[AMD] Resolving bug in per batch size graph capture due to multiple capture runs per batch size (#39840)

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: HAI <hixiao@gmail.com>
This commit is contained in:
mohbasit
2026-10-10 00:47:13 -07:00
committed by GitHub
co-authored by github-actions[bot] Cursor HAI
parent d8c5eca3ac
commit d7d22e8a30
3 changed files with 202 additions and 36 deletions
@@ -835,35 +835,76 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
and not envs.SGLANG_ENABLE_CUDA_GRAPH_CAPTURE_TRACE.get()
)
def _set_profile_trace_tag(
self,
bs: int,
stream_idx: Optional[int],
variant_label: Optional[str],
attention_variant: Optional[str],
) -> None:
"""Name the trace for one capture. bs alone is not unique: the capture
loop multiplies it by the lora and attention variants (DSA captures
dense + sparse per bs). Absent variants are omitted so single-variant
runners keep the historical `bs_<n>` stem.
"""
if getattr(self, "_profiler", None) is None:
return
parts = [f"bs_{bs}"]
if attention_variant is not None:
parts.append(attention_variant)
if variant_label is not None:
parts.append(variant_label)
if stream_idx is not None:
parts.append(f"stream_{stream_idx}")
self._profile_trace_tag = "_".join(parts)
def _profile_runner_name(self) -> str:
"""Runner name used in capture-trace filenames. DSPARK / DFLASH draft
workers capture their verify graphs with this base class too, at the
same shapes as the target, so the name alone would collide; prefix it.
Draft-specific subclasses (EAGLEDraft*, Uno*, ...) are already unique.
"""
name = type(self).__name__
if type(self) is DecodeCudaGraphRunner and self.model_runner.is_draft_worker:
return f"Draft{name}"
return name
def _init_profile_context_and_memory_record(self):
if self._graph_batch_capture_active():
# Per-batch-size capture traces (SGLANG_GRAPH_BATCH_CAPTURE): a
# scheduled profiler is stepped once per batch size (see
# FullCudaGraphBackend.capture_one) and on_trace_ready writes one
# chrome trace per bs.
# SGLANG_GRAPH_BATCH_CAPTURE: the profiler is stepped once per
# captured shape (see FullCudaGraphBackend.capture_one), so
# on_trace_ready writes one chrome trace per shape.
rank = get_parallel().tp_rank
runner_name = type(self).__name__
runner_name = self._profile_runner_name()
trace_dir = graph_capture_profile_dir()
os.makedirs(trace_dir, exist_ok=True)
# Track which BS is currently being captured for trace file naming
self._profile_bs_list = list(reversed(self.capture_bs))
self._profile_bs_idx = 0
self._profile_trace_tag = None
def on_trace_ready(prof):
bs = self._profile_bs_list[self._profile_bs_idx]
tag = self._profile_trace_tag
if tag is None:
# Reachable only if a subclass overrides _capture_one_stream
# without tagging; exporting would collide every flush on
# one name.
logger.warning(
"%s: capture trace flushed with no shape tag; skipping "
"export. A subclass likely overrides _capture_one_stream "
"without calling _set_profile_trace_tag.",
runner_name,
)
return
trace_file = os.path.join(
trace_dir, f"{runner_name}_bs_{bs}_rank{rank}.json.gz"
trace_dir, f"{runner_name}_{tag}_rank{rank}.json.gz"
)
prof.export_chrome_trace(trace_file)
logger.info(f"Saved trace for bs={bs} to {trace_file}")
self._profile_bs_idx += 1
logger.info(f"Saved trace for {tag} to {trace_file}")
profile_context = profile(
activities=[ProfilerActivity.CPU, ProfilerActivity.CUDA],
# Schedule: wait=2 (skip 2 dummy runs), warmup=0, active=1
# (capture run); repeat=0 repeats the cycle so each batch size
# gets its own trace.
# (capture run); repeat=0 repeats the cycle so each captured
# shape gets its own trace.
schedule=torch.profiler.schedule(wait=2, warmup=0, active=1, repeat=0),
record_shapes=True,
with_stack=True,
@@ -905,7 +946,7 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
# and the per-bs on_trace_ready handles export instead.
export_cuda_graph_capture_trace(
prof_context,
runner_name=type(self).__name__,
runner_name=self._profile_runner_name(),
)
def capture_prepare(
@@ -1173,6 +1214,9 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
_set_capture_lora_variant(variant_label)
for attention_variant in attention_variants:
_set_capture_attention_variant(attention_variant)
self._set_profile_trace_tag(
bs, stream_idx, variant_label, attention_variant
)
with torch_compile_decoration.patch_model(
self.model_runner.model,
bs in self.compile_bs,
@@ -145,7 +145,7 @@ class FullCudaGraphBackend(BaseCudaGraphBackend):
# SGLANG_GRAPH_BATCH_CAPTURE), the runner created a scheduled
# torch profiler (wait=2, active=1) and exposed it as _profiler. We step()
# past the two warmup runs so only the capture run is recorded, and each
# batch size produces its own trace via the profiler's on_trace_ready.
# captured shape produces its own trace via the profiler's on_trace_ready.
# With --enable-profile-cuda-graph alone the runner leaves _profiler None
# (its unscheduled profiler records the whole capture in one pass), so no
# stepping happens here.