diff --git a/python/sglang/srt/disaggregation/prefill.py b/python/sglang/srt/disaggregation/prefill.py index ea128956b1..fae1e9f87d 100644 --- a/python/sglang/srt/disaggregation/prefill.py +++ b/python/sglang/srt/disaggregation/prefill.py @@ -21,6 +21,7 @@ from __future__ import annotations import hashlib import logging +import os from array import array from collections import deque from http import HTTPStatus @@ -105,6 +106,63 @@ logger = logging.getLogger(__name__) _is_npu = is_npu() +# DEBUG-ONLY (p2 wedge trap 2026-10-13): per-step batch-decision fingerprint +# and event-loop phase logging. Same env as custom_all_reduce_v2 ARDBG. +# 1 = sampled/one-line, 2 = + stacks on ONE_SHOT_PUSH. Off by default. +_AR_SEQ_DEBUG = int(os.environ.get("SGLANG_AR_SEQ_DEBUG", "0")) +_plan_seq = 0 +_plan_none_seq = 0 + + +def _dbg_plan(self, batch) -> None: + global _plan_seq, _plan_none_seq + if not _AR_SEQ_DEBUG: + return + _plan_seq += 1 + rank = torch.distributed.get_rank() + if batch is None: + # 空转步采样记录,避免 idle 自旋刷爆日志 + _plan_none_seq += 1 + if _plan_none_seq & 511: + return + logger.info("[ARDBG] plan rank=%s step=%d batch=None", rank, _plan_seq) + return + reqs = batch.reqs + fp = 0 + ext_total = 0 + for r in reqs: + fp = (fp * 131 + hash(r.rid)) & 0xFFFFFFFFFFFFFFFF + er = getattr(r, "extend_range", None) + el = int(er.length) if er is not None else -1 + fp = (fp * 131 + el) & 0xFFFFFFFFFFFFFFFF + fp = (fp * 131 + int(getattr(r, "prefix_len", -1))) & 0xFFFFFFFFFFFFFFFF + if el > 0: + ext_total += el + logger.info( + "[ARDBG] plan rank=%s step=%d nreq=%d ext=%d fp=%016x", + rank, + _plan_seq, + len(reqs), + ext_total, + fp, + ) + + +def _dbg_phase(label: str, batch=None) -> None: + if not _AR_SEQ_DEBUG: + return + if batch is None and label in ("inflight_enter", "inflight_exit"): + # 空转迭代的 inflight 相位价值低,采样记录 + if _plan_seq & 255: + return + logger.info( + "[ARDBG] phase rank=%s step=%s %s nreq=%s", + torch.distributed.get_rank(), + _plan_seq, + label, + len(batch.reqs) if batch is not None else "-", + ) + def should_force_retry(req: Req) -> bool: """Test hook to force a request into optimistic prefill retry.""" @@ -628,6 +686,7 @@ class SchedulerDisaggregationPrefillMixin: if batch: set_schedule_time_batch(batch) + _dbg_plan(self, batch) return NextBatchPlan(batch_to_run=batch, running_batch=running_batch) @torch.no_grad() @@ -658,13 +717,17 @@ class SchedulerDisaggregationPrefillMixin: if batch: if self.enable_staging: self.maybe_prefetch_staging_for_batch(batch) + _dbg_phase("run_batch_enter", batch) result = self.run_batch(batch) + _dbg_phase("run_batch_exit", batch) self.process_batch_result(batch, result) else: self._sched_idled = True self.on_idle() + _dbg_phase("inflight_enter") self.process_disagg_prefill_inflight_queue() + _dbg_phase("inflight_exit") # Update last_batch self.last_batch = batch @@ -698,7 +761,9 @@ class SchedulerDisaggregationPrefillMixin: if batch: if self.enable_staging: self.maybe_prefetch_staging_for_batch(batch) + _dbg_phase("run_batch_enter", batch) batch_result = self.run_batch(batch) + _dbg_phase("run_batch_exit", batch) self._apply_war_barrier() self.result_queue.append((batch.copy(), batch_result)) else: @@ -713,7 +778,9 @@ class SchedulerDisaggregationPrefillMixin: # When the server is idle, do self-check and re-init some states self.on_idle() + _dbg_phase("inflight_enter") self.process_disagg_prefill_inflight_queue() + _dbg_phase("inflight_exit") # Run sample of the current batch # It depends on the result of the last batch (e.g., grammar), so we run it after the last batch is processed. diff --git a/python/sglang/srt/disaggregation/utils.py b/python/sglang/srt/disaggregation/utils.py index bf2964e118..e605885874 100644 --- a/python/sglang/srt/disaggregation/utils.py +++ b/python/sglang/srt/disaggregation/utils.py @@ -60,6 +60,9 @@ def _dbg_log_host_ar(phase: str, polls) -> None: return seq = _host_ar_seq _host_ar_seq += 1 + # 空转迭代(无 poller)采样 1/256,避免 idle 自旋刷爆日志 + if (not polls) and seq & 255: + return logger.info( "[ARDBG] host rank=%s seq=%d phase=%s polls=%s", dist.get_rank(), diff --git a/python/sglang/srt/distributed/device_communicators/custom_all_reduce_v2.py b/python/sglang/srt/distributed/device_communicators/custom_all_reduce_v2.py index 75c2b3f19b..6fd0bfd59f 100644 --- a/python/sglang/srt/distributed/device_communicators/custom_all_reduce_v2.py +++ b/python/sglang/srt/distributed/device_communicators/custom_all_reduce_v2.py @@ -387,22 +387,24 @@ class CustomAllReduceV2: if _AR_SEQ_DEBUG: seq = self._dbg_seq self._dbg_seq += 1 - logger.info( - "[ARDBG] dev rank=%s seq=%d nbytes=%d algo=%s graph=%s capturing=%s", - self._dbg_rank, - seq, - nbytes, - getattr(algo, "name", algo), - use_graph, - torch.cuda.is_current_stream_capturing(), - ) - if _AR_SEQ_DEBUG >= 2 and getattr(algo, "name", algo) == "ONE_SHOT_PUSH": + # 采样 1/1024:生产高频路径下全量日志会拖垮 P 节点 + if seq & 1023 == 0: logger.info( - "[ARDBG] stack rank=%s seq=%d\n%s", + "[ARDBG] dev rank=%s seq=%d nbytes=%d algo=%s graph=%s capturing=%s", self._dbg_rank, seq, - "".join(traceback.format_stack(limit=15)), + nbytes, + getattr(algo, "name", algo), + use_graph, + torch.cuda.is_current_stream_capturing(), ) + if _AR_SEQ_DEBUG >= 2 and getattr(algo, "name", algo) == "ONE_SHOT_PUSH": + logger.info( + "[ARDBG] stack rank=%s seq=%d\n%s", + self._dbg_rank, + seq, + "".join(traceback.format_stack(limit=15)), + ) return custom_all_reduce( self.obj, input, diff --git a/python/sglang/srt/mem_cache/unified_radix_cache.py b/python/sglang/srt/mem_cache/unified_radix_cache.py index dd658df956..614c5ce7e7 100644 --- a/python/sglang/srt/mem_cache/unified_radix_cache.py +++ b/python/sglang/srt/mem_cache/unified_radix_cache.py @@ -2,6 +2,7 @@ from __future__ import annotations import atexit import logging +import os import threading import time from dataclasses import replace @@ -132,6 +133,11 @@ COMPONENT_REGISTRY: dict[ComponentType, type[TreeComponent]] = { logger = logging.getLogger(__name__) +# DEBUG-ONLY (p2 wedge trap 2026-10-13): env-gated logging for the per-step +# hicache event sync. Same SGLANG_AR_SEQ_DEBUG switch as the ARDBG sites. +_AR_SEQ_DEBUG = int(os.environ.get("SGLANG_AR_SEQ_DEBUG", "0")) +_hc_step = 0 + class _OngoingWriteThrough(NamedTuple): """Tracks an in-flight D→H write-through operation.""" @@ -3533,6 +3539,16 @@ class UnifiedRadixCache(BasePrefixCache): def check_hicache_events(self) -> None: """Called per scheduler step to poll async HiCache events.""" + global _hc_step + if _AR_SEQ_DEBUG: + _hc_step += 1 + if _hc_step & 255 == 0: + logger.info( + "[ARDBG] hcev rank=%s step=%d linker=%s", + torch.distributed.get_rank(), + _hc_step, + self.linker is not None, + ) if self.linker is not None: finish_counts = torch.tensor( [ @@ -3566,6 +3582,15 @@ class UnifiedRadixCache(BasePrefixCache): storage_queue_sizes, extra_pool_names, ) = self._sync_hicache_ready_counts() + if _AR_SEQ_DEBUG and _hc_step & 255 == 0: + logger.info( + "[ARDBG] hcsync rank=%s step=%d write=%s load=%s storage=%s", + torch.distributed.get_rank(), + _hc_step, + write_finish_count, + load_finish_count, + storage_queue_sizes, + ) self.writing_check(finish_count=write_finish_count) self.loading_check(finish_count=load_finish_count)