DEBUG-ONLY: ARDBG batch fingerprint + phase + hicache sync logs, sampled device AR (1/1024) + idle spin sampling (p2 wedge trap)

This commit is contained in:
2026-10-13 18:01:36 +08:00
parent f3ac513d92
commit db1e8cdbe3
4 changed files with 109 additions and 12 deletions
@@ -21,6 +21,7 @@ from __future__ import annotations
import hashlib
import logging
import os
from array import array
from collections import deque
from http import HTTPStatus
@@ -105,6 +106,63 @@ logger = logging.getLogger(__name__)
_is_npu = is_npu()
# DEBUG-ONLY (p2 wedge trap 2026-10-13): per-step batch-decision fingerprint
# and event-loop phase logging. Same env as custom_all_reduce_v2 ARDBG.
# 1 = sampled/one-line, 2 = + stacks on ONE_SHOT_PUSH. Off by default.
_AR_SEQ_DEBUG = int(os.environ.get("SGLANG_AR_SEQ_DEBUG", "0"))
_plan_seq = 0
_plan_none_seq = 0
def _dbg_plan(self, batch) -> None:
global _plan_seq, _plan_none_seq
if not _AR_SEQ_DEBUG:
return
_plan_seq += 1
rank = torch.distributed.get_rank()
if batch is None:
# 空转步采样记录,避免 idle 自旋刷爆日志
_plan_none_seq += 1
if _plan_none_seq & 511:
return
logger.info("[ARDBG] plan rank=%s step=%d batch=None", rank, _plan_seq)
return
reqs = batch.reqs
fp = 0
ext_total = 0
for r in reqs:
fp = (fp * 131 + hash(r.rid)) & 0xFFFFFFFFFFFFFFFF
er = getattr(r, "extend_range", None)
el = int(er.length) if er is not None else -1
fp = (fp * 131 + el) & 0xFFFFFFFFFFFFFFFF
fp = (fp * 131 + int(getattr(r, "prefix_len", -1))) & 0xFFFFFFFFFFFFFFFF
if el > 0:
ext_total += el
logger.info(
"[ARDBG] plan rank=%s step=%d nreq=%d ext=%d fp=%016x",
rank,
_plan_seq,
len(reqs),
ext_total,
fp,
)
def _dbg_phase(label: str, batch=None) -> None:
if not _AR_SEQ_DEBUG:
return
if batch is None and label in ("inflight_enter", "inflight_exit"):
# 空转迭代的 inflight 相位价值低,采样记录
if _plan_seq & 255:
return
logger.info(
"[ARDBG] phase rank=%s step=%s %s nreq=%s",
torch.distributed.get_rank(),
_plan_seq,
label,
len(batch.reqs) if batch is not None else "-",
)
def should_force_retry(req: Req) -> bool:
"""Test hook to force a request into optimistic prefill retry."""
@@ -628,6 +686,7 @@ class SchedulerDisaggregationPrefillMixin:
if batch:
set_schedule_time_batch(batch)
_dbg_plan(self, batch)
return NextBatchPlan(batch_to_run=batch, running_batch=running_batch)
@torch.no_grad()
@@ -658,13 +717,17 @@ class SchedulerDisaggregationPrefillMixin:
if batch:
if self.enable_staging:
self.maybe_prefetch_staging_for_batch(batch)
_dbg_phase("run_batch_enter", batch)
result = self.run_batch(batch)
_dbg_phase("run_batch_exit", batch)
self.process_batch_result(batch, result)
else:
self._sched_idled = True
self.on_idle()
_dbg_phase("inflight_enter")
self.process_disagg_prefill_inflight_queue()
_dbg_phase("inflight_exit")
# Update last_batch
self.last_batch = batch
@@ -698,7 +761,9 @@ class SchedulerDisaggregationPrefillMixin:
if batch:
if self.enable_staging:
self.maybe_prefetch_staging_for_batch(batch)
_dbg_phase("run_batch_enter", batch)
batch_result = self.run_batch(batch)
_dbg_phase("run_batch_exit", batch)
self._apply_war_barrier()
self.result_queue.append((batch.copy(), batch_result))
else:
@@ -713,7 +778,9 @@ class SchedulerDisaggregationPrefillMixin:
# When the server is idle, do self-check and re-init some states
self.on_idle()
_dbg_phase("inflight_enter")
self.process_disagg_prefill_inflight_queue()
_dbg_phase("inflight_exit")
# Run sample of the current batch
# It depends on the result of the last batch (e.g., grammar), so we run it after the last batch is processed.
@@ -60,6 +60,9 @@ def _dbg_log_host_ar(phase: str, polls) -> None:
return
seq = _host_ar_seq
_host_ar_seq += 1
# 空转迭代(无 poller)采样 1/256,避免 idle 自旋刷爆日志
if (not polls) and seq & 255:
return
logger.info(
"[ARDBG] host rank=%s seq=%d phase=%s polls=%s",
dist.get_rank(),
@@ -387,22 +387,24 @@ class CustomAllReduceV2:
if _AR_SEQ_DEBUG:
seq = self._dbg_seq
self._dbg_seq += 1
logger.info(
"[ARDBG] dev rank=%s seq=%d nbytes=%d algo=%s graph=%s capturing=%s",
self._dbg_rank,
seq,
nbytes,
getattr(algo, "name", algo),
use_graph,
torch.cuda.is_current_stream_capturing(),
)
if _AR_SEQ_DEBUG >= 2 and getattr(algo, "name", algo) == "ONE_SHOT_PUSH":
# 采样 1/1024:生产高频路径下全量日志会拖垮 P 节点
if seq & 1023 == 0:
logger.info(
"[ARDBG] stack rank=%s seq=%d\n%s",
"[ARDBG] dev rank=%s seq=%d nbytes=%d algo=%s graph=%s capturing=%s",
self._dbg_rank,
seq,
"".join(traceback.format_stack(limit=15)),
nbytes,
getattr(algo, "name", algo),
use_graph,
torch.cuda.is_current_stream_capturing(),
)
if _AR_SEQ_DEBUG >= 2 and getattr(algo, "name", algo) == "ONE_SHOT_PUSH":
logger.info(
"[ARDBG] stack rank=%s seq=%d\n%s",
self._dbg_rank,
seq,
"".join(traceback.format_stack(limit=15)),
)
return custom_all_reduce(
self.obj,
input,
@@ -2,6 +2,7 @@ from __future__ import annotations
import atexit
import logging
import os
import threading
import time
from dataclasses import replace
@@ -132,6 +133,11 @@ COMPONENT_REGISTRY: dict[ComponentType, type[TreeComponent]] = {
logger = logging.getLogger(__name__)
# DEBUG-ONLY (p2 wedge trap 2026-10-13): env-gated logging for the per-step
# hicache event sync. Same SGLANG_AR_SEQ_DEBUG switch as the ARDBG sites.
_AR_SEQ_DEBUG = int(os.environ.get("SGLANG_AR_SEQ_DEBUG", "0"))
_hc_step = 0
class _OngoingWriteThrough(NamedTuple):
"""Tracks an in-flight D→H write-through operation."""
@@ -3533,6 +3539,16 @@ class UnifiedRadixCache(BasePrefixCache):
def check_hicache_events(self) -> None:
"""Called per scheduler step to poll async HiCache events."""
global _hc_step
if _AR_SEQ_DEBUG:
_hc_step += 1
if _hc_step & 255 == 0:
logger.info(
"[ARDBG] hcev rank=%s step=%d linker=%s",
torch.distributed.get_rank(),
_hc_step,
self.linker is not None,
)
if self.linker is not None:
finish_counts = torch.tensor(
[
@@ -3566,6 +3582,15 @@ class UnifiedRadixCache(BasePrefixCache):
storage_queue_sizes,
extra_pool_names,
) = self._sync_hicache_ready_counts()
if _AR_SEQ_DEBUG and _hc_step & 255 == 0:
logger.info(
"[ARDBG] hcsync rank=%s step=%d write=%s load=%s storage=%s",
torch.distributed.get_rank(),
_hc_step,
write_finish_count,
load_finish_count,
storage_queue_sizes,
)
self.writing_check(finish_count=write_finish_count)
self.loading_check(finish_count=load_finish_count)