DEBUG-ONLY: ARDBG batch fingerprint + phase + hicache sync logs, sampled device AR (1/1024) + idle spin sampling (p2 wedge trap)
This commit is contained in:
@@ -21,6 +21,7 @@ from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import os
|
||||
from array import array
|
||||
from collections import deque
|
||||
from http import HTTPStatus
|
||||
@@ -105,6 +106,63 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
_is_npu = is_npu()
|
||||
|
||||
# DEBUG-ONLY (p2 wedge trap 2026-10-13): per-step batch-decision fingerprint
|
||||
# and event-loop phase logging. Same env as custom_all_reduce_v2 ARDBG.
|
||||
# 1 = sampled/one-line, 2 = + stacks on ONE_SHOT_PUSH. Off by default.
|
||||
_AR_SEQ_DEBUG = int(os.environ.get("SGLANG_AR_SEQ_DEBUG", "0"))
|
||||
_plan_seq = 0
|
||||
_plan_none_seq = 0
|
||||
|
||||
|
||||
def _dbg_plan(self, batch) -> None:
|
||||
global _plan_seq, _plan_none_seq
|
||||
if not _AR_SEQ_DEBUG:
|
||||
return
|
||||
_plan_seq += 1
|
||||
rank = torch.distributed.get_rank()
|
||||
if batch is None:
|
||||
# 空转步采样记录,避免 idle 自旋刷爆日志
|
||||
_plan_none_seq += 1
|
||||
if _plan_none_seq & 511:
|
||||
return
|
||||
logger.info("[ARDBG] plan rank=%s step=%d batch=None", rank, _plan_seq)
|
||||
return
|
||||
reqs = batch.reqs
|
||||
fp = 0
|
||||
ext_total = 0
|
||||
for r in reqs:
|
||||
fp = (fp * 131 + hash(r.rid)) & 0xFFFFFFFFFFFFFFFF
|
||||
er = getattr(r, "extend_range", None)
|
||||
el = int(er.length) if er is not None else -1
|
||||
fp = (fp * 131 + el) & 0xFFFFFFFFFFFFFFFF
|
||||
fp = (fp * 131 + int(getattr(r, "prefix_len", -1))) & 0xFFFFFFFFFFFFFFFF
|
||||
if el > 0:
|
||||
ext_total += el
|
||||
logger.info(
|
||||
"[ARDBG] plan rank=%s step=%d nreq=%d ext=%d fp=%016x",
|
||||
rank,
|
||||
_plan_seq,
|
||||
len(reqs),
|
||||
ext_total,
|
||||
fp,
|
||||
)
|
||||
|
||||
|
||||
def _dbg_phase(label: str, batch=None) -> None:
|
||||
if not _AR_SEQ_DEBUG:
|
||||
return
|
||||
if batch is None and label in ("inflight_enter", "inflight_exit"):
|
||||
# 空转迭代的 inflight 相位价值低,采样记录
|
||||
if _plan_seq & 255:
|
||||
return
|
||||
logger.info(
|
||||
"[ARDBG] phase rank=%s step=%s %s nreq=%s",
|
||||
torch.distributed.get_rank(),
|
||||
_plan_seq,
|
||||
label,
|
||||
len(batch.reqs) if batch is not None else "-",
|
||||
)
|
||||
|
||||
|
||||
def should_force_retry(req: Req) -> bool:
|
||||
"""Test hook to force a request into optimistic prefill retry."""
|
||||
@@ -628,6 +686,7 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
if batch:
|
||||
set_schedule_time_batch(batch)
|
||||
|
||||
_dbg_plan(self, batch)
|
||||
return NextBatchPlan(batch_to_run=batch, running_batch=running_batch)
|
||||
|
||||
@torch.no_grad()
|
||||
@@ -658,13 +717,17 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
if batch:
|
||||
if self.enable_staging:
|
||||
self.maybe_prefetch_staging_for_batch(batch)
|
||||
_dbg_phase("run_batch_enter", batch)
|
||||
result = self.run_batch(batch)
|
||||
_dbg_phase("run_batch_exit", batch)
|
||||
self.process_batch_result(batch, result)
|
||||
else:
|
||||
self._sched_idled = True
|
||||
self.on_idle()
|
||||
|
||||
_dbg_phase("inflight_enter")
|
||||
self.process_disagg_prefill_inflight_queue()
|
||||
_dbg_phase("inflight_exit")
|
||||
|
||||
# Update last_batch
|
||||
self.last_batch = batch
|
||||
@@ -698,7 +761,9 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
if batch:
|
||||
if self.enable_staging:
|
||||
self.maybe_prefetch_staging_for_batch(batch)
|
||||
_dbg_phase("run_batch_enter", batch)
|
||||
batch_result = self.run_batch(batch)
|
||||
_dbg_phase("run_batch_exit", batch)
|
||||
self._apply_war_barrier()
|
||||
self.result_queue.append((batch.copy(), batch_result))
|
||||
else:
|
||||
@@ -713,7 +778,9 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
# When the server is idle, do self-check and re-init some states
|
||||
self.on_idle()
|
||||
|
||||
_dbg_phase("inflight_enter")
|
||||
self.process_disagg_prefill_inflight_queue()
|
||||
_dbg_phase("inflight_exit")
|
||||
|
||||
# Run sample of the current batch
|
||||
# It depends on the result of the last batch (e.g., grammar), so we run it after the last batch is processed.
|
||||
|
||||
@@ -60,6 +60,9 @@ def _dbg_log_host_ar(phase: str, polls) -> None:
|
||||
return
|
||||
seq = _host_ar_seq
|
||||
_host_ar_seq += 1
|
||||
# 空转迭代(无 poller)采样 1/256,避免 idle 自旋刷爆日志
|
||||
if (not polls) and seq & 255:
|
||||
return
|
||||
logger.info(
|
||||
"[ARDBG] host rank=%s seq=%d phase=%s polls=%s",
|
||||
dist.get_rank(),
|
||||
|
||||
@@ -387,22 +387,24 @@ class CustomAllReduceV2:
|
||||
if _AR_SEQ_DEBUG:
|
||||
seq = self._dbg_seq
|
||||
self._dbg_seq += 1
|
||||
logger.info(
|
||||
"[ARDBG] dev rank=%s seq=%d nbytes=%d algo=%s graph=%s capturing=%s",
|
||||
self._dbg_rank,
|
||||
seq,
|
||||
nbytes,
|
||||
getattr(algo, "name", algo),
|
||||
use_graph,
|
||||
torch.cuda.is_current_stream_capturing(),
|
||||
)
|
||||
if _AR_SEQ_DEBUG >= 2 and getattr(algo, "name", algo) == "ONE_SHOT_PUSH":
|
||||
# 采样 1/1024:生产高频路径下全量日志会拖垮 P 节点
|
||||
if seq & 1023 == 0:
|
||||
logger.info(
|
||||
"[ARDBG] stack rank=%s seq=%d\n%s",
|
||||
"[ARDBG] dev rank=%s seq=%d nbytes=%d algo=%s graph=%s capturing=%s",
|
||||
self._dbg_rank,
|
||||
seq,
|
||||
"".join(traceback.format_stack(limit=15)),
|
||||
nbytes,
|
||||
getattr(algo, "name", algo),
|
||||
use_graph,
|
||||
torch.cuda.is_current_stream_capturing(),
|
||||
)
|
||||
if _AR_SEQ_DEBUG >= 2 and getattr(algo, "name", algo) == "ONE_SHOT_PUSH":
|
||||
logger.info(
|
||||
"[ARDBG] stack rank=%s seq=%d\n%s",
|
||||
self._dbg_rank,
|
||||
seq,
|
||||
"".join(traceback.format_stack(limit=15)),
|
||||
)
|
||||
return custom_all_reduce(
|
||||
self.obj,
|
||||
input,
|
||||
|
||||
@@ -2,6 +2,7 @@ from __future__ import annotations
|
||||
|
||||
import atexit
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import replace
|
||||
@@ -132,6 +133,11 @@ COMPONENT_REGISTRY: dict[ComponentType, type[TreeComponent]] = {
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# DEBUG-ONLY (p2 wedge trap 2026-10-13): env-gated logging for the per-step
|
||||
# hicache event sync. Same SGLANG_AR_SEQ_DEBUG switch as the ARDBG sites.
|
||||
_AR_SEQ_DEBUG = int(os.environ.get("SGLANG_AR_SEQ_DEBUG", "0"))
|
||||
_hc_step = 0
|
||||
|
||||
|
||||
class _OngoingWriteThrough(NamedTuple):
|
||||
"""Tracks an in-flight D→H write-through operation."""
|
||||
@@ -3533,6 +3539,16 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
|
||||
def check_hicache_events(self) -> None:
|
||||
"""Called per scheduler step to poll async HiCache events."""
|
||||
global _hc_step
|
||||
if _AR_SEQ_DEBUG:
|
||||
_hc_step += 1
|
||||
if _hc_step & 255 == 0:
|
||||
logger.info(
|
||||
"[ARDBG] hcev rank=%s step=%d linker=%s",
|
||||
torch.distributed.get_rank(),
|
||||
_hc_step,
|
||||
self.linker is not None,
|
||||
)
|
||||
if self.linker is not None:
|
||||
finish_counts = torch.tensor(
|
||||
[
|
||||
@@ -3566,6 +3582,15 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
storage_queue_sizes,
|
||||
extra_pool_names,
|
||||
) = self._sync_hicache_ready_counts()
|
||||
if _AR_SEQ_DEBUG and _hc_step & 255 == 0:
|
||||
logger.info(
|
||||
"[ARDBG] hcsync rank=%s step=%d write=%s load=%s storage=%s",
|
||||
torch.distributed.get_rank(),
|
||||
_hc_step,
|
||||
write_finish_count,
|
||||
load_finish_count,
|
||||
storage_queue_sizes,
|
||||
)
|
||||
self.writing_check(finish_count=write_finish_count)
|
||||
self.loading_check(finish_count=load_finish_count)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user