Lift forward_ct/cur_batch and use direct access in watchdog (#25442)
This commit is contained in:
@@ -349,6 +349,9 @@ class Scheduler(
|
|||||||
dp_rank: Optional[int],
|
dp_rank: Optional[int],
|
||||||
):
|
):
|
||||||
self.is_initializing = True
|
self.is_initializing = True
|
||||||
|
# init_soft_watchdog starts a daemon thread that reads these on its first tick.
|
||||||
|
self.forward_ct: int = 0
|
||||||
|
self.cur_batch: Optional[ScheduleBatch] = None
|
||||||
self.init_soft_watchdog(server_args)
|
self.init_soft_watchdog(server_args)
|
||||||
|
|
||||||
# Parse args
|
# Parse args
|
||||||
|
|||||||
@@ -594,9 +594,8 @@ def create_scheduler_watchdog(
|
|||||||
|
|
||||||
return WatchdogRaw(
|
return WatchdogRaw(
|
||||||
debug_name="Scheduler",
|
debug_name="Scheduler",
|
||||||
get_counter=lambda: getattr(scheduler, "forward_ct", 0),
|
get_counter=lambda: scheduler.forward_ct,
|
||||||
is_active=lambda: scheduler.is_initializing
|
is_active=lambda: scheduler.is_initializing or scheduler.cur_batch is not None,
|
||||||
or getattr(scheduler, "cur_batch", None) is not None,
|
|
||||||
watchdog_timeout=watchdog_timeout,
|
watchdog_timeout=watchdog_timeout,
|
||||||
soft=soft,
|
soft=soft,
|
||||||
dump_info=dump_info,
|
dump_info=dump_info,
|
||||||
|
|||||||
Reference in New Issue
Block a user