Change naming for graph mode on multiplatform (#17469)
This commit is contained in:
@@ -383,8 +383,15 @@ class SchedulerMetricsMixin:
|
|||||||
msg += f"#transfer-req: {len(self.disagg_decode_transfer_queue.queue)}, "
|
msg += f"#transfer-req: {len(self.disagg_decode_transfer_queue.queue)}, "
|
||||||
msg += f"#retracted-req: {len(self.disagg_decode_prealloc_queue.retracted_queue)}, "
|
msg += f"#retracted-req: {len(self.disagg_decode_prealloc_queue.retracted_queue)}, "
|
||||||
|
|
||||||
|
graph_backend = defaultdict(
|
||||||
|
lambda: "cuda graph",
|
||||||
|
{
|
||||||
|
"cpu": "cpu graph",
|
||||||
|
"npu": "npu graph",
|
||||||
|
},
|
||||||
|
)
|
||||||
msg += (
|
msg += (
|
||||||
f"{'cuda graph' if self.device == 'cuda' else 'cpu graph'}: {can_run_cuda_graph}, "
|
f"{graph_backend[self.device]}: {can_run_cuda_graph}, "
|
||||||
f"gen throughput (token/s): {self.last_gen_throughput:.2f}, "
|
f"gen throughput (token/s): {self.last_gen_throughput:.2f}, "
|
||||||
f"#queue-req: {len(self.waiting_queue)}, "
|
f"#queue-req: {len(self.waiting_queue)}, "
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -1997,8 +1997,15 @@ class ModelRunner(ModelRunnerKVCacheMixin):
|
|||||||
|
|
||||||
tic = time.perf_counter()
|
tic = time.perf_counter()
|
||||||
before_mem = get_available_gpu_memory(self.device, self.gpu_id)
|
before_mem = get_available_gpu_memory(self.device, self.gpu_id)
|
||||||
|
graph_backend = defaultdict(
|
||||||
|
lambda: "cuda graph",
|
||||||
|
{
|
||||||
|
"cpu": "cpu graph",
|
||||||
|
"npu": "npu graph",
|
||||||
|
},
|
||||||
|
)
|
||||||
logger.info(
|
logger.info(
|
||||||
f"Capture {'cpu graph' if self.device == 'cpu' else 'cuda graph'} begin. This can take up to several minutes. avail mem={before_mem:.2f} GB"
|
f"Capture {graph_backend[self.device]} begin. This can take up to several minutes. avail mem={before_mem:.2f} GB"
|
||||||
)
|
)
|
||||||
graph_runners = defaultdict(
|
graph_runners = defaultdict(
|
||||||
lambda: CudaGraphRunner,
|
lambda: CudaGraphRunner,
|
||||||
@@ -2012,7 +2019,7 @@ class ModelRunner(ModelRunnerKVCacheMixin):
|
|||||||
after_mem = get_available_gpu_memory(self.device, self.gpu_id)
|
after_mem = get_available_gpu_memory(self.device, self.gpu_id)
|
||||||
self.graph_mem_usage = before_mem - after_mem
|
self.graph_mem_usage = before_mem - after_mem
|
||||||
logger.info(
|
logger.info(
|
||||||
f"Capture {'cpu graph' if self.device == 'cpu' else 'cuda graph'} end. Time elapsed: {time.perf_counter() - tic:.2f} s. "
|
f"Capture {graph_backend[self.device]} end. Time elapsed: {time.perf_counter() - tic:.2f} s. "
|
||||||
f"mem usage={self.graph_mem_usage:.2f} GB. avail mem={after_mem:.2f} GB."
|
f"mem usage={self.graph_mem_usage:.2f} GB. avail mem={after_mem:.2f} GB."
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user