[diffusion] fix: fix stages not logged when perf_dump_path is provided (#16016)
This commit is contained in:
@@ -16,7 +16,6 @@ from dataclasses import dataclass
|
|||||||
from enum import Enum, auto
|
from enum import Enum, auto
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from sglang.multimodal_gen import envs
|
|
||||||
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
|
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
|
||||||
from sglang.multimodal_gen.utils import StoreBoolean
|
from sglang.multimodal_gen.utils import StoreBoolean
|
||||||
|
|
||||||
@@ -208,10 +207,6 @@ class SamplingParams:
|
|||||||
if env_steps is not None and self.num_inference_steps is not None:
|
if env_steps is not None and self.num_inference_steps is not None:
|
||||||
self.num_inference_steps = int(env_steps)
|
self.num_inference_steps = int(env_steps)
|
||||||
|
|
||||||
# Auto-enable stage logging if dump path is provided
|
|
||||||
if self.perf_dump_path:
|
|
||||||
envs.SGLANG_DIFFUSION_STAGE_LOGGING = True
|
|
||||||
|
|
||||||
def _validate(self):
|
def _validate(self):
|
||||||
"""
|
"""
|
||||||
check if the sampling params is correct by itself
|
check if the sampling params is correct by itself
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ from typing import List
|
|||||||
import torch
|
import torch
|
||||||
from setproctitle import setproctitle
|
from setproctitle import setproctitle
|
||||||
|
|
||||||
|
from sglang.multimodal_gen import envs
|
||||||
from sglang.multimodal_gen.runtime.distributed import (
|
from sglang.multimodal_gen.runtime.distributed import (
|
||||||
get_sp_group,
|
get_sp_group,
|
||||||
maybe_init_distributed_environment_and_model_parallel,
|
maybe_init_distributed_environment_and_model_parallel,
|
||||||
@@ -31,10 +32,7 @@ from sglang.multimodal_gen.runtime.utils.logging_utils import (
|
|||||||
globally_suppress_loggers,
|
globally_suppress_loggers,
|
||||||
init_logger,
|
init_logger,
|
||||||
)
|
)
|
||||||
from sglang.multimodal_gen.runtime.utils.perf_logger import (
|
from sglang.multimodal_gen.runtime.utils.perf_logger import PerformanceLogger
|
||||||
PerformanceLogger,
|
|
||||||
StageProfiler,
|
|
||||||
)
|
|
||||||
|
|
||||||
logger = init_logger(__name__)
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
@@ -129,7 +127,8 @@ class GPUWorker:
|
|||||||
duration_ms = (time.monotonic() - start_time) * 1000
|
duration_ms = (time.monotonic() - start_time) * 1000
|
||||||
output_batch.timings.total_duration_ms = duration_ms
|
output_batch.timings.total_duration_ms = duration_ms
|
||||||
|
|
||||||
if StageProfiler.metrics_enabled():
|
# TODO: extract to avoid duplication
|
||||||
|
if req.perf_dump_path is not None or envs.SGLANG_DIFFUSION_STAGE_LOGGING:
|
||||||
PerformanceLogger.log_request_summary(timings=output_batch.timings)
|
PerformanceLogger.log_request_summary(timings=output_batch.timings)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(
|
logger.error(
|
||||||
|
|||||||
@@ -201,7 +201,12 @@ class PipelineStage(ABC):
|
|||||||
raise
|
raise
|
||||||
|
|
||||||
# Execute the actual stage logic with unified profiling
|
# Execute the actual stage logic with unified profiling
|
||||||
with StageProfiler(stage_name, logger=logger, timings=batch.timings):
|
with StageProfiler(
|
||||||
|
stage_name,
|
||||||
|
logger=logger,
|
||||||
|
timings=batch.timings,
|
||||||
|
perf_dump_path_provided=batch.perf_dump_path is not None,
|
||||||
|
):
|
||||||
result = self.forward(batch, server_args)
|
result = self.forward(batch, server_args)
|
||||||
|
|
||||||
# Post-execution output verification
|
# Post-execution output verification
|
||||||
|
|||||||
@@ -959,7 +959,10 @@ class DenoisingStage(PipelineStage):
|
|||||||
with self.progress_bar(total=num_inference_steps) as progress_bar:
|
with self.progress_bar(total=num_inference_steps) as progress_bar:
|
||||||
for i, t_host in enumerate(timesteps_cpu):
|
for i, t_host in enumerate(timesteps_cpu):
|
||||||
with StageProfiler(
|
with StageProfiler(
|
||||||
f"denoising_step_{i}", logger=logger, timings=batch.timings
|
f"denoising_step_{i}",
|
||||||
|
logger=logger,
|
||||||
|
timings=batch.timings,
|
||||||
|
perf_dump_path_provided=batch.perf_dump_path is not None,
|
||||||
):
|
):
|
||||||
t_int = int(t_host.item())
|
t_int = int(t_host.item())
|
||||||
t_device = timesteps[i]
|
t_device = timesteps[i]
|
||||||
|
|||||||
@@ -98,7 +98,10 @@ class DmdDenoisingStage(DenoisingStage):
|
|||||||
continue
|
continue
|
||||||
|
|
||||||
with StageProfiler(
|
with StageProfiler(
|
||||||
f"denoising_step_{i}", logger=logger, timings=batch.timings
|
f"denoising_step_{i}",
|
||||||
|
logger=logger,
|
||||||
|
timings=batch.timings,
|
||||||
|
perf_dump_path_provided=batch.perf_dump_path is not None,
|
||||||
):
|
):
|
||||||
# Expand latents for I2V
|
# Expand latents for I2V
|
||||||
noise_latents = latents.clone()
|
noise_latents = latents.clone()
|
||||||
|
|||||||
@@ -131,31 +131,26 @@ class StageProfiler:
|
|||||||
logger: _SGLDiffusionLogger,
|
logger: _SGLDiffusionLogger,
|
||||||
timings: Optional["RequestTimings"],
|
timings: Optional["RequestTimings"],
|
||||||
simple_log: bool = False,
|
simple_log: bool = False,
|
||||||
|
perf_dump_path_provided: bool = False,
|
||||||
):
|
):
|
||||||
self.stage_name = stage_name
|
self.stage_name = stage_name
|
||||||
self.timings = timings
|
self.timings = timings
|
||||||
self.logger = logger
|
self.logger = logger
|
||||||
self.simple_log = simple_log
|
self.simple_log = simple_log
|
||||||
self.start_time = 0.0
|
self.start_time = 0.0
|
||||||
|
self.enabled = perf_dump_path_provided or envs.SGLANG_DIFFUSION_STAGE_LOGGING
|
||||||
self._metrics_enabled = StageProfiler.metrics_enabled()
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def metrics_enabled():
|
|
||||||
# Check env var at runtime to ensure we pick up changes (e.g. from CLI args)
|
|
||||||
return envs.SGLANG_DIFFUSION_STAGE_LOGGING
|
|
||||||
|
|
||||||
def __enter__(self):
|
def __enter__(self):
|
||||||
if self.simple_log:
|
if self.simple_log:
|
||||||
self.logger.info(f"[{self.stage_name}] started...")
|
self.logger.info(f"[{self.stage_name}] started...")
|
||||||
|
|
||||||
if (self._metrics_enabled and self.timings) or self.simple_log:
|
if (self.enabled and self.timings) or self.simple_log:
|
||||||
self.start_time = time.perf_counter()
|
self.start_time = time.perf_counter()
|
||||||
|
|
||||||
return self
|
return self
|
||||||
|
|
||||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||||
if not ((self._metrics_enabled and self.timings) or self.simple_log):
|
if not ((self.enabled and self.timings) or self.simple_log):
|
||||||
return False
|
return False
|
||||||
|
|
||||||
execution_time_s = time.perf_counter() - self.start_time
|
execution_time_s = time.perf_counter() - self.start_time
|
||||||
@@ -175,7 +170,7 @@ class StageProfiler:
|
|||||||
f"[{self.stage_name}] finished in {execution_time_s:.4f} seconds",
|
f"[{self.stage_name}] finished in {execution_time_s:.4f} seconds",
|
||||||
)
|
)
|
||||||
|
|
||||||
if self._metrics_enabled and self.timings:
|
if self.enabled and self.timings:
|
||||||
if "denoising_step_" in self.stage_name:
|
if "denoising_step_" in self.stage_name:
|
||||||
index = int(self.stage_name[len("denoising_step_") :])
|
index = int(self.stage_name[len("denoising_step_") :])
|
||||||
self.timings.record_steps(index, execution_time_s)
|
self.timings.record_steps(index, execution_time_s)
|
||||||
|
|||||||
Reference in New Issue
Block a user