fix: change performance log directory to cache path (#13482)

Co-authored-by: Mick <mickjagger19@icloud.com>
This commit is contained in:
Cheng Wan
2025-11-18 15:18:43 +08:00
committed by GitHub
co-authored by Mick
parent 4e41edcb9c
commit a5ad0069b2
4 changed files with 52 additions and 90 deletions
+1 -60
View File
@@ -21,9 +21,6 @@ concurrency:
group: pr-test-${{ github.ref }} group: pr-test-${{ github.ref }}
cancel-in-progress: true cancel-in-progress: true
env:
SGLANG_IS_IN_CI: true
jobs: jobs:
call-gate: call-gate:
uses: ./.github/workflows/pr-gate.yml uses: ./.github/workflows/pr-gate.yml
@@ -169,11 +166,6 @@ jobs:
cd sgl-kernel cd sgl-kernel
pytest tests/ pytest tests/
- name: Cleanup logs directory
if: always()
run: |
rm -rf python/sglang/logs || true
sgl-kernel-mla-test: sgl-kernel-mla-test:
needs: [check-changes, sgl-kernel-build-wheels] needs: [check-changes, sgl-kernel-build-wheels]
if: needs.check-changes.outputs.sgl_kernel == 'true' if: needs.check-changes.outputs.sgl_kernel == 'true'
@@ -205,11 +197,6 @@ jobs:
cd test/srt cd test/srt
python3 test_mla_deepseek_v3.py python3 test_mla_deepseek_v3.py
- name: Cleanup logs directory
if: always()
run: |
rm -rf python/sglang/logs || true
sgl-kernel-benchmark-test: sgl-kernel-benchmark-test:
needs: [check-changes, sgl-kernel-build-wheels] needs: [check-changes, sgl-kernel-build-wheels]
if: needs.check-changes.outputs.sgl_kernel == 'true' if: needs.check-changes.outputs.sgl_kernel == 'true'
@@ -254,11 +241,6 @@ jobs:
echo "All benchmark tests completed!" echo "All benchmark tests completed!"
- name: Cleanup logs directory
if: always()
run: |
rm -rf python/sglang/logs || true
# =============================================== multimodal_gen ==================================================== # =============================================== multimodal_gen ====================================================
multimodal-gen-test: multimodal-gen-test:
needs: [check-changes] needs: [check-changes]
@@ -278,12 +260,6 @@ jobs:
cd python cd python
pytest -s -v --log-cli-level=INFO sglang/multimodal_gen/test/server/test_server_performance.py pytest -s -v --log-cli-level=INFO sglang/multimodal_gen/test/server/test_server_performance.py
- name: Cleanup logs directory
if: always()
run: |
# Remove logs directory to prevent NFS lock files
rm -rf python/sglang/logs || true
# Adding a single CUDA13 smoke test to verify that the kernel builds and runs # Adding a single CUDA13 smoke test to verify that the kernel builds and runs
# TODO: Add back this test when it can pass on CI # TODO: Add back this test when it can pass on CI
# cuda13-kernel-smoke-test: # cuda13-kernel-smoke-test:
@@ -387,11 +363,6 @@ jobs:
cd test/srt cd test/srt
python3 run_suite.py --suite per-commit-1-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 15 python3 run_suite.py --suite per-commit-1-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 15
- name: Cleanup logs directory
if: always()
run: |
rm -rf python/sglang/logs || true
unit-test-backend-2-gpu: unit-test-backend-2-gpu:
needs: [check-changes, unit-test-backend-1-gpu, sgl-kernel-build-wheels] needs: [check-changes, unit-test-backend-1-gpu, sgl-kernel-build-wheels]
if: always() && !failure() && !cancelled() && if: always() && !failure() && !cancelled() &&
@@ -425,11 +396,6 @@ jobs:
cd test/srt cd test/srt
python3 run_suite.py --suite per-commit-2-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 python3 run_suite.py --suite per-commit-2-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 2
- name: Cleanup logs directory
if: always()
run: |
rm -rf python/sglang/logs || true
unit-test-backend-4-gpu: unit-test-backend-4-gpu:
needs: [check-changes, unit-test-backend-2-gpu, sgl-kernel-build-wheels] needs: [check-changes, unit-test-backend-2-gpu, sgl-kernel-build-wheels]
if: always() && !failure() && !cancelled() && if: always() && !failure() && !cancelled() &&
@@ -591,11 +557,6 @@ jobs:
python3 -m unittest test_bench_serving.TestBenchServing.test_lora_online_latency python3 -m unittest test_bench_serving.TestBenchServing.test_lora_online_latency
python3 -m unittest test_bench_serving.TestBenchServing.test_lora_online_latency_with_concurrent_adapter_updates python3 -m unittest test_bench_serving.TestBenchServing.test_lora_online_latency_with_concurrent_adapter_updates
- name: Cleanup logs directory
if: always()
run: |
rm -rf python/sglang/logs || true
performance-test-1-gpu-part-2: performance-test-1-gpu-part-2:
needs: [check-changes, sgl-kernel-build-wheels, stage-a-test-1] needs: [check-changes, sgl-kernel-build-wheels, stage-a-test-1]
if: always() && !failure() && !cancelled() && if: always() && !failure() && !cancelled() &&
@@ -649,11 +610,6 @@ jobs:
cd test/srt cd test/srt
python3 -m unittest test_bench_serving.TestBenchServing.test_vlm_online_latency python3 -m unittest test_bench_serving.TestBenchServing.test_vlm_online_latency
- name: Cleanup logs directory
if: always()
run: |
rm -rf python/sglang/logs || true
performance-test-1-gpu-part-3: performance-test-1-gpu-part-3:
needs: [check-changes, sgl-kernel-build-wheels, stage-a-test-1] needs: [check-changes, sgl-kernel-build-wheels, stage-a-test-1]
if: always() && !failure() && !cancelled() && if: always() && !failure() && !cancelled() &&
@@ -765,11 +721,6 @@ jobs:
cd test/srt cd test/srt
python3 -m unittest test_bench_serving.TestBenchServing.test_pp_long_context_prefill python3 -m unittest test_bench_serving.TestBenchServing.test_pp_long_context_prefill
- name: Cleanup logs directory
if: always()
run: |
rm -rf python/sglang/logs || true
accuracy-test-1-gpu: accuracy-test-1-gpu:
needs: [check-changes, sgl-kernel-build-wheels, stage-a-test-1] needs: [check-changes, sgl-kernel-build-wheels, stage-a-test-1]
if: always() && !failure() && !cancelled() && if: always() && !failure() && !cancelled() &&
@@ -802,11 +753,6 @@ jobs:
cd test/srt cd test/srt
python3 test_eval_accuracy_large.py python3 test_eval_accuracy_large.py
- name: Cleanup logs directory
if: always()
run: |
rm -rf python/sglang/logs || true
accuracy-test-2-gpu: accuracy-test-2-gpu:
needs: [check-changes, accuracy-test-1-gpu, sgl-kernel-build-wheels] needs: [check-changes, accuracy-test-1-gpu, sgl-kernel-build-wheels]
if: always() && !failure() && !cancelled() && if: always() && !failure() && !cancelled() &&
@@ -839,11 +785,6 @@ jobs:
cd test/srt cd test/srt
python3 test_moe_eval_accuracy_large.py python3 test_moe_eval_accuracy_large.py
- name: Cleanup logs directory
if: always()
run: |
rm -rf python/sglang/logs || true
unit-test-deepep-4-gpu: unit-test-deepep-4-gpu:
needs: [check-changes, unit-test-backend-2-gpu, sgl-kernel-build-wheels] needs: [check-changes, unit-test-backend-2-gpu, sgl-kernel-build-wheels]
if: always() && !failure() && !cancelled() && if: always() && !failure() && !cancelled() &&
@@ -978,7 +919,7 @@ jobs:
sgl-kernel-mla-test, sgl-kernel-mla-test,
sgl-kernel-benchmark-test, sgl-kernel-benchmark-test,
# multimodal-gen-test, multimodal-gen-test,
stage-a-test-1, stage-a-test-1,
unit-test-backend-1-gpu, unit-test-backend-1-gpu,
@@ -10,13 +10,28 @@ from typing import Any
from dateutil.tz import UTC from dateutil.tz import UTC
LOG_DIR = os.environ.get("SGLANG_PERF_LOG_DIR")
if LOG_DIR: def get_diffusion_perf_log_dir() -> str:
LOG_DIR = os.path.abspath(LOG_DIR) """
elif LOG_DIR is None: # Not set Determines the directory for performance logs, centralizing the logic.
project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../"))
LOG_DIR = os.path.join(project_root, "logs") Resolution order:
# if LOG_DIR is "", it will remain "", disabling file logging. 1. SGLANG_PERF_LOG_DIR environment variable, if set and not empty.
2. Default to ~/.cache/sglang/logs if the environment variable is not set.
3. Returns an empty string if SGLANG_PERF_LOG_DIR is set to an empty string,
which effectively disables file logging.
"""
log_dir = os.environ.get("SGLANG_PERF_LOG_DIR")
if log_dir:
return os.path.abspath(log_dir)
if log_dir is None:
# Not set, use default
return os.path.join(os.path.expanduser("~/.cache/sglang"), "logs")
# Is set, but is an empty string
return ""
LOG_DIR = get_diffusion_perf_log_dir()
# Configure a specific logger for performance metrics # Configure a specific logger for performance metrics
perf_logger = logging.getLogger("performance") perf_logger = logging.getLogger("performance")
@@ -105,7 +105,7 @@ class ServerManager:
def start(self) -> ServerContext: def start(self) -> ServerContext:
"""Start the diffusion server and wait for readiness.""" """Start the diffusion server and wait for readiness."""
log_dir, perf_log_path = prepare_perf_log(Path(__file__)) log_dir, perf_log_path = prepare_perf_log()
safe_model_name = self.model.replace("/", "_") safe_model_name = self.model.replace("/", "_")
stdout_path = ( stdout_path = (
@@ -17,6 +17,9 @@ from PIL import Image
from sglang.multimodal_gen.configs.sample.base import DataType from sglang.multimodal_gen.configs.sample.base import DataType
from sglang.multimodal_gen.runtime.utils.common import get_bool_env_var from sglang.multimodal_gen.runtime.utils.common import get_bool_env_var
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
from sglang.multimodal_gen.runtime.utils.performance_logger import (
get_diffusion_perf_log_dir,
)
logger = init_logger(__name__) logger = init_logger(__name__)
@@ -107,12 +110,15 @@ def check_image_size(ut, image, width, height):
ut.assertEqual(image.size, (width, height)) ut.assertEqual(image.size, (width, height))
def get_perf_log_dir(start_file: Path) -> Path: def get_perf_log_dir() -> Path:
"""Mirror runtime/utils/performance_logger.py behaviour for locating logs.""" """Gets the performance log directory from the centralized sglang utility."""
this_file = start_file.resolve() log_dir_str = get_diffusion_perf_log_dir()
root_logs = this_file.parents[3] / "logs" if not log_dir_str:
fallback = this_file.parents[2] / "logs" raise RuntimeError(
return root_logs if root_logs.exists() or not fallback.exists() else fallback "Performance logging is disabled (SGLANG_PERF_LOG_DIR is empty), "
"but a test tried to access the log directory."
)
return Path(log_dir_str)
def _ensure_log_path(log_dir: Path) -> Path: def _ensure_log_path(log_dir: Path) -> Path:
@@ -129,9 +135,9 @@ def clear_perf_log(log_dir: Path) -> Path:
return log_path return log_path
def prepare_perf_log(start_file: Path) -> tuple[Path, Path]: def prepare_perf_log() -> tuple[Path, Path]:
"""Convenience helper to resolve and clear the perf log in one call.""" """Convenience helper to resolve and clear the perf log in one call."""
log_dir = get_perf_log_dir(start_file) log_dir = get_perf_log_dir()
log_path = clear_perf_log(log_dir) log_path = clear_perf_log(log_dir)
return log_dir, log_path return log_dir, log_path