fix: change performance log directory to cache path (#13482)
Co-authored-by: Mick <mickjagger19@icloud.com>
This commit is contained in:
@@ -21,9 +21,6 @@ concurrency:
|
|||||||
group: pr-test-${{ github.ref }}
|
group: pr-test-${{ github.ref }}
|
||||||
cancel-in-progress: true
|
cancel-in-progress: true
|
||||||
|
|
||||||
env:
|
|
||||||
SGLANG_IS_IN_CI: true
|
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
call-gate:
|
call-gate:
|
||||||
uses: ./.github/workflows/pr-gate.yml
|
uses: ./.github/workflows/pr-gate.yml
|
||||||
@@ -169,11 +166,6 @@ jobs:
|
|||||||
cd sgl-kernel
|
cd sgl-kernel
|
||||||
pytest tests/
|
pytest tests/
|
||||||
|
|
||||||
- name: Cleanup logs directory
|
|
||||||
if: always()
|
|
||||||
run: |
|
|
||||||
rm -rf python/sglang/logs || true
|
|
||||||
|
|
||||||
sgl-kernel-mla-test:
|
sgl-kernel-mla-test:
|
||||||
needs: [check-changes, sgl-kernel-build-wheels]
|
needs: [check-changes, sgl-kernel-build-wheels]
|
||||||
if: needs.check-changes.outputs.sgl_kernel == 'true'
|
if: needs.check-changes.outputs.sgl_kernel == 'true'
|
||||||
@@ -205,11 +197,6 @@ jobs:
|
|||||||
cd test/srt
|
cd test/srt
|
||||||
python3 test_mla_deepseek_v3.py
|
python3 test_mla_deepseek_v3.py
|
||||||
|
|
||||||
- name: Cleanup logs directory
|
|
||||||
if: always()
|
|
||||||
run: |
|
|
||||||
rm -rf python/sglang/logs || true
|
|
||||||
|
|
||||||
sgl-kernel-benchmark-test:
|
sgl-kernel-benchmark-test:
|
||||||
needs: [check-changes, sgl-kernel-build-wheels]
|
needs: [check-changes, sgl-kernel-build-wheels]
|
||||||
if: needs.check-changes.outputs.sgl_kernel == 'true'
|
if: needs.check-changes.outputs.sgl_kernel == 'true'
|
||||||
@@ -254,35 +241,24 @@ jobs:
|
|||||||
|
|
||||||
echo "All benchmark tests completed!"
|
echo "All benchmark tests completed!"
|
||||||
|
|
||||||
- name: Cleanup logs directory
|
|
||||||
if: always()
|
|
||||||
run: |
|
|
||||||
rm -rf python/sglang/logs || true
|
|
||||||
|
|
||||||
# =============================================== multimodal_gen ====================================================
|
# =============================================== multimodal_gen ====================================================
|
||||||
multimodal-gen-test:
|
multimodal-gen-test:
|
||||||
needs: [check-changes]
|
needs: [check-changes]
|
||||||
if: needs.check-changes.outputs.multimodal_gen == 'true'
|
if: needs.check-changes.outputs.multimodal_gen == 'true'
|
||||||
runs-on: 1-gpu-runner
|
runs-on: 1-gpu-runner
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout code
|
- name: Checkout code
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
- name: Install dependencies
|
- name: Install dependencies
|
||||||
run: |
|
run: |
|
||||||
CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/ci_install_dependency.sh diffusion
|
CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/ci_install_dependency.sh diffusion
|
||||||
|
|
||||||
- name: Run diffusion server tests
|
- name: Run diffusion server tests
|
||||||
timeout-minutes: 60
|
timeout-minutes: 60
|
||||||
run: |
|
run: |
|
||||||
cd python
|
cd python
|
||||||
pytest -s -v --log-cli-level=INFO sglang/multimodal_gen/test/server/test_server_performance.py
|
pytest -s -v --log-cli-level=INFO sglang/multimodal_gen/test/server/test_server_performance.py
|
||||||
|
|
||||||
- name: Cleanup logs directory
|
|
||||||
if: always()
|
|
||||||
run: |
|
|
||||||
# Remove logs directory to prevent NFS lock files
|
|
||||||
rm -rf python/sglang/logs || true
|
|
||||||
|
|
||||||
# Adding a single CUDA13 smoke test to verify that the kernel builds and runs
|
# Adding a single CUDA13 smoke test to verify that the kernel builds and runs
|
||||||
# TODO: Add back this test when it can pass on CI
|
# TODO: Add back this test when it can pass on CI
|
||||||
@@ -387,11 +363,6 @@ jobs:
|
|||||||
cd test/srt
|
cd test/srt
|
||||||
python3 run_suite.py --suite per-commit-1-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 15
|
python3 run_suite.py --suite per-commit-1-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 15
|
||||||
|
|
||||||
- name: Cleanup logs directory
|
|
||||||
if: always()
|
|
||||||
run: |
|
|
||||||
rm -rf python/sglang/logs || true
|
|
||||||
|
|
||||||
unit-test-backend-2-gpu:
|
unit-test-backend-2-gpu:
|
||||||
needs: [check-changes, unit-test-backend-1-gpu, sgl-kernel-build-wheels]
|
needs: [check-changes, unit-test-backend-1-gpu, sgl-kernel-build-wheels]
|
||||||
if: always() && !failure() && !cancelled() &&
|
if: always() && !failure() && !cancelled() &&
|
||||||
@@ -425,11 +396,6 @@ jobs:
|
|||||||
cd test/srt
|
cd test/srt
|
||||||
python3 run_suite.py --suite per-commit-2-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 2
|
python3 run_suite.py --suite per-commit-2-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 2
|
||||||
|
|
||||||
- name: Cleanup logs directory
|
|
||||||
if: always()
|
|
||||||
run: |
|
|
||||||
rm -rf python/sglang/logs || true
|
|
||||||
|
|
||||||
unit-test-backend-4-gpu:
|
unit-test-backend-4-gpu:
|
||||||
needs: [check-changes, unit-test-backend-2-gpu, sgl-kernel-build-wheels]
|
needs: [check-changes, unit-test-backend-2-gpu, sgl-kernel-build-wheels]
|
||||||
if: always() && !failure() && !cancelled() &&
|
if: always() && !failure() && !cancelled() &&
|
||||||
@@ -591,11 +557,6 @@ jobs:
|
|||||||
python3 -m unittest test_bench_serving.TestBenchServing.test_lora_online_latency
|
python3 -m unittest test_bench_serving.TestBenchServing.test_lora_online_latency
|
||||||
python3 -m unittest test_bench_serving.TestBenchServing.test_lora_online_latency_with_concurrent_adapter_updates
|
python3 -m unittest test_bench_serving.TestBenchServing.test_lora_online_latency_with_concurrent_adapter_updates
|
||||||
|
|
||||||
- name: Cleanup logs directory
|
|
||||||
if: always()
|
|
||||||
run: |
|
|
||||||
rm -rf python/sglang/logs || true
|
|
||||||
|
|
||||||
performance-test-1-gpu-part-2:
|
performance-test-1-gpu-part-2:
|
||||||
needs: [check-changes, sgl-kernel-build-wheels, stage-a-test-1]
|
needs: [check-changes, sgl-kernel-build-wheels, stage-a-test-1]
|
||||||
if: always() && !failure() && !cancelled() &&
|
if: always() && !failure() && !cancelled() &&
|
||||||
@@ -649,11 +610,6 @@ jobs:
|
|||||||
cd test/srt
|
cd test/srt
|
||||||
python3 -m unittest test_bench_serving.TestBenchServing.test_vlm_online_latency
|
python3 -m unittest test_bench_serving.TestBenchServing.test_vlm_online_latency
|
||||||
|
|
||||||
- name: Cleanup logs directory
|
|
||||||
if: always()
|
|
||||||
run: |
|
|
||||||
rm -rf python/sglang/logs || true
|
|
||||||
|
|
||||||
performance-test-1-gpu-part-3:
|
performance-test-1-gpu-part-3:
|
||||||
needs: [check-changes, sgl-kernel-build-wheels, stage-a-test-1]
|
needs: [check-changes, sgl-kernel-build-wheels, stage-a-test-1]
|
||||||
if: always() && !failure() && !cancelled() &&
|
if: always() && !failure() && !cancelled() &&
|
||||||
@@ -765,11 +721,6 @@ jobs:
|
|||||||
cd test/srt
|
cd test/srt
|
||||||
python3 -m unittest test_bench_serving.TestBenchServing.test_pp_long_context_prefill
|
python3 -m unittest test_bench_serving.TestBenchServing.test_pp_long_context_prefill
|
||||||
|
|
||||||
- name: Cleanup logs directory
|
|
||||||
if: always()
|
|
||||||
run: |
|
|
||||||
rm -rf python/sglang/logs || true
|
|
||||||
|
|
||||||
accuracy-test-1-gpu:
|
accuracy-test-1-gpu:
|
||||||
needs: [check-changes, sgl-kernel-build-wheels, stage-a-test-1]
|
needs: [check-changes, sgl-kernel-build-wheels, stage-a-test-1]
|
||||||
if: always() && !failure() && !cancelled() &&
|
if: always() && !failure() && !cancelled() &&
|
||||||
@@ -802,11 +753,6 @@ jobs:
|
|||||||
cd test/srt
|
cd test/srt
|
||||||
python3 test_eval_accuracy_large.py
|
python3 test_eval_accuracy_large.py
|
||||||
|
|
||||||
- name: Cleanup logs directory
|
|
||||||
if: always()
|
|
||||||
run: |
|
|
||||||
rm -rf python/sglang/logs || true
|
|
||||||
|
|
||||||
accuracy-test-2-gpu:
|
accuracy-test-2-gpu:
|
||||||
needs: [check-changes, accuracy-test-1-gpu, sgl-kernel-build-wheels]
|
needs: [check-changes, accuracy-test-1-gpu, sgl-kernel-build-wheels]
|
||||||
if: always() && !failure() && !cancelled() &&
|
if: always() && !failure() && !cancelled() &&
|
||||||
@@ -839,11 +785,6 @@ jobs:
|
|||||||
cd test/srt
|
cd test/srt
|
||||||
python3 test_moe_eval_accuracy_large.py
|
python3 test_moe_eval_accuracy_large.py
|
||||||
|
|
||||||
- name: Cleanup logs directory
|
|
||||||
if: always()
|
|
||||||
run: |
|
|
||||||
rm -rf python/sglang/logs || true
|
|
||||||
|
|
||||||
unit-test-deepep-4-gpu:
|
unit-test-deepep-4-gpu:
|
||||||
needs: [check-changes, unit-test-backend-2-gpu, sgl-kernel-build-wheels]
|
needs: [check-changes, unit-test-backend-2-gpu, sgl-kernel-build-wheels]
|
||||||
if: always() && !failure() && !cancelled() &&
|
if: always() && !failure() && !cancelled() &&
|
||||||
@@ -978,7 +919,7 @@ jobs:
|
|||||||
sgl-kernel-mla-test,
|
sgl-kernel-mla-test,
|
||||||
sgl-kernel-benchmark-test,
|
sgl-kernel-benchmark-test,
|
||||||
|
|
||||||
# multimodal-gen-test,
|
multimodal-gen-test,
|
||||||
|
|
||||||
stage-a-test-1,
|
stage-a-test-1,
|
||||||
unit-test-backend-1-gpu,
|
unit-test-backend-1-gpu,
|
||||||
|
|||||||
@@ -10,13 +10,28 @@ from typing import Any
|
|||||||
|
|
||||||
from dateutil.tz import UTC
|
from dateutil.tz import UTC
|
||||||
|
|
||||||
LOG_DIR = os.environ.get("SGLANG_PERF_LOG_DIR")
|
|
||||||
if LOG_DIR:
|
def get_diffusion_perf_log_dir() -> str:
|
||||||
LOG_DIR = os.path.abspath(LOG_DIR)
|
"""
|
||||||
elif LOG_DIR is None: # Not set
|
Determines the directory for performance logs, centralizing the logic.
|
||||||
project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../"))
|
|
||||||
LOG_DIR = os.path.join(project_root, "logs")
|
Resolution order:
|
||||||
# if LOG_DIR is "", it will remain "", disabling file logging.
|
1. SGLANG_PERF_LOG_DIR environment variable, if set and not empty.
|
||||||
|
2. Default to ~/.cache/sglang/logs if the environment variable is not set.
|
||||||
|
3. Returns an empty string if SGLANG_PERF_LOG_DIR is set to an empty string,
|
||||||
|
which effectively disables file logging.
|
||||||
|
"""
|
||||||
|
log_dir = os.environ.get("SGLANG_PERF_LOG_DIR")
|
||||||
|
if log_dir:
|
||||||
|
return os.path.abspath(log_dir)
|
||||||
|
if log_dir is None:
|
||||||
|
# Not set, use default
|
||||||
|
return os.path.join(os.path.expanduser("~/.cache/sglang"), "logs")
|
||||||
|
# Is set, but is an empty string
|
||||||
|
return ""
|
||||||
|
|
||||||
|
|
||||||
|
LOG_DIR = get_diffusion_perf_log_dir()
|
||||||
|
|
||||||
# Configure a specific logger for performance metrics
|
# Configure a specific logger for performance metrics
|
||||||
perf_logger = logging.getLogger("performance")
|
perf_logger = logging.getLogger("performance")
|
||||||
|
|||||||
@@ -105,7 +105,7 @@ class ServerManager:
|
|||||||
|
|
||||||
def start(self) -> ServerContext:
|
def start(self) -> ServerContext:
|
||||||
"""Start the diffusion server and wait for readiness."""
|
"""Start the diffusion server and wait for readiness."""
|
||||||
log_dir, perf_log_path = prepare_perf_log(Path(__file__))
|
log_dir, perf_log_path = prepare_perf_log()
|
||||||
|
|
||||||
safe_model_name = self.model.replace("/", "_")
|
safe_model_name = self.model.replace("/", "_")
|
||||||
stdout_path = (
|
stdout_path = (
|
||||||
|
|||||||
@@ -17,6 +17,9 @@ from PIL import Image
|
|||||||
from sglang.multimodal_gen.configs.sample.base import DataType
|
from sglang.multimodal_gen.configs.sample.base import DataType
|
||||||
from sglang.multimodal_gen.runtime.utils.common import get_bool_env_var
|
from sglang.multimodal_gen.runtime.utils.common import get_bool_env_var
|
||||||
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
|
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
|
||||||
|
from sglang.multimodal_gen.runtime.utils.performance_logger import (
|
||||||
|
get_diffusion_perf_log_dir,
|
||||||
|
)
|
||||||
|
|
||||||
logger = init_logger(__name__)
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
@@ -107,12 +110,15 @@ def check_image_size(ut, image, width, height):
|
|||||||
ut.assertEqual(image.size, (width, height))
|
ut.assertEqual(image.size, (width, height))
|
||||||
|
|
||||||
|
|
||||||
def get_perf_log_dir(start_file: Path) -> Path:
|
def get_perf_log_dir() -> Path:
|
||||||
"""Mirror runtime/utils/performance_logger.py behaviour for locating logs."""
|
"""Gets the performance log directory from the centralized sglang utility."""
|
||||||
this_file = start_file.resolve()
|
log_dir_str = get_diffusion_perf_log_dir()
|
||||||
root_logs = this_file.parents[3] / "logs"
|
if not log_dir_str:
|
||||||
fallback = this_file.parents[2] / "logs"
|
raise RuntimeError(
|
||||||
return root_logs if root_logs.exists() or not fallback.exists() else fallback
|
"Performance logging is disabled (SGLANG_PERF_LOG_DIR is empty), "
|
||||||
|
"but a test tried to access the log directory."
|
||||||
|
)
|
||||||
|
return Path(log_dir_str)
|
||||||
|
|
||||||
|
|
||||||
def _ensure_log_path(log_dir: Path) -> Path:
|
def _ensure_log_path(log_dir: Path) -> Path:
|
||||||
@@ -129,9 +135,9 @@ def clear_perf_log(log_dir: Path) -> Path:
|
|||||||
return log_path
|
return log_path
|
||||||
|
|
||||||
|
|
||||||
def prepare_perf_log(start_file: Path) -> tuple[Path, Path]:
|
def prepare_perf_log() -> tuple[Path, Path]:
|
||||||
"""Convenience helper to resolve and clear the perf log in one call."""
|
"""Convenience helper to resolve and clear the perf log in one call."""
|
||||||
log_dir = get_perf_log_dir(start_file)
|
log_dir = get_perf_log_dir()
|
||||||
log_path = clear_perf_log(log_dir)
|
log_path = clear_perf_log(log_dir)
|
||||||
return log_dir, log_path
|
return log_dir, log_path
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user