CI: fix NFS EBUSY error in PR test workflow (#13460)

Co-authored-by: Kangyan-Zhou <zky314343421@gmail.com>
Co-authored-by: Mick <mickjagger19@icloud.com>

.github/workflows/pr-test.yml

@@ -21,6 +21,9 @@ concurrency:
   group: pr-test-${{ github.ref }}
   cancel-in-progress: true
 
+env:
+  SGLANG_IS_IN_CI: true
+
 jobs:
   # =============================================== check changes ====================================================
   check-changes:
@@ -232,6 +235,11 @@ jobs:
           cd sgl-kernel
           pytest tests/
 
+      - name: Cleanup logs directory
+        if: always()
+        run: |
+          rm -rf python/sglang/logs || true
+
   sgl-kernel-mla-test:
     needs: [check-changes, sgl-kernel-build-wheels]
     if: needs.check-changes.outputs.sgl_kernel == 'true'
@@ -263,6 +271,11 @@ jobs:
           cd test/srt
           python3 test_mla_deepseek_v3.py
 
+      - name: Cleanup logs directory
+        if: always()
+        run: |
+          rm -rf python/sglang/logs || true
+
   sgl-kernel-benchmark-test:
     needs: [check-changes, sgl-kernel-build-wheels]
     if: needs.check-changes.outputs.sgl_kernel == 'true'
@@ -307,24 +320,35 @@ jobs:
 
           echo "All benchmark tests completed!"
 
+      - name: Cleanup logs directory
+        if: always()
+        run: |
+          rm -rf python/sglang/logs || true
+
   # =============================================== multimodal_gen ====================================================
-  # multimodal-gen-test:
+  multimodal-gen-test:
-  #   needs: [check-changes]
+   needs: [check-changes]
-  #   if: needs.check-changes.outputs.multimodal_gen == 'true'
+   if: needs.check-changes.outputs.multimodal_gen == 'true'
-  #   runs-on: 1-gpu-runner
+   runs-on: 1-gpu-runner
-  #   steps:
+   steps:
-  #     - name: Checkout code
+     - name: Checkout code
-  #       uses: actions/checkout@v4
+       uses: actions/checkout@v4
 
-  #     - name: Install dependencies
+     - name: Install dependencies
-  #       run: |
+       run: |
-  #         CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/ci_install_dependency.sh diffusion
+         CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/ci_install_dependency.sh diffusion
 
-  #     - name: Run diffusion server tests
+     - name: Run diffusion server tests
-  #       timeout-minutes: 60
+       timeout-minutes: 60
-  #       run: |
+       run: |
-  #         cd python
+         cd python
-  #         pytest -s -v --log-cli-level=INFO sglang/multimodal_gen/test/server/test_server_performance.py
+         pytest -s -v --log-cli-level=INFO sglang/multimodal_gen/test/server/test_server_performance.py
+
+     - name: Cleanup logs directory
+       if: always()
+       run: |
+         # Remove logs directory to prevent NFS lock files
+         rm -rf python/sglang/logs || true
 
   # Adding a single CUDA13 smoke test to verify that the kernel builds and runs
   # TODO: Add back this test when it can pass on CI
@@ -390,6 +414,11 @@ jobs:
           # temporarily put backend-independent cpu tests here
           python3 run_suite.py --hw cpu --suite default
 
+      - name: Cleanup logs directory
+        if: always()
+        run: |
+          rm -rf python/sglang/logs || true
+
   unit-test-backend-1-gpu:
     needs: [check-changes, stage-a-test-1, sgl-kernel-build-wheels]
     if: always() && !failure() && !cancelled() &&
@@ -424,6 +453,11 @@ jobs:
           cd test/srt
           python3 run_suite.py --suite per-commit-1-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 15
 
+      - name: Cleanup logs directory
+        if: always()
+        run: |
+          rm -rf python/sglang/logs || true
+
   unit-test-backend-2-gpu:
     needs: [check-changes, unit-test-backend-1-gpu, sgl-kernel-build-wheels]
     if: always() && !failure() && !cancelled() &&
@@ -457,6 +491,11 @@ jobs:
           cd test/srt
           python3 run_suite.py --suite per-commit-2-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 2
 
+      - name: Cleanup logs directory
+        if: always()
+        run: |
+          rm -rf python/sglang/logs || true
+
   unit-test-backend-4-gpu:
     needs: [check-changes, unit-test-backend-2-gpu, sgl-kernel-build-wheels]
     if: always() && !failure() && !cancelled() &&
@@ -618,6 +657,11 @@ jobs:
           python3 -m unittest test_bench_serving.TestBenchServing.test_lora_online_latency
           python3 -m unittest test_bench_serving.TestBenchServing.test_lora_online_latency_with_concurrent_adapter_updates
 
+      - name: Cleanup logs directory
+        if: always()
+        run: |
+          rm -rf python/sglang/logs || true
+
   performance-test-1-gpu-part-2:
     needs: [check-changes, sgl-kernel-build-wheels, stage-a-test-1]
     if: always() && !failure() && !cancelled() &&
@@ -671,6 +715,11 @@ jobs:
           cd test/srt
           python3 -m unittest test_bench_serving.TestBenchServing.test_vlm_online_latency
 
+      - name: Cleanup logs directory
+        if: always()
+        run: |
+          rm -rf python/sglang/logs || true
+
   performance-test-1-gpu-part-3:
     needs: [check-changes, sgl-kernel-build-wheels, stage-a-test-1]
     if: always() && !failure() && !cancelled() &&
@@ -718,6 +767,11 @@ jobs:
           cd test/srt
           python3 -m unittest test_bench_serving.TestBenchServing.test_embeddings_api_batch_scaling
 
+      - name: Cleanup logs directory
+        if: always()
+        run: |
+          rm -rf python/sglang/logs || true
+
   performance-test-2-gpu:
     needs: [check-changes, unit-test-backend-2-gpu, sgl-kernel-build-wheels]
     if: always() && !failure() && !cancelled() &&
@@ -777,6 +831,11 @@ jobs:
           cd test/srt
           python3 -m unittest test_bench_serving.TestBenchServing.test_pp_long_context_prefill
 
+      - name: Cleanup logs directory
+        if: always()
+        run: |
+          rm -rf python/sglang/logs || true
+
   accuracy-test-1-gpu:
     needs: [check-changes, sgl-kernel-build-wheels, stage-a-test-1]
     if: always() && !failure() && !cancelled() &&
@@ -809,6 +868,11 @@ jobs:
           cd test/srt
           python3 test_eval_accuracy_large.py
 
+      - name: Cleanup logs directory
+        if: always()
+        run: |
+          rm -rf python/sglang/logs || true
+
   accuracy-test-2-gpu:
     needs: [check-changes, accuracy-test-1-gpu, sgl-kernel-build-wheels]
     if: always() && !failure() && !cancelled() &&
@@ -841,6 +905,11 @@ jobs:
           cd test/srt
           python3 test_moe_eval_accuracy_large.py
 
+      - name: Cleanup logs directory
+        if: always()
+        run: |
+          rm -rf python/sglang/logs || true
+
   unit-test-deepep-4-gpu:
     needs: [check-changes, unit-test-backend-2-gpu, sgl-kernel-build-wheels]
     if: always() && !failure() && !cancelled() &&

python/sglang/multimodal_gen/runtime/utils/performance_logger.py

@@ -13,23 +13,66 @@ from dateutil.tz import UTC
 LOG_DIR = os.environ.get("SGLANG_PERF_LOG_DIR")
 if LOG_DIR:
     LOG_DIR = os.path.abspath(LOG_DIR)
-else:
+elif LOG_DIR is None:  # Not set
     project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../"))
     LOG_DIR = os.path.join(project_root, "logs")
+# if LOG_DIR is "", it will remain "", disabling file logging.
 
 # Configure a specific logger for performance metrics
 perf_logger = logging.getLogger("performance")
 perf_logger.setLevel(logging.INFO)
 perf_logger.propagate = False  # Prevent perf logs from going to the main logger
 
-# Ensure the logs directory exists
+_perf_logger_initialized = False
-if not os.path.exists(LOG_DIR):
-    os.makedirs(LOG_DIR)
 
-# Set up a file handler for the performance logger
+
-handler = logging.FileHandler(os.path.join(LOG_DIR, "performance.log"))
+class OnDemandFileHandler(logging.Handler):
-handler.setFormatter(logging.Formatter("%(message)s"))
+    """
-perf_logger.addHandler(handler)
+    A logging handler that opens the file for each log record, writes, and closes it.
+    This is less performant than FileHandler but avoids long-lived file handles,
+    which can be problematic on certain filesystems like NFS.
+    """
+
+    def __init__(self, filename: str, mode: str = "a", encoding: str | None = None):
+        super().__init__()
+        self.baseFilename = os.path.abspath(filename)
+        self.mode = mode
+        self.encoding = encoding
+        self.terminator = "\n"
+
+    def emit(self, record: logging.LogRecord):
+        """Emit a record."""
+        try:
+            msg = self.format(record)
+            with open(
+                self.baseFilename, self.mode, encoding=self.encoding, errors="replace"
+            ) as f:
+                f.write(msg + self.terminator)
+        except Exception:
+            self.handleError(record)
+
+
+def _initialize_perf_logger():
+    """Initialize the performance logger with a file handler."""
+    global _perf_logger_initialized
+    if _perf_logger_initialized or not LOG_DIR:
+        return
+
+    try:
+        # Ensure the logs directory exists
+        if not os.path.exists(LOG_DIR):
+            os.makedirs(LOG_DIR)
+
+        # Set up a file handler for the performance logger
+        handler = OnDemandFileHandler(os.path.join(LOG_DIR, "performance.log"))
+        handler.setFormatter(logging.Formatter("%(message)s"))
+        perf_logger.addHandler(handler)
+    except (OSError, PermissionError) as e:
+        perf_logger.warning(f"Failed to initialize performance logger: {e}")
+        # Disable file logging if initialization fails
+        globals()["LOG_DIR"] = ""
+    finally:
+        _perf_logger_initialized = True
 
 
 def get_git_commit_hash() -> str:
@@ -69,6 +112,7 @@ class PerformanceLogger:
 
     def log_total_duration(self, tag: str):
         """Logs the total duration of the operation and all recorded steps."""
+        _initialize_perf_logger()
         total_duration = time.monotonic() - self.start_time
         log_entry = {
             "timestamp": datetime.now(UTC).isoformat(),
@@ -82,6 +126,7 @@ class PerformanceLogger:
 
     def log_stage_metric(self, stage_name: str, duration_ms: float):
         """Logs a single pipeline stage timing entry."""
+        _initialize_perf_logger()
         log_entry = {
             "timestamp": datetime.now(UTC).isoformat(),
             "request_id": self.request_id,
@@ -100,6 +145,7 @@ class PerformanceLogger:
             stages: Either a PipelineLoggingInfo instance or any object exposing
                 a mapping of stage metadata via a `stages` attribute/dict.
         """
+        _initialize_perf_logger()
         if stages is None:
             return