diff --git a/docs/docs/sglang-diffusion/api/cli.mdx b/docs/docs/sglang-diffusion/api/cli.mdx index 570833db8e..2e3dfcdf2e 100644 --- a/docs/docs/sglang-diffusion/api/cli.mdx +++ b/docs/docs/sglang-diffusion/api/cli.mdx @@ -492,6 +492,11 @@ export SGLANG_S3_ENDPOINT_URL=https://minio.example.com See [Environment Variables](/docs/sglang-diffusion/environment_variables) for the full set of storage options. +Cancellation waits for active uploads to finish before request cleanup can remove +their source files. Allow for the storage SDK's timeouts and retries in your +shutdown grace period. Cancellation does not roll back objects already uploaded; +use your bucket's lifecycle rules for object retention. + ## Component Path Overrides Override individual pipeline components such as `vae`, `transformer`, or `text_encoder` with `---path`. diff --git a/python/sglang/multimodal_gen/runtime/entrypoints/openai/image_api.py b/python/sglang/multimodal_gen/runtime/entrypoints/openai/image_api.py index b842c17690..2fa1e39570 100644 --- a/python/sglang/multimodal_gen/runtime/entrypoints/openai/image_api.py +++ b/python/sglang/multimodal_gen/runtime/entrypoints/openai/image_api.py @@ -79,9 +79,15 @@ def _b64_encode_images(images: list[bytes]) -> list[str]: async def _upload_and_cleanup_images(paths: list[str]) -> list[str | None]: - return await asyncio.gather( - *(cloud_storage.upload_and_cleanup(path) for path in paths) + results = await asyncio.gather( + *(cloud_storage.upload_and_cleanup(path) for path in paths), + return_exceptions=True, ) + # join every upload before a cancellation or failure permits directory cleanup + for result in results: + if isinstance(result, BaseException): + raise result + return results def _fallback_image_urls( diff --git a/python/sglang/multimodal_gen/runtime/entrypoints/openai/storage.py b/python/sglang/multimodal_gen/runtime/entrypoints/openai/storage.py index 68f66827c2..5cccd89dee 100644 --- a/python/sglang/multimodal_gen/runtime/entrypoints/openai/storage.py +++ b/python/sglang/multimodal_gen/runtime/entrypoints/openai/storage.py @@ -1,5 +1,6 @@ import asyncio import os +from contextlib import suppress from typing import Optional from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger @@ -69,8 +70,16 @@ class CloudStorage: ) try: - # Offload the blocking I/O call to a thread executor - await asyncio.get_running_loop().run_in_executor(None, _sync_upload) + upload = asyncio.get_running_loop().run_in_executor(None, _sync_upload) + await asyncio.shield(upload) + except asyncio.CancelledError: + # the SDK may still read local files; finish before caller cleanup + while not upload.done(): + with suppress(asyncio.CancelledError, Exception): + await asyncio.shield(upload) + if not upload.cancelled(): + upload.exception() + raise except Exception as e: # If upload fails, log the error and return None for fallback logger.error(f"Upload failed for {destination_key}: {e}") diff --git a/python/sglang/multimodal_gen/runtime/entrypoints/openai/video_api.py b/python/sglang/multimodal_gen/runtime/entrypoints/openai/video_api.py index 43bc9019be..a300e0a407 100644 --- a/python/sglang/multimodal_gen/runtime/entrypoints/openai/video_api.py +++ b/python/sglang/multimodal_gen/runtime/entrypoints/openai/video_api.py @@ -361,11 +361,15 @@ async def _save_first_input_image( async def _upload_and_cleanup_videos(paths: list[str]) -> list[str | None]: - return list( - await asyncio.gather( - *(cloud_storage.upload_and_cleanup(path) for path in paths) - ) + results = await asyncio.gather( + *(cloud_storage.upload_and_cleanup(path) for path in paths), + return_exceptions=True, ) + # join every upload before a cancellation or failure permits directory cleanup + for result in results: + if isinstance(result, BaseException): + raise result + return results async def _dispatch_job_async( diff --git a/python/sglang/multimodal_gen/test/unit/test_storage.py b/python/sglang/multimodal_gen/test/unit/test_storage.py index 49233268d5..e7ca70d041 100644 --- a/python/sglang/multimodal_gen/test/unit/test_storage.py +++ b/python/sglang/multimodal_gen/test/unit/test_storage.py @@ -7,9 +7,16 @@ Tests verify file upload, cleanup, URL generation, and error handling. import asyncio import os import sys +import threading +from contextlib import suppress +from pathlib import Path +from tempfile import TemporaryDirectory from types import SimpleNamespace +import pytest + import sglang.multimodal_gen.runtime.entrypoints.openai.storage as storage_mod +from sglang.multimodal_gen.runtime.entrypoints.openai import image_api, video_api from sglang.multimodal_gen.runtime.entrypoints.openai.storage import CloudStorage @@ -228,3 +235,80 @@ def test_env_configured_storage_uploads_and_cleans_up(tmp_path, monkeypatch): {"ExtraArgs": {"ContentType": "image/png"}}, ) ] + + +@pytest.mark.parametrize("kind", ["single", "images", "videos"]) +@pytest.mark.parametrize("upload_fails", [False, True]) +def test_cancelled_upload_finishes_reading_before_request_cleanup( + tmp_path, monkeypatch, kind, upload_fails +): + """Cancellation must not delete files still read by one or more SDK threads.""" + monkeypatch.delenv("SGLANG_CLOUD_STORAGE_TYPE", raising=False) + storage = CloudStorage() + storage.enabled = True + storage.bucket_name = "test" + storage.endpoint_url = "https://s3.example.com" + storage.region_name = None + monkeypatch.setattr(image_api, "cloud_storage", storage) + monkeypatch.setattr(video_api, "cloud_storage", storage) + count = 1 if kind == "single" else 2 + release = [threading.Event() for _ in range(count)] + read = [] + + async def run_test(): + loop = asyncio.get_running_loop() + started = [asyncio.Event() for _ in range(count)] + finished = [asyncio.Event() for _ in range(count)] + + def upload(local_path, bucket, key, **kwargs): + index = int(Path(local_path).stem) + loop.call_soon_threadsafe(started[index].set) + assert release[index].wait(10) + try: + read.append(Path(local_path).read_bytes()) + if upload_fails: + raise OSError("storage unavailable") + finally: + loop.call_soon_threadsafe(finished[index].set) + + storage.client = SimpleNamespace(upload_file=upload) + + async def request(): + with TemporaryDirectory(dir=tmp_path) as directory: + paths = [Path(directory) / f"{i}.mp4" for i in range(count)] + for path in paths: + path.write_bytes(b"media") + paths = [str(path) for path in paths] + if kind == "single": + await storage.upload_and_cleanup(paths[0]) + elif kind == "images": + await image_api._upload_and_cleanup_images(paths) + else: + await video_api._upload_and_cleanup_videos(paths) + + task = asyncio.create_task(request()) + try: + await asyncio.wait_for( + asyncio.gather(*(event.wait() for event in started)), timeout=10 + ) + for _ in range(2): + task.cancel() + for _ in range(5): + await asyncio.sleep(0) + assert not task.done(), "cleanup raced with the active upload" + if count > 1: + release[0].set() + await asyncio.wait_for(finished[0].wait(), timeout=10) + for _ in range(10): + await asyncio.sleep(0) + assert not task.done(), "cleanup did not wait for every output" + finally: + for event in release: + event.set() + with suppress(asyncio.CancelledError): + await task + assert task.cancelled() + assert read == [b"media"] * count + assert not list(tmp_path.iterdir()) + + asyncio.run(run_test())