[diffusion] Don't report video jobs completed after their outputs are deleted (#43084)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Xiaoyu Zhang <1182563586@qq.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Xiaoyu Zhang
parent
aa5c969dc3
commit
a1500b10e3
@@ -103,12 +103,13 @@ class VideoResponse(BaseModel):
|
||||
seconds: str = "4"
|
||||
quality: str = "standard"
|
||||
url: Optional[str] = None
|
||||
urls: Optional[List[Optional[str]]] = None
|
||||
remixed_from_video_id: Optional[str] = None
|
||||
completed_at: Optional[int] = None
|
||||
expires_at: Optional[int] = None
|
||||
error: Optional[Dict[str, Any]] = None
|
||||
file_path: Optional[str] = None
|
||||
file_paths: Optional[List[str]] = None
|
||||
file_paths: Optional[List[Optional[str]]] = None
|
||||
num_outputs: Optional[int] = None
|
||||
peak_memory_mb: Optional[float] = None
|
||||
inference_time_s: Optional[float] = None
|
||||
|
||||
@@ -360,6 +360,14 @@ async def _save_first_input_image(
|
||||
)
|
||||
|
||||
|
||||
async def _upload_and_cleanup_videos(paths: list[str]) -> list[str | None]:
|
||||
return list(
|
||||
await asyncio.gather(
|
||||
*(cloud_storage.upload_and_cleanup(path) for path in paths)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
async def _dispatch_job_async(
|
||||
job_id: str,
|
||||
batch: Req,
|
||||
@@ -397,19 +405,31 @@ async def _dispatch_job_async(
|
||||
)
|
||||
raise
|
||||
|
||||
cloud_url = await cloud_storage.upload_and_cleanup(save_file_path)
|
||||
cloud_urls = await _upload_and_cleanup_videos(save_file_path_list)
|
||||
if not output_persistent and not all(cloud_urls):
|
||||
# The temp output dir is removed in `finally`, so an output that was
|
||||
# not uploaded would be unretrievable once the job reports completed.
|
||||
raise RuntimeError(
|
||||
"Video output could not be published: no output_path is configured "
|
||||
"and cloud storage upload is disabled or failed."
|
||||
)
|
||||
|
||||
persistent_path = (
|
||||
save_file_path if not cloud_url and output_persistent else None
|
||||
)
|
||||
update_fields = {
|
||||
"status": "completed",
|
||||
"progress": 100,
|
||||
"completed_at": int(time.time()),
|
||||
"url": cloud_url,
|
||||
"file_path": persistent_path,
|
||||
"url": cloud_urls[0],
|
||||
"urls": cloud_urls,
|
||||
"file_path": (
|
||||
os.path.abspath(save_file_path)
|
||||
if not cloud_urls[0] and output_persistent
|
||||
else None
|
||||
),
|
||||
"file_paths": (
|
||||
[os.path.abspath(path) for path in save_file_path_list]
|
||||
[
|
||||
None if url else os.path.abspath(path)
|
||||
for path, url in zip(save_file_path_list, cloud_urls)
|
||||
]
|
||||
if output_persistent
|
||||
else None
|
||||
),
|
||||
@@ -428,6 +448,7 @@ async def _dispatch_job_async(
|
||||
"status": "failed",
|
||||
"error": {"message": str(e)},
|
||||
"url": None,
|
||||
"urls": None,
|
||||
"file_path": None,
|
||||
"file_paths": None,
|
||||
"num_outputs": None,
|
||||
@@ -829,17 +850,33 @@ async def delete_video(video_id: str = Path(...)):
|
||||
return VideoResponse(**job)
|
||||
|
||||
|
||||
def _parse_video_variant_index(variant: str | None) -> int | None:
|
||||
try:
|
||||
return 0 if variant is None else int(variant)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _select_video_variant_url(job: dict, variant: str | None) -> str | None:
|
||||
urls = job.get("urls")
|
||||
if urls:
|
||||
variant_index = _parse_video_variant_index(variant)
|
||||
if variant_index is not None and 0 <= variant_index < len(urls):
|
||||
return urls[variant_index]
|
||||
return None
|
||||
if _parse_video_variant_index(variant) != 0:
|
||||
return None
|
||||
return job.get("url")
|
||||
|
||||
|
||||
def _select_video_variant_path(job: dict, variant: str | None) -> str | None:
|
||||
file_paths = job.get("file_paths")
|
||||
if file_paths:
|
||||
try:
|
||||
variant_index = 0 if variant is None else int(variant)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if 0 <= variant_index < len(file_paths):
|
||||
variant_index = _parse_video_variant_index(variant)
|
||||
if variant_index is not None and 0 <= variant_index < len(file_paths):
|
||||
return file_paths[variant_index]
|
||||
return None
|
||||
if variant not in (None, "0", 0):
|
||||
if _parse_video_variant_index(variant) != 0:
|
||||
return None
|
||||
return job.get("file_path")
|
||||
|
||||
@@ -852,10 +889,11 @@ async def download_video_content(
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Video not found")
|
||||
|
||||
if job.get("url"):
|
||||
cloud_url = _select_video_variant_url(job, variant)
|
||||
if cloud_url:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Video has been uploaded to cloud storage. Please use the cloud URL: {job.get('url')}",
|
||||
detail=f"Video has been uploaded to cloud storage. Please use the cloud URL: {cloud_url}",
|
||||
)
|
||||
|
||||
file_path = _select_video_variant_path(job, variant)
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import asyncio
|
||||
import os
|
||||
|
||||
from sglang.multimodal_gen.runtime.entrypoints.openai import video_api
|
||||
|
||||
@@ -45,3 +46,105 @@ def test_shutdown_video_jobs_cancels_and_awaits_cleanup():
|
||||
assert "job-id" not in video_api._VIDEO_JOB_TASKS
|
||||
|
||||
asyncio.run(run_test())
|
||||
|
||||
|
||||
class _FakeSamplingParams:
|
||||
def validate_video_final_outputs(self, paths, batch):
|
||||
return {}
|
||||
|
||||
def cleanup_video_request(self, batch):
|
||||
pass
|
||||
|
||||
|
||||
class _FakeBatch:
|
||||
sampling_params = _FakeSamplingParams()
|
||||
|
||||
|
||||
def _run_dispatch(monkeypatch, tmp_path, *, output_persistent, upload):
|
||||
output_dir = tmp_path / "out"
|
||||
output_dir.mkdir()
|
||||
paths = []
|
||||
for i in range(2):
|
||||
path = output_dir / f"video_{i}.mp4"
|
||||
path.write_bytes(b"video")
|
||||
paths.append(str(path))
|
||||
|
||||
async def process_generation_batch(client, batch, scheduler_batches=None):
|
||||
return paths, object()
|
||||
|
||||
async def upload_and_cleanup(path):
|
||||
url = upload(path)
|
||||
if url:
|
||||
os.remove(path)
|
||||
return url
|
||||
|
||||
monkeypatch.setattr(video_api, "process_generation_batch", process_generation_batch)
|
||||
monkeypatch.setattr(
|
||||
video_api.cloud_storage, "upload_and_cleanup", upload_and_cleanup
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
video_api,
|
||||
"add_common_data_to_response",
|
||||
lambda response, request_id, result: response,
|
||||
)
|
||||
|
||||
async def run_test():
|
||||
await video_api.VIDEO_STORE.upsert("job-id", {"id": "job-id"})
|
||||
await video_api._dispatch_job_async(
|
||||
"job-id",
|
||||
_FakeBatch(),
|
||||
temp_dirs=[] if output_persistent else [str(output_dir)],
|
||||
output_persistent=output_persistent,
|
||||
)
|
||||
return await video_api.VIDEO_STORE.pop("job-id")
|
||||
|
||||
return asyncio.run(run_test()), paths
|
||||
|
||||
|
||||
def test_temp_output_without_cloud_upload_fails_job(monkeypatch, tmp_path):
|
||||
"""A job whose temp outputs are deleted unpublished must not report completed."""
|
||||
job, _ = _run_dispatch(
|
||||
monkeypatch, tmp_path, output_persistent=False, upload=lambda path: None
|
||||
)
|
||||
assert job["status"] == "failed"
|
||||
assert job["url"] is None and job["file_paths"] is None
|
||||
|
||||
|
||||
def test_temp_output_with_partial_cloud_upload_fails_job(monkeypatch, tmp_path):
|
||||
job, _ = _run_dispatch(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
output_persistent=False,
|
||||
upload=lambda path: "s3://a" if path.endswith("_0.mp4") else None,
|
||||
)
|
||||
assert job["status"] == "failed"
|
||||
|
||||
|
||||
def test_temp_output_uploads_every_variant(monkeypatch, tmp_path):
|
||||
job, paths = _run_dispatch(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
output_persistent=False,
|
||||
upload=lambda path: f"s3://{os.path.basename(path)}",
|
||||
)
|
||||
assert job["status"] == "completed"
|
||||
assert job["url"] == "s3://video_0.mp4"
|
||||
assert job["urls"] == ["s3://video_0.mp4", "s3://video_1.mp4"]
|
||||
assert job["file_path"] is None and job["file_paths"] is None
|
||||
assert video_api._select_video_variant_url(job, "1") == "s3://video_1.mp4"
|
||||
|
||||
|
||||
def test_persistent_output_reports_only_local_files(monkeypatch, tmp_path):
|
||||
job, paths = _run_dispatch(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
output_persistent=True,
|
||||
upload=lambda path: "s3://a" if path.endswith("_0.mp4") else None,
|
||||
)
|
||||
assert job["status"] == "completed"
|
||||
assert job["urls"] == ["s3://a", None]
|
||||
assert job["file_path"] is None
|
||||
assert job["file_paths"] == [None, paths[1]]
|
||||
assert os.path.exists(paths[1])
|
||||
assert video_api._select_video_variant_url(job, "1") is None
|
||||
assert video_api._select_video_variant_path(job, "1") == paths[1]
|
||||
|
||||
Reference in New Issue
Block a user