[diffusion] Don't report video jobs completed after their outputs are deleted (#43084)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Xiaoyu Zhang <1182563586@qq.com>
This commit is contained in:
Avaya Aggarwal
2026-10-11 12:42:38 +08:00
committed by GitHub
co-authored by Claude Opus 5.5 Xiaoyu Zhang
parent aa5c969dc3
commit a1500b10e3
3 changed files with 158 additions and 16 deletions
@@ -103,12 +103,13 @@ class VideoResponse(BaseModel):
seconds: str = "4"
quality: str = "standard"
url: Optional[str] = None
urls: Optional[List[Optional[str]]] = None
remixed_from_video_id: Optional[str] = None
completed_at: Optional[int] = None
expires_at: Optional[int] = None
error: Optional[Dict[str, Any]] = None
file_path: Optional[str] = None
file_paths: Optional[List[str]] = None
file_paths: Optional[List[Optional[str]]] = None
num_outputs: Optional[int] = None
peak_memory_mb: Optional[float] = None
inference_time_s: Optional[float] = None
@@ -360,6 +360,14 @@ async def _save_first_input_image(
)
async def _upload_and_cleanup_videos(paths: list[str]) -> list[str | None]:
return list(
await asyncio.gather(
*(cloud_storage.upload_and_cleanup(path) for path in paths)
)
)
async def _dispatch_job_async(
job_id: str,
batch: Req,
@@ -397,19 +405,31 @@ async def _dispatch_job_async(
)
raise
cloud_url = await cloud_storage.upload_and_cleanup(save_file_path)
cloud_urls = await _upload_and_cleanup_videos(save_file_path_list)
if not output_persistent and not all(cloud_urls):
# The temp output dir is removed in `finally`, so an output that was
# not uploaded would be unretrievable once the job reports completed.
raise RuntimeError(
"Video output could not be published: no output_path is configured "
"and cloud storage upload is disabled or failed."
)
persistent_path = (
save_file_path if not cloud_url and output_persistent else None
)
update_fields = {
"status": "completed",
"progress": 100,
"completed_at": int(time.time()),
"url": cloud_url,
"file_path": persistent_path,
"url": cloud_urls[0],
"urls": cloud_urls,
"file_path": (
os.path.abspath(save_file_path)
if not cloud_urls[0] and output_persistent
else None
),
"file_paths": (
[os.path.abspath(path) for path in save_file_path_list]
[
None if url else os.path.abspath(path)
for path, url in zip(save_file_path_list, cloud_urls)
]
if output_persistent
else None
),
@@ -428,6 +448,7 @@ async def _dispatch_job_async(
"status": "failed",
"error": {"message": str(e)},
"url": None,
"urls": None,
"file_path": None,
"file_paths": None,
"num_outputs": None,
@@ -829,17 +850,33 @@ async def delete_video(video_id: str = Path(...)):
return VideoResponse(**job)
def _parse_video_variant_index(variant: str | None) -> int | None:
try:
return 0 if variant is None else int(variant)
except (TypeError, ValueError):
return None
def _select_video_variant_url(job: dict, variant: str | None) -> str | None:
urls = job.get("urls")
if urls:
variant_index = _parse_video_variant_index(variant)
if variant_index is not None and 0 <= variant_index < len(urls):
return urls[variant_index]
return None
if _parse_video_variant_index(variant) != 0:
return None
return job.get("url")
def _select_video_variant_path(job: dict, variant: str | None) -> str | None:
file_paths = job.get("file_paths")
if file_paths:
try:
variant_index = 0 if variant is None else int(variant)
except (TypeError, ValueError):
return None
if 0 <= variant_index < len(file_paths):
variant_index = _parse_video_variant_index(variant)
if variant_index is not None and 0 <= variant_index < len(file_paths):
return file_paths[variant_index]
return None
if variant not in (None, "0", 0):
if _parse_video_variant_index(variant) != 0:
return None
return job.get("file_path")
@@ -852,10 +889,11 @@ async def download_video_content(
if not job:
raise HTTPException(status_code=404, detail="Video not found")
if job.get("url"):
cloud_url = _select_video_variant_url(job, variant)
if cloud_url:
raise HTTPException(
status_code=400,
detail=f"Video has been uploaded to cloud storage. Please use the cloud URL: {job.get('url')}",
detail=f"Video has been uploaded to cloud storage. Please use the cloud URL: {cloud_url}",
)
file_path = _select_video_variant_path(job, variant)
@@ -1,4 +1,5 @@
import asyncio
import os
from sglang.multimodal_gen.runtime.entrypoints.openai import video_api
@@ -45,3 +46,105 @@ def test_shutdown_video_jobs_cancels_and_awaits_cleanup():
assert "job-id" not in video_api._VIDEO_JOB_TASKS
asyncio.run(run_test())
class _FakeSamplingParams:
def validate_video_final_outputs(self, paths, batch):
return {}
def cleanup_video_request(self, batch):
pass
class _FakeBatch:
sampling_params = _FakeSamplingParams()
def _run_dispatch(monkeypatch, tmp_path, *, output_persistent, upload):
output_dir = tmp_path / "out"
output_dir.mkdir()
paths = []
for i in range(2):
path = output_dir / f"video_{i}.mp4"
path.write_bytes(b"video")
paths.append(str(path))
async def process_generation_batch(client, batch, scheduler_batches=None):
return paths, object()
async def upload_and_cleanup(path):
url = upload(path)
if url:
os.remove(path)
return url
monkeypatch.setattr(video_api, "process_generation_batch", process_generation_batch)
monkeypatch.setattr(
video_api.cloud_storage, "upload_and_cleanup", upload_and_cleanup
)
monkeypatch.setattr(
video_api,
"add_common_data_to_response",
lambda response, request_id, result: response,
)
async def run_test():
await video_api.VIDEO_STORE.upsert("job-id", {"id": "job-id"})
await video_api._dispatch_job_async(
"job-id",
_FakeBatch(),
temp_dirs=[] if output_persistent else [str(output_dir)],
output_persistent=output_persistent,
)
return await video_api.VIDEO_STORE.pop("job-id")
return asyncio.run(run_test()), paths
def test_temp_output_without_cloud_upload_fails_job(monkeypatch, tmp_path):
"""A job whose temp outputs are deleted unpublished must not report completed."""
job, _ = _run_dispatch(
monkeypatch, tmp_path, output_persistent=False, upload=lambda path: None
)
assert job["status"] == "failed"
assert job["url"] is None and job["file_paths"] is None
def test_temp_output_with_partial_cloud_upload_fails_job(monkeypatch, tmp_path):
job, _ = _run_dispatch(
monkeypatch,
tmp_path,
output_persistent=False,
upload=lambda path: "s3://a" if path.endswith("_0.mp4") else None,
)
assert job["status"] == "failed"
def test_temp_output_uploads_every_variant(monkeypatch, tmp_path):
job, paths = _run_dispatch(
monkeypatch,
tmp_path,
output_persistent=False,
upload=lambda path: f"s3://{os.path.basename(path)}",
)
assert job["status"] == "completed"
assert job["url"] == "s3://video_0.mp4"
assert job["urls"] == ["s3://video_0.mp4", "s3://video_1.mp4"]
assert job["file_path"] is None and job["file_paths"] is None
assert video_api._select_video_variant_url(job, "1") == "s3://video_1.mp4"
def test_persistent_output_reports_only_local_files(monkeypatch, tmp_path):
job, paths = _run_dispatch(
monkeypatch,
tmp_path,
output_persistent=True,
upload=lambda path: "s3://a" if path.endswith("_0.mp4") else None,
)
assert job["status"] == "completed"
assert job["urls"] == ["s3://a", None]
assert job["file_path"] is None
assert job["file_paths"] == [None, paths[1]]
assert os.path.exists(paths[1])
assert video_api._select_video_variant_url(job, "1") is None
assert video_api._select_video_variant_path(job, "1") == paths[1]