Co-authored-by: Mick Qian <mickqian@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
467 lines
16 KiB
Python
467 lines
16 KiB
Python
# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo
|
|
|
|
import asyncio
|
|
import base64
|
|
import os
|
|
import signal
|
|
import uuid
|
|
from contextlib import asynccontextmanager, suppress
|
|
from typing import TYPE_CHECKING
|
|
|
|
import httpx
|
|
import torch
|
|
from fastapi import APIRouter, FastAPI, Request, Response
|
|
from fastapi.middleware.cors import CORSMiddleware
|
|
|
|
from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams
|
|
from sglang.multimodal_gen.runtime.entrypoints.action import api as action_api
|
|
from sglang.multimodal_gen.runtime.entrypoints.action import openpi
|
|
from sglang.multimodal_gen.runtime.entrypoints.openai import image_api, video_api
|
|
from sglang.multimodal_gen.runtime.entrypoints.openai.prompt_enhancement import (
|
|
PromptEnhancer,
|
|
)
|
|
from sglang.multimodal_gen.runtime.entrypoints.openai.protocol import (
|
|
VertexGenerateReqInput,
|
|
)
|
|
from sglang.multimodal_gen.runtime.entrypoints.openai.realtime import (
|
|
realtime_video_api,
|
|
)
|
|
from sglang.multimodal_gen.runtime.entrypoints.openai.utils import build_sampling_params
|
|
from sglang.multimodal_gen.runtime.entrypoints.post_training import (
|
|
rollout_api,
|
|
weights_api,
|
|
)
|
|
from sglang.multimodal_gen.runtime.entrypoints.utils import (
|
|
prepare_request,
|
|
save_outputs,
|
|
)
|
|
from sglang.multimodal_gen.runtime.observability.metrics import configure_metrics
|
|
from sglang.multimodal_gen.runtime.scheduler_client import async_scheduler_client
|
|
from sglang.multimodal_gen.runtime.server_args import ServerArgs, get_global_server_args
|
|
from sglang.multimodal_gen.runtime.server_warmup import (
|
|
run_async_client_warmup,
|
|
should_run_synthetic_server_warmup,
|
|
)
|
|
from sglang.multimodal_gen.runtime.utils.logging_utils import (
|
|
globally_suppress_loggers,
|
|
init_logger,
|
|
)
|
|
from sglang.srt.utils.common import (
|
|
add_prometheus_middleware,
|
|
add_prometheus_track_response_middleware,
|
|
)
|
|
from sglang.srt.utils.json_response import orjson_response
|
|
from sglang.version import __version__
|
|
|
|
if TYPE_CHECKING:
|
|
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
|
|
|
|
logger = init_logger(__name__)
|
|
|
|
VERTEX_ROUTE = os.environ.get("AIP_PREDICT_ROUTE", "/vertex_generate")
|
|
SERVER_WARMUP_BYPASS_PATHS = (
|
|
"/metrics",
|
|
"/liveness",
|
|
"/health",
|
|
"/health_generate",
|
|
"/model_info",
|
|
"/server_info",
|
|
)
|
|
|
|
|
|
async def _wait_until_http_live(server_args: ServerArgs) -> None:
|
|
"""for server warmup"""
|
|
liveness_url = f"{server_args.url()}/liveness"
|
|
# Probe the local server directly: a loopback liveness check must never be
|
|
# routed through an HTTP proxy. trust_env=False also avoids crashing startup
|
|
# on a malformed proxy env var, since httpx parses *_PROXY/NO_PROXY when the
|
|
# client is constructed (raising httpx.InvalidURL before any request). See #28493.
|
|
async with httpx.AsyncClient(trust_env=False) as client:
|
|
for _ in range(120):
|
|
try:
|
|
response = await client.get(liveness_url, timeout=5.0)
|
|
if response.status_code == 200:
|
|
return
|
|
except httpx.HTTPError:
|
|
pass
|
|
await asyncio.sleep(1.0)
|
|
raise RuntimeError(f"HTTP server did not become live at {liveness_url}")
|
|
|
|
|
|
async def _run_server_warmup_after_http_live(
|
|
server_args: ServerArgs, warmup_done: asyncio.Event
|
|
) -> None:
|
|
try:
|
|
if not should_run_synthetic_server_warmup(server_args):
|
|
warmup_done.set()
|
|
return
|
|
|
|
await _wait_until_http_live(server_args)
|
|
|
|
await run_async_client_warmup(
|
|
server_args,
|
|
async_scheduler_client.forward,
|
|
fail_open=server_args.warmup_resolutions is None,
|
|
)
|
|
logger.info("The server is fired up and ready to roll!")
|
|
warmup_done.set()
|
|
except asyncio.CancelledError:
|
|
raise
|
|
except Exception as e:
|
|
logger.error("Server warmup failed; aborting startup: %s", e, exc_info=True)
|
|
os.kill(os.getpid(), signal.SIGTERM)
|
|
|
|
|
|
@asynccontextmanager
|
|
async def lifespan(app: FastAPI):
|
|
from sglang.multimodal_gen.runtime.entrypoints.openai.video_api import (
|
|
shutdown_video_jobs,
|
|
)
|
|
from sglang.multimodal_gen.runtime.scheduler_client import (
|
|
async_scheduler_client,
|
|
run_zeromq_broker,
|
|
)
|
|
|
|
# 1. Initialize the singleton client that connects to the backend Scheduler
|
|
server_args = app.state.server_args
|
|
async_scheduler_client.initialize(server_args)
|
|
warmup_done = asyncio.Event()
|
|
app.state.server_warmup_done = warmup_done
|
|
|
|
# 2. Start the ZMQ Broker in the background to handle offline requests
|
|
broker_task = asyncio.create_task(run_zeromq_broker(server_args))
|
|
warmup_task = None
|
|
if server_args.warmup_mode == "server":
|
|
warmup_task = asyncio.create_task(
|
|
_run_server_warmup_after_http_live(server_args, warmup_done)
|
|
)
|
|
else:
|
|
warmup_done.set()
|
|
|
|
try:
|
|
app.state.prompt_enhancer = (
|
|
PromptEnhancer.from_file(server_args.prompt_enhancer_config)
|
|
if server_args.prompt_enhancer_config is not None
|
|
else None
|
|
)
|
|
yield
|
|
finally:
|
|
if warmup_task is not None and not warmup_task.done():
|
|
warmup_task.cancel()
|
|
with suppress(asyncio.CancelledError):
|
|
await warmup_task
|
|
|
|
# On shutdown
|
|
logger.info("FastAPI app is shutting down...")
|
|
await shutdown_video_jobs()
|
|
if app.state.prompt_enhancer is not None:
|
|
await app.state.prompt_enhancer.close()
|
|
broker_task.cancel()
|
|
with suppress(asyncio.CancelledError):
|
|
await broker_task
|
|
async_scheduler_client.close()
|
|
|
|
|
|
# Health router
|
|
health_router = APIRouter()
|
|
|
|
|
|
@health_router.get("/liveness")
|
|
async def liveness():
|
|
"""Report that the HTTP server is accepting requests."""
|
|
return {"status": "ok"}
|
|
|
|
|
|
@health_router.get("/health")
|
|
async def health(request: Request):
|
|
"""Report readiness for normal inference traffic."""
|
|
if not request.app.state.server_warmup_done.is_set():
|
|
return Response(status_code=503)
|
|
return {"status": "ok"}
|
|
|
|
|
|
@health_router.get("/models", deprecated=True)
|
|
async def get_models(request: Request):
|
|
"""
|
|
Get information about the model served by this server.
|
|
|
|
.. deprecated::
|
|
Use /v1/models instead for OpenAI-compatible model discovery.
|
|
This endpoint will be removed in a future version.
|
|
"""
|
|
from sglang.multimodal_gen.runtime.entrypoints.openai.common_api import (
|
|
get_served_pipeline_class,
|
|
)
|
|
|
|
server_args: ServerArgs = request.app.state.server_args
|
|
pipeline_cls = get_served_pipeline_class(server_args)
|
|
|
|
response = {
|
|
"model_path": server_args.model_path,
|
|
"num_gpus": server_args.num_gpus,
|
|
"task_type": server_args.pipeline_config.task_type.name,
|
|
"supported_task_types": [
|
|
task.name for task in server_args.pipeline_config.get_supported_task_types()
|
|
],
|
|
"dit_precision": server_args.pipeline_config.dit_precision,
|
|
"vae_precision": server_args.pipeline_config.vae_precision,
|
|
"vae_decode_precision": server_args.pipeline_config.vae_decode_precision,
|
|
}
|
|
|
|
if pipeline_cls:
|
|
response["pipeline_name"] = pipeline_cls.pipeline_name
|
|
response["pipeline_class"] = pipeline_cls.__name__
|
|
|
|
return response
|
|
|
|
|
|
@health_router.get("/server_info")
|
|
async def server_info_endpoint(request: Request):
|
|
"""Get server information.
|
|
|
|
Returns fields compatible with the LLM engine's /server_info so that
|
|
the model gateway can discover diffusion workers.
|
|
"""
|
|
server_args: ServerArgs = request.app.state.server_args
|
|
|
|
return {
|
|
"model_path": server_args.model_path,
|
|
"served_model_name": server_args.served_model_name,
|
|
"tp_size": server_args.tp_size,
|
|
"dp_size": server_args.dp_size,
|
|
"version": __version__,
|
|
}
|
|
|
|
|
|
@health_router.get("/model_info")
|
|
async def model_info_endpoint(request: Request):
|
|
"""Get model information.
|
|
|
|
Returns fields compatible with the LLM engine's /model_info so that
|
|
the model gateway can detect capabilities for diffusion workers.
|
|
"""
|
|
from sglang.multimodal_gen.runtime.entrypoints.openai.common_api import (
|
|
get_served_pipeline_class,
|
|
)
|
|
|
|
server_args: ServerArgs = request.app.state.server_args
|
|
task_type = server_args.pipeline_config.task_type
|
|
supported_tasks = server_args.pipeline_config.get_supported_task_types()
|
|
|
|
try:
|
|
pipeline_cls = get_served_pipeline_class(server_args)
|
|
except Exception:
|
|
logger.warning("Failed to resolve model info from registry", exc_info=True)
|
|
pipeline_cls = None
|
|
|
|
return {
|
|
# Fields consumed by the model gateway for worker discovery
|
|
"model_path": server_args.model_path,
|
|
"is_generation": True,
|
|
"model_type": "diffusion",
|
|
"architectures": [pipeline_cls.__name__] if pipeline_cls else None,
|
|
# Fields matching the LLM engine's /model_info shape
|
|
"has_image_understanding": any(
|
|
task.accepts_image_input() for task in supported_tasks
|
|
),
|
|
"has_audio_understanding": False,
|
|
# Diffusion-specific fields
|
|
"task_type": task_type.name,
|
|
"supported_task_types": [task.name for task in supported_tasks],
|
|
"output_types": sorted({task.data_type().name for task in supported_tasks}),
|
|
"is_image_gen": task_type.is_image_gen(),
|
|
}
|
|
|
|
|
|
@health_router.get("/health_generate")
|
|
async def health_generate(request: Request):
|
|
"""Compatibility readiness endpoint; no generation is issued."""
|
|
return await health(request)
|
|
|
|
|
|
@health_router.get("/stats")
|
|
async def stats_endpoint(request: Request):
|
|
"""Get runtime statistics including disagg pipeline metrics.
|
|
|
|
Returns queue depth, request counts, latency, throughput, etc.
|
|
Sends a GetDisaggStatsReq to the scheduler via ZMQ and returns the result.
|
|
"""
|
|
from sglang.multimodal_gen.runtime.entrypoints.control_requests import (
|
|
GetDisaggStatsReq,
|
|
)
|
|
|
|
server_args: ServerArgs = request.app.state.server_args
|
|
response: dict = {
|
|
"status": "ok",
|
|
"model_path": server_args.model_path,
|
|
}
|
|
|
|
# Query the scheduler for disagg metrics
|
|
try:
|
|
stats_response = await async_scheduler_client.forward(GetDisaggStatsReq())
|
|
if hasattr(stats_response, "output") and stats_response.output is not None:
|
|
response["disagg"] = stats_response.output
|
|
except Exception as e:
|
|
response["disagg"] = {"error": str(e)}
|
|
|
|
return response
|
|
|
|
|
|
def make_serializable(obj):
|
|
"""Recursively converts Tensors to None for JSON serialization."""
|
|
if isinstance(obj, torch.Tensor):
|
|
return None
|
|
if isinstance(obj, dict):
|
|
return {k: make_serializable(v) for k, v in obj.items()}
|
|
if isinstance(obj, list):
|
|
return [make_serializable(v) for v in obj]
|
|
return obj
|
|
|
|
|
|
def encode_video_to_base64(file_path: str):
|
|
if not os.path.exists(file_path):
|
|
return None
|
|
with open(file_path, "rb") as f:
|
|
return base64.b64encode(f.read()).decode("utf-8")
|
|
|
|
|
|
async def forward_to_scheduler(
|
|
req_obj: "Req",
|
|
sp: SamplingParams,
|
|
):
|
|
"""Forwards request to scheduler and processes the result."""
|
|
try:
|
|
response = await async_scheduler_client.forward(req_obj)
|
|
if response.output is None and response.output_file_paths is None:
|
|
raise RuntimeError("Model generation returned no output.")
|
|
|
|
if response.output_file_paths:
|
|
output_file_path = response.output_file_paths[0]
|
|
else:
|
|
output_file_path = sp.output_file_path()
|
|
save_outputs(
|
|
[response.output[0]],
|
|
sp.data_type,
|
|
sp.fps,
|
|
True,
|
|
lambda _idx: output_file_path,
|
|
audio=response.audio,
|
|
audio_sample_rate=response.audio_sample_rate,
|
|
output_compression=sp.output_compression,
|
|
x264_preset=sp.x264_preset,
|
|
enable_frame_interpolation=sp.enable_frame_interpolation,
|
|
frame_interpolation_exp=sp.frame_interpolation_exp,
|
|
frame_interpolation_scale=sp.frame_interpolation_scale,
|
|
frame_interpolation_model_path=sp.frame_interpolation_model_path,
|
|
enable_upscaling=sp.enable_upscaling,
|
|
upscaling_model_path=sp.upscaling_model_path,
|
|
upscaling_scale=sp.upscaling_scale,
|
|
)
|
|
|
|
if hasattr(response, "model_dump"):
|
|
data = response.model_dump()
|
|
else:
|
|
data = response if isinstance(response, dict) else vars(response)
|
|
|
|
if output_file_path:
|
|
logger.info("Processing output file: %s", output_file_path)
|
|
b64_video = encode_video_to_base64(output_file_path)
|
|
|
|
if b64_video:
|
|
data["output"] = b64_video
|
|
data.pop("video_data", None)
|
|
data.pop("video_tensor", None)
|
|
|
|
return make_serializable(data)
|
|
|
|
except Exception as e:
|
|
logger.error("Error during generation: %s", e, exc_info=True)
|
|
return {"error": str(e)}
|
|
|
|
|
|
vertex_router = APIRouter()
|
|
|
|
|
|
@vertex_router.post(VERTEX_ROUTE)
|
|
async def vertex_generate(vertex_req: VertexGenerateReqInput):
|
|
if not vertex_req.instances:
|
|
return orjson_response({"predictions": []})
|
|
|
|
server_args = get_global_server_args()
|
|
params = vertex_req.parameters or {}
|
|
|
|
futures = []
|
|
|
|
for inst in vertex_req.instances:
|
|
rid = f"vertex_{uuid.uuid4()}"
|
|
|
|
sp = build_sampling_params(
|
|
rid,
|
|
prompt=inst.get("prompt") or inst.get("text"),
|
|
image_path=inst.get("image") or inst.get("image_url"),
|
|
num_frames=params.get("num_frames"),
|
|
fps=params.get("fps"),
|
|
width=params.get("width"),
|
|
height=params.get("height"),
|
|
guidance_scale=params.get("guidance_scale"),
|
|
save_output=params.get("save_output"),
|
|
)
|
|
|
|
backend_req = prepare_request(server_args, sampling_params=sp)
|
|
futures.append(forward_to_scheduler(backend_req, sp))
|
|
|
|
results = await asyncio.gather(*futures)
|
|
|
|
return orjson_response({"predictions": results})
|
|
|
|
|
|
def create_app(server_args: ServerArgs):
|
|
"""
|
|
Create and configure the FastAPI application instance.
|
|
"""
|
|
globally_suppress_loggers()
|
|
app = FastAPI(lifespan=lifespan)
|
|
if server_args.enable_metrics:
|
|
configure_metrics()
|
|
add_prometheus_middleware(app)
|
|
add_prometheus_track_response_middleware(app)
|
|
app.add_middleware(
|
|
CORSMiddleware,
|
|
allow_origins=["*"],
|
|
allow_credentials=True,
|
|
allow_methods=["*"],
|
|
allow_headers=["*"],
|
|
)
|
|
|
|
@app.middleware("http")
|
|
async def wait_for_server_warmup(request: Request, call_next):
|
|
warmup_done = getattr(request.app.state, "server_warmup_done", None)
|
|
if (
|
|
warmup_done is not None
|
|
and not warmup_done.is_set()
|
|
and request.url.path not in SERVER_WARMUP_BYPASS_PATHS
|
|
):
|
|
await warmup_done.wait()
|
|
return await call_next(request)
|
|
|
|
app.include_router(health_router)
|
|
app.include_router(vertex_router)
|
|
|
|
from sglang.multimodal_gen.runtime.entrypoints.openai import common_api, mesh_api
|
|
|
|
app.include_router(common_api.router)
|
|
app.include_router(image_api.router)
|
|
app.include_router(video_api.router)
|
|
app.include_router(realtime_video_api.router)
|
|
if server_args.pipeline_config.supports_action_endpoint():
|
|
app.include_router(action_api.router)
|
|
if server_args.pipeline_config.supports_openpi_endpoint():
|
|
app.include_router(openpi.router)
|
|
app.include_router(mesh_api.router)
|
|
app.include_router(weights_api.router)
|
|
app.include_router(rollout_api.router)
|
|
|
|
app.state.server_args = server_args
|
|
app.state.prompt_enhancer = None
|
|
return app
|