diff --git a/python/pyproject.toml b/python/pyproject.toml index 0f5a6de6e..9be8cb833 100755 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools>=61.0", "setuptools-scm>=8.0", "setuptools-rust>=1.10", "wheel"] +requires = ["setuptools>=61.0", "setuptools-rust>=1.10", "setuptools-scm>=8.0", "wheel"] build-backend = "setuptools.build_meta" [project] @@ -14,35 +14,40 @@ classifiers = [ "License :: OSI Approved :: Apache Software License", ] +# Please keep dependency lists in this file sorted alphabetically by package name. dependencies = [ - "IPython", "aiohttp", - "apache-tvm-ffi==0.1.9", "anthropic>=0.20.0", + "apache-tvm-ffi==0.1.9", + "av ; sys_platform == 'linux' and (platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'armv7l')", "blobfile==3.0.0", "build", "compressed-tensors", "cuda-python>=13.0", - "decord2 ; sys_platform == 'linux' and (platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'armv7l')", "datasets", + "decord2 ; sys_platform == 'linux' and (platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'armv7l')", "distro", + "easydict", # Required by remote model code (e.g. DeepSeek-OCR) loaded via trust_remote_code; validated by transformers 5.4+ check_imports "einops", "fastapi", - "flashinfer_python[cu13]==0.6.12", # keep it aligned with jit-cache version in Dockerfile + "flash-attn-4>=4.0.0b9", "flashinfer_cubin==0.6.12", + "flashinfer_python[cu13]==0.6.12", # keep it aligned with jit-cache version in Dockerfile "gguf", "interegular", + "IPython", + "kernels>=0.14.1,<0.15", "llguidance>=0.7.11,<0.8.0", + "mistral_common>=1.11.0", "modelscope", "msgspec", "ninja", - "easydict", # Required by remote model code (e.g. DeepSeek-OCR) loaded via trust_remote_code; validated by transformers 5.4+ check_imports "numpy", "nvidia-cutlass-dsl[cu13]==4.5.2", - "nvidia-ml-py", "nvidia-mathdx==25.6.0", - "openai-harmony==0.0.4", + "nvidia-ml-py", "openai==2.6.1", + "openai-harmony==0.0.4", "orjson", "outlines==0.1.11", "packaging", @@ -60,30 +65,26 @@ dependencies = [ "scipy", "sentencepiece", "setproctitle", - "flash-attn-4>=4.0.0b9", "sgl-deep-gemm==0.1.3", "sglang-kernel==0.4.4", + "smg-grpc-servicer>=0.5.0", "soundfile==0.13.1", "tiktoken", "tilelang==0.1.8", "timm==1.0.16", "tokenspeed_mla==0.1.6", - "torch_memory_saver>=0.0.9.post1", "torch==2.11.0", + "torch_memory_saver>=0.0.9.post1", "torchao==0.17.0", "torchaudio==2.11.0", "torchcodec==0.11.1 ; sys_platform != 'linux' or (sys_platform == 'linux' and platform_machine != 'aarch64' and platform_machine != 'arm64' and platform_machine != 'armv7l')", # torchcodec 0.11.1 for torch 2.11.x (0.10 is ABI-incompatible: references the pre-2.11 c10::MessageLogger ctor signature). Not available on Linux ARM. - "av ; sys_platform == 'linux' and (platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'armv7l')", "torchvision", "tqdm", - "mistral_common>=1.11.0", "transformers==5.8.1", "uvicorn", "uvloop", "watchfiles", "xgrammar==0.2.1", - "smg-grpc-servicer>=0.5.0", - "kernels>=0.14.1,<0.15", ] [[tool.uv.index]] @@ -95,7 +96,9 @@ default = true checkpoint-engine = ["checkpoint-engine==0.1.2"] runai = ["runai-model-streamer[s3,gcs,azure]>=0.15.7"] diffusion = [ - "PyYAML==6.0.1", + "addict==2.4.0", + "av==16.1.0", + "cache-dit==1.3.0", "cloudpickle==3.1.2", "diffusers==0.37.0", "imageio==2.36.0", @@ -103,15 +106,13 @@ diffusion = [ "moviepy>=2.0.0", "nvidia-modelopt", "opencv-python-headless==4.10.0.84", + "PyYAML==6.0.1", "remote-pdb==2.1.0", - "st_attn==0.0.7 ; platform_machine != 'aarch64' and platform_machine != 'arm64'", - "vsa==0.0.4 ; platform_machine != 'aarch64' and platform_machine != 'arm64'", "runai_model_streamer>=0.15.7", - "cache-dit==1.3.0", - "addict==2.4.0", - "av==16.1.0", "scikit-image==0.25.2", + "st_attn==0.0.7 ; platform_machine != 'aarch64' and platform_machine != 'arm64'", "trimesh>=4.0.0", + "vsa==0.0.4 ; platform_machine != 'aarch64' and platform_machine != 'arm64'", "websockets", "xatlas", ] @@ -139,7 +140,9 @@ test = [ "accelerate", "addict", "bitsandbytes", + "diff-cover", "expecttest", + "granian>=2.6.0", "jsonlines", "lm-eval[api]>=0.4.9.2", "matplotlib", @@ -149,24 +152,22 @@ test = [ "polars", "pytest", "pytest-cov", - "diff-cover", "sentence_transformers", "sglang[fastokens]", "tabulate", - "granian>=2.6.0", ] dev = ["sglang[test]"] all = [ "sglang[diffusion]", - "sglang[tracing]", "sglang[http2]", + "sglang[tracing]", ] [tool.uv.extra-build-dependencies] -st-attn = ["torch", "setuptools"] -vsa = ["torch", "setuptools"] +st-attn = ["setuptools", "torch"] +vsa = ["setuptools", "torch"] [project.urls] "Homepage" = "https://github.com/sgl-project/sglang" diff --git a/python/pyproject_cpu.toml b/python/pyproject_cpu.toml index 58f03cb0a..f40064c55 100644 --- a/python/pyproject_cpu.toml +++ b/python/pyproject_cpu.toml @@ -15,27 +15,29 @@ classifiers = [ "License :: OSI Approved :: Apache Software License", ] +# Please keep dependency lists in this file sorted alphabetically by package name. dependencies = [ - "IPython", "aiohttp", "anthropic>=0.20.0", "blobfile==3.0.0", "build", "compressed-tensors", "datasets", + "easydict", "einops", "fastapi", "gguf", "intel-openmp; platform_machine == 'x86_64'", "interegular", + "IPython", "llguidance>=0.7.11,<0.8.0", + "mistral_common>=1.11.0", "modelscope", "msgspec", - "easydict", "ninja", "numpy", - "openai-harmony==0.0.4", "openai==2.6.1", + "openai-harmony==0.0.4", "orjson", "outlines", "packaging", @@ -52,6 +54,7 @@ dependencies = [ "scipy", "sentencepiece", "setproctitle", + "smg-grpc-servicer>=0.5.0", "soundfile==0.13.1", "tabulate", "tiktoken", @@ -61,41 +64,39 @@ dependencies = [ "torchaudio==2.11.0", "torchvision==0.27.0", "tqdm", - "mistral_common>=1.11.0", "transformers==5.8.1", "triton==3.7.0", "uvicorn", "uvloop", "xgrammar==0.2.1", - "smg-grpc-servicer>=0.5.0", ] [project.optional-dependencies] diffusion = [ - "PyYAML==6.0.1", + "addict==2.4.0", + "av==16.1.0", + "cache-dit==1.3.0", "cloudpickle==3.1.2", "diffusers==0.37.0", "imageio==2.36.0", "imageio-ffmpeg==0.5.1", "moviepy>=2.0.0", "opencv-python-headless==4.10.0.84", + "PyYAML==6.0.1", "remote-pdb==2.1.0", - "st_attn==0.0.7 ; platform_machine != 'aarch64' and platform_machine != 'arm64'", - "vsa==0.0.4 ; platform_machine != 'aarch64' and platform_machine != 'arm64'", "runai_model_streamer>=0.15.5", - "cache-dit==1.3.0", - "addict==2.4.0", - "av==16.1.0", "scikit-image==0.25.2", + "st_attn==0.0.7 ; platform_machine != 'aarch64' and platform_machine != 'arm64'", "trimesh>=4.0.0", + "vsa==0.0.4 ; platform_machine != 'aarch64' and platform_machine != 'arm64'", "xatlas", ] tracing = [ - "opentelemetry-sdk", "opentelemetry-api", "opentelemetry-exporter-otlp", "opentelemetry-exporter-otlp-proto-grpc", + "opentelemetry-sdk", ] test = [ "accelerate", diff --git a/python/pyproject_npu.toml b/python/pyproject_npu.toml index 79c846b08..1fdda9a3a 100644 --- a/python/pyproject_npu.toml +++ b/python/pyproject_npu.toml @@ -14,30 +14,32 @@ classifiers = [ "License :: OSI Approved :: Apache Software License", ] +# Please keep dependency lists in this file sorted alphabetically by package name. dependencies = [ - "IPython", "aiohttp", "anthropic>=0.20.0", - "blobfile==3.0.0", "av", + "blobfile==3.0.0", "build", "compressed-tensors", "datasets", "decord2 ; sys_platform == 'linux' and (platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'armv7l')", + "easydict", "einops", "fastapi", - "easydict", "gguf", "hf_transfer", "huggingface_hub", "interegular", + "IPython", "llguidance>=0.7.11,<0.8.0", + "mistral_common>=1.11.0", "modelscope", "msgspec", "ninja", "numpy", - "openai-harmony==0.0.4", "openai==2.6.1", + "openai-harmony==0.0.4", "orjson", "outlines==0.1.11", "packaging", @@ -54,35 +56,34 @@ dependencies = [ "scipy", "sentencepiece", "setproctitle", + "smg-grpc-servicer>=0.5.0", "soundfile==0.13.1", "tiktoken", "timm==1.0.16", "torchao==0.9.0", "tqdm", - "mistral_common>=1.11.0", "transformers==5.8.1", "uvicorn", "uvloop", "xgrammar==0.2.1", - "smg-grpc-servicer>=0.5.0", ] [project.optional-dependencies] checkpoint-engine = ["checkpoint-engine==0.1.2"] diffusion = [ - "PyYAML==6.0.1", - "cloudpickle", - "diffusers==0.37.0", - "imageio==2.36.0", - "imageio-ffmpeg==0.5.1", - "moviepy>=2.0.0", - "opencv-python==4.10.0.84", - "remote-pdb", - "cache-dit==1.3.5", - "addict", - "scikit-image==0.25.2", - "trimesh>=4.0.0", - "xatlas", + "addict", + "cache-dit==1.3.5", + "cloudpickle", + "diffusers==0.37.0", + "imageio==2.36.0", + "imageio-ffmpeg==0.5.1", + "moviepy>=2.0.0", + "opencv-python==4.10.0.84", + "PyYAML==6.0.1", + "remote-pdb", + "scikit-image==0.25.2", + "trimesh>=4.0.0", + "xatlas", ] tracing = [ diff --git a/python/pyproject_other.toml b/python/pyproject_other.toml index 5334cb404..aa73f89ab 100755 --- a/python/pyproject_other.toml +++ b/python/pyproject_other.toml @@ -13,16 +13,16 @@ classifiers = [ "Programming Language :: Python :: 3", "License :: OSI Approved :: Apache Software License", ] -dependencies = ["aiohttp", "requests", "tqdm", "numpy", "IPython", "setproctitle"] +# Please keep dependency lists in this file sorted alphabetically by package name. +dependencies = ["aiohttp", "IPython", "numpy", "requests", "setproctitle", "tqdm"] [project.optional-dependencies] runtime_common = [ - "IPython", "aiohttp", - "apache-tvm-ffi", "anthropic>=0.20.0", - "blobfile==3.0.0", + "apache-tvm-ffi", "av", + "blobfile==3.0.0", "build", "compressed-tensors", "datasets", @@ -31,13 +31,15 @@ runtime_common = [ "fastapi", "gguf", "interegular", + "IPython", "llguidance>=0.7.11,<0.8.0", + "mistral_common>=1.11.0", "modelscope", "msgspec", "ninja", "numpy", - "openai-harmony==0.0.4", "openai==2.6.1", + "openai-harmony==0.0.4", "orjson", "outlines==0.1.11", "packaging", @@ -54,61 +56,60 @@ runtime_common = [ "scipy", "sentencepiece", "setproctitle", + "smg-grpc-servicer>=0.5.0", "soundfile==0.13.1", "tiktoken", "timm==1.0.16", "torchao==0.9.0", "tqdm", - "mistral_common>=1.11.0", "transformers==5.8.1", "uvicorn", "uvloop", "xgrammar==0.2.1", - "smg-grpc-servicer>=0.5.0", ] diffusion_common = [ - "PyYAML==6.0.1", + "addict", "cloudpickle", "diffusers==0.37.0", "imageio==2.36.0", "imageio-ffmpeg==0.5.1", "moviepy>=2.0.0", "opencv-python-headless==4.10.0.84", + "PyYAML==6.0.1", "remote-pdb", - "addict", "scikit-image==0.25.2", "trimesh>=4.0.0", "xatlas", ] tracing = [ - "opentelemetry-sdk", "opentelemetry-api", "opentelemetry-exporter-otlp", "opentelemetry-exporter-otlp-proto-grpc", + "opentelemetry-sdk", ] # HIP (Heterogeneous-computing Interface for Portability) for AMD # => base docker rocm/vllm-dev:20250114, not from public vllm whl srt_hip = [ - "sglang[runtime_common]", - "torch", - "petit_kernel==0.0.2", - "wave-lang==3.8.2", # Pin to 0.15.0: 0.16.0 needs torch>=2.10 (incompatible with ROCm torch # 2.9.1). An open-ended `<0.16.0` made pip backtrack into an unbuildable # ancient setuptools sdist; an exact pin keeps the resolver converging. "compressed-tensors==0.15.0", + "petit_kernel==0.0.2", + "sglang[runtime_common]", + "torch", + "wave-lang==3.8.2", ] diffusion_hip = [ - "sglang[diffusion_common]", + "cache-dit==1.3.0", "peft>=0.18.0,<0.19.0", # Pin to <0.19.0 due to torchao incompatibility + "runai_model_streamer>=0.15.5", + "sglang[diffusion_common]", "st_attn==0.0.7", "vsa==0.0.4", - "runai_model_streamer>=0.15.5", - "cache-dit==1.3.0", ] # For Intel Gaudi(device : hpu) follow the installation guide @@ -117,44 +118,44 @@ srt_hpu = ["sglang[runtime_common]"] # https://docs.sglang.io/platforms/mthreads_gpu.md srt_musa = [ + "deep-gemm>=0.1.3", + "flash_attn_3>=0.1.4", + "mate>=0.2.0", + "mthreads-ml-py", + "numpy<2.0", "sglang[runtime_common]", "torch", "torch_musa", "torchada>=0.1.59", - "mthreads-ml-py", - "mate>=0.2.0", - "deep-gemm>=0.1.3", - "flash_attn_3>=0.1.4", - "numpy<2.0", ] diffusion_musa = [ + "cache-dit==1.1.8", + "runai_model_streamer>=0.15.5", "sglang[diffusion_common]", "st_attn==0.0.7", "vsa==0.0.4", - "runai_model_streamer>=0.15.5", - "cache-dit==1.1.8", ] # https://docs.sglang.io/platforms/mps.md srt_mps = [ + "mlx", + "mlx-lm", "sglang[runtime_common]", "torch==2.11.0", "torchao==0.9.0", "torchaudio==2.11.0", "torchvision", - "mlx", - "mlx-lm", ] diffusion_mps = [ - "sglang[diffusion_common]", - "cloudpickle==3.1.2", - "remote-pdb==2.1.0", - "cache-dit==1.2.3", "addict==2.4.0", "av==16.1.0", + "cache-dit==1.2.3", + "cloudpickle==3.1.2", + "remote-pdb==2.1.0", "scikit-image==0.25.2", + "sglang[diffusion_common]", "trimesh>=4.0.0", "xatlas", ] @@ -172,10 +173,10 @@ test = [ "tabulate", ] -all_hip = ["sglang[srt_hip]", "sglang[diffusion_hip]", "sglang[tracing]"] +all_hip = ["sglang[diffusion_hip]", "sglang[srt_hip]", "sglang[tracing]"] all_hpu = ["sglang[srt_hpu]"] -all_musa = ["sglang[srt_musa]", "sglang[diffusion_musa]"] -all_mps = ["sglang[srt_mps]", "sglang[diffusion_mps]"] +all_musa = ["sglang[diffusion_musa]", "sglang[srt_musa]"] +all_mps = ["sglang[diffusion_mps]", "sglang[srt_mps]"] dev_hip = ["sglang[all_hip]", "sglang[test]"] dev_hpu = ["sglang[all_hpu]", "sglang[test]"] diff --git a/python/pyproject_xpu.toml b/python/pyproject_xpu.toml index b47aa707f..989fec648 100644 --- a/python/pyproject_xpu.toml +++ b/python/pyproject_xpu.toml @@ -14,33 +14,30 @@ classifiers = [ "License :: OSI Approved :: Apache Software License", ] +# Please keep dependency lists in this file sorted alphabetically by package name. dependencies = [ - "torch==2.12.0+xpu", - "torchcodec==0.12.0 ; sys_platform != 'linux' or (sys_platform == 'linux' and platform_machine != 'aarch64' and platform_machine != 'arm64' and platform_machine != 'armv7l')", # torch==2.12.0 on XPU uses torchcodec 0.12.0 - "av ; sys_platform == 'linux' and (platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'armv7l')", - "torchaudio==2.11.0+xpu", - "torchvision==0.27.0+xpu", - "sgl-kernel @ git+https://github.com/sgl-project/sgl-kernel-xpu.git", - "IPython", + "addict", "aiohttp", "anthropic>=0.20.0", + "av ; sys_platform == 'linux' and (platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'armv7l')", "blobfile==3.0.0", "build", "compressed-tensors", - "addict", "datasets", "easydict", "einops", "fastapi", "gguf", "interegular", + "IPython", "llguidance>=0.7.11,<0.8.0", + "mistral_common>=1.11.0", "modelscope", "msgspec", "ninja", "numpy", - "openai-harmony==0.0.4", "openai==2.6.1", + "openai-harmony==0.0.4", "orjson", "outlines==0.1.11", "packaging", @@ -57,35 +54,39 @@ dependencies = [ "scipy", "sentencepiece", "setproctitle", + "sgl-kernel @ git+https://github.com/sgl-project/sgl-kernel-xpu.git", + "smg-grpc-servicer>=0.5.0", "soundfile==0.13.1", "tiktoken", "timm==1.0.16", + "torch==2.12.0+xpu", "torchao==0.17.0+xpu", + "torchaudio==2.11.0+xpu", + "torchcodec==0.12.0 ; sys_platform != 'linux' or (sys_platform == 'linux' and platform_machine != 'aarch64' and platform_machine != 'arm64' and platform_machine != 'armv7l')", # torch==2.12.0 on XPU uses torchcodec 0.12.0 + "torchvision==0.27.0+xpu", "tqdm", - "mistral_common>=1.11.0", "transformers==5.8.1", "uvicorn", "uvloop", # "xgrammar==0.2.1", xgrammar depends on CUDA PyTorch and Triton only - "smg-grpc-servicer>=0.5.0", ] [project.optional-dependencies] diffusion = [ - "PyYAML==6.0.1", + "addict==2.4.0", + "av==16.1.0", + "cache-dit==1.3.0", "cloudpickle==3.1.2", "diffusers==0.38.0", "imageio==2.36.0", "imageio-ffmpeg==0.5.1", "moviepy>=2.0.0", "opencv-python==4.10.0.84", + "PyYAML==6.0.1", "remote-pdb==2.1.0", - "st_attn==0.0.7 ; platform_machine != 'aarch64' and platform_machine != 'arm64'", "runai_model_streamer>=0.15.5", - "cache-dit==1.3.0", - "addict==2.4.0", - "av==16.1.0", "scikit-image==0.25.2", + "st_attn==0.0.7 ; platform_machine != 'aarch64' and platform_machine != 'arm64'", "trimesh>=4.0.0", "xatlas", ] diff --git a/python/sglang/launch_server.py b/python/sglang/launch_server.py index 6b80e1330..04a5ef919 100644 --- a/python/sglang/launch_server.py +++ b/python/sglang/launch_server.py @@ -34,6 +34,7 @@ def run_server(server_args): asyncio.run(serve_grpc(server_args)) elif server_args.use_ray: + # Ray mode: HTTP mode with Ray backend. try: from sglang.srt.ray.http_server import launch_server except ImportError: