From c4db64c16bba9cb549d510735a064272c87779fa Mon Sep 17 00:00:00 2001 From: "Jiaxin(Jackson) Deng" <49501057+JiaxinD@users.noreply.github.com> Date: Tue, 24 Mar 2026 13:48:26 -0700 Subject: [PATCH] Add Lychee Doc Links Check to Local and CI (#19742) Co-authored-by: Zijie Xia Co-authored-by: Zijie Xia Co-authored-by: zijiexia <37504505+zijiexia@users.noreply.github.com> --- .github/linters/lychee-ci.toml | 42 +++++++++++++++++++ .github/linters/lychee.toml | 18 ++++++++ .github/workflows/lint.yml | 5 +++ .github/workflows/nightly-link-check.yml | 32 ++++++++++++++ .pre-commit-config.yaml | 13 ++++++ benchmark/blog_v0_2/README.md | 2 +- .../structured_outputs.ipynb | 9 ++-- docs/basic_usage/deepseek_v3.md | 2 +- docs/basic_usage/gpt_oss.md | 2 +- docs/developer_guide/contribution_guide.md | 20 ++++++++- docs/developer_guide/setup_github_runner.md | 2 +- docs/diffusion/installation.md | 2 +- docs/performance_dashboard/index.html | 2 +- docs/platforms/ascend_contribution_guide.md | 4 +- docs/platforms/ascend_npu.md | 2 +- docs/references/custom_chat_template.md | 2 +- .../multi_node_deployment/multi_node.md | 2 +- .../extending/support_new_models.md | 4 +- .../extending/transformers_fallback.md | 2 +- .../text_generation/generative_models.md | 8 ++-- .../multimodal_language_models.md | 2 +- examples/runtime/README.md | 2 +- python/sglang/multimodal_gen/README.md | 2 +- .../apps/ComfyUI_SGLDiffusion/README.md | 3 +- .../multimodal_gen/docs/quantization.md | 2 +- 25 files changed, 154 insertions(+), 32 deletions(-) create mode 100644 .github/linters/lychee-ci.toml create mode 100644 .github/linters/lychee.toml create mode 100644 .github/workflows/nightly-link-check.yml diff --git a/.github/linters/lychee-ci.toml b/.github/linters/lychee-ci.toml new file mode 100644 index 000000000..50919dcd3 --- /dev/null +++ b/.github/linters/lychee-ci.toml @@ -0,0 +1,42 @@ +no_progress = true +verbose = "warn" +timeout = 20 +max_concurrency = 8 +retry_wait_time = 2 +max_retries = 2 + +# CI should validate external links over the network. +offline = false +scheme = ["http", "https"] + +exclude_path = [ + # Exclude generated Sphinx build artifacts. + # - "(\\./)?" allows both "docs/..." and "./docs/..." + # - "[/\\\\]" supports both slash styles in CI environments + "^(\\./)?docs[/\\\\]_build[/\\\\]", +] + +exclude = [ + # Local-only endpoints referenced in docs/examples. + # These are expected to be unreachable in GitHub-hosted CI. + "^https?://localhost(:[0-9]+)?(/|$)", + "^http://127\\.0\\.0\\.1(:[0-9]+)?(/|$)", + # Vendor pages that frequently block/deny CI user-agents (transient 403/anti-bot). + "^https://www\\.intel\\.com/content/www/us/en/ark/products/series/240391/intel-arc-b-series-graphics\\.html$", + "^https://www\\.intel\\.com/content/www/us/en/ark/products/series/242616/intel-arc-pro-b-series-graphics\\.html$", + "^https://www\\.intel\\.com/content/www/us/en/products/sku/241598/intel-arc-b580-graphics/specifications\\.html$", + + # Non-routable bind address used in examples, never externally reachable. + "^http://0\\.0\\.0\\.0(/|$)", + + # Large doc portals with anti-bot/rate-limit behavior in CI. + # We keep API docs references in content but do not fail CI on access policy. + "^https://platform\\.openai\\.com/docs/", + "^https://gamma\\.app/docs/Optimizing-RL-with-SGLang-y0kqgj877k34779$", + "^https://aflah02\\.substack\\.com/p/multi-node-llm-inference-with-sglang/?$", + + # Known noisy image URLs used in notebook-rendered examples. + "^https://github\\.com/sgl-project/sglang/blob/main/examples/assets/example_image\\.png\\?raw=true$", + "^https://raw\\.githubusercontent\\.com/sgl-project/sglang/main/examples/assets/example_image\\.png/?$", + "^https://raw\\.githubusercontent\\.com/sgl-project/sglang/main/assets/logo\\.png/?$", +] diff --git a/.github/linters/lychee.toml b/.github/linters/lychee.toml new file mode 100644 index 000000000..cae63984d --- /dev/null +++ b/.github/linters/lychee.toml @@ -0,0 +1,18 @@ +# .github/linters/lychee.toml +no_progress = true +verbose = "warn" +timeout = 20 +max_concurrency = 8 + +offline = true + +# Ignore generated docs output; check source docs only. +exclude_path = [ + "^(\\./)?docs[/\\\\]_build[/\\\\]", +] + +exclude = [ + "^https?://localhost(:[0-9]+)?(/|$)", + "^http://127\\.0\\.0\\.1(:[0-9]+)?(/|$)", + "^http://0\\.0\\.0\\.0(/|$)", +] diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 80569a220..72a67d25b 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -25,6 +25,11 @@ jobs: - name: Run pre-commit checks run: SKIP=no-commit-to-branch pre-commit run --all-files --show-diff-on-failure + - name: Run lychee docs checks (offline references) + uses: lycheeverse/lychee-action@8646ba30535128ac92d33dfc9133794bfdd9b411 # v2 + with: + args: --config .github/linters/lychee.toml README.md "docs/**/*.md" "docs/**/*.rst" "docs/**/*.ipynb" + - name: Run sgl-kernel clang-format checks uses: DoozyX/clang-format-lint-action@v0.20 with: diff --git a/.github/workflows/nightly-link-check.yml b/.github/workflows/nightly-link-check.yml new file mode 100644 index 000000000..63d905cda --- /dev/null +++ b/.github/workflows/nightly-link-check.yml @@ -0,0 +1,32 @@ +name: Nightly Link Check + +on: + schedule: + - cron: "0 2 * * *" + workflow_dispatch: + +concurrency: + group: nightly-link-check-${{ github.ref }} + cancel-in-progress: true + +jobs: + lychee-online: + if: github.repository == 'sgl-project/sglang' + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Run lychee online link checks + uses: lycheeverse/lychee-action@8646ba30535128ac92d33dfc9133794bfdd9b411 # v2 + with: + fail: true + args: >- + --config .github/linters/lychee-ci.toml + README.md + docs/**/*.md + docs/**/*.rst + docs/**/*.ipynb + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f088453f3..a14997b9c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -81,3 +81,16 @@ repos: language: system files: ^\.github/CI_PERMISSIONS\.json$ pass_filenames: false + - repo: https://github.com/lycheeverse/lychee.git + rev: lychee-v0.22.0 + hooks: + - id: lychee + name: check doc links (offline) + args: ["--config", ".github/linters/lychee.toml"] + stages: [manual] + exclude: ^docs/_build/ + files: | + (?x)^( + README\.md| + docs/.*\.(md|rst|ipynb) + )$ diff --git a/benchmark/blog_v0_2/README.md b/benchmark/blog_v0_2/README.md index 7448554ee..c8f0f123b 100644 --- a/benchmark/blog_v0_2/README.md +++ b/benchmark/blog_v0_2/README.md @@ -73,7 +73,7 @@ cat online.jsonl | cut -d':' -f9 | cut -d',' -f1 We tried using vLLM 0.5.3.post1, but it often crashes under high loads, and it seems to have similar or worse performance compared to vLLM 0.5.2 from our partial benchmarking, so we are using the older version, vLLM 0.5.2. -Preparation for TensorRT LLM can refer to https://github.com/sgl-project/tensorrt-demo. Specifically, we used a batch size of 512, a max input length of 8192, and a max number of tokens of 8192. The instance count for preprocessing and postprocessing in Triton Server is 16. +For TensorRT LLM preparation, follow your internal TensorRT-LLM deployment guide. Specifically, we used a batch size of 512, a max input length of 8192, and a max number of tokens of 8192. The instance count for preprocessing and postprocessing in Triton Server is 16. ```bash # vLLM diff --git a/docs/advanced_features/structured_outputs.ipynb b/docs/advanced_features/structured_outputs.ipynb index ec603e4e6..8902c9497 100644 --- a/docs/advanced_features/structured_outputs.ipynb +++ b/docs/advanced_features/structured_outputs.ipynb @@ -356,8 +356,7 @@ "outputs": [], "source": [ "# Support for XGrammar latest structural tag format\n", - "# https://xgrammar.mlc.ai/docs/tutorials/structural_tag.html\n", - "\n", + "# \n", "response = client.chat.completions.create(\n", " model=\"meta-llama/Meta-Llama-3.1-8B-Instruct\",\n", " messages=messages,\n", @@ -645,8 +644,7 @@ "outputs": [], "source": [ "# Support for XGrammar latest structural tag format\n", - "# https://xgrammar.mlc.ai/docs/tutorials/structural_tag.html\n", - "\n", + "# \n", "payload = {\n", " \"text\": text,\n", " \"sampling_params\": {\n", @@ -925,8 +923,7 @@ "outputs": [], "source": [ "# Support for XGrammar latest structural tag format\n", - "# https://xgrammar.mlc.ai/docs/tutorials/structural_tag.html\n", - "\n", + "# \n", "sampling_params = {\n", " \"temperature\": 0.8,\n", " \"top_p\": 0.95,\n", diff --git a/docs/basic_usage/deepseek_v3.md b/docs/basic_usage/deepseek_v3.md index 92ffe6ca8..b558c2223 100644 --- a/docs/basic_usage/deepseek_v3.md +++ b/docs/basic_usage/deepseek_v3.md @@ -86,7 +86,7 @@ Please refer to [the example](https://github.com/sgl-project/sglang/tree/main/be - [Deploying DeepSeek on GB200 NVL72 with PD and Large Scale EP](https://lmsys.org/blog/2025-06-16-gb200-part-1/) ([Part I](https://lmsys.org/blog/2025-06-16-gb200-part-1/), [Part II](https://lmsys.org/blog/2025-09-25-gb200-part-2/)) - Comprehensive guide on GB200 optimizations. -- [Deploying DeepSeek with PD Disaggregation and Large-Scale Expert Parallelism on 96 H100 GPUs](https://lmsys.org/blog/2025-05-05-deepseek-pd-ep/) - Guide on PD disaggregation and large-scale EP. +- [Deploying DeepSeek with PD Disaggregation and Large-Scale Expert Parallelism on 96 H100 GPUs](https://lmsys.org/blog/2025-05-05-large-scale-ep/) - Guide on PD disaggregation and large-scale EP. - [Serving with two H20*8 nodes](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-two-h208-nodes). diff --git a/docs/basic_usage/gpt_oss.md b/docs/basic_usage/gpt_oss.md index 9f81da4f4..da8e778b2 100644 --- a/docs/basic_usage/gpt_oss.md +++ b/docs/basic_usage/gpt_oss.md @@ -25,7 +25,7 @@ GPT‑OSS can call built‑in tools for web search and Python execution. You can ### Tool & Reasoning Parser -- We support OpenAI Reasoning and Tool Call parser, as well as our SGLang native api for tool call and reasoning. Refer to [reasoning parser](../advanced_features/separate_reasoning.ipynb) and [tool call parser](../advanced_features/function_calling.ipynb) for more details. +- We support OpenAI Reasoning and Tool Call parser, as well as our SGLang native api for tool call and reasoning. Refer to [reasoning parser](../advanced_features/separate_reasoning.ipynb) and [tool parser](../advanced_features/tool_parser.ipynb) for more details. ## Notes diff --git a/docs/developer_guide/contribution_guide.md b/docs/developer_guide/contribution_guide.md index b1468e1d2..d8f0d974d 100644 --- a/docs/developer_guide/contribution_guide.md +++ b/docs/developer_guide/contribution_guide.md @@ -28,6 +28,22 @@ pre-commit run --all-files - **`pre-commit run --all-files`** manually runs all configured checks, applying fixes if possible. If it fails the first time, re-run it to ensure lint errors are fully resolved. Make sure your code passes all checks **before** creating a Pull Request. - **Do not commit** directly to the `main` branch. Always create a new branch (e.g., `feature/my-new-feature`), push your changes, and open a PR from that branch. +- Link checking with lychee is **enforced in CI**. By default, it is not blocking local commits. +- To run local link checks manually, use: `pre-commit run --hook-stage manual lychee --all-files`. + +### Link check guidance (lychee) + +- If your PR changes `docs/` or `README.md`, we recommend running local link checks before pushing. +- Local lychee is optional (CI is the source of truth), but if you want a system installation, see the official project: [lycheeverse/lychee](https://github.com/lycheeverse/lychee). +- Recommended local commands: + +```bash +# Fast local/offline check (pre-commit config) +pre-commit run --hook-stage manual lychee --all-files + +# CI-like online check (external links over network) +lychee --config .github/linters/lychee-ci.toml README.md "docs/**/*.md" "docs/**/*.rst" "docs/**/*.ipynb" +``` ## Run and add unit tests @@ -88,8 +104,8 @@ Also, do not rely on the "Latency/Output throughput" from this script, as it is GSM8K is too easy for state-of-the-art models nowadays. Please try your own more challenging accuracy tests. You can find additional accuracy eval examples in: -- [test_eval_accuracy_large.py](https://github.com/sgl-project/sglang/blob/main/test/srt/test_eval_accuracy_large.py) -- [test_gpt_oss_1gpu.py](https://github.com/sgl-project/sglang/blob/main/test/srt/test_gpt_oss_1gpu.py) +- [test_eval_accuracy_large.py](https://github.com/sgl-project/sglang/blob/main/test/registered/eval/test_eval_accuracy_large.py) +- [test_gpt_oss_1gpu.py](https://github.com/sgl-project/sglang/blob/main/test/registered/core/test_gpt_oss_1gpu.py) ## Benchmark the speed Refer to [Benchmark and Profiling](../developer_guide/benchmark_and_profiling.md). diff --git a/docs/developer_guide/setup_github_runner.md b/docs/developer_guide/setup_github_runner.md index 36342c8a6..49221acc9 100644 --- a/docs/developer_guide/setup_github_runner.md +++ b/docs/developer_guide/setup_github_runner.md @@ -27,7 +27,7 @@ pip install --upgrade pip export RUNNER_ALLOW_RUNASROOT=1 ``` -Then follow https://github.com/sgl-project/sglang/settings/actions/runners/new?arch=x64&os=linux to run `config.sh` +Then follow https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners/adding-self-hosted-runners to run `config.sh` **Notes** - Do not need to specify the runner group diff --git a/docs/diffusion/installation.md b/docs/diffusion/installation.md index 130ef03cd..2e4c23cc8 100644 --- a/docs/diffusion/installation.md +++ b/docs/diffusion/installation.md @@ -65,7 +65,7 @@ docker run --device=/dev/kfd --device=/dev/dri --ipc=host \ sglang generate --model-path black-forest-labs/FLUX.1-dev --prompt "A logo With Bold Large text: SGL Diffusion" --save-output ``` -For detailed ROCm system configuration and installation from source, see [AMD GPUs](../../platforms/amd_gpu.md). +For detailed ROCm system configuration and installation from source, see [AMD GPUs](../platforms/amd_gpu.md). ## Platform-Specific: MUSA (Moore Threads GPUs) diff --git a/docs/performance_dashboard/index.html b/docs/performance_dashboard/index.html index 1c5b57baf..e680f981a 100644 --- a/docs/performance_dashboard/index.html +++ b/docs/performance_dashboard/index.html @@ -936,7 +936,7 @@

SGLang Performance Dashboard — GitHub · - Documentation + Documentation

diff --git a/docs/platforms/ascend_contribution_guide.md b/docs/platforms/ascend_contribution_guide.md index 545da4029..fa87161ff 100644 --- a/docs/platforms/ascend_contribution_guide.md +++ b/docs/platforms/ascend_contribution_guide.md @@ -60,8 +60,8 @@ Also, do not rely on the "Latency/Output throughput" from this script, as it is GSM8K is too easy for state-of-the-art models nowadays. Please try your own more challenging accuracy tests. You can find additional accuracy eval examples in: -- [test_eval_accuracy_large.py](https://github.com/sgl-project/sglang/blob/main/test/srt/test_eval_accuracy_large.py) -- [test_moe_eval_accuracy_large.py](https://github.com/sgl-project/sglang/blob/main/test/srt/test_moe_eval_accuracy_large.py) +- [test_eval_accuracy_large.py](https://github.com/sgl-project/sglang/blob/main/test/registered/eval/test_eval_accuracy_large.py) +- [test_moe_eval_accuracy_large.py](https://github.com/sgl-project/sglang/blob/main/test/registered/eval/test_moe_eval_accuracy_large.py) ## Benchmark the speed Refer to [Benchmark and Profiling](../developer_guide/benchmark_and_profiling.md). diff --git a/docs/platforms/ascend_npu.md b/docs/platforms/ascend_npu.md index b2cce472f..860eb0a7d 100644 --- a/docs/platforms/ascend_npu.md +++ b/docs/platforms/ascend_npu.md @@ -6,7 +6,7 @@ You can install SGLang using any of the methods below. Please go through `System ## Component Version Mapping For SGLang | Component | Version | Obtain Way | |-------------------|-------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| HDK | 25.3.RC1 | [link](https://hiascend.com/hardware/firmware-drivers/commercial?product=7&model=33) | +| HDK | 25.3.RC1 | [link](https://www.hiascend.com/hardware/firmware-drivers/commercial?product=7&model=33) | | CANN | 8.5.0 | [Obtain Images](#obtain-cann-image) | | Pytorch Adapter | 7.3.0 | [link](https://gitcode.com/Ascend/pytorch/releases) | | MemFabric | 1.0.5 | `pip install memfabric-hybrid==1.0.5` | diff --git a/docs/references/custom_chat_template.md b/docs/references/custom_chat_template.md index f22ee8bec..870d09c1c 100644 --- a/docs/references/custom_chat_template.md +++ b/docs/references/custom_chat_template.md @@ -1,6 +1,6 @@ # Custom Chat Template -**NOTE**: There are two chat template systems in SGLang project. This document is about setting a custom chat template for the OpenAI-compatible API server (defined at [conversation.py](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/conversation.py)). It is NOT related to the chat template used in the SGLang language frontend (defined at [chat_template.py](https://github.com/sgl-project/sglang/blob/main/python/sglang/lang/chat_template.py)). +**NOTE**: There are two chat template systems in SGLang project. This document is about setting a custom chat template for the OpenAI-compatible API server (defined at [conversation.py](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/parser/conversation.py)). It is NOT related to the chat template used in the SGLang language frontend (defined at [chat_template.py](https://github.com/sgl-project/sglang/blob/main/python/sglang/lang/chat_template.py)). By default, the server uses the chat template specified in the model tokenizer from Hugging Face. It should just work for most official models such as Llama-2/Llama-3. diff --git a/docs/references/multi_node_deployment/multi_node.md b/docs/references/multi_node_deployment/multi_node.md index e6e5b5344..bdd0ca23d 100644 --- a/docs/references/multi_node_deployment/multi_node.md +++ b/docs/references/multi_node_deployment/multi_node.md @@ -30,7 +30,7 @@ python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-405B-Instr ## DeepSeek V3/R1 -Please refer to [DeepSeek documents for reference](https://docs.sglang.io/basic_usage/deepseek.html#running-examples-on-multi-node). +Please refer to [DeepSeek documents for reference](https://docs.sglang.io/basic_usage/deepseek_v3.html#running-examples-on-multi-node). ## Multi-Node Inference on SLURM diff --git a/docs/supported_models/extending/support_new_models.md b/docs/supported_models/extending/support_new_models.md index bd30fdd62..7951631e9 100644 --- a/docs/supported_models/extending/support_new_models.md +++ b/docs/supported_models/extending/support_new_models.md @@ -22,7 +22,7 @@ standard LLM support: to return `True` for your model. 2. **Register a new chat-template**: - Only when your default chat-template is unable to accept images as input: Register a new chat template in [conversation.py](https://github.com/sgl-project/sglang/tree/main/python/sglang/srt/conversation.py) and the corresponding matching function. + Only when your default chat-template is unable to accept images as input: Register a new chat template in [conversation.py](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/parser/conversation.py) and the corresponding matching function. 3. **Multimodal Data Processor**: Define a new `Processor` class that inherits from `BaseMultimodalProcessor` and register this processor as your @@ -68,7 +68,7 @@ To ensure the new model is well maintained, add it to the test suite by includin the [test_generation_models.py](https://github.com/sgl-project/sglang/blob/main/test/registered/models/test_generation_models.py) file, test the new model on your local machine and report the results on demonstrative benchmarks (GSM8K, MMLU, MMMU, MMMU-Pro, etc.) in your PR. \\ -For VLMs, also include a test in `test_vision_openai_server_{x}.py` (e.g. [test_vision_openai_server_a.py](https://github.com/sgl-project/sglang/blob/main/test/srt/test_vision_openai_server_a.py), [test_vision_openai_server_b.py](https://github.com/sgl-project/sglang/blob/main/test/srt/test_vision_openai_server_b.py)). +For VLMs, also include a test in `test_vision_openai_server_{x}.py` (e.g. [test_vision_openai_server_a.py](https://github.com/sgl-project/sglang/blob/main/test/registered/vlm/test_vision_openai_server_a.py)). This is an example command to run to test a new model on your local machine: diff --git a/docs/supported_models/extending/transformers_fallback.md b/docs/supported_models/extending/transformers_fallback.md index 3c7dd961c..cd80d5612 100644 --- a/docs/supported_models/extending/transformers_fallback.md +++ b/docs/supported_models/extending/transformers_fallback.md @@ -18,7 +18,7 @@ python3 -m sglang.launch_server \ ### Quantization -Transformers fall back has supported most of available quantization in SGLang (except GGUF). See [Quantization page](../advanced_features/quantization.md) for more information about supported quantization in SGLang. +Transformers fall back has supported most of available quantization in SGLang (except GGUF). See [Quantization page](../../advanced_features/quantization.md) for more information about supported quantization in SGLang. ### Remote code diff --git a/docs/supported_models/text_generation/generative_models.md b/docs/supported_models/text_generation/generative_models.md index f73aa200f..45ff7d5c5 100644 --- a/docs/supported_models/text_generation/generative_models.md +++ b/docs/supported_models/text_generation/generative_models.md @@ -25,12 +25,12 @@ in the GitHub search bar. | Model Family (Variants) | Example HuggingFace Identifier | Description | |-------------------------------------|--------------------------------------------------|----------------------------------------------------------------------------------------| -| **DeepSeek** (v1, v2, v3/R1) | `deepseek-ai/DeepSeek-R1` | Series of advanced reasoning-optimized models (including a 671B MoE) trained with reinforcement learning; top performance on complex reasoning, math, and code tasks. [SGLang provides Deepseek v3/R1 model-specific optimizations](../basic_usage/deepseek.md) and [Reasoning Parser](../advanced_features/separate_reasoning.ipynb)| -| **Kimi K2** (Thinking, Instruct) | `moonshotai/Kimi-K2-Instruct` | Moonshot AI's 1 trillion parameter MoE model (32B active) with 128K–256K context; state-of-the-art agentic intelligence with stable long-horizon agency across 200–300 sequential tool calls. Features MLA attention and native INT4 quantization. [See Reasoning Parser docs](../advanced_features/separate_reasoning.ipynb)| +| **DeepSeek** (v1, v2, v3/R1) | `deepseek-ai/DeepSeek-R1` | Series of advanced reasoning-optimized models (including a 671B MoE) trained with reinforcement learning; top performance on complex reasoning, math, and code tasks. [SGLang provides Deepseek v3/R1 model-specific optimizations](../../basic_usage/deepseek_v3.md) and [Reasoning Parser](../../advanced_features/separate_reasoning.ipynb)| +| **Kimi K2** (Thinking, Instruct) | `moonshotai/Kimi-K2-Instruct` | Moonshot AI's 1 trillion parameter MoE model (32B active) with 128K–256K context; state-of-the-art agentic intelligence with stable long-horizon agency across 200–300 sequential tool calls. Features MLA attention and native INT4 quantization. [See Reasoning Parser docs](../../advanced_features/separate_reasoning.ipynb)| | **Kimi Linear** (48B-A3B) | `moonshotai/Kimi-Linear-48B-A3B-Instruct` | Moonshot AI's hybrid linear attention model (48B total, 3B active) with 1M token context; features Kimi Delta Attention (KDA) for up to 6× faster decoding and 75% KV cache reduction vs full attention. | | **GPT-OSS** | `openai/gpt-oss-20b`, `openai/gpt-oss-120b` | OpenAI’s latest GPT-OSS series for complex reasoning, agentic tasks, and versatile developer use cases.| -| **Qwen** (3.5, 3, 3MoE, 3Next, 2.5, 2 series) | `Qwen/Qwen3.5-397B-A17B`, `Qwen/Qwen3-0.6B`, `Qwen/Qwen3-30B-A3B` | Alibaba’s latest Qwen3 series for complex reasoning, language understanding, and generation tasks; Support for MoE variants along with previous generation 2.5, 2, etc. [SGLang provides Qwen3 specific reasoning parser](../advanced_features/separate_reasoning.ipynb)| -| **Llama** (2, 3.x, 4 series) | `meta-llama/Llama-4-Scout-17B-16E-Instruct` | Meta's open LLM series, spanning 7B to 400B parameters (Llama 2, 3, and new Llama 4) with well-recognized performance. [SGLang provides Llama-4 model-specific optimizations](../basic_usage/llama4.md) | +| **Qwen** (3.5, 3, 3MoE, 3Next, 2.5, 2 series) | `Qwen/Qwen3.5-397B-A17B`, `Qwen/Qwen3-0.6B`, `Qwen/Qwen3-30B-A3B`, `Qwen/Qwen3-Next-80B-A3B-Instruct` | Alibaba’s latest Qwen3 series for complex reasoning, language understanding, and generation tasks; Support for MoE variants along with previous generation 2.5, 2, etc. [SGLang provides Qwen3 specific reasoning parser](../../advanced_features/separate_reasoning.ipynb)| +| **Llama** (2, 3.x, 4 series) | `meta-llama/Llama-4-Scout-17B-16E-Instruct` | Meta's open LLM series, spanning 7B to 400B parameters (Llama 2, 3, and new Llama 4) with well-recognized performance. [SGLang provides Llama-4 model-specific optimizations](../../basic_usage/llama4.md) | | **Mistral** (Mixtral, NeMo, Small3) | `mistralai/Mistral-7B-Instruct-v0.2` | Open 7B LLM by Mistral AI with strong performance; extended into MoE (“Mixtral”) and NeMo Megatron variants for larger scale. | | **Gemma** (v1, v2, v3) | `google/gemma-3-1b-it` | Google’s family of efficient multilingual models (1B–27B); Gemma 3 offers a 128K context window, and its larger (4B+) variants support vision input. | | **Phi** (Phi-1.5, Phi-2, Phi-3, Phi-4, Phi-MoE series) | `microsoft/Phi-4-multimodal-instruct`, `microsoft/Phi-3.5-MoE-instruct` | Microsoft’s Phi family of small models (1.3B–5.6B); Phi-4-multimodal (5.6B) processes text, images, and speech, Phi-4-mini is a high-accuracy text model and Phi-3.5-MoE is a mixture-of-experts model. | diff --git a/docs/supported_models/text_generation/multimodal_language_models.md b/docs/supported_models/text_generation/multimodal_language_models.md index 0dab3a28a..1b94db932 100644 --- a/docs/supported_models/text_generation/multimodal_language_models.md +++ b/docs/supported_models/text_generation/multimodal_language_models.md @@ -63,7 +63,7 @@ SGLang supports video input for Vision-Language Models (VLMs), enabling temporal | **GLM-4v** (4.5V, 4.1V, MOE) | `zai-org/GLM-4.5V` | Video clips are read with Decord, converted to tensors, and passed to the model alongside metadata for rotary-position handling. | | **NVILA** (Full & Lite) | `Efficient-Large-Model/NVILA-8B` | The runtime samples eight frames per clip and attaches them to the multimodal request when `video_data` is present. | | **LLaVA video variants** (LLaVA-NeXT-Video, LLaVA-OneVision) | `lmms-lab/LLaVA-NeXT-Video-7B` | The processor routes video prompts to the LlavaVid video-enabled architecture, and the provided example shows how to query it with `sgl.video(...)` clips. | -| **NVIDIA Nemotron Nano 2.0 VL** | `nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16` | The processor samples at 2 FPS, at a max of 128 frames, as per model training. The model uses [EVS](../../python/sglang/srt/multimodal/evs/README.md), a pruning method that removes redundant tokens from video embeddings. By default `video_pruning_rate=0.7`. Change this by providing: `--json-model-override-args '{"video_pruning_rate": 0.0}'` to disable EVS, for example. | +| **NVIDIA Nemotron Nano 2.0 VL** | `nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16` | The processor samples at 2 FPS, at a max of 128 frames, as per model training. The model uses [EVS](../../../python/sglang/srt/multimodal/evs/README.md), a pruning method that removes redundant tokens from video embeddings. By default `video_pruning_rate=0.7`. Change this by providing: `--json-model-override-args '{"video_pruning_rate": 0.0}'` to disable EVS, for example. | | **JetVLM** | | The runtime samples eight frames per clip and attaches them to the multimodal request when `video_data` is present. | Use `sgl.video(path, num_frames)` when building prompts to attach clips from your SGLang programs. diff --git a/examples/runtime/README.md b/examples/runtime/README.md index 8b623fc34..0bc400d47 100644 --- a/examples/runtime/README.md +++ b/examples/runtime/README.md @@ -5,7 +5,7 @@ The below examples will mostly need you to start a server in a separate terminal ## Native API * `lora.py`: An example how to use LoRA adapters. -* `multimodal_embedding.py`: An example how perform [multi modal embedding](Alibaba-NLP/gme-Qwen2-VL-2B-Instruct). +* `multimodal_embedding.py`: An example how perform [multi modal embedding](https://huggingface.co/Alibaba-NLP/gme-Qwen2-VL-2B-Instruct). * `openai_batch_chat.py`: An example how to process batch requests for chat completions. * `openai_batch_complete.py`: An example how to process batch requests for text completions. * **`openai_chat_with_response_prefill.py`**: diff --git a/python/sglang/multimodal_gen/README.md b/python/sglang/multimodal_gen/README.md index aad97b3f4..8dceb1934 100644 --- a/python/sglang/multimodal_gen/README.md +++ b/python/sglang/multimodal_gen/README.md @@ -89,7 +89,7 @@ sglang generate \ --save-output ``` -For more usage examples (e.g. OpenAI compatible API, server mode), check [cli.md](https://github.com/sgl-project/sglang/tree/main/docs/diffusion/cli.md). +For more usage examples (e.g. OpenAI compatible API, server mode), check [cli.md](https://github.com/sgl-project/sglang/tree/main/docs/diffusion/api/cli.md). ## Contributing diff --git a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/README.md b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/README.md index 3d1146121..18a10cf5c 100644 --- a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/README.md +++ b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/README.md @@ -4,7 +4,7 @@ A ComfyUI plugin for integrating with SGLang Diffusion server, supporting image ## Installation -1. **Install SGLang**: Follow the [Installation Guide](../../docs/install.md) to install `sglang[diffusion]`. +1. **Install SGLang**: Follow the [Installation Guide](../../../../../docs/diffusion/installation.md) to install `sglang[diffusion]`. 2. **Install Plugin**: Copy this entire directory (`ComfyUI_SGLDiffusion`) to your ComfyUI `custom_nodes/` folder. 3. **Restart ComfyUI**: Restart ComfyUI to load the plugin. @@ -53,7 +53,6 @@ To use these workflows: 3. Adjust the parameters and model paths as needed. 4. Run the workflow. - ## Current Implementation This plugin provides a high-performance backend for diffusion models in ComfyUI. By leveraging SGLang's optimized kernels and parallelization techniques (Tensor Parallelism, TeaCache, etc.), it significantly accelerates the sampling process, especially for large models like FLUX. diff --git a/python/sglang/multimodal_gen/docs/quantization.md b/python/sglang/multimodal_gen/docs/quantization.md index b8588a46d..635b2375f 100644 --- a/python/sglang/multimodal_gen/docs/quantization.md +++ b/python/sglang/multimodal_gen/docs/quantization.md @@ -44,7 +44,7 @@ For more installation information, please refer to the [Nunchaku Official Docume Nunchaku provides pre-quantized model weights available on Hugging Face: - [nunchaku-ai/nunchaku-qwen-image](https://huggingface.co/nunchaku-ai/nunchaku-qwen-image) -- [nunchaku-ai/nunchaku-flux](https://huggingface.co/nunchaku-ai/nunchaku-flux) +- [Nunchaku FLUX.1 collection](https://huggingface.co/collections/nunchaku-ai/nunchaku-flux1) Taking Qwen-Image as an example, several quantized models with different configurations are provided: