diff --git a/.claude/skills/add-jit-kernel/SKILL.md b/.claude/skills/add-jit-kernel/SKILL.md index 8095ea5f1..9d6ab1e06 100644 --- a/.claude/skills/add-jit-kernel/SKILL.md +++ b/.claude/skills/add-jit-kernel/SKILL.md @@ -618,7 +618,7 @@ cd test && python3 run_suite.py --hw cuda --suite base-b-kernel-benchmark-test-1 ## References -- `docs/developer_guide/development_jit_kernel_guide.md` +- `docs_new/docs/developer_guide/development_jit_kernel_guide.mdx` - `test/run_suite.py` — suite names, discovery of `test/registered/`, execution entrypoint for CI - `python/sglang/test/ci/ci_register.py` — `register_cuda_ci` and AST registration rules - `python/sglang/jit_kernel/utils.py` — `cache_once`, `load_jit`, `make_cpp_args`, `should_run_full_tests`, `get_ci_test_range` diff --git a/.claude/skills/llm-torch-profiler-analysis/references/source-map.md b/.claude/skills/llm-torch-profiler-analysis/references/source-map.md index f460a45fa..a2ff8b0dd 100644 --- a/.claude/skills/llm-torch-profiler-analysis/references/source-map.md +++ b/.claude/skills/llm-torch-profiler-analysis/references/source-map.md @@ -32,7 +32,7 @@ Use these upstream files when the workflow or behavior needs to be justified fro ## Documentation and tests -- `docs/developer_guide/benchmark_and_profiling.md` +- `docs_new/docs/developer_guide/benchmark_and_profiling.mdx` - canonical profiling docs - `test/registered/profiling/test_start_profile.py` diff --git a/.github/linters/lychee-ci.toml b/.github/linters/lychee-ci.toml index 50919dcd3..232ebe17b 100644 --- a/.github/linters/lychee-ci.toml +++ b/.github/linters/lychee-ci.toml @@ -9,14 +9,9 @@ max_retries = 2 offline = false scheme = ["http", "https"] -exclude_path = [ - # Exclude generated Sphinx build artifacts. - # - "(\\./)?" allows both "docs/..." and "./docs/..." - # - "[/\\\\]" supports both slash styles in CI environments - "^(\\./)?docs[/\\\\]_build[/\\\\]", -] - exclude = [ + # Slack invite link returns 403 to CI user-agents (anti-bot); the link is valid. + "^https://slack\\.sglang\\.io(/|$)", # Local-only endpoints referenced in docs/examples. # These are expected to be unreachable in GitHub-hosted CI. "^https?://localhost(:[0-9]+)?(/|$)", diff --git a/.github/linters/lychee.toml b/.github/linters/lychee.toml index cae63984d..b2d6a0cb3 100644 --- a/.github/linters/lychee.toml +++ b/.github/linters/lychee.toml @@ -6,11 +6,6 @@ max_concurrency = 8 offline = true -# Ignore generated docs output; check source docs only. -exclude_path = [ - "^(\\./)?docs[/\\\\]_build[/\\\\]", -] - exclude = [ "^https?://localhost(:[0-9]+)?(/|$)", "^http://127\\.0\\.0\\.1(:[0-9]+)?(/|$)", diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index b9a7d8ea9..368186bb6 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -13,29 +13,6 @@ jobs: with: fetch-depth: 0 - - name: Reject changes under legacy docs/ - if: github.event_name == 'pull_request' - run: | - set -euo pipefail - BASE_REF="${{ github.base_ref }}" - # Refresh origin/ with full history; --depth=1 would - # shallow the ref and break the merge-base used by `...`. - git fetch --no-tags origin "$BASE_REF" - # First, verify the diff itself succeeds and check whether any - # files changed. A silent failure here would let docs/ changes - # through, so the explicit `if !` guard is important. - if ! CHANGED=$(git diff --name-only --diff-filter=ACMRDTUXB "origin/${BASE_REF}...HEAD"); then - echo "git diff origin/${BASE_REF}...HEAD failed; aborting." >&2 - exit 2 - fi - if [ -z "$CHANGED" ]; then - echo "No changed files vs origin/${BASE_REF}; skipping." - exit 0 - fi - # Re-emit with -z so xargs can safely handle whitespace in paths. - git diff -z --name-only --diff-filter=ACMRDTUXB "origin/${BASE_REF}...HEAD" \ - | xargs -0 python3 scripts/ci/check_no_docs_changes.py - - name: Set up Python uses: actions/setup-python@v4 with: @@ -52,7 +29,24 @@ jobs: - name: Run lychee docs checks (offline references) uses: lycheeverse/lychee-action@8646ba30535128ac92d33dfc9133794bfdd9b411 # v2 with: - args: --config .github/linters/lychee.toml README.md "docs/**/*.md" "docs/**/*.rst" "docs/**/*.ipynb" + args: --config .github/linters/lychee.toml README.md + + # docs_new/ (Mintlify) internal-link check. Runs only in CI — contributors + # are NOT required to install `mint` for local pre-commit. External links + # are covered separately by the nightly lychee job (no --check-external here). + - name: Set up Node + uses: actions/setup-node@v4 + with: + node-version: "20" + + # Hard gate: docs_new internal links/anchors/redirects must resolve. + # mint is pinned for reproducibility (its heading-slug rules are + # version-sensitive); bump deliberately after re-verifying locally. + - name: Check docs_new links (Mintlify broken-links) + run: | + npm install -g mint@4.2.559 + cd docs_new + mint broken-links --check-anchors --check-redirects - name: Run sgl-kernel clang-format checks uses: DoozyX/clang-format-lint-action@v0.20 diff --git a/.github/workflows/nightly-link-check.yml b/.github/workflows/nightly-link-check.yml index 63d905cda..f5a8384b9 100644 --- a/.github/workflows/nightly-link-check.yml +++ b/.github/workflows/nightly-link-check.yml @@ -25,8 +25,5 @@ jobs: args: >- --config .github/linters/lychee-ci.toml README.md - docs/**/*.md - docs/**/*.rst - docs/**/*.ipynb env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/release-branch-cut.yml b/.github/workflows/release-branch-cut.yml index 7adde9bae..b818904ba 100644 --- a/.github/workflows/release-branch-cut.yml +++ b/.github/workflows/release-branch-cut.yml @@ -112,14 +112,14 @@ jobs: VERSION=$(echo "$BRANCH_NAME" | sed 's/release\///') # Update git clone version references in docs - sed -i "s/git clone -b v[0-9]\+\.[0-9]\+\.[0-9]\+\.\?post\?[0-9]*/git clone -b $VERSION/" docs/get_started/install.md - sed -i "s/git clone -b v[0-9]\+\.[0-9]\+\.[0-9]\+\.\?post\?[0-9]*/git clone -b $VERSION/" docs/platforms/amd_gpu.md + sed -i "s/git clone -b v[0-9]\+\.[0-9]\+\.[0-9]\+\.\?post\?[0-9]*/git clone -b $VERSION/" docs_new/docs/get-started/install.mdx + sed -i "s/git clone -b v[0-9]\+\.[0-9]\+\.[0-9]\+\.\?post\?[0-9]*/git clone -b $VERSION/" docs_new/docs/hardware-platforms/amd_gpu.mdx # Check if any changes were made if git diff --quiet; then echo "No version references needed updating" else - git add docs/get_started/install.md docs/platforms/amd_gpu.md + git add docs_new/docs/get-started/install.mdx docs_new/docs/hardware-platforms/amd_gpu.mdx git commit -m "docs: update version references to $VERSION" echo "Updated version references to $VERSION" fi diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9b29ba409..bd5365fc0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -37,7 +37,7 @@ repos: args: - --select=F401,F821,UP037 - --fix - files: ^(benchmark/|docs/|examples/|python/sglang/|sgl-model-gateway/py_*|test/) + files: ^(benchmark/|docs_new/|examples/|python/sglang/|sgl-model-gateway/py_*|test/) exclude: | (?x)^( .*/__init__\.py$| @@ -105,13 +105,6 @@ repos: language: system files: ^python/sglang/.*\.py$ pass_filenames: false - - id: check-no-docs-changes - name: reject changes under legacy docs/ - entry: python3 scripts/ci/check_no_docs_changes.py - language: system - pass_filenames: false - always_run: true - stages: [pre-commit] - id: rustfmt-sgl-model-gateway name: rustfmt sgl-model-gateway (nightly) entry: bash -c 'rustup component add --toolchain nightly rustfmt >/dev/null && cd sgl-model-gateway && cargo +nightly fmt -- --check' @@ -124,6 +117,13 @@ repos: language: system files: ^experimental/sgl-router/.*\.rs$ pass_filenames: false + - id: mint-broken-links + name: check docs_new links (Mintlify; opt-in, requires `mint` on PATH) + entry: bash -c 'cd docs_new && mint broken-links --check-anchors --check-redirects' + language: system + pass_filenames: false + always_run: true + stages: [manual] - repo: https://github.com/lycheeverse/lychee.git rev: lychee-v0.22.0 hooks: @@ -131,9 +131,7 @@ repos: name: check doc links (offline) args: ["--config", ".github/linters/lychee.toml"] stages: [manual] - exclude: ^docs/_build/ files: | (?x)^( - README\.md| - docs/.*\.(md|rst|ipynb) + README\.md )$ diff --git a/benchmark/mmmu/README.md b/benchmark/mmmu/README.md index 61fea8bc4..9cf47527a 100644 --- a/benchmark/mmmu/README.md +++ b/benchmark/mmmu/README.md @@ -46,4 +46,4 @@ python benchmark/mmmu/bench_hf.py --model-path Qwen/Qwen2-VL-7B-Instruct ``` # Profiling MMMU -You should use the standard instructions found in the [dedicated profiling doc](../../docs/developer_guide/benchmark_and_profiling.md) if running this benchmark with the profile option. We recommend using `--concurrency 1` for consistency, which makes profiling and debugging easier. +You should use the standard instructions found in the [dedicated profiling doc](https://docs.sglang.io/docs/developer_guide/benchmark_and_profiling) if running this benchmark with the profile option. We recommend using `--concurrency 1` for consistency, which makes profiling and debugging easier. diff --git a/docs/Makefile b/docs/Makefile deleted file mode 100644 index 716160e56..000000000 --- a/docs/Makefile +++ /dev/null @@ -1,98 +0,0 @@ -# Minimal Makefile for Sphinx documentation -SPHINXOPTS ?= -SPHINXBUILD ?= sphinx-build -SPHINXAUTOBUILD ?= sphinx-autobuild -SOURCEDIR = . -BUILDDIR = _build -PORT ?= 8003 - -help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - @echo "" - @echo "Additional targets:" - @echo " serve to build and serve documentation with auto-build and live reload" - -# Compile Notebook files and record execution time -compile: - @set -e; \ - echo "Starting Notebook compilation..."; \ - mkdir -p logs; \ - echo "Notebook execution timings:" > logs/timing.log; \ - START_TOTAL=$$(date +%s); \ - find $(SOURCEDIR) -path "*/_build/*" -prune -o -name "*.ipynb" -print0 | \ - parallel -0 -j3 --halt soon,fail=1 ' \ - NB_NAME=$$(basename {}); \ - START_TIME=$$(date +%s); \ - retry --delay=0 --times=2 -- \ - jupyter nbconvert --to notebook --execute --inplace "{}" \ - --ExecutePreprocessor.timeout=600 \ - --ExecutePreprocessor.kernel_name=python3; \ - RET_CODE=$$?; \ - END_TIME=$$(date +%s); \ - ELAPSED_TIME=$$((END_TIME - START_TIME)); \ - echo "$${NB_NAME}: $${ELAPSED_TIME}s" >> logs/timing.log; \ - exit $$RET_CODE' || exit 1; \ - END_TOTAL=$$(date +%s); \ - TOTAL_ELAPSED=$$((END_TOTAL - START_TOTAL)); \ - echo "---------------------------------" >> logs/timing.log; \ - echo "Total execution time: $${TOTAL_ELAPSED}s" >> logs/timing.log; \ - echo "All Notebook execution timings:" && cat logs/timing.log - -# Convert Notebook files to Markdown artifacts (no execution) -markdown: - @set -e; \ - echo "Exporting docs to Markdown..."; \ - mkdir -p "$(BUILDDIR)/html/markdown"; \ - \ - # 1) Copy .md and .rst files as-is; additionally convert .rst -> .md \ - find $(SOURCEDIR) -path "*/_build/*" -prune -o \( -name "*.md" -o -name "*.rst" \) -print0 | \ - parallel -0 -j3 --halt soon,fail=1 ' \ - SRC="{}"; \ - REL_DIR=$$(dirname "$$SRC"); \ - OUT_DIR="$(BUILDDIR)/html/markdown/$$REL_DIR"; \ - mkdir -p "$$OUT_DIR"; \ - cp -f "$$SRC" "$$OUT_DIR/"; \ - case "$$SRC" in \ - *.rst) \ - BASE=$$(basename "$$SRC" .rst); \ - pandoc -f rst -t gfm "$$SRC" -o "$$OUT_DIR/$$BASE.md" ;; \ - esac \ - ' || exit 1; \ - \ - # 2) Convert .ipynb -> .md \ - find $(SOURCEDIR) -path "*/_build/*" -prune -o -name "*.ipynb" -print0 | \ - parallel -0 -j3 --halt soon,fail=1 ' \ - NB_SRC="{}"; \ - REL_DIR=$$(dirname "$$NB_SRC"); \ - NB_NAME=$$(basename "$$NB_SRC"); \ - NB_BASE=$${NB_NAME%.ipynb}; \ - OUT_DIR="$(BUILDDIR)/html/markdown/$$REL_DIR"; \ - mkdir -p "$$OUT_DIR"; \ - jupyter nbconvert --to markdown "$$NB_SRC" \ - --output "$$NB_BASE.md" \ - --output-dir "$$OUT_DIR" \ - >/dev/null; \ - ' || exit 1; \ - \ - echo "Markdown artifacts written to: $(BUILDDIR)/html/markdown" - - - -# Serve documentation with auto-build and live reload -serve: - @echo "Starting auto-build server at http://0.0.0.0:$(PORT)" - @$(SPHINXAUTOBUILD) "$(SOURCEDIR)" "$(BUILDDIR)/html" \ - --host 0.0.0.0 \ - --port $(PORT) \ - --watch $(SOURCEDIR) \ - --re-ignore ".*\.(ipynb_checkpoints|pyc|pyo|pyd|git)" - -.PHONY: help Makefile compile clean serve - -%: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -clean: - find . -name "*.ipynb" -exec nbstripout {} \; - rm -rf $(BUILDDIR) - rm -rf logs diff --git a/docs/README.md b/docs/README.md deleted file mode 100644 index 7764169b1..000000000 --- a/docs/README.md +++ /dev/null @@ -1,131 +0,0 @@ -# SGLang Documentation - -This is the documentation website for the SGLang project (https://github.com/sgl-project/sglang). - -We recommend new contributors start from writing documentation, which helps you quickly understand SGLang codebase. -Most documentation files are located under the `docs/` folder. - -## Docs Workflow - -### Install Dependency - -**Linux:** -```bash -apt-get update && apt-get install -y pandoc parallel retry -pip install -r requirements.txt -``` - -**macOS:** -```bash -brew install pandoc parallel retry -pip install -r requirements.txt -``` - -### Update Documentation - -Update your Jupyter notebooks in the appropriate subdirectories under `docs/`. If you add new files, remember to update `index.rst` (or relevant `.rst` files) accordingly. - -- **`pre-commit run --all-files`** manually runs all configured checks, applying fixes if possible. If it fails the first time, re-run it to ensure lint errors are fully resolved. Make sure your code passes all checks **before** creating a Pull Request. - -```bash -# 1) Compile all Jupyter notebooks -make compile # This step can take a long time (10+ mins). You can consider skipping this step if you can make sure your added files are correct. -make html - -# 2) Compile and Preview documentation locally with auto-build -# This will automatically rebuild docs when files change -# Open your browser at the displayed port to view the docs -bash serve.sh - -# 2a) Alternative ways to serve documentation -# Directly use make serve -make serve -# With custom port -PORT=8080 make serve - -# 3) Clean notebook outputs -# nbstripout removes notebook outputs so your PR stays clean -pip install nbstripout -find . -name '*.ipynb' -exec nbstripout {} \; - -# 4) Pre-commit checks and create a PR -# After these checks pass, push your changes and open a PR on your branch -pre-commit run --all-files -``` - -## Documentation Style Guidelines - -- For common functionalities, we prefer **Jupyter Notebooks** over Markdown so that all examples can be executed and validated by our docs CI pipeline. For complex features (e.g., distributed serving), Markdown is preferred. -- Keep in mind the documentation execution time when writing interactive Jupyter notebooks. Each interactive notebook will be run and compiled against every commit to ensure they are runnable, so it is important to apply some tips to reduce the documentation compilation time: - - Use small models (e.g., `qwen/qwen2.5-0.5b-instruct`) for most cases to reduce server launch time. - - Reuse the launched server as much as possible to reduce server launch time. -- Do not use absolute links (e.g., `https://docs.sglang.io/get_started/install.html`). Always prefer relative links (e.g., `../get_started/install.md`). -- Follow the existing examples to learn how to launch a server, send a query and other common styles. - -## Documentation Build, Deployment, and CI - -The SGLang documentation pipeline is based on **Sphinx** and supports rendering Jupyter notebooks (`.ipynb`) into HTML/Markdown for web display. Detailed logits can be found in the [Makefile](./Makefile). - -### Notebook Execution (`make compile`) - -The `make compile` target is responsible for executing notebooks before rendering: - -* Finds all `.ipynb` files under `docs/` (excluding `_build/`) -* Executes notebooks in parallel using GNU Parallel, with a relatively small `--mem-fraction-static` -* Wraps execution with `retry` to reduce flaky failures -* Executes notebooks via `jupyter nbconvert --execute --inplace` -* Records execution timing in `logs/timing.log` - -This step ensures notebooks contain up-to-date outputs with each commit in the main branch before rendering. - -### Web Rendering (`make html`) - -After compilation, Sphinx builds the website: - -* Reads Markdown, reStructuredText, and Jupyter notebooks -* Renders them into HTML pages -* Outputs the website into: - -``` -docs/_build/html/ -``` - -This directory is the source for online documentation hosting. - -### Markdown Export (`make markdown`) - -To support downstream consumers, we add a **new Makefile target**: - -```bash -make markdown -``` - -This target: - -* Does **not modify** `make compile` -* Scans all `.ipynb` files (excluding `_build/`) -* Converts notebooks directly to Markdown using `jupyter nbconvert --to markdown` -* Writes Markdown artifacts into the existing build directory: - -``` -docs/_build/html/markdown/.md -``` - -Example: - -``` -docs/advanced_features/lora.ipynb -→ docs/_build/html/markdown/advanced_features/lora.md -``` - -### CI Execution - -In our [CI](https://github.com/sgl-project/sglang/blob/main/.github/workflows/release-docs.yml), the documentation pipeline first gets all the executed results and renders HTML and Markdown by: - -```bash -make compile # execute notebooks (ensure outputs are up to date) -make html # build website as usual -make markdown # export markdown artifacts into _build/html/markdown -``` - -Then, the compiled results are forced pushed to [sgl-project.io](https://github.com/sgl-project/sgl-project.github.io) for rendering. In other words, sgl-project.io is push-only. All the changes of SGLang docs should be made directly in SGLang main repo, then push to the sgl-project.io. diff --git a/docs/_static/css/custom_log.css b/docs/_static/css/custom_log.css deleted file mode 100644 index 57d0cf6d1..000000000 --- a/docs/_static/css/custom_log.css +++ /dev/null @@ -1,53 +0,0 @@ -.output_area { - color: #615656; -} - -table.autosummary td { - width: 50% - } - - img.align-center { - display: block; - margin-left: auto; - margin-right: auto; -} - -.output_area.stderr { - color: #d3d3d3 !important; -} - -.output_area.stdout { - color: #d3d3d3 !important; -} - -div.output_area.stderr { - color: #d3d3d3 !important; -} - -div.output_area.stdout { - color: #d3d3d3 !important; -} - -.sglang-docs-deprecation-banner { - background: #fff4cc; - border-bottom: 1px solid #d8a21f; - color: #2f2a1f; - font-size: 0.95rem; - line-height: 1.45; - overflow-wrap: anywhere; - padding: 0.75rem 1.25rem; - position: relative; - text-align: center; - z-index: 1030; -} - -.sglang-docs-deprecation-banner a { - color: #1f5fbf; - font-weight: 600; - text-decoration: underline; -} - -.sglang-docs-deprecation-banner a:focus, -.sglang-docs-deprecation-banner a:hover { - color: #143f80; -} diff --git a/docs/_static/css/readthedocs.css b/docs/_static/css/readthedocs.css deleted file mode 100644 index aca6649b4..000000000 --- a/docs/_static/css/readthedocs.css +++ /dev/null @@ -1,9 +0,0 @@ -table.autosummary td { - width: 50% -} - -img.align-center { - display: block; - margin-left: auto; - margin-right: auto; -} diff --git a/docs/_static/image/dpa.png b/docs/_static/image/dpa.png deleted file mode 100644 index 672e02218..000000000 Binary files a/docs/_static/image/dpa.png and /dev/null differ diff --git a/docs/_static/image/logo.ico b/docs/_static/image/logo.ico deleted file mode 100644 index 7a0b5b2f0..000000000 Binary files a/docs/_static/image/logo.ico and /dev/null differ diff --git a/docs/_static/image/logo.png b/docs/_static/image/logo.png deleted file mode 100644 index 2a8bc258f..000000000 Binary files a/docs/_static/image/logo.png and /dev/null differ diff --git a/docs/_static/js/deprecation_banner.js b/docs/_static/js/deprecation_banner.js deleted file mode 100644 index 87c8d73fa..000000000 --- a/docs/_static/js/deprecation_banner.js +++ /dev/null @@ -1,49 +0,0 @@ -(function () { - "use strict"; - - var oldOrigin = "https://sgl-project.github.io"; - var newOrigin = "https://docs.sglang.io"; - - function buildNewDocsUrl() { - var href = window.location.href; - - if (href === oldOrigin || href.indexOf(oldOrigin + "/") === 0) { - return href.replace(oldOrigin, newOrigin); - } - - return newOrigin + window.location.pathname + window.location.search + window.location.hash; - } - - function addDeprecationBanner() { - if (document.getElementById("sglang-docs-deprecation-banner")) { - return; - } - - var link = document.createElement("a"); - link.href = buildNewDocsUrl(); - link.textContent = link.href; - - var banner = document.createElement("div"); - banner.id = "sglang-docs-deprecation-banner"; - banner.className = "sglang-docs-deprecation-banner"; - banner.setAttribute("role", "status"); - banner.setAttribute("aria-live", "polite"); - - var prefix = document.createTextNode( - "This legacy documentation site will be deprecated soon. Please use the new SGLang documentation at " - ); - var suffix = document.createTextNode("."); - - banner.appendChild(prefix); - banner.appendChild(link); - banner.appendChild(suffix); - - document.body.insertBefore(banner, document.body.firstChild); - } - - if (document.readyState === "loading") { - document.addEventListener("DOMContentLoaded", addDeprecationBanner); - } else { - addDeprecationBanner(); - } -})(); diff --git a/docs/advanced_features/adaptive_speculative_decoding.md b/docs/advanced_features/adaptive_speculative_decoding.md deleted file mode 100644 index 64a31f3d8..000000000 --- a/docs/advanced_features/adaptive_speculative_decoding.md +++ /dev/null @@ -1,156 +0,0 @@ -# Adaptive Speculative Decoding - -Adaptive speculative decoding lets SGLang adjust `speculative_num_steps/speculative_num_draft_tokens` at runtime instead of keeping a single fixed value for the whole server lifetime. -It is designed for workloads whose accept length changes over time, where one static step count is rarely optimal. - -## Current support - -- Only `--speculative-algorithm EAGLE` -- Only `--speculative-eagle-topk 1` -- If either condition is not met, SGLang falls back to static speculative settings - -## Why adaptive steps help - -`speculative_num_steps` controls how many draft-model autoregressive steps run in each speculative round. In practice, the best value depends on the current workload. - -- If `num_steps` is too small, the draft model could have produced more accepted tokens, but the round stops too early. -- If `num_steps` is too large, the draft model produces many candidate tokens that the target model rejects, so extra draft work is wasted. - -Real traffic often moves between high-acceptance and low-acceptance phases, so one fixed step count is usually a compromise. Adaptive mode tries to follow the workload instead of hard-coding a single global `num_steps`. - -## Design overview - -The adaptive mechanism has three pieces: - -- `AdaptiveSpeculativeParams`: the EMA-based policy -- `SpecRuntimeState`: the per-tier runtime state bundle -- `AdaptiveController`: the coordinator that chooses a tier and activates the matching runtime state - -At startup, SGLang pre-builds one runtime state per candidate tier. By default, the candidate tiers are `candidate_steps = [1, 3, 7]`. - -```text -┌──────────────────────────────────────────────────────────┐ -│ SpecRuntimeState │ -│ │ -│ speculative_num_steps / speculative_num_draft_tokens │ -│ │ -│ ┌────────────────┐ ┌────────────────┐ ┌──────────────┐ │ -│ │ Draft stage │ │ Verify stage │ │ Extend stage │ │ -│ │ │ │ │ │ │ │ -│ │ attn_backend │ │ attn_backend │ │ attn_backend │ │ -│ │ cuda_graph │ │ cuda_graph │ │ cuda_graph │ │ -│ └────────────────┘ └────────────────┘ └──────────────┘ │ -└──────────────────────────────────────────────────────────┘ -``` - -This matters because `CudaGraphRunner` is shape-dependent. Each candidate tier owns its own graph and backend state, so runtime switching is a reference swap, not an online graph recapture. - -## Runtime flow - -The adaptive update happens after verify and affects the next round, not the current one: - -```text -┌─────────────────────────────────────────────────────────────────────┐ -│ EAGLEWorker.forward_batch_generation() — decode path │ -│ │ -│ ① draft(batch) │ -│ │ draft model multi-step generation with current tier │ -│ v │ -│ ② verify(batch, spec_info) │ -│ │ target model tree verification │ -│ │ → produces accept_length_per_req │ -│ v │ -│ ③ forward_draft_extend_after_decode(batch) │ -│ │ draft model KV-cache catch-up │ -│ v │ -│ ④ adaptive_controller.on_verify_complete(accept_lengths) │ -│ │ │ -│ │ update EMA, apply warmup / interval / hysteresis gates │ -│ │ if tier changed, select a pre-built state from pool │ -│ v │ -│ worker.apply_runtime_state(state) │ -│ │ -│ Tier switch happens after the current round completes. │ -│ Backends and CUDA graphs are never swapped mid-round. │ -└─────────────────────────────────────────────────────────────────────┘ -``` - -## How the policy decides - -After each verify pass, SGLang reads the accepted draft length per request, computes the batch average, smooths it with an exponential moving average (EMA), and switches among the pre-built candidate tiers `[1, 3, 7]` by default. - -The decision logic is intentionally conservative: - -- `warmup_batches` skips the first few batches -- `update_interval` avoids switching every batch -- `down_hysteresis` and `up_hysteresis` reduce oscillation - -Conceptually, the policy probes one step beyond the observed acceptance: - -```text -target_steps ≈ clamp(round(ema_accept_len) + 1, min(candidate_steps), max(candidate_steps)) -``` - -So if recent requests consistently accept more drafted tokens, the policy tends to move up. If they start rejecting earlier, it tends to move down. - -## Usage - -`--speculative-adaptive-config` is optional, but the speculative setup still needs to be valid for adaptive mode. - -```bash -python3 -m sglang.launch_server \ - --model meta-llama/Llama-2-7b-chat-hf \ - --speculative-algorithm EAGLE \ - --speculative-draft-model-path lmsys/sglang-EAGLE-llama2-chat-7B \ - --speculative-eagle-topk 1 \ - --speculative-num-steps 3 \ - --speculative-num-draft-tokens 4 \ - --speculative-adaptive -``` - -If you want to override the defaults, add `--speculative-adaptive-config /path/to/adaptive_spec.json`. - -Example config: - -```json -{ - "candidate_steps": [1, 3, 7], - "ema_alpha": 0.2, - "warmup_batches": 10, - "update_interval": 5 -} -``` - -## Config file reference - -The config file is optional. Any omitted keys use defaults. - -| Key | Default | Meaning | -|---|---|---| -| `candidate_steps` | `[1, 3, 7]` | Discrete `speculative_num_steps` tiers that adaptive mode can switch between | -| `ema_alpha` | `0.2` | EMA smoothing factor for accepted draft length | -| `update_interval` | `5` | Recompute interval, in verify batches, after warmup | -| `warmup_batches` | `10` | Number of verify batches to observe before switching | -| `down_hysteresis` | `-0.25` | Extra margin before moving to a smaller step | -| `up_hysteresis` | `0.0` | Extra margin before moving to a larger step | - -The initial `--speculative-num-steps` is snapped to the nearest value in `candidate_steps`. - -## Monitoring - -You can inspect the active tier and acceptance metric via `/server_info`: - -```bash -curl -s http://127.0.0.1:30000/server_info | jq '.internal_states[0] | {speculative_num_steps, avg_spec_accept_length}' -``` - -- `speculative_num_steps` is the current active tier -- `avg_spec_accept_length` helps explain whether the server is likely to move up or down - -## Tuning tips - -- Start with the default candidate tiers `[1, 3, 7]` -- Use fewer tiers if you want lower startup and graph-memory overhead -- Increase `ema_alpha` to react faster, or lower it for more stability -- Increase `warmup_batches` or `update_interval` if tier switching is too noisy -- If your workload is already stable and one static setting is well tuned, adaptive mode may not help much diff --git a/docs/advanced_features/attention_backend.md b/docs/advanced_features/attention_backend.md deleted file mode 100644 index b6b604c28..000000000 --- a/docs/advanced_features/attention_backend.md +++ /dev/null @@ -1,372 +0,0 @@ -# Attention Backend - -SGLang supports a large variety of attention backends. Each of them has different pros and cons. -You can test them according to your needs. - -```{important} -Selecting an optimal attention backend is crucial for maximizing your performance. Different backends excel in various scenarios, so choose based on your model, hardware, and use case. Not all backends are supported on all platforms and model architectures. - -If you don't specify `--attention-backend`, SGLang makes a best effort to automatically select the most performant backend based on your hardware and model architecture. -``` - -## Support Matrix - -The support matrix is split into two parts: MHA (standard attention) and MLA (multi-head latent attention). For an explanation of the key differences between MHA and MLA, please see the [SGLang documentation on DeepSeek MLA](../basic_usage/deepseek_v3.md#multi-head-latent-attention-mla-throughput-optimizations) and the original [DeepSeek MLA paper](https://arxiv.org/pdf/2405.04434). - -### MHA Backends - -| **Backend** | **Page Size > 1 (native)** | **FP8 KV Cache** | **FP4 KV Cache** | **Spec topk=1** | **Spec topk>1** | **Sliding Window** | **MultiModal** | -|---------------------------------|-----------------------------|------------------|-----------------|-----------------|-----------------|--------------------|----------------| -| **FlashInfer** | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | -| **FA3 (FlashAttention 3)** | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | -| **FA4 (FlashAttention 4)** | 128 | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | -| **Triton** | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| **Torch Native (SDPA)** | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ✅ | -| **FlexAttention (PyTorch)** | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | -| **TRTLLM MHA** | 16, 32 or 64 | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | -| **Dual Chunk FlashAttention** | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| **AITER (ROCm)** | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | -| **Wave (ROCm)** | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| **Ascend (NPU)** | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | -| **Intel XPU** | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | -| **Intel AMX (CPU)** | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | - -### MLA Backends - -| **Backend** | **Native Page Sizes** | **FP8 KV Cache** | **FP4 KV Cache** | **Chunked Prefix Cache** | **Spec topk=1** | **Spec topk>1** | -|----------------------------|---------------------------|------------------|------------------|--------------------------|-----------------|-----------------| -| **FlashInfer MLA** | 1 | ❌ | ✅ | ✅ | ✅ | ❌ | -| **FlashMLA** | 64 | ✅ | ✅ | ✅ | ✅ | ❌ | -| **Cutlass MLA** | 128 | ✅ | ✅ | ✅ | ✅ | ❌ | -| **TRTLLM MLA (Blackwell)** | 32 or 64 | ✅ | ✅ | ✅ | ✅ | ❌ | -| **FA3 (FlashAttention 3)** | n/a | ❌ | ❌ | ✅ | ✅ | ⚠️ (page_size=1 only) | -| **Triton** | n/a | ❌ | ❌ | ❌ | ✅ | ⚠️ (page_size=1 only) | -| **FA4** | 1 | ❌ | ✅ | ✅ | ❌ | ❌ | -| **Ascend MLA (NPU)** | 128 | ❌ | ❌ | ❌ | ❌ | ❌ | - -```{note} -Multimodal attention is selected by `--mm-attention-backend`. The "MultiModal" column indicates whether a corresponding multimodal implementation exists for that backend family. -``` - -```{note} -- FlashAttention 4 supports both prefill and decode on SM90 (Hopper) and SM100 (Blackwell). FA4 MLA supports `page_size = 1`; FA4 MHA requires `page_size = 128`. On SM100, this is auto-enforced by the server; on SM90, users must set `--page-size 128` manually. -- DSA is specifically designed for [DeepSeek V3.2](https://lmsys.org/blog/2025-09-29-deepseek-V32/). See the [DSA Attention Backend](#dsa-attention-backend) section and [DeepSeek V3.2 deployment guide](../basic_usage/deepseek_v32.md) for details. -``` - -```{warning} -**FA4 on Hopper (SM90):** FA4 decode speed decreases as sequence length grows due to lack of SplitKV support. At batch=1 compared to FA3 on H100: ~-10% at 2K tokens, ~-18% at 4K, ~-31% at 8K, ~-49% at 16K. Larger batch sizes reduce the gap (e.g., batch=8: ~-2% at 2K, ~-8% at 4K). Blackwell (SM100) is not affected. -``` - -```{note} -For the KV4 FA4 scenario, FA4 requires using a different --decode-attention-backend to run. Except for trtllm_mha being incompatible with FA4, all other decode backends behave as shown in the table. -``` - -```{tip} -Speculative decoding topk: `topk` is the number of draft tokens sampled per step from the draft model. `topk = 1` follows classic EAGLE; `topk > 1` explores multiple branches and requires backend support in both draft and verification paths. -``` - -```{note} -**Speculative Decoding V2 (Spec V2):** Spec V2 uses overlap scheduling (`SGLANG_ENABLE_SPEC_V2=True`) that benefits various attention backends. Requires `--speculative-eagle-topk 1` and currently applies to EAGLE and EAGLE3. - -**Verified backends:** TRTLLM MLA, TRTLLM MHA, FA3, Ascend (NPU), Triton. - -**Limited support:** FlashInfer can run under Spec V2, but its plan stream (used for split-KV optimization) introduces a synchronization point that limits overlap benefits. -``` - -```{tip} -Page size controls how many tokens are grouped into a KV cache block. For the prefix cache to take effect, the number of tokens must fill at least one complete page. For example, if your prompt is only 32 tokens and `page_size = 64`, it won't fill a complete page and cannot be matched in the prefix cache (pages cannot be padded). With 65 tokens and `page_size = 64`, only the first page of 64 tokens will be cached and matched; the remaining 1 token is discarded. Use `page_size = 1` for maximum prefix reuse (token-level matching). Note that higher page sizes generally improve attention kernel performance, so prefer `page_size > 1` when prefix cache reuse is not critical. -``` - -Many backends that do not natively operate on pages can emulate `page_size > 1` at the wrapper layer by expanding page tables to per-token indices. The "Page Size > 1 (native)" column indicates true in-kernel paging. Some backends require fixed native page sizes and cannot be reduced/emulated differently: TRTLLM MHA (16/32/64), TRTLLM MLA (32/64), FlashMLA (64), Cutlass MLA (128), Ascend (128). - -MLA page-size constraints: -- FlashInfer MLA: page_size = 1. -- FlashMLA: page_size = 64. -- Cutlass MLA: page_size = 128. -- TRTLLM MLA: page_size ∈ {32, 64}. - -### GDN Attention Backends - -GDN (Gated Delta Network) is a linear attention mechanism with O(n) complexity, used in hybrid models that alternate GDN linear attention layers with standard full attention layers. GDN is **not** selected via `--attention-backend`; it is automatically activated when the model architecture requires it (e.g., Qwen 3.5, Qwen 3 Next, Jet Nemotron, Jet VLM). - -The GDN linear attention layers have their own kernel backends, selected via `--linear-attn-backend` (default: `triton`). You can override the kernel per phase with `--linear-attn-decode-backend` and `--linear-attn-prefill-backend`. - -| **Backend** | **Decode** | **Prefill / Extend** | **Spec Decoding (Target Verify)** | -|--------------------------|------------|----------------------|-----------------------------------| -| **Triton (CUDA)** | ✅ | ✅ | ✅ | -| **Triton (AMD/ROCm)** | ✅ | ✅ | ✅ | -| **Triton (NPU)** | ✅ | ✅ | ❌ | -| **Triton (CPU)** | ✅ | ✅ | ❌ | -| **CuTe DSL (CUDA only)**| ✅ | ❌ | ❌ | - -```{important} -GDN models are hybrid: the full-attention layers still require a standard `--attention-backend`. Platform constraints for the full-attention backend on hybrid GDN models: -- **Blackwell (e.g., B200)**: `triton`, `trtllm_mha`, or `fa4` only. -- **NPU (Ascend)**: `ascend` only. -- **AMD (ROCm)**: `triton` recommended. -- **Other CUDA (Hopper, Ampere, etc.)**: auto-selection works; no special constraints. -``` - -### DSA Attention Backend - -DSA (Deepseek Sparse Attention) is a native sparse attention mechanism used by [DeepSeek V3.2](https://lmsys.org/blog/2025-09-29-deepseek-V32/). It is activated automatically when the model architecture requires it and is selected via `--attention-backend dsa` (the legacy alias `--attention-backend nsa` is deprecated and kept for one release). - -Internally, the DSA backend dispatches to different sub-backends for prefill and decode phases. You can override these with `--dsa-prefill-backend` and `--dsa-decode-backend` (the `--nsa-prefill-backend` / `--nsa-decode-backend` aliases are deprecated): - -| **Sub-backend** | **Prefill** | **Decode** | **Notes** | -|-----------------------|-------------|------------|-----------------------------------------------| -| **flashmla_sparse** | ✅ | ✅ | Default prefill on Hopper and Blackwell (bf16) | -| **flashmla_kv** | ✅ | ✅ | Default decode for FP8 on Blackwell with DP | -| **flashmla_auto** | ✅ | ❌ | Auto-selects flashmla_sparse or flashmla_kv based on kv_cache_dtype | -| **fa3** | ✅ | ✅ | Default decode on Hopper (bf16) | -| **trtllm** | ✅ | ✅ | Default decode on Blackwell (bf16); default for both on Blackwell without DP | -| **tilelang** | ✅ | ✅ | Default on AMD (ROCm) | -| **aiter** | ✅ | ✅ | AMD-specific kernel library (requires aiter package) | - -For deployment examples, see the [DeepSeek V3.2 deployment guide](../basic_usage/deepseek_v32.md). - -### Hybrid attention (different backends for prefill vs decode) (Experimental) - -```{warning} -Hybrid attention is an experimental feature. -``` - -You can mix-and-match attention backends for prefill and decode. This is useful when one backend excels at prefill and another excels at decode. For the implementation details, please see `python/sglang/srt/layers/attention/hybrid_attn_backend.py`. - -```bash -# Example: Prefill with FA4, Decode with TRTLLM MLA (Blackwell) -python3 -m sglang.launch_server \ - --model-path nvidia/DeepSeek-R1-FP4 \ - --tp 8 \ - --attention-backend trtllm_mla \ - --moe-runner-backend flashinfer_trtllm \ - --quantization modelopt_fp4 \ - --prefill-attention-backend fa4 -``` - -#### Speculative decoding with hybrid attention - -Hybrid attention also works with speculative decoding. The backend used for draft decoding and target verification depends on `--speculative-attention-mode`: - -- `--speculative-attention-mode decode` (recommended): draft/verify use the decode backend. -- `--speculative-attention-mode prefill` (default): draft/verify use the prefill backend. - -Constraints when combining hybrid attention with speculative decoding: - -- If any attention backend is `trtllm_mha`, speculative decoding supports only `--speculative-eagle-topk 1`. -- For paged MHA backends with `--page-size > 1` and `--speculative-eagle-topk > 1`, only `flashinfer` is supported. -- CUDA Graph: the decode backend is always captured; the prefill backend is captured only when `--speculative-attention-mode prefill`. - - -```{tip} -If you set only one of `--prefill-attention-backend` or `--decode-attention-backend`, the unspecified phase inherits `--attention-backend`. -If both are specified and differ, SGLang automatically enables a hybrid wrapper to dispatch to the chosen backend per phase. -``` - -## Attention Backend Selection Guide (CUDA) - -If the `--attention-backend` argument is not specified, SGLang automatically selects the best backend based on the hardware (CUDA) and model architecture. - -### Automatic Selection Logic - -**1. MHA Models (e.g., Llama, Qwen)** -- **Hopper (e.g., H100, H200)**: Defaults to `fa3` if using CUDA 12.3+ and the model configuration is supported. -- **Blackwell (e.g., B200)**: Defaults to `trtllm_mha`, unless using speculative decoding with `topk > 1`. -- **Other Architectures (Ampere, Ada, etc.)**: Defaults to `flashinfer` if available; otherwise falls back to `triton`. - -**2. MLA Models (e.g., DeepSeek V3)** -- **Hopper**: Defaults to `fa3` (requires CUDA 12.3+). -- **Blackwell**: Defaults to `flashinfer`; `trtllm_mla` is auto-selected for DeepSeek V3 models specifically. -- **Other Architectures**: Defaults to `triton`. - - -## User Guide - -### Launch Command for Different Attention Backends - -- FlashInfer (Default for Non-Hopper Machines, e.g., A100, A40) -```bash -python3 -m sglang.launch_server \ - --model meta-llama/Meta-Llama-3.1-8B-Instruct \ - --attention-backend flashinfer -python3 -m sglang.launch_server \ - --tp 8 \ - --model deepseek-ai/DeepSeek-V3 \ - --attention-backend flashinfer \ - --trust-remote-code -``` - -- FlashAttention 3 (Default for Hopper Machines, e.g., H100, H200, H20) -```bash -python3 -m sglang.launch_server \ - --model meta-llama/Meta-Llama-3.1-8B-Instruct \ - --attention-backend fa3 -python3 -m sglang.launch_server \ - --tp 8 \ - --model deepseek-ai/DeepSeek-V3 \ - --trust-remote-code \ - --attention-backend fa3 -``` - -- Triton -```bash -python3 -m sglang.launch_server \ - --model meta-llama/Meta-Llama-3.1-8B-Instruct \ - --attention-backend triton -python3 -m sglang.launch_server \ - --tp 8 \ - --model deepseek-ai/DeepSeek-V3 \ - --attention-backend triton \ - --trust-remote-code -``` - -- FlashMLA -```bash -python3 -m sglang.launch_server \ - --tp 8 \ - --model deepseek-ai/DeepSeek-R1 \ - --attention-backend flashmla \ - --trust-remote-code -python3 -m sglang.launch_server \ - --tp 8 \ - --model deepseek-ai/DeepSeek-R1 \ - --attention-backend flashmla \ - --kv-cache-dtype fp8_e4m3 \ - --trust-remote-code -``` - -- TRTLLM MLA (Optimized for Blackwell Architecture, e.g., B200) -```bash -python3 -m sglang.launch_server \ - --tp 8 \ - --model deepseek-ai/DeepSeek-R1 \ - --attention-backend trtllm_mla \ - --trust-remote-code -``` - -- TRTLLM MLA with FP8 KV Cache (Higher concurrency, lower memory footprint) -```bash -python3 -m sglang.launch_server \ - --tp 8 \ - --model deepseek-ai/DeepSeek-R1 \ - --attention-backend trtllm_mla \ - --kv-cache-dtype fp8_e4m3 \ - --trust-remote-code -``` - -- TRTLLM MHA (Optimized for Blackwell Architecture, e.g., B200) -```bash -python3 -m sglang.launch_server \ - --tp 4 \ - --model Qwen/Qwen3.5-35B-A3B-FP8 \ - --attention-backend trtllm_mha \ - --trust-remote-code -``` - -- TRTLLM MHA (XQA backend) (Optimized for SM90 and SM120, e.g., H20, H200, 5090) - Note that TRTLLM XQA backend only works well for pagesize 64. -```bash -python3 -m sglang.launch_server \ - --tp 4 \ - --model Qwen/Qwen3.5-35B-A3B-FP8 \ - --decode-attention-backend trtllm_mha \ - --trust-remote-code -``` - -- FlashAttention 4 (MHA & MLA) -```bash -# FA4 for both prefill and decode on SM90/SM100 -python3 -m sglang.launch_server \ - --model-path Qwen/Qwen3-30B-A3B-Instruct-2507-FP8 \ - --attention-backend fa4 \ - --page-size 128 \ - --trust-remote-code - -python3 -m sglang.launch_server \ - --tp 8 \ - --model deepseek-ai/DeepSeek-R1 \ - --prefill-attention-backend fa4 \ - --trust-remote-code -``` - -- Cutlass MLA -```bash -python3 -m sglang.launch_server \ - --tp 8 \ - --model deepseek-ai/DeepSeek-R1 \ - --attention-backend cutlass_mla \ - --trust-remote-code -``` - -- Ascend -```bash -python3 -m sglang.launch_server \ - --model meta-llama/Meta-Llama-3.1-8B-Instruct \ - --attention-backend ascend -``` - -- Intel XPU -```bash -python3 -m sglang.launch_server \ - --model meta-llama/Meta-Llama-3.1-8B-Instruct \ - --attention-backend intel_xpu -``` - -- Wave -```bash -python3 -m sglang.launch_server \ - --model meta-llama/Meta-Llama-3.1-8B-Instruct \ - --attention-backend wave -``` - -- FlexAttention -```bash -python3 -m sglang.launch_server \ - --model meta-llama/Meta-Llama-3.1-8B-Instruct \ - --attention-backend flex_attention -``` - -- Dual Chunk FlashAttention -```bash -python3 -m sglang.launch_server \ - --model Qwen/Qwen2.5-14B-Instruct-1M \ - --attention-backend dual_chunk_flash_attn -``` - -- Torch Native -```bash -python3 -m sglang.launch_server \ - --model meta-llama/Meta-Llama-3.1-8B-Instruct \ - --attention-backend torch_native -``` - -## Steps to add a new attention backend -To add a new attention backend, you can learn from the existing backends -(`python/sglang/srt/layers/attention/triton_backend.py`, `python/sglang/srt/layers/attention/flashattention_backend.py`) -and follow the steps below. - -```{note} -Linear attention kernel backends (GDN, KDA) follow a different pattern. They implement `LinearAttnKernelBase` in `python/sglang/srt/layers/attention/linear/kernels/` and are dispatched by `GDNKernelDispatcher` / `KDAKernelDispatcher` rather than registered via `@register_attention_backend`. -``` - -1. Run without cuda graph. Support the two forward functions -- forward_extend - - Will be used for prefill, prefill with KV cache, and target verification - - It will be called once per layer -- forward_decode - - Will be used for normal decode, and draft decode - - It will be called once per layer -- init_forward_metadata - - Initialize the class and common metadata shared by all layers - - Call the plan function for optimizations like split_kv - - It will be called once per forward -2. Run with cuda graph. It has two phases (capture and replay) and you need to implement three functions -- init_cuda_graph_state - - It will be called once during life time - - Create all common shared buffers -- init_forward_metadata_capture_cuda_graph - - It will be called before capturing a cuda graph - - It is similar to init_forward_metadata but write the medatada to some pre-defined buffers -- init_forward_metadata_replay_cuda_graph - - It will be called before replaying a cuda graph - - This function is in the critical path and needs to be fast diff --git a/docs/advanced_features/breakable_cuda_graph.md b/docs/advanced_features/breakable_cuda_graph.md deleted file mode 100644 index 4fb2c090c..000000000 --- a/docs/advanced_features/breakable_cuda_graph.md +++ /dev/null @@ -1,139 +0,0 @@ -# Breakable CUDA Graph - -## Motivation - -Standard CUDA graphs capture an entire forward pass as a single, opaque graph. This is great for performance, but creates two problems: - -1. **Debugging is hard.** When something goes wrong inside a captured graph (wrong outputs, numerical mismatches, crashes), there is no way to step through the operations or insert print statements because the graph replays as a monolithic unit. - -2. **Some ops are incompatible.** Certain operations — dynamic control flow, host-device synchronization, JIT compilation, or ops that change behavior across iterations — cannot be captured into a CUDA graph at all. Today, the only workaround is to disable CUDA graphs entirely, which sacrifices the kernel launch overhead savings for the rest of the model. - -**Breakable CUDA Graph** solves both problems by allowing graph breaks to be inserted at specific points. The computation is split into multiple captured graph segments with eager (non-graph) execution in between. This preserves most of the CUDA graph performance benefit while allowing targeted operations to run outside the graph. - -## Usage - -### Debug Mode: Run Everything Eagerly - -The simplest use case is debugging. The `--debug-cuda-graph` flag wraps the entire decode forward pass in a graph break, so every operation runs eagerly while still going through the full CUDA graph capture/replay code path. This lets you debug CUDA graph issues without changing model code. - -```bash -python -m sglang.launch_server \ - --model meta-llama/Llama-3.1-8B-Instruct \ - --debug-cuda-graph -``` - -This mode is intended for debugging only — it eliminates the performance benefit of CUDA graphs since every op runs eagerly. - -### Selective Graph Breaks in Model Code - -For production use, you can mark specific functions as "non-graphable" using the `@eager_on_graph` decorator. During CUDA graph capture, these functions run eagerly between captured graph segments. Outside of capture, they behave normally. - -```python -from sglang.srt.model_executor.breakable_cuda_graph.breakable_cuda_graph import eager_on_graph - -@eager_on_graph(enable=True) -def my_dynamic_op(x): - # This op is incompatible with CUDA graph capture - return some_dynamic_operation(x) -``` - -You can also insert a bare graph break (no computation) using the `break_graph()` helper: - -```python -from sglang.srt.model_executor.breakable_cuda_graph.breakable_cuda_graph import break_graph - -def forward(self, x): - x = self.layer1(x) - break_graph() # force a segment split here - x = self.layer2(x) - return x -``` - -To enable breakable CUDA graph at the environment level (without debug mode), set the environment variable: - -```bash -export SGLANG_USE_BREAKABLE_CUDA_GRAPH=1 -python -m sglang.launch_server \ - --model meta-llama/Llama-3.1-8B-Instruct -``` - -### Server Args - -| Argument | Default | Description | -|---|---|---| -| `--debug-cuda-graph` | `False` | Enable debug/eager mode. Wraps the entire forward pass in a graph break so every op runs eagerly through the capture/replay path. | -| `SGLANG_USE_BREAKABLE_CUDA_GRAPH` | `0` | Environment variable. Enables breakable CUDA graph without debug mode. Required for `@eager_on_graph` decorators to take effect. | - -## How It Works - -### Capture - -Breakable CUDA graph extends PyTorch's `torch.cuda.CUDAGraph` by splitting a single capture into multiple segments separated by graph breaks. - -During capture, the flow is: - -``` -Begin capture (segment 1) - ... graphable ops ... - @eager_on_graph function encountered: - 1. End current capture segment - 2. Run the function eagerly (allocates output tensors) - 3. Record the function for later replay - 4. Begin new capture segment - ... more graphable ops ... -End capture (segment N) -``` - -Each segment is independently instantiated as a CUDA graph executable. The non-graph functions and their argument references are stored for replay. - -### Replay - -During replay: - -``` -For each segment i: - 1. Launch CUDA graph segment i - 2. Run the recorded non-graph function i eagerly -Launch final CUDA graph segment -``` - -The non-graph functions are re-invoked with the same tensor references as capture time. Since these references point to the CUDA graph's static input/output buffers, they see updated values on each replay. - -### Output Writeback - -When a non-graph function produces output during replay, the result must be written back into the same tensor buffers that downstream graph segments reference. The mechanism handles: - -- **Plain tensors**: In-place `copy_()` into the original buffer. -- **Structured outputs** (dataclasses, objects with tensor attributes): Tensor fields are copied in-place; non-tensor fields are replaced. -- **Dicts of tensors**: Tensor values are copied in-place; non-tensor values are replaced. - -### Stream Fork/Join Tracking - -Some models fork work onto secondary CUDA streams (e.g., for overlapped computation). Breakable CUDA graph hooks `torch.cuda.Stream.wait_stream` to track which streams are forked from the capture stream. When a graph break occurs, all forked streams are automatically joined back before ending the capture segment, and re-forked after beginning the next segment. - -## Compatibility - -- **NVIDIA CUDA only.** Breakable CUDA graph is not supported on ROCm/HIP or other non-CUDA platforms. On unsupported platforms, `--debug-cuda-graph` is automatically disabled with a warning. -- **Requires `cuda-python`.** The `cuda.bindings` package must be installed (`pip install cuda-python`). -- **Not compatible with memory saver mode.** Cannot be used together with `SGLANG_MEMORY_SAVER_CUDA_GRAPH`. - -## Performance - -When no graph breaks are inserted, breakable CUDA graph has minimal overhead compared to standard CUDA graph — the capture/replay path is nearly identical. - -Each graph break adds: -- One `cudaGraphLaunch` call (to replay the segment before the break) -- One eager Python function call -- One `cudaStreamBeginCapture` / `cudaStreamEndCapture` pair during capture - -For typical use cases with a small number of graph breaks, the overhead is negligible compared to the saved kernel launch overhead from the captured segments. - -## Code Reference - -| File | Description | -|---|---| -| `python/sglang/srt/model_executor/breakable_cuda_graph/breakable_cuda_graph.py` | Core implementation: `eager_on_graph`, `BreakableCUDAGraph`, `BreakableCUDAGraphCapture` | -| `python/sglang/srt/model_executor/breakable_cuda_graph/cuda_utils.py` | CUDA runtime binding utilities | -| `python/sglang/srt/model_executor/cuda_graph_runner.py` | Integration with the main CUDA graph runner | -| `python/sglang/srt/server_args.py` | `--debug-cuda-graph` flag and environment variable handling | -| `python/sglang/srt/environ.py` | `SGLANG_USE_BREAKABLE_CUDA_GRAPH` environment variable definition | diff --git a/docs/advanced_features/checkpoint_engine.md b/docs/advanced_features/checkpoint_engine.md deleted file mode 100644 index 5e39a7ee2..000000000 --- a/docs/advanced_features/checkpoint_engine.md +++ /dev/null @@ -1,254 +0,0 @@ -# Checkpoint Engine Integration - -The SGLang checkpoint engine integration provides an efficient way to load model weights using a distributed checkpoint loading system. This feature significantly reduces model loading time, especially for large models and multi-node setups, by parallelizing the weight loading process across multiple processes and nodes. - -## Overview - -The checkpoint engine integration allows SGLang to: -- Load model weights in parallel using multiple processes -- Distribute weight loading across multiple nodes to increase effective disk bandwidth -- Overlap weight loading with other initialization tasks like CUDA graph capture -- Support both single-node and multi-node deployments - -## Installation - -First, install the checkpoint engine package: - -```bash -pip install 'checkpoint-engine[p2p]' -``` - -## Architecture - -The system consists of two main components: - -1. **SGLang Server**: Runs with `--wait-for-initial-weights` flag to wait for weights before becoming ready -2. **Checkpoint Engine Workers**: Separate processes (managed by torchrun) that load and distribute model weights - -The checkpoint engine uses a parameter server architecture with support for: -- **Broadcast mode**: Weights are broadcast from loading processes to inference processes -- **P2P mode**: Direct peer-to-peer weight transfer between processes -- **All mode**: Combination of both broadcast and P2P methods - -## Usage Examples - -### Single Node Setup - -**Terminal 1 - Launch SGLang Server:** -```bash -python -m sglang.launch_server \ - --model-path Qwen/Qwen3-8B \ - --tp 8 \ - --load-format dummy \ - --wait-for-initial-weights -``` - -**Terminal 2 - Run Checkpoint Engine:** - -Using sglang entrypoint: -```bash -python -m sglang.srt.checkpoint_engine.update \ - --update-method broadcast \ - --checkpoint-path /path/to/Qwen/Qwen3-8B/ \ - --inference-parallel-size 8 -``` - -Using torchrun directly: -```bash -torchrun --nproc-per-node 8 \ - examples/checkpoint_engine/update.py \ - --update-method broadcast \ - --checkpoint-path /path/to/Qwen/Qwen3-8B/ \ - --inference-parallel-size 8 -``` - -### Multi-Node Setup (2 Nodes) - -**Node 0:** - -Launch SGLang server: -```bash -python -m sglang.launch_server \ - --model-path Qwen/Qwen3-8B \ - --tp 8 \ - --load-format dummy \ - --wait-for-initial-weights \ - --host [IP] -``` - -Run checkpoint engine: - -Using sglang entrypoint (recommended): -```bash -python -m sglang.srt.checkpoint_engine.update \ - --update-method broadcast \ - --checkpoint-path /path/to/Qwen/Qwen3-8B/ \ - --inference-parallel-size 8 -``` - -Using torchrun directly: -```bash -torchrun --nproc-per-node 8 \ - --nnodes 2 \ - --node-rank 0 \ - --master-addr [IP] \ - --master-port 29500 \ - examples/checkpoint_engine/update.py \ - --update-method broadcast \ - --checkpoint-path /path/to/Qwen/Qwen3-8B/ \ - --inference-parallel-size 8 -``` - -**Node 1:** - -Launch SGLang server: -```bash -python -m sglang.launch_server \ - --model-path Qwen/Qwen3-8B \ - --tp 8 \ - --load-format dummy \ - --wait-for-initial-weights \ - --host [IP] -``` - -Run checkpoint engine: - -Using sglang entrypoint (recommended): -```bash -python -m sglang.srt.checkpoint_engine.update \ - --update-method broadcast \ - --checkpoint-path /path/to/Qwen/Qwen3-8B/ \ - --inference-parallel-size 8 -``` - -Using torchrun directly: -```bash -torchrun --nproc-per-node 8 \ - --nnodes 2 \ - --node-rank 1 \ - --master-addr [IP] \ - --master-port 29500 \ - examples/checkpoint_engine/update.py \ - --update-method broadcast \ - --checkpoint-path /path/to/Qwen/Qwen3-8B/ \ - --inference-parallel-size 8 -``` - -### Multi-Node Setup with Tensor Parallelism (TP=16) - -**Node 0:** - -Launch SGLang server: -```bash -python -m sglang.launch_server \ - --model-path Qwen/Qwen3-8B \ - --tp 8 \ - --load-format dummy \ - --wait-for-initial-weights \ - --host [IP] \ - --dist-init-addr [IP]:9120 \ - --nnodes 2 \ - --node-rank 0 -``` - -Run checkpoint engine: - -Using sglang entrypoint (recommended): -```bash -python -m sglang.srt.checkpoint_engine.update \ - --update-method broadcast \ - --checkpoint-path /path/to/Qwen/Qwen3-8B/ \ - --inference-parallel-size 16 -``` - -Using torchrun directly: -```bash -torchrun --nproc-per-node 8 \ - --nnodes 2 \ - --node-rank 0 \ - --master-addr [IP] \ - --master-port 29500 \ - examples/checkpoint_engine/update.py \ - --update-method broadcast \ - --checkpoint-path /path/to/Qwen/Qwen3-8B/ \ - --inference-parallel-size 16 -``` - -**Node 1:** - -Launch SGLang server: -```bash -python -m sglang.launch_server \ - --model-path Qwen/Qwen3-8B \ - --tp 8 \ - --load-format dummy \ - --wait-for-initial-weights \ - --host [IP] \ - --dist-init-addr [IP]:9120 \ - --nnodes 2 \ - --node-rank 1 -``` - -Run checkpoint engine: - -Using sglang entrypoint (recommended): -```bash -python -m sglang.srt.checkpoint_engine.update \ - --update-method broadcast \ - --checkpoint-path /path/to/Qwen/Qwen3-8B/ \ - --inference-parallel-size 16 -``` - -Using torchrun directly: -```bash -torchrun --nproc-per-node 8 \ - --nnodes 2 \ - --node-rank 1 \ - --master-addr [IP] \ - --master-port 29500 \ - examples/checkpoint_engine/update.py \ - --update-method broadcast \ - --checkpoint-path /path/to/Qwen/Qwen3-8B/ \ - --inference-parallel-size 16 -``` - -## Configuration Options - -### SGLang Server Options - -- `--load-format dummy`: Use dummy format for initial loading (allows overlapping with other tasks) -- `--wait-for-initial-weights`: Wait for checkpoint engine to provide weights before becoming ready -- `--host`: Host address for multi-node setups -- `--dist-init-addr`: Distributed initialization address for tensor parallelism - -### Checkpoint Engine Options - -- `--update-method`: Weight update method (`broadcast`, `p2p`, or `all`) -- `--checkpoint-path`: Path to model checkpoint directory -- `--inference-parallel-size`: Number of inference parallel processes -- `--endpoint`: SGLang server endpoint (default: `http://localhost:19730`) -- `--checkpoint-name`: Name for the checkpoint (default: `my-checkpoint-iter-0`) -- `--save-metas-file`: File to save checkpoint metadata -- `--load-metas-file`: File to load checkpoint metadata from -- `--uds`: Unix domain socket path for communication -- `--weight-version`: Version identifier for weights - -## Performance Benefits - -The checkpoint engine provides significant time savings in two main aspects: - -1. **Multi-node Loading**: Each node only loads a portion of weights from disk, effectively increasing disk bandwidth. More participating nodes provide greater acceleration. Preliminary tests show 20-second acceleration when loading DeepSeek-R1 on H20-3e with two nodes. - -2. **Single Process Optimization**: Using dummy format allows overlapping disk-to-CPU transfer with CUDA graph capture and other initialization tasks, providing additional time savings. - -## Troubleshooting - -- Ensure checkpoint engine package is installed: `pip install 'checkpoint-engine[p2p]'` -- Verify network connectivity between nodes in multi-node setups -- Check that the checkpoint path contains valid model files -- Monitor logs for connection errors between SGLang server and checkpoint engine -- Use `--sleep-time` parameter to add delays if needed for debugging - -## References - -- [Checkpoint Engine Repository](https://github.com/MoonshotAI/checkpoint-engine) diff --git a/docs/advanced_features/cuda_graph_for_multi_modal_encoder.md b/docs/advanced_features/cuda_graph_for_multi_modal_encoder.md deleted file mode 100644 index 184995ff1..000000000 --- a/docs/advanced_features/cuda_graph_for_multi_modal_encoder.md +++ /dev/null @@ -1,73 +0,0 @@ -# Cuda Graph for Multi-Modal Encoder in SGLang - -## Motivation - -In multimodal reasoning services, the visual encoder (ViT / Vision Transformer) typically has a few characteristic traits: - -Many layers, fragmented operators: Each layer includes LN, QKV projections, attention, MLP, residual connections, etc., resulting in extremely frequent kernel launches. - -Server-side “small batch / low latency” is common: The batch size is very small (sometimes it looks like 1 after “flattening” the batch), so kernel launch overhead accounts for a large portion of end-to-end latency. - -Input token count (number of patches) varies frequently: Different image/video resolutions and different batch composition lead to different sequence lengths -S — and this is precisely the biggest obstacle for CUDA Graph (unstable shapes). - -The value of CUDA Graph: It captures a long sequence of GPU kernels with fixed shapes and fixed memory addresses into a graph; later, for the same shapes, it can replay the graph directly, dramatically reducing launch overhead and making GPU scheduling more compact. - -This led us to seek a CUDA Graph enabled feature for ViT in order to improve ViT performance. - -## Design and Restrictions - -The new CUDA Graph enabled ViT logic is built on ViTCudaGraphRunner. This runner captures the "blocks + merger + deepstack merger (optional)" part of a vision transformer into a CUDA graph and replays it for identical shapes. See the following design consideration and restrictions for more details. - -### Dynamic inputs to fit static constraints of CUDA Graph - -Variable sequence length S is very common in ViT. While CUDA Graph requires fixed shapes. The solution is to build a graph cache by S(e.g., graph_key = S). The first time create a new S, and then capture a graph; afterwards, replay it. - -If there are many distinct S values, we need to increase VRAM usage which is graph-private memory pools for many graphs. - -### Stable addresses - -Everything "parameter-like" becomes a static buffer: - -- block_input / block_ws / block_output -- cu_full_len / cu_window_len and their kk variants -- sin_cos_ws - -In this way to solve the underlying requirement: during replay, not allowed to swap tensors, can only modify tensor contents. - -### Attention backend arguments -Attention backend arguments are fixed inside the graph: - -TritonAttn expects [cu_seqlens, cu_seqlens_kk, max_len] -FA3 expects [cu_seqlens, max_len] - -max_len is frozen as an int constant. -cu_seqlens is cached into a dict during create_graph(), and its contents are not updated during subsequent replays. - -For the same graph_key = S, you not only require the input shape to match, but also require the segmentation pattern in cu_seqlens (and window seqlens) to be identical. Otherwise, attention will segment the sequence incorrectly. - -### Rotary buffer management -The feature reallocates a larger sin_cos_ws when seq_len increases. -The max_content_len is used to make sure the maximum size of the allocated rotary buffer. - - -## Command Example -You can enable CUDA Graph for ViT by setting env variable `SGLANG_VIT_ENABLE_CUDA_GRAPH=1`, for example: -``` -SGLANG_VIT_ENABLE_CUDA_GRAPH=1 \ -python3 -m sglang.launch_server \ - --model Qwen/Qwen3-VL-8B-Instruct -``` -Or you can run CUDA Graph for ViT together with Piecewise CUDA Graph feature by both setting env variable `SGLANG_VIT_ENABLE_CUDA_GRAPH=1` and setting `--enable-piecewise-cuda-graph`, for example: -``` -SGLANG_VIT_ENABLE_CUDA_GRAPH=1 \ -python3 -m sglang.launch_server \ - --model Qwen/Qwen3-VL-8B-Instruct \ - --piecewise-cuda-graph-max-tokens 4096 \ - --enable-piecewise-cuda-graph \ - --piecewise-cuda-graph-compiler eager -``` - -## Known supported models -- Qwen2.5-VL (https://github.com/sgl-project/sglang/pull/14422) -- Qwen3-VL (https://github.com/sgl-project/sglang/pull/15320) diff --git a/docs/advanced_features/deterministic_inference.md b/docs/advanced_features/deterministic_inference.md deleted file mode 100644 index b5b6b5216..000000000 --- a/docs/advanced_features/deterministic_inference.md +++ /dev/null @@ -1,154 +0,0 @@ -# Deterministic Inference - -## Why Deterministic Inference Matters - -Deterministic inference ensures consistent LLM outputs across runs, which is critical for: -- **Reinforcement Learning**: Ensures consistent logprobs across runs, reducing stochastic noise and making RL training more stable, reproducible, and debuggable. -- **Testing & Debugging**: Enables reproducible validation -- **Production**: Improves reliability and user experience - -Even with `temperature=0`, standard LLM inference can produce different outputs due to dynamic batching and varying reduction orders in GPU kernels. - -## The Root Cause of Non-Determinism - -The main source is **varying batch sizes**. Different batch sizes cause GPU kernels to split reduction operations differently, leading to different addition orders. Due to floating-point non-associativity (`(a + b) + c ≠ a + (b + c)`), this produces different results even for identical inputs. - - -## SGLang's Solution - -Building on [Thinking Machines Lab's batch-invariant operators](https://github.com/thinking-machines-lab/batch_invariant_ops), SGLang achieves fully deterministic inference while maintaining compatibility with chunked prefill, CUDA graphs, radix cache, and non-greedy sampling. The development roadmap for deterministic inference features can be found in this [issue](https://github.com/sgl-project/sglang/issues/10278). - -### Supported Backends - -Deterministic inference is only supported with the following three attention backends: **FlashInfer**, **FlashAttention 3 (FA3)**, and **Triton**. - -The following table shows feature compatibility for deterministic inference across different attention backends: - -| Attention Backend | CUDA Graph | Chunked Prefill | Radix Cache | Non-greedy Sampling (Temp > 0) | -|-------------------|------------|-----------------|-------------|---------------------| -| **FlashInfer** | ✅ Yes | ✅ Yes | ❌ No | ✅ Yes | -| **FlashAttention 3 (FA3)** | ✅ Yes | ✅ Yes | ✅ Yes | ✅ Yes | -| **Triton** | ✅ Yes | ✅ Yes | ✅ Yes | ✅ Yes | - -## Usage - -### Basic Usage - -Enable deterministic inference by adding the `--enable-deterministic-inference` flag: - -```bash -python3 -m sglang.launch_server \ - --model-path Qwen/Qwen3-8B \ - --attention-backend fa3 \ - --enable-deterministic-inference -``` - -### Server Arguments - -| Argument | Type/Default | Description | -|----------|--------------|-------------| -| `--enable-deterministic-inference` | flag; default: disabled | Enable deterministic inference with batch-invariant operations | -| `--attention-backend` | string; default: fa3 | Choose attention backend (flashinfer, fa3, or triton) | - -### Example Configurations - -#### Qwen3-8B -```bash -python3 -m sglang.launch_server \ - --model-path Qwen/Qwen3-8B \ - --attention-backend flashinfer \ - --enable-deterministic-inference -``` - -#### Llama Models -```bash -python3 -m sglang.launch_server \ - --model-path meta-llama/Llama-3.1-8B-Instruct \ - --attention-backend fa3 \ - --enable-deterministic-inference -``` - -#### Qwen3-30B-A3B (MoE Model) -```bash -python3 -m sglang.launch_server \ - --model-path Qwen/Qwen3-30B-A3B \ - --attention-backend fa3 \ - --enable-deterministic-inference -``` - -### Deterministic Inference with Non-Greedy Sampling (Temperature > 0) - -SGLang supports deterministic inference even with non-greedy sampling by using sampling seeds. This is particularly useful for reinforcement learning scenarios like GRPO (Group Relative Policy Optimization) where you need multiple diverse but reproducible responses. - -#### Default Behavior - -By default, SGLang uses a sampling seed of `42` for reproducible sampling: - -```python -import requests - -response = requests.post( - "http://localhost:30000/generate", - json={ - "text": "Tell me a joke", - "sampling_params": { - "temperature": 0.8, # Non-greedy sampling - "max_new_tokens": 128, - }, - }, -) -print(response.json()) -# This will always produce the same response across runs -``` - -#### Generating Multiple Reproducible Responses - -To sample different responses from the same prompt while maintaining reproducibility (e.g., for GRPO training), provide different sampling seeds in your requests: - -```python -import requests - -# Prepare a list of sampling seeds for different responses -sampling_seeds = [42, 43, 44, 45, 46] - -responses = [] -for seed in sampling_seeds: - response = requests.post( - "http://localhost:30000/generate", - json={ - "text": "Tell me a joke", - "sampling_params": { - "temperature": 0.8, - "max_new_tokens": 128, - "sampling_seed": seed, # Specify sampling seed - }, - }, - ) - responses.append(response.json()) - -# Each seed will produce a different but reproducible response -# Using the same seed will always produce the same response -``` - -This approach ensures that: -- Different seeds produce diverse responses -- The same seed always produces the same response across different runs -- Results are reproducible for debugging and evaluation - - -## Verification - -Run deterministic tests to verify consistent outputs: - -```bash -# Single test: same prompt, varying batch sizes -python3 -m sglang.test.test_deterministic --test-mode single --n-trials 50 - -# Prefix test: prompts with different prefix lengths -python3 -m sglang.test.test_deterministic --test-mode prefix --n-trials 50 - -# Radix Cache Consistency mode: test radix cache determinism (cached vs uncached prefill) -python3 -m sglang.test.test_deterministic --test-mode radix_cache -``` - -Expected result: All tests should show `Unique samples: 1` (perfectly deterministic). diff --git a/docs/advanced_features/dp_dpa_smg_guide.md b/docs/advanced_features/dp_dpa_smg_guide.md deleted file mode 100644 index 9ec5df648..000000000 --- a/docs/advanced_features/dp_dpa_smg_guide.md +++ /dev/null @@ -1,373 +0,0 @@ -# DP, DPA and SGLang DP Router - -This guide explains the difference between Data Parallelism (DP) and Data Parallelism Attention (DPA), how to enable each mode correctly, and how to use the SGLang Model Gateway (SMG) for production-grade DP deployments. - -## Data Parallelism (DP) - -**Data Parallelism (DP)** is the most common parallelism strategy that replicates the entire model across multiple GPU sets and processes different batches of requests in parallel. Each GPU set handles independent requests. With dedicated routing strategies, as we will introduce later, with those proper routing algorithms in SGLang Model Gateway, the throughput of your serving system could be multiplied nearly linearly. - -### Key characteristics - -- Each replica has a full copy of the model -- Requests are distributed/scattered across replicas -- No inter-replica communication during one request's inference (for simple DP) - -## Data Parallelism Attention (DPA) - -**Data Parallelism Attention (DPA)**, also known as DP Attention, is an advanced parallelism strategy. While DPA provides the most significant benefits for **Multi-Head Latent Attention (MLA)** models (such as DeepSeek, MiniMax, Kimi-K2), it also supports **standard attention models** like Qwen. - -### The Problem with Tensor Parallelism for MLA Models - -The most common parallelism strategy for inference is **Tensor Parallelism (TP)**. However, TP might not be the most efficient strategy for certain models. For example, DeepSeek models use MLA and only have **one KV head**. If we use tensor parallelism on 8 GPUs, it will lead to: - -- **Duplicated KV cache** across all GPUs -- **Unwanted memory usage** that limits batch size -- **Reduced throughput** due to memory constraints - -### How DPA Works - -DPA addresses these limitations by applying **data parallelism specifically to the attention component**. - - - - - - -
-DPA + EP Architecture - - -**Each DP replica:** - -- Processes different batches independently (can be in different forward modes: prefill, decode, or idle) -- Maintains its own KV cache (no duplication) -- Enables significantly larger batch sizes due to memory savings - -**Communication patterns in DPA + EP:** -- -- **All2All (Dispatch)**: Routes tokens to expert sub-groups based on gating decisions -- **All2All (Combine)**: Gathers computed results from experts back to original token positions - -
- -### Key benefits of DPA - -1. **Significantly reduced KV cache memory**: Each DP replica only stores KV cache for its own batches -2. **Larger batch sizes**: Memory savings enable larger batch sizes -3. **Improved decoding throughput**: Significant throughput gains for MLA-based models -4. **Independent forward modes**: Each DP replica can be in different forward modes (prefill, decode, or idle) and handles its assigned batches independently during attention computation - -### DPA with Expert Parallelism for MoE - -For MoE models like DeepSeek, DPA is **often** paired with Expert Parallelism (EP) for best throughput at scale. However, **DPA does not require EP**: you can enable DPA without EP if your deployment does not need expert sharding. - -- Distribute 256+ expert weights across GPUs (cannot fit on a single GPU) -- Enable efficient all-to-all token routing via DeepEP -- Scale to large clusters (up to 5x throughput improvement over vanilla TP) - -### Recommended setup for DeepSeek - -```bash -python -m sglang.launch_server \ - --model-path deepseek-ai/DeepSeek-V3 \ - --tp 8 \ - --dp-size 8 \ - --ep 8 \ - --enable-dp-attention \ - --moe-a2a-backend deepep \ - --moe-runner-backend deep_gemm -``` - -> **Note**: `--dp-size` must be explicitly set when using `--enable-dp-attention`. If `dp_size` is 1 (default), DPA will be disabled. - -For detailed EP configuration (DeepEP, Two-Batch Overlap, EPLB), see [Expert Parallelism](expert_parallelism.md). - -### Target Models - -DPA supports the following model architectures: - -- **MLA (Multi-Head Latent Attention) models** - where DPA provides the most significant benefits: - - DeepSeek family (DeepSeek-V2, DeepSeek-V3, DeepSeek-R1) - - MiniMax models - - Kimi-K2 - - Other models using MLA architecture - -- **Standard attention models** - also supported: - - Qwen models (see [PR #6121](https://github.com/sgl-project/sglang/pull/6121)) - -For models like Llama, with standard GQA, standard DP, or TP is typically recommended. - -To enable DPA, add `--enable-dp-attention` to your server launch command. - -### Activation Logic - -DPA is enabled explicitly via server arguments (CLI or config). You must set both `--dp-size` and `--enable-dp-attention`: - -```bash -python -m sglang.launch_server \ - --model-path deepseek-ai/DeepSeek-V3 \ - --tp 8 \ - --dp-size 8 \ - --enable-dp-attention -``` - -**Important**: `--dp-size` must be greater than 1 for DPA to work. When `dp_size == 1` (default), `--enable-dp-attention` is automatically disabled. The constraint `tp_size % dp_size == 0` must also be satisfied. - -### Standard DP for MLA models - -Note that MLA models, of course, also support DP. Suppose you want to enable standard DP for MLA models. First, launch each MLA model's replica independently. You may launch these replicas one by one with DPA enabled. After launching each MLA model's replica, launch an SMG and connect all the replicas to the SMG. A detailed explanation of SMG is as follows. - -## Modern Data Parallelism SGLang Model Gateway (SMG) - -### Native DP Mode - -Native DP (built-in Data Parallelism) in SGLang creates multiple worker processes within a single SGLang instance, under the control of `DataParallelController` with the launching parameter of `dp-size`. - - -```bash -# Native DP mode -python -m sglang.launch_server \ - --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \ - --dp-size 4 -``` - -**Limitations:** - -- Built-in in-process load balancing only (e.g., `round_robin`, `total_requests`, `total_tokens`) -- No cache-aware routing -- Limited observability and metrics -- No fault tolerance or circuit breakers -- Not suitable for production workloads - -⚠️ Native DP is **highly not recommended for use right now**. It is only used in some ancient/outdated RL frameworks. You can use SGLang Model Gateway (SMG) to power up your data parallelism in any use case. - -### SMG-Based DP (Recommended) - -Starting from September 2024, SGLang Model Gateway, i.e., SMG, formerly named as SGLang DP Router, was built especially as a production-ready DP routing system with Rust. It starts from DP routing, but later we further expanded its scope to coordinate RL, PD Disaggregation, and other scenarios. This doc only discusses SMG's usage in DP routing. For other usage, please refer to [SGLang Model Gateway Documentation](sgl_model_gateway.md). - -> To achieve the best production-level routing performance and reduce the overhead to an extreme extent, we use Rust to build SMG, but not Python, since Python is never FAST enough. - -**We strongly recommend using the SGLang Model Gateway (SMG) for production-grade Data Parallelism.** SMG provides significant advantages over native DP mode. - -```bash -# SMG-based DP mode (Recommended) -python -m sglang_router.launch_server \ - --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \ - --dp-size 4 -``` - -⚠️ Note that **SMG and Naive DP share the same launching parameter, `--dp-size`**. But the entrypoint of Naive DP is `python -m sglang.launch_server`, and SMG's entrypoint is `python -m sglang_router.launch_server`. - -**Advantages of SMG-Based DP:** - -| Feature | Native DP | SMG-Based DP | -|---------|-----------|--------------| -| **Load Balancing** | Built-in in-process methods | Advanced policies (cache-aware, power-of-two, etc.) | -| **Cache Awareness** | ❌ No | ✅ Yes - significantly higher cache hit rate | -| **Throughput** | Baseline | Significant improvement | -| **Multi-Node Support** | Limited | ✅ Full support | -| **Worker Health Monitoring** | Basic | ✅ Circuit breakers, health checks | -| **Reliability** | Basic | ✅ Retries, rate limiting, queuing | -| **Observability** | Basic metrics | ✅ 40+ Prometheus metrics, OpenTelemetry | -| **Hot Worker Add/Remove** | ❌ No | ✅ Yes | - -### SMG's Performance - -The cache-aware routing policy in SMG significantly improves performance for workloads with shared prefixes: - -| Metric | Without Cache-Aware | With Cache-Aware SMG | -|--------|---------------------|----------------------| -| Throughput (token/s) | 82,665 | 158,596 (+92%) | -| Cache Hit Rate | 20% | 75% (+275%) | - -*Benchmark from [SGLang v0.4 blog](https://lmsys.org/blog/2024-12-04-sglang-v0-4/), workload with multiple long prefix groups, 8x A100 80GB GPUs, dp-size=8* - -### When to Use Each - -**Use Native DP when:** - -- ~Never use Native/Naive DP~ -- Learning material of DP routing - -**Use SMG-Based DP when:** - -- In any case, when you think DP is needed -- Production deployments -- Multi-node distributed setups -- Workloads with shared prefixes (high cache reuse potential) -- You need high availability and reliability features -- You require detailed observability and metrics -- You want to have highly efficient RL rollout systems - -Note that for RL rollout systems, **there are four crucial reasons that SMG-Based DP is far better than naive DP routing**. Details can be found at [Load Balancing Router in RL](./sglang_for_rl.md#load-balancing-router). - -### Quick Start For SMG - -**Installation** - -```bash -pip install sglang-router -# or -pip install "sglang[all]" -``` - -**Option A: Co-launch Workers and SMG (Simplest)** - -This is the easiest way to get started - SMG and workers are launched together: - -```bash -python -m sglang_router.launch_server \ - --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \ - --dp-size 4 \ - --host 0.0.0.0 \ - --port 30000 -``` - -**Option B: Separate Launch (Multi-Node)** - -For distributed deployments across multiple machines: - -1. Launch workers on each node - -```bash -# Node 1 -python -m sglang.launch_server \ - --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \ - --port 8000 - -# Node 2 -python -m sglang.launch_server \ - --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \ - --port 8000 -``` - -2. Launch SMG pointing to workers - -```bash -python -m sglang_router.launch_router \ - --worker-urls http://node1:8000 http://node2:8000 \ - --policy cache_aware \ - --host 0.0.0.0 \ - --port 30000 -``` - -**Option C: Dynamic Worker Registration** - -For elastic deployments where workers can be added/removed dynamically: - -```bash -# Launch SMG first -python -m sglang_router.launch_router \ - --policy cache_aware \ - --host 0.0.0.0 \ - --port 30000 - -# Register workers dynamically -curl -X POST http://localhost:30000/workers \ - -H "Content-Type: application/json" \ - -d '{"url": "http://worker1:8000"}' - -curl -X POST http://localhost:30000/workers \ - -H "Content-Type: application/json" \ - -d '{"url": "http://worker2:8000"}' -``` - -### Load Balancing Policies - -SMG supports multiple load balancing policies: - -| Policy | Description | Best For | -|--------|-------------|----------| -| `cache_aware` | Combines cache locality with load balancing | **Recommended for most workloads** | -| `round_robin` | Cycles through workers in order | Simple, predictable distribution | -| `random` | Random worker selection | Baseline, testing | -| `power_of_two` | Samples two workers, picks lighter one | Low latency requirements | - -**Cache-Aware Policy (Default, Recommended)** - -The cache-aware policy provides the best performance for most workloads: - -```bash -python -m sglang_router.launch_router \ - --worker-urls http://worker1:8000 http://worker2:8000 \ - --policy cache_aware \ - --cache-threshold 0.5 \ - --balance-abs-threshold 32 \ - --balance-rel-threshold 1.5 \ - --eviction-interval-secs 120 \ - --max-tree-size 67108864 -``` - -**How it works:** - -1. Maintains an approximate radix tree for each worker based on request history -2. Routes requests to workers with the highest prefix match (cache hit) -3. Falls back to shortest-queue routing when load is imbalanced -4. Automatically evicts old entries to prevent memory overflow - -### Best Practices - -1. **Start with `cache_aware` policy** - It provides the best balance between cache locality and load distribution for most workloads -2. **Use SMG for production** - Prefer `sglang_router.launch_server` over `sglang.launch_server` for better reliability and observability -3. **Enable health checks** - Configure `--router-health-check-interval-secs` to detect and remove unhealthy workers automatically - -**Recommended command with best practices applied:** - -```bash -python -m sglang_router.launch_server \ - --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \ - --dp-size 4 \ - --router-policy cache_aware \ - --router-health-check-interval-secs 30 \ - --router-prometheus-port 10001 \ - --host 0.0.0.0 \ - --port 30000 -``` - -For advanced configuration (circuit breakers, retries, Prometheus metrics, K8s integration), see [SGLang Model Gateway Documentation](sgl_model_gateway.md). - -### Verifying Traffic Distribution - -After launching SMG, verify that traffic is being distributed correctly: - -**1. Check worker status:** - -```bash -curl http://localhost:30000/workers -``` - -**2. Check load distribution:** - -```bash -curl http://localhost:30000/get_loads -``` - -**3. Monitor metrics (if Prometheus enabled):** - -```bash -# Key metrics to check -smg_router_requests_total{model="..."} -smg_worker_requests_active{worker="..."} -sglang_cache_hit_rate{source="..."} -``` - -For detailed metrics and monitoring setup, see [SGLang Model Gateway Documentation](sgl_model_gateway.md). - -## Reference - -| Strategy | Use Case | Key Benefit | -|----------|----------|-------------| -| **Native DP** (`--dp-size`) | Never | Easy to understand, not rust based | -| **SMG-Based DP** | **Production (recommended)** | Cache-aware routing, high availability | -| **DPA** (`--dp-size N --enable-dp-attention`) | DeepSeek/MLA models | Eliminates KV cache duplication, improved throughput | -| **DPA + EP** | DeepSeek MoE models | Significant throughput improvement vs vanilla TP | - -**Recommended production setup for DeepSeek:** -1. Enable **DPA** for attention layers (`--dp-size 8 --enable-dp-attention`) -2. Enable **EP** for MoE layers (`--ep 8 --moe-a2a-backend deepep`) -3. Use **SMG** with **cache_aware** policy - -**Related documentation:** -- [Expert Parallelism](expert_parallelism.md) - DeepEP, Two-Batch Overlap, EPLB -- [SGLang Model Gateway Documentation](sgl_model_gateway.md) - SMG configuration & troubleshooting -- [Large-Scale EP Blog](https://lmsys.org/blog/2025-05-05-large-scale-ep/) - 96 GPU deployment guide diff --git a/docs/advanced_features/dp_for_multi_modal_encoder.md b/docs/advanced_features/dp_for_multi_modal_encoder.md deleted file mode 100644 index a100e0688..000000000 --- a/docs/advanced_features/dp_for_multi_modal_encoder.md +++ /dev/null @@ -1,30 +0,0 @@ -# DP for Multi-Modal Encoder in SGLang - -A typical VLM architecture involves two main components: an multi-modal encoder and a text decoder. - -Most VLMs utilize a Vision Transformer (ViT) as their multi-modal encoder, it is responsible for processing visual data, extracting features (objects, colors, textures, etc.), and transforming them into a format that can be understood by the model. - -The text decoder is based on LLM. It processes textual data and generates output based on the encoded visual features. - -However, since the size of ViT is very small compared to language decoders, -there is relatively little gain from TP. On the other hand, TP incurs significant communication -overhead because of all-reduce being performed after every layer. - -Placing the ViT in data parallel while keeping the LLM in tensor parallel consistently lowers TTFT and boosts end-to-end throughput. In this hybrid layout, the vision front-end becomes parallel and lightweight, while scarce interconnect bandwidth and collective ops are reserved for the LLM. - -Data parallelism replicates the entire model across multiple GPU sets and processes different batches of requests in parallel. - -## Command Example -You can enable batch-level DP by setting `mm-enable-dp-encoder`, for example: -``` -python3 -m sglang.launch_server \ - --model-path Qwen/Qwen2.5-VL-7B-Instruct \ - --tp 2 \ - --mm-enable-dp-encoder -``` - -## Known supported models -- Qwen2.5-VL () -- Qwen3-VL () -- InternVL () -- GLM-4.5V & GLM-4.6V () diff --git a/docs/advanced_features/epd_disaggregation.md b/docs/advanced_features/epd_disaggregation.md deleted file mode 100644 index d07898361..000000000 --- a/docs/advanced_features/epd_disaggregation.md +++ /dev/null @@ -1,194 +0,0 @@ -# EPD Disaggregation - -## Why and What is EPD Disaggregation? - -In modern Vision-Language Model (VLM) inference, request execution naturally decomposes into three distinct stages: Encoder, Prefill, and Decode. -The Encoder stage performs vision preprocessing and ViT-based image encoding, which is highly compute-intensive but only required during request initialization. The Prefill stage processes the full multimodal input sequence to initialize the language model’s Key-Value (KV) cache, while the Decode stage is dominated by memory bandwidth and KV cache access for autoregressive token generation. - -Existing deployments typically colocate these stages within a unified execution engine, or at best apply Prefill–Decode (PD) disaggregation. However, such designs still tightly couple vision encoding with language prefill, leading to inefficient resource utilization, limited scalability for image-heavy workloads, and suboptimal scheduling under load. - -To address these challenges, we introduce Encoder–Prefill–Decode (EPD) Disaggregation in SGLang. EPD further separates vision encoding from language processing, enabling independent horizontal scaling of encoder servers, improved load balancing for multimodal requests, and seamless integration with existing PD disaggregation to form a fully decoupled three-tier inference architecture. - -### Usage - -You can launch a language-only model using `--language-only`, or an encoder-only model using `--encoder-only`. -When launching a language-only model, you must additionally specify the encoder service endpoints via `--encoder-urls`. - -We support multiple encoder transfer backends, including zmq_to_scheduler, zmq_to_tokenizer, and mooncake (the default is zmq_to_scheduler). The backend can be selected using `--encoder-transfer-backend`. - -### Encoder transfer with Mooncake - -`--encoder-transfer-backend mooncake` controls **how encoder outputs are transferred** between encoder and language/prefill services. It is an encoder transfer option and can be used independently of the global multimodal embedding cache. - -Example: - -```bash -# encoder -python -m sglang.launch_server \ - --model-path Qwen/Qwen3-VL-8B-Instruct \ - --encoder-only \ - --encoder-transfer-backend mooncake \ - --port 30000 - -# language-only server -python -m sglang.launch_server \ - --model-path Qwen/Qwen3-VL-8B-Instruct \ - --language-only \ - --encoder-urls http://127.0.0.1:30000 \ - --encoder-transfer-backend mooncake \ - --port 30002 -``` - -### Global multimodal embedding cache with Mooncake - -SGLang also supports a Mooncake-backed **global multimodal embedding cache** for EPD workloads. When enabled on encoder servers, repeated image inputs can reuse previously computed ViT embeddings across instances instead of running the vision encoder again. - -This feature is useful when: - -- the deployment serves repeated or overlapping image inputs, -- encoder compute is the bottleneck, and -- Mooncake is already available in the cluster. - -At a high level, the encoder checks whether the image embedding already exists in Mooncake. Cache hits are prefetched from the global store, while misses are encoded normally and inserted into the cache in the background. - -To enable it: - -- install and configure Mooncake in the same way as other SGLang Mooncake integrations, -- add `--enable-mm-global-cache` on the encoder server. - -`--enable-mm-global-cache` controls **whether multimodal embeddings are looked up and stored in the global Mooncake cache**. It is separate from `--encoder-transfer-backend`, which only controls encoder output transport. - -For Mooncake deployment and configuration details, see [HiCache best practices](hicache_best_practices.md#deployment-with-mooncake) and the [Mooncake backend README](../../python/sglang/srt/mem_cache/storage/mooncake_store/README.md). - -Example: - -```bash -# Shared Mooncake configuration -export MOONCAKE_TE_META_DATA_SERVER="http://127.0.0.1:8080/metadata" -export MOONCAKE_MASTER="127.0.0.1:50051" -export MOONCAKE_PROTOCOL="rdma" -export MOONCAKE_GLOBAL_SEGMENT_SIZE="4gb" - -# encoder with global multimodal cache enabled -python -m sglang.launch_server \ - --model-path Qwen/Qwen3-VL-8B-Instruct \ - --encoder-only \ - --enable-mm-global-cache \ - --port 30000 - -# language-only server -python -m sglang.launch_server \ - --model-path Qwen/Qwen3-VL-8B-Instruct \ - --language-only \ - --encoder-urls http://127.0.0.1:30000 \ - --port 30002 -``` - -Notes: - -- This cache is for **multimodal encoder embeddings**, not the language model KV cache. -- The feature currently uses Mooncake as the shared backing store. -- It can be enabled regardless of which `--encoder-transfer-backend` you use. -- It is most relevant for EPD or encoder-disaggregated VLM deployments where the same images are likely to appear across requests or instances. - -#### Qwen VL - -- EP Disaggregation - -```bash -# encoder 0 -python -m sglang.launch_server \ - --model-path Qwen/Qwen3-VL-8B-Instruct \ - --encoder-only \ - --encoder-transfer-backend zmq_to_scheduler \ - --port 30000 -# encoder 1 -python -m sglang.launch_server \ - --model-path Qwen/Qwen3-VL-8B-Instruct \ - --encoder-only \ - --encoder-transfer-backend zmq_to_scheduler \ - --port 30001 -# language-only server -python -m sglang.launch_server \ - --model-path Qwen/Qwen3-VL-8B-Instruct \ - --language-only \ - --encoder-urls http://127.0.0.1:30000 http://127.0.0.1:30001 \ - --encoder-transfer-backend zmq_to_scheduler \ - --port 30002 -``` - -- EPD Disaggregation - -```bash -# encoder 0 -python -m sglang.launch_server \ - --model-path Qwen/Qwen3-VL-8B-Instruct \ - --encoder-only \ - --encoder-transfer-backend zmq_to_scheduler \ - --port 30000 -# encoder 1 -python -m sglang.launch_server \ - --model-path Qwen/Qwen3-VL-8B-Instruct \ - --encoder-only \ - --encoder-transfer-backend zmq_to_scheduler \ - --port 30001 -# prefill 0 -python -m sglang.launch_server \ - --model-path Qwen/Qwen3-VL-8B-Instruct \ - --disaggregation-mode prefill \ - --language-only \ - --encoder-urls http://127.0.0.1:30000 http://127.0.0.1:30001 \ - --encoder-transfer-backend zmq_to_scheduler \ - --port 30002 -# decode 0 -python -m sglang.launch_server \ - --model-path Qwen/Qwen3-VL-8B-Instruct \ - --disaggregation-mode decode \ - --port 30003 -# router -python -m sglang_router.launch_router \ - --pd-disaggregation \ - --prefill http://$PREFILL_HOST:30002 \ - --decode http://$DECODE_HOST:30003 \ - --port 8000 - -``` - -#### gRPC Encoder (EPD) - -You can run the encoder as a gRPC server while keeping prefill/decode as HTTP. -When using gRPC encoders, set `SGLANG_ENCODER_MM_RECEIVER_MODE=grpc` for the -prefill process so it uses the gRPC receiver. - -```bash -# gRPC encoder -python -m sglang.launch_server \ - --model-path Qwen/Qwen3-VL-8B-Instruct \ - --encoder-only \ - --grpc-mode \ - --encoder-transfer-backend zmq_to_scheduler \ - --port 30000 - -# prefill (HTTP) - tell it to use gRPC receiver -SGLANG_ENCODER_MM_RECEIVER_MODE=grpc \ -python -m sglang.launch_server \ - --model-path Qwen/Qwen3-VL-8B-Instruct \ - --disaggregation-mode prefill \ - --language-only \ - --encoder-urls grpc://127.0.0.1:30000 \ - --encoder-transfer-backend zmq_to_scheduler \ - --port 30002 - -# decode (HTTP) -python -m sglang.launch_server \ - --model-path Qwen/Qwen3-VL-8B-Instruct \ - --disaggregation-mode decode \ - --port 30003 - -# router -python -m sglang_router.launch_router \ - --pd-disaggregation \ - --prefill http://$PREFILL_HOST:30002 \ - --decode http://$DECODE_HOST:30003 \ - --port 8000 -``` diff --git a/docs/advanced_features/expert_parallelism.md b/docs/advanced_features/expert_parallelism.md deleted file mode 100644 index 5c052114b..000000000 --- a/docs/advanced_features/expert_parallelism.md +++ /dev/null @@ -1,200 +0,0 @@ -# Expert Parallelism - -Expert Parallelism (EP) in SGLang distributes expert weights across multiple devices in Mixture-of-Experts (MoE) models, addressing memory bottlenecks and enabling efficient scaling for high-performance inference. It is particularly vital for serving large-scale MoE models where tokens are dynamically routed to specialized experts across GPUs. By leveraging optimized all-to-all communication and grouped matrix multiplications (GEMMs), EP reduces latency, boosts throughput, and minimizes idle GPU time. SGLang's EP offers strong extensibility through its modular framework, allowing seamless integration of custom kernels, backends, and optimizations without refactoring core logic, supporting diverse hardware and quantization schemes. - -## Supported Backends and Selection Guidance - -SGLang's EP integrates diverse, highly efficient backends for different use cases, allowing fine-grained control over performance trade-offs. Users specify backends via command-line flags: -- `--moe-a2a-backend`: Selects the backend for all-to-all communication. -- `--moe-runner-backend`: Selects the backend for MoE computation. - -### Backends for All-to-All Communication - -| Backend | Description | Use Cases | -|--------------|-----------------------------------------------------------------------------|------------------------------------| -| **`none` (default)** | Disables all-to-all for EP. Uses All-Reduce or All-Gather for token dispatch. | Hybrid EP and TP setups. | -| `deepep` | DeepEP, a communication library for efficient token shuffling in MoE models. | Large-scale EP deployments. | -| `mooncake` | An extension of DeepEP for elastic inference, leveraging RDMA for high-performance data transfers. | Elastic EP serving. | -| `nixl` | [NIXL-EP](https://github.com/ai-dynamo/nixl/tree/main/examples/device/ep), an elastic EP communication library built on NVIDIA's [NIXL](https://github.com/ai-dynamo/nixl) framework with native RDMA and NVLink support. | Elastic EP serving with fault tolerance and dynamic scaling. | -| `mori` | MORI-EP, AMD's native all-to-all communication implementation optimized for ROCm. | AMD GPU deployments. | -| `flashinfer` | Flashinfer implementation of all-to-all. | Large-scale EP deployments. | -| `ascend_fuseep` | Ascend NPU native fused all-to-all communication. | Ascend NPU deployments. | - -DeepEP and Mooncake backends support two modes for token dispatch: `normal` mode (optimized for prefill workloads with high throughput) and `low_latency` mode (optimized for decode workloads with low latency and CUDA Graph compatibility). MORI backend only supports `normal` mode now. NIXL-EP currently operates in low-latency mode with CUDA Graph support. Users are recommended to set `--deepep-mode auto` to enable automatic dispatch mode switching during runtime. Setting `--deepep-mode normal` or `--deepep-mode low_latency` is useful for debugging or development purposes. - -Currently, DeepEP, Mooncake, NIXL-EP, `ascend_fuseep` and MORI only support cases where `ep_size = tp_size`. For hybrid EP and TP (i.e., `ep_size < tp_size`), only the `none` backend (All-Reduce or All-Gather-based dispatching) is supported. - -### Backends for MoE Computation - -| Backend | Description | Use Cases | -|--------------------------|-----------------------------------------------------------------------------|------------------------------------| -| **`auto` (default)** | Automatically selects the optimal backend based on model architecture, hardware (e.g., NVIDIA architecture like Ampere, Hopper, Blackwell), quantization scheme (e.g., FP8, FP4), and runtime conditions. | General-purpose deployments; ensures compatibility and performance without user intervention. | -| `triton` | Triton-based implementation for grouped GEMMs. To achieve higher performance, it's highly recommended to create [tuned configurations](https://github.com/sgl-project/sglang/blob/main/benchmark/kernels/fused_moe_triton/README.md). | Custom kernel development or scenarios requiring high extensibility with Torch compilation support. | -| `deep_gemm` | DeepGEMM backend optimized for MoE matrix multiplications, supporting contiguous layouts for prefill and masked layouts for decode; often JIT-compiled for performance. | Large-scale EP deployments with FP8 block-wise quantization. | -| `cutlass` | CUTLASS-based backend for efficient GEMMs. | NVIDIA architectures with CUTLASS support. | -| `flashinfer_trtllm` | FlashInfer integrated with TensorRT-LLM for accelerated MoE computations, supporting FP4 communication operators and high-performance GEMMs. | Blackwell with TRT-LLM. | -| `flashinfer_trtllm_routed` | FlashInfer integrated with TensorRT-LLM for accelerated routed MoE computations, consuming SGLang-computed top-k expert assignments and weights. | Blackwell with TRT-LLM. | -| `flashinfer_cutlass` | FlashInfer combined with CUTLASS for high-performance grouped GEMMs in MoE layers, handling FP4/FP8 quantization efficiently. | Blackwell with FP4/FP8 models. | -| `flashinfer_mxfp4` | FlashInfer variant optimized for MXFP4 (mixed FP4) quantization in MoE runners, focusing on memory-efficient low-precision inference. | Low-precision models with MXFP4. | -| `flashinfer_cutedsl` | FlashInfer with a custom DSL for flexible and efficient MoE kernel generation, integrated with ModelOpt FP4 quantization. | Low-precision models with NVFP4. | - -### Examples - -Launch with DeepEP and DeepGEMM for DeepSeek-V3: - -```bash -python -m sglang.launch_server --model-path deepseek-ai/DeepSeek-V3 --moe-a2a-backend deepep --moe-runner-backend deep_gemm --tp 8 --ep 8 -``` - -## Extensible EP Framework - -SGLang's EP framework provides modular abstractions for easy integration of custom kernels, backends, and optimizations. It decouples the MoE forward pass into stages (dispatch → pre-permute → core runner → post-permute → combine), enabling seamless extensions without refactoring core logic. - -### Framework Overview - -The framework centers on `FusedMoE` as the unified entry point for a single, extensible structure. Key components include: -- **Dispatcher**: Manages dispatch/combine for backends like DeepEP (implements `BaseDispatcher` subclasses). -- **MoeRunner**: Orchestrates grouped-GEMM execution via `MoeRunnerCore` implementations (e.g., `TritonRunnerCore`). -- **PermuteMethodPool**: Auto-registers layout conversions (e.g., pre/post-permute via `register_pre_permute` and `register_post_permute` for dynamic modes, or `register_fused_func` for static, torch.compile-compatible fused operations). -- **TopK Router**: Backend-agnostic expert selection. - -This design supports multiple backends via `--moe-a2a-backend` and `--moe-runner-backend`, with quantization integrated through a standardized `apply()` method. The computation flow ensures modularity: - -``` -[input_hidden_states] - | - v - TopK.forward -> select_experts / triton_kernels.routing / bypass - | - v - [TopKOutput] - | - v - FusedMoE.forward -> Dispatcher.dispatch -> DeepEP / bypass - | | - | v - | [DispatchOutput] - | | - | v - | quant_method.apply -> MoeRunner.forward - | | | - | | v - | | pre-permute + grouped_gemm + post-permute - | | | - | |-------------- - | v - | [CombineInput] - | | - | v - | Dispatcher.combine -> DeepEP / bypass - | | - |--------------------- - v -[final_hidden_states] -``` - -For details, see the [MoE Refactor Roadmap](https://github.com/sgl-project/sglang/issues/8715). - -### Implementing New Backends - -To add a new backend: -1. For a new all-to-all dispatcher, implement a `BaseDispatcher` subclass with `dispatch` and `combine` methods. -2. For a new MoE runner backend, define a `MoeRunnerCore` subclass for core operations (e.g., grouped GEMMs). -3. Define new input/output formats for the dispatcher or model runner (e.g., `RunnerInput`, `RunnerOutput`). -4. Register permute/unpermute methods to ensure compatibility: - - **Fused Mode** (static, torch.compile-compatible): Use `register_fused_func` for end-to-end operations. - - **Permute Mode** (dynamic): Register `register_pre_permute` and `register_post_permute` for flexible layouts. - -See the [MoE Refactor Implementation PR](https://github.com/sgl-project/sglang/pull/9269) for full changes, including type hints and config expansions. - -### Examples - -For an example implementation, see [moe_runner/triton.py](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/layers/moe/moe_runner/triton.py), which demonstrates Triton-based grouped GEMMs with registered fused and permutation functions. - -## Computation and Communication Overlap - -SGLang's EP employs advanced overlap techniques to hide communication latency behind computation, maximizing GPU utilization in MoE layers. - -### Two-Batch Overlap (TBO) - -TBO splits requests into micro-batches, interleaving attention computation with dispatch/combine operations. Yield points in the execution graph allow pausing for overlaps, increasing overall throughput without peak memory spikes: - -```python -operations = [ - self._forward_attn, - YieldOperation(), # Overlap with dispatch of prior micro-batch - self._forward_dispatch, - self._forward_mlp, - YieldOperation(), # Overlap with combine - self._forward_combine, -] -``` - -Users need to specify `--enable-two-batch-overlap` to unlock up to 2x throughput. For details, see the [Large-Scale EP Blog](https://lmsys.org/blog/2025-05-05-large-scale-ep/#two-batch-overlap). - -### Single-Batch Overlap (SBO) - -SGLang introduces a dispatcher-hook system for Single-Batch Overlap (SBO), enabling the overlap of operations within a single batch—such as shared experts computation with communication—while decentralizing logic to enhance modularity. These hooks execute before and after the `dispatch` and `combine` operations without modifying core MoE modules. This design simplifies interfaces, reduces coupling, and improves extensibility. For implementation details and an example of overlapping shared experts with DeepEP's combine operation, refer to [PR #13327](https://github.com/sgl-project/sglang/pull/13327). Users can set `--enable-single-batch-overlap` to enable this feature. - - -## Workload Balancer - -SGLang integrates the [Expert Parallelism Load Balancer (EPLB)](https://github.com/deepseek-ai/EPLB) from DeepSeek to address routing imbalances in MoE models. By analyzing expert activation statistics, EPLB computes an optimal expert arrangement, strategically placing or replicating experts to minimize GPU utilization variance, reduce idle cycles, and enhance scalability. - -To enable EPLB, use the flags `--enable-eplb`. For optimal performance, increase batch sizes to stabilize activation statistics and configure periodic rebalancing (e.g., every 1000 requests) to adapt to evolving workloads. Simulations demonstrate significant improvements in load balancedness (ratio of mean to max computation time), correlating strongly with throughput gains. - -For more details, refer to the [EPLB Section in the Large-Scale EP Blog](https://lmsys.org/blog/2025-05-05-large-scale-ep/#expert-parallelism-load-balancer) and the [EPLB Repository](https://github.com/deepseek-ai/eplb). - - -## EP with Spectulative Decoding - - -When utilizing speculative decoding with MTP on MoE architectures, use the `--speculative-moe-runner-backend` and `--speculative-moe-a2a-backend` arguments to customize the MoE layer behavior for the draft model. While they default to the target model’s settings, users can differentiate them for varying precisions between target and draft models. - -For model like `nvidia/DeepSeek-R1-0528-NVFP4-v2`, the target model uses NVFP4 precision while the draft model uses BF16. To apply `flashinfer_trtllm` kernel for target MoE layer while falling back to triton fused MoE kernel for draft MoE layer, users can set the arguments as follows: -``` -... ---moe-runner-backend flashinfer_trtllm \ ---speculative-moe-runner-backend triton \ -... -``` - - -## Ascend NPU Guidance - - -### Guidance on SGLang configuration in Ascend NPU -- `--moe-a2a-backend` only supports `deepep` and `ascend_fuseep` backends, - - `deepep`: The mechanism is consistent with the above description. - - `ascend_fuseep`: Offer a large fused operator which integrates all operations between dispatch and combine to boost MoE computation. Only used for decode stage in PD Disaggregation Mode. -- `--moe-runner-backend` parameter does not need to be configured. -- `--deepep-mode`: - - In PD mixed mode, please set `--deepep-mode auto`. - - In PD Disaggregation Mode, prefill instance sets `--deepep-mode normal`, and decode instance sets `--deepep-mode low_latency`. - - -### DeepEP Ascend Introduction - -DeepEP Ascend is the adapted version of the DeepEP communication library for Huawei Ascend NPUs, specifically designed for Mixture-of-Experts (MoE) model Expert Parallelism (EP). -It supports the Ant-moving Function (Split the sequence length into rounds for streaming batch transmission) to optimize the buffer size occupied during collective communication in prefill stage, especially for long sequences. - -Ant-moving Function can be enabled for both the dispatch and combine phases via the following environment variables: -- `DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS`: Enable ant-moving function in dispatch stage. Indicates the number of tokens transmitted per round on each rank, default 8192. -- `DEEPEP_NORMAL_LONG_SEQ_ROUND`: Enable ant-moving function in dispatch stage. Indicates the number of rounds transmitted on each rank, default 1. -- `DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ`: Enable ant-moving function in combine stage, default 0 (means disabled). - -`DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS * DEEPEP_NORMAL_LONG_SEQ_ROUND` means input sequence length. When the input sequence length exceeds 8192, it is recommended to enable the ant-moving function in both dispatch and combine phase. - -The environment variable `HCCL_BUFFSIZE` is used to configure the buffer size (MB) actually allocated. Its calculation formula is as follows: -```angular2html -# Enable Ant-moving Function -HCCL_BUFFSIZE >= 2 * (102MB + 4MB + DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS * (hidden_size + hidden_size + hidden_size) * topk) + PADDING_BUFFSIZE - -# Disable Ant-moving Function -HCCL_BUFFSIZE >= 2 * (102MB + 4MB + TOTAL_SEQ_LEN * (hidden_size + hidden_size) * topk) + PADDING_BUFFSIZE -``` -Wherein the parameters are described as follows: -- `hidden_size`: hidden size in model config. -- `topk`: The number of selected routing experts. -- `TOTAL_SEQ_LEN`: input sequence length. -- `PADDING_BUFFSIZE`: A value of 20 or greater is recommended. diff --git a/docs/advanced_features/forward_hooks.md b/docs/advanced_features/forward_hooks.md deleted file mode 100644 index 4dc170b33..000000000 --- a/docs/advanced_features/forward_hooks.md +++ /dev/null @@ -1,297 +0,0 @@ -## Model Hooks - -SGLang supports attaching PyTorch forward hooks to specific submodules in the loaded model, configured entirely via `server_args` JSON. - -This is useful for: - -* Logging intermediate activations -* Debugging model internals -* Exporting hidden states to external tooling - -Hooks are attached once during `ModelRunner.initialize` and run on every forward pass. - ---- - -### Configuration overview - -Hooks are configured via a `ServerArgs` field: - -```python -class ServerArgs: - ... - # For forward hooks - forward_hooks: Optional[List[dict[str, Any]]] = None -```` - -In JSON form, a minimal configuration looks like: - -```jsonc -{ - "forward_hooks": [ - { - "name": "outer_linear_hooks", - "target_modules": ["outer.0", "outer.1"], - "hook_factory": "my_project.hooks:dummy_hook_factory", - "config": { - "tag": "outer-layer" - } - } - ] -} -``` - -#### Top-level fields - -* `forward_hooks` (optional list of objects) - Each element is a hook spec describing: - - * Which modules to target - * Which Python factory to call - * What configuration to pass into that factory - ---- - -### Hook spec schema - -Each entry in `forward_hooks` is a JSON object with the following shape: - -```jsonc -{ - "name": "optional-descriptive-name", - "target_modules": ["pattern1", "pattern2", "..."], - "hook_factory": "module.submodule:factory_name", - "config": { - "...": "arbitrary JSON" - } -} -``` - -#### `name` (optional) - -* Human-readable name for logging. -* Used only in log messages such as: - - ```text - Registered forward hook 'outer_linear_hooks' on outer.0 - ``` - -#### `target_modules` (required) - -* List of **module name patterns** used to match entries in `model.named_modules()`. -* Patterns are matched using `fnmatch.fnmatch`, so: - - * `"outer.0"` matches exactly `"outer.0"`. - * `"outer.*"` matches `"outer.0"`, `"outer.1"`, `"outer.inner"`, etc. - * `"outer.inner.*"` matches children under `outer.inner`. - -> If no modules match the given patterns, hook registration does **not** fail. -> Instead, SGLang logs a warning and continues: -> -> ```text -> No modules matched hook spec 'name' patterns=['...'] -> ``` - -#### `hook_factory` (required) - -* String path to the Python factory function that creates the hook. -* Supported formats: - - * `"package.module:factory_name"` - * `"package.module.submodule.factory_name"` - -The path is resolved via: - -```python -def resolve_callable(path: Optional[str]) -> Optional[Callable]: - if path is None: - return None - - if ":" in path: - module_name, fn_name = path.split(":", 1) - else: - parts = path.split(".") - if len(parts) < 2: - raise ValueError( - f"Invalid hook callable path '{path}'. " - "Expected 'module.submodule:factory' or 'module.submodule.factory'." - ) - *mod_parts, fn_name = parts - module_name = ".".join(mod_parts) - - module = importlib.import_module(module_name) - try: - return getattr(module, fn_name) - except AttributeError as e: - raise AttributeError( - f"Module '{module_name}' has no attribute '{fn_name}' " - f"(from hook path '{path}')" - ) from e -``` - -**Failure modes**: - -* If the path is malformed (not enough dots and no `:`), a `ValueError` is raised at startup. -* If the module imports but the attribute is missing, an `AttributeError` is raised with a clear error message. -* If the hook factory returns `None`, a warning is logged and no hook is registered for that spec (initialization continues). - -The first two cause initialization to fail fast with a descriptive error; the last one is non-fatal. - -#### `config` (optional) - -* Arbitrary JSON object. -* Passed directly to the hook factory as a Python `dict`. -* This lets you parameterize hook behavior from config (e.g. tags, log levels, sampling rates, etc.). - ---- - -### Hook lifecycle and behavior - -Hooks are registered in `ModelRunner.initialize()`: - -```python -if server_args.forward_hooks: - register_forward_hooks(self.model, server_args.forward_hooks) -``` - -The actual registration logic is implemented by `register_forward_hooks`: - -```python -def register_forward_hooks(model: nn.Module, hook_specs: List[dict[str, Any]]) -> None: - """ - hook_specs is a list of dicts from server_args.forward_hooks. - Attaches forward hooks to the matching modules. - """ - name_to_module = dict(model.named_modules()) - - for spec in hook_specs: - spec_name = spec.get("name", "") - target_patterns = spec.get("target_modules", []) - if not target_patterns: - logger.warning( - f"Hook spec '{spec_name}' has no 'target_modules', skipping" - ) - continue - - hook_factory_path = spec.get("hook_factory") - if not hook_factory_path: - logger.warning( - f"Hook spec '{spec_name}' has no 'hook_factory', skipping" - ) - continue - - config = spec.get("config") or {} - hook_factory = resolve_callable(hook_factory_path) - - hook = hook_factory(config) if hook_factory else None - if hook is None: - logger.warning( - f"Hook factory '{hook_factory_path}' for spec '{spec_name}' " - "returned None, not registering any hook" - ) - continue - - # Resolve patterns like "model.layers.*.mlp" - matched = [] - for name, module in name_to_module.items(): - if any(fnmatch.fnmatch(name, pattern) for pattern in target_patterns): - matched.append((name, module)) - - if not matched: - logger.warning( - f"No modules matched hook spec '{spec_name}' " - f"patterns={target_patterns}" - ) - continue - - for module_name, module in matched: - if hook: - _ = module.register_forward_hook(hook) - logger.info( - f"Registered forward hook '{spec_name}' " - f"on {module_name}" - ) -``` - -Key points: - -* Hooks are **forward hooks only** (via `module.register_forward_hook`). -* They are attached once at initialization. -* Hook handles are currently not stored on `ModelRunner` (they cannot be removed later via this API). -* Failure to match any modules is non-fatal; a warning is logged instead. -* If a hook factory returns `None`, a warning is logged and that spec is skipped. - ---- - -### Writing a hook factory - -A hook factory is a regular Python function: - -* Takes a `config: dict` (from JSON) -* Returns a forward hook function with signature `(module, inputs, output)` - -Example: - -```python -HOOK_CALLS = [] - -def dummy_hook_factory(config): - """Factory that returns a forward hook capturing a tag from config.""" - tag = config.get("tag", "default") - - def hook(module, inputs, output): - HOOK_CALLS.append( - { - "module_type": type(module).__name__, - "tag": tag, - "shape": tuple(output.shape), - } - ) - return output # must return output if you don’t want to modify the tensor - - return hook -``` - -In JSON: - -```jsonc -{ - "forward_hooks": [ - { - "name": "capture_outer", - "target_modules": ["outer.0", "outer.1"], - "hook_factory": "my_project.hooks:dummy_hook_factory", - "config": { - "tag": "outer" - } - } - ] -} -``` - -This will: - -* Resolve `my_project.hooks:dummy_hook_factory` to a Python callable. -* Call it with `config = {"tag": "outer"}`. -* Use the returned hook for all modules matching `outer.0` and `outer.1`. -* Append metadata about each call to `HOOK_CALLS`. - ---- - -### Summary - -* Define `forward_hooks` as a list of specs in `ServerArgs` to turn on the feature. - -* Each spec: - - * selects modules via `target_modules` (glob patterns over `model.named_modules()`), - * points to a hook factory via `hook_factory`, - * passes arbitrary `config` into that factory. - -* Hook factories are resolved via `resolve_callable`, which supports `module:factory` and `module.submodule.factory`. - -* Hooks are standard PyTorch forward hooks, attached once at startup and invoked on every forward pass. - -* Misconfiguration is either: - - * **fatal and explicit** (bad path / missing attribute), or - * **non-fatal with clear warnings** (no targets matched, or factory returned `None`). diff --git a/docs/advanced_features/hicache.rst b/docs/advanced_features/hicache.rst deleted file mode 100644 index e7d83211d..000000000 --- a/docs/advanced_features/hicache.rst +++ /dev/null @@ -1,9 +0,0 @@ -Hierarchical KV Caching (HiCache) -================================= - -.. toctree:: - :maxdepth: 1 - - hicache_best_practices.md - hicache_design.md - hicache_storage_runtime_attach_detach.md diff --git a/docs/advanced_features/hicache_best_practices.md b/docs/advanced_features/hicache_best_practices.md deleted file mode 100644 index 104c2b0e2..000000000 --- a/docs/advanced_features/hicache_best_practices.md +++ /dev/null @@ -1,217 +0,0 @@ -# SGLang HiCache Best Practices - -## Why HiCache Matters - -SGLang HiCache extends the traditional RadixAttention with a three-tier hierarchical KV caching system that dramatically improves performance for long-context and multi-turn conversation scenarios. By intelligently managing KV caches across GPU memory, host memory, and external storage backends, HiCache addresses the fundamental capacity bottleneck that limits cache hit rates in conventional systems. - -## Configuration Guidelines - -## Core HiCache Parameters - -```bash -# Essential HiCache flags ---page-size 64 # Page size for cache management ---enable-hierarchical-cache # Enable HiCache ---hicache-ratio 2 # Host memory ratio (2x GPU memory) ---hicache-size 100 # Host memory size in GBs, will override the above ratio ---hicache-io-backend kernel # The I/O backend of moving data between CPU and GPU ---hicache-write-policy write_through # Cache write policy from GPU to CPU ---hicache-storage-backend # Optional storage backend (e.g., hf3fs, mooncake, etc.) -``` - -Notes: - -- Besides configuring `--hicache-storage-backend` at startup, SGLang also supports **runtime attach/detach** of the HiCache storage backend (no restart required) via HTTP admin endpoints. See [Runtime Attach/Detach HiCache Storage Backend](hicache_storage_runtime_attach_detach.md). - -## Key Configurations with Storage Backends Enabled - -### Memory Layout Optimization - -```bash -# Page-first: Optimized for I/O efficiency with zero-copy (recommended with kernel backend) ---hicache-mem-layout page_first -# Page-first-direct: Optimized for direct I/O operations (Compatible with fa3 and same zero-copy performance as page_first) ---hicache-mem-layout page_first_direct -# Layer-first ---hicache-mem-layout layer_first -``` -**Layout Compatibility:** -- `page_first`: Only compatible with `kernel` I/O backend, automatically switches to `layer_first` with `direct` backend -- `page_first_direct`: Specifically designed for `direct` I/O backend with optimized memory organization - -### Heterogeneous TP Support (GQA/MHA models) - -HiCache storage supports cross-cluster KV reuse when different deployments use different TP sizes (for example, `tp=4` and `tp=8`) and share the same storage backend namespace. - -Use `tp_lcm_size` in `--hicache-storage-backend-extra-config`: - -```bash -# Example: heterogeneous TP = {4, 8}, so lcm = 8 ---hicache-storage-backend-extra-config '{"tp_lcm_size": 8}' -``` - -Guidelines: - -- Set `tp_lcm_size` to the least common multiple (LCM) of all TP sizes that will share the same HiCache storage. -- For MHA models with Mooncake and `page_head` layout, HiCache will split head shards based on `tp_lcm_size` to make keys reusable across heterogeneous TP deployments. -- If all clusters use the same TP size, this option is not needed. - -### Prefetch Policies - -```bash -# Best-effort: Terminate prefetch when needed ---hicache-storage-prefetch-policy best_effort -# Wait-complete: Ensure complete prefetch, higher cache reuse ---hicache-storage-prefetch-policy wait_complete -# Timeout: Balance between completion and best-effort ---hicache-storage-prefetch-policy timeout -``` - -### Integration with PD Disaggregation - -HiCache works seamlessly with PD Disaggregation. You can choose between two configurations: - -1. **Prefill-only HiCache**: Enable HiCache only on Prefill nodes, allowing KV cache sharing among Prefill instances -2. **Full HiCache with async offloading**: Enable HiCache on Prefill nodes and async KV cache offloading on Decode nodes, allowing Prefill nodes to reuse KV caches from Decode nodes in multi-turn dialogue scenarios - -```bash -# Prefill node with HiCache enabled for cross-prefill sharing (ideal for SystemPrompt scenarios) -python3 -m sglang.launch_server \ - --model-path /xxx/DeepSeek-R1/ \ - --tp 8 \ - --host 0.0.0.0 \ - --port 10000 \ - --enable-metrics \ - --enable-cache-report \ - --mem-fraction-static 0.85 \ - --page-size 64 \ - --enable-hierarchical-cache \ - --hicache-ratio 2 \ - --hicache-size 0 \ - --hicache-mem-layout page_first_direct \ - --hicache-io-backend direct \ - --hicache-write-policy write_through \ - --hicache-storage-backend hf3fs \ - --hicache-storage-prefetch-policy wait_complete \ - --disaggregation-ib-device mlx5_0 \ - --disaggregation-mode prefill \ - --disaggregation-transfer-backend mooncake - -# Decode node with async offloading enabled for KV cache reuse by Prefill (ideal for multi-turn conversations) -python3 -m sglang.launch_server \ - --model-path /xxx/DeepSeek-R1/ \ - --tp 8 \ - --host 0.0.0.0 \ - --port 10000 \ - --enable-metrics \ - --enable-cache-report \ - --page-size 64 \ - --hicache-ratio 2 \ - --hicache-size 0 \ - --hicache-mem-layout page_first_direct \ - --hicache-io-backend direct \ - --hicache-write-policy write_through \ - --hicache-storage-backend hf3fs \ - --hicache-storage-prefetch-policy wait_complete \ - --disaggregation-decode-enable-offload-kvcache \ # Enable async KV cache offloading in decode node - --disaggregation-ib-device mlx5_0 \ - --disaggregation-mode decode \ - --disaggregation-transfer-backend mooncake -``` - - -### Deployment with HF3FS - -Here is an example of deploying DeepSeek-R1 with HiCache-HF3FS. For more details, see the [HF3FS Documentation](../../python/sglang/srt/mem_cache/storage/hf3fs/docs/README.md). - -```bash -python3 -m sglang.launch_server \ - --model-path /xxx/DeepSeek-R1/ \ - --log-level info \ - --tp 8 \ - --host 0.0.0.0 \ - --port 10000 \ - --enable-metrics \ - --enable-cache-report \ - --page-size 64 \ - --mem-fraction-static 0.85 \ - --enable-hierarchical-cache \ - --hicache-ratio 2 \ - --hicache-size 0 \ - --hicache-mem-layout page_first_direct \ - --hicache-io-backend direct \ - --hicache-write-policy write_through \ - --hicache-storage-backend hf3fs \ - --hicache-storage-prefetch-policy wait_complete \ -``` - -### Deployment with Mooncake - -Here is an example of deploying Qwen3-235B-A22B-Instruct-2507 with Mooncake. For more details, see the [Mooncake Documentation](../../python/sglang/srt/mem_cache/storage/mooncake_store/README.md). - -```bash -# Set Mooncake environment variables -export MOONCAKE_TE_META_DATA_SERVER="http://127.0.0.1:8080/metadata" -export MOONCAKE_GLOBAL_SEGMENT_SIZE=816043786240 -export MOONCAKE_PROTOCOL="rdma" -export MOONCAKE_DEVICE="$DEVICE_LIST" -export MOONCAKE_MASTER=127.0.0.1:50051 - -# Launch SGLang server with Mooncake backend -python3 -m sglang.launch_server \ - --model-path $MODEL_PATH \ - --tp 8 \ - --page-size 64 \ - --enable-hierarchical-cache \ - --hicache-ratio 2 \ - --hicache-mem-layout page_first_direct \ - --hicache-io-backend direct \ - --hicache-storage-backend mooncake \ - --hicache-write-policy write_through \ - --hicache-storage-prefetch-policy timeout -``` - - -## Custom Storage Backend Integration - -To integrate a new storage backend: - -1. **Implement three core methods:** - - `get(key)`: Retrieve value by key - - `exists(key)`: Check key existence - - `set(key, value)`: Store key-value pair - -2. **Register your backend:** Add your storage backend to the HiCache [BackendFactory](../../python/sglang/srt/mem_cache/storage/backend_factory.py#L188) - -The HiCache controller handles all scheduling and synchronization automatically. - -### Dynamic Backend Loading - -Alternatively, you can use dynamic loading to avoid hard-coding your backend in the repository: - -```bash -python3 -m sglang.launch_server \ - --model-path your-model \ - --enable-hierarchical-cache \ - --hicache-storage-backend dynamic \ - --hicache-storage-backend-extra-config '{"backend_name":"custom_backend_name", "module_path": "your_module_path", "class_name": "YourHiCacheClassName"}' -``` - -**Configuration Parameters:** -- `--hicache-storage-backend`: Set to `dynamic` -- `--hicache-storage-backend-extra-config`: JSON configuration with: - - `backend_name`: Custom backend identifier - - `module_path`: Python module path to your implementation - - `class_name`: Your HiCache implementation class name - - `interface_v1`: 0 (disable) or 1 (enable) to control usage of batch_get_v1 and batch_set_v1 methods - - -## Community and Support - -- **GitHub Issues**: Report bugs and feature requests -- **Slack Channel**: Join community discussions in #sgl-kv-cache-store -- **Documentation**: Refer to storage backend-specific guides - ---- - -*This document will be continuously updated based on community feedback and new features. Contributions and suggestions are welcome!* diff --git a/docs/advanced_features/hicache_design.md b/docs/advanced_features/hicache_design.md deleted file mode 100644 index 8996dc6c7..000000000 --- a/docs/advanced_features/hicache_design.md +++ /dev/null @@ -1,157 +0,0 @@ -# HiCache System Design and Optimization - -This document provides a comprehensive overview of SGLang HiCache, covering its system architecture, workflow and key components. It also details configuration parameters, optimization techniques, and integration with various L3 storage backends, serving as a complete reference for users and developers to understand and tune HiCache for efficient LLM inference. - -## Why and What is HiCache? - -In large language model inference, the prefill phase is often time-consuming: input sequences need to be first converted into Key-Value cache (KV cache) for subsequent decoding. When multiple requests share the same prefix, the KV cache for that prefix is identical. By caching and reusing these shared KV caches, redundant computation can be avoided. To address this, SGLang introduced RadixAttention, which leverages idle GPU memory to cache and reuse prefix KV caches, and **HiCache**, which extends this idea to host memory and distributed storage. - -Inspired by the classic three-level cache design of modern CPUs, HiCache organizes GPU memory as L1, host memory as L2, and distributed storage as L3. This hierarchy enables HiCache to fully exploit the "idle" storage space of GPUs and CPUs, while integrating distributed cache systems such as Mooncake, 3FS, NIXL, and AIBrix KVCache for global KV cache storage and scheduling. As a result, HiCache significantly expands KV cache capacity while maintaining strong read performance—especially in workloads such as multi-QA and long-context inference, where KV cache reuse is frequent. For detailed benchmark results, see [this blog](https://lmsys.org/blog/2025-09-10-sglang-hicache/). - - -## System Design - -### Overall Architecture - -In many modern CPU architectures, the small but fast L1 and L2 caches are private to each core, enabling rapid access to the hottest data, while the larger L3 cache is shared across all cores to significantly reduce redundancy within the cache. Similarly, in HiCache, the L1 and L2 KV caches are private to each inference instance, whereas the L3 KV cache is shared among all inference instances within the cluster. - -### HiRadixTree: Metadata Organization in HiCache - -For KV cache data organization, HiCache builds upon the RadixTree structure introduced in RadixAttention and proposes HiRadixTree. In RadixAttention, each node of the RadixTree corresponds to the KV cache of a consecutive span of tokens in GPU memory. A path from the root to a leaf node represents the prefix of a request, and shared prefixes across multiple requests can reuse the same nodes, thereby avoiding redundant storage. - -HiRadixTree extends this idea: each node corresponds to the KV cache of a span of consecutive tokens and records where that KV cache is stored—whether in local GPU memory, CPU memory, L3 storage, or multiple of these tiers. If stored locally, HiRadixTree maintains precise metadata, including the exact storage address. However, to reduce overhead, HiRadixTree does not store or continuously synchronize metadata for L3 KV cache. Instead, when accessing L3 data, it queries the backend in real time to retrieve the necessary metadata, such as whether the data exists and on which server and location it resides. - -### Overall Workflow - -The workflow of HiCache mainly involves three key operations: **local match**, **prefetch** and **write-back**. When the system receives a new request, it first searches the local L1 and L2 caches for matching KV caches. For parts not found locally, it attempts to prefetch from L3. After prefetching, all required KV caches are loaded into the GPU for computation. Once the prefill computation is complete, the system considers storing the newly generated data into L2 or L3. - -![HiCache Workflow](https://lmsys.org/images/blog/hicache/hicache_overview.png) - -### Local Match - -Local matching is the first step in HiCache's workflow, where incoming request tokens are matched against the HiRadixTree to locate cached KV data in local memory tiers (L1 GPU memory and L2 host memory). - -The matching algorithm traverses the HiRadixTree from the root node, following child nodes that match the token sequence prefix. At each node, the incoming token sequence is compared with the node’s stored token sequence. When `page_size > 1`, matching is performed at the page granularity to optimize memory access patterns. If a match terminates within a node’s stored sequence, the node is automatically split to create an exact boundary, improving the efficiency of future matches. - -The algorithm returns a continuous prefix of the request, with the first part residing in L1 and the latter part in L2. - -Since the process only requires traversing the local HiRadixTree and does not involve any actual data copying, local matching is extremely fast. - -### Prefetch from L3 - -Data prefetching is one of HiCache’s core optimization techniques, designed to proactively load KV caches from L3 storage into local L2 memory, thereby reducing access latency during subsequent operations. - -**Prefetch Trigger Conditions**: -After local matching, for the parts not found in L1 or L2, the system queries L3 to retrieve metadata for the next continuous matching KV caches. If the length of hit cache in L3 exceeds a threshold (default: 256 tokens, configurable), a prefetch operation is triggered. - -**Prefetch Strategies**: HiCache provides three different prefetch termination strategies to address different scenario needs: -- **best_effort**: Terminates immediately when GPU can execute prefill computation, with no waiting time, suitable for scenarios extremely sensitive to latency. -- **wait_complete**: Must wait for all prefetch operations to complete, suitable for scenarios requiring high cache hit rates. -- **timeout**: Terminates after specified time or when complete, balancing latency and cache hit rate needs. - -After prefetching stops, the data already fetched is used together with the local data for the prefill computation. - -For **timeout** strategy, HiCache introduces two configuration parameters to support fine-grained control over prefetch timeout conditions: - -* `prefetch_timeout_base`: the base timeout, representing overhead unrelated to the number of tokens (e.g., scheduling and synchronization). -* `prefetch_timeout_per_ki_token`: the incremental timeout per thousand tokens. - -The timeout is computed as: - -``` -timeout = prefetch_timeout_base + prefetch_timeout_per_ki_token * num_token_to_fetch / 1024 -``` - -### Data Write-back - -The write-back mechanism is responsible for moving frequently accessed KV caches from L1 to L2 and L3, enabling larger and longer-term storage as well as cache sharing across instances. - -**Configurable Write-back Policies**: HiCache supports three write-back strategies: - -* **write_through**: Every access is immediately written back to the next level. When bandwidth is sufficient, this strategy provides the strongest caching benefit. -* **write_through_selective**: Data is written back only after the access frequency exceeds a threshold. This strategy backs up only hot data, reducing I/O overhead. -* **write_back**: Data is written back to the next level only when it is evicted from the upper level. This strategy alleviates storage pressure and is suitable for scenarios where storage capacity is limited but memory utilization must be maximized. - -**Cross-instance Sharing**: When data is written back from L2 to L3, only data not already present in L3 is transferred. KV caches stored in L3 can then be shared across all SGLang instances in the cluster (depending on the L3 backend implementation), significantly improving cache hit rates within the same memory budget. - -### Multi-Rank Synchronization - -During multi-GPU parallel computation, such as tensor parallelism (TP), HiCache must ensure consistent states across different ranks. Therefore, critical computation steps require the use of `all_reduce` for state synchronization. - -For example, during prefetching, `all_reduce(op=min)` is used to ensure that all ranks obtain the same number of L3 hits, preventing inconsistent judgments about whether the prefetch threshold has been reached. Similarly, after prefetching completes or terminates, `all_reduce(op=min)` is again required to guarantee consensus among ranks on the prefix length of the successfully retrieved KV cache. - -### Data Transfer Optimization - -**Zero-Copy Data Transfers**: Both prefetching and write-back involve substantial data movement. Minimizing the number of data copies can significantly improve system performance. HiCache supports passing memory addresses and sizes directly when transferring data from L2 memory to an L3 backend. - -**“Batch-Oriented” Data Organization**: The granularity of data reads and writes has a major impact on performance. To address this, HiCache L3 stores and transfers KV cache data at the granularity of **pages** and supports different data layouts beyond the existing `layer first` scheme, including `page first` and `page first direct`. Under the `page first` and `page first direct` layouts, all KV cache data belonging to the same page is placed in contiguous memory, allowing it to be passed as a single object to L3 using zero-copy transfers. - -![HiCache L2 MEM layout](https://lmsys.org/images/blog/hicache/hicache_layout.png) - -However, because GPU KV computation is naturally performed layer by layer, the GPU inherently operates in a `layer first` layout. When transferring `page first` data from L2 to the GPU, data must be transferred at the granularity of one token per layer. The `page first direct` layout mitigates this issue by grouping together all tokens of a given layer within a page, allowing transfers from L2 to GPU to be aggregated at the page-layer level. - -**CPU-to-GPU Transfer Optimizations**: In HiCache, moving data from CPU memory to GPU is as performance-critical as prefetching data from L3 to L2. HiCache employs several optimizations for this process: - -* **Compute-Transfer Overlap**: During the prefill phase, when transferring data from CPU to GPU, HiCache overlaps layers by concurrently loading the KV cache of layer N+1 while computing layer N. This effectively hides data transfer latency. -* **GPU-assisted I/O Kernels**: On top of `cudaMemcpyAsync`, HiCache implements a set of GPU-assisted I/O kernels specifically optimized for KV cache transfers between CPU and GPU. Compared to the baseline approach, these kernels achieve up to 3x higher transfer speed. - -**Write-back Optimization for MLA**: For MHA (Multi-Head Attention) models under multi-TP, each rank holds `1/tp_size` of a token’s KV data. In contrast, for MLA (Multi-Layer Attention) models, all ranks hold the complete and identical KV data for each token. HiCache includes a dedicated optimization for MLA: only one rank initiates the write-back operation, ensuring that data is not redundantly stored across ranks. - -### Integration with PD-Disaggregation Deployment Mode - -SGLang supports a PD (Prefill-Decode) disaggregation deployment mode through the Mooncake TransferEngine (for details, see [this doc](https://docs.sglang.io/advanced_features/pd_disaggregation.html)). In the PD-disaggregation deployment mode, HiCache can be enabled on both the prefill nodes and decode nodes to optimize prefill performance. If enabled on decode nodes, the decode output will also be written back to L3. - -### Unified Interfaces and Rich L3 Storage Backends - -HiCache encapsulates all read, write, and query operations on L3 backends within the `class HiCacheStorage(ABC)`, exposing a set of simple and consistent interfaces. This design supports a wide range of L3 storage backends and allows users to select the one that best fits their specific use cases. - -- **Mooncake**: Mooncake is a high-performance caching system for LLM inference that leverages RDMA and multi-NIC resources to enable zero-copy, ultra-fast data transfers. Try Mooncake [here](https://github.com/sgl-project/sglang/tree/main/python/sglang/srt/mem_cache/storage/mooncake_store). - -- **DeepSeek 3FS (HF3FS)**: HF3FS is a Kubernetes-native distributed storage solution with operator-based deployment. Try HF3FS [here](https://github.com/sgl-project/sglang/tree/main/python/sglang/srt/mem_cache/storage/hf3fs). - -- **NIXL**: NIXL provides a unified API for accessing various storage plugins, including but not limited to DeepSeek's 3FS, GPU Direct Storage (GDS) and Amazon S3-compatible object storage. Try NIXL [here](https://github.com/sgl-project/sglang/tree/main/python/sglang/srt/mem_cache/storage/nixl). - -- **AIBrix KVCache**: AIBrix KVCache is a production-ready KVCache Offloading Framework, which enables efficient memory tiering and low-overhead cross-engine reuse. Try AIBrix KVCache [here](https://github.com/sgl-project/sglang/tree/main/python/sglang/srt/mem_cache/storage/aibrix_kvcache). - -- **HiCacheFile**: A simple file-based storage backend for demonstration purposes. - -Specifically, **LMCache**, an efficient KV cache layer for enterprise-scale LLM inference, provides an alternative solution to HiCache. Try LMCache [here](https://github.com/sgl-project/sglang/tree/main/python/sglang/srt/mem_cache/storage/lmcache). - -## Related Parameters - -- **`--enable-hierarchical-cache`**: Enable hierarchical cache functionality. This is required to use HiCache. - -- **`--hicache-ratio HICACHE_RATIO`**: The ratio of the size of host KV cache memory pool to the size of device pool. For example, a value of 2 means the host memory pool is twice as large as the device memory pool. The value of this parameter must be greater than 1, as the current implementation requires the host memory allocated for the KV cache to be larger than the device memory allocated for the KV cache. - -- **`--hicache-size HICACHE_SIZE`**: The size of host KV cache memory pool in gigabytes. This parameter overrides `hicache-ratio` if set. For example, `--hicache-size 30` allocates 30GB (1GB = 1e9 bytes) for the host memory pool **for each rank**. If there are 8 ranks, then the total memory size is 240GB. Just like `hicache-ratio`, the value of this parameter must be larger than the size of device memory allocated for KV cache. - -**Note**: `--hicache-ratio` and `--hicache-size` are two critical parameters. In general, a larger HiCache size leads to a higher cache hit rate, which improves prefill performance. However, the relationship between cache size and hit rate is not linear. Once most reusable KV data—especially hot tokens—are already cached, further increasing the size may yield only marginal performance gains. Users can set these parameters based on their workload characteristics and performance requirements. - -- **`--page-size PAGE_SIZE`**: The number of tokens per page. This parameter determines the granularity of KV cache storage and retrieval. Larger page sizes reduce metadata overhead and improve I/O efficiency for storage backends, but may lower the cache hit rate when only part of a page matches the stored KV cache. For workloads with long common prefixes, larger pages can improve performance, while workloads with more diverse prefixes may benefit from smaller pages. See [Data Transfer Optimization](#data-transfer-optimization) for how page granularity affects I/O performance. - -- **`--hicache-storage-prefetch-policy {best_effort,wait_complete,timeout}`**: Controls when prefetching from storage should stop. See [Prefetch from L3](#prefetch-from-l3) for details. - - `best_effort`: Prefetch as much as possible without blocking - - `wait_complete`: Wait for prefetch to complete before proceeding - - `timeout`: Terminates after specified time or when complete (Recommended for production environments, as setting an appropriate timeout helps the system meet required SLOs) - -- **`--hicache-write-policy {write_back,write_through,write_through_selective}`**: Controls how data is written from faster to slower memory tiers. See [Data Write-back](#data-write-back) for details. - - `write_through`: Immediately writes data to all tiers (strongest caching benefits) - - `write_through_selective`: Uses hit-count tracking to back up only frequently accessed data - - `write_back`: Writes data back to slower tiers only when eviction is needed (reduces I/O load) - -- **`--hicache-io-backend {direct,kernel}`**: Choose the I/O backend for KV cache transfer between CPU and GPU. See [Data Transfer Optimization](#data-transfer-optimization) for details. - - `direct`: Standard CUDA memory copy operations - - `kernel`: GPU-assisted I/O kernels (recommended for better performance) - -- **`--hicache-mem-layout {layer_first,page_first,page_first_direct}`**: Memory layout for the host memory pool. See [Data Transfer Optimization](#data-transfer-optimization) for details. - - `layer_first`: Compatible with GPU computation kernels (default for GPU memory) - - `page_first`: Optimized for I/O efficiency - - `page_first_direct`: Groups all tokens of a given layer within a page, allowing transfers from L2 to GPU to be aggregated at the page-layer level - -- **`--hicache-storage-backend {file,mooncake,hf3fs,nixl,aibrix,dynamic}`**: Choose the storage backend for the L3 tier. Built-in backends: file, mooncake, hf3fs, nixl, aibrix. For dynamic backend, use --hicache-storage-backend-extra-config to specify: `backend_name` (custom name), `module_path` (Python module path), `class_name` (backend class name). See [Unified Interfaces and Rich L3 Storage Backends](#unified-interfaces-and-rich-l3-storage-backends) for available backends. - -- **`--enable-lmcache`**: Using LMCache as an alternative hierarchical cache solution. - -- **`--hicache-storage-backend-extra-config HICACHE_STORAGE_BACKEND_EXTRA_CONFIG`**: the extra config can be either - - a JSON string containing extra configuration for the storage backend, e.g., `--hicache-storage-backend-extra-config '{"prefetch_threshold":512, "prefetch_timeout_base": 0.5, "prefetch_timeout_per_ki_token": 0.25}' `, or - - a TOML or JSON or YAML file specifying the extra configuration for the storage backend (to differentiate from the JSON string input, prepend a `@` in front of the file name), e.g., `--hicache-storage-backend-extra-config "@config.toml"` where `config.toml` is the config file containing the complex configurations. This can be useful when the configuration consists of many or complex key-value pairs (for instance, it is preferred to use a config file for NIXL backend as its configurations can be complex). diff --git a/docs/advanced_features/hicache_storage_runtime_attach_detach.md b/docs/advanced_features/hicache_storage_runtime_attach_detach.md deleted file mode 100644 index 555d799c2..000000000 --- a/docs/advanced_features/hicache_storage_runtime_attach_detach.md +++ /dev/null @@ -1,132 +0,0 @@ -# Runtime Attach/Detach HiCache Storage Backend (No Restart) - -This document explains how to **dynamically attach/detach the HiCache L3 storage backend at runtime** (e.g., `mooncake` / `hf3fs` / `nixl` / `file` / `aibrix` / `eic`) while **SGLang is already running and serving traffic**, without restarting the process. - -For safety and consistency, the current implementation **strictly requires** these operations to happen only when the service is **idle**: - -- **No running requests** -- **No waiting/queued requests** - -If the idle condition is not met, the API will fail fast (HTTP 400) and **will not modify** the current service state. - ---- - -## 1. Background and implementation overview - -### 1.1 Architecture / control path - -The control path is: - -1. **HTTP Server** (`python/sglang/srt/entrypoints/http_server.py`) - - Exposes `PUT /hicache/storage-backend`, `DELETE /hicache/storage-backend`, `GET /hicache/storage-backend` -2. **TokenizerManager** (`python/sglang/srt/managers/tokenizer_control_mixin.py`) - - Sends the request to the Scheduler via `FanOutCommunicator` -3. **Scheduler** (`python/sglang/srt/managers/scheduler.py`) - - Performs a **strict idle check** - - Calls `tree_cache.attach_storage_backend(...)` / `detach_storage_backend(...)` -4. **HiRadixCache** (`python/sglang/srt/mem_cache/hiradix_cache.py`) - - Parses `hicache_storage_backend_extra_config_json` (supports both backend config and prefetch knobs) - - Calls `cache_controller.attach_storage_backend(...)` / `detach_storage_backend(...)` -5. **HiCacheController** (`python/sglang/srt/managers/cache_controller.py`) - - Creates/destroys the storage backend instance (via `StorageBackendFactory`) - - Starts/stops backend background threads at runtime (prefetch/backup) - ---- - -## 2. Idle-state requirement (strict) - -The Scheduler uses `is_fully_idle()` which checks: - -- No running batches (including chunked prefill, overlap, pipeline-parallel, and disaggregation paths) -- No waiting requests in any queue (waiting, grammar, disagg bootstrap/prealloc/transfer/inflight) -- No DLLM staging requests - -If the condition is not met, attach/detach returns an error like: - -- `Reject attach: scheduler is not idle. #queue-req=... #running-req=...` - -> Tip: before switching, drain upstream traffic and wait for the server to become idle, then call attach/detach. - -### 2.1 DP (data parallel) semantics - -When `dp_size > 1`, the tokenizer dispatches the request to **all DP scheduler instances** and aggregates their responses: - -- The final `success` is **true only if all DP ranks return success** -- The final `message` concatenates messages from all DP ranks - -This is intended to prevent “silent partial success”, but it also means you may see: - -- Overall **failure** even though **some ranks already succeeded** - -Currently there is **no automatic partial rollback** across DP ranks (see TODO in code). Operationally: - -- Prefer to keep backend config identical across ranks -- If attach fails, immediately call detach (best-effort/idempotent), fix config, then retry attach - ---- - -## 3. How to use (HTTP Admin API) - -The examples below assume your SGLang HTTP server is at `http://127.0.0.1:30000`. - -### 3.1 Query current storage backend status - -```bash -curl -s http://127.0.0.1:30000/hicache/storage-backend -``` - -Example response: - -```json -{ - "hicache_storage_backend": "mooncake", - "hicache_storage_backend_extra_config": "{\"master_server_address\":\"127.0.0.1:50051\", ...}" -} -``` - -### 3.2 Attach (enable) a storage backend -```bash -curl -s -X PUT http://127.0.0.1:30000/hicache/storage-backend \ - -H 'Content-Type: application/json' \ - -d '{ - "hicache_storage_backend": "mooncake" - }' -``` - -```bash -curl -s -X PUT http://127.0.0.1:30000/hicache/storage-backend \ - -H 'Content-Type: application/json' \ - -d '{ - "hicache_storage_backend": "mooncake", - "hicache_storage_backend_extra_config_json": "{\"master_server_address\":\"127.0.0.1:50051\",\"protocol\":\"tcp\",\"global_segment_size\":\"4gb\",\"prefetch_threshold\":256}", - "hicache_storage_prefetch_policy": "timeout" - }' -``` - -Notes: - -- `hicache_storage_backend_extra_config_json` can include both: - - **Backend configuration** (e.g., Mooncake master/metadata/protocol, etc.) - - **Prefetch configuration** (`prefetch_threshold`, `prefetch_timeout_base`, `prefetch_timeout_per_ki_token`, `hicache_storage_pass_prefix_keys`) - -### 3.3 Detach (disable) the storage backend - -```bash -curl -s -X DELETE http://127.0.0.1:30000/hicache/storage-backend -``` - -Notes: - -- Detach only makes SGLang **stop using** the L3 storage backend and stops prefetch/backup threads -- It **does not automatically delete** data stored in Mooncake/HF3FS (or other remote backends) - ---- - -## 4. Behavior and caveats - -- **No restart required**: attach/detach switches in-process at runtime -- **Must be idle**: otherwise the request is rejected to avoid consistency issues -- **Host KV layout constraints still apply**: for example, Mooncake still requires layouts like `page_first/page_first_direct/page_head`; if the server's HiCache host-memory layout does not satisfy the backend requirements, attach will fail with an error -- **Observability**: - - After attach, `server_args.hicache_storage_backend*` is updated on both the tokenizer and scheduler sides - - If metrics are enabled, attach will create a storage metrics collector in `HiRadixCache` on demand diff --git a/docs/advanced_features/hisparse_guide.md b/docs/advanced_features/hisparse_guide.md deleted file mode 100644 index 675bbb05e..000000000 --- a/docs/advanced_features/hisparse_guide.md +++ /dev/null @@ -1,136 +0,0 @@ -# HiSparse: Hierarchical Sparse Attention - -HiSparse reduces per-request GPU memory consumption during the decode phase by maintaining only a small "hot" KV buffer on GPU while keeping complete KV data in CPU pinned memory. Combined with PD disaggregation, it enables significantly higher decode concurrency. - -> **Prerequisites**: HiSparse works with models that use **DeepSeek Sparse Attention (DSA)** architectures (e.g., DeepSeek-V3.2, GLM-5.1) and **DeepSeek V4**. These models natively select a subset of tokens for attention, making it possible to keep only the top-k KV on GPU while storing the full KV in host memory — without accuracy loss. Additionally, HiSparse currently requires **PD disaggregation mode** and is enabled on the **decode instance** only. - -## Why HiSparse? - -In long-context LLM inference, each decoding request holds a full-length KV cache on GPU, limiting the number of concurrent requests a decode instance can serve. HiSparse addresses this by: - -- **Reducing GPU memory per request**: Each request occupies only a fixed-size device buffer (e.g., 4KB tokens) instead of the full sequence length. -- **On-demand swap-in**: A CUDA kernel dynamically loads the top-k most relevant KV entries from host memory based on attention scores. -- **Transparent to prefill**: HiSparse is entirely a decode-side optimization; the prefill instance requires no changes. - -## Design Overview - -### Decode Workflow - -Each decode step follows this flow: - -1. **Forward decode** — generate the next token -2. **Top-k selection** — select the most relevant token positions via attention scores -3. **Swap-in** — the CUDA kernel loads top-k KV entries from host to device buffer: - - *Short sequences* (`seq_len ≤ device_buffer_size`): fast path, all KV already in buffer - - *Long sequences*: hit detection → LRU reordering → miss handling (host → device copy) -4. **Decode attention** — compute attention using the top-k device locations -5. **Eager backup** — asynchronously copy the previous token's KV from device to host - -### PD Disaggregation Integration (Direct-to-Host) - -In PD disaggregation mode, the prefill instance transfers KV cache directly into the decode instance's host pool via RDMA, bypassing the GPU entirely on the decode side. This eliminates the transient GPU memory spike during KV transfer and removes the staging DMA step. - -``` -Prefill GPU ──RDMA──▶ Decode Host Pool (CPU pinned memory) - │ - ▼ - alloc device buffer (4KB) - │ - ▼ - swap-in kernel (on-demand top-k) -``` - -For DeepSeek V4, the direct-to-host path writes only C4 KV into the decode host pool. The c4_indexer and C128 KV remain device-to-device transfers. - -## Server Arguments - -| Argument | Type / Default | Description | -|----------|---------------|-------------| -| `--enable-hisparse` | flag; default: disabled | Enable HiSparse on the decode instance | -| `--hisparse-config` | JSON string | Configuration for HiSparse (see below) | - -### HiSparse Config Parameters - -Pass as a JSON string via `--hisparse-config`: - -| Parameter | Type / Default | Description | -|-----------|---------------|-------------| -| `top_k` | int | Number of topk entries | -| `device_buffer_size` | int | Number of token slots in the per-request GPU device buffer | -| `host_to_device_ratio` | int | Ratio of logical pool size to device pool size, determining host memory capacity | - -Example: `--hisparse-config='{"top_k": 2048, "device_buffer_size": 6144, "host_to_device_ratio": 10}'` - -## Deployment - -HiSparse currently requires **PD disaggregation mode** and is enabled only on the **decode instance**. - -### Prefill Instance - -```bash -python3 -m sglang.launch_server \ - --model-path /path/to/model \ - --trust-remote-code \ - --port 8000 --host 0.0.0.0 \ - --context-length 81920 \ - --chunked-prefill-size 65536 \ - --tp-size 8 --dp-size 8 --enable-dp-attention \ - --mem-fraction-static 0.85 \ - --disaggregation-mode prefill \ - --disaggregation-ib-device mlx5_0,mlx5_1,mlx5_2,mlx5_3 \ - --nnodes 1 --node-rank 0 -``` - -### Decode Instance (with HiSparse) - -```bash -python3 -m sglang.launch_server \ - --model-path /path/to/model \ - --trust-remote-code \ - --port 8000 --host 0.0.0.0 \ - --context-length 81920 \ - --tp-size 8 --dp-size 8 --enable-dp-attention \ - --mem-fraction-static 0.85 \ - --disable-radix-cache \ - --disaggregation-mode decode \ - --disaggregation-ib-device mlx5_0,mlx5_1,mlx5_2,mlx5_3 \ - --dist-init-addr 127.0.0.1:5757 \ - --nnodes 1 --node-rank 0 \ - --enable-hisparse \ - --hisparse-config='{"top_k": 2048, "device_buffer_size": 6144, "host_to_device_ratio": 10}' -``` - -> **Note**: For DSA models, `--kv-cache-dtype` defaults to `auto`, which resolves to `fp8_e4m3` on SM100+ (Blackwell) and `bfloat16` on older architectures. The DSA decode backend is automatically selected based on KV dtype (`bfloat16` → `flashmla_sparse`, `fp8_e4m3` → `flashmla_kv`). DSA backend flags apply only to DSA models; DeepSeek V4 uses its own `dsv4` attention backend. - -### Benchmark - -```bash -python3 -m sglang.bench_serving \ - --backend sglang \ - --dataset-path /path/to/ShareGPT_V3_unfiltered_cleaned_split.json \ - --dataset-name random \ - --random-input 40000 \ - --random-output 20000 \ - --num-prompts 200 \ - --max-concurrency 200 \ - --request-rate 40 \ - --random-range-ratio 1.0 \ - --host 127.0.0.1 \ - --port 20000 \ - --model /path/to/model \ - --flush-cache \ -``` - -### Key Notes - -- The prefill instance does not need `--enable-hisparse`; it is unaware of HiSparse. -- On the decode instance, `--enable-hisparse` and `--hisparse-config` are required for HiSparse. -- For DSA models, `--kv-cache-dtype bfloat16` uses `flashmla_sparse`, and `--kv-cache-dtype fp8_e4m3` uses `flashmla_kv`. -- For DeepSeek V4, DSA backend flags are not applicable. DeepSeek V4 uses the `dsv4` attention backend and `fp8_e4m3` KV cache by default. -- `host_to_device_ratio` should be configured based on the host machine's available memory. For example: - - **~1 TB** host memory → `host_to_device_ratio: 5` - - **~2 TB** host memory → `host_to_device_ratio: 10` - -## Acknowledgments - -We would like to thank the SGLang team and community for the implementation and generous support, especially Zhiqiang Xie, Zhangheng Huang, Tingwei Huang, Shangming Cai, Teng Ma, and many others. We also thank the Alibaba Cloud TairKVCache team and the AntGroup SCT Inference team for their valuable contributions. diff --git a/docs/advanced_features/hyperparameter_tuning.md b/docs/advanced_features/hyperparameter_tuning.md deleted file mode 100644 index 2cdae90b6..000000000 --- a/docs/advanced_features/hyperparameter_tuning.md +++ /dev/null @@ -1,77 +0,0 @@ -# Hyperparameter Tuning - -## Achieving high throughput for offline batch inference - -Achieving a large batch size is the most important thing for attaining high throughput in offline batch inference. -When the server is running at full load in a steady state, look for the following in the log: - -```Decode batch. #running-req: 233, #token: 370959, token usage: 0.82, cuda graph: True, gen throughput (token/s): 4594.01, #queue-req: 317``` - -### Adjust the request submission speed to control `#queue-req` - -`#queue-req` indicates the number of requests in the queue. -If you frequently see `#queue-req: 0`, it suggests that your client code is submitting requests too slowly. -A healthy range for `#queue-req` is `100 - 2000`. -However, avoid making `#queue-req` too large, as this will increase the scheduling overhead on the server. - -### Achieve a high `token usage` - -`token usage` indicates the KV cache memory utilization of the server. `token usage > 0.9` means good utilization. - -If you frequently see `token usage < 0.9` and `#queue-req > 0`, it means the server is too conservative about taking in new requests. You can decrease `--schedule-conservativeness` to a value like 0.3. -The case of a server being too conservative can happen when users send many requests with a large `max_new_tokens` but the requests stop very early due to EOS or stop strings. - -On the other hand, if you see `token usage` very high and you frequently see warnings like -`KV cache pool is full. Retract requests. #retracted_reqs: 1, #new_token_ratio: 0.9998 -> 1.0000`, you can increase `--schedule-conservativeness` to a value like 1.3. -If you see `KV cache pool is full. Retract requests.` occasionally but not frequently (~1 time per minute), it is okay. - -### Tune `--mem-fraction-static` to increase KV cache pool capacity -SGLang allocates memory as follows: - -Total memory usage = model weights + KV cache pool + CUDA graph buffers + activations - -The `--mem-fraction-static` parameter determines how much memory is allocated to the first two components: - -mem_fraction_static = (model weights + KV cache pool) / GPU memory capacity - -To support higher concurrency, you should maximize the KV cache pool capacity by setting `--mem-fraction-static` as high as possible while still reserving enough memory for activations and CUDA graph buffers. - -SGLang uses simple heuristics to set the default value of `--mem-fraction-static`, but you can optimize it for your use cases. -As a rule of thumb, reserving 5–8 GB of memory for activations is typically sufficient. You can check this by inspecting the logs just before the server is ready. -Look for log entries like this: - -``` -[2025-08-11 17:17:03] max_total_num_tokens=665690, chunked_prefill_size=8192, max_prefill_tokens=16384, max_running_requests=4096, context_len=65536, available_gpu_mem=13.50 GB -``` - -Check the `available_gpu_mem` value. -- If it is between 5–8 GB, the setting is good. -- If it is too high (e.g., 10 - 20 GB), increase `--mem-fraction-static` to allocate more memory to the KV cache. -- If it is too low, you risk out-of-memory (OOM) errors later, so decrease `--mem-fraction-static`. - -Another straightforward approach is to increase `--mem-fraction-static` in increments of 0.01 until you encounter OOM errors for your workloads. - -### Avoid out-of-memory errors by tuning `--chunked-prefill-size`, `--mem-fraction-static`, and `--max-running-requests` - -If you encounter out-of-memory (OOM) errors, you can adjust the following parameters: - -- If OOM occurs during prefill, try reducing `--chunked-prefill-size` to `4096` or `2048`. This saves memory but slows down the prefill speed for long prompts. -- If OOM occurs during decoding, try lowering `--max-running-requests`. -- You can also reduce `--mem-fraction-static` to a smaller value, such as 0.8 or 0.7. This decreases the memory usage of the KV cache memory pool and helps prevent OOM errors during both prefill and decoding. However, it limits maximum concurrency and reduces peak throughput. - -### Tune `--cuda-graph-max-bs` -By default, CUDA graph is enabled only for small batch sizes (e.g., less than 160 or 256). -However, for some models, especially at large tensor parallelism sizes, CUDA graph can be useful for batch sizes up to 512 or 768. -Therefore, it may be beneficial to increase `--cuda-graph-max-bs` to a larger value. -Note that CUDA graph consumes more memory, so you may need to reduce `--mem-fraction-static` at the same time. - -### Tune `--dp-size` and `--tp-size` - -Data parallelism is better for throughput. When there is enough GPU memory, always favor data parallelism for throughput. Refer to [SGLang Model Gateway (former Router)](../advanced_features/sgl_model_gateway.md) for a better data parallelism rather than using `dp_size` parameter. - -### Try other options - -- `torch.compile` accelerates small models on small batch sizes. You can enable it with `--enable-torch-compile`. -- Try other quantization (e.g. FP8 quantization with `--quantization fp8`) -- Try other parallelism strategies (e.g. [expert parallelism](https://lmsys.org/blog/2025-05-05-large-scale-ep/)) or DP attention for deepseek models (with `--enable-dp-attention --dp-size 8`). -- If the workload has many shared prefixes, try `--schedule-policy lpm`. Here, `lpm` stands for longest prefix match. It reorders requests to encourage more cache hits but introduces more scheduling overhead. diff --git a/docs/advanced_features/lora.ipynb b/docs/advanced_features/lora.ipynb deleted file mode 100644 index 230bd700f..000000000 --- a/docs/advanced_features/lora.ipynb +++ /dev/null @@ -1,716 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# LoRA Serving" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "SGLang enables the use of [LoRA adapters](https://arxiv.org/abs/2106.09685) with a base model. By incorporating techniques from [S-LoRA](https://arxiv.org/pdf/2311.03285) and [Punica](https://arxiv.org/pdf/2310.18547), SGLang can efficiently support multiple LoRA adapters for different sequences within a single batch of inputs." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Arguments for LoRA Serving" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The following server arguments are relevant for multi-LoRA serving:\n", - "\n", - "* `enable_lora`: Enable LoRA support for the model. This argument is automatically set to True if `--lora-paths` is provided for backward compatibility.\n", - "\n", - "* `enable_lora_overlap_loading`: Enable asynchronous LoRA weight loading in order to overlap H2D transfers with GPU compute. This should be enabled if you find that your LoRA workloads are bottlenecked by adapter weight loading, for example when frequently loading large LoRA adapters.\n", - "\n", - "* `lora_paths`: The list of LoRA adapters to load. Each adapter must be specified in one of the following formats: | = | JSON with schema {\"lora_name\":str,\"lora_path\":str,\"pinned\":bool}.\n", - "\n", - "* `max_loras_per_batch`: Maximum number of adaptors used by each batch. This argument can affect the amount of GPU memory reserved for multi-LoRA serving, so it should be set to a smaller value when memory is scarce. Defaults to be 8.\n", - "\n", - "* `max_loaded_loras`: If specified, it limits the maximum number of LoRA adapters loaded in CPU memory at a time. The value must be greater than or equal to `max-loras-per-batch`.\n", - "\n", - "* `lora_eviction_policy`: LoRA adapter eviction policy when GPU memory pool is full. `lru`: Least Recently Used (default, better cache efficiency). `fifo`: First-In-First-Out.\n", - "\n", - "* `lora_backend`: The backend of running GEMM kernels for Lora modules. Currently we support Triton LoRA backend (`triton`) and Chunked SGMV backend (`csgmv`). In the future, faster backend built upon Cutlass or Cuda kernels will be added.\n", - "\n", - "* `max_lora_rank`: The maximum LoRA rank that should be supported. If not specified, it will be automatically inferred from the adapters provided in `--lora-paths`. This argument is needed when you expect to dynamically load adapters of larger LoRA rank after server startup.\n", - "\n", - "* `lora_target_modules`: The union set of all target modules where LoRA should be applied (e.g., `q_proj`, `k_proj`, `gate_proj`). If not specified, it will be automatically inferred from the adapters provided in `--lora-paths`. This argument is needed when you expect to dynamically load adapters of different target modules after server startup. You can also set it to `all` to enable LoRA for all supported modules. However, enabling LoRA on additional modules introduces a minor performance overhead. If your application is performance-sensitive, we recommend only specifying the modules for which you plan to load adapters.\n", - "\n", - "* `--max-lora-chunk-size`: Maximum chunk size for the ChunkedSGMV LoRA backend. Only used when --lora-backend is 'csgmv'. Choosing a larger value might improve performance. Please tune this value based on your hardware and workload as needed. Defaults to 16.\n", - "\n", - "* `lora_drain_wait_threshold`: When any LoRA adapter request waits longer than this threshold (in seconds), the scheduler will selectively drain one running adapter to make room. This mitigates extreme tail latency under high or skewed workloads by preventing a small set of adapters from monopolizing batch slots. Set to 0 to disable draining (default).\n", - "\n", - "* `tp_size`: LoRA serving along with Tensor Parallelism is supported by SGLang. `tp_size` controls the number of GPUs for tensor parallelism. More details on the tensor sharding strategy can be found in [S-Lora](https://arxiv.org/pdf/2311.03285) paper.\n", - "\n", - "From client side, the user needs to provide a list of strings as input batch, and a list of adaptor names that each input sequence corresponds to." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Usage\n", - "\n", - "### Serving Single Adaptor" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**Note:** SGLang supports LoRA adapters through two APIs:\n", - "\n", - "1. **OpenAI-Compatible API** (`/v1/chat/completions`, `/v1/completions`): Use the `model:adapter-name` syntax. See [OpenAI API with LoRA](../basic_usage/openai_api_completions.ipynb#Using-LoRA-Adapters) for examples.\n", - "\n", - "2. **Native API** (`/generate`): Pass `lora_path` in the request body (shown below)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import json\n", - "import requests\n", - "\n", - "from sglang.test.doc_patch import launch_server_cmd\n", - "from sglang.utils import wait_for_server, terminate_process" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "server_process, port = launch_server_cmd(\n", - " # Here we set max-loras-per-batch to 2: one slot for adaptor and another one for base model\n", - " \"\"\"\n", - "python3 -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \\\n", - " --enable-lora \\\n", - " --lora-paths lora0=algoprog/fact-generation-llama-3.1-8b-instruct-lora \\\n", - " --max-loras-per-batch 2 \\\n", - " --log-level warning \\\n", - "\"\"\"\n", - ")\n", - "\n", - "wait_for_server(f\"http://localhost:{port}\", process=server_process)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "url = f\"http://127.0.0.1:{port}\"\n", - "json_data = {\n", - " \"text\": [\n", - " \"List 3 countries and their capitals.\",\n", - " \"List 3 countries and their capitals.\",\n", - " ],\n", - " \"sampling_params\": {\"max_new_tokens\": 32, \"temperature\": 0},\n", - " # The first input uses lora0, and the second input uses the base model\n", - " \"lora_path\": [\"lora0\", None],\n", - "}\n", - "response = requests.post(\n", - " url + \"/generate\",\n", - " json=json_data,\n", - ")\n", - "print(f\"Output 0: {response.json()[0]['text']}\")\n", - "print(f\"Output 1: {response.json()[1]['text']}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "terminate_process(server_process)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Serving Multiple Adaptors" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "server_process, port = launch_server_cmd(\"\"\"\n", - "python3 -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \\\n", - " --enable-lora \\\n", - " --lora-paths lora0=algoprog/fact-generation-llama-3.1-8b-instruct-lora \\\n", - " lora1=Nutanix/Meta-Llama-3.1-8B-Instruct_SFT_lora_4_alpha_16_humaneval_raw_json \\\n", - " --max-loras-per-batch 2 \\\n", - " --log-level warning \\\n", - "\"\"\")\n", - "\n", - "wait_for_server(f\"http://localhost:{port}\", process=server_process)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "url = f\"http://127.0.0.1:{port}\"\n", - "json_data = {\n", - " \"text\": [\n", - " \"List 3 countries and their capitals.\",\n", - " \"List 3 countries and their capitals.\",\n", - " ],\n", - " \"sampling_params\": {\"max_new_tokens\": 32, \"temperature\": 0},\n", - " # The first input uses lora0, and the second input uses lora1\n", - " \"lora_path\": [\"lora0\", \"lora1\"],\n", - "}\n", - "response = requests.post(\n", - " url + \"/generate\",\n", - " json=json_data,\n", - ")\n", - "print(f\"Output 0: {response.json()[0]['text']}\")\n", - "print(f\"Output 1: {response.json()[1]['text']}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "terminate_process(server_process)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Dynamic LoRA loading" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Instead of specifying all adapters during server startup via `--lora-paths`. You can also load & unload LoRA adapters dynamically via the `/load_lora_adapter` and `/unload_lora_adapter` API.\n", - "\n", - "When using dynamic LoRA loading, it's recommended to explicitly specify both `--max-lora-rank` and `--lora-target-modules` at startup. For backward compatibility, SGLang will infer these values from `--lora-paths` if they are not explicitly provided. However, in that case, you would have to ensure that all dynamically loaded adapters share the same shape (rank and target modules) as those in the initial `--lora-paths` or are strictly \"smaller\"." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "lora0 = \"Nutanix/Meta-Llama-3.1-8B-Instruct_SFT_lora_4_alpha_16_humaneval_raw_json\" # rank - 4, target modules - q_proj, k_proj, v_proj, o_proj, gate_proj\n", - "lora1 = \"algoprog/fact-generation-llama-3.1-8b-instruct-lora\" # rank - 64, target modules - q_proj, k_proj, v_proj, o_proj, gate_proj, up_proj, down_proj\n", - "lora0_new = \"philschmid/code-llama-3-1-8b-text-to-sql-lora\" # rank - 256, target modules - q_proj, k_proj, v_proj, o_proj, gate_proj, up_proj, down_proj\n", - "\n", - "\n", - "# The `--target-lora-modules` param below is technically not needed, as the server will infer it from lora0 which already has all the target modules specified.\n", - "# We are adding it here just to demonstrate usage.\n", - "server_process, port = launch_server_cmd(\"\"\"\n", - " python3 -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \\\n", - " --enable-lora \\\n", - " --cuda-graph-max-bs 2 \\\n", - " --max-loras-per-batch 2 \\\n", - " --max-lora-rank 256\n", - " --lora-target-modules all\n", - " --log-level warning\n", - " \"\"\")\n", - "\n", - "url = f\"http://127.0.0.1:{port}\"\n", - "wait_for_server(url, process=server_process)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Load adapter lora0" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "response = requests.post(\n", - " url + \"/load_lora_adapter\",\n", - " json={\n", - " \"lora_name\": \"lora0\",\n", - " \"lora_path\": lora0,\n", - " },\n", - ")\n", - "\n", - "if response.status_code == 200:\n", - " print(\"LoRA adapter loaded successfully.\", response.json())\n", - "else:\n", - " print(\"Failed to load LoRA adapter.\", response.json())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Load adapter lora1:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "response = requests.post(\n", - " url + \"/load_lora_adapter\",\n", - " json={\n", - " \"lora_name\": \"lora1\",\n", - " \"lora_path\": lora1,\n", - " },\n", - ")\n", - "\n", - "if response.status_code == 200:\n", - " print(\"LoRA adapter loaded successfully.\", response.json())\n", - "else:\n", - " print(\"Failed to load LoRA adapter.\", response.json())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Check inference output:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "url = f\"http://127.0.0.1:{port}\"\n", - "json_data = {\n", - " \"text\": [\n", - " \"List 3 countries and their capitals.\",\n", - " \"List 3 countries and their capitals.\",\n", - " ],\n", - " \"sampling_params\": {\"max_new_tokens\": 32, \"temperature\": 0},\n", - " # The first input uses lora0, and the second input uses lora1\n", - " \"lora_path\": [\"lora0\", \"lora1\"],\n", - "}\n", - "response = requests.post(\n", - " url + \"/generate\",\n", - " json=json_data,\n", - ")\n", - "print(f\"Output from lora0: \\n{response.json()[0]['text']}\\n\")\n", - "print(f\"Output from lora1 (updated): \\n{response.json()[1]['text']}\\n\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Unload lora0 and replace it with a different adapter:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "response = requests.post(\n", - " url + \"/unload_lora_adapter\",\n", - " json={\n", - " \"lora_name\": \"lora0\",\n", - " },\n", - ")\n", - "\n", - "response = requests.post(\n", - " url + \"/load_lora_adapter\",\n", - " json={\n", - " \"lora_name\": \"lora0\",\n", - " \"lora_path\": lora0_new,\n", - " },\n", - ")\n", - "\n", - "if response.status_code == 200:\n", - " print(\"LoRA adapter loaded successfully.\", response.json())\n", - "else:\n", - " print(\"Failed to load LoRA adapter.\", response.json())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Check output again:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "url = f\"http://127.0.0.1:{port}\"\n", - "json_data = {\n", - " \"text\": [\n", - " \"List 3 countries and their capitals.\",\n", - " \"List 3 countries and their capitals.\",\n", - " ],\n", - " \"sampling_params\": {\"max_new_tokens\": 32, \"temperature\": 0},\n", - " # The first input uses lora0, and the second input uses lora1\n", - " \"lora_path\": [\"lora0\", \"lora1\"],\n", - "}\n", - "response = requests.post(\n", - " url + \"/generate\",\n", - " json=json_data,\n", - ")\n", - "print(f\"Output from lora0: \\n{response.json()[0]['text']}\\n\")\n", - "print(f\"Output from lora1 (updated): \\n{response.json()[1]['text']}\\n\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "terminate_process(server_process)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### OpenAI-compatible API usage\n", - "\n", - "You can use LoRA adapters via the OpenAI-compatible APIs by specifying the adapter in the `model` field using the `base-model:adapter-name` syntax (for example, `qwen/qwen2.5-0.5b-instruct:adapter_a`). For more details and examples, see the “Using LoRA Adapters” section in the OpenAI API documentation: [openai_api_completions.ipynb](../basic_usage/openai_api_completions.ipynb).\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### LoRA GPU Pinning" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Another advanced option is to specify adapters as `pinned` during loading. When an adapter is pinned, it is permanently assigned to one of the available GPU pool slots (as configured by `--max-loras-per-batch`) and will not be evicted from GPU memory during runtime. Instead, it remains resident until it is explicitly unloaded.\n", - "\n", - "This can improve performance in scenarios where the same adapter is frequently used across requests, by avoiding repeated memory transfers and reinitialization overhead. However, since GPU pool slots are limited, pinning adapters reduces the flexibility of the system to dynamically load other adapters on demand. If too many adapters are pinned, it may lead to degraded performance, or in the most extreme case (`Number of pinned adapters == max-loras-per-batch`), halt all unpinned requests. Therefore, currently SGLang limits maximal number of pinned adapters to `max-loras-per-batch - 1` to prevent unexpected starvations. \n", - "\n", - "In the example below, we start a server with `lora1` loaded as pinned, `lora2` and `lora3` loaded as regular (unpinned) adapters. Please note that, we intentionally specify `lora2` and `lora3` in two different formats to demonstrate that both are supported." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "server_process, port = launch_server_cmd(\"\"\"\n", - " python3 -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \\\n", - " --enable-lora \\\n", - " --cuda-graph-max-bs 8 \\\n", - " --max-loras-per-batch 3 \\\n", - " --max-lora-rank 256 \\\n", - " --lora-target-modules all \\\n", - " --lora-paths \\\n", - " {\"lora_name\":\"lora0\",\"lora_path\":\"Nutanix/Meta-Llama-3.1-8B-Instruct_SFT_lora_4_alpha_16_humaneval_raw_json\",\"pinned\":true} \\\n", - " {\"lora_name\":\"lora1\",\"lora_path\":\"algoprog/fact-generation-llama-3.1-8b-instruct-lora\"} \\\n", - " lora2=philschmid/code-llama-3-1-8b-text-to-sql-lora\n", - " --log-level warning\n", - " \"\"\")\n", - "\n", - "\n", - "url = f\"http://127.0.0.1:{port}\"\n", - "wait_for_server(url, process=server_process)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You can also specify adapter as pinned during dynamic adapter loading. In the example below, we reload `lora2` as pinned adapter:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "response = requests.post(\n", - " url + \"/unload_lora_adapter\",\n", - " json={\n", - " \"lora_name\": \"lora1\",\n", - " },\n", - ")\n", - "\n", - "response = requests.post(\n", - " url + \"/load_lora_adapter\",\n", - " json={\n", - " \"lora_name\": \"lora1\",\n", - " \"lora_path\": \"algoprog/fact-generation-llama-3.1-8b-instruct-lora\",\n", - " \"pinned\": True, # Pin the adapter to GPU\n", - " },\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Verify that the results are expected:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "url = f\"http://127.0.0.1:{port}\"\n", - "json_data = {\n", - " \"text\": [\n", - " \"List 3 countries and their capitals.\",\n", - " \"List 3 countries and their capitals.\",\n", - " \"List 3 countries and their capitals.\",\n", - " ],\n", - " \"sampling_params\": {\"max_new_tokens\": 32, \"temperature\": 0},\n", - " # The first input uses lora0, and the second input uses lora1\n", - " \"lora_path\": [\"lora0\", \"lora1\", \"lora2\"],\n", - "}\n", - "response = requests.post(\n", - " url + \"/generate\",\n", - " json=json_data,\n", - ")\n", - "print(f\"Output from lora0 (pinned): \\n{response.json()[0]['text']}\\n\")\n", - "print(f\"Output from lora1 (pinned): \\n{response.json()[1]['text']}\\n\")\n", - "print(f\"Output from lora2 (not pinned): \\n{response.json()[2]['text']}\\n\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "terminate_process(server_process)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Choosing LoRA Backend\n", - "\n", - "SGLang supports two LoRA backends that you can choose from using the `--lora-backend` argument:\n", - "\n", - "- `triton`: Basic Triton-based backend.\n", - "- `csgmv`: Default chunked SGMV backend optimized for high concurrency scenarios.\n", - "\n", - "The `csgmv` backend was recently introduced to improve performance especially at high-concurrency scenarios. Our benchmark shows that it achieves 20% to 80% latency improvements over the basic triton backend." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "server_process, port = launch_server_cmd(\"\"\"\n", - " python3 -m sglang.launch_server \\\n", - " --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \\\n", - " --enable-lora \\\n", - " --lora-backend csgmv \\\n", - " --max-loras-per-batch 16 \\\n", - " --lora-paths lora1=path/to/lora1 lora2=path/to/lora2\n", - " \"\"\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "terminate_process(server_process)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## LoRA Overlap Loading" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "By using the `--enable-lora-overlap-loading` server argument, the SGLang engine is able to overlap the loading of LoRA weights with prefill and decode compute, essentially hiding the data movement for LoRA weights behind GPU computation. Our benchmarks show that under adversarial conditions, enabling this feature can result in a ~35% reduction in median TTFT - (see the [LoRA overlap loading PR](https://github.com/sgl-project/sglang/pull/15512) for detailed benchmarks)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "lora0 = \"Nutanix/Meta-Llama-3.1-8B-Instruct_SFT_lora_4_alpha_16_humaneval_raw_json\"\n", - "lora1 = \"algoprog/fact-generation-llama-3.1-8b-instruct-lora\"\n", - "lora2 = \"philschmid/code-llama-3-1-8b-text-to-sql-lora\"\n", - "\n", - "\n", - "server_process, port = launch_server_cmd(\"\"\"\n", - " python3 -m sglang.launch_server \\\n", - " --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \\\n", - " --enable-lora \\\n", - " --enable-lora-overlap-loading \\\n", - " --lora-paths lora0=Nutanix/Meta-Llama-3.1-8B-Instruct_SFT_lora_4_alpha_16_humaneval_raw_json \\\n", - " lora1=algoprog/fact-generation-llama-3.1-8b-instruct-lora \\\n", - " lora2=philschmid/code-llama-3-1-8b-text-to-sql-lora \\\n", - " --max-lora-rank 256 \\\n", - " --max-loras-per-batch 2 \\\n", - " --max-loaded-loras 4\n", - " \"\"\")\n", - "\n", - "url = f\"http://127.0.0.1:{port}\"\n", - "wait_for_server(url, process=server_process)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "json_data = {\n", - " \"text\": [\n", - " \"Write a very long fairy-tale.\",\n", - " \"List 3 countries and their capitals.\",\n", - " \"List 3 countries and their capitals.\",\n", - " ],\n", - " \"sampling_params\": [\n", - " {\"max_new_tokens\": 1024, \"temperature\": 0},\n", - " {\"max_new_tokens\": 64, \"temperature\": 0},\n", - " {\"max_new_tokens\": 64, \"temperature\": 0},\n", - " ],\n", - " \"lora_path\": [\"lora0\", \"lora1\", \"lora2\"],\n", - "}\n", - "\n", - "# lora0 and lora1 will be loaded into the memory pool first, and because max_loras_per_batch = 2, lora2's request will remain in the queue.\n", - "# lora1's request will likely finish first, and once it does, lora2 will be loaded. With --enable-lora-overlap-loading, this loading will\n", - "# occur asynchronously and thus decoding for lora0's request won't be blocked.\n", - "response = requests.post(\n", - " url + \"/generate\",\n", - " json=json_data,\n", - ")\n", - "\n", - "for i in range(3):\n", - " print(f\"Output from lora{i}: \\n{response.json()[i]['text']}\\n\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "terminate_process(server_process)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Limitations of LoRA Overlap Loading" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "However, LoRA overlap loading is not free and comes with two important caveats:\n", - "\n", - "1. **Pinned CPU memory requirement**:\n", - " Asynchronous H2D memory copies require LoRA weights to be pinned in CPU memory, which is a finite system resource. To mitigate excessive pinned-memory usage, SGLang currently restricts `max_loaded_loras` to be at most 2× `max_loras_per_batch` when LoRA overlap loading is enabled.\n", - "\n", - "2. **Reduced multi-adapter prefill batching**:\n", - " With overlap loading, adapters become available on the GPU at different times because each adapter is loaded asynchronously. This can reduce the scheduler’s ability to form multi-adapter prefill batches, since only requests whose adapters are currently loaded can be grouped together. As a result, requests for different adapters will be scheduled in separate (or smaller) prefill batches, which can increase TTFT when adapter load time is small compared to prefill compute time. This is why LoRA overlap loading is disabled by default: it should only be enabled when users have determined that LoRA weight loading is a bottleneck (EG high adapter churn, heavy adapter weights, or PCIe-bottlenecked workloads).\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Example When Overlap Loading Results in Higher Latency" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "For instance, suppose we have four LoRA adapters: `lora0`, `lora1`, `lora2`, and `lora3`. Loading any adapter takes 2ms, while the prefill step for requests for that adapter takes 20ms.\n", - "\n", - "1. **Baseline**:\n", - " The engine loads all four adapters synchronously, then runs one combined prefill batch, giving us a total time of ≈ `2 * 4 + 20 = 28ms`\n", - "\n", - "2. **With LoRA overlap loading enabled**:\n", - " The engine begins loading `lora0` and, once it is ready, schedules a prefill batch containing only `lora0` while `lora1` loads in the background. Then it schedules `lora1`’s prefill while `lora2` loads, and so on. In the worst case where prefill cannot be batched across adapters, total time is ≈ `2 + 4 * 20 = 82ms`\n", - "\n", - "In this scenario, overlap loading reduces adapter-load overhead, but the loss of multi-adapter prefill batching dominates and leads to higher TTFT." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Future Works\n", - "\n", - "The development roadmap for LoRA-related features can be found in this [issue](https://github.com/sgl-project/sglang/issues/2929). Other features, including Embedding Layer, Unified Paging, Cutlass backend are still under development." - ] - } - ], - "metadata": { - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/docs/advanced_features/object_storage.md b/docs/advanced_features/object_storage.md deleted file mode 100644 index 957ecdbaf..000000000 --- a/docs/advanced_features/object_storage.md +++ /dev/null @@ -1,108 +0,0 @@ -# Loading Models from Object Storage - -SGLang supports direct loading of models from object storage (S3 and Google Cloud Storage) without requiring a full local download. This feature uses the `runai_streamer` load format to stream model weights directly from cloud storage, significantly reducing startup time and local storage requirements. - -## Overview - -When loading models from object storage, SGLang uses a two-phase approach: - -1. **Metadata Download** (once, before process launch): Configuration files and tokenizer files are downloaded to a local cache -2. **Weight Streaming** (lazy, during model loading): Model weights are streamed directly from object storage as needed - -## Supported Storage Backends - -1. **Amazon S3**: `s3://bucket-name/path/to/model/` -2. **Google Cloud Storage**: `gs://bucket-name/path/to/model/` -3. **Azure Blob**: `az://some-azure-container/path/` -4. **S3 compatible**: `s3://bucket-name/path/to/model/` - -## Quick Start - -### Basic Usage - -Simply provide an object storage URI as the model path: - -```bash -# S3 -python -m sglang.launch_server \ - --model-path s3://my-bucket/models/llama-3-8b/ \ - --load-format runai_streamer - -# Google Cloud Storage -python -m sglang.launch_server \ - --model-path gs://my-bucket/models/llama-3-8b/ \ - --load-format runai_streamer -``` - -**Note**: The `--load-format runai_streamer` is automatically detected when using object storage URIs, so you can omit it: - -```bash -python -m sglang.launch_server \ - --model-path s3://my-bucket/models/llama-3-8b/ -``` - -### With Tensor Parallelism - -```bash -python -m sglang.launch_server \ - --model-path gs://my-bucket/models/llama-70b/ \ - --tp 4 \ - --model-loader-extra-config '{"distributed": true}' -``` - -## Configuration - -### Load Format - -The `runai_streamer` load format is specifically designed for object storage, ssd and shared file systems - -```bash -python -m sglang.launch_server \ - --model-path s3://bucket/model/ \ - --load-format runai_streamer -``` - -### Extended Configuration Parameters - -Use `--model-loader-extra-config` to pass additional configuration as a JSON string: - -```bash -python -m sglang.launch_server \ - --model-path s3://bucket/model/ \ - --model-loader-extra-config '{ - "distributed": true, - "concurrency": 8, - "memory_limit": 2147483648 - }' -``` - -#### Available Parameters - -| Parameter | Type | Description | Default | -|-----------|------|-------------|---------| -| `distributed` | bool | Enable distributed streaming for multi-GPU setups. Automatically set to `true` for object storage paths and cuda alike devices. | Auto-detected | -| `concurrency` | int | Number of concurrent download streams. Higher values can improve throughput for large models. | 4 | -| `memory_limit` | int | Memory limit (in bytes) for the streaming buffer. | System-dependent | - - -## Performance Considerations - -### Distributed Streaming - -For multi-GPU setups, enable distributed streaming to parallelize weight loading between the processes: - -```bash -python -m sglang.launch_server \ - --model-path s3://bucket/model/ \ - --tp 8 \ - --model-loader-extra-config '{"distributed": true}' -``` - -## Limitations - -- **Supported Formats**: Currently only supports `.safetensors` weight format (recommended format) -- **Supported Device**: Distributed streaming is supported on cuda alike devices. Otherwise fallback to non distributed streaming - -## See Also - -- [Runai model streamer documentation](https://github.com/run-ai/runai-model-streamer) diff --git a/docs/advanced_features/observability.md b/docs/advanced_features/observability.md deleted file mode 100644 index 9c5d2e175..000000000 --- a/docs/advanced_features/observability.md +++ /dev/null @@ -1,35 +0,0 @@ -# Observability - -## Production Metrics -SGLang exposes the following metrics via Prometheus. You can enable them by adding `--enable-metrics` when launching the server. -You can query them by: -``` -curl http://localhost:30000/metrics -``` - -See [Production Metrics](../references/production_metrics.md) and [Production Request Tracing](../references/production_request_trace.md) for more details. - -## Logging - -By default, SGLang does not log any request contents. You can log them by using `--log-requests`. -You can control the verbosity by using `--log-request-level`. -See [Logging](server_arguments.md#logging) for more details. - -## Request Dump and Replay - -You can dump all requests and replay them later for benchmarking or other purposes. - -To start dumping, use the following command to send a request to a server: -``` -python3 -m sglang.srt.managers.configure_logging --url http://localhost:30000 --dump-requests-folder /tmp/sglang_request_dump --dump-requests-threshold 100 -``` -The server will dump the requests into a pickle file for every 100 requests. - -To replay the request dump, use `scripts/playground/replay_request_dump.py`. - -## Crash Dump and Replay -Sometimes the server might crash, and you may want to debug the cause of the crash. -SGLang supports crash dumping, which will dump all requests from the 5 minutes before the crash, allowing you to replay the requests and debug the reason later. - -To enable crash dumping, use `--crash-dump-folder /tmp/crash_dump`. -To replay the crash dump, use `scripts/playground/replay_request_dump.py`. diff --git a/docs/advanced_features/pd_disaggregation.md b/docs/advanced_features/pd_disaggregation.md deleted file mode 100644 index 7c12c26e0..000000000 --- a/docs/advanced_features/pd_disaggregation.md +++ /dev/null @@ -1,410 +0,0 @@ -# PD Disaggregation - -## Why and What is PD Disaggregation? - -Large Language Model (LLM) inference comprises two distinct phases: **Prefill** and **Decode**. The Prefill phase is computation-intensive, processing the entire input sequence, while the Decode phase is memory-intensive, managing the Key-Value (KV) cache for token generation. Traditionally, these phases are handled within a unified engine, where combined scheduling of prefill and decode batches introduces inefficiencies. To address these challenges, we introduce **Prefill and Decoding (PD) Disaggregation** in SGLang. - -### Issues with Unified Scheduling - -The conventional unified engine, which processes prefill and decode batches together, results in two significant problems: - -1. **Prefill Interruption**: Incoming prefill batches frequently interrupt ongoing decode batches, causing substantial delays in token generation. -2. **DP Attention Imbalance**: In data-parallel (DP) attention, one DP worker may process a prefill batch while another handles a decode batch simultaneously, leading to increased decode latency. - -PD Disaggregation resolves these by separating the two stages, enabling tailored optimizations for each. - -For the design details, please refer to [link](https://docs.google.com/document/d/1rQXJwKd5b9b1aOzLh98mnyMhBMhlxXA5ATZTHoQrwvc/edit?tab=t.0). - -Currently, we support Mooncake and NIXL as the transfer engine. - -## Profiling in PD Disaggregation Mode - -When you need to profile prefill or decode workers in PD disaggregation mode, please refer to the [Profile In PD Disaggregation Mode](https://docs.sglang.io/developer_guide/benchmark_and_profiling.html#profile-in-pd-disaggregation-mode) section in the Benchmark and Profiling guide. Due to torch profiler limitations, prefill and decode workers must be profiled separately using dedicated command-line options. - -## Router Integration - -For deploying PD disaggregation at scale with load balancing and fault tolerance, SGLang provides a router. The router can distribute requests between prefill and decode instances using various routing policies. For detailed information on setting up routing with PD disaggregation, including configuration options and deployment patterns, see the [SGLang Model Gateway (former Router)](../advanced_features/sgl_model_gateway.md#prefill-decode-disaggregation). - - -## Mooncake -### Requirements - -```bash -uv pip install mooncake-transfer-engine -``` - -### Usage - -### Llama Single Node - -```bash -python -m sglang.launch_server \ - --model-path meta-llama/Llama-3.1-8B-Instruct \ - --disaggregation-mode prefill \ - --port 30000 \ - --disaggregation-ib-device mlx5_roce0 -python -m sglang.launch_server \ - --model-path meta-llama/Llama-3.1-8B-Instruct \ - --disaggregation-mode decode \ - --port 30001 \ - --base-gpu-id 1 \ - --disaggregation-ib-device mlx5_roce0 -python -m sglang_router.launch_router --pd-disaggregation --prefill http://127.0.0.1:30000 --decode http://127.0.0.1:30001 --host 0.0.0.0 --port 8000 -``` - -### DeepSeek Multi-Node - -```bash -# prefill 0 -python -m sglang.launch_server \ - --model-path deepseek-ai/DeepSeek-V3-0324 \ - --disaggregation-ib-device ${device_name} \ - --disaggregation-mode prefill \ - --host ${local_ip} \ - --port 30000 \ - --trust-remote-code \ - --dist-init-addr ${prefill_master_ip}:5000 \ - --nnodes 2 \ - --node-rank 0 \ - --tp-size 16 \ - --dp-size 8 \ - --enable-dp-attention \ - --moe-a2a-backend deepep \ - --mem-fraction-static 0.8 -# prefill 1 -python -m sglang.launch_server \ - --model-path deepseek-ai/DeepSeek-V3-0324 \ - --disaggregation-ib-device ${device_name} \ - --disaggregation-mode prefill \ - --host ${local_ip} \ - --port 30000 \ - --trust-remote-code \ - --dist-init-addr ${prefill_master_ip}:5000 \ - --nnodes 2 \ - --node-rank 1 \ - --tp-size 16 \ - --dp-size 8 \ - --enable-dp-attention \ - --moe-a2a-backend deepep \ - --mem-fraction-static 0.8 -# decode 0 -python -m sglang.launch_server \ - --model-path deepseek-ai/DeepSeek-V3-0324 \ - --disaggregation-ib-device ${device_name} \ - --disaggregation-mode decode \ - --host ${local_ip} \ - --port 30001 \ - --trust-remote-code \ - --dist-init-addr ${decode_master_ip}:5000 \ - --nnodes 2 \ - --node-rank 0 \ - --tp-size 16 \ - --dp-size 8 \ - --enable-dp-attention \ - --moe-a2a-backend deepep \ - --mem-fraction-static 0.8 \ - --max-running-requests 128 -# decode 1 -python -m sglang.launch_server \ - --model-path deepseek-ai/DeepSeek-V3-0324 \ - --disaggregation-ib-device ${device_name} \ - --disaggregation-mode decode \ - --host ${local_ip} \ - --port 30001 \ - --trust-remote-code \ - --dist-init-addr ${decode_master_ip}:5000 \ - --nnodes 2 \ - --node-rank 1 \ - --tp-size 16 \ - --dp-size 8 \ - --enable-dp-attention \ - --moe-a2a-backend deepep \ - --mem-fraction-static 0.8 \ - --max-running-requests 128 -``` -### Advanced Configuration - -PD Disaggregation with Mooncake supports the following environment variables for fine-grained control over system behavior. - -#### NVLink Transport Configuration -To enable NVLink transport for KV cache transfers with the mooncake backend (recommended for NVL72 deployments), set the following environment variables. Note that auxiliary data transfer will still use TCP as a temporary workaround. - -```bash -export SGLANG_MOONCAKE_CUSTOM_MEM_POOL=NVLINK -export MC_FORCE_MNNVL=True -``` -To utilize Intra-Node NVLink for KV cache transfers with the Mooncake backend (recommended for A100, H20, H100, etc.), set the following environment variables. Please note that auxiliary data still needs to be transferred via TCP. - -```bash -export SGLANG_MOONCAKE_CUSTOM_MEM_POOL=INTRA_NODE_NVLINK -export MC_INTRANODE_NVLINK=true -``` - -The `SGLANG_MOONCAKE_CUSTOM_MEM_POOL` environment variable enables the custom memory pool. Supported values are `NVLINK` (or `True`), `BAREX`, and `INTRA_NODE_NVLINK`. - -#### Prefill Server Configuration -| Variable | Description | Default | -|:--------:|:-----------:|:--------: -| **`SGLANG_DISAGGREGATION_THREAD_POOL_SIZE`** | Controls the total number of worker threads for KVCache transfer operations per TP rank | A dynamic value calculated by `int(0.75 * os.cpu_count()) // 8)`, which is limited to be larger than 4 and less than 12 to ensure efficiency and prevent thread race conditions | -| **`SGLANG_DISAGGREGATION_QUEUE_SIZE`** | Sets the number of parallel transfer queues. KVCache transfer requests from multiple decode instances will be sharded into these queues so that they can share the threads and the transfer bandwidth at the same time. If it is set to `1`, then we transfer requests one by one according to fcfs strategy | `4` | -| **`SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT`** | Timeout (seconds) for receiving destination KV indices during request initialization | `300` | -| **`SGLANG_DISAGGREGATION_BOOTSTRAP_ENTRY_CLEANUP_INTERVAL`** | Interval (seconds) between cleanups of bootstrap entries | `120` | - -If a greater mean TTFT is acceptable, you can `export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600` (10 minutes) to relax the timeout condition. -Please be aware that this setting will cause prefill instances to take a longer time to clean up the affected memory resources when a running decode node loses connection. - -#### Decode Server Configuration -| Variable | Description | Default | -|:--------:|:-----------:|:--------: -| **`SGLANG_DISAGGREGATION_HEARTBEAT_INTERVAL`** | Interval (seconds) between health checks to prefill bootstrap servers | `5.0` | -| **`SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE`** | Consecutive heartbeat failures before marking prefill server offline | `2` | -| **`SGLANG_DISAGGREGATION_WAITING_TIMEOUT`** | Timeout (seconds) for receiving KV Cache after request initialization | `300` | - -If a greater mean TTFT is acceptable, you can `export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=600` (10 minutes) to relax the timeout condition. - - -## Heterogeneous TP with GPU Staging Buffer - -When prefill and decode use different tensor parallelism (TP) sizes (e.g., prefill TP=4, decode DP attention with TP=1), the KV cache memory layout differs between the two sides. The **GPU staging buffer** solves this by gathering KV head slices into a contiguous buffer on the prefill side, performing bulk RDMA transfer, then scattering into the correct KV cache pages on the decode side. This provides **2–5x throughput improvement** over the default per-token slice approach at high concurrency and matches homogeneous TP baselines within ~5%. - -Enable the staging buffer when prefill and decode use **different TP sizes** with the **Mooncake** transfer backend. When both sides use the same TP size, staging is automatically bypassed even if enabled. - -> **Note:** The staging buffer is designed for non-MLA models (e.g. GQA, MHA). MLA models (e.g. DeepSeek-V2/V3) should not enable this flag. - -### Environment Variables - -| Variable | Description | Default | -|:---------|:------------|:-------:| -| **`SGLANG_DISAGG_STAGING_BUFFER`** | Enable GPU staging buffer for heterogeneous TP KV transfer | `False` | -| **`SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB`** | Prefill-side per-worker staging buffer size in MB | `64` | -| **`SGLANG_DISAGG_STAGING_POOL_SIZE_MB`** | Decode-side ring buffer pool total size in MB | `4096` | - -### Usage Example - -```bash -# Set staging buffer environment variables on BOTH prefill and decode -export SGLANG_DISAGG_STAGING_BUFFER=1 -export SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB=64 -export SGLANG_DISAGG_STAGING_POOL_SIZE_MB=4096 - -# Prefill with TP=4 -python -m sglang.launch_server \ - --model-path $MODEL_PATH \ - --disaggregation-mode prefill \ - --port 30000 \ - --tp 4 \ - --trust-remote-code \ - --disaggregation-ib-device mlx5_1,mlx5_2 - -# Decode with TP=1 (or DP attention with effective attention TP=1) -python -m sglang.launch_server \ - --model-path $MODEL_PATH \ - --disaggregation-mode decode \ - --port 30001 \ - --tp 4 \ - --dp 4 \ - --enable-dp-attention \ - --trust-remote-code \ - --disaggregation-ib-device mlx5_3,mlx5_4 - -# Router -python -m sglang_router.launch_router \ - --pd-disaggregation \ - --prefill http://127.0.0.1:30000 \ - --decode http://127.0.0.1:30001 \ - --host 0.0.0.0 --port 8000 -``` - -## NIXL -### Requirements - -Install via pip. - -```bash -pip install nixl -``` - -Or build from source - may be required if you already have UCX installed. - -```bash -git clone https://github.com/ai-dynamo/nixl.git -cd nixl -pip install . --config-settings=setup-args="-Ducx_path=/path/to/ucx" -``` - - -### Usage - -### Llama Single Node - -```bash -python -m sglang.launch_server \ - --model-path meta-llama/Llama-3.1-8B-Instruct \ - --disaggregation-mode prefill \ - --port 30000 \ - --disaggregation-transfer-backend nixl -python -m sglang.launch_server \ - --model-path meta-llama/Llama-3.1-8B-Instruct \ - --disaggregation-mode decode \ - --port 30001 \ - --base-gpu-id 1 \ - --disaggregation-transfer-backend nixl -python -m sglang_router.launch_router --pd-disaggregation --prefill http://127.0.0.1:30000 --decode http://127.0.0.1:30001 --host 0.0.0.0 --port 8000 -``` - -### DeepSeek Multi-Node - -```bash -# prefill 0 -python -m sglang.launch_server \ - --model-path deepseek-ai/DeepSeek-V3-0324 \ - --disaggregation-transfer-backend nixl \ - --disaggregation-mode prefill \ - --host ${local_ip} \ - --port 30000 \ - --trust-remote-code \ - --dist-init-addr ${prefill_master_ip}:5000 \ - --nnodes 2 \ - --node-rank 0 \ - --tp-size 16 \ - --dp-size 8 \ - --enable-dp-attention \ - --moe-a2a-backend deepep \ - --mem-fraction-static 0.8 -# prefill 1 -python -m sglang.launch_server \ - --model-path deepseek-ai/DeepSeek-V3-0324 \ - --disaggregation-transfer-backend nixl \ - --disaggregation-mode prefill \ - --host ${local_ip} \ - --port 30000 \ - --trust-remote-code \ - --dist-init-addr ${prefill_master_ip}:5000 \ - --nnodes 2 \ - --node-rank 1 \ - --tp-size 16 \ - --dp-size 8 \ - --enable-dp-attention \ - --moe-a2a-backend deepep \ - --mem-fraction-static 0.8 -# decode 0 -python -m sglang.launch_server \ - --model-path deepseek-ai/DeepSeek-V3-0324 \ - --disaggregation-transfer-backend nixl \ - --disaggregation-mode decode \ - --host ${local_ip} \ - --port 30001 \ - --trust-remote-code \ - --dist-init-addr ${decode_master_ip}:5000 \ - --nnodes 2 \ - --node-rank 0 \ - --tp-size 16 \ - --dp-size 8 \ - --enable-dp-attention \ - --moe-a2a-backend deepep \ - --mem-fraction-static 0.8 \ - --max-running-requests 128 -# decode 1 -python -m sglang.launch_server \ - --model-path deepseek-ai/DeepSeek-V3-0324 \ - --disaggregation-transfer-backend nixl \ - --disaggregation-mode decode \ - --host ${local_ip} \ - --port 30001 \ - --trust-remote-code \ - --dist-init-addr ${decode_master_ip}:5000 \ - --nnodes 2 \ - --node-rank 1 \ - --tp-size 16 \ - --dp-size 8 \ - --enable-dp-attention \ - --moe-a2a-backend deepep \ - --mem-fraction-static 0.8 \ - --max-running-requests 128 -``` - -### Advanced Configuration - -#### NIXL Backend Selection - -By default, NIXL uses the **UCX** backend for KV cache transfers. You can select a different NIXL plugin backend depending on your infrastructure using the environment variable `SGLANG_DISAGGREGATION_NIXL_BACKEND`. - -Example: `export SGLANG_DISAGGREGATION_NIXL_BACKEND=LIBFABRIC` - -**Available backends:** UCX (default), LIBFABRIC, or any installed NIXL plugin. - -Example usage: -```bash -export SGLANG_DISAGGREGATION_NIXL_BACKEND=LIBFABRIC -python -m sglang.launch_server \ - --model-path meta-llama/Llama-3.1-8B-Instruct \ - --disaggregation-mode prefill \ - --disaggregation-transfer-backend nixl \ - --port 30000 -``` - -## ASCEND - -### Usage - -Use ascend backend with [memfabric_hybrid](https://gitcode.com/Ascend/memfabric_hybrid) and ASCEND_MF_STORE_URL being set - -```bash -pip install memfabric-hybrid==1.0.0 -export ASCEND_MF_STORE_URL="tcp://xxx.xx.xxx.xxx:xxxx" -``` -Use mooncake backend, more details can be found in mooncake section. -```bash -export ENABLE_ASCEND_TRANSFER_WITH_MOONCAKE=true -``` -ASCEND_NPU_PHY_ID need to be set in container env -```bash -export ASCEND_NPU_PHY_ID=xxx -``` - - -### Llama Single Node - -```bash -python -m sglang.launch_server \ - --model-path meta-llama/Llama-3.1-8B-Instruct \ - --disaggregation-mode prefill \ - --port 30000 \ - --disaggregation-transfer-backend ascend -python -m sglang.launch_server \ - --model-path meta-llama/Llama-3.1-8B-Instruct \ - --disaggregation-mode decode \ - --port 30001 \ - --base-gpu-id 1 \ - --disaggregation-transfer-backend ascend -python -m sglang_router.launch_router --pd-disaggregation --prefill http://127.0.0.1:30000 --decode http://127.0.0.1:30001 --host 0.0.0.0 --port 8000 -``` - -### DeepSeek Multi-Node - -```bash -# prefill 0 -python -m sglang.launch_server \ - --model-path deepseek-ai/DeepSeek-V3-0324 \ - --disaggregation-transfer-backend ascend \ - --disaggregation-mode prefill \ - --host ${local_ip} \ - --port 30000 \ - --trust-remote-code \ - --dist-init-addr ${prefill_master_ip}:5000 \ - --nnodes 1 \ - --node-rank 0 \ - --tp-size 16 -# decode 0 -python -m sglang.launch_server \ - --model-path deepseek-ai/DeepSeek-V3-0324 \ - --disaggregation-transfer-backend ascend \ - --disaggregation-mode decode \ - --host ${local_ip} \ - --port 30001 \ - --trust-remote-code \ - --dist-init-addr ${decode_master_ip}:5000 \ - --nnodes 1 \ - --node-rank 0 \ - --tp-size 16 -``` diff --git a/docs/advanced_features/piecewise_cuda_graph.md b/docs/advanced_features/piecewise_cuda_graph.md deleted file mode 100644 index e0bb47af9..000000000 --- a/docs/advanced_features/piecewise_cuda_graph.md +++ /dev/null @@ -1,189 +0,0 @@ -# Piecewise CUDA Graph - -## Motivation - -Standard CUDA graphs capture the entire model forward pass as a single graph. This works well for decode (fixed batch size), but not for extend/prefill where the number of tokens varies across iterations. - -Piecewise CUDA Graph (PCG) solves this by splitting the model's computation graph into pieces (roughly one per layer) at "split points" (e.g., MoE dispatch ops). Each piece is captured as a separate CUDA graph for a set of pre-defined token lengths. At runtime, the input is padded to the nearest captured size, and each piece is replayed. This eliminates kernel launch overhead for prefill/extend while still supporting dynamic shapes. - -Recently we **enabled PCG by default**, which means that the old `--enable-piecewise-cuda-graph` flag is deprecated. Use `--disable-piecewise-cuda-graph` to turn it off. - -## Usage - -PCG is enabled by default for supported configurations. No extra flags needed: - -```bash -python3 -m sglang.launch_server \ - --model-path meta-llama/Llama-3.1-8B-Instruct -``` - -### Disable PCG - -```bash -python3 -m sglang.launch_server \ - --model-path meta-llama/Llama-3.1-8B-Instruct \ - --disable-piecewise-cuda-graph -``` - -### Custom capture sizes - -```bash -python3 -m sglang.launch_server \ - --model-path meta-llama/Llama-3.1-8B-Instruct \ - --piecewise-cuda-graph-max-tokens 2048 -``` - -### Server Args - -| Argument | Default | Description | -|---|---|---| -| `--disable-piecewise-cuda-graph` | `False` | Disable PCG for extend/prefill. | -| `--enforce-piecewise-cuda-graph` | `False` | Force-enable PCG, skipping all auto-disable conditions. For testing only. | -| `--piecewise-cuda-graph-max-tokens` | `None` (auto) | Maximum token count to capture. Defaults to `chunked_prefill_size` (non-MLA) or `2048` (MLA). | -| `--piecewise-cuda-graph-tokens` | `None` (auto) | Explicit list of token lengths to capture. Auto-generated if not set. | -| `--piecewise-cuda-graph-compiler` | `"eager"` | Compiler backend for the captured subgraphs. Choices: `eager`, `inductor`. | -| ~~`--enable-piecewise-cuda-graph`~~ | — | **Deprecated.** PCG is now enabled by default. Use `--enforce-piecewise-cuda-graph` to skip auto-disable conditions. | - -## Bug Report - -PCG is enabled by default but is still in an experimental stage. Since PCG relies on `torch.compile` to trace the model's forward pass, most bugs are introduced by torch compile tracing failures (e.g., untraceable ops, dynamic control flow, or graph breaks). If you encounter any issues related to PCG, please disable it by adding `--disable-piecewise-cuda-graph` to your launch command and report the bug at [GitHub Issues](https://github.com/sgl-project/sglang/issues/new/choose). We greatly appreciate your help in improving this feature. - -### For Users - -If you see an error message like the following during server startup, it is a PCG bug: - -``` -Piecewise CUDA Graph is enabled by default as an experimental feature. -To work around this error, add --disable-piecewise-cuda-graph to your launch command. -Please report this issue at https://github.com/sgl-project/sglang/issues/new/choose -``` - -To work around it, add `--disable-piecewise-cuda-graph` to your launch command. When filing a bug report, please include: -1. The full error traceback -2. Model name and quantization method -3. Launch command with all arguments -4. GPU type and driver version - -### For Developers - -Since PCG relies on `torch.compile` to trace the model's forward pass, newly developed CUDA kernels (both JIT kernels and sgl-kernels) are typically not compatible with `torch.compile` out of the box. The tracing will fail on untraceable operations such as JIT compilation, file I/O, or dynamic module loading inside the kernel. - -To make a kernel compatible with PCG, you need to register it as a custom op using `register_custom_op` from `sglang.srt.utils.custom_op`. This wraps the kernel as an opaque node in the compiled graph so that `torch.compile` will not trace inside it. - -**Example usage (JIT kernel):** - -```python -from sglang.srt.utils.custom_op import register_custom_op - -# Inplace operator (no return value) -@register_custom_op(mutates_args=["output_q", "output_s"]) -def per_token_group_quant_8bit( - input: torch.Tensor, - output_q: torch.Tensor, - output_s: torch.Tensor, -) -> None: - # kernel implementation ... -``` - -**Example usage (operator with output):** - -```python -# out_shape indicates which argument has the same shape as the output -@register_custom_op(mutates_args=["x"], out_shape=0) -def add(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor: - return x.add_(y) -``` - -For wrapping external library functions (e.g., FlashInfer kernels), use `register_custom_op_from_extern` instead. See `python/sglang/srt/utils/custom_op.py` for full API documentation. - -## How it works -### Torch compile backend - -PCG uses `torch.compile` with a custom backend (`SGLangBackend`) to split and compile the model's forward pass. The flow is: - -``` -model.forward wrapper -→ torch.compile(..., backend=SGLangBackend) -→ FX graph -→ split_graph() at registered split ops -→ split_gm (top-level graph that chains the pieces) -→ replace capturable submodules with CUDAPiecewiseBackend -→ runtime dispatch: eager split ops + per-piece capture/replay -``` - -- **Install**: `install_torch_compiled()` replaces `model.forward` with a wrapper function. When `is_in_piecewise_cuda_graph()` returns True, the wrapper dispatches to the compiled callable; otherwise it falls back to the original forward. The first invocation through this path triggers Dynamo tracing and graph compilation — CUDA graph replay only happens after the capture phase completes. - -- **Split**: When `torch.compile` traces the model, `SGLangBackend` receives the FX graph and calls `split_graph()`. Ops listed in `CompilationConfig.split_ops` are treated as split points, so the graph is cut at each one. These split-op submodules are left to run eagerly at runtime, while the surrounding submodules are compiled and wrapped by `CUDAPiecewiseBackend`. The result is a top-level "stitching graph" (`split_gm`) with children such as `submod_0`, `submod_1`, … interleaving capturable subgraphs and eager split-op submodules. - -- **Replace**: `PiecewiseCompileInterpreter` iterates over each capturable submodule in `split_gm`, compiles it for general (dynamic) shapes, and replaces it in-place with a `CUDAPiecewiseBackend` instance. Split-op submodules (e.g., attention, all-reduce) are left as-is and run eagerly at runtime. - -- **Dispatch**: At runtime, calling `split_gm` executes the stitching graph, which calls each submodule in order. Split-op submodules run eagerly. Each `CUDAPiecewiseBackend` submodule goes through three phases: - - **Compile warmup** — runs the general-shape compiled path. - - **Capture** — for each capture size, runs one warmup pass then records a CUDA graph. - - **Steady-state replay** — replays the captured CUDA graph for each forward pass. - -### Piecewise cuda graph runner - -`PiecewiseCudaGraphRunner` orchestrates the full lifecycle through three phases: - -- **Compile** — Warms up JIT kernels with a dummy forward pass, then wraps the model with `torch.compile`, triggering Dynamo tracing to split the FX graph and create `CUDAPiecewiseBackend` instances for each subgraph piece. - -- **Capture** — Iterates over capture sizes in reverse order (largest first). For each size, runs the forward pass twice (one warmup, one CUDA graph capture). - -- **Replay** — At runtime, finds the smallest captured size >= actual token count via binary search, copies inputs into static buffers with zero-padding, replays the captured CUDA graphs, and slices outputs back to the actual token count. - -### Memory optimization - -The memory cost of PCG comes from two parts: **torch memory allocator** and **non-torch memory**. - -The torch memory allocator overhead is trivial thanks to several optimizations: a global shared memory pool is reused across all CUDA graph runners and capture sizes, capture is done in reverse order (large to small) so smaller graphs reuse memory allocated by larger ones, and output tensors of the last subgraph are stored as weak references to maximize memory reuse. - -The main memory overhead comes from non-torch memory — the CUDA graph objects themselves require GPU memory to store the recorded kernel launch parameters and internal state. This overhead scales with the number of captured sizes, which is why `piecewise_cuda_graph_max_tokens` is capped conservatively by default. - -### Shape configuration -Piecewise CUDA graph pre-captures graphs for a set of token counts. At runtime, the actual token count is rounded up to the nearest captured size (via binary search), and the corresponding graph is replayed. If the token count exceeds the largest captured size, the runtime falls back to the normal (non-graph) forward path. - -The default capture schedule is auto-generated with increasing granularity: - -| Token range | Step size | -|-------------|-----------| -| 4 – 32 | 4 | -| 48 – 256 | 16 | -| 288 – 512 | 32 | -| 576 – 1024 | 64 | -| 1280 – 4096 | 256 | -| 4096+ | 512 | - -For the auto-generated schedule, sizes are capped at `--piecewise-cuda-graph-max-tokens`. The default cap is `chunked_prefill_size` for non-MLA models and `2048` for MLA backend models. If `--max-total-tokens` is set, the cap is further limited to not exceed it. Additionally, Llama-2 models are auto-capped at 4096 tokens as a temporary workaround. - -## Compatibility - -PCG is auto-disabled in the following scenarios. We are actively working on expanding compatibility — support for many of these will be coming soon. - -- Disabled model architectures (e.g., `DeepseekV32ForCausalLM`) -- Speculative decoding -- DP attention -- Pipeline parallelism (`pp_size > 1`) -- Non-CUDA hardware (AMD ROCm, Ascend NPU) -- MoE A2A backend -- LoRA -- Multimodal / VLM models -- DLLM (diffusion LLM) -- Deterministic inference -- PD disaggregation -- Expert distribution recorder / EPLB - -Use `--enforce-piecewise-cuda-graph` to skip all auto-disable checks (for testing/debugging only). - -## Code Reference - -| File | Description | -|---|---| -| `python/sglang/srt/model_executor/piecewise_cuda_graph_runner.py` | Main runner: init, capture, replay | -| `python/sglang/srt/compilation/compile.py` | `install_torch_compiled` trampoline | -| `python/sglang/srt/compilation/backend.py` | `SGLangBackend`, graph splitting, piecewise compilation | -| `python/sglang/srt/compilation/cuda_piecewise_backend.py` | Per-subgraph CUDA graph capture/replay | -| `python/sglang/srt/compilation/piecewise_context_manager.py` | Global context flags and `ForwardContext` | -| `python/sglang/srt/compilation/compilation_config.py` | Capture sizes, split ops, compiler config | -| `python/sglang/srt/utils/custom_op.py` | `register_custom_op` for torch.compile compatibility | -| `python/sglang/srt/server_args.py` | Server arguments and auto-disable logic | diff --git a/docs/advanced_features/pipeline_parallelism.md b/docs/advanced_features/pipeline_parallelism.md deleted file mode 100644 index 9a565f602..000000000 --- a/docs/advanced_features/pipeline_parallelism.md +++ /dev/null @@ -1,116 +0,0 @@ -# Pipeline Parallelism for Long Context - -## Why Pipeline Parallelism? - -As Large Language Models (LLMs) scale toward trillion-parameter architectures and "infinite" context windows, the underlying serving infrastructure must evolve toward more granular, cross-node parallelization strategies. While KV cache techniques effectively mitigate redundant computation, they cannot circumvent the prohibitive Time to First Token (TTFT) inherent in ultra-long sequences with extremely large initial Input Token Length (ITL). Although Tensor Parallelism (TP) remains the conventional approach for intra-node scaling, it frequently encounters communication bottlenecks during multi-node deployments. On the other hand, pipeline parallelism only requires cross-node communication at the boundaries of each pipeline stage, which can achieve better computation-communication overlap compared to a large TP. Therefore, it is also a promising parallelization strategy for improving throughput. - -Detailed analysis can be found in this [blog](https://lmsys.org/blog/2026-01-15-chunked-pipeline/). - -## Implementation Refactoring based on Async Communication -With Dynamic Chunked Prefill, pipeline parallelism has the potential to reduce the TTFT of long-context inputs. For each request, its input tokens can be partitioned into multiple chunks, each no longer than the chunked prefill size. Different chunks of the same request can be processed simultaneously by different nodes, thus parallelizing the processing and reducing TTFT. SGLang has supported Pipeline Parallelism (#5724) for some time and made it compatible with the PD Disaggregation feature (#8846), but the implementation was not perfect and had significant room for performance improvements. - -To eliminate this performance hazard, SGLang implements a Micro-batching Event Loop with non-blocking asynchronous peer-to-peer (P2P) communication to overlap GPU computation with CPU metadata processing and PP communication. This ensures that while one micro-batch is being computed on the GPU, the next one is already being prepared and moved into position effectively, ensuring the pipeline remains as saturated as possible. This approach was first proposed in #7979 and has been redesigned and included in #11852. - -The key mechanisms of the implementation include: - -* **Decoupled Sync/Async Logic in the Event Loop:** The scheduler uses `async_send` in `_pp_send_pyobj_to_next_stage`. Instead of waiting for a transfer to complete, it returns a `P2PWork` handle. The actual synchronization (`P2PWork.work.wait()`) is deferred until `_pp_commit_comm_work` is called, allowing the CPU to perform other work—like scheduling the next batch or processing metadata—while data is in flight. -* **Multi-Stream Execution:** In addition to the main `default_stream`, which serves as the synchronization stream, SGLang utilizes dedicated `forward_stream` and `copy_stream` to execute forward pass GPU computation and Data-to-Host (D2H) memory transfers separately for better overlapping. While `_pp_launch_batch` is executing the current micro-batch on the GPU for the current stage, the CPU processes the previous micro-batch's results using `_pp_process_batch_result`. - -## Guidance about Dynamic Chunking - -### Why Dynamic Chunking -Chunked prefill with a fixed size can cause bubbles in the pipeline, especially when the pp size is large. The main reason behind this phenomenon is that the model has a non-uniform running time, even though each chunk size is identical (brought by the Transformer structure). The larger the prefix sequence length, the longer the running time of the chunk. And these bubbles will be propagated to the next stage, and will significantly degrade the scale efficiency of larger pp ranks. - -To address this issue, SGLang introduces a dynamic chunking mechanism to predict the optimal size for the next chunk such that it satisfies this condition: - -Runtime(L + Next Chunk Size) - Runtime(L) = Runtime(Initial Chunk Size) - -where ***L*** denotes the Prefix Sequence Length. By profiling a series of requests with different ITLs, we model the cumulative runtime as a quadratic function of sequence length. Using this model, we solve the optimal next chunk size for any given prefix length ***L***. Since the computation complexity of the Attention mechanism scales with ***L***, the next chunk size will be progressively reduced as ***L*** grows to maintain an aligned chunk execution time across pipeline stages. - -Based on this method, the scheduler can predict and dynamically reduce the chunk size during runtime to minimize the bubbles caused by the stage misalignment. To be noticed, the scheduler does not use the raw predicted value. To facilitate efficient KVCache memory management and ensure affinity with hardware execution efficiency, the value is aligned downward to the nearest multiple of max(`--page-size`, 64). - - -### Chunked Prefill Size and Smoothing Factor - -When `--enable-dynamic-chunking` is enabled, each chunk size of a sequence is determined dynamically based on the quadratic model that predicts the next chunk size based on the estimated runtime of the initial chunk length. In this case, we use `--chunked-prefill-size` to set up the initial chunk size. When switching to the dynamic chunking mode, the initial chunk size (`--chunked-prefill-size`) should be set to a larger value comparable to the original chunked prefill size, so that there won't be too many chunks. - -**`SGLANG_DYNAMIC_CHUNKING_SMOOTH_FACTOR`** is an environmental variable that controls the smoothing factor for the dynamic chunking algorithm, defaulting to 0.75. It determines how much the chunk size can change during the prefill phase. A larger value means a more aggressive chunk size change, which may lead to better performance but also to greater chunk size changes (the chunk size at the end may become very small, which could lead to performance degradation) and more total chunks. When it is set to 1, the chunk size will be adjusted strictly based on the aforementioned quadratic model that predicts the next chunk size. A smaller value means a more conservative chunk size change, which may lead to smaller chunk size changes and fewer total chunks. When it is set to 0, the chunk size will not be adjusted dynamically, so it is identical to the traditional way with a fixed chunked prefill size. - -Due to the variation in hardware, models, and target workloads, a static configuration is seldom optimal across all scenarios. Consequently, achieving peak performance necessitates a degree of hyperparameter tuning when switching to the dynamic chunking mode. - -**Tuning Guidance for Dynamic Chunked Prefill** - -* **Step 1 \- Iterate to find the optimal fixed chunked prefill size for the targeted PP size**: Different PP sizes for targeted ITL may have different optimal chunked prefill sizes. Therefore, users should iterate to obtain the baseline according to the available resources for scaling. -* **Step 2 \- Initial Chunk Size Selection for Dynamic Chunking**: Set the initial size to 2× or 3× the optimal fixed chunked prefill size. This reduces the total number of chunks and prevents "tail chunks" from underutilizing hardware. To maintain efficiency for extremely large Input Token Lengths (ITL), the dynamic predictor automatically ensures subsequent chunks are at least 1/4 of this initial size. In addition, it is recommended to use a larger initial chunk size (e.g., 4× the optimal fixed chunked prefill size) for such cases as well. -* **Step 3 \- Smooth Factor Adjustment**: This factor controls how strictly the chunk size adjusts the prediction given by the quadratic performance fitting model. - * 1.0: Follows the model strictly. - * **0.6 – 0.85 (Recommended)**: Typical range for the best balance between dynamic scaling and hardware stability. Through experiments, we find that a range between 0.6 and 0.85 typically yields the best performance for dynamic chunking. - * 0: Disables dynamic adjustment, reverting to traditional fixed-size chunking. -* **Another small optimization tip:** Put the larger partition in the higher PP rank when the layers are not evenly divisible across ranks. It can increase the GPU utilization when a larger PP rank is waiting for the previous stage’s result, hence reducing the bubbles on higher PP ranks. If we take DeepSeek-V3.1 as an example, `SGLANG_PP_LAYER_PARTITION=15,15,15,16` usually performs better than `16,15,15,15`. - -## Best Practice for Long Context - -### Tuning the Chunked Prefill Size -Optimizing the chunked prefill size is crucial for balancing pipeline efficiency and resource utilization. The ideal size depends on factors including model architecture, hardware configuration, and typical input lengths. We recommend starting with a small chunk size, such as 4K, and gradually increasing it until you find the optimal size for your specific use case (Different targeted ITL and PP Sizes may have different optimal chunked prefill sizes. Therefore, users should iterate to obtain the baseline according to the available resources for scaling). Alternatively, you can analyze the hardware capacity and determine the optimal chunk size based on the roofline model. - -### Enable Dynamic Chunking and Adjust Smoothing Factor for Ultra-long ITL -SGLang also offers a dynamic chunking solution that could further improve performance. This feature is currently an experimental feature that requires a certain amount of tuning experimentation and may not be suitable for all workloads. In addition, fine-tuning the smoothing factor can help optimize performance for specific workloads and model characteristics. - -### Case Study on NVIDIA H20 - -When evaluating pipeline parallelism with fixed chunked prefill sizes from 2K to 16K, experiment results show that a 4K chunk size delivered optimal prefill TTFT performance for the DeepSeek-V3.1, and a 6K chunk size delivered optimal prefill TTFT performance for the Qwen3-235B-A22B-FP8. - -When enabling dynamic chunking, we first scale the optimal fixed chunked prefill size by a factor of 3 as the initial chunk size. Through experimentation, we found that a multiplier of 2-3 provides an appropriate balance—avoiding excessive initial pipeline bubbles while ensuring that subsequent chunks don't become too small as context length increases. With the default dynamic chunking smoothing factor of 0.75, we performed parameter tuning and determined that a value of 0.65 works optimally with the 12K initial chunk size for the DeepSeek-V3.1, while a value of 0.8 works optimally with the 18K initial chunk size for the Qwen3-235B-A22B-FP8. - -#### DeepSeek-V3.1 with 128K Input Token Length -```bash -# prefill node 0 (fixed chunked prefill size) -python3 -m sglang.launch_server \ - --model-path deepseek-ai/DeepSeek-V3.1 --trust-remote-code \ - --nnodes 4 --node-rank 0 --tp 8 --pp-size 4 \ - --port 30000 --dist-init-addr \ - --disable-radix-cache --mem-fraction-static 0.8 \ - --attention-backend fa3 --host 0.0.0.0 --watchdog-timeout 3600 \ - --max-running-requests 128 --chunked-prefill-size 4096 -``` - -```bash -# prefill node 0 (with dynamic chunking) -export SGLANG_DYNAMIC_CHUNKING_SMOOTH_FACTOR=0.65 -python3 -m sglang.launch_server \ - --model-path deepseek-ai/DeepSeek-V3.1 --trust-remote-code \ - --nnodes 4 --node-rank 0 --tp 8 --pp-size 4 \ - --port 30000 --dist-init-addr \ - --disable-radix-cache --mem-fraction-static 0.8 \ - --attention-backend fa3 --host 0.0.0.0 --watchdog-timeout 3600 \ - --max-running-requests 128 --chunked-prefill-size 12288 --enable-dynamic-chunking -``` - -#### Qwen3-235B-A22B-FP8 with 128K Input Token Length -```bash -# prefill node 0 (fixed chunked prefill size) -python3 -m sglang.launch_server \ - --model-path Qwen/Qwen3-235B-A22B-FP8 --trust-remote-code \ - --nnodes 4 --node-rank 0 --tp 4 --pp-size 8 \ - --port 30000 --dist-init-addr \ - --disable-radix-cache --mem-fraction-static 0.8 \ - --attention-backend fa3 --host 0.0.0.0 --watchdog-timeout 3600 \ - --max-running-requests 128 --chunked-prefill-size 6144 -``` - -```bash -# prefill node 0 (with dynamic chunking) -export SGLANG_DYNAMIC_CHUNKING_SMOOTH_FACTOR=0.8 -python3 -m sglang.launch_server \ - --model-path Qwen/Qwen3-235B-A22B-FP8 --trust-remote-code \ - --nnodes 4 --node-rank 0 --tp 4 --pp-size 8 \ - --port 30000 --dist-init-addr \ - --disable-radix-cache --mem-fraction-static 0.8 \ - --attention-backend fa3 --host 0.0.0.0 --watchdog-timeout 3600 \ - --max-running-requests 128 --chunked-prefill-size 18432 --enable-dynamic-chunking -``` - -Note: `--disable-radix-cache` is enabled only for reproducible benchmarking purposes. It is not recommended to use it in production. - -## Best Practice for Pipeline Parallelism with PD Disaggregation -To be added. Stay tuned for the latest updates on Pipeline Parallelism with PD Disaggregation. diff --git a/docs/advanced_features/quantization.md b/docs/advanced_features/quantization.md deleted file mode 100644 index 8e68d5d10..000000000 --- a/docs/advanced_features/quantization.md +++ /dev/null @@ -1,634 +0,0 @@ -# Quantization - -SGLang supports various quantization methods, including offline quantization and online dynamic quantization. - -Offline quantization loads pre-quantized model weights directly during inference. This is required for quantization methods -such as GPTQ and AWQ, which collect and pre-compute various statistics from the original weights using the calibration dataset. - -Online quantization dynamically computes scaling parameters—such as the maximum/minimum values of model weights—during runtime. -Like NVIDIA FP8 training's [delayed scaling](https://docs.nvidia.com/deeplearning/transformer-engine/user-guide/examples/fp8_primer.html#Mixed-precision-training-with-FP8) mechanism, online quantization calculates the appropriate scaling factors -on-the-fly to convert high-precision weights into a lower-precision format. - -**Note: For better performance, usability and convenience, offline quantization is recommended over online quantization.** - -If you use a pre-quantized model, do not add `--quantization` to enable online quantization at the same time. -For popular pre-quantized models, please visit [Unsloth](https://huggingface.co/unsloth), [NVIDIA ModelOpt](https://huggingface.co/collections/nvidia/inference-optimized-checkpoints-with-model-optimizer) -or [NeuralMagic](https://huggingface.co/collections/neuralmagic) collections on HF for some -popular quality validated quantized models. Quantized models must be validated via benchmarks post-quantization -to guard against abnormal quantization loss regressions. - -## Platform Compatibility - -The following table summarizes quantization method support across NVIDIA and AMD GPUs, Ascend NPUs. - -| Method | NVIDIA GPUs | AMD GPUs (MI300X/MI325X/MI350X) | Ascend NPUs (A2/A3) | Notes | -|--------|:-----------:|:-------------------------------:|:-----------------------:|-------| -| `fp8` | Yes | Yes | WIP | Aiter or Triton backend on AMD | -| `mxfp4` | Yes | Yes | WIP | Requires CDNA3/CDNA4 with MXFP support; uses Aiter | -| `blockwise_int8` | Yes | Yes | No | Triton-based, works on both platforms | -| `w8a8_int8` | Yes | Yes | No | | -| `w8a8_fp8` | Yes | Yes | No | Aiter or Triton FP8 on AMD | -| `awq` | Yes | Yes | Yes | Uses Triton dequantize on AMD (vs. optimized CUDA kernels on NVIDIA). Uses CANN kernels on Ascend| -| `gptq` | Yes | Yes | Yes | Uses Triton or vLLM kernels on AMD. Uses CANN kernels on Ascend| -| `compressed-tensors` | Yes | Yes | Partial | Aiter paths for FP8/MoE on AMD. Uses CANN kernels on Ascend, `FP8` not supported yet| -| `quark` | Yes | Yes | No | AMD Quark quantization; Aiter GEMM paths on AMD | -| `auto-round` | Yes | Yes | Partial | Platform-agnostic (Intel auto-round). Uses CANN kernels on Ascend| -| `quark_int4fp8_moe` | No | Yes | No | AMD-only; online INT4-to-FP8 MoE quantization (CDNA3/CDNA4) | -| `awq_marlin` | Yes | No | No | Marlin kernels are CUDA-only | -| `gptq_marlin` | Yes | No | No | Marlin kernels are CUDA-only | -| `gguf` | Yes | No | Yes | CUDA-only kernels in sgl-kernel; Pre-dequantized on Ascend | -| `modelopt` / `modelopt_fp8` | Yes (Hopper/SM90+) | No | No | [NVIDIA ModelOpt](https://github.com/NVIDIA/Model-Optimizer); requires NVIDIA hardware | -| `modelopt_fp4` | Yes (Blackwell/SM100+) | No | No | [NVIDIA ModelOpt](https://github.com/NVIDIA/Model-Optimizer); native FP4 on Blackwell (B200, GB200) | -| `petit_nvfp4` | No | Yes (MI250/MI300X/MI325X) | No | Enables NVFP4 on ROCm via [Petit](https://github.com/causalflow-ai/petit-kernel); use `modelopt_fp4` on NVIDIA Blackwell. Auto-selected when loading NVFP4 models on AMD. See [LMSYS blog](https://lmsys.org/blog/2025-09-21-petit-amdgpu/) and [AMD ROCm blog](https://rocm.blogs.amd.com/artificial-intelligence/fp4-mixed-precision/README.html). | -| `bitsandbytes` | Yes | Experimental | No | Depends on bitsandbytes ROCm support | -| `torchao` (`int4wo`, etc.) | Yes | Partial | No | `int4wo` not supported on AMD; other methods may work | -| `modelslim` | No | No | Yes | Ascend quantization; Uses CANN kernels | -| `mxfp8` (diffusion) | No | No | Yes (A2/A3) | Ascend NPU only; online MXFP8 quantization for diffusion models (e.g., Wan2.2); requires CANN ≥ 8.0.RC3 | - -On AMD, several of these methods use [Aiter](https://github.com/ROCm/aiter) for acceleration -- set `SGLANG_USE_AITER=1` where noted. See [AMD GPU setup](../platforms/amd_gpu.md) for installation and configuration details. - -On Ascend, various layers quantization configurations are supported, see [Ascend NPU quantization](../platforms/ascend/ascend_npu_quantization.md) for details. - -## GEMM Backends for FP4/FP8 Quantization - -:::{note} -Backend selection is supported only for **blockwise FP8** and **NVFP4** GEMM. When running FP8 or FP4 quantized models, you can select the GEMM backend via `--fp8-gemm-backend` and `--fp4-gemm-backend`. -::: - -### `--fp8-gemm-backend` (Blockwise FP8 GEMM) - -| Backend | Hardware | Description | -|---------|----------|-------------| -| `auto` | All | Auto-selects based on hardware | -| `deep_gemm` | SM90, SM100 | JIT-compiled; enabled when DeepGEMM is installed | -| `flashinfer_trtllm` | SM100 | FlashInfer TensorRT-LLM backend; optimal for low-latency | -| `flashinfer_cutlass` | SM100/120 | FlashInfer CUTLASS groupwise FP8 GEMM | -| `flashinfer_deepgemm` | SM90 | Uses swapAB optimization for small M dimensions in decoding | -| `cutlass` | SM90, SM100/120 | sgl-kernel CUTLASS | -| `triton` | All | Fallback; widely compatible | -| `aiter` | ROCm | AMD AITER backend | - -**`auto` selection order:** 1) DeepGEMM (SM90/SM100, installed); 2) FlashInfer TRTLLM (SM100, FlashInfer available); 3) CUTLASS (SM90/SM100/120); 4) AITER (AMD); 5) Triton. **Exception:** SM120 always resolves to Triton. - -### `--fp4-gemm-backend` (NVFP4 GEMM) - -| Backend | Hardware | Description | -|---------|----------|-------------| -| `auto` | SM100/120 | Auto-selects: `flashinfer_cudnn` on SM120; `flashinfer_cutlass` on SM100 | -| `cutlass` | SM100/120 | SGLang CUTLASS kernel | -| `flashinfer_cutlass` | SM100/120 | FlashInfer CUTLASS backend | -| `flashinfer_cudnn` | SM100/120 (CUDA 13+, cuDNN 9.15+) | FlashInfer cuDNN backend; used on SM120 for performance | -| `flashinfer_trtllm` | SM100 | FlashInfer TensorRT-LLM backend | - -When FlashInfer is unavailable for NVFP4, the SGLang CUTLASS kernel is used as an automatic fallback. - -## Offline Quantization - -To load already quantized models, simply load the model weights and config. **Again, if the model has been quantized offline, -there's no need to add `--quantization` argument when starting the engine. The quantization method will be parsed from the -downloaded Hugging Face or msModelSlim config. For example, DeepSeek V3/R1 models are already in FP8, so do not add redundant parameters.** - -```bash -python3 -m sglang.launch_server \ - --model-path hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4 \ - --port 30000 --host 0.0.0.0 -``` - -Take note, if your model is **per-channel quantized (INT8 or FP8) with per-token dynamic quantization activation**, you can opt to include `--quantization w8a8_int8` or `--quantization w8a8_fp8` to invoke the corresponding CUTLASS int8_kernel or fp8_kernel in sgl-kernel. This action will ignore the Hugging Face config's quantization settings. For instance, with `neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8-dynamic`, if you execute with `--quantization w8a8_fp8`, the system will use the `W8A8Fp8Config` from SGLang to invoke the sgl-kernel, rather than the `CompressedTensorsConfig` for vLLM kernels. - -```bash -python3 -m sglang.launch_server \ - --model-path neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8-dynamic \ - --quantization w8a8_fp8 \ - --port 30000 --host 0.0.0.0 -``` - -### Examples of Offline Model Quantization - -#### Using [Unsloth](https://docs.unsloth.ai/basics/inference-and-deployment/sglang-guide) - -We strongly suggest the use of Unsloth to quantize and load the model. Please refer to [SGLang Deployment & Inference Guide with Unsloth](https://docs.unsloth.ai/basics/inference-and-deployment/sglang-guide). - -#### Using [auto-round](https://github.com/intel/auto-round) - -```bash -# Install -pip install auto-round -``` - -- LLM quantization - -```py -# for LLM -from auto_round import AutoRound -model_id = "meta-llama/Llama-3.2-1B-Instruct" -quant_path = "Llama-3.2-1B-Instruct-autoround-4bit" -# Scheme examples: "W2A16", "W3A16", "W4A16", "W8A16", "NVFP4", "MXFP4" (no real kernels), "GGUF:Q4_K_M", etc. -scheme = "W4A16" -format = "auto_round" -autoround = AutoRound(model_id, scheme=scheme) -autoround.quantize_and_save(quant_path, format=format) # quantize and save - -``` - -- VLM quantization -```py -# for VLMs -from auto_round import AutoRoundMLLM -model_name = "Qwen/Qwen2-VL-2B-Instruct" -quant_path = "Qwen2-VL-2B-Instruct-autoround-4bit" -scheme = "W4A16" -format = "auto_round" -autoround = AutoRoundMLLM(model_name, scheme) -autoround.quantize_and_save(quant_path, format=format) # quantize and save - -``` - -- Command Line Usage (Gaudi/CPU/Intel GPU/CUDA) - -```bash -auto-round \ - --model meta-llama/Llama-3.2-1B-Instruct \ - --bits 4 \ - --group_size 128 \ - --format "auto_round" \ - --output_dir ./tmp_autoround -``` - -- known issues - -Several limitations currently affect offline quantized model loading in sglang, These issues might be resolved in future updates of sglang. If you experience any problems, consider using Hugging Face Transformers as an alternative. - -1. Mixed-bit Quantization Limitations - - Mixed-bit quantization is not fully supported. Due to vLLM's layer fusion (e.g., QKV fusion), applying different bit-widths to components within the same fused layer can lead to compatibility issues. - - -2. Limited Support for Quantized MoE Models - - Quantized MoE models may encounter inference issues due to kernel limitations (e.g., lack of support for mlp.gate layer quantization). please try to skip quantizing these layers to avoid such errors. - - -3. Limited Support for Quantized VLMs -
- VLM failure cases - - Qwen2.5-VL-7B - - auto_round:auto_gptq format: Accuracy is close to zero. - - GPTQ format: Fails with: - ``` - The output size is not aligned with the quantized weight shape - ``` - auto_round:auto_awq and AWQ format: These work as expected. -
- -#### Using [GPTQModel](https://github.com/ModelCloud/GPTQModel) - -```bash -# install -pip install gptqmodel --no-build-isolation -v -``` - -```py -from datasets import load_dataset -from gptqmodel import GPTQModel, QuantizeConfig - -model_id = "meta-llama/Llama-3.2-1B-Instruct" -quant_path = "Llama-3.2-1B-Instruct-gptqmodel-4bit" - -calibration_dataset = load_dataset( - "allenai/c4", data_files="en/c4-train.00001-of-01024.json.gz", - split="train" - ).select(range(1024))["text"] - -quant_config = QuantizeConfig(bits=4, group_size=128) # quantization config -model = GPTQModel.load(model_id, quant_config) # load model - -model.quantize(calibration_dataset, batch_size=2) # quantize -model.save(quant_path) # save model -``` - -#### Using [LLM Compressor](https://github.com/vllm-project/llm-compressor/) - -```bash -# install -pip install llmcompressor -``` - -Here, we take quantize `meta-llama/Meta-Llama-3-8B-Instruct` to `FP8` as an example to elaborate on how to do offline quantization. - -```python -from transformers import AutoTokenizer -from llmcompressor.transformers import SparseAutoModelForCausalLM -from llmcompressor.transformers import oneshot -from llmcompressor.modifiers.quantization import QuantizationModifier - -# Step 1: Load the original model. -MODEL_ID = "meta-llama/Meta-Llama-3-8B-Instruct" - -model = SparseAutoModelForCausalLM.from_pretrained( - MODEL_ID, device_map="auto", torch_dtype="auto") -tokenizer = AutoTokenizer.from_pretrained(MODEL_ID) - -# Step 2: Perform offline quantization. -# Step 2.1: Configure the simple PTQ quantization. -recipe = QuantizationModifier( - targets="Linear", scheme="FP8_DYNAMIC", ignore=["lm_head"]) - -# Step 2.2: Apply the quantization algorithm. -oneshot(model=model, recipe=recipe) - -# Step 3: Save the model. -SAVE_DIR = MODEL_ID.split("/")[1] + "-FP8-Dynamic" -model.save_pretrained(SAVE_DIR) -tokenizer.save_pretrained(SAVE_DIR) -``` - -Then, you can directly use the quantized model with `SGLang`, by using the following command: - -```bash -python3 -m sglang.launch_server \ - --model-path $PWD/Meta-Llama-3-8B-Instruct-FP8-Dynamic \ - --port 30000 --host 0.0.0.0 -``` - -#### Using [NVIDIA ModelOpt](https://github.com/NVIDIA/Model-Optimizer) - -NVIDIA Model Optimizer (ModelOpt) provides advanced quantization techniques optimized for NVIDIA hardware. - -**Offline vs. Online Quantization:** - -SGLang supports two modes for ModelOpt. - -* **Offline Quantization (pre-quantized):** - * **Usage:** Download a pre-quantized model from Hugging Face or run `hf_ptq.py` once to create a new quantized checkpoint. Then load this quantized checkpoint. - * **Pros:** Fast server startup, quantization can be validated before deployment, efficient resource usage. - * **Cons:** Requires an extra preparation step. - -* **Online Quantization (quant and serve):** - * **Usage:** Load a standard BF16/FP16 model and add a flag. The engine applies quantization *on startup*. - * **Pros:** Convenient (no new checkpoint needed). - * **Cons:** **High startup time**, increases VRAM usage during initialization (risk of OOM). - -The following sections guide you through using the Offline path: loading pre-quantized models or creating your own checkpoints. - -##### Using Pre-Quantized Checkpoints - -If a model is already quantized (e.g., from Hugging Face), you can load it directly. - -* **FP8 Models:** - Use `--quantization modelopt_fp8`. - ```bash - python3 -m sglang.launch_server \ - --model-path nvidia/Llama-3.1-8B-Instruct-FP8 \ - --quantization modelopt_fp8 \ - --port 30000 - ``` - -* **FP4 Models:** - Use `--quantization modelopt_fp4`. - ```bash - python3 -m sglang.launch_server \ - --model-path nvidia/Llama-3.3-70B-Instruct-NVFP4 \ - --quantization modelopt_fp4 \ - --port 30000 - ``` - -##### Creating Your Own Quantized Checkpoints - -If a pre-quantized checkpoint is not available for your model, you can create one using NVIDIA Model Optimizer's `hf_ptq.py` script. - -**Why quantize?** -- Reduce VRAM usage -- Higher throughput and lower latency -- More flexible deployment (on smaller GPUs) - -**What can be quantized?** -- The entire model -- MLP layers only -- KV cache - -**Key options in `hf_ptq.py`:** - -`--qformat`: Quantization formats `fp8`, `nvfp4`, `nvfp4_mlp_only` - -`--kv_cache_qformat`: KV cache quantization format (default: `fp8`) - -**Note:** The default `kv_cache_qformat` may not be optimal for all use cases. Consider setting this explicitly. - -**Hardware requirements:** Hopper and higher are recommended. Insufficient GPU memory may cause weight offloading, resulting in extremely long quantization time. - -For detailed usage and supported model architectures, see [NVIDIA Model Optimizer LLM PTQ](https://github.com/NVIDIA/Model-Optimizer/tree/main/examples/llm_ptq). - -SGLang includes a streamlined workflow for quantizing models with ModelOpt and automatically exporting them for deployment. - -##### Installation - -First, install ModelOpt: - -```bash -pip install nvidia-modelopt -``` - -##### Quantization and Export Workflow - -SGLang provides an example script that demonstrates the complete ModelOpt quantization and export workflow. Run from the SGLang repository root (see [modelopt_quantize_and_export.py](https://github.com/sgl-project/sglang/blob/main/examples/usage/modelopt_quantize_and_export.py)): - -```bash -# Quantize and export a model using ModelOpt FP8 quantization -python examples/usage/modelopt_quantize_and_export.py quantize \ - --model-path TinyLlama/TinyLlama-1.1B-Chat-v1.0 \ - --export-dir ./quantized_tinyllama_fp8 \ - --quantization-method modelopt_fp8 - -# For FP4 quantization (requires Blackwell GPU) -python examples/usage/modelopt_quantize_and_export.py quantize \ - --model-path TinyLlama/TinyLlama-1.1B-Chat-v1.0 \ - --export-dir ./quantized_tinyllama_fp4 \ - --quantization-method modelopt_fp4 -``` - -##### Available Quantization Methods - -- `modelopt_fp8`: FP8 quantization with optimal performance on NVIDIA Hopper and Blackwell GPUs -- `modelopt_fp4`: FP4 quantization with optimal performance on Nvidia Blackwell GPUs - -##### Python API Usage - -You can also use ModelOpt quantization programmatically: - -```python -import sglang as sgl -from sglang.srt.configs.device_config import DeviceConfig -from sglang.srt.configs.load_config import LoadConfig -from sglang.srt.configs.model_config import ModelConfig -from sglang.srt.model_loader.loader import get_model_loader - -# Configure model with ModelOpt quantization and export -model_config = ModelConfig( - model_path="TinyLlama/TinyLlama-1.1B-Chat-v1.0", - quantization="modelopt_fp8", # or "modelopt_fp4" - trust_remote_code=True, -) - -load_config = LoadConfig( - modelopt_export_path="./exported_model", - modelopt_checkpoint_save_path="./checkpoint.pth", # optional, fake quantized checkpoint -) -device_config = DeviceConfig(device="cuda") - -# Load and quantize the model (export happens automatically) -model_loader = get_model_loader(load_config, model_config) -quantized_model = model_loader.load_model( - model_config=model_config, - device_config=device_config, -) -``` - -##### Deploying Quantized Models - -After quantization and export, you can deploy the model with SGLang: - -```bash -# Deploy the exported quantized model -python -m sglang.launch_server \ - --model-path ./quantized_tinyllama_fp8 \ - --quantization modelopt \ - --port 30000 --host 0.0.0.0 -``` - -Or using the Python API (use the same path as `modelopt_export_path` from the quantize step): - -```python -import sglang as sgl - -def main(): - # Deploy exported ModelOpt quantized model - # Path must match modelopt_export_path from quantize step (e.g., ./exported_model) - llm = sgl.Engine( - model_path="./exported_model", - quantization="modelopt", - ) - - # Run inference - prompts = [ - "Hello, how are you?", - "What is the capital of France?", - ] - sampling_params = { - "temperature": 0.8, - "top_p": 0.95, - "max_new_tokens": 100, - } - - outputs = llm.generate(prompts, sampling_params) - - for i, output in enumerate(outputs): - print(f"Prompt: {prompts[i]}") - print(f"Output: {output['text']}") - -if __name__ == "__main__": - main() - -``` - -##### Advanced Features - -**Checkpoint Management**: Save and restore fake quantized checkpoints for reuse: - -```bash -# Save the fake quantized checkpoint during quantization -python examples/usage/modelopt_quantize_and_export.py quantize \ - --model-path meta-llama/Llama-3.2-1B-Instruct \ - --export-dir ./quantized_model \ - --quantization-method modelopt_fp8 \ - --checkpoint-save-path ./my_checkpoint.pth - -# The checkpoint can be reused for future quantization runs and skip calibration -``` - -**Export-only Workflow**: If you have a pre-existing fake quantized ModelOpt checkpoint, you can export it directly. See [LoadConfig](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/configs/load_config.py) for the full API: - -```python -from sglang.srt.configs.device_config import DeviceConfig -from sglang.srt.configs.load_config import LoadConfig -from sglang.srt.configs.model_config import ModelConfig -from sglang.srt.model_loader.loader import get_model_loader - -model_config = ModelConfig( - model_path="meta-llama/Llama-3.2-1B-Instruct", - quantization="modelopt_fp8", - trust_remote_code=True, -) - -load_config = LoadConfig( - modelopt_checkpoint_restore_path="./my_checkpoint.pth", - modelopt_export_path="./exported_model", -) - -# Load and export the model (DeviceConfig defaults to device="cuda") -model_loader = get_model_loader(load_config, model_config) -model_loader.load_model(model_config=model_config, device_config=DeviceConfig()) -``` - -##### Benefits of ModelOpt - -- **Hardware Optimization**: Specifically optimized for NVIDIA GPU architectures -- **Advanced Quantization**: Supports cutting-edge FP8 and FP4 quantization techniques -- **Seamless Integration**: Automatic export to HuggingFace format for easy deployment -- **Calibration-based**: Uses calibration datasets for optimal quantization quality -- **Production Ready**: Enterprise-grade quantization with NVIDIA support - -#### Using [ModelSlim](https://gitcode.com/Ascend/msmodelslim) -MindStudio-ModelSlim (msModelSlim) is a model offline quantization compression tool launched by MindStudio and optimized for Ascend hardware. - -- **Installation** - - ```bash - # Clone repo and install msmodelslim: - git clone https://gitcode.com/Ascend/msmodelslim.git - cd msmodelslim - bash install.sh - ``` - -- **LLM quantization** - - Download the original floating-point weights of the large model. Taking Qwen3-32B as an example, you can go to [Qwen3-32B](https://huggingface.co/Qwen/Qwen3-32B) to obtain the original model weights. Then install other dependencies (related to the model, refer to the huggingface model card). - > Note: You can find pre-quantized validated models on [modelscope/Eco-Tech](https://modelscope.cn/models/Eco-Tech). - - _Traditional quantification methods require the preparation of calibration data files (```.jsonl``` formats) for calibration in the quantification process._ - ```bash - Qwen3-32B/ # floating-point model downloaded from official HF (or modelscope) repo - msmodelslim/ # msmodelslim repo - |----- lab_calib # calibration date folder (put your dataset here in ```.jsonl``` format or use pre-prepared ones) - |----- some file (such as laos_calib.jsonl) - |----- lab_practice # best practice folder with configs for quantization - |----- model folder (such as qwen3_5_moe folder) # folder with quantization configs - |----- quant_config (such as qwen3_5_moe_w8a8.yaml) # quantization config - |----- another folders - output_folder/ # generated by below command - |----- quant_model_weights-00001-of-0001.safetensors # quantized weights - |----- quant_model_description.json # file with description of the quantization methods for each layer (```W4A4_DYNAMIC```, etc.) - |----- another files (such as config.json, tokenizer.json, etc.) - ``` - Run quantization using one-click quantization (recommended): - ```bash - msmodelslim quant \ - --model_path ${MODEL_PATH} \ - --save_path ${SAVE_PATH} \ - --device npu:0,1 \ - --model_type Qwen3-32B \ - --quant_type w8a8 \ - --trust_remote_code True - ``` - -- **Usage Example** - ```bash - python3 -m sglang.launch_server \ - --model-path $PWD/Qwen3-32B-w8a8 \ - --port 30000 --host 0.0.0.0 - ``` - -- **Available Quantization Methods**: - - [x] ```W4A4_DYNAMIC``` linear with online quantization of activations - - [x] ```W8A8``` linear with offline quantization of activations - - [x] ```W8A8_DYNAMIC``` linear with online quantization of activations - - [x] ```W4A4_DYNAMIC``` MOE with online quantization of activations - - [x] ```W4A8_DYNAMIC``` MOE with online quantization of activations - - [x] ```W8A8_DYNAMIC``` MOE with online quantization of activations - - [ ] ```W4A8``` linear TBD - - [ ] ```W4A16``` linear TBD - - [ ] ```W48A16``` linear TBD - - [ ] ```W4A16``` MoE in progress - - [ ] ```W8A16``` MoE in progress - - [ ] ```KV Cache``` in progress - - [ ] ```Attention``` in progress - - -For more detailed examples of quantization of models, as well as information about their support, see the [examples](https://gitcode.com/Ascend/msmodelslim/blob/master/example/README.md) section in ModelSLim repo. - -## Online Quantization - -To enable online quantization, you can simply specify `--quantization` in the command line. For example, you can launch the server with the following command to enable `FP8` quantization for model `meta-llama/Meta-Llama-3.1-8B-Instruct`: - -```bash -python3 -m sglang.launch_server \ - --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \ - --quantization fp8 \ - --port 30000 --host 0.0.0.0 -``` - -Our team is working on supporting more online quantization methods. SGLang will soon support methods including but not limited to `["awq", "gptq", "marlin", "gptq_marlin", "awq_marlin", "bitsandbytes", "gguf"]`. - -### torchao online quantization method - -SGLang also supports quantization methods based on [torchao](https://github.com/pytorch/ao). You can simply specify `--torchao-config` in the command line to support this feature. For example, if you want to enable `int4wo-128` for model `meta-llama/Meta-Llama-3.1-8B-Instruct`, you can launch the server with the following command: - -```bash -python3 -m sglang.launch_server \ - --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \ - --torchao-config int4wo-128 \ - --port 30000 --host 0.0.0.0 -``` - -SGLang supports the following quantization methods based on torchao `["int8dq", "int8wo", "fp8wo", "fp8dq-per_tensor", "fp8dq-per_row", "int4wo-32", "int4wo-64", "int4wo-128", "int4wo-256"]`. - -Note: According to [this issue](https://github.com/sgl-project/sglang/issues/2219#issuecomment-2561890230), `"int8dq"` method currently has some bugs when using together with cuda graph capture. So we suggest to disable cuda graph capture when using `"int8dq"` method. Namely, please use the following command: - -```bash -python3 -m sglang.launch_server \ - --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \ - --torchao-config int8dq \ - --disable-cuda-graph \ - --port 30000 --host 0.0.0.0 -``` - -### `quark_int4fp8_moe` online quantization method - -SGLang running on AMD GPUs (CDNA3 or CDNA4 architecture) supports the quantization method `--quantization quark_int4fp8_moe`, that will replace [MoE layers](https://github.com/sgl-project/sglang/blob/v0.4.8/python/sglang/srt/layers/moe/fused_moe_triton/layer.py#L271) originally in high precision (bfloat16, float16 or float32) to use weights dynamically quantized to int4, that are upcasted to float8 during inference to run compute in float8 precision with activations dynamically quantized on the fly to float8. - -Other layers (e.g. projections in the attention layers) have their weights quantized online to float8 directly. - -## Diffusion Model Quantization on Ascend NPU - -SGLang-Diffusion supports MXFP8 quantization for diffusion models (such as Wan2.2) on Ascend A5 NPUs, in both online and offline (ModelSlim) modes. This is separate from the LLM serving path and uses the `sglang serve` / `sglang generate` CLI. - -**Requirements:** Ascend A5, CANN ≥ 8.0.RC3 - -### Online MXFP8 - -Pass `--quantization mxfp8` to dynamically quantize FP16/BF16 transformer weights to MXFP8 at load time: - -```bash -sglang serve \ - --model-path Wan-AI/Wan2.2-T2V-A14B-Diffusers \ - --quantization mxfp8 \ - --num-gpus 4 -``` - -### Offline MXFP8 (ModelSlim) - -Pre-quantize with [msModelSlim](https://gitcode.com/Ascend/msmodelslim) and load the checkpoint directly — the quantization scheme is auto-detected from `quant_model_description.json`: - -```bash -sglang generate \ - --model-path /path/to/wan2_2_mxfp8_diffusers \ - --prompt "a beautiful sunset" \ - --save-output -``` - -For the full quantization + format conversion workflow and a complete list of supported schemes, see [Diffusion Quantization on Ascend NPU](../platforms/ascend/ascend_npu_quantization.md#diffusion-model-quantization-on-ascend-npu) and [SGLang-Diffusion Quantization](../diffusion/quantization.md#modelslim). - -## Reference - -- [GPTQModel](https://github.com/ModelCloud/GPTQModel) -- [LLM Compressor](https://github.com/vllm-project/llm-compressor/) -- [NVIDIA Model Optimizer (ModelOpt)](https://github.com/NVIDIA/Model-Optimizer) -- [NVIDIA Model Optimizer LLM PTQ](https://github.com/NVIDIA/Model-Optimizer/tree/main/examples/llm_ptq) -- [Petit: NVFP4 on ROCm](https://github.com/causalflow-ai/petit-kernel) — [LMSYS blog](https://lmsys.org/blog/2025-09-21-petit-amdgpu/), [AMD ROCm blog](https://rocm.blogs.amd.com/artificial-intelligence/fp4-mixed-precision/README.html) -- [Torchao: PyTorch Architecture Optimization](https://github.com/pytorch/ao) -- [vLLM Quantization](https://docs.vllm.ai/en/latest/quantization/) -- [auto-round](https://github.com/intel/auto-round) -- [ModelSlim](https://gitcode.com/Ascend/msmodelslim) diff --git a/docs/advanced_features/quantized_kv_cache.md b/docs/advanced_features/quantized_kv_cache.md deleted file mode 100644 index d2c858492..000000000 --- a/docs/advanced_features/quantized_kv_cache.md +++ /dev/null @@ -1,162 +0,0 @@ -# Quantized KV Cache - -Quantized KV cache reduces the memory footprint of key-value cache storage by using lower-precision data types (FP8 or FP4) instead of the default model precision in BF16. During autoregressive generation, LLMs cache previously computed key-value pairs to avoid redundant calculations. The KV cache typically consumes a significant portion of GPU memory, especially for long sequences. - -Quantized KV cache is a memory optimization technique that primarily benefits throughput by allowing more tokens to be cached, but may introduce minimal accuracy degradation depending on the quantization format used. - -```{warning} -**Performance Warning**: When quantized KV cache must be dequantized before use in attention operations, performance can be extremely slow if dequantization is not fused with the attention kernel. Always verify that your chosen attention backend supports quantized KV cache. Backends without fused support may experience significant throughput degradation, potentially negating the memory benefits. - -**Backend Support**: Not all attention backends support quantized KV cache. Refer to [Attention Backend](attention_backend.md) for which backends support it. -``` - -## Supported Formats - -SGLang supports the following quantized KV cache formats: - -### FP8 Format - -[OCP (Open Compute Project)](https://www.opencompute.org) specifies two common 8-bit floating point formats: - -- **E5M2** (5 exponent bits, 2 mantissa bits): Larger dynamic range (±57344.0), lower precision -- **E4M3** (4 exponent bits, 3 mantissa bits): Higher precision, smaller dynamic range (±240.0) - -### FP4 Format - -```{warning} -FP4 quantization is currently experimental. -``` - -[OCP (Open Compute Project)](https://www.opencompute.org) specifies MXFP4 (Microscaling FP4), a 4-bit floating-point format: - -- **E2M1** (1 sign bit, 2 exponent bits, 1 mantissa bit): Uses block-based microscaling where tensors are divided into blocks of consecutive elements, with each block sharing a single 8-bit exponential scaling factor. While OCP specifies blocks of 32 elements, SGLang's current implementation uses blocks of 16 elements for KV cache quantization. - -## Usage - -### Enabling Quantized KV Cache - -To enable quantized KV cache, use the `--kv-cache-dtype` argument when launching the server: - -```bash -# Enable FP8 E5M2 KV cache -python3 -m sglang.launch_server \ - --model-path deepseek-ai/DeepSeek-R1-0528 \ - --kv-cache-dtype fp8_e5m2 \ - -# Enable FP8 E4M3 KV cache -python3 -m sglang.launch_server \ - --model-path deepseek-ai/DeepSeek-R1-0528 \ - --kv-cache-dtype fp8_e4m3 \ - -# Enable FP4 E2M1 KV cache -python3 -m sglang.launch_server \ - --model-path nvidia/DeepSeek-R1-0528-NVFP4 \ - --kv-cache-dtype fp4_e2m1 \ -``` - -### Scaling Factors - -FP8 quantization requires scaling factors to properly quantize and dequantize the KV cache. - -```{note} -Currently, only per-tensor (scalar) scaling factors are supported. -``` - -Scaling factors can be: - -- **Loaded from checkpoints**: Pre-quantized models (e.g., ModelOpt) may include `k_scale` and `v_scale` parameters that are automatically loaded -- **Provided via JSON**: Supply scaling factors via `--quantization-param-path`. - -The JSON file should follow this format: - -```json -{ - "kv_cache": { - "dtype": "float8_e4m3fn", - "scaling_factor": { - "0": { - "0": 1.0, - "1": 1.0 - } - } - } -} -``` - -Where the outer keys in `scaling_factor` are tensor parallel ranks and inner keys are layer indices. - -```{warning} -If scaling factors are not provided and not found in the checkpoint, it will default to 1.0, which may cause accuracy issues. -``` - -```{tip} -**FP4 (MXFP4)**: Unlike FP8, FP4 quantization handles scaling factors automatically on-the-fly during quantization and dequantization. No pre-quantized models or external scaling factor files are required—the block-based scaling factors are computed dynamically as needed. -``` - -## Performance Considerations - -### Memory Savings - -Quantized KV cache provides significant memory savings: -- **BF16 → FP4**: Supports approximately 3.56× more tokens than BF16 (accounting for scaling factor overhead) - -```{note} -FP4 and FP8 quantization require additional memory for block-based scaling factors, which reduces the effective memory savings compared to the raw bit-width reduction. FP4 with block size 16 supports approximately 1.78× more tokens than FP8, and approximately 3.56× more tokens than BF16. The relative token capacity between FP8 and BF16 can be derived from these ratios. -``` - -This enables longer context lengths or more concurrent requests within the same memory budget. - -### Accuracy Impact - -#### FP8 Accuracy - -FP8 E4M3 quantization typically introduces minimal accuracy degradation. The impact depends on model architecture, sequence length, and quantization format (generally, E4M3 has better accuracy than E5M2). - -#### FP4 Accuracy - -FP4 (MXFP4) quantization provides significant memory savings with varying accuracy impact depending on model size and dataset complexity. Preliminary accuracy test results from [PR #10078](https://github.com/sgl-project/sglang/pull/10078) (MLA) and [PR #12612](https://github.com/sgl-project/sglang/pull/12612) (MHA) show: - -**Large Models (e.g., Qwen3-235B-A22B, DeepSeek-R1-0528)** - -On large-scale models, FP4 maintains accuracy close to FP8/BF16, especially on simpler datasets: - -| Model | Dataset | KV16 | KV8 (FP8 E4M3) | KV4 (FP4 E2M1) | -|-------|---------|------|----------------|----------------| -| Qwen3-235B-A22B | gsm8k | 0.9168 | 0.9181 | 0.9186 | -| Qwen3-235B-A22B | aime25 | 0.7733 | 0.7333 | 0.6000 | -| Qwen3-235B-A22B | gpqa_diamond | 0.7010 | 0.6899 | 0.6778 | -| DeepSeek-R1-0528 | gsm8k | 0.9157 | 0.9154 | 0.9124 | -| DeepSeek-R1-0528 | aime25 | 0.5067 | 0.4934 | 0.4000 | -| DeepSeek-R1-0528 | gpqa_diamond | 0.7707 | 0.7697 | 0.7273 | - -**Smaller Models (e.g., GPT-OSS-120B)** - -On smaller models, FP4 shows more pronounced accuracy drops, particularly on challenging datasets: - -| Model | Dataset | KV16 | KV8 (FP8 E4M3) | KV4 (FP4 E2M1) | -|-------|---------|------|----------------|----------------| -| GPT-OSS-120B | gsm8k | 0.9161 | 0.9163 | 0.9152 | -| GPT-OSS-120B | aime25 | 0.7533 | 0.7667 | 0.3533 | -| GPT-OSS-120B | gpqa_diamond | 0.5081 | 0.5434 | 0.3202 | - -**Key Observations:** - -- **Simple datasets (e.g., gsm8k)**: FP4 maintains accuracy close to FP8/BF16 across model sizes -- **Model size matters**: Large models (200B+ parameters) generally tolerate FP4 quantization better than smaller models -- **Context length**: Accuracy degradation may be more pronounced in long-context scenarios, as the accumulation of the quantization error may become significant. - -```{tip} -Evaluate FP4 accuracy on your specific model and workload. Large models on simpler tasks typically show minimal degradation, while smaller models or complex reasoning tasks may require FP8 or BF16 for acceptable accuracy. -``` - -## Best Practices - -- **Use pre-quantized models**: Prefer models quantized offline with scaling factors included in the checkpoint. -- **Choose the right format**: Use `fp8_e4m3` for better accuracy (recommended), `fp8_e5m2` for larger dynamic range, or `fp4_e2m1` for maximum memory savings (experimental) -- **Check backend compatibility**: Verify that your chosen attention backend supports quantized KV cache - -```{seealso} -- [Quantization](quantization.md) -- [Attention Backend](attention_backend.md) -- [Server Arguments](server_arguments.md) -``` diff --git a/docs/advanced_features/rfork.md b/docs/advanced_features/rfork.md deleted file mode 100644 index e4b513328..000000000 --- a/docs/advanced_features/rfork.md +++ /dev/null @@ -1,72 +0,0 @@ -# R-Fork - -R-Fork (Tensor Remote Fork) is a novel weight loading methodology that leverages efficient inter-node GPU-to-GPU data transfer path to load tensors from a running SGLang instance to a new instance with zero-copy. It can significantly optimize the SGLang instance boot-up time by reducing model weights loading from several minutes to mere seconds. - -To learn more details about R-Fork, please check ** R-Fork blog ** - -## Usage - -| Argument | Usage | -|--------------|--------------------------------------------| -| load-format | set to `remote_instance` to enable R-Fork. | -| remote-instance-weight-loader-backend | `nccl`, `transfer_engine`, or `modelexpress`. Default is `nccl`. | -| remote-instance-weight-loader-seed-instance-ip | IP address of the seed instance who will provide the model weight. Used by `nccl` and `transfer_engine` backends. | -| remote-instance-weight-loader-seed-instance-service-port | the port that the seed instance's HTTP server is listening on. Used by `nccl` and `transfer_engine` backends. | -| remote-instance-weight-loader-send-weights-group-ports | the list of available ports on the seed instance that will be used to build NCCL communication groups between seed and client instance. Only needed by `nccl` backend. | -| remote-instance-weight-loader-start-seed-via-transfer-engine | set to start seed service that supports TransferEngine as backend. Needed for seed instances when using `transfer_engine` as backend. | -| modelexpress-config | JSON config for `modelexpress` backend. Keys: `"url"` (required, gRPC host:port of ModelExpress server), `"model_name"` (optional, defaults to `--model-path`), `"source"` (optional bool, `true` for seed mode). | - -### NCCL as backend - -seed instance: -```shell -python -m sglang.launch_server [args] -``` - -client instance: -```shell -python -m sglang.launch_server [args] \ - --load-format remote_instance \ - --remote-instance-weight-loader-seed-instance-ip [seed_instance_ip] \ - --remote-instance-weight-loader-seed-instance-service-port [seed_instance_service_port] \ - --remote-instance-weight-loader-send-weights-group-ports [send_weights_nccl_group_ports_list] \ - --remote-instance-weight-loader-backend nccl -``` - -### TransferEngine as backend - -seed instance: -```shell -python -m sglang.launch_server [args] \ - --remote-instance-weight-loader-start-seed-via-transfer-engine -``` - -```shell -python -m sglang.launch_server [args] \ - --load-format remote_instance \ - --remote-instance-weight-loader-seed-instance-ip [seed_instance_ip] \ - --remote-instance-weight-loader-seed-instance-service-port [seed_instance_service_port] \ - --remote-instance-weight-loader-backend transfer_engine -``` - -### ModelExpress as backend - -[ModelExpress](https://github.com/ai-dynamo/modelexpress) is a coordination service that manages P2P weight transfer metadata. It removes the need for direct seed IP/port configuration by providing a centralized registry that seeds publish to and clients discover from. Under the hood it uses TransferEngine (Mooncake) for the actual RDMA data transfer. - -A running ModelExpress server is required. See the [ModelExpress documentation](https://github.com/ai-dynamo/modelexpress) for setup instructions. - -seed instance: -```shell -python -m sglang.launch_server [args] \ - --modelexpress-config '{"url": "[modelexpress_grpc_host:port]", "model_name": "[model_name]", "source": true}' -``` - -client instance: -```shell -python -m sglang.launch_server [args] \ - --load-format remote_instance \ - --remote-instance-weight-loader-backend modelexpress \ - --modelexpress-config '{"url": "[modelexpress_grpc_host:port]", "model_name": "[model_name]"}' -``` - -The seed publishes its TransferEngine session ID and tensor layout to ModelExpress. The client queries ModelExpress to discover the seed, then pulls weights directly via RDMA. This enables dynamic seed discovery without hardcoding IPs, and supports multiple models through a single ModelExpress instance. diff --git a/docs/advanced_features/separate_reasoning.ipynb b/docs/advanced_features/separate_reasoning.ipynb deleted file mode 100644 index 6277dd8bd..000000000 --- a/docs/advanced_features/separate_reasoning.ipynb +++ /dev/null @@ -1,377 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Reasoning Parser\n", - "\n", - "SGLang supports parsing reasoning content out from \"normal\" content for reasoning models such as [DeepSeek R1](https://huggingface.co/deepseek-ai/DeepSeek-R1).\n", - "\n", - "## Supported Models & Parsers\n", - "\n", - "| Model | Reasoning tags | Parser | Notes |\n", - "|---------|-----------------------------|------------------|-------|\n", - "| [DeepSeek‑R1 series](https://huggingface.co/collections/deepseek-ai/deepseek-r1-678e1e131c0169c0bc89728d) | `` … `` | `deepseek-r1` | Supports all variants (R1, R1-0528, R1-Distill) |\n", - "| [DeepSeek‑V3 series](https://huggingface.co/deepseek-ai/DeepSeek-V3.1) | `` … `` | `deepseek-v3` | Including [DeepSeek‑V3.2](https://huggingface.co/deepseek-ai/DeepSeek-V3.2-Exp). Supports `thinking` parameter |\n", - "| [Standard Qwen3 models](https://huggingface.co/collections/Qwen/qwen3-67dd247413f0e2e4f653967f) | `` … `` | `qwen3` | Supports `enable_thinking` parameter |\n", - "| [Qwen3-Thinking models](https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507) | `` … `` | `qwen3` or `qwen3-thinking` | Always generates thinking content |\n", - "| [Kimi K2 Thinking](https://huggingface.co/moonshotai/Kimi-K2-Thinking) | `◁think▷` … `◁/think▷` | `kimi_k2` | Uses special thinking delimiters. Also requires `--tool-call-parser kimi_k2` for tool use. |\n", - "| [GPT OSS](https://huggingface.co/openai/gpt-oss-120b) | `<\\|channel\\|>analysis<\\|message\\|>` … `<\\|end\\|>` | `gpt-oss` | N/A |\n", - "### Model-Specific Behaviors\n", - "\n", - "**DeepSeek-R1 Family:**\n", - "- DeepSeek-R1: No `` start tag, jumps directly to thinking content\n", - "- DeepSeek-R1-0528: Generates both `` start and `` end tags\n", - "- Both are handled by the same `deepseek-r1` parser\n", - "\n", - "**DeepSeek-V3 Family:**\n", - "- DeepSeek-V3.1/V3.2: Hybrid model supporting both thinking and non-thinking modes, use the `deepseek-v3` parser and `thinking` parameter (NOTE: not `enable_thinking`)\n", - "\n", - "**Qwen3 Family:**\n", - "- Standard Qwen3 (e.g., Qwen3-2507): Use `qwen3` parser, supports `enable_thinking` in chat templates\n", - "- Qwen3-Thinking (e.g., Qwen3-235B-A22B-Thinking-2507): Use `qwen3` or `qwen3-thinking` parser, always thinks\n", - "\n", - "**Kimi K2:**\n", - "- Kimi K2 Thinking: Uses special `◁think▷` and `◁/think▷` tags. For agentic tool use, also specify `--tool-call-parser kimi_k2`.\n", - "\n", - "**GPT OSS:**\n", - "- GPT OSS: Uses special `<|channel|>analysis<|message|>` and `<|end|>` tags" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Usage\n", - "\n", - "### Launching the Server" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Specify the `--reasoning-parser` option." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import requests\n", - "from openai import OpenAI\n", - "from sglang.test.doc_patch import launch_server_cmd\n", - "from sglang.utils import wait_for_server, print_highlight, terminate_process\n", - "\n", - "server_process, port = launch_server_cmd(\n", - " \"python3 -m sglang.launch_server --model-path deepseek-ai/DeepSeek-R1-Distill-Qwen-7B --host 0.0.0.0 --reasoning-parser deepseek-r1 --log-level warning\"\n", - ")\n", - "\n", - "wait_for_server(f\"http://localhost:{port}\", process=server_process)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Note that `--reasoning-parser` defines the parser used to interpret responses." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### OpenAI Compatible API\n", - "\n", - "Using the OpenAI compatible API, the contract follows the [DeepSeek API design](https://api-docs.deepseek.com/guides/reasoning_model) established with the release of DeepSeek-R1:\n", - "\n", - "- `reasoning_content`: The content of the CoT.\n", - "- `content`: The content of the final answer." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Initialize OpenAI-like client\n", - "client = OpenAI(api_key=\"None\", base_url=f\"http://0.0.0.0:{port}/v1\")\n", - "model_name = client.models.list().data[0].id\n", - "\n", - "messages = [\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"What is 1+3?\",\n", - " }\n", - "]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Non-Streaming Request" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "response_non_stream = client.chat.completions.create(\n", - " model=model_name,\n", - " messages=messages,\n", - " temperature=0.6,\n", - " top_p=0.95,\n", - " stream=False, # Non-streaming\n", - " extra_body={\"separate_reasoning\": True},\n", - ")\n", - "print_highlight(\"==== Reasoning ====\")\n", - "print_highlight(response_non_stream.choices[0].message.reasoning_content)\n", - "\n", - "print_highlight(\"==== Text ====\")\n", - "print_highlight(response_non_stream.choices[0].message.content)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Streaming Request" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "response_stream = client.chat.completions.create(\n", - " model=model_name,\n", - " messages=messages,\n", - " temperature=0.6,\n", - " top_p=0.95,\n", - " stream=True, # Non-streaming\n", - " extra_body={\"separate_reasoning\": True},\n", - ")\n", - "\n", - "reasoning_content = \"\"\n", - "content = \"\"\n", - "for chunk in response_stream:\n", - " if chunk.choices[0].delta.content:\n", - " content += chunk.choices[0].delta.content\n", - " if chunk.choices[0].delta.reasoning_content:\n", - " reasoning_content += chunk.choices[0].delta.reasoning_content\n", - "\n", - "print_highlight(\"==== Reasoning ====\")\n", - "print_highlight(reasoning_content)\n", - "\n", - "print_highlight(\"==== Text ====\")\n", - "print_highlight(content)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Optionally, you can buffer the reasoning content to the last reasoning chunk (or the first chunk after the reasoning content)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "response_stream = client.chat.completions.create(\n", - " model=model_name,\n", - " messages=messages,\n", - " temperature=0.6,\n", - " top_p=0.95,\n", - " stream=True, # Non-streaming\n", - " extra_body={\"separate_reasoning\": True, \"stream_reasoning\": False},\n", - ")\n", - "\n", - "reasoning_content = \"\"\n", - "content = \"\"\n", - "for chunk in response_stream:\n", - " if chunk.choices[0].delta.content:\n", - " content += chunk.choices[0].delta.content\n", - " if chunk.choices[0].delta.reasoning_content:\n", - " reasoning_content += chunk.choices[0].delta.reasoning_content\n", - "\n", - "print_highlight(\"==== Reasoning ====\")\n", - "print_highlight(reasoning_content)\n", - "\n", - "print_highlight(\"==== Text ====\")\n", - "print_highlight(content)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The reasoning separation is enable by default when specify . \n", - "**To disable it, set the `separate_reasoning` option to `False` in request.**" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "response_non_stream = client.chat.completions.create(\n", - " model=model_name,\n", - " messages=messages,\n", - " temperature=0.6,\n", - " top_p=0.95,\n", - " stream=False, # Non-streaming\n", - " extra_body={\"separate_reasoning\": False},\n", - ")\n", - "\n", - "print_highlight(\"==== Original Output ====\")\n", - "print_highlight(response_non_stream.choices[0].message.content)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### SGLang Native API " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from transformers import AutoTokenizer\n", - "\n", - "tokenizer = AutoTokenizer.from_pretrained(\"deepseek-ai/DeepSeek-R1-Distill-Qwen-7B\")\n", - "input = tokenizer.apply_chat_template(\n", - " messages, tokenize=False, add_generation_prompt=True, return_dict=False\n", - ")\n", - "\n", - "gen_url = f\"http://localhost:{port}/generate\"\n", - "gen_data = {\n", - " \"text\": input,\n", - " \"sampling_params\": {\n", - " \"skip_special_tokens\": False,\n", - " \"max_new_tokens\": 1024,\n", - " \"temperature\": 0.6,\n", - " \"top_p\": 0.95,\n", - " },\n", - "}\n", - "gen_response = requests.post(gen_url, json=gen_data).json()[\"text\"]\n", - "\n", - "print_highlight(\"==== Original Output ====\")\n", - "print_highlight(gen_response)\n", - "\n", - "parse_url = f\"http://localhost:{port}/separate_reasoning\"\n", - "separate_reasoning_data = {\n", - " \"text\": gen_response,\n", - " \"reasoning_parser\": \"deepseek-r1\",\n", - "}\n", - "separate_reasoning_response_json = requests.post(\n", - " parse_url, json=separate_reasoning_data\n", - ").json()\n", - "print_highlight(\"==== Reasoning ====\")\n", - "print_highlight(separate_reasoning_response_json[\"reasoning_text\"])\n", - "print_highlight(\"==== Text ====\")\n", - "print_highlight(separate_reasoning_response_json[\"text\"])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "terminate_process(server_process)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Offline Engine API" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import sglang as sgl\n", - "from sglang.srt.parser.reasoning_parser import ReasoningParser\n", - "from sglang.utils import print_highlight\n", - "\n", - "llm = sgl.Engine(model_path=\"deepseek-ai/DeepSeek-R1-Distill-Qwen-7B\")\n", - "tokenizer = AutoTokenizer.from_pretrained(\"deepseek-ai/DeepSeek-R1-Distill-Qwen-7B\")\n", - "input = tokenizer.apply_chat_template(\n", - " messages, tokenize=False, add_generation_prompt=True, return_dict=False\n", - ")\n", - "sampling_params = {\n", - " \"max_new_tokens\": 1024,\n", - " \"skip_special_tokens\": False,\n", - " \"temperature\": 0.6,\n", - " \"top_p\": 0.95,\n", - "}\n", - "result = llm.generate(prompt=input, sampling_params=sampling_params)\n", - "\n", - "generated_text = result[\"text\"] # Assume there is only one prompt\n", - "\n", - "print_highlight(\"==== Original Output ====\")\n", - "print_highlight(generated_text)\n", - "\n", - "parser = ReasoningParser(\"deepseek-r1\")\n", - "reasoning_text, text = parser.parse_non_stream(generated_text)\n", - "print_highlight(\"==== Reasoning ====\")\n", - "print_highlight(reasoning_text)\n", - "print_highlight(\"==== Text ====\")\n", - "print_highlight(text)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "llm.shutdown()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Supporting New Reasoning Model Schemas\n", - "\n", - "For future reasoning models, you can implement the reasoning parser as a subclass of `BaseReasoningFormatDetector` in `python/sglang/srt/reasoning_parser.py` and specify the reasoning parser for new reasoning model schemas accordingly." - ] - } - ], - "metadata": { - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/docs/advanced_features/server_arguments.md b/docs/advanced_features/server_arguments.md deleted file mode 100644 index 8a88b0ff2..000000000 --- a/docs/advanced_features/server_arguments.md +++ /dev/null @@ -1,574 +0,0 @@ -# Server Arguments - -This page provides a list of server arguments used in the command line to configure the behavior -and performance of the language model server during deployment. These arguments enable users to -customize key aspects of the server, including model selection, parallelism policies, -memory management, and optimization techniques. -You can find all arguments by `python3 -m sglang.launch_server --help` - -## Common launch commands - -- To use a configuration file, create a YAML file with your server arguments and specify it with `--config`. CLI arguments will override config file values. - - ```bash - # Create config.yaml - cat > config.yaml << EOF - model-path: meta-llama/Meta-Llama-3-8B-Instruct - host: 0.0.0.0 - port: 30000 - tensor-parallel-size: 2 - enable-metrics: true - log-requests: true - EOF - - # Launch server with config file - python -m sglang.launch_server --config config.yaml - ``` - -- To enable multi-GPU tensor parallelism, add `--tp 2`. If it reports the error "peer access is not supported between these two devices", add `--enable-p2p-check` to the server launch command. - - ```bash - python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3-8B-Instruct --tp 2 - ``` - -- To enable multi-GPU data parallelism, add `--dp 2`. Data parallelism is better for throughput if there is enough memory. It can also be used together with tensor parallelism. The following command uses 4 GPUs in total. We recommend [SGLang Model Gateway (former Router)](../advanced_features/sgl_model_gateway.md) for data parallelism. - - ```bash - python -m sglang_router.launch_server --model-path meta-llama/Meta-Llama-3-8B-Instruct --dp 2 --tp 2 - ``` - -- If you see out-of-memory errors during serving, try to reduce the memory usage of the KV cache pool by setting a smaller value of `--mem-fraction-static`. The default value is `0.9`. - - ```bash - python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3-8B-Instruct --mem-fraction-static 0.7 - ``` - -- See [hyperparameter tuning](hyperparameter_tuning.md) on tuning hyperparameters for better performance. -- For docker and Kubernetes runs, you need to set up shared memory which is used for communication between processes. See `--shm-size` for docker and `/dev/shm` size update for Kubernetes manifests. -- If you see out-of-memory errors during prefill for long prompts, try to set a smaller chunked prefill size. - - ```bash - python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3-8B-Instruct --chunked-prefill-size 4096 - ``` -- To enable fp8 weight quantization, add `--quantization fp8` on a fp16 checkpoint or directly load a fp8 checkpoint without specifying any arguments. -- To enable fp8 kv cache quantization, add `--kv-cache-dtype fp8_e4m3` or `--kv-cache-dtype fp8_e5m2`. -- To enable deterministic inference and batch invariant operations, add `--enable-deterministic-inference`. More details can be found in [deterministic inference document](../advanced_features/deterministic_inference.md). -- If the model does not have a chat template in the Hugging Face tokenizer, you can specify a [custom chat template](../references/custom_chat_template.md). If the tokenizer has multiple named templates (e.g., 'default', 'tool_use'), you can select one using `--hf-chat-template-name tool_use`. -- To run tensor parallelism on multiple nodes, add `--nnodes 2`. If you have two nodes with two GPUs on each node and want to run TP=4, let `sgl-dev-0` be the hostname of the first node and `50000` be an available port, you can use the following commands. If you meet deadlock, please try to add `--disable-cuda-graph` -- (Note: This feature is out of maintenance and might cause error) To enable `torch.compile` acceleration, add `--enable-torch-compile`. It accelerates small models on small batch sizes. By default, the cache path is located at `/tmp/torchinductor_root`, you can customize it using environment variable `TORCHINDUCTOR_CACHE_DIR`. For more details, please refer to [PyTorch official documentation](https://pytorch.org/tutorials/recipes/torch_compile_caching_tutorial.html) and [Enabling cache for torch.compile](https://docs.sglang.io/references/torch_compile_cache.html). - ```bash - # Node 0 - python -m sglang.launch_server \ - --model-path meta-llama/Meta-Llama-3-8B-Instruct \ - --tp 4 \ - --dist-init-addr sgl-dev-0:50000 \ - --nnodes 2 \ - --node-rank 0 - - # Node 1 - python -m sglang.launch_server \ - --model-path meta-llama/Meta-Llama-3-8B-Instruct \ - --tp 4 \ - --dist-init-addr sgl-dev-0:50000 \ - --nnodes 2 \ - --node-rank 1 - ``` - -Please consult the documentation below and [server_args.py](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/server_args.py) to learn more about the arguments you may provide when launching a server. - -## Model and tokenizer -| Argument | Description | Defaults | Options | -| --- | --- | --- | --- | -| `--model-path`
`--model` | The path of the model weights. This can be a local folder or a Hugging Face repo ID. | `None` | Type: str | -| `--tokenizer-path` | The path of the tokenizer. | `None` | Type: str | -| `--tokenizer-mode` | Tokenizer mode. 'auto' will use the fast tokenizer if available, and 'slow' will always use the slow tokenizer. | `auto` | `auto`, `slow` | -| `--tokenizer-worker-num` | The worker num of the tokenizer manager. | `1` | Type: int | -| `--skip-tokenizer-init` | If set, skip init tokenizer and pass input_ids in generate request. | `False` | bool flag (set to enable) | -| `--load-format` | The format of the model weights to load. "auto" will try to load the weights in the safetensors format and fall back to the pytorch bin format if safetensors format is not available. "pt" will load the weights in the pytorch bin format. "safetensors" will load the weights in the safetensors format. "npcache" will load the weights in pytorch format and store a numpy cache to speed up the loading. "dummy" will initialize the weights with random values, which is mainly for profiling."gguf" will load the weights in the gguf format. "bitsandbytes" will load the weights using bitsandbytes quantization."layered" loads weights layer by layer so that one can quantize a layer before loading another to make the peak memory envelope smaller. "flash_rl" will load the weights in flash_rl format. "fastsafetensors" and "private" are also supported. "runai_streamer" enables direct model loading from object storage and shared file systems.| `auto` | `auto`, `pt`, `safetensors`, `npcache`, `dummy`, `sharded_state`, `gguf`, `bitsandbytes`, `layered`, `flash_rl`, `remote`, `remote_instance`, `fastsafetensors`, `private`, `runai_streamer` | -| `--model-loader-extra-config` | Extra config for model loader. This will be passed to the model loader corresponding to the chosen load_format. | `{}` | Type: str | -| `--trust-remote-code` | Whether or not to allow for custom models defined on the Hub in their own modeling files. | `False` | bool flag (set to enable) | -| `--context-length` | The model's maximum context length. Defaults to None (will use the value from the model's config.json instead). | `None` | Type: int | -| `--is-embedding` | Whether to use a CausalLM as an embedding model. | `False` | bool flag (set to enable) | -| `--enable-multimodal` | Enable the multimodal functionality for the served model. If the model being served is not multimodal, nothing will happen | `None` | bool flag (set to enable) | -| `--revision` | The specific model version to use. It can be a branch name, a tag name, or a commit id. If unspecified, will use the default version. | `None` | Type: str | -| `--model-impl` | Which implementation of the model to use. * "auto" will try to use the SGLang implementation if it exists and fall back to the Transformers implementation if no SGLang implementation is available. * "sglang" will use the SGLang model implementation. * "transformers" will use the Transformers model implementation. | `auto` | Type: str | - -## HTTP server -| Argument | Description | Defaults | Options | -| --- | --- | --- | --- | -| `--host` | The host of the HTTP server. | `127.0.0.1` | Type: str | -| `--port` | The port of the HTTP server. | `30000` | Type: int | -| `--fastapi-root-path` | App is behind a path based routing proxy. | `""` | Type: str | -| `--grpc-mode` | If set, use gRPC server instead of HTTP server. | `False` | bool flag (set to enable) | -| `--skip-server-warmup` | If set, skip warmup. | `False` | bool flag (set to enable) | -| `--warmups` | Specify custom warmup functions (csv) to run before server starts eg. --warmups=warmup_name1,warmup_name2 will run the functions `warmup_name1` and `warmup_name2` specified in warmup.py before the server starts listening for requests | `None` | Type: str | -| `--nccl-port` | The port for NCCL distributed environment setup. Defaults to a random port. | `None` | Type: int | -| `--checkpoint-engine-wait-weights-before-ready` | If set, the server will wait for initial weights to be loaded via checkpoint-engine or other update methods before serving inference requests. | `False` | bool flag (set to enable) | - -## Quantization and data type -| Argument | Description | Defaults | Options | -| --- | --- | --- | --- | -| `--dtype` | Data type for model weights and activations. * "auto" will use FP16 precision for FP32 and FP16 models, and BF16 precision for BF16 models. * "half" for FP16. Recommended for AWQ quantization. * "float16" is the same as "half". * "bfloat16" for a balance between precision and range. * "float" is shorthand for FP32 precision. * "float32" for FP32 precision. | `auto` | `auto`, `half`, `float16`, `bfloat16`, `float`, `float32` | -| `--quantization` | The quantization method. | `None` | `awq`, `fp8`, `gptq`, `marlin`, `gptq_marlin`, `awq_marlin`, `bitsandbytes`, `gguf`, `modelopt`, `modelopt_fp8`, `modelopt_fp4`, `petit_nvfp4`, `w8a8_int8`, `w8a8_fp8`, `moe_wna16`, `qoq`, `w4afp8`, `mxfp4`, `mxfp8`, `auto-round`, `compressed-tensors`, `modelslim`, `quark_int4fp8_moe` | -| `--quantization-param-path` | Path to the JSON file containing the KV cache scaling factors. This should generally be supplied, when KV cache dtype is FP8. Otherwise, KV cache scaling factors default to 1.0, which may cause accuracy issues. | `None` | Type: Optional[str] | -| `--kv-cache-dtype` | Data type for kv cache storage. "auto" will use model data type. "bf16" or "bfloat16" for BF16 KV cache. "fp8_e5m2" and "fp8_e4m3" are supported for CUDA 11.8+. "fp4_e2m1" (only mxfp4) is supported for CUDA 12.8+ and PyTorch 2.8.0+ | `auto` | `auto`, `fp8_e5m2`, `fp8_e4m3`, `bf16`, `bfloat16`, `fp4_e2m1` | -| `--enable-fp32-lm-head` | If set, the LM head outputs (logits) are in FP32. | `False` | bool flag (set to enable) | -| `--modelopt-quant` | The ModelOpt quantization configuration. Supported values: 'fp8', 'int4_awq', 'w4a8_awq', 'nvfp4', 'nvfp4_awq'. This requires the NVIDIA Model Optimizer library to be installed: pip install nvidia-modelopt | `None` | Type: str | -| `--modelopt-checkpoint-restore-path` | Path to restore a previously saved ModelOpt quantized checkpoint. If provided, the quantization process will be skipped and the model will be loaded from this checkpoint. | `None` | Type: str | -| `--modelopt-checkpoint-save-path` | Path to save the ModelOpt quantized checkpoint after quantization. This allows reusing the quantized model in future runs. | `None` | Type: str | -| `--modelopt-export-path` | Path to export the quantized model in HuggingFace format after ModelOpt quantization. The exported model can then be used directly with SGLang for inference. If not provided, the model will not be exported. | `None` | Type: str | -| `--quantize-and-serve` | Quantize the model with ModelOpt and immediately serve it without exporting. This is useful for development and prototyping. For production, it's recommended to use separate quantization and deployment steps. | `False` | bool flag (set to enable) | -| `--rl-quant-profile` | Path to the FlashRL quantization profile. Required when using --load-format flash_rl. | `None` | Type: str | -| `--enable-quant-communications` | Enable INT8 quantization of TP communications (Supported only for NPU for Qwen3 series). | `False` | bool flag (set to enable) | - -## Memory and scheduling -| Argument | Description | Defaults | Options | -| --- | --- | --- | --- | -| `--mem-fraction-static` | The fraction of the memory used for static allocation (model weights and KV cache memory pool). Use a smaller value if you see out-of-memory errors. | `None` | Type: float | -| `--max-running-requests` | The maximum number of running requests. | `None` | Type: int | -| `--max-queued-requests` | The maximum number of queued requests. This option is ignored when using disaggregation-mode. | `None` | Type: int | -| `--max-total-tokens` | The maximum number of tokens in the memory pool. If not specified, it will be automatically calculated based on the memory usage fraction. This option is typically used for development and debugging purposes. | `None` | Type: int | -| `--chunked-prefill-size` | The maximum number of tokens in a chunk for the chunked prefill. Setting this to -1 means disabling chunked prefill. | `None` | Type: int | -| `--prefill-max-requests` | The maximum number of requests in a prefill batch. If not specified, there is no limit. | `None` | Type: int | -| `--enable-dynamic-chunking` | Enable dynamic chunk size adjustment for pipeline parallelism. When enabled, chunk sizes are dynamically calculated based on fitted function to maintain consistent execution time across chunks. | `False` | bool flag (set to enable) | -| `--max-prefill-tokens` | The maximum number of tokens in a prefill batch. The real bound will be the maximum of this value and the model's maximum context length. | `16384` | Type: int | -| `--schedule-policy` | The scheduling policy of the requests. | `fcfs` | `lpm`, `random`, `fcfs`, `dfs-weight`, `lof`, `priority`, `routing-key` | -| `--enable-priority-scheduling` | Enable priority scheduling. Requests with higher priority integer values will be scheduled first by default. | `False` | bool flag (set to enable) | -| `--abort-on-priority-when-disabled` | If set, abort requests that specify a priority when priority scheduling is disabled. | `False` | bool flag (set to enable) | -| `--schedule-low-priority-values-first` | If specified with --enable-priority-scheduling, the scheduler will schedule requests with lower priority integer values first. | `False` | bool flag (set to enable) | -| `--priority-scheduling-preemption-threshold` | Minimum difference in priorities for an incoming request to have to preempt running request(s). | `10` | Type: int | -| `--schedule-conservativeness` | How conservative the schedule policy is. A larger value means more conservative scheduling. Use a larger value if you see requests being retracted frequently. | `1.0` | Type: float | -| `--page-size` | The number of tokens in a page. | `1` | Type: int | -| `--swa-full-tokens-ratio` | The ratio of SWA layer KV tokens / full layer KV tokens, regardless of the number of swa:full layers. It should be between 0 and 1. E.g. 0.5 means if each swa layer has 50 tokens, then each full layer has 100 tokens. | `0.8` | Type: float | -| `--disable-hybrid-swa-memory` | Disable the hybrid SWA memory. | `False` | bool flag (set to enable) | -| `--radix-eviction-policy` | The eviction policy of radix trees. 'lru' stands for Least Recently Used, 'lfu' stands for Least Frequently Used. | `lru` | `lru`, `lfu` | -| `--enable-prefill-delayer` | Enable prefill delayer for DP attention to reduce idle time. | `False` | bool flag (set to enable) | -| `--prefill-delayer-max-delay-passes` | Maximum forward passes to delay prefill. | `30` | Type: int | -| `--prefill-delayer-token-usage-low-watermark` | Token usage low watermark for prefill delayer. | `None` | Type: float | -| `--prefill-delayer-queue-min-ratio` | Opt-in to the adaptive queue-based delay trigger (independent of the slot-based one). Defers prefill until the waiting queue reaches `min(running_req * ratio, max_prefill_bs)` so small fragments batch into a larger prefill. Unset keeps the original slot-only behavior. Typical: `0.1`–`0.5`. | `None` | Type: float | -| `--prefill-delayer-max-delay-ms` | Wall-clock cap (ms) on a single queue-trigger delay; once exceeded, prefill is force-released to bound worst-case TTFT. Only consulted when `--prefill-delayer-queue-min-ratio` is set. Typical: `1000`–`5000`. | `5000` | Type: float | -| `--prefill-delayer-forward-passes-buckets` | Custom buckets for prefill delayer forward passes histogram. 0 and max_delay_passes-1 will be auto-added. | `None` | List[float] | -| `--prefill-delayer-wait-seconds-buckets` | Custom buckets for prefill delayer wait seconds histogram. 0 will be auto-added. | `None` | List[float] | - -## Runtime options -| Argument | Description | Defaults | Options | -| --- | --- | --- | --- | -| `--device` | The device to use ('cuda', 'xpu', 'hpu', 'npu', 'cpu'). Defaults to auto-detection if not specified. | `None` | Type: str | -| `--tensor-parallel-size`
`--tp-size` | The tensor parallelism size. | `1` | Type: int | -| `--pipeline-parallel-size`
`--pp-size` | The pipeline parallelism size. | `1` | Type: int | -| `--attention-context-parallel-size`
`--attn-cp-size`| The attention context parallelism size. | `1` | Type: int| -| `--moe-data-parallel-size`
`--moe-dp-size`| The moe data parallelism size. | `1` | Type: int| -| `--pp-max-micro-batch-size` | The maximum micro batch size in pipeline parallelism. | `None` | Type: int | -| `--pp-async-batch-depth` | The async batch depth of pipeline parallelism. | `0` | Type: int | -| `--stream-interval` | The interval (or buffer size) for streaming in terms of the token length. A smaller value makes streaming smoother, while a larger value makes the throughput higher | `1` | Type: int | -| `--incremental-streaming-output` | Whether to output as a sequence of disjoint segments. | `False` | bool flag (set to enable) | -| `--random-seed` | The random seed. | `None` | Type: int | -| `--constrained-json-whitespace-pattern` | (outlines and llguidance backends only) Regex pattern for syntactic whitespaces allowed in JSON constrained output. For example, to allow the model to generate consecutive whitespaces, set the pattern to [\n\t ]* | `None` | Type: str | -| `--constrained-json-disable-any-whitespace` | (xgrammar and llguidance backends only) Enforce compact representation in JSON constrained output. | `False` | bool flag (set to enable) | -| `--watchdog-timeout` | Set watchdog timeout in seconds. If a forward batch takes longer than this, the server will crash to prevent hanging. | `300` | Type: float | -| `--soft-watchdog-timeout` | Set soft watchdog timeout in seconds. If a forward batch takes longer than this, the server will dump information for debugging. | `None` | Type: float | -| `--dist-timeout` | Set timeout for torch.distributed initialization. | `None` | Type: int | -| `--download-dir` | Model download directory for huggingface. | `None` | Type: str | -| `--model-checksum` | Model file integrity verification. If provided without value, uses model-path as HF repo ID. Otherwise, provide checksums JSON file path or HuggingFace repo ID. | `None` | Type: str | -| `--base-gpu-id` | The base GPU ID to start allocating GPUs from. Useful when running multiple instances on the same machine. | `0` | Type: int | -| `--gpu-id-step` | The delta between consecutive GPU IDs that are used. For example, setting it to 2 will use GPU 0,2,4,... | `1` | Type: int | -| `--sleep-on-idle` | Reduce CPU usage when sglang is idle. | `False` | bool flag (set to enable) | -| `--custom-sigquit-handler` | Register a custom sigquit handler so you can do additional cleanup after the server is shutdown. This is only available for Engine, not for CLI. | `None` | Type: str | - -## Logging -| Argument | Description | Defaults | Options | -| --- | --- | --- | --- | -| `--log-level` | The logging level of all loggers. | `info` | Type: str | -| `--log-level-http` | The logging level of HTTP server. If not set, reuse --log-level by default. | `None` | Type: str | -| `--log-requests` | Log metadata, inputs, outputs of all requests. The verbosity is decided by --log-requests-level | `False` | bool flag (set to enable) | -| `--log-requests-level` | 0: Log metadata (no sampling parameters). 1: Log metadata and sampling parameters. 2: Log metadata, sampling parameters and partial input/output. 3: Log every input/output. | `2` | `0`, `1`, `2`, `3` | -| `--log-requests-format` | Format for request logging: 'text' (human-readable) or 'json' (structured) | `text` | `text`, `json` | -| `--log-requests-target` | Target(s) for request logging: 'stdout' and/or directory path(s) for file output. Can specify multiple targets, e.g., '--log-requests-target stdout /my/path'. | `None` | List[str] | -| `--uvicorn-access-log-exclude-prefixes` | Exclude uvicorn access logs whose request path starts with any of these prefixes. Defaults to empty (disabled). | `[]` | List[str] | -| `--crash-dump-folder` | Folder path to dump requests from the last 5 min before a crash (if any). If not specified, crash dumping is disabled. | `None` | Type: str | -| `--show-time-cost` | Show time cost of custom marks. | `False` | bool flag (set to enable) | -| `--enable-metrics` | Enable log prometheus metrics. | `False` | bool flag (set to enable) | -| `--enable-mfu-metrics` | Enable estimated MFU-related prometheus metrics. | `False` | bool flag (set to enable) | -| `--enable-metrics-for-all-schedulers` | Enable --enable-metrics-for-all-schedulers when you want schedulers on all TP ranks (not just TP 0) to record request metrics separately. This is especially useful when dp_attention is enabled, as otherwise all metrics appear to come from TP 0. | `False` | bool flag (set to enable) | -| `--tokenizer-metrics-custom-labels-header` | Specify the HTTP header for passing custom labels for tokenizer metrics. | `x-custom-labels` | Type: str | -| `--tokenizer-metrics-allowed-custom-labels` | The custom labels allowed for tokenizer metrics. The labels are specified via a dict in '--tokenizer-metrics-custom-labels-header' field in HTTP requests, e.g., {'label1': 'value1', 'label2': 'value2'} is allowed if '--tokenizer-metrics-allowed-custom-labels label1 label2' is set. | `None` | List[str] | -| `--bucket-time-to-first-token` | The buckets of time to first token, specified as a list of floats. | `None` | List[float] | -| `--bucket-inter-token-latency` | The buckets of inter-token latency, specified as a list of floats. | `None` | List[float] | -| `--bucket-e2e-request-latency` | The buckets of end-to-end request latency, specified as a list of floats. | `None` | List[float] | -| `--collect-tokens-histogram` | Collect prompt/generation tokens histogram. | `False` | bool flag (set to enable) | -| `--prompt-tokens-buckets` | The buckets rule of prompt tokens. Supports 3 rule types: 'default' uses predefined buckets; 'tse ' generates two sides exponential distributed buckets (e.g., 'tse 1000 2 8' generates buckets [984.0, 992.0, 996.0, 998.0, 1000.0, 1002.0, 1004.0, 1008.0, 1016.0]).); 'custom ...' uses custom bucket values (e.g., 'custom 10 50 100 500'). | `None` | List[str] | -| `--generation-tokens-buckets` | The buckets rule for generation tokens histogram. Supports 3 rule types: 'default' uses predefined buckets; 'tse ' generates two sides exponential distributed buckets (e.g., 'tse 1000 2 8' generates buckets [984.0, 992.0, 996.0, 998.0, 1000.0, 1002.0, 1004.0, 1008.0, 1016.0]).); 'custom ...' uses custom bucket values (e.g., 'custom 10 50 100 500'). | `None` | List[str] | -| `--gc-warning-threshold-secs` | The threshold for long GC warning. If a GC takes longer than this, a warning will be logged. Set to 0 to disable. | `0.0` | Type: float | -| `--decode-log-interval` | The log interval of decode batch. | `40` | Type: int | -| `--enable-request-time-stats-logging` | Enable per request time stats logging | `False` | bool flag (set to enable) | -| `--kv-events-config` | Config in json format for NVIDIA dynamo KV event publishing. Publishing will be enabled if this flag is used. | `None` | Type: str | -| `--enable-trace` | Enable opentelemetry trace | `False` | bool flag (set to enable) | -| `--otlp-traces-endpoint` | Config opentelemetry collector endpoint if --enable-trace is set. format: : | `localhost:4317` | Type: str | - -## RequestMetricsExporter configuration -| Argument | Description | Defaults | Options | -| --- | --- | --- | --- | -| `--export-metrics-to-file` | Export performance metrics for each request to local file (e.g. for forwarding to external systems). | `False` | bool flag (set to enable) | -| `--export-metrics-to-file-dir` | Directory path for writing performance metrics files (required when --export-metrics-to-file is enabled). | `None` | Type: str | - -## API related -| Argument | Description | Defaults | Options | -| --- | --- | --- | --- | -| `--api-key` | Set API key of the server. It is also used in the OpenAI API compatible server. | `None` | Type: str | -| `--admin-api-key` | Set **admin API key** for administrative/control endpoints (e.g., weights update, cache flush, `/server_info`). Endpoints marked as admin-only require `Authorization: Bearer ` when this is set. | `None` | Type: str | -| `--served-model-name` | Override the model name returned by the v1/models endpoint in OpenAI API server. | `None` | Type: str | -| `--weight-version` | Version identifier for the model weights. Defaults to 'default' if not specified. | `default` | Type: str | -| `--chat-template` | The builtin chat template name or the path of the chat template file. This is only used for OpenAI-compatible API server. | `None` | Type: str | -| `--hf-chat-template-name` | When the HuggingFace tokenizer has multiple chat templates (e.g., 'default', 'tool_use', 'rag'), specify which named template to use. If not set, the first available template is used. | `None` | Type: str | -| `--completion-template` | The builtin completion template name or the path of the completion template file. This is only used for OpenAI-compatible API server. only for code completion currently. | `None` | Type: str | -| `--file-storage-path` | The path of the file storage in backend. | `sglang_storage` | Type: str | -| `--enable-cache-report` | Return number of cached tokens in usage.prompt_tokens_details for each openai request. | `False` | bool flag (set to enable) | -| `--reasoning-parser` | Specify the parser for reasoning models. Supported parsers: [deepseek-r1, deepseek-v3, glm45, gpt-oss, kimi, qwen3, qwen3-thinking, step3]. | `None` | `deepseek-r1`, `deepseek-v3`, `glm45`, `gpt-oss`, `kimi`, `qwen3`, `qwen3-thinking`, `step3` | -| `--tool-call-parser` | Specify the parser for handling tool-call interactions. Supported parsers: [deepseekv3, deepseekv31, glm, glm45, glm47, gpt-oss, kimi_k2, llama3, mistral, pythonic, qwen, qwen25, qwen3_coder, step3]. | `None` | `deepseekv3`, `deepseekv31`, `glm`, `glm45`, `glm47`, `gpt-oss`, `kimi_k2`, `llama3`, `mistral`, `pythonic`, `qwen`, `qwen25`, `qwen3_coder`, `step3`, `gigachat3` | -| `--tool-server` | Either 'demo' or a comma-separated list of tool server urls to use for the model. If not specified, no tool server will be used. | `None` | Type: str | -| `--sampling-defaults` | Where to get default sampling parameters. 'openai' uses SGLang/OpenAI defaults (temperature=1.0, top_p=1.0, etc.). 'model' uses the model's generation_config.json to get the recommended sampling parameters if available. Default is 'model'. | `model` | `openai`, `model` | - -## Data parallelism -| Argument | Description | Defaults | Options | -| --- | --- | --- | --- | -| `--data-parallel-size`
`--dp-size` | The data parallelism size. | `1` | Type: int | -| `--load-balance-method` | The load balancing strategy for data parallelism. The `total_tokens` algorithm can only be used when DP attention is applied. This algorithm performs load balancing based on the real-time token load of the DP workers. | `auto` | `auto`, `round_robin`, `follow_bootstrap_room`, `total_requests`, `total_tokens` | - -## Multi-node distributed serving -| Argument | Description | Defaults | Options | -| --- | --- | --- | --- | -| `--dist-init-addr`
`--nccl-init-addr` | The host address for initializing distributed backend (e.g., `192.168.0.2:25000`). | `None` | Type: str | -| `--nnodes` | The number of nodes. | `1` | Type: int | -| `--node-rank` | The node rank. | `0` | Type: int | - -## Model override args -| Argument | Description | Defaults | Options | -| --- | --- | --- | --- | -| `--json-model-override-args` | A dictionary in JSON string format used to override default model configurations. | `{}` | Type: str | -| `--preferred-sampling-params` | json-formatted sampling settings that will be returned in /get_model_info | `None` | Type: str | - -## LoRA -| Argument | Description | Defaults | Options | -| --- | --- | --- | --- | -| `--enable-lora` | Enable LoRA support for the model. This argument is automatically set to `True` if `--lora-paths` is provided for backward compatibility. | `False` | Bool flag (set to enable) | -| `--enable-lora-overlap-loading` | Enable asynchronous LoRA weight loading in order to overlap H2D transfers with GPU compute. This should be enabled if you find that your LoRA workloads are bottlenecked by adapter weight loading, for example when frequently loading large LoRA adapters. | `False` | Bool flag (set to enable) -| `--max-lora-rank` | The maximum LoRA rank that should be supported. If not specified, it will be automatically inferred from the adapters provided in `--lora-paths`. This argument is needed when you expect to dynamically load adapters of larger LoRA rank after server startup. | `None` | Type: int | -| `--lora-target-modules` | The union set of all target modules where LoRA should be applied (e.g., `q_proj`, `k_proj`, `gate_proj`). If not specified, it will be automatically inferred from the adapters provided in `--lora-paths`. You can also set it to `all` to enable LoRA for all supported modules; note this may introduce minor performance overhead. | `None` | `q_proj`, `k_proj`, `v_proj`, `o_proj`, `gate_proj`, `up_proj`, `down_proj`, `qkv_proj`, `gate_up_proj`, `all` | -| `--lora-paths` | The list of LoRA adapters to load. Each adapter must be specified in one of the following formats: `` \| `=` \| JSON with schema `{"lora_name": str, "lora_path": str, "pinned": bool}`. | `None` | Type: List[str] / JSON objects | -| `--max-loras-per-batch` | Maximum number of adapters for a running batch, including base-only requests. | `8` | Type: int | -| `--max-loaded-loras` | If specified, limits the maximum number of LoRA adapters loaded in CPU memory at a time. Must be ≥ `--max-loras-per-batch`. | `None` | Type: int | -| `--lora-eviction-policy` | LoRA adapter eviction policy when the GPU memory pool is full. | `lru` | `lru`, `fifo` | -| `--lora-backend` | Choose the kernel backend for multi-LoRA serving. | `csgmv` | `triton`, `csgmv`, `ascend`, `torch_native` | -| `--max-lora-chunk-size` | Maximum chunk size for the ChunkedSGMV LoRA backend. Only used when `--lora-backend` is `csgmv`. Larger values may improve performance. | `16` | `16`, `32`, `64`, `128` | -| `--lora-drain-wait-threshold` | When any LoRA adapter request waits longer than this threshold (in seconds), the scheduler will selectively drain one running adapter to make room. This mitigates extreme tail latency under high or skewed workloads by preventing a small set of adapters from monopolizing batch slots. Set to 0 to disable draining (default). | `0.0` | Type: float | - -## Kernel Backends (Attention, Sampling, Grammar, GEMM) -| Argument | Description | Defaults | Options | -| --- | --- | --- | --- | -| `--attention-backend` | Choose the kernels for attention layers. | `None` | `triton`, `torch_native`, `flex_attention`, `dsa` (canonical; `nsa` is a deprecated alias), `cutlass_mla`, `fa3`, `fa4`, `flashinfer`, `flashmla`, `trtllm_mla`, `trtllm_mha`, `dual_chunk_flash_attn`, `aiter`, `wave`, `intel_amx`, `ascend` | -| `--prefill-attention-backend` | Choose the kernels for prefill attention layers (have priority over --attention-backend). | `None` | `triton`, `torch_native`, `flex_attention`, `dsa` (canonical; `nsa` is a deprecated alias), `cutlass_mla`, `fa3`, `fa4`, `flashinfer`, `flashmla`, `trtllm_mla`, `trtllm_mha`, `dual_chunk_flash_attn`, `aiter`, `wave`, `intel_amx`, `ascend` | -| `--decode-attention-backend` | Choose the kernels for decode attention layers (have priority over --attention-backend). | `None` | `triton`, `torch_native`, `flex_attention`, `dsa` (canonical; `nsa` is a deprecated alias), `cutlass_mla`, `fa3`, `fa4`, `flashinfer`, `flashmla`, `trtllm_mla`, `trtllm_mha`, `dual_chunk_flash_attn`, `aiter`, `wave`, `intel_amx`, `ascend` | -| `--sampling-backend` | Choose the kernels for sampling layers. | `None` | `flashinfer`, `pytorch`, `ascend` | -| `--grammar-backend` | Choose the backend for grammar-guided decoding. | `None` | `xgrammar`, `outlines`, `llguidance`, `none` | -| `--mm-attention-backend` | Set multimodal attention backend. | `None` | `sdpa`, `fa3`, `fa4`, `triton_attn`, `ascend_attn`, `aiter_attn` | -| `--dsa-prefill-backend` | Choose the DSA backend for the prefill stage (overrides `--attention-backend` when running DeepSeek DSA-style attention). `--nsa-prefill-backend` is a deprecated alias. | `flashmla_sparse` | `flashmla_sparse`, `flashmla_kv`, `flashmla_auto`, `fa3`, `tilelang`, `aiter`, `trtllm` | -| `--dsa-decode-backend` | Choose the DSA backend for the decode stage when running DeepSeek DSA-style attention. Overrides `--attention-backend` for decoding. `--nsa-decode-backend` is a deprecated alias. | `fa3` | `flashmla_sparse`, `flashmla_kv`, `fa3`, `tilelang`, `aiter`, `trtllm` | -| `--dsa-topk-backend` | Choose the DSA indexer top-k backend. The `torch` backend currently requires `SGLANG_DSA_FUSE_TOPK=false`. | `sgl-kernel` | `sgl-kernel`, `torch`, `flashinfer` | -| `--fp8-gemm-backend` | Choose the runner backend for Blockwise FP8 GEMM operations. Options: 'auto' (default, auto-selects based on hardware), 'deep_gemm' (JIT-compiled; enabled by default on NVIDIA Hopper (SM90) and Blackwell (SM100) when DeepGEMM is installed), 'flashinfer_trtllm' (FlashInfer TRTLLM backend; SM100/SM103 only), 'flashinfer_cutlass' (FlashInfer CUTLASS backend, SM120 only), 'flashinfer_deepgemm' (Hopper SM90 only, uses swapAB optimization for small M dimensions in decoding), 'cutlass' (optimal for Hopper/Blackwell GPUs and high-throughput), 'triton' (fallback, widely compatible), 'aiter' (ROCm only).| `auto` | `auto`, `deep_gemm`, `flashinfer_trtllm`, `flashinfer_cutlass`, `flashinfer_deepgemm`, `cutlass`, `triton`, `aiter` | -| `--fp4-gemm-backend` | Choose the runner backend for NVFP4 GEMM operations. Options: 'flashinfer_cutlass' (default), 'auto' (auto-selects between flashinfer_cudnn/flashinfer_cutlass based on CUDA/cuDNN version), 'flashinfer_cudnn' (FlashInfer cuDNN backend, optimal on CUDA 13+ with cuDNN 9.15+), 'flashinfer_trtllm' (FlashInfer TensorRT-LLM backend, requires different weight preparation with shuffling). All backends are from FlashInfer; when FlashInfer is unavailable, sgl-kernel CUTLASS is used as an automatic fallback.| `flashinfer_cutlass` | `auto`, `flashinfer_cudnn`, `flashinfer_cutlass`, `flashinfer_trtllm` | -| `--disable-flashinfer-autotune` | Flashinfer autotune is enabled by default. Set this flag to disable the autotune. | `False` | bool flag (set to enable) | - -## Speculative decoding -| Argument | Description | Defaults | Options | -| --- | --- | --- | --- | -| `--speculative-algorithm` | Speculative algorithm. | `None` | `EAGLE`, `EAGLE3`, `NEXTN`, `STANDALONE`, `NGRAM` | -| `--speculative-draft-model-path`
`--speculative-draft-model` | The path of the draft model weights. This can be a local folder or a Hugging Face repo ID. | `None` | Type: str | -| `--speculative-draft-model-revision` | The specific draft model version to use. It can be a branch name, a tag name, or a commit id. If unspecified, will use the default version. | `None` | Type: str | -| `--speculative-draft-load-format` | The format of the draft model weights to load. If not specified, will use the same format as --load-format. Use 'dummy' to initialize draft model weights with random values for profiling. | `None` | Same as --load-format options | -| `--speculative-num-steps` | The number of steps sampled from draft model in Speculative Decoding. | `None` | Type: int | -| `--speculative-eagle-topk` | The number of tokens sampled from the draft model in eagle2 each step. | `None` | Type: int | -| `--speculative-num-draft-tokens` | The number of tokens sampled from the draft model in Speculative Decoding. | `None` | Type: int | -| `--speculative-accept-threshold-single` | Accept a draft token if its probability in the target model is greater than this threshold. | `1.0` | Type: float | -| `--speculative-accept-threshold-acc` | The accept probability of a draft token is raised from its target probability p to min(1, p / threshold_acc). | `1.0` | Type: float | -| `--speculative-token-map` | The path of the draft model's small vocab table. | `None` | Type: str | -| `--speculative-attention-mode` | Attention backend for speculative decoding operations (both target verify and draft extend). Can be one of 'prefill' (default) or 'decode'. | `prefill` | `prefill`, `decode` | -| `--speculative-draft-attention-backend` | Attention backend for speculative decoding drafting. | `None` | Same as attention backend options | -| `--speculative-moe-runner-backend` | MOE backend for EAGLE speculative decoding, see --moe-runner-backend for options. Same as moe runner backend if unset. | `None` | Same as --moe-runner-backend options | -| `--speculative-moe-a2a-backend` | MOE A2A backend for EAGLE speculative decoding, see --moe-a2a-backend for options. Same as moe a2a backend if unset. | `None` | Same as --moe-a2a-backend options | -| `--speculative-draft-model-quantization` | The quantization method for speculative model. | `None` | Same as --quantization options | - -## Ngram speculative decoding -| Argument | Description | Defaults | Options | -| --- | --- | --- | --- | -| `--speculative-ngram-min-bfs-breadth` | The minimum breadth for BFS (Breadth-First Search) in ngram speculative decoding. | `1` | Type: int | -| `--speculative-ngram-max-bfs-breadth` | The maximum breadth for BFS (Breadth-First Search) in ngram speculative decoding. | `10` | Type: int | -| `--speculative-ngram-match-type` | Ngram tree-building mode. `BFS` selects recency-based expansion and `PROB` selects frequency-based expansion. This setting is forwarded to the ngram cache implementation. | `BFS` | `BFS`, `PROB` | -| `--speculative-ngram-max-trie-depth` | Maximum suffix length stored and matched by the ngram trie. | `18` | Type: int | -| `--speculative-ngram-capacity` | The cache capacity for ngram speculative decoding. | `10000000` | Type: int | - -## Multi-layer Eagle speculative decoding -| Argument | Description | Defaults | Options | -| --- | --- | --- | --- | -| `--enable-multi-layer-eagle` | Enable multi-layer Eagle speculative decoding. | `False` | bool flag (set to enable) | - -## MoE -| Argument | Description | Defaults | Options | -| --- | --- | --- | --- | -| `--expert-parallel-size`
`--ep-size`
`--ep` | The expert parallelism size. | `1` | Type: int | -| `--moe-a2a-backend` | Select the backend for all-to-all communication for expert parallelism. | `none` | `none`, `deepep`, `mooncake`, `mori`, `nixl`, `ascend_fuseep`| -| `--moe-runner-backend` | Choose the runner backend for MoE. | `auto` | `auto`, `deep_gemm`, `triton`, `triton_kernel`, `flashinfer_trtllm`, `flashinfer_trtllm_routed`, `flashinfer_cutlass`, `flashinfer_mxfp4`, `flashinfer_cutedsl`, `cutlass` | -| `--flashinfer-mxfp4-moe-precision` | Choose the computation precision of flashinfer mxfp4 moe | `default` | `default`, `bf16` | -| `--enable-flashinfer-allreduce-fusion` | Enable FlashInfer allreduce fusion with Residual RMSNorm. | `False` | bool flag (set to enable) | -| `--enable-aiter-allreduce-fusion` | Enable aiter allreduce fusion with Residual RMSNorm. | `False` | bool flag (set to enable) | -| `--deepep-mode` | Select the mode when enable DeepEP MoE, could be `normal`, `low_latency` or `auto`. Default is `auto`, which means `low_latency` for decode batch and `normal` for prefill batch. | `auto` | `normal`, `low_latency`, `auto` | -| `--deepep-dispatcher-output-dtype` | Select DeepEP dispather output dtype, could be `bf16`, `fp8`, `int8` (only Ascend A2/A3 NPU), `nvfp4` or `auto`. Default is `auto`, which follows a priority order (server argument → deprecated env var → input_global_scale check → dispatcher_output_dtype from quant_config → flashinfer/cutlass backend → NPU BF16 default → GPU FP8 default) | `auto` | `bf16`, `fp8`, `int8`, `nvfp4`, `auto` | -| `--ep-num-redundant-experts` | Allocate this number of redundant experts in expert parallel. | `0` | Type: int | -| `--ep-dispatch-algorithm` | The algorithm to choose ranks for redundant experts in expert parallel. | `None` | Type: str | -| `--init-expert-location` | Initial location of EP experts. | `trivial` | Type: str | -| `--enable-eplb` | Enable EPLB algorithm | `False` | bool flag (set to enable) | -| `--eplb-algorithm` | Chosen EPLB algorithm | `auto` | Type: str | -| `--eplb-rebalance-num-iterations` | Number of iterations to automatically trigger a EPLB re-balance. | `1000` | Type: int | -| `--eplb-rebalance-layers-per-chunk` | Number of layers to rebalance per forward pass. | `None` | Type: int | -| `--eplb-min-rebalancing-utilization-threshold` | Minimum threshold for GPU average utilization to trigger EPLB rebalancing. Must be in the range [0.0, 1.0]. | `1.0` | Type: float | -| `--expert-distribution-recorder-mode` | Mode of expert distribution recorder. | `None` | Type: str | -| `--expert-distribution-recorder-buffer-size` | Circular buffer size of expert distribution recorder. Set to -1 to denote infinite buffer. | `None` | Type: int | -| `--enable-expert-distribution-metrics` | Enable logging metrics for expert balancedness | `False` | bool flag (set to enable) | -| `--deepep-config` | Tuned DeepEP config suitable for your own cluster. It can be either a string with JSON content or a file path. | `None` | Type: str | -| `--moe-dense-tp-size` | TP size for MoE dense MLP layers. This flag is useful when, with large TP size, there are errors caused by weights in MLP layers having dimension smaller than the min dimension GEMM supports. | `None` | Type: int | -| `--elastic-ep-backend` | Specify the collective communication backend for elastic EP. Currently supports 'mooncake'. | `none` | `none`, `mooncake` | -| `--enable-elastic-expert-backup` | Enable elastic EP backend to backup expert weights in DRAM feature. Currently supports 'mooncake'.| `False` | bool flag (set to enable) | -| `--mooncake-ib-device` | The InfiniBand devices for Mooncake Backend transfer, accepts multiple comma-separated devices (e.g., --mooncake-ib-device mlx5_0,mlx5_1). Default is None, which triggers automatic device detection when Mooncake Backend is enabled. | `None` | Type: str | -| `--enable-deepep-waterfill` | Enable DeepEP Waterfill: dispatch the shared expert as the 9th routed expert to the least-loaded EP rank. Automatically sets `--moe-a2a-backend deepep`, implicitly enables shared-expert fusion, and supports `--deepep-mode auto`, `normal`, or `low_latency`. Use `auto` or `low_latency` for production decode so CUDA graph remains enabled. Supported on DeepSeek-V3/R1 with EP >= 2. By default, Waterfill uses the static local-batch path; set `SGLANG_DISABLE_STATIC_WATERFILL=1` to force dynamic Waterfill with runtime EP all-reduce. | `False` | bool flag (set to enable) | -| `--elastic-ep-rejoin` | Indicates that this process is a relaunched elastic EP rank that should rejoin an existing process group during rank recovery. | `False` | bool flag (set to enable) | - -## Mamba Cache -| Argument | Description | Defaults | Options | -| --- | --- | --- | --- | -| `--max-mamba-cache-size` | The maximum size of the mamba cache. | `None` | Type: int | -| `--mamba-ssm-dtype` | The data type of the SSM states in mamba cache. | `float32` | `float32`, `bfloat16`, `float16` | -| `--mamba-full-memory-ratio` | The ratio of mamba state memory to full kv cache memory. | `0.9` | Type: float | -| `--mamba-scheduler-strategy` | The strategy to use for mamba scheduler. `auto` currently defaults to `no_buffer`. 1. `no_buffer` does not support overlap scheduler due to not allocating extra mamba state buffers. Branching point caching support is feasible but not implemented. 2. `extra_buffer` supports overlap schedule by allocating extra mamba state buffers to track mamba state for caching (mamba state usage per running req becomes `2x` for non-spec; `1+(1/(2+speculative_num_draft_tokens))x` for spec dec (e.g. 1.16x if speculative_num_draft_tokens==4)). 2a. `extra_buffer` is strictly better for non-KV-cache-bound cases; for KV-cache-bound cases, the tradeoff depends on whether enabling overlap outweighs reduced max running requests. 2b. mamba caching at radix cache branching point is strictly better than non-branch but requires kernel support, currently only extra_buffer supports branching. | `auto` | `auto`, `no_buffer`, `extra_buffer` | -| `--mamba-track-interval` | The interval (in tokens) to track the mamba state during decode. Only used when `--mamba-scheduler-strategy` is `extra_buffer`. Must be divisible by page_size if set, and must be >= speculative_num_draft_tokens when using speculative decoding. | `256` | Type: int | - -## Hierarchical cache -| Argument | Description | Defaults | Options | -| --- | --- | --- | --- | -| `--enable-hierarchical-cache` | Enable hierarchical cache | `False` | bool flag (set to enable) | -| `--hicache-ratio` | The ratio of the size of host KV cache memory pool to the size of device pool. | `2.0` | Type: float | -| `--hicache-size` | The size of host KV cache memory pool in gigabytes, which will override the hicache_ratio if set. | `0` | Type: int | -| `--hicache-write-policy` | The write policy of hierarchical cache. | `write_through` | `write_back`, `write_through`, `write_through_selective` | -| `--hicache-io-backend` | The IO backend for KV cache transfer between CPU and GPU | `kernel` | `direct`, `kernel`, `kernel_ascend` | -| `--hicache-mem-layout` | The layout of host memory pool for hierarchical cache. | `layer_first` | `layer_first`, `page_first`, `page_first_direct`, `page_first_kv_split`, `page_head` | -| `--hicache-storage-backend` | The storage backend for hierarchical KV cache. Built-in backends: file, mooncake, hf3fs, nixl, aibrix. For dynamic backend, use --hicache-storage-backend-extra-config to specify: backend_name (custom name), module_path (Python module path), class_name (backend class name). | `None` | `file`, `mooncake`, `hf3fs`, `nixl`, `aibrix`, `dynamic`, `eic` | -| `--hicache-storage-prefetch-policy` | Control when prefetching from the storage backend should stop. | `best_effort` | `best_effort`, `wait_complete`, `timeout` | -| `--hicache-storage-backend-extra-config` | A dictionary in JSON string format, or a string starting with a `@` followed by a config file in JSON/YAML/TOML format, containing extra configuration for the storage backend. | `None` | Type: str | - -## Hierarchical sparse attention -| Argument | Description | Defaults | Options | -| --- | --- | --- | --- | -| `--hierarchical-sparse-attention-extra-config` | A dictionary in JSON string format for hierarchical sparse attention configuration. Required fields: `algorithm` (str), `backend` (str). All other fields are algorithm-specific and passed to the algorithm constructor. | `None` | Type: str | - -## LMCache -| Argument | Description | Defaults | Options | -| --- | --- | --- | --- | -| `--enable-lmcache` | Using LMCache as an alternative hierarchical cache solution | `False` | bool flag (set to enable) | - -## Ktransformers -| Argument | Description | Defaults | Options | -| --- | --- | --- | --- | -| `--kt-weight-path` | [ktransformers parameter] The path of the quantized expert weights for amx kernel. A local folder. | `None` | Type: str | -| `--kt-method` | [ktransformers parameter] Quantization formats for CPU execution. | `AMXINT4` | Type: str | -| `--kt-cpuinfer` | [ktransformers parameter] The number of CPUInfer threads. | `None` | Type: int | -| `--kt-threadpool-count` | [ktransformers parameter] One-to-one with the number of NUMA nodes (one thread pool per NUMA). | `2` | Type: int | -| `--kt-num-gpu-experts` | [ktransformers parameter] The number of GPU experts. | `None` | Type: int | -| `--kt-max-deferred-experts-per-token` | [ktransformers parameter] Maximum number of experts deferred to CPU per token. All MoE layers except the final one use this value; the final layer always uses 0. | `None` | Type: int | - -## Diffusion LLM - -| Argument | Description | Defaults | Options | -| --- | --- | --- | --- | -| `--dllm-algorithm` | The diffusion LLM algorithm, such as LowConfidence. | `None` | Type: str | -| `--dllm-algorithm-config` | The diffusion LLM algorithm configurations. Must be a YAML file. | `None` | Type: str | - -## Offloading -| Argument | Description | Defaults | Options | -| --- | --- | --- | --- | -| `--cpu-offload-gb` | How many GBs of RAM to reserve for CPU offloading. | `0` | Type: int | -| `--offload-group-size` | Number of layers per group in offloading. | `-1` | Type: int | -| `--offload-num-in-group` | Number of layers to be offloaded within a group. | `1` | Type: int | -| `--offload-prefetch-step` | Steps to prefetch in offloading. | `1` | Type: int | -| `--offload-mode` | Mode of offloading. | `cpu` | Type: str | - -## Args for multi-item scoring -| Argument | Description | Defaults | Options | -| --- | --- | --- | --- | -| `--multi-item-scoring-delimiter` | Delimiter token ID for multi-item scoring. Used to combine Query and Items into a single sequence: QueryItem1Item2... This enables efficient batch processing of multiple items against a single query. | `None` | Type: int | - -## Optimization/debug options -| Argument | Description | Defaults | Options | -| --- | --- | --- | --- | -| `--disable-radix-cache` | Disable RadixAttention for prefix caching. | `False` | bool flag (set to enable) | -| `--cuda-graph-max-bs` | Set the maximum batch size for cuda graph. It will extend the cuda graph capture batch size to this value. | `None` | Type: int | -| `--cuda-graph-bs` | Set the list of batch sizes for cuda graph. | `None` | List[int] | -| `--disable-cuda-graph` | Disable cuda graph. | `False` | bool flag (set to enable) | -| `--disable-cuda-graph-padding` | Disable cuda graph when padding is needed. Still uses cuda graph when padding is not needed. | `False` | bool flag (set to enable) | -| `--enable-profile-cuda-graph` | Enable profiling of cuda graph capture. | `False` | bool flag (set to enable) | -| `--enable-cudagraph-gc` | Enable garbage collection during CUDA graph capture. If disabled (default), GC is frozen during capture to speed up the process. | `False` | bool flag (set to enable) | -| `--enable-layerwise-nvtx-marker` | Enable layerwise NVTX profiling annotations for the model. This adds NVTX markers to every layer for detailed per-layer performance analysis with Nsight Systems. | `False` | bool flag (set to enable) | -| `--enable-nccl-nvls` | Enable NCCL NVLS for prefill heavy requests when available. | `False` | bool flag (set to enable) | -| `--enable-symm-mem` | Enable NCCL symmetric memory for fast collectives. | `False` | bool flag (set to enable) | -| `--disable-flashinfer-cutlass-moe-fp4-allgather` | Disables quantize before all-gather for flashinfer cutlass moe. | `False` | bool flag (set to enable) | -| `--enable-tokenizer-batch-encode` | Enable batch tokenization for improved performance when processing multiple text inputs. Do not use with image inputs, pre-tokenized input_ids, or input_embeds. | `False` | bool flag (set to enable) | -| `--disable-tokenizer-batch-decode` | Disable batch decoding when decoding multiple completions. | `False` | bool flag (set to enable) | -| `--disable-outlines-disk-cache` | Disable disk cache of outlines to avoid possible crashes related to file system or high concurrency. | `False` | bool flag (set to enable) | -| `--disable-custom-all-reduce` | Disable the custom all-reduce kernel and fall back to NCCL. | `False` | bool flag (set to enable) | -| `--enable-mscclpp` | Enable using mscclpp for small messages for all-reduce kernel and fall back to NCCL. | `False` | bool flag (set to enable) | -| `--enable-torch-symm-mem` | Enable using torch symm mem for all-reduce kernel and fall back to NCCL. Only supports CUDA device SM90 and above. SM90 supports world size 4, 6, 8. SM10 supports world size 6, 8. | `False` | bool flag (set to enable) | -| `--disable-overlap-schedule` | Disable the overlap scheduler, which overlaps the CPU scheduler with GPU model worker. | `False` | bool flag (set to enable) | -| `--enable-mixed-chunk` | Enabling mixing prefill and decode in a batch when using chunked prefill. | `False` | bool flag (set to enable) | -| `--enable-dp-attention` | Enabling data parallelism for attention and tensor parallelism for FFN. The dp size should be equal to the tp size. Currently DeepSeek-V2 and Qwen 2/3 MoE models are supported. | `False` | bool flag (set to enable) | -| `--enable-dp-lm-head` | Enable vocabulary parallel across the attention TP group to avoid all-gather across DP groups, optimizing performance under DP attention. | `False` | bool flag (set to enable) | -| `--enable-two-batch-overlap` | Enabling two micro batches to overlap. | `False` | bool flag (set to enable) | -| `--enable-single-batch-overlap` | Let computation and communication overlap within one micro batch. | `False` | bool flag (set to enable) | -| `--tbo-token-distribution-threshold` | The threshold of token distribution between two batches in micro-batch-overlap, determines whether to two-batch-overlap or two-chunk-overlap. Set to 0 denote disable two-chunk-overlap. | `0.48` | Type: float | -| `--enable-torch-compile` | Optimize the model with torch.compile. Experimental feature. | `False` | bool flag (set to enable) | -| `--enable-torch-compile-debug-mode` | Enable debug mode for torch compile. | `False` | bool flag (set to enable) | -| `--disable-piecewise-cuda-graph` | Disable piecewise cuda graph for extend/prefill. PCG is enabled by default. | `False` | bool flag (set to disable) | -| `--enforce-piecewise-cuda-graph` | Enforce piecewise cuda graph, skipping all auto-disable conditions. For testing only. | `False` | bool flag (set to enable) | -| `--piecewise-cuda-graph-tokens` | Set the list of tokens when using piecewise cuda graph. | `None` | Type: JSON list | -| `--piecewise-cuda-graph-compiler` | Set the compiler for piecewise cuda graph. Choices are: eager, inductor. | `eager` | `eager`, `inductor` | -| `--torch-compile-max-bs` | Set the maximum batch size when using torch compile. | `32` | Type: int | -| `--piecewise-cuda-graph-max-tokens` | Set the maximum tokens when using piecewise cuda graph. | `4096` | Type: int | -| `--torchao-config` | Optimize the model with torchao. Experimental feature. Current choices are: int8dq, int8wo, int4wo-, fp8wo, fp8dq-per_tensor, fp8dq-per_row | `` | Type: str | -| `--enable-nan-detection` | Enable the NaN detection for debugging purposes. | `False` | bool flag (set to enable) | -| `--enable-p2p-check` | Enable P2P check for GPU access, otherwise the p2p access is allowed by default. | `False` | bool flag (set to enable) | -| `--triton-attention-reduce-in-fp32` | Cast the intermediate attention results to fp32 to avoid possible crashes related to fp16. This only affects Triton attention kernels. | `False` | bool flag (set to enable) | -| `--triton-attention-num-kv-splits` | The number of KV splits in flash decoding Triton kernel. Larger value is better in longer context scenarios. The default value is 8. | `8` | Type: int | -| `--triton-attention-split-tile-size` | The size of split KV tile in flash decoding Triton kernel. Used for deterministic inference. | `None` | Type: int | -| `--num-continuous-decode-steps` | Run multiple continuous decoding steps to reduce scheduling overhead. This can potentially increase throughput but may also increase time-to-first-token latency. The default value is 1, meaning only run one decoding step at a time. | `1` | Type: int | -| `--delete-ckpt-after-loading` | Delete the model checkpoint after loading the model. | `False` | bool flag (set to enable) | -| `--enable-memory-saver` | Allow saving memory using release_memory_occupation and resume_memory_occupation | `False` | bool flag (set to enable) | -| `--enable-weights-cpu-backup` | Save model weights to CPU memory during release_weights_occupation and resume_weights_occupation | `False` | bool flag (set to enable) | -| `--enable-draft-weights-cpu-backup` | Save draft model weights to CPU memory during release_weights_occupation and resume_weights_occupation | `False` | bool flag (set to enable) | -| `--allow-auto-truncate` | Allow automatically truncating requests that exceed the maximum input length instead of returning an error. | `False` | bool flag (set to enable) | -| `--enable-custom-logit-processor` | Enable users to pass custom logit processors to the server (disabled by default for security) | `False` | bool flag (set to enable) | -| `--flashinfer-mla-disable-ragged` | Not using ragged prefill wrapper when running flashinfer mla | `False` | bool flag (set to enable) | -| `--disable-shared-experts-fusion` | Disable shared experts fusion optimization for deepseek v3/r1. | `False` | bool flag (set to enable) | -| `--disable-chunked-prefix-cache` | Disable chunked prefix cache feature for deepseek, which should save overhead for short sequences. | `False` | bool flag (set to enable) | -| `--disable-fast-image-processor` | Adopt base image processor instead of fast image processor. | `False` | bool flag (set to enable) | -| `--keep-mm-feature-on-device` | Keep multimodal feature tensors on device after processing to save D2H copy. | `False` | bool flag (set to enable) | -| `--enable-return-hidden-states` | Enable returning hidden states with responses. | `False` | bool flag (set to enable) | -| `--enable-return-routed-experts` | Enable returning routed experts of each layer with responses. | `False` | bool flag (set to enable) | -| `--scheduler-recv-interval` | The interval to poll requests in scheduler. Can be set to >1 to reduce the overhead of this. | `1` | Type: int | -| `--numa-node` | Sets the numa node for the subprocesses. i-th element corresponds to i-th subprocess. | `None` | List[int] | -| `--enable-deterministic-inference` | Enable deterministic inference mode with batch invariant ops. | `False` | bool flag (set to enable) | -| `--rl-on-policy-target` | The training system that SGLang needs to match for true on-policy. | `None` | `fsdp` | -| `--enable-attn-tp-input-scattered` | Allow input of attention to be scattered when only using tensor parallelism, to reduce the computational load of operations such as qkv latent. | `False` | bool flag (set to enable) | -| `--enable-dsa-prefill-context-parallel` | Enable context parallelism used in the long sequence prefill phase of DeepSeek v3.2. (`--enable-nsa-prefill-context-parallel` is a deprecated alias.) | `False` | bool flag (set to enable) | -| `--dsa-prefill-cp-mode` | Token splitting mode for the prefill phase of DeepSeek v3.2 under context parallelism. Optional values: `round-robin-split`(default),`in-seq-split`. `round-robin-split` distributes tokens across ranks based on `token_idx % cp_size`. It supports multi-batch prefill, fused MoE, and FP8 KV cache. (`--nsa-prefill-cp-mode` is a deprecated alias.) | `in-seq-split` | `in-seq-split`, `round-robin-split` | -| `--enable-fused-qk-norm-rope` | Enable fused qk normalization and rope rotary embedding. | `False` | bool flag (set to enable) | -| `--enable-precise-embedding-interpolation` | Enable corner alignment for resize of embeddings grid to ensure more accurate(but slower) evaluation of interpolated embedding values. | `False` | bool flag (set to enable) | - -## Dynamic batch tokenizer -| Argument | Description | Defaults | Options | -| --- | --- | --- | --- | -| `--enable-dynamic-batch-tokenizer` | Enable async dynamic batch tokenizer for improved performance when multiple requests arrive concurrently. | `False` | bool flag (set to enable) | -| `--dynamic-batch-tokenizer-batch-size` | [Only used if --enable-dynamic-batch-tokenizer is set] Maximum batch size for dynamic batch tokenizer. | `32` | Type: int | -| `--dynamic-batch-tokenizer-batch-timeout` | [Only used if --enable-dynamic-batch-tokenizer is set] Timeout in seconds for batching tokenization requests. | `0.002` | Type: float | - -## Debug tensor dumps -| Argument | Description | Defaults | Options | -| --- | --- | --- | --- | -| `--debug-tensor-dump-output-folder` | The output folder for dumping tensors. | `None` | Type: str | -| `--debug-tensor-dump-layers` | The layer ids to dump. Dump all layers if not specified. | `None` | Type: JSON list | -| `--debug-tensor-dump-input-file` | The input filename for dumping tensors | `None` | Type: str | -| `--debug-tensor-dump-inject` | Inject the outputs from jax as the input of every layer. | `False` | Type: str | - -## PD disaggregation -| Argument | Description | Defaults | Options | -| --- | --- | --- | --- | -| `--disaggregation-mode` | Only used for PD disaggregation. "prefill" for prefill-only server, and "decode" for decode-only server. If not specified, it is not PD disaggregated | `null` | `null`, `prefill`, `decode` | -| `--disaggregation-transfer-backend` | The backend for disaggregation transfer. Default is mooncake. | `mooncake` | `mooncake`, `nixl`, `ascend`, `fake` | -| `--disaggregation-bootstrap-port` | Bootstrap server port on the prefill server. Default is 8998. | `8998` | Type: int | -| `--disaggregation-ib-device` | The InfiniBand devices for disaggregation transfer, accepts single device (e.g., --disaggregation-ib-device mlx5_0) or multiple comma-separated devices (e.g., --disaggregation-ib-device mlx5_0,mlx5_1). Default is None, which triggers automatic device detection when mooncake backend is enabled. | `None` | Type: str | -| `--disaggregation-decode-enable-offload-kvcache` | Enable async KV cache offloading on decode server (PD mode). | `False` | bool flag (set to enable) | -| `--num-reserved-decode-tokens` | Number of decode tokens that will have memory reserved when adding new request to the running batch. | `512` | Type: int | -| `--disaggregation-decode-polling-interval` | The interval to poll requests in decode server. Can be set to >1 to reduce the overhead of this. | `1` | Type: int | - -## Encode prefill disaggregation -| Argument | Description | Defaults | Options | -| --- | --- | --- | --- | -| `--encoder-only` | For MLLM with an encoder, launch an encoder-only server | `False` | bool flag (set to enable) | -| `--language-only` | For VLM, load weights for the language model only. | `False` | bool flag (set to enable) | -| `--encoder-transfer-backend` | The backend for encoder disaggregation transfer. Default is zmq_to_scheduler. | `zmq_to_scheduler` | `zmq_to_scheduler`, `zmq_to_tokenizer`, `mooncake` | -| `--encoder-urls` | List of encoder server urls. | `[]` | Type: JSON list | - -## Custom weight loader -| Argument | Description | Defaults | Options | -| --- | --- | --- | --- | -| `--custom-weight-loader` | The custom dataloader which used to update the model. Should be set with a valid import path, such as my_package.weight_load_func | `None` | List[str] | -| `--weight-loader-disable-mmap` | Disable mmap while loading weight using safetensors. | `False` | bool flag (set to enable) | -| `--weight-loader-prefetch-checkpoints` | Prefetch checkpoint files into OS page cache before loading. Each rank prefetches a fraction of the shards in a background thread, reducing total network I/O on shared filesystems (NFS/Lustre) from N\*checkpoint to 1\*checkpoint. Recommended for models on network storage. | `False` | bool flag (set to enable) | -| `--weight-loader-prefetch-num-threads` | Number of threads per rank for checkpoint prefetching. | `4` | Type: int | -| `--remote-instance-weight-loader-seed-instance-ip` | The ip of the seed instance for loading weights from remote instance. | `None` | Type: str | -| `--remote-instance-weight-loader-seed-instance-service-port` | The service port of the seed instance for loading weights from remote instance. | `None` | Type: int | -| `--remote-instance-weight-loader-send-weights-group-ports` | The communication group ports for loading weights from remote instance. | `None` | Type: JSON list | -| `--remote-instance-weight-loader-backend` | The backend for loading weights from remote instance. Can be 'transfer_engine' or 'nccl'. Default is 'nccl'. | `nccl` | `transfer_engine`, `nccl` | -| `--remote-instance-weight-loader-start-seed-via-transfer-engine` | Start seed server via transfer engine backend for remote instance weight loader. | `False` | bool flag (set to enable) | - -## For PD-Multiplexing -| Argument | Description | Defaults | Options | -| --- | --- | --- | --- | -| `--enable-pdmux` | Enable PD-Multiplexing, PD running on greenctx stream. | `False` | bool flag (set to enable) | -| `--pdmux-config-path` | The path of the PD-Multiplexing config file. | `None` | Type: str | -| `--sm-group-num` | Number of sm partition groups. | `8` | Type: int | - -## Configuration file support -| Argument | Description | Defaults | Options | -| --- | --- | --- | --- | -| `--config` | Read CLI options from a config file. Must be a YAML file with configuration options. | `None` | Type: str | - -## For Multi-Modal -| Argument | Description | Defaults | Options | -| --- | --- | --- | --- | -| `--mm-max-concurrent-calls` | The max concurrent calls for async mm data processing. | `32` | Type: int | -| `--mm-per-request-timeout` | The timeout for each multi-modal request in seconds. | `10.0` | Type: int | -| `--enable-broadcast-mm-inputs-process` | Enable broadcast mm-inputs process in scheduler. | `False` | bool flag (set to enable) | -| `--mm-process-config` | Multimodal preprocessing config, a json config contains keys: `image`, `video`, `audio`. | `{}` | Type: JSON / Dict | -| `--mm-enable-dp-encoder` | Enabling data parallelism for mm encoder. The dp size will be set to the tp size automatically. | `False` | bool flag (set to enable) | -| `--limit-mm-data-per-request` | Limit the number of multimodal inputs per request. e.g. '{"image": 1, "video": 1, "audio": 1}' | `None` | Type: JSON / Dict | -| `--enable-mm-global-cache` | Enable Mooncake-backed global multimodal embedding cache on encoder servers so repeated images can reuse cached ViT embeddings instead of recomputing them. | `False` | bool flag (set to enable) | - -## For checkpoint decryption -| Argument | Description | Defaults | Options | -| --- | --- | --- | --- | -| `--decrypted-config-file` | The path of the decrypted config file. | `None` | Type: str | -| `--decrypted-draft-config-file` | The path of the decrypted draft config file. | `None` | Type: str | -| `--enable-prefix-mm-cache` | Enable prefix multimodal cache. Currently only supports mm-only. | `False` | bool flag (set to enable) | - -## Forward hooks -| Argument | Description | Defaults | Options | -| --- | --- | --- | --- | -| `--forward-hooks` | JSON-formatted list of forward hook specifications. Each element must include `target_modules` (list of glob patterns matched against `model.named_modules()` names) and `hook_factory` (Python import path to a factory, e.g. `my_package.hooks:make_hook`). An optional `name` field is used for logging, and an optional `config` object is passed as a `dict` to the factory. | `None` | Type: JSON list | - -## For MindStudio-probe(msProbe) dump -| Argument | Description | Defaults | Options | -| --- | --- | --- | --- | -| `--msprobe-dump-config` | The path of the JSON configuration file for msProbe. If specified, enables msProbe dump. | `None` | Type: str | - -## Deprecated arguments -| Argument | Description | Defaults | Options | -| --- | --- | --- | --- | -| `--enable-ep-moe` | NOTE: --enable-ep-moe is deprecated. Please set `--ep-size` to the same value as `--tp-size` instead. | `None` | N/A | -| `--enable-deepep-moe` | NOTE: --enable-deepep-moe is deprecated. Please set `--moe-a2a-backend` to 'deepep' instead. | `None` | N/A | -| `--prefill-round-robin-balance` | Note: Note: --prefill-round-robin-balance is deprecated now. | `None` | N/A | -| `--enable-flashinfer-cutlass-moe` | NOTE: --enable-flashinfer-cutlass-moe is deprecated. Please set `--moe-runner-backend` to 'flashinfer_cutlass' instead. | `None` | N/A | -| `--enable-flashinfer-cutedsl-moe` | NOTE: --enable-flashinfer-cutedsl-moe is deprecated. Please set `--moe-runner-backend` to 'flashinfer_cutedsl' instead. | `None` | N/A | -| `--enable-flashinfer-trtllm-moe` | NOTE: --enable-flashinfer-trtllm-moe is deprecated. Please set `--moe-runner-backend` to 'flashinfer_trtllm' instead. | `None` | N/A | -| `--enable-triton-kernel-moe` | NOTE: --enable-triton-kernel-moe is deprecated. Please set `--moe-runner-backend` to 'triton_kernel' instead. | `None` | N/A | -| `--enable-flashinfer-mxfp4-moe` | NOTE: --enable-flashinfer-mxfp4-moe is deprecated. Please set `--moe-runner-backend` to 'flashinfer_mxfp4' instead. | `None` | N/A | -| `--crash-on-nan` | Crash the server on nan logprobs. | `False` | Type: str | -| `--hybrid-kvcache-ratio` | Mix ratio in [0,1] between uniform and hybrid kv buffers (0.0 = pure uniform: swa_size / full_size = 1)(1.0 = pure hybrid: swa_size / full_size = local_attention_size / context_length) | `None` | Optional[float] | -| `--load-watch-interval` | The interval of load watching in seconds. | `0.1` | Type: float | -| `--nsa-prefill` | Deprecated alias for `--dsa-prefill-backend`. Choose the DSA backend for the prefill stage (overrides `--attention-backend` when running DeepSeek DSA-style attention). | `flashmla_sparse` | `flashmla_sparse`, `flashmla_decode`, `fa3`, `tilelang`, `aiter` | -| `--nsa-decode` | Deprecated alias for `--dsa-decode-backend`. Choose the DSA backend for the decode stage when running DeepSeek DSA-style attention. Overrides `--attention-backend` for decoding. | `flashmla_kv` | `flashmla_prefill`, `flashmla_kv`, `fa3`, `tilelang`, `aiter` | diff --git a/docs/advanced_features/sgl_model_gateway.md b/docs/advanced_features/sgl_model_gateway.md deleted file mode 100644 index 0f2da5b47..000000000 --- a/docs/advanced_features/sgl_model_gateway.md +++ /dev/null @@ -1,1736 +0,0 @@ -# SGLang Model Gateway - -SGLang Model Gateway is a high-performance model-routing gateway for large-scale LLM deployments. It centralizes worker lifecycle management, balances traffic across heterogeneous protocols (HTTP, gRPC, OpenAI-compatible), and provides enterprise-ready control over history storage, MCP tooling, and privacy-sensitive workflows. The gateway is deeply optimized for the SGLang serving runtime, but can route to any OpenAI-compatible backend. - ---- - -## Table of Contents - -1. [Overview](#overview) -2. [Architecture](#architecture) - - [Control Plane](#control-plane) - - [Data Plane](#data-plane) - - [Storage and Privacy](#storage-and-privacy) -3. [Installation](#installation) -4. [Quick Start](#quick-start) -5. [Deployment Modes](#deployment-modes) - - [Co-launch Router and Workers](#co-launch-router-and-workers) - - [Separate Launch (HTTP)](#separate-launch-http) - - [gRPC Launch](#grpc-launch) - - [Prefill-Decode Disaggregation](#prefill-decode-disaggregation) - - [OpenAI Backend Proxy](#openai-backend-proxy) - - [Multi-Model Inference Gateway](#multi-model-inference-gateway) -6. [API Reference](#api-reference) - - [Inference Endpoints](#inference-endpoints) - - [Tokenization Endpoints](#tokenization-endpoints) - - [Parser Endpoints](#parser-endpoints) - - [Classification API](#classification-api) - - [Conversation and Response APIs](#conversation-and-response-apis) - - [Worker Management APIs](#worker-management-apis) - - [Admin and Health Endpoints](#admin-and-health-endpoints) -7. [Load Balancing Policies](#load-balancing-policies) -8. [Reliability and Flow Control](#reliability-and-flow-control) - - [Retries](#retries) - - [Circuit Breaker](#circuit-breaker) - - [Rate Limiting and Queuing](#rate-limiting-and-queuing) - - [Health Checks](#health-checks) -9. [Reasoning Parser Integration](#reasoning-parser-integration) -10. [Tool Call Parsing](#tool-call-parsing) -11. [Tokenizer Management](#tokenizer-management) -12. [MCP Integration](#mcp-integration) -13. [Service Discovery (Kubernetes)](#service-discovery-kubernetes) -14. [History and Data Connectors](#history-and-data-connectors) -15. [WASM Middleware](#wasm-middleware) -16. [Language Bindings](#language-bindings) -17. [Security and Authentication](#security-and-authentication) - - [TLS (HTTPS) for Gateway Server](#tls-https-for-gateway-server) - - [mTLS for Worker Communication](#mtls-for-worker-communication) -18. [Observability](#observability) - - [Prometheus Metrics](#prometheus-metrics) - - [OpenTelemetry Tracing](#opentelemetry-tracing) - - [Logging](#logging) -19. [Production Recommendations](#production-recommendations) - - [Security Best Practices](#security-best-practices) - - [High Availability](#high-availability) - - [Performance](#performance) - - [Kubernetes Deployment](#kubernetes-deployment) - - [Monitoring with PromQL](#monitoring-with-promql) -20. [Configuration Reference](#configuration-reference) -21. [Troubleshooting](#troubleshooting) - ---- - -## Overview - -- **Unified control plane** for registering, monitoring, and orchestrating regular, prefill, and decode workers across heterogeneous model fleets. -- **Multi-protocol data plane** that routes traffic across HTTP, PD (prefill/decode), gRPC, and OpenAI-compatible backends with shared reliability primitives. -- **Industry-first gRPC pipeline** with native Rust tokenization, reasoning parsers, and tool-call execution for high-throughput, OpenAI-compatible serving; supports both single-stage and PD topologies. -- **Inference Gateway Mode (`--enable-igw`)** dynamically instantiates multiple router stacks (HTTP regular/PD, gRPC) and applies per-model policies for multi-tenant deployments. -- **Conversation & responses connectors** centralize chat history inside the router so the same context can be reused across models and MCP loops without leaking data to upstream vendors (memory, none, Oracle ATP, PostgreSQL). -- **Enterprise privacy**: agentic multi-turn `/v1/responses`, native MCP client (STDIO/HTTP/SSE/Streamable), and history storage all operate within the router boundary. -- **Reliability core**: retries with jitter, worker-scoped circuit breakers, token-bucket rate limiting with queuing, background health checks, and cache-aware load monitoring. -- **Comprehensive observability**: 40+ Prometheus metrics, OpenTelemetry distributed tracing, structured logging, and request ID propagation. - ---- - -## Architecture - -### Control Plane - -- **Worker Manager** discovers capabilities (`/server_info`, `/get_model_info`), tracks load, and registers/removes workers in the shared registry. -- **Job Queue** serializes add/remove requests and exposes status (`/workers/{worker_id}`) so clients can track onboarding progress. -- **Load Monitor** feeds cache-aware and power-of-two policies with live worker load statistics. -- **Health Checker** continuously probes workers and updates readiness, circuit breaker state, and router metrics. -- **Tokenizer Registry** manages dynamically registered tokenizers with async loading from HuggingFace or local paths. - -### Data Plane - -- **HTTP routers** (regular & PD) implement `/generate`, `/v1/chat/completions`, `/v1/completions`, `/v1/responses`, `/v1/embeddings`, `/v1/rerank`, `/v1/classify`, `/v1/tokenize`, `/v1/detokenize`, and associated admin endpoints. -- **gRPC router** streams tokenized requests directly to SRT gRPC workers, running fully in Rust—tokenizer, reasoning parser, and tool parser all reside in-process. Supports both single-stage and PD routing, including embeddings and classification. -- **OpenAI router** proxies OpenAI-compatible endpoints to external vendors (OpenAI, xAI, etc.) while keeping chat history and multi-turn orchestration local. - -### Storage and Privacy - -- Conversation and response history is stored at the router tier (memory, none, Oracle ATP, or PostgreSQL). The same history can power multiple models or MCP loops without sending data to upstream vendors. -- `/v1/responses` agentic flows, MCP sessions, and conversation APIs share the same storage layer, enabling compliance for regulated workloads. - ---- - -## Installation - -### Docker - -Pre-built Docker images are available on Docker Hub with multi-architecture support (x86_64 and ARM64): - -```bash -docker pull lmsysorg/sgl-model-gateway:latest -``` - -### Prerequisites - -- **Rust and Cargo** - ```bash - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh - source "$HOME/.cargo/env" - rustc --version - cargo --version - ``` -- **Python** with `pip` and virtualenv tooling available. - -### Rust Binary - -```bash -cd sgl-model-gateway -cargo build --release -``` - -### Python Package - -```bash -pip install maturin - -# Fast development mode -cd sgl-model-gateway/bindings/python -maturin develop - -# Production build -maturin build --release --out dist --features vendored-openssl -pip install --force-reinstall dist/*.whl -``` - ---- - -## Quick Start - -### Regular HTTP Routing - -```bash -# Rust binary -./target/release/sgl-model-gateway \ - --worker-urls http://worker1:8000 http://worker2:8000 \ - --policy cache_aware - -# Python launcher -python -m sglang_router.launch_router \ - --worker-urls http://worker1:8000 http://worker2:8000 \ - --policy cache_aware -``` - -### gRPC Routing - -```bash -python -m sglang_router.launch_router \ - --worker-urls grpc://127.0.0.1:20000 \ - --model-path meta-llama/Llama-3.1-8B-Instruct \ - --reasoning-parser deepseek-r1 \ - --tool-call-parser json \ - --host 0.0.0.0 --port 8080 -``` - ---- - -## Deployment Modes - -### Co-launch Router and Workers - -Launch the router and a fleet of SGLang workers in one process: - -```bash -python -m sglang_router.launch_server \ - --model meta-llama/Meta-Llama-3.1-8B-Instruct \ - --dp-size 4 \ - --host 0.0.0.0 \ - --port 30000 -``` - -Comprehensive example with router arguments (prefixed with `--router-`): - -```bash -python -m sglang_router.launch_server \ - --host 0.0.0.0 \ - --port 8080 \ - --model meta-llama/Llama-3.1-8B-Instruct \ - --tp-size 1 \ - --dp-size 8 \ - --grpc-mode \ - --log-level debug \ - --router-prometheus-port 10001 \ - --router-tool-call-parser llama \ - --router-model-path meta-llama/Llama-3.1-8B-Instruct \ - --router-policy round_robin \ - --router-log-level debug -``` - -### Separate Launch (HTTP) - -Run workers independently and point the router at their HTTP endpoints: - -```bash -# Worker nodes -python -m sglang.launch_server --model meta-llama/Meta-Llama-3.1-8B-Instruct --port 8000 -python -m sglang.launch_server --model meta-llama/Meta-Llama-3.1-8B-Instruct --port 8001 - -# Router node -python -m sglang_router.launch_router \ - --worker-urls http://worker1:8000 http://worker2:8001 \ - --policy cache_aware \ - --host 0.0.0.0 --port 30000 -``` - -### gRPC Launch - -Use SRT gRPC workers to unlock the highest throughput and access native reasoning/tool pipelines: - -```bash -# Workers expose gRPC endpoints -python -m sglang.launch_server \ - --model meta-llama/Llama-3.1-8B-Instruct \ - --grpc-mode \ - --port 20000 - -# Router -python -m sglang_router.launch_router \ - --worker-urls grpc://127.0.0.1:20000 \ - --model-path meta-llama/Llama-3.1-8B-Instruct \ - --reasoning-parser deepseek-r1 \ - --tool-call-parser json \ - --host 0.0.0.0 --port 8080 -``` - -The gRPC router supports both regular HTTP-equivalent serving and PD (prefill/decode) serving. Provide `--tokenizer-path` or `--model-path` (HuggingFace ID or local directory) whenever connection mode resolves to gRPC. - -### Prefill-Decode Disaggregation - -Split prefill and decode workers for PD-aware caching and balancing: - -```bash -python -m sglang_router.launch_router \ - --pd-disaggregation \ - --prefill http://prefill1:30001 9001 \ - --decode http://decode1:30011 \ - --prefill-policy cache_aware \ - --decode-policy power_of_two -``` - -Prefill entries accept an optional bootstrap port. PD mode merges prefill metadata with decode outputs and streams results back to the client. - -### OpenAI Backend Proxy - -Proxy OpenAI-compatible endpoints while keeping history and MCP sessions local: - -```bash -python -m sglang_router.launch_router \ - --backend openai \ - --worker-urls https://api.openai.com \ - --history-backend memory -``` - -OpenAI backend mode expects exactly one `--worker-urls` entry per router instance. - -### Multi-Model Inference Gateway - -Enable IGW mode to route multiple models through a single router: - -```bash -./target/release/sgl-model-gateway \ - --enable-igw \ - --policy cache_aware \ - --max-concurrent-requests 512 - -# Register workers dynamically -curl -X POST http://localhost:30000/workers \ - -H "Content-Type: application/json" \ - -d '{ - "url": "http://worker-a:8000", - "model_id": "mistral", - "priority": 10, - "labels": {"tier": "gold"} - }' -``` - ---- - -## API Reference - -### Inference Endpoints - -| Method | Path | Description | -|--------|------|-------------| -| `POST` | `/generate` | SGLang generate API | -| `POST` | `/v1/chat/completions` | OpenAI-compatible chat completions (streaming/tool calls) | -| `POST` | `/v1/completions` | OpenAI-compatible text completions | -| `POST` | `/v1/embeddings` | Embedding generation (HTTP and gRPC) | -| `POST` | `/v1/rerank`, `/rerank` | Reranking requests | -| `POST` | `/v1/classify` | Text classification | - -### Tokenization Endpoints - -The gateway provides HTTP endpoints for text tokenization with batch support, designed to mirror the SGLang Python tokenization API. - -| Method | Path | Description | -|--------|------|-------------| -| `POST` | `/v1/tokenize` | Tokenize text to token IDs (single or batch) | -| `POST` | `/v1/detokenize` | Convert token IDs back to text (single or batch) | -| `POST` | `/v1/tokenizers` | Register a new tokenizer (async, returns job status) | -| `GET` | `/v1/tokenizers` | List all registered tokenizers | -| `GET` | `/v1/tokenizers/{id}` | Get tokenizer info by UUID | -| `GET` | `/v1/tokenizers/{id}/status` | Check async tokenizer loading status | -| `DELETE` | `/v1/tokenizers/{id}` | Remove a tokenizer from the registry | - -#### Tokenize Request - -```json -{ - "model": "meta-llama/Llama-3.1-8B-Instruct", - "prompt": "Hello, world!" -} -``` - -#### Batch Tokenize Request - -```json -{ - "model": "meta-llama/Llama-3.1-8B-Instruct", - "prompt": ["Hello", "World", "How are you?"] -} -``` - -#### Tokenize Response - -```json -{ - "tokens": [15339, 11, 1917, 0], - "count": 4, - "char_count": 13 -} -``` - -#### Detokenize Request - -```json -{ - "model": "meta-llama/Llama-3.1-8B-Instruct", - "tokens": [15339, 11, 1917, 0], - "skip_special_tokens": true -} -``` - -#### Detokenize Response - -```json -{ - "text": "Hello, world!" -} -``` - -#### Add Tokenizer (Async) - -```bash -curl -X POST http://localhost:30000/v1/tokenizers \ - -H "Content-Type: application/json" \ - -d '{"name": "llama3", "source": "meta-llama/Llama-3.1-8B-Instruct"}' -``` - -Response: -```json -{ - "id": "550e8400-e29b-41d4-a716-446655440000", - "status": "pending", - "message": "Tokenizer registration queued" -} -``` - -Check status: -```bash -curl http://localhost:30000/v1/tokenizers/550e8400-e29b-41d4-a716-446655440000/status -``` - -### Parser Endpoints - -The gateway provides admin endpoints for parsing reasoning content and function calls from LLM outputs. - -| Method | Path | Description | -|--------|------|-------------| -| `POST` | `/parse/reasoning` | Separate reasoning (``) from normal text | -| `POST` | `/parse/function_call` | Parse function/tool calls from text | - -#### Separate Reasoning Request - -```json -{ - "text": "Let me analyze this step by step...The answer is 42.", - "parser": "deepseek-r1" -} -``` - -#### Response - -```json -{ - "normal_text": "The answer is 42.", - "reasoning_text": "Let me analyze this step by step..." -} -``` - -#### Function Call Parsing - -```json -{ - "text": "{\"name\": \"get_weather\", \"arguments\": {\"city\": \"NYC\"}}", - "parser": "json" -} -``` - -### Classification API - -The `/v1/classify` endpoint provides text classification using sequence classification models (e.g., `Qwen2ForSequenceClassification`, `BertForSequenceClassification`). - -#### Request - -```bash -curl http://localhost:30000/v1/classify \ - -H "Content-Type: application/json" \ - -d '{ - "model": "jason9693/Qwen2.5-1.5B-apeach", - "input": "I love this product!" - }' -``` - -#### Response - -```json -{ - "id": "classify-a1b2c3d4-5678-90ab-cdef-1234567890ab", - "object": "list", - "created": 1767034308, - "model": "jason9693/Qwen2.5-1.5B-apeach", - "data": [ - { - "index": 0, - "label": "positive", - "probs": [0.12, 0.88], - "num_classes": 2 - } - ], - "usage": { - "prompt_tokens": 6, - "completion_tokens": 0, - "total_tokens": 6 - } -} -``` - -#### Response Fields - -| Field | Description | -|-------|-------------| -| `label` | Predicted class label (from model's `id2label` config, or `LABEL_N` fallback) | -| `probs` | Probability distribution over all classes (softmax of logits) | -| `num_classes` | Number of classification classes | - -#### Notes - -- Classification reuses the embedding backend—the scheduler returns logits which are converted to probabilities via softmax -- Labels come from the model's HuggingFace config (`id2label` field); models without this mapping use generic labels (`LABEL_0`, `LABEL_1`, etc.) -- Both HTTP and gRPC routers support classification - -### Conversation and Response APIs - -| Method | Path | Description | -|--------|------|-------------| -| `POST` | `/v1/responses` | Create background responses (agentic loops) | -| `GET` | `/v1/responses/{id}` | Retrieve stored response | -| `POST` | `/v1/responses/{id}/cancel` | Cancel background response | -| `DELETE` | `/v1/responses/{id}` | Delete response | -| `GET` | `/v1/responses/{id}/input_items` | List response input items | -| `POST` | `/v1/conversations` | Create conversation | -| `GET` | `/v1/conversations/{id}` | Get conversation | -| `POST` | `/v1/conversations/{id}` | Update conversation | -| `DELETE` | `/v1/conversations/{id}` | Delete conversation | -| `GET` | `/v1/conversations/{id}/items` | List conversation items | -| `POST` | `/v1/conversations/{id}/items` | Add items to conversation | -| `GET` | `/v1/conversations/{id}/items/{item_id}` | Get conversation item | -| `DELETE` | `/v1/conversations/{id}/items/{item_id}` | Delete conversation item | - -### Worker Management APIs - -| Method | Path | Description | -|--------|------|-------------| -| `POST` | `/workers` | Queue worker registration (returns 202 Accepted) | -| `GET` | `/workers` | List workers with health, load, and policy metadata | -| `GET` | `/workers/{worker_id}` | Inspect specific worker or job queue entry | -| `PUT` | `/workers/{worker_id}` | Queue worker update | -| `DELETE` | `/workers/{worker_id}` | Queue worker removal | - -#### Add Worker - -```bash -curl -X POST http://localhost:30000/workers \ - -H "Content-Type: application/json" \ - -d '{"url":"grpc://0.0.0.0:31000","worker_type":"regular"}' -``` - -#### List Workers - -```bash -curl http://localhost:30000/workers -``` - -Response: -```json -{ - "workers": [ - { - "id": "2f3a0c3e-3a7b-4c3f-8c70-1b7d4c3a6e1f", - "url": "http://0.0.0.0:31378", - "model_id": "mistral", - "priority": 50, - "cost": 1.0, - "worker_type": "regular", - "is_healthy": true, - "load": 0, - "connection_mode": "Http" - } - ], - "total": 1, - "stats": { - "prefill_count": 0, - "decode_count": 0, - "regular_count": 1 - } -} -``` - -### Admin and Health Endpoints - -| Method | Path | Description | -|--------|------|-------------| -| `GET` | `/liveness` | Health check (always returns OK) | -| `GET` | `/readiness` | Readiness check (checks healthy worker availability) | -| `GET` | `/health` | Alias for liveness | -| `GET` | `/health_generate` | Health generate test | -| `GET` | `/engine_metrics` | Engine-level metrics from workers | -| `GET` | `/v1/models` | List available models | -| `GET` | `/get_model_info` | Get model information | -| `GET` | `/server_info` | Get server information | -| `POST` | `/flush_cache` | Clear all caches | -| `GET` | `/get_loads` | Get all worker loads | -| `POST` | `/wasm` | Upload WASM module | -| `GET` | `/wasm` | List WASM modules | -| `DELETE` | `/wasm/{module_uuid}` | Remove WASM module | - ---- - -## Load Balancing Policies - -| Policy | Description | Usage | -|--------|-------------|-------| -| `random` | Uniform random selection | `--policy random` | -| `round_robin` | Cycles through workers in order | `--policy round_robin` | -| `power_of_two` | Samples two workers and picks the lighter one | `--policy power_of_two` | -| `cache_aware` | Combines cache locality with load balancing (default) | `--policy cache_aware` | -| `bucket` | Divides workers into load buckets with dynamic boundaries | `--policy bucket` | - -### Cache-Aware Policy Tuning - -```bash ---cache-threshold 0.5 \ ---balance-abs-threshold 32 \ ---balance-rel-threshold 1.5 \ ---eviction-interval-secs 120 \ ---max-tree-size 67108864 -``` - -| Parameter | Default | Description | -|-----------|---------|-------------| -| `--cache-threshold` | 0.3 | Minimum prefix match ratio for cache hit | -| `--balance-abs-threshold` | 64 | Absolute load difference before rebalancing | -| `--balance-rel-threshold` | 1.5 | Relative load ratio before rebalancing | -| `--eviction-interval-secs` | 120 | Cache eviction cadence in seconds | -| `--max-tree-size` | 67108864 | Maximum nodes in cache tree | - ---- - -## Reliability and Flow Control - -### HTTP Client - -Configure upstream HTTP client connection settings: - -| Parameter | Default | Description | -|-----------|---------|-------------| -| `--pool-idle-timeout-secs` | 50 | Idle timeout in seconds for pooled upstream HTTP connections. Can also be set with `SMG_POOL_IDLE_TIMEOUT_SECS`. | -| `--connect-timeout-secs` | 10 | Timeout in seconds for new upstream HTTP connections. Can also be set with `SMG_CONNECT_TIMEOUT_SECS`. | -| `--pool-max-idle-per-host` | 500 | Maximum idle upstream HTTP connections to keep per host. Can also be set with `SMG_POOL_MAX_IDLE_PER_HOST`. | -| `--tcp-keepalive-secs` | 30 | TCP keepalive idle time in seconds for upstream HTTP connections. Can also be set with `SMG_TCP_KEEPALIVE_SECS`. | - -### Retries - -Configure exponential backoff retries: - -```bash -python -m sglang_router.launch_router \ - --worker-urls http://worker1:8000 http://worker2:8001 \ - --retry-max-retries 5 \ - --retry-initial-backoff-ms 50 \ - --retry-max-backoff-ms 30000 \ - --retry-backoff-multiplier 1.5 \ - --retry-jitter-factor 0.2 -``` - -| Parameter | Default | Description | -|-----------|---------|-------------| -| `--retry-max-retries` | 5 | Maximum retry attempts | -| `--retry-initial-backoff-ms` | 50 | Initial backoff duration (ms) | -| `--retry-max-backoff-ms` | 5000 | Maximum backoff duration (ms) | -| `--retry-backoff-multiplier` | 2.0 | Exponential backoff multiplier | -| `--retry-jitter-factor` | 0.1 | Random jitter factor (0.0-1.0) | -| `--disable-retries` | false | Disable retries entirely | - -**Retryable Status Codes:** 408, 429, 500, 502, 503, 504 - -### Circuit Breaker - -Per-worker circuit breakers prevent cascading failures: - -```bash -python -m sglang_router.launch_router \ - --worker-urls http://worker1:8000 http://worker2:8001 \ - --cb-failure-threshold 5 \ - --cb-success-threshold 2 \ - --cb-timeout-duration-secs 30 \ - --cb-window-duration-secs 60 -``` - -| Parameter | Default | Description | -|-----------|---------|-------------| -| `--cb-failure-threshold` | 5 | Consecutive failures to open circuit | -| `--cb-success-threshold` | 2 | Successes to close from half-open | -| `--cb-timeout-duration-secs` | 30 | Time before half-open attempt | -| `--cb-window-duration-secs` | 60 | Failure counting window | -| `--disable-circuit-breaker` | false | Disable circuit breaker | - -**Circuit Breaker States:** -- **Closed**: Normal operation, requests allowed -- **Open**: Failing, requests rejected immediately -- **Half-Open**: Testing recovery, limited requests allowed - -### Rate Limiting and Queuing - -```bash -python -m sglang_router.launch_router \ - --worker-urls http://worker1:8000 http://worker2:8001 \ - --max-concurrent-requests 256 \ - --rate-limit-tokens-per-second 512 \ - --queue-size 128 \ - --queue-timeout-secs 30 -``` - -Requests beyond the concurrency limit wait in a FIFO queue. Returns: -- `429 Too Many Requests` when queue is full -- `408 Request Timeout` when queue timeout expires - -### Health Checks - -```bash ---health-check-interval-secs 30 \ ---health-check-timeout-secs 10 \ ---health-success-threshold 2 \ ---health-failure-threshold 3 \ ---health-check-endpoint /health -``` - ---- - -## Reasoning Parser Integration - -The gateway includes built-in reasoning parsers for models that use Chain-of-Thought (CoT) reasoning with explicit thinking blocks. - -### Supported Parsers - -| Parser ID | Model Family | Think Tokens | -|-----------|--------------|--------------| -| `deepseek-r1` | DeepSeek-R1 | `...` (initial reasoning) | -| `qwen3` | Qwen-3 | `...` | -| `qwen3-thinking` | Qwen-3 Thinking | `...` (initial reasoning) | -| `kimi` | Kimi K2 | Unicode think tokens | -| `glm45` | GLM-4.5/4.6/4.7 | `...` | -| `step3` | Step-3 | `...` | -| `minimax` | MiniMax | `...` | - -### Usage - -```bash -python -m sglang_router.launch_router \ - --worker-urls grpc://127.0.0.1:20000 \ - --model-path deepseek-ai/DeepSeek-R1 \ - --reasoning-parser deepseek-r1 -``` - -The gRPC router automatically: -1. Detects reasoning blocks in streaming output -2. Separates reasoning content from normal text -3. Applies incremental streaming parsing with buffer management -4. Handles partial token detection for correct streaming behavior - ---- - -## Tool Call Parsing - -The gateway supports parsing function/tool calls from LLM outputs in multiple formats. - -### Supported Formats - -| Parser | Format | Description | -|--------|--------|-------------| -| `json` | JSON | Standard JSON tool calls | -| `python` | Pythonic | Python function call syntax | -| `xml` | XML | XML-formatted tool calls | - -### Usage - -```bash -python -m sglang_router.launch_router \ - --worker-urls grpc://127.0.0.1:20000 \ - --model-path meta-llama/Llama-3.1-8B-Instruct \ - --tool-call-parser json -``` - ---- - -## Tokenizer Management - -### Tokenizer Sources - -The gateway supports multiple tokenizer backends: -- **HuggingFace**: Load from HuggingFace Hub by model ID -- **Local**: Load from local `tokenizer.json` or directory -- **Tiktoken**: Auto-detect OpenAI GPT models (gpt-4, davinci, etc.) - -### Configuration - -```bash -# HuggingFace model ---model-path meta-llama/Llama-3.1-8B-Instruct - -# Local tokenizer ---tokenizer-path /path/to/tokenizer.json - -# With chat template override ---chat-template /path/to/template.jinja -``` - -### Tokenizer Caching - -Two-level caching for optimal performance: - -| Cache | Type | Description | -|-------|------|-------------| -| L0 | Exact match | Whole-string caching for repeated prompts | -| L1 | Prefix match | Prefix boundary matching for incremental prompts | - -```bash ---enable-l0-cache \ ---l0-max-entries 10000 \ ---enable-l1-cache \ ---l1-max-memory 52428800 # 50MB -``` - ---- - -## MCP Integration - -The gateway provides native Model Context Protocol (MCP) client integration for tool execution. - -### Supported Transports - -| Transport | Description | -|-----------|-------------| -| STDIO | Local process execution | -| SSE | Server-Sent Events (HTTP) | -| Streamable | Bidirectional streaming | - -### Configuration - -```bash -python -m sglang_router.launch_router \ - --mcp-config-path /path/to/mcp-config.yaml \ - --worker-urls http://worker1:8000 -``` - -### MCP Configuration File - -```yaml -servers: - - name: "filesystem" - command: "npx" - args: ["-y", "@modelcontextprotocol/server-filesystem", "/tmp"] - protocol: "stdio" - required: false - - - name: "github" - url: "https://api.github.com/mcp" - token: "ghp_xxxxx" - protocol: "sse" - required: false - - - name: "custom-tools" - url: "https://tools.example.com/mcp" - protocol: "streamable" - required: true - -pool: - max_connections: 100 - idle_timeout: 300 - -proxy: - http: "http://proxy.internal:8080" - https: "https://proxy.internal:8443" - no_proxy: "localhost,127.0.0.1,*.internal" - -inventory: - enable_refresh: true - tool_ttl: 300 - refresh_interval: 300 -``` - ---- - -## Service Discovery (Kubernetes) - -Enable automatic worker discovery via Kubernetes pod selectors: - -```bash -python -m sglang_router.launch_router \ - --service-discovery \ - --selector app=sglang-worker role=inference \ - --service-discovery-namespace production \ - --service-discovery-port 8000 -``` - -### PD Mode Discovery - -```bash ---pd-disaggregation \ ---prefill-selector app=sglang component=prefill \ ---decode-selector app=sglang component=decode \ ---service-discovery -``` - -Prefill pods can expose bootstrap ports via the `sglang.ai/bootstrap-port` annotation. RBAC must allow `get`, `list`, and `watch` on pods. - ---- - -## History and Data Connectors - -| Backend | Description | Usage | -|---------|-------------|-------| -| `memory` | In-memory storage (default) | `--history-backend memory` | -| `none` | No persistence | `--history-backend none` | -| `oracle` | Oracle Autonomous Database | `--history-backend oracle` | -| `postgres` | PostgreSQL Database | `--history-backend postgres` | -| `redis` | Redis | `--history-backend redis` | - -### Oracle Configuration - -```bash -# Connection descriptor -export ATP_DSN="(description=(address=(protocol=tcps)(port=1522)(host=adb.region.oraclecloud.com))(connect_data=(service_name=service_name)))" - -# Or TNS alias (requires wallet) -export ATP_TNS_ALIAS="sglroutertestatp_high" -export ATP_WALLET_PATH="/path/to/wallet" - -# Credentials -export ATP_USER="admin" -export ATP_PASSWORD="secret" -export ATP_POOL_MIN=4 -export ATP_POOL_MAX=32 - -python -m sglang_router.launch_router \ - --backend openai \ - --worker-urls https://api.openai.com \ - --history-backend oracle -``` - -### PostgreSQL Configuration - -```bash -export POSTGRES_DB_URL="postgres://user:password@host:5432/dbname" - -python -m sglang_router.launch_router \ - --backend openai \ - --worker-urls https://api.openai.com \ - --history-backend postgres -``` - -### Redis Configuration - -```bash -export REDIS_URL="redis://localhost:6379" -export REDIS_POOL_MAX=16 -export REDIS_RETENTION_DAYS=30 - -python -m sglang_router.launch_router \ - --backend openai \ - --worker-urls https://api.openai.com \ - --history-backend redis \ - --redis-retention-days 30 -``` - -Use `--redis-retention-days -1` for persistent storage (default is 30 days). - ---- - -## WASM Middleware - -The gateway supports WebAssembly (WASM) middleware modules for custom request/response processing. This enables organization-specific logic for authentication, rate limiting, billing, logging, and more—without modifying or recompiling the gateway. - -### Overview - -WASM middleware runs in a sandboxed environment with memory isolation, no network/filesystem access, and configurable resource limits. - -| Attach Point | When Executed | Use Cases | -|--------------|---------------|-----------| -| `OnRequest` | Before forwarding to workers | Auth, rate limiting, request modification | -| `OnResponse` | After receiving worker response | Logging, response modification, error handling | - -| Action | Description | -|--------|-------------| -| `Continue` | Proceed without modification | -| `Reject(status)` | Reject request with HTTP status code | -| `Modify(...)` | Modify headers, body, or status | - -### Examples - -Complete working examples are available in `examples/wasm/`: - -| Example | Description | -|---------|-------------| -| `auth/` | API key authentication for protected routes | -| `rate_limit/` | Per-client rate limiting (requests/minute) | -| `logging/` | Request tracking headers and response modification | - -The interface definition is located at `src/wasm/interface`. - -### Building Modules - -```bash -# Prerequisites -rustup target add wasm32-wasip2 -cargo install wasm-tools - -# Build -cargo build --target wasm32-wasip2 --release - -# Convert to component format -wasm-tools component new \ - target/wasm32-wasip2/release/my_middleware.wasm \ - -o my_middleware.component.wasm -``` - -### Deploying Modules - -```bash -# Enable WASM support -python -m sglang_router.launch_router \ - --worker-urls http://worker1:8000 \ - --enable-wasm - -# Upload module -curl -X POST http://localhost:30000/wasm \ - -H "Content-Type: application/json" \ - -d '{ - "modules": [{ - "name": "auth-middleware", - "file_path": "/absolute/path/to/auth.component.wasm", - "module_type": "Middleware", - "attach_points": [{"Middleware": "OnRequest"}] - }] - }' - -# List modules -curl http://localhost:30000/wasm - -# Remove module -curl -X DELETE http://localhost:30000/wasm/{module_uuid} -``` - -### Runtime Configuration - -| Parameter | Default | Description | -|-----------|---------|-------------| -| `max_memory_pages` | 1024 (64MB) | Maximum WASM memory | -| `max_execution_time_ms` | 1000 | Execution timeout | -| `max_stack_size` | 1MB | Stack size limit | -| `module_cache_size` | 10 | Cached modules per worker | - -**Note:** Rate limiting state is per-worker thread and not shared across gateway replicas. For production, consider implementing rate limiting at a shared layer (e.g., Redis) - ---- - -## Language Bindings - -SGLang Model Gateway provides official language bindings for Python and Go, enabling integration with different technology stacks and organizational requirements. - -### Python Bindings - -The Python bindings provide a PyO3-based wrapper around the Rust gateway library. This is a straightforward binding that calls the gateway server startup from Python. - -#### Installation - -```bash -# From PyPI -pip install sglang-router - -# Development build -cd sgl-model-gateway/bindings/python -pip install maturin && maturin develop --features vendored-openssl -``` - -#### Usage - -The Python bindings are used throughout this documentation. See the [Quick Start](#quick-start) and [Deployment Modes](#deployment-modes) sections for detailed examples. - -Key components: -- `RouterArgs` dataclass with 50+ configuration options -- `Router.from_args()` for programmatic startup -- CLI commands: `smg launch`, `smg server`, `python -m sglang_router.launch_router` - -### Go Bindings - -The Go bindings provide a high-performance gRPC client library for organizations with Go-based infrastructure. This is ideal for: - -- Integration with internal Go services and tooling -- High-performance client applications -- Building custom OpenAI-compatible proxy servers - -#### Architecture - -``` -┌─────────────────────────────────────────┐ -│ High-Level Go API │ -│ (client.go - OpenAI-style interface) │ -├─────────────────────────────────────────┤ -│ gRPC Layer │ -├─────────────────────────────────────────┤ -│ Rust FFI Layer │ -│ (Tokenization, Parsing, Conversion) │ -└─────────────────────────────────────────┘ -``` - -**Key Features:** -- Native Rust tokenization via FFI (thread-safe, lock-free) -- Full streaming support with context cancellation -- Configurable channel buffer sizes for high concurrency -- Built-in tool call parsing and chat template application - -#### Installation - -```bash -# Build the FFI library first -cd sgl-model-gateway/bindings/golang -make build && make lib - -# Then use in your Go project -go get github.com/sgl-project/sgl-go-sdk -``` - -**Requirements:** Go 1.24+, Rust toolchain - -#### Examples - -Complete working examples are available in `bindings/golang/examples/`: - -| Example | Description | -|---------|-------------| -| `simple/` | Non-streaming chat completion | -| `streaming/` | Streaming chat completion with SSE | -| `oai_server/` | Full OpenAI-compatible HTTP server | - -```bash -# Run examples -cd sgl-model-gateway/bindings/golang/examples/simple && ./run.sh -cd sgl-model-gateway/bindings/golang/examples/streaming && ./run.sh -cd sgl-model-gateway/bindings/golang/examples/oai_server && ./run.sh -``` - -#### Testing - -```bash -cd sgl-model-gateway/bindings/golang - -# Unit tests -go test -v ./... - -# Integration tests (requires running SGLang server) -export SGL_GRPC_ENDPOINT=grpc://localhost:20000 -export SGL_TOKENIZER_PATH=/path/to/tokenizer -go test -tags=integration -v ./... -``` - -### Comparison - -| Feature | Python | Go | -|---------|--------|-----| -| **Primary Use** | Gateway server launcher | gRPC client library | -| **CLI Support** | Full CLI (smg, sglang-router) | Library only | -| **K8s Discovery** | Native support | N/A (client library) | -| **PD Mode** | Built-in | N/A (client library) | - -**When to Use Python:** Launching and managing the gateway server, service discovery, PD disaggregation. - -**When to Use Go:** Building custom client applications, integration with Go microservices, OpenAI-compatible proxy servers - ---- - -## Security and Authentication - -### Router API Key - -```bash -python -m sglang_router.launch_router \ - --api-key "your-router-api-key" \ - --worker-urls http://worker1:8000 -``` - -Clients must supply `Authorization: Bearer ` for protected endpoints. - -### Worker API Keys - -```bash -# Add worker with explicit key -curl -H "Authorization: Bearer router-key" \ - -X POST http://localhost:8080/workers \ - -H "Content-Type: application/json" \ - -d '{"url":"http://worker:8000","api_key":"worker-key"}' -``` - -### Security Configurations - -1. **No Authentication** (default): Use only in trusted environments -2. **Router-only Authentication**: Clients authenticate to router -3. **Worker-only Authentication**: Router open, workers require keys -4. **Full Authentication**: Both router and workers protected - -### TLS (HTTPS) for Gateway Server - -Enable TLS to serve the gateway over HTTPS: - -```bash -python -m sglang_router.launch_router \ - --worker-urls http://worker1:8000 \ - --tls-cert-path /path/to/server.crt \ - --tls-key-path /path/to/server.key -``` - -| Parameter | Description | -|-----------|-------------| -| `--tls-cert-path` | Path to server certificate (PEM format) | -| `--tls-key-path` | Path to server private key (PEM format) | - -Both parameters must be provided together. The gateway uses rustls with the ring crypto provider for TLS termination. If TLS is not configured, the gateway falls back to plain HTTP. - -### mTLS for Worker Communication - -Enable mutual TLS (mTLS) for secure communication with workers in HTTP mode: - -```bash -python -m sglang_router.launch_router \ - --worker-urls https://worker1:8443 https://worker2:8443 \ - --client-cert-path /path/to/client.crt \ - --client-key-path /path/to/client.key \ - --ca-cert-path /path/to/ca.crt -``` - -| Parameter | Description | -|-----------|-------------| -| `--client-cert-path` | Path to client certificate for mTLS (PEM format) | -| `--client-key-path` | Path to client private key for mTLS (PEM format) | -| `--ca-cert-path` | Path to CA certificate for verifying worker TLS (PEM format, repeatable) | - -**Key Points:** -- Client certificate and key must be provided together -- Multiple CA certificates can be added with multiple `--ca-cert-path` flags -- Uses rustls backend when TLS is configured -- Single HTTP client is created for all workers (assumes single security domain) -- TCP keepalive (30 seconds) is enabled for long-lived connections - -### Full TLS Configuration Example - -Gateway HTTPS + Worker mTLS + API Key authentication: - -```bash -python -m sglang_router.launch_router \ - --worker-urls https://worker1:8443 https://worker2:8443 \ - --tls-cert-path /etc/certs/server.crt \ - --tls-key-path /etc/certs/server.key \ - --client-cert-path /etc/certs/client.crt \ - --client-key-path /etc/certs/client.key \ - --ca-cert-path /etc/certs/ca.crt \ - --api-key "secure-api-key" \ - --policy cache_aware -``` - ---- - -## Observability - -### Prometheus Metrics - -Enable with `--prometheus-host`/`--prometheus-port` (defaults to `0.0.0.0:29000`). - -#### Metric Categories (40+ metrics) - -| Layer | Prefix | Metrics | -|-------|--------|---------| -| HTTP | `smg_http_*` | `requests_total`, `request_duration_seconds`, `responses_total`, `connections_active`, `rate_limit_total` | -| Router | `smg_router_*` | `requests_total`, `request_duration_seconds`, `request_errors_total`, `stage_duration_seconds`, `upstream_responses_total` | -| Inference | `smg_router_*` | `ttft_seconds`, `tpot_seconds`, `tokens_total`, `generation_duration_seconds` | -| Worker | `smg_worker_*` | `pool_size`, `connections_active`, `requests_active`, `health_checks_total`, `selection_total`, `errors_total` | -| Circuit Breaker | `smg_worker_cb_*` | `state`, `transitions_total`, `outcomes_total`, `consecutive_failures`, `consecutive_successes` | -| Retry | `smg_worker_*` | `retries_total`, `retries_exhausted_total`, `retry_backoff_seconds` | -| Discovery | `smg_discovery_*` | `registrations_total`, `deregistrations_total`, `sync_duration_seconds`, `workers_discovered` | -| MCP | `smg_mcp_*` | `tool_calls_total`, `tool_duration_seconds`, `servers_active`, `tool_iterations_total` | -| Database | `smg_db_*` | `operations_total`, `operation_duration_seconds`, `connections_active`, `items_stored` | - -#### Key Inference Metrics (gRPC mode) - -| Metric | Type | Description | -|--------|------|-------------| -| `smg_router_ttft_seconds` | Histogram | Time to first token | -| `smg_router_tpot_seconds` | Histogram | Time per output token | -| `smg_router_tokens_total` | Counter | Total tokens (input/output) | -| `smg_router_generation_duration_seconds` | Histogram | End-to-end generation time | - -#### Duration Buckets - -1ms, 5ms, 10ms, 25ms, 50ms, 100ms, 250ms, 500ms, 1s, 2.5s, 5s, 10s, 15s, 30s, 45s, 60s, 90s, 120s, 180s, 240s - -### OpenTelemetry Tracing - -Enable distributed tracing with OTLP export: - -```bash -python -m sglang_router.launch_router \ - --worker-urls http://worker1:8000 \ - --enable-trace \ - --otlp-traces-endpoint localhost:4317 -``` - -#### Features - -- OTLP/gRPC exporter (default port 4317) -- W3C Trace Context propagation for HTTP and gRPC -- Batch span processing (500ms delay, 64 span batch size) -- Custom filtering to reduce noise -- Trace context injection into upstream worker requests -- Service name: `sgl-router` - -### Logging - -```bash -python -m sglang_router.launch_router \ - --worker-urls http://worker1:8000 \ - --log-level debug \ - --log-dir ./router_logs -``` - -Structured tracing with optional file sink. Log levels: `debug`, `info`, `warn`, `error`. - -### Request ID Propagation - -```bash ---request-id-headers x-request-id x-trace-id x-correlation-id -``` - -Responses include `x-request-id` header for correlation. - ---- - -## Production Recommendations - -This section provides guidance for deploying SGLang Model Gateway in production environments. - -### Security Best Practices - -**Always enable TLS in production:** - -```bash -python -m sglang_router.launch_router \ - --worker-urls https://worker1:8443 https://worker2:8443 \ - --tls-cert-path /etc/certs/server.crt \ - --tls-key-path /etc/certs/server.key \ - --client-cert-path /etc/certs/client.crt \ - --client-key-path /etc/certs/client.key \ - --ca-cert-path /etc/certs/ca.crt \ - --api-key "${ROUTER_API_KEY}" -``` - -**Security Checklist:** -- Enable TLS for gateway HTTPS termination -- Enable mTLS for worker communication when workers are on untrusted networks -- Set `--api-key` to protect router endpoints -- Use Kubernetes Secrets or a secrets manager for credentials -- Rotate certificates and API keys periodically -- Restrict network access with firewalls or network policies - -### High Availability - -**Scaling Strategy:** - -The gateway supports running multiple replicas behind a load balancer for high availability. However, there are important considerations: - -| Component | Shared Across Replicas | Impact | -|-----------|----------------------|--------| -| Worker Registry | No (independent) | Each replica discovers workers independently | -| Radix Cache Tree | No (independent) | Cache hits may decrease by 10-20% | -| Circuit Breaker State | No (independent) | Each replica tracks failures independently | -| Rate Limiting | No (independent) | Limits apply per-replica, not globally | - -**Recommendations:** - -1. **Prefer horizontal scaling over vertical scaling**: Deploy multiple smaller gateway replicas rather than one large instance with excessive CPU and memory. This provides: - - Better fault tolerance (single replica failure doesn't take down the gateway) - - More predictable resource usage - - Easier capacity planning - -2. **Use Kubernetes Service Discovery**: Let the gateway automatically discover and manage workers: - ```bash - python -m sglang_router.launch_router \ - --service-discovery \ - --selector app=sglang-worker \ - --service-discovery-namespace production - ``` - -3. **Accept cache efficiency trade-off**: With multiple replicas, the cache-aware routing policy's radix tree is not synchronized across replicas. This means: - - Each replica builds its own cache tree - - Requests from the same user may hit different replicas - - Expected cache hit rate reduction: **10-20%** - - This is often acceptable given the HA benefits - -4. **Configure session affinity (optional)**: If cache efficiency is critical, configure your load balancer for session affinity based on a consistent hash of the request (e.g., user ID or API key). - -**Example HA Architecture:** -``` - ┌─────────────────┐ - │ Load Balancer │ - │ (L4/L7) │ - └────────┬────────┘ - ┌──────────────┼──────────────┐ - │ │ │ - ┌─────▼─────┐ ┌─────▼─────┐ ┌─────▼─────┐ - │ Gateway │ │ Gateway │ │ Gateway │ - │ Replica 1 │ │ Replica 2 │ │ Replica 3 │ - └─────┬─────┘ └─────┬─────┘ └─────┬─────┘ - │ │ │ - └──────────────┼──────────────┘ - │ - ┌──────────────┼──────────────┐ - │ │ │ - ┌─────▼─────┐ ┌─────▼─────┐ ┌─────▼─────┐ - │ Worker │ │ Worker │ │ Worker │ - │ Pod 1 │ │ Pod 2 │ │ Pod N │ - └───────────┘ └───────────┘ └───────────┘ -``` - -### Performance - -**Use gRPC mode for high throughput:** - -gRPC mode provides the highest performance for SGLang workers: - -```bash -# Start workers in gRPC mode -python -m sglang.launch_server \ - --model meta-llama/Llama-3.1-8B-Instruct \ - --grpc-mode \ - --port 20000 - -# Configure gateway for gRPC -python -m sglang_router.launch_router \ - --worker-urls grpc://worker1:20000 grpc://worker2:20000 \ - --model-path meta-llama/Llama-3.1-8B-Instruct \ - --policy cache_aware -``` - -**Performance Benefits of gRPC:** -- Native Rust tokenization (no Python overhead) -- Streaming with lower latency -- Built-in reasoning parser execution -- Tool call parsing in the gateway -- Reduced serialization overhead - -**Tuning Recommendations:** - -| Parameter | Recommendation | Reason | -|-----------|---------------|--------| -| `--policy` | `cache_aware` | Best for repeated prompts, ~30% latency reduction | -| `--max-concurrent-requests` | 2-4x worker count | Prevent overload while maximizing throughput | -| `--queue-size` | 2x max-concurrent | Buffer for burst traffic | -| `--request-timeout-secs` | Based on max generation length | Prevent stuck requests | - -### Kubernetes Deployment - -**Pod Labeling for Service Discovery:** - -For the gateway to discover workers automatically, label your worker pods consistently: - -```yaml -# Worker Deployment (Regular Mode) -apiVersion: apps/v1 -kind: Deployment -metadata: - name: sglang-worker - namespace: production -spec: - replicas: 4 - selector: - matchLabels: - app: sglang-worker - component: inference - template: - metadata: - labels: - app: sglang-worker - component: inference - model: llama-3-8b - spec: - containers: - - name: worker - image: lmsysorg/sglang:latest - ports: - - containerPort: 8000 - name: http - - containerPort: 20000 - name: grpc -``` - -**Gateway configuration for discovery:** -```bash -python -m sglang_router.launch_router \ - --service-discovery \ - --selector app=sglang-worker component=inference \ - --service-discovery-namespace production \ - --service-discovery-port 8000 -``` - -**PD (Prefill/Decode) Mode Labeling:** - -```yaml -# Prefill Worker -metadata: - labels: - app: sglang-worker - component: prefill - annotations: - sglang.ai/bootstrap-port: "9001" - -# Decode Worker -metadata: - labels: - app: sglang-worker - component: decode -``` - -**Gateway configuration for PD discovery:** -```bash -python -m sglang_router.launch_router \ - --service-discovery \ - --pd-disaggregation \ - --prefill-selector app=sglang-worker component=prefill \ - --decode-selector app=sglang-worker component=decode \ - --service-discovery-namespace production -``` - -**RBAC Requirements:** - -The gateway needs permissions to watch pods: - -```yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: sglang-gateway - namespace: production -rules: -- apiGroups: [""] - resources: ["pods"] - verbs: ["get", "list", "watch"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: sglang-gateway - namespace: production -subjects: -- kind: ServiceAccount - name: sglang-gateway - namespace: production -roleRef: - kind: Role - name: sglang-gateway - apiGroup: rbac.authorization.k8s.io -``` - -### Monitoring with PromQL - -Configure Prometheus to scrape the gateway metrics endpoint (default: `:29000/metrics`). - -**Essential Dashboards:** - -**1. Request Rate and Latency:** -```promql -# Request rate by endpoint -sum(rate(smg_http_requests_total[5m])) by (path, method) - -# P50 latency -histogram_quantile(0.50, sum(rate(smg_http_request_duration_seconds_bucket[5m])) by (le)) - -# P99 latency -histogram_quantile(0.99, sum(rate(smg_http_request_duration_seconds_bucket[5m])) by (le)) - -# Error rate -sum(rate(smg_http_responses_total{status=~"5.."}[5m])) / sum(rate(smg_http_responses_total[5m])) -``` - -**2. Worker Health:** -```promql -# Healthy workers -sum(smg_worker_pool_size) - -# Active connections per worker -smg_worker_connections_active - -# Worker health check failures -sum(rate(smg_worker_health_checks_total{result="failure"}[5m])) by (worker_id) -``` - -**3. Circuit Breaker Status:** -```promql -# Circuit breaker states (0=closed, 1=open, 2=half-open) -smg_worker_cb_state - -# Circuit breaker transitions -sum(rate(smg_worker_cb_transitions_total[5m])) by (worker_id, from_state, to_state) - -# Workers with open circuits -count(smg_worker_cb_state == 1) -``` - -**4. Inference Performance (gRPC mode):** -```promql -# Time to first token (P50) -histogram_quantile(0.50, sum(rate(smg_router_ttft_seconds_bucket[5m])) by (le, model)) - -# Time per output token (P99) -histogram_quantile(0.99, sum(rate(smg_router_tpot_seconds_bucket[5m])) by (le, model)) - -# Token throughput -sum(rate(smg_router_tokens_total[5m])) by (model, direction) - -# Generation duration P95 -histogram_quantile(0.95, sum(rate(smg_router_generation_duration_seconds_bucket[5m])) by (le)) -``` - -**5. Rate Limiting and Queuing:** -```promql -# Rate limit rejections -sum(rate(smg_http_rate_limit_total{decision="rejected"}[5m])) - -# Queue depth (if using concurrency limiting) -smg_worker_requests_active - -# Retry attempts -sum(rate(smg_worker_retries_total[5m])) by (worker_id) - -# Exhausted retries (failures after all retries) -sum(rate(smg_worker_retries_exhausted_total[5m])) -``` - -**6. MCP Tool Execution:** -```promql -# Tool call rate -sum(rate(smg_mcp_tool_calls_total[5m])) by (server, tool) - -# Tool latency P95 -histogram_quantile(0.95, sum(rate(smg_mcp_tool_duration_seconds_bucket[5m])) by (le, tool)) - -# Active MCP server connections -smg_mcp_servers_active -``` - -**Alerting Rules Example:** - -```yaml -groups: -- name: sglang-gateway - rules: - - alert: HighErrorRate - expr: | - sum(rate(smg_http_responses_total{status=~"5.."}[5m])) - / sum(rate(smg_http_responses_total[5m])) > 0.05 - for: 5m - labels: - severity: critical - annotations: - summary: "High error rate on SGLang Gateway" - - - alert: CircuitBreakerOpen - expr: count(smg_worker_cb_state == 1) > 0 - for: 2m - labels: - severity: warning - annotations: - summary: "Worker circuit breaker is open" - - - alert: HighLatency - expr: | - histogram_quantile(0.99, sum(rate(smg_http_request_duration_seconds_bucket[5m])) by (le)) > 30 - for: 5m - labels: - severity: warning - annotations: - summary: "P99 latency exceeds 30 seconds" - - - alert: NoHealthyWorkers - expr: sum(smg_worker_pool_size) == 0 - for: 1m - labels: - severity: critical - annotations: - summary: "No healthy workers available" -``` - ---- - -## Configuration Reference - -### Core Settings - -| Parameter | Type | Default | Description | -|-----------|------|---------|-------------| -| `--host` | str | 127.0.0.1 | Router host | -| `--port` | int | 30000 | Router port | -| `--worker-urls` | list | [] | Worker URLs (HTTP or gRPC) | -| `--policy` | str | cache_aware | Routing policy | -| `--max-concurrent-requests` | int | -1 | Concurrency limit (-1 disables) | -| `--request-timeout-secs` | int | 600 | Request timeout | -| `--max-payload-size` | int | 512MB | Maximum request payload | - -### Prefill/Decode - -| Parameter | Type | Default | Description | -|-----------|------|---------|-------------| -| `--pd-disaggregation` | flag | false | Enable PD mode | -| `--prefill` | list | [] | Prefill URLs + optional bootstrap ports | -| `--decode` | list | [] | Decode URLs | -| `--prefill-policy` | str | None | Override policy for prefill nodes | -| `--decode-policy` | str | None | Override policy for decode nodes | -| `--worker-startup-timeout-secs` | int | 600 | Worker init timeout | - -### Kubernetes Discovery - -| Parameter | Type | Description | -|-----------|------|-------------| -| `--service-discovery` | flag | Enable discovery | -| `--selector` | list | Label selectors (key=value) | -| `--prefill-selector` / `--decode-selector` | list | PD mode selectors | -| `--service-discovery-namespace` | str | Namespace to watch | -| `--service-discovery-port` | int | Worker port (default 80) | -| `--bootstrap-port-annotation` | str | Annotation for bootstrap ports | - -### TLS Configuration - -| Parameter | Type | Description | -|-----------|------|-------------| -| `--tls-cert-path` | str | Server certificate for gateway HTTPS (PEM) | -| `--tls-key-path` | str | Server private key for gateway HTTPS (PEM) | -| `--client-cert-path` | str | Client certificate for worker mTLS (PEM) | -| `--client-key-path` | str | Client private key for worker mTLS (PEM) | -| `--ca-cert-path` | str | CA certificate for verifying workers (PEM, repeatable) | - ---- - -## Troubleshooting - -### Workers Never Ready - -Increase `--worker-startup-timeout-secs` or ensure health probes respond before router startup. - -### Load Imbalance / Hot Workers - -Inspect `smg_router_requests_total` by worker and tune cache-aware thresholds (`--balance-*`, `--cache-threshold`). - -### Circuit Breaker Flapping - -Increase `--cb-failure-threshold` or extend the timeout/window durations. Consider temporarily disabling retries. - -### Queue Overflow (429) - -Increase `--queue-size` or reduce client concurrency. Ensure `--max-concurrent-requests` matches downstream capacity. - -### Memory Growth - -Reduce `--max-tree-size` or lower `--eviction-interval-secs` for more aggressive cache pruning. - -### Debugging - -```bash -python -m sglang_router.launch_router \ - --worker-urls http://worker1:8000 \ - --log-level debug \ - --log-dir ./router_logs -``` - -### gRPC Connection Issues - -Ensure workers are started with `--grpc-mode` and verify `--model-path` or `--tokenizer-path` is provided to the router. - -### Tokenizer Loading Failures - -Check HuggingFace Hub credentials (`HF_TOKEN` environment variable) for private models. Verify local paths are accessible. - ---- - -SGLang Model Gateway continues to evolve alongside the SGLang runtime. Keep CLI flags, integrations, and documentation aligned when adopting new features or contributing improvements. diff --git a/docs/advanced_features/sglang_for_rl.md b/docs/advanced_features/sglang_for_rl.md deleted file mode 100644 index 12eb41540..000000000 --- a/docs/advanced_features/sglang_for_rl.md +++ /dev/null @@ -1,271 +0,0 @@ -# SGLang for RL Systems - -This document is a practical guide for infrastructure teams integrating SGLang into RL and post-training systems. It focuses on the operational pain points in the loop (rollout, evaluation, training, weight sync) and maps them to concrete SGLang APIs, flags, and integration patterns. The focus is on maximizing rollout efficiency, accuracy and stability while keeping rollout-serving behavior aligned in production environments. - -## Why SGLang for RL Lifecycle? - -Let's embrace a guiding principle from early DeepMind's RL engineering: - -**Be a library, not a framework.** - -This philosophy empowers innovation by providing SGLang as flexible tools, not rigid structures. Here are five reasons to use SGLang for your RL lifecycle: - -* **Fine-Grained Engine Sleep and Wake Up**: facilitate maximum-powered rollout and training -* **Open-To-Use Refit Functionality**: diverse methods for co-location or disaggregation -* **Easy To Postpone Generation**: enable partial rollout and dedicated rollout control -* **Deterministic Inference**: achieve deterministic inference to enable zero training-inference mismatch -* **Load Balancing Router**: cache-aware load-balancing for high-throughput rollout - -The following sections cover these aspects in detail. - -## Fine-Grained Engine Sleep and Wake Up - -Rollout and training are both memory-intensive, and co-locating them on the same GPUs often leads to memory pressure and slow handoffs. SGLang provides a memory-aware sleep/wake mechanism that releases KV cache and weights while keeping the server process alive, then resumes them for rollout without a full restart. This avoids repeated disk I/O and CUDA graph recapture during each RL step. - -Under the hood, the RL team uses CUDA-graph-aware weight offload via [torch_memory_saver](https://github.com/fzyzcjy/torch_memory_saver) to preserve virtual memory addresses for graph replay. For details, see: [Efficient RL Training - Optimizing Memory Usage in verl](https://hebiao064.github.io/rl-memory-management). - -### Server flag - -Enable memory saver support when launching the server: - -``` ---enable-memory-saver -``` - -### Release Memory - -**Endpoint:** `POST /release_memory_occupation` - -**Request body:** - -| Field | Description | Defaults | Options | -| --- | --- | --- | --- | -| `tags` | Which memory regions to release. If omitted, all are released. | `None` | Type: list[str], values: `kv_cache`, `weights` | - -**Behavior notes:** - -- This call asserts there are no ongoing requests. Ensure the engine is idle before calling it. -- If `kv_cache` is released, SGLang flushes cache; subsequent requests will rebuild KV cache as needed. - -### Resume Memory - -**Endpoint:** `POST /resume_memory_occupation` - -**Request body:** - -| Field | Description | Defaults | Options | -| --- | --- | --- | --- | -| `tags` | Which memory regions to resume. If omitted, all are resumed. | `None` | Type: list[str], values: `kv_cache`, `weights` | - - -## Open-To-Use Refit Functionality - -After training completes each step, rollout engines must be refit with new weights. SGLang supports three refit strategies so you can match your infrastructure style (co-located vs disaggregated) and scaling needs. Each strategy maps to a concrete API with clear request schemas. For a deeper dive into SGLang's weight update utilities, see [RL System Deep Thinking: Weight Update Mechanisms](https://github.com/zhaochenyang20/Awesome-ML-SYS-Tutorial/blob/main/rlhf/sys-design/readme-1-EN.md). - -**How to choose:** - -- **From disk** is simplest and best for elastic rollout scaling and checkpointing. -- **From tensor** is best for co-located training/rollout when you can pass in-memory tensors. -- **From distributed** is best for disaggregated training/rollout with dedicated communication groups (NCCL/IB). - -### Update Weights from Disk - -**When to use:** - -- Save checkpoint to disk and update weights from disk -- Dynamic scaling (new rollout instances can load from the same checkpoint) - -**Why it works well:** - -This path trades some I/O overhead for simplicity and flexibility. It integrates naturally with checkpointing and makes it trivial to add new rollout engines: point them at the same checkpoint and call the API. It is also the safest option for high availability because the checkpoint itself is the source of truth. - -**Endpoint:** `POST /update_weights_from_disk` - -**Request body:** - -| Field | Description | Defaults | Options | -| --- | --- | --- | --- | -| `model_path` | The model path with the new weights. | Required | Type: str | -| `load_format` | The format to load the weights. | `None` | Type: str | -| `abort_all_requests` | Abort all running requests before update. | `False` | Type: bool | -| `weight_version` | Optional weight version label tracked by the server. | `None` | Type: str | -| `is_async` | Perform weight load asynchronously. | `False` | Type: bool | -| `torch_empty_cache` | Empty torch cache. | `False` | Type: bool | -| `keep_pause` | Keep scheduler paused after update. | `False` | Type: bool | -| `recapture_cuda_graph` | Recapture CUDA graphs after update. | `False` | Type: bool | -| `token_step` | Trainer step id for rollout bookkeeping. | `0` | Type: int | -| `flush_cache` | Flush KV cache after update. | `True` | Type: bool | - -**Response body:** - -| Field | Description | Defaults | Options | -| --- | --- | --- | --- | -| `success` | Whether the update succeeded. | - | Type: bool | -| `message` | Status / error message. | - | Type: str | -| `num_paused_requests` | Number of paused requests during update. | `0` | Type: int | - -**Python Engine API:** `engine.update_weights_from_disk(model_path, load_format=None)` - -**Diffusion engine (SGLang-Diffusion):** The diffusion engine exposes the same `POST /update_weights_from_disk` endpoint with the following behavior: - -- **All-or-nothing with rollback:** if any module fails to load, all previously updated modules are rolled back to the original weights by reloading from the original model path. No partial updates are left behind. If rollback itself fails, the exception propagates so the caller knows the model is in an inconsistent state. -- **Offload-aware:** when layerwise offload (`--dit-layerwise-offload`) is enabled, the diffusion offload manager replaces GPU parameters with small `torch.empty((1,))` placeholders while real weights live in consolidated pinned CPU buffers. A naive `param.data.copy_()` would fail with a shape mismatch. Instead, the updater dynamically detects active offload managers and writes new weights directly into their CPU buffers, bypassing the placeholders entirely. For any layer that happens to be prefetched on GPU at update time, the live GPU tensor is also updated so the change takes effect immediately. This requires no extra GPU memory and does not disturb the offload state. -- **DTensor-aware:** parameters distributed via `torch.distributed.tensor` (tensor parallelism) are updated through `distribute_tensor` so that each shard is correctly placed on the right device mesh. - -**Request body:** - -| Field | Description | Defaults | Options | -| --- | --- | --- | --- | -| `model_path` | The model path with the new weights. | Required | Type: str | -| `flush_cache` | Flush TeaCache state after update. | `True` | Type: bool | -| `target_modules` | List of module names to update (e.g. `["transformer"]`). If omitted, all `nn.Module` components are updated. | `None` | Type: list[str] | - -**Response body:** - -| Field | Description | Defaults | Options | -| --- | --- | --- | --- | -| `success` | Whether the update succeeded. | - | Type: bool | -| `message` | Status / error message. | - | Type: str | - -> **Note:** The diffusion engine (SGLang-Diffusion) does not currently support hot refit (updating weights while inference is in progress). The diffusion scheduler processes one request at a time and completes the entire inference before handling the next request, so weight updates and inference never run concurrently. - -### Update Weights from Tensor - -**When to use:** - -- Co-located training and rollout, where training can provide tensors directly -- Fast in-memory updates - -**Important constraints:** - -This strategy requires the training process and rollout engine to share access to the tensors. Co-located setups must keep the model on GPU; moving tensors to CPU will break the update path. For high-performance MoE or specialized attention kernels, co-location may limit some optimizations compared to disaggregated rollouts. - -**Endpoint:** `POST /update_weights_from_tensor` - -**Request body:** - -| Field | Description | Defaults | Options | -| --- | --- | --- | --- | -| `serialized_named_tensors` | Per-TP serialized tensor payloads. | Required | Type: list[str|bytes] | -| `load_format` | Optional load format selector. | `None` | `None`, `direct`, `flattened_bucket`, or a custom loader path string | -| `flush_cache` | Flush KV cache after update. | `True` | Type: bool | -| `abort_all_requests` | Abort all running requests before update. | `False` | Type: bool | -| `weight_version` | Optional version label tracked by the server. | `None` | Type: str | - -**Note:** The serialized tensor payloads must be created with `MultiprocessingSerializer.serialize(...)` and should be base64-safe strings. - -**Python Engine API:** `engine.update_weights_from_tensor(named_tensors, load_format=None, flush_cache=True)` - -### Update Weights from Distributed Group - -**When to use:** - -- Disaggregated training and rollout -- NCCL or IB-backed weight broadcast from training workers to rollout workers - -**How it works:** - -Training workers gather weights (typically on TP rank 0), broadcast them to the rollout group, and each rollout TP shard loads the parameters it needs. This avoids disk I/O and keeps training and rollout decoupled, at the cost of managing a dedicated communication group. - -**Initialize weight update group** - -**Endpoint:** `POST /init_weights_update_group` - -**Request body:** - -| Field | Description | Defaults | Options | -| --- | --- | --- | --- | -| `master_address` | Group master address. | Required | Type: str | -| `master_port` | Group master port. | Required | Type: int | -| `rank_offset` | Offset for local rank mapping. | Required | Type: int | -| `world_size` | Total world size. | Required | Type: int | -| `group_name` | Group name. | `weight_update_group` | Type: str | -| `backend` | Communication backend. | `nccl` | Type: str | - -**Update weight** - -**Endpoint:** `POST /update_weights_from_distributed` - -**Request body:** - -| Field | Description | Defaults | Options | -| --- | --- | --- | --- | -| `names` | Parameter names to update. | Required | Type: list[str] | -| `dtypes` | Dtype strings for each parameter. | Required | Type: list[str] | -| `shapes` | Tensor shapes. | Required | Type: list[list[int]] | -| `group_name` | Group name. | `weight_update_group` | Type: str | -| `flush_cache` | Flush KV cache after update. | `True` | Type: bool | -| `abort_all_requests` | Abort all running requests before update. | `False` | Type: bool | -| `weight_version` | Optional version label. | `None` | Type: str | -| `load_format` | Optional format selector. | `None` | `None` or `flattened_bucket` | - -**Destroy weights update group** - -**Endpoint:** `POST /destroy_weights_update_group` - -**Request body:** - -| Field | Description | Defaults | Options | -| --- | --- | --- | --- | -| `group_name` | Group name. | `weight_update_group` | Type: str | - -**Python Engine APIs:** - -- `engine.init_weights_update_group(...)` -- `engine.update_weights_from_distributed(names, dtypes, shapes, ...)` -- `engine.destroy_weights_update_group(group_name)` - -## Easy To Postpone Generation - -Multi-turn RL rollouts often suffer from long-tail requests that block the entire batch. A small number of slow interactions can stall all GPUs, and the long-tail behavior makes profiling and monitoring difficult. - -SGLang exposes explicit pause/resume APIs so you can pause slow requests and continue them later. This pattern matches systems like [APRIL](https://arxiv.org/abs/2509.18521), terminate once enough responses are collected, and recycle incomplete responses in the next step. The result is higher GPU utilization without discarding partial work. - -`pause_generation` --- update weights --- `continue_generation` is the correct execution flow when updating weights from training. An update can only happen when SGLang is not actively processing inference tasks. - -### Pause Generation - -**Endpoint:** `POST /pause_generation` - -**Request body:** - -| Field | Description | Defaults | Options | -| --- | --- | --- | --- | -| `mode` | Pause mode. | `abort` | `abort`, `retract`, `in_place` | - -**Modes:** - -- `abort`: Default behavior, identical to `abort` endpoint with `abort_all` set. Pending requests from `waiting_queue` and `running_queue` will be returned immediately to the caller. -- `retract`: Put engine in "paused" state. Move running requests back to waiting queue. KV cache can be flushed and recomputed later. -- `in_place`: Put engine in "paused" state without changing states of the requests. Running requests rely on availability of KV caches to continue, so any subsequent `flush_cache` call will be unsuccessful. - -### Continue Generation - -**Endpoint:** `POST /continue_generation` - -## Deterministic Inference - -In many RL stacks, rollout and training are implemented with different kernels or batching behavior. Even when weights are identical, token probabilities can drift, silently breaking the on-policy assumption. This is the training–inference mismatch problem. - -SGLang supports a deterministic inference mode that reduces non-determinism across batch shapes. This mitigates variance introduced by runtime batching and kernel selection. To further achieve true on-policy training, you need to modify the training engine to use the same deterministic kernels. For implementation details, see these miles examples: [True On-Policy](https://github.com/radixark/miles/tree/main/examples/true_on_policy) and [True On-Policy for VLM](https://github.com/radixark/miles/tree/main/examples/true_on_policy_vlm). For additional context, see the blog post [Let Speed Be With Stability: All-In-One Solution to Training-Inference Mismatch with Miles](https://github.com/zhaochenyang20/Awesome-ML-SYS-Tutorial/blob/main/rlhf/slime/mismatch/blog-en.md). - -**Server flag:** - -``` ---enable-deterministic-inference -``` - -For more details, see [Deterministic Inference](deterministic_inference.md) - -## Load Balancing Router - -SGLang Model Gateway is the recommended control plane for large‑scale RL rollouts. It provides async, non‑blocking request handling, cache‑aware load balancing, and fault‑tolerant routing across rollout and reward servers. This lets you keep GPUs saturated while avoiding long‑tail stalls and brittle, engine‑local concurrency logic. It has been deployed in the training of GLM 4.5+ models and proven to be highly efficient in production-level large-scale RL workloads. - -Key benefits for RL infrastructure: - -- **Async non-blocking efficiency**: SGLang’s native async server/router architecture (HTTPS/gRPC) manages concurrency automatically. This guarantees maximum GPU saturation and effective continuous batching without requiring complex, manual implementation by engineers. -- **Elasticity and fault tolerance**: By encapsulating the reward model and rollout as independent servers, SGLang decouples them logically and physically. This architecture provides robust disaster recovery for large-scale distributed training; if a server fails, the router automatically redirects traffic to healthy nodes, ensuring the training process continues without interruption. -- **Training–Inference alignment**: Using the SGLang Model Gateway for both training and inference ensures "What You See Is What You Get." This eliminates score discrepancies and the painful backend alignment issues often caused by using different engines for training versus deployment. -- **Dynamic load balancing and long-tail mitigation**: Unlike static partitioning, the SGLang Model Gateway enables request-level dynamic dispatching for multi-turn RL. It can distribute different turns of a conversation across different servers to balance workloads and eliminate long-tail latency caused by varying sequence lengths. - -For deployment and configuration, see: [SGLang Model Gateway](sgl_model_gateway.md) diff --git a/docs/advanced_features/speculative_decoding.md b/docs/advanced_features/speculative_decoding.md deleted file mode 100644 index 8acaf4fcf..000000000 --- a/docs/advanced_features/speculative_decoding.md +++ /dev/null @@ -1,565 +0,0 @@ -# Speculative Decoding - -SGLang provides several speculative decoding options, including EAGLE-2/EAGLE-3, MTP, classic draft-model decoding, and an NGRAM-based variant. Our implementation aims to maximize speed and efficiency and is considered to be among the fastest in open-source LLM engines. - -## Summary - -### Jump to sections - -- [EAGLE Decoding](#eagle-decoding) - - [EAGLE-2 Decoding](#eagle-2-decoding) - - [EAGLE-2 Decoding with torch.compile](#eagle-2-decoding-with-torchcompile) - - [EAGLE-2 Decoding via Frequency-Ranked Speculative Sampling](#eagle-2-decoding-via-frequency-ranked-speculative-sampling) - - [EAGLE-3 Decoding](#eagle-3-decoding) -- [Multi Token Prediction](#multi-token-prediction) -- [Standalone Speculative Decoding (Small Draft Model)](#standalone-speculative-decoding-small-draft-model) -- [Speculative Decoding V2 (Overlap Scheduler)](#speculative-decoding-v2-overlap-scheduler) -- [Ngram Speculative Decoding](#ngram-speculative-decoding) -- [Full Parameter Reference](#full-parameter-reference) -- [OOM Troubleshooting](#oom-troubleshooting) -- [References](#references) - -### Quick guidance - -- **Best speed/quality (recommended)**: Use **EAGLE-3** with `--speculative-algorithm EAGLE3`. -- **Strong default / broad compatibility**: Use **EAGLE-2** with `--speculative-algorithm EAGLE`. -- **Workload acceptance changes over time**: Use [**Adaptive speculative decoding**](adaptive_speculative_decoding.md) on top of **EAGLE** with `--speculative-eagle-topk 1`. -- **Lower `lm_head` overhead for EAGLE-2**: Enable **FR-Spec** with `--speculative-token-map`. -- **Model is MTP-enabled**: Use **MTP via speculative decoding** (often with small `speculative_num_steps/topk/num_draft_tokens`, see the example section). -- **You have a smaller draft LLM**: Use **STANDALONE** (`--speculative-algorithm STANDALONE`). -- **No extra model available**: Use **NGRAM** (`--speculative-algorithm NGRAM`, CUDA-only). -- **Want overlap scheduler (experimental)**: Enable **SpecV2** with `SGLANG_ENABLE_SPEC_V2=True` (requires `--speculative-eagle-topk 1`). - -### Method comparison (mini table) - -| Method | Draft source | Separate draft model? | How to enable | Notes / constraints | -|---|---|---:|---|---| -| EAGLE-2 | EAGLE draft model (feature drafting + tree) | Typically yes | `--speculative-algorithm EAGLE` + `--speculative-draft-model-path ...` | Tune `--speculative-num-steps`, `--speculative-eagle-topk`, `--speculative-num-draft-tokens` | -| EAGLE-2 + `torch.compile` | Same as EAGLE-2 | Typically yes | Add `--enable-torch-compile` (optionally `--torch-compile-max-bs`) | Benefit varies by hardware/model; benchmark to verify | -| EAGLE-2 + FR-Spec | Same as EAGLE-2 + token subset | Typically yes | Add `--speculative-token-map ...` | Reduces `lm_head` overhead with high-frequency token vocab | -| EAGLE-3 | EAGLE3 draft model | Yes | `--speculative-algorithm EAGLE3` + `--speculative-draft-model-path ...` | Best throughput in the benchmark below | -| MTP | Built-in multi-token heads (model-specific) | Often no | See **Multi Token Prediction** section | Uses speculative workflow; draft path may be auto-handled for some models | -| STANDALONE | Smaller draft LLM (token-level) | Yes | `--speculative-algorithm STANDALONE` + `--speculative-draft-model-path ...` | Does **not** support `--enable-dp-attention` | -| SpecV2 (experimental) | V2 workers + overlap scheduler | N/A | `SGLANG_ENABLE_SPEC_V2=True` | Only supports `--speculative-eagle-topk 1`; applies to `EAGLE`, `EAGLE3`, `STANDALONE` | -| NGRAM | Ngram cache from previous tokens | No | `--speculative-algorithm NGRAM` | CUDA-only; no `--enable-dp-attention`; disables overlap scheduler & mixed chunked prefill | - -### Performance Highlights - -Please see below for the huge improvements on throughput for LLaMA-Instruct 3.1 8B tested on MT bench that can be achieved via EAGLE3 decoding. -For further details please see the [EAGLE3 paper](https://arxiv.org/pdf/2503.01840). - -| Method | Throughput (tokens/s) | -|--------|----------------| -| SGLang (w/o speculative, 1x H100) | 158.34 tokens/s | -| SGLang + EAGLE-2 (1x H100) | 244.10 tokens/s | -| SGLang + EAGLE-3 (1x H100) | 373.25 tokens/s | - ---- - -## EAGLE Decoding - -To enable EAGLE speculative decoding the following parameters are relevant: - -| Parameter | Description | Default | -|---|---|---| -| `--speculative-draft-model-path` | Draft model path/weights. **Typically required** for EAGLE/EAGLE3 and STANDALONE. For some MTP-enabled models, this can be omitted. | `None` | -| `--speculative-num-steps` | Depth of autoregressive drafting. Increases speculation range but risks rejection cascades. | Auto (`5` for Llama/Grok; `3` for many other models) | -| `--speculative-eagle-topk` | Branching factor per step. Improves candidate diversity and acceptance rate, but increases memory/compute consumption. | Auto (`4` for Llama/Grok; `1` for many other models) | -| `--speculative-num-draft-tokens` | Maximum parallel verification capacity. Allows deeper tree evaluation but increases GPU memory usage. | Auto (`8` for Llama/Grok; `4` for many other models). If `topk=1`, it is adjusted to `num_steps + 1`. | -| `--speculative-accept-threshold-single` | Acceptance threshold for single-token verification. Lower values accept more aggressively. | `1.0` | -| `--speculative-accept-threshold-acc` | Accumulated acceptance threshold across steps. | `1.0` | -| `--speculative-attention-mode` | Attention mode for speculative operations (`prefill` or `decode`), affecting both target verification and draft extension. | `"prefill"` | -| `--speculative-draft-attention-backend` | Override attention backend for the draft model. | `None` (same as target) | -| `--speculative-draft-model-quantization` | Quantization method for the draft model. Use `"unquant"` to force no quantization even when the target model is quantized. | Same as target model | -| `--speculative-draft-model-revision` | Specific revision/commit of the draft model to load. | `None` (auto-set to `"main"` when `--speculative-draft-model-path` is set and revision is omitted) | -| `--speculative-draft-load-format` | Load format for the draft model weights. | `None` | - -These parameters are mostly the same for EAGLE-2 and EAGLE-3. `--speculative-token-map` is ignored for EAGLE-3 models. -For `--speculative-num-steps`, `--speculative-eagle-topk`, and `--speculative-num-draft-tokens`: leave all three unset to use auto-tuning, or set all three explicitly when tuning. -If you use EAGLE with `--speculative-eagle-topk 1` and your acceptance rate varies across requests, see [Adaptive Speculative Decoding](adaptive_speculative_decoding.md). - -You can find the best combinations of these parameters with [bench_speculative.py](https://github.com/sgl-project/sglang/blob/main/scripts/playground/bench_speculative.py). - - -### EAGLE-2 Decoding - -You can enable EAGLE-2 Decoding by setting `--speculative-algorithm EAGLE` and choosing an appropriate model. - -**Launch the server:** - -```bash -python3 -m sglang.launch_server \ - --model meta-llama/Llama-2-7b-chat-hf \ - --speculative-algorithm EAGLE \ - --speculative-draft-model-path lmsys/sglang-EAGLE-llama2-chat-7B \ - --speculative-num-steps 3 \ - --speculative-eagle-topk 4 \ - --speculative-num-draft-tokens 16 \ - --mem-fraction-static 0.7 \ - --cuda-graph-max-bs 8 \ - --log-level warning -``` - -**Send a request:** - -```python -import openai - -client = openai.Client(base_url="http://127.0.0.1:30000/v1", api_key="None") - -response = client.chat.completions.create( - model="meta-llama/Llama-2-7b-chat-hf", - messages=[ - {"role": "user", "content": "List 3 countries and their capitals."}, - ], - temperature=0, - max_tokens=64, -) - -print(response.choices[0].message.content) -``` - ---- - -### EAGLE-2 Decoding with `torch.compile` - -You can optionally enable `torch.compile` to apply kernel-level optimizations (operator fusion, autotune) to the draft model. The actual speedup depends on your hardware, model architecture, and batch size. In some configurations (e.g., small draft models on H100 where cuBLAS is already optimal and CUDA graphs are enabled), the benefit may be negligible. We recommend benchmarking with and without this flag on your specific setup to verify whether it helps. - -To enable it, add `--enable-torch-compile` and optionally set `--torch-compile-max-bs`: - -```bash -python3 -m sglang.launch_server \ - --model meta-llama/Llama-2-7b-chat-hf \ - --speculative-algorithm EAGLE \ - --speculative-draft-model-path lmsys/sglang-EAGLE-llama2-chat-7B \ - --speculative-num-steps 3 \ - --speculative-eagle-topk 4 \ - --speculative-num-draft-tokens 16 \ - --mem-fraction-static 0.7 \ - --enable-torch-compile \ - --torch-compile-max-bs 8 \ - --log-level warning -``` - -**Send a request:** - -```python -import openai - -client = openai.Client(base_url="http://127.0.0.1:30000/v1", api_key="None") - -response = client.chat.completions.create( - model="meta-llama/Llama-2-7b-chat-hf", - messages=[ - {"role": "user", "content": "List 3 countries and their capitals."}, - ], - temperature=0, - max_tokens=64, -) - -print(response.choices[0].message.content) -``` - ---- - -### EAGLE-2 Decoding via Frequency-Ranked Speculative Sampling - -By employing a truncated high-frequency token vocabulary in the draft model, EAGLE speculative decoding reduces `lm_head` computational overhead while accelerating the pipeline without quality degradation. For more details, check out [the paper](https://arxiv.org/pdf/2502.14856). - -In our implementation, set `--speculative-token-map` to enable the optimization. You can get the high-frequency tokens in FR-Spec from [this model](https://huggingface.co/thunlp/LLaMA3-Instruct-8B-FR-Spec). Or you can obtain high-frequency tokens by directly downloading these tokens from [this repo](https://github.com/thunlp/FR-Spec/tree/main?tab=readme-ov-file#prepare-fr-spec-vocabulary-subset). - -Thanks for the contribution from [Weilin Zhao](https://github.com/Achazwl) and [Zhousx](https://github.com/Zhou-sx). - -```bash -python3 -m sglang.launch_server \ - --model meta-llama/Meta-Llama-3-8B-Instruct \ - --speculative-algorithm EAGLE \ - --speculative-draft-model-path lmsys/sglang-EAGLE-LLaMA3-Instruct-8B \ - --speculative-num-steps 3 \ - --speculative-eagle-topk 4 \ - --speculative-num-draft-tokens 16 \ - --speculative-token-map thunlp/LLaMA3-Instruct-8B-FR-Spec/freq_32768.pt \ - --mem-fraction-static 0.7 \ - --cuda-graph-max-bs 8 \ - --dtype float16 \ - --log-level warning -``` - -**Send a request:** - -```python -import openai - -client = openai.Client(base_url="http://127.0.0.1:30000/v1", api_key="None") - -response = client.chat.completions.create( - model="meta-llama/Meta-Llama-3-8B-Instruct", - messages=[ - {"role": "user", "content": "List 3 countries and their capitals."}, - ], - temperature=0, - max_tokens=64, -) - -print(response.choices[0].message.content) -``` - ---- - -### EAGLE-3 Decoding - -You can enable EAGLE-3 decoding by setting `--speculative-algorithm EAGLE3` and choosing an appropriate model. - -```bash -python3 -m sglang.launch_server \ - --model meta-llama/Meta-Llama-3.1-8B-Instruct \ - --speculative-algorithm EAGLE3 \ - --speculative-draft-model-path jamesliu1/sglang-EAGLE3-Llama-3.1-Instruct-8B \ - --speculative-num-steps 3 \ - --speculative-eagle-topk 4 \ - --speculative-num-draft-tokens 16 \ - --mem-fraction-static 0.7 \ - --cuda-graph-max-bs 8 \ - --dtype float16 \ - --log-level warning -``` - -**Send a request:** - -```python -import openai - -client = openai.Client(base_url="http://127.0.0.1:30000/v1", api_key="None") - -response = client.chat.completions.create( - model="meta-llama/Meta-Llama-3.1-8B-Instruct", - messages=[ - {"role": "user", "content": "List 3 countries and their capitals."}, - ], - temperature=0, - max_tokens=64, -) - -print(response.choices[0].message.content) -``` - ---- - -## Multi Token Prediction - -We support [MTP (Multi-Token Prediction)](https://arxiv.org/pdf/2404.19737) in SGLang by using speculative decoding. We use `XiaomiMiMo/MiMo-7B-RL` as an example here (for DeepSeek MTP usage, refer to [deepseek_v32 doc](../basic_usage/deepseek_v32.md#multi-token-prediction)). - -```bash -python3 -m sglang.launch_server \ - --model XiaomiMiMo/MiMo-7B-RL \ - --host 0.0.0.0 \ - --trust-remote-code \ - --speculative-algorithm EAGLE \ - --speculative-num-steps 1 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 2 \ - --mem-fraction-static 0.7 \ - --cuda-graph-max-bs 8 \ - --log-level warning -``` - -**Send a request:** - -```python -import requests - -url = "http://localhost:30000/v1/chat/completions" - -data = { - "model": "XiaomiMiMo/MiMo-7B-RL", - "messages": [{"role": "user", "content": "What is the capital of France?"}], -} - -response = requests.post(url, json=data) -print(response.json()) -``` - ---- - -## Standalone Speculative Decoding (Small Draft Model) - -Besides EAGLE/MTP, SGLang also supports **token-level speculative decoding** using a smaller **draft model**. Enable it with `--speculative-algorithm STANDALONE` and provide a draft model via `--speculative-draft-model-path`. - -Relevant parameters: - -| Parameter | Description | Default | -|---|---|---| -| `--speculative-draft-model-path` | Draft model weights (smaller than the target model). | `None` | -| `--speculative-num-steps` | Draft depth (how many steps the draft model runs autoregressively). | `3` (auto default for STANDALONE) | -| `--speculative-eagle-topk` | Branching factor (token candidates per step). | `1` (auto default for STANDALONE) | -| `--speculative-num-draft-tokens` | Verification capacity. | `4` (auto default for STANDALONE) | -| `--speculative-draft-model-quantization` | Quantization for the draft model. Use `"unquant"` to disable quantization on the draft even when the target is quantized. | Same as target | - -> **Note:** Standalone speculative decoding currently **does not support** `--enable-dp-attention`. - -```bash -python3 -m sglang.launch_server \ - --model Qwen/Qwen2.5-7B-Instruct \ - --speculative-algorithm STANDALONE \ - --speculative-draft-model-path Qwen/Qwen2.5-1.5B-Instruct \ - --speculative-num-steps 4 \ - --speculative-eagle-topk 2 \ - --speculative-num-draft-tokens 7 \ - --mem-fraction-static 0.7 \ - --cuda-graph-max-bs 8 \ - --log-level warning -``` - -**Send a request:** - -```python -import openai - -client = openai.Client(base_url="http://127.0.0.1:30000/v1", api_key="None") - -response = client.chat.completions.create( - model="Qwen/Qwen2.5-7B-Instruct", - messages=[ - {"role": "user", "content": "List 3 countries and their capitals."}, - ], - temperature=0, - max_tokens=64, -) - -print(response.choices[0].message.content) -``` - ---- - -## Speculative Decoding V2 (Overlap Scheduler) - -SGLang provides an **experimental Speculative Decoding V2** implementation that enables an overlap scheduler and uses V2 speculative workers (e.g. `StandaloneWorkerV2`, `EAGLEWorkerV2`). - -To enable it, set the environment variable: -- `SGLANG_ENABLE_SPEC_V2=True` - -Notes: -- SpecV2 currently only supports `--speculative-eagle-topk 1`. When SpecV2 is enabled, **set `--speculative-eagle-topk 1` explicitly**. -- If you explicitly set `--speculative-eagle-topk > 1`, the server will error. -- If you omit `--speculative-eagle-topk`, auto-tuning may pick `topk > 1` for some models (e.g. Llama). This is incompatible with SpecV2 and may not always trigger an immediate config error, so set `--speculative-eagle-topk 1` explicitly. -- This applies to `EAGLE`, `EAGLE3`, and `STANDALONE`. - -```bash -SGLANG_ENABLE_SPEC_V2=True python3 -m sglang.launch_server \ - --model Qwen/Qwen2.5-7B-Instruct \ - --speculative-algorithm STANDALONE \ - --speculative-draft-model-path Qwen/Qwen2.5-1.5B-Instruct \ - --speculative-num-steps 4 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 5 \ - --mem-fraction-static 0.7 \ - --cuda-graph-max-bs 8 \ - --log-level warning -``` - -**Send a request:** - -```python -import openai - -client = openai.Client(base_url="http://127.0.0.1:30000/v1", api_key="None") - -response = client.chat.completions.create( - model="Qwen/Qwen2.5-7B-Instruct", - messages=[ - {"role": "user", "content": "List 3 countries and their capitals."}, - ], - temperature=0, - max_tokens=64, -) - -print(response.choices[0].message.content) -``` - ---- - -## Ngram Speculative Decoding - -SGLang also supports **ngram-based speculative decoding** (no separate draft model). It retrieves draft tokens from an ngram cache built from previously generated tokens, and then verifies them with the target model. - -Enable it with: -- `--speculative-algorithm NGRAM` - -### Ngram-specific parameters - -| Parameter | Description | Default | -|---|---|---| -| `--speculative-num-draft-tokens` | Number of draft tokens verified per step. If omitted, defaults to `min(--speculative-ngram-max-trie-depth, 12)`. | `12` (with default ngram settings) | -| `--speculative-ngram-min-bfs-breadth` | Minimum BFS breadth. | `1` | -| `--speculative-ngram-max-bfs-breadth` | Maximum BFS breadth. | `10` | -| `--speculative-ngram-match-type` | Ngram tree-building mode: `"BFS"` for recency-based expansion or `"PROB"` for frequency-based expansion. | `"BFS"` | -| `--speculative-ngram-max-trie-depth` | Maximum suffix length stored and matched by the ngram trie. | `18` | -| `--speculative-ngram-capacity` | Cache capacity (number of entries). | `10,000,000` | - -Notes: -- Ngram speculative decoding **only supports CUDA**. -- It currently **does not support** `--enable-dp-attention`. -- It disables the overlap scheduler and mixed chunked prefill. -- If `--speculative-ngram-max-bfs-breadth > 1` (thus `speculative_eagle_topk > 1`) and `page_size > 1`, use `--attention-backend flashinfer`; otherwise the server will error. -- Optional: set `SGLANG_NGRAM_FORCE_GREEDY_VERIFY=True` to force greedy verification. - -```bash -python3 -m sglang.launch_server \ - --model Qwen/Qwen2.5-7B-Instruct \ - --speculative-algorithm NGRAM \ - --speculative-num-draft-tokens 16 \ - --speculative-ngram-max-bfs-breadth 10 \ - --mem-fraction-static 0.7 \ - --cuda-graph-max-bs 8 \ - --log-level warning -``` - -**Send a request:** - -```python -import openai - -client = openai.Client(base_url="http://127.0.0.1:30000/v1", api_key="None") - -response = client.chat.completions.create( - model="Qwen/Qwen2.5-7B-Instruct", - messages=[ - {"role": "user", "content": "List 3 countries and their capitals."}, - ], - temperature=0, - max_tokens=64, -) - -print(response.choices[0].message.content) -``` - ---- - -## Full Parameter Reference - -Below is a comprehensive list of all speculative decoding parameters available in SGLang: - -### Core parameters - -| Parameter | Type | Default | Description | -|---|---|---|---| -| `--speculative-algorithm` | `str` | `None` | Algorithm to use: `EAGLE`, `EAGLE3`, `STANDALONE`, `NGRAM`, `NEXTN` (alias of `EAGLE`) | -| `--speculative-draft-model-path` | `str` | `None` | Path to the draft model weights | -| `--speculative-draft-model-revision` | `str` | `None` | Specific revision/commit of the draft model (`"main"` is auto-used when draft path is set and revision is omitted) | -| `--speculative-draft-load-format` | `str` | `None` | Load format for draft model weights | -| `--speculative-num-steps` | `int` | `None` (auto-chosen when omitted) | Autoregressive drafting depth | -| `--speculative-eagle-topk` | `int` | `None` (auto-chosen when omitted) | Branching factor per drafting step | -| `--speculative-num-draft-tokens` | `int` | `None` (auto-chosen when omitted) | Maximum number of draft tokens for verification | -| `--speculative-accept-threshold-single` | `float` | `1.0` | Single-token acceptance threshold | -| `--speculative-accept-threshold-acc` | `float` | `1.0` | Accumulated acceptance threshold | -| `--speculative-token-map` | `str` | `None` | Path to FR-Spec high-frequency token map | -| `--speculative-attention-mode` | `str` | `"prefill"` | Attention mode for speculative operations (`"prefill"` or `"decode"`) | -| `--speculative-draft-attention-backend` | `str` | `None` | Override attention backend for the draft model | -| `--speculative-moe-runner-backend` | `str` | `None` | MoE runner backend for the draft model | -| `--speculative-moe-a2a-backend` | `str` | `None` | MoE all-to-all backend for the draft model | -| `--speculative-draft-model-quantization` | `str` | Same as target | Quantization for the draft model (`"unquant"` to disable) | - -### Ngram-specific parameters - -| Parameter | Type | Default | Description | -|---|---|---|---| -| `--speculative-ngram-min-bfs-breadth` | `int` | `1` | Minimum BFS breadth | -| `--speculative-ngram-max-bfs-breadth` | `int` | `10` | Maximum BFS breadth | -| `--speculative-ngram-match-type` | `str` | `"BFS"` | Ngram tree-building mode: `"BFS"` for recency-based expansion or `"PROB"` for frequency-based expansion | -| `--speculative-ngram-max-trie-depth` | `int` | `18` | Maximum suffix length stored and matched by the ngram trie | -| `--speculative-ngram-capacity` | `int` | `10,000,000` | Cache capacity | - -### Environment variables - -| Variable | Default | Description | -|---|---|---| -| `SGLANG_ENABLE_SPEC_V2` | `False` | Enable Speculative Decoding V2 (overlap scheduler) | -| `SGLANG_NGRAM_FORCE_GREEDY_VERIFY` | `False` | Force greedy verification for ngram decoding | - -### Other related flags - -| Parameter | Description | -|---|---| -| `--enable-multi-layer-eagle` | Enable multi-layer EAGLE (auto-enabled for MiMoV2 and Step3p5 models) | -| `--enable-torch-compile` | Enable `torch.compile` for kernel-level optimizations | -| `--torch-compile-max-bs` | Maximum batch size for `torch.compile` | - ---- - -## OOM Troubleshooting - -> [!WARNING] -> **Out of Memory (OOM)?** Speculative decoding may increase GPU memory usage because the draft tree, CUDA graphs, and verification-related buffers consume additional VRAM. If you encounter OOM errors, try the following adjustments. - -### Step 1: Lower static memory fraction (most effective) - -```bash ---mem-fraction-static 0.5 # when omitted, this value is auto-computed -``` - -- `--mem-fraction-static` controls the memory budget for model weights + KV cache pool. -- Lowering it directly increases dynamic headroom for activations and CUDA graph buffers. -- If omitted, SGLang auto-estimates this value from other settings, and those auto settings can still be too aggressive for some workloads. - -### Step 2: Reduce CUDA graph batch size - -```bash -# Fewer CUDA graph captures = less memory reserved ---cuda-graph-max-bs 4 # or even 2 for tight memory situations -``` - -- If omitted, `--cuda-graph-max-bs` is auto-selected based on GPU memory and TP size, and can be much larger on high-memory GPUs. - -### Step 3: Reduce draft tree size - -These three parameters directly control how much memory the draft tree consumes: - -```bash -# Before (aggressive, high memory) ---speculative-num-steps 5 --speculative-eagle-topk 8 --speculative-num-draft-tokens 64 - -# After (conservative, lower memory) ---speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 -``` - -### Step 4: Limit concurrent requests - -```bash -# Fewer concurrent requests lowers in-flight load and can reduce OOM risk ---max-running-requests 4 -``` - -### Quick OOM recovery recipe - -If you're hitting OOM and just want something that works, start with this minimal configuration and scale up: - -```bash -python3 -m sglang.launch_server \ - --model \ - --speculative-algorithm EAGLE \ - --speculative-draft-model-path \ - --speculative-num-steps 3 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 \ - --cuda-graph-max-bs 2 \ - --mem-fraction-static 0.5 \ - --max-running-requests 4 \ - --log-level warning -``` - -Then gradually increase `--speculative-num-draft-tokens`, `--speculative-eagle-topk`, and `--cuda-graph-max-bs`. Increase `--mem-fraction-static` last, only after the run is stable. - ---- - -## References - -EAGLE process is as follows: - -- Within EAGLE the draft model predicts the next feature vector, i.e. the last hidden state of the original LLM, using the feature sequence $(f_1, ..., f_k)$ and the token sequence $(t_2, ..., t_{k+1})$. -- The next token is then sampled from $p_{k+2}=\text{LMHead}(f_{k+1})$. Afterwards, the two sequences are extended in a tree style—branching out multiple potential continuations, with the branching factor per step controlled by the `speculative_eagle_topk` parameter—to ensure a more coherent connection of context, and are given as input again. -- In SGLang's EAGLE-2 implementation, the draft tree is expanded for the configured steps and then reranked to select the top `speculative_num_draft_tokens` final nodes as draft tokens. -- EAGLE-3 removes the feature prediction objective, incorporates low and mid-layer features, and is trained in an on-policy manner. - -This enhances drafting accuracy by operating on features instead of tokens for more regular inputs and by additionally passing tokens from the next timestep to reduce sampling randomness. For more details, see the [EAGLE-2](https://arxiv.org/abs/2406.16858) and [EAGLE-3](https://arxiv.org/abs/2503.01840) papers. - -For guidance on how to train your own EAGLE model please see the [EAGLE repo](https://github.com/SafeAILab/EAGLE/tree/main?tab=readme-ov-file#train). For EAGLE-3 training specifically, check out [SpecForge](https://github.com/sgl-project/SpecForge), the SGLang team's training framework designed for EAGLE-3 speculative decoding models with seamless porting to SGLang serving. See the [SpecForge documentation](https://docs.sglang.ai/SpecForge/) and [blog post](https://lmsys.org/blog/2025-07-25-spec-forge) for details. diff --git a/docs/advanced_features/structured_outputs.ipynb b/docs/advanced_features/structured_outputs.ipynb deleted file mode 100644 index 8902c9497..000000000 --- a/docs/advanced_features/structured_outputs.ipynb +++ /dev/null @@ -1,994 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Structured Outputs" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You can specify a JSON schema, [regular expression](https://en.wikipedia.org/wiki/Regular_expression) or [EBNF](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form) to constrain the model output. The model output will be guaranteed to follow the given constraints. Only one constraint parameter (`json_schema`, `regex`, or `ebnf`) can be specified for a request.\n", - "\n", - "SGLang supports three grammar backends:\n", - "\n", - "- [XGrammar](https://github.com/mlc-ai/xgrammar)(default): Supports JSON schema, regular expression, and EBNF constraints.\n", - "- [Outlines](https://github.com/dottxt-ai/outlines): Supports JSON schema and regular expression constraints.\n", - "- [Llguidance](https://github.com/guidance-ai/llguidance): Supports JSON schema, regular expression, and EBNF constraints.\n", - "\n", - "We suggest using XGrammar for its better performance and utility. XGrammar currently uses the [GGML BNF format](https://github.com/ggerganov/llama.cpp/blob/master/grammars/README.md). For more details, see [XGrammar technical overview](https://blog.mlc.ai/2024/11/22/achieving-efficient-flexible-portable-structured-generation-with-xgrammar).\n", - "\n", - "To use Outlines, simply add `--grammar-backend outlines` when launching the server.\n", - "To use llguidance, add `--grammar-backend llguidance` when launching the server.\n", - "If no backend is specified, XGrammar will be used as the default.\n", - "\n", - "For better output quality, **It's advisable to explicitly include instructions in the prompt to guide the model to generate the desired format.** For example, you can specify, 'Please generate the output in the following JSON format: ...'.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## OpenAI Compatible API" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import openai\n", - "import os\n", - "\n", - "from sglang.test.doc_patch import launch_server_cmd\n", - "from sglang.utils import wait_for_server, print_highlight, terminate_process\n", - "\n", - "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n", - "\n", - "\n", - "server_process, port = launch_server_cmd(\n", - " \"python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-8B-Instruct --host 0.0.0.0 --log-level warning\"\n", - ")\n", - "\n", - "wait_for_server(f\"http://localhost:{port}\", process=server_process)\n", - "client = openai.Client(base_url=f\"http://127.0.0.1:{port}/v1\", api_key=\"None\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### JSON\n", - "\n", - "you can directly define a JSON schema or use [Pydantic](https://docs.pydantic.dev/latest/) to define and validate the response." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**Using Pydantic**" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from pydantic import BaseModel, Field\n", - "\n", - "\n", - "# Define the schema using Pydantic\n", - "class CapitalInfo(BaseModel):\n", - " name: str = Field(..., pattern=r\"^\\w+$\", description=\"Name of the capital city\")\n", - " population: int = Field(..., description=\"Population of the capital city\")\n", - "\n", - "\n", - "response = client.chat.completions.create(\n", - " model=\"meta-llama/Meta-Llama-3.1-8B-Instruct\",\n", - " messages=[\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"Please generate the information of the capital of France in the JSON format.\",\n", - " },\n", - " ],\n", - " temperature=0,\n", - " max_tokens=128,\n", - " response_format={\n", - " \"type\": \"json_schema\",\n", - " \"json_schema\": {\n", - " \"name\": \"foo\",\n", - " # convert the pydantic model to json schema\n", - " \"schema\": CapitalInfo.model_json_schema(),\n", - " },\n", - " },\n", - ")\n", - "\n", - "response_content = response.choices[0].message.content\n", - "# validate the JSON response by the pydantic model\n", - "capital_info = CapitalInfo.model_validate_json(response_content)\n", - "print_highlight(f\"Validated response: {capital_info.model_dump_json()}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**JSON Schema Directly**\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import json\n", - "\n", - "json_schema = json.dumps(\n", - " {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"name\": {\"type\": \"string\", \"pattern\": \"^[\\\\w]+$\"},\n", - " \"population\": {\"type\": \"integer\"},\n", - " },\n", - " \"required\": [\"name\", \"population\"],\n", - " }\n", - ")\n", - "\n", - "response = client.chat.completions.create(\n", - " model=\"meta-llama/Meta-Llama-3.1-8B-Instruct\",\n", - " messages=[\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"Give me the information of the capital of France in the JSON format.\",\n", - " },\n", - " ],\n", - " temperature=0,\n", - " max_tokens=128,\n", - " response_format={\n", - " \"type\": \"json_schema\",\n", - " \"json_schema\": {\"name\": \"foo\", \"schema\": json.loads(json_schema)},\n", - " },\n", - ")\n", - "\n", - "print_highlight(response.choices[0].message.content)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### EBNF" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "ebnf_grammar = \"\"\"\n", - "root ::= city | description\n", - "city ::= \"London\" | \"Paris\" | \"Berlin\" | \"Rome\"\n", - "description ::= city \" is \" status\n", - "status ::= \"the capital of \" country\n", - "country ::= \"England\" | \"France\" | \"Germany\" | \"Italy\"\n", - "\"\"\"\n", - "\n", - "response = client.chat.completions.create(\n", - " model=\"meta-llama/Meta-Llama-3.1-8B-Instruct\",\n", - " messages=[\n", - " {\"role\": \"system\", \"content\": \"You are a helpful geography bot.\"},\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"Give me the information of the capital of France.\",\n", - " },\n", - " ],\n", - " temperature=0,\n", - " max_tokens=32,\n", - " extra_body={\"ebnf\": ebnf_grammar},\n", - ")\n", - "\n", - "print_highlight(response.choices[0].message.content)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Regular expression" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "response = client.chat.completions.create(\n", - " model=\"meta-llama/Meta-Llama-3.1-8B-Instruct\",\n", - " messages=[\n", - " {\"role\": \"user\", \"content\": \"What is the capital of France?\"},\n", - " ],\n", - " temperature=0,\n", - " max_tokens=128,\n", - " extra_body={\"regex\": \"(Paris|London)\"},\n", - ")\n", - "\n", - "print_highlight(response.choices[0].message.content)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Structural Tag" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "tool_get_current_weather = {\n", - " \"type\": \"function\",\n", - " \"function\": {\n", - " \"name\": \"get_current_weather\",\n", - " \"description\": \"Get the current weather in a given location\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"city\": {\n", - " \"type\": \"string\",\n", - " \"description\": \"The city to find the weather for, e.g. 'San Francisco'\",\n", - " },\n", - " \"state\": {\n", - " \"type\": \"string\",\n", - " \"description\": \"the two-letter abbreviation for the state that the city is\"\n", - " \" in, e.g. 'CA' which would mean 'California'\",\n", - " },\n", - " \"unit\": {\n", - " \"type\": \"string\",\n", - " \"description\": \"The unit to fetch the temperature in\",\n", - " \"enum\": [\"celsius\", \"fahrenheit\"],\n", - " },\n", - " },\n", - " \"required\": [\"city\", \"state\", \"unit\"],\n", - " },\n", - " },\n", - "}\n", - "\n", - "tool_get_current_date = {\n", - " \"type\": \"function\",\n", - " \"function\": {\n", - " \"name\": \"get_current_date\",\n", - " \"description\": \"Get the current date and time for a given timezone\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"timezone\": {\n", - " \"type\": \"string\",\n", - " \"description\": \"The timezone to fetch the current date and time for, e.g. 'America/New_York'\",\n", - " }\n", - " },\n", - " \"required\": [\"timezone\"],\n", - " },\n", - " },\n", - "}\n", - "\n", - "schema_get_current_weather = tool_get_current_weather[\"function\"][\"parameters\"]\n", - "schema_get_current_date = tool_get_current_date[\"function\"][\"parameters\"]\n", - "\n", - "\n", - "def get_messages():\n", - " return [\n", - " {\n", - " \"role\": \"system\",\n", - " \"content\": f\"\"\"\n", - "# Tool Instructions\n", - "- Always execute python code in messages that you share.\n", - "- When looking for real time information use relevant functions if available else fallback to brave_search\n", - "You have access to the following functions:\n", - "Use the function 'get_current_weather' to: Get the current weather in a given location\n", - "{tool_get_current_weather[\"function\"]}\n", - "Use the function 'get_current_date' to: Get the current date and time for a given timezone\n", - "{tool_get_current_date[\"function\"]}\n", - "If a you choose to call a function ONLY reply in the following format:\n", - "<{{start_tag}}={{function_name}}>{{parameters}}{{end_tag}}\n", - "where\n", - "start_tag => ` a JSON dict with the function argument name as key and function argument value as value.\n", - "end_tag => ``\n", - "Here is an example,\n", - "{{\"example_name\": \"example_value\"}}\n", - "Reminder:\n", - "- Function calls MUST follow the specified format\n", - "- Required parameters MUST be specified\n", - "- Only call one function at a time\n", - "- Put the entire function call reply on one line\n", - "- Always add your sources when using search results to answer the user query\n", - "You are a helpful assistant.\"\"\",\n", - " },\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"You are in New York. Please get the current date and time, and the weather.\",\n", - " },\n", - " ]\n", - "\n", - "\n", - "messages = get_messages()\n", - "\n", - "response = client.chat.completions.create(\n", - " model=\"meta-llama/Meta-Llama-3.1-8B-Instruct\",\n", - " messages=messages,\n", - " response_format={\n", - " \"type\": \"structural_tag\",\n", - " \"structures\": [\n", - " {\n", - " \"begin\": \"\",\n", - " \"schema\": schema_get_current_weather,\n", - " \"end\": \"\",\n", - " },\n", - " {\n", - " \"begin\": \"\",\n", - " \"schema\": schema_get_current_date,\n", - " \"end\": \"\",\n", - " },\n", - " ],\n", - " \"triggers\": [\"\n", - "response = client.chat.completions.create(\n", - " model=\"meta-llama/Meta-Llama-3.1-8B-Instruct\",\n", - " messages=messages,\n", - " response_format={\n", - " \"type\": \"structural_tag\",\n", - " \"format\": {\n", - " \"type\": \"triggered_tags\",\n", - " \"triggers\": [\"\",\n", - " \"content\": {\n", - " \"type\": \"json_schema\",\n", - " \"json_schema\": schema_get_current_weather,\n", - " },\n", - " \"end\": \"\",\n", - " },\n", - " {\n", - " \"begin\": \"\",\n", - " \"content\": {\n", - " \"type\": \"json_schema\",\n", - " \"json_schema\": schema_get_current_date,\n", - " },\n", - " \"end\": \"\",\n", - " },\n", - " ],\n", - " \"at_least_one\": False,\n", - " \"stop_after_first\": False,\n", - " },\n", - " },\n", - ")\n", - "\n", - "print_highlight(response.choices[0].message.content)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Native API and SGLang Runtime (SRT)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### JSON" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**Using Pydantic**" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import requests\n", - "import json\n", - "from pydantic import BaseModel, Field\n", - "\n", - "from transformers import AutoTokenizer\n", - "\n", - "tokenizer = AutoTokenizer.from_pretrained(\"meta-llama/Meta-Llama-3.1-8B-Instruct\")\n", - "\n", - "\n", - "# Define the schema using Pydantic\n", - "class CapitalInfo(BaseModel):\n", - " name: str = Field(..., pattern=r\"^\\w+$\", description=\"Name of the capital city\")\n", - " population: int = Field(..., description=\"Population of the capital city\")\n", - "\n", - "\n", - "# Make API request\n", - "messages = [\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"Here is the information of the capital of France in the JSON format.\\n\",\n", - " }\n", - "]\n", - "text = tokenizer.apply_chat_template(\n", - " messages, tokenize=False, add_generation_prompt=True, return_dict=False\n", - ")\n", - "response = requests.post(\n", - " f\"http://localhost:{port}/generate\",\n", - " json={\n", - " \"text\": text,\n", - " \"sampling_params\": {\n", - " \"temperature\": 0,\n", - " \"max_new_tokens\": 64,\n", - " \"json_schema\": json.dumps(CapitalInfo.model_json_schema()),\n", - " },\n", - " },\n", - ")\n", - "print_highlight(response.json())\n", - "\n", - "\n", - "response_data = json.loads(response.json()[\"text\"])\n", - "# validate the response by the pydantic model\n", - "capital_info = CapitalInfo.model_validate(response_data)\n", - "print_highlight(f\"Validated response: {capital_info.model_dump_json()}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**JSON Schema Directly**" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "json_schema = json.dumps(\n", - " {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"name\": {\"type\": \"string\", \"pattern\": \"^[\\\\w]+$\"},\n", - " \"population\": {\"type\": \"integer\"},\n", - " },\n", - " \"required\": [\"name\", \"population\"],\n", - " }\n", - ")\n", - "\n", - "# JSON\n", - "response = requests.post(\n", - " f\"http://localhost:{port}/generate\",\n", - " json={\n", - " \"text\": text,\n", - " \"sampling_params\": {\n", - " \"temperature\": 0,\n", - " \"max_new_tokens\": 64,\n", - " \"json_schema\": json_schema,\n", - " },\n", - " },\n", - ")\n", - "\n", - "print_highlight(response.json())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### EBNF" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "messages = [\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"Give me the information of the capital of France.\",\n", - " }\n", - "]\n", - "text = tokenizer.apply_chat_template(\n", - " messages, tokenize=False, add_generation_prompt=True, return_dict=False\n", - ")\n", - "response = requests.post(\n", - " f\"http://localhost:{port}/generate\",\n", - " json={\n", - " \"text\": text,\n", - " \"sampling_params\": {\n", - " \"max_new_tokens\": 128,\n", - " \"temperature\": 0,\n", - " \"n\": 3,\n", - " \"ebnf\": (\n", - " \"root ::= city | description\\n\"\n", - " 'city ::= \"London\" | \"Paris\" | \"Berlin\" | \"Rome\"\\n'\n", - " 'description ::= city \" is \" status\\n'\n", - " 'status ::= \"the capital of \" country\\n'\n", - " 'country ::= \"England\" | \"France\" | \"Germany\" | \"Italy\"'\n", - " ),\n", - " },\n", - " \"stream\": False,\n", - " \"return_logprob\": False,\n", - " },\n", - ")\n", - "\n", - "print_highlight(response.json())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Regular expression" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "messages = [\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"Paris is the capital of\",\n", - " }\n", - "]\n", - "text = tokenizer.apply_chat_template(\n", - " messages, tokenize=False, add_generation_prompt=True, return_dict=False\n", - ")\n", - "response = requests.post(\n", - " f\"http://localhost:{port}/generate\",\n", - " json={\n", - " \"text\": text,\n", - " \"sampling_params\": {\n", - " \"temperature\": 0,\n", - " \"max_new_tokens\": 64,\n", - " \"regex\": \"(France|England)\",\n", - " },\n", - " },\n", - ")\n", - "print_highlight(response.json())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Structural Tag" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from transformers import AutoTokenizer\n", - "\n", - "# generate an answer\n", - "tokenizer = AutoTokenizer.from_pretrained(\"meta-llama/Meta-Llama-3.1-8B-Instruct\")\n", - "\n", - "text = tokenizer.apply_chat_template(\n", - " messages, tokenize=False, add_generation_prompt=True, return_dict=False\n", - ")\n", - "payload = {\n", - " \"text\": text,\n", - " \"sampling_params\": {\n", - " \"structural_tag\": json.dumps(\n", - " {\n", - " \"type\": \"structural_tag\",\n", - " \"structures\": [\n", - " {\n", - " \"begin\": \"\",\n", - " \"schema\": schema_get_current_weather,\n", - " \"end\": \"\",\n", - " },\n", - " {\n", - " \"begin\": \"\",\n", - " \"schema\": schema_get_current_date,\n", - " \"end\": \"\",\n", - " },\n", - " ],\n", - " \"triggers\": [\"\n", - "payload = {\n", - " \"text\": text,\n", - " \"sampling_params\": {\n", - " \"structural_tag\": json.dumps(\n", - " {\n", - " \"type\": \"structural_tag\",\n", - " \"format\": {\n", - " \"type\": \"triggered_tags\",\n", - " \"triggers\": [\"\",\n", - " \"content\": {\n", - " \"type\": \"json_schema\",\n", - " \"json_schema\": schema_get_current_weather,\n", - " },\n", - " \"end\": \"\",\n", - " },\n", - " {\n", - " \"begin\": \"\",\n", - " \"content\": {\n", - " \"type\": \"json_schema\",\n", - " \"json_schema\": schema_get_current_date,\n", - " },\n", - " \"end\": \"\",\n", - " },\n", - " ],\n", - " \"at_least_one\": False,\n", - " \"stop_after_first\": False,\n", - " },\n", - " }\n", - " )\n", - " },\n", - "}\n", - "\n", - "\n", - "# Send POST request to the API endpoint\n", - "response = requests.post(f\"http://localhost:{port}/generate\", json=payload)\n", - "print_highlight(response.json())" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "terminate_process(server_process)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Offline Engine API" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import sglang as sgl\n", - "\n", - "llm = sgl.Engine(\n", - " model_path=\"meta-llama/Meta-Llama-3.1-8B-Instruct\", grammar_backend=\"xgrammar\"\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### JSON" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**Using Pydantic**" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import json\n", - "from pydantic import BaseModel, Field\n", - "\n", - "prompts = [\n", - " \"Give me the information of the capital of China in the JSON format.\",\n", - " \"Give me the information of the capital of France in the JSON format.\",\n", - " \"Give me the information of the capital of Ireland in the JSON format.\",\n", - "]\n", - "\n", - "\n", - "# Define the schema using Pydantic\n", - "class CapitalInfo(BaseModel):\n", - " name: str = Field(..., pattern=r\"^\\w+$\", description=\"Name of the capital city\")\n", - " population: int = Field(..., description=\"Population of the capital city\")\n", - "\n", - "\n", - "sampling_params = {\n", - " \"temperature\": 0.1,\n", - " \"top_p\": 0.95,\n", - " \"json_schema\": json.dumps(CapitalInfo.model_json_schema()),\n", - "}\n", - "\n", - "outputs = llm.generate(prompts, sampling_params)\n", - "for prompt, output in zip(prompts, outputs):\n", - " print_highlight(\"===============================\")\n", - " print_highlight(f\"Prompt: {prompt}\") # validate the output by the pydantic model\n", - " capital_info = CapitalInfo.model_validate_json(output[\"text\"])\n", - " print_highlight(f\"Validated output: {capital_info.model_dump_json()}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**JSON Schema Directly**" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "prompts = [\n", - " \"Give me the information of the capital of China in the JSON format.\",\n", - " \"Give me the information of the capital of France in the JSON format.\",\n", - " \"Give me the information of the capital of Ireland in the JSON format.\",\n", - "]\n", - "\n", - "json_schema = json.dumps(\n", - " {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"name\": {\"type\": \"string\", \"pattern\": \"^[\\\\w]+$\"},\n", - " \"population\": {\"type\": \"integer\"},\n", - " },\n", - " \"required\": [\"name\", \"population\"],\n", - " }\n", - ")\n", - "\n", - "sampling_params = {\"temperature\": 0.1, \"top_p\": 0.95, \"json_schema\": json_schema}\n", - "\n", - "outputs = llm.generate(prompts, sampling_params)\n", - "for prompt, output in zip(prompts, outputs):\n", - " print_highlight(\"===============================\")\n", - " print_highlight(f\"Prompt: {prompt}\\nGenerated text: {output['text']}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### EBNF\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "prompts = [\n", - " \"Give me the information of the capital of France.\",\n", - " \"Give me the information of the capital of Germany.\",\n", - " \"Give me the information of the capital of Italy.\",\n", - "]\n", - "\n", - "sampling_params = {\n", - " \"temperature\": 0.8,\n", - " \"top_p\": 0.95,\n", - " \"ebnf\": (\n", - " \"root ::= city | description\\n\"\n", - " 'city ::= \"London\" | \"Paris\" | \"Berlin\" | \"Rome\"\\n'\n", - " 'description ::= city \" is \" status\\n'\n", - " 'status ::= \"the capital of \" country\\n'\n", - " 'country ::= \"England\" | \"France\" | \"Germany\" | \"Italy\"'\n", - " ),\n", - "}\n", - "\n", - "outputs = llm.generate(prompts, sampling_params)\n", - "for prompt, output in zip(prompts, outputs):\n", - " print_highlight(\"===============================\")\n", - " print_highlight(f\"Prompt: {prompt}\\nGenerated text: {output['text']}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Regular expression" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "prompts = [\n", - " \"Please provide information about London as a major global city:\",\n", - " \"Please provide information about Paris as a major global city:\",\n", - "]\n", - "\n", - "sampling_params = {\"temperature\": 0.8, \"top_p\": 0.95, \"regex\": \"(France|England)\"}\n", - "\n", - "outputs = llm.generate(prompts, sampling_params)\n", - "for prompt, output in zip(prompts, outputs):\n", - " print_highlight(\"===============================\")\n", - " print_highlight(f\"Prompt: {prompt}\\nGenerated text: {output['text']}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Structural Tag" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "text = tokenizer.apply_chat_template(\n", - " messages, tokenize=False, add_generation_prompt=True, return_dict=False\n", - ")\n", - "prompts = [text]\n", - "\n", - "\n", - "sampling_params = {\n", - " \"temperature\": 0.8,\n", - " \"top_p\": 0.95,\n", - " \"structural_tag\": json.dumps(\n", - " {\n", - " \"type\": \"structural_tag\",\n", - " \"structures\": [\n", - " {\n", - " \"begin\": \"\",\n", - " \"schema\": schema_get_current_weather,\n", - " \"end\": \"\",\n", - " },\n", - " {\n", - " \"begin\": \"\",\n", - " \"schema\": schema_get_current_date,\n", - " \"end\": \"\",\n", - " },\n", - " ],\n", - " \"triggers\": [\"\n", - "sampling_params = {\n", - " \"temperature\": 0.8,\n", - " \"top_p\": 0.95,\n", - " \"structural_tag\": json.dumps(\n", - " {\n", - " \"type\": \"structural_tag\",\n", - " \"format\": {\n", - " \"type\": \"triggered_tags\",\n", - " \"triggers\": [\"\",\n", - " \"content\": {\n", - " \"type\": \"json_schema\",\n", - " \"json_schema\": schema_get_current_weather,\n", - " },\n", - " \"end\": \"\",\n", - " },\n", - " {\n", - " \"begin\": \"\",\n", - " \"content\": {\n", - " \"type\": \"json_schema\",\n", - " \"json_schema\": schema_get_current_date,\n", - " },\n", - " \"end\": \"\",\n", - " },\n", - " ],\n", - " \"at_least_one\": False,\n", - " \"stop_after_first\": False,\n", - " },\n", - " }\n", - " ),\n", - "}\n", - "\n", - "\n", - "# Send POST request to the API endpoint\n", - "outputs = llm.generate(prompts, sampling_params)\n", - "for prompt, output in zip(prompts, outputs):\n", - " print_highlight(\"===============================\")\n", - " print_highlight(f\"Prompt: {prompt}\\nGenerated text: {output['text']}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "llm.shutdown()" - ] - } - ], - "metadata": { - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/docs/advanced_features/structured_outputs_for_reasoning_models.ipynb b/docs/advanced_features/structured_outputs_for_reasoning_models.ipynb deleted file mode 100644 index cfc07fd01..000000000 --- a/docs/advanced_features/structured_outputs_for_reasoning_models.ipynb +++ /dev/null @@ -1,841 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Structured Outputs For Reasoning Models\n", - "\n", - "When working with reasoning models that use special tokens like `...` to denote reasoning sections, you might want to allow free-form text within these sections while still enforcing grammar constraints on the rest of the output.\n", - "\n", - "SGLang provides a feature to disable grammar restrictions within reasoning sections. This is particularly useful for models that need to perform complex reasoning steps before providing a structured output.\n", - "\n", - "To enable this feature, use the `--reasoning-parser` flag which decide the think_end_token, such as ``, when launching the server. You can also specify the reasoning parser using the `--reasoning-parser` flag.\n", - "\n", - "## Supported Models\n", - "\n", - "Currently, SGLang supports the following reasoning models:\n", - "- [DeepSeek R1 series](https://huggingface.co/collections/deepseek-ai/deepseek-r1-678e1e131c0169c0bc89728d): The reasoning content is wrapped with `` and `` tags.\n", - "- [QwQ](https://huggingface.co/Qwen/QwQ-32B): The reasoning content is wrapped with `` and `` tags.\n", - "\n", - "\n", - "## Usage\n", - "\n", - "## OpenAI Compatible API" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Specify the `--grammar-backend`, `--reasoning-parser` option." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import openai\n", - "import os\n", - "\n", - "from sglang.test.doc_patch import launch_server_cmd\n", - "from sglang.utils import wait_for_server, print_highlight, terminate_process\n", - "\n", - "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n", - "\n", - "\n", - "server_process, port = launch_server_cmd(\n", - " \"python -m sglang.launch_server --model-path deepseek-ai/DeepSeek-R1-Distill-Qwen-7B --host 0.0.0.0 --reasoning-parser deepseek-r1 --log-level warning\"\n", - ")\n", - "\n", - "wait_for_server(f\"http://localhost:{port}\", process=server_process)\n", - "client = openai.Client(base_url=f\"http://127.0.0.1:{port}/v1\", api_key=\"None\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### JSON\n", - "\n", - "you can directly define a JSON schema or use [Pydantic](https://docs.pydantic.dev/latest/) to define and validate the response." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**Using Pydantic**" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from pydantic import BaseModel, Field\n", - "\n", - "\n", - "# Define the schema using Pydantic\n", - "class CapitalInfo(BaseModel):\n", - " name: str = Field(..., pattern=r\"^\\w+$\", description=\"Name of the capital city\")\n", - " population: int = Field(..., description=\"Population of the capital city\")\n", - "\n", - "\n", - "response = client.chat.completions.create(\n", - " model=\"deepseek-ai/DeepSeek-R1-Distill-Qwen-7B\",\n", - " messages=[\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"Give me the information and population of the capital of France in the JSON format.\",\n", - " },\n", - " ],\n", - " temperature=0,\n", - " max_tokens=2048,\n", - " response_format={\n", - " \"type\": \"json_schema\",\n", - " \"json_schema\": {\n", - " \"name\": \"foo\",\n", - " # convert the pydantic model to json schema\n", - " \"schema\": CapitalInfo.model_json_schema(),\n", - " },\n", - " },\n", - ")\n", - "\n", - "print_highlight(\n", - " f\"reasoing_content: {response.choices[0].message.reasoning_content}\\n\\ncontent: {response.choices[0].message.content}\"\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**JSON Schema Directly**\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import json\n", - "\n", - "json_schema = json.dumps(\n", - " {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"name\": {\"type\": \"string\", \"pattern\": \"^[\\\\w]+$\"},\n", - " \"population\": {\"type\": \"integer\"},\n", - " },\n", - " \"required\": [\"name\", \"population\"],\n", - " }\n", - ")\n", - "\n", - "response = client.chat.completions.create(\n", - " model=\"deepseek-ai/DeepSeek-R1-Distill-Qwen-7B\",\n", - " messages=[\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"Give me the information and population of the capital of France in the JSON format.\",\n", - " },\n", - " ],\n", - " temperature=0,\n", - " max_tokens=2048,\n", - " response_format={\n", - " \"type\": \"json_schema\",\n", - " \"json_schema\": {\"name\": \"foo\", \"schema\": json.loads(json_schema)},\n", - " },\n", - ")\n", - "\n", - "print_highlight(\n", - " f\"reasoing_content: {response.choices[0].message.reasoning_content}\\n\\ncontent: {response.choices[0].message.content}\"\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### EBNF" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "ebnf_grammar = \"\"\"\n", - "root ::= city | description\n", - "city ::= \"London\" | \"Paris\" | \"Berlin\" | \"Rome\"\n", - "description ::= city \" is \" status\n", - "status ::= \"the capital of \" country\n", - "country ::= \"England\" | \"France\" | \"Germany\" | \"Italy\"\n", - "\"\"\"\n", - "\n", - "response = client.chat.completions.create(\n", - " model=\"deepseek-ai/DeepSeek-R1-Distill-Qwen-7B\",\n", - " messages=[\n", - " {\"role\": \"system\", \"content\": \"You are a helpful geography bot.\"},\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"Give me the information and population of the capital of France in the JSON format.\",\n", - " },\n", - " ],\n", - " temperature=0,\n", - " max_tokens=2048,\n", - " extra_body={\"ebnf\": ebnf_grammar},\n", - ")\n", - "\n", - "print_highlight(\n", - " f\"reasoing_content: {response.choices[0].message.reasoning_content}\\n\\ncontent: {response.choices[0].message.content}\"\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Regular expression" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "response = client.chat.completions.create(\n", - " model=\"deepseek-ai/DeepSeek-R1-Distill-Qwen-7B\",\n", - " messages=[\n", - " {\"role\": \"assistant\", \"content\": \"What is the capital of France?\"},\n", - " ],\n", - " temperature=0,\n", - " max_tokens=2048,\n", - " extra_body={\"regex\": \"(Paris|London)\"},\n", - ")\n", - "\n", - "print_highlight(\n", - " f\"reasoing_content: {response.choices[0].message.reasoning_content}\\n\\ncontent: {response.choices[0].message.content}\"\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Structural Tag" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "tool_get_current_weather = {\n", - " \"type\": \"function\",\n", - " \"function\": {\n", - " \"name\": \"get_current_weather\",\n", - " \"description\": \"Get the current weather in a given location\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"city\": {\n", - " \"type\": \"string\",\n", - " \"description\": \"The city to find the weather for, e.g. 'San Francisco'\",\n", - " },\n", - " \"state\": {\n", - " \"type\": \"string\",\n", - " \"description\": \"the two-letter abbreviation for the state that the city is\"\n", - " \" in, e.g. 'CA' which would mean 'California'\",\n", - " },\n", - " \"unit\": {\n", - " \"type\": \"string\",\n", - " \"description\": \"The unit to fetch the temperature in\",\n", - " \"enum\": [\"celsius\", \"fahrenheit\"],\n", - " },\n", - " },\n", - " \"required\": [\"city\", \"state\", \"unit\"],\n", - " },\n", - " },\n", - "}\n", - "\n", - "tool_get_current_date = {\n", - " \"type\": \"function\",\n", - " \"function\": {\n", - " \"name\": \"get_current_date\",\n", - " \"description\": \"Get the current date and time for a given timezone\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"timezone\": {\n", - " \"type\": \"string\",\n", - " \"description\": \"The timezone to fetch the current date and time for, e.g. 'America/New_York'\",\n", - " }\n", - " },\n", - " \"required\": [\"timezone\"],\n", - " },\n", - " },\n", - "}\n", - "\n", - "schema_get_current_weather = tool_get_current_weather[\"function\"][\"parameters\"]\n", - "schema_get_current_date = tool_get_current_date[\"function\"][\"parameters\"]\n", - "\n", - "\n", - "def get_messages():\n", - " return [\n", - " {\n", - " \"role\": \"system\",\n", - " \"content\": f\"\"\"\n", - "# Tool Instructions\n", - "- Always execute python code in messages that you share.\n", - "- When looking for real time information use relevant functions if available else fallback to brave_search\n", - "You have access to the following functions:\n", - "Use the function 'get_current_weather' to: Get the current weather in a given location\n", - "{tool_get_current_weather[\"function\"]}\n", - "Use the function 'get_current_date' to: Get the current date and time for a given timezone\n", - "{tool_get_current_date[\"function\"]}\n", - "If a you choose to call a function ONLY reply in the following format:\n", - "<{{start_tag}}={{function_name}}>{{parameters}}{{end_tag}}\n", - "where\n", - "start_tag => ` a JSON dict with the function argument name as key and function argument value as value.\n", - "end_tag => ``\n", - "Here is an example,\n", - "{{\"example_name\": \"example_value\"}}\n", - "Reminder:\n", - "- Function calls MUST follow the specified format\n", - "- Required parameters MUST be specified\n", - "- Only call one function at a time\n", - "- Put the entire function call reply on one line\n", - "- Always add your sources when using search results to answer the user query\n", - "You are a helpful assistant.\"\"\",\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"You are in New York. Please get the current date and time, and the weather.\",\n", - " },\n", - " ]\n", - "\n", - "\n", - "messages = get_messages()\n", - "\n", - "response = client.chat.completions.create(\n", - " model=\"deepseek-ai/DeepSeek-R1-Distill-Qwen-7B\",\n", - " messages=messages,\n", - " response_format={\n", - " \"type\": \"structural_tag\",\n", - " \"max_new_tokens\": 2048,\n", - " \"structures\": [\n", - " {\n", - " \"begin\": \"\",\n", - " \"schema\": schema_get_current_weather,\n", - " \"end\": \"\",\n", - " },\n", - " {\n", - " \"begin\": \"\",\n", - " \"schema\": schema_get_current_date,\n", - " \"end\": \"\",\n", - " },\n", - " ],\n", - " \"triggers\": [\" Note: For native API, as a work-around, you need to set `require_reasoning` argument to `True` to ensure the model will think before generating the structured output. It's not required for chat-completion API." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### JSON" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**Using Pydantic**" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import requests\n", - "from pydantic import BaseModel, Field\n", - "from transformers import AutoTokenizer\n", - "\n", - "tokenizer = AutoTokenizer.from_pretrained(\"deepseek-ai/DeepSeek-R1-Distill-Qwen-7B\")\n", - "\n", - "\n", - "# Define the schema using Pydantic\n", - "class CapitalInfo(BaseModel):\n", - " name: str = Field(..., pattern=r\"^\\w+$\", description=\"Name of the capital city\")\n", - " population: int = Field(..., description=\"Population of the capital city\")\n", - "\n", - "\n", - "messages = [\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"Give me the information and population of the capital of France in the JSON format.\",\n", - " },\n", - "]\n", - "text = tokenizer.apply_chat_template(\n", - " messages, tokenize=False, add_generation_prompt=True, return_dict=False\n", - ")\n", - "# Make API request\n", - "response = requests.post(\n", - " f\"http://localhost:{port}/generate\",\n", - " json={\n", - " \"text\": text,\n", - " \"require_reasoning\": True,\n", - " \"sampling_params\": {\n", - " \"temperature\": 0,\n", - " \"max_new_tokens\": 2048,\n", - " \"json_schema\": json.dumps(CapitalInfo.model_json_schema()),\n", - " },\n", - " },\n", - ")\n", - "print(response.json())\n", - "\n", - "\n", - "reasoing_content = response.json()[\"text\"].split(\"\")[0]\n", - "content = response.json()[\"text\"].split(\"\")[1]\n", - "print_highlight(f\"reasoing_content: {reasoing_content}\\n\\ncontent: {content}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**JSON Schema Directly**" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "json_schema = json.dumps(\n", - " {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"name\": {\"type\": \"string\", \"pattern\": \"^[\\\\w]+$\"},\n", - " \"population\": {\"type\": \"integer\"},\n", - " },\n", - " \"required\": [\"name\", \"population\"],\n", - " }\n", - ")\n", - "\n", - "# JSON\n", - "text = tokenizer.apply_chat_template(\n", - " messages, tokenize=False, add_generation_prompt=True, return_dict=False\n", - ")\n", - "response = requests.post(\n", - " f\"http://localhost:{port}/generate\",\n", - " json={\n", - " \"text\": text,\n", - " \"require_reasoning\": True,\n", - " \"sampling_params\": {\n", - " \"temperature\": 0,\n", - " \"max_new_tokens\": 2048,\n", - " \"json_schema\": json_schema,\n", - " },\n", - " },\n", - ")\n", - "\n", - "print_highlight(response.json())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### EBNF" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "response = requests.post(\n", - " f\"http://localhost:{port}/generate\",\n", - " json={\n", - " \"text\": \"Give me the information of the capital of France.\",\n", - " \"require_reasoning\": True,\n", - " \"sampling_params\": {\n", - " \"max_new_tokens\": 2048,\n", - " \"temperature\": 0,\n", - " \"n\": 3,\n", - " \"ebnf\": (\n", - " \"root ::= city | description\\n\"\n", - " 'city ::= \"London\" | \"Paris\" | \"Berlin\" | \"Rome\"\\n'\n", - " 'description ::= city \" is \" status\\n'\n", - " 'status ::= \"the capital of \" country\\n'\n", - " 'country ::= \"England\" | \"France\" | \"Germany\" | \"Italy\"'\n", - " ),\n", - " },\n", - " \"stream\": False,\n", - " \"return_logprob\": False,\n", - " },\n", - ")\n", - "\n", - "print(response.json())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Regular expression" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "response = requests.post(\n", - " f\"http://localhost:{port}/generate\",\n", - " json={\n", - " \"text\": \"Paris is the capital of\",\n", - " \"require_reasoning\": True,\n", - " \"sampling_params\": {\n", - " \"temperature\": 0,\n", - " \"max_new_tokens\": 2048,\n", - " \"regex\": \"(France|England)\",\n", - " },\n", - " },\n", - ")\n", - "print(response.json())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Structural Tag" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "text = tokenizer.apply_chat_template(\n", - " messages, tokenize=False, add_generation_prompt=True, return_dict=False\n", - ")\n", - "payload = {\n", - " \"text\": text,\n", - " \"require_reasoning\": True,\n", - " \"sampling_params\": {\n", - " \"max_new_tokens\": 2048,\n", - " \"structural_tag\": json.dumps(\n", - " {\n", - " \"type\": \"structural_tag\",\n", - " \"structures\": [\n", - " {\n", - " \"begin\": \"\",\n", - " \"schema\": schema_get_current_weather,\n", - " \"end\": \"\",\n", - " },\n", - " {\n", - " \"begin\": \"\",\n", - " \"schema\": schema_get_current_date,\n", - " \"end\": \"\",\n", - " },\n", - " ],\n", - " \"triggers\": [\"\",\n", - " \"schema\": schema_get_current_weather,\n", - " \"end\": \"\",\n", - " },\n", - " {\n", - " \"begin\": \"\",\n", - " \"schema\": schema_get_current_date,\n", - " \"end\": \"\",\n", - " },\n", - " ],\n", - " \"triggers\": [\" is not trimmed.\n", - "\n", - "sampling_params = {\n", - " \"max_new_tokens\": 1024,\n", - " \"temperature\": 0,\n", - " \"top_p\": 0.95,\n", - " \"skip_special_tokens\": False,\n", - "}\n", - "\n", - "# 1) Offline generation\n", - "result = llm.generate(input_ids=input_ids, sampling_params=sampling_params)\n", - "generated_text = result[\"text\"] # Assume there is only one prompt\n", - "\n", - "print_highlight(\"=== Offline Engine Output Text ===\")\n", - "print_highlight(generated_text)\n", - "\n", - "\n", - "# 2) Parse using FunctionCallParser\n", - "def convert_dict_to_tool(tool_dict: dict) -> Tool:\n", - " function_dict = tool_dict.get(\"function\", {})\n", - " return Tool(\n", - " type=tool_dict.get(\"type\", \"function\"),\n", - " function=Function(\n", - " name=function_dict.get(\"name\"),\n", - " description=function_dict.get(\"description\"),\n", - " parameters=function_dict.get(\"parameters\"),\n", - " ),\n", - " )\n", - "\n", - "\n", - "tools = [convert_dict_to_tool(raw_tool) for raw_tool in tools]\n", - "\n", - "parser = FunctionCallParser(tools=tools, tool_call_parser=\"qwen25\")\n", - "normal_text, calls = parser.parse_non_stream(generated_text)\n", - "\n", - "print_highlight(\"=== Parsing Result ===\")\n", - "print(\"Normal text portion:\", normal_text)\n", - "print_highlight(\"Function call portion:\")\n", - "for call in calls:\n", - " # call: ToolCallItem\n", - " print_highlight(f\" - tool name: {call.name}\")\n", - " print_highlight(f\" parameters: {call.parameters}\")\n", - "\n", - "# 3) If needed, perform additional logic on the parsed functions, such as automatically calling the corresponding function to obtain a return value, etc." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "llm.shutdown()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Tool Choice Mode\n", - "\n", - "SGLang supports OpenAI's `tool_choice` parameter to control when and which tools the model should call. This feature is implemented using EBNF (Extended Backus-Naur Form) grammar to ensure reliable tool calling behavior.\n", - "\n", - "### Supported Tool Choice Options\n", - "\n", - "- **`tool_choice=\"required\"`**: Forces the model to call at least one tool\n", - "- **`tool_choice={\"type\": \"function\", \"function\": {\"name\": \"specific_function\"}}`**: Forces the model to call a specific function\n", - "\n", - "### Backend Compatibility\n", - "\n", - "Tool choice is fully supported with the **Xgrammar backend**, which is the default grammar backend (`--grammar-backend xgrammar`). However, it may not be fully supported with other backends such as `outlines`.\n", - "\n", - "### Example: Required Tool Choice" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from openai import OpenAI\n", - "from sglang.utils import wait_for_server, print_highlight, terminate_process\n", - "from sglang.test.doc_patch import launch_server_cmd\n", - "\n", - "# Start a new server session for tool choice examples\n", - "server_process_tool_choice, port_tool_choice = launch_server_cmd(\n", - " \"python3 -m sglang.launch_server --model-path Qwen/Qwen2.5-7B-Instruct --tool-call-parser qwen25 --host 0.0.0.0 --log-level warning\"\n", - ")\n", - "wait_for_server(\n", - " f\"http://localhost:{port_tool_choice}\", process=server_process_tool_choice\n", - ")\n", - "\n", - "# Initialize client for tool choice examples\n", - "client_tool_choice = OpenAI(\n", - " api_key=\"None\", base_url=f\"http://0.0.0.0:{port_tool_choice}/v1\"\n", - ")\n", - "model_name_tool_choice = client_tool_choice.models.list().data[0].id\n", - "\n", - "# Example with tool_choice=\"required\" - forces the model to call a tool\n", - "messages_required = [\n", - " {\"role\": \"user\", \"content\": \"Hello, what is the capital of France?\"}\n", - "]\n", - "\n", - "# Define tools\n", - "tools = [\n", - " {\n", - " \"type\": \"function\",\n", - " \"function\": {\n", - " \"name\": \"get_current_weather\",\n", - " \"description\": \"Get the current weather in a given location\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"city\": {\n", - " \"type\": \"string\",\n", - " \"description\": \"The city to find the weather for, e.g. 'San Francisco'\",\n", - " },\n", - " \"unit\": {\n", - " \"type\": \"string\",\n", - " \"description\": \"The unit to fetch the temperature in\",\n", - " \"enum\": [\"celsius\", \"fahrenheit\"],\n", - " },\n", - " },\n", - " \"required\": [\"city\", \"unit\"],\n", - " },\n", - " },\n", - " }\n", - "]\n", - "\n", - "response_required = client_tool_choice.chat.completions.create(\n", - " model=model_name_tool_choice,\n", - " messages=messages_required,\n", - " temperature=0,\n", - " max_tokens=1024,\n", - " tools=tools,\n", - " tool_choice=\"required\", # Force the model to call a tool\n", - ")\n", - "\n", - "print_highlight(\"Response with tool_choice='required':\")\n", - "print(\"Content:\", response_required.choices[0].message.content)\n", - "print(\"Tool calls:\", response_required.choices[0].message.tool_calls)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Example: Specific Function Choice\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Example with specific function choice - forces the model to call a specific function\n", - "messages_specific = [\n", - " {\"role\": \"user\", \"content\": \"What are the most attactive places in France?\"}\n", - "]\n", - "\n", - "response_specific = client_tool_choice.chat.completions.create(\n", - " model=model_name_tool_choice,\n", - " messages=messages_specific,\n", - " temperature=0,\n", - " max_tokens=1024,\n", - " tools=tools,\n", - " tool_choice={\n", - " \"type\": \"function\",\n", - " \"function\": {\"name\": \"get_current_weather\"},\n", - " }, # Force the model to call the specific get_current_weather function\n", - ")\n", - "\n", - "print_highlight(\"Response with specific function choice:\")\n", - "print(\"Content:\", response_specific.choices[0].message.content)\n", - "print(\"Tool calls:\", response_specific.choices[0].message.tool_calls)\n", - "\n", - "if response_specific.choices[0].message.tool_calls:\n", - " tool_call = response_specific.choices[0].message.tool_calls[0]\n", - " print_highlight(f\"Called function: {tool_call.function.name}\")\n", - " print_highlight(f\"Arguments: {tool_call.function.arguments}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "terminate_process(server_process_tool_choice)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Pythonic Tool Call Format (Llama-3.2 / Llama-3.3 / Llama-4)\n", - "\n", - "Some Llama models (such as Llama-3.2-1B, Llama-3.2-3B, Llama-3.3-70B, and Llama-4) support a \"pythonic\" tool call format, where the model outputs function calls as Python code, e.g.:\n", - "\n", - "```python\n", - "[get_current_weather(city=\"San Francisco\", state=\"CA\", unit=\"celsius\")]\n", - "```\n", - "\n", - "- The output is a Python list of function calls, with arguments as Python literals (not JSON).\n", - "- Multiple tool calls can be returned in the same list:\n", - "```python\n", - "[get_current_weather(city=\"San Francisco\", state=\"CA\", unit=\"celsius\"),\n", - " get_current_weather(city=\"New York\", state=\"NY\", unit=\"fahrenheit\")]\n", - "```\n", - "\n", - "For more information, refer to Meta’s documentation on [Zero shot function calling](https://github.com/meta-llama/llama-models/blob/main/models/llama4/prompt_format.md#zero-shot-function-calling---system-message).\n", - "\n", - "Note that this feature is still under development on Blackwell.\n", - "\n", - "### How to enable\n", - "- Launch the server with `--tool-call-parser pythonic`\n", - "- You may also specify --chat-template with the improved template for the model (e.g., `--chat-template=examples/chat_template/tool_chat_template_llama4_pythonic.jinja`).\n", - "This is recommended because the model expects a special prompt format to reliably produce valid pythonic tool call outputs. The template ensures that the prompt structure (e.g., special tokens, message boundaries like `<|eom|>`, and function call delimiters) matches what the model was trained or fine-tuned on. If you do not use the correct chat template, tool calling may fail or produce inconsistent results.\n", - "\n", - "#### Forcing Pythonic Tool Call Output Without a Chat Template\n", - "If you don't want to specify a chat template, you must give the model extremely explicit instructions in your messages to enforce pythonic output. For example, for `Llama-3.2-1B-Instruct`, you need:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import openai\n", - "\n", - "server_process, port = launch_server_cmd(\n", - " \" python3 -m sglang.launch_server --model-path meta-llama/Llama-3.2-1B-Instruct --tool-call-parser pythonic --tp 1 --log-level warning\" # llama-3.2-1b-instruct\n", - ")\n", - "wait_for_server(f\"http://localhost:{port}\", process=server_process)\n", - "\n", - "tools = [\n", - " {\n", - " \"type\": \"function\",\n", - " \"function\": {\n", - " \"name\": \"get_weather\",\n", - " \"description\": \"Get the current weather for a given location.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"location\": {\n", - " \"type\": \"string\",\n", - " \"description\": \"The name of the city or location.\",\n", - " }\n", - " },\n", - " \"required\": [\"location\"],\n", - " },\n", - " },\n", - " },\n", - " {\n", - " \"type\": \"function\",\n", - " \"function\": {\n", - " \"name\": \"get_tourist_attractions\",\n", - " \"description\": \"Get a list of top tourist attractions for a given city.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"city\": {\n", - " \"type\": \"string\",\n", - " \"description\": \"The name of the city to find attractions for.\",\n", - " }\n", - " },\n", - " \"required\": [\"city\"],\n", - " },\n", - " },\n", - " },\n", - "]\n", - "\n", - "\n", - "def get_messages():\n", - " return [\n", - " {\n", - " \"role\": \"system\",\n", - " \"content\": (\n", - " \"You are a travel assistant. \"\n", - " \"When asked to call functions, ALWAYS respond ONLY with a python list of function calls, \"\n", - " \"using this format: [func_name1(param1=value1, param2=value2), func_name2(param=value)]. \"\n", - " \"Do NOT use JSON, do NOT use variables, do NOT use any other format. \"\n", - " \"Here is an example:\\n\"\n", - " '[get_weather(location=\"Paris\"), get_tourist_attractions(city=\"Paris\")]'\n", - " ),\n", - " },\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": (\n", - " \"I'm planning a trip to Tokyo next week. What's the weather like and what are some top tourist attractions? \"\n", - " \"Propose parallel tool calls at once, using the python list of function calls format as shown above.\"\n", - " ),\n", - " },\n", - " ]\n", - "\n", - "\n", - "messages = get_messages()\n", - "\n", - "client = openai.Client(base_url=f\"http://localhost:{port}/v1\", api_key=\"xxxxxx\")\n", - "model_name = client.models.list().data[0].id\n", - "\n", - "\n", - "response_non_stream = client.chat.completions.create(\n", - " model=model_name,\n", - " messages=messages,\n", - " temperature=0,\n", - " top_p=0.9,\n", - " stream=False, # Non-streaming\n", - " tools=tools,\n", - ")\n", - "print_highlight(\"Non-stream response:\")\n", - "print_highlight(response_non_stream)\n", - "\n", - "response_stream = client.chat.completions.create(\n", - " model=model_name,\n", - " messages=messages,\n", - " temperature=0,\n", - " top_p=0.9,\n", - " stream=True,\n", - " tools=tools,\n", - ")\n", - "texts = \"\"\n", - "tool_calls = []\n", - "name = \"\"\n", - "arguments = \"\"\n", - "\n", - "for chunk in response_stream:\n", - " if chunk.choices[0].delta.content:\n", - " texts += chunk.choices[0].delta.content\n", - " if chunk.choices[0].delta.tool_calls:\n", - " tool_calls.append(chunk.choices[0].delta.tool_calls[0])\n", - "\n", - "print_highlight(\"Streaming Response:\")\n", - "print_highlight(\"==== Text ====\")\n", - "print_highlight(texts)\n", - "\n", - "print_highlight(\"==== Tool Call ====\")\n", - "for tool_call in tool_calls:\n", - " print_highlight(tool_call)\n", - "\n", - "terminate_process(server_process)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "> **Note:** \n", - "> The model may still default to JSON if it was heavily finetuned on that format. Prompt engineering (including examples) is the only way to increase the chance of pythonic output if you are not using a chat template." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## How to support a new model?\n", - "1. Update the TOOLS_TAG_LIST in sglang/srt/function_call_parser.py with the model’s tool tags. Currently supported tags include:\n", - "```\n", - "\tTOOLS_TAG_LIST = [\n", - "\t “<|plugin|>“,\n", - "\t ““,\n", - "\t “<|python_tag|>“,\n", - "\t “[TOOL_CALLS]”\n", - "\t]\n", - "```\n", - "2. Create a new detector class in sglang/srt/function_call_parser.py that inherits from BaseFormatDetector. The detector should handle the model’s specific function call format. For example:\n", - "```\n", - " class NewModelDetector(BaseFormatDetector):\n", - "```\n", - "3. Add the new detector to the MultiFormatParser class that manages all the format detectors." - ] - } - ], - "metadata": { - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/docs/advanced_features/vlm_query.ipynb b/docs/advanced_features/vlm_query.ipynb deleted file mode 100644 index 24bd7a90b..000000000 --- a/docs/advanced_features/vlm_query.ipynb +++ /dev/null @@ -1,379 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "0", - "metadata": {}, - "source": [ - "# Query VLM with Offline Engine\n", - "\n", - "This tutorial demonstrates how to use SGLang's **offline Engine API** to query VLMs. We will demonstrate usage with Qwen2.5-VL and Llama 4. This section demonstrates three different calling approaches:\n", - "\n", - "1. **Basic Call**: Directly pass images and text.\n", - "2. **Processor Output**: Use HuggingFace processor for data preprocessing.\n", - "3. **Precomputed Embeddings**: Pre-calculate image features to improve inference efficiency." - ] - }, - { - "cell_type": "markdown", - "id": "1", - "metadata": {}, - "source": [ - "## Understanding the Three Input Formats\n", - "\n", - "SGLang supports three ways to pass visual data, each optimized for different scenarios:\n", - "\n", - "### 1. **Raw Images** - Simplest approach\n", - "- Pass PIL Images, file paths, URLs, or base64 strings directly\n", - "- SGLang handles all preprocessing automatically\n", - "- Best for: Quick prototyping, simple applications\n", - "\n", - "### 2. **Processor Output** - For custom preprocessing\n", - "- Pre-process images with HuggingFace processor\n", - "- Pass the complete processor output dict with `format: \"processor_output\"`\n", - "- Best for: Custom image transformations, integration with existing pipelines\n", - "- Requirement: Must use `input_ids` instead of text prompt\n", - "\n", - "### 3. **Precomputed Embeddings** - For maximum performance\n", - "- Pre-calculate visual embeddings using the vision encoder\n", - "- Pass embeddings with `format: \"precomputed_embedding\"`\n", - "- Best for: Repeated queries on same images, caching, high-throughput serving\n", - "- Performance gain: Avoids redundant vision encoder computation (30-50% speedup)\n", - "\n", - "**Key Rule**: Within a single request, use only one format for all images. Don't mix formats.\n", - "\n", - "The examples below demonstrate all three approaches with both Qwen2.5-VL and Llama 4 models." - ] - }, - { - "cell_type": "markdown", - "id": "2", - "metadata": {}, - "source": [ - "## Querying Qwen2.5-VL Model" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3", - "metadata": {}, - "outputs": [], - "source": [ - "import nest_asyncio\n", - "\n", - "nest_asyncio.apply()\n", - "\n", - "import sglang.test.doc_patch # noqa: F401\n", - "\n", - "model_path = \"Qwen/Qwen2.5-VL-3B-Instruct\"\n", - "chat_template = \"qwen2-vl\"\n", - "example_image_url = \"https://raw.githubusercontent.com/sgl-project/sglang/main/examples/assets/example_image.png\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4", - "metadata": {}, - "outputs": [], - "source": [ - "from io import BytesIO\n", - "import requests\n", - "from PIL import Image\n", - "\n", - "from sglang.srt.parser.conversation import chat_templates\n", - "\n", - "image = Image.open(BytesIO(requests.get(example_image_url).content))\n", - "\n", - "conv = chat_templates[chat_template].copy()\n", - "conv.append_message(conv.roles[0], f\"What's shown here: {conv.image_token}?\")\n", - "conv.append_message(conv.roles[1], \"\")\n", - "conv.image_data = [image]\n", - "\n", - "print(\"Generated prompt text:\")\n", - "print(conv.get_prompt())\n", - "print(f\"\\nImage size: {image.size}\")\n", - "image" - ] - }, - { - "cell_type": "markdown", - "id": "5", - "metadata": {}, - "source": [ - "### Basic Offline Engine API Call" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6", - "metadata": {}, - "outputs": [], - "source": [ - "from sglang import Engine\n", - "\n", - "llm = Engine(model_path=model_path, chat_template=chat_template, log_level=\"warning\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7", - "metadata": {}, - "outputs": [], - "source": [ - "out = llm.generate(prompt=conv.get_prompt(), image_data=[image])\n", - "print(\"Model response:\")\n", - "print(out[\"text\"])" - ] - }, - { - "cell_type": "markdown", - "id": "8", - "metadata": {}, - "source": [ - "### Call with Processor Output\n", - "\n", - "Using a HuggingFace processor to preprocess text and images, and passing the `processor_output` directly into `Engine.generate`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9", - "metadata": {}, - "outputs": [], - "source": [ - "from transformers import AutoProcessor\n", - "\n", - "processor = AutoProcessor.from_pretrained(model_path, use_fast=True)\n", - "processor_output = processor(\n", - " images=[image], text=conv.get_prompt(), return_tensors=\"pt\"\n", - ")\n", - "\n", - "out = llm.generate(\n", - " input_ids=processor_output[\"input_ids\"][0].detach().cpu().tolist(),\n", - " image_data=[dict(processor_output, format=\"processor_output\")],\n", - ")\n", - "print(\"Response using processor output:\")\n", - "print(out[\"text\"])" - ] - }, - { - "cell_type": "markdown", - "id": "10", - "metadata": {}, - "source": [ - "### Call with Precomputed Embeddings\n", - "\n", - "You can pre-calculate image features to avoid repeated visual encoding processes." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "11", - "metadata": {}, - "outputs": [], - "source": [ - "from transformers import AutoProcessor\n", - "from transformers import Qwen2_5_VLForConditionalGeneration\n", - "\n", - "processor = AutoProcessor.from_pretrained(model_path, use_fast=True)\n", - "model = Qwen2_5_VLForConditionalGeneration.from_pretrained(model_path).eval()\n", - "vision = model.model.visual.cuda()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "12", - "metadata": {}, - "outputs": [], - "source": [ - "processor_output = processor(\n", - " images=[image], text=conv.get_prompt(), return_tensors=\"pt\"\n", - ")\n", - "\n", - "input_ids = processor_output[\"input_ids\"][0].detach().cpu().tolist()\n", - "\n", - "precomputed_embeddings = vision(\n", - " processor_output[\"pixel_values\"].cuda(), processor_output[\"image_grid_thw\"].cuda()\n", - ")\n", - "precomputed_embeddings = precomputed_embeddings.pooler_output\n", - "\n", - "multi_modal_item = dict(\n", - " processor_output,\n", - " format=\"precomputed_embedding\",\n", - " feature=precomputed_embeddings,\n", - ")\n", - "\n", - "out = llm.generate(input_ids=input_ids, image_data=[multi_modal_item])\n", - "print(\"Response using precomputed embeddings:\")\n", - "print(out[\"text\"])\n", - "\n", - "llm.shutdown()" - ] - }, - { - "cell_type": "markdown", - "id": "13", - "metadata": {}, - "source": [ - "## Querying Llama 4 Vision Model\n", - "\n", - "```python\n", - "model_path = \"meta-llama/Llama-4-Scout-17B-16E-Instruct\"\n", - "chat_template = \"llama-4\"\n", - "\n", - "from io import BytesIO\n", - "import requests\n", - "from PIL import Image\n", - "\n", - "from sglang.srt.parser.conversation import chat_templates\n", - "\n", - "# Download the same example image\n", - "image = Image.open(BytesIO(requests.get(example_image_url).content))\n", - "\n", - "conv = chat_templates[chat_template].copy()\n", - "conv.append_message(conv.roles[0], f\"What's shown here: {conv.image_token}?\")\n", - "conv.append_message(conv.roles[1], \"\")\n", - "conv.image_data = [image]\n", - "\n", - "print(\"Llama 4 generated prompt text:\")\n", - "print(conv.get_prompt())\n", - "print(f\"Image size: {image.size}\")\n", - "\n", - "image\n", - "```" - ] - }, - { - "cell_type": "markdown", - "id": "14", - "metadata": {}, - "source": [ - "### Llama 4 Basic Call\n", - "\n", - "Llama 4 requires more computational resources, so it's configured with multi-GPU parallelism (tp_size=4) and larger context length.\n", - "\n", - "```python\n", - "llm = Engine(\n", - " model_path=model_path,\n", - " enable_multimodal=True,\n", - " attention_backend=\"fa3\",\n", - " tp_size=4,\n", - " context_length=65536,\n", - ")\n", - "\n", - "out = llm.generate(prompt=conv.get_prompt(), image_data=[image])\n", - "print(\"Llama 4 response:\")\n", - "print(out[\"text\"])\n", - "```" - ] - }, - { - "cell_type": "markdown", - "id": "15", - "metadata": {}, - "source": [ - "### Call with Processor Output\n", - "\n", - "Using HuggingFace processor to preprocess data can reduce computational overhead during inference.\n", - "\n", - "```python\n", - "from transformers import AutoProcessor\n", - "\n", - "processor = AutoProcessor.from_pretrained(model_path, use_fast=True)\n", - "processor_output = processor(\n", - " images=[image], text=conv.get_prompt(), return_tensors=\"pt\"\n", - ")\n", - "\n", - "out = llm.generate(\n", - " input_ids=processor_output[\"input_ids\"][0].detach().cpu().tolist(),\n", - " image_data=[dict(processor_output, format=\"processor_output\")],\n", - ")\n", - "print(\"Response using processor output:\")\n", - "print(out)\n", - "```" - ] - }, - { - "cell_type": "markdown", - "id": "16", - "metadata": {}, - "source": [ - "### Call with Precomputed Embeddings\n", - "\n", - "```python\n", - "from transformers import AutoProcessor\n", - "from transformers import Llama4ForConditionalGeneration\n", - "\n", - "processor = AutoProcessor.from_pretrained(model_path, use_fast=True)\n", - "model = Llama4ForConditionalGeneration.from_pretrained(\n", - " model_path, torch_dtype=\"auto\"\n", - ").eval()\n", - "\n", - "vision = model.vision_model.cuda()\n", - "multi_modal_projector = model.multi_modal_projector.cuda()\n", - "\n", - "print(f'Image pixel values shape: {processor_output[\"pixel_values\"].shape}')\n", - "input_ids = processor_output[\"input_ids\"][0].detach().cpu().tolist()\n", - "\n", - "# Process image through vision encoder\n", - "image_outputs = vision(\n", - " processor_output[\"pixel_values\"].to(\"cuda\"), \n", - " aspect_ratio_ids=processor_output[\"aspect_ratio_ids\"].to(\"cuda\"),\n", - " aspect_ratio_mask=processor_output[\"aspect_ratio_mask\"].to(\"cuda\"),\n", - " output_hidden_states=False\n", - ")\n", - "image_features = image_outputs.last_hidden_state\n", - "\n", - "# Flatten image features and pass through multimodal projector\n", - "vision_flat = image_features.view(-1, image_features.size(-1))\n", - "precomputed_embeddings = multi_modal_projector(vision_flat)\n", - "\n", - "# Build precomputed embedding data item\n", - "mm_item = dict(\n", - " processor_output, \n", - " format=\"precomputed_embedding\", \n", - " feature=precomputed_embeddings\n", - ")\n", - "\n", - "# Use precomputed embeddings for efficient inference\n", - "out = llm.generate(input_ids=input_ids, image_data=[mm_item])\n", - "print(\"Llama 4 precomputed embedding response:\")\n", - "print(out[\"text\"])\n", - "```" - ] - } - ], - "metadata": { - "jupytext": { - "cell_metadata_filter": "-all", - "custom_cell_magics": "kql", - "encoding": "# -*- coding: utf-8 -*-", - "text_representation": { - "extension": ".py", - "format_name": "light", - "format_version": "1.5", - "jupytext_version": "1.16.1" - } - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/docs/basic_usage/deepseek_ocr.md b/docs/basic_usage/deepseek_ocr.md deleted file mode 100644 index 6f62713eb..000000000 --- a/docs/basic_usage/deepseek_ocr.md +++ /dev/null @@ -1,54 +0,0 @@ -# DeepSeek OCR (OCR-1 / OCR-2) - -DeepSeek OCR models are multimodal (image + text) models for OCR and document understanding. - -## Launch server - -```shell -python -m sglang.launch_server \ - --model-path deepseek-ai/DeepSeek-OCR-2 \ - --trust-remote-code \ - --host 0.0.0.0 \ - --port 30000 -``` - -> You can replace `deepseek-ai/DeepSeek-OCR-2` with `deepseek-ai/DeepSeek-OCR`. - -## Prompt examples - -Recommended prompts from the model card: - -``` - -<|grounding|>Convert the document to markdown. -``` - -``` - -Free OCR. -``` - -## OpenAI-compatible request example - -```python -import requests - -url = "http://localhost:30000/v1/chat/completions" - -data = { - "model": "deepseek-ai/DeepSeek-OCR-2", - "messages": [ - { - "role": "user", - "content": [ - {"type": "text", "text": "\n<|grounding|>Convert the document to markdown."}, - {"type": "image_url", "image_url": {"url": "https://example.com/your_image.jpg"}}, - ], - } - ], - "max_tokens": 512, -} - -response = requests.post(url, json=data) -print(response.text) -``` diff --git a/docs/basic_usage/deepseek_v3.md b/docs/basic_usage/deepseek_v3.md deleted file mode 100644 index 9770c2882..000000000 --- a/docs/basic_usage/deepseek_v3.md +++ /dev/null @@ -1,306 +0,0 @@ -# DeepSeek V3/V3.1/R1 Usage - -SGLang provides many optimizations specifically designed for the DeepSeek models, making it the inference engine recommended by the official [DeepSeek team](https://github.com/deepseek-ai/DeepSeek-V3/tree/main?tab=readme-ov-file#62-inference-with-sglang-recommended) from Day 0. - -This document outlines current optimizations for DeepSeek. -For an overview of the implemented features see the completed [Roadmap](https://github.com/sgl-project/sglang/issues/2591). - -## Launch DeepSeek V3.1/V3/R1 with SGLang - -To run DeepSeek V3.1/V3/R1 models, the recommended settings are as follows: - -| Weight Type | Configuration | -|------------|-------------------| -| **Full precision [FP8](https://huggingface.co/deepseek-ai/DeepSeek-R1-0528)**
*(recommended)* | 8 x H200 | -| | 8 x B200 | -| | 8 x MI300X | -| | 2 x 8 x H100/800/20 | -| | Xeon 6980P CPU | -| **Full precision ([BF16](https://huggingface.co/unsloth/DeepSeek-R1-0528-BF16))** (upcast from original FP8) | 2 x 8 x H200 | -| | 2 x 8 x MI300X | -| | 4 x 8 x H100/800/20 | -| | 4 x 8 x A100/A800 | -| **Quantized weights ([INT8](https://huggingface.co/meituan/DeepSeek-R1-Channel-INT8))** | 16 x A100/800 | -| | 32 x L40S | -| | Xeon 6980P CPU | -| | 4 x Atlas 800I A3 | -| **Quantized weights ([W4A8](https://huggingface.co/novita/Deepseek-R1-0528-W4AFP8))** | 8 x H20/100, 4 x H200 | -| **Quantized weights ([AWQ](https://huggingface.co/QuixiAI/DeepSeek-R1-0528-AWQ))** | 8 x H100/800/20 | -| | 8 x A100/A800 | -| **Quantized weights ([MXFP4](https://huggingface.co/amd/DeepSeek-R1-MXFP4-Preview))** | 8, 4 x MI355X/350X | -| **Quantized weights ([NVFP4](https://huggingface.co/nvidia/DeepSeek-R1-0528-NVFP4-v2))** | 8, 4 x B200 | - - - -```{important} -The official DeepSeek V3 is already in FP8 format, so you should not run it with any quantization arguments like `--quantization fp8`. -``` - -Detailed commands for reference: - -- [8 x H200](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#using-docker-recommended) -- [4 x B200, 8 x B200](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-one-b200-node) -- [8 x MI300X](../platforms/amd_gpu.md#running-deepseek-v3) -- [2 x 8 x H200](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-two-h2008-nodes-and-docker) -- [4 x 8 x A100](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-four-a1008-nodes) -- [8 x A100 (AWQ)](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-8-a100a800-with-awq-quantization) -- [16 x A100 (INT8)](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-16-a100a800-with-int8-quantization) -- [32 x L40S (INT8)](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-32-l40s-with-int8-quantization) -- [Xeon 6980P CPU](../platforms/cpu_server.md#example-running-deepseek-r1) -- [4 x Atlas 800I A3 (int8)](../platforms/ascend/ascend_npu_deepseek_example.md#running-deepseek-with-pd-disaggregation-on-4-x-atlas-800i-a3) - -### Download Weights -If you encounter errors when starting the server, ensure the weights have finished downloading. It's recommended to download them beforehand or restart multiple times until all weights are downloaded. Please refer to [DeepSeek V3](https://huggingface.co/deepseek-ai/DeepSeek-V3-Base#61-inference-with-deepseek-infer-demo-example-only) official guide to download the weights. - -### Launch with one node of 8 x H200 -Please refer to [the example](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#installation--launch). - -### Running examples on Multi-Node - -- [Deploying DeepSeek on GB200 NVL72 with PD and Large Scale EP](https://lmsys.org/blog/2025-06-16-gb200-part-1/) ([Part I](https://lmsys.org/blog/2025-06-16-gb200-part-1/), [Part II](https://lmsys.org/blog/2025-09-25-gb200-part-2/)) - Comprehensive guide on GB200 optimizations. - -- [Deploying DeepSeek with PD Disaggregation and Large-Scale Expert Parallelism on 96 H100 GPUs](https://lmsys.org/blog/2025-05-05-large-scale-ep/) - Guide on PD disaggregation and large-scale EP. - -- [Serving with two H20*8 nodes](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-two-h208-nodes). - -- [Best Practices for Serving DeepSeek-R1 on H20](https://lmsys.org/blog/2025-09-26-sglang-ant-group/) - Comprehensive guide on H20 optimizations, deployment and performance. - -- [Serving with two H200*8 nodes and docker](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-two-h2008-nodes-and-docker). - -- [Serving with four A100*8 nodes](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-four-a1008-nodes). - -## Optimizations - -### Multi-head Latent Attention (MLA) Throughput Optimizations - -**Description**: [MLA](https://arxiv.org/pdf/2405.04434) is an innovative attention mechanism introduced by the DeepSeek team, aimed at improving inference efficiency. SGLang has implemented specific optimizations for this, including: - -- **Weight Absorption**: By applying the associative law of matrix multiplication to reorder computation steps, this method balances computation and memory access and improves efficiency in the decoding phase. - -- **MLA Attention Backends**: Currently SGLang supports different optimized MLA attention backends, including [FlashAttention3](https://github.com/Dao-AILab/flash-attention), [Flashinfer](https://docs.flashinfer.ai/api/attention.html#flashinfer-mla), [FlashMLA](https://github.com/deepseek-ai/FlashMLA), [CutlassMLA](https://github.com/sgl-project/sglang/pull/5390), **TRTLLM MLA** (optimized for Blackwell architecture), and [Triton](https://github.com/triton-lang/triton) backends. The default FA3 provides good performance across wide workloads. - -- **FP8 Quantization**: W8A8 FP8 and KV Cache FP8 quantization enables efficient FP8 inference. Additionally, we have implemented Batched Matrix Multiplication (BMM) operator to facilitate FP8 inference in MLA with weight absorption. - -- **CUDA Graph & Torch.compile**: Both MLA and Mixture of Experts (MoE) are compatible with CUDA Graph and Torch.compile, which reduces latency and accelerates decoding speed for small batch sizes. - -- **Chunked Prefix Cache**: Chunked prefix cache optimization can increase throughput by cutting prefix cache into chunks, processing them with multi-head attention and merging their states. Its improvement can be significant when doing chunked prefill on long sequences. Currently this optimization is only available for FlashAttention3 backend. - -Overall, with these optimizations, we have achieved up to **7x** acceleration in output throughput compared to the previous version. - -

- Multi-head Latent Attention for DeepSeek Series Models -

- -**Usage**: MLA optimization is enabled by default. - -**Reference**: Check [Blog](https://lmsys.org/blog/2024-09-04-sglang-v0-3/#deepseek-multi-head-latent-attention-mla-throughput-optimizations) and [Slides](https://github.com/sgl-project/sgl-learning-materials/blob/main/slides/lmsys_1st_meetup_deepseek_mla.pdf) for more details. - -### Data Parallelism Attention - -**Description**: This optimization involves data parallelism (DP) for the MLA attention mechanism of DeepSeek Series Models, which allows for a significant reduction in the KV cache size, enabling larger batch sizes. Each DP worker independently handles different types of batches (prefill, decode, idle), which are then synchronized before and after processing through the Mixture-of-Experts (MoE) layer. If you do not use DP attention, KV cache will be duplicated among all TP ranks. - -

- Data Parallelism Attention for DeepSeek Series Models -

- -With data parallelism attention enabled, we have achieved up to **1.9x** decoding throughput improvement compared to the previous version. - -

- Data Parallelism Attention Performance Comparison -

- -**Usage**: -- Append `--enable-dp-attention --tp 8 --dp 8` to the server arguments when using 8 H200 GPUs. This optimization improves peak throughput in high batch size scenarios where the server is limited by KV cache capacity. -- DP and TP attention can be flexibly combined. For example, to deploy DeepSeek-V3/R1 on 2 nodes with 8 H100 GPUs each, you can specify `--enable-dp-attention --tp 16 --dp 2`. This configuration runs attention with 2 DP groups, each containing 8 TP GPUs. - -```{caution} -Data parallelism attention is not recommended for low-latency, small-batch use cases. It is optimized for high-throughput scenarios with large batch sizes. -``` - -**Reference**: Check [Blog](https://lmsys.org/blog/2024-12-04-sglang-v0-4/#data-parallelism-attention-for-deepseek-models). - -### Multi-Node Tensor Parallelism - -**Description**: For users with limited memory on a single node, SGLang supports serving DeepSeek Series Models, including DeepSeek V3, across multiple nodes using tensor parallelism. This approach partitions the model parameters across multiple GPUs or nodes to handle models that are too large for one node's memory. - -**Usage**: Check [here](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-two-h2008-nodes-and-docker) for usage examples. - -### Block-wise FP8 - -**Description**: SGLang implements block-wise FP8 quantization with two key optimizations: - -- **Activation**: E4M3 format using per-token-per-128-channel sub-vector scales with online casting. - -- **Weight**: Per-128x128-block quantization for better numerical stability. - -- **DeepGEMM**: The [DeepGEMM](https://github.com/deepseek-ai/DeepGEMM) kernel library optimized for FP8 matrix multiplications. - -**Usage**: The activation and weight optimization above are turned on by default for DeepSeek V3 models. DeepGEMM is enabled by default on NVIDIA Hopper/Blackwell GPUs and disabled by default on other devices. DeepGEMM can also be manually turned off by setting the environment variable `SGLANG_ENABLE_JIT_DEEPGEMM=0`. - -```{tip} -Before serving the DeepSeek model, precompile the DeepGEMM kernels to improve first-run performance. The precompilation process typically takes around 10 minutes to complete. -``` - -```bash -python3 -m sglang.compile_deep_gemm --model deepseek-ai/DeepSeek-V3 --tp 8 --trust-remote-code -``` - -### Multi-token Prediction -**Description**: SGLang implements DeepSeek V3 Multi-Token Prediction (MTP) based on [EAGLE speculative decoding](https://docs.sglang.io/advanced_features/speculative_decoding.html#EAGLE-Decoding). With this optimization, the decoding speed can be improved by **1.8x** for batch size 1 and **1.5x** for batch size 32 respectively on H200 TP8 setting. - -**Usage**: -Add `--speculative-algorithm EAGLE`. Other flags, like `--speculative-num-steps`, `--speculative-eagle-topk` and `--speculative-num-draft-tokens` are optional. For example: -``` -python3 -m sglang.launch_server \ - --model-path deepseek-ai/DeepSeek-V3-0324 \ - --speculative-algorithm EAGLE \ - --trust-remote-code \ - --tp 8 -``` -- The default configuration for DeepSeek models is `--speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4`. The best configuration for `--speculative-num-steps`, `--speculative-eagle-topk` and `--speculative-num-draft-tokens` can be searched with [bench_speculative.py](https://github.com/sgl-project/sglang/blob/main/scripts/playground/bench_speculative.py) script for given batch size. The minimum configuration is `--speculative-num-steps 1 --speculative-eagle-topk 1 --speculative-num-draft-tokens 2`, which can achieve speedup for larger batch sizes. -- Most MLA attention backends fully support MTP usage. See [MLA Backends](../advanced_features/attention_backend.md#mla-backends) for details. - -```{note} -To enable DeepSeek MTP for large batch sizes (>48), you need to adjust some parameters (Reference [this discussion](https://github.com/sgl-project/sglang/issues/4543#issuecomment-2737413756)): -- Adjust `--max-running-requests` to a larger number. The default value is `48` for MTP. For larger batch sizes, you should increase this value beyond the default value. -- Set `--cuda-graph-bs`. It's a list of batch sizes for cuda graph capture. The [default captured batch sizes for speculative decoding](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/server_args.py#L888-L895) is 48. You can customize this by including more batch sizes. -``` - -```{tip} -To enable the experimental overlap scheduler for EAGLE speculative decoding, set the environment variable `SGLANG_ENABLE_SPEC_V2=1`. This can improve performance by enabling overlap scheduling between draft and verification stages. -``` - - -### Reasoning Content for DeepSeek R1 & V3.1 - -See [Reasoning Parser](https://docs.sglang.io/advanced_features/separate_reasoning.html) and [Thinking Parameter for DeepSeek V3.1](https://docs.sglang.io/basic_usage/openai_api_completions.html#Example:-DeepSeek-V3-Models). - - -### Function calling for DeepSeek Models - -Add arguments `--tool-call-parser deepseekv3` and `--chat-template ./examples/chat_template/tool_chat_template_deepseekv3.jinja`(recommended) to enable this feature. For example (running on 1 * H20 node): - -``` -python3 -m sglang.launch_server \ - --model deepseek-ai/DeepSeek-V3-0324 \ - --tp 8 \ - --port 30000 \ - --host 0.0.0.0 \ - --mem-fraction-static 0.9 \ - --tool-call-parser deepseekv3 \ - --chat-template ./examples/chat_template/tool_chat_template_deepseekv3.jinja -``` - -Sample Request: - -``` -curl "http://127.0.0.1:30000/v1/chat/completions" \ --H "Content-Type: application/json" \ --d '{"temperature": 0, "max_tokens": 100, "model": "deepseek-ai/DeepSeek-V3-0324", "tools": [{"type": "function", "function": {"name": "query_weather", "description": "Get weather of a city, the user should supply a city first", "parameters": {"type": "object", "properties": {"city": {"type": "string", "description": "The city, e.g. Beijing"}}, "required": ["city"]}}}], "messages": [{"role": "user", "content": "How'\''s the weather like in Qingdao today"}]}' -``` - -Expected Response - -``` -{"id":"6501ef8e2d874006bf555bc80cddc7c5","object":"chat.completion","created":1745993638,"model":"deepseek-ai/DeepSeek-V3-0324","choices":[{"index":0,"message":{"role":"assistant","content":null,"reasoning_content":null,"tool_calls":[{"id":"0","index":null,"type":"function","function":{"name":"query_weather","arguments":"{\"city\": \"Qingdao\"}"}}]},"logprobs":null,"finish_reason":"tool_calls","matched_stop":null}],"usage":{"prompt_tokens":116,"total_tokens":138,"completion_tokens":22,"prompt_tokens_details":null}} - -``` -Sample Streaming Request: -``` -curl "http://127.0.0.1:30000/v1/chat/completions" \ --H "Content-Type: application/json" \ --d '{"temperature": 0, "max_tokens": 100, "model": "deepseek-ai/DeepSeek-V3-0324","stream":true,"tools": [{"type": "function", "function": {"name": "query_weather", "description": "Get weather of a city, the user should supply a city first", "parameters": {"type": "object", "properties": {"city": {"type": "string", "description": "The city, e.g. Beijing"}}, "required": ["city"]}}}], "messages": [{"role": "user", "content": "How'\''s the weather like in Qingdao today"}]}' -``` -Expected Streamed Chunks (simplified for clarity): -``` -data: {"choices":[{"delta":{"tool_calls":[{"function":{"arguments":"{\""}}]}}]} -data: {"choices":[{"delta":{"tool_calls":[{"function":{"arguments":"city"}}]}}]} -data: {"choices":[{"delta":{"tool_calls":[{"function":{"arguments":"\":\""}}]}}]} -data: {"choices":[{"delta":{"tool_calls":[{"function":{"arguments":"Q"}}]}}]} -data: {"choices":[{"delta":{"tool_calls":[{"function":{"arguments":"ing"}}]}}]} -data: {"choices":[{"delta":{"tool_calls":[{"function":{"arguments":"dao"}}]}}]} -data: {"choices":[{"delta":{"tool_calls":[{"function":{"arguments":"\"}"}}]}}]} -data: {"choices":[{"delta":{"tool_calls":null}}], "finish_reason": "tool_calls"} -data: [DONE] -``` -The client needs to concatenate all arguments fragments to reconstruct the complete tool call: -``` -{"city": "Qingdao"} -``` - -```{important} -1. Use a lower `"temperature"` value for better results. -2. To receive more consistent tool call results, it is recommended to use `--chat-template examples/chat_template/tool_chat_template_deepseekv3.jinja`. It provides an improved unified prompt. -``` - - -### Thinking Budget for DeepSeek R1 - -In SGLang, we can implement thinking budget with `CustomLogitProcessor`. - -Launch a server with `--enable-custom-logit-processor` flag on. - -``` -python3 -m sglang.launch_server --model deepseek-ai/DeepSeek-R1 --tp 8 --port 30000 --host 0.0.0.0 --mem-fraction-static 0.9 --disable-cuda-graph --reasoning-parser deepseek-r1 --enable-custom-logit-processor -``` - -Sample Request: - -```python -import openai -from rich.pretty import pprint -from sglang.srt.sampling.custom_logit_processor import DeepSeekR1ThinkingBudgetLogitProcessor - - -client = openai.Client(base_url="http://127.0.0.1:30000/v1", api_key="*") -response = client.chat.completions.create( - model="deepseek-ai/DeepSeek-R1", - messages=[ - { - "role": "user", - "content": "Question: Is Paris the Capital of France?", - } - ], - max_tokens=1024, - extra_body={ - "custom_logit_processor": DeepSeekR1ThinkingBudgetLogitProcessor().to_str(), - "custom_params": { - "thinking_budget": 512, - }, - }, -) -pprint(response) -``` - -## FAQ - -**Q: Model loading is taking too long, and I'm encountering an NCCL timeout. What should I do?** - -A: If you're experiencing extended model loading times and an NCCL timeout, you can try increasing the timeout duration. Add the argument `--dist-timeout 3600` when launching your model. This will set the timeout to one hour, which often resolves the issue. diff --git a/docs/basic_usage/deepseek_v32.md b/docs/basic_usage/deepseek_v32.md deleted file mode 100644 index 7bd0a447b..000000000 --- a/docs/basic_usage/deepseek_v32.md +++ /dev/null @@ -1,476 +0,0 @@ -# DeepSeek V3.2/GLM-5 Usage - -DeepSeek-V3.2 model family equips DeepSeek-V3.1-Terminus with DeepSeek Sparse Attention (DSA) through continued training. With DSA, a fine-grained sparse attention mechanism powered by a lightning indexer, DeepSeek-V3.2 achieves efficiency improvements in long-context scenarios. - - -Note: This document is originally written for the usage of [DeepSeek-V3.2-Exp](https://huggingface.co/deepseek-ai/DeepSeek-V3.2-Exp) model. The usage of [DeepSeek-V3.2](https://huggingface.co/deepseek-ai/DeepSeek-V3.2) or [DeepSeek-V3.2-Speciale](https://huggingface.co/deepseek-ai/DeepSeek-V3.2-Speciale) is the same as DeepSeek-V3.2-Exp except for the tool call parser. [GLM-5](https://huggingface.co/zai-org/GLM-5) model also applies DSA (DeepSeek Sparse Attention) structure, so it can share most of the usage here, except for the reasoning parser and tool call parser. - - -## Installation - -### Docker - -```bash -# H200/B200 -docker pull lmsysorg/sglang:latest - -# MI350/MI355 -docker pull lmsysorg/sglang:v0.5.8-rocm700-mi35x - -# MI300 -# v0.5.8-rocm700-mi30x does not include PR #17504. Prefer the newest MI30x ROCm -# image tag from Docker Hub when available, or build from source (below). -docker pull lmsysorg/sglang:v0.5.8-rocm700-mi30x - - -# NPUs -docker pull lmsysorg/sglang:dsv32-a2 -docker pull lmsysorg/sglang:dsv32-a3 -``` - -### Build From Source - -```bash -# Install SGLang -git clone https://github.com/sgl-project/sglang -cd sglang -pip3 install pip --upgrade -pip3 install -e "python" -``` - -## Launch DeepSeek V3.2/GLM-5 with SGLang - -To serve [DeepSeek-V3.2-Exp](https://huggingface.co/deepseek-ai/DeepSeek-V3.2-Exp) on 8xH200/B200 GPUs: - -```bash -# Launch with TP + DP (Recommended) -python -m sglang.launch_server --model deepseek-ai/DeepSeek-V3.2-Exp --tp 8 --dp 8 --enable-dp-attention - -# Launch with EP + DP -python -m sglang.launch_server --model deepseek-ai/DeepSeek-V3.2-Exp --tp 8 --ep 8 --dp 8 --enable-dp-attention - -# Launch with Pure TP -python -m sglang.launch_server --model deepseek-ai/DeepSeek-V3.2-Exp --tp 8 - -# Launch with TP on MI30x/MI35x -python3 -m sglang.launch_server --model deepseek-ai/DeepSeek-V3.2-Exp --tp 8 --dsa-prefill-backend tilelang --dsa-decode-backend tilelang -``` - -To serve GLM-5, just replace the `--model` argument with `zai-org/GLM-5-FP8`. - -### Configuration Tips -- **DP Attention**: To enable [DP Attention](../advanced_features/dp_dpa_smg_guide.md), please include `--enable-dp-attention --dp ` in command. DP Attention is better for large concurrency scenarios. -- **TP Attention**: Launching with TP attention is also supported. TP attention is better for low latency scenarios. -- **Short-sequence MHA prefill (adaptive)**: For short prefill sequences (default threshold: **2048 tokens**), the DSA backend uses standard MHA automatically (no extra flags). On H200 (SM90) this path uses the FlashAttention variable-length kernel; on B200 (SM100) it uses TRT-LLM ragged MHA. MHA uses `MHA_ONE_SHOT` for best performance, which computes multi-head attention over all tokens (both cached prefix and newly extended tokens) in a single kernel invocation, avoiding the overhead of chunked KV cache processing. This achieves optimal throughput for short sequences where total sequence length fits within the chunk capacity limit. -- **MHA prefill threshold relaxation**: To apply MHA attention to requests longer than 2048 tokens, please set the flag `SGLANG_DSA_PREFILL_DENSE_ATTN_KV_LEN_THRESHOLD` to a value larger than 2048 (`SGLANG_NSA_PREFILL_DENSE_ATTN_KV_LEN_THRESHOLD` is a deprecated alias). As threshold grows larger, the prefill performance can be improved, but at the cost of potential accuracy drop. -- **Choices of Attention Kernels**: The attention backend is automatically set to `dsa` attention backend for DeepSeek V3.2 model (the deprecated `nsa` alias also works). In this backend, different kernels for sparse prefilling/decoding are implemented, which can be specified by `--dsa-prefill-backend` and `--dsa-decode-backend` server arguments (the deprecated `--nsa-prefill-backend` / `--nsa-decode-backend` aliases also work). The choices of dsa prefill/decode attention kernels include: - - `flashmla_sparse`: `flash_mla_sparse_fwd` kernel from `flash_mla` library. Can run on both Hopper and Blackwell GPUs. It requires bf16 q, kv inputs. - - `flashmla_kv`: `flash_mla_with_kvcache` kernel from `flash_mla` library. Can run on both Hopper and Blackwell GPUs. It requires bf16 q, fp8 k_cache inputs. - - `flashmla_auto`: enables automatic selection of either `flashmla_sparse` or `flashmla_kv` kernel for prefill based on KV cache dtype, hardware, and heuristics. With BF16 KV cache, `flashmla_sparse` is always used on both Hopper and Blackwell. With FP8 KV cache: On Hopper (SM90), it unconditionally uses `flashmla_kv`; On Blackwell (SM100), it uses `flashmla_sparse` when `total_kv_tokens < total_q_tokens * 512`, otherwise falls back to `flashmla_kv`. The heuristics may need to be tuned if the performance of either kernel changes significantly. - - `fa3`: `flash_attn_with_kvcache` kernel from `flash_attn` library. Can only run on Hopper GPUs. It requires bf16 q, kv inputs. - - `tilelang`: `tilelang` implementation that can run on GPU, HPU and NPU. - - `aiter`: Aiter kernel on AMD HPUs. Can only be used as decode kernel. - - `trtllm`: `trtllm-mla` sparse kernel from flashinfer library. Only run on blackwell GPUs. It requires q,k,v to be uniformly bf16 or fp8_e4m3 format. - - On the basis of performance benchmarks, the default configuration of DSA kernels on Hopper and Blackwell are set as follows : - - Bfloat 16 kv cache: On Hopper, `flashmla_sparse` prefill attention, `fa3` decode attention; On Blackwell, `flashmla_sparse` prefill attention, `trtllm` decode attention - - Float8_e4m3fn KV cache: On Hopper, `flashmla_kv` prefill attention, `flashmla_kv` decode attention; On Blackwell, `trtllm` prefill attention and `trtllm` decode attention. -- **Index Cache**: Introduce in [this paper](https://arxiv.org/abs/2603.12201), IndexCache improves speed by reusing the result of indexer across different layers, only at cost of negligible accuracy loss. For **GLM-5** model, we recommend appending `--json-model-override-args '{"index_topk_pattern": "FFSFSSSFSSFFFSSSFFFSFSSSSSSFFSFFSFFSSFFFFFFSFFFFFSFFSSSSSSFSFFFSFSSSFSFFSFFSSS"}'` to command for better tradeoff between speedup and performance. - -## Multi-token Prediction -SGLang implements Multi-Token Prediction (MTP) for DeepSeek V3.2 based on [EAGLE speculative decoding](https://docs.sglang.io/advanced_features/speculative_decoding.html#EAGLE-Decoding). With this optimization, the decoding speed can be improved significantly on small batch sizes. Please look at [this PR](https://github.com/sgl-project/sglang/pull/11652) for more information. - -Example usage with DP Attention: -```bash -python -m sglang.launch_server --model deepseek-ai/DeepSeek-V3.2-Exp --tp 8 --dp 8 --enable-dp-attention --speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 -``` - -Example usage with Pure TP: -```bash -python -m sglang.launch_server --model deepseek-ai/DeepSeek-V3.2-Exp --tp 8 --speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 -``` - -- The best configuration for `--speculative-num-steps`, `--speculative-eagle-topk` and `--speculative-num-draft-tokens` can be searched with [bench_speculative.py](https://github.com/sgl-project/sglang/blob/main/scripts/playground/bench_speculative.py) script for given batch size. The minimum configuration is `--speculative-num-steps 1 --speculative-eagle-topk 1 --speculative-num-draft-tokens 2`, which can achieve speedup for larger batch sizes. -- The default value of `--max-running-requests` is set to `48` for MTP. For larger batch sizes, this value should be increased beyond the default value. - -```{tip} -To enable overlap scheduler for EAGLE speculative decoding, we recommend setting the environment variable `SGLANG_ENABLE_SPEC_V2=1`. This can improve performance by enabling overlap scheduling between draft and verification stages. -``` - - -## Function Calling and Reasoning Parser -The usage of function calling and reasoning parser is the same as DeepSeek V3.1. Please refer to [Reasoning Parser](https://docs.sglang.io/advanced_features/separate_reasoning.html) and [Tool Parser](https://docs.sglang.io/advanced_features/tool_parser.html) documents. - -To launch `DeepSeek-V3.2-Exp` with function calling and reasoning parser: -> Note: It is recommended to specify the chat-template, ensuring that you are within the sglang's root directory. -```bash -python3 -m sglang.launch_server \ - --model-path deepseek-ai/DeepSeek-V3.2-Exp \ - --trust-remote-code \ - --tp-size 8 --dp-size 8 --enable-dp-attention \ - --tool-call-parser deepseekv31 \ - --reasoning-parser deepseek-v3 \ - --chat-template ./examples/chat_template/tool_chat_template_deepseekv32.jinja -``` - -To launch `DeepSeek-V3.2` with function calling and reasoning parser: -```bash -python3 -m sglang.launch_server \ - --model-path deepseek-ai/DeepSeek-V3.2 \ - --trust-remote-code \ - --tp-size 8 --dp-size 8 --enable-dp-attention \ - --tool-call-parser deepseekv32 \ - --reasoning-parser deepseek-v3 -``` - -`DeepSeek-V3.2-Speciale` does not support tool calling, so it can only be launched with the reasoning parser: -```bash -python3 -m sglang.launch_server \ - --model-path deepseek-ai/DeepSeek-V3.2-Speciale \ - --trust-remote-code \ - --tp-size 8 --dp-size 8 --enable-dp-attention \ - --reasoning-parser deepseek-v3 -``` - -To launch `GLM-5` with function calling and reasoning parser: -```bash -python -m sglang.launch_server \ - --model zai-org/GLM-5-FP8 \ - --tp-size 8 --dp-size 8 --enable-dp-attention \ - --tool-call-parser glm47 \ - --reasoning-parser glm45 \ -``` - -## NVFP4 Checkpoint - -To launch deepseek v3.2 [NVFP4 checkpoint](https://huggingface.co/nvidia/DeepSeek-V3.2-NVFP4) on Blackwell devices, the user needs to specify the quantization method as `modelopt_fp4`, and moe runner backend as one of `flashinfer_trtllm`(recommended), `flashinfer_cutlass` and `flashinfer_cutedsl`. Any other usage (parallelism, reasoning parser, ...) is the same as FP8 checkpoint. - -An example launching command can be: -```bash -python -m sglang.launch_server --model nvidia/DeepSeek-V3.2-NVFP4 --tp 4 --quantization modelopt_fp4 --moe-runner-backend flashinfer_trtllm --tool-call-parser deepseekv32 --reasoning-parser deepseek-v3 -``` - -## PD Disaggregation - -Prefill Command: -```bash -python -m sglang.launch_server \ - --model-path deepseek-ai/DeepSeek-V3.2-Exp \ - --disaggregation-mode prefill \ - --host $LOCAL_IP \ - --port $PORT \ - --tp 8 \ - --dp 8 \ - --enable-dp-attention \ - --dist-init-addr ${HOST}:${DIST_PORT} \ - --trust-remote-code \ - --disaggregation-bootstrap-port 8998 \ - --mem-fraction-static 0.9 \ -``` - -Decode command: -```bash -python -m sglang.launch_server \ - --model-path deepseek-ai/DeepSeek-V3.2-Exp \ - --disaggregation-mode decode \ - --host $LOCAL_IP \ - --port $PORT \ - --tp 8 \ - --dp 8 \ - --enable-dp-attention \ - --dist-init-addr ${HOST}:${DIST_PORT} \ - --trust-remote-code \ - --mem-fraction-static 0.9 \ -``` - -Router command: -```bash -python -m sglang_router.launch_router --pd-disaggregation \ - --prefill $PREFILL_ADDR 8998 \ - --decode $DECODE_ADDR \ - --host 127.0.0.1 \ - --port 8000 \ -``` - -If you need more advanced deployment methods or production-ready deployment methods, such as RBG or LWS-based deployment, please refer to [references/multi_node_deployment/rbg_pd/deepseekv32_pd.md](../references/multi_node_deployment/rbg_pd/deepseekv32_pd.md). Additionally, you can also find startup commands for DeepEP-based EP parallelism in the aforementioned documentation. - - -## Benchmarking Results - -### Accuracy Test with `gsm8k` -A simple accuracy benchmark can be tested with `gsm8k` dataset: -```bash -python3 benchmark/gsm8k/bench_sglang.py --num-shots 8 --num-questions 1319 --parallel 1319 -``` - -The result is 0.956, which matches our expectation: -```bash -Accuracy: 0.956 -Invalid: 0.000 -Latency: 25.109 s -Output throughput: 5226.235 token/s -``` - -To test long-context accuracy, run gsm8k with `--num-shots 20`. The results are very close to the 8 shots results: -``` -Accuracy: 0.956 -Invalid: 0.000 -Latency: 29.545 s -Output throughput: 4418.617 token/s -``` - - -### Accuracy Test with `gpqa-diamond` - -Accuracy benchmark on long context can be tested on GPQA-diamond dataset with long output tokens and thinking enabled: -```bash -python3 -m sglang.test.run_eval --port 30000 --eval-name gpqa --num-examples 198 --max-tokens 128000 --repeat 8 --thinking-mode deepseek-v3 -``` - -The mean accuracy over 8 runs shows 0.797, which matches the number 0.799 in official tech report. -```bash -Repeat: 8, mean: 0.797 -Scores: ['0.808', '0.798', '0.808', '0.798', '0.783', '0.788', '0.803', '0.793'] -``` - -For DeepSeek V3.2, DeepSeek recommends setting the sampling parameters to temperature = 1.0, top_p = 0.95: - -```bash -python3 -m sglang.test.run_eval --port 30000 --eval-name gpqa --num-examples 198 --max-tokens 128000 --repeat 8 --top-p 0.95 --temperature 1.0 --thinking-mode deepseek-v3 - -Repeat: 8, mean: 0.840 -Scores: ['0.848', '0.808', '0.848', '0.838', '0.879', '0.813', '0.838', '0.848'] -``` -which matches the official score, 0.824, as reported in the [DeepSeek-V3.2 technical report](https://huggingface.co/deepseek-ai/DeepSeek-V3.2/blob/main/assets/paper.pdf). - -### Accuracy Test with `aime 2025` - -Prepare the environment by installing NeMo-Skills in the docker or your own virtual environment: - - ``` - pip install git+https://github.com/NVIDIA/NeMo-Skills.git --ignore-installed blinker - ``` - -Then launch the SGLang server: -``` -python -m sglang.launch_server --model deepseek-ai/DeepSeek-V3.2-Exp --tp 8 --dp 8 --enable-dp-attention -``` - -**For `DeepSeek-V3.2` and `DeepSeek-V3.2-Speciale`**: - -``` -python3 -m sglang.launch_server --model-path deepseek-ai/DeepSeek-V3.2 --trust-remote-code --tp-size 8 --dp-size 8 --enable-dp-attention --tool-call-parser deepseekv32 --reasoning-parser deepseek-v3 -``` - -Run the following script to evaluate AIME 2025: -``` -#! /bin/bash -export NEMO_SKILLS_DISABLE_UNCOMMITTED_CHANGES_CHECK=1 - -ns prepare_data aime25 - -PORT=30000 -BACKEND=sglang -MODEL="deepseek-ai/DeepSeek-V3.2-Exp" # Should be changed to the model name -MODEL_NAME="dsv32-fp8" - -echo "Starting AIME25 evaluation with model $MODEL on port $PORT using backend $BACKEND..." -ns eval \ - --benchmarks=aime25:4 \ - --server_type=$BACKEND \ - --model=$MODEL \ - --server_address=http://localhost:${PORT}/v1 \ - --output_dir=nemo_skills_aime25_${MODEL_NAME}_output_${BACKEND}_$(date +%Y%m%d_%H%M%S) \ - ++chat_template_kwargs.thinking=true \ - ++inference.temperature=1.0 \ - ++inference.top_p=0.95 \ - ++inference.tokens_to_generate=64000 - # ++inference.tokens_to_generate=120000 for Speciale model -``` - -Test results (8*B200): - -DeepSeek-V3.2-Exp: - -| evaluation_mode | num_entries | avg_tokens | gen_seconds | symbolic_correct | no_answer | -|--------------------|-------------|------------|-------------|-----------------------|-----------| -| pass@1[avg-of-4] | 30 | 15040 | 1673 | 87.50% ± 1.67% | 0.00% | -| majority@4 | 30 | 15040 | 1673 | 90.00% | 0.00% | -| pass@4 | 30 | 15040 | 1673 | 90.00% | 0.00% | - - -DeepSeek-V3.2: -| evaluation_mode | num_entries | avg_tokens | gen_seconds | symbolic_correct | no_answer | -|--------------------|-------------|------------|-------------|-----------------------|-----------| -| pass@1[avg-of-4] | 30 | 13550 | 1632 | 92.50% ± 1.67% | 0.00% | -| majority@4 | 30 | 13550 | 1632 | 94.71% | 0.00% | -| pass@4 | 30 | 13550 | 1632 | 96.67% | 0.00% | - - -DeepSeek-V3.2-Speciale: -| evaluation_mode | num_entries | avg_tokens | gen_seconds | symbolic_correct | no_answer | -|--------------------|-------------|------------|-------------|-----------------------|-----------| -| pass@1[avg-of-4] | 30 | 24155 | 3583 | 95.00% ± 1.92% | 0.00% | -| majority@4 | 30 | 24155 | 3583 | 95.83% | 0.00% | -| pass@4 | 30 | 24155 | 3583 | 100.00% | 0.00% | - - - -## DSA long sequence context parallel optimization(experimental) - -**Note: This feature is only verified on Hopper machines** - -For context parallel in DeepSeek V3.2 model, we provide two different modes of splitting tokens, which can be controlled with argument `--dsa-prefill-cp-mode` (the deprecated `--nsa-prefill-cp-mode` alias also works). - -### In sequence splitting - -The first mode can be enabled by `--dsa-prefill-cp-mode in-seq-split`. This mode implements context parallel for DSA by splitting the sequence uniformly between context parallel ranks. At attention stage, each cp rank computes the indexer results of sharded sequence, and collects the whole kv cache through all gather operator. Add `attn_cp_size` for communication group for context parallel. - -Note that the in-sequence splitting mode has the following restrictions: -- The batch size is restricted to 1 for prefill batches -- `moe_dense_tp_size=1`, `moe_a2a_backend = "deepep"` -- To ensure `cp_size > 1`, the passed in `tp_size` must be larger than `dp_size` - -For more details, please refer to PR https://github.com/sgl-project/sglang/pull/12065. - -Example: -```bash -# In-seq splitting mode launched with EP + DP -python -m sglang.launch_server --model deepseek-ai/DeepSeek-V3.2-Exp --tp 8 --ep 8 --dp 2 --enable-dp-attention --enable-dsa-prefill-context-parallel --attn-cp-size 4 --dsa-prefill-cp-mode in-seq-split --max-running-requests 32 -``` - -### Round robin splitting (default setting) - -This mode can be enabled by specifying the parameter `--dsa-prefill-cp-mode round-robin-split`, which distributes tokens across ranks based on `token_idx % cp_size`. - -In this scenario, compared to the in-sequence splitting method, it additionally supports the fused MoE backend (the fused MoE backend may deliver better performance than DeepEP in single-machine scenarios), FP8 KV-cache, and multi-batch prefill inference. However, it cannot be enabled with DP attention together. - -For more details, please refer to PR https://github.com/sgl-project/sglang/pull/13959. - -Example usage: -```bash -# Launch with FusedMoe + CP8 -python -m sglang.launch_server --model deepseek-ai/DeepSeek-V3.2-Exp --tp 8 --enable-dsa-prefill-context-parallel --attn-cp-size 8 --dsa-prefill-cp-mode round-robin-split --max-running-requests 32 -``` -### Pipeline Parallel + Context Parallel (PP + CP) - -This mode combines Pipeline Parallelism (PP) and Context Parallelism (CP) to scale across multiple nodes, which can achieve better throughput and Time To First Token (TTFT). Note that this method has only been tested on H20 96G. - -#### Standard Usage - -To launch with PP=2 and CP (via `round-robin-split` mode) on 2 nodes. This configuration uses the fused MoE kernel by default, which generally provides better performance. - -For related development details, please refer to: -- Fused MoE + CP support: [PR #13959](https://github.com/sgl-project/sglang/pull/13959) -- PP + CP support: [Issue #15358](https://github.com/sgl-project/sglang/issues/15358) and [PR #16380](https://github.com/sgl-project/sglang/pull/16380) - -Node 0: -```bash -export SGLANG_PP_LAYER_PARTITION=30,31 -python3 -m sglang.launch_server \ - --model-path deepseek-ai/DeepSeek-V3.2-Exp \ - --nnodes 2 --node-rank 0 \ - --dist-init-addr :62001 \ - --tp 8 --pp-size 2 \ - --dp-size 1 --moe-dense-tp-size 1 \ - --enable-dsa-prefill-context-parallel \ - --attn-cp-size 8 \ - --dsa-prefill-cp-mode round-robin-split \ - --trust-remote-code \ - --disable-radix-cache \ - --mem-fraction-static 0.8 \ - --max-running-requests 128 \ - --chunked-prefill-size 16384 \ - --cuda-graph-max-bs 8 \ - --page-size 64 \ - --watchdog-timeout 3600 \ - --host 0.0.0.0 --port 8000 \ - --tool-call-parser deepseekv32 -``` - -Node 1: -```bash -export SGLANG_PP_LAYER_PARTITION=30,31 -python3 -m sglang.launch_server \ - --model-path deepseek-ai/DeepSeek-V3.2-Exp \ - --nnodes 2 --node-rank 1 \ - --dist-init-addr :62001 \ - --tp 8 --pp-size 2 \ - --dp-size 1 --moe-dense-tp-size 1 \ - --enable-dsa-prefill-context-parallel \ - --attn-cp-size 8 \ - --dsa-prefill-cp-mode round-robin-split \ - --trust-remote-code \ - --disable-radix-cache \ - --mem-fraction-static 0.8 \ - --max-running-requests 128 \ - --chunked-prefill-size 16384 \ - --cuda-graph-max-bs 8 \ - --page-size 64 \ - --watchdog-timeout 3600 \ - --host 0.0.0.0 --port 8000 \ - --tool-call-parser deepseekv32 -``` - -#### PD Disaggregation with PP + CP - -If using PD (Prefill-Decode) Disaggregation, the Prefill nodes can be configured with PP + CP as follows. - -Prefill Node 0: -```bash -python -m sglang.launch_server \ - --model-path deepseek-ai/DeepSeek-V3.2-Exp \ - --served-model-name deepseek-v32 \ - --nnodes 2 --node-rank 0 \ - --dist-init-addr :20102 \ - --tp 8 --pp-size 2 \ - --dp-size 1 --moe-dense-tp-size 1 \ - --enable-dsa-prefill-context-parallel \ - --attn-cp-size 8 \ - --dsa-prefill-cp-mode round-robin-split \ - --disaggregation-ib-device mlx5_bond_0,mlx5_bond_1,mlx5_bond_2,mlx5_bond_3 \ - --trust-remote-code \ - --disable-radix-cache \ - --max-running-requests 512 \ - --chunked-prefill-size 4096 \ - --context-length 131072 \ - --mem-fraction-static 0.9 \ - --page-size 64 \ - --enable-metrics \ - --collect-tokens-histogram \ - --tokenizer-worker-num 8 \ - --host 0.0.0.0 --port 30000 -``` - -Prefill Node 1: -```bash -python -m sglang.launch_server \ - --model-path deepseek-ai/DeepSeek-V3.2-Exp \ - --served-model-name deepseek-v32-prefill \ - --nnodes 2 --node-rank 1 \ - --dist-init-addr :20102 \ - --tp 8 --pp-size 2 \ - --dp-size 1 --moe-dense-tp-size 1 \ - --enable-dsa-prefill-context-parallel \ - --attn-cp-size 8 \ - --dsa-prefill-cp-mode round-robin-split \ - --disaggregation-ib-device mlx5_bond_0,mlx5_bond_1,mlx5_bond_2,mlx5_bond_3 \ - --trust-remote-code \ - --disable-radix-cache \ - --max-running-requests 512 \ - --chunked-prefill-size 4096 \ - --context-length 131072 \ - --mem-fraction-static 0.9 \ - --page-size 64 \ - --enable-metrics \ - --collect-tokens-histogram \ - --tokenizer-worker-num 8 \ - --host 0.0.0.0 --port 30000 -``` - -For the Decode nodes, it is recommended to use the **EP mode**. - -## HiSparse: Hierarchical Sparse Attention for DSA (experimental) - -HiSparse reduces per-request GPU memory during decode by keeping only a small "hot" KV buffer on GPU while storing complete KV data in CPU pinned memory. A CUDA kernel dynamically swaps in the top-k most relevant KV entries from host memory on each decode step. This enables significantly higher decode concurrency for long-context DSA models. - -HiSparse currently requires PD disaggregation mode and is enabled on the decode instance only. For detailed design, configuration, and deployment instructions, see the [HiSparse Guide](../advanced_features/hisparse_guide.md). diff --git a/docs/basic_usage/glm45.md b/docs/basic_usage/glm45.md deleted file mode 100644 index aaf8e13b4..000000000 --- a/docs/basic_usage/glm45.md +++ /dev/null @@ -1,70 +0,0 @@ -## Launch GLM-4.5 / GLM-4.6 / GLM-4.7 with SGLang - -To serve GLM-4.5 / GLM-4.6 FP8 models on 8xH100/H200 GPUs: - -```bash -python3 -m sglang.launch_server --model zai-org/GLM-4.6-FP8 --tp 8 -``` - -### EAGLE Speculative Decoding - -**Description**: SGLang has supported GLM-4.5 / GLM-4.6 models -with [EAGLE speculative decoding](https://docs.sglang.io/advanced_features/speculative_decoding.html#EAGLE-Decoding). - -**Usage**: -Add arguments `--speculative-algorithm`, `--speculative-num-steps`, `--speculative-eagle-topk` and -`--speculative-num-draft-tokens` to enable this feature. For example: - -``` bash -python3 -m sglang.launch_server \ - --model-path zai-org/GLM-4.6-FP8 \ - --tp-size 8 \ - --tool-call-parser glm45 \ - --reasoning-parser glm45 \ - --speculative-algorithm EAGLE \ - --speculative-num-steps 3 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 \ - --mem-fraction-static 0.9 \ - --served-model-name glm-4.6-fp8 \ - --enable-custom-logit-processor -``` - -```{tip} -To enable the experimental overlap scheduler for EAGLE speculative decoding, set the environment variable `SGLANG_ENABLE_SPEC_V2=1`. This can improve performance by enabling overlap scheduling between draft and verification stages. -``` - -### Thinking Budget for GLM-4.5 / GLM-4.6 -**Note**: For GLM-4.7, `--tool-call-parser` should be set to `glm47`, for GLM-4.5 and GLM-4.6, it should be set to `glm45`. - -In SGLang, we can implement thinking budget with `CustomLogitProcessor`. - -Launch a server with `--enable-custom-logit-processor` flag on. - -Sample Request: - -```python -import openai -from rich.pretty import pprint -from sglang.srt.sampling.custom_logit_processor import Glm4MoeThinkingBudgetLogitProcessor - - -client = openai.Client(base_url="http://127.0.0.1:30000/v1", api_key="*") -response = client.chat.completions.create( - model="zai-org/GLM-4.6", - messages=[ - { - "role": "user", - "content": "Question: Is Paris the Capital of France?", - } - ], - max_tokens=1024, - extra_body={ - "custom_logit_processor": Glm4MoeThinkingBudgetLogitProcessor().to_str(), - "custom_params": { - "thinking_budget": 512, - }, - }, -) -pprint(response) -``` diff --git a/docs/basic_usage/glmv.md b/docs/basic_usage/glmv.md deleted file mode 100644 index ad36cea26..000000000 --- a/docs/basic_usage/glmv.md +++ /dev/null @@ -1,136 +0,0 @@ -# GLM-4.6V / GLM-4.5V Usage - -## Launch commands for SGLang - -Below are suggested launch commands tailored for different hardware / precision modes - -### FP8 (quantised) mode - -For high memory-efficiency and latency optimized deployments (e.g., on H100, H200) where FP8 checkpoint is supported: - -```bash -python3 -m sglang.launch_server \ - --model-path zai-org/GLM-4.6V-FP8 \ - --tp 2 \ - --ep 2 \ - --host 0.0.0.0 \ - --port 30000 \ - --keep-mm-feature-on-device -``` - -### Non-FP8 (BF16 / full precision) mode -For deployments on A100/H100 where BF16 is used (or FP8 snapshot not used): -```bash -python3 -m sglang.launch_server \ - --model-path zai-org/GLM-4.6V \ - --tp 4 \ - --ep 4 \ - --host 0.0.0.0 \ - --port 30000 -``` - -## Hardware-specific notes / recommendations - -- On H100 with FP8: Use the FP8 checkpoint for best memory efficiency. -- On A100 / H100 with BF16 (non-FP8): It’s recommended to use `--mm-max-concurrent-calls` to control parallel throughput and GPU memory usage during image/video inference. -- On H200 & B200: The model can be run “out of the box”, supporting full context length plus concurrent image + video processing. - -## Sending Image/Video Requests - -### Image input: - -```python -import requests - -url = f"http://localhost:30000/v1/chat/completions" - -data = { - "model": "zai-org/GLM-4.6V", - "messages": [ - { - "role": "user", - "content": [ - {"type": "text", "text": "What’s in this image?"}, - { - "type": "image_url", - "image_url": { - "url": "https://github.com/sgl-project/sglang/blob/main/examples/assets/example_image.png?raw=true" - }, - }, - ], - } - ], - "max_tokens": 300, -} - -response = requests.post(url, json=data) -print(response.text) -``` - -### Video Input: - -```python -import requests - -url = f"http://localhost:30000/v1/chat/completions" - -data = { - "model": "zai-org/GLM-4.6V", - "messages": [ - { - "role": "user", - "content": [ - {"type": "text", "text": "What’s happening in this video?"}, - { - "type": "video_url", - "video_url": { - "url": "https://github.com/sgl-project/sgl-test-files/raw/refs/heads/main/videos/jobs_presenting_ipod.mp4" - }, - }, - ], - } - ], - "max_tokens": 300, -} - -response = requests.post(url, json=data) -print(response.text) -``` - -## Important Server Parameters and Flags - -When launching the model server for **multimodal support**, you can use the following command-line arguments to fine-tune performance and behavior: - -- `--mm-attention-backend`: Specify multimodal attention backend. Eg. `fa3`(Flash Attention 3) -- `--mm-max-concurrent-calls `: Specifies the **maximum number of concurrent asynchronous multimodal data processing calls** allowed on the server. Use this to control parallel throughput and GPU memory usage during image/video inference. -- `--mm-per-request-timeout `: Defines the **timeout duration (in seconds)** for each multimodal request. If a request exceeds this time limit (e.g., for very large video inputs), it will be automatically terminated. -- `--keep-mm-feature-on-device`: Instructs the server to **retain multimodal feature tensors on the GPU** after processing. This avoids device-to-host (D2H) memory copies and improves performance for repeated or high-frequency inference workloads. -- `--mm-enable-dp-encoder`: Placing the ViT in data parallel while keeping the LLM in tensor parallel consistently lowers TTFT and boosts end-to-end throughput. -- `SGLANG_USE_CUDA_IPC_TRANSPORT=1`: Shared memory pool based CUDA IPC for multi-modal data transport. For significantly improving e2e latency. - -### Example usage with the above optimizations: -```bash -SGLANG_USE_CUDA_IPC_TRANSPORT=1 \ -SGLANG_VLM_CACHE_SIZE_MB=0 \ -python -m sglang.launch_server \ - --model-path zai-org/GLM-4.6V \ - --host 0.0.0.0 \ - --port 30000 \ - --trust-remote-code \ - --tp-size 8 \ - --enable-cache-report \ - --log-level info \ - --max-running-requests 64 \ - --mem-fraction-static 0.65 \ - --chunked-prefill-size 8192 \ - --attention-backend fa3 \ - --mm-attention-backend fa3 \ - --mm-enable-dp-encoder \ - --enable-metrics -``` - -### Thinking Budget for GLM-4.5V / GLM-4.6V - -In SGLang, we can implement thinking budget with `CustomLogitProcessor`. - -Launch a server with the `--enable-custom-logit-processor` flag. Then, use `Glm4MoeThinkingBudgetLogitProcessor` in the request, similar to the `GLM-4.6` example in [glm45.md](./glm45.md). diff --git a/docs/basic_usage/gpt_oss.md b/docs/basic_usage/gpt_oss.md deleted file mode 100644 index da8e778b2..000000000 --- a/docs/basic_usage/gpt_oss.md +++ /dev/null @@ -1,147 +0,0 @@ -# GPT OSS Usage - -Please refer to [https://github.com/sgl-project/sglang/issues/8833](https://github.com/sgl-project/sglang/issues/8833). - -## Responses API & Built-in Tools - -### Responses API - -GPT‑OSS is compatible with the OpenAI Responses API. Use `client.responses.create(...)` with `model`, `instructions`, `input`, and optional `tools` to enable built‑in tool use. You can set reasoning level via `instructions`, e.g., "Reasoning: high" (also supports "medium" and "low") — levels: low (fast), medium (balanced), high (deep). - -### Built-in Tools - -GPT‑OSS can call built‑in tools for web search and Python execution. You can use the demo tool server or connect to external MCP tool servers. - -#### Python Tool - -- Executes short Python snippets for calculations, parsing, and quick scripts. -- By default runs in a Docker-based sandbox. To run on the host, set `PYTHON_EXECUTION_BACKEND=UV` (this executes model-generated code locally; use with care). -- Ensure Docker is available if you are not using the UV backend. It is recommended to run `docker pull python:3.11` in advance. - -#### Web Search Tool - -- Uses the Exa backend for web search. -- Requires an Exa API key; set `EXA_API_KEY` in your environment. Create a key at `https://exa.ai`. - -### Tool & Reasoning Parser - -- We support OpenAI Reasoning and Tool Call parser, as well as our SGLang native api for tool call and reasoning. Refer to [reasoning parser](../advanced_features/separate_reasoning.ipynb) and [tool parser](../advanced_features/tool_parser.ipynb) for more details. - - -## Notes - -- Use **Python 3.12** for the demo tools. And install the required `gpt-oss` packages. -- The default demo integrates the web search tool (Exa backend) and a demo Python interpreter via Docker. -- For search, set `EXA_API_KEY`. For Python execution, either have Docker available or set `PYTHON_EXECUTION_BACKEND=UV`. - -Examples: -```bash -export EXA_API_KEY=YOUR_EXA_KEY -# Optional: run Python tool locally instead of Docker (use with care) -export PYTHON_EXECUTION_BACKEND=UV -``` - -Launch the server with the demo tool server: - -```bash -python3 -m sglang.launch_server \ - --model-path openai/gpt-oss-120b \ - --tool-server demo \ - --tp 2 -``` - -For production usage, sglang can act as an MCP client for multiple services. An [example tool server](https://github.com/openai/gpt-oss/tree/main/gpt-oss-mcp-server) is provided. Start the servers and point sglang to them: -```bash -mcp run -t sse browser_server.py:mcp -mcp run -t sse python_server.py:mcp - -python -m sglang.launch_server ... --tool-server ip-1:port-1,ip-2:port-2 -``` -The URLs should be MCP SSE servers that expose server information and well-documented tools. These tools are added to the system prompt so the model can use them. - -## Speculative Decoding - -SGLang supports speculative decoding for GPT-OSS models using EAGLE3 algorithm. This can significantly improve decoding speed, especially for small batch sizes. - -**Usage**: -Add `--speculative-algorithm EAGLE3` along with the draft model path. -```bash -python3 -m sglang.launch_server \ - --model-path openai/gpt-oss-120b \ - --speculative-algorithm EAGLE3 \ - --speculative-draft-model-path lmsys/EAGLE3-gpt-oss-120b-bf16 \ - --tp 2 -``` - -```{tip} -To enable the experimental overlap scheduler for EAGLE3 speculative decoding, set the environment variable `SGLANG_ENABLE_SPEC_V2=1`. This can improve performance by enabling overlap scheduling between draft and verification stages. -``` - -### Quick Demo - -```python -from openai import OpenAI - -client = OpenAI( - base_url="http://localhost:30000/v1", - api_key="sk-123456" -) - -tools = [ - {"type": "code_interpreter"}, - {"type": "web_search_preview"}, -] - -# Reasoning level example -response = client.responses.create( - model="openai/gpt-oss-120b", - instructions="You are a helpful assistant." - reasoning_effort="high" # Supports high, medium, or low - input="In one sentence, explain the transformer architecture.", -) -print("====== reasoning: high ======") -print(response.output_text) - -# Test python tool -response = client.responses.create( - model="openai/gpt-oss-120b", - instructions="You are a helpful assistant, you could use python tool to execute code.", - input="Use python tool to calculate the sum of 29138749187 and 29138749187", # 58,277,498,374 - tools=tools -) -print("====== test python tool ======") -print(response.output_text) - -# Test browser tool -response = client.responses.create( - model="openai/gpt-oss-120b", - instructions="You are a helpful assistant, you could use browser to search the web", - input="Search the web for the latest news about Nvidia stock price", - tools=tools -) -print("====== test browser tool ======") -print(response.output_text) -``` - -Example output: -``` -====== test python tool ====== -The sum of 29,138,749,187 and 29,138,749,187 is **58,277,498,374**. -====== test browser tool ====== -**Recent headlines on Nvidia (NVDA) stock** - -| Date (2025) | Source | Key news points | Stock‑price detail | -|-------------|--------|----------------|--------------------| -| **May 13** | Reuters | The market data page shows Nvidia trading “higher” at **$116.61** with no change from the previous close. | **$116.61** – latest trade (delayed ≈ 15 min)【14†L34-L38】 | -| **Aug 18** | CNBC | Morgan Stanley kept an **overweight** rating and lifted its price target to **$206** (up from $200), implying a 14 % upside from the Friday close. The firm notes Nvidia shares have already **jumped 34 % this year**. | No exact price quoted, but the article signals strong upside expectations【9†L27-L31】 | -| **Aug 20** | The Motley Fool | Nvidia is set to release its Q2 earnings on Aug 27. The article lists the **current price of $175.36**, down 0.16 % on the day (as of 3:58 p.m. ET). | **$175.36** – current price on Aug 20【10†L12-L15】【10†L53-L57】 | - -**What the news tells us** - -* Nvidia’s share price has risen sharply this year – up roughly a third according to Morgan Stanley – and analysts are still raising targets (now $206). -* The most recent market quote (Reuters, May 13) was **$116.61**, but the stock has surged since then, reaching **$175.36** by mid‑August. -* Upcoming earnings on **Aug 27** are a focal point; both the Motley Fool and Morgan Stanley expect the results could keep the rally going. - -**Bottom line:** Nvidia’s stock is on a strong upward trajectory in 2025, with price targets climbing toward $200‑$210 and the market price already near $175 as of late August. - -``` diff --git a/docs/basic_usage/hy3_preview.md b/docs/basic_usage/hy3_preview.md deleted file mode 100644 index b7f23937e..000000000 --- a/docs/basic_usage/hy3_preview.md +++ /dev/null @@ -1,191 +0,0 @@ -# Hy3-preview Usage - -Hy3-preview is a large-scale language model (295B parameters, 21B active parameters) from Tencent Hunyuan team. SGLang supports serving Hy3-preview. This guide describes how to run Hy3-preview with native BF16. - -## Installation - -### Docker - -```bash -docker pull lmsysorg/sglang:hy3-preview -``` - -### Build from Source - -```bash -# Install SGLang -git clone https://github.com/sgl-project/sglang -cd sglang -pip3 install pip --upgrade -pip3 install "transformers>=5.6.0" -pip3 install -e "python" -``` - -## Launch Hy3-preview with SGLang - -To serve the [Hy3-preview](https://huggingface.co/tencent/Hy3-preview) model on 8 GPUs. On 8x96GB H20, SGLang can barely deploy the BF16 model and can only run small batch sizes or short requests. Use larger-memory GPUs such as H20-3e when possible. - -```bash -python3 -m sglang.launch_server \ - --model tencent/Hy3-preview \ - --tp 8 \ - --tool-call-parser hunyuan \ - --reasoning-parser hunyuan \ - --served-model-name hy3-preview -``` - -### EAGLE Speculative Decoding - -**Description**: SGLang supports Hy3-preview models with [EAGLE speculative decoding](https://docs.sglang.io/advanced_features/speculative_decoding.html#eagle-decoding). - -**Usage**: -Add `--speculative-algorithm`, `--speculative-num-steps`, `--speculative-eagle-topk`, and `--speculative-num-draft-tokens` to enable this feature. For example: - -```bash -python3 -m sglang.launch_server \ - --model tencent/Hy3-preview \ - --tp 8 \ - --tool-call-parser hunyuan \ - --reasoning-parser hunyuan \ - --speculative-num-steps 1 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 2 \ - --speculative-algorithm EAGLE \ - --served-model-name hy3-preview -``` - -## OpenAI Client Example - -First, install the OpenAI Python client: - -```bash -uv pip install -U openai -``` - -You can use the OpenAI client as follows to verify thinking-mode responses. - -```python -from openai import OpenAI - -# If running SGLang locally with its default OpenAI-compatible port: -# http://localhost:30000/v1 -openai_api_key = "EMPTY" -openai_api_base = "http://localhost:30000/v1" - -client = OpenAI( - api_key=openai_api_key, - base_url=openai_api_base, -) -messages = [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "Hello."}, -] - -# Thinking mode is disabled by default (no need to pass chat_template_kwargs). -resp = client.chat.completions.create( - model="hy3-preview", - messages=messages, - temperature=1, - max_tokens=4096, -) -print(resp.choices[0].message.content) - -# Thinking mode is enabled only if 'reasoning_effort' and 'interleaved_thinking' are set in 'chat_template_kwargs'. -# 'reasoning_effort' supports: 'high', 'low', 'no_think'. -resp_think = client.chat.completions.create( - model="hy3-preview", - messages=messages, - temperature=1, - max_tokens=4096, - extra_body={ - "chat_template_kwargs": { - "reasoning_effort": "high", - "interleaved_thinking": True - }, - }, -) -output_msg = resp_think.choices[0].message -# thinking content -print(output_msg.reasoning_content) -# response content -print(output_msg.content) -``` - -### cURL Usage - -```bash -curl http://localhost:30000/v1/chat/completions \ - -H "Content-Type: application/json" \ - -d '{ - "model": "hy3-preview", - "messages": [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "Hello."} - ], - "temperature": 1, - "max_tokens": 4096 - }' -``` - -## Benchmarking Results - -For benchmarking, disable prefix caching by adding `--disable-radix-cache` to the server command. - -The following example runs the benchmark on 8 H20 GPUs with 96 GB memory each. - -```bash -python3 -m sglang.bench_serving \ - --backend sglang \ - --flush-cache \ - --dataset-name random \ - --random-range-ratio 1.0 \ - --random-input-len 4096 \ - --random-output-len 4096 \ - --num-prompts 5 \ - --max-concurrency 1 \ - --output-file hy3_preview_h20.jsonl \ - --model tencent/Hy3-preview \ - --served-model-name hy3-preview -``` - -If successful, you will see the following output. - -```shell -============ Serving Benchmark Result ============ -Backend: sglang -Traffic request rate: inf -Max request concurrency: 1 -Successful requests: 5 -Benchmark duration (s): 176.41 -Total input tokens: 20480 -Total input text tokens: 20480 -Total generated tokens: 20480 -Total generated tokens (retokenized): 20480 -Request throughput (req/s): 0.03 -Input token throughput (tok/s): 116.09 -Output token throughput (tok/s): 116.09 -Peak output token throughput (tok/s): 118.00 -Peak concurrent requests: 2 -Total token throughput (tok/s): 232.19 -Concurrency: 1.00 -----------------End-to-End Latency---------------- -Mean E2E Latency (ms): 35279.06 -Median E2E Latency (ms): 35275.60 -P90 E2E Latency (ms): 35294.13 -P99 E2E Latency (ms): 35294.41 ----------------Time to First Token---------------- -Mean TTFT (ms): 355.93 -Median TTFT (ms): 309.28 -P99 TTFT (ms): 518.36 ------Time per Output Token (excl. 1st token)------ -Mean TPOT (ms): 8.53 -Median TPOT (ms): 8.54 -P99 TPOT (ms): 8.54 ----------------Inter-Token Latency---------------- -Mean ITL (ms): 8.53 -Median ITL (ms): 8.54 -P95 ITL (ms): 8.62 -P99 ITL (ms): 8.74 -Max ITL (ms): 31.70 -================================================== -``` diff --git a/docs/basic_usage/llama4.md b/docs/basic_usage/llama4.md deleted file mode 100644 index 05ffb2c60..000000000 --- a/docs/basic_usage/llama4.md +++ /dev/null @@ -1,92 +0,0 @@ -# Llama4 Usage - -[Llama 4](https://github.com/meta-llama/llama-models/blob/main/models/llama4/MODEL_CARD.md) is Meta's latest generation of open-source LLM model with industry-leading performance. - -SGLang has supported Llama 4 Scout (109B) and Llama 4 Maverick (400B) since [v0.4.5](https://github.com/sgl-project/sglang/releases/tag/v0.4.5). - -Ongoing optimizations are tracked in the [Roadmap](https://github.com/sgl-project/sglang/issues/5118). - -## Launch Llama 4 with SGLang - -To serve Llama 4 models on 8xH100/H200 GPUs: - -```bash -python3 -m sglang.launch_server \ - --model-path meta-llama/Llama-4-Scout-17B-16E-Instruct \ - --tp 8 \ - --context-length 1000000 -``` - -### Configuration Tips - -- **OOM Mitigation**: Adjust `--context-length` to avoid a GPU out-of-memory issue. For the Scout model, we recommend setting this value up to 1M on 8\*H100 and up to 2.5M on 8\*H200. For the Maverick model, we don't need to set context length on 8\*H200. When hybrid kv cache is enabled, `--context-length` can be set up to 5M on 8\*H100 and up to 10M on 8\*H200 for the Scout model. - -- **Attention Backend Auto-Selection**: SGLang automatically selects the optimal attention backend for Llama 4 based on your hardware. You typically don't need to specify `--attention-backend` manually: - - **Blackwell GPUs (B200/GB200)**: `trtllm_mha` - - **Hopper GPUs (H100/H200)**: `fa3` - - **AMD GPUs**: `aiter` - - **Intel XPU**: `intel_xpu` - - **Other platforms**: `triton` (fallback) - - To override the auto-selection, explicitly specify `--attention-backend` with one of the supported backends: `fa3`, `aiter`, `triton`, `trtllm_mha`, or `intel_xpu`. - -- **Chat Template**: Add `--chat-template llama-4` for chat completion tasks. -- **Enable Multi-Modal**: Add `--enable-multimodal` for multi-modal capabilities. -- **Enable Hybrid-KVCache**: Set `--swa-full-tokens-ratio` to adjust the ratio of SWA layer (for Llama4, it's local attention layer) KV tokens / full layer KV tokens. (default: 0.8, range: 0-1) - - -### EAGLE Speculative Decoding -**Description**: SGLang has supported Llama 4 Maverick (400B) with [EAGLE speculative decoding](https://docs.sglang.io/advanced_features/speculative_decoding.html#EAGLE-Decoding). - -**Usage**: -Add arguments `--speculative-draft-model-path`, `--speculative-algorithm`, `--speculative-num-steps`, `--speculative-eagle-topk` and `--speculative-num-draft-tokens` to enable this feature. For example: -``` -python3 -m sglang.launch_server \ - --model-path meta-llama/Llama-4-Maverick-17B-128E-Instruct \ - --speculative-algorithm EAGLE3 \ - --speculative-draft-model-path nvidia/Llama-4-Maverick-17B-128E-Eagle3 \ - --speculative-num-steps 3 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 \ - --trust-remote-code \ - --tp 8 \ - --context-length 1000000 -``` - -- **Note** The Llama 4 draft model *nvidia/Llama-4-Maverick-17B-128E-Eagle3* can only recognize conversations in chat mode. - -## Benchmarking Results - -### Accuracy Test with `lm_eval` - -The accuracy on SGLang for both Llama4 Scout and Llama4 Maverick can match the [official benchmark numbers](https://ai.meta.com/blog/llama-4-multimodal-intelligence/). - -Benchmark results on MMLU Pro dataset with 8*H100: -| | Llama-4-Scout-17B-16E-Instruct | Llama-4-Maverick-17B-128E-Instruct | -|--------------------|--------------------------------|-------------------------------------| -| Official Benchmark | 74.3 | 80.5 | -| SGLang | 75.2 | 80.7 | - -Commands: - -```bash -# Llama-4-Scout-17B-16E-Instruct model -python -m sglang.launch_server \ - --model-path meta-llama/Llama-4-Scout-17B-16E-Instruct \ - --port 30000 \ - --tp 8 \ - --mem-fraction-static 0.8 \ - --context-length 65536 -lm_eval --model local-chat-completions --model_args model=meta-llama/Llama-4-Scout-17B-16E-Instruct,base_url=http://localhost:30000/v1/chat/completions,num_concurrent=128,timeout=999999,max_gen_toks=2048 --tasks mmlu_pro --batch_size 128 --apply_chat_template --num_fewshot 0 - -# Llama-4-Maverick-17B-128E-Instruct -python -m sglang.launch_server \ - --model-path meta-llama/Llama-4-Maverick-17B-128E-Instruct \ - --port 30000 \ - --tp 8 \ - --mem-fraction-static 0.8 \ - --context-length 65536 -lm_eval --model local-chat-completions --model_args model=meta-llama/Llama-4-Maverick-17B-128E-Instruct,base_url=http://localhost:30000/v1/chat/completions,num_concurrent=128,timeout=999999,max_gen_toks=2048 --tasks mmlu_pro --batch_size 128 --apply_chat_template --num_fewshot 0 -``` - -Details can be seen in [this PR](https://github.com/sgl-project/sglang/pull/5092). diff --git a/docs/basic_usage/minimax_m2.md b/docs/basic_usage/minimax_m2.md deleted file mode 100644 index 7ca6ed809..000000000 --- a/docs/basic_usage/minimax_m2.md +++ /dev/null @@ -1,85 +0,0 @@ -# MiniMax M2.5/M2.1/M2 Usage - -[MiniMax-M2.5](https://huggingface.co/MiniMaxAI/MiniMax-M2.5), [MiniMax-M2.1](https://huggingface.co/MiniMaxAI/MiniMax-M2.1), and [MiniMax-M2](https://huggingface.co/MiniMaxAI/MiniMax-M2) are advanced large language models created by [MiniMax](https://www.minimax.io/). - -The MiniMax-M2 series redefines efficiency for agents. These compact, fast, and cost-effective MoE models (230 billion total parameters with 10 billion active parameters) are built for elite performance in coding and agentic tasks, all while maintaining powerful general intelligence. With just 10 billion activated parameters, the MiniMax-M2 series provides sophisticated, end-to-end tool use performance expected from today's leading models, but in a streamlined form factor that makes deployment and scaling easier than ever. - -## Supported Models - -This guide applies to the following models. You only need to update the model name during deployment. The following examples use **MiniMax-M2**: - -- [MiniMaxAI/MiniMax-M2.5](https://huggingface.co/MiniMaxAI/MiniMax-M2.5) -- [MiniMaxAI/MiniMax-M2.1](https://huggingface.co/MiniMaxAI/MiniMax-M2.1) -- [MiniMaxAI/MiniMax-M2](https://huggingface.co/MiniMaxAI/MiniMax-M2) - -## System Requirements - -The following are recommended configurations; actual requirements should be adjusted based on your use case: - -- 4x 96GB GPUs: Supported context length of up to 400K tokens. -- 8x 144GB GPUs: Supported context length of up to 3M tokens. - -## Deployment with Python - -4-GPU deployment command: - -```bash -python -m sglang.launch_server \ - --model-path MiniMaxAI/MiniMax-M2 \ - --tp-size 4 \ - --tool-call-parser minimax-m2 \ - --reasoning-parser minimax-append-think \ - --host 0.0.0.0 \ - --trust-remote-code \ - --port 8000 \ - --mem-fraction-static 0.85 -``` - -8-GPU deployment command: - -```bash -python -m sglang.launch_server \ - --model-path MiniMaxAI/MiniMax-M2 \ - --tp-size 8 \ - --ep-size 8 \ - --tool-call-parser minimax-m2 \ - --reasoning-parser minimax-append-think \ - --host 0.0.0.0 \ - --trust-remote-code \ - --port 8000 \ - --mem-fraction-static 0.85 -``` - -### AMD GPUs (MI300X/MI325X/MI355X) - -8-GPU deployment command: - -```bash -SGLANG_USE_AITER=1 python -m sglang.launch_server \ - --model-path MiniMaxAI/MiniMax-M2.5 \ - --tp-size 8 \ - --ep-size 8 \ - --attention-backend aiter \ - --tool-call-parser minimax-m2 \ - --reasoning-parser minimax-append-think \ - --host 0.0.0.0 \ - --trust-remote-code \ - --port 8000 \ - --mem-fraction-static 0.85 -``` - -## Testing Deployment - -After startup, you can test the SGLang OpenAI-compatible API with the following command: - -```bash -curl http://localhost:8000/v1/chat/completions \ - -H "Content-Type: application/json" \ - -d '{ - "model": "MiniMaxAI/MiniMax-M2", - "messages": [ - {"role": "system", "content": [{"type": "text", "text": "You are a helpful assistant."}]}, - {"role": "user", "content": [{"type": "text", "text": "Who won the world series in 2020?"}]} - ] - }' -``` diff --git a/docs/basic_usage/native_api.ipynb b/docs/basic_usage/native_api.ipynb deleted file mode 100644 index d3ead5e34..000000000 --- a/docs/basic_usage/native_api.ipynb +++ /dev/null @@ -1,675 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# SGLang Native APIs\n", - "\n", - "Apart from the OpenAI compatible APIs, the SGLang Runtime also provides its native server APIs. We introduce the following APIs:\n", - "\n", - "- `/generate` (text generation model)\n", - "- `/get_model_info`\n", - "- `/server_info`\n", - "- `/health`\n", - "- `/health_generate`\n", - "- `/flush_cache`\n", - "- `/update_weights`\n", - "- `/encode`(embedding model)\n", - "- `/v1/rerank`(cross encoder rerank model)\n", - "- `/v1/score`(decoder-only scoring)\n", - "- `/classify`(reward model)\n", - "- `/start_expert_distribution_record`\n", - "- `/stop_expert_distribution_record`\n", - "- `/dump_expert_distribution_record`\n", - "- `/tokenize`\n", - "- `/detokenize`\n", - "- A full list of these APIs can be found at [http_server.py](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/entrypoints/http_server.py)\n", - "\n", - "We mainly use `requests` to test these APIs in the following examples. You can also use `curl`.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Launch A Server" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from sglang.test.doc_patch import launch_server_cmd\n", - "from sglang.utils import wait_for_server, print_highlight, terminate_process\n", - "\n", - "server_process, port = launch_server_cmd(\n", - " \"python3 -m sglang.launch_server --model-path qwen/qwen2.5-0.5b-instruct --host 0.0.0.0 --log-level warning\"\n", - ")\n", - "\n", - "wait_for_server(f\"http://localhost:{port}\", process=server_process)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Generate (text generation model)\n", - "Generate completions. This is similar to the `/v1/completions` in OpenAI API. Detailed parameters can be found in the [sampling parameters](sampling_params.md)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import requests\n", - "\n", - "url = f\"http://localhost:{port}/generate\"\n", - "data = {\"text\": \"What is the capital of France?\"}\n", - "\n", - "response = requests.post(url, json=data)\n", - "print_highlight(response.json())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Get Model Info\n", - "\n", - "Get the information of the model.\n", - "\n", - "- `model_path`: The path/name of the model.\n", - "- `is_generation`: Whether the model is used as generation model or embedding model.\n", - "- `tokenizer_path`: The path/name of the tokenizer.\n", - "- `preferred_sampling_params`: The default sampling params specified via `--preferred-sampling-params`. `None` is returned in this example as we did not explicitly configure it in server args.\n", - "- `weight_version`: This field contains the version of the model weights. This is often used to track changes or updates to the model’s trained parameters.\n", - "- `has_image_understanding`: Whether the model has image-understanding capability.\n", - "- `has_audio_understanding`: Whether the model has audio-understanding capability.\n", - "- `model_type`: The model type from the HuggingFace config (e.g., \"qwen2\", \"llama\").\n", - "- `architectures`: The model architectures from the HuggingFace config (e.g., [\"Qwen2ForCausalLM\"])." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "url = f\"http://localhost:{port}/get_model_info\"\n", - "\n", - "response = requests.get(url)\n", - "response_json = response.json()\n", - "print_highlight(response_json)\n", - "assert response_json[\"model_path\"] == \"qwen/qwen2.5-0.5b-instruct\"\n", - "assert response_json[\"is_generation\"] is True\n", - "assert response_json[\"tokenizer_path\"] == \"qwen/qwen2.5-0.5b-instruct\"\n", - "assert response_json[\"preferred_sampling_params\"] is None\n", - "assert response_json.keys() == {\n", - " \"model_path\",\n", - " \"is_generation\",\n", - " \"tokenizer_path\",\n", - " \"preferred_sampling_params\",\n", - " \"weight_version\",\n", - " \"has_image_understanding\",\n", - " \"has_audio_understanding\",\n", - " \"model_type\",\n", - " \"architectures\",\n", - "}" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Get Server Info\n", - "Gets the server information including CLI arguments, token limits, and memory pool sizes.\n", - "- Note: `get_server_info` merges the following deprecated endpoints:\n", - " - `get_server_args`\n", - " - `get_memory_pool_size`\n", - " - `get_max_total_num_tokens`" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "url = f\"http://localhost:{port}/server_info\"\n", - "\n", - "response = requests.get(url)\n", - "print_highlight(response.text)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Health Check\n", - "- `/health`: Check the health of the server.\n", - "- `/health_generate`: Check the health of the server by generating one token." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "url = f\"http://localhost:{port}/health_generate\"\n", - "\n", - "response = requests.get(url)\n", - "print_highlight(response.text)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "url = f\"http://localhost:{port}/health\"\n", - "\n", - "response = requests.get(url)\n", - "print_highlight(response.text)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Flush Cache\n", - "\n", - "Flush the radix cache. It will be automatically triggered when the model weights are updated by the `/update_weights` API.\n", - "\n", - "Parameters:\n", - "- `timeout` (query, float, default `0`, unit: seconds): Wait time for idle state before flushing. `0` means fail fast if not idle. When HiCache async operations are in-flight, a non-zero timeout allows the server to wait until idle before flushing, avoiding unnecessary 400 errors.\n", - "\n", - "```bash\n", - "# With timeout (wait up to 30s for idle state)\n", - "curl -s -X POST \"http://127.0.0.1:30000/flush_cache?timeout=30\"\n", - "```" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "url = f\"http://localhost:{port}/flush_cache\"\n", - "\n", - "response = requests.post(url)\n", - "print_highlight(response.text)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Update Weights From Disk\n", - "\n", - "Update model weights from disk without restarting the server. Only applicable for models with the same architecture and parameter size.\n", - "\n", - "SGLang support `update_weights_from_disk` API for continuous evaluation during training (save checkpoint to disk and update weights from disk).\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# successful update with same architecture and size\n", - "\n", - "url = f\"http://localhost:{port}/update_weights_from_disk\"\n", - "data = {\"model_path\": \"qwen/qwen2.5-0.5b-instruct\"}\n", - "\n", - "response = requests.post(url, json=data)\n", - "print_highlight(response.text)\n", - "assert response.json()[\"success\"] is True\n", - "assert response.json()[\"message\"] == \"Succeeded to update model weights.\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# failed update with different parameter size or wrong name\n", - "\n", - "url = f\"http://localhost:{port}/update_weights_from_disk\"\n", - "data = {\"model_path\": \"qwen/qwen2.5-0.5b-instruct-wrong\"}\n", - "\n", - "response = requests.post(url, json=data)\n", - "response_json = response.json()\n", - "print_highlight(response_json)\n", - "assert response_json[\"success\"] is False\n", - "assert response_json[\"message\"] == (\n", - " \"Failed to get weights iterator: \"\n", - " \"qwen/qwen2.5-0.5b-instruct-wrong\"\n", - " \" (repository not found).\"\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "terminate_process(server_process)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Encode (embedding model)\n", - "\n", - "Encode text into embeddings. Note that this API is only available for [embedding models](openai_api_embeddings.ipynb) and will raise an error for generation models.\n", - "Therefore, we launch a new server to server an embedding model." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "embedding_process, port = launch_server_cmd(\"\"\"\n", - "python3 -m sglang.launch_server --model-path Alibaba-NLP/gte-Qwen2-1.5B-instruct \\\n", - " --host 0.0.0.0 --is-embedding --log-level warning\n", - "\"\"\")\n", - "\n", - "wait_for_server(f\"http://localhost:{port}\", process=embedding_process)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# successful encode for embedding model\n", - "\n", - "url = f\"http://localhost:{port}/encode\"\n", - "data = {\"model\": \"Alibaba-NLP/gte-Qwen2-1.5B-instruct\", \"text\": \"Once upon a time\"}\n", - "\n", - "response = requests.post(url, json=data)\n", - "response_json = response.json()\n", - "print_highlight(f\"Text embedding (first 10): {response_json['embedding'][:10]}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "terminate_process(embedding_process)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## v1/rerank (cross encoder rerank model)\n", - "Rerank a list of documents given a query using a cross-encoder model. Note that this API is only available for cross encoder model like [BAAI/bge-reranker-v2-m3](https://huggingface.co/BAAI/bge-reranker-v2-m3) with `attention-backend` `triton` and `torch_native`.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "reranker_process, port = launch_server_cmd(\"\"\"\n", - "python3 -m sglang.launch_server --model-path BAAI/bge-reranker-v2-m3 \\\n", - " --host 0.0.0.0 --disable-radix-cache --chunked-prefill-size -1 --attention-backend triton --is-embedding --log-level warning\n", - "\"\"\")\n", - "\n", - "wait_for_server(f\"http://localhost:{port}\", process=reranker_process)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# compute rerank scores for query and documents\n", - "\n", - "url = f\"http://localhost:{port}/v1/rerank\"\n", - "data = {\n", - " \"model\": \"BAAI/bge-reranker-v2-m3\",\n", - " \"query\": \"what is panda?\",\n", - " \"documents\": [\n", - " \"hi\",\n", - " \"The giant panda (Ailuropoda melanoleuca), sometimes called a panda bear or simply panda, is a bear species endemic to China.\",\n", - " ],\n", - "}\n", - "\n", - "response = requests.post(url, json=data)\n", - "response_json = response.json()\n", - "for item in response_json:\n", - " print_highlight(f\"Score: {item['score']:.2f} - Document: '{item['document']}'\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "terminate_process(reranker_process)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## v1/score (decoder-only scoring)\n", - "\n", - "Compute token probabilities for specified tokens given a query and items. This is useful for classification tasks, scoring responses, or computing log-probabilities.\n", - "\n", - "Parameters:\n", - "- `query`: Query text\n", - "- `items`: Item text(s) to score\n", - "- `label_token_ids`: Token IDs to compute probabilities for\n", - "- `apply_softmax`: Whether to apply softmax to get normalized probabilities (default: False)\n", - "- `item_first`: Whether items come first in concatenation order (default: False)\n", - "- `model`: Model name\n", - "\n", - "The response contains `scores` - a list of probability lists, one per item, each in the order of `label_token_ids`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "score_process, port = launch_server_cmd(\"\"\"\n", - "python3 -m sglang.launch_server --model-path qwen/qwen2.5-0.5b-instruct \\\n", - " --host 0.0.0.0 --log-level warning\n", - "\"\"\")\n", - "\n", - "wait_for_server(f\"http://localhost:{port}\", process=score_process)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Score the probability of different completions given a query\n", - "query = \"The capital of France is\"\n", - "items = [\"Paris\", \"London\", \"Berlin\"]\n", - "\n", - "url = f\"http://localhost:{port}/v1/score\"\n", - "data = {\n", - " \"model\": \"qwen/qwen2.5-0.5b-instruct\",\n", - " \"query\": query,\n", - " \"items\": items,\n", - " \"label_token_ids\": [9454, 2753], # e.g. \"Yes\" and \"No\" token ids\n", - " \"apply_softmax\": True, # Normalize probabilities to sum to 1\n", - "}\n", - "\n", - "response = requests.post(url, json=data)\n", - "response_json = response.json()\n", - "\n", - "# Display scores for each item\n", - "for item, scores in zip(items, response_json[\"scores\"]):\n", - " print_highlight(f\"Item '{item}': probabilities = {[f'{s:.4f}' for s in scores]}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "terminate_process(score_process)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Classify (reward model)\n", - "\n", - "SGLang Runtime also supports reward models. Here we use a reward model to classify the quality of pairwise generations." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Note that SGLang now treats embedding models and reward models as the same type of models.\n", - "# This will be updated in the future.\n", - "\n", - "reward_process, port = launch_server_cmd(\"\"\"\n", - "python3 -m sglang.launch_server --model-path Skywork/Skywork-Reward-Llama-3.1-8B-v0.2 --host 0.0.0.0 --is-embedding --log-level warning\n", - "\"\"\")\n", - "\n", - "wait_for_server(f\"http://localhost:{port}\", process=reward_process)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from transformers import AutoTokenizer\n", - "\n", - "PROMPT = (\n", - " \"What is the range of the numeric output of a sigmoid node in a neural network?\"\n", - ")\n", - "\n", - "RESPONSE1 = \"The output of a sigmoid node is bounded between -1 and 1.\"\n", - "RESPONSE2 = \"The output of a sigmoid node is bounded between 0 and 1.\"\n", - "\n", - "CONVS = [\n", - " [{\"role\": \"user\", \"content\": PROMPT}, {\"role\": \"assistant\", \"content\": RESPONSE1}],\n", - " [{\"role\": \"user\", \"content\": PROMPT}, {\"role\": \"assistant\", \"content\": RESPONSE2}],\n", - "]\n", - "\n", - "tokenizer = AutoTokenizer.from_pretrained(\"Skywork/Skywork-Reward-Llama-3.1-8B-v0.2\")\n", - "prompts = tokenizer.apply_chat_template(CONVS, tokenize=False, return_dict=False)\n", - "\n", - "url = f\"http://localhost:{port}/classify\"\n", - "data = {\"model\": \"Skywork/Skywork-Reward-Llama-3.1-8B-v0.2\", \"text\": prompts}\n", - "\n", - "responses = requests.post(url, json=data).json()\n", - "for response in responses:\n", - " print_highlight(f\"reward: {response['embedding'][0]}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "terminate_process(reward_process)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Capture expert selection distribution in MoE models\n", - "\n", - "SGLang Runtime supports recording the number of times an expert is selected in a MoE model run for each expert in the model. This is useful when analyzing the throughput of the model and plan for optimization.\n", - "\n", - "*Note: We only print out the first 10 lines of the csv below for better readability. Please adjust accordingly if you want to analyze the results more deeply.*" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "expert_record_server_process, port = launch_server_cmd(\n", - " \"python3 -m sglang.launch_server --model-path Qwen/Qwen1.5-MoE-A2.7B --host 0.0.0.0 --expert-distribution-recorder-mode stat --log-level warning\"\n", - ")\n", - "\n", - "wait_for_server(f\"http://localhost:{port}\", process=expert_record_server_process)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "response = requests.post(f\"http://localhost:{port}/start_expert_distribution_record\")\n", - "print_highlight(response)\n", - "\n", - "url = f\"http://localhost:{port}/generate\"\n", - "data = {\"text\": \"What is the capital of France?\"}\n", - "\n", - "response = requests.post(url, json=data)\n", - "print_highlight(response.json())\n", - "\n", - "response = requests.post(f\"http://localhost:{port}/stop_expert_distribution_record\")\n", - "print_highlight(response)\n", - "\n", - "response = requests.post(f\"http://localhost:{port}/dump_expert_distribution_record\")\n", - "print_highlight(response)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "terminate_process(expert_record_server_process)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Tokenize/Detokenize Example (Round Trip)\n", - "\n", - "This example demonstrates how to use the /tokenize and /detokenize endpoints together. We first tokenize a string, then detokenize the resulting IDs to reconstruct the original text. This workflow is useful when you need to handle tokenization externally but still leverage the server for detokenization." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "tokenizer_free_server_process, port = launch_server_cmd(\"\"\"\n", - "python3 -m sglang.launch_server --model-path qwen/qwen2.5-0.5b-instruct\n", - "\"\"\")\n", - "\n", - "wait_for_server(f\"http://localhost:{port}\", process=tokenizer_free_server_process)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import requests\n", - "from sglang.utils import print_highlight\n", - "\n", - "base_url = f\"http://localhost:{port}\"\n", - "tokenize_url = f\"{base_url}/tokenize\"\n", - "detokenize_url = f\"{base_url}/detokenize\"\n", - "\n", - "model_name = \"qwen/qwen2.5-0.5b-instruct\"\n", - "input_text = \"SGLang provides efficient tokenization endpoints.\"\n", - "print_highlight(f\"Original Input Text:\\n'{input_text}'\")\n", - "\n", - "# --- tokenize the input text ---\n", - "tokenize_payload = {\n", - " \"model\": model_name,\n", - " \"prompt\": input_text,\n", - " \"add_special_tokens\": False,\n", - "}\n", - "try:\n", - " tokenize_response = requests.post(tokenize_url, json=tokenize_payload)\n", - " tokenize_response.raise_for_status()\n", - " tokenization_result = tokenize_response.json()\n", - " token_ids = tokenization_result.get(\"tokens\")\n", - "\n", - " if not token_ids:\n", - " raise ValueError(\"Tokenization returned empty tokens.\")\n", - "\n", - " print_highlight(f\"\\nTokenized Output (IDs):\\n{token_ids}\")\n", - " print_highlight(f\"Token Count: {tokenization_result.get('count')}\")\n", - " print_highlight(f\"Max Model Length: {tokenization_result.get('max_model_len')}\")\n", - "\n", - " # --- detokenize the obtained token IDs ---\n", - " detokenize_payload = {\n", - " \"model\": model_name,\n", - " \"tokens\": token_ids,\n", - " \"skip_special_tokens\": True,\n", - " }\n", - "\n", - " detokenize_response = requests.post(detokenize_url, json=detokenize_payload)\n", - " detokenize_response.raise_for_status()\n", - " detokenization_result = detokenize_response.json()\n", - " reconstructed_text = detokenization_result.get(\"text\")\n", - "\n", - " print_highlight(f\"\\nDetokenized Output (Text):\\n'{reconstructed_text}'\")\n", - "\n", - " if input_text == reconstructed_text:\n", - " print_highlight(\n", - " \"\\nRound Trip Successful: Original and reconstructed text match.\"\n", - " )\n", - " else:\n", - " print_highlight(\n", - " \"\\nRound Trip Mismatch: Original and reconstructed text differ.\"\n", - " )\n", - "\n", - "except requests.exceptions.RequestException as e:\n", - " print_highlight(f\"\\nHTTP Request Error: {e}\")\n", - "except Exception as e:\n", - " print_highlight(f\"\\nAn error occurred: {e}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "terminate_process(tokenizer_free_server_process)" - ] - } - ], - "metadata": { - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/docs/basic_usage/offline_engine_api.ipynb b/docs/basic_usage/offline_engine_api.ipynb deleted file mode 100644 index fe8a9e304..000000000 --- a/docs/basic_usage/offline_engine_api.ipynb +++ /dev/null @@ -1,235 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Offline Engine API\n", - "\n", - "SGLang provides a direct inference engine without the need for an HTTP server, especially for use cases where additional HTTP server adds unnecessary complexity or overhead. Here are two general use cases:\n", - "\n", - "- Offline Batch Inference\n", - "- Custom Server on Top of the Engine\n", - "\n", - "This document focuses on the offline batch inference, demonstrating four different inference modes:\n", - "\n", - "- Non-streaming synchronous generation\n", - "- Streaming synchronous generation\n", - "- Non-streaming asynchronous generation\n", - "- Streaming asynchronous generation\n", - "\n", - "Additionally, you can easily build a custom server on top of the SGLang offline engine. A detailed example working in a python script can be found in [custom_server](https://github.com/sgl-project/sglang/blob/main/examples/runtime/engine/custom_server.py).\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Nest Asyncio\n", - "Note that if you want to use **Offline Engine** in ipython or some other nested loop code, you need to add the following code:\n", - "```python\n", - "import nest_asyncio\n", - "\n", - "nest_asyncio.apply()\n", - "\n", - "```" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Advanced Usage\n", - "\n", - "The engine supports [vlm inference](https://github.com/sgl-project/sglang/blob/main/examples/runtime/engine/offline_batch_inference_vlm.py) as well as [extracting hidden states](https://github.com/sgl-project/sglang/blob/main/examples/runtime/hidden_states). \n", - "\n", - "Please see [the examples](https://github.com/sgl-project/sglang/tree/main/examples/runtime/engine) for further use cases." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Offline Batch Inference\n", - "\n", - "SGLang offline engine supports batch inference with efficient scheduling." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# launch the offline engine\n", - "import asyncio\n", - "\n", - "import sglang as sgl\n", - "import sglang.test.doc_patch # noqa: F401\n", - "from sglang.utils import async_stream_and_merge, stream_and_merge\n", - "\n", - "llm = sgl.Engine(model_path=\"qwen/qwen2.5-0.5b-instruct\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Non-streaming Synchronous Generation" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "prompts = [\n", - " \"Hello, my name is\",\n", - " \"The president of the United States is\",\n", - " \"The capital of France is\",\n", - " \"The future of AI is\",\n", - "]\n", - "\n", - "sampling_params = {\"temperature\": 0.8, \"top_p\": 0.95}\n", - "\n", - "outputs = llm.generate(prompts, sampling_params)\n", - "for prompt, output in zip(prompts, outputs):\n", - " print(\"===============================\")\n", - " print(f\"Prompt: {prompt}\\nGenerated text: {output['text']}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Streaming Synchronous Generation" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "prompts = [\n", - " \"Write a short, neutral self-introduction for a fictional character. Hello, my name is\",\n", - " \"Provide a concise factual statement about France’s capital city. The capital of France is\",\n", - " \"Explain possible future trends in artificial intelligence. The future of AI is\",\n", - "]\n", - "\n", - "sampling_params = {\n", - " \"temperature\": 0.2,\n", - " \"top_p\": 0.9,\n", - "}\n", - "\n", - "print(\"\\n=== Testing synchronous streaming generation with overlap removal ===\\n\")\n", - "\n", - "for prompt in prompts:\n", - " print(f\"Prompt: {prompt}\")\n", - " merged_output = stream_and_merge(llm, prompt, sampling_params)\n", - " print(\"Generated text:\", merged_output)\n", - " print()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Non-streaming Asynchronous Generation" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "prompts = [\n", - " \"Write a short, neutral self-introduction for a fictional character. Hello, my name is\",\n", - " \"Provide a concise factual statement about France’s capital city. The capital of France is\",\n", - " \"Explain possible future trends in artificial intelligence. The future of AI is\",\n", - "]\n", - "\n", - "sampling_params = {\"temperature\": 0.8, \"top_p\": 0.95}\n", - "\n", - "print(\"\\n=== Testing asynchronous batch generation ===\")\n", - "\n", - "\n", - "async def main():\n", - " outputs = await llm.async_generate(prompts, sampling_params)\n", - "\n", - " for prompt, output in zip(prompts, outputs):\n", - " print(f\"\\nPrompt: {prompt}\")\n", - " print(f\"Generated text: {output['text']}\")\n", - "\n", - "\n", - "asyncio.run(main())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Streaming Asynchronous Generation" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "prompts = [\n", - " \"Write a short, neutral self-introduction for a fictional character. Hello, my name is\",\n", - " \"Provide a concise factual statement about France’s capital city. The capital of France is\",\n", - " \"Explain possible future trends in artificial intelligence. The future of AI is\",\n", - "]\n", - "\n", - "sampling_params = {\"temperature\": 0.8, \"top_p\": 0.95}\n", - "\n", - "print(\"\\n=== Testing asynchronous streaming generation (no repeats) ===\")\n", - "\n", - "\n", - "async def main():\n", - " for prompt in prompts:\n", - " print(f\"\\nPrompt: {prompt}\")\n", - " print(\"Generated text: \", end=\"\", flush=True)\n", - "\n", - " # Replace direct calls to async_generate with our custom overlap-aware version\n", - " async for cleaned_chunk in async_stream_and_merge(llm, prompt, sampling_params):\n", - " print(cleaned_chunk, end=\"\", flush=True)\n", - "\n", - " print() # New line after each prompt\n", - "\n", - "\n", - "asyncio.run(main())" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "llm.shutdown()" - ] - } - ], - "metadata": { - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/docs/basic_usage/ollama_api.md b/docs/basic_usage/ollama_api.md deleted file mode 100644 index c7b302c96..000000000 --- a/docs/basic_usage/ollama_api.md +++ /dev/null @@ -1,91 +0,0 @@ -# Ollama-Compatible API - -SGLang provides Ollama API compatibility, allowing you to use the Ollama CLI and Python library with SGLang as the inference backend. - -## Prerequisites - -```bash -# Install the Ollama Python library (for Python client usage) -pip install ollama -``` - -> **Note**: You don't need the Ollama server installed - SGLang acts as the backend. You only need the `ollama` CLI or Python library as the client. - -## Endpoints - -| Endpoint | Method | Description | -|----------|--------|-------------| -| `/` | GET, HEAD | Health check for Ollama CLI | -| `/api/tags` | GET | List available models | -| `/api/chat` | POST | Chat completions (streaming & non-streaming) | -| `/api/generate` | POST | Text generation (streaming & non-streaming) | -| `/api/show` | POST | Model information | - -## Quick Start - -### 1. Launch SGLang Server - -```bash -python -m sglang.launch_server \ - --model Qwen/Qwen2.5-1.5B-Instruct \ - --port 30001 \ - --host 0.0.0.0 -``` - -> **Note**: The model name used with `ollama run` must match exactly what you passed to `--model`. - -### 2. Use Ollama CLI - -```bash -# List available models -OLLAMA_HOST=http://localhost:30001 ollama list - -# Interactive chat -OLLAMA_HOST=http://localhost:30001 ollama run "Qwen/Qwen2.5-1.5B-Instruct" -``` - -If connecting to a remote server behind a firewall: - -```bash -# SSH tunnel -ssh -L 30001:localhost:30001 user@gpu-server -N & - -# Then use Ollama CLI as above -OLLAMA_HOST=http://localhost:30001 ollama list -``` - -### 3. Use Ollama Python Library - -```python -import ollama - -client = ollama.Client(host='http://localhost:30001') - -# Non-streaming -response = client.chat( - model='Qwen/Qwen2.5-1.5B-Instruct', - messages=[{'role': 'user', 'content': 'Hello!'}] -) -print(response['message']['content']) - -# Streaming -stream = client.chat( - model='Qwen/Qwen2.5-1.5B-Instruct', - messages=[{'role': 'user', 'content': 'Tell me a story'}], - stream=True -) -for chunk in stream: - print(chunk['message']['content'], end='', flush=True) -``` - -## Smart Router - -For intelligent routing between local Ollama (fast) and remote SGLang (powerful) using an LLM judge, see the [Smart Router documentation](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/entrypoints/ollama/README.md). - -## Summary - -| Component | Purpose | -|-----------|---------| -| **Ollama API** | Familiar CLI/API that developers already know | -| **SGLang Backend** | High-performance inference engine | -| **Smart Router** | Intelligent routing - fast local for simple tasks, powerful remote for complex tasks | diff --git a/docs/basic_usage/openai_api.rst b/docs/basic_usage/openai_api.rst deleted file mode 100644 index 370abe99c..000000000 --- a/docs/basic_usage/openai_api.rst +++ /dev/null @@ -1,9 +0,0 @@ -OpenAI-Compatible APIs -====================== - -.. toctree:: - :maxdepth: 1 - - openai_api_completions.ipynb - openai_api_vision.ipynb - openai_api_embeddings.ipynb diff --git a/docs/basic_usage/openai_api_completions.ipynb b/docs/basic_usage/openai_api_completions.ipynb deleted file mode 100644 index ffa576ae5..000000000 --- a/docs/basic_usage/openai_api_completions.ipynb +++ /dev/null @@ -1,552 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# OpenAI APIs - Completions\n", - "\n", - "SGLang provides OpenAI-compatible APIs to enable a smooth transition from OpenAI services to self-hosted local models.\n", - "A complete reference for the API is available in the [OpenAI API Reference](https://platform.openai.com/docs/api-reference).\n", - "\n", - "This tutorial covers the following popular APIs:\n", - "\n", - "- `chat/completions`\n", - "- `completions`\n", - "\n", - "Check out other tutorials to learn about [vision APIs](openai_api_vision.ipynb) for vision-language models and [embedding APIs](openai_api_embeddings.ipynb) for embedding models." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Launch A Server\n", - "\n", - "Launch the server in your terminal and wait for it to initialize." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from sglang.test.doc_patch import launch_server_cmd\n", - "from sglang.utils import wait_for_server, print_highlight, terminate_process\n", - "\n", - "server_process, port = launch_server_cmd(\n", - " \"python3 -m sglang.launch_server --model-path qwen/qwen2.5-0.5b-instruct --host 0.0.0.0 --log-level warning\"\n", - ")\n", - "\n", - "wait_for_server(f\"http://localhost:{port}\", process=server_process)\n", - "print(f\"Server started on http://localhost:{port}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Chat Completions\n", - "\n", - "### Usage\n", - "\n", - "The server fully implements the OpenAI API.\n", - "It will automatically apply the chat template specified in the Hugging Face tokenizer, if one is available.\n", - "You can also specify a custom chat template with `--chat-template` when launching the server." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import openai\n", - "\n", - "client = openai.Client(base_url=f\"http://127.0.0.1:{port}/v1\", api_key=\"None\")\n", - "\n", - "response = client.chat.completions.create(\n", - " model=\"qwen/qwen2.5-0.5b-instruct\",\n", - " messages=[\n", - " {\"role\": \"user\", \"content\": \"List 3 countries and their capitals.\"},\n", - " ],\n", - " temperature=0,\n", - " max_tokens=64,\n", - ")\n", - "\n", - "print_highlight(f\"Response: {response}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Model Thinking/Reasoning Support\n", - "\n", - "Some models support internal reasoning or thinking processes that can be exposed in the API response. SGLang provides unified support for various reasoning models through the `chat_template_kwargs` parameter and compatible reasoning parsers.\n", - "\n", - "#### Supported Models and Configuration\n", - "\n", - "| Model Family | Chat Template Parameter | Reasoning Parser | Notes |\n", - "|--------------|------------------------|------------------|--------|\n", - "| DeepSeek-R1 (R1, R1-0528, R1-Distill) | `enable_thinking` | `--reasoning-parser deepseek-r1` | Standard reasoning models |\n", - "| DeepSeek-V3.1 | `thinking` | `--reasoning-parser deepseek-v3` | Hybrid model (thinking/non-thinking modes) |\n", - "| Qwen3 (standard) | `enable_thinking` | `--reasoning-parser qwen3` | Hybrid model (thinking/non-thinking modes) |\n", - "| Qwen3-Thinking | N/A (always enabled) | `--reasoning-parser qwen3-thinking` | Always generates reasoning |\n", - "| Kimi | N/A (always enabled) | `--reasoning-parser kimi` | Kimi thinking models |\n", - "| Gpt-Oss | N/A (always enabled) | `--reasoning-parser gpt-oss` | Gpt-Oss thinking models |\n", - "\n", - "#### Basic Usage\n", - "\n", - "To enable reasoning output, you need to:\n", - "1. Launch the server with the appropriate reasoning parser\n", - "2. Set the model-specific parameter in `chat_template_kwargs`\n", - "3. Optionally use `separate_reasoning: False` to not get reasoning content separately (default to `True`)\n", - "\n", - "**Note for Qwen3-Thinking models:** These models always generate thinking content and do not support the `enable_thinking` parameter. Use `--reasoning-parser qwen3-thinking` or `--reasoning-parser qwen3` to parse the thinking content.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Example: Qwen3 Models\n", - "\n", - "```python\n", - "# Launch server:\n", - "# python3 -m sglang.launch_server --model Qwen/Qwen3-4B --reasoning-parser qwen3\n", - "\n", - "from openai import OpenAI\n", - "\n", - "client = OpenAI(\n", - " api_key=\"EMPTY\",\n", - " base_url=f\"http://127.0.0.1:30000/v1\",\n", - ")\n", - "\n", - "model = \"Qwen/Qwen3-4B\"\n", - "messages = [{\"role\": \"user\", \"content\": \"How many r's are in 'strawberry'?\"}]\n", - "\n", - "response = client.chat.completions.create(\n", - " model=model,\n", - " messages=messages,\n", - " extra_body={\n", - " \"chat_template_kwargs\": {\"enable_thinking\": True},\n", - " \"separate_reasoning\": True\n", - " }\n", - ")\n", - "\n", - "print(\"Reasoning:\", response.choices[0].message.reasoning_content)\n", - "print(\"-\"*100)\n", - "print(\"Answer:\", response.choices[0].message.content)\n", - "```\n", - "\n", - "**ExampleOutput:**\n", - "```\n", - "Reasoning: Okay, so the user is asking how many 'r's are in the word 'strawberry'. Let me think. First, I need to make sure I have the word spelled correctly. Strawberry... S-T-R-A-W-B-E-R-R-Y. Wait, is that right? Let me break it down.\n", - "\n", - "Starting with 'strawberry', let's write out the letters one by one. S, T, R, A, W, B, E, R, R, Y. Hmm, wait, that's 10 letters. Let me check again. S (1), T (2), R (3), A (4), W (5), B (6), E (7), R (8), R (9), Y (10). So the letters are S-T-R-A-W-B-E-R-R-Y. \n", - "...\n", - "Therefore, the answer should be three R's in 'strawberry'. But I need to make sure I'm not counting any other letters as R. Let me check again. S, T, R, A, W, B, E, R, R, Y. No other R's. So three in total. Yeah, that seems right.\n", - "\n", - "----------------------------------------------------------------------------------------------------\n", - "Answer: The word \"strawberry\" contains **three** letters 'r'. Here's the breakdown:\n", - "\n", - "1. **S-T-R-A-W-B-E-R-R-Y** \n", - " - The **third letter** is 'R'. \n", - " - The **eighth and ninth letters** are also 'R's. \n", - "\n", - "Thus, the total count is **3**. \n", - "\n", - "**Answer:** 3.\n", - "```\n", - "\n", - "**Note:** Setting `\"enable_thinking\": False` (or omitting it) will result in `reasoning_content` being `None`. Qwen3-Thinking models always generate reasoning content and don't support the `enable_thinking` parameter.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Logit Bias Support\n", - "\n", - "SGLang supports the `logit_bias` parameter for both chat completions and completions APIs. This parameter allows you to modify the likelihood of specific tokens being generated by adding bias values to their logits. The bias values can range from -100 to 100, where:\n", - "\n", - "- **Positive values** (0 to 100) increase the likelihood of the token being selected\n", - "- **Negative values** (-100 to 0) decrease the likelihood of the token being selected\n", - "- **-100** effectively prevents the token from being generated\n", - "\n", - "The `logit_bias` parameter accepts a dictionary where keys are token IDs (as strings) and values are the bias amounts (as floats).\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Getting Token IDs\n", - "\n", - "To use `logit_bias` effectively, you need to know the token IDs for the words you want to bias. Here's how to get token IDs:\n", - "\n", - "```python\n", - "# Get tokenizer to find token IDs\n", - "import tiktoken\n", - "\n", - "# For OpenAI models, use the appropriate encoding\n", - "tokenizer = tiktoken.encoding_for_model(\"gpt-3.5-turbo\") # or your model\n", - "\n", - "# Get token IDs for specific words\n", - "word = \"sunny\"\n", - "token_ids = tokenizer.encode(word)\n", - "print(f\"Token IDs for '{word}': {token_ids}\")\n", - "\n", - "# For SGLang models, you can access the tokenizer through the client\n", - "# and get token IDs for bias\n", - "```\n", - "\n", - "**Important:** The `logit_bias` parameter uses token IDs as string keys, not the actual words.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Example: DeepSeek-V3 Models\n", - "\n", - "DeepSeek-V3 models support thinking mode through the `thinking` parameter:\n", - "\n", - "```python\n", - "# Launch server:\n", - "# python3 -m sglang.launch_server --model deepseek-ai/DeepSeek-V3.1 --tp 8 --reasoning-parser deepseek-v3\n", - "\n", - "from openai import OpenAI\n", - "\n", - "client = OpenAI(\n", - " api_key=\"EMPTY\",\n", - " base_url=f\"http://127.0.0.1:30000/v1\",\n", - ")\n", - "\n", - "model = \"deepseek-ai/DeepSeek-V3.1\"\n", - "messages = [{\"role\": \"user\", \"content\": \"How many r's are in 'strawberry'?\"}]\n", - "\n", - "response = client.chat.completions.create(\n", - " model=model,\n", - " messages=messages,\n", - " extra_body={\n", - " \"chat_template_kwargs\": {\"thinking\": True},\n", - " \"separate_reasoning\": True\n", - " }\n", - ")\n", - "\n", - "print(\"Reasoning:\", response.choices[0].message.reasoning_content)\n", - "print(\"-\"*100)\n", - "print(\"Answer:\", response.choices[0].message.content)\n", - "```\n", - "\n", - "**Example Output:**\n", - "```\n", - "Reasoning: First, the question is: \"How many r's are in 'strawberry'?\"\n", - "\n", - "I need to count the number of times the letter 'r' appears in the word \"strawberry\".\n", - "\n", - "Let me write out the word: S-T-R-A-W-B-E-R-R-Y.\n", - "\n", - "Now, I'll go through each letter and count the 'r's.\n", - "...\n", - "So, I have three 'r's in \"strawberry\".\n", - "\n", - "I should double-check. The word is spelled S-T-R-A-W-B-E-R-R-Y. The letters are at positions: 3, 8, and 9 are 'r's. Yes, that's correct.\n", - "\n", - "Therefore, the answer should be 3.\n", - "----------------------------------------------------------------------------------------------------\n", - "Answer: The word \"strawberry\" contains **3** instances of the letter \"r\". Here's a breakdown for clarity:\n", - "\n", - "- The word is spelled: S-T-R-A-W-B-E-R-R-Y\n", - "- The \"r\" appears at the 3rd, 8th, and 9th positions.\n", - "```\n", - "\n", - "**Note:** DeepSeek-V3 models use the `thinking` parameter (not `enable_thinking`) to control reasoning output.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Example with logit_bias parameter\n", - "# Note: You need to get the actual token IDs from your tokenizer\n", - "# For demonstration, we'll use some example token IDs\n", - "response = client.chat.completions.create(\n", - " model=\"qwen/qwen2.5-0.5b-instruct\",\n", - " messages=[\n", - " {\"role\": \"user\", \"content\": \"Complete this sentence: The weather today is\"}\n", - " ],\n", - " temperature=0.7,\n", - " max_tokens=20,\n", - " logit_bias={\n", - " \"12345\": 50, # Increase likelihood of token ID 12345\n", - " \"67890\": -50, # Decrease likelihood of token ID 67890\n", - " \"11111\": 25, # Slightly increase likelihood of token ID 11111\n", - " },\n", - ")\n", - "\n", - "print_highlight(f\"Response with logit bias: {response.choices[0].message.content}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Parameters\n", - "\n", - "The chat completions API accepts OpenAI Chat Completions API's parameters. Refer to [OpenAI Chat Completions API](https://platform.openai.com/docs/api-reference/chat/create) for more details.\n", - "\n", - "SGLang extends the standard API with the `extra_body` parameter, allowing for additional customization. One key option within `extra_body` is `chat_template_kwargs`, which can be used to pass arguments to the chat template processor." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "response = client.chat.completions.create(\n", - " model=\"qwen/qwen2.5-0.5b-instruct\",\n", - " messages=[\n", - " {\n", - " \"role\": \"system\",\n", - " \"content\": \"You are a knowledgeable historian who provides concise responses.\",\n", - " },\n", - " {\"role\": \"user\", \"content\": \"Tell me about ancient Rome\"},\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"Ancient Rome was a civilization centered in Italy.\",\n", - " },\n", - " {\"role\": \"user\", \"content\": \"What were their major achievements?\"},\n", - " ],\n", - " temperature=0.3, # Lower temperature for more focused responses\n", - " max_tokens=128, # Reasonable length for a concise response\n", - " top_p=0.95, # Slightly higher for better fluency\n", - " presence_penalty=0.2, # Mild penalty to avoid repetition\n", - " frequency_penalty=0.2, # Mild penalty for more natural language\n", - " n=1, # Single response is usually more stable\n", - " seed=42, # Keep for reproducibility\n", - ")\n", - "\n", - "print_highlight(response.choices[0].message.content)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Streaming mode is also supported." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Logit Bias Support\n", - "\n", - "The completions API also supports the `logit_bias` parameter with the same functionality as described in the chat completions section above.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "stream = client.chat.completions.create(\n", - " model=\"qwen/qwen2.5-0.5b-instruct\",\n", - " messages=[{\"role\": \"user\", \"content\": \"Say this is a test\"}],\n", - " stream=True,\n", - ")\n", - "for chunk in stream:\n", - " if chunk.choices[0].delta.content is not None:\n", - " print(chunk.choices[0].delta.content, end=\"\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Returning Routed Experts (MoE Models)\n", - "\n", - "For MoE models, set `return_routed_experts: true` in `extra_body` to return expert routing data. Requires `--enable-return-routed-experts` server flag. The `routed_experts` field will be returned in the `sgl_ext` object on each choice, containing base64-encoded int32 expert IDs as a flattened array with logical shape `[num_tokens, num_layers, top_k]`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Example with logit_bias parameter for completions API\n", - "# Note: You need to get the actual token IDs from your tokenizer\n", - "# For demonstration, we'll use some example token IDs\n", - "response = client.completions.create(\n", - " model=\"qwen/qwen2.5-0.5b-instruct\",\n", - " prompt=\"The best programming language for AI is\",\n", - " temperature=0.7,\n", - " max_tokens=20,\n", - " logit_bias={\n", - " \"12345\": 75, # Strongly favor token ID 12345\n", - " \"67890\": -100, # Completely avoid token ID 67890\n", - " \"11111\": -25, # Slightly discourage token ID 11111\n", - " },\n", - ")\n", - "\n", - "print_highlight(f\"Response with logit bias: {response.choices[0].text}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Completions\n", - "\n", - "### Usage\n", - "Completions API is similar to Chat Completions API, but without the `messages` parameter or chat templates." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "response = client.completions.create(\n", - " model=\"qwen/qwen2.5-0.5b-instruct\",\n", - " prompt=\"List 3 countries and their capitals.\",\n", - " temperature=0,\n", - " max_tokens=64,\n", - " n=1,\n", - " stop=None,\n", - ")\n", - "\n", - "print_highlight(f\"Response: {response}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Parameters\n", - "\n", - "The completions API accepts OpenAI Completions API's parameters. Refer to [OpenAI Completions API](https://platform.openai.com/docs/api-reference/completions/create) for more details.\n", - "\n", - "Here is an example of a detailed completions request:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "response = client.completions.create(\n", - " model=\"qwen/qwen2.5-0.5b-instruct\",\n", - " prompt=\"Write a short story about a space explorer.\",\n", - " temperature=0.7, # Moderate temperature for creative writing\n", - " max_tokens=150, # Longer response for a story\n", - " top_p=0.9, # Balanced diversity in word choice\n", - " stop=[\"\\n\\n\", \"THE END\"], # Multiple stop sequences\n", - " presence_penalty=0.3, # Encourage novel elements\n", - " frequency_penalty=0.3, # Reduce repetitive phrases\n", - " n=1, # Generate one completion\n", - " seed=123, # For reproducible results\n", - ")\n", - "\n", - "print_highlight(f\"Response: {response}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Returning Routed Experts (MoE Models)\n", - "\n", - "For MoE models, set `return_routed_experts: true` in `extra_body` to return expert routing data. Requires `--enable-return-routed-experts` server flag. The `routed_experts` field will be returned in the `sgl_ext` object on each choice, containing base64-encoded int32 expert IDs as a flattened array with logical shape `[num_tokens, num_layers, top_k]`." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Structured Outputs (JSON, Regex, EBNF)\n", - "\n", - "For OpenAI compatible structured outputs API, refer to [Structured Outputs](../advanced_features/structured_outputs.ipynb) for more details.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Using LoRA Adapters\n", - "\n", - "SGLang supports LoRA (Low-Rank Adaptation) adapters with OpenAI-compatible APIs. You can specify which adapter to use directly in the `model` parameter using the `base-model:adapter-name` syntax.\n", - "\n", - "**Server Setup:**\n", - "```bash\n", - "python -m sglang.launch_server \\\n", - " --model-path qwen/qwen2.5-0.5b-instruct \\\n", - " --enable-lora \\\n", - " --lora-paths adapter_a=/path/to/adapter_a adapter_b=/path/to/adapter_b\n", - "```\n", - "\n", - "For more details on LoRA serving configuration, see the [LoRA documentation](../advanced_features/lora.ipynb).\n", - "\n", - "**API Call:**\n", - "\n", - "(Recommended) Use the `model:adapter` syntax to specify which adapter to use:\n", - "```python\n", - "response = client.chat.completions.create(\n", - " model=\"qwen/qwen2.5-0.5b-instruct:adapter_a\", # ← base-model:adapter-name\n", - " messages=[{\"role\": \"user\", \"content\": \"Convert to SQL: show all users\"}],\n", - " max_tokens=50,\n", - ")\n", - "```\n", - "\n", - "**Backward Compatible: Using `extra_body`**\n", - "\n", - "The old `extra_body` method is still supported for backward compatibility:\n", - "```python\n", - "# Backward compatible method\n", - "response = client.chat.completions.create(\n", - " model=\"qwen/qwen2.5-0.5b-instruct\",\n", - " messages=[{\"role\": \"user\", \"content\": \"Convert to SQL: show all users\"}],\n", - " extra_body={\"lora_path\": \"adapter_a\"}, # ← old method\n", - " max_tokens=50,\n", - ")\n", - "```\n", - "**Note:** When both `model:adapter` and `extra_body[\"lora_path\"]` are specified, the `model:adapter` syntax takes precedence." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "terminate_process(server_process)" - ] - } - ], - "metadata": { - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/docs/basic_usage/openai_api_embeddings.ipynb b/docs/basic_usage/openai_api_embeddings.ipynb deleted file mode 100644 index a6c90c06b..000000000 --- a/docs/basic_usage/openai_api_embeddings.ipynb +++ /dev/null @@ -1,193 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# OpenAI APIs - Embedding\n", - "\n", - "SGLang provides OpenAI-compatible APIs to enable a smooth transition from OpenAI services to self-hosted local models.\n", - "A complete reference for the API is available in the [OpenAI API Reference](https://platform.openai.com/docs/guides/embeddings).\n", - "\n", - "This tutorial covers the embedding APIs for embedding models. For a list of the supported models see the [corresponding overview page](../supported_models/retrieval_ranking/embedding_models.md)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Launch A Server\n", - "\n", - "Launch the server in your terminal and wait for it to initialize. Remember to add `--is-embedding` to the command." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from sglang.test.doc_patch import launch_server_cmd\n", - "from sglang.utils import wait_for_server, print_highlight, terminate_process\n", - "\n", - "embedding_process, port = launch_server_cmd(\"\"\"\n", - "python3 -m sglang.launch_server --model-path Alibaba-NLP/gte-Qwen2-1.5B-instruct \\\n", - " --host 0.0.0.0 --is-embedding --log-level warning\n", - "\"\"\")\n", - "\n", - "wait_for_server(f\"http://localhost:{port}\", process=embedding_process)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Using cURL" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import subprocess, json\n", - "\n", - "text = \"Once upon a time\"\n", - "\n", - "curl_text = f\"\"\"curl -s http://localhost:{port}/v1/embeddings \\\n", - " -H \"Content-Type: application/json\" \\\n", - " -d '{{\"model\": \"Alibaba-NLP/gte-Qwen2-1.5B-instruct\", \"input\": \"{text}\"}}'\"\"\"\n", - "\n", - "result = subprocess.check_output(curl_text, shell=True)\n", - "\n", - "print(result)\n", - "\n", - "text_embedding = json.loads(result)[\"data\"][0][\"embedding\"]\n", - "\n", - "print_highlight(f\"Text embedding (first 10): {text_embedding[:10]}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Using Python Requests" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import requests\n", - "\n", - "text = \"Once upon a time\"\n", - "\n", - "response = requests.post(\n", - " f\"http://localhost:{port}/v1/embeddings\",\n", - " json={\"model\": \"Alibaba-NLP/gte-Qwen2-1.5B-instruct\", \"input\": text},\n", - ")\n", - "\n", - "text_embedding = response.json()[\"data\"][0][\"embedding\"]\n", - "\n", - "print_highlight(f\"Text embedding (first 10): {text_embedding[:10]}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Using OpenAI Python Client" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import openai\n", - "\n", - "client = openai.Client(base_url=f\"http://127.0.0.1:{port}/v1\", api_key=\"None\")\n", - "\n", - "# Text embedding example\n", - "response = client.embeddings.create(\n", - " model=\"Alibaba-NLP/gte-Qwen2-1.5B-instruct\",\n", - " input=text,\n", - ")\n", - "\n", - "embedding = response.data[0].embedding[:10]\n", - "print_highlight(f\"Text embedding (first 10): {embedding}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Using Input IDs\n", - "\n", - "SGLang also supports `input_ids` as input to get the embedding." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import json\n", - "import os\n", - "from transformers import AutoTokenizer\n", - "\n", - "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n", - "\n", - "tokenizer = AutoTokenizer.from_pretrained(\"Alibaba-NLP/gte-Qwen2-1.5B-instruct\")\n", - "input_ids = tokenizer.encode(text)\n", - "\n", - "curl_ids = f\"\"\"curl -s http://localhost:{port}/v1/embeddings \\\n", - " -H \"Content-Type: application/json\" \\\n", - " -d '{{\"model\": \"Alibaba-NLP/gte-Qwen2-1.5B-instruct\", \"input\": {json.dumps(input_ids)}}}'\"\"\"\n", - "\n", - "input_ids_embedding = json.loads(subprocess.check_output(curl_ids, shell=True))[\"data\"][\n", - " 0\n", - "][\"embedding\"]\n", - "\n", - "print_highlight(f\"Input IDs embedding (first 10): {input_ids_embedding[:10]}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "terminate_process(embedding_process)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Multi-Modal Embedding Model\n", - "Please refer to [Multi-Modal Embedding Model](../supported_models/retrieval_ranking/embedding_models.md)" - ] - } - ], - "metadata": { - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/docs/basic_usage/openai_api_vision.ipynb b/docs/basic_usage/openai_api_vision.ipynb deleted file mode 100644 index b6e6a1a24..000000000 --- a/docs/basic_usage/openai_api_vision.ipynb +++ /dev/null @@ -1,253 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# OpenAI APIs - Vision\n", - "\n", - "SGLang provides OpenAI-compatible APIs to enable a smooth transition from OpenAI services to self-hosted local models.\n", - "A complete reference for the API is available in the [OpenAI API Reference](https://platform.openai.com/docs/guides/vision).\n", - "This tutorial covers the vision APIs for vision language models.\n", - "\n", - "SGLang supports various vision language models such as Llama 3.2, LLaVA-OneVision, Qwen2.5-VL, Gemma3 and [more](../supported_models/text_generation/multimodal_language_models.md).\n", - "\n", - "As an alternative to the OpenAI API, you can also use the [SGLang offline engine](https://github.com/sgl-project/sglang/blob/main/examples/runtime/engine/offline_batch_inference_vlm.py)." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Launch A Server\n", - "\n", - "Launch the server in your terminal and wait for it to initialize." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from sglang.test.doc_patch import launch_server_cmd\n", - "from sglang.utils import wait_for_server, print_highlight, terminate_process\n", - "\n", - "example_image_url = \"https://raw.githubusercontent.com/sgl-project/sglang/main/examples/assets/example_image.png\"\n", - "logo_image_url = (\n", - " \"https://raw.githubusercontent.com/sgl-project/sglang/main/assets/logo.png\"\n", - ")\n", - "\n", - "vision_process, port = launch_server_cmd(\"\"\"\n", - "python3 -m sglang.launch_server --model-path Qwen/Qwen2.5-VL-7B-Instruct --log-level warning\n", - "\"\"\")\n", - "\n", - "wait_for_server(f\"http://localhost:{port}\", process=vision_process)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Using cURL\n", - "\n", - "Once the server is up, you can send test requests using curl or requests." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import subprocess\n", - "\n", - "curl_command = f\"\"\"\n", - "curl -s http://localhost:{port}/v1/chat/completions \\\\\n", - " -H \"Content-Type: application/json\" \\\\\n", - " -d '{{\n", - " \"model\": \"Qwen/Qwen2.5-VL-7B-Instruct\",\n", - " \"messages\": [\n", - " {{\n", - " \"role\": \"user\",\n", - " \"content\": [\n", - " {{\n", - " \"type\": \"text\",\n", - " \"text\": \"What’s in this image?\"\n", - " }},\n", - " {{\n", - " \"type\": \"image_url\",\n", - " \"image_url\": {{\n", - " \"url\": \"{example_image_url}\"\n", - " }}\n", - " }}\n", - " ]\n", - " }}\n", - " ],\n", - " \"max_tokens\": 300\n", - " }}'\n", - "\"\"\"\n", - "\n", - "response = subprocess.check_output(curl_command, shell=True).decode()\n", - "print_highlight(response)\n", - "\n", - "\n", - "response = subprocess.check_output(curl_command, shell=True).decode()\n", - "print_highlight(response)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Using Python Requests" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import requests\n", - "\n", - "url = f\"http://localhost:{port}/v1/chat/completions\"\n", - "\n", - "data = {\n", - " \"model\": \"Qwen/Qwen2.5-VL-7B-Instruct\",\n", - " \"messages\": [\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": [\n", - " {\"type\": \"text\", \"text\": \"What’s in this image?\"},\n", - " {\n", - " \"type\": \"image_url\",\n", - " \"image_url\": {\"url\": example_image_url},\n", - " },\n", - " ],\n", - " }\n", - " ],\n", - " \"max_tokens\": 300,\n", - "}\n", - "\n", - "response = requests.post(url, json=data)\n", - "print_highlight(response.text)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Using OpenAI Python Client" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from openai import OpenAI\n", - "\n", - "client = OpenAI(base_url=f\"http://localhost:{port}/v1\", api_key=\"None\")\n", - "\n", - "response = client.chat.completions.create(\n", - " model=\"Qwen/Qwen2.5-VL-7B-Instruct\",\n", - " messages=[\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"text\",\n", - " \"text\": \"What is in this image?\",\n", - " },\n", - " {\n", - " \"type\": \"image_url\",\n", - " \"image_url\": {\"url\": example_image_url},\n", - " },\n", - " ],\n", - " }\n", - " ],\n", - " max_tokens=300,\n", - ")\n", - "\n", - "print_highlight(response.choices[0].message.content)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Multiple-Image Inputs\n", - "\n", - "The server also supports multiple images and interleaved text and images if the model supports it." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from openai import OpenAI\n", - "\n", - "client = OpenAI(base_url=f\"http://localhost:{port}/v1\", api_key=\"None\")\n", - "\n", - "response = client.chat.completions.create(\n", - " model=\"Qwen/Qwen2.5-VL-7B-Instruct\",\n", - " messages=[\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"image_url\",\n", - " \"image_url\": {\n", - " \"url\": example_image_url,\n", - " },\n", - " },\n", - " {\n", - " \"type\": \"image_url\",\n", - " \"image_url\": {\n", - " \"url\": logo_image_url,\n", - " },\n", - " },\n", - " {\n", - " \"type\": \"text\",\n", - " \"text\": \"I have two very different images. They are not related at all. \"\n", - " \"Please describe the first image in one sentence, and then describe the second image in another sentence.\",\n", - " },\n", - " ],\n", - " }\n", - " ],\n", - " temperature=0,\n", - ")\n", - "\n", - "print_highlight(response.choices[0].message.content)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "terminate_process(vision_process)" - ] - } - ], - "metadata": { - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/docs/basic_usage/popular_model_usage.rst b/docs/basic_usage/popular_model_usage.rst deleted file mode 100644 index ec0268ed7..000000000 --- a/docs/basic_usage/popular_model_usage.rst +++ /dev/null @@ -1,19 +0,0 @@ -Popular Model Usage (DeepSeek, GPT-OSS, GLM, Llama, MiniMax, Qwen, and more) -=============================================================== - -For more usage examples and recipes, visit the `SGLang Cookbook `_. - -.. toctree:: - :maxdepth: 1 - - deepseek_v3.md - deepseek_v32.md - glm45.md - glmv.md - gpt_oss.md - minimax_m2.md - qwen3.md - qwen3_5.md - qwen3_vl.md - deepseek_ocr.md - llama4.md diff --git a/docs/basic_usage/qwen3.md b/docs/basic_usage/qwen3.md deleted file mode 100644 index c0364176c..000000000 --- a/docs/basic_usage/qwen3.md +++ /dev/null @@ -1,39 +0,0 @@ -# Qwen3-Next Usage - -SGLang has supported Qwen3-Next-80B-A3B-Instruct and Qwen3-Next-80B-A3B-Thinking since [this PR](https://github.com/sgl-project/sglang/pull/10233). - -## Launch Qwen3-Next with SGLang - -To serve Qwen3-Next models on 4xH100/H200 GPUs: - -```bash -python3 -m sglang.launch_server --model Qwen/Qwen3-Next-80B-A3B-Instruct --tp 4 -``` - -### Configuration Tips -- `--max-mamba-cache-size`: Adjust `--max-mamba-cache-size` to increase mamba cache space and max running requests capability. It will decrease KV cache space as a trade-off. You can adjust it according to workload. -- `--mamba-ssm-dtype`: `bfloat16` or `float32`, use `bfloat16` to save mamba cache size and `float32` to get more accurate results. The default setting is `float32`. -- `--mamba-full-memory-ratio`: The ratio of mamba state memory to full kv cache memory. The default is 0.9. - -### Mamba Radix Cache -SGLang supports prefix caching for Qwen3-Next models named `MambaRadixCache`, which improves inference speed by reusing computation results. There are two versions of `MambaRadixCache`: -- `no_buffer`: The default version, which is also other hybrid linear models' choice. When it is enabled, SGLang will automatically close overlap schedule for compatibility reasons. -- `extra_buffer`: An optimized version that is compatible with features like page size > 1, overlap schedule, and speculative decoding. It also supports storing mamba state in branching positions. However, it requires two extra mamba spaces for a ping-pong buffer for each request. To enable it, add the argument `--mamba-scheduler-strategy extra_buffer` when launching the server. - -### EAGLE Speculative Decoding -**Description**: SGLang has supported Qwen3-Next models with [EAGLE speculative decoding](https://docs.sglang.io/advanced_features/speculative_decoding.html#EAGLE-Decoding). - -**Usage**: -Add arguments `--speculative-algorithm`, `--speculative-num-steps`, `--speculative-eagle-topk` and `--speculative-num-draft-tokens` to enable this feature. For example: - -``` bash -python3 -m sglang.launch_server \ - --model Qwen/Qwen3-Next-80B-A3B-Instruct \ - --tp 4 \ - --speculative-num-steps 3 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 \ - --speculative-algo NEXTN -``` - -Details can be seen in [this PR](https://github.com/sgl-project/sglang/pull/10233). diff --git a/docs/basic_usage/qwen3_5.md b/docs/basic_usage/qwen3_5.md deleted file mode 100644 index 06f7b615e..000000000 --- a/docs/basic_usage/qwen3_5.md +++ /dev/null @@ -1,76 +0,0 @@ -# Qwen 3.5 Usage - -Qwen 3.5 is Alibaba's latest generation LLM featuring a hybrid attention architecture, advanced MoE with shared experts, and native multimodal capabilities. - -Key architecture features: -- **Hybrid Attention**: Gated Delta Networks (linear, O(n) complexity) combined with full attention every 4th layer for high associative recall -- **MoE with Shared Experts**: Top-8 active out of 64 routed experts plus a dedicated shared expert for universal features -- **Multimodal**: DeepStack Vision Transformer with Conv3d for native image and video understanding - -## Launch Qwen 3.5 with SGLang - -### Dense Model - -To serve `Qwen/Qwen3.5-397B-A17B` on 8 GPUs: - -```bash -python3 -m sglang.launch_server \ - --model-path Qwen/Qwen3.5-397B-A17B \ - --tp 8 \ - --trust-remote-code -``` - -### AMD GPU (MI300X / MI325X / MI35X) - -On AMD Instinct GPUs, use the `triton` attention backend. Both the full attention layers and the Gated Delta Net (linear attention) layers use Triton-based kernels on ROCm: - -```bash -SGLANG_USE_AITER=1 python3 -m sglang.launch_server \ - --model-path Qwen/Qwen3.5-397B-A17B \ - --tp 8 \ - --attention-backend triton \ - --trust-remote-code -``` - -```{tip} -Set `SGLANG_USE_AITER=1` to enable AMD's optimized aiter kernels for MoE and GEMM operations. -``` - -### Configuration Tips - -- `--attention-backend`: Use `triton` on AMD GPUs for Qwen 3.5. The hybrid attention architecture (Gated Delta Networks + full attention) works best with the Triton backend on ROCm. The linear attention (GDN) layers always use Triton kernels internally via the `GDNAttnBackend`. -- `--watchdog-timeout`: Increase to `1200` or higher for this large model, as weight loading takes significant time. -- `--model-loader-extra-config '{"enable_multithread_load": true}'`: Enables parallel weight loading for faster startup. - -### Reasoning and Tool Calling - -Qwen 3.5 supports reasoning and tool calling via the Qwen3 parsers: - -```bash -python3 -m sglang.launch_server \ - --model-path Qwen/Qwen3.5-397B-A17B \ - --tp 8 \ - --trust-remote-code \ - --reasoning-parser qwen3 \ - --tool-call-parser qwen3_coder -``` - -## Accuracy Evaluation - -You can evaluate the model accuracy using `lm-eval`: - -```bash -pip install lm-eval[api] - -lm_eval --model local-completions \ - --model_args '{"base_url": "http://localhost:8000/v1/completions", "model": "Qwen/Qwen3.5-397B-A17B", "num_concurrent": 256, "max_retries": 10, "max_gen_toks": 2048}' \ - --tasks gsm8k \ - --batch_size auto \ - --num_fewshot 5 \ - --trust_remote_code -``` - -## Additional Resources - -- [AMD Day 0 Support for Qwen 3.5 on AMD Instinct GPUs](https://www.amd.com/en/developer/resources/technical-articles/2026/day-0-support-for-qwen-3-5-on-amd-instinct-gpus.html) -- [HuggingFace Model Card](https://huggingface.co/Qwen/Qwen3.5-397B-A17B) diff --git a/docs/basic_usage/qwen3_vl.md b/docs/basic_usage/qwen3_vl.md deleted file mode 100644 index f05e7832a..000000000 --- a/docs/basic_usage/qwen3_vl.md +++ /dev/null @@ -1,130 +0,0 @@ -# Qwen3-VL Usage - -[Qwen3-VL](https://huggingface.co/collections/Qwen/qwen3-vl) -is Alibaba’s latest multimodal large language model with strong text, vision, and reasoning capabilities. -SGLang supports Qwen3-VL Family of models with Image and Video input support. - -## Launch commands for SGLang - -Below are suggested launch commands tailored for different hardware / precision modes - -### FP8 (quantised) mode -For high memory-efficiency and latency optimized deployments (e.g., on H100, H200) where FP8 checkpoint is supported: -```bash -python3 -m sglang.launch_server \ - --model-path Qwen/Qwen3-VL-235B-A22B-Instruct-FP8 \ - --tp 8 \ - --ep 8 \ - --host 0.0.0.0 \ - --port 30000 \ - --keep-mm-feature-on-device -``` - -### Non-FP8 (BF16 / full precision) mode -For deployments on A100/H100 where BF16 is used (or FP8 snapshot not used): -```bash -python3 -m sglang.launch_server \ - --model-path Qwen/Qwen3-VL-235B-A22B-Instruct \ - --tp 8 \ - --ep 8 \ - --host 0.0.0.0 \ - --port 30000 \ -``` - -## Hardware-specific notes / recommendations - -- On H100 with FP8: Use the FP8 checkpoint for best memory efficiency. -- On A100 / H100 with BF16 (non-FP8): It’s recommended to use `--mm-max-concurrent-calls` to control parallel throughput and GPU memory usage during image/video inference. -- On H200 & B200: The model can be run “out of the box”, supporting full context length plus concurrent image + video processing. - -## Sending Image/Video Requests - -### Image input: - -```python -import requests - -url = f"http://localhost:30000/v1/chat/completions" - -data = { - "model": "Qwen/Qwen3-VL-30B-A3B-Instruct", - "messages": [ - { - "role": "user", - "content": [ - {"type": "text", "text": "What’s in this image?"}, - { - "type": "image_url", - "image_url": { - "url": "https://github.com/sgl-project/sglang/blob/main/examples/assets/example_image.png?raw=true" - }, - }, - ], - } - ], - "max_tokens": 300, -} - -response = requests.post(url, json=data) -print(response.text) -``` - -### Video Input: - -```python -import requests - -url = f"http://localhost:30000/v1/chat/completions" - -data = { - "model": "Qwen/Qwen3-VL-30B-A3B-Instruct", - "messages": [ - { - "role": "user", - "content": [ - {"type": "text", "text": "What’s happening in this video?"}, - { - "type": "video_url", - "video_url": { - "url": "https://github.com/sgl-project/sgl-test-files/raw/refs/heads/main/videos/jobs_presenting_ipod.mp4" - }, - }, - ], - } - ], - "max_tokens": 300, -} - -response = requests.post(url, json=data) -print(response.text) -``` - -## Important Server Parameters and Flags - -When launching the model server for **multimodal support**, you can use the following command-line arguments to fine-tune performance and behavior: - -- `--mm-attention-backend`: Specify multimodal attention backend. Eg. `fa3`(Flash Attention 3) -- `--mm-max-concurrent-calls `: Specifies the **maximum number of concurrent asynchronous multimodal data processing calls** allowed on the server. Use this to control parallel throughput and GPU memory usage during image/video inference. -- `--mm-per-request-timeout `: Defines the **timeout duration (in seconds)** for each multimodal request. If a request exceeds this time limit (e.g., for very large video inputs), it will be automatically terminated. -- `--keep-mm-feature-on-device`: Instructs the server to **retain multimodal feature tensors on the GPU** after processing. This avoids device-to-host (D2H) memory copies and improves performance for repeated or high-frequency inference workloads. -- `SGLANG_USE_CUDA_IPC_TRANSPORT=1`: Shared memory pool based CUDA IPC for multi-modal data transport. For significantly improving e2e latency. - -### Example usage with the above optimizations: -```bash -SGLANG_USE_CUDA_IPC_TRANSPORT=1 \ -SGLANG_VLM_CACHE_SIZE_MB=0 \ -python -m sglang.launch_server \ - --model-path Qwen/Qwen3-VL-235B-A22B-Instruct \ - --host 0.0.0.0 \ - --port 30000 \ - --trust-remote-code \ - --tp-size 8 \ - --enable-cache-report \ - --log-level info \ - --max-running-requests 64 \ - --mem-fraction-static 0.65 \ - --chunked-prefill-size 8192 \ - --attention-backend fa3 \ - --mm-attention-backend fa3 \ - --enable-metrics -``` diff --git a/docs/basic_usage/sampling_params.md b/docs/basic_usage/sampling_params.md deleted file mode 100644 index 9508c0492..000000000 --- a/docs/basic_usage/sampling_params.md +++ /dev/null @@ -1,347 +0,0 @@ -# Sampling Parameters - -This doc describes the sampling parameters of the SGLang Runtime. It is the low-level endpoint of the runtime. -If you want a high-level endpoint that can automatically handle chat templates, consider using the [OpenAI Compatible API](openai_api_completions.ipynb). - -## `/generate` Endpoint - -The `/generate` endpoint accepts the following parameters in JSON format. For detailed usage, see the [native API doc](native_api.ipynb). The object is defined at `io_struct.py::GenerateReqInput`. You can also read the source code to find more arguments and docs. - -| Argument | Type/Default | Description | -|----------------------------|------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------| -| text | `Optional[Union[List[str], str]] = None` | The input prompt. Can be a single prompt or a batch of prompts. | -| input_ids | `Optional[Union[List[List[int]], List[int]]] = None` | The token IDs for text; one can specify either text or input_ids. | -| input_embeds | `Optional[Union[List[List[List[float]]], List[List[float]]]] = None` | The embeddings for input_ids; one can specify either text, input_ids, or input_embeds. | -| image_data | `Optional[Union[List[List[ImageDataItem]], List[ImageDataItem], ImageDataItem]] = None` | The image input. Supports three formats: (1) **Raw images**: PIL Image, file path, URL, or base64 string; (2) **Processor output**: Dict with `format: "processor_output"` containing HuggingFace processor outputs; (3) **Precomputed embeddings**: Dict with `format: "precomputed_embedding"` and `feature` containing pre-calculated visual embeddings. Can be a single image, list of images, or list of lists of images. See [Multimodal Input Formats](#multimodal-input-formats) for details. | -| audio_data | `Optional[Union[List[AudioDataItem], AudioDataItem]] = None` | The audio input. Can be a file name, URL, or base64 encoded string. | -| sampling_params | `Optional[Union[List[Dict], Dict]] = None` | The sampling parameters as described in the sections below. | -| rid | `Optional[Union[List[str], str]] = None` | The request ID. | -| return_logprob | `Optional[Union[List[bool], bool]] = None` | Whether to return log probabilities for tokens. | -| logprob_start_len | `Optional[Union[List[int], int]] = None` | If return_logprob, the start location in the prompt for returning logprobs. Default is "-1", which returns logprobs for output tokens only. | -| top_logprobs_num | `Optional[Union[List[int], int]] = None` | If return_logprob, the number of top logprobs to return at each position. | -| token_ids_logprob | `Optional[Union[List[List[int]], List[int]]] = None` | If return_logprob, the token IDs to return logprob for. | -| return_text_in_logprobs | `bool = False` | Whether to detokenize tokens in text in the returned logprobs. | -| stream | `bool = False` | Whether to stream output. | -| lora_path | `Optional[Union[List[Optional[str]], Optional[str]]] = None` | The path to the LoRA. | -| custom_logit_processor | `Optional[Union[List[Optional[str]], str]] = None` | Custom logit processor for advanced sampling control. Must be a serialized instance of `CustomLogitProcessor` using its `to_str()` method. For usage see below. | -| return_hidden_states | `Union[List[bool], bool] = False` | Whether to return hidden states. | -| return_routed_experts | `bool = False` | Whether to return routed experts for MoE models. Requires `--enable-return-routed-experts` server flag. Returns base64-encoded int32 expert IDs as a flattened array with logical shape `[num_tokens, num_layers, top_k]`. | - -## Sampling parameters - -The object is defined at `sampling_params.py::SamplingParams`. You can also read the source code to find more arguments and docs. - -### Note on defaults - -By default, SGLang initializes several sampling parameters from the model's `generation_config.json` (when the server is launched with `--sampling-defaults model`, which is the default). To use SGLang/OpenAI constant defaults instead, start the server with `--sampling-defaults openai`. You can always override any parameter per request via `sampling_params`. - -```bash -# Use model-provided defaults from generation_config.json (default behavior) -python -m sglang.launch_server --model-path --sampling-defaults model - -# Use SGLang/OpenAI constant defaults instead -python -m sglang.launch_server --model-path --sampling-defaults openai -``` - -### Core parameters - -| Argument | Type/Default | Description | -|-----------------|----------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------| -| max_new_tokens | `int = 128` | The maximum output length measured in tokens. | -| stop | `Optional[Union[str, List[str]]] = None` | One or multiple [stop words](https://platform.openai.com/docs/api-reference/chat/create#chat-create-stop). Generation will stop if one of these words is sampled. | -| stop_token_ids | `Optional[List[int]] = None` | Provide stop words in the form of token IDs. Generation will stop if one of these token IDs is sampled. | -| stop_regex | `Optional[Union[str, List[str]]] = None` | Stop when hitting any of the regex patterns in this list | -| temperature | `float (model default; fallback 1.0)` | [Temperature](https://platform.openai.com/docs/api-reference/chat/create#chat-create-temperature) when sampling the next token. `temperature = 0` corresponds to greedy sampling, a higher temperature leads to more diversity. | -| top_p | `float (model default; fallback 1.0)` | [Top-p](https://platform.openai.com/docs/api-reference/chat/create#chat-create-top_p) selects tokens from the smallest sorted set whose cumulative probability exceeds `top_p`. When `top_p = 1`, this reduces to unrestricted sampling from all tokens. | -| top_k | `int (model default; fallback -1)` | [Top-k](https://developer.nvidia.com/blog/how-to-get-better-outputs-from-your-large-language-model/#predictability_vs_creativity) randomly selects from the `k` highest-probability tokens. | -| min_p | `float (model default; fallback 0.0)` | [Min-p](https://github.com/huggingface/transformers/issues/27670) samples from tokens with probability larger than `min_p * highest_token_probability`. | - -### Penalizers - -| Argument | Type/Default | Description | -|--------------------|------------------------|------------------------------------------------------------------------------------------------------------------------------------------------| -| frequency_penalty | `float = 0.0` | Penalizes tokens based on their frequency in generation so far. Must be between `-2` and `2` where negative numbers encourage repeatment of tokens and positive number encourages sampling of new tokens. The scaling of penalization grows linearly with each appearance of a token. | -| presence_penalty | `float = 0.0` | Penalizes tokens if they appeared in the generation so far. Must be between `-2` and `2` where negative numbers encourage repeatment of tokens and positive number encourages sampling of new tokens. The scaling of the penalization is constant if a token occurred. | -| repetition_penalty | `float = 1.0` | Scales the logits of previously generated tokens to discourage (values > 1) or encourage (values < 1) repetition. Valid range is `(0, 2]`; `1.0` leaves probabilities unchanged. | -| min_new_tokens | `int = 0` | Forces the model to generate at least `min_new_tokens` until a stop word or EOS token is sampled. Note that this might lead to unintended behavior, for example, if the distribution is highly skewed towards these tokens. | - -### Constrained decoding - -Please refer to our dedicated guide on [constrained decoding](../advanced_features/structured_outputs.ipynb) for the following parameters. - -| Argument | Type/Default | Description | -|-----------------|---------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------| -| json_schema | `Optional[str] = None` | JSON schema for structured outputs. | -| regex | `Optional[str] = None` | Regex for structured outputs. | -| ebnf | `Optional[str] = None` | EBNF for structured outputs. | -| structural_tag | `Optional[str] = None` | The structural tag for structured outputs. | - -### Other options - -| Argument | Type/Default | Description | -|-------------------------------|---------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------| -| n | `int = 1` | Specifies the number of output sequences to generate per request. (Generating multiple outputs in one request (n > 1) is discouraged; repeating the same prompts several times offers better control and efficiency.) | -| ignore_eos | `bool = False` | Don't stop generation when EOS token is sampled. | -| skip_special_tokens | `bool = True` | Remove special tokens during decoding. | -| spaces_between_special_tokens | `bool = True` | Whether or not to add spaces between special tokens during detokenization. | -| no_stop_trim | `bool = False` | Don't trim stop words or EOS token from the generated text. | -| custom_params | `Optional[List[Optional[Dict[str, Any]]]] = None` | Used when employing `CustomLogitProcessor`. For usage, see below. | - -## Examples - -### Normal - -Launch a server: - -```bash -python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3-8B-Instruct --port 30000 -``` - -Send a request: - -```python -import requests - -response = requests.post( - "http://localhost:30000/generate", - json={ - "text": "The capital of France is", - "sampling_params": { - "temperature": 0, - "max_new_tokens": 32, - }, - }, -) -print(response.json()) -``` - -Detailed example in [send request](./send_request.ipynb). - -### Streaming - -Send a request and stream the output: - -```python -import requests, json - -response = requests.post( - "http://localhost:30000/generate", - json={ - "text": "The capital of France is", - "sampling_params": { - "temperature": 0, - "max_new_tokens": 32, - }, - "stream": True, - }, - stream=True, -) - -prev = 0 -for chunk in response.iter_lines(decode_unicode=False): - chunk = chunk.decode("utf-8") - if chunk and chunk.startswith("data:"): - if chunk == "data: [DONE]": - break - data = json.loads(chunk[5:].strip("\n")) - output = data["text"].strip() - print(output[prev:], end="", flush=True) - prev = len(output) -print("") -``` - -Detailed example in [openai compatible api](openai_api_completions.ipynb). - -### Multimodal - -Launch a server: - -```bash -python3 -m sglang.launch_server --model-path lmms-lab/llava-onevision-qwen2-7b-ov -``` - -Download an image: - -```bash -curl -o example_image.png -L https://github.com/sgl-project/sglang/blob/main/examples/assets/example_image.png?raw=true -``` - -Send a request: - -```python -import requests - -response = requests.post( - "http://localhost:30000/generate", - json={ - "text": "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n" - "<|im_start|>user\n\nDescribe this image in a very short sentence.<|im_end|>\n" - "<|im_start|>assistant\n", - "image_data": "example_image.png", - "sampling_params": { - "temperature": 0, - "max_new_tokens": 32, - }, - }, -) -print(response.json()) -``` - -The `image_data` can be a file name, a URL, or a base64 encoded string. See also `python/sglang/srt/utils.py:load_image`. - -Streaming is supported in a similar manner as [above](#streaming). - -Detailed example in [OpenAI API Vision](openai_api_vision.ipynb). - -### Structured Outputs (JSON, Regex, EBNF) - -You can specify a JSON schema, regular expression or [EBNF](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form) to constrain the model output. The model output will be guaranteed to follow the given constraints. Only one constraint parameter (`json_schema`, `regex`, or `ebnf`) can be specified for a request. - -SGLang supports two grammar backends: - -- [XGrammar](https://github.com/mlc-ai/xgrammar) (default): Supports JSON schema, regular expression, and EBNF constraints. - - XGrammar currently uses the [GGML BNF format](https://github.com/ggerganov/llama.cpp/blob/master/grammars/README.md). -- [Outlines](https://github.com/dottxt-ai/outlines): Supports JSON schema and regular expression constraints. - -If instead you want to initialize the Outlines backend, you can use `--grammar-backend outlines` flag: - -```bash -python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \ ---port 30000 --host 0.0.0.0 --grammar-backend [xgrammar|outlines] # xgrammar or outlines (default: xgrammar) -``` - -```python -import json -import requests - -json_schema = json.dumps({ - "type": "object", - "properties": { - "name": {"type": "string", "pattern": "^[\\w]+$"}, - "population": {"type": "integer"}, - }, - "required": ["name", "population"], -}) - -# JSON (works with both Outlines and XGrammar) -response = requests.post( - "http://localhost:30000/generate", - json={ - "text": "Here is the information of the capital of France in the JSON format.\n", - "sampling_params": { - "temperature": 0, - "max_new_tokens": 64, - "json_schema": json_schema, - }, - }, -) -print(response.json()) - -# Regular expression (Outlines backend only) -response = requests.post( - "http://localhost:30000/generate", - json={ - "text": "Paris is the capital of", - "sampling_params": { - "temperature": 0, - "max_new_tokens": 64, - "regex": "(France|England)", - }, - }, -) -print(response.json()) - -# EBNF (XGrammar backend only) -response = requests.post( - "http://localhost:30000/generate", - json={ - "text": "Write a greeting.", - "sampling_params": { - "temperature": 0, - "max_new_tokens": 64, - "ebnf": 'root ::= "Hello" | "Hi" | "Hey"', - }, - }, -) -print(response.json()) -``` - -Detailed example in [structured outputs](../advanced_features/structured_outputs.ipynb). - -### Custom logit processor - -Launch a server with `--enable-custom-logit-processor` flag on. - -```bash -python -m sglang.launch_server \ - --model-path meta-llama/Meta-Llama-3-8B-Instruct \ - --port 30000 \ - --enable-custom-logit-processor -``` - -Define a custom logit processor that will always sample a specific token id. - -```python -from sglang.srt.sampling.custom_logit_processor import CustomLogitProcessor - -class DeterministicLogitProcessor(CustomLogitProcessor): - """A dummy logit processor that changes the logits to always - sample the given token id. - """ - - def __call__(self, logits, custom_param_list): - # Check that the number of logits matches the number of custom parameters - assert logits.shape[0] == len(custom_param_list) - key = "token_id" - - for i, param_dict in enumerate(custom_param_list): - # Mask all other tokens - logits[i, :] = -float("inf") - # Assign highest probability to the specified token - logits[i, param_dict[key]] = 0.0 - return logits -``` - -Send a request: - -```python -import requests - -response = requests.post( - "http://localhost:30000/generate", - json={ - "text": "The capital of France is", - "custom_logit_processor": DeterministicLogitProcessor().to_str(), - "sampling_params": { - "temperature": 0.0, - "max_new_tokens": 32, - "custom_params": {"token_id": 5}, - }, - }, -) -print(response.json()) -``` - -Send an OpenAI chat completion request: - -```python -import openai -from sglang.utils import print_highlight - -client = openai.Client(base_url="http://127.0.0.1:30000/v1", api_key="None") - -response = client.chat.completions.create( - model="meta-llama/Meta-Llama-3-8B-Instruct", - messages=[ - {"role": "user", "content": "List 3 countries and their capitals."}, - ], - temperature=0.0, - max_tokens=32, - extra_body={ - "custom_logit_processor": DeterministicLogitProcessor().to_str(), - "custom_params": {"token_id": 5}, - }, -) - -print_highlight(f"Response: {response}") -``` diff --git a/docs/basic_usage/send_request.ipynb b/docs/basic_usage/send_request.ipynb deleted file mode 100644 index 968a23b8d..000000000 --- a/docs/basic_usage/send_request.ipynb +++ /dev/null @@ -1,251 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Sending Requests\n", - "This notebook provides a quick-start guide to use SGLang in chat completions after installation. Once your server is running, API documentation is available at `http://localhost:30000/docs` (Swagger UI), `http://localhost:30000/redoc` (ReDoc), or `http://localhost:30000/openapi.json` (OpenAPI spec, useful for AI agents). Replace `30000` with your port if using a different one.\n", - "\n", - "- For Vision Language Models, see [OpenAI APIs - Vision](openai_api_vision.ipynb).\n", - "- For Embedding Models, see [OpenAI APIs - Embedding](openai_api_embeddings.ipynb) and [Encode (embedding model)](native_api.html#Encode-(embedding-model)).\n", - "- For Reward Models, see [Classify (reward model)](native_api.html#Classify-(reward-model))." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Launch A Server" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from sglang.test.doc_patch import launch_server_cmd\n", - "from sglang.utils import wait_for_server, print_highlight, terminate_process\n", - "\n", - "# This is equivalent to running the following command in your terminal\n", - "# python3 -m sglang.launch_server --model-path qwen/qwen2.5-0.5b-instruct --host 0.0.0.0\n", - "\n", - "server_process, port = launch_server_cmd(\"\"\"\n", - "python3 -m sglang.launch_server --model-path qwen/qwen2.5-0.5b-instruct \\\n", - " --host 0.0.0.0 --log-level warning\n", - "\"\"\")\n", - "\n", - "wait_for_server(f\"http://localhost:{port}\", process=server_process)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Using cURL\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import subprocess, json\n", - "\n", - "curl_command = f\"\"\"\n", - "curl -s http://localhost:{port}/v1/chat/completions \\\n", - " -H \"Content-Type: application/json\" \\\n", - " -d '{{\"model\": \"qwen/qwen2.5-0.5b-instruct\", \"messages\": [{{\"role\": \"user\", \"content\": \"What is the capital of France?\"}}]}}'\n", - "\"\"\"\n", - "\n", - "response = json.loads(subprocess.check_output(curl_command, shell=True))\n", - "print_highlight(response)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Using Python Requests" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import requests\n", - "\n", - "url = f\"http://localhost:{port}/v1/chat/completions\"\n", - "\n", - "data = {\n", - " \"model\": \"qwen/qwen2.5-0.5b-instruct\",\n", - " \"messages\": [{\"role\": \"user\", \"content\": \"What is the capital of France?\"}],\n", - "}\n", - "\n", - "response = requests.post(url, json=data)\n", - "print_highlight(response.json())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Using OpenAI Python Client" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import openai\n", - "\n", - "client = openai.Client(base_url=f\"http://127.0.0.1:{port}/v1\", api_key=\"None\")\n", - "\n", - "response = client.chat.completions.create(\n", - " model=\"qwen/qwen2.5-0.5b-instruct\",\n", - " messages=[\n", - " {\"role\": \"user\", \"content\": \"List 3 countries and their capitals.\"},\n", - " ],\n", - " temperature=0,\n", - " max_tokens=64,\n", - ")\n", - "print_highlight(response)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Streaming" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import openai\n", - "\n", - "client = openai.Client(base_url=f\"http://127.0.0.1:{port}/v1\", api_key=\"None\")\n", - "\n", - "# Use stream=True for streaming responses\n", - "response = client.chat.completions.create(\n", - " model=\"qwen/qwen2.5-0.5b-instruct\",\n", - " messages=[\n", - " {\"role\": \"user\", \"content\": \"List 3 countries and their capitals.\"},\n", - " ],\n", - " temperature=0,\n", - " max_tokens=64,\n", - " stream=True,\n", - ")\n", - "\n", - "# Handle the streaming output\n", - "for chunk in response:\n", - " if chunk.choices[0].delta.content:\n", - " print(chunk.choices[0].delta.content, end=\"\", flush=True)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Using Native Generation APIs\n", - "\n", - "You can also use the native `/generate` endpoint with requests, which provides more flexibility. An API reference is available at [Sampling Parameters](sampling_params.md)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import requests\n", - "\n", - "response = requests.post(\n", - " f\"http://localhost:{port}/generate\",\n", - " json={\n", - " \"text\": \"The capital of France is\",\n", - " \"sampling_params\": {\n", - " \"temperature\": 0,\n", - " \"max_new_tokens\": 32,\n", - " },\n", - " },\n", - ")\n", - "\n", - "print_highlight(response.json())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Streaming" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import requests, json\n", - "\n", - "response = requests.post(\n", - " f\"http://localhost:{port}/generate\",\n", - " json={\n", - " \"text\": \"The capital of France is\",\n", - " \"sampling_params\": {\n", - " \"temperature\": 0,\n", - " \"max_new_tokens\": 32,\n", - " },\n", - " \"stream\": True,\n", - " },\n", - " stream=True,\n", - ")\n", - "\n", - "prev = 0\n", - "for chunk in response.iter_lines(decode_unicode=False):\n", - " chunk = chunk.decode(\"utf-8\")\n", - " if chunk and chunk.startswith(\"data:\"):\n", - " if chunk == \"data: [DONE]\":\n", - " break\n", - " data = json.loads(chunk[5:].strip(\"\\n\"))\n", - " output = data[\"text\"]\n", - " print(output[prev:], end=\"\", flush=True)\n", - " prev = len(output)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "terminate_process(server_process)" - ] - } - ], - "metadata": { - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/docs/conf.py b/docs/conf.py deleted file mode 100644 index 6140b47f8..000000000 --- a/docs/conf.py +++ /dev/null @@ -1,206 +0,0 @@ -import os -import sys -from datetime import datetime - -sys.path.insert(0, os.path.abspath("../..")) - -version_file = "../python/sglang/version.py" -with open(version_file, "r") as f: - exec(compile(f.read(), version_file, "exec")) -__version__ = locals()["__version__"] - -project = "SGLang" -copyright = f"2023-{datetime.now().year}, SGLang" -author = "SGLang Team" - -version = __version__ -release = __version__ - -extensions = [ - "sphinx.ext.autodoc", - "sphinx.ext.autosummary", - "sphinx.ext.napoleon", - "sphinx.ext.viewcode", - "sphinx.ext.autosectionlabel", - "sphinx.ext.intersphinx", - "sphinx_tabs.tabs", - "myst_parser", - "sphinx_copybutton", - "sphinxcontrib.mermaid", - "nbsphinx", - "sphinx.ext.mathjax", -] - -nbsphinx_allow_errors = True -nbsphinx_execute = "never" - -autosectionlabel_prefix_document = True -nbsphinx_allow_directives = True - - -myst_enable_extensions = [ - "dollarmath", - "amsmath", - "deflist", - "colon_fence", - "html_image", - "linkify", - "substitution", -] - -myst_heading_anchors = 3 - -nbsphinx_kernel_name = "python3" -nbsphinx_execute_arguments = [ - "--InlineBackend.figure_formats={'svg', 'pdf'}", - "--InlineBackend.rc={'figure.dpi': 96}", -] - - -nb_render_priority = { - "html": ( - "application/vnd.jupyter.widget-view+json", - "application/javascript", - "text/html", - "image/svg+xml", - "image/png", - "image/jpeg", - "text/markdown", - "text/latex", - "text/plain", - ) -} - -myst_enable_extensions = [ - "dollarmath", - "amsmath", - "deflist", - "colon_fence", - "html_image", - "linkify", - "substitution", -] - -myst_heading_anchors = 3 -myst_ref_domains = ["std", "py"] - -templates_path = ["_templates"] - -source_suffix = { - ".rst": "restructuredtext", - ".md": "markdown", -} - -master_doc = "index" - -language = "en" - -exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] - -pygments_style = "sphinx" - -html_theme = "sphinx_book_theme" -html_logo = "_static/image/logo.png" -html_favicon = "_static/image/logo.ico" -html_title = project -html_copy_source = True -html_last_updated_fmt = "" - -html_theme_options = { - "repository_url": "https://github.com/sgl-project/sgl-project.github.io", - "repository_branch": "main", - "show_navbar_depth": 3, - "max_navbar_depth": 4, - "collapse_navbar": True, - "use_edit_page_button": True, - "use_source_button": True, - "use_issues_button": True, - "use_repository_button": True, - "use_download_button": True, - "use_sidenotes": True, - "show_toc_level": 2, -} - -html_context = { - "display_github": True, - "github_user": "sgl-project", - "github_repo": "sgl-project.github.io", - "github_version": "main", - "conf_py_path": "/docs/", -} - -html_static_path = ["_static"] -html_css_files = ["css/custom_log.css"] -html_js_files = ["js/deprecation_banner.js"] - - -def setup(app): - app.add_css_file("css/custom_log.css") - - -myst_enable_extensions = [ - "dollarmath", - "amsmath", - "deflist", - "colon_fence", -] -myst_heading_anchors = 5 - -htmlhelp_basename = "sglangdoc" - -latex_elements = {} - -latex_documents = [ - (master_doc, "sglang.tex", "sglang Documentation", "SGLang Team", "manual"), -] - -man_pages = [(master_doc, "sglang", "sglang Documentation", [author], 1)] - -texinfo_documents = [ - ( - master_doc, - "sglang", - "sglang Documentation", - author, - "sglang", - "One line description of project.", - "Miscellaneous", - ), -] - -epub_title = project - -epub_exclude_files = ["search.html"] - -copybutton_prompt_text = r">>> |\.\.\. " -copybutton_prompt_is_regexp = True - -autodoc_preserve_defaults = True -navigation_with_keys = False - -autodoc_mock_imports = [ - "torch", - "transformers", - "triton", -] - -intersphinx_mapping = { - "python": ("https://docs.python.org/3.12", None), - "typing_extensions": ("https://typing-extensions.readthedocs.io/en/latest", None), - "pillow": ("https://pillow.readthedocs.io/en/stable", None), - "numpy": ("https://numpy.org/doc/stable", None), - "torch": ("https://pytorch.org/docs/stable", None), -} - -html_theme = "sphinx_book_theme" - - -nbsphinx_prolog = """ -.. raw:: html - - -""" diff --git a/docs/deploy.py b/docs/deploy.py deleted file mode 100644 index 75b7ea7f2..000000000 --- a/docs/deploy.py +++ /dev/null @@ -1,22 +0,0 @@ -# Deploy the documents - -import os -from datetime import datetime - - -def run_cmd(cmd): - print(cmd) - os.system(cmd) - - -run_cmd("cd $DOC_SITE_PATH; git pull") - -# (Optional) Remove old files -# run_cmd("rm -rf $ALPA_SITE_PATH/*") - -run_cmd("cp -r _build/html/* $DOC_SITE_PATH") - -cmd_message = f"Update {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}" -run_cmd( - f"cd $DOC_SITE_PATH; git add .; git commit -m '{cmd_message}'; git push origin main" -) diff --git a/docs/developer_guide/bench_serving.md b/docs/developer_guide/bench_serving.md deleted file mode 100644 index bc13765d0..000000000 --- a/docs/developer_guide/bench_serving.md +++ /dev/null @@ -1,390 +0,0 @@ -# Bench Serving Guide - -This guide explains how to benchmark online serving throughput and latency using `python -m sglang.bench_serving`. It supports multiple inference backends via OpenAI-compatible and native endpoints, and produces both console metrics and optional JSONL outputs. - -### What it does - -- Generates synthetic or dataset-driven prompts and submits them to a target serving endpoint -- Measures throughput, time-to-first-token (TTFT), inter-token latency (ITL), per-request end-to-end latency, and more -- Supports streaming or non-streaming modes, rate control, and concurrency limits - -### Supported backends and endpoints - -- `sglang` / `sglang-native`: `POST /generate` -- `sglang-oai`, `vllm`, `lmdeploy`: `POST /v1/completions` -- `sglang-oai-chat`, `vllm-chat`, `lmdeploy-chat`: `POST /v1/chat/completions` -- `trt` (TensorRT-LLM): `POST /v2/models/ensemble/generate_stream` -- `gserver`: Custom server (Not Implemented yet in this script) -- `truss`: `POST /v1/models/model:predict` - -If `--base-url` is provided, requests are sent to it. Otherwise, `--host` and `--port` are used. When `--model` is not provided, the script will attempt to query `GET /v1/models` for an available model ID (OpenAI-compatible endpoints). - -### Prerequisites - -- Python 3.10+ -- Dependencies typically used by this script: `aiohttp`, `numpy`, `requests`, `tqdm`, `transformers`, and for some datasets `datasets`, `pillow`, `pybase64`. Install as needed. -- An inference server running and reachable via the endpoints above -- If your server requires authentication, set environment variable `OPENAI_API_KEY` (used as `Authorization: Bearer `) - -### Quick start - -Run a basic benchmark against an sglang server exposing `/generate`: - -```bash -python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct -``` - -```bash -python3 -m sglang.bench_serving \ - --backend sglang \ - --host 127.0.0.1 --port 30000 \ - --num-prompts 1000 \ - --model meta-llama/Llama-3.1-8B-Instruct -``` - -Or, using an OpenAI-compatible endpoint (completions): - -```bash -python3 -m sglang.bench_serving \ - --backend vllm \ - --base-url http://127.0.0.1:8000 \ - --num-prompts 1000 \ - --model meta-llama/Llama-3.1-8B-Instruct -``` - -### Datasets - -Select with `--dataset-name`: - -- `sharegpt` (default): loads ShareGPT-style pairs; optionally restrict with `--sharegpt-context-len` and override outputs with `--sharegpt-output-len` -- `random`: random text lengths; sampled from ShareGPT token space -- `random-ids`: random token ids (can lead to gibberish) -- `image`: generates images and wraps them in chat messages; supports custom resolutions, multiple formats, and different content types -- `generated-shared-prefix`: synthetic dataset with shared long system prompts and short questions -- `mmmu`: samples from MMMU (Math split) and includes images - -Common dataset flags: - -- `--num-prompts N`: number of requests -- `--random-input-len`, `--random-output-len`, `--random-range-ratio`: for random/random-ids/image -- `--image-count`: Number of images per request (for `image` dataset). - -- `--apply-chat-template`: apply tokenizer chat template when constructing prompts -- `--dataset-path PATH`: file path for ShareGPT json; if blank and missing, it will be downloaded and cached - -Generated Shared Prefix flags (for `generated-shared-prefix`): - -- `--gsp-num-groups` -- `--gsp-prompts-per-group` -- `--gsp-system-prompt-len` -- `--gsp-question-len` -- `--gsp-output-len` - -Image dataset flags (for `image`): - -- `--image-count`: Number of images per request -- `--image-resolution`: Image resolution; supports presets (4k, 1080p, 720p, 360p) or custom 'heightxwidth' format (e.g., 1080x1920, 512x768) -- `--image-format`: Image format (jpeg or png) -- `--image-content`: Image content type (random or blank) - -### Examples - -1. To benchmark image dataset with 3 images per request, 500 prompts, 512 input length, and 512 output length, you can run: - -```bash -python -m sglang.launch_server --model-path Qwen/Qwen2.5-VL-3B-Instruct --disable-radix-cache -``` - -```bash -python -m sglang.bench_serving \ - --backend sglang-oai-chat \ - --dataset-name image \ - --num-prompts 500 \ - --image-count 3 \ - --image-resolution 720p \ - --random-input-len 512 \ - --random-output-len 512 -``` - -2. To benchmark random dataset with 3000 prompts, 1024 input length, and 1024 output length, you can run: - -```bash -python -m sglang.launch_server --model-path Qwen/Qwen2.5-3B-Instruct -``` - -```bash -python3 -m sglang.bench_serving \ - --backend sglang \ - --dataset-name random \ - --num-prompts 3000 \ - --random-input 1024 \ - --random-output 1024 \ - --random-range-ratio 0.5 -``` - -### Choosing model and tokenizer - -- `--model` is required unless the backend exposes `GET /v1/models`, in which case the first model ID is auto-selected. -- `--tokenizer` defaults to `--model`. Both can be HF model IDs or local paths. -- For ModelScope workflows, setting `SGLANG_USE_MODELSCOPE=true` enables fetching via ModelScope (weights are skipped for speed). -- If your tokenizer lacks a chat template, the script warns because token counting can be less robust for gibberish outputs. - -### Rate, concurrency, and streaming - -- `--request-rate`: requests per second. `inf` sends all immediately (burst). Non-infinite rate uses a Poisson process for arrival times. -- `--max-concurrency`: caps concurrent in-flight requests regardless of arrival rate. -- `--disable-stream`: switch to non-streaming mode when supported; TTFT then equals total latency for chat completions. - -### Other key options - -- `--output-file FILE.jsonl`: append JSONL results to file; auto-named if unspecified -- `--output-details`: include per-request arrays (generated texts, errors, ttfts, itls, input/output lens) -- `--extra-request-body '{"top_p":0.9,"temperature":0.6}'`: merged into payload (sampling params, etc.) -- `--disable-ignore-eos`: pass through EOS behavior (varies by backend) -- `--warmup-requests N`: run warmup requests with short output first (default 1) -- `--flush-cache`: call `/flush_cache` (sglang) before main run -- `--profile`: call `/start_profile` and `/stop_profile` (requires server to enable profiling, e.g., `SGLANG_TORCH_PROFILER_DIR`) -- `--lora-name name1 name2 ...`: randomly pick one per request and pass to backend (e.g., `lora_path` for sglang) -- `--tokenize-prompt`: send integer IDs instead of text (currently supports `--backend sglang` only) - -### Authentication - -If your target endpoint requires OpenAI-style auth, set: - -```bash -export OPENAI_API_KEY=sk-...yourkey... -``` - -The script will add `Authorization: Bearer $OPENAI_API_KEY` automatically for OpenAI-compatible routes. - -### Metrics explained - -Printed after each run: - -- Request throughput (req/s) -- Input token throughput (tok/s) - includes both text and vision tokens -- Output token throughput (tok/s) -- Total token throughput (tok/s) - includes both text and vision tokens -- Total input text tokens and Total input vision tokens - per-modality breakdown -- Concurrency: aggregate time of all requests divided by wall time -- End-to-End Latency (ms): mean/median/std/p99 per-request total latency -- Time to First Token (TTFT, ms): mean/median/std/p99 for streaming mode -- Inter-Token Latency (ITL, ms): mean/median/std/p95/p99/max between tokens -- TPOT (ms): Token processing time after first token, i.e., `(latency - ttft)/(tokens-1)` -- Accept length (sglang-only, if available): speculative decoding accept length - -The script also retokenizes generated text with the configured tokenizer and reports "retokenized" counts. - -### JSONL output format - -When `--output-file` is set, one JSON object is appended per run. Base fields: - -- Arguments summary: backend, dataset, request_rate, max_concurrency, etc. -- Duration and totals: completed, total_input_tokens, total_output_tokens, retokenized totals -- Throughputs and latency statistics as printed in the console -- `accept_length` when available (sglang) - -With `--output-details`, an extended object also includes arrays: - -- `input_lens`, `output_lens` -- `ttfts`, `itls` (per request: ITL arrays) -- `generated_texts`, `errors` - -### End-to-end examples - -1) sglang native `/generate` (streaming): - -```bash -python3 -m sglang.bench_serving \ - --backend sglang \ - --host 127.0.0.1 --port 30000 \ - --model meta-llama/Llama-3.1-8B-Instruct \ - --dataset-name random \ - --random-input-len 1024 --random-output-len 1024 --random-range-ratio 0.5 \ - --num-prompts 2000 \ - --request-rate 100 \ - --max-concurrency 512 \ - --output-file sglang_random.jsonl --output-details -``` - -2) OpenAI-compatible Completions (e.g., vLLM): - -```bash -python3 -m sglang.bench_serving \ - --backend vllm \ - --base-url http://127.0.0.1:8000 \ - --model meta-llama/Llama-3.1-8B-Instruct \ - --dataset-name sharegpt \ - --num-prompts 1000 \ - --sharegpt-output-len 256 -``` - -3) OpenAI-compatible Chat Completions (streaming): - -```bash -python3 -m sglang.bench_serving \ - --backend vllm-chat \ - --base-url http://127.0.0.1:8000 \ - --model meta-llama/Llama-3.1-8B-Instruct \ - --dataset-name random \ - --num-prompts 500 \ - --apply-chat-template -``` - -4) Images (VLM) with chat template: - -```bash -python3 -m sglang.bench_serving \ - --backend sglang \ - --host 127.0.0.1 --port 30000 \ - --model your-vlm-model \ - --dataset-name image \ - --image-count 2 \ - --image-resolution 720p \ - --random-input-len 128 --random-output-len 256 \ - --num-prompts 200 \ - --apply-chat-template -``` - -4a) Images with custom resolution: - -```bash -python3 -m sglang.bench_serving \ - --backend sglang \ - --host 127.0.0.1 --port 30000 \ - --model your-vlm-model \ - --dataset-name image \ - --image-count 1 \ - --image-resolution 512x768 \ - --random-input-len 64 --random-output-len 128 \ - --num-prompts 100 \ - --apply-chat-template -``` - -4b) 1080p images with PNG format and blank content: - -```bash -python3 -m sglang.bench_serving \ - --backend sglang \ - --host 127.0.0.1 --port 30000 \ - --model your-vlm-model \ - --dataset-name image \ - --image-count 1 \ - --image-resolution 1080p \ - --image-format png \ - --image-content blank \ - --random-input-len 64 --random-output-len 128 \ - --num-prompts 100 \ - --apply-chat-template -``` - -5) Generated shared prefix (long system prompts + short questions): - -```bash -python3 -m sglang.bench_serving \ - --backend sglang \ - --host 127.0.0.1 --port 30000 \ - --model meta-llama/Llama-3.1-8B-Instruct \ - --dataset-name generated-shared-prefix \ - --gsp-num-groups 64 --gsp-prompts-per-group 16 \ - --gsp-system-prompt-len 2048 --gsp-question-len 128 --gsp-output-len 256 \ - --num-prompts 1024 -``` - -6) Tokenized prompts (ids) for strict length control (sglang only): - -```bash -python3 -m sglang.bench_serving \ - --backend sglang \ - --host 127.0.0.1 --port 30000 \ - --model meta-llama/Llama-3.1-8B-Instruct \ - --dataset-name random \ - --tokenize-prompt \ - --random-input-len 2048 --random-output-len 256 --random-range-ratio 0.2 -``` - -7) Profiling and cache flush (sglang): - -```bash -python3 -m sglang.bench_serving \ - --backend sglang \ - --host 127.0.0.1 --port 30000 \ - --model meta-llama/Llama-3.1-8B-Instruct \ - --profile \ - --flush-cache -``` - -8) TensorRT-LLM streaming endpoint: - -```bash -python3 -m sglang.bench_serving \ - --backend trt \ - --base-url http://127.0.0.1:8000 \ - --model your-trt-llm-model \ - --dataset-name random \ - --num-prompts 100 \ - --disable-ignore-eos -``` - -9) Evaluating large-scale KVCache sharing with mooncake trace (sglang only): - -```bash -python3 -m sglang.bench_serving \ - --backend sglang \ - --host 127.0.0.1 --port 30000 \ - --model model-name \ - --dataset-name mooncake \ - --mooncake-slowdown-factor 1.0 \ - --mooncake-num-rounds 1000 \ - --mooncake-workload conversation|mooncake|agent|synthetic - --use-trace-timestamps true \ - --random-output-len 256 -``` - -10) Fake decode stress testing (PD disaggregation, decode-only): - -When benchmarking pure decode performance in a PD disaggregation setup, you can bypass the prefill node entirely by using `--fake-prefill`. This requires the decode server to be started with `--disaggregation-transfer-backend fake`: - -```bash -# Step 1: Start a decode-only server with fake transfer backend -python -m sglang.launch_server \ - --model-path meta-llama/Llama-3.1-8B-Instruct \ - --disaggregation-mode decode \ - --disaggregation-transfer-backend fake \ - --port 30001 - -# Step 2: Run bench_serving with --fake-prefill -python3 -m sglang.bench_serving \ - --backend sglang \ - --host 127.0.0.1 --port 30001 \ - --model meta-llama/Llama-3.1-8B-Instruct \ - --dataset-name random \ - --num-prompts 500 \ - --random-input-len 1024 --random-output-len 256 \ - --fake-prefill -``` - -Similarly, `bench_one_batch_server` also supports `--fake-prefill`: - -```bash -python3 -m sglang.bench_one_batch_server \ - --base-url http://127.0.0.1:30001 \ - --model-path meta-llama/Llama-3.1-8B-Instruct \ - --batch-size 32 --input-len 1024 --output-len 256 \ - --fake-prefill -``` - -The `--fake-prefill` flag automatically injects special sentinel values into each request, telling the decode server to skip real KV transfer and generate fake KV data locally. - -### Troubleshooting - -- All requests failed: verify `--backend`, server URL/port, `--model`, and authentication. Check warmup errors printed by the script. -- Throughput seems too low: adjust `--request-rate` and `--max-concurrency`; verify server batch size/scheduling; ensure streaming is enabled if appropriate. -- Token counts look odd: prefer chat/instruct models with proper chat templates; otherwise tokenization of gibberish may be inconsistent. -- Image/MMMU datasets: ensure you installed extra deps (`pillow`, `datasets`, `pybase64`). -- Authentication errors (401/403): set `OPENAI_API_KEY` or disable auth on your server. - -### Notes - -- The script raises the file descriptor soft limit (`RLIMIT_NOFILE`) to help with many concurrent connections. -- For sglang, `/server_info` is queried post-run to report speculative decoding accept length when available. diff --git a/docs/developer_guide/benchmark_and_profiling.md b/docs/developer_guide/benchmark_and_profiling.md deleted file mode 100644 index 3a3539440..000000000 --- a/docs/developer_guide/benchmark_and_profiling.md +++ /dev/null @@ -1,473 +0,0 @@ -# Benchmark and Profiling - -## Benchmark - -SGLang provides four benchmark tools that operate at different levels of the stack. The table below summarizes their key differences: - -| Tool | HTTP Server | Scheduler | Use Case | -| -------------------------- | --------------------------------------------- | --------------------------------------- | -------------------------------------------------------------------------- | -| `bench_serving` | Yes (async HTTP client to a running server) | Yes (indirectly, via server) | Realistic online serving benchmarks with latency metrics (TTFT, TPOT, ITL) | -| `bench_one_batch_server` | Yes (sends HTTP requests to a running server) | Yes (indirectly, via server) | End-to-end single-batch latency including HTTP and scheduler overhead | -| `bench_offline_throughput` | No | Yes (directly uses `Engine` in-process) | Maximum throughput measurement without HTTP overhead | -| `bench_one_batch` | No | No (directly calls `ModelRunner`) | Kernel-level latency profiling of a single static batch | - -Use `bench_serving` by default unless there are specific needs. - -**`bench_serving`** is an async HTTP load-testing client that sends requests at controlled rates with configurable concurrency to a running server. It measures realistic online serving metrics including time-to-first-token (TTFT), time-per-output-token (TPOT), inter-token latency (ITL), and throughput. Use `num-prompts >= 5 * max-concurrency` to measure steady-state performance. Launch a server with `sglang.launch_server` first. - - ```bash - python3 -m sglang.bench_serving --backend sglang --max-concurrency 16 --num-prompts 80 --random-input-len 256 --random-output-len 32 --dataset-name random - ``` - -**`bench_one_batch_server`** sends a single batch as one HTTP request to a running server. Due to only having a single batch, the server is never in a steady-state and metrics will be biased. Launch a server with `sglang.launch_server` first. - - ```bash - python3 -m sglang.bench_one_batch_server --base-url http://127.0.0.1:30000 --model-path meta-llama/Meta-Llama-3.1-8B-Instruct --batch-size 32 --input-len 256 --output-len 32 - ``` - - - Pass `--enable-multi-batch` and set `--batch-size` to a multiple of the server's `--max-running-requests` to stabilize throughput measurements. Surplus requests are queued by the scheduler and promoted batch-by-batch, amortizing per-request prefill and first-step transients into steady-state decode. Under this flag, only `overall_throughput` is authoritative; `input_throughput`, `output_throughput`, `last_ttft`, and ITL include cross-batch queueing in their denominators and should be treated as informational. - - Pass `--lora-name ` to route every prompt through a pre-loaded LoRA adapter. Requires the server to be launched with `--enable-lora --lora-paths =`. - -**`bench_offline_throughput`** directly instantiates the `Engine` object in-process (no HTTP server) and submits all requests at once via `engine.generate()`. The engine's scheduler handles batching and execution. This measures maximum achievable throughput without any network overhead. - - ```bash - python3 -m sglang.bench_offline_throughput --model-path meta-llama/Meta-Llama-3.1-8B-Instruct --num-prompts 10 - ``` - -**`bench_one_batch`** is the lowest-level tool. It directly instantiates a `ModelRunner` and calls `extend()` / `decode()` on a fixed static batch, bypassing the scheduler entirely. The prefill and decode phases are run separately, making profiling easier but rendering the metrics unrealistic. Because there is no dynamic batching, it may run out of memory for batch sizes that a real server can handle (a real server chunks prefill into smaller batches). This is best suited for profiling individual kernel performance. - - ```bash - python3 -m sglang.bench_one_batch --model-path meta-llama/Meta-Llama-3.1-8B-Instruct --batch-size 32 --input-len 256 --output-len 32 - ``` - -## Profile with PyTorch Profiler - -[Pytorch Profiler](https://pytorch.org/tutorials/recipes/recipes/profiler_recipe.html) is a convenient basic tool to inspect kernel execution time, call stack, and kernel overlap and occupancy. - -### Profile a server with `sglang.bench_serving` - -```bash -# set trace path -export SGLANG_TORCH_PROFILER_DIR=/root/sglang/profile_log - -# start server -python -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct - -# send profiling request from client -python -m sglang.bench_serving --backend sglang --model meta-llama/Llama-3.1-8B-Instruct --num-prompts 10 --sharegpt-output-len 100 --profile -``` - -For `bench_serving --profile`, the output directory is selected on the client side from `--profile-output-dir` or `SGLANG_TORCH_PROFILER_DIR` (fallback: `/tmp`), then sent in the `/start_profile` request. -If you call `/start_profile` directly and do not provide `output_dir`, the server uses its own `SGLANG_TORCH_PROFILER_DIR` (fallback: `/tmp`). - -Setting `SGLANG_TORCH_PROFILER_DIR` on both server and client is still recommended to avoid confusion about where traces are written. - -For more details, please refer to [Bench Serving Guide](./bench_serving.md). - -### Profile In PD Disaggregation Mode - -When profiling in PD disaggregation mode, prefill and decode workers **must be profiled separately** due to torch profiler limitations. The `bench_serving` command provides dedicated options for this: - -#### Profile Prefill Workers - -```bash -# set trace path -export SGLANG_TORCH_PROFILER_DIR=/root/sglang/profile_log - -# start prefill and decode servers (see PD disaggregation docs for setup) -python -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --disaggregation-mode prefill -python -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --disaggregation-mode decode --port 30001 --base-gpu-id 1 - -# start router -python -m sglang_router.launch_router --pd-disaggregation --prefill http://127.0.0.1:30000 --decode http://127.0.0.1:30001 --host 0.0.0.0 --port 8000 - -# send profiling request targeting prefill workers -python -m sglang.bench_serving --backend sglang --model meta-llama/Llama-3.1-8B-Instruct --num-prompts 10 --sharegpt-output-len 100 --profile --pd-separated --profile-prefill-url http://127.0.0.1:30000 -``` - -#### Profile Decode Workers - -```bash -# send profiling request targeting decode workers -python -m sglang.bench_serving --backend sglang --model meta-llama/Llama-3.1-8B-Instruct --num-prompts 10 --sharegpt-output-len 100 --profile --pd-separated --profile-decode-url http://127.0.0.1:30001 -``` - -#### Important Notes - -- `--profile-prefill-url` and `--profile-decode-url` are **mutually exclusive** - you cannot profile both at the same time -- Both options support multiple worker URLs for multi-instance setups: - ```bash - # Profile multiple prefill workers - python -m sglang.bench_serving --backend sglang --model meta-llama/Llama-3.1-8B-Instruct --num-prompts 10 --profile --pd-separated --profile-prefill-url http://127.0.0.1:30000 http://127.0.0.1:30002 - - # Profile multiple decode workers - python -m sglang.bench_serving --backend sglang --model meta-llama/Llama-3.1-8B-Instruct --num-prompts 10 --profile --pd-separated --profile-decode-url http://127.0.0.1:30001 http://127.0.0.1:30003 - ``` -- Make sure `SGLANG_TORCH_PROFILER_DIR` is set on all worker nodes before starting the servers -- For more details on setting up PD disaggregation, see [PD Disaggregation Guide](../advanced_features/pd_disaggregation.md) - -### Profile a server with `sglang.bench_offline_throughput` -```bash -export SGLANG_TORCH_PROFILER_DIR=/root/sglang/profile_log - -# profile one batch with bench_one_batch.py -# batch size can be controlled with --batch argument -python3 -m sglang.bench_one_batch --model-path meta-llama/Llama-3.1-8B-Instruct --batch 32 --input-len 1024 --output-len 10 --profile - -# profile multiple batches with bench_offline_throughput.py -python -m sglang.bench_offline_throughput --model-path meta-llama/Llama-3.1-8B-Instruct --dataset-name random --num-prompts 10 --profile --mem-frac=0.8 -``` - -### Profile a server with `sglang.profiler` - -When the server is running (e.g., processing a decoding request), you can start live profiling immediately by sending a profile request to the server. - -You can do this by running `python3 -m sglang.profiler`. For example: - -``` -# Terminal 1: Send a generation request -python3 -m sglang.test.send_one - -# Terminal 2: Before the above request finishes, quickly launch the following command in a separate terminal. -# It will generate a profile of the above request for several decoding batches. -python3 -m sglang.profiler -``` - -You can also combine the above operations into a single command - -``` -python3 -m sglang.test.send_one --profile -``` - -### Profile a server with HTTP API endpoints - -SGLang provides HTTP API endpoints to control profiling on a running server. This allows you to start and stop profiling programmatically, which is useful for capturing specific workload patterns. - -#### Using `/start_profile` endpoint - -The `/start_profile` endpoint starts profiling on the server. You can control when profiling begins and how long it runs using the following parameters: - -**Basic usage:** - -```bash -# Start profiling immediately for 10 steps -curl -X POST http://127.0.0.1:30000/start_profile \ - -H "Content-Type: application/json" \ - -d '{ - "num_steps": 10 - }' -``` - -**Parameters:** - -- `output_dir` (optional): Directory where profile traces will be saved. If not specified, uses `SGLANG_TORCH_PROFILER_DIR` environment variable, or `/tmp` as the default -- `num_steps` (optional): Number of steps to profile. If not specified, profiling continues until manually stopped with `/stop_profile` -- `start_step` (optional): Step number at which to start profiling (inclusive). Useful for skipping warmup iterations -- `activities` (optional): List of activities to profile, e.g., `["CPU", "GPU"]`. Default is `["CPU", "GPU"]` -- `merge_profiles` (optional): Whether to merge distributed traces. Default is `false` - -**Note on step ranges:** Profiling starts at `start_step` (inclusive) and continues for `num_steps` iterations. For example, with `start_step=3` and `num_steps=10`, profiling captures steps 3, 4, 5, 6, 7, 8, 9, 10, 11, and 12 (10 steps total, starting from step 3). - -**Advanced usage with `start_step`:** - -```bash -# Wait 5 steps (warmup), then profile for 10 steps -curl -X POST http://127.0.0.1:30000/start_profile \ - -H "Content-Type: application/json" \ - -d '{ - "output_dir": "/tmp/profiles", - "start_step": 5, - "num_steps": 10, - "activities": ["CPU", "GPU"] - }' -``` - -**Continuous profiling (manual stop):** - -```bash -# Start profiling without num_steps - must manually stop with /stop_profile -curl -X POST http://127.0.0.1:30000/start_profile -``` - -#### Using `/stop_profile` endpoint - -The `/stop_profile` endpoint stops an ongoing profiling session and saves the trace file. - -```bash -# Stop profiling and save traces -curl -X POST http://127.0.0.1:30000/stop_profile -``` - -This is only needed when you start profiling without specifying `num_steps`. If `num_steps` is specified, profiling will automatically stop after that many steps. - -#### Example workflow - -```bash -# Terminal 1: Start the server -export SGLANG_TORCH_PROFILER_DIR=/tmp/profiles -python -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct - -# Terminal 2: Start continuous profiling -curl -X POST http://127.0.0.1:30000/start_profile \ - -H "Content-Type: application/json" \ - -d '{ - "start_step": 3 - }' - -# Terminal 3: Send requests to generate load -python -m sglang.bench_serving --backend sglang --num-prompts 100 - -# Terminal 2: Stop profiling when done -curl -X POST http://127.0.0.1:30000/stop_profile -``` - -### Profiler Trace Merger for Distributed Traces - -SGLang now supports automatic merging of profiling traces from distributed setups with multiple parallelism types (TP, DP, PP, EP). This feature is particularly useful for analyzing performance across distributed runs. - -#### Multi-Node Profiling and Shared Storage Considerations - -Single-node profiler output merging is completely supported. When profiling in distributed environments spanning multiple nodes, shared storage (e.g., NFS, Lustre) should be accessible by all nodes for the output directory to enable merging of trace files. - -If there is no shared storage accessible across nodes, automatic merging of trace files during profiling is not supported directly as of now. - -#### HTTP API Usage - -```bash -# Start profiling with automatic trace merging enabled -curl -X POST /start_profile \ - -H "Content-Type: application/json" \ - -d '{ - "output_dir": "/tmp/profiles", # where to store profile traces - "num_steps": 10, - "activities": ["CPU", "GPU"], - "merge_profiles": true # optional argument to merge profile traces (default=False) - }' -``` - -#### Command Line Usage - -```bash -# Start profiling with merge enabled -python -m sglang.profiler \ - --num-steps 10 \ - --cpu \ - --gpu \ - --output-dir /tmp/profiles \ - --merge-profiles # optional argument to merge profile traces (default=False) -``` - -#### Output Files - -The profile merger generates: -- Individual rank trace files: `{profile_id}-TP-{tp}-DP-{dp}-PP-{pp}-EP-{ep}.trace.json.gz` -- Merged trace file: `merged-{profile_id}.trace.json.gz` - -### Possible PyTorch bugs -If in any cases you encounter the following error (for example, using qwen 2.5 VL): -```bash -RuntimeError: !stack.empty() INTERNAL ASSERT FAILED at "/pytorch/torch/csrc/autograd/profiler_python.cpp":983, please report a bug to PyTorch. Python replay stack is empty. -``` -This is likely a PyTorch Bug reported in [Bug: vLLM Profiler](https://github.com/vllm-project/vllm/issues/18240) and [Bug: torch.profiler.profile](https://github.com/pytorch/pytorch/issues/101632). As a workaround, you may disable `with_stack` with an environment variable such as follows: -```bash -export SGLANG_PROFILE_WITH_STACK=False -python -m sglang.bench_offline_throughput --model-path meta-llama/Llama-3.1-8B-Instruct --dataset-name random --num-prompts 10 --profile --mem-frac=0.8 -``` - -### View traces - -Trace files can be loaded and visualized from: - -1. https://ui.perfetto.dev/ (any browser) -2. chrome://tracing (Chrome browser only) - -If browser cannot open trace file due to its large size, -client can generate a small trace file (<100MB) by controlling number of prompts and lengths of prompt outputs. -For example, when profiling a server, - -```bash -python -m sglang.bench_serving --backend sglang --model meta-llama/Llama-3.1-8B-Instruct --num-prompts 2 --sharegpt-output-len 100 --profile -``` - -This command sets the number of prompts to 2 with `--num-prompts` argument and limits the length of output sequences to 100 with `--sharegpt-output-len` argument, which can generate a small trace file for browser to open smoothly. - -Additionally, if you want to locate the SGLang Python source code through the cuda kernel in Trace, you need to disable CUDA Graph when starting the service. This can be done by using the `--disable-cuda-graph` parameter in the command to start the service. - -## Profile with Nsight - -[Nsight systems](https://docs.nvidia.com/nsight-systems/) is an advanced tool that exposes more profiling details, such as register and shared memory usage, annotated code regions and low-level CUDA APIs and events. - -1. Prerequisite: - - Install using apt, or run inside a [NVIDIA Docker container](https://catalog.ngc.nvidia.com/orgs/nvidia/containers/pytorch/tags) or [SGLang Docker container](https://github.com/sgl-project/sglang/tree/main/docker). - - ```bash - # install nsys - # https://docs.nvidia.com/nsight-systems/InstallationGuide/index.html - apt update - apt install -y --no-install-recommends gnupg - echo "deb http://developer.download.nvidia.com/devtools/repos/ubuntu$(source /etc/lsb-release; echo "$DISTRIB_RELEASE" | tr -d .)/$(dpkg --print-architecture) /" | tee /etc/apt/sources.list.d/nvidia-devtools.list - apt-key adv --fetch-keys http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/7fa2af80.pub - apt update - apt install nsight-systems-cli - ``` - -2. To profile a single batch, use - - ```bash - nsys profile --trace-fork-before-exec=true --cuda-graph-trace=node python3 -m sglang.bench_one_batch --model meta-llama/Meta-Llama-3-8B --batch-size 64 --input-len 512 - ``` - -3. To profile a server, e.g. - - ```bash - # launch the server, set the delay and duration times according to needs - # after the duration time has been used up, server will be killed by nsys - - nsys profile --trace-fork-before-exec=true --cuda-graph-trace=node -o sglang.out --delay 60 --duration 70 python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --disable-radix-cache - - # client - python3 -m sglang.bench_serving --backend sglang --num-prompts 1000 --dataset-name random --random-input 1024 --random-output 512 - ``` - - In practice, we recommend users to set `--duration` argument to a large value. Whenever user wants the server to stop profiling. Firstly run: - - ```bash - nsys sessions list - ``` - - to get the session id in the form of `profile-XXXXX`, then run: - - ```bash - nsys stop --session=profile-XXXXX - ``` - - to manually kill the profiler and generate `nsys-rep` files instantly. - -4. Use NVTX to annotate code regions, e.g. to see their execution time. - - ```bash - # install nvtx - pip install nvtx - ``` - - ```python - # code snippets - import nvtx - with nvtx.annotate("description", color="color"): - # some critical code - ``` - -### Layer-wise NVTX Profiling with Nsight Systems - -SGLang provides built-in layerwise NVTX annotations that can be combined with the CUDA Profiler for detailed per-layer profiling in Nsight Systems. This is particularly useful for identifying performance bottlenecks at the layer level. - -#### Using `--enable-layerwise-nvtx-marker` with Nsight Systems and `/start_profile` - -The `--enable-layerwise-nvtx-marker` flag automatically adds NVTX markers to every layer in your model. This is particularly powerful when combined with Nsight Systems profiling to see detailed per-layer performance. - -**Method 1: Using `/start_profile` with CUDA_PROFILER (for programmatic control)** - -This method allows you to control exactly when profiling starts/stops via HTTP API while Nsight Systems is running. - -1. Launch the server with layerwise NVTX enabled under Nsight Systems: - - ```bash - # Terminal 1: Start server with nsys and capture-range option - nsys profile --trace-fork-before-exec=true \ - --cuda-graph-trace=node \ - --capture-range=cudaProfilerApi \ - --capture-range-end=stop \ - -o layerwise_profile \ - python -m sglang.launch_server \ - --model-path meta-llama/Llama-3.1-8B-Instruct \ - --enable-layerwise-nvtx-marker \ - --disable-cuda-graph - ``` - - Note: NVTX markers are not emitted for kernel launches captured by CUDA graphs. Use `--disable-cuda-graph` to ensure all layerwise NVTX markers are emitted in the trace. - -2. In another terminal, control profiling via `/start_profile` with `CUDA_PROFILER` activity: - - ```bash - # Terminal 2: Wait for server to be ready, then start CUDA profiling - # Wait 3 steps for warmup, then profile for 10 steps - curl -X POST http://127.0.0.1:30000/start_profile \ - -H "Content-Type: application/json" \ - -d '{ - "start_step": 3, - "num_steps": 10, - "activities": ["CUDA_PROFILER"] - }' - ``` - -3. Send requests to generate load: - - ```bash - # Terminal 3: Generate workload - python -m sglang.bench_serving --backend sglang --num-prompts 100 - ``` - -4. Profiling will automatically stop after 10 steps (due to `num_steps: 10`). If you hadn't specified `num_steps`, you would need to manually stop it: - - ```bash - # Terminal 2: Only needed if num_steps was not specified - curl -X POST http://127.0.0.1:30000/stop_profile - ``` - -The `--capture-range=cudaProfilerApi` option tells Nsight Systems to only capture data between `cudaProfilerStart()` and `cudaProfilerStop()` calls (triggered by `/start_profile` and `/stop_profile`), reducing overhead and file size. The `start_step` parameter skips the first 3 steps to avoid capturing warmup overhead. - -**Method 2: Simpler approach without `/start_profile` API** - -For simpler use cases where you don't need fine-grained control over profiling start/stop, you can profile with Nsight Systems capturing the entire workload: - -```bash -# Terminal 1: Start server with layerwise NVTX -# Note: --disable-cuda-graph ensures all NVTX markers are emitted -python -m sglang.launch_server \ - --model-path meta-llama/Llama-3.1-8B-Instruct \ - --enable-layerwise-nvtx-marker \ - --disable-cuda-graph - -# Terminal 2: Profile the benchmarking client -nsys profile --trace-fork-before-exec=true \ - --cuda-graph-trace=node \ - -o layerwise_profile \ - python -m sglang.bench_serving --backend sglang --num-prompts 10 -``` - -This approach profiles the entire client execution, including all server interactions. The layerwise NVTX markers will be visible in the Nsight Systems timeline. - -**Viewing the profiling results:** - -Open the generated `.qdrep` file with Nsight Systems: - -```bash -nsys-ui layerwise_profile.qdrep -``` - -In the Nsight Systems GUI, you'll see: -- **NVTX ranges**: Each layer appears as a labeled range in the timeline with detailed information in the marker metadata -- **CUDA kernels**: All GPU kernels are shown alongside the layer annotations -- **Layer hierarchy**: The full module path (e.g., `meta-llama/Meta-Llama-3.1-8B-Instruct.model.layers.0.self_attn.qkv_proj`) helps identify specific layers. The prefix uses the full model path from `--model-path`. -- **Tensor shapes**: Input/output dimensions and parameter shapes are included in the NVTX marker data - -**Benefits of layerwise NVTX profiling:** - -- **Granular visibility**: See exactly which layers are taking the most time -- **Memory tracking**: Identify layers with large memory allocations -- **Bottleneck identification**: Quickly locate inefficient operations -- **Communication overhead**: In multi-GPU setups, see per-layer communication costs -- **Development debugging**: Validate that model architecture changes have the expected performance impact - -## Other tips - -1. You can benchmark a model using dummy weights by only providing the config.json file. This allows for quick testing of model variants without training. To do so, add `--load-format dummy` to the above commands and then you only need a correct `config.json` under the checkpoint folder. -2. You can benchmark a model with modified configs (e.g., less layers) by using `--json-model-override-args`. For example, you can benchmark a model with only 2 layers and 2 kv heads using: - - ```bash - python -m sglang.bench_one_batch --model-path meta-llama/Meta-Llama-3.1-8B-Instruct --batch 32 --input-len 256 --output-len 32 --load-format dummy --json-model-override-args '{"num_hidden_layers": 1, "num_key_value_heads": 1}' - ``` - -3. You can use `--python-backtrace=cuda` to see python call stack for all CUDA kernels, as in PyTorch Profiler. (Caveat: this can cause inaccurately long kernel runtimes for CUDA event based timing) -4. For more arguments see [Nsight Systems User Guide](https://docs.nvidia.com/nsight-systems/UserGuide/index.html). diff --git a/docs/developer_guide/contribution_guide.md b/docs/developer_guide/contribution_guide.md deleted file mode 100644 index 1e999e5a1..000000000 --- a/docs/developer_guide/contribution_guide.md +++ /dev/null @@ -1,185 +0,0 @@ -# Contribution Guide - -Welcome to **SGLang**! We appreciate your interest in contributing. This guide provides a concise overview of how to set up your environment, run tests, build documentation, and open a Pull Request (PR). Whether you’re fixing a small bug or developing a major feature, we encourage following these steps for a smooth contribution process. - -## Install SGLang from Source - -### Fork and clone the repository - -**Note**: New contributors do **not** have the write permission to push to the official SGLang repo. Please fork the repository under your GitHub account, then clone your fork locally. - -```bash -git clone https://github.com//sglang.git -``` - -### Build from source - -Refer to [Install SGLang from Source](../get_started/install.md#method-2-from-source). - -## Format code with pre-commit - -We use [pre-commit](https://pre-commit.com/) to maintain consistent code style checks. Before pushing your changes, please run: - -```bash -pip3 install pre-commit -pre-commit install -pre-commit run --all-files -``` - -- **`pre-commit run --all-files`** manually runs all configured checks, applying fixes if possible. If it fails the first time, re-run it to ensure lint errors are fully resolved. Make sure your code passes all checks **before** creating a Pull Request. -- **Do not commit** directly to the `main` branch. Always create a new branch (e.g., `feature/my-new-feature`), push your changes, and open a PR from that branch. -- Link checking with lychee is **enforced in CI**. By default, it is not blocking local commits. -- To run local link checks manually, use: `pre-commit run --hook-stage manual lychee --all-files`. - -## Run and add unit tests - -If you add a new feature or fix a bug, please add corresponding unit tests to ensure coverage and prevent regression. - -### Unit tests (no server required) - -Unit tests live under [`test/registered/unit/`](https://github.com/sgl-project/sglang/tree/main/test/registered/unit), organized to mirror the `python/sglang/srt/` source tree. These tests validate component logic **without** launching a server or loading real model weights. -SGLang uses Python's built-in [unittest](https://docs.python.org/3/library/unittest.html) framework with [pytest](https://docs.pytest.org/) as the test runner. - -**When to add a unit test:** If you modify a file under `python/sglang/srt/`, check whether a corresponding test exists in `test/registered/unit/` and add coverage for your changes. For example: - -``` -srt/mem_cache/radix_cache.py → unit/mem_cache/test_radix_cache.py -srt/sampling/sampling_params.py → unit/sampling/test_sampling_params.py -``` - -**Run unit tests locally:** - -```bash -pytest test/registered/unit/ -v # all unit tests -pytest test/registered/unit/mem_cache/ -v # one module -``` - -**Run with coverage:** - -```bash -pytest test/registered/unit/ --cov --cov-config=.coveragerc -v -``` - -For conventions on CI registration, test structure, and examples, see [`test/registered/unit/README.md`](https://github.com/sgl-project/sglang/tree/main/test/registered/unit/README.md). - -### E2E tests (server required) - -For tests that require launching a server, refer to [`test/registered/README.md`](https://github.com/sgl-project/sglang/tree/main/test/registered/README.md) for guidance on where to place your test. - -For detailed instructions on running tests and integrating them into CI, refer to [test/README.md](https://github.com/sgl-project/sglang/tree/main/test/README.md). - -## Write documentations - -We recommend new contributors start from writing documentation, which helps you quickly understand SGLang codebase. -For more details, please refer to [docs/README.md](https://github.com/sgl-project/sglang/tree/main/docs/README.md). - -## Test the accuracy -If your code changes the model output, please run the accuracy tests. A quick sanity check is the few-shot GSM8K. - -``` -# Launch a server -python3 -m sglang.launch_server --model Qwen/Qwen2-7B-Instruct - -# Evaluate -python3 -m sglang.test.few_shot_gsm8k --num-questions 200 -``` - -Please note that the above script is primarily a sanity check, not a rigorous accuracy or speed test. -This test can have significant variance (1%–5%) in accuracy due to batching and the non-deterministic nature of the inference engine. -Also, do not rely on the "Latency/Output throughput" from this script, as it is not a proper speed test. - -GSM8K is too easy for state-of-the-art models nowadays. Please try your own more challenging accuracy tests. -You can find additional accuracy eval examples in: -- [test_eval_accuracy_large.py](https://github.com/sgl-project/sglang/blob/main/test/registered/eval/test_eval_accuracy_large.py) -- [test_gpt_oss_1gpu.py](https://github.com/sgl-project/sglang/blob/main/test/registered/core/test_gpt_oss_1gpu.py) - -## Benchmark the speed -Refer to [Benchmark and Profiling](../developer_guide/benchmark_and_profiling.md). - -## Requesting a review for merge -You can follow the pull request merge process described in [MAINTAINER.md](https://github.com/sgl-project/sglang/blob/main/.github/MAINTAINER.md). -You will need to work with the Merge Oncall, Codeowner, and other reviewers to get their approvals. -Then your PR can be merged. - -## How to Trigger CI Tests - -We have a lot of open PRs but limited CI machines, so only top and trusted contributors have permission to trigger CI tests. -Users with permission are listed in the [CI_PERMISSIONS.json](https://github.com/sgl-project/sglang/blob/main/.github/CI_PERMISSIONS.json) - -**PR authors** can always use `/rerun-failed-ci` on their own PRs, even if they are not listed in `CI_PERMISSIONS.json`. - -For CI to run on a pull request, it must have the "run-ci" label. Authorized users can add the label or rerun failed tests by commenting on the PR with one of these commands: - -- `/tag-run-ci-label`: Adds the "run-ci" label. Only **future** commits trigger CI; the current commit is unaffected. -- `/rerun-failed-ci`: Reruns workflows from the latest commit with conclusion **failed, flaky, or skipped**. -- `/tag-and-rerun-ci`: Runs both. Use this on a fresh PR to kick off CI on the current commit — `/tag-run-ci-label` alone won't. -- `/rerun-stage `: Reruns a single test stage without waiting for its dependencies. Useful for quickly validating a specific test fix instead of waiting ~30 minutes for preceding stages. - -If you have permission, the [Slash Command Handler](https://github.com/sgl-project/sglang/actions/workflows/slash-command-handler.yml) will run your command and react with a 👍 to your comment. It may take up to a few minutes for the reaction to appear. Here’s a usage [example](https://github.com/sgl-project/sglang/pull/14253#issuecomment-3599509302). - -To avoid spamming a PR with too many `/rerun-failed-ci` comments, you can also trigger the command by editing an existing comment and adding any suffix (e.g., `/rerun-failed-ci try again`). - -Example of rerunning a single test stage: `/rerun-stage unit-test-backend-4-gpu`. - -If you don’t have permission and you’re not the PR author, please ask maintainers to trigger CI for you. - -### CI rate limits - -Due to CI scheduling and limited resources, higher-priority PRs may preempt running jobs. In such cases, you may need to rerun the tests. -We apply CI rate limits to prevent abuse and ensure fair usage of our CI resources. - -Each CI workflow has a default limit defined in its workflow configuration file. For example, in [pr-gate.yml](https://github.com/sgl-project/sglang/blob/main/.github/workflows/pr-gate.yml), the default cooldown period is 120 minutes, and each workflow can override it via the `cool-down-minutes` input parameter: - -```yaml -cool-down-minutes: - description: "Default cooldown period in minutes; 0 disables rate limiting" - type: number - default: 120 -``` - -Users listed in [CI_PERMISSIONS.json](https://github.com/sgl-project/sglang/blob/main/.github/CI_PERMISSIONS.json) may have a per-user cooldown interval. In practice, we use the minimum of the workflow’s default window and the user-specific interval. - -## Code style guidance -- Avoid code duplication. If the same code snippet (more than five lines) appears multiple times, extract it into a shared function. -- Minimize device synchronization. Reduce expensive CPU-GPU synchronization operations, such as `tensor.item()` or `tensor.cpu()`, whenever possible. Use vectorized code. -- Prioritize extreme efficiency. SGLang is a runtime, and most of your code runs on the critical path for every request. Optimize all minor overheads as much as possible, especially in the model forward code. - - A common pattern is some runtime checks in the model forward pass (e.g., [this](https://github.com/sgl-project/sglang/blob/f1b0eda55c2c4838e8ab90a0fac7fb1e3d7064ab/python/sglang/srt/models/deepseek_v2.py#L486-L491)). These are very likely the same for every layer. Please cache the result as a single boolean value in `__init__` whenever possible. -- Make functions as pure as possible. Avoid in-place modification of arguments. -- Keep files concise. If a file exceeds 2,000 lines of code, split it into multiple smaller files. (e.g., `scheduler.py`, `scheduler_output_processor_mixin.py`) -- In a file, put core data structures at the top of the file. Put utility functions at the bottom of the file. -- Keep tests run fast. - - If a single test file run longer than 500 seconds, split it into multiple smaller files (e.g., `test_eagle_infer_a.py`, `test_eagle_infer_b.py`). - - If a single job in a github workflow runs longer than 30 mins, split it into smaller jobs/steps. - - Reuse server launches in your unit tests to make tests run faster. -- Never use `pickle.loads()`, `pickle.load()`, or `recv_pyobj()` to deserialize untrusted or network-received data. Python's [pickle module is not secure](https://docs.python.org/3/library/pickle.html) — it can execute arbitrary code during deserialization. Use safe serialization formats such as [msgpack](https://github.com/jcrist/msgspec) or JSON instead. -- When supporting new hardware or features, follow these guidelines: - - Do not drastically change existing code. - - Always prefer new files to introduce specific components for your new hardware (e.g., `allocator_ascend.py`). - - If you write multiple if/else blocks for new features, ensure the common path (e.g., NVIDIA hardware or the existing code path) is the first branch. - -## How to update sgl-kernel -Since sglang and the `sglang-kernel` (prior `sgl-kernel`) distribution are separate Python packages, our current GitHub CI infrastructure does not support updating a kernel and using it immediately within the same pull request (PR). -To add a new kernel or modify an existing one in the `sgl-kernel/` source tree, you must use multiple PRs. - -Follow these steps: - -1. Submit a PR to update the sgl-kernel source code without using it in sglang python package (e.g., [#8884](https://github.com/sgl-project/sglang/pull/8884/files)). -2. Bump the version of the kernel package (e.g., [#9220](https://github.com/sgl-project/sglang/pull/9220/files)). - - Once merged, this will trigger an automatic release of the `sglang-kernel` wheel to PyPI. - - If not urgent, you can wait for other people to release the wheel. A new version will typically be released within one week. -3. Apply the changes: - - Update the `sglang-kernel` version in `sglang/python/pyproject.toml` to use the modified kernels. - - Update the related caller code in the sglang to use the new kernel. - -## Tips for newcomers - -If you want to contribute but don’t have a specific idea in mind, pick issues labeled [“good first issue” or “help wanted”](https://github.com/sgl-project/sglang/issues?q=is%3Aissue+label%3A%22good+first+issue%22%2C%22help+wanted%22). These tasks typically have lower complexity and provide an excellent introduction to the codebase. - -Also check out the following materials as startup guide: -- [Mini-SGLang](https://github.com/sgl-project/mini-sglang) for a quick overview on the structure of sglang. -- [Code Walk-through](https://github.com/zhaochenyang20/Awesome-ML-SYS-Tutorial/tree/main/sglang/code-walk-through) for a deeper look into SGLang’s workflow. -- [GTC-2026 Training Lab](https://drive.google.com/file/d/1mwOZEtipNLJzrflCTodj34KhuOZEoEw5/view?usp=drive_link) for hands-on practices of how to do optimization, benchmarking, or profiling on a launched SGLang instance. - -If you have any questions or want to start a discussion, please feel free to ask in our [Slack channel](https://slack.sglang.io). - -Thank you for your interest in SGLang. Happy coding! diff --git a/docs/developer_guide/development_guide_using_docker.md b/docs/developer_guide/development_guide_using_docker.md deleted file mode 100644 index a833011c6..000000000 --- a/docs/developer_guide/development_guide_using_docker.md +++ /dev/null @@ -1,108 +0,0 @@ -# Development Guide Using Docker - -## Setup VSCode on a Remote Host -(Optional - you can skip this step if you plan to run sglang dev container locally) - -1. In the remote host, download `code` from [Https://code.visualstudio.com/docs/?dv=linux64cli](https://code.visualstudio.com/download) and run `code tunnel` in a shell. - -Example -```bash -wget https://vscode.download.prss.microsoft.com/dbazure/download/stable/fabdb6a30b49f79a7aba0f2ad9df9b399473380f/vscode_cli_alpine_x64_cli.tar.gz -tar xf vscode_cli_alpine_x64_cli.tar.gz - -# https://code.visualstudio.com/docs/remote/tunnels -./code tunnel -``` - -2. In your local machine, press F1 in VSCode and choose "Remote Tunnels: Connect to Tunnel". - -## Setup Docker Container - -### Option 1. Use the default dev container automatically from VSCode -There is a `.devcontainer` folder in the sglang repository root folder to allow VSCode to automatically start up within dev container. You can read more about this VSCode extension in VSCode official document [Developing inside a Container](https://code.visualstudio.com/docs/devcontainers/containers). -![image](https://github.com/user-attachments/assets/6a245da8-2d4d-4ea8-8db1-5a05b3a66f6d) -(*Figure 1: Diagram from VSCode official documentation [Developing inside a Container](https://code.visualstudio.com/docs/devcontainers/containers).*) - -To enable this, you only need to: -1. Start Visual Studio Code and install [VSCode dev container extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers). -2. Press F1, type and choose "Dev Container: Open Folder in Container. -3. Input the `sglang` local repo path in your machine and press enter. - -The first time you open it in dev container might take longer due to docker pull and build. Once it's successful, you should set on your status bar at the bottom left displaying that you are in a dev container: - -![image](https://github.com/user-attachments/assets/650bba0b-c023-455f-91f9-ab357340106b) - -Now when you run `sglang.launch_server` in the VSCode terminal or start debugging using F5, sglang server will be started in the dev container with all your local changes applied automatically: - -![image](https://github.com/user-attachments/assets/748c85ba-7f8c-465e-8599-2bf7a8dde895) - - -### Option 2. Start up containers manually (advanced) - -The following startup command is an example for internal development by the SGLang team. You can **modify or add directory mappings as needed**, especially for model weight downloads, to prevent repeated downloads by different Docker containers. - -❗️ **Note on RDMA** - - 1. `--network host` and `--privileged` are required by RDMA. If you don't need RDMA, you can remove them but keeping them there does not harm. Thus, we enable these two flags by default in the commands below. - 2. You may need to set `NCCL_IB_GID_INDEX` if you are using RoCE, for example: `export NCCL_IB_GID_INDEX=3`. - -```bash -# Change the name to yours -docker run -itd --shm-size 32g --gpus all -v --ipc=host --network=host --privileged --name sglang_dev lmsysorg/sglang:dev /bin/zsh -docker exec -it sglang_dev /bin/zsh -``` -Some useful volumes to mount are: -1. **Huggingface model cache**: mounting model cache can avoid re-download every time docker restarts. Default location on Linux is `~/.cache/huggingface/`. -2. **SGLang repository**: code changes in the SGLang local repository will be automatically synced to the .devcontainer. - -Example 1: Mounting local cache folder `/opt/dlami/nvme/.cache` but not the SGLang repo. Use this when you prefer to manually transfer local code changes to the devcontainer. -```bash -docker run -itd --shm-size 32g --gpus all -v /opt/dlami/nvme/.cache:/root/.cache --ipc=host --network=host --privileged --name sglang_zhyncs lmsysorg/sglang:dev /bin/zsh -docker exec -it sglang_zhyncs /bin/zsh -``` -Example 2: Mounting both HuggingFace cache and local SGLang repo. Local code changes are automatically synced to the devcontainer as the SGLang is installed in editable mode in the dev image. -```bash -docker run -itd --shm-size 32g --gpus all -v $HOME/.cache/huggingface/:/root/.cache/huggingface -v $HOME/src/sglang:/sgl-workspace/sglang --ipc=host --network=host --privileged --name sglang_zhyncs lmsysorg/sglang:dev /bin/zsh -docker exec -it sglang_zhyncs /bin/zsh -``` -## Debug SGLang with VSCode Debugger -1. (Create if not exist) open `launch.json` in VSCode. -2. Add the following config and save. Please note that you can edit the script as needed to apply different parameters or debug a different program (e.g. benchmark script). - ```JSON - { - "version": "0.2.0", - "configurations": [ - { - "name": "Python Debugger: launch_server", - "type": "debugpy", - "request": "launch", - "module": "sglang.launch_server", - "console": "integratedTerminal", - "args": [ - "--model-path", "meta-llama/Llama-3.2-1B", - "--host", "0.0.0.0", - "--port", "30000", - "--trust-remote-code", - ], - "justMyCode": false - } - ] - } - ``` - -3. Press "F5" to start. VSCode debugger will ensure that the program will pause at the breakpoints even if the program is running at remote SSH/Tunnel host + dev container. - -## Profile - -```bash -# Change batch size, input, output and add `disable-cuda-graph` (for easier analysis) -# e.g. DeepSeek V3 -nsys profile -o deepseek_v3 python3 -m sglang.bench_one_batch --batch-size 1 --input 128 --output 256 --model deepseek-ai/DeepSeek-V3 --trust-remote-code --tp 8 --disable-cuda-graph -``` - -## Evaluation - -```bash -# e.g. gsm8k 8 shot -python3 benchmark/gsm8k/bench_sglang.py --num-questions 2000 --parallel 2000 --num-shots 8 -``` diff --git a/docs/developer_guide/development_jit_kernel_guide.md b/docs/developer_guide/development_jit_kernel_guide.md deleted file mode 100644 index 0059a82bd..000000000 --- a/docs/developer_guide/development_jit_kernel_guide.md +++ /dev/null @@ -1,315 +0,0 @@ -# Development Guide for JIT Kernels - -## Environment Setup - -We strongly recommend using `clangd` as the language server for JIT kernel development. -For Ubuntu/Debian, you can download clangd from [apt.llvm.org](https://apt.llvm.org/). -If you are using VS Code, we recommend installing the `clangd` extension for better IDE integration. - -All JIT-related files are located in `python/sglang/jit_kernel`. -Unlike `sgl-kernel`, which compiles CUDA/C++ binaries ahead of time (AOT), just-in-time (JIT) kernels are compiled at runtime. -Consequently, a static `compile_commands.json` cannot be generated. -To enable code completion with `clangd`, run `python -m sglang.jit_kernel` to generate a `.clangd` configuration file in your current directory. -After generating the file, restart the clangd language server. It should now recognize all JIT kernel files. - -## Code Structure - -### C++ Implementation - -C++ source code is located in `python/sglang/jit_kernel/csrc`. -Reusable functions should be placed in `python/sglang/jit_kernel/include`. - -We use [tvm-ffi](https://github.com/apache/tvm-ffi) for efficient foreign language bindings. -Refer to the [documentation](https://tvm.apache.org/ffi/) for advanced usage, such as exporting C++ objects. -Typically, `tvm::ffi::TensorView` is sufficient for passing PyTorch Tensors from Python. - -### Python Interface - -Python interfaces are defined in `python/sglang/jit_kernel`. -The `load_jit` utility function in `python/sglang/jit_kernel/utils.py` loads and returns the compiled module. -To export a C++ function (e.g., `cpp_func`), pass `cuda_wrappers=[("func", "cpp_func")]` to `load_jit`. -The function can then be called in Python as `module.func`. - -For caching compiled modules, prefer `sglang.jit_kernel.utils.cache_once` over `functools.lru_cache`. -`functools.lru_cache` is not compatible with `torch.compile`. - -### C++ Utilities - -The following C++ utilities are available: - -#### Integer Range - -Similar to PyTorch, we provide an `irange` function to represent an integer range. - -```C++ -#include - -void test() { - for (auto i : host::irange(100)) { // [0, 100) - // do something - } - for (auto i : host::irange(0, 100)) { // [0, 100) - // do something - } -} - -``` - -#### Runtime Checking - -`RuntimeCheck` validates conditions at runtime. It accepts optional arguments for error reporting. -If the check fails, these arguments are output to aid debugging. -`RuntimeDeviceCheck` verifies the status of the last kernel launch. - -```C++ -#include -#include - -void test() { - host::RuntimeCheck(1 + 1 == 2, 1 + 1, " != ", 2); - host::RuntimeDeviceCheck(); - // check the provided `cudaError_t` - host::RuntimeDeviceCheck(cudaGetLastError()); -} - -``` - -#### Tensor Checking - -`TensorMatcher` provides a readable way to validate and extract tensor shape information. - -```cpp -#include - -void test(const tvm::ffi::TensorView k_cache, const tvm::ffi::TensorView v_cache) { - using namespace host; - - auto D = SymbolicSize{"D"}; // cache dimension - auto N = SymbolicSize{"N"}; // kvcache stride - auto dtype = SymbolicDType{}; - auto device = SymbolicDevice{}; - - TensorMatcher({-1, D}) // - .with_strides({N, 1}) - .with_dtype(dtype) - .with_device(device) - .verify(k_cache) - .verify(v_cache); -} -``` - -Configure the `TensorMatcher` with expected stride, dtype, and device properties before verification. -- If `with_strides` is omitted, the tensor is expected to be contiguous. -- Template arguments in `with_dtype` restrict the allowed data types. -- Template arguments in `with_device` restrict the allowed devices. -- Values passed to `with_xxx` methods enforce equality checks. -- Passing `-1` for size or stride allows matching any value. - -A `Symbolic` variable must resolve to the same value across all verifications. -Use `.unwrap()` to retrieve the matched value after verification. - -> Note: `TensorMatcher` is a temporary expression and should not be stored in a variable. - -> Tip: Add `//` at the end of the `TensorMatcher` chain to enforce proper indentation. - -#### Kernel Launching - -`LaunchKernel::resolve_device` retrieves the current `cudaStream` from PyTorch. -Kernels can also be launched directly using `LaunchKernel`. - -```cpp -#include - -#include - -__global__ void kernel() {} - -void test() { - const auto num_blocks = 1; - const auto num_threads = 32; - const auto dynamic_smem = 0; - - DLDevice dev; // suppose this is initialized properly - host::LaunchKernel(num_blocks, num_threads, dev)(kernel); - - cudaStream_t stream = host::LaunchKernel::resolve_device(dev); - host::LaunchKernel(num_blocks, num_threads, stream, dynamic_smem)(kernel); -} - -``` - -## Add new kernels - -This section walks through a complete, end-to-end example of adding a new JIT kernel to the system. -We use a simple add_constant kernel as a running example, which adds a constant integer value to every element of an input tensor. - -Conceptually, the Python interface looks like this: - -```python -def add_constant(src: torch.Tensor, c: int): - return src + c -``` - -### STEP 1: Write the C++ kernel - -Write your CUDA kernel in [jit_kernel/csrc/add_constant.cuh](../../python/sglang/jit_kernel/csrc/add_constant.cuh). For demonstration purposes, we pass the constant value as a template parameter. - -```cpp -#include // For TensorMatcher, SymbolicSize, SymbolicDevice -#include // For LaunchKernel -#include // For div_ceil, RuntimeCheck - -#include -#include - -#include -#include - -namespace { - -template -__global__ void add_constant_kernel(int32_t* dst, const int32_t* src, size_t length) { - size_t idx = blockIdx.x * blockDim.x + threadIdx.x; - if (idx < length) { - dst[idx] = src[idx] + kConstant; - } -} - -constexpr size_t kBlockSize = 256; - -// You can also use struct with static method as an alternative -template -void add_constant(tvm::ffi::TensorView dst, tvm::ffi::TensorView src) { - using namespace host; - - // 1. Validate input tensors - SymbolicSize N = {"num_elements"}; - SymbolicDevice device_; - TensorMatcher({N}) // 1D tensor, must be contiguous - .with_dtype() // must be int32 - .with_device(device_) // must be on CUDA device - .verify(dst) // check tensor dst - .verify(src); // check tensor src - - // 2. Extract required parameters, prepare for kernel launch - const size_t num_elements = N.unwrap(); - const size_t grid_size = div_ceil(num_elements, kBlockSize); - const DLDevice device = device_.unwrap(); - // some extra runtime checks using host::RuntimeCheck - RuntimeCheck(num_elements > 0, "We only support non-empty tensors, got num_elements = ", num_elements); - - // 3. Launch the kernel. Error code will be automatically checked. - LaunchKernel(grid_size, kBlockSize, device /*, dynamic_smem*/)( - // kernel function - add_constant_kernel, - // kernel arguments - static_cast(dst.data_ptr()), - static_cast(src.data_ptr()), - num_elements); -} - -} // namespace - -``` - -### STEP 2: Create Python Interfaces - -Next, expose the kernel through a Python wrapper. -Create a new file at [jit_kernel/add_constant.py](../../python/sglang/jit_kernel/add_constant.py) and expose the needed interfaces. - -```python -from __future__ import annotations -from typing import TYPE_CHECKING - -import torch - -from sglang.jit_kernel.utils import cache_once, load_jit, make_cpp_args - -if TYPE_CHECKING: - from tvm_ffi.module import Module - - -@cache_once -def _jit_add_constant_module(constant: int) -> Module: - args = make_cpp_args(constant) # pass all the template argument - return load_jit( - "add_constant", - *args, - cuda_files=["add_constant.cuh"], - cuda_wrappers=[("add_constant", f"add_constant<{args}>")], - ) - - -def add_constant(src: torch.Tensor, constant: int) -> torch.Tensor: - if not src.is_cuda: - raise RuntimeError("src must be a CUDA tensor") - if src.dtype != torch.int32: - raise RuntimeError(f"Unsupported dtype {src.dtype}. Supported: int32") - dst = torch.empty_like(src) - module = _jit_add_constant_module(constant) - module.add_constant(dst, src) - return dst - -``` - -Keep the Python wrapper thin, but still validate the basic invariants such as device and dtype before dispatch. In the current JIT/FFI path, invalid tensors are not always rejected safely before launch. - -### STEP 3: Use your kernel - -Finally, import and use the kernel like a regular Python function: - -```python -from sglang.jit_kernel.add_constant import add_constant -``` - -For a complete, runnable example, refer to [test_add_constant.py](../../test/registered/jit/test_add_constant.py). - -## C++ Include Library Reference - -The JIT kernel framework provides a set of reusable C++ headers in -`python/sglang/jit_kernel/include/sgl_kernel/`. Each header is designed -to be lightweight and self-contained. Below is a summary of each header -and its key APIs. - -### Core Utilities - -| Header | Namespace | Purpose | -|--------|-----------|---------| -| `utils.h` | `host` | Host-side essentials: `RuntimeCheck`, `Panic`, `div_ceil`, `irange` | -| `utils.cuh` | `device` / `host` | Type aliases (`fp16_t`, `bf16_t`, ...), `SGL_DEVICE` macro, PDL helpers, `LaunchKernel`, `RuntimeDeviceCheck` | -| `source_location.h` | (global) | Portable `std::source_location` wrapper for error reporting | -| `runtime.cuh` | `host::runtime` | CUDA runtime queries: `get_blocks_per_sm`, `get_sm_count`, `get_cc_major`, `get_runtime_version`, `get_available_dynamic_smem_per_block` | - -### Tensor Validation - -| Header | Namespace | Purpose | -|--------|-----------|---------| -| `tensor.h` | `host` | `TensorMatcher`, `SymbolicSize`, `SymbolicDType`, `SymbolicDevice` | - -### Math & Type System - -| Header | Namespace | Purpose | -|--------|-----------|---------| -| `math.cuh` | `device::math` | `max`, `min`, `abs`, `sqrt`, `rsqrt`, `exp`, `sin`, `cos`, constants | -| `type.cuh` | (global) / `device` | `dtype_trait`, `packed_t`, `device::cast(from)` | - -### Memory Access - -| Header | Namespace | Purpose | -|--------|-----------|---------| -| `vec.cuh` | `device` | `AlignedVector` - vectorized load/store (up to 128-bit; 256-bit requires Blackwell GPUs) | -| `tile.cuh` | `device::tile` | `Memory` - cooperative tiled memory I/O (thread/warp/CTA) | - -### Parallel Primitives - -| Header | Namespace | Purpose | -|--------|-----------|---------| -| `warp.cuh` | `device::warp` | `reduce_sum`, `reduce_max` via `__shfl_xor_sync` | -| `cta.cuh` | `device::cta` | `reduce_max` across warps via shared memory | -| `atomic.cuh` | `device::atomic` | `max` - atomic float max (CUDA + ROCm fallback) | - -### Reusable Kernel Templates - -| Header | Namespace | Purpose | -|--------|-----------|---------| -| `impl/norm.cuh` | `host::norm` / `device::norm` | RMSNorm building blocks (warp & CTA paths, `StorageType`) | diff --git a/docs/developer_guide/evaluating_new_models.md b/docs/developer_guide/evaluating_new_models.md deleted file mode 100644 index f3126c9a0..000000000 --- a/docs/developer_guide/evaluating_new_models.md +++ /dev/null @@ -1,146 +0,0 @@ -# Evaluating New Models with SGLang - -This document provides commands for evaluating models' accuracy and performance. Before open-sourcing new models, we strongly suggest running these commands to verify whether the score matches your internal benchmark results. - -**For cross verification, please submit commands for installation, server launching, and benchmark running with all the scores and hardware requirements when open-sourcing your models.** - -[Reference: MiniMax M2](https://github.com/sgl-project/sglang/pull/12129) - -## Accuracy - -### LLMs - -SGLang provides built-in scripts to evaluate common benchmarks. - -**MMLU** - -```bash -python -m sglang.test.run_eval \ - --eval-name mmlu \ - --port 30000 \ - --num-examples 1000 \ - --max-tokens 8192 -``` - -**GSM8K** - -```bash -python -m sglang.test.few_shot_gsm8k \ - --host 127.0.0.1 \ - --port 30000 \ - --num-questions 200 \ - --num-shots 5 -``` - -**HellaSwag** - -```bash -python benchmark/hellaswag/bench_sglang.py \ - --host 127.0.0.1 \ - --port 30000 \ - --num-questions 200 \ - --num-shots 20 -``` - -**GPQA** - -```bash -python -m sglang.test.run_eval \ - --eval-name gpqa \ - --port 30000 \ - --num-examples 198 \ - --max-tokens 120000 \ - --repeat 8 -``` - -```{tip} -For reasoning models, add `--thinking-mode ` (e.g., `qwen3`, `deepseek-v3`). You may skip it if the model has forced thinking enabled. -``` - -**HumanEval** - -```bash -pip install human_eval - -python -m sglang.test.run_eval \ - --eval-name humaneval \ - --num-examples 10 \ - --port 30000 -``` - -### VLMs - -**MMMU** - -```bash -python benchmark/mmmu/bench_sglang.py \ - --port 30000 \ - --concurrency 64 -``` - -```{tip} -You can set max tokens by passing `--extra-request-body '{"max_tokens": 4096}'`. -``` - -For models capable of processing video, we recommend extending the evaluation to include `VideoMME`, `MVBench`, and other relevant benchmarks. - -## Performance - -Performance benchmarks measure **Latency** (Time To First Token - TTFT) and **Throughput** (tokens/second). - -### LLMs - -**Latency-Sensitive Benchmark** - -This simulates a scenario with low concurrency (e.g., single user) to measure latency. - -```bash -python -m sglang.bench_serving \ - --backend sglang \ - --host 0.0.0.0 \ - --port 30000 \ - --dataset-name random \ - --num-prompts 10 \ - --max-concurrency 1 -``` - -**Throughput-Sensitive Benchmark** - -This simulates a high-traffic scenario to measure maximum system throughput. - -```bash -python -m sglang.bench_serving \ - --backend sglang \ - --host 0.0.0.0 \ - --port 30000 \ - --dataset-name random \ - --num-prompts 1000 \ - --max-concurrency 100 -``` - -**Single Batch Performance** - -You can also benchmark the performance of processing a single batch offline. - -```bash -python -m sglang.bench_one_batch_server \ - --model \ - --batch-size 8 \ - --input-len 1024 \ - --output-len 1024 -``` - -You can run more granular benchmarks: - -- **Low Concurrency**: `--num-prompts 10 --max-concurrency 1` -- **Medium Concurrency**: `--num-prompts 80 --max-concurrency 16` -- **High Concurrency**: `--num-prompts 500 --max-concurrency 100` - -## Reporting Results - -For each evaluation, please report: - -1. **Metric Score**: Accuracy % (LLMs and VLMs); Latency (ms) and Throughput (tok/s) (LLMs only). -2. **Environment settings**: GPU type/count, SGLang commit hash. -3. **Launch configuration**: Model path, TP size, and any special flags. -4. **Evaluation parameters**: Number of shots, examples, max tokens. diff --git a/docs/developer_guide/msprobe_debugging_guide.md b/docs/developer_guide/msprobe_debugging_guide.md deleted file mode 100644 index ee0d8496e..000000000 --- a/docs/developer_guide/msprobe_debugging_guide.md +++ /dev/null @@ -1,598 +0,0 @@ -# MSProbe Debugging Guide - -## Introduction to MSProbe - -MSProbe is a debugging tool for AI models that diagnoses accuracy anomalies and -numerical errors during model training and inference. It captures and monitors intermediate data (feature maps, weights, -activations, layer outputs) and contextual metadata (prompts, tensor dtypes, hardware configuration), and supports -visual analysis to systematically trace the root cause of accuracy degradation or numerical errors (e.g., NaN/Inf, -output drift, mismatched predictions). - -## Basic Details - -### Background Concepts: MSProbe Dumping Levels - -MSProbe supports three accuracy levels for data dumping, each for different debugging needs: - -- **L0**: Dumps tensors/statistics at the **module level** and generates `construct.json` (for network structure - reconstruction in visualization). Requires passing a model/submodule handle. -- **L1**: Dumps tensors/statistics at the **torch API level**, suitable for fine-grained API-level numerical checking. -- **mix**: Combines L0 + L1, ideal for scenarios that require both **graph reconstruction** and **numerical comparison**. - -### Prerequisites: Install MSProbe - -Install MSProbe with pip: - -```shell -pip install mindstudio-probe --pre -``` - -### Key Configuration Parameters - -MSProbe uses a JSON configuration file for customized data dumping. All core parameters are listed in the table below, -with the default JSON configuration provided for reference. - -#### Configuration Parameter Table - -| Field | Description | Required | -|:------------:|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:--------:| -| `task` | Type of dump task. Common PyTorch values include `"statistics"` and `"tensor"`. A statistics task collects tensor statistics (mean, variance, max, min, etc.) while a tensor task captures arbitrary tensors. | Yes | -| `dump_path` | Directory where dump results are stored. When omitted, `MSProbe` uses its default path. | No | -| `rank` | Ranks to sample. An empty list collects every rank. For single-card tasks you must set this field to `[]`. | No | -| `step` | Token iteration(s) to sample. An empty list means every iteration. | No | -| `level` | Dump level string (`"L0"`, `"L1"`, or `"mix"`). `L0` targets `nn.Module`, `L1` targets `torch.api`, and `mix` collects both. | Yes | -| `async_dump` | Whether to enable asynchronous dump (supported for PyTorch `statistics`/`tensor` tasks). Defaults to `false`. | No | -| `scope` | Customize the scope of dump. Provide two module or API names that follow the tool's naming convention to lock a range, only data between the two names will be dumped. An empty list dumps every module or torch API.

Examples:
`"scope": ["Module.conv1.Conv2d.forward.0", "Module.fc2.Linear.forward.0"]`
`"scope": ["Tensor.add.0.forward", "Functional.square.2.forward"]`

The `level` setting determines what can be provided—modules when `level=L0`, APIs when `level=L1`, and either modules or APIs when `level=mix`. | No | -| `list` | Customize dump list, only dumps elements from the list. An empty list dumps every module or torch API. Options include:

򴎲Supply the full names of specific APIs in PyTorch pynative scenarios to only dump those APIs. Example: `"list": ["Tensor.permute.1.forward", "Tensor.transpose.2.forward", "Torch.relu.3.backward"]`.
򴎲When `level=mix`, you can provide module names so that the dump expands to everything produced while the module is running. Example: `"list": ["Module.module.language_model.encoder.layers.0.mlp.ParallelMlp.forward.0"]`.
򴎲Provide a substring such as `"list": ["relu"]` to dump every API whose name contains the substring. When `level=mix`, modules whose names contain the substring are also expanded. | No | - -#### Default configuration - -```json -{ - "task": "statistics", - "dump_path": "./dump_path", - "rank": [], - "step": [], - "level": "L1", - "async_dump": false, - "statistics": { - "scope": [], - "list": [], - "data_mode": [ - "all" - ], - "summary_mode": "statistics" - }, - "tensor": { - "scope": [], - "list": [], - "data_mode": [ - "all" - ], - "file_format": "npy" - }, - "acc_check": { - "white_list": [], - "black_list": [], - "error_data_path": "./" - } -} -``` - -#### Outputs - -Dump files are written into `dump_path` you defined. They usually contain: - -- `dump.json`, which records metadata such as dtype, shape, min, max, mean, L2 norm, and `requires_grad`. -- `construct.json`, hierarchical structure description, when `level` is `L0` or `mix` (required for visualization), its - content is not empty. -- `stack.json`, record the call stack information of API/Module. -- `dump_tensor_data`, generated when `task` is `tensor` and save the collected tensor data. - -See [dump directory description](#dump-directory-description) for details. - -> **Note**: When MSProbe is enabled, cuda graph is disabled(disable_cuda_graph=True) because MSProbe only supports dump -> in eager mode, warmup is disabled(skip_server_warmup=True) because there is no need to dump data for this stage. - -## End-to-End Examples - -MSProbe’s full debugging workflow follows **Enable → Collect Data → Visualize → Analyze Root Cause**. Below is a common -E2E example for SGLang-based model inference debugging. - -### Example : Advanced Debugging with Custom Configuration - -Suitable for targeted debugging (e.g., only collect statistics data for specific ranks/steps, enable mix level for graph -reconstruction + numerical comparison) and root cause analysis via **problem vs. benchmark comparison**. - -#### Step 1: Enable -##### Prepare Custom Configuration JSON - -Create `msprobe-config.json` (dump statistics data for rank0/1, step0/1, mix level): - -```json -{ - "task": "statistics", - "dump_path": "./problem_dump", - "rank": [ - 0, - 1 - ], - "step": [ - 0, - 1 - ], - "level": "mix", - "async_dump": false, - "statistics": { - "scope": [], - "list": [], - "data_mode": [ - "all" - ], - "summary_mode": "statistics" - } -} -``` - -##### Enable MSProbe with Custom Configuration in SGLang - -Launch the SGLang server and specify the configuration file path with `--msprobe-dump-config`: - -```bash -python3 -m sglang.launch_server \ - --model-path Qwen/Qwen2.5-0.5B-Instruct \ - --host 127.0.0.1 \ - --port 1027 \ - --msprobe-dump-config /home/msprobe-config.json -``` -#### Step 2: Collect Data -##### Collect Dump Data for Problem & Benchmark Sides - -Send normal inference requests to trigger model running (MSProbe automatically collects data during request processing): - -```bash -curl -H "Content-type: application/json" \ - -X POST \ - -d '{ - "model": "Qwen/Qwen2.5-0.5B-Instruct", - "messages": [ - { - "role": "user", - "content": "Hello, my name is" - } - ], - "max_tokens": 10 - }' \ - http://127.0.0.1:1027/v1/chat/completions -``` - -- **Problem side**: Run the above SGLang server (with the accuracy/numerical issue) and send inference request; dump - data is saved to `./problem_dump`. -- **Benchmark side**: Launch a normal SGLang server (without the issue, e.g., stable framework version/operator) with - the **same custom configuration** and send the **same inference request**; rename the dump directory - to `./bench_dump`. - -> **Key Requirement**: Problem and benchmark dumps must use the same inputs and sampling points (rank/step) -> for valid comparison. - -##### Check Generated Dump Files - -Dump files are saved to `./problem_dump` and `./bench_dump` you defined and include core files for subsequent analysis: - -- `dump.json`: Records tensor metadata of APIs and modules (dtype, shape, min/max/mean, L2 norm, `requires_grad`, etc.). -- `stack.json`: Logs call stack information of APIs and modules. -- `construct.json`: hierarchical structure description, required for visualization, its content is not empty. - -#### Step 3: Visualize -##### Visualize Problem vs. Benchmark Comparison (Multi-Rank) - -Generate a multi-rank comparison visualization file (mix level generates `construct.json` for graph reconstruction): - -```shell -msprobe graph_visualize -tp ./problem_dump/step0 -gp ./bench_dump/step0 -o ./graph_output -``` - -- `-tp`: Path to problem-side dump data -- `-gp`: Path to benchmark-side dump data -- `-o`: Output directory for visualization files - -If you want overflow check (for NaN/Inf detection), please specify the parameter `-oc` - -```shell -msprobe graph_visualize -tp ./problem_dump/step0 -gp ./bench_dump/step0 -o ./graph_output -oc -``` - -After the comparison or build task finishes, a `compare_{timestamp}.vis.db` file is created under `graph_output`. - -##### Launch TensorBoard - -Start TensorBoard: -```bash -tensorboard --logdir ./graph_output --bind_all --port 6006 -``` -#### Step 4: Analyze Root Cause -##### Locate Root Cause - -Root Cause Analysis in TensorBoard: -- Divergent nodes (with accuracy/numerical differences) are highlighted in **red** (darker red = larger difference). -- Click on divergent nodes to view detailed tensor data (inputs/outputs, parameters) and API/module call stacks. -- Use the **search/filter** function to quickly locate key layers/APIs (e.g., "relu", "conv"). -- Switch between ranks/steps via the UI to check cross-rank/cross-step divergence. -- Check the **overflow check** tab for NaN/Inf values in specific nodes (the direct cause of numerical instability). - -##### Verify the Root Cause - -After locating the divergent node (e.g., a specific Conv layer or torch API with abnormal tensor values), verify by: - -- Narrowing the dump scope to this node (via `scope`/`list` in the configuration file) for fine-grained data collection. -- Modifying the problematic layer/API (e.g., replacing the operator, adjusting the dtype) and re-running the debugging - workflow to confirm the issue is resolved. - -## Troubleshooting - -### No Dump Files Generated - -1. To confirm if MSProbe is installed, use `pip show mindstudio_probe` to troubleshoot. If it is installed, the MSProbe - version information will be printed. If it is confirmed that it has not been installed, please - use `pip install mindstudio-probe --pre` for installation; -2. Confirm the `--msprobe-dump-config` parameter points to the **correct JSON file path**. - -### Dump Files Are Too Large (Excessive Data) - -1. Start with `task: "statistics"` instead of `"tensor"` to collect only tensor statistics (avoids raw tensor dump); -2. Narrow the dump range with the `scope` field (specify start/end module/API); -3. Filter dump targets with the `list` field (only dump specific modules/APIs or substrings); -4. Sample specific `rank` and `step` (avoid dumping all ranks/iterations). - -### TensorBoard Visualization Fails - -1. Confirm `construct.json` is not empty (requires `level: L0` or `mix` – L1 does not generate graph files); -2. Check that the `-tp` (problem dump) and `-gp` (benchmark dump) paths point to **valid rank/step subdirectories** ( - e.g., `srep0/rank0`); -3. Ensure the MSProbe version is up-to-date (reinstall with `pip install mindstudio-probe --pre --upgrade`); -4. Verify TensorBoard is installed and the `--logdir` parameter points to the directory containing `.vis.db` files (not - the file itself). - -### Numerical Comparison Shows No Divergence But Model Accuracy Is Low - -1. Expand the dump `step` range (check more token iterations for late-stage divergence); -2. Switch to `task: "tensor"` (statistics may mask subtle numerical differences in raw tensor data); -3. Ensure the problem and benchmark dumps use **the same input data/hardware configuration** (different inputs lead to - invalid comparisons); -4. Use the `manual mapping` feature in TensorBoard (automatic mapping may miss some nodes for custom models). - ---- - -## Appendix - -### Dump directory description - -```text -├── problem_dump or bench_dump -│ ├── step0 -│ │ ├── rank0 -│ │ │ ├── dump_tensor_data -│ │ │ │ ├── Tensor.permute.1.forward.pt -│ │ │ │ ├── Functional.linear.5.backward.output.pt # Format: {api_type}.{api_name}.{call_count}.{forward/backward}.{input/output}.{arg_index}. -│ │ │ │ │ # arg_index is the nth input or output of the API. If an input is a list, keep numbering with decimals (e.g., 1.1 is the first element of the first argument). -│ │ │ │ ├── Module.conv1.Conv2d.forward.0.input.0.pt # Format: {Module}.{module_name}.{class_name}.{forward/backward}.{call_count}.{input/output}.{arg_index}. -│ │ │ │ ├── Module.conv1.Conv2d.forward.0.parameters.bias.pt # Module parameter data: {Module}.{module_name}.{class_name}.forward.{call_count}.parameters.{parameter_name}. -│ │ │ │ └── Module.conv1.Conv2d.parameters_grad.weight.pt # Module parameter gradients: {Module}.{module_name}.{class_name}.parameters_grad.{parameter_name}. Gradients do not include call_count because the same gradient updates all invocations. -│ │ │ │ # When the `model` argument passed to dump is a List[torch.nn.Module] or Tuple[torch.nn.Module], module-level data names also include the index inside the list ({Module}.{index}.*), e.g., Module.0.conv1.Conv2d.forward.0.input.0.pt. -│ │ │ ├── dump.json -│ │ │ ├── stack.json -│ │ │ ├── dump_error_info.log -│ │ │ └── construct.json -│ │ ├── rank1 -│ │ │ ├── dump_tensor_data -│ │ │ │ └── ... -│ │ │ ├── dump.json -│ │ │ ├── stack.json -│ │ │ ├── dump_error_info.log -│ │ │ └── construct.json -│ │ ├── ... -│ │ │ -│ │ └── rank7 -│ ├── step1 -│ │ ├── ... -│ ├── step2 -``` - -- `rank`: Device ID. Each card writes its data to the corresponding `rank{ID}` directory. In non-distributed scenarios - the directory is simply named `rank`. -- `dump_tensor_data`: Save the collected tensor data. -- `dump.json`: Statistics for the forward data of each API or module, including names, dtype, shape, max, min, mean, L2 - norm (square root of the L2 variance), and CRC-32 when `summary_mode="md5"`. - See [dump.json file description](#dumpjson-file-description) for details. -- `dump_error_info.log`: Present only when the dump tool encountered an error and records the failure log. -- `stack.json`: Call stacks for APIs/modules. -- `construct.json`: Hierarchical structure description. Empty when `level=L1`. - -### dump.json file description - -#### L0 level - -An L0 `dump.json` contains forward/backward I/O for modules together with parameters and parameter gradients. Using -PyTorch's `Conv2d` as an example, the network code looks like: - -`output = self.conv2(input) # self.conv2 = torch.nn.Conv2d(64, 128, 5, padding=2, bias=True)` - -`dump.json` contains the following entries: - -- `Module.conv2.Conv2d.forward.0`: Forward data of the module. `input_args` represents positional inputs, `input_kwargs` - represents keyword inputs, `output` stores forward outputs, and `parameters` stores weights/biases. -- `Module.conv2.Conv2d.parameters_grad`: Parameter gradients (weight and bias). -- `Module.conv2.Conv2d.backward.0`: Backward data of the module. `input` represents gradients that flow into the - module (gradients of the forward outputs) and `output` represents gradients that flow out (gradients of the module - inputs). - -**Note**: When the `model` parameter passed to the dump API is `List[torch.nn.Module]` or `Tuple[torch.nn.Module]`, -module-level names include the index inside the list (`{Module}.{index}.*`). Example: `Module.0.conv1.Conv2d.forward.0`. - -
- -L0 dump.json - -```json -{ - "task": "tensor", - "level": "L0", - "framework": "pytorch", - "dump_data_dir": "/dump/path", - "data": { - "Module.conv2.Conv2d.forward.0": { - "input_args": [ - { - "type": "torch.Tensor", - "dtype": "torch.float32", - "shape": [ - 8, - 16, - 14, - 14 - ], - "Max": 1.638758659362793, - "Min": 0.0, - "Mean": 0.2544615864753723, - "Norm": 70.50277709960938, - "requires_grad": true, - "data_name": "Module.conv2.Conv2d.forward.0.input.0.pt" - } - ], - "input_kwargs": {}, - "output": [ - { - "type": "torch.Tensor", - "dtype": "torch.float32", - "shape": [ - 8, - 32, - 10, - 10 - ], - "Max": 1.6815717220306396, - "Min": -1.5120246410369873, - "Mean": -0.025344856083393097, - "Norm": 149.65576171875, - "requires_grad": true, - "data_name": "Module.conv2.Conv2d.forward.0.output.0.pt" - } - ], - "parameters": { - "weight": { - "type": "torch.Tensor", - "dtype": "torch.float32", - "shape": [ - 32, - 16, - 5, - 5 - ], - "Max": 0.05992485210299492, - "Min": -0.05999220535159111, - "Mean": -0.0006165213999338448, - "Norm": 3.421217441558838, - "requires_grad": true, - "data_name": "Module.conv2.Conv2d.forward.0.parameters.weight.pt" - }, - "bias": { - "type": "torch.Tensor", - "dtype": "torch.float32", - "shape": [ - 32 - ], - "Max": 0.05744686722755432, - "Min": -0.04894155263900757, - "Mean": 0.006410328671336174, - "Norm": 0.17263513803482056, - "requires_grad": true, - "data_name": "Module.conv2.Conv2d.forward.0.parameters.bias.pt" - } - } - }, - "Module.conv2.Conv2d.parameters_grad": { - "weight": [ - { - "type": "torch.Tensor", - "dtype": "torch.float32", - "shape": [ - 32, - 16, - 5, - 5 - ], - "Max": 0.018550323322415352, - "Min": -0.008627401664853096, - "Mean": 0.0006675920449197292, - "Norm": 0.26084786653518677, - "requires_grad": false, - "data_name": "Module.conv2.Conv2d.parameters_grad.weight.pt" - } - ], - "bias": [ - { - "type": "torch.Tensor", - "dtype": "torch.float32", - "shape": [ - 32 - ], - "Max": 0.014914230443537235, - "Min": -0.006656786892563105, - "Mean": 0.002657240955159068, - "Norm": 0.029451673850417137, - "requires_grad": false, - "data_name": "Module.conv2.Conv2d.parameters_grad.bias.pt" - } - ] - }, - "Module.conv2.Conv2d.backward.0": { - "input": [ - { - "type": "torch.Tensor", - "dtype": "torch.float32", - "shape": [ - 8, - 32, - 10, - 10 - ], - "Max": 0.0015069986693561077, - "Min": -0.001139344065450132, - "Mean": 3.3215508210560074e-06, - "Norm": 0.020567523315548897, - "requires_grad": false, - "data_name": "Module.conv2.Conv2d.backward.0.input.0.pt" - } - ], - "output": [ - { - "type": "torch.Tensor", - "dtype": "torch.float32", - "shape": [ - 8, - 16, - 14, - 14 - ], - "Max": 0.0007466732058674097, - "Min": -0.00044813455315306783, - "Mean": 6.814070275140693e-06, - "Norm": 0.01474067009985447, - "requires_grad": false, - "data_name": "Module.conv2.Conv2d.backward.0.output.0.pt" - } - ] - } - } -} -``` - -
- -#### L1 level - -An L1 `dump.json` records forward/backward I/O for APIs. Using PyTorch's `relu` function as an -example (`output = torch.nn.functional.relu(input)`), the file contains: - -- `Functional.relu.0.forward`: Forward data of the API. `input_args` are positional inputs, `input_kwargs` are keyword - inputs, and `output` stores the forward outputs. -- `Functional.relu.0.backward`: Backward data of the API. `input` represents the gradients of the forward outputs, - and `output` represents the gradients that flow back to the forward inputs. - -
- -L1 dump.json - -```json -{ - "task": "tensor", - "level": "L1", - "framework": "pytorch", - "dump_data_dir": "/dump/path", - "data": { - "Functional.relu.0.forward": { - "input_args": [ - { - "type": "torch.Tensor", - "dtype": "torch.float32", - "shape": [ - 32, - 16, - 28, - 28 - ], - "Max": 1.3864083290100098, - "Min": -1.3364859819412231, - "Mean": 0.03711778670549393, - "Norm": 236.20692443847656, - "requires_grad": true, - "data_name": "Functional.relu.0.forward.input.0.pt" - } - ], - "input_kwargs": {}, - "output": [ - { - "type": "torch.Tensor", - "dtype": "torch.float32", - "shape": [ - 32, - 16, - 28, - 28 - ], - "Max": 1.3864083290100098, - "Min": 0.0, - "Mean": 0.16849493980407715, - "Norm": 175.23345947265625, - "requires_grad": true, - "data_name": "Functional.relu.0.forward.output.0.pt" - } - ] - }, - "Functional.relu.0.backward": { - "input": [ - { - "type": "torch.Tensor", - "dtype": "torch.float32", - "shape": [ - 32, - 16, - 28, - 28 - ], - "Max": 0.0001815402356442064, - "Min": -0.00013352684618439525, - "Mean": 0.00011915402356442064, - "Norm": 0.007598237134516239, - "requires_grad": false, - "data_name": "Functional.relu.0.backward.input.0.pt" - } - ], - "output": [ - { - "type": "torch.Tensor", - "dtype": "torch.float32", - "shape": [ - 32, - 16, - 28, - 28 - ], - "Max": 0.0001815402356442064, - "Min": -0.00012117840378778055, - "Mean": 2.0098118724831693e-08, - "Norm": 0.006532244384288788, - "requires_grad": false, - "data_name": "Functional.relu.0.backward.output.0.pt" - } - ] - } - } -} -``` - -
- -#### mix level - -A `mix` dump.json contains both L0 and L1 level data; the file format is the same as the examples above. diff --git a/docs/developer_guide/release_process.md b/docs/developer_guide/release_process.md deleted file mode 100644 index e817a256e..000000000 --- a/docs/developer_guide/release_process.md +++ /dev/null @@ -1,18 +0,0 @@ -# PyPI Package Release Process - -## Update the version in code -Update the package version in `python/pyproject.toml` and `python/sglang/__init__.py`. - -## Upload the PyPI package - -``` -pip install build twine -``` - -``` -cd python -bash upload_pypi.sh -``` - -## Make a release in GitHub -Make a new release https://github.com/sgl-project/sglang/releases/new. diff --git a/docs/developer_guide/setup_github_runner.md b/docs/developer_guide/setup_github_runner.md deleted file mode 100644 index 49221acc9..000000000 --- a/docs/developer_guide/setup_github_runner.md +++ /dev/null @@ -1,51 +0,0 @@ -# Set Up Self-Hosted Runners for GitHub Actions - -## Add a Runner - -### Step 1: Start a docker container. - -**You can mount a folder for the shared huggingface model weights cache. ** -The command below uses `/tmp/huggingface` as an example. - -``` -docker pull nvidia/cuda:12.9.1-devel-ubuntu22.04 -# Nvidia -docker run --shm-size 128g -it -v /tmp/huggingface:/hf_home --gpus all nvidia/cuda:12.9.1-devel-ubuntu22.04 /bin/bash -# AMD -docker run --rm --device=/dev/kfd --device=/dev/dri --group-add video --shm-size 128g -it -v /tmp/huggingface:/hf_home lmsysorg/sglang:v0.5.8-rocm700-mi30x /bin/bash -# AMD just the last 2 GPUs -docker run --rm --device=/dev/kfd --device=/dev/dri/renderD176 --device=/dev/dri/renderD184 --group-add video --shm-size 128g -it -v /tmp/huggingface:/hf_home lmsysorg/sglang:v0.5.8-rocm700-mi30x /bin/bash -``` - -### Step 2: Configure the runner by `config.sh` - -Run these commands inside the container. - -``` -apt update && apt install -y curl python3-pip git -pip install --upgrade pip -export RUNNER_ALLOW_RUNASROOT=1 -``` - -Then follow https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners/adding-self-hosted-runners to run `config.sh` - -**Notes** -- Do not need to specify the runner group -- Give it a name (e.g., `test-sgl-gpu-0`) and some labels (e.g., `1-gpu-h100`). The labels can be edited later in Github Settings. -- Do not need to change the work folder. - -### Step 3: Run the runner by `run.sh` - -- Set up environment variables -``` -export HF_HOME=/hf_home -export SGLANG_IS_IN_CI=true -export HF_TOKEN=hf_xxx -export OPENAI_API_KEY=sk-xxx -export CUDA_VISIBLE_DEVICES=0 -``` - -- Run it forever -``` -while true; do ./run.sh; echo "Restarting..."; sleep 2; done -``` diff --git a/docs/diffusion/api/cli.md b/docs/diffusion/api/cli.md deleted file mode 100644 index d91d1281a..000000000 --- a/docs/diffusion/api/cli.md +++ /dev/null @@ -1,274 +0,0 @@ -# SGLang Diffusion CLI - -Use the CLI for one-off generation with `sglang generate` or to start a persistent HTTP server with `sglang serve`. - -### Overlay repos for non-diffusers models - -If `--model-path` points to a supported non-diffusers source repo, SGLang can resolve it -through a self-hosted overlay repo. - -SGLang first checks a built-in overlay registry. Concrete built-in mappings can be added over time without changing the CLI surface. - -Override example: - -```bash -export SGLANG_DIFFUSION_MODEL_OVERLAY_REGISTRY='{ - "Wan-AI/Wan2.2-S2V-14B": { - "overlay_repo_id": "your-org/Wan2.2-S2V-14B-overlay", - "overlay_revision": "main" - } -}' - -sglang generate \ - --model-path Wan-AI/Wan2.2-S2V-14B \ - --config configs/wan_s2v.yaml -``` - -The overlay repo should be a complete diffusers-style/componentized repo - -You can also pass the overlay repo itself as `--model-path` if it contains `_overlay/overlay_manifest.json`. - -Notes: -1. `SGLANG_DIFFUSION_MODEL_OVERLAY_REGISTRY` is only an optional override for -development and debugging. It accepts either a JSON object or a path to a JSON -file, and can extend or replace built-in entries for the current process. -2. On the first load, SGLang will: - - download overlay metadata from the overlay repo - - download the required files from the original source repo - - materialize a local standard component repo under `~/.cache/sgl_diffusion/materialized_models/` -3. Later loads reuse the materialized local repo. The materialized repo is what the runtime loads as a normal componentized model directory. - - -## Quick Start - -### Generate - -```bash -sglang generate \ - --model-path Qwen/Qwen-Image \ - --prompt "A beautiful sunset over the mountains" \ - --save-output -``` - -### Serve - -```bash -sglang serve \ - --model-path Wan-AI/Wan2.1-T2V-1.3B-Diffusers \ - --num-gpus 4 \ - --ulysses-degree 2 \ - --ring-degree 2 \ - --port 30010 -``` - -For request and response examples, see [OpenAI-Compatible API](openai_api.md). - -```{tip} -Use `sglang generate --help` and `sglang serve --help` for the full argument list. The CLI help output is the source of truth for exhaustive flags. -``` - -## Common Options - -### Model and runtime - -- `--model-path {MODEL}`: model path or Hugging Face model ID -- `--lora-path {PATH}` and `--lora-nickname {NAME}`: load a LoRA adapter -- `--lora-merge-mode {auto|merge|dynamic}`: choose how LoRA is applied. `auto` statically merges regular weights and uses dynamic LoRA for FSDP-sharded weights to avoid full-gather peaks. -- `--num-gpus {N}`: number of GPUs to use -- `--performance-mode {manual|auto|speed|memory}` / `--mode`: preset for latency/throughput and memory defaults. `auto` is the default and keeps safe offload defaults, using FSDP only for validated DiT-offload replacement paths; use `manual` to keep performance-related server args under explicit user control. Explicit offload, FSDP, and parallelism flags take precedence in all modes. -- `--tp-size {N}`: tensor parallelism size, mainly for encoders -- `--sp-degree {N}`: sequence parallelism size -- `--ulysses-degree {N}` and `--ring-degree {N}`: USP parallelism controls -- `--enable-cfg-parallel {true|false}`: enable or explicitly disable CFG parallelism -- `--attention-backend {BACKEND}`: attention backend for native SGLang pipelines -- `--component-attention-backends {MAP}`: per-component attention backend overrides, for example `text_encoder=torch_sdpa,transformer=fa` -- `--attention-backend-config {CONFIG}`: attention backend configuration - -### Sampling and output - -- `--prompt {PROMPT}` and `--negative-prompt {PROMPT}` -- `--image-path {PATH} [{PATH} ...]`: input image(s) for image-to-video or image-to-image generation -- `--num-inference-steps {STEPS}` and `--seed {SEED}` -- `--height {HEIGHT}`, `--width {WIDTH}`, `--num-frames {N}`, `--fps {FPS}` -- `--output-path {PATH}`, `--output-file-name {NAME}`, `--save-output`, `--return-frames` - -For frame interpolation and upscaling, see [Post-Processing](post_processing.md). - -### Quantized transformers - -For quantized transformer checkpoints, prefer: - -- `--model-path` for the base pipeline -- `--transformer-path` for a quantized `transformers` transformer component folder -- `--transformer-weights-path` for a quantized safetensors file, directory, or repo -- `--quantization` for online quantization (apply quantization to unquantized models at load time, activations are quantized dynamically) -- `--quantization-ignored-layers` layer name patterns to keep unquantized (e.g. `attention.to_`) - -See [Quantization](../quantization.md) for supported quantization families and examples. - -## Configuration Files - -Use `--config` to load JSON or YAML configuration. Command-line flags override values from the config file. - -```bash -sglang generate --config config.yaml -``` - -Example: - -```yaml -model_path: FastVideo/FastHunyuan-diffusers -prompt: A beautiful woman in a red dress walking down a street -output_path: outputs/ -num_gpus: 2 -sp_size: 2 -tp_size: 1 -num_frames: 45 -height: 720 -width: 1280 -num_inference_steps: 6 -seed: 1024 -fps: 24 -precision: bf16 -vae_precision: fp16 -vae_tiling: true -vae_sp: true -enable_torch_compile: false -``` - -## Generate - -`sglang generate` runs a single generation job and exits when the job finishes. - -```bash -sglang generate \ - --model-path Wan-AI/Wan2.2-T2V-A14B-Diffusers \ - --text-encoder-cpu-offload \ - --pin-cpu-memory \ - --num-gpus 4 \ - --ulysses-degree 2 \ - --ring-degree 2 \ - --prompt "A curious raccoon" \ - --save-output \ - --output-path outputs \ - --output-file-name "a-curious-raccoon.mp4" -``` - -```{note} -HTTP server-only arguments are ignored by `sglang generate`. -``` - -For diffusers pipelines, Cache-DiT can be enabled with `SGLANG_CACHE_DIT_ENABLED=true` or `--cache-dit-config`. See [Cache-DiT](../performance/cache/cache_dit.md). - -### Layerwise Offload - -Use layerwise offload when a large component does not fit comfortably in GPU memory. By default, `--dit-layerwise-offload` only applies to legacy DiT components. Use `--layerwise-offload-components` to select pipeline component names explicitly (`--layerwise-offload-modules` is accepted as an alias): - -```bash -sglang generate \ - --model-path Wan-AI/Wan2.2-T2V-A14B-Diffusers \ - --dit-layerwise-offload \ - --layerwise-offload-components transformer text_encoder \ - --dit-offload-prefetch-size 0 \ - --prompt "A quiet city street after rain" -``` - -The values must match keys in the selected pipeline's `pipeline.modules`, such as `transformer`, `text_encoder`, `image_encoder`, `vae`, `condition_image_encoder`, `spatial_upsampler`, or `vocoder`. Use `all` to select every layerwise-offloadable component. Prefer the smallest component set that solves the memory issue because layerwise offload can increase latency. - -## Serve - -`sglang serve` starts the HTTP server and keeps the model loaded for repeated requests. - -```bash -sglang serve \ - --model-path Wan-AI/Wan2.1-T2V-1.3B-Diffusers \ - --text-encoder-cpu-offload \ - --pin-cpu-memory \ - --num-gpus 4 \ - --ulysses-degree 2 \ - --ring-degree 2 \ - --port 30010 -``` - -### Cloud Storage - -SGLang Diffusion can upload generated images and videos to S3-compatible object storage after generation. - -```bash -export SGLANG_CLOUD_STORAGE_TYPE=s3 -export SGLANG_S3_BUCKET_NAME=my-bucket -export SGLANG_S3_ACCESS_KEY_ID=your-access-key -export SGLANG_S3_SECRET_ACCESS_KEY=your-secret-key -export SGLANG_S3_ENDPOINT_URL=https://minio.example.com -``` - -See [Environment Variables](../environment_variables.md) for the full set of storage options. - -## Component Path Overrides - -Override individual pipeline components such as `vae`, `transformer`, or `text_encoder` with `---path`. - -```bash -sglang serve \ - --model-path black-forest-labs/FLUX.2-dev \ - --vae-path fal/FLUX.2-Tiny-AutoEncoder -``` - -The component key must match the key in the model's `model_index.json`, and the path must be either a Hugging Face repo ID or a complete component directory. - -## Component Attention Backend Overrides - -Use `--component-attention-backends` when one pipeline component needs a different native attention backend from the global `--attention-backend`. - -```bash -sglang generate \ - --model-path Lightricks/LTX-2.3 \ - --attention-backend fa \ - --component-attention-backends text_encoder=torch_sdpa -``` - -The component key must match a pipeline module key such as `text_encoder`, `text_encoder_2`, `transformer`, `transformer_2`, or `connectors`. Component overrides take precedence over the global `--attention-backend` only while that component is being constructed. - -You can also pass dotted CLI entries: - -```bash -sglang generate \ - --model-path \ - --component-attention-backends.text_encoder torch_sdpa \ - --component-attention-backends.transformer fa -``` - -## Diffusers Backend - -Use `--backend diffusers` to force vanilla diffusers pipelines when no native SGLang implementation exists or when a model requires a custom pipeline class. - -### Key Options - -| Argument | Values | Description | -|----------|--------|-------------| -| `--backend` | `auto`, `sglang`, `diffusers` | Choose native SGLang, force native, or force diffusers | -| `--diffusers-attention-backend` | `flash`, `_flash_3_hub`, `sage`, `xformers`, `native` | Attention backend for diffusers pipelines | -| `--trust-remote-code` | flag | Required for models with custom pipeline classes | -| `--vae-tiling` and `--vae-slicing` | flag | Lower memory usage for VAE decode | -| `--dit-precision` and `--vae-precision` | `fp16`, `bf16`, `fp32` | Precision controls | -| `--enable-torch-compile` | flag | Enable `torch.compile` | -| `--cache-dit-config` | `{PATH}` | Cache-DiT config for diffusers pipelines | - -### Example - -```bash -sglang generate \ - --model-path AIDC-AI/Ovis-Image-7B \ - --backend diffusers \ - --trust-remote-code \ - --diffusers-attention-backend flash \ - --prompt "A serene Japanese garden with cherry blossoms" \ - --height 1024 \ - --width 1024 \ - --num-inference-steps 30 \ - --save-output \ - --output-path outputs \ - --output-file-name ovis_garden.png -``` - -For pipeline-specific arguments not exposed in the CLI, pass `diffusers_kwargs` in a config file. diff --git a/docs/diffusion/api/openai_api.md b/docs/diffusion/api/openai_api.md deleted file mode 100644 index 99bbea056..000000000 --- a/docs/diffusion/api/openai_api.md +++ /dev/null @@ -1,449 +0,0 @@ -# SGLang Diffusion OpenAI API - -The SGLang diffusion HTTP server implements an OpenAI-compatible API for image and video generation, as well as LoRA adapter management. - -## Prerequisites - -- Python 3.11+ if you plan to use the OpenAI Python SDK. - -## Serve - -Launch the server using the `sglang serve` command. - -### Start the server - -```bash -SERVER_ARGS=( - --model-path Wan-AI/Wan2.1-T2V-1.3B-Diffusers - --text-encoder-cpu-offload - --pin-cpu-memory - --num-gpus 4 - --ulysses-degree=2 - --ring-degree=2 - --port 30010 -) - -sglang serve "${SERVER_ARGS[@]}" -``` - -- **--model-path**: Path to the model or model ID. -- **--port**: HTTP port to listen on (default: `30000`). - -**Get Model Information** - -**Endpoint:** `GET /models` - -Returns information about the model served by this server, including model path, task type, pipeline configuration, and precision settings. - -**Curl Example:** - -```bash -curl -sS -X GET "http://localhost:30010/models" -``` - -**Response Example:** - -```json -{ - "model_path": "Wan-AI/Wan2.1-T2V-1.3B-Diffusers", - "task_type": "T2V", - "pipeline_name": "wan_pipeline", - "pipeline_class": "WanPipeline", - "num_gpus": 4, - "dit_precision": "bf16", - "vae_precision": "fp16" -} -``` - ---- - -## Endpoints - -### Image Generation - -The server implements an OpenAI-compatible Images API under the `/v1/images` namespace. - -**Create an image** - -**Endpoint:** `POST /v1/images/generations` - -**Python Example (b64_json response):** - -```python -import base64 -from openai import OpenAI - -client = OpenAI(api_key="sk-proj-1234567890", base_url="http://localhost:30010/v1") - -img = client.images.generate( - prompt="A calico cat playing a piano on stage", - size="1024x1024", - n=1, - response_format="b64_json", -) - -image_bytes = base64.b64decode(img.data[0].b64_json) -with open("output.png", "wb") as f: - f.write(image_bytes) -``` - -**Curl Example:** - -```bash -curl -sS -X POST "http://localhost:30010/v1/images/generations" \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer sk-proj-1234567890" \ - -d '{ - "prompt": "A calico cat playing a piano on stage", - "size": "1024x1024", - "n": 1, - "response_format": "b64_json" - }' -``` - -> **Note** -> If `response_format=url` is used and cloud storage is not configured, the API returns -> a relative URL like `/v1/images//content`. - -**Edit an image** - -**Endpoint:** `POST /v1/images/edits` - -This endpoint accepts a multipart form upload with input images and a text prompt. The server can return either a base64-encoded image or a URL to download the image. - -**Curl Example (b64_json response):** - -```bash -curl -sS -X POST "http://localhost:30010/v1/images/edits" \ - -H "Authorization: Bearer sk-proj-1234567890" \ - -F "image=@local_input_image.png" \ - -F "url=image_url.jpg" \ - -F "prompt=A calico cat playing a piano on stage" \ - -F "size=1024x1024" \ - -F "response_format=b64_json" -``` - -**Curl Example (URL response):** - -```bash -curl -sS -X POST "http://localhost:30010/v1/images/edits" \ - -H "Authorization: Bearer sk-proj-1234567890" \ - -F "image=@local_input_image.png" \ - -F "url=image_url.jpg" \ - -F "prompt=A calico cat playing a piano on stage" \ - -F "size=1024x1024" \ - -F "response_format=url" -``` - -**Download image content** - -When `response_format=url` is used with `POST /v1/images/generations` or `POST /v1/images/edits`, -the API returns a relative URL like `/v1/images//content`. - -**Endpoint:** `GET /v1/images/{image_id}/content` - -**Curl Example:** - -```bash -curl -sS -L "http://localhost:30010/v1/images//content" \ - -H "Authorization: Bearer sk-proj-1234567890" \ - -o output.png -``` - -### Video Generation - -The server implements a subset of the OpenAI Videos API under the `/v1/videos` namespace. - -**Create a video (text-to-video)** - -**Endpoint:** `POST /v1/videos` - -**Python Example:** - -```python -from openai import OpenAI - -client = OpenAI(api_key="sk-proj-1234567890", base_url="http://localhost:30010/v1") - -video = client.videos.create( - prompt="A calico cat playing a piano on stage", - size="1280x720" -) -print(f"Video ID: {video.id}, Status: {video.status}") -``` - -**Curl Example:** - -```bash -curl -sS -X POST "http://localhost:30010/v1/videos" \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer sk-proj-1234567890" \ - -d '{ - "prompt": "A calico cat playing a piano on stage", - "size": "1280x720" - }' -``` - -**Create a video (image-to-video)** - -For I2V or TI2V models (e.g., Wan2.1 I2V, LTX-2.3 two-stage), pass an input image via multipart form upload or a reference URL. - -**Curl Example (multipart form upload):** - -```bash -curl -sS -X POST "http://localhost:30010/v1/videos" \ - -H "Authorization: Bearer sk-proj-1234567890" \ - -F "prompt=A cat playing a piano" \ - -F "input_reference=@input_image.png" \ - -F "size=1280x720" -``` - -**Curl Example (reference URL):** - -```bash -curl -sS -X POST "http://localhost:30010/v1/videos" \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer sk-proj-1234567890" \ - -d '{ - "prompt": "A cat playing a piano", - "reference_url": "https://example.com/input_image.png", - "size": "1280x720" - }' -``` - -**List videos** - -**Endpoint:** `GET /v1/videos` - -**Python Example:** - -```python -videos = client.videos.list() -for item in videos.data: - print(item.id, item.status) -``` - -**Curl Example:** - -```bash -curl -sS -X GET "http://localhost:30010/v1/videos" \ - -H "Authorization: Bearer sk-proj-1234567890" -``` - -**Download video content** - -**Endpoint:** `GET /v1/videos/{video_id}/content` - -**Python Example:** - -```python -import time - -# Poll for completion -while True: - page = client.videos.list() - item = next((v for v in page.data if v.id == video_id), None) - if item and item.status == "completed": - break - time.sleep(5) - -# Download content -resp = client.videos.download_content(video_id=video_id) -with open("output.mp4", "wb") as f: - f.write(resp.read()) -``` - -**Curl Example:** - -```bash -curl -sS -L "http://localhost:30010/v1/videos//content" \ - -H "Authorization: Bearer sk-proj-1234567890" \ - -o output.mp4 -``` - ---- - -### LoRA Management - -The server supports dynamic loading, merging, and unmerging of LoRA adapters. - -**Important Notes:** -- Mutual Exclusion: Only one LoRA configuration can be active per target at a time -- Switching: To switch LoRAs, deactivate the current LoRA with `unmerge_lora_weights`, then `set` the new one -- Caching: The server caches loaded LoRA weights in memory. Switching back to a previously loaded LoRA (same path) has little cost - -**Set LoRA Adapter** - -Loads one or more LoRA adapters and applies them to the model. By default, regular weights are statically merged, while FSDP-sharded weights use dynamic LoRA to avoid full-gather memory peaks. - -**Endpoint:** `POST /v1/set_lora` - -**Parameters:** -- `lora_nickname` (string or list of strings, required): A unique identifier for the LoRA adapter(s). Can be a single string or a list of strings for multiple LoRAs -- `lora_path` (string or list of strings/None, optional): Path to the `.safetensors` file(s) or Hugging Face repo ID(s). Required for the first load; optional if re-activating a cached nickname. If a list, must match the length of `lora_nickname` -- `target` (string or list of strings, optional): Which transformer(s) to apply the LoRA to. If a list, must match the length of `lora_nickname`. Valid values: - - `"all"` (default): Apply to all transformers - - `"transformer"`: Apply only to the primary transformer (high noise for Wan2.2) - - `"transformer_2"`: Apply only to transformer_2 (low noise for Wan2.2) - - `"critic"`: Apply only to the critic model -- `strength` (float or list of floats, optional): LoRA strength for merge, default 1.0. If a list, must match the length of `lora_nickname`. Values < 1.0 reduce the effect, values > 1.0 amplify the effect -- `merge_mode` (string, optional): `"auto"` (default server policy), `"merge"` (force static merge), or `"dynamic"` (apply LoRA at forward time) - -**Single LoRA Example:** - -```bash -curl -X POST http://localhost:30010/v1/set_lora \ - -H "Content-Type: application/json" \ - -d '{ - "lora_nickname": "lora_name", - "lora_path": "/path/to/lora.safetensors", - "target": "all", - "strength": 0.8 - }' -``` - -**Multiple LoRA Example:** - -```bash -curl -X POST http://localhost:30010/v1/set_lora \ - -H "Content-Type: application/json" \ - -d '{ - "lora_nickname": ["lora_1", "lora_2"], - "lora_path": ["/path/to/lora1.safetensors", "/path/to/lora2.safetensors"], - "target": ["transformer", "transformer_2"], - "strength": [0.8, 1.0] - }' -``` - -**Multiple LoRA with Same Target:** - -```bash -curl -X POST http://localhost:30010/v1/set_lora \ - -H "Content-Type: application/json" \ - -d '{ - "lora_nickname": ["style_lora", "character_lora"], - "lora_path": ["/path/to/style.safetensors", "/path/to/character.safetensors"], - "target": "all", - "strength": [0.7, 0.9] - }' -``` - -> [!NOTE] -> When using multiple LoRAs: -> - All list parameters (`lora_nickname`, `lora_path`, `target`, `strength`) must have the same length -> - If `target` or `strength` is a single value, it will be applied to all LoRAs -> - Multiple LoRAs applied to the same target are applied in order - - -**Merge LoRA Weights** - -Manually merges the currently set LoRA weights into the base model. - -> [!NOTE] -> With FSDP-sharded weights, manual merge may require a full-gather and can OOM. Use `set_lora` with `merge_mode="auto"` or `"dynamic"` for the lower-peak path. - -**Endpoint:** `POST /v1/merge_lora_weights` - -**Parameters:** -- `target` (string, optional): Which transformer(s) to merge. One of "all" (default), "transformer", "transformer_2", "critic" -- `strength` (float, optional): LoRA strength for merge, default 1.0. Values < 1.0 reduce the effect, values > 1.0 amplify the effect - -**Curl Example:** - -```bash -curl -X POST http://localhost:30010/v1/merge_lora_weights \ - -H "Content-Type: application/json" \ - -d '{"strength": 0.8}' -``` - - -**Unmerge LoRA Weights** - -Unmerges the currently active LoRA weights from the base model, restoring it to its original state. This **must** be called before setting a different LoRA. - -**Endpoint:** `POST /v1/unmerge_lora_weights` - -**Curl Example:** - -```bash -curl -X POST http://localhost:30010/v1/unmerge_lora_weights \ - -H "Content-Type: application/json" -``` - -**List LoRA Adapters** - -Returns loaded LoRA adapters and current application status per module. - -**Endpoint:** `GET /v1/list_loras` - -**Curl Example:** - -```bash -curl -sS -X GET "http://localhost:30010/v1/list_loras" -``` - -**Response Example:** - -```json -{ - "loaded_adapters": [ - { "nickname": "lora_a", "path": "/weights/lora_a.safetensors" }, - { "nickname": "lora_b", "path": "/weights/lora_b.safetensors" } - ], - "active": { - "transformer": [ - { - "nickname": "lora2", - "path": "tarn59/pixel_art_style_lora_z_image_turbo", - "merged": true, - "mode": "merged", - "strength": 1.0 - } - ] - } -} -``` - -Notes: -- If LoRA is not enabled for the current pipeline, the server will return an error. -- `num_lora_layers_with_weights` counts only layers that have LoRA weights applied for the active adapter. - -### Example: Switching LoRAs - -1. Set LoRA A: - ```bash - curl -X POST http://localhost:30010/v1/set_lora -d '{"lora_nickname": "lora_a", "lora_path": "path/to/A"}' - ``` -2. Generate with LoRA A... -3. Unmerge LoRA A: - ```bash - curl -X POST http://localhost:30010/v1/unmerge_lora_weights - ``` -4. Set LoRA B: - ```bash - curl -X POST http://localhost:30010/v1/set_lora -d '{"lora_nickname": "lora_b", "lora_path": "path/to/B"}' - ``` -5. Generate with LoRA B... - -### Adjust Output Quality - -The server supports adjusting output quality and compression levels for both image and video generation through the `output-quality` and `output-compression` parameters. - -#### Parameters - -- **`output-quality`** (string, optional): Preset quality level that automatically sets compression. **Default is `"default"`**. Valid values: - - `"maximum"`: Highest quality (100) - - `"high"`: High quality (90) - - `"medium"`: Medium quality (55) - - `"low"`: Lower quality (35) - - `"default"`: Auto-adjust based on media type (50 for video, 75 for image) - -- **`output-compression`** (integer, optional): Direct compression level override (0-100). **Default is `None`**. When provided (not `None`), takes precedence over `output-quality`. - - `0`: Lowest quality, smallest file size - - `100`: Highest quality, largest file size - -#### Notes - -- **Precedence**: When both `output-quality` and `output-compression` are provided, `output-compression` takes precedence -- **Format Support**: Quality settings apply to JPEG, and video formats. PNG uses lossless compression and ignores these settings -- **File Size vs Quality**: Lower compression values (or "low" quality preset) produce smaller files but may show visible artifacts diff --git a/docs/diffusion/api/post_processing.md b/docs/diffusion/api/post_processing.md deleted file mode 100644 index d832f4af2..000000000 --- a/docs/diffusion/api/post_processing.md +++ /dev/null @@ -1,148 +0,0 @@ -# Post-Processing - -SGLang diffusion supports optional post-processing steps that run after -generation to improve temporal smoothness (frame interpolation) or spatial -resolution (upscaling). These steps are independent of the diffusion model and -can be combined in a single run. - -When both are enabled, **frame interpolation runs first** (increasing the frame -count), then **upscaling runs on every frame** (increasing the spatial -resolution). - ---- - -## Frame Interpolation (video only) - -Frame interpolation synthesizes new frames between each pair of consecutive -generated frames, producing smoother motion without re-running the diffusion -model. - -The `--frame-interpolation-exp` flag controls how many rounds of interpolation -to apply: each round inserts one new frame into every gap between adjacent -frames, so the output frame count follows the formula: - -> **(N − 1) × 2^exp + 1** -> -> e.g. 5 original frames with `exp=1` → 4 gaps × 1 new frame + 5 originals = **9** frames; -> with `exp=2` → **17** frames. - -### CLI Arguments - -| Argument | Description | -|----------|-------------| -| `--enable-frame-interpolation` | Enable frame interpolation. Model weights are downloaded automatically on first use. | -| `--frame-interpolation-exp {EXP}` | Interpolation exponent — `1` = 2× temporal resolution, `2` = 4×, etc. (default: `1`) | -| `--frame-interpolation-scale {SCALE}` | RIFE inference scale; use `0.5` for high-resolution inputs to save memory (default: `1.0`) | -| `--frame-interpolation-model-path {PATH}` | Local directory or HuggingFace repo ID containing RIFE `flownet.pkl` weights (default: `elfgum/RIFE-4.22.lite`, downloaded automatically) | - -### Supported Models - -Frame interpolation uses the [RIFE](https://github.com/hzwer/Practical-RIFE) -(Real-Time Intermediate Flow Estimation) architecture. Only **RIFE 4.22.lite** -(`IFNet` with 4-scale `IFBlock` backbone) is supported. The network topology is -hard-coded, so custom weights provided via `--frame-interpolation-model-path` -must be a `flownet.pkl` checkpoint that is compatible with this architecture. - -Other RIFE versions (e.g., older `v4.x` variants with different block counts) -or entirely different frame interpolation methods (FILM, AMT, etc.) are **not -supported**. - -| Weight | HuggingFace Repo | Description | -|--------|------------------|-------------| -| RIFE 4.22.lite *(default)* | [`elfgum/RIFE-4.22.lite`](https://huggingface.co/elfgum/RIFE-4.22.lite) | Lightweight model, downloaded automatically on first use | - -### Example - -Generate a 5-frame video and interpolate to 9 frames ((5 − 1) × 2¹ + 1 = 9): - -```bash -sglang generate \ - --model-path Wan-AI/Wan2.2-T2V-A14B-Diffusers \ - --prompt "A dog running through a park" \ - --num-frames 5 \ - --enable-frame-interpolation \ - --frame-interpolation-exp 1 \ - --save-output -``` - ---- - -## Upscaling (image and video) - -Upscaling increases the spatial resolution of generated images or video frames -using [Real-ESRGAN](https://github.com/xinntao/Real-ESRGAN). The model weights -are downloaded automatically on first use and cached for subsequent runs. - -### CLI Arguments - -| Argument | Description | -|----------|-------------| -| `--enable-upscaling` | Enable post-generation upscaling using Real-ESRGAN. | -| `--upscaling-scale {SCALE}` | Desired upscaling factor (default: `4`). The 4× model is used internally; if a different scale is requested, a bicubic resize is applied after the network output. | -| `--upscaling-model-path {PATH}` | Local `.pth` file, HuggingFace repo ID, or `repo_id:filename` for Real-ESRGAN weights (default: `ai-forever/Real-ESRGAN` with `RealESRGAN_x4.pth`, downloaded automatically). Use the `repo_id:filename` format to specify a custom weight file from a HuggingFace repo (e.g. `my-org/my-esrgan:weights.pth`). | - -### Supported Models - -Upscaling supports two Real-ESRGAN network architectures. The correct -architecture is **auto-detected** from the checkpoint keys, so you only need to -point `--upscaling-model-path` at a valid `.pth` file: - -| Architecture | Example Weights | Description | -|--------------|-----------------|-------------| -| **RRDBNet** | `RealESRGAN_x4plus.pth` | Heavier model with higher quality; best for photos | -| **SRVGGNetCompact** | `RealESRGAN_x4.pth` *(default)*, `realesr-animevideov3.pth`, `realesr-general-x4v3.pth` | Lightweight model; faster inference, good for video | - -The default weight file is -[`ai-forever/Real-ESRGAN`](https://huggingface.co/ai-forever/Real-ESRGAN) with -`RealESRGAN_x4.pth` (SRVGGNetCompact, 4× native scale). - -Other super-resolution models (e.g., SwinIR, HAT, BSRGAN) are **not supported** -— only Real-ESRGAN checkpoints using the two architectures above are -compatible. - -### Examples - -Generate a 1024×1024 image and upscale to 4096×4096: - -```bash -sglang generate \ - --model-path black-forest-labs/FLUX.2-dev \ - --prompt "A cat sitting on a windowsill" \ - --output-size 1024x1024 \ - --enable-upscaling \ - --save-output -``` - -Generate a video and upscale each frame by 4×: - -```bash -sglang generate \ - --model-path Wan-AI/Wan2.1-T2V-1.3B-Diffusers \ - --prompt "A curious raccoon" \ - --enable-upscaling \ - --upscaling-scale 4 \ - --save-output -``` - ---- - -## Combining Frame Interpolation and Upscaling - -Frame interpolation and upscaling can be combined in a single run. -Interpolation is applied first (increasing the frame count), then upscaling is -applied to every frame (increasing the spatial resolution). - -Example — generate 5 frames, interpolate to 9 frames, and upscale each frame -by 4×: - -```bash -sglang generate \ - --model-path Wan-AI/Wan2.1-T2V-1.3B-Diffusers \ - --prompt "A curious raccoon" \ - --num-frames 5 \ - --enable-frame-interpolation \ - --frame-interpolation-exp 1 \ - --enable-upscaling \ - --upscaling-scale 4 \ - --save-output -``` diff --git a/docs/diffusion/ci_perf.md b/docs/diffusion/ci_perf.md deleted file mode 100644 index f8bb2316b..000000000 --- a/docs/diffusion/ci_perf.md +++ /dev/null @@ -1,31 +0,0 @@ -# CI Performance - -## Perf Baseline Generation Script - -`python/sglang/multimodal_gen/test/scripts/gen_perf_baselines.py` starts a local diffusion server, issues requests for selected test cases, aggregates stage/denoise-step/E2E timings from the perf log, and writes the results back to the `scenarios` section of `perf_baselines.json`. - -### Usage - -Update a single case: - -```bash -python python/sglang/multimodal_gen/test/scripts/gen_perf_baselines.py --case qwen_image_t2i -``` - -Select by regex: - -```bash -python python/sglang/multimodal_gen/test/scripts/gen_perf_baselines.py --match 'qwen_image_.*' -``` - -Run all keys from the baseline file `scenarios`: - -```bash -python python/sglang/multimodal_gen/test/scripts/gen_perf_baselines.py --all-from-baseline -``` - -Specify input/output paths and timeout: - -```bash -python python/sglang/multimodal_gen/test/scripts/gen_perf_baselines.py --baseline python/sglang/multimodal_gen/test/server/perf_baselines.json --out /tmp/perf_baselines.json --timeout 600 -``` diff --git a/docs/diffusion/compatibility_matrix.md b/docs/diffusion/compatibility_matrix.md deleted file mode 100644 index 7f760575a..000000000 --- a/docs/diffusion/compatibility_matrix.md +++ /dev/null @@ -1,200 +0,0 @@ -# Compatibility Matrix - -The table below shows every supported model and the optimizations supported for them. - -The symbols used have the following meanings: - -- ✅ = Full compatibility -- ❌ = No compatibility -- ⭕ = Does not apply to this model - -## Models x Optimization - -The `HuggingFace Model ID` can be passed directly to `from_pretrained()` methods, and sglang-diffusion will use the -optimal -default parameters when initializing and generating videos. - -### Video Generation Models - -| Model Name | Hugging Face Model ID | Resolutions | TeaCache | Sliding Tile Attn | Sage Attn | Video Sparse Attention (VSA) | Sparse Linear Attention (SLA) | Sage Sparse Linear Attention (SageSLA) | Sparse Video Gen 2 (SVG2) | Laser Attn | Block Sparse Attn | Rain Fusion Attn | -|:-----------------------------|:--------------------------------------------------|:---------------------|:--------:|:-----------------:|:---------:|:----------------------------:|:-----------------------------:|:--------------------------------------:|:-------------------------:|:-------------------------:|:-------------------------:|:-------------------------:| -| FastWan2.1 T2V 1.3B | `FastVideo/FastWan2.1-T2V-1.3B-Diffusers` | 480p | ⭕ | ⭕ | ⭕ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| FastWan2.2 TI2V 5B Full Attn | `FastVideo/FastWan2.2-TI2V-5B-FullAttn-Diffusers` | 720p | ⭕ | ⭕ | ⭕ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| Wan2.2 TI2V 5B | `Wan-AI/Wan2.2-TI2V-5B-Diffusers` | 720p | ⭕ | ⭕ | ✅ | ⭕ | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ | -| Wan2.2 T2V A14B | `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | 480p
720p | ❌ | ❌ | ✅ | ⭕ | ✅ | ❌ | ✅ | -| Wan2.2 I2V A14B | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | 480p
720p | ❌ | ❌ | ✅ | ⭕ | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ | -| HunyuanVideo | `hunyuanvideo-community/HunyuanVideo` | 720×1280
544×960 | ❌ | ✅ | ✅ | ⭕ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | -| FastHunyuan | `FastVideo/FastHunyuan-diffusers` | 720×1280
544×960 | ❌ | ✅ | ✅ | ⭕ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | -| Wan2.1 T2V 1.3B | `Wan-AI/Wan2.1-T2V-1.3B-Diffusers` | 480p | ✅ | ✅ | ✅ | ⭕ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | -| Wan2.1 T2V 14B | `Wan-AI/Wan2.1-T2V-14B-Diffusers` | 480p, 720p | ✅ | ✅ | ✅ | ⭕ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | -| Wan2.1 I2V 480P | `Wan-AI/Wan2.1-I2V-14B-480P-Diffusers` | 480p | ✅ | ✅ | ✅ | ⭕ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | -| Wan2.1 I2V 720P | `Wan-AI/Wan2.1-I2V-14B-720P-Diffusers` | 720p | ✅ | ✅ | ✅ | ⭕ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | -| TurboWan2.1 T2V 1.3B | `IPostYellow/TurboWan2.1-T2V-1.3B-Diffusers` | 480p | ✅ | ❌ | ❌ | ❌ | ✅ | ✅ | ⭕ | -| TurboWan2.1 T2V 14B | `IPostYellow/TurboWan2.1-T2V-14B-Diffusers` | 480p | ✅ | ❌ | ❌ | ❌ | ✅ | ✅ | ⭕ | ❌ | ❌ | ❌ | -| TurboWan2.1 T2V 14B 720P | `IPostYellow/TurboWan2.1-T2V-14B-720P-Diffusers` | 720p | ✅ | ❌ | ❌ | ❌ | ✅ | ✅ | ⭕ | -| TurboWan2.2 I2V A14B | `IPostYellow/TurboWan2.2-I2V-A14B-Diffusers` | 720p | ✅ | ❌ | ❌ | ❌ | ✅ | ✅ | ⭕ | -| Wan2.1 Fun 1.3B InP | `weizhou03/Wan2.1-Fun-1.3B-InP-Diffusers` | 480p | ✅ | ✅ | ✅ | ⭕ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | -| Helios Base | `BestWishYsh/Helios-Base` | 720p | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| Helios Mid | `BestWishYsh/Helios-Mid` | 720p | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| Helios Distilled | `BestWishYsh/Helios-Distilled` | 720p | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| LTX-2 (one/two-stage/TI2V) | `Lightricks/LTX-2` | 768×512
1536×1024 | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| LTX-2.3 (one/two-stage/TI2V/HQ) | `Lightricks/LTX-2.3` | 768×512
1536×1024
1920×1088 (HQ default) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| Cosmos3-Nano (T2V / I2V / T2I) | `nvidia/Cosmos3-Nano` | 720p · 480p
1024×1024 (T2I) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| Cosmos3-Super (T2V / I2V / T2I) | `nvidia/Cosmos3-Super` | 720p · 480p
1024×1024 (T2I) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | - -**Note**: - -1. Wan2.2 TI2V 5B has some quality issues when performing I2V generation. We are working on fixing this issue. -2. SageSLA is based on SpargeAttn. Install it first with `pip install git+https://github.com/thu-ml/SpargeAttn.git --no-build-isolation` -3. LTX pipeline selection: - - One-stage: `--pipeline-class-name LTX2Pipeline` - - Two-stage: `--pipeline-class-name LTX2TwoStagePipeline` - - Two-stage HQ: `--pipeline-class-name LTX2TwoStageHQPipeline` (HQ defaults to 1920×1088; you can still override `--width/--height`) - - LTX-2 and LTX-2.3 support both T2V and TI2V (`--image-path`) on one-stage and two-stage pipelines (including HQ). - - The spatial upsampler and distilled LoRA are auto-resolved from the model snapshot by default, and can still be overridden with `--spatial-upsampler-path` and `--distilled-lora-path`. - - For LTX models, the `Resolutions` column uses output video `width×height` semantics, matching `sglang generate --width ... --height ...`. -4. LTX-2 / LTX-2.3 two-stage also supports `--ltx2-two-stage-device-mode {original,snapshot,resident}`: - - `snapshot` is the default and recommended mode. - - `resident` usually provides the best latency/throughput but uses much more VRAM. - - `original` keeps official two-stage semantics without the premerged stage-2 transformer path. - - Example (one prior run): `original` `154.67s`, `snapshot` `114.05s`, `resident` `75.71s`; peak VRAM trend is `original < snapshot < resident`. -5. Cosmos3 ships in two sizes — `nvidia/Cosmos3-Nano` (8B) and - `nvidia/Cosmos3-Super` (32B). Both share the same pipeline; the only - difference is transformer depth and width, picked up from - `transformer/config.json` at load time. A single checkpoint serves T2V, - I2V (`--image-path`), and T2I (`--num-frames 1`). - -### Image Generation Models - -| Model Name | HuggingFace Model ID | -|:--------------------------|:---------------------------------------------------------| -| FLUX.1-dev | `black-forest-labs/FLUX.1-dev` | -| FLUX.2-dev | `black-forest-labs/FLUX.2-dev` | -| FLUX.2-dev-NVFP4 | `black-forest-labs/FLUX.2-dev-NVFP4` | -| FLUX.2-Klein-4B | `black-forest-labs/FLUX.2-klein-4B` | -| FLUX.2-Klein-9B | `black-forest-labs/FLUX.2-klein-9B` | -| Z-Image | `Tongyi-MAI/Z-Image` | -| Z-Image-Turbo | `Tongyi-MAI/Z-Image-Turbo` | -| GLM-Image | `zai-org/GLM-Image` | -| Qwen Image | `Qwen/Qwen-Image` | -| Qwen Image 2512 | `Qwen/Qwen-Image-2512` | -| Qwen Image Edit | `Qwen/Qwen-Image-Edit` | -| Qwen Image Edit 2509 | `Qwen/Qwen-Image-Edit-2509` | -| Qwen Image Edit 2511 | `Qwen/Qwen-Image-Edit-2511` | -| Qwen Image Layered | `Qwen/Qwen-Image-Layered` | -| SD3 Medium | `stabilityai/stable-diffusion-3-medium-diffusers` | -| SD3.5 Medium | `stabilityai/stable-diffusion-3.5-medium-diffusers` | -| SD3.5 Large | `stabilityai/stable-diffusion-3.5-large-diffusers` | -| Hunyuan3D-2 | `tencent/Hunyuan3D-2` | -| SANA 1.5 1.6B | `Efficient-Large-Model/SANA1.5_1.6B_1024px_diffusers` | -| SANA 1.5 4.8B | `Efficient-Large-Model/SANA1.5_4.8B_1024px_diffusers` | -| SANA 1600M 1024px | `Efficient-Large-Model/Sana_1600M_1024px_diffusers` | -| SANA 600M 1024px | `Efficient-Large-Model/Sana_600M_1024px_diffusers` | -| SANA 1600M 512px | `Efficient-Large-Model/Sana_1600M_512px_diffusers` | -| SANA 600M 512px | `Efficient-Large-Model/Sana_600M_512px_diffusers` | -| FireRed-Image-Edit 1.0 | `FireRedTeam/FireRed-Image-Edit-1.0` | -| FireRed-Image-Edit 1.1 | `FireRedTeam/FireRed-Image-Edit-1.1` | -| ERNIE-Image | `baidu/ERNIE-Image` | -| ERNIE-Image-Turbo | `baidu/ERNIE-Image-Turbo` | - -## Supported Components - -SGLang Diffusion supports overriding individual pipeline components with -`---path`. The value can be either a Hugging Face repo ID or a local -component directory. - -The same overrides can also be provided in config files through -`component_paths.`. - -### Common Syntax - -CLI: - -```bash -sglang generate \ - --model-path black-forest-labs/FLUX.2-dev \ - --vae-path black-forest-labs/FLUX.2-small-decoder \ - --transformer-path /models/flux2/transformer -``` - -Config file: - -```yaml -model_path: black-forest-labs/FLUX.2-dev -component_paths: - vae: black-forest-labs/FLUX.2-small-decoder - transformer: /models/flux2/transformer -``` - -Use the component name from the pipeline's `model_index.json` or the native pipeline's registered module name: - -| Component Type | Supported Keys | Notes | -|:------------------|:---------------------------------------------------------------------------------------------------------------------------|:--------------------------------------------------------------| -| VAE | `vae`, `video_vae`, `audio_vae` | `vae` is the common image-generation override | -| Transformer / DiT | `transformer`, `video_dit`, `audio_dit` | `transformer` is the standard override for the main denoiser | -| Text / Preprocess | `text_encoder`, `text_encoder_2`, `tokenizer`, `processor`, `image_processor` | Replacement encoders often need matching preprocessing assets | -| Auxiliary | `scheduler`, `spatial_upsampler`, `vocoder`, `connectors`, `dual_tower_bridge`, `image_encoder`, `vision_language_encoder` | Only valid for pipelines that expose these components | - -### Known Component Repos - -The table below lists concrete Hugging Face component repos that are already used in SGLang Diffusion docs or tests. It is not an exhaustive catalog of all compatible component repos. - -| Base Model | Override Key | Example Repo | Notes | -|:-------------------------------|:--------------|:-----------------------------------------|:------------------------------------------| -| `black-forest-labs/FLUX.2-dev` | `vae` | `black-forest-labs/FLUX.2-small-decoder` | Decoder-only FLUX.2 VAE override | -| `black-forest-labs/FLUX.2-dev` | `vae` | `fal/FLUX.2-Tiny-AutoEncoder` | Existing tested custom VAE path | - -### VAE - -- `--vae-path` is the common image-generation override. -- `--video-vae-path` and `--audio-vae-path` are only relevant for pipelines with separate video or audio VAEs. - -### Transformer / DiT - -- `--transformer-path` is the standard override for the main denoising transformer. -- For quantized transformers, prefer `--transformer-path` or `--transformer-weights-path`; see `quantization.md`. -- `--video-dit-path` and `--audio-dit-path` are only for pipelines that split denoisers by modality. - -### Text Encoders and Preprocessors - -- `--text-encoder-path` and `--text-encoder-2-path` override primary and secondary text encoders. -- `--tokenizer-path`, `--processor-path`, and `--image-processor-path` are useful when the replacement encoder requires matching preprocessing assets. - -### Auxiliary Components - -- `--scheduler-path` is only relevant when the pipeline exposes a scheduler component. -- `--spatial-upsampler-path` is mainly for two-stage pipelines such as `LTX2TwoStagePipeline`. -- `--vocoder-path`, `--connectors-path`, `--dual-tower-bridge-path`, `--image-encoder-path`, and `--vision-language-encoder-path` are only valid for pipelines that expose those components. - -### Notes - -1. Component overrides are only valid when the target pipeline actually uses - that component. -2. The override key should match the component name in the pipeline's - `model_index.json` or the native pipeline's registered module name. - -## Verified LoRA Examples - -This section lists example LoRAs that have been explicitly tested and verified with each base model in the **SGLang Diffusion** pipeline. - -> Important: -> LoRAs that are not listed here are not necessarily incompatible. -> In practice, most standard LoRAs are expected to work, especially those following common Diffusers or SD-style conventions. -> The entries below simply reflect configurations that have been manually validated by the SGLang team. - -### Verified LoRAs by Base Model - -| Base Model | Supported LoRAs | -|:----------------|:---------------------------------------------------------------------------------------------------------------------------------------------------| -| Wan2.2 | `lightx2v/Wan2.2-Distill-Loras`
`Cseti/wan2.2-14B-Arcane_Jinx-lora-v1` | -| Wan2.1 | `lightx2v/Wan2.1-Distill-Loras` | -| Z-Image-Turbo | `tarn59/pixel_art_style_lora_z_image_turbo`
`wcde/Z-Image-Turbo-DeJPEG-Lora` | -| Qwen-Image | `lightx2v/Qwen-Image-Lightning`
`flymy-ai/qwen-image-realism-lora`
`prithivMLmods/Qwen-Image-HeadshotX`
`starsfriday/Qwen-Image-EVA-LoRA` | -| Qwen-Image-Edit | `ostris/qwen_image_edit_inpainting`
`lightx2v/Qwen-Image-Edit-2511-Lightning` | -| Flux | `dvyio/flux-lora-simple-illustration`
`XLabs-AI/flux-furry-lora`
`XLabs-AI/flux-RealismLora` | - -## Special requirements - -### Sliding Tile Attention - -- Currently, only Hopper GPUs (H100s) are supported. diff --git a/docs/diffusion/contributing.md b/docs/diffusion/contributing.md deleted file mode 100644 index 9b960aec9..000000000 --- a/docs/diffusion/contributing.md +++ /dev/null @@ -1,79 +0,0 @@ -# Contributing to SGLang Diffusion - -This guide outlines the requirements for contributing to the SGLang Diffusion module (`sglang.multimodal_gen`). - -## Contributor Guides - -- [Support New Models](support_new_models.md): implementation guide for adding new diffusion pipelines -- [CI Performance](ci_perf.md): update and regenerate perf baselines - -```{toctree} -:maxdepth: 1 - -support_new_models -ci_perf -``` - -## On AI-Assisted ("Vibe Coding") PRs - -Vibe-coded PRs are welcome — we judge code quality, not how it was produced. The bar is the same for all PRs: - -- **No over-commenting.** If the name says it all, skip the docstring. -- **No over-catching.** Don't guard against errors that virtually never happen in practice. -- **Test before submitting.** AI-generated code can be subtly wrong — verify correctness end-to-end. - -## Commit Message Convention - -We follow a structured commit message format to maintain a clean history. - -**Format:** -```text -[diffusion] : -``` - -**Examples:** -- `[diffusion] cli: add --perf-dump-path argument` -- `[diffusion] scheduler: fix deadlock in batch processing` -- `[diffusion] model: support Stable Diffusion 3.5` - -**Rules:** -- **Prefix**: Always start with `[diffusion]`. -- **Scope** (Optional): `cli`, `scheduler`, `model`, `pipeline`, `docs`, etc. -- **Subject**: Imperative mood, short and clear (e.g., "add feature" not "added feature"). - -## Performance Reporting - -For PRs that impact **latency**, **throughput**, or **memory usage**, you **should** provide a performance comparison report. - -### How to Generate a Report - -1. **Baseline**: run the benchmark (for a single generation task) - ```bash - $ sglang generate --model-path --prompt "A benchmark prompt" --perf-dump-path baseline.json - ``` - -2. **New**: run the same benchmark, without modifying any server_args or sampling_params - ```bash - $ sglang generate --model-path --prompt "A benchmark prompt" --perf-dump-path new.json - ``` - -3. **Compare**: run the compare script, which will print a Markdown table to the console - ```bash - $ python python/sglang/multimodal_gen/benchmarks/compare_perf.py baseline.json new.json [new2.json ...] - ### Performance Comparison Report - ... - ``` -4. **Paste**: paste the table into the PR description - -## CI-Based Change Protection - -Consider adding tests to the `pr-test` or `nightly-test` suites to safeguard your changes, especially for PRs that: - -- support a new model - - add a testcase for this new model to `testcase_configs.py` -- support or fix important features -- significantly improve performance - -Please run the according testcase, then update/add the baseline to `perf_baselines.json` by following the instruction in console if applicable. - -See [test](https://github.com/sgl-project/sglang/tree/main/python/sglang/multimodal_gen/test) for examples diff --git a/docs/diffusion/development.md b/docs/diffusion/development.md deleted file mode 100644 index afed2fb8d..000000000 --- a/docs/diffusion/development.md +++ /dev/null @@ -1,5 +0,0 @@ -# Development - -This page collects lower-level development material for SGLang Diffusion. - -- [Contributing](contributing.md): contribution workflow, adding new models, and CI perf baselines diff --git a/docs/diffusion/disaggregation.md b/docs/diffusion/disaggregation.md deleted file mode 100644 index 57bc2c4f1..000000000 --- a/docs/diffusion/disaggregation.md +++ /dev/null @@ -1,237 +0,0 @@ -# Disaggregated Diffusion Pipeline - -Split a monolithic text-to-video/image pipeline into independent **Encoder**, **Denoiser**, and **Decoder** roles, each running on its own GPU(s). A central **DiffusionServer** routes requests through the pipeline. - -## Quick Start - -Disaggregation is controlled by a single flag: `--disagg-role`. Each component is launched independently, just like LLM PD disaggregation. - -| `--disagg-role` | What it runs | -|----------------|--------------| -| `monolithic` | (Default) Standard single-server mode | -| `encoder` | All stages with the default `RoleType.ENCODER` affinity: `InputValidationStage`, `TextEncodingStage` (plus `ImageEncodingStage` / `ImageVAEEncodingStage` for image-conditioned pipelines), `LatentPreparationStage`, `TimestepPreparationStage`, and any model-specific "before denoising" stage (e.g. `QwenImageLayeredBeforeDenoisingStage`, `GlmImageBeforeDenoisingStage`). | -| `denoiser` | `DenoisingStage` (and its subclasses: `CausalDMDDenoisingStage`, `DmdDenoisingStage`, `LTX2AVDenoisingStage`, `LTX2RefinementStage`, `Hunyuan3DShapeDenoisingStage`, ...) — the DiT forward loop plus the scheduler stepping it drives. | -| `decoder` | `DecodingStage` (VAE decode) and its subclasses (`LTX2AVDecodingStage`, `HeliosDecodingStage`, ...). | -| `server` | DiffusionServer head node + HTTP server (no GPU) | - -> Each stage declares its role via the `role_affinity` property on `PipelineStage` (default `ENCODER`). When `--disagg-role` is not `monolithic`, the pipeline only instantiates stages whose affinity matches, so the above table is the source of truth for what actually runs in each process. - -### Single-Machine Example (Verified) - -The following commands have been tested end-to-end on an 8×H200 machine with -`Wan-AI/Wan2.1-T2V-1.3B-Diffusers`. Each role runs on a separate GPU via -`--base-gpu-id`; the `server` head node requires no GPU. - -```bash -# Terminal 1: Encoder (GPU 0) -sglang serve --model-path Wan-AI/Wan2.1-T2V-1.3B-Diffusers \ - --disagg-role encoder \ - --disagg-server-addr tcp://127.0.0.1:19655 \ - --scheduler-port 19000 \ - --num-gpus 1 --base-gpu-id 0 - -# Terminal 2: Denoiser (GPU 1) -sglang serve --model-path Wan-AI/Wan2.1-T2V-1.3B-Diffusers \ - --disagg-role denoiser \ - --disagg-server-addr tcp://127.0.0.1:19655 \ - --scheduler-port 19001 \ - --num-gpus 1 --base-gpu-id 1 - -# Terminal 3: Decoder (GPU 2) -sglang serve --model-path Wan-AI/Wan2.1-T2V-1.3B-Diffusers \ - --disagg-role decoder \ - --disagg-server-addr tcp://127.0.0.1:19655 \ - --scheduler-port 19002 \ - --num-gpus 1 --base-gpu-id 2 - -# Terminal 4: DiffusionServer head (no GPU, receives HTTP requests) -sglang serve --model-path Wan-AI/Wan2.1-T2V-1.3B-Diffusers \ - --disagg-role server \ - --encoder-urls "tcp://127.0.0.1:19000" \ - --denoiser-urls "tcp://127.0.0.1:19001" \ - --decoder-urls "tcp://127.0.0.1:19002" \ - --host 0.0.0.0 --port 22000 \ - --scheduler-port 19655 - -# Send request (video generation) -curl http://127.0.0.1:22000/v1/videos \ - -H "Content-Type: application/json" \ - -d '{"model": "Wan-AI/Wan2.1-T2V-1.3B-Diffusers", "prompt": "A curious raccoon exploring a garden, cinematic", "size": "832x480"}' -``` - -> **Tested result (8×H200):** -> Encoder 2.3 s (TextEncoding) → Denoiser 312.8 s (50 steps, layerwise offload) → Decoder 7.1 s (VAE decode). -> Total ~322 s for 81-frame 1024×1024 video. - -> **Tip:** `--base-gpu-id` controls which physical GPU the role uses. -> Encoder and Decoder can share a GPU (e.g. both `--base-gpu-id 0`) to save resources, -> but make sure the combined GPU memory is sufficient. - -### Multi-Machine Example - -The exact same CLI pattern — just replace `127.0.0.1` with actual IPs and add -RDMA flags for direct transfer: - -```bash -# Machine A (10.0.0.1): Encoder -sglang serve --model-path Wan-AI/Wan2.1-T2V-14B-Diffusers \ - --disagg-role encoder \ - --disagg-server-addr tcp://10.0.0.4:19655 \ - --scheduler-port 19000 \ - --num-gpus 1 \ - --disagg-p2p-hostname 10.0.0.1 --disagg-ib-device mlx5_0 - -# Machine B (10.0.0.2): Denoiser (4 GPUs with SP) -sglang serve --model-path Wan-AI/Wan2.1-T2V-14B-Diffusers \ - --disagg-role denoiser \ - --disagg-server-addr tcp://10.0.0.4:19655 \ - --scheduler-port 19001 \ - --num-gpus 4 --denoiser-sp 4 --denoiser-ulysses 2 --denoiser-ring 2 \ - --disagg-p2p-hostname 10.0.0.2 --disagg-ib-device mlx5_0 - -# Machine C (10.0.0.3): Decoder -sglang serve --model-path Wan-AI/Wan2.1-T2V-14B-Diffusers \ - --disagg-role decoder \ - --disagg-server-addr tcp://10.0.0.4:19655 \ - --scheduler-port 19002 \ - --num-gpus 1 \ - --disagg-p2p-hostname 10.0.0.3 --disagg-ib-device mlx5_0 - -# Machine D (10.0.0.4): DiffusionServer head -sglang serve --model-path Wan-AI/Wan2.1-T2V-14B-Diffusers \ - --disagg-role server \ - --encoder-urls "tcp://10.0.0.1:19000" \ - --denoiser-urls "tcp://10.0.0.2:19001" \ - --decoder-urls "tcp://10.0.0.3:19002" \ - --host 0.0.0.0 --port 30000 \ - --scheduler-port 19655 \ - --disagg-dispatch-policy max_free_slots -``` - -> ZMQ handles startup order gracefully — instances and head can start in any order. - -## Multiple Instances per Role - -Use semicolons in `--*-urls` to register multiple instances: - -```bash -# 2 encoders + 2 denoisers (4-GPU SP each) + 1 decoder -sglang serve --model-path ... --disagg-role server \ - --encoder-urls "tcp://10.0.0.1:35000;tcp://10.0.0.2:35000" \ - --denoiser-urls "tcp://10.0.0.3:35000;tcp://10.0.0.4:35000" \ - --decoder-urls "tcp://10.0.0.5:35000" -``` - -## Port Convention - -Result endpoints are derived deterministically from the head node's `--scheduler-port` (default: 5555): - -| Socket | Port | -|--------|------| -| DS frontend (ROUTER) | `scheduler_port` | -| Encoder result (PULL) | `scheduler_port + 1` | -| Denoiser result (PULL) | `scheduler_port + 2` | -| Decoder result (PULL) | `scheduler_port + 3` | - -Role instances derive their result endpoint automatically from `--disagg-server-addr`. No manual endpoint configuration needed. - -## Transfer Mechanism - -Tensor data between roles (encoder→denoiser, denoiser→decoder) is transferred via a P2P transfer engine. The DiffusionServer only routes lightweight control messages (alloc/push/ready); actual tensor data flows directly between instances. - -**mooncake-transfer-engine** is required for disaggregated diffusion. It provides RDMA for direct GPU-to-GPU data movement. - -```bash -pip install mooncake-transfer-engine -``` - -### Transfer Flow - -1. **Sender** (encoder/denoiser) stages tensors: async copy to transfer buffer (GPU or CPU pinned, depending on GPUDirect support), overlapped with metadata JSON serialization. -2. **Sender** sends `transfer_staged` control message to DiffusionServer (metadata only, no tensor data). -3. **DiffusionServer** sends `transfer_alloc` to receiver → receiver allocates buffer slot → replies `transfer_allocated`. -4. **DiffusionServer** sends `transfer_push` to receiver with sender's address info. -5. **Receiver** pulls data via transfer engine (Mooncake RDMA or mock), sends `transfer_ready`. -6. **Receiver** loads tensors async on a dedicated transfer stream, overlapped with the previous request's compute. - -Decoder results (final output) flow back through DiffusionServer as raw ZMQ frames to the HTTP client. - -### RDMA Flags - -| Flag | Default | Description | -|------|---------|-------------| -| `--disagg-p2p-hostname` | `127.0.0.1` | RDMA-reachable hostname/IP of this instance | -| `--disagg-ib-device` | `None` | InfiniBand device (e.g., `mlx5_0`, `mlx5_roce0`) | -| `--disagg-transfer-pool-size` | 256 MiB | Pinned memory pool per instance | - -Set `--disagg-p2p-hostname` to the actual IP on each machine. For multi-machine, `--disagg-ib-device` specifies the RDMA NIC. - -## Per-Role Parallelism - -| Flag | Description | -|------|-------------| -| `--encoder-tp` | Encoder tensor parallelism | -| `--denoiser-tp` / `--denoiser-sp` / `--denoiser-ulysses` / `--denoiser-ring` | Denoiser parallelism | -| `--decoder-tp` | Decoder tensor parallelism | - -If not specified, parallelism is auto-derived from `--num-gpus`. - -## Other Options - -| Flag | Default | Description | -|------|---------|-------------| -| `--disagg-timeout` | `600` | Timeout (seconds) for pending requests | -| `--disagg-dispatch-policy` | `round_robin` | `round_robin` or `max_free_slots` | - -## Python API - -For programmatic single-machine deployment, `launch_pool_disagg_server()` is available: - -```python -from sglang.multimodal_gen.runtime.server_args import ServerArgs -from sglang.multimodal_gen.runtime.launch_server import launch_pool_disagg_server - -server_args = ServerArgs.from_kwargs( - model_path="Wan-AI/Wan2.1-T2V-14B-Diffusers", - denoiser_sp=4, denoiser_ulysses=2, denoiser_ring=2, - disagg_ib_device="mlx5_0", -) - -launch_pool_disagg_server( - server_args, - encoder_gpus=[[0]], - denoiser_gpus=[[1, 2, 3, 4], [5, 6, 7, 8]], - decoder_gpus=[[0]], -) -``` - -## Architecture - -``` -Client ─── HTTP (port 30000) ──► FastAPI Server - │ - ▼ - DiffusionServer (ROUTER, scheduler_port) - ┌───────┼───────┐ - PUSH work │ │ │ PUSH work - ▼ │ ▼ - Encoder[0..N] │ Decoder[0..K] - │ │ ▲ - P2P tensor │ │ │ P2P tensor - transfer ▼ │ │ transfer - Denoiser[0..M] ─────┘ - │ - PULL results ◄────┘ (decoder → DS → client) -``` - -### Request State Machine - -``` -PENDING → ENCODER_WAITING → ENCODER_RUNNING → ENCODER_DONE - │ - DENOISING_WAITING → DENOISING_RUNNING → DENOISING_DONE - │ - DECODER_WAITING → DECODER_RUNNING → DONE -``` - -Any state can transition to `FAILED` or `TIMED_OUT`. diff --git a/docs/diffusion/environment_variables.md b/docs/diffusion/environment_variables.md deleted file mode 100644 index a9ba2d250..000000000 --- a/docs/diffusion/environment_variables.md +++ /dev/null @@ -1,101 +0,0 @@ -# Environment Variables - -## Runtime - -| Environment Variable | Default | Description | -|----------------------|---------|-------------| -| `SGLANG_DIFFUSION_TARGET_DEVICE` | `cuda` | Target device for inference (`cuda`, `rocm`, `xpu`, `npu`, `musa`, `mps`, `cpu`) | -| `SGLANG_DIFFUSION_ATTENTION_BACKEND` | not set | Override attention backend via env var (e.g. `fa`, `torch_sdpa`, `sage_attn`) | -| `SGLANG_DIFFUSION_ATTENTION_CONFIG` | not set | Path to attention backend configuration file (JSON/YAML) | -| `SGLANG_DIFFUSION_STAGE_LOGGING` | false | Enable per-stage timing logs | -| `SGLANG_DIFFUSION_SERVER_DEV_MODE` | false | Enable dev-only HTTP endpoints for debugging | -| `SGLANG_DIFFUSION_TORCH_PROFILER_DIR` | not set | Directory for torch profiler traces (absolute path). Enables profiling when set | -| `SGLANG_DIFFUSION_CACHE_ROOT` | `~/.cache/sgl_diffusion` | Root directory for cache files | -| `SGLANG_DIFFUSION_CONFIG_ROOT` | `~/.config/sgl_diffusion` | Root directory for configuration files | -| `SGLANG_DIFFUSION_LOGGING_LEVEL` | `INFO` | Default logging level | -| `SGLANG_DIFFUSION_WORKER_MULTIPROC_METHOD` | `fork` | Multiprocess context for workers (`fork` or `spawn`) | -| `SGLANG_USE_RUNAI_MODEL_STREAMER` | true | Use Run:AI model streamer for model loading | - -## Platform-Specific - -### Apple MPS - -| Environment Variable | Default | Description | -|----------------------|---------|--------------------------------------------------------------| -| `SGLANG_USE_MLX` | not set | Set to `1` to enable MLX fused Metal kernels for norm ops on MPS | - -### ROCm (AMD GPUs) - -| Environment Variable | Default | Description | -|----------------------|---------|-------------| -| `SGLANG_USE_ROCM_VAE` | false | Use AITer GroupNorm in VAE for improved performance on ROCm | -| `SGLANG_USE_ROCM_CUDNN_BENCHMARK` | false | Enable MIOpen auto-tuning for VAE conv layers on ROCm | - -### Quantization - -| Environment Variable | Default | Description | -|----------------------|---------|-------------| -| `SGLANG_DIFFUSION_FLASHINFER_FP4_GEMM_BACKEND` | not set | Optional FlashInfer FP4 GEMM backend override for diffusion NVFP4. When unset, SGLang defaults to `flashinfer_trtllm`. | - -## Caching Acceleration - -These variables configure caching acceleration for Diffusion Transformer (DiT) models. -SGLang supports multiple caching strategies - see [caching documentation](performance/cache/index.md) for an overview. - -### Cache-DiT Configuration - -See [cache-dit documentation](performance/cache/cache_dit.md) for detailed configuration. - -| Environment Variable | Default | Description | -|-------------------------------------|---------|------------------------------------------| -| `SGLANG_CACHE_DIT_ENABLED` | false | Enable Cache-DiT acceleration | -| `SGLANG_CACHE_DIT_FN` | 1 | First N blocks to always compute | -| `SGLANG_CACHE_DIT_BN` | 0 | Last N blocks to always compute | -| `SGLANG_CACHE_DIT_WARMUP` | 4 | Warmup steps before caching | -| `SGLANG_CACHE_DIT_RDT` | 0.24 | Residual difference threshold | -| `SGLANG_CACHE_DIT_MC` | 3 | Max continuous cached steps | -| `SGLANG_CACHE_DIT_TAYLORSEER` | false | Enable TaylorSeer calibrator | -| `SGLANG_CACHE_DIT_TS_ORDER` | 1 | TaylorSeer order (1 or 2) | -| `SGLANG_CACHE_DIT_SCM_PRESET` | none | SCM preset (none/slow/medium/fast/ultra) | -| `SGLANG_CACHE_DIT_SCM_POLICY` | dynamic | SCM caching policy | -| `SGLANG_CACHE_DIT_SCM_COMPUTE_BINS` | not set | Custom SCM compute bins | -| `SGLANG_CACHE_DIT_SCM_CACHE_BINS` | not set | Custom SCM cache bins | - -### Cache-DiT Secondary Transformer - -For dual-transformer models (e.g., Wan2.2 with high/low-noise experts), these variables configure caching for the secondary transformer. Each falls back to its primary counterpart if not set. - -| Environment Variable | Default | Description | -|-------------------------------------|---------|------------------------------------------| -| `SGLANG_CACHE_DIT_SECONDARY_FN` | (from primary) | First N blocks to always compute | -| `SGLANG_CACHE_DIT_SECONDARY_BN` | (from primary) | Last N blocks to always compute | -| `SGLANG_CACHE_DIT_SECONDARY_WARMUP` | (from primary) | Warmup steps before caching | -| `SGLANG_CACHE_DIT_SECONDARY_RDT` | (from primary) | Residual difference threshold | -| `SGLANG_CACHE_DIT_SECONDARY_MC` | (from primary) | Max continuous cached steps | -| `SGLANG_CACHE_DIT_SECONDARY_TAYLORSEER` | (from primary) | Enable TaylorSeer calibrator | -| `SGLANG_CACHE_DIT_SECONDARY_TS_ORDER` | (from primary) | TaylorSeer order (1 or 2) | - -## Cloud Storage - -These variables configure S3-compatible cloud storage for automatically uploading generated images and videos. - -| Environment Variable | Default | Description | -|---------------------------------|---------|--------------------------------------------------------| -| `SGLANG_CLOUD_STORAGE_TYPE` | not set | Set to `s3` to enable cloud storage | -| `SGLANG_S3_BUCKET_NAME` | not set | The name of the S3 bucket | -| `SGLANG_S3_ENDPOINT_URL` | not set | Custom endpoint URL (for MinIO, OSS, etc.) | -| `SGLANG_S3_REGION_NAME` | us-east-1 | AWS region name | -| `SGLANG_S3_ACCESS_KEY_ID` | not set | AWS Access Key ID | -| `SGLANG_S3_SECRET_ACCESS_KEY` | not set | AWS Secret Access Key | - -## CUDA Crash Debugging - -These variables enable kernel API logging and optional input/output dumps around diffusion CUDA kernel call boundaries. They are useful when tracking down CUDA crashes such as illegal memory access, device-side assert, or shape mismatches in custom kernels. - -| Environment Variable | Default | Description | -|----------------------|---------|-------------| -| `SGLANG_KERNEL_API_LOGLEVEL` | `0` | Controls crash-debug kernel API logging. `1` logs API names, `3` logs tensor metadata, `5` adds tensor statistics, and `10` also writes dump snapshots. | -| `SGLANG_KERNEL_API_LOGDEST` | `stdout` | Destination for crash-debug kernel API logs. Use `stdout`, `stderr`, or a file path. `%i` is replaced with the process PID. | -| `SGLANG_KERNEL_API_DUMP_DIR` | `sglang_kernel_api_dumps` | Output directory for level-10 kernel API dumps. `%i` is replaced with the process PID. | -| `SGLANG_KERNEL_API_DUMP_INCLUDE` | not set | Comma-separated wildcard patterns for kernel API names to include in level-10 dumps. | -| `SGLANG_KERNEL_API_DUMP_EXCLUDE` | not set | Comma-separated wildcard patterns for kernel API names to exclude from level-10 dumps. | diff --git a/docs/diffusion/index.md b/docs/diffusion/index.md deleted file mode 100644 index 91ec58d39..000000000 --- a/docs/diffusion/index.md +++ /dev/null @@ -1,53 +0,0 @@ -# SGLang Diffusion - -SGLang Diffusion is a high-performance inference framework for image and video generation. It provides native SGLang pipelines, diffusers backend support, an OpenAI-compatible server, and an optimized kernel stack built on both precompiled `sgl-kernel` operators and JIT kernels for key inference paths. - -## Key Features - -- Broad model support across Wan, Hunyuan, Cosmos3, Qwen-Image, FLUX, Z-Image, GLM-Image, and more -- Fast inference with `sgl-kernel`, JIT kernels, scheduler improvements, and caching acceleration -- Multiple interfaces: `sglang generate`, `sglang serve`, and an OpenAI-compatible API -- Multi-platform support for NVIDIA, AMD, Intel XPU, Ascend, Apple Silicon, and Moore Threads - -## Quick Start - -```bash -uv pip install "sglang[diffusion]" --prerelease=allow -``` - -```bash -sglang generate --model-path Qwen/Qwen-Image \ - --prompt "A beautiful sunset over the mountains" \ - --save-output -``` - -```bash -sglang serve --model-path Qwen/Qwen-Image --port 30010 -``` - -## Start Here - -- [Installation](installation.md): install SGLang Diffusion and platform dependencies -- [Compatibility Matrix](compatibility_matrix.md): check model, optimization, and component override support -- [CLI](api/cli.md): run one-off generation jobs or launch a persistent server -- [OpenAI-Compatible API](api/openai_api.md): send image and video requests to the HTTP server -- [Attention Backends](performance/attention_backends.md): choose the best backend for your model and hardware -- [Caching Acceleration](performance/cache/index.md): use Cache-DiT or TeaCache to reduce denoising cost -- [Quantization](quantization.md): load quantized transformer checkpoints -- [Contributing](contributing.md): contribution workflow, adding new models, and CI perf baselines - -## Additional Documentation - -- [Post-Processing](api/post_processing.md): frame interpolation and upscaling -- [Performance Overview](performance/index.md): overview of attention, caching, and profiling -- [Environment Variables](environment_variables.md): platform, caching, storage, and debugging configuration -- [Support New Models](support_new_models.md): implementation guide for new diffusion pipelines -- [CI Performance](ci_perf.md): performance baseline generation - -## References - -- [SGLang GitHub](https://github.com/sgl-project/sglang) -- [Cache-DiT](https://github.com/vipshop/cache-dit) -- [FastVideo](https://github.com/hao-ai-lab/FastVideo) -- [xDiT](https://github.com/xdit-project/xDiT) -- [Diffusers](https://github.com/huggingface/diffusers) diff --git a/docs/diffusion/installation.md b/docs/diffusion/installation.md deleted file mode 100644 index 46fbab063..000000000 --- a/docs/diffusion/installation.md +++ /dev/null @@ -1,128 +0,0 @@ -# Install SGLang-Diffusion - -You can install SGLang-Diffusion using one of the methods below. The standard installation already includes SGLang's optimized kernel stack, including both `sgl-kernel` and JIT kernels used by diffusion workloads. - -## Standard Installation (NVIDIA GPUs) - -### Method 1: With pip or uv - -It is recommended to use uv for a faster installation: - -```bash -pip install --upgrade pip -pip install uv -uv pip install "sglang[diffusion]" --prerelease=allow -``` - -### Method 2: From source - -```bash -# Use the latest release branch -git clone https://github.com/sgl-project/sglang.git -cd sglang - -# Install the Python packages -pip install --upgrade pip -pip install -e "python[diffusion]" - -# With uv -uv pip install -e "python[diffusion]" --prerelease=allow -``` - -### Method 3: Using Docker - -The Docker images are available on Docker Hub at [lmsysorg/sglang](https://hub.docker.com/r/lmsysorg/sglang), built from the [Dockerfile](https://github.com/sgl-project/sglang/blob/main/docker/Dockerfile). -Replace `` below with your HuggingFace Hub [token](https://huggingface.co/docs/hub/en/security-tokens). - -```bash -docker run --gpus all \ - --shm-size 32g \ - -p 30000:30000 \ - -v ~/.cache/huggingface:/root/.cache/huggingface \ - --env "HF_TOKEN=" \ - --ipc=host \ - lmsysorg/sglang:dev \ - zsh -c '\ - echo "Installing diffusion dependencies..." && \ - pip install -e "python[diffusion]" && \ - echo "Starting SGLang-Diffusion..." && \ - sglang generate \ - --model-path black-forest-labs/FLUX.1-dev \ - --prompt "A logo With Bold Large text: SGL Diffusion" \ - --save-output \ - ' -``` - -## Platform-Specific: ROCm (AMD GPUs) - -For AMD Instinct GPUs (e.g., MI300X), you can use the ROCm-enabled Docker image: - -```bash -docker run --device=/dev/kfd --device=/dev/dri --ipc=host \ - -v ~/.cache/huggingface:/root/.cache/huggingface \ - --env HF_TOKEN= \ - lmsysorg/sglang:v0.5.5.post2-rocm700-mi30x \ - sglang generate --model-path black-forest-labs/FLUX.1-dev --prompt "A logo With Bold Large text: SGL Diffusion" --save-output -``` - -For detailed ROCm system configuration and installation from source, see [AMD GPUs](../platforms/amd_gpu.md). - -## Platform-Specific: MUSA (Moore Threads GPUs) - -For Moore Threads GPUs (MTGPU) with the MUSA software stack, please follow the instructions below to install from source: - -```bash -# Clone the repository -git clone https://github.com/sgl-project/sglang.git -cd sglang - -# Install the Python packages -pip install --upgrade pip -rm -f python/pyproject.toml && mv python/pyproject_other.toml python/pyproject.toml -pip install -e "python[all_musa]" -``` - -## Platform-Specific: Intel XPU - -For Intel Data Center GPU Max or Arc GPUs, follow the [XPU installation guide](../platforms/xpu.md) to set up the base environment, then install diffusion dependencies: - -```bash -pip install -e "python[diffusion]" -``` - -## Platform-Specific: Ascend NPU - -For Ascend NPU, please follow the [NPU installation guide](../platforms/ascend/ascend_npu.md). - -Quick test: - -```bash -sglang generate --model-path black-forest-labs/FLUX.1-dev \ - --prompt "A logo With Bold Large text: SGL Diffusion" \ - --save-output -``` - -## Platform-Specific: Apple MPS - -For Apple MPS, please follow the instructions below to install from source: - -```bash -# Install ffmpeg -brew install ffmpeg - -# Install uv -brew install uv - -# Clone the repository -git clone https://github.com/sgl-project/sglang.git -cd sglang - -# Create and activate a virtual environment -uv venv -p 3.11 sglang-diffusion -source sglang-diffusion/bin/activate - -# Install the Python packages -uv pip install --upgrade pip -rm -f python/pyproject.toml && mv python/pyproject_other.toml python/pyproject.toml -uv pip install -e "python[all_mps]" -``` diff --git a/docs/diffusion/performance/attention_backends.md b/docs/diffusion/performance/attention_backends.md deleted file mode 100644 index 9318c98a8..000000000 --- a/docs/diffusion/performance/attention_backends.md +++ /dev/null @@ -1,166 +0,0 @@ -# Attention Backends - -This document describes the attention backends available in sglang diffusion (`sglang.multimodal_gen`) and how to select them. - -## Overview - -Attention backends are defined by `AttentionBackendEnum` (`sglang.multimodal_gen.runtime.platforms.interface.AttentionBackendEnum`) and selected via the CLI flag `--attention-backend`. - -Backend selection is performed by the shared attention layers (e.g. `LocalAttention` / `USPAttention` / `UlyssesAttention` in `sglang.multimodal_gen.runtime.layers.attention.layer`) and therefore applies to any model component using these layers (e.g. diffusion transformer / DiT and encoders). - -When using the diffusers backend, `--attention-backend` is passed through to diffusers' -`set_attention_backend` (e.g., `flash`, `_flash_3_hub`, `sage`, `xformers`, `native`). - -- **CUDA**: prefers FlashAttention (FA3/FA4) when supported; otherwise falls back to PyTorch SDPA. -- **ROCm**: uses FlashAttention when available; otherwise falls back to PyTorch SDPA. -- **Intel XPU**: uses XPU Flash Attention backend (fp16/bf16, head sizes 64/96/128/192/256); otherwise falls back to PyTorch SDPA. -- **MUSA**: uses FlashAttention when available; otherwise falls back to PyTorch SDPA. -- **MPS**: always uses PyTorch SDPA. -- **NPU**: for ring attention uses FA otherwise uses PyTorch SDPA. - -## Backend options - -For SGLang-native pipelines, the CLI accepts the lowercase names of `AttentionBackendEnum`. The table below lists the backends implemented by the built-in platforms. `fa3`/`fa4` are accepted as aliases for `fa`. - -| CLI value | Enum value | Notes | -|---|---|---| -| `fa` / `fa3` / `fa4` | `FA` | FlashAttention. `fa3/fa4` are normalized to `fa` during argument parsing (`ServerArgs.__post_init__`). | -| `torch_sdpa` | `TORCH_SDPA` | PyTorch `scaled_dot_product_attention`. | -| `sliding_tile_attn` | `SLIDING_TILE_ATTN` | Sliding Tile Attention (STA). Requires `st_attn`. Configure via `--attention-backend-config`. | -| `sage_attn` | `SAGE_ATTN` | Requires `sageattention`. Upstream SageAttention CUDA extensions target SM80/SM86/SM89/SM90/SM120 (compute capability 8.0/8.6/8.9/9.0/12.0); see upstream `setup.py`: https://github.com/thu-ml/SageAttention/blob/main/setup.py. | -| `sage_attn_3` | `SAGE_ATTN_3` | Requires SageAttention3 installed per upstream instructions. | -| `video_sparse_attn` | `VIDEO_SPARSE_ATTN` | Requires `vsa`. Configure `sparsity` via `--attention-backend-config`. | -| `vmoba_attn` | `VMOBA_ATTN` | Requires `kernel.attn.vmoba_attn.vmoba`. Configure via `--attention-backend-config`. | -| `aiter` | `AITER` | Requires `aiter`. | -| `aiter_sage` | `AITER_SAGE` | Requires `aiter`. | -| `sla_attn` | `SLA_ATTN` | Sparse Linear Attention. Requires `SpargeAttn`. Install with `pip install git+https://github.com/thu-ml/SpargeAttn.git --no-build-isolation`. | -| `sage_sla_attn` | `SAGE_SLA_ATTN` | SageAttention + Sparse Linear Attention. Requires `SpargeAttn` (same install as SLA). | -| `sparse_video_gen_2_attn` | `SPARSE_VIDEO_GEN_2_ATTN` | Requires `svg`. See installation instructions at https://github.com/svg-project/Sparse-VideoGen. | -| `laser_attn` | `LASER_ATTN` | Requires `attentions` which can be installed with `sgl_kernel_npu`; available only for NPU. | -| `block_sparse_attn` | `BLOCK_SPARSE_ATTN` | Requires `attentions` which can be installed with `sgl_kernel_npu`; available only for NPU. | -| `rain_fusion_attn` | `RAIN_FUSION_ATTN` | Requires `attentions` which can be installed with `sgl_kernel_npu`; available only for NPU. | - -## Selection priority - -The selection order in `runtime/layers/attention/selector.py` is: - -1. `global_force_attn_backend(...)` / `global_force_attn_backend_context_manager(...)` -2. Component override from `--component-attention-backends` while that component is being constructed -3. CLI `--attention-backend` (`ServerArgs.attention_backend`) -4. Auto selection (platform capability, dtype, and installed packages) - -## Configuration - -Some backends require additional configuration. You can pass these parameters via `--attention-backend-config`. This argument accepts: -- A path to a JSON or YAML configuration file. -- A JSON string (e.g., `'{"sparsity": 0.5}'`). -- Key-value pairs (e.g., `"sparsity=0.5,enable_x=true"`). - -### Supported Configuration Parameters - -**Sliding Tile Attention (`sliding_tile_attn`)** - -| Parameter | Type | Description | Default | -| :--- | :--- | :--- | :--- | -| `mask_strategy_file_path` | `str` | **Required.** Path to the mask strategy JSON file. | - | -| `sta_mode` | `str` | Mode of STA. | `STA_inference` | -| `skip_time_steps` | `int` | Number of steps to use full attention before switching to sparse attention. | `15` | - -**Video Sparse Attention (`video_sparse_attn`)** - -| Parameter | Type | Description | Default | -| :--- | :--- | :--- | :--- | -| `sparsity` | `float` | Validation sparsity (0.0 - 1.0). | `0.0` | - -**V-MoBA (`vmoba_attn`)** - -| Parameter | Type | Description | Default | -| :--- | :--- | :--- | :--- | -| `temporal_chunk_size` | `int` | Chunk size for temporal dimension. | - | -| `temporal_topk` | `int` | Top-K tokens to select in temporal dimension. | - | -| `spatial_chunk_size` | `list[int]` | Chunk size for spatial dimension (H, W). | - | -| `spatial_topk` | `int` | Top-K tokens to select in spatial dimension. | - | -| `st_chunk_size` | `list[int]` | Chunk size for spatiotemporal dimension (T, H, W). | - | -| `st_topk` | `int` | Top-K tokens to select in spatiotemporal dimension. | - | -| `moba_select_mode` | `str` | Selection mode (e.g., `threshold`). | `threshold` | -| `moba_threshold` | `float` | Threshold value for selection. | `0.25` | -| `moba_threshold_type` | `str` | Type of thresholding (e.g., `query_head`). | `query_head` | -| `first_full_step` | `int` | Number of initial steps to use full attention. | `12` | -| `first_full_layer` | `int` | Number of initial layers to use full attention. | `0` | -| `temporal_layer` | `int` | Number of temporal layers. | `1` | -| `spatial_layer` | `int` | Number of spatial layers. | `1` | -| `st_layer` | `int` | Number of spatiotemporal layers. | `1` | - -**Block Sparse attention ( `block_sparse_attn` ) and Rain Fusion attention ( `rain_fusion_attn` )** -| Parameter | Type | Description | Default | -| :--- | :--- | :--- | :--- | -| `skip_first_steps` | `int` | Number of steps to use laser attention before switching to sparse attention. | `10` | -| `sparsity` | `float` | The sparsity coefficient must be in the range (0, 1). | `0.2` | - -## Platform support matrix - -| Backend | CUDA | ROCm | XPU | MUSA | MPS | NPU | Notes | -|---|---:|---:|---:|---:|---:|---:|---| -| `fa` | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | CUDA requires SM80+ and fp16/bf16. XPU uses its own flash attention backend. FlashAttention is only used when the required runtime is installed; otherwise it falls back to `torch_sdpa`. No extra installations are required for NPU | -| `torch_sdpa` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | Most compatible option across platforms. | -| `sliding_tile_attn` | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | CUDA-only. Requires `st_attn`. Configure via `--attention-backend-config`. | -| `sage_attn` | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | CUDA-only (optional dependency). | -| `sage_attn_3` | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | CUDA-only (optional dependency). | -| `video_sparse_attn` | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | CUDA-only. Requires `vsa`. Configure `sparsity` via `--attention-backend-config`. | -| `sla_attn` | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | CUDA-only. Requires `SpargeAttn`. | -| `sage_sla_attn` | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | CUDA-only. Requires `SpargeAttn`. | -| `vmoba_attn` | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | CUDA-only. Requires `kernel.attn.vmoba_attn.vmoba`. Configure via `--attention-backend-config`. | -| `aiter` | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | Requires `aiter`. | -| `aiter_sage` | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | Requires `aiter`. | -| `sparse_video_gen_2_attn` | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | CUDA-only. Requires `svg`. | -| `laser_attn` | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | NPU-only. Requires `attentions` from `sgl_kernel_npu`. Uses SDPA if seqlen is less than 2048. | -| `block_sparse_attn` | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | NPU-only. Requires `attentions` from `sgl_kernel_npu`. Configure via `--attention-backend-config`. | -| `rain_fusion_attn` | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | NPU-only. Requires `attentions` from `sgl_kernel_npu`. Configure via `--attention-backend-config`. | - -## Usage - -### Select a backend via CLI - -```bash -sglang generate \ - --model-path \ - --prompt "..." \ - --attention-backend fa -``` - -```bash -sglang generate \ - --model-path \ - --prompt "..." \ - --attention-backend torch_sdpa -``` - -### Override one component - -Use component overrides when a specific module needs different attention semantics from the main transformer: - -```bash -sglang generate \ - --model-path \ - --prompt "..." \ - --attention-backend fa \ - --component-attention-backends text_encoder=torch_sdpa -``` - -Component keys match pipeline module names from `model_index.json`, such as `text_encoder`, `text_encoder_2`, `transformer`, `transformer_2`, or `connectors`. - -### Using Sliding Tile Attention (STA) - -```bash -# Pass the mask strategy file path via config -sglang generate \ - --model-path \ - --prompt "..." \ - --attention-backend sliding_tile_attn \ - --attention-backend-config "mask_strategy_file_path=/abs/path/to/mask_strategy.json" -``` - -### Notes for ROCm / MPS - -- ROCm: use `--attention-backend torch_sdpa` or `fa` depending on what is available in your environment. -- MPS: the platform implementation always uses `torch_sdpa`. diff --git a/docs/diffusion/performance/cache/cache_dit.md b/docs/diffusion/performance/cache/cache_dit.md deleted file mode 100644 index 9f804ce54..000000000 --- a/docs/diffusion/performance/cache/cache_dit.md +++ /dev/null @@ -1,418 +0,0 @@ -# Cache-DiT - -SGLang integrates [Cache-DiT](https://github.com/vipshop/cache-dit), a caching acceleration engine for Diffusion Transformers (DiT), to achieve up to **1.69x inference speedup** with minimal quality loss. - -## Overview - -**Cache-DiT** uses intelligent caching strategies to skip redundant computation in the denoising loop: - -- **DBCache (Dual Block Cache)**: Dynamically decides when to cache transformer blocks based on residual differences -- **TaylorSeer**: Uses Taylor expansion for calibration to optimize caching decisions -- **SCM (Step Computation Masking)**: Step-level caching control for additional speedup - -## Basic Usage - -Enable Cache-DiT by exporting the environment variable and using `sglang generate` or `sglang serve` : - -```bash -SGLANG_CACHE_DIT_ENABLED=true \ -sglang generate --model-path Qwen/Qwen-Image \ - --prompt "A beautiful sunset over the mountains" -``` - -## Diffusers Backend - -Cache-DiT supports loading acceleration configs from a custom YAML file. For -diffusers pipelines (`diffusers` backend), pass the YAML/JSON path via `--cache-dit-config`. This -flow requires cache-dit >= 1.2.0 (`cache_dit.load_configs`). - -### Single GPU inference - -Define a `cache.yaml` file that contains: - -- DBCache + TaylorSeer - -```yaml -cache_config: - max_warmup_steps: 8 - warmup_interval: 2 - max_cached_steps: -1 - max_continuous_cached_steps: 2 - Fn_compute_blocks: 1 - Bn_compute_blocks: 0 - residual_diff_threshold: 0.12 - enable_taylorseer: true - taylorseer_order: 1 -``` - -Then apply the config with: - -```bash -sglang generate \ - --backend diffusers \ - --model-path Qwen/Qwen-Image \ - --cache-dit-config cache.yaml \ - --prompt "A beautiful sunset over the mountains" -``` - -- DBCache + TaylorSeer + SCM (Step Computation Mask) - -```yaml -cache_config: - max_warmup_steps: 8 - warmup_interval: 2 - max_cached_steps: -1 - max_continuous_cached_steps: 2 - Fn_compute_blocks: 1 - Bn_compute_blocks: 0 - residual_diff_threshold: 0.12 - enable_taylorseer: true - taylorseer_order: 1 - # Must set the num_inference_steps for SCM. The SCM will automatically - # generate the steps computation mask based on the num_inference_steps. - # Reference: https://cache-dit.readthedocs.io/en/latest/user_guide/CACHE_API/#scm-steps-computation-masking - num_inference_steps: 28 - steps_computation_mask: fast -``` - -- DBCache + TaylorSeer + SCM (Step Computation Mask) + Cache CFG - -```yaml -cache_config: - max_warmup_steps: 8 - warmup_interval: 2 - max_cached_steps: -1 - max_continuous_cached_steps: 2 - Fn_compute_blocks: 1 - Bn_compute_blocks: 0 - residual_diff_threshold: 0.12 - enable_taylorseer: true - taylorseer_order: 1 - num_inference_steps: 28 - steps_computation_mask: fast - enable_sperate_cfg: true # e.g, Qwen-Image, Wan, Chroma, Ovis-Image, etc. -``` - -### Distributed inference - -- 1D Parallelism - -Define a parallelism only config yaml `parallel.yaml` file that contains: - -```yaml -parallelism_config: - ulysses_size: auto - attention_backend: native -``` - -Then, apply the distributed inference acceleration config from yaml. `ulysses_size: auto` means that cache-dit will auto detect the `world_size` as the ulysses_size. Otherwise, you should manually set it as specific int number, e.g, 4. - -Then apply the distributed config with: (Note: please add `--num-gpus N` to specify the number of gpus for distributed inference) - -```bash -sglang generate \ - --backend diffusers \ - --num-gpus 4 \ - --model-path Qwen/Qwen-Image \ - --cache-dit-config parallel.yaml \ - --prompt "A futuristic cityscape at sunset" -``` - -- 2D Parallelism - -You can also define a 2D parallelism config yaml `parallel_2d.yaml` file that contains: - -```yaml -parallelism_config: - ulysses_size: auto - tp_size: 2 - attention_backend: native -``` -Then, apply the 2D parallelism config from yaml. Here `tp_size: 2` means using tensor parallelism with size 2. The `ulysses_size: auto` means that cache-dit will auto detect the `world_size // tp_size` as the ulysses_size. - -- 3D Parallelism - -You can also define a 3D parallelism config yaml `parallel_3d.yaml` file that contains: - -```yaml -parallelism_config: - ulysses_size: 2 - ring_size: 2 - tp_size: 2 - attention_backend: native -``` -Then, apply the 3D parallelism config from yaml. Here `ulysses_size: 2`, `ring_size: 2`, `tp_size: 2` means using ulysses parallelism with size 2, ring parallelism with size 2 and tensor parallelism with size 2. - -- Ulysses Anything Attention - -To enable Ulysses Anything Attention, you can define a parallelism config yaml `parallel_uaa.yaml` file that contains: - -```yaml -parallelism_config: - ulysses_size: auto - attention_backend: native - ulysses_anything: true -``` - -- Ulysses FP8 Communication - -For device that don't have NVLink support, you can enable Ulysses FP8 Communication to further reduce the communication overhead. You can define a parallelism config yaml `parallel_fp8.yaml` file that contains: - -```yaml -parallelism_config: - ulysses_size: auto - attention_backend: native - ulysses_float8: true -``` - -- Async Ulysses CP - -You can also enable async ulysses CP to overlap the communication and computation. Define a parallelism config yaml `parallel_async.yaml` file that contains: - -```yaml -parallelism_config: - ulysses_size: auto - attention_backend: native - ulysses_async: true # Now, only support for FLUX.1, Qwen-Image, Ovis-Image and Z-Image. -``` -Then, apply the config from yaml. Here `ulysses_async: true` means enabling async ulysses CP. - -- TE-P and VAE-P - -You can also specify the extra parallel modules in the yaml config. For example, define a parallelism config yaml `parallel_extra.yaml` file that contains: - -```yaml -parallelism_config: - ulysses_size: auto - attention_backend: native - extra_parallel_modules: ["text_encoder", "vae"] -``` - - -### Hybrid Cache and Parallelism - -Define a hybrid cache and parallel acceleration config yaml `hybrid.yaml` file that contains: - -```yaml -cache_config: - max_warmup_steps: 8 - warmup_interval: 2 - max_cached_steps: -1 - max_continuous_cached_steps: 2 - Fn_compute_blocks: 1 - Bn_compute_blocks: 0 - residual_diff_threshold: 0.12 - enable_taylorseer: true - taylorseer_order: 1 -parallelism_config: - ulysses_size: auto - attention_backend: native - extra_parallel_modules: ["text_encoder", "vae"] -``` - -Then, apply the hybrid cache and parallel acceleration config from yaml. - -```bash -sglang generate \ - --backend diffusers \ - --num-gpus 4 \ - --model-path Qwen/Qwen-Image \ - --cache-dit-config hybrid.yaml \ - --prompt "A beautiful sunset over the mountains" -``` - -### Attention Backend - -In some cases, users may want to only specify the attention backend without any other optimization configs. In this case, you can define a yaml file `attention.yaml` that only contains: - -```yaml -attention_backend: "flash" # '_flash_3' for Hopper -``` - -### Quantization - -You can also specify the quantization config in the yaml file, required `torchao>=0.16.0`. For example, define a yaml file `quantize.yaml` that contains: - -```yaml -quantize_config: # quantization configuration for transformer modules - # float8 (DQ), float8_weight_only, float8_blockwise, int8 (DQ), int8_weight_only, etc. - quant_type: "float8" - # layers to exclude from quantization (transformer). layers that contains any of the - # keywords in the exclude_layers list will be excluded from quantization. This is useful - # for some sensitive layers that are not robust to quantization, e.g., embedding layers. - exclude_layers: - - "embedder" - - "embed" - verbose: false # whether to print verbose logs during quantization -``` -Then, apply the quantization config from yaml. Please also enable torch.compile for better performance if you are using quantization. For example: - -```bash -sglang generate \ - --backend diffusers \ - --model-path Qwen/Qwen-Image \ - --warmup \ - --cache-dit-config quantize.yaml \ - --enable-torch-compile \ - --dit-cpu-offload false \ - --text-encoder-cpu-offload false \ - --prompt "A beautiful sunset over the mountains" -``` - -### Combined Configs: Cache + Parallelism + Quantization - -You can also combine all the above configs together in a single yaml file `combined.yaml` that contains: - -```yaml -cache_config: - max_warmup_steps: 8 - warmup_interval: 2 - max_cached_steps: -1 - max_continuous_cached_steps: 2 - Fn_compute_blocks: 1 - Bn_compute_blocks: 0 - residual_diff_threshold: 0.12 - enable_taylorseer: true - taylorseer_order: 1 -parallelism_config: - ulysses_size: auto - attention_backend: native - extra_parallel_modules: ["text_encoder", "vae"] -quantize_config: - quant_type: "float8" - exclude_layers: - - "embedder" - - "embed" - verbose: false -``` -Then, apply the combined cache, parallelism and quantization config from yaml. Please also enable torch.compile for better performance if you are using quantization. - -## Advanced Configuration - -### DBCache Parameters - -DBCache controls block-level caching behavior: - -| Parameter | Env Variable | Default | Description | -|-----------|---------------------------|---------|------------------------------------------| -| Fn | `SGLANG_CACHE_DIT_FN` | 1 | Number of first blocks to always compute | -| Bn | `SGLANG_CACHE_DIT_BN` | 0 | Number of last blocks to always compute | -| W | `SGLANG_CACHE_DIT_WARMUP` | 4 | Warmup steps before caching starts | -| R | `SGLANG_CACHE_DIT_RDT` | 0.24 | Residual difference threshold | -| MC | `SGLANG_CACHE_DIT_MC` | 3 | Maximum continuous cached steps | - -### TaylorSeer Configuration - -TaylorSeer improves caching accuracy using Taylor expansion: - -| Parameter | Env Variable | Default | Description | -|-----------|-------------------------------|---------|---------------------------------| -| Enable | `SGLANG_CACHE_DIT_TAYLORSEER` | false | Enable TaylorSeer calibrator | -| Order | `SGLANG_CACHE_DIT_TS_ORDER` | 1 | Taylor expansion order (1 or 2) | - -### Combined Configuration Example - -DBCache and TaylorSeer are complementary strategies that work together, you can configure both sets of parameters -simultaneously: - -```bash -SGLANG_CACHE_DIT_ENABLED=true \ -SGLANG_CACHE_DIT_FN=2 \ -SGLANG_CACHE_DIT_BN=1 \ -SGLANG_CACHE_DIT_WARMUP=4 \ -SGLANG_CACHE_DIT_RDT=0.4 \ -SGLANG_CACHE_DIT_MC=4 \ -SGLANG_CACHE_DIT_TAYLORSEER=true \ -SGLANG_CACHE_DIT_TS_ORDER=2 \ -sglang generate --model-path black-forest-labs/FLUX.1-dev \ - --prompt "A curious raccoon in a forest" -``` - -### SCM (Step Computation Masking) - -SCM provides step-level caching control for additional speedup. It decides which denoising steps to compute fully and -which to use cached results. - -**SCM Presets** - -SCM is configured with presets: - -| Preset | Compute Ratio | Speed | Quality | -|----------|---------------|----------|------------| -| `none` | 100% | Baseline | Best | -| `slow` | ~75% | ~1.3x | High | -| `medium` | ~50% | ~2x | Good | -| `fast` | ~35% | ~3x | Acceptable | -| `ultra` | ~25% | ~4x | Lower | - -**Usage** - -```bash -SGLANG_CACHE_DIT_ENABLED=true \ -SGLANG_CACHE_DIT_SCM_PRESET=medium \ -sglang generate --model-path Qwen/Qwen-Image \ - --prompt "A futuristic cityscape at sunset" -``` - -**Custom SCM Bins** - -For fine-grained control over which steps to compute vs cache: - -```bash -SGLANG_CACHE_DIT_ENABLED=true \ -SGLANG_CACHE_DIT_SCM_COMPUTE_BINS="8,3,3,2,2" \ -SGLANG_CACHE_DIT_SCM_CACHE_BINS="1,2,2,2,3" \ -sglang generate --model-path Qwen/Qwen-Image \ - --prompt "A futuristic cityscape at sunset" -``` - -**SCM Policy** - -| Policy | Env Variable | Description | -|-----------|---------------------------------------|---------------------------------------------| -| `dynamic` | `SGLANG_CACHE_DIT_SCM_POLICY=dynamic` | Adaptive caching based on content (default) | -| `static` | `SGLANG_CACHE_DIT_SCM_POLICY=static` | Fixed caching pattern | - -## Environment Variables - -All Cache-DiT parameters can be configured via environment variables. -See [Environment Variables](../../environment_variables.md) for the complete list. - -## Supported Models - -SGLang Diffusion x Cache-DiT supports almost all models originally supported in SGLang Diffusion: - -| Model Family | Example Models | -|--------------|-----------------------------| -| Wan | Wan2.1, Wan2.2 | -| Flux | FLUX.1-dev, FLUX.2-dev | -| Z-Image | Z-Image-Turbo | -| Qwen | Qwen-Image, Qwen-Image-Edit | -| Hunyuan | HunyuanVideo | - -## Performance Tips - -1. **Start with defaults**: The default parameters work well for most models -2. **Use TaylorSeer**: It typically improves both speed and quality -3. **Tune R threshold**: Lower values = better quality, higher values = faster -4. **SCM for extra speed**: Use `medium` preset for good speed/quality balance -5. **Warmup matters**: Higher warmup = more stable caching decisions - -## Limitations - -- **SGLang-native pipelines**: Distributed support (TP/SP) is not yet validated; Cache-DiT will be automatically - disabled when `world_size > 1`. -- **SCM minimum steps**: SCM requires >= 8 inference steps to be effective -- **Model support**: Only models registered in Cache-DiT's BlockAdapterRegister are supported - -## Troubleshooting - -### SCM disabled for low step count - -For models with < 8 inference steps (e.g., DMD distilled models), SCM will be automatically disabled. DBCache -acceleration still works. - -## References - -- [Cache-DiT](https://github.com/vipshop/cache-dit) -- [SGLang Diffusion](../index.md) diff --git a/docs/diffusion/performance/cache/index.md b/docs/diffusion/performance/cache/index.md deleted file mode 100644 index c7f8f53ef..000000000 --- a/docs/diffusion/performance/cache/index.md +++ /dev/null @@ -1,65 +0,0 @@ -# Caching Acceleration - -SGLang provides two complementary caching strategies for Diffusion Transformer (DiT) models. Both reduce denoising cost by skipping redundant computation, but they operate at different levels. - -## Overview - -SGLang supports two complementary caching approaches: - -| Strategy | Scope | Mechanism | Best For | -|----------|-------|-----------|----------| -| **Cache-DiT** | Block-level | Skip individual transformer blocks dynamically | Advanced, higher speedup | -| **TeaCache** | Timestep-level | Skip entire denoising steps based on L1 similarity | Simple, built-in | - -## Cache-DiT - -[Cache-DiT](https://github.com/vipshop/cache-dit) provides block-level caching with -advanced strategies like DBCache and TaylorSeer. It can achieve up to **1.69x speedup**. - -See [cache_dit.md](cache_dit.md) for detailed configuration. - -### Quick Start - -```bash -SGLANG_CACHE_DIT_ENABLED=true \ -sglang generate --model-path Qwen/Qwen-Image \ - --prompt "A beautiful sunset over the mountains" -``` - -### Key Features - -- **DBCache**: Dynamic block-level caching based on residual differences -- **TaylorSeer**: Taylor expansion-based calibration for optimized caching -- **SCM**: Step-level computation masking for additional speedup - -## TeaCache - -TeaCache (Temporal similarity-based caching) accelerates diffusion inference by detecting when consecutive denoising steps are similar enough to skip computation entirely. - -See [teacache.md](teacache.md) for detailed documentation. - -### Quick Overview - -- Tracks L1 distance between modulated inputs across timesteps -- When accumulated distance is below threshold, reuses cached residual -- Supports CFG with separate positive/negative caches - -### Supported Models - -- Wan (wan2.1, wan2.2) -- Hunyuan (HunyuanVideo) -- Z-Image - -For Flux and Qwen models, TeaCache is automatically disabled when CFG is enabled. - -```{toctree} -:maxdepth: 1 - -cache_dit -teacache -``` - -## References - -- [Cache-DiT Repository](https://github.com/vipshop/cache-dit) -- [TeaCache Paper](https://arxiv.org/abs/2411.14324) diff --git a/docs/diffusion/performance/cache/teacache.md b/docs/diffusion/performance/cache/teacache.md deleted file mode 100644 index dd9691c43..000000000 --- a/docs/diffusion/performance/cache/teacache.md +++ /dev/null @@ -1,84 +0,0 @@ -# TeaCache - -> **Note**: This is one of two caching strategies available in SGLang. -> For an overview of all caching options, see [caching](../index.md). - -TeaCache (Temporal similarity-based caching) accelerates diffusion inference by detecting when consecutive denoising steps are similar enough to skip computation entirely. - -## Overview - -TeaCache works by: -1. Tracking the L1 distance between modulated inputs across consecutive timesteps -2. Accumulating the rescaled L1 distance over steps -3. When accumulated distance is below a threshold, reusing the cached residual -4. Supporting CFG (Classifier-Free Guidance) with separate positive/negative caches - -## How It Works - -### L1 Distance Tracking - -At each denoising step, TeaCache computes the relative L1 distance between the current and previous modulated inputs: - -``` -rel_l1 = |current - previous|.mean() / |previous|.mean() -``` - -This distance is then rescaled using polynomial coefficients and accumulated: - -``` -accumulated += poly(coefficients)(rel_l1) -``` - -### Cache Decision - -- If `accumulated >= threshold`: Force computation, reset accumulator -- If `accumulated < threshold`: Skip computation, use cached residual - -### CFG Support - -For models that support CFG cache separation (Wan, Hunyuan, Z-Image), TeaCache maintains separate caches for positive and negative branches: -- `previous_modulated_input` / `previous_residual` for positive branch -- `previous_modulated_input_negative` / `previous_residual_negative` for negative branch - -For models that don't support CFG separation (Flux, Qwen), TeaCache is automatically disabled when CFG is enabled. - -## Configuration - -TeaCache is configured via `TeaCacheParams` in the sampling parameters: - -```python -from sglang.multimodal_gen.configs.sample.teacache import TeaCacheParams - -params = TeaCacheParams( - teacache_thresh=0.1, # Threshold for accumulated L1 distance - coefficients=[1.0, 0.0, 0.0], # Polynomial coefficients for L1 rescaling -) -``` - -### Parameters - -| Parameter | Type | Description | -|-----------|------|-------------| -| `teacache_thresh` | float | Threshold for accumulated L1 distance. Lower = more caching, faster but potentially lower quality | -| `coefficients` | list[float] | Polynomial coefficients for L1 rescaling. Model-specific tuning | - -### Model-Specific Configurations - -Different models may have different optimal configurations. The coefficients are typically tuned per-model to balance speed and quality. - -## Supported Models - -TeaCache is built into the following model families: - -| Model Family | CFG Cache Separation | Notes | -|--------------|---------------------|-------| -| Wan (wan2.1, wan2.2) | Yes | Full support | -| Hunyuan (HunyuanVideo) | Yes | To be supported | -| Z-Image | Yes | To be supported | -| Flux | No | To be supported | -| Qwen | No | To be supported | - - -## References - -- [TeaCache: Accelerating Diffusion Models with Temporal Similarity](https://arxiv.org/abs/2411.14324) diff --git a/docs/diffusion/performance/deployment_cookbook.md b/docs/diffusion/performance/deployment_cookbook.md deleted file mode 100644 index d45798b1c..000000000 --- a/docs/diffusion/performance/deployment_cookbook.md +++ /dev/null @@ -1,94 +0,0 @@ -# Deployment Cookbook - -This page gives practical defaults for choosing CPU offload, FSDP, CFG parallelism, SP, and TP. - -## Quick Rule - -Use the simplest setting that fits your memory target: - -| Goal | Recommended setting | -|--------------------------------------------|------------------------------------------------------------------------------------| -| Fastest single-GPU run when the model fits | Disable CPU offload and do not use FSDP. | -| Lower single-GPU memory usage | Use component CPU offload, or layerwise DiT offload for supported Wan/MOVA models. | -| Faster multi-GPU Qwen/Wan CFG generation | Use FSDP with CFG parallelism and disable CPU offload. | -| Sequence length or video-shape scaling | Use SP/Ulysses/Ring when the model benefits from sequence parallelism. | -| TP compatibility or encoder-heavy paths | Set TP explicitly; do not treat TP as the default latency optimization. | - -Base the decision on available memory on the selected GPU(s). - -- For multi-GPU deployment: the least-free selected GPU is the bottleneck. A busy 80GiB GPU can behave like a much smaller GPU. -- For single-GPU deployment: FSDP shards DiT weights across multiple GPUs. It is not useful for keeping a single-GPU deployment on one GPU; for that case use CPU offload. - -## Performance Modes - -`--performance-mode` applies safe presets without overriding explicit offload, FSDP, or parallelism flags. `auto` is the default. Use `manual` when you need to keep performance-related server args under explicit user control. `--mode` is a short alias. - -| Mode | Meaning | -|------------|---------------------------------------------------------------------------------------------------------------------------| -| `manual` | Keeps performance-related server args under explicit user control. | -| `auto` | Default. Keeps legacy safe offload defaults and uses FSDP/CFG only on validated multi-GPU deployments where FSDP can replace DiT offload. | -| `speed` | Favors GPU-resident execution for lower latency and higher throughput. Disables CPU offload when unset; may OOM. | -| `memory` | Favors lower GPU memory. Uses component offload, or Wan/MOVA layerwise DiT offload when supported. | - -`auto` checks selected GPU memory before applying FSDP. In multi-GPU runs it uses the least available memory across selected GPUs, and only turns on FSDP automatically when doing so can replace DiT offload. Text encoder, image encoder, and other component residency still follow the offload policy unless the model marks a high-memory resident path as safe. When the model default uses CFG and the user did not set a parallelism policy, `auto` may also enable CFG parallelism. `speed` intentionally does not check memory; it is the mode for users who prefer latency/throughput and accept OOM risk. - -The modes tune residency for native pipeline components declared to the component residency manager. Today this covers the major DiT, text/image encoder, VAE, vocoder, and upsampler components; DiT can use layerwise offload when supported, while text encoders use either resident execution or component CPU offload. Do not assume text-encoder layerwise offload unless a model implements and validates it. - -NOTE: -The preset is intentionally coarse. A future continuous value such as `0.0` to `1.0` could express the speed-memory tradeoff more precisely, but it would need model-specific memory models and clearer user expectations. Until then, use the preset plus explicit flags for overrides. - -Examples: - -```bash -sglang generate \ - --model-path Qwen/Qwen-Image \ - --num-gpus 2 \ - --performance-mode auto -``` - -```bash -sglang generate \ - --model-path Wan-AI/Wan2.1-T2V-1.3B-Diffusers \ - --performance-mode memory -``` - -Explicit flags win over the mode: - -```bash -sglang generate \ - --model-path Qwen/Qwen-Image \ - --num-gpus 2 \ - --performance-mode auto \ - --use-fsdp-inference false -``` - -In this example, `auto` will not re-enable FSDP. The same applies to parallelism; for example, `--enable-cfg-parallel false` keeps CFG parallelism disabled. - -## Interpreting The Levers - -**No offload** keeps model components resident on GPU. It is usually fastest when memory is sufficient. - -**Component CPU offload** lowers GPU memory by moving large components to CPU. It is simple and robust, but it usually trades latency for memory. - -**Layerwise DiT offload** lowers DiT memory further for supported Wan/MOVA models by moving DiT layers between CPU and GPU. It can be the best single-GPU memory mode, but may increase latency and lower throughput. - -**FSDP** shards DiT weights across multiple GPUs and all-gathers weights during forward. It can reduce DiT CPU offload cost on multi-GPU deployments, especially for validated Wan I2V workloads. - -FSDP sharding granularity matters. SGLang Diffusion prefers sharding direct repeated transformer block entries such as `transformer_blocks.0` or `blocks.0`. Coarser sharding lowers wrapper count but can increase all-gather peak memory; finer sharding can reduce transient memory but adds communication and scheduling overhead. If a model does not define an explicit sharding rule, the loader falls back to repeated block class names and common direct numbered block paths. - -**CFG parallelism** splits positive and negative CFG branches across GPUs. For Qwen/Wan workloads with normal step counts, this is the most reliable multi-GPU speedup observed so far. - -**SP/Ulysses/Ring** splits sequence work. It can help video workloads, but validated Qwen/Wan runs showed CFG parallelism outperforming SP for latency. - -**TP** is supported for compatibility and some model structures, but current measurements do not make it the default latency path for Qwen/Wan. - -## Current Benchmark Takeaways - -Observed regular-scale trends: - -- Z-Image: single-GPU no-offload was faster than FSDP/SP in the tested setting; keep FSDP off unless memory or parallelism requires it. -- Qwen-Image: keep the default non-FSDP path unless a specific FSDP/SP/Ring setting has been benchmarked on the target hardware. -- Wan: FSDP can replace DiT offload on validated multi-GPU workloads, while text/image encoders may still need component offload. Keep model-specific precision checks before making FSDP automatic for a path. -- Component offload mainly reduced memory; it did not improve latency in the tested no-offload-vs-offload runs. - -Always benchmark with your actual resolution, frame count, step count, and GPU type before locking production defaults. diff --git a/docs/diffusion/performance/index.md b/docs/diffusion/performance/index.md deleted file mode 100644 index 0723e15b9..000000000 --- a/docs/diffusion/performance/index.md +++ /dev/null @@ -1,44 +0,0 @@ -# Performance - -This section covers the main performance levers for SGLang Diffusion: attention backends, caching acceleration, and profiling. - -## Overview - -| Optimization | Type | Description | -|--------------|------|-------------| -| **Cache-DiT** | Caching | Block-level caching with DBCache, TaylorSeer, and SCM | -| **TeaCache** | Caching | Timestep-level caching based on temporal similarity | -| **Attention Backends** | Kernel | Optimized attention implementations (FlashAttention, SageAttention, etc.) | -| **Profiling** | Diagnostics | PyTorch Profiler and Nsight Systems guidance | - -## Start Here - -- Use [Attention Backends](attention_backends.md) to choose the best backend for your model and hardware. -- Use [Deployment Cookbook](deployment_cookbook.md) to choose CPU offload, FSDP, CFG parallelism, SP, and TP. -- Use [Caching Acceleration](cache/index.md) to reduce denoising cost with Cache-DiT or TeaCache. -- Use [Profiling](profiling.md) when you need to diagnose a bottleneck rather than guess. - -## Caching at a Glance - -- [Cache-DiT](cache/cache_dit.md) is block-level caching for diffusers pipelines and higher speedup-oriented tuning. -- [TeaCache](cache/teacache.md) is timestep-level caching built into SGLang model families. - -```{toctree} -:maxdepth: 1 - -attention_backends -deployment_cookbook -cache/index -profiling -``` - -## Current Baseline Snapshot - -For Ring SP benchmark details, see: - -- [Ring SP Performance](ring_sp_performance.md) - -## References - -- [Cache-DiT Repository](https://github.com/vipshop/cache-dit) -- [TeaCache Paper](https://arxiv.org/abs/2411.14324) diff --git a/docs/diffusion/performance/profiling.md b/docs/diffusion/performance/profiling.md deleted file mode 100644 index b445138c0..000000000 --- a/docs/diffusion/performance/profiling.md +++ /dev/null @@ -1,136 +0,0 @@ -# Profiling Multimodal Generation - -This guide covers profiling techniques for multimodal generation pipelines in SGLang. - -## PyTorch Profiler - -PyTorch Profiler provides detailed kernel execution time, call stack, and GPU utilization metrics. - -### Denoising Stage Profiling - -Profile the denoising stage with sampled timesteps (default: 5 steps after 1 warmup step): - -```bash -sglang generate \ - --model-path Qwen/Qwen-Image \ - --prompt "A Logo With Bold Large Text: SGL Diffusion" \ - --seed 0 \ - --profile -``` - -**Parameters:** -- `--profile`: Enable profiling for the denoising stage -- `--num-profiled-timesteps N`: Number of timesteps to profile after warmup (default: 5) - - Smaller values reduce trace file size - - Example: `--num-profiled-timesteps 10` profiles 10 steps after 1 warmup step - -### Full Pipeline Profiling - -Profile all pipeline stages (text encoding, denoising, VAE decoding, etc.): - -```bash -sglang generate \ - --model-path Qwen/Qwen-Image \ - --prompt "A Logo With Bold Large Text: SGL Diffusion" \ - --seed 0 \ - --profile \ - --profile-all-stages -``` - -**Parameters:** -- `--profile-all-stages`: Used with `--profile`, profile all pipeline stages instead of just denoising - -### Output Location - -By default, trace files are saved in the ./logs/ directory. - -The exact output file path will be shown in the console output, for example: - -```bash -[mm-dd hh:mm:ss] Saved profiler traces to: /sgl-workspace/sglang/logs/mocked_fake_id_for_offline_generate-5_steps-global-rank0.trace.json.gz -``` - -### View Traces - -Load and visualize trace files at: -- https://ui.perfetto.dev/ (recommended) -- chrome://tracing (Chrome only) - -For large trace files, reduce `--num-profiled-timesteps` or avoid using `--profile-all-stages`. - - -### `--perf-dump-path` (Stage/Step Timing Dump) - -Besides profiler traces, you can also dump a lightweight JSON report that contains: -- stage-level timing breakdown for the full pipeline -- step-level timing breakdown for the denoising stage (per diffusion step) - -This is useful to quickly identify which stage dominates end-to-end latency, and whether denoising steps have uniform runtimes (and if not, which step has an abnormal spike). - -The dumped JSON contains a `denoise_steps_ms` field formatted as an array of objects, each with a `step` key (the step index) and a `duration_ms` key. - -Example: - -```bash -sglang generate \ - --model-path \ - --prompt "" \ - --perf-dump-path perf.json -``` - -## Nsight Systems - -Nsight Systems provides low-level CUDA profiling with kernel details, register usage, and memory access patterns. - -### Installation - -See the [SGLang profiling guide](https://github.com/sgl-project/sglang/blob/main/docs/developer_guide/benchmark_and_profiling.md#profile-with-nsight) for installation instructions. - -### Basic Profiling - -Profile the entire pipeline execution: - -```bash -nsys profile \ - --trace-fork-before-exec=true \ - --cuda-graph-trace=node \ - --force-overwrite=true \ - -o QwenImage \ - sglang generate \ - --model-path Qwen/Qwen-Image \ - --prompt "A Logo With Bold Large Text: SGL Diffusion" \ - --seed 0 -``` - -### Targeted Stage Profiling - -Use `--delay` and `--duration` to capture specific stages and reduce file size: - -```bash -nsys profile \ - --trace-fork-before-exec=true \ - --cuda-graph-trace=node \ - --force-overwrite=true \ - --delay 10 \ - --duration 30 \ - -o QwenImage_denoising \ - sglang generate \ - --model-path Qwen/Qwen-Image \ - --prompt "A Logo With Bold Large Text: SGL Diffusion" \ - --seed 0 -``` - -**Parameters:** -- `--delay N`: Wait N seconds before starting capture (skip initialization overhead) -- `--duration N`: Capture for N seconds (focus on specific stages) -- `--force-overwrite`: Overwrite existing output files - -## Notes - -- **Reduce trace size**: Use `--num-profiled-timesteps` with smaller values or `--delay`/`--duration` with Nsight Systems -- **Stage-specific analysis**: Use `--profile` alone for denoising stage, add `--profile-all-stages` for full pipeline -- **Multiple runs**: Profile with different prompts and resolutions to identify bottlenecks across workloads - -## FAQ - -- If you are profiling `sglang generate` with Nsight Systems and find that the generated profiler file did not capture any CUDA kernels, you can resolve this issue by increasing the model's inference steps to extend the execution time. diff --git a/docs/diffusion/performance/ring_sp_performance.md b/docs/diffusion/performance/ring_sp_performance.md deleted file mode 100644 index 138698bfc..000000000 --- a/docs/diffusion/performance/ring_sp_performance.md +++ /dev/null @@ -1,67 +0,0 @@ -# Ring SP Benchmark: Wan2.2-TI2V-5B (u1r2 vs Baseline) - -This page reports Ring-SP performance for `Wan2.2-TI2V-5B-Diffusers` using: - -- Parallel config: `sp=2, ulysses=1, ring=2` (short: `u1r2`) -- Baseline config: `sp=1, ulysses=1, ring=1` (short: `u1r1`) - -## Benchmark Setup - -- Model: `Wan2.2-TI2V-5B-Diffusers` -- GPU: `48G RTX40 series * 2` - -## Online Serving - -### Ring SP (`u1r2`) - -```bash -sglang serve \ - --model-type diffusion \ - --model-path /model/HuggingFace/Wan-AI/Wan2.2-TI2V-5B-Diffusers \ - --num-gpus 2 --sp-degree 2 --ulysses-degree 1 --ring-degree 2 \ - --port 8898 -``` - -### Baseline (`u1r1`) - -```bash -sglang serve \ - --model-type diffusion \ - --model-path /model/HuggingFace/Wan-AI/Wan2.2-TI2V-5B-Diffusers \ - --num-gpus 1 --sp-degree 1 --ulysses-degree 1 --ring-degree 1 \ - --port 8898 -``` - -## Benchmarks - -### Benchmark Disclaimer - -These benchmarks are provided for reference under one specific setup and command configuration. Actual performance may vary with model settings, runtime environment, and request patterns. - -### Stage Time Breakdown - -| Stage / Metric | `u1r2` (s) | `u1r1` baseline (s) | Speedup | -|---|---:|---:|---:| -| InputValidation | 0.1060 | 0.1029 | 0.97x | -| TextEncoding | 1.3965 | 2.2261 | 1.59x | -| LatentPreparation | 0.0002 | 0.0002 | 1.00x | -| TimestepPreparation | 0.0003 | 0.0004 | 1.33x | -| Denoising | 52.6358 | 71.6785 | 1.36x | -| Decoding | 7.6708 | 13.4314 | 1.75x | -| **Total** | **63.74** | **90.63** | **1.42x** | - -### Memory Usage - -| Memory Metric | `u1r2` (GB) | `u1r1` baseline (GB) | Delta | -|---|---:|---:|---:| -| Peak GPU Memory | 20.07 | 27.40 | -7.33 | -| Peak Allocated | 13.35 | 20.40 | -7.05 | -| Memory Overhead | 6.72 | 7.00 | -0.28 | -| Overhead Ratio | 33.5% | 25.6% | +7.9pp | - -## Summary - -- End-to-end latency improves from `90.63s` to `63.74s` (`1.42x`). -- Main gains come from `Denoising` (`1.36x`) and `Decoding` (`1.75x`). -- Absolute memory usage drops noticeably on Ring-SP (`Peak GPU Memory -7.33GB`, `Peak Allocated -7.05GB`). -- Overhead ratio rises (`+7.9pp`), so future tuning can focus on reducing communication/runtime overhead while preserving the latency gain. diff --git a/docs/diffusion/quantization.md b/docs/diffusion/quantization.md deleted file mode 100644 index dfcd3b3f3..000000000 --- a/docs/diffusion/quantization.md +++ /dev/null @@ -1,447 +0,0 @@ -# Quantization - -SGLang-Diffusion supports quantized transformer checkpoints. In most cases, keep -the base model and the quantized transformer override separate. - -## Quick Reference - -Use these paths: - -- `--model-path`: the base or original model -- `--transformer-path`: a quantized transformers-style transformer component directory that already contains its own `config.json` -- `--transformer-weights-path`: quantized transformer weights provided as a single safetensors file, a sharded safetensors directory, a local path, or a Hugging Face repo ID -- `--quantization`: apply online quantization to unquantized models at load time (activations are quantized dynamically) -- `--quantization-ignored-layers` layer name patterns to keep unquantized (e.g. `attention.to_`) - -Recommended example for pre-quantized checkpoints: - -```bash -sglang generate \ - --model-path black-forest-labs/FLUX.2-dev \ - --transformer-weights-path black-forest-labs/FLUX.2-dev-NVFP4 \ - --prompt "a curious pikachu" -``` - -For quantized transformers-style transformer component folders: - -```bash -sglang generate \ - --model-path /path/to/base-model \ - --transformer-path /path/to/quantized-transformer \ - --prompt "A Logo With Bold Large Text: SGL Diffusion" -``` - -NOTE: Some model-specific integrations also accept a quantized repo or local -directory directly as `--model-path`, but that is a compatibility path. If a -repo contains multiple candidate checkpoints, pass -`--transformer-weights-path` explicitly. - -## Quant Families - -Here, `quant_family` means a checkpoint and loading family with shared CLI -usage and loader behavior. It is not just the numeric precision or a kernel -backend. - -| quant_family | checkpoint form | canonical CLI | supported models | extra dependency | platform / notes | -|------------------|--------------------------------------------------------------------------------------------|------------------------------------------------------|--------------------------------------------------------------|---------------------------------------|-----------------------------------------------------------------------------------------------------------------------| -| `fp8` / `mxfp4` (online quantization) | Unquantized checkpoint (offline via AMD Quark coming soon) | `--quantization {fp8,mxfp4}` | Z-Image-Turbo (validated), others likely work. More support coming soon. | MXFP4: `aiter` on ROCm | MXFP4 requires ROCm and MI350+ (gfx95x). Weights quantized at load time, activations quantized to `fp8` / `mxfp4` dynamically. | -| `fp8` (offline quantization) | Quantized transformer component folder, or safetensors with `quantization_config` metadata | `--transformer-path` or `--transformer-weights-path` | ALL | None | Component-folder and single-file flows are both supported | -| `modelopt-fp8` | Converted ModelOpt FP8 transformer directory or repo with `config.json` | `--transformer-path` | FLUX.1, FLUX.2, Wan2.2, HunyuanVideo, Qwen Image, Qwen Image Edit | None | Serialized config stays `quant_method=modelopt` with `quant_algo=FP8`; `dit_layerwise_offload` is supported and `dit_cpu_offload` stays disabled | -| `modelopt-nvfp4` | Mixed transformer directory/repo with `config.json`, or raw NVFP4 safetensors export/repo | `--transformer-path` for mixed overrides; `--transformer-weights-path` for raw exports | FLUX.1, FLUX.2, Wan2.2 | None | Mixed override repos keep the base model separate; raw exports such as `black-forest-labs/FLUX.2-dev-NVFP4` still use the weights-path flow | -| `nunchaku-svdq` | Pre-quantized Nunchaku transformer weights, usually named `svdq-{int4\|fp4}_r{rank}-...` | `--transformer-weights-path` | Model-specific support such as Qwen-Image, FLUX, and Z-Image | `nunchaku` | SGLang can infer precision and rank from the filename and supports both `int4` and `nvfp4` | -| `msmodelslim` | Pre-quantized msmodelslim transformer weights | `--model-path` | Wan2.2 family | None | Currently only compatible with the Ascend NPU family and supports both `w8a8` and `w4a4` | - -## Online Quantization - -Online quantization applies quantization to unquantized models at load time. This is useful for when pre-quantized checkpoints are not available. - -### FP8 Online Quantization - -Apply FP8 quantization to any unquantized model: - -```bash -sglang generate \ - --model-path Tongyi-MAI/Z-Image-Turbo \ - --quantization fp8 \ - --prompt "a beautiful sunset" \ - --save-output -``` - -### MXFP4 Online Quantization - -MXFP4 provides aggressive 4-bit compression with online quantization. **Note: Requires ROCm and MI350+ (gfx95x) GPU.** - -```bash -sglang generate \ - --model-path Tongyi-MAI/Z-Image-Turbo \ - --quantization mxfp4 \ - --prompt "a beautiful sunset" \ - --save-output -``` -**Note:** Requires `aiter` package with MXFP4 kernel support - -### Skipping Layers - -By default, online quantization quantizes every linear layer in -the transformer. However, `--quantization-ignored-layers` can be used to keep specific layers in their original precision: - -```bash -sglang generate \ - --model-path Tongyi-MAI/Z-Image-Turbo \ - --quantization fp8 \ - --quantization-ignored-layers attention.to_ \ - --prompt "a beautiful sunset" \ - --save-output - -sglang generate \ - --model-path Tongyi-MAI/Z-Image-Turbo \ - --quantization mxfp4 \ - --quantization-ignored-layers attention.to_ \ - --prompt "a beautiful sunset" \ - --save-output -``` - -Each pattern is matched against the full layer prefix (e.g. `layers.0.attention.to_q`). A layer is skipped and left unquantizd if its prefix contains any of the given patterns. - -## Validated ModelOpt Checkpoints - -This section is the canonical support matrix for the nine diffusion ModelOpt -checkpoints currently wired up in SGLang docs and validation coverage. - -Published checkpoints keep the serialized quantization config as -`quant_method=modelopt`; the FP8 vs NVFP4 split below is a documentation label -derived from `quant_algo`. - -Eight of the nine repos live under `lmsys/*`. The FLUX.2 NVFP4 entry keeps the -official `black-forest-labs/FLUX.2-dev-NVFP4` repo. - -| Quant Algo | Base Model | Preferred CLI | HF Repo | Current Scope | Notes | -| --- | --- | --- | --- | --- | --- | -| `FP8` | `black-forest-labs/FLUX.1-dev` | `--transformer-path` | `lmsys/flux1-dev-modelopt-fp8-sglang-transformer` | single-transformer override, deterministic latent/image comparison, H100 benchmark, torch-profiler trace | SGLang converter keeps a validated BF16 fallback set for modulation and FF projection layers; use `--model-id FLUX.1-dev` for local mirrors | -| `FP8` | `black-forest-labs/FLUX.2-dev` | `--transformer-path` | `lmsys/flux2-dev-modelopt-fp8-sglang-transformer` | single-transformer override load and generation path | published SGLang-ready transformer override | -| `FP8` | `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | `--transformer-path` | `lmsys/wan22-t2v-a14b-modelopt-fp8-sglang-transformer` | primary `transformer` quantized, `transformer_2` kept BF16 | primary-transformer-only path; keep `transformer_2` on the base checkpoint, and do not describe this as dual-transformer full-model FP8 unless that path is validated separately | -| `FP8` | `hunyuanvideo-community/HunyuanVideo` | `--transformer-path` | `lmsys/hunyuanvideo-modelopt-fp8-sglang-transformer` | single-transformer override, BF16-vs-FP8 video comparison, H100 benchmark, torch-profiler trace | HunyuanVideo uses different ModelOpt/diffusers and SGLang runtime module names; the converter maps those names before writing FP8 scale tensors and BF16 fallback ignores | -| `FP8` | `Qwen/Qwen-Image` | `--transformer-path` | `lmsys/qwen-image-modelopt-fp8-sglang-transformer` | single-transformer override, BF16-vs-FP8 image comparison, H100 benchmark, torch-profiler trace | shares the Qwen Image FP8 fallback preset; keep `img_in`, `txt_in`, timestep embedder, `norm_out.linear`, `proj_out`, `img_mod`/`txt_mod`, and `img_mlp.net.2` in BF16 | -| `FP8` | `Qwen/Qwen-Image-Edit-2511` | `--transformer-path` | `lmsys/qwen-image-edit-modelopt-fp8-sglang-transformer` | TI2I edit path, BF16-vs-FP8 image comparison, H100 benchmark | shares `QwenImageTransformer2DModel` with Qwen Image and uses the same Qwen Image FP8 fallback preset | -| `NVFP4` | `black-forest-labs/FLUX.1-dev` | `--transformer-path` | `lmsys/flux1-dev-modelopt-nvfp4-sglang-transformer` | mixed BF16+NVFP4 transformer override, correctness validation, 4x RTX 5090 benchmark, torch-profiler trace | use `build_modelopt_nvfp4_transformer.py`; validated builder keeps selected FLUX.1 modules in BF16 and sets `swap_weight_nibbles=false` | -| `NVFP4` | `black-forest-labs/FLUX.2-dev` | `--transformer-weights-path` | `black-forest-labs/FLUX.2-dev-NVFP4` | packed-QKV load path | official raw export repo; validated packed export detection and runtime layout handling | -| `NVFP4` | `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | `--transformer-path` | `lmsys/wan22-t2v-a14b-modelopt-nvfp4-sglang-transformer` | primary `transformer` quantized with ModelOpt NVFP4, `transformer_2` kept BF16 | primary-transformer-only path; keep `transformer_2` on the base checkpoint; the default FP4 GEMM backend is `flashinfer_trtllm` | - -These nine checkpoints are also the intended case set for the B200 diffusion -CI job (`multimodal-gen-test-1-b200`). - -## ModelOpt FP8 - -### Usage Examples - -Converted ModelOpt FP8 checkpoints should be loaded as transformer component -overrides. If the repo or local directory already contains `config.json`, use -`--transformer-path`. - -```bash -sglang generate \ - --model-path black-forest-labs/FLUX.2-dev \ - --transformer-path lmsys/flux2-dev-modelopt-fp8-sglang-transformer \ - --prompt "A Logo With Bold Large Text: SGL Diffusion" \ - --save-output -``` - -```bash -sglang generate \ - --model-path Wan-AI/Wan2.2-T2V-A14B-Diffusers \ - --transformer-path lmsys/wan22-t2v-a14b-modelopt-fp8-sglang-transformer \ - --prompt "a fox walking through neon rain" \ - --save-output -``` - -```bash -sglang generate \ - --model-path hunyuanvideo-community/HunyuanVideo \ - --transformer-path lmsys/hunyuanvideo-modelopt-fp8-sglang-transformer \ - --height 544 --width 960 --num-frames 17 \ - --prompt "A cinematic shot of a red sports car driving through rain at night" \ - --save-output -``` - -```bash -sglang generate \ - --model-path Qwen/Qwen-Image \ - --transformer-path lmsys/qwen-image-modelopt-fp8-sglang-transformer \ - --prompt "A tiny astronaut reading a book under a glass greenhouse" \ - --save-output -``` - -```bash -sglang generate \ - --model-path Qwen/Qwen-Image-Edit-2511 \ - --transformer-path lmsys/qwen-image-edit-modelopt-fp8-sglang-transformer \ - --image-path /path/to/input.png \ - --prompt "Turn the scene into a warm watercolor illustration" \ - --save-output -``` - -### Notes - -- `--transformer-path` is the canonical flag for converted ModelOpt FP8 - transformer component repos or directories that already carry `config.json`. -- If the override repo or local directory contains its own `config.json`, - SGLang reads the quantization config from that override instead of relying on - the base model config. -- `--transformer-weights-path` still works when you intentionally point at raw - weight files or a directory that should be metadata-probed as weights first. -- `dit_layerwise_offload` is supported for ModelOpt FP8 checkpoints. -- `dit_cpu_offload` still stays disabled for ModelOpt FP8 checkpoints. -- The layerwise offload path now preserves the non-contiguous FP8 weight stride - expected by the runtime FP8 GEMM path. -- On disk, the quantization config stays `quant_method=modelopt` with - `quant_algo=FP8`; the `modelopt-fp8` label in this document is a support - family name, not a serialized config key. -- `hunyuanvideo-community/HunyuanVideo` uses the `hunyuan-video` converter - preset. Use `--model-type hunyuan-video` to force it, or rely on - auto-detection from `_class_name=HunyuanVideoTransformer3DModel`. -- The validated HunyuanVideo FP8 fallback preset keeps `context_embedder`, - `x_embedder.proj`, timestep/guidance/text embedder linear layers, - `norm_out.linear`, `proj_out`, double-block modulation linear layers, and - single-block modulation linear layers in BF16. -- HunyuanVideo ModelOpt exports use diffusers module names that do not match - SGLang runtime module names for fused QKV and fused QKV+MLP layers. The - converter maps the names before selecting scale tensors and before writing - the runtime ignore list. -- `Qwen/Qwen-Image` and `Qwen/Qwen-Image-Edit-2511` share the `qwen-image` - converter preset. Use `--model-type qwen-image` to force it, or rely on - auto-detection from `_class_name=QwenImageTransformer2DModel`. -- The validated Qwen Image FP8 fallback preset keeps `img_in`, `txt_in`, - timestep embedder linear layers, `norm_out.linear`, `proj_out`, - `transformer_blocks.*.(img_mod|txt_mod)`, and - `transformer_blocks.*.img_mlp.net.2` in BF16. -- For Qwen Image FP8 conversion, write explicit BF16 fallback tensors before - honoring ModelOpt ignored weights. Otherwise converter stats can report a - fallback while the output checkpoint still retains the source FP8 tensor. -- To build the converted checkpoint yourself from a ModelOpt diffusers export, - use `python -m sglang.multimodal_gen.tools.build_modelopt_fp8_transformer`. - -## ModelOpt NVFP4 - -### Usage Examples - -For mixed ModelOpt NVFP4 transformer overrides that already contain -`config.json`, keep the base model and quantized transformer separate and use -`--transformer-path`: - -```bash -sglang generate \ - --model-path black-forest-labs/FLUX.1-dev \ - --transformer-path lmsys/flux1-dev-modelopt-nvfp4-sglang-transformer \ - --prompt "A Logo With Bold Large Text: SGL Diffusion" \ - --save-output -``` - -For raw NVFP4 exports such as the official FLUX.2 release, use -`--transformer-weights-path`: - -```bash -sglang generate \ - --model-path black-forest-labs/FLUX.2-dev \ - --transformer-weights-path black-forest-labs/FLUX.2-dev-NVFP4 \ - --prompt "A Logo With Bold Large Text: SGL Diffusion" \ - --save-output -``` - -SGLang also supports passing the NVFP4 repo or local directory directly as -`--model-path`: - -```bash -sglang generate \ - --model-path black-forest-labs/FLUX.2-dev-NVFP4 \ - --prompt "A Logo With Bold Large Text: SGL Diffusion" \ - --save-output -``` - -For a dual-transformer Wan2.2 export where only the primary `transformer` -was quantized: - -```bash -sglang generate \ - --model-path Wan-AI/Wan2.2-T2V-A14B-Diffusers \ - --transformer-path lmsys/wan22-t2v-a14b-modelopt-nvfp4-sglang-transformer \ - --prompt "a fox walking through neon rain" \ - --save-output -``` - -### Notes - -- Use `--transformer-path` for mixed ModelOpt NVFP4 transformer repos or local - directories that already include `config.json`. -- Use `--transformer-weights-path` for raw NVFP4 exports, individual - safetensors files, or repo layouts that should be treated as weights first. -- For dual-transformer pipelines such as `Wan2.2-T2V-A14B-Diffusers`, the - primary `--transformer-path` override targets only `transformer`. Use a - per-component override such as `--transformer-2-path` only when you - intentionally want a non-default `transformer_2`. -- On Blackwell, the diffusion ModelOpt NVFP4 path defaults to FlashInfer - TensorRT-LLM FP4 GEMM (`flashinfer_trtllm`). -- Direct `--model-path` loading is a compatibility path for FLUX.2 NVFP4-style - repos or local directories. -- If `--transformer-weights-path` is provided explicitly, it takes precedence - over the compatibility `--model-path` flow. -- For local directories, SGLang first looks for `*-mixed.safetensors`, then - falls back to loading from the directory. -- To force the diffusion ModelOpt FP4 path onto a different FlashInfer - backend, set `SGLANG_DIFFUSION_FLASHINFER_FP4_GEMM_BACKEND`. Supported values - include `flashinfer_cudnn`, `flashinfer_cutlass`, and `flashinfer_trtllm`. -- On disk, the quantization config stays `quant_method=modelopt` with - `quant_algo=NVFP4`; the `modelopt-nvfp4` label here is again a documentation - family name rather than a serialized config key. - -## Nunchaku (SVDQuant) - -### Install - -Install the runtime dependency first: - -```bash -pip install nunchaku -``` - -For platform-specific installation methods and troubleshooting, see the -[Nunchaku installation guide](https://nunchaku.tech/docs/nunchaku/installation/installation.html). - -### File Naming and Auto-Detection - -For Nunchaku checkpoints, `--model-path` should still point to the original -base model, while `--transformer-weights-path` points to the quantized -transformer weights. - -If the basename of `--transformer-weights-path` contains the pattern -`svdq-(int4|fp4)_r{rank}`, SGLang will automatically: -- enable SVDQuant -- infer `--quantization-precision` -- infer `--quantization-rank` - -Examples: - -| checkpoint name fragment | inferred precision | inferred rank | notes | -|--------------------------|--------------------|---------------|-------| -| `svdq-int4_r32` | `int4` | `32` | Standard INT4 checkpoint | -| `svdq-int4_r128` | `int4` | `128` | Higher-quality INT4 checkpoint | -| `svdq-fp4_r32` | `nvfp4` | `32` | `fp4` in the filename maps to CLI value `nvfp4` | -| `svdq-fp4_r128` | `nvfp4` | `128` | Higher-quality NVFP4 checkpoint | - -Common filenames: - -| filename | precision | rank | typical use | -|----------|-----------|------|-------------| -| `svdq-int4_r32-qwen-image.safetensors` | `int4` | `32` | Balanced default | -| `svdq-int4_r128-qwen-image.safetensors` | `int4` | `128` | Quality-focused | -| `svdq-fp4_r32-qwen-image.safetensors` | `nvfp4` | `32` | RTX 50-series / NVFP4 path | -| `svdq-fp4_r128-qwen-image.safetensors` | `nvfp4` | `128` | Quality-focused NVFP4 | -| `svdq-int4_r32-qwen-image-lightningv1.0-4steps.safetensors` | `int4` | `32` | Lightning 4-step | -| `svdq-int4_r128-qwen-image-lightningv1.1-8steps.safetensors` | `int4` | `128` | Lightning 8-step | - -If your checkpoint name does not follow this convention, pass -`--enable-svdquant`, `--quantization-precision`, and `--quantization-rank` -explicitly. - -### Usage Examples - -Recommended auto-detected flow: - -```bash -sglang generate \ - --model-path Qwen/Qwen-Image \ - --transformer-weights-path /path/to/svdq-int4_r32-qwen-image.safetensors \ - --prompt "a beautiful sunset" \ - --save-output -``` - -Manual override when the filename does not encode the quant settings: - -```bash -sglang generate \ - --model-path Qwen/Qwen-Image \ - --transformer-weights-path /path/to/custom_nunchaku_checkpoint.safetensors \ - --enable-svdquant \ - --quantization-precision int4 \ - --quantization-rank 128 \ - --prompt "a beautiful sunset" \ - --save-output -``` - -### Notes - -- `--transformer-weights-path` is the canonical flag for Nunchaku checkpoints. - Older config names such as `quantized_model_path` are treated as - compatibility aliases. -- Auto-detection only happens when the checkpoint basename matches - `svdq-(int4|fp4)_r{rank}`. -- The CLI values are `int4` and `nvfp4`. In filenames, the NVFP4 variant is - written as `fp4`. -- Lightning checkpoints usually expect matching `--num-inference-steps`, such - as `4` or `8`. -- Current runtime validation only allows Nunchaku on NVIDIA CUDA Ampere (SM8x) - or SM12x GPUs. Hopper (SM90) is currently rejected. - -## [ModelSlim](https://gitcode.com/Ascend/msmodelslim) -MindStudio-ModelSlim (msModelSlim) is a model offline quantization compression tool launched by MindStudio and optimized for Ascend hardware. - -- **Installation** - - ```bash - # Clone repo and install msmodelslim: - git clone https://gitcode.com/Ascend/msmodelslim.git - cd msmodelslim - bash install.sh - ``` - -- **Multimodal_sd quantization** - - Download the original floating-point weights of the large model. Taking Wan2.2-T2V-A14B as an example, you can go to [Wan2.2-T2V-A14B](https://modelscope.cn/models/Wan-AI/Wan2.2-T2V-A14B) to obtain the original model weights. Then install other dependencies (related to the model, refer to the modelscope model card). - > Note: You can find pre-quantized validated models on [modelscope/Eco-Tech](https://modelscope.cn/models/Eco-Tech). - - Run quantization using one-click quantization (recommended): - - ```bash - msmodelslim quant \ - --model_path /path/to/wan2_2_float_weights \ - --save_path /path/to/wan2_2_quantized_weights \ - --device npu \ - --model_type Wan2_2 \ - --quant_type w8a8 \ - --trust_remote_code True - ``` - - For more detailed examples of quantization of models, as well as information about their support, see the [examples](https://gitcode.com/Ascend/msmodelslim/blob/master/example/multimodal_sd/README.md) section in ModelSLim repo. - - > Note: SGLang does not support quantized embeddings, please disable this option when quantizing using msmodelslim. - -- **Auto-Detection and different formats** - - For msmodelslim checkpoints, it's enough to specify only ```--model-path```, the detection of quantization occurs automatically for each layer using parsing of `quant_model_description.json` config. - - In the case of `Wan2.2` only `Diffusers` weights storage format are supported, whereas modelslim saves the quantized model in the original `Wan2.2` format, - for conversion in use `python/sglang/multimodal_gen/tools/wan_repack.py` script: - - ```bash - python wan_repack.py \ - --input-path {path_to_quantized_model} \ - --output-path {path_to_converted_model} - ``` - - After that, please copy all files from original `Diffusers` checkpoint (instead of `transformer`/`tranfsormer_2` folders) - -- **Usage Example** - - With auto-detected flow: - - ```bash - sglang generate \ - --model-path Eco-Tech/Wan2.2-T2V-A14B-Diffusers-w8a8 \ - --prompt "a beautiful sunset" \ - --save-output - ``` - -- **Available Quantization Methods**: - - [x] ```W4A4_DYNAMIC``` linear with online quantization of activations - - [x] ```W8A8``` linear with offline quantization of activations - - [x] ```W8A8_DYNAMIC``` linear with online quantization of activations - - [x] ```mxfp8``` linear with online/offline MXFP8 quantization (Ascend A5, CANN ≥ 8.0.RC3; see [Ascend NPU quantization](../platforms/ascend/ascend_npu_quantization.md#diffusion-model-quantization-on-ascend-npu)) diff --git a/docs/diffusion/reference.md b/docs/diffusion/reference.md deleted file mode 100644 index 2005a91c7..000000000 --- a/docs/diffusion/reference.md +++ /dev/null @@ -1,11 +0,0 @@ -# Reference - -Reference material for environment-based configuration and runtime behavior. - -- [Environment Variables](environment_variables.md): platform, caching, cloud storage, and debugging variables - -```{toctree} -:maxdepth: 1 - -environment_variables -``` diff --git a/docs/diffusion/support_new_models.md b/docs/diffusion/support_new_models.md deleted file mode 100644 index 42f33c72b..000000000 --- a/docs/diffusion/support_new_models.md +++ /dev/null @@ -1,388 +0,0 @@ -# How to Support New Diffusion Models - -This document explains how to add support for new diffusion models in SGLang Diffusion. - -## Architecture Overview - -SGLang Diffusion is engineered for both performance and flexibility, built upon a pipeline architecture. This -design allows developers to construct pipelines for various diffusion models while keeping the core generation -loop standardized for optimization. - -At its core, the architecture revolves around two key concepts, as highlighted in our [blog post](https://lmsys.org/blog/2025-11-07-sglang-diffusion/#architecture): - -- **`ComposedPipeline`**: This class orchestrates a series of `PipelineStage`s to define the complete generation process for a specific model. It acts as the main entry point for a model and manages the data flow between the different stages of the diffusion process. -- **`PipelineStage`**: Each stage is a modular component that encapsulates a function within the diffusion process. Examples include prompt encoding, the denoising loop, or VAE decoding. - -### Two Pipeline Styles - -SGLang Diffusion supports two pipeline composition styles. Both are valid; choose the one that best fits your model. - -#### Style A: Hybrid Monolithic Pipeline (Recommended Default) - -The recommended default for most new models. Uses a three-stage structure: - -``` -BeforeDenoisingStage (model-specific) → DenoisingStage (standard) → DecodingStage (standard) -``` - -| Stage | Ownership | Responsibility | -|-------|-----------|----------------| -| `{Model}BeforeDenoisingStage` | Model-specific | All pre-processing: input validation, text/image encoding, latent preparation, timestep computation | -| `DenoisingStage` | Framework-standard | The denoising loop (DiT/UNet forward passes), shared across all models | -| `DecodingStage` | Framework-standard | VAE decoding from latent space to pixel space, shared across all models | - -**Why recommended?** Modern diffusion models often have highly heterogeneous pre-processing requirements — different text encoders, different latent formats, different conditioning mechanisms. The Hybrid approach keeps pre-processing isolated per model, avoids fragile shared stages with excessive conditional logic, and lets developers port Diffusers reference code quickly. - -#### Style B: Modular Composition Style - -Uses the framework's fine-grained standard stages (`TextEncodingStage`, `LatentPreparationStage`, `TimestepPreparationStage`, etc.) to build the pipeline by composition. Convenience methods like `add_standard_t2i_stages()` and `add_standard_ti2i_stages()` make this very concise. - -This style is appropriate when: -- **The new model's pre-processing can largely reuse existing stages** — e.g., a model that uses standard CLIP/T5 text encoding + standard latent preparation with minimal customization. -- **A model-specific optimization needs to be extracted as a standalone stage** — e.g., a specialized encoding or conditioning step that benefits from being a separate stage for profiling, parallelism control, or reuse across multiple pipeline variants. - -#### How to Choose - -| Situation | Recommended Style | -|-----------|-------------------| -| Model has unique/complex pre-processing (VLM captioning, AR token generation, custom latent packing, etc.) | **Hybrid** — consolidate into a BeforeDenoisingStage | -| Model fits neatly into standard text-to-image or text+image-to-image pattern | **Modular** — use `add_standard_t2i_stages()` / `add_standard_ti2i_stages()` | -| Porting a Diffusers pipeline with many custom steps | **Hybrid** — copy the `__call__` logic into a single stage | -| Adding a variant of an existing model that shares most logic | **Modular** — reuse existing stages, customize via PipelineConfig callbacks | -| A specific pre-processing step needs special parallelism or profiling isolation | **Modular** — extract that step as a dedicated stage | - -## Key Components for Implementation - -To add support for a new diffusion model, you will need to define or configure the following components: - -1. **`PipelineConfig`**: A dataclass holding static configurations for your model pipeline — precision settings, model architecture parameters, and callback methods used by the standard `DenoisingStage` and `DecodingStage`. Each model has its own subclass. - -2. **`SamplingParams`**: A dataclass defining runtime generation parameters — `prompt`, `negative_prompt`, `guidance_scale`, `num_inference_steps`, `seed`, `height`, `width`, etc. - -3. **Pre-processing stage(s)**: Either a single model-specific `{Model}BeforeDenoisingStage` (Hybrid style) or a combination of standard stages (Modular style). See [Two Pipeline Styles](#two-pipeline-styles) above. - -4. **`ComposedPipeline`**: A class that wires together your pre-processing stage(s) with the standard `DenoisingStage` and `DecodingStage`. See base definitions: - - [`ComposedPipelineBase`](https://github.com/sgl-project/sglang/blob/main/python/sglang/multimodal_gen/runtime/pipelines_core/composed_pipeline_base.py) - - [`PipelineStage`](https://github.com/sgl-project/sglang/blob/main/python/sglang/multimodal_gen/runtime/pipelines_core/stages/base.py) - - [Central registry](https://github.com/sgl-project/sglang/blob/main/python/sglang/multimodal_gen/registry.py) - -5. **Modules (model components)**: Each pipeline references modules loaded from the model repository (e.g., Diffusers `model_index.json`): - - `text_encoder`: Encodes text prompts into embeddings. - - `tokenizer`: Tokenizes raw text input for the text encoder(s). - - `processor`: Preprocesses images and extracts features; often used in image-to-image tasks. - - `image_encoder`: Specialized image feature extractor. - - `dit/transformer`: The core denoising network (DiT/UNet architecture) operating in latent space. - - `scheduler`: Controls the timestep schedule and denoising dynamics. - - `vae`: Variational Autoencoder for encoding/decoding between pixel space and latent space. - -## Pipeline Stages Reference - -### Core Stages (used by all pipelines) - -| Stage Class | Description | -| -------------------------------- | ------------------------------------------------------------------------------------------------------- | -| `DenoisingStage` | Executes the main denoising loop, iteratively applying the model (DiT/UNet) to refine the latents. | -| `DecodingStage` | Decodes the final latent tensor back into pixel space using the VAE. | -| `DmdDenoisingStage` | A specialized denoising stage for DMD model architectures. | -| `CausalDMDDenoisingStage` | A specialized causal denoising stage for specific video models. | - -### Pre-processing Stages (for Modular Composition Style) - -The following fine-grained stages can be composed to build the pre-processing portion of a pipeline. They are best suited for models whose pre-processing largely fits the standard patterns. If your model requires significant customization, consider the Hybrid style with a single `BeforeDenoisingStage` instead. - -| Stage Class | Description | -| -------------------------------- | ------------------------------------------------------------------------------------------------------- | -| `InputValidationStage` | Validates user-provided `SamplingParams`. | -| `TextEncodingStage` | Encodes text prompts into embeddings using one or more text encoders. | -| `ImageEncodingStage` | Encodes input images into embeddings, often used in image-to-image tasks. | -| `ImageVAEEncodingStage` | Encodes an input image into latent space using the VAE. | -| `TimestepPreparationStage` | Prepares the scheduler's timesteps for the diffusion process. | -| `LatentPreparationStage` | Creates the initial noisy latent tensor that will be denoised. | - -## Implementation Guide - -### Step 1: Obtain and Study the Reference Implementation - -Before writing any code, obtain the model's original implementation or Diffusers pipeline code: -- The model's Diffusers pipeline source (e.g., the `pipeline_*.py` file from the `diffusers` library or HuggingFace repo) -- Or the model's official reference implementation (e.g., from the model author's GitHub repo) -- Or the HuggingFace model ID to look up `model_index.json` and the associated pipeline class - -Once you have the reference code, study it thoroughly: - -1. Find the model's `model_index.json` to identify required modules. -2. Read the Diffusers pipeline's `__call__` method to understand: - - How text prompts are encoded - - How latents are prepared (shape, dtype, scaling) - - How timesteps/sigmas are computed - - What conditioning kwargs the DiT expects - - How the denoising loop works - - How VAE decoding is done - -### Step 2: Evaluate Reuse of Existing Pipelines and Stages - -Before creating any new files, check whether an existing pipeline or stage can be reused or extended. Only create new pipelines/stages when the existing ones would need substantial structural changes or when no architecturally similar implementation exists. - -- **Compare against existing pipelines** (Flux, Wan, Qwen-Image, GLM-Image, HunyuanVideo, LTX, etc.). If the new model shares most of its structure with an existing one, prefer adding a new config variant or reusing existing stages. -- **Check existing stages** in `runtime/pipelines_core/stages/` and `stages/model_specific_stages/`. -- **Check existing model components** — many models share VAEs (e.g., `AutoencoderKL`), text encoders (CLIP, T5), and schedulers. Reuse these directly. - -### Step 3: Implement Model Components - -Adapt the model's core components: - -- **DiT/Transformer**: Implement in [`runtime/models/dits/`](https://github.com/sgl-project/sglang/blob/main/python/sglang/multimodal_gen/runtime/models/dits/) -- **Encoders**: Implement in [`runtime/models/encoders/`](https://github.com/sgl-project/sglang/blob/main/python/sglang/multimodal_gen/runtime/models/encoders/) -- **VAEs**: Implement in [`runtime/models/vaes/`](https://github.com/sgl-project/sglang/blob/main/python/sglang/multimodal_gen/runtime/models/vaes/) -- **Schedulers**: Implement in [`runtime/models/schedulers/`](https://github.com/sgl-project/sglang/blob/main/python/sglang/multimodal_gen/runtime/models/schedulers/) if needed - -Use SGLang's fused kernels where possible (see `LayerNormScaleShift`, `RMSNormScaleShift`, `apply_qk_norm`, etc.). - -**Tensor Parallel (TP) and Sequence Parallel (SP)**: For multi-GPU deployment, it is recommended to add TP/SP support to the DiT model. This can be done incrementally after the single-GPU implementation is verified. Reference implementations: -- **Wan model** (`runtime/models/dits/wanvideo.py`) — Full TP + SP: `ColumnParallelLinear`/`RowParallelLinear` for attention, sequence dimension sharding via `get_sp_world_size()` -- **Qwen-Image model** (`runtime/models/dits/qwen_image.py`) — SP via `USPAttention` (Ulysses + Ring Attention) - -### Step 4: Create Configs - -- **DiT Config**: `configs/models/dits/{model_name}.py` -- **VAE Config**: `configs/models/vaes/{model_name}.py` -- **SamplingParams**: `configs/sample/{model_name}.py` - -### Step 5: Create PipelineConfig - -The `PipelineConfig` provides callbacks that the standard `DenoisingStage` and `DecodingStage` use: - -```python -# python/sglang/multimodal_gen/configs/pipeline_configs/my_model.py - -@dataclass -class MyModelPipelineConfig(ImagePipelineConfig): - task_type: ModelTaskType = ModelTaskType.T2I - vae_precision: str = "bf16" - should_use_guidance: bool = True - dit_config: DiTConfig = field(default_factory=MyModelDitConfig) - vae_config: VAEConfig = field(default_factory=MyModelVAEConfig) - - def get_freqs_cis(self, batch, device, rotary_emb, dtype): - """Prepare rotary position embeddings for the DiT.""" - ... - - def prepare_pos_cond_kwargs(self, batch, latent_model_input, t, **kwargs): - """Build positive conditioning kwargs for each denoising step.""" - return { - "hidden_states": latent_model_input, - "encoder_hidden_states": batch.prompt_embeds[0], - "timestep": t, - } - - def prepare_neg_cond_kwargs(self, batch, latent_model_input, t, **kwargs): - """Build negative conditioning kwargs for CFG.""" - return { - "hidden_states": latent_model_input, - "encoder_hidden_states": batch.negative_prompt_embeds[0], - "timestep": t, - } - - def get_decode_scale_and_shift(self): - """Return (scale, shift) for latent denormalization before VAE decode.""" - ... -``` - -### Step 6: Implement Pre-processing - -Choose based on your model's needs (see [How to Choose](#how-to-choose)): - -#### Option A: BeforeDenoisingStage (Hybrid Style) - -Create a single stage that handles all pre-processing. Best when the model has custom/complex pre-processing logic. - -```python -# python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/my_model.py - -class MyModelBeforeDenoisingStage(PipelineStage): - """Monolithic pre-processing stage for MyModel. - - Consolidates: input validation, text/image encoding, latent - preparation, and timestep computation. - """ - - def __init__(self, vae, text_encoder, tokenizer, transformer, scheduler): - super().__init__() - self.vae = vae - self.text_encoder = text_encoder - self.tokenizer = tokenizer - self.transformer = transformer - self.scheduler = scheduler - - @torch.no_grad() - def forward(self, batch: Req, server_args: ServerArgs) -> Req: - device = get_local_torch_device() - - # 1. Encode prompt (model-specific logic) - prompt_embeds, negative_prompt_embeds = self._encode_prompt(...) - - # 2. Prepare latents - latents = self._prepare_latents(...) - - # 3. Prepare timesteps - timesteps, sigmas = self._prepare_timesteps(...) - - # 4. Populate batch for DenoisingStage - batch.prompt_embeds = [prompt_embeds] - batch.negative_prompt_embeds = [negative_prompt_embeds] - batch.latents = latents - batch.timesteps = timesteps - batch.num_inference_steps = len(timesteps) - batch.sigmas = sigmas.tolist() - batch.generator = generator - batch.raw_latent_shape = latents.shape - return batch -``` - -#### Option B: Standard Stages (Modular Style) - -Skip creating a custom stage entirely — configure via `PipelineConfig` callbacks and use framework helpers. Best when the model fits standard patterns. - -(This option has no separate stage file; the pipeline class in Step 7 calls `add_standard_t2i_stages()` directly.) - -**Key batch fields that `DenoisingStage` expects** (regardless of which option you choose): - -| Field | Type | Description | -|-------|------|-------------| -| `batch.latents` | `torch.Tensor` | Initial noisy latent tensor | -| `batch.timesteps` | `torch.Tensor` | Timestep schedule | -| `batch.num_inference_steps` | `int` | Number of denoising steps | -| `batch.sigmas` | `list[float]` | Sigma schedule (must be a Python list, not numpy) | -| `batch.prompt_embeds` | `list[torch.Tensor]` | Positive prompt embeddings (wrapped in a list) | -| `batch.negative_prompt_embeds` | `list[torch.Tensor]` | Negative prompt embeddings (wrapped in a list) | -| `batch.generator` | `torch.Generator` | RNG generator for reproducibility | -| `batch.raw_latent_shape` | `tuple` | Original latent shape before any packing | - -### Step 7: Define the Pipeline Class - -#### Hybrid Style - -```python -# python/sglang/multimodal_gen/runtime/pipelines/my_model.py - -class MyModelPipeline(LoRAPipeline, ComposedPipelineBase): - pipeline_name = "MyModelPipeline" # Must match model_index.json _class_name - - _required_config_modules = [ - "text_encoder", "tokenizer", "vae", "transformer", "scheduler", - ] - - def create_pipeline_stages(self, server_args: ServerArgs): - # 1. Monolithic pre-processing (model-specific) - self.add_stage( - MyModelBeforeDenoisingStage( - vae=self.get_module("vae"), - text_encoder=self.get_module("text_encoder"), - tokenizer=self.get_module("tokenizer"), - transformer=self.get_module("transformer"), - scheduler=self.get_module("scheduler"), - ), - ) - - # 2. Standard denoising loop (framework-provided) - self.add_stage( - DenoisingStage( - transformer=self.get_module("transformer"), - scheduler=self.get_module("scheduler"), - ), - ) - - # 3. Standard VAE decoding (framework-provided) - self.add_standard_decoding_stage() - - -EntryClass = [MyModelPipeline] -``` - -#### Modular Style - -```python -# python/sglang/multimodal_gen/runtime/pipelines/my_model.py - -class MyModelPipeline(LoRAPipeline, ComposedPipelineBase): - pipeline_name = "MyModelPipeline" - - _required_config_modules = [ - "text_encoder", "tokenizer", "vae", "transformer", "scheduler", - ] - - def create_pipeline_stages(self, server_args: ServerArgs): - # All pre-processing + denoising + decoding in one call - self.add_standard_t2i_stages( - prepare_extra_timestep_kwargs=[prepare_mu], # model-specific hooks - ) - - -EntryClass = [MyModelPipeline] -``` - -### Step 8: Register the Model - -Register your configs in [`registry.py`](https://github.com/sgl-project/sglang/blob/main/python/sglang/multimodal_gen/registry.py): - -```python -register_configs( - model_family="my_model", - sampling_param_cls=MyModelSamplingParams, - pipeline_config_cls=MyModelPipelineConfig, - hf_model_paths=["org/my-model-name"], -) -``` - -The `EntryClass` in your pipeline file is automatically discovered by the registry — no additional registration needed for the pipeline class itself. - -### Step 9: Verify Output Quality - -After implementation, verify that the generated output is not noise. A noisy or garbled output is the most common sign of an incorrect implementation. Common causes include: - -- Incorrect latent scale/shift factors -- Wrong timestep/sigma schedule (order, dtype, or value range) -- Mismatched conditioning kwargs -- Rotary embedding style mismatch (`is_neox_style`) - -Debug by comparing intermediate tensor values against the Diffusers reference pipeline with the same seed. - -## Reference Implementations - -### Hybrid Style - -| Model | Pipeline | BeforeDenoisingStage | PipelineConfig | -|-------|----------|---------------------|----------------| -| GLM-Image | `runtime/pipelines/glm_image.py` | `stages/model_specific_stages/glm_image.py` | `configs/pipeline_configs/glm_image.py` | -| Qwen-Image-Layered | `runtime/pipelines/qwen_image.py` | `stages/model_specific_stages/qwen_image_layered.py` | `configs/pipeline_configs/qwen_image.py` | - -### Modular Style - -| Model | Pipeline | Notes | -|-------|----------|-------| -| Qwen-Image (T2I) | `runtime/pipelines/qwen_image.py` | Uses `add_standard_t2i_stages()` | -| Qwen-Image-Edit | `runtime/pipelines/qwen_image.py` | Uses `add_standard_ti2i_stages()` | -| Flux | `runtime/pipelines/flux.py` | Uses `add_standard_t2i_stages()` with custom `prepare_mu` | -| Wan | `runtime/pipelines/wan_pipeline.py` | Uses `add_standard_ti2v_stages()` | - -## Checklist - -Before submitting your implementation, verify: - -**Common (both styles):** -- [ ] **Pipeline file** at `runtime/pipelines/{model_name}.py` with `EntryClass` -- [ ] **PipelineConfig** at `configs/pipeline_configs/{model_name}.py` -- [ ] **SamplingParams** at `configs/sample/{model_name}.py` -- [ ] **DiT model** at `runtime/models/dits/{model_name}.py` -- [ ] **Model configs** (DiT, VAE) at `configs/models/dits/` and `configs/models/vaes/` -- [ ] **Registry entry** in `registry.py` via `register_configs()` -- [ ] `pipeline_name` matches Diffusers `model_index.json` `_class_name` -- [ ] `_required_config_modules` lists all modules from `model_index.json` -- [ ] `PipelineConfig` callbacks (`prepare_pos_cond_kwargs`, etc.) match the DiT's `forward()` signature -- [ ] Uses framework-standard `DenoisingStage` and `DecodingStage` (not custom denoising loops) -- [ ] **TP/SP support** considered for DiT model (recommended; reference `wanvideo.py` for TP+SP, `qwen_image.py` for USPAttention) -- [ ] **Output quality verified** — generated images/videos are not noise; compared against Diffusers reference output - -**Hybrid style only:** -- [ ] **BeforeDenoisingStage** at `stages/model_specific_stages/{model_name}.py` -- [ ] `BeforeDenoisingStage.forward()` populates all batch fields required by `DenoisingStage` diff --git a/docs/diffusion/usage.md b/docs/diffusion/usage.md deleted file mode 100644 index 78b0a545d..000000000 --- a/docs/diffusion/usage.md +++ /dev/null @@ -1,17 +0,0 @@ -# Usage - -Use this section for day-to-day inference workflows with SGLang Diffusion. - -- [CLI](api/cli.md): run one-off jobs with `sglang generate` or start a server with `sglang serve` -- [OpenAI-Compatible API](api/openai_api.md): request format, endpoints, and SDK examples -- [Post-Processing](api/post_processing.md): frame interpolation and upscaling -- [Quantization](quantization.md): quantized transformer checkpoints and supported quantization families - -```{toctree} -:maxdepth: 1 - -api/cli -api/openai_api -api/post_processing -quantization -``` diff --git a/docs/get_started/install.md b/docs/get_started/install.md deleted file mode 100644 index 091bf4ae7..000000000 --- a/docs/get_started/install.md +++ /dev/null @@ -1,230 +0,0 @@ -# Install SGLang - -You can install SGLang using one of the methods below. -This page primarily applies to common NVIDIA GPU platforms. -For other or newer platforms, please refer to the dedicated pages for [AMD GPUs](../platforms/amd_gpu.md), [Intel Xeon CPUs](../platforms/cpu_server.md), [TPU](../platforms/tpu.md), [NVIDIA DGX Spark](https://lmsys.org/blog/2025-11-03-gpt-oss-on-nvidia-dgx-spark/), [NVIDIA Jetson](../platforms/nvidia_jetson.md), [Ascend NPUs](../platforms/ascend/ascend_npu.md), and [Intel XPU](../platforms/xpu.md). - -## Method 1: With pip or uv - -It is recommended to use uv for faster installation: - -```bash -pip install --upgrade pip -pip install uv -uv pip install sglang -``` - -### For CUDA 13 - -Docker is recommended (see Method 3 note on B300/GB300/CUDA 13). If you do not have Docker access, follow these steps: - -1. Install PyTorch with CUDA 13 support first: -```bash -# Replace X.Y.Z with the version by your SGLang install -uv pip install torch==X.Y.Z torchvision torchaudio --index-url https://download.pytorch.org/whl/cu130 -``` - -2. Install sglang: -```bash -uv pip install sglang -``` - -3. Install the `sglang-kernel` wheel for CUDA 13 from [the sgl-project whl releases](https://github.com/sgl-project/whl/blob/gh-pages/cu130/sglang-kernel/index.html). Replace `X.Y.Z` with the `sglang-kernel` version required by your SGLang install (you can find this by running `uv pip show sglang-kernel`). Examples: -```bash -# x86_64 -uv pip install "https://github.com/sgl-project/whl/releases/download/vX.Y.Z/sglang_kernel-X.Y.Z+cu130-cp310-abi3-manylinux2014_x86_64.whl" - -# aarch64 -uv pip install "https://github.com/sgl-project/whl/releases/download/vX.Y.Z/sglang_kernel-X.Y.Z+cu130-cp310-abi3-manylinux2014_aarch64.whl" -``` - -4. If you encounter `ptxas fatal : Value 'sm_103a' is not defined for option 'gpu-name'` on B300/GB300, fix it with: -```bash -export TRITON_PTXAS_PATH=/usr/local/cuda/bin/ptxas -``` - -### **Quick fixes to common problems** -- If you encounter `OSError: CUDA_HOME environment variable is not set`. Please set it to your CUDA install root with either of the following solutions: - 1. Use `export CUDA_HOME=/usr/local/cuda-` to set the `CUDA_HOME` environment variable. - 2. Install FlashInfer first following [FlashInfer installation doc](https://docs.flashinfer.ai/installation.html), then install SGLang as described above. - -## Method 2: From source - -```bash -# Use the last release branch -git clone -b v0.5.9 https://github.com/sgl-project/sglang.git -cd sglang - -# Install the python packages -pip install --upgrade pip -pip install -e "python" -``` - -**Quick fixes to common problems** - -- If you want to develop SGLang, you can try the dev docker image. Please refer to [setup docker container](../developer_guide/development_guide_using_docker.md#setup-docker-container). The docker image is `lmsysorg/sglang:dev`. - -## Method 3: Using docker - -The docker images are available on Docker Hub at [lmsysorg/sglang](https://hub.docker.com/r/lmsysorg/sglang/tags), built from [Dockerfile](https://github.com/sgl-project/sglang/tree/main/docker). -Replace `` below with your huggingface hub [token](https://huggingface.co/docs/hub/en/security-tokens). - -```bash -docker run --gpus all \ - --shm-size 32g \ - -p 30000:30000 \ - -v ~/.cache/huggingface:/root/.cache/huggingface \ - --env "HF_TOKEN=" \ - --ipc=host \ - lmsysorg/sglang:latest \ - python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --host 0.0.0.0 --port 30000 -``` - -For production deployments, use the `runtime` variant which is significantly smaller (~40% reduction) by excluding build tools and development dependencies: - -```bash -docker run --gpus all \ - --shm-size 32g \ - -p 30000:30000 \ - -v ~/.cache/huggingface:/root/.cache/huggingface \ - --env "HF_TOKEN=" \ - --ipc=host \ - lmsysorg/sglang:latest-runtime \ - python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --host 0.0.0.0 --port 30000 -``` - -You can also find the nightly docker images [here](https://hub.docker.com/r/lmsysorg/sglang/tags?name=nightly). - -Notes: -- On B300/GB300 (SM103) or CUDA 13 environment, we recommend using the nightly image at `lmsysorg/sglang:dev-cu13` or stable image at `lmsysorg/sglang:latest-cu130-runtime`. Please, do not re-install the project as editable inside the docker image, since it will override the version of libraries specified by the cu13 docker image. - -## Method 4: Using Kubernetes - -Please check out [OME](https://github.com/sgl-project/ome), a Kubernetes operator for enterprise-grade management and serving of large language models (LLMs). - -
-More - -1. Option 1: For single node serving (typically when the model size fits into GPUs on one node) - - Execute command `kubectl apply -f docker/k8s-sglang-service.yaml`, to create k8s deployment and service, with llama-31-8b as example. - -2. Option 2: For multi-node serving (usually when a large model requires more than one GPU node, such as `DeepSeek-R1`) - - Modify the LLM model path and arguments as necessary, then execute command `kubectl apply -f docker/k8s-sglang-distributed-sts.yaml`, to create two nodes k8s statefulset and serving service. - -
- -## Method 5: Using docker compose - -
-More - -> This method is recommended if you plan to serve it as a service. -> A better approach is to use the [k8s-sglang-service.yaml](https://github.com/sgl-project/sglang/blob/main/docker/k8s-sglang-service.yaml). - -1. Copy the [compose.yml](https://github.com/sgl-project/sglang/blob/main/docker/compose.yaml) to your local machine -2. Execute the command `docker compose up -d` in your terminal. -
- -## Method 6: Run on Kubernetes or Clouds with SkyPilot - -
-More - -To deploy on Kubernetes or 12+ clouds, you can use [SkyPilot](https://github.com/skypilot-org/skypilot). - -1. Install SkyPilot and set up Kubernetes cluster or cloud access: see [SkyPilot's documentation](https://skypilot.readthedocs.io/en/latest/getting-started/installation.html). -2. Deploy on your own infra with a single command and get the HTTP API endpoint: -
-SkyPilot YAML: sglang.yaml - -```yaml -# sglang.yaml -envs: - HF_TOKEN: null - -resources: - image_id: docker:lmsysorg/sglang:latest - accelerators: A100 - ports: 30000 - -run: | - conda deactivate - python3 -m sglang.launch_server \ - --model-path meta-llama/Llama-3.1-8B-Instruct \ - --host 0.0.0.0 \ - --port 30000 -``` - -
- -```bash -# Deploy on any cloud or Kubernetes cluster. Use --cloud to select a specific cloud provider. -HF_TOKEN= sky launch -c sglang --env HF_TOKEN sglang.yaml - -# Get the HTTP API endpoint -sky status --endpoint 30000 sglang -``` - -3. To further scale up your deployment with autoscaling and failure recovery, check out the [SkyServe + SGLang guide](https://github.com/skypilot-org/skypilot/tree/master/llm/sglang#serving-llama-2-with-sglang-for-more-traffic-using-skyserve). - -
- -## Method 7: Run on AWS SageMaker - -
-More - -To deploy on SGLang on AWS SageMaker, check out [AWS SageMaker Inference](https://aws.amazon.com/sagemaker/ai/deploy) - -Amazon Web Services provide supports for SGLang containers along with routine security patching. For available SGLang containers, check out [AWS SGLang DLCs](https://github.com/aws/deep-learning-containers/blob/master/available_images.md#sglang-containers) - -To host a model with your own container, follow the following steps: - -1. Build a docker container with [sagemaker.Dockerfile](https://github.com/sgl-project/sglang/blob/main/docker/sagemaker.Dockerfile) alongside the [serve](https://github.com/sgl-project/sglang/blob/main/docker/serve) script. -2. Push your container onto AWS ECR. - -
-Dockerfile Build Script: build-and-push.sh - -```bash -#!/bin/bash -AWS_ACCOUNT="" -AWS_REGION="" -REPOSITORY_NAME="" -IMAGE_TAG="" - -ECR_REGISTRY="${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com" -IMAGE_URI="${ECR_REGISTRY}/${REPOSITORY_NAME}:${IMAGE_TAG}" - -echo "Starting build and push process..." - -# Login to ECR -echo "Logging into ECR..." -aws ecr get-login-password --region ${AWS_REGION} | docker login --username AWS --password-stdin ${ECR_REGISTRY} - -# Build the image -echo "Building Docker image..." -docker build -t ${IMAGE_URI} -f sagemaker.Dockerfile . - -echo "Pushing ${IMAGE_URI}" -docker push ${IMAGE_URI} - -echo "Build and push completed successfully!" -``` - -
- -3. Deploy a model for serving on AWS Sagemaker, refer to [deploy_and_serve_endpoint.py](https://github.com/sgl-project/sglang/blob/main/examples/sagemaker/deploy_and_serve_endpoint.py). For more information, check out [sagemaker-python-sdk](https://github.com/aws/sagemaker-python-sdk). - 1. By default, the model server on SageMaker will run with the following command: `python3 -m sglang.launch_server --model-path opt/ml/model --host 0.0.0.0 --port 8080`. This is optimal for hosting your own model with SageMaker. - 2. To modify your model serving parameters, the [serve](https://github.com/sgl-project/sglang/blob/main/docker/serve) script allows for all available options within `python3 -m sglang.launch_server --help` cli by specifying environment variables with prefix `SM_SGLANG_`. - 3. The serve script will automatically convert all environment variables with prefix `SM_SGLANG_` from `SM_SGLANG_INPUT_ARGUMENT` into `--input-argument` to be parsed into `python3 -m sglang.launch_server` cli. - 4. For example, to run [Qwen/Qwen3-0.6B](https://huggingface.co/Qwen/Qwen3-0.6B) with reasoning parser, simply add additional environment variables `SM_SGLANG_MODEL_PATH=Qwen/Qwen3-0.6B` and `SM_SGLANG_REASONING_PARSER=qwen3`. - -
- -## Common Notes - -- [FlashInfer](https://github.com/flashinfer-ai/flashinfer) is the default attention kernel backend. It only supports sm75 and above. If you encounter any FlashInfer-related issues on sm75+ devices (e.g., T4, A10, A100, L4, L40S, H100), please switch to other kernels by adding `--attention-backend triton --sampling-backend pytorch` and open an issue on GitHub. -- To reinstall flashinfer locally, use the following command: `pip3 install --upgrade flashinfer-python --force-reinstall --no-deps` and then delete the cache with `rm -rf ~/.cache/flashinfer`. diff --git a/docs/index.rst b/docs/index.rst deleted file mode 100644 index 4a892226c..000000000 --- a/docs/index.rst +++ /dev/null @@ -1,140 +0,0 @@ -SGLang Documentation -==================== - -.. raw:: html - - Star - Fork - -

- -SGLang is a high-performance serving framework for large language models and multimodal models. -It is designed to deliver low-latency and high-throughput inference across a wide range of setups, from a single GPU to large distributed clusters. -Its core features include: - -- **Fast Runtime**: Provides efficient serving with RadixAttention for prefix caching, a zero-overhead CPU scheduler, prefill-decode disaggregation, speculative decoding, continuous batching, paged attention, tensor/pipeline/expert/data parallelism, structured outputs, chunked prefill, quantization (FP4/FP8/INT4/AWQ/GPTQ), and multi-LoRA batching. -- **Broad Model Support**: Supports a wide range of language models (Llama, Qwen, DeepSeek, Kimi, GLM, GPT, Gemma, Mistral, etc.), embedding models (e5-mistral, gte, mcdse), reward models (Skywork), and diffusion models (WAN, Qwen-Image), with easy extensibility for adding new models. Compatible with most Hugging Face models and OpenAI APIs. -- **Extensive Hardware Support**: Runs on NVIDIA GPUs (GB200/B300/H100/A100/Spark/5090), AMD GPUs (MI355/MI300), Intel Xeon CPUs, Google TPUs, Ascend NPUs, and more. -- **Active Community**: SGLang is open-source and supported by a vibrant community with widespread industry adoption, powering over 400,000 GPUs worldwide. -- **RL & Post-Training Backbone**: SGLang is a proven rollout backend used for training many frontier models, with native RL integrations and adoption by well-known post-training frameworks such as AReaL, Miles, slime, Tunix, verl and more. - -.. toctree:: - :maxdepth: 1 - :caption: Get Started - - get_started/install.md - -.. toctree:: - :maxdepth: 1 - :caption: Basic Usage - - basic_usage/send_request.ipynb - basic_usage/openai_api.rst - basic_usage/ollama_api.md - basic_usage/offline_engine_api.ipynb - basic_usage/native_api.ipynb - basic_usage/sampling_params.md - basic_usage/popular_model_usage.rst - -.. toctree:: - :maxdepth: 1 - :caption: Advanced Features - - advanced_features/server_arguments.md - advanced_features/object_storage.md - advanced_features/hyperparameter_tuning.md - advanced_features/attention_backend.md - advanced_features/speculative_decoding.ipynb - advanced_features/adaptive_speculative_decoding.md - advanced_features/structured_outputs.ipynb - advanced_features/structured_outputs_for_reasoning_models.ipynb - advanced_features/tool_parser.ipynb - advanced_features/separate_reasoning.ipynb - advanced_features/quantization.md - advanced_features/quantized_kv_cache.md - advanced_features/expert_parallelism.md - advanced_features/dp_dpa_smg_guide.md - advanced_features/lora.ipynb - advanced_features/pd_disaggregation.md - advanced_features/epd_disaggregation.md - advanced_features/pipeline_parallelism.md - advanced_features/hicache.rst - advanced_features/pd_multiplexing.md - advanced_features/vlm_query.ipynb - advanced_features/dp_for_multi_modal_encoder.md - advanced_features/cuda_graph_for_multi_modal_encoder.md - advanced_features/piecewise_cuda_graph.md - advanced_features/breakable_cuda_graph.md - advanced_features/sgl_model_gateway.md - advanced_features/deterministic_inference.md - advanced_features/observability.md - advanced_features/checkpoint_engine.md - advanced_features/sglang_for_rl.md - -.. toctree:: - :maxdepth: 2 - :caption: Supported Models - - supported_models/text_generation/index - supported_models/retrieval_ranking/index - supported_models/specialized/index - supported_models/extending/index - -.. toctree:: - :maxdepth: 2 - :caption: SGLang Diffusion - - diffusion/index - diffusion/installation - diffusion/compatibility_matrix - diffusion/api/cli - diffusion/api/openai_api - diffusion/performance/index - diffusion/performance/ring_sp_performance - diffusion/performance/attention_backends - diffusion/performance/cache/index - diffusion/quantization - diffusion/contributing - -.. toctree:: - :maxdepth: 1 - :caption: Hardware Platforms - - platforms/amd_gpu.md - platforms/cpu_server.md - platforms/tpu.md - platforms/nvidia_jetson.md - platforms/ascend/ascend_npu_support.rst - platforms/xpu.md - -.. toctree:: - :maxdepth: 1 - :caption: Developer Guide - - developer_guide/contribution_guide.md - developer_guide/development_guide_using_docker.md - developer_guide/development_jit_kernel_guide.md - developer_guide/benchmark_and_profiling.md - developer_guide/bench_serving.md - developer_guide/evaluating_new_models.md - -.. toctree:: - :maxdepth: 1 - :caption: References - - references/faq.md - references/environment_variables.md - references/production_metrics.md - references/production_request_trace.md - references/multi_node_deployment/multi_node_index.rst - references/custom_chat_template.md - references/frontend/frontend_index.rst - references/post_training_integration.md - references/release_lookup - references/learn_more.md - -.. toctree:: - :maxdepth: 1 - :caption: Security Acknowledgement - - security/acknowledgements.md diff --git a/docs/performance_dashboard/README.md b/docs/performance_dashboard/README.md deleted file mode 100644 index 857dc26a8..000000000 --- a/docs/performance_dashboard/README.md +++ /dev/null @@ -1,147 +0,0 @@ -# SGLang Performance Dashboard - -A web-based dashboard for visualizing SGLang nightly test performance metrics. - -## Features - -- **Performance Trends**: View throughput, latency, and TTFT trends over time -- **Model Comparison**: Compare performance across different models and configurations -- **Filtering**: Filter by GPU configuration, model, variant, and batch size -- **Interactive Charts**: Zoom, pan, and hover for detailed metrics -- **Run History**: View recent benchmark runs with links to GitHub Actions - -## Quick Start - -### Option 1: Run with Local Server (Recommended) - -For live data from GitHub Actions artifacts: - -```bash -# Install requirements -pip install requests - -# Run the server -python server.py --fetch-on-start - -# Visit http://localhost:8000 -``` - -The server provides: -- Automatic fetching of metrics from GitHub -- Caching to reduce API calls -- `/api/metrics` endpoint for the frontend - -### Option 2: Fetch Data Manually - -Use the fetch script to download metrics data: - -```bash -# Fetch last 30 days of metrics -python fetch_metrics.py --output metrics_data.json - -# Fetch a specific run -python fetch_metrics.py --run-id 21338741812 --output single_run.json - -# Fetch only scheduled (nightly) runs -python fetch_metrics.py --scheduled-only --days 7 -``` - -## GitHub Token - -To download artifacts from GitHub, you need authentication: - -1. **Using `gh` CLI** (recommended): - ```bash - gh auth login - ``` - -2. **Using environment variable**: - ```bash - export GITHUB_TOKEN=your_token_here - ``` - -Without a token, the dashboard will show run metadata but not detailed benchmark results. - -## Data Structure - -The metrics JSON has this structure: - -```json -{ - "run_id": "21338741812", - "run_date": "2026-01-25T22:24:02.090218+00:00", - "commit_sha": "5cdb391...", - "branch": "main", - "results": [ - { - "gpu_config": "8-gpu-h200", - "partition": 0, - "model": "deepseek-ai/DeepSeek-V3.1", - "variant": "TP8+MTP", - "benchmarks": [ - { - "batch_size": 1, - "input_len": 4096, - "output_len": 512, - "latency_ms": 2400.72, - "input_throughput": 21408.64, - "output_throughput": 231.74, - "overall_throughput": 1919.43, - "ttft_ms": 191.32, - "acc_length": 3.19 - } - ] - } - ] -} -``` - -## Deployment - -### GitHub Pages - -The dashboard can be deployed to GitHub Pages for public access: - -1. Copy the dashboard files to `docs/performance_dashboard/` -2. Enable GitHub Pages in repository settings -3. Set up a GitHub Action to periodically update metrics data - -### Self-Hosted - -For a self-hosted deployment with live data: - -1. Set up a server running `server.py` -2. Configure a cron job or systemd timer to refresh data -3. Optionally put behind nginx/caddy for SSL - -## Metrics Explained - -- **Overall Throughput**: Total tokens (input + output) processed per second -- **Input Throughput**: Input tokens processed per second (prefill speed) -- **Output Throughput**: Output tokens generated per second (decode speed) -- **Latency**: End-to-end time to complete the request -- **TTFT**: Time to First Token - time until the first output token -- **Acc Length**: Acceptance length for speculative decoding (MTP variants) - -## Contributing - -To add support for new metrics or visualizations: - -1. Update `fetch_metrics.py` if data collection needs changes -2. Modify `app.js` to add new chart types or filters -3. Update `index.html` for UI changes - -## Troubleshooting - -**No data displayed** -- Check browser console for errors -- Verify GitHub API is accessible -- Try running with `server.py --fetch-on-start` - -**API rate limits** -- Use a GitHub token for higher limits -- The server caches data for 5 minutes - -**Charts not rendering** -- Ensure Chart.js is loading from CDN -- Check for JavaScript errors in console diff --git a/docs/performance_dashboard/app.js b/docs/performance_dashboard/app.js deleted file mode 100644 index 8bfb12b2e..000000000 --- a/docs/performance_dashboard/app.js +++ /dev/null @@ -1,1056 +0,0 @@ -// SGLang Performance Dashboard Application - -const GITHUB_REPO = 'sgl-project/sglang'; -const WORKFLOW_NAME = 'nightly-test-nvidia.yml'; -const ARTIFACT_PREFIX = 'consolidated-metrics-'; - -// Chart instances (array for batch-separated charts) -let activeCharts = []; - -// Data storage -let allMetricsData = []; -let currentModel = null; -let currentMetricType = 'throughput'; // throughput, latency, ttft, inputThroughput - -// Metric type definitions -const metricTypes = { - // Text/VLM metrics - throughput: { label: 'Overall Throughput', unit: 'tokens/sec', field: 'throughput', type: 'text' }, - outputThroughput: { label: 'Output Throughput', unit: 'tokens/sec', field: 'outputThroughput', type: 'text' }, - inputThroughput: { label: 'Input Throughput', unit: 'tokens/sec', field: 'inputThroughput', type: 'text' }, - latency: { label: 'Latency', unit: 'ms', field: 'latency', type: 'text' }, - ttft: { label: 'Time to First Token', unit: 'ms', field: 'ttft', type: 'text' }, - accLength: { label: 'Accept Length', unit: 'tokens', field: 'accLength', filterInvalid: true, type: 'text' }, - // Diffusion metrics - e2eMs: { label: 'End-to-End Time', unit: 'ms', field: 'e2e_ms', type: 'diffusion' }, - avgDenoiseMs: { label: 'Avg Denoise Time', unit: 'ms', field: 'avg_denoise_ms', type: 'diffusion' }, - medianDenoiseMs: { label: 'Median Denoise Time', unit: 'ms', field: 'median_denoise_ms', type: 'diffusion' } -}; - -// Chart.js default configuration for dark theme -Chart.defaults.color = '#94a3b8'; -Chart.defaults.borderColor = '#1e293b'; - -const chartColors = [ - '#22d3ee', '#34d399', '#fbbf24', '#f87171', '#a78bfa', - '#67e8f9', '#6ee7b7', '#fcd34d', '#fca5a5', '#c4b5fd' -]; - -// Initialize the dashboard -async function init() { - try { - await loadData(); - document.getElementById('loading').style.display = 'none'; - document.getElementById('content').style.display = 'block'; - populateFilters(); - updateStats(); - updateCharts(); - updateRunsTable(); - } catch (error) { - console.error('Failed to initialize dashboard:', error); - document.getElementById('loading').style.display = 'none'; - document.getElementById('error').style.display = 'block'; - document.getElementById('error-message').textContent = error.message; - } -} - -// Load data from local server API or GitHub -async function loadData() { - // Try local server API first (if running server.py) - try { - const response = await fetch('/api/metrics', { headers: getAuthHeaders() }); - if (response.ok) { - const data = await response.json(); - if (data.length > 0 && data[0].results && data[0].results.length > 0) { - allMetricsData = data; - console.log(`Loaded ${data.length} records from local API`); - allMetricsData.sort((a, b) => new Date(b.run_date) - new Date(a.run_date)); - return; - } - } - } catch (error) { - console.log('Local API not available, trying GitHub API'); - } - - // Try to load from GitHub API - const runs = await fetchWorkflowRuns(); - const metricsPromises = runs.map(run => fetchMetricsForRun(run)); - const results = await Promise.allSettled(metricsPromises); - - allMetricsData = results - .filter(r => r.status === 'fulfilled' && r.value !== null) - .map(r => r.value); - - if (allMetricsData.length === 0) { - throw new Error('No metrics data available. Please run the server.py with --fetch-on-start to fetch data from GitHub.'); - } - - // Sort by date descending - allMetricsData.sort((a, b) => new Date(b.run_date) - new Date(a.run_date)); -} - -// Fetch workflow runs from GitHub API -async function fetchWorkflowRuns() { - const response = await fetch( - `https://api.github.com/repos/${GITHUB_REPO}/actions/workflows/${WORKFLOW_NAME}/runs?status=completed&per_page=30`, - { - headers: { - 'Accept': 'application/vnd.github.v3+json' - } - } - ); - - if (!response.ok) { - throw new Error(`GitHub API error: ${response.status}`); - } - - const data = await response.json(); - return data.workflow_runs || []; -} - -// Fetch metrics artifact for a specific run -async function fetchMetricsForRun(run) { - try { - // Get artifacts for this run - const artifactsResponse = await fetch( - `https://api.github.com/repos/${GITHUB_REPO}/actions/runs/${run.id}/artifacts`, - { - headers: { - 'Accept': 'application/vnd.github.v3+json' - } - } - ); - - if (!artifactsResponse.ok) return null; - - const artifactsData = await artifactsResponse.json(); - const metricsArtifact = artifactsData.artifacts.find( - a => a.name.startsWith(ARTIFACT_PREFIX) - ); - - if (!metricsArtifact) return null; - - // Note: GitHub API doesn't allow direct artifact download without authentication - // For public access, we would need to use a proxy or pre-process the data - // For now, return run metadata - in production, use a backend to fetch artifacts - return { - run_id: run.id.toString(), - run_date: run.created_at, - commit_sha: run.head_sha, - branch: run.head_branch, - artifact_id: metricsArtifact.id, - results: [] // Would be populated from artifact content - }; - } catch (error) { - console.warn(`Failed to fetch metrics for run ${run.id}:`, error); - return null; - } -} - -// Helper function to detect if result is diffusion type -function isDiffusionResult(result) { - return result.test_type === 'diffusion' || (result.tests && !result.benchmarks); -} - -// Populate filter dropdowns -function populateFilters() { - const gpuConfigs = new Set(); - const models = new Set(); - const testNames = new Set(); // For diffusion tests - const batchSizes = new Set(); - const ioLengths = new Set(); - - allMetricsData.forEach(run => { - run.results.forEach(result => { - gpuConfigs.add(result.gpu_config); - - // Handle diffusion results - if (isDiffusionResult(result)) { - models.add(result.test_suite || 'diffusion'); - if (result.tests) { - result.tests.forEach(test => { - testNames.add(test.test_name); - }); - } - } - // Handle text/VLM results - else { - models.add(result.model); - // Try new structure first (benchmarks_by_io_len), fall back to flat benchmarks - if (result.benchmarks_by_io_len) { - Object.entries(result.benchmarks_by_io_len).forEach(([ioKey, ioData]) => { - ioLengths.add(ioKey); - ioData.benchmarks.forEach(bench => { - batchSizes.add(bench.batch_size); - }); - }); - } else if (result.benchmarks) { - result.benchmarks.forEach(bench => { - batchSizes.add(bench.batch_size); - if (bench.input_len && bench.output_len) { - ioLengths.add(`${bench.input_len}_${bench.output_len}`); - } - }); - } - } - }); - }); - - // No "all" option for GPU and Model - populate with first value selected - const gpuArray = Array.from(gpuConfigs).sort(); - const modelArray = Array.from(models).sort(); - - populateSelectNoAll('gpu-filter', gpuArray); - populateSelectNoAll('model-filter', modelArray); - populateSelect('batch-filter', Array.from(batchSizes).sort((a, b) => a - b)); - populateSelectWithLabels('io-len-filter', sortIoLengths(Array.from(ioLengths)), formatIoLenLabel); - - // Set initial values (first option) - if (gpuArray.length > 0) { - document.getElementById('gpu-filter').value = gpuArray[0]; - } - if (modelArray.length > 0) { - document.getElementById('model-filter').value = modelArray[0]; - currentModel = modelArray[0]; - } - - // Update variants based on selected model - updateVariantFilter(); - // Update IO length filter based on selected GPU/model - updateIoLenFilter(); - - // Create metric type tabs - createMetricTabs(); -} - -// Format input/output length key for display -function formatIoLenLabel(ioKey) { - if (!ioKey) return 'Unknown'; - const parts = ioKey.split('_'); - if (parts.length === 2) { - return `In: ${parts[0]}, Out: ${parts[1]}`; - } - return ioKey; -} - -// Sort IO length keys numerically (by input length, then output length) -function sortIoLengths(ioLengths) { - return ioLengths.filter(key => key && key.includes('_')).sort((a, b) => { - const [aIn, aOut] = a.split('_').map(Number); - const [bIn, bOut] = b.split('_').map(Number); - if (isNaN(aIn) || isNaN(bIn)) return 0; - return (aIn - bIn) || (aOut - bOut); - }); -} - -// Populate select with custom label formatting -function populateSelectWithLabels(selectId, options, labelFormatter) { - const select = document.getElementById(selectId); - options.forEach(option => { - const opt = document.createElement('option'); - opt.value = option; - opt.textContent = labelFormatter ? labelFormatter(option) : option; - select.appendChild(opt); - }); -} - -// Update IO length filter based on selected GPU and model -function updateIoLenFilter() { - const gpuFilterEl = document.getElementById('gpu-filter'); - const modelFilterEl = document.getElementById('model-filter'); - const ioLenSelect = document.getElementById('io-len-filter'); - if (!gpuFilterEl || !modelFilterEl || !ioLenSelect) return; - - const gpuFilter = gpuFilterEl.value; - const modelFilter = modelFilterEl.value; - - const ioLengths = new Set(); - - allMetricsData.forEach(run => { - run.results.forEach(result => { - if (result.gpu_config === gpuFilter && result.model === modelFilter) { - if (result.benchmarks_by_io_len) { - Object.keys(result.benchmarks_by_io_len).forEach(ioKey => { - ioLengths.add(ioKey); - }); - } else if (result.benchmarks) { - result.benchmarks.forEach(bench => { - if (bench.input_len && bench.output_len) { - ioLengths.add(`${bench.input_len}_${bench.output_len}`); - } - }); - } - } - }); - }); - - const ioLenArray = sortIoLengths(Array.from(ioLengths)); - const currentIoLen = ioLenSelect.value; - - // Clear and repopulate - ioLenSelect.innerHTML = ''; - ioLenArray.forEach(ioLen => { - const opt = document.createElement('option'); - opt.value = ioLen; - opt.textContent = formatIoLenLabel(ioLen); - ioLenSelect.appendChild(opt); - }); - - // Try to restore previous selection if still valid - if (ioLenArray.includes(currentIoLen)) { - ioLenSelect.value = currentIoLen; - } else { - ioLenSelect.value = 'all'; - } -} - -// Update variant filter based on selected GPU and model -function updateVariantFilter() { - const gpuFilter = document.getElementById('gpu-filter').value; - const modelFilter = document.getElementById('model-filter').value; - - const variants = new Set(); - - allMetricsData.forEach(run => { - run.results.forEach(result => { - if (result.gpu_config === gpuFilter && result.model === modelFilter) { - // Use 'default' for null/undefined variants - variants.add(result.variant || 'default'); - } - }); - }); - - const variantArray = Array.from(variants).sort(); - const variantSelect = document.getElementById('variant-filter'); - const currentVariant = variantSelect.value; - - // Clear and repopulate - variantSelect.innerHTML = ''; - variantArray.forEach(variant => { - const opt = document.createElement('option'); - opt.value = variant; - opt.textContent = variant; - variantSelect.appendChild(opt); - }); - - // Try to restore previous selection if still valid - if (variantArray.includes(currentVariant)) { - variantSelect.value = currentVariant; - } else { - variantSelect.value = 'all'; - } -} - -function populateSelect(selectId, options) { - const select = document.getElementById(selectId); - options.forEach(option => { - const opt = document.createElement('option'); - opt.value = option; - opt.textContent = option; - select.appendChild(opt); - }); -} - -function populateSelectNoAll(selectId, options) { - const select = document.getElementById(selectId); - // Remove the "all" option if present - while (select.options.length > 0) { - select.remove(0); - } - options.forEach(option => { - const opt = document.createElement('option'); - opt.value = option; - opt.textContent = option; - select.appendChild(opt); - }); -} - -function createMetricTabs() { - const tabsContainer = document.getElementById('metric-tabs'); - tabsContainer.innerHTML = ''; - - // Detect if current data is diffusion or text - const isDiffusion = detectCurrentDataType() === 'diffusion'; - const dataType = isDiffusion ? 'diffusion' : 'text'; - - // Filter metrics based on data type - const relevantMetrics = Object.entries(metricTypes).filter(([key, metric]) => - metric.type === dataType - ); - - relevantMetrics.forEach(([key, metric], index) => { - const tab = document.createElement('div'); - tab.className = index === 0 ? 'tab active' : 'tab'; - tab.textContent = metric.label; - tab.dataset.metric = key; - tab.onclick = () => selectMetricTab(key, tab); - tabsContainer.appendChild(tab); - }); - - // Set initial metric type - if (relevantMetrics.length > 0) { - currentMetricType = relevantMetrics[0][0]; - } -} - -function detectCurrentDataType() { - // Check if currently selected model/GPU config has diffusion data - const gpuFilter = document.getElementById('gpu-filter')?.value; - const modelFilter = currentModel; - - if (!gpuFilter || !modelFilter) return 'text'; - - for (const run of allMetricsData) { - for (const result of run.results) { - if (result.gpu_config === gpuFilter) { - const resultModel = result.test_suite || result.model; - if (resultModel === modelFilter && isDiffusionResult(result)) { - return 'diffusion'; - } - } - } - } - return 'text'; -} - -function selectMetricTab(metricKey, tabElement) { - document.querySelectorAll('.tab').forEach(t => t.classList.remove('active')); - tabElement.classList.add('active'); - currentMetricType = metricKey; - - // Update chart title - const metric = metricTypes[metricKey]; - document.getElementById('metric-title').textContent = `${metric.label} (${metric.unit})`; - - updateCharts(); -} - -// Handle model filter dropdown change -function handleModelFilterChange(model) { - currentModel = model; - // Update variant filter based on new model selection - updateVariantFilter(); - // Update IO length filter based on new model selection - updateIoLenFilter(); - // Recreate metric tabs in case data type changed (text vs diffusion) - createMetricTabs(); - updateCharts(); -} - -// Handle GPU filter change -function handleGpuFilterChange() { - // Update variant filter based on new GPU selection - updateVariantFilter(); - // Update IO length filter based on new GPU selection - updateIoLenFilter(); - // Recreate metric tabs in case data type changed (text vs diffusion) - createMetricTabs(); - updateCharts(); -} - -// Update summary stats -function updateStats() { - const statsRow = document.getElementById('stats-row'); - const latestRun = allMetricsData[0]; - - if (!latestRun) { - statsRow.innerHTML = ''; - const noDataDiv = document.createElement('div'); - noDataDiv.className = 'no-data'; - noDataDiv.textContent = 'No data available'; - statsRow.appendChild(noDataDiv); - return; - } - - const totalModels = new Set(latestRun.results.map(r => r.model)).size; - const totalBenchmarks = latestRun.results.reduce((sum, r) => { - // Count benchmarks from either structure - if (r.benchmarks_by_io_len) { - return sum + Object.values(r.benchmarks_by_io_len).reduce( - (ioSum, ioData) => ioSum + ioData.benchmarks.length, 0 - ); - } - return sum + (r.benchmarks ? r.benchmarks.length : 0); - }, 0); - - statsRow.innerHTML = ''; // Clear previous stats - - const addStat = (label, value) => { - const card = document.createElement('div'); - card.className = 'stat-card'; - const labelEl = document.createElement('div'); - labelEl.className = 'label'; - labelEl.textContent = label; - const valueEl = document.createElement('div'); - valueEl.className = 'value'; - valueEl.textContent = value; - card.appendChild(labelEl); - card.appendChild(valueEl); - statsRow.appendChild(card); - }; - - addStat('Total Runs', allMetricsData.length); - addStat('Models Tested', totalModels); - addStat('Benchmarks', totalBenchmarks); -} - -// Update charts based on current filters and selected metric type -function updateCharts() { - const gpuFilter = document.getElementById('gpu-filter').value; - const modelFilter = currentModel; - const variantFilter = document.getElementById('variant-filter').value; - const ioLenFilter = document.getElementById('io-len-filter').value; - const batchFilter = document.getElementById('batch-filter').value; - - // Prepare data for charts - grouped by batch size - const chartDataByBatch = prepareChartDataByBatch(gpuFilter, modelFilter, variantFilter, ioLenFilter, batchFilter); - - // Update chart for the selected metric type - updateMetricChart(chartDataByBatch, currentMetricType); -} - -function prepareChartData(gpuFilter, modelFilter, variantFilter, ioLenFilter, batchFilter) { - const seriesMap = new Map(); - - allMetricsData.forEach(run => { - const runDate = new Date(run.run_date); - - run.results.forEach(result => { - // Apply filters - if (result.gpu_config !== gpuFilter) return; - if (result.model !== modelFilter) return; - if (variantFilter !== 'all' && result.variant !== variantFilter) return; - - // Helper function to process a benchmark entry - const processBenchmark = (bench, ioKey) => { - if (batchFilter !== 'all' && bench.batch_size !== parseInt(batchFilter)) return; - - const ioLabel = ioKey ? `, ${formatIoLenLabel(ioKey)}` : ''; - const seriesKey = `${result.model.split('/').pop()} (${result.variant}, BS=${bench.batch_size}${ioLabel})`; - - if (!seriesMap.has(seriesKey)) { - seriesMap.set(seriesKey, { - label: seriesKey, - data: [], - model: result.model, - variant: result.variant, - batchSize: bench.batch_size, - ioKey: ioKey - }); - } - - seriesMap.get(seriesKey).data.push({ - x: runDate, - throughput: bench.overall_throughput, - outputThroughput: bench.output_throughput, - latency: bench.latency_ms, - ttft: bench.ttft_ms, - inputThroughput: bench.input_throughput, - accLength: bench.acc_length, - runId: run.run_id - }); - }; - - // Use benchmarks_by_io_len if available - if (result.benchmarks_by_io_len) { - Object.entries(result.benchmarks_by_io_len).forEach(([ioKey, ioData]) => { - if (ioLenFilter !== 'all' && ioKey !== ioLenFilter) return; - ioData.benchmarks.forEach(bench => processBenchmark(bench, ioKey)); - }); - } else if (result.benchmarks) { - result.benchmarks.forEach(bench => { - const benchIoKey = bench.input_len && bench.output_len - ? `${bench.input_len}_${bench.output_len}` - : null; - if (ioLenFilter !== 'all' && benchIoKey !== ioLenFilter) return; - processBenchmark(bench, benchIoKey); - }); - } - }); - }); - - // Sort data points by date - seriesMap.forEach(series => { - series.data.sort((a, b) => a.x - b.x); - }); - - return Array.from(seriesMap.values()); -} - -// Prepare chart data grouped by batch size - each batch size is a separate series -function prepareChartDataByBatch(gpuFilter, modelFilter, variantFilter, ioLenFilter, batchFilter) { - const batchDataMap = new Map(); // batch_size -> Map of variant -> data - const testDataMap = new Map(); // For diffusion: test_name -> data - - allMetricsData.forEach(run => { - const runDate = new Date(run.run_date); - - run.results.forEach(result => { - // Apply filters - GPU and Model are required (no "all" option) - if (result.gpu_config !== gpuFilter) return; - - // Handle diffusion results - if (isDiffusionResult(result)) { - const resultModel = result.test_suite || 'diffusion'; - if (resultModel !== modelFilter) return; - - if (result.tests) { - result.tests.forEach(test => { - const testName = test.test_name; - if (!testDataMap.has(testName)) { - testDataMap.set(testName, { - label: testName, - data: [], - model: resultModel, - testName: testName - }); - } - - testDataMap.get(testName).data.push({ - x: runDate, - e2e_ms: test.e2e_ms, - avg_denoise_ms: test.avg_denoise_ms, - median_denoise_ms: test.median_denoise_ms, - runId: run.run_id - }); - }); - } - return; - } - - // Handle text/VLM results - if (result.model !== modelFilter) return; - if (variantFilter !== 'all' && result.variant !== variantFilter) return; - - // Use benchmarks_by_io_len if available, otherwise fall back to flat benchmarks - if (result.benchmarks_by_io_len) { - Object.entries(result.benchmarks_by_io_len).forEach(([ioKey, ioData]) => { - // Apply IO length filter - if (ioLenFilter !== 'all' && ioKey !== ioLenFilter) return; - - ioData.benchmarks.forEach(bench => { - if (batchFilter !== 'all' && bench.batch_size !== parseInt(batchFilter)) return; - - const batchSize = bench.batch_size; - const variantLabel = result.variant || 'default'; - // Include IO length in series key when showing all lengths - const seriesKey = ioLenFilter === 'all' - ? `${variantLabel} (${formatIoLenLabel(ioKey)})` - : variantLabel; - - if (!batchDataMap.has(batchSize)) { - batchDataMap.set(batchSize, new Map()); - } - - const variantMap = batchDataMap.get(batchSize); - if (!variantMap.has(seriesKey)) { - variantMap.set(seriesKey, { - label: seriesKey, - data: [], - model: result.model, - variant: result.variant, - batchSize: batchSize, - ioKey: ioKey - }); - } - - variantMap.get(seriesKey).data.push({ - x: runDate, - throughput: bench.overall_throughput, - outputThroughput: bench.output_throughput, - latency: bench.latency_ms, - ttft: bench.ttft_ms, - inputThroughput: bench.input_throughput, - accLength: bench.acc_length, - runId: run.run_id - }); - }); - }); - } else if (result.benchmarks) { - // Fall back to flat benchmarks for backward compatibility - result.benchmarks.forEach(bench => { - // Apply IO length filter using flat structure - const benchIoKey = bench.input_len && bench.output_len - ? `${bench.input_len}_${bench.output_len}` - : null; - if (ioLenFilter !== 'all' && benchIoKey !== ioLenFilter) return; - if (batchFilter !== 'all' && bench.batch_size !== parseInt(batchFilter)) return; - - const batchSize = bench.batch_size; - const variantLabel = result.variant || 'default'; - // Include IO length in series key when showing all lengths - const seriesKey = ioLenFilter === 'all' && benchIoKey - ? `${variantLabel} (${formatIoLenLabel(benchIoKey)})` - : variantLabel; - - if (!batchDataMap.has(batchSize)) { - batchDataMap.set(batchSize, new Map()); - } - - const variantMap = batchDataMap.get(batchSize); - if (!variantMap.has(seriesKey)) { - variantMap.set(seriesKey, { - label: seriesKey, - data: [], - model: result.model, - variant: result.variant, - batchSize: batchSize, - ioKey: benchIoKey - }); - } - - variantMap.get(seriesKey).data.push({ - x: runDate, - throughput: bench.overall_throughput, - outputThroughput: bench.output_throughput, - latency: bench.latency_ms, - ttft: bench.ttft_ms, - inputThroughput: bench.input_throughput, - accLength: bench.acc_length, - runId: run.run_id - }); - }); - } - }); - }); - - // Sort data points by date and convert to array format - const result = {}; - - // For diffusion data, use test names as "batch sizes" - if (testDataMap.size > 0) { - testDataMap.forEach((series, testName) => { - series.data.sort((a, b) => a.x - b.x); - result[testName] = [series]; // Each test is its own series - }); - return result; - } - - // For text/VLM data, use batch sizes - batchDataMap.forEach((variantMap, batchSize) => { - variantMap.forEach(series => { - series.data.sort((a, b) => a.x - b.x); - }); - result[batchSize] = Array.from(variantMap.values()); - }); - - return result; -} - -// Unified chart update function for any metric type -function updateMetricChart(chartDataByBatch, metricType) { - const container = document.getElementById('charts-container'); - container.innerHTML = ''; - - // Destroy existing charts - activeCharts.forEach(chart => chart.destroy()); - activeCharts = []; - - const metric = metricTypes[metricType]; - const isDiffusion = metric.type === 'diffusion'; - - // For diffusion, keys are test names; for text, keys are batch sizes - const keys = Object.keys(chartDataByBatch); - if (!isDiffusion) { - keys.sort((a, b) => parseInt(a) - parseInt(b)); - } else { - keys.sort(); // Alphabetical sort for test names - } - const batchSizes = keys; // Keep variable name for compatibility - - if (batchSizes.length === 0) { - container.innerHTML = '
No data available for the selected filters
'; - return; - } - - let hasAnyData = false; - - batchSizes.forEach(batchSize => { - const chartData = chartDataByBatch[batchSize]; - - const ctx_datasets = chartData.map((series, index) => { - // Filter data points - for metrics like accLength, exclude invalid values (-1 or null) - let dataPoints = series.data.map(d => ({ x: d.x, y: d[metric.field] })); - if (metric.filterInvalid) { - dataPoints = dataPoints.filter(d => d.y != null && d.y !== -1 && d.y > 0); - } - return { - label: series.label, - data: dataPoints, - borderColor: chartColors[index % chartColors.length], - backgroundColor: chartColors[index % chartColors.length] + '20', - tension: 0.1, - fill: false - }; - }).filter(dataset => dataset.data.length > 0); // Remove empty datasets - - // Skip this batch size if no valid data - if (ctx_datasets.length === 0) { - return; - } - - hasAnyData = true; - - const chartWrapper = document.createElement('div'); - chartWrapper.className = 'batch-chart-wrapper'; - - const title = document.createElement('div'); - title.className = 'batch-chart-title'; - // For diffusion, show test name; for text, show batch size - title.textContent = isDiffusion ? `Test: ${batchSize}` : `Batch Size: ${batchSize}`; - chartWrapper.appendChild(title); - - const chartContainer = document.createElement('div'); - chartContainer.className = 'chart-container'; - const canvas = document.createElement('canvas'); - chartContainer.appendChild(canvas); - chartWrapper.appendChild(chartContainer); - container.appendChild(chartWrapper); - - const ctx = canvas.getContext('2d'); - - const chart = new Chart(ctx, { - type: 'line', - data: { datasets: ctx_datasets }, - options: getChartOptions(metric.unit) - }); - activeCharts.push(chart); - }); - - // Show message if no valid data for this metric - if (!hasAnyData) { - container.innerHTML = `
No valid ${metric.label.toLowerCase()} data available for the selected filters
`; - } -} - -function getChartOptions(yAxisLabel) { - return { - responsive: true, - maintainAspectRatio: false, - interaction: { - mode: 'index', - intersect: false - }, - plugins: { - legend: { - position: 'bottom', - labels: { - boxWidth: 12, - padding: 10, - font: { size: 11 } - } - }, - tooltip: { - backgroundColor: '#1a2332', - borderColor: 'rgba(148, 163, 184, 0.1)', - borderWidth: 1, - titleFont: { size: 13, family: "'DM Sans', sans-serif" }, - bodyFont: { size: 12, family: "'JetBrains Mono', monospace" }, - padding: 14, - cornerRadius: 8 - } - }, - scales: { - x: { - type: 'time', - time: { - unit: 'day', - displayFormats: { - day: 'MMM d' - } - }, - grid: { - color: 'rgba(148, 163, 184, 0.06)' - } - }, - y: { - title: { - display: true, - text: yAxisLabel - }, - grid: { - color: 'rgba(148, 163, 184, 0.06)' - } - } - } - }; -} - -// Escape HTML to prevent XSS -function escapeHtml(text) { - const div = document.createElement('div'); - div.textContent = text; - return div.innerHTML; -} - -// Update runs table -function updateRunsTable() { - const tbody = document.getElementById('runs-table-body'); - tbody.innerHTML = ''; - - allMetricsData.slice(0, 10).forEach(run => { - const models = new Set(run.results.map(r => r.model.split('/').pop())); - const date = new Date(run.run_date); - - const row = document.createElement('tr'); - - // Create cells safely to prevent XSS - const dateCell = document.createElement('td'); - dateCell.textContent = `${date.toLocaleDateString()} ${date.toLocaleTimeString()}`; - - const runIdCell = document.createElement('td'); - const runLink = document.createElement('a'); - runLink.href = `https://github.com/${GITHUB_REPO}/actions/runs/${encodeURIComponent(run.run_id)}`; - runLink.target = '_blank'; - runLink.className = 'run-link'; - runLink.textContent = run.run_id; - runIdCell.appendChild(runLink); - - const commitCell = document.createElement('td'); - const commitCode = document.createElement('code'); - commitCode.textContent = run.commit_sha.substring(0, 7); - commitCell.appendChild(commitCode); - - const branchCell = document.createElement('td'); - branchCell.textContent = run.branch; - - const modelsCell = document.createElement('td'); - Array.from(models).forEach((model, index) => { - if (index > 0) modelsCell.appendChild(document.createTextNode(' ')); - const badge = document.createElement('span'); - badge.className = 'model-badge'; - badge.textContent = model; - modelsCell.appendChild(badge); - }); - - row.appendChild(dateCell); - row.appendChild(runIdCell); - row.appendChild(commitCell); - row.appendChild(branchCell); - row.appendChild(modelsCell); - - tbody.appendChild(row); - }); -} - -// Refresh data -async function refreshData() { - document.getElementById('content').style.display = 'none'; - document.getElementById('loading').style.display = 'flex'; - await init(); -} - -// Format numbers for display -function formatNumber(num) { - if (num >= 1000) { - return (num / 1000).toFixed(1) + 'k'; - } - return num.toFixed(1); -} - -// Authentication state -let authToken = sessionStorage.getItem('dashboard_auth_token') || null; - -// Get auth headers for API requests -function getAuthHeaders() { - const headers = {}; - if (authToken) { - headers['Authorization'] = `Bearer ${authToken}`; - } - return headers; -} - -// Check if server requires authentication and show/hide login accordingly -async function checkAuthAndInit() { - const loginOverlay = document.getElementById('login-overlay'); - const dashboardContainer = document.getElementById('dashboard-container'); - - try { - const response = await fetch('/api/auth-check'); - if (response.ok) { - const data = await response.json(); - if (!data.auth_required) { - // No auth required - skip login, show dashboard directly - loginOverlay.style.display = 'none'; - dashboardContainer.style.display = 'block'; - init(); - return; - } - } - } catch (e) { - // Server not available (e.g. static hosting) - skip login - loginOverlay.style.display = 'none'; - dashboardContainer.style.display = 'block'; - init(); - return; - } - - // Auth is required - check if we have a valid token from a previous session - if (authToken) { - try { - const testResponse = await fetch('/api/metrics', { - headers: getAuthHeaders() - }); - if (testResponse.ok) { - loginOverlay.style.display = 'none'; - dashboardContainer.style.display = 'block'; - init(); - return; - } - } catch (e) { - // Token invalid or expired - } - // Clear invalid token - authToken = null; - sessionStorage.removeItem('dashboard_auth_token'); - } - - // Show login form - loginOverlay.style.display = 'flex'; - dashboardContainer.style.display = 'none'; -} - -// Handle login form submission -async function handleLogin(event) { - event.preventDefault(); - - const username = document.getElementById('login-username').value; - const password = document.getElementById('login-password').value; - const errorEl = document.getElementById('login-error'); - const loginBtn = document.getElementById('login-btn'); - - errorEl.textContent = ''; - loginBtn.disabled = true; - loginBtn.textContent = 'Signing in...'; - - try { - const response = await fetch('/api/login', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ username, password }) - }); - - const data = await response.json(); - - if (response.ok && data.token) { - authToken = data.token; - sessionStorage.setItem('dashboard_auth_token', authToken); - - document.getElementById('login-overlay').style.display = 'none'; - document.getElementById('dashboard-container').style.display = 'block'; - init(); - } else { - errorEl.textContent = data.error || 'Invalid username or password'; - } - } catch (e) { - errorEl.textContent = 'Unable to connect to server'; - } finally { - loginBtn.disabled = false; - loginBtn.textContent = 'Sign In'; - } - - return false; -} - -// Initialize on page load -document.addEventListener('DOMContentLoaded', checkAuthAndInit); diff --git a/docs/performance_dashboard/fetch_metrics.py b/docs/performance_dashboard/fetch_metrics.py deleted file mode 100755 index 264e7f334..000000000 --- a/docs/performance_dashboard/fetch_metrics.py +++ /dev/null @@ -1,272 +0,0 @@ -#!/usr/bin/env python3 -""" -Fetch and process SGLang nightly test metrics from GitHub Actions artifacts. - -This script fetches consolidated metrics from GitHub Actions workflow runs -and outputs them as JSON for the performance dashboard. - -Usage: - python fetch_metrics.py --output metrics_data.json - python fetch_metrics.py --output metrics_data.json --days 30 - python fetch_metrics.py --output metrics_data.json --run-id 21338741812 -""" - -import argparse -import io -import json -import os -import sys -import zipfile -from datetime import datetime, timedelta, timezone -from pathlib import Path -from typing import Optional - -import requests - -GITHUB_REPO = "sgl-project/sglang" -WORKFLOW_NAME = "nightly-test-nvidia.yml" -ARTIFACT_PREFIX = "consolidated-metrics-" - - -def get_github_token() -> Optional[str]: - """Get GitHub token from environment or gh CLI.""" - # Check environment variable first - token = os.environ.get("GITHUB_TOKEN") - if token: - return token - - # Try gh CLI - try: - import subprocess - - result = subprocess.run( - ["gh", "auth", "token"], - capture_output=True, - text=True, - check=True, - ) - return result.stdout.strip() - except (subprocess.CalledProcessError, FileNotFoundError): - pass - - return None - - -def get_headers(token: Optional[str]) -> dict: - """Get request headers with optional authentication.""" - headers = { - "Accept": "application/vnd.github.v3+json", - } - if token: - headers["Authorization"] = f"Bearer {token}" - return headers - - -def fetch_workflow_runs( - token: Optional[str], - days: int = 30, - event: Optional[str] = None, -) -> list: - """Fetch completed workflow runs from GitHub Actions.""" - url = f"https://api.github.com/repos/{GITHUB_REPO}/actions/workflows/{WORKFLOW_NAME}/runs" - - params = { - "status": "completed", - "per_page": 100, - } - - if event: - params["event"] = event - - response = requests.get(url, headers=get_headers(token), params=params, timeout=30) - response.raise_for_status() - - runs = response.json().get("workflow_runs", []) - - # Filter by date - cutoff = datetime.now(timezone.utc) - timedelta(days=days) - runs = [ - run - for run in runs - if datetime.fromisoformat(run["created_at"].replace("Z", "+00:00")) > cutoff - ] - - return runs - - -def fetch_run_artifacts(token: Optional[str], run_id: int) -> list: - """Fetch artifacts for a specific workflow run.""" - url = f"https://api.github.com/repos/{GITHUB_REPO}/actions/runs/{run_id}/artifacts" - - response = requests.get(url, headers=get_headers(token), timeout=30) - response.raise_for_status() - - return response.json().get("artifacts", []) - - -def download_artifact(token: Optional[str], artifact_id: int) -> Optional[bytes]: - """Download an artifact by ID.""" - if not token: - print(f"Warning: GitHub token required to download artifacts", file=sys.stderr) - return None - - url = f"https://api.github.com/repos/{GITHUB_REPO}/actions/artifacts/{artifact_id}/zip" - - headers = get_headers(token) - response = requests.get(url, headers=headers, allow_redirects=True, timeout=60) - - if response.status_code == 200: - return response.content - - print( - f"Failed to download artifact {artifact_id}: {response.status_code}", - file=sys.stderr, - ) - return None - - -def extract_metrics_from_zip(zip_content: bytes) -> Optional[dict]: - """Extract metrics JSON from a zip file.""" - try: - with zipfile.ZipFile(io.BytesIO(zip_content)) as zf: - # Find the JSON file in the archive - json_files = [f for f in zf.namelist() if f.endswith(".json")] - if not json_files: - return None - - with zf.open(json_files[0]) as f: - return json.load(f) - except (zipfile.BadZipFile, json.JSONDecodeError) as e: - print(f"Failed to extract metrics: {e}", file=sys.stderr) - return None - - -def fetch_metrics_for_run(token: Optional[str], run: dict) -> Optional[dict]: - """Fetch metrics for a single workflow run.""" - run_id = run["id"] - print(f"Fetching metrics for run {run_id}...", file=sys.stderr) - - artifacts = fetch_run_artifacts(token, run_id) - - # Find consolidated metrics artifact - metrics_artifact = None - for artifact in artifacts: - if artifact["name"].startswith(ARTIFACT_PREFIX): - metrics_artifact = artifact - break - - if not metrics_artifact: - print(f"No consolidated metrics found for run {run_id}", file=sys.stderr) - return None - - # Download and extract - zip_content = download_artifact(token, metrics_artifact["id"]) - if not zip_content: - return None - - metrics = extract_metrics_from_zip(zip_content) - if not metrics: - return None - - # Ensure required fields are present - if "run_id" not in metrics: - metrics["run_id"] = str(run_id) - if "run_date" not in metrics: - metrics["run_date"] = run["created_at"] - if "commit_sha" not in metrics: - metrics["commit_sha"] = run["head_sha"] - if "branch" not in metrics: - metrics["branch"] = run["head_branch"] - - return metrics - - -def fetch_single_run(token: Optional[str], run_id: int) -> Optional[dict]: - """Fetch metrics for a single run by ID.""" - url = f"https://api.github.com/repos/{GITHUB_REPO}/actions/runs/{run_id}" - - response = requests.get(url, headers=get_headers(token), timeout=30) - response.raise_for_status() - - run = response.json() - return fetch_metrics_for_run(token, run) - - -def main(): - parser = argparse.ArgumentParser( - description="Fetch SGLang nightly test metrics from GitHub Actions" - ) - parser.add_argument( - "--output", - "-o", - type=str, - default="metrics_data.json", - help="Output JSON file path", - ) - parser.add_argument( - "--days", - type=int, - default=30, - help="Number of days to fetch (default: 30)", - ) - parser.add_argument( - "--run-id", - type=int, - help="Fetch a specific run by ID", - ) - parser.add_argument( - "--event", - type=str, - choices=["schedule", "workflow_dispatch", "push"], - help="Filter by trigger event type", - ) - parser.add_argument( - "--scheduled-only", - action="store_true", - help="Only fetch scheduled (nightly) runs", - ) - - args = parser.parse_args() - - token = get_github_token() - if not token: - print( - "Warning: No GitHub token found. Some features may be limited.", - file=sys.stderr, - ) - print( - "Set GITHUB_TOKEN env var or login with 'gh auth login'", - file=sys.stderr, - ) - - all_metrics = [] - - if args.run_id: - # Fetch single run - metrics = fetch_single_run(token, args.run_id) - if metrics: - all_metrics.append(metrics) - else: - # Fetch multiple runs - event = "schedule" if args.scheduled_only else args.event - runs = fetch_workflow_runs(token, days=args.days, event=event) - print(f"Found {len(runs)} workflow runs", file=sys.stderr) - - for run in runs: - metrics = fetch_metrics_for_run(token, run) - if metrics: - all_metrics.append(metrics) - - # Sort by date descending - all_metrics.sort(key=lambda x: x.get("run_date", ""), reverse=True) - - # Write output - output_path = Path(args.output) - with open(output_path, "w") as f: - json.dump(all_metrics, f, indent=2) - - print(f"Wrote {len(all_metrics)} metrics records to {output_path}", file=sys.stderr) - - -if __name__ == "__main__": - main() diff --git a/docs/performance_dashboard/index.html b/docs/performance_dashboard/index.html deleted file mode 100644 index e680f981a..000000000 --- a/docs/performance_dashboard/index.html +++ /dev/null @@ -1,946 +0,0 @@ - - - - - - SGLang Performance Dashboard - - - - - - - - - - - - - - - - diff --git a/docs/performance_dashboard/server.py b/docs/performance_dashboard/server.py deleted file mode 100755 index 1e025ce85..000000000 --- a/docs/performance_dashboard/server.py +++ /dev/null @@ -1,422 +0,0 @@ -#!/usr/bin/env python3 -""" -Simple development server for the SGLang Performance Dashboard. - -This server: -1. Serves the static HTML/JS files -2. Provides an API endpoint to fetch metrics from GitHub -3. Caches metrics data to reduce API calls - -Usage: - python server.py - python server.py --port 8080 - python server.py --host 0.0.0.0 # Allow external access - python server.py --fetch-on-start - python server.py --username admin --password secret # Enable authentication - DASHBOARD_USERNAME=admin DASHBOARD_PASSWORD=secret python server.py # Via env vars - python server.py --refresh-interval 12 # Auto-refresh data every 12 hours -""" - -import argparse -import hashlib -import hmac -import http.server -import io -import json -import os -import secrets -import socketserver -import threading -import time -import zipfile -from datetime import datetime, timedelta, timezone -from pathlib import Path -from urllib.parse import urlparse - -import requests - -GITHUB_REPO = "sgl-project/sglang" -WORKFLOW_NAME = "nightly-test-nvidia.yml" -ARTIFACT_PREFIX = "consolidated-metrics-" - -# Cache for metrics data with thread-safe lock -cache_lock = threading.Lock() -metrics_cache = { - "data": [], - "last_updated": None, - "updating": False, -} - -CACHE_TTL = 300 # 5 minutes -REQUEST_TIMEOUT = 30 # seconds - -# Authentication configuration (set via CLI flags) -auth_config = { - "enabled": False, - "username": None, - "password_hash": None, # SHA-256 hash of the password - "active_tokens": {}, # token -> expiry timestamp -} -auth_lock = threading.Lock() -AUTH_TOKEN_TTL = 3600 # 1 hour - - -def hash_password(password): - """Hash a password using SHA-256 for constant-time comparison.""" - return hashlib.sha256(password.encode("utf-8")).hexdigest() - - -def create_auth_token(): - """Create a new session token.""" - token = secrets.token_hex(32) - with auth_lock: - # Clean up expired tokens - now = time.time() - auth_config["active_tokens"] = { - t: exp for t, exp in auth_config["active_tokens"].items() if exp > now - } - auth_config["active_tokens"][token] = now + AUTH_TOKEN_TTL - return token - - -def verify_auth_token(token): - """Verify a session token is valid and not expired.""" - if not token: - return False - with auth_lock: - expiry = auth_config["active_tokens"].get(token) - if expiry and expiry > time.time(): - return True - # Remove expired token - auth_config["active_tokens"].pop(token, None) - return False - - -def get_github_token(): - """Get GitHub token from environment or gh CLI.""" - token = os.environ.get("GITHUB_TOKEN") - if token: - return token - - try: - import subprocess - - result = subprocess.run( - ["gh", "auth", "token"], - capture_output=True, - text=True, - check=True, - ) - return result.stdout.strip() - except (subprocess.CalledProcessError, FileNotFoundError): - pass - - return None - - -def fetch_metrics_from_github(days=30): - """Fetch metrics from GitHub Actions artifacts.""" - token = get_github_token() - headers = {"Accept": "application/vnd.github.v3+json"} - if token: - headers["Authorization"] = f"Bearer {token}" - - # Get workflow runs - only scheduled (nightly) runs, not workflow_dispatch - url = f"https://api.github.com/repos/{GITHUB_REPO}/actions/workflows/{WORKFLOW_NAME}/runs" - params = {"status": "completed", "per_page": 50, "event": "schedule"} - - try: - response = requests.get( - url, headers=headers, params=params, timeout=REQUEST_TIMEOUT - ) - if not response.ok: - print(f"Failed to fetch workflow runs: {response.status_code}") - return [] - except requests.exceptions.RequestException as e: - print(f"Network error fetching workflow runs: {e}") - return [] - - runs = response.json().get("workflow_runs", []) - - # Filter by date - cutoff = datetime.now(timezone.utc) - timedelta(days=days) - runs = [ - run - for run in runs - if datetime.fromisoformat(run["created_at"].replace("Z", "+00:00")) > cutoff - ] - - all_metrics = [] - - for run in runs[:20]: # Limit to 20 most recent - run_id = run["id"] - - # Get artifacts - artifacts_url = f"https://api.github.com/repos/{GITHUB_REPO}/actions/runs/{run_id}/artifacts" - try: - artifacts_resp = requests.get( - artifacts_url, headers=headers, timeout=REQUEST_TIMEOUT - ) - if not artifacts_resp.ok: - continue - except requests.exceptions.RequestException as e: - print(f"Network error fetching artifacts for run {run_id}: {e}") - continue - - artifacts = artifacts_resp.json().get("artifacts", []) - - # Find consolidated metrics - for artifact in artifacts: - if artifact["name"].startswith(ARTIFACT_PREFIX): - if not token: - # Without token, we can't download - return metadata only - all_metrics.append( - { - "run_id": str(run_id), - "run_date": run["created_at"], - "commit_sha": run["head_sha"], - "branch": run["head_branch"], - "results": [], - } - ) - break - - # Download artifact - download_url = f"https://api.github.com/repos/{GITHUB_REPO}/actions/artifacts/{artifact['id']}/zip" - try: - download_resp = requests.get( - download_url, - headers=headers, - allow_redirects=True, - timeout=REQUEST_TIMEOUT, - ) - except requests.exceptions.RequestException as e: - print(f"Network error downloading artifact: {e}") - break - - if download_resp.ok: - try: - with zipfile.ZipFile(io.BytesIO(download_resp.content)) as zf: - json_files = [ - f for f in zf.namelist() if f.endswith(".json") - ] - if json_files: - with zf.open(json_files[0]) as f: - metrics = json.load(f) - # Ensure required fields - metrics.setdefault("run_id", str(run_id)) - metrics.setdefault("run_date", run["created_at"]) - metrics.setdefault("commit_sha", run["head_sha"]) - metrics.setdefault("branch", run["head_branch"]) - all_metrics.append(metrics) - except (zipfile.BadZipFile, json.JSONDecodeError) as e: - print(f"Failed to process artifact: {e}") - break - - return all_metrics - - -def update_cache_async(): - """Update the metrics cache in background with thread safety.""" - with cache_lock: - if metrics_cache["updating"]: - return - metrics_cache["updating"] = True - - try: - data = fetch_metrics_from_github() - with cache_lock: - metrics_cache["data"] = data - metrics_cache["last_updated"] = time.time() - print(f"Cache updated with {len(data)} metrics records") - finally: - with cache_lock: - metrics_cache["updating"] = False - - -def start_periodic_refresh(interval_hours): - """Start a background thread that refreshes the cache periodically.""" - interval_seconds = interval_hours * 3600 - - def refresh_loop(): - while True: - time.sleep(interval_seconds) - print(f"Periodic refresh triggered (every {interval_hours}h)") - update_cache_async() - - thread = threading.Thread(target=refresh_loop, daemon=True) - thread.start() - print(f"Periodic refresh enabled: every {interval_hours} hours") - - -class DashboardHandler(http.server.SimpleHTTPRequestHandler): - """HTTP request handler for the dashboard.""" - - def __init__(self, *args, directory=None, **kwargs): - super().__init__(*args, directory=directory, **kwargs) - - def _send_json(self, data, status=200): - """Send a JSON response.""" - self.send_response(status) - self.send_header("Content-Type", "application/json") - self.send_header("Access-Control-Allow-Origin", "*") - self.end_headers() - self.wfile.write(json.dumps(data).encode()) - - def _check_auth(self): - """Check if request is authenticated. Returns True if OK, sends 401 and returns False otherwise.""" - if not auth_config["enabled"]: - return True - auth_header = self.headers.get("Authorization", "") - if auth_header.startswith("Bearer "): - token = auth_header[7:] - if verify_auth_token(token): - return True - self._send_json({"error": "Unauthorized"}, status=401) - return False - - def do_GET(self): - parsed = urlparse(self.path) - - # Prevent directory traversal attacks - if ".." in parsed.path or parsed.path.startswith("//"): - self.send_error(400, "Invalid path") - return - - if parsed.path == "/api/auth-check": - self.handle_auth_check() - elif parsed.path == "/api/metrics": - if self._check_auth(): - self.handle_metrics_api(parsed) - elif parsed.path == "/api/refresh": - if self._check_auth(): - self.handle_refresh_api() - else: - super().do_GET() - - def do_POST(self): - parsed = urlparse(self.path) - - if parsed.path == "/api/login": - self.handle_login() - else: - self.send_error(404, "Not Found") - - def handle_auth_check(self): - """Tell the frontend whether authentication is required.""" - self._send_json({"auth_required": auth_config["enabled"]}) - - def handle_login(self): - """Validate username/password and return a session token.""" - content_length = int(self.headers.get("Content-Length", 0)) - if content_length == 0 or content_length > 4096: - self._send_json({"error": "Invalid request"}, status=400) - return - - try: - body = json.loads(self.rfile.read(content_length)) - except (json.JSONDecodeError, ValueError): - self._send_json({"error": "Invalid JSON"}, status=400) - return - - username = body.get("username", "") - password = body.get("password", "") - - if hmac.compare_digest( - username, auth_config["username"] - ) and hmac.compare_digest( - hash_password(password), auth_config["password_hash"] - ): - token = create_auth_token() - self._send_json({"token": token}) - else: - self._send_json({"error": "Invalid username or password"}, status=401) - - def handle_metrics_api(self, parsed): - """Handle /api/metrics endpoint.""" - # Check cache with thread safety - with cache_lock: - cache_valid = ( - metrics_cache["last_updated"] - and time.time() - metrics_cache["last_updated"] < CACHE_TTL - ) - data = metrics_cache["data"].copy() - - if not cache_valid: - # Trigger background update - threading.Thread(target=update_cache_async, daemon=True).start() - - self._send_json(data) - - def handle_refresh_api(self): - """Handle /api/refresh endpoint.""" - threading.Thread(target=update_cache_async, daemon=True).start() - self._send_json({"status": "refreshing"}) - - def log_message(self, format, *args): - """Custom log format.""" - print(f"[{self.log_date_time_string()}] {args[0]}") - - -def main(): - parser = argparse.ArgumentParser(description="SGLang Performance Dashboard Server") - parser.add_argument("--port", type=int, default=8000, help="Port to serve on") - parser.add_argument( - "--host", - default="127.0.0.1", - help="Host to bind to (use 0.0.0.0 for external access)", - ) - parser.add_argument( - "--fetch-on-start", action="store_true", help="Fetch metrics on startup" - ) - parser.add_argument( - "--refresh-interval", - type=float, - default=12, - help="Auto-refresh interval in hours (default: 12, set to 0 to disable)", - ) - parser.add_argument( - "--username", - default=os.environ.get("DASHBOARD_USERNAME"), - help="Username for dashboard authentication (or set DASHBOARD_USERNAME env var)", - ) - parser.add_argument( - "--password", - default=os.environ.get("DASHBOARD_PASSWORD"), - help="Password for dashboard authentication (or set DASHBOARD_PASSWORD env var)", - ) - args = parser.parse_args() - - # Configure authentication if both username and password are provided - if args.username and args.password: - auth_config["enabled"] = True - auth_config["username"] = args.username - auth_config["password_hash"] = hash_password(args.password) - print(f"Authentication enabled for user: {args.username}") - elif args.username or args.password: - parser.error("Both --username and --password must be provided together") - - # Change to dashboard directory - dashboard_dir = Path(__file__).parent - os.chdir(dashboard_dir) - - if args.fetch_on_start: - print("Fetching initial metrics data...") - update_cache_async() - - if args.refresh_interval > 0: - start_periodic_refresh(args.refresh_interval) - - handler = lambda *a, **kw: DashboardHandler(*a, directory=str(dashboard_dir), **kw) - - with socketserver.TCPServer((args.host, args.port), handler) as httpd: - print(f"Serving dashboard at http://{args.host}:{args.port}") - print("Press Ctrl+C to stop") - try: - httpd.serve_forever() - except KeyboardInterrupt: - print("\nShutting down...") - - -if __name__ == "__main__": - main() diff --git a/docs/platforms/amd_gpu.md b/docs/platforms/amd_gpu.md deleted file mode 100644 index ca427d38a..000000000 --- a/docs/platforms/amd_gpu.md +++ /dev/null @@ -1,195 +0,0 @@ -# AMD GPUs - -This document describes how to run SGLang on AMD GPUs. If you encounter issues or have questions, please [open an issue](https://github.com/sgl-project/sglang/issues). - -## System Configuration - -When using AMD GPUs (such as MI300X), certain system-level optimizations help ensure stable performance. Here we take MI300X as an example. AMD provides official documentation for MI300X optimization and system tuning: - -- [AMD MI300X Tuning Guides](https://rocm.docs.amd.com/en/latest/how-to/tuning-guides/mi300x/index.html) -- [LLM inference performance validation on AMD Instinct MI300X](https://rocm.docs.amd.com/en/latest/how-to/rocm-for-ai/inference/vllm-benchmark.html) -- [AMD Instinct MI300X System Optimization](https://rocm.docs.amd.com/en/latest/how-to/system-optimization/mi300x.html) -- [AMD Instinct MI300X Workload Optimization](https://rocm.docs.amd.com/en/latest/how-to/rocm-for-ai/inference-optimization/workload.html) -- [Supercharge DeepSeek-R1 Inference on AMD Instinct MI300X](https://rocm.blogs.amd.com/artificial-intelligence/DeepSeekR1-Part2/README.html) - -**NOTE:** We strongly recommend reading these docs and guides entirely to fully utilize your system. - -Below are a few key settings to confirm or enable for SGLang: - -### Update GRUB Settings - -In `/etc/default/grub`, append the following to `GRUB_CMDLINE_LINUX`: - -```text -pci=realloc=off iommu=pt -``` - -Afterward, run `sudo update-grub` (or your distro’s equivalent) and reboot. - -### Disable NUMA Auto-Balancing - -```bash -sudo sh -c 'echo 0 > /proc/sys/kernel/numa_balancing' -``` - -You can automate or verify this change using [this helpful script](https://github.com/ROCm/triton/blob/rocm_env/scripts/amd/env_check.sh). - -Again, please go through the entire documentation to confirm your system is using the recommended configuration. - -## Install SGLang - -You can install SGLang using one of the methods below. - -### Install from Source - -```bash -# Use the last release branch -git clone -b v0.5.9 https://github.com/sgl-project/sglang.git -cd sglang - -# Compile sgl-kernel -pip install --upgrade pip -cd sgl-kernel -python setup_rocm.py install - -# Install sglang python package along with diffusion support -cd .. -rm -rf python/pyproject.toml && mv python/pyproject_other.toml python/pyproject.toml -pip install -e "python[all_hip]" -``` - -### Install Using Docker (Recommended) - -The docker images are available on Docker Hub at [lmsysorg/sglang](https://hub.docker.com/r/lmsysorg/sglang/tags), built from [rocm.Dockerfile](https://github.com/sgl-project/sglang/tree/main/docker). - -The steps below show how to build and use an image. - -1. Build the docker image. - If you use pre-built images, you can skip this step and replace `sglang_image` with the pre-built image names in the steps below. - - ```bash - docker build -t sglang_image -f rocm.Dockerfile . - ``` - -2. Create a convenient alias. - - ```bash - alias drun='docker run -it --rm --network=host --privileged --device=/dev/kfd --device=/dev/dri \ - --ipc=host --shm-size 16G --group-add video --cap-add=SYS_PTRACE \ - --security-opt seccomp=unconfined \ - -v $HOME/dockerx:/dockerx \ - -v /data:/data' - ``` - - If you are using RDMA, please note that: - - `--network host` and `--privileged` are required by RDMA. If you don't need RDMA, you can remove them. - - You may need to set `NCCL_IB_GID_INDEX` if you are using RoCE, for example: `export NCCL_IB_GID_INDEX=3`. - -3. Launch the server. - - **NOTE:** Replace `` below with your [huggingface hub token](https://huggingface.co/docs/hub/en/security-tokens). - - ```bash - drun -p 30000:30000 \ - -v ~/.cache/huggingface:/root/.cache/huggingface \ - --env "HF_TOKEN=" \ - sglang_image \ - python3 -m sglang.launch_server \ - --model-path NousResearch/Meta-Llama-3.1-8B \ - --host 0.0.0.0 \ - --port 30000 - ``` - -4. To verify the utility, you can run a benchmark in another terminal or refer to [other docs](https://docs.sglang.io/basic_usage/openai_api_completions.html) to send requests to the engine. - - ```bash - drun sglang_image \ - python3 -m sglang.bench_serving \ - --backend sglang \ - --dataset-name random \ - --num-prompts 4000 \ - --random-input 128 \ - --random-output 128 - ``` - -With your AMD system properly configured and SGLang installed, you can now fully leverage AMD hardware to power SGLang’s machine learning capabilities. - -## Quantization on AMD GPUs - -The [Quantization documentation](../advanced_features/quantization.md#platform-compatibility) has a full compatibility matrix. The short version: FP8, AWQ, MXFP4, W8A8, GPTQ, compressed-tensors, Quark, and **petit_nvfp4** (NVFP4 on ROCm via [Petit](https://github.com/causalflow-ai/petit-kernel)) all work on AMD. Methods that depend on Marlin or NVIDIA-specific kernels (`awq_marlin`, `gptq_marlin`, `gguf`, `modelopt_fp8`, `modelopt_fp4`) do not. - -A few things to keep in mind: - -- FP8 works via Aiter or Triton. Pre-quantized FP8 models like DeepSeek-V3/R1 work out of the box. -- AWQ uses Triton dequantization kernels on AMD. The faster Marlin path is not available. -- MXFP4 requires CDNA3/CDNA4 and `SGLANG_USE_AITER=1`. -- `petit_nvfp4` enables NVFP4 models (e.g., [Llama 3.3 70B FP4](https://huggingface.co/nvidia/Llama-3.3-70B-Instruct-FP4)) on MI250/MI300X via [Petit](https://github.com/causalflow-ai/petit-kernel). Install with `pip install petit-kernel`; no `--quantization` flag needed when loading pre-quantized NVFP4 models. -- `quark_int4fp8_moe` is an AMD-only online quantization method for MoE models on CDNA3/CDNA4. - -Several of these backends are accelerated by [Aiter](https://github.com/ROCm/aiter). Enable it with: - -```bash -export SGLANG_USE_AITER=1 -``` - -Example -- serving an AWQ model: - -```bash -python3 -m sglang.launch_server \ - --model-path hugging-quants/Mixtral-8x7B-Instruct-v0.1-AWQ-INT4 \ - --trust-remote-code \ - --port 30000 --host 0.0.0.0 -``` - -Example -- FP8 online quantization: - -```bash -python3 -m sglang.launch_server \ - --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \ - --quantization fp8 \ - --port 30000 --host 0.0.0.0 -``` - -## Examples - -### Running DeepSeek-V3 - -The only difference when running DeepSeek-V3 is in how you start the server. Here's an example command: - -```bash -drun -p 30000:30000 \ - -v ~/.cache/huggingface:/root/.cache/huggingface \ - --ipc=host \ - --env "HF_TOKEN=" \ - sglang_image \ - python3 -m sglang.launch_server \ - --model-path deepseek-ai/DeepSeek-V3 \ # <- here - --tp 8 \ - --trust-remote-code \ - --host 0.0.0.0 \ - --port 30000 -``` - -[Running DeepSeek-R1 on a single NDv5 MI300X VM](https://techcommunity.microsoft.com/blog/azurehighperformancecomputingblog/running-deepseek-r1-on-a-single-ndv5-mi300x-vm/4372726) could also be a good reference. - -### Running Llama3.1 - -Running Llama3.1 is nearly identical to running DeepSeek-V3. The only difference is in the model specified when starting the server, shown by the following example command: - -```bash -drun -p 30000:30000 \ - -v ~/.cache/huggingface:/root/.cache/huggingface \ - --ipc=host \ - --env "HF_TOKEN=" \ - sglang_image \ - python3 -m sglang.launch_server \ - --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \ # <- here - --tp 8 \ - --trust-remote-code \ - --host 0.0.0.0 \ - --port 30000 -``` - -### Warmup Step - -When the server displays `The server is fired up and ready to roll!`, it means the startup is successful. diff --git a/docs/platforms/apple_metal.md b/docs/platforms/apple_metal.md deleted file mode 100644 index 9f388d768..000000000 --- a/docs/platforms/apple_metal.md +++ /dev/null @@ -1,74 +0,0 @@ -# Apple Silicon with Metal (MLX) - -This document describes how run SGLang on Apple Silicon using [Metal (MLX)](https://opensource.apple.com/projects/mlx/). If you encounter issues or have questions, please [open an issue](https://github.com/sgl-project/sglang/issues). - -## Install SGLang - -You can install SGLang using one of the methods below. - -### Install from Source - -```bash -# Use the default branch -git clone https://github.com/sgl-project/sglang.git -cd sglang - -# Install sglang python package -pip install --upgrade pip -rm -f python/pyproject.toml && mv python/pyproject_other.toml python/pyproject.toml -uv pip install -e "python[all_mps]" -``` - -## Launch of the Serving Engine - -Launch the server with: - -```bash -SGLANG_USE_MLX=1 python -m sglang.launch_server \ - --model \ - --disable-cuda-graph \ - --host 0.0.0.0 -``` - -**Key Parameters Explained:** - -1. `SGLANG_USE_MLX=1` - Enables the use of MLX as the SGLang runtime backend (if disabled, SGLang will fall back to `torch.mps`, which has less support) -2. `--disable-cuda-graph` - Disables usage of CUDA graph, which is not relevant for Apple Metal. -3. `--disable-overlap-schedule` - Disables overlap scheduling (enabled/not present by default) achieved using MLX's `async_eval()` - - -## Benchmarking with Requests - -`sglang.benchmark_one_batch` calls the synchronous prefill/decode methods directly without going through the scheduler and the overlap code path. - -`sglang.benchmark_offline_throughput` can toggle overlap scheduling as it uses the scheduler and the overlap code path by using the flag `--disable-overlap-schedule`. - -### Throughput Testing - -Basic synchronous one batch throughput: -```bash -SGLANG_USE_MLX=1 python -m sglang.bench_one_batch \ - --model-path \ - --disable-cuda-graph \ - --tp-size 1 \ - --batch-size 1 \ - --input-len 60 \ - --output-len 10 -``` - -Synchronous offline throughput: -```bash -SGLANG_USE_MLX=1 python -m sglang.bench_offline_throughput \ - --model-path \ - --disable-cuda-graph \ - --num-prompts 1 \ - --disable-overlap-schedule -``` - -Asynchronous offline throughput: -```bash -SGLANG_USE_MLX=1 python -m sglang.bench_offline_throughput \ - --model-path \ - --disable-cuda-graph \ - --num-prompts 1 -``` diff --git a/docs/platforms/ascend/ascend_contribution_guide.md b/docs/platforms/ascend/ascend_contribution_guide.md deleted file mode 100644 index 3b2451713..000000000 --- a/docs/platforms/ascend/ascend_contribution_guide.md +++ /dev/null @@ -1,163 +0,0 @@ -# Contribution Guide - -Welcome to **SGLang**! We appreciate your interest in contributing. This guide provides a concise overview of how to set up your environment, run tests, build documentation, and open a Pull Request (PR). Whether you’re fixing a small bug or developing a major feature, we encourage following these steps for a smooth contribution process. - -## Install SGLang from Source - -### Prepare Environment - -Before contributing, please ensure that your environment is set up correctly. Follow the steps in the [Installation Guide](ascend_npu.md) to install the necessary dependencies. We recommend [using docker](ascend_npu.md#method-2-using-docker-image) to build the environment. - -### Fork and clone the repository - -**Note**: New contributors do **not** have the write permission to push to the official SGLang repo. Please fork the repository under your GitHub account, then clone your fork locally. - -```bash -git clone https://github.com//sglang.git -# if you are using docker, the environment is already set up. -cd sglang -export PYTHONPATH=$PWD/python:$PYTHONPATH -``` - -## Format code with pre-commit - -We use [pre-commit](https://pre-commit.com/) to maintain consistent code style checks. Before pushing your changes, please run: - -```bash -pip3 install pre-commit -pre-commit install -pre-commit run --all-files -``` - -- **`pre-commit run --all-files`** manually runs all configured checks, applying fixes if possible. If it fails the first time, re-run it to ensure lint errors are fully resolved. Make sure your code passes all checks **before** creating a Pull Request. -- **Do not commit** directly to the `main` branch. Always create a new branch (e.g., `feature/my-new-feature`), push your changes, and open a PR from that branch. - -## Run and add unit tests - -If you add a new feature or fix a bug, please add corresponding unit tests to ensure coverage and prevent regression. -SGLang uses Python's built-in [unittest](https://docs.python.org/3/library/unittest.html) framework. -For detailed instructions on running tests and integrating them into CI, refer to [test/README.md](https://github.com/sgl-project/sglang/tree/main/test/README.md). - -If you need to use model which is not in `python/sglang/test/ascend/test_ascend_utils.py` list. Follow these steps: -1. Register account and upload your model to [modelscope](https://modelscope.cn/models). -2. Make sure your model is pre-cached on the CI server and is on the way "/data/ascend-ci-share-pkking-sglang/modelscope/hub/models/{your_model_repo}/{your_model}". -If this is not the case, use following command on CI server: - ```bash - modelscope download - --model {your_model_repo}/{your_model} - --local_dir /data/ascend-ci-share-pkking-sglang/modelscope/hub/models/{your_model_repo}/{your_model} - ``` - > Note: If you don’t have access to CI server, please ask maintainers (zl19940307@163.com) to download your model. -4. Add model to ```python/sglang/test/ascend/test_ascend_utils.py``` (use docker ```"/root/.cache/modelscope/hub/models/{your_model_repo}/{your_model}"``` path). - -## Write documentations - -We recommend new contributors start from writing documentation, which helps you quickly understand SGLang codebase. -For more details, please refer to [docs/README.md](https://github.com/sgl-project/sglang/tree/main/docs/README.md). - -## Test the accuracy -If your code changes the model output, please run the accuracy tests. A quick sanity check is the few-shot GSM8K. - -``` -# Launch a server -python3 -m sglang.launch_server --model Qwen/Qwen2-7B-Instruct - -# Evaluate -python3 -m sglang.test.few_shot_gsm8k --num-questions 200 -``` - -Please note that the above script is primarily a sanity check, not a rigorous accuracy or speed test. -This test can have significant variance (1%–5%) in accuracy due to batching and the non-deterministic nature of the inference engine. -Also, do not rely on the "Latency/Output throughput" from this script, as it is not a proper speed test. - -GSM8K is too easy for state-of-the-art models nowadays. Please try your own more challenging accuracy tests. -You can find additional accuracy eval examples in: -- [test_eval_accuracy_large.py](https://github.com/sgl-project/sglang/blob/main/test/registered/eval/test_eval_accuracy_large.py) -- [test_gpt_oss_1gpu.py](https://github.com/sgl-project/sglang/blob/main/test/registered/core/test_gpt_oss_1gpu.py) - -## Benchmark the speed -Refer to [Benchmark and Profiling](../../developer_guide/benchmark_and_profiling.md). - -## Requesting a review for merge -You can follow the pull request merge process described in [MAINTAINER.md](https://github.com/sgl-project/sglang/blob/main/.github/MAINTAINER.md). -You will need to work with the Merge Oncall, Codeowner, and other reviewers to get their approvals. -Then your PR can be merged. - -## How to Trigger CI Tests - -We have a lot of open PRs but limited CI machines, so only top and trusted contributors have permission to trigger CI tests. -Users with permission are listed in the [CI_PERMISSIONS.json](https://github.com/sgl-project/sglang/blob/main/.github/CI_PERMISSIONS.json) - -For CI to run on a pull request, it must have the "run-ci" label. Authorized users can add the label or rerun failed tests by commenting on the PR with one of these commands: - -- `/tag-run-ci-label`: Adds the "run-ci" label. Only **future** commits trigger CI; the current commit is unaffected. -- `/rerun-failed-ci`: Reruns workflows from the latest commit with conclusion **failed, flaky, or skipped**. -- `/tag-and-rerun-ci`: Runs both. Use this on a fresh PR to kick off CI on the current commit — `/tag-run-ci-label` alone won't. -- `/rerun-stage `: Reruns a single test stage without waiting for its dependencies. Useful for quickly validating a specific test fix instead of waiting ~30 minutes for preceding stages. - -If you have permission, the [Slash Command Handler](https://github.com/sgl-project/sglang/actions/workflows/slash-command-handler.yml) will run your command and react with a 👍 to your comment. It may take up to a few minutes for the reaction to appear. Here’s a usage [example](https://github.com/sgl-project/sglang/pull/14253#issuecomment-3599509302). - -To avoid spamming a PR with too many `/rerun-failed-ci` comments, you can also trigger the command by editing an existing comment and adding any suffix (e.g., `/rerun-failed-ci try again`). - -Example of rerunning a single test stage: `/rerun-stage unit-test-backend-4-gpu`. - -If you don’t have permission, please ask maintainers to trigger CI for you. - -### CI rate limits - -Due to CI scheduling and limited resources, higher-priority PRs may preempt running jobs. In such cases, you may need to rerun the tests. - -We apply CI rate limits to prevent abuse and ensure fair usage of our CI resources. - -Each CI workflow has a default limit defined in its workflow configuration file. For example, in [pr-gate.yml](https://github.com/sgl-project/sglang/blob/main/.github/workflows/pr-gate.yml), the default cooldown period is 120 minutes, and each workflow can override it via the `cool-down-minutes` input parameter: - -```yaml -cool-down-minutes: - description: "Cooldown period in minutes for low-permission users; 0 disables rate limiting" - type: number - default: 120 -``` - -Users listed in [CI_PERMISSIONS.json](https://github.com/sgl-project/sglang/blob/main/.github/CI_PERMISSIONS.json) may have a per-user cooldown interval. In practice, we use the minimum of the workflow’s default window and the user-specific interval. - -## Code style guidance -- Avoid code duplication. If the same code snippet (more than five lines) appears multiple times, extract it into a shared function. -- Minimize device synchronization. Reduce expensive CPU-GPU synchronization operations, such as `tensor.item()` or `tensor.cpu()`, whenever possible. Use vectorized code. -- Prioritize extreme efficiency. SGLang is a runtime, and most of your code runs on the critical path for every request. Optimize all minor overheads as much as possible, especially in the model forward code. - - A common pattern is some runtime checks in the model forward pass (e.g., [this](https://github.com/sgl-project/sglang/blob/f1b0eda55c2c4838e8ab90a0fac7fb1e3d7064ab/python/sglang/srt/models/deepseek_v2.py#L486-L491)). These are very likely the same for every layer. Please cache the result as a single boolean value whenever possible. -- Make functions as pure as possible. Avoid in-place modification of arguments. -- Keep files concise. If a file exceeds 2,000 lines of code, split it into multiple smaller files. (e.g., `scheduler.py`, `scheduler_output_processor_mixin.py`) -- Keep tests run fast. - - If a single test file run longer than 500 seconds, split it into multiple smaller files (e.g., `test_eagle_infer_a.py`, `test_eagle_infer_b.py`). - - If a single job in a github workflow runs longer than 30 mins, split it into smaller jobs/steps. - - Reuse server launches in your unit tests to make tests run faster. -- When supporting new hardware or features, follow these guidelines: - - Do not drastically change existing code. - - Always prefer new files to introduce specific components for your new hardware (e.g., `allocator_npu.py`). - - If you write multiple if/else blocks for new features, ensure the common path (e.g., NVIDIA hardware or the existing code path) is the first branch. - -## How to update sgl-kernel -Since sglang and sgl-kernel are separate Python packages, our current GitHub CI infrastructure does not support updating a kernel and using it immediately within the same pull request (PR). -To add a new kernel or modify an existing one in the `sgl-kernel/` source tree, you must use multiple PRs. - -Follow these steps: - -1. Submit a PR to update the sgl-kernel source code without using it in sglang python package (e.g., [#8884](https://github.com/sgl-project/sglang/pull/8884/files)). -2. Bump the version of the kernel package (e.g., [#9220](https://github.com/sgl-project/sglang/pull/9220/files)). - - Once merged, this will trigger an automatic release of the `sglang-kernel` wheel to PyPI. - - If not urgent, you can wait for other people to release the wheel. A new version will typically be released within one week. -3. Apply the changes: - - Update the `sglang-kernel` version in `sglang/python/pyproject.toml` to use the modified kernels. - - Update the related caller code in the sglang to use the new kernel. - -## How to update sgl-kernel-npu - -Sgl-kernel-npu is the kernel package for Ascend NPU and is maintained in the [sgl-kernel-npu](https://github.com/sgl-project/sgl-kernel-npu) repository. if you want to add a new kernel and want to use it in sglang, please follow the steps in [Contribution Guide](https://github.com/sgl-project/sgl-kernel-npu/blob/main/docs/developer_guide/contribution_guide.md). - -## Tips for newcomers - -If you want to contribute but don’t have a specific idea in mind, pick issues labeled [“good first issue” or “help wanted”](https://github.com/sgl-project/sglang/issues?q=is%3Aissue+label%3A%22good+first+issue%22%2C%22help+wanted%22). These tasks typically have lower complexity and provide an excellent introduction to the codebase. Also check out this [code walk-through](https://github.com/zhaochenyang20/Awesome-ML-SYS-Tutorial/tree/main/sglang/code-walk-through) for a deeper look into SGLang’s workflow. - -If you have any questions or want to start a discussion, please feel free to ask in our [Slack channel](https://slack.sglang.io). - -Thank you for your interest in SGLang. Happy coding! diff --git a/docs/platforms/ascend/ascend_npu.md b/docs/platforms/ascend/ascend_npu.md deleted file mode 100644 index b6f1fcf30..000000000 --- a/docs/platforms/ascend/ascend_npu.md +++ /dev/null @@ -1,256 +0,0 @@ - -# SGLang installation with NPUs support - -You can install SGLang using any of the methods below. Please go through `System Settings` section to ensure the clusters are roaring at max performance. Feel free to leave an issue [here at sglang](https://github.com/sgl-project/sglang/issues) if you encounter any issues or have any problems. - -## Component Version Mapping For SGLang -| Component | Version | Obtain Way | -|-------------------|-------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| HDK | 25.5.2 | [link](https://www.hiascend.com/hardware/firmware-drivers/commercial?product=7&model=33) | -| CANN | 8.5.0 | [Obtain Images](#obtain-cann-image) | -| Pytorch Adapter | 7.3.0 | [link](https://gitcode.com/Ascend/pytorch/releases) | -| MemFabric | 1.0.5 | `pip install memfabric-hybrid==1.0.5` | -| Triton | 3.2.0 | `pip install triton-ascend`| -| SGLang NPU Kernel | NA | [link](https://github.com/sgl-project/sgl-kernel-npu/releases) | - - -### Obtain CANN Image -You can obtain the dependency of a specified version of CANN through an image. -```shell -# for Atlas 800I A3 and Ubuntu OS -docker pull quay.io/ascend/cann:8.5.0-a3-ubuntu22.04-py3.11 -# for Atlas 800I A2 and Ubuntu OS -docker pull quay.io/ascend/cann:8.5.0-910b-ubuntu22.04-py3.11 -``` - -## Preparing the Running Environment - -### Method 1: Installing from source with prerequisites - -#### Python Version - -Only `python==3.11` is supported currently. If you don't want to break system pre-installed python, try installing with [conda](https://github.com/conda/conda). - -```shell -conda create --name sglang_npu python=3.11 -conda activate sglang_npu -``` - -#### CANN - -Prior to start work with SGLang on Ascend you need to install CANN Toolkit, Kernels operator package and NNAL version 8.5.0, check the [installation guide](https://www.hiascend.com/document/detail/zh/CANNCommunityEdition/850/softwareinst/instg/instg_0008.html?Mode=PmIns&InstallType=local&OS=openEuler&Software=cannToolKit) - -#### MemFabric-Hybrid - -If you want to use PD disaggregation mode, you need to install MemFabric-Hybrid. MemFabric-Hybrid is a drop-in replacement of Mooncake Transfer Engine that enables KV cache transfer on Ascend NPU clusters. - -```shell -pip install memfabric-hybrid==1.0.5 -``` - -#### Pytorch and Pytorch Framework Adaptor on Ascend - -```shell -PYTORCH_VERSION=2.8.0 -TORCHVISION_VERSION=0.23.0 -TORCH_NPU_VERSION=2.8.0.post2 -pip install torch==$PYTORCH_VERSION torchvision==$TORCHVISION_VERSION --index-url https://download.pytorch.org/whl/cpu -pip install torch_npu==$TORCH_NPU_VERSION -``` - -If you are using other versions of `torch` and install `torch_npu`, check [installation guide](https://github.com/Ascend/pytorch/blob/master/README.md) - -#### Triton on Ascend - -We provide our own implementation of Triton for Ascend. - -```shell -pip install triton-ascend -``` -For installation of Triton on Ascend nightly builds or from sources, follow [installation guide](https://gitcode.com/Ascend/triton-ascend/blob/master/docs/sources/getting-started/installation.md) - -#### SGLang Kernels NPU -We provide SGL kernels for Ascend NPU, check [installation guide](https://github.com/sgl-project/sgl-kernel-npu/blob/main/python/sgl_kernel_npu/README.md). - -#### DeepEP-compatible Library -We provide a DeepEP-compatible Library as a drop-in replacement of deepseek-ai's DeepEP library, check the [installation guide](https://github.com/sgl-project/sgl-kernel-npu/blob/main/python/deep_ep/README.md). - -#### Some other dependencies - -```shell -# libGL -apt update -apt install libgl1 libglib2.0-0 - -# ensure setuptools contains pkg_resources module -pip install "setuptools<80" -``` - -#### Installing SGLang from source - -```shell -# Use the last release branch -git clone https://github.com/sgl-project/sglang.git -cd sglang -mv python/pyproject_npu.toml python/pyproject.toml -pip install -e python[all_npu] -``` - -### Method 2: Using Docker Image -#### Obtain Image -You can download the SGLang image or build an image based on Dockerfile to obtain the Ascend NPU image. -1. Download SGLang image -```angular2html -dockerhub: docker.io/lmsysorg/sglang:$tag -# Main-based tag, change main to specific version like v0.5.6, -# you can get image for specific version -Atlas 800I A3 : {main}-cann8.5.0-a3 -Atlas 800I A2: {main}-cann8.5.0-910b -``` -2. Build an image based on Dockerfile -```shell -# Clone the SGLang repository -git clone https://github.com/sgl-project/sglang.git -cd sglang/docker - -# Build the docker image -# If there are network errors, please modify the Dockerfile to use offline dependencies or use a proxy -# is the target architecture of the image, e.g. amd64, arm64 -docker build --build-arg TARGETARCH= -t -f npu.Dockerfile . -``` - -#### Create Docker -__Notice:__ `--privileged` and `--network=host` are required by RDMA, which is typically needed by Ascend NPU clusters. - -__Notice:__ The following docker command is based on Atlas 800I A3 machines. If you are using Atlas 800I A2, make sure only `davinci[0-7]` are mapped into container. - -```shell - -alias drun='docker run -it --rm --privileged --network=host --ipc=host --shm-size=16g \ - --device=/dev/davinci0 --device=/dev/davinci1 --device=/dev/davinci2 --device=/dev/davinci3 \ - --device=/dev/davinci4 --device=/dev/davinci5 --device=/dev/davinci6 --device=/dev/davinci7 \ - --device=/dev/davinci8 --device=/dev/davinci9 --device=/dev/davinci10 --device=/dev/davinci11 \ - --device=/dev/davinci12 --device=/dev/davinci13 --device=/dev/davinci14 --device=/dev/davinci15 \ - --device=/dev/davinci_manager --device=/dev/hisi_hdc \ - --volume /usr/local/sbin:/usr/local/sbin --volume /usr/local/Ascend/driver:/usr/local/Ascend/driver \ - --volume /usr/local/Ascend/firmware:/usr/local/Ascend/firmware \ - --volume /etc/ascend_install.info:/etc/ascend_install.info \ - --volume /var/queue_schedule:/var/queue_schedule --volume ~/.cache/:/root/.cache/' - -# Add HF_TOKEN env for download model by SGLang. -drun --env "HF_TOKEN=" \ - \ - python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --attention-backend ascend -``` - -## System Settings - -### CPU performance power scheme - -The default power scheme on Ascend hardware is `ondemand` which could affect performance, changing it to `performance` is recommended. - -```shell -echo performance | sudo tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor - -# Make sure changes are applied successfully -cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor # shows performance -``` - -### Disable NUMA balancing - -```shell -sudo sysctl -w kernel.numa_balancing=0 -# Check -cat /proc/sys/kernel/numa_balancing # shows 0 -``` - -### Prevent swapping out system memory - -```shell -sudo sysctl -w vm.swappiness=10 - -# Check -cat /proc/sys/vm/swappiness # shows 10 -``` - -## Running SGLang Service -### Running Service For Large Language Models -#### PD Mixed Scene -```shell -# Enabling CPU Affinity -export SGLANG_SET_CPU_AFFINITY=1 -python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --attention-backend ascend -``` - -#### PD Disaggregation Scene -1. Launch Prefill Server -```shell -# Enabling CPU Affinity -export SGLANG_SET_CPU_AFFINITY=1 - -# PIP: recommended to config first Prefill Server IP -# PORT: one free port -# all sglang servers need to be config the same PIP and PORT, -export ASCEND_MF_STORE_URL="tcp://PIP:PORT" -# if you are Atlas 800I A2 hardware and use rdma for kv cache transfer, add this parameter -export ASCEND_MF_TRANSFER_PROTOCOL="device_rdma" -python3 -m sglang.launch_server \ - --model-path meta-llama/Llama-3.1-8B-Instruct \ - --disaggregation-mode prefill \ - --disaggregation-transfer-backend ascend \ - --disaggregation-bootstrap-port 8995 \ - --attention-backend ascend \ - --device npu \ - --base-gpu-id 0 \ - --tp-size 1 \ - --host 127.0.0.1 \ - --port 8000 -``` - -2. Launch Decode Server -```shell -# PIP: recommended to config first Prefill Server IP -# PORT: one free port -# all sglang servers need to be config the same PIP and PORT, -export ASCEND_MF_STORE_URL="tcp://PIP:PORT" -# if you are Atlas 800I A2 hardware and use rdma for kv cache transfer, add this parameter -export ASCEND_MF_TRANSFER_PROTOCOL="device_rdma" -python3 -m sglang.launch_server \ - --model-path meta-llama/Llama-3.1-8B-Instruct \ - --disaggregation-mode decode \ - --disaggregation-transfer-backend ascend \ - --attention-backend ascend \ - --device npu \ - --base-gpu-id 1 \ - --tp-size 1 \ - --host 127.0.0.1 \ - --port 8001 -``` - -3. Launch Router -```shell -python3 -m sglang_router.launch_router \ - --pd-disaggregation \ - --policy cache_aware \ - --prefill http://127.0.0.1:8000 8995 \ - --decode http://127.0.0.1:8001 \ - --host 127.0.0.1 \ - --port 6688 -``` - -### Running Service For Multimodal Language Models -#### PD Mixed Scene -```shell -python3 -m sglang.launch_server \ - --model-path Qwen3-VL-30B-A3B-Instruct \ - --host 127.0.0.1 \ - --port 8000 \ - --tp 4 \ - --device npu \ - --attention-backend ascend \ - --mm-attention-backend ascend_attn \ - --disable-radix-cache \ - --trust-remote-code \ - --enable-multimodal \ - --sampling-backend ascend -``` diff --git a/docs/platforms/ascend/ascend_npu_best_practice.md b/docs/platforms/ascend/ascend_npu_best_practice.md deleted file mode 100644 index 943170f13..000000000 --- a/docs/platforms/ascend/ascend_npu_best_practice.md +++ /dev/null @@ -1,3922 +0,0 @@ -# Best Practice on Ascend NPU - -This section describes the best practice data of mainstream LLM models such as DeepSeek and Qwen on the Ascend NPU. If -you encounter issues or have any questions, please [open an issue](https://github.com/sgl-project/sglang/issues). - -## DeepSeek Series Models - -### Low Latency - -| Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration | -|-------------------|---------------|-------|-------------------|-----------|------|--------------|-------------------------------------------------------------------------------------------| -| Deepseek-R1 | Atlas 800I A3 | 32 | PD Disaggregation | 6K+1.6K | 20ms | W8A8 INT8 | [Optimal Configuration](#deepseek-r1-6k-1_6k-20ms-on-a3-32-cards-disaggregation-mode) | -| Deepseek-R1 | Atlas 800I A3 | 32 | PD Disaggregation | 3.9K+1K | 19ms | W8A8 INT8 | [Optimal Configuration](#deepseek-r1-3_9k-1k-19ms-on-a3-32-cards-disaggregation-mode) | -| Deepseek-R1 | Atlas 800I A3 | 32 | PD Disaggregation | 3.5K+1.5K | 19ms | W8A8 INT8 | [Optimal Configuration](#deepseek-r1-3_5k-1_5k-19ms-on-a3-32-cards-disaggregation-mode) | -| Deepseek-R1 | Atlas 800I A3 | 32 | PD Disaggregation | 3.5K+1K | 19ms | W8A8 INT8 | [Optimal Configuration](#deepseek-r1-3_5k-1k-19ms-on-a3-32-cards-disaggregation-mode) | -| DeepSeek-V3.2 | Atlas 800I A3 | 32 | PD Disaggregation | 128K+1K | 26ms | W8A8 INT8 | [Optimal Configuration](#deepseek-v32-128k-1k-26ms-on-a3-32-cards-disaggregation-mode) | - -### High Throughput - -| Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration | -|-------------|---------------|-------|-------------------|-----------|------|--------------|-----------------------------------------------------------------------------------------| -| Deepseek-R1 | Atlas 800I A3 | 32 | PD Disaggregation | 3.5K+1.5K | 50ms | W8A8 INT8 | [Optimal Configuration](#deepseek-r1-3_5k-1_5k-50ms-on-a3-32-cards-disaggregation-mode) | -| Deepseek-R1 | Atlas 800I A3 | 24 | PD Disaggregation | 2K+2K | 50ms | W8A8 INT8 | [Optimal Configuration](#deepseek-r1-2k-2k-50ms-on-a3-24-cards-disaggregation-mode) | -| Deepseek-R1 | Atlas 800I A3 | 8 | PD Mixed | 2K+2K | 50ms | W4A8 INT8 | [Optimal Configuration](#deepseek-r1-2k-2k-50ms-on-a3-8-cards-mixed-mode) | -| Deepseek-R1 | Atlas 800I A3 | 16 | PD Disaggregation | 2K+2K | 50ms | W4A8 INT8 | [Optimal Configuration](#deepseek-r1-2k-2k-50ms-on-a3-16-cards-disaggregation-mode) | -| Deepseek-R1 | Atlas 800I A3 | 8 | PD Mixed | 3.5K+1.5K | 50ms | W4A8 INT8 | [Optimal Configuration](#deepseek-r1-3_5k-1_5k-50ms-on-a3-8-cards-mixed-mode) | -| Deepseek-R1 | Atlas 800I A3 | 16 | PD Disaggregation | 3.5K+1.5K | 50ms | W4A8 INT8 | [Optimal Configuration](#deepseek-r1-3_5k-1_5k-50ms-on-a3-16-cards-disaggregation-mode) | - -## Qwen Series Models - -### Low Latency - -| Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration | -|-------------------------|---------------|-------|-------------|-----------|---------|--------------|------------------------------------------------------------------------------------| -| Qwen3-235B-A22B | Atlas 800I A3 | 8 | PD Mixed | 11K+1K | 10ms | BF16 | [Optimal Configuration](#qwen3-235b-a22b-11k-1k-10ms-on-a3-8-cards-mixed-mode) | -| Qwen3-32B | Atlas 800I A3 | 4 | PD Mixed | 6K+1.5K | 18ms | BF16 | [Optimal Configuration](#qwen3-32b-6k-1_5k-18ms-on-a3-4-cards-mixed-mode) | -| Qwen3-32B | Atlas 800I A3 | 4 | PD Mixed | 4K+1.5K | 11ms | BF16 | [Optimal Configuration](#qwen3-32b-4k-1_5k-11ms-on-a3-4-cards-mixed-mode) | -| Qwen3-32B | Atlas 800I A3 | 8 | PD Mixed | 18K+4K | 6ms | BF16 | [Optimal Configuration](#qwen3-32b-18k-4k-6ms-on-a3-8-cards-mixed-mode) | -| Qwen3-32B | Atlas 800I A2 | 8 | PD Mixed | 6K+1.5K | 18ms | W8A8 INT8 | [Optimal Configuration](#qwen3-32b-6k-1_5k-18ms-on-a2-8-cards-mixed-mode) | -| Qwen3-32B | Atlas 800I A2 | 8 | PD Mixed | 4K+1.5K | 11ms | BF16 | [Optimal Configuration](#qwen3-32b-4k-1_5k-11ms-on-a2-8-cards-mixed-mode) | -| Qwen3-32B | Atlas 800I A3 | 2 | PD Mixed | 1K+0.3K | 12ms | W8A8 INT8 | [Optimal Configuration](#qwen3-32b-1k-0_3k-12ms-on-a3-2-cards-mixed-mode) | -| Qwen3-32B | Atlas 800I A3 | 2 | PD Mixed | 6K+1.5K | 17ms | W8A8 INT8 | [Optimal Configuration](#qwen3-32b-6k-1_5k-17ms-on-a3-2-cards-mixed-mode) | -| Qwen3-8B | Atlas 800I A3 | 1 | PD Mixed | 1K+0.3K | 7ms | W8A8 INT8 | [Optimal Configuration](#qwen3-8b-1k-0_3k-7ms-on-a3-1-cards-mixed-mode) | -| Qwen3-8B | Atlas 800I A3 | 1 | PD Mixed | 6K+1.5K | 12ms | W8A8 INT8 | [Optimal Configuration](#qwen3-8b-6k-1_5k-12ms-on-a3-1-cards-mixed-mode) | -| Qwen3-8B | Atlas 800I A3 | 1 | PD Mixed | 3.5K+1.5K | 5ms | W8A8 INT8 | [Optimal Configuration](#qwen3-8b-3_5k-1_5k-5ms-on-a3-1-cards-mixed-mode) | -| Qwen3-30B-A3B | Atlas 800I A3 | 1 | PD Mixed | 6K+1.5K | 10ms | W8A8 INT8 | [Optimal Configuration](#qwen3-30b-a3b-6k-1_5k-10ms-on-a3-1-cards-mixed-mode) | -| Qwen3-30B-A3B | Atlas 800I A3 | 1 | PD Mixed | 1K+0.3K | 7ms | W8A8 INT8 | [Optimal Configuration](#qwen3-30b-a3b-1k-0_3k-7ms-on-a3-1-cards-mixed-mode) | -| Qwen3-Next-A3B-Instruct | Atlas 800I A3 | 2 | PD Mixed | 1K+0.3K | 14.21ms | W8A8 INT8 | [Optimal Configuration](#qwen3-next-1k-0_3k-14_21ms-on-a3-2-cards-mixed-mode) | -| Qwen3-Next-A3B-Instruct | Atlas 800I A3 | 2 | PD Mixed | 6K+1.5K | 15.62ms | W8A8 INT8 | [Optimal Configuration](#qwen3-next-6k-1_5k-15_62ms-on-a3-2-cards-mixed-mode) | -| Qwen3-Next-A3B-Instruct | Atlas 800I A3 | 2 | PD Mixed | 3.5K+1.5K | 20ms | W8A8 INT8 | [Optimal Configuration](#qwen3-next-3_5k-1_5k-20ms-on-a3-2-cards-mixed-mode) | -| Qwen3-14B | Atlas 800I A3 | 1 | PD Mixed | 3.5K+1.5K | 9ms | W8A8 INT8 | [Optimal Configuration](#qwen3-14b-3_5k-1_5k-9ms-on-a3-1-cards-mixed-mode) | -| Qwen3.5-397B-A17B | Atlas 800I A3 | 8 | PD Mixed | 3.5K+1.5K | 22ms | W4A8 | [Optimal Configuration](#qwen35-397B-a17b-3_5k-1_5k-22ms-on-a3-8-cards-mixed-mode) | - -### High Throughput - -| Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration | -|--------------------------------|---------------|-------|-------------------|-----------|-------|--------------|------------------------------------------------------------------------------------------------------------| -| Qwen3-235B-A22B | Atlas 800I A3 | 24 | PD Disaggregation | 3.5K+1.5K | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-235b-a22b-3_5k-1_5k-50ms-on-a3-24-cards-disaggregation-mode) | -| Qwen3-235B-A22B | Atlas 800I A3 | 8 | PD Mixed | 3.5K+1.5K | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-235b-a22b-3_5k-1_5k-50ms-on-a3-8-cards-mixed-mode) | -| Qwen3-235B-A22B | Atlas 800I A3 | 8 | PD Mixed | 2K+2K | 100ms | W8A8 INT8 | [Optimal Configuration](#qwen3-235b-a22b-2k-2k-100ms-on-a3-8-cards-mixed-mode) | -| Qwen3-235B-A22B | Atlas 800I A3 | 8 | PD Mixed | 2K+2K | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-235b-a22b-2k-2k-50ms-on-a3-8-cards-mixed-mode) | -| Qwen3-235B-A22B | Atlas 800I A3 | 16 | PD Mixed | 2K+2K | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-235b-a22b-2k-2k-50ms-on-a3-16-cards-mixed-mode) | -| Qwen3-32B | Atlas 800I A3 | 2 | PD Mixed | 3.5K+1.5K | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-32b-3_5k-1_5k-50ms-on-a3-2-cards-mixed-mode) | -| Qwen3-32B | Atlas 800I A3 | 2 | PD Mixed | 2K+2K | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-32b-2k-2k-50ms-on-a3-2-cards-mixed-mode) | -| Qwen3-30B-A3B | Atlas 800I A3 | 1 | PD Mixed | 3.5K+1.5K | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-30b-a3b-3_5k-1_5k-50ms-on-a3-1-card-mixed-mode) | -| Qwen3-Coder-480B-A35B-Instruct | Atlas 800I A3 | 24 | PD Disaggregation | 3.5K+1.5K | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-coder-480b-a35b-instruct-3_5k-1_5k-50ms-on-a3-24-cards-disaggregation-mode) | -| Qwen3-Coder-480B-A35B-Instruct | Atlas 800I A3 | 16 | PD Mixed | 3.5K+1.5K | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-coder-480b-a35b-instruct-3_5k-1_5k-50ms-on-a3-16-cards-mixed-mode) | -| Qwen3-Coder-480B-A35B-Instruct | Atlas 800I A3 | 8 | PD Mixed | 3.5K+1.5K | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-coder-480b-a35b-instruct-3_5k-1_5k-50ms-on-a3-8-cards-mixed-mode) | -| Qwen3-Next-80B-A3B-Instruct | Atlas 800I A3 | 2 | PD Mixed | 3.5K+1.5K | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-next-80B-a3b-instruct-3_5k-1_5k-50ms-on-a3-2-cards-mixed-mode) | -| Qwen3-32B | Atlas 800I A2 | 8 | PD Mixed | 3.5K+1.5K | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-32b-3_5k-1_5k-50ms-on-a2-8-cards-mixed-mode) | -| Qwen3-32B | Atlas 800I A2 | 8 | PD Mixed | 2K+2K | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-32b-2k-2k-50ms-on-a2-8-cards-mixed-mode) | -| Qwen3-14B | Atlas 800I A3 | 1 | PD Mixed | 3.5K+1.5K | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-14b-3_5k-1_5k-50ms-on-a3-1-cards-mixed-mode) | -| Qwen3-8B | Atlas 800I A3 | 1 | PD Mixed | 3.5K+1.5K | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-8b-3_5k-1_5k-50ms-on-a3-1-cards-mixed-mode) | -| Qwen3.5-397B-A17B | Atlas 800I A3 | 8 | PD Mixed | 3.5K+1.5K | 50ms | W4A8 | [Optimal Configuration](#qwen35-397B-a17b-3_5k-1_5k-50ms-on-a3-8-cards-mixed-mode) | - -## Optimal Configuration - -### DeepSeek-R1 3_5K-1_5K 50ms on A3 32 Cards Disaggregation Mode - -Model: Deepseek R1 - -Hardware: Atlas 800I A3 32Card - -DeployMode: PD Disaggregation - -Dataset: random - -Input Output Length: 3.5K+1.5K - -TPOT: 50ms - -#### Model Deployment - -```shell -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 -export SGLANG_SET_CPU_AFFINITY=1 -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export STREAMS_PER_DEVICE=32 -export HCCL_OP_EXPANSION_MODE=AIV -export SGLANG_NPU_USE_MLAPO=1 -export SGLANG_USE_FIA_NZ=1 -export SGLANG_NPU_USE_MULTI_STREAM=1 - -export ASCEND_MF_STORE_URL="tcp://your prefill ip1:24669" - -P_IP=('your prefill ip1' 'your prefill ip2') - -D_IP=('your decode ip1' 'your decode ip2') - -MODEL_PATH=xxx - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" -# prefill -for i in "${!P_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; - then - echo "${P_IP[$i]}" - export SGLANG_USE_AG_AFTER_QLORA=1 - export HCCL_BUFFSIZE=800 - export TASK_QUEUE_ENABLE=2 - export SGLANG_NPU_FUSED_MOE_MODE=2 - export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=131072 - - export HCCL_SOCKET_IFNAME=lo - export GLOO_SOCKET_IFNAME=lo - python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode prefill --host ${P_IP[$i]} \ - --port 8000 --disaggregation-bootstrap-port $((8998+$i)) --trust-remote-code --nnodes 1 --node-rank 0 \ - --tp-size 16 --mem-fraction-static 0.778 --attention-backend ascend --device npu \ - --disaggregation-transfer-backend ascend --max-running-requests 16 --disable-radix-cache \ - --chunked-prefill-size -1 --max-prefill-tokens 60000 --moe-a2a-backend ascend_fuseep --deepep-mode normal \ - --speculative-algorithm NEXTN --speculative-num-steps 1 --speculative-eagle-topk 1 --speculative-num-draft-tokens 2 \ - --dp-size 4 --enable-dp-attention --disable-shared-experts-fusion --dtype bfloat16 --enable-attn-tp-input-scattered - NODE_RANK=$i - break - fi -done - -# decode -for i in "${!D_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; - then - echo "${D_IP[$i]}" - export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - export SGLANG_ENABLE_SPEC_V2=1 - export HCCL_BUFFSIZE=600 - export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64 - export TASK_QUEUE_ENABLE=1 - export SGLANG_NPU_FUSED_MOE_MODE=1 - export SGLANG_LM_HEAD_TP=8 - export HCCL_SOCKET_IFNAME=xxx - export GLOO_SOCKET_IFNAME=xxx - python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode decode --host ${D_IP[$i]} \ - --port 8001 --trust-remote-code --dist-init-addr ${D_IP[0]}:5000 --nnodes 2 --node-rank $i --tp-size 32 --dp-size 32 \ - --mem-fraction-static 0.82 --max-running-requests 1024 --attention-backend ascend --device npu --quantization modelslim \ - --moe-a2a-backend ascend_fuseep --enable-dp-attention --deepep-mode low_latency --moe-dense-tp 1 \ - --cuda-graph-bs 2 4 6 8 10 12 14 16 18 20 22 24 26 28 30 32 --disaggregation-transfer-backend ascend --watchdog-timeout 9000 --context-length 8192 \ - --speculative-algorithm NEXTN --speculative-num-steps 1 --speculative-eagle-topk 1 --speculative-num-draft-tokens 2 \ - --tokenizer-worker-num 4 --disable-shared-experts-fusion --dtype bfloat16 \ - --load-balance-method round_robin - NODE_RANK=$i - break - fi -done - -``` - -```shell -export SGLANG_DP_ROUND_ROBIN=1 -python -m sglang_router.launch_router \ - --pd-disaggregation \ - --policy cache_aware \ - --prefill http://P_IP:8000 8998 \ - --prefill http://P_IP:8000 8999 \ - --decode http://D_IP:8001 \ - --host 127.0.0.1 \ - --port 6688 \ - --mini-lb -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell -python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6688 --max-concurrency 768 --random-input-len 3500 --random-output-len 1500 --num-prompts 3072 --random-range-ratio 1 --request-rate 16 -``` - -### DeepSeek-R1 2K-2K 50ms on A3 24 Cards Disaggregation Mode - -Model: Deepseek R1 - -Hardware: Atlas 800I A3 24Card - -DeployMode: PD Disaggregation - -Dataset: random - -Input Output Length: 2K+2K - -TPOT: 50ms - -#### Model Deployment - -```shell -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -export SGLANG_SET_CPU_AFFINITY=1 -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export STREAMS_PER_DEVICE=32 -export SGLANG_NPU_USE_MLAPO=1 -export SGLANG_USE_FIA_NZ=1 - -export ASCEND_MF_STORE_URL="tcp://your prefill ip1:24669" - -P_IP=('your prefill ip1') -D_IP=('your decode ip1' 'your decode ip2') - -MODEL_PATH=xxx - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" -# prefill -for i in "${!P_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; - then - echo "${P_IP[$i]}" - export HCCL_BUFFSIZE=1600 - export TASK_QUEUE_ENABLE=2 - export SGLANG_USE_AG_AFTER_QLORA=1 - export HCCL_SOCKET_IFNAME=lo - export GLOO_SOCKET_IFNAME=lo - - python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode prefill --host ${P_IP[$i]} \ - --port 8000 --disaggregation-bootstrap-port $((8998+$i)) --trust-remote-code --nnodes 1 --node-rank 0 \ - --tp-size 16 --mem-fraction-static 0.8 --attention-backend ascend --device npu --quantization modelslim \ - --disaggregation-transfer-backend ascend --max-running-requests 20 --context-length 8192 --disable-radix-cache \ - --chunked-prefill-size -1 --max-prefill-tokens 28680 --moe-a2a-backend deepep --deepep-mode normal \ - --speculative-algorithm NEXTN --speculative-num-steps 1 --speculative-eagle-topk 1 --speculative-num-draft-tokens 2 \ - --dp-size 4 --enable-dp-attention --disable-shared-experts-fusion --dtype bfloat16 --enable-attn-tp-input-scattered - NODE_RANK=$i - break - fi -done - -# decode -for i in "${!D_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; - then - echo "${D_IP[$i]}" - export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - export SGLANG_ENABLE_SPEC_V2=1 - export HCCL_BUFFSIZE=800 - export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=102 - export TASK_QUEUE_ENABLE=1 - export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1 - export SGLANG_NPU_FUSED_MOE_MODE=1 - export HCCL_SOCKET_IFNAME=xxx - export GLOO_SOCKET_IFNAME=xxx - - python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode decode --host ${D_IP[$i]} \ - --port 8001 --trust-remote-code --dist-init-addr ${D_IP[0]}:5000 --nnodes 2 --node-rank $i --tp-size 32 --dp-size 32 \ - --mem-fraction-static 0.81 --max-running-requests 1088 --attention-backend ascend --device npu --quantization modelslim \ - --moe-a2a-backend ascend_fuseep --enable-dp-attention --deepep-mode low_latency --enable-dp-lm-head --moe-dense-tp 1 \ - --cuda-graph-bs 2 4 6 8 10 12 14 16 18 20 22 24 26 28 30 32 34 --disaggregation-transfer-backend ascend --watchdog-timeout 9000 --context-length 8192 \ - --speculative-algorithm NEXTN --speculative-num-steps 2 --speculative-eagle-topk 1 --speculative-num-draft-tokens 3 \ - --tokenizer-worker-num 4 --disable-shared-experts-fusion --dtype bfloat16 \ - --load-balance-method round_robin - NODE_RANK=$i - break - fi -done - -``` - -```shell -python -m sglang_router.launch_router \ - --pd-disaggregation \ - --policy cache_aware \ - --prefill http://P_IP:8000 8998 \ - --prefill http://P_IP:8000 8999 \ - --decode http://D_IP:8001 \ - --host 127.0.0.1 \ - --port 6688 \ - --mini-lb -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell -python -m sglang.bench_serving --dataset-name random --backend sglang \ ---host 127.0.0.1 \ ---port 6688 \ ---max-concurrency 1088 \ ---random-input-len 2048 \ ---random-output-len 2048 \ ---num-prompts 12800 \ ---random-range-ratio 1 \ ---request-rate 24 -``` - -### DeepSeek-R1 6K-1_6K 20ms on A3 32 Cards Disaggregation Mode - -Model: Deepseek R1 - -Hardware: Atlas 800I A3 32Card - -DeployMode: PD Disaggregation - -Dataset: random - -Input Output Length: 6K+1.6K - -TPOT: 20ms - -#### Model Deployment - -```shell -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 -export SGLANG_SET_CPU_AFFINITY=1 -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export STREAMS_PER_DEVICE=32 -export ASCEND_MF_STORE_URL="tcp://your prefill ip1:24669" - -P_IP=('your prefill ip1' 'your prefill ip2') - -D_IP=('your decode ip1' 'your decode ip2') - -MODEL_PATH=xxx - -export SGLANG_NPU_USE_MLAPO=1 -export SGLANG_USE_FIA_NZ=1 - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" -# prefill -for i in "${!P_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; - then - echo "${P_IP[$i]}" - export HCCL_BUFFSIZE=1536 - export TASK_QUEUE_ENABLE=2 - - export HCCL_SOCKET_IFNAME=lo - export GLOO_SOCKET_IFNAME=lo - python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode prefill --host ${P_IP[$i]} \ - --port 8000 --disaggregation-bootstrap-port $((8998+$i)) --trust-remote-code --nnodes 1 --node-rank 0 \ - --tp-size 16 --mem-fraction-static 0.81 --attention-backend ascend --device npu --quantization modelslim \ - --disaggregation-transfer-backend ascend --max-running-requests 4 --disable-radix-cache \ - --chunked-prefill-size -1 --max-prefill-tokens 28680 --moe-a2a-backend deepep --deepep-mode normal \ - --speculative-algorithm NEXTN --speculative-num-steps 1 --speculative-eagle-topk 1 --speculative-num-draft-tokens 2 \ - --dp-size 2 --enable-dp-attention --disable-shared-experts-fusion --dtype bfloat16 --enable-attn-tp-input-scattered - NODE_RANK=$i - break - fi -done - -# decode -for i in "${!D_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; - then - echo "${D_IP[$i]}" - export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - export SGLANG_ENABLE_SPEC_V2=1 - export HCCL_BUFFSIZE=650 - export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=16 - export TASK_QUEUE_ENABLE=1 - export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1 - export HCCL_SOCKET_IFNAME=xxx - export GLOO_SOCKET_IFNAME=xxx - python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode decode --host ${D_IP[$i]} \ - --port 8001 --trust-remote-code --dist-init-addr DIP1:5000 --nnodes 2 --node-rank $i --tp-size 32 --dp-size 8 \ - --mem-fraction-static 0.75 --max-running-requests 32 --attention-backend ascend --device npu --quantization modelslim \ - --moe-a2a-backend deepep --enable-dp-attention --deepep-mode low_latency --enable-dp-lm-head --moe-dense-tp 1 \ - --cuda-graph-bs 2 4 6 --disaggregation-transfer-backend ascend --watchdog-timeout 9000 \ - --speculative-algorithm NEXTN --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ - --tokenizer-worker-num 4 --disable-shared-experts-fusion --dtype bfloat16 \ - --load-balance-method round_robin - NODE_RANK=$i - break - fi -done - -``` - -```shell -export SGLANG_DP_ROUND_ROBIN=1 -python -m sglang_router.launch_router \ - --pd-disaggregation \ - --policy cache_aware \ - --prefill http://P_IP:8000 8998 \ - --prefill http://P_IP:8000 8999 \ - --decode http://D_IP:8001 \ - --host 127.0.0.1 \ - --port 6688 \ - --mini-lb -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell -python -m sglang.bench_serving --dataset-name random --backend sglang \ - --host 127.0.0.1 \ - --port 6688 \ - --max-concurrency 32 \ - --random-input-len 6000 \ - --random-output-len 1600 \ - --num-prompts 32 \ - --random-range-ratio 1 \ - --request-rate 16 -``` - -### DeepSeek-R1 3_9K-1K 19ms on A3 32 Cards Disaggregation Mode - -Model: Deepseek R1 - -Hardware: Atlas 800I A3 32Card - -DeployMode: PD Disaggregation - -Dataset: random - -Input Output Length: 3.9K+1K - -TPOT: 19ms - -#### Model Deployment - -```shell -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -export SGLANG_SET_CPU_AFFINITY=1 -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export STREAMS_PER_DEVICE=32 -export SGLANG_NPU_USE_MLAPO=1 -export SGLANG_USE_FIA_NZ=1 -export ASCEND_MF_STORE_URL="tcp://your prefill ip1:24669" - -P_IP=('your prefill ip1' 'your prefill ip2') -D_IP=('your decode ip1' 'your decode ip2') - -MODEL_PATH=xxx - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -# prefill -for i in "${!P_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; - then - echo "${P_IP[$i]}" - export HCCL_BUFFSIZE=1536 - export TASK_QUEUE_ENABLE=2 - export HCCL_SOCKET_IFNAME=lo - export GLOO_SOCKET_IFNAME=lo - - python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode prefill --host ${P_IP[$i]} \ - --port 8000 --disaggregation-bootstrap-port $((8998+$i)) --trust-remote-code --nnodes 1 --node-rank 0 \ - --tp-size 16 --mem-fraction-static 0.81 --attention-backend ascend --device npu --quantization modelslim \ - --disaggregation-transfer-backend ascend --max-running-requests 4 --context-length 8192 --disable-radix-cache \ - --chunked-prefill-size -1 --max-prefill-tokens 28680 --moe-a2a-backend deepep --deepep-mode normal \ - --speculative-algorithm NEXTN --speculative-num-steps 1 --speculative-eagle-topk 1 --speculative-num-draft-tokens 2 \ - --dp-size 2 --enable-dp-attention --disable-shared-experts-fusion --dtype bfloat16 --enable-attn-tp-input-scattered - NODE_RANK=$i - break - fi -done - -# decode -for i in "${!D_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; - then - echo "${D_IP[$i]}" - export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - export SGLANG_ENABLE_SPEC_V2=1 - export HCCL_BUFFSIZE=650 - export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=12 - export TASK_QUEUE_ENABLE=1 - export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1 - export HCCL_SOCKET_IFNAME=xxx - export GLOO_SOCKET_IFNAME=xxx - python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode decode --host ${D_IP[$i]} \ - --port 8001 --trust-remote-code --dist-init-addr DIP1:5000 --nnodes 2 --node-rank $i --tp-size 32 --dp-size 16 \ - --mem-fraction-static 0.75 --max-running-requests 32 --attention-backend ascend --device npu --quantization modelslim \ - --moe-a2a-backend deepep --enable-dp-attention --deepep-mode low_latency --enable-dp-lm-head --moe-dense-tp 1 \ - --cuda-graph-bs 2 4 6 --disaggregation-transfer-backend ascend --watchdog-timeout 9000 --context-length 8192 \ - --speculative-algorithm NEXTN --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ - --tokenizer-worker-num 4 --disable-shared-experts-fusion --dtype bfloat16 \ - --load-balance-method round_robin - NODE_RANK=$i - break - fi -done -``` - -```shell -export SGLANG_DP_ROUND_ROBIN=1 -python -m sglang_router.launch_router \ - --pd-disaggregation \ - --policy cache_aware \ - --prefill http://P_IP:8000 8998 \ - --prefill http://P_IP:8000 8999 \ - --decode http://D_IP:8001 \ - --host 127.0.0.1 \ - --port 6688 \ - --mini-lb -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell -python -m sglang.bench_serving --dataset-name random --backend sglang \ - --host 127.0.0.1 \ - --port 6688 \ - --max-concurrency 32 \ - --random-input-len 3900 \ - --random-output-len 1024 \ - --num-prompts 32 \ - --random-range-ratio 1 \ - --request-rate 16 -``` - -### DeepSeek-R1 3_5K-1_5K 19ms on A3 32 Cards Disaggregation Mode - -Model: Deepseek R1 - -Hardware: Atlas 800I A3 32Card - -DeployMode: PD Disaggregation - -Dataset: random - -Input Output Length: 3.5K+1.5K - -TPOT: 19ms - -#### Model Deployment - -Please Turn to [DeepSeek-R1 3_9K-1K 19ms on A3 32 Cards Disaggregation Mode](#deepseek-r1-3_9k-1k-19ms-on-a3-32-cards-disaggregation-mode) - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell -python -m sglang.bench_serving --dataset-name random --backend sglang \ - --host 127.0.0.1 \ - --port 6688 \ - --max-concurrency 32 \ - --random-input-len 3500 \ - --random-output-len 1500 \ - --num-prompts 32 \ - --random-range-ratio 1 \ - --request-rate 16 -``` - -### DeepSeek-R1 3_5K-1K 19ms on A3 32 Cards Disaggregation Mode - -Model: Deepseek R1 - -Hardware: Atlas 800I A3 32Card - -DeployMode: PD Disaggregation - -Dataset: random - -Input Output Length: 3.5K+1K - -TPOT: 19ms - -#### Model Deployment - -Please Turn to [DeepSeek-R1 3_9K-1K 19ms on A3 32 Cards Disaggregation Mode](#deepseek-r1-3_9k-1k-19ms-on-a3-32-cards-disaggregation-mode) - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell -python -m sglang.bench_serving --dataset-name random --backend sglang \ - --host 127.0.0.1 \ - --port 6688 \ - --max-concurrency 32 \ - --random-input-len 3500 \ - --random-output-len 1024 \ - --num-prompts 32 \ - --random-range-ratio 1 \ - --request-rate 16 -``` - -### DeepSeek-R1 2K-2K 50ms on A3 8 Cards Mixed Mode - -Model: Deepseek R1 - -Hardware: Atlas 800I A3 8Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 2K+2K - -TPOT: 50ms - -#### Model Deployment - -```shell -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -export SGLANG_SET_CPU_AFFINITY=1 -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export STREAMS_PER_DEVICE=32 -export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 -export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200 - -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo - -export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=88 -export HCCL_BUFFSIZE=1600 -export DEEPEP_NORMAL_LONG_SEQ_ROUND=10 -export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=512 - -MODEL_PATH=xxx - -export SGLANG_NPU_USE_MLAPO=1 -export SGLANG_ENABLE_SPEC_V2=1 -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_USE_FIA_NZ=1 - -python3 -m sglang.launch_server --model-path ${MODEL_PATH} \ ---tp 16 \ ---trust-remote-code \ ---attention-backend ascend \ ---device npu \ ---quantization modelslim \ ---watchdog-timeout 9000 \ ---host 127.0.0.1 --port 6699 \ ---cuda-graph-bs 4 8 20 21 22 \ ---mem-fraction-static 0.78 \ ---max-running-requests 352 \ ---disable-radix-cache --chunked-prefill-size -1 --max-prefill-tokens 1500 \ ---moe-a2a-backend deepep --deepep-mode auto \ ---enable-dp-attention --dp-size 16 --enable-dp-lm-head \ ---speculative-algorithm NEXTN --speculative-num-steps 2 --speculative-eagle-topk 1 --speculative-num-draft-tokens 3 \ ---dtype bfloat16 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell -python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6699 --max-concurrency 352 --random-input-len 2048 --random-output-len 2048 --num-prompts 1408 --random-range-ratio 1 -``` - -### DeepSeek-R1 2K-2K 50ms on A3 16 Cards Disaggregation Mode - -Model: Deepseek R1 - -Hardware: Atlas 800I A3 16Card - -DeployMode: PD Disaggregation - -Dataset: random - -Input Output Length: 2K+2K - -TPOT: 50ms - -#### Model Deployment - -```shell -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -export SGLANG_SET_CPU_AFFINITY=1 -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export STREAMS_PER_DEVICE=32 - -export ASCEND_MF_STORE_URL="tcp://your prefill ip1:24667" - -P_IP=('your prefill ip1') - -D_IP=('your decode ip1') - -MODEL_PATH=xxx - -export SGLANG_NPU_USE_MLAPO=1 -export SGLANG_USE_FIA_NZ=1 -export ENABLE_MOE_NZ=1 - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -# prefill -for i in "${!P_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; - then - echo "${P_IP[$i]}" - export HCCL_BUFFSIZE=2600 - export TASK_QUEUE_ENABLE=2 - - export HCCL_SOCKET_IFNAME=lo - export GLOO_SOCKET_IFNAME=lo - python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode prefill --host ${P_IP[$i]} \ - --port 8000 --disaggregation-bootstrap-port $((8998+$i)) --trust-remote-code --nnodes 1 --node-rank 0 \ - --tp-size 16 --mem-fraction-static 0.7 --attention-backend ascend --device npu --quantization modelslim \ - --disaggregation-transfer-backend ascend --max-running-requests 32 --context-length 8192 --disable-radix-cache \ - --chunked-prefill-size -1 --max-prefill-tokens 10240 --moe-a2a-backend deepep --deepep-mode normal \ - --speculative-algorithm NEXTN --speculative-num-steps 1 --speculative-eagle-topk 1 --speculative-num-draft-tokens 2 \ - --dp-size 8 --enable-dp-attention --disable-shared-experts-fusion --dtype bfloat16 - NODE_RANK=$i - break - fi -done - -# decode -for i in "${!D_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; - then - echo "${D_IP[$i]}" - export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - export SGLANG_ENABLE_SPEC_V2=1 - export HCCL_BUFFSIZE=900 - export SGLANG_DP_ROUND_ROBIN=1 - export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=112 - export TASK_QUEUE_ENABLE=1 - export HCCL_SOCKET_IFNAME=xxx - export GLOO_SOCKET_IFNAME=xxx - python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode decode --host ${D_IP[$i]} \ - --port 8001 --trust-remote-code --nnodes 1 --node-rank 0 --tp-size 16 --dp-size 16 \ - --mem-fraction-static 0.8 --max-running-requests 448 --attention-backend ascend --device npu --quantization modelslim \ - --moe-a2a-backend deepep --enable-dp-attention --deepep-mode low_latency --enable-dp-lm-head \ - --cuda-graph-bs 2 4 6 8 10 12 14 16 18 20 22 24 26 28 --disaggregation-transfer-backend ascend --watchdog-timeout 9000 --context-length 8192 \ - --speculative-algorithm NEXTN --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ - --disable-shared-experts-fusion --dtype bfloat16 --tokenizer-worker-num 4 \ - --load-balance-method round_robin - NODE_RANK=$i - break - fi -done - -``` - -```shell -python -m sglang_router.launch_router \ - --pd-disaggregation \ - --policy cache_aware \ - --prefill http://P_IP:8000 8998 \ - --decode http://D_IP:8001 \ - --host 127.0.0.1 \ - --port 6688 \ - --mini-lb -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell -python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6688 --max-concurrency 448 --random-input-len 2048 --random-output-len 2048 --num-prompts 1792 --random-range-ratio 1 --request-rate 32 -``` - -### DeepSeek-R1 3_5K-1_5K 50ms on A3 8 Cards Mixed Mode - -Model: Deepseek R1 - -Hardware: Atlas 800I A3 8Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 3.5K+1.5K - -TPOT: 50ms - -#### Model Deployment - -```shell -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True - -export STREAMS_PER_DEVICE=32 -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 -export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200 -export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=56 -export HCCL_BUFFSIZE=1200 -export DEEPEP_NORMAL_LONG_SEQ_ROUND=10 -export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=512 -export SGLANG_NPU_USE_MLAPO=1 -export SGLANG_ENABLE_SPEC_V2=1 -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_USE_FIA_NZ=1 - -MODEL_PATH=xxx - -python3 -m sglang.launch_server --model-path ${MODEL_PATH} \ ---tp 16 \ ---trust-remote-code \ ---attention-backend ascend \ ---device npu \ ---quantization modelslim \ ---watchdog-timeout 9000 \ ---host 127.0.0.1 --port 6699 \ ---cuda-graph-bs 4 8 12 14 \ ---mem-fraction-static 0.77 \ ---max-running-requests 224 \ ---context-length 8188 --disable-radix-cache --chunked-prefill-size -1 --max-prefill-tokens 3000 \ ---moe-a2a-backend deepep --deepep-mode auto \ ---enable-dp-attention --dp-size 16 --enable-dp-lm-head \ ---speculative-algorithm NEXTN --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ ---dtype bfloat16 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell -python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6699 --max-concurrency 224 --random-input-len 3500 --random-output-len 1500 --num-prompts 896 --random-range-ratio 1 -``` - -### DeepSeek-R1 3_5K-1_5K 50ms on A3 16 Cards Disaggregation Mode - -Model: Deepseek R1 - -Hardware: Atlas 800I A3 16Card - -DeployMode: PD Disaggregation - -Dataset: random - -Input Output Length: 3.5K+1.5K - -TPOT: 50ms - -#### Model Deployment - -```shell -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -export SGLANG_SET_CPU_AFFINITY=1 -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export STREAMS_PER_DEVICE=32 - -export ASCEND_MF_STORE_URL="tcp://your prefill ip1:24667" - -P_IP=('your prefill ip1') - -D_IP=('your decode ip1') - -MODEL_PATH=xxx - -export SGLANG_NPU_USE_MLAPO=1 -export SGLANG_USE_FIA_NZ=1 -export ENABLE_MOE_NZ=1 - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -# prefill -for i in "${!P_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; - then - echo "${P_IP[$i]}" - export HCCL_BUFFSIZE=3500 - export TASK_QUEUE_ENABLE=2 - - export HCCL_SOCKET_IFNAME=lo - export GLOO_SOCKET_IFNAME=lo - python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode prefill --host ${P_IP[$i]} \ - --port 8000 --disaggregation-bootstrap-port $((8998+$i)) --trust-remote-code --nnodes 1 --node-rank 0 \ - --tp-size 16 --mem-fraction-static 0.62 --attention-backend ascend --device npu --quantization modelslim \ - --disaggregation-transfer-backend ascend --max-running-requests 32 --context-length 8192 --disable-radix-cache \ - --chunked-prefill-size -1 --max-prefill-tokens 20480 --moe-a2a-backend deepep --deepep-mode normal \ - --speculative-algorithm NEXTN --speculative-num-steps 1 --speculative-eagle-topk 1 --speculative-num-draft-tokens 2 \ - --dp-size 8 --enable-dp-attention --disable-shared-experts-fusion --dtype bfloat16 - NODE_RANK=$i - break - fi -done - -# decode -for i in "${!D_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; - then - echo "${D_IP[$i]}" - export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - export SGLANG_ENABLE_SPEC_V2=1 - export HCCL_BUFFSIZE=800 - export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=78 - export TASK_QUEUE_ENABLE=1 - export HCCL_SOCKET_IFNAME=xxx - export GLOO_SOCKET_IFNAME=xxx - python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode decode --host ${D_IP[$i]} \ - --port 8001 --trust-remote-code --nnodes 1 --node-rank 0 --tp-size 16 --dp-size 16 \ - --mem-fraction-static 0.805 --max-running-requests 416 --attention-backend ascend --device npu --quantization modelslim \ - --moe-a2a-backend deepep --enable-dp-attention --deepep-mode low_latency --enable-dp-lm-head \ - --cuda-graph-bs 2 4 6 8 10 12 14 16 18 20 22 24 26 --disaggregation-transfer-backend ascend --watchdog-timeout 9000 --context-length 8192 \ - --speculative-algorithm NEXTN --speculative-num-steps 2 --speculative-eagle-topk 1 --speculative-num-draft-tokens 3 \ - --disable-shared-experts-fusion --dtype bfloat16 --tokenizer-worker-num 4 \ - --load-balance-method round_robin - NODE_RANK=$i - break - fi -done - -``` - -```shell -python -m sglang_router.launch_router \ - --pd-disaggregation \ - --policy cache_aware \ - --prefill http://P_IP:8000 8998 \ - --decode http://D_IP:8001 \ - --host 127.0.0.1 \ - --port 6688 \ - --mini-lb -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell -python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6688 --max-concurrency 416 --random-input-len 3500 --random-output-len 1500 --num-prompts 1664 --random-range-ratio 1 -``` - -### DeepSeek-V3.2 128K-1K 26ms on A3 32 Cards Disaggregation Mode - -Model: DeepSeek-V3.2-W8A8 - -Hardware: Atlas 800I A3 32Card - -DeployMode: PD Disaggregation - -Dataset: random - -Input Output Length: 128K+1K - -TPOT: 26ms - -#### Model Deployment - -```shell -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING - -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -export LD_LIBRARY_PATH=/usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/op_api/lib/:${LD_LIBRARY_PATH} -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -export SGLANG_SET_CPU_AFFINITY=1 -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export STREAMS_PER_DEVICE=32 -export ASCEND_MF_STORE_URL="tcp://your prefill ip1:24670" - -P_IP=('your prefill ip1' 'your prefill ip2') -D_IP=('your decode ip1' 'your decode ip2') -MODEL_PATH=xxx - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -# prefill -for i in "${!P_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; - then - echo "${P_IP[$i]}" - export HCCL_BUFFSIZE=1200 - export TASK_QUEUE_ENABLE=2 - export HCCL_SOCKET_IFNAME=xxx - export GLOO_SOCKET_IFNAME=xxx - - python3 -m sglang.launch_server --model-path ${MODEL_PATH} \ - --tp 32 \ - --trust-remote-code \ - --attention-backend ascend \ - --device npu \ - --watchdog-timeout 9000 \ - --host ${P_IP[$i]} --port 8000 \ - --mem-fraction-static 0.73 \ - --disable-radix-cache --chunked-prefill-size -1 --max-prefill-tokens 68000 \ - --max-running-requests 1 \ - --moe-a2a-backend deepep --deepep-mode normal \ - --quantization modelslim \ - --disaggregation-transfer-backend ascend \ - --disaggregation-mode prefill \ - --disable-cuda-graph \ - --nnodes 2 --node-rank $i \ - --disaggregation-bootstrap-port 8995 \ - --moe-dense-tp-size 1 \ - --enable-dsa-prefill-context-parallel \ - --dsa-prefill-cp-mode in-seq-split \ - --attn-cp-size 32 \ - --speculative-algorithm NEXTN --speculative-num-steps 1 --speculative-eagle-topk 1 --speculative-num-draft-tokens 2 \ - --dist-init-addr ${P_IP[0]}:10000 - break - fi -done - - -# decode -for i in "${!D_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; - then - echo "${D_IP[$i]}" - export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - export SGLANG_ENABLE_SPEC_V2=1 - - export TASK_QUEUE_ENABLE=0 - export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1 - - export HCCL_SOCKET_IFNAME=xxx - export GLOO_SOCKET_IFNAME=xxx - - DP=8 - export HCCL_BUFFSIZE=400 - export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=8 - - python3 -m sglang.launch_server --model-path ${MODEL_PATH} \ - --tp 32 \ - --dp ${DP} \ - --ep 32 \ - --moe-dense-tp-size 1 \ - --enable-dp-attention \ - --enable-dp-lm-head \ - --trust-remote-code \ - --attention-backend ascend \ - --device npu \ - --watchdog-timeout 9000 \ - --host ${D_IP[$i]} --port 8001 \ - --mem-fraction-static 0.79 \ - --disable-radix-cache \ - --chunked-prefill-size -1 --max-prefill-tokens 68000 \ - --max-running-requests 32 \ - --cuda-graph-max-bs 4 \ - --moe-a2a-backend deepep \ - --deepep-mode low_latency \ - --quantization modelslim \ - --speculative-algorithm NEXTN --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ - --disaggregation-transfer-backend ascend \ - --disaggregation-mode decode \ - --nnodes 2 --node-rank $i \ - --dist-init-addr ${D_IP[0]}:10000 - break - fi -done -``` - - -```shell -python -m sglang_router.launch_router \ - --pd-disaggregation \ - --policy cache_aware \ - --prefill http://P_IP1:8000 8995 \ - --decode http://D_IP1:8001 \ - --host 127.0.0.1 \ - --port 6688 \ - --mini-lb -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell -python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6688 --max-concurrency 8 --random-input-len 131076 --random-output-len 1024 --num-prompts 8 --random-range-ratio 1 -``` - -### Qwen3-235B-A22B 3_5K-1_5K 50ms on A3 24 Cards Disaggregation Mode - -Model: Qwen3-235B-A22B-W8A8 - -Hardware: Atlas 800I A3 24Card - -DeployMode: PD Disaggregation - -Dataset: random - -Input Output Length: 3.5K+1.5K - -TPOT: 50ms - -#### Model Deployment - -```shell -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING - -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash - -export SGLANG_SET_CPU_AFFINITY=1 -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_ENABLE_SPEC_V2=1 -export SGLANG_DP_ROUND_ROBIN=1 -export SGLANG_NPU_FUSED_MOE_MODE=2 - -MODEL_PATH=xxx -export ASCEND_MF_STORE_URL="tcp://your prefill ip1:24667" -P_IP=('your prefill ip1') -D_IP=('your decode ip1' 'your decode ip2') - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - - -for i in "${!P_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; - then - echo "${P_IP[$i]}" - source /usr/local/Ascend/ascend-toolkit/set_env.sh - source /usr/local/Ascend/nnal/atb/set_env.sh - export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=188416 - export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=1024 - export DEEPEP_NORMAL_LONG_SEQ_ROUND=16 - export HCCL_BUFFSIZE=4300 - export TASK_QUEUE_ENABLE=2 - export HCCL_SOCKET_IFNAME=lo - export GLOO_SOCKET_IFNAME=lo - export STREAMS_PER_DEVICE=32 - - # P节点 - python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode prefill \ - --host ${P_IP[$i]} --port 8000 --disaggregation-bootstrap-port 8995 --trust-remote-code \ - --nnodes 1 --node-rank $i --tp-size 16 --dp-size 16 --mem-fraction-static 0.6 \ - --disable-radix-cache \ - --attention-backend ascend --device npu --quantization modelslim --disaggregation-transfer-backend ascend \ - --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ - --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ - --speculative-draft-model-quantization unquant \ - --max-running-requests 128 --chunked-prefill-size 94208 --max-prefill-tokens 262144 \ - --enable-dp-attention \ - --moe-a2a-backend ascend_fuseep --dtype bfloat16 - NODE_RANK=$i - break - fi -done - - -for i in "${!D_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; - then - echo "${D_IP[$i]}" - source /usr/local/Ascend/ascend-toolkit/set_env.sh - source /usr/local/Ascend/nnal/atb/set_env.sh - export DP_ROUND_ROBIN=1 - export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=65536 - export HCCL_BUFFSIZE=800 - export HCCL_SOCKET_IFNAME=data0.3001 - export GLOO_SOCKET_IFNAME=data0.3001 - export STREAMS_PER_DEVICE=32 - - python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode decode \ - --host ${D_IP[$i]} --port 8001 --trust-remote-code \ - --nnodes 2 --node-rank $i --tp-size 32 --dp-size 32 --mem-fraction-static 0.83 --max-running-requests 768 \ - --attention-backend ascend --device npu --quantization modelslim --enable-dp-attention \ - --moe-a2a-backend ascend_fuseep --cuda-graph-bs 6 8 12 15 18 20 22 24 \ - --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ - --speculative-draft-model-quantization unquant \ - --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ - --dist-init-addr xxx:5000 \ - --disaggregation-transfer-backend ascend --watchdog-timeout 9000 --context-length 8192 \ - --enable-dp-lm-head --dtype bfloat16 --tokenizer-worker-num 4 \ - --load-balance-method round_robin - NODE_RANK=$i - break - fi -done - -``` - -```shell -export SGLANG_DP_ROUND_ROBIN=1 -python -m sglang_router.launch_router \ - --pd-disaggregation \ - --policy cache_aware \ - --prefill http://PIP:8000 8995 \ - --decode http://DIP:8001 \ - --host 127.0.0.1 \ - --port 6688 \ - --mini-lb -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell -python -m sglang.bench_serving --dataset-name random --backend sglang-oai --host 127.0.0.1 --port 7239 --max-concurrency 860 --random-input-len 3500 --random-output-len 1500 --num-prompts 3440 --random-range-ratio 1 -``` - -### Qwen3-235B-A22B 3_5K-1_5K 50ms on A3 8 Cards Mixed Mode - -Model: Qwen3-235B-A22B-W8A8 - -Hardware: Atlas 800I A3 8Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 3.5K+1.5K - -TPOT: 50ms - -#### Model Deployment - -```shell -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -export SGLANG_SET_CPU_AFFINITY=1 -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True - -MODEL_PATH=xxx - -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -export HCCL_BUFFSIZE=570 -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export HCCL_OP_EXPANSION_MODE="AIV" -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_ENABLE_SPEC_V2=1 -export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 -export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=100 - -export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=188416 -export SGLANG_NPU_FUSED_MOE_MODE=2 - -python -m sglang.launch_server --model-path $MODEL_PATH \ - --host 127.0.0.1 --port 7439 --trust-remote-code --nnodes 1 --node-rank 0 \ - --attention-backend ascend --device npu --quantization modelslim \ - --max-running-requests 432 --context-length 8192 --dtype bfloat16 \ - --chunked-prefill-size 94208 --max-prefill-tokens 458880 --sampling-backend ascend \ - --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ - --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ - --disable-radix-cache --moe-a2a-backend ascend_fuseep --speculative-draft-model-quantization unquant \ - --tp 16 --dp-size 16 --enable-dp-attention --enable-dp-lm-head --mem-fraction-static 0.8 --cuda-graph-bs 1 2 4 8 16 20 24 26 27 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell -python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7439 --max-concurrency 272 --random-input-len 3500 --random-output-len 1500 --num-prompts 1088 --random-range-ratio 1 -``` - -### Qwen3-235B-A22B 2K-2K 100ms on A3 8 Cards Mixed Mode - -Model: Qwen3-235B-A22B-W8A8 - -Hardware: Atlas 800I A3 8Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 2K+2K - -TPOT: 100ms - -#### Model Deployment - -```shell -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -export SGLANG_SET_CPU_AFFINITY=1 -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 - -MODEL_PATH=xxx - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -export HCCL_BUFFSIZE=1200 -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export HCCL_OP_EXPANSION_MODE="AIV" -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_ENABLE_SPEC_V2=1 -export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=144 - -python -m sglang.launch_server --model-path $MODEL_PATH \ - --host 127.0.0.1 --port 7439 --trust-remote-code --nnodes 1 --node-rank 0 \ - --attention-backend ascend --device npu --quantization modelslim \ - --max-running-requests 576 --context-length 8192 --dtype bfloat16 \ - --chunked-prefill-size 32768 --max-prefill-tokens 458880 \ - --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ - --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ - --disable-radix-cache --moe-a2a-backend deepep --deepep-mode auto --speculative-draft-model-quantization unquant \ - --tp 16 --dp-size 16 --enable-dp-attention --enable-dp-lm-head --mem-fraction-static 0.84 --cuda-graph-bs 8 16 20 24 32 36 - -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell -python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7439 --max-concurrency 576 --random-input-len 2000 --random-output-len 2000 --num-prompts 576 --random-range-ratio 1 -``` - -### Qwen3-235B-A22B 2K-2K 50ms on A3 8 Cards Mixed Mode - -Model: Qwen3-235B-A22B-W8A8 - -Hardware: Atlas 800I A3 8Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 2K+2K - -TPOT: 50ms - -#### Model Deployment - -```shell -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -export SGLANG_SET_CPU_AFFINITY=1 -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 - -MODEL_PATH=xxx - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -export HCCL_BUFFSIZE=450 -export HCCL_SOCKET_IFNAME=xxx -export GLOO_SOCKET_IFNAME=xxx -export HCCL_OP_EXPANSION_MODE="AIV" -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_ENABLE_SPEC_V2=1 -export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 -export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=100 -export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=147456 -export SGLANG_NPU_FUSED_MOE_MODE=2 - -python -m sglang.launch_server --model-path $MODEL_PATH \ - --host 127.0.0.1 --port 7439 --trust-remote-code --nnodes 1 --node-rank 0 \ - --attention-backend ascend --device npu --quantization modelslim \ - --max-running-requests 624 --context-length 8192 --dtype bfloat16 \ - --chunked-prefill-size 73728 --max-prefill-tokens 458880 --speculative-draft-model-quantization unquant \ - --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ - --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ - --disable-radix-cache --moe-a2a-backend ascend_fuseep \ - --tp 16 --dp-size 16 --enable-dp-attention --enable-dp-lm-head --mem-fraction-static 0.83 --cuda-graph-bs 4 8 16 24 28 29 30 32 34 36 37 38 39 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell -python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7439 --max-concurrency 480 --random-input-len 2048 --random-output-len 2048 --num-prompts 480 --random-range-ratio 1 -``` - -### Qwen3-235B-A22B 2K-2K 50ms on A3 16 Cards Mixed Mode - -Model: Qwen3-235B-A22B-W8A8 - -Hardware: Atlas 800I A3 16Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 2K+2K - -TPOT: 50ms - -#### Model Deployment - -```shell -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash - -export SGLANG_SET_CPU_AFFINITY=1 -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 - -MODEL_PATH=xxx - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -export HCCL_BUFFSIZE=1600 -export HCCL_SOCKET_IFNAME=xxx -export GLOO_SOCKET_IFNAME=xxx -export HCCL_OP_EXPANSION_MODE="AIV" - -MIX_IP=('IP1' 'IP2') - -for i in "${!MIX_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${MIX_IP[$i]}" || "$LOCAL_HOST2" == "${MIX_IP[$i]}" ]]; - then - echo "${MIX_IP[$i]}" - export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - export SGLANG_ENABLE_SPEC_V2=1 - - python -m sglang.launch_server --model-path ${MODEL_PATH} \ - --host 127.0.0.1 --port 7439 --trust-remote-code \ - --nnodes 2 --node-rank $i --tp-size 32 --dp-size 32 --mem-fraction-static 0.8 --max-running-requests 768 \ - --attention-backend ascend --device npu --quantization modelslim --enable-dp-attention \ - --moe-a2a-backend deepep --deepep-mode auto --cuda-graph-bs 6 8 10 12 18 24 \ - --dist-init-addr ${MIX_IP[0]}:5000 --chunked-prefill-size 131072 --max-prefill-tokens 458880 \ - --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx --speculative-draft-model-quantization= unquant \ - --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ - --context-length 8192 --disable-radix-cache \ - --enable-dp-lm-head --dtype bfloat16 - NODE_RANK=$i - break - fi -done - -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell -python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7439 --max-concurrency 768 --random-input-len 2000 --random-output-len 2000 --num-prompts 768 --random-range-ratio 1 -``` - -### Qwen3-235B-A22B 11K-1K 10ms on A3 8 Cards Mixed Mode - -Model: Qwen3-235B-A22B-W8A8 - -Hardware: Atlas 800I A3 8Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 11K+1K - -TPOT: 10ms - -#### Model Deployment - -```shell -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -export SGLANG_SET_CPU_AFFINITY=1 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True - -MODEL_PATH=xxx - -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -export HCCL_BUFFSIZE=1600 -export HCCL_SOCKET_IFNAME=xxx -export GLOO_SOCKET_IFNAME=xxx -export HCCL_OP_EXPANSION_MODE="AIV" -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_ENABLE_SPEC_V2=1 - -python -m sglang.launch_server --model-path $MODEL_PATH \ - --host 127.0.0.1 --port 7439 --trust-remote-code --nnodes 1 --node-rank 0 \ - --attention-backend ascend --device npu --quantization modelslim \ - --max-running-requests 1 --dtype bfloat16 \ - --chunked-prefill-size -1 --max-prefill-tokens 16384 --speculative-draft-model-quantization unquant \ - --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ - --speculative-num-steps 4 --speculative-eagle-topk 1 --speculative-num-draft-tokens 5 \ - --disable-radix-cache --enable-dp-lm-head \ - --tp 16 --mem-fraction-static 0.78 --cuda-graph-bs 1 - -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell -python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7439 --max-concurrency 1 --random-input-len 11000 --random-output-len 1000 --num-prompts 1 --random-range-ratio 1 -``` - -### Qwen3-32B 6K-1_5K 18ms on A3 4 Cards Mixed Mode - -Model: Qwen3-32B - -Hardware: Atlas 800I A3 4Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 6K+1.5K - -TPOT: 18ms - -#### Model Deployment - -```shell -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -export SGLANG_SET_CPU_AFFINITY=1 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -MODEL_PATH=xxx - -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -export HCCL_BUFFSIZE=400 -export HCCL_SOCKET_IFNAME=xxx -export GLOO_SOCKET_IFNAME=xxx -export HCCL_OP_EXPANSION_MODE="AIV" -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_ENABLE_SPEC_V2=1 - -python -m sglang.launch_server --model-path $MODEL_PATH \ - --host 127.0.0.1 --port 7439 --trust-remote-code --nnodes 1 --node-rank 0 \ - --attention-backend ascend --device npu \ - --max-running-requests 32 \ - --disable-radix-cache \ - --chunked-prefill-size 24576 --max-prefill-tokens 65536 \ - --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ - --speculative-num-steps 4 --speculative-eagle-topk 1 --speculative-num-draft-tokens 5 \ - --tp-size 8 --mem-fraction-static 0.72 --cuda-graph-bs 8 16 24 32 --dtype bfloat16 - -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell -python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7439 --max-concurrency 32 --random-output-len 1500 --random-input-len 6000 --num-prompts 32 --random-range-ratio 1 -``` - -### Qwen3-32B 4K-1_5K 11ms on A3 4 Cards Mixed Mode - -Model: Qwen3-32B - -Hardware: Atlas 800I A3 4Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 4K+1.5K - -TPOT: 11ms - -#### Model Deployment - -```shell -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -export SGLANG_SET_CPU_AFFINITY=1 -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -MODEL_PATH=xxx - -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -export HCCL_BUFFSIZE=400 -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export HCCL_OP_EXPANSION_MODE="AIV" -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_ENABLE_SPEC_V2=1 - -python -m sglang.launch_server --model-path $MODEL_PATH \ - --host 127.0.0.1 --port 7339 --trust-remote-code --nnodes 1 --node-rank 0 \ - --attention-backend ascend --device npu \ - --max-running-requests 1 \ - --disable-radix-cache \ - --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ - --speculative-num-steps 4 --speculative-eagle-topk 1 --speculative-num-draft-tokens 5 \ - --chunked-prefill-size 24576 --max-prefill-tokens 65536 \ - --tp-size 8 --mem-fraction-static 0.72 --cuda-graph-bs 1 --dtype bfloat16 - -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell -python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7239 --random-range-ratio 1 --max-concurrency 1 --random-output-len 1500 --random-input-len 4096 --num-prompts 4 -``` - -### Qwen3-32B 18K-4K 6ms on A3 8 Cards Mixed Mode - -Model: Qwen3-32B - -Hardware: Atlas 800I A3 8Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 18K+4K - -TPOT: 6ms - -#### Model Deployment - -```shell -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -export SGLANG_SET_CPU_AFFINITY=1 -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -MODEL_PATH=xxx - -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -export HCCL_BUFFSIZE=400 -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export HCCL_OP_EXPANSION_MODE="AIV" -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_ENABLE_SPEC_V2=1 - -python -m sglang.launch_server --model-path $MODEL_PATH \ - --host 127.0.0.1 --port 7339 --trust-remote-code --nnodes 1 --node-rank 0 \ - --attention-backend ascend --device npu \ - --max-running-requests 1 \ - --disable-radix-cache --speculative-draft-model-quantization unquant \ - --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ - --speculative-num-steps 4 --speculative-eagle-topk 1 --speculative-num-draft-tokens 5 \ - --chunked-prefill-size -1 --max-prefill-tokens 65536 \ - --tp-size 16 --mem-fraction-static 0.72 --cuda-graph-bs 1 --dtype bfloat16 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell -python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7339 --random-range-ratio 1 --max-concurrency 1 --random-output-len 18000 --random-input-len 4000 --num-prompts 1 -``` - -### Qwen3-32B 3_5K-1_5K 50ms on A3 2 Cards Mixed Mode - -Model: Qwen3-32B - -Hardware: Atlas 800I A3 2Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 3.5K+1.5K - -TPOT: 50ms - -#### Model Deployment - -```shell -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -export SGLANG_SET_CPU_AFFINITY=1 -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - - -MODEL_PATH=xxx - -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -export HCCL_BUFFSIZE=400 -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export HCCL_OP_EXPANSION_MODE="AIV" -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_ENABLE_SPEC_V2=1 - -python -m sglang.launch_server --model-path $MODEL_PATH \ - --host 127.0.0.1 --port 7239 --trust-remote-code --nnodes 1 --node-rank 0 \ - --attention-backend ascend --device npu --quantization modelslim \ - --max-running-requests 78 \ - --disable-radix-cache --speculative-draft-model-quantization unquant \ - --chunked-prefill-size -1 --max-prefill-tokens 49152 \ - --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ - --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ - --tp-size 4 --mem-fraction-static 0.72 --cuda-graph-bs 16 32 64 68 72 78 --dtype bfloat16 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell -python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7239 --max-concurrency 78 --random-output-len 1500 --random-input-len 3500 --num-prompts 312 --random-range-ratio 1 -``` - -### Qwen3-32B 2K-2K 50ms on A3 2 Cards Mixed Mode - -Model: Qwen3-32B - -Hardware: Atlas 800I A3 2Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 2K+2K - -TPOT: 50ms - -#### Model Deployment - -```shell -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -export SGLANG_SET_CPU_AFFINITY=1 -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -MODEL_PATH=xxx - -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -export HCCL_BUFFSIZE=400 -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export HCCL_OP_EXPANSION_MODE="AIV" -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_ENABLE_SPEC_V2=1 - -python -m sglang.launch_server --model-path $MODEL_PATH \ - --host 127.0.0.1 --port 7239 --trust-remote-code --nnodes 1 --node-rank 0 \ - --attention-backend ascend --device npu --quantization modelslim \ - --max-running-requests 120 \ - --disable-radix-cache --speculative-draft-model-quantization unquant \ - --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ - --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ - --chunked-prefill-size -1 --max-prefill-tokens 49152 \ - --tp-size 4 --mem-fraction-static 0.7 --cuda-graph-bs 54 60 66 72 78 84 90 108 114 120 --dtype bfloat16 - -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell -python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7239 --max-concurrency 120 --random-output-len 2000 --random-input-len 2000 --num-prompts 480 --random-range-ratio 1 -``` - -### Qwen3-30B-A3B 3_5K-1_5K 50ms on A3 1 Card Mixed Mode - -Model: Qwen3-30B-A3B-Instruct-2507 - -Hardware: Atlas 800I A3 1Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 3.5K+1.5K - -TPOT: 50ms - -#### Model Deployment - -```shell -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -MODEL_PATH=xxx - -export SGLANG_SET_CPU_AFFINITY=1 -export ASCEND_LAUNCH_BLOCKING=0 -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -export HCCL_BUFFSIZE=400 -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export HCCL_OP_EXPANSION_MODE="AIV" -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 -export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200 -export SGLANG_ENABLE_SPEC_V2=1 - -python -m sglang.launch_server --model-path $MODEL_PATH \ - --host 127.0.0.1 --port 7239 --trust-remote-code --nnodes 1 --node-rank 0 \ - --attention-backend ascend --device npu --quantization modelslim \ - --max-running-requests 162 \ - --disable-radix-cache \ - --speculative-draft-model-quantization unquant \ - --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ - --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ - --chunked-prefill-size -1 --max-prefill-tokens 35000 \ - --tp-size 2 --mem-fraction-static 0.87 --cuda-graph-bs 1 5 15 40 70 100 120 130 140 146 150 154 156 158 160 162 \ - --dtype bfloat16 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell -python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7239 --max-concurrency 156 --random-input-len 3500 --random-output-len 1500 --num-prompts 624 --random-range-ratio 1 -``` - -### Qwen3-Coder-480B-A35B-Instruct 3_5K-1_5K 50ms on A3 24 Cards Disaggregation Mode - -Model: Qwen3-Coder-480B-A35B-Instruct - -Hardware: Atlas 800I A3 24Card - -DeployMode: PD Disaggregation - -Dataset: random - -Input Output Length: 3.5K+1.5K - -TPOT: 50ms - -#### Model Deployment - -```shell -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING - -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash - -export SGLANG_SET_CPU_AFFINITY=1 -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export STREAMS_PER_DEVICE=32 -export SGLANG_NPU_FUSED_MOE_MODE=2 - -MODEL_PATH=xxx -export ASCEND_MF_STORE_URL="tcp://PIP:24667" -P_IP=('PIP') -D_IP=('DIP1' 'DIP2') -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - - -for i in "${!P_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; - then - echo "${P_IP[$i]}" - source /usr/local/Ascend/ascend-toolkit/set_env.sh - source /usr/local/Ascend/nnal/atb/set_env.sh - export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=327680 - export HCCL_BUFFSIZE=1550 - export TASK_QUEUE_ENABLE=2 - export HCCL_SOCKET_IFNAME=lo - export GLOO_SOCKET_IFNAME=lo - - python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode prefill \ - --host ${P_IP[$i]} --port 8000 --disaggregation-bootstrap-port 8995 --trust-remote-code \ - --nnodes 1 --node-rank $i --tp-size 16 --dp-size 2 --mem-fraction-static 0.7 \ - --disable-radix-cache \ - --attention-backend ascend --device npu --quantization modelslim --disaggregation-transfer-backend ascend \ - --max-running-requests 16 --chunked-prefill-size 20480 --max-prefill-tokens 20480 \ - --enable-dp-attention \ - --moe-a2a-backend ascend_fuseep --dtype bfloat16 \ - --disable-overlap-schedule - NODE_RANK=$i - break - fi -done - -for i in "${!D_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; - then - echo "${D_IP[$i]}" - source /usr/local/Ascend/ascend-toolkit/set_env.sh - source /usr/local/Ascend/nnal/atb/set_env.sh - export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=65536 - export HCCL_BUFFSIZE=600 - export SGLANG_NPU_FUSED_MOE_MODE=2 - export HCCL_SOCKET_IFNAME=xxx - export GLOO_SOCKET_IFNAME=xxx - - python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode decode \ - --host ${D_IP[$i]} --port 8001 --trust-remote-code \ - --nnodes 2 --node-rank $i --tp-size 32 --dp-size 4 --mem-fraction-static 0.75 --max-running-requests 544 \ - --attention-backend ascend --device npu --quantization modelslim --enable-dp-attention \ - --moe-a2a-backend ascend_fuseep --cuda-graph-bs 16 32 56 72 80 88 96 104 112 120 128 136 \ - --dist-init-addr DIP1:5000 \ - --disaggregation-transfer-backend ascend --watchdog-timeout 9000 --context-length 8192 \ - --enable-dp-lm-head --dtype bfloat16 --tokenizer-worker-num 4 --load-balance-method round_robin - NODE_RANK=$i - break - fi -done - -``` - -```shell -python -m sglang_router.launch_router \ - --pd-disaggregation \ - --policy cache_aware \ - --prefill http://PIP:8000 8995 \ - --decode http://DIP:8001 \ - --host 127.0.0.1 \ - --port 6688 \ - --mini-lb -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell -python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7239 --max-concurrency 410 --random-input-len 3500 --random-output-len 1500 --num-prompts 1640 --random-range-ratio 1 --request-rate 8 -``` - -### Qwen3-Coder-480B-A35B-Instruct 3_5K-1_5K 50ms on A3 16 Cards Mixed Mode - -Model: Qwen3-Coder-480B-A35B-Instruct - -Hardware: Atlas 800I A3 16Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 3.5K+1.5K - -TPOT: 50ms - -#### Model Deployment - -```shell -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash - -export SGLANG_SET_CPU_AFFINITY=1 -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=72 -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 - -MODEL_PATH=xxx - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -export HCCL_BUFFSIZE=1800 -export HCCL_SOCKET_IFNAME=xxx -export GLOO_SOCKET_IFNAME=xxx -export HCCL_OP_EXPANSION_MODE="AIV" - -MIX_IP=('IP1' 'IP2') - -for i in "${!MIX_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${MIX_IP[$i]}" || "$LOCAL_HOST2" == "${MIX_IP[$i]}" ]]; - then - echo "${MIX_IP[$i]}" - - python -m sglang.launch_server --model-path $MODEL_PATH \ - --host 127.0.0.1 --port 7439 --trust-remote-code --nnodes 2 --node-rank $i \ - --dist-init-addr 141.61.133.128:5000 \ - --attention-backend ascend --device npu --quantization modelslim \ - --max-running-requests 288 --context-length 8192 --dtype bfloat16 \ - --chunked-prefill-size 114688 --max-prefill-tokens 458880 \ - --disable-radix-cache --moe-a2a-backend deepep --deepep-mode auto \ - --tp 32 --dp-size 4 --enable-dp-attention --enable-dp-lm-head --mem-fraction-static 0.7 --cuda-graph-bs 56 64 72 - NODE_RANK=$i - break - fi -done -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell -python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7439 --max-concurrency 288 --random-input-len 3500 --random-output-len 1500 --num-prompts 1152 --random-range-ratio 1 --request-rate 20 -``` - -### Qwen3-Coder-480B-A35B-Instruct 3_5K-1_5K 50ms on A3 8 Cards Mixed Mode - -Model: Qwen3-Coder-480B-A35B-Instruct - -Hardware: Atlas 800I A3 8Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 3.5K+1.5K - -TPOT: 50ms - -#### Model Deployment - -```shell -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -export SGLANG_SET_CPU_AFFINITY=1 -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 - -MODEL_PATH=xxx - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -export HCCL_BUFFSIZE=2100 -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export HCCL_OP_EXPANSION_MODE="AIV" - -python -m sglang.launch_server --model-path $MODEL_PATH \ ---host 127.0.0.1 --port 7439 --trust-remote-code --nnodes 1 --node-rank 0 \ ---attention-backend ascend --device npu --quantization modelslim \ ---max-running-requests 80 --context-length 8192 --dtype bfloat16 \ ---chunked-prefill-size 28672 --max-prefill-tokens 458880 \ ---disable-radix-cache --moe-a2a-backend deepep --deepep-mode auto --enable-dp-attention --enable-dp-lm-head \ ---tp 16 --dp-size 4 --mem-fraction-static 0.7 --cuda-graph-bs 16 20 24 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell -python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7439 --max-concurrency 80 --random-input-len 3500 --random-output-len 1500 --num-prompts 320 --random-range-ratio 1 -``` - -### Qwen3-Next-80B-A3B-Instruct 3_5K-1_5K 50ms on A3 2 Cards Mixed Mode - -Model: Qwen3-Next-80B-A3B-Instruct - -Hardware: Atlas 800I A3 2Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 3.5K+1.5K - -TPOT: 50ms - -#### Model Deployment - -```shell -export cann_path=/usr/local/Ascend/ascend-toolkit/latest -source /usr/local/Ascend/driver/bin/setenv.bash -source ${cann_path}/../set_env.sh -source ${cann_path}/../../nnal/atb/set_env.sh -source ${cann_path}/opp/vendors/customize/bin/set_env.bash -export ASCEND_HOME_PATH=${cann_path} -source /usr/local/Ascend/8.5.0/bisheng_toolkit/set_env.sh - -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -export SGLANG_SET_CPU_AFFINITY=1 - -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export STREAMS_PER_DEVICE=32 -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo - -export HCCL_OP_EXPANSION_MODE=AIV -export HCCL_ALGO="level0:NA;level1:ring" - -export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=20 -export HCCL_BUFFSIZE=2000 - -python -m sglang.launch_server \ - --model-path /path/to/Qwen3-Next-80B-A3B-Instruct-W8A8-3 \ - --host 127.0.0.1 \ - --port 6699 \ - --tp-size 4 \ - --device npu \ - --attention-backend ascend \ - --mem-fraction-static 0.685 \ - --max-running-requests 80 \ - --watchdog-timeout 3600 \ - --disable-radix-cache \ - --cuda-graph-bs 80 \ - --max-prefill-tokens 28672 --max-total-tokens 450560 \ - --moe-a2a-backend deepep --deepep-mode auto \ - --quantization modelslim \ - --chunked-prefill-size -1 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell -python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6699 --max-concurrency 80 --random-output-len 1536 --random-input-len 3584 --num-prompts 160 --random-range-ratio 1 -``` - -### Qwen3-32B 6K-1_5K 18ms on A2 8 Cards Mixed Mode - -Model: Qwen3-32B - -Hardware: Atlas 800I A2 8Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 6K+1.5K - -TPOT: 18ms - -#### Model Deployment - -```shell -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -export SGLANG_SET_CPU_AFFINITY=1 -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -MODEL_PATH=xxx - -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -export HCCL_BUFFSIZE=400 -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export HCCL_OP_EXPANSION_MODE="AIV" -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_ENABLE_SPEC_V2=1 - -python -m sglang.launch_server --model-path $MODEL_PATH \ - --host 127.0.0.1 --port 7439 --trust-remote-code --nnodes 1 --node-rank 0 \ - --attention-backend ascend --device npu --quantization modelslim \ - --max-running-requests 32 \ - --disable-radix-cache \ - --chunked-prefill-size 24576 --max-prefill-tokens 65536 \ - --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ - --speculative-num-steps 4 --speculative-eagle-topk 1 --speculative-num-draft-tokens 5 \ - --tp-size 8 --mem-fraction-static 0.72 --cuda-graph-bs 8 16 24 32 --dtype bfloat16 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell -python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7439 --max-concurrency 32 --random-output-len 1500 --random-input-len 6000 --num-prompts 32 --random-range-ratio 1 -``` - -### Qwen3-32B 4K-1_5K 11ms on A2 8 Cards Mixed Mode - -Model: Qwen3-32B - -Hardware: Atlas 800I A2 8Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 4K+1.5K - -TPOT: 11ms - -#### Model Deployment - -```shell -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -export SGLANG_SET_CPU_AFFINITY=1 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True - -MODEL_PATH=xxx - -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -export HCCL_BUFFSIZE=400 -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export HCCL_OP_EXPANSION_MODE="AIV" -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_ENABLE_SPEC_V2=1 - -python -m sglang.launch_server --model-path $MODEL_PATH \ - --host 127.0.0.1 --port 7339 --trust-remote-code --nnodes 1 --node-rank 0 \ - --attention-backend ascend --device npu \ - --max-running-requests 32 \ - --disable-radix-cache \ - --speculative-draft-model-quantization unquant \ - --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ - --speculative-num-steps 4 --speculative-eagle-topk 1 --speculative-num-draft-tokens 5 \ - --chunked-prefill-size -1 --max-prefill-tokens 65536 \ - --tp-size 8 --mem-fraction-static 0.72 --cuda-graph-bs 1 4 6 12 18 24 30 32 --dtype bfloat16 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell -python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7339 --random-range-ratio 1 --max-concurrency 1 --random-output-len 1500 --random-input-len 4096 --num-prompts 4 -``` - -### Qwen3-32B 1K-0_3K 12ms on A3 2 Cards Mixed Mode - -Model: Qwen3-32B - -Hardware: Atlas 800I A3 2Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 1K+0.3K - -TPOT: 12ms - -#### Model Deployment - -```shell -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -export SGLANG_SET_CPU_AFFINITY=1 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True - -MODEL_PATH=xxx - -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export HCCL_OP_EXPANSION_MODE="AIV" -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_ENABLE_SPEC_V2=1 - -python -m sglang.launch_server --model-path $MODEL_PATH \ - --host 127.0.0.1 --port 7339 --trust-remote-code --nnodes 1 --node-rank 0 \ - --attention-backend ascend --device npu --quantization modelslim \ - --max-running-requests 16 \ - --disable-radix-cache \ - --speculative-draft-model-quantization unquant \ - --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ - --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ - --chunked-prefill-size -1 --max-prefill-tokens 16384 \ - --tp-size 4 --mem-fraction-static 0.843 --cuda-graph-bs 1 4 8 16 --dtype bfloat16 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell -python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7339 --random-range-ratio 1 --max-concurrency 16 --random-output-len 300 --random-input-len 1024 --num-prompts 16 -``` - -### Qwen3-32B 6K-1_5K 17ms on A3 2 Cards Mixed Mode - -Model: Qwen3-32B - -Hardware: Atlas 800I A3 2Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 6K+1.5K - -TPOT: 17ms - -#### Model Deployment - -```shell -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -export SGLANG_SET_CPU_AFFINITY=1 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True - -MODEL_PATH=xxx - -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export HCCL_OP_EXPANSION_MODE="AIV" -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_ENABLE_SPEC_V2=1 - -python -m sglang.launch_server --model-path $MODEL_PATH \ - --host 127.0.0.1 --port 7339 --trust-remote-code --nnodes 1 --node-rank 0 \ - --attention-backend ascend --device npu --quantization modelslim \ - --max-running-requests 16 \ - --disable-radix-cache \ - --speculative-draft-model-quantization unquant \ - --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ - --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ - --chunked-prefill-size -1 --max-prefill-tokens 16384 \ - --tp-size 4 --mem-fraction-static 0.843 --cuda-graph-bs 1 4 10 15 16 --dtype bfloat16 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell -python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7339 --random-range-ratio 1 --max-concurrency 16 --random-output-len 1500 --random-input-len 6144 --num-prompts 16 -``` - -### Qwen3-8B 1K-0_3K 7ms on A3 1 Cards Mixed Mode - -Model: Qwen3-8B - -Hardware: Atlas 800I A3 1Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 1K+0.3K - -TPOT: 7ms - -#### Model Deployment - -```shell -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -export SGLANG_SET_CPU_AFFINITY=1 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True - -MODEL_PATH=xxx - -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export HCCL_OP_EXPANSION_MODE="AIV" -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_ENABLE_SPEC_V2=1 - -python -m sglang.launch_server --model-path $MODEL_PATH \ - --host 127.0.0.1 --port 7339 --trust-remote-code --nnodes 1 --node-rank 0 \ - --attention-backend ascend --device npu --quantization modelslim \ - --max-running-requests 16 \ - --disable-radix-cache \ - --speculative-draft-model-quantization unquant \ - --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ - --speculative-num-steps 4 --speculative-eagle-topk 1 --speculative-num-draft-tokens 5 \ - --chunked-prefill-size -1 --max-prefill-tokens 16384 \ - --tp-size 2 --mem-fraction-static 0.894 --cuda-graph-bs 1 2 4 6 9 10 15 16 --dtype bfloat16 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell -python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7339 --random-range-ratio 1 --max-concurrency 16 --random-output-len 300 --random-input-len 1024 --num-prompts 16 -``` - -### Qwen3-8B 6K-1_5K 12ms on A3 1 Cards Mixed Mode - -Model: Qwen3-8B - -Hardware: Atlas 800I A3 1Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 6K+1.5K - -TPOT: 12ms - -#### Model Deployment - -```shell -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -export SGLANG_SET_CPU_AFFINITY=1 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True - -MODEL_PATH=xxx - -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export HCCL_OP_EXPANSION_MODE="AIV" -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_ENABLE_SPEC_V2=1 - -python -m sglang.launch_server --model-path $MODEL_PATH \ - --host 127.0.0.1 --port 7339 --trust-remote-code --nnodes 1 --node-rank 0 \ - --attention-backend ascend --device npu --quantization modelslim \ - --max-running-requests 16 \ - --disable-radix-cache \ - --speculative-draft-model-quantization unquant \ - --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ - --speculative-num-steps 4 --speculative-eagle-topk 1 --speculative-num-draft-tokens 5 \ - --chunked-prefill-size -1 --max-prefill-tokens 16384 \ - --tp-size 2 --mem-fraction-static 0.894 --cuda-graph-bs 1 5 15 16 --dtype bfloat16 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell -python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7339 --random-range-ratio 1 --max-concurrency 16 --random-output-len 1500 --random-input-len 6144 --num-prompts 16 -``` - -### Qwen3-32B 3_5K-1_5K 50ms on A2 8 Cards Mixed Mode - -Model: Qwen3-32B - -Hardware: Atlas 800I A2 8Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 3.5K+1.5K - -TPOT: 50ms - -#### Model Deployment - -```shell -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -export SGLANG_SET_CPU_AFFINITY=1 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -MODEL_PATH=xxx - -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -export HCCL_BUFFSIZE=400 -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export HCCL_OP_EXPANSION_MODE="AIV" -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_ENABLE_SPEC_V2=1 - -python -m sglang.launch_server --model-path $MODEL_PATH \ - --host 127.0.0.1 --port 7239 --trust-remote-code --nnodes 1 --node-rank 0 \ - --attention-backend ascend --device npu --quantization modelslim \ - --max-running-requests 78 \ - --disable-radix-cache --speculative-draft-model-quantization unquant \ - --chunked-prefill-size -1 --max-prefill-tokens 65536 \ - --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ - --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ - --tp-size 4 --mem-fraction-static 0.72 --cuda-graph-bs 1 4 8 16 32 64 68 72 78 --dtype bfloat16 --base-gpu-id 4 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell -python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7239 --max-concurrency 78 --random-output-len 1500 --random-input-len 3500 --num-prompts 312 --random-range-ratio 1 -``` - -### Qwen3-32B 2K-2K 50ms on A2 8 Cards Mixed Mode - -Model: Qwen3-32B - -Hardware: Atlas 800I A2 8Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 2K+2K - -TPOT: 50ms - -#### Model Deployment - -```shell -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -export SGLANG_SET_CPU_AFFINITY=1 -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -MODEL_PATH=xxx - -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -export HCCL_BUFFSIZE=400 -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export HCCL_OP_EXPANSION_MODE="AIV" -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_ENABLE_SPEC_V2=1 - -python -m sglang.launch_server --model-path $MODEL_PATH \ - --host 127.0.0.1 --port 7239 --trust-remote-code --nnodes 1 --node-rank 0 \ - --attention-backend ascend --device npu --quantization modelslim \ - --max-running-requests 120 \ - --disable-radix-cache \ - --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ - --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 --speculative-draft-model-quantization unquant \ - --chunked-prefill-size -1 --max-prefill-tokens 49152 --base-gpu-id 4 \ - --tp-size 4 --mem-fraction-static 0.7 --cuda-graph-bs 54 60 66 72 78 84 90 108 114 120 --dtype bfloat16 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell -python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7239 --max-concurrency 120 --random-output-len 2000 --random-input-len 2000 --num-prompts 120 --random-range-ratio 1 -``` - -### Qwen3-30B-A3B 6K-1_5K 10ms on A3 1 Cards Mixed Mode - -Model: Qwen3-30B-A3B - -Hardware: Atlas 800I A3 1Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 6K+1.5K - -TPOT: 10ms - -#### Model Deployment - -```shell -export SGLANG_SET_CPU_AFFINITY=1 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING - -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True - -MODEL_PATH=xxx - -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -export HCCL_BUFFSIZE=400 -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export HCCL_OP_EXPANSION_MODE="AIV" -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_ENABLE_SPEC_V2=1 - -python -m sglang.launch_server --model-path $MODEL_PATH \ - --host 127.0.0.1 --port 7339 --trust-remote-code --nnodes 1 --node-rank 0 \ - --attention-backend ascend --device npu --quantization modelslim \ - --max-running-requests 16 \ - --disable-radix-cache \ - --speculative-draft-model-quantization unquant \ - --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ - --speculative-num-steps 4 --speculative-eagle-topk 1 --speculative-num-draft-tokens 5 \ - --chunked-prefill-size -1 --max-prefill-tokens 35000 \ - --tp-size 2 --mem-fraction-static 0.6 --cuda-graph-bs 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 --dtype bfloat16 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell -python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7339 --random-range-ratio 1 --max-concurrency 16 --random-output-len 1500 --random-input-len 6144 --num-prompts 16 -``` - -### Qwen3-30B-A3B 1K-0_3K 7ms on A3 1 Cards Mixed Mode - -Model: Qwen3-30B-A3B - -Hardware: Atlas 800I A3 1Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 1K+0.3K - -TPOT: 7ms - -#### Model Deployment - -```shell -export SGLANG_SET_CPU_AFFINITY=1 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING - -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True - -MODEL_PATH=xxx - -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -export HCCL_BUFFSIZE=400 -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export HCCL_OP_EXPANSION_MODE="AIV" -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_ENABLE_SPEC_V2=1 - -python -m sglang.launch_server --model-path $MODEL_PATH \ - --host 127.0.0.1 --port 7339 --trust-remote-code --nnodes 1 --node-rank 0 \ - --attention-backend ascend --device npu --quantization modelslim \ - --max-running-requests 8 \ - --disable-radix-cache \ - --speculative-draft-model-quantization unquant \ - --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ - --speculative-num-steps 4 --speculative-eagle-topk 1 --speculative-num-draft-tokens 5 \ - --chunked-prefill-size -1 --max-prefill-tokens 35000 \ - --tp-size 2 --mem-fraction-static 0.7 --cuda-graph-bs 1 2 3 4 5 6 7 8 --dtype bfloat16 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell -python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7339 --random-range-ratio 1 --max-concurrency 8 --random-output-len 300 --random-input-len 1024 --num-prompts 8 -``` - -### Qwen3-Next 1K-0_3K 14_21ms on A3 2 Cards Mixed Mode - -Model: Qwen3-Next-80B-A3B-Instruct - -Hardware: Atlas 800I A3 2Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 1K+0.3K - -TPOT: 14.21ms - -#### Model Deployment - -```shell -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -export SGLANG_SET_CPU_AFFINITY=1 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export STREAMS_PER_DEVICE=32 -export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=330 -export DEEPEP_NORMAL_LONG_SEQ_ROUND=5 -export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=3000 -export DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ=1 - -export ASCEND_USE_FIA=1 -export SGLANG_NPU_USE_MULTI_STREAM=1 - -export SGLANG_WARMUP_TIMEOUT=3600 -export SGLANG_ENABLE_SPEC_V2=1 -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export FORCE_DRAFT_MODEL_NON_QUANT=1 - -MODEL_PATH=xxx - -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -export HCCL_BUFFSIZE=2000 -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export HCCL_OP_EXPANSION_MODE="AIV" -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_ENABLE_SPEC_V2=1 - -python3 -m sglang.launch_server --model-path ${MODEL_PATH} \ - --page-size 128 \ - --tp-size 4 \ - --trust-remote-code \ - --attention-backend ascend \ - --device npu \ - --watchdog-timeout 9000 \ - --host 127.0.0.1 --port 6699 \ - --mem-fraction-static 0.75 \ - --disable-radix-cache --max-prefill-tokens 14080 --context-length 26384 \ - --speculative-algorithm NEXTN --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 --speculative-draft-model-quantization unquant \ - --chunked-prefill-size -1 --max-running-requests 312 \ - --cuda-graph-bs 2 4 16 32 48 64 80 96 128 140 156 \ - --mamba-ssm-dtype bfloat16 \ - --base-gpu-id 0 \ - --speculative-draft-model-path /home/weights/Qwen3-Next-80B-A3B-Instruct \ - --moe-a2a-backend deepep --deepep-mode auto \ -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell -python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6699 --random-range-ratio 1 --max-concurrency 16 --random-output-len 300 --random-input-len 1024 --num-prompts 16 -``` - -### Qwen3-Next 6K-1_5K 15_62ms on A3 2 Cards Mixed Mode - -Model: Qwen3-Next-80B-A3B-Instruct - -Hardware: Atlas 800I A3 2Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 6K+1.5K - -TPOT: 15.62ms - -#### Model Deployment - -```shell -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -export SGLANG_SET_CPU_AFFINITY=1 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export STREAMS_PER_DEVICE=32 -export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=330 -export DEEPEP_NORMAL_LONG_SEQ_ROUND=5 -export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=3000 -export DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ=1 - -export ASCEND_USE_FIA=1 -export SGLANG_NPU_USE_MULTI_STREAM=1 - -export SGLANG_WARMUP_TIMEOUT=3600 -export SGLANG_ENABLE_SPEC_V2=1 -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export FORCE_DRAFT_MODEL_NON_QUANT=1 - -MODEL_PATH=xxx - -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -export HCCL_BUFFSIZE=2000 -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export HCCL_OP_EXPANSION_MODE="AIV" -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_ENABLE_SPEC_V2=1 - -python3 -m sglang.launch_server --model-path ${MODEL_PATH} \ - --page-size 128 \ - --tp-size 4 \ - --trust-remote-code \ - --attention-backend ascend \ - --device npu \ - --watchdog-timeout 9000 \ - --host 127.0.0.1 --port 6699 \ - --mem-fraction-static 0.75 \ - --disable-radix-cache --max-prefill-tokens 14080 --context-length 26384 \ - --speculative-algorithm NEXTN --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 --speculative-draft-model-quantization unquant \ - --chunked-prefill-size -1 --max-running-requests 312 \ - --cuda-graph-bs 2 4 16 32 48 64 80 96 128 140 156 \ - --mamba-ssm-dtype bfloat16 \ - --base-gpu-id 0 \ - --speculative-draft-model-path /home/weights/Qwen3-Next-80B-A3B-Instruct \ - --quantization modelslim \ - --moe-a2a-backend deepep --deepep-mode auto \ -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell -python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6699 --random-range-ratio 1 --max-concurrency 16 --random-output-len 1500 --random-input-len 6144 --num-prompts 16 -``` - -### Qwen3-14B 3_5K-1_5K 9ms on A3 1 Cards Mixed Mode - -Model: Qwen3-14B - -Hardware: Atlas 800I A3 1Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 3.5K+1.5K - -TPOT: 9ms - -#### Model Deployment - -```shell -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -MODEL_PATH=xxx - -export SGLANG_SET_CPU_AFFINITY=1 -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export HCCL_OP_EXPANSION_MODE="AIV" -export STREAMS_PER_DEVICE=32 -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export ASCEND_USE_FIA=0 -export SGLANG_ENABLE_SPEC_V2=1 -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -python -m sglang.launch_server --model-path $MODEL_PATH \ - --host 127.0.0.1 --port 7239 --trust-remote-code --nnodes 1 --node-rank 0 \ - --attention-backend ascend --device npu --quantization modelslim \ - --disable-radix-cache --mem-fraction-static 0.8 \ - --tp-size 1 --dp-size 1 \ - --sampling-backend ascend --max-running-requests 8 \ - --served-model-name Qwen3-14B \ - --chunked-prefill-size -1 \ - --cuda-graph-bs 8 \ - --dtype bfloat16 \ - --speculative-draft-model-quantization unquant \ - --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ - --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ - --schedule-conservativeness 0.01 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell -python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7239 --max-concurrency 1 --random-output-len 1500 --random-input-len 3500 --num-prompts 8 --random-range-ratio 1 -``` - -### Qwen3-14B 3_5K-1_5K 50ms on A3 1 Cards Mixed Mode - -Model: Qwen3-14B - -Hardware: Atlas 800I A3 1Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 3.5K+1.5K - -TPOT: 50ms - -#### Model Deployment - -```shell -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -MODEL_PATH=xxx - -export SGLANG_SET_CPU_AFFINITY=1 -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export HCCL_OP_EXPANSION_MODE="AIV" -export STREAMS_PER_DEVICE=32 -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export ASCEND_USE_FIA=0 -export SGLANG_ENABLE_SPEC_V2=1 -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 -export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200 - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -python -m sglang.launch_server --model-path $MODEL_PATH \ - --host 127.0.0.1 --port 7239 --trust-remote-code --nnodes 1 --node-rank 0 \ - --attention-backend ascend --device npu --quantization modelslim \ - --disable-radix-cache --mem-fraction-static 0.89 \ - --tp-size 1 --dp-size 2 \ - --sampling-backend ascend --max-running-requests 144 \ - --max-prefill-tokens 12288 \ - --served-model-name Qwen3-14B \ - --chunked-prefill-size -1 \ - --cuda-graph-bs 8 16 32 44 48 50 52 \ - --dtype bfloat16 \ - --speculative-draft-model-quantization unquant \ - --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ - --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ - --schedule-conservativeness 0.01 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell -python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7239 --max-concurrency 144 --random-output-len 1500 --random-input-len 3500 --num-prompts 576 --random-range-ratio 1 -``` - -### Qwen3-8B 3_5K-1_5K 50ms on A3 1 Cards Mixed Mode - -Model: Qwen3-8B - -Hardware: Atlas 800I A3 1Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 3.5K+1.5K - -TPOT: 50ms - -#### Model Deployment - -```shell -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -MODEL_PATH=xxx - -export SGLANG_SET_CPU_AFFINITY=1 -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export HCCL_OP_EXPANSION_MODE="AIV" -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_ENABLE_SPEC_V2=1 -export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 -export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=50 -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -python -m sglang.launch_server --model-path $MODEL_PATH \ - --host 127.0.0.1 --port 7239 --trust-remote-code --nnodes 1 --node-rank 0 \ - --attention-backend ascend --device npu --quantization modelslim \ - --disable-radix-cache --mem-fraction-static 0.9 \ - --tp-size 1 \ - --max-running-requests 70 \ - --max-prefill-tokens 16384 \ - --served-model-name Qwen3-8B \ - --chunked-prefill-size 16384 \ - --cuda-graph-bs 8 12 24 36 48 51 55 60 63 64 66 68 70 \ - --dtype bfloat16 \ - --speculative-draft-model-quantization unquant \ - --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ - --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell -python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7239 --max-concurrency 64 --random-output-len 1500 --random-input-len 3500 --num-prompts 256 --random-range-ratio 1 -``` - -### Qwen3-8B 3_5K-1_5K 5ms on A3 1 Cards Mixed Mode - -Model: Qwen3-8B - -Hardware: Atlas 800I A3 1Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 3.5K+1.5K - -TPOT: 5ms - -#### Model Deployment - -```shell -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -MODEL_PATH=xxx - -export SGLANG_SET_CPU_AFFINITY=1 -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export HCCL_OP_EXPANSION_MODE="AIV" -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_ENABLE_SPEC_V2=1 -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -python -m sglang.launch_server --model-path $MODEL_PATH \ - --host 127.0.0.1 --port 7239 --trust-remote-code --nnodes 1 --node-rank 0 \ - --attention-backend ascend --device npu --quantization modelslim \ - --disable-radix-cache --mem-fraction-static 0.894 \ - --tp-size 2 \ - --max-running-requests 1 \ - --max-prefill-tokens 16384 \ - --served-model-name Qwen3-8B \ - --chunked-prefill-size -1 \ - --cuda-graph-bs 1 \ - --dtype bfloat16 \ - --speculative-draft-model-quantization unquant \ - --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ - --speculative-num-steps 4 --speculative-eagle-topk 1 --speculative-num-draft-tokens 5 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell -python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7239 --max-concurrency 1 --random-output-len 1500 --random-input-len 3500 --num-prompts 4 --random-range-ratio 1 -``` - -### Qwen3-Next 3_5K-1_5K 20ms on A3 2 Cards Mixed Mode - -Model: Qwen3-Next-80B-A3B-Instruct - -Hardware: Atlas 800I A3 2Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 3.5K+1.5K - -TPOT: 20ms - -#### Model Deployment - -```shell -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -export SGLANG_SET_CPU_AFFINITY=1 -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export STREAMS_PER_DEVICE=32 -export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=400 -export DEEPEP_NORMAL_LONG_SEQ_ROUND=10 -export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=2048 -export HCCL_OP_EXPANSION_MODE="AIV" -export TASK_QUEUE_ENABLE=1 -export ASCEND_USE_FIA=1 -export SGLANG_NPU_USE_MULTI_STREAM=0 -export SGLANG_WARMUP_TIMEOUT=3600 -export SGLANG_ENABLE_SPEC_V2=1 -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export FORCE_DRAFT_MODEL_NON_QUANT=1 -export HCCL_BUFFSIZE=2000 -export ZBCCL_LOCAL_MEM_SIZE=60416 -export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 - -export ZBCCL_BOOTSTRAP_URL=tcp://127.0.0.1:24669 -export ZBCCL_NPU_ALLOC_CONF=use_vmm_for_static_memory:True -export ZBCCL_ENABLE_GRAPH=1 - -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo - -MODEL_PATH=xxx - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -python3 -m sglang.launch_server --model-path ${MODEL_PATH} \ - --page-size 128 \ - --tp-size 4 --dp-size 2 \ - --trust-remote-code \ - --attention-backend ascend \ - --device npu \ - --quantization modelslim \ - --watchdog-timeout 9000 \ - --host 127.0.0.1 --port 6699 \ - --mem-fraction-static 0.85 \ - --disable-radix-cache --max-prefill-tokens 28672 --context-length 26384 --max-total-tokens 122304 \ - --enable-dp-attention --enable-dp-lm-head \ - --speculative-algorithm NEXTN --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 --speculative-draft-model-quantization unquant \ - --chunked-prefill-size -1 --max-running-requests 16 \ - --cuda-graph-bs 2 4 8 \ - --mamba-ssm-dtype bfloat16 \ - --speculative-draft-model-path /path/to/Qwen3-Next-80B-A3B-Instruct -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell -python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6699 --random-range-ratio 1 --max-concurrency 1 --random-output-len 1500 --random-input-len 3500 --num-prompts 1 -``` - -## Qwen3.5-397B-A17B 3_5K-1_5K 22ms on A3 8 Cards Mixed Mode - -Model: Qwen3.5-397B-A17B - -Hardware: Atlas 800I A3 8Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 3.5K+1.5K - -TPOT: 22ms - -#### Model Deployment - -```shell -# high performance cpu -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 -# bind cpu -export SGLANG_SET_CPU_AFFINITY=1 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -export ASCEND_USE_FIA=1 -export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128 -export HCCL_BUFFSIZE=3000 -export DEEPEP_NORMAL_LONG_SEQ_ROUND=32 -export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=3584 -export STREAMS_PER_DEVICE=32 -export HCCL_OP_EXPANSION_MODE=AIV -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export SGLANG_ENABLE_SPEC_V2=1 -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_NPU_USE_MULTI_STREAM=1 - -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export SGLANG_ZBAL_LOCAL_MEM_SIZE=58624 -export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 -export SGLANG_ZBAL_BOOTSTRAP_URL="tcp://127.0.0.1:24669" -export ZBAL_NPU_ALLOC_CONF=use_vmm_for_static_memory:True -export ZBAL_ENABLE_GRAPH=1 - -MODEL_PATH=xxx - -python3 -m sglang.launch_server \ ---model-path $MODEL_PATH \ ---attention-backend ascend \ ---device npu \ ---tp-size 16 \ ---chunked-prefill-size -1 --max-prefill-tokens 35000 \ ---disable-radix-cache \ ---trust-remote-code \ ---host 127.0.0.1 --max-running-requests 160 \ ---mem-fraction-static 0.8 \ ---port 6699 \ ---cuda-graph-bs 2 4 6 8 10 12 14 16 18 20 \ ---quantization modelslim \ ---enable-multimodal --moe-a2a-backend deepep --deepep-mode auto \ ---mm-attention-backend ascend_attn \ ---dtype bfloat16 --mamba-ssm-dtype bfloat16 --max-total-tokens 128000 \ ---speculative-algorithm NEXTN --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ ---speculative-draft-model-quantization unquant \ ---dp-size 8 --enable-dp-attention --enable-dp-lm-head \ ---enable-prefill-delayer --prefill-delayer-max-delay-passes 100 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell -python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6699 --random-range-ratio 1 --max-concurrency 120 --random-output-len 1500 --random-input-len 3500 --num-prompts 480 -``` - -## Qwen3.5-397B-A17B 3_5K-1_5K 50ms on A3 8 Cards Mixed Mode - -Model: Qwen3.5-397B-A17B - -Hardware: Atlas 800I A3 8Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 3.5K+1.5K - -TPOT: 50ms - -#### Model Deployment - -```shell -# high performance cpu -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 -# bind cpu -export SGLANG_SET_CPU_AFFINITY=1 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -export ASCEND_USE_FIA=1 -export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128 -export HCCL_BUFFSIZE=3000 -export DEEPEP_NORMAL_LONG_SEQ_ROUND=32 -export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=3584 -export STREAMS_PER_DEVICE=32 -export HCCL_OP_EXPANSION_MODE=AIV -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export SGLANG_ENABLE_SPEC_V2=1 -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_NPU_USE_MULTI_STREAM=1 - -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export SGLANG_ZBAL_LOCAL_MEM_SIZE=59648 -export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 -export SGLANG_ZBAL_BOOTSTRAP_URL="tcp://127.0.0.1:24669" -export ZBAL_NPU_ALLOC_CONF=use_vmm_for_static_memory:True -export ZBAL_ENABLE_GRAPH=1 - -MODEL_PATH=xxx - -python3 -m sglang.launch_server \ ---model-path $MODEL_PATH \ ---attention-backend ascend \ ---device npu \ ---tp-size 16 \ ---chunked-prefill-size -1 --max-prefill-tokens 17500 \ ---disable-radix-cache \ ---trust-remote-code \ ---host 127.0.0.1 --max-running-requests 432 \ ---mem-fraction-static 0.75 \ ---port 6699 \ ---cuda-graph-bs 2 4 6 8 12 16 20 24 28 32 36 40 44 48 52 56 \ ---quantization modelslim \ ---enable-multimodal --moe-a2a-backend deepep --deepep-mode auto \ ---mm-attention-backend ascend_attn \ ---dtype bfloat16 --mamba-ssm-dtype bfloat16 --max-total-tokens 280000 \ ---dp-size 8 --enable-dp-attention --enable-dp-lm-head \ ---speculative-algorithm NEXTN --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ ---speculative-draft-model-quantization unquant \ ---enable-prefill-delayer --prefill-delayer-max-delay-passes 200 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell -python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6699 --random-range-ratio 1 --max-concurrency 176 --random-output-len 1500 --random-input-len 3500 --num-prompts 352 -``` diff --git a/docs/platforms/ascend/ascend_npu_deepseek_example.md b/docs/platforms/ascend/ascend_npu_deepseek_example.md deleted file mode 100644 index 0312633b7..000000000 --- a/docs/platforms/ascend/ascend_npu_deepseek_example.md +++ /dev/null @@ -1,294 +0,0 @@ -## DeepSeek examples - -### Running DeepSeek-V3 - -#### Running DeepSeek in PD mixed mode on 1 x Atlas 800I A3. - -W4A8 Model weights could be found [here](https://modelers.cn/models/Modelers_Park/DeepSeek-R1-0528-w4a8). - -```shell -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export STREAMS_PER_DEVICE=32 - -#Deepep communication settings -export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=32 -export HCCL_BUFFSIZE=1600 - -#spec overlap -export SGLANG_ENABLE_SPEC_V2=1 -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - -#npu acceleration operator -export SGLANG_NPU_USE_MLAPO=1 -export SGLANG_USE_FIA_NZ=1 - -python3 -m sglang.launch_server \ - --model-path ${MODEL_PATH} \ - --tp 16 \ - --trust-remote-code \ - --attention-backend ascend \ - --device npu \ - --quantization modelslim \ - --watchdog-timeout 9000 \ - --cuda-graph-bs 8 16 24 28 32 \ - --mem-fraction-static 0.68 \ - --max-running-requests 128 \ - --context-length 8188 \ - --disable-radix-cache \ - --chunked-prefill-size -1 \ - --max-prefill-tokens 16384 \ - --moe-a2a-backend deepep \ - --deepep-mode auto \ - --enable-dp-attention \ - --dp-size 4 \ - --enable-dp-lm-head \ - --speculative-algorithm NEXTN \ - --speculative-num-steps 3 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 \ - --dtype bfloat16 -``` - -#### Running DeepSeek with PD disaggregation mode on 2 x Atlas 800I A3. - -W4A8 Model weights could be found [here](https://modelers.cn/models/Modelers_Park/DeepSeek-R1-0528-w4a8). - -1. Prefill: - -```shell -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export STREAMS_PER_DEVICE=32 - -#memfabric config store -export ASCEND_MF_STORE_URL="tcp://:" - -#Deepep communication settings -export HCCL_BUFFSIZE=1536 - -#npu acceleration operator -export SGLANG_NPU_USE_MLAPO=1 -export SGLANG_USE_FIA_NZ=1 -export TASK_QUEUE_ENABLE=2 - -python -m sglang.launch_server \ - --model-path ${MODEL_PATH} \ - --host $PREFILL_HOST_IP \ - --port 8000 \ - --disaggregation-mode prefill \ - --disaggregation-bootstrap-port 8996 \ - --disaggregation-transfer-backend ascend \ - --trust-remote-code \ - --nnodes 1 \ - --node-rank 0 \ - --tp-size 16 \ - --mem-fraction-static 0.6 \ - --attention-backend ascend \ - --device npu \ - --quantization modelslim \ - --load-balance-method round_robin \ - --max-running-requests 8 \ - --context-length 8192 \ - --disable-radix-cache \ - --chunked-prefill-size -1 \ - --max-prefill-tokens 28680 \ - --moe-a2a-backend deepep \ - --deepep-mode normal \ - --speculative-algorithm NEXTN \ - --speculative-num-steps 3 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 \ - --dp-size 2 \ - --enable-dp-attention \ - --disable-shared-experts-fusion \ - --dtype bfloat16 -``` - -2. Decode: - -```shell -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export STREAMS_PER_DEVICE=32 - -#memfabric config store -export ASCEND_MF_STORE_URL="tcp://:" - -#Deepep communication settings -export HCCL_BUFFSIZE=720 -export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=88 - -#spec overlap -export SGLANG_ENABLE_SPEC_V2=1 -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - -#npu acceleration operator -unset TASK_QUEUE_ENABLE -export SGLANG_NPU_USE_MLAPO=1 -export SGLANG_USE_FIA_NZ=1 - -# suggest max-running-requests <= max-cuda-graph-bs * dp_size, Because when this value is exceeded, performance will significantly degrade. -python -m sglang.launch_server \ - --model-path ${MODEL_PATH} \ - --disaggregation-mode decode \ - --host $DECODE_HOST_IP \ - --port 8001 \ - --trust-remote-code \ - --nnodes 1 \ - --node-rank 0 \ - --tp-size 16 \ - --dp-size 16 \ - --mem-fraction-static 0.8 \ - --max-running-requests 352 \ - --attention-backend ascend \ - --device npu \ - --quantization modelslim \ - --moe-a2a-backend deepep \ - --enable-dp-attention \ - --deepep-mode low_latency \ - --enable-dp-lm-head \ - --cuda-graph-bs 8 10 12 14 16 18 20 22 \ - --disaggregation-transfer-backend ascend \ - --watchdog-timeout 9000 \ - --context-length 8192 \ - --speculative-algorithm NEXTN \ - --speculative-num-steps 3 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 \ - --disable-shared-experts-fusion \ - --dtype bfloat16 \ - --tokenizer-worker-num 4 -``` - -3. SGLang Model Gateway (former Router) - -```shell -python -m sglang_router.launch_router \ - --pd-disaggregation \ - --policy cache_aware \ - --prefill http://:8000 8996 \ - --decode http://:8001 \ - --host 127.0.0.1 \ - --port 6688 -``` - -#### Running DeepSeek with PD disaggregation on 4 x Atlas 800I A3. - -W8A8 Model weights could be found [here](https://modelers.cn/models/State_Cloud/Deepseek-R1-bf16-hfd-w8a8). - -1. Prefill & Decode: - -```shell -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 -export SGLANG_SET_CPU_AFFINITY=1 -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export STREAMS_PER_DEVICE=32 - -export ASCEND_MF_STORE_URL="tcp://your prefill ip1:24669" - -P_IP=('your prefill ip1' 'your prefill ip2') - -D_IP=('your decode ip1' 'your decode ip2') - -MODEL_PATH=xxx - -export SGLANG_NPU_USE_MLAPO=1 -export SGLANG_USE_FIA_NZ=1 - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" -# prefill -for i in "${!P_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; - then - echo "${P_IP[$i]}" - export HCCL_BUFFSIZE=1536 - export TASK_QUEUE_ENABLE=2 - - export HCCL_SOCKET_IFNAME=lo - export GLOO_SOCKET_IFNAME=lo - python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode prefill --host ${P_IP[$i]} \ - --port 8000 --disaggregation-bootstrap-port $((8998+$i)) --trust-remote-code --nnodes 1 --node-rank 0 \ - --tp-size 16 --mem-fraction-static 0.81 --attention-backend ascend --device npu --quantization modelslim \ - --disaggregation-transfer-backend ascend --max-running-requests 8 --context-length 8192 --disable-radix-cache \ - --chunked-prefill-size -1 --max-prefill-tokens 28680 --moe-a2a-backend deepep --deepep-mode normal \ - --speculative-algorithm NEXTN --speculative-num-steps 1 --speculative-eagle-topk 1 --speculative-num-draft-tokens 2 \ - --dp-size 2 --enable-dp-attention --disable-shared-experts-fusion --dtype bfloat16 --enable-attn-tp-input-scattered - NODE_RANK=$i - break - fi -done - -# decode -for i in "${!D_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; - then - echo "${D_IP[$i]}" - export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - export SGLANG_ENABLE_SPEC_V2=1 - export HCCL_BUFFSIZE=650 - export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=78 - export TASK_QUEUE_ENABLE=1 - export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1 - export HCCL_SOCKET_IFNAME=xxx - export GLOO_SOCKET_IFNAME=xxx - python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode decode --host ${D_IP[$i]} \ - --port 8001 --trust-remote-code --dist-init-addr ${D_IP[0]}:5000 --nnodes 2 --node-rank $i --tp-size 32 --dp-size 32 \ - --mem-fraction-static 0.815 --max-running-requests 832 --attention-backend ascend --device npu --quantization modelslim \ - --moe-a2a-backend deepep --enable-dp-attention --deepep-mode low_latency --enable-dp-lm-head --moe-dense-tp 1 \ - --cuda-graph-bs 12 14 16 18 20 22 24 26 --disaggregation-transfer-backend ascend --watchdog-timeout 9000 --context-length 8192 \ - --speculative-algorithm NEXTN --speculative-num-steps 2 --speculative-eagle-topk 1 --speculative-num-draft-tokens 3 \ - --tokenizer-worker-num 4 --disable-shared-experts-fusion --dtype bfloat16 \ - --load-balance-method decode_round_robin - NODE_RANK=$i - break - fi -done -``` - -2. SGLang Model Gateway (former Router): - -```shell -python -m sglang_router.launch_router \ - --pd-disaggregation \ - --policy cache_aware \ - --prefill http://P_IP:8000 8998 \ - --prefill http://P_IP:8000 8999 \ - --decode http://D_IP:8001 \ - --host 127.0.0.1 \ - --port 6688 \ - --mini-lb -``` - -#### test gsm8k - -```python -from types import SimpleNamespace -from sglang.test.few_shot_gsm8k import run_eval - -def gsm8k(): - args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=32, - host=f"http://127.0.0.1", - port=6688, - ) - metrics = run_eval(args) - print(f"{metrics=}") - print(f"{metrics['accuracy']=}") -if __name__ == "__main__": - gsm8k() -``` diff --git a/docs/platforms/ascend/ascend_npu_environment_variables.md b/docs/platforms/ascend/ascend_npu_environment_variables.md deleted file mode 100644 index c638298b5..000000000 --- a/docs/platforms/ascend/ascend_npu_environment_variables.md +++ /dev/null @@ -1,38 +0,0 @@ -# Environment Variables - -SGLang supports various environment variables related to Ascend NPU that can be used to configure its runtime behavior. -This document provides a list of commonly used environment variables and aims to stay updated over time. - -## Directly Used in SGLang - -| Environment Variable | Description | Default Value | -|--------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------| -| `SGLANG_NPU_USE_MLAPO` | Adopts the `MLAPO` fusion operator in attention
preprocessing stage of the MLA model. | `false` | -| `SGLANG_USE_FIA_NZ` | Reshapes KV Cache for FIA NZ format.
`SGLANG_USE_FIA_NZ` must be enabled with `SGLANG_NPU_USE_MLAPO` | `false` | -| `SGLANG_NPU_USE_MULTI_STREAM` | Enable dual-stream computation of shared experts
and routing experts in DeepSeek models.
Enable dual-stream computation in DeepSeek DSA Indexer. | `false` | -| `SGLANG_NPU_DISABLE_ACL_FORMAT_WEIGHT` | Disable cast model weight tensor to a specific NPU
ACL format. | `false` | -| `SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK` | The maximum number of dispatched tokens on each rank. | `128` | - -## Used in DeepEP Ascend - -| Environment Variable | Description | Default Value | -|-------------------------------------------|------------------------------------------------------------------------------------------------------------------------|---------------| -| `DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS` | Enable ant-moving function in dispatch stage. Indicates
the number of tokens transmitted per round on each rank. | `8192` | -| `DEEPEP_NORMAL_LONG_SEQ_ROUND` | Enable ant-moving function in dispatch stage. Indicates
the number of rounds transmitted on each rank. | `1` | -| `DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ` | Enable ant-moving function in combine stage.
The value `0` means disabled. | `0` | -| `MOE_ENABLE_TOPK_NEG_ONE` | Needs to be enabled when the expert ID to be processed by
DEEPEP contains -1. | `0` | - -## Others - -| Environment Variable | Description | Default Value | -|--------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------| -| `TASK_QUEUE_ENABLE` | Used to control the optimization level of the dispatch queue
about the task_queue operator. [Detail](https://www.hiascend.com/document/detail/zh/Pytorch/730/comref/Envvariables/docs/zh/environment_variable_reference/TASK_QUEUE_ENABLE.md) | `1` | -| `INF_NAN_MODE_ENABLE` | Controls whether the chip uses saturation mode or INF_NAN mode. [Detail](https://www.hiascend.com/document/detail/zh/CANNCommunityEdition/800alpha001/apiref/envref/envref_07_0056.html) | `1` | -| `STREAMS_PER_DEVICE` | Configures the maximum number of streams for the stream pool. [Detail](https://www.hiascend.com/document/detail/zh/Pytorch/720/comref/Envvariables/Envir_041.html) | `32` | -| `PYTORCH_NPU_ALLOC_CONF` | Controls the behavior of the cache allocator.
This variable changes memory usage and may cause performance fluctuations. [Detail](https://www.hiascend.com/document/detail/zh/Pytorch/700/comref/Envvariables/Envir_012.html) | | -| `ASCEND_MF_STORE_URL` | The address of config store in MemFabric during PD separation,
which is generally set to the IP address of the P primary node
with an arbitrary port number. | | -| `ASCEND_LAUNCH_BLOCKING` | Controls whether synchronous mode is enabled during operator execution. [Detail](https://www.hiascend.com/document/detail/zh/Pytorch/710/comref/Envvariables/Envir_006.html) | `0` | -| `HCCL_OP_EXPANSION_MODE` | Configures the expansion position for communication algorithm scheduling. [Detail](https://www.hiascend.com/document/detail/zh/CANNCommunityEdition/800alpha001/apiref/envref/envref_07_0094.html) | | -| `HCCL_BUFFSIZE` | Controls the size of the buffer area for shared data between two NPUs.
The unit is MB, and the value must be greater than or equal to 1. [Detail](https://www.hiascend.com/document/detail/zh/Pytorch/60RC3/ptmoddevg/trainingmigrguide/performance_tuning_0047.html) | `200` | -| `HCCL_SOCKET_IFNAME` | Configures the name of the network card used by the Host
during HCCL initialization. [Detail](https://www.hiascend.com/document/detail/zh/canncommercial/81RC1/apiref/envvar/envref_07_0075.html) | | -| `GLOO_SOCKET_IFNAME` | Configures the network interface name for GLOO communication. | | diff --git a/docs/platforms/ascend/ascend_npu_glm5_examples.md b/docs/platforms/ascend/ascend_npu_glm5_examples.md deleted file mode 100644 index d83f670fc..000000000 --- a/docs/platforms/ascend/ascend_npu_glm5_examples.md +++ /dev/null @@ -1,200 +0,0 @@ -# GLM-5 examples - -## Introduction - -The GLM (General Language Model) series is an open-source bilingual large language model family jointly developed by the KEG Laboratory of Tsinghua University and Zhipu AI. This series of models has performed outstandingly in the field of Chinese NLP with its unique unified pre-training framework and bilingual capabilities. [GLM-5](https://huggingface.co/zai-org/GLM-5) adopts the DeepSeek-V3/V3.2 architecture, including the sparse attention (DSA) and multi-token prediction (MTP). Ascend supports GLM-5 with 0Day based on the SGLang inference framework, achieving low-code seamless enablement and compatibility with the mainstream distributed parallel capabilities within the current SGLang framework. We welcome developers to download and experience it. - -## Environment Preparation - -### Model Weight - -- `GLM-5.0`(BF16 version): [Download model weight](https://www.modelscope.cn/models/ZhipuAI/GLM-5). -- `GLM-5.0-w4a8`(Quantized version without mtp): [Download model weight](https://modelers.cn/models/Eco-Tech/GLM-5-w4a8). -- You can use [msmodelslim](https://gitcode.com/Ascend/msmodelslim) to quantify the model naively. - - -### Installation - -The dependencies required for the NPU runtime environment have been integrated into a Docker image and uploaded to the online platform. You can directly pull it. - -```{code-block} bash -#Atlas 800 A3 -docker pull swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:cann8.5.0-a3-glm5 -#Atlas 800 A2 -docker pull swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:cann8.5.0-910b-glm5 - -#start container -docker run -itd --shm-size=16g --privileged=true --name ${NAME} \ ---privileged=true --net=host \ --v /var/queue_schedule:/var/queue_schedule \ --v /etc/ascend_install.info:/etc/ascend_install.info \ --v /usr/local/sbin:/usr/local/sbin \ --v /usr/local/Ascend/driver:/usr/local/Ascend/driver \ --v /usr/local/Ascend/firmware:/usr/local/Ascend/firmware \ ---device=/dev/davinci0:/dev/davinci0 \ ---device=/dev/davinci1:/dev/davinci1 \ ---device=/dev/davinci2:/dev/davinci2 \ ---device=/dev/davinci3:/dev/davinci3 \ ---device=/dev/davinci4:/dev/davinci4 \ ---device=/dev/davinci5:/dev/davinci5 \ ---device=/dev/davinci6:/dev/davinci6 \ ---device=/dev/davinci7:/dev/davinci7 \ ---device=/dev/davinci8:/dev/davinci8 \ ---device=/dev/davinci9:/dev/davinci9 \ ---device=/dev/davinci10:/dev/davinci10 \ ---device=/dev/davinci11:/dev/davinci11 \ ---device=/dev/davinci12:/dev/davinci12 \ ---device=/dev/davinci13:/dev/davinci13 \ ---device=/dev/davinci14:/dev/davinci14 \ ---device=/dev/davinci15:/dev/davinci15 \ ---device=/dev/davinci_manager:/dev/davinci_manager \ ---device=/dev/hisi_hdc:/dev/hisi_hdc \ ---entrypoint=bash \ -swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:${TAG} -``` - -### Best Practices -Note: Using this image for **best practices**, you need to update transformers to version 5.3.0 -``` shell -# reinstall transformers - -# Install transformers version 5.3.0 from PyPI -pip install transformers==5.3.0 - -# Install from GitHub v5.3.0 tag from GitHub -pip install git+https://github.com/huggingface/transformers.git@v5.3.0 -``` - -## Deployment - -### Single-node Deployment - -- Quantized model `glm5_w4a8` can be deployed on 1 Atlas 800 A3 (64G × 16) . - -Run the following script to execute online inference. - -```shell -# high performance cpu -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 -# bind cpu -export SGLANG_SET_CPU_AFFINITY=1 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -# cann -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh - -export STREAMS_PER_DEVICE=32 -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 -export SGLANG_ENABLE_SPEC_V2=1 -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_NPU_USE_MULTI_STREAM=1 -export HCCL_BUFFSIZE=1000 -export HCCL_OP_EXPANSION_MODE=AIV -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo - -python3 -m sglang.launch_server \ - --model-path $MODEL_PATH \ - --attention-backend ascend \ - --device npu \ - --tp-size 16 --nnodes 1 --node-rank 0 \ - --chunked-prefill-size 16384 --max-prefill-tokens 280000 \ - --trust-remote-code \ - --host 127.0.0.1 \ - --mem-fraction-static 0.7 \ - --port 8000 \ - --served-model-name glm-5 \ - --cuda-graph-bs 16 \ - --quantization modelslim \ - --moe-a2a-backend deepep --deepep-mode auto -``` - -### Multi-node Deployment - -- `GLM-5-bf16`: require at least 2 Atlas 800 A3 (64G × 16). - -**A3 series** - -Modify the IP of 2 nodes, then run the same scripts on two nodes. - -**node 0/1** - -```shell -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 -# bind cpu -export SGLANG_SET_CPU_AFFINITY=1 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -# cann -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh - -export STREAMS_PER_DEVICE=32 -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 -export SGLANG_ENABLE_SPEC_V2=1 -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_NPU_USE_MULTI_STREAM=1 -export HCCL_BUFFSIZE=1000 -export HCCL_OP_EXPANSION_MODE=AIV - -# Run command ifconfig on two nodes, find out which inet addr has same IP with your node IP. That is your public interface, which should be added here -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo - - -P_IP=('your ip1' 'your ip2') -P_MASTER="${P_IP[0]}:your port" -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 - - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` -for i in "${!P_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; - then - echo "${P_IP[$i]}" - python3 -m sglang.launch_server \ - --model-path $MODEL_PATH \ - --attention-backend ascend \ - --device npu \ - --tp-size 32 --nnodes 2 --node-rank $i --dist-init-addr $P_MASTER \ - --chunked-prefill-size 16384 --max-prefill-tokens 131072 \ - --trust-remote-code \ - --host 127.0.0.1 \ - --mem-fraction-static 0.8\ - --port 8000 \ - --served-model-name glm-5 \ - --cuda-graph-max-bs 32 \ - --moe-a2a-backend deepep \ - --deepep-mode auto \ - --disable-radix-cache - NODE_RANK=$i - break - fi -done - -``` - -### Prefill-Decode Disaggregation - -Not test yet. - -### Using Benchmark - -Refer to [Benchmark and Profiling](../../developer_guide/benchmark_and_profiling.md) for details. diff --git a/docs/platforms/ascend/ascend_npu_quantization.md b/docs/platforms/ascend/ascend_npu_quantization.md deleted file mode 100644 index e60173850..000000000 --- a/docs/platforms/ascend/ascend_npu_quantization.md +++ /dev/null @@ -1,134 +0,0 @@ -# Quantization on Ascend - -To load already quantized models, simply load the model weights and config. Again, if the model has been quantized offline, there's no need to add `--quantization` argument when starting the engine. The quantization method will be automatically parsed from the downloaded `quant_model_description.json` or `config.json` config. - -SGLang support **mix-bits** quantization (independently defines and loads each layer depending on the type of quantification specified in the `quant_model_description'.json`). [Advanced mix-bits for MoE](https://github.com/sgl-project/sglang/pull/17361) in progress, will add independent quantization determination for the w13 (up-gate) and w2 (down) layers. - -[ModelSlim on Ascend support](https://github.com/sgl-project/sglang/pull/14504) -| Quantization scheme | `quant_type` in JSON | Scheme class | Layer type | A2 Supported | A3 Supported | A5 Supported | Diffusion models | -|-----------------------------------------------------------|----------------------|--------------------------|--------------------------|:----------------------------------------:|:----------------------------------------:|:------------------------------------------:|:------------------------------------------:| -| W4A4 dynamic | `W4A4_DYNAMIC` | `ModelSlimW4A4Int4` | Linear | **√** | **√** | **TBD** | **√** | -| W8A8 static | `W8A8` | `ModelSlimW8A8Int8` | Linear | **√** | **√** | **TBD** | **√** | -| W8A8 dynamic | `W8A8_DYNAMIC` | `ModelSlimW8A8Int8` | Linear | **√** | **√** | **TBD** | **√** | -| [MXFP8](https://github.com/sgl-project/sglang/pull/20922) | `W8A8_MXFP8` | `ModelSlimMXFP8Scheme` | Linear | **x** | **x** | **WIP** | **√** (A5) | -| W4A4 dynamic | `W4A4_DYNAMIC` | `ModelSlimW4A4Int4` | MoE | **√** | **√** | **TBD** | **x** | -| W4A8 dynamic | `W4A8_DYNAMIC` | `ModelSlimW4A8Int8MoE` | MoE | **√** | **√** | **TBD** | **x** | -| W8A8 dynamic | `W8A8_DYNAMIC` | `ModelSlimW8A8Int8` | MoE | **√** | **√** | **TBD** | **x** | -| [MXFP8](https://github.com/sgl-project/sglang/pull/20922) | `W8A8_MXFP8` | `ModelSlimMXFP8Scheme` | MoE | **x** | **x** | **WIP** | **x** | - -[AWQ on Ascend support](https://github.com/sgl-project/sglang/pull/10158): -| Quantization scheme | Layer type | A2 Supported | A3 Supported | A5 Supported | -|--------------------------------|--------------------------|:----------------------------------------:|:----------------------------------------:|:------------------------------------------:| -| W4A16 | Linear | **√** | **√** | **TBD** | -| W8A16 | Linear | **√** | **√** | **TBD** | -| W4A16 | MoE | **√** | **√** | **TBD** | - -GPTQ on Ascend support -| Quantization scheme | Layer type | A2 Supported | A3 Supported | A5 Supported | -|----------------------------------------------------------------------------|--------------------------|:----------------------------------------:|:----------------------------------------:|:-----------------------------------------:| -| [W4A16](https://github.com/sgl-project/sglang/pull/15203) | Linear | **√** | **√** | **TBD** | -| [W8A16](https://github.com/sgl-project/sglang/pull/15203) | Linear | **√** | **√** | **TBD** | -| [W4A16 MOE](https://github.com/sgl-project/sglang/pull/16364) | MoE | **√** | **√** | **TBD** | -| [W8A16 MOE](https://github.com/sgl-project/sglang/pull/16364) | MoE | **√** | **√** | **TBD** | - -[Auto-round on Ascend support](https://github.com/sgl-project/sglang/pull/16699) -| Quantization scheme | Layer type | A2 Supported | A3 Supported | A5 Supported | -|--------------------------------|--------------------------|:----------------------------------------:|:----------------------------------------:|:-----------------------------------------:| -| W4A16 | Linear | **√** | **√** | **TBD** | -| W8A16 | Linear | **√** | **√** | **TBD** | -| W4A16 | MoE | **√** | **√** | **TBD** | -| W8A16 | MoE | **√** | **√** | **TBD** | - -Compressed-tensors (LLM Compressor) on Ascend support: -| Quantization scheme | Layer type | A2 Supported | A3 Supported | A5 Supported | -|-----------------------------------------------------------------------------------------------|--------------------------|:----------------------------------------:|:----------------------------------------:|:-----------------------------------------:| -| [W8A8 dynamic](https://github.com/sgl-project/sglang/pull/14504) | Linear | **√** | **√** | **TBD** | -| [W4A8 dynamic with/without activation clip](https://github.com/sgl-project/sglang/pull/14736) | MoE | **√** | **√** | **TBD** | -| [W4A16 MOE](https://github.com/sgl-project/sglang/pull/12759) | MoE | **√** | **√** | **TBD** | -| [W8A8 dynamic](https://github.com/sgl-project/sglang/pull/14504) | MoE | **√** | **√** | **TBD** | - -[GGUF on Ascend support](https://github.com/sgl-project/sglang/pull/17883) -| Quantization scheme | Layer type | A2 Supported | A3 Supported | A5 Supported | -|-----------------------------------------------------------|--------------------------|:----------------------------------------:|:----------------------------------------:|:-----------------------------------------:| -| [GGUF (all types)](https://github.com/sgl-project/sglang/pull/17883) | Linear | **√** | **√** | **TBD** | -| [GGUF (all types)](https://github.com/sgl-project/sglang/pull/17883) | MoE | **√** | **√** | **TBD** | - -> Note: On Ascend, GGUF weights are pre-dequantized to FP16/BF16 during model loading to ensure optimal inference performance. This enables support for all GGUF quantization types (Q2_K, Q4_K_M, IQ4_XS, etc.) while maintaining high inference speed. - -in progress - -## Diffusion Model Quantization on Ascend NPU - -SGLang-Diffusion supports MXFP8 online and offline quantization for diffusion models (such as Wan2.2) on Ascend NPUs. MXFP8 requires A5; the ModelSlim W8A8/W4A4 schemes work on A2/A3. - -**Requirements for MXFP8:** CANN ≥ 8.0.RC3, Ascend A5 - -| Quantization method | `quant_type` in JSON | Scheme class | Mode | A2/A3 Supported | A5 Supported | Trigger | -|---------------------|-----------------------|-------------------------------|---------|:--------------------------------------------:|:----------------------------------------:|---------------------------------------------------| -| MXFP8 (W8A8) | — | `MXFP8Config` | Online | **x** | **√** | `--quantization mxfp8` | -| MXFP8 (W8A8) | `W8A8_MXFP8` | `ModelSlimMXFP8Scheme` | Offline | **x** | **√** | auto-detected from `quant_model_description.json` | -| W8A8 static | `W8A8` | `ModelSlimW8A8Int8` | Offline | **√** | **TBD** | auto-detected from `quant_model_description.json` | -| W8A8 dynamic | `W8A8_DYNAMIC` | `ModelSlimW8A8Int8` | Offline | **√** | **TBD** | auto-detected from `quant_model_description.json` | -| W4A4 dynamic | `W4A4_DYNAMIC` | `ModelSlimW4A4Int4` | Offline | **√** | **TBD** | auto-detected from `quant_model_description.json` | - -### Online MXFP8 Quantization - -Online quantization dynamically quantizes FP16/BF16 weights to MXFP8 at load time using `npu_dynamic_mx_quant` + `npu_quant_matmul` CANN kernels. Pass `--quantization mxfp8` to override auto-detection. - -```bash -# Start the diffusion server with online MXFP8 quantization -sglang serve \ - --model-path Wan-AI/Wan2.2-T2V-A14B-Diffusers \ - --quantization mxfp8 \ - --num-gpus 4 -``` - -```bash -# One-shot generation -sglang generate \ - --model-path Wan-AI/Wan2.2-T2V-A14B-Diffusers \ - --quantization mxfp8 \ - --prompt "a beautiful sunset over the mountains" \ - --save-output -``` - -### Offline MXFP8 Quantization (ModelSlim) - -For offline quantization, pre-quantize the model with msModelSlim and load the resulting checkpoint. The quantization scheme is auto-detected from `quant_model_description.json`, so no extra `--quantization` flag is needed. - -**Step 1: Quantize with msModelSlim** - -```bash -msmodelslim quant \ - --model_path /path/to/wan2_2_float_weights \ - --save_path /path/to/wan2_2_mxfp8_weights \ - --device npu \ - --model_type Wan2_2 \ - --quant_type mxfp8 \ - --trust_remote_code True -``` - -> Note: SGLang does not support quantized embeddings; disable embedding quantization when using msmodelslim. - -**Step 2: Convert to Diffusers format** - -msModelSlim saves quantized Wan2.2 weights in the original Wan format. Convert to Diffusers format using the provided repack script: - -```bash -python python/sglang/multimodal_gen/tools/wan_repack.py \ - --input-path /path/to/wan2_2_mxfp8_weights \ - --output-path /path/to/wan2_2_mxfp8_diffusers -``` - -Then copy all files from the original Diffusers checkpoint (except the `transformer`/`transformer_2` folders) into the output directory. - -**Step 3: Run inference** - -```bash -sglang generate \ - --model-path /path/to/wan2_2_mxfp8_diffusers \ - --prompt "a beautiful sunset over the mountains" \ - --save-output -``` - -For pre-quantized checkpoints available on ModelScope, see [modelscope/Eco-Tech](https://modelscope.cn/models/Eco-Tech). diff --git a/docs/platforms/ascend/ascend_npu_quick_start.md b/docs/platforms/ascend/ascend_npu_quick_start.md deleted file mode 100644 index 7f0bef6e8..000000000 --- a/docs/platforms/ascend/ascend_npu_quick_start.md +++ /dev/null @@ -1,103 +0,0 @@ -# Ascend NPU Quickstart - -## Prerequisites - -### Supported Devices - -- Atlas 800I A2 inference series (Atlas 800I A2) -- Atlas 800I A3 inference series (Atlas 800I A3) - -## Setup environment using container - -__Notice:__ The following commands are based on Atlas 800I A3 machines. If you are using Atlas 800I A2, some changes are needed. - -- The image tag needs to be `main-cann8.5.0-a3` for Atlas 800I A3 and `main-cann8.5.0-910b` for Atlas 800I A2. -- The device mapping in `docker run` command needs to be changed to `davinci[0-7]` for Atlas 800I A2. - -```shell -# For Atlas 800I A3 -export IMAGE=quay.io/ascend/sglang:main-cann8.5.0-a3 - -docker run -it --rm --privileged --network=host --ipc=host --shm-size=16g \ - --device=/dev/davinci0 --device=/dev/davinci1 --device=/dev/davinci2 --device=/dev/davinci3 \ - --device=/dev/davinci4 --device=/dev/davinci5 --device=/dev/davinci6 --device=/dev/davinci7 \ - --device=/dev/davinci8 --device=/dev/davinci9 --device=/dev/davinci10 --device=/dev/davinci11 \ - --device=/dev/davinci12 --device=/dev/davinci13 --device=/dev/davinci14 --device=/dev/davinci15 \ - --device=/dev/davinci_manager \ - --device=/dev/hisi_hdc \ - --volume /usr/local/sbin:/usr/local/sbin \ - --volume /usr/local/Ascend/driver:/usr/local/Ascend/driver \ - --volume /usr/local/Ascend/firmware:/usr/local/Ascend/firmware \ - --volume /etc/ascend_install.info:/etc/ascend_install.info \ - --volume /var/queue_schedule:/var/queue_schedule \ - --volume ~/.cache/:/root/.cache/ \ - --entrypoint=bash \ - $IMAGE -``` - -## Usage - -The SGLang server is installed in the container by default. You can use `pip show sglang` to check the version. - -### Start SGLang server - -SGLang will automatically download the model from Hugging Face. - -```shell -# Set HF_ENDPOINT to a mirror site if network is not available -export HF_ENDPOINT=https://hf-mirror.com - -# Set your own HF_TOKEN to download restricted models -export HF_TOKEN= - -# Start SGLang server -# It may take several minutes to download the model on the first run -sglang serve --model-path Qwen/Qwen2.5-7B-Instruct --attention-backend ascend & -``` - -If you see output like the following, the server is running. - -```log -INFO: Waiting for application startup. -INFO: Application startup complete. -INFO: Uvicorn running on http://127.0.0.1:30000 (Press CTRL+C to quit) -The server is fired up and ready to roll! -``` - -### Send a test request - -You can do inference using the server: - -```shell -curl -X POST http://localhost:30000/generate \ - -H "Content-Type: application/json" \ - -d '{ - "text": "The capital of France is", - "sampling_params": { - "temperature": 0, - "max_new_tokens": 16 - } - }' -``` - -If the "text" field in the response contains "Paris", the server is working as expected. - -### Stop server and exit container - -The SGLang server is running as a background process. You can send a `SIGINT` signal to stop it. - -```shell -SGLANG_PID=$(pgrep -f "sglang serve") -kill -SIGINT $SGLANG_PID -``` - -The output should be like the following: - -```log -INFO: Shutting down -INFO: Waiting for application shutdown. -INFO: Application shutdown complete. -INFO: Finished server process [25310] -``` - -The server has now stopped. You can verify it with `ps -ef | grep sglang`, then exit the container by pressing `Ctrl+D`. diff --git a/docs/platforms/ascend/ascend_npu_qwen3_5_examples.md b/docs/platforms/ascend/ascend_npu_qwen3_5_examples.md deleted file mode 100644 index 8660f17cc..000000000 --- a/docs/platforms/ascend/ascend_npu_qwen3_5_examples.md +++ /dev/null @@ -1,231 +0,0 @@ -# Qwen3.5 examples - -## Environment Preparation - -### Installation - -The dependencies required for the NPU runtime environment have been integrated into a Docker image and uploaded to the quay.io platform. You can directly pull it. - -```{code-block} bash -#Atlas 800 A3 -docker pull quay.io/ascend/sglang:main-cann8.5.0-a3 -#Atlas 800 A2 -docker pull quay.io/ascend/sglang:main-cann8.5.0-910b - -#start container -docker run -itd --shm-size=16g --privileged=true --name ${NAME} \ ---privileged=true --net=host \ --v /var/queue_schedule:/var/queue_schedule \ --v /etc/ascend_install.info:/etc/ascend_install.info \ --v /usr/local/sbin:/usr/local/sbin \ --v /usr/local/Ascend/driver:/usr/local/Ascend/driver \ --v /usr/local/Ascend/firmware:/usr/local/Ascend/firmware \ ---device=/dev/davinci0:/dev/davinci0 \ ---device=/dev/davinci1:/dev/davinci1 \ ---device=/dev/davinci2:/dev/davinci2 \ ---device=/dev/davinci3:/dev/davinci3 \ ---device=/dev/davinci4:/dev/davinci4 \ ---device=/dev/davinci5:/dev/davinci5 \ ---device=/dev/davinci6:/dev/davinci6 \ ---device=/dev/davinci7:/dev/davinci7 \ ---device=/dev/davinci8:/dev/davinci8 \ ---device=/dev/davinci9:/dev/davinci9 \ ---device=/dev/davinci10:/dev/davinci10 \ ---device=/dev/davinci11:/dev/davinci11 \ ---device=/dev/davinci12:/dev/davinci12 \ ---device=/dev/davinci13:/dev/davinci13 \ ---device=/dev/davinci14:/dev/davinci14 \ ---device=/dev/davinci15:/dev/davinci15 \ ---device=/dev/davinci_manager:/dev/davinci_manager \ ---device=/dev/hisi_hdc:/dev/hisi_hdc \ ---entrypoint=bash \ -quay.io/ascend/sglang:${tag} -``` - -## Deployment - -### Single-node Deployment - -Run the following script to execute online inference. - -#### Qwen3.5 397B - -```shell -# high performance cpu -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 -# bind cpu -export SGLANG_SET_CPU_AFFINITY=1 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -# cann -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh - -export STREAMS_PER_DEVICE=32 -export HCCL_BUFFSIZE=1000 -export HCCL_OP_EXPANSION_MODE=AIV -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo - -python3 -m sglang.launch_server \ - --model-path $MODEL_PATH \ - --attention-backend ascend \ - --device npu \ - --tp-size 16 --nnodes 1 --node-rank 0 \ - --chunked-prefill-size 4096 --max-prefill-tokens 280000 \ - --disable-radix-cache \ - --trust-remote-code \ - --host 127.0.0.1 \ - --mem-fraction-static 0.7 \ - --port 8000 \ - --cuda-graph-bs 16 \ - --quantization modelslim \ - --enable-multimodal \ - --mm-attention-backend ascend_attn \ - --dtype bfloat16 -``` - -#### Qwen3.5 122B - -```shell -# high performance cpu -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 -# bind cpu -export SGLANG_SET_CPU_AFFINITY=1 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -# cann -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh - -export STREAMS_PER_DEVICE=32 -export HCCL_BUFFSIZE=1000 -export HCCL_OP_EXPANSION_MODE=AIV -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo - -python3 -m sglang.launch_server \ - --model-path $MODEL_PATH \ - --attention-backend ascend \ - --device npu \ - --tp-size 8 --nnodes 1 --node-rank 0 \ - --chunked-prefill-size 4096 --max-prefill-tokens 280000 \ - --disable-radix-cache \ - --trust-remote-code \ - --host 127.0.0.1 \ - --mem-fraction-static 0.7 \ - --port 8000 \ - --cuda-graph-bs 16 \ - --quantization modelslim \ - --enable-multimodal \ - --mm-attention-backend ascend_attn \ - --dtype bfloat16 -``` - -#### Qwen3.5 35B - -```shell -# high performance cpu -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 -# bind cpu -export SGLANG_SET_CPU_AFFINITY=1 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -# cann -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh - -export STREAMS_PER_DEVICE=32 -export HCCL_BUFFSIZE=1000 -export HCCL_OP_EXPANSION_MODE=AIV -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo - -python3 -m sglang.launch_server \ - --model-path $MODEL_PATH \ - --attention-backend ascend \ - --device npu \ - --tp-size 2 --nnodes 1 --node-rank 0 \ - --chunked-prefill-size 4096 --max-prefill-tokens 280000 \ - --disable-radix-cache \ - --trust-remote-code \ - --host 127.0.0.1 \ - --mem-fraction-static 0.7 \ - --port 8000 \ - --cuda-graph-bs 16 \ - --quantization modelslim \ - --enable-multimodal \ - --mm-attention-backend ascend_attn \ - --dtype bfloat16 -``` - -#### Qwen3.5 27B - -```shell -# high performance cpu -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 -# bind cpu -export SGLANG_SET_CPU_AFFINITY=1 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -# cann -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh - -export STREAMS_PER_DEVICE=32 -export HCCL_BUFFSIZE=1000 -export HCCL_OP_EXPANSION_MODE=AIV -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo - -python3 -m sglang.launch_server \ - --model-path $MODEL_PATH \ - --attention-backend ascend \ - --device npu \ - --tp-size 2 \ - --chunked-prefill-size -1 --max-prefill-tokens 120000 \ - --disable-radix-cache \ - --trust-remote-code \ - --host 127.0.0.1 \ - --mem-fraction-static 0.8 \ - --port 8000 \ - --cuda-graph-bs 32 \ - --enable-multimodal \ - --mm-attention-backend ascend_attn -``` - -### Prefill-Decode Disaggregation - -Not test yet. - -### Using Benchmark - -Refer to [Benchmark and Profiling](../../developer_guide/benchmark_and_profiling.md) for details. diff --git a/docs/platforms/ascend/ascend_npu_qwen3_examples.md b/docs/platforms/ascend/ascend_npu_qwen3_examples.md deleted file mode 100644 index 059a2a041..000000000 --- a/docs/platforms/ascend/ascend_npu_qwen3_examples.md +++ /dev/null @@ -1,283 +0,0 @@ -## Qwen3 examples - -### Running Qwen3 - -#### Running Qwen3-32B on 1 x Atlas 800I A3. - -Model weights could be found [here](https://huggingface.co/Qwen/Qwen3-32B) - -```shell -export SGLANG_SET_CPU_AFFINITY=1 -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export STREAMS_PER_DEVICE=32 -export HCCL_BUFFSIZE=1536 -export HCCL_OP_EXPANSION_MODE=AIV - -python -m sglang.launch_server \ - --device npu \ - --attention-backend ascend \ - --trust-remote-code \ - --tp-size 4 \ - --model-path Qwen/Qwen3-32B \ - --mem-fraction-static 0.8 -``` - -#### Running Qwen3-32B on 1 x Atlas 800I A3 with Qwen3-32B-Eagle3. - -Model weights could be found [here](https://huggingface.co/Qwen/Qwen3-32B) - -Speculative model weights could be found [here](https://huggingface.co/Zhihu-ai/Zhi-Create-Qwen3-32B-Eagle3) - -```shell -export SGLANG_SET_CPU_AFFINITY=1 -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export STREAMS_PER_DEVICE=32 -export HCCL_OP_EXPANSION_MODE=AIV -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_ENABLE_SPEC_V2=1 - -python -m sglang.launch_server \ - --device npu \ - --attention-backend ascend \ - --trust-remote-code \ - --tp-size 4 \ - --model-path Qwen/Qwen3-32B \ - --mem-fraction-static 0.8 \ - --speculative-algorithm EAGLE3 \ - --speculative-draft-model-path Qwen/Qwen3-32B-Eagle3 \ - --speculative-num-steps 1 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 2 -``` - -#### Running Qwen3-30B-A3B MOE on 1 x Atlas 800I A3. - -Model weights could be found [here](https://huggingface.co/Qwen/Qwen3-30B-A3B) - -```shell -export SGLANG_SET_CPU_AFFINITY=1 -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export STREAMS_PER_DEVICE=32 -export HCCL_BUFFSIZE=1536 -export HCCL_OP_EXPANSION_MODE=AIV -export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=32 - -python -m sglang.launch_server \ - --device npu \ - --attention-backend ascend \ - --trust-remote-code \ - --tp-size 4 \ - --model-path Qwen/Qwen3-30B-A3B \ - --mem-fraction-static 0.8 -``` - -#### Running Qwen3-235B-A22B-Instruct-2507 MOE on 1 x Atlas 800I A3. - -Model weights could be found [here](https://huggingface.co/Qwen/Qwen3-235B-A22B-Instruct-2507) - -```shell -export SGLANG_SET_CPU_AFFINITY=1 -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export STREAMS_PER_DEVICE=32 -export HCCL_BUFFSIZE=1536 -export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=32 - -python -m sglang.launch_server \ - --model-path Qwen/Qwen3-235B-A22B-Instruct-2507 \ - --tp-size 16 \ - --trust-remote-code \ - --attention-backend ascend \ - --device npu \ - --watchdog-timeout 9000 \ - --mem-fraction-static 0.8 -``` - -#### Running Qwen3-235B-A22B-Instruct-2507 with 256K long sequence on 2 x Atlas 800I A3 without CP - -This example uses **PD disaggregation** for long-sequence inference and keeps **context parallel disabled**. - -Set the shared environment variables on both nodes first: - -```shell -export ASCEND_USE_FIA=1 -export SGLANG_SET_CPU_AFFINITY=1 -export ASCEND_MF_STORE_URL="tcp://:12345" -export HCCL_SOCKET_IFNAME= -export GLOO_SOCKET_IFNAME= - -MODEL_PATH=/root/.cache/modelscope/hub/models/zcgy26/Qwen3-235B-A22B-Instruct-2507-w8a8 -``` - -**Prefill node:** - -```shell -export ASCEND_LAUNCH_BLOCKING=1 -export HCCL_BUFFSIZE=1500 -export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=1024 -export DEEPEP_NORMAL_LONG_SEQ_ROUND=128 -export DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ=1 - -python3 -m sglang.launch_server \ - --model-path ${MODEL_PATH} \ - --disaggregation-mode prefill \ - --disaggregation-transfer-backend ascend \ - --disaggregation-bootstrap-port 8995 \ - --attention-backend ascend \ - --disable-radix-cache \ - --quantization modelslim \ - --chunked-prefill-size -1 \ - --skip-server-warmup \ - --device npu \ - --tp-size 16 \ - --mem-fraction-static 0.45 \ - --max-running-requests 1 \ - --host \ - --port 8000 \ - --dist-init-addr :5000 \ - --nnodes 1 \ - --node-rank 0 \ - --moe-a2a-backend deepep \ - --deepep-mode normal -``` - -**Decode node:** - -```shell -export HCCL_BUFFSIZE=4000 -export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=4096 -export DEEPEP_NORMAL_LONG_SEQ_ROUND=16 - -python3 -m sglang.launch_server \ - --model-path ${MODEL_PATH} \ - --disaggregation-mode decode \ - --disaggregation-transfer-backend ascend \ - --attention-backend ascend \ - --mem-fraction-static 0.8 \ - --disable-cuda-graph \ - --device npu \ - --disable-radix-cache \ - --quantization modelslim \ - --chunked-prefill-size 8192 \ - --skip-server-warmup \ - --tp-size 16 \ - --max-running-requests 1 \ - --host \ - --port 8232 \ - --moe-a2a-backend deepep \ - --deepep-mode low_latency \ - --disable-overlap-schedule -``` - -**Router:** - -```shell -python3 -m sglang_router.launch_router \ - --pd-disaggregation \ - --policy cache_aware \ - --prefill http://:8000 8995 \ - --decode http://:8232 \ - --host \ - --port 6689 \ - --prometheus-port 29010 -``` - -#### Running Qwen3-235B-A22B-Instruct-2507-W8A8 with Prefill Context Parallel (CP) on 2 x Atlas 800I A3 - -This example enables **Prefill Context Parallel** (`--enable-prefill-context-parallel`) to split the context across CP ranks during prefill, reducing per-device memory pressure and improving TTFT for long sequences. PD disaggregation is required. - -> **Constraints** -> - Prefill side must set `--max-running-requests 1` (PCP only supports batch_size=1) -> - `--attn-cp-size` must evenly divide `--tp-size`; each CP rank occupies `tp_size / cp_size` NPUs - -**Prefill node :** - -```shell -export SGLANG_SET_CPU_AFFINITY=1 -export ASCEND_MF_STORE_URL="tcp://:23456" -export ASCEND_USE_FIA=True - -python3 -m sglang.launch_server \ - --model-path /mnt/share/weights/Qwen3-235B-A22B-Instruct-2507-W8A8 \ - --trust-remote-code \ - --disaggregation-mode prefill \ - --disaggregation-transfer-backend ascend \ - --disaggregation-bootstrap-port 8995 \ - --quantization modelslim \ - --attention-backend ascend \ - --skip-server-warmup \ - --mem-fraction-static 0.7 \ - --chunked-prefill-size 32768 \ - --device npu \ - --base-gpu-id 0 \ - --tp-size 16 \ - --enable-prefill-context-parallel \ - --attn-cp-size 2 \ - --moe-dp-size 2 \ - --max-running-requests 1 \ - --host \ - --port 8000 \ - --nnodes 1 \ - --node-rank 0 \ - --dist-init-addr :6688 -``` - -Key parameters for PCP: - -| Parameter | Value | Description | -|-----------|-------|-------------| -| `--enable-prefill-context-parallel` | flag | Enable PCP feature | -| `--attn-cp-size` | 2 | Split context across 2 CP ranks (each rank handles half the sequence) | -| `--moe-dp-size` | 2 | MoE DP size, should match `--attn-cp-size` | -| `--max-running-requests` | 1 | Required by PCP (batch_size=1 constraint) | - -**Decode node ():** - -```shell -export ASCEND_MF_STORE_URL="tcp://141.61.39.231:23456" -export ASCEND_USE_FIA=True - -python3 -m sglang.launch_server \ - --model-path /mnt/share/weights/Qwen3-235B-A22B-Instruct-2507-W8A8 \ - --trust-remote-code \ - --disaggregation-mode decode \ - --disaggregation-transfer-backend ascend \ - --quantization modelslim \ - --attention-backend ascend \ - --disable-radix-cache \ - --disable-cuda-graph \ - --mem-fraction-static 0.7 \ - --chunked-prefill-size 32768 \ - --skip-server-warmup \ - --device npu \ - --base-gpu-id 0 \ - --tp-size 8 \ - --max-running-requests 32 \ - --host \ - --port 8001 \ - --nnodes 1 \ - --node-rank 0 \ - --dist-init-addr :6688 -``` - -> **Note:** `ASCEND_MF_STORE_URL` on both nodes must point to the same KV store (typically the Prefill node IP). `ASCEND_USE_FIA=True` enables fast interconnect aggregation for KV transfer. PCP is a Prefill-only feature; the Decode side needs no CP-related flags. - -#### Running Qwen3-VL-8B-Instruct on 1 x Atlas 800I A3. - -Model weights could be found [here](https://huggingface.co/Qwen/Qwen3-VL-8B-Instruct) - -```shell -export SGLANG_SET_CPU_AFFINITY=1 -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export STREAMS_PER_DEVICE=32 -export HCCL_BUFFSIZE=1536 -export HCCL_OP_EXPANSION_MODE=AIV - -python -m sglang.launch_server \ - --enable-multimodal \ - --attention-backend ascend \ - --mm-attention-backend ascend_attn \ - --trust-remote-code \ - --tp-size 4 \ - --model-path Qwen/Qwen3-VL-8B-Instruct \ - --mem-fraction-static 0.8 -``` diff --git a/docs/platforms/ascend/ascend_npu_support.rst b/docs/platforms/ascend/ascend_npu_support.rst deleted file mode 100644 index 1c0bbc276..000000000 --- a/docs/platforms/ascend/ascend_npu_support.rst +++ /dev/null @@ -1,20 +0,0 @@ -Ascend NPUs -=============================================================== - -.. toctree:: - :maxdepth: 1 - - ascend_npu_quick_start.md - ascend_npu.md - ascend_npu_support_features.md - ascend_npu_support_models.md - ascend_npu_quantization.md - ascend_npu_deepseek_example.md - ascend_npu_qwen3_examples.md - mindspore_backend.md - ascend_contribution_guide.md - ascend_npu_best_practice.md - ascend_npu_ring_sp_performance.md - ascend_npu_qwen3_5_examples.md - ascend_npu_glm5_examples.md - ascend_npu_environment_variables.md diff --git a/docs/platforms/ascend/ascend_npu_support_features.md b/docs/platforms/ascend/ascend_npu_support_features.md deleted file mode 100644 index a28c51add..000000000 --- a/docs/platforms/ascend/ascend_npu_support_features.md +++ /dev/null @@ -1,483 +0,0 @@ -# Support Features on Ascend NPU - -This section describes the basic functions and features supported by the Ascend NPU.If you encounter issues or have any -questions, please [open an issue](https://github.com/sgl-project/sglang/issues). - -If you want to know the meaning and usage of each parameter, -click [Server Arguments](https://docs.sglang.io/advanced_features/server_arguments.html). - -## Model and tokenizer - -| Argument | Defaults | Options | Server supported | -|----------------------------------------|----------|---------------------------------------|:----------------:| -| `--model-path`
`--model` | `None` | Type: str | A2, A3 | -| `--tokenizer-path` | `None` | Type: str | A2, A3 | -| `--tokenizer-mode` | `auto` | `auto`, `slow` | A2, A3 | -| `--tokenizer-worker-num` | `1` | Type: int | A2, A3 | -| `--skip-tokenizer-init` | `False` | bool flag (set to enable) | A2, A3 | -| `--load-format` | `auto` | `auto`, `safetensors` | A2, A3 | -| `--model-loader-`
`extra-config` | `{}` | Type: str | A2, A3 | -| `--trust-remote-code` | `False` | bool flag (set to enable) | A2, A3 | -| `--context-length` | `None` | Type: int | A2, A3 | -| `--is-embedding` | `False` | bool flag (set to enable) | A2, A3 | -| `--enable-multimodal` | `None` | bool flag (set to enable) | A2, A3 | -| `--revision` | `None` | Type: str | A2, A3 | -| `--model-impl` | `auto` | `auto`, `sglang`,
`transformers` | A2, A3 | - -## HTTP server - -| Argument | Defaults | Options | Server supported | -|------------------------|-------------|---------------------------|:----------------:| -| `--host` | `127.0.0.1` | Type: str | A2, A3 | -| `--port` | `30000` | Type: int | A2, A3 | -| `--skip-server-warmup` | `False` | bool flag (set to enable) | A2, A3 | -| `--warmups` | `None` | Type: str | A2, A3 | -| `--nccl-port` | `None` | Type: int | A2, A3 | -| `--fastapi-root-path` | `None` | Type: str | A2, A3 | -| `--grpc-mode` | `False` | `False` | Planned | - -## Quantization and data type - -| Argument | Defaults | Options | Server supported | -|---------------------------------------------|----------|-----------------------------------------|:----------------:| -| `--dtype` | `auto` | `auto`,
`float16`,
`bfloat16` | A2, A3 | -| `--quantization` | `None` | `modelslim` | A2, A3 | -| `--quantization-param-path` | `None` | Type: str | Special For GPU | -| `--kv-cache-dtype` | `auto` | `auto` | A2, A3 | -| `--enable-fp32-lm-head` | `False` | bool flag
(set to enable) | A2, A3 | -| `--modelopt-quant` | `None` | Type: str | Special For GPU | -| `--modelopt-checkpoint-`
`restore-path` | `None` | Type: str | Special For GPU | -| `--modelopt-checkpoint-`
`save-path` | `None` | Type: str | Special For GPU | -| `--modelopt-export-path` | `None` | Type: str | Special For GPU | -| `--quantize-and-serve` | `False` | bool flag
(set to enable) | Special For GPU | -| `--rl-quant-profile` | `None` | Type: str | Special For GPU | - -## Memory and scheduling - -| Argument | Defaults | Options | Server supported | -|-----------------------------------------------------|----------|--------------------------------|:----------------:| -| `--mem-fraction-static` | `None` | Type: float | A2, A3 | -| `--max-running-requests` | `None` | Type: int | A2, A3 | -| `--prefill-max-requests` | `None` | Type: int | A2, A3 | -| `--max-queued-requests` | `None` | Type: int | A2, A3 | -| `--max-total-tokens` | `None` | Type: int | A2, A3 | -| `--chunked-prefill-size` | `None` | Type: int | A2, A3 | -| `--max-prefill-tokens` | `16384` | Type: int | A2, A3 | -| `--schedule-policy` | `fcfs` | `lpm`, `fcfs` | A2, A3 | -| `--enable-priority-`
`scheduling` | `False` | bool flag
(set to enable) | A2, A3 | -| `--schedule-low-priority-`
`values-first` | `False` | bool flag
(set to enable) | A2, A3 | -| `--priority-scheduling-`
`preemption-threshold` | `10` | Type: int | A2, A3 | -| `--schedule-conservativeness` | `1.0` | Type: float | A2, A3 | -| `--page-size` | `128` | Type: int | A2, A3 | -| `--swa-full-tokens-ratio` | `0.8` | Type: float | Planned | -| `--disable-hybrid-swa-memory` | `False` | bool flag
(set to enable) | Planned | -| `--radix-eviction-policy` | `lru` | `lru`,
`lfu` | A2, A3 | -| `--enable-prefill-delayer` | `False` | bool flag
(set to enable) | A2, A3 | -| `--prefill-delayer-max-delay-passes` | `30` | Type: int | A2, A3 | -| `--prefill-delayer-token-usage-low-watermark` | `None` | Type: float | A2, A3 | -| `--prefill-delayer-forward-passes-buckets` | `None` | List[float] | A2, A3 | -| `--prefill-delayer-wait-seconds-buckets` | `None` | List[float] | A2, A3 | -| `--abort-on-priority-`
`when-disabled` | `False` | bool flag
(set to enable) | A2, A3 | -| `--enable-dynamic-chunking` | `False` | bool flag
(set to enable) | Experimental | - -## Runtime options - -| Argument | Defaults | Options | Server supported | -|----------------------------------------------------------|----------|----------------------------------------|:----------------:| -| `--device` | `None` | Type: str | A2, A3 | -| `--tensor-parallel-size`
`--tp-size` | `1` | Type: int | A2, A3 | -| `--pipeline-parallel-size`
`--pp-size` | `1` | Type: int; Currently `2` not supported | Experimental | -| `--attention-context-parallel-size`
`--attn-cp-size` | `1` | Type: int; must be equal to --tp-size | A2, A3 | -| `--moe-data-parallel-size`
`--moe-dp-size` | `1` | Type: int | Planned | -| `--pp-max-micro-batch-size` | `None` | Type: int | Experimental | -| `--pp-async-batch-depth` | `None` | Type: int | Experimental | -| `--stream-interval` | `1` | Type: int | A2, A3 | -| `--incremental-streaming-output` | `False` | bool flag (set to enable) | A2, A3 | -| `--random-seed` | `None` | Type: int | A2, A3 | -| `--constrained-json-`
`whitespace-pattern` | `None` | Type: str | A2, A3 | -| `--constrained-json-`
`disable-any-whitespace` | `False` | bool flag (set to enable) | A2, A3 | -| `--watchdog-timeout` | `300` | Type: float | A2, A3 | -| `--soft-watchdog-timeout` | `300` | Type: float | A2, A3 | -| `--dist-timeout` | `None` | Type: int | A2, A3 | -| `--download-dir` | `None` | Type: str | A2, A3 | -| `--model-checksum` | `None` | Type: str | Planned | -| `--base-gpu-id` | `0` | Type: int | A2, A3 | -| `--gpu-id-step` | `1` | Type: int | A2, A3 | -| `--sleep-on-idle` | `False` | bool flag (set to enable) | A2, A3 | - -## Logging - -| Argument | Defaults | Options | Server supported | -|----------------------------------------------------|-------------------|--------------------------------|:----------------:| -| `--log-level` | `info` | Type: str | A2, A3 | -| `--log-level-http` | `None` | Type: str | A2, A3 | -| `--log-requests` | `False` | bool flag
(set to enable) | A2, A3 | -| `--log-requests-level` | `2` | `0`, `1`, `2`, `3` | A2, A3 | -| `--log-requests-format` | `text` | `text`, `json` | A2, A3 | -| `--crash-dump-folder` | `None` | Type: str | A2, A3 | -| `--enable-metrics` | `False` | bool flag
(set to enable) | A2, A3 | -| `--enable-metrics-for-`
`all-schedulers` | `False` | bool flag
(set to enable) | A2, A3 | -| `--tokenizer-metrics-`
`custom-labels-header` | `x-custom-labels` | Type: str | A2, A3 | -| `--tokenizer-metrics-`
`allowed-custom-labels` | `None` | List[str] | A2, A3 | -| `--bucket-time-to-`
`first-token` | `None` | List[float] | A2, A3 | -| `--bucket-inter-token-`
`latency` | `None` | List[float] | A2, A3 | -| `--bucket-e2e-request-`
`latency` | `None` | List[float] | A2, A3 | -| `--collect-tokens-`
`histogram` | `False` | bool flag
(set to enable) | A2, A3 | -| `--prompt-tokens-buckets` | `None` | List[str] | A2, A3 | -| `--generation-tokens-buckets` | `None` | List[str] | A2, A3 | -| `--gc-warning-threshold-secs` | `0.0` | Type: float | A2, A3 | -| `--decode-log-interval` | `40` | Type: int | A2, A3 | -| `--enable-request-time-`
`stats-logging` | `False` | bool flag
(set to enable) | A2, A3 | -| `--kv-events-config` | `None` | Type: str | Special for GPU | -| `--enable-trace` | `False` | bool flag
(set to enable) | A2, A3 | -| `--oltp-traces-endpoint` | `localhost:4317` | Type: str | A2, A3 | -| `--log-requests-target` | `None` | Type: str | A2, A3 | -| `--uvicorn-access-log-exclude-prefixes` | `[]` | List[str] | A2, A3 | - -## RequestMetricsExporter configuration - -| Argument | Defaults | Options | Server supported | -|---------------------------------------|----------|--------------------------------|:----------------:| -| `--export-metrics-to-`
`file` | `False` | bool flag
(set to enable) | A2, A3 | -| `--export-metrics-to-`
`file-dir` | `None` | Type: str | A2, A3 | - -## API related - -| Argument | Defaults | Options | Server supported | -|---------------------------|-----------|-------------------------------------------------------------------------------------------------------------------|:----------------:| -| `--api-key` | `None` | Type: str | A2, A3 | -| `--admin-api-key` | `None` | Type: str | A2, A3 | -| `--served-model-name` | `None` | Type: str | A2, A3 | -| `--weight-version` | `default` | Type: str | A2, A3 | -| `--chat-template` | `None` | Type: str | A2, A3 | -| `--hf-chat-template-name` | `None` | Type: str | A2, A3 | -| `--completion-template` | `None` | Type: str | A2, A3 | -| `--enable-cache-report` | `False` | bool flag
(set to enable) | A2, A3 | -| `--reasoning-parser` | `None` | `deepseek-r1`
`deepseek-v3`
`glm45`
`gpt-oss`
`kimi`
`qwen3`
`qwen3-thinking`
`step3` | A2, A3 | -| `--tool-call-parser` | `None` | `llama3`
`pythonic`
`qwen`
`qwen3_coder` | A2, A3 | -| `--sampling-defaults` | `model` | `openai`, `model` | A2, A3 | - -## Data parallelism - -| Argument | Defaults | Options | Server supported | -|----------------------------------------|---------------|-----------------------------------------------------------|:----------------:| -| `--data-parallel-size`
`--dp-size` | `1` | Type: int | A2, A3 | -| `--load-balance-method` | `auto` | `auto`,
`round_robin`,
`follow_bootstrap_room`,
`total_requests`,
`total_tokens` | A2, A3 | - -## Multi-node distributed serving - -| Argument | Defaults | Options | Server supported | -|-------------------------------------------|----------|-----------|:----------------:| -| `--dist-init-addr`
`--nccl-init-addr` | `None` | Type: str | A2, A3 | -| `--nnodes` | `1` | Type: int | A2, A3 | -| `--node-rank` | `0` | Type: int | A2, A3 | - -## Model override args - -| Argument | Defaults | Options | Server supported | -|--------------------------------------|----------|-----------|:----------------:| -| `--json-model-override-`
`args` | `{}` | Type: str | A2, A3 | -| `--preferred-sampling-`
`params` | `None` | Type: str | A2, A3 | - -## LoRA - -| Argument | Defaults | Options | Server supported | -|--------------------------|----------|-------------------------------------|:----------------:| -| `--enable-lora` | `False` | Bool flag
(set to enable) | A2, A3 | -| `--enable-lora-overlap-loading` | `False` | Bool flag
(set to enable) | A2, A3 | -| `--max-lora-rank` | `None` | Type: int | A2, A3 | -| `--lora-target-modules` | `None` | `all` | A2, A3 | -| `--lora-paths` | `None` | Type: List[str] /
JSON objects | A2, A3 | -| `--max-loras-per-batch` | `8` | Type: int | A2, A3 | -| `--max-loaded-loras` | `None` | Type: int | A2, A3 | -| `--lora-eviction-policy` | `lru` | `lru`,
`fifo` | A2, A3 | -| `--lora-backend` | `csgmv` | `triton`,
`csgmv`,
`ascend`,
`torch_native` | A2, A3 | -| `--max-lora-chunk-size` | `16` | `16`, `32`,
`64`, `128` | Special for GPU | - -## Kernel Backends (Attention, Sampling, Grammar, GEMM) - -| Argument | Defaults | Options | Server supported | -|----------------------------------------|-------------------|------------------------------------------------------------------------------------------------|:----------------:| -| `--attention-backend` | `None` | `ascend` | A2, A3 | -| `--prefill-attention-backend` | `None` | `ascend` | A2, A3 | -| `--decode-attention-backend` | `None` | `ascend` | A2, A3 | -| `--sampling-backend` | `None` | `pytorch`,
`ascend` | A2, A3 | -| `--grammar-backend` | `None` | `xgrammar` | A2, A3 | -| `--mm-attention-backend` | `None` | `ascend_attn` | A2, A3 | -| `--dsa-prefill-backend` | `flashmla_sparse` | `flashmla_sparse`,
`flashmla_decode`,
`fa3`,
`tilelang`,
`aiter` | Special for GPU | -| `--dsa-decode-backend` | `fa3` | `flashmla_prefill`,
`flashmla_kv`,
`fa3`,
`tilelang`,
`aiter` | Special for GPU | -| `--fp8-gemm-backend` | `auto` | `auto`,
`deep_gemm`,
`flashinfer_trtllm`,
`flashinfer_cutlass`,
`flashinfer_deepgemm`,
`cutlass`,
`triton`,
`aiter` | Special for GPU | -| `--disable-flashinfer-`
`autotune` | `False` | bool flag
(set to enable) | Special for GPU | - -## Speculative decoding - -| Argument | Defaults | Options | Server supported | -|------------------------------------------------------------------|-----------|------------------------------------------------------------------|:----------------:| -| `--speculative-algorithm` | `None` | `EAGLE3`,
`NEXTN` | A2, A3 | -| `--speculative-draft-model-path`
`--speculative-draft-model` | `None` | Type: str | A2, A3 | -| `--speculative-draft-model-`
`revision` | `None` | Type: str,
`branch name`,
`tag name`,
`commit id` | A2, A3 | -| `--speculative-draft-load-format` | `auto` | `auto`,
`dummy` | A2, A3 | -| `--speculative-num-steps` | `None` | Type: int | A2, A3 | -| `--speculative-eagle-topk` | `None` | Type: int | A2, A3 | -| `--speculative-num-draft-tokens` | `None` | Type: int | A2, A3 | -| `--speculative-accept-`
`threshold-single` | `1.0` | Type: float | Special for GPU | -| `--speculative-accept-`
`threshold-acc` | `1.0` | Type: float | Special for GPU | -| `--speculative-token-map` | `None` | Type: str | A2, A3 | -| `--speculative-attention-`
`mode` | `prefill` | `prefill`,
`decode` | A2, A3 | -| `--speculative-moe-runner-`
`backend` | `None` | `auto` | A2, A3 | -| `--speculative-moe-a2a-`
`backend` | `None` | `ascend_fuseep` | A2, A3 | -| `--speculative-draft-attention-backend` | `None` | `ascend` | A2, A3 | -| `--speculative-draft-model-quantization` | `None` | `unquant` | A2, A3 | - -## Ngram speculative decoding - -| Argument | Defaults | Options | Server supported | -|----------------------------------------------------|------------|--------------------|:----------------:| -| `--speculative-ngram-`
`min-match-window-size` | `1` | Type: int | Experimental | -| `--speculative-ngram-`
`max-match-window-size` | `12` | Type: int | Experimental | -| `--speculative-ngram-`
`min-bfs-breadth` | `1` | Type: int | Experimental | -| `--speculative-ngram-`
`max-bfs-breadth` | `10` | Type: int | Experimental | -| `--speculative-ngram-`
`match-type` | `BFS` | `BFS`,
`PROB` | Experimental. `BFS` uses recency-based expansion; `PROB` uses frequency-based expansion. | -| `--speculative-ngram-`
`max-trie-depth` | `18` | Type: int | Experimental | -| `--speculative-ngram-`
`capacity` | `10000000` | Type: int | Experimental | - -## Expert parallelism - -| Argument | Defaults | Options | Server supported | -|-------------------------------------------------------|-----------|---------------------------------------------------------------------------|:----------------:| -| `--expert-parallel-size`
`--ep-size`
`--ep` | `1` | Type: int | A2, A3 | -| `--moe-a2a-backend` | `none` | `none`,
`deepep`,
`ascend_fuseep`(It is incompatible with eplb) | A2, A3 | -| `--moe-runner-backend` | `auto` | `auto`, `triton` | A2, A3 | -| `--flashinfer-mxfp4-`
`moe-precision` | `default` | `default`,
`bf16` | Special for GPU | -| `--enable-flashinfer-`
`allreduce-fusion` | `False` | bool flag
(set to enable) | Special for GPU | -| `--deepep-mode` | `auto` | `normal`,
`low_latency`,
`auto` | A2, A3 | -| `--deepep-config` | `None` | Type: str | Special for GPU | -| `--ep-num-redundant-experts` | `0` | Type: int | A2, A3 | -| `--ep-dispatch-algorithm` | `None` | `static`,
`dynamic`,
`fake` | A2, A3 | -| `--init-expert-location` | `trivial` | `trivial`,
``,
``,
`` | A2, A3 | -| `--enable-eplb` | `False` | bool flag
(set to enable) | A2, A3 | -| `--eplb-algorithm` | `deepseek`| `auto`,
`deepseek` | A2, A3 | -| `--eplb-rebalance-num-iterations` | `1000` | Type: int | A2, A3 | -| `--eplb-rebalance-layers-`
`per-chunk` | `None` | Type: int | A2, A3 | -| `--eplb-min-rebalancing-`
`utilization-threshold` | `1.0` | Type: float | A2, A3 | -| `--expert-distribution-`
`recorder-mode` | `None` | `stat`,
`stat_approx`,
`per_pass`,
`per_token` | A2, A3 | -| `--expert-distribution-`
`recorder-buffer-size` | `None` | Type: int | A2, A3 | -| `--enable-expert-distribution-`
`metrics` | `False` | bool flag (set to enable) | A2, A3 | -| `--moe-dense-tp-size` | `None` | `1` | A2, A3 | -| `--elastic-ep-backend` | `None` | `none`, `mooncake` | Special for GPU | -| `--mooncake-ib-device` | `None` | Type: str | Special for GPU | - -## Mamba Cache - -| Argument | Defaults | Options | Server supported | -|------------------------------|-----------|-----------------------------------------------|:----------------:| -| `--max-mamba-cache-size` | `None` | Type: int | A2, A3 | -| `--mamba-ssm-dtype` | `float32` | `float32`,
`bfloat16`,
`float16` | A2, A3 | -| `--mamba-full-memory-ratio` | `0.9` | Type: float | A2, A3 | -| `--mamba-scheduler-strategy` | `auto` | `auto`,
`no_buffer`,
`extra_buffer` | A2, A3 | -| `--mamba-track-interval` | `256` | Type: int | A2, A3 | - -## Hierarchical cache - -| Argument | Defaults | Options | Server supported | -|-------------------------------------------------|-----------------|-------------------------------------------------------------------------------|:----------------:| -| `--enable-hierarchical-`
`cache` | `False` | bool flag
(set to enable).
Currently, mamba cache is not supported. | A2, A3 | -| `--hicache-ratio` | `2.0` | Type: float | A2, A3 | -| `--hicache-size` | `0` | Type: int | A2, A3 | -| `--hicache-write-policy` | `write_through` | Currently only `write_back` supported | A2, A3 | -| `--hicache-io-backend` | `kernel` | `kernel_ascend`,
`direct` | A2, A3 | -| `--hicache-mem-layout` | `layer_first` | `page_first_direct`,
`page_first_kv_split` | A2, A3 | -| `--hicache-storage-`
`backend` | `None` | `file` | A2, A3 | -| `--hicache-storage-`
`prefetch-policy` | `best_effort` | `best_effort`,
`wait_complete`,
`timeout` | Special for GPU | -| `--hicache-storage-`
`backend-extra-config` | `None` | Type: str | Special for GPU | - -## LMCache - -| Argument | Defaults | Options | Server supported | -|--------------------|----------|--------------------------------|:----------------:| -| `--enable-lmcache` | `False` | bool flag
(set to enable) | Special for GPU | - -## Offloading (must be used with `--disable-cuda-graph`) - -| Argument | Defaults | Options | Server supported | -|---------------------------|----------|-----------|:----------------:| -| `--cpu-offload-gb` | `0` | Type: int | A2, A3 | -| `--offload-group-size` | `-1` | Type: int (DeepSeek only) | A2, A3 | -| `--offload-num-in-group` | `1` | Type: int (DeepSeek only) | A2, A3 | -| `--offload-prefetch-step` | `1` | Type: int (DeepSeek only) | A2, A3 | -| `--offload-mode` | `cpu` | `cpu` (DeepSeek only)
`meta` (DeepSeek only)
`sharded_gpu` (DeepSeek only, only support tp=1 dp>1) | A2, A3 | - -## Args for multi-item scoring - -| Argument | Defaults | Options | Server supported | -|----------------------------------|----------|-----------|:----------------:| -| `--multi-item-scoring-delimiter` | `None` | Type: int | A2, A3 | - -## Optimization/debug options - -| Argument | Defaults | Options | Server supported | -|---------------------------------------------------------|----------|----------------------------------------------------------------------------------------------------------------------|:----------------:| -| `--disable-radix-cache` | `False` | bool flag
(set to enable) | A2, A3 | -| `--cuda-graph-max-bs` | `None` | Type: int | A2, A3 | -| `--cuda-graph-bs` | `None` | List[int] | A2, A3 | -| `--disable-cuda-graph` | `False` | bool flag
(set to enable) | A2, A3 | -| `--disable-cuda-graph-`
`padding` | `False` | bool flag
(set to enable) | A2, A3 | -| `--enable-profile-`
`cuda-graph` | `False` | bool flag
(set to enable) | A2, A3 | -| `--enable-cudagraph-gc` | `False` | bool flag
(set to enable) | A2, A3 | -| `--enable-nccl-nvls` | `False` | bool flag
(set to enable) | Special for GPU | -| `--enable-symm-mem` | `False` | bool flag
(set to enable) | Special for GPU | -| `--disable-flashinfer-`
`cutlass-moe-fp4-allgather` | `False` | bool flag
(set to enable) | Special for GPU | -| `--enable-tokenizer-`
`batch-encode` | `False` | bool flag
(set to enable) | A2, A3 | -| `--disable-tokenizer-`
`batch-decode` | `False` | bool flag
(set to enable) | A2, A3 | -| `--disable-custom-`
`all-reduce` | `False` | bool flag
(set to enable) | Special for GPU | -| `--enable-mscclpp` | `False` | bool flag
(set to enable) | Special for GPU | -| `--enable-torch-`
`symm-mem` | `False` | bool flag
(set to enable) | Special for GPU | -| `--disable-overlap`
`-schedule` | `False` | bool flag
(set to enable) | A2, A3 | -| `--enable-mixed-`
`chunk` | `False` | bool flag
(set to enable) | A2, A3 | -| `--enable-dp-attention` | `False` | bool flag
(set to enable) | A2, A3 | -| `--enable-dp-lm-head` | `False` | bool flag
(set to enable) | A2, A3 | -| `--enable-two-`
`batch-overlap` | `False` | bool flag
(set to enable) | Planned | -| `--enable-single-`
`batch-overlap` | `False` | bool flag
(set to enable) | A2, A3 | -| `--tbo-token-`
`distribution-threshold` | `0.48` | Type: float | Planned | -| `--enable-torch-`
`compile` | `False` | bool flag
(set to enable) | A2, A3 | -| `--enable-torch-`
`compile-debug-mode` | `False` | bool flag
(set to enable) | A2, A3 | -| `--enforce-piecewise-`
`cuda-graph` | `False` | bool flag
(set to enable);
Currently, Llama-3.1-8B-Instruct and Qwen2.5-7B-Instruct models are supported. | A2, A3 | -| `--piecewise-cuda-`
`graph-tokens` | `None` | Type: JSON
list | A2, A3 | -| `--piecewise-cuda-`
`graph-compiler` | `eager` | `eager` | A2, A3 | -| `--torch-compile-max-bs` | `32` | Type: int | A2, A3 | -| `--piecewise-cuda-`
`graph-max-tokens` | `None` | Type: int | A2, A3 | -| `--torchao-config` | `` | Type: str | Special for GPU | -| `--enable-nan-detection` | `False` | bool flag
(set to enable) | A2, A3 | -| `--enable-p2p-check` | `False` | bool flag
(set to enable) | Special for GPU | -| `--triton-attention-`
`reduce-in-fp32` | `False` | bool flag
(set to enable) | Special for GPU | -| `--triton-attention-`
`num-kv-splits` | `8` | Type: int | Special for GPU | -| `--triton-attention-`
`split-tile-size` | `None` | Type: int | Special for GPU | -| `--delete-ckpt-`
`after-loading` | `False` | bool flag
(set to enable) | A2, A3 | -| `--enable-memory-saver` | `False` | bool flag
(set to enable) | A2, A3 | -| `--enable-weights-`
`cpu-backup` | `False` | bool flag
(set to enable) | A2, A3 | -| `--enable-draft-weights-`
`cpu-backup` | `False` | bool flag
(set to enable) | A2, A3 | -| `--allow-auto-truncate` | `False` | bool flag
(set to enable) | A2, A3 | -| `--enable-custom-`
`logit-processor` | `False` | bool flag
(set to enable) | A2, A3 | -| `--flashinfer-mla-`
`disable-ragged` | `False` | bool flag
(set to enable) | Special for GPU | -| `--disable-shared-`
`experts-fusion` | `True` | bool flag
(set to enable) | A2, A3 | -| `--disable-chunked-`
`prefix-cache` | `True` | bool flag
(set to enable) | A2, A3 | -| `--disable-fast-`
`image-processor` | `False` | bool flag
(set to enable) | A2, A3 | -| `--keep-mm-feature-`
`on-device` | `False` | bool flag
(set to enable) | A2, A3 | -| `--enable-return-`
`hidden-states` | `False` | bool flag
(set to enable) | A2, A3 | -| `--enable-return-`
`routed-experts` | `False` | bool flag
(set to enable) | A2, A3 | -| `--scheduler-recv-`
`interval` | `1` | Type: int | A2, A3 | -| `--numa-node` | `None` | List[int] | A2, A3 | -| `--enable-deterministic-`
`inference` | `False` | bool flag
(set to enable) | Planned | -| `--rl-on-policy-target` | `None` | `fsdp` | Planned | -| `--enable-layerwise-`
`nvtx-marker` | `False` | bool flag
(set to enable) | Special for GPU | -| `--enable-attn-tp-`
`input-scattered` | `False` | bool flag
(set to enable) | Experimental | -| `--enable-dsa-prefill-`
`context-parallel` | `False` | bool flag
(set to enable) | A2, A3 | -| `--enable-fused-qk-`
`norm-rope` | `False` | bool flag
(set to enable) | Special for GPU | - -## Dynamic batch tokenizer - -| Argument | Defaults | Options | Server supported | -|--------------------------------------------------|----------|--------------------------------|:----------------:| -| `--enable-dynamic-`
`batch-tokenizer` | `False` | bool flag
(set to enable) | A2, A3 | -| `--dynamic-batch-`
`tokenizer-batch-size` | `32` | Type: int | A2, A3 | -| `--dynamic-batch-`
`tokenizer-batch-timeout` | `0.002` | Type: float | A2, A3 | - -## Debug tensor dumps - -| Argument | Defaults | Options | Server supported | -|--------------------------------------------|----------|-----------|:----------------:| -| `--debug-tensor-dump-`
`output-folder` | `None` | Type: str | A2, A3 | -| `--debug-tensor-dump-`
`layers` | `None` | List[int] | A2, A3 | -| `--debug-tensor-dump-`
`input-file` | `None` | Type: str | A2, A3 | - -## PD disaggregation - -| Argument | Defaults | Options | Server supported | -|---------------------------------------------------------|------------|---------------------------------------|:----------------:| -| `--disaggregation-mode` | `null` | `null`,
`prefill`,
`decode` | A2, A3 | -| `--disaggregation-transfer-backend` | `mooncake` | `ascend` | A2, A3 | -| `--disaggregation-bootstrap-port` | `8998` | Type: int | A2, A3 | -| `--disaggregation-ib-device` | `None` | Type: str | Special for GPU | -| `--disaggregation-decode-`
`enable-offload-kvcache` | `False` | `False` | A2, A3 | -| `--num-reserved-decode-tokens` | `512` | Type: int | A2, A3 | -| `--disaggregation-decode-`
`polling-interval` | `1` | Type: int | A2, A3 | - -## Encode prefill disaggregation - -| Argument | Defaults | Options | Server supported | -| --------------------------------------- | ------------------ | ---------------------------------------------------------------- |:----------------:| -| `--enable-adaptive-dispatch-to-encoder` | `False` | bool flag
(set to enable adaptively dispatch) | A2, A3 | -| `--encoder-only` | `False` | bool flag
(set to launch an encoder-only server) | A2, A3 | -| `--language-only` | `False` | bool flag
(set to load weights for the language model only) | A2, A3 | -| `--encoder-transfer-backend` | `zmq_to_scheduler` | `zmq_to_scheduler`,
`zmq_to_tokenizer`,
`mooncake` | A2, A3 | -| `--encoder-urls` | `[]` | List[str]
(List of encoder server urls) | A2, A3 | - -## Custom weight loader - -| Argument | Defaults | Options | Server supported | -|-------------------------------------------------------------------------|----------|---------------------------------|:----------------:| -| `--custom-weight-loader` | `None` | List[str] | A2, A3 | -| `--weight-loader-disable-`
`mmap` | `False` | bool flag
(set to enable) | A2, A3 | -| `--remote-instance-weight-`
`loader-seed-instance-ip` | `None` | Type: str | A2, A3 | -| `--remote-instance-weight-`
`loader-seed-instance-service-port` | `None` | Type: int | A2, A3 | -| `--remote-instance-weight-`
`loader-send-weights-group-ports` | `None` | Type: JSON
list | A2, A3 | -| `--remote-instance-weight-`
`loader-backend` | `nccl` | `transfer_engine`,
`nccl` | A2, A3 | -| `--remote-instance-weight-`
`loader-start-seed-via-transfer-engine` | `False` | bool flag
(set to enable) | Special for GPU | - -## For PD-Multiplexing - -| Argument | Defaults | Options | Server supported | -|-----------------------|----------|--------------------------------|:----------------:| -| `--enable-pdmux` | `False` | bool flag
(set to enable) | Special for GPU | -| `--pdmux-config-path` | `None` | Type: str | Special for GPU | -| `--sm-group-num` | `8` | Type: int | Special for GPU | - -## For Multi-Modal - -| Argument | Defaults | Options | Server supported | -|-----------------------------------------------|----------|--------------------------------|:----------------:| -| `--enable-broadcast-mm-`
`inputs-process` | `False` | bool flag
(set to enable) | A2, A3 | -| `--mm-process-config` | `None` | Type: JSON / Dict | A2, A3 | -| `--mm-enable-dp-encoder` | `False` | bool flag
(set to enable) | A2, A3 | -| `--limit-mm-data-per-request` | `None` | Type: JSON / Dict | A2, A3 | - -## For checkpoint decryption - -| Argument | Defaults | Options | Server supported | -|---------------------------------|----------|--------------------------------|:----------------:| -| `--decrypted-config-file` | `None` | Type: str | A2, A3 | -| `--decrypted-draft-config-file` | `None` | Type: str | A2, A3 | -| `--enable-prefix-mm-cache` | `False` | bool flag
(set to enable) | A2, A3 | - -## Forward hooks - -| Argument | Defaults | Options | Server supported | -|-------------------|----------|-----------------|:----------------:| -| `--forward-hooks` | `None` | Type: JSON list | A2, A3 | - -## Configuration file support - -| Argument | Defaults | Options | Server supported | -|------------|----------|-----------|:----------------:| -| `--config` | `None` | Type: str | A2, A3 | - -## Other Params - -The following parameters are not supported because the third-party components that depend on are not compatible with the -NPU, like Ktransformer, checkpoint-engine etc. - -| Argument | Defaults | Options | -|-------------------------------------------------------------------|-----------|---------------------------| -| `--checkpoint-engine-`
`wait-weights-`
`before-ready` | `False` | bool flag (set to enable) | -| `--kt-weight-path` | `None` | Type: str | -| `--kt-method` | `AMXINT4` | Type: str | -| `--kt-cpuinfer` | `None` | Type: int | -| `--kt-threadpool-count` | `2` | Type: int | -| `--kt-num-gpu-experts` | `None` | Type: int | -| `--kt-max-deferred-`
`experts-per-token` | `None` | Type: int | - -The following parameters have some functional deficiencies on community - -| Argument | Defaults | Options | -|---------------------------------------|----------|--------------------------------| -| `--tool-server` | `None` | Type: str | diff --git a/docs/platforms/ascend/ascend_npu_support_models.md b/docs/platforms/ascend/ascend_npu_support_models.md deleted file mode 100644 index b1ee29fb4..000000000 --- a/docs/platforms/ascend/ascend_npu_support_models.md +++ /dev/null @@ -1,110 +0,0 @@ -# Support Models on Ascend NPU - -This section describes the models supported on the Ascend NPU, including Large Language Models, Multimodal Language -Models, Embedding Models, Reward Models and Rerank Models. Mainstream DeepSeek/Qwen/GLM series are included. -You are welcome to enable various models based on your business requirements. - -## Large Language Models - -| Models | Model Family | A2 Supported | A3 Supported | -|--------------------------------------------|--------------------------------|:----------------------------------------:|:----------------------------------------:| -| DeepSeek V3/V3.1 | DeepSeek | **√** | **√** | -| DeepSeek-V3.2-W8A8 | DeepSeek | **√** | **√** | -| DeepSeek-R1-0528-W8A8 | DeepSeek | **√** | **√** | -| DeepSeek-V2-Lite-W8A8 | DeepSeek | **√** | **√** | -| Qwen/Qwen3-30B-A3B-Instruct-2507 | Qwen | **√** | **√** | -| Qwen/Qwen3-32B | Qwen | **√** | **√** | -| Qwen/Qwen3-0.6B | Qwen | **√** | **√** | -| Qwen3-235B-A22B-W8A8 | Qwen | **√** | **√** | -| Qwen/Qwen3-Next-80B-A3B-Instruct | Qwen | **√** | **√** | -| Qwen3-Coder-480B-A35B-Instruct-w8a8-QuaRot | Qwen | **√** | **√** | -| Qwen/Qwen2.5-7B-Instruct | Qwen | **√** | **√** | -| QWQ-32B-W8A8 | Qwen | **√** | **√** | -| meta-llama/Llama-4-Scout-17B-16E-Instruct | Llama | **√** | **√** | -| AI-ModelScope/Llama-3.1-8B-Instruct | Llama | **√** | **√** | -| LLM-Research/llama-2-7b | Llama | **√** | **√** | -| LLM-Research/Llama-3.2-1B-Instruct | Llama | **√** | **√** | -| mistralai/Mistral-7B-Instruct-v0.2 | Mistral | **√** | **√** | -| google/gemma-3-4b-it | Gemma | **√** | **√** | -| microsoft/Phi-4-multimodal-instruct | Phi | **√** | **√** | -| allenai/OLMoE-1B-7B-0924 | OLMoE | **√** | **√** | -| stabilityai/stablelm-2-1_6b | StableLM | **√** | **√** | -| CohereForAI/c4ai-command-r-v01 | Command-R | **√** | **√** | -| huihui-ai/grok-2 | Grok | **√** | **√** | -| ZhipuAI/chatglm2-6b | ChatGLM | **√** | **√** | -| Shanghai_AI_Laboratory/internlm2-7b | InternLM 2 | **√** | **√** | -| LGAI-EXAONE/EXAONE-3.5-7.8B-Instruct | ExaONE 3 | **√** | **√** | -| xverse/XVERSE-MoE-A36B | XVERSE | **√** | **√** | -| HuggingFaceTB/SmolLM-1.7B | SmolLM | **√** | **√** | -| ZhipuAI/glm-4-9b-chat | GLM-4 | **√** | **√** | -| XiaomiMiMo/MiMo-7B-RL | MiMo | **√** | **√** | -| arcee-ai/AFM-4.5B-Base | Arcee AFM-4.5B | **√** | **√** | -| Howeee/persimmon-8b-chat | Persimmon | **√** | **√** | -| inclusionAI/Ling-lite | Ling | **√** | **√** | -| ibm-granite/granite-3.1-8b-instruct | Granite | **√** | **√** | -| ibm-granite/granite-3.0-3b-a800m-instruct | Granite MoE | **√** | **√** | -| AI-ModelScope/dbrx-instruct | DBRX (Databricks) | **√** | **√** | -| baichuan-inc/Baichuan2-13B-Chat | Baichuan 2 (7B, 13B) | **√** | **√** | -| baidu/ERNIE-4.5-21B-A3B-PT | ERNIE-4.5 (4.5, 4.5MoE series) | **√** | **√** | -| OpenBMB/MiniCPM3-4B | MiniCPM (v3, 4B) | **√** | **√** | -| moonshotai/Kimi-K2-Thinking | Kimi | **√** | **√** | -| eigen-ai-labs/gpt-oss-120b-bf16 | GPTOSS | **√** | **√** | -| allenai/OLMo-2-1124-7B-Instruct | OLMo | **√** | **√** | -| cyankiwi/MiniMax-M2-BF16 | MiniMax-M2 | **√** | **√** | -| upstage/SOLAR-10.7B-Instruct-v1.0 | Solar | **√** | **√** | -| bigcode/starcoder2-7b | StarCoder2 | **√** | **√** | -| arcee-ai/Trinity-Mini | Trinity (Nano, Mini) | **√** | **√** | - -## Multimodal Language Models - -| Models | Model Family (Variants) | A2 Supported | A3 Supported | -|-----------------------------------------------|---------------------------|:----------------------------------------:|:----------------------------------------:| -| Qwen/Qwen2.5-VL-3B-Instruct | Qwen-VL | **√** | **√** | -| Qwen/Qwen2.5-VL-72B-Instruct | Qwen-VL | **√** | **√** | -| Qwen/Qwen3-VL-30B-A3B-Instruct | Qwen-VL | **√** | **√** | -| Qwen/Qwen3-VL-8B-Instruct | Qwen-VL | **√** | **√** | -| Qwen/Qwen3-VL-4B-Instruct | Qwen-VL | **√** | **√** | -| Qwen/Qwen3-VL-235B-A22B-Instruct | Qwen-VL | **√** | **√** | -| deepseek-ai/deepseek-vl2 | DeepSeek-VL2 | **√** | **√** | -| deepseek-ai/Janus-Pro-1B | Janus-Pro (1B, 7B) | **√** | **√** | -| deepseek-ai/Janus-Pro-7B | Janus-Pro (1B, 7B) | **√** | **√** | -| openbmb/MiniCPM-V-2_6 | MiniCPM-V / MiniCPM-o | **√** | **√** | -| openbmb/MiniCPM-o-2_6 | MiniCPM-V / MiniCPM-o | **√** | **√** | -| google/gemma-3-4b-it | Gemma 3 (Multimodal) | **√** | **√** | -| mistralai/Mistral-Small-3.1-24B-Instruct-2503 | Mistral-Small-3.1-24B | **√** | **√** | -| microsoft/Phi-4-multimodal-instruct | Phi-4-multimodal-instruct | **√** | **√** | -| XiaomiMiMo/MiMo-VL-7B-RL | MiMo-VL (7B) | **√** | **√** | -| AI-ModelScope/llava-v1.6-34b | LLaVA (v1.5 & v1.6) | **√** | **√** | -| lmms-lab/llava-next-72b | LLaVA-NeXT (8B, 72B) | **√** | **√** | -| lmms-lab/llava-onevision-qwen2-7b-ov | LLaVA-OneVision | **√** | **√** | -| moonshotai/Kimi-VL-A3B-Instruct | Kimi-VL (A3B) | **√** | **√** | -| ZhipuAI/GLM-4.5V | GLM-4.5V (106B) | **√** | **√** | -| LLM-Research/Llama-3.2-11B-Vision-Instruct | Llama 3.2 Vision (11B) | **√** | **√** | -| rednote-hilab/dots.ocr | DotsVLM-OCR | **√** | **√** | - -## Embedding Models - -| Models | Model Family | A2 Supported | A3 Supported | -|-------------------------------------------|--------------------------|:----------------------------------------:|:----------------------------------------:| -| intfloat/e5-mistral-7b-instruct | E5 (Llama/Mistral based) | **√** | **√** | -| iic/gte_Qwen2-1.5B-instruct | GTE-Qwen2 | **√** | **√** | -| Qwen/Qwen3-Embedding-8B | Qwen3-Embedding | **√** | **√** | -| Alibaba-NLP/gme-Qwen2-VL-2B-Instruct | GME (Multimodal) | **√** | **√** | -| AI-ModelScope/clip-vit-large-patch14-336 | CLIP | **√** | **√** | -| BAAI/bge-large-en-v1.5 | BGE | **√** | **√** | - -## Reward Models - -| Models | Model Family | A2 Supported | A3 Supported | -|------------------------------------------------|---------------------------|------------------------------------------|:----------------------------------------:| -| Skywork/Skywork-Reward-Llama-3.1-8B-v0.2 | Llama3.1 Reward | **√** | **√** | -| Shanghai_AI_Laboratory/internlm2-7b-reward | InternLM 2 Reward | **√** | **√** | -| Qwen/Qwen2.5-Math-RM-72B | Qwen2.5 Reward - Math | **√** | **√** | -| Howeee/Qwen2.5-1.5B-apeach | Qwen2.5 Reward - Sequence | **√** | **√** | -| AI-ModelScope/Skywork-Reward-Gemma-2-27B-v0.2 | Gemma 2-27B Reward | **√** | **√** | - -## Rerank Models - -| Models | Model Family | A2 Supported | A3 Supported | -|-------------------------|--------------|:----------------------------------------:|:----------------------------------------:| -| BAAI/bge-reranker-v2-m3 | BGE-Reranker | **√** | **√** | diff --git a/docs/platforms/ascend/mindspore_backend.md b/docs/platforms/ascend/mindspore_backend.md deleted file mode 100644 index d0df08ea3..000000000 --- a/docs/platforms/ascend/mindspore_backend.md +++ /dev/null @@ -1,151 +0,0 @@ -# MindSpore Models - -## Introduction - -MindSpore is a high-performance AI framework optimized for Ascend NPUs. This doc guides users to run MindSpore models in SGLang. - -## Requirements - -MindSpore currently only supports Ascend NPU devices. Users need to first install Ascend CANN software packages. -The CANN software packages can be downloaded from the [Ascend Official Website](https://www.hiascend.com). The recommended version is 8.3.RC2. - -## Supported Models - -Currently, the following models are supported: - -- **Qwen3**: Dense and MoE models -- **DeepSeek V3/R1** -- *More models coming soon...* - -## Installation - -> **Note**: Currently, MindSpore models are provided by an independent package `sgl-mindspore`. Support for MindSpore is built upon current SGLang support for Ascend NPU platform. Please first [install SGLang for Ascend NPU](ascend_npu.md) and then install `sgl-mindspore`: - -```shell -git clone https://github.com/mindspore-lab/sgl-mindspore.git -cd sgl-mindspore -pip install -e . -``` - - -## Run Model - -Current SGLang-MindSpore supports Qwen3 and DeepSeek V3/R1 models. This doc uses Qwen3-8B as an example. - -### Offline infer - -Use the following script for offline infer: - -```python -import sglang as sgl - -# Initialize the engine with MindSpore backend -llm = sgl.Engine( - model_path="/path/to/your/model", # Local model path - device="npu", # Use NPU device - model_impl="mindspore", # MindSpore implementation - attention_backend="ascend", # Attention backend - tp_size=1, # Tensor parallelism size - dp_size=1 # Data parallelism size -) - -# Generate text -prompts = [ - "Hello, my name is", - "The capital of France is", - "The future of AI is" -] - -sampling_params = {"temperature": 0, "top_p": 0.9} -outputs = llm.generate(prompts, sampling_params) - -for prompt, output in zip(prompts, outputs): - print(f"Prompt: {prompt}") - print(f"Generated: {output['text']}") - print("---") -``` - -### Start server - -Launch a server with MindSpore backend: - -```bash -# Basic server startup -python3 -m sglang.launch_server \ - --model-path /path/to/your/model \ - --host 0.0.0.0 \ - --device npu \ - --model-impl mindspore \ - --attention-backend ascend \ - --tp-size 1 \ - --dp-size 1 -``` - -For distributed server with multiple nodes: - -```bash -# Multi-node distributed server -python3 -m sglang.launch_server \ - --model-path /path/to/your/model \ - --host 0.0.0.0 \ - --device npu \ - --model-impl mindspore \ - --attention-backend ascend \ - --dist-init-addr 127.0.0.1:29500 \ - --nnodes 2 \ - --node-rank 0 \ - --tp-size 4 \ - --dp-size 2 -``` - -## Troubleshooting - -#### Debug Mode - -Enable sglang debug logging by log-level argument. - -```bash -python3 -m sglang.launch_server \ - --model-path /path/to/your/model \ - --host 0.0.0.0 \ - --device npu \ - --model-impl mindspore \ - --attention-backend ascend \ - --log-level DEBUG -``` - -Enable mindspore info and debug logging by setting environments. - -```bash -export GLOG_v=1 # INFO -export GLOG_v=0 # DEBUG -``` - -#### Explicitly select devices - -Use the following environment variable to explicitly select the devices to use. - -```shell -export ASCEND_RT_VISIBLE_DEVICES=4,5,6,7 # to set device -``` - -#### Some communication environment issues - -In case of some environment with special communication environment, users need set some environment variables. - -```shell -export MS_ENABLE_LCCL=off # current not support LCCL communication mode in SGLang-MindSpore -``` - -#### Some dependencies of protobuf - -In case of some environment with special protobuf version, users need set some environment variables to avoid binary version mismatch. - -```shell -export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python # to avoid protobuf binary version mismatch -``` - -## Support -For MindSpore-specific issues: - -- Refer to the [MindSpore documentation](https://www.mindspore.cn/) diff --git a/docs/platforms/ascend_npu_ring_sp_performance.md b/docs/platforms/ascend_npu_ring_sp_performance.md deleted file mode 100644 index 014328aef..000000000 --- a/docs/platforms/ascend_npu_ring_sp_performance.md +++ /dev/null @@ -1,55 +0,0 @@ -# Ascend NPU Ring-SP Performance (Wan2.1-T2V-1.3B) - -This page reports Ring-SP performance on Ascend NPU with `torch_npu==2.10.0`. - -- Baseline config: `ulysses=1, ring=1` (short: `u1r1`) -- Ring-SP config: `ulysses=1, ring=2` (short: `u1r2`) - -## Benchmark Setup - -- Model: `Wan2.1-T2V-1.3B-Diffusers` -- Prompt: `"a cat is playing piano"` -- Framework command: `sglang generate` -- Runtime: `torch_npu==2.10.0` - -## Generate Commands - -### Baseline (`u1r1`) - -```bash -sglang generate --model-path /nas/disk1/Wan2.1-T2V-1.3B-Diffusers \ - --prompt "a cat is playing piano" --num-gpus 1 --ring-degree 1 \ - --save-output -``` - -### Ring-SP (`u1r2`) - -```bash -sglang generate --model-path /nas/disk1/Wan2.1-T2V-1.3B-Diffusers \ - --prompt "a cat is playing piano" --num-gpus 2 --ring-degree 2 \ - --save-output -``` - -## Benchmarks - -Benchmark Disclaimer - -These numbers are from one fixed setup and one prompt case. Actual performance may vary by model settings, environment, and workload. - -### Stage Time Breakdown - -| Stage / Metric | `u1r2` (s) | `u1r1` baseline (s) | Speedup | -|---|---:|---:|---:| -| InputValidation | 0.0003 | 0.0002 | 0.67x | -| TextEncoding | 3.5936 | 3.5820 | 1.00x | -| LatentPreparation | 0.0007 | 0.0055 | 7.86x | -| TimestepPreparation | 0.0008 | 0.0007 | 0.88x | -| Denoising | 121.2788 | 239.2580 | 1.97x | -| Decoding | 13.8685 | 16.4969 | 1.19x | -| **Total (Pixel data generated)** | **141.86** | **266.50** | **1.88x** | - -## Summary - -- With `torch_npu==2.10.0`, Ring-SP (`u1r2`) runs successfully on NPU for this case. -- End-to-end generation time improves from `266.50s` to `141.86s` (`1.88x`). -- The main gain comes from `DenoisingStage` (`1.97x`), while decoding also improves (`1.19x`). diff --git a/docs/platforms/cpu_server.md b/docs/platforms/cpu_server.md deleted file mode 100644 index b954163e5..000000000 --- a/docs/platforms/cpu_server.md +++ /dev/null @@ -1,335 +0,0 @@ -# CPU Servers - -The document addresses how to set up the [SGLang](https://github.com/sgl-project/sglang) environment and run LLM inference on CPU servers. -SGLang is enabled and optimized on the CPUs equipped with Intel® AMX® Instructions, -which are 4th generation or newer Intel® Xeon® Scalable Processors. - -## Optimized Model List - -A list of popular LLMs are optimized and run efficiently on CPU, -including the most notable open-source models like Llama series, Qwen series, -and DeepSeek series like DeepSeek-R1 and DeepSeek-V3.1-Terminus. - -| Model Name | BF16 | W8A8_INT8 | FP8 | -|:---:|:---:|:---:|:---:| -| DeepSeek-R1 | | [meituan/DeepSeek-R1-Channel-INT8](https://huggingface.co/meituan/DeepSeek-R1-Channel-INT8) | [deepseek-ai/DeepSeek-R1](https://huggingface.co/deepseek-ai/DeepSeek-R1) | -| DeepSeek-V3.1-Terminus | | [IntervitensInc/DeepSeek-V3.1-Terminus-Channel-int8](https://huggingface.co/IntervitensInc/DeepSeek-V3.1-Terminus-Channel-int8) | [deepseek-ai/DeepSeek-V3.1-Terminus](https://huggingface.co/deepseek-ai/DeepSeek-V3.1-Terminus) | -| Llama-3.2-3B | [meta-llama/Llama-3.2-3B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct) | [RedHatAI/Llama-3.2-3B-quantized.w8a8](https://huggingface.co/RedHatAI/Llama-3.2-3B-Instruct-quantized.w8a8) | | -| Llama-3.1-8B | [meta-llama/Llama-3.1-8B-Instruct](https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct) | [RedHatAI/Meta-Llama-3.1-8B-quantized.w8a8](https://huggingface.co/RedHatAI/Meta-Llama-3.1-8B-quantized.w8a8) | | -| QwQ-32B | | [RedHatAI/QwQ-32B-quantized.w8a8](https://huggingface.co/RedHatAI/QwQ-32B-quantized.w8a8) | | -| DeepSeek-Distilled-Llama | | [RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w8a8](https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w8a8) | | -| Qwen3-235B | | | [Qwen/Qwen3-235B-A22B-FP8](https://huggingface.co/Qwen/Qwen3-235B-A22B-FP8) | - -**Note:** The model identifiers listed in the table above -have been verified on 6th Gen Intel® Xeon® P-core platforms. - -## Installation - -### Install Using Docker - -It is recommended to use Docker for setting up the SGLang environment. -A [Dockerfile](https://github.com/sgl-project/sglang/blob/main/docker/xeon.Dockerfile) is provided to facilitate the installation. -Replace `` below with your [HuggingFace access token](https://huggingface.co/docs/hub/en/security-tokens). - -```bash -# Clone the SGLang repository -git clone https://github.com/sgl-project/sglang.git -cd sglang/docker - -# Build the docker image -docker build -t sglang-cpu:latest -f xeon.Dockerfile . - -# Initiate a docker container -docker run \ - -it \ - --privileged \ - --ipc=host \ - --network=host \ - -v /dev/shm:/dev/shm \ - -v ~/.cache/huggingface:/root/.cache/huggingface \ - -p 30000:30000 \ - -e "HF_TOKEN=" \ - sglang-cpu:latest /bin/bash -``` - -### Install From Source - -If you prefer to install SGLang in a bare metal environment, -the setup process is as follows: - -Please install the required packages and libraries beforehand if -they are not already present on your system. -You can refer to the Ubuntu-based installation commands in -[the Dockerfile](https://github.com/sgl-project/sglang/blob/main/docker/xeon.Dockerfile#L11) -for guidance. - -1. Install `uv` package manager, then create and activate a virtual environment: - -```bash -# Taking '/opt' as the example uv env folder, feel free to change it as needed -cd /opt -curl -LsSf https://astral.sh/uv/install.sh | sh -source $HOME/.local/bin/env -uv venv --python 3.12 -source .venv/bin/activate -``` - -2. Create a config file to direct the installation channel - (a.k.a. index-url) of `torch` related packages: - -```bash -vim .venv/uv.toml -``` - -Press 'a' to enter insert mode of `vim`, paste the following content into the created file - -```file -[[index]] -name = "torch" -url = "https://download.pytorch.org/whl/cpu" - -[[index]] -name = "torchvision" -url = "https://download.pytorch.org/whl/cpu" - -[[index]] -name = "torchaudio" -url = "https://download.pytorch.org/whl/cpu" - -[[index]] -name = "triton" -url = "https://download.pytorch.org/whl/cpu" - -``` - -Save the file (in `vim`, press 'esc' to exit insert mode, then ':x+Enter'), -and set it as the default `uv` config. - -```bash -export UV_CONFIG_FILE=/opt/.venv/uv.toml -``` - -3. Clone the `sglang` source code and build the packages - -```bash -# Clone the SGLang code -git clone https://github.com/sgl-project/sglang.git -cd sglang -git checkout - -# Use dedicated toml file -cd python -cp pyproject_cpu.toml pyproject.toml -# Install SGLang dependent libs, and build SGLang main package -uv pip install --upgrade pip setuptools -uv pip install . - -# Build the CPU backend kernels -cd ../sgl-kernel -cp pyproject_cpu.toml pyproject.toml -uv pip install . -``` - -4. Set the required environment variables - -```bash -export SGLANG_USE_CPU_ENGINE=1 - -# Set 'LD_LIBRARY_PATH' and 'LD_PRELOAD' to ensure the libs can be loaded by sglang processes -export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu -export LD_PRELOAD=${LD_PRELOAD}:/opt/.venv/lib/libiomp5.so:${LD_LIBRARY_PATH}/libtcmalloc.so.4:${LD_LIBRARY_PATH}/libtbbmalloc.so.2 -``` - -Notes: - -- Note that the environment variable `SGLANG_USE_CPU_ENGINE=1` - is required to enable the SGLang service with the CPU engine. - -- If you encounter code compilation issues during the `sgl-kernel` building process, - please check your `gcc` and `g++` versions and upgrade them if they are outdated. - It is recommended to use `gcc-13` and `g++-13` as they have been verified - in the official Docker container. - -- The system library path is typically located in one of the following directories: - `~/.local/lib/`, `/usr/local/lib/`, `/usr/local/lib64/`, `/usr/lib/`, `/usr/lib64/` - and `/usr/lib/x86_64-linux-gnu/`. In the above example commands, `/usr/lib/x86_64-linux-gnu` - is used. Please adjust the path according to your server configuration. - -- It is recommended to add the following to your `~/.bashrc` file to - avoid setting these variables every time you open a new terminal: - - ```bash - source .venv/bin/activate - export SGLANG_USE_CPU_ENGINE=1 - export LD_LIBRARY_PATH= - export LD_PRELOAD= - ``` - -## Launch of the Serving Engine - -Example command to launch SGLang serving: - -```bash -python -m sglang.launch_server \ - --model \ - --trust-remote-code \ - --disable-overlap-schedule \ - --device cpu \ - --host 0.0.0.0 \ - --tp 6 -``` - -Notes: - -1. For running W8A8 quantized models, please add the flag `--quantization w8a8_int8`. - -2. The flag `--tp 6` specifies that tensor parallelism will be applied using 6 ranks (TP6). - The number of TP specified is how many TP ranks will be used during the execution. - On a CPU platform, a TP rank means a sub-NUMA cluster (SNC). - Usually we can get the SNC information (How many available) from the Operating System with e.g. `lscpu` command. - - If the specified TP rank number differs from the total SNC count, - the system will automatically utilize the first `n` SNCs. - Note that `n` cannot exceed the total SNC number, doing so will result in an error. - - `SGLANG_CPU_OMP_THREADS_BIND` allows explicit control of CPU cores for each tensor parallel (TP) rank. - - **example 1**: Run SGLang service with TP=6, using the first 40 cores of each SNC on a Xeon® 6980P server, - which has 43-43-42 cores on the 3 SNCs of a socket, we should set: - - ```bash - export SGLANG_CPU_OMP_THREADS_BIND="0-39|43-82|86-125|128-167|171-210|214-253" - ``` - This configuration is equivalent to: - - rank 0: `numactl -C 0-39 -m 0` - - rank 1: `numactl -C 43-82 -m 1` - - rank 2: `numactl -C 86-125 -m 2` - - rank 3: `numactl -C 128-167 -m 3` - - rank 4: `numactl -C 171-210 -m 4` - - rank 5: `numactl -C 214-253 -m 5` - - - **example 2**: Run SGLang service with TP=2, using 96 cores cross 3 SNCs on a Xeon® 6972P server, - which has 32-32-32 cores on the 3 SNCs in a socket, we should set: - ```bash - export SGLANG_CPU_OMP_THREADS_BIND="0-95|96-191" - ``` - This configuration is equivalent to: - - rank 0: `numactl -C 0-95 -m 0-2` - - rank 1: `numactl -C 96-191 -m 3-5` - - Please beware that with SGLANG_CPU_OMP_THREADS_BIND set, - the available memory amounts of the ranks may not be determined in prior. - You may need to set proper `--max-total-tokens` to avoid the out-of-memory error. - -3. For optimizing decoding with torch.compile, please add the flag `--enable-torch-compile`. - To specify the maximum batch size when using `torch.compile`, set the flag `--torch-compile-max-bs`. - For example, `--enable-torch-compile --torch-compile-max-bs 4` means using `torch.compile` - and setting the maximum batch size to 4. - -4. A warmup step is automatically triggered when the service is started. - The server is ready when you see the log `The server is fired up and ready to roll!`. - -## Benchmarking with Requests - -You can benchmark the performance via the `bench_serving` script. -Run the command in another terminal. An example command would be: - -```bash -python -m sglang.bench_serving \ - --dataset-name random \ - --random-input-len 1024 \ - --random-output-len 1024 \ - --num-prompts 1 \ - --request-rate inf \ - --random-range-ratio 1.0 -``` - -Detailed parameter descriptions are available via the command: - -```bash -python -m sglang.bench_serving -h -``` - -Additionally, requests can be formatted using -[the OpenAI Completions API](https://docs.sglang.io/basic_usage/openai_api_completions.html) -and sent via the command line (e.g., using `curl`) or through your own scripts. - -## Example Usage Commands - -Large Language Models can range from fewer than 1 billion to several hundred billion parameters. -Dense models larger than 20B are expected to run on flagship 6th Gen Intel® Xeon® processors -with dual sockets and a total of 6 sub-NUMA clusters. Dense models of approximately 10B parameters or fewer, -or MoE (Mixture of Experts) models with fewer than 10B activated parameters, can run on more common -4th generation or newer Intel® Xeon® processors, or utilize a single socket of the flagship 6th Gen Intel® Xeon® processors. - -### Example: Running DeepSeek-V3.1-Terminus - -An example command to launch service of W8A8_INT8 DeepSeek-V3.1-Terminus on a Xeon® 6980P server: - -```bash -python -m sglang.launch_server \ - --model IntervitensInc/DeepSeek-V3.1-Terminus-Channel-int8 \ - --trust-remote-code \ - --disable-overlap-schedule \ - --device cpu \ - --quantization w8a8_int8 \ - --host 0.0.0.0 \ - --enable-torch-compile \ - --torch-compile-max-bs 4 \ - --tp 6 -``` - -Similarly, an example command to launch service of FP8 DeepSeek-V3.1-Terminus would be: - -```bash -python -m sglang.launch_server \ - --model deepseek-ai/DeepSeek-V3.1-Terminus \ - --trust-remote-code \ - --disable-overlap-schedule \ - --device cpu \ - --host 0.0.0.0 \ - --enable-torch-compile \ - --torch-compile-max-bs 4 \ - --tp 6 -``` - -Note: Please set `--torch-compile-max-bs` to the maximum desired batch size for your deployment, -which can be up to 16. The value `4` in the examples is illustrative. - -### Example: Running Llama-3.2-3B - -An example command to launch service of Llama-3.2-3B with BF16 precision: - -```bash -python -m sglang.launch_server \ - --model meta-llama/Llama-3.2-3B-Instruct \ - --trust-remote-code \ - --disable-overlap-schedule \ - --device cpu \ - --host 0.0.0.0 \ - --enable-torch-compile \ - --torch-compile-max-bs 16 \ - --tp 2 -``` - -The example command to launch service of W8A8_INT8 version of Llama-3.2-3B: - -```bash -python -m sglang.launch_server \ - --model RedHatAI/Llama-3.2-3B-quantized.w8a8 \ - --trust-remote-code \ - --disable-overlap-schedule \ - --device cpu \ - --quantization w8a8_int8 \ - --host 0.0.0.0 \ - --enable-torch-compile \ - --torch-compile-max-bs 16 \ - --tp 2 -``` - -Note: The `--torch-compile-max-bs` and `--tp` settings are examples that should be adjusted for your setup. -For instance, use `--tp 3` to utilize 1 socket with 3 sub-NUMA clusters on an Intel® Xeon® 6980P server. - -Once the server have been launched, you can test it using the `bench_serving` command or create -your own commands or scripts following [the benchmarking example](#benchmarking-with-requests). diff --git a/docs/platforms/mthreads_gpu.md b/docs/platforms/mthreads_gpu.md deleted file mode 100644 index 3538df318..000000000 --- a/docs/platforms/mthreads_gpu.md +++ /dev/null @@ -1,25 +0,0 @@ -# Moore Threads GPUs - -This document describes how run SGLang on Moore Threads GPUs. If you encounter issues or have questions, please [open an issue](https://github.com/sgl-project/sglang/issues). - -## Install SGLang - -You can install SGLang using one of the methods below. - -### Install from Source - -```bash -# Use the default branch -git clone https://github.com/sgl-project/sglang.git -cd sglang - -# Compile sgl-kernel -pip install --upgrade pip -cd sgl-kernel -python setup_musa.py install - -# Install sglang python package -cd .. -rm -f python/pyproject.toml && mv python/pyproject_other.toml python/pyproject.toml -pip install -e "python[all_musa]" -``` diff --git a/docs/platforms/nvidia_jetson.md b/docs/platforms/nvidia_jetson.md deleted file mode 100644 index ba3b68ae8..000000000 --- a/docs/platforms/nvidia_jetson.md +++ /dev/null @@ -1,80 +0,0 @@ -# NVIDIA Jetson Orin - -## Prerequisites - -Before starting, ensure the following: - -- [**NVIDIA Jetson AGX Orin Devkit**](https://www.nvidia.com/en-us/autonomous-machines/embedded-systems/jetson-orin/) is set up with **JetPack 6.1** or later. -- **CUDA Toolkit** and **cuDNN** are installed. -- Verify that the Jetson AGX Orin is in **high-performance mode**: -```bash -sudo nvpmodel -m 0 -``` -* * * * * -## Installing and running SGLang with Jetson Containers -Clone the jetson-containers github repository: -``` -git clone https://github.com/dusty-nv/jetson-containers.git -``` -Run the installation script: -``` -bash jetson-containers/install.sh -``` -Build the container image: -``` -jetson-containers build sglang -``` -Run the container: -``` -jetson-containers run $(autotag sglang) -``` -Or you can also manually run a container with this command: -``` -docker run --runtime nvidia -it --rm --network=host IMAGE_NAME -``` -* * * * * - -Running Inference ------------------------------------------ - -Launch the server: -```bash -python -m sglang.launch_server \ - --model-path deepseek-ai/DeepSeek-R1-Distill-Llama-8B \ - --device cuda \ - --dtype half \ - --attention-backend flashinfer \ - --mem-fraction-static 0.8 \ - --context-length 8192 -``` -The quantization and limited context length (`--dtype half --context-length 8192`) are due to the limited computational resources in [Nvidia jetson kit](https://www.nvidia.com/en-us/autonomous-machines/embedded-systems/jetson-orin/). A detailed explanation can be found in [Server Arguments](../advanced_features/server_arguments.md). - -After launching the engine, refer to [Chat completions](https://docs.sglang.io/basic_usage/openai_api_completions.html#Usage) to test the usability. -* * * * * -Running quantization with TorchAO -------------------------------------- -TorchAO is suggested to NVIDIA Jetson Orin. -```bash -python -m sglang.launch_server \ - --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \ - --device cuda \ - --dtype bfloat16 \ - --attention-backend flashinfer \ - --mem-fraction-static 0.8 \ - --context-length 8192 \ - --torchao-config int4wo-128 -``` -This enables TorchAO's int4 weight-only quantization with a 128-group size. The usage of `--torchao-config int4wo-128` is also for memory efficiency. - - -* * * * * -Structured output with XGrammar -------------------------------- -Please refer to [SGLang doc structured output](../advanced_features/structured_outputs.ipynb). -* * * * * - -Thanks to the support from [Nurgaliyev Shakhizat](https://github.com/shahizat), [Dustin Franklin](https://github.com/dusty-nv) and [Johnny Núñez Cano](https://github.com/johnnynunez). - -References ----------- -- [NVIDIA Jetson AGX Orin Documentation](https://developer.nvidia.com/embedded/jetson-agx-orin) diff --git a/docs/platforms/plugin.md b/docs/platforms/plugin.md deleted file mode 100644 index a956ebffa..000000000 --- a/docs/platforms/plugin.md +++ /dev/null @@ -1,414 +0,0 @@ -# SGLang Plugin System - -## Overview - -Allows hardware vendors and developers to extend SGLang **without modifying the main repository code**. - -The framework provides two plugin types, both discovered via Python's standard `setuptools` entry_points: - -| Plugin Type | Entry Point Group | Purpose | -|---|---|---| -| **Hardware Platform Plugin** | `sglang.srt.platforms` | Register a custom hardware platform (device operations, KV cache pools, attention backends, graph capture, compilation backends, etc.) | -| **General Plugin** | `sglang.srt.plugins` | Inject hooks (before/after/around/replace) into any function/method, or replace entire classes | - -### Principles - -- **Non-intrusive**: Existing CUDA/ROCm/NPU/XPU code remains unchanged. OOT code paths are added alongside existing hardware-specific logic. -- **Zero configuration**: Plugins are automatically discovered after `pip install`, no sglang code changes required. -- **Environment variable control**: `SGLANG_PLATFORM` selects or validates the active platform plugin; `SGLANG_PLUGINS` (comma-separated) controls which general plugins to load. - -### Current Scope & Future Direction - -The plugin system currently targets **out-of-tree (OOT) hardware platforms** — enabling new devices to integrate with SGLang without any changes to the main repository. The main-repo hardware paths (CUDA, ROCm, NPU, XPU, etc.) continue to use the existing `is_cuda()`/`is_npu()`/… utility functions. - -As the plugin interfaces mature and stabilize, in-tree hardware backends can be gradually migrated to the same plugin architecture. This would replace the scattered `if device == "cuda" … elif device == "npu" …` branches throughout the codebase with a single polymorphic dispatch through the platform interface, making each hardware backend self-contained and the core engine hardware-agnostic. - -## Architecture - -### Platform Hierarchy - -The platform hierarchy uses a DeviceMixin pattern to share device operations between SRT (LLM inference) and Multimodal subsystems: - -``` -DeviceMixin (shared device identity + operations) -├── SRTPlatform(DeviceMixin) # + graph runner, KV pool, … -│ └── MySRTPlatform(SRTPlatform, MyDeviceMixin) # OOT plugin -└── MMPlatform(DeviceMixin) # + attention backend, VAE, … (future) - └── MyMMPlatform(MMPlatform, MyDeviceMixin) # OOT plugin -``` - -Key design points: -- **DeviceMixin** provides platform identity queries (`is_cuda()`, `is_npu()`, etc.) and device operations (`set_device()`, `get_device_name()`, etc.) -- **SRTPlatform** adds SRT-specific factory methods, capability flags, and lifecycle hooks -- OOT plugins implement a **device mixin** (vendor-specific operations) and compose it with **SRTPlatform** via multiple inheritance -- All methods are **instance methods** (not classmethods), called through the `current_platform` singleton -- Device operations and factory methods raise `NotImplementedError` by default (fail-fast) -- Capability flags use safe conservative defaults (`False`/`pass`) -- Methods are annotated `[Active]` (called by SGLang core) or `[Planned]` (reserved for future migration) - -### Platform Discovery (`current_platform`) - -`current_platform` is a **lazy singleton** in `sglang.srt.platforms`. On first access it resolves the active platform through the following priority chain: - -``` -entry_points("sglang.srt.platforms") → Enumerate ALL plugins by name (metadata only) - │ - ├─ SGLANG_PLATFORM set (front-loading filter): - │ ├─ Name not found in discovered → RuntimeError - │ ├─ activate() returns non-None → load that platform - │ └─ activate() returns None → RuntimeError (hardware unavailable) - │ - └─ SGLANG_PLATFORM unset (auto-discover, activate all): - ├─ 0 activated → fallback base SRTPlatform - ├─ 1 activated → use it - └─ N activated → RuntimeError (must set SGLANG_PLATFORM) -``` - -### Plugin Loading Flow - -`load_plugins()` discovers and executes general plugins, then applies all registered hooks. It is called at four points: - -| Call Site | Process | Timing | -|---------|------|------| -| `cli/serve.py` serve() | Main | Before `prepare_server_args()` | -| `launch_server.py` `__main__` | Main | Before `prepare_server_args()` | -| `engine.py` `_launch_subprocesses()` | Main | Before `server_args.check_server_args()` | -| `scheduler.py` `run_scheduler_process()` | Subprocess | Before `Scheduler()` construction | - -> **Note**: `load_plugins()` is idempotent (guarded by `_plugins_loaded` flag). In spawn'd subprocesses the flag resets, so plugins are correctly re-loaded. - -``` -load_plugins() - ├── _get_excluded_dists() → compute dists to skip (via SGLANG_PLATFORM) - ├── load_plugins_by_group("sglang.srt.plugins", → discover entry_points, filter by SGLANG_PLUGINS - │ excluded_dists=...) skip plugins from unselected platform packages - ├── for each plugin: → set _current_plugin_source context var - │ func() side effects (register hooks with source tracking) - └── HookRegistry.apply_hooks() → monkey-patch targets -``` - ---- - -## Plugin Type 1: Hardware Platform Plugin - -### Description - -A hardware platform plugin registers an `SRTPlatform` subclass that tells SGLang how to interact with a specific hardware backend. - -### Quick Start - -**1. Create a minimal package:** - -``` -my_platform_plugin/ -├── pyproject.toml -└── my_platform_plugin/ - ├── __init__.py # activate() function - ├── device.py # MyDeviceMixin - └── platform.py # MySRTPlatform -``` - -**2. `pyproject.toml`:** - -```toml -[build-system] -requires = ["setuptools"] -build-backend = "setuptools.build_meta" - -[project] -name = "my-platform-plugin" -version = "0.1.0" - -[project.entry-points."sglang.srt.platforms"] -my_device = "my_platform_plugin:activate" -``` - -**3. `__init__.py`** — activation function: - -```python -def activate(): - """Return fully-qualified class name to activate, or None to skip.""" - if _my_device_is_available(): - return "my_platform_plugin.platform.MySRTPlatform" - return None -``` - -**4. `device.py`** — device mixin: - -```python -from sglang.srt.platforms.device_mixin import DeviceMixin, PlatformEnum - -class MyDeviceMixin(DeviceMixin): - _enum = PlatformEnum.OOT - device_name = "my_device" - device_type = "my_device" # torch device type - - def set_device(self, device) -> None: ... - def get_device_name(self, device_id=0) -> str: ... - def get_device_total_memory(self, device_id=0) -> int: ... - def get_current_memory_usage(self, device=None) -> float: ... - def get_device_capability(self, device_id=0): ... - def get_torch_distributed_backend_str(self) -> str: ... -``` - -**5. `platform.py`** — SRT platform: - -```python -from sglang.srt.platforms.interface import SRTPlatform -from my_platform_plugin.device import MyDeviceMixin - -class MySRTPlatform(SRTPlatform, MyDeviceMixin): - def get_default_attention_backend(self) -> str: ... - def support_cuda_graph(self) -> bool: ... - # ... override other methods as needed -``` - -**6. Install and verify:** - -```bash -pip install -e my_platform_plugin/ -python -c "from sglang.srt.platforms import current_platform; print(current_platform)" -``` - -### Platform Interface Reference - -#### Identity Queries (from DeviceMixin) - -| Method | Default | Description | -|---|---|---| -| `is_cuda()` | Based on `_enum` | Whether this is an NVIDIA CUDA platform | -| `is_rocm()` | Based on `_enum` | Whether this is an AMD ROCm platform | -| `is_npu()` | Based on `_enum` | Whether this is a Huawei NPU platform | -| `is_cpu()` | Based on `_enum` | Whether this is a CPU-only platform | -| `is_xpu()` | Based on `_enum` | Whether this is an Intel XPU platform | -| `is_musa()` | Based on `_enum` | Whether this is a Moore Threads MUSA platform | -| `is_cuda_alike()` | CUDA+ROCM+MUSA | True if the hardware supports CUDA-like APIs | -| `is_out_of_tree()` | `True` for OOT | Automatically detected based on `_enum = PlatformEnum.OOT` | - -#### Device Operations (from DeviceMixin) - -> Methods annotated **[Active]** are called by SGLang core through `current_platform` — OOT implementations take effect immediately. -> Methods annotated **[Planned]** are reserved interfaces — SGLang core still uses hardcoded calls (e.g. `torch.cuda.empty_cache()`). OOT implementations will NOT take effect until the core is migrated in a future PR. - -| Method | Default | Status | Description | -|---|---|---|---| -| `get_device(local_rank)` | `raise NotImplementedError` | Planned | Return `torch.device` for a given local rank | -| `set_device(device)` | `raise NotImplementedError` | Planned | Set the current device | -| `get_device_name(device_id)` | `raise NotImplementedError` | Planned | Get human-readable device name | -| `get_device_uuid(device_id)` | `raise NotImplementedError` | Planned | Get unique device identifier | -| `get_device_capability(device_id)` | `raise NotImplementedError` | Planned | Get `DeviceCapability(major, minor)`. None if N/A | -| `empty_cache()` | `pass` | Planned | Release cached device memory | -| `synchronize()` | `pass` | Planned | Synchronize device operations | -| `get_device_total_memory(device_id)` | `raise NotImplementedError` | **Active** | Get total device memory in bytes | -| `get_available_memory(device_id)` | `raise NotImplementedError` | Planned | Return `(free_bytes, total_bytes)` | -| `get_current_memory_usage(device)` | `raise NotImplementedError` | **Active** | Get current peak memory usage in bytes | -| `get_torch_distributed_backend_str()` | `raise NotImplementedError` | Planned | Distributed backend string (e.g. "nccl", "hccl") | -| `get_communicator_class()` | `None` | Planned | Platform-specific communicator class | -| `inference_mode()` | `torch.inference_mode(True)` | Planned | Return inference mode context manager | -| `seed_everything(seed)` | Set random/np/torch seeds | Planned | Set random seeds for reproducibility | -| `verify_quantization(quant)` | `pass` | Planned | Validate quantization method support | -| `get_cpu_architecture()` | Auto-detect x86/arm | Planned | Detect CPU architecture (`CpuArchEnum`) | - -#### Types (from DeviceMixin) - -| Type | Description | -|---|---| -| `PlatformEnum` | Enumeration of platform types: CUDA, ROCM, CPU, XPU, MUSA, NPU, TPU, MPS, OOT, UNSPECIFIED | -| `CpuArchEnum` | CPU architecture: X86, ARM, UNSPECIFIED | -| `DeviceCapability` | `NamedTuple(major, minor)` with comparison support. Methods: `as_version_str()`, `to_int()` | - -#### Capability Flags (from SRTPlatform) - -| Method | Default | Description | -|---|---|---| -| `support_cuda_graph()` | `False` | Whether device graph capture is supported (plain CUDA graph) | -| `support_piecewise_cuda_graph()` | `False` | Whether piecewise CUDA graph (torch.compile backend) is supported | -| `supports_fp8()` | `False` | Whether FP8 quantization is supported | -| `is_pin_memory_available()` | `True` | Whether pinned memory is available | - -#### Subsystem Factory Methods (from SRTPlatform) - -| Method | Default | Description | -|---|---|---| -| `get_default_attention_backend()` | `raise NotImplementedError` | Default attention backend name | -| `get_graph_runner_cls()` | `raise NotImplementedError` | Graph Runner class | -| `get_mha_kv_pool_cls()` | `raise NotImplementedError` | MHA KV cache pool class | -| `get_mla_kv_pool_cls()` | `raise NotImplementedError` | MLA KV cache pool class | -| `get_dsa_kv_pool_cls()` | `raise NotImplementedError` | DSA KV cache pool class (DeepSeek V3.2) | -| `get_paged_allocator_cls()` | `raise NotImplementedError` | Paged allocator class | -| `get_piecewise_backend_cls()` | `raise NotImplementedError` | Piecewise compilation backend class | -| `get_compile_backend(mode)` | `"inductor"` | Compilation backend string | -| `get_dispatch_key_name()` | `"native"` | MultiPlatformOp dispatch key name | - -#### Lifecycle Hooks (from SRTPlatform) - -| Method | Invocation Timing | Purpose | -|---|---|---| -| `apply_server_args_defaults(server_args)` | After ServerArgs parsing, in `__post_init__` | Set platform-specific defaults | -| `init_backend()` | In each worker, before model construction | One-time backend initialization | - -### Environment Variables - -| Variable | Description | -|---|---| -| `SGLANG_PLATFORM` | Select the platform plugin by entry_point name (e.g. `kunlun`, `demo_cuda`). When set, **only** the named plugin's `activate()` is called (front-loading filter) — other plugins are not touched. Additionally, general plugins (`sglang.srt.plugins`) from unselected platform packages are automatically skipped to avoid importing their dependencies. Required when multiple plugins would activate. Errors if the name is not found or if the plugin's hardware is unavailable. | -| `SGLANG_PLUGINS` | Comma-separated whitelist of general plugin names to load (group: `sglang.srt.plugins`). If unset, all discovered general plugins are loaded. | - ---- - -## Plugin Type 2: General Plugin - -### Description - -General function plugins inject behavior into sglang **without requiring a custom platform**. Use cases include: - -- **Observability**: Add logging, metrics, and tracing to any function -- **Behavior modification**: Modify function arguments or return values -- **Performance profiling**: Add timing to critical functions -- **A/B testing**: Replace implementations at runtime - -### Quick Start - -**1. Create a minimal package:** - -``` -my_general_plugin/ -├── pyproject.toml -└── my_general_plugin/ - └── __init__.py # register() function -``` - -**2. `pyproject.toml`:** - -```toml -[build-system] -requires = ["setuptools"] -build-backend = "setuptools.build_meta" - -[project] -name = "my-general-plugin" -version = "0.1.0" - -[project.entry-points."sglang.srt.plugins"] -my_plugin = "my_general_plugin:register" -``` - -**3. `__init__.py`** — register hooks: - -```python -from sglang.srt.plugins.hook_registry import HookRegistry, HookType - -def register(): - """Entry point called by load_plugins().""" - HookRegistry.register( - "sglang.srt.managers.scheduler.Scheduler.__init__", - my_hook, - HookType.AROUND, - ) - -def my_hook(original_fn, self, *args, **kwargs): - result = original_fn(self, *args, **kwargs) - print(f"Scheduler initialized! gpu_id={self.gpu_id}") - return result -``` - -**4. Install and run:** - -```bash -pip install -e my_general_plugin/ -sglang serve --model-path [options] -# Look for "Scheduler initialized!" in logs -``` - -### Hook Types - -`HookRegistry` supports four hook types: - -| Hook Type | Signature | Description | -|---|---|---| -| **BEFORE** | `fn(*args, **kwargs) -> (args, kwargs) \| None` | Runs before the original. Return `None` to keep args unchanged, or `(args, kwargs)` to modify. | -| **AFTER** | `fn(result, *args, **kwargs) -> new_result \| None` | Runs after the original. Return `None` to keep result, or a new value to replace. | -| **AROUND** | `fn(original_fn, *args, **kwargs) -> result` | Wraps the original. You must call `original_fn` yourself. Full control over execution. | -| **REPLACE** | `fn(*args, **kwargs) -> result` or `class` | Replace the original function or class entirely. For class targets, pass a replacement class directly — it is substituted via `setattr` preserving `isinstance()`/`issubclass()` semantics. | - -> **Note**: Only `REPLACE` accepts a class as the hook. Passing a class to `BEFORE`/`AFTER`/`AROUND` raises `TypeError` at registration time. - -### Registration API - -Hooks can be registered using the **imperative API** or the **decorator API**: - -```python -# --- Imperative API --- -from sglang.srt.plugins.hook_registry import HookRegistry, HookType - -def my_timer(original_fn, *args, **kwargs): - start = time.perf_counter() - result = original_fn(*args, **kwargs) - print(f"Elapsed: {time.perf_counter() - start:.3f}s") - return result - -HookRegistry.register( - "sglang.srt.managers.scheduler.Scheduler.get_next_batch_to_run", - my_timer, - HookType.AROUND, -) - -# --- Decorator API --- -from sglang.srt.plugins.hook_registry import plugin_hook, HookType - -@plugin_hook( - "sglang.srt.managers.scheduler.Scheduler.get_next_batch_to_run", - type=HookType.AROUND, -) -def my_timer(original_fn, *args, **kwargs): - start = time.perf_counter() - result = original_fn(*args, **kwargs) - print(f"Elapsed: {time.perf_counter() - start:.3f}s") - return result - -# --- Class replacement (REPLACE) --- -from sglang.srt.plugins.hook_registry import plugin_hook, HookType -from sglang.srt.managers.scheduler import Scheduler - -@plugin_hook( - "sglang.srt.managers.scheduler.Scheduler", - type=HookType.REPLACE, -) -class MyScheduler(Scheduler): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - print("Enhanced scheduler initialized!") -``` - -### Hook Target Resolution - -Target paths use fully-qualified dotted notation. Both formats are supported: - -- **Dotted**: `sglang.srt.managers.scheduler.Scheduler.__init__` -- **Entry-points style**: `sglang.srt.managers.scheduler:Scheduler.__init__` (colon treated as dot) - -### Common Hook Targets - -| Target | Description | -|---|---| -| `sglang.srt.server_args.ServerArgs.add_cli_args` | Add custom CLI arguments | -| `sglang.srt.server_args.ServerArgs.__post_init__` | Modify ServerArgs after parsing | -| `sglang.srt.server_args.ServerArgs.check_server_args` | Add/relax validation | -| `sglang.srt.managers.scheduler.Scheduler.__init__` | Custom scheduler state | -| `sglang.srt.managers.scheduler.Scheduler.get_next_batch_to_run` | Custom scheduling policy | -| `sglang.srt.managers.scheduler.Scheduler.run_batch` | Profiling / inspection | -| `sglang.srt.managers.scheduler.Scheduler.process_batch_result` | Custom metrics | -| `sglang.srt.managers.tp_worker.TpModelWorker.__init__` | Custom worker state | -| `sglang.srt.managers.tp_worker.TpModelWorker.forward_batch_generation` | Forward pass wrapping | - ---- - -## File Reference - -| File | Description | -|---|---| -| `sglang/srt/platforms/device_mixin.py` | `PlatformEnum` + `DeviceMixin` base class | -| `sglang/srt/platforms/interface.py` | `SRTPlatform` base class (extends DeviceMixin) | -| `sglang/srt/platforms/__init__.py` | `current_platform` lazy singleton + discovery logic | -| `sglang/srt/plugins/__init__.py` | `load_plugins()` + `load_plugins_by_group()` | -| `sglang/srt/plugins/hook_registry.py` | `HookRegistry`, `HookType`, `plugin_hook` decorator | diff --git a/docs/platforms/tpu.md b/docs/platforms/tpu.md deleted file mode 100644 index ede533513..000000000 --- a/docs/platforms/tpu.md +++ /dev/null @@ -1,477 +0,0 @@ -# TPU - -SGLang supports high-performance TPU inference through the SGLang-JAX backend, which is specifically optimized for Google Cloud TPUs. The JAX-based implementation delivers exceptional throughput and low latency for Large Language Model (LLM) serving workloads on TPU hardware. - -For TPU-specific issues or feature requests, please visit the [sglang-jax GitHub issues page](https://github.com/sgl-project/sglang-jax/issues). - -**NOTE:** SGLang TPU support is implemented via the SGLang-JAX backend, a dedicated JAX-based inference engine maintained as a separate repository at [https://github.com/sgl-project/sglang-jax](https://github.com/sgl-project/sglang-jax). - -## System Requirements - -### Supported TPU Hardware - -| TPU Type | HBM Memory | Availability | -|----------|-----------|--------------| -| TPU v6e | 32 GB | Google Cloud | -| TPU v7 | 96 GB per core | Google Cloud | - -### Software Requirements - -- **Python:** 3.12 or higher -- **JAX:** Latest version with TPU support -- **Environment:** Google Cloud TPU VM or compatible TPU runtime -- **Optional:** SkyPilot for simplified cloud deployment - -## Feature Support Matrix - -SGLang-JAX provides comprehensive TPU-optimized features for production LLM serving: - -| Feature | Support Status | Description | -|---------|---------------|-------------| -| High-Throughput Continuous Batching | ✅ | Dynamic request batching for maximum TPU utilization | -| Radix Tree KV Cache | ✅ | Memory-efficient prefix sharing between requests | -| FlashAttention Backend | ✅ | TPU-optimized attention kernel for long sequences | -| Tensor Parallelism | ✅ | Distribute models across multiple TPU cores | -| Paged Attention | ✅ | Flexible KV cache management with paging | -| Speculative Decoding (EAGLE/EAGLE3) | ✅ | 20-40% throughput improvement for compatible models | -| Chunked Prefill | ✅ | Mixed prefill-decode batching | -| OpenAI-Compatible API | ✅ | Drop-in replacement for OpenAI API | -| Data Parallel Attention | 🚧 | In development - Attention computation with data parallelism | -| Quantization | 🚧 | In development - Model quantization for reduced memory usage | -| Multi-LoRA | 🚧 | In development - Serve multiple LoRA adapters simultaneously | - -### Attention Backend Comparison - -| Backend | Paged Attention | Spec Decoding | MLA | Sliding Window | -|---------|----------------|---------------|-----|----------------| -| FlashAttention (fa) | ✅ | ✅ | ❌ | ✅ | -| Native | ❌ | ❌ | ❌ | ❌ | - -**NOTE:** FlashAttention backend is recommended for production workloads due to superior memory efficiency and performance. - -## Optimized Model List - -The following models have been tested and optimized for TPU deployment: - -| Model Family | Performance Status | -|--------------|-------------------| -| [Qwen 3](https://huggingface.co/Qwen) | ⭐ Recommended for production | -| [Qwen 3 MoE](https://huggingface.co/Qwen) | ⭐ Best performance | -| [Qwen 2](https://huggingface.co/Qwen) | Needs improvement | -| [Qwen 2 MoE](https://huggingface.co/Qwen) | Needs improvement | -| [Qwen 1.5](https://huggingface.co/Qwen) | Needs improvement | -| [Llama/LLaMA](https://huggingface.co/meta-llama) | Needs improvement | -| [Grok-2](https://huggingface.co/xai-org) | Needs improvement | -| [Gemma 2](https://huggingface.co/google) | Verified on TPU | -| Bailing MoE | Needs improvement | - -## Installation - -### Method 1: Using PyPI (Recommended) - -```bash -pip install sglang-jax -``` - -### Method 2: From Source - -```bash -git clone https://github.com/sgl-project/sglang-jax -cd sglang-jax -uv venv --python 3.12 && source .venv/bin/activate -uv pip install -e "python[all]" -``` - -### Method 3: Using Docker - -**NOTE:** Docker support for TPU is currently under development. Please use PyPI or source installation methods. - -### Method 4: Cloud TPU with SkyPilot - -[SkyPilot](https://github.com/skypilot-org/skypilot) provides simplified deployment on Google Cloud TPU: - -1. Install SkyPilot and configure GCP access (see [SkyPilot documentation](https://skypilot.readthedocs.io/)) - -2. Create a SkyPilot configuration file: - -
-SkyPilot YAML: sglang-jax.sky.yaml - -```yaml -# sglang-jax.sky.yaml -resources: - accelerators: tpu-v6e-4 - accelerator_args: - tpu_vm: True - runtime_version: v2-alpha-tpuv6e - -run: | - git clone https://github.com/sgl-project/sglang-jax.git - cd sglang-jax - uv venv --python 3.12 - source .venv/bin/activate - uv pip install -e "python[all]" -``` - -
- -3. Launch your TPU cluster: - -```bash -# Standard deployment -sky launch -c sglang-jax sglang-jax.sky.yaml --infra=gcp - -# With spot instances for cost savings -sky launch -c sglang-jax sglang-jax.sky.yaml --infra=gcp --use-spot -``` - -## Launch of the Serving Engine - -### Basic Example: Qwen-7B - -```bash -JAX_COMPILATION_CACHE_DIR=/tmp/jit_cache python3 -u -m sgl_jax.launch_server \ - --model-path Qwen/Qwen-7B-Chat \ - --trust-remote-code \ - --dist-init-addr=0.0.0.0:10011 \ - --nnodes=1 \ - --tp-size=4 \ - --device=tpu \ - --random-seed=3 \ - --node-rank=0 \ - --mem-fraction-static=0.8 \ - --max-prefill-tokens=8192 \ - --download-dir=/tmp \ - --dtype=bfloat16 \ - --skip-server-warmup \ - --host 0.0.0.0 \ - --port 30000 -``` - -**Key Parameters Explained:** - -1. `JAX_COMPILATION_CACHE_DIR=/tmp/jit_cache` - Enables JIT compilation caching to accelerate server startup on subsequent runs -2. `--tp-size=4` - Tensor parallelism size; match this to your TPU core count (typically 1, 4, or 8) -3. `--device=tpu` - Specifies TPU device (this is the default for sglang-jax) -4. `--dtype=bfloat16` - Uses bfloat16 precision, which TPUs are optimized for -5. `--mem-fraction-static=0.8` - Allocates 80% of TPU HBM for static memory (adjustable from 0.2 to 0.9) -6. `--max-prefill-tokens=8192` - Maximum number of tokens processed in the prefill phase - -### High-Performance Configuration: Qwen3-8B - -For production workloads with optimal throughput: - -```bash -python3 -u -m sgl_jax.launch_server \ - --model-path Qwen/Qwen3-8B \ - --trust-remote-code \ - --tp-size=4 \ - --device=tpu \ - --mem-fraction-static=0.8 \ - --chunked-prefill-size=2048 \ - --dtype=bfloat16 \ - --max-running-requests=256 \ - --page-size=128 \ - --attention-backend=fa -``` - -### Advanced: Speculative Decoding (EAGLE3) - -Speculative decoding can improve throughput by 20-40% for compatible models: - -```bash -python3 -u -m sgl_jax.launch_server \ - --model-path Qwen/Qwen3-32B \ - --trust-remote-code \ - --device=tpu \ - --tp-size=4 \ - --mem-fraction-static=0.8 \ - --max-prefill-tokens=4096 \ - --attention-backend=fa \ - --dtype=bfloat16 \ - --port=30000 \ - --host=0.0.0.0 \ - --disable-overlap-schedule \ - --speculative-algorithm=EAGLE3 \ - --speculative-draft-model-path=AngelSlim/Qwen3-32B_eagle3 \ - --page-size=64 \ - --speculative-eagle-topk=1 \ - --speculative-num-steps=3 \ - --speculative-num-draft-tokens=4 -``` - -**NOTE:** Speculative decoding is currently supported for Qwen3 and LLaMA model families. See the [Speculative Decoding documentation](https://github.com/sgl-project/sglang-jax/blob/main/docs/features/speculative_decoding.md) for detailed configuration guidance. - - -### Multi-Node Distributed Serving - -For large models requiring multiple TPU VMs: - -```bash -# Node 0 (coordinator) -python3 -m sgl_jax.launch_server \ - --model-path MODEL_PATH \ - --dist-init-addr=NODE0_IP:10011 \ - --nnodes=2 \ - --node-rank=0 \ - --tp-size=8 \ - [other parameters...] - -# Node 1 (worker) -python3 -m sgl_jax.launch_server \ - --model-path MODEL_PATH \ - --dist-init-addr=NODE0_IP:10011 \ - --nnodes=2 \ - --node-rank=1 \ - --tp-size=8 \ - [other parameters...] -``` - -## Benchmarking with Requests - -### Throughput Testing - -Basic throughput benchmark: - -```bash -python3 -m sgl_jax.bench_serving \ - --backend sgl-jax \ - --dataset-name random \ - --num-prompts=100 \ - --random-input=512 \ - --random-output=128 \ - --max-concurrency=8 \ - --random-range-ratio=1 \ - --warmup-requests=0 -``` - -### Latency Testing - -Measure single-batch latency: - -```bash -python3 -m sgl_jax.bench_one_batch_server \ - --base-url http://127.0.0.1:30000 \ - --model-path Qwen/Qwen-7B-Chat \ - --batch-size=32 \ - --input-len=256 \ - --output-len=32 -``` - -### Comprehensive Benchmark Script - -For systematic performance evaluation across different configurations: - -```bash -#!/bin/bash -set -e - -backend=${1:-sgl-jax} -num_prompts_per_concurrency=3 -input_seq_lens=(1024 4096 8192) -output_seq_lens=(1 1024) -max_concurrencies=(8 16 32 64 128 256) - -for input_seq_len in "${input_seq_lens[@]}"; do - for output_seq_len in "${output_seq_lens[@]}"; do - echo "=======================================" - echo "Testing ISL/OSL: $input_seq_len/$output_seq_len" - echo "=======================================" - for max_concurrency in "${max_concurrencies[@]}"; do - num_prompts=$((num_prompts_per_concurrency * max_concurrency)) - python3 -m sgl_jax.bench_serving \ - --backend ${backend} \ - --dataset-name random \ - --num-prompts ${num_prompts} \ - --random-input ${input_seq_len} \ - --random-output ${output_seq_len} \ - --max-concurrency ${max_concurrency} \ - --random-range-ratio 1 \ - --disable-ignore-eos \ - --warmup-requests 0 - done - done -done -``` - -For detailed help on all benchmark parameters: - -```bash -python3 -m sgl_jax.bench_serving --help -``` - -See the [Benchmark and Profiling Guide](https://github.com/sgl-project/sglang-jax/blob/main/docs/developer_guide/benchmark_and_profiling.md) for advanced benchmarking techniques and profiling with JAX Profiler. - -## Performance Optimization - -### Memory Optimization - -**Reduce memory usage:** -- Lower `--mem-fraction-static` (from 0.8 → 0.5 → 0.3) -- Decrease `--max-prefill-tokens` (from 16384 → 8192 → 4096) -- Reduce `--max-running-requests` - -**Handle OOM errors:** -- Start with conservative memory settings (`--mem-fraction-static=0.5`) -- Gradually increase until you find the optimal balance -- Increase `--page-size` for better memory locality (1 → 16 → 64 → 128) - -### Throughput Optimization - -To maximize tokens per second: - -- Use FlashAttention backend: `--attention-backend=fa` -- Enable speculative decoding (EAGLE3) for Qwen3 models (20-40% improvement) -- Increase `--max-running-requests` to 256+ -- Set `--mem-fraction-static` to 0.8+ (if memory allows) -- Use larger page sizes (64-128) -- Enable chunked prefill: `--chunked-prefill-size=2048` - -### Latency Optimization - -To minimize time-to-first-token (TTFT) and inter-token latency: - -- Reduce `--page-size` to 1-4 -- Lower `--max-running-requests` (16-32) for smaller batches -- Reduce `--chunked-prefill-size` -- Use conservative memory settings to avoid GC pauses - -### TPU-Specific Optimizations - -1. **JIT Compilation Cache:** - ```bash - export JAX_COMPILATION_CACHE_DIR=/tmp/jit_cache - ``` - Always set this environment variable to cache compiled kernels and accelerate server startup. - -2. **Data Type Optimization:** - Use `--dtype=bfloat16` for TPU native optimization. TPUs are specifically designed for bfloat16 computations. - -3. **Tensor Parallelism:** - Match `--tp-size` to your TPU core configuration (1, 4, or 8) for optimal model distribution. - -4. **Attention Backend:** - Always use `--attention-backend=fa` (FlashAttention) for production workloads. - -## Troubleshooting - -### OOM (Out of Memory) Errors - -If you encounter out-of-memory errors: - -1. Reduce `--mem-fraction-static` from 0.8 to 0.5 or lower -2. Decrease `--max-prefill-tokens` from 8192 to 4096 or 2048 -3. Lower `--max-running-requests` to reduce concurrent batch size -4. Increase `--page-size` for better memory layout efficiency - -### Compilation Long-Time - -If the server takes too long to start: - -1. Ensure `JAX_COMPILATION_CACHE_DIR` is properly set -2. Understand that the first run requires JIT compilation (this is normal) -3. Subsequent runs will be significantly faster with cached compilations -4. Consider using `--skip-server-warmup` to defer compilation until first request - -### Low Throughput - -If you're not achieving expected throughput: - -1. Verify `--tp-size` matches your TPU core configuration -2. Check that `--attention-backend=fa` is enabled -3. Increase `--max-running-requests` to enable larger batch formation -4. Consider enabling speculative decoding for compatible models -5. Ensure memory settings allow for sufficient batch sizes - -### Connection Issues - -If clients cannot connect to the server: - -1. Ensure `--host=0.0.0.0` for external access (not just `127.0.0.1`) -2. Verify firewall rules allow traffic on the specified port (default: 30000) -3. Check that the server process is running: `curl http://localhost:30000/health` - -## Advanced Features - -### Speculative Decoding - -SGLang-JAX supports EAGLE and EAGLE3 speculative decoding algorithms for Qwen3 and LLaMA model families. Speculative decoding can improve throughput by 20-40% without affecting output quality. - -See the [Speculative Decoding documentation](https://github.com/sgl-project/sglang-jax/blob/main/docs/features/speculative_decoding.md) for detailed configuration and supported model combinations. - -### Chunked Prefill - -Enable mixed prefill-decode batching for better TPU utilization: - -```bash ---chunked-prefill-size=2048 --enable-mixed-chunk -``` - -This allows the scheduler to mix prefill operations with decode operations in the same batch, improving overall throughput. - -### Custom Attention Backends - -SGLang-JAX supports a plugin-based attention backend system. You can implement custom attention kernels optimized for specific use cases. - -See the [Attention Backend documentation](https://github.com/sgl-project/sglang-jax/blob/main/docs/features/attention_backend.md) for implementation details. - -### Environment Verification - -Verify your TPU setup before deploying: - -```bash -python -c "from sgl_jax import check_env; check_env.check_env()" -``` - -This command checks: -- Installed package versions -- TPU device availability and specifications -- System resources and configuration -- Compatibility of settings - -## Contributing - -We welcome contributions to improve TPU support in SGLang-JAX! - -### Areas for Contribution - -**Check the [Development Roadmap](https://github.com/sgl-project/sglang-jax/issues/190)** to see planned features and find opportunities to contribute new functionality. - -Current contribution areas include: - -- Performance optimizations for specific TPU generations -- Support for additional model architectures -- Documentation improvements and examples -- Bug reports and fixes -- Benchmark results and performance analysis - -### How to Contribute - -1. Visit the [sglang-jax repository](https://github.com/sgl-project/sglang-jax) -2. Read the [Contribution Guide](https://github.com/sgl-project/sglang-jax/blob/main/docs/developer_guide/contribution_guide.md) -3. Join the [SGL-JAX Slack community](https://sgl-fru7574.slack.com/archives/C09EBE5HT5X) for discussions -4. Report issues at [sglang-jax/issues](https://github.com/sgl-project/sglang-jax/issues) - -### Testing on TPU - -For contributors who need TPU access for testing: - -- Refer to the [TPU Resources Guide](https://github.com/sgl-project/sglang-jax/blob/main/docs/developer_guide/tpu_resources_guide.md) for information on accessing TPU hardware -- Use SkyPilot with spot instances for cost-effective testing -- Follow the [Benchmark and Profiling Guide](https://github.com/sgl-project/sglang-jax/blob/main/docs/developer_guide/benchmark_and_profiling.md) for performance validation - -## References - -### Documentation - -- [SGLang-JAX Repository](https://github.com/sgl-project/sglang-jax) -- [SGLang-JAX Installation Guide](https://github.com/sgl-project/sglang-jax/blob/main/docs/get_started/install.md) -- [Qwen Models Quick Start](https://github.com/sgl-project/sglang-jax/blob/main/docs/basic_usage/qwen.md) -- [Benchmark and Profiling Guide](https://github.com/sgl-project/sglang-jax/blob/main/docs/developer_guide/benchmark_and_profiling.md) -- [Speculative Decoding](https://github.com/sgl-project/sglang-jax/blob/main/docs/features/speculative_decoding.md) - -### External Resources - -- [JAX Documentation](https://jax.readthedocs.io/) -- [Google Cloud TPU Documentation](https://cloud.google.com/tpu/docs) -- [SkyPilot Documentation](https://skypilot.readthedocs.io/) diff --git a/docs/platforms/xpu.md b/docs/platforms/xpu.md deleted file mode 100644 index 1ba56b192..000000000 --- a/docs/platforms/xpu.md +++ /dev/null @@ -1,143 +0,0 @@ -# XPU - -The document addresses how to set up the [SGLang](https://github.com/sgl-project/sglang) environment and run LLM inference on Intel GPU, [see more context about Intel GPU support within PyTorch ecosystem](https://docs.pytorch.org/docs/stable/notes/get_start_xpu.html). - -Specifically, SGLang is optimized for [Intel® Arc™ Pro B-Series Graphics](https://www.intel.com/content/www/us/en/ark/products/series/242616/intel-arc-pro-b-series-graphics.html) and [ -Intel® Arc™ B-Series Graphics](https://www.intel.com/content/www/us/en/ark/products/series/240391/intel-arc-b-series-graphics.html). - -## Optimized Model List - -A list of LLMs have been optimized on Intel GPU, and more are on the way: - -| Model Name | BF16 | -|:---:|:---:| -| Llama-3.2-3B | [meta-llama/Llama-3.2-3B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct) | -| Llama-3.1-8B | [meta-llama/Llama-3.1-8B-Instruct](https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct) | -| Qwen2.5-1.5B | [Qwen/Qwen2.5-1.5B](https://huggingface.co/Qwen/Qwen2.5-1.5B) | - -**Note:** The model identifiers listed in the table above -have been verified on [Intel® Arc™ B580 Graphics](https://www.intel.com/content/www/us/en/products/sku/241598/intel-arc-b580-graphics/specifications.html). - -## Installation - -### Install From Source - -Currently SGLang XPU only supports installation from source. Please refer to ["Getting Started on Intel GPU"](https://docs.pytorch.org/docs/stable/notes/get_start_xpu.html) to install XPU dependency. - -```bash -# Create and activate a conda environment -conda create -n sgl-xpu python=3.12 -y -conda activate sgl-xpu - -# Set PyTorch XPU as primary pip install channel to avoid installing the larger CUDA-enabled version and prevent potential runtime issues. -pip3 install torch==2.11.0+xpu torchao torchvision torchaudio==2.11.0+xpu --index-url https://download.pytorch.org/whl/xpu -pip3 install xgrammar --no-deps # xgrammar will introduce CUDA-enabled triton which might conflict with XPU - -# Clone the SGLang code -git clone https://github.com/sgl-project/sglang.git -cd sglang -git checkout - -# Use dedicated toml file -cd python -cp pyproject_xpu.toml pyproject.toml -# Install SGLang dependent libs, and build SGLang main package -pip install --upgrade pip setuptools -pip install -v . --extra-index-url https://download.pytorch.org/whl/xpu -``` - -### Install Using Docker - -The docker for XPU is under active development. Please stay tuned. - -## Launch of the Serving Engine - -Example command to launch SGLang serving: - -```bash -python -m sglang.launch_server \ - --model \ - --trust-remote-code \ - --disable-overlap-schedule \ - --device xpu \ - --host 0.0.0.0 \ - --tp 2 \ # using multi GPUs - --attention-backend intel_xpu \ # using intel optimized XPU attention backend - --page-size \ # intel_xpu attention backend supports [32, 64, 128] -``` - -## Benchmarking with Requests - -You can benchmark the performance via the `bench_serving` script. -Run the command in another terminal. - -```bash -python -m sglang.bench_serving \ - --dataset-name random \ - --random-input-len 1024 \ - --random-output-len 1024 \ - --num-prompts 1 \ - --request-rate inf \ - --random-range-ratio 1.0 -``` - -The detail explanations of the parameters can be looked up by the command: - -```bash -python -m sglang.bench_serving -h -``` - -Additionally, the requests can be formed with -[OpenAI Completions API](https://docs.sglang.io/basic_usage/openai_api_completions.html) -and sent via the command line (e.g. using `curl`) or via your own script. - -## Prefill-Decode (P/D) Disaggregation on Intel XPU [Experimental] - -SGLang supports prefill-decode disaggregation on Intel XPU using the [NIXL](https://github.com/ai-dynamo/nixl) KV-transfer backend. - -**Tested models:** - -| Model | Notes | -|:---:|:---:| -| [Qwen/Qwen3-0.6B](https://huggingface.co/Qwen/Qwen3-0.6B) | Used in integration tests; verified on Intel XPU with homogeneous P/D (XPU prefill + XPU decode) | -| [Qwen/Qwen2.5-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-7B-Instruct) | Verified on Intel XPU with homogeneous P/D (XPU prefill + XPU decode) | - -**Prerequisites:** `pip install nixl sglang-router` - -**Start the prefill server (GPU 0):** - -```bash -ZE_AFFINITY_MASK=0 UCX_POSIX_USE_PROC_LINK=n python -m sglang.launch_server \ - --model-path Qwen/Qwen3-0.6B --trust-remote-code --device xpu \ - --disaggregation-mode prefill --disaggregation-transfer-backend nixl \ - --disaggregation-bootstrap-port 12335 --host 0.0.0.0 --port 30000 -``` - -**Start the decode server (GPU 1):** - -```bash -ZE_AFFINITY_MASK=1 UCX_POSIX_USE_PROC_LINK=n python -m sglang.launch_server \ - --model-path Qwen/Qwen3-0.6B --trust-remote-code --device xpu \ - --disaggregation-mode decode --disaggregation-transfer-backend nixl \ - --disaggregation-bootstrap-port 12335 --host 0.0.0.0 --port 30001 -``` - -**Start the router:** - -```bash -python -m sglang_router.launch_router \ - --pd-disaggregation \ - --prefill http://127.0.0.1:30000 \ - --decode http://127.0.0.1:30001 \ - --host 0.0.0.0 --port 8000 -``` - -**Send a request:** - -```bash -curl http://127.0.0.1:8000/v1/completions \ - -H "Content-Type: application/json" \ - -d '{"model": "Qwen/Qwen3-0.6B", "prompt": "The capital of France is", "max_tokens": 32}' -``` - -> **Note:** `UCX_POSIX_USE_PROC_LINK=n` is required on Intel XPU to avoid UCX shared-memory transport issues. diff --git a/docs/references/custom_chat_template.md b/docs/references/custom_chat_template.md deleted file mode 100644 index 870d09c1c..000000000 --- a/docs/references/custom_chat_template.md +++ /dev/null @@ -1,51 +0,0 @@ -# Custom Chat Template - -**NOTE**: There are two chat template systems in SGLang project. This document is about setting a custom chat template for the OpenAI-compatible API server (defined at [conversation.py](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/parser/conversation.py)). It is NOT related to the chat template used in the SGLang language frontend (defined at [chat_template.py](https://github.com/sgl-project/sglang/blob/main/python/sglang/lang/chat_template.py)). - -By default, the server uses the chat template specified in the model tokenizer from Hugging Face. -It should just work for most official models such as Llama-2/Llama-3. - -If needed, you can also override the chat template when launching the server: - -```bash -python -m sglang.launch_server \ - --model-path meta-llama/Llama-2-7b-chat-hf \ - --port 30000 \ - --chat-template llama-2 -``` - -If the chat template you are looking for is missing, you are welcome to contribute it or load it from a file. - -## JSON Format - -You can load the JSON format, which is defined by `conversation.py`. - -```json -{ - "name": "my_model", - "system": "<|im_start|>system", - "user": "<|im_start|>user", - "assistant": "<|im_start|>assistant", - "sep_style": "CHATML", - "sep": "<|im_end|>", - "stop_str": ["<|im_end|>", "<|im_start|>"] -} -``` - -```bash -python -m sglang.launch_server \ - --model-path meta-llama/Llama-2-7b-chat-hf \ - --port 30000 \ - --chat-template ./my_model_template.json -``` - -## Jinja Format - -You can also use the [Jinja template format](https://huggingface.co/docs/transformers/main/en/chat_templating) as defined by Hugging Face Transformers. - -```bash -python -m sglang.launch_server \ - --model-path meta-llama/Llama-2-7b-chat-hf \ - --port 30000 \ - --chat-template ./my_model_template.jinja -``` diff --git a/docs/references/environment_variables.md b/docs/references/environment_variables.md deleted file mode 100644 index 63cb9c837..000000000 --- a/docs/references/environment_variables.md +++ /dev/null @@ -1,199 +0,0 @@ -# Environment Variables - -SGLang supports various environment variables that can be used to configure its runtime behavior. This document provides a comprehensive list and aims to stay updated over time. - -*Note: SGLang uses two prefixes for environment variables: `SGL_` and `SGLANG_`. This is likely due to historical reasons. While both are currently supported for different settings, future versions might consolidate them.* - -## General Configuration - -| Environment Variable | Description | Default Value | -|-------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------|------------------------------| -| `SGLANG_USE_MODELSCOPE` | Enable using models from ModelScope | `false` | -| `SGLANG_HOST_IP` | Host IP address for the server | `0.0.0.0` | -| `SGLANG_PORT` | Port for the server | auto-detected | -| `SGLANG_LOGGING_CONFIG_PATH` | Custom logging configuration path | Not set | -| `SGLANG_LOG_REQUEST_HEADERS` | Comma-separated list of additional HTTP headers to log when `--log-requests` is enabled. Appends to the default `x-smg-routing-key`. | Not set | -| `SGLANG_HEALTH_CHECK_TIMEOUT` | Timeout for health check in seconds | `20` | -| `SGLANG_EPLB_HEATMAP_COLLECTION_INTERVAL` | The interval of passes to collect the metric of selected count of physical experts on each layer and GPU rank. 0 means disabled. | `0` | -| `SGLANG_FORWARD_UNKNOWN_TOOLS` | Forward unknown tool calls to clients instead of dropping them | `false` (drop unknown tools) | -| `SGLANG_REQ_WAITING_TIMEOUT` | Timeout (in seconds) for requests waiting in the queue before being scheduled | `-1` | -| `SGLANG_REQ_RUNNING_TIMEOUT` | Timeout (in seconds) for requests running in the decode batch | `-1` | -| `SGLANG_CACHE_DIR` | Cache directory for model weights and other data | `~/.cache/sglang` | -| `SGLANG_PREFETCH_BLOCK_SIZE_MB` | Block size (in MB) for sequential checkpoint prefetch reads that warm the OS page cache before workers load weights via mmap | `16` | - -## Performance Tuning - -| Environment Variable | Description | Default Value | -| --- | --- | --- | -| `SGLANG_ENABLE_TORCH_INFERENCE_MODE` | Control whether to use torch.inference_mode | `false` | -| `SGLANG_ENABLE_TORCH_COMPILE` | Enable torch.compile | `false` | -| `SGLANG_SET_CPU_AFFINITY` | Enable CPU affinity setting (often set to `1` in Docker builds) | `false` | -| `SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN` | Allows the scheduler to overwrite longer context length requests (often set to `1` in Docker builds) | `false` | -| `SGLANG_IS_FLASHINFER_AVAILABLE` | Control FlashInfer availability check | `true` | -| `SGLANG_SKIP_P2P_CHECK` | Skip P2P (peer-to-peer) access check | `false` | -| `SGLANG_CHUNKED_PREFIX_CACHE_THRESHOLD` | Sets the threshold for enabling chunked prefix caching | `8192` | -| `SGLANG_MAX_KV_CHUNK_CAPACITY` | Maximum number of tokens in each KV chunk for DeepSeek MHA chunked prefix cache | `131072` | -| `SGLANG_FUSED_MLA_ENABLE_ROPE_FUSION` | Enable RoPE fusion in Fused Multi-Layer Attention | `1` | -| `SGLANG_DISABLE_CONSECUTIVE_PREFILL_OVERLAP` | Disable overlap schedule for consecutive prefill batches | `false` | -| `SGLANG_SCHEDULER_MAX_RECV_PER_POLL` | Set the maximum number of requests per poll, with a negative value indicating no limit | `-1` | -| `SGLANG_DISABLE_FA4_WARMUP` | Disable Flash Attention 4 warmup passes (set to `1`, `true`, `yes`, or `on` to disable) | `false` | -| `SGLANG_DATA_PARALLEL_BUDGET_INTERVAL` | Interval for DPBudget updates | `1` | -| `SGLANG_SCHEDULER_RECV_SKIPPER_WEIGHT_DEFAULT` | Default weight value for scheduler recv skipper counter (used when forward mode doesn't match specific modes). Only active when `--scheduler-recv-interval > 1`. The counter accumulates weights and triggers request polling when reaching the interval threshold. | `1000` | -| `SGLANG_SCHEDULER_RECV_SKIPPER_WEIGHT_DECODE` | Weight increment for decode forward mode in scheduler recv skipper. Works with `--scheduler-recv-interval` to control polling frequency during decode phase. | `1` | -| `SGLANG_SCHEDULER_RECV_SKIPPER_WEIGHT_TARGET_VERIFY` | Weight increment for target verify forward mode in scheduler recv skipper. Works with `--scheduler-recv-interval` to control polling frequency during verification phase. | `1` | -| `SGLANG_SCHEDULER_RECV_SKIPPER_WEIGHT_NONE` | Weight increment when forward mode is None in scheduler recv skipper. Works with `--scheduler-recv-interval` to control polling frequency when no specific forward mode is active. | `1` | -| `SGLANG_MM_BUFFER_SIZE_MB` | Size of preallocated GPU buffer (in MB) for multi-modal feature hashing optimization. When set to a positive value, temporarily moves features to GPU for faster hash computation, then moves them back to CPU to save GPU memory. Larger features benefit more from GPU hashing. Set to `0` to disable. | `0` | -| `SGLANG_MM_PRECOMPUTE_HASH` | Enable precomputing of hash values for MultimodalDataItem | `false` | -| `SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH` | Enable NCCL for gathering when preparing mlp sync batch under overlap scheduler (without this flag gloo is used for gathering) | `false` | -| `SGLANG_SYMM_MEM_PREALLOC_GB_SIZE` | Size of preallocated GPU buffer (in GB) for NCCL symmetric memory pool to limit memory fragmentation. Only have an effect when server arg `--enable-symm-mem` is set. | `-1` | -| `SGLANG_CUSTOM_ALLREDUCE_ALGO` | The algorithm of custom all-reduce. Set to `oneshot` or `1stage` to force use one-shot. Set to `twoshot` or `2stage` to force use two-shot. | `` | -| `SGLANG_SKIP_SOFTMAX_PREFILL_THRESHOLD_SCALE_FACTOR` | Skip-softmax threshold scale factor for TRT-LLM prefill attention in flashinfer. `None` means standard attention. See https://arxiv.org/abs/2512.12087 | `None` | -| `SGLANG_SKIP_SOFTMAX_DECODE_THRESHOLD_SCALE_FACTOR` | Skip-softmax threshold scale factor for TRT-LLM decode attention in flashinfer. `None` means standard attention. See https://arxiv.org/abs/2512.12087 | `None` | -| `SGLANG_USE_SGL_FA3_KERNEL` | Use sgl-kernel implementation for FlashAttention v3 | `true` | - - -## DeepGEMM Configuration (Advanced Optimization) - -| Environment Variable | Description | Default Value | -| --- | --- | --- | -| `SGLANG_ENABLE_JIT_DEEPGEMM` | Enable Just-In-Time compilation of DeepGEMM kernels (enabled by default on NVIDIA Hopper (SM90) and Blackwell (SM100) GPUs when the DeepGEMM package is installed; set to `"0"` to disable) | `"true"` | -| `SGLANG_JIT_DEEPGEMM_PRECOMPILE` | Enable precompilation of DeepGEMM kernels | `"true"` | -| `SGLANG_JIT_DEEPGEMM_COMPILE_WORKERS` | Number of workers for parallel DeepGEMM kernel compilation | `4` | -| `SGLANG_IN_DEEPGEMM_PRECOMPILE_STAGE` | Indicator flag used during the DeepGEMM precompile script | `"false"` | -| `SGLANG_DG_CACHE_DIR` | Directory for caching compiled DeepGEMM kernels | `~/.cache/deep_gemm` | -| `SGLANG_DG_USE_NVRTC` | Use NVRTC (instead of Triton) for JIT compilation (Experimental) | `"false"` | -| `SGLANG_USE_DEEPGEMM_BMM` | Use DeepGEMM for Batched Matrix Multiplication (BMM) operations | `"false"` | -| `SGLANG_JIT_DEEPGEMM_FAST_WARMUP` | Precompile less kernels during warmup, which reduces the warmup time from 30min to less than 3min. Might cause performance degradation during runtime. | `"false"` | - -## DeepEP Configuration - -| Environment Variable | Description | Default Value | -| --- | --- | --- | -| `SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK` | The maximum number of dispatched tokens on each GPU | `"128"` | -| `SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK` | The maximum number of dispatched tokens on each GPU for --moe-a2a-backend=flashinfer | `"1024"` | -| `SGLANG_DEEPEP_LL_COMBINE_SEND_NUM_SMS` | Number of SMs used for DeepEP combine when single batch overlap is enabled | `"32"` | -| `SGLANG_BLACKWELL_OVERLAP_SHARED_EXPERTS_OUTSIDE_SBO` | Run shared experts on an alternate stream when single batch overlap is enabled on GB200. When not setting this flag, shared experts and down gemm will be overlapped with DeepEP combine together. | `"false"` | - -## MORI Configuration - -| Environment Variable | Description | Default Value | -| --- | --- | --- | -| `SGLANG_MORI_DISPATCH_DTYPE` | Override MoRI-EP dispatch quantization type. `auto` uses auto-detection from weight dtype; `bf16`/`fp8`/`fp4` forces the specified type for all layers | `"auto"` | -| `SGLANG_MORI_FP8_COMB` | Use FP8 for combine | `"false"` | -| `SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK` | Maximum number of dispatch tokens per rank for MORI-EP buffer allocation | `4096` | -| `SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD` | Threshold for switching between `InterNodeV1` and `InterNodeV1LL` kernel types. `InterNodeV1LL` is used if `SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK` is less than or equal to this threshold; otherwise, `InterNodeV1` is used. | `256` | -| `SGLANG_MORI_PREALLOC_MAX_RECV_TOKENS` | This argument devives `SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK` which indicates customized amount of tokens preallocated for a rank, valid range from 1 to world_size*SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK, by default `0` means maximum. Setting a smaller value will reduce memory footprint but too small value could cause buffer overflow. | `0` | -| `SGLANG_MORI_MOE_MAX_INPUT_TOKENS` | Truncate the dispatch buffer to this many rows before MoE computation, reducing kernel overhead on padding tokens. The value must be >= the actual number of received tokens (`totalRecvTokenNum`); setting it too small causes incorrect results. `0` disables truncation (use full buffer). | `0` | -| `SGLANG_MORI_QP_PER_TRANSFER` | Number of RDMA Queue Pairs (QPs) used per transfer operation | `1` | -| `SGLANG_MORI_POST_BATCH_SIZE` | Number of RDMA work requests posted in a single batch to each QP | `-1` | -| `SGLANG_MORI_NUM_WORKERS` | Number of worker threads in the RDMA executor thread pool | `1` | - -## DSA Backend Configuration (For DeepSeek V3.2) - - - -| Environment Variable | Description | Default Value | -| --- | --- | --- | -| `SGLANG_DSA_FUSE_TOPK` | Fuse the operation of picking topk logits and picking topk indices from page table (`SGLANG_NSA_FUSE_TOPK` is a deprecated alias) | `true` | -| `SGLANG_DSA_TOPK_FLASHINFER_DETERMINISTIC` | Use deterministic FlashInfer topk kernels when `--dsa-topk-backend=flashinfer` | `false` | -| `SGLANG_DSA_TOPK_FLASHINFER_TIE_BREAK` | Tie-break mode for FlashInfer DSA topk when `--dsa-topk-backend=flashinfer`: unset disables explicit tie-breaking, `small` prefers the smaller candidate index for equal scores, and `large` prefers the larger candidate index for equal scores. Setting this variable makes FlashInfer use deterministic topk. | `unset` | -| `SGLANG_DSA_ENABLE_MTP_PRECOMPUTE_METADATA` | Precompute metadata that can be shared among different draft steps when MTP is enabled (`SGLANG_NSA_ENABLE_MTP_PRECOMPUTE_METADATA` is a deprecated alias) | `true` | -| `SGLANG_USE_FUSED_METADATA_COPY` | Control whether to use fused metadata copy kernel for cuda graph replay | `true` | -| `SGLANG_DSA_PREFILL_DENSE_ATTN_KV_LEN_THRESHOLD` | When the maximum kv len in current prefill batch exceeds this value, the sparse mla kernel will be applied, else it falls back to dense MHA implementation. Default to the index topk of model (2048 for DeepSeek V3.2) (`SGLANG_NSA_PREFILL_DENSE_ATTN_KV_LEN_THRESHOLD` is a deprecated alias) | `2048` | - - -## Memory Management - -| Environment Variable | Description | Default Value | -| --- | --- | --- | -| `SGLANG_DEBUG_MEMORY_POOL` | Enable memory pool debugging | `false` | -| `SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION` | Clip max new tokens estimation for memory planning | `4096` | -| `SGLANG_DETOKENIZER_MAX_STATES` | Maximum states for detokenizer | Default value based on system | -| `SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK` | Enable checks for memory imbalance across Tensor Parallel ranks | `true` | -| `SGLANG_MOONCAKE_CUSTOM_MEM_POOL` | Configure the custom memory pool type for Mooncake. Supports `NVLINK`, `BAREX`, `INTRA_NODE_NVLINK`. If set to `true`, it defaults to `NVLINK`. | `None` | - -## Model-Specific Options - -| Environment Variable | Description | Default Value | -| --- | --- | --- | -| `SGLANG_USE_AITER` | Use AITER optimize implementation | `false` | -| `SGLANG_ROCM_USE_MULTI_STREAM` | Allocate alt CUDA/HIP stream on ROCm/AITER to overlap shared and routed experts in DeepseekV2 MoE. Requires the HIP env `GPU_MAX_HW_QUEUES>=5` (default `4`, the cap on HSA/ROCr HW queues HIP creates) so the alt stream gets its own queue instead of serializing with the main stream. Best paired with `--deepep-mode low_latency` so Mori's AsyncLL kernel offloads dispatch/combine to copy engines and frees CUs. | `false` | -| `SGLANG_MOE_PADDING` | Enable MoE padding (sets padding size to 128 if value is `1`, often set to `1` in Docker builds) | `false` | -| `SGLANG_CUTLASS_MOE` (deprecated) | Use Cutlass FP8 MoE kernel on Blackwell GPUs (deprecated, use --moe-runner-backend=cutlass) | `false` | - -## Quantization - -| Environment Variable | Description | Default Value | -| --- | --- | --- | -| `SGLANG_INT4_WEIGHT` | Enable INT4 weight quantization | `false` | -| `SGLANG_FORCE_FP8_MARLIN` | Force using FP8 MARLIN kernels even if other FP8 kernels are available | `false` | -| `SGLANG_NVFP4_CKPT_FP8_GEMM_IN_ATTN` | Quantize q_b_proj from BF16 to FP8 when launching DeepSeek NVFP4 checkpoint | `false` | -| `SGLANG_MOE_NVFP4_DISPATCH` | Use nvfp4 for moe dispatch (on flashinfer_cutlass or flashinfer_cutedsl moe runner backend) | `"false"` | -| `SGLANG_FLASHINFER_NVFP4_PER_TOKEN_ACTIVATION` | Enable FlashInfer TRTLLM per-token NVFP4 activation scaling; ignores checkpoint activation FP32 scale by treating it as `1` | `false` | -| `SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE` | Quantize moe of nextn layer from BF16 to FP8 when launching DeepSeek NVFP4 checkpoint | `false` | -| `SGLANG_QUANT_ALLOW_DOWNCASTING` | Allow weight dtype downcasting during loading (e.g., fp32 → fp16). By default, SGLang rejects this kind of downcasting when using quantization. | `false` | -| `SGLANG_FP8_IGNORED_LAYERS` | A comma-separated list of layer names to ignore during FP8 quantization. For example: `model.layers.0,model.layers.1.,qkv_proj`. | `""` | - - -## Distributed Computing - -| Environment Variable | Description | Default Value | -| --- | --- | --- | -| `SGLANG_BLOCK_NONZERO_RANK_CHILDREN` | Control blocking of non-zero rank children processes | `1` | -| `SGLANG_IS_FIRST_RANK_ON_NODE` | Indicates if the current process is the first rank on its node | `"true"` | -| `SGLANG_PP_LAYER_PARTITION` | Pipeline parallel layer partition specification | Not set | -| `SGLANG_ONE_VISIBLE_DEVICE_PER_PROCESS` | Set one visible device per process for distributed computing | `false` | - -## PD Disaggregation — Staging Buffer (Heterogeneous TP) - -| Environment Variable | Description | Default Value | -| --- | --- | --- | -| `SGLANG_DISAGG_STAGING_BUFFER` | Enable GPU staging buffer for heterogeneous TP KV transfer. Required when prefill and decode use different TP/attention-TP sizes. Only for non-MLA models (e.g. GQA, MHA). | `false` | -| `SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB` | Prefill-side per-worker staging buffer size in MB. Used for gathering KV head slices before bulk RDMA transfer. | `64` | -| `SGLANG_DISAGG_STAGING_POOL_SIZE_MB` | Decode-side ring buffer pool total size in MB. Shared buffer receiving RDMA data from all prefill ranks. Larger values support higher concurrency. | `4096` | -| `SGLANG_STAGING_USE_TORCH` | Force using PyTorch gather/scatter fallback instead of Triton fused kernels for staging operations. Useful for debugging. | `false` | - -## Testing & Debugging (Internal/CI) - -*These variables are primarily used for internal testing, continuous integration, or debugging.* - -| Environment Variable | Description | Default Value | -| --- | --- | --- | -| `SGLANG_IS_IN_CI` | Indicates if running in CI environment | `false` | -| `SGLANG_IS_IN_CI_AMD` | Indicates running in AMD CI environment | `false` | -| `SGLANG_TEST_RETRACT` | Enable retract decode testing | `false` | -| `SGLANG_TEST_RETRACT_NO_PREFILL_BS` | When SGLANG_TEST_RETRACT is enabled, no prefill is performed if the batch size exceeds SGLANG_TEST_RETRACT_NO_PREFILL_BS. | `2 ** 31` | -| `SGLANG_RECORD_STEP_TIME` | Record step time for profiling | `false` | -| `SGLANG_TEST_REQUEST_TIME_STATS` | Test request time statistics | `false` | -| `SGLANG_DEBUG_SYMM_MEM` | Enable debug checks that verify tensors passed to NCCL communication ops are allocated in the symmetric memory pool. Logs warnings (rank 0 only) with stack traces for any tensor not in the pool. | `false` | -| `SGLANG_KERNEL_API_LOGLEVEL` | Controls crash-debug kernel API logging. `0` disables logging, `1` logs API names, `3` logs tensor metadata, `5` adds tensor statistics, and `10` also writes pre-call dump snapshots. | `0` | -| `SGLANG_KERNEL_API_LOGDEST` | Destination for crash-debug kernel API logs. Use `stdout`, `stderr`, or a file path. `%i` is replaced with the process PID. | `stdout` | -| `SGLANG_KERNEL_API_DUMP_DIR` | Output directory for level-10 kernel API input/output dumps. `%i` is replaced with the process PID. | `sglang_kernel_api_dumps` | -| `SGLANG_KERNEL_API_DUMP_INCLUDE` | Comma-separated wildcard patterns for kernel API names to include in level-10 dumps. | Not set | -| `SGLANG_KERNEL_API_DUMP_EXCLUDE` | Comma-separated wildcard patterns for kernel API names to exclude from level-10 dumps. | Not set | - -## Profiling & Benchmarking - -| Environment Variable | Description | Default Value | -| --- | --- | --- | -| `SGLANG_TORCH_PROFILER_DIR` | Directory for PyTorch profiler output | `/tmp` | -| `SGLANG_PROFILE_WITH_STACK` | Set `with_stack` option (bool) for PyTorch profiler (capture stack trace) | `true` | -| `SGLANG_PROFILE_RECORD_SHAPES` | Set `record_shapes` option (bool) for PyTorch profiler (record shapes) | `true` | -| `SGLANG_OTLP_EXPORTER_SCHEDULE_DELAY_MILLIS` | Config BatchSpanProcessor.schedule_delay_millis if tracing is enabled | `500` | -| `SGLANG_OTLP_EXPORTER_MAX_EXPORT_BATCH_SIZE` | Config BatchSpanProcessor.max_export_batch_size if tracing is enabled | `64` | - -## Storage & Caching - -| Environment Variable | Description | Default Value | -| --- | --- | --- | -| `SGLANG_WAIT_WEIGHTS_READY_TIMEOUT` | Timeout period for waiting on weights | `120` | -| `SGLANG_DISABLE_OUTLINES_DISK_CACHE` | Disable Outlines disk cache | `false` | -| `SGLANG_USE_CUSTOM_TRITON_KERNEL_CACHE` | Use SGLang's custom Triton kernel cache implementation for lower overheads (automatically enabled on CUDA) | `false` | -| `SGLANG_HICACHE_DECODE_OFFLOAD_STRIDE` | Decode-side incremental KV cache offload stride. Rounded down to a multiple of `--page-size` (min is `--page-size`). If unset/invalid/<=0, it falls back to `--page-size`. | Not set (uses `--page-size`) | - - -## Function Calling / Tool Use - -| Environment Variable | Description | Default Value | -| --- | --- | --- | -| `SGLANG_TOOL_STRICT_LEVEL` | Controls the strictness level of tool call parsing and validation.
**Level 0**: Off - No strict validation
**Level 1**: Function strict - Enables structural tag constraints for all tools (even if none have `strict=True` set)
**Level 2**: Parameter strict - Enforces strict parameter validation for all tools, treating them as if they all have `strict=True` set | `0` | diff --git a/docs/references/faq.md b/docs/references/faq.md deleted file mode 100644 index ffa1a7c54..000000000 --- a/docs/references/faq.md +++ /dev/null @@ -1,39 +0,0 @@ -# Troubleshooting and Frequently Asked Questions - -## Troubleshooting - -This page lists common errors and tips for resolving them. - -### CUDA Out of Memory -If you encounter out-of-memory (OOM) errors, you can adjust the following parameters: - -- If OOM occurs during prefill, try reducing `--chunked-prefill-size` to `4096` or `2048`. This saves memory but slows down the prefill speed for long prompts. -- If OOM occurs during decoding, try lowering `--max-running-requests`. -- You can also decrease `--mem-fraction-static` to a smaller value, such as 0.8 or 0.7. This decreases the memory usage of the KV cache memory pool and helps prevent OOM errors during both prefill and decoding. However, it limits maximum concurrency and reduces peak throughput. -- Another common case for OOM is requesting input logprobs for a long prompt as it requires significant memory. To address this, set `logprob_start_len` in your sampling parameters to include only the necessary parts. If you do need input logprobs for a long prompt, try reducing `--mem-fraction-static`. - -### CUDA Error: Illegal Memory Access Encountered -This error may result from kernel errors or out-of-memory issues: -- If it is a kernel error, resolving it may be challenging. Please file an issue on GitHub. -- If it is an out-of-memory issue, it may sometimes be reported as this error instead of "Out of Memory." Refer to the section above for guidance on avoiding OOM issues. - -### The server hangs -- If the server hangs during initialization or running, it can be memory issues (out of memory), network issues (nccl errors), or other bugs in sglang. - - If it is out of memory, you might see that `avail mem` is very low during the initialization or right after initialization. In this case, - you can try to decrease `--mem-fraction-static`, decrease `--cuda-graph-max-bs`, or decrease `--chunked-prefill-size`. -- Other bugs, please file an issue on GitHub. - - -## Frequently Asked Questions - -### The results are not deterministic, even with a temperature of 0 - -You may notice that when you send the same request twice, the results from the engine will be slightly different, even when the temperature is set to 0. - -From our initial investigation, this indeterminism arises from two factors: dynamic batching and prefix caching. Roughly speaking, dynamic batching accounts for about 95% of the indeterminism, while prefix caching accounts for the remaining portion. The server runs dynamic batching under the hood. Different batch sizes can cause PyTorch/CuBLAS to dispatch to different CUDA kernels, which can lead to slight numerical differences. This difference accumulates across many layers, resulting in nondeterministic output when the batch size changes. Similarly, when prefix caching is enabled, it can also dispatch to different kernels. Even when the computations are mathematically equivalent, small numerical differences from different kernel implementations lead to the final nondeterministic outputs. - -To achieve more deterministic outputs in the current code, you can add `--disable-radix-cache` and send only one request at a time. The results will be mostly deterministic under this setting. - -**Update**: -Recently, we also introduced a deterministic mode, you can enable it with `--enable-deterministic-inference`. -Please find more details in this blog post: https://lmsys.org/blog/2025-09-22-sglang-deterministic/ diff --git a/docs/references/frontend/choices_methods.md b/docs/references/frontend/choices_methods.md deleted file mode 100644 index 30a0a1814..000000000 --- a/docs/references/frontend/choices_methods.md +++ /dev/null @@ -1,77 +0,0 @@ -# Choices Methods in SGLang -This doc describes the choices methods supported by SGLang. - -The optional `choices_method` arg determines how options supplied to SGLang's `choices` primitive are selected. Only the `RuntimeEndpoint` backend supports the `choices_method` arg. Other backends, such as `OpenAI`, have bespoke selection implementations due to API limitations. - -## Methods - -### Token Length Normalized - -Token length normalized is the default SGLang choices method. It selects the option with the highest average logprob across all of its tokens. - -Usage example (alternatively, simply omit the `choices_method` arg): -```python -@sgl.function -def example(s): - s += sgl.user("What is the capital of France?") - s += sgl.assistant( - sgl.gen( - "answer", - choices=["London", "Paris", "Berlin"], - choices_method=sgl.token_length_normalized, - ) - ) -``` - - -This can perform poorly if an option contains many tokens, where its later tokens are predicted with high confidence based on its earlier tokens. For instance, even strong models will fail the above example if the specified options are `["Paris", "Antidisestablishmentarianism"]`. - -### Greedy Token Selection - -Greedy token selection simply selects the option with the highest logprob for its initial token. For overlapping options where one option is a subset of a longer option, the logprobs of the shorter option are extended using its average logprob for comparison against the longer option. - -Usage example: -```python -@sgl.function -def example(s): - s += sgl.user("What is the capital of France?") - s += sgl.assistant( - sgl.gen( - "answer", - choices=["London", "Paris", "Berlin"], - choices_method=sgl.greedy_token_selection, - ) - ) -``` - -This can perform poorly if an option misleads the model down a bad path based on an attractive initial token. For instance, greedy selection will result in an incorrect response for this example: -```python -@sgl.function -def us_president_example(s): - s += sgl.user("Name a US president.") - s += sgl.assistant( - sgl.gen( - "answer", - choices=["Donald Duck", "Millard Fillmore"], - choices_method=sgl.greedy_token_selection, - ) - ) -``` - -### Unconditional Likelihood Normalized - -Unconditional likelihood normalized selects the option with the highest average token logprob once normalized by the unconditional token logprobs, as described in [this EleutherAI blogpost](https://blog.eleuther.ai/multiple-choice-normalization/). This method incurs an additional LLM call to obtain the unconditional likelihoods. - -Usage example: -```python -@sgl.function -def example(s): - s += sgl.user("What is the capital of France?") - s += sgl.assistant( - sgl.gen( - "answer", - choices=["London", "Paris", "Berlin"], - choices_method=sgl.unconditional_likelihood_normalized, - ) - ) -``` diff --git a/docs/references/frontend/frontend_index.rst b/docs/references/frontend/frontend_index.rst deleted file mode 100644 index 62544cba5..000000000 --- a/docs/references/frontend/frontend_index.rst +++ /dev/null @@ -1,9 +0,0 @@ -Frontend Language -================= - -.. toctree:: - :maxdepth: 1 - :caption: Frontend Language - - frontend_tutorial.ipynb - choices_methods.md diff --git a/docs/references/frontend/frontend_tutorial.ipynb b/docs/references/frontend/frontend_tutorial.ipynb deleted file mode 100644 index 9c4da052c..000000000 --- a/docs/references/frontend/frontend_tutorial.ipynb +++ /dev/null @@ -1,456 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# SGLang Frontend Language" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "SGLang frontend language can be used to define simple and easy prompts in a convenient, structured way." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Launch A Server\n", - "\n", - "Launch the server in your terminal and wait for it to initialize." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from sglang import assistant_begin, assistant_end\n", - "from sglang import assistant, function, gen, system, user\n", - "from sglang import image\n", - "from sglang import RuntimeEndpoint\n", - "from sglang.lang.api import set_default_backend\n", - "from sglang.srt.utils import load_image\n", - "from sglang.test.doc_patch import launch_server_cmd\n", - "from sglang.utils import print_highlight, terminate_process, wait_for_server\n", - "\n", - "server_process, port = launch_server_cmd(\n", - " \"python -m sglang.launch_server --model-path Qwen/Qwen2.5-7B-Instruct --host 0.0.0.0 --log-level warning\"\n", - ")\n", - "\n", - "wait_for_server(f\"http://localhost:{port}\", process=server_process)\n", - "print(f\"Server started on http://localhost:{port}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Set the default backend. Note: Besides the local server, you may use also `OpenAI` or other API endpoints." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "set_default_backend(RuntimeEndpoint(f\"http://localhost:{port}\"))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Basic Usage\n", - "\n", - "The most simple way of using SGLang frontend language is a simple question answer dialog between a user and an assistant." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "@function\n", - "def basic_qa(s, question):\n", - " s += system(f\"You are a helpful assistant than can answer questions.\")\n", - " s += user(question)\n", - " s += assistant(gen(\"answer\", max_tokens=512))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "state = basic_qa(\"List 3 countries and their capitals.\")\n", - "print_highlight(state[\"answer\"])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Multi-turn Dialog\n", - "\n", - "SGLang frontend language can also be used to define multi-turn dialogs." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "@function\n", - "def multi_turn_qa(s):\n", - " s += system(f\"You are a helpful assistant than can answer questions.\")\n", - " s += user(\"Please give me a list of 3 countries and their capitals.\")\n", - " s += assistant(gen(\"first_answer\", max_tokens=512))\n", - " s += user(\"Please give me another list of 3 countries and their capitals.\")\n", - " s += assistant(gen(\"second_answer\", max_tokens=512))\n", - " return s\n", - "\n", - "\n", - "state = multi_turn_qa()\n", - "print_highlight(state[\"first_answer\"])\n", - "print_highlight(state[\"second_answer\"])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Control flow\n", - "\n", - "You may use any Python code within the function to define more complex control flows." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "@function\n", - "def tool_use(s, question):\n", - " s += assistant(\n", - " \"To answer this question: \"\n", - " + question\n", - " + \". I need to use a \"\n", - " + gen(\"tool\", choices=[\"calculator\", \"search engine\"])\n", - " + \". \"\n", - " )\n", - "\n", - " if s[\"tool\"] == \"calculator\":\n", - " s += assistant(\"The math expression is: \" + gen(\"expression\"))\n", - " elif s[\"tool\"] == \"search engine\":\n", - " s += assistant(\"The key word to search is: \" + gen(\"word\"))\n", - "\n", - "\n", - "state = tool_use(\"What is 2 * 2?\")\n", - "print_highlight(state[\"tool\"])\n", - "print_highlight(state[\"expression\"])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Parallelism\n", - "\n", - "Use `fork` to launch parallel prompts. Because `sgl.gen` is non-blocking, the for loop below issues two generation calls in parallel." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "@function\n", - "def tip_suggestion(s):\n", - " s += assistant(\n", - " \"Here are two tips for staying healthy: \"\n", - " \"1. Balanced Diet. 2. Regular Exercise.\\n\\n\"\n", - " )\n", - "\n", - " forks = s.fork(2)\n", - " for i, f in enumerate(forks):\n", - " f += assistant(\n", - " f\"Now, expand tip {i+1} into a paragraph:\\n\"\n", - " + gen(\"detailed_tip\", max_tokens=256, stop=\"\\n\\n\")\n", - " )\n", - "\n", - " s += assistant(\"Tip 1:\" + forks[0][\"detailed_tip\"] + \"\\n\")\n", - " s += assistant(\"Tip 2:\" + forks[1][\"detailed_tip\"] + \"\\n\")\n", - " s += assistant(\n", - " \"To summarize the above two tips, I can say:\\n\" + gen(\"summary\", max_tokens=512)\n", - " )\n", - "\n", - "\n", - "state = tip_suggestion()\n", - "print_highlight(state[\"summary\"])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Constrained Decoding\n", - "\n", - "Use `regex` to specify a regular expression as a decoding constraint. This is only supported for local models." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "@function\n", - "def regular_expression_gen(s):\n", - " s += user(\"What is the IP address of the Google DNS servers?\")\n", - " s += assistant(\n", - " gen(\n", - " \"answer\",\n", - " temperature=0,\n", - " regex=r\"((25[0-5]|2[0-4]\\d|[01]?\\d\\d?).){3}(25[0-5]|2[0-4]\\d|[01]?\\d\\d?)\",\n", - " )\n", - " )\n", - "\n", - "\n", - "state = regular_expression_gen()\n", - "print_highlight(state[\"answer\"])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Use `regex` to define a `JSON` decoding schema." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "character_regex = (\n", - " r\"\"\"\\{\\n\"\"\"\n", - " + r\"\"\" \"name\": \"[\\w\\d\\s]{1,16}\",\\n\"\"\"\n", - " + r\"\"\" \"house\": \"(Gryffindor|Slytherin|Ravenclaw|Hufflepuff)\",\\n\"\"\"\n", - " + r\"\"\" \"blood status\": \"(Pure-blood|Half-blood|Muggle-born)\",\\n\"\"\"\n", - " + r\"\"\" \"occupation\": \"(student|teacher|auror|ministry of magic|death eater|order of the phoenix)\",\\n\"\"\"\n", - " + r\"\"\" \"wand\": \\{\\n\"\"\"\n", - " + r\"\"\" \"wood\": \"[\\w\\d\\s]{1,16}\",\\n\"\"\"\n", - " + r\"\"\" \"core\": \"[\\w\\d\\s]{1,16}\",\\n\"\"\"\n", - " + r\"\"\" \"length\": [0-9]{1,2}\\.[0-9]{0,2}\\n\"\"\"\n", - " + r\"\"\" \\},\\n\"\"\"\n", - " + r\"\"\" \"alive\": \"(Alive|Deceased)\",\\n\"\"\"\n", - " + r\"\"\" \"patronus\": \"[\\w\\d\\s]{1,16}\",\\n\"\"\"\n", - " + r\"\"\" \"bogart\": \"[\\w\\d\\s]{1,16}\"\\n\"\"\"\n", - " + r\"\"\"\\}\"\"\"\n", - ")\n", - "\n", - "\n", - "@function\n", - "def character_gen(s, name):\n", - " s += user(\n", - " f\"{name} is a character in Harry Potter. Please fill in the following information about this character.\"\n", - " )\n", - " s += assistant(gen(\"json_output\", max_tokens=256, regex=character_regex))\n", - "\n", - "\n", - "state = character_gen(\"Harry Potter\")\n", - "print_highlight(state[\"json_output\"])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Batching \n", - "\n", - "Use `run_batch` to run a batch of prompts." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "@function\n", - "def text_qa(s, question):\n", - " s += user(question)\n", - " s += assistant(gen(\"answer\", stop=\"\\n\"))\n", - "\n", - "\n", - "states = text_qa.run_batch(\n", - " [\n", - " {\"question\": \"What is the capital of the United Kingdom?\"},\n", - " {\"question\": \"What is the capital of France?\"},\n", - " {\"question\": \"What is the capital of Japan?\"},\n", - " ],\n", - " progress_bar=True,\n", - ")\n", - "\n", - "for i, state in enumerate(states):\n", - " print_highlight(f\"Answer {i+1}: {states[i]['answer']}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Streaming \n", - "\n", - "Use `stream` to stream the output to the user." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "@function\n", - "def text_qa(s, question):\n", - " s += user(question)\n", - " s += assistant(gen(\"answer\", stop=\"\\n\"))\n", - "\n", - "\n", - "state = text_qa.run(\n", - " question=\"What is the capital of France?\", temperature=0.1, stream=True\n", - ")\n", - "\n", - "for out in state.text_iter():\n", - " print(out, end=\"\", flush=True)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Complex Prompts\n", - "\n", - "You may use `{system|user|assistant}_{begin|end}` to define complex prompts." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "@function\n", - "def chat_example(s):\n", - " s += system(\"You are a helpful assistant.\")\n", - " # Same as: s += s.system(\"You are a helpful assistant.\")\n", - "\n", - " with s.user():\n", - " s += \"Question: What is the capital of France?\"\n", - "\n", - " s += assistant_begin()\n", - " s += \"Answer: \" + gen(\"answer\", max_tokens=100, stop=\"\\n\")\n", - " s += assistant_end()\n", - "\n", - "\n", - "state = chat_example()\n", - "print_highlight(state[\"answer\"])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "terminate_process(server_process)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Multi-modal Generation\n", - "\n", - "You may use SGLang frontend language to define multi-modal prompts.\n", - "See [here](https://docs.sglang.io/supported_models/text_generation/multimodal_language_models.html) for supported models." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "server_process, port = launch_server_cmd(\n", - " \"python -m sglang.launch_server --model-path Qwen/Qwen2.5-VL-7B-Instruct --host 0.0.0.0 --log-level warning\"\n", - ")\n", - "\n", - "wait_for_server(f\"http://localhost:{port}\", process=server_process)\n", - "print(f\"Server started on http://localhost:{port}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "set_default_backend(RuntimeEndpoint(f\"http://localhost:{port}\"))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Ask a question about an image." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "@function\n", - "def image_qa(s, image_file, question):\n", - " s += user(image(image_file) + question)\n", - " s += assistant(gen(\"answer\", max_tokens=256))\n", - "\n", - "\n", - "image_url = \"https://raw.githubusercontent.com/sgl-project/sglang/main/examples/assets/example_image.png\"\n", - "image_bytes, _ = load_image(image_url)\n", - "state = image_qa(image_bytes, \"What is in the image?\")\n", - "print_highlight(state[\"answer\"])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "terminate_process(server_process)" - ] - } - ], - "metadata": { - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/docs/references/learn_more.md b/docs/references/learn_more.md deleted file mode 100644 index f0d6ffb8b..000000000 --- a/docs/references/learn_more.md +++ /dev/null @@ -1,9 +0,0 @@ -# Learn More and Join the Community - -- The development roadmap: [https://roadmap.sglang.io](https://roadmap.sglang.io) -- Join weekly public development meeting: [https://meet.sglang.io](https://meet.sglang.io) -- Join Slack: [https://slack.sglang.io/](https://slack.sglang.io/) -- Follow on X (formerly Twitter): [https://x.com/lmsysorg](https://x.com/lmsysorg) -- Follow on LinkedIn: [https://www.linkedin.com/company/sgl-project/](https://www.linkedin.com/company/sgl-project/) -- The latest SGLang features and updates are shared through the [LMSYS blog](https://lmsys.org/blog/) -- More blogs, slides, and videos about SGLang at [https://github.com/sgl-project/sgl-learning-materials](https://github.com/sgl-project/sgl-learning-materials) diff --git a/docs/references/multi_node_deployment/deploy_on_k8s.md b/docs/references/multi_node_deployment/deploy_on_k8s.md deleted file mode 100644 index cfc099f56..000000000 --- a/docs/references/multi_node_deployment/deploy_on_k8s.md +++ /dev/null @@ -1,337 +0,0 @@ -# Deploy On Kubernetes - -This document is for deploying a RoCE network-based SGLang two-node inference service on a Kubernetes (K8S) cluster. - -[LeaderWorkerSet (LWS)](https://github.com/kubernetes-sigs/lws) is a Kubernetes API that aims to address common deployment patterns of AI/ML inference workloads. A major use case is for multi-host/multi-node distributed inference. - -SGLang can also be deployed with LWS on Kubernetes for distributed model serving. - -Please see this guide for more details on deploying SGLang on Kubernetes using LWS. - -Here we take the deployment of DeepSeek-R1 as an example. - -## Prerequisites - -1. At least two Kubernetes nodes, each with two H20 systems and eight GPUs, are required. - -2. Make sure your K8S cluster has LWS correctly installed. If it hasn't been set up yet, please follow the [installation instructions](https://github.com/kubernetes-sigs/lws/blob/main/site/content/en/docs/installation/_index.md). **Note:** For LWS versions ≤0.5.x, you must use the Downward API to obtain `LWS_WORKER_INDEX`, as native support for this feature was introduced in v0.6.0. - -## Basic example - -For the basic example documentation, refer to [Deploy Distributed Inference Service with SGLang and LWS on GPUs](https://github.com/kubernetes-sigs/lws/tree/main/docs/examples/sglang). - -However, that document only covers the basic NCCL socket mode. - -In this section, we’ll make some simple modifications to adapt the setup to the RDMA scenario. - -## RDMA RoCE case - -* Check your env: - -```bash -[root@node1 ~]# ibstatus -Infiniband device 'mlx5_bond_0' port 1 status: - default gid: fe80:0000:0000:0000:0225:9dff:fe64:c79a - base lid: 0x0 - sm lid: 0x0 - state: 4: ACTIVE - phys state: 5: LinkUp - rate: 200 Gb/sec (2X NDR) - link_layer: Ethernet - -Infiniband device 'mlx5_bond_1' port 1 status: - default gid: fe80:0000:0000:0000:0225:9dff:fe6e:c3ec - base lid: 0x0 - sm lid: 0x0 - state: 4: ACTIVE - phys state: 5: LinkUp - rate: 200 Gb/sec (2X NDR) - link_layer: Ethernet - -Infiniband device 'mlx5_bond_2' port 1 status: - default gid: fe80:0000:0000:0000:0225:9dff:fe73:0dd7 - base lid: 0x0 - sm lid: 0x0 - state: 4: ACTIVE - phys state: 5: LinkUp - rate: 200 Gb/sec (2X NDR) - link_layer: Ethernet - -Infiniband device 'mlx5_bond_3' port 1 status: - default gid: fe80:0000:0000:0000:0225:9dff:fe36:f7ff - base lid: 0x0 - sm lid: 0x0 - state: 4: ACTIVE - phys state: 5: LinkUp - rate: 200 Gb/sec (2X NDR) - link_layer: Ethernet -``` - -* Prepare the `lws.yaml` file for deploying on k8s. - -```yaml -apiVersion: leaderworkerset.x-k8s.io/v1 -kind: LeaderWorkerSet -metadata: - name: sglang -spec: - replicas: 1 - leaderWorkerTemplate: - size: 2 - restartPolicy: RecreateGroupOnPodRestart - leaderTemplate: - metadata: - labels: - role: leader - spec: - dnsPolicy: ClusterFirstWithHostNet - hostNetwork: true - hostIPC: true - containers: - - name: sglang-leader - image: sglang:latest - securityContext: - privileged: true - env: - - name: NCCL_IB_GID_INDEX - value: "3" - command: - - python3 - - -m - - sglang.launch_server - - --model-path - - /work/models - - --mem-fraction-static - - "0.93" - - --torch-compile-max-bs - - "8" - - --max-running-requests - - "20" - - --tp - - "16" # Size of Tensor Parallelism - - --dist-init-addr - - $(LWS_LEADER_ADDRESS):20000 - - --nnodes - - $(LWS_GROUP_SIZE) - - --node-rank - - $(LWS_WORKER_INDEX) - - --trust-remote-code - - --host - - "0.0.0.0" - - --port - - "40000" - resources: - limits: - nvidia.com/gpu: "8" - ports: - - containerPort: 40000 - readinessProbe: - tcpSocket: - port: 40000 - initialDelaySeconds: 15 - periodSeconds: 10 - volumeMounts: - - mountPath: /dev/shm - name: dshm - - name: model - mountPath: /work/models - - name: ib - mountPath: /dev/infiniband - volumes: - - name: dshm - emptyDir: - medium: Memory - - name: model - hostPath: - path: '< your models dir >' # modify it according your models dir - - name: ib - hostPath: - path: /dev/infiniband - workerTemplate: - spec: - dnsPolicy: ClusterFirstWithHostNet - hostNetwork: true - hostIPC: true - containers: - - name: sglang-worker - image: sglang:latest - securityContext: - privileged: true - env: - - name: NCCL_IB_GID_INDEX - value: "3" - command: - - python3 - - -m - - sglang.launch_server - - --model-path - - /work/models - - --mem-fraction-static - - "0.93" - - --torch-compile-max-bs - - "8" - - --max-running-requests - - "20" - - --tp - - "16" # Size of Tensor Parallelism - - --dist-init-addr - - $(LWS_LEADER_ADDRESS):20000 - - --nnodes - - $(LWS_GROUP_SIZE) - - --node-rank - - $(LWS_WORKER_INDEX) - - --trust-remote-code - resources: - limits: - nvidia.com/gpu: "8" - volumeMounts: - - mountPath: /dev/shm - name: dshm - - name: model - mountPath: /work/models - - name: ib - mountPath: /dev/infiniband - volumes: - - name: dshm - emptyDir: - medium: Memory - - name: ib - hostPath: - path: /dev/infiniband - - name: model - hostPath: - path: /data1/models/deepseek_v3_moe ---- -apiVersion: v1 -kind: Service -metadata: - name: sglang-leader -spec: - selector: - leaderworkerset.sigs.k8s.io/name: sglang - role: leader - ports: - - protocol: TCP - port: 40000 - targetPort: 40000 - -``` - -* Then use `kubectl apply -f lws.yaml` you will get this output. - -```text -NAME READY STATUS RESTARTS AGE -sglang-0 0/1 Running 0 9s -sglang-0-1 1/1 Running 0 9s -``` - -Wait for the sglang leader (`sglang-0`) status to change to 1/1, which indicates it is `Ready`. - -You can use the command `kubectl logs -f sglang-0` to view the logs of the leader node. - -Once successful, you should see output like this: - -```text -[2025-02-17 05:27:24 TP1] Capture cuda graph end. Time elapsed: 84.89 s -[2025-02-17 05:27:24 TP6] max_total_num_tokens=712400, chunked_prefill_size=8192, max_prefill_tokens=16384, max_running_requests=50, context_len=163840 -[2025-02-17 05:27:24 TP0] max_total_num_tokens=712400, chunked_prefill_size=8192, max_prefill_tokens=16384, max_running_requests=50, context_len=163840 -[2025-02-17 05:27:24 TP7] max_total_num_tokens=712400, chunked_prefill_size=8192, max_prefill_tokens=16384, max_running_requests=50, context_len=163840 -[2025-02-17 05:27:24 TP3] max_total_num_tokens=712400, chunked_prefill_size=8192, max_prefill_tokens=16384, max_running_requests=50, context_len=163840 -[2025-02-17 05:27:24 TP2] max_total_num_tokens=712400, chunked_prefill_size=8192, max_prefill_tokens=16384, max_running_requests=50, context_len=163840 -[2025-02-17 05:27:24 TP4] max_total_num_tokens=712400, chunked_prefill_size=8192, max_prefill_tokens=16384, max_running_requests=50, context_len=163840 -[2025-02-17 05:27:24 TP1] max_total_num_tokens=712400, chunked_prefill_size=8192, max_prefill_tokens=16384, max_running_requests=50, context_len=163840 -[2025-02-17 05:27:24 TP5] max_total_num_tokens=712400, chunked_prefill_size=8192, max_prefill_tokens=16384, max_running_requests=50, context_len=163840 -[2025-02-17 05:27:24] INFO: Started server process [1] -[2025-02-17 05:27:24] INFO: Waiting for application startup. -[2025-02-17 05:27:24] INFO: Application startup complete. -[2025-02-17 05:27:24] INFO: Uvicorn running on http://0.0.0.0:40000 (Press CTRL+C to quit) -[2025-02-17 05:27:25] INFO: 127.0.0.1:48908 - "GET /get_model_info HTTP/1.1" 200 OK -[2025-02-17 05:27:25 TP0] Prefill batch. #new-seq: 1, #new-token: 7, #cached-token: 0, cache hit rate: 0.00%, token usage: 0.00, #running-req: 0, #queue-req: 0 -[2025-02-17 05:27:32] INFO: 127.0.0.1:48924 - "POST /generate HTTP/1.1" 200 OK -[2025-02-17 05:27:32] The server is fired up and ready to roll! -``` - -If it doesn’t start up successfully, please follow these steps to check for any remaining issues. Thanks! - -### Debug - -* Set `NCCL_DEBUG=TRACE` to check if it is a NCCL communication problem. - -This should resolve most NCCL-related issues. - -***Notice: If you find that NCCL_DEBUG=TRACE is not effective in the container environment, but the process is stuck or you encounter hard-to-diagnose issues, try switching to a different container image. Some images may not handle standard error output properly.*** - -#### RoCE scenario - -* Please make sure that RDMA devices are available in the cluster environment. -* Please make sure that the nodes in the cluster have Mellanox NICs with RoCE. In this example, we use Mellanox ConnectX 5 model NICs, and the proper OFED driver has been installed. If not, please refer to the document [Install OFED Driver](https://docs.nvidia.com/networking/display/mlnxofedv461000/installing+mellanox+ofed) to install the driver. -* Check your env: - - ```shell - $ lspci -nn | grep Eth | grep Mellanox - 0000:7f:00.0 Ethernet controller [0200]: Mellanox Technologies MT43244 BlueField-3 integrated ConnectX-7 network controller [15b3:a2dc] (rev 01) - 0000:7f:00.1 Ethernet controller [0200]: Mellanox Technologies MT43244 BlueField-3 integrated ConnectX-7 network controller [15b3:a2dc] (rev 01) - 0000:c7:00.0 Ethernet controller [0200]: Mellanox Technologies MT43244 BlueField-3 integrated ConnectX-7 network controller [15b3:a2dc] (rev 01) - 0000:c7:00.1 Ethernet controller [0200]: Mellanox Technologies MT43244 BlueField-3 integrated ConnectX-7 network controller [15b3:a2dc] (rev 01) - 0001:08:00.0 Ethernet controller [0200]: Mellanox Technologies MT43244 BlueField-3 integrated ConnectX-7 network controller [15b3:a2dc] (rev 01) - 0001:08:00.1 Ethernet controller [0200]: Mellanox Technologies MT43244 BlueField-3 integrated ConnectX-7 network controller [15b3:a2dc] (rev 01) - 0001:a2:00.0 Ethernet controller [0200]: Mellanox Technologies MT43244 BlueField-3 integrated ConnectX-7 network controller [15b3:a2dc] (rev 01) - 0001:a2:00.1 Ethernet controller [0200]: Mellanox Technologies MT43244 BlueField-3 integrated ConnectX-7 network controller [15b3:a2dc] (rev 01) - ``` - -* Check the OFED driver: - - ```shell - ofed_info -s - OFED-internal-23.07-0.5.0: - ``` - -* Show RDMA link status and check IB devices: - - ```shell - $ rdma link show - 8/1: mlx5_bond_0/1: state ACTIVE physical_state LINK_UP netdev reth0 - 9/1: mlx5_bond_1/1: state ACTIVE physical_state LINK_UP netdev reth2 - 10/1: mlx5_bond_2/1: state ACTIVE physical_state LINK_UP netdev reth4 - 11/1: mlx5_bond_3/1: state ACTIVE physical_state LINK_UP netdev reth6 - - $ ibdev2netdev - 8/1: mlx5_bond_0/1: state ACTIVE physical_state LINK_UP netdev reth0 - 9/1: mlx5_bond_1/1: state ACTIVE physical_state LINK_UP netdev reth2 - 10/1: mlx5_bond_2/1: state ACTIVE physical_state LINK_UP netdev reth4 - 11/1: mlx5_bond_3/1: state ACTIVE physical_state LINK_UP netdev reth6 - ``` - -* Test RoCE network speed on the host: - - ```shell - yum install qperf - # for server： - execute qperf - # for client - qperf -t 60 -cm1 rc_rdma_write_bw - ``` - -* Check RDMA accessible in your container: - - ```shell - # ibv_devices - # ibv_devinfo - ``` - -## Keys to success - -* In the YAML configuration above, pay attention to the NCCL environment variable. For older versions of NCCL, you should check the NCCL_IB_GID_INDEX environment setting. -* NCCL_SOCKET_IFNAME is also crucial, but in a containerized environment, this typically isn’t an issue. -* In some cases, it’s necessary to configure GLOO_SOCKET_IFNAME correctly. -* NCCL_DEBUG is essential for troubleshooting, but I've found that sometimes it doesn't show error logs within containers. This could be related to the Docker image you're using. You may want to try switching images if needed. -* Avoid using Docker images based on Ubuntu 18.04, as they tend to have compatibility issues. - -## Remaining issues - -* In Kubernetes, Docker, or Containerd environments, we use hostNetwork to prevent performance degradation. -* We utilize privileged mode, which isn’t secure. Additionally, in containerized environments, full GPU isolation cannot be achieved. - -## TODO - -* Integrated with [k8s-rdma-shared-dev-plugin](https://github.com/Mellanox/k8s-rdma-shared-dev-plugin). diff --git a/docs/references/multi_node_deployment/lws_pd/lws-examples/d-svc.yaml b/docs/references/multi_node_deployment/lws_pd/lws-examples/d-svc.yaml deleted file mode 100644 index 27f98009e..000000000 --- a/docs/references/multi_node_deployment/lws_pd/lws-examples/d-svc.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: deepseekr10528-decode-main -spec: - selector: - leaderworkerset.sigs.k8s.io/name: deepseekr10528-decode-main - role: leader - ports: - - protocol: TCP - port: 30000 - targetPort: 30000 diff --git a/docs/references/multi_node_deployment/lws_pd/lws-examples/d.yaml b/docs/references/multi_node_deployment/lws_pd/lws-examples/d.yaml deleted file mode 100644 index dbb51b519..000000000 --- a/docs/references/multi_node_deployment/lws_pd/lws-examples/d.yaml +++ /dev/null @@ -1,290 +0,0 @@ -apiVersion: leaderworkerset.x-k8s.io/v1 -kind: LeaderWorkerSet -metadata: - name: deepseekr10528-decode-main -spec: - leaderWorkerTemplate: - leaderTemplate: - metadata: - labels: - role: leader - spec: - containers: - - command: - - python3 - - -m - - sglang.launch_server - - --port - - "30000" - - --host - - "0.0.0.0" - - --model-path - - /work/models - - --chunked-prefill-size - - "262144" - - --page-size - - "64" - - --enable-dp-attention - - --enable-dp-lm-head - - --dp-size - - "16" - - --moe-a2a-backend - - deepep - - --disaggregation-mode - - decode - - --mem-fraction-static - - "0.849" - - --context-length - - "32768" - - --disaggregation-ib-device - - "mlx5_bond_0,mlx5_bond_1,mlx5_bond_2,mlx5_bond_3" - - --cuda-graph-max-bs - - "64" - - --max-running-requests - - "2048" - - --tp-size - - "16" # Size of Tensor Parallelism - - --dist-init-addr - - $(LWS_LEADER_ADDRESS):20102 - - --nnodes - - $(LWS_GROUP_SIZE) - - --node-rank - - $(LWS_WORKER_INDEX) - - --trust-remote-code - - --ep-num-redundant-experts - - "32" - - --moe-dense-tp-size - - "1" - env: - - name: CUDA_LAUNCH_BLOCKING - value: "0" - - name: NVSHMEM_IB_GID_INDEX - value: "3" - - name: NVSHMEM_ENABLE_NIC_PE_MAPPING - value: "1" - - name: NVSHMEM_HCA_PE_MAPPING - value: "mlx5_bond_0:1:2,mlx5_bond_1:1:2,mlx5_bond_2:1:2,mlx5_bond_3:1:2" - - name: NCCL_IB_QPS_PER_CONNECTION - value: "8" - - name: NCCL_IB_SPLIT_DATA_ON_QPS - value: "1" - - name: NCCL_NET_PLUGIN - value: "none" - - name: NCCL_IB_TC - value: "136" - - name: NCCL_MIN_NCHANNELS - value: "4" - - name: NCCL_IB_SL - value: "5" - - name: MC_TE_METRIC - value: "true" - - name: SGLANG_MOONCAKE_TRANS_THREAD - value: "16" - - name: SGLANG_ENABLE_JIT_DEEPGEMM - value: "1" - - name: NCCL_IB_HCA - value: ^=mlx5_0,mlx5_5,mlx5_6 - - name: LWS_WORKER_INDEX - valueFrom: - fieldRef: - fieldPath: metadata.labels['leaderworkerset.sigs.k8s.io/worker-index'] - image: lmsysorg/sglang:latest - name: sglang-leader - ports: - - containerPort: 30000 - protocol: TCP - readinessProbe: - periodSeconds: 30 - tcpSocket: - port: 30000 - resources: - limits: - nvidia.com/gpu: "8" - securityContext: - capabilities: - add: - - IPC_LOCK - privileged: true - volumeMounts: - - mountPath: /root/.cache - name: sgl-cache - - mountPath: /dev/shm - name: dshm - - mountPath: /work/models - name: model - - mountPath: /dev/infiniband - name: ib - - mountPath: /sgl-workspace/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs - name: cf - dnsPolicy: ClusterFirstWithHostNet - hostIPC: true - hostNetwork: true - nodeSelector: - # should modify according your deployment env - pd: "yes" - tolerations: - # should modify according your deployment env - - key: bopd - operator: Exists - - key: node-role - operator: Exists - volumes: - - hostPath: - path: /data1/sgl_cache1 - type: DirectoryOrCreate - name: sgl-cache - - emptyDir: - medium: Memory - name: dshm - - hostPath: - path: /data1/maas_hosted_models/models/DeepSeek-R1-0528/deepseek_r1_0528 - name: model - - hostPath: - path: /dev/infiniband - name: ib - - hostPath: - path: /data1/maas_hosted_models/models/fused_moe_triton/configs - name: cf - restartPolicy: RecreateGroupOnPodRestart - size: 2 - workerTemplate: - metadata: {} - spec: - containers: - - command: - - python3 - - -m - - sglang.launch_server - - --model-path - - /work/models - - --chunked-prefill-size - - "262144" - - --page-size - - "64" - - --enable-dp-attention - - --enable-dp-lm-head - - --dp-size - - "16" - - --moe-a2a-backend - - deepep - - --disaggregation-mode - - decode - - --mem-fraction-static - - "0.849" - - --context-length - - "32768" - - --disaggregation-ib-device - - "mlx5_bond_0,mlx5_bond_1,mlx5_bond_2,mlx5_bond_3" - - --cuda-graph-max-bs - - "64" - - --max-running-requests - - "2048" - - --tp-size - - "16" # Size of Tensor Parallelism - - --dist-init-addr - - $(LWS_LEADER_ADDRESS):20102 - - --nnodes - - $(LWS_GROUP_SIZE) - - --node-rank - - $(LWS_WORKER_INDEX) - - --trust-remote-code - - --ep-num-redundant-experts - - "32" - - --moe-dense-tp-size - - "1" - env: - - name: NVSHMEM_IB_TRAFFIC_CLASS - value: "16" - - name: NVSHMEM_IB_GID_INDEX - value: "3" - - name: NVSHMEM_ENABLE_NIC_PE_MAPPING - value: "1" - - name: NVSHMEM_HCA_PE_MAPPING - value: "mlx5_bond_0:1:2,mlx5_bond_1:1:2,mlx5_bond_2:1:2,mlx5_bond_3:1:2" - - name: NCCL_IB_QPS_PER_CONNECTION - value: "8" - - name: NCCL_IB_SPLIT_DATA_ON_QPS - value: "1" - - name: NCCL_NET_PLUGIN - value: "none" - - name: NCCL_IB_TC - value: "136" - - name: NCCL_MIN_NCHANNELS - value: "4" - - name: MC_TE_METRIC - value: "true" - - name: NCCL_IB_SL - value: "5" - - name: SGLANG_MOONCAKE_TRANS_THREAD - value: "16" - - name: SGLANG_ENABLE_JIT_DEEPGEMM - value: "1" - - name: NCCL_IB_HCA - value: ^=mlx5_0,mlx5_5,mlx5_6 - - name: LWS_WORKER_INDEX - valueFrom: - fieldRef: - fieldPath: metadata.labels['leaderworkerset.sigs.k8s.io/worker-index'] - image: lmsysorg/sglang:latest - name: sglang-worker - ports: - - containerPort: 30001 - resources: - limits: - nvidia.com/gpu: "8" - securityContext: - capabilities: - add: - - IPC_LOCK - privileged: true - volumeMounts: - - mountPath: /root/.cache - name: sgl-cache - - mountPath: /dev/shm - name: dshm - - mountPath: /work/models - name: model - - mountPath: /dev/infiniband - name: ib - - mountPath: /sgl-workspace/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs - name: cf - dnsPolicy: ClusterFirstWithHostNet - hostIPC: true - hostNetwork: true - nodeSelector: - # should modify according your deployment env - pd: "yes" - tolerations: - # should modify according your deployment env - - key: bopd - operator: Exists - - key: node-role - operator: Exists - volumes: - - hostPath: - path: /data1/sgl_cache1 - type: DirectoryOrCreate - name: sgl-cache - - emptyDir: - medium: Memory - name: dshm - - hostPath: - path: /dev/infiniband - name: ib - - hostPath: - # modify according to you deployment env - path: /data1/maas_hosted_models/models/DeepSeek-R1-0528/deepseek_r1_0528 - name: model - - hostPath: - # modify according to you deployment env - path: /data1/maas_hosted_models/models/fused_moe_triton/configs - name: cf - networkConfig: - subdomainPolicy: Shared - replicas: 1 - rolloutStrategy: - rollingUpdateConfiguration: - maxSurge: 0 - maxUnavailable: 1 - type: RollingUpdate - startupPolicy: LeaderCreated diff --git a/docs/references/multi_node_deployment/lws_pd/lws-examples/lb.yaml b/docs/references/multi_node_deployment/lws_pd/lws-examples/lb.yaml deleted file mode 100644 index 4ca690969..000000000 --- a/docs/references/multi_node_deployment/lws_pd/lws-examples/lb.yaml +++ /dev/null @@ -1,56 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: deepseekr10528-lb-main - labels: - app: deepseekr10528-lb -spec: - replicas: 1 - selector: - matchLabels: - app: deepseekr10528-lb - template: - metadata: - labels: - app: deepseekr10528-lb - spec: - nodeSelector: - bo: "yes" - tolerations: - - key: bopd - operator: Exists - - key: node-role - operator: Exists - containers: - - name: sgl-minilb - image: lmsysorg/sglang:latest - command: - - python - - -m - - sglang_router.launch_router - - --pd-disaggregation - - --prefill - - http://deepseekr10528-prefill-main:30000 - - --decode - - http://deepseekr10528-decode-main:30000 - - --host - - 0.0.0.0 - - --port - - "8000" - ports: - - containerPort: 8000 - ---- -apiVersion: v1 -kind: Service -metadata: - name: deepseekr10528-lb-service -spec: - type: NodePort # NodePort is easy to test, you can also specify `ClusterIP` - selector: - app: deepseekr10528-lb - ports: - - protocol: TCP - port: 8000 # Service Port（In-Cluster） - targetPort: 8000 # Exposed Container - nodePort: 30800 diff --git a/docs/references/multi_node_deployment/lws_pd/lws-examples/p-svc.yaml b/docs/references/multi_node_deployment/lws_pd/lws-examples/p-svc.yaml deleted file mode 100644 index 6826a13df..000000000 --- a/docs/references/multi_node_deployment/lws_pd/lws-examples/p-svc.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: deepseekr10528-prefill-main -spec: - selector: - leaderworkerset.sigs.k8s.io/name: deepseekr10528-prefill-main - role: leader - ports: - - protocol: TCP - port: 30000 - targetPort: 30000 diff --git a/docs/references/multi_node_deployment/lws_pd/lws-examples/p.yaml b/docs/references/multi_node_deployment/lws_pd/lws-examples/p.yaml deleted file mode 100644 index 11bbcd30a..000000000 --- a/docs/references/multi_node_deployment/lws_pd/lws-examples/p.yaml +++ /dev/null @@ -1,304 +0,0 @@ -apiVersion: leaderworkerset.x-k8s.io/v1 -kind: LeaderWorkerSet -metadata: - name: deepseekr10528-prefill-main -spec: - leaderWorkerTemplate: - leaderTemplate: - metadata: - labels: - role: leader - spec: - containers: - - command: - - python3 - - -m - - sglang.launch_server - - --port - - "30000" - - --host - - "0.0.0.0" - - --model-path - - /work/models - - --disaggregation-ib-device - # should modify according your rdma env - - mlx5_bond_0,mlx5_bond_1,mlx5_bond_2,mlx5_bond_3 - - --chunked-prefill-size - - "524288" - - --max-prefill-tokens - - "32768" - - --page-size - - "64" - - --ep-dispatch-algorithm - - dynamic - - --eplb-algorithm - - deepseek - - --enable-dp-lm-head - - --enable-dp-attention - - --dp-size - - "16" - - --disable-radix-cache - - --moe-a2a-backend - - deepep - - --disaggregation-mode - - prefill - - --mem-fraction-static - - "0.7" - - --context-length - - "32768" - - --tp - - "16" - - --dist-init-addr - - $(LWS_LEADER_ADDRESS):20102 - - --nnodes - - $(LWS_GROUP_SIZE) - - --node-rank - - $(LWS_WORKER_INDEX) - - --trust-remote-code - - --ep-num-redundant-experts - - "32" - - --moe-dense-tp-size - - "1" - - --max-running-requests - - "1024" - env: - - name: NVSHMEM_HCA_PE_MAPPING - # should modify according your rdma env - value: "mlx5_bond_0:1:2,mlx5_bond_1:1:2,mlx5_bond_2:1:2,mlx5_bond_3:1:2" - - name: NVSHMEM_IB_GID_INDEX - value: "3" - - name: NVSHMEM_ENABLE_NIC_PE_MAPPING - value: "1" - - name: SGLANG_SET_CPU_AFFINITY - value: "true" - - name: SGLANG_ENABLE_JIT_DEEPGEMM - value: "1" - - name: NCCL_IB_QPS_PER_CONNECTION - value: "8" - - name: NCCL_IB_SPLIT_DATA_ON_QPS - value: "1" - - name: NCCL_NET_PLUGIN - value: none - - name: NCCL_IB_TC - value: "136" - - name: NCCL_MIN_NCHANNELS - value: "4" - - name: MC_TE_METRIC - value: "false" - - name: NCCL_IB_SL - value: "5" - - name: NCCL_IB_HCA - value: ^=mlx5_0,mlx5_5,mlx5_6 - - name: LWS_WORKER_INDEX - valueFrom: - fieldRef: - fieldPath: metadata.labels['leaderworkerset.sigs.k8s.io/worker-index'] - image: lmsysorg/sglang:latest - name: sglang-leader - ports: - - containerPort: 30000 - protocol: TCP - readinessProbe: - periodSeconds: 30 - tcpSocket: - port: 30000 - resources: - limits: - nvidia.com/gpu: "8" - securityContext: - capabilities: - add: - - IPC_LOCK - privileged: true - volumeMounts: - - mountPath: /dev/shm - name: dshm - - mountPath: /work/models - name: model - - mountPath: /dev/infiniband - name: ib - - mountPath: /sgl-workspace/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs - name: cf - - mountPath: /root/.cache - name: sgl-cache - dnsPolicy: ClusterFirstWithHostNet - hostIPC: true - hostNetwork: true - nodeSelector: - # should modify according your deployment env - pd: "yes" - tolerations: - # should modify according your deployment env - - key: bopd - operator: Exists - - key: node-role - operator: Exists - volumes: - - emptyDir: - medium: Memory - name: dshm - - hostPath: - path: /data1/maas_hosted_models/models/DeepSeek-R1-0528/deepseek_r1_0528 - name: model - - hostPath: - path: /dev/infiniband - name: ib - - hostPath: - path: /data1/maas_hosted_models/models/fused_moe_triton/configs - name: cf - - hostPath: - path: /data1/sgl_cache - type: DirectoryOrCreate - name: sgl-cache - restartPolicy: RecreateGroupOnPodRestart - size: 2 - workerTemplate: - metadata: {} - spec: - containers: - - command: - - python3 - - -m - - sglang.launch_server - - --model-path - - /work/models - - --disaggregation-ib-device - # should modify according your rdma env - - mlx5_bond_0,mlx5_bond_1,mlx5_bond_2,mlx5_bond_3 - - --chunked-prefill-size - - "524288" - - --max-prefill-tokens - - "32768" - - --page-size - - "64" - - --ep-dispatch-algorithm - - dynamic - - --eplb-algorithm - - deepseek - # - --deepep-config - # - /home/aiges/tuned/tuned_8sms.json - # can be tuned using deepep test scripts - - --enable-dp-lm-head - - --enable-dp-attention - - --dp-size - - "16" - - --disable-radix-cache - - --moe-a2a-backend - - deepep - - --disaggregation-mode - - prefill - - --mem-fraction-static - - "0.7" - - --context-length - - "32768" - - --tp - - "16" - - --dist-init-addr - - $(LWS_LEADER_ADDRESS):20102 - - --nnodes - - $(LWS_GROUP_SIZE) - - --node-rank - - $(LWS_WORKER_INDEX) - - --trust-remote-code - - --ep-num-redundant-experts - - "32" - - --moe-dense-tp-size - - "1" - - --max-running-requests - - "1024" - env: - - name: SGLANG_SET_CPU_AFFINITY - value: "true" - - name: NVSHMEM_HCA_PE_MAPPING - # should modify according your rdma env - value: "mlx5_bond_0:1:2,mlx5_bond_1:1:2,mlx5_bond_2:1:2,mlx5_bond_3:1:2" - - name: NCCL_IB_HCA - value: ^=mlx5_0,mlx5_5,mlx5_6 - - name: NVSHMEM_IB_TRAFFIC_CLASS - value: "16" - - name: NVSHMEM_IB_GID_INDEX - value: "3" - - name: NVSHMEM_ENABLE_NIC_PE_MAPPING - value: "1" - - name: CUDA_LAUNCH_BLOCKING - value: "0" - - name: SGLANG_MOONCAKE_TRANS_THREAD - value: "8" - - name: SGLANG_ENABLE_JIT_DEEPGEMM - value: "1" - - name: SGLANG_CHUNKED_PREFIX_CACHE_THRESHOLD - value: "0" - - name: NCCL_IB_QPS_PER_CONNECTION - value: "8" - - name: NCCL_IB_SPLIT_DATA_ON_QPS - value: "1" - - name: NCCL_NET_PLUGIN - value: none - - name: NCCL_IB_TC - value: "136" - - name: NCCL_MIN_NCHANNELS - value: "4" - - name: MC_TE_METRIC - value: "true" - - name: NCCL_IB_SL - value: "5" - - name: LWS_WORKER_INDEX - valueFrom: - fieldRef: - fieldPath: metadata.labels['leaderworkerset.sigs.k8s.io/worker-index'] - image: lmsysorg/sglang:latest - name: sglang-worker - ports: - - containerPort: 30001 - protocol: TCP - resources: - limits: - nvidia.com/gpu: "8" - securityContext: - capabilities: - add: - - IPC_LOCK - privileged: true - volumeMounts: - - mountPath: /root/.cache - name: sgl-cache - - mountPath: /dev/shm - name: dshm - - mountPath: /work/models - name: model - - mountPath: /dev/infiniband - name: ib - - mountPath: /sgl-workspace/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs - name: cf - dnsPolicy: ClusterFirstWithHostNet - hostIPC: true - hostNetwork: true - nodeSelector: - # should modify according your deployment env - pd: "yes" - tolerations: - # should modify according your deployment env - - key: bopd - operator: Exists - - key: node-role - operator: Exists - volumes: - - emptyDir: - medium: Memory - name: dshm - - hostPath: - path: /dev/infiniband - name: ib - - hostPath: - # modify according to you deployment env - path: /data1/maas_hosted_models/models/DeepSeek-R1-0528/deepseek_r1_0528 - name: model - - hostPath: - # modify according to you deployment env - path: /data1/maas_hosted_models/models/fused_moe_triton/configs - name: cf - - hostPath: - # modify according to you deployment env - path: /data1/sgl_cache - type: DirectoryOrCreate - name: sgl-cache diff --git a/docs/references/multi_node_deployment/lws_pd/lws_pd_deploy.md b/docs/references/multi_node_deployment/lws_pd/lws_pd_deploy.md deleted file mode 100644 index 419474a4e..000000000 --- a/docs/references/multi_node_deployment/lws_pd/lws_pd_deploy.md +++ /dev/null @@ -1,783 +0,0 @@ -# LWS Based PD Deploy - -## 0. Prerequisites - -1. k8s >=1.26 -2. lws installed on k8s. - -## 1. Image Preparation - -`lmsysorg/sglang:deepep` - -## 2. Deployment Manifest Files - -***Notice: We will package all deployment files into Helm Chart format in the near future. Interested community members can contact us to contribute*** - -### Prefill - -Prefill manifest file [prefill.yaml](lws-examples/p.yaml) - -*Note: The NodeSelector section, model location section, and taint toleration section can be adjusted according to your actual deployment environment* - -```yaml -apiVersion: leaderworkerset.x-k8s.io/v1 -kind: LeaderWorkerSet -metadata: - name: deepseekr10528-prefill-main -spec: - leaderWorkerTemplate: - leaderTemplate: - metadata: - labels: - role: leader - spec: - containers: - - command: - - python3 - - -m - - sglang.launch_server - - --port - - "30000" - - --host - - "0.0.0.0" - - --model-path - - /work/models - - --disaggregation-ib-device - # should modify according your rdma env - - mlx5_bond_0,mlx5_bond_1,mlx5_bond_2,mlx5_bond_3 - - --chunked-prefill-size - - "524288" - - --max-prefill-tokens - - "32768" - - --page-size - - "64" - # - --init-expert-location - # - /home/aiges/tuned/attachment_ep_statistics/prefill_in1024.json - - --ep-dispatch-algorithm - - dynamic - - --eplb-algorithm - - deepseek - # - --deepep-config - # - /home/aiges/tuned/tuned_8sms.json - - --enable-dp-lm-head - - --enable-dp-attention - - --dp-size - - "16" - - --disable-radix-cache - - --moe-a2a-backend - - deepep - - --disaggregation-mode - - prefill - - --mem-fraction-static - - "0.7" - - --context-length - - "32768" - - --tp - - "16" - - --dist-init-addr - - $(LWS_LEADER_ADDRESS):20102 - - --nnodes - - $(LWS_GROUP_SIZE) - - --node-rank - - $(LWS_WORKER_INDEX) - - --trust-remote-code - - --ep-num-redundant-experts - - "32" - - --moe-dense-tp-size - - "1" - - --max-running-requests - - "1024" - env: -# - name: NVSHMEM_HCA_PE_MAPPING -# value: "mlx5_bond_0:1:2,mlx5_bond_1:1:2,mlx5_bond_2:1:2,mlx5_bond_3:1:2" -# - name: NVSHMEM_HCA_LIST -# value: "mlx5_bond_0:1,mlx5_bond_1:1,mlx5_bond_2:1,mlx5_bond_3:1" - - name: NVSHMEM_IB_GID_INDEX - value: "3" - - name: NVSHMEM_ENABLE_NIC_PE_MAPPING - value: "1" - - name: SGLANG_SET_CPU_AFFINITY - value: "true" - - name: SGLANG_ENABLE_JIT_DEEPGEMM - value: "1" - - name: NCCL_IB_QPS_PER_CONNECTION - value: "8" - - name: NCCL_IB_SPLIT_DATA_ON_QPS - value: "1" - - name: NCCL_NET_PLUGIN - value: none - - name: NCCL_IB_TC - value: "136" - - name: NCCL_MIN_NCHANNELS - value: "4" - - name: MC_TE_METRIC - value: "false" - - name: NCCL_IB_SL - value: "5" - - name: NCCL_IB_HCA - value: ^=mlx5_0,mlx5_5,mlx5_6 - - name: LWS_WORKER_INDEX - valueFrom: - fieldRef: - fieldPath: metadata.labels['leaderworkerset.sigs.k8s.io/worker-index'] - image: lmsysorg/sglang:deepep - name: sglang-leader - ports: - - containerPort: 30000 - protocol: TCP - readinessProbe: - periodSeconds: 30 - tcpSocket: - port: 30000 - resources: - limits: - nvidia.com/gpu: "8" - securityContext: - capabilities: - add: - - IPC_LOCK - privileged: true - volumeMounts: - - mountPath: /dev/shm - name: dshm - - mountPath: /work/models - name: model - - mountPath: /dev/infiniband - name: ib - - mountPath: /sgl-workspace/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs - name: cf - - mountPath: /root/.cache - name: sgl-cache - dnsPolicy: ClusterFirstWithHostNet - hostIPC: true - hostNetwork: true - nodeSelector: - pd: "yes" - tolerations: - - key: pd - operator: Exists - - key: node-role - operator: Exists - volumes: - - emptyDir: - medium: Memory - name: dshm - - hostPath: - # modify according to you deployment env - path: /data1/maas_hosted_models/models/DeepSeek-R1-0528/deepseek_r1_0528 - name: model - - hostPath: - path: /dev/infiniband - name: ib - - hostPath: - # modify according to you deployment env - path: /data1/maas_hosted_models/models/fused_moe_triton/configs - name: cf - - hostPath: - # modify according to you deployment env - path: /data1/sgl_cache - type: DirectoryOrCreate - name: sgl-cache - restartPolicy: RecreateGroupOnPodRestart - size: 2 - workerTemplate: - metadata: {} - spec: - containers: - - command: - - python3 - - -m - - sglang.launch_server - - --model-path - - /work/models - - --disaggregation-ib-device - - mlx5_bond_0,mlx5_bond_1,mlx5_bond_2,mlx5_bond_3 - - --chunked-prefill-size - - "524288" - - --max-prefill-tokens - - "32768" - - --page-size - - "64" - #- --init-expert-location - #- /home/aiges/tuned/attachment_ep_statistics/prefill_in1024.json - - --ep-dispatch-algorithm - - dynamic - - --eplb-algorithm - - deepseek -# - --deepep-config -# - /home/aiges/tuned/tuned_8sms.json - - --enable-dp-lm-head - - --enable-dp-attention - - --dp-size - - "16" - - --disable-radix-cache - - --moe-a2a-backend - - deepep - - --disaggregation-mode - - prefill - - --mem-fraction-static - - "0.7" - - --context-length - - "32768" - - --tp - - "16" - - --dist-init-addr - - $(LWS_LEADER_ADDRESS):20102 - - --nnodes - - $(LWS_GROUP_SIZE) - - --node-rank - - $(LWS_WORKER_INDEX) - - --trust-remote-code - - --ep-num-redundant-experts - - "32" - - --moe-dense-tp-size - - "1" - - --max-running-requests - - "1024" - env: - - name: SGLANG_SET_CPU_AFFINITY - value: "true" - - name: SGLANG_HACK_DEEPEP_NUM_SMS - value: "8" - - name: SGLANG_HACK_DEEPEP_NEW_MODE - value: "0" -# - name: NVSHMEM_HCA_PE_MAPPING -# value: "mlx5_bond_0:1:2,mlx5_bond_1:1:2,mlx5_bond_2:1:2,mlx5_bond_3:1:2" -# - name: NVSHMEM_HCA_LIST -# value: "mlx5_bond_0:1,mlx5_bond_1:1,mlx5_bond_2:1,mlx5_bond_3:1" - - name: NCCL_IB_HCA - value: ^=mlx5_0,mlx5_5,mlx5_6 - - name: NVSHMEM_IB_TRAFFIC_CLASS - value: "16" - - name: NVSHMEM_IB_GID_INDEX - value: "3" - - name: NVSHMEM_ENABLE_NIC_PE_MAPPING - value: "1" - - name: CUDA_LAUNCH_BLOCKING - value: "0" - - name: SGLANG_MOONCAKE_TRANS_THREAD - value: "8" - - name: SGLANG_ENABLE_JIT_DEEPGEMM - value: "1" - - name: SGLANG_CHUNKED_PREFIX_CACHE_THRESHOLD - value: "0" - - name: NCCL_IB_QPS_PER_CONNECTION - value: "8" - - name: NCCL_IB_SPLIT_DATA_ON_QPS - value: "1" - - name: NCCL_NET_PLUGIN - value: none - - name: NCCL_IB_TC - value: "136" - - name: NCCL_MIN_NCHANNELS - value: "4" - - name: MC_TE_METRIC - value: "true" - - name: NCCL_IB_SL - value: "5" - - name: LWS_WORKER_INDEX - valueFrom: - fieldRef: - fieldPath: metadata.labels['leaderworkerset.sigs.k8s.io/worker-index'] - image: lmsysorg/sglang:deepep - name: sglang-worker - ports: - - containerPort: 30001 - protocol: TCP - resources: - limits: - nvidia.com/gpu: "8" - securityContext: - capabilities: - add: - - IPC_LOCK - privileged: true - volumeMounts: - - - mountPath: /root/.cache - name: sgl-cache - - mountPath: /dev/shm - name: dshm - - mountPath: /work/models - name: model - - mountPath: /dev/infiniband - name: ib - - mountPath: /sgl-workspace/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs - name: cf - dnsPolicy: ClusterFirstWithHostNet - hostIPC: true - hostNetwork: true - nodeSelector: - pd: "yes" - tolerations: - - key: pd - operator: Exists - - key: node-role - operator: Exists - volumes: - - emptyDir: - medium: Memory - name: dshm - - hostPath: - path: /dev/infiniband - name: ib - - hostPath: - path: /data1/maas_hosted_models/models/DeepSeek-R1-0528/deepseek_r1_0528 - name: model - - hostPath: - path: /data1/maas_hosted_models/models/fused_moe_triton/configs - name: cf - - hostPath: - path: /data1/sgl_cache - type: DirectoryOrCreate - name: sgl-cache - -``` - -### Decode - -Decode node deployment manifest file [decode.yaml](lws-examples/d.yaml) - -*Note: The NodeSelector section, model location section, and taint toleration section can be adjusted according to your actual deployment environment* - -```yaml -apiVersion: leaderworkerset.x-k8s.io/v1 -kind: LeaderWorkerSet -metadata: - name: deepseekr10528-decode-main -spec: - leaderWorkerTemplate: - leaderTemplate: - metadata: - labels: - role: leader - spec: - containers: - - command: - - python3 - - -m - - sglang.launch_server - - --port - - "30000" - - --host - - "0.0.0.0" - - --model-path - - /work/models - - --chunked-prefill-size - - "262144" - - --page-size - - "64" - - --enable-dp-attention - - --enable-dp-lm-head - - --dp-size - - "16" - - --moe-a2a-backend - - deepep - - --disaggregation-mode - - decode - - --mem-fraction-static - - "0.849" - - --context-length - - "32768" - - --disaggregation-ib-device - - "mlx5_bond_0,mlx5_bond_1,mlx5_bond_2,mlx5_bond_3" - - --cuda-graph-max-bs - - "64" - - --max-running-requests - - "2048" - - --tp-size - - "16" # Size of Tensor Parallelism - - --dist-init-addr - - $(LWS_LEADER_ADDRESS):20102 - - --nnodes - - $(LWS_GROUP_SIZE) - - --node-rank - - $(LWS_WORKER_INDEX) - - --trust-remote-code - - --ep-num-redundant-experts - - "32" - - --moe-dense-tp-size - - "1" - env: - - name: CUDA_LAUNCH_BLOCKING - value: "0" - - name: NVSHMEM_IB_GID_INDEX - value: "3" - - name: NVSHMEM_ENABLE_NIC_PE_MAPPING - value: "1" - - name: NCCL_IB_QPS_PER_CONNECTION - value: "8" - - name: NCCL_IB_SPLIT_DATA_ON_QPS - value: "1" - - name: NCCL_NET_PLUGIN - value: "none" - - name: NCCL_IB_TC - value: "136" - - name: NCCL_MIN_NCHANNELS - value: "4" - - name: NCCL_IB_SL - value: "5" - - name: MC_TE_METRIC - value: "true" - - name: SGLANG_MOONCAKE_TRANS_THREAD - value: "16" - - name: SGLANG_ENABLE_JIT_DEEPGEMM - value: "1" - - name: NCCL_IB_HCA - value: ^=mlx5_0,mlx5_5,mlx5_6 - - name: LWS_WORKER_INDEX - valueFrom: - fieldRef: - fieldPath: metadata.labels['leaderworkerset.sigs.k8s.io/worker-index'] - image: lmsysorg/sglang:deepep - name: sglang-leader - ports: - - containerPort: 30000 - protocol: TCP - readinessProbe: - periodSeconds: 30 - tcpSocket: - port: 30000 - resources: - limits: - nvidia.com/gpu: "8" - securityContext: - capabilities: - add: - - IPC_LOCK - privileged: true - volumeMounts: - - mountPath: /root/.cache - name: sgl-cache - - mountPath: /dev/shm - name: dshm - - mountPath: /work/models - name: model - - mountPath: /dev/infiniband - name: ib - - mountPath: /sgl-workspace/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs - name: cf - dnsPolicy: ClusterFirstWithHostNet - hostIPC: true - hostNetwork: true - nodeSelector: - pd: "yes" - tolerations: - - key: pd - operator: Exists - - key: node-role - operator: Exists - volumes: - - hostPath: - path: /data1/sgl_cache1 - type: DirectoryOrCreate - name: sgl-cache - - emptyDir: - medium: Memory - name: dshm - - hostPath: - path: /data1/maas_hosted_models/models/DeepSeek-R1-0528/deepseek_r1_0528 - name: model - - hostPath: - path: /dev/infiniband - name: ib - - hostPath: - path: /data1/maas_hosted_models/models/fused_moe_triton/configs - name: cf - restartPolicy: RecreateGroupOnPodRestart - size: 2 - workerTemplate: - metadata: {} - spec: - containers: - - command: - - python3 - - -m - - sglang.launch_server - - --model-path - - /work/models - - --chunked-prefill-size - - "262144" - - --page-size - - "64" - - --enable-dp-attention - - --enable-dp-lm-head - #- --enable-two-batch-overlap - - --dp-size - - "16" - - --moe-a2a-backend - - deepep - - --disaggregation-mode - - decode - - --mem-fraction-static - - "0.849" - - --context-length - - "32768" - - --disaggregation-ib-device - # should modify according your rdma env - - "mlx5_bond_0,mlx5_bond_1,mlx5_bond_2,mlx5_bond_3" - - --cuda-graph-max-bs - - "64" - - --max-running-requests - - "2048" - - --tp-size - - "16" # Size of Tensor Parallelism - - --dist-init-addr - - $(LWS_LEADER_ADDRESS):20102 - - --nnodes - - $(LWS_GROUP_SIZE) - - --node-rank - - $(LWS_WORKER_INDEX) - - --trust-remote-code - - --ep-num-redundant-experts - - "32" - - --moe-dense-tp-size - - "1" - env: - - name: SGLANG_HACK_DEEPEP_NUM_SMS - value: "24" - - name: SGLANG_HACK_DEEPEP_NEW_MODE - value: "0" - - name: NVSHMEM_IB_TRAFFIC_CLASS - value: "16" - - name: NVSHMEM_IB_GID_INDEX - value: "3" - - name: NVSHMEM_ENABLE_NIC_PE_MAPPING - value: "1" - - name: NCCL_IB_QPS_PER_CONNECTION - value: "8" - - name: NCCL_IB_SPLIT_DATA_ON_QPS - value: "1" - - name: NCCL_NET_PLUGIN - value: "none" - - name: NCCL_IB_TC - value: "136" - - name: NCCL_MIN_NCHANNELS - value: "4" - - name: MC_TE_METRIC - value: "true" - - name: NCCL_IB_SL - value: "5" - - name: SGLANG_MOONCAKE_TRANS_THREAD - value: "16" - - name: SGLANG_ENABLE_JIT_DEEPGEMM - value: "1" - - name: NCCL_IB_HCA - value: ^=mlx5_0,mlx5_5,mlx5_6 - - name: LWS_WORKER_INDEX - valueFrom: - fieldRef: - fieldPath: metadata.labels['leaderworkerset.sigs.k8s.io/worker-index'] - image: lmsysorg/sglang:deepep - name: sglang-worker - ports: - - containerPort: 30001 - resources: - limits: - nvidia.com/gpu: "8" - securityContext: - capabilities: - add: - - IPC_LOCK - privileged: true - volumeMounts: - - mountPath: /root/.cache - name: sgl-cache - - mountPath: /dev/shm - name: dshm - - mountPath: /work/models - name: model - - mountPath: /dev/infiniband - name: ib - - mountPath: /sgl-workspace/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs - name: cf - dnsPolicy: ClusterFirstWithHostNet - hostIPC: true - hostNetwork: true - nodeSelector: - pd: "yes" - tolerations: - - key: pd - operator: Exists - - key: node-role - operator: Exists - volumes: - - hostPath: - path: /data1/sgl_cache1 - type: DirectoryOrCreate - name: sgl-cache - - emptyDir: - medium: Memory - name: dshm - - hostPath: - path: /dev/infiniband - name: ib - - hostPath: - # modify according to you deployment env - path: /data1/maas_hosted_models/models/DeepSeek-R1-0528/deepseek_r1_0528 - name: model - - hostPath: - # modify according to you deployment env - path: /data1/maas_hosted_models/models/fused_moe_triton/configs - name: cf - networkConfig: - subdomainPolicy: Shared - replicas: 1 - rolloutStrategy: - rollingUpdateConfiguration: - maxSurge: 0 - maxUnavailable: 1 - type: RollingUpdate - startupPolicy: LeaderCreated -``` - -Execute separately: - -```bash -kubectl apply -f p.yaml -kubectl apply -f d.yaml -``` - -At this point, we have completed the deployment of the 1P1D SGLang engine part. - -To allow our users to directly experience the model API, we still need a load balancer to handle sequential calls between prefill and decode. Different companies implement LBs differently, and the community will also officially release a new LB component written in Rust in the near future. - -Currently, we use a static K8S service + minilb approach to implement model API calls. - -### Creating Service for Prefill and Decode - -#### Create prefill k8s service -[p-svc.yaml](lws-examples/p-svc.yaml) -```yaml -apiVersion: v1 -kind: Service -metadata: - name: deepseekr10528-prefill-main -spec: - selector: - leaderworkerset.sigs.k8s.io/name: deepseekr10528-prefill-main - role: leader - ports: - - protocol: TCP - port: 30000 - targetPort: 30000 -``` -Execute `kubectl apply -f p-svc.yaml` - -#### Create decode k8s service -[d-svc.yaml](lws-examples/d-svc.yaml) -```yaml -apiVersion: v1 -kind: Service -metadata: - name: deepseekr10528-decode-main -spec: - selector: - leaderworkerset.sigs.k8s.io/name: deepseekr10528-decode-main - role: leader - ports: - - protocol: TCP - port: 30000 - targetPort: 30000 -``` -Execute `kubectl apply -f d-svc.yaml` - -#### Deploy minilb and lb service -[lb.yaml](lws-examples/lb.yaml) -```yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: deepseekr10528-lb-main - labels: - app: deepseekr10528-lb -spec: - replicas: 1 - selector: - matchLabels: - app: deepseekr10528-lb - template: - metadata: - labels: - app: deepseekr10528-lb - spec: - nodeSelector: - pd: "yes" - tolerations: - - key: pd - operator: Exists - - key: node-role - operator: Exists - containers: - - name: sgl-minilb - image: lmsysorg/sglang:deepep - command: - - python - - -m - - sglang_router.launch_router - - --pd-disaggregation - - --prefill - - http://deepseekr10528-prefill-main:30000 - - --decode - - http://deepseekr10528-decode-main:30000 - - --host - - 0.0.0.0 - - --port - - "8000" - ports: - - containerPort: 8000 ---- -apiVersion: v1 -kind: Service -metadata: - name: deepseekr10528-lb-service -spec: - type: NodePort - selector: - app: deepseekr10528-lb - ports: - - protocol: TCP - port: 8000 # Service Port（In-Cluster） - targetPort: 8000 # Exposed Container - nodePort: 30800 -``` -Execute `kubectl apply -f lb.yaml` - -After waiting for all model deployments to succeed, you will get the following output: - -```bash -[root@ecs-001]# kubectl get po -deepseekr10528-decode-main-0 1/1 Running 0 74m -deepseekr10528-decode-main-0-1 1/1 Running 0 74m -deepseekr10528-lb-main-9c5dbfc57-6lcbd 1/1 Running 0 22m -deepseekr10528-prefill-main-0 1/1 Running 0 74m -deepseekr10528-prefill-main-0-1 1/1 Running 0 74m -[root@ecs-cbm-x1-pd-cpu-001 main_doc]# kubectl get svc |grep dee -deepseekr10528-decode-main ClusterIP None 97m -deepseekr10528-lb-service NodePort 172.16.242.169 8000:30800/TCP 22m -deepseekr10528-prefill-main ClusterIP None 97m -``` - -At this point, select a nodePort:30800 to access: - -```bash -[root@ecs-001]# curl -X POST "http://{nodePort}:30800/v1/chat/completions" \ -> -H "Content-Type: application/json" \ -> -H "Authorization: Bearer None" \ -> -d '{ -> "rid":"ccccdd", -> "model": "r1", -> "messages": [ -> {"role": "system", "content": "0: You are a helpful AI assistant"}, -> {"role": "user", "content": "你是谁？."} -> ], -> "max_tokens":221 -> }' -{"id":"ccccdd","object":"chat.completion","created":1750252498,"model":"qwen2","choices":[{"index":0,"message":{"role":"assistant","content":"\n嗯，用户问了一个很基础的自我介绍问题"你是谁？"。这可能是第一次互动时的常规开场白，也可能是想确认我的身份和功能范围。\n\n用户没有提供任何背景信息，语气简洁中性。这种场景下新用户的可能性较高，需要给出清晰友好的自我介绍，同时突出实用价值来降低陌生感。\n\n考虑到中文用户，应该用简体中文回复。重点要说明三点：身份归属（深度求索）、功能定位（AI助手）、服务范围（学习/工作/生活）。结尾用开放性问题引导对话很关键——既能了解需求，又能避免让用户面对空白输入框时不知所措。\n\n用波浪线结尾可以软化语气，那个笑脸表情😊刚好能中和AI的机械感。不过要控制表情符号数量，避免显得轻浮。\n\n你好呀！我是你的AI助手，由深度求索公司（DeepSeek）开发的语言模型，名字叫 **DeepSeek-R1**。你可以把我当成一个知识丰富、随叫随到的小帮手～😊\n\n我的任务就是陪你聊天、解答问题、","reasoning_content":null,"tool_calls":null},"logprobs":null,"finish_reason":"length","matched_stop":null}],"usage":{"prompt_tokens":14,"total_tokens":235,"completion_tokens":221,"prompt_tokens_details":null}} - -``` -## FAQ - -1. The current deployment startup parameters may not be fully compatible with all RDMA scenarios. Different RDMA NCCL-related environment configurations may be needed in different network environments. - -2. Some preset, optimized configurations for EPLB are not used here. You can adjust them according to [6017](https://github.com/sgl-project/sglang/issues/6017) as needed. diff --git a/docs/references/multi_node_deployment/multi_node.md b/docs/references/multi_node_deployment/multi_node.md deleted file mode 100644 index bdd0ca23d..000000000 --- a/docs/references/multi_node_deployment/multi_node.md +++ /dev/null @@ -1,100 +0,0 @@ -# Multi-Node Deployment - -## Llama 3.1 405B - -**Run 405B (fp16) on Two Nodes** - -```bash -# replace 172.16.4.52:20000 with your own node ip address and port of the first node - -python3 -m sglang.launch_server \ - --model-path meta-llama/Meta-Llama-3.1-405B-Instruct \ - --tp 16 \ - --dist-init-addr 172.16.4.52:20000 \ - --nnodes 2 \ - --node-rank 0 - -python3 -m sglang.launch_server \ - --model-path meta-llama/Meta-Llama-3.1-405B-Instruct \ - --tp 16 \ - --dist-init-addr 172.16.4.52:20000 \ - --nnodes 2 \ - --node-rank 1 -``` - -Note that LLama 405B (fp8) can also be launched on a single node. - -```bash -python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-405B-Instruct-FP8 --tp 8 -``` - -## DeepSeek V3/R1 - -Please refer to [DeepSeek documents for reference](https://docs.sglang.io/basic_usage/deepseek_v3.html#running-examples-on-multi-node). - -## Multi-Node Inference on SLURM - -This example showcases how to serve SGLang server across multiple nodes by SLURM. Submit the following job to the SLURM cluster. - -``` -#!/bin/bash -l - -#SBATCH -o SLURM_Logs/%x_%j_master.out -#SBATCH -e SLURM_Logs/%x_%j_master.err -#SBATCH -D ./ -#SBATCH -J Llama-405B-Online-Inference-TP16-SGL - -#SBATCH --nodes=2 -#SBATCH --ntasks=2 -#SBATCH --ntasks-per-node=1 # Ensure 1 task per node -#SBATCH --cpus-per-task=18 -#SBATCH --mem=224GB -#SBATCH --partition="lmsys.org" -#SBATCH --gres=gpu:8 -#SBATCH --time=12:00:00 - -echo "[INFO] Activating environment on node $SLURM_PROCID" -if ! source ENV_FOLDER/bin/activate; then - echo "[ERROR] Failed to activate environment" >&2 - exit 1 -fi - -# Define parameters -model=MODEL_PATH -tp_size=16 - -echo "[INFO] Running inference" -echo "[INFO] Model: $model" -echo "[INFO] TP Size: $tp_size" - -# Set NCCL initialization address using the hostname of the head node -HEAD_NODE=$(scontrol show hostname "$SLURM_NODELIST" | head -n 1) -NCCL_INIT_ADDR="${HEAD_NODE}:8000" -echo "[INFO] NCCL_INIT_ADDR: $NCCL_INIT_ADDR" - -# Launch the model server on each node using SLURM -srun --ntasks=2 --nodes=2 --output="SLURM_Logs/%x_%j_node$SLURM_NODEID.out" \ - --error="SLURM_Logs/%x_%j_node$SLURM_NODEID.err" \ - python3 -m sglang.launch_server \ - --model-path "$model" \ - --grammar-backend "xgrammar" \ - --tp "$tp_size" \ - --dist-init-addr "$NCCL_INIT_ADDR" \ - --nnodes 2 \ - --node-rank "$SLURM_NODEID" & - -# Wait for the NCCL server to be ready on port 30000 -while ! nc -z "$HEAD_NODE" 30000; do - sleep 1 - echo "[INFO] Waiting for $HEAD_NODE:30000 to accept connections" -done - -echo "[INFO] $HEAD_NODE:30000 is ready to accept connections" - -# Keep the script running until the SLURM job times out -wait -``` - -Then, you can test the server by sending requests following other [documents](https://docs.sglang.io/basic_usage/openai_api_completions.html). - -Thanks for [aflah02](https://github.com/aflah02) for providing the example, based on his [blog post](https://aflah02.substack.com/p/multi-node-llm-inference-with-sglang). diff --git a/docs/references/multi_node_deployment/multi_node_index.rst b/docs/references/multi_node_deployment/multi_node_index.rst deleted file mode 100644 index 78636869e..000000000 --- a/docs/references/multi_node_deployment/multi_node_index.rst +++ /dev/null @@ -1,14 +0,0 @@ -Multi-Node Deployment -===================== - -.. toctree:: - :maxdepth: 1 - :caption: Multi-Node Deployment - - multi_node.md - deploy_on_k8s.md - lws_pd/lws_pd_deploy.md - rbg_pd/deepseekv32_pd.md - -- `Deploying DeepSeek with PD Disaggregation and Large-Scale Expert Parallelism on 96 H100 GPUs `_ -- `Deploying Kimi K2 with PD Disaggregation and Large-Scale Expert Parallelism on 128 H200 GPUs `_ diff --git a/docs/references/multi_node_deployment/rbg_pd/deepseekv32_pd.md b/docs/references/multi_node_deployment/rbg_pd/deepseekv32_pd.md deleted file mode 100644 index f3d4562b7..000000000 --- a/docs/references/multi_node_deployment/rbg_pd/deepseekv32_pd.md +++ /dev/null @@ -1,567 +0,0 @@ -# DeepSeekV32-Exp RBG Based PD Deploy - -## 0. Prerequisites - -1. k8s >=1.26 -2. lws installed on k8s. -3. rbg installed on k8s. - -For RBG installation, please refer to: https://github.com/sgl-project/rbg - -## 1. Image Preparation - -`lmsysorg/sglang:latest` - - -### 2. All In One manifest file - -*Note: The NodeSelector section, model location section, and taint toleration section can be adjusted according to your actual deployment environment* - -rbg-dsv32.yml - -```yaml -apiVersion: workloads.x-k8s.io/v1alpha1 -kind: RoleBasedGroup -metadata: - name: deepseek-rbg-32exp - namespace: default -spec: - roles: - - name: prefill - replicas: 1 - workload: - apiVersion: leaderworkerset.x-k8s.io/v1 - kind: LeaderWorkerSet - restartPolicy: None - leaderWorkerSet: - size: 1 - patchLeaderTemplate: - metadata: - labels: - role: leader - pd_role: prefill - spec: - containers: - - command: - - python3 - - -m - - sglang.launch_server - - --model-path - - /work/models - - --port - - "30000" - - --trust-remote - - --host - - 0.0.0.0 - - --disaggregation-ib-device - - mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_6,mlx5_7 - - --disable-radix-cache - - --chunked-prefill-size - - "131072" - - --page-size - - "64" - # - --enable-eplb - - --ep-dispatch-algorithm - - dynamic - - --eplb-algorithm - - deepseek - - --enable-dp-lm-head - - --enable-dp-attention - - --dp-size - - "8" - - --moe-a2a-backend - - deepep - - --deepep-mode - - normal - - --disaggregation-mode - - prefill - - --mem-fraction-static - - "0.8" - - --max-prefill-tokens - - "32768" - - --context-length - - "32768" - - --tp - - "8" - - --dist-init-addr - - $(LWS_LEADER_ADDRESS):20102 - - --nnodes - - $(LWS_GROUP_SIZE) - - --node-rank - - $(LWS_WORKER_INDEX) - - --trust-remote-code - - --ep-num-redundant-experts - - "32" - - --moe-dense-tp-size - - "1" - - --max-running-requests - - "1024" - env: - - name: LWS_WORKER_INDEX - valueFrom: - fieldRef: - fieldPath: metadata.labels['leaderworkerset.sigs.k8s.io/worker-index'] - livenessProbe: - failureThreshold: 3000 - httpGet: - path: /health - port: 30000 - initialDelaySeconds: 300 - periodSeconds: 60 - successThreshold: 1 - timeoutSeconds: 10 - readinessProbe: - failureThreshold: 20 - httpGet: - path: /health - port: 30000 - periodSeconds: 30 - successThreshold: 1 - timeoutSeconds: 10 - name: sglang - ports: - - containerPort: 30000 - name: sglang-http - protocol: TCP - - patchWorkerTemplate: {} - template: - metadata: - labels: - inference-framework: sglang - inference-stack.io/monitoring: "enabled" - spec: - containers: - - name: sglang - image: lmsysorg/sglang:latest - env: - - name: SGLANG_SKIP_SGL_KERNEL_VERSION_CHECK - value: "1" - - name: CUDA_LAUNCH_BLOCKING - value: "0" - - name: SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT - value: "1000000000" - - name: NVSHMEM_IB_TRAFFIC_CLASS - value: "16" - - name: NVSHMEM_DISABLE_P2P - value: "0" - - name: ENABLE_METRICS - value: "true" - - name: NVSHMEM_IB_GID_INDEX - value: "3" - - name: NVSHMEM_IB_SL - value: "5" - - name: SGLANG_SET_CPU_AFFINITY - value: "true" - - name: SGL_ENABLE_JIT_DEEPGEMM - value: "1" - - name: NCCL_IB_QPS_PER_CONNECTION - value: "8" - - name: NCCL_IB_SPLIT_DATA_ON_QPS - value: "1" - - name: NCCL_NET_PLUGIN - value: "none" - - name: NCCL_IB_TC - value: "136" - - name: NCCL_IB_SL - value: "5" - - name: NCCL_IB_TIMEOUT - value: "22" - - name: NCCL_IB_GID_INDEX - value: "3" - - name: NCCL_MIN_NCHANNELS - value: "4" - - name: NCCL_SOCKET_IFNAME - value: bond0 - - name: GLOO_SOCKET_IFNAME - value: bond0 - - name: NCCL_IB_HCA - value: ^=mlx5_0,mlx5_5,mlx5_6 - - name: NVSHMEM_BOOTSTRAP_UID_SOCK_IFNAME - value: "bond0" - - name: MC_TE_METRIC - value: "false" - resources: - limits: - nvidia.com/gpu: "8" - securityContext: - capabilities: - add: - - IPC_LOCK - privileged: true - volumeMounts: - - mountPath: /root/.cache - name: sgl-cache - - mountPath: /dev/shm - name: dshm - - mountPath: /work/models - name: model - - mountPath: /dev/infiniband - name: ib - - mountPath: /sgl-workspace/sglang - name: src - - dnsPolicy: ClusterFirstWithHostNet - hostIPC: true - hostNetwork: true - nodeSelector: - pd: "yes" - tolerations: - - key: pd - operator: Exists - volumes: - - hostPath: - path: /var/run/sys-topology - name: topo - - hostPath: - path: /data1/sgl_cache4 - type: DirectoryOrCreate - name: sgl-cache - - emptyDir: - medium: Memory - name: dshm - - hostPath: - path: /data/DeepSeek-V3.2-Exp - name: model - - hostPath: - path: /dev/infiniband - name: ib - - hostPath: - path: /data/src/sglang - type: DirectoryOrCreate - name: src - - - name: decode - replicas: 1 - workload: - apiVersion: leaderworkerset.x-k8s.io/v1 - kind: LeaderWorkerSet - leaderWorkerSet: - size: 1 - patchLeaderTemplate: - metadata: - labels: - role: leader - pd_role: decode - spec: - containers: - - command: - - python3 - - -m - - sglang.launch_server - - --model-path - - /work/models - - --port - - "30000" - - --trust-remote - - --host - - 0.0.0.0 - - --disaggregation-ib-device - - mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_6,mlx5_7 - - --chunked-prefill-size - - "131072" - - --eplb-rebalance-layers-per-chunk - - "29" - - --page-size - - "64" - - --enable-dp-attention - - --enable-dp-lm-head - - --dp-size - - "8" - - --moe-a2a-backend - - deepep - - --deepep-mode - - low_latency - - --disaggregation-mode - - decode - - --mem-fraction-static - - "0.8" - - --context-length - - "32768" - - --max-running-requests - - "2048" - - --tp-size - - "8" # Size of Tensor Parallelism - - --cuda-graph-max-bs - - "16" - - --dist-init-addr - - $(LWS_LEADER_ADDRESS):20102 - - --nnodes - - $(LWS_GROUP_SIZE) - - --node-rank - - $(LWS_WORKER_INDEX) - - --trust-remote-code - - --ep-num-redundant-experts - - "32" - - --moe-dense-tp-size - - "1" - env: - - name: LWS_WORKER_INDEX - valueFrom: - fieldRef: - fieldPath: metadata.labels['leaderworkerset.sigs.k8s.io/worker-index'] - livenessProbe: - failureThreshold: 30000 - httpGet: - path: /health - port: 30000 - initialDelaySeconds: 300 - periodSeconds: 60 - successThreshold: 1 - timeoutSeconds: 10 - name: sglang - readinessProbe: - failureThreshold: 20 - httpGet: - path: /health - port: 30000 - periodSeconds: 30 - successThreshold: 1 - timeoutSeconds: 10 - patchWorkerTemplate: - spec: - containers: - - command: - - python3 - - -m - - sglang.launch_server - - --model-path - - /work/models - - --crash-dump-folder - - /log - - --chunked-prefill-size - - "262144" - - --eplb-rebalance-layers-per-chunk - - "29" - - --page-size - - "64" - - --enable-dp-attention - - --enable-dp-lm-head - - --dp-size - - "32" - - --moe-a2a-backend - - "deepep" - - --deepep-mode - - low_latency - - --disaggregation-mode - - decode - - --mem-fraction-static - - "0.849" - - --context-length - - "32768" - - --disaggregation-ib-device - - mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_6,mlx5_7 - - --max-running-requests - - "4096" - - --cuda-graph-max-bs - - "16" - - --tp-size - - "8" # Size of Tensor Parallelism - - --dist-init-addr - - $(LWS_LEADER_ADDRESS):20102 - - --nnodes - - $(LWS_GROUP_SIZE) - - --node-rank - - $(LWS_WORKER_INDEX) - - --trust-remote-code - - --ep-num-redundant-experts - - "32" - - --moe-dense-tp-size - - "1" - env: - - name: LWS_WORKER_INDEX - valueFrom: - fieldRef: - fieldPath: metadata.labels['leaderworkerset.sigs.k8s.io/worker-index'] - name: sglang - template: - metadata: - labels: - inference-framework: sglang-unuse - inference-stack.io/monitoring: "enabled" - spec: - containers: - - image: lmsysorg/sglang:latest - name: sglang - resources: - limits: - nvidia.com/gpu: "8" - securityContext: - capabilities: - add: - - IPC_LOCK - privileged: true - volumeMounts: - - mountPath: /root/.cache - name: sgl-cache - - mountPath: /dev/shm - name: dshm - - mountPath: /work/models - name: model - - mountPath: /dev/infiniband - name: ib - - mountPath: /sgl-workspace/sglang - name: src - env: - - name: SGLANG_SKIP_SGL_KERNEL_VERSION_CHECK - value: "1" - - name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT - value: "100000000" - - name: NVSHMEM_DISABLE_P2P - value: "0" - - name: NVSHMEM_IB_TRAFFIC_CLASS - value: "16" - - name: NVSHMEM_IB_SL - value: "5" - - name: ENABLE_METRICS - value: "true" - - name: CUDA_LAUNCH_BLOCKING - value: "0" - - name: NVSHMEM_IB_GID_INDEX - value: "3" - - name: NCCL_IB_QPS_PER_CONNECTION - value: "8" - - name: NCCL_IB_SPLIT_DATA_ON_QPS - value: "1" - - name: NCCL_NET_PLUGIN - value: "none" - - name: NCCL_IB_TC - value: "136" - - name: NCCL_IB_SL - value: "5" - - name: NCCL_IB_TIMEOUT - value: "22" - - name: NCCL_IB_GID_INDEX - value: "3" - - name: NCCL_MIN_NCHANNELS - value: "4" - - name: NCCL_SOCKET_IFNAME - value: bond0 - - name: GLOO_SOCKET_IFNAME - value: bond0 - - name: NVSHMEM_BOOTSTRAP_UID_SOCK_IFNAME - value: "bond0" - - name: NCCL_IB_HCA - value: ^=mlx5_0,mlx5_5,mlx5_6 - - name: MC_TE_METRIC - value: "false" - - name: SGL_ENABLE_JIT_DEEPGEMM - value: "1" - dnsPolicy: ClusterFirstWithHostNet - hostIPC: true - hostNetwork: true - nodeSelector: - pd: "yes" - tolerations: - - key: pd - operator: Exists - volumes: - - hostPath: - path: /var/run/sys-topology - name: topo - - hostPath: - path: /data1/sgl_cache4 - type: DirectoryOrCreate - name: sgl-cache - - hostPath: - path: /data/src/sglang - type: DirectoryOrCreate - name: src - - emptyDir: - medium: Memory - name: dshm - - hostPath: - path: /data/DeepSeek-V3.2-Exp - name: model - - hostPath: - path: /dev/infiniband - name: ib - - name: router - replicas: 1 - dependencies: [ "decode", "prefill" ] - template: - spec: - containers: - - name: scheduler - image: lmsysorg/sglang:latest - command: - - sh - - -c - - > - python3 -m sglang_router.launch_router - --host 0.0.0.0 - --port 8080 - --pd-disaggregation - --policy random - --service-discovery - --service-discovery-namespace ${NAMESPACE} - --service-discovery-port 30000 - --prefill-selector pd_role=prefill - --decode-selector pd_role=decode - --max-payload-size 2147483648 - --worker-startup-timeout-secs 1200 - env: - - name: NAMESPACE - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app: deepseek-rbg-32exp - name: deepseek-rbg-32exp - namespace: default -spec: - ports: - - name: http - port: 8080 - protocol: TCP - targetPort: 8080 - nodePort: 30080 - - selector: - rolebasedgroup.workloads.x-k8s.io/name: deepseek-rbg-32exp - rolebasedgroup.workloads.x-k8s.io/role: router - type: NodePort - -``` - -```bash -[root@ecs-001]# kubectl get po -n default -deepseek-rbg-32exp-decode-main-0 1/1 Running 0 74m -deepseek-rbg-32exp-decode-0-1 1/1 Running 0 74m -deepseek-rbg-32exp-router-9c5dbfc57 1/1 Running 0 22m -deepseek-rbg-32exp-prefill-0 1/1 Running 0 74m - -[root@ecs-cbm-x1-pd-cpu-001 main_doc]# kubectl get svc |grep dee -deepseek-rbg-32exp-decode ClusterIP None 97m -deepseek-rbg-32exp-router-service NodePort 172.16.242.169 8000:30800/TCP 22m -deepseek-rbg-32exp-prefill ClusterIP None 97m -``` - -At this point, select a nodePort:30800 to access: - -```bash -[root@ecs-001]# curl -X POST "http://{nodePort}:30800/v1/chat/completions" \ -> -H "Content-Type: application/json" \ -> -H "Authorization: Bearer None" \ -> -d '{ -> "rid":"ccccdd", -> "model": "dsv32", -> "messages": [ -> {"role": "system", "content": "0: You are a helpful AI assistant"}, -> {"role": "user", "content": "你是谁？."} -> ], -> "max_tokens":221 -> }' -{"id":"ccccdd","object":"chat.completion","created":1750252498,"model":"qwen2","choices":[{"index":0,"message":{"role":"assistant","content":"\n嗯，用户问了一个很基础的自我介绍问题"你是谁？"。这可能是第一次互动时的常规开场白，也可能是想确认我的身份和功能范围。\n\n用户没有提供任何背景信息，语气简洁中性。这种场景下新用户的可能性较高，需要给出清晰友好的自我介绍，同时突出实用价值来降低陌生感。\n\n考虑到中文用户，应该用简体中文回复。重点要说明三点：身份归属（深度求索）、功能定位（AI助手）、服务范围（学习/工作/生活）。结尾用开放性问题引导对话很关键——既能了解需求，又能避免让用户面对空白输入框时不知所措。\n\n用波浪线结尾可以软化语气，那个笑脸表情😊刚好能中和AI的机械感。不过要控制表情符号数量，避免显得轻浮。\n\n你好呀！我是你的AI助手，由深度求索公司（DeepSeek）开发的语言模型，名字叫 **DeepSeek-V32**。你可以把我当成一个知识丰富、随叫随到的小帮手～😊\n\n我的任务就是陪你聊天、解答问题、","reasoning_content":null,"tool_calls":null},"logprobs":null,"finish_reason":"length","matched_stop":null}],"usage":{"prompt_tokens":14,"total_tokens":235,"completion_tokens":221,"prompt_tokens_details":null}} - -``` -## FAQ - -1. The current deployment startup parameters may not be fully compatible with all RDMA scenarios. Different RDMA NCCL-related environment configurations may be needed in different network environments. - -2. Please ensure that the sglang code in the image has incorporated the changes from [PR #10912](https://github.com/sgl-project/sglang/pull/10912). diff --git a/docs/references/post_training_integration.md b/docs/references/post_training_integration.md deleted file mode 100644 index 4dddf5905..000000000 --- a/docs/references/post_training_integration.md +++ /dev/null @@ -1,31 +0,0 @@ -# Post-Training Integration - -SGLang has become the de facto inference backend for modern LLM training frameworks, powering state-of-the-art models across the industry. From GLM-4.6 to Qwen3, leading models leverage SGLang's high-performance inference during reinforcement learning and post-training workflows. - -What makes SGLang essential for post-training? - -- Open-To-Use Refit Functionality: diverse method for colocate or disaggregate -- Easy To Postpone Generation: enable partial rollout and dedicated rollout control -- Fine-Grained Engine Sleep And Wake Up: facilitate maximum-powered rollout and training -- Training Serving Alignment: ensure the performance consistency in training and serving -- Load Balancing Router: cache-aware load-balancing for high-throughput rollout -- Deterministic Inference: ensure zero kl divergence between rollout and training - -These capabilities, combined with native integration support across major frameworks, have established SGLang as the infrastructure backbone for modern LLM/VLMs post-training. We also share our latest work in this slide, [Optimizing Large-Scale RL with SGLang](https://gamma.app/docs/Optimizing-RL-with-SGLang-y0kqgj877k34779). - -## Adoption - -- [**Miles**](https://github.com/radixark/miles): Enterprise-scale RL framework for large MoE models with SGLang-native rollout, speculative training, and production-grade stability -- [**slime**](https://github.com/THUDM/slime): Post-training framework combining Megatron and SGLang, used to train GLM-4.6 -- [**AReaL**](https://github.com/inclusionAI/AReaL): Fully asynchronous RL system achieving 2.77x speedup with SGLang backend for continuous rollout generation -- [**ROLL**](https://github.com/alibaba/ROLL): ROLL is an efficient and user-friendly RL library designed for Large Language Models utilizing Large Scale GPU resources -- [**verl**](https://github.com/volcengine/verl): Full-stack RLHF framework supporting PPO, GRPO, and ReMax with modular SGLang integration -- [**Unsloth**](https://docs.unsloth.ai/basics/inference-and-deployment/sglang-guide): 2x faster fine-tuning with optimized kernels, deploys seamlessly with SGLang inference -- [**LLaMA Factory**](https://github.com/hiyouga/LLaMA-Factory): Unified framework for training 100+ LLMs with LoRA, QLoRA, and full fine-tuning methods -- [**Tunix**](https://github.com/google/tunix): Google's JAX-native library for LLM post-training with SFT, DPO, PPO, and GRPO support -- [**RL2**](https://github.com/ChenmienTan/RL2): Ray Less Reinforcement Learning, a concise library of post-training for large language models - - -## Collaboration - -Due to the privacy of the design partners, we cannot list the companies that adopt SGLang for post-training. However, we are happy to share the details with you if you are interested and trust the choice among 10+ top companies and frontier labs across US and China. If you are interested in integrating SGLang with your training framework or need technical support, we're here to help! Reach out to us at **rl_team@lmsys.org** for partnerships, integration guidance, and custom feature development. diff --git a/docs/references/production_metrics.md b/docs/references/production_metrics.md deleted file mode 100644 index d104584ee..000000000 --- a/docs/references/production_metrics.md +++ /dev/null @@ -1,267 +0,0 @@ -# Production Metrics - -SGLang exposes the following metrics via Prometheus. You can enable it by adding `--enable-metrics` when you launch the server. - -An example of the monitoring dashboard is available in [examples/monitoring/grafana.json](https://github.com/sgl-project/sglang/blob/main/examples/monitoring/grafana/dashboards/json/sglang-dashboard.json). - -Here is an example of the metrics: - -``` -$ curl http://localhost:30000/metrics -# HELP sglang:prompt_tokens_total Number of prefill tokens processed. -# TYPE sglang:prompt_tokens_total counter -sglang:prompt_tokens_total{model_name="meta-llama/Llama-3.1-8B-Instruct"} 8.128902e+06 -# HELP sglang:generation_tokens_total Number of generation tokens processed. -# TYPE sglang:generation_tokens_total counter -sglang:generation_tokens_total{model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.557572e+06 -# HELP sglang:token_usage The token usage -# TYPE sglang:token_usage gauge -sglang:token_usage{model_name="meta-llama/Llama-3.1-8B-Instruct"} 0.28 -# HELP sglang:cache_hit_rate The cache hit rate -# TYPE sglang:cache_hit_rate gauge -sglang:cache_hit_rate{model_name="meta-llama/Llama-3.1-8B-Instruct"} 0.007507552643049313 -# HELP sglang:time_to_first_token_seconds Histogram of time to first token in seconds. -# TYPE sglang:time_to_first_token_seconds histogram -sglang:time_to_first_token_seconds_sum{model_name="meta-llama/Llama-3.1-8B-Instruct"} 2.3518979474117756e+06 -sglang:time_to_first_token_seconds_bucket{le="0.001",model_name="meta-llama/Llama-3.1-8B-Instruct"} 0.0 -sglang:time_to_first_token_seconds_bucket{le="0.005",model_name="meta-llama/Llama-3.1-8B-Instruct"} 0.0 -sglang:time_to_first_token_seconds_bucket{le="0.01",model_name="meta-llama/Llama-3.1-8B-Instruct"} 0.0 -sglang:time_to_first_token_seconds_bucket{le="0.02",model_name="meta-llama/Llama-3.1-8B-Instruct"} 0.0 -sglang:time_to_first_token_seconds_bucket{le="0.04",model_name="meta-llama/Llama-3.1-8B-Instruct"} 1.0 -sglang:time_to_first_token_seconds_bucket{le="0.06",model_name="meta-llama/Llama-3.1-8B-Instruct"} 3.0 -sglang:time_to_first_token_seconds_bucket{le="0.08",model_name="meta-llama/Llama-3.1-8B-Instruct"} 6.0 -sglang:time_to_first_token_seconds_bucket{le="0.1",model_name="meta-llama/Llama-3.1-8B-Instruct"} 6.0 -sglang:time_to_first_token_seconds_bucket{le="0.25",model_name="meta-llama/Llama-3.1-8B-Instruct"} 6.0 -sglang:time_to_first_token_seconds_bucket{le="0.5",model_name="meta-llama/Llama-3.1-8B-Instruct"} 6.0 -sglang:time_to_first_token_seconds_bucket{le="0.75",model_name="meta-llama/Llama-3.1-8B-Instruct"} 6.0 -sglang:time_to_first_token_seconds_bucket{le="1.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 27.0 -sglang:time_to_first_token_seconds_bucket{le="2.5",model_name="meta-llama/Llama-3.1-8B-Instruct"} 140.0 -sglang:time_to_first_token_seconds_bucket{le="5.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 314.0 -sglang:time_to_first_token_seconds_bucket{le="7.5",model_name="meta-llama/Llama-3.1-8B-Instruct"} 941.0 -sglang:time_to_first_token_seconds_bucket{le="10.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 1330.0 -sglang:time_to_first_token_seconds_bucket{le="15.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 1970.0 -sglang:time_to_first_token_seconds_bucket{le="20.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 2326.0 -sglang:time_to_first_token_seconds_bucket{le="25.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 2417.0 -sglang:time_to_first_token_seconds_bucket{le="30.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 2513.0 -sglang:time_to_first_token_seconds_bucket{le="+Inf",model_name="meta-llama/Llama-3.1-8B-Instruct"} 11008.0 -sglang:time_to_first_token_seconds_count{model_name="meta-llama/Llama-3.1-8B-Instruct"} 11008.0 -# HELP sglang:e2e_request_latency_seconds Histogram of End-to-end request latency in seconds -# TYPE sglang:e2e_request_latency_seconds histogram -sglang:e2e_request_latency_seconds_sum{model_name="meta-llama/Llama-3.1-8B-Instruct"} 3.116093850019932e+06 -sglang:e2e_request_latency_seconds_bucket{le="0.3",model_name="meta-llama/Llama-3.1-8B-Instruct"} 0.0 -sglang:e2e_request_latency_seconds_bucket{le="0.5",model_name="meta-llama/Llama-3.1-8B-Instruct"} 6.0 -sglang:e2e_request_latency_seconds_bucket{le="0.8",model_name="meta-llama/Llama-3.1-8B-Instruct"} 6.0 -sglang:e2e_request_latency_seconds_bucket{le="1.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 6.0 -sglang:e2e_request_latency_seconds_bucket{le="1.5",model_name="meta-llama/Llama-3.1-8B-Instruct"} 6.0 -sglang:e2e_request_latency_seconds_bucket{le="2.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 6.0 -sglang:e2e_request_latency_seconds_bucket{le="2.5",model_name="meta-llama/Llama-3.1-8B-Instruct"} 6.0 -sglang:e2e_request_latency_seconds_bucket{le="5.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.0 -sglang:e2e_request_latency_seconds_bucket{le="10.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 10.0 -sglang:e2e_request_latency_seconds_bucket{le="15.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 11.0 -sglang:e2e_request_latency_seconds_bucket{le="20.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 14.0 -sglang:e2e_request_latency_seconds_bucket{le="30.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 247.0 -sglang:e2e_request_latency_seconds_bucket{le="40.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 486.0 -sglang:e2e_request_latency_seconds_bucket{le="50.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 845.0 -sglang:e2e_request_latency_seconds_bucket{le="60.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 1513.0 -sglang:e2e_request_latency_seconds_bucket{le="+Inf",model_name="meta-llama/Llama-3.1-8B-Instruct"} 11228.0 -sglang:e2e_request_latency_seconds_count{model_name="meta-llama/Llama-3.1-8B-Instruct"} 11228.0 -# HELP sglang:time_per_output_token_seconds Histogram of time per output token in seconds. -# TYPE sglang:time_per_output_token_seconds histogram -sglang:time_per_output_token_seconds_sum{model_name="meta-llama/Llama-3.1-8B-Instruct"} 866964.5791549598 -sglang:time_per_output_token_seconds_bucket{le="0.005",model_name="meta-llama/Llama-3.1-8B-Instruct"} 1.0 -sglang:time_per_output_token_seconds_bucket{le="0.01",model_name="meta-llama/Llama-3.1-8B-Instruct"} 73.0 -sglang:time_per_output_token_seconds_bucket{le="0.015",model_name="meta-llama/Llama-3.1-8B-Instruct"} 382.0 -sglang:time_per_output_token_seconds_bucket{le="0.02",model_name="meta-llama/Llama-3.1-8B-Instruct"} 593.0 -sglang:time_per_output_token_seconds_bucket{le="0.025",model_name="meta-llama/Llama-3.1-8B-Instruct"} 855.0 -sglang:time_per_output_token_seconds_bucket{le="0.03",model_name="meta-llama/Llama-3.1-8B-Instruct"} 1035.0 -sglang:time_per_output_token_seconds_bucket{le="0.04",model_name="meta-llama/Llama-3.1-8B-Instruct"} 1815.0 -sglang:time_per_output_token_seconds_bucket{le="0.05",model_name="meta-llama/Llama-3.1-8B-Instruct"} 11685.0 -sglang:time_per_output_token_seconds_bucket{le="0.075",model_name="meta-llama/Llama-3.1-8B-Instruct"} 433413.0 -sglang:time_per_output_token_seconds_bucket{le="0.1",model_name="meta-llama/Llama-3.1-8B-Instruct"} 4.950195e+06 -sglang:time_per_output_token_seconds_bucket{le="0.15",model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.039435e+06 -sglang:time_per_output_token_seconds_bucket{le="0.2",model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.171662e+06 -sglang:time_per_output_token_seconds_bucket{le="0.3",model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.266055e+06 -sglang:time_per_output_token_seconds_bucket{le="0.4",model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.296752e+06 -sglang:time_per_output_token_seconds_bucket{le="0.5",model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.312226e+06 -sglang:time_per_output_token_seconds_bucket{le="0.75",model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.339675e+06 -sglang:time_per_output_token_seconds_bucket{le="1.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.357747e+06 -sglang:time_per_output_token_seconds_bucket{le="2.5",model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.389414e+06 -sglang:time_per_output_token_seconds_bucket{le="+Inf",model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.400757e+06 -sglang:time_per_output_token_seconds_count{model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.400757e+06 -# HELP sglang:func_latency_seconds Function latency in seconds -# TYPE sglang:func_latency_seconds histogram -sglang:func_latency_seconds_sum{name="generate_request"} 4.514771912145079 -sglang:func_latency_seconds_bucket{le="0.05",name="generate_request"} 14006.0 -sglang:func_latency_seconds_bucket{le="0.07500000000000001",name="generate_request"} 14006.0 -sglang:func_latency_seconds_bucket{le="0.1125",name="generate_request"} 14006.0 -sglang:func_latency_seconds_bucket{le="0.16875",name="generate_request"} 14006.0 -sglang:func_latency_seconds_bucket{le="0.253125",name="generate_request"} 14006.0 -sglang:func_latency_seconds_bucket{le="0.3796875",name="generate_request"} 14006.0 -sglang:func_latency_seconds_bucket{le="0.56953125",name="generate_request"} 14006.0 -sglang:func_latency_seconds_bucket{le="0.8542968750000001",name="generate_request"} 14006.0 -sglang:func_latency_seconds_bucket{le="1.2814453125",name="generate_request"} 14006.0 -sglang:func_latency_seconds_bucket{le="1.9221679687500002",name="generate_request"} 14006.0 -sglang:func_latency_seconds_bucket{le="2.8832519531250003",name="generate_request"} 14006.0 -sglang:func_latency_seconds_bucket{le="4.3248779296875",name="generate_request"} 14007.0 -sglang:func_latency_seconds_bucket{le="6.487316894531251",name="generate_request"} 14007.0 -sglang:func_latency_seconds_bucket{le="9.730975341796876",name="generate_request"} 14007.0 -sglang:func_latency_seconds_bucket{le="14.596463012695313",name="generate_request"} 14007.0 -sglang:func_latency_seconds_bucket{le="21.89469451904297",name="generate_request"} 14007.0 -sglang:func_latency_seconds_bucket{le="32.84204177856446",name="generate_request"} 14007.0 -sglang:func_latency_seconds_bucket{le="49.26306266784668",name="generate_request"} 14007.0 -sglang:func_latency_seconds_bucket{le="+Inf",name="generate_request"} 14007.0 -sglang:func_latency_seconds_count{name="generate_request"} 14007.0 -# HELP sglang:num_running_reqs The number of running requests -# TYPE sglang:num_running_reqs gauge -sglang:num_running_reqs{model_name="meta-llama/Llama-3.1-8B-Instruct"} 162.0 -# HELP sglang:num_used_tokens The number of used tokens -# TYPE sglang:num_used_tokens gauge -sglang:num_used_tokens{model_name="meta-llama/Llama-3.1-8B-Instruct"} 123859.0 -# HELP sglang:gen_throughput The generate throughput (token/s) -# TYPE sglang:gen_throughput gauge -sglang:gen_throughput{model_name="meta-llama/Llama-3.1-8B-Instruct"} 86.50814177726902 -# HELP sglang:num_queue_reqs The number of requests in the waiting queue -# TYPE sglang:num_queue_reqs gauge -sglang:num_queue_reqs{model_name="meta-llama/Llama-3.1-8B-Instruct"} 2826.0 -``` - -## Setup Guide - -This section describes how to set up the monitoring stack (Prometheus + Grafana) provided in the `examples/monitoring` directory. - -### Prerequisites - -- Docker and Docker Compose installed -- SGLang server running with metrics enabled - -### Usage - -1. **Start your SGLang server with metrics enabled:** - - ```bash - python -m sglang.launch_server \ - --model-path \ - --port 30000 \ - --enable-metrics \ - --enable-mfu-metrics - ``` - Replace `` with the actual path to your model (e.g., `meta-llama/Meta-Llama-3.1-8B-Instruct`). Ensure the server is accessible from the monitoring stack (you might need `--host 0.0.0.0` if running in Docker). By default, the metrics endpoint will be available at `http://:30000/metrics`. - -2. **Navigate to the monitoring example directory:** - ```bash - cd examples/monitoring - ``` - -3. **Start the monitoring stack:** - ```bash - docker compose up -d - ``` - This command will start Prometheus and Grafana in the background. - -4. **Access the monitoring interfaces:** - * **Grafana:** Open your web browser and go to [http://localhost:3000](http://localhost:3000). - * **Prometheus:** Open your web browser and go to [http://localhost:9090](http://localhost:9090). - -5. **Log in to Grafana:** - * Default Username: `admin` - * Default Password: `admin` - You will be prompted to change the password upon your first login. - -6. **View the Dashboard:** - The SGLang dashboard is pre-configured and should be available automatically. Navigate to `Dashboards` -> `Browse` -> `SGLang Monitoring` folder -> `SGLang Dashboard`. - -### Troubleshooting - -* **Port Conflicts:** If you encounter errors like "port is already allocated," check if other services (including previous instances of Prometheus/Grafana) are using ports `9090` or `3000`. Use `docker ps` to find running containers and `docker stop ` to stop them, or use `lsof -i :` to find other processes using the ports. You might need to adjust the ports in the `docker-compose.yaml` file if they permanently conflict with other essential services on your system. - -To modify Grafana's port to the other one(like 3090) in your Docker Compose file, you need to explicitly specify the port mapping under the grafana service. - - Option 1: Add GF_SERVER_HTTP_PORT to the environment section: - ``` - environment: - - GF_AUTH_ANONYMOUS_ENABLED=true - - GF_SERVER_HTTP_PORT=3090 # <-- Add this line - ``` - Option 2: Use port mapping: - ``` - grafana: - image: grafana/grafana:latest - container_name: grafana - ports: - - "3090:3000" # <-- Host:Container port mapping - ``` -* **Connection Issues:** - * Ensure both Prometheus and Grafana containers are running (`docker ps`). - * Verify the Prometheus data source configuration in Grafana (usually auto-configured via `grafana/datasources/datasource.yaml`). Go to `Connections` -> `Data sources` -> `Prometheus`. The URL should point to the Prometheus service (e.g., `http://prometheus:9090`). - * Confirm that your SGLang server is running and the metrics endpoint (`http://:30000/metrics`) is accessible *from the Prometheus container*. If SGLang is running on your host machine and Prometheus is in Docker, use `host.docker.internal` (on Docker Desktop) or your machine's network IP instead of `localhost` in the `prometheus.yaml` scrape configuration. -* **No Data on Dashboard:** - * Generate some traffic to your SGLang server to produce metrics. For example, run a benchmark: - ```bash - python3 -m sglang.bench_serving --backend sglang --dataset-name random --num-prompts 100 --random-input 128 --random-output 128 - ``` - * Check the Prometheus UI (`http://localhost:9090`) under `Status` -> `Targets` to see if the SGLang endpoint is being scraped successfully. - * Verify the `model_name` and `instance` labels in your Prometheus metrics match the variables used in the Grafana dashboard. You might need to adjust the Grafana dashboard variables or the labels in your Prometheus configuration. - -### Configuration Files - -The monitoring setup is defined by the following files within the `examples/monitoring` directory: - -* `docker-compose.yaml`: Defines the Prometheus and Grafana services. -* `prometheus.yaml`: Prometheus configuration, including scrape targets. -* `grafana/datasources/datasource.yaml`: Configures the Prometheus data source for Grafana. -* `grafana/dashboards/config/dashboard.yaml`: Tells Grafana to load dashboards from the specified path. -* `grafana/dashboards/json/sglang-dashboard.json`: The actual Grafana dashboard definition in JSON format. - -You can customize the setup by modifying these files. For instance, you might need to update the `static_configs` target in `prometheus.yaml` if your SGLang server runs on a different host or port. - -#### Check if the metrics are being collected - -Run: -``` -python3 -m sglang.bench_serving \ - --backend sglang \ - --dataset-name random \ - --num-prompts 3000 \ - --random-input 1024 \ - --random-output 1024 \ - --random-range-ratio 0.5 -``` - -to generate some requests. - -Then you should be able to see the metrics in the Grafana dashboard. - -## Estimated Performance Metrics (MFU-related) - -SGLang exports the following estimated per-GPU counters that can be used to derive -Model FLOPs Utilization (MFU)-related signals: - -- `sglang:estimated_flops_per_gpu_total`: Estimated floating-point operations. -- `sglang:estimated_read_bytes_per_gpu_total`: Estimated bytes read from memory. -- `sglang:estimated_write_bytes_per_gpu_total`: Estimated bytes written to memory. - -These metrics are available when both `--enable-metrics` and -`--enable-mfu-metrics` are enabled. - -These are cumulative counters. Use Prometheus `rate(...)` to get per-second values. - -### PromQL examples - -Average TFLOPS per GPU: - -```promql -rate(sglang:estimated_flops_per_gpu_total[1m]) / 1e12 -``` - -Average estimated memory bandwidth in GB/s: - -```promql -(rate(sglang:estimated_read_bytes_per_gpu_total[1m]) + - rate(sglang:estimated_write_bytes_per_gpu_total[1m])) / 1e9 -``` - -### Notes - -- These metrics are estimates intended for observability and trend analysis. -- Estimated memory bytes reflect modeled traffic and are not a direct hardware - counter from GPU profilers. diff --git a/docs/references/production_request_trace.md b/docs/references/production_request_trace.md deleted file mode 100644 index 4fbe12584..000000000 --- a/docs/references/production_request_trace.md +++ /dev/null @@ -1,138 +0,0 @@ -# Production Request Tracing - -SGLang exports request trace data based on the OpenTelemetry Collector. You can enable tracing by adding the `--enable-trace` and configure the OpenTelemetry Collector endpoint using `--otlp-traces-endpoint` when launching the server. - -You can find example screenshots of the visualization in https://github.com/sgl-project/sglang/issues/8965. - -## Setup Guide -This section explains how to configure the request tracing and export the trace data. -1. Install the required packages and tools - * install Docker and Docker Compose - * install the dependencies - ```bash - # enter the SGLang root directory - pip install -e "python[tracing]" - - # or manually install the dependencies using pip - pip install opentelemetry-sdk opentelemetry-api opentelemetry-exporter-otlp opentelemetry-exporter-otlp-proto-grpc - ``` - -2. Launch OpenTelemetry collector and Jaeger - ```bash - docker compose -f examples/monitoring/tracing_compose.yaml up -d - ``` - -3. Start your SGLang server with tracing enabled - ```bash - # set env variables - export SGLANG_OTLP_EXPORTER_SCHEDULE_DELAY_MILLIS=500 - export SGLANG_OTLP_EXPORTER_MAX_EXPORT_BATCH_SIZE=64 - # start the prefill and decode server - python -m sglang.launch_server --enable-trace --otlp-traces-endpoint 0.0.0.0:4317 - # start the model-gate-way - python -m sglang_router.launch_router --enable-trace --otlp-traces-endpoint 0.0.0.0:4317 - ``` - - Replace `0.0.0.0:4317` with the actual endpoint of the OpenTelemetry collector. If you launched the openTelemetry collector with tracing_compose.yaml, the default receiving port is 4317. - - To use the HTTP/protobuf span exporter, set the following environment variable and point to an HTTP endpoint, for example, `http://0.0.0.0:4318/v1/traces`. - ```bash - export OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=http/protobuf - ``` - - -4. Raise some requests -5. Observe whether trace data is being exported - * Access port 16686 of Jaeger using a web browser to visualize the request traces. - * The OpenTelemetry Collector also exports trace data in JSON format to /tmp/otel_trace.json. In a follow-up patch, we will provide a tool to convert this data into a Perfetto-compatible format, enabling visualization of requests in the Perfetto UI. - -6. Dynamically adjust trace level - The trace level accepts configurable values from `0` to `3`. The meanings of different trace level values are as follows: - ``` - 0: disable tracing - 1: Trace important slices - 2: Trace all slices except nested ones - 3: Trace all slices (default) - ``` - **At startup** — set `SGLANG_TRACE_LEVEL` before launching the server: - ```bash - SGLANG_TRACE_LEVEL=2 python -m sglang.launch_server --enable-trace --otlp-traces-endpoint 0.0.0.0:4317 - ``` - - **At runtime** — dynamically adjust via HTTP API without restarting: - ```bash - curl http://0.0.0.0:30000/set_trace_level?level=2 - ``` - Replace `0.0.0.0:30000` with your actual server address, and replace `level=2` with the level you want to set. - - **Note**: You must set the parameter `--enable-trace`; otherwise, the trace capability will not be enabled regardless of any dynamic adjustments to the trace level. - -## How to add Tracing for slices you're interested in?(API introduction) -We have already inserted instrumentation points in the tokenizer and scheduler main threads. If you wish to trace additional request execution segments or perform finer-grained tracing, please use the APIs from the tracing package as described below. - -**All of the following implementations are done in python/sglang/srt/observability/req_time_stats.py. If you want to add another slice, please do it here.** - -1. Initialization - - Every process involved in tracing during the initialization phase should execute: - ```python - process_tracing_init(otlp_traces_endpoint, server_name) - ``` - The otlp_traces_endpoint is obtained from the arguments, and you can set server_name freely, but it should remain consistent across all processes. - - Every thread involved in tracing during the initialization phase should execute: - ```python - trace_set_thread_info("thread label", tp_rank, dp_rank) - ``` - The "thread label" can be regarded as the name of the thread, used to distinguish different threads in the visualization view. - -2. Create a trace context for a request - Each request needs to call `TraceReqContext()` to initialize a request context, which is used to generate slice spans and record request stage info. You can either store it within the request object or maintain it as a global variable. - -3. Mark the beginning and end of a request - ``` - trace_ctx.trace_req_start(). - trace_ctx.trace_req_finish() - ``` - trace_req_start() and trace_req_finish() must be called within the same process, for example, in the tokenizer. - -4. Add tracing for a slice - - * Add slice tracing normally: - ```python - trace_ctx.trace_slice_start(RequestStage.TOKENIZER.stage_name) - trace_ctx.trace_slice_end(RequestStage.TOKENIZER.stage_name) - - or - trace_ctx.trace_slice(slice: TraceSliceContext) - ``` - - - The end of the last slice in a thread must be marked with thread_finish_flag=True, or explicitly call trace_ctx.abort(); otherwise, the thread's span will not be properly generated. - ```python - trace_ctx.slice_end(RequestStage.D.stage_name, thread_finish_flag = True) - trace_ctx.abort() - ``` - -5. When the request execution flow transfers to another thread, the thread context needs to be explicitly rebuilt. - - receiver: Execute the following code after receiving the request via ZMQ - ```python - trace_ctx.rebuild_thread_context() - ``` - -## How to Extend the Tracing Framework to Support Complex Tracing Scenarios - -The currently provided tracing package still has potential for further development. If you wish to build more advanced features upon it, you must first understand its existing design principles. - -The core of the tracing framework's implementation lies in the design of the span structure and the trace context. To aggregate scattered slices and enable concurrent tracking of multiple requests, we have designed a three-level trace context structure or span structure: `TraceReqContext`, `TraceThreadContext` and `TraceSliceContext`. Their relationship is as follows: -``` -TraceReqContext (req_id="req-123") -├── TraceThreadContext(thread_label="scheduler", tp_rank=0) -| └── TraceSliceContext(slice_name="prefill") -| -└── TraceThreadContext(thread_label="scheduler", tp_rank=1) - └── TraceSliceContext(slice_name="prefill") -``` - -Each traced request maintains a global `TraceReqContext` and creates a corresponding request span. For every thread that processes the request, a `TraceThreadContext` is recorded and a thread span is created. The `TraceThreadContext` is nested within the `TraceReqContext`, and each currently traced code slice—potentially nested—is stored in its associated `TraceThreadContext`. - -In addition to the above hierarchy, each slice also records its previous slice via Span.add_link(), which can be used to trace the execution flow. diff --git a/docs/references/release_lookup.rst b/docs/references/release_lookup.rst deleted file mode 100644 index 2e8833f6c..000000000 --- a/docs/references/release_lookup.rst +++ /dev/null @@ -1,325 +0,0 @@ -Release Lookup -============== - -Find which SGLang release first included a specific PR or commit. - -.. raw:: html - - - -
-
- - -
- -
-
Initializing…
-
- - diff --git a/docs/references/torch_compile_cache.md b/docs/references/torch_compile_cache.md deleted file mode 100644 index f2bb257f4..000000000 --- a/docs/references/torch_compile_cache.md +++ /dev/null @@ -1,13 +0,0 @@ -# Enabling cache for torch.compile - -SGLang uses `max-autotune-no-cudagraphs` mode of torch.compile. The auto-tuning can be slow. -If you want to deploy a model on many different machines, you can ship the torch.compile cache to these machines and skip the compilation steps. - -This is based on https://pytorch.org/tutorials/recipes/torch_compile_caching_tutorial.html - - -1. Generate the cache by setting TORCHINDUCTOR_CACHE_DIR and running the model once. -``` -TORCHINDUCTOR_CACHE_DIR=/root/inductor_root_cache python3 -m sglang.launch_server --model meta-llama/Llama-3.1-8B-Instruct --enable-torch-compile -``` -2. Copy the cache folder to other machines and launch the server with `TORCHINDUCTOR_CACHE_DIR`. diff --git a/docs/release_lookup/README.md b/docs/release_lookup/README.md deleted file mode 100644 index 3472ded2f..000000000 --- a/docs/release_lookup/README.md +++ /dev/null @@ -1,39 +0,0 @@ -# SGLang Release Lookup Tool - -This tool allows users to find the earliest release that contains a specific PR or commit. -It runs entirely in the browser using a static JSON index generated from the git history. - -## Usage - -1. **Generate the Index**: - Run the Python script to generate the `release_index.json` file from your local git repository. - - ```bash - python3 generate_index.py --output release_index.json - ``` - - This script: - - Finds all tags matching `v*` and `gateway-v*`. - - Sorts them by creation date. - - Traverses the history to find which release first introduced each commit and PR. - - Extracts PR numbers from commit messages. - -2. **Open the Tool**: - Open `index.html` in your browser. - - ```bash - # You can open it directly if your browser supports local file fetch (Firefox usually does), - # or serve it locally: - python3 -m http.server - # Then go to http://localhost:8000/index.html - ``` - -## Files - -- `index.html`: The UI for the lookup tool. -- `generate_index.py`: Script to build the index. -- `release_index.json`: The index file used by the UI. - -## Logic - -The tool determines the "earliest release" based on the tag creation date. It traverses tags from oldest to newest. Any commit reachable from a tag (that wasn't reachable from a previous tag) is assigned to that release. diff --git a/docs/release_lookup/generate_index.py b/docs/release_lookup/generate_index.py deleted file mode 100644 index d8415e41d..000000000 --- a/docs/release_lookup/generate_index.py +++ /dev/null @@ -1,222 +0,0 @@ -import argparse -import json -import os -import re -import subprocess -import sys -from datetime import datetime - -# Short hash length for commits (7 is git's default short hash) -SHORT_HASH_LEN = 8 -COMMIT_CHUNK_SIZE = 1000 - - -def run_git(cmd): - try: - output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) - return output.decode("utf-8", errors="replace").strip() - except subprocess.CalledProcessError as e: - print(f"Error running cmd: {cmd}\n{e.output.decode('utf-8', errors='replace')}") - sys.exit(1) - - -def is_stable_release(tag_name): - """Check if tag is a stable release (not rc/alpha/beta).""" - # Skip release candidates, alpha, beta versions - if re.search(r"(rc|alpha|beta)\d*$", tag_name, re.IGNORECASE): - return False - return True - - -def get_tags(): - # Get tags sorted by creator date - cmd = [ - "git", - "tag", - "--list", - "v*", - "gateway-v*", - "--sort=creatordate", - "--format=%(refname:short)|%(creatordate:iso8601)|%(objectname)", - ] - raw = run_git(cmd) - tags = [] - if not raw: - return [] - for line in raw.split("\n"): - parts = line.split("|") - if len(parts) >= 3: - name, date, commit = parts[0], parts[1], parts[2] - # Skip non-stable releases (rc, alpha, beta) - if not is_stable_release(name): - continue - tag_type = "gateway" if name.startswith("gateway-") else "main" - tags.append( - {"name": name, "date": date, "commit": commit, "type": tag_type} - ) - return tags - - -def extract_pr_num(message): - lines = message.strip().split("\n") - first_line = lines[0] - - m = re.search(r"\(#(\d+)\)$", first_line) - if m: - return m.group(1) - - m = re.search(r"Merge pull request #(\d+)", message) - if m: - return m.group(1) - - return None - - -def process_tag_line(tags, commit_map, pr_map, tag_type, tag_to_idx): - """Process a single release line (main or gateway) independently.""" - seen_commits = set() - - for tag in tags: - tag_name = tag["name"] - print(f"Processing {tag_name}...") - - commits = run_git(["git", "rev-list", tag_name]).split("\n") - - new_commits = [] - for c in commits: - c = c.strip() - if not c: - continue - if c in seen_commits: - continue - new_commits.append(c) - seen_commits.add(c) - - if not new_commits: - continue - - for i in range(0, len(new_commits), COMMIT_CHUNK_SIZE): - chunk = new_commits[i : i + COMMIT_CHUNK_SIZE] - - cmd = ["git", "show", "-s", "--format=%H|%B%n--END-COMMIT--"] + chunk - raw_logs = run_git(cmd) - - entries = raw_logs.split("--END-COMMIT--\n") - for log_entry in entries: - if not log_entry.strip(): - continue - parts = log_entry.split("|", 1) - if len(parts) < 2: - continue - sha = parts[0].strip() - msg = parts[1].strip() - - tag_idx = tag_to_idx[tag_name] - - # Store release index using full SHA as key - if sha not in commit_map: - commit_map[sha] = {} - commit_map[sha][tag_type] = tag_idx - - pr = extract_pr_num(msg) - if pr: - if pr not in pr_map: - pr_map[pr] = {} - if tag_type not in pr_map[pr]: - pr_map[pr][tag_type] = tag_idx - - -def main(): - parser = argparse.ArgumentParser( - description="Generate lookup index for sglang releases" - ) - parser.add_argument( - "--output", default="release_index.json", help="Output JSON file" - ) - args = parser.parse_args() - - tags = get_tags() - print(f"Found {len(tags)} tags.") - - main_tags = [t for t in tags if t["type"] == "main"] - gateway_tags = [t for t in tags if t["type"] == "gateway"] - - print(f" - {len(main_tags)} main tags") - print(f" - {len(gateway_tags)} gateway tags") - - # Build tag list and index mapping - # Tags array: [name, date, type] for each tag - tag_list = [] - tag_to_idx = {} - - for tag in tags: - tag_to_idx[tag["name"]] = len(tag_list) - # Compact format: [name, date, type (0=main, 1=gateway)] - tag_list.append( - [tag["name"], tag["date"], 1 if tag["type"] == "gateway" else 0] - ) - - pr_map = {} - commit_map_full = {} - - process_tag_line(main_tags, commit_map_full, pr_map, "m", tag_to_idx) - process_tag_line(gateway_tags, commit_map_full, pr_map, "g", tag_to_idx) - - # Convert full SHAs to short SHAs, checking for collisions - commit_map = {} - short_to_full_map = {} - for full_sha, data in commit_map_full.items(): - short_sha = full_sha[:SHORT_HASH_LEN] - if short_sha in short_to_full_map and short_to_full_map[short_sha] != full_sha: - print( - f"CRITICAL: Short SHA collision detected for '{short_sha}'\n" - f" Commit 1: {short_to_full_map[short_sha]}\n" - f" Commit 2: {full_sha}\n" - "Please increase SHORT_HASH_LEN and re-run.", - file=sys.stderr, - ) - sys.exit(1) - commit_map[short_sha] = data - short_to_full_map[short_sha] = full_sha - - # Compact output format: - # - tags: array of [name, date, type] - # - prs: {pr_num: tag_idx} or {pr_num: {m: idx, g: idx}} - # - commits: {short_hash: tag_idx} or {short_hash: {m: idx, g: idx}} - - # Simplify single-entry dicts to just the value - def simplify_map(m): - result = {} - for k, v in m.items(): - if len(v) == 1: - # Single entry: just store the index directly with type prefix - key_type, idx = list(v.items())[0] - result[k] = f"{key_type}{idx}" - else: - # Multiple entries: keep as dict - result[k] = v - return result - - output_data = { - "t": tag_list, # tags - "p": simplify_map(pr_map), # prs - "c": simplify_map(commit_map), # commits - "g": datetime.now().isoformat(), # generated_at - } - - # Write minified JSON with a trailing newline for formatter compatibility. - json_str = json.dumps(output_data, separators=(",", ":")) - - with open(args.output, "w", encoding="utf-8") as f: - f.write(json_str) - f.write("\n") - - json_size = os.path.getsize(args.output) - - print(f"Index generated at {args.output}") - print(f"Stats: {len(tag_list)} tags, {len(pr_map)} PRs, {len(commit_map)} commits.") - print(f"Size: {json_size/1024:.1f} KB") - - -if __name__ == "__main__": - main() diff --git a/docs/release_lookup/index.html b/docs/release_lookup/index.html deleted file mode 100644 index dc8219de5..000000000 --- a/docs/release_lookup/index.html +++ /dev/null @@ -1,515 +0,0 @@ - - - - - - SGLang Release Lookup - - - - -
-

Release Lookup

-

Find which SGLang release first included your PR or commit.

- -
- - -
- - - -
- -
Initializing...
-
- - - - - diff --git a/docs/release_lookup/release_index.json b/docs/release_lookup/release_index.json deleted file mode 100644 index 4a8606e24..000000000 --- a/docs/release_lookup/release_index.json +++ /dev/null @@ -1 +0,0 @@ -{"t":[["v0.1.3","2024-01-16 05:55:25 +0000",0],["v0.1.5","2024-01-17 18:37:02 -0800",0],["v0.1.6","2024-01-21 01:45:02 -0800",0],["v0.1.7","2024-01-21 10:31:02 +0000",0],["v0.1.8","2024-01-24 03:33:34 -0800",0],["v0.1.9","2024-01-24 11:37:25 +0000",0],["v0.1.10","2024-01-30 15:37:52 +0000",0],["v0.1.11","2024-02-03 02:50:13 -0800",0],["v0.1.12","2024-02-11 06:43:45 -0800",0],["v0.1.13","2024-03-11 05:49:27 -0700",0],["v0.1.14","2024-03-22 13:42:22 -0700",0],["v0.1.15","2024-05-12 14:22:33 -0700",0],["v0.1.16","2024-05-13 17:29:17 -0700",0],["v0.1.17","2024-06-07 19:49:18 -0700",0],["v0.1.18","2024-07-04 06:27:29 +0000",0],["v0.1.19","2024-07-09 02:23:14 -0700",0],["v0.1.20","2024-07-13 17:27:55 -0700",0],["v0.1.21","2024-07-15 13:10:53 -0700",0],["v0.1.22","2024-07-20 03:39:50 -0700",0],["v0.1.23","2024-07-23 13:49:34 -0700",0],["v0.1.24","2024-07-24 15:55:01 -0700",0],["v0.2.0","2024-07-25 08:03:36 -0700",0],["v0.2.5","2024-07-27 05:56:30 +1000",0],["v0.2.6","2024-07-27 20:29:33 -0700",0],["v0.2.7","2024-07-30 20:41:10 +1000",0],["v0.2.8","2024-08-01 14:18:26 -0700",0],["v0.2.9","2024-08-02 01:45:48 -0700",0],["v0.2.9.post1","2024-08-02 12:08:00 -0700",0],["v0.2.10","2024-08-04 16:52:51 -0700",0],["v0.2.11","2024-08-07 20:47:53 +0800",0],["v0.2.12","2024-08-12 20:59:38 +1000",0],["v0.2.13","2024-08-16 03:50:43 +1000",0],["v0.2.14","2024-08-27 00:28:24 +1000",0],["v0.2.14.post1","2024-08-28 21:16:47 +1000",0],["v0.2.14.post2","2024-08-28 18:46:33 +0000",0],["v0.2.15","2024-09-01 22:22:38 -0700",0],["v0.3.0","2024-09-04 04:21:21 -0700",0],["v0.3.1.post1","2024-09-17 01:47:31 -0700",0],["v0.3.1.post2","2024-09-19 02:03:38 -0700",0],["v0.3.1.post3","2024-09-21 11:17:45 +0800",0],["v0.3.2","2024-09-25 14:17:09 +0800",0],["v0.3.3","2024-10-08 12:58:41 -0700",0],["v0.3.3.post1","2024-10-11 07:56:16 -0700",0],["v0.3.4","2024-10-19 08:17:41 -0700",0],["v0.3.4.post1","2024-10-21 21:16:43 -0700",0],["v0.3.4.post2","2024-10-25 11:07:19 -0700",0],["v0.3.5","2024-11-03 13:48:11 -0800",0],["v0.3.5.post1","2024-11-13 10:27:12 -0800",0],["v0.3.5.post2","2024-11-15 06:54:00 -0800",0],["v0.3.6","2024-11-22 19:27:30 +0800",0],["v0.3.6.post1","2024-11-25 17:31:37 -0800",0],["v0.3.6.post2","2024-11-27 03:35:30 -0800",0],["v0.3.6.post3","2024-11-30 01:41:16 +0800",0],["v0.4.0","2024-12-03 11:55:41 -0800",0],["v0.4.0.post1","2024-12-06 06:08:19 -0800",0],["v0.4.0.post2","2024-12-21 21:16:34 +0800",0],["v0.4.1","2024-12-26 07:14:51 +0800",0],["v0.4.1.post1","2024-12-28 00:11:06 +0800",0],["v0.4.1.post2","2024-12-30 00:11:46 +0800",0],["v0.4.1.post3","2024-12-29 14:25:53 -0800",0],["v0.4.1.post4","2025-01-06 01:29:54 +0800",0],["v0.4.1.post5","2025-01-11 23:10:02 +0800",0],["v0.4.1.post6","2025-01-15 16:23:42 +0800",0],["v0.4.1.post7","2025-01-20 21:50:55 +0800",0],["v0.4.2","2025-01-27 21:42:05 +0800",0],["v0.4.2.post1","2025-01-31 20:35:55 +0800",0],["v0.4.2.post2","2025-02-05 17:35:02 +0800",0],["v0.4.2.post3","2025-02-07 08:20:03 -0800",0],["v0.4.2.post4","2025-02-10 14:12:16 +0800",0],["v0.4.3","2025-02-14 09:43:14 +0800",0],["v0.4.3.post1","2025-02-17 21:58:19 +0800",0],["v0.4.3.post2","2025-02-18 02:48:30 +0800",0],["v0.4.3.post3","2025-03-05 17:26:10 -0800",0],["v0.4.3.post4","2025-03-06 12:50:28 -0800",0],["v0.4.4","2025-03-13 02:49:58 -0700",0],["v0.4.4.post1","2025-03-13 17:53:46 -0700",0],["v0.4.4.post2","2025-03-26 19:58:00 -0700",0],["v0.4.4.post3","2025-03-28 23:21:24 -0700",0],["v0.4.4.post4","2025-04-05 15:36:17 -0700",0],["v0.4.5","2025-04-07 00:35:00 -0700",0],["v0.4.5.post1","2025-04-15 23:00:07 -0700",0],["v0.4.5.post2","2025-04-20 14:12:37 -0700",0],["v0.4.5.post3","2025-04-21 18:16:20 -0700",0],["v0.4.6","2025-04-27 14:07:05 -0700",0],["v0.4.6.post1","2025-04-28 12:57:08 -0700",0],["v0.4.6.post2","2025-04-30 22:04:40 -0700",0],["v0.4.6.post3","2025-05-09 15:38:47 -0700",0],["v0.4.6.post4","2025-05-13 01:57:51 -0700",0],["v0.4.6.post5","2025-05-24 00:48:05 -0700",0],["v0.4.7","2025-06-10 01:56:20 -0700",0],["v0.4.7.post1","2025-06-16 15:20:29 -0700",0],["v0.4.8","2025-06-23 23:14:22 -0700",0],["v0.4.8.post1","2025-06-26 02:21:12 -0700",0],["v0.4.9","2025-07-05 17:40:29 -0700",0],["gateway-v0.1.5","2025-07-06 22:54:17 -0700",1],["v0.4.9.post1","2025-07-09 00:28:17 -0700",0],["v0.4.9.post2","2025-07-11 21:11:20 -0700",0],["gateway-v0.1.6","2025-07-20 23:13:20 -0700",1],["v0.4.9.post3","2025-07-22 15:55:48 -0700",0],["v0.4.9.post4","2025-07-25 17:12:47 -0700",0],["v0.4.9.post5","2025-07-28 02:11:06 -0700",0],["v0.4.9.post6","2025-07-29 02:30:07 -0700",0],["v0.4.10","2025-07-31 20:50:17 +0800",0],["gateway-v0.1.7","2025-07-31 11:24:12 -0700",1],["gateway-v0.1.8","2025-07-31 19:00:23 -0700",1],["v0.4.10.post1","2025-08-01 12:07:30 +0800",0],["v0.4.10.post2","2025-08-03 03:43:29 -0700",0],["gateway-v0.1.9","2025-08-07 09:29:12 -0700",1],["v0.5.1","2025-08-23 07:09:26 -0700",0],["v0.5.1.post1","2025-08-24 01:14:17 -0700",0],["v0.5.1.post2","2025-08-25 03:45:09 -0700",0],["v0.5.1.post3","2025-08-27 15:42:42 -0700",0],["v0.5.2","2025-09-11 16:09:20 -0700",0],["v0.5.3","2025-10-06 20:07:02 +0800",0],["v0.5.3.post1","2025-10-09 15:19:59 -0700",0],["gateway-v0.2.0","2025-10-14 22:10:30 -0400",1],["v0.5.3.post2","2025-10-15 16:49:14 -0700",0],["v0.5.3.post3","2025-10-16 13:14:55 -0700",0],["gateway-v0.2.1","2025-10-20 21:08:45 -0700",1],["v0.5.4","2025-10-23 18:01:40 -0700",0],["v0.5.4.post1","2025-10-27 09:35:20 +0800",0],["gateway-v0.2.2","2025-10-30 14:40:13 -0700",1],["v0.5.4.post2","2025-10-31 17:38:50 -0700",0],["v0.5.4.post3","2025-11-04 18:32:11 -0800",0],["v0.5.5","2025-11-07 00:46:19 +0800",0],["v0.5.5.post1","2025-11-10 11:53:43 -0800",0],["v0.5.5.post2","2025-11-12 20:35:20 +0800",0],["gateway-v0.2.3","2025-11-14 19:04:20 -0800",1],["v0.5.5.post3","2025-11-16 17:55:38 -0800",0],["v0.5.6","2025-12-02 17:17:13 -0800",0],["v0.5.6.post1","2025-12-08 13:41:01 -0800",0],["gateway-v0.2.4","2025-12-09 16:36:17 -0800",1],["v0.5.6.post2","2025-12-11 12:29:52 -0800",0],["gateway-v0.3.0","2025-12-24 16:25:05 -0500",1],["v0.5.7","2026-01-01 10:59:48 +0800",0],["gateway-v0.3.1","2026-01-08 21:50:34 -0800",1],["v0.5.8","2026-01-23 09:58:11 -0800",0],["v0.5.8.post1","2026-02-05 20:56:52 +0800",0]],"p":{"10":{"m":0,"g":94},"8":{"m":0,"g":94},"7":{"m":0,"g":94},"6":{"m":0,"g":94},"4":{"m":0,"g":94},"3":{"m":0,"g":94},"2":{"m":0,"g":94},"1":{"m":0,"g":94},"32":{"m":1,"g":94},"18":{"m":1,"g":94},"30":{"m":1,"g":94},"20":{"m":1,"g":94},"19":{"m":1,"g":94},"9":{"m":1,"g":94},"17":{"m":1,"g":94},"16":{"m":1,"g":94},"15":{"m":1,"g":94},"12":{"m":1,"g":94},"11":{"m":1,"g":94},"68":{"m":2,"g":94},"67":{"m":2,"g":94},"64":{"m":2,"g":94},"63":{"m":2,"g":94},"58":{"m":2,"g":94},"57":{"m":2,"g":94},"36":{"m":2,"g":94},"52":{"m":2,"g":94},"50":{"m":2,"g":94},"49":{"m":2,"g":94},"47":{"m":2,"g":94},"46":{"m":2,"g":94},"45":{"m":2,"g":94},"42":{"m":2,"g":94},"34":{"m":2,"g":94},"33":{"m":2,"g":94},"93":{"m":4,"g":94},"92":{"m":4,"g":94},"90":{"m":4,"g":94},"87":{"m":4,"g":94},"84":{"m":4,"g":94},"83":{"m":4,"g":94},"82":{"m":4,"g":94},"75":{"m":4,"g":94},"80":{"m":4,"g":94},"72":{"m":4,"g":94},"37":{"m":4,"g":94},"71":{"m":4,"g":94},"113":{"m":6,"g":94},"121":{"m":6,"g":94},"120":{"m":6,"g":94},"118":{"m":6,"g":94},"114":{"m":6,"g":94},"117":{"m":6,"g":94},"108":{"m":6,"g":94},"103":{"m":6,"g":94},"101":{"m":6,"g":94},"98":{"m":6,"g":94},"48":{"m":6,"g":94},"97":{"m":6,"g":94},"95":{"m":6,"g":94},"134":{"m":7,"g":94},"133":{"m":7,"g":94},"132":{"m":7,"g":94},"112":{"m":7,"g":94},"129":{"m":7,"g":94},"125":{"m":7,"g":94},"119":{"m":7,"g":94},"116":{"m":7,"g":94},"178":{"m":8,"g":94},"177":{"m":8,"g":94},"172":{"m":8,"g":94},"174":{"m":8,"g":94},"168":{"m":8,"g":94},"170":{"m":8,"g":94},"162":{"m":8,"g":94},"160":{"m":8,"g":94},"156":{"m":8,"g":94},"155":{"m":8,"g":94},"130":{"m":8,"g":94},"141":{"m":8,"g":94},"153":{"m":8,"g":94},"148":{"m":8,"g":94},"146":{"m":8,"g":94},"144":{"m":8,"g":94},"142":{"m":8,"g":94},"137":{"m":8,"g":94},"136":{"m":8,"g":94},"280":{"m":9,"g":94},"279":{"m":9,"g":94},"230":{"m":9,"g":94},"277":{"m":9,"g":94},"278":{"m":9,"g":94},"256":{"m":9,"g":94},"222":{"m":9,"g":94},"261":{"m":9,"g":94},"275":{"m":9,"g":94},"263":{"m":9,"g":94},"201":{"m":9,"g":94},"224":{"m":9,"g":94},"253":{"m":9,"g":94},"226":{"m":9,"g":94},"195":{"m":9,"g":94},"198":{"m":9,"g":94},"225":{"m":9,"g":94},"219":{"m":9,"g":94},"193":{"m":9,"g":94},"210":{"m":9,"g":94},"207":{"m":9,"g":94},"200":{"m":9,"g":94},"196":{"m":9,"g":94},"189":{"m":9,"g":94},"186":{"m":9,"g":94},"184":{"m":9,"g":94},"181":{"m":9,"g":94},"182":{"m":9,"g":94},"324":{"m":10,"g":94},"323":{"m":10,"g":94},"301":{"m":10,"g":94},"304":{"m":10,"g":94},"311":{"m":10,"g":94},"291":{"m":10,"g":94},"290":{"m":10,"g":94},"288":{"m":10,"g":94},"286":{"m":10,"g":94},"287":{"m":10,"g":94},"282":{"m":10,"g":94},"242":{"m":10,"g":94},"281":{"m":10,"g":94},"431":{"m":11,"g":94},"430":{"m":11,"g":94},"429":{"m":11,"g":94},"428":{"m":11,"g":94},"427":{"m":11,"g":94},"422":{"m":11,"g":94},"420":{"m":11,"g":94},"380":{"m":11,"g":94},"416":{"m":11,"g":94},"415":{"m":11,"g":94},"412":{"m":11,"g":94},"411":{"m":11,"g":94},"381":{"m":11,"g":94},"392":{"m":11,"g":94},"390":{"m":11,"g":94},"406":{"m":11,"g":94},"399":{"m":11,"g":94},"395":{"m":11,"g":94},"394":{"m":11,"g":94},"382":{"m":11,"g":94},"385":{"m":11,"g":94},"378":{"m":11,"g":94},"372":{"m":11,"g":94},"375":{"m":11,"g":94},"364":{"m":11,"g":94},"370":{"m":11,"g":94},"368":{"m":11,"g":94},"369":{"m":11,"g":94},"358":{"m":11,"g":94},"355":{"m":11,"g":94},"354":{"m":11,"g":94},"346":{"m":11,"g":94},"338":{"m":11,"g":94},"345":{"m":11,"g":94},"343":{"m":11,"g":94},"315":{"m":11,"g":94},"332":{"m":11,"g":94},"337":{"m":11,"g":94},"331":{"m":11,"g":94},"329":{"m":11,"g":94},"293":{"m":11,"g":94},"327":{"m":11,"g":94},"326":{"m":11,"g":94},"298":{"m":11,"g":94},"438":{"m":12,"g":94},"437":{"m":12,"g":94},"426":{"m":12,"g":94},"436":{"m":12,"g":94},"434":{"m":12,"g":94},"418":{"m":12,"g":94},"433":{"m":12,"g":94},"363":{"m":12,"g":94},"432":{"m":12,"g":94},"515":{"m":13,"g":94},"514":{"m":13,"g":94},"505":{"m":13,"g":94},"512":{"m":13,"g":94},"502":{"m":13,"g":94},"500":{"m":13,"g":94},"511":{"m":13,"g":94},"493":{"m":13,"g":94},"491":{"m":13,"g":94},"492":{"m":13,"g":94},"488":{"m":13,"g":94},"486":{"m":13,"g":94},"480":{"m":13,"g":94},"484":{"m":13,"g":94},"477":{"m":13,"g":94},"475":{"m":13,"g":94},"476":{"m":13,"g":94},"440":{"m":13,"g":94},"471":{"m":13,"g":94},"463":{"m":13,"g":94},"470":{"m":13,"g":94},"460":{"m":13,"g":94},"459":{"m":13,"g":94},"458":{"m":13,"g":94},"457":{"m":13,"g":94},"456":{"m":13,"g":94},"250":{"m":13,"g":94},"451":{"m":13,"g":94},"449":{"m":13,"g":94},"448":{"m":13,"g":94},"447":{"m":13,"g":94},"446":{"m":13,"g":94},"441":{"m":13,"g":94},"419":{"m":13,"g":94},"579":{"m":14,"g":94},"585":{"m":14,"g":94},"583":{"m":14,"g":94},"578":{"m":14,"g":94},"577":{"m":14,"g":94},"576":{"m":14,"g":94},"574":{"m":14,"g":94},"545":{"m":14,"g":94},"571":{"m":14,"g":94},"569":{"m":14,"g":94},"568":{"m":14,"g":94},"567":{"m":14,"g":94},"566":{"m":14,"g":94},"564":{"m":14,"g":94},"563":{"m":14,"g":94},"561":{"m":14,"g":94},"560":{"m":14,"g":94},"559":{"m":14,"g":94},"558":{"m":14,"g":94},"557":{"m":14,"g":94},"556":{"m":14,"g":94},"554":{"m":14,"g":94},"550":{"m":14,"g":94},"553":{"m":14,"g":94},"551":{"m":14,"g":94},"546":{"m":14,"g":94},"542":{"m":14,"g":94},"540":{"m":14,"g":94},"539":{"m":14,"g":94},"538":{"m":14,"g":94},"517":{"m":14,"g":94},"531":{"m":14,"g":94},"516":{"m":14,"g":94},"526":{"m":14,"g":94},"525":{"m":14,"g":94},"524":{"m":14,"g":94},"518":{"m":14,"g":94},"605":{"m":15,"g":94},"503":{"m":15,"g":94},"530":{"m":15,"g":94},"603":{"m":15,"g":94},"598":{"m":15,"g":94},"602":{"m":15,"g":94},"604":{"m":15,"g":94},"601":{"m":15,"g":94},"600":{"m":15,"g":94},"599":{"m":15,"g":94},"586":{"m":15,"g":94},"594":{"m":15,"g":94},"593":{"m":15,"g":94},"592":{"m":15,"g":94},"588":{"m":15,"g":94},"618":{"m":16,"g":94},"616":{"m":16,"g":94},"615":{"m":16,"g":94},"614":{"m":16,"g":94},"613":{"m":16,"g":94},"612":{"m":16,"g":94},"611":{"m":16,"g":94},"610":{"m":16,"g":94},"609":{"m":16,"g":94},"607":{"m":16,"g":94},"626":{"m":17,"g":94},"625":{"m":17,"g":94},"623":{"m":17,"g":94},"621":{"m":17,"g":94},"620":{"m":17,"g":94},"619":{"m":17,"g":94},"677":{"m":18,"g":94},"676":{"m":18,"g":94},"675":{"m":18,"g":94},"664":{"m":18,"g":94},"673":{"m":18,"g":94},"671":{"m":18,"g":94},"669":{"m":18,"g":94},"668":{"m":18,"g":94},"667":{"m":18,"g":94},"640":{"m":18,"g":94},"666":{"m":18,"g":94},"665":{"m":18,"g":94},"663":{"m":18,"g":94},"662":{"m":18,"g":94},"661":{"m":18,"g":94},"660":{"m":18,"g":94},"659":{"m":18,"g":94},"655":{"m":18,"g":94},"657":{"m":18,"g":94},"658":{"m":18,"g":94},"656":{"m":18,"g":94},"654":{"m":18,"g":94},"653":{"m":18,"g":94},"651":{"m":18,"g":94},"650":{"m":18,"g":94},"648":{"m":18,"g":94},"647":{"m":18,"g":94},"649":{"m":18,"g":94},"646":{"m":18,"g":94},"645":{"m":18,"g":94},"643":{"m":18,"g":94},"642":{"m":18,"g":94},"617":{"m":18,"g":94},"638":{"m":18,"g":94},"637":{"m":18,"g":94},"636":{"m":18,"g":94},"635":{"m":18,"g":94},"632":{"m":18,"g":94},"633":{"m":18,"g":94},"624":{"m":18,"g":94},"630":{"m":18,"g":94},"631":{"m":18,"g":94},"629":{"m":18,"g":94},"628":{"m":18,"g":94},"627":{"m":18,"g":94},"705":{"m":19,"g":94},"704":{"m":19,"g":94},"701":{"m":19,"g":94},"702":{"m":19,"g":94},"700":{"m":19,"g":94},"698":{"m":19,"g":94},"697":{"m":19,"g":94},"696":{"m":19,"g":94},"695":{"m":19,"g":94},"694":{"m":19,"g":94},"692":{"m":19,"g":94},"691":{"m":19,"g":94},"690":{"m":19,"g":94},"689":{"m":19,"g":94},"688":{"m":19,"g":94},"687":{"m":19,"g":94},"686":{"m":19,"g":94},"685":{"m":19,"g":94},"684":{"m":19,"g":94},"682":{"m":19,"g":94},"681":{"m":19,"g":94},"679":{"m":19,"g":94},"670":{"m":19,"g":94},"678":{"m":19,"g":94},"718":{"m":20,"g":94},"717":{"m":20,"g":94},"716":{"m":20,"g":94},"715":{"m":20,"g":94},"714":{"m":20,"g":94},"713":{"m":20,"g":94},"712":{"m":20,"g":94},"711":{"m":20,"g":94},"708":{"m":20,"g":94},"709":{"m":20,"g":94},"707":{"m":20,"g":94},"706":{"m":20,"g":94},"730":{"m":21,"g":94},"729":{"m":21,"g":94},"728":{"m":21,"g":94},"727":{"m":21,"g":94},"726":{"m":21,"g":94},"725":{"m":21,"g":94},"720":{"m":21,"g":94},"723":{"m":21,"g":94},"724":{"m":21,"g":94},"722":{"m":21,"g":94},"721":{"m":21,"g":94},"719":{"m":21,"g":94},"755":{"m":22,"g":94},"754":{"m":22,"g":94},"753":{"m":22,"g":94},"752":{"m":22,"g":94},"751":{"m":22,"g":94},"740":{"m":22,"g":94},"743":{"m":22,"g":94},"742":{"m":22,"g":94},"739":{"m":22,"g":94},"741":{"m":22,"g":94},"736":{"m":22,"g":94},"734":{"m":22,"g":94},"733":{"m":22,"g":94},"731":{"m":22,"g":94},"779":{"m":23,"g":94},"778":{"m":23,"g":94},"776":{"m":23,"g":94},"775":{"m":23,"g":94},"774":{"m":23,"g":94},"773":{"m":23,"g":94},"772":{"m":23,"g":94},"766":{"m":23,"g":94},"770":{"m":23,"g":94},"769":{"m":23,"g":94},"767":{"m":23,"g":94},"761":{"m":23,"g":94},"763":{"m":23,"g":94},"762":{"m":23,"g":94},"760":{"m":23,"g":94},"757":{"m":23,"g":94},"693":{"m":23,"g":94},"830":{"m":24,"g":94},"829":{"m":24,"g":94},"828":{"m":24,"g":94},"825":{"m":24,"g":94},"826":{"m":24,"g":94},"823":{"m":24,"g":94},"824":{"m":24,"g":94},"822":{"m":24,"g":94},"821":{"m":24,"g":94},"820":{"m":24,"g":94},"819":{"m":24,"g":94},"807":{"m":24,"g":94},"817":{"m":24,"g":94},"814":{"m":24,"g":94},"815":{"m":24,"g":94},"812":{"m":24,"g":94},"809":{"m":24,"g":94},"699":{"m":24,"g":94},"806":{"m":24,"g":94},"802":{"m":24,"g":94},"805":{"m":24,"g":94},"803":{"m":24,"g":94},"800":{"m":24,"g":94},"799":{"m":24,"g":94},"797":{"m":24,"g":94},"793":{"m":24,"g":94},"796":{"m":24,"g":94},"795":{"m":24,"g":94},"794":{"m":24,"g":94},"792":{"m":24,"g":94},"791":{"m":24,"g":94},"790":{"m":24,"g":94},"789":{"m":24,"g":94},"788":{"m":24,"g":94},"787":{"m":24,"g":94},"786":{"m":24,"g":94},"785":{"m":24,"g":94},"784":{"m":24,"g":94},"783":{"m":24,"g":94},"781":{"m":24,"g":94},"877":{"m":25,"g":94},"872":{"m":25,"g":94},"873":{"m":25,"g":94},"871":{"m":25,"g":94},"870":{"m":25,"g":94},"869":{"m":25,"g":94},"864":{"m":25,"g":94},"862":{"m":25,"g":94},"861":{"m":25,"g":94},"860":{"m":25,"g":94},"811":{"m":25,"g":94},"852":{"m":25,"g":94},"858":{"m":25,"g":94},"856":{"m":25,"g":94},"855":{"m":25,"g":94},"850":{"m":25,"g":94},"848":{"m":25,"g":94},"843":{"m":25,"g":94},"842":{"m":25,"g":94},"838":{"m":25,"g":94},"840":{"m":25,"g":94},"890":{"m":26,"g":94},"886":{"m":26,"g":94},"889":{"m":26,"g":94},"888":{"m":26,"g":94},"883":{"m":26,"g":94},"882":{"m":26,"g":94},"880":{"m":26,"g":94},"879":{"m":26,"g":94},"749":{"m":26,"g":94},"878":{"m":26,"g":94},"876":{"m":26,"g":94},"875":{"m":26,"g":94},"899":{"m":27,"g":94},"896":{"m":27,"g":94},"895":{"m":27,"g":94},"894":{"m":27,"g":94},"884":{"m":27,"g":94},"891":{"m":27,"g":94},"923":{"m":28,"g":94},"916":{"m":28,"g":94},"920":{"m":28,"g":94},"918":{"m":28,"g":94},"917":{"m":28,"g":94},"915":{"m":28,"g":94},"905":{"m":28,"g":94},"914":{"m":28,"g":94},"912":{"m":28,"g":94},"911":{"m":28,"g":94},"909":{"m":28,"g":94},"866":{"m":28,"g":94},"904":{"m":28,"g":94},"908":{"m":28,"g":94},"900":{"m":28,"g":94},"970":{"m":29,"g":94},"966":{"m":29,"g":94},"967":{"m":29,"g":94},"960":{"m":29,"g":94},"965":{"m":29,"g":94},"964":{"m":29,"g":94},"963":{"m":29,"g":94},"932":{"m":29,"g":94},"936":{"m":29,"g":94},"957":{"m":29,"g":94},"953":{"m":29,"g":94},"948":{"m":29,"g":94},"941":{"m":29,"g":94},"940":{"m":29,"g":94},"835":{"m":29,"g":94},"934":{"m":29,"g":94},"935":{"m":29,"g":94},"928":{"m":29,"g":94},"927":{"m":29,"g":94},"926":{"m":29,"g":94},"925":{"m":29,"g":94},"921":{"m":29,"g":94},"1048":{"m":30,"g":94},"1052":{"m":30,"g":94},"1051":{"m":30,"g":94},"1049":{"m":30,"g":94},"1050":{"m":30,"g":94},"1033":{"m":30,"g":94},"1046":{"m":30,"g":94},"1047":{"m":30,"g":94},"1044":{"m":30,"g":94},"1045":{"m":30,"g":94},"1039":{"m":30,"g":94},"1037":{"m":30,"g":94},"1038":{"m":30,"g":94},"1025":{"m":30,"g":94},"1034":{"m":30,"g":94},"1031":{"m":30,"g":94},"1027":{"m":30,"g":94},"1029":{"m":30,"g":94},"1028":{"m":30,"g":94},"1022":{"m":30,"g":94},"1024":{"m":30,"g":94},"907":{"m":30,"g":94},"1021":{"m":30,"g":94},"1020":{"m":30,"g":94},"1019":{"m":30,"g":94},"990":{"m":30,"g":94},"1014":{"m":30,"g":94},"1010":{"m":30,"g":94},"1009":{"m":30,"g":94},"1007":{"m":30,"g":94},"959":{"m":30,"g":94},"997":{"m":30,"g":94},"1005":{"m":30,"g":94},"1002":{"m":30,"g":94},"1001":{"m":30,"g":94},"994":{"m":30,"g":94},"995":{"m":30,"g":94},"988":{"m":30,"g":94},"993":{"m":30,"g":94},"973":{"m":30,"g":94},"992":{"m":30,"g":94},"985":{"m":30,"g":94},"981":{"m":30,"g":94},"987":{"m":30,"g":94},"983":{"m":30,"g":94},"984":{"m":30,"g":94},"982":{"m":30,"g":94},"971":{"m":30,"g":94},"980":{"m":30,"g":94},"977":{"m":30,"g":94},"968":{"m":30,"g":94},"969":{"m":30,"g":94},"976":{"m":30,"g":94},"975":{"m":30,"g":94},"1111":{"m":31,"g":94},"1113":{"m":31,"g":94},"1112":{"m":31,"g":94},"1110":{"m":31,"g":94},"1107":{"m":31,"g":94},"1040":{"m":31,"g":94},"1106":{"m":31,"g":94},"1077":{"m":31,"g":94},"1104":{"m":31,"g":94},"1092":{"m":31,"g":94},"1103":{"m":31,"g":94},"1090":{"m":31,"g":94},"1082":{"m":31,"g":94},"1099":{"m":31,"g":94},"1095":{"m":31,"g":94},"1098":{"m":31,"g":94},"1096":{"m":31,"g":94},"1094":{"m":31,"g":94},"1088":{"m":31,"g":94},"1086":{"m":31,"g":94},"1084":{"m":31,"g":94},"1056":{"m":31,"g":94},"1081":{"m":31,"g":94},"1006":{"m":31,"g":94},"1079":{"m":31,"g":94},"1078":{"m":31,"g":94},"1074":{"m":31,"g":94},"1053":{"m":31,"g":94},"1070":{"m":31,"g":94},"1060":{"m":31,"g":94},"1066":{"m":31,"g":94},"1068":{"m":31,"g":94},"1057":{"m":31,"g":94},"1155":{"m":32,"g":94},"1201":{"m":32,"g":94},"1219":{"m":32,"g":94},"1212":{"m":32,"g":94},"1218":{"m":32,"g":94},"1217":{"m":32,"g":94},"1215":{"m":32,"g":94},"1214":{"m":32,"g":94},"1204":{"m":32,"g":94},"1213":{"m":32,"g":94},"1210":{"m":32,"g":94},"1211":{"m":32,"g":94},"1209":{"m":32,"g":94},"1208":{"m":32,"g":94},"1186":{"m":32,"g":94},"1205":{"m":32,"g":94},"1207":{"m":32,"g":94},"1199":{"m":32,"g":94},"1202":{"m":32,"g":94},"1198":{"m":32,"g":94},"1194":{"m":32,"g":94},"1193":{"m":32,"g":94},"1123":{"m":32,"g":94},"1185":{"m":32,"g":94},"1184":{"m":32,"g":94},"1180":{"m":32,"g":94},"1168":{"m":32,"g":94},"1179":{"m":32,"g":94},"1167":{"m":32,"g":94},"1170":{"m":32,"g":94},"1177":{"m":32,"g":94},"1171":{"m":32,"g":94},"1157":{"m":32,"g":94},"1166":{"m":32,"g":94},"1154":{"m":32,"g":94},"1165":{"m":32,"g":94},"1148":{"m":32,"g":94},"1164":{"m":32,"g":94},"1134":{"m":32,"g":94},"1138":{"m":32,"g":94},"1035":{"m":32,"g":94},"1144":{"m":32,"g":94},"1143":{"m":32,"g":94},"1141":{"m":32,"g":94},"1140":{"m":32,"g":94},"1139":{"m":32,"g":94},"1136":{"m":32,"g":94},"1133":{"m":32,"g":94},"1131":{"m":32,"g":94},"1013":{"m":32,"g":94},"1122":{"m":32,"g":94},"1119":{"m":32,"g":94},"1115":{"m":32,"g":94},"1114":{"m":32,"g":94},"1242":{"m":33,"g":94},"1239":{"m":33,"g":94},"1233":{"m":33,"g":94},"1237":{"m":33,"g":94},"1225":{"m":33,"g":94},"1236":{"m":33,"g":94},"1231":{"m":33,"g":94},"1230":{"m":33,"g":94},"1227":{"m":33,"g":94},"1222":{"m":33,"g":94},"1223":{"m":33,"g":94},"1125":{"m":33,"g":94},"1250":{"m":34,"g":94},"1252":{"m":34,"g":94},"1249":{"m":34,"g":94},"1234":{"m":34,"g":94},"1247":{"m":34,"g":94},"1232":{"m":34,"g":94},"1244":{"m":34,"g":94},"1243":{"m":34,"g":94},"1295":{"m":35,"g":94},"1297":{"m":35,"g":94},"1296":{"m":35,"g":94},"1294":{"m":35,"g":94},"1293":{"m":35,"g":94},"1291":{"m":35,"g":94},"1290":{"m":35,"g":94},"1277":{"m":35,"g":94},"1284":{"m":35,"g":94},"1288":{"m":35,"g":94},"1286":{"m":35,"g":94},"1289":{"m":35,"g":94},"1285":{"m":35,"g":94},"1280":{"m":35,"g":94},"1262":{"m":35,"g":94},"1282":{"m":35,"g":94},"1276":{"m":35,"g":94},"1256":{"m":35,"g":94},"1269":{"m":35,"g":94},"1267":{"m":35,"g":94},"1258":{"m":35,"g":94},"1261":{"m":35,"g":94},"1260":{"m":35,"g":94},"1253":{"m":35,"g":94},"1255":{"m":35,"g":94},"1254":{"m":35,"g":94},"1327":{"m":36,"g":94},"1326":{"m":36,"g":94},"1320":{"m":36,"g":94},"1319":{"m":36,"g":94},"1318":{"m":36,"g":94},"1317":{"m":36,"g":94},"1313":{"m":36,"g":94},"1299":{"m":36,"g":94},"1308":{"m":36,"g":94},"1306":{"m":36,"g":94},"1304":{"m":36,"g":94},"1445":{"m":37,"g":94},"1444":{"m":37,"g":94},"1442":{"m":37,"g":94},"1420":{"m":37,"g":94},"1441":{"m":37,"g":94},"1440":{"m":37,"g":94},"1438":{"m":37,"g":94},"1432":{"m":37,"g":94},"1433":{"m":37,"g":94},"1431":{"m":37,"g":94},"1430":{"m":37,"g":94},"1428":{"m":37,"g":94},"1429":{"m":37,"g":94},"1427":{"m":37,"g":94},"1422":{"m":37,"g":94},"1426":{"m":37,"g":94},"1425":{"m":37,"g":94},"1418":{"m":37,"g":94},"1392":{"m":37,"g":94},"1414":{"m":37,"g":94},"1412":{"m":37,"g":94},"1411":{"m":37,"g":94},"1409":{"m":37,"g":94},"1408":{"m":37,"g":94},"1407":{"m":37,"g":94},"1406":{"m":37,"g":94},"1307":{"m":37,"g":94},"1397":{"m":37,"g":94},"1402":{"m":37,"g":94},"1403":{"m":37,"g":94},"1401":{"m":37,"g":94},"1399":{"m":37,"g":94},"1393":{"m":37,"g":94},"1381":{"m":37,"g":94},"1390":{"m":37,"g":94},"1389":{"m":37,"g":94},"1367":{"m":37,"g":94},"1385":{"m":37,"g":94},"1378":{"m":37,"g":94},"1380":{"m":37,"g":94},"1379":{"m":37,"g":94},"1376":{"m":37,"g":94},"1375":{"m":37,"g":94},"1373":{"m":37,"g":94},"1371":{"m":37,"g":94},"1370":{"m":37,"g":94},"1368":{"m":37,"g":94},"1300":{"m":37,"g":94},"1363":{"m":37,"g":94},"1360":{"m":37,"g":94},"1361":{"m":37,"g":94},"1341":{"m":37,"g":94},"1357":{"m":37,"g":94},"1298":{"m":37,"g":94},"1346":{"m":37,"g":94},"1281":{"m":37,"g":94},"1345":{"m":37,"g":94},"1339":{"m":37,"g":94},"1340":{"m":37,"g":94},"1337":{"m":37,"g":94},"1336":{"m":37,"g":94},"1328":{"m":37,"g":94},"1470":{"m":38,"g":94},"1469":{"m":38,"g":94},"1464":{"m":38,"g":94},"1458":{"m":38,"g":94},"1457":{"m":38,"g":94},"1454":{"m":38,"g":94},"1453":{"m":38,"g":94},"1452":{"m":38,"g":94},"1449":{"m":38,"g":94},"1451":{"m":38,"g":94},"1450":{"m":38,"g":94},"1448":{"m":38,"g":94},"1447":{"m":38,"g":94},"1483":{"m":39,"g":94},"1484":{"m":39,"g":94},"1482":{"m":39,"g":94},"1476":{"m":39,"g":94},"1475":{"m":39,"g":94},"1305":{"m":39,"g":94},"1472":{"m":39,"g":94},"1512":{"m":40,"g":94},"1511":{"m":40,"g":94},"1508":{"m":40,"g":94},"1510":{"m":40,"g":94},"1503":{"m":40,"g":94},"1499":{"m":40,"g":94},"1502":{"m":40,"g":94},"1500":{"m":40,"g":94},"1497":{"m":40,"g":94},"1496":{"m":40,"g":94},"1494":{"m":40,"g":94},"1490":{"m":40,"g":94},"1492":{"m":40,"g":94},"1491":{"m":40,"g":94},"1489":{"m":40,"g":94},"1456":{"m":40,"g":94},"1488":{"m":40,"g":94},"1486":{"m":40,"g":94},"1481":{"m":40,"g":94},"1605":{"m":41,"g":94},"1606":{"m":41,"g":94},"1604":{"m":41,"g":94},"1598":{"m":41,"g":94},"1603":{"m":41,"g":94},"1597":{"m":41,"g":94},"1594":{"m":41,"g":94},"1596":{"m":41,"g":94},"1595":{"m":41,"g":94},"1593":{"m":41,"g":94},"1567":{"m":41,"g":94},"1592":{"m":41,"g":94},"1591":{"m":41,"g":94},"1590":{"m":41,"g":94},"1589":{"m":41,"g":94},"1587":{"m":41,"g":94},"1586":{"m":41,"g":94},"1585":{"m":41,"g":94},"1584":{"m":41,"g":94},"1573":{"m":41,"g":94},"1582":{"m":41,"g":94},"1583":{"m":41,"g":94},"1581":{"m":41,"g":94},"1576":{"m":41,"g":94},"1561":{"m":41,"g":94},"1572":{"m":41,"g":94},"1577":{"m":41,"g":94},"1580":{"m":41,"g":94},"1574":{"m":41,"g":94},"1563":{"m":41,"g":94},"1569":{"m":41,"g":94},"1568":{"m":41,"g":94},"1566":{"m":41,"g":94},"1562":{"m":41,"g":94},"1559":{"m":41,"g":94},"1536":{"m":41,"g":94},"1557":{"m":41,"g":94},"1556":{"m":41,"g":94},"1555":{"m":41,"g":94},"1553":{"m":41,"g":94},"1554":{"m":41,"g":94},"1549":{"m":41,"g":94},"1552":{"m":41,"g":94},"1550":{"m":41,"g":94},"1548":{"m":41,"g":94},"1547":{"m":41,"g":94},"1545":{"m":41,"g":94},"1544":{"m":41,"g":94},"1543":{"m":41,"g":94},"1541":{"m":41,"g":94},"1539":{"m":41,"g":94},"1538":{"m":41,"g":94},"1537":{"m":41,"g":94},"1534":{"m":41,"g":94},"1531":{"m":41,"g":94},"1532":{"m":41,"g":94},"1530":{"m":41,"g":94},"1495":{"m":41,"g":94},"1520":{"m":41,"g":94},"1521":{"m":41,"g":94},"1528":{"m":41,"g":94},"1525":{"m":41,"g":94},"1529":{"m":41,"g":94},"1524":{"m":41,"g":94},"1513":{"m":41,"g":94},"1636":{"m":42,"g":94},"1635":{"m":42,"g":94},"1634":{"m":42,"g":94},"1633":{"m":42,"g":94},"1632":{"m":42,"g":94},"1631":{"m":42,"g":94},"1626":{"m":42,"g":94},"1579":{"m":42,"g":94},"1611":{"m":42,"g":94},"1607":{"m":42,"g":94},"1629":{"m":42,"g":94},"1625":{"m":42,"g":94},"1619":{"m":42,"g":94},"1620":{"m":42,"g":94},"1615":{"m":42,"g":94},"1714":{"m":43,"g":94},"1713":{"m":43,"g":94},"1712":{"m":43,"g":94},"1710":{"m":43,"g":94},"1709":{"m":43,"g":94},"1707":{"m":43,"g":94},"1706":{"m":43,"g":94},"1705":{"m":43,"g":94},"1704":{"m":43,"g":94},"1703":{"m":43,"g":94},"1684":{"m":43,"g":94},"1702":{"m":43,"g":94},"1701":{"m":43,"g":94},"1700":{"m":43,"g":94},"1699":{"m":43,"g":94},"1694":{"m":43,"g":94},"1697":{"m":43,"g":94},"1696":{"m":43,"g":94},"1690":{"m":43,"g":94},"1679":{"m":43,"g":94},"1689":{"m":43,"g":94},"1688":{"m":43,"g":94},"1599":{"m":43,"g":94},"1687":{"m":43,"g":94},"1686":{"m":43,"g":94},"1685":{"m":43,"g":94},"1677":{"m":43,"g":94},"1676":{"m":43,"g":94},"1681":{"m":43,"g":94},"1674":{"m":43,"g":94},"1672":{"m":43,"g":94},"1671":{"m":43,"g":94},"1670":{"m":43,"g":94},"1658":{"m":43,"g":94},"1667":{"m":43,"g":94},"1666":{"m":43,"g":94},"1665":{"m":43,"g":94},"1459":{"m":43,"g":94},"1663":{"m":43,"g":94},"1662":{"m":43,"g":94},"1661":{"m":43,"g":94},"1659":{"m":43,"g":94},"1650":{"m":43,"g":94},"1656":{"m":43,"g":94},"1652":{"m":43,"g":94},"1654":{"m":43,"g":94},"1653":{"m":43,"g":94},"1651":{"m":43,"g":94},"1648":{"m":43,"g":94},"1480":{"m":43,"g":94},"1645":{"m":43,"g":94},"1642":{"m":43,"g":94},"1638":{"m":43,"g":94},"1614":{"m":43,"g":94},"1749":{"m":44,"g":94},"1748":{"m":44,"g":94},"1551":{"m":44,"g":94},"1746":{"m":44,"g":94},"1737":{"m":44,"g":94},"1738":{"m":44,"g":94},"1743":{"m":44,"g":94},"1741":{"m":44,"g":94},"1740":{"m":44,"g":94},"1736":{"m":44,"g":94},"1735":{"m":44,"g":94},"1734":{"m":44,"g":94},"1727":{"m":44,"g":94},"1726":{"m":44,"g":94},"1725":{"m":44,"g":94},"1724":{"m":44,"g":94},"1722":{"m":44,"g":94},"1721":{"m":44,"g":94},"1720":{"m":44,"g":94},"1718":{"m":44,"g":94},"1716":{"m":44,"g":94},"1796":{"m":45,"g":94},"1795":{"m":45,"g":94},"1797":{"m":45,"g":94},"1794":{"m":45,"g":94},"1787":{"m":45,"g":94},"1793":{"m":45,"g":94},"1780":{"m":45,"g":94},"1789":{"m":45,"g":94},"1785":{"m":45,"g":94},"1783":{"m":45,"g":94},"1778":{"m":45,"g":94},"1782":{"m":45,"g":94},"1779":{"m":45,"g":94},"1776":{"m":45,"g":94},"1774":{"m":45,"g":94},"1773":{"m":45,"g":94},"1772":{"m":45,"g":94},"1771":{"m":45,"g":94},"1769":{"m":45,"g":94},"1768":{"m":45,"g":94},"1766":{"m":45,"g":94},"1767":{"m":45,"g":94},"1765":{"m":45,"g":94},"1760":{"m":45,"g":94},"1758":{"m":45,"g":94},"1747":{"m":45,"g":94},"1908":{"m":46,"g":94},"1907":{"m":46,"g":94},"1906":{"m":46,"g":94},"1902":{"m":46,"g":94},"1905":{"m":46,"g":94},"1904":{"m":46,"g":94},"1903":{"m":46,"g":94},"1899":{"m":46,"g":94},"1896":{"m":46,"g":94},"1895":{"m":46,"g":94},"1894":{"m":46,"g":94},"1892":{"m":46,"g":94},"1890":{"m":46,"g":94},"1888":{"m":46,"g":94},"1889":{"m":46,"g":94},"1886":{"m":46,"g":94},"1885":{"m":46,"g":94},"1883":{"m":46,"g":94},"1873":{"m":46,"g":94},"1882":{"m":46,"g":94},"1881":{"m":46,"g":94},"1879":{"m":46,"g":94},"1878":{"m":46,"g":94},"1877":{"m":46,"g":94},"1875":{"m":46,"g":94},"1871":{"m":46,"g":94},"1867":{"m":46,"g":94},"1866":{"m":46,"g":94},"1754":{"m":46,"g":94},"1856":{"m":46,"g":94},"1859":{"m":46,"g":94},"1860":{"m":46,"g":94},"1861":{"m":46,"g":94},"1858":{"m":46,"g":94},"1855":{"m":46,"g":94},"1852":{"m":46,"g":94},"1851":{"m":46,"g":94},"1846":{"m":46,"g":94},"1850":{"m":46,"g":94},"1847":{"m":46,"g":94},"1845":{"m":46,"g":94},"1842":{"m":46,"g":94},"1836":{"m":46,"g":94},"1838":{"m":46,"g":94},"1840":{"m":46,"g":94},"1839":{"m":46,"g":94},"1827":{"m":46,"g":94},"1833":{"m":46,"g":94},"1835":{"m":46,"g":94},"1834":{"m":46,"g":94},"1822":{"m":46,"g":94},"1823":{"m":46,"g":94},"1830":{"m":46,"g":94},"1825":{"m":46,"g":94},"1790":{"m":46,"g":94},"1821":{"m":46,"g":94},"1820":{"m":46,"g":94},"1819":{"m":46,"g":94},"1810":{"m":46,"g":94},"1817":{"m":46,"g":94},"1816":{"m":46,"g":94},"1813":{"m":46,"g":94},"1811":{"m":46,"g":94},"1809":{"m":46,"g":94},"1808":{"m":46,"g":94},"1807":{"m":46,"g":94},"1805":{"m":46,"g":94},"1804":{"m":46,"g":94},"1803":{"m":46,"g":94},"1802":{"m":46,"g":94},"1801":{"m":46,"g":94},"1800":{"m":46,"g":94},"1786":{"m":46,"g":94},"1799":{"m":46,"g":94},"1798":{"m":46,"g":94},"1752":{"m":46,"g":94},"2022":{"m":47,"g":94},"2020":{"m":47,"g":94},"2018":{"m":47,"g":94},"1996":{"m":47,"g":94},"2015":{"m":47,"g":94},"2014":{"m":47,"g":94},"2013":{"m":47,"g":94},"1998":{"m":47,"g":94},"2011":{"m":47,"g":94},"2010":{"m":47,"g":94},"2009":{"m":47,"g":94},"2008":{"m":47,"g":94},"2006":{"m":47,"g":94},"2005":{"m":47,"g":94},"1994":{"m":47,"g":94},"2003":{"m":47,"g":94},"2004":{"m":47,"g":94},"2002":{"m":47,"g":94},"2001":{"m":47,"g":94},"2000":{"m":47,"g":94},"1999":{"m":47,"g":94},"1995":{"m":47,"g":94},"1997":{"m":47,"g":94},"1934":{"m":47,"g":94},"1980":{"m":47,"g":94},"1990":{"m":47,"g":94},"1988":{"m":47,"g":94},"1984":{"m":47,"g":94},"1986":{"m":47,"g":94},"1983":{"m":47,"g":94},"1981":{"m":47,"g":94},"1982":{"m":47,"g":94},"1977":{"m":47,"g":94},"1972":{"m":47,"g":94},"1976":{"m":47,"g":94},"1975":{"m":47,"g":94},"1974":{"m":47,"g":94},"1745":{"m":47,"g":94},"1973":{"m":47,"g":94},"1963":{"m":47,"g":94},"1966":{"m":47,"g":94},"1962":{"m":47,"g":94},"1961":{"m":47,"g":94},"1958":{"m":47,"g":94},"1957":{"m":47,"g":94},"1956":{"m":47,"g":94},"1955":{"m":47,"g":94},"1954":{"m":47,"g":94},"1933":{"m":47,"g":94},"1952":{"m":47,"g":94},"1951":{"m":47,"g":94},"1939":{"m":47,"g":94},"1941":{"m":47,"g":94},"1949":{"m":47,"g":94},"1940":{"m":47,"g":94},"1942":{"m":47,"g":94},"1891":{"m":47,"g":94},"1926":{"m":47,"g":94},"1922":{"m":47,"g":94},"1853":{"m":47,"g":94},"1924":{"m":47,"g":94},"1920":{"m":47,"g":94},"1916":{"m":47,"g":94},"1893":{"m":47,"g":94},"1915":{"m":47,"g":94},"1910":{"m":47,"g":94},"1909":{"m":47,"g":94},"2046":{"m":48,"g":94},"2044":{"m":48,"g":94},"2043":{"m":48,"g":94},"2030":{"m":48,"g":94},"2042":{"m":48,"g":94},"2038":{"m":48,"g":94},"1968":{"m":48,"g":94},"2039":{"m":48,"g":94},"2036":{"m":48,"g":94},"2034":{"m":48,"g":94},"2033":{"m":48,"g":94},"2031":{"m":48,"g":94},"2027":{"m":48,"g":94},"2028":{"m":48,"g":94},"2026":{"m":48,"g":94},"2024":{"m":48,"g":94},"2023":{"m":48,"g":94},"2120":{"m":49,"g":94},"2125":{"m":49,"g":94},"2122":{"m":49,"g":94},"2110":{"m":49,"g":94},"2118":{"m":49,"g":94},"2106":{"m":49,"g":94},"2115":{"m":49,"g":94},"2055":{"m":49,"g":94},"2111":{"m":49,"g":94},"2116":{"m":49,"g":94},"2107":{"m":49,"g":94},"2105":{"m":49,"g":94},"2104":{"m":49,"g":94},"2103":{"m":49,"g":94},"2073":{"m":49,"g":94},"2100":{"m":49,"g":94},"2067":{"m":49,"g":94},"2096":{"m":49,"g":94},"2095":{"m":49,"g":94},"2093":{"m":49,"g":94},"2094":{"m":49,"g":94},"2091":{"m":49,"g":94},"2088":{"m":49,"g":94},"2089":{"m":49,"g":94},"2086":{"m":49,"g":94},"2085":{"m":49,"g":94},"2083":{"m":49,"g":94},"2078":{"m":49,"g":94},"2069":{"m":49,"g":94},"2075":{"m":49,"g":94},"2074":{"m":49,"g":94},"2072":{"m":49,"g":94},"2071":{"m":49,"g":94},"2070":{"m":49,"g":94},"2068":{"m":49,"g":94},"2062":{"m":49,"g":94},"2056":{"m":49,"g":94},"2066":{"m":49,"g":94},"2061":{"m":49,"g":94},"2065":{"m":49,"g":94},"2064":{"m":49,"g":94},"2063":{"m":49,"g":94},"1849":{"m":49,"g":94},"2053":{"m":49,"g":94},"2051":{"m":49,"g":94},"1970":{"m":49,"g":94},"2050":{"m":49,"g":94},"2049":{"m":49,"g":94},"1876":{"m":49,"g":94},"2048":{"m":49,"g":94},"2047":{"m":49,"g":94},"2189":{"m":50,"g":94},"2188":{"m":50,"g":94},"2187":{"m":50,"g":94},"2052":{"m":50,"g":94},"2184":{"m":50,"g":94},"2176":{"m":50,"g":94},"2186":{"m":50,"g":94},"2171":{"m":50,"g":94},"2185":{"m":50,"g":94},"2183":{"m":50,"g":94},"2173":{"m":50,"g":94},"2182":{"m":50,"g":94},"2180":{"m":50,"g":94},"2175":{"m":50,"g":94},"2174":{"m":50,"g":94},"2170":{"m":50,"g":94},"2169":{"m":50,"g":94},"2167":{"m":50,"g":94},"2164":{"m":50,"g":94},"2163":{"m":50,"g":94},"2162":{"m":50,"g":94},"2158":{"m":50,"g":94},"2161":{"m":50,"g":94},"2159":{"m":50,"g":94},"2156":{"m":50,"g":94},"2154":{"m":50,"g":94},"2157":{"m":50,"g":94},"2155":{"m":50,"g":94},"2153":{"m":50,"g":94},"2152":{"m":50,"g":94},"2148":{"m":50,"g":94},"2147":{"m":50,"g":94},"2146":{"m":50,"g":94},"2144":{"m":50,"g":94},"2143":{"m":50,"g":94},"2142":{"m":50,"g":94},"2114":{"m":50,"g":94},"2139":{"m":50,"g":94},"2138":{"m":50,"g":94},"2136":{"m":50,"g":94},"2137":{"m":50,"g":94},"2134":{"m":50,"g":94},"2081":{"m":50,"g":94},"2121":{"m":50,"g":94},"2130":{"m":50,"g":94},"2124":{"m":50,"g":94},"2092":{"m":50,"g":94},"2127":{"m":50,"g":94},"2077":{"m":50,"g":94},"2126":{"m":50,"g":94},"2214":{"m":51,"g":94},"2222":{"m":51,"g":94},"2221":{"m":51,"g":94},"2217":{"m":51,"g":94},"2210":{"m":51,"g":94},"2212":{"m":51,"g":94},"2208":{"m":51,"g":94},"2207":{"m":51,"g":94},"2204":{"m":51,"g":94},"2206":{"m":51,"g":94},"2201":{"m":51,"g":94},"2199":{"m":51,"g":94},"2196":{"m":51,"g":94},"2198":{"m":51,"g":94},"2195":{"m":51,"g":94},"2197":{"m":51,"g":94},"2259":{"m":52,"g":94},"2257":{"m":52,"g":94},"2256":{"m":52,"g":94},"2254":{"m":52,"g":94},"2253":{"m":52,"g":94},"2252":{"m":52,"g":94},"2251":{"m":52,"g":94},"2250":{"m":52,"g":94},"2239":{"m":52,"g":94},"2242":{"m":52,"g":94},"2238":{"m":52,"g":94},"2231":{"m":52,"g":94},"2233":{"m":52,"g":94},"2235":{"m":52,"g":94},"2234":{"m":52,"g":94},"2232":{"m":52,"g":94},"2228":{"m":52,"g":94},"2123":{"m":52,"g":94},"2223":{"m":52,"g":94},"2224":{"m":52,"g":94},"2226":{"m":52,"g":94},"2191":{"m":52,"g":94},"2218":{"m":52,"g":94},"2338":{"m":53,"g":94},"2339":{"m":53,"g":94},"2300":{"m":53,"g":94},"2335":{"m":53,"g":94},"2327":{"m":53,"g":94},"2324":{"m":53,"g":94},"2328":{"m":53,"g":94},"2329":{"m":53,"g":94},"2325":{"m":53,"g":94},"2281":{"m":53,"g":94},"2319":{"m":53,"g":94},"2318":{"m":53,"g":94},"2314":{"m":53,"g":94},"2311":{"m":53,"g":94},"2310":{"m":53,"g":94},"2309":{"m":53,"g":94},"2279":{"m":53,"g":94},"2306":{"m":53,"g":94},"2305":{"m":53,"g":94},"2304":{"m":53,"g":94},"2301":{"m":53,"g":94},"2302":{"m":53,"g":94},"2299":{"m":53,"g":94},"2298":{"m":53,"g":94},"2241":{"m":53,"g":94},"2295":{"m":53,"g":94},"2292":{"m":53,"g":94},"2179":{"m":53,"g":94},"2293":{"m":53,"g":94},"2290":{"m":53,"g":94},"2244":{"m":53,"g":94},"2288":{"m":53,"g":94},"2289":{"m":53,"g":94},"2287":{"m":53,"g":94},"2284":{"m":53,"g":94},"2286":{"m":53,"g":94},"2285":{"m":53,"g":94},"2282":{"m":53,"g":94},"2215":{"m":53,"g":94},"2280":{"m":53,"g":94},"2274":{"m":53,"g":94},"2266":{"m":53,"g":94},"2265":{"m":53,"g":94},"2269":{"m":53,"g":94},"2243":{"m":53,"g":94},"2268":{"m":53,"g":94},"2225":{"m":53,"g":94},"2261":{"m":53,"g":94},"2375":{"m":54,"g":94},"2377":{"m":54,"g":94},"2363":{"m":54,"g":94},"2374":{"m":54,"g":94},"2373":{"m":54,"g":94},"2369":{"m":54,"g":94},"2357":{"m":54,"g":94},"2360":{"m":54,"g":94},"2370":{"m":54,"g":94},"2371":{"m":54,"g":94},"2368":{"m":54,"g":94},"2359":{"m":54,"g":94},"2364":{"m":54,"g":94},"2355":{"m":54,"g":94},"2342":{"m":54,"g":94},"2352":{"m":54,"g":94},"2308":{"m":54,"g":94},"2323":{"m":54,"g":94},"2350":{"m":54,"g":94},"2340":{"m":54,"g":94},"2349":{"m":54,"g":94},"2341":{"m":54,"g":94},"2348":{"m":54,"g":94},"2525":{"m":55,"g":94},"2528":{"m":55,"g":94},"2524":{"m":55,"g":94},"2517":{"m":55,"g":94},"2516":{"m":55,"g":94},"2515":{"m":55,"g":94},"2502":{"m":55,"g":94},"2500":{"m":55,"g":94},"2499":{"m":55,"g":94},"2438":{"m":55,"g":94},"2426":{"m":55,"g":94},"2457":{"m":55,"g":94},"2467":{"m":55,"g":94},"2495":{"m":55,"g":94},"2494":{"m":55,"g":94},"2493":{"m":55,"g":94},"2492":{"m":55,"g":94},"2491":{"m":55,"g":94},"2476":{"m":55,"g":94},"2490":{"m":55,"g":94},"2489":{"m":55,"g":94},"2486":{"m":55,"g":94},"2487":{"m":55,"g":94},"2481":{"m":55,"g":94},"2485":{"m":55,"g":94},"2484":{"m":55,"g":94},"2483":{"m":55,"g":94},"2479":{"m":55,"g":94},"2473":{"m":55,"g":94},"2469":{"m":55,"g":94},"2464":{"m":55,"g":94},"2466":{"m":55,"g":94},"2463":{"m":55,"g":94},"2462":{"m":55,"g":94},"2459":{"m":55,"g":94},"2456":{"m":55,"g":94},"2444":{"m":55,"g":94},"2455":{"m":55,"g":94},"2454":{"m":55,"g":94},"2442":{"m":55,"g":94},"2453":{"m":55,"g":94},"2452":{"m":55,"g":94},"2437":{"m":55,"g":94},"2449":{"m":55,"g":94},"2448":{"m":55,"g":94},"2425":{"m":55,"g":94},"2447":{"m":55,"g":94},"2436":{"m":55,"g":94},"2441":{"m":55,"g":94},"2440":{"m":55,"g":94},"2435":{"m":55,"g":94},"2434":{"m":55,"g":94},"2433":{"m":55,"g":94},"2424":{"m":55,"g":94},"2412":{"m":55,"g":94},"2422":{"m":55,"g":94},"2419":{"m":55,"g":94},"2417":{"m":55,"g":94},"2416":{"m":55,"g":94},"2410":{"m":55,"g":94},"2393":{"m":55,"g":94},"2413":{"m":55,"g":94},"2411":{"m":55,"g":94},"2398":{"m":55,"g":94},"2409":{"m":55,"g":94},"2408":{"m":55,"g":94},"2407":{"m":55,"g":94},"2406":{"m":55,"g":94},"2405":{"m":55,"g":94},"2404":{"m":55,"g":94},"2403":{"m":55,"g":94},"2401":{"m":55,"g":94},"2397":{"m":55,"g":94},"2394":{"m":55,"g":94},"2382":{"m":55,"g":94},"2330":{"m":55,"g":94},"2392":{"m":55,"g":94},"2391":{"m":55,"g":94},"2388":{"m":55,"g":94},"2390":{"m":55,"g":94},"2387":{"m":55,"g":94},"2380":{"m":55,"g":94},"2379":{"m":55,"g":94},"2378":{"m":55,"g":94},"2582":{"m":56,"g":94},"2581":{"m":56,"g":94},"2580":{"m":56,"g":94},"2579":{"m":56,"g":94},"2575":{"m":56,"g":94},"2566":{"m":56,"g":94},"2563":{"m":56,"g":94},"2553":{"m":56,"g":94},"2545":{"m":56,"g":94},"2547":{"m":56,"g":94},"2543":{"m":56,"g":94},"2509":{"m":56,"g":94},"2523":{"m":56,"g":94},"2529":{"m":56,"g":94},"2541":{"m":56,"g":94},"2616":{"m":57,"g":94},"2617":{"m":57,"g":94},"2615":{"m":57,"g":94},"2610":{"m":57,"g":94},"2611":{"m":57,"g":94},"2612":{"m":57,"g":94},"2608":{"m":57,"g":94},"2606":{"m":57,"g":94},"2586":{"m":57,"g":94},"2605":{"m":57,"g":94},"2603":{"m":57,"g":94},"2564":{"m":57,"g":94},"2570":{"m":57,"g":94},"2521":{"m":57,"g":94},"2598":{"m":57,"g":94},"2574":{"m":57,"g":94},"2565":{"m":57,"g":94},"2597":{"m":57,"g":94},"2596":{"m":57,"g":94},"2526":{"m":57,"g":94},"2557":{"m":57,"g":94},"2594":{"m":57,"g":94},"2555":{"m":57,"g":94},"2560":{"m":57,"g":94},"2592":{"m":57,"g":94},"2590":{"m":57,"g":94},"2589":{"m":57,"g":94},"2643":{"m":58,"g":94},"2641":{"m":58,"g":94},"2637":{"m":58,"g":94},"2635":{"m":58,"g":94},"2640":{"m":58,"g":94},"2639":{"m":58,"g":94},"2638":{"m":58,"g":94},"2609":{"m":58,"g":94},"2544":{"m":58,"g":94},"2631":{"m":58,"g":94},"2628":{"m":58,"g":94},"2633":{"m":58,"g":94},"2614":{"m":58,"g":94},"2626":{"m":58,"g":94},"2624":{"m":58,"g":94},"2625":{"m":58,"g":94},"2623":{"m":58,"g":94},"2622":{"m":58,"g":94},"2475":{"m":58,"g":94},"2618":{"m":58,"g":94},"2647":{"m":59,"g":94},"2648":{"m":59,"g":94},"2646":{"m":59,"g":94},"2636":{"m":59,"g":94},"2645":{"m":59,"g":94},"2644":{"m":59,"g":94},"2713":{"m":60,"g":94},"2688":{"m":60,"g":94},"2735":{"m":60,"g":94},"2733":{"m":60,"g":94},"2731":{"m":60,"g":94},"2571":{"m":60,"g":94},"2726":{"m":60,"g":94},"2727":{"m":60,"g":94},"2722":{"m":60,"g":94},"2717":{"m":60,"g":94},"2601":{"m":60,"g":94},"2716":{"m":60,"g":94},"2711":{"m":60,"g":94},"2714":{"m":60,"g":94},"2704":{"m":60,"g":94},"2707":{"m":60,"g":94},"2712":{"m":60,"g":94},"2150":{"m":60,"g":94},"2709":{"m":60,"g":94},"2695":{"m":60,"g":94},"2705":{"m":60,"g":94},"2663":{"m":60,"g":94},"2697":{"m":60,"g":94},"2692":{"m":60,"g":94},"2691":{"m":60,"g":94},"2690":{"m":60,"g":94},"2689":{"m":60,"g":94},"2685":{"m":60,"g":94},"2684":{"m":60,"g":94},"2683":{"m":60,"g":94},"2682":{"m":60,"g":94},"2680":{"m":60,"g":94},"2678":{"m":60,"g":94},"2679":{"m":60,"g":94},"2676":{"m":60,"g":94},"2674":{"m":60,"g":94},"2672":{"m":60,"g":94},"2670":{"m":60,"g":94},"2667":{"m":60,"g":94},"2669":{"m":60,"g":94},"2664":{"m":60,"g":94},"2642":{"m":60,"g":94},"2666":{"m":60,"g":94},"2654":{"m":60,"g":94},"2655":{"m":60,"g":94},"2656":{"m":60,"g":94},"2652":{"m":60,"g":94},"2651":{"m":60,"g":94},"2650":{"m":60,"g":94},"2649":{"m":60,"g":94},"2840":{"m":61,"g":94},"2837":{"m":61,"g":94},"2836":{"m":61,"g":94},"2804":{"m":61,"g":94},"2730":{"m":61,"g":94},"2826":{"m":61,"g":94},"2835":{"m":61,"g":94},"2822":{"m":61,"g":94},"2819":{"m":61,"g":94},"2833":{"m":61,"g":94},"2830":{"m":61,"g":94},"2787":{"m":61,"g":94},"2816":{"m":61,"g":94},"2813":{"m":61,"g":94},"2809":{"m":61,"g":94},"2792":{"m":61,"g":94},"2789":{"m":61,"g":94},"2773":{"m":61,"g":94},"2784":{"m":61,"g":94},"2723":{"m":61,"g":94},"2780":{"m":61,"g":94},"2779":{"m":61,"g":94},"2771":{"m":61,"g":94},"2774":{"m":61,"g":94},"2761":{"m":61,"g":94},"2770":{"m":61,"g":94},"2767":{"m":61,"g":94},"2758":{"m":61,"g":94},"2757":{"m":61,"g":94},"2513":{"m":61,"g":94},"2535":{"m":61,"g":94},"2756":{"m":61,"g":94},"2745":{"m":61,"g":94},"2748":{"m":61,"g":94},"2752":{"m":61,"g":94},"2751":{"m":61,"g":94},"2750":{"m":61,"g":94},"2899":{"m":62,"g":94},"2887":{"m":62,"g":94},"2888":{"m":62,"g":94},"2881":{"m":62,"g":94},"2885":{"m":62,"g":94},"2879":{"m":62,"g":94},"2878":{"m":62,"g":94},"2875":{"m":62,"g":94},"2630":{"m":62,"g":94},"2870":{"m":62,"g":94},"2869":{"m":62,"g":94},"2863":{"m":62,"g":94},"2868":{"m":62,"g":94},"2867":{"m":62,"g":94},"2862":{"m":62,"g":94},"2866":{"m":62,"g":94},"2865":{"m":62,"g":94},"2828":{"m":62,"g":94},"2859":{"m":62,"g":94},"2858":{"m":62,"g":94},"2861":{"m":62,"g":94},"2857":{"m":62,"g":94},"2860":{"m":62,"g":94},"2856":{"m":62,"g":94},"2851":{"m":62,"g":94},"2853":{"m":62,"g":94},"2854":{"m":62,"g":94},"2852":{"m":62,"g":94},"2786":{"m":62,"g":94},"2848":{"m":62,"g":94},"2850":{"m":62,"g":94},"2846":{"m":62,"g":94},"2843":{"m":62,"g":94},"2841":{"m":62,"g":94},"3009":{"m":63,"g":94},"2993":{"m":63,"g":94},"3010":{"m":63,"g":94},"3006":{"m":63,"g":94},"3008":{"m":63,"g":94},"3003":{"m":63,"g":94},"2998":{"m":63,"g":94},"3005":{"m":63,"g":94},"3004":{"m":63,"g":94},"3001":{"m":63,"g":94},"2996":{"m":63,"g":94},"2997":{"m":63,"g":94},"2995":{"m":63,"g":94},"2991":{"m":63,"g":94},"2992":{"m":63,"g":94},"2990":{"m":63,"g":94},"2396":{"m":63,"g":94},"2983":{"m":63,"g":94},"2988":{"m":63,"g":94},"2839":{"m":63,"g":94},"2982":{"m":63,"g":94},"2986":{"m":63,"g":94},"2987":{"m":63,"g":94},"2985":{"m":63,"g":94},"2984":{"m":63,"g":94},"2981":{"m":63,"g":94},"2975":{"m":63,"g":94},"2980":{"m":63,"g":94},"2979":{"m":63,"g":94},"2978":{"m":63,"g":94},"2976":{"m":63,"g":94},"2974":{"m":63,"g":94},"2973":{"m":63,"g":94},"2972":{"m":63,"g":94},"2958":{"m":63,"g":94},"2956":{"m":63,"g":94},"2901":{"m":63,"g":94},"2971":{"m":63,"g":94},"2785":{"m":63,"g":94},"2966":{"m":63,"g":94},"2967":{"m":63,"g":94},"2964":{"m":63,"g":94},"2963":{"m":63,"g":94},"2960":{"m":63,"g":94},"2941":{"m":63,"g":94},"2894":{"m":63,"g":94},"2942":{"m":63,"g":94},"2944":{"m":63,"g":94},"2954":{"m":63,"g":94},"2952":{"m":63,"g":94},"2951":{"m":63,"g":94},"2947":{"m":63,"g":94},"2948":{"m":63,"g":94},"2949":{"m":63,"g":94},"2950":{"m":63,"g":94},"2945":{"m":63,"g":94},"2907":{"m":63,"g":94},"2938":{"m":63,"g":94},"2937":{"m":63,"g":94},"2806":{"m":63,"g":94},"2876":{"m":63,"g":94},"2930":{"m":63,"g":94},"2928":{"m":63,"g":94},"2920":{"m":63,"g":94},"2926":{"m":63,"g":94},"2927":{"m":63,"g":94},"2925":{"m":63,"g":94},"2924":{"m":63,"g":94},"2923":{"m":63,"g":94},"2821":{"m":63,"g":94},"2910":{"m":63,"g":94},"2922":{"m":63,"g":94},"2919":{"m":63,"g":94},"2917":{"m":63,"g":94},"2915":{"m":63,"g":94},"2911":{"m":63,"g":94},"2909":{"m":63,"g":94},"2908":{"m":63,"g":94},"2511":{"m":63,"g":94},"2906":{"m":63,"g":94},"2904":{"m":63,"g":94},"2902":{"m":63,"g":94},"2872":{"m":63,"g":94},"2897":{"m":63,"g":94},"3180":{"m":64,"g":94},"3179":{"m":64,"g":94},"3178":{"m":64,"g":94},"3175":{"m":64,"g":94},"3176":{"m":64,"g":94},"3174":{"m":64,"g":94},"3146":{"m":64,"g":94},"3173":{"m":64,"g":94},"3170":{"m":64,"g":94},"3167":{"m":64,"g":94},"3156":{"m":64,"g":94},"3134":{"m":64,"g":94},"3162":{"m":64,"g":94},"3144":{"m":64,"g":94},"3155":{"m":64,"g":94},"2700":{"m":64,"g":94},"3154":{"m":64,"g":94},"3153":{"m":64,"g":94},"3152":{"m":64,"g":94},"3150":{"m":64,"g":94},"3151":{"m":64,"g":94},"3149":{"m":64,"g":94},"3147":{"m":64,"g":94},"3145":{"m":64,"g":94},"3085":{"m":64,"g":94},"3047":{"m":64,"g":94},"3143":{"m":64,"g":94},"3139":{"m":64,"g":94},"3135":{"m":64,"g":94},"3138":{"m":64,"g":94},"3113":{"m":64,"g":94},"3133":{"m":64,"g":94},"3132":{"m":64,"g":94},"3130":{"m":64,"g":94},"3129":{"m":64,"g":94},"3128":{"m":64,"g":94},"3127":{"m":64,"g":94},"3126":{"m":64,"g":94},"3125":{"m":64,"g":94},"3124":{"m":64,"g":94},"3121":{"m":64,"g":94},"3110":{"m":64,"g":94},"3109":{"m":64,"g":94},"3107":{"m":64,"g":94},"3096":{"m":64,"g":94},"3037":{"m":64,"g":94},"3105":{"m":64,"g":94},"3097":{"m":64,"g":94},"3095":{"m":64,"g":94},"3094":{"m":64,"g":94},"3070":{"m":64,"g":94},"3093":{"m":64,"g":94},"2742":{"m":64,"g":94},"3087":{"m":64,"g":94},"3086":{"m":64,"g":94},"3084":{"m":64,"g":94},"3083":{"m":64,"g":94},"3081":{"m":64,"g":94},"3080":{"m":64,"g":94},"3079":{"m":64,"g":94},"3078":{"m":64,"g":94},"3074":{"m":64,"g":94},"3030":{"m":64,"g":94},"3071":{"m":64,"g":94},"3069":{"m":64,"g":94},"3068":{"m":64,"g":94},"3067":{"m":64,"g":94},"3061":{"m":64,"g":94},"3062":{"m":64,"g":94},"3063":{"m":64,"g":94},"2989":{"m":64,"g":94},"3060":{"m":64,"g":94},"3045":{"m":64,"g":94},"3038":{"m":64,"g":94},"3058":{"m":64,"g":94},"3057":{"m":64,"g":94},"3055":{"m":64,"g":94},"3056":{"m":64,"g":94},"3054":{"m":64,"g":94},"3053":{"m":64,"g":94},"3052":{"m":64,"g":94},"3051":{"m":64,"g":94},"3048":{"m":64,"g":94},"3046":{"m":64,"g":94},"3039":{"m":64,"g":94},"3036":{"m":64,"g":94},"3035":{"m":64,"g":94},"3033":{"m":64,"g":94},"3027":{"m":64,"g":94},"3026":{"m":64,"g":94},"3025":{"m":64,"g":94},"3014":{"m":64,"g":94},"3018":{"m":64,"g":94},"2939":{"m":64,"g":94},"3022":{"m":64,"g":94},"3021":{"m":64,"g":94},"3017":{"m":64,"g":94},"3015":{"m":64,"g":94},"3020":{"m":64,"g":94},"3019":{"m":64,"g":94},"3016":{"m":64,"g":94},"3013":{"m":64,"g":94},"3012":{"m":64,"g":94},"3233":{"m":65,"g":94},"3231":{"m":65,"g":94},"3232":{"m":65,"g":94},"3224":{"m":65,"g":94},"3230":{"m":65,"g":94},"3229":{"m":65,"g":94},"3227":{"m":65,"g":94},"3218":{"m":65,"g":94},"3217":{"m":65,"g":94},"3216":{"m":65,"g":94},"3214":{"m":65,"g":94},"3213":{"m":65,"g":94},"3212":{"m":65,"g":94},"2977":{"m":65,"g":94},"3190":{"m":65,"g":94},"3169":{"m":65,"g":94},"3192":{"m":65,"g":94},"3183":{"m":65,"g":94},"3166":{"m":65,"g":94},"3171":{"m":65,"g":94},"3181":{"m":65,"g":94},"3313":{"m":66,"g":94},"3312":{"m":66,"g":94},"3306":{"m":66,"g":94},"3305":{"m":66,"g":94},"3299":{"m":66,"g":94},"3294":{"m":66,"g":94},"3293":{"m":66,"g":94},"3292":{"m":66,"g":94},"3287":{"m":66,"g":94},"3288":{"m":66,"g":94},"3161":{"m":66,"g":94},"3273":{"m":66,"g":94},"3276":{"m":66,"g":94},"3274":{"m":66,"g":94},"3272":{"m":66,"g":94},"3270":{"m":66,"g":94},"3269":{"m":66,"g":94},"3268":{"m":66,"g":94},"3205":{"m":66,"g":94},"3207":{"m":66,"g":94},"3259":{"m":66,"g":94},"3261":{"m":66,"g":94},"3114":{"m":66,"g":94},"3255":{"m":66,"g":94},"3252":{"m":66,"g":94},"3251":{"m":66,"g":94},"3250":{"m":66,"g":94},"3249":{"m":66,"g":94},"3248":{"m":66,"g":94},"3246":{"m":66,"g":94},"3221":{"m":66,"g":94},"3242":{"m":66,"g":94},"3240":{"m":66,"g":94},"3238":{"m":66,"g":94},"3236":{"m":66,"g":94},"3235":{"m":66,"g":94},"3228":{"m":66,"g":94},"3369":{"m":67,"g":94},"3378":{"m":67,"g":94},"3376":{"m":67,"g":94},"3374":{"m":67,"g":94},"3373":{"m":67,"g":94},"3372":{"m":67,"g":94},"3366":{"m":67,"g":94},"3356":{"m":67,"g":94},"3300":{"m":67,"g":94},"3355":{"m":67,"g":94},"3314":{"m":67,"g":94},"3350":{"m":67,"g":94},"3347":{"m":67,"g":94},"3352":{"m":67,"g":94},"3349":{"m":67,"g":94},"3338":{"m":67,"g":94},"3337":{"m":67,"g":94},"3335":{"m":67,"g":94},"3332":{"m":67,"g":94},"3325":{"m":67,"g":94},"3327":{"m":67,"g":94},"3324":{"m":67,"g":94},"3168":{"m":67,"g":94},"3317":{"m":67,"g":94},"3309":{"m":67,"g":94},"3459":{"m":68,"g":94},"3457":{"m":68,"g":94},"3413":{"m":68,"g":94},"3453":{"m":68,"g":94},"3452":{"m":68,"g":94},"3442":{"m":68,"g":94},"3441":{"m":68,"g":94},"3440":{"m":68,"g":94},"3439":{"m":68,"g":94},"3437":{"m":68,"g":94},"3410":{"m":68,"g":94},"3435":{"m":68,"g":94},"3433":{"m":68,"g":94},"3431":{"m":68,"g":94},"3430":{"m":68,"g":94},"3425":{"m":68,"g":94},"3422":{"m":68,"g":94},"3421":{"m":68,"g":94},"3408":{"m":68,"g":94},"3415":{"m":68,"g":94},"3411":{"m":68,"g":94},"3412":{"m":68,"g":94},"3407":{"m":68,"g":94},"3404":{"m":68,"g":94},"3346":{"m":68,"g":94},"3382":{"m":68,"g":94},"3386":{"m":68,"g":94},"3275":{"m":68,"g":94},"3556":{"m":69,"g":94},"3555":{"m":69,"g":94},"3550":{"m":69,"g":94},"3553":{"m":69,"g":94},"3543":{"m":69,"g":94},"3534":{"m":69,"g":94},"3541":{"m":69,"g":94},"3536":{"m":69,"g":94},"3529":{"m":69,"g":94},"3530":{"m":69,"g":94},"3267":{"m":69,"g":94},"3450":{"m":69,"g":94},"3503":{"m":69,"g":94},"3493":{"m":69,"g":94},"3523":{"m":69,"g":94},"3522":{"m":69,"g":94},"3405":{"m":69,"g":94},"3502":{"m":69,"g":94},"3420":{"m":69,"g":94},"3495":{"m":69,"g":94},"3496":{"m":69,"g":94},"3499":{"m":69,"g":94},"3498":{"m":69,"g":94},"3497":{"m":69,"g":94},"3500":{"m":69,"g":94},"3492":{"m":69,"g":94},"3490":{"m":69,"g":94},"3418":{"m":69,"g":94},"3364":{"m":69,"g":94},"3473":{"m":69,"g":94},"3469":{"m":69,"g":94},"3468":{"m":69,"g":94},"3466":{"m":69,"g":94},"3638":{"m":70,"g":94},"3636":{"m":70,"g":94},"3634":{"m":70,"g":94},"3632":{"m":70,"g":94},"3619":{"m":70,"g":94},"3617":{"m":70,"g":94},"3260":{"m":70,"g":94},"3532":{"m":70,"g":94},"3597":{"m":70,"g":94},"3535":{"m":70,"g":94},"3258":{"m":70,"g":94},"3598":{"m":70,"g":94},"3564":{"m":70,"g":94},"3594":{"m":70,"g":94},"3592":{"m":70,"g":94},"3591":{"m":70,"g":94},"3589":{"m":70,"g":94},"3587":{"m":70,"g":94},"3582":{"m":70,"g":94},"3548":{"m":70,"g":94},"3584":{"m":70,"g":94},"3505":{"m":70,"g":94},"3581":{"m":70,"g":94},"3563":{"m":70,"g":94},"3363":{"m":70,"g":94},"3558":{"m":70,"g":94},"3557":{"m":70,"g":94},"3645":{"m":71,"g":94},"3644":{"m":71,"g":94},"3643":{"m":71,"g":94},"3639":{"m":71,"g":94},"4114":{"m":72,"g":94},"4099":{"m":72,"g":94},"4101":{"m":72,"g":94},"3211":{"m":72,"g":94},"3029":{"m":72,"g":94},"4110":{"m":72,"g":94},"4105":{"m":72,"g":94},"4109":{"m":72,"g":94},"4103":{"m":72,"g":94},"4108":{"m":72,"g":94},"4107":{"m":72,"g":94},"4102":{"m":72,"g":94},"4100":{"m":72,"g":94},"4012":{"m":72,"g":94},"4081":{"m":72,"g":94},"3986":{"m":72,"g":94},"4075":{"m":72,"g":94},"3990":{"m":72,"g":94},"3790":{"m":72,"g":94},"3941":{"m":72,"g":94},"4077":{"m":72,"g":94},"4074":{"m":72,"g":94},"4066":{"m":72,"g":94},"4071":{"m":72,"g":94},"4065":{"m":72,"g":94},"4016":{"m":72,"g":94},"3607":{"m":72,"g":94},"3712":{"m":72,"g":94},"3948":{"m":72,"g":94},"3954":{"m":72,"g":94},"4030":{"m":72,"g":94},"4051":{"m":72,"g":94},"4046":{"m":72,"g":94},"4053":{"m":72,"g":94},"4052":{"m":72,"g":94},"4023":{"m":72,"g":94},"4000":{"m":72,"g":94},"4049":{"m":72,"g":94},"4044":{"m":72,"g":94},"4043":{"m":72,"g":94},"4033":{"m":72,"g":94},"4039":{"m":72,"g":94},"3999":{"m":72,"g":94},"4034":{"m":72,"g":94},"4032":{"m":72,"g":94},"4027":{"m":72,"g":94},"4025":{"m":72,"g":94},"3264":{"m":72,"g":94},"4031":{"m":72,"g":94},"4029":{"m":72,"g":94},"4021":{"m":72,"g":94},"3988":{"m":72,"g":94},"4014":{"m":72,"g":94},"3826":{"m":72,"g":94},"4010":{"m":72,"g":94},"4008":{"m":72,"g":94},"3987":{"m":72,"g":94},"3822":{"m":72,"g":94},"3993":{"m":72,"g":94},"3406":{"m":72,"g":94},"3994":{"m":72,"g":94},"3992":{"m":72,"g":94},"3991":{"m":72,"g":94},"3893":{"m":72,"g":94},"3989":{"m":72,"g":94},"3985":{"m":72,"g":94},"3982":{"m":72,"g":94},"3979":{"m":72,"g":94},"3976":{"m":72,"g":94},"3977":{"m":72,"g":94},"3975":{"m":72,"g":94},"3967":{"m":72,"g":94},"3966":{"m":72,"g":94},"3963":{"m":72,"g":94},"3852":{"m":72,"g":94},"3566":{"m":72,"g":94},"3678":{"m":72,"g":94},"3950":{"m":72,"g":94},"3870":{"m":72,"g":94},"3613":{"m":72,"g":94},"3866":{"m":72,"g":94},"3861":{"m":72,"g":94},"3934":{"m":72,"g":94},"3933":{"m":72,"g":94},"3593":{"m":72,"g":94},"3922":{"m":72,"g":94},"3925":{"m":72,"g":94},"3914":{"m":72,"g":94},"3905":{"m":72,"g":94},"3897":{"m":72,"g":94},"3907":{"m":72,"g":94},"3898":{"m":72,"g":94},"3903":{"m":72,"g":94},"3791":{"m":72,"g":94},"3900":{"m":72,"g":94},"3894":{"m":72,"g":94},"3298":{"m":72,"g":94},"3860":{"m":72,"g":94},"3845":{"m":72,"g":94},"3602":{"m":72,"g":94},"3865":{"m":72,"g":94},"3843":{"m":72,"g":94},"3519":{"m":72,"g":94},"3841":{"m":72,"g":94},"3857":{"m":72,"g":94},"3709":{"m":72,"g":94},"3803":{"m":72,"g":94},"3787":{"m":72,"g":94},"3237":{"m":72,"g":94},"3641":{"m":72,"g":94},"3741":{"m":72,"g":94},"3799":{"m":72,"g":94},"3801":{"m":72,"g":94},"3821":{"m":72,"g":94},"3829":{"m":72,"g":94},"3828":{"m":72,"g":94},"3818":{"m":72,"g":94},"3730":{"m":72,"g":94},"3785":{"m":72,"g":94},"3813":{"m":72,"g":94},"3809":{"m":72,"g":94},"2693":{"m":72,"g":94},"3795":{"m":72,"g":94},"3116":{"m":72,"g":94},"3115":{"m":72,"g":94},"3562":{"m":72,"g":94},"3117":{"m":72,"g":94},"3766":{"m":72,"g":94},"3777":{"m":72,"g":94},"3348":{"m":72,"g":94},"3223":{"m":72,"g":94},"3772":{"m":72,"g":94},"3773":{"m":72,"g":94},"3771":{"m":72,"g":94},"3733":{"m":72,"g":94},"3754":{"m":72,"g":94},"3432":{"m":72,"g":94},"3761":{"m":72,"g":94},"3740":{"m":72,"g":94},"3680":{"m":72,"g":94},"3747":{"m":72,"g":94},"3737":{"m":72,"g":94},"3588":{"m":72,"g":94},"3652":{"m":72,"g":94},"3732":{"m":72,"g":94},"3705":{"m":72,"g":94},"3731":{"m":72,"g":94},"3722":{"m":72,"g":94},"3727":{"m":72,"g":94},"3710":{"m":72,"g":94},"3677":{"m":72,"g":94},"3601":{"m":72,"g":94},"3692":{"m":72,"g":94},"3700":{"m":72,"g":94},"3706":{"m":72,"g":94},"3628":{"m":72,"g":94},"3698":{"m":72,"g":94},"3657":{"m":72,"g":94},"3665":{"m":72,"g":94},"3635":{"m":72,"g":94},"3676":{"m":72,"g":94},"3663":{"m":72,"g":94},"3629":{"m":72,"g":94},"3654":{"m":72,"g":94},"3624":{"m":72,"g":94},"3567":{"m":72,"g":94},"3616":{"m":72,"g":94},"3650":{"m":72,"g":94},"4140":{"m":73,"g":94},"4142":{"m":73,"g":94},"4147":{"m":73,"g":94},"4134":{"m":73,"g":94},"4135":{"m":73,"g":94},"4138":{"m":73,"g":94},"4137":{"m":73,"g":94},"4132":{"m":73,"g":94},"4128":{"m":73,"g":94},"4126":{"m":73,"g":94},"4129":{"m":73,"g":94},"4131":{"m":73,"g":94},"4038":{"m":73,"g":94},"4111":{"m":73,"g":94},"4113":{"m":73,"g":94},"4117":{"m":73,"g":94},"4121":{"m":73,"g":94},"4041":{"m":74,"g":94},"4381":{"m":74,"g":94},"4377":{"m":74,"g":94},"4376":{"m":74,"g":94},"4374":{"m":74,"g":94},"4367":{"m":74,"g":94},"4086":{"m":74,"g":94},"4356":{"m":74,"g":94},"3679":{"m":74,"g":94},"3814":{"m":74,"g":94},"3835":{"m":74,"g":94},"3844":{"m":74,"g":94},"3896":{"m":74,"g":94},"3959":{"m":74,"g":94},"3980":{"m":74,"g":94},"3962":{"m":74,"g":94},"3961":{"m":74,"g":94},"4026":{"m":74,"g":94},"4079":{"m":74,"g":94},"4359":{"m":74,"g":94},"4212":{"m":74,"g":94},"4295":{"m":74,"g":94},"4342":{"m":74,"g":94},"4335":{"m":74,"g":94},"4329":{"m":74,"g":94},"4362":{"m":74,"g":94},"4348":{"m":74,"g":94},"4326":{"m":74,"g":94},"4354":{"m":74,"g":94},"4355":{"m":74,"g":94},"4352":{"m":74,"g":94},"4350":{"m":74,"g":94},"4278":{"m":74,"g":94},"4082":{"m":74,"g":94},"3203":{"m":74,"g":94},"4340":{"m":74,"g":94},"4337":{"m":74,"g":94},"3911":{"m":74,"g":94},"4334":{"m":74,"g":94},"4331":{"m":74,"g":94},"4333":{"m":74,"g":94},"4104":{"m":74,"g":94},"4215":{"m":74,"g":94},"4327":{"m":74,"g":94},"4297":{"m":74,"g":94},"4323":{"m":74,"g":94},"4321":{"m":74,"g":94},"4317":{"m":74,"g":94},"4229":{"m":74,"g":94},"4220":{"m":74,"g":94},"4311":{"m":74,"g":94},"4299":{"m":74,"g":94},"4287":{"m":74,"g":94},"4290":{"m":74,"g":94},"4199":{"m":74,"g":94},"4291":{"m":74,"g":94},"4136":{"m":74,"g":94},"4288":{"m":74,"g":94},"4284":{"m":74,"g":94},"4277":{"m":74,"g":94},"4279":{"m":74,"g":94},"4275":{"m":74,"g":94},"4272":{"m":74,"g":94},"4261":{"m":74,"g":94},"4256":{"m":74,"g":94},"4267":{"m":74,"g":94},"4262":{"m":74,"g":94},"4258":{"m":74,"g":94},"4255":{"m":74,"g":94},"4231":{"m":74,"g":94},"4144":{"m":74,"g":94},"4252":{"m":74,"g":94},"4206":{"m":74,"g":94},"3958":{"m":74,"g":94},"4165":{"m":74,"g":94},"4250":{"m":74,"g":94},"4230":{"m":74,"g":94},"4238":{"m":74,"g":94},"4243":{"m":74,"g":94},"4242":{"m":74,"g":94},"4241":{"m":74,"g":94},"4237":{"m":74,"g":94},"4235":{"m":74,"g":94},"4228":{"m":74,"g":94},"4217":{"m":74,"g":94},"3148":{"m":74,"g":94},"4218":{"m":74,"g":94},"4224":{"m":74,"g":94},"4193":{"m":74,"g":94},"3631":{"m":74,"g":94},"4225":{"m":74,"g":94},"4223":{"m":74,"g":94},"4213":{"m":74,"g":94},"4222":{"m":74,"g":94},"4219":{"m":74,"g":94},"4124":{"m":74,"g":94},"4216":{"m":74,"g":94},"4211":{"m":74,"g":94},"4210":{"m":74,"g":94},"3749":{"m":74,"g":94},"4203":{"m":74,"g":94},"4181":{"m":74,"g":94},"4200":{"m":74,"g":94},"4198":{"m":74,"g":94},"4197":{"m":74,"g":94},"4164":{"m":74,"g":94},"4195":{"m":74,"g":94},"4194":{"m":74,"g":94},"4189":{"m":74,"g":94},"4185":{"m":74,"g":94},"4187":{"m":74,"g":94},"4186":{"m":74,"g":94},"4178":{"m":74,"g":94},"4174":{"m":74,"g":94},"4179":{"m":74,"g":94},"4177":{"m":74,"g":94},"4176":{"m":74,"g":94},"4170":{"m":74,"g":94},"4168":{"m":74,"g":94},"4166":{"m":74,"g":94},"4162":{"m":74,"g":94},"4163":{"m":74,"g":94},"3888":{"m":74,"g":94},"4089":{"m":74,"g":94},"3786":{"m":74,"g":94},"3694":{"m":74,"g":94},"4152":{"m":74,"g":94},"4154":{"m":74,"g":94},"4151":{"m":74,"g":94},"4148":{"m":74,"g":94},"4402":{"m":75,"g":94},"4397":{"m":75,"g":94},"4399":{"m":75,"g":94},"4269":{"m":75,"g":94},"4320":{"m":75,"g":94},"4398":{"m":75,"g":94},"4393":{"m":75,"g":94},"4390":{"m":75,"g":94},"4392":{"m":75,"g":94},"4375":{"m":75,"g":94},"4669":{"m":76,"g":94},"4743":{"m":76,"g":94},"4797":{"m":76,"g":94},"4782":{"m":76,"g":94},"4777":{"m":76,"g":94},"4775":{"m":76,"g":94},"4784":{"m":76,"g":94},"4566":{"m":76,"g":94},"4728":{"m":76,"g":94},"4755":{"m":76,"g":94},"4753":{"m":76,"g":94},"4752":{"m":76,"g":94},"4751":{"m":76,"g":94},"4310":{"m":76,"g":94},"4705":{"m":76,"g":94},"4435":{"m":76,"g":94},"4695":{"m":76,"g":94},"4691":{"m":76,"g":94},"4738":{"m":76,"g":94},"4735":{"m":76,"g":94},"4744":{"m":76,"g":94},"3023":{"m":76,"g":94},"4737":{"m":76,"g":94},"3899":{"m":76,"g":94},"4609":{"m":76,"g":94},"4736":{"m":76,"g":94},"4716":{"m":76,"g":94},"4721":{"m":76,"g":94},"4731":{"m":76,"g":94},"4720":{"m":76,"g":94},"4396":{"m":76,"g":94},"4605":{"m":76,"g":94},"4680":{"m":76,"g":94},"4631":{"m":76,"g":94},"4698":{"m":76,"g":94},"4064":{"m":76,"g":94},"4525":{"m":76,"g":94},"4661":{"m":76,"g":94},"4610":{"m":76,"g":94},"4608":{"m":76,"g":94},"4685":{"m":76,"g":94},"4679":{"m":76,"g":94},"4643":{"m":76,"g":94},"3984":{"m":76,"g":94},"4660":{"m":76,"g":94},"4676":{"m":76,"g":94},"4670":{"m":76,"g":94},"4677":{"m":76,"g":94},"4674":{"m":76,"g":94},"4556":{"m":76,"g":94},"4665":{"m":76,"g":94},"4596":{"m":76,"g":94},"4639":{"m":76,"g":94},"4664":{"m":76,"g":94},"4582":{"m":76,"g":94},"4654":{"m":76,"g":94},"4637":{"m":76,"g":94},"4641":{"m":76,"g":94},"4613":{"m":76,"g":94},"4640":{"m":76,"g":94},"4558":{"m":76,"g":94},"4622":{"m":76,"g":94},"4592":{"m":76,"g":94},"4571":{"m":76,"g":94},"3446":{"m":76,"g":94},"4577":{"m":76,"g":94},"4549":{"m":76,"g":94},"4583":{"m":76,"g":94},"4514":{"m":76,"g":94},"4232":{"m":76,"g":94},"4515":{"m":76,"g":94},"4557":{"m":76,"g":94},"4553":{"m":76,"g":94},"4274":{"m":76,"g":94},"4521":{"m":76,"g":94},"4247":{"m":76,"g":94},"4532":{"m":76,"g":94},"4441":{"m":76,"g":94},"4538":{"m":76,"g":94},"4541":{"m":76,"g":94},"4542":{"m":76,"g":94},"4500":{"m":76,"g":94},"4531":{"m":76,"g":94},"3682":{"m":76,"g":94},"4458":{"m":76,"g":94},"4486":{"m":76,"g":94},"4507":{"m":76,"g":94},"4522":{"m":76,"g":94},"4520":{"m":76,"g":94},"4505":{"m":76,"g":94},"4482":{"m":76,"g":94},"4517":{"m":76,"g":94},"4513":{"m":76,"g":94},"4510":{"m":76,"g":94},"4499":{"m":76,"g":94},"4495":{"m":76,"g":94},"4446":{"m":76,"g":94},"4480":{"m":76,"g":94},"4067":{"m":76,"g":94},"4485":{"m":76,"g":94},"4418":{"m":76,"g":94},"4493":{"m":76,"g":94},"2798":{"m":76,"g":94},"4372":{"m":76,"g":94},"4483":{"m":76,"g":94},"4386":{"m":76,"g":94},"2797":{"m":76,"g":94},"3612":{"m":76,"g":94},"4448":{"m":76,"g":94},"4465":{"m":76,"g":94},"4474":{"m":76,"g":94},"4479":{"m":76,"g":94},"4424":{"m":76,"g":94},"4202":{"m":76,"g":94},"4484":{"m":76,"g":94},"4481":{"m":76,"g":94},"4477":{"m":76,"g":94},"4472":{"m":76,"g":94},"4363":{"m":76,"g":94},"4470":{"m":76,"g":94},"4469":{"m":76,"g":94},"4383":{"m":76,"g":94},"4468":{"m":76,"g":94},"4467":{"m":76,"g":94},"4466":{"m":76,"g":94},"4449":{"m":76,"g":94},"4464":{"m":76,"g":94},"4460":{"m":76,"g":94},"4368":{"m":76,"g":94},"4459":{"m":76,"g":94},"4447":{"m":76,"g":94},"4423":{"m":76,"g":94},"4391":{"m":76,"g":94},"4413":{"m":76,"g":94},"4454":{"m":76,"g":94},"4453":{"m":76,"g":94},"4455":{"m":76,"g":94},"4452":{"m":76,"g":94},"4451":{"m":76,"g":94},"4442":{"m":76,"g":94},"4439":{"m":76,"g":94},"4438":{"m":76,"g":94},"4437":{"m":76,"g":94},"4302":{"m":76,"g":94},"4427":{"m":76,"g":94},"4419":{"m":76,"g":94},"3964":{"m":76,"g":94},"4400":{"m":76,"g":94},"4009":{"m":76,"g":94},"4403":{"m":76,"g":94},"4878":{"m":77,"g":94},"4874":{"m":77,"g":94},"4873":{"m":77,"g":94},"4831":{"m":77,"g":94},"4872":{"m":77,"g":94},"4768":{"m":77,"g":94},"4871":{"m":77,"g":94},"4866":{"m":77,"g":94},"4749":{"m":77,"g":94},"4864":{"m":77,"g":94},"4772":{"m":77,"g":94},"4834":{"m":77,"g":94},"4492":{"m":77,"g":94},"4863":{"m":77,"g":94},"4855":{"m":77,"g":94},"4853":{"m":77,"g":94},"4840":{"m":77,"g":94},"4740":{"m":77,"g":94},"4750":{"m":77,"g":94},"4687":{"m":77,"g":94},"4729":{"m":77,"g":94},"4712":{"m":77,"g":94},"4704":{"m":77,"g":94},"4688":{"m":77,"g":94},"4681":{"m":77,"g":94},"4648":{"m":77,"g":94},"4832":{"m":77,"g":94},"4528":{"m":77,"g":94},"4597":{"m":77,"g":94},"4487":{"m":77,"g":94},"3949":{"m":77,"g":94},"4844":{"m":77,"g":94},"4846":{"m":77,"g":94},"4843":{"m":77,"g":94},"4799":{"m":77,"g":94},"4788":{"m":77,"g":94},"4809":{"m":77,"g":94},"4837":{"m":77,"g":94},"3969":{"m":77,"g":94},"4835":{"m":77,"g":94},"4815":{"m":77,"g":94},"4819":{"m":77,"g":94},"4770":{"m":77,"g":94},"4833":{"m":77,"g":94},"4830":{"m":77,"g":94},"4694":{"m":77,"g":94},"4826":{"m":77,"g":94},"4825":{"m":77,"g":94},"4828":{"m":77,"g":94},"4827":{"m":77,"g":94},"4823":{"m":77,"g":94},"4341":{"m":77,"g":94},"4638":{"m":77,"g":94},"4813":{"m":77,"g":94},"4764":{"m":77,"g":94},"4706":{"m":77,"g":94},"4745":{"m":77,"g":94},"4565":{"m":77,"g":94},"4804":{"m":77,"g":94},"4506":{"m":77,"g":94},"4388":{"m":77,"g":94},"4628":{"m":77,"g":94},"4719":{"m":77,"g":94},"5091":{"m":78,"g":94},"5080":{"m":78,"g":94},"5089":{"m":78,"g":94},"5079":{"m":78,"g":94},"5088":{"m":78,"g":94},"5052":{"m":78,"g":94},"5050":{"m":78,"g":94},"5074":{"m":78,"g":94},"4535":{"m":78,"g":94},"5072":{"m":78,"g":94},"4918":{"m":78,"g":94},"4996":{"m":78,"g":94},"4995":{"m":78,"g":94},"4994":{"m":78,"g":94},"5060":{"m":78,"g":94},"5057":{"m":78,"g":94},"5056":{"m":78,"g":94},"4625":{"m":78,"g":94},"5049":{"m":78,"g":94},"5051":{"m":78,"g":94},"5039":{"m":78,"g":94},"4796":{"m":78,"g":94},"5046":{"m":78,"g":94},"5048":{"m":78,"g":94},"5036":{"m":78,"g":94},"4992":{"m":78,"g":94},"5005":{"m":78,"g":94},"5024":{"m":78,"g":94},"5030":{"m":78,"g":94},"5020":{"m":78,"g":94},"4727":{"m":78,"g":94},"5009":{"m":78,"g":94},"5011":{"m":78,"g":94},"4817":{"m":78,"g":94},"5008":{"m":78,"g":94},"4951":{"m":78,"g":94},"4861":{"m":78,"g":94},"4989":{"m":78,"g":94},"4581":{"m":78,"g":94},"4915":{"m":78,"g":94},"4977":{"m":78,"g":94},"4958":{"m":78,"g":94},"4767":{"m":78,"g":94},"4959":{"m":78,"g":94},"4954":{"m":78,"g":94},"4953":{"m":78,"g":94},"4950":{"m":78,"g":94},"4754":{"m":78,"g":94},"4944":{"m":78,"g":94},"4928":{"m":78,"g":94},"4913":{"m":78,"g":94},"4936":{"m":78,"g":94},"4883":{"m":78,"g":94},"4925":{"m":78,"g":94},"4933":{"m":78,"g":94},"4932":{"m":78,"g":94},"4931":{"m":78,"g":94},"4902":{"m":78,"g":94},"4930":{"m":78,"g":94},"4927":{"m":78,"g":94},"4926":{"m":78,"g":94},"4896":{"m":78,"g":94},"4914":{"m":78,"g":94},"4908":{"m":78,"g":94},"4909":{"m":78,"g":94},"4890":{"m":78,"g":94},"4899":{"m":78,"g":94},"4898":{"m":78,"g":94},"4530":{"m":78,"g":94},"4891":{"m":78,"g":94},"4889":{"m":78,"g":94},"4886":{"m":78,"g":94},"4795":{"m":78,"g":94},"4845":{"m":78,"g":94},"4882":{"m":78,"g":94},"5117":{"m":79,"g":94},"5106":{"m":79,"g":94},"5092":{"m":79,"g":94},"5097":{"m":79,"g":94},"5445":{"m":80,"g":94},"5113":{"m":80,"g":94},"5425":{"m":80,"g":94},"5397":{"m":80,"g":94},"5398":{"m":80,"g":94},"5038":{"m":80,"g":94},"5211":{"m":80,"g":94},"5264":{"m":80,"g":94},"5436":{"m":80,"g":94},"5344":{"m":80,"g":94},"5431":{"m":80,"g":94},"5434":{"m":80,"g":94},"5430":{"m":80,"g":94},"5419":{"m":80,"g":94},"5420":{"m":80,"g":94},"5423":{"m":80,"g":94},"5422":{"m":80,"g":94},"5415":{"m":80,"g":94},"5416":{"m":80,"g":94},"5351":{"m":80,"g":94},"5412":{"m":80,"g":94},"5352":{"m":80,"g":94},"5214":{"m":80,"g":94},"5406":{"m":80,"g":94},"5401":{"m":80,"g":94},"5400":{"m":80,"g":94},"5381":{"m":80,"g":94},"5399":{"m":80,"g":94},"5395":{"m":80,"g":94},"5279":{"m":80,"g":94},"5291":{"m":80,"g":94},"5368":{"m":80,"g":94},"5393":{"m":80,"g":94},"5263":{"m":80,"g":94},"5392":{"m":80,"g":94},"5371":{"m":80,"g":94},"5385":{"m":80,"g":94},"5370":{"m":80,"g":94},"5384":{"m":80,"g":94},"5326":{"m":80,"g":94},"5003":{"m":80,"g":94},"5367":{"m":80,"g":94},"5364":{"m":80,"g":94},"5277":{"m":80,"g":94},"5360":{"m":80,"g":94},"5359":{"m":80,"g":94},"5161":{"m":80,"g":94},"5328":{"m":80,"g":94},"5357":{"m":80,"g":94},"5342":{"m":80,"g":94},"5343":{"m":80,"g":94},"5322":{"m":80,"g":94},"5341":{"m":80,"g":94},"5337":{"m":80,"g":94},"5336":{"m":80,"g":94},"5333":{"m":80,"g":94},"5332":{"m":80,"g":94},"5331":{"m":80,"g":94},"5327":{"m":80,"g":94},"4848":{"m":80,"g":94},"5294":{"m":80,"g":94},"5321":{"m":80,"g":94},"4884":{"m":80,"g":94},"5210":{"m":80,"g":94},"5317":{"m":80,"g":94},"5316":{"m":80,"g":94},"5315":{"m":80,"g":94},"5120":{"m":80,"g":94},"5299":{"m":80,"g":94},"5311":{"m":80,"g":94},"5142":{"m":80,"g":94},"5271":{"m":80,"g":94},"5065":{"m":80,"g":94},"5310":{"m":80,"g":94},"5308":{"m":80,"g":94},"5307":{"m":80,"g":94},"5306":{"m":80,"g":94},"5304":{"m":80,"g":94},"5303":{"m":80,"g":94},"5302":{"m":80,"g":94},"5298":{"m":80,"g":94},"5301":{"m":80,"g":94},"5300":{"m":80,"g":94},"5290":{"m":80,"g":94},"5292":{"m":80,"g":94},"5289":{"m":80,"g":94},"5288":{"m":80,"g":94},"5287":{"m":80,"g":94},"5286":{"m":80,"g":94},"5280":{"m":80,"g":94},"5193":{"m":80,"g":94},"5244":{"m":80,"g":94},"5265":{"m":80,"g":94},"5254":{"m":80,"g":94},"5262":{"m":80,"g":94},"5127":{"m":80,"g":94},"5228":{"m":80,"g":94},"5259":{"m":80,"g":94},"5245":{"m":80,"g":94},"5216":{"m":80,"g":94},"5167":{"m":80,"g":94},"5213":{"m":80,"g":94},"5215":{"m":80,"g":94},"5204":{"m":80,"g":94},"4880":{"m":80,"g":94},"5086":{"m":80,"g":94},"5190":{"m":80,"g":94},"4444":{"m":80,"g":94},"5209":{"m":80,"g":94},"5196":{"m":80,"g":94},"5207":{"m":80,"g":94},"5171":{"m":80,"g":94},"5144":{"m":80,"g":94},"5102":{"m":80,"g":94},"5194":{"m":80,"g":94},"5128":{"m":80,"g":94},"5185":{"m":80,"g":94},"5189":{"m":80,"g":94},"4058":{"m":80,"g":94},"5179":{"m":80,"g":94},"5180":{"m":80,"g":94},"5176":{"m":80,"g":94},"5110":{"m":80,"g":94},"5068":{"m":80,"g":94},"5173":{"m":80,"g":94},"5175":{"m":80,"g":94},"5174":{"m":80,"g":94},"3972":{"m":80,"g":94},"5159":{"m":80,"g":94},"4938":{"m":80,"g":94},"5139":{"m":80,"g":94},"4911":{"m":80,"g":94},"5150":{"m":80,"g":94},"5155":{"m":80,"g":94},"4686":{"m":80,"g":94},"5158":{"m":80,"g":94},"5151":{"m":80,"g":94},"5115":{"m":80,"g":94},"5152":{"m":80,"g":94},"4760":{"m":80,"g":94},"5103":{"m":80,"g":94},"5147":{"m":80,"g":94},"5083":{"m":80,"g":94},"5140":{"m":80,"g":94},"5145":{"m":80,"g":94},"4984":{"m":80,"g":94},"5137":{"m":80,"g":94},"5133":{"m":80,"g":94},"5090":{"m":80,"g":94},"5126":{"m":80,"g":94},"5582":{"m":81,"g":94},"5581":{"m":81,"g":94},"4947":{"m":81,"g":94},"5571":{"m":81,"g":94},"5564":{"m":81,"g":94},"5568":{"m":81,"g":94},"5432":{"m":81,"g":94},"5149":{"m":81,"g":94},"5562":{"m":81,"g":94},"5543":{"m":81,"g":94},"5561":{"m":81,"g":94},"5546":{"m":81,"g":94},"5549":{"m":81,"g":94},"5504":{"m":81,"g":94},"5475":{"m":81,"g":94},"5460":{"m":81,"g":94},"5547":{"m":81,"g":94},"5534":{"m":81,"g":94},"5545":{"m":81,"g":94},"5548":{"m":81,"g":94},"5340":{"m":81,"g":94},"5540":{"m":81,"g":94},"5544":{"m":81,"g":94},"5542":{"m":81,"g":94},"4693":{"m":81,"g":94},"5476":{"m":81,"g":94},"5497":{"m":81,"g":94},"5461":{"m":81,"g":94},"5473":{"m":81,"g":94},"5518":{"m":81,"g":94},"5440":{"m":81,"g":94},"4836":{"m":81,"g":94},"5512":{"m":81,"g":94},"5373":{"m":81,"g":94},"5426":{"m":81,"g":94},"5511":{"m":81,"g":94},"5500":{"m":81,"g":94},"5503":{"m":81,"g":94},"5496":{"m":81,"g":94},"5493":{"m":81,"g":94},"5205":{"m":81,"g":94},"5479":{"m":81,"g":94},"4887":{"m":81,"g":94},"5480":{"m":81,"g":94},"5481":{"m":81,"g":94},"5484":{"m":81,"g":94},"5489":{"m":81,"g":94},"4982":{"m":81,"g":94},"5345":{"m":81,"g":94},"5467":{"m":81,"g":94},"5463":{"m":81,"g":94},"5447":{"m":81,"g":94},"5449":{"m":81,"g":94},"5444":{"m":81,"g":94},"5611":{"m":82,"g":94},"5510":{"m":82,"g":94},"5610":{"m":82,"g":94},"5580":{"m":82,"g":94},"5604":{"m":82,"g":94},"5609":{"m":82,"g":94},"5608":{"m":82,"g":94},"5477":{"m":82,"g":94},"5589":{"m":82,"g":94},"5598":{"m":82,"g":94},"5488":{"m":82,"g":94},"5037":{"m":82,"g":94},"5021":{"m":82,"g":94},"4980":{"m":82,"g":94},"4937":{"m":82,"g":94},"4718":{"m":82,"g":94},"4590":{"m":82,"g":94},"3443":{"m":82,"g":94},"5348":{"m":82,"g":94},"5570":{"m":82,"g":94},"5318":{"m":82,"g":94},"5590":{"m":82,"g":94},"5319":{"m":82,"g":94},"5241":{"m":82,"g":94},"5188":{"m":82,"g":94},"5141":{"m":82,"g":94},"5019":{"m":82,"g":94},"5016":{"m":82,"g":94},"4859":{"m":82,"g":94},"4852":{"m":82,"g":94},"4675":{"m":82,"g":94},"4733":{"m":82,"g":94},"5378":{"m":82,"g":94},"5224":{"m":82,"g":94},"4226":{"m":82,"g":94},"5433":{"m":82,"g":94},"5588":{"m":82,"g":94},"5452":{"m":82,"g":94},"5586":{"m":82,"g":94},"5575":{"m":82,"g":94},"5417":{"m":82,"g":94},"5531":{"m":82,"g":94},"5526":{"m":82,"g":94},"5521":{"m":82,"g":94},"5559":{"m":82,"g":94},"5560":{"m":82,"g":94},"5567":{"m":82,"g":94},"5574":{"m":82,"g":94},"5795":{"m":83,"g":94},"5691":{"m":83,"g":94},"5790":{"m":83,"g":94},"5791":{"m":83,"g":94},"5769":{"m":83,"g":94},"5789":{"m":83,"g":94},"5787":{"m":83,"g":94},"5786":{"m":83,"g":94},"5785":{"m":83,"g":94},"5779":{"m":83,"g":94},"5777":{"m":83,"g":94},"5774":{"m":83,"g":94},"5776":{"m":83,"g":94},"5772":{"m":83,"g":94},"3744":{"m":83,"g":94},"4986":{"m":83,"g":94},"4971":{"m":83,"g":94},"4870":{"m":83,"g":94},"5633":{"m":83,"g":94},"5599":{"m":83,"g":94},"5565":{"m":83,"g":94},"5509":{"m":83,"g":94},"5687":{"m":83,"g":94},"5592":{"m":83,"g":94},"5607":{"m":83,"g":94},"5730":{"m":83,"g":94},"5697":{"m":83,"g":94},"5748":{"m":83,"g":94},"5682":{"m":83,"g":94},"5685":{"m":83,"g":94},"5716":{"m":83,"g":94},"5720":{"m":83,"g":94},"5722":{"m":83,"g":94},"5728":{"m":83,"g":94},"5733":{"m":83,"g":94},"5736":{"m":83,"g":94},"5756":{"m":83,"g":94},"5760":{"m":83,"g":94},"5552":{"m":83,"g":94},"5718":{"m":83,"g":94},"5719":{"m":83,"g":94},"5737":{"m":83,"g":94},"5740":{"m":83,"g":94},"5753":{"m":83,"g":94},"5754":{"m":83,"g":94},"5750":{"m":83,"g":94},"5723":{"m":83,"g":94},"5704":{"m":83,"g":94},"5738":{"m":83,"g":94},"5715":{"m":83,"g":94},"5078":{"m":83,"g":94},"4491":{"m":83,"g":94},"5706":{"m":83,"g":94},"5648":{"m":83,"g":94},"5684":{"m":83,"g":94},"5707":{"m":83,"g":94},"5349":{"m":83,"g":94},"5688":{"m":83,"g":94},"5686":{"m":83,"g":94},"5683":{"m":83,"g":94},"5667":{"m":83,"g":94},"5677":{"m":83,"g":94},"5530":{"m":83,"g":94},"5671":{"m":83,"g":94},"5670":{"m":83,"g":94},"5435":{"m":83,"g":94},"5669":{"m":83,"g":94},"5666":{"m":83,"g":94},"5281":{"m":83,"g":94},"5601":{"m":83,"g":94},"5619":{"m":83,"g":94},"5628":{"m":83,"g":94},"5649":{"m":83,"g":94},"5646":{"m":83,"g":94},"5638":{"m":83,"g":94},"5634":{"m":83,"g":94},"5272":{"m":83,"g":94},"5641":{"m":83,"g":94},"5632":{"m":83,"g":94},"5640":{"m":83,"g":94},"5624":{"m":83,"g":94},"5622":{"m":83,"g":94},"5620":{"m":83,"g":94},"5618":{"m":83,"g":94},"5615":{"m":83,"g":94},"5578":{"m":83,"g":94},"5845":{"m":84,"g":94},"5849":{"m":84,"g":94},"5854":{"m":84,"g":94},"5823":{"m":84,"g":94},"5847":{"m":84,"g":94},"5816":{"m":84,"g":94},"5798":{"m":84,"g":94},"5850":{"m":84,"g":94},"5851":{"m":84,"g":94},"5846":{"m":84,"g":94},"5726":{"m":84,"g":94},"5839":{"m":84,"g":94},"5842":{"m":84,"g":94},"5833":{"m":84,"g":94},"5838":{"m":84,"g":94},"5551":{"m":84,"g":94},"5825":{"m":84,"g":94},"5276":{"m":84,"g":94},"5482":{"m":84,"g":94},"5771":{"m":84,"g":94},"5809":{"m":84,"g":94},"5807":{"m":84,"g":94},"5690":{"m":84,"g":94},"5390":{"m":84,"g":94},"5788":{"m":84,"g":94},"5643":{"m":84,"g":94},"5796":{"m":84,"g":94},"5797":{"m":84,"g":94},"5939":{"m":85,"g":94},"5934":{"m":85,"g":94},"5881":{"m":85,"g":94},"5915":{"m":85,"g":94},"5930":{"m":85,"g":94},"5724":{"m":85,"g":94},"5783":{"m":85,"g":94},"5933":{"m":85,"g":94},"5932":{"m":85,"g":94},"5912":{"m":85,"g":94},"5909":{"m":85,"g":94},"5917":{"m":85,"g":94},"5919":{"m":85,"g":94},"5910":{"m":85,"g":94},"5905":{"m":85,"g":94},"5383":{"m":85,"g":94},"5903":{"m":85,"g":94},"5900":{"m":85,"g":94},"5899":{"m":85,"g":94},"5870":{"m":85,"g":94},"5830":{"m":85,"g":94},"5901":{"m":85,"g":94},"5898":{"m":85,"g":94},"5861":{"m":85,"g":94},"5696":{"m":85,"g":94},"5725":{"m":85,"g":94},"5893":{"m":85,"g":94},"5793":{"m":85,"g":94},"5896":{"m":85,"g":94},"5746":{"m":85,"g":94},"5895":{"m":85,"g":94},"5841":{"m":85,"g":94},"5836":{"m":85,"g":94},"5894":{"m":85,"g":94},"5880":{"m":85,"g":94},"5875":{"m":85,"g":94},"5859":{"m":85,"g":94},"4115":{"m":85,"g":94},"4949":{"m":85,"g":94},"5820":{"m":85,"g":94},"5868":{"m":85,"g":94},"5860":{"m":85,"g":94},"5801":{"m":85,"g":94},"5857":{"m":85,"g":94},"6165":{"m":86,"g":94},"5778":{"m":86,"g":94},"6162":{"m":86,"g":94},"6089":{"m":86,"g":94},"6141":{"m":86,"g":94},"5822":{"m":86,"g":94},"6101":{"m":86,"g":94},"5745":{"m":86,"g":94},"6132":{"m":86,"g":94},"6131":{"m":86,"g":94},"6112":{"m":86,"g":94},"6129":{"m":86,"g":94},"6123":{"m":86,"g":94},"6097":{"m":86,"g":94},"5662":{"m":86,"g":94},"5764":{"m":86,"g":94},"6119":{"m":86,"g":94},"5626":{"m":86,"g":94},"6091":{"m":86,"g":94},"5572":{"m":86,"g":94},"5232":{"m":86,"g":94},"5219":{"m":86,"g":94},"5121":{"m":86,"g":94},"6077":{"m":86,"g":94},"6111":{"m":86,"g":94},"6038":{"m":86,"g":94},"6079":{"m":86,"g":94},"6034":{"m":86,"g":94},"6102":{"m":86,"g":94},"6105":{"m":86,"g":94},"5993":{"m":86,"g":94},"6075":{"m":86,"g":94},"6063":{"m":86,"g":94},"5233":{"m":86,"g":94},"5014":{"m":86,"g":94},"6084":{"m":86,"g":94},"3853":{"m":86,"g":94},"6010":{"m":86,"g":94},"6039":{"m":86,"g":94},"5655":{"m":86,"g":94},"6062":{"m":86,"g":94},"6004":{"m":86,"g":94},"6045":{"m":86,"g":94},"5885":{"m":86,"g":94},"5751":{"m":86,"g":94},"6057":{"m":86,"g":94},"6048":{"m":86,"g":94},"6047":{"m":86,"g":94},"6046":{"m":86,"g":94},"5081":{"m":86,"g":94},"5752":{"m":86,"g":94},"5996":{"m":86,"g":94},"5428":{"m":86,"g":94},"5555":{"m":86,"g":94},"5587":{"m":86,"g":94},"5781":{"m":86,"g":94},"6018":{"m":86,"g":94},"6002":{"m":86,"g":94},"5997":{"m":86,"g":94},"5679":{"m":86,"g":94},"5957":{"m":86,"g":94},"6012":{"m":86,"g":94},"5992":{"m":86,"g":94},"5998":{"m":86,"g":94},"5991":{"m":86,"g":94},"5986":{"m":86,"g":94},"5977":{"m":86,"g":94},"5975":{"m":86,"g":94},"5681":{"m":86,"g":94},"5969":{"m":86,"g":94},"5968":{"m":86,"g":94},"5350":{"m":86,"g":94},"5967":{"m":86,"g":94},"5908":{"m":86,"g":94},"5960":{"m":86,"g":94},"5782":{"m":86,"g":94},"5956":{"m":86,"g":94},"5945":{"m":86,"g":94},"5944":{"m":86,"g":94},"5952":{"m":86,"g":94},"5953":{"m":86,"g":94},"5834":{"m":86,"g":94},"5921":{"m":86,"g":94},"6245":{"m":87,"g":94},"6259":{"m":87,"g":94},"6252":{"m":87,"g":94},"5084":{"m":87,"g":94},"5657":{"m":87,"g":94},"6247":{"m":87,"g":94},"6235":{"m":87,"g":94},"6251":{"m":87,"g":94},"6042":{"m":87,"g":94},"6248":{"m":87,"g":94},"6225":{"m":87,"g":94},"5922":{"m":87,"g":94},"6241":{"m":87,"g":94},"6243":{"m":87,"g":94},"6244":{"m":87,"g":94},"6223":{"m":87,"g":94},"6206":{"m":87,"g":94},"6209":{"m":87,"g":94},"6231":{"m":87,"g":94},"6212":{"m":87,"g":94},"6201":{"m":87,"g":94},"6213":{"m":87,"g":94},"5558":{"m":87,"g":94},"6154":{"m":87,"g":94},"6043":{"m":87,"g":94},"6204":{"m":87,"g":94},"6202":{"m":87,"g":94},"6178":{"m":87,"g":94},"6198":{"m":87,"g":94},"6192":{"m":87,"g":94},"6188":{"m":87,"g":94},"6032":{"m":87,"g":94},"6199":{"m":87,"g":94},"5621":{"m":87,"g":94},"6196":{"m":87,"g":94},"6195":{"m":87,"g":94},"6073":{"m":87,"g":94},"6169":{"m":87,"g":94},"6191":{"m":87,"g":94},"6190":{"m":87,"g":94},"6186":{"m":87,"g":94},"5654":{"m":87,"g":94},"6180":{"m":87,"g":94},"6179":{"m":87,"g":94},"6184":{"m":87,"g":94},"6183":{"m":87,"g":94},"4701":{"m":87,"g":94},"6181":{"m":87,"g":94},"6146":{"m":87,"g":94},"6114":{"m":87,"g":94},"6118":{"m":87,"g":94},"6016":{"m":87,"g":94},"6566":{"m":88,"g":94},"6567":{"m":88,"g":94},"6485":{"m":88,"g":94},"6560":{"m":88,"g":94},"6550":{"m":88,"g":94},"6533":{"m":88,"g":94},"6562":{"m":88,"g":94},"6524":{"m":88,"g":94},"6347":{"m":88,"g":94},"6558":{"m":88,"g":94},"6521":{"m":88,"g":94},"6507":{"m":88,"g":94},"6474":{"m":88,"g":94},"6452":{"m":88,"g":94},"6404":{"m":88,"g":94},"6493":{"m":88,"g":94},"6355":{"m":88,"g":94},"6535":{"m":88,"g":94},"6536":{"m":88,"g":94},"6059":{"m":88,"g":94},"6532":{"m":88,"g":94},"6120":{"m":88,"g":94},"6522":{"m":88,"g":94},"6520":{"m":88,"g":94},"6469":{"m":88,"g":94},"6482":{"m":88,"g":94},"6308":{"m":88,"g":94},"6388":{"m":88,"g":94},"6492":{"m":88,"g":94},"6504":{"m":88,"g":94},"6019":{"m":88,"g":94},"6457":{"m":88,"g":94},"6499":{"m":88,"g":94},"6510":{"m":88,"g":94},"6508":{"m":88,"g":94},"5759":{"m":88,"g":94},"6419":{"m":88,"g":94},"6275":{"m":88,"g":94},"6503":{"m":88,"g":94},"5573":{"m":88,"g":94},"6445":{"m":88,"g":94},"6461":{"m":88,"g":94},"6467":{"m":88,"g":94},"6468":{"m":88,"g":94},"6476":{"m":88,"g":94},"6311":{"m":88,"g":94},"6487":{"m":88,"g":94},"5339":{"m":88,"g":94},"6381":{"m":88,"g":94},"6214":{"m":88,"g":94},"6475":{"m":88,"g":94},"6472":{"m":88,"g":94},"6447":{"m":88,"g":94},"6385":{"m":88,"g":94},"6444":{"m":88,"g":94},"6405":{"m":88,"g":94},"6429":{"m":88,"g":94},"6412":{"m":88,"g":94},"6387":{"m":88,"g":94},"6386":{"m":88,"g":94},"6326":{"m":88,"g":94},"6438":{"m":88,"g":94},"6321":{"m":88,"g":94},"4957":{"m":88,"g":94},"6440":{"m":88,"g":94},"6431":{"m":88,"g":94},"6098":{"m":88,"g":94},"6414":{"m":88,"g":94},"6430":{"m":88,"g":94},"6417":{"m":88,"g":94},"6137":{"m":88,"g":94},"6401":{"m":88,"g":94},"6400":{"m":88,"g":94},"5974":{"m":88,"g":94},"6325":{"m":88,"g":94},"6323":{"m":88,"g":94},"6333":{"m":88,"g":94},"6397":{"m":88,"g":94},"6396":{"m":88,"g":94},"6395":{"m":88,"g":94},"6339":{"m":88,"g":94},"6383":{"m":88,"g":94},"6392":{"m":88,"g":94},"6331":{"m":88,"g":94},"6391":{"m":88,"g":94},"6365":{"m":88,"g":94},"6250":{"m":88,"g":94},"6187":{"m":88,"g":94},"6379":{"m":88,"g":94},"6377":{"m":88,"g":94},"6362":{"m":88,"g":94},"6364":{"m":88,"g":94},"6330":{"m":88,"g":94},"6290":{"m":88,"g":94},"6041":{"m":88,"g":94},"6284":{"m":88,"g":94},"6257":{"m":88,"g":94},"6108":{"m":88,"g":94},"6134":{"m":88,"g":94},"6107":{"m":88,"g":94},"4741":{"m":88,"g":94},"6348":{"m":88,"g":94},"6211":{"m":88,"g":94},"6175":{"m":88,"g":94},"6366":{"m":88,"g":94},"6373":{"m":88,"g":94},"6356":{"m":88,"g":94},"6368":{"m":88,"g":94},"6324":{"m":88,"g":94},"6316":{"m":88,"g":94},"5099":{"m":88,"g":94},"6361":{"m":88,"g":94},"6360":{"m":88,"g":94},"6358":{"m":88,"g":94},"6359":{"m":88,"g":94},"6121":{"m":88,"g":94},"5694":{"m":88,"g":94},"6334":{"m":88,"g":94},"6136":{"m":88,"g":94},"6336":{"m":88,"g":94},"6327":{"m":88,"g":94},"6302":{"m":88,"g":94},"6147":{"m":88,"g":94},"6317":{"m":88,"g":94},"6216":{"m":88,"g":94},"6298":{"m":88,"g":94},"6109":{"m":88,"g":94},"5914":{"m":88,"g":94},"6009":{"m":88,"g":94},"6274":{"m":88,"g":94},"6283":{"m":88,"g":94},"6300":{"m":88,"g":94},"6138":{"m":88,"g":94},"6282":{"m":88,"g":94},"6276":{"m":88,"g":94},"6273":{"m":88,"g":94},"6115":{"m":88,"g":94},"7038":{"m":89,"g":94},"6833":{"m":89,"g":94},"7029":{"m":89,"g":94},"7027":{"m":89,"g":94},"6980":{"m":89,"g":94},"6964":{"m":89,"g":94},"7023":{"m":89,"g":94},"7018":{"m":89,"g":94},"7017":{"m":89,"g":94},"6741":{"m":89,"g":94},"6987":{"m":89,"g":94},"7015":{"m":89,"g":94},"7013":{"m":89,"g":94},"6884":{"m":89,"g":94},"6992":{"m":89,"g":94},"6998":{"m":89,"g":94},"7008":{"m":89,"g":94},"7007":{"m":89,"g":94},"6958":{"m":89,"g":94},"6557":{"m":89,"g":94},"6990":{"m":89,"g":94},"6960":{"m":89,"g":94},"6973":{"m":89,"g":94},"6983":{"m":89,"g":94},"6929":{"m":89,"g":94},"6977":{"m":89,"g":94},"6967":{"m":89,"g":94},"6981":{"m":89,"g":94},"6979":{"m":89,"g":94},"6976":{"m":89,"g":94},"6970":{"m":89,"g":94},"6937":{"m":89,"g":94},"6965":{"m":89,"g":94},"6974":{"m":89,"g":94},"6966":{"m":89,"g":94},"6956":{"m":89,"g":94},"6963":{"m":89,"g":94},"6926":{"m":89,"g":94},"6968":{"m":89,"g":94},"6853":{"m":89,"g":94},"6957":{"m":89,"g":94},"6955":{"m":89,"g":94},"6916":{"m":89,"g":94},"6885":{"m":89,"g":94},"6950":{"m":89,"g":94},"6953":{"m":89,"g":94},"6220":{"m":89,"g":94},"6866":{"m":89,"g":94},"6895":{"m":89,"g":94},"6874":{"m":89,"g":94},"6915":{"m":89,"g":94},"6924":{"m":89,"g":94},"6945":{"m":89,"g":94},"6369":{"m":89,"g":94},"5955":{"m":89,"g":94},"6912":{"m":89,"g":94},"6910":{"m":89,"g":94},"6944":{"m":89,"g":94},"6943":{"m":89,"g":94},"6942":{"m":89,"g":94},"6939":{"m":89,"g":94},"6767":{"m":89,"g":94},"6879":{"m":89,"g":94},"6922":{"m":89,"g":94},"6932":{"m":89,"g":94},"6934":{"m":89,"g":94},"6931":{"m":89,"g":94},"6930":{"m":89,"g":94},"6838":{"m":89,"g":94},"6458":{"m":89,"g":94},"6877":{"m":89,"g":94},"6887":{"m":89,"g":94},"6890":{"m":89,"g":94},"6764":{"m":89,"g":94},"6170":{"m":89,"g":94},"6837":{"m":89,"g":94},"6868":{"m":89,"g":94},"6865":{"m":89,"g":94},"6851":{"m":89,"g":94},"6846":{"m":89,"g":94},"6820":{"m":89,"g":94},"6277":{"m":89,"g":94},"6861":{"m":89,"g":94},"6878":{"m":89,"g":94},"6736":{"m":89,"g":94},"6852":{"m":89,"g":94},"6460":{"m":89,"g":94},"6659":{"m":89,"g":94},"6858":{"m":89,"g":94},"6745":{"m":89,"g":94},"5929":{"m":89,"g":94},"6735":{"m":89,"g":94},"6816":{"m":89,"g":94},"6818":{"m":89,"g":94},"6671":{"m":89,"g":94},"6456":{"m":89,"g":94},"6766":{"m":89,"g":94},"6093":{"m":89,"g":94},"6812":{"m":89,"g":94},"6811":{"m":89,"g":94},"6815":{"m":89,"g":94},"6813":{"m":89,"g":94},"6780":{"m":89,"g":94},"5382":{"m":89,"g":94},"6805":{"m":89,"g":94},"6699":{"m":89,"g":94},"6803":{"m":89,"g":94},"6804":{"m":89,"g":94},"6799":{"m":89,"g":94},"6800":{"m":89,"g":94},"5981":{"m":89,"g":94},"6421":{"m":89,"g":94},"6797":{"m":89,"g":94},"6795":{"m":89,"g":94},"6787":{"m":89,"g":94},"6794":{"m":89,"g":94},"6788":{"m":89,"g":94},"6791":{"m":89,"g":94},"6792":{"m":89,"g":94},"6734":{"m":89,"g":94},"6408":{"m":89,"g":94},"6786":{"m":89,"g":94},"6785":{"m":89,"g":94},"6782":{"m":89,"g":94},"6784":{"m":89,"g":94},"6679":{"m":89,"g":94},"6772":{"m":89,"g":94},"6289":{"m":89,"g":94},"6509":{"m":89,"g":94},"6737":{"m":89,"g":94},"6761":{"m":89,"g":94},"6765":{"m":89,"g":94},"6727":{"m":89,"g":94},"6265":{"m":89,"g":94},"6748":{"m":89,"g":94},"6746":{"m":89,"g":94},"6742":{"m":89,"g":94},"6728":{"m":89,"g":94},"6680":{"m":89,"g":94},"6437":{"m":89,"g":94},"6729":{"m":89,"g":94},"6545":{"m":89,"g":94},"6705":{"m":89,"g":94},"6715":{"m":89,"g":94},"6725":{"m":89,"g":94},"6718":{"m":89,"g":94},"6720":{"m":89,"g":94},"6726":{"m":89,"g":94},"6676":{"m":89,"g":94},"6709":{"m":89,"g":94},"6719":{"m":89,"g":94},"6479":{"m":89,"g":94},"6668":{"m":89,"g":94},"6682":{"m":89,"g":94},"6711":{"m":89,"g":94},"6710":{"m":89,"g":94},"6712":{"m":89,"g":94},"6706":{"m":89,"g":94},"6703":{"m":89,"g":94},"6697":{"m":89,"g":94},"6649":{"m":89,"g":94},"6685":{"m":89,"g":94},"6689":{"m":89,"g":94},"6693":{"m":89,"g":94},"6655":{"m":89,"g":94},"6260":{"m":89,"g":94},"6627":{"m":89,"g":94},"6687":{"m":89,"g":94},"6582":{"m":89,"g":94},"6672":{"m":89,"g":94},"6678":{"m":89,"g":94},"6380":{"m":89,"g":94},"6665":{"m":89,"g":94},"6673":{"m":89,"g":94},"6007":{"m":89,"g":94},"6473":{"m":89,"g":94},"6677":{"m":89,"g":94},"6661":{"m":89,"g":94},"6660":{"m":89,"g":94},"6638":{"m":89,"g":94},"6606":{"m":89,"g":94},"6662":{"m":89,"g":94},"6652":{"m":89,"g":94},"6658":{"m":89,"g":94},"6403":{"m":89,"g":94},"6601":{"m":89,"g":94},"6640":{"m":89,"g":94},"6450":{"m":89,"g":94},"6650":{"m":89,"g":94},"6585":{"m":89,"g":94},"6648":{"m":89,"g":94},"6634":{"m":89,"g":94},"6646":{"m":89,"g":94},"6643":{"m":89,"g":94},"6547":{"m":89,"g":94},"6631":{"m":89,"g":94},"6603":{"m":89,"g":94},"6263":{"m":89,"g":94},"6639":{"m":89,"g":94},"6635":{"m":89,"g":94},"6620":{"m":89,"g":94},"6629":{"m":89,"g":94},"6628":{"m":89,"g":94},"6617":{"m":89,"g":94},"6306":{"m":89,"g":94},"6599":{"m":89,"g":94},"6611":{"m":89,"g":94},"6598":{"m":89,"g":94},"6597":{"m":89,"g":94},"6581":{"m":89,"g":94},"6610":{"m":89,"g":94},"6575":{"m":89,"g":94},"6609":{"m":89,"g":94},"6594":{"m":89,"g":94},"6587":{"m":89,"g":94},"6596":{"m":89,"g":94},"6595":{"m":89,"g":94},"6593":{"m":89,"g":94},"6586":{"m":89,"g":94},"6571":{"m":89,"g":94},"6439":{"m":89,"g":94},"6527":{"m":89,"g":94},"6588":{"m":89,"g":94},"6570":{"m":89,"g":94},"6546":{"m":89,"g":94},"6537":{"m":89,"g":94},"6494":{"m":89,"g":94},"5961":{"m":89,"g":94},"6543":{"m":89,"g":94},"4068":{"m":89,"g":94},"6577":{"m":89,"g":94},"6578":{"m":89,"g":94},"6477":{"m":89,"g":94},"6564":{"m":89,"g":94},"6576":{"m":89,"g":94},"7248":{"m":90,"g":94},"7244":{"m":90,"g":94},"7247":{"m":90,"g":94},"6058":{"m":90,"g":94},"7234":{"m":90,"g":94},"7245":{"m":90,"g":94},"7231":{"m":90,"g":94},"7239":{"m":90,"g":94},"7232":{"m":90,"g":94},"7207":{"m":90,"g":94},"7213":{"m":90,"g":94},"7228":{"m":90,"g":94},"7221":{"m":90,"g":94},"7218":{"m":90,"g":94},"6378":{"m":90,"g":94},"7217":{"m":90,"g":94},"7215":{"m":90,"g":94},"7214":{"m":90,"g":94},"7210":{"m":90,"g":94},"7163":{"m":90,"g":94},"7205":{"m":90,"g":94},"7204":{"m":90,"g":94},"7202":{"m":90,"g":94},"7200":{"m":90,"g":94},"7196":{"m":90,"g":94},"7198":{"m":90,"g":94},"7195":{"m":90,"g":94},"7186":{"m":90,"g":94},"7180":{"m":90,"g":94},"7189":{"m":90,"g":94},"7191":{"m":90,"g":94},"7190":{"m":90,"g":94},"7184":{"m":90,"g":94},"7181":{"m":90,"g":94},"7177":{"m":90,"g":94},"7178":{"m":90,"g":94},"7175":{"m":90,"g":94},"7172":{"m":90,"g":94},"7173":{"m":90,"g":94},"7161":{"m":90,"g":94},"7150":{"m":90,"g":94},"7153":{"m":90,"g":94},"7170":{"m":90,"g":94},"7165":{"m":90,"g":94},"7156":{"m":90,"g":94},"7020":{"m":90,"g":94},"7157":{"m":90,"g":94},"6814":{"m":90,"g":94},"7154":{"m":90,"g":94},"7155":{"m":90,"g":94},"7152":{"m":90,"g":94},"7146":{"m":90,"g":94},"7145":{"m":90,"g":94},"7056":{"m":90,"g":94},"7058":{"m":90,"g":94},"7140":{"m":90,"g":94},"7134":{"m":90,"g":94},"6026":{"m":90,"g":94},"7092":{"m":90,"g":94},"7126":{"m":90,"g":94},"7119":{"m":90,"g":94},"7115":{"m":90,"g":94},"6919":{"m":90,"g":94},"7093":{"m":90,"g":94},"6994":{"m":90,"g":94},"6824":{"m":90,"g":94},"7079":{"m":90,"g":94},"6106":{"m":90,"g":94},"7091":{"m":90,"g":94},"7097":{"m":90,"g":94},"6870":{"m":90,"g":94},"7067":{"m":90,"g":94},"7076":{"m":90,"g":94},"7046":{"m":90,"g":94},"7054":{"m":90,"g":94},"7073":{"m":90,"g":94},"6031":{"m":90,"g":94},"7071":{"m":90,"g":94},"6579":{"m":90,"g":94},"7066":{"m":90,"g":94},"6716":{"m":90,"g":94},"7064":{"m":90,"g":94},"7049":{"m":90,"g":94},"7063":{"m":90,"g":94},"6947":{"m":90,"g":94},"7057":{"m":90,"g":94},"7061":{"m":90,"g":94},"7060":{"m":90,"g":94},"7021":{"m":90,"g":94},"7053":{"m":90,"g":94},"7037":{"m":90,"g":94},"7051":{"m":90,"g":94},"6999":{"m":90,"g":94},"7045":{"m":90,"g":94},"7043":{"m":90,"g":94},"7040":{"m":90,"g":94},"7493":{"m":91,"g":94},"7490":{"m":91,"g":94},"7376":{"m":91,"g":94},"7487":{"m":91,"g":94},"7347":{"m":91,"g":94},"7269":{"m":91,"g":94},"7449":{"m":91,"g":94},"7469":{"m":91,"g":94},"7378":{"m":91,"g":94},"7481":{"m":91,"g":94},"7382":{"m":91,"g":94},"7480":{"m":91,"g":94},"7456":{"m":91,"g":94},"7479":{"m":91,"g":94},"7397":{"m":91,"g":94},"7472":{"m":91,"g":94},"7457":{"m":91,"g":94},"7290":{"m":91,"g":94},"6821":{"m":91,"g":94},"7454":{"m":91,"g":94},"7451":{"m":91,"g":94},"7445":{"m":91,"g":94},"7361":{"m":91,"g":94},"7391":{"m":91,"g":94},"7441":{"m":91,"g":94},"7327":{"m":91,"g":94},"7406":{"m":91,"g":94},"7414":{"m":91,"g":94},"7408":{"m":91,"g":94},"7412":{"m":91,"g":94},"7351":{"m":91,"g":94},"7420":{"m":91,"g":94},"7409":{"m":91,"g":94},"7425":{"m":91,"g":94},"6984":{"m":91,"g":94},"7394":{"m":91,"g":94},"7396":{"m":91,"g":94},"7400":{"m":91,"g":94},"7329":{"m":91,"g":94},"7401":{"m":91,"g":94},"7403":{"m":91,"g":94},"7402":{"m":91,"g":94},"7285":{"m":91,"g":94},"7219":{"m":91,"g":94},"7360":{"m":91,"g":94},"7399":{"m":91,"g":94},"7398":{"m":91,"g":94},"6389":{"m":91,"g":94},"7372":{"m":91,"g":94},"5485":{"m":91,"g":94},"7393":{"m":91,"g":94},"7356":{"m":91,"g":94},"7326":{"m":91,"g":94},"7322":{"m":91,"g":94},"7371":{"m":91,"g":94},"7364":{"m":91,"g":94},"7159":{"m":91,"g":94},"7370":{"m":91,"g":94},"7343":{"m":91,"g":94},"7366":{"m":91,"g":94},"7362":{"m":91,"g":94},"7242":{"m":91,"g":94},"7363":{"m":91,"g":94},"7303":{"m":91,"g":94},"7354":{"m":91,"g":94},"7099":{"m":91,"g":94},"7333":{"m":91,"g":94},"7331":{"m":91,"g":94},"7284":{"m":91,"g":94},"7096":{"m":91,"g":94},"7319":{"m":91,"g":94},"7003":{"m":91,"g":94},"7301":{"m":91,"g":94},"7297":{"m":91,"g":94},"7251":{"m":91,"g":94},"7300":{"m":91,"g":94},"6614":{"m":91,"g":94},"7267":{"m":91,"g":94},"7264":{"m":91,"g":94},"7237":{"m":91,"g":94},"7289":{"m":91,"g":94},"7288":{"m":91,"g":94},"7286":{"m":91,"g":94},"6842":{"m":91,"g":94},"7283":{"m":91,"g":94},"7164":{"m":91,"g":94},"7022":{"m":91,"g":94},"6081":{"m":91,"g":94},"7265":{"m":91,"g":94},"7160":{"m":91,"g":94},"7179":{"m":91,"g":94},"7167":{"m":91,"g":94},"7252":{"m":91,"g":94},"7122":{"m":91,"g":94},"7233":{"m":91,"g":94},"7125":{"m":91,"g":94},"7559":{"m":92,"g":94},"7541":{"m":92,"g":94},"7542":{"m":92,"g":94},"7544":{"m":92,"g":94},"7549":{"m":92,"g":94},"7543":{"m":92,"g":94},"7527":{"m":92,"g":94},"7531":{"m":92,"g":94},"7148":{"m":92,"g":94},"7499":{"m":92,"g":94},"7507":{"m":92,"g":94},"7513":{"m":92,"g":94},"7521":{"m":92,"g":94},"7520":{"m":92,"g":94},"6793":{"m":92,"g":94},"6721":{"m":92,"g":94},"7386":{"m":92,"g":94},"7522":{"m":92,"g":94},"6641":{"m":92,"g":94},"6626":{"m":92,"g":94},"7498":{"m":92,"g":94},"7512":{"m":92,"g":94},"7510":{"m":92,"g":94},"7516":{"m":92,"g":94},"7508":{"m":92,"g":94},"7437":{"m":92,"g":94},"7489":{"m":92,"g":94},"7505":{"m":92,"g":94},"6717":{"m":92,"g":94},"7277":{"m":92,"g":94},"7422":{"m":92,"g":94},"7439":{"m":92,"g":94},"7236":{"m":92,"g":94},"7423":{"m":92,"g":94},"7268":{"m":92,"g":94},"7802":{"m":93,"g":94},"7801":{"m":93,"g":94},"7799":{"m":93,"g":94},"7792":{"m":93,"g":94},"7800":{"m":93,"g":94},"7756":{"m":93,"g":94},"7790":{"m":93,"g":94},"7757":{"m":93,"g":94},"7786":{"m":93,"g":94},"7222":{"m":93,"g":94},"7787":{"m":93,"g":94},"7784":{"m":93,"g":94},"7444":{"m":93,"g":94},"7623":{"m":93,"g":94},"7782":{"m":93,"g":94},"7596":{"m":93,"g":94},"7772":{"m":93,"g":94},"7705":{"m":93,"g":94},"7745":{"m":93,"g":94},"7778":{"m":93,"g":94},"7419":{"m":93,"g":94},"7418":{"m":93,"g":94},"7748":{"m":93,"g":94},"7764":{"m":93,"g":94},"7741":{"m":93,"g":94},"7729":{"m":93,"g":94},"7390":{"m":93,"g":94},"7759":{"m":93,"g":94},"7751":{"m":93,"g":94},"7754":{"m":93,"g":94},"7755":{"m":93,"g":94},"7744":{"m":93,"g":94},"7752":{"m":93,"g":94},"7723":{"m":93,"g":94},"7673":{"m":93,"g":94},"7740":{"m":93,"g":94},"7681":{"m":93,"g":94},"6771":{"m":93,"g":94},"7647":{"m":93,"g":94},"7750":{"m":93,"g":94},"7722":{"m":93,"g":94},"7738":{"m":93,"g":94},"7278":{"m":93,"g":94},"7731":{"m":93,"g":94},"7735":{"m":93,"g":94},"6770":{"m":93,"g":94},"7734":{"m":93,"g":94},"7714":{"m":93,"g":94},"6698":{"m":93,"g":94},"7462":{"m":93,"g":94},"6549":{"m":93,"g":94},"7621":{"m":93,"g":94},"6512":{"m":93,"g":94},"7292":{"m":93,"g":94},"7416":{"m":93,"g":94},"7717":{"m":93,"g":94},"7677":{"m":93,"g":94},"7683":{"m":93,"g":94},"7697":{"m":93,"g":94},"7635":{"m":93,"g":94},"7642":{"m":93,"g":94},"7698":{"m":93,"g":94},"7684":{"m":93,"g":94},"7688":{"m":93,"g":94},"7676":{"m":93,"g":94},"7629":{"m":93,"g":94},"6985":{"m":93,"g":94},"7675":{"m":93,"g":94},"7486":{"m":93,"g":94},"7671":{"m":93,"g":94},"7648":{"m":93,"g":94},"7318":{"m":93,"g":94},"7663":{"m":93,"g":94},"7627":{"m":93,"g":94},"7632":{"m":93,"g":94},"7524":{"m":93,"g":94},"7643":{"m":93,"g":94},"7640":{"m":93,"g":94},"7539":{"m":93,"g":94},"7176":{"m":93,"g":94},"7580":{"m":93,"g":94},"7628":{"m":93,"g":94},"7619":{"m":93,"g":94},"7432":{"m":93,"g":94},"7636":{"m":93,"g":94},"7630":{"m":93,"g":94},"7624":{"m":93,"g":94},"7625":{"m":93,"g":94},"7036":{"m":93,"g":94},"7620":{"m":93,"g":94},"7310":{"m":93,"g":94},"7309":{"m":93,"g":94},"7618":{"m":93,"g":94},"7598":{"m":93,"g":94},"7446":{"m":93,"g":94},"7584":{"m":93,"g":94},"7552":{"m":93,"g":94},"7308":{"m":93,"g":94},"6769":{"m":93,"g":94},"6563":{"m":93,"g":94},"7612":{"m":93,"g":94},"7588":{"m":93,"g":94},"7610":{"m":93,"g":94},"7581":{"m":93,"g":94},"7225":{"m":93,"g":94},"7577":{"m":93,"g":94},"7540":{"m":93,"g":94},"7208":{"m":93,"g":94},"7569":{"m":93,"g":94},"7573":{"m":93,"g":94},"7575":{"m":93,"g":94},"7330":{"m":93,"g":94},"7882":{"m":95,"g":97},"7880":{"m":95,"g":97},"7660":{"m":95,"g":97},"7846":{"m":95,"g":97},"7818":{"m":95,"g":97},"7866":{"m":95,"g":97},"7724":{"m":95,"g":97},"7830":{"m":95,"g":97},"7579":{"m":95,"g":97},"7840":{"m":95,"g":97},"7864":{"m":95,"g":97},"7860":{"m":95,"g":97},"7832":{"m":95,"g":97},"7853":{"m":95,"g":97},"7850":{"m":95,"g":97},"7129":{"m":95,"g":97},"7762":{"m":95,"g":97},"7821":{"m":95,"g":97},"7187":{"m":95,"g":97},"7816":{"m":95,"g":97},"7798":{"m":95,"g":94},"7797":{"m":95,"g":94},"7313":{"m":95,"g":94},"7689":{"m":95,"g":94},"7813":{"m":95,"g":94},"6094":{"m":95,"g":94},"7794":{"m":95,"g":94},"7812":{"m":95,"g":94},"7733":{"m":95,"g":94},"5246":{"m":95,"g":94},"7785":{"m":95,"g":94},"7709":{"m":95,"g":94},"7793":{"m":95,"g":94},"7803":{"m":95,"g":94},"7796":{"m":95,"g":94},"7963":{"m":96,"g":97},"7971":{"m":96,"g":97},"7960":{"m":96,"g":97},"7969":{"m":96,"g":97},"7970":{"m":96,"g":97},"7962":{"m":96,"g":97},"7968":{"m":96,"g":97},"7964":{"m":96,"g":97},"7932":{"m":96,"g":97},"7961":{"m":96,"g":97},"7953":{"m":96,"g":97},"7795":{"m":96,"g":97},"7940":{"m":96,"g":97},"7775":{"m":96,"g":97},"7791":{"m":96,"g":97},"6449":{"m":96,"g":97},"7922":{"m":96,"g":97},"7907":{"m":96,"g":97},"5888":{"m":96,"g":97},"7904":{"m":96,"g":97},"7608":{"m":96,"g":97},"7899":{"m":96,"g":97},"7898":{"m":96,"g":97},"7838":{"m":96,"g":97},"7885":{"m":96,"g":97},"7872":{"m":96,"g":97},"7895":{"m":96,"g":97},"8265":{"m":98,"g":103},"8260":{"m":98,"g":103},"7822":{"m":98,"g":103},"8059":{"m":98,"g":103},"8257":{"m":98,"g":103},"7484":{"m":98,"g":103},"8221":{"m":98,"g":103},"8237":{"m":98,"g":103},"8231":{"m":98,"g":103},"8202":{"m":98,"g":103},"8204":{"m":98,"g":103},"8209":{"m":98,"g":97},"8208":{"m":98,"g":97},"8107":{"m":98,"g":97},"8193":{"m":98,"g":97},"8200":{"m":98,"g":97},"8195":{"m":98,"g":97},"7935":{"m":98,"g":97},"8184":{"m":98,"g":97},"8197":{"m":98,"g":97},"8067":{"m":98,"g":97},"8163":{"m":98,"g":97},"8183":{"m":98,"g":97},"7983":{"m":98,"g":97},"8182":{"m":98,"g":97},"8181":{"m":98,"g":97},"7825":{"m":98,"g":97},"8178":{"m":98,"g":97},"8176":{"m":98,"g":97},"7312":{"m":98,"g":97},"6230":{"m":98,"g":97},"7999":{"m":98,"g":97},"8115":{"m":98,"g":97},"8175":{"m":98,"g":97},"8172":{"m":98,"g":97},"8019":{"m":98,"g":97},"8167":{"m":98,"g":97},"8170":{"m":98,"g":97},"8169":{"m":98,"g":97},"8103":{"m":98,"g":97},"8161":{"m":98,"g":97},"8171":{"m":98,"g":97},"8168":{"m":98,"g":97},"8166":{"m":98,"g":97},"8165":{"m":98,"g":97},"7966":{"m":98,"g":97},"8157":{"m":98,"g":97},"8160":{"m":98,"g":97},"8158":{"m":98,"g":97},"8028":{"m":98,"g":97},"7931":{"m":98,"g":97},"8048":{"m":98,"g":97},"7302":{"m":98,"g":97},"6881":{"m":98,"g":97},"8155":{"m":98,"g":97},"7661":{"m":98,"g":97},"7987":{"m":98,"g":97},"8113":{"m":98,"g":97},"8147":{"m":98,"g":97},"8142":{"m":98,"g":97},"8136":{"m":98,"g":97},"8141":{"m":98,"g":97},"7704":{"m":98,"g":97},"7820":{"m":98,"g":97},"7889":{"m":98,"g":97},"7506":{"m":98,"g":97},"7959":{"m":98,"g":97},"8127":{"m":98,"g":97},"7924":{"m":98,"g":97},"7030":{"m":98,"g":97},"8117":{"m":98,"g":97},"8102":{"m":98,"g":97},"8046":{"m":98,"g":97},"7884":{"m":98,"g":97},"7989":{"m":98,"g":97},"8105":{"m":98,"g":97},"8110":{"m":98,"g":97},"8108":{"m":98,"g":97},"8100":{"m":98,"g":97},"7597":{"m":98,"g":97},"8075":{"m":98,"g":97},"7992":{"m":98,"g":97},"7634":{"m":98,"g":97},"8098":{"m":98,"g":97},"8090":{"m":98,"g":97},"8086":{"m":98,"g":97},"8077":{"m":98,"g":97},"8001":{"m":98,"g":97},"7760":{"m":98,"g":97},"8029":{"m":98,"g":97},"8058":{"m":98,"g":97},"5163":{"m":98,"g":97},"8045":{"m":98,"g":97},"8047":{"m":98,"g":97},"7943":{"m":98,"g":97},"8052":{"m":98,"g":97},"6556":{"m":98,"g":97},"8022":{"m":98,"g":97},"8002":{"m":98,"g":97},"8023":{"m":98,"g":97},"8044":{"m":98,"g":97},"7887":{"m":98,"g":97},"8035":{"m":98,"g":97},"7897":{"m":98,"g":97},"7982":{"m":98,"g":97},"8006":{"m":98,"g":97},"7649":{"m":98,"g":97},"7653":{"m":98,"g":97},"8005":{"m":98,"g":97},"7874":{"m":98,"g":97},"8021":{"m":98,"g":97},"8010":{"m":98,"g":97},"7902":{"m":98,"g":97},"7862":{"m":98,"g":97},"7844":{"m":98,"g":97},"7997":{"m":98,"g":97},"7367":{"m":98,"g":97},"7749":{"m":98,"g":97},"7952":{"m":98,"g":97},"7988":{"m":98,"g":97},"7814":{"m":98,"g":97},"7978":{"m":98,"g":97},"7985":{"m":98,"g":97},"7975":{"m":98,"g":97},"7972":{"m":98,"g":97},"7950":{"m":98,"g":97},"8305":{"m":99,"g":103},"8370":{"m":99,"g":103},"8333":{"m":99,"g":103},"8367":{"m":99,"g":103},"8363":{"m":99,"g":103},"8359":{"m":99,"g":103},"8332":{"m":99,"g":103},"8357":{"m":99,"g":103},"8344":{"m":99,"g":103},"7858":{"m":99,"g":103},"8353":{"m":99,"g":103},"8341":{"m":99,"g":103},"8000":{"m":99,"g":103},"7135":{"m":99,"g":103},"8266":{"m":99,"g":103},"8280":{"m":99,"g":103},"6619":{"m":99,"g":103},"8233":{"m":99,"g":103},"8334":{"m":99,"g":103},"8307":{"m":99,"g":103},"8300":{"m":99,"g":103},"8299":{"m":99,"g":103},"8301":{"m":99,"g":103},"8310":{"m":99,"g":103},"8298":{"m":99,"g":103},"8315":{"m":99,"g":103},"8303":{"m":99,"g":103},"8317":{"m":99,"g":103},"8235":{"m":99,"g":103},"8070":{"m":99,"g":103},"7562":{"m":99,"g":103},"8043":{"m":99,"g":103},"7685":{"m":99,"g":103},"8304":{"m":99,"g":103},"8240":{"m":99,"g":103},"8262":{"m":99,"g":103},"7708":{"m":99,"g":103},"8133":{"m":99,"g":103},"8302":{"m":99,"g":103},"8295":{"m":99,"g":103},"8130":{"m":99,"g":103},"8288":{"m":99,"g":103},"8282":{"m":99,"g":103},"8264":{"m":99,"g":103},"8261":{"m":99,"g":103},"8284":{"m":99,"g":103},"8272":{"m":99,"g":103},"8458":{"m":100,"g":103},"8457":{"m":100,"g":103},"8449":{"m":100,"g":103},"8456":{"m":100,"g":103},"8445":{"m":100,"g":103},"8441":{"m":100,"g":103},"8442":{"m":100,"g":103},"8224":{"m":100,"g":103},"8352":{"m":100,"g":103},"8416":{"m":100,"g":103},"6338":{"m":100,"g":103},"8415":{"m":100,"g":103},"8422":{"m":100,"g":103},"8425":{"m":100,"g":103},"8419":{"m":100,"g":103},"8417":{"m":100,"g":103},"8316":{"m":100,"g":103},"7603":{"m":100,"g":103},"8213":{"m":100,"g":103},"8414":{"m":100,"g":103},"8062":{"m":100,"g":103},"8258":{"m":100,"g":103},"8406":{"m":100,"g":103},"8407":{"m":100,"g":103},"8405":{"m":100,"g":103},"8156":{"m":100,"g":103},"8241":{"m":100,"g":103},"8397":{"m":100,"g":103},"7720":{"m":100,"g":103},"8351":{"m":100,"g":103},"8395":{"m":100,"g":103},"8382":{"m":100,"g":103},"8392":{"m":100,"g":103},"8036":{"m":100,"g":103},"7739":{"m":100,"g":103},"7974":{"m":100,"g":103},"7976":{"m":100,"g":103},"8403":{"m":100,"g":103},"8401":{"m":100,"g":103},"8372":{"m":100,"g":103},"8394":{"m":100,"g":103},"8396":{"m":100,"g":103},"8314":{"m":100,"g":103},"8350":{"m":100,"g":103},"8381":{"m":100,"g":103},"6003":{"m":100,"g":103},"7737":{"m":100,"g":103},"8267":{"m":100,"g":103},"8343":{"m":100,"g":103},"7000":{"m":100,"g":103},"8356":{"m":100,"g":103},"8374":{"m":100,"g":103},"8517":{"m":101,"g":103},"8489":{"m":101,"g":103},"8482":{"m":101,"g":103},"7973":{"m":101,"g":103},"8426":{"m":101,"g":103},"8413":{"m":101,"g":103},"8486":{"m":101,"g":103},"8485":{"m":101,"g":103},"8477":{"m":101,"g":103},"8480":{"m":101,"g":103},"8478":{"m":101,"g":103},"8476":{"m":101,"g":103},"8469":{"m":101,"g":103},"8473":{"m":101,"g":103},"8421":{"m":101,"g":103},"7273":{"m":101,"g":103},"8453":{"m":101,"g":103},"8467":{"m":101,"g":103},"7565":{"m":101,"g":103},"8465":{"m":101,"g":103},"8125":{"m":101,"g":103},"8608":{"m":102,"g":103},"8590":{"m":102,"g":103},"8583":{"m":102,"g":103},"8604":{"m":102,"g":103},"8515":{"m":102,"g":103},"8603":{"m":102,"g":103},"8550":{"m":102,"g":103},"7211":{"m":102,"g":103},"8533":{"m":102,"g":103},"8404":{"m":102,"g":103},"8599":{"m":102,"g":103},"8514":{"m":102,"g":103},"8365":{"m":102,"g":103},"8544":{"m":102,"g":103},"8541":{"m":102,"g":103},"8564":{"m":102,"g":103},"8479":{"m":102,"g":103},"7280":{"m":102,"g":103},"8584":{"m":102,"g":103},"8154":{"m":102,"g":103},"8461":{"m":102,"g":103},"6869":{"m":102,"g":103},"8562":{"m":102,"g":103},"8545":{"m":102,"g":103},"8560":{"m":102,"g":103},"8516":{"m":102,"g":103},"8498":{"m":102,"g":103},"8448":{"m":102,"g":103},"8431":{"m":102,"g":103},"8537":{"m":102,"g":103},"8483":{"m":102,"g":103},"8531":{"m":102,"g":103},"8535":{"m":102,"g":103},"8499":{"m":102,"g":103},"8528":{"m":102,"g":103},"8527":{"m":102,"g":103},"8652":{"m":105,"g":107},"8051":{"m":105,"g":107},"8318":{"m":105,"g":107},"8636":{"m":105,"g":107},"8450":{"m":105,"g":107},"8645":{"m":105,"g":104},"8640":{"m":105,"g":104},"8644":{"m":105,"g":104},"8308":{"m":105,"g":104},"8270":{"m":105,"g":104},"8083":{"m":105,"g":104},"8642":{"m":105,"g":104},"8532":{"m":105,"g":104},"8632":{"m":105,"g":104},"8598":{"m":105,"g":104},"8630":{"m":105,"g":104},"8634":{"m":105,"g":104},"8488":{"m":105,"g":104},"8633":{"m":105,"g":104},"6227":{"m":105,"g":104},"8577":{"m":105,"g":104},"8628":{"m":105,"g":103},"8629":{"m":105,"g":103},"8626":{"m":105,"g":103},"8623":{"m":105,"g":103},"8611":{"m":105,"g":103},"8595":{"m":105,"g":103},"8727":{"m":106,"g":107},"8723":{"m":106,"g":107},"8579":{"m":106,"g":107},"8567":{"m":106,"g":107},"8718":{"m":106,"g":107},"8444":{"m":106,"g":107},"8547":{"m":106,"g":107},"8683":{"m":106,"g":107},"8631":{"m":106,"g":107},"7379":{"m":106,"g":107},"8719":{"m":106,"g":107},"8306":{"m":106,"g":107},"8650":{"m":106,"g":107},"8721":{"m":106,"g":107},"8524":{"m":106,"g":107},"8709":{"m":106,"g":107},"8722":{"m":106,"g":107},"7369":{"m":106,"g":107},"8714":{"m":106,"g":107},"8705":{"m":106,"g":107},"8693":{"m":106,"g":107},"8717":{"m":106,"g":107},"8713":{"m":106,"g":107},"8701":{"m":106,"g":107},"8711":{"m":106,"g":107},"8706":{"m":106,"g":107},"8704":{"m":106,"g":107},"8691":{"m":106,"g":107},"8512":{"m":106,"g":107},"7434":{"m":106,"g":107},"8688":{"m":106,"g":107},"8694":{"m":106,"g":107},"8364":{"m":106,"g":107},"8238":{"m":106,"g":107},"8618":{"m":106,"g":107},"8522":{"m":106,"g":107},"8668":{"m":106,"g":107},"8648":{"m":106,"g":107},"8679":{"m":106,"g":107},"8686":{"m":106,"g":107},"8684":{"m":106,"g":107},"8685":{"m":106,"g":107},"8647":{"m":106,"g":107},"8094":{"m":106,"g":107},"8664":{"m":106,"g":107},"8543":{"m":106,"g":107},"8665":{"m":106,"g":107},"8013":{"m":106,"g":107},"8643":{"m":106,"g":107},"8658":{"m":106,"g":107},"8511":{"m":106,"g":107},"8635":{"m":106,"g":107},"8653":{"m":106,"g":107},"9533":{"m":108,"g":115},"9532":{"m":108,"g":115},"9372":{"m":108,"g":115},"9485":{"m":108,"g":115},"9478":{"m":108,"g":115},"8034":{"m":108,"g":115},"9473":{"m":108,"g":115},"9525":{"m":108,"g":115},"9530":{"m":108,"g":115},"8946":{"m":108,"g":115},"9004":{"m":108,"g":115},"9241":{"m":108,"g":115},"9211":{"m":108,"g":115},"9503":{"m":108,"g":115},"9519":{"m":108,"g":115},"9456":{"m":108,"g":115},"7699":{"m":108,"g":115},"9200":{"m":108,"g":115},"9516":{"m":108,"g":115},"9127":{"m":108,"g":115},"9513":{"m":108,"g":115},"8624":{"m":108,"g":115},"8865":{"m":108,"g":115},"9109":{"m":108,"g":115},"9452":{"m":108,"g":115},"9507":{"m":108,"g":115},"9303":{"m":108,"g":115},"9331":{"m":108,"g":115},"9497":{"m":108,"g":115},"9494":{"m":108,"g":115},"9475":{"m":108,"g":115},"9480":{"m":108,"g":115},"9487":{"m":108,"g":115},"9491":{"m":108,"g":115},"9482":{"m":108,"g":115},"9492":{"m":108,"g":115},"9483":{"m":108,"g":115},"9333":{"m":108,"g":115},"9474":{"m":108,"g":115},"8616":{"m":108,"g":115},"9356":{"m":108,"g":115},"8593":{"m":108,"g":115},"9468":{"m":108,"g":115},"9467":{"m":108,"g":115},"9470":{"m":108,"g":115},"9469":{"m":108,"g":115},"9455":{"m":108,"g":115},"9463":{"m":108,"g":115},"9464":{"m":108,"g":115},"9462":{"m":108,"g":115},"9461":{"m":108,"g":115},"9458":{"m":108,"g":115},"9454":{"m":108,"g":115},"9427":{"m":108,"g":115},"9433":{"m":108,"g":115},"7604":{"m":108,"g":115},"8521":{"m":108,"g":115},"9392":{"m":108,"g":115},"9395":{"m":108,"g":115},"9238":{"m":108,"g":115},"9384":{"m":108,"g":115},"9430":{"m":108,"g":115},"9346":{"m":108,"g":115},"9399":{"m":108,"g":115},"9251":{"m":108,"g":115},"9388":{"m":108,"g":115},"9261":{"m":108,"g":115},"9420":{"m":108,"g":115},"9416":{"m":108,"g":115},"9415":{"m":108,"g":115},"9413":{"m":108,"g":115},"9371":{"m":108,"g":115},"9339":{"m":108,"g":115},"9357":{"m":108,"g":115},"9377":{"m":108,"g":115},"9381":{"m":108,"g":115},"9404":{"m":108,"g":115},"9359":{"m":108,"g":115},"9336":{"m":108,"g":115},"9249":{"m":108,"g":115},"9409":{"m":108,"g":115},"8690":{"m":108,"g":115},"9278":{"m":108,"g":115},"9391":{"m":108,"g":115},"9106":{"m":108,"g":115},"7375":{"m":108,"g":115},"9385":{"m":108,"g":115},"9383":{"m":108,"g":115},"9378":{"m":108,"g":115},"9380":{"m":108,"g":115},"9376":{"m":108,"g":115},"9350":{"m":108,"g":115},"9368":{"m":108,"g":115},"9344":{"m":108,"g":115},"9369":{"m":108,"g":115},"9367":{"m":108,"g":115},"9370":{"m":108,"g":115},"9364":{"m":108,"g":115},"9360":{"m":108,"g":115},"9335":{"m":108,"g":115},"9361":{"m":108,"g":115},"9354":{"m":108,"g":115},"6295":{"m":108,"g":115},"9353":{"m":108,"g":115},"9348":{"m":108,"g":115},"9327":{"m":108,"g":115},"9332":{"m":108,"g":115},"9326":{"m":108,"g":115},"8990":{"m":108,"g":115},"7019":{"m":108,"g":115},"9321":{"m":108,"g":115},"9317":{"m":108,"g":115},"9299":{"m":108,"g":115},"9322":{"m":108,"g":115},"9306":{"m":108,"g":115},"9320":{"m":108,"g":115},"9059":{"m":108,"g":115},"8936":{"m":108,"g":115},"9284":{"m":108,"g":115},"9313":{"m":108,"g":115},"9316":{"m":108,"g":115},"9315":{"m":108,"g":115},"8829":{"m":108,"g":115},"9011":{"m":108,"g":115},"9289":{"m":108,"g":115},"9310":{"m":108,"g":115},"9307":{"m":108,"g":115},"9298":{"m":108,"g":115},"9276":{"m":108,"g":115},"9293":{"m":108,"g":115},"6307":{"m":108,"g":115},"9245":{"m":108,"g":115},"9287":{"m":108,"g":115},"9286":{"m":108,"g":115},"8289":{"m":108,"g":115},"9281":{"m":108,"g":115},"8520":{"m":108,"g":115},"9272":{"m":108,"g":115},"9271":{"m":108,"g":115},"9279":{"m":108,"g":115},"9131":{"m":108,"g":115},"9242":{"m":108,"g":115},"9260":{"m":108,"g":115},"9268":{"m":108,"g":115},"9067":{"m":108,"g":115},"9264":{"m":108,"g":115},"9237":{"m":108,"g":115},"9232":{"m":108,"g":115},"9006":{"m":108,"g":115},"8893":{"m":108,"g":115},"9049":{"m":108,"g":115},"8846":{"m":108,"g":115},"9252":{"m":108,"g":115},"8027":{"m":108,"g":115},"9258":{"m":108,"g":115},"7758":{"m":108,"g":115},"9165":{"m":108,"g":115},"7667":{"m":108,"g":115},"9247":{"m":108,"g":115},"9246":{"m":108,"g":115},"8663":{"m":108,"g":115},"8268":{"m":108,"g":115},"9243":{"m":108,"g":115},"9236":{"m":108,"g":115},"9201":{"m":108,"g":115},"9198":{"m":108,"g":115},"9231":{"m":108,"g":115},"9220":{"m":108,"g":115},"9223":{"m":108,"g":115},"9222":{"m":108,"g":115},"8777":{"m":108,"g":115},"8790":{"m":108,"g":115},"9215":{"m":108,"g":115},"9218":{"m":108,"g":115},"9214":{"m":108,"g":115},"9208":{"m":108,"g":115},"9213":{"m":108,"g":115},"9207":{"m":108,"g":115},"9177":{"m":108,"g":115},"8849":{"m":108,"g":115},"9206":{"m":108,"g":115},"9205":{"m":108,"g":115},"9183":{"m":108,"g":115},"9204":{"m":108,"g":115},"9203":{"m":108,"g":115},"9202":{"m":108,"g":115},"9197":{"m":108,"g":115},"9008":{"m":108,"g":115},"8795":{"m":108,"g":115},"9191":{"m":108,"g":115},"9194":{"m":108,"g":115},"9060":{"m":108,"g":115},"8913":{"m":108,"g":115},"9185":{"m":108,"g":115},"8112":{"m":108,"g":115},"9065":{"m":108,"g":115},"8018":{"m":108,"g":115},"7687":{"m":108,"g":115},"7631":{"m":108,"g":115},"7004":{"m":108,"g":115},"8852":{"m":108,"g":115},"8808":{"m":108,"g":115},"8818":{"m":108,"g":115},"9154":{"m":108,"g":115},"9101":{"m":108,"g":115},"9162":{"m":108,"g":115},"9136":{"m":108,"g":115},"9171":{"m":108,"g":115},"9169":{"m":108,"g":115},"9159":{"m":108,"g":115},"8951":{"m":108,"g":115},"9161":{"m":108,"g":115},"8840":{"m":108,"g":115},"9134":{"m":108,"g":115},"9042":{"m":108,"g":115},"7957":{"m":108,"g":115},"9069":{"m":108,"g":115},"9028":{"m":108,"g":115},"8910":{"m":108,"g":115},"9149":{"m":108,"g":115},"9133":{"m":108,"g":115},"9126":{"m":108,"g":115},"9150":{"m":108,"g":115},"8484":{"m":108,"g":115},"9111":{"m":108,"g":115},"9146":{"m":108,"g":115},"9093":{"m":108,"g":115},"9088":{"m":108,"g":115},"8588":{"m":108,"g":115},"9137":{"m":108,"g":115},"8884":{"m":108,"g":115},"8651":{"m":108,"g":115},"9130":{"m":108,"g":115},"9129":{"m":108,"g":115},"8660":{"m":108,"g":115},"9119":{"m":108,"g":115},"8619":{"m":108,"g":115},"8610":{"m":108,"g":115},"8700":{"m":108,"g":115},"9125":{"m":108,"g":115},"9121":{"m":108,"g":115},"9014":{"m":108,"g":115},"9118":{"m":108,"g":115},"9122":{"m":108,"g":115},"9107":{"m":108,"g":115},"9113":{"m":108,"g":115},"9114":{"m":108,"g":115},"9021":{"m":108,"g":115},"9103":{"m":108,"g":115},"9077":{"m":108,"g":115},"9096":{"m":108,"g":115},"9005":{"m":108,"g":115},"9075":{"m":108,"g":115},"9097":{"m":108,"g":115},"9032":{"m":108,"g":115},"8766":{"m":108,"g":115},"9087":{"m":108,"g":115},"8293":{"m":108,"g":115},"9095":{"m":108,"g":115},"9084":{"m":108,"g":115},"8992":{"m":108,"g":115},"9089":{"m":108,"g":115},"9043":{"m":108,"g":115},"9086":{"m":108,"g":115},"9030":{"m":108,"g":115},"9053":{"m":108,"g":115},"8638":{"m":108,"g":115},"8731":{"m":108,"g":115},"9083":{"m":108,"g":115},"8752":{"m":108,"g":115},"9081":{"m":108,"g":115},"9082":{"m":108,"g":115},"8866":{"m":108,"g":115},"9080":{"m":108,"g":115},"8973":{"m":108,"g":115},"9063":{"m":108,"g":115},"9079":{"m":108,"g":115},"9066":{"m":108,"g":115},"7216":{"m":108,"g":115},"9051":{"m":108,"g":115},"9047":{"m":108,"g":115},"9057":{"m":108,"g":115},"9050":{"m":108,"g":115},"9054":{"m":108,"g":115},"9048":{"m":108,"g":115},"8997":{"m":108,"g":115},"9046":{"m":108,"g":115},"9044":{"m":108,"g":115},"9031":{"m":108,"g":115},"9037":{"m":108,"g":115},"9036":{"m":108,"g":115},"9034":{"m":108,"g":115},"9035":{"m":108,"g":115},"9033":{"m":108,"g":115},"8079":{"m":108,"g":115},"8794":{"m":108,"g":115},"9024":{"m":108,"g":115},"9027":{"m":108,"g":115},"9029":{"m":108,"g":115},"7626":{"m":108,"g":115},"9022":{"m":108,"g":115},"8940":{"m":108,"g":115},"8996":{"m":108,"g":115},"9018":{"m":108,"g":115},"9017":{"m":108,"g":115},"9019":{"m":108,"g":115},"8340":{"m":108,"g":115},"8991":{"m":108,"g":115},"8915":{"m":108,"g":115},"8245":{"m":108,"g":115},"9013":{"m":108,"g":115},"9007":{"m":108,"g":115},"9012":{"m":108,"g":115},"9003":{"m":108,"g":115},"9010":{"m":108,"g":115},"9001":{"m":108,"g":115},"8329":{"m":108,"g":115},"8355":{"m":108,"g":115},"8877":{"m":108,"g":115},"8995":{"m":108,"g":115},"8998":{"m":108,"g":115},"8878":{"m":108,"g":115},"6752":{"m":108,"g":115},"8673":{"m":108,"g":115},"8798":{"m":108,"g":115},"8687":{"m":108,"g":115},"8600":{"m":108,"g":115},"8966":{"m":108,"g":115},"8851":{"m":108,"g":115},"8984":{"m":108,"g":115},"8962":{"m":108,"g":115},"8987":{"m":108,"g":115},"8994":{"m":108,"g":115},"8993":{"m":108,"g":115},"8989":{"m":108,"g":115},"8667":{"m":108,"g":115},"8983":{"m":108,"g":115},"8980":{"m":108,"g":115},"8330":{"m":108,"g":115},"8770":{"m":108,"g":115},"8724":{"m":108,"g":115},"8988":{"m":108,"g":115},"8986":{"m":108,"g":115},"8785":{"m":108,"g":115},"8371":{"m":108,"g":115},"8982":{"m":108,"g":115},"8978":{"m":108,"g":115},"8981":{"m":108,"g":115},"8772":{"m":108,"g":115},"8971":{"m":108,"g":115},"8968":{"m":108,"g":115},"8972":{"m":108,"g":115},"7279":{"m":108,"g":115},"8941":{"m":108,"g":115},"8959":{"m":108,"g":115},"8757":{"m":108,"g":115},"8960":{"m":108,"g":115},"8958":{"m":108,"g":115},"8692":{"m":108,"g":115},"6555":{"m":108,"g":115},"8894":{"m":108,"g":115},"8957":{"m":108,"g":115},"8955":{"m":108,"g":115},"7657":{"m":108,"g":115},"8944":{"m":108,"g":115},"8799":{"m":108,"g":115},"8932":{"m":108,"g":115},"8952":{"m":108,"g":115},"8953":{"m":108,"g":115},"8947":{"m":108,"g":115},"8950":{"m":108,"g":115},"8720":{"m":108,"g":115},"8703":{"m":108,"g":115},"8923":{"m":108,"g":115},"8850":{"m":108,"g":115},"8933":{"m":108,"g":115},"8937":{"m":108,"g":115},"8929":{"m":108,"g":115},"8928":{"m":108,"g":115},"8925":{"m":108,"g":115},"8927":{"m":108,"g":115},"8908":{"m":108,"g":115},"8844":{"m":108,"g":107},"8916":{"m":108,"g":107},"8912":{"m":108,"g":107},"8698":{"m":108,"g":107},"8869":{"m":108,"g":107},"8898":{"m":108,"g":107},"8895":{"m":108,"g":107},"8041":{"m":108,"g":107},"5949":{"m":108,"g":107},"8888":{"m":108,"g":107},"8292":{"m":108,"g":107},"8787":{"m":108,"g":107},"8369":{"m":108,"g":107},"8697":{"m":108,"g":107},"8834":{"m":108,"g":107},"8883":{"m":108,"g":107},"8847":{"m":108,"g":107},"8539":{"m":108,"g":107},"8837":{"m":108,"g":107},"8880":{"m":108,"g":107},"8881":{"m":108,"g":107},"8811":{"m":108,"g":107},"8872":{"m":108,"g":107},"8861":{"m":108,"g":107},"8860":{"m":108,"g":107},"8868":{"m":108,"g":107},"8815":{"m":108,"g":107},"8859":{"m":108,"g":107},"8853":{"m":108,"g":107},"8843":{"m":108,"g":107},"8753":{"m":108,"g":107},"8751":{"m":108,"g":107},"8838":{"m":108,"g":107},"8144":{"m":108,"g":107},"8680":{"m":108,"g":107},"8839":{"m":108,"g":107},"8828":{"m":108,"g":107},"8836":{"m":108,"g":107},"8824":{"m":108,"g":107},"8809":{"m":108,"g":107},"8832":{"m":108,"g":107},"8681":{"m":108,"g":107},"8827":{"m":108,"g":107},"8823":{"m":108,"g":107},"8817":{"m":108,"g":107},"8804":{"m":108,"g":107},"8782":{"m":108,"g":107},"8802":{"m":108,"g":107},"8800":{"m":108,"g":107},"8797":{"m":108,"g":107},"8596":{"m":108,"g":107},"8779":{"m":108,"g":107},"8780":{"m":108,"g":107},"8744":{"m":108,"g":107},"8571":{"m":108,"g":107},"8212":{"m":108,"g":107},"8255":{"m":108,"g":107},"8776":{"m":108,"g":107},"8773":{"m":108,"g":107},"8771":{"m":108,"g":107},"8762":{"m":108,"g":107},"8768":{"m":108,"g":107},"8639":{"m":108,"g":107},"8552":{"m":108,"g":107},"8749":{"m":108,"g":107},"8294":{"m":108,"g":107},"8738":{"m":108,"g":107},"8437":{"m":108,"g":107},"8745":{"m":108,"g":107},"8733":{"m":108,"g":107},"8735":{"m":108,"g":107},"8737":{"m":108,"g":107},"8662":{"m":108,"g":107},"7114":{"m":108,"g":107},"8678":{"m":108,"g":107},"8732":{"m":108,"g":107},"8729":{"m":108,"g":107},"8676":{"m":108,"g":107},"8699":{"m":108,"g":107},"9558":{"m":109,"g":115},"9557":{"m":109,"g":115},"9549":{"m":109,"g":115},"9544":{"m":109,"g":115},"9547":{"m":109,"g":115},"9546":{"m":109,"g":115},"9592":{"m":110,"g":115},"9591":{"m":110,"g":115},"9589":{"m":110,"g":115},"9587":{"m":110,"g":115},"9581":{"m":110,"g":115},"9578":{"m":110,"g":115},"9229":{"m":110,"g":115},"9536":{"m":110,"g":115},"9535":{"m":110,"g":115},"9559":{"m":110,"g":115},"9560":{"m":110,"g":115},"7317":{"m":110,"g":115},"9576":{"m":110,"g":115},"9429":{"m":110,"g":115},"9565":{"m":110,"g":115},"9498":{"m":110,"g":115},"9716":{"m":111,"g":115},"9708":{"m":111,"g":115},"9340":{"m":111,"g":115},"9703":{"m":111,"g":115},"9702":{"m":111,"g":115},"9695":{"m":111,"g":115},"9683":{"m":111,"g":115},"9700":{"m":111,"g":115},"9676":{"m":111,"g":115},"9694":{"m":111,"g":115},"9693":{"m":111,"g":115},"9679":{"m":111,"g":115},"9678":{"m":111,"g":115},"9397":{"m":111,"g":115},"9495":{"m":111,"g":115},"9677":{"m":111,"g":115},"9446":{"m":111,"g":115},"9071":{"m":111,"g":115},"9597":{"m":111,"g":115},"9555":{"m":111,"g":115},"9583":{"m":111,"g":115},"9564":{"m":111,"g":115},"9658":{"m":111,"g":115},"9665":{"m":111,"g":115},"9648":{"m":111,"g":115},"9637":{"m":111,"g":115},"9649":{"m":111,"g":115},"9647":{"m":111,"g":115},"9656":{"m":111,"g":115},"9523":{"m":111,"g":115},"9606":{"m":111,"g":115},"9635":{"m":111,"g":115},"9630":{"m":111,"g":115},"9640":{"m":111,"g":115},"9636":{"m":111,"g":115},"9301":{"m":111,"g":115},"8328":{"m":111,"g":115},"9632":{"m":111,"g":115},"9629":{"m":111,"g":115},"9628":{"m":111,"g":115},"9623":{"m":111,"g":115},"9622":{"m":111,"g":115},"9608":{"m":111,"g":115},"8901":{"m":111,"g":115},"9613":{"m":111,"g":115},"9190":{"m":111,"g":115},"9554":{"m":111,"g":115},"9500":{"m":111,"g":115},"9436":{"m":111,"g":115},"9568":{"m":111,"g":115},"10221":{"m":112,"g":115},"10340":{"m":112,"g":115},"10303":{"m":112,"g":115},"10331":{"m":112,"g":115},"10339":{"m":112,"g":115},"10330":{"m":112,"g":115},"10327":{"m":112,"g":115},"10338":{"m":112,"g":115},"10254":{"m":112,"g":115},"10264":{"m":112,"g":115},"10280":{"m":112,"g":115},"10335":{"m":112,"g":115},"10322":{"m":112,"g":115},"10328":{"m":112,"g":115},"10326":{"m":112,"g":115},"10233":{"m":112,"g":115},"10297":{"m":112,"g":115},"10314":{"m":112,"g":115},"10311":{"m":112,"g":115},"10310":{"m":112,"g":115},"10299":{"m":112,"g":115},"9090":{"m":112,"g":115},"10229":{"m":112,"g":115},"10239":{"m":112,"g":115},"9881":{"m":112,"g":115},"10294":{"m":112,"g":115},"10292":{"m":112,"g":115},"10282":{"m":112,"g":115},"10184":{"m":112,"g":115},"10241":{"m":112,"g":115},"9662":{"m":112,"g":115},"10252":{"m":112,"g":115},"9940":{"m":112,"g":115},"10251":{"m":112,"g":115},"10256":{"m":112,"g":115},"10250":{"m":112,"g":115},"10262":{"m":112,"g":115},"9954":{"m":112,"g":115},"10173":{"m":112,"g":115},"10060":{"m":112,"g":115},"10253":{"m":112,"g":115},"10093":{"m":112,"g":115},"10240":{"m":112,"g":115},"8803":{"m":112,"g":115},"10246":{"m":112,"g":115},"9795":{"m":112,"g":115},"10245":{"m":112,"g":115},"10242":{"m":112,"g":115},"10236":{"m":112,"g":115},"10234":{"m":112,"g":115},"10213":{"m":112,"g":115},"10238":{"m":112,"g":115},"10210":{"m":112,"g":115},"10220":{"m":112,"g":115},"10214":{"m":112,"g":115},"9960":{"m":112,"g":115},"10208":{"m":112,"g":115},"10212":{"m":112,"g":115},"10209":{"m":112,"g":115},"10207":{"m":112,"g":115},"10205":{"m":112,"g":115},"9300":{"m":112,"g":115},"10193":{"m":112,"g":115},"10127":{"m":112,"g":115},"10188":{"m":112,"g":115},"10165":{"m":112,"g":115},"10169":{"m":112,"g":115},"4422":{"m":112,"g":115},"9900":{"m":112,"g":115},"10191":{"m":112,"g":115},"7995":{"m":112,"g":115},"10185":{"m":112,"g":115},"10149":{"m":112,"g":115},"9522":{"m":112,"g":115},"10182":{"m":112,"g":115},"10181":{"m":112,"g":115},"9839":{"m":112,"g":115},"10176":{"m":112,"g":115},"9595":{"m":112,"g":115},"10166":{"m":112,"g":115},"9925":{"m":112,"g":115},"10156":{"m":112,"g":115},"10161":{"m":112,"g":115},"10159":{"m":112,"g":115},"10131":{"m":112,"g":115},"10028":{"m":112,"g":115},"10155":{"m":112,"g":115},"9871":{"m":112,"g":115},"9434":{"m":112,"g":115},"10148":{"m":112,"g":115},"6226":{"m":112,"g":115},"10013":{"m":112,"g":115},"10147":{"m":112,"g":115},"9981":{"m":112,"g":115},"9989":{"m":112,"g":115},"10108":{"m":112,"g":115},"10123":{"m":112,"g":115},"7843":{"m":112,"g":115},"10090":{"m":112,"g":115},"10104":{"m":112,"g":115},"8801":{"m":112,"g":115},"10040":{"m":112,"g":115},"10141":{"m":112,"g":115},"10095":{"m":112,"g":115},"10144":{"m":112,"g":115},"9971":{"m":112,"g":115},"10134":{"m":112,"g":115},"10135":{"m":112,"g":115},"10074":{"m":112,"g":115},"10128":{"m":112,"g":115},"10126":{"m":112,"g":115},"10113":{"m":112,"g":115},"10096":{"m":112,"g":115},"10056":{"m":112,"g":115},"10101":{"m":112,"g":115},"9969":{"m":112,"g":115},"9741":{"m":112,"g":115},"9477":{"m":112,"g":115},"10117":{"m":112,"g":115},"10102":{"m":112,"g":115},"10116":{"m":112,"g":115},"10068":{"m":112,"g":115},"9956":{"m":112,"g":115},"10041":{"m":112,"g":115},"10058":{"m":112,"g":115},"10107":{"m":112,"g":115},"10100":{"m":112,"g":115},"9834":{"m":112,"g":115},"9861":{"m":112,"g":115},"9764":{"m":112,"g":115},"9269":{"m":112,"g":115},"9620":{"m":112,"g":115},"6905":{"m":112,"g":115},"10032":{"m":112,"g":115},"10097":{"m":112,"g":115},"10029":{"m":112,"g":115},"10039":{"m":112,"g":115},"10092":{"m":112,"g":115},"10086":{"m":112,"g":115},"10057":{"m":112,"g":115},"10047":{"m":112,"g":115},"9842":{"m":112,"g":115},"9965":{"m":112,"g":115},"10069":{"m":112,"g":115},"9884":{"m":112,"g":115},"10087":{"m":112,"g":115},"8622":{"m":112,"g":115},"8555":{"m":112,"g":115},"10080":{"m":112,"g":115},"10079":{"m":112,"g":115},"10043":{"m":112,"g":115},"10007":{"m":112,"g":115},"7182":{"m":112,"g":115},"8725":{"m":112,"g":115},"8867":{"m":112,"g":115},"9567":{"m":112,"g":115},"5255":{"m":112,"g":115},"10006":{"m":112,"g":115},"9534":{"m":112,"g":115},"9934":{"m":112,"g":115},"9931":{"m":112,"g":115},"9801":{"m":112,"g":115},"10049":{"m":112,"g":115},"10055":{"m":112,"g":115},"9964":{"m":112,"g":115},"10052":{"m":112,"g":115},"10050":{"m":112,"g":115},"8677":{"m":112,"g":115},"10008":{"m":112,"g":115},"9951":{"m":112,"g":115},"9957":{"m":112,"g":115},"9938":{"m":112,"g":115},"9634":{"m":112,"g":115},"9886":{"m":112,"g":115},"9973":{"m":112,"g":115},"9846":{"m":112,"g":115},"10003":{"m":112,"g":115},"10004":{"m":112,"g":115},"9997":{"m":112,"g":115},"10016":{"m":112,"g":115},"9993":{"m":112,"g":115},"9994":{"m":112,"g":115},"9999":{"m":112,"g":115},"10000":{"m":112,"g":115},"9996":{"m":112,"g":115},"9988":{"m":112,"g":115},"9986":{"m":112,"g":115},"9733":{"m":112,"g":115},"9314":{"m":112,"g":115},"9978":{"m":112,"g":115},"9914":{"m":112,"g":115},"9460":{"m":112,"g":115},"9958":{"m":112,"g":115},"9906":{"m":112,"g":115},"9953":{"m":112,"g":115},"9937":{"m":112,"g":115},"9959":{"m":112,"g":115},"9955":{"m":112,"g":115},"9755":{"m":112,"g":115},"9952":{"m":112,"g":115},"9671":{"m":112,"g":115},"7912":{"m":112,"g":115},"9895":{"m":112,"g":115},"9905":{"m":112,"g":115},"9927":{"m":112,"g":115},"9946":{"m":112,"g":115},"9912":{"m":112,"g":115},"9869":{"m":112,"g":115},"8747":{"m":112,"g":115},"9939":{"m":112,"g":115},"9929":{"m":112,"g":115},"9932":{"m":112,"g":115},"9705":{"m":112,"g":115},"9909":{"m":112,"g":115},"9920":{"m":112,"g":115},"9921":{"m":112,"g":115},"9879":{"m":112,"g":115},"9919":{"m":112,"g":115},"9916":{"m":112,"g":115},"9907":{"m":112,"g":115},"9844":{"m":112,"g":115},"9913":{"m":112,"g":115},"9902":{"m":112,"g":115},"8118":{"m":112,"g":115},"9875":{"m":112,"g":115},"9893":{"m":112,"g":115},"9878":{"m":112,"g":115},"9803":{"m":112,"g":115},"9876":{"m":112,"g":115},"9874":{"m":112,"g":115},"9783":{"m":112,"g":115},"9882":{"m":112,"g":115},"9857":{"m":112,"g":115},"9862":{"m":112,"g":115},"9864":{"m":112,"g":115},"8964":{"m":112,"g":115},"9794":{"m":112,"g":115},"9858":{"m":112,"g":115},"9852":{"m":112,"g":115},"9847":{"m":112,"g":115},"9073":{"m":112,"g":115},"9850":{"m":112,"g":115},"9797":{"m":112,"g":115},"9661":{"m":112,"g":115},"9841":{"m":112,"g":115},"9750":{"m":112,"g":115},"9709":{"m":112,"g":115},"8909":{"m":112,"g":115},"9840":{"m":112,"g":115},"9824":{"m":112,"g":115},"9835":{"m":112,"g":115},"9837":{"m":112,"g":115},"9836":{"m":112,"g":115},"9831":{"m":112,"g":115},"9830":{"m":112,"g":115},"9761":{"m":112,"g":115},"9828":{"m":112,"g":115},"9827":{"m":112,"g":115},"9826":{"m":112,"g":115},"9822":{"m":112,"g":115},"9802":{"m":112,"g":115},"9820":{"m":112,"g":115},"9746":{"m":112,"g":115},"9817":{"m":112,"g":115},"9815":{"m":112,"g":115},"9807":{"m":112,"g":115},"8345":{"m":112,"g":115},"9809":{"m":112,"g":115},"9670":{"m":112,"g":115},"9556":{"m":112,"g":115},"9675":{"m":112,"g":115},"9712":{"m":112,"g":115},"9793":{"m":112,"g":115},"9216":{"m":112,"g":115},"8375":{"m":112,"g":115},"9663":{"m":112,"g":115},"9715":{"m":112,"g":115},"9692":{"m":112,"g":115},"9776":{"m":112,"g":115},"9792":{"m":112,"g":115},"9786":{"m":112,"g":115},"9789":{"m":112,"g":115},"9788":{"m":112,"g":115},"9757":{"m":112,"g":115},"9784":{"m":112,"g":115},"9777":{"m":112,"g":115},"9749":{"m":112,"g":115},"9772":{"m":112,"g":115},"8750":{"m":112,"g":115},"6287":{"m":112,"g":115},"6407":{"m":112,"g":115},"9355":{"m":112,"g":115},"9770":{"m":112,"g":115},"8236":{"m":112,"g":115},"9759":{"m":112,"g":115},"9745":{"m":112,"g":115},"9721":{"m":112,"g":115},"9735":{"m":112,"g":115},"9573":{"m":112,"g":115},"9673":{"m":112,"g":115},"9740":{"m":112,"g":115},"9739":{"m":112,"g":115},"9684":{"m":112,"g":115},"9732":{"m":112,"g":115},"9730":{"m":112,"g":115},"9728":{"m":112,"g":115},"9615":{"m":112,"g":115},"9505":{"m":112,"g":115},"9724":{"m":112,"g":115},"9720":{"m":112,"g":115},"11263":{"m":113,"g":115},"11259":{"m":113,"g":115},"11061":{"m":113,"g":115},"11235":{"m":113,"g":115},"11240":{"m":113,"g":115},"11209":{"m":113,"g":115},"11254":{"m":113,"g":115},"11242":{"m":113,"g":115},"11252":{"m":113,"g":115},"11251":{"m":113,"g":115},"10048":{"m":113,"g":115},"10042":{"m":113,"g":115},"11248":{"m":113,"g":115},"11247":{"m":113,"g":115},"10996":{"m":113,"g":115},"11206":{"m":113,"g":115},"11228":{"m":113,"g":115},"11237":{"m":113,"g":115},"11222":{"m":113,"g":115},"11174":{"m":113,"g":115},"11162":{"m":113,"g":115},"10571":{"m":113,"g":115},"11229":{"m":113,"g":115},"11137":{"m":113,"g":115},"9624":{"m":113,"g":115},"11194":{"m":113,"g":115},"11225":{"m":113,"g":115},"11063":{"m":113,"g":115},"11217":{"m":113,"g":115},"11215":{"m":113,"g":115},"11140":{"m":113,"g":115},"11213":{"m":113,"g":115},"11012":{"m":113,"g":115},"11096":{"m":113,"g":115},"11011":{"m":113,"g":115},"11178":{"m":113,"g":115},"11196":{"m":113,"g":115},"10741":{"m":113,"g":115},"11198":{"m":113,"g":115},"10517":{"m":113,"g":115},"10838":{"m":113,"g":115},"10859":{"m":113,"g":115},"10609":{"m":113,"g":115},"10855":{"m":113,"g":115},"11090":{"m":113,"g":115},"10780":{"m":113,"g":115},"10892":{"m":113,"g":115},"11166":{"m":113,"g":115},"11192":{"m":113,"g":115},"11189":{"m":113,"g":115},"10873":{"m":113,"g":115},"11173":{"m":113,"g":115},"11167":{"m":113,"g":115},"10637":{"m":113,"g":115},"11185":{"m":113,"g":115},"11161":{"m":113,"g":115},"9537":{"m":113,"g":115},"10830":{"m":113,"g":115},"10133":{"m":113,"g":115},"11138":{"m":113,"g":115},"11179":{"m":113,"g":115},"11176":{"m":113,"g":115},"11159":{"m":113,"g":115},"11170":{"m":113,"g":115},"11175":{"m":113,"g":115},"11124":{"m":113,"g":115},"11171":{"m":113,"g":115},"11164":{"m":113,"g":115},"11130":{"m":113,"g":115},"11163":{"m":113,"g":115},"10837":{"m":113,"g":115},"11152":{"m":113,"g":115},"10988":{"m":113,"g":115},"11160":{"m":113,"g":115},"10422":{"m":113,"g":115},"10263":{"m":113,"g":115},"11156":{"m":113,"g":115},"10508":{"m":113,"g":115},"10779":{"m":113,"g":115},"10768":{"m":113,"g":115},"11132":{"m":113,"g":115},"11148":{"m":113,"g":115},"11149":{"m":113,"g":115},"10559":{"m":113,"g":115},"11135":{"m":113,"g":115},"10720":{"m":113,"g":115},"11145":{"m":113,"g":115},"11123":{"m":113,"g":115},"11143":{"m":113,"g":115},"11120":{"m":113,"g":115},"10512":{"m":113,"g":115},"10271":{"m":113,"g":115},"11005":{"m":113,"g":115},"10760":{"m":113,"g":115},"11128":{"m":113,"g":115},"11075":{"m":113,"g":115},"10985":{"m":113,"g":115},"11111":{"m":113,"g":115},"11115":{"m":113,"g":115},"11114":{"m":113,"g":115},"10735":{"m":113,"g":115},"11112":{"m":113,"g":115},"10972":{"m":113,"g":115},"11080":{"m":113,"g":115},"11113":{"m":113,"g":115},"11071":{"m":113,"g":115},"11081":{"m":113,"g":115},"11102":{"m":113,"g":115},"11101":{"m":113,"g":115},"10846":{"m":113,"g":115},"11094":{"m":113,"g":115},"11067":{"m":113,"g":115},"11099":{"m":113,"g":115},"11087":{"m":113,"g":115},"10991":{"m":113,"g":115},"11085":{"m":113,"g":115},"11070":{"m":113,"g":115},"11092":{"m":113,"g":115},"10729":{"m":113,"g":115},"9642":{"m":113,"g":115},"10816":{"m":113,"g":115},"11083":{"m":113,"g":115},"10875":{"m":113,"g":115},"11082":{"m":113,"g":115},"11079":{"m":113,"g":115},"10611":{"m":113,"g":115},"11076":{"m":113,"g":115},"11073":{"m":113,"g":115},"10975":{"m":113,"g":115},"11069":{"m":113,"g":115},"11056":{"m":113,"g":115},"10976":{"m":113,"g":115},"11054":{"m":113,"g":115},"11050":{"m":113,"g":115},"11022":{"m":113,"g":115},"9614":{"m":113,"g":115},"11010":{"m":113,"g":115},"10591":{"m":113,"g":115},"11015":{"m":113,"g":115},"10940":{"m":113,"g":115},"10701":{"m":113,"g":115},"11036":{"m":113,"g":115},"11038":{"m":113,"g":115},"10986":{"m":113,"g":115},"11033":{"m":113,"g":115},"11017":{"m":113,"g":115},"11003":{"m":113,"g":115},"11013":{"m":113,"g":115},"10543":{"m":113,"g":115},"10555":{"m":113,"g":115},"10964":{"m":113,"g":115},"11009":{"m":113,"g":115},"10999":{"m":113,"g":115},"10978":{"m":113,"g":115},"10995":{"m":113,"g":115},"10997":{"m":113,"g":115},"10550":{"m":113,"g":115},"10565":{"m":113,"g":115},"10616":{"m":113,"g":115},"10751":{"m":113,"g":115},"10930":{"m":113,"g":115},"10981":{"m":113,"g":115},"10112":{"m":113,"g":115},"10980":{"m":113,"g":115},"10982":{"m":113,"g":115},"10551":{"m":113,"g":115},"10965":{"m":113,"g":115},"10944":{"m":113,"g":115},"10971":{"m":113,"g":115},"10941":{"m":113,"g":115},"10495":{"m":113,"g":115},"10372":{"m":113,"g":115},"10970":{"m":113,"g":115},"10947":{"m":113,"g":115},"10968":{"m":113,"g":115},"10967":{"m":113,"g":115},"10963":{"m":113,"g":115},"10960":{"m":113,"g":115},"10958":{"m":113,"g":115},"10956":{"m":113,"g":115},"10955":{"m":113,"g":115},"10749":{"m":113,"g":115},"10927":{"m":113,"g":115},"10936":{"m":113,"g":115},"10192":{"m":113,"g":115},"10939":{"m":113,"g":115},"10935":{"m":113,"g":115},"10929":{"m":113,"g":115},"10932":{"m":113,"g":115},"10898":{"m":113,"g":115},"10612":{"m":113,"g":115},"10923":{"m":113,"g":115},"10926":{"m":113,"g":115},"10899":{"m":113,"g":115},"10883":{"m":113,"g":115},"10910":{"m":113,"g":115},"10924":{"m":113,"g":115},"10132":{"m":113,"g":115},"10881":{"m":113,"g":115},"10894":{"m":113,"g":115},"10915":{"m":113,"g":115},"10376":{"m":113,"g":115},"10778":{"m":113,"g":115},"10872":{"m":113,"g":115},"10885":{"m":113,"g":115},"10895":{"m":113,"g":115},"10845":{"m":113,"g":115},"10880":{"m":113,"g":115},"10572":{"m":113,"g":115},"10861":{"m":113,"g":115},"10876":{"m":113,"g":115},"10877":{"m":113,"g":115},"10534":{"m":113,"g":115},"10827":{"m":113,"g":115},"10832":{"m":113,"g":115},"10786":{"m":113,"g":115},"10860":{"m":113,"g":115},"10718":{"m":113,"g":115},"10829":{"m":113,"g":115},"10828":{"m":113,"g":115},"10825":{"m":113,"g":115},"10826":{"m":113,"g":115},"10822":{"m":113,"g":115},"10824":{"m":113,"g":115},"10823":{"m":113,"g":115},"10787":{"m":113,"g":115},"10820":{"m":113,"g":115},"10794":{"m":113,"g":115},"10799":{"m":113,"g":115},"10818":{"m":113,"g":115},"10540":{"m":113,"g":115},"10504":{"m":113,"g":115},"10761":{"m":113,"g":115},"10814":{"m":113,"g":115},"10812":{"m":113,"g":115},"10259":{"m":113,"g":115},"10323":{"m":113,"g":115},"10581":{"m":113,"g":115},"10773":{"m":113,"g":115},"10792":{"m":113,"g":115},"10791":{"m":113,"g":115},"10770":{"m":113,"g":115},"10783":{"m":113,"g":115},"10782":{"m":113,"g":115},"10715":{"m":113,"g":115},"10705":{"m":113,"g":115},"10776":{"m":113,"g":115},"10777":{"m":113,"g":115},"10771":{"m":113,"g":115},"10774":{"m":113,"g":115},"10767":{"m":113,"g":115},"10756":{"m":113,"g":115},"10765":{"m":113,"g":115},"10574":{"m":113,"g":115},"10556":{"m":113,"g":115},"10130":{"m":113,"g":115},"10762":{"m":113,"g":115},"10541":{"m":113,"g":115},"10281":{"m":113,"g":115},"10759":{"m":113,"g":115},"10300":{"m":113,"g":115},"10755":{"m":113,"g":115},"10732":{"m":113,"g":115},"10758":{"m":113,"g":115},"10757":{"m":113,"g":115},"10727":{"m":113,"g":115},"10754":{"m":113,"g":115},"10724":{"m":113,"g":115},"10753":{"m":113,"g":115},"10737":{"m":113,"g":115},"10728":{"m":113,"g":115},"10730":{"m":113,"g":115},"9849":{"m":113,"g":115},"10731":{"m":113,"g":115},"10709":{"m":113,"g":115},"10699":{"m":113,"g":115},"10678":{"m":113,"g":115},"10695":{"m":113,"g":115},"10694":{"m":113,"g":115},"10717":{"m":113,"g":115},"10714":{"m":113,"g":115},"10716":{"m":113,"g":115},"10385":{"m":113,"g":115},"10317":{"m":113,"g":115},"10706":{"m":113,"g":115},"10592":{"m":113,"g":115},"10697":{"m":113,"g":115},"10696":{"m":113,"g":115},"10651":{"m":113,"g":115},"10688":{"m":113,"g":115},"10686":{"m":113,"g":115},"10685":{"m":113,"g":115},"10673":{"m":113,"g":115},"10684":{"m":113,"g":115},"10680":{"m":113,"g":115},"10681":{"m":113,"g":115},"10683":{"m":113,"g":115},"10645":{"m":113,"g":115},"10677":{"m":113,"g":115},"10679":{"m":113,"g":115},"10648":{"m":113,"g":115},"10671":{"m":113,"g":115},"10675":{"m":113,"g":115},"10666":{"m":113,"g":115},"10670":{"m":113,"g":115},"10668":{"m":113,"g":115},"10664":{"m":113,"g":115},"10661":{"m":113,"g":115},"10522":{"m":113,"g":115},"10653":{"m":113,"g":115},"10634":{"m":113,"g":115},"10650":{"m":113,"g":115},"10321":{"m":113,"g":115},"10647":{"m":113,"g":115},"10081":{"m":113,"g":115},"10633":{"m":113,"g":115},"10319":{"m":113,"g":115},"10630":{"m":113,"g":115},"10631":{"m":113,"g":115},"10632":{"m":113,"g":115},"10586":{"m":113,"g":115},"10553":{"m":113,"g":115},"9873":{"m":113,"g":115},"10629":{"m":113,"g":115},"10628":{"m":113,"g":115},"10621":{"m":113,"g":115},"9947":{"m":113,"g":115},"10579":{"m":113,"g":115},"10595":{"m":113,"g":115},"10610":{"m":113,"g":115},"10622":{"m":113,"g":115},"10624":{"m":113,"g":115},"10222":{"m":113,"g":115},"9979":{"m":113,"g":115},"8274":{"m":113,"g":115},"10604":{"m":113,"g":115},"10525":{"m":113,"g":115},"10596":{"m":113,"g":115},"10273":{"m":113,"g":115},"10563":{"m":113,"g":115},"10190":{"m":113,"g":115},"10558":{"m":113,"g":115},"10526":{"m":113,"g":115},"9987":{"m":113,"g":115},"10584":{"m":113,"g":115},"9976":{"m":113,"g":115},"10171":{"m":113,"g":115},"10548":{"m":113,"g":115},"8813":{"m":113,"g":115},"10545":{"m":113,"g":115},"10523":{"m":113,"g":115},"10459":{"m":113,"g":115},"10529":{"m":113,"g":115},"8746":{"m":113,"g":115},"10538":{"m":113,"g":115},"10494":{"m":113,"g":115},"9928":{"m":113,"g":115},"10474":{"m":113,"g":115},"10530":{"m":113,"g":115},"10528":{"m":113,"g":115},"10524":{"m":113,"g":115},"10511":{"m":113,"g":115},"10506":{"m":113,"g":115},"10515":{"m":113,"g":115},"10491":{"m":113,"g":115},"10500":{"m":113,"g":115},"10507":{"m":113,"g":115},"10466":{"m":113,"g":115},"10498":{"m":113,"g":115},"10499":{"m":113,"g":115},"10493":{"m":113,"g":115},"10487":{"m":113,"g":115},"10230":{"m":113,"g":115},"10336":{"m":113,"g":115},"10203":{"m":113,"g":115},"10434":{"m":113,"g":115},"8863":{"m":113,"g":115},"10486":{"m":113,"g":115},"10484":{"m":113,"g":115},"10286":{"m":113,"g":115},"10481":{"m":113,"g":115},"10478":{"m":113,"g":115},"10479":{"m":113,"g":115},"10475":{"m":113,"g":115},"10473":{"m":113,"g":115},"10476":{"m":113,"g":115},"8189":{"m":113,"g":115},"9657":{"m":113,"g":115},"10375":{"m":113,"g":115},"9887":{"m":113,"g":115},"8710":{"m":113,"g":115},"10471":{"m":113,"g":115},"10470":{"m":113,"g":115},"10468":{"m":113,"g":115},"10465":{"m":113,"g":115},"10440":{"m":113,"g":115},"10456":{"m":113,"g":115},"10439":{"m":113,"g":115},"10463":{"m":113,"g":115},"10458":{"m":113,"g":115},"10457":{"m":113,"g":115},"10401":{"m":113,"g":115},"10358":{"m":113,"g":115},"10449":{"m":113,"g":115},"9343":{"m":113,"g":115},"10452":{"m":113,"g":115},"10450":{"m":113,"g":115},"10445":{"m":113,"g":115},"9626":{"m":113,"g":115},"9768":{"m":113,"g":115},"10143":{"m":113,"g":115},"10441":{"m":113,"g":115},"10437":{"m":113,"g":115},"9338":{"m":113,"g":115},"10435":{"m":113,"g":115},"10201":{"m":113,"g":115},"10432":{"m":113,"g":115},"10129":{"m":113,"g":115},"10433":{"m":113,"g":115},"10426":{"m":113,"g":115},"10425":{"m":113,"g":115},"10429":{"m":113,"g":115},"10431":{"m":113,"g":115},"10428":{"m":113,"g":115},"9962":{"m":113,"g":115},"10076":{"m":113,"g":115},"8627":{"m":113,"g":115},"10419":{"m":113,"g":115},"6539":{"m":113,"g":115},"10270":{"m":113,"g":115},"9948":{"m":113,"g":115},"10157":{"m":113,"g":115},"10313":{"m":113,"g":115},"10369":{"m":113,"g":115},"10318":{"m":113,"g":115},"10404":{"m":113,"g":115},"10228":{"m":113,"g":115},"10414":{"m":113,"g":115},"10410":{"m":113,"g":115},"10411":{"m":113,"g":115},"10412":{"m":113,"g":115},"9748":{"m":113,"g":115},"9382":{"m":113,"g":115},"10406":{"m":113,"g":115},"10392":{"m":113,"g":115},"10403":{"m":113,"g":115},"10400":{"m":113,"g":115},"10397":{"m":113,"g":115},"10398":{"m":113,"g":115},"9984":{"m":113,"g":115},"10395":{"m":113,"g":115},"10394":{"m":113,"g":115},"10332":{"m":113,"g":115},"10377":{"m":113,"g":115},"10379":{"m":113,"g":115},"10380":{"m":113,"g":115},"10387":{"m":113,"g":115},"10244":{"m":113,"g":115},"10386":{"m":113,"g":115},"10391":{"m":113,"g":115},"10361":{"m":113,"g":115},"10390":{"m":113,"g":115},"10388":{"m":113,"g":115},"10343":{"m":113,"g":115},"10333":{"m":113,"g":115},"9023":{"m":113,"g":115},"10099":{"m":113,"g":115},"10219":{"m":113,"g":115},"10370":{"m":113,"g":115},"10368":{"m":113,"g":115},"10180":{"m":113,"g":115},"10355":{"m":113,"g":115},"8215":{"m":113,"g":115},"8778":{"m":113,"g":115},"10351":{"m":113,"g":115},"10362":{"m":113,"g":115},"10359":{"m":113,"g":115},"10360":{"m":113,"g":115},"10356":{"m":113,"g":115},"10031":{"m":113,"g":115},"10283":{"m":113,"g":115},"10346":{"m":113,"g":115},"10352":{"m":113,"g":115},"9774":{"m":113,"g":115},"10296":{"m":113,"g":115},"9199":{"m":113,"g":115},"10345":{"m":113,"g":115},"10349":{"m":113,"g":115},"10347":{"m":113,"g":115},"11324":{"m":114,"g":115},"11369":{"m":114,"g":115},"11364":{"m":114,"g":115},"11394":{"m":114,"g":115},"11387":{"m":114,"g":115},"11376":{"m":114,"g":115},"11375":{"m":114,"g":115},"11373":{"m":114,"g":115},"11309":{"m":114,"g":115},"11366":{"m":114,"g":115},"11359":{"m":114,"g":115},"11353":{"m":114,"g":115},"10979":{"m":114,"g":115},"11350":{"m":114,"g":115},"11327":{"m":114,"g":115},"11342":{"m":114,"g":115},"11339":{"m":114,"g":115},"11341":{"m":114,"g":115},"11340":{"m":114,"g":115},"11336":{"m":114,"g":115},"11323":{"m":114,"g":115},"10909":{"m":114,"g":115},"11264":{"m":114,"g":115},"9812":{"m":114,"g":115},"11318":{"m":114,"g":115},"11007":{"m":114,"g":115},"11321":{"m":114,"g":115},"10937":{"m":114,"g":115},"11312":{"m":114,"g":115},"11211":{"m":114,"g":115},"9545":{"m":114,"g":115},"11314":{"m":114,"g":115},"11316":{"m":114,"g":115},"11126":{"m":114,"g":115},"11315":{"m":114,"g":115},"10710":{"m":114,"g":115},"11230":{"m":114,"g":115},"11200":{"m":114,"g":115},"11304":{"m":114,"g":115},"11310":{"m":114,"g":115},"11311":{"m":114,"g":115},"11297":{"m":114,"g":115},"11205":{"m":114,"g":115},"11307":{"m":114,"g":115},"11223":{"m":114,"g":115},"11306":{"m":114,"g":115},"11305":{"m":114,"g":115},"11001":{"m":114,"g":115},"11027":{"m":114,"g":115},"11288":{"m":114,"g":115},"11303":{"m":114,"g":115},"11302":{"m":114,"g":115},"11068":{"m":114,"g":115},"11301":{"m":114,"g":115},"11300":{"m":114,"g":115},"11290":{"m":114,"g":115},"11210":{"m":114,"g":115},"11231":{"m":114,"g":115},"11294":{"m":114,"g":115},"10949":{"m":114,"g":115},"11095":{"m":114,"g":115},"11283":{"m":114,"g":115},"11281":{"m":114,"g":115},"11286":{"m":114,"g":115},"11282":{"m":114,"g":115},"11261":{"m":114,"g":115},"11238":{"m":114,"g":115},"11279":{"m":114,"g":115},"11280":{"m":114,"g":115},"11276":{"m":114,"g":115},"11277":{"m":114,"g":115},"11182":{"m":114,"g":115},"11268":{"m":114,"g":115},"11274":{"m":114,"g":115},"11270":{"m":114,"g":115},"7149":{"m":114,"g":115},"11262":{"m":114,"g":115},"11219":{"m":114,"g":115},"11680":{"m":116,"g":118},"11676":{"m":116,"g":118},"11684":{"m":116,"g":118},"11667":{"m":116,"g":118},"11674":{"m":116,"g":118},"11681":{"m":116,"g":118},"11621":{"m":116,"g":118},"11367":{"m":116,"g":118},"11653":{"m":116,"g":118},"11660":{"m":116,"g":118},"11659":{"m":116,"g":118},"11585":{"m":116,"g":118},"11293":{"m":116,"g":118},"11458":{"m":116,"g":118},"11590":{"m":116,"g":118},"11636":{"m":116,"g":118},"11579":{"m":116,"g":118},"8247":{"m":116,"g":118},"10423":{"m":116,"g":118},"11642":{"m":116,"g":115},"11638":{"m":116,"g":115},"11628":{"m":116,"g":115},"11639":{"m":116,"g":115},"11351":{"m":116,"g":115},"11627":{"m":116,"g":115},"11633":{"m":116,"g":115},"11631":{"m":116,"g":115},"11622":{"m":116,"g":115},"11625":{"m":116,"g":115},"11623":{"m":116,"g":115},"11624":{"m":116,"g":115},"11619":{"m":116,"g":115},"11605":{"m":116,"g":115},"11620":{"m":116,"g":115},"11617":{"m":116,"g":115},"11453":{"m":116,"g":115},"11561":{"m":116,"g":115},"11434":{"m":116,"g":115},"10721":{"m":116,"g":115},"11586":{"m":116,"g":115},"11556":{"m":116,"g":115},"11603":{"m":116,"g":115},"11593":{"m":116,"g":115},"11601":{"m":116,"g":115},"11449":{"m":116,"g":115},"11600":{"m":116,"g":115},"11597":{"m":116,"g":115},"11598":{"m":116,"g":115},"11566":{"m":116,"g":115},"11591":{"m":116,"g":115},"11588":{"m":116,"g":115},"11587":{"m":116,"g":115},"11580":{"m":116,"g":115},"11583":{"m":116,"g":115},"11582":{"m":116,"g":115},"11041":{"m":116,"g":115},"11542":{"m":116,"g":115},"11535":{"m":116,"g":115},"11565":{"m":116,"g":115},"11413":{"m":116,"g":115},"11539":{"m":116,"g":115},"11572":{"m":116,"g":115},"11573":{"m":116,"g":115},"11534":{"m":116,"g":115},"11571":{"m":116,"g":115},"11564":{"m":116,"g":115},"11537":{"m":116,"g":115},"11538":{"m":116,"g":115},"11521":{"m":116,"g":115},"11562":{"m":116,"g":115},"11308":{"m":116,"g":115},"11557":{"m":116,"g":115},"11441":{"m":116,"g":115},"11483":{"m":116,"g":115},"11531":{"m":116,"g":115},"11549":{"m":116,"g":115},"11553":{"m":116,"g":115},"11547":{"m":116,"g":115},"11507":{"m":116,"g":115},"11419":{"m":116,"g":115},"11444":{"m":116,"g":115},"11442":{"m":116,"g":115},"11548":{"m":116,"g":115},"11530":{"m":116,"g":115},"11527":{"m":116,"g":115},"11201":{"m":116,"g":115},"11528":{"m":116,"g":115},"11457":{"m":116,"g":115},"11544":{"m":116,"g":115},"11460":{"m":116,"g":115},"11505":{"m":116,"g":115},"11385":{"m":116,"g":115},"11214":{"m":116,"g":115},"11493":{"m":116,"g":115},"11512":{"m":116,"g":115},"11432":{"m":116,"g":115},"11485":{"m":116,"g":115},"11511":{"m":116,"g":115},"5889":{"m":116,"g":115},"11520":{"m":116,"g":115},"11516":{"m":116,"g":115},"11498":{"m":116,"g":115},"11474":{"m":116,"g":115},"11515":{"m":116,"g":115},"11514":{"m":116,"g":115},"11331":{"m":116,"g":115},"11509":{"m":116,"g":115},"11443":{"m":116,"g":115},"11452":{"m":116,"g":115},"11503":{"m":116,"g":115},"11502":{"m":116,"g":115},"11497":{"m":116,"g":115},"11501":{"m":116,"g":115},"11500":{"m":116,"g":115},"11332":{"m":116,"g":115},"11499":{"m":116,"g":115},"11465":{"m":116,"g":115},"10577":{"m":116,"g":115},"11479":{"m":116,"g":115},"11221":{"m":116,"g":115},"10172":{"m":116,"g":115},"11478":{"m":116,"g":115},"11481":{"m":116,"g":115},"11476":{"m":116,"g":115},"11489":{"m":116,"g":115},"10062":{"m":116,"g":115},"11398":{"m":116,"g":115},"10635":{"m":116,"g":115},"9804":{"m":116,"g":115},"11454":{"m":116,"g":115},"11019":{"m":116,"g":115},"8919":{"m":116,"g":115},"11462":{"m":116,"g":115},"11428":{"m":116,"g":115},"11470":{"m":116,"g":115},"11427":{"m":116,"g":115},"11467":{"m":116,"g":115},"11448":{"m":116,"g":115},"10312":{"m":116,"g":115},"9991":{"m":116,"g":115},"11455":{"m":116,"g":115},"11450":{"m":116,"g":115},"11360":{"m":116,"g":115},"11368":{"m":116,"g":115},"11445":{"m":116,"g":115},"11438":{"m":116,"g":115},"11439":{"m":116,"g":115},"11399":{"m":116,"g":115},"11435":{"m":116,"g":115},"11313":{"m":116,"g":115},"10745":{"m":116,"g":115},"11411":{"m":116,"g":115},"11433":{"m":116,"g":115},"11437":{"m":116,"g":115},"11436":{"m":116,"g":115},"11345":{"m":116,"g":115},"9256":{"m":116,"g":115},"11381":{"m":116,"g":115},"11361":{"m":116,"g":115},"11420":{"m":116,"g":115},"11144":{"m":116,"g":115},"10734":{"m":116,"g":115},"10969":{"m":116,"g":115},"9045":{"m":116,"g":115},"11414":{"m":116,"g":115},"11388":{"m":116,"g":115},"11363":{"m":116,"g":115},"11365":{"m":116,"g":115},"11389":{"m":116,"g":115},"11401":{"m":116,"g":115},"11285":{"m":116,"g":115},"11693":{"m":117,"g":118},"11543":{"m":117,"g":118},"11687":{"m":117,"g":118},"11706":{"m":117,"g":118},"11510":{"m":117,"g":118},"11488":{"m":117,"g":118},"11370":{"m":117,"g":118},"11692":{"m":117,"g":118},"11663":{"m":117,"g":118},"10912":{"m":117,"g":118},"11679":{"m":117,"g":118},"11689":{"m":117,"g":118},"11686":{"m":117,"g":118},"10248":{"m":117,"g":118},"9493":{"m":117,"g":118},"12027":{"m":119,"g":121},"12009":{"m":119,"g":121},"12030":{"m":119,"g":121},"12029":{"m":119,"g":121},"11616":{"m":119,"g":121},"12028":{"m":119,"g":121},"9366":{"m":119,"g":121},"11891":{"m":119,"g":121},"10158":{"m":119,"g":121},"12024":{"m":119,"g":121},"11765":{"m":119,"g":121},"12022":{"m":119,"g":121},"12018":{"m":119,"g":121},"12021":{"m":119,"g":121},"11755":{"m":119,"g":121},"11981":{"m":119,"g":121},"12015":{"m":119,"g":121},"12014":{"m":119,"g":121},"11937":{"m":119,"g":121},"11988":{"m":119,"g":121},"11866":{"m":119,"g":121},"11821":{"m":119,"g":121},"12004":{"m":119,"g":121},"11985":{"m":119,"g":121},"11944":{"m":119,"g":121},"10652":{"m":119,"g":121},"11965":{"m":119,"g":121},"11906":{"m":119,"g":121},"11990":{"m":119,"g":121},"11299":{"m":119,"g":121},"11322":{"m":119,"g":121},"11811":{"m":119,"g":121},"11955":{"m":119,"g":121},"11978":{"m":119,"g":121},"10869":{"m":119,"g":121},"11921":{"m":119,"g":121},"11563":{"m":119,"g":121},"11980":{"m":119,"g":121},"11977":{"m":119,"g":121},"10750":{"m":119,"g":121},"11956":{"m":119,"g":121},"11723":{"m":119,"g":121},"11967":{"m":119,"g":121},"11953":{"m":119,"g":121},"9651":{"m":119,"g":121},"10606":{"m":119,"g":121},"11908":{"m":119,"g":121},"10154":{"m":119,"g":121},"11929":{"m":119,"g":121},"11717":{"m":119,"g":121},"11922":{"m":119,"g":121},"11945":{"m":119,"g":121},"11790":{"m":119,"g":121},"11926":{"m":119,"g":121},"11940":{"m":119,"g":121},"11935":{"m":119,"g":121},"11934":{"m":119,"g":121},"11377":{"m":119,"g":121},"11933":{"m":119,"g":121},"11876":{"m":119,"g":121},"11844":{"m":119,"g":121},"11287":{"m":119,"g":121},"11918":{"m":119,"g":121},"11915":{"m":119,"g":121},"11702":{"m":119,"g":121},"11482":{"m":119,"g":121},"10700":{"m":119,"g":121},"11902":{"m":119,"g":121},"11295":{"m":119,"g":121},"11416":{"m":119,"g":121},"11895":{"m":119,"g":121},"11570":{"m":119,"g":121},"11487":{"m":119,"g":121},"11878":{"m":119,"g":121},"11885":{"m":119,"g":118},"11664":{"m":119,"g":118},"10656":{"m":119,"g":118},"11843":{"m":119,"g":118},"11845":{"m":119,"g":118},"11859":{"m":119,"g":118},"11887":{"m":119,"g":118},"11838":{"m":119,"g":118},"11886":{"m":119,"g":118},"11826":{"m":119,"g":118},"11882":{"m":119,"g":118},"11875":{"m":119,"g":118},"11881":{"m":119,"g":118},"11868":{"m":119,"g":118},"11807":{"m":119,"g":118},"11867":{"m":119,"g":118},"11823":{"m":119,"g":118},"11847":{"m":119,"g":118},"11862":{"m":119,"g":118},"10691":{"m":119,"g":118},"11776":{"m":119,"g":118},"11822":{"m":119,"g":118},"11849":{"m":119,"g":118},"11396":{"m":119,"g":118},"11846":{"m":119,"g":118},"11747":{"m":119,"g":118},"10801":{"m":119,"g":118},"11722":{"m":119,"g":118},"11594":{"m":119,"g":118},"11780":{"m":119,"g":118},"11733":{"m":119,"g":118},"10510":{"m":119,"g":118},"11508":{"m":119,"g":118},"11787":{"m":119,"g":118},"11832":{"m":119,"g":118},"11778":{"m":119,"g":118},"11612":{"m":119,"g":118},"11810":{"m":119,"g":118},"11831":{"m":119,"g":118},"11815":{"m":119,"g":118},"11606":{"m":119,"g":118},"11835":{"m":119,"g":118},"10994":{"m":119,"g":118},"11147":{"m":119,"g":118},"11833":{"m":119,"g":118},"11834":{"m":119,"g":118},"11652":{"m":119,"g":118},"11819":{"m":119,"g":118},"11827":{"m":119,"g":118},"11805":{"m":119,"g":118},"5162":{"m":119,"g":118},"11786":{"m":119,"g":118},"11804":{"m":119,"g":118},"11808":{"m":119,"g":118},"11091":{"m":119,"g":118},"11818":{"m":119,"g":118},"10788":{"m":119,"g":118},"11817":{"m":119,"g":118},"11328":{"m":119,"g":118},"11555":{"m":119,"g":118},"11618":{"m":119,"g":118},"11670":{"m":119,"g":118},"11688":{"m":119,"g":118},"11773":{"m":119,"g":118},"11813":{"m":119,"g":118},"11000":{"m":119,"g":118},"11710":{"m":119,"g":118},"11749":{"m":119,"g":118},"11772":{"m":119,"g":118},"11506":{"m":119,"g":118},"11797":{"m":119,"g":118},"11781":{"m":119,"g":118},"11803":{"m":119,"g":118},"11801":{"m":119,"g":118},"10152":{"m":119,"g":118},"11669":{"m":119,"g":118},"11793":{"m":119,"g":118},"11665":{"m":119,"g":118},"11799":{"m":119,"g":118},"11798":{"m":119,"g":118},"11794":{"m":119,"g":118},"11784":{"m":119,"g":118},"11783":{"m":119,"g":118},"11614":{"m":119,"g":118},"11788":{"m":119,"g":118},"9170":{"m":119,"g":118},"11666":{"m":119,"g":118},"11613":{"m":119,"g":118},"11611":{"m":119,"g":118},"11607":{"m":119,"g":118},"11685":{"m":119,"g":118},"11519":{"m":119,"g":118},"11782":{"m":119,"g":118},"11777":{"m":119,"g":118},"11682":{"m":119,"g":118},"11775":{"m":119,"g":118},"11738":{"m":119,"g":118},"10725":{"m":119,"g":118},"11540":{"m":119,"g":118},"11767":{"m":119,"g":118},"11768":{"m":119,"g":118},"11766":{"m":119,"g":118},"11735":{"m":119,"g":118},"11730":{"m":119,"g":118},"11643":{"m":119,"g":118},"11062":{"m":119,"g":118},"11724":{"m":119,"g":118},"11746":{"m":119,"g":118},"11739":{"m":119,"g":118},"11740":{"m":119,"g":118},"11732":{"m":119,"g":118},"11734":{"m":119,"g":118},"11541":{"m":119,"g":118},"11728":{"m":119,"g":118},"11731":{"m":119,"g":118},"11727":{"m":119,"g":118},"11729":{"m":119,"g":118},"11677":{"m":119,"g":118},"10911":{"m":119,"g":118},"12169":{"m":120,"g":121},"12177":{"m":120,"g":121},"12170":{"m":120,"g":121},"12167":{"m":120,"g":121},"12171":{"m":120,"g":121},"12164":{"m":120,"g":121},"12168":{"m":120,"g":121},"12129":{"m":120,"g":121},"12166":{"m":120,"g":121},"11047":{"m":120,"g":121},"11632":{"m":120,"g":121},"10399":{"m":120,"g":121},"12142":{"m":120,"g":121},"12106":{"m":120,"g":121},"12156":{"m":120,"g":121},"12155":{"m":120,"g":121},"12152":{"m":120,"g":121},"12154":{"m":120,"g":121},"11615":{"m":120,"g":121},"11494":{"m":120,"g":121},"12113":{"m":120,"g":121},"12116":{"m":120,"g":121},"12136":{"m":120,"g":121},"12147":{"m":120,"g":121},"12141":{"m":120,"g":121},"11991":{"m":120,"g":121},"12097":{"m":120,"g":121},"12139":{"m":120,"g":121},"12138":{"m":120,"g":121},"12118":{"m":120,"g":121},"12133":{"m":120,"g":121},"11936":{"m":120,"g":121},"12132":{"m":120,"g":121},"11993":{"m":120,"g":121},"12130":{"m":120,"g":121},"12125":{"m":120,"g":121},"12101":{"m":120,"g":121},"11814":{"m":120,"g":121},"12127":{"m":120,"g":121},"12115":{"m":120,"g":121},"12126":{"m":120,"g":121},"12098":{"m":120,"g":121},"11962":{"m":120,"g":121},"12119":{"m":120,"g":121},"12124":{"m":120,"g":121},"11869":{"m":120,"g":121},"12110":{"m":120,"g":121},"12096":{"m":120,"g":121},"12083":{"m":120,"g":121},"12103":{"m":120,"g":121},"12105":{"m":120,"g":121},"12087":{"m":120,"g":121},"9501":{"m":120,"g":121},"11379":{"m":120,"g":121},"12058":{"m":120,"g":121},"12054":{"m":120,"g":121},"11877":{"m":120,"g":121},"12070":{"m":120,"g":121},"8464":{"m":120,"g":121},"12093":{"m":120,"g":121},"12071":{"m":120,"g":121},"12000":{"m":120,"g":121},"12091":{"m":120,"g":121},"12089":{"m":120,"g":121},"12086":{"m":120,"g":121},"11560":{"m":120,"g":121},"12084":{"m":120,"g":121},"12034":{"m":120,"g":121},"11924":{"m":120,"g":121},"11884":{"m":120,"g":121},"12025":{"m":120,"g":121},"11958":{"m":120,"g":121},"12067":{"m":120,"g":121},"11999":{"m":120,"g":121},"12046":{"m":120,"g":121},"12049":{"m":120,"g":121},"12063":{"m":120,"g":121},"12064":{"m":120,"g":121},"12053":{"m":120,"g":121},"11800":{"m":120,"g":121},"12056":{"m":120,"g":121},"12019":{"m":120,"g":121},"11853":{"m":120,"g":121},"12031":{"m":120,"g":121},"12041":{"m":120,"g":121},"10953":{"m":120,"g":121},"11759":{"m":120,"g":121},"12042":{"m":120,"g":121},"12037":{"m":120,"g":121},"11745":{"m":120,"g":121},"12038":{"m":120,"g":121},"11909":{"m":120,"g":121},"11816":{"m":120,"g":121},"11795":{"m":120,"g":121},"12003":{"m":120,"g":121},"9936":{"m":120,"g":121},"11964":{"m":120,"g":121},"12439":{"m":122,"g":127},"11874":{"m":122,"g":127},"12475":{"m":122,"g":127},"12469":{"m":122,"g":127},"11987":{"m":122,"g":127},"12430":{"m":122,"g":127},"12473":{"m":122,"g":127},"12297":{"m":122,"g":127},"12341":{"m":122,"g":127},"12428":{"m":122,"g":127},"12429":{"m":122,"g":127},"12066":{"m":122,"g":127},"12275":{"m":122,"g":127},"11757":{"m":122,"g":127},"11931":{"m":122,"g":127},"12470":{"m":122,"g":127},"12415":{"m":122,"g":127},"12266":{"m":122,"g":127},"12463":{"m":122,"g":127},"12328":{"m":122,"g":127},"12369":{"m":122,"g":127},"12256":{"m":122,"g":127},"12449":{"m":122,"g":127},"12452":{"m":122,"g":127},"12413":{"m":122,"g":127},"10889":{"m":122,"g":127},"12422":{"m":122,"g":127},"12436":{"m":122,"g":127},"12437":{"m":122,"g":127},"12410":{"m":122,"g":127},"12384":{"m":122,"g":127},"12307":{"m":122,"g":127},"10566":{"m":122,"g":127},"12405":{"m":122,"g":127},"12425":{"m":122,"g":127},"12401":{"m":122,"g":127},"12300":{"m":122,"g":127},"11224":{"m":122,"g":127},"11116":{"m":122,"g":127},"12399":{"m":122,"g":121},"12290":{"m":122,"g":121},"12242":{"m":122,"g":121},"12368":{"m":122,"g":121},"12403":{"m":122,"g":121},"12386":{"m":122,"g":121},"12409":{"m":122,"g":121},"12375":{"m":122,"g":121},"12404":{"m":122,"g":121},"12281":{"m":122,"g":121},"12185":{"m":122,"g":121},"12012":{"m":122,"g":121},"12364":{"m":122,"g":121},"12395":{"m":122,"g":121},"11960":{"m":122,"g":121},"12377":{"m":122,"g":121},"11897":{"m":122,"g":121},"11969":{"m":122,"g":121},"12394":{"m":122,"g":121},"11806":{"m":122,"g":121},"12319":{"m":122,"g":121},"12123":{"m":122,"g":121},"12358":{"m":122,"g":121},"12362":{"m":122,"g":121},"12135":{"m":122,"g":121},"12378":{"m":122,"g":121},"12174":{"m":122,"g":121},"12340":{"m":122,"g":121},"12195":{"m":122,"g":121},"11910":{"m":122,"g":121},"12216":{"m":122,"g":121},"12050":{"m":122,"g":121},"12153":{"m":122,"g":121},"12094":{"m":122,"g":121},"12182":{"m":122,"g":121},"12354":{"m":122,"g":121},"12350":{"m":122,"g":121},"12348":{"m":122,"g":121},"11737":{"m":122,"g":121},"12346":{"m":122,"g":121},"12325":{"m":122,"g":121},"12095":{"m":122,"g":121},"12347":{"m":122,"g":121},"12345":{"m":122,"g":121},"11709":{"m":122,"g":121},"12343":{"m":122,"g":121},"12315":{"m":122,"g":121},"12338":{"m":122,"g":121},"11673":{"m":122,"g":121},"12002":{"m":122,"g":121},"12336":{"m":122,"g":121},"12312":{"m":122,"g":121},"12317":{"m":122,"g":121},"12276":{"m":122,"g":121},"12294":{"m":122,"g":121},"12314":{"m":122,"g":121},"12144":{"m":122,"g":121},"10874":{"m":122,"g":121},"12269":{"m":122,"g":121},"9825":{"m":122,"g":121},"12313":{"m":122,"g":121},"12311":{"m":122,"g":121},"12259":{"m":122,"g":121},"12241":{"m":122,"g":121},"12308":{"m":122,"g":121},"12299":{"m":122,"g":121},"12271":{"m":122,"g":121},"12296":{"m":122,"g":121},"12295":{"m":122,"g":121},"12285":{"m":122,"g":121},"12233":{"m":122,"g":121},"11928":{"m":122,"g":121},"12188":{"m":122,"g":121},"12283":{"m":122,"g":121},"12231":{"m":122,"g":121},"12284":{"m":122,"g":121},"12274":{"m":122,"g":121},"12257":{"m":122,"g":121},"12268":{"m":122,"g":121},"12267":{"m":122,"g":121},"12206":{"m":122,"g":121},"12247":{"m":122,"g":121},"10804":{"m":122,"g":121},"12230":{"m":122,"g":121},"12229":{"m":122,"g":121},"12252":{"m":122,"g":121},"12249":{"m":122,"g":121},"7873":{"m":122,"g":121},"10567":{"m":122,"g":121},"11177":{"m":122,"g":121},"11655":{"m":122,"g":121},"12245":{"m":122,"g":121},"11517":{"m":122,"g":121},"10654":{"m":122,"g":121},"12222":{"m":122,"g":121},"11994":{"m":122,"g":121},"12176":{"m":122,"g":121},"11708":{"m":122,"g":121},"12235":{"m":122,"g":121},"12161":{"m":122,"g":121},"12234":{"m":122,"g":121},"11142":{"m":122,"g":121},"12006":{"m":122,"g":121},"11592":{"m":122,"g":121},"11656":{"m":122,"g":121},"12186":{"m":122,"g":121},"12209":{"m":122,"g":121},"12205":{"m":122,"g":121},"12107":{"m":122,"g":121},"12112":{"m":122,"g":121},"10153":{"m":122,"g":121},"12117":{"m":122,"g":121},"12080":{"m":122,"g":121},"9403":{"m":122,"g":121},"12192":{"m":122,"g":121},"12173":{"m":122,"g":121},"12159":{"m":122,"g":121},"12057":{"m":122,"g":121},"12639":{"m":123,"g":127},"12572":{"m":123,"g":127},"12656":{"m":123,"g":127},"12456":{"m":123,"g":127},"12585":{"m":123,"g":127},"12648":{"m":123,"g":127},"12650":{"m":123,"g":127},"12640":{"m":123,"g":127},"12645":{"m":123,"g":127},"12647":{"m":123,"g":127},"12642":{"m":123,"g":127},"12641":{"m":123,"g":127},"12628":{"m":123,"g":127},"12634":{"m":123,"g":127},"12633":{"m":123,"g":127},"12632":{"m":123,"g":127},"12593":{"m":123,"g":127},"12616":{"m":123,"g":127},"12594":{"m":123,"g":127},"12592":{"m":123,"g":127},"11456":{"m":123,"g":127},"12615":{"m":123,"g":127},"12599":{"m":123,"g":127},"12522":{"m":123,"g":127},"10183":{"m":123,"g":127},"12598":{"m":123,"g":127},"6318":{"m":123,"g":127},"11131":{"m":123,"g":127},"11974":{"m":123,"g":127},"12580":{"m":123,"g":127},"12597":{"m":123,"g":127},"11760":{"m":123,"g":127},"12462":{"m":123,"g":127},"12165":{"m":123,"g":127},"12111":{"m":123,"g":127},"12547":{"m":123,"g":127},"12270":{"m":123,"g":127},"12044":{"m":123,"g":127},"12519":{"m":123,"g":127},"12571":{"m":123,"g":127},"12301":{"m":123,"g":127},"12569":{"m":123,"g":127},"12550":{"m":123,"g":127},"12549":{"m":123,"g":127},"12553":{"m":123,"g":127},"12524":{"m":123,"g":127},"12560":{"m":123,"g":127},"12227":{"m":123,"g":127},"12548":{"m":123,"g":127},"11330":{"m":123,"g":127},"12564":{"m":123,"g":127},"12561":{"m":123,"g":127},"12060":{"m":123,"g":127},"12536":{"m":123,"g":127},"12541":{"m":123,"g":127},"12532":{"m":123,"g":127},"12530":{"m":123,"g":127},"12523":{"m":123,"g":127},"12367":{"m":123,"g":127},"12502":{"m":123,"g":127},"12521":{"m":123,"g":127},"12515":{"m":123,"g":127},"12453":{"m":123,"g":127},"12481":{"m":123,"g":127},"12506":{"m":123,"g":127},"11917":{"m":123,"g":127},"12511":{"m":123,"g":127},"10078":{"m":123,"g":127},"12505":{"m":123,"g":127},"12507":{"m":123,"g":127},"11133":{"m":123,"g":127},"11052":{"m":123,"g":127},"12499":{"m":123,"g":127},"12391":{"m":123,"g":127},"12488":{"m":123,"g":127},"12412":{"m":123,"g":127},"11966":{"m":123,"g":127},"12238":{"m":123,"g":127},"12423":{"m":123,"g":127},"12500":{"m":123,"g":127},"12480":{"m":123,"g":127},"12485":{"m":123,"g":127},"12435":{"m":123,"g":127},"12483":{"m":123,"g":127},"12482":{"m":123,"g":127},"12226":{"m":123,"g":127},"12334":{"m":123,"g":127},"12739":{"m":124,"g":127},"12778":{"m":124,"g":127},"12440":{"m":124,"g":127},"12760":{"m":124,"g":127},"12565":{"m":124,"g":127},"12674":{"m":124,"g":127},"12646":{"m":124,"g":127},"12240":{"m":124,"g":127},"12508":{"m":124,"g":127},"12737":{"m":124,"g":127},"12693":{"m":124,"g":127},"12752":{"m":124,"g":127},"12744":{"m":124,"g":127},"12736":{"m":124,"g":127},"12748":{"m":124,"g":127},"12741":{"m":124,"g":127},"12742":{"m":124,"g":127},"12738":{"m":124,"g":127},"11892":{"m":124,"g":127},"12721":{"m":124,"g":127},"12734":{"m":124,"g":127},"12723":{"m":124,"g":127},"12716":{"m":124,"g":127},"12732":{"m":124,"g":127},"12611":{"m":124,"g":127},"12728":{"m":124,"g":127},"12729":{"m":124,"g":127},"12718":{"m":124,"g":127},"12651":{"m":124,"g":127},"12711":{"m":124,"g":127},"12713":{"m":124,"g":127},"12714":{"m":124,"g":127},"12712":{"m":124,"g":127},"12658":{"m":124,"g":127},"12673":{"m":124,"g":127},"12710":{"m":124,"g":127},"12709":{"m":124,"g":127},"12406":{"m":124,"g":127},"12586":{"m":124,"g":127},"12631":{"m":124,"g":127},"12699":{"m":124,"g":127},"12484":{"m":124,"g":127},"12677":{"m":124,"g":127},"12696":{"m":124,"g":127},"12708":{"m":124,"g":127},"12609":{"m":124,"g":127},"12702":{"m":124,"g":127},"12455":{"m":124,"g":127},"12691":{"m":124,"g":127},"12687":{"m":124,"g":127},"11641":{"m":124,"g":127},"12680":{"m":124,"g":127},"10044":{"m":124,"g":127},"12668":{"m":124,"g":127},"12175":{"m":124,"g":127},"12670":{"m":124,"g":127},"8784":{"m":124,"g":127},"12486":{"m":124,"g":127},"12638":{"m":124,"g":127},"12353":{"m":124,"g":127},"13000":{"m":125,"g":127},"12908":{"m":125,"g":127},"12952":{"m":125,"g":127},"13010":{"m":125,"g":127},"11850":{"m":125,"g":127},"12781":{"m":125,"g":127},"12224":{"m":125,"g":127},"13013":{"m":125,"g":127},"13009":{"m":125,"g":127},"13001":{"m":125,"g":127},"13005":{"m":125,"g":127},"12996":{"m":125,"g":127},"12999":{"m":125,"g":127},"12966":{"m":125,"g":127},"12984":{"m":125,"g":127},"12982":{"m":125,"g":127},"10225":{"m":125,"g":127},"10702":{"m":125,"g":127},"11719":{"m":125,"g":127},"12916":{"m":125,"g":127},"12239":{"m":125,"g":127},"12883":{"m":125,"g":127},"12912":{"m":125,"g":127},"12604":{"m":125,"g":127},"12934":{"m":125,"g":127},"9528":{"m":125,"g":127},"12931":{"m":125,"g":127},"12959":{"m":125,"g":127},"12926":{"m":125,"g":127},"12803":{"m":125,"g":127},"12957":{"m":125,"g":127},"12943":{"m":125,"g":127},"12834":{"m":125,"g":127},"12956":{"m":125,"g":127},"12554":{"m":125,"g":127},"12946":{"m":125,"g":127},"12948":{"m":125,"g":127},"12940":{"m":125,"g":127},"11812":{"m":125,"g":127},"12928":{"m":125,"g":127},"12927":{"m":125,"g":127},"12839":{"m":125,"g":127},"10775":{"m":125,"g":127},"12917":{"m":125,"g":127},"12920":{"m":125,"g":127},"12332":{"m":125,"g":127},"12919":{"m":125,"g":127},"12448":{"m":125,"g":127},"12906":{"m":125,"g":127},"12907":{"m":125,"g":127},"12911":{"m":125,"g":127},"12895":{"m":125,"g":127},"12905":{"m":125,"g":127},"12904":{"m":125,"g":127},"12865":{"m":125,"g":127},"12900":{"m":125,"g":127},"12896":{"m":125,"g":127},"12891":{"m":125,"g":127},"12897":{"m":125,"g":127},"12889":{"m":125,"g":127},"12870":{"m":125,"g":127},"12361":{"m":125,"g":127},"12811":{"m":125,"g":127},"12888":{"m":125,"g":127},"12832":{"m":125,"g":127},"12843":{"m":125,"g":127},"12886":{"m":125,"g":127},"12798":{"m":125,"g":127},"12868":{"m":125,"g":127},"12853":{"m":125,"g":127},"12846":{"m":125,"g":127},"12582":{"m":125,"g":127},"12805":{"m":125,"g":127},"12849":{"m":125,"g":127},"12859":{"m":125,"g":127},"12852":{"m":125,"g":127},"12851":{"m":125,"g":127},"12856":{"m":125,"g":127},"12801":{"m":125,"g":127},"12822":{"m":125,"g":127},"12431":{"m":125,"g":127},"12825":{"m":125,"g":127},"12836":{"m":125,"g":127},"12374":{"m":125,"g":127},"12812":{"m":125,"g":127},"12816":{"m":125,"g":127},"12090":{"m":125,"g":127},"12758":{"m":125,"g":127},"12520":{"m":125,"g":127},"12765":{"m":125,"g":127},"12761":{"m":125,"g":127},"12763":{"m":125,"g":127},"12776":{"m":125,"g":127},"12772":{"m":125,"g":127},"12788":{"m":125,"g":127},"12576":{"m":125,"g":127},"12782":{"m":125,"g":127},"12794":{"m":125,"g":127},"12715":{"m":125,"g":127},"12795":{"m":125,"g":127},"12724":{"m":125,"g":127},"12279":{"m":125,"g":127},"12717":{"m":125,"g":127},"8243":{"m":125,"g":127},"11904":{"m":125,"g":127},"12684":{"m":125,"g":127},"12764":{"m":125,"g":127},"12363":{"m":125,"g":127},"11051":{"m":125,"g":127},"12749":{"m":125,"g":127},"13129":{"m":126,"g":127},"10808":{"m":126,"g":127},"12617":{"m":126,"g":127},"7906":{"m":126,"g":127},"13149":{"m":126,"g":127},"7886":{"m":126,"g":127},"9790":{"m":126,"g":127},"13120":{"m":126,"g":127},"12458":{"m":126,"g":127},"11961":{"m":126,"g":127},"13137":{"m":126,"g":127},"12860":{"m":126,"g":127},"13136":{"m":126,"g":127},"13135":{"m":126,"g":127},"13077":{"m":126,"g":127},"13132":{"m":126,"g":127},"13131":{"m":126,"g":127},"13133":{"m":126,"g":127},"12942":{"m":126,"g":127},"12817":{"m":126,"g":127},"13095":{"m":126,"g":127},"12666":{"m":126,"g":127},"12396":{"m":126,"g":127},"13118":{"m":126,"g":127},"12872":{"m":126,"g":127},"12997":{"m":126,"g":127},"12863":{"m":126,"g":127},"13114":{"m":126,"g":127},"13039":{"m":126,"g":127},"11856":{"m":126,"g":127},"13105":{"m":126,"g":127},"13056":{"m":126,"g":127},"12583":{"m":126,"g":127},"13093":{"m":126,"g":127},"12660":{"m":126,"g":127},"13090":{"m":126,"g":127},"12866":{"m":126,"g":127},"12915":{"m":126,"g":127},"13018":{"m":126,"g":127},"13092":{"m":126,"g":127},"11645":{"m":126,"g":127},"13041":{"m":126,"g":127},"10862":{"m":126,"g":127},"13088":{"m":126,"g":127},"12814":{"m":126,"g":127},"12941":{"m":126,"g":127},"12994":{"m":126,"g":127},"13076":{"m":126,"g":127},"13015":{"m":126,"g":127},"13063":{"m":126,"g":127},"12199":{"m":126,"g":127},"12983":{"m":126,"g":127},"13037":{"m":126,"g":127},"12689":{"m":126,"g":127},"11609":{"m":126,"g":127},"12976":{"m":126,"g":127},"13050":{"m":126,"g":127},"12980":{"m":126,"g":127},"11938":{"m":126,"g":127},"13057":{"m":126,"g":127},"13053":{"m":126,"g":127},"13036":{"m":126,"g":127},"13043":{"m":126,"g":127},"13029":{"m":126,"g":127},"12885":{"m":126,"g":127},"12518":{"m":126,"g":127},"13035":{"m":126,"g":127},"13027":{"m":126,"g":127},"13028":{"m":126,"g":127},"12218":{"m":126,"g":127},"12753":{"m":126,"g":127},"12869":{"m":126,"g":127},"12993":{"m":126,"g":127},"13012":{"m":126,"g":127},"13366":{"m":128,"g":131},"13389":{"m":128,"g":131},"13387":{"m":128,"g":131},"12903":{"m":128,"g":131},"13388":{"m":128,"g":131},"12874":{"m":128,"g":131},"13386":{"m":128,"g":131},"13385":{"m":128,"g":131},"13384":{"m":128,"g":131},"13381":{"m":128,"g":131},"13335":{"m":128,"g":131},"13228":{"m":128,"g":131},"13371":{"m":128,"g":131},"13339":{"m":128,"g":131},"12978":{"m":128,"g":131},"13332":{"m":128,"g":131},"13263":{"m":128,"g":131},"13375":{"m":128,"g":131},"13344":{"m":128,"g":131},"13373":{"m":128,"g":131},"13372":{"m":128,"g":131},"13336":{"m":128,"g":131},"13369":{"m":128,"g":131},"13348":{"m":128,"g":131},"13199":{"m":128,"g":131},"13179":{"m":128,"g":131},"12310":{"m":128,"g":131},"11870":{"m":128,"g":131},"13358":{"m":128,"g":131},"13101":{"m":128,"g":131},"13321":{"m":128,"g":131},"13355":{"m":128,"g":131},"13351":{"m":128,"g":131},"13181":{"m":128,"g":131},"13341":{"m":128,"g":131},"13325":{"m":128,"g":131},"12329":{"m":128,"g":131},"13337":{"m":128,"g":131},"12001":{"m":128,"g":131},"12692":{"m":128,"g":131},"12443":{"m":128,"g":131},"13331":{"m":128,"g":131},"13330":{"m":128,"g":131},"13329":{"m":128,"g":131},"10568":{"m":128,"g":131},"13306":{"m":128,"g":131},"13297":{"m":128,"g":131},"13326":{"m":128,"g":131},"13323":{"m":128,"g":131},"13287":{"m":128,"g":131},"13322":{"m":128,"g":131},"7415":{"m":128,"g":131},"13286":{"m":128,"g":131},"13285":{"m":128,"g":131},"13259":{"m":128,"g":131},"13320":{"m":128,"g":131},"13295":{"m":128,"g":131},"13318":{"m":128,"g":131},"13314":{"m":128,"g":131},"13226":{"m":128,"g":131},"13278":{"m":128,"g":131},"13279":{"m":128,"g":131},"12612":{"m":128,"g":131},"12871":{"m":128,"g":131},"13317":{"m":128,"g":131},"13294":{"m":128,"g":131},"13316":{"m":128,"g":131},"13315":{"m":128,"g":131},"13312":{"m":128,"g":127},"13091":{"m":128,"g":127},"13100":{"m":128,"g":127},"13311":{"m":128,"g":127},"13310":{"m":128,"g":127},"13045":{"m":128,"g":127},"13293":{"m":128,"g":127},"13305":{"m":128,"g":127},"13170":{"m":128,"g":127},"13298":{"m":128,"g":127},"13274":{"m":128,"g":127},"13235":{"m":128,"g":127},"13272":{"m":128,"g":127},"13260":{"m":128,"g":127},"10573":{"m":128,"g":127},"10665":{"m":128,"g":127},"13236":{"m":128,"g":127},"12777":{"m":128,"g":127},"13277":{"m":128,"g":127},"13288":{"m":128,"g":127},"13254":{"m":128,"g":127},"13284":{"m":128,"g":127},"13283":{"m":128,"g":127},"13242":{"m":128,"g":127},"12191":{"m":128,"g":127},"12605":{"m":128,"g":127},"12623":{"m":128,"g":127},"12622":{"m":128,"g":127},"12620":{"m":128,"g":127},"13113":{"m":128,"g":127},"13247":{"m":128,"g":127},"13237":{"m":128,"g":127},"13265":{"m":128,"g":127},"11589":{"m":128,"g":127},"13261":{"m":128,"g":127},"13256":{"m":128,"g":127},"13257":{"m":128,"g":127},"13255":{"m":128,"g":127},"13096":{"m":128,"g":127},"13221":{"m":128,"g":127},"13213":{"m":128,"g":127},"13246":{"m":128,"g":127},"13243":{"m":128,"g":127},"13186":{"m":128,"g":127},"13239":{"m":128,"g":127},"13097":{"m":128,"g":127},"12392":{"m":128,"g":127},"13222":{"m":128,"g":127},"13188":{"m":128,"g":127},"13218":{"m":128,"g":127},"13087":{"m":128,"g":127},"13142":{"m":128,"g":127},"11595":{"m":128,"g":127},"13210":{"m":128,"g":127},"12774":{"m":128,"g":127},"13211":{"m":128,"g":127},"13220":{"m":128,"g":127},"13171":{"m":128,"g":127},"13215":{"m":128,"g":127},"13212":{"m":128,"g":127},"10485":{"m":128,"g":127},"12543":{"m":128,"g":127},"12201":{"m":128,"g":127},"12376":{"m":128,"g":127},"13155":{"m":128,"g":127},"13148":{"m":128,"g":127},"13190":{"m":128,"g":127},"13178":{"m":128,"g":127},"13102":{"m":128,"g":127},"13154":{"m":128,"g":127},"12975":{"m":128,"g":127},"13163":{"m":128,"g":127},"13172":{"m":128,"g":127},"13150":{"m":128,"g":127},"10973":{"m":128,"g":127},"12288":{"m":128,"g":127},"13162":{"m":128,"g":127},"12215":{"m":128,"g":127},"13104":{"m":128,"g":127},"13164":{"m":128,"g":127},"13127":{"m":128,"g":127},"12979":{"m":128,"g":127},"13128":{"m":128,"g":127},"12998":{"m":128,"g":127},"13075":{"m":128,"g":127},"10907":{"m":128,"g":127},"13153":{"m":128,"g":127},"12214":{"m":128,"g":127},"14316":{"m":129,"g":131},"14324":{"m":129,"g":131},"14323":{"m":129,"g":131},"14317":{"m":129,"g":131},"14309":{"m":129,"g":131},"14319":{"m":129,"g":131},"14262":{"m":129,"g":131},"14315":{"m":129,"g":131},"14249":{"m":129,"g":131},"11423":{"m":129,"g":131},"14278":{"m":129,"g":131},"14299":{"m":129,"g":131},"13089":{"m":129,"g":131},"14133":{"m":129,"g":131},"14269":{"m":129,"g":131},"14281":{"m":129,"g":131},"14287":{"m":129,"g":131},"14286":{"m":129,"g":131},"14283":{"m":129,"g":131},"14252":{"m":129,"g":131},"14244":{"m":129,"g":131},"14279":{"m":129,"g":131},"14276":{"m":129,"g":131},"13738":{"m":129,"g":131},"14047":{"m":129,"g":131},"14274":{"m":129,"g":131},"14257":{"m":129,"g":131},"14254":{"m":129,"g":131},"13700":{"m":129,"g":131},"14267":{"m":129,"g":131},"14261":{"m":129,"g":131},"14172":{"m":129,"g":131},"14263":{"m":129,"g":131},"14259":{"m":129,"g":131},"14260":{"m":129,"g":131},"14222":{"m":129,"g":131},"14232":{"m":129,"g":131},"13968":{"m":129,"g":131},"14256":{"m":129,"g":131},"14255":{"m":129,"g":131},"13880":{"m":129,"g":131},"13794":{"m":129,"g":131},"14247":{"m":129,"g":131},"13843":{"m":129,"g":131},"14250":{"m":129,"g":131},"14245":{"m":129,"g":131},"14243":{"m":129,"g":131},"14241":{"m":129,"g":131},"14240":{"m":129,"g":131},"14237":{"m":129,"g":131},"14152":{"m":129,"g":131},"14179":{"m":129,"g":131},"14229":{"m":129,"g":131},"14230":{"m":129,"g":131},"13693":{"m":129,"g":131},"14228":{"m":129,"g":131},"14122":{"m":129,"g":131},"14088":{"m":129,"g":131},"13887":{"m":129,"g":131},"14219":{"m":129,"g":131},"14218":{"m":129,"g":131},"14214":{"m":129,"g":131},"14212":{"m":129,"g":131},"14211":{"m":129,"g":131},"14173":{"m":129,"g":131},"14165":{"m":129,"g":131},"14123":{"m":129,"g":131},"14186":{"m":129,"g":131},"14180":{"m":129,"g":131},"14167":{"m":129,"g":131},"14044":{"m":129,"g":131},"14182":{"m":129,"g":131},"14183":{"m":129,"g":131},"14181":{"m":129,"g":131},"14003":{"m":129,"g":131},"14034":{"m":129,"g":131},"14153":{"m":129,"g":131},"14187":{"m":129,"g":131},"12181":{"m":129,"g":131},"14155":{"m":129,"g":131},"14104":{"m":129,"g":131},"13873":{"m":129,"g":131},"14059":{"m":129,"g":131},"14140":{"m":129,"g":131},"13646":{"m":129,"g":131},"13841":{"m":129,"g":131},"14171":{"m":129,"g":131},"13907":{"m":129,"g":131},"14065":{"m":129,"g":131},"14166":{"m":129,"g":131},"14005":{"m":129,"g":131},"12494":{"m":129,"g":131},"14163":{"m":129,"g":131},"14156":{"m":129,"g":131},"14148":{"m":129,"g":131},"14161":{"m":129,"g":131},"14052":{"m":129,"g":131},"14150":{"m":129,"g":131},"14157":{"m":129,"g":131},"14154":{"m":129,"g":131},"14147":{"m":129,"g":131},"14146":{"m":129,"g":131},"14145":{"m":129,"g":131},"14136":{"m":129,"g":131},"13956":{"m":129,"g":131},"13759":{"m":129,"g":131},"14151":{"m":129,"g":131},"14135":{"m":129,"g":131},"14130":{"m":129,"g":131},"14119":{"m":129,"g":131},"14131":{"m":129,"g":131},"14129":{"m":129,"g":131},"14121":{"m":129,"g":131},"12306":{"m":129,"g":131},"14124":{"m":129,"g":131},"14113":{"m":129,"g":131},"14117":{"m":129,"g":131},"13377":{"m":129,"g":131},"13488":{"m":129,"g":131},"14111":{"m":129,"g":131},"12558":{"m":129,"g":131},"10712":{"m":129,"g":131},"14106":{"m":129,"g":131},"14067":{"m":129,"g":131},"14096":{"m":129,"g":131},"14094":{"m":129,"g":131},"13724":{"m":129,"g":131},"13904":{"m":129,"g":131},"14076":{"m":129,"g":131},"13936":{"m":129,"g":131},"14006":{"m":129,"g":131},"14082":{"m":129,"g":131},"13205":{"m":129,"g":131},"14079":{"m":129,"g":131},"14036":{"m":129,"g":131},"13944":{"m":129,"g":131},"13749":{"m":129,"g":131},"14069":{"m":129,"g":131},"13946":{"m":129,"g":131},"14048":{"m":129,"g":131},"13960":{"m":129,"g":131},"14057":{"m":129,"g":131},"13425":{"m":129,"g":131},"13854":{"m":129,"g":131},"14002":{"m":129,"g":131},"13855":{"m":129,"g":131},"14040":{"m":129,"g":131},"13895":{"m":129,"g":131},"13976":{"m":129,"g":131},"13814":{"m":129,"g":131},"13965":{"m":129,"g":131},"14026":{"m":129,"g":131},"14017":{"m":129,"g":131},"14033":{"m":129,"g":131},"14030":{"m":129,"g":131},"14028":{"m":129,"g":131},"13761":{"m":129,"g":131},"14027":{"m":129,"g":131},"14025":{"m":129,"g":131},"13824":{"m":129,"g":131},"12277":{"m":129,"g":131},"13941":{"m":129,"g":131},"13983":{"m":129,"g":131},"14018":{"m":129,"g":131},"14007":{"m":129,"g":131},"14022":{"m":129,"g":131},"14019":{"m":129,"g":131},"14021":{"m":129,"g":131},"13937":{"m":129,"g":131},"13990":{"m":129,"g":131},"14020":{"m":129,"g":131},"13966":{"m":129,"g":131},"13872":{"m":129,"g":131},"14016":{"m":129,"g":131},"14013":{"m":129,"g":131},"14015":{"m":129,"g":131},"14014":{"m":129,"g":131},"14012":{"m":129,"g":131},"13151":{"m":129,"g":131},"13892":{"m":129,"g":131},"14000":{"m":129,"g":131},"14009":{"m":129,"g":131},"13766":{"m":129,"g":131},"13754":{"m":129,"g":131},"12491":{"m":129,"g":131},"13994":{"m":129,"g":131},"13991":{"m":129,"g":131},"13977":{"m":129,"g":131},"13203":{"m":129,"g":131},"12588":{"m":129,"g":131},"13922":{"m":129,"g":131},"13852":{"m":129,"g":131},"13963":{"m":129,"g":131},"10071":{"m":129,"g":131},"13961":{"m":129,"g":131},"13962":{"m":129,"g":131},"13958":{"m":129,"g":131},"7725":{"m":129,"g":131},"13925":{"m":129,"g":131},"12786":{"m":129,"g":131},"13954":{"m":129,"g":131},"13951":{"m":129,"g":131},"13950":{"m":129,"g":131},"13945":{"m":129,"g":131},"13942":{"m":129,"g":131},"12969":{"m":129,"g":131},"13910":{"m":129,"g":131},"13866":{"m":129,"g":131},"13903":{"m":129,"g":131},"13421":{"m":129,"g":131},"13851":{"m":129,"g":131},"13544":{"m":129,"g":131},"13938":{"m":129,"g":131},"13935":{"m":129,"g":131},"13933":{"m":129,"g":131},"13859":{"m":129,"g":131},"13931":{"m":129,"g":131},"13928":{"m":129,"g":131},"13927":{"m":129,"g":131},"13657":{"m":129,"g":131},"13081":{"m":129,"g":131},"12078":{"m":129,"g":131},"13921":{"m":129,"g":131},"13905":{"m":129,"g":131},"13916":{"m":129,"g":131},"13642":{"m":129,"g":131},"13908":{"m":129,"g":131},"13848":{"m":129,"g":131},"13793":{"m":129,"g":131},"13901":{"m":129,"g":131},"13827":{"m":129,"g":131},"13889":{"m":129,"g":131},"13890":{"m":129,"g":131},"13888":{"m":129,"g":131},"11893":{"m":129,"g":131},"13891":{"m":129,"g":131},"13870":{"m":129,"g":131},"13874":{"m":129,"g":131},"13860":{"m":129,"g":131},"13871":{"m":129,"g":131},"13572":{"m":129,"g":131},"10275":{"m":129,"g":131},"13487":{"m":129,"g":131},"13786":{"m":129,"g":131},"13834":{"m":129,"g":131},"13822":{"m":129,"g":131},"13783":{"m":129,"g":131},"13763":{"m":129,"g":131},"13752":{"m":129,"g":131},"13864":{"m":129,"g":131},"13865":{"m":129,"g":131},"13853":{"m":129,"g":131},"10027":{"m":129,"g":131},"13745":{"m":129,"g":131},"13858":{"m":129,"g":131},"13612":{"m":129,"g":131},"11871":{"m":129,"g":131},"13713":{"m":129,"g":131},"13508":{"m":129,"g":131},"13846":{"m":129,"g":131},"13819":{"m":129,"g":131},"13245":{"m":129,"g":131},"13751":{"m":129,"g":131},"13833":{"m":129,"g":131},"13831":{"m":129,"g":131},"13792":{"m":129,"g":131},"13829":{"m":129,"g":131},"13800":{"m":129,"g":131},"13656":{"m":129,"g":131},"13820":{"m":129,"g":131},"13201":{"m":129,"g":131},"13650":{"m":129,"g":131},"13816":{"m":129,"g":131},"13601":{"m":129,"g":131},"13810":{"m":129,"g":131},"13802":{"m":129,"g":131},"13815":{"m":129,"g":131},"13813":{"m":129,"g":131},"13687":{"m":129,"g":131},"13806":{"m":129,"g":131},"13781":{"m":129,"g":131},"13791":{"m":129,"g":131},"13180":{"m":129,"g":131},"13718":{"m":129,"g":131},"13787":{"m":129,"g":131},"13764":{"m":129,"g":131},"13709":{"m":129,"g":131},"13776":{"m":129,"g":131},"13777":{"m":129,"g":131},"13727":{"m":129,"g":131},"13676":{"m":129,"g":131},"13690":{"m":129,"g":131},"13547":{"m":129,"g":131},"13533":{"m":129,"g":131},"13769":{"m":129,"g":131},"13478":{"m":129,"g":131},"13736":{"m":129,"g":131},"13706":{"m":129,"g":131},"13768":{"m":129,"g":131},"13704":{"m":129,"g":131},"13720":{"m":129,"g":131},"13714":{"m":129,"g":131},"12759":{"m":129,"g":131},"9405":{"m":129,"g":131},"13506":{"m":129,"g":131},"13756":{"m":129,"g":131},"13669":{"m":129,"g":131},"13729":{"m":129,"g":131},"13702":{"m":129,"g":131},"13746":{"m":129,"g":131},"13484":{"m":129,"g":131},"12690":{"m":129,"g":131},"13701":{"m":129,"g":131},"12949":{"m":129,"g":131},"13707":{"m":129,"g":131},"13694":{"m":129,"g":131},"13466":{"m":129,"g":131},"13739":{"m":129,"g":131},"13737":{"m":129,"g":131},"13735":{"m":129,"g":131},"13734":{"m":129,"g":131},"13733":{"m":129,"g":131},"13407":{"m":129,"g":131},"13649":{"m":129,"g":131},"13705":{"m":129,"g":131},"13630":{"m":129,"g":131},"13719":{"m":129,"g":131},"13327":{"m":129,"g":131},"13647":{"m":129,"g":131},"13708":{"m":129,"g":131},"13498":{"m":129,"g":131},"13590":{"m":129,"g":131},"13679":{"m":129,"g":131},"13564":{"m":129,"g":131},"13686":{"m":129,"g":131},"13177":{"m":129,"g":131},"13665":{"m":129,"g":131},"13675":{"m":129,"g":131},"13640":{"m":129,"g":131},"13587":{"m":129,"g":131},"13596":{"m":129,"g":131},"13555":{"m":129,"g":131},"13619":{"m":129,"g":131},"13301":{"m":129,"g":131},"12672":{"m":129,"g":131},"13683":{"m":129,"g":131},"13685":{"m":129,"g":131},"13627":{"m":129,"g":131},"13678":{"m":129,"g":131},"13677":{"m":129,"g":131},"13659":{"m":129,"g":131},"13667":{"m":129,"g":131},"13610":{"m":129,"g":131},"11526":{"m":129,"g":131},"13038":{"m":129,"g":131},"13600":{"m":129,"g":131},"13459":{"m":129,"g":131},"13666":{"m":129,"g":131},"12964":{"m":129,"g":131},"13655":{"m":129,"g":131},"13524":{"m":129,"g":131},"11577":{"m":129,"g":131},"13663":{"m":129,"g":131},"13637":{"m":129,"g":131},"13634":{"m":129,"g":131},"13644":{"m":129,"g":131},"13197":{"m":129,"g":131},"13617":{"m":129,"g":131},"13633":{"m":129,"g":131},"13453":{"m":129,"g":131},"13614":{"m":129,"g":131},"13554":{"m":129,"g":131},"13583":{"m":129,"g":131},"13328":{"m":129,"g":131},"13253":{"m":129,"g":131},"13248":{"m":129,"g":131},"12379":{"m":129,"g":131},"13528":{"m":129,"g":131},"13613":{"m":129,"g":131},"13429":{"m":129,"g":131},"13562":{"m":129,"g":131},"13055":{"m":129,"g":131},"13603":{"m":129,"g":131},"13604":{"m":129,"g":131},"13357":{"m":129,"g":131},"13570":{"m":129,"g":131},"13577":{"m":129,"g":131},"13465":{"m":129,"g":131},"13049":{"m":129,"g":131},"13448":{"m":129,"g":131},"13589":{"m":129,"g":131},"13567":{"m":129,"g":131},"13568":{"m":129,"g":131},"13413":{"m":129,"g":131},"13558":{"m":129,"g":131},"13557":{"m":129,"g":131},"13542":{"m":129,"g":131},"13481":{"m":129,"g":131},"12740":{"m":129,"g":131},"13551":{"m":129,"g":131},"13452":{"m":129,"g":131},"9234":{"m":129,"g":131},"13047":{"m":129,"g":131},"13548":{"m":129,"g":131},"13543":{"m":129,"g":131},"13541":{"m":129,"g":131},"13489":{"m":129,"g":131},"13495":{"m":129,"g":131},"13540":{"m":129,"g":131},"13537":{"m":129,"g":131},"13534":{"m":129,"g":131},"13536":{"m":129,"g":131},"13532":{"m":129,"g":131},"13527":{"m":129,"g":131},"13474":{"m":129,"g":131},"13525":{"m":129,"g":131},"13522":{"m":129,"g":131},"13521":{"m":129,"g":131},"13519":{"m":129,"g":131},"13516":{"m":129,"g":131},"12962":{"m":129,"g":131},"13513":{"m":129,"g":131},"13512":{"m":129,"g":131},"13510":{"m":129,"g":131},"13509":{"m":129,"g":131},"13168":{"m":129,"g":131},"13157":{"m":129,"g":131},"13501":{"m":129,"g":131},"13126":{"m":129,"g":131},"13496":{"m":129,"g":131},"13374":{"m":129,"g":131},"13491":{"m":129,"g":131},"13482":{"m":129,"g":131},"13486":{"m":129,"g":131},"13393":{"m":129,"g":131},"13460":{"m":129,"g":131},"13094":{"m":129,"g":131},"13479":{"m":129,"g":131},"13476":{"m":129,"g":131},"12149":{"m":129,"g":131},"13289":{"m":129,"g":131},"13473":{"m":129,"g":131},"13258":{"m":129,"g":131},"13229":{"m":129,"g":131},"13462":{"m":129,"g":131},"13444":{"m":129,"g":131},"13458":{"m":129,"g":131},"13449":{"m":129,"g":131},"13455":{"m":129,"g":131},"13140":{"m":129,"g":131},"13463":{"m":129,"g":131},"13264":{"m":129,"g":131},"12359":{"m":129,"g":131},"13461":{"m":129,"g":131},"13457":{"m":129,"g":131},"13173":{"m":129,"g":131},"13456":{"m":129,"g":131},"13273":{"m":129,"g":131},"13022":{"m":129,"g":131},"13450":{"m":129,"g":131},"13447":{"m":129,"g":131},"13445":{"m":129,"g":131},"13443":{"m":129,"g":131},"13418":{"m":129,"g":131},"13217":{"m":129,"g":131},"5879":{"m":129,"g":131},"11900":{"m":129,"g":131},"13144":{"m":129,"g":131},"13379":{"m":129,"g":131},"13282":{"m":129,"g":131},"13420":{"m":129,"g":131},"13416":{"m":129,"g":131},"13415":{"m":129,"g":131},"13399":{"m":129,"g":131},"13004":{"m":129,"g":131},"13324":{"m":129,"g":131},"13112":{"m":129,"g":131},"11644":{"m":129,"g":131},"13398":{"m":129,"g":131},"13396":{"m":129,"g":131},"13345":{"m":129,"g":131},"13338":{"m":129,"g":131},"12065":{"m":129,"g":131},"13391":{"m":129,"g":131},"13383":{"m":129,"g":131},"14670":{"m":130,"g":131},"14650":{"m":130,"g":131},"14457":{"m":130,"g":131},"14657":{"m":130,"g":131},"14667":{"m":130,"g":131},"14634":{"m":130,"g":131},"14664":{"m":130,"g":131},"14663":{"m":130,"g":131},"14658":{"m":130,"g":131},"14497":{"m":130,"g":131},"14651":{"m":130,"g":131},"14649":{"m":130,"g":131},"14356":{"m":130,"g":131},"14629":{"m":130,"g":131},"14558":{"m":130,"g":131},"12527":{"m":130,"g":131},"14632":{"m":130,"g":131},"14606":{"m":130,"g":131},"12551":{"m":130,"g":131},"14625":{"m":130,"g":131},"14556":{"m":130,"g":131},"14585":{"m":130,"g":131},"14618":{"m":130,"g":131},"14203":{"m":130,"g":131},"14452":{"m":130,"g":131},"14612":{"m":130,"g":131},"14609":{"m":130,"g":131},"14604":{"m":130,"g":131},"14608":{"m":130,"g":131},"14605":{"m":130,"g":131},"14600":{"m":130,"g":131},"14591":{"m":130,"g":131},"14386":{"m":130,"g":131},"14573":{"m":130,"g":131},"14551":{"m":130,"g":131},"14141":{"m":130,"g":131},"14590":{"m":130,"g":131},"14588":{"m":130,"g":131},"14587":{"m":130,"g":131},"14586":{"m":130,"g":131},"14517":{"m":130,"g":131},"14455":{"m":130,"g":131},"14553":{"m":130,"g":131},"13573":{"m":130,"g":131},"14132":{"m":130,"g":131},"14185":{"m":130,"g":131},"14576":{"m":130,"g":131},"14577":{"m":130,"g":131},"13725":{"m":130,"g":131},"13998":{"m":130,"g":131},"14569":{"m":130,"g":131},"14412":{"m":130,"g":131},"14544":{"m":130,"g":131},"14561":{"m":130,"g":131},"14560":{"m":130,"g":131},"14559":{"m":130,"g":131},"14337":{"m":130,"g":131},"14555":{"m":130,"g":131},"14494":{"m":130,"g":131},"14476":{"m":130,"g":131},"14205":{"m":130,"g":131},"14557":{"m":130,"g":131},"14447":{"m":130,"g":131},"14552":{"m":130,"g":131},"14518":{"m":130,"g":131},"14538":{"m":130,"g":131},"14520":{"m":130,"g":131},"14535":{"m":130,"g":131},"14493":{"m":130,"g":131},"14464":{"m":130,"g":131},"14543":{"m":130,"g":131},"13897":{"m":130,"g":131},"14505":{"m":130,"g":131},"14539":{"m":130,"g":131},"13115":{"m":130,"g":131},"14533":{"m":130,"g":131},"12324":{"m":130,"g":131},"14290":{"m":130,"g":131},"14528":{"m":130,"g":131},"14530":{"m":130,"g":131},"11791":{"m":130,"g":131},"14522":{"m":130,"g":131},"14465":{"m":130,"g":131},"14521":{"m":130,"g":131},"14291":{"m":130,"g":131},"14427":{"m":130,"g":131},"14516":{"m":130,"g":131},"14514":{"m":130,"g":131},"14513":{"m":130,"g":131},"14512":{"m":130,"g":131},"14312":{"m":130,"g":131},"14405":{"m":130,"g":131},"14420":{"m":130,"g":131},"14460":{"m":130,"g":131},"14508":{"m":130,"g":131},"13607":{"m":130,"g":131},"14507":{"m":130,"g":131},"12471":{"m":130,"g":131},"14093":{"m":130,"g":131},"14234":{"m":130,"g":131},"13434":{"m":130,"g":131},"14506":{"m":130,"g":131},"14471":{"m":130,"g":131},"14459":{"m":130,"g":131},"14466":{"m":130,"g":131},"14456":{"m":130,"g":131},"14097":{"m":130,"g":131},"14499":{"m":130,"g":131},"13584":{"m":130,"g":131},"14364":{"m":130,"g":131},"13861":{"m":130,"g":131},"13996":{"m":130,"g":131},"14472":{"m":130,"g":131},"14484":{"m":130,"g":131},"14444":{"m":130,"g":131},"14475":{"m":130,"g":131},"14463":{"m":130,"g":131},"14473":{"m":130,"g":131},"14468":{"m":130,"g":131},"13836":{"m":130,"g":131},"14421":{"m":130,"g":131},"14432":{"m":130,"g":131},"14251":{"m":130,"g":131},"14450":{"m":130,"g":131},"14445":{"m":130,"g":131},"14446":{"m":130,"g":131},"8287":{"m":130,"g":131},"14348":{"m":130,"g":131},"14350":{"m":130,"g":131},"14441":{"m":130,"g":131},"14325":{"m":130,"g":131},"14440":{"m":130,"g":131},"14438":{"m":130,"g":131},"14143":{"m":130,"g":131},"14434":{"m":130,"g":131},"14366":{"m":130,"g":131},"14430":{"m":130,"g":131},"14429":{"m":130,"g":131},"14225":{"m":130,"g":131},"14409":{"m":130,"g":131},"14213":{"m":130,"g":131},"14224":{"m":130,"g":131},"14334":{"m":130,"g":131},"14399":{"m":130,"g":131},"12446":{"m":130,"g":131},"13359":{"m":130,"g":131},"14383":{"m":130,"g":131},"14394":{"m":130,"g":131},"14381":{"m":130,"g":131},"12309":{"m":130,"g":131},"14393":{"m":130,"g":131},"12316":{"m":130,"g":131},"14292":{"m":130,"g":131},"14392":{"m":130,"g":131},"14272":{"m":130,"g":131},"13731":{"m":130,"g":131},"14359":{"m":130,"g":131},"14377":{"m":130,"g":131},"14330":{"m":130,"g":131},"14277":{"m":130,"g":131},"14375":{"m":130,"g":131},"14374":{"m":130,"g":131},"14253":{"m":130,"g":131},"14372":{"m":130,"g":131},"14226":{"m":130,"g":131},"14371":{"m":130,"g":131},"14326":{"m":130,"g":131},"9660":{"m":130,"g":131},"12330":{"m":130,"g":131},"14355":{"m":130,"g":131},"13585":{"m":130,"g":131},"14362":{"m":130,"g":131},"14271":{"m":130,"g":131},"14295":{"m":130,"g":131},"13980":{"m":130,"g":131},"14347":{"m":130,"g":131},"14333":{"m":130,"g":131},"12441":{"m":130,"g":131},"14344":{"m":130,"g":131},"14265":{"m":130,"g":131},"14335":{"m":130,"g":131},"14336":{"m":130,"g":131},"13350":{"m":130,"g":131},"14266":{"m":130,"g":131},"14329":{"m":130,"g":131},"13812":{"m":130,"g":131},"14195":{"m":130,"g":131},"14321":{"m":130,"g":131},"13710":{"m":130,"g":131},"14858":{"m":132,"g":133},"14620":{"m":132,"g":133},"14304":{"m":132,"g":133},"14917":{"m":132,"g":133},"14307":{"m":132,"g":133},"14887":{"m":132,"g":133},"14911":{"m":132,"g":133},"14910":{"m":132,"g":133},"14852":{"m":132,"g":133},"14889":{"m":132,"g":133},"14890":{"m":132,"g":133},"14900":{"m":132,"g":133},"14899":{"m":132,"g":133},"12287":{"m":132,"g":133},"14878":{"m":132,"g":133},"14541":{"m":132,"g":133},"13641":{"m":132,"g":133},"14828":{"m":132,"g":133},"14827":{"m":132,"g":133},"14853":{"m":132,"g":133},"14876":{"m":132,"g":133},"14880":{"m":132,"g":133},"14877":{"m":132,"g":133},"14856":{"m":132,"g":133},"14875":{"m":132,"g":133},"14861":{"m":132,"g":133},"14845":{"m":132,"g":133},"14871":{"m":132,"g":133},"14313":{"m":132,"g":133},"14554":{"m":132,"g":133},"14865":{"m":132,"g":133},"14811":{"m":132,"g":133},"14836":{"m":132,"g":133},"14848":{"m":132,"g":133},"14854":{"m":132,"g":133},"14849":{"m":132,"g":133},"14844":{"m":132,"g":133},"14851":{"m":132,"g":133},"14850":{"m":132,"g":133},"14847":{"m":132,"g":133},"14442":{"m":132,"g":133},"14841":{"m":132,"g":133},"14796":{"m":132,"g":133},"14638":{"m":132,"g":133},"14823":{"m":132,"g":133},"14801":{"m":132,"g":133},"14837":{"m":132,"g":133},"14045":{"m":132,"g":133},"14833":{"m":132,"g":133},"14829":{"m":132,"g":133},"14769":{"m":132,"g":133},"14712":{"m":132,"g":133},"14716":{"m":132,"g":133},"14830":{"m":132,"g":133},"14834":{"m":132,"g":133},"14812":{"m":132,"g":133},"14831":{"m":132,"g":133},"14806":{"m":132,"g":133},"14822":{"m":132,"g":133},"14819":{"m":132,"g":133},"14710":{"m":132,"g":133},"14807":{"m":132,"g":133},"14770":{"m":132,"g":133},"14793":{"m":132,"g":133},"14808":{"m":132,"g":133},"14697":{"m":132,"g":133},"14720":{"m":132,"g":133},"14803":{"m":132,"g":133},"14788":{"m":132,"g":133},"14794":{"m":132,"g":133},"9650":{"m":132,"g":133},"14786":{"m":132,"g":133},"14784":{"m":132,"g":133},"14725":{"m":132,"g":133},"14777":{"m":132,"g":133},"14761":{"m":132,"g":133},"14759":{"m":132,"g":133},"14064":{"m":132,"g":133},"14768":{"m":132,"g":133},"14756":{"m":132,"g":133},"14744":{"m":132,"g":133},"14687":{"m":132,"g":133},"14763":{"m":132,"g":131},"14177":{"m":132,"g":131},"14758":{"m":132,"g":131},"14669":{"m":132,"g":131},"14740":{"m":132,"g":131},"14753":{"m":132,"g":131},"14698":{"m":132,"g":131},"14379":{"m":132,"g":131},"14752":{"m":132,"g":131},"14751":{"m":132,"g":131},"14745":{"m":132,"g":131},"12953":{"m":132,"g":131},"14743":{"m":132,"g":131},"14738":{"m":132,"g":131},"14733":{"m":132,"g":131},"12039":{"m":132,"g":131},"13432":{"m":132,"g":131},"14461":{"m":132,"g":131},"14686":{"m":132,"g":131},"14601":{"m":132,"g":131},"14622":{"m":132,"g":131},"14714":{"m":132,"g":131},"14707":{"m":132,"g":131},"14699":{"m":132,"g":131},"14647":{"m":132,"g":131},"14648":{"m":132,"g":131},"14683":{"m":132,"g":131},"14678":{"m":132,"g":131},"14676":{"m":132,"g":131},"14529":{"m":132,"g":131},"14689":{"m":132,"g":131},"14627":{"m":132,"g":131},"14679":{"m":132,"g":131},"14469":{"m":132,"g":131},"14614":{"m":132,"g":131},"14653":{"m":132,"g":131},"13147":{"m":132,"g":131},"14652":{"m":132,"g":131},"13334":{"m":132,"g":131},"14489":{"m":132,"g":131},"14675":{"m":132,"g":131},"14671":{"m":132,"g":131},"16253":{"m":134,"g":135},"16107":{"m":134,"g":135},"16244":"m134","16241":{"m":134,"g":135},"16211":{"m":134,"g":135},"16153":{"m":134,"g":135},"15942":{"m":134,"g":135},"16140":{"m":134,"g":135},"16142":{"m":134,"g":135},"16141":{"m":134,"g":135},"16129":{"m":134,"g":135},"10959":{"m":134,"g":135},"15888":{"m":134,"g":135},"16114":{"m":134,"g":135},"16131":{"m":134,"g":135},"16133":{"m":134,"g":135},"16053":{"m":134,"g":135},"16130":{"m":134,"g":135},"16105":{"m":134,"g":135},"15187":{"m":134,"g":135},"16123":{"m":134,"g":135},"15813":{"m":134,"g":135},"16081":{"m":134,"g":135},"15896":{"m":134,"g":135},"15877":{"m":134,"g":135},"15800":{"m":134,"g":135},"15985":{"m":134,"g":135},"16103":{"m":134,"g":135},"16099":{"m":134,"g":135},"15921":{"m":134,"g":135},"16101":{"m":134,"g":135},"16100":{"m":134,"g":135},"16098":{"m":134,"g":135},"16097":{"m":134,"g":135},"16094":{"m":134,"g":135},"16096":{"m":134,"g":135},"16093":{"m":134,"g":135},"15057":{"m":134,"g":135},"14838":{"m":134,"g":135},"16061":{"m":134,"g":135},"16066":{"m":134,"g":135},"16087":{"m":134,"g":135},"16069":{"m":134,"g":135},"16085":{"m":134,"g":135},"16062":{"m":134,"g":135},"14414":{"m":134,"g":135},"16047":{"m":134,"g":135},"15805":{"m":134,"g":135},"16054":{"m":134,"g":135},"16003":{"m":134,"g":135},"16046":{"m":134,"g":135},"16051":{"m":134,"g":135},"16038":{"m":134,"g":135},"16041":{"m":134,"g":135},"16039":{"m":134,"g":135},"16037":{"m":134,"g":135},"16035":{"m":134,"g":135},"16036":{"m":134,"g":135},"16010":{"m":134,"g":135},"14280":{"m":134,"g":135},"16028":{"m":134,"g":135},"16017":{"m":134,"g":135},"14873":{"m":134,"g":135},"15922":{"m":134,"g":135},"16016":{"m":134,"g":135},"15939":{"m":134,"g":135},"15998":{"m":134,"g":135},"15928":{"m":134,"g":135},"16008":{"m":134,"g":135},"16002":{"m":134,"g":135},"16013":{"m":134,"g":135},"16004":{"m":134,"g":135},"15615":{"m":134,"g":135},"15992":{"m":134,"g":135},"15991":{"m":134,"g":135},"16001":{"m":134,"g":135},"15945":{"m":134,"g":135},"15990":{"m":134,"g":135},"15988":{"m":134,"g":135},"15987":{"m":134,"g":135},"15891":{"m":134,"g":135},"15216":{"m":134,"g":135},"15693":{"m":134,"g":135},"15353":{"m":134,"g":135},"15835":{"m":134,"g":135},"15806":{"m":134,"g":135},"15937":{"m":134,"g":135},"15986":{"m":134,"g":135},"12596":{"m":134,"g":135},"15919":{"m":134,"g":135},"15947":{"m":134,"g":135},"15925":{"m":134,"g":135},"15936":{"m":134,"g":135},"15886":{"m":134,"g":135},"15943":{"m":134,"g":135},"15935":{"m":134,"g":135},"15934":{"m":134,"g":135},"15933":{"m":134,"g":135},"14736":{"m":134,"g":135},"15923":{"m":134,"g":135},"15907":{"m":134,"g":135},"15887":{"m":134,"g":135},"15920":{"m":134,"g":135},"15918":{"m":134,"g":135},"15905":{"m":134,"g":135},"15850":{"m":134,"g":135},"15915":{"m":134,"g":135},"15398":{"m":134,"g":135},"15914":{"m":134,"g":135},"15916":{"m":134,"g":135},"15913":{"m":134,"g":135},"15911":{"m":134,"g":135},"15910":{"m":134,"g":135},"14750":{"m":134,"g":135},"15906":{"m":134,"g":135},"15778":{"m":134,"g":135},"15844":{"m":134,"g":135},"15812":{"m":134,"g":135},"15842":{"m":134,"g":135},"15881":{"m":134,"g":135},"15874":{"m":134,"g":135},"15889":{"m":134,"g":135},"15846":{"m":134,"g":135},"14209":{"m":134,"g":135},"15849":{"m":134,"g":135},"15817":{"m":134,"g":135},"15870":{"m":134,"g":135},"15867":{"m":134,"g":135},"14644":{"m":134,"g":135},"15518":{"m":134,"g":135},"15821":{"m":134,"g":135},"15369":{"m":134,"g":135},"15858":{"m":134,"g":135},"15857":{"m":134,"g":135},"15820":{"m":134,"g":135},"15851":{"m":134,"g":135},"15701":{"m":134,"g":135},"15791":{"m":134,"g":135},"15847":{"m":134,"g":135},"15522":{"m":134,"g":135},"15796":{"m":134,"g":135},"15826":{"m":134,"g":135},"15822":{"m":134,"g":135},"15772":{"m":134,"g":135},"15356":{"m":134,"g":135},"15759":{"m":134,"g":135},"15827":{"m":134,"g":135},"15488":{"m":134,"g":135},"15815":{"m":134,"g":135},"15818":{"m":134,"g":135},"15802":{"m":134,"g":135},"15666":{"m":134,"g":135},"15709":{"m":134,"g":135},"14741":{"m":134,"g":135},"15803":{"m":134,"g":135},"15409":{"m":134,"g":135},"15798":{"m":134,"g":135},"15811":{"m":134,"g":135},"15720":{"m":134,"g":135},"15736":{"m":134,"g":135},"15801":{"m":134,"g":135},"15555":{"m":134,"g":135},"15770":{"m":134,"g":135},"11469":{"m":134,"g":135},"15586":{"m":134,"g":135},"15596":{"m":134,"g":135},"14032":{"m":134,"g":135},"15787":{"m":134,"g":135},"15700":{"m":134,"g":135},"15781":{"m":134,"g":133},"15149":{"m":134,"g":133},"15775":{"m":134,"g":133},"15782":{"m":134,"g":133},"15758":{"m":134,"g":133},"15745":{"m":134,"g":133},"15750":{"m":134,"g":133},"15780":{"m":134,"g":133},"15741":{"m":134,"g":133},"14137":{"m":134,"g":133},"15390":{"m":134,"g":133},"15718":{"m":134,"g":133},"15769":{"m":134,"g":133},"15768":{"m":134,"g":133},"15653":{"m":134,"g":133},"15652":{"m":134,"g":133},"15752":{"m":134,"g":133},"15747":{"m":134,"g":133},"15748":{"m":134,"g":133},"15743":{"m":134,"g":133},"15740":{"m":134,"g":133},"15655":{"m":134,"g":133},"15706":{"m":134,"g":133},"15459":{"m":134,"g":133},"15689":{"m":134,"g":133},"15593":{"m":134,"g":133},"15704":{"m":134,"g":133},"15691":{"m":134,"g":133},"15656":{"m":134,"g":133},"15717":{"m":134,"g":133},"15715":{"m":134,"g":133},"15722":{"m":134,"g":133},"15716":{"m":134,"g":133},"15719":{"m":134,"g":133},"11828":{"m":134,"g":133},"15500":{"m":134,"g":133},"15622":{"m":134,"g":133},"15624":{"m":134,"g":133},"15705":{"m":134,"g":133},"15177":{"m":134,"g":133},"15702":{"m":134,"g":133},"15538":{"m":134,"g":133},"15667":{"m":134,"g":133},"15695":{"m":134,"g":133},"15696":{"m":134,"g":133},"15606":{"m":134,"g":133},"15273":{"m":134,"g":133},"15672":{"m":134,"g":133},"15694":{"m":134,"g":133},"15469":{"m":134,"g":133},"12968":{"m":134,"g":133},"15644":{"m":134,"g":133},"15692":{"m":134,"g":133},"14570":{"m":134,"g":133},"15091":{"m":134,"g":133},"15646":{"m":134,"g":133},"15633":{"m":134,"g":133},"15688":{"m":134,"g":133},"15684":{"m":134,"g":133},"15312":{"m":134,"g":133},"15600":{"m":134,"g":133},"15463":{"m":134,"g":133},"15460":{"m":134,"g":133},"14983":{"m":134,"g":133},"15563":{"m":134,"g":133},"15582":{"m":134,"g":133},"14628":{"m":134,"g":133},"15539":{"m":134,"g":133},"15570":{"m":134,"g":133},"15635":{"m":134,"g":133},"15590":{"m":134,"g":133},"13576":{"m":134,"g":133},"15632":{"m":134,"g":133},"15621":{"m":134,"g":133},"15368":{"m":134,"g":133},"15611":{"m":134,"g":133},"15610":{"m":134,"g":133},"15572":{"m":134,"g":133},"15573":{"m":134,"g":133},"15616":{"m":134,"g":133},"15613":{"m":134,"g":133},"15607":{"m":134,"g":133},"15612":{"m":134,"g":133},"15164":{"m":134,"g":133},"15566":{"m":134,"g":133},"15599":{"m":134,"g":133},"15589":{"m":134,"g":133},"15588":{"m":134,"g":133},"15587":{"m":134,"g":133},"15565":{"m":134,"g":133},"15581":{"m":134,"g":133},"15585":{"m":134,"g":133},"15230":{"m":134,"g":133},"15427":{"m":134,"g":133},"15553":{"m":134,"g":133},"15583":{"m":134,"g":133},"15580":{"m":134,"g":133},"15569":{"m":134,"g":133},"14901":{"m":134,"g":133},"15564":{"m":134,"g":133},"15578":{"m":134,"g":133},"15579":{"m":134,"g":133},"15509":{"m":134,"g":133},"15540":{"m":134,"g":133},"15568":{"m":134,"g":133},"15558":{"m":134,"g":133},"12162":{"m":134,"g":133},"15531":{"m":134,"g":133},"15111":{"m":134,"g":133},"15432":{"m":134,"g":133},"15554":{"m":134,"g":133},"15556":{"m":134,"g":133},"15537":{"m":134,"g":133},"15520":{"m":134,"g":133},"15447":{"m":134,"g":133},"15324":{"m":134,"g":133},"15436":{"m":134,"g":133},"14134":{"m":134,"g":133},"15552":{"m":134,"g":133},"15526":{"m":134,"g":133},"15547":{"m":134,"g":133},"15413":{"m":134,"g":133},"15515":{"m":134,"g":133},"15544":{"m":134,"g":133},"15542":{"m":134,"g":133},"15418":{"m":134,"g":133},"15479":{"m":134,"g":133},"15533":{"m":134,"g":133},"15534":{"m":134,"g":133},"15530":{"m":134,"g":133},"15511":{"m":134,"g":133},"15536":{"m":134,"g":133},"15320":{"m":134,"g":133},"14091":{"m":134,"g":133},"15464":{"m":134,"g":133},"15484":{"m":134,"g":133},"15172":{"m":134,"g":133},"15178":{"m":134,"g":133},"15498":{"m":134,"g":133},"15333":{"m":134,"g":133},"15267":{"m":134,"g":133},"15507":{"m":134,"g":133},"15510":{"m":134,"g":133},"15485":{"m":134,"g":133},"15473":{"m":134,"g":133},"15505":{"m":134,"g":133},"15504":{"m":134,"g":133},"15503":{"m":134,"g":133},"15497":{"m":134,"g":133},"15040":{"m":134,"g":133},"15296":{"m":134,"g":133},"15496":{"m":134,"g":133},"15495":{"m":134,"g":133},"15494":{"m":134,"g":133},"15491":{"m":134,"g":133},"15022":{"m":134,"g":133},"14164":{"m":134,"g":133},"15348":{"m":134,"g":133},"15406":{"m":134,"g":133},"15483":{"m":134,"g":133},"15166":{"m":134,"g":133},"14138":{"m":134,"g":133},"15408":{"m":134,"g":133},"15416":{"m":134,"g":133},"15219":{"m":134,"g":133},"15478":{"m":134,"g":133},"15382":{"m":134,"g":133},"12995":{"m":134,"g":133},"15394":{"m":134,"g":133},"15458":{"m":134,"g":133},"15262":{"m":134,"g":133},"15437":{"m":134,"g":133},"14395":{"m":134,"g":133},"15253":{"m":134,"g":133},"15415":{"m":134,"g":133},"15433":{"m":134,"g":133},"13402":{"m":134,"g":133},"13760":{"m":134,"g":133},"15340":{"m":134,"g":133},"13782":{"m":134,"g":133},"15423":{"m":134,"g":133},"15425":{"m":134,"g":133},"15287":{"m":134,"g":133},"15207":{"m":134,"g":133},"15410":{"m":134,"g":133},"15371":{"m":134,"g":133},"15429":{"m":134,"g":133},"15431":{"m":134,"g":133},"14723":{"m":134,"g":133},"14843":{"m":134,"g":133},"14353":{"m":134,"g":133},"15424":{"m":134,"g":133},"14354":{"m":134,"g":133},"15318":{"m":134,"g":133},"14781":{"m":134,"g":133},"15421":{"m":134,"g":133},"15352":{"m":134,"g":133},"15395":{"m":134,"g":133},"15401":{"m":134,"g":133},"15407":{"m":134,"g":133},"15400":{"m":134,"g":133},"15396":{"m":134,"g":133},"15404":{"m":134,"g":133},"15397":{"m":134,"g":133},"12921":{"m":134,"g":133},"15298":{"m":134,"g":133},"15141":{"m":134,"g":133},"15379":{"m":134,"g":133},"15306":{"m":134,"g":133},"14270":{"m":134,"g":133},"15384":{"m":134,"g":133},"15361":{"m":134,"g":133},"15372":{"m":134,"g":133},"12967":{"m":134,"g":133},"15290":{"m":134,"g":133},"14501":{"m":134,"g":133},"15049":{"m":134,"g":133},"15354":{"m":134,"g":133},"15337":{"m":134,"g":133},"15278":{"m":134,"g":133},"15131":{"m":134,"g":133},"15205":{"m":134,"g":133},"15307":{"m":134,"g":133},"14860":{"m":134,"g":133},"15176":{"m":134,"g":133},"15277":{"m":134,"g":133},"15328":{"m":134,"g":133},"15120":{"m":134,"g":133},"15241":{"m":134,"g":133},"15308":{"m":134,"g":133},"15336":{"m":134,"g":133},"15316":{"m":134,"g":133},"15335":{"m":134,"g":133},"15329":{"m":134,"g":133},"15186":{"m":134,"g":133},"15222":{"m":134,"g":133},"15198":{"m":134,"g":133},"15189":{"m":134,"g":133},"15326":{"m":134,"g":133},"15237":{"m":134,"g":133},"15138":{"m":134,"g":133},"14918":{"m":134,"g":133},"11914":{"m":134,"g":133},"15304":{"m":134,"g":133},"15223":{"m":134,"g":133},"15233":{"m":134,"g":133},"15314":{"m":134,"g":133},"12333":{"m":134,"g":133},"15071":{"m":134,"g":133},"15284":{"m":134,"g":133},"14449":{"m":134,"g":133},"14357":{"m":134,"g":133},"15088":{"m":134,"g":133},"14376":{"m":134,"g":133},"15155":{"m":134,"g":133},"13571":{"m":134,"g":133},"15283":{"m":134,"g":133},"15218":{"m":134,"g":133},"15297":{"m":134,"g":133},"15258":{"m":134,"g":133},"15280":{"m":134,"g":133},"15293":{"m":134,"g":133},"15292":{"m":134,"g":133},"15291":{"m":134,"g":133},"15282":{"m":134,"g":133},"15242":{"m":134,"g":133},"14997":{"m":134,"g":133},"14934":{"m":134,"g":133},"15281":{"m":134,"g":133},"14857":{"m":134,"g":133},"14975":{"m":134,"g":133},"14936":{"m":134,"g":133},"15234":{"m":134,"g":133},"15270":{"m":134,"g":133},"15232":{"m":134,"g":133},"15192":{"m":134,"g":133},"15100":{"m":134,"g":133},"9337":{"m":134,"g":133},"15239":{"m":134,"g":133},"15220":{"m":134,"g":133},"14866":{"m":134,"g":133},"14415":{"m":134,"g":133},"15180":{"m":134,"g":133},"15225":{"m":134,"g":133},"15162":{"m":134,"g":133},"14990":{"m":134,"g":133},"15229":{"m":134,"g":133},"15231":{"m":134,"g":133},"15204":{"m":134,"g":133},"15092":{"m":134,"g":133},"15224":{"m":134,"g":133},"13410":{"m":134,"g":133},"15212":{"m":134,"g":133},"15185":{"m":134,"g":133},"15153":{"m":134,"g":133},"14820":{"m":134,"g":133},"15201":{"m":134,"g":133},"15127":{"m":134,"g":133},"15191":{"m":134,"g":133},"15190":{"m":134,"g":133},"15196":{"m":134,"g":133},"15193":{"m":134,"g":133},"15160":{"m":134,"g":133},"15047":{"m":134,"g":133},"15017":{"m":134,"g":133},"15163":{"m":134,"g":133},"15152":{"m":134,"g":133},"14906":{"m":134,"g":133},"15116":{"m":134,"g":133},"14862":{"m":134,"g":133},"15174":{"m":134,"g":133},"9324":{"m":134,"g":133},"15170":{"m":134,"g":133},"15005":{"m":134,"g":133},"13914":{"m":134,"g":133},"15158":{"m":134,"g":133},"15156":{"m":134,"g":133},"15154":{"m":134,"g":133},"15144":{"m":134,"g":133},"14778":{"m":134,"g":133},"15147":{"m":134,"g":133},"15146":{"m":134,"g":133},"15130":{"m":134,"g":133},"14907":{"m":134,"g":133},"14764":{"m":134,"g":133},"14792":{"m":134,"g":133},"15142":{"m":134,"g":133},"15139":{"m":134,"g":133},"15101":{"m":134,"g":133},"14938":{"m":134,"g":133},"15134":{"m":134,"g":133},"15058":{"m":134,"g":133},"15086":{"m":134,"g":133},"15099":{"m":134,"g":133},"15098":{"m":134,"g":133},"14953":{"m":134,"g":133},"15052":{"m":134,"g":133},"15044":{"m":134,"g":133},"15125":{"m":134,"g":133},"15110":{"m":134,"g":133},"14791":{"m":134,"g":133},"15113":{"m":134,"g":133},"15117":{"m":134,"g":133},"15124":{"m":134,"g":133},"15121":{"m":134,"g":133},"14874":{"m":134,"g":133},"15106":{"m":134,"g":133},"15090":{"m":134,"g":133},"15114":{"m":134,"g":133},"12263":{"m":134,"g":133},"13969":{"m":134,"g":133},"14961":{"m":134,"g":133},"15062":{"m":134,"g":133},"15053":{"m":134,"g":133},"14881":{"m":134,"g":133},"15108":{"m":134,"g":133},"15048":{"m":134,"g":133},"14294":{"m":134,"g":133},"15084":{"m":134,"g":133},"14969":{"m":134,"g":133},"15096":{"m":134,"g":133},"15061":{"m":134,"g":133},"15095":{"m":134,"g":133},"15094":{"m":134,"g":133},"15093":{"m":134,"g":133},"14943":{"m":134,"g":133},"15087":{"m":134,"g":133},"14993":{"m":134,"g":133},"14956":{"m":134,"g":133},"15056":{"m":134,"g":133},"15066":{"m":134,"g":133},"15085":{"m":134,"g":133},"15064":{"m":134,"g":133},"14935":{"m":134,"g":133},"15065":{"m":134,"g":133},"14998":{"m":134,"g":133},"14201":{"m":134,"g":133},"14940":{"m":134,"g":133},"15054":{"m":134,"g":133},"15055":{"m":134,"g":133},"15080":{"m":134,"g":133},"15079":{"m":134,"g":133},"14742":{"m":134,"g":133},"15074":{"m":134,"g":133},"15072":{"m":134,"g":133},"13740":{"m":134,"g":133},"15069":{"m":134,"g":133},"15060":{"m":134,"g":133},"15059":{"m":134,"g":133},"14422":{"m":134,"g":133},"14855":{"m":134,"g":133},"14423":{"m":134,"g":133},"15027":{"m":134,"g":133},"13989":{"m":134,"g":133},"14924":{"m":134,"g":133},"14659":{"m":134,"g":133},"15002":{"m":134,"g":133},"15034":{"m":134,"g":133},"14485":{"m":134,"g":133},"13876":{"m":134,"g":133},"15037":{"m":134,"g":133},"15036":{"m":134,"g":133},"15032":{"m":134,"g":133},"15031":{"m":134,"g":133},"15030":{"m":134,"g":133},"15028":{"m":134,"g":133},"15035":{"m":134,"g":133},"14939":{"m":134,"g":133},"15024":{"m":134,"g":133},"15033":{"m":134,"g":133},"15009":{"m":134,"g":133},"14957":{"m":134,"g":133},"14955":{"m":134,"g":133},"15015":{"m":134,"g":133},"15023":{"m":134,"g":133},"15021":{"m":134,"g":133},"14992":{"m":134,"g":133},"14976":{"m":134,"g":133},"14966":{"m":134,"g":133},"14933":{"m":134,"g":133},"14795":{"m":134,"g":133},"15020":{"m":134,"g":133},"15010":{"m":134,"g":133},"15014":{"m":134,"g":133},"14869":{"m":134,"g":133},"14989":{"m":134,"g":133},"14958":{"m":134,"g":133},"14951":{"m":134,"g":133},"15004":{"m":134,"g":133},"15001":{"m":134,"g":133},"15000":{"m":134,"g":133},"14999":{"m":134,"g":133},"14996":{"m":134,"g":133},"14995":{"m":134,"g":133},"14987":{"m":134,"g":133},"14945":{"m":134,"g":133},"14985":{"m":134,"g":133},"14534":{"m":134,"g":133},"14916":{"m":134,"g":133},"14959":{"m":134,"g":133},"14937":{"m":134,"g":133},"13798":{"m":134,"g":133},"14572":{"m":134,"g":133},"14949":{"m":134,"g":133},"14842":{"m":134,"g":133},"13699":{"m":134,"g":133},"14944":{"m":134,"g":133},"14888":{"m":134,"g":133},"14893":{"m":134,"g":133},"14892":{"m":134,"g":133},"14941":{"m":134,"g":133},"14894":{"m":134,"g":133},"14839":{"m":134,"g":133},"11852":{"m":134,"g":133},"14870":{"m":134,"g":133},"14358":{"m":134,"g":133},"14923":{"m":134,"g":133},"14891":{"m":134,"g":133},"14909":{"m":134,"g":133},"14467":{"m":134,"g":133},"14927":{"m":134,"g":133},"14931":{"m":134,"g":133},"13730":{"m":134,"g":133},"14932":{"m":134,"g":133},"14748":{"m":134,"g":133},"14525":{"m":134,"g":133},"14771":{"m":134,"g":133},"14074":{"m":134,"g":133},"14922":{"m":134,"g":133},"13125":{"m":134,"g":133},"14921":{"m":134,"g":133},"14928":{"m":134,"g":133},"14925":{"m":134,"g":133},"17458":"m136","17569":"m136","17591":"m136","17553":"m136","15859":"m136","17460":"m136","17474":"m136","17541":"m136","17518":"m136","17539":"m136","16366":"m136","17514":"m136","17536":"m136","17529":"m136","17534":"m136","17442":"m136","17493":"m136","17528":"m136","17524":"m136","16927":"m136","17486":"m136","17519":"m136","17416":"m136","17490":"m136","17517":"m136","17394":"m136","17498":"m136","16034":"m136","17510":"m136","17166":"m136","16919":"m136","17108":"m136","17417":"m136","16670":"m136","17355":"m136","17399":"m136","17400":"m136","17397":"m136","17372":"m136","16396":"m136","17457":"m136","17462":"m136","17466":"m136","17425":"m136","17465":"m136","17452":"m136","17386":"m136","17293":"m136","17455":"m136","17251":"m136","17444":"m136","17443":"m136","17439":"m136","17290":"m136","17291":"m136","17313":"m136","17436":"m136","17428":"m136","17289":"m136","17334":"m136","17403":"m136","17429":"m136","17247":"m136","17205":"m136","17288":"m136","17419":"m136","17414":"m136","17409":"m136","17358":"m136","11657":"m136","17382":"m136","17160":"m136","17179":"m136","17327":"m136","17385":"m136","17302":"m136","17339":"m136","15325":"m136","17364":"m136","16880":"m136","17378":"m136","17376":"m136","17370":"m136","17043":"m136","17088":"m136","17336":"m136","17367":"m136","17366":"m136","17363":"m136","17329":"m136","17049":"m136","17345":"m136","17116":"m136","17142":"m136","17220":"m136","16567":"m136","17305":"m136","17309":"m136","17158":"m136","17177":"m136","17332":"m136","17238":"m136","17245":"m136","17241":"m136","16412":"m136","17325":"m136","16744":"m136","17317":"m136","17319":"m136","16961":"m136","15347":"m136","17315":"m136","17306":"m136","15512":"m136","17308":"m136","17235":"m136","15631":"m136","14197":"m136","16649":"m136","17296":"m136","17212":"m136","16534":"m136","17295":"m136","17236":"m136","17281":"m136","16974":"m136","17287":"m136","17182":"m136","17264":"m136","16152":"m136","17261":"m136","17250":"m136","17234":"m136","17225":"m136","17256":"m136","17257":"m136","17048":"m136","14883":"m136","17191":"m136","17252":"m136","16561":"m136","17248":"m136","17249":"m136","16879":"m136","17045":"m136","17047":"m136","17044":"m136","16951":"m136","17242":"m136","16354":"m136","16817":"m136","16824":"m136","17232":"m136","17230":"m136","17233":"m136","17165":"m136","16925":"m136","17187":"m136","17217":"m136","16842":"m136","13672":"m136","15551":"m136","17133":"m136","17016":"m136","17200":"m136","17038":"m136","13216":"m136","17143":"m136","17105":"m136","17184":"m136","17173":"m136","17051":"m136","16934":"m136","17186":"m136","16121":"m136","17180":"m136","15513":"m136","17041":"m136","14565":"m136","17100":"m136","14579":"m136","17178":"m136","16882":"m136","16826":"m136","16369":"m136","17176":"m136","17174":"m136","11028":"m136","15455":"m136","17170":"m136","10598":"m136","17091":"m136","17168":"m136","17167":"m136","16568":"m136","15789":"m136","17163":"m136","17099":"m136","17126":"m136","16965":"m136","17111":"m136","17113":"m136","17020":"m136","17064":"m136","17103":"m136","17092":"m136","16949":"m136","17056":"m136","17075":"m136","17061":"m136","17028":"m136","17101":"m136","17052":"m136","12497":"m136","14504":"m136","17087":"m136","16278":"m136","16971":"m136","16976":"m136","16403":"m136","17058":"m136","17077":"m136","16264":"m136","7392":"m136","16732":"m136","16899":"m136","16569":"m136","17013":"m136","16941":"m136","16962":"m136","16888":"m136","15908":"m136","16989":"m136","17046":"m136","17030":"m136","17027":"m136","17054":"m136","16898":"m136","16994":"m136","16841":"m136","17002":"m136","14108":"m136","17022":"m136","17005":"m136","16982":"m136","16894":"m136","16953":"m136","17019":"m136","16886":"m136","16259":"m136","16924":"m136","16986":"m136","16790":"m136","16935":"m136","16884":"m136","16967":"m136","16766":"m136","16564":"m136","16767":"m136","16648":"m136","16252":"m136","16940":"m136","15853":"m136","16765":"m136","16978":"m136","16981":"m136","16980":"m136","16979":"m136","16977":"m136","16973":"m136","16970":"m136","16192":"m136","16348":"m136","16963":"m136","15271":"m136","16933":"m136","16480":"m136","16922":"m136","16916":"m136","16912":"m136","16019":"m136","16559":"m136","16677":"m136","16930":"m136","16932":"m136","16536":"m136","16850":"m136","16915":"m136","16896":"m136","16820":"m136","15268":"m136","12909":"m136","16908":"m136","16906":"m136","16851":"m136","14867":"m136","16895":"m136","16300":"m136","16864":"m136","16889":"m136","15182":"m136","16876":"m136","16258":"m136","16835":"m136","16345":"m136","16878":"m136","16867":"m136","16273":"m136","16757":"m136","16865":"m136","16877":"m136","16863":"m136","16667":"m136","16788":"m136","16397":"m136","16737":"m136","16872":"m136","16871":"m136","16870":"m136","15790":"m136","15927":"m136","15227":"m136","16226":"m136","16844":"m136","16838":"m136","16849":"m136","16333":"m136","16779":"m136","16854":"m136","16862":"m136","16572":"m136","16805":"m136","16723":"m136","13715":"m136","16853":"m136","16637":"m136","16852":"m136","16847":"m136","16848":"m136","16845":"m136","16698":"m136","16825":"m136","16837":"m136","16840":"m136","16839":"m136","16831":"m136","16830":"m136","16679":"m136","16810":"m136","16783":"m136","16587":"m136","16821":"m136","16804":"m136","15753":"m136","13947":"m136","16275":"m136","16814":"m136","16813":"m136","16812":"m136","16811":"m136","14655":"m136","16325":"m136","16708":"m136","16760":"m136","16792":"m136","11349":"m136","16458":"m136","16743":"m136","16768":"m136","16721":"m136","16778":"m136","16774":"m136","16686":"m136","16378":"m136","16588":"m136","16446":"m136","16773":"m136","16380":"m136","16254":{"m":136,"g":135},"16720":{"m":136,"g":135},"16560":{"m":136,"g":135},"16764":{"m":136,"g":135},"16763":{"m":136,"g":135},"16759":{"m":136,"g":135},"16715":{"m":136,"g":135},"16756":{"m":136,"g":135},"16754":{"m":136,"g":135},"16752":{"m":136,"g":135},"16751":{"m":136,"g":135},"16749":{"m":136,"g":135},"16748":{"m":136,"g":135},"16675":{"m":136,"g":135},"16746":{"m":136,"g":135},"16409":{"m":136,"g":135},"13681":{"m":136,"g":135},"16745":{"m":136,"g":135},"16741":{"m":136,"g":135},"16014":{"m":136,"g":135},"16739":{"m":136,"g":135},"16719":{"m":136,"g":135},"16668":{"m":136,"g":135},"16738":{"m":136,"g":135},"16709":{"m":136,"g":135},"16735":{"m":136,"g":135},"16622":{"m":136,"g":135},"16635":{"m":136,"g":135},"16733":{"m":136,"g":135},"16730":{"m":136,"g":135},"16729":{"m":136,"g":135},"16519":{"m":136,"g":135},"16706":{"m":136,"g":135},"16115":{"m":136,"g":135},"16634":{"m":136,"g":135},"16535":{"m":136,"g":135},"16452":{"m":136,"g":135},"16533":{"m":136,"g":135},"16531":{"m":136,"g":135},"16529":{"m":136,"g":135},"15627":{"m":136,"g":135},"16608":{"m":136,"g":135},"16693":{"m":136,"g":135},"16155":{"m":136,"g":135},"16505":{"m":136,"g":135},"16697":{"m":136,"g":135},"16555":{"m":136,"g":135},"16669":{"m":136,"g":135},"16625":{"m":136,"g":135},"16695":{"m":136,"g":135},"16678":{"m":136,"g":135},"16692":{"m":136,"g":135},"16629":{"m":136,"g":135},"16680":{"m":136,"g":135},"16445":{"m":136,"g":135},"16306":{"m":136,"g":135},"16652":{"m":136,"g":135},"15343":{"m":136,"g":135},"16095":{"m":136,"g":135},"15151":{"m":136,"g":135},"16681":{"m":136,"g":135},"16674":{"m":136,"g":135},"16457":{"m":136,"g":135},"16426":{"m":136,"g":135},"16676":{"m":136,"g":135},"16658":{"m":136,"g":135},"16672":{"m":136,"g":135},"16633":{"m":136,"g":135},"16671":{"m":136,"g":135},"15712":{"m":136,"g":135},"16660":{"m":136,"g":135},"16664":{"m":136,"g":135},"16657":{"m":136,"g":135},"16661":{"m":136,"g":135},"16618":{"m":136,"g":135},"16179":{"m":136,"g":135},"16599":{"m":136,"g":135},"15238":{"m":136,"g":135},"16532":{"m":136,"g":135},"16654":{"m":136,"g":135},"16651":{"m":136,"g":135},"16203":{"m":136,"g":135},"13602":{"m":136,"g":135},"16620":{"m":136,"g":135},"16514":{"m":136,"g":135},"16631":{"m":136,"g":135},"16630":{"m":136,"g":135},"16619":{"m":136,"g":135},"16527":{"m":136,"g":135},"15938":{"m":136,"g":135},"16582":{"m":136,"g":135},"16418":{"m":136,"g":135},"16459":{"m":136,"g":135},"16465":{"m":136,"g":135},"16468":{"m":136,"g":135},"16566":{"m":136,"g":135},"16617":{"m":136,"g":135},"16597":{"m":136,"g":135},"16504":{"m":136,"g":135},"16593":{"m":136,"g":135},"16118":{"m":136,"g":135},"15439":{"m":136,"g":135},"16609":{"m":136,"g":135},"16453":{"m":136,"g":135},"16499":{"m":136,"g":135},"16606":{"m":136,"g":135},"16603":{"m":136,"g":135},"16576":{"m":136,"g":135},"16596":{"m":136,"g":135},"16598":{"m":136,"g":135},"16601":{"m":136,"g":135},"16594":{"m":136,"g":135},"16442":{"m":136,"g":135},"16422":{"m":136,"g":135},"16223":{"m":136,"g":135},"16425":{"m":136,"g":135},"16589":{"m":136,"g":135},"16592":{"m":136,"g":135},"16583":{"m":136,"g":135},"16585":{"m":136,"g":135},"16591":{"m":136,"g":135},"16326":{"m":136,"g":135},"16523":{"m":136,"g":135},"16420":{"m":136,"g":135},"16548":{"m":136,"g":135},"16549":{"m":136,"g":135},"16575":{"m":136,"g":135},"16507":{"m":136,"g":135},"16570":{"m":136,"g":135},"16463":{"m":136,"g":135},"14215":{"m":136,"g":135},"13518":{"m":136,"g":135},"16455":{"m":136,"g":135},"16496":{"m":136,"g":135},"16540":{"m":136,"g":135},"16201":{"m":136,"g":135},"15456":{"m":136,"g":135},"16421":{"m":136,"g":135},"16466":{"m":136,"g":135},"16539":{"m":136,"g":135},"16417":{"m":136,"g":135},"16502":{"m":136,"g":135},"16415":{"m":136,"g":135},"16467":{"m":136,"g":135},"16399":{"m":136,"g":135},"14112":{"m":136,"g":135},"15492":{"m":136,"g":135},"16538":{"m":136,"g":135},"6135":{"m":136,"g":135},"16528":{"m":136,"g":135},"16416":{"m":136,"g":135},"16419":{"m":136,"g":135},"16434":{"m":136,"g":135},"16525":{"m":136,"g":135},"16520":{"m":136,"g":135},"16086":{"m":136,"g":135},"16524":{"m":136,"g":135},"15677":{"m":136,"g":135},"16477":{"m":136,"g":135},"16356":{"m":136,"g":135},"16513":{"m":136,"g":135},"15663":{"m":136,"g":135},"16516":{"m":136,"g":135},"16482":{"m":136,"g":135},"16511":{"m":136,"g":135},"16280":{"m":136,"g":135},"16509":{"m":136,"g":135},"16508":{"m":136,"g":135},"16492":{"m":136,"g":135},"16469":{"m":136,"g":135},"16138":{"m":136,"g":135},"16481":{"m":136,"g":135},"14474":{"m":136,"g":135},"16367":{"m":136,"g":135},"16475":{"m":136,"g":135},"16478":{"m":136,"g":135},"16479":{"m":136,"g":135},"16474":{"m":136,"g":135},"16447":{"m":136,"g":135},"16471":{"m":136,"g":135},"16456":{"m":136,"g":135},"16382":{"m":136,"g":135},"14051":{"m":136,"g":135},"16042":{"m":136,"g":135},"16424":{"m":136,"g":135},"16460":{"m":136,"g":135},"16464":{"m":136,"g":135},"16454":{"m":136,"g":135},"16088":{"m":136,"g":135},"16387":{"m":136,"g":135},"16386":{"m":136,"g":135},"16451":{"m":136,"g":135},"16450":{"m":136,"g":135},"16448":{"m":136,"g":135},"16389":{"m":136,"g":135},"16180":{"m":136,"g":135},"16441":{"m":136,"g":135},"16444":{"m":136,"g":135},"16437":{"m":136,"g":135},"16310":{"m":136,"g":135},"16435":{"m":136,"g":135},"16433":{"m":136,"g":135},"16432":{"m":136,"g":135},"16375":{"m":136,"g":135},"16330":{"m":136,"g":135},"15836":{"m":136,"g":135},"16430":{"m":136,"g":135},"16414":{"m":136,"g":135},"16429":{"m":136,"g":135},"16427":{"m":136,"g":135},"16413":{"m":136,"g":135},"16408":{"m":136,"g":135},"16411":{"m":136,"g":135},"16334":{"m":136,"g":135},"16127":{"m":136,"g":135},"16323":{"m":136,"g":135},"14066":{"m":136,"g":135},"16405":{"m":136,"g":135},"16406":{"m":136,"g":135},"16401":{"m":136,"g":135},"16400":{"m":136,"g":135},"16347":{"m":136,"g":135},"16349":{"m":136,"g":135},"16390":{"m":136,"g":135},"16392":{"m":136,"g":135},"16374":{"m":136,"g":135},"16391":{"m":136,"g":135},"16381":{"m":136,"g":135},"16376":{"m":136,"g":135},"16373":{"m":136,"g":135},"16359":{"m":136,"g":135},"16268":{"m":136,"g":135},"16365":{"m":136,"g":135},"16368":{"m":136,"g":135},"16363":{"m":136,"g":135},"16358":{"m":136,"g":135},"16343":{"m":136,"g":135},"16357":{"m":136,"g":135},"16355":{"m":136,"g":135},"16352":{"m":136,"g":135},"16064":{"m":136,"g":135},"15434":{"m":136,"g":135},"16353":{"m":136,"g":135},"16341":{"m":136,"g":135},"16351":{"m":136,"g":135},"16269":{"m":136,"g":135},"16350":{"m":136,"g":135},"16340":{"m":136,"g":135},"16339":{"m":136,"g":135},"16335":{"m":136,"g":135},"16344":{"m":136,"g":135},"15941":{"m":136,"g":135},"16342":{"m":136,"g":135},"16164":{"m":136,"g":135},"16304":{"m":136,"g":135},"16303":{"m":136,"g":135},"16338":{"m":136,"g":135},"16219":{"m":136,"g":135},"16337":{"m":136,"g":135},"15640":{"m":136,"g":135},"16311":{"m":136,"g":135},"16332":{"m":136,"g":135},"15175":{"m":136,"g":135},"16328":{"m":136,"g":135},"16009":{"m":136,"g":135},"13592":{"m":136,"g":135},"16272":{"m":136,"g":135},"16283":{"m":136,"g":135},"16257":{"m":136,"g":135},"16177":{"m":136,"g":135},"15560":{"m":136,"g":135},"16324":{"m":136,"g":135},"16317":{"m":136,"g":135},"16296":{"m":136,"g":135},"16321":{"m":136,"g":135},"16316":{"m":136,"g":135},"16313":{"m":136,"g":135},"16318":{"m":136,"g":135},"16320":{"m":136,"g":135},"16319":{"m":136,"g":135},"16314":{"m":136,"g":135},"16315":{"m":136,"g":135},"16312":{"m":136,"g":135},"16287":{"m":136,"g":135},"16277":{"m":136,"g":135},"16308":{"m":136,"g":135},"16092":{"m":136,"g":135},"16128":{"m":136,"g":135},"16305":{"m":136,"g":135},"16301":{"m":136,"g":135},"13959":{"m":136,"g":135},"16248":{"m":136,"g":135},"16292":{"m":136,"g":135},"16298":{"m":136,"g":135},"16227":{"m":136,"g":135},"16213":{"m":136,"g":135},"16267":{"m":136,"g":135},"16144":{"m":136,"g":135},"16222":{"m":136,"g":135},"15345":{"m":136,"g":135},"16270":{"m":136,"g":135},"16285":{"m":136,"g":135},"14636":{"m":136,"g":135},"16263":{"m":136,"g":135},"16265":{"m":136,"g":135},"15995":{"m":136,"g":135},"16162":{"m":136,"g":135},"16262":{"m":136,"g":135},"16261":{"m":136,"g":135},"16260":{"m":136,"g":135},"16251":{"m":136,"g":135},"16247":{"m":136,"g":135},"16250":{"m":136,"g":135},"16243":{"m":136,"g":135},"14085":{"m":136,"g":135},"16171":{"m":136,"g":135},"16245":{"m":136,"g":135},"16238":{"m":136,"g":135},"15814":{"m":136,"g":135},"16239":{"m":136,"g":135},"16240":{"m":136,"g":135},"16236":{"m":136,"g":135},"16233":{"m":136,"g":135},"16202":{"m":136,"g":135},"16228":{"m":136,"g":135},"16018":{"m":136,"g":135},"16195":{"m":136,"g":135},"16111":{"m":136,"g":135},"13394":{"m":136,"g":135},"16204":{"m":136,"g":135},"16214":{"m":136,"g":135},"16221":{"m":136,"g":135},"16178":{"m":136,"g":135},"16187":{"m":136,"g":135},"16209":{"m":136,"g":135},"15597":{"m":136,"g":135},"16117":{"m":136,"g":135},"15878":{"m":136,"g":135},"16200":{"m":136,"g":135},"15946":{"m":136,"g":135},"16198":{"m":136,"g":135},"16172":{"m":136,"g":135},"15575":{"m":136,"g":135},"16174":{"m":136,"g":135},"16156":{"m":136,"g":135},"15754":{"m":136,"g":135},"14920":{"m":136,"g":135},"16050":{"m":136,"g":135},"16181":{"m":136,"g":135},"16183":{"m":136,"g":135},"16182":{"m":136,"g":135},"14416":{"m":136,"g":135},"16168":{"m":136,"g":135},"16175":{"m":136,"g":135},"16166":{"m":136,"g":135},"16161":{"m":136,"g":135},"16163":{"m":136,"g":135},"16165":{"m":136,"g":135},"16110":{"m":136,"g":135},"16173":{"m":136,"g":135},"16159":{"m":136,"g":135},"16150":{"m":136,"g":135},"16160":{"m":136,"g":135},"16158":{"m":136,"g":135},"16149":{"m":136,"g":135},"16124":{"m":136,"g":135},"15730":{"m":136,"g":135},"13978":{"m":136,"g":135},"12625":{"m":136,"g":135},"16154":{"m":136,"g":135},"18298":"m137","18111":"m137"},"c":{"2ccd9fd8":{"m":0,"g":94},"46b7ea7c":{"m":0,"g":94},"70359bf3":{"m":0,"g":94},"01ca82d7":{"m":0,"g":94},"4bd8233f":{"m":0,"g":94},"08ab2a16":{"m":0,"g":94},"f652494d":{"m":0,"g":94},"30720e73":{"m":0,"g":94},"331848de":{"m":0,"g":94},"93eeb543":{"m":0,"g":94},"ead5b39f":{"m":0,"g":94},"22085081":{"m":0,"g":94},"f6d40df0":{"m":0,"g":94},"22ec7bc2":{"m":1,"g":94},"c0454b32":{"m":1,"g":94},"8024fc5e":{"m":1,"g":94},"70528762":{"m":1,"g":94},"71d30d6d":{"m":1,"g":94},"f9d72381":{"m":1,"g":94},"bf51ddc6":{"m":1,"g":94},"fd7c4792":{"m":1,"g":94},"c4707f1b":{"m":1,"g":94},"ffe4aaee":{"m":1,"g":94},"5b27a1dc":{"m":1,"g":94},"e71d4ab3":{"m":1,"g":94},"fbf42263":{"m":1,"g":94},"cc3ada98":{"m":2,"g":94},"a837166e":{"m":2,"g":94},"11f3cca6":{"m":2,"g":94},"ca13f3b8":{"m":2,"g":94},"0b2efc2a":{"m":2,"g":94},"f30abd09":{"m":2,"g":94},"40ab1f01":{"m":2,"g":94},"199e82a1":{"m":2,"g":94},"23471f9a":{"m":2,"g":94},"61d4c939":{"m":2,"g":94},"98a3e8ef":{"m":2,"g":94},"2b079f89":{"m":2,"g":94},"05b4c398":{"m":2,"g":94},"dafafe5b":{"m":2,"g":94},"b240f751":{"m":2,"g":94},"501f9444":{"m":2,"g":94},"723f0421":{"m":3,"g":94},"585eabab":{"m":3,"g":94},"c70b3cfa":{"m":4,"g":94},"489796c7":{"m":4,"g":94},"fa7a696d":{"m":4,"g":94},"bef0b359":{"m":4,"g":94},"c6576e82":{"m":4,"g":94},"99258181":{"m":4,"g":94},"3de54a1b":{"m":4,"g":94},"7358fa64":{"m":4,"g":94},"9a16fea0":{"m":4,"g":94},"9e037c82":{"m":4,"g":94},"9076386d":{"m":4,"g":94},"959c4174":{"m":4,"g":94},"94e05770":{"m":4,"g":94},"63e97e5e":{"m":4,"g":94},"e08bca28":{"m":4,"g":94},"cd3ccb2e":{"m":4,"g":94},"3f5c2f4c":{"m":4,"g":94},"007eeb4e":{"m":4,"g":94},"e8f2b155":{"m":4,"g":94},"6dceab4d":{"m":5,"g":94},"a49dc52b":{"m":6,"g":94},"873d0e85":{"m":6,"g":94},"1d0fbe8e":{"m":6,"g":94},"97aa9b32":{"m":6,"g":94},"06175286":{"m":6,"g":94},"4ea92f83":{"m":6,"g":94},"6b0af285":{"m":6,"g":94},"6f560c76":{"m":6,"g":94},"cd687233":{"m":6,"g":94},"81561f8e":{"m":6,"g":94},"3a581e99":{"m":6,"g":94},"0147f940":{"m":6,"g":94},"23950056":{"m":6,"g":94},"93414c82":{"m":6,"g":94},"ed7c7eca":{"m":6,"g":94},"0c457bae":{"m":6,"g":94},"d3fc86a4":{"m":6,"g":94},"01ee0fbc":{"m":6,"g":94},"711d3435":{"m":6,"g":94},"f6bfe3aa":{"m":7,"g":94},"e095b162":{"m":7,"g":94},"67be11c7":{"m":7,"g":94},"cd8c3ccd":{"m":7,"g":94},"9c121f2a":{"m":7,"g":94},"03e04b23":{"m":7,"g":94},"86442530":{"m":7,"g":94},"79cb018e":{"m":7,"g":94},"c7af9f73":{"m":7,"g":94},"876db8dc":{"m":7,"g":94},"ad82bac6":{"m":7,"g":94},"71b54eea":{"m":7,"g":94},"74b3bfaa":{"m":7,"g":94},"4a634cf6":{"m":7,"g":94},"624b21e7":{"m":8,"g":94},"c51020cf":{"m":8,"g":94},"50afed4e":{"m":8,"g":94},"4d303c4f":{"m":8,"g":94},"37b42297":{"m":8,"g":94},"cba50273":{"m":8,"g":94},"a6aa46dd":{"m":8,"g":94},"405f26b0":{"m":8,"g":94},"b1a3a454":{"m":8,"g":94},"79e6b84b":{"m":8,"g":94},"26c34941":{"m":8,"g":94},"cb8e1982":{"m":8,"g":94},"23f05005":{"m":8,"g":94},"a7334aee":{"m":8,"g":94},"ee1df26a":{"m":8,"g":94},"3ae78a09":{"m":8,"g":94},"ccbe1e67":{"m":8,"g":94},"e2bf732b":{"m":8,"g":94},"322421fa":{"m":8,"g":94},"8ff870bf":{"m":8,"g":94},"26f0bedc":{"m":8,"g":94},"82fa69b3":{"m":8,"g":94},"8fb7459e":{"m":8,"g":94},"bb3a3b66":{"m":8,"g":94},"45d6592d":{"m":8,"g":94},"4aa5dd2c":{"m":9,"g":94},"13662fd5":{"m":9,"g":94},"d5ae2eba":{"m":9,"g":94},"1b355479":{"m":9,"g":94},"faba293a":{"m":9,"g":94},"89885b31":{"m":9,"g":94},"64fe3115":{"m":9,"g":94},"a7ace9c8":{"m":9,"g":94},"a833de05":{"m":9,"g":94},"30d67b2b":{"m":9,"g":94},"b0b722ee":{"m":9,"g":94},"01b07ea3":{"m":9,"g":94},"dfb13ac4":{"m":9,"g":94},"ec90b9c0":{"m":9,"g":94},"9759d927":{"m":9,"g":94},"8d0a7fae":{"m":9,"g":94},"c4e9ebe3":{"m":9,"g":94},"3c2c5869":{"m":9,"g":94},"4cb9aaed":{"m":9,"g":94},"9de9a468":{"m":9,"g":94},"ce3b2610":{"m":9,"g":94},"91e03633":{"m":9,"g":94},"2a74748b":{"m":9,"g":94},"63ba630b":{"m":9,"g":94},"6493256b":{"m":9,"g":94},"06008bc2":{"m":9,"g":94},"bb824da4":{"m":9,"g":94},"93121324":{"m":9,"g":94},"c97fdae4":{"m":9,"g":94},"51104cd4":{"m":10,"g":94},"e2b2f0a2":{"m":10,"g":94},"b57abe16":{"m":10,"g":94},"e57f0792":{"m":10,"g":94},"08df63a6":{"m":10,"g":94},"77835756":{"m":10,"g":94},"ed315799":{"m":10,"g":94},"92e2d74f":{"m":10,"g":94},"d9b3b018":{"m":10,"g":94},"745ea007":{"m":10,"g":94},"ad1dd746":{"m":10,"g":94},"eb4308c4":{"m":10,"g":94},"b2eb0805":{"m":10,"g":94},"72bb3443":{"m":11,"g":94},"2d580e7a":{"m":11,"g":94},"3fc97f67":{"m":11,"g":94},"abc548c7":{"m":11,"g":94},"aee4f523":{"m":11,"g":94},"7023f413":{"m":11,"g":94},"09deb20d":{"m":11,"g":94},"33b242df":{"m":11,"g":94},"a511a2d0":{"m":11,"g":94},"6ec65f45":{"m":11,"g":94},"e2c31fca":{"m":11,"g":94},"d5de20a3":{"m":11,"g":94},"4a1c6ae2":{"m":11,"g":94},"14522e6a":{"m":11,"g":94},"183df472":{"m":11,"g":94},"5c5aba59":{"m":11,"g":94},"ba67101f":{"m":11,"g":94},"95c4e0df":{"m":11,"g":94},"19818b9c":{"m":11,"g":94},"9216b106":{"m":11,"g":94},"da19434c":{"m":11,"g":94},"150d7020":{"m":11,"g":94},"9acc6e35":{"m":11,"g":94},"cf9d8efd":{"m":11,"g":94},"1bf1cf19":{"m":11,"g":94},"e822e590":{"m":11,"g":94},"ca4f1ab8":{"m":11,"g":94},"2b6d9991":{"m":11,"g":94},"65501a9c":{"m":11,"g":94},"db611066":{"m":11,"g":94},"c93293c5":{"m":11,"g":94},"62b3812b":{"m":11,"g":94},"550a4f78":{"m":11,"g":94},"ff99c38a":{"m":11,"g":94},"c9de3e16":{"m":11,"g":94},"ed27a6b9":{"m":11,"g":94},"463c6632":{"m":11,"g":94},"b0890631":{"m":11,"g":94},"cb389c91":{"m":11,"g":94},"eddaa2b5":{"m":11,"g":94},"2af565b3":{"m":11,"g":94},"3842eba5":{"m":11,"g":94},"24e59f53":{"m":11,"g":94},"75235419":{"m":11,"g":94},"64ee9c03":{"m":11,"g":94},"30d17840":{"m":11,"g":94},"ce216c80":{"m":11,"g":94},"e0ae5d42":{"m":12,"g":94},"32de16ce":{"m":12,"g":94},"0992d85f":{"m":12,"g":94},"5dc55a5f":{"m":12,"g":94},"4231a42f":{"m":12,"g":94},"455c9ccc":{"m":12,"g":94},"39191c85":{"m":12,"g":94},"562b8857":{"m":12,"g":94},"04c0b214":{"m":12,"g":94},"6e09cf6a":{"m":12,"g":94},"e8a2327d":{"m":13,"g":94},"91f93f14":{"m":13,"g":94},"f70f7258":{"m":13,"g":94},"c0ae70c8":{"m":13,"g":94},"87260b7b":{"m":13,"g":94},"651a23ee":{"m":13,"g":94},"bf3e271f":{"m":13,"g":94},"3bc01ac1":{"m":13,"g":94},"9f009261":{"m":13,"g":94},"159cc741":{"m":13,"g":94},"7d1ebc2d":{"m":13,"g":94},"83525a1d":{"m":13,"g":94},"80a33ce8":{"m":13,"g":94},"1a57e416":{"m":13,"g":94},"adc97426":{"m":13,"g":94},"0463f7fb":{"m":13,"g":94},"565d7274":{"m":13,"g":94},"09de730d":{"m":13,"g":94},"55c16436":{"m":13,"g":94},"2b605ab1":{"m":13,"g":94},"947bda73":{"m":13,"g":94},"f06e90c2":{"m":13,"g":94},"2cea6146":{"m":13,"g":94},"44c998fc":{"m":13,"g":94},"3167d8da":{"m":13,"g":94},"0fafc560":{"m":13,"g":94},"19d2135c":{"m":13,"g":94},"ced77c66":{"m":13,"g":94},"8dbdc018":{"m":13,"g":94},"3e684be7":{"m":13,"g":94},"ec380dfd":{"m":13,"g":94},"5b647543":{"m":13,"g":94},"8210ec60":{"m":13,"g":94},"5be9eb8a":{"m":13,"g":94},"c05956e5":{"m":13,"g":94},"d75dc20f":{"m":13,"g":94},"690d162d":{"m":13,"g":94},"664287b2":{"m":13,"g":94},"2f11936f":{"m":14,"g":94},"63fbef98":{"m":14,"g":94},"2a754e57":{"m":14,"g":94},"96c503eb":{"m":14,"g":94},"441cca77":{"m":14,"g":94},"c7709d3a":{"m":14,"g":94},"9380f50f":{"m":14,"g":94},"95dc093b":{"m":14,"g":94},"d9ac6392":{"m":14,"g":94},"26294b2f":{"m":14,"g":94},"75b31a2a":{"m":14,"g":94},"11616fc6":{"m":14,"g":94},"9ce89bc1":{"m":14,"g":94},"badf3fa0":{"m":14,"g":94},"945aa9be":{"m":14,"g":94},"2e6e62e1":{"m":14,"g":94},"a385ee27":{"m":14,"g":94},"eb1ae6ae":{"m":14,"g":94},"2187f362":{"m":14,"g":94},"9465b668":{"m":14,"g":94},"05471f21":{"m":14,"g":94},"1fa15099":{"m":14,"g":94},"303ef888":{"m":14,"g":94},"92cb93f3":{"m":14,"g":94},"e94e60d6":{"m":14,"g":94},"d2f8bfb2":{"m":14,"g":94},"b7e2f800":{"m":14,"g":94},"09593e9b":{"m":14,"g":94},"53a7ebd8":{"m":14,"g":94},"ad5f04d6":{"m":14,"g":94},"bbec01c9":{"m":14,"g":94},"40e53d65":{"m":14,"g":94},"fb9296f0":{"m":14,"g":94},"1374334d":{"m":14,"g":94},"94aead9e":{"m":14,"g":94},"9c902b19":{"m":14,"g":94},"111991fe":{"m":14,"g":94},"a8c787d2":{"m":14,"g":94},"5f283991":{"m":14,"g":94},"b6667a53":{"m":14,"g":94},"542bc733":{"m":14,"g":94},"f6dbd240":{"m":14,"g":94},"ad872feb":{"m":15,"g":94},"da2e5d65":{"m":15,"g":94},"ce62dc73":{"m":15,"g":94},"02b72586":{"m":15,"g":94},"d557e9f3":{"m":15,"g":94},"740c46a1":{"m":15,"g":94},"b3868722":{"m":15,"g":94},"710f614e":{"m":15,"g":94},"f25b76c0":{"m":15,"g":94},"f4e885b7":{"m":15,"g":94},"0877f1e7":{"m":15,"g":94},"5304b4ef":{"m":15,"g":94},"26908d95":{"m":15,"g":94},"c0982ac5":{"m":15,"g":94},"dc1b8bcf":{"m":15,"g":94},"5a57b8ad":{"m":15,"g":94},"d737da5f":{"m":15,"g":94},"ac113887":{"m":15,"g":94},"dc8cef1d":{"m":15,"g":94},"5d264a90":{"m":16,"g":94},"5949b1ca":{"m":16,"g":94},"0feca02d":{"m":16,"g":94},"10143e1a":{"m":16,"g":94},"65c65776":{"m":16,"g":94},"66581596":{"m":16,"g":94},"396a6924":{"m":16,"g":94},"af4e7910":{"m":16,"g":94},"519e20cf":{"m":16,"g":94},"d9a69029":{"m":16,"g":94},"56f5fc4a":{"m":17,"g":94},"6a2941f4":{"m":17,"g":94},"5ac8b806":{"m":17,"g":94},"bae9541e":{"m":17,"g":94},"a56858ba":{"m":17,"g":94},"564a898a":{"m":17,"g":94},"2b4c6462":{"m":18,"g":94},"f424e76d":{"m":18,"g":94},"490a1f39":{"m":18,"g":94},"06487f12":{"m":18,"g":94},"39c57317":{"m":18,"g":94},"9592a1f3":{"m":18,"g":94},"35759efa":{"m":18,"g":94},"8f4b1559":{"m":18,"g":94},"e3046ea3":{"m":18,"g":94},"49c5e0ec":{"m":18,"g":94},"ec2150b2":{"m":18,"g":94},"7620cd37":{"m":18,"g":94},"50a53887":{"m":18,"g":94},"11c8efff":{"m":18,"g":94},"e87c7fd5":{"m":18,"g":94},"630479c3":{"m":18,"g":94},"51fda143":{"m":18,"g":94},"dc4e4a6a":{"m":18,"g":94},"2d96da81":{"m":18,"g":94},"c126a6cc":{"m":18,"g":94},"ac971ff6":{"m":18,"g":94},"e1792cca":{"m":18,"g":94},"1b7adbb5":{"m":18,"g":94},"a9ef49c1":{"m":18,"g":94},"21ba3a88":{"m":18,"g":94},"9c5cac24":{"m":18,"g":94},"5960a6e5":{"m":18,"g":94},"b050d928":{"m":18,"g":94},"6a4dc996":{"m":18,"g":94},"d774acad":{"m":18,"g":94},"d93388da":{"m":18,"g":94},"476584cb":{"m":18,"g":94},"abd5385a":{"m":18,"g":94},"3de2f30a":{"m":18,"g":94},"4efcc59d":{"m":18,"g":94},"2e341cd4":{"m":18,"g":94},"a8552cb1":{"m":18,"g":94},"a470e60c":{"m":18,"g":94},"5f90e076":{"m":18,"g":94},"8832ecb1":{"m":18,"g":94},"5ff60eda":{"m":18,"g":94},"c1930022":{"m":18,"g":94},"fe3be159":{"m":18,"g":94},"0aa189f1":{"m":18,"g":94},"f6b29f69":{"m":18,"g":94},"c9ee3d35":{"m":18,"g":94},"41d1f677":{"m":18,"g":94},"444a0244":{"m":19,"g":94},"fa7ccb33":{"m":19,"g":94},"26868443":{"m":19,"g":94},"824a77d0":{"m":19,"g":94},"cf99eab7":{"m":19,"g":94},"9fdea29d":{"m":19,"g":94},"df7c4c19":{"m":19,"g":94},"c3f1aac8":{"m":19,"g":94},"d198791f":{"m":19,"g":94},"c07526e4":{"m":19,"g":94},"7b597475":{"m":19,"g":94},"5303c1ed":{"m":19,"g":94},"65bd1338":{"m":19,"g":94},"eedc12e1":{"m":19,"g":94},"5a4ef2b5":{"m":19,"g":94},"9dab947d":{"m":19,"g":94},"33ee97b0":{"m":19,"g":94},"6a846bb1":{"m":19,"g":94},"0fdb3127":{"m":19,"g":94},"5ad033a0":{"m":19,"g":94},"77e592e8":{"m":19,"g":94},"caaad53b":{"m":19,"g":94},"69d19188":{"m":19,"g":94},"4b4a67f8":{"m":19,"g":94},"0ac94c36":{"m":19,"g":94},"459abad2":{"m":20,"g":94},"30d8e130":{"m":20,"g":94},"08a3bd19":{"m":20,"g":94},"321a963b":{"m":20,"g":94},"e17deb27":{"m":20,"g":94},"2d3ae4e1":{"m":20,"g":94},"75f4ccb7":{"m":20,"g":94},"83d2b30d":{"m":20,"g":94},"4367f4bb":{"m":20,"g":94},"00e4baa7":{"m":20,"g":94},"4cd64b8e":{"m":20,"g":94},"01d66ae2":{"m":20,"g":94},"a523a3c1":{"m":20,"g":94},"9f94728f":{"m":20,"g":94},"1a491d00":{"m":21,"g":94},"8fbba3de":{"m":21,"g":94},"ae0f6130":{"m":21,"g":94},"60105897":{"m":21,"g":94},"926ac01b":{"m":21,"g":94},"25c881a0":{"m":21,"g":94},"04ec6ba2":{"m":21,"g":94},"d63f13c1":{"m":21,"g":94},"fded6744":{"m":21,"g":94},"6e453940":{"m":21,"g":94},"97e0f7d2":{"m":21,"g":94},"d5146bae":{"m":21,"g":94},"5bd06b45":{"m":22,"g":94},"9a611827":{"m":22,"g":94},"eeb24821":{"m":22,"g":94},"3e455b01":{"m":22,"g":94},"8628ab9c":{"m":22,"g":94},"1b77670f":{"m":22,"g":94},"768e05d0":{"m":22,"g":94},"01fbb11b":{"m":22,"g":94},"05d216da":{"m":22,"g":94},"6b32bb1c":{"m":22,"g":94},"40facad5":{"m":22,"g":94},"da504445":{"m":22,"g":94},"252e0f7b":{"m":22,"g":94},"7f6f2f0f":{"m":22,"g":94},"7802df1e":{"m":22,"g":94},"bc1154c3":{"m":23,"g":94},"752e6430":{"m":23,"g":94},"30db99b3":{"m":23,"g":94},"0a409bd4":{"m":23,"g":94},"e4db4e5b":{"m":23,"g":94},"bbc07c41":{"m":23,"g":94},"a036d419":{"m":23,"g":94},"f95e6617":{"m":23,"g":94},"de854fb5":{"m":23,"g":94},"f64b2a9b":{"m":23,"g":94},"9f95dcc6":{"m":23,"g":94},"0736b270":{"m":23,"g":94},"3fdab919":{"m":23,"g":94},"ba29504b":{"m":23,"g":94},"a72342f1":{"m":23,"g":94},"c3c74bf8":{"m":23,"g":94},"d9fccfef":{"m":23,"g":94},"679ebcbb":{"m":23,"g":94},"1edd4e07":{"m":24,"g":94},"62c673c4":{"m":24,"g":94},"377c5dc9":{"m":24,"g":94},"f52eda35":{"m":24,"g":94},"b579ecf0":{"m":24,"g":94},"e7487b08":{"m":24,"g":94},"ae5c0fc4":{"m":24,"g":94},"a30d5d75":{"m":24,"g":94},"17af39c5":{"m":24,"g":94},"daf593a3":{"m":24,"g":94},"bece265f":{"m":24,"g":94},"cdcbde5f":{"m":24,"g":94},"21e22b9e":{"m":24,"g":94},"a50c8a14":{"m":24,"g":94},"db6089e6":{"m":24,"g":94},"3520f75f":{"m":24,"g":94},"c8e9fed8":{"m":24,"g":94},"084fa54d":{"m":24,"g":94},"eba458bd":{"m":24,"g":94},"3d1cb0af":{"m":24,"g":94},"7d352b4f":{"m":24,"g":94},"87064015":{"m":24,"g":94},"7cd4f244":{"m":24,"g":94},"98111fbe":{"m":24,"g":94},"2ec39ab7":{"m":24,"g":94},"8f6274c8":{"m":24,"g":94},"325a06c2":{"m":24,"g":94},"79f81629":{"m":24,"g":94},"b688fd85":{"m":24,"g":94},"5bd89924":{"m":24,"g":94},"8d908a93":{"m":24,"g":94},"dd7e8b94":{"m":24,"g":94},"1f013d64":{"m":24,"g":94},"628e1fa7":{"m":24,"g":94},"c71880f8":{"m":24,"g":94},"bcb6611a":{"m":24,"g":94},"fa2aa0db":{"m":24,"g":94},"6a387a69":{"m":24,"g":94},"27f5ce0a":{"m":24,"g":94},"94862579":{"m":24,"g":94},"68e52626":{"m":24,"g":94},"e4d3333c":{"m":25,"g":94},"6f221d4c":{"m":25,"g":94},"aba6f51f":{"m":25,"g":94},"7f6c690b":{"m":25,"g":94},"40e6f513":{"m":25,"g":94},"40756776":{"m":25,"g":94},"9e8d2c7f":{"m":25,"g":94},"c9bff5fc":{"m":25,"g":94},"b04444ac":{"m":25,"g":94},"3d617a21":{"m":25,"g":94},"c020f9ce":{"m":25,"g":94},"ca600e8c":{"m":25,"g":94},"0c0c8137":{"m":25,"g":94},"90286d85":{"m":25,"g":94},"5e7dd984":{"m":25,"g":94},"bc3eaac2":{"m":25,"g":94},"a78d98de":{"m":25,"g":94},"7d5ed7c6":{"m":25,"g":94},"a6c7ebbb":{"m":25,"g":94},"bb0501c0":{"m":25,"g":94},"6b0f2e90":{"m":25,"g":94},"30a9b2ef":{"m":26,"g":94},"3cadecf0":{"m":26,"g":94},"e90e3a50":{"m":26,"g":94},"fbd6b94d":{"m":26,"g":94},"4c8093c8":{"m":26,"g":94},"ae7ee01a":{"m":26,"g":94},"76e59088":{"m":26,"g":94},"12ce3bef":{"m":26,"g":94},"4013a4e1":{"m":26,"g":94},"60340a36":{"m":26,"g":94},"70c78cfb":{"m":26,"g":94},"72b6ea88":{"m":26,"g":94},"b906c015":{"m":27,"g":94},"9319cd13":{"m":27,"g":94},"046c2b33":{"m":27,"g":94},"6b8f66ef":{"m":27,"g":94},"7937a886":{"m":27,"g":94},"2e218b9e":{"m":27,"g":94},"141e8c71":{"m":28,"g":94},"d53dcf9c":{"m":28,"g":94},"bb66cc4c":{"m":28,"g":94},"975adb80":{"m":28,"g":94},"0d4f3a9f":{"m":28,"g":94},"afd411d0":{"m":28,"g":94},"e1eae1fd":{"m":28,"g":94},"f4d9953d":{"m":28,"g":94},"4f005250":{"m":28,"g":94},"995af5a5":{"m":28,"g":94},"53985645":{"m":28,"g":94},"70cc0749":{"m":28,"g":94},"7dd8a7e6":{"m":28,"g":94},"947402c8":{"m":28,"g":94},"8c5382e6":{"m":28,"g":94},"001b0bdd":{"m":28,"g":94},"dc9d06d8":{"m":29,"g":94},"c31f084c":{"m":29,"g":94},"a01ddd96":{"m":29,"g":94},"7fa54a1a":{"m":29,"g":94},"05abd126":{"m":29,"g":94},"5f6fa04a":{"m":29,"g":94},"58a09708":{"m":29,"g":94},"ff68ae85":{"m":29,"g":94},"795eab6d":{"m":29,"g":94},"41bb1ab1":{"m":29,"g":94},"87e8c090":{"m":29,"g":94},"ad56e684":{"m":29,"g":94},"ffb15744":{"m":29,"g":94},"a9c833d5":{"m":29,"g":94},"94e01151":{"m":29,"g":94},"b216a545":{"m":29,"g":94},"fde83405":{"m":29,"g":94},"fd7926e4":{"m":29,"g":94},"399cad91":{"m":29,"g":94},"0a4f5f9b":{"m":29,"g":94},"3bc99e6f":{"m":29,"g":94},"ebf69964":{"m":29,"g":94},"b0ad0c1b":{"m":30,"g":94},"c877292c":{"m":30,"g":94},"0c1c72a0":{"m":30,"g":94},"41598e0d":{"m":30,"g":94},"89f23a51":{"m":30,"g":94},"cb99ba4f":{"m":30,"g":94},"32f61443":{"m":30,"g":94},"fb1f28cb":{"m":30,"g":94},"fb7421db":{"m":30,"g":94},"14b64930":{"m":30,"g":94},"82076370":{"m":30,"g":94},"7de60345":{"m":30,"g":94},"d84c5e70":{"m":30,"g":94},"d7854120":{"m":30,"g":94},"7b6a5332":{"m":30,"g":94},"4080e822":{"m":30,"g":94},"c245b789":{"m":30,"g":94},"9dae4078":{"m":30,"g":94},"fcc0f5ed":{"m":30,"g":94},"a97df791":{"m":30,"g":94},"33d61356":{"m":30,"g":94},"94752ac8":{"m":30,"g":94},"43fbb6d9":{"m":30,"g":94},"54fb1c80":{"m":30,"g":94},"b68c4c07":{"m":30,"g":94},"e712837d":{"m":30,"g":94},"7599bade":{"m":30,"g":94},"62757db6":{"m":30,"g":94},"73fa2d49":{"m":30,"g":94},"61728884":{"m":30,"g":94},"9cf0a5ba":{"m":30,"g":94},"b16e856f":{"m":30,"g":94},"05c50a82":{"m":30,"g":94},"b568df5d":{"m":30,"g":94},"10bca45b":{"m":30,"g":94},"b91a4cb1":{"m":30,"g":94},"95a28019":{"m":30,"g":94},"e040a245":{"m":30,"g":94},"9f662501":{"m":30,"g":94},"ab787594":{"m":30,"g":94},"228cf475":{"m":30,"g":94},"3a79613c":{"m":30,"g":94},"1ac304ee":{"m":30,"g":94},"20a4f927":{"m":30,"g":94},"0de7c2d0":{"m":30,"g":94},"6ed4e3b8":{"m":30,"g":94},"00023d62":{"m":30,"g":94},"c62d560c":{"m":30,"g":94},"2b8257f3":{"m":30,"g":94},"7623091d":{"m":30,"g":94},"f724f1f1":{"m":30,"g":94},"6db27f7b":{"m":30,"g":94},"4d929107":{"m":30,"g":94},"fbe0c818":{"m":30,"g":94},"5bd95374":{"m":31,"g":94},"0cb099e2":{"m":31,"g":94},"93d4e354":{"m":31,"g":94},"9195d136":{"m":31,"g":94},"14cb544d":{"m":31,"g":94},"e86b1ccb":{"m":31,"g":94},"8d2d876f":{"m":31,"g":94},"326df4ba":{"m":31,"g":94},"6767e222":{"m":31,"g":94},"73cf6834":{"m":31,"g":94},"1c2b5f52":{"m":31,"g":94},"96a2093e":{"m":31,"g":94},"a34dd86a":{"m":31,"g":94},"67c0d832":{"m":31,"g":94},"a59636bb":{"m":31,"g":94},"fe502432":{"m":31,"g":94},"f14569f6":{"m":31,"g":94},"8f790ac1":{"m":31,"g":94},"616b59f3":{"m":31,"g":94},"c8423ca3":{"m":31,"g":94},"e205527c":{"m":31,"g":94},"0909bb0d":{"m":31,"g":94},"ad3e4f16":{"m":31,"g":94},"312e8492":{"m":31,"g":94},"95f5fbf1":{"m":31,"g":94},"cebd78d8":{"m":31,"g":94},"0076f115":{"m":31,"g":94},"f7fb68d2":{"m":31,"g":94},"396a13e6":{"m":31,"g":94},"65915f9f":{"m":31,"g":94},"162f3ccb":{"m":31,"g":94},"65e89bae":{"m":31,"g":94},"6a38efa8":{"m":31,"g":94},"c5fe11a8":{"m":32,"g":94},"75ce37f4":{"m":32,"g":94},"97589a60":{"m":32,"g":94},"632d506d":{"m":32,"g":94},"3579162a":{"m":32,"g":94},"7514b9f8":{"m":32,"g":94},"158e8f1e":{"m":32,"g":94},"d3efcb39":{"m":32,"g":94},"2c615d12":{"m":32,"g":94},"61bb223e":{"m":32,"g":94},"15f1a49d":{"m":32,"g":94},"308d0240":{"m":32,"g":94},"ab4990e4":{"m":32,"g":94},"90227800":{"m":32,"g":94},"30b4f771":{"m":32,"g":94},"66e7dcaf":{"m":32,"g":94},"bc4c7a35":{"m":32,"g":94},"1cb4da5c":{"m":32,"g":94},"e61d13ac":{"m":32,"g":94},"b20daf98":{"m":32,"g":94},"f6af3a65":{"m":32,"g":94},"c9064e6f":{"m":32,"g":94},"a5b14ad0":{"m":32,"g":94},"5fafcac0":{"m":32,"g":94},"364d3d72":{"m":32,"g":94},"5623826f":{"m":32,"g":94},"83e23c69":{"m":32,"g":94},"ac1b74fa":{"m":32,"g":94},"068e9eae":{"m":32,"g":94},"d6aeb9fa":{"m":32,"g":94},"1fb94599":{"m":32,"g":94},"bea2bb9e":{"m":32,"g":94},"cd10654e":{"m":32,"g":94},"350a8160":{"m":32,"g":94},"6242c399":{"m":32,"g":94},"04707b09":{"m":32,"g":94},"ff2cfdb1":{"m":32,"g":94},"a8ae6403":{"m":32,"g":94},"d8476818":{"m":32,"g":94},"df191254":{"m":32,"g":94},"b997a18d":{"m":32,"g":94},"d8627ed1":{"m":32,"g":94},"fa13b95d":{"m":32,"g":94},"3c1f5a92":{"m":32,"g":94},"57d0bd91":{"m":32,"g":94},"cdc8d607":{"m":32,"g":94},"9208591f":{"m":32,"g":94},"5d0d40d0":{"m":32,"g":94},"f624f6a6":{"m":32,"g":94},"3694f8f9":{"m":32,"g":94},"5a261bd0":{"m":32,"g":94},"6aa8ad14":{"m":32,"g":94},"26e9c12c":{"m":32,"g":94},"87a0db82":{"m":32,"g":94},"f25f4dfd":{"m":33,"g":94},"184ae1c6":{"m":33,"g":94},"198974cd":{"m":33,"g":94},"6cc38b2b":{"m":33,"g":94},"1ece2cda":{"m":33,"g":94},"c8a9e791":{"m":33,"g":94},"3602692c":{"m":33,"g":94},"909f3436":{"m":33,"g":94},"5ff25cdf":{"m":33,"g":94},"2f1d9283":{"m":33,"g":94},"c61a1b6f":{"m":33,"g":94},"9935f97b":{"m":33,"g":94},"13ac95b8":{"m":34,"g":94},"492143bf":{"m":34,"g":94},"0a97d796":{"m":34,"g":94},"c411f32e":{"m":34,"g":94},"bf53bf51":{"m":34,"g":94},"b1a540ec":{"m":34,"g":94},"66975360":{"m":34,"g":94},"6c498313":{"m":34,"g":94},"99994427":{"m":35,"g":94},"6def9b01":{"m":35,"g":94},"47f20da2":{"m":35,"g":94},"4a9f8ea4":{"m":35,"g":94},"58fa6076":{"m":35,"g":94},"6487ef64":{"m":35,"g":94},"9b080524":{"m":35,"g":94},"32a4141d":{"m":35,"g":94},"08360553":{"m":35,"g":94},"00b19f19":{"m":35,"g":94},"6cb32ef9":{"m":35,"g":94},"761b2ceb":{"m":35,"g":94},"54772f78":{"m":35,"g":94},"1b5d56f7":{"m":35,"g":94},"d134c139":{"m":35,"g":94},"6cc9c525":{"m":35,"g":94},"52cefdbf":{"m":35,"g":94},"51c554d8":{"m":35,"g":94},"79ece2c5":{"m":35,"g":94},"55f5976b":{"m":35,"g":94},"b7f83410":{"m":35,"g":94},"f414352a":{"m":35,"g":94},"a362340b":{"m":35,"g":94},"381dd57b":{"m":35,"g":94},"8153168c":{"m":35,"g":94},"6c34d633":{"m":35,"g":94},"5ab9418f":{"m":36,"g":94},"843e63d8":{"m":36,"g":94},"a63c8275":{"m":36,"g":94},"dc67d976":{"m":36,"g":94},"1e495e08":{"m":36,"g":94},"12cb115d":{"m":36,"g":94},"c500f96b":{"m":36,"g":94},"474317f2":{"m":36,"g":94},"f64eae3a":{"m":36,"g":94},"a5a134f3":{"m":36,"g":94},"2561ed01":{"m":36,"g":94},"90a26be3":{"m":37,"g":94},"1f4b5f77":{"m":37,"g":94},"76524b70":{"m":37,"g":94},"3a6e0418":{"m":37,"g":94},"2fa5cec7":{"m":37,"g":94},"27b557ae":{"m":37,"g":94},"93dffd69":{"m":37,"g":94},"2abe4f1c":{"m":37,"g":94},"37963394":{"m":37,"g":94},"899cf5c4":{"m":37,"g":94},"e79f6cd7":{"m":37,"g":94},"9ba1f097":{"m":37,"g":94},"282681b8":{"m":37,"g":94},"58cafe23":{"m":37,"g":94},"9463bc13":{"m":37,"g":94},"e3fc4658":{"m":37,"g":94},"33b54e7c":{"m":37,"g":94},"30b404ce":{"m":37,"g":94},"70b68029":{"m":37,"g":94},"f3d32f88":{"m":37,"g":94},"8779da95":{"m":37,"g":94},"ad0ff62a":{"m":37,"g":94},"9a903a87":{"m":37,"g":94},"68be2f6d":{"m":37,"g":94},"b912de11":{"m":37,"g":94},"eb02c161":{"m":37,"g":94},"71221692":{"m":37,"g":94},"c33d82a2":{"m":37,"g":94},"8234e663":{"m":37,"g":94},"debbdb51":{"m":37,"g":94},"3efa7981":{"m":37,"g":94},"2a71be5e":{"m":37,"g":94},"44621377":{"m":37,"g":94},"fec185ce":{"m":37,"g":94},"c03cece4":{"m":37,"g":94},"15c75e41":{"m":37,"g":94},"224200e3":{"m":37,"g":94},"8c0efa51":{"m":37,"g":94},"144bc70f":{"m":37,"g":94},"46094e0c":{"m":37,"g":94},"3a6e8b6d":{"m":37,"g":94},"fbb4754c":{"m":37,"g":94},"6c7cb903":{"m":37,"g":94},"dff2860a":{"m":37,"g":94},"e72275cf":{"m":37,"g":94},"fec2d122":{"m":37,"g":94},"8d1095db":{"m":37,"g":94},"743007e1":{"m":37,"g":94},"9144ed10":{"m":37,"g":94},"69b3bb9a":{"m":37,"g":94},"689ff588":{"m":37,"g":94},"a7c47e0f":{"m":37,"g":94},"e4d68afc":{"m":37,"g":94},"c9b75917":{"m":37,"g":94},"662ecd93":{"m":37,"g":94},"8e6bdf85":{"m":37,"g":94},"05bea688":{"m":37,"g":94},"ab4a83b2":{"m":37,"g":94},"62f15eea":{"m":37,"g":94},"79794af5":{"m":37,"g":94},"3494b32c":{"m":37,"g":94},"eda7c090":{"m":37,"g":94},"5ce55aee":{"m":38,"g":94},"2d346a57":{"m":38,"g":94},"446ea332":{"m":38,"g":94},"8f527e29":{"m":38,"g":94},"7f24ea95":{"m":38,"g":94},"1acccb36":{"m":38,"g":94},"aa2750be":{"m":38,"g":94},"5e62a6b7":{"m":38,"g":94},"5752f25e":{"m":38,"g":94},"7c162fa9":{"m":38,"g":94},"36078fb2":{"m":38,"g":94},"b3710d2c":{"m":38,"g":94},"c6b6d2e7":{"m":38,"g":94},"82136eb0":{"m":39,"g":94},"b8ccaf4d":{"m":39,"g":94},"a68cb201":{"m":39,"g":94},"014982b5":{"m":39,"g":94},"a6db8862":{"m":39,"g":94},"b4408b0d":{"m":39,"g":94},"2cd7e181":{"m":39,"g":94},"37c5899f":{"m":40,"g":94},"f39a0197":{"m":40,"g":94},"3c93187c":{"m":40,"g":94},"fb2d0680":{"m":40,"g":94},"067d8e16":{"m":40,"g":94},"e6692bf4":{"m":40,"g":94},"28b4d8e1":{"m":40,"g":94},"bc068e96":{"m":40,"g":94},"8d4ed42a":{"m":40,"g":94},"2854a5ea":{"m":40,"g":94},"42a2d82b":{"m":40,"g":94},"e4780cf8":{"m":40,"g":94},"39bb49d1":{"m":40,"g":94},"6f3cf129":{"m":40,"g":94},"13f1357e":{"m":40,"g":94},"2a99993c":{"m":40,"g":94},"167591e8":{"m":40,"g":94},"441c22db":{"m":40,"g":94},"ce636ac4":{"m":40,"g":94},"7b69d91b":{"m":41,"g":94},"e8613df0":{"m":41,"g":94},"c5325aba":{"m":41,"g":94},"ebbc42d9":{"m":41,"g":94},"3ff64113":{"m":41,"g":94},"2b302b93":{"m":41,"g":94},"68f8b60d":{"m":41,"g":94},"6a5b352a":{"m":41,"g":94},"565b05f0":{"m":41,"g":94},"b6aad70a":{"m":41,"g":94},"551a3a9d":{"m":41,"g":94},"91877a9f":{"m":41,"g":94},"f7cce751":{"m":41,"g":94},"17e998f1":{"m":41,"g":94},"c98e84c2":{"m":41,"g":94},"9c064bf7":{"m":41,"g":94},"58d1082e":{"m":41,"g":94},"4d086719":{"m":41,"g":94},"9244f27f":{"m":41,"g":94},"2422de51":{"m":41,"g":94},"521f862d":{"m":41,"g":94},"34c32d28":{"m":41,"g":94},"dde8bb16":{"m":41,"g":94},"8ac3ccc0":{"m":41,"g":94},"9b0926ce":{"m":41,"g":94},"1c1bdc76":{"m":41,"g":94},"6bfdb403":{"m":41,"g":94},"f8fb4ce9":{"m":41,"g":94},"5d0ba403":{"m":41,"g":94},"04b262cd":{"m":41,"g":94},"2432ad40":{"m":41,"g":94},"45473d4b":{"m":41,"g":94},"114bbc86":{"m":41,"g":94},"32eb6e96":{"m":41,"g":94},"e0b5dbce":{"m":41,"g":94},"e6852b0d":{"m":41,"g":94},"4ae0969c":{"m":41,"g":94},"317631ca":{"m":41,"g":94},"b5648353":{"m":41,"g":94},"2c7d0a5b":{"m":41,"g":94},"8cdc76f6":{"m":41,"g":94},"f202ed97":{"m":41,"g":94},"100f5b8b":{"m":41,"g":94},"619bb6dd":{"m":41,"g":94},"b88ea90d":{"m":41,"g":94},"99ec439d":{"m":41,"g":94},"0f4fb19b":{"m":41,"g":94},"63ba2f8d":{"m":41,"g":94},"36d5acfc":{"m":41,"g":94},"3f0fe08d":{"m":41,"g":94},"55b974f9":{"m":41,"g":94},"f86c1e61":{"m":41,"g":94},"acaffd23":{"m":41,"g":94},"04868543":{"m":41,"g":94},"fd9ad817":{"m":41,"g":94},"e165a9fc":{"m":41,"g":94},"4e4459b9":{"m":41,"g":94},"065bb947":{"m":41,"g":94},"f42e9bfb":{"m":41,"g":94},"840c5dbc":{"m":41,"g":94},"63e845d0":{"m":41,"g":94},"9aa6553d":{"m":41,"g":94},"b1e330bc":{"m":41,"g":94},"4353acb4":{"m":41,"g":94},"9ae1db0b":{"m":41,"g":94},"00c7e636":{"m":42,"g":94},"23cc66f7":{"m":42,"g":94},"5d09ca57":{"m":42,"g":94},"81c33274":{"m":42,"g":94},"f13d86f9":{"m":42,"g":94},"aba9eae4":{"m":42,"g":94},"bbd72bfc":{"m":42,"g":94},"b503881b":{"m":42,"g":94},"58093b86":{"m":42,"g":94},"8275049c":{"m":42,"g":94},"5476ccad":{"m":42,"g":94},"b040ed71":{"m":42,"g":94},"c9e66586":{"m":42,"g":94},"e11ab79e":{"m":42,"g":94},"01fdb2f3":{"m":42,"g":94},"c996e8cc":{"m":42,"g":94},"087257ea":{"m":43,"g":94},"736f0402":{"m":43,"g":94},"769bf11c":{"m":43,"g":94},"3db43d1b":{"m":43,"g":94},"f0f8a769":{"m":43,"g":94},"2bcfba1b":{"m":43,"g":94},"bc12d403":{"m":43,"g":94},"392f2863":{"m":43,"g":94},"6d0fa73e":{"m":43,"g":94},"9e0dac1a":{"m":43,"g":94},"a95d5589":{"m":43,"g":94},"d17d19e5":{"m":43,"g":94},"dd3809fa":{"m":43,"g":94},"7feba415":{"m":43,"g":94},"30ee3630":{"m":43,"g":94},"e5db40dc":{"m":43,"g":94},"b1709305":{"m":43,"g":94},"5ab20cce":{"m":43,"g":94},"02f7f3e4":{"m":43,"g":94},"2782132b":{"m":43,"g":94},"d19cc0b9":{"m":43,"g":94},"b0facb33":{"m":43,"g":94},"ecb8bad2":{"m":43,"g":94},"dbec2f18":{"m":43,"g":94},"e4b367ba":{"m":43,"g":94},"d10b933a":{"m":43,"g":94},"9116b289":{"m":43,"g":94},"a5114b6f":{"m":43,"g":94},"b6b40946":{"m":43,"g":94},"f1088e0f":{"m":43,"g":94},"175afed3":{"m":43,"g":94},"4a292f67":{"m":43,"g":94},"cd0be748":{"m":43,"g":94},"56503d9b":{"m":43,"g":94},"02bc9579":{"m":43,"g":94},"24f3e151":{"m":43,"g":94},"6790240c":{"m":43,"g":94},"061e5463":{"m":43,"g":94},"0c1e8796":{"m":43,"g":94},"869f1c02":{"m":43,"g":94},"2725f8da":{"m":43,"g":94},"da1ffed6":{"m":43,"g":94},"48761171":{"m":43,"g":94},"c3f2fc5a":{"m":43,"g":94},"7ee6c259":{"m":43,"g":94},"9610fcd4":{"m":43,"g":94},"31fad29a":{"m":43,"g":94},"9da5a60b":{"m":43,"g":94},"69aa937a":{"m":43,"g":94},"5d638c92":{"m":43,"g":94},"e37cdab0":{"m":43,"g":94},"1d9deeac":{"m":43,"g":94},"dafb6a52":{"m":43,"g":94},"862cd265":{"m":43,"g":94},"1f26e8b8":{"m":44,"g":94},"5e1558f1":{"m":44,"g":94},"94cde109":{"m":44,"g":94},"00611286":{"m":44,"g":94},"e68b9e76":{"m":44,"g":94},"7ce36068":{"m":44,"g":94},"efb099cd":{"m":44,"g":94},"09603c6d":{"m":44,"g":94},"cf470fea":{"m":44,"g":94},"45d5af24":{"m":44,"g":94},"b121bc03":{"m":44,"g":94},"e12358dc":{"m":44,"g":94},"554fbf93":{"m":44,"g":94},"b48edff6":{"m":44,"g":94},"593b19f2":{"m":44,"g":94},"59cbf476":{"m":44,"g":94},"95946271":{"m":44,"g":94},"5c4ce656":{"m":44,"g":94},"cbbc82b7":{"m":44,"g":94},"8bee20f8":{"m":44,"g":94},"12cad0fe":{"m":44,"g":94},"b6cd9036":{"m":44,"g":94},"30643fed":{"m":45,"g":94},"e646c590":{"m":45,"g":94},"c555ce2c":{"m":45,"g":94},"40900bae":{"m":45,"g":94},"a2f5e755":{"m":45,"g":94},"2148914e":{"m":45,"g":94},"def55bc8":{"m":45,"g":94},"86a2c473":{"m":45,"g":94},"1701b0db":{"m":45,"g":94},"384d85ba":{"m":45,"g":94},"60597219":{"m":45,"g":94},"fc82f5a7":{"m":45,"g":94},"0089c4bc":{"m":45,"g":94},"72e7b57a":{"m":45,"g":94},"87a7cfa0":{"m":45,"g":94},"8f8f96a6":{"m":45,"g":94},"05b3bf5e":{"m":45,"g":94},"3f5ac88d":{"m":45,"g":94},"0d800090":{"m":45,"g":94},"b7d05594":{"m":45,"g":94},"80a90547":{"m":45,"g":94},"9af7b88e":{"m":45,"g":94},"fbcbb263":{"m":45,"g":94},"2fce449b":{"m":45,"g":94},"ad4125d1":{"m":45,"g":94},"17536e7e":{"m":45,"g":94},"65859754":{"m":46,"g":94},"2ce32db6":{"m":46,"g":94},"793b79db":{"m":46,"g":94},"1363b519":{"m":46,"g":94},"0abbf289":{"m":46,"g":94},"c17c5781":{"m":46,"g":94},"916b3cdd":{"m":46,"g":94},"838dcda1":{"m":46,"g":94},"efbc116a":{"m":46,"g":94},"6aed0445":{"m":46,"g":94},"908dd7f9":{"m":46,"g":94},"f4cd8040":{"m":46,"g":94},"be7986e0":{"m":46,"g":94},"5a5f1843":{"m":46,"g":94},"7b394e5f":{"m":46,"g":94},"3b60558d":{"m":46,"g":94},"5a9a4f41":{"m":46,"g":94},"72e979bf":{"m":46,"g":94},"146f6134":{"m":46,"g":94},"660ecb73":{"m":46,"g":94},"2565cb0f":{"m":46,"g":94},"066e8a4e":{"m":46,"g":94},"2134f089":{"m":46,"g":94},"a54f278d":{"m":46,"g":94},"d1b31b06":{"m":46,"g":94},"d59a4782":{"m":46,"g":94},"104bf260":{"m":46,"g":94},"3bf3d011":{"m":46,"g":94},"d86a2d65":{"m":46,"g":94},"16eb33ff":{"m":46,"g":94},"61cf00e1":{"m":46,"g":94},"b9fd178f":{"m":46,"g":94},"d8e9d61f":{"m":46,"g":94},"a2e0424a":{"m":46,"g":94},"8ce202a4":{"m":46,"g":94},"d913d52c":{"m":46,"g":94},"0ab7bcaf":{"m":46,"g":94},"438526a8":{"m":46,"g":94},"f7102fbd":{"m":46,"g":94},"a7a0a688":{"m":46,"g":94},"2d4ce1b7":{"m":46,"g":94},"4ba815b8":{"m":46,"g":94},"5f65e2b8":{"m":46,"g":94},"4e2af03c":{"m":46,"g":94},"3184aa95":{"m":46,"g":94},"b548801d":{"m":46,"g":94},"539df95d":{"m":46,"g":94},"5e00ddeb":{"m":46,"g":94},"54dd3ea1":{"m":46,"g":94},"d04899d7":{"m":46,"g":94},"5010e0d2":{"m":46,"g":94},"5e6c3265":{"m":46,"g":94},"680cad20":{"m":46,"g":94},"0a24eb85":{"m":46,"g":94},"3839be29":{"m":46,"g":94},"6e13b650":{"m":46,"g":94},"6fcd6d7d":{"m":46,"g":94},"c77762d5":{"m":46,"g":94},"51c81e33":{"m":46,"g":94},"eaade87a":{"m":46,"g":94},"86fc0d79":{"m":46,"g":94},"1be853ee":{"m":46,"g":94},"86e0dde5":{"m":46,"g":94},"2b809788":{"m":46,"g":94},"9d6fb084":{"m":46,"g":94},"ced362f7":{"m":46,"g":94},"9084a864":{"m":46,"g":94},"6aa94b96":{"m":46,"g":94},"c2650748":{"m":46,"g":94},"07bf2e84":{"m":46,"g":94},"a628dd8e":{"m":46,"g":94},"1e890341":{"m":46,"g":94},"715b16c1":{"m":46,"g":94},"9ce8e1a9":{"m":46,"g":94},"fb99aaa5":{"m":46,"g":94},"b77a02cd":{"m":46,"g":94},"f407fcf9":{"m":47,"g":94},"54479d6f":{"m":47,"g":94},"ba069a24":{"m":47,"g":94},"125b1199":{"m":47,"g":94},"eff468dd":{"m":47,"g":94},"a1bd7190":{"m":47,"g":94},"78c1d644":{"m":47,"g":94},"027e6524":{"m":47,"g":94},"b808a383":{"m":47,"g":94},"602ebc66":{"m":47,"g":94},"530ae1bd":{"m":47,"g":94},"befc6beb":{"m":47,"g":94},"59a5ba9b":{"m":47,"g":94},"86c37d01":{"m":47,"g":94},"f18b9c72":{"m":47,"g":94},"3e335743":{"m":47,"g":94},"0d94f1dd":{"m":47,"g":94},"e728258d":{"m":47,"g":94},"239eafbd":{"m":47,"g":94},"9d427265":{"m":47,"g":94},"00ffde20":{"m":47,"g":94},"ddeb9d42":{"m":47,"g":94},"aaf0a315":{"m":47,"g":94},"f9633fa9":{"m":47,"g":94},"087ab832":{"m":47,"g":94},"8169c6f4":{"m":47,"g":94},"3d043319":{"m":47,"g":94},"a8aad935":{"m":47,"g":94},"47ffe7af":{"m":47,"g":94},"b3523af8":{"m":47,"g":94},"1929c067":{"m":47,"g":94},"ed53ac84":{"m":47,"g":94},"520f0094":{"m":47,"g":94},"9c939a3d":{"m":47,"g":94},"549e8b83":{"m":47,"g":94},"a1f32867":{"m":47,"g":94},"760552e0":{"m":47,"g":94},"d9aada9d":{"m":47,"g":94},"f11eb90f":{"m":47,"g":94},"95a4ed12":{"m":47,"g":94},"d1150e9a":{"m":47,"g":94},"e3126e3c":{"m":47,"g":94},"a5095520":{"m":47,"g":94},"7ef0084b":{"m":47,"g":94},"f9a377f6":{"m":47,"g":94},"4ade15dd":{"m":47,"g":94},"8dc84da0":{"m":47,"g":94},"f16eb15d":{"m":47,"g":94},"5bc2508b":{"m":47,"g":94},"a71a44f2":{"m":47,"g":94},"691808d5":{"m":47,"g":94},"d32fba2a":{"m":47,"g":94},"67c424cc":{"m":47,"g":94},"1ae270c5":{"m":47,"g":94},"c77c1e05":{"m":47,"g":94},"dca87ec3":{"m":47,"g":94},"4b1d7a25":{"m":47,"g":94},"a5e0defb":{"m":47,"g":94},"96766101":{"m":47,"g":94},"a146d999":{"m":47,"g":94},"f5113e50":{"m":47,"g":94},"02755768":{"m":47,"g":94},"463d56bf":{"m":47,"g":94},"530ff541":{"m":47,"g":94},"3cd28092":{"m":47,"g":94},"704f8e8e":{"m":47,"g":94},"1853c352":{"m":47,"g":94},"32c9a7ec":{"m":48,"g":94},"b01df48c":{"m":48,"g":94},"c29b98e0":{"m":48,"g":94},"954f4e6b":{"m":48,"g":94},"2558d6a6":{"m":48,"g":94},"29ebe3df":{"m":48,"g":94},"f6dd6486":{"m":48,"g":94},"ea53c63b":{"m":48,"g":94},"a10d5309":{"m":48,"g":94},"aae5434b":{"m":48,"g":94},"c3eac1b0":{"m":48,"g":94},"b275ce00":{"m":48,"g":94},"13ce3e4b":{"m":48,"g":94},"df246e69":{"m":48,"g":94},"fb9fb351":{"m":48,"g":94},"c722d9bd":{"m":48,"g":94},"218ab361":{"m":48,"g":94},"9a00e6f4":{"m":49,"g":94},"4f8c3aea":{"m":49,"g":94},"2369e882":{"m":49,"g":94},"ad30d5cf":{"m":49,"g":94},"dfec7fca":{"m":49,"g":94},"8048c28c":{"m":49,"g":94},"30af7dfb":{"m":49,"g":94},"f6f71379":{"m":49,"g":94},"f35cb46c":{"m":49,"g":94},"7f8fcd39":{"m":49,"g":94},"5c6a41fa":{"m":49,"g":94},"722530fa":{"m":49,"g":94},"56a347f7":{"m":49,"g":94},"3295cd8a":{"m":49,"g":94},"5942dfc0":{"m":49,"g":94},"63a395b9":{"m":49,"g":94},"7d671e4a":{"m":49,"g":94},"699384cb":{"m":49,"g":94},"ffd20fcd":{"m":49,"g":94},"55bd97f3":{"m":49,"g":94},"e57c3e12":{"m":49,"g":94},"f239268f":{"m":49,"g":94},"929c7621":{"m":49,"g":94},"b7a065ea":{"m":49,"g":94},"b1104538":{"m":49,"g":94},"3b44bbee":{"m":49,"g":94},"80e2c4a8":{"m":49,"g":94},"66318ffe":{"m":49,"g":94},"76619261":{"m":49,"g":94},"2a3992b6":{"m":49,"g":94},"4af3f889":{"m":49,"g":94},"df7fe452":{"m":49,"g":94},"a7164b62":{"m":49,"g":94},"11668533":{"m":49,"g":94},"a9e90b4b":{"m":49,"g":94},"8c280cee":{"m":49,"g":94},"9c745d07":{"m":49,"g":94},"ebaa2f31":{"m":49,"g":94},"62832bb2":{"m":49,"g":94},"11f881d1":{"m":49,"g":94},"38625e21":{"m":49,"g":94},"c1f401fc":{"m":49,"g":94},"3b878863":{"m":49,"g":94},"f719d9ae":{"m":49,"g":94},"edad3731":{"m":49,"g":94},"976bc302":{"m":49,"g":94},"2f2e0743":{"m":49,"g":94},"2ffe0a73":{"m":49,"g":94},"cf248976":{"m":49,"g":94},"e5c67150":{"m":49,"g":94},"023d0a73":{"m":49,"g":94},"ac5a0f04":{"m":50,"g":94},"ea34350d":{"m":50,"g":94},"1605ae12":{"m":50,"g":94},"1aea19f6":{"m":50,"g":94},"1f76fc6e":{"m":50,"g":94},"7f076c2c":{"m":50,"g":94},"3c5538f7":{"m":50,"g":94},"10189d08":{"m":50,"g":94},"c4336b2b":{"m":50,"g":94},"4d62bca5":{"m":50,"g":94},"e1e595d7":{"m":50,"g":94},"5ada33ff":{"m":50,"g":94},"254fd130":{"m":50,"g":94},"538fa0ae":{"m":50,"g":94},"55842eb8":{"m":50,"g":94},"a866b65e":{"m":50,"g":94},"4b0a1c93":{"m":50,"g":94},"8e1adb84":{"m":50,"g":94},"dd44173d":{"m":50,"g":94},"8912b763":{"m":50,"g":94},"be0124bd":{"m":50,"g":94},"fe5d3e81":{"m":50,"g":94},"731146f6":{"m":50,"g":94},"fa271613":{"m":50,"g":94},"5652c565":{"m":50,"g":94},"e3938b2f":{"m":50,"g":94},"c211e7b6":{"m":50,"g":94},"d90c3d6b":{"m":50,"g":94},"9e8f8fbf":{"m":50,"g":94},"b509db58":{"m":50,"g":94},"dbe17293":{"m":50,"g":94},"84a1698d":{"m":50,"g":94},"32293a29":{"m":50,"g":94},"79216908":{"m":50,"g":94},"bbb81c24":{"m":50,"g":94},"52f58fc4":{"m":50,"g":94},"145c0ddc":{"m":50,"g":94},"505d7f71":{"m":50,"g":94},"cbedd1db":{"m":50,"g":94},"ad47749b":{"m":50,"g":94},"751c3a03":{"m":50,"g":94},"60769be1":{"m":50,"g":94},"a78d8f8d":{"m":50,"g":94},"c5f86501":{"m":50,"g":94},"d98fa1e9":{"m":50,"g":94},"865233e2":{"m":50,"g":94},"66d4859a":{"m":50,"g":94},"e1b63624":{"m":50,"g":94},"c35cd1f8":{"m":50,"g":94},"72f87b72":{"m":50,"g":94},"62a4a339":{"m":50,"g":94},"2797bc34":{"m":50,"g":94},"fed4c694":{"m":51,"g":94},"fb6e04a0":{"m":51,"g":94},"6997e28f":{"m":51,"g":94},"a0e58740":{"m":51,"g":94},"37c8a576":{"m":51,"g":94},"c754652f":{"m":51,"g":94},"0b46b951":{"m":51,"g":94},"2763c0a7":{"m":51,"g":94},"de3b67b7":{"m":51,"g":94},"19f33b32":{"m":51,"g":94},"30ce5b59":{"m":51,"g":94},"bc1f6fda":{"m":51,"g":94},"867e092f":{"m":51,"g":94},"88c7763f":{"m":51,"g":94},"e4118b15":{"m":51,"g":94},"ba4ee37f":{"m":51,"g":94},"fae4e5e9":{"m":52,"g":94},"afe1e465":{"m":52,"g":94},"f50a6cf4":{"m":52,"g":94},"fe97a2d4":{"m":52,"g":94},"8b48496a":{"m":52,"g":94},"4057ea82":{"m":52,"g":94},"4f2ee48e":{"m":52,"g":94},"71ff2728":{"m":52,"g":94},"b7038fec":{"m":52,"g":94},"65fdb289":{"m":52,"g":94},"b2ccf36d":{"m":52,"g":94},"d4fc1a70":{"m":52,"g":94},"db674e3d":{"m":52,"g":94},"fb915bd1":{"m":52,"g":94},"09798b36":{"m":52,"g":94},"b79fffdc":{"m":52,"g":94},"cd51758f":{"m":52,"g":94},"91e5dbf5":{"m":52,"g":94},"dd5eba4c":{"m":52,"g":94},"a4fd2f9b":{"m":52,"g":94},"92d1253e":{"m":52,"g":94},"a9ca297d":{"m":52,"g":94},"2a02185c":{"m":52,"g":94},"f8b03269":{"m":53,"g":94},"04957965":{"m":53,"g":94},"1228f7ca":{"m":53,"g":94},"fda628d8":{"m":53,"g":94},"07ec07ad":{"m":53,"g":94},"83b340e3":{"m":53,"g":94},"0639bf15":{"m":53,"g":94},"aa47f642":{"m":53,"g":94},"3ddb1c46":{"m":53,"g":94},"480e38a7":{"m":53,"g":94},"69e2d4fb":{"m":53,"g":94},"85e1a6f3":{"m":53,"g":94},"33deca81":{"m":53,"g":94},"18108abe":{"m":53,"g":94},"c54bda30":{"m":53,"g":94},"3c79ad35":{"m":53,"g":94},"983bfcf3":{"m":53,"g":94},"28bc60dc":{"m":53,"g":94},"7301a39b":{"m":53,"g":94},"47eb139f":{"m":53,"g":94},"5c18a037":{"m":53,"g":94},"5c91a315":{"m":53,"g":94},"3dbd73d3":{"m":53,"g":94},"e9a6203d":{"m":53,"g":94},"62c516ac":{"m":53,"g":94},"fc78640e":{"m":53,"g":94},"906d795f":{"m":53,"g":94},"118b6af3":{"m":53,"g":94},"9449a954":{"m":53,"g":94},"5f12f0e7":{"m":53,"g":94},"d5b95cbb":{"m":53,"g":94},"0303ca91":{"m":53,"g":94},"00181098":{"m":53,"g":94},"4936be8a":{"m":53,"g":94},"1bfa511b":{"m":53,"g":94},"f5b5f2bf":{"m":53,"g":94},"7e4c6dd8":{"m":53,"g":94},"d622851d":{"m":53,"g":94},"883c9554":{"m":53,"g":94},"0d6a49bd":{"m":53,"g":94},"ccaf1f99":{"m":53,"g":94},"7d1485d3":{"m":53,"g":94},"7d5d1d3d":{"m":53,"g":94},"b53d6cbd":{"m":53,"g":94},"01017d4c":{"m":53,"g":94},"94e167ea":{"m":53,"g":94},"262e370f":{"m":53,"g":94},"419a57e7":{"m":53,"g":94},"e5f227c0":{"m":54,"g":94},"0e7409ad":{"m":54,"g":94},"3cde5eb6":{"m":54,"g":94},"f5b2a3aa":{"m":54,"g":94},"f6817596":{"m":54,"g":94},"67b65794":{"m":54,"g":94},"37ee906f":{"m":54,"g":94},"34b364e0":{"m":54,"g":94},"84d96b3a":{"m":54,"g":94},"3d32e4a3":{"m":54,"g":94},"64fceab8":{"m":54,"g":94},"71e2a277":{"m":54,"g":94},"4a63c181":{"m":54,"g":94},"2b0fc594":{"m":54,"g":94},"9cc733b3":{"m":54,"g":94},"d693ec04":{"m":54,"g":94},"18ea841f":{"m":54,"g":94},"786be44d":{"m":54,"g":94},"2db44698":{"m":54,"g":94},"ed45e509":{"m":54,"g":94},"ec52464d":{"m":54,"g":94},"eb0c1f53":{"m":54,"g":94},"b2986d7a":{"m":54,"g":94},"8f4d04e5":{"m":55,"g":94},"feb2b768":{"m":55,"g":94},"d95a5f5b":{"m":55,"g":94},"4b83db24":{"m":55,"g":94},"64456cf0":{"m":55,"g":94},"bb4a9220":{"m":55,"g":94},"21e9e63a":{"m":55,"g":94},"1fc84cf6":{"m":55,"g":94},"361ea8d9":{"m":55,"g":94},"33c5ff28":{"m":55,"g":94},"5ce9daea":{"m":55,"g":94},"ce094a5d":{"m":55,"g":94},"e2102669":{"m":55,"g":94},"bd619616":{"m":55,"g":94},"56198b45":{"m":55,"g":94},"ba36b552":{"m":55,"g":94},"9cd9dc83":{"m":55,"g":94},"7a1aecb9":{"m":55,"g":94},"82699474":{"m":55,"g":94},"7154b4b1":{"m":55,"g":94},"b532a5fd":{"m":55,"g":94},"a0592c05":{"m":55,"g":94},"e8dbdf75":{"m":55,"g":94},"e04d3f28":{"m":55,"g":94},"5f2595be":{"m":55,"g":94},"0ba2c589":{"m":55,"g":94},"fccbfa37":{"m":55,"g":94},"2f9bd0fa":{"m":55,"g":94},"5282a473":{"m":55,"g":94},"f0ed9c35":{"m":55,"g":94},"e3b3acfa":{"m":55,"g":94},"2673fa29":{"m":55,"g":94},"dedaf8cd":{"m":55,"g":94},"32ed0160":{"m":55,"g":94},"6efa9e4a":{"m":55,"g":94},"7791fd99":{"m":55,"g":94},"2ac36b9a":{"m":55,"g":94},"2d60a5ee":{"m":55,"g":94},"2e4a5907":{"m":55,"g":94},"c0ee46fe":{"m":55,"g":94},"9208618b":{"m":55,"g":94},"864bf2ba":{"m":55,"g":94},"a4cca7fc":{"m":55,"g":94},"993956c6":{"m":55,"g":94},"f8548295":{"m":55,"g":94},"959735fc":{"m":55,"g":94},"f6772394":{"m":55,"g":94},"626a99ac":{"m":55,"g":94},"ece72491":{"m":55,"g":94},"0fb88aaa":{"m":55,"g":94},"d4de9a62":{"m":55,"g":94},"7310aede":{"m":55,"g":94},"5de9a58e":{"m":55,"g":94},"56fcd8e8":{"m":55,"g":94},"2b340adf":{"m":55,"g":94},"8586b72d":{"m":55,"g":94},"641b7d0a":{"m":55,"g":94},"0ce091a8":{"m":55,"g":94},"835f8afc":{"m":55,"g":94},"3844feb9":{"m":55,"g":94},"27f7bed7":{"m":55,"g":94},"6387098f":{"m":55,"g":94},"2a717c50":{"m":55,"g":94},"a1e697b2":{"m":55,"g":94},"a6ca736c":{"m":55,"g":94},"f62055b5":{"m":55,"g":94},"74bc9184":{"m":55,"g":94},"0f8eb153":{"m":55,"g":94},"67470bbb":{"m":55,"g":94},"cc858953":{"m":55,"g":94},"6128f7cf":{"m":55,"g":94},"a2486eb5":{"m":55,"g":94},"61dec545":{"m":55,"g":94},"96db0f66":{"m":55,"g":94},"7dc66fcb":{"m":55,"g":94},"1f09e84b":{"m":55,"g":94},"63dfab1b":{"m":55,"g":94},"ef995dae":{"m":55,"g":94},"75ae9689":{"m":55,"g":94},"95f93f49":{"m":55,"g":94},"aaac33fd":{"m":55,"g":94},"d332aa3b":{"m":55,"g":94},"c36736c8":{"m":55,"g":94},"1bf9e347":{"m":55,"g":94},"499c85f1":{"m":55,"g":94},"efc52f85":{"m":56,"g":94},"60e2fdcf":{"m":56,"g":94},"d7c0e872":{"m":56,"g":94},"31548116":{"m":56,"g":94},"53aed988":{"m":56,"g":94},"8a56b431":{"m":56,"g":94},"e835a500":{"m":56,"g":94},"23e5e50f":{"m":56,"g":94},"25e5d589":{"m":56,"g":94},"41b1db69":{"m":56,"g":94},"84967019":{"m":56,"g":94},"7d672d27":{"m":56,"g":94},"d4b17481":{"m":56,"g":94},"19ba2b0e":{"m":56,"g":94},"4e1e3cff":{"m":56,"g":94},"ef5b0ff9":{"m":57,"g":94},"6e530515":{"m":57,"g":94},"77d1210b":{"m":57,"g":94},"70dc2fbe":{"m":57,"g":94},"b438a2e5":{"m":57,"g":94},"7ca751ff":{"m":57,"g":94},"c75adfec":{"m":57,"g":94},"7722c11c":{"m":57,"g":94},"b2ed5c8e":{"m":57,"g":94},"f46f394f":{"m":57,"g":94},"2125898a":{"m":57,"g":94},"44f011d2":{"m":57,"g":94},"ed91e003":{"m":57,"g":94},"531d6ea9":{"m":57,"g":94},"dc3bee48":{"m":57,"g":94},"a74d1941":{"m":57,"g":94},"3169e66c":{"m":57,"g":94},"77395154":{"m":57,"g":94},"637de9e8":{"m":57,"g":94},"acb34072":{"m":57,"g":94},"08effbff":{"m":57,"g":94},"60bd3272":{"m":57,"g":94},"e7ebecf8":{"m":57,"g":94},"9a23c484":{"m":57,"g":94},"635a0426":{"m":57,"g":94},"2dccecf4":{"m":57,"g":94},"75ad0a14":{"m":57,"g":94},"3ccf566b":{"m":58,"g":94},"afa0341e":{"m":58,"g":94},"30828e71":{"m":58,"g":94},"e0e09fce":{"m":58,"g":94},"9c05c689":{"m":58,"g":94},"3464e57b":{"m":58,"g":94},"3815b23c":{"m":58,"g":94},"fd34f2da":{"m":58,"g":94},"8ee9a850":{"m":58,"g":94},"fd28640d":{"m":58,"g":94},"7863e436":{"m":58,"g":94},"333e3bfd":{"m":58,"g":94},"239c9d4d":{"m":58,"g":94},"855d0ba3":{"m":58,"g":94},"9254a33a":{"m":58,"g":94},"8a2681e2":{"m":58,"g":94},"5276a675":{"m":58,"g":94},"751e5ca2":{"m":58,"g":94},"7a7ac6be":{"m":58,"g":94},"d9e6ee38":{"m":58,"g":94},"03d5fbfd":{"m":59,"g":94},"1703d766":{"m":59,"g":94},"09e6e2aa":{"m":59,"g":94},"fad29f7f":{"m":59,"g":94},"35bdb485":{"m":59,"g":94},"b085e06b":{"m":59,"g":94},"763dd55d":{"m":59,"g":94},"2f0d3864":{"m":60,"g":94},"3900a94a":{"m":60,"g":94},"ded9fcd0":{"m":60,"g":94},"bc6ad367":{"m":60,"g":94},"3a22a303":{"m":60,"g":94},"bdb3929d":{"m":60,"g":94},"f5d0865b":{"m":60,"g":94},"afdee7b1":{"m":60,"g":94},"cb34d848":{"m":60,"g":94},"0f9cc6d8":{"m":60,"g":94},"c7ae474a":{"m":60,"g":94},"bdf946bf":{"m":60,"g":94},"8c8779cd":{"m":60,"g":94},"1775b963":{"m":60,"g":94},"dd2e2d27":{"m":60,"g":94},"a990daff":{"m":60,"g":94},"ba5112ff":{"m":60,"g":94},"815dce05":{"m":60,"g":94},"ad20b795":{"m":60,"g":94},"9183c23e":{"m":60,"g":94},"148254d4":{"m":60,"g":94},"a4d6d6f1":{"m":60,"g":94},"062c48d2":{"m":60,"g":94},"b6e0cfb5":{"m":60,"g":94},"0d8d97b8":{"m":60,"g":94},"0a765bbc":{"m":60,"g":94},"286cad3e":{"m":60,"g":94},"dc7eb01f":{"m":60,"g":94},"b0524c37":{"m":60,"g":94},"6c42fa22":{"m":60,"g":94},"d49b13c6":{"m":60,"g":94},"bedc4c7a":{"m":60,"g":94},"f44d1439":{"m":60,"g":94},"b6b57fc2":{"m":60,"g":94},"b4403985":{"m":60,"g":94},"339c69a2":{"m":60,"g":94},"f7074700":{"m":60,"g":94},"21ec66e5":{"m":60,"g":94},"c5210dfa":{"m":60,"g":94},"a29dd950":{"m":60,"g":94},"9c6ba248":{"m":60,"g":94},"b02da24a":{"m":60,"g":94},"bdd2827a":{"m":60,"g":94},"8c3b420e":{"m":60,"g":94},"e6f523b5":{"m":60,"g":94},"32318178":{"m":60,"g":94},"a11f8d5f":{"m":60,"g":94},"098d659c":{"m":60,"g":94},"76d14f8c":{"m":60,"g":94},"b08c308e":{"m":60,"g":94},"f624901c":{"m":61,"g":94},"f0e15dc6":{"m":61,"g":94},"f1769586":{"m":61,"g":94},"5d6e9467":{"m":61,"g":94},"a47bf391":{"m":61,"g":94},"b1706469":{"m":61,"g":94},"5413ec2b":{"m":61,"g":94},"f290bd43":{"m":61,"g":94},"8f157893":{"m":61,"g":94},"2db03a04":{"m":61,"g":94},"5cc11705":{"m":61,"g":94},"11fffbc9":{"m":61,"g":94},"4f077c01":{"m":61,"g":94},"679c3bca":{"m":61,"g":94},"656aed58":{"m":61,"g":94},"b5fb4ef5":{"m":61,"g":94},"2e6346fc":{"m":61,"g":94},"977f785d":{"m":61,"g":94},"8a690612":{"m":61,"g":94},"694e4192":{"m":61,"g":94},"b22f3f64":{"m":61,"g":94},"6fb57683":{"m":61,"g":94},"51caee74":{"m":61,"g":94},"58f9060e":{"m":61,"g":94},"bdc1acf6":{"m":61,"g":94},"6d08ce2a":{"m":61,"g":94},"380930a9":{"m":61,"g":94},"9dec582d":{"m":61,"g":94},"b01febdc":{"m":61,"g":94},"1acbaf1b":{"m":61,"g":94},"287427e2":{"m":61,"g":94},"b8574f69":{"m":61,"g":94},"2855caa4":{"m":61,"g":94},"2329e1dd":{"m":61,"g":94},"0f3eb1d2":{"m":61,"g":94},"06dd2eab":{"m":61,"g":94},"439f6580":{"m":61,"g":94},"b3e99dfb":{"m":62,"g":94},"f005758f":{"m":62,"g":94},"f5c6c667":{"m":62,"g":94},"cc0485be":{"m":62,"g":94},"b8cd09f2":{"m":62,"g":94},"c19d8482":{"m":62,"g":94},"80002562":{"m":62,"g":94},"46d44318":{"m":62,"g":94},"923f5183":{"m":62,"g":94},"d08c77c4":{"m":62,"g":94},"c1e097ca":{"m":62,"g":94},"6ec75e62":{"m":62,"g":94},"d855653b":{"m":62,"g":94},"336ff5b9":{"m":62,"g":94},"3b141e15":{"m":62,"g":94},"6249e4a1":{"m":62,"g":94},"f3516c28":{"m":62,"g":94},"17de02f9":{"m":62,"g":94},"51ab3ccf":{"m":62,"g":94},"67008f4b":{"m":62,"g":94},"4536d724":{"m":62,"g":94},"41d7e5b7":{"m":62,"g":94},"20a9f5df":{"m":62,"g":94},"42f39099":{"m":62,"g":94},"72c77763":{"m":62,"g":94},"4093aa46":{"m":62,"g":94},"e808c1df":{"m":62,"g":94},"a18ab81d":{"m":62,"g":94},"0bb0f763":{"m":62,"g":94},"85b2e057":{"m":62,"g":94},"a879c2fb":{"m":62,"g":94},"e2b16c47":{"m":62,"g":94},"c4f9707e":{"m":62,"g":94},"197cbf9b":{"m":62,"g":94},"e94fb7cb":{"m":63,"g":94},"b5caa22d":{"m":63,"g":94},"73401fd0":{"m":63,"g":94},"89cd9235":{"m":63,"g":94},"dc188132":{"m":63,"g":94},"10bfce71":{"m":63,"g":94},"583697cd":{"m":63,"g":94},"2584f6d9":{"m":63,"g":94},"51e87f6f":{"m":63,"g":94},"09bcbe01":{"m":63,"g":94},"03464890":{"m":63,"g":94},"44a96697":{"m":63,"g":94},"1a820e38":{"m":63,"g":94},"0ffcfdf4":{"m":63,"g":94},"cd493b5a":{"m":63,"g":94},"61f42b57":{"m":63,"g":94},"e403d237":{"m":63,"g":94},"3bcf5ece":{"m":63,"g":94},"2c05f81f":{"m":63,"g":94},"d77caa2b":{"m":63,"g":94},"8b6a4486":{"m":63,"g":94},"a69cb5cf":{"m":63,"g":94},"def5c318":{"m":63,"g":94},"3fc2b625":{"m":63,"g":94},"6ada05d0":{"m":63,"g":94},"24cafe31":{"m":63,"g":94},"5a176c92":{"m":63,"g":94},"4719c1d0":{"m":63,"g":94},"ef18b0ed":{"m":63,"g":94},"53cc91e5":{"m":63,"g":94},"d33cbb7e":{"m":63,"g":94},"23196d52":{"m":63,"g":94},"93b77c8e":{"m":63,"g":94},"7906d1d2":{"m":63,"g":94},"81d27c8e":{"m":63,"g":94},"4d4cdb3f":{"m":63,"g":94},"2bd18e2d":{"m":63,"g":94},"83452dbb":{"m":63,"g":94},"3d93f84a":{"m":63,"g":94},"c2f212d6":{"m":63,"g":94},"e2cdc8a5":{"m":63,"g":94},"2add697d":{"m":63,"g":94},"6f98c586":{"m":63,"g":94},"656dcc1a":{"m":63,"g":94},"8af7048d":{"m":63,"g":94},"d3024f4f":{"m":63,"g":94},"13387e6b":{"m":63,"g":94},"120c3634":{"m":63,"g":94},"78e5b22f":{"m":63,"g":94},"7a15e9ad":{"m":63,"g":94},"dc2ac0cb":{"m":63,"g":94},"d47c5101":{"m":63,"g":94},"033c715b":{"m":63,"g":94},"d06c1ab5":{"m":63,"g":94},"c5644cac":{"m":63,"g":94},"53e6552f":{"m":63,"g":94},"5dc54f1a":{"m":63,"g":94},"f3e9b489":{"m":63,"g":94},"6a7973ad":{"m":63,"g":94},"63051738":{"m":63,"g":94},"a8ccacc8":{"m":63,"g":94},"0427416b":{"m":63,"g":94},"bf3edc2c":{"m":63,"g":94},"78e974b2":{"m":63,"g":94},"bc6915e3":{"m":63,"g":94},"a883f079":{"m":63,"g":94},"8b6ce52e":{"m":63,"g":94},"58f3f2b8":{"m":63,"g":94},"93d69061":{"m":63,"g":94},"e00e5385":{"m":63,"g":94},"a2f602b5":{"m":63,"g":94},"8f2c522a":{"m":63,"g":94},"75964177":{"m":63,"g":94},"2dc957d4":{"m":63,"g":94},"bf8d07a6":{"m":63,"g":94},"ab317936":{"m":63,"g":94},"b7f3fec1":{"m":63,"g":94},"58f42b1d":{"m":63,"g":94},"767c9dec":{"m":63,"g":94},"a53454c5":{"m":63,"g":94},"6cb3974e":{"m":63,"g":94},"f65c13b5":{"m":63,"g":94},"b803b395":{"m":63,"g":94},"bfbda62c":{"m":63,"g":94},"4ab43cfb":{"m":64,"g":94},"2f79f588":{"m":64,"g":94},"8a96f749":{"m":64,"g":94},"827aa873":{"m":64,"g":94},"f8ca66fb":{"m":64,"g":94},"53cef815":{"m":64,"g":94},"351a72d4":{"m":64,"g":94},"514f37c3":{"m":64,"g":94},"52c03f16":{"m":64,"g":94},"741fccd7":{"m":64,"g":94},"1e3e5215":{"m":64,"g":94},"fb11a439":{"m":64,"g":94},"af02f99b":{"m":64,"g":94},"9472e699":{"m":64,"g":94},"1acc1f56":{"m":64,"g":94},"b045841b":{"m":64,"g":94},"f265d15b":{"m":64,"g":94},"02431b9a":{"m":64,"g":94},"1dda8c5e":{"m":64,"g":94},"7e097613":{"m":64,"g":94},"f4a92f4b":{"m":64,"g":94},"318260c0":{"m":64,"g":94},"4a612531":{"m":64,"g":94},"d1a08632":{"m":64,"g":94},"f8b28e46":{"m":64,"g":94},"82392da8":{"m":64,"g":94},"95f789ad":{"m":64,"g":94},"4f118a39":{"m":64,"g":94},"66283dbc":{"m":64,"g":94},"822bae8c":{"m":64,"g":94},"8e48ca8c":{"m":64,"g":94},"27acf63b":{"m":64,"g":94},"da6f8081":{"m":64,"g":94},"9286740e":{"m":64,"g":94},"896c0744":{"m":64,"g":94},"c23d5706":{"m":64,"g":94},"67ad4338":{"m":64,"g":94},"3cab5f71":{"m":64,"g":94},"14e754a8":{"m":64,"g":94},"98522149":{"m":64,"g":94},"5d9d15e7":{"m":64,"g":94},"665e5e85":{"m":64,"g":94},"a22f60a3":{"m":64,"g":94},"04f0b4cb":{"m":64,"g":94},"4505a436":{"m":64,"g":94},"685a5738":{"m":64,"g":94},"153b414e":{"m":64,"g":94},"6619f48e":{"m":64,"g":94},"3ed0a547":{"m":64,"g":94},"8d8ef849":{"m":64,"g":94},"9a0cc2e9":{"m":64,"g":94},"7bad7e75":{"m":64,"g":94},"1c4e0d24":{"m":64,"g":94},"54bac8af":{"m":64,"g":94},"5de4051b":{"m":64,"g":94},"e0cd65c2":{"m":64,"g":94},"f1b68618":{"m":64,"g":94},"0da0989a":{"m":64,"g":94},"07a22cbb":{"m":64,"g":94},"3d0bfa3e":{"m":64,"g":94},"1f6cf0d4":{"m":64,"g":94},"553f5a3f":{"m":64,"g":94},"ac2dc35d":{"m":64,"g":94},"3e032c07":{"m":64,"g":94},"44e12ce4":{"m":64,"g":94},"a547aad6":{"m":64,"g":94},"ea535dc5":{"m":64,"g":94},"862bcff8":{"m":64,"g":94},"8b84e69f":{"m":64,"g":94},"5de50653":{"m":64,"g":94},"c0bf9bf1":{"m":64,"g":94},"022614d2":{"m":64,"g":94},"b8ab989f":{"m":64,"g":94},"b3393e94":{"m":64,"g":94},"ddc2001f":{"m":64,"g":94},"806a3002":{"m":64,"g":94},"0d2148ef":{"m":64,"g":94},"bf669606":{"m":64,"g":94},"b2bd8f44":{"m":64,"g":94},"9d9b482a":{"m":64,"g":94},"7353fb9b":{"m":64,"g":94},"bcda0c9e":{"m":64,"g":94},"9f8f2c7f":{"m":64,"g":94},"6fc37bd8":{"m":64,"g":94},"3d8f1c9b":{"m":64,"g":94},"a42213db":{"m":64,"g":94},"0ac019f1":{"m":64,"g":94},"5a0d680a":{"m":64,"g":94},"a4331cd2":{"m":64,"g":94},"ec1c21cd":{"m":64,"g":94},"6c856b4f":{"m":64,"g":94},"287d07a6":{"m":64,"g":94},"d2571dd5":{"m":64,"g":94},"b730aa6b":{"m":64,"g":94},"60b2a44a":{"m":64,"g":94},"949b3fbf":{"m":64,"g":94},"da4e8b38":{"m":64,"g":94},"af6c5357":{"m":64,"g":94},"3ad4cd49":{"m":64,"g":94},"3a8428ec":{"m":64,"g":94},"0311ce8e":{"m":64,"g":94},"5dfcacfc":{"m":64,"g":94},"41a0ccd4":{"m":64,"g":94},"cf0f7eaf":{"m":65,"g":94},"b49d6d0f":{"m":65,"g":94},"c02e3139":{"m":65,"g":94},"734daedd":{"m":65,"g":94},"3ee62235":{"m":65,"g":94},"9829e77e":{"m":65,"g":94},"cde4bbd5":{"m":65,"g":94},"9602c2aa":{"m":65,"g":94},"e81d7f11":{"m":65,"g":94},"222ce6f1":{"m":65,"g":94},"468d23cf":{"m":65,"g":94},"c38b5fb4":{"m":65,"g":94},"20453cef":{"m":65,"g":94},"9f635ea5":{"m":65,"g":94},"76285fde":{"m":65,"g":94},"988d0a4b":{"m":65,"g":94},"81262c7b":{"m":65,"g":94},"27aeb4b7":{"m":65,"g":94},"7b9b4f44":{"m":65,"g":94},"08104b56":{"m":65,"g":94},"cf142b6e":{"m":65,"g":94},"7aad8d18":{"m":66,"g":94},"76fa2d15":{"m":66,"g":94},"7ab84948":{"m":66,"g":94},"4885b908":{"m":66,"g":94},"c2723a42":{"m":66,"g":94},"c7256ca8":{"m":66,"g":94},"6186a8f8":{"m":66,"g":94},"a07364cc":{"m":66,"g":94},"2c1a695f":{"m":66,"g":94},"d39899e8":{"m":66,"g":94},"70817a7e":{"m":66,"g":94},"7b5a3741":{"m":66,"g":94},"4b6f62e2":{"m":66,"g":94},"897e2e25":{"m":66,"g":94},"d54cee14":{"m":66,"g":94},"00fa7d04":{"m":66,"g":94},"013021b6":{"m":66,"g":94},"3c8ac78d":{"m":66,"g":94},"455bfe8d":{"m":66,"g":94},"28b0a62b":{"m":66,"g":94},"566d61d9":{"m":66,"g":94},"55f5fc68":{"m":66,"g":94},"c27c378a":{"m":66,"g":94},"d9eb9358":{"m":66,"g":94},"959dca4f":{"m":66,"g":94},"f2b3a318":{"m":66,"g":94},"ad674097":{"m":66,"g":94},"8db776f0":{"m":66,"g":94},"4eb4b401":{"m":66,"g":94},"17dbf976":{"m":66,"g":94},"53179026":{"m":66,"g":94},"d7c0b32f":{"m":66,"g":94},"7b020cca":{"m":66,"g":94},"7876279e":{"m":66,"g":94},"34e405e0":{"m":66,"g":94},"1ebe1d6d":{"m":66,"g":94},"7811bfda":{"m":66,"g":94},"656f7fc1":{"m":66,"g":94},"c1f5f99f":{"m":67,"g":94},"fa82dfcc":{"m":67,"g":94},"5da3d21c":{"m":67,"g":94},"f2870376":{"m":67,"g":94},"f9905d59":{"m":67,"g":94},"45c87e08":{"m":67,"g":94},"2b1808ce":{"m":67,"g":94},"e868d0b6":{"m":67,"g":94},"591e751e":{"m":67,"g":94},"40022d07":{"m":67,"g":94},"823148e7":{"m":67,"g":94},"76ca91df":{"m":67,"g":94},"cdae77b0":{"m":67,"g":94},"adeee152":{"m":67,"g":94},"6792411e":{"m":67,"g":94},"7348d962":{"m":67,"g":94},"25ed22b6":{"m":67,"g":94},"200d3b16":{"m":67,"g":94},"ad349985":{"m":67,"g":94},"32de54ed":{"m":67,"g":94},"2d9c3195":{"m":67,"g":94},"07e58a2d":{"m":67,"g":94},"04d8cd20":{"m":67,"g":94},"a322051e":{"m":67,"g":94},"de553334":{"m":67,"g":94},"cddb1cdf":{"m":68,"g":94},"fa1b40e0":{"m":68,"g":94},"c45cab1c":{"m":68,"g":94},"27c4c9cf":{"m":68,"g":94},"52a492a1":{"m":68,"g":94},"36f6fc50":{"m":68,"g":94},"d8727275":{"m":68,"g":94},"6239d0b2":{"m":68,"g":94},"4cfd3add":{"m":68,"g":94},"20cf910d":{"m":68,"g":94},"0af1d239":{"m":68,"g":94},"85986bb9":{"m":68,"g":94},"64c87135":{"m":68,"g":94},"1646149a":{"m":68,"g":94},"bc72e5bd":{"m":68,"g":94},"014cab4d":{"m":68,"g":94},"4d2dbeac":{"m":68,"g":94},"29daf498":{"m":68,"g":94},"6702592d":{"m":68,"g":94},"60abdb3e":{"m":68,"g":94},"7b4e61ff":{"m":68,"g":94},"6222e1c2":{"m":68,"g":94},"fad315cb":{"m":68,"g":94},"f90db8bc":{"m":68,"g":94},"d8ad5970":{"m":68,"g":94},"849f58d6":{"m":68,"g":94},"64480df4":{"m":68,"g":94},"4530136e":{"m":68,"g":94},"0a6f18f0":{"m":68,"g":94},"e0b9a423":{"m":69,"g":94},"e0821425":{"m":69,"g":94},"70f894b8":{"m":69,"g":94},"368de366":{"m":69,"g":94},"20de05a7":{"m":69,"g":94},"f076328b":{"m":69,"g":94},"bf2a7087":{"m":69,"g":94},"871a4aa1":{"m":69,"g":94},"98eecbda":{"m":69,"g":94},"4430c0a5":{"m":69,"g":94},"640363ad":{"m":69,"g":94},"8616357a":{"m":69,"g":94},"8adbc78b":{"m":69,"g":94},"45e3a7bc":{"m":69,"g":94},"b96e92e6":{"m":69,"g":94},"693c2600":{"m":69,"g":94},"ced68066":{"m":69,"g":94},"b8318aec":{"m":69,"g":94},"2f482210":{"m":69,"g":94},"d81ac443":{"m":69,"g":94},"2491cc92":{"m":69,"g":94},"67c5de92":{"m":69,"g":94},"1e2cf2b5":{"m":69,"g":94},"9490d157":{"m":69,"g":94},"eefcbdd3":{"m":69,"g":94},"7e6d5fc6":{"m":69,"g":94},"cadd5dbe":{"m":69,"g":94},"bb418ced":{"m":69,"g":94},"fdf04a14":{"m":69,"g":94},"5f0e7de3":{"m":69,"g":94},"2f47d710":{"m":69,"g":94},"4fe92bfc":{"m":69,"g":94},"d23cb9a0":{"m":69,"g":94},"2d611323":{"m":69,"g":94},"e782eb7e":{"m":70,"g":94},"e319153b":{"m":70,"g":94},"32b44d2f":{"m":70,"g":94},"5f1a485d":{"m":70,"g":94},"c9565e49":{"m":70,"g":94},"d03c4c25":{"m":70,"g":94},"8f13377d":{"m":70,"g":94},"3d4a8f9b":{"m":70,"g":94},"7474bed8":{"m":70,"g":94},"03caefeb":{"m":70,"g":94},"bcc213df":{"m":70,"g":94},"39416e39":{"m":70,"g":94},"231c40d8":{"m":70,"g":94},"bbc47c34":{"m":70,"g":94},"dfce9269":{"m":70,"g":94},"6718b109":{"m":70,"g":94},"7711ac6e":{"m":70,"g":94},"7443197a":{"m":70,"g":94},"862dd76c":{"m":70,"g":94},"fb4c9c3a":{"m":70,"g":94},"d973c78e":{"m":70,"g":94},"6ce6eabb":{"m":70,"g":94},"4e23c961":{"m":70,"g":94},"3efbdf68":{"m":70,"g":94},"6cc30955":{"m":70,"g":94},"31eec35b":{"m":70,"g":94},"ac963be2":{"m":70,"g":94},"a5375adc":{"m":71,"g":94},"75d171a9":{"m":71,"g":94},"714f3e63":{"m":71,"g":94},"c38f3aed":{"m":71,"g":94},"2e6be53e":{"m":71,"g":94},"fc671f66":{"m":72,"g":94},"197751e9":{"m":72,"g":94},"d2d0d061":{"m":72,"g":94},"25482edb":{"m":72,"g":94},"62b362b1":{"m":72,"g":94},"44d76463":{"m":72,"g":94},"cd85b78f":{"m":72,"g":94},"0aaccbbf":{"m":72,"g":94},"357671e2":{"m":72,"g":94},"e70fa279":{"m":72,"g":94},"abe74b7b":{"m":72,"g":94},"70b3c6ee":{"m":72,"g":94},"ef9d3b3c":{"m":72,"g":94},"fc91d08a":{"m":72,"g":94},"71ab0dab":{"m":72,"g":94},"d3d4d767":{"m":72,"g":94},"5be8f1ed":{"m":72,"g":94},"e5760bc4":{"m":72,"g":94},"56a724eb":{"m":72,"g":94},"583d6af7":{"m":72,"g":94},"e074d84e":{"m":72,"g":94},"4725e3f6":{"m":72,"g":94},"77a3954b":{"m":72,"g":94},"03b0364f":{"m":72,"g":94},"2dd7d0c5":{"m":72,"g":94},"0d4e3228":{"m":72,"g":94},"926f8efc":{"m":72,"g":94},"9545bfb2":{"m":72,"g":94},"37373ef2":{"m":72,"g":94},"61261b39":{"m":72,"g":94},"19120f71":{"m":72,"g":94},"2415ec38":{"m":72,"g":94},"87f671ab":{"m":72,"g":94},"51d25405":{"m":72,"g":94},"e0a2c963":{"m":72,"g":94},"12f2e6c3":{"m":72,"g":94},"95575aa7":{"m":72,"g":94},"11eea69e":{"m":72,"g":94},"1baa9e6c":{"m":72,"g":94},"911fcd09":{"m":72,"g":94},"9fafa62d":{"m":72,"g":94},"146ac8df":{"m":72,"g":94},"57a404fd":{"m":72,"g":94},"2796fbb5":{"m":72,"g":94},"935cda94":{"m":72,"g":94},"110e0066":{"m":72,"g":94},"6b45a21d":{"m":72,"g":94},"a7000a76":{"m":72,"g":94},"1a8f995c":{"m":72,"g":94},"a3ab768a":{"m":72,"g":94},"66301e12":{"m":72,"g":94},"ac238727":{"m":72,"g":94},"0194948f":{"m":72,"g":94},"b4d34cd3":{"m":72,"g":94},"728e175f":{"m":72,"g":94},"9e1014cf":{"m":72,"g":94},"fa561067":{"m":72,"g":94},"7fbab730":{"m":72,"g":94},"b7e274f2":{"m":72,"g":94},"9cf40772":{"m":72,"g":94},"d3fe9bae":{"m":72,"g":94},"00ce7e31":{"m":72,"g":94},"50f28f65":{"m":72,"g":94},"90a55e25":{"m":72,"g":94},"407e2b92":{"m":72,"g":94},"40782f05":{"m":72,"g":94},"18bb216c":{"m":72,"g":94},"6b859e7d":{"m":72,"g":94},"930da877":{"m":72,"g":94},"3f8a4414":{"m":72,"g":94},"aceb4201":{"m":72,"g":94},"90a4b7d9":{"m":72,"g":94},"f3b99f73":{"m":72,"g":94},"9e74ee91":{"m":72,"g":94},"77a6c9d2":{"m":72,"g":94},"e3e0bc50":{"m":72,"g":94},"bac414ab":{"m":72,"g":94},"eec3f6d1":{"m":72,"g":94},"90bc26a8":{"m":72,"g":94},"ec0a72c2":{"m":72,"g":94},"1c96fa86":{"m":72,"g":94},"bc20e93f":{"m":72,"g":94},"d3887852":{"m":72,"g":94},"564bdf29":{"m":72,"g":94},"5d860168":{"m":72,"g":94},"d2815879":{"m":72,"g":94},"b0df5d24":{"m":72,"g":94},"3e02526b":{"m":72,"g":94},"d8a98a2c":{"m":72,"g":94},"0519269d":{"m":72,"g":94},"d6898dd2":{"m":72,"g":94},"71ed0183":{"m":72,"g":94},"8b681d77":{"m":72,"g":94},"194eea17":{"m":72,"g":94},"acd1a159":{"m":72,"g":94},"7c1692aa":{"m":72,"g":94},"8f019c7d":{"m":72,"g":94},"7551498a":{"m":72,"g":94},"44a2c4bd":{"m":72,"g":94},"c9fc4a9d":{"m":72,"g":94},"21463e32":{"m":72,"g":94},"3dc9ff3c":{"m":72,"g":94},"06427dfa":{"m":72,"g":94},"60524920":{"m":72,"g":94},"10771026":{"m":72,"g":94},"4606e2a3":{"m":72,"g":94},"127998cc":{"m":72,"g":94},"c0bb9eb3":{"m":72,"g":94},"7036d6fc":{"m":72,"g":94},"6ce9dbe8":{"m":72,"g":94},"3758d209":{"m":72,"g":94},"faf29e0b":{"m":72,"g":94},"b0743ea0":{"m":72,"g":94},"60b771c8":{"m":72,"g":94},"d7934cde":{"m":72,"g":94},"62bbd343":{"m":72,"g":94},"f2388f6b":{"m":72,"g":94},"c9745ee0":{"m":72,"g":94},"1a6e9757":{"m":72,"g":94},"b1100846":{"m":72,"g":94},"27a46317":{"m":72,"g":94},"c9795808":{"m":72,"g":94},"6c7a152c":{"m":72,"g":94},"4d2a88bd":{"m":72,"g":94},"45360b2f":{"m":72,"g":94},"3f41b184":{"m":72,"g":94},"45205d88":{"m":72,"g":94},"90876940":{"m":72,"g":94},"a3339d8c":{"m":72,"g":94},"14d90617":{"m":72,"g":94},"d37f9551":{"m":72,"g":94},"c66b2c9c":{"m":72,"g":94},"20b765a2":{"m":72,"g":94},"e3107222":{"m":72,"g":94},"e074e76b":{"m":72,"g":94},"4592afc2":{"m":72,"g":94},"9af0e21e":{"m":72,"g":94},"c7c79b16":{"m":72,"g":94},"d8d75d25":{"m":72,"g":94},"1df6eabd":{"m":72,"g":94},"0c227ee3":{"m":72,"g":94},"5c54ef03":{"m":72,"g":94},"c6a48521":{"m":72,"g":94},"4f678c87":{"m":72,"g":94},"e79f7420":{"m":72,"g":94},"ac053100":{"m":72,"g":94},"d5d80ab4":{"m":72,"g":94},"ddcf9fe3":{"m":72,"g":94},"6252ade9":{"m":72,"g":94},"1eb8eade":{"m":72,"g":94},"3c7bfd7e":{"m":72,"g":94},"bb121214":{"m":72,"g":94},"55de40f7":{"m":72,"g":94},"6b0aeb58":{"m":72,"g":94},"bb3e5268":{"m":72,"g":94},"f93e9158":{"m":72,"g":94},"55a7ec38":{"m":72,"g":94},"fe0673f1":{"m":72,"g":94},"99c1b9d2":{"m":72,"g":94},"634a3561":{"m":72,"g":94},"424848d2":{"m":72,"g":94},"e5ce395a":{"m":72,"g":94},"f983213a":{"m":72,"g":94},"67fc595b":{"m":72,"g":94},"07ab4d4a":{"m":72,"g":94},"522e18ea":{"m":72,"g":94},"c51dc2cc":{"m":72,"g":94},"ddf39d3f":{"m":72,"g":94},"2eab1132":{"m":72,"g":94},"058d199d":{"m":72,"g":94},"9c58e68b":{"m":73,"g":94},"d03b3467":{"m":73,"g":94},"ab7fba0e":{"m":73,"g":94},"bc1534ff":{"m":73,"g":94},"3a391812":{"m":73,"g":94},"800bf018":{"m":73,"g":94},"b16af90b":{"m":73,"g":94},"98c73d71":{"m":73,"g":94},"fcc2e37f":{"m":73,"g":94},"0804dd11":{"m":73,"g":94},"55dc8e4d":{"m":73,"g":94},"02e9e9f1":{"m":73,"g":94},"8f0b6313":{"m":73,"g":94},"b9b3b098":{"m":73,"g":94},"aee30630":{"m":73,"g":94},"286e6540":{"m":73,"g":94},"718c391f":{"m":73,"g":94},"6aaeb848":{"m":74,"g":94},"3623b6a7":{"m":74,"g":94},"4ff12642":{"m":74,"g":94},"2a4cbad8":{"m":74,"g":94},"2937387a":{"m":74,"g":94},"cf721fde":{"m":74,"g":94},"45de8971":{"m":74,"g":94},"71046fcd":{"m":74,"g":94},"c76040e3":{"m":74,"g":94},"2f6bacee":{"m":74,"g":94},"40148041":{"m":74,"g":94},"ad46550d":{"m":74,"g":94},"14344caa":{"m":74,"g":94},"f7f88b70":{"m":74,"g":94},"18c27131":{"m":74,"g":94},"ccdd10c8":{"m":74,"g":94},"76f6c0eb":{"m":74,"g":94},"959a3143":{"m":74,"g":94},"6412c5e4":{"m":74,"g":94},"0c020860":{"m":74,"g":94},"85ef7f64":{"m":74,"g":94},"f1cf6eef":{"m":74,"g":94},"0a59a465":{"m":74,"g":94},"aff79f10":{"m":74,"g":94},"01603318":{"m":74,"g":94},"56c39a05":{"m":74,"g":94},"4068e012":{"m":74,"g":94},"817d4370":{"m":74,"g":94},"c550e52f":{"m":74,"g":94},"e35a93fa":{"m":74,"g":94},"2c3656f2":{"m":74,"g":94},"d40ee62b":{"m":74,"g":94},"91b19949":{"m":74,"g":94},"7c866711":{"m":74,"g":94},"10b544ae":{"m":74,"g":94},"01090e8a":{"m":74,"g":94},"6f43a9b9":{"m":74,"g":94},"0540fef7":{"m":74,"g":94},"481f608b":{"m":74,"g":94},"ed91561f":{"m":74,"g":94},"6e7239f9":{"m":74,"g":94},"0a3960f2":{"m":74,"g":94},"07f94463":{"m":74,"g":94},"e0917e6b":{"m":74,"g":94},"7130a7ce":{"m":74,"g":94},"8f1f614e":{"m":74,"g":94},"7140ba35":{"m":74,"g":94},"d1da58e2":{"m":74,"g":94},"1cf63485":{"m":74,"g":94},"ff2ce0b8":{"m":74,"g":94},"0f2a2e3c":{"m":74,"g":94},"690e1f23":{"m":74,"g":94},"00f42707":{"m":74,"g":94},"6a02b32d":{"m":74,"g":94},"3a08f546":{"m":74,"g":94},"dce303e2":{"m":74,"g":94},"4d27eb9a":{"m":74,"g":94},"d3ecd632":{"m":74,"g":94},"cd909455":{"m":74,"g":94},"bde24ab3":{"m":74,"g":94},"bf2eefc0":{"m":74,"g":94},"5524e7d0":{"m":74,"g":94},"e187a3d5":{"m":74,"g":94},"3dd4feae":{"m":74,"g":94},"2ac189ed":{"m":74,"g":94},"5a6400ee":{"m":74,"g":94},"cf0ccd40":{"m":74,"g":94},"3d56585a":{"m":74,"g":94},"00d25a7f":{"m":74,"g":94},"1a5023e0":{"m":74,"g":94},"23308a90":{"m":74,"g":94},"ac698850":{"m":74,"g":94},"aa957102":{"m":74,"g":94},"007f8b3d":{"m":74,"g":94},"4455b26e":{"m":74,"g":94},"c553e160":{"m":74,"g":94},"7c0541b3":{"m":74,"g":94},"e8a69e4d":{"m":74,"g":94},"fbd56002":{"m":74,"g":94},"730d084f":{"m":74,"g":94},"4a05bdfa":{"m":74,"g":94},"eb06dbcb":{"m":74,"g":94},"9dfafa74":{"m":74,"g":94},"f1d09a65":{"m":74,"g":94},"df84ab2a":{"m":74,"g":94},"34c88987":{"m":74,"g":94},"0dd6cda2":{"m":74,"g":94},"9fb48f95":{"m":74,"g":94},"89ccb533":{"m":74,"g":94},"dceb256f":{"m":74,"g":94},"0e90ae62":{"m":74,"g":94},"1361ab9e":{"m":74,"g":94},"5c7dd14b":{"m":74,"g":94},"8abf74e3":{"m":74,"g":94},"ee132a45":{"m":74,"g":94},"79a321af":{"m":74,"g":94},"6eec3cdc":{"m":74,"g":94},"48473684":{"m":74,"g":94},"b3251e9f":{"m":74,"g":94},"2cadd51d":{"m":74,"g":94},"4a893d14":{"m":74,"g":94},"8d323e95":{"m":74,"g":94},"0fe7c13b":{"m":74,"g":94},"08c4d764":{"m":74,"g":94},"96d0e37f":{"m":74,"g":94},"90bb2be2":{"m":74,"g":94},"b93ef5e5":{"m":74,"g":94},"d4017a6b":{"m":74,"g":94},"d052f4c8":{"m":74,"g":94},"e1aaa79a":{"m":74,"g":94},"20c81199":{"m":74,"g":94},"70866b6f":{"m":74,"g":94},"eb61f5c9":{"m":74,"g":94},"0beea450":{"m":74,"g":94},"c827c671":{"m":74,"g":94},"b55a621f":{"m":74,"g":94},"ffa1b3e3":{"m":74,"g":94},"7e3bb527":{"m":74,"g":94},"96263f27":{"m":74,"g":94},"9376ac36":{"m":74,"g":94},"94a2b9d3":{"m":74,"g":94},"3c3eb374":{"m":74,"g":94},"d557319a":{"m":74,"g":94},"95085d65":{"m":74,"g":94},"c7f25446":{"m":74,"g":94},"63ee26d1":{"m":74,"g":94},"ad55f171":{"m":74,"g":94},"361971b8":{"m":74,"g":94},"13bc39c5":{"m":74,"g":94},"9854a18a":{"m":74,"g":94},"ebddb65a":{"m":74,"g":94},"19fd57bc":{"m":74,"g":94},"ba80c102":{"m":75,"g":94},"fbdb5050":{"m":75,"g":94},"f0afaf52":{"m":75,"g":94},"85d2365d":{"m":75,"g":94},"5fe79605":{"m":75,"g":94},"c6d7f8d3":{"m":75,"g":94},"a5a892ff":{"m":75,"g":94},"8e66fbec":{"m":75,"g":94},"f141298a":{"m":75,"g":94},"4fea040c":{"m":75,"g":94},"1099f6c9":{"m":76,"g":94},"04e3ff69":{"m":76,"g":94},"45fdf1f7":{"m":76,"g":94},"d89c0e4b":{"m":76,"g":94},"fa3c9e06":{"m":76,"g":94},"0d658ac3":{"m":76,"g":94},"ced35a06":{"m":76,"g":94},"26f07294":{"m":76,"g":94},"34e07a65":{"m":76,"g":94},"15ddd843":{"m":76,"g":94},"52029bd1":{"m":76,"g":94},"eb934bdf":{"m":76,"g":94},"e45ae444":{"m":76,"g":94},"ac3fae84":{"m":76,"g":94},"2d1b83e5":{"m":76,"g":94},"199bb01d":{"m":76,"g":94},"6b7038ba":{"m":76,"g":94},"57eec0bf":{"m":76,"g":94},"f01b0925":{"m":76,"g":94},"14269198":{"m":76,"g":94},"9b7cf9ee":{"m":76,"g":94},"1e86457c":{"m":76,"g":94},"64129fa6":{"m":76,"g":94},"e9f8e423":{"m":76,"g":94},"22c3702e":{"m":76,"g":94},"4c584fc6":{"m":76,"g":94},"77cf771e":{"m":76,"g":94},"8154de5a":{"m":76,"g":94},"c11cfda0":{"m":76,"g":94},"64edeb79":{"m":76,"g":94},"65c24c28":{"m":76,"g":94},"3980ff1b":{"m":76,"g":94},"5d7edc8e":{"m":76,"g":94},"af6535e7":{"m":76,"g":94},"93cf7fc5":{"m":76,"g":94},"2a206b22":{"m":76,"g":94},"4d253057":{"m":76,"g":94},"11577ced":{"m":76,"g":94},"ca75741e":{"m":76,"g":94},"c6d549e7":{"m":76,"g":94},"3c09548d":{"m":76,"g":94},"8796cebb":{"m":76,"g":94},"c2bd094d":{"m":76,"g":94},"f8f9244a":{"m":76,"g":94},"ecbfe58b":{"m":76,"g":94},"8f163b16":{"m":76,"g":94},"e7a8610d":{"m":76,"g":94},"a2cc62a6":{"m":76,"g":94},"fb888603":{"m":76,"g":94},"321ab756":{"m":76,"g":94},"38f25e87":{"m":76,"g":94},"8cd42504":{"m":76,"g":94},"6a384d5c":{"m":76,"g":94},"f69e0696":{"m":76,"g":94},"f6ab4ca6":{"m":76,"g":94},"c7c7dbeb":{"m":76,"g":94},"417fc72f":{"m":76,"g":94},"c6ec7029":{"m":76,"g":94},"4c56e5db":{"m":76,"g":94},"7b5fc719":{"m":76,"g":94},"ad4e58bf":{"m":76,"g":94},"bfb03c61":{"m":76,"g":94},"b36ab493":{"m":76,"g":94},"9e93ef3f":{"m":76,"g":94},"fad86a68":{"m":76,"g":94},"df7014a8":{"m":76,"g":94},"49420741":{"m":76,"g":94},"ba52fd18":{"m":76,"g":94},"b6944f97":{"m":76,"g":94},"f44db16c":{"m":76,"g":94},"f9c53cbb":{"m":76,"g":94},"90532b76":{"m":76,"g":94},"c0e9a36c":{"m":76,"g":94},"588865f0":{"m":76,"g":94},"3196999f":{"m":76,"g":94},"9e0186f3":{"m":76,"g":94},"8baf9a0c":{"m":76,"g":94},"c7872985":{"m":76,"g":94},"45212ce1":{"m":76,"g":94},"c16b33cc":{"m":76,"g":94},"2d004512":{"m":76,"g":94},"804d250a":{"m":76,"g":94},"dd865bef":{"m":76,"g":94},"d373a48c":{"m":76,"g":94},"98be3bd3":{"m":76,"g":94},"a98290ae":{"m":76,"g":94},"9b81f9bd":{"m":76,"g":94},"f81a27f6":{"m":76,"g":94},"988ab646":{"m":76,"g":94},"3ded4b21":{"m":76,"g":94},"f4d7ab7a":{"m":76,"g":94},"c38ca4fc":{"m":76,"g":94},"82dec1f7":{"m":76,"g":94},"5f9b2c62":{"m":76,"g":94},"5493c334":{"m":76,"g":94},"f2ab37e5":{"m":76,"g":94},"91ba98fe":{"m":76,"g":94},"c614dbdf":{"m":76,"g":94},"927ca935":{"m":76,"g":94},"ef3c2dd0":{"m":76,"g":94},"75b65648":{"m":76,"g":94},"0f52fb55":{"m":76,"g":94},"d6d21640":{"m":76,"g":94},"0212d2e2":{"m":76,"g":94},"8cc300f5":{"m":76,"g":94},"452db508":{"m":76,"g":94},"d1112d85":{"m":76,"g":94},"48efec7b":{"m":76,"g":94},"9b8333d9":{"m":76,"g":94},"f5bbf603":{"m":76,"g":94},"5cbd709e":{"m":76,"g":94},"2e4a1e2d":{"m":76,"g":94},"9d02bb3e":{"m":76,"g":94},"402db5c5":{"m":76,"g":94},"754a0e82":{"m":76,"g":94},"799fb5f4":{"m":76,"g":94},"25e1816e":{"m":76,"g":94},"a53fe428":{"m":76,"g":94},"1b859295":{"m":76,"g":94},"9971dc22":{"m":76,"g":94},"3db35c1a":{"m":76,"g":94},"52a34d74":{"m":76,"g":94},"06d12b39":{"m":76,"g":94},"c30976fb":{"m":76,"g":94},"1a3fa75f":{"m":76,"g":94},"81f431ed":{"m":76,"g":94},"65b7c9b7":{"m":76,"g":94},"2c4f5cca":{"m":76,"g":94},"15843047":{"m":76,"g":94},"8ec2ce07":{"m":76,"g":94},"1fd0cf8a":{"m":76,"g":94},"bf63ee54":{"m":76,"g":94},"22c96f78":{"m":76,"g":94},"2892b9bb":{"m":76,"g":94},"470b4740":{"m":76,"g":94},"26c372c1":{"m":76,"g":94},"86d9baed":{"m":76,"g":94},"21d485f8":{"m":76,"g":94},"035ac2ab":{"m":76,"g":94},"e1a5e7e4":{"m":76,"g":94},"ad1ae7f7":{"m":76,"g":94},"e73167ad":{"m":76,"g":94},"862fe522":{"m":76,"g":94},"61e4433c":{"m":76,"g":94},"660305c3":{"m":76,"g":94},"642ab418":{"m":76,"g":94},"1ce4878d":{"m":76,"g":94},"977d7cd2":{"m":76,"g":94},"0e0ec702":{"m":76,"g":94},"bb378556":{"m":76,"g":94},"19e96e59":{"m":77,"g":94},"aa08aeac":{"m":77,"g":94},"d8a136a1":{"m":77,"g":94},"20c90be2":{"m":77,"g":94},"ec3ee028":{"m":77,"g":94},"92941ce7":{"m":77,"g":94},"2bb0e7cf":{"m":77,"g":94},"72549263":{"m":77,"g":94},"044c3159":{"m":77,"g":94},"4db29e82":{"m":77,"g":94},"c483377e":{"m":77,"g":94},"74e0ac1d":{"m":77,"g":94},"ef9a378a":{"m":77,"g":94},"6dea5c96":{"m":77,"g":94},"6ffb6bd4":{"m":77,"g":94},"47e6628a":{"m":77,"g":94},"7907f9eb":{"m":77,"g":94},"8c04f0f2":{"m":77,"g":94},"265e7564":{"m":77,"g":94},"d3f71f5e":{"m":77,"g":94},"5eae67cb":{"m":77,"g":94},"6dbf9998":{"m":77,"g":94},"e0166f8a":{"m":77,"g":94},"53a2c3b4":{"m":77,"g":94},"550586ef":{"m":77,"g":94},"cf29fe9e":{"m":77,"g":94},"26c0f131":{"m":77,"g":94},"f9970bd1":{"m":77,"g":94},"2e0f94ab":{"m":77,"g":94},"18317ddc":{"m":77,"g":94},"e2e2ab70":{"m":77,"g":94},"0d3e3072":{"m":77,"g":94},"62dd9587":{"m":77,"g":94},"72031173":{"m":77,"g":94},"9fdc6d6a":{"m":77,"g":94},"42a45df0":{"m":77,"g":94},"04eb6062":{"m":77,"g":94},"e84f4ba0":{"m":77,"g":94},"b149b393":{"m":77,"g":94},"31dfff7d":{"m":77,"g":94},"10a9ab7b":{"m":77,"g":94},"bb0fd749":{"m":77,"g":94},"7f19e083":{"m":77,"g":94},"98a2cfa9":{"m":77,"g":94},"2a882e8f":{"m":77,"g":94},"e6e4d022":{"m":77,"g":94},"188105a2":{"m":77,"g":94},"b3953258":{"m":77,"g":94},"5fa3058f":{"m":77,"g":94},"bbab97a6":{"m":77,"g":94},"0bc0bf57":{"m":77,"g":94},"f60f2931":{"m":77,"g":94},"17000d2b":{"m":77,"g":94},"668ecc6c":{"m":77,"g":94},"886fcbdd":{"m":77,"g":94},"8bf6d7f4":{"m":77,"g":94},"1b9175cb":{"m":77,"g":94},"92bb49a7":{"m":77,"g":94},"6f5cc5eb":{"m":77,"g":94},"c913ed40":{"m":77,"g":94},"1afe3d07":{"m":77,"g":94},"44f47d3e":{"m":77,"g":94},"ae25d36d":{"m":77,"g":94},"35e0856b":{"m":78,"g":94},"aba5ca15":{"m":78,"g":94},"496dde84":{"m":78,"g":94},"bcbbf519":{"m":78,"g":94},"0d99adb7":{"m":78,"g":94},"efbae697":{"m":78,"g":94},"ca8d02ab":{"m":78,"g":94},"3f287b85":{"m":78,"g":94},"7ed77d6b":{"m":78,"g":94},"4c54f442":{"m":78,"g":94},"924ca7c9":{"m":78,"g":94},"6ff9c6a5":{"m":78,"g":94},"77e929a1":{"m":78,"g":94},"febe21ce":{"m":78,"g":94},"a995a773":{"m":78,"g":94},"31035dda":{"m":78,"g":94},"913e38df":{"m":78,"g":94},"d95269f9":{"m":78,"g":94},"e53bf190":{"m":78,"g":94},"3289c120":{"m":78,"g":94},"69df9761":{"m":78,"g":94},"98f768d1":{"m":78,"g":94},"d7954b76":{"m":78,"g":94},"74885a84":{"m":78,"g":94},"b8b6008f":{"m":78,"g":94},"8e10fec9":{"m":78,"g":94},"e8999b13":{"m":78,"g":94},"772d2a19":{"m":78,"g":94},"9d0b36c4":{"m":78,"g":94},"7d8c0ce7":{"m":78,"g":94},"e41549c3":{"m":78,"g":94},"cccfc10e":{"m":78,"g":94},"a2aea59b":{"m":78,"g":94},"2c8fd993":{"m":78,"g":94},"31da75ab":{"m":78,"g":94},"e983e432":{"m":78,"g":94},"e9c6ce46":{"m":78,"g":94},"3fadc647":{"m":78,"g":94},"e119f042":{"m":78,"g":94},"9eb49e87":{"m":78,"g":94},"12047f5e":{"m":78,"g":94},"fda6bb78":{"m":78,"g":94},"23c764b1":{"m":78,"g":94},"87fafa01":{"m":78,"g":94},"1c63e797":{"m":78,"g":94},"ee47a6c1":{"m":78,"g":94},"6384d317":{"m":78,"g":94},"5cb552b1":{"m":78,"g":94},"c7457191":{"m":78,"g":94},"51ac297a":{"m":78,"g":94},"a169b9f8":{"m":78,"g":94},"4a63bc32":{"m":78,"g":94},"a303325f":{"m":78,"g":94},"42873eac":{"m":78,"g":94},"4814ecaf":{"m":78,"g":94},"e62d60fe":{"m":78,"g":94},"032f8faa":{"m":78,"g":94},"37c66ec8":{"m":78,"g":94},"9adf178c":{"m":78,"g":94},"f842853a":{"m":78,"g":94},"195a09f5":{"m":78,"g":94},"9fccda31":{"m":78,"g":94},"4ede6770":{"m":78,"g":94},"b26bc86b":{"m":78,"g":94},"5ec5eaf7":{"m":78,"g":94},"0d7fe866":{"m":78,"g":94},"54b9a2de":{"m":78,"g":94},"8e7b3154":{"m":78,"g":94},"45dcfc2e":{"m":78,"g":94},"ddf8981d":{"m":78,"g":94},"400ad660":{"m":78,"g":94},"05625b97":{"m":78,"g":94},"736502d4":{"m":78,"g":94},"8690c40b":{"m":78,"g":94},"b1cfb4e9":{"m":78,"g":94},"57f99608":{"m":79,"g":94},"81992474":{"m":79,"g":94},"f04c80dc":{"m":79,"g":94},"d1bb1711":{"m":79,"g":94},"5b5c7237":{"m":80,"g":94},"a42736bb":{"m":80,"g":94},"dd83e7e9":{"m":80,"g":94},"0769b14b":{"m":80,"g":94},"b64b88e7":{"m":80,"g":94},"bc24205b":{"m":80,"g":94},"3efc8e2d":{"m":80,"g":94},"27a009bb":{"m":80,"g":94},"8ec0bb7d":{"m":80,"g":94},"fa909dc3":{"m":80,"g":94},"e8f62b20":{"m":80,"g":94},"88defc4d":{"m":80,"g":94},"6f509d55":{"m":80,"g":94},"12ef7e3b":{"m":80,"g":94},"838fa0f2":{"m":80,"g":94},"f1b3b75f":{"m":80,"g":94},"33b16ad1":{"m":80,"g":94},"ffde65a0":{"m":80,"g":94},"471650de":{"m":80,"g":94},"d06a83fb":{"m":80,"g":94},"5d134401":{"m":80,"g":94},"f88f7e19":{"m":80,"g":94},"3dfc6023":{"m":80,"g":94},"15e91d72":{"m":80,"g":94},"8aab7fdb":{"m":80,"g":94},"e940dc4f":{"m":80,"g":94},"388e15c0":{"m":80,"g":94},"11421a3f":{"m":80,"g":94},"6c41fcf0":{"m":80,"g":94},"ee9d6ca6":{"m":80,"g":94},"2dd64894":{"m":80,"g":94},"61e7c4dd":{"m":80,"g":94},"dae79444":{"m":80,"g":94},"f6772f14":{"m":80,"g":94},"ac5b78ba":{"m":80,"g":94},"38076dea":{"m":80,"g":94},"5e0a9b09":{"m":80,"g":94},"bdde2375":{"m":80,"g":94},"e9fc2ac7":{"m":80,"g":94},"44afde82":{"m":80,"g":94},"072df753":{"m":80,"g":94},"defede50":{"m":80,"g":94},"fc728719":{"m":80,"g":94},"14e8bd88":{"m":80,"g":94},"adca585b":{"m":80,"g":94},"39d90449":{"m":80,"g":94},"39e41138":{"m":80,"g":94},"5fbafbb8":{"m":80,"g":94},"a9499885":{"m":80,"g":94},"f7655790":{"m":80,"g":94},"f58b929a":{"m":80,"g":94},"c1270aab":{"m":80,"g":94},"8311b07f":{"m":80,"g":94},"c1380257":{"m":80,"g":94},"b62e7e99":{"m":80,"g":94},"7d3b7c87":{"m":80,"g":94},"75015bb6":{"m":80,"g":94},"b371f7cd":{"m":80,"g":94},"812e82f3":{"m":80,"g":94},"4879e50c":{"m":80,"g":94},"bc92107b":{"m":80,"g":94},"3e4794aa":{"m":80,"g":94},"690ec205":{"m":80,"g":94},"2074a2e6":{"m":80,"g":94},"57de7c6b":{"m":80,"g":94},"115ae2e7":{"m":80,"g":94},"aea98512":{"m":80,"g":94},"e4155e96":{"m":80,"g":94},"1b1b47a9":{"m":80,"g":94},"3c9740d2":{"m":80,"g":94},"2eb55770":{"m":80,"g":94},"f65b8d5c":{"m":80,"g":94},"5ad05719":{"m":80,"g":94},"34ef6c81":{"m":80,"g":94},"61172091":{"m":80,"g":94},"4f288113":{"m":80,"g":94},"136b8e6a":{"m":80,"g":94},"034c5256":{"m":80,"g":94},"c1dd773c":{"m":80,"g":94},"6f859379":{"m":80,"g":94},"f774a0d2":{"m":80,"g":94},"60bcbf2a":{"m":80,"g":94},"a0a9f6d6":{"m":80,"g":94},"80aa8ca8":{"m":80,"g":94},"4aa6bab0":{"m":80,"g":94},"c35dcfdb":{"m":80,"g":94},"c163bf4f":{"m":80,"g":94},"55986343":{"m":80,"g":94},"b75275b6":{"m":80,"g":94},"7074e9ca":{"m":80,"g":94},"fc14cca0":{"m":80,"g":94},"e7beff8a":{"m":80,"g":94},"4d2e3051":{"m":80,"g":94},"e53a0b3d":{"m":80,"g":94},"038bc5d5":{"m":80,"g":94},"aee62d74":{"m":80,"g":94},"cd7e32e2":{"m":80,"g":94},"88799448":{"m":80,"g":94},"a879811c":{"m":80,"g":94},"a222945d":{"m":80,"g":94},"ed01b451":{"m":80,"g":94},"d050df36":{"m":80,"g":94},"76f44c2a":{"m":80,"g":94},"1078396f":{"m":80,"g":94},"7e4f72dd":{"m":80,"g":94},"4c31ae9f":{"m":80,"g":94},"f730362e":{"m":80,"g":94},"e3c4bd31":{"m":80,"g":94},"5db37c86":{"m":80,"g":94},"4cb53ecd":{"m":80,"g":94},"456b008b":{"m":80,"g":94},"ebf495f0":{"m":80,"g":94},"7f875f12":{"m":80,"g":94},"fbebcb7a":{"m":80,"g":94},"87eddedf":{"m":80,"g":94},"40652482":{"m":80,"g":94},"86a876d8":{"m":80,"g":94},"92823069":{"m":80,"g":94},"d2e507df":{"m":80,"g":94},"61970b08":{"m":80,"g":94},"76c48a09":{"m":80,"g":94},"90caf06c":{"m":80,"g":94},"6669d127":{"m":80,"g":94},"f2b70afd":{"m":80,"g":94},"bc3f6db2":{"m":80,"g":94},"aac531c5":{"m":80,"g":94},"39efad4f":{"m":80,"g":94},"466899e6":{"m":80,"g":94},"11d760d5":{"m":80,"g":94},"5039d547":{"m":80,"g":94},"d09a51f1":{"m":80,"g":94},"f8194b26":{"m":80,"g":94},"6d3b35fa":{"m":80,"g":94},"a73c4df4":{"m":80,"g":94},"89a55418":{"m":80,"g":94},"2695ab05":{"m":80,"g":94},"88d6fd9a":{"m":80,"g":94},"cc88d98a":{"m":80,"g":94},"3033c11a":{"m":80,"g":94},"fd5a55cf":{"m":80,"g":94},"804d9f2e":{"m":80,"g":94},"a7c3f74b":{"m":80,"g":94},"5a144a8a":{"m":80,"g":94},"27f8e6b9":{"m":80,"g":94},"afb752bc":{"m":80,"g":94},"9731eca7":{"m":80,"g":94},"7c5658c1":{"m":80,"g":94},"9798e72b":{"m":80,"g":94},"ade714a6":{"m":80,"g":94},"93470a14":{"m":80,"g":94},"db452760":{"m":80,"g":94},"fbdc94ba":{"m":81,"g":94},"b54b5a96":{"m":81,"g":94},"bca832c7":{"m":81,"g":94},"d9dd5298":{"m":81,"g":94},"0a0dd34e":{"m":81,"g":94},"80ac527d":{"m":81,"g":94},"99456bca":{"m":81,"g":94},"d07e797a":{"m":81,"g":94},"c555d794":{"m":81,"g":94},"e2574ee9":{"m":81,"g":94},"ab4b5606":{"m":81,"g":94},"20f1c8e3":{"m":81,"g":94},"613b197e":{"m":81,"g":94},"d58e3544":{"m":81,"g":94},"bf86c5e9":{"m":81,"g":94},"dca90f1d":{"m":81,"g":94},"0961feef":{"m":81,"g":94},"59dd090f":{"m":81,"g":94},"569b032c":{"m":81,"g":94},"f6a71139":{"m":81,"g":94},"1e0806f3":{"m":81,"g":94},"2c11f9c2":{"m":81,"g":94},"a6f892e5":{"m":81,"g":94},"08b518d5":{"m":81,"g":94},"4db463b1":{"m":81,"g":94},"bfa39224":{"m":81,"g":94},"e465b08d":{"m":81,"g":94},"bed05878":{"m":81,"g":94},"b2a189dd":{"m":81,"g":94},"f28d8299":{"m":81,"g":94},"8e09b370":{"m":81,"g":94},"53dcf388":{"m":81,"g":94},"1effba4c":{"m":81,"g":94},"a0fc5bc1":{"m":81,"g":94},"27e9538a":{"m":81,"g":94},"211c7b31":{"m":81,"g":94},"c08a717c":{"m":81,"g":94},"f13d65a7":{"m":81,"g":94},"06d0a3d9":{"m":81,"g":94},"22c2a79d":{"m":81,"g":94},"8beb356f":{"m":81,"g":94},"c776234b":{"m":81,"g":94},"3bface15":{"m":81,"g":94},"6fb29ffd":{"m":81,"g":94},"4fb05583":{"m":81,"g":94},"81c89111":{"m":81,"g":94},"92d1561b":{"m":81,"g":94},"8f783c19":{"m":81,"g":94},"90faf901":{"m":81,"g":94},"177320a5":{"m":81,"g":94},"d7bc19a4":{"m":81,"g":94},"85ec0440":{"m":81,"g":94},"06a1656e":{"m":81,"g":94},"6aca5834":{"m":81,"g":94},"b9c87e78":{"m":82,"g":94},"968ef515":{"m":82,"g":94},"13432002":{"m":82,"g":94},"c2942907":{"m":82,"g":94},"e69a2190":{"m":82,"g":94},"bf98d2e3":{"m":82,"g":94},"e65b9f21":{"m":82,"g":94},"4dce1cc6":{"m":82,"g":94},"deded17f":{"m":82,"g":94},"f29a718f":{"m":82,"g":94},"3f57b00a":{"m":82,"g":94},"453d412c":{"m":82,"g":94},"dc86f25a":{"m":82,"g":94},"08289eaa":{"m":82,"g":94},"3b6d539f":{"m":82,"g":94},"57131dd9":{"m":82,"g":94},"a7591ecf":{"m":82,"g":94},"c44f2869":{"m":82,"g":94},"685d8980":{"m":82,"g":94},"70645f4d":{"m":82,"g":94},"188f0955":{"m":82,"g":94},"eef9433b":{"m":82,"g":94},"97cb762b":{"m":82,"g":94},"11951820":{"m":82,"g":94},"5239d795":{"m":82,"g":94},"f0815419":{"m":82,"g":94},"2b3bdc93":{"m":82,"g":94},"5fc4b600":{"m":82,"g":94},"b868526d":{"m":82,"g":94},"502524e2":{"m":82,"g":94},"4c764007":{"m":82,"g":94},"9f3bd2ad":{"m":82,"g":94},"8de53da9":{"m":82,"g":94},"fac17acf":{"m":82,"g":94},"8b39274e":{"m":82,"g":94},"5156d5a4":{"m":82,"g":94},"c951d312":{"m":82,"g":94},"dcb82325":{"m":82,"g":94},"66c0ff9e":{"m":82,"g":94},"9a7e83e8":{"m":82,"g":94},"417b44eb":{"m":82,"g":94},"475e2e37":{"m":82,"g":94},"fba86b6b":{"m":82,"g":94},"072b4d03":{"m":82,"g":94},"9c434777":{"m":82,"g":94},"fa2f677e":{"m":82,"g":94},"463d4b74":{"m":82,"g":94},"9924bbe1":{"m":82,"g":94},"84022c0e":{"m":83,"g":94},"f9fb33ef":{"m":83,"g":94},"a38f6932":{"m":83,"g":94},"beb65c74":{"m":83,"g":94},"621e96bf":{"m":83,"g":94},"35ca04d2":{"m":83,"g":94},"3c4e0ee6":{"m":83,"g":94},"9c088829":{"m":83,"g":94},"005aad32":{"m":83,"g":94},"4d23ba08":{"m":83,"g":94},"6e313c1b":{"m":83,"g":94},"a45a4b23":{"m":83,"g":94},"981a2619":{"m":83,"g":94},"8ba31330":{"m":83,"g":94},"02102063":{"m":83,"g":94},"7e944246":{"m":83,"g":94},"a086a113":{"m":83,"g":94},"bdbe5f81":{"m":83,"g":94},"9ad28f63":{"m":83,"g":94},"d7b1ce65":{"m":83,"g":94},"f55933e1":{"m":83,"g":94},"408ba022":{"m":83,"g":94},"094891c0":{"m":83,"g":94},"a21ef363":{"m":83,"g":94},"3c4dc38a":{"m":83,"g":94},"d8fbc7c0":{"m":83,"g":94},"c5e1026f":{"m":83,"g":94},"799c4bb5":{"m":83,"g":94},"02723e1b":{"m":83,"g":94},"df2cf583":{"m":83,"g":94},"133ded03":{"m":83,"g":94},"f87a6ab3":{"m":83,"g":94},"eebfdb94":{"m":83,"g":94},"dfb32264":{"m":83,"g":94},"63c13a2c":{"m":83,"g":94},"4d1e52ab":{"m":83,"g":94},"155890e4":{"m":83,"g":94},"1f963d7f":{"m":83,"g":94},"04d0123f":{"m":83,"g":94},"feda9b11":{"m":83,"g":94},"c3948ba6":{"m":83,"g":94},"269c457e":{"m":83,"g":94},"18ce468d":{"m":83,"g":94},"21514ff5":{"m":83,"g":94},"5641a094":{"m":83,"g":94},"3dd3538c":{"m":83,"g":94},"93c6fb12":{"m":83,"g":94},"11e27d09":{"m":83,"g":94},"50eda839":{"m":83,"g":94},"c55550cb":{"m":83,"g":94},"43fb95c2":{"m":83,"g":94},"7d9679b7":{"m":83,"g":94},"b5be5694":{"m":83,"g":94},"d2b8d0b8":{"m":83,"g":94},"a14654dd":{"m":83,"g":94},"5d93a950":{"m":83,"g":94},"c998d04b":{"m":83,"g":94},"7d0edf3c":{"m":83,"g":94},"ce4ecba4":{"m":83,"g":94},"b1f6d89b":{"m":83,"g":94},"7c99103f":{"m":83,"g":94},"de071366":{"m":83,"g":94},"e0673969":{"m":83,"g":94},"127ff898":{"m":83,"g":94},"8777a1d2":{"m":83,"g":94},"711efe78":{"m":83,"g":94},"fbb5f229":{"m":83,"g":94},"15fabcc0":{"m":83,"g":94},"e62c4955":{"m":83,"g":94},"71d1785f":{"m":83,"g":94},"3f87f831":{"m":83,"g":94},"ce5412b6":{"m":83,"g":94},"7282ab74":{"m":83,"g":94},"b0feda09":{"m":83,"g":94},"6b6e7487":{"m":83,"g":94},"91732486":{"m":83,"g":94},"2ed96c7a":{"m":83,"g":94},"2aa3f5e2":{"m":83,"g":94},"76d17c7e":{"m":83,"g":94},"70d040f9":{"m":83,"g":94},"4418f599":{"m":83,"g":94},"04f2abcb":{"m":83,"g":94},"506be6b8":{"m":83,"g":94},"2343d8df":{"m":83,"g":94},"92bb64bc":{"m":83,"g":94},"11b23ae9":{"m":83,"g":94},"dcae1fb2":{"m":84,"g":94},"a0251a3f":{"m":84,"g":94},"663037a7":{"m":84,"g":94},"f4a9f60c":{"m":84,"g":94},"ee71ed8a":{"m":84,"g":94},"d364b9b0":{"m":84,"g":94},"849c83a0":{"m":84,"g":94},"d73ddeb1":{"m":84,"g":94},"f48b007c":{"m":84,"g":94},"74cb12a8":{"m":84,"g":94},"c6c62640":{"m":84,"g":94},"92ab0a20":{"m":84,"g":94},"e132cba2":{"m":84,"g":94},"0045f4b2":{"m":84,"g":94},"8601300b":{"m":84,"g":94},"6fa6f38e":{"m":84,"g":94},"693723d1":{"m":84,"g":94},"966eb908":{"m":84,"g":94},"644ed409":{"m":84,"g":94},"3029889c":{"m":84,"g":94},"ef15dcda":{"m":84,"g":94},"ad4df307":{"m":84,"g":94},"41ac0c6d":{"m":84,"g":94},"84810da4":{"m":84,"g":94},"40d9b8ac":{"m":84,"g":94},"f0365820":{"m":84,"g":94},"86317c09":{"m":84,"g":94},"daed453e":{"m":84,"g":94},"ded04b2e":{"m":84,"g":94},"9858113c":{"m":85,"g":94},"8441baad":{"m":85,"g":94},"256c4c25":{"m":85,"g":94},"9f21e754":{"m":85,"g":94},"7bcd8b1c":{"m":85,"g":94},"11383cec":{"m":85,"g":94},"e97e57e6":{"m":85,"g":94},"9a6ad891":{"m":85,"g":94},"d353d08b":{"m":85,"g":94},"08acdb5c":{"m":85,"g":94},"2afba1b1":{"m":85,"g":94},"e330f2b8":{"m":85,"g":94},"3ddf5b9d":{"m":85,"g":94},"3cff9633":{"m":85,"g":94},"d50e36a7":{"m":85,"g":94},"8fefdd32":{"m":85,"g":94},"403b855a":{"m":85,"g":94},"1698e94e":{"m":85,"g":94},"58195dd5":{"m":85,"g":94},"799789af":{"m":85,"g":94},"cc4a80ca":{"m":85,"g":94},"3c8a5231":{"m":85,"g":94},"a043f7f2":{"m":85,"g":94},"e3a53044":{"m":85,"g":94},"28b26dbf":{"m":85,"g":94},"2b06484b":{"m":85,"g":94},"e4b6133b":{"m":85,"g":94},"dd408ee4":{"m":85,"g":94},"9419e75d":{"m":85,"g":94},"2c7dbb7c":{"m":85,"g":94},"9a62191b":{"m":85,"g":94},"ae523675":{"m":85,"g":94},"5c08aa49":{"m":85,"g":94},"f4c191a7":{"m":85,"g":94},"771669cb":{"m":85,"g":94},"1468769b":{"m":85,"g":94},"91dda4cd":{"m":85,"g":94},"8e5a6d34":{"m":85,"g":94},"8465f035":{"m":85,"g":94},"8c0cfca8":{"m":85,"g":94},"2c3ea294":{"m":85,"g":94},"5bb0accb":{"m":85,"g":94},"8d463fe3":{"m":85,"g":94},"26fc32d1":{"m":85,"g":94},"1cc32603":{"m":85,"g":94},"05ee2192":{"m":85,"g":94},"678d8cc9":{"m":86,"g":94},"d2cb3024":{"m":86,"g":94},"1940cdec":{"m":86,"g":94},"63484f9f":{"m":86,"g":94},"dff0ab92":{"m":86,"g":94},"e30c273b":{"m":86,"g":94},"0ab3f437":{"m":86,"g":94},"cec98f10":{"m":86,"g":94},"8dc4efd0":{"m":86,"g":94},"6578cf27":{"m":86,"g":94},"087751a8":{"m":86,"g":94},"911f3ba6":{"m":86,"g":94},"f6f96b05":{"m":86,"g":94},"2a936a84":{"m":86,"g":94},"5e023301":{"m":86,"g":94},"fa7d7fd9":{"m":86,"g":94},"f1ff736d":{"m":86,"g":94},"acc816d8":{"m":86,"g":94},"a05bd83a":{"m":86,"g":94},"cef91b1e":{"m":86,"g":94},"6450c122":{"m":86,"g":94},"b6cf3532":{"m":86,"g":94},"3b2680a4":{"m":86,"g":94},"79961afa":{"m":86,"g":94},"cfca4e0e":{"m":86,"g":94},"e88dd482":{"m":86,"g":94},"73600673":{"m":86,"g":94},"8f508cc7":{"m":86,"g":94},"9bddf1c8":{"m":86,"g":94},"24c13ca9":{"m":86,"g":94},"b70957fc":{"m":86,"g":94},"e444c13f":{"m":86,"g":94},"fee37d9e":{"m":86,"g":94},"c68de479":{"m":86,"g":94},"4c7b4242":{"m":86,"g":94},"38053c33":{"m":86,"g":94},"00c2c1f0":{"m":86,"g":94},"cb691945":{"m":86,"g":94},"d25398cb":{"m":86,"g":94},"8a828666":{"m":86,"g":94},"aff584fa":{"m":86,"g":94},"6f566147":{"m":86,"g":94},"bdd17998":{"m":86,"g":94},"c9abd7be":{"m":86,"g":94},"a3e4e9bf":{"m":86,"g":94},"6d4d3bc8":{"m":86,"g":94},"5f300141":{"m":86,"g":94},"1c05425b":{"m":86,"g":94},"b26cb1c5":{"m":86,"g":94},"f8e46093":{"m":86,"g":94},"683707c3":{"m":86,"g":94},"a68ed766":{"m":86,"g":94},"82653f66":{"m":86,"g":94},"22da3d97":{"m":86,"g":94},"b8559764":{"m":86,"g":94},"56f6589e":{"m":86,"g":94},"1232f7e8":{"m":86,"g":94},"3008db9c":{"m":86,"g":94},"357fb2db":{"m":86,"g":94},"95c231e5":{"m":86,"g":94},"3042f1da":{"m":86,"g":94},"2b63798c":{"m":86,"g":94},"bf203cb7":{"m":86,"g":94},"8ebde73f":{"m":86,"g":94},"6b0fae79":{"m":86,"g":94},"141a4596":{"m":86,"g":94},"d8ab6011":{"m":86,"g":94},"6579cd7d":{"m":86,"g":94},"97ac42b6":{"m":86,"g":94},"1acca3a2":{"m":86,"g":94},"6ea1e6ac":{"m":86,"g":94},"3409aaab":{"m":86,"g":94},"73dcf2b3":{"m":86,"g":94},"170d1f21":{"m":86,"g":94},"73bc1d00":{"m":86,"g":94},"c5645e92":{"m":86,"g":94},"d33955d2":{"m":86,"g":94},"6fc17596":{"m":86,"g":94},"ad506a4e":{"m":86,"g":94},"ebaba856":{"m":86,"g":94},"de2faef9":{"m":86,"g":94},"67b7d5b1":{"m":86,"g":94},"4322c31e":{"m":86,"g":94},"16267d4f":{"m":87,"g":94},"0f5cb8ca":{"m":87,"g":94},"17299f08":{"m":87,"g":94},"5380cd7e":{"m":87,"g":94},"b2e95f62":{"m":87,"g":94},"1ab14c4c":{"m":87,"g":94},"3c32895c":{"m":87,"g":94},"ac2324c1":{"m":87,"g":94},"ef8ec07b":{"m":87,"g":94},"f24fc5b8":{"m":87,"g":94},"d18c6b33":{"m":87,"g":94},"f1c89600":{"m":87,"g":94},"983c663d":{"m":87,"g":94},"f94543d2":{"m":87,"g":94},"e8e18dcd":{"m":87,"g":94},"bad7c26f":{"m":87,"g":94},"12319a67":{"m":87,"g":94},"d738ab52":{"m":87,"g":94},"3ee40ff9":{"m":87,"g":94},"0f334945":{"m":87,"g":94},"fba8eccd":{"m":87,"g":94},"7d3a3d45":{"m":87,"g":94},"25c83fff":{"m":87,"g":94},"9f2c9568":{"m":87,"g":94},"3f2702ae":{"m":87,"g":94},"6ea05950":{"m":87,"g":94},"e7dd906c":{"m":87,"g":94},"6e2da515":{"m":87,"g":94},"e9a47f4c":{"m":87,"g":94},"03227c5f":{"m":87,"g":94},"01bdbf7f":{"m":87,"g":94},"94d42b67":{"m":87,"g":94},"69276f61":{"m":87,"g":94},"41a645f5":{"m":87,"g":94},"23010630":{"m":87,"g":94},"45b4dcf0":{"m":87,"g":94},"213e8c7d":{"m":87,"g":94},"41273fd7":{"m":87,"g":94},"e9bebafb":{"m":87,"g":94},"4d1c9db6":{"m":87,"g":94},"17c36c55":{"m":87,"g":94},"31d1f6e7":{"m":87,"g":94},"a823c6e8":{"m":87,"g":94},"2ce87935":{"m":87,"g":94},"de167cf5":{"m":87,"g":94},"4319978c":{"m":87,"g":94},"03dd785c":{"m":87,"g":94},"66fc63d6":{"m":87,"g":94},"921e4a81":{"m":87,"g":94},"9d8ec2e6":{"m":87,"g":94},"c178abda":{"m":87,"g":94},"b29a026e":{"m":87,"g":94},"7e257cd6":{"m":88,"g":94},"c4831e2f":{"m":88,"g":94},"2e37fa07":{"m":88,"g":94},"2d831c6e":{"m":88,"g":94},"ed0c3035":{"m":88,"g":94},"e6f11356":{"m":88,"g":94},"7b02c326":{"m":88,"g":94},"fefa19fe":{"m":88,"g":94},"9c574585":{"m":88,"g":94},"8233cc10":{"m":88,"g":94},"1b2e8f76":{"m":88,"g":94},"d2e0881a":{"m":88,"g":94},"2f427491":{"m":88,"g":94},"d8189660":{"m":88,"g":94},"3ded6235":{"m":88,"g":94},"4ba1eea8":{"m":88,"g":94},"4685fbb8":{"m":88,"g":94},"0a4fc73b":{"m":88,"g":94},"a6970a17":{"m":88,"g":94},"a6ae3af1":{"m":88,"g":94},"0b07c4a9":{"m":88,"g":94},"fc0e3b91":{"m":88,"g":94},"d71f3f0a":{"m":88,"g":94},"58f10679":{"m":88,"g":94},"7a80f565":{"m":88,"g":94},"9484eba4":{"m":88,"g":94},"e9feb488":{"m":88,"g":94},"fc992a09":{"m":88,"g":94},"121f92c5":{"m":88,"g":94},"3bde1010":{"m":88,"g":94},"75135580":{"m":88,"g":94},"4d643f6c":{"m":88,"g":94},"6ce0ed07":{"m":88,"g":94},"969660c7":{"m":88,"g":94},"16d4f680":{"m":88,"g":94},"ada268fd":{"m":88,"g":94},"cfe48c59":{"m":88,"g":94},"d4c038da":{"m":88,"g":94},"55f6005f":{"m":88,"g":94},"7222e1da":{"m":88,"g":94},"505eec4d":{"m":88,"g":94},"ccfe5c00":{"m":88,"g":94},"a071dc40":{"m":88,"g":94},"a40aecc5":{"m":88,"g":94},"d6e1d28c":{"m":88,"g":94},"7c347259":{"m":88,"g":94},"669caa0a":{"m":88,"g":94},"4024e1d2":{"m":88,"g":94},"5c0b38f3":{"m":88,"g":94},"30ca18f4":{"m":88,"g":94},"03886917":{"m":88,"g":94},"66324895":{"m":88,"g":94},"13feffd0":{"m":88,"g":94},"e98afbe0":{"m":88,"g":94},"69af3ec3":{"m":88,"g":94},"32cc66ef":{"m":88,"g":94},"83f2d9d4":{"m":88,"g":94},"6317c5c6":{"m":88,"g":94},"cba1cdbc":{"m":88,"g":94},"c471d39e":{"m":88,"g":94},"d0443275":{"m":88,"g":94},"17d080b7":{"m":88,"g":94},"1b19df4b":{"m":88,"g":94},"f0653886":{"m":88,"g":94},"b1465557":{"m":88,"g":94},"b06215da":{"m":88,"g":94},"7adf245b":{"m":88,"g":94},"299fd22f":{"m":88,"g":94},"506e5de8":{"m":88,"g":94},"844e2f22":{"m":88,"g":94},"4f39bcf7":{"m":88,"g":94},"31c9569b":{"m":88,"g":94},"1be6956d":{"m":88,"g":94},"626ccb7d":{"m":88,"g":94},"72bfb0ba":{"m":88,"g":94},"15521495":{"m":88,"g":94},"ebe58d54":{"m":88,"g":94},"066cf445":{"m":88,"g":94},"6dc6b306":{"m":88,"g":94},"1f30c05d":{"m":88,"g":94},"5dd62c3a":{"m":88,"g":94},"f11481b9":{"m":88,"g":94},"9d24c3ff":{"m":88,"g":94},"24161c59":{"m":88,"g":94},"eabcf82a":{"m":88,"g":94},"c47a51db":{"m":88,"g":94},"11553c1a":{"m":88,"g":94},"01dd39ba":{"m":88,"g":94},"b3f3d610":{"m":88,"g":94},"f07c6a00":{"m":88,"g":94},"4bb816d4":{"m":88,"g":94},"c250939e":{"m":88,"g":94},"b6909aa2":{"m":88,"g":94},"f8728357":{"m":88,"g":94},"73187152":{"m":88,"g":94},"40865665":{"m":88,"g":94},"fd08c048":{"m":88,"g":94},"26ebb849":{"m":88,"g":94},"02973cd9":{"m":88,"g":94},"6d95a35a":{"m":88,"g":94},"01d2838c":{"m":88,"g":94},"e3b8a722":{"m":88,"g":94},"3cf1473a":{"m":88,"g":94},"27168308":{"m":88,"g":94},"e3bed74a":{"m":88,"g":94},"e9ef39d2":{"m":88,"g":94},"205d5cb4":{"m":88,"g":94},"3d7f7a43":{"m":88,"g":94},"2df9d40a":{"m":88,"g":94},"8dc191f2":{"m":88,"g":94},"64825b83":{"m":88,"g":94},"69748d08":{"m":88,"g":94},"dcc0a456":{"m":88,"g":94},"c2b7ddca":{"m":88,"g":94},"abebd939":{"m":88,"g":94},"4bd2952a":{"m":88,"g":94},"6fc93575":{"m":88,"g":94},"839fb31e":{"m":88,"g":94},"f19a9204":{"m":88,"g":94},"c23a7072":{"m":88,"g":94},"e07a6977":{"m":88,"g":94},"cd8d4b9d":{"m":88,"g":94},"f194e14f":{"m":88,"g":94},"cfc9f9ab":{"m":88,"g":94},"fb4959b2":{"m":88,"g":94},"9a405274":{"m":88,"g":94},"2e4babdb":{"m":88,"g":94},"44a3783d":{"m":88,"g":94},"f3bf6110":{"m":88,"g":94},"198b9056":{"m":88,"g":94},"73eb67c0":{"m":88,"g":94},"9a91fa0e":{"m":88,"g":94},"cd7c8a8d":{"m":88,"g":94},"3e350a93":{"m":88,"g":94},"fb71725c":{"m":88,"g":94},"912788c0":{"m":88,"g":94},"0f75b907":{"m":88,"g":94},"4f723edd":{"m":89,"g":94},"fcde67b0":{"m":89,"g":94},"81372f3b":{"m":89,"g":94},"baa6624d":{"m":89,"g":94},"c2b16795":{"m":89,"g":94},"f6ebba53":{"m":89,"g":94},"6716b417":{"m":89,"g":94},"1c8b42c8":{"m":89,"g":94},"f20f7000":{"m":89,"g":94},"f40942ad":{"m":89,"g":94},"dc0705a5":{"m":89,"g":94},"a968c888":{"m":89,"g":94},"a979daac":{"m":89,"g":94},"f1569876":{"m":89,"g":94},"3465d7ae":{"m":89,"g":94},"e58423b2":{"m":89,"g":94},"7059ae16":{"m":89,"g":94},"51d9a597":{"m":89,"g":94},"56ccd3c2":{"m":89,"g":94},"98c00a2d":{"m":89,"g":94},"451ffe74":{"m":89,"g":94},"b1e5a33a":{"m":89,"g":94},"9d5fa68b":{"m":89,"g":94},"2c186425":{"m":89,"g":94},"18efb5e8":{"m":89,"g":94},"de1350ea":{"m":89,"g":94},"86fe943b":{"m":89,"g":94},"9ecb1856":{"m":89,"g":94},"cc74499d":{"m":89,"g":94},"0c1f03a2":{"m":89,"g":94},"3712abfa":{"m":89,"g":94},"971a0dfa":{"m":89,"g":94},"2fc12995":{"m":89,"g":94},"20d3ad3b":{"m":89,"g":94},"fa3592cf":{"m":89,"g":94},"608668e1":{"m":89,"g":94},"6c0a4828":{"m":89,"g":94},"47402883":{"m":89,"g":94},"1fb76ebb":{"m":89,"g":94},"c2c4f57f":{"m":89,"g":94},"23881fa6":{"m":89,"g":94},"8db3ac55":{"m":89,"g":94},"3e56f557":{"m":89,"g":94},"62fec60d":{"m":89,"g":94},"e7759778":{"m":89,"g":94},"77e928d0":{"m":89,"g":94},"515ef4fa":{"m":89,"g":94},"f5599ef1":{"m":89,"g":94},"c499591a":{"m":89,"g":94},"e1ce44cd":{"m":89,"g":94},"f1114e7f":{"m":89,"g":94},"bae4fdc7":{"m":89,"g":94},"6153f2ff":{"m":89,"g":94},"8b5f83ed":{"m":89,"g":94},"2a413829":{"m":89,"g":94},"d5c097a2":{"m":89,"g":94},"9736cd3b":{"m":89,"g":94},"2f715f51":{"m":89,"g":94},"d664ca18":{"m":89,"g":94},"22fe7878":{"m":89,"g":94},"c4ffbeca":{"m":89,"g":94},"f8eaaab8":{"m":89,"g":94},"697b0f71":{"m":89,"g":94},"132dad87":{"m":89,"g":94},"60fdad7c":{"m":89,"g":94},"61ce91ed":{"m":89,"g":94},"e6b7053b":{"m":89,"g":94},"5f91c825":{"m":89,"g":94},"b819381f":{"m":89,"g":94},"562f279a":{"m":89,"g":94},"8b247489":{"m":89,"g":94},"0df6765c":{"m":89,"g":94},"35b65cf0":{"m":89,"g":94},"dd1012fc":{"m":89,"g":94},"44aab7f9":{"m":89,"g":94},"43baba64":{"m":89,"g":94},"0166403c":{"m":89,"g":94},"bcf66ef3":{"m":89,"g":94},"0de5e7d4":{"m":89,"g":94},"72a110f6":{"m":89,"g":94},"5aff1e93":{"m":89,"g":94},"8e3797be":{"m":89,"g":94},"4474eaf5":{"m":89,"g":94},"499f5e62":{"m":89,"g":94},"81964328":{"m":89,"g":94},"f0f84975":{"m":89,"g":94},"3f1e4339":{"m":89,"g":94},"cf9815ba":{"m":89,"g":94},"bd75690f":{"m":89,"g":94},"180ff5ee":{"m":89,"g":94},"37f15475":{"m":89,"g":94},"8a548052":{"m":89,"g":94},"b6d0ce9f":{"m":89,"g":94},"0ea330ca":{"m":89,"g":94},"27e327b4":{"m":89,"g":94},"ff00895c":{"m":89,"g":94},"ff914748":{"m":89,"g":94},"eb38c7d1":{"m":89,"g":94},"df7f61ee":{"m":89,"g":94},"ef21729c":{"m":89,"g":94},"f5159315":{"m":89,"g":94},"6d7b6696":{"m":89,"g":94},"6376b632":{"m":89,"g":94},"e05e29d1":{"m":89,"g":94},"a2cb5913":{"m":89,"g":94},"55444ed6":{"m":89,"g":94},"20fd53b8":{"m":89,"g":94},"6a47b730":{"m":89,"g":94},"c429919d":{"m":89,"g":94},"1da8d230":{"m":89,"g":94},"2f7420bc":{"m":89,"g":94},"c6a0cacc":{"m":89,"g":94},"0a9bfc20":{"m":89,"g":94},"34c63731":{"m":89,"g":94},"2d72fc47":{"m":89,"g":94},"b520d028":{"m":89,"g":94},"7dc0e394":{"m":89,"g":94},"fb507b7b":{"m":89,"g":94},"f90945c4":{"m":89,"g":94},"094fbdac":{"m":89,"g":94},"888cb175":{"m":89,"g":94},"e39bca07":{"m":89,"g":94},"a2bb8565":{"m":89,"g":94},"ced3c07a":{"m":89,"g":94},"f18b068f":{"m":89,"g":94},"4fac524b":{"m":89,"g":94},"b581b225":{"m":89,"g":94},"69dd878b":{"m":89,"g":94},"22630ca2":{"m":89,"g":94},"d279d499":{"m":89,"g":94},"6cb00c63":{"m":89,"g":94},"62cac2c4":{"m":89,"g":94},"2c3b71d6":{"m":89,"g":94},"51cdd81f":{"m":89,"g":94},"73def253":{"m":89,"g":94},"d9d35def":{"m":89,"g":94},"6df81e8a":{"m":89,"g":94},"3ab7d9b5":{"m":89,"g":94},"7e5071c9":{"m":89,"g":94},"78689d33":{"m":89,"g":94},"1dc6864f":{"m":89,"g":94},"485a023b":{"m":89,"g":94},"7e412900":{"m":89,"g":94},"c673727e":{"m":89,"g":94},"f4d4f939":{"m":89,"g":94},"f2bd3515":{"m":89,"g":94},"c459536b":{"m":89,"g":94},"535c8386":{"m":89,"g":94},"2163586e":{"m":89,"g":94},"e06b0761":{"m":89,"g":94},"844a8f42":{"m":89,"g":94},"791b3bfa":{"m":89,"g":94},"31589e17":{"m":89,"g":94},"ae6a5b29":{"m":89,"g":94},"4839999b":{"m":89,"g":94},"541a985f":{"m":89,"g":94},"5170b010":{"m":89,"g":94},"d63e76f7":{"m":89,"g":94},"e9fd11c0":{"m":89,"g":94},"c7588d59":{"m":89,"g":94},"6b231325":{"m":89,"g":94},"b1c8d4e9":{"m":89,"g":94},"c25231c6":{"m":89,"g":94},"fba03b29":{"m":89,"g":94},"461a7302":{"m":89,"g":94},"07610353":{"m":89,"g":94},"c087ddd6":{"m":89,"g":94},"f4a8987f":{"m":89,"g":94},"41ba767f":{"m":89,"g":94},"f127355a":{"m":89,"g":94},"bdb962d7":{"m":89,"g":94},"0b9557fc":{"m":89,"g":94},"87068b5c":{"m":89,"g":94},"a564e001":{"m":89,"g":94},"2103b806":{"m":89,"g":94},"e806f708":{"m":89,"g":94},"fa6723f0":{"m":89,"g":94},"673ff668":{"m":89,"g":94},"447be242":{"m":89,"g":94},"183d9f96":{"m":89,"g":94},"63195028":{"m":89,"g":94},"a3d7f4b6":{"m":89,"g":94},"b18416fb":{"m":89,"g":94},"ce9d690e":{"m":89,"g":94},"bdaefbbf":{"m":89,"g":94},"45a31a82":{"m":89,"g":94},"1aa0fbf4":{"m":89,"g":94},"7a0bbe6a":{"m":89,"g":94},"ae335842":{"m":89,"g":94},"477a101c":{"m":89,"g":94},"1a8f5f68":{"m":89,"g":94},"32cd7070":{"m":89,"g":94},"ebd1ed49":{"m":89,"g":94},"f77da699":{"m":89,"g":94},"d6864ce6":{"m":89,"g":94},"755a3661":{"m":89,"g":94},"79a39ac0":{"m":89,"g":94},"3ce94f71":{"m":89,"g":94},"ca95556c":{"m":89,"g":94},"eb8f02dd":{"m":89,"g":94},"0ca3e568":{"m":89,"g":94},"5c7aa009":{"m":89,"g":94},"fe386aca":{"m":89,"g":94},"14d1075f":{"m":89,"g":94},"006ead9d":{"m":89,"g":94},"0d503090":{"m":89,"g":94},"501efc3d":{"m":89,"g":94},"f9bab3d5":{"m":89,"g":94},"16f69b1f":{"m":89,"g":94},"65f09131":{"m":89,"g":94},"fc419b62":{"m":89,"g":94},"7eb9d8e5":{"m":89,"g":94},"84147254":{"m":89,"g":94},"6bebef60":{"m":89,"g":94},"25be63d0":{"m":89,"g":94},"d502dae0":{"m":89,"g":94},"93e53f6e":{"m":89,"g":94},"a191a0e4":{"m":89,"g":94},"8c7279c2":{"m":89,"g":94},"0ca18117":{"m":89,"g":94},"2c3a6fe1":{"m":89,"g":94},"8b33d8df":{"m":89,"g":94},"e235be16":{"m":89,"g":94},"5ccf8fe1":{"m":89,"g":94},"3f23d8cd":{"m":89,"g":94},"1a399799":{"m":89,"g":94},"022012aa":{"m":89,"g":94},"681e7af3":{"m":89,"g":94},"681fdc26":{"m":89,"g":94},"0d477880":{"m":89,"g":94},"f4560373":{"m":89,"g":94},"b2388433":{"m":89,"g":94},"a38376fa":{"m":89,"g":94},"7a5e6ce1":{"m":89,"g":94},"24c035f2":{"m":89,"g":94},"f9dc9dd2":{"m":90,"g":94},"62a7aa2e":{"m":90,"g":94},"5ca07eed":{"m":90,"g":94},"e30ef368":{"m":90,"g":94},"91a066ec":{"m":90,"g":94},"c4943867":{"m":90,"g":94},"53a525bf":{"m":90,"g":94},"7ddf8e83":{"m":90,"g":94},"8321f8e4":{"m":90,"g":94},"cfceb83d":{"m":90,"g":94},"b1286a11":{"m":90,"g":94},"21615cc3":{"m":90,"g":94},"0ae1e9a7":{"m":90,"g":94},"e07d0647":{"m":90,"g":94},"3c2274fb":{"m":90,"g":94},"d2679f51":{"m":90,"g":94},"96be97bf":{"m":90,"g":94},"88f9c347":{"m":90,"g":94},"fff10809":{"m":90,"g":94},"5f1ab327":{"m":90,"g":94},"7df7c679":{"m":90,"g":94},"38af4f68":{"m":90,"g":94},"a6305c7d":{"m":90,"g":94},"a023856b":{"m":90,"g":94},"db0cc57e":{"m":90,"g":94},"349bb2c9":{"m":90,"g":94},"0b8939bc":{"m":90,"g":94},"ed89837c":{"m":90,"g":94},"55561e25":{"m":90,"g":94},"44733203":{"m":90,"g":94},"0bd67ba2":{"m":90,"g":94},"7d316991":{"m":90,"g":94},"ab1a4fa5":{"m":90,"g":94},"ed54bf9d":{"m":90,"g":94},"b57d87c2":{"m":90,"g":94},"98538822":{"m":90,"g":94},"f47a1b1d":{"m":90,"g":94},"93cec433":{"m":90,"g":94},"ba589b88":{"m":90,"g":94},"50876abc":{"m":90,"g":94},"b4c41f72":{"m":90,"g":94},"8b8f2e74":{"m":90,"g":94},"0fc3d992":{"m":90,"g":94},"be2d985d":{"m":90,"g":94},"5b1afa78":{"m":90,"g":94},"c49c1d92":{"m":90,"g":94},"0f1dfa1e":{"m":90,"g":94},"e3ec6bf4":{"m":90,"g":94},"b04df75a":{"m":90,"g":94},"bec3e484":{"m":90,"g":94},"8ab7d93c":{"m":90,"g":94},"5c66c442":{"m":90,"g":94},"aa46ed34":{"m":90,"g":94},"2f4ec752":{"m":90,"g":94},"da47621c":{"m":90,"g":94},"22a6b9fc":{"m":90,"g":94},"b02df20a":{"m":90,"g":94},"bd7cfbd2":{"m":90,"g":94},"4b9971e4":{"m":90,"g":94},"dcc79d32":{"m":90,"g":94},"7046e0fa":{"m":90,"g":94},"930746d9":{"m":90,"g":94},"84727a51":{"m":90,"g":94},"ef326774":{"m":90,"g":94},"021f76e4":{"m":90,"g":94},"777688b8":{"m":90,"g":94},"0ca594ed":{"m":90,"g":94},"31d6dee5":{"m":90,"g":94},"02543b54":{"m":90,"g":94},"25a6a9aa":{"m":90,"g":94},"83d87685":{"m":90,"g":94},"2a5f0100":{"m":90,"g":94},"dbdf76ca":{"m":90,"g":94},"f2a75a66":{"m":90,"g":94},"6b12d6a8":{"m":90,"g":94},"0f218731":{"m":90,"g":94},"14c18d25":{"m":90,"g":94},"90bd3e32":{"m":90,"g":94},"ca929118":{"m":90,"g":94},"344adb00":{"m":90,"g":94},"b56de8f9":{"m":90,"g":94},"ce5ee3bd":{"m":90,"g":94},"a0e4d4eb":{"m":90,"g":94},"2f584455":{"m":90,"g":94},"fe55947a":{"m":90,"g":94},"19995dd7":{"m":90,"g":94},"3b014bc1":{"m":90,"g":94},"d7c3e8e9":{"m":90,"g":94},"8ea7df61":{"m":90,"g":94},"4a102a2b":{"m":90,"g":94},"6406408a":{"m":90,"g":94},"019851d0":{"m":90,"g":94},"2dae104d":{"m":90,"g":94},"cef6655b":{"m":90,"g":94},"27196d41":{"m":90,"g":94},"bb185b0e":{"m":90,"g":94},"7c3a12c0":{"m":91,"g":94},"e846d95e":{"m":91,"g":94},"15f34013":{"m":91,"g":94},"d04163b3":{"m":91,"g":94},"7732bbe4":{"m":91,"g":94},"ed0a0b69":{"m":91,"g":94},"fa42e419":{"m":91,"g":94},"e5afb88b":{"m":91,"g":94},"e5ddeb04":{"m":91,"g":94},"bdbb8d00":{"m":91,"g":94},"34c3f9b2":{"m":91,"g":94},"76139bfb":{"m":91,"g":94},"f8d48fd3":{"m":91,"g":94},"34b6b842":{"m":91,"g":94},"25549433":{"m":91,"g":94},"d6dddc19":{"m":91,"g":94},"55e03b10":{"m":91,"g":94},"8aa68ed5":{"m":91,"g":94},"506c4928":{"m":91,"g":94},"30ceccc7":{"m":91,"g":94},"ac5010e0":{"m":91,"g":94},"3cee035e":{"m":91,"g":94},"30f2a44a":{"m":91,"g":94},"bd4f5818":{"m":91,"g":94},"50f1b6d6":{"m":91,"g":94},"5962e70d":{"m":91,"g":94},"edc21cc8":{"m":91,"g":94},"05c9bc89":{"m":91,"g":94},"b7a2df0a":{"m":91,"g":94},"1998ce40":{"m":91,"g":94},"72676cd6":{"m":91,"g":94},"02bf31ef":{"m":91,"g":94},"5ea5d221":{"m":91,"g":94},"fdfd5224":{"m":91,"g":94},"7f3ee861":{"m":91,"g":94},"bec58910":{"m":91,"g":94},"9edf6608":{"m":91,"g":94},"ab74f8f0":{"m":91,"g":94},"5e7fdc79":{"m":91,"g":94},"4d8d9b8e":{"m":91,"g":94},"5041df2d":{"m":91,"g":94},"256801e9":{"m":91,"g":94},"73b13e69":{"m":91,"g":94},"8609e637":{"m":91,"g":94},"dea2b84b":{"m":91,"g":94},"cfb2fb5a":{"m":91,"g":94},"22bfed75":{"m":91,"g":94},"e879d8b7":{"m":91,"g":94},"09988080":{"m":91,"g":94},"794be55a":{"m":91,"g":94},"187b85b7":{"m":91,"g":94},"ceba0ce4":{"m":91,"g":94},"1ab6be1b":{"m":91,"g":94},"4df5fc21":{"m":91,"g":94},"a06912ad":{"m":91,"g":94},"97011abc":{"m":91,"g":94},"1d6515ef":{"m":91,"g":94},"dea8aa7a":{"m":91,"g":94},"906dbc34":{"m":91,"g":94},"fadf18fd":{"m":91,"g":94},"f88e7085":{"m":91,"g":94},"4f838c09":{"m":91,"g":94},"d20a073b":{"m":91,"g":94},"47367b76":{"m":91,"g":94},"650127a1":{"m":91,"g":94},"3774f078":{"m":91,"g":94},"9179ea15":{"m":91,"g":94},"20a503c7":{"m":91,"g":94},"ffd1a26e":{"m":91,"g":94},"09ae5b20":{"m":91,"g":94},"712bf9ec":{"m":91,"g":94},"9c6a0656":{"m":91,"g":94},"2ae809c5":{"m":91,"g":94},"1de4db9b":{"m":91,"g":94},"31fccf5a":{"m":91,"g":94},"b783c1cb":{"m":91,"g":94},"094c116f":{"m":91,"g":94},"e56685ac":{"m":91,"g":94},"c26d7349":{"m":91,"g":94},"ceaa85c9":{"m":91,"g":94},"0650e517":{"m":91,"g":94},"fc554105":{"m":91,"g":94},"4f204db5":{"m":91,"g":94},"3eb4a800":{"m":91,"g":94},"e7261315":{"m":91,"g":94},"8c16da33":{"m":91,"g":94},"a39d9287":{"m":91,"g":94},"10d60cd4":{"m":91,"g":94},"8a10c4c3":{"m":91,"g":94},"405780bc":{"m":91,"g":94},"1dffee31":{"m":91,"g":94},"70c471a8":{"m":91,"g":94},"1a9c2c92":{"m":91,"g":94},"873ae12c":{"m":91,"g":94},"c64290dc":{"m":91,"g":94},"8e2363dc":{"m":91,"g":94},"69183f88":{"m":92,"g":94},"9b00990b":{"m":92,"g":94},"4d67025a":{"m":92,"g":94},"0e05fe8c":{"m":92,"g":94},"2390a2bc":{"m":92,"g":94},"16d76b9f":{"m":92,"g":94},"5c214257":{"m":92,"g":94},"b8df43ab":{"m":92,"g":94},"a1c1ebe9":{"m":92,"g":94},"fe2a0f96":{"m":92,"g":94},"20beb370":{"m":92,"g":94},"00fbd8a4":{"m":92,"g":94},"802815e4":{"m":92,"g":94},"4c6675c4":{"m":92,"g":94},"e21aa1df":{"m":92,"g":94},"f3cbd245":{"m":92,"g":94},"506a2d59":{"m":92,"g":94},"a07f8ae4":{"m":92,"g":94},"7eb47b0f":{"m":92,"g":94},"bc2e5645":{"m":92,"g":94},"3abc3036":{"m":92,"g":94},"afeed465":{"m":92,"g":94},"587b4c6e":{"m":92,"g":94},"7b9a174a":{"m":92,"g":94},"03c039c4":{"m":92,"g":94},"57ab7769":{"m":92,"g":94},"112b496a":{"m":92,"g":94},"3562256b":{"m":92,"g":94},"5f527834":{"m":92,"g":94},"9f1787fa":{"m":92,"g":94},"8ecad0b1":{"m":92,"g":94},"7151194b":{"m":92,"g":94},"2ed68d7a":{"m":92,"g":94},"e984d507":{"m":92,"g":94},"755f3147":{"m":92,"g":94},"ec5f9c62":{"m":93,"g":94},"62f5522f":{"m":93,"g":94},"01f98730":{"m":93,"g":94},"199d6218":{"m":93,"g":94},"f200af0d":{"m":93,"g":94},"5589b750":{"m":93,"g":94},"c04a8a82":{"m":93,"g":94},"6c903611":{"m":93,"g":94},"77cfea68":{"m":93,"g":94},"8fc910db":{"m":93,"g":94},"75354d9a":{"m":93,"g":94},"4fece12b":{"m":93,"g":94},"c7973222":{"m":93,"g":94},"ef8a29c4":{"m":93,"g":94},"8e9fb43d":{"m":93,"g":94},"83646089":{"m":93,"g":94},"da3890e8":{"m":93,"g":94},"cb432f17":{"m":93,"g":94},"1964c325":{"m":93,"g":94},"af564774":{"m":93,"g":94},"af46f299":{"m":93,"g":94},"16a6b1d8":{"m":93,"g":94},"14229ccf":{"m":93,"g":94},"975a5ec6":{"m":93,"g":94},"1e3e3add":{"m":93,"g":94},"8c298031":{"m":93,"g":94},"4de03953":{"m":93,"g":94},"8b1942c6":{"m":93,"g":94},"489934be":{"m":93,"g":94},"43f93f63":{"m":93,"g":94},"aca1101a":{"m":93,"g":94},"2998c4bd":{"m":93,"g":94},"6840a7bb":{"m":93,"g":94},"c01a1df5":{"m":93,"g":94},"00991723":{"m":93,"g":94},"264dc6e7":{"m":93,"g":94},"646cef2e":{"m":93,"g":94},"1dce6c48":{"m":93,"g":94},"9fcc9a80":{"m":93,"g":94},"ac49dac0":{"m":93,"g":94},"1e0e5497":{"m":93,"g":94},"b5822651":{"m":93,"g":94},"2c4feaf3":{"m":93,"g":94},"2ff572e2":{"m":93,"g":94},"84f2e4a0":{"m":93,"g":94},"8f844db6":{"m":93,"g":94},"36cc3ffd":{"m":93,"g":94},"1bebd315":{"m":93,"g":94},"d3c275b1":{"m":93,"g":94},"b044400d":{"m":93,"g":94},"40e5cb7a":{"m":93,"g":94},"8e64140e":{"m":93,"g":94},"82f021e2":{"m":93,"g":94},"0626f678":{"m":93,"g":94},"09e699bb":{"m":93,"g":94},"b116b21a":{"m":93,"g":94},"88f484ce":{"m":93,"g":94},"8e03b641":{"m":93,"g":94},"b3fa5dc3":{"m":93,"g":94},"00aec6ad":{"m":93,"g":94},"1a08358a":{"m":93,"g":94},"f18a8fdd":{"m":93,"g":94},"a7efbb27":{"m":93,"g":94},"93b6785d":{"m":93,"g":94},"f9eb04dd":{"m":93,"g":94},"3a911b85":{"m":93,"g":94},"886d3449":{"m":93,"g":94},"637bfee4":{"m":93,"g":94},"6005ecee":{"m":93,"g":94},"ff2e9c94":{"m":93,"g":94},"3e34e900":{"m":93,"g":94},"7349717e":{"m":93,"g":94},"392e441a":{"m":93,"g":94},"7248272c":{"m":93,"g":94},"22352d47":{"m":93,"g":94},"c5131f7a":{"m":93,"g":94},"78700893":{"m":93,"g":94},"663c04f7":{"m":93,"g":94},"3b3f1e3a":{"m":93,"g":94},"b691dcc4":{"m":93,"g":94},"0c9c6c75":{"m":93,"g":94},"e3f9b548":{"m":93,"g":94},"b3cff365":{"m":93,"g":94},"8f335b5b":{"m":93,"g":94},"b2264076":{"m":93,"g":94},"04b35190":{"m":93,"g":94},"071a1f51":{"m":93,"g":94},"7c0db3a6":{"m":93,"g":94},"c45e49d8":{"m":93,"g":94},"d8053929":{"m":93,"g":94},"00c7b1ad":{"m":93,"g":94},"82eccae4":{"m":93,"g":94},"a8c10aee":{"m":93,"g":94},"eb429b88":{"m":93,"g":94},"49538d11":{"m":93,"g":94},"cfe2edac":{"m":93,"g":94},"2373faa3":{"m":93,"g":94},"9efb2993":{"m":93,"g":94},"a5317b2f":{"m":93,"g":94},"eb6c2c16":{"m":93,"g":94},"357921aa":{"m":93,"g":94},"c071198c":{"m":93,"g":94},"d7374d74":{"m":93,"g":94},"ce3a3e87":{"m":93,"g":94},"41650b0d":{"m":93,"g":94},"1b951620":{"m":93,"g":94},"29bd4c81":{"m":93,"g":94},"031f64aa":{"m":93,"g":94},"3d7cdb2e":{"m":93,"g":94},"604efe07":{"m":93,"g":94},"1b8cf77b":{"m":93,"g":94},"bb9b608c":{"m":93,"g":94},"066f4ec9":{"m":95,"g":97},"b6b6268c":{"m":95,"g":97},"08702321":{"m":95,"g":97},"64c5907e":{"m":95,"g":97},"128f16a8":{"m":95,"g":97},"49861046":{"m":95,"g":97},"a37e1247":{"m":95,"g":97},"136c6e04":{"m":95,"g":97},"43e20c06":{"m":95,"g":97},"4bab50a6":{"m":95,"g":97},"2e7ab862":{"m":95,"g":97},"51ae4030":{"m":95,"g":97},"653b873b":{"m":95,"g":97},"d379bda4":{"m":95,"g":97},"2b0e1d1c":{"m":95,"g":97},"659907e3":{"m":95,"g":97},"cb9d91ea":{"m":95,"g":97},"6a6e0bb7":{"m":95,"g":97},"076313bd":{"m":95,"g":97},"9abe1163":{"m":95,"g":97},"3646f6bb":{"m":95,"g":94},"35724aa1":{"m":95,"g":94},"2fc824b8":{"m":95,"g":94},"253454de":{"m":95,"g":94},"ea3e7ffe":{"m":95,"g":94},"8d4a01cb":{"m":95,"g":94},"a3398d84":{"m":95,"g":94},"ba69c153":{"m":95,"g":94},"3589aa79":{"m":95,"g":94},"e00715eb":{"m":95,"g":94},"ea4bf122":{"m":95,"g":94},"a291439a":{"m":95,"g":94},"54411f6a":{"m":95,"g":94},"625018d2":{"m":95,"g":94},"5732d904":{"m":95,"g":94},"eb118d88":{"m":96,"g":97},"732fc8e4":{"m":96,"g":97},"f2d5c492":{"m":96,"g":97},"2a2d3478":{"m":96,"g":97},"aa205609":{"m":96,"g":97},"61bb2858":{"m":96,"g":97},"880221bd":{"m":96,"g":97},"8f3173d0":{"m":96,"g":97},"26118a13":{"m":96,"g":97},"475a249b":{"m":96,"g":97},"191d836f":{"m":96,"g":97},"86044712":{"m":96,"g":97},"61555307":{"m":96,"g":97},"49a5915f":{"m":96,"g":97},"766392c6":{"m":96,"g":97},"4a0d1919":{"m":96,"g":97},"57482415":{"m":96,"g":97},"2d54d4bb":{"m":96,"g":97},"b5e3d603":{"m":96,"g":97},"4ed57807":{"m":96,"g":97},"dd445a41":{"m":96,"g":97},"7590f522":{"m":96,"g":97},"f9df11ae":{"m":96,"g":97},"d389bedf":{"m":96,"g":97},"ac80f4da":{"m":96,"g":97},"d487555f":{"m":96,"g":97},"e5888edd":{"m":96,"g":97},"01c00004":{"m":98,"g":103},"0dfe2491":{"m":98,"g":103},"ff45ab7a":{"m":98,"g":103},"0f8b5386":{"m":98,"g":103},"c33499a6":{"m":98,"g":103},"e50109f2":{"m":98,"g":103},"69adc4f8":{"m":98,"g":103},"11483785":{"m":98,"g":103},"7b68d271":{"m":98,"g":103},"74f59ae5":{"m":98,"g":103},"6936be32":{"m":98,"g":103},"9b5de6cb":{"m":98,"g":97},"5c8365a0":{"m":98,"g":97},"8430bfe3":{"m":98,"g":97},"c9e8613c":{"m":98,"g":97},"429bb0ef":{"m":98,"g":97},"7eebd440":{"m":98,"g":97},"93d124ef":{"m":98,"g":97},"1fc455e8":{"m":98,"g":97},"465968b2":{"m":98,"g":97},"750838ad":{"m":98,"g":97},"99aefa03":{"m":98,"g":97},"bbcfbc1a":{"m":98,"g":97},"83c104b1":{"m":98,"g":97},"2db6719c":{"m":98,"g":97},"55381a46":{"m":98,"g":97},"a589a071":{"m":98,"g":97},"f62d75b6":{"m":98,"g":97},"0f9b11e3":{"m":98,"g":97},"877e35d7":{"m":98,"g":97},"cbdfb771":{"m":98,"g":97},"282eb59f":{"m":98,"g":97},"4540a466":{"m":98,"g":97},"abda2542":{"m":98,"g":97},"8cddfa56":{"m":98,"g":97},"4e3defe5":{"m":98,"g":97},"60468da4":{"m":98,"g":97},"41d33e47":{"m":98,"g":97},"bfdd226f":{"m":98,"g":97},"3de617a7":{"m":98,"g":97},"bb0e8a32":{"m":98,"g":97},"1b427dae":{"m":98,"g":97},"f3d97361":{"m":98,"g":97},"561dd7b2":{"m":98,"g":97},"f98e88b9":{"m":98,"g":97},"15ad6c90":{"m":98,"g":97},"cfab0ff6":{"m":98,"g":97},"b763cf7e":{"m":98,"g":97},"8fcc55cf":{"m":98,"g":97},"610381b7":{"m":98,"g":97},"1403ea56":{"m":98,"g":97},"b7e951a6":{"m":98,"g":97},"d918ab79":{"m":98,"g":97},"3964b352":{"m":98,"g":97},"9c7a4618":{"m":98,"g":97},"7750b91c":{"m":98,"g":97},"c8f31042":{"m":98,"g":97},"1f76fc87":{"m":98,"g":97},"6737671c":{"m":98,"g":97},"fd63b62e":{"m":98,"g":97},"719b29f2":{"m":98,"g":97},"d0510f08":{"m":98,"g":97},"9d33fcfb":{"m":98,"g":97},"7891bac1":{"m":98,"g":97},"48c1fa7b":{"m":98,"g":97},"8aa5ae6b":{"m":98,"g":97},"8a323557":{"m":98,"g":97},"6e92da8f":{"m":98,"g":97},"e1020dc5":{"m":98,"g":97},"3586b4ce":{"m":98,"g":97},"42960214":{"m":98,"g":97},"01857fab":{"m":98,"g":97},"519ff5c8":{"m":98,"g":97},"af1cc8fe":{"m":98,"g":97},"49b87774":{"m":98,"g":97},"02404a1e":{"m":98,"g":97},"5c08a36c":{"m":98,"g":97},"9069884b":{"m":98,"g":97},"8a7a7770":{"m":98,"g":97},"795668dc":{"m":98,"g":97},"4395c87a":{"m":98,"g":97},"c28ad199":{"m":98,"g":97},"570d3343":{"m":98,"g":97},"d9eb5efc":{"m":98,"g":97},"6dc4af49":{"m":98,"g":97},"b188a89a":{"m":98,"g":97},"497efe74":{"m":98,"g":97},"69f453e5":{"m":98,"g":97},"3bc43c68":{"m":98,"g":97},"7498522f":{"m":98,"g":97},"194841e3":{"m":98,"g":97},"ebff5fcb":{"m":98,"g":97},"f06bd210":{"m":98,"g":97},"14f1f151":{"m":98,"g":97},"38216cf0":{"m":98,"g":97},"4a883795":{"m":98,"g":97},"f1f1d1d4":{"m":98,"g":97},"9120e83d":{"m":98,"g":97},"6e923dbd":{"m":98,"g":97},"c268c11c":{"m":98,"g":97},"e6d59884":{"m":98,"g":97},"9b560c3e":{"m":98,"g":97},"5dc5866e":{"m":98,"g":97},"64e78bb3":{"m":98,"g":97},"7c39e8a1":{"m":98,"g":97},"d969504d":{"m":98,"g":97},"1ebec1a8":{"m":98,"g":97},"d4d0c7c3":{"m":98,"g":97},"8d2cf38c":{"m":98,"g":97},"2117f82d":{"m":98,"g":97},"c07f647c":{"m":98,"g":97},"07452cbe":{"m":98,"g":97},"a562c8a3":{"m":98,"g":97},"cb736df8":{"m":98,"g":97},"e2ed9d04":{"m":98,"g":97},"b5dd5e87":{"m":98,"g":97},"9379da77":{"m":98,"g":97},"0c55cbcf":{"m":98,"g":97},"c46e069d":{"m":98,"g":97},"42fc4410":{"m":98,"g":97},"5f6756b0":{"m":98,"g":97},"98aa836b":{"m":98,"g":97},"22bd857c":{"m":98,"g":97},"ccfa0841":{"m":98,"g":97},"bcc5ba94":{"m":98,"g":97},"cee9f329":{"m":98,"g":97},"2272c2a5":{"m":99,"g":103},"3ec0b212":{"m":99,"g":103},"58c468f4":{"m":99,"g":103},"f8ca2368":{"m":99,"g":103},"d8ee1564":{"m":99,"g":103},"7181ec8c":{"m":99,"g":103},"ed2e313e":{"m":99,"g":103},"f8260f25":{"m":99,"g":103},"12cb760a":{"m":99,"g":103},"1b9cea5a":{"m":99,"g":103},"9045cc1e":{"m":99,"g":103},"70e37b97":{"m":99,"g":103},"15d27591":{"m":99,"g":103},"af4b9bae":{"m":99,"g":103},"7ad6b766":{"m":99,"g":103},"c0fb25e9":{"m":99,"g":103},"28d4d472":{"m":99,"g":103},"f4674df6":{"m":99,"g":103},"d40846d4":{"m":99,"g":103},"145482f4":{"m":99,"g":103},"39fe1e88":{"m":99,"g":103},"33c4b4d0":{"m":99,"g":103},"8d1c5b94":{"m":99,"g":103},"a167fd0b":{"m":99,"g":103},"2f86f3ad":{"m":99,"g":103},"bfb118c0":{"m":99,"g":103},"f6e07f27":{"m":99,"g":103},"5dd0f870":{"m":99,"g":103},"f7e102d5":{"m":99,"g":103},"0e5fa677":{"m":99,"g":103},"624a3b8d":{"m":99,"g":103},"01079e17":{"m":99,"g":103},"0e7a5b26":{"m":99,"g":103},"4953f4ca":{"m":99,"g":103},"38000a5f":{"m":99,"g":103},"70251e93":{"m":99,"g":103},"c87d4fec":{"m":99,"g":103},"a99801e0":{"m":99,"g":103},"4c605235":{"m":99,"g":103},"6f8f4aee":{"m":99,"g":103},"0c8dab9e":{"m":99,"g":103},"f39037ff":{"m":99,"g":103},"ce86e201":{"m":99,"g":103},"b4326330":{"m":99,"g":103},"8abd3e77":{"m":99,"g":103},"e885bfdc":{"m":99,"g":103},"e2d66f60":{"m":99,"g":103},"45bc170b":{"m":100,"g":103},"22623699":{"m":100,"g":103},"fb4ce17d":{"m":100,"g":103},"25f73c6c":{"m":100,"g":103},"581e7dcb":{"m":100,"g":103},"484d0e02":{"m":100,"g":103},"5922c0cb":{"m":100,"g":103},"6d6a8bc2":{"m":100,"g":103},"2fd5c704":{"m":100,"g":103},"4ad97370":{"m":100,"g":103},"28103384":{"m":100,"g":103},"fe6a445d":{"m":100,"g":103},"dd487e55":{"m":100,"g":103},"bb81daef":{"m":100,"g":103},"58dd95fb":{"m":100,"g":103},"b47eda33":{"m":100,"g":103},"e983d666":{"m":100,"g":103},"b58c3c28":{"m":100,"g":103},"df906455":{"m":100,"g":103},"95217a9b":{"m":100,"g":103},"22e00eeb":{"m":100,"g":103},"b3eac168":{"m":100,"g":103},"10ee8955":{"m":100,"g":103},"bf3352c5":{"m":100,"g":103},"4d921f2b":{"m":100,"g":103},"44d600cd":{"m":100,"g":103},"5c9c275b":{"m":100,"g":103},"bf0f448f":{"m":100,"g":103},"36d6f0ba":{"m":100,"g":103},"2a1936de":{"m":100,"g":103},"2ab97023":{"m":100,"g":103},"0bcc195f":{"m":100,"g":103},"91e3d154":{"m":100,"g":103},"85486b6f":{"m":100,"g":103},"e34cf6ad":{"m":100,"g":103},"62222bd2":{"m":100,"g":103},"ed0fdbf3":{"m":100,"g":103},"b602f423":{"m":100,"g":103},"426b7493":{"m":100,"g":103},"528bd1ed":{"m":100,"g":103},"62a6b7c7":{"m":100,"g":103},"76154631":{"m":100,"g":103},"5c705b1d":{"m":100,"g":103},"b7094a5e":{"m":100,"g":103},"da0c0260":{"m":100,"g":103},"3212c2ad":{"m":100,"g":103},"53475674":{"m":100,"g":103},"ce32bc2b":{"m":100,"g":103},"e236d8fe":{"m":100,"g":103},"4fa44d63":{"m":100,"g":103},"e6312d27":{"m":100,"g":103},"8af145b7":{"m":100,"g":103},"6478831b":{"m":101,"g":103},"2e1d2d7e":{"m":101,"g":103},"fb16fbaf":{"m":101,"g":103},"0ce84c82":{"m":101,"g":103},"59d0bf01":{"m":101,"g":103},"7df2c0c2":{"m":101,"g":103},"69712e6f":{"m":101,"g":103},"001bffca":{"m":101,"g":103},"7c969717":{"m":101,"g":103},"8240a6b0":{"m":101,"g":103},"3a04aa4b":{"m":101,"g":103},"bd516949":{"m":101,"g":103},"74e7e457":{"m":101,"g":103},"1466c1b8":{"m":101,"g":103},"9c138a04":{"m":101,"g":103},"c8f549d9":{"m":101,"g":103},"134fa43e":{"m":101,"g":103},"ccfe52a0":{"m":101,"g":103},"747dd450":{"m":101,"g":103},"b5821592":{"m":101,"g":103},"a9dd3ec3":{"m":101,"g":103},"02328864":{"m":102,"g":103},"7a1f7fc5":{"m":102,"g":103},"51c38163":{"m":102,"g":103},"09f1a247":{"m":102,"g":103},"32fa1e9c":{"m":102,"g":103},"e7dc163f":{"m":102,"g":103},"e179e0b7":{"m":102,"g":103},"d9049592":{"m":102,"g":103},"26c8a310":{"m":102,"g":103},"5963e505":{"m":102,"g":103},"43118f5f":{"m":102,"g":103},"a5f5ab40":{"m":102,"g":103},"59aab76f":{"m":102,"g":103},"659bfd10":{"m":102,"g":103},"67e53b16":{"m":102,"g":103},"9b9e8253":{"m":102,"g":103},"66a398f4":{"m":102,"g":103},"29980334":{"m":102,"g":103},"a79a5d70":{"m":102,"g":103},"ec5f9442":{"m":102,"g":103},"3bdcdd13":{"m":102,"g":103},"a730ce81":{"m":102,"g":103},"55ecdc0a":{"m":102,"g":103},"e3f08c77":{"m":102,"g":103},"a9fd8033":{"m":102,"g":103},"2fbb754e":{"m":102,"g":103},"a85ebf50":{"m":102,"g":103},"9effeb5b":{"m":102,"g":103},"1992ef9b":{"m":102,"g":103},"c0fd77e8":{"m":102,"g":103},"a4c3b121":{"m":102,"g":103},"5973675b":{"m":102,"g":103},"4d16c88b":{"m":102,"g":103},"7a4309cc":{"m":102,"g":103},"81367066":{"m":102,"g":103},"263c9236":{"m":102,"g":103},"33f0de33":{"m":105,"g":107},"e7e5a305":{"m":105,"g":107},"dd7ca006":{"m":105,"g":107},"9305ea6c":{"m":105,"g":107},"aa4c66b5":{"m":105,"g":107},"39decec1":{"m":105,"g":104},"f6f46f46":{"m":105,"g":104},"2886e23d":{"m":105,"g":104},"99795d61":{"m":105,"g":104},"fe5086fd":{"m":105,"g":104},"04913430":{"m":105,"g":104},"0ad098b4":{"m":105,"g":104},"4a6e7a66":{"m":105,"g":104},"4b04998d":{"m":105,"g":104},"3dde8619":{"m":105,"g":104},"b7170cc8":{"m":105,"g":104},"5c14515f":{"m":105,"g":104},"2cd2e27f":{"m":105,"g":104},"743638bc":{"m":105,"g":104},"061c8959":{"m":105,"g":104},"4acf6902":{"m":105,"g":104},"aee0ef52":{"m":105,"g":103},"ae807774":{"m":105,"g":103},"8fbcfd07":{"m":105,"g":103},"3c307dc0":{"m":105,"g":103},"5d15fb8c":{"m":105,"g":103},"016fd251":{"m":105,"g":103},"8cd34458":{"m":106,"g":107},"0e0eef00":{"m":106,"g":107},"cb099d20":{"m":106,"g":107},"7a913301":{"m":106,"g":107},"5ce5093b":{"m":106,"g":107},"6f9baf10":{"m":106,"g":107},"a31b7a70":{"m":106,"g":107},"7ed8e51b":{"m":106,"g":107},"32f28154":{"m":106,"g":107},"f7b2853f":{"m":106,"g":107},"b0add2da":{"m":106,"g":107},"0305c505":{"m":106,"g":107},"8675bdf2":{"m":106,"g":107},"a437aa99":{"m":106,"g":107},"0e612dbf":{"m":106,"g":107},"9f47d686":{"m":106,"g":107},"d9def43d":{"m":106,"g":107},"e273aa6d":{"m":106,"g":107},"828a4fe9":{"m":106,"g":107},"8ada1ab6":{"m":106,"g":107},"e314b084":{"m":106,"g":107},"403566bc":{"m":106,"g":107},"0a56b721":{"m":106,"g":107},"603f5ce0":{"m":106,"g":107},"6d4fd882":{"m":106,"g":107},"f9f0138f":{"m":106,"g":107},"ac6962cc":{"m":106,"g":107},"4ca43b06":{"m":106,"g":107},"ea93079b":{"m":106,"g":107},"4bec99ec":{"m":106,"g":107},"89caf7a3":{"m":106,"g":107},"b27b1191":{"m":106,"g":107},"f642524f":{"m":106,"g":107},"82e6c3a6":{"m":106,"g":107},"b89d37cb":{"m":106,"g":107},"5deab128":{"m":106,"g":107},"d1c4d51c":{"m":106,"g":107},"1fe691a4":{"m":106,"g":107},"e2521926":{"m":106,"g":107},"07e46eca":{"m":106,"g":107},"ab9b893e":{"m":106,"g":107},"6a7528e6":{"m":106,"g":107},"2ae95d17":{"m":106,"g":107},"2d401bd9":{"m":106,"g":107},"b17c5b01":{"m":106,"g":107},"db7343c9":{"m":106,"g":107},"533cb5b2":{"m":106,"g":107},"6bdd2786":{"m":106,"g":107},"46e9d1c7":{"m":106,"g":107},"6c88f6c8":{"m":106,"g":107},"c8d3a402":{"m":106,"g":107},"7e831efe":{"m":106,"g":107},"20b5563e":{"m":106,"g":107},"97a38ee8":{"m":108,"g":115},"86d10d22":{"m":108,"g":115},"83871aa1":{"m":108,"g":115},"b1b3f0b3":{"m":108,"g":115},"34e5e11f":{"m":108,"g":115},"2600fc0d":{"m":108,"g":115},"ccd3fb94":{"m":108,"g":115},"c9dd70fb":{"m":108,"g":115},"6b2b8bf0":{"m":108,"g":115},"4edbe0d5":{"m":108,"g":115},"0374304a":{"m":108,"g":115},"127d4b0d":{"m":108,"g":115},"7e880286":{"m":108,"g":115},"446c8e4c":{"m":108,"g":115},"5ef545e6":{"m":108,"g":115},"c4500233":{"m":108,"g":115},"f445a1d9":{"m":108,"g":115},"e5638573":{"m":108,"g":115},"f556ac8b":{"m":108,"g":115},"110a6598":{"m":108,"g":115},"49f9d025":{"m":108,"g":115},"0f587e80":{"m":108,"g":115},"6078d5fc":{"m":108,"g":115},"70cf4abc":{"m":108,"g":115},"cebf4599":{"m":108,"g":115},"9c0c1e30":{"m":108,"g":115},"a1f011d0":{"m":108,"g":115},"9ec314c6":{"m":108,"g":115},"fedfe91c":{"m":108,"g":115},"988accbc":{"m":108,"g":115},"b6b2287e":{"m":108,"g":115},"243e745d":{"m":108,"g":115},"61a0e600":{"m":108,"g":115},"0f8cee8c":{"m":108,"g":115},"816c4c85":{"m":108,"g":115},"13ec8d42":{"m":108,"g":115},"05bd7897":{"m":108,"g":115},"5fd311d3":{"m":108,"g":115},"53e2cd46":{"m":108,"g":115},"9708d353":{"m":108,"g":115},"704ced1b":{"m":108,"g":115},"3cc3d9b9":{"m":108,"g":115},"0b3a5b11":{"m":108,"g":115},"6c855db8":{"m":108,"g":115},"0f9318f7":{"m":108,"g":115},"849957bc":{"m":108,"g":115},"cded039b":{"m":108,"g":115},"275f9df3":{"m":108,"g":115},"e8449ab5":{"m":108,"g":115},"4746aaea":{"m":108,"g":115},"10d34f74":{"m":108,"g":115},"9ba72530":{"m":108,"g":115},"9c8e4f69":{"m":108,"g":115},"78ae1758":{"m":108,"g":115},"dae9a80f":{"m":108,"g":115},"e85cb1ce":{"m":108,"g":115},"55d336cb":{"m":108,"g":115},"de4990a5":{"m":108,"g":115},"029e0af3":{"m":108,"g":115},"64574ef8":{"m":108,"g":115},"18da2c96":{"m":108,"g":115},"9b5f0f64":{"m":108,"g":115},"70bb066e":{"m":108,"g":115},"2c4b4b78":{"m":108,"g":115},"7cd2ee06":{"m":108,"g":115},"eb19ccad":{"m":108,"g":115},"25ef53f0":{"m":108,"g":115},"c674bf9c":{"m":108,"g":115},"af1973b8":{"m":108,"g":115},"5cfbb4c1":{"m":108,"g":115},"e6523102":{"m":108,"g":115},"3828db43":{"m":108,"g":115},"88fbc31b":{"m":108,"g":115},"8f5b9910":{"m":108,"g":115},"ef3004d9":{"m":108,"g":115},"84719b52":{"m":108,"g":115},"e99729c9":{"m":108,"g":115},"c10b8e6a":{"m":108,"g":115},"d4bce297":{"m":108,"g":115},"b0980af8":{"m":108,"g":115},"24eaebeb":{"m":108,"g":115},"a91e90d9":{"m":108,"g":115},"f96413c4":{"m":108,"g":115},"08ebdf79":{"m":108,"g":115},"42c87045":{"m":108,"g":115},"c9bf3877":{"m":108,"g":115},"de2dd738":{"m":108,"g":115},"1ec97697":{"m":108,"g":115},"d8ed60f2":{"m":108,"g":115},"f1b0eda5":{"m":108,"g":115},"f20b6a3f":{"m":108,"g":115},"3680d6f8":{"m":108,"g":115},"f5154495":{"m":108,"g":115},"e0ce171d":{"m":108,"g":115},"fe43e889":{"m":108,"g":115},"5ae5ecaa":{"m":108,"g":115},"5fbad308":{"m":108,"g":115},"7638f5e4":{"m":108,"g":115},"b45f753c":{"m":108,"g":115},"c5057262":{"m":108,"g":115},"46fe8b8c":{"m":108,"g":115},"0b95a01a":{"m":108,"g":115},"a3b810eb":{"m":108,"g":115},"94959237":{"m":108,"g":115},"f4fafacc":{"m":108,"g":115},"01d47a27":{"m":108,"g":115},"ecc9f3e4":{"m":108,"g":115},"7e8187e0":{"m":108,"g":115},"e483ab6d":{"m":108,"g":115},"720cd308":{"m":108,"g":115},"ce67b2d5":{"m":108,"g":115},"3c2c9f6c":{"m":108,"g":115},"a31ea448":{"m":108,"g":115},"439df454":{"m":108,"g":115},"5626e20b":{"m":108,"g":115},"c6c379ab":{"m":108,"g":115},"c2fbf60f":{"m":108,"g":115},"98b44e9e":{"m":108,"g":115},"6805f6da":{"m":108,"g":115},"ca533580":{"m":108,"g":115},"886454e8":{"m":108,"g":115},"0cf3fbeb":{"m":108,"g":115},"2256d62d":{"m":108,"g":115},"6cdcbcc6":{"m":108,"g":115},"c480a3f6":{"m":108,"g":115},"6e316588":{"m":108,"g":115},"24247b41":{"m":108,"g":115},"4c0bb411":{"m":108,"g":115},"968e1818":{"m":108,"g":115},"d08663ee":{"m":108,"g":115},"716e6827":{"m":108,"g":115},"84b30d9e":{"m":108,"g":115},"ff0cf51c":{"m":108,"g":115},"a1c7f742":{"m":108,"g":115},"ebbb75e9":{"m":108,"g":115},"b341b7db":{"m":108,"g":115},"b498cd21":{"m":108,"g":115},"0fc54b97":{"m":108,"g":115},"b3c1f2e4":{"m":108,"g":115},"be1a3cd9":{"m":108,"g":115},"4b74c3fc":{"m":108,"g":115},"ce3ca9b0":{"m":108,"g":115},"4d98e486":{"m":108,"g":115},"3d77a318":{"m":108,"g":115},"845d12a9":{"m":108,"g":115},"e47800e1":{"m":108,"g":115},"bb10e3a1":{"m":108,"g":115},"fda762a2":{"m":108,"g":115},"1df84ff4":{"m":108,"g":115},"66d6be08":{"m":108,"g":115},"1c1f8a11":{"m":108,"g":115},"384f8ab5":{"m":108,"g":115},"6a9d6ca3":{"m":108,"g":115},"94371dbb":{"m":108,"g":115},"740f0630":{"m":108,"g":115},"81da16f6":{"m":108,"g":115},"bc938ea1":{"m":108,"g":115},"eff4eb3f":{"m":108,"g":115},"87dab548":{"m":108,"g":115},"5121af46":{"m":108,"g":115},"983aa496":{"m":108,"g":115},"9c3e95d9":{"m":108,"g":115},"e52c3866":{"m":108,"g":115},"da53e13c":{"m":108,"g":115},"d7e38b2f":{"m":108,"g":115},"21b88460":{"m":108,"g":115},"0c8594e6":{"m":108,"g":115},"c186feed":{"m":108,"g":115},"84b006b2":{"m":108,"g":115},"8ca07bd9":{"m":108,"g":115},"4fc09e0d":{"m":108,"g":115},"a3d99d6d":{"m":108,"g":115},"189af908":{"m":108,"g":115},"f8644a56":{"m":108,"g":115},"e3e75a78":{"m":108,"g":115},"d4db9b02":{"m":108,"g":115},"f7dd651d":{"m":108,"g":115},"9d54c6e6":{"m":108,"g":115},"1f9d65f5":{"m":108,"g":115},"29589512":{"m":108,"g":115},"584e1ab2":{"m":108,"g":115},"392de007":{"m":108,"g":115},"004f7f19":{"m":108,"g":115},"d2fbf2de":{"m":108,"g":115},"fab0f6e7":{"m":108,"g":115},"27985c27":{"m":108,"g":115},"ac474869":{"m":108,"g":115},"0b1e04f0":{"m":108,"g":115},"c1c7dc45":{"m":108,"g":115},"2cc9eeab":{"m":108,"g":115},"63d82a77":{"m":108,"g":115},"53dcc750":{"m":108,"g":115},"432f2053":{"m":108,"g":115},"1fea998a":{"m":108,"g":115},"5aa1ebd2":{"m":108,"g":115},"4dbf4360":{"m":108,"g":115},"3d6be1fb":{"m":108,"g":115},"4063234c":{"m":108,"g":115},"83feef5b":{"m":108,"g":115},"2871eacc":{"m":108,"g":115},"ac15bdc1":{"m":108,"g":115},"d6451c3f":{"m":108,"g":115},"841810f2":{"m":108,"g":115},"733446dd":{"m":108,"g":115},"4c22897a":{"m":108,"g":115},"1bc183c6":{"m":108,"g":115},"b87aacb5":{"m":108,"g":115},"b3363cc1":{"m":108,"g":115},"98457c04":{"m":108,"g":115},"0fc8bf2c":{"m":108,"g":115},"a669bc2f":{"m":108,"g":115},"6b7c2471":{"m":108,"g":115},"a027a9b4":{"m":108,"g":115},"9e426466":{"m":108,"g":115},"2f20f430":{"m":108,"g":115},"65736dc5":{"m":108,"g":115},"7b56e494":{"m":108,"g":115},"0ff6d1fc":{"m":108,"g":115},"4a16a71c":{"m":108,"g":115},"a16923ef":{"m":108,"g":115},"6337d905":{"m":108,"g":115},"71fb8c95":{"m":108,"g":115},"94f44b88":{"m":108,"g":115},"3b3b3baf":{"m":108,"g":115},"35e6bc92":{"m":108,"g":115},"9394ed63":{"m":108,"g":115},"930fe467":{"m":108,"g":115},"13c48dcf":{"m":108,"g":115},"8723b4f1":{"m":108,"g":115},"62f99e08":{"m":108,"g":115},"86a0be65":{"m":108,"g":115},"0edda320":{"m":108,"g":115},"924827c3":{"m":108,"g":115},"c81daf83":{"m":108,"g":115},"25caa7a8":{"m":108,"g":115},"03d11449":{"m":108,"g":115},"83123f48":{"m":108,"g":115},"48afa8f1":{"m":108,"g":115},"2ecbd8b8":{"m":108,"g":115},"305b27c1":{"m":108,"g":115},"1ce30dd1":{"m":108,"g":115},"c9ee7385":{"m":108,"g":115},"1f9ec653":{"m":108,"g":115},"ad359d1c":{"m":108,"g":115},"5f5b3b24":{"m":108,"g":115},"4caca4f6":{"m":108,"g":115},"f2a5de28":{"m":108,"g":115},"445f9dca":{"m":108,"g":115},"3a9afe2a":{"m":108,"g":115},"9aea2555":{"m":108,"g":115},"fcc11e5e":{"m":108,"g":115},"5190ba7f":{"m":108,"g":115},"5438886c":{"m":108,"g":115},"9c83d74d":{"m":108,"g":115},"b4ac2b9c":{"m":108,"g":115},"83262dcb":{"m":108,"g":115},"c46c75f8":{"m":108,"g":115},"2aaf22c4":{"m":108,"g":115},"29a610b4":{"m":108,"g":115},"5ded39ca":{"m":108,"g":115},"4093d460":{"m":108,"g":115},"9d68bdb2":{"m":108,"g":115},"a2184901":{"m":108,"g":115},"0eec4cb6":{"m":108,"g":115},"ff1f6825":{"m":108,"g":115},"9f78f391":{"m":108,"g":115},"f508cd3c":{"m":108,"g":115},"44e86480":{"m":108,"g":115},"8c07fabd":{"m":108,"g":115},"90f44b74":{"m":108,"g":115},"38907fe6":{"m":108,"g":115},"f9afa7dc":{"m":108,"g":115},"0d9e89ec":{"m":108,"g":115},"3d64fda3":{"m":108,"g":115},"3bffe112":{"m":108,"g":115},"44426e54":{"m":108,"g":115},"9f24dfef":{"m":108,"g":115},"89f1d4f5":{"m":108,"g":115},"75e6a7cd":{"m":108,"g":115},"6f81a710":{"m":108,"g":115},"a6452b71":{"m":108,"g":115},"f4ae50e9":{"m":108,"g":115},"84cb449e":{"m":108,"g":115},"f003cd35":{"m":108,"g":115},"9d834fdc":{"m":108,"g":115},"b3279251":{"m":108,"g":115},"067068f2":{"m":108,"g":115},"6beeff41":{"m":108,"g":115},"2e8e7e35":{"m":108,"g":115},"2449a0af":{"m":108,"g":115},"0f229c07":{"m":108,"g":115},"dd001a54":{"m":108,"g":115},"4ea9d74a":{"m":108,"g":115},"dd949ace":{"m":108,"g":115},"f2887498":{"m":108,"g":115},"8ecf6b9d":{"m":108,"g":115},"0418b9d4":{"m":108,"g":115},"e322a94d":{"m":108,"g":115},"2c7f01bc":{"m":108,"g":115},"b58ae7a2":{"m":108,"g":115},"6345069f":{"m":108,"g":115},"ce9cf353":{"m":108,"g":115},"f8a173bb":{"m":108,"g":115},"6b847a9a":{"m":108,"g":115},"473400e4":{"m":108,"g":115},"dd665f96":{"m":108,"g":115},"3817a37d":{"m":108,"g":115},"7ba5ad57":{"m":108,"g":115},"19bc77f0":{"m":108,"g":115},"86497d99":{"m":108,"g":115},"5c31b35d":{"m":108,"g":115},"ef48d554":{"m":108,"g":115},"a886564a":{"m":108,"g":115},"9a44b643":{"m":108,"g":115},"41d71ca4":{"m":108,"g":115},"20cfc5a2":{"m":108,"g":115},"48b8b4c1":{"m":108,"g":115},"323bc2f5":{"m":108,"g":115},"137e75da":{"m":108,"g":115},"52e1f52f":{"m":108,"g":115},"50188092":{"m":108,"g":115},"326a901d":{"m":108,"g":115},"6e0b6468":{"m":108,"g":115},"4a9f3eef":{"m":108,"g":115},"1b7afad0":{"m":108,"g":115},"f29aba8c":{"m":108,"g":115},"faa25df1":{"m":108,"g":115},"7b81f956":{"m":108,"g":115},"d3e67deb":{"m":108,"g":115},"442534aa":{"m":108,"g":115},"de8b8b6e":{"m":108,"g":115},"3f2e315f":{"m":108,"g":115},"6e215118":{"m":108,"g":115},"a47baff1":{"m":108,"g":115},"fd7e15b7":{"m":108,"g":115},"fc42ff7b":{"m":108,"g":115},"7c0db868":{"m":108,"g":115},"706bd69c":{"m":108,"g":115},"23f2afb2":{"m":108,"g":115},"a60f88b5":{"m":108,"g":115},"591c232f":{"m":108,"g":115},"f352b793":{"m":108,"g":115},"6642e3a2":{"m":108,"g":115},"67a7d1f6":{"m":108,"g":115},"92cbef59":{"m":108,"g":115},"b3359dc9":{"m":108,"g":115},"7b7e5615":{"m":108,"g":115},"1a8706c8":{"m":108,"g":115},"7d3af603":{"m":108,"g":115},"4e7f0252":{"m":108,"g":115},"36bfddec":{"m":108,"g":115},"91e2f902":{"m":108,"g":115},"a59cbea9":{"m":108,"g":115},"53f7874a":{"m":108,"g":115},"61a46804":{"m":108,"g":115},"9020f7fc":{"m":108,"g":115},"dd650e0e":{"m":108,"g":115},"a9471542":{"m":108,"g":115},"41357e51":{"m":108,"g":115},"e2fd2b9c":{"m":108,"g":115},"7490e3f6":{"m":108,"g":115},"6ee6619b":{"m":108,"g":115},"54ea57f2":{"m":108,"g":115},"b4c9f38a":{"m":108,"g":115},"11325474":{"m":108,"g":115},"1d24db83":{"m":108,"g":115},"44401358":{"m":108,"g":115},"9c7e3924":{"m":108,"g":115},"08fab2b0":{"m":108,"g":115},"0d1e27a0":{"m":108,"g":115},"774b47f3":{"m":108,"g":115},"76915d68":{"m":108,"g":115},"39fd1788":{"m":108,"g":115},"ed0a3dd5":{"m":108,"g":115},"2e901e89":{"m":108,"g":115},"d3be9710":{"m":108,"g":115},"3e7ff1ab":{"m":108,"g":115},"aaf0ad8c":{"m":108,"g":115},"361379b5":{"m":108,"g":115},"1ac16add":{"m":108,"g":115},"c3a5fb3b":{"m":108,"g":115},"4bf6e5a6":{"m":108,"g":115},"3ae33fcd":{"m":108,"g":115},"500b15c9":{"m":108,"g":107},"16a4c66d":{"m":108,"g":107},"89e6521c":{"m":108,"g":107},"fd05b567":{"m":108,"g":107},"482c3db2":{"m":108,"g":107},"47824c14":{"m":108,"g":107},"c36a6693":{"m":108,"g":107},"62f8eb48":{"m":108,"g":107},"b7cd7430":{"m":108,"g":107},"a69b6370":{"m":108,"g":107},"2d120f8b":{"m":108,"g":107},"4f2e1490":{"m":108,"g":107},"3fa3c6cd":{"m":108,"g":107},"6210e2c4":{"m":108,"g":107},"6ad6c8c9":{"m":108,"g":107},"5b6acc14":{"m":108,"g":107},"4373df55":{"m":108,"g":107},"c0e84297":{"m":108,"g":107},"92cc32d9":{"m":108,"g":107},"cbbd685a":{"m":108,"g":107},"78aad910":{"m":108,"g":107},"288ae41f":{"m":108,"g":107},"01c99a99":{"m":108,"g":107},"b114a810":{"m":108,"g":107},"0475448e":{"m":108,"g":107},"399e7ec8":{"m":108,"g":107},"1bd53168":{"m":108,"g":107},"aeac900c":{"m":108,"g":107},"4fc5f2f9":{"m":108,"g":107},"168033d5":{"m":108,"g":107},"cbbb7383":{"m":108,"g":107},"89588179":{"m":108,"g":107},"8c7bb39d":{"m":108,"g":107},"ca47e24f":{"m":108,"g":107},"d26ca84f":{"m":108,"g":107},"8128e08d":{"m":108,"g":107},"5d62b56f":{"m":108,"g":107},"3ae8e3ea":{"m":108,"g":107},"c1d2061f":{"m":108,"g":107},"556e4143":{"m":108,"g":107},"4ef47839":{"m":108,"g":107},"32d9e39a":{"m":108,"g":107},"4f4e0e41":{"m":108,"g":107},"901ab758":{"m":108,"g":107},"8e8545ca":{"m":108,"g":107},"a4b0d5c9":{"m":108,"g":107},"40e3b2be":{"m":108,"g":107},"75df31b6":{"m":108,"g":107},"194561f2":{"m":108,"g":107},"5e91fed1":{"m":108,"g":107},"873f384a":{"m":108,"g":107},"b01eeb80":{"m":108,"g":107},"1ea94d3b":{"m":108,"g":107},"354ac435":{"m":108,"g":107},"d98a4913":{"m":108,"g":107},"08f8f490":{"m":108,"g":107},"d4bf5a85":{"m":108,"g":107},"7cb20754":{"m":108,"g":107},"6d0646da":{"m":108,"g":107},"02bc1c7d":{"m":108,"g":107},"fc8c8e50":{"m":108,"g":107},"9bd4872a":{"m":108,"g":107},"2fa0462c":{"m":108,"g":107},"915140fd":{"m":108,"g":107},"36fc9260":{"m":108,"g":107},"fee0ab0f":{"m":108,"g":107},"f57d2dc1":{"m":108,"g":107},"f2d68ded":{"m":108,"g":107},"3b87a9e8":{"m":108,"g":107},"f024795e":{"m":108,"g":107},"b102353f":{"m":108,"g":107},"7a27e798":{"m":108,"g":107},"76ba5bbe":{"m":108,"g":107},"ed6f7597":{"m":108,"g":107},"e67276ec":{"m":108,"g":107},"0242bb9c":{"m":108,"g":107},"760286e3":{"m":108,"g":107},"3435a24e":{"m":108,"g":107},"00da9065":{"m":108,"g":107},"e0ab167d":{"m":109,"g":115},"c807cd7c":{"m":109,"g":115},"327f7b7c":{"m":109,"g":115},"80425e59":{"m":109,"g":115},"af9d4eb0":{"m":109,"g":115},"fb107cfd":{"m":109,"g":115},"e3e97a12":{"m":110,"g":115},"05106867":{"m":110,"g":115},"9dcdf5da":{"m":110,"g":115},"f8b757bc":{"m":110,"g":115},"ebd9dbe7":{"m":110,"g":115},"938e986e":{"m":110,"g":115},"17d5eda8":{"m":110,"g":115},"71a7f1d8":{"m":110,"g":115},"433266c1":{"m":110,"g":115},"fda47926":{"m":110,"g":115},"a0b22f2f":{"m":110,"g":115},"b5c6529e":{"m":110,"g":115},"ca4b86c5":{"m":110,"g":115},"dd6ec029":{"m":110,"g":115},"bf863e3b":{"m":110,"g":115},"9e169ea8":{"m":110,"g":115},"bc80dc4c":{"m":111,"g":115},"b962a296":{"m":111,"g":115},"aa3eba8e":{"m":111,"g":115},"07ee0ab7":{"m":111,"g":115},"5c06dcb7":{"m":111,"g":115},"6f6beca4":{"m":111,"g":115},"68a54e06":{"m":111,"g":115},"fd18995c":{"m":111,"g":115},"db0831e0":{"m":111,"g":115},"6e4e1c8c":{"m":111,"g":115},"9768c50d":{"m":111,"g":115},"fd71b11b":{"m":111,"g":115},"ae7428a8":{"m":111,"g":115},"a3aee7c3":{"m":111,"g":115},"79e6a8a6":{"m":111,"g":115},"8f7b1c31":{"m":111,"g":115},"b9683be6":{"m":111,"g":115},"a85363c1":{"m":111,"g":115},"b21fdd53":{"m":111,"g":115},"c04c17ed":{"m":111,"g":115},"16a6d21b":{"m":111,"g":115},"a530b3ff":{"m":111,"g":115},"603b3446":{"m":111,"g":115},"b6c14ec0":{"m":111,"g":115},"43de1d73":{"m":111,"g":115},"79ce3688":{"m":111,"g":115},"44ffe2cb":{"m":111,"g":115},"1a0896e9":{"m":111,"g":115},"90313fb0":{"m":111,"g":115},"3578eb1e":{"m":111,"g":115},"0936c766":{"m":111,"g":115},"0ef583b7":{"m":111,"g":115},"f7881a27":{"m":111,"g":115},"fdff3167":{"m":111,"g":115},"cbc0e4d7":{"m":111,"g":115},"4cd08dc5":{"m":111,"g":115},"f92b729d":{"m":111,"g":115},"e2e378ca":{"m":111,"g":115},"dc1decc6":{"m":111,"g":115},"03680f33":{"m":111,"g":115},"d4c5e534":{"m":111,"g":115},"817c62a0":{"m":111,"g":115},"0ff72419":{"m":111,"g":115},"80dc76e1":{"m":111,"g":115},"9b08d975":{"m":111,"g":115},"a0a77d93":{"m":111,"g":115},"24a8cee6":{"m":111,"g":115},"3affa9dc":{"m":111,"g":115},"ea0696b9":{"m":111,"g":115},"3aec3d4f":{"m":111,"g":115},"b0d25e72":{"m":112,"g":115},"a2424068":{"m":112,"g":115},"c5d2b01c":{"m":112,"g":115},"46ccbed2":{"m":112,"g":115},"fe68c148":{"m":112,"g":115},"70c0c1f9":{"m":112,"g":115},"760b788a":{"m":112,"g":115},"1ee11df8":{"m":112,"g":115},"dee197e1":{"m":112,"g":115},"ab795ae8":{"m":112,"g":115},"480d1b8b":{"m":112,"g":115},"6c18ab46":{"m":112,"g":115},"4a0e0be2":{"m":112,"g":115},"64f296f8":{"m":112,"g":115},"956d805d":{"m":112,"g":115},"30c6e1f5":{"m":112,"g":115},"bfe01a5e":{"m":112,"g":115},"3dd6420a":{"m":112,"g":115},"532f998b":{"m":112,"g":115},"de15d140":{"m":112,"g":115},"37367da6":{"m":112,"g":115},"ef959d7b":{"m":112,"g":115},"4aa1e69b":{"m":112,"g":115},"dc491b39":{"m":112,"g":115},"5b64f006":{"m":112,"g":115},"5b7448de":{"m":112,"g":115},"6d55f60e":{"m":112,"g":115},"033b75f5":{"m":112,"g":115},"f3b5db6e":{"m":112,"g":115},"2286e85e":{"m":112,"g":115},"91b3555d":{"m":112,"g":115},"9e2f7252":{"m":112,"g":115},"21176b00":{"m":112,"g":115},"94100294":{"m":112,"g":115},"cda7e47c":{"m":112,"g":115},"e903f695":{"m":112,"g":115},"27760fc1":{"m":112,"g":115},"0ac809de":{"m":112,"g":115},"4efe2c57":{"m":112,"g":115},"5be8c2f7":{"m":112,"g":115},"737d73ed":{"m":112,"g":115},"ebd0e1c1":{"m":112,"g":115},"a1d03892":{"m":112,"g":115},"dccf52f9":{"m":112,"g":115},"676a7b51":{"m":112,"g":115},"15f99347":{"m":112,"g":115},"bcf1955f":{"m":112,"g":115},"a06bf664":{"m":112,"g":115},"bf72b801":{"m":112,"g":115},"8cbe1538":{"m":112,"g":115},"8471e5e6":{"m":112,"g":115},"4582931a":{"m":112,"g":115},"d352c29a":{"m":112,"g":115},"d3ee7098":{"m":112,"g":115},"71fc7b7f":{"m":112,"g":115},"9ab72f98":{"m":112,"g":115},"f3817cb0":{"m":112,"g":115},"71133a04":{"m":112,"g":115},"2cd94dd0":{"m":112,"g":115},"f5f6b3b4":{"m":112,"g":115},"94fb4e9e":{"m":112,"g":115},"d1d4074c":{"m":112,"g":115},"718f25ae":{"m":112,"g":115},"948b01a0":{"m":112,"g":115},"cdc56ef6":{"m":112,"g":115},"16ff3d4b":{"m":112,"g":115},"83d55ac5":{"m":112,"g":115},"2fe17735":{"m":112,"g":115},"97fff98c":{"m":112,"g":115},"ba066ca0":{"m":112,"g":115},"96784a65":{"m":112,"g":115},"df5407fb":{"m":112,"g":115},"8ad700f7":{"m":112,"g":115},"148022fc":{"m":112,"g":115},"7a40e4f4":{"m":112,"g":115},"19d64f2b":{"m":112,"g":115},"a02071a1":{"m":112,"g":115},"45b3a6a2":{"m":112,"g":115},"9a18aa54":{"m":112,"g":115},"91f0fd95":{"m":112,"g":115},"8085aca7":{"m":112,"g":115},"0096798e":{"m":112,"g":115},"2c2b19b1":{"m":112,"g":115},"72f9fc5f":{"m":112,"g":115},"ec99668a":{"m":112,"g":115},"78f13981":{"m":112,"g":115},"bfd7a18d":{"m":112,"g":115},"5dd8c644":{"m":112,"g":115},"ee21817c":{"m":112,"g":115},"b7d1f17b":{"m":112,"g":115},"c8295d23":{"m":112,"g":115},"b67c277f":{"m":112,"g":115},"8116804e":{"m":112,"g":115},"8c5930f0":{"m":112,"g":115},"3b99f23c":{"m":112,"g":115},"ee0b3c5b":{"m":112,"g":115},"6049ca20":{"m":112,"g":115},"7577f0e4":{"m":112,"g":115},"8cda5a62":{"m":112,"g":115},"400d3b97":{"m":112,"g":115},"37d83c6e":{"m":112,"g":115},"7802586c":{"m":112,"g":115},"bc5fc332":{"m":112,"g":115},"f3440adc":{"m":112,"g":115},"5a7e10fe":{"m":112,"g":115},"33467c05":{"m":112,"g":115},"b0fcbb74":{"m":112,"g":115},"76a2c86b":{"m":112,"g":115},"e719bb0e":{"m":112,"g":115},"06724683":{"m":112,"g":115},"617aa2b2":{"m":112,"g":115},"111b1379":{"m":112,"g":115},"41628dc1":{"m":112,"g":115},"a12061df":{"m":112,"g":115},"85ed8e0a":{"m":112,"g":115},"dd1e2689":{"m":112,"g":115},"9a7ced4e":{"m":112,"g":115},"cb3918a0":{"m":112,"g":115},"f3b67602":{"m":112,"g":115},"9eb50ecc":{"m":112,"g":115},"b3e7a2ce":{"m":112,"g":115},"00974e4f":{"m":112,"g":115},"5f1eb204":{"m":112,"g":115},"039cef76":{"m":112,"g":115},"4c22ebe2":{"m":112,"g":115},"a5a03209":{"m":112,"g":115},"21af5c04":{"m":112,"g":115},"012584ec":{"m":112,"g":115},"90dfe3de":{"m":112,"g":115},"9a719b7a":{"m":112,"g":115},"3fa62da7":{"m":112,"g":115},"dbb1235d":{"m":112,"g":115},"ad26f298":{"m":112,"g":115},"8d114f25":{"m":112,"g":115},"0e78c63c":{"m":112,"g":115},"1a3d6f31":{"m":112,"g":115},"0b8c5721":{"m":112,"g":115},"beac202b":{"m":112,"g":115},"21b9a4b4":{"m":112,"g":115},"db37422c":{"m":112,"g":115},"ab62b135":{"m":112,"g":115},"273b2834":{"m":112,"g":115},"f84db115":{"m":112,"g":115},"efb0de2c":{"m":112,"g":115},"0f6ac5e2":{"m":112,"g":115},"29850900":{"m":112,"g":115},"e678cc71":{"m":112,"g":115},"4efe844a":{"m":112,"g":115},"bde73ee4":{"m":112,"g":115},"4f0e28d7":{"m":112,"g":115},"045ab92d":{"m":112,"g":115},"bd7f8821":{"m":112,"g":115},"5e5c30d9":{"m":112,"g":115},"9f00ec44":{"m":112,"g":115},"8e85ee88":{"m":112,"g":115},"adf73175":{"m":112,"g":115},"13705dae":{"m":112,"g":115},"df97b31f":{"m":112,"g":115},"339f8eef":{"m":112,"g":115},"afd9f2f5":{"m":112,"g":115},"f40038fb":{"m":112,"g":115},"bebd0576":{"m":112,"g":115},"f9836660":{"m":112,"g":115},"8b3b995a":{"m":112,"g":115},"6e95f5e5":{"m":112,"g":115},"0e9387a9":{"m":112,"g":115},"fa9c82d3":{"m":112,"g":115},"918e3d4c":{"m":112,"g":115},"e9697374":{"m":112,"g":115},"93088b69":{"m":112,"g":115},"453511ac":{"m":112,"g":115},"d0730487":{"m":112,"g":115},"b32ab070":{"m":112,"g":115},"75ee0011":{"m":112,"g":115},"ec15c836":{"m":112,"g":115},"106c2b31":{"m":112,"g":115},"c6756949":{"m":112,"g":115},"27e8ffed":{"m":112,"g":115},"4dbb34fe":{"m":112,"g":115},"1e18a341":{"m":112,"g":115},"2c562fd2":{"m":112,"g":115},"b648d862":{"m":112,"g":115},"bbf261ae":{"m":112,"g":115},"4f8a982d":{"m":112,"g":115},"d966b902":{"m":112,"g":115},"de921733":{"m":112,"g":115},"397448eb":{"m":112,"g":115},"66d5d042":{"m":112,"g":115},"73179b76":{"m":112,"g":115},"8cbf71dc":{"m":112,"g":115},"56eb5d0a":{"m":112,"g":115},"4ed9053e":{"m":112,"g":115},"5e19b159":{"m":112,"g":115},"788b19a5":{"m":112,"g":115},"f78b7fd1":{"m":112,"g":115},"b1fb7e45":{"m":112,"g":115},"1b2ff4fb":{"m":112,"g":115},"2c7ca33a":{"m":112,"g":115},"df397a72":{"m":112,"g":115},"5dfcd6c2":{"m":112,"g":115},"0dfd54d1":{"m":112,"g":115},"bcbeed71":{"m":112,"g":115},"cc9a31c6":{"m":112,"g":115},"d631290e":{"m":112,"g":115},"37565b7f":{"m":112,"g":115},"6243c367":{"m":112,"g":115},"60e37f80":{"m":112,"g":115},"369b1433":{"m":112,"g":115},"03dbf1aa":{"m":112,"g":115},"11dcabc5":{"m":112,"g":115},"4d89389c":{"m":112,"g":115},"9491d6e5":{"m":112,"g":115},"f64b8e3e":{"m":112,"g":115},"53976fce":{"m":112,"g":115},"18f91eb6":{"m":112,"g":115},"8766b3ac":{"m":112,"g":115},"1db649ac":{"m":112,"g":115},"a1e5d781":{"m":112,"g":115},"b7361cc4":{"m":112,"g":115},"a96c5b5c":{"m":112,"g":115},"b9eb0d9c":{"m":112,"g":115},"1fbfdebe":{"m":112,"g":115},"a25e8e42":{"m":112,"g":115},"d4a93841":{"m":112,"g":115},"21e1bc47":{"m":112,"g":115},"9a0cac1b":{"m":112,"g":115},"b5245064":{"m":112,"g":115},"9d9fa9a5":{"m":112,"g":115},"58d06fdc":{"m":112,"g":115},"cb9e0e41":{"m":112,"g":115},"9db80253":{"m":112,"g":115},"598c0bc1":{"m":112,"g":115},"b361750a":{"m":112,"g":115},"16e56ea6":{"m":112,"g":115},"349b491c":{"m":112,"g":115},"5f77e129":{"m":112,"g":115},"4750cddf":{"m":112,"g":115},"065e523d":{"m":112,"g":115},"7de2ce45":{"m":112,"g":115},"8c2ffaaf":{"m":112,"g":115},"20445327":{"m":112,"g":115},"6d3c20cf":{"m":112,"g":115},"8b6966d0":{"m":112,"g":115},"a391f73a":{"m":112,"g":115},"25c73959":{"m":112,"g":115},"f05c6873":{"m":112,"g":115},"9a0d0b75":{"m":112,"g":115},"ba861293":{"m":112,"g":115},"c112bcc4":{"m":112,"g":115},"5e194b21":{"m":112,"g":115},"fd5ce576":{"m":112,"g":115},"92d79646":{"m":112,"g":115},"f9076a5a":{"m":112,"g":115},"646076b7":{"m":112,"g":115},"0d040089":{"m":112,"g":115},"05e47872":{"m":112,"g":115},"1e61b496":{"m":112,"g":115},"300676af":{"m":112,"g":115},"7fe89f7c":{"m":112,"g":115},"9970e3bf":{"m":112,"g":115},"70eedb58":{"m":112,"g":115},"9c99949e":{"m":112,"g":115},"c5082f0f":{"m":112,"g":115},"836873b9":{"m":112,"g":115},"8abe8dea":{"m":112,"g":115},"1e85589d":{"m":112,"g":115},"c2a26e72":{"m":112,"g":115},"591e6c59":{"m":112,"g":115},"42f34437":{"m":112,"g":115},"5c34b4f1":{"m":112,"g":115},"ff9b5618":{"m":112,"g":115},"fcd72bd1":{"m":112,"g":115},"3d8fc434":{"m":112,"g":115},"87a0f7d2":{"m":112,"g":115},"839c93bd":{"m":112,"g":115},"f1e9bbaf":{"m":112,"g":115},"3fd1431d":{"m":112,"g":115},"161e9dc5":{"m":112,"g":115},"54e872d3":{"m":112,"g":115},"e5b29bf1":{"m":112,"g":115},"9a7c8842":{"m":112,"g":115},"7a16db9b":{"m":112,"g":115},"09a1df22":{"m":112,"g":115},"4b7034dd":{"m":112,"g":115},"a23c3020":{"m":112,"g":115},"a7d825fc":{"m":112,"g":115},"38cd5fb1":{"m":112,"g":115},"001f5194":{"m":112,"g":115},"5ad296bd":{"m":112,"g":115},"9f81d741":{"m":112,"g":115},"a38c1497":{"m":112,"g":115},"74dd4249":{"m":112,"g":115},"dc20c22f":{"m":112,"g":115},"711390a9":{"m":112,"g":115},"53430588":{"m":112,"g":115},"fce7ae33":{"m":112,"g":115},"6b39f9cf":{"m":112,"g":115},"07c9d8fb":{"m":112,"g":115},"4a4772ae":{"m":112,"g":115},"c3779233":{"m":112,"g":115},"f84b57c8":{"m":112,"g":115},"aee094e4":{"m":112,"g":115},"55349e36":{"m":112,"g":115},"e1f7cf57":{"m":112,"g":115},"2bb9d454":{"m":112,"g":115},"d0934a51":{"m":112,"g":115},"3f2d0cef":{"m":112,"g":115},"8b30bec2":{"m":112,"g":115},"4aeba40d":{"m":112,"g":115},"28684f90":{"m":112,"g":115},"a4a3d823":{"m":113,"g":115},"0b13cbb7":{"m":113,"g":115},"efbc687c":{"m":113,"g":115},"292a867a":{"m":113,"g":115},"8fd41eae":{"m":113,"g":115},"0cd1996e":{"m":113,"g":115},"b6b4b563":{"m":113,"g":115},"f8924ad7":{"m":113,"g":115},"2f80bd9f":{"m":113,"g":115},"366a603e":{"m":113,"g":115},"baee0860":{"m":113,"g":115},"c7a104c1":{"m":113,"g":115},"97d966a7":{"m":113,"g":115},"8e66d87f":{"m":113,"g":115},"a20fc7b7":{"m":113,"g":115},"6b30e097":{"m":113,"g":115},"d645ae90":{"m":113,"g":115},"41763ba0":{"m":113,"g":115},"652c24a6":{"m":113,"g":115},"5e142484":{"m":113,"g":115},"c560410d":{"m":113,"g":115},"590f2da0":{"m":113,"g":115},"148d8d48":{"m":113,"g":115},"1a599509":{"m":113,"g":115},"36a6b8db":{"m":113,"g":115},"e0b2d3ee":{"m":113,"g":115},"4cb5a523":{"m":113,"g":115},"85c1f793":{"m":113,"g":115},"48e9e719":{"m":113,"g":115},"31b49c0b":{"m":113,"g":115},"d736e0b6":{"m":113,"g":115},"ffd03a9b":{"m":113,"g":115},"666da3d5":{"m":113,"g":115},"d01b9214":{"m":113,"g":115},"c70e58e8":{"m":113,"g":115},"c61b9a1d":{"m":113,"g":115},"3c3d6255":{"m":113,"g":115},"546914fa":{"m":113,"g":115},"4726c919":{"m":113,"g":115},"a0010bf4":{"m":113,"g":115},"307fc060":{"m":113,"g":115},"586e81a2":{"m":113,"g":115},"fad7ca73":{"m":113,"g":115},"08af8ffb":{"m":113,"g":115},"2c7f4ca2":{"m":113,"g":115},"03def5e3":{"m":113,"g":115},"6ae3f05b":{"m":113,"g":115},"fdc4e1e5":{"m":113,"g":115},"04b86b3c":{"m":113,"g":115},"d6777a70":{"m":113,"g":115},"8c574902":{"m":113,"g":115},"34151f17":{"m":113,"g":115},"6794d210":{"m":113,"g":115},"1a31229c":{"m":113,"g":115},"de89ef49":{"m":113,"g":115},"b00a0c78":{"m":113,"g":115},"a2faf894":{"m":113,"g":115},"7e61737d":{"m":113,"g":115},"3c699772":{"m":113,"g":115},"e8100774":{"m":113,"g":115},"963175d5":{"m":113,"g":115},"0618ad6d":{"m":113,"g":115},"6a261aac":{"m":113,"g":115},"7ff740a6":{"m":113,"g":115},"bfcd9b24":{"m":113,"g":115},"458611de":{"m":113,"g":115},"3511b370":{"m":113,"g":115},"afcd3e10":{"m":113,"g":115},"12d68183":{"m":113,"g":115},"b65db028":{"m":113,"g":115},"948278f1":{"m":113,"g":115},"7d004799":{"m":113,"g":115},"083629c2":{"m":113,"g":115},"b658be6f":{"m":113,"g":115},"5e786cca":{"m":113,"g":115},"0b9dfba7":{"m":113,"g":115},"6a290034":{"m":113,"g":115},"2ac453b0":{"m":113,"g":115},"f35def86":{"m":113,"g":115},"d61615fe":{"m":113,"g":115},"b1ccaf01":{"m":113,"g":115},"097725bb":{"m":113,"g":115},"44b1fbe2":{"m":113,"g":115},"c0dbbdd1":{"m":113,"g":115},"25e7dbe8":{"m":113,"g":115},"0b2aa8a7":{"m":113,"g":115},"609f65ba":{"m":113,"g":115},"2d62af6b":{"m":113,"g":115},"a28b394f":{"m":113,"g":115},"96fe2d0f":{"m":113,"g":115},"bfa27438":{"m":113,"g":115},"86cb4db0":{"m":113,"g":115},"2e130b76":{"m":113,"g":115},"ac1f2928":{"m":113,"g":115},"195a59fe":{"m":113,"g":115},"47488cc3":{"m":113,"g":115},"61305291":{"m":113,"g":115},"a9ce2bcb":{"m":113,"g":115},"5dddb331":{"m":113,"g":115},"01a26544":{"m":113,"g":115},"73d4a5f8":{"m":113,"g":115},"7fb551a7":{"m":113,"g":115},"1193f131":{"m":113,"g":115},"84a9f5d6":{"m":113,"g":115},"8ce830a8":{"m":113,"g":115},"fb367acf":{"m":113,"g":115},"a6cc86df":{"m":113,"g":115},"229d2b95":{"m":113,"g":115},"9710f718":{"m":113,"g":115},"91847e38":{"m":113,"g":115},"5a290a56":{"m":113,"g":115},"580051c5":{"m":113,"g":115},"1237aa19":{"m":113,"g":115},"59911195":{"m":113,"g":115},"424591d5":{"m":113,"g":115},"d1676cd4":{"m":113,"g":115},"33b3c0f8":{"m":113,"g":115},"e5281f84":{"m":113,"g":115},"d17986f8":{"m":113,"g":115},"8831c55c":{"m":113,"g":115},"2bc61dd1":{"m":113,"g":115},"6535fda1":{"m":113,"g":115},"3713eb61":{"m":113,"g":115},"5937a56d":{"m":113,"g":115},"f065e5be":{"m":113,"g":115},"9de1320b":{"m":113,"g":115},"dda34c2f":{"m":113,"g":115},"4eeaff74":{"m":113,"g":115},"a17e70f5":{"m":113,"g":115},"816b3a43":{"m":113,"g":115},"3a641d90":{"m":113,"g":115},"6f16bf9d":{"m":113,"g":115},"5942fdb4":{"m":113,"g":115},"af4ab656":{"m":113,"g":115},"11965b0d":{"m":113,"g":115},"71959545":{"m":113,"g":115},"24f7cb1e":{"m":113,"g":115},"e05555fa":{"m":113,"g":115},"43fa9f22":{"m":113,"g":115},"e98d9346":{"m":113,"g":115},"0c917410":{"m":113,"g":115},"25728863":{"m":113,"g":115},"dba751a8":{"m":113,"g":115},"2e763398":{"m":113,"g":115},"336e9a60":{"m":113,"g":115},"abb67815":{"m":113,"g":115},"07440f5f":{"m":113,"g":115},"9816989b":{"m":113,"g":115},"42245551":{"m":113,"g":115},"2a9d995c":{"m":113,"g":115},"a9050b5c":{"m":113,"g":115},"66face35":{"m":113,"g":115},"5519766a":{"m":113,"g":115},"72392f29":{"m":113,"g":115},"c1c8dd1d":{"m":113,"g":115},"f6bc3f52":{"m":113,"g":115},"8cc27fdc":{"m":113,"g":115},"9c339d6b":{"m":113,"g":115},"e23e280e":{"m":113,"g":115},"51f7c6bd":{"m":113,"g":115},"62e2e99d":{"m":113,"g":115},"8ebf72fe":{"m":113,"g":115},"82605747":{"m":113,"g":115},"37f3325b":{"m":113,"g":115},"bd95944c":{"m":113,"g":115},"c8a5d12a":{"m":113,"g":115},"2387c22b":{"m":113,"g":115},"592ddf37":{"m":113,"g":115},"0c3db889":{"m":113,"g":115},"2bdaf482":{"m":113,"g":115},"777eb538":{"m":113,"g":115},"05a35266":{"m":113,"g":115},"e56c64bf":{"m":113,"g":115},"fff7fbab":{"m":113,"g":115},"aae7ead2":{"m":113,"g":115},"a7fe6e10":{"m":113,"g":115},"be059b83":{"m":113,"g":115},"5d4fe1ce":{"m":113,"g":115},"1b011e68":{"m":113,"g":115},"5c0efa56":{"m":113,"g":115},"1e57b947":{"m":113,"g":115},"a5095d62":{"m":113,"g":115},"6c2c467d":{"m":113,"g":115},"c3d2ad4e":{"m":113,"g":115},"7ec5b4e8":{"m":113,"g":115},"60885482":{"m":113,"g":115},"172bcf01":{"m":113,"g":115},"37158f20":{"m":113,"g":115},"3e95aa1a":{"m":113,"g":115},"c4197e99":{"m":113,"g":115},"0ac61146":{"m":113,"g":115},"7dcd689b":{"m":113,"g":115},"f7bab41a":{"m":113,"g":115},"f68dd998":{"m":113,"g":115},"35ec2a45":{"m":113,"g":115},"0035f1ce":{"m":113,"g":115},"5e21d6ae":{"m":113,"g":115},"cd4da1f1":{"m":113,"g":115},"91678474":{"m":113,"g":115},"d511b2d9":{"m":113,"g":115},"77830a26":{"m":113,"g":115},"fce17048":{"m":113,"g":115},"3d40794f":{"m":113,"g":115},"c1f39013":{"m":113,"g":115},"3e43eb13":{"m":113,"g":115},"458c0219":{"m":113,"g":115},"a73eb8cd":{"m":113,"g":115},"e7387035":{"m":113,"g":115},"fe531d6f":{"m":113,"g":115},"c4e314f9":{"m":113,"g":115},"7a06ef98":{"m":113,"g":115},"4a87ba21":{"m":113,"g":115},"d7b20dd6":{"m":113,"g":115},"c3faf2d6":{"m":113,"g":115},"9209b209":{"m":113,"g":115},"adba172f":{"m":113,"g":115},"cd641a99":{"m":113,"g":115},"71f24ef8":{"m":113,"g":115},"b1f0fc1c":{"m":113,"g":115},"32d89373":{"m":113,"g":115},"f47a2c67":{"m":113,"g":115},"ee704e62":{"m":113,"g":115},"f4e3ebeb":{"m":113,"g":115},"312bfc4c":{"m":113,"g":115},"e290303e":{"m":113,"g":115},"aab35bcc":{"m":113,"g":115},"42aedb02":{"m":113,"g":115},"984730b7":{"m":113,"g":115},"23632d35":{"m":113,"g":115},"08b8c0c3":{"m":113,"g":115},"d42975c6":{"m":113,"g":115},"adc24a3a":{"m":113,"g":115},"7ff93e61":{"m":113,"g":115},"b24b2e7e":{"m":113,"g":115},"7135db5d":{"m":113,"g":115},"4b5ef300":{"m":113,"g":115},"4f564b9e":{"m":113,"g":115},"98c3b04f":{"m":113,"g":115},"ddab4fc7":{"m":113,"g":115},"d21c3522":{"m":113,"g":115},"4a762041":{"m":113,"g":115},"ea338676":{"m":113,"g":115},"b06db198":{"m":113,"g":115},"8c1ef0f9":{"m":113,"g":115},"f5a2faf2":{"m":113,"g":115},"1c82d9db":{"m":113,"g":115},"9241f4fd":{"m":113,"g":115},"063c3791":{"m":113,"g":115},"632b7d8c":{"m":113,"g":115},"16adf3dc":{"m":113,"g":115},"c3a1d775":{"m":113,"g":115},"89971c4c":{"m":113,"g":115},"113f8f65":{"m":113,"g":115},"e22f3a5e":{"m":113,"g":115},"095093ee":{"m":113,"g":115},"d27a6f70":{"m":113,"g":115},"0753ef83":{"m":113,"g":115},"662393f2":{"m":113,"g":115},"b1bb8e74":{"m":113,"g":115},"38c00ed7":{"m":113,"g":115},"d4041a5e":{"m":113,"g":115},"2f555c4c":{"m":113,"g":115},"e53df7c0":{"m":113,"g":115},"9c53dad8":{"m":113,"g":115},"7ca1bea6":{"m":113,"g":115},"97c38239":{"m":113,"g":115},"60dbbd08":{"m":113,"g":115},"aa1c5cf5":{"m":113,"g":115},"592caab6":{"m":113,"g":115},"2101d93b":{"m":113,"g":115},"70e4b218":{"m":113,"g":115},"944f1ea0":{"m":113,"g":115},"9d7e82a0":{"m":113,"g":115},"f0580551":{"m":113,"g":115},"635ccda6":{"m":113,"g":115},"1c3dbad8":{"m":113,"g":115},"e2ac7888":{"m":113,"g":115},"86527a47":{"m":113,"g":115},"134b4f7e":{"m":113,"g":115},"f67d1f45":{"m":113,"g":115},"0f04a5f4":{"m":113,"g":115},"2f18602f":{"m":113,"g":115},"56321e9f":{"m":113,"g":115},"12d6cf18":{"m":113,"g":115},"fc3e5420":{"m":113,"g":115},"08ecd0aa":{"m":113,"g":115},"720c1c8c":{"m":113,"g":115},"d403c143":{"m":113,"g":115},"cba0d8c3":{"m":113,"g":115},"f1d78923":{"m":113,"g":115},"7c876de7":{"m":113,"g":115},"ba94b829":{"m":113,"g":115},"2b7417bf":{"m":113,"g":115},"f1116495":{"m":113,"g":115},"bd7eb020":{"m":113,"g":115},"74cd6e39":{"m":113,"g":115},"b17e67df":{"m":113,"g":115},"8ecef73f":{"m":113,"g":115},"1d1ce624":{"m":113,"g":115},"60e2a7ce":{"m":113,"g":115},"d88ef4a3":{"m":113,"g":115},"6f993e8b":{"m":113,"g":115},"03ce92e5":{"m":113,"g":115},"00eb5eb7":{"m":113,"g":115},"dab4663b":{"m":113,"g":115},"610a6d6e":{"m":113,"g":115},"36efd5be":{"m":113,"g":115},"68cdc189":{"m":113,"g":115},"7f399e4b":{"m":113,"g":115},"873d858b":{"m":113,"g":115},"3fa3c22a":{"m":113,"g":115},"4f2055ad":{"m":113,"g":115},"616a3e20":{"m":113,"g":115},"ac2a723b":{"m":113,"g":115},"56b991b1":{"m":113,"g":115},"780d6a22":{"m":113,"g":115},"8b713c72":{"m":113,"g":115},"5bfafdfc":{"m":113,"g":115},"8c52de6f":{"m":113,"g":115},"c1815a99":{"m":113,"g":115},"4e6c4923":{"m":113,"g":115},"b91cb67e":{"m":113,"g":115},"e7bc6003":{"m":113,"g":115},"2a2ff9a8":{"m":113,"g":115},"5291f32d":{"m":113,"g":115},"67073dde":{"m":113,"g":115},"9a5c42f9":{"m":113,"g":115},"388c05d5":{"m":113,"g":115},"fc809665":{"m":113,"g":115},"6fd4816d":{"m":113,"g":115},"1344ebc8":{"m":113,"g":115},"e07b21ce":{"m":113,"g":115},"52f248cd":{"m":113,"g":115},"93f75778":{"m":113,"g":115},"4039c626":{"m":113,"g":115},"db71c38f":{"m":113,"g":115},"7a68b422":{"m":113,"g":115},"60fc5b51":{"m":113,"g":115},"a13dd1e4":{"m":113,"g":115},"d500eb91":{"m":113,"g":115},"1ccd59c7":{"m":113,"g":115},"c32fb7a2":{"m":113,"g":115},"1ba137e9":{"m":113,"g":115},"de28f8e7":{"m":113,"g":115},"56405076":{"m":113,"g":115},"b73ac629":{"m":113,"g":115},"77098aea":{"m":113,"g":115},"5ccf0b03":{"m":113,"g":115},"a77564e0":{"m":113,"g":115},"4f9e71df":{"m":113,"g":115},"541551ce":{"m":113,"g":115},"124097fc":{"m":113,"g":115},"e1d45bc2":{"m":113,"g":115},"14fdd527":{"m":113,"g":115},"f949ad57":{"m":113,"g":115},"c49484a6":{"m":113,"g":115},"a2f7218a":{"m":113,"g":115},"311de47b":{"m":113,"g":115},"373080ea":{"m":113,"g":115},"7f028b07":{"m":113,"g":115},"0abb41c7":{"m":113,"g":115},"925dbb32":{"m":113,"g":115},"8df7353a":{"m":113,"g":115},"ae4be601":{"m":113,"g":115},"9b876889":{"m":113,"g":115},"c0c6f543":{"m":113,"g":115},"edd6a07b":{"m":113,"g":115},"b6dd4bcb":{"m":113,"g":115},"b2435be6":{"m":113,"g":115},"5fe39e85":{"m":113,"g":115},"fa5d0bf6":{"m":113,"g":115},"16e93359":{"m":113,"g":115},"f1c692f6":{"m":113,"g":115},"80572c83":{"m":113,"g":115},"4bb08f6e":{"m":113,"g":115},"ec272dda":{"m":113,"g":115},"a220c14f":{"m":113,"g":115},"35ef3f29":{"m":113,"g":115},"31fb19a0":{"m":113,"g":115},"3f41b48c":{"m":113,"g":115},"2689f0bf":{"m":113,"g":115},"52074240":{"m":113,"g":115},"c3c26f76":{"m":113,"g":115},"2cf811a9":{"m":113,"g":115},"3b25dc12":{"m":113,"g":115},"5c08d7d2":{"m":113,"g":115},"a45d9a4e":{"m":113,"g":115},"28c79dc8":{"m":113,"g":115},"1fcccda4":{"m":113,"g":115},"79acec4f":{"m":113,"g":115},"b1721edb":{"m":113,"g":115},"57234d0c":{"m":113,"g":115},"b93acd70":{"m":113,"g":115},"86a32bb5":{"m":113,"g":115},"5afd0365":{"m":113,"g":115},"059c13de":{"m":113,"g":115},"50dc0c1e":{"m":113,"g":115},"76becc1d":{"m":113,"g":115},"2a37b24d":{"m":113,"g":115},"f73aae0b":{"m":113,"g":115},"69b35793":{"m":113,"g":115},"957482c8":{"m":113,"g":115},"3795b6a4":{"m":113,"g":115},"7eccbe99":{"m":113,"g":115},"0549f21c":{"m":113,"g":115},"b354e3c9":{"m":113,"g":115},"65e6f48c":{"m":113,"g":115},"0ec580a8":{"m":113,"g":115},"0b14159f":{"m":113,"g":115},"1489cd6c":{"m":113,"g":115},"fc2c3a3d":{"m":113,"g":115},"8f6a1758":{"m":113,"g":115},"01018138":{"m":113,"g":115},"4844fac9":{"m":113,"g":115},"b7d385e8":{"m":113,"g":115},"305c9e8c":{"m":113,"g":115},"ca63f075":{"m":113,"g":115},"f9ee6ae1":{"m":113,"g":115},"dcee42c2":{"m":113,"g":115},"258d02c8":{"m":113,"g":115},"60d7beda":{"m":113,"g":115},"2f8ba6fe":{"m":113,"g":115},"7ce6c10e":{"m":113,"g":115},"55025b92":{"m":113,"g":115},"4c21b090":{"m":113,"g":115},"165abeeb":{"m":113,"g":115},"21ca4c3a":{"m":113,"g":115},"e3cf812f":{"m":113,"g":115},"4da55336":{"m":113,"g":115},"ac964d2e":{"m":113,"g":115},"fa46e2bd":{"m":113,"g":115},"b047b553":{"m":113,"g":115},"a0f844ed":{"m":113,"g":115},"2df532ef":{"m":113,"g":115},"abea9250":{"m":113,"g":115},"b3c97762":{"m":113,"g":115},"b8347b40":{"m":113,"g":115},"72dfa96a":{"m":113,"g":115},"05b01ef4":{"m":113,"g":115},"55a6e644":{"m":113,"g":115},"6897e06b":{"m":113,"g":115},"a360511d":{"m":113,"g":115},"94d0f656":{"m":113,"g":115},"eca59f96":{"m":113,"g":115},"97528610":{"m":113,"g":115},"297d3745":{"m":113,"g":115},"31e9d3a5":{"m":113,"g":115},"6f4676ef":{"m":113,"g":115},"c9ec4cae":{"m":113,"g":115},"99757cc3":{"m":113,"g":115},"cdddab05":{"m":113,"g":115},"7c5a0a1b":{"m":113,"g":115},"49f169d5":{"m":113,"g":115},"7fce2fd9":{"m":113,"g":115},"16cd550c":{"m":113,"g":115},"d5e2a374":{"m":113,"g":115},"366043db":{"m":113,"g":115},"2f173ea0":{"m":113,"g":115},"321fecab":{"m":113,"g":115},"9d775b1a":{"m":113,"g":115},"78b7465c":{"m":113,"g":115},"07bcad7f":{"m":113,"g":115},"98adac8e":{"m":113,"g":115},"cef11e9a":{"m":113,"g":115},"2269cf1e":{"m":113,"g":115},"151e287d":{"m":113,"g":115},"8c86595c":{"m":113,"g":115},"4634fd59":{"m":113,"g":115},"efedbe6c":{"m":113,"g":115},"3a77c80b":{"m":113,"g":115},"36acd2ff":{"m":113,"g":115},"fe6cdf89":{"m":113,"g":115},"30d20ce8":{"m":113,"g":115},"1b1701f1":{"m":113,"g":115},"6d403089":{"m":113,"g":115},"24dc2bee":{"m":113,"g":115},"fac07c9b":{"m":113,"g":115},"b3839a7f":{"m":113,"g":115},"4aa39d72":{"m":113,"g":115},"b4c2c421":{"m":113,"g":115},"53ca1552":{"m":113,"g":115},"a23bdeaf":{"m":113,"g":115},"27778010":{"m":113,"g":115},"46d8fb1c":{"m":113,"g":115},"c7e85f53":{"m":113,"g":115},"3df05f4d":{"m":113,"g":115},"7b141f81":{"m":113,"g":115},"7bc5fb0d":{"m":113,"g":115},"144ee5f3":{"m":113,"g":115},"758b887a":{"m":114,"g":115},"eb7d9261":{"m":114,"g":115},"44cb0607":{"m":114,"g":115},"88bb627d":{"m":114,"g":115},"b520958e":{"m":114,"g":115},"fa7e2c30":{"m":114,"g":115},"8f2cd177":{"m":114,"g":115},"ab926dd6":{"m":114,"g":115},"a4b424c6":{"m":114,"g":115},"a0557642":{"m":114,"g":115},"84768d10":{"m":114,"g":115},"368fd206":{"m":114,"g":115},"53bd00d9":{"m":114,"g":115},"e22b13c5":{"m":114,"g":115},"a3c2ea44":{"m":114,"g":115},"fccac7d1":{"m":114,"g":115},"7ac6b900":{"m":114,"g":115},"a1080b72":{"m":114,"g":115},"a65ca739":{"m":114,"g":115},"677aa0e2":{"m":114,"g":115},"01c9ee1a":{"m":114,"g":115},"d6837aea":{"m":114,"g":115},"c882b5ae":{"m":114,"g":115},"e3bb7f5a":{"m":114,"g":115},"92473e2e":{"m":114,"g":115},"6c0bb327":{"m":114,"g":115},"0a7c4bde":{"m":114,"g":115},"edefab0c":{"m":114,"g":115},"97cd38e5":{"m":114,"g":115},"3c06b673":{"m":114,"g":115},"7c3f07db":{"m":114,"g":115},"edd86b88":{"m":114,"g":115},"4b4dc132":{"m":114,"g":115},"5a9170d9":{"m":114,"g":115},"c4d77774":{"m":114,"g":115},"832c84fb":{"m":114,"g":115},"64d1505c":{"m":114,"g":115},"f3764c26":{"m":114,"g":115},"7ba3de0e":{"m":114,"g":115},"fde9b963":{"m":114,"g":115},"f094e0a4":{"m":114,"g":115},"4ed67c27":{"m":114,"g":115},"cd4b39a9":{"m":114,"g":115},"420c99ac":{"m":114,"g":115},"e3c7f091":{"m":114,"g":115},"6f1e03a4":{"m":114,"g":115},"f4affd4d":{"m":114,"g":115},"df08bf9b":{"m":114,"g":115},"69efdd27":{"m":114,"g":115},"64582caa":{"m":114,"g":115},"2fcd56ea":{"m":114,"g":115},"0958a397":{"m":114,"g":115},"4f42c8cd":{"m":114,"g":115},"3ddd7dc9":{"m":114,"g":115},"501dfa6b":{"m":114,"g":115},"79d34951":{"m":114,"g":115},"1519a89c":{"m":114,"g":115},"24bc3fb0":{"m":114,"g":115},"8a8a608a":{"m":114,"g":115},"533e58a1":{"m":114,"g":115},"9b4c4497":{"m":114,"g":115},"a578d300":{"m":114,"g":115},"8c967037":{"m":114,"g":115},"fb27d383":{"m":114,"g":115},"fd8a0b29":{"m":114,"g":115},"afc35ccc":{"m":114,"g":115},"a57f0e3d":{"m":114,"g":115},"708f4ff4":{"m":114,"g":115},"e2daeb35":{"m":114,"g":115},"0e7b3530":{"m":114,"g":115},"b07c9c76":{"m":114,"g":115},"748f86f3":{"m":114,"g":115},"73ea484a":{"m":114,"g":115},"4aeb193f":{"m":114,"g":115},"466992b2":{"m":114,"g":115},"155cbb51":{"m":114,"g":115},"eb30b888":{"m":114,"g":115},"5ee777c9":{"m":114,"g":115},"baf277a9":{"m":116,"g":118},"f5d30dae":{"m":116,"g":118},"2479b894":{"m":116,"g":118},"54644572":{"m":116,"g":118},"6c01844f":{"m":116,"g":118},"f226d3da":{"m":116,"g":118},"d2478cd4":{"m":116,"g":118},"30ea4c46":{"m":116,"g":118},"6d036468":{"m":116,"g":118},"8221f9ae":{"m":116,"g":118},"ab9187a2":{"m":116,"g":118},"6b143d62":{"m":116,"g":118},"6bc503af":{"m":116,"g":118},"b2c85669":{"m":116,"g":118},"32803fb2":{"m":116,"g":118},"91fc5bb5":{"m":116,"g":118},"780fbf2f":{"m":116,"g":118},"825432fc":{"m":116,"g":118},"a40229f6":{"m":116,"g":118},"74737b28":{"m":116,"g":115},"40e0082d":{"m":116,"g":115},"e9e120ac":{"m":116,"g":115},"e0c2af2a":{"m":116,"g":115},"1d7f7835":{"m":116,"g":115},"32595146":{"m":116,"g":115},"86373b9e":{"m":116,"g":115},"d314bf60":{"m":116,"g":115},"e28c9e52":{"m":116,"g":115},"b98cf398":{"m":116,"g":115},"27d71045":{"m":116,"g":115},"c224a4c6":{"m":116,"g":115},"49345a68":{"m":116,"g":115},"94d26d85":{"m":116,"g":115},"9e8a15a7":{"m":116,"g":115},"3962e39d":{"m":116,"g":115},"eb8cac6f":{"m":116,"g":115},"5ea96ac7":{"m":116,"g":115},"56222658":{"m":116,"g":115},"dc965db0":{"m":116,"g":115},"817e46f4":{"m":116,"g":115},"5a33c3aa":{"m":116,"g":115},"9767a1e4":{"m":116,"g":115},"1d086539":{"m":116,"g":115},"a04efc49":{"m":116,"g":115},"642fa966":{"m":116,"g":115},"da7fac1b":{"m":116,"g":115},"28ad2297":{"m":116,"g":115},"f7f9f8ec":{"m":116,"g":115},"4b62af92":{"m":116,"g":115},"0b9915c1":{"m":116,"g":115},"27ef1459":{"m":116,"g":115},"e4358a45":{"m":116,"g":115},"ba2ce28f":{"m":116,"g":115},"98923880":{"m":116,"g":115},"f792e3c5":{"m":116,"g":115},"28f80b12":{"m":116,"g":115},"88a6f9da":{"m":116,"g":115},"cb8ed2c0":{"m":116,"g":115},"38473363":{"m":116,"g":115},"aaf7af1b":{"m":116,"g":115},"932e2637":{"m":116,"g":115},"43f80884":{"m":116,"g":115},"60b05032":{"m":116,"g":115},"dc48c4c0":{"m":116,"g":115},"6dc9ca8c":{"m":116,"g":115},"887c2b45":{"m":116,"g":115},"065ce815":{"m":116,"g":115},"8e51049f":{"m":116,"g":115},"cb8f3d90":{"m":116,"g":115},"4b694e7d":{"m":116,"g":115},"9f1f699a":{"m":116,"g":115},"c9cff2b9":{"m":116,"g":115},"b6fb5d76":{"m":116,"g":115},"f4aa7880":{"m":116,"g":115},"5e3f7e7f":{"m":116,"g":115},"728af887":{"m":116,"g":115},"7b59b0b8":{"m":116,"g":115},"acc2327b":{"m":116,"g":115},"bfadb5ea":{"m":116,"g":115},"9cc1e065":{"m":116,"g":115},"b8c430f1":{"m":116,"g":115},"f35f120d":{"m":116,"g":115},"54a46a26":{"m":116,"g":115},"7c94eaee":{"m":116,"g":115},"13d596c9":{"m":116,"g":115},"c7867b67":{"m":116,"g":115},"516738b0":{"m":116,"g":115},"0b6f535f":{"m":116,"g":115},"c5fe3c0b":{"m":116,"g":115},"318424e2":{"m":116,"g":115},"6806c4e6":{"m":116,"g":115},"0c0779d6":{"m":116,"g":115},"a55cf530":{"m":116,"g":115},"19ba16aa":{"m":116,"g":115},"a2b3d9b9":{"m":116,"g":115},"9a30914e":{"m":116,"g":115},"8e776c78":{"m":116,"g":115},"63e84352":{"m":116,"g":115},"a20e7df8":{"m":116,"g":115},"1bdd0102":{"m":116,"g":115},"6cd29694":{"m":116,"g":115},"2ac46e94":{"m":116,"g":115},"0aa65f94":{"m":116,"g":115},"0ecb4261":{"m":116,"g":115},"05f015f6":{"m":116,"g":115},"1083e7e3":{"m":116,"g":115},"2157d12a":{"m":116,"g":115},"9f2b457c":{"m":116,"g":115},"f5b34a51":{"m":116,"g":115},"5a6ec8f9":{"m":116,"g":115},"6a653bb1":{"m":116,"g":115},"548a57b1":{"m":116,"g":115},"88e73ed0":{"m":116,"g":115},"4b15fa00":{"m":116,"g":115},"f4941906":{"m":116,"g":115},"01e59e82":{"m":116,"g":115},"99a0704a":{"m":116,"g":115},"ec1cd90a":{"m":116,"g":115},"1103dc62":{"m":116,"g":115},"a220536f":{"m":116,"g":115},"7b064f04":{"m":116,"g":115},"43190bec":{"m":116,"g":115},"be740acd":{"m":116,"g":115},"2db2cddd":{"m":116,"g":115},"9b5efe34":{"m":116,"g":115},"4ac8e09d":{"m":116,"g":115},"20a6c0a6":{"m":116,"g":115},"47c606d3":{"m":116,"g":115},"9fcf7306":{"m":116,"g":115},"0a304870":{"m":116,"g":115},"8fdcd98e":{"m":116,"g":115},"b5dcfd41":{"m":116,"g":115},"5061b8fd":{"m":116,"g":115},"c8452551":{"m":116,"g":115},"bf3e7149":{"m":116,"g":115},"f5754d12":{"m":116,"g":115},"739daa63":{"m":116,"g":115},"d957177a":{"m":116,"g":115},"21337b22":{"m":116,"g":115},"129d2992":{"m":116,"g":115},"8b85926a":{"m":116,"g":115},"451d15c4":{"m":116,"g":115},"c80a96da":{"m":116,"g":115},"eae9a9fb":{"m":116,"g":115},"2674c1d2":{"m":116,"g":115},"61055cb3":{"m":116,"g":115},"92777135":{"m":116,"g":115},"c4958331":{"m":116,"g":115},"2eeb2751":{"m":116,"g":115},"b36afed4":{"m":116,"g":115},"9aa4502d":{"m":116,"g":115},"a0835c3a":{"m":116,"g":115},"55b14656":{"m":116,"g":115},"b4408e60":{"m":116,"g":115},"52fcbbb8":{"m":116,"g":115},"af96ca11":{"m":116,"g":115},"9082a7d3":{"m":116,"g":115},"3b9d97f3":{"m":116,"g":115},"a1a20b4c":{"m":116,"g":115},"4299aebd":{"m":116,"g":115},"0babd487":{"m":116,"g":115},"f19613e6":{"m":116,"g":115},"8df49455":{"m":116,"g":115},"ee3bd8a1":{"m":116,"g":115},"d8467db7":{"m":116,"g":115},"b5044fbf":{"m":116,"g":115},"70fbb3ad":{"m":116,"g":115},"9a7e7a65":{"m":116,"g":115},"0fe87213":{"m":116,"g":115},"1f106ee3":{"m":116,"g":115},"9b8ebb27":{"m":116,"g":115},"85ebeecf":{"m":117,"g":118},"0dd6cf16":{"m":117,"g":118},"0975ba99":{"m":117,"g":118},"1de3924b":{"m":117,"g":118},"3cceaa38":{"m":117,"g":118},"b0d20cde":{"m":117,"g":118},"cbac4997":{"m":117,"g":118},"476c67d7":{"m":117,"g":118},"3289da5b":{"m":117,"g":118},"868403f6":{"m":117,"g":118},"97d857c0":{"m":117,"g":118},"52a54a26":{"m":117,"g":118},"cd7e1bd5":{"m":117,"g":118},"729b7edf":{"m":117,"g":118},"4c03dbaa":{"m":117,"g":118},"1053e1be":{"m":119,"g":121},"9a71500c":{"m":119,"g":121},"6d6e24bc":{"m":119,"g":121},"2c057fbf":{"m":119,"g":121},"dbd9435d":{"m":119,"g":121},"8ae9d4bb":{"m":119,"g":121},"1c304aa9":{"m":119,"g":121},"770529a7":{"m":119,"g":121},"39c237f0":{"m":119,"g":121},"28b8a406":{"m":119,"g":121},"8bd26dd4":{"m":119,"g":121},"ab07cd3e":{"m":119,"g":121},"a9849683":{"m":119,"g":121},"a4b637d8":{"m":119,"g":121},"96a5e4dd":{"m":119,"g":121},"b0b4f716":{"m":119,"g":121},"6c18addb":{"m":119,"g":121},"32852fe9":{"m":119,"g":121},"53c2934d":{"m":119,"g":121},"e321c971":{"m":119,"g":121},"d6fee73d":{"m":119,"g":121},"36a4cad7":{"m":119,"g":121},"65d376b4":{"m":119,"g":121},"c23eda85":{"m":119,"g":121},"138ff231":{"m":119,"g":121},"13fb8b54":{"m":119,"g":121},"81fd2b0e":{"m":119,"g":121},"007b849b":{"m":119,"g":121},"8612811d":{"m":119,"g":121},"e7aa4664":{"m":119,"g":121},"4d4feccb":{"m":119,"g":121},"99c92ff2":{"m":119,"g":121},"6ade6a02":{"m":119,"g":121},"983ef22c":{"m":119,"g":121},"164302c7":{"m":119,"g":121},"5dccf697":{"m":119,"g":121},"eec9e471":{"m":119,"g":121},"6d535b71":{"m":119,"g":121},"fdcb1d13":{"m":119,"g":121},"d7e834d6":{"m":119,"g":121},"200a3c0b":{"m":119,"g":121},"77258ce0":{"m":119,"g":121},"1d097aac":{"m":119,"g":121},"7fceeef5":{"m":119,"g":121},"88568c01":{"m":119,"g":121},"904655c5":{"m":119,"g":121},"e028af69":{"m":119,"g":121},"80b2b320":{"m":119,"g":121},"4b65ed42":{"m":119,"g":121},"23afdfd1":{"m":119,"g":121},"9d61205d":{"m":119,"g":121},"590bc4b7":{"m":119,"g":121},"63cfe1b0":{"m":119,"g":121},"70f6309c":{"m":119,"g":121},"70416001":{"m":119,"g":121},"87a92e45":{"m":119,"g":121},"c461e771":{"m":119,"g":121},"fde2decf":{"m":119,"g":121},"9792b9d7":{"m":119,"g":121},"ef4a8097":{"m":119,"g":121},"ebff4ee6":{"m":119,"g":121},"2b1da821":{"m":119,"g":121},"97710ccd":{"m":119,"g":121},"f3cd5d25":{"m":119,"g":121},"c61b0b29":{"m":119,"g":121},"e8640ee9":{"m":119,"g":121},"d0a64c7e":{"m":119,"g":121},"05d3667a":{"m":119,"g":121},"260fe755":{"m":119,"g":121},"dbb16bed":{"m":119,"g":121},"c1e16003":{"m":119,"g":121},"852c0578":{"m":119,"g":121},"7e6191c0":{"m":119,"g":121},"6f9b66bd":{"m":119,"g":121},"8a801ee3":{"m":119,"g":118},"d9a20fd2":{"m":119,"g":118},"b113c72e":{"m":119,"g":118},"fb6cc7b0":{"m":119,"g":118},"8374a96e":{"m":119,"g":118},"74de76c6":{"m":119,"g":118},"9c0b1eb5":{"m":119,"g":118},"01f14a7a":{"m":119,"g":118},"11110303":{"m":119,"g":118},"28ddfb37":{"m":119,"g":118},"e69094df":{"m":119,"g":118},"43ad0590":{"m":119,"g":118},"b4948512":{"m":119,"g":118},"ddcba74b":{"m":119,"g":118},"0917c5da":{"m":119,"g":118},"184a4df6":{"m":119,"g":118},"f7b1d8c5":{"m":119,"g":118},"bfc3b3f7":{"m":119,"g":118},"da5bde4d":{"m":119,"g":118},"276e7b3e":{"m":119,"g":118},"296f6892":{"m":119,"g":118},"9edb7b51":{"m":119,"g":118},"e53bf442":{"m":119,"g":118},"d383e661":{"m":119,"g":118},"984fbeb1":{"m":119,"g":118},"a2ba0bc3":{"m":119,"g":118},"6d2d0ce2":{"m":119,"g":118},"271d3d0d":{"m":119,"g":118},"c4e81e64":{"m":119,"g":118},"c726d44c":{"m":119,"g":118},"283c8ba0":{"m":119,"g":118},"cae39565":{"m":119,"g":118},"27a223ab":{"m":119,"g":118},"53529f46":{"m":119,"g":118},"24ed3f32":{"m":119,"g":118},"44f0ece9":{"m":119,"g":118},"be0058bc":{"m":119,"g":118},"9e3be1fa":{"m":119,"g":118},"a8ba3279":{"m":119,"g":118},"3b80232d":{"m":119,"g":118},"252dc4e1":{"m":119,"g":118},"cbb5fc2e":{"m":119,"g":118},"53fb229f":{"m":119,"g":118},"4fff1ec1":{"m":119,"g":118},"7a020e0f":{"m":119,"g":118},"48738af7":{"m":119,"g":118},"efa47334":{"m":119,"g":118},"d658f049":{"m":119,"g":118},"57e25de7":{"m":119,"g":118},"12eb02e9":{"m":119,"g":118},"002d0373":{"m":119,"g":118},"a27825ae":{"m":119,"g":118},"ce399e15":{"m":119,"g":118},"ea6275df":{"m":119,"g":118},"eb7318f1":{"m":119,"g":118},"6058fb52":{"m":119,"g":118},"80407b04":{"m":119,"g":118},"b288f4f4":{"m":119,"g":118},"6d6ea5af":{"m":119,"g":118},"1dacedd2":{"m":119,"g":118},"b5e14b2b":{"m":119,"g":118},"d513ee93":{"m":119,"g":118},"a7ae61ed":{"m":119,"g":118},"fda0cb2a":{"m":119,"g":118},"ebda73dc":{"m":119,"g":118},"f4f8a1b4":{"m":119,"g":118},"c44e985d":{"m":119,"g":118},"f9a7d9b3":{"m":119,"g":118},"a93f10a7":{"m":119,"g":118},"585e1223":{"m":119,"g":118},"a7043c6f":{"m":119,"g":118},"67e34c56":{"m":119,"g":118},"1d726528":{"m":119,"g":118},"f4488e9d":{"m":119,"g":118},"e68a2b5b":{"m":119,"g":118},"31b9f19e":{"m":119,"g":118},"547003bd":{"m":119,"g":118},"f7ab9554":{"m":119,"g":118},"dbbd4e18":{"m":119,"g":118},"ca240eef":{"m":119,"g":118},"6c7c92eb":{"m":119,"g":118},"5b214b50":{"m":119,"g":118},"13219e1e":{"m":119,"g":118},"33e9bbec":{"m":119,"g":118},"dcb8f090":{"m":119,"g":118},"9eefe2c0":{"m":119,"g":118},"69fe3c97":{"m":119,"g":118},"8af84912":{"m":119,"g":118},"505329ca":{"m":119,"g":118},"8a382fd3":{"m":119,"g":118},"62797440":{"m":119,"g":118},"2614adf9":{"m":119,"g":118},"fdd7c69d":{"m":119,"g":118},"b9a54e09":{"m":119,"g":118},"20b8d230":{"m":119,"g":118},"d1984e21":{"m":119,"g":118},"b79f75fd":{"m":119,"g":118},"8fcc69e7":{"m":119,"g":118},"f440baa1":{"m":119,"g":118},"2bc3fcd4":{"m":119,"g":118},"a5978a20":{"m":119,"g":118},"e483c1ea":{"m":119,"g":118},"da681f35":{"m":119,"g":118},"9b0f725b":{"m":119,"g":118},"cde5a6e3":{"m":119,"g":118},"3e4c7da2":{"m":119,"g":118},"d88ac9bc":{"m":119,"g":118},"ce11dd82":{"m":119,"g":118},"9e87b60f":{"m":119,"g":118},"7780230a":{"m":119,"g":118},"dc01313d":{"m":119,"g":118},"7a7f99be":{"m":119,"g":118},"fd389df9":{"m":119,"g":118},"b0d1d717":{"m":119,"g":118},"c7962868":{"m":119,"g":118},"4f24ab17":{"m":119,"g":118},"64affab4":{"m":119,"g":118},"4c9bcb9d":{"m":119,"g":118},"86b04d25":{"m":119,"g":118},"55d75e11":{"m":120,"g":121},"3f4cc0af":{"m":120,"g":121},"cadfae66":{"m":120,"g":121},"da1766e4":{"m":120,"g":121},"a124b517":{"m":120,"g":121},"d05a968b":{"m":120,"g":121},"94aad0de":{"m":120,"g":121},"7ebc28f5":{"m":120,"g":121},"b89111d6":{"m":120,"g":121},"0b3b3e9a":{"m":120,"g":121},"a1d5bc4c":{"m":120,"g":121},"a8023891":{"m":120,"g":121},"0103f374":{"m":120,"g":121},"96a5a949":{"m":120,"g":121},"ea385ae8":{"m":120,"g":121},"9e949e58":{"m":120,"g":121},"6dbb569b":{"m":120,"g":121},"5994e6c3":{"m":120,"g":121},"3e6281d0":{"m":120,"g":121},"6371f7af":{"m":120,"g":121},"8491c794":{"m":120,"g":121},"bda3758f":{"m":120,"g":121},"7b36c47b":{"m":120,"g":121},"773d89da":{"m":120,"g":121},"03e7d949":{"m":120,"g":121},"ff604064":{"m":120,"g":121},"212f5e48":{"m":120,"g":121},"fe527812":{"m":120,"g":121},"97828878":{"m":120,"g":121},"c001deba":{"m":120,"g":121},"b4d2da10":{"m":120,"g":121},"b72f9f08":{"m":120,"g":121},"8e70064c":{"m":120,"g":121},"d98b81e2":{"m":120,"g":121},"bcecf27e":{"m":120,"g":121},"4b0ac1d5":{"m":120,"g":121},"8e987fa2":{"m":120,"g":121},"9e656dd3":{"m":120,"g":121},"3862661c":{"m":120,"g":121},"c8492978":{"m":120,"g":121},"428710c2":{"m":120,"g":121},"d9b31011":{"m":120,"g":121},"d0cff78f":{"m":120,"g":121},"e8b71445":{"m":120,"g":121},"4caca1ba":{"m":120,"g":121},"ceb105a7":{"m":120,"g":121},"22cbc9c0":{"m":120,"g":121},"3865afc5":{"m":120,"g":121},"4ea42f7c":{"m":120,"g":121},"ea13cb14":{"m":120,"g":121},"a04212f1":{"m":120,"g":121},"ce869793":{"m":120,"g":121},"22f55e1b":{"m":120,"g":121},"89824189":{"m":120,"g":121},"433c622e":{"m":120,"g":121},"20bd2271":{"m":120,"g":121},"64994980":{"m":120,"g":121},"729b2429":{"m":120,"g":121},"d7056c52":{"m":120,"g":121},"13bf565d":{"m":120,"g":121},"e51046be":{"m":120,"g":121},"4eeeae1e":{"m":120,"g":121},"f4b78d13":{"m":120,"g":121},"4463e90d":{"m":120,"g":121},"229f236d":{"m":120,"g":121},"5983e5bd":{"m":120,"g":121},"4b046a72":{"m":120,"g":121},"770d6312":{"m":120,"g":121},"14203432":{"m":120,"g":121},"d7f0d88f":{"m":120,"g":121},"fc86b18b":{"m":120,"g":121},"0bfa394a":{"m":120,"g":121},"e04340bf":{"m":120,"g":121},"84701338":{"m":120,"g":121},"93ef9a09":{"m":120,"g":121},"b04cd3d4":{"m":120,"g":121},"7ef5d8af":{"m":120,"g":121},"71d41212":{"m":120,"g":121},"b9fb74f3":{"m":120,"g":121},"e15b63a1":{"m":120,"g":121},"4060ed37":{"m":120,"g":121},"2342605e":{"m":120,"g":121},"dbf17a83":{"m":120,"g":121},"0f0c430e":{"m":120,"g":121},"8e797a47":{"m":120,"g":121},"aa3003f1":{"m":120,"g":121},"4793ec7d":{"m":120,"g":121},"92009bd2":{"m":120,"g":121},"4ef981e2":{"m":120,"g":121},"69ed8b67":{"m":120,"g":121},"1801cd19":{"m":120,"g":121},"ffc722a6":{"m":120,"g":121},"49afb3d9":{"m":120,"g":121},"f80371ff":{"m":120,"g":121},"62eff37b":{"m":120,"g":121},"47e12e08":{"m":120,"g":121},"823b4429":{"m":120,"g":121},"14a4d80e":{"m":120,"g":121},"41c10e67":{"m":122,"g":127},"0bfe1d14":{"m":122,"g":127},"5f98b7fe":{"m":122,"g":127},"a4bf5c6a":{"m":122,"g":127},"30ad1070":{"m":122,"g":127},"a80bcb5a":{"m":122,"g":127},"f7f9e41b":{"m":122,"g":127},"263eab9f":{"m":122,"g":127},"25257d8e":{"m":122,"g":127},"cf0c2415":{"m":122,"g":127},"5538e05c":{"m":122,"g":127},"c30ebb93":{"m":122,"g":127},"41efcaeb":{"m":122,"g":127},"70562969":{"m":122,"g":127},"b57dc169":{"m":122,"g":127},"0095e018":{"m":122,"g":127},"68486481":{"m":122,"g":127},"410225b7":{"m":122,"g":127},"2c9aebea":{"m":122,"g":127},"bc741073":{"m":122,"g":127},"2f6af1a3":{"m":122,"g":127},"50b6842b":{"m":122,"g":127},"2d5605e8":{"m":122,"g":127},"300b4c21":{"m":122,"g":127},"c0652d90":{"m":122,"g":127},"2e48584b":{"m":122,"g":127},"57cc5385":{"m":122,"g":127},"5cc0d25a":{"m":122,"g":127},"a076ec1a":{"m":122,"g":127},"72b5f3d0":{"m":122,"g":127},"2f766f38":{"m":122,"g":127},"069e490b":{"m":122,"g":127},"ab95d35f":{"m":122,"g":127},"34c286b8":{"m":122,"g":127},"9416ee60":{"m":122,"g":127},"d4a09ec9":{"m":122,"g":127},"662725b9":{"m":122,"g":127},"82cfcd3b":{"m":122,"g":127},"6c1a3f0c":{"m":122,"g":127},"62377548":{"m":122,"g":121},"96ac24c0":{"m":122,"g":121},"c0d02cf4":{"m":122,"g":121},"7d121448":{"m":122,"g":121},"6a63a985":{"m":122,"g":121},"4d2f17bd":{"m":122,"g":121},"7cd716f7":{"m":122,"g":121},"b7fdde4b":{"m":122,"g":121},"69bf8011":{"m":122,"g":121},"d8fcbaa3":{"m":122,"g":121},"2cf3d0f8":{"m":122,"g":121},"1ed1abfd":{"m":122,"g":121},"ecb9fa14":{"m":122,"g":121},"700daa34":{"m":122,"g":121},"39cee0fe":{"m":122,"g":121},"04e5b6fa":{"m":122,"g":121},"ce6b17c0":{"m":122,"g":121},"cafebef1":{"m":122,"g":121},"73dfd2df":{"m":122,"g":121},"df5192cf":{"m":122,"g":121},"78c43d88":{"m":122,"g":121},"7e28c67d":{"m":122,"g":121},"3edba9bc":{"m":122,"g":121},"e5ec9764":{"m":122,"g":121},"621dfb88":{"m":122,"g":121},"fb52d35f":{"m":122,"g":121},"2b71531a":{"m":122,"g":121},"25c50498":{"m":122,"g":121},"8e2ac2e6":{"m":122,"g":121},"17a57fd8":{"m":122,"g":121},"32438eba":{"m":122,"g":121},"fed02a49":{"m":122,"g":121},"03b3e89a":{"m":122,"g":121},"9ff9fa7f":{"m":122,"g":121},"7ed8ba05":{"m":122,"g":121},"df08f346":{"m":122,"g":121},"db15148c":{"m":122,"g":121},"5259becd":{"m":122,"g":121},"ed1044ac":{"m":122,"g":121},"d717e73e":{"m":122,"g":121},"a1816187":{"m":122,"g":121},"e39628fd":{"m":122,"g":121},"bacb3825":{"m":122,"g":121},"b53d9e11":{"m":122,"g":121},"8a683821":{"m":122,"g":121},"52694b60":{"m":122,"g":121},"400bddf2":{"m":122,"g":121},"1e90fe2e":{"m":122,"g":121},"caa5d296":{"m":122,"g":121},"750940ae":{"m":122,"g":121},"42f8ea40":{"m":122,"g":121},"14cbe42f":{"m":122,"g":121},"685c0645":{"m":122,"g":121},"1357397a":{"m":122,"g":121},"42e1a72e":{"m":122,"g":121},"83a7c89c":{"m":122,"g":121},"0380ca82":{"m":122,"g":121},"ec92b0ce":{"m":122,"g":121},"e03b6bee":{"m":122,"g":121},"5e36a0b4":{"m":122,"g":121},"0297773a":{"m":122,"g":121},"587deb15":{"m":122,"g":121},"83087247":{"m":122,"g":121},"334543ff":{"m":122,"g":121},"c143f416":{"m":122,"g":121},"b48354c5":{"m":122,"g":121},"29195aaa":{"m":122,"g":121},"0ee831de":{"m":122,"g":121},"8d6ab1cb":{"m":122,"g":121},"84a9d0ea":{"m":122,"g":121},"737b58d6":{"m":122,"g":121},"77225d60":{"m":122,"g":121},"9c6e25d2":{"m":122,"g":121},"2a3763c3":{"m":122,"g":121},"fdd00295":{"m":122,"g":121},"25e73640":{"m":122,"g":121},"0da9845e":{"m":122,"g":121},"92885441":{"m":122,"g":121},"64cf868e":{"m":122,"g":121},"ea399527":{"m":122,"g":121},"41a11335":{"m":122,"g":121},"a1f2dc90":{"m":122,"g":121},"ea961060":{"m":122,"g":121},"b1e13e7c":{"m":122,"g":121},"cc7b04a2":{"m":122,"g":121},"d85d6dba":{"m":122,"g":121},"c5642a7a":{"m":122,"g":121},"691c8534":{"m":122,"g":121},"d2b8c412":{"m":122,"g":121},"bf8f7a94":{"m":122,"g":121},"81a632ac":{"m":122,"g":121},"83b22400":{"m":122,"g":121},"285a8e69":{"m":122,"g":121},"813bd6f8":{"m":122,"g":121},"729f612d":{"m":122,"g":121},"899453ac":{"m":122,"g":121},"ce832d70":{"m":122,"g":121},"88596739":{"m":122,"g":121},"a6ea3add":{"m":122,"g":121},"326c84c4":{"m":122,"g":121},"8da608cc":{"m":122,"g":121},"9fc3e8aa":{"m":122,"g":121},"c11b34d5":{"m":122,"g":121},"05ad28f2":{"m":122,"g":121},"0cae873f":{"m":122,"g":121},"a8b91f6b":{"m":122,"g":121},"959d1ab8":{"m":122,"g":121},"6c1c1933":{"m":122,"g":121},"3029d301":{"m":122,"g":121},"f389f017":{"m":122,"g":121},"caa4819b":{"m":122,"g":121},"a88b006e":{"m":122,"g":121},"ce112c07":{"m":122,"g":121},"f7dc2f33":{"m":122,"g":121},"cd784faf":{"m":122,"g":121},"75c09e1f":{"m":122,"g":121},"09af0a7b":{"m":122,"g":121},"c8d385ce":{"m":122,"g":121},"09938e1f":{"m":123,"g":127},"23407983":{"m":123,"g":127},"1357ab02":{"m":123,"g":127},"44da7377":{"m":123,"g":127},"fb9582c4":{"m":123,"g":127},"d22d0447":{"m":123,"g":127},"887742a1":{"m":123,"g":127},"34f7564d":{"m":123,"g":127},"1cfbbc42":{"m":123,"g":127},"55dfb539":{"m":123,"g":127},"42889acb":{"m":123,"g":127},"211f4070":{"m":123,"g":127},"befa41a1":{"m":123,"g":127},"30b26ee9":{"m":123,"g":127},"aa797d01":{"m":123,"g":127},"7cee07a0":{"m":123,"g":127},"bb517fe3":{"m":123,"g":127},"0e82fd3d":{"m":123,"g":127},"b7d70411":{"m":123,"g":127},"ff0b64e1":{"m":123,"g":127},"d84790db":{"m":123,"g":127},"0678beaa":{"m":123,"g":127},"c2d4716d":{"m":123,"g":127},"c14cc47e":{"m":123,"g":127},"dbcf85b7":{"m":123,"g":127},"83804bc6":{"m":123,"g":127},"d5fa019c":{"m":123,"g":127},"fef3a6b6":{"m":123,"g":127},"173e0f70":{"m":123,"g":127},"f600866a":{"m":123,"g":127},"93be7e86":{"m":123,"g":127},"60b0754c":{"m":123,"g":127},"0b24af4d":{"m":123,"g":127},"a209fb05":{"m":123,"g":127},"48d6bea1":{"m":123,"g":127},"1689c0e3":{"m":123,"g":127},"193fbb0b":{"m":123,"g":127},"e607850f":{"m":123,"g":127},"15efbcb4":{"m":123,"g":127},"243c064d":{"m":123,"g":127},"0b41a293":{"m":123,"g":127},"d31d48b3":{"m":123,"g":127},"88342607":{"m":123,"g":127},"fd7a72d6":{"m":123,"g":127},"21a8fa16":{"m":123,"g":127},"7a21d8b2":{"m":123,"g":127},"d36639ee":{"m":123,"g":127},"6ef23b98":{"m":123,"g":127},"385599cb":{"m":123,"g":127},"952fbe47":{"m":123,"g":127},"edb25693":{"m":123,"g":127},"3529c061":{"m":123,"g":127},"ffb32a85":{"m":123,"g":127},"14d80648":{"m":123,"g":127},"ab8b83f7":{"m":123,"g":127},"de0b10cf":{"m":123,"g":127},"6e29446e":{"m":123,"g":127},"0c3543d7":{"m":123,"g":127},"6a3b9fd0":{"m":123,"g":127},"65f1d065":{"m":123,"g":127},"9434a0e5":{"m":123,"g":127},"20315697":{"m":123,"g":127},"c9db7911":{"m":123,"g":127},"15ed27d7":{"m":123,"g":127},"66fb9b13":{"m":123,"g":127},"819fc591":{"m":123,"g":127},"7efd8b3d":{"m":123,"g":127},"a920b9da":{"m":123,"g":127},"76196b3c":{"m":123,"g":127},"95191ebd":{"m":123,"g":127},"9a512cf9":{"m":123,"g":127},"3451fc32":{"m":123,"g":127},"c550ab91":{"m":123,"g":127},"086f0b79":{"m":123,"g":127},"0afd6832":{"m":123,"g":127},"6f858930":{"m":123,"g":127},"229256c5":{"m":123,"g":127},"6b634493":{"m":123,"g":127},"756ad9ce":{"m":123,"g":127},"d2a8f71c":{"m":123,"g":127},"2b7bf11b":{"m":123,"g":127},"566ade03":{"m":123,"g":127},"69193f71":{"m":123,"g":127},"d5b6e50f":{"m":123,"g":127},"9632e48f":{"m":123,"g":127},"59cce594":{"m":123,"g":127},"795e98f8":{"m":123,"g":127},"358ae356":{"m":123,"g":127},"0c006b88":{"m":124,"g":127},"cd135bfe":{"m":124,"g":127},"fc84b073":{"m":124,"g":127},"8be0e1bc":{"m":124,"g":127},"8e1d6756":{"m":124,"g":127},"0da30dbc":{"m":124,"g":127},"58095cb0":{"m":124,"g":127},"fd3034da":{"m":124,"g":127},"fbbe16fa":{"m":124,"g":127},"6a1a64fa":{"m":124,"g":127},"e2715cf8":{"m":124,"g":127},"74630ba3":{"m":124,"g":127},"73e9a2ef":{"m":124,"g":127},"837b08eb":{"m":124,"g":127},"bb6a21cd":{"m":124,"g":127},"32ec68fa":{"m":124,"g":127},"3c0a6df8":{"m":124,"g":127},"2104d20e":{"m":124,"g":127},"f235498e":{"m":124,"g":127},"149dc9aa":{"m":124,"g":127},"9a954982":{"m":124,"g":127},"97be66c3":{"m":124,"g":127},"74243dff":{"m":124,"g":127},"4ea4c48b":{"m":124,"g":127},"cf5d27e3":{"m":124,"g":127},"9ec6031d":{"m":124,"g":127},"a5affb0c":{"m":124,"g":127},"5925d3d7":{"m":124,"g":127},"7ef1964a":{"m":124,"g":127},"b0476a06":{"m":124,"g":127},"ffba61a1":{"m":124,"g":127},"3c219eb0":{"m":124,"g":127},"1ffdcdc4":{"m":124,"g":127},"c7d57d5b":{"m":124,"g":127},"80802c4c":{"m":124,"g":127},"83b104ee":{"m":124,"g":127},"14127804":{"m":124,"g":127},"82f39dc1":{"m":124,"g":127},"627bac64":{"m":124,"g":127},"3651cfbf":{"m":124,"g":127},"c8547ecd":{"m":124,"g":127},"7bc1dae0":{"m":124,"g":127},"4fe53e58":{"m":124,"g":127},"fb2e816e":{"m":124,"g":127},"7c45b8b4":{"m":124,"g":127},"ba5b6823":{"m":124,"g":127},"508d2f7a":{"m":124,"g":127},"a889c854":{"m":124,"g":127},"4d84f886":{"m":124,"g":127},"dc4f5418":{"m":124,"g":127},"0648eb48":{"m":124,"g":127},"b88fab31":{"m":124,"g":127},"36942660":{"m":124,"g":127},"9f5e7018":{"m":124,"g":127},"cbf23dbb":{"m":124,"g":127},"6dade6c3":{"m":124,"g":127},"b419e20c":{"m":124,"g":127},"48641435":{"m":124,"g":127},"44b1b394":{"m":124,"g":127},"0711d150":{"m":124,"g":127},"303cc957":{"m":125,"g":127},"661c1c97":{"m":125,"g":127},"c022107f":{"m":125,"g":127},"56c83e0f":{"m":125,"g":127},"b51d46d0":{"m":125,"g":127},"10864731":{"m":125,"g":127},"665416f6":{"m":125,"g":127},"838bcb0d":{"m":125,"g":127},"f1f4c451":{"m":125,"g":127},"b0ee99dd":{"m":125,"g":127},"ddfcb7c8":{"m":125,"g":127},"58b12ccb":{"m":125,"g":127},"547de8c7":{"m":125,"g":127},"37c40a87":{"m":125,"g":127},"1240ac13":{"m":125,"g":127},"5639145f":{"m":125,"g":127},"afee2843":{"m":125,"g":127},"6f084880":{"m":125,"g":127},"611a4fd0":{"m":125,"g":127},"9ea2c686":{"m":125,"g":127},"e2a784ec":{"m":125,"g":127},"05559a4a":{"m":125,"g":127},"7bffc5dc":{"m":125,"g":127},"a5e5088d":{"m":125,"g":127},"ac19ce7e":{"m":125,"g":127},"95876d75":{"m":125,"g":127},"a30f1907":{"m":125,"g":127},"dc8a5a1c":{"m":125,"g":127},"e123648b":{"m":125,"g":127},"90401cf7":{"m":125,"g":127},"9cfe78dd":{"m":125,"g":127},"307e7a61":{"m":125,"g":127},"ddd1440d":{"m":125,"g":127},"d1be60c3":{"m":125,"g":127},"583bb180":{"m":125,"g":127},"1f2a6c69":{"m":125,"g":127},"61c7fe7a":{"m":125,"g":127},"83f89cc6":{"m":125,"g":127},"db24d346":{"m":125,"g":127},"885cfca2":{"m":125,"g":127},"4e916f98":{"m":125,"g":127},"4f65a646":{"m":125,"g":127},"f5b3ccd9":{"m":125,"g":127},"bb00e24f":{"m":125,"g":127},"210a9cab":{"m":125,"g":127},"b8ac4fcb":{"m":125,"g":127},"877cb528":{"m":125,"g":127},"3633f8b0":{"m":125,"g":127},"93cf60fc":{"m":125,"g":127},"b5e04173":{"m":125,"g":127},"8a821af7":{"m":125,"g":127},"4b1d163b":{"m":125,"g":127},"c21a3ec2":{"m":125,"g":127},"b142831a":{"m":125,"g":127},"d1340963":{"m":125,"g":127},"f290e801":{"m":125,"g":127},"9299a62f":{"m":125,"g":127},"49543be9":{"m":125,"g":127},"52362903":{"m":125,"g":127},"b2b26d43":{"m":125,"g":127},"d3a03aee":{"m":125,"g":127},"5f02b918":{"m":125,"g":127},"49653c88":{"m":125,"g":127},"44f594d8":{"m":125,"g":127},"2b6c4257":{"m":125,"g":127},"243ea585":{"m":125,"g":127},"6fee2c53":{"m":125,"g":127},"f1a9c72d":{"m":125,"g":127},"190002c6":{"m":125,"g":127},"0296f1cd":{"m":125,"g":127},"e039ff38":{"m":125,"g":127},"b8ddc296":{"m":125,"g":127},"0b88d520":{"m":125,"g":127},"d3d7f960":{"m":125,"g":127},"e4341872":{"m":125,"g":127},"fe19a580":{"m":125,"g":127},"55e8e399":{"m":125,"g":127},"32f79828":{"m":125,"g":127},"0f76976c":{"m":125,"g":127},"ae622790":{"m":125,"g":127},"5c9273c0":{"m":125,"g":127},"e316bcac":{"m":125,"g":127},"0fe9c1f7":{"m":125,"g":127},"61bfd9fa":{"m":125,"g":127},"c67fce16":{"m":125,"g":127},"bef37d6d":{"m":125,"g":127},"bc25ea67":{"m":125,"g":127},"1fa788ec":{"m":125,"g":127},"125f76ea":{"m":125,"g":127},"d8736c75":{"m":125,"g":127},"3b1cc466":{"m":125,"g":127},"0ee5ab5a":{"m":125,"g":127},"ed5e905c":{"m":125,"g":127},"d9c812d8":{"m":125,"g":127},"7257525c":{"m":125,"g":127},"34ba10ef":{"m":125,"g":127},"a119363f":{"m":125,"g":127},"fb314d7b":{"m":125,"g":127},"3a64844a":{"m":125,"g":127},"585c417f":{"m":125,"g":127},"b0d1c21d":{"m":125,"g":127},"b07c5e40":{"m":125,"g":127},"1772671b":{"m":125,"g":127},"6e6009fb":{"m":125,"g":127},"88a2a340":{"m":125,"g":127},"78c58621":{"m":125,"g":127},"c3bb348d":{"m":125,"g":127},"4e234b4c":{"m":125,"g":127},"4cc725ac":{"m":125,"g":127},"2cb42dc1":{"m":125,"g":127},"ebaf86d4":{"m":126,"g":127},"8359f185":{"m":126,"g":127},"5324f37a":{"m":126,"g":127},"2864c49f":{"m":126,"g":127},"d26ec39f":{"m":126,"g":127},"9c546bfd":{"m":126,"g":127},"c9b58164":{"m":126,"g":127},"e5e65e3d":{"m":126,"g":127},"ffeb28ba":{"m":126,"g":127},"b40f605f":{"m":126,"g":127},"3cdec20c":{"m":126,"g":127},"d28caaf6":{"m":126,"g":127},"ad8d24c3":{"m":126,"g":127},"018123b5":{"m":126,"g":127},"4983b7e7":{"m":126,"g":127},"f825137f":{"m":126,"g":127},"ae68158f":{"m":126,"g":127},"a7cc02e3":{"m":126,"g":127},"8ece99a9":{"m":126,"g":127},"44e391b6":{"m":126,"g":127},"1a5c313f":{"m":126,"g":127},"7ea5b42d":{"m":126,"g":127},"5ded5e27":{"m":126,"g":127},"33d1aeb0":{"m":126,"g":127},"dd909a51":{"m":126,"g":127},"60cb7167":{"m":126,"g":127},"151e1368":{"m":126,"g":127},"2f9952cd":{"m":126,"g":127},"3e7cc273":{"m":126,"g":127},"8f01a12d":{"m":126,"g":127},"28b8c579":{"m":126,"g":127},"7b877ab8":{"m":126,"g":127},"0d4a4184":{"m":126,"g":127},"2ca25a8a":{"m":126,"g":127},"cc2e36c3":{"m":126,"g":127},"d8f7816a":{"m":126,"g":127},"99e25805":{"m":126,"g":127},"4a2768a8":{"m":126,"g":127},"9b247f73":{"m":126,"g":127},"36d14712":{"m":126,"g":127},"e0e6a6ef":{"m":126,"g":127},"e8114102":{"m":126,"g":127},"14a339fc":{"m":126,"g":127},"5f662e78":{"m":126,"g":127},"7f5055ed":{"m":126,"g":127},"63728b11":{"m":126,"g":127},"5de25f78":{"m":126,"g":127},"d52800db":{"m":126,"g":127},"38a704bc":{"m":126,"g":127},"a06c44f9":{"m":126,"g":127},"527b7d3f":{"m":126,"g":127},"f09eee03":{"m":126,"g":127},"e38994dd":{"m":126,"g":127},"4a78031a":{"m":126,"g":127},"ea10a9d1":{"m":126,"g":127},"71aea45c":{"m":126,"g":127},"e3b38d71":{"m":126,"g":127},"5c0cadd0":{"m":126,"g":127},"39b1d048":{"m":126,"g":127},"8db7fc41":{"m":126,"g":127},"fe92d4d8":{"m":126,"g":127},"fc8cda14":{"m":126,"g":127},"6a7322ff":{"m":126,"g":127},"c751cb38":{"m":126,"g":127},"3594815a":{"m":126,"g":127},"9caca6a4":{"m":126,"g":127},"08c805a8":{"m":126,"g":127},"f18ec927":{"m":126,"g":127},"aea88fa7":{"m":126,"g":127},"2fe4e69f":{"m":126,"g":127},"012bfc4f":{"m":126,"g":127},"0493775b":{"m":126,"g":127},"40b26b45":{"m":126,"g":127},"9840bf4f":{"m":126,"g":127},"7b2fb3d4":{"m":128,"g":131},"d64dd3e1":{"m":128,"g":131},"3ccd7fa6":{"m":128,"g":131},"254f62d8":{"m":128,"g":131},"2b8b9d84":{"m":128,"g":131},"e970892f":{"m":128,"g":131},"d5fa58c4":{"m":128,"g":131},"9f011f61":{"m":128,"g":131},"3b18fd4c":{"m":128,"g":131},"1869f25c":{"m":128,"g":131},"e019f233":{"m":128,"g":131},"b1c688fb":{"m":128,"g":131},"8e3663d4":{"m":128,"g":131},"9edb0e0d":{"m":128,"g":131},"95f43669":{"m":128,"g":131},"50691d7b":{"m":128,"g":131},"6afe3963":{"m":128,"g":131},"191f5c77":{"m":128,"g":131},"d7246708":{"m":128,"g":131},"efc5d8f5":{"m":128,"g":131},"ef32a252":{"m":128,"g":131},"9509c4cc":{"m":128,"g":131},"4e19c1d5":{"m":128,"g":131},"78a4b446":{"m":128,"g":131},"f9696641":{"m":128,"g":131},"f35f7f12":{"m":128,"g":131},"12c789eb":{"m":128,"g":131},"597d4160":{"m":128,"g":131},"1ca205f6":{"m":128,"g":131},"24a25ffa":{"m":128,"g":131},"db7299aa":{"m":128,"g":131},"13366843":{"m":128,"g":131},"be353ffd":{"m":128,"g":131},"20e59f95":{"m":128,"g":131},"0d116b9a":{"m":128,"g":131},"4a56fa5c":{"m":128,"g":131},"51f9b962":{"m":128,"g":131},"daf494b6":{"m":128,"g":131},"37e8724e":{"m":128,"g":131},"10592e9c":{"m":128,"g":131},"2aec8b6e":{"m":128,"g":131},"0d41ddfb":{"m":128,"g":131},"37c87615":{"m":128,"g":131},"3f400f25":{"m":128,"g":131},"d91b16eb":{"m":128,"g":131},"4a10e37b":{"m":128,"g":131},"b051d76d":{"m":128,"g":131},"d52d992a":{"m":128,"g":131},"d971f228":{"m":128,"g":131},"1d3d42bd":{"m":128,"g":131},"8e9f05ec":{"m":128,"g":131},"bc083521":{"m":128,"g":131},"33f08a98":{"m":128,"g":131},"8e6083bf":{"m":128,"g":131},"2fbc78a0":{"m":128,"g":131},"f0b5ccf5":{"m":128,"g":131},"b732ffa4":{"m":128,"g":131},"56fc4830":{"m":128,"g":131},"2a96e302":{"m":128,"g":131},"8a437340":{"m":128,"g":131},"f0021c0d":{"m":128,"g":131},"7ee3e364":{"m":128,"g":131},"6d5e16fb":{"m":128,"g":131},"67e6f143":{"m":128,"g":131},"34851471":{"m":128,"g":131},"c2083116":{"m":128,"g":131},"10285ec2":{"m":128,"g":131},"9b3fc186":{"m":128,"g":131},"172c71a2":{"m":128,"g":127},"af373636":{"m":128,"g":127},"a5be6ef9":{"m":128,"g":127},"8f4e18a2":{"m":128,"g":127},"e9681444":{"m":128,"g":127},"eae59b33":{"m":128,"g":127},"14dc0523":{"m":128,"g":127},"b2236691":{"m":128,"g":127},"dcc47a56":{"m":128,"g":127},"6448b4cd":{"m":128,"g":127},"5ae0ac42":{"m":128,"g":127},"22f641ab":{"m":128,"g":127},"0997c78d":{"m":128,"g":127},"fd3be107":{"m":128,"g":127},"665f43bd":{"m":128,"g":127},"a53f2d6c":{"m":128,"g":127},"a7002e61":{"m":128,"g":127},"84e151ac":{"m":128,"g":127},"875a25dd":{"m":128,"g":127},"fc55b45e":{"m":128,"g":127},"0050ff25":{"m":128,"g":127},"af9f71f9":{"m":128,"g":127},"15264232":{"m":128,"g":127},"f8d3d80f":{"m":128,"g":127},"5027739f":{"m":128,"g":127},"3701f34d":{"m":128,"g":127},"821fb060":{"m":128,"g":127},"ace27c0c":{"m":128,"g":127},"ed1d18d4":{"m":128,"g":127},"e7b57b0d":{"m":128,"g":127},"49141df9":{"m":128,"g":127},"7b79cc4f":{"m":128,"g":127},"385ff0e5":{"m":128,"g":127},"fc5da1e8":{"m":128,"g":127},"04848ba7":{"m":128,"g":127},"9bc6a9ad":{"m":128,"g":127},"7cdaedb8":{"m":128,"g":127},"1f134f85":{"m":128,"g":127},"5f72d36d":{"m":128,"g":127},"7a2254b2":{"m":128,"g":127},"b5904999":{"m":128,"g":127},"922525ee":{"m":128,"g":127},"ee3e337c":{"m":128,"g":127},"e523e216":{"m":128,"g":127},"2ce23777":{"m":128,"g":127},"19f6a33c":{"m":128,"g":127},"e9c0c558":{"m":128,"g":127},"9b41f31a":{"m":128,"g":127},"9db3add3":{"m":128,"g":127},"c9e5799b":{"m":128,"g":127},"2966367a":{"m":128,"g":127},"87791007":{"m":128,"g":127},"dd192a55":{"m":128,"g":127},"bfe638f7":{"m":128,"g":127},"4ac65e3c":{"m":128,"g":127},"0779c3d1":{"m":128,"g":127},"5c2d72ba":{"m":128,"g":127},"9bd511a5":{"m":128,"g":127},"85b8c5c4":{"m":128,"g":127},"c8b7516f":{"m":128,"g":127},"67e9d287":{"m":128,"g":127},"e7e89349":{"m":128,"g":127},"aead0ef5":{"m":128,"g":127},"66640835":{"m":128,"g":127},"c2d69e8b":{"m":128,"g":127},"4c1e909a":{"m":128,"g":127},"2bb0317e":{"m":128,"g":127},"86255f27":{"m":128,"g":127},"e4b29370":{"m":128,"g":127},"7a8524b4":{"m":128,"g":127},"909d0d38":{"m":128,"g":127},"e42df37d":{"m":128,"g":127},"6d21392b":{"m":128,"g":127},"a1cb717d":{"m":128,"g":127},"c9456491":{"m":128,"g":127},"c4b74c1d":{"m":128,"g":127},"7aa44390":{"m":128,"g":127},"4eda9969":{"m":128,"g":127},"03a7e6f4":{"m":128,"g":127},"2cdde3d4":{"m":128,"g":127},"401ed0c5":{"m":128,"g":127},"4ef43905":{"m":128,"g":127},"d646cf63":{"m":128,"g":127},"4edb2401":{"m":128,"g":127},"706502ff":{"m":128,"g":127},"c2e56dad":{"m":128,"g":127},"9c1c5c6d":{"m":128,"g":127},"2d531946":{"m":128,"g":127},"7ae368ef":{"m":129,"g":131},"c4e20cad":{"m":129,"g":131},"5dad1ff1":{"m":129,"g":131},"ca52ed42":{"m":129,"g":131},"5c03aa3e":{"m":129,"g":131},"253be18e":{"m":129,"g":131},"084b06e7":{"m":129,"g":131},"fc6fb550":{"m":129,"g":131},"7c38eca1":{"m":129,"g":131},"427b08e2":{"m":129,"g":131},"0141ca37":{"m":129,"g":131},"25a6be49":{"m":129,"g":131},"df1f3124":{"m":129,"g":131},"c5947ecd":{"m":129,"g":131},"9530b766":{"m":129,"g":131},"3067b3f0":{"m":129,"g":131},"e0ec42c7":{"m":129,"g":131},"9c9d7091":{"m":129,"g":131},"51a86ce6":{"m":129,"g":131},"64092c8b":{"m":129,"g":131},"63b9300f":{"m":129,"g":131},"21ec99be":{"m":129,"g":131},"383689e3":{"m":129,"g":131},"236a7c23":{"m":129,"g":131},"3dabd609":{"m":129,"g":131},"73df5253":{"m":129,"g":131},"e6420100":{"m":129,"g":131},"c9e20901":{"m":129,"g":131},"106df4ea":{"m":129,"g":131},"427a19b6":{"m":129,"g":131},"1f930cd2":{"m":129,"g":131},"11ce0516":{"m":129,"g":131},"cd4151ab":{"m":129,"g":131},"8fe8b635":{"m":129,"g":131},"8a7b1b83":{"m":129,"g":131},"26aebf83":{"m":129,"g":131},"3ab8ae68":{"m":129,"g":131},"03888b9d":{"m":129,"g":131},"796d82b1":{"m":129,"g":131},"1da59e83":{"m":129,"g":131},"1d66a14c":{"m":129,"g":131},"02af51e4":{"m":129,"g":131},"eb500884":{"m":129,"g":131},"079b1738":{"m":129,"g":131},"57f933fd":{"m":129,"g":131},"1f2b84d2":{"m":129,"g":131},"e7d6027e":{"m":129,"g":131},"07821352":{"m":129,"g":131},"edbeaf3b":{"m":129,"g":131},"d9dca282":{"m":129,"g":131},"9325f945":{"m":129,"g":131},"41b7aab8":{"m":129,"g":131},"491f4fe8":{"m":129,"g":131},"34035d8c":{"m":129,"g":131},"92ca6295":{"m":129,"g":131},"ec92d7f1":{"m":129,"g":131},"79b389da":{"m":129,"g":131},"3de09aad":{"m":129,"g":131},"2e8f54e6":{"m":129,"g":131},"e55731b6":{"m":129,"g":131},"45264554":{"m":129,"g":131},"c4293f59":{"m":129,"g":131},"a2423052":{"m":129,"g":131},"bc3d2a85":{"m":129,"g":131},"d815d002":{"m":129,"g":131},"fa9021b2":{"m":129,"g":131},"9c800728":{"m":129,"g":131},"630a6930":{"m":129,"g":131},"7ce8faae":{"m":129,"g":131},"de153cf7":{"m":129,"g":131},"f4a0c5c7":{"m":129,"g":131},"0f8e5394":{"m":129,"g":131},"e8ba5a66":{"m":129,"g":131},"a2960bdd":{"m":129,"g":131},"487c8d4d":{"m":129,"g":131},"f87b8eab":{"m":129,"g":131},"e8542db5":{"m":129,"g":131},"6df1e8d6":{"m":129,"g":131},"4addb602":{"m":129,"g":131},"bd0e6908":{"m":129,"g":131},"0825d7f4":{"m":129,"g":131},"0b9dbea5":{"m":129,"g":131},"982db4eb":{"m":129,"g":131},"f5f3a5d9":{"m":129,"g":131},"f138ae57":{"m":129,"g":131},"decb4896":{"m":129,"g":131},"c72f0756":{"m":129,"g":131},"f1115cf5":{"m":129,"g":131},"412160f4":{"m":129,"g":131},"7b03cc64":{"m":129,"g":131},"9872a677":{"m":129,"g":131},"c15c864b":{"m":129,"g":131},"dc7bdc73":{"m":129,"g":131},"0a9d6453":{"m":129,"g":131},"340c613a":{"m":129,"g":131},"36b729c2":{"m":129,"g":131},"67e6ef4b":{"m":129,"g":131},"65ba5ab8":{"m":129,"g":131},"990023e5":{"m":129,"g":131},"5ddd2f6b":{"m":129,"g":131},"0ae4b1ad":{"m":129,"g":131},"94cd64a7":{"m":129,"g":131},"b870271a":{"m":129,"g":131},"22ee9b01":{"m":129,"g":131},"9d0e5f1f":{"m":129,"g":131},"1d3d8b34":{"m":129,"g":131},"155a9e72":{"m":129,"g":131},"d7cb08c5":{"m":129,"g":131},"3339c810":{"m":129,"g":131},"d6c88d51":{"m":129,"g":131},"f03ea34a":{"m":129,"g":131},"4cafc835":{"m":129,"g":131},"c6a52f44":{"m":129,"g":131},"c6d34a06":{"m":129,"g":131},"848ee570":{"m":129,"g":131},"ce6b7dfc":{"m":129,"g":131},"0fe74af5":{"m":129,"g":131},"0a362d65":{"m":129,"g":131},"143b57b8":{"m":129,"g":131},"f446b51c":{"m":129,"g":131},"a102a050":{"m":129,"g":131},"6bad6a36":{"m":129,"g":131},"11b6217a":{"m":129,"g":131},"0b0b2607":{"m":129,"g":131},"841eb29d":{"m":129,"g":131},"45cf5758":{"m":129,"g":131},"0e8ce1e8":{"m":129,"g":131},"ea1e9f6b":{"m":129,"g":131},"f6e37d3e":{"m":129,"g":131},"ab9a46d4":{"m":129,"g":131},"621061f0":{"m":129,"g":131},"7daddcdb":{"m":129,"g":131},"91d249cd":{"m":129,"g":131},"95102896":{"m":129,"g":131},"3543a04a":{"m":129,"g":131},"e12c78aa":{"m":129,"g":131},"bce40fa2":{"m":129,"g":131},"051ad833":{"m":129,"g":131},"4c9f7c97":{"m":129,"g":131},"63b05621":{"m":129,"g":131},"21af8e73":{"m":129,"g":131},"7ab548ef":{"m":129,"g":131},"bab033b9":{"m":129,"g":131},"63500426":{"m":129,"g":131},"25758647":{"m":129,"g":131},"2bc8ee8b":{"m":129,"g":131},"ab843ced":{"m":129,"g":131},"6edffc63":{"m":129,"g":131},"077ca70e":{"m":129,"g":131},"7cb04dc0":{"m":129,"g":131},"5443db87":{"m":129,"g":131},"9f340ab1":{"m":129,"g":131},"70c6f951":{"m":129,"g":131},"d941a3be":{"m":129,"g":131},"b12c9e5c":{"m":129,"g":131},"e9e90460":{"m":129,"g":131},"6330d664":{"m":129,"g":131},"91e8dc37":{"m":129,"g":131},"231df4b0":{"m":129,"g":131},"b087ef8b":{"m":129,"g":131},"5155016b":{"m":129,"g":131},"082b54c6":{"m":129,"g":131},"a8ef4d18":{"m":129,"g":131},"15ff6982":{"m":129,"g":131},"9adef42c":{"m":129,"g":131},"685b9d82":{"m":129,"g":131},"a223402f":{"m":129,"g":131},"44d0a848":{"m":129,"g":131},"5b7da0f5":{"m":129,"g":131},"697a77bf":{"m":129,"g":131},"0a186924":{"m":129,"g":131},"b6312e62":{"m":129,"g":131},"779cbc6e":{"m":129,"g":131},"67c8c867":{"m":129,"g":131},"69a03bc3":{"m":129,"g":131},"66f242b9":{"m":129,"g":131},"8a9b8b84":{"m":129,"g":131},"7e964b51":{"m":129,"g":131},"a0d9f6cd":{"m":129,"g":131},"8308cd36":{"m":129,"g":131},"e0e8a996":{"m":129,"g":131},"5102d009":{"m":129,"g":131},"0dd759e0":{"m":129,"g":131},"5e70880e":{"m":129,"g":131},"9dab534b":{"m":129,"g":131},"262c3c1f":{"m":129,"g":131},"6c190cbd":{"m":129,"g":131},"eff6a07c":{"m":129,"g":131},"b704b0a9":{"m":129,"g":131},"15729dbc":{"m":129,"g":131},"21b0582d":{"m":129,"g":131},"5795da5e":{"m":129,"g":131},"540d6fee":{"m":129,"g":131},"5a8adca9":{"m":129,"g":131},"007c3e23":{"m":129,"g":131},"7130ad3a":{"m":129,"g":131},"35a4c21a":{"m":129,"g":131},"846ba3c6":{"m":129,"g":131},"18fb5158":{"m":129,"g":131},"ca5c8b16":{"m":129,"g":131},"f33e5d1e":{"m":129,"g":131},"13e5beea":{"m":129,"g":131},"c53e729d":{"m":129,"g":131},"03a26557":{"m":129,"g":131},"873382a9":{"m":129,"g":131},"391a863b":{"m":129,"g":131},"36b1bcd2":{"m":129,"g":131},"64a11303":{"m":129,"g":131},"1ab6ce0e":{"m":129,"g":131},"e99ca6ac":{"m":129,"g":131},"fcccaf90":{"m":129,"g":131},"215a97fa":{"m":129,"g":131},"5eed5fc0":{"m":129,"g":131},"808b6dfd":{"m":129,"g":131},"4852aa05":{"m":129,"g":131},"f922bfd5":{"m":129,"g":131},"dfd7ab96":{"m":129,"g":131},"46673b42":{"m":129,"g":131},"3421d049":{"m":129,"g":131},"d3d404d3":{"m":129,"g":131},"64225a8a":{"m":129,"g":131},"d64bf6c6":{"m":129,"g":131},"432ecf84":{"m":129,"g":131},"0b3f002d":{"m":129,"g":131},"6f094def":{"m":129,"g":131},"59464dbf":{"m":129,"g":131},"1f7fcc10":{"m":129,"g":131},"dbab5d50":{"m":129,"g":131},"760c20b3":{"m":129,"g":131},"407cb3ce":{"m":129,"g":131},"7cc43bd4":{"m":129,"g":131},"cce2d748":{"m":129,"g":131},"c1dd9a95":{"m":129,"g":131},"ed8786b0":{"m":129,"g":131},"f9fe0630":{"m":129,"g":131},"8ff3ef1f":{"m":129,"g":131},"da182e4b":{"m":129,"g":131},"83e72077":{"m":129,"g":131},"a2c388ba":{"m":129,"g":131},"9384fa27":{"m":129,"g":131},"173e73fa":{"m":129,"g":131},"a164259e":{"m":129,"g":131},"b0a26ba6":{"m":129,"g":131},"de430b67":{"m":129,"g":131},"4b45d556":{"m":129,"g":131},"db0ffc09":{"m":129,"g":131},"eb1d8854":{"m":129,"g":131},"bf108692":{"m":129,"g":131},"e83bd1fa":{"m":129,"g":131},"9dc15d85":{"m":129,"g":131},"fafaa2cc":{"m":129,"g":131},"9b4b3441":{"m":129,"g":131},"94216a9c":{"m":129,"g":131},"9535015d":{"m":129,"g":131},"a3b578fc":{"m":129,"g":131},"b60e769d":{"m":129,"g":131},"a95a3807":{"m":129,"g":131},"98b38de3":{"m":129,"g":131},"a146f833":{"m":129,"g":131},"1dd9a6ae":{"m":129,"g":131},"8ef11569":{"m":129,"g":131},"ecefc790":{"m":129,"g":131},"aeac6220":{"m":129,"g":131},"04b52fa8":{"m":129,"g":131},"e5c0f591":{"m":129,"g":131},"981ca831":{"m":129,"g":131},"414248e0":{"m":129,"g":131},"f56b9b42":{"m":129,"g":131},"b2f7b08c":{"m":129,"g":131},"75222bfe":{"m":129,"g":131},"9ea19533":{"m":129,"g":131},"dbf22152":{"m":129,"g":131},"d5e03468":{"m":129,"g":131},"a22104a6":{"m":129,"g":131},"4683e244":{"m":129,"g":131},"9054e844":{"m":129,"g":131},"18403f6b":{"m":129,"g":131},"2892265d":{"m":129,"g":131},"c9bd1aca":{"m":129,"g":131},"618ca238":{"m":129,"g":131},"5c291549":{"m":129,"g":131},"aaa40a9b":{"m":129,"g":131},"dd70cf99":{"m":129,"g":131},"d4593964":{"m":129,"g":131},"53fffefd":{"m":129,"g":131},"b964ce61":{"m":129,"g":131},"ac5505b0":{"m":129,"g":131},"a90435c0":{"m":129,"g":131},"5354d7b7":{"m":129,"g":131},"e0148677":{"m":129,"g":131},"04793508":{"m":129,"g":131},"cad78789":{"m":129,"g":131},"b29769f3":{"m":129,"g":131},"3990b84b":{"m":129,"g":131},"5a4394a3":{"m":129,"g":131},"dd303614":{"m":129,"g":131},"86312468":{"m":129,"g":131},"5625e32c":{"m":129,"g":131},"ca548d83":{"m":129,"g":131},"3397bcee":{"m":129,"g":131},"3e804bb0":{"m":129,"g":131},"a22de641":{"m":129,"g":131},"ac438226":{"m":129,"g":131},"a92afb00":{"m":129,"g":131},"0eea17e3":{"m":129,"g":131},"38052432":{"m":129,"g":131},"8bfce9b0":{"m":129,"g":131},"b41afa37":{"m":129,"g":131},"94ae816f":{"m":129,"g":131},"53620a1b":{"m":129,"g":131},"59b4d7f8":{"m":129,"g":131},"a56f7702":{"m":129,"g":131},"1b48e1b9":{"m":129,"g":131},"a24aefe5":{"m":129,"g":131},"45c572c5":{"m":129,"g":131},"681b9e64":{"m":129,"g":131},"dab06b50":{"m":129,"g":131},"85ffce30":{"m":129,"g":131},"e94ef9fc":{"m":129,"g":131},"964cdedc":{"m":129,"g":131},"aa6e2c8a":{"m":129,"g":131},"1776dce5":{"m":129,"g":131},"99e13d18":{"m":129,"g":131},"dc836909":{"m":129,"g":131},"323fed5c":{"m":129,"g":131},"eff7df6d":{"m":129,"g":131},"5e7f91d4":{"m":129,"g":131},"a34d3abb":{"m":129,"g":131},"6d0e0b9b":{"m":129,"g":131},"589d9ad5":{"m":129,"g":131},"a244c030":{"m":129,"g":131},"1bb063aa":{"m":129,"g":131},"b30f63c4":{"m":129,"g":131},"90a01335":{"m":129,"g":131},"43602790":{"m":129,"g":131},"b537ac0d":{"m":129,"g":131},"eda2f700":{"m":129,"g":131},"8c212a20":{"m":129,"g":131},"d754ce97":{"m":129,"g":131},"475962a1":{"m":129,"g":131},"bfcf15a1":{"m":129,"g":131},"fb04d434":{"m":129,"g":131},"6be65ae4":{"m":129,"g":131},"750084ae":{"m":129,"g":131},"64480ec7":{"m":129,"g":131},"db2d362d":{"m":129,"g":131},"81e86992":{"m":129,"g":131},"c4db77f8":{"m":129,"g":131},"c0a2513b":{"m":129,"g":131},"3ae664d7":{"m":129,"g":131},"c56fc424":{"m":129,"g":131},"3f1cfd87":{"m":129,"g":131},"b5344b31":{"m":129,"g":131},"6b262ac8":{"m":129,"g":131},"ada8ce1f":{"m":129,"g":131},"5a2c7039":{"m":129,"g":131},"42028af6":{"m":129,"g":131},"7291c72e":{"m":129,"g":131},"6bc30628":{"m":129,"g":131},"fa924410":{"m":129,"g":131},"fc9efdcb":{"m":129,"g":131},"2dec555d":{"m":129,"g":131},"acde21d8":{"m":129,"g":131},"4528cb7d":{"m":129,"g":131},"a352e833":{"m":129,"g":131},"852eb6ce":{"m":129,"g":131},"7af9b88c":{"m":129,"g":131},"2847e5c4":{"m":129,"g":131},"c8ede0e9":{"m":129,"g":131},"19729f72":{"m":129,"g":131},"b51f9bbe":{"m":129,"g":131},"4a8442af":{"m":129,"g":131},"7dcf910d":{"m":129,"g":131},"c7b37b70":{"m":129,"g":131},"10e0b83a":{"m":129,"g":131},"bc42c8c4":{"m":129,"g":131},"2e3a69ae":{"m":129,"g":131},"21370ef7":{"m":129,"g":131},"c3c4da71":{"m":129,"g":131},"dc694624":{"m":129,"g":131},"48ca9f75":{"m":129,"g":131},"127d59cd":{"m":129,"g":131},"af6bcadc":{"m":129,"g":131},"67fca6b2":{"m":129,"g":131},"f88b2aa6":{"m":129,"g":131},"17b24aca":{"m":129,"g":131},"bfaf0b86":{"m":129,"g":131},"83756a4b":{"m":129,"g":131},"a3557949":{"m":129,"g":131},"e72cf136":{"m":129,"g":131},"196b940a":{"m":129,"g":131},"a1e1e533":{"m":129,"g":131},"d4a4dcdf":{"m":129,"g":131},"97ba2c2d":{"m":129,"g":131},"e197bef5":{"m":129,"g":131},"8900f996":{"m":129,"g":131},"ba9102f9":{"m":129,"g":131},"b638abba":{"m":129,"g":131},"37980559":{"m":129,"g":131},"075ba74d":{"m":129,"g":131},"f7be98e1":{"m":129,"g":131},"9a1a9a42":{"m":129,"g":131},"6c2e5fcd":{"m":129,"g":131},"0d2d6878":{"m":129,"g":131},"9f59194f":{"m":129,"g":131},"cf1f0166":{"m":129,"g":131},"10969ae4":{"m":129,"g":131},"92ad2ff9":{"m":129,"g":131},"9b64f6f3":{"m":129,"g":131},"c0d1a338":{"m":129,"g":131},"a9d22b75":{"m":129,"g":131},"6b9459e8":{"m":129,"g":131},"3a6ec47b":{"m":129,"g":131},"b8e32e79":{"m":129,"g":131},"f5566acc":{"m":129,"g":131},"d79e1294":{"m":129,"g":131},"6e9b1549":{"m":129,"g":131},"109f27ba":{"m":129,"g":131},"c1a30aa7":{"m":129,"g":131},"2e1dbdb2":{"m":129,"g":131},"6d025fd3":{"m":129,"g":131},"518467be":{"m":129,"g":131},"63807079":{"m":129,"g":131},"f6cfe9f1":{"m":129,"g":131},"7bc99d41":{"m":129,"g":131},"e2d67468":{"m":129,"g":131},"6beb6e99":{"m":129,"g":131},"7e88b9c1":{"m":129,"g":131},"cfcf2758":{"m":129,"g":131},"ac81db66":{"m":129,"g":131},"33905005":{"m":129,"g":131},"820e13c9":{"m":129,"g":131},"595adf6d":{"m":129,"g":131},"a5ad0069":{"m":129,"g":131},"4e41edcb":{"m":129,"g":131},"67071f55":{"m":129,"g":131},"0c966779":{"m":129,"g":131},"f3386077":{"m":129,"g":131},"4ce8fb3c":{"m":129,"g":131},"4c3573e4":{"m":129,"g":131},"f1be8aa0":{"m":129,"g":131},"aa8ecbda":{"m":129,"g":131},"9188fecc":{"m":129,"g":131},"26ca0746":{"m":129,"g":131},"90c18a16":{"m":129,"g":131},"d7984f31":{"m":129,"g":131},"7119d188":{"m":129,"g":131},"fe3bbfb4":{"m":129,"g":131},"9846f8ed":{"m":129,"g":131},"85ae508e":{"m":129,"g":131},"d879e37f":{"m":129,"g":131},"e2c9a590":{"m":129,"g":131},"a1e37b02":{"m":129,"g":131},"e389f91d":{"m":129,"g":131},"aac07bf7":{"m":129,"g":131},"ea89a3a0":{"m":129,"g":131},"2bc7c5eb":{"m":129,"g":131},"a63f433b":{"m":129,"g":131},"58f8f4e4":{"m":129,"g":131},"25acbbc6":{"m":129,"g":131},"a8fcbf6f":{"m":129,"g":131},"e486308c":{"m":129,"g":131},"60420109":{"m":129,"g":131},"ff00b6ad":{"m":129,"g":131},"7b445260":{"m":129,"g":131},"c236d05f":{"m":129,"g":131},"df561392":{"m":129,"g":131},"15db5497":{"m":129,"g":131},"9ba3597d":{"m":129,"g":131},"80797c2a":{"m":129,"g":131},"7afff8fd":{"m":129,"g":131},"ac406d43":{"m":129,"g":131},"b436113f":{"m":129,"g":131},"8b5e2c53":{"m":129,"g":131},"b24235b8":{"m":129,"g":131},"ae7698fb":{"m":129,"g":131},"a3e4fe4b":{"m":129,"g":131},"f3e9336d":{"m":129,"g":131},"290fcd89":{"m":129,"g":131},"1dcde539":{"m":129,"g":131},"15bc1f5c":{"m":129,"g":131},"4d597616":{"m":129,"g":131},"ab63f3c5":{"m":129,"g":131},"147b7823":{"m":129,"g":131},"d368c745":{"m":129,"g":131},"7e626d12":{"m":129,"g":131},"2a577344":{"m":129,"g":131},"6abb8051":{"m":130,"g":131},"2e3946d8":{"m":130,"g":131},"32f8b606":{"m":130,"g":131},"b9bef31a":{"m":130,"g":131},"8550822d":{"m":130,"g":131},"8810152e":{"m":130,"g":131},"7bf16c63":{"m":130,"g":131},"39f9a9c2":{"m":130,"g":131},"d69ecc19":{"m":130,"g":131},"763888b5":{"m":130,"g":131},"9a327bdf":{"m":130,"g":131},"2de98010":{"m":130,"g":131},"8200fb56":{"m":130,"g":131},"cb4cdb43":{"m":130,"g":131},"80cfca50":{"m":130,"g":131},"7871593c":{"m":130,"g":131},"4a62a0e3":{"m":130,"g":131},"12a08efc":{"m":130,"g":131},"06836ad0":{"m":130,"g":131},"f72a7703":{"m":130,"g":131},"aeff0d38":{"m":130,"g":131},"cf0478d6":{"m":130,"g":131},"a2ca9bd4":{"m":130,"g":131},"36361adc":{"m":130,"g":131},"661e9775":{"m":130,"g":131},"2970f229":{"m":130,"g":131},"c08b780f":{"m":130,"g":131},"8fbf7dd5":{"m":130,"g":131},"1915a1f8":{"m":130,"g":131},"85d0ccfa":{"m":130,"g":131},"a4ffd665":{"m":130,"g":131},"559202b5":{"m":130,"g":131},"f57d4fe7":{"m":130,"g":131},"b7b7524e":{"m":130,"g":131},"6799847e":{"m":130,"g":131},"03b835e7":{"m":130,"g":131},"aff1238e":{"m":130,"g":131},"5e2cda61":{"m":130,"g":131},"b0bbc7f5":{"m":130,"g":131},"673c11ba":{"m":130,"g":131},"3b47973a":{"m":130,"g":131},"f6423b62":{"m":130,"g":131},"84efe54b":{"m":130,"g":131},"948b6ace":{"m":130,"g":131},"f124539a":{"m":130,"g":131},"c8683ae3":{"m":130,"g":131},"125e17ef":{"m":130,"g":131},"88c459c6":{"m":130,"g":131},"ae6a6630":{"m":130,"g":131},"26d95008":{"m":130,"g":131},"f2b5dcc9":{"m":130,"g":131},"9abcab3f":{"m":130,"g":131},"e5135b73":{"m":130,"g":131},"41d61faa":{"m":130,"g":131},"3c7886ec":{"m":130,"g":131},"0e4d8790":{"m":130,"g":131},"6d5d76ad":{"m":130,"g":131},"91c9c14c":{"m":130,"g":131},"32a32cf7":{"m":130,"g":131},"be4a3ec3":{"m":130,"g":131},"ff6e3ea9":{"m":130,"g":131},"dd91d38e":{"m":130,"g":131},"5f6f550a":{"m":130,"g":131},"5edbe351":{"m":130,"g":131},"d2b42477":{"m":130,"g":131},"9dfa01a4":{"m":130,"g":131},"e592ee65":{"m":130,"g":131},"cee93a6f":{"m":130,"g":131},"bc388471":{"m":130,"g":131},"3e40c636":{"m":130,"g":131},"80122e4f":{"m":130,"g":131},"e12c6b32":{"m":130,"g":131},"6d417918":{"m":130,"g":131},"35a9a073":{"m":130,"g":131},"ea177372":{"m":130,"g":131},"42fcf543":{"m":130,"g":131},"d257bf87":{"m":130,"g":131},"7b0c7ad1":{"m":130,"g":131},"d30d6b36":{"m":130,"g":131},"d881f314":{"m":130,"g":131},"2ac5b983":{"m":130,"g":131},"a0dde90a":{"m":130,"g":131},"b988c18e":{"m":130,"g":131},"e41664ba":{"m":130,"g":131},"3d1b591a":{"m":130,"g":131},"e11f795f":{"m":130,"g":131},"959a1746":{"m":130,"g":131},"b72f0268":{"m":130,"g":131},"09376fd7":{"m":130,"g":131},"aed835e3":{"m":130,"g":131},"49dfa1d8":{"m":130,"g":131},"1ea6b740":{"m":130,"g":131},"e73173b0":{"m":130,"g":131},"16e8463a":{"m":130,"g":131},"cf9a774c":{"m":130,"g":131},"ec7b2c16":{"m":130,"g":131},"1569fc7f":{"m":130,"g":131},"5a46fb15":{"m":130,"g":131},"38daa294":{"m":130,"g":131},"66984a8b":{"m":130,"g":131},"889b46ea":{"m":130,"g":131},"05284378":{"m":130,"g":131},"a8904560":{"m":130,"g":131},"66280987":{"m":130,"g":131},"205f041e":{"m":130,"g":131},"7235a7fb":{"m":130,"g":131},"8fce9e7b":{"m":130,"g":131},"53477322":{"m":130,"g":131},"2ce121a1":{"m":130,"g":131},"35ba6fe1":{"m":130,"g":131},"498ea41c":{"m":130,"g":131},"7c744d13":{"m":130,"g":131},"46b05ef5":{"m":130,"g":131},"beec8eed":{"m":130,"g":131},"b76e303e":{"m":130,"g":131},"80a575e4":{"m":130,"g":131},"4c5074eb":{"m":130,"g":131},"41429a8c":{"m":130,"g":131},"532037df":{"m":130,"g":131},"fa0ca976":{"m":130,"g":131},"b5d39985":{"m":130,"g":131},"2ecee757":{"m":130,"g":131},"6d37e708":{"m":130,"g":131},"c1006fd8":{"m":130,"g":131},"29c6c2ea":{"m":130,"g":131},"eb85fa6d":{"m":130,"g":131},"0e6441b4":{"m":130,"g":131},"88d1bab5":{"m":130,"g":131},"922756aa":{"m":130,"g":131},"d8faf2f3":{"m":130,"g":131},"7dfcc781":{"m":130,"g":131},"7f3308bc":{"m":130,"g":131},"fdc2ef58":{"m":130,"g":131},"1808df48":{"m":130,"g":131},"b01fc161":{"m":130,"g":131},"788628b5":{"m":130,"g":131},"11d33c0e":{"m":130,"g":131},"441420e1":{"m":130,"g":131},"29a2d4b5":{"m":130,"g":131},"079ac237":{"m":130,"g":131},"84280784":{"m":130,"g":131},"af35023e":{"m":130,"g":131},"cb8df87f":{"m":130,"g":131},"e3ab23c1":{"m":130,"g":131},"70d25873":{"m":130,"g":131},"894c0dc5":{"m":130,"g":131},"d6c49019":{"m":130,"g":131},"fa78c44a":{"m":130,"g":131},"654a78f9":{"m":130,"g":131},"f90b4004":{"m":130,"g":131},"78647e08":{"m":130,"g":131},"46f21a59":{"m":130,"g":131},"b2b09f5f":{"m":130,"g":131},"04df80a9":{"m":130,"g":131},"4f73e53d":{"m":130,"g":131},"16ff892c":{"m":130,"g":131},"df026bb1":{"m":130,"g":131},"d42c167b":{"m":130,"g":131},"38815105":{"m":130,"g":131},"9d823402":{"m":130,"g":131},"8ab5d8b4":{"m":130,"g":131},"03575ce3":{"m":130,"g":131},"80518bea":{"m":130,"g":131},"7e78825d":{"m":130,"g":131},"5bbd83a2":{"m":130,"g":131},"abf6272b":{"m":130,"g":131},"46d7b35e":{"m":130,"g":131},"20aad5b5":{"m":130,"g":131},"974c562a":{"m":130,"g":131},"aca0d01d":{"m":130,"g":131},"dc163502":{"m":130,"g":131},"24903b88":{"m":130,"g":131},"443d7bcd":{"m":130,"g":131},"16d8de22":{"m":130,"g":131},"d122e324":{"m":130,"g":131},"96cc1083":{"m":130,"g":131},"77512ae0":{"m":130,"g":131},"c233e9d7":{"m":130,"g":131},"58ac3f31":{"m":130,"g":131},"93452a82":{"m":130,"g":131},"65c8568c":{"m":130,"g":131},"4bcc5879":{"m":130,"g":131},"d5ea8c71":{"m":130,"g":131},"42271376":{"m":130,"g":131},"84e0abb7":{"m":130,"g":131},"043f1317":{"m":130,"g":131},"f764c691":{"m":130,"g":131},"92205407":{"m":130,"g":131},"7d1a130c":{"m":130,"g":131},"5c8bd8b5":{"m":132,"g":133},"b05b346a":{"m":132,"g":133},"5c961756":{"m":132,"g":133},"c5f1e861":{"m":132,"g":133},"2c4d376d":{"m":132,"g":133},"d0f756ae":{"m":132,"g":133},"6f99dc97":{"m":132,"g":133},"cd1c1fa5":{"m":132,"g":133},"5b0872d2":{"m":132,"g":133},"ba88f1ca":{"m":132,"g":133},"60560c07":{"m":132,"g":133},"ca114421":{"m":132,"g":133},"543d62d1":{"m":132,"g":133},"27032cec":{"m":132,"g":133},"5d804a37":{"m":132,"g":133},"388018a5":{"m":132,"g":133},"fca8e88f":{"m":132,"g":133},"45eeeb9a":{"m":132,"g":133},"a368df28":{"m":132,"g":133},"f85460fb":{"m":132,"g":133},"e52cf30e":{"m":132,"g":133},"a076d75e":{"m":132,"g":133},"8348725d":{"m":132,"g":133},"28566241":{"m":132,"g":133},"b62fe850":{"m":132,"g":133},"624725cb":{"m":132,"g":133},"1a96e664":{"m":132,"g":133},"32829b16":{"m":132,"g":133},"e54307f2":{"m":132,"g":133},"8642dbe4":{"m":132,"g":133},"7dcad45c":{"m":132,"g":133},"7c985331":{"m":132,"g":133},"bd7824b2":{"m":132,"g":133},"312df1d6":{"m":132,"g":133},"25e97380":{"m":132,"g":133},"b6523a4f":{"m":132,"g":133},"c51efb8b":{"m":132,"g":133},"bcc5483e":{"m":132,"g":133},"ccf26027":{"m":132,"g":133},"a4992873":{"m":132,"g":133},"c97ce391":{"m":132,"g":133},"c032b559":{"m":132,"g":133},"da9b801e":{"m":132,"g":133},"0e54a695":{"m":132,"g":133},"e99ee0c6":{"m":132,"g":133},"c1bd5ee8":{"m":132,"g":133},"ef1ab230":{"m":132,"g":133},"d6598737":{"m":132,"g":133},"6c5ebc0e":{"m":132,"g":133},"5b5571a8":{"m":132,"g":133},"1698c234":{"m":132,"g":133},"3d82c0f1":{"m":132,"g":133},"617e9b3b":{"m":132,"g":133},"83e35a7c":{"m":132,"g":133},"2543666c":{"m":132,"g":133},"f732f8ea":{"m":132,"g":133},"503880db":{"m":132,"g":133},"b8cfa02c":{"m":132,"g":133},"d85fecb5":{"m":132,"g":133},"6634f67b":{"m":132,"g":133},"5eccaf77":{"m":132,"g":133},"d7f6320b":{"m":132,"g":133},"766476f5":{"m":132,"g":133},"12b7a4fa":{"m":132,"g":133},"02f1e81e":{"m":132,"g":133},"908c7186":{"m":132,"g":133},"03836d85":{"m":132,"g":133},"87dbdddc":{"m":132,"g":133},"56e5c074":{"m":132,"g":133},"6c9c8da6":{"m":132,"g":133},"21028b55":{"m":132,"g":133},"b0a25d09":{"m":132,"g":133},"793c98af":{"m":132,"g":133},"b1cbfce6":{"m":132,"g":133},"b0f531ad":{"m":132,"g":133},"01835998":{"m":132,"g":133},"4285e99d":{"m":132,"g":133},"f0774368":{"m":132,"g":133},"5e8f544d":{"m":132,"g":133},"c8d74feb":{"m":132,"g":133},"cbc7dcda":{"m":132,"g":133},"a6dc7d29":{"m":132,"g":133},"390406c4":{"m":132,"g":131},"0c63fb94":{"m":132,"g":131},"9ad02b79":{"m":132,"g":131},"18bd8e8d":{"m":132,"g":131},"036e64da":{"m":132,"g":131},"7c6fb3aa":{"m":132,"g":131},"8b0b6a45":{"m":132,"g":131},"55504df2":{"m":132,"g":131},"73df7a4e":{"m":132,"g":131},"8b98bb76":{"m":132,"g":131},"15bc8cbd":{"m":132,"g":131},"9496f12d":{"m":132,"g":131},"ab004879":{"m":132,"g":131},"6ec77680":{"m":132,"g":131},"13680e55":{"m":132,"g":131},"98c430e1":{"m":132,"g":131},"fe7f91ef":{"m":132,"g":131},"cef5ba65":{"m":132,"g":131},"53d17088":{"m":132,"g":131},"f0e948a0":{"m":132,"g":131},"9a426fc5":{"m":132,"g":131},"66772aa2":{"m":132,"g":131},"b6263344":{"m":132,"g":131},"0f8bd55f":{"m":132,"g":131},"da3dc497":{"m":132,"g":131},"817daba0":{"m":132,"g":131},"af60cad0":{"m":132,"g":131},"ce4e836b":{"m":132,"g":131},"0e0b0c05":{"m":132,"g":131},"e6f0ddda":{"m":132,"g":131},"af20657c":{"m":132,"g":131},"08da4c26":{"m":132,"g":131},"ef3f8c97":{"m":132,"g":131},"e5201bda":{"m":132,"g":131},"60d36e7b":{"m":132,"g":131},"6f657070":{"m":132,"g":131},"c106b54b":{"m":132,"g":131},"119fd956":{"m":132,"g":131},"eac5b664":{"m":132,"g":131},"07404d76":{"m":132,"g":131},"93043f7b":{"m":132,"g":131},"edde5e5d":{"m":132,"g":131},"232982a0":"m134","9e88c0a2":"m134","b7e0d54e":"m134","5dde0a57":"m134","9e5ab903":"m134","98225be6":{"m":134,"g":135},"94bcc19b":{"m":134,"g":135},"db3821a9":{"m":134,"g":135},"8c5d91b8":{"m":134,"g":135},"6a3e7092":{"m":134,"g":135},"c2601f0d":{"m":134,"g":135},"1048803c":{"m":134,"g":135},"8a84b1e7":{"m":134,"g":135},"5e20e7a6":{"m":134,"g":135},"3946dad6":{"m":134,"g":135},"ac03ec08":{"m":134,"g":135},"94164646":{"m":134,"g":135},"5fb734f1":{"m":134,"g":135},"60f1ca69":{"m":134,"g":135},"9e263c21":{"m":134,"g":135},"0d003e34":{"m":134,"g":135},"f253f43c":{"m":134,"g":135},"1e453201":{"m":134,"g":135},"26e17f90":{"m":134,"g":135},"3de23274":{"m":134,"g":135},"f4ec6f8e":{"m":134,"g":135},"9c4eb460":{"m":134,"g":135},"c2e0913e":{"m":134,"g":135},"8c6f865a":{"m":134,"g":135},"269aa27b":{"m":134,"g":135},"f39382c6":{"m":134,"g":135},"2ff289e2":{"m":134,"g":135},"b2a3f055":{"m":134,"g":135},"684e148e":{"m":134,"g":135},"f44c4b37":{"m":134,"g":135},"7380ec9d":{"m":134,"g":135},"ac78f96e":{"m":134,"g":135},"278012ca":{"m":134,"g":135},"7f587998":{"m":134,"g":135},"162d1cf9":{"m":134,"g":135},"8e08207c":{"m":134,"g":135},"c31f6272":{"m":134,"g":135},"b5d9fc87":{"m":134,"g":135},"88f3de25":{"m":134,"g":135},"24616c52":{"m":134,"g":135},"d48723b7":{"m":134,"g":135},"f3d73b01":{"m":134,"g":135},"4ab66d95":{"m":134,"g":135},"de2799f3":{"m":134,"g":135},"f784cbfa":{"m":134,"g":135},"09733090":{"m":134,"g":135},"a44d0079":{"m":134,"g":135},"8305dc17":{"m":134,"g":135},"ec8c831d":{"m":134,"g":135},"f13949e5":{"m":134,"g":135},"c236a3fd":{"m":134,"g":135},"41a1d16b":{"m":134,"g":135},"9884c9fd":{"m":134,"g":135},"a435f55d":{"m":134,"g":135},"2ec6fa3c":{"m":134,"g":135},"c58a573a":{"m":134,"g":135},"b840d6aa":{"m":134,"g":135},"ef4b3c0e":{"m":134,"g":135},"6f9d0a89":{"m":134,"g":135},"d7a3336e":{"m":134,"g":135},"7d02c8e5":{"m":134,"g":135},"e6d5a213":{"m":134,"g":135},"9f8e2307":{"m":134,"g":135},"d90f9bfc":{"m":134,"g":135},"3881bc8d":{"m":134,"g":135},"208e6a9d":{"m":134,"g":135},"be3828a1":{"m":134,"g":135},"5969be2f":{"m":134,"g":135},"8fab4895":{"m":134,"g":135},"7ccaec64":{"m":134,"g":135},"c457aad5":{"m":134,"g":135},"c7e7bfa3":{"m":134,"g":135},"bf90ea9c":{"m":134,"g":135},"26c50912":{"m":134,"g":135},"325a4c19":{"m":134,"g":135},"0294844f":{"m":134,"g":135},"0e536600":{"m":134,"g":135},"d70c2655":{"m":134,"g":135},"656f4d69":{"m":134,"g":135},"8e43980e":{"m":134,"g":135},"474a4699":{"m":134,"g":135},"b4a00ed2":{"m":134,"g":135},"f55d608c":{"m":134,"g":135},"349ce2dd":{"m":134,"g":135},"183b6519":{"m":134,"g":135},"2af955e1":{"m":134,"g":135},"0cd2b719":{"m":134,"g":135},"39d56196":{"m":134,"g":135},"41addd2e":{"m":134,"g":135},"5c393e81":{"m":134,"g":135},"3645ed0f":{"m":134,"g":135},"ca740a41":{"m":134,"g":135},"0e25aa43":{"m":134,"g":135},"60a230b1":{"m":134,"g":135},"aa89c6a7":{"m":134,"g":135},"171912a9":{"m":134,"g":135},"faecd37e":{"m":134,"g":135},"9ad546d7":{"m":134,"g":135},"29ce7b36":{"m":134,"g":135},"a8380ded":{"m":134,"g":135},"4edee695":{"m":134,"g":135},"67caea6f":{"m":134,"g":135},"cd3289c7":{"m":134,"g":135},"acddb8e0":{"m":134,"g":135},"2ec57cef":{"m":134,"g":135},"988b14ca":{"m":134,"g":135},"93495dca":{"m":134,"g":135},"886e0383":{"m":134,"g":135},"01bd0d3e":{"m":134,"g":135},"43e1bbc0":{"m":134,"g":135},"59b12996":{"m":134,"g":135},"b7091496":{"m":134,"g":135},"51dbdb22":{"m":134,"g":135},"8dc6f0fc":{"m":134,"g":135},"cf34d0ab":{"m":134,"g":135},"3778c2fc":{"m":134,"g":135},"5d421db8":{"m":134,"g":135},"fe3d47fc":{"m":134,"g":135},"ef92b4eb":{"m":134,"g":135},"73c0c66f":{"m":134,"g":135},"a1e9b4ed":{"m":134,"g":135},"e75657c8":{"m":134,"g":135},"c28c536c":{"m":134,"g":135},"086813ae":{"m":134,"g":135},"3fd232ad":{"m":134,"g":135},"cb181295":{"m":134,"g":135},"a91e072f":{"m":134,"g":135},"0271fc34":{"m":134,"g":135},"68bece8c":{"m":134,"g":135},"7b7e357f":{"m":134,"g":135},"2f66b067":{"m":134,"g":135},"48051181":{"m":134,"g":135},"f2ccc442":{"m":134,"g":135},"caa95c7e":{"m":134,"g":135},"9d878c1f":{"m":134,"g":135},"8087ef12":{"m":134,"g":135},"6ef543f9":{"m":134,"g":135},"a3559119":{"m":134,"g":135},"f3ba7116":{"m":134,"g":135},"5c243ba5":{"m":134,"g":135},"b6702d72":{"m":134,"g":135},"c1256727":{"m":134,"g":135},"bb9e6cdf":{"m":134,"g":135},"de03b0cd":{"m":134,"g":135},"2a8a7856":{"m":134,"g":135},"f4e835af":{"m":134,"g":135},"e6ce16a4":{"m":134,"g":135},"de2f2880":{"m":134,"g":135},"a89e85e7":{"m":134,"g":135},"cbf9f134":{"m":134,"g":135},"eb3da9c1":{"m":134,"g":135},"72a980c6":{"m":134,"g":135},"ccf2330b":{"m":134,"g":135},"b9af8d2e":{"m":134,"g":135},"10a9573e":{"m":134,"g":135},"49ab72f8":{"m":134,"g":135},"8865424f":{"m":134,"g":135},"b311c43d":{"m":134,"g":135},"0c39730b":{"m":134,"g":135},"45adad37":{"m":134,"g":135},"1ba897f3":{"m":134,"g":135},"38dd4fbb":{"m":134,"g":135},"ecd2d09a":{"m":134,"g":135},"92ddc468":{"m":134,"g":135},"5454d2a7":{"m":134,"g":133},"17b38f88":{"m":134,"g":133},"ae434f78":{"m":134,"g":133},"643aeefe":{"m":134,"g":133},"186a56f6":{"m":134,"g":133},"17e65466":{"m":134,"g":133},"370bd27f":{"m":134,"g":133},"b27b5a83":{"m":134,"g":133},"2f7c6292":{"m":134,"g":133},"2fb31605":{"m":134,"g":133},"8bf7f240":{"m":134,"g":133},"2c5679f3":{"m":134,"g":133},"159b1283":{"m":134,"g":133},"9338f63f":{"m":134,"g":133},"8196998a":{"m":134,"g":133},"aa21c6e3":{"m":134,"g":133},"fd4a558e":{"m":134,"g":133},"b3b818fd":{"m":134,"g":133},"d5fbbfd9":{"m":134,"g":133},"e245cac0":{"m":134,"g":133},"c6a6ba43":{"m":134,"g":133},"d6108166":{"m":134,"g":133},"ddb3970e":{"m":134,"g":133},"f65fa047":{"m":134,"g":133},"7e027691":{"m":134,"g":133},"cb719c74":{"m":134,"g":133},"ff903a7e":{"m":134,"g":133},"eee3700d":{"m":134,"g":133},"e254cdf3":{"m":134,"g":133},"dfb53574":{"m":134,"g":133},"96655749":{"m":134,"g":133},"ac320a6f":{"m":134,"g":133},"6292c244":{"m":134,"g":133},"5f5a5677":{"m":134,"g":133},"3bf07c68":{"m":134,"g":133},"e7b09efc":{"m":134,"g":133},"99d3bcdf":{"m":134,"g":133},"4d64f150":{"m":134,"g":133},"aef7ca7c":{"m":134,"g":133},"fe712aa3":{"m":134,"g":133},"846953d9":{"m":134,"g":133},"5c64a20d":{"m":134,"g":133},"cf817376":{"m":134,"g":133},"aa6ac966":{"m":134,"g":133},"0d0367e9":{"m":134,"g":133},"bd572360":{"m":134,"g":133},"5f3a47d8":{"m":134,"g":133},"80ae2229":{"m":134,"g":133},"705287b2":{"m":134,"g":133},"76284653":{"m":134,"g":133},"dd620987":{"m":134,"g":133},"53f974b9":{"m":134,"g":133},"6a5764a7":{"m":134,"g":133},"291f11ae":{"m":134,"g":133},"d7301c89":{"m":134,"g":133},"758b9067":{"m":134,"g":133},"ffc23ef8":{"m":134,"g":133},"b3f83cc1":{"m":134,"g":133},"c15fa1c5":{"m":134,"g":133},"fa296698":{"m":134,"g":133},"f9dd90ac":{"m":134,"g":133},"66902e0f":{"m":134,"g":133},"e50f356f":{"m":134,"g":133},"ac42797c":{"m":134,"g":133},"883747ce":{"m":134,"g":133},"989d4b30":{"m":134,"g":133},"bc3ca300":{"m":134,"g":133},"061f41af":{"m":134,"g":133},"82f1d615":{"m":134,"g":133},"5e1a495c":{"m":134,"g":133},"34013d9d":{"m":134,"g":133},"77597167":{"m":134,"g":133},"3c882db3":{"m":134,"g":133},"b736a152":{"m":134,"g":133},"6984837d":{"m":134,"g":133},"2142881b":{"m":134,"g":133},"d77f3fcc":{"m":134,"g":133},"828dec1c":{"m":134,"g":133},"575a49dc":{"m":134,"g":133},"e62e1744":{"m":134,"g":133},"d5431ff8":{"m":134,"g":133},"454a2544":{"m":134,"g":133},"89619a99":{"m":134,"g":133},"cb30d056":{"m":134,"g":133},"677930c2":{"m":134,"g":133},"beae3f96":{"m":134,"g":133},"1167867e":{"m":134,"g":133},"ad7f35fb":{"m":134,"g":133},"f4100732":{"m":134,"g":133},"468931b5":{"m":134,"g":133},"796969ca":{"m":134,"g":133},"122c2503":{"m":134,"g":133},"a3a55223":{"m":134,"g":133},"0bf95e6d":{"m":134,"g":133},"a92de891":{"m":134,"g":133},"b9d78605":{"m":134,"g":133},"1354063a":{"m":134,"g":133},"254de6d2":{"m":134,"g":133},"350fbbf4":{"m":134,"g":133},"a3912667":{"m":134,"g":133},"e1dcd0df":{"m":134,"g":133},"393e2f9b":{"m":134,"g":133},"8766a1dd":{"m":134,"g":133},"1d9ba2ce":{"m":134,"g":133},"ef001fb8":{"m":134,"g":133},"c69c1c4f":{"m":134,"g":133},"bed301a5":{"m":134,"g":133},"8fe3e374":{"m":134,"g":133},"60143655":{"m":134,"g":133},"fc05acc2":{"m":134,"g":133},"1ed94668":{"m":134,"g":133},"d7fbe73b":{"m":134,"g":133},"42bff706":{"m":134,"g":133},"26704c23":{"m":134,"g":133},"43b7c174":{"m":134,"g":133},"96740d69":{"m":134,"g":133},"9a3bdf2c":{"m":134,"g":133},"4b351f6b":{"m":134,"g":133},"7fa4906f":{"m":134,"g":133},"050f108c":{"m":134,"g":133},"47cdb65a":{"m":134,"g":133},"1d90b194":{"m":134,"g":133},"537ef18d":{"m":134,"g":133},"69412ccb":{"m":134,"g":133},"d3885d4b":{"m":134,"g":133},"0a346d3b":{"m":134,"g":133},"bc18cb86":{"m":134,"g":133},"bee8ac5b":{"m":134,"g":133},"41bd76e1":{"m":134,"g":133},"c6ca1b3a":{"m":134,"g":133},"8999ce75":{"m":134,"g":133},"dce2ed44":{"m":134,"g":133},"019517a3":{"m":134,"g":133},"1f1f05a8":{"m":134,"g":133},"165f5c04":{"m":134,"g":133},"3e01f3a5":{"m":134,"g":133},"51e2eaa4":{"m":134,"g":133},"6468cb58":{"m":134,"g":133},"e220da17":{"m":134,"g":133},"b82c7a0a":{"m":134,"g":133},"c0f9b519":{"m":134,"g":133},"5529ab58":{"m":134,"g":133},"74a3349b":{"m":134,"g":133},"b9ebf0ed":{"m":134,"g":133},"3c116d5e":{"m":134,"g":133},"71a60288":{"m":134,"g":133},"61405b3d":{"m":134,"g":133},"0adfc42b":{"m":134,"g":133},"ba72e759":{"m":134,"g":133},"6afc5d49":{"m":134,"g":133},"2ee6c810":{"m":134,"g":133},"bd16244d":{"m":134,"g":133},"b5eb0214":{"m":134,"g":133},"d72e908b":{"m":134,"g":133},"50cad014":{"m":134,"g":133},"ef908aeb":{"m":134,"g":133},"9d0347b3":{"m":134,"g":133},"05eb0bcc":{"m":134,"g":133},"5dccd9bd":{"m":134,"g":133},"933cef16":{"m":134,"g":133},"241ae17b":{"m":134,"g":133},"2c5a4460":{"m":134,"g":133},"f3705b01":{"m":134,"g":133},"5a0ad731":{"m":134,"g":133},"5045aa34":{"m":134,"g":133},"ff1e2ce2":{"m":134,"g":133},"1e582488":{"m":134,"g":133},"46be74b4":{"m":134,"g":133},"1c658026":{"m":134,"g":133},"ba410808":{"m":134,"g":133},"89512029":{"m":134,"g":133},"92e6b3c3":{"m":134,"g":133},"6559e43f":{"m":134,"g":133},"af780c59":{"m":134,"g":133},"a21aa87e":{"m":134,"g":133},"fb178457":{"m":134,"g":133},"65c09859":{"m":134,"g":133},"4bf06635":{"m":134,"g":133},"f2d64e67":{"m":134,"g":133},"0e869f08":{"m":134,"g":133},"17394092":{"m":134,"g":133},"f228b662":{"m":134,"g":133},"a36142aa":{"m":134,"g":133},"160a06ca":{"m":134,"g":133},"a0985dd5":{"m":134,"g":133},"f6c9db4b":{"m":134,"g":133},"e88e75a9":{"m":134,"g":133},"4b4050e2":{"m":134,"g":133},"b2803ff2":{"m":134,"g":133},"9e0ef04e":{"m":134,"g":133},"216067c0":{"m":134,"g":133},"e72b02db":{"m":134,"g":133},"29e8f7f9":{"m":134,"g":133},"e0963a6c":{"m":134,"g":133},"e0026f7c":{"m":134,"g":133},"9749d3e3":{"m":134,"g":133},"2b0ddf89":{"m":134,"g":133},"17e81c75":{"m":134,"g":133},"88a405cc":{"m":134,"g":133},"602fe3b2":{"m":134,"g":133},"ad9616f1":{"m":134,"g":133},"c5f4e20f":{"m":134,"g":133},"2c196f95":{"m":134,"g":133},"9a7641d7":{"m":134,"g":133},"793c96c3":{"m":134,"g":133},"d1f00632":{"m":134,"g":133},"4792d1f4":{"m":134,"g":133},"fea2d521":{"m":134,"g":133},"56d12b4a":{"m":134,"g":133},"374ad4cc":{"m":134,"g":133},"8b0a68f1":{"m":134,"g":133},"70607e55":{"m":134,"g":133},"ee1ca51d":{"m":134,"g":133},"ef7c29ac":{"m":134,"g":133},"58c840db":{"m":134,"g":133},"3d42b7e7":{"m":134,"g":133},"9970ee34":{"m":134,"g":133},"9e7656be":{"m":134,"g":133},"9d4f066f":{"m":134,"g":133},"41683536":{"m":134,"g":133},"891ee822":{"m":134,"g":133},"8fa3dc36":{"m":134,"g":133},"d20699a3":{"m":134,"g":133},"169a75df":{"m":134,"g":133},"4128d4f5":{"m":134,"g":133},"011d8d89":{"m":134,"g":133},"5290cef9":{"m":134,"g":133},"726fe3e7":{"m":134,"g":133},"5d087891":{"m":134,"g":133},"8451e227":{"m":134,"g":133},"53e15194":{"m":134,"g":133},"d747147a":{"m":134,"g":133},"0c002207":{"m":134,"g":133},"eeb2b9b2":{"m":134,"g":133},"c4aed389":{"m":134,"g":133},"9d04b570":{"m":134,"g":133},"3e690cce":{"m":134,"g":133},"b12b40de":{"m":134,"g":133},"6c4bf8a0":{"m":134,"g":133},"533851fb":{"m":134,"g":133},"0071fe9c":{"m":134,"g":133},"ffa7e035":{"m":134,"g":133},"712f44ee":{"m":134,"g":133},"8c34e181":{"m":134,"g":133},"e9abb525":{"m":134,"g":133},"88859433":{"m":134,"g":133},"feb8e30b":{"m":134,"g":133},"45a959d3":{"m":134,"g":133},"cdce5163":{"m":134,"g":133},"79ab57bd":{"m":134,"g":133},"4b8901ac":{"m":134,"g":133},"435d1c83":{"m":134,"g":133},"2bdbaef1":{"m":134,"g":133},"31d48d7f":{"m":134,"g":133},"0129c911":{"m":134,"g":133},"03f9eb25":{"m":134,"g":133},"7ec678eb":{"m":134,"g":133},"9d64a7b2":{"m":134,"g":133},"46ad4b98":{"m":134,"g":133},"71cb9037":{"m":134,"g":133},"c8c64876":{"m":134,"g":133},"0861dca8":{"m":134,"g":133},"da58df6b":{"m":134,"g":133},"49237e26":{"m":134,"g":133},"d92c1f8c":{"m":134,"g":133},"93070586":{"m":134,"g":133},"ccc8f3b2":{"m":134,"g":133},"a4c76281":{"m":134,"g":133},"28a19e49":{"m":134,"g":133},"0261c4af":{"m":134,"g":133},"8ac350f3":{"m":134,"g":133},"99401e7b":{"m":134,"g":133},"66824751":{"m":134,"g":133},"f95729b0":{"m":134,"g":133},"9f4ed93d":{"m":134,"g":133},"ecb401ed":{"m":134,"g":133},"b399e3ac":{"m":134,"g":133},"e27635a0":{"m":134,"g":133},"272c5fe4":{"m":134,"g":133},"3c8dc448":{"m":134,"g":133},"5e96beb3":{"m":134,"g":133},"9327482b":{"m":134,"g":133},"36fcf71f":{"m":134,"g":133},"6292d971":{"m":134,"g":133},"c8434195":{"m":134,"g":133},"3e4d431a":{"m":134,"g":133},"538e733e":{"m":134,"g":133},"22587bc0":{"m":134,"g":133},"02d24244":{"m":134,"g":133},"1da5cd63":{"m":134,"g":133},"a9a2cdd8":{"m":134,"g":133},"4733fcff":{"m":134,"g":133},"3ffa2604":{"m":134,"g":133},"61f362c6":{"m":134,"g":133},"e7157c9b":{"m":134,"g":133},"30da2f05":{"m":134,"g":133},"49016931":{"m":134,"g":133},"3d484be5":{"m":134,"g":133},"c0d94440":{"m":134,"g":133},"1dedb638":{"m":134,"g":133},"7bc8b153":{"m":134,"g":133},"9003a436":{"m":134,"g":133},"3518b331":{"m":134,"g":133},"b098b1ae":{"m":134,"g":133},"abd3e048":{"m":134,"g":133},"92c29d43":{"m":134,"g":133},"bf643814":{"m":134,"g":133},"f03bfa4c":{"m":134,"g":133},"89ad3908":{"m":134,"g":133},"2ea844ec":{"m":134,"g":133},"1e2d7538":{"m":134,"g":133},"d16ff357":{"m":134,"g":133},"af49e302":{"m":134,"g":133},"01b955ac":{"m":134,"g":133},"16e6bc20":{"m":134,"g":133},"37250764":{"m":134,"g":133},"7b9156c7":{"m":134,"g":133},"21cfebac":{"m":134,"g":133},"702426b0":{"m":134,"g":133},"9e9a6169":{"m":134,"g":133},"bd9c3a47":{"m":134,"g":133},"1e641ee4":{"m":134,"g":133},"fb96669f":{"m":134,"g":133},"3912ee49":{"m":134,"g":133},"1ab9b8e0":{"m":134,"g":133},"e61dabf5":{"m":134,"g":133},"36e7c8c5":{"m":134,"g":133},"037c3982":{"m":134,"g":133},"62b3fdae":{"m":134,"g":133},"1cd0c3bf":{"m":134,"g":133},"2c899431":{"m":134,"g":133},"4513f549":{"m":134,"g":133},"c9690307":{"m":134,"g":133},"4449c170":{"m":134,"g":133},"5ca962ce":{"m":134,"g":133},"bab20a84":{"m":134,"g":133},"0e4108ba":{"m":134,"g":133},"99cb2ed9":{"m":134,"g":133},"0612175c":{"m":134,"g":133},"8c96fcda":{"m":134,"g":133},"ea7c69ce":{"m":134,"g":133},"8102e36b":{"m":134,"g":133},"3f048217":{"m":134,"g":133},"b11af135":{"m":134,"g":133},"f9bceea0":{"m":134,"g":133},"997ea57e":{"m":134,"g":133},"47633c19":{"m":134,"g":133},"64b5c3ab":{"m":134,"g":133},"d277a86d":{"m":134,"g":133},"9acb21ae":{"m":134,"g":133},"a9ce1623":{"m":134,"g":133},"6f0c77d7":{"m":134,"g":133},"e3f51e82":{"m":134,"g":133},"fdfabb7a":{"m":134,"g":133},"19c16748":{"m":134,"g":133},"5c75907e":{"m":134,"g":133},"4ea36422":{"m":134,"g":133},"f50af32d":{"m":134,"g":133},"72952919":{"m":134,"g":133},"2ae5bed1":{"m":134,"g":133},"54df514b":{"m":134,"g":133},"681c68cf":{"m":134,"g":133},"74ea45cc":{"m":134,"g":133},"0fa044ad":{"m":134,"g":133},"7d8e42c9":{"m":134,"g":133},"6abdf73f":{"m":134,"g":133},"20ce9938":{"m":134,"g":133},"168a31eb":{"m":134,"g":133},"69cfb17b":{"m":134,"g":133},"a7a4b175":{"m":134,"g":133},"fdc93b01":{"m":134,"g":133},"fd37cc5d":{"m":134,"g":133},"96705514":{"m":134,"g":133},"ab3ffd1c":{"m":134,"g":133},"2285afff":{"m":134,"g":133},"a81cc1b8":{"m":134,"g":133},"3134d2b2":{"m":134,"g":133},"06b58c5d":{"m":134,"g":133},"ea07a283":{"m":134,"g":133},"ea91a720":{"m":134,"g":133},"5d9c6bac":{"m":134,"g":133},"e048ee90":{"m":134,"g":133},"ed52d01b":{"m":134,"g":133},"90e7d4f7":{"m":134,"g":133},"c20d43d2":{"m":134,"g":133},"d977dd2e":{"m":134,"g":133},"0c23331e":{"m":134,"g":133},"993278b4":{"m":134,"g":133},"9e9d9107":{"m":134,"g":133},"3b8a824b":{"m":134,"g":133},"f1bbd26f":{"m":134,"g":133},"d36299ad":{"m":134,"g":133},"0e7d7969":{"m":134,"g":133},"80554598":{"m":134,"g":133},"b2e240bc":{"m":134,"g":133},"dcc5f5c0":{"m":134,"g":133},"4eda4194":{"m":134,"g":133},"875f84db":{"m":134,"g":133},"f6031adf":{"m":134,"g":133},"2a39cfe0":{"m":134,"g":133},"bf17e769":{"m":134,"g":133},"9a5d6a84":{"m":134,"g":133},"31c23e5f":{"m":134,"g":133},"06617a9e":{"m":134,"g":133},"e79ca959":{"m":134,"g":133},"9d3b411c":{"m":134,"g":133},"05325db3":{"m":134,"g":133},"01e3b3f3":{"m":134,"g":133},"86988674":{"m":134,"g":133},"29139654":{"m":134,"g":133},"665cb020":{"m":134,"g":133},"71602838":{"m":134,"g":133},"77873343":{"m":134,"g":133},"267170bf":{"m":134,"g":133},"313f59ad":{"m":134,"g":133},"487cf81a":{"m":134,"g":133},"df111bc0":{"m":134,"g":133},"6d2b3324":{"m":134,"g":133},"8cc77261":{"m":134,"g":133},"d143b020":{"m":134,"g":133},"9b9d2131":{"m":134,"g":133},"44fd7017":{"m":134,"g":133},"9a56273a":{"m":134,"g":133},"b737a125":{"m":134,"g":133},"1b5e9034":{"m":134,"g":133},"171b442a":{"m":134,"g":133},"4b7b5af3":{"m":134,"g":133},"ec242f51":{"m":134,"g":133},"b2431546":{"m":134,"g":133},"526fd008":{"m":134,"g":133},"56d0ad47":{"m":134,"g":133},"306e5b8d":{"m":134,"g":133},"10c68f62":{"m":134,"g":133},"c7c837cd":{"m":134,"g":133},"3e1e7157":{"m":134,"g":133},"8fa8d9d7":{"m":134,"g":133},"c8cf1caf":{"m":134,"g":133},"d71baa72":{"m":134,"g":133},"82e33170":{"m":134,"g":133},"94e12511":{"m":134,"g":133},"4dabfbc8":{"m":134,"g":133},"d7ed8a8c":{"m":134,"g":133},"c05d3afb":{"m":134,"g":133},"edb172e9":{"m":134,"g":133},"fe6d38d2":{"m":134,"g":133},"dab31e4c":{"m":134,"g":133},"a7fa31ff":{"m":134,"g":133},"bd91f882":{"m":134,"g":133},"b47adb80":{"m":134,"g":133},"b2b5bdb0":{"m":134,"g":133},"22fe5da1":{"m":134,"g":133},"1834401e":{"m":134,"g":133},"198c8ecf":{"m":134,"g":133},"c01b2ee0":{"m":134,"g":133},"8f5adac8":{"m":134,"g":133},"76743a98":{"m":134,"g":133},"8bf10e71":{"m":134,"g":133},"e4873d04":{"m":134,"g":133},"c660d8df":{"m":134,"g":133},"10146af0":{"m":134,"g":133},"b62e7e3b":{"m":134,"g":133},"6ce36b12":{"m":134,"g":133},"e9e7f15e":{"m":134,"g":133},"0aa3dec5":{"m":134,"g":133},"4885f8b9":{"m":134,"g":133},"f832994c":{"m":134,"g":133},"e59435c3":{"m":134,"g":133},"d6bd2d11":{"m":134,"g":133},"9975acf5":{"m":134,"g":133},"70758d45":{"m":134,"g":133},"fd1ebbb0":{"m":134,"g":133},"3d98bd5e":{"m":134,"g":133},"aa3716b2":{"m":134,"g":133},"6107268f":{"m":134,"g":133},"0189f41c":"m136","b6e4893a":"m136","3eb7da53":"m136","cb53ddc9":"m136","0c265321":"m136","6469c964":"m136","71705394":"m136","67589c16":"m136","a702c8f1":"m136","f2ae066a":"m136","0c2993ee":"m136","590969ee":"m136","e6ccb294":"m136","d6ea2c52":"m136","9be2a3a9":"m136","b74a57a8":"m136","95f59c13":"m136","85d9af51":"m136","858f317f":"m136","cf893516":"m136","1fdf5cac":"m136","cda43ffa":"m136","39089854":"m136","b827e9d3":"m136","d725487d":"m136","a95c9f5b":"m136","19089aa4":"m136","4f6f5d25":"m136","458fe5a3":"m136","2ff0880a":"m136","2c1b164a":"m136","bcc6d84f":"m136","a618202f":"m136","7520b929":"m136","e7224e96":"m136","e776239a":"m136","1b97fa76":"m136","236772c0":"m136","0d49b13f":"m136","0a7a2017":"m136","0a9099e1":"m136","0050c476":"m136","8251a74d":"m136","a54d75bf":"m136","3321eb4e":"m136","be5121b4":"m136","c3f9c30f":"m136","54a82179":"m136","aca354bc":"m136","aea57b33":"m136","823a046e":"m136","648aab0c":"m136","1e309030":"m136","60927215":"m136","38c233fd":"m136","20ed3822":"m136","4ecd9afd":"m136","eb38d644":"m136","6ea491e4":"m136","16802fb6":"m136","d97066d2":"m136","ce2d686e":"m136","76b06bee":"m136","612026ad":"m136","55c61642":"m136","91a4cd86":"m136","23d765d1":"m136","db2425a0":"m136","603f386c":"m136","f7a5e425":"m136","d50dcd9b":"m136","e6b7c049":"m136","a3addd62":"m136","6988a0f5":"m136","1b192cf1":"m136","c560e142":"m136","71cb9d03":"m136","8fb45523":"m136","84aef378":"m136","17c04b10":"m136","55c4288b":"m136","9f8b79f1":"m136","7dc3cbe7":"m136","79ddc34c":"m136","09a9d214":"m136","7e40d526":"m136","057b07fc":"m136","91d8c52d":"m136","e9a44ea6":"m136","c1282da2":"m136","71279e31":"m136","1a053a81":"m136","2ea02f06":"m136","20b0523e":"m136","ce8a6ac6":"m136","ebca5879":"m136","cc410a10":"m136","f374623f":"m136","5c022177":"m136","8916b9d0":"m136","a3d9a218":"m136","fb88fb67":"m136","2d72e168":"m136","64946679":"m136","5836324c":"m136","9fe56cd0":"m136","858a4d65":"m136","e619f531":"m136","fc4b932f":"m136","d2105d4a":"m136","84c83905":"m136","ea879c77":"m136","0227db89":"m136","ad1b4e47":"m136","51f147ad":"m136","d3eafc73":"m136","e00b4344":"m136","733de6be":"m136","93433726":"m136","330605cc":"m136","bb6055b4":"m136","4df74eb5":"m136","f3a7c7dc":"m136","2069050d":"m136","1fe0c82f":"m136","a45e0e5d":"m136","f78201f3":"m136","8fd33998":"m136","088758c1":"m136","6d29d8ab":"m136","e499258e":"m136","09491a9b":"m136","7edb0615":"m136","e486a4da":"m136","90399cbc":"m136","53609e5e":"m136","9c253064":"m136","d2c86387":"m136","737a1183":"m136","dc743fe4":"m136","dd99f818":"m136","8ce64aa1":"m136","eb768189":"m136","2cdd4370":"m136","a7b5f75d":"m136","305c1a57":"m136","c824ddd5":"m136","e18e0057":"m136","166396ca":"m136","43779f27":"m136","8b9e9357":"m136","d36f6f04":"m136","b0701f02":"m136","4229de3b":"m136","2e144079":"m136","d2ec128b":"m136","7f8353af":"m136","a7f5677a":"m136","3e968ab3":"m136","b4fce995":"m136","ec9b48ea":"m136","a0467589":"m136","82a1b645":"m136","6f10e17b":"m136","c771933d":"m136","9d8bbd42":"m136","daea5138":"m136","3355b6e2":"m136","a1dd3d48":"m136","d9ed80b9":"m136","7c39ea68":"m136","daa4841e":"m136","968c4f55":"m136","669d309a":"m136","21ee597e":"m136","6ee970a3":"m136","8ec160ed":"m136","2740ed1a":"m136","d44f09ad":"m136","e7dc85c5":"m136","c81bad1b":"m136","0e86de7c":"m136","e3a95077":"m136","146b5fcc":"m136","8b22deef":"m136","69822c72":"m136","8b99af9a":"m136","72e2f70e":"m136","3d72944f":"m136","7dde3438":"m136","77fc4c4a":"m136","655d2c7c":"m136","3f44268f":"m136","f7ec8174":"m136","cd23c2f0":"m136","d1110e1c":"m136","9227d9f6":"m136","4c59782e":"m136","dda35ccb":"m136","d11e2dc6":"m136","16831ab6":"m136","c9a45b7e":"m136","e7df8bdc":"m136","e9979950":"m136","6586f44a":"m136","43fe3a4d":"m136","98096b5e":"m136","68e8d0f6":"m136","000ad422":"m136","9d5f16d4":"m136","7f8a58ff":"m136","6b065298":"m136","c020d300":"m136","4346db5f":"m136","424a3800":"m136","f0918583":"m136","5b1215d9":"m136","aa2b4f76":"m136","b3a3f513":"m136","de94d793":"m136","0d904ef4":"m136","969faaa4":"m136","48c2aca9":"m136","5af84c8a":"m136","feae615b":"m136","e75299a1":"m136","72bacc88":"m136","ba625c2d":"m136","b025cff4":"m136","030496eb":"m136","c86ca128":"m136","cd336945":"m136","c5e363e8":"m136","9479eca7":"m136","a5348eac":"m136","95240402":"m136","2122fea3":"m136","e2c8a50b":"m136","a4825ed5":"m136","afe285f7":"m136","cf25852a":"m136","5938c3b0":"m136","b8806071":"m136","339915ce":"m136","075c5a57":"m136","a0b4ba90":"m136","2a7b67ad":"m136","1d811094":"m136","2ab3ed3e":"m136","250477d2":"m136","af1232b2":"m136","7a869045":"m136","888d7e54":"m136","3cb1fbae":"m136","ba9f6d8f":"m136","740d3c0b":"m136","87165898":"m136","ff3ddb9d":"m136","9d3018f4":"m136","a8348427":"m136","47d485f3":"m136","2b423099":"m136","ae0baefb":"m136","1f0e3d7f":"m136","d3c08fb0":"m136","c6a64e9f":"m136","e0ac559a":"m136","6620548f":"m136","6e158e55":"m136","ed729d22":"m136","fa51b854":"m136","559ff9ec":"m136","7b682de8":"m136","d0092dec":"m136","76f69b77":"m136","9a628744":"m136","53dca74f":"m136","2dadf635":"m136","aab640c9":"m136","2b3791ed":"m136","9f5cd80a":"m136","b1ee75ae":"m136","f44c63ee":"m136","c54c70ab":"m136","aab906a3":"m136","feb39f77":"m136","38b30c7b":"m136","c581b5ed":"m136","a1c48943":"m136","5b7bed7c":"m136","38a88479":"m136","503c3d95":"m136","934ae89a":"m136","cf1426a7":"m136","17cb3c8e":"m136","2f4a6add":"m136","f9fc50ac":"m136","3c16c586":"m136","7b089ae4":"m136","8b5d4263":"m136","b5493f65":"m136","09e2571e":"m136","c0248d6f":"m136","cc25f9df":"m136","cf14feba":"m136","7c25687c":"m136","ff978142":"m136","d112f6a2":"m136","a2c2c09d":"m136","2a9344d3":"m136","78c41758":"m136","206db66f":"m136","5c72be1e":"m136","76d48817":"m136","d1ec93e3":"m136","bdb76b34":"m136","dae6a409":"m136","a0899bdb":"m136","641830c1":"m136","3fd88ea9":"m136","145bd54f":"m136","2d088b85":"m136","3a8b44fe":"m136","aeb480c1":"m136","9fd2358c":"m136","3c358736":"m136","6327dff2":"m136","675acece":"m136","ad201273":"m136","7f393d95":"m136","4b14f622":"m136","94fc26aa":"m136","67b61a4e":"m136","d27f16f3":"m136","c89949bb":"m136","20abaee2":"m136","32a569fb":"m136","3ed3b7ef":"m136","1f9d4795":"m136","e6d40bff":"m136","fbc128a3":"m136","9c64a15a":"m136","e91a7176":"m136","1f0ea4f9":"m136","15da3061":"m136","6406a596":"m136","cec19b56":"m136","ef35d8fe":"m136","08636f72":"m136","a6c29d4c":"m136","84ab32a2":"m136","70667115":"m136","bd1afeb5":"m136","8ef5b905":"m136","76b3c698":"m136","5dcff947":"m136","8eeffbe9":"m136","71e9c31c":"m136","7656d267":"m136","dd24ba90":"m136","068abe7e":"m136","64a31d4b":"m136","2babf88f":"m136","d56d14e5":"m136","0c4e155a":"m136","d6d5c3fd":"m136","e46f7943":"m136","9d4d57db":"m136","41609b52":"m136","ccd0fb32":"m136","87ee6b5e":"m136","f7c1d24b":"m136","cceb5e6a":"m136","c1c13c84":"m136","fcec35dc":"m136","75da784d":"m136","77d35665":"m136","05dfef92":"m136","74602407":{"m":136,"g":135},"f7f5c389":{"m":136,"g":135},"9e3a032a":{"m":136,"g":135},"1bc7aa58":{"m":136,"g":135},"8726d30c":{"m":136,"g":135},"a1b243d7":{"m":136,"g":135},"a9799277":{"m":136,"g":135},"ee71e773":{"m":136,"g":135},"55a8dd00":{"m":136,"g":135},"cf242321":{"m":136,"g":135},"9d03af91":{"m":136,"g":135},"064ae341":{"m":136,"g":135},"49305fa1":{"m":136,"g":135},"4e999404":{"m":136,"g":135},"fbc24886":{"m":136,"g":135},"16880235":{"m":136,"g":135},"cda35611":{"m":136,"g":135},"8a45a9c6":{"m":136,"g":135},"aecd5f5f":{"m":136,"g":135},"05ab110e":{"m":136,"g":135},"c8dc4d2d":{"m":136,"g":135},"82a8d77b":{"m":136,"g":135},"294ff71d":{"m":136,"g":135},"f52ae586":{"m":136,"g":135},"2f8a3634":{"m":136,"g":135},"b6e8a0d8":{"m":136,"g":135},"fb7609f1":{"m":136,"g":135},"d2ea44f7":{"m":136,"g":135},"20ca2c6e":{"m":136,"g":135},"83abecd0":{"m":136,"g":135},"d54f0a10":{"m":136,"g":135},"fb04e7e3":{"m":136,"g":135},"1e5de05e":{"m":136,"g":135},"7dd679cb":{"m":136,"g":135},"3d51ae18":{"m":136,"g":135},"f9c04266":{"m":136,"g":135},"48b8dcd4":{"m":136,"g":135},"41b434a7":{"m":136,"g":135},"4935344f":{"m":136,"g":135},"63cc97f4":{"m":136,"g":135},"ab7d5829":{"m":136,"g":135},"261860e1":{"m":136,"g":135},"154740bd":{"m":136,"g":135},"1c09cbe3":{"m":136,"g":135},"e14f5ec8":{"m":136,"g":135},"8867d248":{"m":136,"g":135},"6b3f93c4":{"m":136,"g":135},"5a5cece5":{"m":136,"g":135},"d566739b":{"m":136,"g":135},"4c46ecde":{"m":136,"g":135},"12a0292b":{"m":136,"g":135},"a08dc5aa":{"m":136,"g":135},"eec7dbd3":{"m":136,"g":135},"109fe03a":{"m":136,"g":135},"bb798a1c":{"m":136,"g":135},"38dc5839":{"m":136,"g":135},"5e867f60":{"m":136,"g":135},"65bed838":{"m":136,"g":135},"156d97b2":{"m":136,"g":135},"24b30f77":{"m":136,"g":135},"3a4767da":{"m":136,"g":135},"6037267f":{"m":136,"g":135},"0241e046":{"m":136,"g":135},"0c474273":{"m":136,"g":135},"3e73e124":{"m":136,"g":135},"f4742558":{"m":136,"g":135},"7385834c":{"m":136,"g":135},"b5a94f8a":{"m":136,"g":135},"c356ed03":{"m":136,"g":135},"ee4d2287":{"m":136,"g":135},"153c69f6":{"m":136,"g":135},"55b79365":{"m":136,"g":135},"7fc12e0b":{"m":136,"g":135},"8729ad5e":{"m":136,"g":135},"e4320573":{"m":136,"g":135},"4d902c82":{"m":136,"g":135},"2ff87231":{"m":136,"g":135},"fd16c91c":{"m":136,"g":135},"32a6540a":{"m":136,"g":135},"62d0280f":{"m":136,"g":135},"d4b717c0":{"m":136,"g":135},"98a107d4":{"m":136,"g":135},"b86bbf84":{"m":136,"g":135},"48381c3b":{"m":136,"g":135},"8bce0853":{"m":136,"g":135},"6b8a9d70":{"m":136,"g":135},"973116e6":{"m":136,"g":135},"820e97d6":{"m":136,"g":135},"4c85f9d0":{"m":136,"g":135},"7d757d6f":{"m":136,"g":135},"5c04088b":{"m":136,"g":135},"f066036c":{"m":136,"g":135},"52de807d":{"m":136,"g":135},"70933f34":{"m":136,"g":135},"ce453fa4":{"m":136,"g":135},"3be1e734":{"m":136,"g":135},"38895a00":{"m":136,"g":135},"d8b81981":{"m":136,"g":135},"913b688f":{"m":136,"g":135},"951d16c8":{"m":136,"g":135},"53846746":{"m":136,"g":135},"534ac384":{"m":136,"g":135},"2a8d5493":{"m":136,"g":135},"90eac38a":{"m":136,"g":135},"badcd028":{"m":136,"g":135},"d874c8bb":{"m":136,"g":135},"9a21d89c":{"m":136,"g":135},"4c9ac856":{"m":136,"g":135},"fb5b71d0":{"m":136,"g":135},"05b54b6d":{"m":136,"g":135},"4f443f44":{"m":136,"g":135},"dce8b060":{"m":136,"g":135},"399d5283":{"m":136,"g":135},"2e0527dd":{"m":136,"g":135},"6beb50d6":{"m":136,"g":135},"18e2ef09":{"m":136,"g":135},"3271e0e7":{"m":136,"g":135},"d415d22d":{"m":136,"g":135},"a49b9a64":{"m":136,"g":135},"53497642":{"m":136,"g":135},"d57d8e7e":{"m":136,"g":135},"0cbd8f32":{"m":136,"g":135},"6e3fff13":{"m":136,"g":135},"2210155a":{"m":136,"g":135},"a3656cbb":{"m":136,"g":135},"21da2dc1":{"m":136,"g":135},"ed307a40":{"m":136,"g":135},"02722b91":{"m":136,"g":135},"f959250f":{"m":136,"g":135},"95934379":{"m":136,"g":135},"bc2f40be":{"m":136,"g":135},"2724b110":{"m":136,"g":135},"176266f3":{"m":136,"g":135},"5e5b1183":{"m":136,"g":135},"9bf76c11":{"m":136,"g":135},"1d7ad4af":{"m":136,"g":135},"fba785c4":{"m":136,"g":135},"5cfa901b":{"m":136,"g":135},"f27c6cdc":{"m":136,"g":135},"5097e1e8":{"m":136,"g":135},"861a35fb":{"m":136,"g":135},"6ffe1fc0":{"m":136,"g":135},"73398e22":{"m":136,"g":135},"17958c5f":{"m":136,"g":135},"3aa11ca7":{"m":136,"g":135},"7be1a8c7":{"m":136,"g":135},"84d13c54":{"m":136,"g":135},"c80c0e0f":{"m":136,"g":135},"c105a312":{"m":136,"g":135},"4cf2bbd0":{"m":136,"g":135},"d7b706be":{"m":136,"g":135},"4a9537a4":{"m":136,"g":135},"ca922d4b":{"m":136,"g":135},"402a0bd6":{"m":136,"g":135},"76c71d1d":{"m":136,"g":135},"2d02c150":{"m":136,"g":135},"b98bd9a5":{"m":136,"g":135},"ce694b2b":{"m":136,"g":135},"6c0fb189":{"m":136,"g":135},"9a9f996f":{"m":136,"g":135},"4221b7c5":{"m":136,"g":135},"c371df2f":{"m":136,"g":135},"1751c75b":{"m":136,"g":135},"23849eba":{"m":136,"g":135},"51541404":{"m":136,"g":135},"45ef8344":{"m":136,"g":135},"5a2b1ed4":{"m":136,"g":135},"454dc9e2":{"m":136,"g":135},"1e41069a":{"m":136,"g":135},"2b4d6d81":{"m":136,"g":135},"a3914e3b":{"m":136,"g":135},"130f60ee":{"m":136,"g":135},"4397cda7":{"m":136,"g":135},"d56fd10c":{"m":136,"g":135},"abb06be9":{"m":136,"g":135},"c35eb0fd":{"m":136,"g":135},"7f35c46e":{"m":136,"g":135},"da2f8cc3":{"m":136,"g":135},"4308c25b":{"m":136,"g":135},"4d737db8":{"m":136,"g":135},"9d6029fb":{"m":136,"g":135},"dcfb92dd":{"m":136,"g":135},"bebd625b":{"m":136,"g":135},"b12258bf":{"m":136,"g":135},"012dc586":{"m":136,"g":135},"7f6a678f":{"m":136,"g":135},"399ca037":{"m":136,"g":135},"d93f37a6":{"m":136,"g":135},"e6fe092d":{"m":136,"g":135},"f02d8221":{"m":136,"g":135},"10174e11":{"m":136,"g":135},"2138ff48":{"m":136,"g":135},"e53160bb":{"m":136,"g":135},"4ea6a11c":{"m":136,"g":135},"2181bc9e":{"m":136,"g":135},"520c048d":{"m":136,"g":135},"561a3e04":{"m":136,"g":135},"f84487af":{"m":136,"g":135},"07827047":{"m":136,"g":135},"c63e9cb2":{"m":136,"g":135},"12cde0df":{"m":136,"g":135},"a7fd8108":{"m":136,"g":135},"ca80c19b":{"m":136,"g":135},"1e7b3264":{"m":136,"g":135},"e267ca0b":{"m":136,"g":135},"9a8ba3c1":{"m":136,"g":135},"0fee6bc6":{"m":136,"g":135},"0ff3747c":{"m":136,"g":135},"f8411ded":{"m":136,"g":135},"249c3563":{"m":136,"g":135},"12df1660":{"m":136,"g":135},"55d112dc":{"m":136,"g":135},"a1ed247f":{"m":136,"g":135},"87699d48":{"m":136,"g":135},"52c60434":{"m":136,"g":135},"ff0f370f":{"m":136,"g":135},"26f9e207":{"m":136,"g":135},"828cd893":{"m":136,"g":135},"4436dc0f":{"m":136,"g":135},"cf6800f6":{"m":136,"g":135},"f16606d6":{"m":136,"g":135},"387fad2f":{"m":136,"g":135},"76bc07a3":{"m":136,"g":135},"b328cd20":{"m":136,"g":135},"bf32cd83":{"m":136,"g":135},"27a08305":{"m":136,"g":135},"53479e22":{"m":136,"g":135},"ff978e7d":{"m":136,"g":135},"16e00651":{"m":136,"g":135},"1b2b95d8":{"m":136,"g":135},"9cac3c86":{"m":136,"g":135},"8d58b3dc":{"m":136,"g":135},"5f3eb377":{"m":136,"g":135},"22993880":{"m":136,"g":135},"d7aa0ce7":{"m":136,"g":135},"e797f0c5":{"m":136,"g":135},"5d4b7c78":{"m":136,"g":135},"9bd64d73":{"m":136,"g":135},"24e116ef":{"m":136,"g":135},"8ca95970":{"m":136,"g":135},"216ea910":{"m":136,"g":135},"f4ab2ec5":{"m":136,"g":135},"25fa2ac2":{"m":136,"g":135},"ef5ac6f0":{"m":136,"g":135},"c88aaf22":{"m":136,"g":135},"e139d2aa":{"m":136,"g":135},"2337b1bb":{"m":136,"g":135},"7bc13c90":{"m":136,"g":135},"877c8e3a":{"m":136,"g":135},"66dfb8c1":{"m":136,"g":135},"dcacc492":{"m":136,"g":135},"87ef05e2":{"m":136,"g":135},"b65c9889":{"m":136,"g":135},"c7c0d97f":{"m":136,"g":135},"6bc5a52f":{"m":136,"g":135},"2c09de34":{"m":136,"g":135},"38d48de9":{"m":136,"g":135},"7f2fa216":{"m":136,"g":135},"d0fb24ee":{"m":136,"g":135},"65b0b5b2":{"m":136,"g":135},"9a414b16":{"m":136,"g":135},"5b4f7902":{"m":136,"g":135},"d8ac5eec":{"m":136,"g":135},"bdde9496":{"m":136,"g":135},"b23e7ed1":{"m":136,"g":135},"9821fae5":{"m":136,"g":135},"078d9621":{"m":136,"g":135},"8b111b20":{"m":136,"g":135},"74a166cb":{"m":136,"g":135},"888e126a":{"m":136,"g":135},"bb23a8fe":{"m":136,"g":135},"8b869e32":{"m":136,"g":135},"6256936d":{"m":136,"g":135},"62f73a8c":{"m":136,"g":135},"7c1b4b1c":{"m":136,"g":135},"31ed68e7":{"m":136,"g":135},"24c91001":{"m":136,"g":135},"c4edcac6":{"m":136,"g":135},"e9343389":{"m":136,"g":135},"a2d4f58a":{"m":136,"g":135},"f66b0916":{"m":136,"g":135},"2f623368":{"m":136,"g":135},"bd9a2ced":{"m":136,"g":135},"0ca417d9":{"m":136,"g":135},"30cfb687":{"m":136,"g":135},"b7c7e03d":{"m":136,"g":135},"f0195627":{"m":136,"g":135},"d401d238":{"m":136,"g":135},"17041f46":{"m":136,"g":135},"f07e76b2":{"m":136,"g":135},"1cfd2b2d":{"m":136,"g":135},"0d244116":{"m":136,"g":135},"0eae8317":{"m":136,"g":135},"c483a5f4":{"m":136,"g":135},"f26f6c2c":{"m":136,"g":135},"b0213323":{"m":136,"g":135},"5062537b":{"m":136,"g":135},"698629d1":{"m":136,"g":135},"dd93e445":{"m":136,"g":135},"6c8587b5":{"m":136,"g":135},"02704260":{"m":136,"g":135},"bd48ad5e":{"m":136,"g":135},"749736ba":{"m":136,"g":135},"72549863":{"m":136,"g":135},"00562ee1":{"m":136,"g":135},"d7a8257b":{"m":136,"g":135},"f6f7af40":{"m":136,"g":135},"a3b1e8ef":{"m":136,"g":135},"db499e18":{"m":136,"g":135},"6cf3a6dd":{"m":136,"g":135},"90e24f5c":{"m":136,"g":135},"21de3e14":{"m":136,"g":135},"e4c1e441":{"m":136,"g":135},"b5af283b":{"m":136,"g":135},"130c6911":{"m":136,"g":135},"e0e50848":{"m":136,"g":135},"70a769bc":{"m":136,"g":135},"3a42c5e3":{"m":136,"g":135},"12b89e51":{"m":136,"g":135},"85184557":{"m":136,"g":135},"57d2ba92":{"m":136,"g":135},"417e75a6":{"m":136,"g":135},"0500fea9":{"m":136,"g":135},"2b461c15":{"m":136,"g":135},"5595ae14":{"m":136,"g":135},"ace6f300":{"m":136,"g":135},"2da49eec":{"m":136,"g":135},"d65ae0ec":{"m":136,"g":135},"60d7279c":{"m":136,"g":135},"abdf65d4":{"m":136,"g":135},"c1dfbc77":{"m":136,"g":135},"3b3c5a05":{"m":136,"g":135},"2667c857":{"m":136,"g":135},"fc643ffb":{"m":136,"g":135},"4280a18a":{"m":136,"g":135},"386e5415":{"m":136,"g":135},"b6267de5":{"m":136,"g":135},"e47afa02":{"m":136,"g":135},"5ed384d0":{"m":136,"g":135},"b4ce7a6d":{"m":136,"g":135},"25b48564":{"m":136,"g":135},"1c360bf7":{"m":136,"g":135},"3619ec61":{"m":136,"g":135},"db6b51a8":{"m":136,"g":135},"8a9ca41f":{"m":136,"g":135},"ac11e6a7":{"m":136,"g":135},"47a660d5":{"m":136,"g":135},"c0fc7a89":{"m":136,"g":135},"5bf0d862":{"m":136,"g":135},"75b72eb8":{"m":136,"g":135},"bc8b526e":{"m":136,"g":135},"3dfff6ae":{"m":136,"g":135},"ad2c1ee3":{"m":136,"g":135},"9940c6f5":{"m":136,"g":135},"00e60711":{"m":136,"g":135},"4bc2f2e0":{"m":136,"g":135},"d17b9e63":{"m":136,"g":135},"ba67e006":{"m":136,"g":135},"45f3ad2f":{"m":136,"g":135},"f35b5da5":{"m":136,"g":135},"733a0c1a":{"m":136,"g":135},"b369aaa2":{"m":136,"g":135},"b9732025":{"m":136,"g":135},"cbff7ad9":{"m":136,"g":135},"4de59d83":{"m":136,"g":135},"39ca57cd":{"m":136,"g":135},"34498067":{"m":136,"g":135},"b3817fa9":{"m":136,"g":135},"059428bd":{"m":136,"g":135},"5d200dd8":{"m":136,"g":135},"7f9a3d06":{"m":136,"g":135},"664f611e":{"m":136,"g":135},"49adb37e":{"m":136,"g":135},"7518dc35":{"m":136,"g":135},"b6871ba7":{"m":136,"g":135},"c1f2241a":"m137","8da70e2a":"m137"},"g":"2026-02-12T19:54:15.483852"} diff --git a/docs/requirements.txt b/docs/requirements.txt deleted file mode 100644 index 5d7309675..000000000 --- a/docs/requirements.txt +++ /dev/null @@ -1,20 +0,0 @@ -ipykernel -ipywidgets -jupyter_client -markdown>=3.4.0 -matplotlib -myst-parser -nbconvert -nbsphinx -pandoc -pillow -pydantic -sphinx -sphinx-book-theme -sphinx-copybutton -sphinx-tabs -nbstripout -sphinxcontrib-mermaid -urllib3<2.0.0 -gguf>=0.17.1 -sphinx-autobuild diff --git a/docs/serve.sh b/docs/serve.sh deleted file mode 100644 index 049f767cf..000000000 --- a/docs/serve.sh +++ /dev/null @@ -1,3 +0,0 @@ -# Clean and serve documentation with auto-build -make clean -make serve diff --git a/docs/supported_models/extending/index.rst b/docs/supported_models/extending/index.rst deleted file mode 100644 index dbd5ff6ce..000000000 --- a/docs/supported_models/extending/index.rst +++ /dev/null @@ -1,12 +0,0 @@ -Extending SGLang -================ - -Adding new models and alternative backends. - -.. toctree:: - :maxdepth: 1 - - support_new_models.md - transformers_fallback.md - modelscope.md - mindspore_models.md diff --git a/docs/supported_models/extending/mindspore_models.md b/docs/supported_models/extending/mindspore_models.md deleted file mode 100644 index caa5ade9c..000000000 --- a/docs/supported_models/extending/mindspore_models.md +++ /dev/null @@ -1,151 +0,0 @@ -# MindSpore Models - -## Introduction - -MindSpore is a high-performance AI framework optimized for Ascend NPUs. This doc guides users to run MindSpore models in SGLang. - -## Requirements - -MindSpore currently only supports Ascend NPU devices. Users need to first install Ascend CANN 8.5. -The CANN software packages can be downloaded from the [Ascend Official Website](https://www.hiascend.com). - -## Supported Models - -Currently, the following models are supported: - -- **Qwen3**: Dense and MoE models -- **DeepSeek V3/R1** -- *More models coming soon...* - -## Installation - -> **Note**: Currently, MindSpore models are provided by an independent package `sgl-mindspore`. Support for MindSpore is built upon current SGLang support for Ascend NPU platform. Please first [install SGLang for Ascend NPU](../../platforms/ascend/ascend_npu.md) and then install `sgl-mindspore`: - -```shell -git clone https://github.com/mindspore-lab/sgl-mindspore.git -cd sgl-mindspore -pip install -e . -``` - - -## Run Model - -Current SGLang-MindSpore supports Qwen3 and DeepSeek V3/R1 models. This doc uses Qwen3-8B as an example. - -### Offline inference - -Use the following script for offline inference: - -```python -import sglang as sgl - -# Initialize the engine with MindSpore backend -llm = sgl.Engine( - model_path="/path/to/your/model", # Local model path - device="npu", # Use NPU device - model_impl="mindspore", # MindSpore implementation - attention_backend="ascend", # Attention backend - tp_size=1, # Tensor parallelism size - dp_size=1 # Data parallelism size -) - -# Generate text -prompts = [ - "Hello, my name is", - "The capital of France is", - "The future of AI is" -] - -sampling_params = {"temperature": 0, "top_p": 0.9} -outputs = llm.generate(prompts, sampling_params) - -for prompt, output in zip(prompts, outputs): - print(f"Prompt: {prompt}") - print(f"Generated: {output['text']}") - print("---") -``` - -### Start server - -Launch a server with MindSpore backend: - -```bash -# Basic server startup -python3 -m sglang.launch_server \ - --model-path /path/to/your/model \ - --host 0.0.0.0 \ - --device npu \ - --model-impl mindspore \ - --attention-backend ascend \ - --tp-size 1 \ - --dp-size 1 -``` - -For distributed server with multiple nodes: - -```bash -# Multi-node distributed server -python3 -m sglang.launch_server \ - --model-path /path/to/your/model \ - --host 0.0.0.0 \ - --device npu \ - --model-impl mindspore \ - --attention-backend ascend \ - --dist-init-addr 127.0.0.1:29500 \ - --nnodes 2 \ - --node-rank 0 \ - --tp-size 4 \ - --dp-size 2 -``` - -## Troubleshooting - -#### Debug Mode - -Enable sglang debug logging by log-level argument. - -```bash -python3 -m sglang.launch_server \ - --model-path /path/to/your/model \ - --host 0.0.0.0 \ - --device npu \ - --model-impl mindspore \ - --attention-backend ascend \ - --log-level DEBUG -``` - -Enable mindspore info and debug logging by setting environments. - -```bash -export GLOG_v=1 # INFO -export GLOG_v=0 # DEBUG -``` - -#### Explicitly select devices - -Use the following environment variable to explicitly select the devices to use. - -```shell -export ASCEND_RT_VISIBLE_DEVICES=4,5,6,7 # to set device -``` - -#### Some communication environment issues - -In case of some environment with special communication environment, users need set some environment variables. - -```shell -export MS_ENABLE_LCCL=off # current not support LCCL communication mode in SGLang-MindSpore -``` - -#### Some dependencies of protobuf - -In case of some environment with special protobuf version, users need set some environment variables to avoid binary version mismatch. - -```shell -export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python # to avoid protobuf binary version mismatch -``` - -## Support -For MindSpore-specific issues: - -- Refer to the [MindSpore documentation](https://www.mindspore.cn/) diff --git a/docs/supported_models/extending/modelscope.md b/docs/supported_models/extending/modelscope.md deleted file mode 100644 index 4740c2770..000000000 --- a/docs/supported_models/extending/modelscope.md +++ /dev/null @@ -1,28 +0,0 @@ -# Use Models From ModelScope - -To use a model from [ModelScope](https://www.modelscope.cn), set the environment variable `SGLANG_USE_MODELSCOPE`. - -```bash -export SGLANG_USE_MODELSCOPE=true -``` - -We take [Qwen2-7B-Instruct](https://www.modelscope.cn/models/qwen/qwen2-7b-instruct) as an example. - -Launch the Server: -```bash -python -m sglang.launch_server --model-path qwen/Qwen2-7B-Instruct --port 30000 -``` - -Or start it by docker: - -```bash -docker run --gpus all \ - -p 30000:30000 \ - -v ~/.cache/modelscope:/root/.cache/modelscope \ - --env "SGLANG_USE_MODELSCOPE=true" \ - --ipc=host \ - lmsysorg/sglang:latest \ - python3 -m sglang.launch_server --model-path Qwen/Qwen2.5-7B-Instruct --host 0.0.0.0 --port 30000 -``` - -Note that modelscope uses a different cache directory than huggingface. You may need to set it manually to avoid running out of disk space. diff --git a/docs/supported_models/extending/support_new_models.md b/docs/supported_models/extending/support_new_models.md deleted file mode 100644 index 7951631e9..000000000 --- a/docs/supported_models/extending/support_new_models.md +++ /dev/null @@ -1,520 +0,0 @@ -# How to Support New Models - -This document explains how to add support for new language models and multimodal large language models (MLLMs) in -SGLang. It also covers how to test new models and register external implementations. - -## How to Support a New Language Model - -To support a new model in SGLang, you only need to add a single file under -the [SGLang Models Directory](https://github.com/sgl-project/sglang/tree/main/python/sglang/srt/models). You can learn -from existing model implementations and create a new file for your model. For most models, you should be able to find a -similar model to start with (e.g., starting from Llama). Also refer how -to [port a Model from vLLM to SGLang](#port-a-model-from-vllm-to-sglang) - -## How to Support a New Multimodal Large Language Model - -To support a new multimodal large language model (MLLM) in SGLang, there are several key components in addition to the -standard LLM support: - -1. **Register your new model as multimodal**: - Extend `is_multimodal_model` - in [model_config.py](https://github.com/sgl-project/sglang/blob/0ab3f437aba729b348a683ab32b35b214456efc7/python/sglang/srt/configs/model_config.py#L561) - to return `True` for your model. - -2. **Register a new chat-template**: - Only when your default chat-template is unable to accept images as input: Register a new chat template in [conversation.py](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/parser/conversation.py) and the corresponding matching function. - -3. **Multimodal Data Processor**: - Define a new `Processor` class that inherits from `BaseMultimodalProcessor` and register this processor as your - model’s dedicated processor. - See [multimodal_processor.py](https://github.com/sgl-project/sglang/tree/main/python/sglang/srt/multimodal/processors) - for more details. - -4. **Handle Multimodal Tokens**: - Implement a `pad_input_ids` function for your new model. In this function, multimodal tokens in the prompt should be - expanded (if necessary) and padded with multimodal-data-hashes so that SGLang can recognize different multimodal data - with `RadixAttention`. - -5. **Handle Image Feature Extraction**: - Implement a `get_image_feature` function for your new model, which extracts image features from raw image data and converts them into the embeddings used by the language model. - -6. **Adapt to Vision Attention**: - Adapt the multi-headed `Attention` of ViT with SGLang’s `VisionAttention`. - -You can refer to [Qwen2VL](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/models/qwen2_vl.py) or -other mllm implementations. These models demonstrate how to correctly handle both multimodal and textual inputs. - -## Testing and Debugging - -Please note all your testing and benchmarking results in PR description. - -### Interactive Debugging - -For interactive debugging, compare the outputs of Hugging Face/Transformers and SGLang. The following two commands -should give the same text output and very similar prefill logits: - -- Get the reference output: - ```bash - python3 scripts/playground/reference_hf.py --model-path [new model] --model-type {text,vlm} - ``` -- Get the SGLang output: - ```bash - python3 -m sglang.bench_one_batch --correct --model [new model] - ``` - -### Add the Model to the Test Suite - -To ensure the new model is well maintained, add it to the test suite by including it in the `ALL_OTHER_MODELS` list in -the [test_generation_models.py](https://github.com/sgl-project/sglang/blob/main/test/registered/models/test_generation_models.py) -file, test the new model on your local machine and report the results on demonstrative benchmarks (GSM8K, MMLU, MMMU, -MMMU-Pro, etc.) in your PR. \\ -For VLMs, also include a test in `test_vision_openai_server_{x}.py` (e.g. [test_vision_openai_server_a.py](https://github.com/sgl-project/sglang/blob/main/test/registered/vlm/test_vision_openai_server_a.py)). - -This is an example command to run to test a new model on your local machine: - -```bash -ONLY_RUN=Qwen/Qwen2-1.5B python3 -m unittest test_generation_models.TestGenerationModels.test_others -``` - -### Benchmark - -- **(Required) MMMU**: follow MMMU benchmark [README.md](https://github.com/sgl-project/sglang/blob/main/benchmark/mmmu/README.md) to get SGLang vs. HF Transformer accuracy comparison. The accuracy score from SGLang run should not be much lower than that from HF Transformer run. Similarly, follow https://docs.sglang.io/developer_guide/benchmark_and_profiling.html to get performance comparison: TTFT and throughput must meet or exceed baselines (e.g., HF Transformer). -- **(Optional) Other evals**: If you ran other evals, please note the results in PR description. - -## Port a Model from vLLM to SGLang - -The [vLLM Models Directory](https://github.com/vllm-project/vllm/tree/main/vllm/model_executor/models) is a valuable -resource, as vLLM covers many models. SGLang reuses vLLM’s interface and some layers, making it easier to port models -from vLLM to SGLang. - -To port a model from vLLM to SGLang: - -- Compare these two files for guidance: - - [SGLang Llama Implementation](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/models/llama.py) - - [vLLM Llama Implementation](https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/models/llama.py) -- The major differences include: - - **Replace vLLM’s `Attention` with `RadixAttention`** (ensure you pass `layer_id` to `RadixAttention`). - - **Replace vLLM’s `LogitsProcessor` with SGLang’s `LogitsProcessor`.** - - **Replace the multi-headed `Attention` of ViT with SGLang’s `VisionAttention`.** - - **Replace other vLLM layers** (such as `RMSNorm`, `SiluAndMul`) with SGLang layers. - - **Remove `Sample`.** - - **Change the `forward()` functions** and add a `forward_batch()` method. - - **Add `EntryClass`** at the end. - - **Ensure that the new implementation uses only SGLang components** and does not rely on any vLLM components. - -Note: make sure you add your new model to the supported models list in the supported models documentation. - -## Registering an External Model Implementation - -In addition to the methods above, you can register your new model with the `ModelRegistry` before launching the server. -This allows you to integrate your model without modifying the source code. - -For example: - -```python -from sglang.srt.models.registry import ModelRegistry -from sglang.srt.entrypoints.http_server import launch_server - -# For a single model, add it to the registry: -ModelRegistry.models[model_name] = model_class - -# For multiple models, you can imitate the import_model_classes() function: -from functools import lru_cache - -@lru_cache() -def import_new_model_classes(): - model_arch_name_to_cls = {} - # Populate model_arch_name_to_cls with your new model classes. - ... - return model_arch_name_to_cls - -ModelRegistry.models.update(import_new_model_classes()) - -# Launch the server with your server arguments: -launch_server(server_args) -``` - -## Example: Implementing and Serving a Llama Wrapper Model - -Below is an introductory, step-by-step walkthrough on how to implement a new model end-to-end in SGLang and then run it via the [Offline Engine](https://github.com/sgl-project/sglang/blob/main/docs/basic_usage/offline_engine_api.ipynb). - -### Implementing Our Model - -To keep things simple, this new model will be a simple wrapper around [Llama 3.1-8B-Instruct](https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct), and our goal will be just to bias the output logits for each `forward` call by taking the square root of each individual logit. - -Let's start by defining our model in a file called `llama_wrapper.py`. -The first step is to import the necessary libraries from SRT, which is SGLang's internal backend. - -```python -# In the file `llama_wrapper.py` - -import torch -from transformers import LlamaConfig -from typing import Optional -from sglang.srt.layers.logits_processor import LogitsProcessorOutput -from sglang.srt.layers.quantization.base_config import QuantizationConfig -from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors - -from sglang.srt.models.llama import LlamaForCausalLM -``` - -Next, we declare a new `class` for our model and have it inherit from `LlamaForCausalLM`, which allows our model to access `LlamaForCausalLM`'s predefined modules and layers, such as `LlamaAttention` and `LlamaMLP`. -Note that almost all model implementations take in `config` and `quant_config` as arguments for their `__init__` method; `config` and `quant_config` are passed in via [`model_loader/loader.py`](https://github.com/sgl-project/sglang/blob/bf72b80122fd888bf619d17b96fa3e323ab809fc/python/sglang/srt/model_loader/loader.py#L219). -Because we have inherited from `LlamaForCausalLM`, we can pass our parameters directly to its constructor, which will set the member variables for us. - -```python -class LlamaWrapper(LlamaForCausalLM): - def __init__( - self, - config: LlamaConfig, - quant_config: Optional[QuantizationConfig] = None, - prefix: str = "", - ) -> None: - super().__init__(config=config, quant_config=quant_config, prefix=prefix) -``` - -Now, we want to define the `forward` method, which is what will be called at inference time. -Note that the signature for `forward` is essentially the same for any model; you can take a look at the other models defined in the [`models` directory](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/models/) for references. -To see where exactly `forward` is called in the SGLang runtime's internals, take a look at [`forward_decode`](https://github.com/sgl-project/sglang/blob/bf72b80122fd888bf619d17b96fa3e323ab809fc/python/sglang/srt/model_executor/model_runner.py#L1705) and [`forward_extend`](https://github.com/sgl-project/sglang/blob/bf72b80122fd888bf619d17b96fa3e323ab809fc/python/sglang/srt/model_executor/model_runner.py#L1724) in the [`ModelRunner` class](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/model_executor/model_runner.py). - -```python - @torch.no_grad() - def forward( - self, - input_ids: torch.Tensor, - positions: torch.Tensor, - forward_batch: ForwardBatch, - pp_proxy_tensors: Optional[PPProxyTensors] = None, - input_embeds: Optional[torch.Tensor] = None, - get_embedding: bool = False, - ) -> LogitsProcessorOutput: -``` - -We now call the `__call__` method for `self.model` (which is a member variable that `LlamaForCausalLM` defines in its `__init__` method), which eventually calls `LlamaForCausalLM`'s `forward` method. -After that, we feed the `hidden_states` into our model's `LogitsProcessor` (again defined in `LlamaForCausalLM`). - -```python - hidden_states = self.model( - input_ids, - positions, - forward_batch, - input_embeds, - pp_proxy_tensors=pp_proxy_tensors, - ) - - res: LogitsProcessorOutput = self.logits_processor( - input_ids, - hidden_states, - self.lm_head, - forward_batch, - ) -``` - -After receiving the logits for the next token, we can finally perform our biasing step. - -```python - orig_logits = res.next_token_logits - res.next_token_logits = torch.where( - orig_logits > 0, - orig_logits.sqrt(), - orig_logits - ) - - return res -``` - -Now, our `LlamaWrapper` model is created and ready to be served! - -### Serving Our Model Via SGLang's Offline Engine - -The next step of this walkthrough involves hosting our new model offline, so that it can be served locally and without an HTTP server. - -First, create a new file called `run.py`. -Now, we must ensure that SGLang's `ModelRegistry` can find our model. -To do this, we first download the model's configuration and weights from Huggingface. - -```python -# In the file `run.py` - -import asyncio -from functools import lru_cache -from huggingface_hub import snapshot_download -from llama_wrapper import LlamaWrapper # Make sure to import our new model! -import sglang as sgl -from sglang.srt.models.registry import ModelRegistry - -# Make sure to request access to this model on Huggingface, then export your -# `HF_TOKEN` to download the model snapshot -llama_dir = snapshot_download( - repo_id="meta-llama/Llama-3.1-8B-Instruct", - local_dir="./llama_ckpt", -) -``` - -Now that we have our model on disk, we want to point it to `LlamaWrapper` by changing the `architectures` field in `./llama_ckpt/config.json` to be `LlamaWrapper`. -That way, when we pass in the path of our model checkpoint to SGLang, it will know that we want to use "LlamaWrapper" instead of "LlamaForCausalLM" as our model. - -```python -{ - "architectures": [ - # "LlamaForCausalLM" - "LlamaWrapper" - ], - ... -} -``` - -However, if we don't link our `LlamaWrapper` class to the "LlamaWrapper" registry keyword, then SGLang won't be able to find our model. -Thus, to register our `LlamaWrapper`, we want to follow the steps in the above section titled "Registering an External Model Implementation". - -```python -@lru_cache() -def import_new_model_classes(): - model_arch_name_to_cls = {"LlamaWrapper": LlamaWrapper} - return model_arch_name_to_cls - -ModelRegistry.models.update(import_new_model_classes()) -``` - -Lastly, when we create our `Engine`, we just pass in the path to the local model directory. -Then, our `LlamaWrapper` is ready to be served; for this walkthrough, we will use SGLang `Engine`'s non-streaming asynchronous generation endpoint. - -```python -def main(): - llm = sgl.Engine(model_path="./llama_ckpt") - sampling_params = {"temperature": 0.2, "top_k": 5} - prompts = [ - "Write a short, neutral self-introduction for a fictional character. Hello, my name is", - "Provide a concise factual statement about France’s capital city. The capital of France is", - "Explain possible future trends in artificial intelligence. The future of AI is", - ] - - asyncio.run(run_llm(llm, sampling_params, prompts)) - - llm.shutdown() - -async def run_llm( - llm, - sampling_params, - prompts, -) -> None: - outputs = await llm.async_generate(prompts, sampling_params) - - for prompt, output in zip(prompts, outputs): - print(f"\nPrompt: {prompt}") - print(f"Generated text: {output['text']}") - -if __name__ == "__main__": - main() -``` - -Now, when we call `python run.py`, we will get the outputs of our newly created model! - -## Serving External Models via the Standard CLI - -The previous sections show how to register a model programmatically via `ModelRegistry` and serve it through the Offline Engine. Similar to vLLM model plugin, there is an alternative that lets you keep using the standard `python -m sglang.launch_server` CLI without modifying any SGLang source code: you can register your model using the `SGLANG_EXTERNAL_MODEL_PACKAGE` environment variable. - -### The `EntryClass` Variable - -When SGLang scans a model package, it looks for the variable `EntryClass` at the module level of your Python file. The [model registry](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/models/registry.py) imports your file, checks for `EntryClass`, and registers the class assigned to it. If you are using a model based on HuggingFace, the name of this class needs to match the `"architectures"` field in your model's `config.json`. - -For example, if you are implementing a Llama wrapper, add this line at the end of your model file: - -```python -# This is what "Add EntryClass at the end" means -EntryClass = LlamaWrapper -``` - -### Example: Text-Only Model - -Using the same Llama wrapper from the previous section, here is how to package and serve it via the CLI. - -1. Create your project - -``` -sglang_custom_project/ -|----setup.py -|----custom_llm/ - |----__init__.py - |----llama_wrapper.py -``` - -Write the `setup.py`: - -```python -# sglang_custom_project/setup.py - -from setuptools import setup, find_packages -setup( - name="sglang-custom-plugins", - version="0.1", - packages=find_packages(), -) -``` - -2. Write your model code - -Inside `llama_wrapper.py`, write your model and include `EntryClass`: - -```python -# sglang_custom_project/custom_llm/llama_wrapper.py - -import torch -from typing import Optional -from sglang.srt.layers.logits_processor import LogitsProcessorOutput -from sglang.srt.layers.quantization.base_config import QuantizationConfig -from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors -from sglang.srt.models.llama import LlamaForCausalLM - -class LlamaWrapper(LlamaForCausalLM): - def __init__(self, config, quant_config: Optional[QuantizationConfig] = None, - prefix: str = "") -> None: - super().__init__(config=config, quant_config=quant_config, prefix=prefix) - @torch.no_grad() - def forward(self, input_ids, positions, forward_batch, - pp_proxy_tensors=None, input_embeds=None, get_embedding=False): - hidden_states = self.model( - input_ids, positions, forward_batch, input_embeds, - pp_proxy_tensors=pp_proxy_tensors, - ) - res: LogitsProcessorOutput = self.logits_processor( - input_ids, hidden_states, self.lm_head, forward_batch, - ) - - orig = res.next_token_logits - res.next_token_logits = torch.where(orig > 0, orig.sqrt(), orig) - return res - -# Don't forget to add EntryClass -EntryClass = LlamaWrapper -``` - -3. Install your package - -Run this inside your `sglang_custom_project` directory to install your code into the active Python environment: - -```bash -pip install -e . -``` - -4. Update your `config.json` - -Update the `config.json` under your HuggingFace model checkpoint directory so the `architectures` field matches your class name: - -```json -{ - "architectures": ["LlamaWrapper"], - ... -} -``` - -5. Launch the server - -Set the environment variable before running the CLI: - -```bash -export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_llm -python -m sglang.launch_server \ - --model-path /path/to/Llama-3.1-8B-Instruct \ - --port 8000 -``` - -The `SGLANG_EXTERNAL_MODEL_PACKAGE` should be the parent folder name containing your model-related code. In this example, it should be `custom_llm`. - -### Example: Multimodal Model - -If you are working with multimodal models, setting `SGLANG_EXTERNAL_MODEL_PACKAGE` alone is not enough. SGLang also needs to recognize your architecture as multimodal to enable the image/video processing pipelines, and it needs a custom processor. - -You can handle this by setting two additional environment variables: - -- `SGLANG_EXTERNAL_MM_MODEL_ARCH`: Adds your architecture name to SGLang's internal list of multimodal models. -- `SGLANG_EXTERNAL_MM_PROCESSOR_PACKAGE`: Tells SGLang where to find your custom processor class. - -For example, let's build a custom model based on Qwen2-VL-Instruct that takes the square root of the logits. - -Create the project: - -``` -sglang_custom_project_vl/ -|----setup.py -|----custom_vlm/ - |----__init__.py - |----qwenvl_wrapper.py -``` - -Write `setup.py`: - -```python -# sglang_custom_project_vl/setup.py - -from setuptools import setup, find_packages -setup( - name="sglang-custom-plugins-vl", - version="0.1", - packages=find_packages(), -) -``` - -Write the model in `qwenvl_wrapper.py`: - -```python -# sglang_custom_project_vl/custom_vlm/qwenvl_wrapper.py -import torch -from sglang.srt.models.qwen2_vl import Qwen2VLForConditionalGeneration -from sglang.srt.multimodal.processors.qwen_vl import QwenVLImageProcessor - -class CustomQwen2VL(Qwen2VLForConditionalGeneration): - def forward(self, input_ids, positions, forward_batch, - input_embeds=None, get_embedding=False): - res = super().forward( - input_ids, positions, forward_batch, - input_embeds=input_embeds, get_embedding=get_embedding - ) - if not get_embedding: - orig = res.next_token_logits - res.next_token_logits = torch.where(orig > 0, orig.sqrt(), orig) - return res - -class CustomQwen2VLProcessor(QwenVLImageProcessor): - models = [CustomQwen2VL] - - def __init__(self, hf_config, server_args, _processor, *args, **kwargs): - super().__init__(hf_config, server_args, _processor, *args, **kwargs) - -EntryClass = CustomQwen2VL -``` - -**Note:** you don't need a separate `EntryClass` for the custom processor as long as you associate the processor with the specific model class. - -Install the package, update `config.json`, and launch: - -```bash -pip install -e . -``` - -```json -{ - "architectures": ["CustomQwen2VL"], - ... -} -``` - -```bash -export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_vlm -export SGLANG_EXTERNAL_MM_MODEL_ARCH=CustomQwen2VL -export SGLANG_EXTERNAL_MM_PROCESSOR_PACKAGE=custom_vlm - -python -m sglang.launch_server \ - --model-path /path/to/Qwen2-VL-2B-Instruct \ - --port 8000 \ - --enable-multimodal -``` - -## Documentation - -Add to table of supported models in [generative_models.md](../text_generation/generative_models.md) or [multimodal_language_models.md](../text_generation/multimodal_language_models.md) - ---- - -By following these guidelines, you can add support for new language models and multimodal large language models in -SGLang and ensure they are thoroughly tested and easily integrated into the system. diff --git a/docs/supported_models/extending/transformers_fallback.md b/docs/supported_models/extending/transformers_fallback.md deleted file mode 100644 index cd80d5612..000000000 --- a/docs/supported_models/extending/transformers_fallback.md +++ /dev/null @@ -1,58 +0,0 @@ -# Transformers fallback in SGLang - -`sglang` can fall back to using models that are available in `transformers`. This works for most decoder-style language models and support for vision-language models is coming soon! - -## Example launch Command - -By default, we will use sglang implementation if it is available. Otherwise, we will fall back to transformers one. However, you can switch the implementation by setting `--model-impl` to `transformers`. - -```shell -python3 -m sglang.launch_server \ - --model-path meta-llama/Llama-3.2-1B-Instruct \ - --host 0.0.0.0 \ - --port 30000 \ - --model-impl transformers -``` - -## Supported features - -### Quantization - -Transformers fall back has supported most of available quantization in SGLang (except GGUF). See [Quantization page](../../advanced_features/quantization.md) for more information about supported quantization in SGLang. - -### Remote code - -This fallback also means that any model on the hub that can be used in `transformers` with `trust_remote_code=True` that correctly implements attention can be used in production! - -A model just needs the following two things: - -```python -from transformers import PreTrainedModel -from torch import nn - -class MyAttention(nn.Module): - - def forward(self, hidden_states, **kwargs): # <- kwargs are required - - ... - attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation] - attn_output, attn_weights = attention_interface( - self, - query_states, - key_states, - value_states, - **kwargs, - ) - ... - -class MyModel(PreTrainedModel): - _supports_attention_backend = True -``` - -Here is what happens in the background: - -1. The config is loaded -2. `MyModel` python class is loaded from the `auto_map`, and we check that the model `_supports_attention_backend`. -3. The `TransformersModel` backend is used. See `/srt/models/transformers`, which leverages `self.config._attn_implementation = "sglang"`, thus the need to use `ALL_ATTENTION_FUNCTIONS`. - -That's it! diff --git a/docs/supported_models/index.rst b/docs/supported_models/index.rst deleted file mode 100644 index f90c6fba1..000000000 --- a/docs/supported_models/index.rst +++ /dev/null @@ -1,13 +0,0 @@ -Supported Models -================ - -SGLang supports a wide variety of model architectures for different use cases. -Browse by category below to find models suited for your needs. - -.. toctree:: - :maxdepth: 2 - - text_generation/index - retrieval_ranking/index - specialized/index - extending/index diff --git a/docs/supported_models/retrieval_ranking/classify_models.md b/docs/supported_models/retrieval_ranking/classify_models.md deleted file mode 100644 index 9b3c6a591..000000000 --- a/docs/supported_models/retrieval_ranking/classify_models.md +++ /dev/null @@ -1,162 +0,0 @@ -# Classification API - -This document describes the `/v1/classify` API endpoint implementation in SGLang, which is compatible with vLLM's classification API format. - -## Overview - -The classification API allows you to classify text inputs using classification models. This implementation follows the same format as vLLM's 0.7.0 classification API. - -## API Endpoint - -``` -POST /v1/classify -``` - -## Request Format - -```json -{ - "model": "model_name", - "input": "text to classify" -} -``` - -### Parameters - -- `model` (string, required): The name of the classification model to use -- `input` (string, required): The text to classify -- `user` (string, optional): User identifier for tracking -- `rid` (string, optional): Request ID for tracking -- `priority` (integer, optional): Request priority - -## Response Format - -```json -{ - "id": "classify-9bf17f2847b046c7b2d5495f4b4f9682", - "object": "list", - "created": 1745383213, - "model": "jason9693/Qwen2.5-1.5B-apeach", - "data": [ - { - "index": 0, - "label": "Default", - "probs": [0.565970778465271, 0.4340292513370514], - "num_classes": 2 - } - ], - "usage": { - "prompt_tokens": 10, - "total_tokens": 10, - "completion_tokens": 0, - "prompt_tokens_details": null - } -} -``` - -### Response Fields - -- `id`: Unique identifier for the classification request -- `object`: Always "list" -- `created`: Unix timestamp when the request was created -- `model`: The model used for classification -- `data`: Array of classification results - - `index`: Index of the result - - `label`: Predicted class label - - `probs`: Array of probabilities for each class - - `num_classes`: Total number of classes -- `usage`: Token usage information - - `prompt_tokens`: Number of input tokens - - `total_tokens`: Total number of tokens - - `completion_tokens`: Number of completion tokens (always 0 for classification) - - `prompt_tokens_details`: Additional token details (optional) - -## Example Usage - -### Using curl - -```bash -curl -v "http://127.0.0.1:8000/v1/classify" \ - -H "Content-Type: application/json" \ - -d '{ - "model": "jason9693/Qwen2.5-1.5B-apeach", - "input": "Loved the new café—coffee was great." - }' -``` - -### Using Python - -```python -import requests -import json - -# Make classification request -response = requests.post( - "http://127.0.0.1:8000/v1/classify", - headers={"Content-Type": "application/json"}, - json={ - "model": "jason9693/Qwen2.5-1.5B-apeach", - "input": "Loved the new café—coffee was great." - } -) - -# Parse response -result = response.json() -print(json.dumps(result, indent=2)) -``` - -## Supported Models - -The classification API works with any classification model supported by SGLang, including: - -### Classification Models (Multi-class) -- `LlamaForSequenceClassification` - Multi-class classification -- `Qwen2ForSequenceClassification` - Multi-class classification -- `Qwen3ForSequenceClassification` - Multi-class classification -- `BertForSequenceClassification` - Multi-class classification -- `Gemma2ForSequenceClassification` - Multi-class classification - -**Label Mapping**: The API automatically uses the `id2label` mapping from the model's `config.json` file to provide meaningful label names instead of generic class names. If `id2label` is not available, it falls back to `LABEL_0`, `LABEL_1`, etc., or `Class_0`, `Class_1` as a last resort. - -### Reward Models (Single score) -- `InternLM2ForRewardModel` - Single reward score -- `Qwen2ForRewardModel` - Single reward score -- `LlamaForSequenceClassificationWithNormal_Weights` - Special reward model - -**Note**: The `/classify` endpoint in SGLang was originally designed for reward models but now supports all non-generative models. Our `/v1/classify` endpoint provides a standardized vLLM-compatible interface for classification tasks. - -## Error Handling - -The API returns appropriate HTTP status codes and error messages: - -- `400 Bad Request`: Invalid request format or missing required fields -- `500 Internal Server Error`: Server-side processing error - -Error response format: -```json -{ - "error": "Error message", - "type": "error_type", - "code": 400 -} -``` - -## Implementation Details - -The classification API is implemented using: - -1. **Rust Model Gateway**: Handles routing and request/response models in `sgl-model-gateway/src/protocols/spec.rs` -2. **Python HTTP Server**: Implements the actual endpoint in `python/sglang/srt/entrypoints/http_server.py` -3. **Classification Service**: Handles the classification logic in `python/sglang/srt/entrypoints/openai/serving_classify.py` - -## Testing - -Use the provided test script to verify the implementation: - -```bash -python test_classify_api.py -``` - -## Compatibility - -This implementation is compatible with vLLM's classification API format, allowing seamless migration from vLLM to SGLang for classification tasks. diff --git a/docs/supported_models/retrieval_ranking/embedding_models.md b/docs/supported_models/retrieval_ranking/embedding_models.md deleted file mode 100644 index 906466ac5..000000000 --- a/docs/supported_models/retrieval_ranking/embedding_models.md +++ /dev/null @@ -1,126 +0,0 @@ -# Embedding Models - -SGLang provides robust support for embedding models by integrating efficient serving mechanisms with its flexible programming interface. This integration allows for streamlined handling of embedding tasks, facilitating faster and more accurate retrieval and semantic search operations. SGLang's architecture enables better resource utilization and reduced latency in embedding model deployment. - -```{important} -Embedding models are executed with `--is-embedding` flag and some may require `--trust-remote-code` -``` - -## Quick Start - -### Launch Server - -```shell -python3 -m sglang.launch_server \ - --model-path Qwen/Qwen3-Embedding-4B \ - --is-embedding \ - --host 0.0.0.0 \ - --port 30000 -``` - -### Client Request - -```python -import requests - -url = "http://127.0.0.1:30000" - -payload = { - "model": "Qwen/Qwen3-Embedding-4B", - "input": "What is the capital of France?", - "encoding_format": "float" -} - -response = requests.post(url + "/v1/embeddings", json=payload).json() -print("Embedding:", response["data"][0]["embedding"]) -``` - - - -## Multimodal Embedding Example - -For multimodal models like GME that support both text and images: - -```shell -python3 -m sglang.launch_server \ - --model-path Alibaba-NLP/gme-Qwen2-VL-2B-Instruct \ - --is-embedding \ - --chat-template gme-qwen2-vl \ - --host 0.0.0.0 \ - --port 30000 -``` - -```python -import requests - -url = "http://127.0.0.1:30000" - -text_input = "Represent this image in embedding space." -image_path = "https://huggingface.co/datasets/liuhaotian/llava-bench-in-the-wild/resolve/main/images/023.jpg" - -payload = { - "model": "gme-qwen2-vl", - "input": [ - { - "text": text_input - }, - { - "image": image_path - } - ], -} - -response = requests.post(url + "/v1/embeddings", json=payload).json() - -print("Embeddings:", [x.get("embedding") for x in response.get("data", [])]) -``` - -## Matryoshka Embedding Example - -[Matryoshka Embeddings](https://sbert.net/examples/sentence_transformer/training/matryoshka/README.html#matryoshka-embeddings) or [Matryoshka Representation Learning (MRL)](https://arxiv.org/abs/2205.13147) is a technique used in training embedding models. It allows user to trade off between performance and cost. - -### 1. Launch a Matryoshka‑capable model - -If the model config already includes `matryoshka_dimensions` or `is_matryoshka` then no override is needed. Otherwise, you can use `--json-model-override-args` as below: - -```shell -python3 -m sglang.launch_server \ - --model-path Qwen/Qwen3-Embedding-0.6B \ - --is-embedding \ - --host 0.0.0.0 \ - --port 30000 \ - --json-model-override-args '{"matryoshka_dimensions": [128, 256, 512, 1024, 1536]}' -``` - -1. Setting `"is_matryoshka": true` allows truncating to any dimension. Otherwise, the server will validate that the specified dimension in the request is one of `matryoshka_dimensions`. -2. Omitting `dimensions` in a request returns the full vector. - -### 2. Make requests with different output dimensions - -```python -import requests - -url = "http://127.0.0.1:30000" - -# Request a truncated (Matryoshka) embedding by specifying a supported dimension. -payload = { - "model": "Qwen/Qwen3-Embedding-0.6B", - "input": "Explain diffusion models simply.", - "dimensions": 512 # change to 128 / 1024 / omit for full size -} - -response = requests.post(url + "/v1/embeddings", json=payload).json() -print("Embedding:", response["data"][0]["embedding"]) -``` - - -## Supported Models - -| Model Family | Example Model | Chat Template | Description | -| ------------------------------------------ | -------------------------------------- | ------------- | --------------------------------------------------------------------------- | -| **E5 (Llama/Mistral based)** | `intfloat/e5-mistral-7b-instruct` | N/A | High-quality text embeddings based on Mistral/Llama architectures | -| **GTE-Qwen2** | `Alibaba-NLP/gte-Qwen2-7B-instruct` | N/A | Alibaba's text embedding model with multilingual support | -| **Qwen3-Embedding** | `Qwen/Qwen3-Embedding-4B` | N/A | Latest Qwen3-based text embedding model for semantic representation | -| **BGE** | `BAAI/bge-large-en-v1.5` | N/A | BAAI's text embeddings (requires `attention-backend` triton/torch_native) | -| **GME (Multimodal)** | `Alibaba-NLP/gme-Qwen2-VL-2B-Instruct`| `gme-qwen2-vl`| Multimodal embedding for text and image cross-modal tasks | -| **CLIP** | `openai/clip-vit-large-patch14-336` | N/A | OpenAI's CLIP for image and text embeddings | diff --git a/docs/supported_models/retrieval_ranking/index.rst b/docs/supported_models/retrieval_ranking/index.rst deleted file mode 100644 index e7c669f9b..000000000 --- a/docs/supported_models/retrieval_ranking/index.rst +++ /dev/null @@ -1,11 +0,0 @@ -Retrieval & Ranking -=================== - -Models for embeddings, reranking, and classification. - -.. toctree:: - :maxdepth: 1 - - embedding_models.md - rerank_models.md - classify_models.md diff --git a/docs/supported_models/retrieval_ranking/rerank_models.md b/docs/supported_models/retrieval_ranking/rerank_models.md deleted file mode 100644 index 12f3e05e2..000000000 --- a/docs/supported_models/retrieval_ranking/rerank_models.md +++ /dev/null @@ -1,314 +0,0 @@ -# Rerank Models - -SGLang offers comprehensive support for rerank models by incorporating optimized serving frameworks with a flexible programming interface. This setup enables efficient processing of cross-encoder reranking tasks, improving the accuracy and relevance of search result ordering. SGLang’s design ensures high throughput and low latency during reranker model deployment, making it ideal for semantic-based result refinement in large-scale retrieval systems. - -```{important} -Rerank models in SGLang fall into two categories: - -- **Cross-encoder rerank models**: run with `--is-embedding` (embedding runner). -- **Decoder-only rerank models**: run **without** `--is-embedding` and use next-token logprob scoring (yes/no). - - Text-only (e.g. Qwen3-Reranker) - - Multimodal (e.g. Qwen3-VL-Reranker): also supports image/video content - -Some models may require `--trust-remote-code`. -``` - -## Supported rerank models - -| Model Family (Rerank) | Example HuggingFace Identifier | Chat Template | Description | -|------------------------------------------------|--------------------------------------|---------------|----------------------------------------------------------------------------------------------------------------------------------| -| **BGE-Reranker (BgeRerankModel)** | `BAAI/bge-reranker-v2-m3` | N/A | Currently only support `attention-backend` `triton` and `torch_native`. High-performance cross-encoder reranker model from BAAI. Suitable for reranking search results based on semantic relevance. | -| **Qwen3-Reranker (decoder-only yes/no)** | `Qwen/Qwen3-Reranker-8B` | `examples/chat_template/qwen3_reranker.jinja` | Decoder-only reranker using next-token logprob scoring for labels (yes/no). Launch **without** `--is-embedding`. | -| **Qwen3-VL-Reranker (multimodal yes/no)** | `Qwen/Qwen3-VL-Reranker-2B` | `examples/chat_template/qwen3_vl_reranker.jinja` | Multimodal decoder-only reranker supporting text, images, and videos. Uses yes/no logprob scoring. Launch **without** `--is-embedding`. | - - -## Cross-Encoder Rerank (embedding runner) - -### Launch Command - -```shell -python3 -m sglang.launch_server \ - --model-path BAAI/bge-reranker-v2-m3 \ - --host 0.0.0.0 \ - --disable-radix-cache \ - --chunked-prefill-size -1 \ - --attention-backend triton \ - --is-embedding \ - --port 30000 -``` - -### Example Client Request - -```python -import requests - -url = "http://127.0.0.1:30000/v1/rerank" - -payload = { - "model": "BAAI/bge-reranker-v2-m3", - "query": "what is panda?", - "documents": [ - "hi", - "The giant panda (Ailuropoda melanoleuca), sometimes called a panda bear or simply panda, is a bear species endemic to China." - ], - "top_n": 1, - "return_documents": True -} - -response = requests.post(url, json=payload) -response_json = response.json() - -for item in response_json: - if item.get("document"): - print(f"Score: {item['score']:.2f} - Document: '{item['document']}'") - else: - print(f"Score: {item['score']:.2f} - Index: {item['index']}") -``` - -**Request Parameters:** - -- `query` (required): The query text to rank documents against -- `documents` (required): List of documents to be ranked -- `model` (required): Model to use for reranking -- `top_n` (optional): Maximum number of documents to return. Defaults to returning all documents. If specified value is greater than the total number of documents, all documents will be returned. -- `return_documents` (optional): Whether to return documents in the response. Defaults to `True`. - -## Qwen3-Reranker (decoder-only yes/no rerank) - -### Launch Command - -```shell -python3 -m sglang.launch_server \ - --model-path Qwen/Qwen3-Reranker-0.6B \ - --trust-remote-code \ - --disable-radix-cache \ - --host 0.0.0.0 \ - --port 8001 \ - --chat-template examples/chat_template/qwen3_reranker.jinja -``` - -```{note} -Qwen3-Reranker uses decoder-only logprob scoring (yes/no). Do NOT launch it with `--is-embedding`. -``` - -### Example Client Request (supports optional instruct, top_n, and return_documents) - -```shell -curl -X POST http://127.0.0.1:8001/v1/rerank \ - -H "Content-Type: application/json" \ - -d '{ - "model": "Qwen3-Reranker-0.6B", - "query": "法国首都是哪里？", - "documents": [ - "法国的首都是巴黎。", - "德国的首都是柏林。", - "香蕉是黄色的水果。" - ], - "instruct": "Given a web search query, retrieve relevant passages that answer the query.", - "top_n": 2, - "return_documents": true - }' -``` - -**Request Parameters:** - -- `query` (required): The query text to rank documents against -- `documents` (required): List of documents to be ranked -- `model` (required): Model to use for reranking -- `instruct` (optional): Instruction text for the reranker -- `top_n` (optional): Maximum number of documents to return. Defaults to returning all documents. If specified value is greater than the total number of documents, all documents will be returned. -- `return_documents` (optional): Whether to return documents in the response. Defaults to `True`. - -### Response Format - -`/v1/rerank` returns a list of objects (sorted by descending score): - -- `score`: float, higher means more relevant -- `document`: the original document string (only included when `return_documents` is `true`) -- `index`: the original index in the input `documents` -- `meta_info`: optional debug/usage info (may be present for some models) - -The number of returned results is controlled by the `top_n` parameter. If `top_n` is not specified or is greater than the total number of documents, all documents are returned. - -Example (with `return_documents: true`): - -```json -[ - {"score": 0.99, "document": "法国的首都是巴黎。", "index": 0}, - {"score": 0.01, "document": "德国的首都是柏林。", "index": 1}, - {"score": 0.00, "document": "香蕉是黄色的水果。", "index": 2} -] -``` - -Example (with `return_documents: false`): - -```json -[ - {"score": 0.99, "index": 0}, - {"score": 0.01, "index": 1}, - {"score": 0.00, "index": 2} -] -``` - -Example (with `top_n: 2`): - -```json -[ - {"score": 0.99, "document": "法国的首都是巴黎。", "index": 0}, - {"score": 0.01, "document": "德国的首都是柏林。", "index": 1} -] -``` - -### Common Pitfalls - -- **`--chat-template` is required.** Without `--chat-template examples/chat_template/qwen3_reranker.jinja`, the server does not recognize the model as a decoder-only reranker and returns a 400 error: `"This model does not appear to be an embedding model by default. Please add `--is-embedding`..."`. The fix is to add the chat template flag, NOT `--is-embedding`. -- If you launch Qwen3-Reranker with `--is-embedding`, `/v1/rerank` cannot compute yes/no logprob scores. Relaunch **without** `--is-embedding`. -- If you see a validation error like "score should be a valid number" and the backend returned a list, upgrade to a version that coerces `embedding[0]` into `score` for rerank responses. - -## Qwen3-VL-Reranker (multimodal decoder-only rerank) - -Qwen3-VL-Reranker extends the Qwen3-Reranker to support multimodal content, allowing reranking of documents containing text, images, and videos. - -### Launch Command - -```shell -python3 -m sglang.launch_server \ - --model-path Qwen/Qwen3-VL-Reranker-2B \ - --trust-remote-code \ - --disable-radix-cache \ - --host 0.0.0.0 \ - --port 30000 \ - --chat-template examples/chat_template/qwen3_vl_reranker.jinja -``` - -```{note} -Qwen3-VL-Reranker uses decoder-only logprob scoring (yes/no) like Qwen3-Reranker. Do NOT launch it with `--is-embedding`. -``` - -### Text-Only Reranking (backward compatible) - -```python -import requests - -url = "http://127.0.0.1:30000/v1/rerank" - -payload = { - "model": "Qwen3-VL-Reranker-2B", - "query": "What is machine learning?", - "documents": [ - "Machine learning is a branch of artificial intelligence that enables computers to learn from data.", - "The weather in Paris is usually mild with occasional rain.", - "Deep learning is a subset of machine learning using neural networks with many layers.", - ], - "instruct": "Retrieve passages that answer the question.", - "return_documents": True -} - -response = requests.post(url, json=payload) -results = response.json() - -for item in results: - print(f"Score: {item['score']:.4f} - {item['document'][:60]}...") -``` - -### Image Reranking (text query, image/mixed documents) - -```python -import requests - -url = "http://127.0.0.1:30000/v1/rerank" - -payload = { - "query": "A woman playing with her dog on a beach at sunset.", - "documents": [ - # Document 1: Text description - "A woman shares a joyful moment with her golden retriever on a sun-drenched beach at sunset.", - # Document 2: Image URL - [ - { - "type": "image_url", - "image_url": { - "url": "https://example.com/beach_dog.jpeg" - } - } - ], - # Document 3: Text + Image (mixed) - [ - {"type": "text", "text": "A joyful scene at the beach:"}, - { - "type": "image_url", - "image_url": { - "url": "https://example.com/beach_dog.jpeg" - } - } - ] - ], - "instruct": "Retrieve images or text relevant to the user's query.", - "return_documents": False -} - -response = requests.post(url, json=payload) -results = response.json() - -for item in results: - print(f"Index: {item['index']}, Score: {item['score']:.4f}") -``` - -### Multimodal Query Reranking (query with image) - -```python -import requests - -url = "http://127.0.0.1:30000/v1/rerank" - -payload = { - # Query with text and image - "query": [ - {"type": "text", "text": "Find similar images to this:"}, - { - "type": "image_url", - "image_url": { - "url": "https://example.com/reference_image.jpeg" - } - } - ], - "documents": [ - "A cat sleeping on a couch.", - "A woman and her dog enjoying the sunset at the beach.", - "A busy city street with cars and pedestrians.", - [ - { - "type": "image_url", - "image_url": { - "url": "https://example.com/similar_image.jpeg" - } - } - ] - ], - "instruct": "Find images or descriptions similar to the query image." -} - -response = requests.post(url, json=payload) -results = response.json() - -for item in results: - print(f"Index: {item['index']}, Score: {item['score']:.4f}") -``` - -### Request Parameters (Multimodal) - -- `query` (required): Can be a string (text-only) or a list of content parts: - - `{"type": "text", "text": "..."}` for text - - `{"type": "image_url", "image_url": {"url": "..."}}` for images - - `{"type": "video_url", "video_url": {"url": "..."}}` for videos -- `documents` (required): List where each document can be a string or list of content parts (same format as query) -- `instruct` (optional): Instruction text for the reranker -- `top_n` (optional): Maximum number of documents to return -- `return_documents` (optional): Whether to return documents in the response (default: `false`) - -### Common Pitfalls - -- Always use `--chat-template examples/chat_template/qwen3_vl_reranker.jinja` for Qwen3-VL-Reranker. -- Do NOT launch with `--is-embedding`. -- For best results, use `--disable-radix-cache` to avoid caching issues with multimodal content. -- **Note**: Currently only `Qwen3-VL-Reranker-2B` is tested and supported. The 8B model may have different behavior and is not guaranteed to work with this template. diff --git a/docs/supported_models/specialized/index.rst b/docs/supported_models/specialized/index.rst deleted file mode 100644 index 40d108acb..000000000 --- a/docs/supported_models/specialized/index.rst +++ /dev/null @@ -1,9 +0,0 @@ -Specialized Models -================== - -Models for specialized tasks like reward modeling. - -.. toctree:: - :maxdepth: 1 - - reward_models.md diff --git a/docs/supported_models/specialized/reward_models.md b/docs/supported_models/specialized/reward_models.md deleted file mode 100644 index ef4474637..000000000 --- a/docs/supported_models/specialized/reward_models.md +++ /dev/null @@ -1,28 +0,0 @@ -# Reward Models - -These models output a scalar reward score or classification result, often used in reinforcement learning or content moderation tasks. - -```{important} -They are executed with `--is-embedding` and some may require `--trust-remote-code`. -``` - -## Example launch Command - -```shell -python3 -m sglang.launch_server \ - --model-path Qwen/Qwen2.5-Math-RM-72B \ # example HF/local path - --is-embedding \ - --host 0.0.0.0 \ - --tp-size=4 \ # set for tensor parallelism - --port 30000 \ -``` - -## Supported models - -| Model Family (Reward) | Example HuggingFace Identifier | Description | -|---------------------------------------------------------------------------|-----------------------------------------------------|---------------------------------------------------------------------------------| -| **Llama (3.1 Reward / `LlamaForSequenceClassification`)** | `Skywork/Skywork-Reward-Llama-3.1-8B-v0.2` | Reward model (preference classifier) based on Llama 3.1 (8B) for scoring and ranking responses for RLHF. | -| **Gemma 2 (27B Reward / `Gemma2ForSequenceClassification`)** | `Skywork/Skywork-Reward-Gemma-2-27B-v0.2` | Derived from Gemma‑2 (27B), this model provides human preference scoring for RLHF and multilingual tasks. | -| **InternLM 2 (Reward / `InternLM2ForRewardMode`)** | `internlm/internlm2-7b-reward` | InternLM 2 (7B)–based reward model used in alignment pipelines to guide outputs toward preferred behavior. | -| **Qwen2.5 (Reward - Math / `Qwen2ForRewardModel`)** | `Qwen/Qwen2.5-Math-RM-72B` | A 72B math-specialized RLHF reward model from the Qwen2.5 series, tuned for evaluating and refining responses. | -| **Qwen2.5 (Reward - Sequence / `Qwen2ForSequenceClassification`)** | `jason9693/Qwen2.5-1.5B-apeach` | A smaller Qwen2.5 variant used for sequence classification, offering an alternative RLHF scoring mechanism. | diff --git a/docs/supported_models/text_generation/diffusion_language_models.md b/docs/supported_models/text_generation/diffusion_language_models.md deleted file mode 100644 index 7dbb4828b..000000000 --- a/docs/supported_models/text_generation/diffusion_language_models.md +++ /dev/null @@ -1,111 +0,0 @@ -# Diffusion Language Models - -Diffusion language models have shown promise for non-autoregressive text generation with parallel decoding capabilities. Unlike auto-regressive language models, different diffusion language models require different decoding strategies. - -## Example Launch Command - -SGLang supports different DLLM algorithms such as `LowConfidence` and `JointThreshold`. - -```shell -python3 -m sglang.launch_server \ - --model-path inclusionAI/LLaDA2.0-mini \ # example HF/local path - --dllm-algorithm LowConfidence \ - --dllm-algorithm-config ./config.yaml \ # Optional. Uses the algorithm's default if not set. - --host 0.0.0.0 \ - --port 30000 -``` - -## Example Configuration File - -Depending on the algorithm selected, the configuration parameters vary. - -LowConfidence Config: - -```yaml -# Confidence threshold for accepting predicted tokens -# - Higher values: More conservative, better quality but slower -# - Lower values: More aggressive, faster but potentially lower quality -# Range: 0.0 - 1.0 -threshold: 0.95 - -# Default: 32, for LLaDA2MoeModelLM -block_size: 32 -``` - -JointThreshold Config: - -```yaml -# Decoding threshold for Mask-to-Token (M2T) phase -# - Higher values: More conservative, better quality but slower -# - Lower values: More aggressive, faster but potentially lower quality -# Range: 0.0 - 1.0 -threshold: 0.5 -# Decoding threshold for Token-to-Token (T2T) phase -# Range: 0.0 - 1.0 -# Setting to 0.0 allows full editing (recommended for most cases). -edit_threshold: 0.0 -# Max extra T2T steps after all masks are removed. Prevents infinite loops. -max_post_edit_steps: 16 -# 2-gram repetition penalty (default 0). -# An empirical value of 3 is often sufficient to mitigate most repetitions. -penalty_lambda: 0 -``` - -## Example Client Code Snippet - -Just like other supported models, diffusion language models can be used via the REST API or Python client. - -Python client example for making a generation request to the launched server: - -```python -import sglang as sgl - -def main(): - llm = sgl.Engine(model_path="inclusionAI/LLaDA2.0-mini", - dllm_algorithm="LowConfidence", - max_running_requests=1, - trust_remote_code=True) - - prompts = [ - "SYSTEMdetailed thinking off<|role_end|>HUMAN Write a brief introduction of the great wall <|role_end|>ASSISTANT" - ] - - sampling_params = { - "temperature": 0, - "max_new_tokens": 1024, - } - - outputs = llm.generate(prompts, sampling_params) - print(outputs) - -if __name__ == '__main__': - main() -``` - -Curl example for making a generation request to the launched server: - -```bash -curl -X POST "http://127.0.0.1:30000/generate" \ - -H "Content-Type: application/json" \ - -d '{ - "text": [ - "SYSTEMdetailed thinking off<|role_end|>HUMAN Write the number from 1 to 128 <|role_end|>ASSISTANT", - "SYSTEMdetailed thinking off<|role_end|>HUMAN Write a brief introduction of the great wall <|role_end|>ASSISTANT" - ], - "stream": true, - "sampling_params": { - "temperature": 0, - "max_new_tokens": 1024 - } - }' -``` - -## Supported Models - -Below the supported models are summarized in a table. - -| Model Family | Example Model | Description | -| -------------------------- | ---------------------------- | ---------------------------------------------------------------------------------------------------- | -| **LLaDA2.0 (mini, flash)** | `inclusionAI/LLaDA2.0-flash` | LLaDA2.0-flash is a diffusion language model featuring a 100B Mixture-of-Experts (MoE) architecture. | -| **SDAR (JetLM)** | `JetLM/SDAR-8B-Chat` | SDAR series diffusion language model (Chat), dense architecture. | -| **SDAR (JetLM)** | `JetLM/SDAR-30B-A3B-Chat` | SDAR series diffusion language model (Chat), MoE architecture. | diff --git a/docs/supported_models/text_generation/generative_models.md b/docs/supported_models/text_generation/generative_models.md deleted file mode 100644 index a3e263f68..000000000 --- a/docs/supported_models/text_generation/generative_models.md +++ /dev/null @@ -1,76 +0,0 @@ -# Large Language Models - -These models accept text input and produce text output (e.g., chat completions). They are primarily large language models (LLMs), some with mixture-of-experts (MoE) architectures for scaling. - -## Example launch Command - -```shell -python3 -m sglang.launch_server \ - --model-path meta-llama/Llama-3.2-1B-Instruct \ # example HF/local path - --host 0.0.0.0 \ - --port 30000 \ -``` - -## Supported models - -Below the supported models are summarized in a table. - -If you are unsure if a specific architecture is implemented, you can search for it via GitHub. For example, to search for `Qwen3ForCausalLM`, use the expression: - -``` -repo:sgl-project/sglang path:/^python\/sglang\/srt\/models\// Qwen3ForCausalLM -``` - -in the GitHub search bar. - -| Model Family (Variants) | Example HuggingFace Identifier | Description | -|-------------------------------------|--------------------------------------------------|----------------------------------------------------------------------------------------| -| **DeepSeek** (v1, v2, v3/R1) | `deepseek-ai/DeepSeek-R1` | Series of advanced reasoning-optimized models (including a 671B MoE) trained with reinforcement learning; top performance on complex reasoning, math, and code tasks. [SGLang provides Deepseek v3/R1 model-specific optimizations](../../basic_usage/deepseek_v3.md) and [Reasoning Parser](../../advanced_features/separate_reasoning.ipynb)| -| **Kimi K2** (Thinking, Instruct) | `moonshotai/Kimi-K2-Instruct` | Moonshot AI's 1 trillion parameter MoE model (32B active) with 128K–256K context; state-of-the-art agentic intelligence with stable long-horizon agency across 200–300 sequential tool calls. Features MLA attention and native INT4 quantization. [See Reasoning Parser docs](../../advanced_features/separate_reasoning.ipynb)| -| **Kimi Linear** (48B-A3B) | `moonshotai/Kimi-Linear-48B-A3B-Instruct` | Moonshot AI's hybrid linear attention model (48B total, 3B active) with 1M token context; features Kimi Delta Attention (KDA) for up to 6× faster decoding and 75% KV cache reduction vs full attention. | -| **GPT-OSS** | `openai/gpt-oss-20b`, `openai/gpt-oss-120b` | OpenAI’s latest GPT-OSS series for complex reasoning, agentic tasks, and versatile developer use cases.| -| **Qwen** (3.5, 3, 3MoE, 3Next, 2.5, 2 series) | `Qwen/Qwen3.5-397B-A17B`, `Qwen/Qwen3-0.6B`, `Qwen/Qwen3-30B-A3B`, `Qwen/Qwen3-Next-80B-A3B-Instruct` | Alibaba’s latest Qwen3 series for complex reasoning, language understanding, and generation tasks; Support for MoE variants along with previous generation 2.5, 2, etc. [SGLang provides Qwen3 specific reasoning parser](../../advanced_features/separate_reasoning.ipynb)| -| **Llama** (2, 3.x, 4 series) | `meta-llama/Llama-4-Scout-17B-16E-Instruct` | Meta's open LLM series, spanning 7B to 400B parameters (Llama 2, 3, and new Llama 4) with well-recognized performance. [SGLang provides Llama-4 model-specific optimizations](../../basic_usage/llama4.md) | -| **Mistral** (Mixtral, NeMo, Small3) | `mistralai/Mistral-7B-Instruct-v0.2` | Open 7B LLM by Mistral AI with strong performance; extended into MoE (“Mixtral”) and NeMo Megatron variants for larger scale. | -| **Gemma** (v1, v2, v3) | `google/gemma-3-1b-it` | Google’s family of efficient multilingual models (1B–27B); Gemma 3 offers a 128K context window, and its larger (4B+) variants support vision input. | -| **Phi** (Phi-1.5, Phi-2, Phi-3, Phi-4, Phi-MoE series) | `microsoft/Phi-4-multimodal-instruct`, `microsoft/Phi-3.5-MoE-instruct` | Microsoft’s Phi family of small models (1.3B–5.6B); Phi-4-multimodal (5.6B) processes text, images, and speech, Phi-4-mini is a high-accuracy text model and Phi-3.5-MoE is a mixture-of-experts model. | -| **MiniCPM** (v3, 4B) | `openbmb/MiniCPM3-4B` | OpenBMB’s series of compact LLMs for edge devices; MiniCPM 3 (4B) achieves GPT-3.5-level results in text tasks. | -| **OLMo** (2, 3) | `allenai/OLMo-3-1125-32B`, `allenai/OLMo-3-32B-Think`, `allenai/OLMo-2-1124-7B-Instruct` | Allen AI’s series of Open Language Models designed to enable the science of language models. | -| **OLMoE** (Open MoE) | `allenai/OLMoE-1B-7B-0924` | Allen AI’s open Mixture-of-Experts model (7B total, 1B active parameters) delivering state-of-the-art results with sparse expert activation. | -| **MiniMax-M2** (M2, M2.1, M2.5) | `MiniMaxAI/MiniMax-M2.5`, `MiniMaxAI/MiniMax-M2.1`, `MiniMaxAI/MiniMax-M2` | MiniMax's SOTA LLM for coding & agentic workflows. | -| **StableLM** (3B, 7B) | `stabilityai/stablelm-tuned-alpha-7b` | StabilityAI’s early open-source LLM (3B & 7B) for general text generation; a demonstration model with basic instruction-following ability. | -| **Command-(R,A)** (Cohere) | `CohereLabs/c4ai-command-r-v01`, `CohereLabs/c4ai-command-r7b-12-2024`, `CohereLabs/c4ai-command-a-03-2025` | Cohere’s open conversational LLM (Command series) optimized for long context, retrieval-augmented generation, and tool use. | -| **DBRX** (Databricks) | `databricks/dbrx-instruct` | Databricks’ 132B-parameter MoE model (36B active) trained on 12T tokens; competes with GPT-3.5 quality as a fully open foundation model. | -| **Grok** (xAI) | `xai-org/grok-1` | xAI’s grok-1 model known for vast size(314B parameters) and high quality; integrated in SGLang for high-performance inference. | -| **ChatGLM** (GLM-130B family) | `THUDM/chatglm2-6b` | Zhipu AI’s bilingual chat model (6B) excelling at Chinese-English dialogue; fine-tuned for conversational quality and alignment. | -| **InternLM 2** (7B, 20B) | `internlm/internlm2-7b` | Next-gen InternLM (7B and 20B) from SenseTime, offering strong reasoning and ultra-long context support (up to 200K tokens). | -| **ExaONE 3** (Korean-English) | `LGAI-EXAONE/EXAONE-3.5-7.8B-Instruct` | LG AI Research’s Korean-English model (7.8B) trained on 8T tokens; provides high-quality bilingual understanding and generation. | -| **Baichuan 2** (7B, 13B) | `baichuan-inc/Baichuan2-13B-Chat` | BaichuanAI’s second-generation Chinese-English LLM (7B/13B) with improved performance and an open commercial license. | -| **XVERSE** (MoE) | `xverse/XVERSE-MoE-A36B` | Yuanxiang’s open MoE LLM (XVERSE-MoE-A36B: 255B total, 36B active) supporting ~40 languages; delivers 100B+ dense-level performance via expert routing. | -| **SmolLM** (135M–1.7B) | `HuggingFaceTB/SmolLM-1.7B` | Hugging Face’s ultra-small LLM series (135M–1.7B params) offering surprisingly strong results, enabling advanced AI on mobile/edge devices. | -| **GLM-4** (Multilingual 9B) | `ZhipuAI/glm-4-9b-chat` | Zhipu’s GLM-4 series (up to 9B parameters) – open multilingual models with support for 1M-token context and even a 5.6B multimodal variant (Phi-4V). | -| **MiMo** (7B series) | `XiaomiMiMo/MiMo-7B-RL` | Xiaomi's reasoning-optimized model series, leverages Multiple-Token Prediction for faster inference. | -| **ERNIE-4.5** (4.5, 4.5MoE series) | `baidu/ERNIE-4.5-21B-A3B-PT` | Baidu's ERNIE-4.5 series which consists of MoE with 47B and 3B active parameters, with the largest model having 424B total parameters, as well as a 0.3B dense model. | -| **Arcee AFM-4.5B** | `arcee-ai/AFM-4.5B-Base` | Arcee's foundational model series for real world reliability and edge deployments. | -| **Persimmon** (8B) | `adept/persimmon-8b-chat` | Adept’s open 8B model with a 16K context window and fast inference; trained for broad usability and licensed under Apache 2.0. | -| **Solar** (10.7B) | `upstage/SOLAR-10.7B-Instruct-v1.0` | Upstage's 10.7B parameter model, optimized for instruction-following tasks. This architecture incorporates a depth-up scaling methodology, enhancing model performance. | -| **Tele FLM** (52B-1T) | `CofeAI/Tele-FLM` | BAAI & TeleAI's multilingual model, available in 52-billion and 1-trillion parameter variants. It is a decoder-only transformer trained on ~2T tokens | -| **Ling** (16.8B–290B) | `inclusionAI/Ling-lite`, `inclusionAI/Ling-plus` | InclusionAI’s open MoE models. Ling-Lite has 16.8B total / 2.75B active parameters, and Ling-Plus has 290B total / 28.8B active parameters. They are designed for high performance on NLP and complex reasoning tasks. | -| **Granite 3.0, 3.1** (IBM) | `ibm-granite/granite-3.1-8b-instruct` | IBM's open dense foundation models optimized for reasoning, code, and business AI use cases. Integrated with Red Hat and watsonx systems. | -| **Granite 3.0 MoE** (IBM) | `ibm-granite/granite-3.0-3b-a800m-instruct` | IBM’s Mixture-of-Experts models offering strong performance with cost-efficiency. MoE expert routing designed for enterprise deployment at scale. | -| **GPT-J** (6B) | `EleutherAI/gpt-j-6b` | EleutherAI's GPT-2-like causal language model (6B) trained on the [Pile](https://pile.eleuther.ai/) dataset. | -| **Orion** (14B) | `OrionStarAI/Orion-14B-Base` | A series of open-source multilingual large language models by OrionStarAI, pretrained on a 2.5T token multilingual corpus including Chinese, English, Japanese, Korean, etc, and it exhibits superior performance in these languages. | -| **Llama Nemotron Super** (v1, v1.5, NVIDIA) | `nvidia/Llama-3_3-Nemotron-Super-49B-v1`, `nvidia/Llama-3_3-Nemotron-Super-49B-v1_5` | The [NVIDIA Nemotron](https://www.nvidia.com/en-us/ai-data-science/foundation-models/nemotron/) family of multimodal models provides state-of-the-art reasoning models specifically designed for enterprise-ready AI agents. | -| **Llama Nemotron Ultra** (v1, NVIDIA) | `nvidia/Llama-3_1-Nemotron-Ultra-253B-v1` | The [NVIDIA Nemotron](https://www.nvidia.com/en-us/ai-data-science/foundation-models/nemotron/) family of multimodal models provides state-of-the-art reasoning models specifically designed for enterprise-ready AI agents. | -| **NVIDIA Nemotron Nano 2.0** | `nvidia/NVIDIA-Nemotron-Nano-9B-v2` | The [NVIDIA Nemotron](https://www.nvidia.com/en-us/ai-data-science/foundation-models/nemotron/) family of multimodal models provides state-of-the-art reasoning models specifically designed for enterprise-ready AI agents. `Nemotron-Nano-9B-v2` is a hybrid Mamba-Transformer language model designed to increase throughput for reasoning workloads while achieving state-of-the-art accuracy compared to similarly-sized models. | -| **NVIDIA Nemotron 3 Super** (NVIDIA) | `nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4` | The [NVIDIA Nemotron](https://www.nvidia.com/en-us/ai-data-science/foundation-models/nemotron/) 3 Super is a 120B-parameter MoE model (12B active) delivering high-quality reasoning and generation for enterprise AI agents. | -| **NVIDIA Nemotron 3 Nano** (NVIDIA) | `nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16` | The [NVIDIA Nemotron](https://www.nvidia.com/en-us/ai-data-science/foundation-models/nemotron/) 3 Nano is a compact model designed for efficient edge and enterprise deployment with strong reasoning capabilities. | -| **StarCoder2** (3B-15B) | `bigcode/starcoder2-7b` | StarCoder2 is a family of open large language models (LLMs) specialized for code generation and understanding. It is the successor to StarCoder, jointly developed by the BigCode project (a collaboration between Hugging Face, ServiceNow Research, and other contributors). | -| **Jet-Nemotron** | `jet-ai/Jet-Nemotron-2B` | Jet-Nemotron is a new family of hybrid-architecture language models that surpass state-of-the-art open-source full-attention language models, while achieving significant efficiency gains. | -| **Trinity** (Nano, Mini) | `arcee-ai/Trinity-Mini` | Arcee's foundational MoE Trinity family of models, open weights under Apache 2.0. | -| **LFM2** (350M, 1.2B) | `LiquidAI/LFM2.5-1.2B-Instruct` | Liquid AI's hybrid attention + short convolution language model. | -| **LFM2-MoE** (8B-A1B, 24B-A2B) | `LiquidAI/LFM2-8B-A1B` | Liquid AI's Mixture-of-Experts variant with sigmoid routing and top-k expert selection. | -| **Falcon-H1** (0.5B–34B) | `tiiuae/Falcon-H1-34B-Instruct` | TII's hybrid Mamba-Transformer architecture combining attention and state-space models for efficient long-context inference. | -| **Hunyuan-Large** (389B, MoE) | `tencent/Tencent-Hunyuan-Large` | Tencent's open-source MoE model with 389B total / 52B active parameters, featuring Cross-Layer Attention (CLA) for improved efficiency. | -| **IBM Granite 4.0 (Hybrid, Dense)** | `ibm-granite/granite-4.0-h-micro`, `ibm-granite/granite-4.0-micro` | IBM Granite 4.0 micro models: hybrid Mamba–MoE (`h-micro`) and dense (`micro`) variants. Enterprise-focused reasoning models | -| **Sarvam 2** (30B-A2B, 105B-A10B) | `sarvamai/sarvam-2` | Sarvam's Mixture-of-Experts models. The 105B variant uses MLA (Multi-head Latent Attention) and the 30B variant uses GQA, both with 128 routed experts. | diff --git a/docs/supported_models/text_generation/index.rst b/docs/supported_models/text_generation/index.rst deleted file mode 100644 index e315f83d1..000000000 --- a/docs/supported_models/text_generation/index.rst +++ /dev/null @@ -1,11 +0,0 @@ -Text Generation -=============== - -Models for generating text from text or multimodal inputs. - -.. toctree:: - :maxdepth: 1 - - generative_models.md - multimodal_language_models.md - diffusion_language_models.md diff --git a/docs/supported_models/text_generation/multimodal_language_models.md b/docs/supported_models/text_generation/multimodal_language_models.md deleted file mode 100644 index a12113f6b..000000000 --- a/docs/supported_models/text_generation/multimodal_language_models.md +++ /dev/null @@ -1,166 +0,0 @@ -# Multimodal Language Models - -These models accept multi-modal inputs (e.g., images and text) and generate text output. They augment language models with multimodal encoders. - -## Example launch Command - -```shell -python3 -m sglang.launch_server \ - --model-path meta-llama/Llama-3.2-11B-Vision-Instruct \ # example HF/local path - --host 0.0.0.0 \ - --port 30000 \ -``` - -> See the [OpenAI APIs section](https://docs.sglang.io/basic_usage/openai_api_vision.html) for how to send multimodal requests. - -## Supported models - -Below the supported models are summarized in a table. - -If you are unsure if a specific architecture is implemented, you can search for it via GitHub. For example, to search for `Qwen2_5_VLForConditionalGeneration`, use the expression: - -``` -repo:sgl-project/sglang path:/^python\/sglang\/srt\/models\// Qwen2_5_VLForConditionalGeneration -``` - -in the GitHub search bar. - - -| Model Family (Variants) | Example HuggingFace Identifier | Description | Notes | -|----------------------------|--------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------| -| **Qwen-VL** | `Qwen/Qwen3-VL-235B-A22B-Instruct` | Alibaba's vision-language extension of Qwen; for example, Qwen2.5-VL (7B and larger variants) can analyze and converse about image content. | | -| **DeepSeek-VL2** | `deepseek-ai/deepseek-vl2` | Vision-language variant of DeepSeek (with a dedicated image processor), enabling advanced multimodal reasoning on image and text inputs. | | -| **DeepSeek-OCR / OCR-2** | `deepseek-ai/DeepSeek-OCR-2` | OCR-focused DeepSeek models for document understanding and text extraction. | Use `--trust-remote-code`. | -| **Janus-Pro** (1B, 7B) | `deepseek-ai/Janus-Pro-7B` | DeepSeek's open-source multimodal model capable of both image understanding and generation. Janus-Pro employs a decoupled architecture for separate visual encoding paths, enhancing performance in both tasks. | | -| **MiniCPM-V / MiniCPM-o** | `openbmb/MiniCPM-V-2_6` | MiniCPM-V (2.6, ~8B) supports image inputs, and MiniCPM-o adds audio/video; these multimodal LLMs are optimized for end-side deployment on mobile/edge devices. | | -| **Llama 3.2 Vision** (11B) | `meta-llama/Llama-3.2-11B-Vision-Instruct` | Vision-enabled variant of Llama 3 (11B) that accepts image inputs for visual question answering and other multimodal tasks. | | -| **LLaVA** (v1.5 & v1.6) | *e.g.* `liuhaotian/llava-v1.5-13b` | Open vision-chat models that add an image encoder to LLaMA/Vicuna (e.g. LLaMA2 13B) for following multimodal instruction prompts. | | -| **LLaVA-NeXT** (8B, 72B) | `lmms-lab/llava-next-72b` | Improved LLaVA models (with an 8B Llama3 version and a 72B version) offering enhanced visual instruction-following and accuracy on multimodal benchmarks. | | -| **LLaVA-OneVision** | `lmms-lab/llava-onevision-qwen2-7b-ov` | Enhanced LLaVA variant integrating Qwen as the backbone; supports multiple images (and even video frames) as inputs via an OpenAI Vision API-compatible format. | | -| **Gemma 3 (Multimodal)** | `google/gemma-3-4b-it` | Gemma 3's larger models (4B, 12B, 27B) accept images (each image encoded as 256 tokens) alongside text in a combined 128K-token context. | | -| **Kimi-VL** (A3B) | `moonshotai/Kimi-VL-A3B-Instruct` | Kimi-VL is a multimodal model that can understand and generate text from images. | | -| **Mistral-Small-3.1-24B** | `mistralai/Mistral-Small-3.1-24B-Instruct-2503` | Mistral 3.1 is a multimodal model that can generate text from text or images input. It also supports tool calling and structured output. | | -| **Phi-4-multimodal-instruct** | `microsoft/Phi-4-multimodal-instruct` | Phi-4-multimodal-instruct is the multimodal variant of the Phi-4-mini model, enhanced with LoRA for improved multimodal capabilities. It supports text, vision and audio modalities in SGLang. | | -| **MiMo-VL** (7B) | `XiaomiMiMo/MiMo-VL-7B-RL` | Xiaomi's compact yet powerful vision-language model featuring a native resolution ViT encoder for fine-grained visual details, an MLP projector for cross-modal alignment, and the MiMo-7B language model optimized for complex reasoning tasks. | | -| **GLM-4.5V** (106B) / **GLM-4.1V**(9B) | `zai-org/GLM-4.5V` | GLM-4.5V and GLM-4.1V-Thinking: Towards Versatile Multimodal Reasoning with Scalable Reinforcement Learning | Use `--chat-template glm-4v` | -| **GLM-OCR** | `zai-org/GLM-OCR` | GLM-OCR: A fast and accurate general OCR model | | -| **DotsVLM** (General/OCR) | `rednote-hilab/dots.vlm1.inst` | RedNote's vision-language model built on a 1.2B vision encoder and DeepSeek V3 LLM, featuring NaViT vision encoder trained from scratch with dynamic resolution support and enhanced OCR capabilities through structured image data training. | | -| **DotsVLM-OCR** | `rednote-hilab/dots.ocr` | Specialized OCR variant of DotsVLM optimized for optical character recognition tasks with enhanced text extraction and document understanding capabilities. | Don't use `--trust-remote-code` | -| **NVILA** (8B, 15B, Lite-2B, Lite-8B, Lite-15B) | `Efficient-Large-Model/NVILA-8B` | `chatml` | NVILA explores the full stack efficiency of multi-modal design, achieving cheaper training, faster deployment and better performance. | -| **NVIDIA Nemotron Nano 2.0 VL** | `nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16` | NVIDIA Nemotron Nano v2 VL enables multi-image reasoning and video understanding, along with strong document intelligence, visual Q&A and summarization capabilities. It builds on Nemotron Nano V2, a hybrid Mamba-Transformer LLM, in order to achieve higher inference throughput in long document and video scenarios. | Use `--trust-remote-code`. You may need to adjust `--max-mamba-cache-size` [default is 512] to fit memory constraints. | -| **Ernie4.5-VL** | `baidu/ERNIE-4.5-VL-28B-A3B-PT` | Baidu's vision-language models(28B,424B). Support image and video comprehension, and also support thinking. | | -| **JetVLM** | | JetVLM is an vision-language model designed for high-performance multimodal understanding and generation tasks built upon Jet-Nemotron. | Coming soon | -| **Step3-VL** (10B) | `stepfun-ai/Step3-VL-10B` | StepFun's lightweight open-source 10B parameter VLM for multimodal intelligence, excelling in visual perception, complex reasoning, and human alignment. | | -| **Qwen3-ASR** (0.6B, 1.7B) | `Qwen/Qwen3-ASR-1.7B` | Alibaba's automatic speech recognition models supporting 52 languages. Served via the `/v1/audio/transcriptions` endpoint. | | -| **Qwen3-Omni** | `Qwen/Qwen3-Omni-30B-A3B-Instruct` | Alibaba's omni-modal MoE model. Currently supports the **Thinker** component (multimodal understanding for text, images, audio, and video), while the **Talker** component (audio generation) is not yet supported. | | -| **LFM2-VL** | `LiquidAI/LFM2.5-VL-1.6B` | Liquid AI's vision-language model combining a SigLip2 vision encoder (NaFlex variable-resolution) with the LFM2 hybrid attention + short convolution language model. Supports multi-image inputs. | | - -## Audio Transcription - -SGLang supports audio-only ASR models via the OpenAI-compatible `/v1/audio/transcriptions` endpoint. Upload an audio file and receive a transcription. - -### Launch Command - -```shell -sglang serve \ - --model-path Qwen/Qwen3-ASR-1.7B \ - --served-model-name qwen3-asr \ - --trust-remote-code \ - --host 0.0.0.0 --port 30000 -``` - -### Example Request - -```bash -curl http://localhost:30000/v1/audio/transcriptions \ - -F file=@audio.wav \ - -F model=qwen3-asr \ - -F response_format=verbose_json -``` - -| Model Family | Example Identifier | Notes | -|--------------|--------------------|-------| -| **Whisper** | `openai/whisper-large-v3` | OpenAI's speech recognition model. | -| **Qwen3-ASR** (0.6B, 1.7B) | `Qwen/Qwen3-ASR-1.7B` | Use `--trust-remote-code`. Supports 52 languages. | - -## Video Input Support - -SGLang supports video input for Vision-Language Models (VLMs), enabling temporal reasoning tasks such as video question answering, captioning, and holistic scene understanding. Video clips are decoded, key frames are sampled, and the resulting tensors are batched together with the text prompt, allowing multimodal inference to integrate visual and linguistic context. - -| Model Family | Example Identifier | Video notes | -|--------------|--------------------|-------------| -| **Qwen-VL** (Qwen2-VL, Qwen2.5-VL, Qwen3-VL, Qwen3-Omni) | `Qwen/Qwen3-VL-235B-A22B-Instruct` | The processor gathers `video_data`, runs Qwen's frame sampler, and merges the resulting features with text tokens before inference. | -| **GLM-4v** (4.5V, 4.1V, MOE) | `zai-org/GLM-4.5V` | Video clips are read with Decord, converted to tensors, and passed to the model alongside metadata for rotary-position handling. | -| **NVILA** (Full & Lite) | `Efficient-Large-Model/NVILA-8B` | The runtime samples eight frames per clip and attaches them to the multimodal request when `video_data` is present. | -| **LLaVA video variants** (LLaVA-NeXT-Video, LLaVA-OneVision) | `lmms-lab/LLaVA-NeXT-Video-7B` | The processor routes video prompts to the LlavaVid video-enabled architecture, and the provided example shows how to query it with `sgl.video(...)` clips. | -| **NVIDIA Nemotron Nano 2.0 VL** | `nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16` | The processor samples at 2 FPS, at a max of 128 frames, as per model training. The model uses [EVS](../../../python/sglang/srt/multimodal/evs/README.md), a pruning method that removes redundant tokens from video embeddings. By default `video_pruning_rate=0.7`. Change this by providing: `--json-model-override-args '{"video_pruning_rate": 0.0}'` to disable EVS, for example. | -| **JetVLM** | | The runtime samples eight frames per clip and attaches them to the multimodal request when `video_data` is present. | - -Use `sgl.video(path, num_frames)` when building prompts to attach clips from your SGLang programs. - -Example OpenAI-compatible request that sends a video clip: - -```python -import requests - -url = "http://localhost:30000/v1/chat/completions" - -data = { - "model": "Qwen/Qwen3-VL-30B-A3B-Instruct", - "messages": [ - { - "role": "user", - "content": [ - {"type": "text", "text": "What’s happening in this video?"}, - { - "type": "video_url", - "video_url": { - "url": "https://github.com/sgl-project/sgl-test-files/raw/refs/heads/main/videos/jobs_presenting_ipod.mp4" - }, - }, - ], - } - ], - "max_tokens": 300, -} - -response = requests.post(url, json=data) -print(response.text) -``` - -## Usage Notes - -### Performance Optimization - -For multimodal models, you can use the `--keep-mm-feature-on-device` flag to optimize for latency at the cost of increased GPU memory usage: - -- **Default behavior**: Multimodal feature tensors are moved to CPU after processing to save GPU memory -- **With `--keep-mm-feature-on-device`**: Feature tensors remain on GPU, reducing device-to-host copy overhead and improving latency, but consuming more GPU memory - -Use this flag when you have sufficient GPU memory and want to minimize latency for multimodal inference. - -### Multimodal Inputs Limitation - -- **Use `--mm-process-config '{"image":{"max_pixels":1048576},"video":{"fps":3,"max_pixels":602112,"max_frames":60}}'`**: To set `image`, `video`, and `audio` input limits. - -This can reduce GPU memory usage, improve inference speed, and help to avoid OOM, but may impact model performance, thus set a proper value based on your specific use case. The config entries are passed as `images_kwargs`, `videos_kwargs`, and `audio_kwargs` to the HuggingFace processor, so each modality's settings are kept separate and do not collide. Refer to the HuggingFace documentation for your model's processor to understand the available parameters. - -### Bidirectional Attention in Multimodal Model Serving -**Note for serving the Gemma-3 multimodal model**: - -As mentioned in [Welcome Gemma 3: Google's all new multimodal, multilingual, long context open LLM -](https://huggingface.co/blog/gemma3#multimodality), Gemma-3 employs bidirectional attention between image tokens during the prefill phase. Currently, SGLang only supports bidirectional attention when using the Triton Attention Backend. Note, however, that SGLang's current bidirectional attention implementation is incompatible with both CUDA Graph and Chunked Prefill. - -To enable bidirectional attention, you can use the `TritonAttnBackend` while disabling CUDA Graph and Chunked Prefill. Example launch command: -```shell -python -m sglang.launch_server \ - --model-path google/gemma-3-4b-it \ - --host 0.0.0.0 --port 30000 \ - --enable-multimodal \ - --dtype bfloat16 --triton-attention-reduce-in-fp32 \ - --attention-backend triton \ # Use Triton attention backend - --disable-cuda-graph \ # Disable Cuda Graph - --chunked-prefill-size -1 # Disable Chunked Prefill -``` - -If higher serving performance is required and a certain degree of accuracy loss is acceptable, you may choose to use other attention backends, and you can also enable features like CUDA Graph and Chunked Prefill for better performance, but note that the model will fall back to using causal attention instead of bidirectional attention. diff --git a/docs/wrap_run_llm.py b/docs/wrap_run_llm.py deleted file mode 100644 index 2d21442cf..000000000 --- a/docs/wrap_run_llm.py +++ /dev/null @@ -1,47 +0,0 @@ -import os -import re - - -def insert_runllm_widget(html_content): - # RunLLM Widget script to be inserted - widget_script = """ - - - """ - - # Find the closing body tag and insert the widget script before it - return re.sub(r"", f"{widget_script}\n", html_content) - - -def process_html_files(build_dir): - for root, dirs, files in os.walk(build_dir): - for file in files: - if file.endswith(".html"): - file_path = os.path.join(root, file) - - # Read the HTML file - with open(file_path, "r", encoding="utf-8") as f: - content = f.read() - - # Insert the RunLLM widget - modified_content = insert_runllm_widget(content) - - # Write back the modified content - with open(file_path, "w", encoding="utf-8") as f: - f.write(modified_content) - - -def main(): - # Get the build directory path - build_dir = os.path.join( - os.path.dirname(os.path.abspath(__file__)), "_build", "html" - ) - # Process all HTML files - if os.path.exists(build_dir): - process_html_files(build_dir) - else: - print(f"Build directory not found: {build_dir}") - - -if __name__ == "__main__": - main() diff --git a/docs_new/README.md b/docs_new/README.md index c3b11bce2..a975619b4 100644 --- a/docs_new/README.md +++ b/docs_new/README.md @@ -15,11 +15,17 @@ The official documentation and cookbook for [SGLang](https://github.com/sgl-proj ├── docs/ # Documentation pages │ └── get-started/ │ └── install.mdx # Installation guide -└── cookbook/ # Model deployment recipes - ├── intro.mdx # Cookbook overview and recipe index - └── autoregressive/ # Autoregressive model recipes - └── Qwen/ - └── Qwen3.5.mdx +├── cookbook/ # Model deployment recipes (one .mdx page per model) +│ ├── intro.mdx # Cookbook overview and recipe index +│ └── autoregressive/ # Autoregressive recipes (also: diffusion/, omni/, …) +│ └── DeepSeek/ +│ └── DeepSeek-V4.mdx +└── src/snippets/ # Config-driven cookbook engine + ├── _deployment.jsx # Shared deploy-matrix engine (no model-specific code) + ├── _playground.jsx # Shared override-playground engine + └── configs/ + └── deepseek-ai/ # Per-model config + benchmarks (HF-org folder) + └── deepseek-v4.jsx ``` Pages are `.mdx` files with YAML frontmatter. Navigation is defined in `docs.json`. @@ -36,7 +42,7 @@ Pages are `.mdx` files with YAML frontmatter. Navigation is defined in `docs.jso # Install the CLI npm i -g mint -# Start the dev server (with hot reload) +# From docs_new/ (where docs.json lives), start the dev server (hot reload) mint dev ``` @@ -63,9 +69,9 @@ We welcome contributions! Whether you want to add a recipe for a new model, impr ### Local development workflow ```bash -# 1. Fork and clone the repo -git clone https://github.com//sgl-docs.git -cd sgl-docs +# 1. Fork sgl-project/sglang and clone your fork +git clone https://github.com//sglang.git +cd sglang/docs_new # 2. Create a branch git checkout -b my-changes @@ -86,15 +92,49 @@ git push origin my-changes ### Adding a new cookbook recipe -1. Create a new `.mdx` file under `cookbook/` following the existing directory structure (e.g., `cookbook/llm//.mdx` or `cookbook/vlm//.mdx`) -2. Use an existing recipe like `cookbook/llm/Qwen/Qwen3.5.mdx` as a template -3. Add your page to the navigation in `docs.json` -4. Each recipe should include: - - Model introduction and key specs - - Installation / environment setup - - Deployment configuration (with hardware recommendations) - - Usage examples (basic + advanced) - - Benchmarks (if available) +The autoregressive cookbook is **config-driven**: two shared engines — +`src/snippets/_deployment.jsx` (the deploy matrix) and `src/snippets/_playground.jsx` +(the override playground) — contain **no** model-specific code. Adding a model means adding +*data*: a per-model config (plus optional benchmarks) that both engines consume, and an +`.mdx` page that imports them. Copy [`DeepSeek-V4`](cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx) +as the reference instance. + +**Recommended — use the Claude Code skill `/cookbook-add-model`.** It walks the whole flow +interactively: collect the model card + verified `sglang serve` recipes → instantiate the +template → wire up the nav/card → validate → fill in measured benchmarks. Related skills: +`/cookbook-migrate-model` (port an existing legacy-template page) and `/cookbook-review-pr` +(review a cookbook PR against the checklist). + +**The files it creates / edits — using DeepSeek-V4 as the example:** + +| File | Purpose | +|---|---| +| `src/snippets/configs/deepseek-ai/deepseek-v4.jsx` | Per-model config: `supportedHardware`, `variants`, `quantizations`, `strategies`, the `cells[]` deploy matrix (verified env + flags per `hw × variant × quant × strategy × nodes`), and `playgroundFeatures`. | +| `src/snippets/configs/deepseek-ai/deepseek-v4-benchmarks.jsx` | One entry per cell with measured speed/accuracy + the `sglang_version` it ran on. Optional — skip until you have numbers. | +| `cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx` | The page: imports the engines + config, renders `` / ``, and adds prose (intro + specs + license, config tips, advanced usage). | +| `docs.json` | Nav entry under Cookbook → category → vendor. | +| `cookbook/autoregressive/intro.mdx` | Vendor `` on the category homepage. | + +Note the two folder conventions: under `configs/` the folder is the **HuggingFace org** +(`deepseek-ai`); under `cookbook/` it's the **display vendor** (`DeepSeek`). The page wires +everything together with data only — no engine edits: + +```mdx +import { Deployment } from "/src/snippets/_deployment.jsx"; +import { Playground } from "/src/snippets/_playground.jsx"; +import { config } from "/src/snippets/configs/deepseek-ai/deepseek-v4.jsx"; +import { benchmarks } from "/src/snippets/configs/deepseek-ai/deepseek-v4-benchmarks.jsx"; + + + +``` + +Keep `tag: NEW` on the new page and strip it from same-vendor siblings (≤1 per vendor), then +validate from `docs_new/`: `mint validate`, `mint broken-links`, and `mint dev` for a visual +smoke test. + +> Diffusion / omni / specbundle pages follow their own category structure — don't force the +> autoregressive config-driven template on them. ### Writing guidelines @@ -107,7 +147,7 @@ git push origin my-changes ## Acknowledgements -Thank you to all the authors who contributed to the original documentation in [`sglang/docs/`](https://github.com/sgl-project/sglang/tree/main/docs) and the original cookbook in [`sgl-cookbook`](https://github.com/sgl-project/sgl-cookbook). The migration to the new Mintlify-based documentation was led by the following [ACM-VIT](https://github.com/ACM-VIT) students: +Thank you to all the authors who contributed to the original documentation in `sglang/docs/` and the original cookbook in [`sgl-cookbook`](https://github.com/sgl-project/sgl-cookbook). The migration to the new Mintlify-based documentation was led by the following [ACM-VIT](https://github.com/ACM-VIT) students: [@Adhyan Jain](https://github.com/Adhyan-Jain), [@Maitri-shah29](https://github.com/Maitri-shah29), [@architnigam](https://github.com/architnigam), [@Nakul-Sinha](https://github.com/Nakul-Sinha), [@divyamagrawal06](https://github.com/divyamagrawal06), [@A-Taman](https://github.com/A-Taman), [@nimeshas](https://github.com/nimeshas), [@IshhanKheria](https://github.com/IshhanKheria), [@Krishang-Zinzuwadia](https://github.com/Krishang-Zinzuwadia), [@pokymono](https://github.com/pokymono), [@Ishitajoshii](https://github.com/Ishitajoshii), [@AdityaVKochar](https://github.com/AdityaVKochar) diff --git a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V3.mdx b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V3.mdx index 9c1a32569..550620fdf 100644 --- a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V3.mdx +++ b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V3.mdx @@ -377,7 +377,7 @@ print(final_response.choices[0].message.content) #### 4.2.3 Multi-Token Prediction (EAGLE Speculative Decoding) -SGLang implements DeepSeek V3 Multi-Token Prediction (MTP) based on [EAGLE speculative decoding](../../../docs/advanced_features/speculative_decoding#EAGLE-Decoding). With this optimization, decoding speed improves by up to **1.8×** at batch size 1 and **1.5×** at batch size 32 on H200 TP8. +SGLang implements DeepSeek V3 Multi-Token Prediction (MTP) based on [EAGLE speculative decoding](../../../docs/advanced_features/speculative_decoding#eagle-decoding). With this optimization, decoding speed improves by up to **1.8×** at batch size 1 and **1.5×** at batch size 32 on H200 TP8. **Enable with:** @@ -424,7 +424,7 @@ For multi-node serving and hardware-specific examples: - [8× A100 AWQ](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-8-a100a800-with-awq-quantization) - [16× A100 INT8](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-16-a100a800-with-int8-quantization) - [32× L40S INT8](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-32-l40s-with-int8-quantization) -- [Xeon 6980P CPU](../../../docs/hardware-platforms/cpu_server#example-running-deepseek-r1) +- [Xeon 6980P CPU](../../../docs/hardware-platforms/cpu_server#example-running-deepseek-v3-1-terminus) - [4× Atlas 800I A3 (int8)](../../../docs/hardware-platforms/ascend-npus/model-tutorials/deepseek_r1#multi-node-pd-disaggregation-deployment) **Blog references for large-scale deployment:** diff --git a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2.mdx b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2.mdx index 6db67ff32..cba3154e7 100644 --- a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2.mdx +++ b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2.mdx @@ -348,7 +348,7 @@ print(final_response.choices[0].message.content) #### 4.2.3 Multi-Token Prediction (EAGLE Speculative Decoding) -SGLang implements Multi-Token Prediction (MTP) for DeepSeek V3.2 based on [EAGLE speculative decoding](../../../docs/advanced_features/speculative_decoding#EAGLE-Decoding). This optimization significantly improves decoding speed for small batch sizes. +SGLang implements Multi-Token Prediction (MTP) for DeepSeek V3.2 based on [EAGLE speculative decoding](../../../docs/advanced_features/speculative_decoding#eagle-decoding). This optimization significantly improves decoding speed for small batch sizes. **With DP Attention:** diff --git a/docs_new/cookbook/autoregressive/Google/DiffusionGemma.mdx b/docs_new/cookbook/autoregressive/Google/DiffusionGemma.mdx index da8794ddc..2674f2268 100644 --- a/docs_new/cookbook/autoregressive/Google/DiffusionGemma.mdx +++ b/docs_new/cookbook/autoregressive/Google/DiffusionGemma.mdx @@ -142,7 +142,7 @@ stopping_config: ### 4.1 Deployment -Start the server with the command from [Section 3.1](#31-basic-configuration). +Start the server with the command from [Section 3.1](#3-1-basic-configuration). ### 4.2 Basic Usage diff --git a/docs_new/cookbook/autoregressive/Google/Gemma4.mdx b/docs_new/cookbook/autoregressive/Google/Gemma4.mdx index 4d3b01793..78e1218dd 100644 --- a/docs_new/cookbook/autoregressive/Google/Gemma4.mdx +++ b/docs_new/cookbook/autoregressive/Google/Gemma4.mdx @@ -172,7 +172,7 @@ SGLANG_USE_AITER=0 sglang serve --model-path google/gemma-4-E4B-it \ For `gemma-4-31B-it` and `gemma-4-26B-A4B-it`, the same commands above work on MI300X, MI325X, and MI355X without additional command-line changes. -> **Status**: AMD benchmarks are available in [Section 5.1](#51-speed-benchmark). +> **Status**: AMD benchmarks are available in [Section 5.1](#5-1-speed-benchmark). ## 4. Model Invocation @@ -187,7 +187,7 @@ sglang serve --model-path google/gemma-4-26B-A4B-it \ #### Speculative Decoding (MTP) Server Commands -Each Gemma 4 variant ships with a paired `*-assistant` draft model for NEXTN multi-token prediction. Use the commands below to enable MTP for the corresponding target model. These match the configuration generated when you toggle **Speculative Decoding (MTP) → Enabled** in the [interactive selector](#31-basic-configuration). +Each Gemma 4 variant ships with a paired `*-assistant` draft model for NEXTN multi-token prediction. Use the commands below to enable MTP for the corresponding target model. These match the configuration generated when you toggle **Speculative Decoding (MTP) → Enabled** in the [interactive selector](#3-1-basic-configuration). ```bash Command # Gemma 4 E2B + MTP diff --git a/docs_new/cookbook/autoregressive/LiquidAI/LFM2.5.mdx b/docs_new/cookbook/autoregressive/LiquidAI/LFM2.5.mdx index bedae1ffb..e0a5d5301 100644 --- a/docs_new/cookbook/autoregressive/LiquidAI/LFM2.5.mdx +++ b/docs_new/cookbook/autoregressive/LiquidAI/LFM2.5.mdx @@ -195,7 +195,7 @@ This minimal hybrid layout was selected by a hardware-in-the-loop architecture s -The Deploy panel above covers the eight serving variants; **LFM2.5-1.2B-JP** (original — launch without `--tool-call-parser`) and the **Base** repos (pre-trained only, no post-training — see [§3.5](#35-base-checkpoints)) launch the same way with the model path swapped. +The Deploy panel above covers the eight serving variants; **LFM2.5-1.2B-JP** (original — launch without `--tool-call-parser`) and the **Base** repos (pre-trained only, no post-training — see [§3.5](#3-5-base-checkpoints)) launch the same way with the model path swapped. **Choosing a variant:** @@ -212,7 +212,7 @@ The Deploy panel above covers the eight serving variants; **LFM2.5-1.2B-JP** (or ## 2. Configuration Tips - **Reasoning parser**: LFM2.5 reasoning models wrap their chain-of-thought in `...` tags. The command generator passes `--reasoning-parser qwen3` for **8B-A1B** (it emits an explicit opening ``) and `--reasoning-parser qwen3-thinking` for **1.2B-Thinking** (always-on reasoning). This splits the thinking process into `reasoning_content`; without it the chain-of-thought stays inline in `content`. -- **Tool calling**: `--tool-call-parser lfm2` surfaces LFM2.5's Pythonic `<|tool_call_start|>[...]<|tool_call_end|>` calls as standard `message.tool_calls`. The original **1.2B-JP** does not expose tool calling; **Base** has no post-training (see [§3.5](#35-base-checkpoints)). +- **Tool calling**: `--tool-call-parser lfm2` surfaces LFM2.5's Pythonic `<|tool_call_start|>[...]<|tool_call_end|>` calls as standard `message.tool_calls`. The original **1.2B-JP** does not expose tool calling; **Base** has no post-training (see [§3.5](#3-5-base-checkpoints)). - **Attention backend on Blackwell (B200/sm100)**: SGLang defaults to the `trtllm_mha` backend on sm100, which is fastest for the dense text models. The **8B-A1B** uses a mamba-style state cache that runs on a page-size-1 backend, so the generator picks `--attention-backend flashinfer` for it. The **VL** language model also uses that state cache and offers two backends: `--attention-backend flashinfer` (keeps prefix/radix caching — what the generator emits), or `--attention-backend trtllm_mha --disable-radix-cache` to run the language model on Blackwell `trtllm_mha` attention (`--disable-radix-cache` lifts the page-size-1 requirement, at the cost of prefix caching). Pair either with `--mm-attention-backend fa4` for the vision tower. - **VL vision tower (`--mm-attention-backend`)**: on sm100 the `trtllm_mha` default is fastest for text but applies *causal* attention to image tokens. For the VL model, pass `--mm-attention-backend fa4` on B200/B300 (or `fa3` on H100/H200) to restore bidirectional image-token attention and full vision quality. - **VL multimodal feature transport**: the generator launches the VL models with `SGLANG_USE_CUDA_IPC_TRANSPORT=1 SGLANG_USE_IPC_POOL_HANDLE_CACHE=1`. The first moves the processor→scheduler image-feature handoff onto CUDA IPC instead of serializing tensors between processes; the second ships the pool handle so the scheduler opens it once and caches it, instead of opening a per-item handle on every request. On the image serving workload (1 image @ 720p, measured on VL-1.6B on H100 and B200) this pair is worth roughly 30–50% higher image throughput and 30–40% lower image TTFT vs running without them (measured on VL-1.6B, H100 and B200); decode speed (TPOT) is unaffected. diff --git a/docs_new/cookbook/autoregressive/Llama/Llama3.1.mdx b/docs_new/cookbook/autoregressive/Llama/Llama3.1.mdx index ac7679cf6..52c143506 100644 --- a/docs_new/cookbook/autoregressive/Llama/Llama3.1.mdx +++ b/docs_new/cookbook/autoregressive/Llama/Llama3.1.mdx @@ -230,7 +230,7 @@ for index, tool_call in sorted(tool_calls_accumulator.items()): print() ``` -Reference: [SGLang Tool Parser Documentation](../../../docs/advanced_features/tool_parser#OpenAI-Compatible-API) +Reference: [SGLang Tool Parser Documentation](../../../docs/advanced_features/tool_parser#openai-compatible-api) **Output Example** diff --git a/docs_new/cookbook/autoregressive/NVIDIA/Nemotron3-Nano-Omni.mdx b/docs_new/cookbook/autoregressive/NVIDIA/Nemotron3-Nano-Omni.mdx index bfa029df9..fce05d5cf 100644 --- a/docs_new/cookbook/autoregressive/NVIDIA/Nemotron3-Nano-Omni.mdx +++ b/docs_new/cookbook/autoregressive/NVIDIA/Nemotron3-Nano-Omni.mdx @@ -89,7 +89,7 @@ This section provides a progressive guide from quick deployment to performance t ## 4. Model Invocation -The command below launches the server for a 4×H100 setup with reasoning and tool calling enabled. See [Section 4.8](#48-fp8-and-nvfp4-deployment) for FP8 and NVFP4 variants. +The command below launches the server for a 4×H100 setup with reasoning and tool calling enabled. See [Section 4.8](#4-8-fp8-and-nvfp4-deployment) for FP8 and NVFP4 variants. ```shell Command sglang serve \ diff --git a/docs_new/cookbook/autoregressive/Qwen/Qwen3-Coder.mdx b/docs_new/cookbook/autoregressive/Qwen/Qwen3-Coder.mdx index 862ae951f..2c142bc3b 100644 --- a/docs_new/cookbook/autoregressive/Qwen/Qwen3-Coder.mdx +++ b/docs_new/cookbook/autoregressive/Qwen/Qwen3-Coder.mdx @@ -769,7 +769,7 @@ python3 -m sglang.test.few_shot_gsm8k --num-questions 200 ##### NVIDIA (B200/GB200) -For deployment commands, see [Section 3.1](#31-configuration). +For deployment commands, see [Section 3.1](#3-1-configuration). - Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8 ``` diff --git a/docs_new/cookbook/autoregressive/Qwen/Qwen3-Next.mdx b/docs_new/cookbook/autoregressive/Qwen/Qwen3-Next.mdx index 8f74a30a8..be285a346 100644 --- a/docs_new/cookbook/autoregressive/Qwen/Qwen3-Next.mdx +++ b/docs_new/cookbook/autoregressive/Qwen/Qwen3-Next.mdx @@ -572,7 +572,7 @@ SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1 python -m sglang.launch_server --mod #### 4.2.4 Multi-Token Prediction (NEXTN Speculative Decoding) -Qwen3-Next ships built-in Multi-Token Prediction (MTP) layers and supports [EAGLE-style speculative decoding](../../../docs/advanced_features/speculative_decoding#EAGLE-Decoding) through the `NEXTN` algorithm. The MTP weights are bundled in the main checkpoint, so no separate draft model is required. +Qwen3-Next ships built-in Multi-Token Prediction (MTP) layers and supports [EAGLE-style speculative decoding](../../../docs/advanced_features/speculative_decoding#eagle-decoding) through the `NEXTN` algorithm. The MTP weights are bundled in the main checkpoint, so no separate draft model is required. ```shell Command python3 -m sglang.launch_server \ diff --git a/docs_new/cookbook/autoregressive/Xiaomi/MiMo-V2.5.mdx b/docs_new/cookbook/autoregressive/Xiaomi/MiMo-V2.5.mdx index 8a1d67b34..18436e947 100644 --- a/docs_new/cookbook/autoregressive/Xiaomi/MiMo-V2.5.mdx +++ b/docs_new/cookbook/autoregressive/Xiaomi/MiMo-V2.5.mdx @@ -65,7 +65,7 @@ Refer to the [official SGLang installation guide](../../../docs/get-started/inst > Pull the image matching your GPU's CUDA driver. `lmsysorg/sglang:latest` will not load either checkpoint. -**TPU (sgl-jax):** MiMo-V2.5-Pro can also be served on TPU via the JAX-based [sgl-jax](https://github.com/sgl-project/sglang-jax) runtime. The container image and `pip install` steps are listed in [§3.3 TPU Deployment](#33-tpu-deployment-mimo-v25-pro-sgl-jax). +**TPU (sgl-jax):** MiMo-V2.5-Pro can also be served on TPU via the JAX-based [sgl-jax](https://github.com/sgl-project/sglang-jax) runtime. The container image and `pip install` steps are listed in [§3.3 TPU Deployment](#3-3-tpu-deployment-mimo-v2-5-pro-sgl-jax). ## 3. Model Deployment diff --git a/docs_new/cookbook/intro copy.mdx b/docs_new/cookbook/intro copy.mdx deleted file mode 100644 index c11ccbee6..000000000 --- a/docs_new/cookbook/intro copy.mdx +++ /dev/null @@ -1,228 +0,0 @@ ---- -title: SGLang Cookbook -metatags: - description: The SGLang Cookbook is a practical collection of examples and guides that show developers how to efficiently run SGLang with a variety of models on different platforms. ---- - -[![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) -[![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg)](https://github.com/sgl-project/sgl-cookbook/pulls) - -A community-maintained repository of practical guides and recipes for deploying and using SGLang in production environments. Our mission is simple: answer the question **"How do I use SGLang (and related models) on hardware Y for task Z?"** with clear, actionable solutions. - -## 🎯 What You'll Find Here - -This cookbook aggregates battle-tested SGLang recipes covering: - -- **Models**: Mainstream LLMs and Vision-Language Models (VLMs) -- **Use Cases**: Inference serving, deployment strategies, multimodal applications -- **Hardware**: GPU and CPU configurations, optimization for different accelerators -- **Best Practices**: Configuration templates, performance tuning, troubleshooting guides - -Each recipe provides step-by-step instructions to help you quickly implement SGLang solutions for your specific requirements. - -## Guides - -### Autoregressive Models - -#### Qwen - -- [x] [Qwen3.5](./autoregressive/Qwen/Qwen3.5) NEW -- [x] [Qwen3](./autoregressive/Qwen/Qwen3) -- [x] [Qwen3-Next](./autoregressive/Qwen/Qwen3-Next) -- [x] [Qwen3-VL](./autoregressive/Qwen/Qwen3-VL) -- [x] [Qwen3-Coder](./autoregressive/Qwen/Qwen3-Coder) -- [x] [Qwen3-Coder-Next](./autoregressive/Qwen/Qwen3-Coder-Next) NEW -- [x] [Qwen2.5-VL](./autoregressive/Qwen/Qwen2.5-VL) - -#### DeepSeek - -- [x] [DeepSeek-V3.2](./autoregressive/DeepSeek/DeepSeek-V3_2) -- [x] [DeepSeek-V3.1](./autoregressive/DeepSeek/DeepSeek-V3_1) -- [x] [DeepSeek-V3](./autoregressive/DeepSeek/DeepSeek-V3) -- [x] [DeepSeek-R1](./autoregressive/DeepSeek/DeepSeek-R1) -- [x] [DeepSeek-OCR](./autoregressive/DeepSeek/DeepSeek-OCR) -- [x] [DeepSeek-OCR-2](./autoregressive/DeepSeek/DeepSeek-OCR-2) NEW - -#### Llama - -- [ ] [Llama4-Scout](./autoregressive/Llama/Llama4) -- [x] [Llama3.3-70B](./autoregressive/Llama/Llama3.3-70B) -- [x] [Llama3.1](./autoregressive/Llama/Llama3.1) - -#### GLM - -- [ ] [GLM-Glyph](./autoregressive/GLM/GLM-Glyph) -- [x] [GLM-5](./autoregressive/GLM/GLM-5) NEW -- [x] [GLM-OCR](./autoregressive/GLM/GLM-OCR) NEW -- [x] [GLM-4.5](./autoregressive/GLM/GLM-4.5) -- [x] [GLM-4.5V](./autoregressive/GLM/GLM-4.5V) -- [x] [GLM-4.6](./autoregressive/GLM/GLM-4.6) -- [x] [GLM-4.6V](./autoregressive/GLM/GLM-4.6V) -- [x] [GLM-4.7](./autoregressive/GLM/GLM-4.7) -- [x] [GLM-4.7-Flash](./autoregressive/GLM/GLM-4.7-Flash) NEW - -#### OpenAI - -- [x] [gpt-oss](./autoregressive/OpenAI/GPT-OSS) - -#### Moonshotai - -- [x] [Kimi-K2.7-Code](./autoregressive/Moonshotai/Kimi-K2.7-Code) NEW -- [x] [Kimi-K2.6](./autoregressive/Moonshotai/Kimi-K2.6) -- [x] [Kimi-K2.5](./autoregressive/Moonshotai/Kimi-K2.5) -- [x] [Kimi-K2](./autoregressive/Moonshotai/Kimi-K2) -- [x] [Kimi-Linear](./autoregressive/Moonshotai/Kimi-Linear) - -#### MiniMax - -- [ ] [MiniMax-M2](./autoregressive/MiniMax/MiniMax-M2) -- [x] [MiniMax-M2.5](./autoregressive/MiniMax/MiniMax-M2.5) NEW - -#### NVIDIA - -- [x] [Nemotron 3 Nano Omni](./autoregressive/NVIDIA/Nemotron3-Nano-Omni) -- [x] [Nemotron-Nano-3-30B-A3B](./autoregressive/NVIDIA/Nemotron3-Nano) -- [x] [Nemotron3-Super](./autoregressive/NVIDIA/Nemotron3-Super) -- [x] [Nemotron3-Ultra](./autoregressive/NVIDIA/Nemotron3-Ultra) NEW - -#### Ernie - -- [x] [Ernie4.5](./autoregressive/Ernie/Ernie4.5) -- [ ] [Ernie4.5-VL](./autoregressive/Ernie/Ernie4.5-VL) - -#### InternVL - -- [ ] [InternVL3.5](./autoregressive/InternVL/InternVL3.5) - -#### InternLM - -- [ ] [Intern-S1](./autoregressive/InternLM/Intern-S1) - -#### Jina AI - -- [ ] [Jina-reranker-m0](./autoregressive/Jina/Jina-reranker-m0) - -#### Mistral - -- [ ] [Mistral-3](./autoregressive/Mistral/Ministral-3) -- [x] [Devstral 2](./autoregressive/Mistral/Devstral-2) - -#### Xiaomi - -- [x] [MiMo-V2-Flash](./autoregressive/Xiaomi/MiMo-V2-Flash) - -#### FlashLabs - -- [x] [Chroma 1.0](./autoregressive/FlashLabs/Chroma1.0)NEW - -#### StepFun - -- [x] [Step-3.5-Flash](./autoregressive/StepFun/Step3.5) NEW -- [x] [Step3-VL-10B](./autoregressive/StepFun/Step3-VL-10B) NEW - -#### InclusionAI - -- [x] [Ling-2.5-1T](./autoregressive/InclusionAI/Ling-2.5-1T) NEW -- [x] [Ring-2.5-1T](./autoregressive/InclusionAI/Ring-2.5-1T) NEW -- [x] [LLaDA-2.1](./autoregressive/InclusionAI/LLaDA-2.1) NEW - -### Diffusion Models - -#### FLUX - -- [x] [FLUX](./diffusion/FLUX/FLUX) - -#### Qwen-Image - -- [ ] [Qwen-Image](./diffusion/Qwen-Image/Qwen-Image) -- [x] [Qwen-Image-Edit](./diffusion/Qwen-Image/Qwen-Image-Edit) - -#### Wan - -- [ ] [Wan2.1](./diffusion/Wan/Wan2.1) -- [x] [Wan2.2](./diffusion/Wan/Wan2.2) - -#### Z-Image - -- [x] [Z-Image-Turbo](./diffusion/Z-Image/Z-Image-Turbo) - -### Benchmarks - -- [x] [Diffusion Model Benchmark](./base/benchmarks/diffusion_model_benchmark.mdx) -- [x] [LLM Benchmark](./base/benchmarks/autoregressive_model_benchmark.mdx) - -## Reference - -- [Installation (PyPI)](../docs/get-started/install) - Install SGLang via pip or uv (stable and nightly) -- [Server arguments](./base/reference/server_arguments) - Understanding all the arguments - -## 🚀 Quick Start - -1. Browse the recipe index above to find your model -2. Follow the step-by-step instructions in each guide -3. Adapt configurations to your specific hardware and requirements -4. Join our community to share feedback and improvements - -## 🤝 Contributing - -We believe the best documentation comes from practitioners. Whether you've optimized SGLang for a specific model, solved a tricky deployment challenge, or discovered performance improvements, we encourage you to contribute your recipes! - -**Ways to contribute:** - -- Add a new recipe for a model not yet covered -- Improve existing recipes with additional tips or configurations -- Report issues or suggest enhancements -- Share your production deployment experiences - -**To contribute:** - - -```bash Contribute a Recipe -# Fork the repo and clone locally -git clone https://github.com/YOUR_USERNAME/sglang-cookbook.git -cd sglang-cookbook - -# Create a new branch -git checkout -b add-my-recipe - -# Add your recipe following the template in DeepSeek-V3.2 -# Submit a PR! -``` - - -## 🛠️ Local Development - -### Prerequisites - -- Node.js >= 20.0 -- npm or yarn - -### Setup and Run - -Install dependencies and start the development server: - - -```bash Local Development -# Install dependencies -npm install - -# Start development server (hot reload enabled) -npm start -``` - - -The site will automatically open in your browser at `http://localhost:3000`. - -## 📖 Resources - -- [SGLang GitHub](https://github.com/sgl-project/sglang) -- [SGLang Documentation](https://sgl-project.github.io) -- [Community Slack/Discord](https://discord.gg/MpEEuAeb) - -## 📄 License - -This project is licensed under the Apache License 2.0 - see the [LICENSE](https://github.com/sgl-project/sgl-cookbook/blob/main/LICENSE) file for details. - ---- - -**Let's build this resource together!** 🚀 Star the repo and contribute your recipes to help the SGLang community grow. diff --git a/docs_new/docs/advanced_features/hicache.rst b/docs_new/docs/advanced_features/hicache.rst deleted file mode 100644 index e7d83211d..000000000 --- a/docs_new/docs/advanced_features/hicache.rst +++ /dev/null @@ -1,9 +0,0 @@ -Hierarchical KV Caching (HiCache) -================================= - -.. toctree:: - :maxdepth: 1 - - hicache_best_practices.md - hicache_design.md - hicache_storage_runtime_attach_detach.md diff --git a/docs_new/docs/advanced_features/lora.ipynb b/docs_new/docs/advanced_features/lora.ipynb deleted file mode 100644 index 310c7eba1..000000000 --- a/docs_new/docs/advanced_features/lora.ipynb +++ /dev/null @@ -1,714 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# LoRA Serving" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "SGLang enables the use of [LoRA adapters](https://arxiv.org/abs/2106.09685) with a base model. By incorporating techniques from [S-LoRA](https://arxiv.org/pdf/2311.03285) and [Punica](https://arxiv.org/pdf/2310.18547), SGLang can efficiently support multiple LoRA adapters for different sequences within a single batch of inputs." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Arguments for LoRA Serving" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The following server arguments are relevant for multi-LoRA serving:\n", - "\n", - "* `enable_lora`: Enable LoRA support for the model. This argument is automatically set to True if `--lora-paths` is provided for backward compatibility.\n", - "\n", - "* `enable_lora_overlap_loading`: Enable asynchronous LoRA weight loading in order to overlap H2D transfers with GPU compute. This should be enabled if you find that your LoRA workloads are bottlenecked by adapter weight loading, for example when frequently loading large LoRA adapters.\n", - "\n", - "* `lora_paths`: The list of LoRA adapters to load. Each adapter must be specified in one of the following formats: | = | JSON with schema {\"lora_name\":str,\"lora_path\":str,\"pinned\":bool}.\n", - "\n", - "* `max_loras_per_batch`: Maximum number of adaptors used by each batch. This argument can affect the amount of GPU memory reserved for multi-LoRA serving, so it should be set to a smaller value when memory is scarce. Defaults to be 8.\n", - "\n", - "* `max_loaded_loras`: If specified, it limits the maximum number of LoRA adapters loaded in CPU memory at a time. The value must be greater than or equal to `max-loras-per-batch`.\n", - "\n", - "* `lora_eviction_policy`: LoRA adapter eviction policy when GPU memory pool is full. `lru`: Least Recently Used (default, better cache efficiency). `fifo`: First-In-First-Out.\n", - "\n", - "* `lora_backend`: The backend of running GEMM kernels for Lora modules. Currently we support Triton LoRA backend (`triton`) and Chunked SGMV backend (`csgmv`). In the future, faster backend built upon Cutlass or Cuda kernels will be added.\n", - "\n", - "* `max_lora_rank`: The maximum LoRA rank that should be supported. If not specified, it will be automatically inferred from the adapters provided in `--lora-paths`. This argument is needed when you expect to dynamically load adapters of larger LoRA rank after server startup.\n", - "\n", - "* `lora_target_modules`: The union set of all target modules where LoRA should be applied (e.g., `q_proj`, `k_proj`, `gate_proj`). If not specified, it will be automatically inferred from the adapters provided in `--lora-paths`. This argument is needed when you expect to dynamically load adapters of different target modules after server startup. You can also set it to `all` to enable LoRA for all supported modules. However, enabling LoRA on additional modules introduces a minor performance overhead. If your application is performance-sensitive, we recommend only specifying the modules for which you plan to load adapters.\n", - "\n", - "* `--max-lora-chunk-size`: Maximum chunk size for the ChunkedSGMV LoRA backend. Only used when --lora-backend is 'csgmv'. Choosing a larger value might improve performance. Please tune this value based on your hardware and workload as needed. Defaults to 16.\n", - "\n", - "* `tp_size`: LoRA serving along with Tensor Parallelism is supported by SGLang. `tp_size` controls the number of GPUs for tensor parallelism. More details on the tensor sharding strategy can be found in [S-Lora](https://arxiv.org/pdf/2311.03285) paper.\n", - "\n", - "From client side, the user needs to provide a list of strings as input batch, and a list of adaptor names that each input sequence corresponds to." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Usage\n", - "\n", - "### Serving Single Adaptor" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**Note:** SGLang supports LoRA adapters through two APIs:\n", - "\n", - "1. **OpenAI-Compatible API** (`/v1/chat/completions`, `/v1/completions`): Use the `model:adapter-name` syntax. See [OpenAI API with LoRA](../basic_usage/openai_api_completions.ipynb#Using-LoRA-Adapters) for examples.\n", - "\n", - "2. **Native API** (`/generate`): Pass `lora_path` in the request body (shown below)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import json\n", - "import requests\n", - "\n", - "from sglang.test.doc_patch import launch_server_cmd\n", - "from sglang.utils import wait_for_server, terminate_process" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "server_process, port = launch_server_cmd(\n", - " # Here we set max-loras-per-batch to 2: one slot for adaptor and another one for base model\n", - " \"\"\"\n", - "python3 -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \\\n", - " --enable-lora \\\n", - " --lora-paths lora0=algoprog/fact-generation-llama-3.1-8b-instruct-lora \\\n", - " --max-loras-per-batch 2 \\\n", - " --log-level warning \\\n", - "\"\"\"\n", - ")\n", - "\n", - "wait_for_server(f\"http://localhost:{port}\", process=server_process)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "url = f\"http://127.0.0.1:{port}\"\n", - "json_data = {\n", - " \"text\": [\n", - " \"List 3 countries and their capitals.\",\n", - " \"List 3 countries and their capitals.\",\n", - " ],\n", - " \"sampling_params\": {\"max_new_tokens\": 32, \"temperature\": 0},\n", - " # The first input uses lora0, and the second input uses the base model\n", - " \"lora_path\": [\"lora0\", None],\n", - "}\n", - "response = requests.post(\n", - " url + \"/generate\",\n", - " json=json_data,\n", - ")\n", - "print(f\"Output 0: {response.json()[0]['text']}\")\n", - "print(f\"Output 1: {response.json()[1]['text']}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "terminate_process(server_process)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Serving Multiple Adaptors" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "server_process, port = launch_server_cmd(\"\"\"\n", - "python3 -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \\\n", - " --enable-lora \\\n", - " --lora-paths lora0=algoprog/fact-generation-llama-3.1-8b-instruct-lora \\\n", - " lora1=Nutanix/Meta-Llama-3.1-8B-Instruct_SFT_lora_4_alpha_16_humaneval_raw_json \\\n", - " --max-loras-per-batch 2 \\\n", - " --log-level warning \\\n", - "\"\"\")\n", - "\n", - "wait_for_server(f\"http://localhost:{port}\", process=server_process)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "url = f\"http://127.0.0.1:{port}\"\n", - "json_data = {\n", - " \"text\": [\n", - " \"List 3 countries and their capitals.\",\n", - " \"List 3 countries and their capitals.\",\n", - " ],\n", - " \"sampling_params\": {\"max_new_tokens\": 32, \"temperature\": 0},\n", - " # The first input uses lora0, and the second input uses lora1\n", - " \"lora_path\": [\"lora0\", \"lora1\"],\n", - "}\n", - "response = requests.post(\n", - " url + \"/generate\",\n", - " json=json_data,\n", - ")\n", - "print(f\"Output 0: {response.json()[0]['text']}\")\n", - "print(f\"Output 1: {response.json()[1]['text']}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "terminate_process(server_process)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Dynamic LoRA loading" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Instead of specifying all adapters during server startup via `--lora-paths`. You can also load & unload LoRA adapters dynamically via the `/load_lora_adapter` and `/unload_lora_adapter` API.\n", - "\n", - "When using dynamic LoRA loading, it's recommended to explicitly specify both `--max-lora-rank` and `--lora-target-modules` at startup. For backward compatibility, SGLang will infer these values from `--lora-paths` if they are not explicitly provided. However, in that case, you would have to ensure that all dynamically loaded adapters share the same shape (rank and target modules) as those in the initial `--lora-paths` or are strictly \"smaller\"." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "lora0 = \"Nutanix/Meta-Llama-3.1-8B-Instruct_SFT_lora_4_alpha_16_humaneval_raw_json\" # rank - 4, target modules - q_proj, k_proj, v_proj, o_proj, gate_proj\n", - "lora1 = \"algoprog/fact-generation-llama-3.1-8b-instruct-lora\" # rank - 64, target modules - q_proj, k_proj, v_proj, o_proj, gate_proj, up_proj, down_proj\n", - "lora0_new = \"philschmid/code-llama-3-1-8b-text-to-sql-lora\" # rank - 256, target modules - q_proj, k_proj, v_proj, o_proj, gate_proj, up_proj, down_proj\n", - "\n", - "\n", - "# The `--target-lora-modules` param below is technically not needed, as the server will infer it from lora0 which already has all the target modules specified.\n", - "# We are adding it here just to demonstrate usage.\n", - "server_process, port = launch_server_cmd(\"\"\"\n", - " python3 -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \\\n", - " --enable-lora \\\n", - " --cuda-graph-max-bs-decode 2 \\\n", - " --max-loras-per-batch 2 \\\n", - " --max-lora-rank 256\n", - " --lora-target-modules all\n", - " --log-level warning\n", - " \"\"\")\n", - "\n", - "url = f\"http://127.0.0.1:{port}\"\n", - "wait_for_server(url, process=server_process)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Load adapter lora0" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "response = requests.post(\n", - " url + \"/load_lora_adapter\",\n", - " json={\n", - " \"lora_name\": \"lora0\",\n", - " \"lora_path\": lora0,\n", - " },\n", - ")\n", - "\n", - "if response.status_code == 200:\n", - " print(\"LoRA adapter loaded successfully.\", response.json())\n", - "else:\n", - " print(\"Failed to load LoRA adapter.\", response.json())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Load adapter lora1:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "response = requests.post(\n", - " url + \"/load_lora_adapter\",\n", - " json={\n", - " \"lora_name\": \"lora1\",\n", - " \"lora_path\": lora1,\n", - " },\n", - ")\n", - "\n", - "if response.status_code == 200:\n", - " print(\"LoRA adapter loaded successfully.\", response.json())\n", - "else:\n", - " print(\"Failed to load LoRA adapter.\", response.json())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Check inference output:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "url = f\"http://127.0.0.1:{port}\"\n", - "json_data = {\n", - " \"text\": [\n", - " \"List 3 countries and their capitals.\",\n", - " \"List 3 countries and their capitals.\",\n", - " ],\n", - " \"sampling_params\": {\"max_new_tokens\": 32, \"temperature\": 0},\n", - " # The first input uses lora0, and the second input uses lora1\n", - " \"lora_path\": [\"lora0\", \"lora1\"],\n", - "}\n", - "response = requests.post(\n", - " url + \"/generate\",\n", - " json=json_data,\n", - ")\n", - "print(f\"Output from lora0: \\n{response.json()[0]['text']}\\n\")\n", - "print(f\"Output from lora1 (updated): \\n{response.json()[1]['text']}\\n\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Unload lora0 and replace it with a different adapter:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "response = requests.post(\n", - " url + \"/unload_lora_adapter\",\n", - " json={\n", - " \"lora_name\": \"lora0\",\n", - " },\n", - ")\n", - "\n", - "response = requests.post(\n", - " url + \"/load_lora_adapter\",\n", - " json={\n", - " \"lora_name\": \"lora0\",\n", - " \"lora_path\": lora0_new,\n", - " },\n", - ")\n", - "\n", - "if response.status_code == 200:\n", - " print(\"LoRA adapter loaded successfully.\", response.json())\n", - "else:\n", - " print(\"Failed to load LoRA adapter.\", response.json())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Check output again:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "url = f\"http://127.0.0.1:{port}\"\n", - "json_data = {\n", - " \"text\": [\n", - " \"List 3 countries and their capitals.\",\n", - " \"List 3 countries and their capitals.\",\n", - " ],\n", - " \"sampling_params\": {\"max_new_tokens\": 32, \"temperature\": 0},\n", - " # The first input uses lora0, and the second input uses lora1\n", - " \"lora_path\": [\"lora0\", \"lora1\"],\n", - "}\n", - "response = requests.post(\n", - " url + \"/generate\",\n", - " json=json_data,\n", - ")\n", - "print(f\"Output from lora0: \\n{response.json()[0]['text']}\\n\")\n", - "print(f\"Output from lora1 (updated): \\n{response.json()[1]['text']}\\n\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "terminate_process(server_process)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### OpenAI-compatible API usage\n", - "\n", - "You can use LoRA adapters via the OpenAI-compatible APIs by specifying the adapter in the `model` field using the `base-model:adapter-name` syntax (for example, `qwen/qwen2.5-0.5b-instruct:adapter_a`). For more details and examples, see the “Using LoRA Adapters” section in the OpenAI API documentation: [openai_api_completions.ipynb](../basic_usage/openai_api_completions.ipynb).\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### LoRA GPU Pinning" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Another advanced option is to specify adapters as `pinned` during loading. When an adapter is pinned, it is permanently assigned to one of the available GPU pool slots (as configured by `--max-loras-per-batch`) and will not be evicted from GPU memory during runtime. Instead, it remains resident until it is explicitly unloaded.\n", - "\n", - "This can improve performance in scenarios where the same adapter is frequently used across requests, by avoiding repeated memory transfers and reinitialization overhead. However, since GPU pool slots are limited, pinning adapters reduces the flexibility of the system to dynamically load other adapters on demand. If too many adapters are pinned, it may lead to degraded performance, or in the most extreme case (`Number of pinned adapters == max-loras-per-batch`), halt all unpinned requests. Therefore, currently SGLang limits maximal number of pinned adapters to `max-loras-per-batch - 1` to prevent unexpected starvations. \n", - "\n", - "In the example below, we start a server with `lora1` loaded as pinned, `lora2` and `lora3` loaded as regular (unpinned) adapters. Please note that, we intentionally specify `lora2` and `lora3` in two different formats to demonstrate that both are supported." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "server_process, port = launch_server_cmd(\"\"\"\n", - " python3 -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \\\n", - " --enable-lora \\\n", - " --cuda-graph-max-bs-decode 8 \\\n", - " --max-loras-per-batch 3 \\\n", - " --max-lora-rank 256 \\\n", - " --lora-target-modules all \\\n", - " --lora-paths \\\n", - " {\"lora_name\":\"lora0\",\"lora_path\":\"Nutanix/Meta-Llama-3.1-8B-Instruct_SFT_lora_4_alpha_16_humaneval_raw_json\",\"pinned\":true} \\\n", - " {\"lora_name\":\"lora1\",\"lora_path\":\"algoprog/fact-generation-llama-3.1-8b-instruct-lora\"} \\\n", - " lora2=philschmid/code-llama-3-1-8b-text-to-sql-lora\n", - " --log-level warning\n", - " \"\"\")\n", - "\n", - "\n", - "url = f\"http://127.0.0.1:{port}\"\n", - "wait_for_server(url, process=server_process)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You can also specify adapter as pinned during dynamic adapter loading. In the example below, we reload `lora2` as pinned adapter:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "response = requests.post(\n", - " url + \"/unload_lora_adapter\",\n", - " json={\n", - " \"lora_name\": \"lora1\",\n", - " },\n", - ")\n", - "\n", - "response = requests.post(\n", - " url + \"/load_lora_adapter\",\n", - " json={\n", - " \"lora_name\": \"lora1\",\n", - " \"lora_path\": \"algoprog/fact-generation-llama-3.1-8b-instruct-lora\",\n", - " \"pinned\": True, # Pin the adapter to GPU\n", - " },\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Verify that the results are expected:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "url = f\"http://127.0.0.1:{port}\"\n", - "json_data = {\n", - " \"text\": [\n", - " \"List 3 countries and their capitals.\",\n", - " \"List 3 countries and their capitals.\",\n", - " \"List 3 countries and their capitals.\",\n", - " ],\n", - " \"sampling_params\": {\"max_new_tokens\": 32, \"temperature\": 0},\n", - " # The first input uses lora0, and the second input uses lora1\n", - " \"lora_path\": [\"lora0\", \"lora1\", \"lora2\"],\n", - "}\n", - "response = requests.post(\n", - " url + \"/generate\",\n", - " json=json_data,\n", - ")\n", - "print(f\"Output from lora0 (pinned): \\n{response.json()[0]['text']}\\n\")\n", - "print(f\"Output from lora1 (pinned): \\n{response.json()[1]['text']}\\n\")\n", - "print(f\"Output from lora2 (not pinned): \\n{response.json()[2]['text']}\\n\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "terminate_process(server_process)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Choosing LoRA Backend\n", - "\n", - "SGLang supports two LoRA backends that you can choose from using the `--lora-backend` argument:\n", - "\n", - "- `triton`: Basic Triton-based backend.\n", - "- `csgmv`: Default chunked SGMV backend optimized for high concurrency scenarios.\n", - "\n", - "The `csgmv` backend was recently introduced to improve performance especially at high-concurrency scenarios. Our benchmark shows that it achieves 20% to 80% latency improvements over the basic triton backend." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "server_process, port = launch_server_cmd(\"\"\"\n", - " python3 -m sglang.launch_server \\\n", - " --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \\\n", - " --enable-lora \\\n", - " --lora-backend csgmv \\\n", - " --max-loras-per-batch 16 \\\n", - " --lora-paths lora1=path/to/lora1 lora2=path/to/lora2\n", - " \"\"\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "terminate_process(server_process)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## LoRA Overlap Loading" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "By using the `--enable-lora-overlap-loading` server argument, the SGLang engine is able to overlap the loading of LoRA weights with prefill and decode compute, essentially hiding the data movement for LoRA weights behind GPU computation. Our benchmarks show that under adversarial conditions, enabling this feature can result in a ~35% reduction in median TTFT - (see the [LoRA overlap loading PR](https://github.com/sgl-project/sglang/pull/15512) for detailed benchmarks)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "lora0 = \"Nutanix/Meta-Llama-3.1-8B-Instruct_SFT_lora_4_alpha_16_humaneval_raw_json\"\n", - "lora1 = \"algoprog/fact-generation-llama-3.1-8b-instruct-lora\"\n", - "lora2 = \"philschmid/code-llama-3-1-8b-text-to-sql-lora\"\n", - "\n", - "\n", - "server_process, port = launch_server_cmd(\"\"\"\n", - " python3 -m sglang.launch_server \\\n", - " --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \\\n", - " --enable-lora \\\n", - " --enable-lora-overlap-loading \\\n", - " --lora-paths lora0=Nutanix/Meta-Llama-3.1-8B-Instruct_SFT_lora_4_alpha_16_humaneval_raw_json \\\n", - " lora1=algoprog/fact-generation-llama-3.1-8b-instruct-lora \\\n", - " lora2=philschmid/code-llama-3-1-8b-text-to-sql-lora \\\n", - " --max-lora-rank 256 \\\n", - " --max-loras-per-batch 2 \\\n", - " --max-loaded-loras 4\n", - " \"\"\")\n", - "\n", - "url = f\"http://127.0.0.1:{port}\"\n", - "wait_for_server(url, process=server_process)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "json_data = {\n", - " \"text\": [\n", - " \"Write a very long fairy-tale.\",\n", - " \"List 3 countries and their capitals.\",\n", - " \"List 3 countries and their capitals.\",\n", - " ],\n", - " \"sampling_params\": [\n", - " {\"max_new_tokens\": 1024, \"temperature\": 0},\n", - " {\"max_new_tokens\": 64, \"temperature\": 0},\n", - " {\"max_new_tokens\": 64, \"temperature\": 0},\n", - " ],\n", - " \"lora_path\": [\"lora0\", \"lora1\", \"lora2\"],\n", - "}\n", - "\n", - "# lora0 and lora1 will be loaded into the memory pool first, and because max_loras_per_batch = 2, lora2's request will remain in the queue.\n", - "# lora1's request will likely finish first, and once it does, lora2 will be loaded. With --enable-lora-overlap-loading, this loading will\n", - "# occur asynchronously and thus decoding for lora0's request won't be blocked.\n", - "response = requests.post(\n", - " url + \"/generate\",\n", - " json=json_data,\n", - ")\n", - "\n", - "for i in range(3):\n", - " print(f\"Output from lora{i}: \\n{response.json()[i]['text']}\\n\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "terminate_process(server_process)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Limitations of LoRA Overlap Loading" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "However, LoRA overlap loading is not free and comes with two important caveats:\n", - "\n", - "1. **Pinned CPU memory requirement**:\n", - " Asynchronous H2D memory copies require LoRA weights to be pinned in CPU memory, which is a finite system resource. To mitigate excessive pinned-memory usage, SGLang currently restricts `max_loaded_loras` to be at most 2× `max_loras_per_batch` when LoRA overlap loading is enabled.\n", - "\n", - "2. **Reduced multi-adapter prefill batching**:\n", - " With overlap loading, adapters become available on the GPU at different times because each adapter is loaded asynchronously. This can reduce the scheduler’s ability to form multi-adapter prefill batches, since only requests whose adapters are currently loaded can be grouped together. As a result, requests for different adapters will be scheduled in separate (or smaller) prefill batches, which can increase TTFT when adapter load time is small compared to prefill compute time. This is why LoRA overlap loading is disabled by default: it should only be enabled when users have determined that LoRA weight loading is a bottleneck (EG high adapter churn, heavy adapter weights, or PCIe-bottlenecked workloads).\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Example When Overlap Loading Results in Higher Latency" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "For instance, suppose we have four LoRA adapters: `lora0`, `lora1`, `lora2`, and `lora3`. Loading any adapter takes 2ms, while the prefill step for requests for that adapter takes 20ms.\n", - "\n", - "1. **Baseline**:\n", - " The engine loads all four adapters synchronously, then runs one combined prefill batch, giving us a total time of ≈ `2 * 4 + 20 = 28ms`\n", - "\n", - "2. **With LoRA overlap loading enabled**:\n", - " The engine begins loading `lora0` and, once it is ready, schedules a prefill batch containing only `lora0` while `lora1` loads in the background. Then it schedules `lora1`’s prefill while `lora2` loads, and so on. In the worst case where prefill cannot be batched across adapters, total time is ≈ `2 + 4 * 20 = 82ms`\n", - "\n", - "In this scenario, overlap loading reduces adapter-load overhead, but the loss of multi-adapter prefill batching dominates and leads to higher TTFT." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Future Works\n", - "\n", - "The development roadmap for LoRA-related features can be found in this [issue](https://github.com/sgl-project/sglang/issues/2929). Other features, including Embedding Layer, Unified Paging, Cutlass backend are still under development." - ] - } - ], - "metadata": { - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/docs_new/docs/advanced_features/separate_reasoning.ipynb b/docs_new/docs/advanced_features/separate_reasoning.ipynb deleted file mode 100644 index e6d8873e2..000000000 --- a/docs_new/docs/advanced_features/separate_reasoning.ipynb +++ /dev/null @@ -1,381 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Reasoning Parser\n", - "\n", - "SGLang supports parsing reasoning content out from \"normal\" content for reasoning models such as [DeepSeek R1](https://huggingface.co/deepseek-ai/DeepSeek-R1).\n", - "\n", - "## Supported Models & Parsers\n", - "\n", - "| Model | Reasoning tags | Parser | Notes |\n", - "|---------|-----------------------------|------------------|-------|\n", - "| [Apertus 2509 models](https://huggingface.co/swiss-ai/Apertus-8B-Instruct-2509) | `<\\|inner_prefix\\|>` … `<\\|inner_suffix\\|>` | `apertus2509` | For tool use, also set `--tool-call-parser apertus2509`. |\n", - "| [DeepSeek‑R1 series](https://huggingface.co/collections/deepseek-ai/deepseek-r1-678e1e131c0169c0bc89728d) | `` … `` | `deepseek-r1` | Supports all variants (R1, R1-0528, R1-Distill) |\n", - "| [DeepSeek‑V3 series](https://huggingface.co/deepseek-ai/DeepSeek-V3.1) | `` … `` | `deepseek-v3` | Including [DeepSeek‑V3.2](https://huggingface.co/deepseek-ai/DeepSeek-V3.2-Exp). Supports `thinking` parameter |\n", - "| [Standard Qwen3 models](https://huggingface.co/collections/Qwen/qwen3-67dd247413f0e2e4f653967f) | `` … `` | `qwen3` | Supports `enable_thinking` parameter |\n", - "| [Qwen3-Thinking models](https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507) | `` … `` | `qwen3` or `qwen3-thinking` | Always generates thinking content |\n", - "| [Kimi K2 Thinking](https://huggingface.co/moonshotai/Kimi-K2-Thinking) | `◁think▷` … `◁/think▷` | `kimi_k2` | Uses special thinking delimiters. Also requires `--tool-call-parser kimi_k2` for tool use. |\n", - "| [GPT OSS](https://huggingface.co/openai/gpt-oss-120b) | `<\\|channel\\|>analysis<\\|message\\|>` … `<\\|end\\|>` | `gpt-oss` | N/A |\n", - "### Model-Specific Behaviors\n", - "\n", - "**Apertus 2509:**\n", - "- Uses `<|inner_prefix|>` and `<|inner_suffix|>` to delimit reasoning content. For agentic tool use, also specify `--tool-call-parser apertus2509`.\n", - "\n", - "**DeepSeek-R1 Family:**\n", - "- DeepSeek-R1: No `` start tag, jumps directly to thinking content\n", - "- DeepSeek-R1-0528: Generates both `` start and `` end tags\n", - "- Both are handled by the same `deepseek-r1` parser\n", - "\n", - "**DeepSeek-V3 Family:**\n", - "- DeepSeek-V3.1/V3.2: Hybrid model supporting both thinking and non-thinking modes, use the `deepseek-v3` parser and `thinking` parameter (NOTE: not `enable_thinking`)\n", - "\n", - "**Qwen3 Family:**\n", - "- Standard Qwen3 (e.g., Qwen3-2507): Use `qwen3` parser, supports `enable_thinking` in chat templates\n", - "- Qwen3-Thinking (e.g., Qwen3-235B-A22B-Thinking-2507): Use `qwen3` or `qwen3-thinking` parser, always thinks\n", - "\n", - "**Kimi K2:**\n", - "- Kimi K2 Thinking: Uses special `◁think▷` and `◁/think▷` tags. For agentic tool use, also specify `--tool-call-parser kimi_k2`.\n", - "\n", - "**GPT OSS:**\n", - "- GPT OSS: Uses special `<|channel|>analysis<|message|>` and `<|end|>` tags" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Usage\n", - "\n", - "### Launching the Server" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Specify the `--reasoning-parser` option." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import requests\n", - "from openai import OpenAI\n", - "from sglang.test.doc_patch import launch_server_cmd\n", - "from sglang.utils import wait_for_server, print_highlight, terminate_process\n", - "\n", - "server_process, port = launch_server_cmd(\n", - " \"python3 -m sglang.launch_server --model-path deepseek-ai/DeepSeek-R1-Distill-Qwen-7B --host 0.0.0.0 --reasoning-parser deepseek-r1 --log-level warning\"\n", - ")\n", - "\n", - "wait_for_server(f\"http://localhost:{port}\", process=server_process)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Note that `--reasoning-parser` defines the parser used to interpret responses." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### OpenAI Compatible API\n", - "\n", - "Using the OpenAI compatible API, the contract follows the [DeepSeek API design](https://api-docs.deepseek.com/guides/reasoning_model) established with the release of DeepSeek-R1:\n", - "\n", - "- `reasoning_content`: The content of the CoT.\n", - "- `content`: The content of the final answer." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Initialize OpenAI-like client\n", - "client = OpenAI(api_key=\"None\", base_url=f\"http://0.0.0.0:{port}/v1\")\n", - "model_name = client.models.list().data[0].id\n", - "\n", - "messages = [\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"What is 1+3?\",\n", - " }\n", - "]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Non-Streaming Request" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "response_non_stream = client.chat.completions.create(\n", - " model=model_name,\n", - " messages=messages,\n", - " temperature=0.6,\n", - " top_p=0.95,\n", - " stream=False, # Non-streaming\n", - " extra_body={\"separate_reasoning\": True},\n", - ")\n", - "print_highlight(\"==== Reasoning ====\")\n", - "print_highlight(response_non_stream.choices[0].message.reasoning_content)\n", - "\n", - "print_highlight(\"==== Text ====\")\n", - "print_highlight(response_non_stream.choices[0].message.content)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Streaming Request" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "response_stream = client.chat.completions.create(\n", - " model=model_name,\n", - " messages=messages,\n", - " temperature=0.6,\n", - " top_p=0.95,\n", - " stream=True, # Non-streaming\n", - " extra_body={\"separate_reasoning\": True},\n", - ")\n", - "\n", - "reasoning_content = \"\"\n", - "content = \"\"\n", - "for chunk in response_stream:\n", - " if chunk.choices[0].delta.content:\n", - " content += chunk.choices[0].delta.content\n", - " if chunk.choices[0].delta.reasoning_content:\n", - " reasoning_content += chunk.choices[0].delta.reasoning_content\n", - "\n", - "print_highlight(\"==== Reasoning ====\")\n", - "print_highlight(reasoning_content)\n", - "\n", - "print_highlight(\"==== Text ====\")\n", - "print_highlight(content)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Optionally, you can buffer the reasoning content to the last reasoning chunk (or the first chunk after the reasoning content)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "response_stream = client.chat.completions.create(\n", - " model=model_name,\n", - " messages=messages,\n", - " temperature=0.6,\n", - " top_p=0.95,\n", - " stream=True, # Non-streaming\n", - " extra_body={\"separate_reasoning\": True, \"stream_reasoning\": False},\n", - ")\n", - "\n", - "reasoning_content = \"\"\n", - "content = \"\"\n", - "for chunk in response_stream:\n", - " if chunk.choices[0].delta.content:\n", - " content += chunk.choices[0].delta.content\n", - " if chunk.choices[0].delta.reasoning_content:\n", - " reasoning_content += chunk.choices[0].delta.reasoning_content\n", - "\n", - "print_highlight(\"==== Reasoning ====\")\n", - "print_highlight(reasoning_content)\n", - "\n", - "print_highlight(\"==== Text ====\")\n", - "print_highlight(content)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The reasoning separation is enable by default when specify . \n", - "**To disable it, set the `separate_reasoning` option to `False` in request.**" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "response_non_stream = client.chat.completions.create(\n", - " model=model_name,\n", - " messages=messages,\n", - " temperature=0.6,\n", - " top_p=0.95,\n", - " stream=False, # Non-streaming\n", - " extra_body={\"separate_reasoning\": False},\n", - ")\n", - "\n", - "print_highlight(\"==== Original Output ====\")\n", - "print_highlight(response_non_stream.choices[0].message.content)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### SGLang Native API " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from transformers import AutoTokenizer\n", - "\n", - "tokenizer = AutoTokenizer.from_pretrained(\"deepseek-ai/DeepSeek-R1-Distill-Qwen-7B\")\n", - "input = tokenizer.apply_chat_template(\n", - " messages, tokenize=False, add_generation_prompt=True, return_dict=False\n", - ")\n", - "\n", - "gen_url = f\"http://localhost:{port}/generate\"\n", - "gen_data = {\n", - " \"text\": input,\n", - " \"sampling_params\": {\n", - " \"skip_special_tokens\": False,\n", - " \"max_new_tokens\": 1024,\n", - " \"temperature\": 0.6,\n", - " \"top_p\": 0.95,\n", - " },\n", - "}\n", - "gen_response = requests.post(gen_url, json=gen_data).json()[\"text\"]\n", - "\n", - "print_highlight(\"==== Original Output ====\")\n", - "print_highlight(gen_response)\n", - "\n", - "parse_url = f\"http://localhost:{port}/separate_reasoning\"\n", - "separate_reasoning_data = {\n", - " \"text\": gen_response,\n", - " \"reasoning_parser\": \"deepseek-r1\",\n", - "}\n", - "separate_reasoning_response_json = requests.post(\n", - " parse_url, json=separate_reasoning_data\n", - ").json()\n", - "print_highlight(\"==== Reasoning ====\")\n", - "print_highlight(separate_reasoning_response_json[\"reasoning_text\"])\n", - "print_highlight(\"==== Text ====\")\n", - "print_highlight(separate_reasoning_response_json[\"text\"])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "terminate_process(server_process)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Offline Engine API" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import sglang as sgl\n", - "from sglang.srt.parser.reasoning_parser import ReasoningParser\n", - "from sglang.utils import print_highlight\n", - "\n", - "llm = sgl.Engine(model_path=\"deepseek-ai/DeepSeek-R1-Distill-Qwen-7B\")\n", - "tokenizer = AutoTokenizer.from_pretrained(\"deepseek-ai/DeepSeek-R1-Distill-Qwen-7B\")\n", - "input = tokenizer.apply_chat_template(\n", - " messages, tokenize=False, add_generation_prompt=True, return_dict=False\n", - ")\n", - "sampling_params = {\n", - " \"max_new_tokens\": 1024,\n", - " \"skip_special_tokens\": False,\n", - " \"temperature\": 0.6,\n", - " \"top_p\": 0.95,\n", - "}\n", - "result = llm.generate(prompt=input, sampling_params=sampling_params)\n", - "\n", - "generated_text = result[\"text\"] # Assume there is only one prompt\n", - "\n", - "print_highlight(\"==== Original Output ====\")\n", - "print_highlight(generated_text)\n", - "\n", - "parser = ReasoningParser(\"deepseek-r1\")\n", - "reasoning_text, text = parser.parse_non_stream(generated_text)\n", - "print_highlight(\"==== Reasoning ====\")\n", - "print_highlight(reasoning_text)\n", - "print_highlight(\"==== Text ====\")\n", - "print_highlight(text)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "llm.shutdown()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Supporting New Reasoning Model Schemas\n", - "\n", - "For future reasoning models, you can implement the reasoning parser as a subclass of `BaseReasoningFormatDetector` in `python/sglang/srt/reasoning_parser.py` and specify the reasoning parser for new reasoning model schemas accordingly." - ] - } - ], - "metadata": { - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/docs_new/docs/advanced_features/speculative_decoding.ipynb b/docs_new/docs/advanced_features/speculative_decoding.ipynb deleted file mode 100644 index 2cd02c0e8..000000000 --- a/docs_new/docs/advanced_features/speculative_decoding.ipynb +++ /dev/null @@ -1,360 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Speculative Decoding\n", - "\n", - "SGLang now provides an EAGLE-based (EAGLE-2/EAGLE-3) speculative decoding option. Our implementation aims to maximize speed and efficiency and is considered to be among the fastest in open-source LLM engines.\n", - "\n", - "### Performance Highlights\n", - "\n", - "Please see below for the huge improvements on throughput for LLaMA-Instruct 3.1 8B tested on MT bench that can be achieved via EAGLE3 decoding.\n", - "For further details please see the [EAGLE3 paper](https://arxiv.org/pdf/2503.01840).\n", - "\n", - "| Method | Throughput (tokens/s) |\n", - "|--------|----------------|\n", - "| SGLang (w/o speculative, 1x H100) | 158.34 tokens/s |\n", - "| SGLang + EAGLE-2 (1x H100) | 244.10 tokens/s |\n", - "| SGLang + EAGLE-3 (1x H100) | 373.25 tokens/s |" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## EAGLE Decoding\n", - "\n", - "To enable EAGLE speculative decoding the following parameters are relevant:\n", - "* `speculative_draft_model_path`: Specifies draft model. This parameter is required.\n", - "* `speculative_num_steps`: Depth of autoregressive drafting. Increases speculation range but risks rejection cascades. Default is 5.\n", - "* `speculative_eagle_topk`: Branching factor per step. Improves candidate diversity, will lead to higher acceptance rate, but more lead to higher memory/compute consumption. Default is 4.\n", - "* `speculative_num_draft_tokens`: Maximum parallel verification capacity. Allows deeper tree evaluation but will lead to higher GPU memory usage. Default is 8.\n", - "\n", - "These parameters are the same for EAGLE-2 and EAGLE-3.\n", - "\n", - "You can find the best combinations of these parameters with [bench_speculative.py](https://github.com/sgl-project/sglang/blob/main/scripts/playground/bench_speculative.py).\n", - "\n", - "In the documentation below, we set `--cuda-graph-max-bs-decode` to be a small value for faster engine startup. For your own workloads, please tune the above parameters together with `--cuda-graph-max-bs-decode`, `--max-running-requests`, `--mem-fraction-static` for the best performance. " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### EAGLE-2 decoding\n", - "\n", - "You can enable EAGLE-2 decoding by setting `--speculative-algorithm EAGLE` and choosing an appropriate model." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from sglang.test.doc_patch import launch_server_cmd\n", - "from sglang.utils import wait_for_server, print_highlight, terminate_process\n", - "\n", - "import openai" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "server_process, port = launch_server_cmd(\"\"\"\n", - "python3 -m sglang.launch_server --model meta-llama/Llama-2-7b-chat-hf --speculative-algorithm EAGLE \\\n", - " --speculative-draft-model-path lmsys/sglang-EAGLE-llama2-chat-7B --speculative-num-steps 3 \\\n", - " --speculative-eagle-topk 4 --speculative-num-draft-tokens 16 --cuda-graph-max-bs-decode 8 --log-level warning\n", - "\"\"\")\n", - "\n", - "wait_for_server(f\"http://localhost:{port}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "client = openai.Client(base_url=f\"http://127.0.0.1:{port}/v1\", api_key=\"None\")\n", - "\n", - "response = client.chat.completions.create(\n", - " model=\"meta-llama/Llama-2-7b-chat-hf\",\n", - " messages=[\n", - " {\"role\": \"user\", \"content\": \"List 3 countries and their capitals.\"},\n", - " ],\n", - " temperature=0,\n", - " max_tokens=64,\n", - ")\n", - "\n", - "print_highlight(f\"Response: {response}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "terminate_process(server_process)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### EAGLE-2 Decoding with `torch.compile`\n", - "\n", - "You can also enable `torch.compile` for further optimizations and optionally set `--torch-compile-max-bs`:\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "server_process, port = launch_server_cmd(\"\"\"\n", - "python3 -m sglang.launch_server --model meta-llama/Llama-2-7b-chat-hf --speculative-algorithm EAGLE \\\n", - " --speculative-draft-model-path lmsys/sglang-EAGLE-llama2-chat-7B --speculative-num-steps 5 \\\n", - " --speculative-eagle-topk 8 --speculative-num-draft-tokens 64 --mem-fraction 0.6 \\\n", - " --enable-torch-compile --torch-compile-max-bs 2 --log-level warning\n", - "\"\"\")\n", - "\n", - "wait_for_server(f\"http://localhost:{port}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "client = openai.Client(base_url=f\"http://127.0.0.1:{port}/v1\", api_key=\"None\")\n", - "\n", - "response = client.chat.completions.create(\n", - " model=\"meta-llama/Llama-2-7b-chat-hf\",\n", - " messages=[\n", - " {\"role\": \"user\", \"content\": \"List 3 countries and their capitals.\"},\n", - " ],\n", - " temperature=0,\n", - " max_tokens=64,\n", - ")\n", - "\n", - "print_highlight(f\"Response: {response}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "terminate_process(server_process)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### EAGLE-2 Decoding via Frequency-Ranked Speculative Sampling\n", - "\n", - "By employing a truncated high-frequency token vocabulary in the draft model, Eagle speculative decoding reduces `lm_head` computational overhead while accelerating the pipeline without quality degradation. For more details, checkout [the paper](https://arxiv.org/pdf/arXiv:2502.14856).\n", - "\n", - "In our implementation, set `--speculative-token-map` to enable the optimization. You can get the high-frequency token in FR-Spec from [this model](https://huggingface.co/thunlp/LLaMA3-Instruct-8B-FR-Spec). Or you can obtain high-frequency token by directly downloading these token from [this repo](https://github.com/thunlp/FR-Spec/tree/main?tab=readme-ov-file#prepare-fr-spec-vocabulary-subset).\n", - "\n", - "Thanks for the contribution from [Weilin Zhao](https://github.com/Achazwl) and [Zhousx](https://github.com/Zhou-sx). " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "server_process, port = launch_server_cmd(\"\"\"\n", - "python3 -m sglang.launch_server --model meta-llama/Meta-Llama-3-8B-Instruct --speculative-algorithm EAGLE \\\n", - " --speculative-draft-model-path lmsys/sglang-EAGLE-LLaMA3-Instruct-8B --speculative-num-steps 5 \\\n", - " --speculative-eagle-topk 8 --speculative-num-draft-tokens 64 --speculative-token-map thunlp/LLaMA3-Instruct-8B-FR-Spec/freq_32768.pt \\\n", - " --mem-fraction 0.7 --cuda-graph-max-bs-decode 2 --dtype float16 --log-level warning\n", - "\"\"\")\n", - "\n", - "wait_for_server(f\"http://localhost:{port}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "client = openai.Client(base_url=f\"http://127.0.0.1:{port}/v1\", api_key=\"None\")\n", - "\n", - "response = client.chat.completions.create(\n", - " model=\"meta-llama/Meta-Llama-3-8B-Instruct\",\n", - " messages=[\n", - " {\"role\": \"user\", \"content\": \"List 3 countries and their capitals.\"},\n", - " ],\n", - " temperature=0,\n", - " max_tokens=64,\n", - ")\n", - "\n", - "print_highlight(f\"Response: {response}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "terminate_process(server_process)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### EAGLE-3 Decoding\n", - "\n", - "You can enable EAGLE-3 decoding by setting `--speculative-algorithm EAGLE3` and choosing an appropriate model." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "server_process, port = launch_server_cmd(\"\"\"\n", - "python3 -m sglang.launch_server --model meta-llama/Llama-3.1-8B-Instruct --speculative-algorithm EAGLE3 \\\n", - " --speculative-draft-model-path jamesliu1/sglang-EAGLE3-Llama-3.1-Instruct-8B --speculative-num-steps 5 \\\n", - " --speculative-eagle-topk 8 --speculative-num-draft-tokens 32 --mem-fraction 0.6 \\\n", - " --cuda-graph-max-bs-decode 2 --dtype float16 --log-level warning\n", - "\"\"\")\n", - "\n", - "wait_for_server(f\"http://localhost:{port}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "client = openai.Client(base_url=f\"http://127.0.0.1:{port}/v1\", api_key=\"None\")\n", - "\n", - "response = client.chat.completions.create(\n", - " model=\"meta-llama/Meta-Llama-3.1-8B-Instruct\",\n", - " messages=[\n", - " {\"role\": \"user\", \"content\": \"List 3 countries and their capitals.\"},\n", - " ],\n", - " temperature=0,\n", - " max_tokens=64,\n", - ")\n", - "\n", - "print_highlight(f\"Response: {response}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "terminate_process(server_process)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Multi Token Prediction\n", - "\n", - "We support [MTP(Multi-Token Prediction)](https://arxiv.org/pdf/2404.19737) in SGLang by using speculative decoding. We use Xiaomi/MiMo-7B-RL model as example here (deepseek mtp usage refer to [deepseek doc](../basic_usage/deepseek.md#multi-token-prediction))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "server_process, port = launch_server_cmd(\"\"\"\n", - " python3 -m sglang.launch_server --model-path XiaomiMiMo/MiMo-7B-RL --host 0.0.0.0 --trust-remote-code \\\n", - " --speculative-algorithm EAGLE --speculative-num-steps 1 --speculative-eagle-topk 1 --speculative-num-draft-tokens 2 \\\n", - " --mem-fraction 0.5 --log-level warning\n", - "\"\"\")\n", - "\n", - "wait_for_server(f\"http://localhost:{port}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import requests\n", - "\n", - "url = f\"http://localhost:{port}/v1/chat/completions\"\n", - "\n", - "data = {\n", - " \"model\": \"XiaomiMiMo/MiMo-7B-RL\",\n", - " \"messages\": [{\"role\": \"user\", \"content\": \"What is the capital of France?\"}],\n", - "}\n", - "\n", - "response = requests.post(url, json=data)\n", - "print_highlight(response.json())" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "terminate_process(server_process)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## References\n", - "\n", - "EAGLE process is as follows:\n", - "\n", - "- Within EAGLE the draft model predicts the next feature vector, i.e. the last hidden state of the original LLM, using the feature sequence $(f_1, ..., f_k)$ and the token sequence $(t_2, ..., t_{k+1})$. \n", - "- The next token is then sampled from $p_{k+2}=\\text{LMHead}(f_{k+1})$. Afterwards, the two sequences are extended in a tree style—branching out multiple potential continuations, with the branching factor per step controlled by the `speculative_eagle_topk` parameter—to ensure a more coherent connection of context, and are given as input again.\n", - "- EAGLE-2 additionally uses the draft model to evaluate how probable certain branches in the draft tree are, dynamically stopping the expansion of unlikely branches. After the expansion phase, reranking is employed to select only the top `speculative_num_draft_tokens` final nodes as draft tokens.\n", - "- EAGLE-3 removes the feature prediction objective, incorporates low and mid-layer features, and is trained in an on-policy manner.\n", - "\n", - "This enhances drafting accuracy by operating on the features instead of tokens for more regular inputs and passing the tokens from the next timestep additionally to minimize randomness effects from sampling. Furthermore the dynamic adjustment of the draft tree and selection of reranked final nodes increases acceptance rate of draft tokens further. For more details see [EAGLE-2](https://arxiv.org/abs/2406.16858) and [EAGLE-3](https://arxiv.org/abs/2503.01840) paper.\n", - "\n", - "\n", - "For guidance how to train your own EAGLE model please see the [EAGLE repo](https://github.com/SafeAILab/EAGLE/tree/main?tab=readme-ov-file#train)." - ] - } - ], - "metadata": { - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/docs_new/docs/advanced_features/speculative_decoding.mdx b/docs_new/docs/advanced_features/speculative_decoding.mdx index af009ae83..52c89027b 100644 --- a/docs_new/docs/advanced_features/speculative_decoding.mdx +++ b/docs_new/docs/advanced_features/speculative_decoding.mdx @@ -11,7 +11,7 @@ SGLang provides several speculative decoding options, including EAGLE-2/EAGLE-3, - [EAGLE Decoding](#eagle-decoding) - [EAGLE-2 Decoding](#eagle-2-decoding) - - [EAGLE-2 Decoding with torch.compile](#eagle-2-decoding-with-torchcompile) + - [EAGLE-2 Decoding with torch.compile](#eagle-2-decoding-with-torch-compile) - [EAGLE-2 Decoding via Frequency-Ranked Speculative Sampling](#eagle-2-decoding-via-frequency-ranked-speculative-sampling) - [EAGLE-3 Decoding](#eagle-3-decoding) - [Multi Token Prediction](#multi-token-prediction) diff --git a/docs_new/docs/advanced_features/structured_outputs.ipynb b/docs_new/docs/advanced_features/structured_outputs.ipynb deleted file mode 100644 index 8902c9497..000000000 --- a/docs_new/docs/advanced_features/structured_outputs.ipynb +++ /dev/null @@ -1,994 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Structured Outputs" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You can specify a JSON schema, [regular expression](https://en.wikipedia.org/wiki/Regular_expression) or [EBNF](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form) to constrain the model output. The model output will be guaranteed to follow the given constraints. Only one constraint parameter (`json_schema`, `regex`, or `ebnf`) can be specified for a request.\n", - "\n", - "SGLang supports three grammar backends:\n", - "\n", - "- [XGrammar](https://github.com/mlc-ai/xgrammar)(default): Supports JSON schema, regular expression, and EBNF constraints.\n", - "- [Outlines](https://github.com/dottxt-ai/outlines): Supports JSON schema and regular expression constraints.\n", - "- [Llguidance](https://github.com/guidance-ai/llguidance): Supports JSON schema, regular expression, and EBNF constraints.\n", - "\n", - "We suggest using XGrammar for its better performance and utility. XGrammar currently uses the [GGML BNF format](https://github.com/ggerganov/llama.cpp/blob/master/grammars/README.md). For more details, see [XGrammar technical overview](https://blog.mlc.ai/2024/11/22/achieving-efficient-flexible-portable-structured-generation-with-xgrammar).\n", - "\n", - "To use Outlines, simply add `--grammar-backend outlines` when launching the server.\n", - "To use llguidance, add `--grammar-backend llguidance` when launching the server.\n", - "If no backend is specified, XGrammar will be used as the default.\n", - "\n", - "For better output quality, **It's advisable to explicitly include instructions in the prompt to guide the model to generate the desired format.** For example, you can specify, 'Please generate the output in the following JSON format: ...'.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## OpenAI Compatible API" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import openai\n", - "import os\n", - "\n", - "from sglang.test.doc_patch import launch_server_cmd\n", - "from sglang.utils import wait_for_server, print_highlight, terminate_process\n", - "\n", - "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n", - "\n", - "\n", - "server_process, port = launch_server_cmd(\n", - " \"python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-8B-Instruct --host 0.0.0.0 --log-level warning\"\n", - ")\n", - "\n", - "wait_for_server(f\"http://localhost:{port}\", process=server_process)\n", - "client = openai.Client(base_url=f\"http://127.0.0.1:{port}/v1\", api_key=\"None\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### JSON\n", - "\n", - "you can directly define a JSON schema or use [Pydantic](https://docs.pydantic.dev/latest/) to define and validate the response." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**Using Pydantic**" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from pydantic import BaseModel, Field\n", - "\n", - "\n", - "# Define the schema using Pydantic\n", - "class CapitalInfo(BaseModel):\n", - " name: str = Field(..., pattern=r\"^\\w+$\", description=\"Name of the capital city\")\n", - " population: int = Field(..., description=\"Population of the capital city\")\n", - "\n", - "\n", - "response = client.chat.completions.create(\n", - " model=\"meta-llama/Meta-Llama-3.1-8B-Instruct\",\n", - " messages=[\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"Please generate the information of the capital of France in the JSON format.\",\n", - " },\n", - " ],\n", - " temperature=0,\n", - " max_tokens=128,\n", - " response_format={\n", - " \"type\": \"json_schema\",\n", - " \"json_schema\": {\n", - " \"name\": \"foo\",\n", - " # convert the pydantic model to json schema\n", - " \"schema\": CapitalInfo.model_json_schema(),\n", - " },\n", - " },\n", - ")\n", - "\n", - "response_content = response.choices[0].message.content\n", - "# validate the JSON response by the pydantic model\n", - "capital_info = CapitalInfo.model_validate_json(response_content)\n", - "print_highlight(f\"Validated response: {capital_info.model_dump_json()}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**JSON Schema Directly**\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import json\n", - "\n", - "json_schema = json.dumps(\n", - " {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"name\": {\"type\": \"string\", \"pattern\": \"^[\\\\w]+$\"},\n", - " \"population\": {\"type\": \"integer\"},\n", - " },\n", - " \"required\": [\"name\", \"population\"],\n", - " }\n", - ")\n", - "\n", - "response = client.chat.completions.create(\n", - " model=\"meta-llama/Meta-Llama-3.1-8B-Instruct\",\n", - " messages=[\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"Give me the information of the capital of France in the JSON format.\",\n", - " },\n", - " ],\n", - " temperature=0,\n", - " max_tokens=128,\n", - " response_format={\n", - " \"type\": \"json_schema\",\n", - " \"json_schema\": {\"name\": \"foo\", \"schema\": json.loads(json_schema)},\n", - " },\n", - ")\n", - "\n", - "print_highlight(response.choices[0].message.content)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### EBNF" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "ebnf_grammar = \"\"\"\n", - "root ::= city | description\n", - "city ::= \"London\" | \"Paris\" | \"Berlin\" | \"Rome\"\n", - "description ::= city \" is \" status\n", - "status ::= \"the capital of \" country\n", - "country ::= \"England\" | \"France\" | \"Germany\" | \"Italy\"\n", - "\"\"\"\n", - "\n", - "response = client.chat.completions.create(\n", - " model=\"meta-llama/Meta-Llama-3.1-8B-Instruct\",\n", - " messages=[\n", - " {\"role\": \"system\", \"content\": \"You are a helpful geography bot.\"},\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"Give me the information of the capital of France.\",\n", - " },\n", - " ],\n", - " temperature=0,\n", - " max_tokens=32,\n", - " extra_body={\"ebnf\": ebnf_grammar},\n", - ")\n", - "\n", - "print_highlight(response.choices[0].message.content)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Regular expression" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "response = client.chat.completions.create(\n", - " model=\"meta-llama/Meta-Llama-3.1-8B-Instruct\",\n", - " messages=[\n", - " {\"role\": \"user\", \"content\": \"What is the capital of France?\"},\n", - " ],\n", - " temperature=0,\n", - " max_tokens=128,\n", - " extra_body={\"regex\": \"(Paris|London)\"},\n", - ")\n", - "\n", - "print_highlight(response.choices[0].message.content)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Structural Tag" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "tool_get_current_weather = {\n", - " \"type\": \"function\",\n", - " \"function\": {\n", - " \"name\": \"get_current_weather\",\n", - " \"description\": \"Get the current weather in a given location\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"city\": {\n", - " \"type\": \"string\",\n", - " \"description\": \"The city to find the weather for, e.g. 'San Francisco'\",\n", - " },\n", - " \"state\": {\n", - " \"type\": \"string\",\n", - " \"description\": \"the two-letter abbreviation for the state that the city is\"\n", - " \" in, e.g. 'CA' which would mean 'California'\",\n", - " },\n", - " \"unit\": {\n", - " \"type\": \"string\",\n", - " \"description\": \"The unit to fetch the temperature in\",\n", - " \"enum\": [\"celsius\", \"fahrenheit\"],\n", - " },\n", - " },\n", - " \"required\": [\"city\", \"state\", \"unit\"],\n", - " },\n", - " },\n", - "}\n", - "\n", - "tool_get_current_date = {\n", - " \"type\": \"function\",\n", - " \"function\": {\n", - " \"name\": \"get_current_date\",\n", - " \"description\": \"Get the current date and time for a given timezone\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"timezone\": {\n", - " \"type\": \"string\",\n", - " \"description\": \"The timezone to fetch the current date and time for, e.g. 'America/New_York'\",\n", - " }\n", - " },\n", - " \"required\": [\"timezone\"],\n", - " },\n", - " },\n", - "}\n", - "\n", - "schema_get_current_weather = tool_get_current_weather[\"function\"][\"parameters\"]\n", - "schema_get_current_date = tool_get_current_date[\"function\"][\"parameters\"]\n", - "\n", - "\n", - "def get_messages():\n", - " return [\n", - " {\n", - " \"role\": \"system\",\n", - " \"content\": f\"\"\"\n", - "# Tool Instructions\n", - "- Always execute python code in messages that you share.\n", - "- When looking for real time information use relevant functions if available else fallback to brave_search\n", - "You have access to the following functions:\n", - "Use the function 'get_current_weather' to: Get the current weather in a given location\n", - "{tool_get_current_weather[\"function\"]}\n", - "Use the function 'get_current_date' to: Get the current date and time for a given timezone\n", - "{tool_get_current_date[\"function\"]}\n", - "If a you choose to call a function ONLY reply in the following format:\n", - "<{{start_tag}}={{function_name}}>{{parameters}}{{end_tag}}\n", - "where\n", - "start_tag => ` a JSON dict with the function argument name as key and function argument value as value.\n", - "end_tag => ``\n", - "Here is an example,\n", - "{{\"example_name\": \"example_value\"}}\n", - "Reminder:\n", - "- Function calls MUST follow the specified format\n", - "- Required parameters MUST be specified\n", - "- Only call one function at a time\n", - "- Put the entire function call reply on one line\n", - "- Always add your sources when using search results to answer the user query\n", - "You are a helpful assistant.\"\"\",\n", - " },\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"You are in New York. Please get the current date and time, and the weather.\",\n", - " },\n", - " ]\n", - "\n", - "\n", - "messages = get_messages()\n", - "\n", - "response = client.chat.completions.create(\n", - " model=\"meta-llama/Meta-Llama-3.1-8B-Instruct\",\n", - " messages=messages,\n", - " response_format={\n", - " \"type\": \"structural_tag\",\n", - " \"structures\": [\n", - " {\n", - " \"begin\": \"\",\n", - " \"schema\": schema_get_current_weather,\n", - " \"end\": \"\",\n", - " },\n", - " {\n", - " \"begin\": \"\",\n", - " \"schema\": schema_get_current_date,\n", - " \"end\": \"\",\n", - " },\n", - " ],\n", - " \"triggers\": [\"\n", - "response = client.chat.completions.create(\n", - " model=\"meta-llama/Meta-Llama-3.1-8B-Instruct\",\n", - " messages=messages,\n", - " response_format={\n", - " \"type\": \"structural_tag\",\n", - " \"format\": {\n", - " \"type\": \"triggered_tags\",\n", - " \"triggers\": [\"\",\n", - " \"content\": {\n", - " \"type\": \"json_schema\",\n", - " \"json_schema\": schema_get_current_weather,\n", - " },\n", - " \"end\": \"\",\n", - " },\n", - " {\n", - " \"begin\": \"\",\n", - " \"content\": {\n", - " \"type\": \"json_schema\",\n", - " \"json_schema\": schema_get_current_date,\n", - " },\n", - " \"end\": \"\",\n", - " },\n", - " ],\n", - " \"at_least_one\": False,\n", - " \"stop_after_first\": False,\n", - " },\n", - " },\n", - ")\n", - "\n", - "print_highlight(response.choices[0].message.content)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Native API and SGLang Runtime (SRT)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### JSON" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**Using Pydantic**" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import requests\n", - "import json\n", - "from pydantic import BaseModel, Field\n", - "\n", - "from transformers import AutoTokenizer\n", - "\n", - "tokenizer = AutoTokenizer.from_pretrained(\"meta-llama/Meta-Llama-3.1-8B-Instruct\")\n", - "\n", - "\n", - "# Define the schema using Pydantic\n", - "class CapitalInfo(BaseModel):\n", - " name: str = Field(..., pattern=r\"^\\w+$\", description=\"Name of the capital city\")\n", - " population: int = Field(..., description=\"Population of the capital city\")\n", - "\n", - "\n", - "# Make API request\n", - "messages = [\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"Here is the information of the capital of France in the JSON format.\\n\",\n", - " }\n", - "]\n", - "text = tokenizer.apply_chat_template(\n", - " messages, tokenize=False, add_generation_prompt=True, return_dict=False\n", - ")\n", - "response = requests.post(\n", - " f\"http://localhost:{port}/generate\",\n", - " json={\n", - " \"text\": text,\n", - " \"sampling_params\": {\n", - " \"temperature\": 0,\n", - " \"max_new_tokens\": 64,\n", - " \"json_schema\": json.dumps(CapitalInfo.model_json_schema()),\n", - " },\n", - " },\n", - ")\n", - "print_highlight(response.json())\n", - "\n", - "\n", - "response_data = json.loads(response.json()[\"text\"])\n", - "# validate the response by the pydantic model\n", - "capital_info = CapitalInfo.model_validate(response_data)\n", - "print_highlight(f\"Validated response: {capital_info.model_dump_json()}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**JSON Schema Directly**" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "json_schema = json.dumps(\n", - " {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"name\": {\"type\": \"string\", \"pattern\": \"^[\\\\w]+$\"},\n", - " \"population\": {\"type\": \"integer\"},\n", - " },\n", - " \"required\": [\"name\", \"population\"],\n", - " }\n", - ")\n", - "\n", - "# JSON\n", - "response = requests.post(\n", - " f\"http://localhost:{port}/generate\",\n", - " json={\n", - " \"text\": text,\n", - " \"sampling_params\": {\n", - " \"temperature\": 0,\n", - " \"max_new_tokens\": 64,\n", - " \"json_schema\": json_schema,\n", - " },\n", - " },\n", - ")\n", - "\n", - "print_highlight(response.json())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### EBNF" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "messages = [\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"Give me the information of the capital of France.\",\n", - " }\n", - "]\n", - "text = tokenizer.apply_chat_template(\n", - " messages, tokenize=False, add_generation_prompt=True, return_dict=False\n", - ")\n", - "response = requests.post(\n", - " f\"http://localhost:{port}/generate\",\n", - " json={\n", - " \"text\": text,\n", - " \"sampling_params\": {\n", - " \"max_new_tokens\": 128,\n", - " \"temperature\": 0,\n", - " \"n\": 3,\n", - " \"ebnf\": (\n", - " \"root ::= city | description\\n\"\n", - " 'city ::= \"London\" | \"Paris\" | \"Berlin\" | \"Rome\"\\n'\n", - " 'description ::= city \" is \" status\\n'\n", - " 'status ::= \"the capital of \" country\\n'\n", - " 'country ::= \"England\" | \"France\" | \"Germany\" | \"Italy\"'\n", - " ),\n", - " },\n", - " \"stream\": False,\n", - " \"return_logprob\": False,\n", - " },\n", - ")\n", - "\n", - "print_highlight(response.json())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Regular expression" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "messages = [\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"Paris is the capital of\",\n", - " }\n", - "]\n", - "text = tokenizer.apply_chat_template(\n", - " messages, tokenize=False, add_generation_prompt=True, return_dict=False\n", - ")\n", - "response = requests.post(\n", - " f\"http://localhost:{port}/generate\",\n", - " json={\n", - " \"text\": text,\n", - " \"sampling_params\": {\n", - " \"temperature\": 0,\n", - " \"max_new_tokens\": 64,\n", - " \"regex\": \"(France|England)\",\n", - " },\n", - " },\n", - ")\n", - "print_highlight(response.json())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Structural Tag" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from transformers import AutoTokenizer\n", - "\n", - "# generate an answer\n", - "tokenizer = AutoTokenizer.from_pretrained(\"meta-llama/Meta-Llama-3.1-8B-Instruct\")\n", - "\n", - "text = tokenizer.apply_chat_template(\n", - " messages, tokenize=False, add_generation_prompt=True, return_dict=False\n", - ")\n", - "payload = {\n", - " \"text\": text,\n", - " \"sampling_params\": {\n", - " \"structural_tag\": json.dumps(\n", - " {\n", - " \"type\": \"structural_tag\",\n", - " \"structures\": [\n", - " {\n", - " \"begin\": \"\",\n", - " \"schema\": schema_get_current_weather,\n", - " \"end\": \"\",\n", - " },\n", - " {\n", - " \"begin\": \"\",\n", - " \"schema\": schema_get_current_date,\n", - " \"end\": \"\",\n", - " },\n", - " ],\n", - " \"triggers\": [\"\n", - "payload = {\n", - " \"text\": text,\n", - " \"sampling_params\": {\n", - " \"structural_tag\": json.dumps(\n", - " {\n", - " \"type\": \"structural_tag\",\n", - " \"format\": {\n", - " \"type\": \"triggered_tags\",\n", - " \"triggers\": [\"\",\n", - " \"content\": {\n", - " \"type\": \"json_schema\",\n", - " \"json_schema\": schema_get_current_weather,\n", - " },\n", - " \"end\": \"\",\n", - " },\n", - " {\n", - " \"begin\": \"\",\n", - " \"content\": {\n", - " \"type\": \"json_schema\",\n", - " \"json_schema\": schema_get_current_date,\n", - " },\n", - " \"end\": \"\",\n", - " },\n", - " ],\n", - " \"at_least_one\": False,\n", - " \"stop_after_first\": False,\n", - " },\n", - " }\n", - " )\n", - " },\n", - "}\n", - "\n", - "\n", - "# Send POST request to the API endpoint\n", - "response = requests.post(f\"http://localhost:{port}/generate\", json=payload)\n", - "print_highlight(response.json())" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "terminate_process(server_process)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Offline Engine API" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import sglang as sgl\n", - "\n", - "llm = sgl.Engine(\n", - " model_path=\"meta-llama/Meta-Llama-3.1-8B-Instruct\", grammar_backend=\"xgrammar\"\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### JSON" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**Using Pydantic**" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import json\n", - "from pydantic import BaseModel, Field\n", - "\n", - "prompts = [\n", - " \"Give me the information of the capital of China in the JSON format.\",\n", - " \"Give me the information of the capital of France in the JSON format.\",\n", - " \"Give me the information of the capital of Ireland in the JSON format.\",\n", - "]\n", - "\n", - "\n", - "# Define the schema using Pydantic\n", - "class CapitalInfo(BaseModel):\n", - " name: str = Field(..., pattern=r\"^\\w+$\", description=\"Name of the capital city\")\n", - " population: int = Field(..., description=\"Population of the capital city\")\n", - "\n", - "\n", - "sampling_params = {\n", - " \"temperature\": 0.1,\n", - " \"top_p\": 0.95,\n", - " \"json_schema\": json.dumps(CapitalInfo.model_json_schema()),\n", - "}\n", - "\n", - "outputs = llm.generate(prompts, sampling_params)\n", - "for prompt, output in zip(prompts, outputs):\n", - " print_highlight(\"===============================\")\n", - " print_highlight(f\"Prompt: {prompt}\") # validate the output by the pydantic model\n", - " capital_info = CapitalInfo.model_validate_json(output[\"text\"])\n", - " print_highlight(f\"Validated output: {capital_info.model_dump_json()}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**JSON Schema Directly**" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "prompts = [\n", - " \"Give me the information of the capital of China in the JSON format.\",\n", - " \"Give me the information of the capital of France in the JSON format.\",\n", - " \"Give me the information of the capital of Ireland in the JSON format.\",\n", - "]\n", - "\n", - "json_schema = json.dumps(\n", - " {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"name\": {\"type\": \"string\", \"pattern\": \"^[\\\\w]+$\"},\n", - " \"population\": {\"type\": \"integer\"},\n", - " },\n", - " \"required\": [\"name\", \"population\"],\n", - " }\n", - ")\n", - "\n", - "sampling_params = {\"temperature\": 0.1, \"top_p\": 0.95, \"json_schema\": json_schema}\n", - "\n", - "outputs = llm.generate(prompts, sampling_params)\n", - "for prompt, output in zip(prompts, outputs):\n", - " print_highlight(\"===============================\")\n", - " print_highlight(f\"Prompt: {prompt}\\nGenerated text: {output['text']}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### EBNF\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "prompts = [\n", - " \"Give me the information of the capital of France.\",\n", - " \"Give me the information of the capital of Germany.\",\n", - " \"Give me the information of the capital of Italy.\",\n", - "]\n", - "\n", - "sampling_params = {\n", - " \"temperature\": 0.8,\n", - " \"top_p\": 0.95,\n", - " \"ebnf\": (\n", - " \"root ::= city | description\\n\"\n", - " 'city ::= \"London\" | \"Paris\" | \"Berlin\" | \"Rome\"\\n'\n", - " 'description ::= city \" is \" status\\n'\n", - " 'status ::= \"the capital of \" country\\n'\n", - " 'country ::= \"England\" | \"France\" | \"Germany\" | \"Italy\"'\n", - " ),\n", - "}\n", - "\n", - "outputs = llm.generate(prompts, sampling_params)\n", - "for prompt, output in zip(prompts, outputs):\n", - " print_highlight(\"===============================\")\n", - " print_highlight(f\"Prompt: {prompt}\\nGenerated text: {output['text']}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Regular expression" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "prompts = [\n", - " \"Please provide information about London as a major global city:\",\n", - " \"Please provide information about Paris as a major global city:\",\n", - "]\n", - "\n", - "sampling_params = {\"temperature\": 0.8, \"top_p\": 0.95, \"regex\": \"(France|England)\"}\n", - "\n", - "outputs = llm.generate(prompts, sampling_params)\n", - "for prompt, output in zip(prompts, outputs):\n", - " print_highlight(\"===============================\")\n", - " print_highlight(f\"Prompt: {prompt}\\nGenerated text: {output['text']}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Structural Tag" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "text = tokenizer.apply_chat_template(\n", - " messages, tokenize=False, add_generation_prompt=True, return_dict=False\n", - ")\n", - "prompts = [text]\n", - "\n", - "\n", - "sampling_params = {\n", - " \"temperature\": 0.8,\n", - " \"top_p\": 0.95,\n", - " \"structural_tag\": json.dumps(\n", - " {\n", - " \"type\": \"structural_tag\",\n", - " \"structures\": [\n", - " {\n", - " \"begin\": \"\",\n", - " \"schema\": schema_get_current_weather,\n", - " \"end\": \"\",\n", - " },\n", - " {\n", - " \"begin\": \"\",\n", - " \"schema\": schema_get_current_date,\n", - " \"end\": \"\",\n", - " },\n", - " ],\n", - " \"triggers\": [\"\n", - "sampling_params = {\n", - " \"temperature\": 0.8,\n", - " \"top_p\": 0.95,\n", - " \"structural_tag\": json.dumps(\n", - " {\n", - " \"type\": \"structural_tag\",\n", - " \"format\": {\n", - " \"type\": \"triggered_tags\",\n", - " \"triggers\": [\"\",\n", - " \"content\": {\n", - " \"type\": \"json_schema\",\n", - " \"json_schema\": schema_get_current_weather,\n", - " },\n", - " \"end\": \"\",\n", - " },\n", - " {\n", - " \"begin\": \"\",\n", - " \"content\": {\n", - " \"type\": \"json_schema\",\n", - " \"json_schema\": schema_get_current_date,\n", - " },\n", - " \"end\": \"\",\n", - " },\n", - " ],\n", - " \"at_least_one\": False,\n", - " \"stop_after_first\": False,\n", - " },\n", - " }\n", - " ),\n", - "}\n", - "\n", - "\n", - "# Send POST request to the API endpoint\n", - "outputs = llm.generate(prompts, sampling_params)\n", - "for prompt, output in zip(prompts, outputs):\n", - " print_highlight(\"===============================\")\n", - " print_highlight(f\"Prompt: {prompt}\\nGenerated text: {output['text']}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "llm.shutdown()" - ] - } - ], - "metadata": { - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/docs_new/docs/advanced_features/structured_outputs.mdx b/docs_new/docs/advanced_features/structured_outputs.mdx index 26e450bfd..c41e5a190 100644 --- a/docs_new/docs/advanced_features/structured_outputs.mdx +++ b/docs_new/docs/advanced_features/structured_outputs.mdx @@ -11,7 +11,7 @@ SGLang supports three grammar backends: - [Outlines](https://github.com/dottxt-ai/outlines): Supports JSON schema and regular expression constraints. - [Llguidance](https://github.com/guidance-ai/llguidance): Supports JSON schema, regular expression, and EBNF constraints. -We suggest using XGrammar for its better performance and utility. XGrammar currently uses the [GGML BNF format](https://github.com/ggerganov/llama.cpp/blob/master/grammars/README.md). For more details, see [XGrammar technical overview](https://blog.mlc.ai/2024/11/22/achieving-efficient-flexible-portable-structured-generation-with-xgrammar). +We suggest using XGrammar for its better performance and utility. XGrammar currently uses the [GGML BNF format](https://github.com/ggml-org/llama.cpp/blob/master/grammars/README.md). For more details, see [XGrammar technical overview](https://blog.mlc.ai/2024/11/22/achieving-efficient-flexible-portable-structured-generation-with-xgrammar). To use Outlines, simply add `--grammar-backend outlines` when launching the server. To use llguidance, add `--grammar-backend llguidance` when launching the server. @@ -294,7 +294,7 @@ print_highlight(response.choices[0].message.content) ```python Example # Support for XGrammar latest structural tag format -# https://xgrammar.mlc.ai/docs/tutorials/structural_tag.html +# https://xgrammar.mlc.ai/docs/api/python/structural_tag.html response = client.chat.completions.create( model="meta-llama/Meta-Llama-3.1-8B-Instruct", @@ -531,7 +531,7 @@ print_highlight(response.json()) ```python Example # Support for XGrammar latest structural tag format -# https://xgrammar.mlc.ai/docs/tutorials/structural_tag.html +# https://xgrammar.mlc.ai/docs/api/python/structural_tag.html payload = { "text": text, @@ -753,7 +753,7 @@ for prompt, output in zip(prompts, outputs): ```python Example # Support for XGrammar latest structural tag format -# https://xgrammar.mlc.ai/docs/tutorials/structural_tag.html +# https://xgrammar.mlc.ai/docs/api/python/structural_tag.html sampling_params = { "temperature": 0.8, diff --git a/docs_new/docs/advanced_features/structured_outputs_for_reasoning_models.ipynb b/docs_new/docs/advanced_features/structured_outputs_for_reasoning_models.ipynb deleted file mode 100644 index cfc07fd01..000000000 --- a/docs_new/docs/advanced_features/structured_outputs_for_reasoning_models.ipynb +++ /dev/null @@ -1,841 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Structured Outputs For Reasoning Models\n", - "\n", - "When working with reasoning models that use special tokens like `...` to denote reasoning sections, you might want to allow free-form text within these sections while still enforcing grammar constraints on the rest of the output.\n", - "\n", - "SGLang provides a feature to disable grammar restrictions within reasoning sections. This is particularly useful for models that need to perform complex reasoning steps before providing a structured output.\n", - "\n", - "To enable this feature, use the `--reasoning-parser` flag which decide the think_end_token, such as ``, when launching the server. You can also specify the reasoning parser using the `--reasoning-parser` flag.\n", - "\n", - "## Supported Models\n", - "\n", - "Currently, SGLang supports the following reasoning models:\n", - "- [DeepSeek R1 series](https://huggingface.co/collections/deepseek-ai/deepseek-r1-678e1e131c0169c0bc89728d): The reasoning content is wrapped with `` and `` tags.\n", - "- [QwQ](https://huggingface.co/Qwen/QwQ-32B): The reasoning content is wrapped with `` and `` tags.\n", - "\n", - "\n", - "## Usage\n", - "\n", - "## OpenAI Compatible API" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Specify the `--grammar-backend`, `--reasoning-parser` option." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import openai\n", - "import os\n", - "\n", - "from sglang.test.doc_patch import launch_server_cmd\n", - "from sglang.utils import wait_for_server, print_highlight, terminate_process\n", - "\n", - "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n", - "\n", - "\n", - "server_process, port = launch_server_cmd(\n", - " \"python -m sglang.launch_server --model-path deepseek-ai/DeepSeek-R1-Distill-Qwen-7B --host 0.0.0.0 --reasoning-parser deepseek-r1 --log-level warning\"\n", - ")\n", - "\n", - "wait_for_server(f\"http://localhost:{port}\", process=server_process)\n", - "client = openai.Client(base_url=f\"http://127.0.0.1:{port}/v1\", api_key=\"None\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### JSON\n", - "\n", - "you can directly define a JSON schema or use [Pydantic](https://docs.pydantic.dev/latest/) to define and validate the response." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**Using Pydantic**" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from pydantic import BaseModel, Field\n", - "\n", - "\n", - "# Define the schema using Pydantic\n", - "class CapitalInfo(BaseModel):\n", - " name: str = Field(..., pattern=r\"^\\w+$\", description=\"Name of the capital city\")\n", - " population: int = Field(..., description=\"Population of the capital city\")\n", - "\n", - "\n", - "response = client.chat.completions.create(\n", - " model=\"deepseek-ai/DeepSeek-R1-Distill-Qwen-7B\",\n", - " messages=[\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"Give me the information and population of the capital of France in the JSON format.\",\n", - " },\n", - " ],\n", - " temperature=0,\n", - " max_tokens=2048,\n", - " response_format={\n", - " \"type\": \"json_schema\",\n", - " \"json_schema\": {\n", - " \"name\": \"foo\",\n", - " # convert the pydantic model to json schema\n", - " \"schema\": CapitalInfo.model_json_schema(),\n", - " },\n", - " },\n", - ")\n", - "\n", - "print_highlight(\n", - " f\"reasoing_content: {response.choices[0].message.reasoning_content}\\n\\ncontent: {response.choices[0].message.content}\"\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**JSON Schema Directly**\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import json\n", - "\n", - "json_schema = json.dumps(\n", - " {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"name\": {\"type\": \"string\", \"pattern\": \"^[\\\\w]+$\"},\n", - " \"population\": {\"type\": \"integer\"},\n", - " },\n", - " \"required\": [\"name\", \"population\"],\n", - " }\n", - ")\n", - "\n", - "response = client.chat.completions.create(\n", - " model=\"deepseek-ai/DeepSeek-R1-Distill-Qwen-7B\",\n", - " messages=[\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"Give me the information and population of the capital of France in the JSON format.\",\n", - " },\n", - " ],\n", - " temperature=0,\n", - " max_tokens=2048,\n", - " response_format={\n", - " \"type\": \"json_schema\",\n", - " \"json_schema\": {\"name\": \"foo\", \"schema\": json.loads(json_schema)},\n", - " },\n", - ")\n", - "\n", - "print_highlight(\n", - " f\"reasoing_content: {response.choices[0].message.reasoning_content}\\n\\ncontent: {response.choices[0].message.content}\"\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### EBNF" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "ebnf_grammar = \"\"\"\n", - "root ::= city | description\n", - "city ::= \"London\" | \"Paris\" | \"Berlin\" | \"Rome\"\n", - "description ::= city \" is \" status\n", - "status ::= \"the capital of \" country\n", - "country ::= \"England\" | \"France\" | \"Germany\" | \"Italy\"\n", - "\"\"\"\n", - "\n", - "response = client.chat.completions.create(\n", - " model=\"deepseek-ai/DeepSeek-R1-Distill-Qwen-7B\",\n", - " messages=[\n", - " {\"role\": \"system\", \"content\": \"You are a helpful geography bot.\"},\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"Give me the information and population of the capital of France in the JSON format.\",\n", - " },\n", - " ],\n", - " temperature=0,\n", - " max_tokens=2048,\n", - " extra_body={\"ebnf\": ebnf_grammar},\n", - ")\n", - "\n", - "print_highlight(\n", - " f\"reasoing_content: {response.choices[0].message.reasoning_content}\\n\\ncontent: {response.choices[0].message.content}\"\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Regular expression" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "response = client.chat.completions.create(\n", - " model=\"deepseek-ai/DeepSeek-R1-Distill-Qwen-7B\",\n", - " messages=[\n", - " {\"role\": \"assistant\", \"content\": \"What is the capital of France?\"},\n", - " ],\n", - " temperature=0,\n", - " max_tokens=2048,\n", - " extra_body={\"regex\": \"(Paris|London)\"},\n", - ")\n", - "\n", - "print_highlight(\n", - " f\"reasoing_content: {response.choices[0].message.reasoning_content}\\n\\ncontent: {response.choices[0].message.content}\"\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Structural Tag" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "tool_get_current_weather = {\n", - " \"type\": \"function\",\n", - " \"function\": {\n", - " \"name\": \"get_current_weather\",\n", - " \"description\": \"Get the current weather in a given location\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"city\": {\n", - " \"type\": \"string\",\n", - " \"description\": \"The city to find the weather for, e.g. 'San Francisco'\",\n", - " },\n", - " \"state\": {\n", - " \"type\": \"string\",\n", - " \"description\": \"the two-letter abbreviation for the state that the city is\"\n", - " \" in, e.g. 'CA' which would mean 'California'\",\n", - " },\n", - " \"unit\": {\n", - " \"type\": \"string\",\n", - " \"description\": \"The unit to fetch the temperature in\",\n", - " \"enum\": [\"celsius\", \"fahrenheit\"],\n", - " },\n", - " },\n", - " \"required\": [\"city\", \"state\", \"unit\"],\n", - " },\n", - " },\n", - "}\n", - "\n", - "tool_get_current_date = {\n", - " \"type\": \"function\",\n", - " \"function\": {\n", - " \"name\": \"get_current_date\",\n", - " \"description\": \"Get the current date and time for a given timezone\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"timezone\": {\n", - " \"type\": \"string\",\n", - " \"description\": \"The timezone to fetch the current date and time for, e.g. 'America/New_York'\",\n", - " }\n", - " },\n", - " \"required\": [\"timezone\"],\n", - " },\n", - " },\n", - "}\n", - "\n", - "schema_get_current_weather = tool_get_current_weather[\"function\"][\"parameters\"]\n", - "schema_get_current_date = tool_get_current_date[\"function\"][\"parameters\"]\n", - "\n", - "\n", - "def get_messages():\n", - " return [\n", - " {\n", - " \"role\": \"system\",\n", - " \"content\": f\"\"\"\n", - "# Tool Instructions\n", - "- Always execute python code in messages that you share.\n", - "- When looking for real time information use relevant functions if available else fallback to brave_search\n", - "You have access to the following functions:\n", - "Use the function 'get_current_weather' to: Get the current weather in a given location\n", - "{tool_get_current_weather[\"function\"]}\n", - "Use the function 'get_current_date' to: Get the current date and time for a given timezone\n", - "{tool_get_current_date[\"function\"]}\n", - "If a you choose to call a function ONLY reply in the following format:\n", - "<{{start_tag}}={{function_name}}>{{parameters}}{{end_tag}}\n", - "where\n", - "start_tag => ` a JSON dict with the function argument name as key and function argument value as value.\n", - "end_tag => ``\n", - "Here is an example,\n", - "{{\"example_name\": \"example_value\"}}\n", - "Reminder:\n", - "- Function calls MUST follow the specified format\n", - "- Required parameters MUST be specified\n", - "- Only call one function at a time\n", - "- Put the entire function call reply on one line\n", - "- Always add your sources when using search results to answer the user query\n", - "You are a helpful assistant.\"\"\",\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"You are in New York. Please get the current date and time, and the weather.\",\n", - " },\n", - " ]\n", - "\n", - "\n", - "messages = get_messages()\n", - "\n", - "response = client.chat.completions.create(\n", - " model=\"deepseek-ai/DeepSeek-R1-Distill-Qwen-7B\",\n", - " messages=messages,\n", - " response_format={\n", - " \"type\": \"structural_tag\",\n", - " \"max_new_tokens\": 2048,\n", - " \"structures\": [\n", - " {\n", - " \"begin\": \"\",\n", - " \"schema\": schema_get_current_weather,\n", - " \"end\": \"\",\n", - " },\n", - " {\n", - " \"begin\": \"\",\n", - " \"schema\": schema_get_current_date,\n", - " \"end\": \"\",\n", - " },\n", - " ],\n", - " \"triggers\": [\" Note: For native API, as a work-around, you need to set `require_reasoning` argument to `True` to ensure the model will think before generating the structured output. It's not required for chat-completion API." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### JSON" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**Using Pydantic**" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import requests\n", - "from pydantic import BaseModel, Field\n", - "from transformers import AutoTokenizer\n", - "\n", - "tokenizer = AutoTokenizer.from_pretrained(\"deepseek-ai/DeepSeek-R1-Distill-Qwen-7B\")\n", - "\n", - "\n", - "# Define the schema using Pydantic\n", - "class CapitalInfo(BaseModel):\n", - " name: str = Field(..., pattern=r\"^\\w+$\", description=\"Name of the capital city\")\n", - " population: int = Field(..., description=\"Population of the capital city\")\n", - "\n", - "\n", - "messages = [\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"Give me the information and population of the capital of France in the JSON format.\",\n", - " },\n", - "]\n", - "text = tokenizer.apply_chat_template(\n", - " messages, tokenize=False, add_generation_prompt=True, return_dict=False\n", - ")\n", - "# Make API request\n", - "response = requests.post(\n", - " f\"http://localhost:{port}/generate\",\n", - " json={\n", - " \"text\": text,\n", - " \"require_reasoning\": True,\n", - " \"sampling_params\": {\n", - " \"temperature\": 0,\n", - " \"max_new_tokens\": 2048,\n", - " \"json_schema\": json.dumps(CapitalInfo.model_json_schema()),\n", - " },\n", - " },\n", - ")\n", - "print(response.json())\n", - "\n", - "\n", - "reasoing_content = response.json()[\"text\"].split(\"\")[0]\n", - "content = response.json()[\"text\"].split(\"\")[1]\n", - "print_highlight(f\"reasoing_content: {reasoing_content}\\n\\ncontent: {content}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**JSON Schema Directly**" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "json_schema = json.dumps(\n", - " {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"name\": {\"type\": \"string\", \"pattern\": \"^[\\\\w]+$\"},\n", - " \"population\": {\"type\": \"integer\"},\n", - " },\n", - " \"required\": [\"name\", \"population\"],\n", - " }\n", - ")\n", - "\n", - "# JSON\n", - "text = tokenizer.apply_chat_template(\n", - " messages, tokenize=False, add_generation_prompt=True, return_dict=False\n", - ")\n", - "response = requests.post(\n", - " f\"http://localhost:{port}/generate\",\n", - " json={\n", - " \"text\": text,\n", - " \"require_reasoning\": True,\n", - " \"sampling_params\": {\n", - " \"temperature\": 0,\n", - " \"max_new_tokens\": 2048,\n", - " \"json_schema\": json_schema,\n", - " },\n", - " },\n", - ")\n", - "\n", - "print_highlight(response.json())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### EBNF" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "response = requests.post(\n", - " f\"http://localhost:{port}/generate\",\n", - " json={\n", - " \"text\": \"Give me the information of the capital of France.\",\n", - " \"require_reasoning\": True,\n", - " \"sampling_params\": {\n", - " \"max_new_tokens\": 2048,\n", - " \"temperature\": 0,\n", - " \"n\": 3,\n", - " \"ebnf\": (\n", - " \"root ::= city | description\\n\"\n", - " 'city ::= \"London\" | \"Paris\" | \"Berlin\" | \"Rome\"\\n'\n", - " 'description ::= city \" is \" status\\n'\n", - " 'status ::= \"the capital of \" country\\n'\n", - " 'country ::= \"England\" | \"France\" | \"Germany\" | \"Italy\"'\n", - " ),\n", - " },\n", - " \"stream\": False,\n", - " \"return_logprob\": False,\n", - " },\n", - ")\n", - "\n", - "print(response.json())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Regular expression" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "response = requests.post(\n", - " f\"http://localhost:{port}/generate\",\n", - " json={\n", - " \"text\": \"Paris is the capital of\",\n", - " \"require_reasoning\": True,\n", - " \"sampling_params\": {\n", - " \"temperature\": 0,\n", - " \"max_new_tokens\": 2048,\n", - " \"regex\": \"(France|England)\",\n", - " },\n", - " },\n", - ")\n", - "print(response.json())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Structural Tag" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "text = tokenizer.apply_chat_template(\n", - " messages, tokenize=False, add_generation_prompt=True, return_dict=False\n", - ")\n", - "payload = {\n", - " \"text\": text,\n", - " \"require_reasoning\": True,\n", - " \"sampling_params\": {\n", - " \"max_new_tokens\": 2048,\n", - " \"structural_tag\": json.dumps(\n", - " {\n", - " \"type\": \"structural_tag\",\n", - " \"structures\": [\n", - " {\n", - " \"begin\": \"\",\n", - " \"schema\": schema_get_current_weather,\n", - " \"end\": \"\",\n", - " },\n", - " {\n", - " \"begin\": \"\",\n", - " \"schema\": schema_get_current_date,\n", - " \"end\": \"\",\n", - " },\n", - " ],\n", - " \"triggers\": [\"\",\n", - " \"schema\": schema_get_current_weather,\n", - " \"end\": \"\",\n", - " },\n", - " {\n", - " \"begin\": \"\",\n", - " \"schema\": schema_get_current_date,\n", - " \"end\": \"\",\n", - " },\n", - " ],\n", - " \"triggers\": [\"[{\"tool\": {...}}]<\\|tools_suffix\\|>`. |\n", - "| `deepseekv3` | DeepSeek-v3 (e.g., `deepseek-ai/DeepSeek-V3-0324`) | Recommend adding `--chat-template ./examples/chat_template/tool_chat_template_deepseekv3.jinja` to launch command. |\n", - "| `deepseekv31` | DeepSeek-V3.1 and DeepSeek-V3.2-Exp (e.g. `deepseek-ai/DeepSeek-V3.1`, `deepseek-ai/DeepSeek-V3.2-Exp`) | Recommend adding `--chat-template ./examples/chat_template/tool_chat_template_deepseekv31.jinja` (Or ..deepseekv32.jinja for DeepSeek-V3.2) to launch command. |\n", - "| `deepseekv32` | DeepSeek-V3.2 (`deepseek-ai/DeepSeek-V3.2`) | |\n", - "| `glm` | GLM series (e.g. `zai-org/GLM-4.6`) | |\n", - "| `gpt-oss` | GPT-OSS (e.g., `openai/gpt-oss-120b`, `openai/gpt-oss-20b`, `lmsys/gpt-oss-120b-bf16`, `lmsys/gpt-oss-20b-bf16`) | The gpt-oss tool parser filters out analysis channel events and only preserves normal text. This can cause the content to be empty when explanations are in the analysis channel. To work around this, complete the tool round by returning tool results as `role=\"tool\"` messages, which enables the model to generate the final content. |\n", - "| `kimi_k2` | `moonshotai/Kimi-K2-Instruct` | |\n", - "| `llama3` | Llama 3.1 / 3.2 / 3.3 (e.g. `meta-llama/Llama-3.1-8B-Instruct`, `meta-llama/Llama-3.2-1B-Instruct`, `meta-llama/Llama-3.3-70B-Instruct`) | |\n", - "| `llama4` | Llama 4 (e.g. `meta-llama/Llama-4-Scout-17B-16E-Instruct`) | |\n", - "| `mistral` | Mistral (e.g. `mistralai/Mistral-7B-Instruct-v0.3`, `mistralai/Mistral-Nemo-Instruct-2407`, `mistralai/Mistral-7B-v0.3`) | |\n", - "| `pythonic` | Llama-3.2 / Llama-3.3 / Llama-4 | Model outputs function calls as Python code. Requires `--tool-call-parser pythonic` and is recommended to use with a specific chat template. |\n", - "| `qwen` | Qwen series (e.g. `Qwen/Qwen3-Next-80B-A3B-Instruct`, `Qwen/Qwen3-VL-30B-A3B-Thinking`) except Qwen3-Coder| |\n", - "| `qwen3_coder` | Qwen3-Coder (e.g. `Qwen/Qwen3-Coder-30B-A3B-Instruct`) | |\n", - "| `step3` | Step-3 | |\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## OpenAI Compatible API" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Launching the Server" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import json\n", - "from sglang.test.doc_patch import launch_server_cmd\n", - "from sglang.utils import wait_for_server, print_highlight, terminate_process\n", - "from openai import OpenAI\n", - "\n", - "server_process, port = launch_server_cmd(\n", - " \"python3 -m sglang.launch_server --model-path Qwen/Qwen2.5-7B-Instruct --tool-call-parser qwen25 --host 0.0.0.0 --log-level warning\" # qwen25\n", - ")\n", - "wait_for_server(f\"http://localhost:{port}\", process=server_process)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Note that `--tool-call-parser` defines the parser used to interpret responses." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Define Tools for Function Call\n", - "Below is a Python snippet that shows how to define a tool as a dictionary. The dictionary includes a tool name, a description, and property defined Parameters." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Define tools\n", - "tools = [\n", - " {\n", - " \"type\": \"function\",\n", - " \"function\": {\n", - " \"name\": \"get_current_weather\",\n", - " \"description\": \"Get the current weather in a given location\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"city\": {\n", - " \"type\": \"string\",\n", - " \"description\": \"The city to find the weather for, e.g. 'San Francisco'\",\n", - " },\n", - " \"state\": {\n", - " \"type\": \"string\",\n", - " \"description\": \"the two-letter abbreviation for the state that the city is\"\n", - " \" in, e.g. 'CA' which would mean 'California'\",\n", - " },\n", - " \"unit\": {\n", - " \"type\": \"string\",\n", - " \"description\": \"The unit to fetch the temperature in\",\n", - " \"enum\": [\"celsius\", \"fahrenheit\"],\n", - " },\n", - " },\n", - " \"required\": [\"city\", \"state\", \"unit\"],\n", - " },\n", - " },\n", - " }\n", - "]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Define Messages" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def get_messages():\n", - " return [\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"What's the weather like in Boston today? Output a reasoning before act, then use the tools to help you.\",\n", - " }\n", - " ]\n", - "\n", - "\n", - "messages = get_messages()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Initialize the Client" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Initialize OpenAI-like client\n", - "client = OpenAI(api_key=\"None\", base_url=f\"http://0.0.0.0:{port}/v1\")\n", - "model_name = client.models.list().data[0].id" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Non-Streaming Request" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Non-streaming mode test\n", - "response_non_stream = client.chat.completions.create(\n", - " model=model_name,\n", - " messages=messages,\n", - " temperature=0,\n", - " top_p=0.95,\n", - " max_tokens=1024,\n", - " stream=False, # Non-streaming\n", - " tools=tools,\n", - ")\n", - "print_highlight(\"Non-stream response:\")\n", - "print_highlight(response_non_stream)\n", - "print_highlight(\"==== content ====\")\n", - "print_highlight(response_non_stream.choices[0].message.content)\n", - "print_highlight(\"==== tool_calls ====\")\n", - "print_highlight(response_non_stream.choices[0].message.tool_calls)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Handle Tools\n", - "When the engine determines it should call a particular tool, it will return arguments or partial arguments through the response. You can parse these arguments and later invoke the tool accordingly." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "name_non_stream = response_non_stream.choices[0].message.tool_calls[0].function.name\n", - "arguments_non_stream = (\n", - " response_non_stream.choices[0].message.tool_calls[0].function.arguments\n", - ")\n", - "\n", - "print_highlight(f\"Final streamed function call name: {name_non_stream}\")\n", - "print_highlight(f\"Final streamed function call arguments: {arguments_non_stream}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Streaming Request" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Streaming mode test\n", - "print_highlight(\"Streaming response:\")\n", - "response_stream = client.chat.completions.create(\n", - " model=model_name,\n", - " messages=messages,\n", - " temperature=0,\n", - " top_p=0.95,\n", - " max_tokens=1024,\n", - " stream=True, # Enable streaming\n", - " tools=tools,\n", - ")\n", - "\n", - "texts = \"\"\n", - "tool_calls = []\n", - "name = \"\"\n", - "arguments = \"\"\n", - "for chunk in response_stream:\n", - " if chunk.choices[0].delta.content:\n", - " texts += chunk.choices[0].delta.content\n", - " if chunk.choices[0].delta.tool_calls:\n", - " tool_calls.append(chunk.choices[0].delta.tool_calls[0])\n", - "print_highlight(\"==== Text ====\")\n", - "print_highlight(texts)\n", - "\n", - "print_highlight(\"==== Tool Call ====\")\n", - "for tool_call in tool_calls:\n", - " print_highlight(tool_call)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Handle Tools\n", - "When the engine determines it should call a particular tool, it will return arguments or partial arguments through the response. You can parse these arguments and later invoke the tool accordingly." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Parse and combine function call arguments\n", - "arguments = []\n", - "for tool_call in tool_calls:\n", - " if tool_call.function.name:\n", - " print_highlight(f\"Streamed function call name: {tool_call.function.name}\")\n", - "\n", - " if tool_call.function.arguments:\n", - " arguments.append(tool_call.function.arguments)\n", - "\n", - "# Combine all fragments into a single JSON string\n", - "full_arguments = \"\".join(arguments)\n", - "print_highlight(f\"streamed function call arguments: {full_arguments}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Define a Tool Function" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# This is a demonstration, define real function according to your usage.\n", - "def get_current_weather(city: str, state: str, unit: \"str\"):\n", - " return (\n", - " f\"The weather in {city}, {state} is 85 degrees {unit}. It is \"\n", - " \"partly cloudly, with highs in the 90's.\"\n", - " )\n", - "\n", - "\n", - "available_tools = {\"get_current_weather\": get_current_weather}" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "### Execute the Tool" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "messages.append(response_non_stream.choices[0].message)\n", - "\n", - "# Call the corresponding tool function\n", - "tool_call = messages[-1].tool_calls[0]\n", - "tool_name = tool_call.function.name\n", - "tool_to_call = available_tools[tool_name]\n", - "result = tool_to_call(**(json.loads(tool_call.function.arguments)))\n", - "print_highlight(f\"Function call result: {result}\")\n", - "# messages.append({\"role\": \"tool\", \"content\": result, \"name\": tool_name})\n", - "messages.append(\n", - " {\n", - " \"role\": \"tool\",\n", - " \"tool_call_id\": tool_call.id,\n", - " \"content\": str(result),\n", - " \"name\": tool_name,\n", - " }\n", - ")\n", - "\n", - "print_highlight(f\"Updated message history: {messages}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Send Results Back to Model" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "final_response = client.chat.completions.create(\n", - " model=model_name,\n", - " messages=messages,\n", - " temperature=0,\n", - " top_p=0.95,\n", - " stream=False,\n", - " tools=tools,\n", - ")\n", - "print_highlight(\"Non-stream response:\")\n", - "print_highlight(final_response)\n", - "\n", - "print_highlight(\"==== Text ====\")\n", - "print_highlight(final_response.choices[0].message.content)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Native API and SGLang Runtime (SRT)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from transformers import AutoTokenizer\n", - "import requests\n", - "\n", - "# generate an answer\n", - "tokenizer = AutoTokenizer.from_pretrained(\"Qwen/Qwen2.5-7B-Instruct\")\n", - "\n", - "messages = get_messages()\n", - "\n", - "input = tokenizer.apply_chat_template(\n", - " messages, tokenize=False, add_generation_prompt=True, tools=tools, return_dict=False\n", - ")\n", - "\n", - "gen_url = f\"http://localhost:{port}/generate\"\n", - "gen_data = {\n", - " \"text\": input,\n", - " \"sampling_params\": {\n", - " \"skip_special_tokens\": False,\n", - " \"max_new_tokens\": 1024,\n", - " \"temperature\": 0,\n", - " \"top_p\": 0.95,\n", - " },\n", - "}\n", - "gen_response = requests.post(gen_url, json=gen_data).json()[\"text\"]\n", - "print_highlight(\"==== Response ====\")\n", - "print_highlight(gen_response)\n", - "\n", - "# parse the response\n", - "parse_url = f\"http://localhost:{port}/parse_function_call\"\n", - "\n", - "function_call_input = {\n", - " \"text\": gen_response,\n", - " \"tool_call_parser\": \"qwen25\",\n", - " \"tools\": tools,\n", - "}\n", - "\n", - "function_call_response = requests.post(parse_url, json=function_call_input)\n", - "function_call_response_json = function_call_response.json()\n", - "\n", - "print_highlight(\"==== Text ====\")\n", - "print(function_call_response_json[\"normal_text\"])\n", - "print_highlight(\"==== Calls ====\")\n", - "print(\"function name: \", function_call_response_json[\"calls\"][0][\"name\"])\n", - "print(\"function arguments: \", function_call_response_json[\"calls\"][0][\"parameters\"])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "terminate_process(server_process)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Offline Engine API" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import sglang as sgl\n", - "from sglang.srt.function_call.function_call_parser import FunctionCallParser\n", - "from sglang.srt.managers.io_struct import Tool, Function\n", - "\n", - "llm = sgl.Engine(model_path=\"Qwen/Qwen2.5-7B-Instruct\")\n", - "tokenizer = llm.tokenizer_manager.tokenizer\n", - "input_ids = tokenizer.apply_chat_template(\n", - " messages, tokenize=True, add_generation_prompt=True, tools=tools, return_dict=False\n", - ")\n", - "\n", - "# Note that for gpt-oss tool parser, adding \"no_stop_trim\": True\n", - "# to make sure the tool call token is not trimmed.\n", - "\n", - "sampling_params = {\n", - " \"max_new_tokens\": 1024,\n", - " \"temperature\": 0,\n", - " \"top_p\": 0.95,\n", - " \"skip_special_tokens\": False,\n", - "}\n", - "\n", - "# 1) Offline generation\n", - "result = llm.generate(input_ids=input_ids, sampling_params=sampling_params)\n", - "generated_text = result[\"text\"] # Assume there is only one prompt\n", - "\n", - "print_highlight(\"=== Offline Engine Output Text ===\")\n", - "print_highlight(generated_text)\n", - "\n", - "\n", - "# 2) Parse using FunctionCallParser\n", - "def convert_dict_to_tool(tool_dict: dict) -> Tool:\n", - " function_dict = tool_dict.get(\"function\", {})\n", - " return Tool(\n", - " type=tool_dict.get(\"type\", \"function\"),\n", - " function=Function(\n", - " name=function_dict.get(\"name\"),\n", - " description=function_dict.get(\"description\"),\n", - " parameters=function_dict.get(\"parameters\"),\n", - " ),\n", - " )\n", - "\n", - "\n", - "tools = [convert_dict_to_tool(raw_tool) for raw_tool in tools]\n", - "\n", - "parser = FunctionCallParser(tools=tools, tool_call_parser=\"qwen25\")\n", - "normal_text, calls = parser.parse_non_stream(generated_text)\n", - "\n", - "print_highlight(\"=== Parsing Result ===\")\n", - "print(\"Normal text portion:\", normal_text)\n", - "print_highlight(\"Function call portion:\")\n", - "for call in calls:\n", - " # call: ToolCallItem\n", - " print_highlight(f\" - tool name: {call.name}\")\n", - " print_highlight(f\" parameters: {call.parameters}\")\n", - "\n", - "# 3) If needed, perform additional logic on the parsed functions, such as automatically calling the corresponding function to obtain a return value, etc." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "llm.shutdown()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Tool Choice Mode\n", - "\n", - "SGLang supports OpenAI's `tool_choice` parameter to control when and which tools the model should call. This feature is implemented using EBNF (Extended Backus-Naur Form) grammar to ensure reliable tool calling behavior.\n", - "\n", - "### Supported Tool Choice Options\n", - "\n", - "- **`tool_choice=\"required\"`**: Forces the model to call at least one tool\n", - "- **`tool_choice={\"type\": \"function\", \"function\": {\"name\": \"specific_function\"}}`**: Forces the model to call a specific function\n", - "\n", - "### Backend Compatibility\n", - "\n", - "Tool choice is fully supported with the **Xgrammar backend**, which is the default grammar backend (`--grammar-backend xgrammar`). However, it may not be fully supported with other backends such as `outlines`.\n", - "\n", - "### Example: Required Tool Choice" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from openai import OpenAI\n", - "from sglang.utils import wait_for_server, print_highlight, terminate_process\n", - "from sglang.test.doc_patch import launch_server_cmd\n", - "\n", - "# Start a new server session for tool choice examples\n", - "server_process_tool_choice, port_tool_choice = launch_server_cmd(\n", - " \"python3 -m sglang.launch_server --model-path Qwen/Qwen2.5-7B-Instruct --tool-call-parser qwen25 --host 0.0.0.0 --log-level warning\"\n", - ")\n", - "wait_for_server(\n", - " f\"http://localhost:{port_tool_choice}\", process=server_process_tool_choice\n", - ")\n", - "\n", - "# Initialize client for tool choice examples\n", - "client_tool_choice = OpenAI(\n", - " api_key=\"None\", base_url=f\"http://0.0.0.0:{port_tool_choice}/v1\"\n", - ")\n", - "model_name_tool_choice = client_tool_choice.models.list().data[0].id\n", - "\n", - "# Example with tool_choice=\"required\" - forces the model to call a tool\n", - "messages_required = [\n", - " {\"role\": \"user\", \"content\": \"Hello, what is the capital of France?\"}\n", - "]\n", - "\n", - "# Define tools\n", - "tools = [\n", - " {\n", - " \"type\": \"function\",\n", - " \"function\": {\n", - " \"name\": \"get_current_weather\",\n", - " \"description\": \"Get the current weather in a given location\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"city\": {\n", - " \"type\": \"string\",\n", - " \"description\": \"The city to find the weather for, e.g. 'San Francisco'\",\n", - " },\n", - " \"unit\": {\n", - " \"type\": \"string\",\n", - " \"description\": \"The unit to fetch the temperature in\",\n", - " \"enum\": [\"celsius\", \"fahrenheit\"],\n", - " },\n", - " },\n", - " \"required\": [\"city\", \"unit\"],\n", - " },\n", - " },\n", - " }\n", - "]\n", - "\n", - "response_required = client_tool_choice.chat.completions.create(\n", - " model=model_name_tool_choice,\n", - " messages=messages_required,\n", - " temperature=0,\n", - " max_tokens=1024,\n", - " tools=tools,\n", - " tool_choice=\"required\", # Force the model to call a tool\n", - ")\n", - "\n", - "print_highlight(\"Response with tool_choice='required':\")\n", - "print(\"Content:\", response_required.choices[0].message.content)\n", - "print(\"Tool calls:\", response_required.choices[0].message.tool_calls)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Example: Specific Function Choice\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Example with specific function choice - forces the model to call a specific function\n", - "messages_specific = [\n", - " {\"role\": \"user\", \"content\": \"What are the most attactive places in France?\"}\n", - "]\n", - "\n", - "response_specific = client_tool_choice.chat.completions.create(\n", - " model=model_name_tool_choice,\n", - " messages=messages_specific,\n", - " temperature=0,\n", - " max_tokens=1024,\n", - " tools=tools,\n", - " tool_choice={\n", - " \"type\": \"function\",\n", - " \"function\": {\"name\": \"get_current_weather\"},\n", - " }, # Force the model to call the specific get_current_weather function\n", - ")\n", - "\n", - "print_highlight(\"Response with specific function choice:\")\n", - "print(\"Content:\", response_specific.choices[0].message.content)\n", - "print(\"Tool calls:\", response_specific.choices[0].message.tool_calls)\n", - "\n", - "if response_specific.choices[0].message.tool_calls:\n", - " tool_call = response_specific.choices[0].message.tool_calls[0]\n", - " print_highlight(f\"Called function: {tool_call.function.name}\")\n", - " print_highlight(f\"Arguments: {tool_call.function.arguments}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "terminate_process(server_process_tool_choice)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Pythonic Tool Call Format (Llama-3.2 / Llama-3.3 / Llama-4)\n", - "\n", - "Some Llama models (such as Llama-3.2-1B, Llama-3.2-3B, Llama-3.3-70B, and Llama-4) support a \"pythonic\" tool call format, where the model outputs function calls as Python code, e.g.:\n", - "\n", - "```python\n", - "[get_current_weather(city=\"San Francisco\", state=\"CA\", unit=\"celsius\")]\n", - "```\n", - "\n", - "- The output is a Python list of function calls, with arguments as Python literals (not JSON).\n", - "- Multiple tool calls can be returned in the same list:\n", - "```python\n", - "[get_current_weather(city=\"San Francisco\", state=\"CA\", unit=\"celsius\"),\n", - " get_current_weather(city=\"New York\", state=\"NY\", unit=\"fahrenheit\")]\n", - "```\n", - "\n", - "For more information, refer to Meta’s documentation on [Zero shot function calling](https://github.com/meta-llama/llama-models/blob/main/models/llama4/prompt_format.md#zero-shot-function-calling---system-message).\n", - "\n", - "Note that this feature is still under development on Blackwell.\n", - "\n", - "### How to enable\n", - "- Launch the server with `--tool-call-parser pythonic`\n", - "- You may also specify --chat-template with the improved template for the model (e.g., `--chat-template=examples/chat_template/tool_chat_template_llama4_pythonic.jinja`).\n", - "This is recommended because the model expects a special prompt format to reliably produce valid pythonic tool call outputs. The template ensures that the prompt structure (e.g., special tokens, message boundaries like `<|eom|>`, and function call delimiters) matches what the model was trained or fine-tuned on. If you do not use the correct chat template, tool calling may fail or produce inconsistent results.\n", - "\n", - "#### Forcing Pythonic Tool Call Output Without a Chat Template\n", - "If you don't want to specify a chat template, you must give the model extremely explicit instructions in your messages to enforce pythonic output. For example, for `Llama-3.2-1B-Instruct`, you need:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import openai\n", - "\n", - "server_process, port = launch_server_cmd(\n", - " \" python3 -m sglang.launch_server --model-path meta-llama/Llama-3.2-1B-Instruct --tool-call-parser pythonic --tp 1 --log-level warning\" # llama-3.2-1b-instruct\n", - ")\n", - "wait_for_server(f\"http://localhost:{port}\", process=server_process)\n", - "\n", - "tools = [\n", - " {\n", - " \"type\": \"function\",\n", - " \"function\": {\n", - " \"name\": \"get_weather\",\n", - " \"description\": \"Get the current weather for a given location.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"location\": {\n", - " \"type\": \"string\",\n", - " \"description\": \"The name of the city or location.\",\n", - " }\n", - " },\n", - " \"required\": [\"location\"],\n", - " },\n", - " },\n", - " },\n", - " {\n", - " \"type\": \"function\",\n", - " \"function\": {\n", - " \"name\": \"get_tourist_attractions\",\n", - " \"description\": \"Get a list of top tourist attractions for a given city.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"city\": {\n", - " \"type\": \"string\",\n", - " \"description\": \"The name of the city to find attractions for.\",\n", - " }\n", - " },\n", - " \"required\": [\"city\"],\n", - " },\n", - " },\n", - " },\n", - "]\n", - "\n", - "\n", - "def get_messages():\n", - " return [\n", - " {\n", - " \"role\": \"system\",\n", - " \"content\": (\n", - " \"You are a travel assistant. \"\n", - " \"When asked to call functions, ALWAYS respond ONLY with a python list of function calls, \"\n", - " \"using this format: [func_name1(param1=value1, param2=value2), func_name2(param=value)]. \"\n", - " \"Do NOT use JSON, do NOT use variables, do NOT use any other format. \"\n", - " \"Here is an example:\\n\"\n", - " '[get_weather(location=\"Paris\"), get_tourist_attractions(city=\"Paris\")]'\n", - " ),\n", - " },\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": (\n", - " \"I'm planning a trip to Tokyo next week. What's the weather like and what are some top tourist attractions? \"\n", - " \"Propose parallel tool calls at once, using the python list of function calls format as shown above.\"\n", - " ),\n", - " },\n", - " ]\n", - "\n", - "\n", - "messages = get_messages()\n", - "\n", - "client = openai.Client(base_url=f\"http://localhost:{port}/v1\", api_key=\"xxxxxx\")\n", - "model_name = client.models.list().data[0].id\n", - "\n", - "\n", - "response_non_stream = client.chat.completions.create(\n", - " model=model_name,\n", - " messages=messages,\n", - " temperature=0,\n", - " top_p=0.9,\n", - " stream=False, # Non-streaming\n", - " tools=tools,\n", - ")\n", - "print_highlight(\"Non-stream response:\")\n", - "print_highlight(response_non_stream)\n", - "\n", - "response_stream = client.chat.completions.create(\n", - " model=model_name,\n", - " messages=messages,\n", - " temperature=0,\n", - " top_p=0.9,\n", - " stream=True,\n", - " tools=tools,\n", - ")\n", - "texts = \"\"\n", - "tool_calls = []\n", - "name = \"\"\n", - "arguments = \"\"\n", - "\n", - "for chunk in response_stream:\n", - " if chunk.choices[0].delta.content:\n", - " texts += chunk.choices[0].delta.content\n", - " if chunk.choices[0].delta.tool_calls:\n", - " tool_calls.append(chunk.choices[0].delta.tool_calls[0])\n", - "\n", - "print_highlight(\"Streaming Response:\")\n", - "print_highlight(\"==== Text ====\")\n", - "print_highlight(texts)\n", - "\n", - "print_highlight(\"==== Tool Call ====\")\n", - "for tool_call in tool_calls:\n", - " print_highlight(tool_call)\n", - "\n", - "terminate_process(server_process)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "> **Note:** \n", - "> The model may still default to JSON if it was heavily finetuned on that format. Prompt engineering (including examples) is the only way to increase the chance of pythonic output if you are not using a chat template." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## How to support a new model?\n", - "1. Update the TOOLS_TAG_LIST in sglang/srt/function_call_parser.py with the model’s tool tags. Currently supported tags include:\n", - "```\n", - "\tTOOLS_TAG_LIST = [\n", - "\t “<|plugin|>“,\n", - "\t ““,\n", - "\t “<|python_tag|>“,\n", - "\t “[TOOL_CALLS]”\n", - "\t]\n", - "```\n", - "2. Create a new detector class in sglang/srt/function_call_parser.py that inherits from BaseFormatDetector. The detector should handle the model’s specific function call format. For example:\n", - "```\n", - " class NewModelDetector(BaseFormatDetector):\n", - "```\n", - "3. Add the new detector to the MultiFormatParser class that manages all the format detectors." - ] - } - ], - "metadata": { - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/docs_new/docs/advanced_features/vlm_query.ipynb b/docs_new/docs/advanced_features/vlm_query.ipynb deleted file mode 100644 index 24bd7a90b..000000000 --- a/docs_new/docs/advanced_features/vlm_query.ipynb +++ /dev/null @@ -1,379 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "0", - "metadata": {}, - "source": [ - "# Query VLM with Offline Engine\n", - "\n", - "This tutorial demonstrates how to use SGLang's **offline Engine API** to query VLMs. We will demonstrate usage with Qwen2.5-VL and Llama 4. This section demonstrates three different calling approaches:\n", - "\n", - "1. **Basic Call**: Directly pass images and text.\n", - "2. **Processor Output**: Use HuggingFace processor for data preprocessing.\n", - "3. **Precomputed Embeddings**: Pre-calculate image features to improve inference efficiency." - ] - }, - { - "cell_type": "markdown", - "id": "1", - "metadata": {}, - "source": [ - "## Understanding the Three Input Formats\n", - "\n", - "SGLang supports three ways to pass visual data, each optimized for different scenarios:\n", - "\n", - "### 1. **Raw Images** - Simplest approach\n", - "- Pass PIL Images, file paths, URLs, or base64 strings directly\n", - "- SGLang handles all preprocessing automatically\n", - "- Best for: Quick prototyping, simple applications\n", - "\n", - "### 2. **Processor Output** - For custom preprocessing\n", - "- Pre-process images with HuggingFace processor\n", - "- Pass the complete processor output dict with `format: \"processor_output\"`\n", - "- Best for: Custom image transformations, integration with existing pipelines\n", - "- Requirement: Must use `input_ids` instead of text prompt\n", - "\n", - "### 3. **Precomputed Embeddings** - For maximum performance\n", - "- Pre-calculate visual embeddings using the vision encoder\n", - "- Pass embeddings with `format: \"precomputed_embedding\"`\n", - "- Best for: Repeated queries on same images, caching, high-throughput serving\n", - "- Performance gain: Avoids redundant vision encoder computation (30-50% speedup)\n", - "\n", - "**Key Rule**: Within a single request, use only one format for all images. Don't mix formats.\n", - "\n", - "The examples below demonstrate all three approaches with both Qwen2.5-VL and Llama 4 models." - ] - }, - { - "cell_type": "markdown", - "id": "2", - "metadata": {}, - "source": [ - "## Querying Qwen2.5-VL Model" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3", - "metadata": {}, - "outputs": [], - "source": [ - "import nest_asyncio\n", - "\n", - "nest_asyncio.apply()\n", - "\n", - "import sglang.test.doc_patch # noqa: F401\n", - "\n", - "model_path = \"Qwen/Qwen2.5-VL-3B-Instruct\"\n", - "chat_template = \"qwen2-vl\"\n", - "example_image_url = \"https://raw.githubusercontent.com/sgl-project/sglang/main/examples/assets/example_image.png\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4", - "metadata": {}, - "outputs": [], - "source": [ - "from io import BytesIO\n", - "import requests\n", - "from PIL import Image\n", - "\n", - "from sglang.srt.parser.conversation import chat_templates\n", - "\n", - "image = Image.open(BytesIO(requests.get(example_image_url).content))\n", - "\n", - "conv = chat_templates[chat_template].copy()\n", - "conv.append_message(conv.roles[0], f\"What's shown here: {conv.image_token}?\")\n", - "conv.append_message(conv.roles[1], \"\")\n", - "conv.image_data = [image]\n", - "\n", - "print(\"Generated prompt text:\")\n", - "print(conv.get_prompt())\n", - "print(f\"\\nImage size: {image.size}\")\n", - "image" - ] - }, - { - "cell_type": "markdown", - "id": "5", - "metadata": {}, - "source": [ - "### Basic Offline Engine API Call" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6", - "metadata": {}, - "outputs": [], - "source": [ - "from sglang import Engine\n", - "\n", - "llm = Engine(model_path=model_path, chat_template=chat_template, log_level=\"warning\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7", - "metadata": {}, - "outputs": [], - "source": [ - "out = llm.generate(prompt=conv.get_prompt(), image_data=[image])\n", - "print(\"Model response:\")\n", - "print(out[\"text\"])" - ] - }, - { - "cell_type": "markdown", - "id": "8", - "metadata": {}, - "source": [ - "### Call with Processor Output\n", - "\n", - "Using a HuggingFace processor to preprocess text and images, and passing the `processor_output` directly into `Engine.generate`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9", - "metadata": {}, - "outputs": [], - "source": [ - "from transformers import AutoProcessor\n", - "\n", - "processor = AutoProcessor.from_pretrained(model_path, use_fast=True)\n", - "processor_output = processor(\n", - " images=[image], text=conv.get_prompt(), return_tensors=\"pt\"\n", - ")\n", - "\n", - "out = llm.generate(\n", - " input_ids=processor_output[\"input_ids\"][0].detach().cpu().tolist(),\n", - " image_data=[dict(processor_output, format=\"processor_output\")],\n", - ")\n", - "print(\"Response using processor output:\")\n", - "print(out[\"text\"])" - ] - }, - { - "cell_type": "markdown", - "id": "10", - "metadata": {}, - "source": [ - "### Call with Precomputed Embeddings\n", - "\n", - "You can pre-calculate image features to avoid repeated visual encoding processes." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "11", - "metadata": {}, - "outputs": [], - "source": [ - "from transformers import AutoProcessor\n", - "from transformers import Qwen2_5_VLForConditionalGeneration\n", - "\n", - "processor = AutoProcessor.from_pretrained(model_path, use_fast=True)\n", - "model = Qwen2_5_VLForConditionalGeneration.from_pretrained(model_path).eval()\n", - "vision = model.model.visual.cuda()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "12", - "metadata": {}, - "outputs": [], - "source": [ - "processor_output = processor(\n", - " images=[image], text=conv.get_prompt(), return_tensors=\"pt\"\n", - ")\n", - "\n", - "input_ids = processor_output[\"input_ids\"][0].detach().cpu().tolist()\n", - "\n", - "precomputed_embeddings = vision(\n", - " processor_output[\"pixel_values\"].cuda(), processor_output[\"image_grid_thw\"].cuda()\n", - ")\n", - "precomputed_embeddings = precomputed_embeddings.pooler_output\n", - "\n", - "multi_modal_item = dict(\n", - " processor_output,\n", - " format=\"precomputed_embedding\",\n", - " feature=precomputed_embeddings,\n", - ")\n", - "\n", - "out = llm.generate(input_ids=input_ids, image_data=[multi_modal_item])\n", - "print(\"Response using precomputed embeddings:\")\n", - "print(out[\"text\"])\n", - "\n", - "llm.shutdown()" - ] - }, - { - "cell_type": "markdown", - "id": "13", - "metadata": {}, - "source": [ - "## Querying Llama 4 Vision Model\n", - "\n", - "```python\n", - "model_path = \"meta-llama/Llama-4-Scout-17B-16E-Instruct\"\n", - "chat_template = \"llama-4\"\n", - "\n", - "from io import BytesIO\n", - "import requests\n", - "from PIL import Image\n", - "\n", - "from sglang.srt.parser.conversation import chat_templates\n", - "\n", - "# Download the same example image\n", - "image = Image.open(BytesIO(requests.get(example_image_url).content))\n", - "\n", - "conv = chat_templates[chat_template].copy()\n", - "conv.append_message(conv.roles[0], f\"What's shown here: {conv.image_token}?\")\n", - "conv.append_message(conv.roles[1], \"\")\n", - "conv.image_data = [image]\n", - "\n", - "print(\"Llama 4 generated prompt text:\")\n", - "print(conv.get_prompt())\n", - "print(f\"Image size: {image.size}\")\n", - "\n", - "image\n", - "```" - ] - }, - { - "cell_type": "markdown", - "id": "14", - "metadata": {}, - "source": [ - "### Llama 4 Basic Call\n", - "\n", - "Llama 4 requires more computational resources, so it's configured with multi-GPU parallelism (tp_size=4) and larger context length.\n", - "\n", - "```python\n", - "llm = Engine(\n", - " model_path=model_path,\n", - " enable_multimodal=True,\n", - " attention_backend=\"fa3\",\n", - " tp_size=4,\n", - " context_length=65536,\n", - ")\n", - "\n", - "out = llm.generate(prompt=conv.get_prompt(), image_data=[image])\n", - "print(\"Llama 4 response:\")\n", - "print(out[\"text\"])\n", - "```" - ] - }, - { - "cell_type": "markdown", - "id": "15", - "metadata": {}, - "source": [ - "### Call with Processor Output\n", - "\n", - "Using HuggingFace processor to preprocess data can reduce computational overhead during inference.\n", - "\n", - "```python\n", - "from transformers import AutoProcessor\n", - "\n", - "processor = AutoProcessor.from_pretrained(model_path, use_fast=True)\n", - "processor_output = processor(\n", - " images=[image], text=conv.get_prompt(), return_tensors=\"pt\"\n", - ")\n", - "\n", - "out = llm.generate(\n", - " input_ids=processor_output[\"input_ids\"][0].detach().cpu().tolist(),\n", - " image_data=[dict(processor_output, format=\"processor_output\")],\n", - ")\n", - "print(\"Response using processor output:\")\n", - "print(out)\n", - "```" - ] - }, - { - "cell_type": "markdown", - "id": "16", - "metadata": {}, - "source": [ - "### Call with Precomputed Embeddings\n", - "\n", - "```python\n", - "from transformers import AutoProcessor\n", - "from transformers import Llama4ForConditionalGeneration\n", - "\n", - "processor = AutoProcessor.from_pretrained(model_path, use_fast=True)\n", - "model = Llama4ForConditionalGeneration.from_pretrained(\n", - " model_path, torch_dtype=\"auto\"\n", - ").eval()\n", - "\n", - "vision = model.vision_model.cuda()\n", - "multi_modal_projector = model.multi_modal_projector.cuda()\n", - "\n", - "print(f'Image pixel values shape: {processor_output[\"pixel_values\"].shape}')\n", - "input_ids = processor_output[\"input_ids\"][0].detach().cpu().tolist()\n", - "\n", - "# Process image through vision encoder\n", - "image_outputs = vision(\n", - " processor_output[\"pixel_values\"].to(\"cuda\"), \n", - " aspect_ratio_ids=processor_output[\"aspect_ratio_ids\"].to(\"cuda\"),\n", - " aspect_ratio_mask=processor_output[\"aspect_ratio_mask\"].to(\"cuda\"),\n", - " output_hidden_states=False\n", - ")\n", - "image_features = image_outputs.last_hidden_state\n", - "\n", - "# Flatten image features and pass through multimodal projector\n", - "vision_flat = image_features.view(-1, image_features.size(-1))\n", - "precomputed_embeddings = multi_modal_projector(vision_flat)\n", - "\n", - "# Build precomputed embedding data item\n", - "mm_item = dict(\n", - " processor_output, \n", - " format=\"precomputed_embedding\", \n", - " feature=precomputed_embeddings\n", - ")\n", - "\n", - "# Use precomputed embeddings for efficient inference\n", - "out = llm.generate(input_ids=input_ids, image_data=[mm_item])\n", - "print(\"Llama 4 precomputed embedding response:\")\n", - "print(out[\"text\"])\n", - "```" - ] - } - ], - "metadata": { - "jupytext": { - "cell_metadata_filter": "-all", - "custom_cell_magics": "kql", - "encoding": "# -*- coding: utf-8 -*-", - "text_representation": { - "extension": ".py", - "format_name": "light", - "format_version": "1.5", - "jupytext_version": "1.16.1" - } - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/docs_new/docs/basic_usage/native_api.ipynb b/docs_new/docs/basic_usage/native_api.ipynb deleted file mode 100644 index d3ead5e34..000000000 --- a/docs_new/docs/basic_usage/native_api.ipynb +++ /dev/null @@ -1,675 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# SGLang Native APIs\n", - "\n", - "Apart from the OpenAI compatible APIs, the SGLang Runtime also provides its native server APIs. We introduce the following APIs:\n", - "\n", - "- `/generate` (text generation model)\n", - "- `/get_model_info`\n", - "- `/server_info`\n", - "- `/health`\n", - "- `/health_generate`\n", - "- `/flush_cache`\n", - "- `/update_weights`\n", - "- `/encode`(embedding model)\n", - "- `/v1/rerank`(cross encoder rerank model)\n", - "- `/v1/score`(decoder-only scoring)\n", - "- `/classify`(reward model)\n", - "- `/start_expert_distribution_record`\n", - "- `/stop_expert_distribution_record`\n", - "- `/dump_expert_distribution_record`\n", - "- `/tokenize`\n", - "- `/detokenize`\n", - "- A full list of these APIs can be found at [http_server.py](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/entrypoints/http_server.py)\n", - "\n", - "We mainly use `requests` to test these APIs in the following examples. You can also use `curl`.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Launch A Server" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from sglang.test.doc_patch import launch_server_cmd\n", - "from sglang.utils import wait_for_server, print_highlight, terminate_process\n", - "\n", - "server_process, port = launch_server_cmd(\n", - " \"python3 -m sglang.launch_server --model-path qwen/qwen2.5-0.5b-instruct --host 0.0.0.0 --log-level warning\"\n", - ")\n", - "\n", - "wait_for_server(f\"http://localhost:{port}\", process=server_process)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Generate (text generation model)\n", - "Generate completions. This is similar to the `/v1/completions` in OpenAI API. Detailed parameters can be found in the [sampling parameters](sampling_params.md)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import requests\n", - "\n", - "url = f\"http://localhost:{port}/generate\"\n", - "data = {\"text\": \"What is the capital of France?\"}\n", - "\n", - "response = requests.post(url, json=data)\n", - "print_highlight(response.json())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Get Model Info\n", - "\n", - "Get the information of the model.\n", - "\n", - "- `model_path`: The path/name of the model.\n", - "- `is_generation`: Whether the model is used as generation model or embedding model.\n", - "- `tokenizer_path`: The path/name of the tokenizer.\n", - "- `preferred_sampling_params`: The default sampling params specified via `--preferred-sampling-params`. `None` is returned in this example as we did not explicitly configure it in server args.\n", - "- `weight_version`: This field contains the version of the model weights. This is often used to track changes or updates to the model’s trained parameters.\n", - "- `has_image_understanding`: Whether the model has image-understanding capability.\n", - "- `has_audio_understanding`: Whether the model has audio-understanding capability.\n", - "- `model_type`: The model type from the HuggingFace config (e.g., \"qwen2\", \"llama\").\n", - "- `architectures`: The model architectures from the HuggingFace config (e.g., [\"Qwen2ForCausalLM\"])." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "url = f\"http://localhost:{port}/get_model_info\"\n", - "\n", - "response = requests.get(url)\n", - "response_json = response.json()\n", - "print_highlight(response_json)\n", - "assert response_json[\"model_path\"] == \"qwen/qwen2.5-0.5b-instruct\"\n", - "assert response_json[\"is_generation\"] is True\n", - "assert response_json[\"tokenizer_path\"] == \"qwen/qwen2.5-0.5b-instruct\"\n", - "assert response_json[\"preferred_sampling_params\"] is None\n", - "assert response_json.keys() == {\n", - " \"model_path\",\n", - " \"is_generation\",\n", - " \"tokenizer_path\",\n", - " \"preferred_sampling_params\",\n", - " \"weight_version\",\n", - " \"has_image_understanding\",\n", - " \"has_audio_understanding\",\n", - " \"model_type\",\n", - " \"architectures\",\n", - "}" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Get Server Info\n", - "Gets the server information including CLI arguments, token limits, and memory pool sizes.\n", - "- Note: `get_server_info` merges the following deprecated endpoints:\n", - " - `get_server_args`\n", - " - `get_memory_pool_size`\n", - " - `get_max_total_num_tokens`" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "url = f\"http://localhost:{port}/server_info\"\n", - "\n", - "response = requests.get(url)\n", - "print_highlight(response.text)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Health Check\n", - "- `/health`: Check the health of the server.\n", - "- `/health_generate`: Check the health of the server by generating one token." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "url = f\"http://localhost:{port}/health_generate\"\n", - "\n", - "response = requests.get(url)\n", - "print_highlight(response.text)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "url = f\"http://localhost:{port}/health\"\n", - "\n", - "response = requests.get(url)\n", - "print_highlight(response.text)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Flush Cache\n", - "\n", - "Flush the radix cache. It will be automatically triggered when the model weights are updated by the `/update_weights` API.\n", - "\n", - "Parameters:\n", - "- `timeout` (query, float, default `0`, unit: seconds): Wait time for idle state before flushing. `0` means fail fast if not idle. When HiCache async operations are in-flight, a non-zero timeout allows the server to wait until idle before flushing, avoiding unnecessary 400 errors.\n", - "\n", - "```bash\n", - "# With timeout (wait up to 30s for idle state)\n", - "curl -s -X POST \"http://127.0.0.1:30000/flush_cache?timeout=30\"\n", - "```" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "url = f\"http://localhost:{port}/flush_cache\"\n", - "\n", - "response = requests.post(url)\n", - "print_highlight(response.text)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Update Weights From Disk\n", - "\n", - "Update model weights from disk without restarting the server. Only applicable for models with the same architecture and parameter size.\n", - "\n", - "SGLang support `update_weights_from_disk` API for continuous evaluation during training (save checkpoint to disk and update weights from disk).\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# successful update with same architecture and size\n", - "\n", - "url = f\"http://localhost:{port}/update_weights_from_disk\"\n", - "data = {\"model_path\": \"qwen/qwen2.5-0.5b-instruct\"}\n", - "\n", - "response = requests.post(url, json=data)\n", - "print_highlight(response.text)\n", - "assert response.json()[\"success\"] is True\n", - "assert response.json()[\"message\"] == \"Succeeded to update model weights.\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# failed update with different parameter size or wrong name\n", - "\n", - "url = f\"http://localhost:{port}/update_weights_from_disk\"\n", - "data = {\"model_path\": \"qwen/qwen2.5-0.5b-instruct-wrong\"}\n", - "\n", - "response = requests.post(url, json=data)\n", - "response_json = response.json()\n", - "print_highlight(response_json)\n", - "assert response_json[\"success\"] is False\n", - "assert response_json[\"message\"] == (\n", - " \"Failed to get weights iterator: \"\n", - " \"qwen/qwen2.5-0.5b-instruct-wrong\"\n", - " \" (repository not found).\"\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "terminate_process(server_process)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Encode (embedding model)\n", - "\n", - "Encode text into embeddings. Note that this API is only available for [embedding models](openai_api_embeddings.ipynb) and will raise an error for generation models.\n", - "Therefore, we launch a new server to server an embedding model." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "embedding_process, port = launch_server_cmd(\"\"\"\n", - "python3 -m sglang.launch_server --model-path Alibaba-NLP/gte-Qwen2-1.5B-instruct \\\n", - " --host 0.0.0.0 --is-embedding --log-level warning\n", - "\"\"\")\n", - "\n", - "wait_for_server(f\"http://localhost:{port}\", process=embedding_process)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# successful encode for embedding model\n", - "\n", - "url = f\"http://localhost:{port}/encode\"\n", - "data = {\"model\": \"Alibaba-NLP/gte-Qwen2-1.5B-instruct\", \"text\": \"Once upon a time\"}\n", - "\n", - "response = requests.post(url, json=data)\n", - "response_json = response.json()\n", - "print_highlight(f\"Text embedding (first 10): {response_json['embedding'][:10]}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "terminate_process(embedding_process)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## v1/rerank (cross encoder rerank model)\n", - "Rerank a list of documents given a query using a cross-encoder model. Note that this API is only available for cross encoder model like [BAAI/bge-reranker-v2-m3](https://huggingface.co/BAAI/bge-reranker-v2-m3) with `attention-backend` `triton` and `torch_native`.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "reranker_process, port = launch_server_cmd(\"\"\"\n", - "python3 -m sglang.launch_server --model-path BAAI/bge-reranker-v2-m3 \\\n", - " --host 0.0.0.0 --disable-radix-cache --chunked-prefill-size -1 --attention-backend triton --is-embedding --log-level warning\n", - "\"\"\")\n", - "\n", - "wait_for_server(f\"http://localhost:{port}\", process=reranker_process)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# compute rerank scores for query and documents\n", - "\n", - "url = f\"http://localhost:{port}/v1/rerank\"\n", - "data = {\n", - " \"model\": \"BAAI/bge-reranker-v2-m3\",\n", - " \"query\": \"what is panda?\",\n", - " \"documents\": [\n", - " \"hi\",\n", - " \"The giant panda (Ailuropoda melanoleuca), sometimes called a panda bear or simply panda, is a bear species endemic to China.\",\n", - " ],\n", - "}\n", - "\n", - "response = requests.post(url, json=data)\n", - "response_json = response.json()\n", - "for item in response_json:\n", - " print_highlight(f\"Score: {item['score']:.2f} - Document: '{item['document']}'\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "terminate_process(reranker_process)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## v1/score (decoder-only scoring)\n", - "\n", - "Compute token probabilities for specified tokens given a query and items. This is useful for classification tasks, scoring responses, or computing log-probabilities.\n", - "\n", - "Parameters:\n", - "- `query`: Query text\n", - "- `items`: Item text(s) to score\n", - "- `label_token_ids`: Token IDs to compute probabilities for\n", - "- `apply_softmax`: Whether to apply softmax to get normalized probabilities (default: False)\n", - "- `item_first`: Whether items come first in concatenation order (default: False)\n", - "- `model`: Model name\n", - "\n", - "The response contains `scores` - a list of probability lists, one per item, each in the order of `label_token_ids`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "score_process, port = launch_server_cmd(\"\"\"\n", - "python3 -m sglang.launch_server --model-path qwen/qwen2.5-0.5b-instruct \\\n", - " --host 0.0.0.0 --log-level warning\n", - "\"\"\")\n", - "\n", - "wait_for_server(f\"http://localhost:{port}\", process=score_process)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Score the probability of different completions given a query\n", - "query = \"The capital of France is\"\n", - "items = [\"Paris\", \"London\", \"Berlin\"]\n", - "\n", - "url = f\"http://localhost:{port}/v1/score\"\n", - "data = {\n", - " \"model\": \"qwen/qwen2.5-0.5b-instruct\",\n", - " \"query\": query,\n", - " \"items\": items,\n", - " \"label_token_ids\": [9454, 2753], # e.g. \"Yes\" and \"No\" token ids\n", - " \"apply_softmax\": True, # Normalize probabilities to sum to 1\n", - "}\n", - "\n", - "response = requests.post(url, json=data)\n", - "response_json = response.json()\n", - "\n", - "# Display scores for each item\n", - "for item, scores in zip(items, response_json[\"scores\"]):\n", - " print_highlight(f\"Item '{item}': probabilities = {[f'{s:.4f}' for s in scores]}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "terminate_process(score_process)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Classify (reward model)\n", - "\n", - "SGLang Runtime also supports reward models. Here we use a reward model to classify the quality of pairwise generations." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Note that SGLang now treats embedding models and reward models as the same type of models.\n", - "# This will be updated in the future.\n", - "\n", - "reward_process, port = launch_server_cmd(\"\"\"\n", - "python3 -m sglang.launch_server --model-path Skywork/Skywork-Reward-Llama-3.1-8B-v0.2 --host 0.0.0.0 --is-embedding --log-level warning\n", - "\"\"\")\n", - "\n", - "wait_for_server(f\"http://localhost:{port}\", process=reward_process)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from transformers import AutoTokenizer\n", - "\n", - "PROMPT = (\n", - " \"What is the range of the numeric output of a sigmoid node in a neural network?\"\n", - ")\n", - "\n", - "RESPONSE1 = \"The output of a sigmoid node is bounded between -1 and 1.\"\n", - "RESPONSE2 = \"The output of a sigmoid node is bounded between 0 and 1.\"\n", - "\n", - "CONVS = [\n", - " [{\"role\": \"user\", \"content\": PROMPT}, {\"role\": \"assistant\", \"content\": RESPONSE1}],\n", - " [{\"role\": \"user\", \"content\": PROMPT}, {\"role\": \"assistant\", \"content\": RESPONSE2}],\n", - "]\n", - "\n", - "tokenizer = AutoTokenizer.from_pretrained(\"Skywork/Skywork-Reward-Llama-3.1-8B-v0.2\")\n", - "prompts = tokenizer.apply_chat_template(CONVS, tokenize=False, return_dict=False)\n", - "\n", - "url = f\"http://localhost:{port}/classify\"\n", - "data = {\"model\": \"Skywork/Skywork-Reward-Llama-3.1-8B-v0.2\", \"text\": prompts}\n", - "\n", - "responses = requests.post(url, json=data).json()\n", - "for response in responses:\n", - " print_highlight(f\"reward: {response['embedding'][0]}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "terminate_process(reward_process)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Capture expert selection distribution in MoE models\n", - "\n", - "SGLang Runtime supports recording the number of times an expert is selected in a MoE model run for each expert in the model. This is useful when analyzing the throughput of the model and plan for optimization.\n", - "\n", - "*Note: We only print out the first 10 lines of the csv below for better readability. Please adjust accordingly if you want to analyze the results more deeply.*" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "expert_record_server_process, port = launch_server_cmd(\n", - " \"python3 -m sglang.launch_server --model-path Qwen/Qwen1.5-MoE-A2.7B --host 0.0.0.0 --expert-distribution-recorder-mode stat --log-level warning\"\n", - ")\n", - "\n", - "wait_for_server(f\"http://localhost:{port}\", process=expert_record_server_process)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "response = requests.post(f\"http://localhost:{port}/start_expert_distribution_record\")\n", - "print_highlight(response)\n", - "\n", - "url = f\"http://localhost:{port}/generate\"\n", - "data = {\"text\": \"What is the capital of France?\"}\n", - "\n", - "response = requests.post(url, json=data)\n", - "print_highlight(response.json())\n", - "\n", - "response = requests.post(f\"http://localhost:{port}/stop_expert_distribution_record\")\n", - "print_highlight(response)\n", - "\n", - "response = requests.post(f\"http://localhost:{port}/dump_expert_distribution_record\")\n", - "print_highlight(response)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "terminate_process(expert_record_server_process)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Tokenize/Detokenize Example (Round Trip)\n", - "\n", - "This example demonstrates how to use the /tokenize and /detokenize endpoints together. We first tokenize a string, then detokenize the resulting IDs to reconstruct the original text. This workflow is useful when you need to handle tokenization externally but still leverage the server for detokenization." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "tokenizer_free_server_process, port = launch_server_cmd(\"\"\"\n", - "python3 -m sglang.launch_server --model-path qwen/qwen2.5-0.5b-instruct\n", - "\"\"\")\n", - "\n", - "wait_for_server(f\"http://localhost:{port}\", process=tokenizer_free_server_process)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import requests\n", - "from sglang.utils import print_highlight\n", - "\n", - "base_url = f\"http://localhost:{port}\"\n", - "tokenize_url = f\"{base_url}/tokenize\"\n", - "detokenize_url = f\"{base_url}/detokenize\"\n", - "\n", - "model_name = \"qwen/qwen2.5-0.5b-instruct\"\n", - "input_text = \"SGLang provides efficient tokenization endpoints.\"\n", - "print_highlight(f\"Original Input Text:\\n'{input_text}'\")\n", - "\n", - "# --- tokenize the input text ---\n", - "tokenize_payload = {\n", - " \"model\": model_name,\n", - " \"prompt\": input_text,\n", - " \"add_special_tokens\": False,\n", - "}\n", - "try:\n", - " tokenize_response = requests.post(tokenize_url, json=tokenize_payload)\n", - " tokenize_response.raise_for_status()\n", - " tokenization_result = tokenize_response.json()\n", - " token_ids = tokenization_result.get(\"tokens\")\n", - "\n", - " if not token_ids:\n", - " raise ValueError(\"Tokenization returned empty tokens.\")\n", - "\n", - " print_highlight(f\"\\nTokenized Output (IDs):\\n{token_ids}\")\n", - " print_highlight(f\"Token Count: {tokenization_result.get('count')}\")\n", - " print_highlight(f\"Max Model Length: {tokenization_result.get('max_model_len')}\")\n", - "\n", - " # --- detokenize the obtained token IDs ---\n", - " detokenize_payload = {\n", - " \"model\": model_name,\n", - " \"tokens\": token_ids,\n", - " \"skip_special_tokens\": True,\n", - " }\n", - "\n", - " detokenize_response = requests.post(detokenize_url, json=detokenize_payload)\n", - " detokenize_response.raise_for_status()\n", - " detokenization_result = detokenize_response.json()\n", - " reconstructed_text = detokenization_result.get(\"text\")\n", - "\n", - " print_highlight(f\"\\nDetokenized Output (Text):\\n'{reconstructed_text}'\")\n", - "\n", - " if input_text == reconstructed_text:\n", - " print_highlight(\n", - " \"\\nRound Trip Successful: Original and reconstructed text match.\"\n", - " )\n", - " else:\n", - " print_highlight(\n", - " \"\\nRound Trip Mismatch: Original and reconstructed text differ.\"\n", - " )\n", - "\n", - "except requests.exceptions.RequestException as e:\n", - " print_highlight(f\"\\nHTTP Request Error: {e}\")\n", - "except Exception as e:\n", - " print_highlight(f\"\\nAn error occurred: {e}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "terminate_process(tokenizer_free_server_process)" - ] - } - ], - "metadata": { - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/docs_new/docs/basic_usage/offline_engine_api.ipynb b/docs_new/docs/basic_usage/offline_engine_api.ipynb deleted file mode 100644 index 963bd3f73..000000000 --- a/docs_new/docs/basic_usage/offline_engine_api.ipynb +++ /dev/null @@ -1,291 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Offline Engine API\n", - "\n", - "SGLang provides a direct inference engine without the need for an HTTP server, especially for use cases where additional HTTP server adds unnecessary complexity or overhead. Here are two general use cases:\n", - "\n", - "- Offline Batch Inference\n", - "- Custom Server on Top of the Engine\n", - "\n", - "This document focuses on the offline batch inference, demonstrating four different inference modes:\n", - "\n", - "- Non-streaming synchronous generation\n", - "- Streaming synchronous generation\n", - "- Non-streaming asynchronous generation\n", - "- Streaming asynchronous generation\n", - "\n", - "Additionally, you can easily build a custom server on top of the SGLang offline engine. A detailed example working in a python script can be found in [custom_server](https://github.com/sgl-project/sglang/blob/main/examples/runtime/engine/custom_server.py).\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Nest Asyncio\n", - "Note that if you want to use **Offline Engine** in ipython or some other nested loop code, you need to add the following code:\n", - "```python\n", - "import nest_asyncio\n", - "\n", - "nest_asyncio.apply()\n", - "\n", - "```" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Advanced Usage\n", - "\n", - "The engine supports [vlm inference](https://github.com/sgl-project/sglang/blob/main/examples/runtime/engine/offline_batch_inference_vlm.py) as well as [extracting hidden states](https://github.com/sgl-project/sglang/blob/main/examples/runtime/hidden_states). \n", - "\n", - "Please see [the examples](https://github.com/sgl-project/sglang/tree/main/examples/runtime/engine) for further use cases." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Ray Integration\n", - "\n", - "When running in a Ray cluster, you can use `RayEngine` with a custom placement group for fine-grained GPU placement control.\n", - "\n", - "### Custom Placement Groups\n", - "\n", - "Pass a `placement_group` with 1-GPU-per-bundle bundles to control exactly which GPUs are used. Each bundle should have exactly 1 GPU for deterministic mapping.\n", - "\n", - "```python\n", - "import ray\n", - "from ray.util.placement_group import placement_group\n", - "from sglang.srt.ray.engine import RayEngine\n", - "\n", - "ray.init()\n", - "\n", - "# Create placement group with specific GPU bundles\n", - "pg = placement_group(\n", - " [{\"GPU\": 1} for _ in range(4)], # 4 bundles, each with 1 GPU\n", - " strategy=\"STRICT_PACK\",\n", - ")\n", - "ray.get(pg.ready())\n", - "\n", - "# Launch RayEngine on custom placement group\n", - "engine = RayEngine(\n", - " model_path=\"meta-llama/Meta-Llama-3-8B-Instruct\",\n", - " tp_size=4,\n", - " use_ray=True,\n", - " placement_group=pg,\n", - ")\n", - "\n", - "# Optional: specify exact bundle indices via environment variable\n", - "# export SGLANG_RAY_BUNDLE_INDICES=\"0,1,2,3\"\n", - "```\n", - "\n", - "### Bundle Index Control\n", - "\n", - "Use `SGLANG_RAY_BUNDLE_INDICES` environment variable to specify which placement group bundles to use for each worker rank. This enables:\n", - "- Skipping unhealthy GPUs\n", - "- Topology-aware placement (e.g., NVLink-connected GPUs)\n", - "- Non-sequential bundle assignment\n", - "\n", - "```bash\n", - "# Use bundles 0,1,2,7 (skip bundles 3-6) for tp_size=4\n", - "export SGLANG_RAY_BUNDLE_INDICES=\"0,1,2,7\"\n", - "\n", - "# Place workers on NVLink-connected GPUs\n", - "export SGLANG_RAY_BUNDLE_INDICES=\"0,1,2,3\"\n", - "```\n", - "\n", - "The number of indices must match `world_size` (`tp_size * pp_size * dp_size`, or `tp_size * pp_size` when `enable_dp_attention=True`)." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Offline Batch Inference\n", - "\n", - "SGLang offline engine supports batch inference with efficient scheduling." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# launch the offline engine\n", - "import asyncio\n", - "\n", - "import sglang as sgl\n", - "import sglang.test.doc_patch # noqa: F401\n", - "from sglang.utils import async_stream_and_merge, stream_and_merge\n", - "\n", - "llm = sgl.Engine(model_path=\"qwen/qwen2.5-0.5b-instruct\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Non-streaming Synchronous Generation" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "prompts = [\n", - " \"Hello, my name is\",\n", - " \"The president of the United States is\",\n", - " \"The capital of France is\",\n", - " \"The future of AI is\",\n", - "]\n", - "\n", - "sampling_params = {\"temperature\": 0.8, \"top_p\": 0.95}\n", - "\n", - "outputs = llm.generate(prompts, sampling_params)\n", - "for prompt, output in zip(prompts, outputs):\n", - " print(\"===============================\")\n", - " print(f\"Prompt: {prompt}\\nGenerated text: {output['text']}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Streaming Synchronous Generation" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "prompts = [\n", - " \"Write a short, neutral self-introduction for a fictional character. Hello, my name is\",\n", - " \"Provide a concise factual statement about France’s capital city. The capital of France is\",\n", - " \"Explain possible future trends in artificial intelligence. The future of AI is\",\n", - "]\n", - "\n", - "sampling_params = {\n", - " \"temperature\": 0.2,\n", - " \"top_p\": 0.9,\n", - "}\n", - "\n", - "print(\"\\n=== Testing synchronous streaming generation with overlap removal ===\\n\")\n", - "\n", - "for prompt in prompts:\n", - " print(f\"Prompt: {prompt}\")\n", - " merged_output = stream_and_merge(llm, prompt, sampling_params)\n", - " print(\"Generated text:\", merged_output)\n", - " print()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Non-streaming Asynchronous Generation" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "prompts = [\n", - " \"Write a short, neutral self-introduction for a fictional character. Hello, my name is\",\n", - " \"Provide a concise factual statement about France’s capital city. The capital of France is\",\n", - " \"Explain possible future trends in artificial intelligence. The future of AI is\",\n", - "]\n", - "\n", - "sampling_params = {\"temperature\": 0.8, \"top_p\": 0.95}\n", - "\n", - "print(\"\\n=== Testing asynchronous batch generation ===\")\n", - "\n", - "\n", - "async def main():\n", - " outputs = await llm.async_generate(prompts, sampling_params)\n", - "\n", - " for prompt, output in zip(prompts, outputs):\n", - " print(f\"\\nPrompt: {prompt}\")\n", - " print(f\"Generated text: {output['text']}\")\n", - "\n", - "\n", - "asyncio.run(main())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Streaming Asynchronous Generation" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "prompts = [\n", - " \"Write a short, neutral self-introduction for a fictional character. Hello, my name is\",\n", - " \"Provide a concise factual statement about France’s capital city. The capital of France is\",\n", - " \"Explain possible future trends in artificial intelligence. The future of AI is\",\n", - "]\n", - "\n", - "sampling_params = {\"temperature\": 0.8, \"top_p\": 0.95}\n", - "\n", - "print(\"\\n=== Testing asynchronous streaming generation (no repeats) ===\")\n", - "\n", - "\n", - "async def main():\n", - " for prompt in prompts:\n", - " print(f\"\\nPrompt: {prompt}\")\n", - " print(\"Generated text: \", end=\"\", flush=True)\n", - "\n", - " # Replace direct calls to async_generate with our custom overlap-aware version\n", - " async for cleaned_chunk in async_stream_and_merge(llm, prompt, sampling_params):\n", - " print(cleaned_chunk, end=\"\", flush=True)\n", - "\n", - " print() # New line after each prompt\n", - "\n", - "\n", - "asyncio.run(main())" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "llm.shutdown()" - ] - } - ], - "metadata": { - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/docs_new/docs/basic_usage/ollama_api.mdx b/docs_new/docs/basic_usage/ollama_api.mdx index c92533c3f..17067eb19 100644 --- a/docs_new/docs/basic_usage/ollama_api.mdx +++ b/docs_new/docs/basic_usage/ollama_api.mdx @@ -125,7 +125,7 @@ for chunk in stream: ## Smart Router -For intelligent routing between local Ollama (fast) and remote SGLang (powerful) using an LLM judge, see the [Smart Router documentation](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/entrypoints/ollama/README). +For intelligent routing between local Ollama (fast) and remote SGLang (powerful) using an LLM judge, see the [Smart Router documentation](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/entrypoints/ollama/README.md). ## Summary diff --git a/docs_new/docs/basic_usage/openai_api.rst b/docs_new/docs/basic_usage/openai_api.rst deleted file mode 100644 index 370abe99c..000000000 --- a/docs_new/docs/basic_usage/openai_api.rst +++ /dev/null @@ -1,9 +0,0 @@ -OpenAI-Compatible APIs -====================== - -.. toctree:: - :maxdepth: 1 - - openai_api_completions.ipynb - openai_api_vision.ipynb - openai_api_embeddings.ipynb diff --git a/docs_new/docs/basic_usage/openai_api_completions.ipynb b/docs_new/docs/basic_usage/openai_api_completions.ipynb deleted file mode 100644 index 8d417dab4..000000000 --- a/docs_new/docs/basic_usage/openai_api_completions.ipynb +++ /dev/null @@ -1,552 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# OpenAI APIs - Completions\n", - "\n", - "SGLang provides OpenAI-compatible APIs to enable a smooth transition from OpenAI services to self-hosted local models.\n", - "A complete reference for the API is available in the [OpenAI API Reference](https://platform.openai.com/docs/api-reference).\n", - "\n", - "This tutorial covers the following popular APIs:\n", - "\n", - "- `chat/completions`\n", - "- `completions`\n", - "\n", - "Check out other tutorials to learn about [vision APIs](openai_api_vision.ipynb) for vision-language models and [embedding APIs](openai_api_embeddings.ipynb) for embedding models." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Launch A Server\n", - "\n", - "Launch the server in your terminal and wait for it to initialize." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from sglang.test.doc_patch import launch_server_cmd\n", - "from sglang.utils import wait_for_server, print_highlight, terminate_process\n", - "\n", - "server_process, port = launch_server_cmd(\n", - " \"python3 -m sglang.launch_server --model-path qwen/qwen2.5-0.5b-instruct --host 0.0.0.0 --log-level warning\"\n", - ")\n", - "\n", - "wait_for_server(f\"http://localhost:{port}\", process=server_process)\n", - "print(f\"Server started on http://localhost:{port}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Chat Completions\n", - "\n", - "### Usage\n", - "\n", - "The server fully implements the OpenAI API.\n", - "It will automatically apply the chat template specified in the Hugging Face tokenizer, if one is available.\n", - "You can also specify a custom chat template with `--chat-template` when launching the server." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import openai\n", - "\n", - "client = openai.Client(base_url=f\"http://127.0.0.1:{port}/v1\", api_key=\"None\")\n", - "\n", - "response = client.chat.completions.create(\n", - " model=\"qwen/qwen2.5-0.5b-instruct\",\n", - " messages=[\n", - " {\"role\": \"user\", \"content\": \"List 3 countries and their capitals.\"},\n", - " ],\n", - " temperature=0,\n", - " max_tokens=64,\n", - ")\n", - "\n", - "print_highlight(f\"Response: {response}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Model Thinking/Reasoning Support\n", - "\n", - "Some models support internal reasoning or thinking processes that can be exposed in the API response. SGLang provides unified support for various reasoning models through the `chat_template_kwargs` parameter and compatible reasoning parsers.\n", - "\n", - "#### Supported Models and Configuration\n", - "\n", - "| Model Family | Chat Template Parameter | Reasoning Parser | Notes |\n", - "|--------------|------------------------|------------------|--------|\n", - "| DeepSeek-R1 (R1, R1-0528, R1-Distill) | `enable_thinking` | `--reasoning-parser deepseek-r1` | Standard reasoning models |\n", - "| DeepSeek-V3.1 | `thinking` | `--reasoning-parser deepseek-v3` | Hybrid model (thinking/non-thinking modes) |\n", - "| Qwen3 (standard) | `enable_thinking` | `--reasoning-parser qwen3` | Hybrid model (thinking/non-thinking modes) |\n", - "| Qwen3-Thinking | N/A (always enabled) | `--reasoning-parser qwen3-thinking` | Always generates reasoning |\n", - "| Kimi | N/A (always enabled) | `--reasoning-parser kimi` | Kimi thinking models |\n", - "| Gpt-Oss | N/A (always enabled) | `--reasoning-parser gpt-oss` | Gpt-Oss thinking models |\n", - "\n", - "#### Basic Usage\n", - "\n", - "To enable reasoning output, you need to:\n", - "1. Launch the server with the appropriate reasoning parser\n", - "2. Set the model-specific parameter in `chat_template_kwargs`\n", - "3. Optionally use `separate_reasoning: False` to not get reasoning content separately (default to `True`)\n", - "\n", - "**Note for Qwen3-Thinking models:** These models always generate thinking content and do not support the `enable_thinking` parameter. Use `--reasoning-parser qwen3-thinking` or `--reasoning-parser qwen3` to parse the thinking content.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Example: Qwen3 Models\n", - "\n", - "```python\n", - "# Launch server:\n", - "# python3 -m sglang.launch_server --model Qwen/Qwen3-4B --reasoning-parser qwen3\n", - "\n", - "from openai import OpenAI\n", - "\n", - "client = OpenAI(\n", - " api_key=\"EMPTY\",\n", - " base_url=f\"http://127.0.0.1:30000/v1\",\n", - ")\n", - "\n", - "model = \"Qwen/Qwen3-4B\"\n", - "messages = [{\"role\": \"user\", \"content\": \"How many r's are in 'strawberry'?\"}]\n", - "\n", - "response = client.chat.completions.create(\n", - " model=model,\n", - " messages=messages,\n", - " extra_body={\n", - " \"chat_template_kwargs\": {\"enable_thinking\": True},\n", - " \"separate_reasoning\": True\n", - " }\n", - ")\n", - "\n", - "print(\"Reasoning:\", response.choices[0].message.reasoning_content)\n", - "print(\"-\"*100)\n", - "print(\"Answer:\", response.choices[0].message.content)\n", - "```\n", - "\n", - "**ExampleOutput:**\n", - "```\n", - "Reasoning: Okay, so the user is asking how many 'r's are in the word 'strawberry'. Let me think. First, I need to make sure I have the word spelled correctly. Strawberry... S-T-R-A-W-B-E-R-R-Y. Wait, is that right? Let me break it down.\n", - "\n", - "Starting with 'strawberry', let's write out the letters one by one. S, T, R, A, W, B, E, R, R, Y. Hmm, wait, that's 10 letters. Let me check again. S (1), T (2), R (3), A (4), W (5), B (6), E (7), R (8), R (9), Y (10). So the letters are S-T-R-A-W-B-E-R-R-Y. \n", - "...\n", - "Therefore, the answer should be three R's in 'strawberry'. But I need to make sure I'm not counting any other letters as R. Let me check again. S, T, R, A, W, B, E, R, R, Y. No other R's. So three in total. Yeah, that seems right.\n", - "\n", - "----------------------------------------------------------------------------------------------------\n", - "Answer: The word \"strawberry\" contains **three** letters 'r'. Here's the breakdown:\n", - "\n", - "1. **S-T-R-A-W-B-E-R-R-Y** \n", - " - The **third letter** is 'R'. \n", - " - The **eighth and ninth letters** are also 'R's. \n", - "\n", - "Thus, the total count is **3**. \n", - "\n", - "**Answer:** 3.\n", - "```\n", - "\n", - "**Note:** Setting `\"enable_thinking\": False` (or omitting it) will result in `reasoning_content` being `None`. Qwen3-Thinking models always generate reasoning content and don't support the `enable_thinking` parameter.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Logit Bias Support\n", - "\n", - "SGLang supports the `logit_bias` parameter for both chat completions and completions APIs. This parameter allows you to modify the likelihood of specific tokens being generated by adding bias values to their logits. The bias values can range from -100 to 100, where:\n", - "\n", - "- **Positive values** (0 to 100) increase the likelihood of the token being selected\n", - "- **Negative values** (-100 to 0) decrease the likelihood of the token being selected\n", - "- **-100** effectively prevents the token from being generated\n", - "\n", - "The `logit_bias` parameter accepts a dictionary where keys are token IDs (as strings) and values are the bias amounts (as floats).\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Getting Token IDs\n", - "\n", - "To use `logit_bias` effectively, you need to know the token IDs for the words you want to bias. Here's how to get token IDs:\n", - "\n", - "```python\n", - "# Get tokenizer to find token IDs\n", - "import tiktoken\n", - "\n", - "# For OpenAI models, use the appropriate encoding\n", - "tokenizer = tiktoken.encoding_for_model(\"gpt-3.5-turbo\") # or your model\n", - "\n", - "# Get token IDs for specific words\n", - "word = \"sunny\"\n", - "token_ids = tokenizer.encode(word)\n", - "print(f\"Token IDs for '{word}': {token_ids}\")\n", - "\n", - "# For SGLang models, you can access the tokenizer through the client\n", - "# and get token IDs for bias\n", - "```\n", - "\n", - "**Important:** The `logit_bias` parameter uses token IDs as string keys, not the actual words.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Example: DeepSeek-V3 Models\n", - "\n", - "DeepSeek-V3 models support thinking mode through the `thinking` parameter:\n", - "\n", - "```python\n", - "# Launch server:\n", - "# python3 -m sglang.launch_server --model deepseek-ai/DeepSeek-V3.1 --tp 8 --reasoning-parser deepseek-v3\n", - "\n", - "from openai import OpenAI\n", - "\n", - "client = OpenAI(\n", - " api_key=\"EMPTY\",\n", - " base_url=f\"http://127.0.0.1:30000/v1\",\n", - ")\n", - "\n", - "model = \"deepseek-ai/DeepSeek-V3.1\"\n", - "messages = [{\"role\": \"user\", \"content\": \"How many r's are in 'strawberry'?\"}]\n", - "\n", - "response = client.chat.completions.create(\n", - " model=model,\n", - " messages=messages,\n", - " extra_body={\n", - " \"chat_template_kwargs\": {\"thinking\": True},\n", - " \"separate_reasoning\": True\n", - " }\n", - ")\n", - "\n", - "print(\"Reasoning:\", response.choices[0].message.reasoning_content)\n", - "print(\"-\"*100)\n", - "print(\"Answer:\", response.choices[0].message.content)\n", - "```\n", - "\n", - "**Example Output:**\n", - "```\n", - "Reasoning: First, the question is: \"How many r's are in 'strawberry'?\"\n", - "\n", - "I need to count the number of times the letter 'r' appears in the word \"strawberry\".\n", - "\n", - "Let me write out the word: S-T-R-A-W-B-E-R-R-Y.\n", - "\n", - "Now, I'll go through each letter and count the 'r's.\n", - "...\n", - "So, I have three 'r's in \"strawberry\".\n", - "\n", - "I should double-check. The word is spelled S-T-R-A-W-B-E-R-R-Y. The letters are at positions: 3, 8, and 9 are 'r's. Yes, that's correct.\n", - "\n", - "Therefore, the answer should be 3.\n", - "----------------------------------------------------------------------------------------------------\n", - "Answer: The word \"strawberry\" contains **3** instances of the letter \"r\". Here's a breakdown for clarity:\n", - "\n", - "- The word is spelled: S-T-R-A-W-B-E-R-R-Y\n", - "- The \"r\" appears at the 3rd, 8th, and 9th positions.\n", - "```\n", - "\n", - "**Note:** DeepSeek-V3 models use the `thinking` parameter (not `enable_thinking`) to control reasoning output.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Example with logit_bias parameter\n", - "# Note: You need to get the actual token IDs from your tokenizer\n", - "# For demonstration, we'll use some example token IDs\n", - "response = client.chat.completions.create(\n", - " model=\"qwen/qwen2.5-0.5b-instruct\",\n", - " messages=[\n", - " {\"role\": \"user\", \"content\": \"Complete this sentence: The weather today is\"}\n", - " ],\n", - " temperature=0.7,\n", - " max_tokens=20,\n", - " logit_bias={\n", - " \"12345\": 50, # Increase likelihood of token ID 12345\n", - " \"67890\": -50, # Decrease likelihood of token ID 67890\n", - " \"11111\": 25, # Slightly increase likelihood of token ID 11111\n", - " },\n", - ")\n", - "\n", - "print_highlight(f\"Response with logit bias: {response.choices[0].message.content}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Parameters\n", - "\n", - "The chat completions API accepts OpenAI Chat Completions API's parameters. Refer to [OpenAI Chat Completions API](https://platform.openai.com/docs/api-reference/chat/create) for more details.\n", - "\n", - "SGLang extends the standard API with the `extra_body` parameter, allowing for additional customization. One key option within `extra_body` is `chat_template_kwargs`, which can be used to pass arguments to the chat template processor." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "response = client.chat.completions.create(\n", - " model=\"qwen/qwen2.5-0.5b-instruct\",\n", - " messages=[\n", - " {\n", - " \"role\": \"system\",\n", - " \"content\": \"You are a knowledgeable historian who provides concise responses.\",\n", - " },\n", - " {\"role\": \"user\", \"content\": \"Tell me about ancient Rome\"},\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"Ancient Rome was a civilization centered in Italy.\",\n", - " },\n", - " {\"role\": \"user\", \"content\": \"What were their major achievements?\"},\n", - " ],\n", - " temperature=0.3, # Lower temperature for more focused responses\n", - " max_tokens=128, # Reasonable length for a concise response\n", - " top_p=0.95, # Slightly higher for better fluency\n", - " presence_penalty=0.2, # Mild penalty to avoid repetition\n", - " frequency_penalty=0.2, # Mild penalty for more natural language\n", - " n=1, # Single response is usually more stable\n", - " seed=42, # Keep for reproducibility\n", - ")\n", - "\n", - "print_highlight(response.choices[0].message.content)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Streaming mode is also supported." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Logit Bias Support\n", - "\n", - "The completions API also supports the `logit_bias` parameter with the same functionality as described in the chat completions section above.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "stream = client.chat.completions.create(\n", - " model=\"qwen/qwen2.5-0.5b-instruct\",\n", - " messages=[{\"role\": \"user\", \"content\": \"Say this is a test\"}],\n", - " stream=True,\n", - ")\n", - "for chunk in stream:\n", - " if chunk.choices[0].delta.content is not None:\n", - " print(chunk.choices[0].delta.content, end=\"\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Returning Routed Experts (MoE Models)\n", - "\n", - "For MoE models, set `return_routed_experts: true` in `extra_body` to return expert routing data. Requires `--enable-return-routed-experts` server flag. The `routed_experts` field will be returned in the `sgl_ext` object on each choice, containing base64-encoded int32 expert IDs as a flattened array with logical shape `[num_tokens, num_layers, top_k]`. By default this returns `[0, seqlen - 1)`, the full available sequence, because RL workflows need routed experts for the full sequence. Set `routed_experts_start_len` in `extra_body` to an absolute prefix length to return only `[routed_experts_start_len, seqlen - 1)`. For example, in multi-turn RL rollouts, routed experts for tokens from previous turns have already been collected, so setting this value avoids unnecessary transfer that cause bottlenecks." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Example with logit_bias parameter for completions API\n", - "# Note: You need to get the actual token IDs from your tokenizer\n", - "# For demonstration, we'll use some example token IDs\n", - "response = client.completions.create(\n", - " model=\"qwen/qwen2.5-0.5b-instruct\",\n", - " prompt=\"The best programming language for AI is\",\n", - " temperature=0.7,\n", - " max_tokens=20,\n", - " logit_bias={\n", - " \"12345\": 75, # Strongly favor token ID 12345\n", - " \"67890\": -100, # Completely avoid token ID 67890\n", - " \"11111\": -25, # Slightly discourage token ID 11111\n", - " },\n", - ")\n", - "\n", - "print_highlight(f\"Response with logit bias: {response.choices[0].text}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Completions\n", - "\n", - "### Usage\n", - "Completions API is similar to Chat Completions API, but without the `messages` parameter or chat templates." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "response = client.completions.create(\n", - " model=\"qwen/qwen2.5-0.5b-instruct\",\n", - " prompt=\"List 3 countries and their capitals.\",\n", - " temperature=0,\n", - " max_tokens=64,\n", - " n=1,\n", - " stop=None,\n", - ")\n", - "\n", - "print_highlight(f\"Response: {response}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Parameters\n", - "\n", - "The completions API accepts OpenAI Completions API's parameters. Refer to [OpenAI Completions API](https://platform.openai.com/docs/api-reference/completions/create) for more details.\n", - "\n", - "Here is an example of a detailed completions request:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "response = client.completions.create(\n", - " model=\"qwen/qwen2.5-0.5b-instruct\",\n", - " prompt=\"Write a short story about a space explorer.\",\n", - " temperature=0.7, # Moderate temperature for creative writing\n", - " max_tokens=150, # Longer response for a story\n", - " top_p=0.9, # Balanced diversity in word choice\n", - " stop=[\"\\n\\n\", \"THE END\"], # Multiple stop sequences\n", - " presence_penalty=0.3, # Encourage novel elements\n", - " frequency_penalty=0.3, # Reduce repetitive phrases\n", - " n=1, # Generate one completion\n", - " seed=123, # For reproducible results\n", - ")\n", - "\n", - "print_highlight(f\"Response: {response}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Returning Routed Experts (MoE Models)\n", - "\n", - "For MoE models, set `return_routed_experts: true` in `extra_body` to return expert routing data. Requires `--enable-return-routed-experts` server flag. The `routed_experts` field will be returned in the `sgl_ext` object on each choice, containing base64-encoded int32 expert IDs as a flattened array with logical shape `[num_tokens, num_layers, top_k]`. By default this returns `[0, seqlen - 1)`, the full available sequence, because RL workflows need routed experts for the full sequence. Set `routed_experts_start_len` in `extra_body` to an absolute prefix length to return only `[routed_experts_start_len, seqlen - 1)`. For example, in multi-turn RL rollouts, routed experts for tokens from previous turns have already been collected, so setting this value avoids unnecessary transfer that cause bottlenecks." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Structured Outputs (JSON, Regex, EBNF)\n", - "\n", - "For OpenAI compatible structured outputs API, refer to [Structured Outputs](../advanced_features/structured_outputs.ipynb) for more details.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Using LoRA Adapters\n", - "\n", - "SGLang supports LoRA (Low-Rank Adaptation) adapters with OpenAI-compatible APIs. You can specify which adapter to use directly in the `model` parameter using the `base-model:adapter-name` syntax.\n", - "\n", - "**Server Setup:**\n", - "```bash\n", - "python -m sglang.launch_server \\\n", - " --model-path qwen/qwen2.5-0.5b-instruct \\\n", - " --enable-lora \\\n", - " --lora-paths adapter_a=/path/to/adapter_a adapter_b=/path/to/adapter_b\n", - "```\n", - "\n", - "For more details on LoRA serving configuration, see the [LoRA documentation](../advanced_features/lora.ipynb).\n", - "\n", - "**API Call:**\n", - "\n", - "(Recommended) Use the `model:adapter` syntax to specify which adapter to use:\n", - "```python\n", - "response = client.chat.completions.create(\n", - " model=\"qwen/qwen2.5-0.5b-instruct:adapter_a\", # ← base-model:adapter-name\n", - " messages=[{\"role\": \"user\", \"content\": \"Convert to SQL: show all users\"}],\n", - " max_tokens=50,\n", - ")\n", - "```\n", - "\n", - "**Backward Compatible: Using `extra_body`**\n", - "\n", - "The old `extra_body` method is still supported for backward compatibility:\n", - "```python\n", - "# Backward compatible method\n", - "response = client.chat.completions.create(\n", - " model=\"qwen/qwen2.5-0.5b-instruct\",\n", - " messages=[{\"role\": \"user\", \"content\": \"Convert to SQL: show all users\"}],\n", - " extra_body={\"lora_path\": \"adapter_a\"}, # ← old method\n", - " max_tokens=50,\n", - ")\n", - "```\n", - "**Note:** When both `model:adapter` and `extra_body[\"lora_path\"]` are specified, the `model:adapter` syntax takes precedence." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "terminate_process(server_process)" - ] - } - ], - "metadata": { - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/docs_new/docs/basic_usage/openai_api_embeddings.ipynb b/docs_new/docs/basic_usage/openai_api_embeddings.ipynb deleted file mode 100644 index a6c90c06b..000000000 --- a/docs_new/docs/basic_usage/openai_api_embeddings.ipynb +++ /dev/null @@ -1,193 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# OpenAI APIs - Embedding\n", - "\n", - "SGLang provides OpenAI-compatible APIs to enable a smooth transition from OpenAI services to self-hosted local models.\n", - "A complete reference for the API is available in the [OpenAI API Reference](https://platform.openai.com/docs/guides/embeddings).\n", - "\n", - "This tutorial covers the embedding APIs for embedding models. For a list of the supported models see the [corresponding overview page](../supported_models/retrieval_ranking/embedding_models.md)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Launch A Server\n", - "\n", - "Launch the server in your terminal and wait for it to initialize. Remember to add `--is-embedding` to the command." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from sglang.test.doc_patch import launch_server_cmd\n", - "from sglang.utils import wait_for_server, print_highlight, terminate_process\n", - "\n", - "embedding_process, port = launch_server_cmd(\"\"\"\n", - "python3 -m sglang.launch_server --model-path Alibaba-NLP/gte-Qwen2-1.5B-instruct \\\n", - " --host 0.0.0.0 --is-embedding --log-level warning\n", - "\"\"\")\n", - "\n", - "wait_for_server(f\"http://localhost:{port}\", process=embedding_process)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Using cURL" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import subprocess, json\n", - "\n", - "text = \"Once upon a time\"\n", - "\n", - "curl_text = f\"\"\"curl -s http://localhost:{port}/v1/embeddings \\\n", - " -H \"Content-Type: application/json\" \\\n", - " -d '{{\"model\": \"Alibaba-NLP/gte-Qwen2-1.5B-instruct\", \"input\": \"{text}\"}}'\"\"\"\n", - "\n", - "result = subprocess.check_output(curl_text, shell=True)\n", - "\n", - "print(result)\n", - "\n", - "text_embedding = json.loads(result)[\"data\"][0][\"embedding\"]\n", - "\n", - "print_highlight(f\"Text embedding (first 10): {text_embedding[:10]}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Using Python Requests" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import requests\n", - "\n", - "text = \"Once upon a time\"\n", - "\n", - "response = requests.post(\n", - " f\"http://localhost:{port}/v1/embeddings\",\n", - " json={\"model\": \"Alibaba-NLP/gte-Qwen2-1.5B-instruct\", \"input\": text},\n", - ")\n", - "\n", - "text_embedding = response.json()[\"data\"][0][\"embedding\"]\n", - "\n", - "print_highlight(f\"Text embedding (first 10): {text_embedding[:10]}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Using OpenAI Python Client" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import openai\n", - "\n", - "client = openai.Client(base_url=f\"http://127.0.0.1:{port}/v1\", api_key=\"None\")\n", - "\n", - "# Text embedding example\n", - "response = client.embeddings.create(\n", - " model=\"Alibaba-NLP/gte-Qwen2-1.5B-instruct\",\n", - " input=text,\n", - ")\n", - "\n", - "embedding = response.data[0].embedding[:10]\n", - "print_highlight(f\"Text embedding (first 10): {embedding}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Using Input IDs\n", - "\n", - "SGLang also supports `input_ids` as input to get the embedding." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import json\n", - "import os\n", - "from transformers import AutoTokenizer\n", - "\n", - "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n", - "\n", - "tokenizer = AutoTokenizer.from_pretrained(\"Alibaba-NLP/gte-Qwen2-1.5B-instruct\")\n", - "input_ids = tokenizer.encode(text)\n", - "\n", - "curl_ids = f\"\"\"curl -s http://localhost:{port}/v1/embeddings \\\n", - " -H \"Content-Type: application/json\" \\\n", - " -d '{{\"model\": \"Alibaba-NLP/gte-Qwen2-1.5B-instruct\", \"input\": {json.dumps(input_ids)}}}'\"\"\"\n", - "\n", - "input_ids_embedding = json.loads(subprocess.check_output(curl_ids, shell=True))[\"data\"][\n", - " 0\n", - "][\"embedding\"]\n", - "\n", - "print_highlight(f\"Input IDs embedding (first 10): {input_ids_embedding[:10]}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "terminate_process(embedding_process)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Multi-Modal Embedding Model\n", - "Please refer to [Multi-Modal Embedding Model](../supported_models/retrieval_ranking/embedding_models.md)" - ] - } - ], - "metadata": { - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/docs_new/docs/basic_usage/openai_api_vision.ipynb b/docs_new/docs/basic_usage/openai_api_vision.ipynb deleted file mode 100644 index b6e6a1a24..000000000 --- a/docs_new/docs/basic_usage/openai_api_vision.ipynb +++ /dev/null @@ -1,253 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# OpenAI APIs - Vision\n", - "\n", - "SGLang provides OpenAI-compatible APIs to enable a smooth transition from OpenAI services to self-hosted local models.\n", - "A complete reference for the API is available in the [OpenAI API Reference](https://platform.openai.com/docs/guides/vision).\n", - "This tutorial covers the vision APIs for vision language models.\n", - "\n", - "SGLang supports various vision language models such as Llama 3.2, LLaVA-OneVision, Qwen2.5-VL, Gemma3 and [more](../supported_models/text_generation/multimodal_language_models.md).\n", - "\n", - "As an alternative to the OpenAI API, you can also use the [SGLang offline engine](https://github.com/sgl-project/sglang/blob/main/examples/runtime/engine/offline_batch_inference_vlm.py)." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Launch A Server\n", - "\n", - "Launch the server in your terminal and wait for it to initialize." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from sglang.test.doc_patch import launch_server_cmd\n", - "from sglang.utils import wait_for_server, print_highlight, terminate_process\n", - "\n", - "example_image_url = \"https://raw.githubusercontent.com/sgl-project/sglang/main/examples/assets/example_image.png\"\n", - "logo_image_url = (\n", - " \"https://raw.githubusercontent.com/sgl-project/sglang/main/assets/logo.png\"\n", - ")\n", - "\n", - "vision_process, port = launch_server_cmd(\"\"\"\n", - "python3 -m sglang.launch_server --model-path Qwen/Qwen2.5-VL-7B-Instruct --log-level warning\n", - "\"\"\")\n", - "\n", - "wait_for_server(f\"http://localhost:{port}\", process=vision_process)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Using cURL\n", - "\n", - "Once the server is up, you can send test requests using curl or requests." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import subprocess\n", - "\n", - "curl_command = f\"\"\"\n", - "curl -s http://localhost:{port}/v1/chat/completions \\\\\n", - " -H \"Content-Type: application/json\" \\\\\n", - " -d '{{\n", - " \"model\": \"Qwen/Qwen2.5-VL-7B-Instruct\",\n", - " \"messages\": [\n", - " {{\n", - " \"role\": \"user\",\n", - " \"content\": [\n", - " {{\n", - " \"type\": \"text\",\n", - " \"text\": \"What’s in this image?\"\n", - " }},\n", - " {{\n", - " \"type\": \"image_url\",\n", - " \"image_url\": {{\n", - " \"url\": \"{example_image_url}\"\n", - " }}\n", - " }}\n", - " ]\n", - " }}\n", - " ],\n", - " \"max_tokens\": 300\n", - " }}'\n", - "\"\"\"\n", - "\n", - "response = subprocess.check_output(curl_command, shell=True).decode()\n", - "print_highlight(response)\n", - "\n", - "\n", - "response = subprocess.check_output(curl_command, shell=True).decode()\n", - "print_highlight(response)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Using Python Requests" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import requests\n", - "\n", - "url = f\"http://localhost:{port}/v1/chat/completions\"\n", - "\n", - "data = {\n", - " \"model\": \"Qwen/Qwen2.5-VL-7B-Instruct\",\n", - " \"messages\": [\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": [\n", - " {\"type\": \"text\", \"text\": \"What’s in this image?\"},\n", - " {\n", - " \"type\": \"image_url\",\n", - " \"image_url\": {\"url\": example_image_url},\n", - " },\n", - " ],\n", - " }\n", - " ],\n", - " \"max_tokens\": 300,\n", - "}\n", - "\n", - "response = requests.post(url, json=data)\n", - "print_highlight(response.text)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Using OpenAI Python Client" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from openai import OpenAI\n", - "\n", - "client = OpenAI(base_url=f\"http://localhost:{port}/v1\", api_key=\"None\")\n", - "\n", - "response = client.chat.completions.create(\n", - " model=\"Qwen/Qwen2.5-VL-7B-Instruct\",\n", - " messages=[\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"text\",\n", - " \"text\": \"What is in this image?\",\n", - " },\n", - " {\n", - " \"type\": \"image_url\",\n", - " \"image_url\": {\"url\": example_image_url},\n", - " },\n", - " ],\n", - " }\n", - " ],\n", - " max_tokens=300,\n", - ")\n", - "\n", - "print_highlight(response.choices[0].message.content)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Multiple-Image Inputs\n", - "\n", - "The server also supports multiple images and interleaved text and images if the model supports it." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from openai import OpenAI\n", - "\n", - "client = OpenAI(base_url=f\"http://localhost:{port}/v1\", api_key=\"None\")\n", - "\n", - "response = client.chat.completions.create(\n", - " model=\"Qwen/Qwen2.5-VL-7B-Instruct\",\n", - " messages=[\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"image_url\",\n", - " \"image_url\": {\n", - " \"url\": example_image_url,\n", - " },\n", - " },\n", - " {\n", - " \"type\": \"image_url\",\n", - " \"image_url\": {\n", - " \"url\": logo_image_url,\n", - " },\n", - " },\n", - " {\n", - " \"type\": \"text\",\n", - " \"text\": \"I have two very different images. They are not related at all. \"\n", - " \"Please describe the first image in one sentence, and then describe the second image in another sentence.\",\n", - " },\n", - " ],\n", - " }\n", - " ],\n", - " temperature=0,\n", - ")\n", - "\n", - "print_highlight(response.choices[0].message.content)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "terminate_process(vision_process)" - ] - } - ], - "metadata": { - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/docs_new/docs/basic_usage/sampling_params.mdx b/docs_new/docs/basic_usage/sampling_params.mdx index 1972bd335..1e586ac48 100644 --- a/docs_new/docs/basic_usage/sampling_params.mdx +++ b/docs_new/docs/basic_usage/sampling_params.mdx @@ -42,7 +42,7 @@ The `/generate` endpoint accepts the following parameters in JSON format. For de image_data `Optional[Union[List[List[ImageDataItem]], List[ImageDataItem], ImageDataItem]] = None` - The image input. Supports three formats: (1) **Raw images**: PIL Image, file path, URL, or base64 string; (2) **Processor output**: Dict with `format: "processor_output"` containing HuggingFace processor outputs; (3) **Precomputed embeddings**: Dict with `format: "precomputed_embedding"` and `feature` containing pre-calculated visual embeddings. Can be a single image, list of images, or list of lists of images. See [Multimodal Input Formats](#multimodal-input-formats) for details. + The image input. Supports three formats: (1) **Raw images**: PIL Image, file path, URL, or base64 string; (2) **Processor output**: Dict with `format: "processor_output"` containing HuggingFace processor outputs; (3) **Precomputed embeddings**: Dict with `format: "precomputed_embedding"` and `feature` containing pre-calculated visual embeddings. Can be a single image, list of images, or list of lists of images. See [Multimodal Input Formats](#multimodal) for details. audio_data @@ -434,7 +434,7 @@ You can specify a JSON schema, regular expression or [EBNF](https://en.wikipedia SGLang supports two grammar backends: - [XGrammar](https://github.com/mlc-ai/xgrammar) (default): Supports JSON schema, regular expression, and EBNF constraints. - - XGrammar currently uses the [GGML BNF format](https://github.com/ggerganov/llama.cpp/blob/master/grammars/README). + - XGrammar currently uses the [GGML BNF format](https://github.com/ggml-org/llama.cpp/blob/master/grammars/README.md). - [Outlines](https://github.com/dottxt-ai/outlines): Supports JSON schema and regular expression constraints. If instead you want to initialize the Outlines backend, you can use `--grammar-backend outlines` flag: diff --git a/docs_new/docs/basic_usage/send_request.ipynb b/docs_new/docs/basic_usage/send_request.ipynb deleted file mode 100644 index 968a23b8d..000000000 --- a/docs_new/docs/basic_usage/send_request.ipynb +++ /dev/null @@ -1,251 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Sending Requests\n", - "This notebook provides a quick-start guide to use SGLang in chat completions after installation. Once your server is running, API documentation is available at `http://localhost:30000/docs` (Swagger UI), `http://localhost:30000/redoc` (ReDoc), or `http://localhost:30000/openapi.json` (OpenAPI spec, useful for AI agents). Replace `30000` with your port if using a different one.\n", - "\n", - "- For Vision Language Models, see [OpenAI APIs - Vision](openai_api_vision.ipynb).\n", - "- For Embedding Models, see [OpenAI APIs - Embedding](openai_api_embeddings.ipynb) and [Encode (embedding model)](native_api.html#Encode-(embedding-model)).\n", - "- For Reward Models, see [Classify (reward model)](native_api.html#Classify-(reward-model))." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Launch A Server" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from sglang.test.doc_patch import launch_server_cmd\n", - "from sglang.utils import wait_for_server, print_highlight, terminate_process\n", - "\n", - "# This is equivalent to running the following command in your terminal\n", - "# python3 -m sglang.launch_server --model-path qwen/qwen2.5-0.5b-instruct --host 0.0.0.0\n", - "\n", - "server_process, port = launch_server_cmd(\"\"\"\n", - "python3 -m sglang.launch_server --model-path qwen/qwen2.5-0.5b-instruct \\\n", - " --host 0.0.0.0 --log-level warning\n", - "\"\"\")\n", - "\n", - "wait_for_server(f\"http://localhost:{port}\", process=server_process)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Using cURL\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import subprocess, json\n", - "\n", - "curl_command = f\"\"\"\n", - "curl -s http://localhost:{port}/v1/chat/completions \\\n", - " -H \"Content-Type: application/json\" \\\n", - " -d '{{\"model\": \"qwen/qwen2.5-0.5b-instruct\", \"messages\": [{{\"role\": \"user\", \"content\": \"What is the capital of France?\"}}]}}'\n", - "\"\"\"\n", - "\n", - "response = json.loads(subprocess.check_output(curl_command, shell=True))\n", - "print_highlight(response)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Using Python Requests" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import requests\n", - "\n", - "url = f\"http://localhost:{port}/v1/chat/completions\"\n", - "\n", - "data = {\n", - " \"model\": \"qwen/qwen2.5-0.5b-instruct\",\n", - " \"messages\": [{\"role\": \"user\", \"content\": \"What is the capital of France?\"}],\n", - "}\n", - "\n", - "response = requests.post(url, json=data)\n", - "print_highlight(response.json())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Using OpenAI Python Client" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import openai\n", - "\n", - "client = openai.Client(base_url=f\"http://127.0.0.1:{port}/v1\", api_key=\"None\")\n", - "\n", - "response = client.chat.completions.create(\n", - " model=\"qwen/qwen2.5-0.5b-instruct\",\n", - " messages=[\n", - " {\"role\": \"user\", \"content\": \"List 3 countries and their capitals.\"},\n", - " ],\n", - " temperature=0,\n", - " max_tokens=64,\n", - ")\n", - "print_highlight(response)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Streaming" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import openai\n", - "\n", - "client = openai.Client(base_url=f\"http://127.0.0.1:{port}/v1\", api_key=\"None\")\n", - "\n", - "# Use stream=True for streaming responses\n", - "response = client.chat.completions.create(\n", - " model=\"qwen/qwen2.5-0.5b-instruct\",\n", - " messages=[\n", - " {\"role\": \"user\", \"content\": \"List 3 countries and their capitals.\"},\n", - " ],\n", - " temperature=0,\n", - " max_tokens=64,\n", - " stream=True,\n", - ")\n", - "\n", - "# Handle the streaming output\n", - "for chunk in response:\n", - " if chunk.choices[0].delta.content:\n", - " print(chunk.choices[0].delta.content, end=\"\", flush=True)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Using Native Generation APIs\n", - "\n", - "You can also use the native `/generate` endpoint with requests, which provides more flexibility. An API reference is available at [Sampling Parameters](sampling_params.md)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import requests\n", - "\n", - "response = requests.post(\n", - " f\"http://localhost:{port}/generate\",\n", - " json={\n", - " \"text\": \"The capital of France is\",\n", - " \"sampling_params\": {\n", - " \"temperature\": 0,\n", - " \"max_new_tokens\": 32,\n", - " },\n", - " },\n", - ")\n", - "\n", - "print_highlight(response.json())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Streaming" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import requests, json\n", - "\n", - "response = requests.post(\n", - " f\"http://localhost:{port}/generate\",\n", - " json={\n", - " \"text\": \"The capital of France is\",\n", - " \"sampling_params\": {\n", - " \"temperature\": 0,\n", - " \"max_new_tokens\": 32,\n", - " },\n", - " \"stream\": True,\n", - " },\n", - " stream=True,\n", - ")\n", - "\n", - "prev = 0\n", - "for chunk in response.iter_lines(decode_unicode=False):\n", - " chunk = chunk.decode(\"utf-8\")\n", - " if chunk and chunk.startswith(\"data:\"):\n", - " if chunk == \"data: [DONE]\":\n", - " break\n", - " data = json.loads(chunk[5:].strip(\"\\n\"))\n", - " output = data[\"text\"]\n", - " print(output[prev:], end=\"\", flush=True)\n", - " prev = len(output)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "terminate_process(server_process)" - ] - } - ], - "metadata": { - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/docs_new/docs/developer_guide/contribution_guide.mdx b/docs_new/docs/developer_guide/contribution_guide.mdx index ccb27beeb..1339257fa 100644 --- a/docs_new/docs/developer_guide/contribution_guide.mdx +++ b/docs_new/docs/developer_guide/contribution_guide.mdx @@ -94,8 +94,8 @@ Also, do not rely on the "Latency/Output throughput" from this script, as it is GSM8K is too easy for state-of-the-art models nowadays. Please try your own more challenging accuracy tests. You can find additional accuracy eval examples in: -- [test_eval_accuracy_large.py](https://github.com/sgl-project/sglang/blob/main/test/registered/eval/test_eval_accuracy_large.py) -- [test_gpt_oss_1gpu.py](https://github.com/sgl-project/sglang/blob/main/test/registered/core/test_gpt_oss_1gpu.py) +- [test_eval_accuracy_large.py](https://github.com/sgl-project/sglang/blob/main/test/manual/eval/test_eval_accuracy_large.py) +- [test_gpt_oss_1gpu.py](https://github.com/sgl-project/sglang/blob/main/test/manual/core/test_gpt_oss_1gpu.py) ## Benchmark the speed Refer to [Benchmark and Profiling](./benchmark_and_profiling). diff --git a/docs_new/docs/developer_guide/msprobe_debugging_guide.mdx b/docs_new/docs/developer_guide/msprobe_debugging_guide.mdx index c40600cf2..3b8a69499 100644 --- a/docs_new/docs/developer_guide/msprobe_debugging_guide.mdx +++ b/docs_new/docs/developer_guide/msprobe_debugging_guide.mdx @@ -306,7 +306,7 @@ After locating the divergent node (e.g., a specific Conv layer or torch API with - `dump_tensor_data`: Save the collected tensor data. - `dump.json`: Statistics for the forward data of each API or module, including names, dtype, shape, max, min, mean, L2 norm (square root of the L2 variance), and CRC-32 when `summary_mode="md5"`. - See [dump.json file description](#dumpjson-file-description) for details. + See [dump.json file description](#dump-json-file-description) for details. - `dump_error_info.log`: Present only when the dump tool encountered an error and records the failure log. - `stack.json`: Call stacks for APIs/modules. - `construct.json`: Hierarchical structure description. Empty when `level=L1`. diff --git a/docs_new/docs/hardware-platforms/nvidia_jetson.mdx b/docs_new/docs/hardware-platforms/nvidia_jetson.mdx index 26f8e58d4..015b20f61 100644 --- a/docs_new/docs/hardware-platforms/nvidia_jetson.mdx +++ b/docs_new/docs/hardware-platforms/nvidia_jetson.mdx @@ -51,7 +51,7 @@ python -m sglang.launch_server \ ``` The quantization and limited context length (`--dtype half --context-length 8192`) are due to the limited computational resources in [Nvidia jetson kit](https://www.nvidia.com/en-us/autonomous-machines/embedded-systems/jetson-orin/). A detailed explanation can be found in [Server Arguments](../advanced_features/server_arguments). -After launching the engine, refer to [Chat completions](../basic_usage/openai_api_completions#Usage) to test the usability. +After launching the engine, refer to [Chat completions](../basic_usage/openai_api_completions#usage) to test the usability. * * * * * Running quantization with TorchAO ------------------------------------- diff --git a/docs_new/docs/references/custom_chat_template.mdx b/docs_new/docs/references/custom_chat_template.mdx index 19cae6004..7138d510f 100644 --- a/docs_new/docs/references/custom_chat_template.mdx +++ b/docs_new/docs/references/custom_chat_template.mdx @@ -3,7 +3,7 @@ title: "Custom Chat Template" metatags: description: "SGLang custom chat templates: JSON and Jinja formats for OpenAI-compatible API server. Override tokenizer defaults." --- -**NOTE**: There are two chat template systems in SGLang project. This document is about setting a custom chat template for the OpenAI-compatible API server (defined at [conversation.py](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/conversation.py)). It is NOT related to the chat template used in the SGLang language frontend (defined at [chat_template.py](https://github.com/sgl-project/sglang/blob/main/python/sglang/lang/chat_template.py)). +**NOTE**: There are two chat template systems in SGLang project. This document is about setting a custom chat template for the OpenAI-compatible API server (defined at [conversation.py](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/parser/conversation.py)). It is NOT related to the chat template used in the SGLang language frontend (defined at [chat_template.py](https://github.com/sgl-project/sglang/blob/main/python/sglang/lang/chat_template.py)). By default, the server uses the chat template specified in the model tokenizer from Hugging Face. It should just work for most official models such as Llama-2/Llama-3. diff --git a/docs_new/docs/references/frontend/frontend_index.rst b/docs_new/docs/references/frontend/frontend_index.rst deleted file mode 100644 index 62544cba5..000000000 --- a/docs_new/docs/references/frontend/frontend_index.rst +++ /dev/null @@ -1,9 +0,0 @@ -Frontend Language -================= - -.. toctree:: - :maxdepth: 1 - :caption: Frontend Language - - frontend_tutorial.ipynb - choices_methods.md diff --git a/docs_new/docs/references/frontend/frontend_tutorial.ipynb b/docs_new/docs/references/frontend/frontend_tutorial.ipynb deleted file mode 100644 index 9c4da052c..000000000 --- a/docs_new/docs/references/frontend/frontend_tutorial.ipynb +++ /dev/null @@ -1,456 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# SGLang Frontend Language" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "SGLang frontend language can be used to define simple and easy prompts in a convenient, structured way." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Launch A Server\n", - "\n", - "Launch the server in your terminal and wait for it to initialize." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from sglang import assistant_begin, assistant_end\n", - "from sglang import assistant, function, gen, system, user\n", - "from sglang import image\n", - "from sglang import RuntimeEndpoint\n", - "from sglang.lang.api import set_default_backend\n", - "from sglang.srt.utils import load_image\n", - "from sglang.test.doc_patch import launch_server_cmd\n", - "from sglang.utils import print_highlight, terminate_process, wait_for_server\n", - "\n", - "server_process, port = launch_server_cmd(\n", - " \"python -m sglang.launch_server --model-path Qwen/Qwen2.5-7B-Instruct --host 0.0.0.0 --log-level warning\"\n", - ")\n", - "\n", - "wait_for_server(f\"http://localhost:{port}\", process=server_process)\n", - "print(f\"Server started on http://localhost:{port}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Set the default backend. Note: Besides the local server, you may use also `OpenAI` or other API endpoints." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "set_default_backend(RuntimeEndpoint(f\"http://localhost:{port}\"))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Basic Usage\n", - "\n", - "The most simple way of using SGLang frontend language is a simple question answer dialog between a user and an assistant." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "@function\n", - "def basic_qa(s, question):\n", - " s += system(f\"You are a helpful assistant than can answer questions.\")\n", - " s += user(question)\n", - " s += assistant(gen(\"answer\", max_tokens=512))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "state = basic_qa(\"List 3 countries and their capitals.\")\n", - "print_highlight(state[\"answer\"])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Multi-turn Dialog\n", - "\n", - "SGLang frontend language can also be used to define multi-turn dialogs." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "@function\n", - "def multi_turn_qa(s):\n", - " s += system(f\"You are a helpful assistant than can answer questions.\")\n", - " s += user(\"Please give me a list of 3 countries and their capitals.\")\n", - " s += assistant(gen(\"first_answer\", max_tokens=512))\n", - " s += user(\"Please give me another list of 3 countries and their capitals.\")\n", - " s += assistant(gen(\"second_answer\", max_tokens=512))\n", - " return s\n", - "\n", - "\n", - "state = multi_turn_qa()\n", - "print_highlight(state[\"first_answer\"])\n", - "print_highlight(state[\"second_answer\"])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Control flow\n", - "\n", - "You may use any Python code within the function to define more complex control flows." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "@function\n", - "def tool_use(s, question):\n", - " s += assistant(\n", - " \"To answer this question: \"\n", - " + question\n", - " + \". I need to use a \"\n", - " + gen(\"tool\", choices=[\"calculator\", \"search engine\"])\n", - " + \". \"\n", - " )\n", - "\n", - " if s[\"tool\"] == \"calculator\":\n", - " s += assistant(\"The math expression is: \" + gen(\"expression\"))\n", - " elif s[\"tool\"] == \"search engine\":\n", - " s += assistant(\"The key word to search is: \" + gen(\"word\"))\n", - "\n", - "\n", - "state = tool_use(\"What is 2 * 2?\")\n", - "print_highlight(state[\"tool\"])\n", - "print_highlight(state[\"expression\"])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Parallelism\n", - "\n", - "Use `fork` to launch parallel prompts. Because `sgl.gen` is non-blocking, the for loop below issues two generation calls in parallel." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "@function\n", - "def tip_suggestion(s):\n", - " s += assistant(\n", - " \"Here are two tips for staying healthy: \"\n", - " \"1. Balanced Diet. 2. Regular Exercise.\\n\\n\"\n", - " )\n", - "\n", - " forks = s.fork(2)\n", - " for i, f in enumerate(forks):\n", - " f += assistant(\n", - " f\"Now, expand tip {i+1} into a paragraph:\\n\"\n", - " + gen(\"detailed_tip\", max_tokens=256, stop=\"\\n\\n\")\n", - " )\n", - "\n", - " s += assistant(\"Tip 1:\" + forks[0][\"detailed_tip\"] + \"\\n\")\n", - " s += assistant(\"Tip 2:\" + forks[1][\"detailed_tip\"] + \"\\n\")\n", - " s += assistant(\n", - " \"To summarize the above two tips, I can say:\\n\" + gen(\"summary\", max_tokens=512)\n", - " )\n", - "\n", - "\n", - "state = tip_suggestion()\n", - "print_highlight(state[\"summary\"])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Constrained Decoding\n", - "\n", - "Use `regex` to specify a regular expression as a decoding constraint. This is only supported for local models." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "@function\n", - "def regular_expression_gen(s):\n", - " s += user(\"What is the IP address of the Google DNS servers?\")\n", - " s += assistant(\n", - " gen(\n", - " \"answer\",\n", - " temperature=0,\n", - " regex=r\"((25[0-5]|2[0-4]\\d|[01]?\\d\\d?).){3}(25[0-5]|2[0-4]\\d|[01]?\\d\\d?)\",\n", - " )\n", - " )\n", - "\n", - "\n", - "state = regular_expression_gen()\n", - "print_highlight(state[\"answer\"])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Use `regex` to define a `JSON` decoding schema." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "character_regex = (\n", - " r\"\"\"\\{\\n\"\"\"\n", - " + r\"\"\" \"name\": \"[\\w\\d\\s]{1,16}\",\\n\"\"\"\n", - " + r\"\"\" \"house\": \"(Gryffindor|Slytherin|Ravenclaw|Hufflepuff)\",\\n\"\"\"\n", - " + r\"\"\" \"blood status\": \"(Pure-blood|Half-blood|Muggle-born)\",\\n\"\"\"\n", - " + r\"\"\" \"occupation\": \"(student|teacher|auror|ministry of magic|death eater|order of the phoenix)\",\\n\"\"\"\n", - " + r\"\"\" \"wand\": \\{\\n\"\"\"\n", - " + r\"\"\" \"wood\": \"[\\w\\d\\s]{1,16}\",\\n\"\"\"\n", - " + r\"\"\" \"core\": \"[\\w\\d\\s]{1,16}\",\\n\"\"\"\n", - " + r\"\"\" \"length\": [0-9]{1,2}\\.[0-9]{0,2}\\n\"\"\"\n", - " + r\"\"\" \\},\\n\"\"\"\n", - " + r\"\"\" \"alive\": \"(Alive|Deceased)\",\\n\"\"\"\n", - " + r\"\"\" \"patronus\": \"[\\w\\d\\s]{1,16}\",\\n\"\"\"\n", - " + r\"\"\" \"bogart\": \"[\\w\\d\\s]{1,16}\"\\n\"\"\"\n", - " + r\"\"\"\\}\"\"\"\n", - ")\n", - "\n", - "\n", - "@function\n", - "def character_gen(s, name):\n", - " s += user(\n", - " f\"{name} is a character in Harry Potter. Please fill in the following information about this character.\"\n", - " )\n", - " s += assistant(gen(\"json_output\", max_tokens=256, regex=character_regex))\n", - "\n", - "\n", - "state = character_gen(\"Harry Potter\")\n", - "print_highlight(state[\"json_output\"])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Batching \n", - "\n", - "Use `run_batch` to run a batch of prompts." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "@function\n", - "def text_qa(s, question):\n", - " s += user(question)\n", - " s += assistant(gen(\"answer\", stop=\"\\n\"))\n", - "\n", - "\n", - "states = text_qa.run_batch(\n", - " [\n", - " {\"question\": \"What is the capital of the United Kingdom?\"},\n", - " {\"question\": \"What is the capital of France?\"},\n", - " {\"question\": \"What is the capital of Japan?\"},\n", - " ],\n", - " progress_bar=True,\n", - ")\n", - "\n", - "for i, state in enumerate(states):\n", - " print_highlight(f\"Answer {i+1}: {states[i]['answer']}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Streaming \n", - "\n", - "Use `stream` to stream the output to the user." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "@function\n", - "def text_qa(s, question):\n", - " s += user(question)\n", - " s += assistant(gen(\"answer\", stop=\"\\n\"))\n", - "\n", - "\n", - "state = text_qa.run(\n", - " question=\"What is the capital of France?\", temperature=0.1, stream=True\n", - ")\n", - "\n", - "for out in state.text_iter():\n", - " print(out, end=\"\", flush=True)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Complex Prompts\n", - "\n", - "You may use `{system|user|assistant}_{begin|end}` to define complex prompts." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "@function\n", - "def chat_example(s):\n", - " s += system(\"You are a helpful assistant.\")\n", - " # Same as: s += s.system(\"You are a helpful assistant.\")\n", - "\n", - " with s.user():\n", - " s += \"Question: What is the capital of France?\"\n", - "\n", - " s += assistant_begin()\n", - " s += \"Answer: \" + gen(\"answer\", max_tokens=100, stop=\"\\n\")\n", - " s += assistant_end()\n", - "\n", - "\n", - "state = chat_example()\n", - "print_highlight(state[\"answer\"])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "terminate_process(server_process)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Multi-modal Generation\n", - "\n", - "You may use SGLang frontend language to define multi-modal prompts.\n", - "See [here](https://docs.sglang.io/supported_models/text_generation/multimodal_language_models.html) for supported models." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "server_process, port = launch_server_cmd(\n", - " \"python -m sglang.launch_server --model-path Qwen/Qwen2.5-VL-7B-Instruct --host 0.0.0.0 --log-level warning\"\n", - ")\n", - "\n", - "wait_for_server(f\"http://localhost:{port}\", process=server_process)\n", - "print(f\"Server started on http://localhost:{port}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "set_default_backend(RuntimeEndpoint(f\"http://localhost:{port}\"))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Ask a question about an image." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "@function\n", - "def image_qa(s, image_file, question):\n", - " s += user(image(image_file) + question)\n", - " s += assistant(gen(\"answer\", max_tokens=256))\n", - "\n", - "\n", - "image_url = \"https://raw.githubusercontent.com/sgl-project/sglang/main/examples/assets/example_image.png\"\n", - "image_bytes, _ = load_image(image_url)\n", - "state = image_qa(image_bytes, \"What is in the image?\")\n", - "print_highlight(state[\"answer\"])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "terminate_process(server_process)" - ] - } - ], - "metadata": { - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/docs_new/docs/references/multi_node_deployment/deploy_on_k8s.mdx b/docs_new/docs/references/multi_node_deployment/deploy_on_k8s.mdx index e2070fd1c..2dc940376 100644 --- a/docs_new/docs/references/multi_node_deployment/deploy_on_k8s.mdx +++ b/docs_new/docs/references/multi_node_deployment/deploy_on_k8s.mdx @@ -17,7 +17,7 @@ Here we take the deployment of DeepSeek-R1 as an example. 1. At least two Kubernetes nodes, each with two H20 systems and eight GPUs, are required. -2. Make sure your K8S cluster has LWS correctly installed. If it hasn't been set up yet, please follow the [installation instructions](https://github.com/kubernetes-sigs/lws/blob/main/site/content/en/docs/installation/_index). **Note:** For LWS versions ≤0.5.x, you must use the Downward API to obtain `LWS_WORKER_INDEX`, as native support for this feature was introduced in v0.6.0. +2. Make sure your K8S cluster has LWS correctly installed. If it hasn't been set up yet, please follow the [installation instructions](https://lws.sigs.k8s.io/docs/installation/). **Note:** For LWS versions ≤0.5.x, you must use the Downward API to obtain `LWS_WORKER_INDEX`, as native support for this feature was introduced in v0.6.0. ## Basic example @@ -267,7 +267,7 @@ This should resolve most NCCL-related issues. #### RoCE scenario * Please make sure that RDMA devices are available in the cluster environment. -* Please make sure that the nodes in the cluster have Mellanox NICs with RoCE. In this example, we use Mellanox ConnectX 5 model NICs, and the proper OFED driver has been installed. If not, please refer to the document [Install OFED Driver](https://docs.nvidia.com/networking/display/mlnxofedv461000/installing+mellanox+ofed) to install the driver. +* Please make sure that the nodes in the cluster have Mellanox NICs with RoCE. In this example, we use Mellanox ConnectX 5 model NICs, and the proper OFED driver has been installed. If not, please refer to the document [Install OFED Driver](https://docs.nvidia.com/networking/display/mlnxofedv24102180lts/installing+the+driver) to install the driver. * Check your env: ```shell Command diff --git a/docs_new/docs/references/multi_node_deployment/multi_node_index.rst b/docs_new/docs/references/multi_node_deployment/multi_node_index.rst deleted file mode 100644 index 78636869e..000000000 --- a/docs_new/docs/references/multi_node_deployment/multi_node_index.rst +++ /dev/null @@ -1,14 +0,0 @@ -Multi-Node Deployment -===================== - -.. toctree:: - :maxdepth: 1 - :caption: Multi-Node Deployment - - multi_node.md - deploy_on_k8s.md - lws_pd/lws_pd_deploy.md - rbg_pd/deepseekv32_pd.md - -- `Deploying DeepSeek with PD Disaggregation and Large-Scale Expert Parallelism on 96 H100 GPUs `_ -- `Deploying Kimi K2 with PD Disaggregation and Large-Scale Expert Parallelism on 128 H200 GPUs `_ diff --git a/docs_new/docs_migration_plan.md b/docs_new/docs_migration_plan.md deleted file mode 100644 index 6391c6f9b..000000000 --- a/docs_new/docs_migration_plan.md +++ /dev/null @@ -1,133 +0,0 @@ -# SGLang Documentation Migration Plan - -## Background - -Migrate the new Mintlify-based documentation (currently in the standalone `sgl-docs` repo) into the sglang main repo under `docs_new/`, and point `staging.docs.sglang.io` to it. - -### Current State - -| Item | Location | Stack | Domain | -|------|----------|-------|--------| -| Old docs | `sglang/docs/` | Sphinx + GitHub Pages | `docs.sglang.io` | -| New docs + cookbook | `sgl-project/sgl-docs` repo | Mintlify | `lmsysorg.mintlify.app` (temp preview) | - -- Cookbook is already inside `sgl-docs/cookbook/`, no separate repo needed. -- Old docs CI (`execute-notebook.yml`, `lint.yml`) only watches `docs/**`, will not be triggered by `docs_new/**`. - ---- - -## Phase 1: Git Subtree Merge (Local Experiment) - -> Goal: Merge `sgl-docs` into `sglang` repo's `docs_new/` directory, preserving full commit history and authorship. - -```bash -# 1. Create a new branch (sglang remote is NOT affected) -cd /path/to/sglang -git checkout -b docs-new-migration - -# 2. Add sgl-docs as a remote (sgl-docs repo is NOT affected, read-only fetch) -git remote add sgl-docs git@github.com:sgl-project/sgl-docs.git -git fetch sgl-docs - -# 3. Subtree merge — all sgl-docs content goes into docs_new/, full history preserved -git subtree add --prefix=docs_new sgl-docs main - -# 4. Keep the remote for ongoing sync during migration period -# (remove only after sgl-docs is officially archived) -``` - -### Safety Guarantees - -- `sgl-docs` original repo: **unaffected** (fetch only, no push) -- `sglang` remote: **unaffected** (local branch, no push until ready) -- Rollback: `git checkout main && git branch -D docs-new-migration` - -### Side Effect: Contributors - -`git subtree add` (without `--squash`) imports all original commits. Authors from `sgl-docs` will appear in `sglang`'s git history and GitHub Contributors list. This is intentional — it gives proper credit. - ---- - -## Phase 2: Configure Mintlify for `docs_new/` (on branch) - -> Goal: Make Mintlify read from `sglang` repo's `docs_new/` subdirectory instead of the standalone `sgl-docs` repo. **No need to merge to main first** — Mintlify can point to a specific branch for validation. - -1. Log in to [Mintlify Dashboard](https://dashboard.mintlify.com) -2. Change the project's **GitHub repository** from `sgl-project/sgl-docs` to `sgl-project/sglang` -3. Set **Branch** to `docs-new-migration` (temporarily, for validation) -4. Set **Documentation directory** to `docs_new` (Mintlify supports monorepo subdirectory) -5. `docs.json` (Mintlify config) will be at `docs_new/docs.json` after the subtree merge — paths inside it (e.g., `cookbook/llm/Qwen/Qwen3`) are relative to `docs_new/`, so no changes needed -6. Verify the preview build succeeds on Mintlify - ---- - -## Phase 3: DNS & Custom Domain for `staging.docs.sglang.io` - -> Goal: Make `staging.docs.sglang.io` serve the new Mintlify docs. - -1. **DNS**: Add a CNAME record for `staging.docs.sglang.io` pointing to Mintlify's endpoint (typically `cname.mintlify.dev`) -2. **Mintlify Dashboard**: Settings > Custom Domain > add `staging.docs.sglang.io` -3. Mintlify handles SSL certificate automatically -4. Verify `staging.docs.sglang.io` loads correctly - ---- - -## Phase 4: Ongoing Sync During Migration Period - -> During the transition, `sgl-docs` may still receive updates. Sync them into `docs_new/` as needed. - -```bash -# Pull latest changes from sgl-docs into docs_new/ -git subtree pull --prefix=docs_new sgl-docs main -``` - -Once `sgl-docs` is frozen, this step is no longer needed. - ---- - -## Phase 5: CI/CD (Optional, Post-Migration) - -Current `docs/**` CI workflows will **NOT** trigger for `docs_new/**` changes. This is fine initially since Mintlify has its own GitHub integration for auto-deployment on push to main. - -Optional additions later: -- Link checking (lychee) for `docs_new/**/*.mdx` -- Mintlify broken-link or build validation on PR - ---- - -## Phase 6: Final Cutover - -> Goal: Promote staging to production. - -| Stage | `docs.sglang.io` | `staging.docs.sglang.io` | -|-------|-------------------|--------------------------| -| After Phase 3 | Sphinx (old docs) | Mintlify (new docs) | -| After cutover | Mintlify (new docs) | Keep or remove | - -Cutover steps: -1. Confirm `staging.docs.sglang.io` is stable and content-complete -2. Update DNS: point `docs.sglang.io` CNAME from GitHub Pages to Mintlify (`cname.mintlify.dev`) -3. Update Mintlify Dashboard custom domain to `docs.sglang.io` -4. Remove or archive old resources: - - Delete `sglang/docs/` (old Sphinx docs) - - Delete `.github/workflows/release-docs.yml` and `.github/workflows/execute-notebook.yml` - - Archive `sgl-project/sgl-docs` repo on GitHub - - Remove the `sgl-docs` git remote: `git remote remove sgl-docs` - - Optionally archive `sgl-project/sgl-project.github.io` repo - ---- - -## Execution Order - -> Mintlify supports pointing to a specific branch, so we can validate on `docs-new-migration` **before** merging to main. - -| Step | Action | Who | Dependency | -|------|--------|-----|------------| -| 1 | Phase 1: subtree merge on local branch | Dev | — | -| 2 | Push branch to `sgl-project/sglang` | Dev | Step 1 | -| 3 | Phase 2: configure Mintlify Dashboard to read from `sgl-project/sglang` branch `docs-new-migration` `docs_new/` | Admin (Mintlify access) | Step 2 | -| 4 | Phase 3: DNS CNAME + Mintlify custom domain for `staging.docs.sglang.io` | Admin (DNS access) | Step 3 | -| 5 | Verify staging site | Team | Step 4 | -| 6 | Merge PR to main, switch Mintlify branch back to `main` | Dev + Admin | Step 5 confirmed OK | -| 7 | Phase 4: sync any remaining sgl-docs updates | Dev | As needed | -| 8 | Phase 6: final cutover when ready | Admin | Step 6 done | diff --git a/python/sglang/lang/api.py b/python/sglang/lang/api.py index 745c656ee..a4739fd61 100644 --- a/python/sglang/lang/api.py +++ b/python/sglang/lang/api.py @@ -97,7 +97,7 @@ def gen( regex: Optional[str] = None, json_schema: Optional[str] = None, ): - """Call the model to generate. See the meaning of the arguments in docs/backend/sampling_params.md""" + """Call the model to generate. See the meaning of the arguments in docs_new/docs/basic_usage/sampling_params.mdx""" if choices: return SglSelect( diff --git a/python/sglang/lang/ir.py b/python/sglang/lang/ir.py index f3527d1c9..d74e576d9 100644 --- a/python/sglang/lang/ir.py +++ b/python/sglang/lang/ir.py @@ -473,7 +473,7 @@ class SglGen(SglExpr): regex: Optional[str] = None, json_schema: Optional[str] = None, ): - """Call the model to generate. See the meaning of the arguments in docs/backend/sampling_params.md""" + """Call the model to generate. See the meaning of the arguments in docs_new/docs/basic_usage/sampling_params.mdx""" super().__init__() self.name = name self.sampling_params = SglSamplingParams( diff --git a/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-add-model/SKILL.md b/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-add-model/SKILL.md index 1e9b19d41..44624d50a 100644 --- a/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-add-model/SKILL.md +++ b/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-add-model/SKILL.md @@ -65,7 +65,7 @@ See existing Modular examples: `QwenImagePipeline` (uses `add_standard_t2i_stage | Model/VAE/DiT configs | `python/sglang/multimodal_gen/configs/models/dits/`, `vaes/`, `encoders/` | | Central registry | `python/sglang/multimodal_gen/registry.py` | | Model component registry | `python/sglang/multimodal_gen/runtime/models/registry.py` | -| Current support list | `docs/diffusion/compatibility_matrix.md` | +| Current support list | `docs_new/docs/sglang-diffusion/compatibility_matrix.mdx` | --- diff --git a/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-benchmark-profile/existing-fast-paths.md b/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-benchmark-profile/existing-fast-paths.md index 9f5e9573a..f73f5aa6c 100644 --- a/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-benchmark-profile/existing-fast-paths.md +++ b/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-benchmark-profile/existing-fast-paths.md @@ -32,7 +32,7 @@ framework-specific optimization workflow. - `python/sglang/jit_kernel/norm.py` - `python/sglang/multimodal_gen/runtime/platforms/cuda.py` - `python/sglang/multimodal_gen/runtime/layers/attention/selector.py` -- `docs/diffusion/performance/attention_backends.md` (repo root) +- `docs_new/docs/sglang-diffusion/attention_backends.mdx` (repo root) **Core Fusion Patterns** @@ -121,7 +121,7 @@ framework-specific optimization workflow. - ROCm falls back to native. 2. Attention backend selection (FlashAttention, Sage, SDPA) -- Locations: `platforms/cuda.py`, `attention/selector.py`, `docs/diffusion/performance/attention_backends.md` +- Locations: `platforms/cuda.py`, `attention/selector.py`, `docs_new/docs/sglang-diffusion/attention_backends.mdx` - Behavior: CUDA prefers FlashAttention (FA3/FA4) when supported, otherwise Torch SDPA. Force via `--attention-backend` or `global_force_attn_backend`. 3. FlashInfer RoPE (Q/K inplace) diff --git a/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-modelopt-quant/SKILL.md b/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-modelopt-quant/SKILL.md index 1ac5a5d40..fea581671 100644 --- a/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-modelopt-quant/SKILL.md +++ b/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-modelopt-quant/SKILL.md @@ -25,7 +25,7 @@ This skill owns the ModelOpt-to-SGLang bridge. It is not a generic kernel-tuning - For multi-transformer pipelines, use per-component overrides when different components need different checkpoints. - For B200 NVFP4 validation, keep backend-sensitive environment variables explicit. Wan2.2 NVFP4 is commonly validated with `SGLANG_DIFFUSION_FLASHINFER_FP4_GEMM_BACKEND=cudnn`; benchmark the default CUTLASS path separately if that is what you are evaluating. - When a branch is missing the validated helper tools, refresh `python/sglang/multimodal_gen/tools/build_modelopt_fp8_transformer.py`, `python/sglang/multimodal_gen/tools/build_modelopt_nvfp4_transformer.py`, and `python/sglang/multimodal_gen/tools/compare_diffusion_trajectory_similarity.py` instead of inventing one-off scripts elsewhere. -- After validating a new ModelOpt quant path, update the ModelOpt support matrix in `docs/diffusion/quantization.md` before closing the task. +- After validating a new ModelOpt quant path, update the ModelOpt support matrix in `docs_new/docs/sglang-diffusion/quantization.mdx` before closing the task. ## Read First @@ -71,9 +71,9 @@ Treat a new family, a new precision, or a new checkpoint layout as unsupported u Current B200 CI also contains an Ideogram4 NVFP4 native load case (`ideogram4_nvfp4_t2i` via `Comfy-Org/Ideogram-4`). Treat that as source evidence for an existing NVFP4 path, but do not expand the ModelOpt support -matrix to Ideogram4 unless `docs/diffusion/quantization.md` is updated with the +matrix to Ideogram4 unless `docs_new/docs/sglang-diffusion/quantization.mdx` is updated with the exact checkpoint, loader path, quality check, and benchmark scope. -Before writing CLI examples, re-read the active branch's `docs/diffusion/quantization.md`: FLUX.2 NVFP4 is an official `black-forest-labs/*` repo rather than a `lmsys/*` converted repo, and its preferred flag depends on the current documented loader flow. Use `--transformer-path` for a component override directory with `config.json`; use `--transformer-weights-path` when the repo or path should be probed as raw weights. +Before writing CLI examples, re-read the active branch's `docs_new/docs/sglang-diffusion/quantization.mdx`: FLUX.2 NVFP4 is an official `black-forest-labs/*` repo rather than a `lmsys/*` converted repo, and its preferred flag depends on the current documented loader flow. Use `--transformer-path` for a component override directory with `config.json`; use `--transformer-weights-path` when the repo or path should be probed as raw weights. B200 CI coverage can include loose BF16-vs-quantized quality checks. Inspect the active branch's `run_suite.py` before assuming they are part of the suite; mainline and feature branches may differ. Those checks are intended to catch blank, corrupted, or obviously divergent images, not exact image parity. @@ -100,7 +100,7 @@ accuracy check, and benchmark scope are validated on the active branch. ## Documentation Maintenance -- Keep the validated ModelOpt support matrix in `docs/diffusion/quantization.md`. +- Keep the validated ModelOpt support matrix in `docs_new/docs/sglang-diffusion/quantization.mdx`. - Each row should record the validated scope, the Hugging Face repo or path for the quantized DiT weights, and the key caveats. - If the quantized DiT weights are not published yet, write `unpublished` explicitly instead of leaving the field blank. @@ -402,6 +402,6 @@ When documenting results: | `tools/build_modelopt_fp8_transformer.py` | Build an SGLang-loadable FP8 transformer from a ModelOpt export | | `tools/build_modelopt_nvfp4_transformer.py` | Build mixed BF16+NVFP4 transformer directories when a family needs preserved BF16 layers | | `tools/compare_diffusion_trajectory_similarity.py` | reduced deterministic BF16-vs-quantized validation | -| `docs/diffusion/quantization.md` | public ModelOpt support matrix and CLI examples | +| `docs_new/docs/sglang-diffusion/quantization.mdx` | public ModelOpt support matrix and CLI examples | | `test/server/testcase_configs.py` | reusable ModelOpt testcase constants, thresholds, and helpers | | `test/server/gpu_cases.py` | concrete GPU and B200 ModelOpt CI case lists | diff --git a/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-performance/SKILL.md b/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-performance/SKILL.md index 18be5dae8..2fb75e118 100644 --- a/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-performance/SKILL.md +++ b/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-performance/SKILL.md @@ -42,7 +42,7 @@ These options are intended to preserve output quality. In practice, some paths ( | **FSDP Inference** | `--use-fsdp-inference` | Uses PyTorch FSDP to shard model weights across GPUs with prefetch. Low latency, low VRAM. | Reduces per-GPU VRAM | Mutually exclusive with `--dit-layerwise-offload`. More overhead than SP on high-bandwidth interconnects. | | **CPU Offload (components)** | `--text-encoder-cpu-offload`, `--image-encoder-cpu-offload`, `--vae-cpu-offload`, `--dit-cpu-offload` | Offloads specific pipeline components to CPU when not in use. | Reduces peak VRAM | Adds H2D transfer latency when the component is needed. Auto-enabled for low-VRAM GPUs (<30 GB). **Tip:** after the first request completes, the console prints a peak VRAM analysis with suggestions on which offload flags can be safely disabled — look for the `"Components that could stay resident"` log line. | | **Pin CPU Memory** | `--pin-cpu-memory` | Uses pinned (page-locked) memory for CPU offload transfers. | Faster H2D transfers | Slightly higher host memory usage. Enabled by default; disable only as workaround for CUDA errors. | -| **Attention Backend (lossless)** | `--attention-backend fa` | Selects a lossless attention kernel for SGLang-native pipelines: `fa` (FlashAttention 2/3/4 alias) or `torch_sdpa`. | FA is usually faster than SDPA on long sequences | FA requires compatible GPU (Ampere+). For `--backend diffusers`, valid backend names differ; use the names documented in `docs/diffusion/performance/attention_backends.md`. | +| **Attention Backend (lossless)** | `--attention-backend fa` | Selects a lossless attention kernel for SGLang-native pipelines: `fa` (FlashAttention 2/3/4 alias) or `torch_sdpa`. | FA is usually faster than SDPA on long sequences | FA requires compatible GPU (Ampere+). For `--backend diffusers`, valid backend names differ; use the names documented in `docs_new/docs/sglang-diffusion/attention_backends.mdx`. | | **Parallel Folding** | *(automatic when SP > 1)* | Reuses the SP process group as TP for the T5 text encoder, so text encoding is parallelized "for free". | Faster text encoding on multi-GPU | Automatic; no user action needed. Only applies to T5-based pipelines. | --- diff --git a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/README.md b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/README.md index 18a10cf5c..305190b68 100644 --- a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/README.md +++ b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/README.md @@ -4,7 +4,7 @@ A ComfyUI plugin for integrating with SGLang Diffusion server, supporting image ## Installation -1. **Install SGLang**: Follow the [Installation Guide](../../../../../docs/diffusion/installation.md) to install `sglang[diffusion]`. +1. **Install SGLang**: Follow the [Installation Guide](https://docs.sglang.io/docs/sglang-diffusion/installation) to install `sglang[diffusion]`. 2. **Install Plugin**: Copy this entire directory (`ComfyUI_SGLDiffusion`) to your ComfyUI `custom_nodes/` folder. 3. **Restart ComfyUI**: Restart ComfyUI to load the plugin. diff --git a/python/sglang/srt/disaggregation/ascend/transfer_engine.py b/python/sglang/srt/disaggregation/ascend/transfer_engine.py index 5cef34ae9..435ab8a1d 100644 --- a/python/sglang/srt/disaggregation/ascend/transfer_engine.py +++ b/python/sglang/srt/disaggregation/ascend/transfer_engine.py @@ -31,7 +31,7 @@ class AscendTransferEngine(MooncakeTransferEngine): ): if import_error is not None: logger.warning( - "Please install memfabric_hybrid, for details, see docs/backend/pd_disaggregation.md" + "Please install memfabric_hybrid, for details, see docs_new/docs/advanced_features/pd_disaggregation.mdx" ) raise import_error diff --git a/python/sglang/srt/entrypoints/ollama/README.md b/python/sglang/srt/entrypoints/ollama/README.md index a3f87460a..9141031a7 100644 --- a/python/sglang/srt/entrypoints/ollama/README.md +++ b/python/sglang/srt/entrypoints/ollama/README.md @@ -9,7 +9,7 @@ Ollama API compatibility for SGLang, plus a Smart Router for intelligent routing ## Ollama API -For basic Ollama API usage with SGLang (CLI and Python examples), see the [Ollama API documentation](https://sgl-project.github.io/basic_usage/ollama_api.html). +For basic Ollama API usage with SGLang (CLI and Python examples), see the [Ollama API documentation](https://docs.sglang.io/docs/basic_usage/ollama_api). ## Smart Router diff --git a/python/sglang/srt/sampling/sampling_params.py b/python/sglang/srt/sampling/sampling_params.py index ce4a7c2d8..b30af33bb 100644 --- a/python/sglang/srt/sampling/sampling_params.py +++ b/python/sglang/srt/sampling/sampling_params.py @@ -76,8 +76,7 @@ class SamplingParams(msgspec.Struct, kw_only=True, omit_defaults=True): """ The sampling parameters. - See docs/backend/sampling_params.md or - https://docs.sglang.io/backend/sampling_params.html + See docs_new/docs/basic_usage/sampling_params.mdx for the documentation. """ diff --git a/scripts/ci/check_no_docs_changes.py b/scripts/ci/check_no_docs_changes.py deleted file mode 100755 index b3c02cb5c..000000000 --- a/scripts/ci/check_no_docs_changes.py +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env python3 -"""Reject staged changes under the legacy docs/ tree.""" - -from __future__ import annotations - -import subprocess -import sys - -ERROR_MESSAGE = """\ -Changes under the legacy docs/ directory are not allowed. - -The documentation has been migrated. Please make documentation updates in the -corresponding location under docs_new/ instead. -""" - -LEGACY_DOCS_ALLOWLIST = { - "docs/_static/css/custom_log.css", - "docs/_static/js/deprecation_banner.js", - "docs/conf.py", - # Has relative links into the source tree that the offline lychee check - # validates, so it must be updated when the linked source files move. - "docs/developer_guide/development_jit_kernel_guide.md", -} - - -def staged_paths() -> list[str]: - result = subprocess.run( - [ - "git", - "diff", - "--cached", - "--name-only", - "--diff-filter=ACMRDTUXB", - ], - check=True, - capture_output=True, - text=True, - ) - return [line.strip() for line in result.stdout.splitlines() if line.strip()] - - -def main() -> int: - paths = sys.argv[1:] or staged_paths() - docs_paths = sorted( - path - for path in paths - if (path == "docs" or path.startswith("docs/")) - and path not in LEGACY_DOCS_ALLOWLIST - ) - - if not docs_paths: - return 0 - - print(ERROR_MESSAGE, file=sys.stderr) - print("Detected legacy docs/ changes:", file=sys.stderr) - for path in docs_paths: - print(f" - {path}", file=sys.stderr) - return 1 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/release/README.md b/scripts/release/README.md index eb5af785d..484637695 100644 --- a/scripts/release/README.md +++ b/scripts/release/README.md @@ -16,9 +16,9 @@ python scripts/release/bump_sglang_version.py 0.5.3rc0 - `Makefile` - `benchmark/deepseek_v3/README.md` - `docker/rocm.Dockerfile` -- `docs/get_started/install.md` -- `docs/platforms/amd_gpu.md` -- `docs/platforms/ascend_npu.md` +- `docs_new/docs/get-started/install.mdx` +- `docs_new/docs/hardware-platforms/amd_gpu.mdx` +- `docs_new/docs/hardware-platforms/ascend-npus/ascend_npu.mdx` - `python/pyproject.toml` - `python/pyproject_other.toml` - `python/pyproject_npu.toml` @@ -69,7 +69,7 @@ python scripts/release/bump_kernel_version.py 0.4.0 ```bash grep -r "0.5.4rc0" python/sglang/version.py grep -r "0.5.4rc0" python/pyproject.toml - grep -r "0.5.4rc0" docs/get_started/install.md + grep -r "0.5.4rc0" docs_new/docs/get-started/install.mdx ``` 4. **Reset changes (if testing):**