diff --git a/.claude/skills/ci-test-audit/action-items.md b/.claude/skills/ci-test-audit/action-items.md index cddef4db2..451d0d78f 100644 --- a/.claude/skills/ci-test-audit/action-items.md +++ b/.claude/skills/ci-test-audit/action-items.md @@ -193,7 +193,7 @@ Spot: a command any commenter can trigger; a command that reruns far more than t needs; an unrecognized command that is skipped silently; a reply that names the wrong backend or run. Examples: #35750, #31980, #34057, #37618, #38734, #38736, #36778. -**E5. Fast-fail cascades link only jobs whose failures are correlated.** +**E5. Fail-fast cascades link only jobs whose failures are correlated.** Spot: a scheduled or manually dispatched run whose later jobs are cancelled by an earlier unrelated failure; another platform's lane cancelled by a CUDA failure. Examples: #35392, #35238, #36146. diff --git a/.claude/skills/ci-workflow-guide/SKILL.md b/.claude/skills/ci-workflow-guide/SKILL.md index 0a6ed7b30..55aa38210 100644 --- a/.claude/skills/ci-workflow-guide/SKILL.md +++ b/.claude/skills/ci-workflow-guide/SKILL.md @@ -1,11 +1,11 @@ --- name: ci-workflow-guide -description: Guide to SGLang CI workflow orchestration — stage ordering, fast-fail, gating, partitioning, execution modes, and debugging CI failures. Use when modifying CI workflows, adding stages, debugging CI pipeline issues, or understanding how tests are dispatched and gated across stages. +description: Guide to SGLang CI workflow orchestration — stage ordering, fail-fast, gating, partitioning, execution modes, and debugging CI failures. Use when modifying CI workflows, adding stages, debugging CI pipeline issues, or understanding how tests are dispatched and gated across stages. --- # SGLang CI Workflow Orchestration Guide -This skill covers the CI **infrastructure** layer — how tests are dispatched, gated, and fast-failed across stages. For test authoring (templates, fixtures, registration, model selection), see the [write-sglang-test skill](../write-sglang-test/SKILL.md). +This skill covers the CI **infrastructure** layer — how tests are dispatched, gated, and aborted on failure across stages. For test authoring (templates, fixtures, registration, model selection), see the [write-sglang-test skill](../write-sglang-test/SKILL.md). --- @@ -24,9 +24,10 @@ This skill covers the CI **infrastructure** layer — how tests are dispatched, | `.github/workflows/pr-test.yml` | Main workflow — all stages, jobs, conditions, matrix definitions | | `.github/workflows/pr-test-extra.yml` | Extra workflow — gated by BOTH `run-ci` and `run-ci-extra` labels | | `.github/workflows/pr-gate.yml` | PR gating: draft check, `run-ci` label, per-user rate limiting | -| `.github/actions/check-pr-test-health/action.yml` | Cross-job fast-fail: queries API for any failed job | +| `.github/actions/check-pr-test-health/action.yml` | Cross-job fail-fast: queries API for any failed job | | `.github/actions/wait-for-jobs/action.yml` | Stage gating: polls API until stage jobs complete | | `.github/actions/check-maintenance/action.yml` | Maintenance mode check | +| `.github/scripts/ci-labels.cjs` | Resolves the four CI control labels into dispatch axes | | `test/run_suite.py` | Suite runner: collects, filters, partitions, executes tests | | `python/sglang/test/ci/ci_register.py` | Test registration (AST-parsed markers), LPT auto-partition | | `python/sglang/test/ci/ci_utils.py` | `run_unittest_files()`: execution, retry, continue-on-error | @@ -113,21 +114,21 @@ This skill covers the CI **infrastructure** layer — how tests are dispatched, └─────────────────────────────────────┘ ``` -**Every stage test job** includes a `check-pr-test-health` step after checkout — if any job in the run has already failed, the job fast-fails (red X) with a root cause annotation. +**Every stage test job** includes a `check-pr-test-health` step after checkout — if any job in the run has already failed, the job fails fast (red X) with a root cause annotation. -**Scheduled runs** skip `wait-for-base-*` jobs, running all stages in parallel. Fast-fail is also disabled. +**Scheduled runs** skip `wait-for-base-*` jobs, running all stages in parallel. Fail-fast is also disabled. --- -## Fast-Fail Layers +## Fail-Fast Layers -4 layers of fast-fail, from fine to coarse: +4 layers of fail-fast, from fine to coarse: | Layer | Mechanism | Granularity | Disabled on schedule? | |-------|-----------|-------------|----------------------| | **1. Test method → file** | `unittest -f` (failfast) | One test method fails → entire test file stops immediately | Yes | | **2. File → suite** | `run_unittest_files()` default | One test file fails → entire suite stops (`--continue-on-error` off) | Yes | -| **3. Job → job (same stage)** | `check-pr-test-health` action | One job fails → other waiting jobs in same stage fast-fail (red X) | Yes | +| **3. Job → job (same stage)** | `check-pr-test-health` action | One job fails → other waiting jobs in same stage fail-fast (red X) | Yes | | **4. Stage → stage (cross-stage)** | `wait-for-base-*` + `needs` | Base A fails → base B/C jobs skip entirely (never get a runner) | Yes (wait jobs skipped) | - **Layer 1**: `-f` flag appended to all `python3 -m pytest` / `unittest` invocations in `ci_utils.py` @@ -142,12 +143,17 @@ This skill covers the CI **infrastructure** layer — how tests are dispatched, | Aspect | PR (`pull_request`) | Scheduled (`cron`, every 6h) | Manual dispatch (`workflow_dispatch`) | |--------|---------------------|------------------------------|--------------------------------------| | **Stage ordering** | Sequential: A → B → C via `wait-for-base-*` | Parallel (all at once) | Single target stage only | -| **Cross-job fast-fail** | Yes (`check-pr-test-health`) | Yes | Yes | +| **Cross-job fail-fast** | Yes (`check-pr-test-health`) | Yes | Yes | | **continue-on-error** | No (stop at first failure within suite) | Yes (run all tests) | No | | **Retry** | Enabled | Enabled | Enabled | -| **max_parallel** | 3 (default), 14 if `high priority` label | 14 | 3 (default), 14 if `high priority` | +| **max_parallel** | 3 (default), 14 if `max-concurrency` label | 14 | 3 (default), 14 if `max-concurrency` | | **PR gate** | Yes (draft, label, rate limit) | Skipped | Skipped | -| **Concurrency** | `cancel-in-progress: true` per branch | Queue (no cancel) | Isolated per stage+SHA | +| **Concurrency** | `cancel-in-progress: true` per PR | Queue (no cancel) | Isolated per stage+SHA | + +Four labels relax these limits for one PR: `bypass-fail-fast`, `parallel-stages`, +`max-concurrency`, and `highest-priority` (all three). `.github/scripts/ci-labels.cjs` +resolves them; the [contribution guide](https://docs.sglang.io/developer_guide/contribution_guide.html#ci-control-labels) +describes what each one does. --- @@ -158,7 +164,7 @@ This skill covers the CI **infrastructure** layer — how tests are dispatched, **How it works:** 1. Calls `listJobsForWorkflowRun` to list all jobs in the current run 2. Matches jobs by exact name or prefix (for matrix jobs, e.g., `base-b-test-1-gpu-small (3)`) -3. If any matched job has `conclusion === 'failure'` → fail immediately (fast-fail) +3. If any matched job has `conclusion === 'failure'` → fail immediately (fail-fast) 4. If all matched jobs are completed and count matches `expected_count` → success 5. Otherwise → sleep `poll-interval-seconds` (default: 60s) and retry 6. Timeout after `max-wait-minutes` (240 min for base-a, 480 min for base-b) @@ -179,7 +185,7 @@ This skill covers the CI **infrastructure** layer — how tests are dispatched, --- -## Cross-Job Fast-Fail (`check-pr-test-health` action) +## Cross-Job Fail-Fast (`check-pr-test-health` action) Composite action called after checkout in every stage test job (21 jobs total across `pr-test.yml`, `pr-test-multimodal-gen.yml`, `pr-test-sgl-kernel.yml`, `pr-test-jit-kernel.yml`). @@ -189,7 +195,7 @@ Composite action called after checkout in every stage test job (21 jobs total ac 3. If root cause failures found → calls `core.setFailed()` with the list of root cause job names 4. If none → does nothing (step succeeds) -**Cascade filtering**: When job A fast-fails due to health check, it also has `conclusion: failure`. Without filtering, job B would list both the original failure AND job A's fast-fail. The filter checks each failed job's `steps` array — if the failing step name contains `check-pr-test-health` or `Check PR test health`, it's excluded from the root cause list. +**Cascade filtering**: When job A fails fast due to the health check, it also has `conclusion: failure`. Without filtering, job B would list both the original failure AND job A's fail-fast. The filter checks each failed job's `steps` array — if the failing step name contains `check-pr-test-health` or `Check PR test health`, it's excluded from the root cause list. **Usage pattern:** ```yaml @@ -210,11 +216,11 @@ steps: **Visual effect**: Job shows **red X** (failure) with error annotation showing root cause job names. Subsequent steps are naturally skipped (default `if: success()` is false after a failed step). No per-step `if` guards needed. -**No stage filtering**: Checks ALL jobs in the run, not just the current stage. Any failure anywhere triggers fast-fail. +**No stage filtering**: Checks ALL jobs in the run, not just the current stage. Any failure anywhere triggers fail-fast. **Error message example:** ``` -Fast-fail: skipping — root cause job(s): base-b-test-1-gpu-small (0), base-b-test-1-gpu-small (1) +Fail-fast: skipping — root cause job(s): base-b-test-1-gpu-small (0), base-b-test-1-gpu-small (1) ``` --- diff --git a/.claude/skills/clean-startup-log/SKILL.md b/.claude/skills/clean-startup-log/SKILL.md index c1b9e886f..7f50390a2 100644 --- a/.claude/skills/clean-startup-log/SKILL.md +++ b/.claude/skills/clean-startup-log/SKILL.md @@ -1,282 +1,105 @@ --- name: clean-startup-log -description: Clean up noisy startup warnings and spurious prints in SGLang server logs. Use when users ask to clean up unwanted warnings, deprecation messages, or third-party noise in the server startup output. +description: Audit SGLang startup logs, save evidence, and propose cleanup for user review. With no arguments, run Qwen3-8B at TP1 and TP2 plus gpt-oss-20b at TP1. disable-model-invocation: true --- -# Clean Up SGLang Server Startup Logs - -Goal: ensure the server startup log is clean and minimal, with no spurious warnings, deprecation messages, or unformatted prints from third-party libraries. - -## Workflow - -### 1. Launch a server and capture the log - -```bash -uv run sglang serve --model-path Qwen/Qwen3-8B 2>&1 | tee /tmp/startup_log.txt -``` - -Wait until the server prints `The server is fired up and ready to roll!`, then Ctrl-C. - -For TP>1 testing: -```bash -uv run sglang serve --model-path Qwen/Qwen3-8B --tp 2 2>&1 | tee /tmp/startup_log.txt -``` - -For MoE / hybrid-SWA models (e.g. gpt-oss), test separately — they exercise different code paths: -```bash -uv run sglang serve --model-path openai/gpt-oss-20b 2>&1 | tee /tmp/startup_log.txt -``` - -### 2. Compare against the clean reference log - -Read `/tmp/startup_log.txt` and compare it against the reference log at the bottom of this file. Identify lines that: - -- Do NOT have the `[timestamp]` or `[timestamp TPx]` logger prefix -- Contain `WARNING`, `deprecated`, `is deprecated`, or similar noise -- Are printed by third-party libraries (transformers, torchao, NCCL, Gloo, tqdm, etc.) -- Are duplicate/redundant with information already logged by SGLang -- Appear multiple times due to `ModelConfig` being constructed in multiple processes - -### 3. Classify each noisy line - -For each noisy line, determine: - -| Category | Action | -|----------|--------| -| **SGLang code using wrong API** | Fix the SGLang code (e.g., replace deprecated API with new one) | -| **SGLang code logging at wrong level** | Change log level (e.g., warning -> debug for non-actionable messages) | -| **Duplicated across processes** | Downgrade to debug — info logged in one process becomes noise in 3-4 | -| **Third-party lib prints at import time** | Suppress the logger or redirect stdout during that import | -| **C-level print from .so library** | Redirect fd 1 during the specific C call, or accept it if too invasive | -| **Real warning the user should see** | Keep it | - -### 4. Present findings before fixing - -List all noisy lines with their source and proposed fix. Ask the user to review before making changes. - -### 5. Apply fixes and verify - -After approval, apply fixes one at a time, re-launch the server, and verify each fix works. - -## Key Architecture: Why Logs Repeat - -`ModelConfig` is constructed **3-4 times** during startup across different processes: -1. Main process: `ServerArgs.__post_init__()` → `get_model_config()` → `ModelConfig()` -2. Scheduler subprocess: `Scheduler.init_model_config()` → `ModelConfig.from_server_args()` -3. Scheduler subprocess: `TpModelWorker._init_model_config()` → `ModelConfig.from_server_args()` -4. Main process: `TokenizerManager.init_model_config()` → `ModelConfig.from_server_args()` - -Similarly, `get_tokenizer()` is called **5 times** across processes: -1. `resolve_auto_parsers` (main) — `template_detection.py` -2. `Scheduler.init_tokenizer()` (scheduler subprocess) — `scheduler.py` -3. `DetokenizerManager` (detokenizer subprocess) — `detokenizer_manager.py` -4. `TpModelWorker.__init__()` (scheduler subprocess) — `tp_worker.py` -5. `TokenizerManager` (main) — `tokenizer_manager.py` - -Any `logger.info()` or `logger.warning()` in `ModelConfig.__init__()` or `get_tokenizer()` will appear 3-5 times. **Keep these at `logger.debug()`.** - -## Known Noise Sources and Fixes (from past sessions) - -### 1. torchao "Skipping import of cpp extensions due to incompatible torch version" - -- **Source:** `torchao/__init__.py` — printed via `logger.warning()` when torch version < 2.11.0 -- **Trigger:** `sglang/__init__.py` -> `_apply_hf_patches()` -> `_patch_removed_symbols()` -> `from transformers.models.llama import modeling_llama` -> deep import chain -> `transformers/quantizers/auto.py` -> `from .quantizer_torchao import TorchAoHfQuantizer` -> imports torchao -- **Fix:** In `hf_transformers_patches.py::_patch_removed_symbols()`, temporarily set the `torchao` logger level to `ERROR` around the `modeling_llama` import: - ```python - _torchao_logger = logging.getLogger("torchao") - _prev_level = _torchao_logger.level - _torchao_logger.setLevel(logging.ERROR) - try: - from transformers.models.llama import modeling_llama - finally: - _torchao_logger.setLevel(_prev_level) - ``` - -### 2. "`torch_dtype` is deprecated! Use `dtype` instead!" (PARTIALLY FIXED) - -- **Source:** `transformers/configuration_utils.py` — the `torch_dtype` property warns via `logger.warning_once()` -- **Trigger:** Model files accessing `config.torch_dtype` instead of `config.dtype` -- **Fix applied so far:** Only `models/gpt_oss.py` (lines 222, 471) — tested with `openai/gpt-oss-20b`. -- **Remaining files that still use `config.torch_dtype`** (fix each only after testing with the corresponding model): - - `models/bailing_moe.py` (line 302) - - `models/llada2.py` (line 313) - - `models/qwen3_next.py` (lines 192, 209) - - `models/qwen3_5.py` (line 245) - - `models/nano_nemotron_vl.py` (lines 79, 102, 284) - - `models/llava.py` (lines 732, 734-737) - - `model_loader/loader.py` (line 649) -- **Note:** `common.py` was already fixed in a prior session. If new model files are added with `config.torch_dtype`, the warning will reappear — grep for `\.torch_dtype` to find them. -- **Important:** Only change `config.torch_dtype` → `config.dtype` for models you have actually tested. The `dtype` property should return the same value, but verify per-model to avoid regressions. - -### 3. "`BaseImageProcessorFast` is deprecated" - -- **Source:** `transformers/utils/import_utils.py` — the lazy module `__getattr__` warns when `BaseImageProcessorFast` is accessed -- **Trigger:** `base_processor.py` and `ernie45_vl.py` have `from transformers import BaseImageProcessorFast` at top level. These are imported eagerly via `tokenizer_manager.py` -> `multimodal_processor.py` -> `base_processor.py`, even for non-multimodal models. -- **Fix:** Replace `from transformers import BaseImageProcessorFast` with `from transformers import BaseImageProcessor` and update all `isinstance(..., BaseImageProcessorFast)` checks to `isinstance(..., BaseImageProcessor)` - -### 4. "No platform detected. Using base SRTPlatform with defaults." - -- **Source:** `sglang/srt/platforms/__init__.py` — `logger.warning()` -- **Fix:** Change to `logger.debug()` — this is expected on machines without a platform plugin and not actionable. - -### 5. `NCCL version 2.27.7+cuda13.0` - -- **Source:** C-level print from `libnccl.so` during `ncclCommInitRank()` call -- **Status:** Accepted as-is. SGLang already logs the version via `sglang is using nccl==X.Y.Z`. The C-level print cannot be suppressed without redirecting stdout fd, which is too invasive. `NCCL_DEBUG=WARN` does not suppress it in NCCL 2.27+. - -### 6. `[Gloo] Rank X is connected to Y peer ranks` - -- **Source:** C++ Gloo library print during process group init -- **Status:** Accepted as-is. From C++ code inside PyTorch's Gloo backend. - -### 7. `torchao SyntaxWarning: invalid escape sequence` - -- **Source:** `torchao/quantization/quant_api.py` — a raw string with unescaped `\.` -- **Status:** Upstream torchao bug. Cannot fix from SGLang side. - -### 8. tqdm progress bars (e.g., `Multi-thread loading shards`, `Capturing batches`) - -- **Status:** These are expected and useful. They show progress during weight loading and CUDA graph capture. Keep them. - -### 9. CUTE_DSL "Unexpected error during package walk" — double-logged (FIXED) - -- **Source:** `nvidia-cutlass-dsl` package at `.venv/.../cutlass/cutlass_dsl/cutlass.py`, line 391. Logger named `CUTE_DSL` with its own `StreamHandler`. -- **Trigger:** During CUDA graph capture, cutlass DSL walks packages and hits an unexpected error for `cutlass.cute.experimental`. -- **Root cause of double-logging:** The CUTE_DSL logger has `propagate=True` (default), so the warning is emitted by both the CUTE_DSL handler (with its format) and the root logger (SGLang's format). -- **Fix applied:** In `entrypoints/engine.py`, changed `CUTE_DSL_LOG_LEVEL` from `"30"` (WARNING) to `"40"` (ERROR). This suppresses the WARNING at both the CUTE_DSL logger and root propagation levels. The env var controls both `logger.setLevel()` and `console_handler.setLevel()` in cutlass's `setup_log()`. - -### 10. ModelConfig init logs repeated 3x (FIXED) - -- **Lines:** `"Downcasting torch.float32 to ..."`, `"Hybrid swa model: ..."`, `"DeepGemm is enabled but ..."` -- **Source:** `configs/model_config.py` — `_get_and_verify_dtype()` (line 1457), `_derive_hybrid_model()` (line 497), `_verify_quantization()` (line 1236) -- **Root cause:** `ModelConfig.__init__()` is called 3-4 times in different processes (see "Key Architecture" above). Each construction fires the same log lines. -- **Fix applied:** Downgraded all three from `logger.info()`/`logger.warning()` to `logger.debug()`. The dtype is already visible in `server_args` and `Load weight end`. Hybrid SWA info appears in `Tree cache initialized`. DeepGemm is not actionable. - -### 11. Tokenizer retry/fallback messages repeated 3-4x (FIXED) - -- **Lines:** `"Tokenizer loaded as generic TokenizersBackend ... retrying"`, `"Loading tokenizer ... directly as PreTrainedTokenizerFast"`, `"Tokenizer for ... loaded as generic TokenizersBackend. Set --trust-remote-code"` -- **Source:** `utils/hf_transformers/tokenizer.py` — `_resolve_tokenizers_backend()` (line 215), `_load_tokenizer_by_declared_class()` (line 110), final warning (line 244) -- **Root cause:** 5 separate `get_tokenizer()` calls across processes (see "Key Architecture" above). Each produces 3 log lines. Concurrent subprocess launches cause interleaved/doubled output. -- **Fix applied:** Downgraded all three from `logger.warning()`/`logger.info()` to `logger.debug()`. - -### 12. Template detection logs — 5 lines consolidated to 1 (FIXED) - -- **Lines:** `"Detected reasoning config '...' from template rule '...'"`, `"Detected reasoning parser '...' from template rule '...'"`, `"Detected tool-call parser '...' from template rule '...'"`, `"Auto-detected reasoning parser: ..."`, `"Auto-detected tool-call parser: ..."` -- **Source:** `managers/template_detection.py` (lines 337, 370) logged each detection rule match. `managers/template_manager.py` (lines 177-182) logged summary lines that duplicated the detection logs. -- **Fix applied:** Removed per-rule logs from `template_detection.py`. Consolidated the 5 lines in `template_manager.py` into a single summary: `"Auto-detected template features: reasoning_config=..., reasoning_parser=..., tool_call_parser=..."` - -### 13. KV cache dtype logged separately from allocation (FIXED) - -- **Lines:** `"Using KV cache dtype: torch.bfloat16"` then `"KV Cache is allocated. #tokens: ..., K size: ..., V size: ..."` -- **Source:** `model_executor/model_runner.py` (line 2217) and `mem_cache/memory_pool.py` (line 740) -- **Fix applied:** Removed the standalone dtype log from `model_runner.py`. Added `dtype` field to the allocation log in `memory_pool.py`: `"KV Cache is allocated. dtype: torch.bfloat16, #tokens: ..., K size: ..., V size: ..."` - -### 14. CUTLASS backend warning — B200 → SM100, warning → info (FIXED) - -- **Line:** `"CUTLASS backend is disabled when piecewise cuda graph is enabled due to TMA descriptor initialization issues on B200."` -- **Source:** `layers/attention/flashinfer_backend.py` (line 249) -- **Fix applied:** Changed "B200" to "SM100 GPUs" (the condition checks `is_sm100_supported()` which matches SM10x, not just B200). Downgraded from `logger.warning()` to `logger.info()` since it's an expected automatic fallback. - -### 15. `max_total_num_tokens` and `Tree cache initialized` log ordering - -- **Issue:** `max_total_num_tokens=...` appears before `Tree cache initialized:...` even though tree cache is conceptually part of memory setup. -- **Root cause:** `max_total_num_tokens` is logged inside `init_model_worker()` (scheduler.py:972), which runs before `build_kv_cache()` (scheduler.py:425) where tree cache is created. -- **Status:** Not fixed — reordering was reverted. Acceptable as-is. - -### 16. `Ignore import error when loading sglang.srt.models.midashenglm` - -- **Source:** `models/registry.py` (line 109) — `logger.warning()` during `import_model_classes()` which iterates all model modules via `pkgutil.iter_modules` -- **Trigger:** The `midashenglm` model depends on `torchaudio`, which fails to load -- **Status:** Should be downgraded to `logger.debug()` — not actionable when loading an unrelated model. Same pattern exists in `managers/multimodal_processor.py`, `dllm/algorithm/__init__.py`, `multimodal_gen/runtime/models/registry.py`. - -### 17. `Multiple NUMA nodes found for GPU X` - -- **Source:** `utils/numa_utils.py` (line 112) — `logger.warning()` -- **Status:** Could be downgraded to `logger.info()`. The situation is handled gracefully ("Using the first one") and not actionable. - -### 18. Warmup `/model_info` access log - -- **Source:** Uvicorn access log, triggered by SGLang's own warmup at `entrypoints/http_server.py` (line 1877) -- **Status:** SGLang talking to itself. Could suppress uvicorn access logger during warmup, or exclude `/model_info` from warmup access logging. - -## Investigation Techniques - -### Trace what triggers an import -```python -import sys -_real_import = __builtins__.__import__ -def _tracing_import(name, *args, **kwargs): - if 'TARGET_MODULE' in name: - import traceback - print(f'=== Importing {name} ===') - traceback.print_stack() - return _real_import(name, *args, **kwargs) -__builtins__.__import__ = _tracing_import -``` - -### Trace what triggers a logger warning -```python -import logging, traceback -class TraceHandler(logging.Handler): - def emit(self, record): - if 'SEARCH_STRING' in record.getMessage(): - traceback.print_stack() -h = TraceHandler() -h.setLevel(logging.WARNING) -logging.getLogger('TARGET_LOGGER_NAME').addHandler(h) -``` - -### Find C-level prints in .so files -```bash -strings /path/to/library.so | grep "SEARCH_STRING" -``` - -### Find all config.torch_dtype accesses (for deprecation warning) -```bash -grep -rn '\.torch_dtype' python/sglang/srt/models/ python/sglang/srt/model_loader/ python/sglang/srt/utils/hf_transformers/ -``` - -## Reference: Clean Startup Log (TP=1, Qwen3-8B) - -``` -[2026-05-24 00:52:39] Attention backend not specified. Use trtllm_mha backend by default. -[2026-05-24 00:52:39] TensorRT-LLM MHA only supports page_size of 16, 32 or 64, changing page_size from None to 64. -[2026-05-24 00:52:40] server_args=ServerArgs(model_path='Qwen/Qwen3-8B', ...) -[2026-05-24 00:52:40] Multiple NUMA nodes found for GPU 0: [...]. Using the first one. -[2026-05-24 00:52:42] Using default HuggingFace chat template with detected content format: string -[2026-05-24 00:52:42] Auto-detected template features: reasoning_config=..., reasoning_parser=qwen3, tool_call_parser=qwen -[2026-05-24 00:52:50] Init torch distributed begin. -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[2026-05-24 00:52:50] Init torch distributed ends. elapsed=0.21 s, mem usage=0.10 GB -[2026-05-24 00:52:51] Load weight begin. avail mem=275.75 GB -[2026-05-24 00:52:51] Found local HF snapshot for Qwen/Qwen3-8B at ...; skipping download. -Multi-thread loading shards: 100% Completed | 5/5 [00:01<00:00, 2.62it/s] -[2026-05-24 00:52:54] Load weight end. elapsed=2.62 s, type=Qwen3ForCausalLM, avail mem=260.48 GB, mem usage=15.28 GB. -[2026-05-24 00:52:54] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1707904, K size: 117.28 GB, V size: 117.28 GB -[2026-05-24 00:52:54] Memory pool end. avail mem=25.28 GB -[2026-05-24 00:52:54] CUTLASS backend is disabled when piecewise cuda graph is enabled due to TMA descriptor initialization issues on SM100 GPUs. Using auto backend instead for stability. -[2026-05-24 00:52:54] Capture cuda graph begin. This can take up to several minutes. avail mem=24.16 GB -[2026-05-24 00:52:54] Capture cuda graph bs [1, 2, 4, ...] -Capturing batches (bs=1 avail_mem=23.56 GB): 100% | 52/52 [00:05<00:00, 10.36it/s] -[2026-05-24 00:53:00] Capture cuda graph end. Time elapsed: 5.38 s. mem usage=0.60 GB. avail mem=23.56 GB. -[2026-05-24 00:53:00] Capture piecewise CUDA graph begin. avail mem=23.56 GB -[2026-05-24 00:53:00] Capture cuda graph num tokens [4, 8, 12, ...] -Compiling num tokens (num_tokens=4): 100% | 74/74 [00:09<00:00, 7.44it/s] -Capturing num tokens (num_tokens=4 avail_mem=21.24 GB): 100% | 74/74 [00:07<00:00, 10.44it/s] -[2026-05-24 00:53:18] Capture piecewise CUDA graph end. Time elapsed: 18.18 s. mem usage=2.32 GB. avail mem=21.24 GB. -[2026-05-24 00:53:20] Tree cache initialized: source=default impl=RadixCache hybrid_swa=False hybrid_ssm=False hierarchical=False streaming_wrapped=False -[2026-05-24 00:53:20] max_total_num_tokens=1707904, chunked_prefill_size=16384, max_prefill_tokens=16384, max_running_requests=4096, context_len=40960, available_gpu_mem=21.24 GB -[2026-05-24 00:53:20] INFO: Started server process [1964249] -[2026-05-24 00:53:20] INFO: Waiting for application startup. -[2026-05-24 00:53:20] Using default chat sampling params from model generation config: {'temperature': 0.6, 'top_k': 20, 'top_p': 0.95} -[2026-05-24 00:53:20] INFO: Application startup complete. -[2026-05-24 00:53:20] INFO: Uvicorn running on http://127.0.0.1:30000 (Press CTRL+C to quit) -[2026-05-24 00:53:21] Prefill batch, #new-seq: 1, #new-token: 64, ... -[2026-05-24 00:53:21] INFO: 127.0.0.1:... - "POST /generate HTTP/1.1" 200 OK -[2026-05-24 00:53:21] The server is fired up and ready to roll! -``` - -Note: `[Gloo]` messages and tqdm progress bars are acceptable. The key is no warnings or deprecation messages from transformers, torchao, or other third-party libraries. The `CUTLASS backend is disabled` message is now `info` level, not a warning. +# Audit SGLang Startup Logs + +The default outcome is saved logs and a findings report. Apply runtime changes +only after the user selects them. A request to edit this skill does not itself +launch servers. + +## Default runs + +A bare `$clean-startup-log` invocation runs these cases sequentially without +asking for commands. Explicit commands, models, or TP sizes replace this matrix. + +| Case / log filename | Command | +|---|---| +| `qwen3-8b-tp1.log` | `uv run sglang serve --model-path Qwen/Qwen3-8B` | +| `qwen3-8b-tp2.log` | `uv run sglang serve --model-path Qwen/Qwen3-8B --tp 2` | +| `gpt-oss-20b-tp1.log` | `uv run sglang serve --model-path openai/gpt-oss-20b` | + +These cover dense, tensor-parallel, and MoE/hybrid sliding-window attention +startup. Reuse complete captures from the current audit when code and environment +have not changed. + +## Capture logs + +1. Check the checkout, free GPUs, and ports once. Use the requested command when + resources are free; otherwise select free GPUs with `CUDA_VISIBLE_DEVICES` and + an unused `--port`, recording the adjustments. Leave existing servers alone. +2. Create a unique directory with `mktemp -d /tmp/sglang-startup-audit-XXXXXX`. + Save raw stdout and stderr together in a separate log for each case, for + example with `set -o pipefail` and `COMMAND 2>&1 | tee LOG_PATH`. Record commands, + GPU IDs, ports, commit, relevant overrides, and readiness status. +3. Wait for `The server is fired up and ready to roll!`, then stop that server and + its workers before the next case. First runs can spend many minutes downloading + weights or compiling FlashInfer kernels; check download/compiler activity + before treating a quiet log as a hang. Preserve partial logs for failed or + stalled starts and report the last stage. Continue independent cases when possible. +4. Preserve the user's logging configuration, including `NCCL_DEBUG`. If NCCL + verbosity needs explaining, inspect relevant shell settings, `NCCL_CONF_FILE`, + and `/etc/nccl.conf`. Do not override intentional diagnostics or recommend + `NCCL_DEBUG=WARN` solely because the output is long. Avoid full environment dumps. + +## Investigate efficiently + +- Scan for deprecations, duplicate handler output, unrelated import failures, + unformatted prints, and unexpected warnings. Read representative excerpts and + counts instead of repeatedly dumping `server_args`, progress redraws, or NCCL + diagnostics. Normalize carriage returns for analysis only; preserve raw logs. +- Trace each candidate to its actual emitter with focused `rg` searches. Inspect + its log level: SGLang's formatter may omit severity. Group shared signatures + across cases and distinguish handler duplication from separate GPU/process calls. +- Repetition, WARNING severity, or a different third-party format alone does not + establish a cleanup need. Consider whether the message explains configuration, + progress, resource use, or an operational limitation. +- Consult [noise-source hints](references/noise-sources.md) only for a matching + signature or an unresolved emitter. Verify current code rather than trusting + historical line numbers, fix status, or assumptions about unrelated models. + +## Accepted output + +Preserve these reviewed messages unless the user requests a different policy: + +- NCCL diagnostics enabled by the user's environment or host configuration. +- NUMA permission warnings, including one check per GPU in TP runs. +- GPT-OSS MXFP4 backend-selection warnings and default page-size selection warnings. +- `Init Unified Radix Cache. Components: ... Tree Core: ...`, tree-cache summaries, + SWA allocation details, and per-rank memory/timing records. +- Useful progress bars, warmup HTTP access logs, uv synchronization messages, + isolated NCCL/Gloo startup lines, and one timestamped HF authentication warning. + +These can appear in a clean startup log. Do not repeatedly propose the declined +NUMA deduplication, backend/page-size level changes, or NCCL verbosity override. +Keep real operational warnings visible: for example, a Harmony vocabulary failure +can disable `/v1/responses` even when server readiness and `/generate` succeed. + +## Report before changing code + +Return a compact run table with readiness status and clickable raw-log links. +For each actual cleanup candidate, give an exact representative message, affected +cases/counts, source file/function, and specific proposed behavior. Distinguish +confirmed findings from suspicions and operational failures from logging noise. + +If there are no actionable cleanup findings, say the logs are clean and no +further cleanup is needed. Otherwise, ask which numbered changes to adopt and +wait for the user's selections before editing runtime code or preparing patches. +Honor existing approvals and declined items without asking again. + +## Apply selected changes + +- Batch compatible approved edits, then verify affected cases once. Repeat a + startup only for a new change, failure, or unresolved concern; do not relaunch + after every one-line edit. Save verification logs separately from baselines. +- Preserve useful warnings and application handlers. HF can warn during early + CLI model detection before `configure_logger()`, and spawned processes have + independent logger state. Keep `configure_hf_hub_logger()` in both + `suppress_noisy_warnings()` and `configure_logger()`; make repeated setup safe. +- Keep the legacy compiled-kernel cache migration notice at DEBUG. Avoid broad + library-level suppression or fd redirection for a narrow logging problem. +- Run relevant formatting and focused existing checks. Add tests only when they + verify meaningful behavior, not a log-level spelling. Report changes and + verification; create branches, commits, and PRs when requested. diff --git a/.claude/skills/clean-startup-log/references/noise-sources.md b/.claude/skills/clean-startup-log/references/noise-sources.md new file mode 100644 index 000000000..f59d55f04 --- /dev/null +++ b/.claude/skills/clean-startup-log/references/noise-sources.md @@ -0,0 +1,70 @@ +# Startup noise-source hints + +Read this only for an observed signature that needs investigation. Paths and +behaviors can change; search the current checkout and installed dependency before +proposing a fix. The accepted-output policy in `../SKILL.md` takes precedence. +These are investigation hints, not a list of changes to apply. + +## Common signatures + +| Observed output | Where to investigate | Decision to verify | +|---|---|---| +| HF warning printed in two formats | `utils/common.py`, HF logger handlers and propagation | A library handler plus root propagation can emit the same record twice. Keep one timestamped copy, including before full server logging setup. | +| `Skipping import of cpp extensions due to incompatible torch version` | `torchao/__init__.py`; imports through `hf_transformers_patches.py` and transformers quantizers | Determine whether torchao is needed for the requested model before proposing narrow import-time suppression. | +| `torch_dtype` is deprecated | Model code, loader, and HF utilities accessing `config.torch_dtype` | Prefer `config.dtype` where supported; change and verify only affected, tested models. | +| `BaseImageProcessorFast` is deprecated | Multimodal processor imports and type checks | Check the installed transformers replacement and compatible `isinstance` behavior. | +| `No platform detected. Using base SRTPlatform with defaults.` | Platform detection | Decide whether fallback is expected on this machine or indicates a missing plugin. | +| `Unexpected error during package walk` in CUTE_DSL | Installed cutlass package, its handler, root propagation, and `CUTE_DSL_LOG_LEVEL` | Inspect the actual exception and duplicate-handler path; do not assume the current log level or hide all warnings. | +| Repeated dtype, hybrid-model, or tokenizer fallback messages | Model-config and tokenizer construction in launcher, scheduler, and detokenizer | Multiple constructors may explain repetition. Retain meaningful per-process information and distinguish these from handler duplication. | +| Repeated template detection messages | `managers/template_detection.py` and `managers/template_manager.py` | Check whether a summary already contains the same information. | +| KV cache dtype logged separately from allocation | Model runner and memory pool | Check whether the allocation summary already carries dtype; retain useful allocation and SWA details. | +| CUTLASS backend disabled during graph capture | Attention backend selection | Expected fallback may be useful information; retain the reason and selected behavior. | +| `Ignore import error when loading ...` | Model, multimodal, or algorithm registries | Distinguish an unused optional model dependency from failure to load the requested model. | +| `Multiple NUMA nodes found for GPU ...` | `utils/numa_utils.py` | This differs from the accepted NUMA permission warning; evaluate its context separately. | +| `OpenAI Responses API (/v1/responses) disabled` | `entrypoints/http_server.py`, serving responses, and Harmony vocabulary loading | A real endpoint limitation should remain visible even if generation warmup succeeds. | +| `SyntaxWarning: invalid escape sequence` from a dependency | Installed package file named by the warning | Identify the upstream issue; avoid editing the installed package as a repository fix. | + +Repeated constructors and separate processes are not blanket reasons to downgrade +messages. Constructor counts and import paths vary by model and checkout. Likewise, +cache-summary ordering and native library formats need no change merely because +they differ from a historical reference log. + +## Targeted tracing + +Start with source searches. For example: + +```bash +rg -n 'SEARCH_STRING' python/sglang/srt/FOCUSED_DIRECTORY +rg -n '\.torch_dtype' python/sglang/srt/models/MODEL.py +``` + +If the emitter is clear but its call path is not, attach a temporary handler in an +isolated reproduction: + +```python +import logging +import traceback + + +class TraceHandler(logging.Handler): + def emit(self, record): + if "SEARCH_STRING" in record.getMessage(): + traceback.print_stack() + + +target = logging.getLogger("TARGET_LOGGER_NAME") +handler = TraceHandler() +target.addHandler(handler) +try: + reproduce_observed_warning() +finally: + target.removeHandler(handler) +``` + +For suspected native output, search only the identified shared library: + +```bash +strings /path/to/library.so | rg -F 'SEARCH_STRING' +``` + +Keep diagnostic instrumentation out of committed fixes and baseline logs. diff --git a/.claude/skills/cookbook-add-model/SKILL.md b/.claude/skills/cookbook-add-model/SKILL.md index 2d5e722fd..803ae11c7 100644 --- a/.claude/skills/cookbook-add-model/SKILL.md +++ b/.claude/skills/cookbook-add-model/SKILL.md @@ -169,6 +169,27 @@ least one real deployment or capability boundary. Put orthogonal runtime feature variant/quant/strategy needs its own image), `multiNodeHints` only for fabric-specific hw (e.g. gb200). +5. **Diffusion pages: add the ComfyUI section.** Every diffusion cookbook page ends with + a `## . Run in ComfyUI` section so a reader never has to guess whether the model is + reachable from ComfyUI. It is one component; the per-model facts live in the component, + not the page: + + ```mdx + ## . Run in ComfyUI + + import { ComfyUISupport } from '/src/snippets/diffusion/comfyui-support.jsx'; + + + ``` + + Pick `model` from the table in + `docs/src/snippets/diffusion/comfyui-support.jsx`. Use the model's own key when it has + an entry (its executor or dedicated node differs); otherwise use the generic `image` or + `video`. A model gets its own key only when the plugin actually treats it specially — + an entry in `executor_class_dict` + (`python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/generator.py`) or a + dedicated node. Adding a key without the matching plugin support makes the page lie. + ### Site-wiring (do all three) - **`docs/docs.json`** — add the page under Cookbook → `` → ``, at diff --git a/.claude/skills/cookbook-review-pr/SKILL.md b/.claude/skills/cookbook-review-pr/SKILL.md index d7ccaf46a..27d29f087 100644 --- a/.claude/skills/cookbook-review-pr/SKILL.md +++ b/.claude/skills/cookbook-review-pr/SKILL.md @@ -147,6 +147,18 @@ than restating. equal what the engine emits from the corresponding cell — same flags, same order. Drift here is the most common review miss. +### 5b. ComfyUI section (diffusion pages) +- A diffusion page ends with `## . Run in ComfyUI` rendering ``. A + reader must not have to guess whether the model is reachable from ComfyUI. +- The `model` prop is a key that exists in `docs/src/snippets/diffusion/comfyui-support.jsx`. + A model-specific key is only correct when the plugin really treats it specially — an entry + in `executor_class_dict` + (`python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/generator.py`) or a dedicated + node. Otherwise the generic `image` / `video` key is the honest one; a model-specific key + without matching plugin support makes the page claim support that does not exist. +- Prose describing ComfyUI support inline instead of using the component is a finding: the + facts drift from the plugin. + ### 6. Commands / port - Launch uses `sglang serve` — flag any `python -m sglang.launch_server` / `python3 -m sglang.launch_server` (deprecated). The engine already emits `sglang serve`; diff --git a/.claude/skills/write-sglang-test/SKILL.md b/.claude/skills/write-sglang-test/SKILL.md index 6ae221361..c932523dd 100644 --- a/.claude/skills/write-sglang-test/SKILL.md +++ b/.claude/skills/write-sglang-test/SKILL.md @@ -5,7 +5,7 @@ description: Guide for writing SGLang CI/UT tests. Covers CustomTestCase, CI reg # Writing SGLang CI / UT Tests -This skill covers **how to write and register tests**. For CI pipeline internals (stage ordering, fast-fail, gating, partitioning, debugging CI failures), see the [CI workflow guide](../ci-workflow-guide/SKILL.md). Whether a case is worth adding at all is decided by [`unit-test-admission`](../../rules/unit-test-admission.md) — read it before writing the case, not after. +This skill covers **how to write and register tests**. For CI pipeline internals (stage ordering, fail-fast, gating, partitioning, debugging CI failures), see the [CI workflow guide](../ci-workflow-guide/SKILL.md). Whether a case is worth adding at all is decided by [`unit-test-admission`](../../rules/unit-test-admission.md) — read it before writing the case, not after. ## Core Rules diff --git a/.github/CI_PERMISSIONS.json b/.github/CI_PERMISSIONS.json index 2ae065282..217a5ab7f 100644 --- a/.github/CI_PERMISSIONS.json +++ b/.github/CI_PERMISSIONS.json @@ -731,6 +731,12 @@ "cooldown_interval_minutes": 0, "reason": "top contributor" }, + "iyastreb": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "custom override" + }, "jason-fxz": { "can_tag_run_ci_label": true, "can_rerun_failed_ci": true, @@ -900,6 +906,12 @@ "cooldown_interval_minutes": 60, "reason": "custom override" }, + "lluki": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "custom override" + }, "luccafong": { "can_tag_run_ci_label": true, "can_rerun_failed_ci": true, @@ -978,6 +990,12 @@ "cooldown_interval_minutes": 0, "reason": "custom override" }, + "niehen6174": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 60, + "reason": "custom override" + }, "nvcastet": { "can_tag_run_ci_label": true, "can_rerun_failed_ci": true, @@ -1014,6 +1032,12 @@ "cooldown_interval_minutes": 0, "reason": "custom override" }, + "ovidiusm": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "custom override" + }, "pansicheng": { "can_tag_run_ci_label": true, "can_rerun_failed_ci": true, diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 76d6e687f..014a9a167 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -18,6 +18,7 @@ /python/sglang/srt/disaggregation/ascend @ping1jing2 @iforgetmyname /python/sglang/srt/disaggregation/encoder @ShangmingCai @liusy58 @ZhengWG @gty111 /python/sglang/srt/disaggregation/mori @Duyi-Wang @kkHuang-amd @HaiShaw @Lzy17 @billishyahao +/python/sglang/srt/disaggregation/nixl @iyastreb @ovidiusm @lluki /python/sglang/srt/distributed @yizhang2077 @merrymercy @ch-wan /python/sglang/srt/distributed/device_communicators/mooncake_transfer_engine.py @ShangmingCai @stmatengss /python/sglang/srt/dllm @ClawSeven @btw616 @rwang5203 @@ -55,6 +56,7 @@ /python/sglang/srt/mem_cache/allocator @hnyls2002 @hzh0425 @xiezhq-hermann @ispobock @alphabetc1 @huangtingwei9988 /python/sglang/srt/mem_cache/rust_tree_core @Jialin @hzh0425 @xiezhq-hermann @ispobock @alphabetc1 /python/sglang/srt/mem_cache/storage/mooncake_store @huangtingwei9988 @stmatengss +/python/sglang/srt/mem_cache/storage/nixl @iyastreb @ovidiusm @lluki /python/sglang/srt/mem_cache/embedding_*.py @liusy58 /python/sglang/srt/mem_cache/storage/mooncake_store/mooncake_embedding_store.py @liusy58 /python/sglang/srt/model_executor @merrymercy @Ying1123 @hnyls2002 @Fridge003 @ispobock @@ -73,6 +75,7 @@ /python/sglang/kernels/aot @ispobock @BBuf @yizhang2077 @merrymercy @FlamingoPg @HaiShaw /python/sglang/kernels/aot/csrc/musa @yeahdongcn /rust/sglang-radix-tree @Jialin @hzh0425 @xiezhq-hermann @ispobock @alphabetc1 +/rust/sglang-renderer @sagearc /sgl-model-gateway @slin1237 @CatherineSue /sgl-model-gateway/benches @slin1237 /sgl-model-gateway/bindings/python @CatherineSue @key4ng @slin1237 diff --git a/.github/MAINTAINER.md b/.github/MAINTAINER.md index 4aac1716c..42e863b48 100644 --- a/.github/MAINTAINER.md +++ b/.github/MAINTAINER.md @@ -143,6 +143,17 @@ This section lists the oncalls for each hardware platform. The format is @github This list is based on the current situation. If you or someone you know would like to donate machines for CI, they can serve as the CI oncalls for their machines. Please ping [Lianmin Zheng](https://github.com/merrymercy) and [Ying Sheng](https://github.com/Ying1123) in the Slack channel. They will start a nomination and internal review process. +## CI Control Labels + +`bypass-fail-fast`, `parallel-stages`, `max-concurrency` and `highest-priority` +each relax one of the limits that keep a single PR from monopolizing the +self-hosted GPU runners; `highest-priority` relaxes all of them at once. The +[contribution guide](https://docs.sglang.io/developer_guide/contribution_guide.html#ci-control-labels) +describes what each one does. + +Applying one spends other PRs' runner capacity. `parallel-stages` is the +expensive one: a PR that cannot pass now runs its whole matrix. + ## CI Maintenance Mode When the CI is unhealthy (e.g., the scheduled pr-test on `main` is broken for consecutive runs), the project enters **CI Maintenance Mode** by opening [issue #21065](https://github.com/sgl-project/sglang/issues/21065). While active: - All PR CI runs are paused. Resources are allocated to PRs that fix the CI. diff --git a/.github/actions/check-pr-test-health/action.test.cjs b/.github/actions/check-pr-test-health/action.test.cjs new file mode 100644 index 000000000..d31120f2c --- /dev/null +++ b/.github/actions/check-pr-test-health/action.test.cjs @@ -0,0 +1,76 @@ +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const { test } = require('node:test'); + +const yaml = fs.readFileSync(path.join(__dirname, 'action.yml'), 'utf8'); +const script = yaml.split(' script: |\n')[1] + .split('\n').map(line => line.replace(/^ /, '')).join('\n'); +// `require` and GITHUB_WORKSPACE mirror what actions/github-script injects. +const WORKSPACE = path.join(__dirname, '..', '..', '..'); +const run = new (Object.getPrototypeOf(async function () {}).constructor)( + 'github', 'context', 'core', 'process', 'require', script, +); + +async function check({ snapshot = [], live = [], lint = 'success', event = 'pull_request' } = {}) { + const failures = []; + let labelReads = 0; + let jobReads = 0; + const labels = names => names.map(name => ({ name })); + const github = { + rest: { + checks: { listForRef: async () => ({ data: { check_runs: [ + { app: { slug: 'github-actions' }, status: 'completed', conclusion: lint }, + ] } }) }, + pulls: { get: async args => { + assert.equal(args.pull_number, 42); + labelReads++; + return { data: { labels: labels(live) } }; + } }, + repos: { listPullRequestsAssociatedWithCommit: async () => { + labelReads++; + return { data: [{ labels: labels(live) }] }; + } }, + actions: { listJobsForWorkflowRun: () => {} }, + }, + paginate: async () => { + jobReads++; + return [{ name: 'model-test', status: 'completed', conclusion: 'failure', steps: [] }]; + }, + }; + await run(github, { + eventName: event, repo: { owner: 'owner', repo: 'repo' }, sha: 'head', runId: 1, + payload: event === 'pull_request' + ? { pull_request: { number: 42, head: { sha: 'head' }, labels: labels(snapshot) } } + : {}, + }, { info: () => {}, setFailed: message => failures.push(message) }, + { env: { GITHUB_WORKSPACE: WORKSPACE } }, require); + return { failures, labelReads, jobReads }; +} + +test('a label added after the event bypasses sibling failures on rerun', async () => { + assert.deepEqual(await check({ live: ['bypass-fail-fast'] }), + { failures: [], labelReads: 1, jobReads: 0 }); +}); + +test('a removed label does not continue bypassing sibling failures', async () => { + const result = await check({ snapshot: ['bypass-fail-fast'] }); + assert.equal(result.labelReads, 1); + assert.equal(result.jobReads, 1); + assert.match(result.failures[0], /root cause job\(s\): model-test/); +}); + +test('bypass never skips a failed lint check', async () => { + assert.deepEqual(await check({ live: ['bypass-fail-fast'], lint: 'failure' }), + { failures: ['Fail-fast: lint check failed'], labelReads: 0, jobReads: 0 }); +}); + +test('non-PR events retain associated-PR label lookup', async () => { + assert.deepEqual(await check({ event: 'workflow_dispatch', live: ['bypass-fail-fast'] }), + { failures: [], labelReads: 1, jobReads: 0 }); +}); + +test('scheduled runs remain exempt', async () => { + assert.deepEqual(await check({ event: 'schedule' }), + { failures: [], labelReads: 0, jobReads: 0 }); +}); diff --git a/.github/actions/check-pr-test-health/action.yml b/.github/actions/check-pr-test-health/action.yml index 8cafb66e9..4b1e42f79 100644 --- a/.github/actions/check-pr-test-health/action.yml +++ b/.github/actions/check-pr-test-health/action.yml @@ -1,5 +1,5 @@ name: Check PR Test Health -description: Fail fast if any job in the current workflow run has already failed, or if the lint check (from lint.yml) has failed. Auto-skips for scheduled runs. The jobs-failed check (but not the lint check) is bypassed when the PR carries the `bypass-fastfail` label. +description: Fail fast if any job in the current workflow run has already failed, or if the lint check (from lint.yml) has failed. Auto-skips for scheduled runs. The jobs-failed check (but not the lint check) is bypassed when the PR carries the `bypass-fail-fast` label (or `highest-priority`, which implies it). inputs: github-token: @@ -17,15 +17,17 @@ runs: with: github-token: ${{ inputs.github-token }} script: | + core.info(`[health-check] START -- event=${context.eventName}, runId=${context.runId}`); + // Skip when explicitly requested via env var (e.g. release branch cut) if (process.env.SKIP_PR_TEST_HEALTH_CHECK === 'true') { - core.info('Skipping health check (SKIP_PR_TEST_HEALTH_CHECK=true)'); + core.info('[health-check] SKIP: SKIP_PR_TEST_HEALTH_CHECK=true'); return; } - // Skip for scheduled runs — they should collect all failures, not fast-fail + // Skip for scheduled runs -- they should collect all failures, not fail-fast if (context.eventName === 'schedule') { - core.info('Skipping health check for scheduled run'); + core.info('[health-check] SKIP: scheduled run'); return; } @@ -33,6 +35,7 @@ runs: // listJobsForWorkflowRun only sees jobs within the SAME run, so we use // checks.listForRef which queries by commit SHA across ALL workflows. const ref = context.payload.pull_request?.head?.sha || context.sha; + core.info(`[health-check] Checking lint for ref=${ref}`); const { data } = await github.rest.checks.listForRef({ owner: context.repo.owner, repo: context.repo.repo, @@ -42,28 +45,20 @@ runs: const lintRun = data.check_runs.find( cr => cr.app?.slug === 'github-actions' ); + core.info(`[health-check] Lint check: status=${lintRun?.status}, conclusion=${lintRun?.conclusion}`); if (lintRun?.status === 'completed' && lintRun?.conclusion === 'failure') { - core.setFailed('Fast-fail: lint check failed'); + core.setFailed('Fail-fast: lint check failed'); return; } - // Skip the jobs-failed check when the PR carries the bypass-fastfail label. - // Lint check above still runs. - let labels = []; - if (context.payload.pull_request?.labels) { - labels = context.payload.pull_request.labels.map(l => l.name); - } else { - const { data: prs } = await github.rest.repos.listPullRequestsAssociatedWithCommit({ - owner: context.repo.owner, - repo: context.repo.repo, - commit_sha: ref, - }); - if (prs.length > 0) { - labels = prs[0].labels.map(l => l.name); - } - } - if (labels.includes('bypass-fastfail')) { - core.info('Skipping jobs-failed check (bypass-fastfail label present)'); + // The lint check above is never bypassed; only sibling failures are. + const { resolveCiLabels } = require( + `${process.env.GITHUB_WORKSPACE}/.github/scripts/ci-labels.cjs` + ); + const axes = await resolveCiLabels(github, context); + core.info(`[health-check] PR labels: [${axes.labels.join(', ')}]`); + if (axes.bypassFailFast) { + core.info('[health-check] SKIP jobs-failed check: bypass-fail-fast label present'); return; } @@ -73,26 +68,41 @@ runs: run_id: context.runId, per_page: 100, }); - // Find jobs that failed from a real error, not from fast-fail cascade + core.info(`[health-check] Total jobs in run: ${jobs.length}`); + const failedJobs = jobs.filter(j => j.status === 'completed' && j.conclusion === 'failure'); + core.info(`[health-check] Failed jobs (before filtering): ${failedJobs.map(j => `${j.name}(${j.conclusion})`).join(', ') || 'none'}`); + + // Find jobs that failed from a real error, not from fail-fast cascade const rootCauseFailures = jobs.filter(j => { if (j.status !== 'completed' || j.conclusion !== 'failure') return false; // h20 runners are flaky (dirty GPU state from prior runs); their failures - // should not cascade fast-fail to other stages. j.name shape from + // should not cascade fail-fast to other stages. j.name shape from // listJobsForWorkflowRun: "" + optional " / " // + optional " ()". Split off the base job key before exact // match so we cover both inline + reusable forms without confusing // 'h20' with the 'h200' prefix. const baseName = j.name.split(/[ /]/)[0]; if (baseName === 'base-c-test-8-gpu-h20') { + core.info(`[health-check] Filtered out h20 job: ${j.name}`); return false; } - // If the failing step is the health check, it's a cascade — skip it + // multimodal-gen NPU tests should not cascade fail-fast to perf/accuracy stages. + if (baseName === 'multimodal-gen-test-1-npu-a3' || baseName === 'multimodal-gen-test-2-npu-a3') { + core.info(`[health-check] Filtered out multimodal-gen NPU job: ${j.name}`); + return false; + } + // If the failing step is the health check, it's a cascade -- skip it const failedStep = (j.steps || []).find(s => s.conclusion === 'failure'); if (failedStep && (failedStep.name.includes('check-pr-test-health') || failedStep.name.includes('Check PR test health'))) { + core.info(`[health-check] Filtered out cascade failure: ${j.name} (failed step: ${failedStep.name})`); return false; } return true; }); + core.info(`[health-check] Root cause failures (after filtering): ${rootCauseFailures.map(j => j.name).join(', ') || 'none'}`); + if (rootCauseFailures.length > 0) { - core.setFailed(`Fast-fail: skipping — root cause job(s): ${rootCauseFailures.map(j => j.name).join(', ')}`); + core.setFailed(`Fail-fast: skipping — root cause job(s): ${rootCauseFailures.map(j => j.name).join(', ')}`); + } else { + core.info('[health-check] PASS: no root cause failures detected'); } diff --git a/.github/actions/wait-for-jobs/action.yml b/.github/actions/wait-for-jobs/action.yml index e5093d1f9..846bd9118 100644 --- a/.github/actions/wait-for-jobs/action.yml +++ b/.github/actions/wait-for-jobs/action.yml @@ -1,5 +1,5 @@ name: Wait for Jobs -description: Poll and wait for specified jobs in the current workflow run to complete. Returns success immediately when the PR carries the `bypass-fastfail` label, letting downstream stages dispatch in parallel (same effect as scheduled runs). +description: Poll and wait for specified jobs in the current workflow run to complete. Returns success immediately when the PR carries the `parallel-stages` label (or `highest-priority`, which implies it), letting downstream stages dispatch in parallel (same effect as scheduled runs). inputs: stage-name: @@ -49,28 +49,12 @@ runs: const pollIntervalSeconds = parseInt(process.env.INPUT_POLL_INTERVAL_SECONDS); const maxAttempts = (maxWaitMinutes * 60) / pollIntervalSeconds; - // bypass-fastfail label opts the PR out of stage-to-stage waiting, - // letting all stages dispatch in parallel like scheduled runs do. - let labels = []; - if (context.payload.pull_request?.labels) { - labels = context.payload.pull_request.labels.map(l => l.name); - } else { - const ref = context.payload.pull_request?.head?.sha || context.sha; - try { - const { data: prs } = await github.rest.repos.listPullRequestsAssociatedWithCommit({ - owner: context.repo.owner, - repo: context.repo.repo, - commit_sha: ref, - }); - if (prs.length > 0) { - labels = prs[0].labels.map(l => l.name); - } - } catch (e) { - console.log(`Could not fetch PR labels for ${ref}: ${e.message}`); - } - } - if (labels.includes('bypass-fastfail')) { - console.log(`Skipping ${stageName} wait (bypass-fastfail label present)`); + const { resolveCiLabels } = require( + `${process.env.GITHUB_WORKSPACE}/.github/scripts/ci-labels.cjs` + ); + const { parallelStages } = await resolveCiLabels(github, context); + if (parallelStages) { + console.log(`Skipping ${stageName} wait (parallel-stages label present)`); core.setOutput('result', 'success'); return; } diff --git a/.github/scripts/ci-labels.cjs b/.github/scripts/ci-labels.cjs new file mode 100644 index 000000000..81b1cb55f --- /dev/null +++ b/.github/scripts/ci-labels.cjs @@ -0,0 +1,62 @@ +"use strict"; + +/** + * Labels come from the API, not `context.payload`: a rerun replays the original + * event, so the payload carries the label set from when the run was created. + */ + +const BYPASS_FAIL_FAST = "bypass-fail-fast"; +const PARALLEL_STAGES = "parallel-stages"; +const MAX_CONCURRENCY = "max-concurrency"; +const HIGHEST_PRIORITY = "highest-priority"; + +// Callers gate the whole run on this, so a transient API error must not fail them. +async function readLabels(github, context) { + try { + return await fetchLabels(github, context); + } catch (e) { + console.warn(`Could not read PR labels: ${e.message}`); + return []; + } +} + +async function fetchLabels(github, context) { + const prNumber = context.payload.pull_request?.number; + if (prNumber) { + const { data: pr } = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: prNumber, + }); + return pr.labels.map((l) => l.name); + } + const sha = context.payload.pull_request?.head?.sha || context.sha; + const { data: prs } = + await github.rest.repos.listPullRequestsAssociatedWithCommit({ + owner: context.repo.owner, + repo: context.repo.repo, + commit_sha: sha, + }); + return prs.length > 0 ? prs[0].labels.map((l) => l.name) : []; +} + +async function resolveCiLabels(github, context) { + const labels = await readLabels(github, context); + const has = (name) => labels.includes(name); + const highestPriority = has(HIGHEST_PRIORITY); + return { + labels, + bypassFailFast: highestPriority || has(BYPASS_FAIL_FAST), + parallelStages: highestPriority || has(PARALLEL_STAGES), + maxConcurrency: highestPriority || has(MAX_CONCURRENCY), + highestPriority, + }; +} + +module.exports = { + resolveCiLabels, + BYPASS_FAIL_FAST, + PARALLEL_STAGES, + MAX_CONCURRENCY, + HIGHEST_PRIORITY, +}; diff --git a/.github/workflows/_npu-analyze-failure.yml b/.github/workflows/_npu-analyze-failure.yml new file mode 100644 index 000000000..f801f4557 --- /dev/null +++ b/.github/workflows/_npu-analyze-failure.yml @@ -0,0 +1,96 @@ +# Reusable workflow: cross-reference CI test failures with coverage-based +# test recommendations. Called by pr-test-npu.yml's analyze-failure-report job. +name: Analyze Failure Report + +on: + workflow_call: + inputs: + log_artifact_pattern: + type: string + required: true + description: 'Glob pattern for test log artifacts (e.g. selected-test-logs-*)' + recommendations_content: + type: string + required: false + default: '' + description: 'Content of recommendations from upstream job output' + +defaults: + run: + shell: bash -el {0} + +jobs: + analyze: + name: cross-reference failures with recommendations + continue-on-error: true # Non-blocking on parse/read failure. + runs-on: ubuntu-latest + steps: + - name: Checkout sglang repo + uses: actions/checkout@v4 + + # ----- Step 1: Locate recommendations ----- + - name: Locate recommendations file + id: locate-recs + run: | + mkdir -p ./test-logs + + if [ -n "${{ inputs.recommendations_content }}" ]; then + echo "has_recommendations=true" >> $GITHUB_OUTPUT + echo "Using workflow output recommendations" + echo "${{ inputs.recommendations_content }}" > ./test-logs/_recommended.txt + echo "RECS_FILE=./test-logs/_recommended.txt" >> $GITHUB_ENV + echo "RECS_SOURCE=output" >> $GITHUB_ENV + COUNT=$(wc -l < ./test-logs/_recommended.txt) + echo "Recommendations count: ${COUNT}" + else + echo "has_recommendations=false" >> $GITHUB_OUTPUT + echo "::notice::No recommended test cases detected, skipping subsequent analysis tasks." + fi + + # ----- Step 2: Download test logs ----- + - name: Download test logs + uses: actions/download-artifact@v7 + with: + pattern: ${{ inputs.log_artifact_pattern }} + path: ./test-logs + merge-multiple: true + continue-on-error: true + + - name: Show downloaded logs structure + run: | + echo "Downloaded log files:" + find ./test-logs -type f | head -50 || echo " (no logs found)" + + # ----- Step 3: Run analysis ----- + - name: Ensure regex is available + if: steps.locate-recs.outputs.has_recommendations != 'false' + run: | + set -euo pipefail + if ! python3 -c "import regex" >/dev/null 2>&1; then + echo "regex not found for $(python3 -V); bootstrapping pip and installing regex" + curl -fsSL https://bootstrap.pypa.io/get-pip.py -o get-pip.py + python3 get-pip.py --break-system-packages + python3 -m pip install --break-system-packages regex + fi + python3 -c "import regex; print('regex ok:', regex.__file__)" + + - name: Run failure analysis + if: steps.locate-recs.outputs.has_recommendations != 'false' + id: analysis + continue-on-error: true + run: | + python3 scripts/ci/npu/precise-test/analyze_failure_report.py \ + --log-dir ./test-logs \ + --recommendations-file "${RECS_FILE}" \ + --recommendations-source "${RECS_SOURCE:-none}" \ + --output ./test-logs/failure_report.md + + # ----- Step 4: Upload report ----- + - name: Upload failure analysis report + if: always() + uses: actions/upload-artifact@v7 + with: + name: failure-analysis-report + path: ./test-logs/failure_report.md + if-no-files-found: ignore + retention-days: 14 diff --git a/.github/workflows/_npu-pr-test-stage.yml b/.github/workflows/_npu-pr-test-stage.yml index 42cd4b0e4..fed2b1317 100644 --- a/.github/workflows/_npu-pr-test-stage.yml +++ b/.github/workflows/_npu-pr-test-stage.yml @@ -1,5 +1,7 @@ name: PR Test Stage for NPU -# Reusable workflow for one CUDA test stage. Caller pr-test-npu.yml forwards +# Reusable workflow for one NPU test stage. Caller pr-test-npu.yml forwards +# job parameters (runner, image, partitions) and optionally delegates test +# execution to the coverage runner (use_coverage_runner=true). on: workflow_call: @@ -33,17 +35,25 @@ on: type: string default: '{"size":1,"arr":[0]}' ref: - description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.' + description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the event commit SHA.' type: string default: '' skip_pr_test_health_check: - description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.' + description: 'Set to true to skip the PR test health check (fail-fast gate).' type: string default: 'false' is_nightly_pipeline_job: description: 'Run the test suite with --nightly (collects nightly-registered tests) and --continue-on-error' type: boolean default: false + use_coverage_runner: + description: 'If true, delegate test execution to scripts/ci/npu/precise-test/run_tests_with_coverage.sh instead of running via run_suite.py directly' + type: boolean + default: false + upload_test_logs: + description: 'If true, upload the test logs as a GitHub Actions artifact (only enabled by pr-test-npu)' + type: boolean + default: false github-token: description: 'GitHub token for API calls' type: string @@ -72,118 +82,15 @@ jobs: - name: Checkout code uses: actions/checkout@v4 with: - ref: ${{ inputs.ref || github.ref }} + # Pin to the event SHA (not the branch name) so every job in this run + # checks out the same commit even if the branch advances mid-run. + ref: ${{ inputs.ref || github.sha }} - name: Mark repository safe run: | git config --system --add safe.directory ${GITHUB_WORKSPACE} - - name: Check PR test health - uses: actions/github-script@v8 - env: - SKIP_PR_TEST_HEALTH_CHECK: ${{ env.SKIP_PR_TEST_HEALTH_CHECK }} - with: - github-token: ${{ inputs.github-token || github.token }} - script: | - core.notice(`[health-check] START — event=${context.eventName}, runId=${context.runId}`); - - // Skip when explicitly requested via env var (e.g. release branch cut) - if (process.env.SKIP_PR_TEST_HEALTH_CHECK === 'true') { - core.notice('[health-check] SKIP: SKIP_PR_TEST_HEALTH_CHECK=true'); - return; - } - - // Skip for scheduled runs — they should collect all failures, not fast-fail - if (context.eventName === 'schedule') { - core.notice('[health-check] SKIP: scheduled run'); - return; - } - - // Check lint status from the separate Lint workflow (lint.yml). - // listJobsForWorkflowRun only sees jobs within the SAME run, so we use - // checks.listForRef which queries by commit SHA across ALL workflows. - const ref = context.payload.pull_request?.head?.sha || context.sha; - core.info(`[health-check] Checking lint for ref=${ref}`); - const { data } = await github.rest.checks.listForRef({ - owner: context.repo.owner, - repo: context.repo.repo, - ref: ref, - check_name: 'lint', - }); - const lintRun = data.check_runs.find( - cr => cr.app?.slug === 'github-actions' - ); - core.info(`[health-check] Lint check: status=${lintRun?.status}, conclusion=${lintRun?.conclusion}`); - if (lintRun?.status === 'completed' && lintRun?.conclusion === 'failure') { - core.setFailed('Fast-fail: lint check failed'); - return; - } - - // Skip the jobs-failed check when the PR carries the bypass-fastfail label. - // Lint check above still runs. - let labels = []; - if (context.payload.pull_request?.labels) { - labels = context.payload.pull_request.labels.map(l => l.name); - } else { - const { data: prs } = await github.rest.repos.listPullRequestsAssociatedWithCommit({ - owner: context.repo.owner, - repo: context.repo.repo, - commit_sha: ref, - }); - if (prs.length > 0) { - labels = prs[0].labels.map(l => l.name); - } - } - core.info(`[health-check] PR labels: [${labels.join(', ')}]`); - if (labels.includes('bypass-fastfail')) { - core.notice('[health-check] SKIP jobs-failed check: bypass-fastfail label present'); - return; - } - - const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, { - owner: context.repo.owner, - repo: context.repo.repo, - run_id: context.runId, - per_page: 100, - }); - core.info(`[health-check] Total jobs in run: ${jobs.length}`); - const failedJobs = jobs.filter(j => j.status === 'completed' && j.conclusion === 'failure'); - core.info(`[health-check] Failed jobs (before filtering): ${failedJobs.map(j => `${j.name}(${j.conclusion})`).join(', ') || 'none'}`); - - // Find jobs that failed from a real error, not from fast-fail cascade - const rootCauseFailures = jobs.filter(j => { - if (j.status !== 'completed' || j.conclusion !== 'failure') return false; - // h20 runners are flaky (dirty GPU state from prior runs); their failures - // should not cascade fast-fail to other stages. j.name shape from - // listJobsForWorkflowRun: "" + optional " / " - // + optional " ()". Split off the base job key before exact - // match so we cover both inline + reusable forms without confusing - // 'h20' with the 'h200' prefix. - const baseName = j.name.split(/[ /]/)[0]; - if (baseName === 'base-c-test-8-gpu-h20') { - core.info(`[health-check] Filtered out h20 job: ${j.name}`); - return false; - } - // multimodal-gen NPU tests should not cascade fast-fail to perf/accuracy stages. - if (baseName === 'multimodal-gen-test-1-npu-a3' || baseName === 'multimodal-gen-test-2-npu-a3') { - core.info(`[health-check] Filtered out multimodal-gen NPU job: ${j.name}`); - return false; - } - // If the failing step is the health check, it's a cascade — skip it - const failedStep = (j.steps || []).find(s => s.conclusion === 'failure'); - if (failedStep && (failedStep.name.includes('check-pr-test-health') || failedStep.name.includes('Check PR test health'))) { - core.info(`[health-check] Filtered out cascade failure: ${j.name} (failed step: ${failedStep.name})`); - return false; - } - return true; - }); - core.info(`[health-check] Root cause failures (after filtering): ${rootCauseFailures.map(j => j.name).join(', ') || 'none'}`); - - if (rootCauseFailures.length > 0) { - core.setFailed(`Fast-fail: skipping — root cause job(s): ${rootCauseFailures.map(j => j.name).join(', ')}`); - } else { - core.notice('[health-check] PASS: no root cause failures detected'); - } + - uses: ./.github/actions/check-pr-test-health - name: Install dependencies env: @@ -228,10 +135,12 @@ jobs: timeout-minutes: ${{ fromJson(inputs.run_timeout_minutes) }} env: CONTINUE_ON_ERROR_FLAG: ${{ inputs.continue_on_error == 'true' && '--continue-on-error' || '' }} + TEST_LOG_DIR: /tmp/test-logs shell: bash run: | # Fail fast on any command error, undefined variable, or pipe failure. set -euo pipefail + mkdir -p "${TEST_LOG_DIR}" # Install missing python deps (skip if already importable). PYTHON_FOR_SGLANG="python" @@ -255,8 +164,11 @@ jobs: install_pkg "${pkg}" done - # Install sgl-eval (CLI used by run_eval.py's `sgl-eval run ...`) if missing. - command -v sgl-eval >/dev/null 2>&1 || install_pkg "sgl-eval==0.1.0" + # Fallback for images that predate the pin in pyproject_npu.toml. Checking + # the version, not the command, so a stale wheel is replaced rather than kept. + SGL_EVAL_VERSION=0.1.2 + ${PYTHON_FOR_SGLANG} -c "import sgl_eval, sys; sys.exit(sgl_eval.__version__ != '${SGL_EVAL_VERSION}')" 2>/dev/null \ + || install_pkg "sgl-eval==${SGL_EVAL_VERSION}" cd test # Append `--nightly` and `--continue-on-error` when the job belongs to the nightly pipeline. @@ -266,8 +178,38 @@ jobs: if [ "${{ inputs.is_nightly_pipeline_job }}" = "true" ]; then NIGHTLY_FLAG="--nightly --continue-on-error" fi + + # Optional: delegate execution to scripts/ci/npu/precise-test/run_tests_with_coverage.sh when the + # caller sets use_coverage_runner=true. list_tests.py (same directory) enumerates the + # selected test files; the original run_suite.py execution below is preserved + # unchanged for the default (false) path. + LOG_FILE="${TEST_LOG_DIR}/${{ inputs.self_name }}-part${{ matrix.partition }}.log" + + if [[ "${{ inputs.use_coverage_runner }}" == "true" ]]; then + pip install coverage==7.8.* + python3 ../scripts/ci/npu/precise-test/list_tests.py --hw npu --suite ${{ inputs.self_name }} \ + --auto-partition-id ${{ matrix.partition }} \ + --auto-partition-size ${{ fromJson(inputs.partitions).size }} \ + -o /tmp/selected_tests.txt + echo "Selected test files:" + cat /tmp/selected_tests.txt + if [ -s /tmp/selected_tests.txt ]; then + bash ../scripts/ci/npu/precise-test/run_tests_with_coverage.sh $(cat /tmp/selected_tests.txt) + fi + exit 0 + fi + python3 run_suite.py --hw npu --suite ${{ inputs.self_name }} \ --auto-partition-id ${{ matrix.partition }} \ --auto-partition-size ${{ fromJson(inputs.partitions).size }} \ ${{ inputs.timeout_per_file && format('--timeout-per-file {0}', inputs.timeout_per_file) || '' }} \ - $NIGHTLY_FLAG $CONTINUE_ON_ERROR_FLAG + $NIGHTLY_FLAG $CONTINUE_ON_ERROR_FLAG 2>&1 | tee "${LOG_FILE}" + + - name: Upload test logs + if: always() && inputs.use_coverage_runner != true && inputs.upload_test_logs == true + uses: actions/upload-artifact@v7 + with: + name: selected-test-logs-${{ inputs.self_name }}-part${{ matrix.partition }} + path: /tmp/test-logs/ + if-no-files-found: ignore + retention-days: 14 diff --git a/.github/workflows/_npu-single-node-test-stage.yml b/.github/workflows/_npu-single-node-test-stage.yml index 1b04aec65..9ad0b0f30 100644 --- a/.github/workflows/_npu-single-node-test-stage.yml +++ b/.github/workflows/_npu-single-node-test-stage.yml @@ -43,7 +43,7 @@ on: default: '{}' description: 'JSON run metadata {branch_label, workflow_name, create_time}, recorded once at workflow start' skip_pr_test_health_check: - description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.' + description: 'Set to true to skip the PR test health check (fail-fast gate).' type: string default: 'false' is_nightly_pipeline_job: @@ -54,6 +54,10 @@ on: description: 'Partition config to parallelize a suite across jobs, e.g. {"size":3,"arr":[0,1,2]}. Each element of arr is a partition id; the template expands one matrix job per element. size is passed to run_suite.py --auto-partition-size.' type: string default: '{"size":1,"arr":[0]}' + ref: + description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the event SHA.' + type: string + default: '' github-token: description: 'GitHub token for API calls' type: string @@ -63,6 +67,14 @@ on: type: string default: '300' description: 'timeout-minutes for the Run test step' + use_coverage_runner: + description: 'If true, delegate test execution to scripts/ci/npu/precise-test/run_tests_with_coverage.sh instead of running via run_suite.py directly' + type: boolean + default: false + upload_test_logs: + description: 'If true, upload the test logs as a GitHub Actions artifact (only enabled by pr-test-npu)' + type: boolean + default: false env: SKIP_PR_TEST_HEALTH_CHECK: ${{ inputs.skip_pr_test_health_check }} @@ -84,113 +96,12 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v4 - - - name: Check PR test health - uses: actions/github-script@v8 - env: - SKIP_PR_TEST_HEALTH_CHECK: ${{ env.SKIP_PR_TEST_HEALTH_CHECK }} with: - github-token: ${{ inputs.github-token || github.token }} - script: | - core.notice(`[health-check] START — event=${context.eventName}, runId=${context.runId}`); + # Pin to the event SHA (not the branch name) so every job in this run + # checks out the same commit even if the branch advances mid-run. + ref: ${{ inputs.ref || github.sha }} - // Skip when explicitly requested via env var (e.g. release branch cut) - if (process.env.SKIP_PR_TEST_HEALTH_CHECK === 'true') { - core.notice('[health-check] SKIP: SKIP_PR_TEST_HEALTH_CHECK=true'); - return; - } - - // Skip for scheduled runs — they should collect all failures, not fast-fail - if (context.eventName === 'schedule') { - core.notice('[health-check] SKIP: scheduled run'); - return; - } - - // Check lint status from the separate Lint workflow (lint.yml). - // listJobsForWorkflowRun only sees jobs within the SAME run, so we use - // checks.listForRef which queries by commit SHA across ALL workflows. - const ref = context.payload.pull_request?.head?.sha || context.sha; - core.info(`[health-check] Checking lint for ref=${ref}`); - const { data } = await github.rest.checks.listForRef({ - owner: context.repo.owner, - repo: context.repo.repo, - ref: ref, - check_name: 'lint', - }); - const lintRun = data.check_runs.find( - cr => cr.app?.slug === 'github-actions' - ); - core.info(`[health-check] Lint check: status=${lintRun?.status}, conclusion=${lintRun?.conclusion}`); - if (lintRun?.status === 'completed' && lintRun?.conclusion === 'failure') { - core.setFailed('Fast-fail: lint check failed'); - return; - } - - // Skip the jobs-failed check when the PR carries the bypass-fastfail label. - // Lint check above still runs. - let labels = []; - if (context.payload.pull_request?.labels) { - labels = context.payload.pull_request.labels.map(l => l.name); - } else { - const { data: prs } = await github.rest.repos.listPullRequestsAssociatedWithCommit({ - owner: context.repo.owner, - repo: context.repo.repo, - commit_sha: ref, - }); - if (prs.length > 0) { - labels = prs[0].labels.map(l => l.name); - } - } - core.info(`[health-check] PR labels: [${labels.join(', ')}]`); - if (labels.includes('bypass-fastfail')) { - core.notice('[health-check] SKIP jobs-failed check: bypass-fastfail label present'); - return; - } - - const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, { - owner: context.repo.owner, - repo: context.repo.repo, - run_id: context.runId, - per_page: 100, - }); - core.info(`[health-check] Total jobs in run: ${jobs.length}`); - const failedJobs = jobs.filter(j => j.status === 'completed' && j.conclusion === 'failure'); - core.info(`[health-check] Failed jobs (before filtering): ${failedJobs.map(j => `${j.name}(${j.conclusion})`).join(', ') || 'none'}`); - - // Find jobs that failed from a real error, not from fast-fail cascade - const rootCauseFailures = jobs.filter(j => { - if (j.status !== 'completed' || j.conclusion !== 'failure') return false; - // h20 runners are flaky (dirty GPU state from prior runs); their failures - // should not cascade fast-fail to other stages. j.name shape from - // listJobsForWorkflowRun: "" + optional " / " - // + optional " ()". Split off the base job key before exact - // match so we cover both inline + reusable forms without confusing - // 'h20' with the 'h200' prefix. - const baseName = j.name.split(/[ /]/)[0]; - if (baseName === 'base-c-test-8-gpu-h20') { - core.info(`[health-check] Filtered out h20 job: ${j.name}`); - return false; - } - // multimodal-gen NPU tests should not cascade fast-fail to perf/accuracy stages. - if (baseName === 'multimodal-gen-test-1-npu-a3' || baseName === 'multimodal-gen-test-2-npu-a3') { - core.info(`[health-check] Filtered out multimodal-gen NPU job: ${j.name}`); - return false; - } - // If the failing step is the health check, it's a cascade — skip it - const failedStep = (j.steps || []).find(s => s.conclusion === 'failure'); - if (failedStep && (failedStep.name.includes('check-pr-test-health') || failedStep.name.includes('Check PR test health'))) { - core.info(`[health-check] Filtered out cascade failure: ${j.name} (failed step: ${failedStep.name})`); - return false; - } - return true; - }); - core.info(`[health-check] Root cause failures (after filtering): ${rootCauseFailures.map(j => j.name).join(', ') || 'none'}`); - - if (rootCauseFailures.length > 0) { - core.setFailed(`Fast-fail: skipping — root cause job(s): ${rootCauseFailures.map(j => j.name).join(', ')}`); - } else { - core.notice('[health-check] PASS: no root cause failures detected'); - } + - uses: ./.github/actions/check-pr-test-health - name: Check npu info run: | @@ -227,6 +138,9 @@ jobs: # Fail fast on any command error, undefined variable, or pipe failure. set -euo pipefail + TEST_LOG_DIR="/tmp/test-logs" + mkdir -p "${TEST_LOG_DIR}" + sglang_source_path=$(pwd) echo "Source code path: ${sglang_source_path}" ln -sf ${sglang_source_path} /root/sglang @@ -390,14 +304,29 @@ jobs: fi # Nightly additionally persists the full suite log to ${log_path}/${tc_name}.log - # under the structured path; PR runs only tee to /tmp/test_output.log. + # under the structured path; PR runs tee to ${TEST_LOG_DIR}/${tc_name}.log. # Use an array so the extra target is omitted when empty: passing an empty # string arg makes GNU tee fail (exit code 1) and flips the pipeline exit code. - LOG_TEE_TARGETS=("/tmp/test_output.log") + LOG_TEE_TARGETS=("${TEST_LOG_DIR}/${tc_name}.log") if [ "${{ inputs.is_nightly_pipeline_job }}" = "true" ]; then LOG_TEE_TARGETS+=("${log_path}/${tc_name}.log") fi + # Optional: delegate execution to scripts/ci/npu/precise-test/run_tests_with_coverage.sh when the + # caller sets use_coverage_runner=true. list_tests.py (same directory) enumerates the + # selected test files; the original run_suite.py execution below is preserved + # unchanged for the default (false) path. + if [[ "${{ inputs.use_coverage_runner }}" == "true" ]]; then + pip install coverage==7.8.* + ${PYTHON_FOR_SGLANG} -u ../scripts/ci/npu/precise-test/list_tests.py --hw npu --suite ${test_suite} ${PARTITION_ARGS} -o /tmp/selected_tests.txt + echo "Selected test files:" + cat /tmp/selected_tests.txt + if [ -s /tmp/selected_tests.txt ]; then + bash ../scripts/ci/npu/precise-test/run_tests_with_coverage.sh $(cat /tmp/selected_tests.txt) + fi + exit 0 + fi + # Run NPU test suite with run_suite.py. # Capture mode (--enable-retry --max-attempts 1) keeps test subprocesses off the # tee pipe, so leftover server processes cannot block the pipeline on exit. @@ -418,11 +347,11 @@ jobs: echo "## ${tc_name} ${status_icon} ${test_status}" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY if [ "${{ inputs.is_nightly_pipeline_job }}" != "true" ]; then - metric_count=$(grep -c '\[METRIC\]' /tmp/test_output.log 2>/dev/null || echo 0) + metric_count=$(grep -c '\[METRIC\]' "${TEST_LOG_DIR}/${tc_name}.log" 2>/dev/null || echo 0) if [ "${metric_count}" -gt 0 ]; then echo "| Metric | Value | Pass |" >> $GITHUB_STEP_SUMMARY echo "|--------|-------|------|" >> $GITHUB_STEP_SUMMARY - grep '\[METRIC\]' /tmp/test_output.log | while IFS= read -r line; do + grep '\[METRIC\]' "${TEST_LOG_DIR}/${tc_name}.log" | while IFS= read -r line; do metric_name=$(echo "$line" | sed -E 's/.*\[METRIC\] ([^=]+)=.*/\1/') metric_value=$(echo "$line" | sed -E 's/.*\[METRIC\] [^=]+=([^ ]+).*/\1/') echo "| ${metric_name} | ${metric_value} | ${status_icon} |" >> $GITHUB_STEP_SUMMARY @@ -433,3 +362,12 @@ jobs: echo "" >> $GITHUB_STEP_SUMMARY fi exit ${test_exit_code} + + - name: Upload test logs + if: always() && inputs.use_coverage_runner != true && inputs.upload_test_logs == true + uses: actions/upload-artifact@v7 + with: + name: selected-test-logs-${{ inputs.test_suite }}-part${{ matrix.partition }} + path: /tmp/test-logs/ + if-no-files-found: ignore + retention-days: 14 diff --git a/.github/workflows/_pr-test-check-changes.yml b/.github/workflows/_pr-test-check-changes.yml index 4070b5ca2..8600dcd89 100644 --- a/.github/workflows/_pr-test-check-changes.yml +++ b/.github/workflows/_pr-test-check-changes.yml @@ -136,15 +136,24 @@ jobs: - "test/registered/rust/**" - ".github/workflows/_pr-test-*.yml" + - name: Resolve CI labels + id: ci-labels + if: github.event_name == 'pull_request' + uses: actions/github-script@v8 + with: + script: | + const { resolveCiLabels } = require( + `${process.env.GITHUB_WORKSPACE}/.github/scripts/ci-labels.cjs` + ); + const axes = await resolveCiLabels(github, context); + core.info(`CI labels: [${axes.labels.join(', ')}]`); + core.setOutput('max_concurrency', String(axes.maxConcurrency)); + - name: Determine full-parallel mode id: parallel-mode run: | # `full=true` lifts the matrix-fanout throttle so each suite's - # max_parallel = size. Conditions: - # 1. Scheduled cron run. - # 2. run_all_tests run (manual full dispatch / release) -- a full - # run should mirror the cron's parallelism, not just its test set. - # 3. pull_request event with the `high priority` label. + # max_parallel = size. FULL=false if [[ "${{ github.event_name }}" == "schedule" ]]; then FULL=true @@ -152,9 +161,9 @@ jobs: elif [[ "${{ inputs.run_all_tests }}" == "true" ]]; then FULL=true echo "run_all_tests -> full parallelism" - elif [[ "${{ github.event_name }}" == "pull_request" && "${{ contains(github.event.pull_request.labels.*.name, 'high priority') }}" == "true" ]]; then + elif [[ "${{ steps.ci-labels.outputs.max_concurrency }}" == "true" ]]; then FULL=true - echo "high priority PR -> full parallelism" + echo "max-concurrency PR -> full parallelism" fi echo "full=$FULL" >> "$GITHUB_OUTPUT" @@ -189,10 +198,7 @@ jobs: id: partitions run: | # Emit a single JSON output `partitions` keyed by suite name with - # {size, arr, max_parallel} fields per suite. Replaces the prior - # full/low max-parallel presets; `--full-parallel` keeps the - # `high priority` PR / scheduled cron escape hatch. - # See scripts/ci/utils/compute_partitions.py. + # {size, arr, max_parallel} fields per suite. python3 scripts/ci/utils/compute_partitions.py \ --full-parallel ${{ steps.parallel-mode.outputs.full }} \ --partition-model-file /tmp/partition-model.json \ diff --git a/.github/workflows/_pr-test-simulator-cpu.yml b/.github/workflows/_pr-test-simulator-cpu.yml index a5f050dec..6c47ce9c5 100644 --- a/.github/workflows/_pr-test-simulator-cpu.yml +++ b/.github/workflows/_pr-test-simulator-cpu.yml @@ -88,6 +88,7 @@ jobs: MALLOC_ARENA_MAX: "2" MAX_JOBS: "1" PYTEST_DISABLE_PLUGIN_AUTOLOAD: "1" + PYTEST_ADDOPTS: "-s" TORCH_EXTENSIONS_DIR: ${{ runner.temp }}/torch-extensions run: | python3 -m pytest -q tools/sglang-simulator/test/test_simulation_sglang_runner.py diff --git a/.github/workflows/cancel-unfinished-pr-tests.yml b/.github/workflows/cancel-unfinished-pr-tests.yml index dc08ed9ac..4b0218da8 100644 --- a/.github/workflows/cancel-unfinished-pr-tests.yml +++ b/.github/workflows/cancel-unfinished-pr-tests.yml @@ -8,8 +8,8 @@ on: required: true type: string default: 'pr-test.yml pr-test-extra.yml' - include_high_priority: - description: 'Also cancel runs from high-priority PRs' + include_highest_priority: + description: 'Also cancel runs from PRs labelled highest-priority' required: false type: boolean default: false @@ -36,7 +36,7 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} REPO: ${{ github.repository }} WORKFLOWS: ${{ github.event.inputs.workflows }} - INCLUDE_HIGH_PRIORITY: ${{ github.event.inputs.include_high_priority }} + INCLUDE_HIGHEST_PRIORITY: ${{ github.event.inputs.include_highest_priority }} INCLUDE_RERUN_TEST: ${{ github.event.inputs.include_rerun_test }} shell: bash run: | @@ -49,7 +49,7 @@ jobs: fi echo "Targeting ${#WORKFLOW_FILES[@]} workflow(s): ${WORKFLOW_FILES[*]}" - echo "include_high_priority=$INCLUDE_HIGH_PRIORITY, include_rerun_test=$INCLUDE_RERUN_TEST" + echo "include_highest_priority=$INCLUDE_HIGHEST_PRIORITY, include_rerun_test=$INCLUDE_RERUN_TEST" echo "" # Decide whether to cancel run_id given a PR-lookup endpoint. @@ -99,12 +99,12 @@ jobs: return fi - if echo "$labels" | grep -Fxq "high priority"; then - if [ "$INCLUDE_HIGH_PRIORITY" != "true" ]; then - echo " 🛑 Skipping (high priority label)" + if echo "$labels" | grep -Fxq "highest-priority"; then + if [ "$INCLUDE_HIGHEST_PRIORITY" != "true" ]; then + echo " 🛑 Skipping (highest-priority label)" return fi - echo " ⚠️ High priority PR, but include_high_priority is enabled" + echo " ⚠️ highest-priority PR, but include_highest_priority is enabled" fi echo " 🚫 Cancelling..." diff --git a/.github/workflows/close-stale-prs.yml b/.github/workflows/close-stale-prs.yml index 96f83c806..1965b9960 100644 --- a/.github/workflows/close-stale-prs.yml +++ b/.github/workflows/close-stale-prs.yml @@ -38,7 +38,7 @@ jobs: const QUOTA_IDLE = 7; const STALE_DAYS = 90; - const KEEP_LABELS = new Set(['high priority', 'keep-open', 'good first issue']); + const KEEP_LABELS = new Set(['highest-priority', 'keep-open', 'good first issue']); const WIP_MARKER = /^\s*\[?\s*(wip|do[ _-]?not[ _-]?merge|dnm|draft)\s*\]?/i; // Scheduled runs always act; only manual runs can be dry. diff --git a/.github/workflows/coverage-collection-npu.yml b/.github/workflows/coverage-collection-npu.yml new file mode 100644 index 000000000..52ff15cd4 --- /dev/null +++ b/.github/workflows/coverage-collection-npu.yml @@ -0,0 +1,27 @@ +# Coverage data collection pipeline for the NPU precision test selection. +# Delegates to pr-test-npu.yml with coverage_mode=true, which switches: +# - Runners to the coverage pool (shared /root/.cache with setup-covstub) +# - use_coverage_runner=true on all test jobs +# - Skips pr-gate, base-a (910b), multimodal-gen, recommend, analyze +# - Runs setup-covstub to assemble coverage data and publish it to the +# shared disk at a fixed path (/root/.cache/tests/precise-test/coverage) +# pr-test-npu-finish is the final gate in the called workflow. +# +# Manually triggered only (workflow_dispatch): PRs run the standard +# pr-test-npu.yml instead; this pipeline builds the full coverage baseline +# on demand. +name: Coverage Collection (NPU) + +on: + workflow_dispatch: + +concurrency: + group: npu-coverage-collection-${{ github.ref }} + cancel-in-progress: true + +jobs: + coverage: + uses: ./.github/workflows/pr-test-npu.yml + with: + coverage_mode: true + secrets: inherit diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 4a89b953f..2041ea817 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -76,6 +76,9 @@ jobs: - name: Check cookbook authoring contracts run: node docs/scripts/check_cookbook_configs.mjs + - name: Test PR health action + run: node --test .github/actions/check-pr-test-health/action.test.cjs + - name: Cache mint uses: actions/cache@v4 with: diff --git a/.github/workflows/list-active-pr-runs.yml b/.github/workflows/list-active-pr-runs.yml index 10deab837..4a289af43 100644 --- a/.github/workflows/list-active-pr-runs.yml +++ b/.github/workflows/list-active-pr-runs.yml @@ -127,23 +127,23 @@ jobs: # Get unique PR numbers (exclude NO_PR entries) pr_numbers=$(cut -d'|' -f1 < "$pr_data_file" | grep -v '^NO_PR$' | sort -u || true) - # Separate high priority and normal PRs - high_priority_prs=() + # Separate highest-priority and normal PRs + highest_priority_prs=() normal_prs=() for pr_num in $pr_numbers; do labels=$(gh pr view "$pr_num" --repo "$REPO" --json labels \ | jq -r '.labels[].name' 2>/dev/null || true) - if echo "$labels" | grep -Fxq "high priority"; then - high_priority_prs+=($pr_num) + if echo "$labels" | grep -Fxq "highest-priority"; then + highest_priority_prs+=($pr_num) else normal_prs+=($pr_num) fi done - # Combine: high priority first, then normal - sorted_pr_numbers=("${high_priority_prs[@]}" "${normal_prs[@]}") + # Combine: highest-priority first, then normal + sorted_pr_numbers=("${highest_priority_prs[@]}" "${normal_prs[@]}") pr_count=0 total_running=0 @@ -170,8 +170,8 @@ jobs: # Add priority indicator priority_indicator="" - if echo "$pr_labels" | grep -q "high priority"; then - priority_indicator="🔴 [HIGH PRIORITY] " + if echo "$pr_info" | jq -e '[.labels[].name] | index("highest-priority")' >/dev/null; then + priority_indicator="🔴 [HIGHEST PRIORITY] " fi echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" diff --git a/.github/workflows/nightly-amd-mi355x-disagg.yml b/.github/workflows/nightly-amd-mi355x-disagg.yml index a19189269..3ea8c5bb3 100644 --- a/.github/workflows/nightly-amd-mi355x-disagg.yml +++ b/.github/workflows/nightly-amd-mi355x-disagg.yml @@ -156,11 +156,14 @@ jobs: # mi355x-ci--- # Here we clear THIS runner's leftovers from a crashed previous run, so # we match the RUNNER_NAME prefix (older runs have a different run id). - # Never a blanket `squeue --me`, which would kill a concurrent leg. + # Never a blanket match on all jobs, which would kill a concurrent leg. # %200j: squeue truncates the job name (%j) by default -- widen it so # the grep sees the full name. + # No --me: Spur's squeue does not accept it. The RUNNER_NAME prefix is + # what makes this selective, and that is unique per runner, so the + # filter is equally precise without it. if [ -z "${RUNNER_NAME:-}" ]; then echo "RUNNER_NAME unset; skipping"; exit 0; fi - STALE_JOBS=$(squeue --me --noheader --format="%i %200j" | grep -F "mi355x-ci-${RUNNER_NAME}-" | awk '{print $1}' || true) + STALE_JOBS=$(squeue --noheader --format="%i %200j" | grep -F "mi355x-ci-${RUNNER_NAME}-" | awk '{print $1}' || true) if [ -n "$STALE_JOBS" ]; then echo "Cancelling stale jobs for ${RUNNER_NAME}: $STALE_JOBS" scancel $STALE_JOBS @@ -237,7 +240,7 @@ jobs: # config are unique per matrix leg, so a concurrent leg is never hit. if [ -z "${RUNNER_NAME:-}" ]; then echo "RUNNER_NAME unset; skipping"; exit 0; fi JOB_TAG="mi355x-ci-${RUNNER_NAME}-${GITHUB_RUN_ID}-${MATRIX_CONFIG_NAME}" - ACTIVE_JOBS=$(squeue --me --noheader --format="%i %200j" | grep -F "$JOB_TAG" | awk '{print $1}' || true) + ACTIVE_JOBS=$(squeue --noheader --format="%i %200j" | grep -F "$JOB_TAG" | awk '{print $1}' || true) if [ -n "$ACTIVE_JOBS" ]; then echo "Cancelling jobs for ${JOB_TAG}: $ACTIVE_JOBS" scancel $ACTIVE_JOBS diff --git a/.github/workflows/pr-test-amd-extra.yml b/.github/workflows/pr-test-amd-extra.yml index f06236601..2096b6114 100644 --- a/.github/workflows/pr-test-amd-extra.yml +++ b/.github/workflows/pr-test-amd-extra.yml @@ -243,7 +243,14 @@ jobs: extra-a-test-1-gpu-large-amd, extra-a-test-2-gpu-large-amd, ] - if: always() + # Same `labeled` guard as call-gate: an unrelated label would otherwise finish + # green with nothing executed, over the real run's result. + if: | + always() && + (github.event_name != 'pull_request' || + github.event.action != 'labeled' || + github.event.label.name == 'run-ci' || + github.event.label.name == 'run-ci-extra') runs-on: ubuntu-latest steps: - name: Check all dependent job statuses diff --git a/.github/workflows/pr-test-amd.yml b/.github/workflows/pr-test-amd.yml index e9f7a431c..cf92aa5fa 100644 --- a/.github/workflows/pr-test-amd.yml +++ b/.github/workflows/pr-test-amd.yml @@ -187,19 +187,29 @@ jobs: echo "Run mode: FILTERED (triggered by ${{ github.event_name }})" fi + - name: Resolve CI labels + id: ci-labels + if: github.event_name == 'pull_request' + uses: actions/github-script@v8 + with: + script: | + const { resolveCiLabels } = require( + `${process.env.GITHUB_WORKSPACE}/.github/scripts/ci-labels.cjs` + ); + const axes = await resolveCiLabels(github, context); + core.info(`CI labels: [${axes.labels.join(', ')}]`); + core.setOutput('bypass_fail_fast', String(axes.bypassFailFast)); + - name: Set continue-on-error for schedule/full runs id: set-continue-on-error env: - # `bypass-fastfail` PR label: also disable within-suite fast-fail - # here. The shared actions/wait-for-jobs already honors the same - # label to skip cross-stage waits. - BYPASS_FASTFAIL_LABEL: ${{ contains(github.event.pull_request.labels.*.name, 'bypass-fastfail') }} + BYPASS_FAIL_FAST: ${{ steps.ci-labels.outputs.bypass_fail_fast }} run: | if [[ "${{ steps.run-mode.outputs.run_all_tests }}" == "true" \ || "${{ inputs.continue_on_error }}" == "true" \ - || "$BYPASS_FASTFAIL_LABEL" == "true" ]]; then + || "$BYPASS_FAIL_FAST" == "true" ]]; then echo "continue_on_error=true" >> $GITHUB_OUTPUT - echo "Continue-on-error: ENABLED (run_all_tests=${{ steps.run-mode.outputs.run_all_tests }}, input=${{ inputs.continue_on_error }}, bypass-fastfail=$BYPASS_FASTFAIL_LABEL)" + echo "Continue-on-error: ENABLED (run_all_tests=${{ steps.run-mode.outputs.run_all_tests }}, input=${{ inputs.continue_on_error }}, bypass-fail-fast=$BYPASS_FAIL_FAST)" else echo "continue_on_error=false" >> $GITHUB_OUTPUT echo "Continue-on-error: DISABLED" diff --git a/.github/workflows/pr-test-extra.yml b/.github/workflows/pr-test-extra.yml index 3c4cd168a..cd0faf072 100644 --- a/.github/workflows/pr-test-extra.yml +++ b/.github/workflows/pr-test-extra.yml @@ -50,13 +50,14 @@ on: type: boolean default: false skip_pr_test_health_check: - description: "Skip PR test health check fast-fail (e.g. for release branch cuts)" + description: "Skip PR test health check fail-fast (e.g. for release branch cuts)" required: false type: boolean default: false concurrency: - group: pr-test-extra-${{ github.event_name }}-${{ github.head_ref || github.ref_name || 'default' }}-${{ inputs.git_ref || 'all' }} + # Keys on the PR number: two forks can share a head_ref and would cancel each other. + group: pr-test-extra-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref_name || 'default' }}-${{ inputs.git_ref || 'all' }} cancel-in-progress: ${{ github.event_name != 'workflow_call' }} env: @@ -159,92 +160,32 @@ jobs: rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} # No `secrets: inherit`: this hosted CPU job has no secret consumer. - # =============================================== extra-a (1-/2-gpu) =============================================== - extra-a-test-1-gpu-small: + # =============================================== extra-a / extra-b =============================================== + # `name:` keeps the job names wait-for-jobs and check-pr-test-health match on. + extra-test: + name: ${{ matrix.stage }}-test-${{ matrix.runner_config }} needs: [check-changes, call-gate, sgl-kernel-build-wheels, rust-ext-build] if: ${{ !failure() && !cancelled() && needs.check-changes.result == 'success' && (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') }} + strategy: + fail-fast: false + matrix: + include: + - { stage: extra-a, runner_config: 1-gpu-small, timeout_per_file: '' } + - { stage: extra-a, runner_config: 1-gpu-large, timeout_per_file: '1800' } + - { stage: extra-a, runner_config: 2-gpu-large, timeout_per_file: '' } + - { stage: extra-b, runner_config: 4-gpu-h100, timeout_per_file: '3600' } + - { stage: extra-b, runner_config: 4-gpu-b200, timeout_per_file: '1800' } + - { stage: extra-b, runner_config: 8-gpu-h200, timeout_per_file: '' } + - { stage: extra-b, runner_config: 8-gpu-b300, timeout_per_file: '3600' } uses: ./.github/workflows/_pr-test-stage.yml with: - self_name: extra-a-test-1-gpu-small - runner_config: 1-gpu-small - check_changes: ${{ toJson(needs.check-changes.outputs) }} - caller_inputs: ${{ toJson(inputs) }} - partitions: ${{ needs.check-changes.outputs.partitions }} - run_timeout_minutes: '60' - rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} - secrets: inherit - - extra-a-test-1-gpu-large: - needs: [check-changes, call-gate, sgl-kernel-build-wheels, rust-ext-build] - if: ${{ !failure() && !cancelled() && needs.check-changes.result == 'success' && (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') }} - uses: ./.github/workflows/_pr-test-stage.yml - with: - self_name: extra-a-test-1-gpu-large - runner_config: 1-gpu-large - check_changes: ${{ toJson(needs.check-changes.outputs) }} - caller_inputs: ${{ toJson(inputs) }} - partitions: ${{ needs.check-changes.outputs.partitions }} - run_timeout_minutes: '60' - timeout_per_file: '1800' - rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} - secrets: inherit - - extra-a-test-2-gpu-large: - needs: [check-changes, call-gate, sgl-kernel-build-wheels, rust-ext-build] - if: ${{ !failure() && !cancelled() && needs.check-changes.result == 'success' && (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') }} - uses: ./.github/workflows/_pr-test-stage.yml - with: - self_name: extra-a-test-2-gpu-large - runner_config: 2-gpu-large - check_changes: ${{ toJson(needs.check-changes.outputs) }} - caller_inputs: ${{ toJson(inputs) }} - partitions: ${{ needs.check-changes.outputs.partitions }} - run_timeout_minutes: '60' - rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} - secrets: inherit - - # =============================================== extra-b (4-/8-gpu) =============================================== - extra-b-test-4-gpu-h100: - needs: [check-changes, call-gate, sgl-kernel-build-wheels, rust-ext-build] - if: ${{ !failure() && !cancelled() && needs.check-changes.result == 'success' && (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') }} - uses: ./.github/workflows/_pr-test-stage.yml - with: - self_name: extra-b-test-4-gpu-h100 - runner_config: 4-gpu-h100 - check_changes: ${{ toJson(needs.check-changes.outputs) }} - caller_inputs: ${{ toJson(inputs) }} - partitions: ${{ needs.check-changes.outputs.partitions }} - run_timeout_minutes: '60' - timeout_per_file: '3600' - rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} - secrets: inherit - - extra-b-test-4-gpu-b200: - needs: [check-changes, call-gate, sgl-kernel-build-wheels, rust-ext-build] - if: ${{ !failure() && !cancelled() && needs.check-changes.result == 'success' && (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') }} - uses: ./.github/workflows/_pr-test-stage.yml - with: - self_name: extra-b-test-4-gpu-b200 - runner_config: 4-gpu-b200 - check_changes: ${{ toJson(needs.check-changes.outputs) }} - caller_inputs: ${{ toJson(inputs) }} - partitions: ${{ needs.check-changes.outputs.partitions }} - run_timeout_minutes: '60' - timeout_per_file: '1800' - rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} - secrets: inherit - - extra-b-test-8-gpu-h200: - needs: [check-changes, call-gate, sgl-kernel-build-wheels, rust-ext-build] - if: ${{ !failure() && !cancelled() && needs.check-changes.result == 'success' && (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') }} - uses: ./.github/workflows/_pr-test-stage.yml - with: - self_name: extra-b-test-8-gpu-h200 - runner_config: 8-gpu-h200 + self_name: ${{ matrix.stage }}-test-${{ matrix.runner_config }} + runner_config: ${{ matrix.runner_config }} check_changes: ${{ toJson(needs.check-changes.outputs) }} caller_inputs: ${{ toJson(inputs) }} partitions: ${{ needs.check-changes.outputs.partitions }} run_timeout_minutes: '60' + timeout_per_file: ${{ matrix.timeout_per_file }} rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} secrets: inherit @@ -260,14 +201,16 @@ jobs: sgl-kernel-build-wheels, rust-ext-build, simulator-test-cpu, - extra-a-test-1-gpu-small, - extra-a-test-1-gpu-large, - extra-a-test-2-gpu-large, - extra-b-test-4-gpu-h100, - extra-b-test-4-gpu-b200, - extra-b-test-8-gpu-h200, + extra-test, ] - if: always() + # Without this guard an unrelated label starts a run that finishes green with + # nothing executed, over the real failure; `skipped` is what pr-states.yml reads. + if: | + always() && + (github.event_name != 'pull_request' || + github.event.action != 'labeled' || + github.event.label.name == 'run-ci' || + github.event.label.name == 'run-ci-extra') runs-on: ubuntu-latest steps: - name: Check all dependent job statuses @@ -297,10 +240,14 @@ jobs: # there (initial completion) and pull_request_target (push / label). notify-pr-states: needs: [pr-test-extra-finish] + # Same guard; pr-states.yml subscribes to labeled/unlabeled itself, so nothing is lost. if: | always() && github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name == github.repository + github.event.pull_request.head.repo.full_name == github.repository && + (github.event.action != 'labeled' || + github.event.label.name == 'run-ci' || + github.event.label.name == 'run-ci-extra') runs-on: ubuntu-latest steps: - name: Dispatch pr-states refresh diff --git a/.github/workflows/pr-test-jit-kernel.yml b/.github/workflows/pr-test-jit-kernel.yml index 9a5b17182..e8685122f 100644 --- a/.github/workflows/pr-test-jit-kernel.yml +++ b/.github/workflows/pr-test-jit-kernel.yml @@ -13,9 +13,6 @@ on: description: 'Artifact of prebuilt Rust extension modules, from rust-ext-build. Empty, or a download that fails, falls back to the cache; a miss there compiles during install.' type: string default: '' - runner_config: - required: true - type: string runs_on_map: required: true type: string @@ -43,15 +40,37 @@ env: SKIP_PR_TEST_HEALTH_CHECK: ${{ inputs.skip_pr_test_health_check == true && 'true' || 'false' }} jobs: - jit-kernel-unit-test: + # `name:` reproduces the job names this workflow published before the table. + jit-kernel-test: + # The job name is the suite it runs, with the stage prefix swapped for this + # workflow's, so a row declares a (suite, runner) pair and nothing else. + name: jit-kernel-${{ matrix.suite }}${{ matrix.shard }} # Runs whenever call-jit-kernel-tests dispatches this workflow. That caller is the # single gate (PR jit_kernel changes, or scheduled/parallel-dispatch full runs), so # the sub-jobs no longer re-exclude schedule/parallel-dispatch here. strategy: fail-fast: false matrix: - partition: [0, 1] - runs-on: 1-gpu-h100 + include: + - suite: unit-test-1-gpu-large + runner_config: 1-gpu-large + shard: " (0)" + suite_args: --auto-partition-id 0 --auto-partition-size 2 --fork-worker-batch-size 20 + - suite: unit-test-1-gpu-large + runner_config: 1-gpu-large + shard: " (1)" + suite_args: --auto-partition-id 1 --auto-partition-size 2 --fork-worker-batch-size 20 + - suite: unit-test-1-gpu-small + runner_config: 1-gpu-small + - suite: unit-test-4-gpu-b200 + runner_config: 4-gpu-b200 + - suite: unit-test-8-gpu-h200 + runner_config: 8-gpu-h200 + # Alone among these, it has never run the health check. + skip_health_check: true + - suite: benchmark-test-1-gpu-large + runner_config: 1-gpu-large + runs-on: ${{ fromJson(inputs.runs_on_map)[matrix.runner_config] }} timeout-minutes: 240 steps: - uses: actions/checkout@v4 @@ -59,6 +78,7 @@ jobs: ref: ${{ inputs.git_ref || github.sha }} - uses: ./.github/actions/check-pr-test-health + if: ${{ !matrix.skip_health_check }} - uses: ./.github/actions/check-maintenance @@ -89,138 +109,4 @@ jobs: timeout-minutes: 60 run: | cd test/ - python3 run_suite.py --hw cuda \ - --suite base-b-kernel-unit-test-1-gpu-large \ - --auto-partition-id ${{ matrix.partition }} \ - --auto-partition-size 2 \ - --fork-worker-batch-size 20 - - jit-kernel-multigpu-unit-test: - # Runs whenever call-jit-kernel-tests dispatches this workflow. That caller is the - # single gate (PR jit_kernel changes, or scheduled/parallel-dispatch full runs), so - # the sub-jobs no longer re-exclude schedule/parallel-dispatch here. - runs-on: 8-gpu-h200 - timeout-minutes: 240 - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ inputs.git_ref || github.sha }} - - - uses: ./.github/actions/check-maintenance - - - name: Cleanup - if: inputs.sgl_kernel == 'true' - run: | - ls -alh python/sglang/kernels/aot/dist || true - rm -rf python/sglang/kernels/aot/dist/* || true - - - name: Download artifacts - if: inputs.sgl_kernel == 'true' - uses: actions/download-artifact@v4 - with: - path: python/sglang/kernels/aot/dist/ - merge-multiple: true - pattern: wheel-python3.10-cuda13.0 - - - uses: ./.github/actions/download-rust-ext - with: - artifact_name: ${{ inputs.rust_ext_artifact }} - - - name: Install dependencies - timeout-minutes: 20 - run: | - CUSTOM_BUILD_SGL_KERNEL=${{ inputs.sgl_kernel }} bash scripts/ci/cuda/ci_install_dependency.sh diffusion - - - name: Run multi-GPU test - timeout-minutes: 45 - run: | - cd test/ - python3 run_suite.py --hw cuda --suite base-b-kernel-unit-test-8-gpu-h200 - - jit-kernel-benchmark-test: - # Runs whenever call-jit-kernel-tests dispatches this workflow. That caller is the - # single gate (PR jit_kernel changes, or scheduled/parallel-dispatch full runs), so - # the sub-jobs no longer re-exclude schedule/parallel-dispatch here. - runs-on: 1-gpu-h100 - timeout-minutes: 240 - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ inputs.git_ref || github.sha }} - - - uses: ./.github/actions/check-pr-test-health - - - uses: ./.github/actions/check-maintenance - - - name: Cleanup - if: inputs.sgl_kernel == 'true' - run: | - ls -alh python/sglang/kernels/aot/dist || true - rm -rf python/sglang/kernels/aot/dist/* || true - - - name: Download artifacts - if: inputs.sgl_kernel == 'true' - uses: actions/download-artifact@v4 - with: - path: python/sglang/kernels/aot/dist/ - merge-multiple: true - pattern: wheel-python3.10-cuda13.0 - - - uses: ./.github/actions/download-rust-ext - with: - artifact_name: ${{ inputs.rust_ext_artifact }} - - - name: Install dependencies - timeout-minutes: 20 - run: | - CUSTOM_BUILD_SGL_KERNEL=${{ inputs.sgl_kernel }} bash scripts/ci/cuda/ci_install_dependency.sh diffusion - - - name: Run benchmark tests - timeout-minutes: 45 - run: | - cd test/ - python3 run_suite.py --hw cuda --suite base-b-kernel-benchmark-test-1-gpu-large - - jit-kernel-b200-test: - # Runs whenever call-jit-kernel-tests dispatches this workflow. That caller is the - # single gate (PR jit_kernel changes, or scheduled/parallel-dispatch full runs), so - # the sub-jobs no longer re-exclude schedule/parallel-dispatch here. - runs-on: ${{ fromJson(inputs.runs_on_map)[inputs.runner_config] }} - timeout-minutes: 240 - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ inputs.git_ref || github.sha }} - - - uses: ./.github/actions/check-pr-test-health - - - uses: ./.github/actions/check-maintenance - - - name: Cleanup - if: inputs.sgl_kernel == 'true' - run: | - ls -alh python/sglang/kernels/aot/dist || true - rm -rf python/sglang/kernels/aot/dist/* || true - - - name: Download artifacts - if: inputs.sgl_kernel == 'true' - uses: actions/download-artifact@v4 - with: - path: python/sglang/kernels/aot/dist/ - merge-multiple: true - pattern: wheel-python3.10-cuda13.0 - - - uses: ./.github/actions/download-rust-ext - with: - artifact_name: ${{ inputs.rust_ext_artifact }} - - - name: Install dependencies - timeout-minutes: 20 - run: | - CUSTOM_BUILD_SGL_KERNEL=${{ inputs.sgl_kernel }} bash scripts/ci/cuda/ci_install_dependency.sh diffusion - - - name: Run B200 diffusion test - timeout-minutes: 30 - run: | - cd test/ - python3 run_suite.py --hw cuda --suite base-b-kernel-unit-test-4-gpu-b200 + python3 run_suite.py --hw cuda --suite base-b-kernel-${{ matrix.suite }} ${{ matrix.suite_args }} diff --git a/.github/workflows/pr-test-multimodal-gen.yml b/.github/workflows/pr-test-multimodal-gen.yml index 1baa2c41d..f15e8a45b 100644 --- a/.github/workflows/pr-test-multimodal-gen.yml +++ b/.github/workflows/pr-test-multimodal-gen.yml @@ -130,7 +130,7 @@ jobs: path: | python/sglang/multimodal_gen/test/execution_report_*.json python/diffusion-results.json - retention-days: 1 + retention-days: 7 - name: Upload diffusion failure artifacts if: always() @@ -280,6 +280,9 @@ jobs: fail-fast: false matrix: ${{ fromJson(needs.compute-diffusion-partitions.outputs.matrix-2gpu) }} steps: + - name: Record retry deadline + run: echo "SGLANG_DIFFUSION_RETRY_DEADLINE=$(( $(date +%s) + 2400 ))" >> "$GITHUB_ENV" + - name: Checkout code uses: actions/checkout@v4 with: @@ -327,7 +330,7 @@ jobs: path: | python/sglang/multimodal_gen/test/execution_report_*.json python/diffusion-results.json - retention-days: 1 + retention-days: 7 - name: Upload diffusion failure artifacts if: always() diff --git a/.github/workflows/pr-test-npu.yml b/.github/workflows/pr-test-npu.yml index c676a973a..922d27412 100644 --- a/.github/workflows/pr-test-npu.yml +++ b/.github/workflows/pr-test-npu.yml @@ -7,10 +7,16 @@ on: - cron: '0 12 * * *' # Run daily at 12:00 UTC pull_request: workflow_dispatch: + inputs: + coverage_mode: + description: 'Run in coverage collection mode (uses coverage runners, enables coverage instrumentation, runs setup-covstub)' + required: false + type: boolean + default: false workflow_call: inputs: ref: - description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.' + description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the event commit SHA.' required: false type: string default: '' @@ -19,9 +25,14 @@ on: required: false type: boolean default: false + coverage_mode: + description: 'Run in coverage collection mode (uses coverage runners, enables coverage instrumentation, runs setup-covstub)' + required: false + type: boolean + default: false concurrency: - group: pr-test-npu-${{ inputs.ref || github.ref }} + group: pr-test-npu-${{ inputs.coverage_mode == true && 'coverage-' || '' }}${{ inputs.ref || github.ref }} cancel-in-progress: ${{ github.event_name != 'workflow_call' }} jobs: @@ -36,7 +47,9 @@ jobs: - name: Checkout code uses: actions/checkout@v4 with: - ref: ${{ inputs.ref || github.ref }} + # Pin to the event SHA (not the branch name) so every job in this run + # checks out the same commit even if the branch advances mid-run. + ref: ${{ inputs.ref || github.sha }} - name: Determine run mode id: run-mode @@ -57,10 +70,18 @@ jobs: if: steps.run-mode.outputs.run_all_tests != 'true' with: filters: | + # Everything under python/sglang except multimodal_gen/ and kernels/ops/diffusion/. + # Exclusions have to be extglobs inside a single pattern: paths-filter ORs the + # patterns of a filter (predicate-quantifier defaults to "some"), so a standalone + # "!some/path/**" entry matches every file outside that path and makes the whole + # filter unconditionally true. main_package: - - "python/sglang/**/!(*.md)" - - "!python/sglang/multimodal_gen/**" - - "!python/sglang/kernels/ops/diffusion/**" + - "python/sglang/!(*.md)" + - "python/sglang/!(multimodal_gen|kernels)/**/!(*.md)" + - "python/sglang/kernels/!(*.md)" + - "python/sglang/kernels/!(ops)/**/!(*.md)" + - "python/sglang/kernels/ops/!(*.md)" + - "python/sglang/kernels/ops/!(diffusion)/**/!(*.md)" - "python/pyproject_npu.toml" - "scripts/ci/npu/npu_ci_install_dependency.sh" - "test/registered/npu/**" @@ -77,7 +98,7 @@ jobs: # ==================== PR Gate ==================== # pr-gate: needs: check-changes - if: needs.check-changes.outputs.changes_exist == 'true' + if: ${{ inputs.coverage_mode != true && needs.check-changes.outputs.changes_exist == 'true' }} uses: ./.github/workflows/pr-gate.yml secrets: inherit @@ -96,7 +117,7 @@ jobs: base-a-test-1-npu-a2: needs: [check-changes, pr-gate, set-image-config] - if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }} + if: ${{ !failure() && !cancelled() && inputs.coverage_mode != true && needs.check-changes.outputs.main_package == 'true' }} uses: ./.github/workflows/_npu-pr-test-stage.yml with: self_name: base-a-test-1-npu-a2 @@ -108,68 +129,78 @@ jobs: base-b-test-1-npu-a3: needs: [check-changes, pr-gate, set-image-config, base-a-test-1-npu-a2] - if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }} + if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') }} uses: ./.github/workflows/_npu-pr-test-stage.yml with: self_name: base-b-test-1-npu-a3 - runner_config: linux-aarch64-a3-2- + runner_config: ${{ inputs.coverage_mode == true && 'linux-aarch64-a3-800t-2' || 'linux-aarch64-a3-2-' }} image: ${{ needs.set-image-config.outputs.CANN_image_a3 }} run_timeout_minutes: '60' timeout_per_file: '3600' + use_coverage_runner: ${{ inputs.coverage_mode == true }} + upload_test_logs: ${{ inputs.coverage_mode != true }} secrets: inherit base-b-test-2-npu-a3: needs: [check-changes, pr-gate, set-image-config, base-a-test-1-npu-a2] - if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }} + if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') }} uses: ./.github/workflows/_npu-pr-test-stage.yml with: self_name: base-b-test-2-npu-a3 - runner_config: linux-aarch64-a3-2- + runner_config: ${{ inputs.coverage_mode == true && 'linux-aarch64-a3-800t-2' || 'linux-aarch64-a3-2-' }} image: ${{ needs.set-image-config.outputs.CANN_image_a3 }} run_timeout_minutes: '60' timeout_per_file: '3600' + use_coverage_runner: ${{ inputs.coverage_mode == true }} + upload_test_logs: ${{ inputs.coverage_mode != true }} secrets: inherit base-b-test-4-npu-a3: needs: [check-changes, pr-gate, set-image-config, base-a-test-1-npu-a2] - if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }} + if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') }} uses: ./.github/workflows/_npu-pr-test-stage.yml with: self_name: base-b-test-4-npu-a3 - runner_config: linux-aarch64-a3-4- + runner_config: ${{ inputs.coverage_mode == true && 'linux-aarch64-a3-800t-4' || 'linux-aarch64-a3-4-' }} image: ${{ needs.set-image-config.outputs.CANN_image_a3 }} run_timeout_minutes: '120' timeout_per_file: '3600' partitions: '{"size":2,"arr":[0, 1]}' + use_coverage_runner: ${{ inputs.coverage_mode == true }} + upload_test_logs: ${{ inputs.coverage_mode != true }} secrets: inherit base-b-test-8-npu-a3: needs: [check-changes, pr-gate, set-image-config, base-a-test-1-npu-a2] - if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }} + if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') }} uses: ./.github/workflows/_npu-pr-test-stage.yml with: self_name: base-b-test-8-npu-a3 - runner_config: linux-aarch64-a3-8- + runner_config: ${{ inputs.coverage_mode == true && 'linux-aarch64-a3-800t-8' || 'linux-aarch64-a3-8-' }} image: ${{ needs.set-image-config.outputs.CANN_image_a3 }} run_timeout_minutes: '60' timeout_per_file: '3600' + use_coverage_runner: ${{ inputs.coverage_mode == true }} + upload_test_logs: ${{ inputs.coverage_mode != true }} secrets: inherit base-b-test-16-npu-a3: needs: [check-changes, pr-gate, set-image-config, base-a-test-1-npu-a2] - if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }} + if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') }} uses: ./.github/workflows/_npu-pr-test-stage.yml with: self_name: base-b-test-16-npu-a3 - runner_config: linux-aarch64-a3-16- + runner_config: ${{ inputs.coverage_mode == true && 'linux-aarch64-a3-800t-16' || 'linux-aarch64-a3-16-' }} image: ${{ needs.set-image-config.outputs.CANN_image_a3 }} run_timeout_minutes: '120' timeout_per_file: '3600' + use_coverage_runner: ${{ inputs.coverage_mode == true }} + upload_test_logs: ${{ inputs.coverage_mode != true }} secrets: inherit multimodal-gen-test-1-npu-a3: needs: [check-changes, pr-gate, set-image-config, base-a-test-1-npu-a2] - if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.multimodal_gen == 'true' }} + if: ${{ !failure() && !cancelled() && inputs.coverage_mode != true && needs.check-changes.outputs.multimodal_gen == 'true' }} runs-on: linux-aarch64-a3-800t-2 strategy: fail-fast: false @@ -233,7 +264,7 @@ jobs: multimodal-gen-test-4-npu-a3: needs: [check-changes, pr-gate, set-image-config, base-a-test-1-npu-a2] - if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.multimodal_gen == 'true' }} + if: ${{ !failure() && !cancelled() && inputs.coverage_mode != true && needs.check-changes.outputs.multimodal_gen == 'true' }} runs-on: linux-aarch64-a3-800t-4 strategy: fail-fast: false @@ -298,34 +329,38 @@ jobs: base-c-test-acc-2-npu-a3: name: base-c-test-acc-2-npu-a3 needs: [ check-changes, pr-gate, set-image-config, base-b-test-1-npu-a3, base-b-test-2-npu-a3, base-b-test-4-npu-a3, base-b-test-8-npu-a3, base-b-test-16-npu-a3 ] - if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }} + if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') }} uses: ./.github/workflows/_npu-single-node-test-stage.yml with: - runner: linux-aarch64-a3-2- + runner: ${{ inputs.coverage_mode == true && 'linux-aarch64-a3-800t-2' || 'linux-aarch64-a3-2-' }} test_type: 'accuracy' test_suite: base-c-test-acc-2-npu-a3 image: ${{ needs.set-image-config.outputs.CANN_image_a3 }} install_sglang_deps: true device_type_for_deps: 'a3' partitions: '{"size":2,"arr":[0,1]}' + use_coverage_runner: ${{ inputs.coverage_mode == true }} + upload_test_logs: ${{ inputs.coverage_mode != true }} base-c-test-acc-16-npu-a3: name: base-c-test-acc-16-npu-a3 needs: [ check-changes, pr-gate, set-image-config, base-b-test-1-npu-a3, base-b-test-2-npu-a3, base-b-test-4-npu-a3, base-b-test-8-npu-a3, base-b-test-16-npu-a3 ] - if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }} + if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') }} uses: ./.github/workflows/_npu-single-node-test-stage.yml with: - runner: linux-aarch64-a3-16- + runner: ${{ inputs.coverage_mode == true && 'linux-aarch64-a3-800t-16' || 'linux-aarch64-a3-16-' }} test_type: 'accuracy' test_suite: base-c-test-acc-16-npu-a3 image: ${{ needs.set-image-config.outputs.CANN_image_a3 }} install_sglang_deps: true device_type_for_deps: 'a3' + use_coverage_runner: ${{ inputs.coverage_mode == true }} + upload_test_logs: ${{ inputs.coverage_mode != true }} base-c-test-perf-2-npu-a3: name: base-c-test-perf-2-npu-a3 needs: [ check-changes, pr-gate, set-image-config, base-c-test-acc-2-npu-a3, base-c-test-acc-16-npu-a3 ] - if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }} + if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') }} uses: ./.github/workflows/_npu-single-node-test-stage.yml with: runner: linux-aarch64-a3-800t-2 @@ -334,11 +369,13 @@ jobs: image: ${{ needs.set-image-config.outputs.CANN_image_a3 }} install_sglang_deps: true device_type_for_deps: 'a3' + use_coverage_runner: ${{ inputs.coverage_mode == true }} + upload_test_logs: ${{ inputs.coverage_mode != true }} base-c-test-perf-16-npu-a3: name: base-c-test-perf-16-npu-a3 needs: [ check-changes, pr-gate, set-image-config, base-c-test-acc-2-npu-a3, base-c-test-acc-16-npu-a3 ] - if: ${{ !failure() && !cancelled() && needs.check-changes.outputs.main_package == 'true' }} + if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') }} uses: ./.github/workflows/_npu-single-node-test-stage.yml with: runner: linux-aarch64-a3-800t-16 @@ -348,8 +385,262 @@ jobs: install_sglang_deps: true device_type_for_deps: 'a3' test_timeout_minutes: '180' + use_coverage_runner: ${{ inputs.coverage_mode == true }} + upload_test_logs: ${{ inputs.coverage_mode != true }} + # ==================== Recommend Tests from Coverage ==================== # + # Recommends PR-affected test cases from the pre-built coverage baseline. + # continue-on-error so it never blocks the PR gate. + recommend-tests-from-coverage: + name: Recommend tests from coverage + runs-on: ubuntu-latest + continue-on-error: true + if: ${{ !cancelled() && github.event_name == 'pull_request' && inputs.coverage_mode != true }} + outputs: + coverage_paths: ${{ steps.export-paths.outputs.coverage_paths }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: Checkout sglang scripts + uses: actions/checkout@v4 + with: + sparse-checkout: | + scripts + sparse-checkout-cone-mode: false + + - name: Download test case map + run: | + set -euo pipefail + curl -fsSL \ + --retry 3 \ + --retry-delay 5 \ + --retry-all-errors \ + "https://sglang-npu.obs.cn-southwest-2.myhuaweicloud.com/coverage/test_case_map.json" \ + -o test_case_map.json + test -s test_case_map.json + echo "Downloaded test_case_map.json" + du -h test_case_map.json + + - name: Download coverage package and extract covstub + run: | + set -euo pipefail + curl -fsSL \ + --retry 3 \ + --retry-delay 5 \ + --retry-all-errors \ + "https://sglang-npu.obs.cn-southwest-2.myhuaweicloud.com/coverage/outputs.tar.gz" \ + -o outputs.tar.gz + test -s outputs.tar.gz + echo "Downloaded outputs.tar.gz" + du -h outputs.tar.gz + tar xzf outputs.tar.gz outputs/covstub + test -d outputs/covstub + echo "Extracted outputs/covstub" + find outputs/covstub -maxdepth 2 -type d | head -20 || true + + - name: Recommend tests for current PR + run: | + set -euo pipefail + if ! python3 -c "import regex" >/dev/null 2>&1; then + echo "regex not found for $(python3 -V); bootstrapping pip and installing regex" + curl -fsSL https://bootstrap.pypa.io/get-pip.py -o get-pip.py + python3 get-pip.py --break-system-packages + python3 -m pip install --break-system-packages regex + fi + python3 -c "import regex; print('regex ok:', regex.__file__)" + PR_SPEC="${{ github.repository }}#${{ github.event.pull_request.number }}" + echo "Selecting tests for PR: ${PR_SPEC}" + # test_selector.py resolves relative paths against the script's own dir, + # so pass absolute paths for files downloaded to the workspace root. + python3 scripts/ci/npu/precise-test/test_selector.py \ + --github-pr "${PR_SPEC}" \ + --source-dir "$PWD/outputs/covstub" \ + --map-file "$PWD/test_case_map.json" + + - name: Export coverage paths output + id: export-paths + if: always() + run: | + FILE="scripts/ci/npu/precise-test/recommended_pytest_paths.txt" + if [ -f "$FILE" ]; then + { + echo "coverage_paths<> "$GITHUB_OUTPUT" + echo "Exported $(wc -l < "$FILE") recommended paths" + else + echo "coverage_paths=" >> "$GITHUB_OUTPUT" + echo "::notice::recommended_pytest_paths.txt not found" + fi + + # ==================== Analyze Failure Report ==================== # + # Cross-references test failures with coverage recommendations. + # !cancelled() only: skipped jobs in needs poison success(). + analyze-failure-report: + name: Analyze failure report + needs: + [ + base-a-test-1-npu-a2, + base-b-test-1-npu-a3, + base-b-test-2-npu-a3, + base-b-test-4-npu-a3, + base-b-test-8-npu-a3, + base-b-test-16-npu-a3, + + multimodal-gen-test-1-npu-a3, + multimodal-gen-test-4-npu-a3, + + base-c-test-acc-2-npu-a3, + base-c-test-acc-16-npu-a3, + base-c-test-perf-2-npu-a3, + base-c-test-perf-16-npu-a3, + + recommend-tests-from-coverage, + ] + if: ${{ !cancelled() && inputs.coverage_mode != true }} + uses: ./.github/workflows/_npu-analyze-failure.yml + with: + log_artifact_pattern: selected-test-logs-* + recommendations_content: ${{ needs.recommend-tests-from-coverage.outputs.coverage_paths || '' }} + + # ==================== Coverage Assembly & Publish ==================== # + # Assembles coverage data and publishes the baseline to the shared disk. + # !failure() && !cancelled() (not success()): skipped upstream poisons success(). + setup-covstub: + needs: + [ + set-image-config, + base-b-test-1-npu-a3, + base-b-test-2-npu-a3, + base-b-test-4-npu-a3, + base-b-test-8-npu-a3, + base-b-test-16-npu-a3, + base-c-test-acc-2-npu-a3, + base-c-test-acc-16-npu-a3, + base-c-test-perf-2-npu-a3, + base-c-test-perf-16-npu-a3, + ] + if: ${{ !failure() && !cancelled() && inputs.coverage_mode == true }} + runs-on: linux-aarch64-a3-800t-2 + container: + image: ${{ needs.set-image-config.outputs.CANN_image_a3 }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + # Pin to the event SHA (not the branch name) so every job in this run + # checks out the same commit even if the branch advances mid-run. + ref: ${{ inputs.ref || github.sha }} + + - name: Copy sglang source to covstub + run: | + RUN_DIR="/root/.cache/tests/precise-test/${{ github.run_id }}-attempt-${{ github.run_attempt }}" + COVSTUB="${RUN_DIR}/outputs/covstub" + mkdir -p "${COVSTUB}" + cp -r python/sglang "${COVSTUB}/" + + - name: Build test case map + shell: bash + run: | + set -euo pipefail + + RUN_DIR="/root/.cache/tests/precise-test/${{ github.run_id }}-attempt-${{ github.run_attempt }}" + # Date tag derivation must match scripts/ci/npu/precise-test/run_tests_with_coverage.sh. + # The custom k8s runner only injects explicitly-set env vars, so + # GITHUB_RUN_STARTED_AT may be absent (set -u would abort); fall back + # to local date exactly like run_tests_with_coverage.sh does, so both + # sides derive the same tag. + if [ -n "${GITHUB_RUN_STARTED_AT:-}" ]; then + COV_DATE_TAG="${GITHUB_RUN_STARTED_AT:0:10}" + COV_DATE_TAG="${COV_DATE_TAG//-/}" + else + COV_DATE_TAG="$(date +%Y%m%d)" + fi + COV_ROOT="${RUN_DIR}/outputs/sglang@${COV_DATE_TAG}" + COVSTUB="${RUN_DIR}/outputs/covstub" + # Placed outside outputs/ so it is NOT included in outputs.tar.gz. + MAP_FILE="${RUN_DIR}/test_case_map.json" + + echo "Coverage dir: ${COV_ROOT}" + echo "Source dir: ${COVSTUB}" + + # Coverage data may be absent when all test jobs were skipped + # (e.g. by the changes filter). Skip map building in that case. + if [ ! -d "${COV_ROOT}" ]; then + echo "::warning::Coverage dir not found: ${COV_ROOT}, skip building test case map" + exit 0 + fi + + COVERAGE_FILE_COUNT=$(find "${COV_ROOT}" -name 'coverage.*' -type f | wc -l) + echo "Found ${COVERAGE_FILE_COUNT} coverage files" + if [ "${COVERAGE_FILE_COUNT}" -eq 0 ]; then + echo "::warning::No coverage files found under ${COV_ROOT}, skip building test case map" + exit 0 + fi + + if ! python3 -c "import regex" 2>/dev/null; then + pip install regex \ + -i http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \ + --trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \ + --retries 3 --timeout 60 \ + || pip install regex -i https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple + fi + + python3 scripts/ci/npu/precise-test/test_selector.py \ + --build-map \ + --coverage-dir "${COV_ROOT}" \ + --source-dir "${COVSTUB}" \ + --map-file "${MAP_FILE}" + test -s "${MAP_FILE}" + du -h "${MAP_FILE}" + + - name: Archive outputs + run: | + RUN_DIR="/root/.cache/tests/precise-test/${{ github.run_id }}-attempt-${{ github.run_attempt }}" + tar -czf "${RUN_DIR}/outputs.tar.gz" -C "${RUN_DIR}" outputs + + - name: Publish outputs and map to shared disk + shell: bash + run: | + set -euo pipefail + + RUN_DIR="/root/.cache/tests/precise-test/${{ github.run_id }}-attempt-${{ github.run_attempt }}" + # Fixed publish dir with no RUN_DIR layer: other pipelines have a + # different GITHUB_RUN_ID and must be able to read the baseline. + # Must match PUBLISH_DIR in the recommend-tests-from-coverage job. + PUBLISH_DIR="/root/.cache/tests/precise-test/coverage" + + # Skip publish when no coverage data was collected this run (map + # build was skipped), so we never overwrite the historical + # baseline on the shared disk with an incomplete one. + if [ ! -s "${RUN_DIR}/test_case_map.json" ]; then + echo "::warning::test_case_map.json missing (no coverage data this run), skip publish" + exit 0 + fi + + mkdir -p "${PUBLISH_DIR}" + + # mv (rename) within the same shared filesystem is atomic, so a + # concurrent reader sees either the old or the new baseline, + # never a partial file. + mv "${RUN_DIR}/outputs.tar.gz" "${PUBLISH_DIR}/outputs.tar.gz" + mv "${RUN_DIR}/test_case_map.json" "${PUBLISH_DIR}/test_case_map.json" + + echo "Published baseline to ${PUBLISH_DIR}" + du -h "${PUBLISH_DIR}/outputs.tar.gz" "${PUBLISH_DIR}/test_case_map.json" + + - name: Clean up per-run directory + # Skipped on publish failure, keeping RUN_DIR for manual recovery. + shell: bash + run: | + set -euo pipefail + RUN_DIR="/root/.cache/tests/precise-test/${{ github.run_id }}-attempt-${{ github.run_attempt }}" + rm -rf "${RUN_DIR}" + echo "Removed ${RUN_DIR}" + pr-test-npu-finish: needs: [ @@ -369,6 +660,8 @@ jobs: base-c-test-acc-16-npu-a3, base-c-test-perf-2-npu-a3, base-c-test-perf-16-npu-a3, + + setup-covstub, ] if: always() runs-on: ubuntu-latest diff --git a/.github/workflows/pr-test-xpu.yml b/.github/workflows/pr-test-xpu.yml index d432697f7..9b2530101 100644 --- a/.github/workflows/pr-test-xpu.yml +++ b/.github/workflows/pr-test-xpu.yml @@ -62,6 +62,7 @@ jobs: - "python/sglang/test/!(ascend|observability|mock_model|manual|external_models|kernels)/**/!(*.md)" - "python/pyproject_xpu.toml" - "test/registered/xpu/**/!(*.md)" + - "test/registered/disaggregation/test_disaggregation_xpu.py" - "test/registered/attention/test_chunk_gated_delta_rule.py" - "test/registered/attention/test_deterministic.py" - "test/registered/lora/test_moe_lora_info.py" @@ -251,7 +252,9 @@ jobs: - name: Run diffusion server tests (1-GPU) timeout-minutes: 60 run: | - docker exec ci_sglang_xpu bash -c "source /opt/venv/bin/activate && cd /sglang-checkout/python && python3 sglang/multimodal_gen/test/run_suite.py --suite 1-gpu-xpu" + # xpu_b60.json is seeded with this flag on; without it the SYCL queue + # is deep enough that step timings record host enqueue, not device time. + docker exec -e SGLANG_DIFFUSION_SYNC_STAGE_PROFILING=1 ci_sglang_xpu bash -c "source /opt/venv/bin/activate && cd /sglang-checkout/python && python3 sglang/multimodal_gen/test/run_suite.py --suite 1-gpu-xpu" - name: Cleanup container if: always() diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 1b8ba5f38..27606f848 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -41,17 +41,17 @@ on: type: boolean default: false skip_pr_test_health_check: - description: "Skip PR test health check fast-fail (e.g. for release branch cuts)" + description: "Skip PR test health check fail-fast (e.g. for release branch cuts)" required: false type: boolean default: false concurrency: - # Concurrency group structure: pr-test-{event}-{branch}-{git_ref} # - event_name prevents scheduled runs from colliding with fork PRs whose branch is named 'main' # (without it, both resolve the branch segment to 'main' and block each other) - # - github.head_ref (pull_request) or github.ref_name (workflow_dispatch) normalizes to branch name - group: pr-test-${{ github.event_name }}-${{ github.head_ref || github.ref_name || 'default' }}-${{ inputs.git_ref || 'all' }} + # - a PR keys on its number: github.head_ref is a bare branch name with no owner, + # so two forks using the same name would share one group and cancel each other + group: pr-test-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref_name || 'default' }}-${{ inputs.git_ref || 'all' }} cancel-in-progress: ${{ github.event_name != 'workflow_call' }} env: @@ -88,11 +88,6 @@ jobs: secrets: inherit # =============================================== Wait Jobs for Sequential PR Execution ==================================================== - # These jobs poll GitHub API to wait for previous stages to complete. - # For PR runs: wait jobs run and enforce sequential execution via polling. - # For scheduled runs: wait jobs are skipped, enabling parallel execution for easier retry. - # For PRs with the `bypass-fastfail` label: wait jobs run but return success immediately - # (handled inside the wait-for-jobs action), so downstream stages dispatch in parallel. wait-for-base-a: needs: [check-changes, call-gate] @@ -290,7 +285,6 @@ jobs: needs.check-changes.outputs.jit_kernel == 'true' uses: ./.github/workflows/pr-test-jit-kernel.yml with: - runner_config: 4-gpu-b200 runs_on_map: ${{ needs.check-changes.outputs.runs_on_map }} jit_kernel: ${{ needs.check-changes.outputs.jit_kernel }} # On scheduled/parallel-dispatch runs sgl-kernel-build-wheels is skipped, so the wheel @@ -305,18 +299,27 @@ jobs: # =============================================== primary ==================================================== - # Runs on 5090 (32GB, SM120) - base-a-test-1-gpu-small: + # `name:` is load-bearing: wait-for-jobs gates a stage by job-name prefix, + # and check-pr-test-health carves h20 out by the same name. + base-a-test: + name: base-a-test-${{ matrix.runner_config }} needs: [check-changes, call-gate, sgl-kernel-build-wheels, rust-ext-build] if: ${{ !failure() && !cancelled() }} + strategy: + fail-fast: false + matrix: + include: + # 5090 (32GB, SM120) + - { runner_config: 1-gpu-small, run_timeout: '10', timeout_per_file: '' } uses: ./.github/workflows/_pr-test-stage.yml with: - self_name: base-a-test-1-gpu-small - runner_config: 1-gpu-small + self_name: base-a-test-${{ matrix.runner_config }} + runner_config: ${{ matrix.runner_config }} check_changes: ${{ toJson(needs.check-changes.outputs) }} caller_inputs: ${{ toJson(inputs) }} partitions: ${{ needs.check-changes.outputs.partitions }} - run_timeout_minutes: '10' + run_timeout_minutes: ${{ matrix.run_timeout }} + timeout_per_file: ${{ matrix.timeout_per_file }} rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} secrets: inherit @@ -337,62 +340,28 @@ jobs: # No `secrets: inherit`: this stage has no secret consumer, unlike the GPU # stages' coredump upload. GITHUB_TOKEN and permissions inherit regardless. - # Runs on 5090 (32GB, SM120) - base-b-test-1-gpu-small: + base-b-test: + name: base-b-test-${{ matrix.runner_config }} needs: [check-changes, call-gate, wait-for-base-a, sgl-kernel-build-wheels, rust-ext-build] if: ${{ !failure() && !cancelled() }} + strategy: + fail-fast: false + matrix: + include: + # 5090 (32GB, SM120) + - { runner_config: 1-gpu-small, run_timeout: '30', timeout_per_file: '' } + - { runner_config: 1-gpu-large, run_timeout: '60', timeout_per_file: '1800' } + - { runner_config: 2-gpu-large, run_timeout: '30', timeout_per_file: '' } + - { runner_config: 4-gpu-b200, run_timeout: '40', timeout_per_file: '' } uses: ./.github/workflows/_pr-test-stage.yml with: - self_name: base-b-test-1-gpu-small - runner_config: 1-gpu-small + self_name: base-b-test-${{ matrix.runner_config }} + runner_config: ${{ matrix.runner_config }} check_changes: ${{ toJson(needs.check-changes.outputs) }} caller_inputs: ${{ toJson(inputs) }} partitions: ${{ needs.check-changes.outputs.partitions }} - run_timeout_minutes: '30' - rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} - secrets: inherit - - # Runs on H100 (80GB, SM90) - tests that don't pass on 5090 (FA3, FP8, high VRAM, etc.) - base-b-test-1-gpu-large: - needs: [check-changes, call-gate, wait-for-base-a, sgl-kernel-build-wheels, rust-ext-build] - if: ${{ !failure() && !cancelled() }} - uses: ./.github/workflows/_pr-test-stage.yml - with: - self_name: base-b-test-1-gpu-large - runner_config: 1-gpu-large - check_changes: ${{ toJson(needs.check-changes.outputs) }} - caller_inputs: ${{ toJson(inputs) }} - partitions: ${{ needs.check-changes.outputs.partitions }} - run_timeout_minutes: '60' - timeout_per_file: '1800' - rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} - secrets: inherit - - base-b-test-2-gpu-large: - needs: [check-changes, call-gate, wait-for-base-a, sgl-kernel-build-wheels, rust-ext-build] - if: ${{ !failure() && !cancelled() }} - uses: ./.github/workflows/_pr-test-stage.yml - with: - self_name: base-b-test-2-gpu-large - runner_config: 2-gpu-large - check_changes: ${{ toJson(needs.check-changes.outputs) }} - caller_inputs: ${{ toJson(inputs) }} - partitions: ${{ needs.check-changes.outputs.partitions }} - run_timeout_minutes: '30' - rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} - secrets: inherit - - base-b-test-4-gpu-b200: - needs: [check-changes, call-gate, wait-for-base-a, sgl-kernel-build-wheels, rust-ext-build] - if: ${{ !failure() && !cancelled() }} - uses: ./.github/workflows/_pr-test-stage.yml - with: - self_name: base-b-test-4-gpu-b200 - runner_config: 4-gpu-b200 - check_changes: ${{ toJson(needs.check-changes.outputs) }} - caller_inputs: ${{ toJson(inputs) }} - partitions: ${{ needs.check-changes.outputs.partitions }} - run_timeout_minutes: '40' + run_timeout_minutes: ${{ matrix.run_timeout }} + timeout_per_file: ${{ matrix.timeout_per_file }} rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} secrets: inherit @@ -416,104 +385,70 @@ jobs: skip_pr_test_health_check: ${{ (inputs.skip_pr_test_health_check == true || inputs.test_parallel_dispatch == true || inputs.run_all_tests == true) && 'true' || 'false' }} secrets: inherit - base-c-test-4-gpu-h100: + base-c-test: + name: base-c-test-${{ matrix.runner_config }} needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels, rust-ext-build] if: ${{ !failure() && !cancelled() }} + strategy: + fail-fast: false + matrix: + include: + - { + runner_config: 4-gpu-h100, + run_timeout: '30', + timeout_per_file: '', + warmup_deep_gemm_models: 'lmsys/sglang-ci-dsv3-test:4', + warmup_server_models: 'lmsys/sglang-ci-dsv3-test:4', + } + - { + runner_config: 8-gpu-h200, + run_timeout: '30', + timeout_per_file: '1800', + warmup_deep_gemm_models: 'deepseek-ai/DeepSeek-V3-0324:8 deepseek-ai/DeepSeek-V3.2:8 zai-org/GLM-5-FP8:8 XiaomiMiMo/MiMo-V2-Flash:4 XiaomiMiMo/MiMo-V2.5:8', + warmup_timeout: '60', + } + - { runner_config: 8-gpu-h20, run_timeout: '30', timeout_per_file: '' } + - { runner_config: 4-gpu-b200, run_timeout: '30', timeout_per_file: '1800' } + - { runner_config: 8-gpu-b300, run_timeout: '60', timeout_per_file: '3600' } uses: ./.github/workflows/_pr-test-stage.yml with: - self_name: base-c-test-4-gpu-h100 - runner_config: 4-gpu-h100 + self_name: base-c-test-${{ matrix.runner_config }} + runner_config: ${{ matrix.runner_config }} check_changes: ${{ toJson(needs.check-changes.outputs) }} caller_inputs: ${{ toJson(inputs) }} partitions: ${{ needs.check-changes.outputs.partitions }} - run_timeout_minutes: '30' - warmup_deep_gemm_models: 'lmsys/sglang-ci-dsv3-test:4' - warmup_server_models: 'lmsys/sglang-ci-dsv3-test:4' + run_timeout_minutes: ${{ matrix.run_timeout }} + timeout_per_file: ${{ matrix.timeout_per_file }} + warmup_deep_gemm_models: ${{ matrix.warmup_deep_gemm_models }} + warmup_server_models: ${{ matrix.warmup_server_models }} + # Rows that do not set it keep _pr-test-stage.yml's own default. + warmup_timeout_minutes: ${{ matrix.warmup_timeout || '25' }} rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} secrets: inherit - base-c-test-8-gpu-h200: - needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels, rust-ext-build] - if: ${{ !failure() && !cancelled() }} - uses: ./.github/workflows/_pr-test-stage.yml - with: - self_name: base-c-test-8-gpu-h200 - runner_config: 8-gpu-h200 - check_changes: ${{ toJson(needs.check-changes.outputs) }} - caller_inputs: ${{ toJson(inputs) }} - partitions: ${{ needs.check-changes.outputs.partitions }} - run_timeout_minutes: '30' - timeout_per_file: '1800' - # Per-model TP must match the test's launch in test/registered/ -- see - # FALLBACK_ARGS in scripts/ci/cuda/warmup_deep_gemm.py for extra dp/ep - # flags. Only models that actually invoke DeepGEMM kernels at runtime - # are listed. Cold-cache ~13 min; warm-cache <=30 s via marker file. - # Server CUDA Graph warmup is combined into this step (warmup_server_models unset). - warmup_deep_gemm_models: 'deepseek-ai/DeepSeek-V3-0324:8 deepseek-ai/DeepSeek-V3.2:8 zai-org/GLM-5-FP8:8 XiaomiMiMo/MiMo-V2-Flash:4 XiaomiMiMo/MiMo-V2.5:8' - warmup_timeout_minutes: '60' - rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} - secrets: inherit - - base-c-test-8-gpu-h20: - needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels, rust-ext-build] - if: ${{ !failure() && !cancelled() }} - uses: ./.github/workflows/_pr-test-stage.yml - with: - self_name: base-c-test-8-gpu-h20 - runner_config: 8-gpu-h20 - check_changes: ${{ toJson(needs.check-changes.outputs) }} - caller_inputs: ${{ toJson(inputs) }} - partitions: ${{ needs.check-changes.outputs.partitions }} - run_timeout_minutes: '30' - rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} - secrets: inherit - - base-c-test-4-gpu-b200: - needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels, rust-ext-build] - if: ${{ !failure() && !cancelled() }} - uses: ./.github/workflows/_pr-test-stage.yml - with: - self_name: base-c-test-4-gpu-b200 - runner_config: 4-gpu-b200 - check_changes: ${{ toJson(needs.check-changes.outputs) }} - caller_inputs: ${{ toJson(inputs) }} - partitions: ${{ needs.check-changes.outputs.partitions }} - run_timeout_minutes: '30' - timeout_per_file: '1800' - rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} - secrets: inherit - - base-c-test-4-gpu-gb300: + # Separate only because `needs` cannot vary per matrix row: this stage takes + # the aarch64 build, and neither table should wait on the other's. + base-c-test-aarch64: + name: base-c-test-${{ matrix.runner_config }} needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels, rust-ext-build-aarch64] if: ${{ !failure() && !cancelled() }} + strategy: + fail-fast: false + matrix: + include: + - { runner_config: 4-gpu-gb300, run_timeout: '30', timeout_per_file: '1800' } uses: ./.github/workflows/_pr-test-stage.yml with: - self_name: base-c-test-4-gpu-gb300 - runner_config: 4-gpu-gb300 + self_name: base-c-test-${{ matrix.runner_config }} + runner_config: ${{ matrix.runner_config }} check_changes: ${{ toJson(needs.check-changes.outputs) }} caller_inputs: ${{ toJson(inputs) }} partitions: ${{ needs.check-changes.outputs.partitions }} - run_timeout_minutes: '30' - timeout_per_file: '1800' - # The one aarch64 stage, so it takes the aarch64 build, not rust-ext-build's. + run_timeout_minutes: ${{ matrix.run_timeout }} + timeout_per_file: ${{ matrix.timeout_per_file }} rust_ext_artifact: ${{ needs.rust-ext-build-aarch64.outputs.artifact_name }} secrets: inherit - base-c-test-8-gpu-b300: - needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels, rust-ext-build] - if: ${{ !failure() && !cancelled() }} - uses: ./.github/workflows/_pr-test-stage.yml - with: - self_name: base-c-test-8-gpu-b300 - runner_config: 8-gpu-b300 - check_changes: ${{ toJson(needs.check-changes.outputs) }} - caller_inputs: ${{ toJson(inputs) }} - partitions: ${{ needs.check-changes.outputs.partitions }} - run_timeout_minutes: '60' - timeout_per_file: '3600' - rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} - secrets: inherit - # List every build and test job: `skipped` passes here, so an omission turns that # job's failure into a green run with no tests. pr-test-finish: @@ -536,18 +471,11 @@ jobs: call-multimodal-gen-tests, - base-a-test-1-gpu-small, + base-a-test, base-a-test-cpu, - base-b-test-1-gpu-small, - base-b-test-1-gpu-large, - base-b-test-2-gpu-large, - base-b-test-4-gpu-b200, - base-c-test-4-gpu-h100, - base-c-test-8-gpu-h20, - base-c-test-8-gpu-h200, - base-c-test-4-gpu-b200, - base-c-test-4-gpu-gb300, - base-c-test-8-gpu-b300, + base-b-test, + base-c-test, + base-c-test-aarch64, ] if: always() runs-on: ubuntu-latest diff --git a/3rdparty/amd/wheel/sglang/pyproject.toml b/3rdparty/amd/wheel/sglang/pyproject.toml index d039b1b7d..cbccd1008 100644 --- a/3rdparty/amd/wheel/sglang/pyproject.toml +++ b/3rdparty/amd/wheel/sglang/pyproject.toml @@ -62,7 +62,7 @@ runtime_common = [ "transformers==5.12.1", "uvicorn", "uvloop", - "xgrammar==0.2.1", + "xgrammar==0.2.7", "smg-grpc-servicer>=0.9.0", ] diff --git a/benchmark/disaggregation/bench_cached_prefix.py b/benchmark/disaggregation/bench_cached_prefix.py new file mode 100644 index 000000000..92062c5de --- /dev/null +++ b/benchmark/disaggregation/bench_cached_prefix.py @@ -0,0 +1,113 @@ +"""Measure PD TTFT with a shared cached prefix and unique request suffixes. + +Start a PD router and workers before running this client. Both worker caches are +flushed after warmup; the shared prefix is then warmed on prefill. Decode radix +caching should be disabled to exercise transfer of the entire missing KV range. +Saves per-request responses and client-observed TTFT to the requested JSON file. +""" + +import argparse +import asyncio +import json +import random +import time +from pathlib import Path + +import aiohttp +import numpy as np + + +async def main(args): + rng = random.Random(35762) + prefix = [rng.randrange(1000, 30000) for _ in range(args.prefix)] + prompts = [ + prefix + [rng.randrange(1000, 30000) for _ in range(args.unique)] + for _ in range(args.requests) + ] + timeout = aiohttp.ClientTimeout(total=1800) + async with aiohttp.ClientSession(timeout=timeout) as session: + + async def generate(ids, stream=True): + start = time.perf_counter() + first = None + result = None + body = { + "input_ids": ids, + "sampling_params": { + "temperature": 0, + "max_new_tokens": args.output, + "ignore_eos": True, + }, + "stream": stream, + } + async with session.post(args.url + "/generate", json=body) as response: + response.raise_for_status() + if stream: + async for line in response.content: + if not line.startswith(b"data:"): + continue + payload = line[5:].strip() + if payload == b"[DONE]": + continue + result = json.loads(payload) + if first is None: + first = time.perf_counter() - start + else: + result = await response.json() + elapsed = time.perf_counter() - start + if result is None or "error" in result: + raise RuntimeError(result) + return {"ttft_s": first, "elapsed_s": elapsed, "response": result} + + # Warm both cold-prefill and cached-prefix/concurrent shapes. Clear both + # sides afterwards so measured requests never reuse a unique suffix. + await generate(prompts[0]) + await asyncio.gather(*(generate(p) for p in prompts[1 : args.concurrency + 1])) + for url in args.workers: + async with session.post(url + "/flush_cache", params={"timeout": 30}) as r: + r.raise_for_status() + if prefix: + await generate(prefix) + sem = asyncio.Semaphore(args.concurrency) + + async def run(ids): + async with sem: + return await generate(ids) + + start = time.perf_counter() + records = await asyncio.gather(*(run(ids) for ids in prompts)) + elapsed = time.perf_counter() - start + ttfts = [r["ttft_s"] for r in records] + summary = { + "args": vars(args), + "elapsed_s": elapsed, + "requests_per_s": len(records) / elapsed, + "ttft_mean_ms": float(np.mean(ttfts) * 1000), + "ttft_p50_ms": float(np.percentile(ttfts, 50) * 1000), + "ttft_p99_ms": float(np.percentile(ttfts, 99) * 1000), + "records": records, + } + Path(args.result).write_text(json.dumps(summary, indent=2)) + print(json.dumps({k: v for k, v in summary.items() if k != "records"}), flush=True) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--url", default="http://127.0.0.1:30000") + parser.add_argument( + "--workers", + nargs="+", + default=["http://127.0.0.1:30001", "http://127.0.0.1:30002"], + ) + parser.add_argument("--prefix", type=int, default=83264) + parser.add_argument("--unique", type=int, default=6720) + parser.add_argument("--output", type=int, default=1) + parser.add_argument("--requests", type=int, default=16) + parser.add_argument("--concurrency", type=int, default=1) + parser.add_argument("--result", required=True) + args = parser.parse_args() + if args.prefix < 0 or args.unique <= 0: + parser.error("--prefix must be nonnegative and --unique must be positive") + if min(args.output, args.requests, args.concurrency) <= 0: + parser.error("--output, --requests and --concurrency must be positive") + asyncio.run(main(args)) diff --git a/benchmark/hf3fs/bench_zerocopy.py b/benchmark/hf3fs/bench_zerocopy.py index a4ed66912..8559e45b7 100644 --- a/benchmark/hf3fs/bench_zerocopy.py +++ b/benchmark/hf3fs/bench_zerocopy.py @@ -5,10 +5,10 @@ import torch from tqdm import tqdm from sglang.srt.distributed import ( - get_world_group, init_distributed_environment, initialize_model_parallel, ) +from sglang.srt.distributed.parallel_state import get_world_group from sglang.srt.managers.cache_controller import ( HiCacheController, PrefetchOperation, @@ -17,6 +17,7 @@ from sglang.srt.managers.cache_controller import ( from sglang.srt.mem_cache.allocator import TokenToKVPoolAllocator from sglang.srt.mem_cache.memory_pool import MHATokenToKVPool from sglang.srt.mem_cache.pool_host.mha import MHATokenToKVPoolHost +from sglang.test.test_utils import publish_build_topology init_distributed_environment( world_size=1, @@ -26,10 +27,8 @@ init_distributed_environment( backend="gloo", ) -initialize_model_parallel( - tensor_model_parallel_size=1, - pipeline_model_parallel_size=1, -) +publish_build_topology() +initialize_model_parallel() group = get_world_group().cpu_group diff --git a/benchmark/kernels/all_reduce/benchmark_all_reduce.py b/benchmark/kernels/all_reduce/benchmark_all_reduce.py index 30a7733f3..41dda1930 100644 --- a/benchmark/kernels/all_reduce/benchmark_all_reduce.py +++ b/benchmark/kernels/all_reduce/benchmark_all_reduce.py @@ -21,6 +21,7 @@ from sglang.srt.distributed.parallel_state import ( init_distributed_environment, initialize_model_parallel, ) +from sglang.test.test_utils import publish_build_topology def parse_args(): @@ -85,7 +86,8 @@ def init_dist(backend: str): distributed_init_method=distributed_init_method, local_rank=rank, ) - initialize_model_parallel(tensor_model_parallel_size=world_size) + publish_build_topology(world_rank=rank, tp_size=world_size) + initialize_model_parallel() return dist.group.WORLD diff --git a/benchmark/kernels/all_reduce/benchmark_fused_ar_rms_amd.py b/benchmark/kernels/all_reduce/benchmark_fused_ar_rms_amd.py index 1fa3819cc..107533fb1 100644 --- a/benchmark/kernels/all_reduce/benchmark_fused_ar_rms_amd.py +++ b/benchmark/kernels/all_reduce/benchmark_fused_ar_rms_amd.py @@ -41,6 +41,7 @@ from sglang.srt.distributed.parallel_state import ( initialize_model_parallel, set_custom_all_reduce, ) +from sglang.test.test_utils import publish_build_topology Shape = Tuple[int, int] @@ -381,7 +382,8 @@ def main(): distributed_init_method="env://", backend="nccl", ) - initialize_model_parallel(tensor_model_parallel_size=world_size) + publish_build_topology(world_rank=rank, tp_size=world_size) + initialize_model_parallel() prefill_shapes = parse_shapes(args.prefill_shapes) decode_shapes = parse_shapes(args.decode_shapes) diff --git a/benchmark/kernels/all_reduce/benchmark_fused_ar_rms_quant_amd.py b/benchmark/kernels/all_reduce/benchmark_fused_ar_rms_quant_amd.py index 85cabe94c..4c02a5df4 100644 --- a/benchmark/kernels/all_reduce/benchmark_fused_ar_rms_quant_amd.py +++ b/benchmark/kernels/all_reduce/benchmark_fused_ar_rms_quant_amd.py @@ -47,6 +47,7 @@ from sglang.srt.distributed.parallel_state import ( initialize_model_parallel, set_custom_all_reduce, ) +from sglang.test.test_utils import publish_build_topology Shape = Tuple[int, int] FP8_DTYPE = torch.float8_e4m3fnuz @@ -400,7 +401,8 @@ def main() -> None: distributed_init_method="env://", backend="nccl", ) - initialize_model_parallel(tensor_model_parallel_size=world_size) + publish_build_topology(world_rank=rank, tp_size=world_size) + initialize_model_parallel() if rank == 0: print( diff --git a/benchmark/kernels/all_reduce/benchmark_mscclpp.py b/benchmark/kernels/all_reduce/benchmark_mscclpp.py index 5b72d2f46..87dfd17e5 100644 --- a/benchmark/kernels/all_reduce/benchmark_mscclpp.py +++ b/benchmark/kernels/all_reduce/benchmark_mscclpp.py @@ -30,6 +30,7 @@ from sglang.srt.distributed.parallel_state import ( initialize_model_parallel, set_mscclpp_all_reduce, ) +from sglang.test.test_utils import publish_build_topology def torch_allreduce(torch_input: torch.Tensor, group: ProcessGroup) -> torch.Tensor: @@ -173,7 +174,8 @@ if __name__ == "__main__": rank=rank, local_rank=rank % 8, ) - initialize_model_parallel(tensor_model_parallel_size=world_size) + publish_build_topology(world_rank=rank, tp_size=world_size) + initialize_model_parallel() group = get_tensor_model_parallel_group().device_group cpu_group = get_tensor_model_parallel_group().cpu_group pynccl_comm = get_tensor_model_parallel_group().pynccl_comm diff --git a/benchmark/kernels/all_reduce/benchmark_torch_symm_mem.py b/benchmark/kernels/all_reduce/benchmark_torch_symm_mem.py index 5bdb7f5d6..3d2c65870 100644 --- a/benchmark/kernels/all_reduce/benchmark_torch_symm_mem.py +++ b/benchmark/kernels/all_reduce/benchmark_torch_symm_mem.py @@ -44,6 +44,7 @@ from sglang.srt.distributed.parallel_state import ( initialize_model_parallel, set_torch_symm_mem_all_reduce, ) +from sglang.test.test_utils import publish_build_topology from sglang.utils import is_in_ci IS_CI = is_in_ci() @@ -188,7 +189,8 @@ if __name__ == "__main__": rank=rank, local_rank=rank % 8, ) - initialize_model_parallel(tensor_model_parallel_size=world_size) + publish_build_topology(world_rank=rank, tp_size=world_size) + initialize_model_parallel() group = get_tensor_model_parallel_group().device_group cpu_group = get_tensor_model_parallel_group().cpu_group pynccl_comm = get_tensor_model_parallel_group().pynccl_comm diff --git a/benchmark/kernels/flashinfer_allreduce_fusion/benchmark_fused_collective.py b/benchmark/kernels/flashinfer_allreduce_fusion/benchmark_fused_collective.py index 2905a62a1..9af15ac87 100644 --- a/benchmark/kernels/flashinfer_allreduce_fusion/benchmark_fused_collective.py +++ b/benchmark/kernels/flashinfer_allreduce_fusion/benchmark_fused_collective.py @@ -30,14 +30,16 @@ import torch.distributed as dist # type: ignore from sglang.kernels.ops.quantization.fp8_kernel import fp8_dtype as SGLANG_FP8_DTYPE from sglang.kernels.ops.quantization.fp8_kernel import static_quant_fp8 -from sglang.srt.distributed import get_tp_group, tensor_model_parallel_all_reduce +from sglang.srt.distributed import tensor_model_parallel_all_reduce from sglang.srt.distributed.parallel_state import ( cleanup_dist_env_and_memory, + get_tp_group, graph_capture, init_distributed_environment, initialize_model_parallel, ) from sglang.srt.layers.layernorm import RMSNorm # noqa +from sglang.test.test_utils import publish_build_topology try: from sgl_kernel import fused_add_rmsnorm as SGL_FUSED_ADD_RMS_NORM @@ -1178,7 +1180,8 @@ def main(): local_rank=rank, backend="nccl", ) - initialize_model_parallel(tensor_model_parallel_size=world_size) + publish_build_topology(world_rank=rank, tp_size=world_size) + initialize_model_parallel() # Validate world size (must be > 1 for collective operations) if world_size <= 1: diff --git a/benchmark/kernels/fused_moe_triton/benchmark_sglang_fused_moe_triton.py b/benchmark/kernels/fused_moe_triton/benchmark_sglang_fused_moe_triton.py index 4515ff53b..6934650ad 100644 --- a/benchmark/kernels/fused_moe_triton/benchmark_sglang_fused_moe_triton.py +++ b/benchmark/kernels/fused_moe_triton/benchmark_sglang_fused_moe_triton.py @@ -26,6 +26,7 @@ from sglang.srt.layers.moe.topk import ( select_experts, ) from sglang.srt.server_args import ServerArgs, set_global_server_args_for_scheduler +from sglang.test.test_utils import publish_build_topology def fused_moe_triton_api( @@ -227,10 +228,8 @@ def main(): backend="nccl" if torch.cuda.is_available() else "gloo", ) - initialize_model_parallel( - tensor_model_parallel_size=1, - expert_model_parallel_size=1, - ) + publish_build_topology() + initialize_model_parallel() model_config = get_model_config(args.model, args.tp_size, args.ep_size) benchmark.run( diff --git a/benchmark/kernels/fused_moe_triton/benchmark_vllm_vs_sglang_fused_moe_triton.py b/benchmark/kernels/fused_moe_triton/benchmark_vllm_vs_sglang_fused_moe_triton.py index fc100ce50..1adecbeae 100644 --- a/benchmark/kernels/fused_moe_triton/benchmark_vllm_vs_sglang_fused_moe_triton.py +++ b/benchmark/kernels/fused_moe_triton/benchmark_vllm_vs_sglang_fused_moe_triton.py @@ -15,6 +15,7 @@ from sglang.srt.distributed.parallel_state import ( from sglang.srt.layers.moe.moe_runner.triton_utils.fused_moe import ( fused_moe as fused_moe_sglang, ) +from sglang.test.test_utils import publish_build_topology from .common_utils import get_model_config @@ -243,10 +244,8 @@ def main(): backend="nccl" if torch.cuda.is_available() else "gloo", ) - initialize_model_parallel( - tensor_model_parallel_size=1, - pipeline_model_parallel_size=1, - ) + publish_build_topology() + initialize_model_parallel() shape_configs = get_model_config(args.model, args.tp_size, args.ep_size) benchmark.run( diff --git a/docker/renderer.Dockerfile b/docker/renderer.Dockerfile new file mode 100644 index 000000000..804095194 --- /dev/null +++ b/docker/renderer.Dockerfile @@ -0,0 +1,89 @@ +# syntax=docker/dockerfile:1 + +# Keep the compiler aligned with rust/rust-toolchain.toml. Pin image indexes +# rather than individual architecture manifests so both platforms use this file. +FROM rust:1.92.0-slim-bookworm@sha256:f1f73538ebe623fd3673a35aff3df358ae1084c64c55646516e5b17b321b6c9b AS build + +ARG TARGETARCH +ARG CARGO_BUILD_JOBS=4 +ENV RUSTUP_TOOLCHAIN=1.92.0 \ + CARGO_BUILD_JOBS=${CARGO_BUILD_JOBS} \ + PCRE2_SYS_STATIC=1 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends build-essential pkg-config \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /build +COPY rust/Cargo.toml rust/Cargo.lock rust/rust-toolchain.toml rust/ +# Cargo loads every workspace member even when building only the renderer. +COPY rust/sglang-grpc/Cargo.toml rust/sglang-grpc/ +COPY rust/sglang-grpc/src/ rust/sglang-grpc/src/ +COPY rust/sglang-mm/Cargo.toml rust/sglang-mm/ +COPY rust/sglang-mm/src/ rust/sglang-mm/src/ +COPY rust/sglang-server/Cargo.toml rust/sglang-server/ +COPY rust/sglang-server/src/ rust/sglang-server/src/ +COPY rust/sglang-renderer/Cargo.toml rust/sglang-renderer/ +COPY rust/sglang-renderer/src/ rust/sglang-renderer/src/ + +# Avoid rustup downloading development components from rust-toolchain.toml, +# but fail if the image's compiler and the workspace toolchain drift apart. +RUN channel=$(sed -n 's/^channel = "\([^"]*\)"/\1/p' rust/rust-toolchain.toml) \ + && case "${RUSTUP_TOOLCHAIN}" in "$channel"|"$channel".*) ;; *) exit 1 ;; esac + +RUN --mount=type=cache,id=renderer-registry-${TARGETARCH},target=/usr/local/cargo/registry,sharing=locked \ + --mount=type=cache,id=renderer-git-${TARGETARCH},target=/usr/local/cargo/git,sharing=locked \ + --mount=type=cache,id=renderer-target-${TARGETARCH},target=/build/rust/target,sharing=locked \ + cargo build --manifest-path rust/Cargo.toml -p sglang-renderer \ + --bin sglang-renderer --release --features http --locked \ + && install -D rust/target/release/sglang-renderer /out/sglang-renderer + +# Run the existing unit suite in the same Linux toolchain used for the image. +# This sibling stage is selected by CI and is not a dependency of the runtime. +FROM build AS test +COPY rust/sglang-renderer/tests/ rust/sglang-renderer/tests/ +COPY experimental/sgl-router/tests/fixtures/tiny_tokenizer.json experimental/sgl-router/tests/fixtures/tiny_tokenizer.json +RUN --mount=type=cache,id=renderer-registry-${TARGETARCH},target=/usr/local/cargo/registry,sharing=locked \ + --mount=type=cache,id=renderer-git-${TARGETARCH},target=/usr/local/cargo/git,sharing=locked \ + --mount=type=cache,id=renderer-target-${TARGETARCH},target=/build/rust/target,sharing=locked \ + cargo test --manifest-path rust/Cargo.toml -p sglang-renderer --features http --locked + +FROM debian:bookworm-slim@sha256:88200866dfff7ea7f5cbcb6ec7c8a701889efe6fe859fe64d6990e4b07ea4171 AS runtime + +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates libgcc-s1 \ + && rm -rf /var/lib/apt/lists/* \ + && groupadd --gid 65532 sglang \ + && useradd --uid 65532 --gid 65532 --no-log-init --create-home \ + --home-dir /home/sglang --shell /usr/sbin/nologin sglang \ + && mkdir -p /home/sglang/.cache/huggingface \ + && chown -R 65532:65532 /home/sglang + +COPY --from=build /out/sglang-renderer /usr/local/bin/sglang-renderer +COPY LICENSE /usr/share/licenses/sglang-renderer/LICENSE + +# Metadata changes must not invalidate compilation. +ARG SGLANG_BUILD_COMMIT=unknown +ARG SGLANG_BUILD_URL= +ARG SGLANG_IMAGE_TAG=local/sglang-renderer:dev +ENV HOME=/home/sglang \ + HF_HOME=/home/sglang/.cache/huggingface \ + SGLANG_BUILD_COMMIT=${SGLANG_BUILD_COMMIT} \ + SGLANG_BUILD_URL=${SGLANG_BUILD_URL} \ + SGLANG_IMAGE_TAG=${SGLANG_IMAGE_TAG} +LABEL org.opencontainers.image.source="https://github.com/sgl-project/sglang" \ + org.opencontainers.image.licenses="Apache-2.0" \ + org.opencontainers.image.revision="${SGLANG_BUILD_COMMIT}" \ + org.opencontainers.image.version="${SGLANG_IMAGE_TAG}" \ + org.opencontainers.image.url="${SGLANG_BUILD_URL}" \ + ai.sglang.build.commit="${SGLANG_BUILD_COMMIT}" \ + ai.sglang.build.url="${SGLANG_BUILD_URL}" \ + ai.sglang.image.tag="${SGLANG_IMAGE_TAG}" + +USER 65532:65532 +WORKDIR /home/sglang +EXPOSE 30000 +# The renderer's existing graceful shutdown handler listens for Ctrl-C. +STOPSIGNAL SIGINT +ENTRYPOINT ["/usr/local/bin/sglang-renderer"] +CMD ["--help"] diff --git a/docker/rocm.Dockerfile b/docker/rocm.Dockerfile index 30ee6947f..3682a5264 100644 --- a/docker/rocm.Dockerfile +++ b/docker/rocm.Dockerfile @@ -61,7 +61,7 @@ ENV BUILD_TRITON="0" ENV BUILD_LLVM="0" ENV BUILD_AITER_ALL="1" ENV BUILD_MOONCAKE="1" -ENV AITER_COMMIT_DEFAULT="4ad99832823dde2315b361cbd3b54b1c5c12acd5" +ENV AITER_COMMIT_DEFAULT="acf8fdf9307431ece8ee275971c41cb3d1a7020b" # =============================== # Base image 942 with rocm720 and args @@ -71,7 +71,7 @@ ENV BUILD_TRITON="1" ENV BUILD_LLVM="0" ENV BUILD_AITER_ALL="1" ENV BUILD_MOONCAKE="1" -ENV AITER_COMMIT_DEFAULT="4ad99832823dde2315b361cbd3b54b1c5c12acd5" +ENV AITER_COMMIT_DEFAULT="acf8fdf9307431ece8ee275971c41cb3d1a7020b" ENV TRITON_COMMIT_DEFAULT="42270451990532c67e69d753fbd026f28fcc4840" # =============================== @@ -82,7 +82,7 @@ ENV BUILD_TRITON="1" ENV BUILD_LLVM="0" ENV BUILD_AITER_ALL="1" ENV BUILD_MOONCAKE="1" -ENV AITER_COMMIT_DEFAULT="4ad99832823dde2315b361cbd3b54b1c5c12acd5" +ENV AITER_COMMIT_DEFAULT="acf8fdf9307431ece8ee275971c41cb3d1a7020b" # Pin the ROCm torch stack for every pip invocation in this flavor. The file is # filled in after the torch 2.11 upgrade below; it must already exist (empty is # valid) because pip reads PIP_CONSTRAINT from the first pip call onwards. @@ -106,7 +106,7 @@ ENV BUILD_TRITON="0" ENV BUILD_LLVM="0" ENV BUILD_AITER_ALL="1" ENV BUILD_MOONCAKE="1" -ENV AITER_COMMIT_DEFAULT="4ad99832823dde2315b361cbd3b54b1c5c12acd5" +ENV AITER_COMMIT_DEFAULT="acf8fdf9307431ece8ee275971c41cb3d1a7020b" # =============================== # Base image 950 with rocm720 and args @@ -116,7 +116,7 @@ ENV BUILD_TRITON="1" ENV BUILD_LLVM="0" ENV BUILD_AITER_ALL="1" ENV BUILD_MOONCAKE="1" -ENV AITER_COMMIT_DEFAULT="4ad99832823dde2315b361cbd3b54b1c5c12acd5" +ENV AITER_COMMIT_DEFAULT="acf8fdf9307431ece8ee275971c41cb3d1a7020b" ENV TRITON_COMMIT_DEFAULT="42270451990532c67e69d753fbd026f28fcc4840" # =============================== @@ -127,7 +127,7 @@ ENV BUILD_TRITON="1" ENV BUILD_LLVM="0" ENV BUILD_AITER_ALL="1" ENV BUILD_MOONCAKE="1" -ENV AITER_COMMIT_DEFAULT="4ad99832823dde2315b361cbd3b54b1c5c12acd5" +ENV AITER_COMMIT_DEFAULT="acf8fdf9307431ece8ee275971c41cb3d1a7020b" # Pin the ROCm torch stack for every pip invocation in this flavor. The file is # filled in after the torch 2.11 upgrade below; it must already exist (empty is # valid) because pip reads PIP_CONSTRAINT from the first pip call onwards. @@ -286,7 +286,7 @@ ENV BUILD_TRITON="0" ENV BUILD_LLVM="0" ENV BUILD_AITER_ALL="1" ENV BUILD_MOONCAKE="1" -ENV AITER_COMMIT_DEFAULT="4ad99832823dde2315b361cbd3b54b1c5c12acd5" +ENV AITER_COMMIT_DEFAULT="acf8fdf9307431ece8ee275971c41cb3d1a7020b" # Same reasoning as the rocm724 stages: keep pip from resolving the image's # ROCm torch away to a PyPI CUDA build. Populated after the stack is in place. ENV PIP_CONSTRAINT="/etc/sglang/constraints/torch-rocm.txt" @@ -300,7 +300,7 @@ ENV BUILD_TRITON="0" ENV BUILD_LLVM="0" ENV BUILD_AITER_ALL="1" ENV BUILD_MOONCAKE="1" -ENV AITER_COMMIT_DEFAULT="4ad99832823dde2315b361cbd3b54b1c5c12acd5" +ENV AITER_COMMIT_DEFAULT="acf8fdf9307431ece8ee275971c41cb3d1a7020b" ENV PIP_CONSTRAINT="/etc/sglang/constraints/torch-rocm.txt" RUN mkdir -p /etc/sglang/constraints && : > /etc/sglang/constraints/torch-rocm.txt diff --git a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-OCR-2.mdx b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-OCR-2.mdx index cfe54990b..3aa56939e 100644 --- a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-OCR-2.mdx +++ b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-OCR-2.mdx @@ -30,15 +30,15 @@ For more details, please refer to the [official DeepSeek-OCR-2 repository](https Please refer to the [official SGLang installation guide](../../../docs/get-started/install) for installation instructions. -For SGLang CPU installation, please refer to the [CPU version installation guide](../../../docs/hardware-platforms/cpu_server#installation). - ## 3. Model Deployment This section provides deployment configurations optimized for different hardware platforms and use cases. ### 3.1 Basic Configuration -**Interactive Command Generator**: Use the configuration selector below to automatically generate the appropriate deployment command for your hardware platform, quantization method, and deployment strategy. SGLang supports serving DeepSeek-OCR-2 on NVIDIA H200 and B200, AMD MI300X, MI355X, and MI325X GPUs, as well as Intel Xeon CPUs. +The DeepSeek-OCR-2 series offers models in various sizes and architectures, optimized for different hardware platforms including NVIDIA GPUs, AMD GPUs, Intel Arc Pro B-Series GPUs(codename: BMG (Battlemage)), and Intel Xeon CPUs. The recommended launch configurations vary by hardware and model size. + +**Interactive Command Generator**: Use the configuration selector below to automatically generate the appropriate deployment command for your hardware platform, quantization method, and deployment strategy. SGLang supports serving DeepSeek-OCR-2 on NVIDIA H200 and B200, AMD MI300X, MI355X, and MI325X GPUs, and Intel Arc Pro B-Series GPUs, as well as Intel Xeon CPUs. diff --git a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx index 61f5f5aa8..5c866aaa5 100644 --- a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx +++ b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx @@ -692,7 +692,7 @@ Larger blocks can improve decode latency when acceptance stays high, but they al For every candidate, compare with the same recipe without `--speculative-algorithm DSPARK`. Restart the server between the DSpark and non-speculative legs, keep the request corpus, sampling, concurrency, and warmup identical, and give each `bench_serving` leg its own `--flush-cache`. Leave `--speculative-draft-attention-backend` unset unless a separate profiling run justifies an override. -DSpark requires `pp_size == 1`. It is not compatible with PD disaggregation on current SGLang releases; selecting a prefill or decode role in the Playground automatically removes the inherited DSpark flags. The MI355X Flash Official recipes therefore run target-only, and so do the DP-Attention recipes in the Deploy panel — for DP-Attention configurations that do run DSpark, see the [B200 agentic recipe](#3-6-agentic-long-context-with-hicache-dram-offload-b200-fp4-dspark) and the [MI355X agentic recipe](#3-7-agentic-long-context-with-hicache-dram-offload-mi355x-fp4-dspark). If a larger draft block or concurrency causes graph-capture OOM, lower `--mem-fraction-static`, the draft block size, or the configured maximum running requests, then rerun both performance and accuracy gates. +DSpark requires `pp_size == 1`. Selecting a prefill or decode role in the Playground removes the inherited DSpark flags; for a PD-disaggregated configuration that runs DSpark on both roles, see the [B200 agentic recipe](#3-6-agentic-long-context-with-hicache-dram-offload-b200-fp4-dspark). The MI355X Flash Official recipes run target-only, and so do the DP-Attention recipes in the Deploy panel — for DP-Attention configurations that do run DSpark, see the [B200 agentic recipe](#3-6-agentic-long-context-with-hicache-dram-offload-b200-fp4-dspark) and the [MI355X agentic recipe](#3-7-agentic-long-context-with-hicache-dram-offload-mi355x-fp4-dspark). If a larger draft block or concurrency causes graph-capture OOM, lower `--mem-fraction-static`, the draft block size, or the configured maximum running requests, then rerun both performance and accuracy gates. ### 3.5 Vision (Image Inputs) @@ -737,7 +737,7 @@ Pending update... ### 3.6 Agentic Long-Context with HiCache DRAM Offload (B200 FP4, DSpark) -**TP8, concurrency 8–16:** +**TP8, concurrency 1–8:** ```bash Command SGLANG_ENABLE_UNIFIED_RADIX_TREE=1 \ python3 -m sglang.launch_server \ @@ -761,7 +761,7 @@ python3 -m sglang.launch_server \ --hicache-mem-layout page_first_direct ``` -DSv4 HiCache sizes the host tier with `--hicache-ratio` (host/device token ratio), not `--hicache-size`. Concurrency 1–5 runs the same command without the HiCache flags. +DSv4 HiCache sizes the host tier with `--hicache-ratio` (host/device token ratio), not `--hicache-size`. **DEP8 (DP Attention), concurrency 64–160:** ```bash Command @@ -796,6 +796,87 @@ python3 -m sglang.launch_server \ `--chunked-prefill-size` is a global budget divided by `--dp`, so this keeps 6144 tokens per rank. +**Disaggregated 1P1D (DEP8 prefill / DEP8 decode), concurrency 64–128.** Prefill, on one 8-GPU node: +```bash Command +SGLANG_DSV4_MHC_PREWARM=1 \ +SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=9216 \ +NCCL_MNNVL_ENABLE=1 \ +NCCL_CUMEM_ENABLE=1 \ +python3 -m sglang.launch_server \ + --model-path deepseek-ai/DeepSeek-V4-Pro-0813 \ + --trust-remote-code \ + --tp 8 \ + --dp 8 \ + --enable-dp-attention \ + --enable-dp-lm-head \ + --ep-size 8 \ + --moe-dense-tp-size 1 \ + --moe-a2a-backend megamoe \ + --enable-w4a4-mxfp4-megamoe \ + --enable-deepseek-v4-fp4-indexer \ + --disable-flashinfer-autotune \ + --mem-fraction-static 0.85 \ + --page-size 256 \ + --swa-full-tokens-ratio 0.01 \ + --chunked-prefill-size 65536 \ + --tool-call-parser deepseekv4 \ + --reasoning-parser deepseek-v4 \ + --speculative-algorithm DSPARK \ + --speculative-dspark-block-size 6 \ + --enable-hierarchical-cache \ + --hicache-ratio 8 \ + --hicache-write-policy write_back \ + --hicache-io-backend direct \ + --hicache-mem-layout page_first_direct \ + --disaggregation-mode prefill \ + --disaggregation-transfer-backend mooncake \ + --disaggregation-ib-device mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_10,mlx5_11 +``` + +Decode, on a second 8-GPU node: +```bash Command +SGLANG_DSV4_MHC_PREWARM=1 \ +SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=4096 \ +NCCL_MNNVL_ENABLE=1 \ +NCCL_CUMEM_ENABLE=1 \ +python3 -m sglang.launch_server \ + --model-path deepseek-ai/DeepSeek-V4-Pro-0813 \ + --trust-remote-code \ + --tp 8 \ + --dp 8 \ + --enable-dp-attention \ + --enable-dp-lm-head \ + --ep-size 8 \ + --moe-dense-tp-size 1 \ + --moe-a2a-backend megamoe \ + --enable-w4a4-mxfp4-megamoe \ + --enable-deepseek-v4-fp4-indexer \ + --disable-flashinfer-autotune \ + --mem-fraction-static 0.90 \ + --page-size 256 \ + --swa-full-tokens-ratio 0.02 \ + --tool-call-parser deepseekv4 \ + --reasoning-parser deepseek-v4 \ + --speculative-algorithm DSPARK \ + --speculative-dspark-block-size 6 \ + --disaggregation-mode decode \ + --disaggregation-transfer-backend mooncake \ + --disaggregation-ib-device mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_10,mlx5_11 +``` + +Router: +```bash Command +python3 -m sglang_router.launch_router \ + --pd-disaggregation \ + --prefill http://:30000 8998 \ + --decode http://:30000 \ + --host 0.0.0.0 --port 8000 +``` + +HiCache runs on the prefill role only, with `write_back` rather than the aggregate recipe's `write_through`. Set `--disaggregation-ib-device` to the node's own external HCAs. + +Concurrency 256 uses 2P1D — two prefill workers, one node each, added with a second `--prefill` on the router — with `--swa-full-tokens-ratio 0.02` on prefill and `--mem-fraction-static 0.91 --swa-full-tokens-ratio 0.005` on decode. + ### 3.7 Agentic Long-Context with HiCache DRAM Offload (MI355X FP4, DSpark) DeepSeek-V4-Pro-0813 bundles the DSpark head, so `--speculative-draft-model-path` is not needed. `--speculative-dspark-block-size 6` is the AL-optimal draft length on the committed golden curve (verify window 7). diff --git a/docs/cookbook/autoregressive/GLM/GLM-5.2.mdx b/docs/cookbook/autoregressive/GLM/GLM-5.2.mdx index a3d8a19dc..7953cc30f 100644 --- a/docs/cookbook/autoregressive/GLM/GLM-5.2.mdx +++ b/docs/cookbook/autoregressive/GLM/GLM-5.2.mdx @@ -124,7 +124,7 @@ import { Playground } from "/src/snippets/_playground.jsx"; **gfx950 block-FP8 accuracy: fixed as of the pinned MI355X image (`v0.5.13.post1-rocm720-mi35x-20260618`).** Earlier SGLang ROCm images miscompiled AMD aiter's `gemm_a8w8_blockscale_bpreshuffle` GEMM on gfx950 (ROCm 7.2): the error was small per layer but compounded across all 78 layers and silently corrupted output — in-context reasoning broke (GSM8K ≈ 0) while short factual prompts still looked fine. The root cause was a gfx950/ROCm-7.2 miscompile of the CK kernel (a packed illegal-type FMA that relied on an LLVM coercion pass removed in ROCm 7.2; non-deterministic wrong rows near tile boundaries). This is resolved in the pinned image and newer: GLM-5.2-FP8 on MI350X/MI355X (gfx950) was re-validated at TP4 and TP8 — **GSM8K ≈ 0.96 (0% invalid)** and **15/15 needle-in-haystack retrieval to ~118K tokens**. **MI300X / MI325X (gfx942) were never affected.** If you must run an older image, treat gfx950 FP8 output as unverified. Background: [sgl-project/sglang#28685](https://github.com/sgl-project/sglang/issues/28685) (analysis) and the upstream CK fix [ROCm/rocm-libraries#8639](https://github.com/ROCm/rocm-libraries/pull/8639) (scalar FMA + accumulator anchor; restores correctness and determinism at -O3). -- **MTP / EAGLE speculative decoding on AMD.** Five-step MTP is validated for `amd/GLM-5.2-MXFP4` on MI355X with `lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260916`. Choose **Low-Latency** for the GPU-resident TP8/EP1 topology, or **High-Throughput** for the TP4/EP4 topology. Both emit `--speculative-num-steps 5 --speculative-eagle-topk 1 --speculative-num-draft-tokens 6` with the Triton DSA prefill and decode backends. The TP4/EP4 InferenceX benchmark adds HiCache DRAM offload, but those host-specific knobs are intentionally omitted from the portable cookbook command. MTP remains unvalidated for GLM-5.2 on MI300X/MI325X and for the other MI355X checkpoint precisions. +- **MTP / EAGLE speculative decoding on AMD.** Five-step MTP is validated for `amd/GLM-5.2-MXFP4` on MI355X with `lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260920`. Choose **Low-Latency** for TP8/EP1, or **High-Throughput** for TP4/EP4 with HiCache (`--enable-hierarchical-cache --hicache-ratio 1.0`). HiCache uses the default `kernel`, `page_first`, and `write_through` settings. MTP remains unvalidated for GLM-5.2 on MI300X/MI325X and for the other MI355X checkpoint precisions. ## 3. Advanced Usage diff --git a/docs/cookbook/autoregressive/GLM/GLM-5.3-Flash.mdx b/docs/cookbook/autoregressive/GLM/GLM-5.3-Flash.mdx index 02196da2b..abd56dc27 100644 --- a/docs/cookbook/autoregressive/GLM/GLM-5.3-Flash.mdx +++ b/docs/cookbook/autoregressive/GLM/GLM-5.3-Flash.mdx @@ -10,10 +10,10 @@ tag: NEW -Use an SGLang build that includes GLM-5.3-Flash support. +Use an SGLang build that includes GLM-5.3-Flash support (v0.5.20 or later). ```bash Command -docker pull lmsysorg/sglang:glm-5.3-flash +docker pull lmsysorg/sglang:latest ``` The deployment panel can render a complete `docker run` command for the selected hardware and options. See [Install SGLang with Docker](/docs/get-started/install#method-3-using-docker) for host setup. @@ -104,7 +104,7 @@ The **Speculative** card in the Playground changes the algorithm without leaving - **EAGLE / MTP 5-1-6** is exactly what Low Latency serves, so a Low Latency base starts on this chip. Pick it from a High Throughput base to keep that recipe's other settings and add the MTP head. - **Off (greedy)** strips the whole `--speculative-*` family, which is what High Throughput already starts from. -- **DFlash2** swaps the in-checkpoint MTP head for the trained block-diffusion draft in [`incoai/GLM-5.3-Flash-DFlash2`](https://huggingface.co/incoai/GLM-5.3-Flash-DFlash2). The draft proposes a whole block per step and the target verifies it in one forward pass, so output quality stays the target's. Its block size comes from the draft checkpoint, and the draft runs on `fa4` rather than the target's DSA backends. It needs a build that carries the GLM-5.3-Flash hidden-state capture from [PR #36708](https://github.com/sgl-project/sglang/pull/36708), which is merged into the [PR #36507](https://github.com/sgl-project/sglang/pull/36507) support branch (`xinyuan/glm-5.3-flash-support`) rather than into `main`, so the image pinned above is not enough on its own — pull that branch at its current head, or add #36708's commit on top of an older checkout. The draft repository is also access-gated: request access on its model page, then download it alongside the target before serving. This combination is not yet measured on the cookbook hardware, so treat it as a starting point. +- **DFlash2** swaps the in-checkpoint MTP head for the trained block-diffusion draft in [`incoai/GLM-5.3-Flash-DFlash2`](https://huggingface.co/incoai/GLM-5.3-Flash-DFlash2). The draft proposes a whole block per step and the target verifies it in one forward pass, so output quality stays the target's. Its block size comes from the draft checkpoint, and the draft runs on `fa4` rather than the target's DSA backends. The hidden-state capture it needs ([PR #36708](https://github.com/sgl-project/sglang/pull/36708)) shipped with the GLM-5.3-Flash support in v0.5.20, so the image pinned above is enough. The draft repository is access-gated: request access on its model page, then download it alongside the target before serving. This combination is not yet measured on the cookbook hardware, so treat it as a starting point. Neither algorithm runs with DP-Attention; the card disables the affected chips and names the reason. @@ -134,13 +134,13 @@ The default multimodal feature transport is automatic, and on a single CUDA node ### 3.1 Reasoning -Thinking is enabled by the checkpoint's generation configuration, and generated commands enable `--reasoning-parser glm45` by default. The OpenAI-compatible API then places thinking in `message.reasoning_content` and the final answer in `message.content`. You can disable **Reasoning Parser** in the Playground when an integration needs the raw response format. +Thinking is enabled by the checkpoint's generation configuration, and generated commands enable `--reasoning-parser auto` (which resolves to `glm45` for GLM-5.3-Flash) by default. The OpenAI-compatible API then places thinking in `message.reasoning_content` and the final answer in `message.content`. You can disable **Reasoning Parser** in the Playground when an integration needs the raw response format. To disable thinking for a request, pass `chat_template_kwargs: {"thinking": false}` in the request body. ### 3.2 Tool calling -Generated commands enable `--tool-call-parser glm47` by default, so structured calls are returned in `message.tool_calls`. You can disable **Tool Call Parser** in the Playground when tool calling is not needed. On follow-up turns, read both `reasoning_content` and `content` because a thinking model can use either field around tool execution. +Generated commands enable `--tool-call-parser auto` (which resolves to `glm47` for GLM-5.3-Flash) by default, so structured calls are returned in `message.tool_calls`. You can disable **Tool Call Parser** in the Playground when tool calling is not needed. On follow-up turns, read both `reasoning_content` and `content` because a thinking model can use either field around tool execution. ### 3.3 Multimodal serving diff --git a/docs/cookbook/autoregressive/GLM/GLM-5.3.mdx b/docs/cookbook/autoregressive/GLM/GLM-5.3.mdx index 8b0f63027..bd6b6760b 100644 --- a/docs/cookbook/autoregressive/GLM/GLM-5.3.mdx +++ b/docs/cookbook/autoregressive/GLM/GLM-5.3.mdx @@ -103,7 +103,7 @@ import { Playground } from "/src/snippets/_playground.jsx"; - **DeepSeek Sparse Attention (DSA).** GLM-5.3 uses the `glm_moe_dsa` architecture; SGLang auto-selects the DSA attention backends (`flashmla_sparse` prefill, `fa3` decode, `sgl-kernel` indexer topk). No attention-backend flag is needed on the supported hardware. SGLang also auto-selects the KV-cache dtype for DSA models — `fp8_e4m3` on Blackwell (B200/GB300/B300, which then routes DSA through the TensorRT-LLM backend) and `bf16` on Hopper (H200) — so no `--kv-cache-dtype` flag is required. On Hopper, pairing `--kv-cache-dtype fp8_e4m3` with `--dsa-prefill-backend flashmla_sparse_q8 --dsa-decode-backend flashmla_kv` selects the native FP8 sparse prefill kernel (computes directly on the fp8 KV cache with no fp8→bf16 dequantization round-trip; GLM-5.3's 64 query heads match the kernel's native tile) — see the [DeepSeek-V3.2 page](../DeepSeek/DeepSeek-V3_2) for kernel details; the optional `SGLANG_ENABLE_DSA_Q8KV8_*` performance env vars are documented in `python/sglang/srt/environ.py`. - **MTP / speculative decoding.** The checkpoint ships one nextn layer. Enable EAGLE MTP for lower latency (`--speculative-algorithm EAGLE --speculative-num-steps 5 --speculative-eagle-topk 1 --speculative-num-draft-tokens 6` for low-latency; `1-1-2` for balanced). The config's `index_share_for_mtp_iteration` reuses the DSA indexer's topk across draft steps (effective only at `--speculative-eagle-topk 1`). Watch the server's reported **accept length** and adjust `--speculative-num-steps` / `--speculative-num-draft-tokens`: lower the draft length when rejected draft tokens create excess verification work. -- **DFlash2 (block-diffusion draft).** The **Speculative** card in the [Playground above](#playground) also offers **DFlash2**, which replaces the in-checkpoint MTP layer with the separately trained block-diffusion drafter [`incoai/GLM-5.3-DFlash2`](https://huggingface.co/incoai/GLM-5.3-DFlash2). It proposes a whole block per step and the target verifies the block in one forward pass, so output quality stays the target's. The block size — 8, i.e. 7 draft tokens per verification step — comes from the draft checkpoint's own `dflash_config`, so no `--speculative-num-draft-tokens` is passed; the draft is a small dense model and runs on `fa4` instead of the target's DSA backends. Two prerequisites: the DFlash2 drafter ([PR #35371](https://github.com/sgl-project/sglang/pull/35371)) merged **after v0.5.18**, so install SGLang from `main` (or use a nightly image) rather than the release this page pins; and DFLASH runs on **CUDA/NPU only** and rejects **DP-Attention**, so turn DP-Attention off in the **Attention** card before selecting it on a high-throughput base. The draft repository is public but licensed CC BY-NC-ND 4.0 for research and evaluation. +- **DFlash2 (block-diffusion draft).** The **Speculative** card in the [Playground above](#playground) also offers **DFlash2**, which replaces the in-checkpoint MTP layer with the separately trained block-diffusion drafter [`incoai/GLM-5.3-DFlash2`](https://huggingface.co/incoai/GLM-5.3-DFlash2). It proposes a whole block per step and the target verifies the block in one forward pass, so output quality stays the target's. The block size — 8, i.e. 7 draft tokens per verification step — comes from the draft checkpoint's own `dflash_config`, so no `--speculative-num-draft-tokens` is passed; the draft is a small dense model and runs on `fa4` instead of the target's DSA backends. Note that DFLASH runs on **CUDA/NPU only** and rejects **DP-Attention**, so turn DP-Attention off in the **Attention** card before selecting it on a high-throughput base. The draft repository is public but licensed CC BY-NC-ND 4.0 for research and evaluation. - **Memory.** The FP8 weights are large (MoE total, not active params). Start around `--mem-fraction-static 0.8` on H200 (TP8) and tune up; raise it for the 4-GPU GB300 single-node layout (TP4). - **DP-Attention + DeepEP** for the balanced/high-throughput strategies spreads attention across data-parallel ranks and routes MoE through DeepEP. - **BF16 weights need more GPUs.** The full-precision build (`zai-org/GLM-5.3-BF16`, ~1.5 TB) does not fit a single 8×H200 / 8×B200 / 4×GB300 node. It fits single-node on **8×B300** (TP8, ~2.1 TB HBM); on the smaller GPUs it needs a **multi-node** layout (e.g. 2×8×H200 or 2×8×B200 at TP16, 2×4×GB300 at TP8). FP8 is the recommended deployment. Use the same DSA / MTP / chunked-prefill guidance as FP8. @@ -117,7 +117,7 @@ import { Playground } from "/src/snippets/_playground.jsx"; ### 3.1 Reasoning -GLM-5.3 is a reasoning model. Enable the `glm45` reasoning parser (toggle **Reasoning Parser** in the **Parsers** card of the [Playground above](#playground)) to separate thinking from the final answer — thinking lands in `message.reasoning_content`, the answer in `message.content`. The chat template defaults `clear_thinking` to `false`; for multi-turn chat, pass `chat_template_kwargs: {"clear_thinking": True}` so previous reasoning is cleared before the next response. +GLM-5.3 is a reasoning model, and generated commands enable `--reasoning-parser auto` (which resolves to `glm45` for GLM-5.3) by default so thinking is separated from the final answer — thinking lands in `message.reasoning_content`, the answer in `message.content`. Without the parser the server returns the thinking and the answer as one `content` string with a stray `` between them, because the chat template opens `` in the generation prompt. You can disable **Reasoning Parser** in the **Parsers** card of the [Playground above](#playground) when an integration needs that raw format. The chat template defaults `clear_thinking` to `false`; for multi-turn chat, pass `chat_template_kwargs: {"clear_thinking": True}` so previous reasoning is cleared before the next response. **Reasoning effort.** Pass `chat_template_kwargs: {"reasoning_effort": ...}` to select `low`, `high`, or `max`. If you omit it or pass another value, the template uses `max`. @@ -164,7 +164,7 @@ Here is how you can calculate it: ### 3.2 Tool Calling -Enable the `glm47` tool-call parser (toggle **Tool Call Parser** in the **Parsers** card of the [Playground above](#playground)) to surface structured tool calls via `message.tool_calls`. GLM-5.3 emits the newer `………` format, so it needs the **`glm47`** parser — the older `glm45` parser does not parse it (the call would be left as raw text in `content`). On thinking mode the turn also fills `reasoning_content`, so print both fields. +Generated commands enable `--tool-call-parser auto` by default, so structured calls are returned in `message.tool_calls` with `finish_reason: "tool_calls"`. `auto` resolves to **`glm47`** for GLM-5.3: the model emits the newer `………` format, which the older `glm45` parser does not parse (the call would be left as raw text in `content`). Running with no tool-call parser fails the same way, and `finish_reason` stays `"stop"`, so an agent loop never sees the call. You can disable **Tool Call Parser** in the **Parsers** card of the [Playground above](#playground) when tool calling is not needed. On thinking mode the turn also fills `reasoning_content`, so print both fields. @@ -218,7 +218,7 @@ For long-context, prefix-heavy workloads, enable hierarchical KV caching to spil ### 3.4 Claude Code Integration -GLM-5.3's strong reasoning + tool-calling makes it a good backend for [Claude Code](https://code.claude.com/docs/en/overview), Anthropic's agentic CLI. SGLang exposes the Anthropic-compatible `/v1/messages` endpoint on every server, so Claude Code can talk to a GLM-5.3 server with only environment variables — no code change. Launch the server with `--reasoning-parser glm45 --tool-call-parser glm47` (any recipe from the Deployment panel above works), then: +GLM-5.3's strong reasoning + tool-calling makes it a good backend for [Claude Code](https://code.claude.com/docs/en/overview), Anthropic's agentic CLI. SGLang exposes the Anthropic-compatible `/v1/messages` endpoint on every server, so Claude Code can talk to a GLM-5.3 server with only environment variables — no code change. Launch the server with `--reasoning-parser auto --tool-call-parser auto` (any recipe from the Deployment panel above works), then: ```bash Command export ANTHROPIC_BASE_URL="http://127.0.0.1:30000" diff --git a/docs/cookbook/autoregressive/Google/Gemma4.mdx b/docs/cookbook/autoregressive/Google/Gemma4.mdx index a1e4ef921..81f549969 100644 --- a/docs/cookbook/autoregressive/Google/Gemma4.mdx +++ b/docs/cookbook/autoregressive/Google/Gemma4.mdx @@ -97,6 +97,8 @@ For other installation methods, please refer to the [official SGLang installatio ### 3.1 Basic Configuration +The Gemma 4 series offers models in various sizes and architectures, optimized for different hardware platforms including NVIDIA GPUs, AMD GPUs, and Intel Arc Pro B-Series GPUs(codename: BMG (Battlemage)). The recommended launch configurations vary by hardware and model size. + **Interactive Command Generator**: Use the configuration selector below to automatically generate the appropriate deployment command for your hardware platform and model variant. diff --git a/docs/cookbook/autoregressive/Meta/Llama3.1.mdx b/docs/cookbook/autoregressive/Meta/Llama3.1.mdx index 8827abf67..2bccc224e 100644 --- a/docs/cookbook/autoregressive/Meta/Llama3.1.mdx +++ b/docs/cookbook/autoregressive/Meta/Llama3.1.mdx @@ -22,11 +22,9 @@ SGLang offers multiple installation methods. You can choose the most suitable in Please refer to the [official SGLang installation guide](../../../docs/get-started/install) for installation instructions. -For SGLang CPU installation, please refer to the [CPU version installation guide](../../../docs/hardware-platforms/cpu_server#installation). - ## 3. Model Deployment -This section provides deployment configurations optimized for different hardware platforms and use cases. +This section provides deployment configurations optimized for different hardware platforms including NVIDIA GPUs, AMD GPUs, Intel Arc Pro B-Series GPUs(codename: BMG (Battlemage)), and Intel Xeon CPUs. ### 3.1 Basic Configuration diff --git a/docs/cookbook/autoregressive/Meta/Llama3.3-70B.mdx b/docs/cookbook/autoregressive/Meta/Llama3.3-70B.mdx index 549ca1a8d..ebceaa55f 100644 --- a/docs/cookbook/autoregressive/Meta/Llama3.3-70B.mdx +++ b/docs/cookbook/autoregressive/Meta/Llama3.3-70B.mdx @@ -24,15 +24,13 @@ For more details, please refer to the [official Llama models repository](https:/ Please refer to the [official SGLang installation guide](../../../docs/get-started/install) for installation instructions. -For SGLang CPU installation, please refer to the [CPU version installation guide](../../../docs/hardware-platforms/cpu_server#installation). - ## 3. Model Deployment -This section provides deployment configurations optimized for AMD GPUs (MI300X, MI325X, MI355X) and Intel Xeon CPUs. +This section provides deployment configurations optimized for AMD GPUs (MI300X, MI325X, MI355X), Intel Arc Pro B-Series GPUs(codename: BMG (Battlemage)) and Intel Xeon CPUs. ### 3.1 Interactive Configuration -**Interactive Command Generator**: Use the configuration selector below to automatically generate the appropriate deployment command for your AMD GPU setup. +**Interactive Command Generator**: Use the configuration selector below to automatically generate the appropriate deployment command for your AMD GPU, Intel Arc Pro B-Series GPUs or Intel Xeon CPUs setup. import { Llama33Deployment } from "/src/snippets/autoregressive/llama33-70b-deployment.jsx"; diff --git a/docs/cookbook/autoregressive/MiniMax/MiniMax-M3.mdx b/docs/cookbook/autoregressive/MiniMax/MiniMax-M3.mdx index d24cb14c1..bfdb5af6c 100644 --- a/docs/cookbook/autoregressive/MiniMax/MiniMax-M3.mdx +++ b/docs/cookbook/autoregressive/MiniMax/MiniMax-M3.mdx @@ -129,6 +129,7 @@ The NVIDIA Blackwell recipes are validated single-node: **B200 at `--tp 8`** and - **Memory**: `--mem-fraction-static` reserves GPU memory for weights + KV pool; the rest is prefill **activation headroom**. The value scales with *free* memory per GPU (card capacity minus per-GPU weight), so it tracks the card more than the TP degree: **`0.65` on B200** (180 GB — less headroom once weights are resident) and **`0.75` on the larger-memory B300 / GB300** (`0.80` on AMD). Lower TP packs more weight per GPU, so a tighter config needs a *lower* value — B200 needs `0.65` even at `--tp 4`. Raising it past the validated value is fine only for low-concurrency single-stream serving; it OOMs under high concurrency or long context. - **Long context (32K+)**: keep `--mem-fraction-static` at the platform default and raise `--chunked-prefill-size` to `16384`. Decode TPOT stays roughly flat in context length thanks to sparse attention; 1K–128K prompts are validated. +- **HiSparse for decode capacity**: on NVIDIA CUDA, HiSparse keeps the three dense layers on GPU, moves the 57 sparse-layer K/V caches to pinned host memory, and feeds selected block IDs directly to the swap-in kernel. For the released four-KV-head model, use `--tp 4` or greater, `--disable-radix-cache`, and `device_buffer_size >= 2048`. Enable it with `--enable-hisparse --hisparse-config='{"device_buffer_size":4096,"host_to_device_ratio":2}'` on the Triton launch command. - **Scaling TP**: B200 is documented at `--tp 8`; B300 / GB200 / GB300 at `--tp 4` (the single-node cross-family common denominator). On an 8-GPU B300 host you can also raise to `--tp 8` for more throughput / KV headroom. - **Expert parallelism**: to trade latency for throughput add `--ep` (see [Expert Parallelism Deployment](../../../docs/advanced_features/expert_parallelism)). On AMD, set `--ep` equal to `--tp`. Shared-experts fusion is automatically disabled when EP > 1; on AMD standard EP the server also disables `--enable-aiter-allreduce-fusion` automatically to preserve accuracy. - `--trust-remote-code` is required to load the MiniMax config / processor classes. diff --git a/docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx b/docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx index e316b9b01..6025fa67f 100644 --- a/docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx +++ b/docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx @@ -30,7 +30,7 @@ Then run the **Python** output of the command panel below in that environment. ```bash Command docker pull lmsysorg/sglang:latest # NVIDIA (CUDA) -docker pull lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260910 # AMD MI350X / MI355X (ROCm) +docker pull lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260916 # AMD MI350X / MI355X (ROCm) ``` For how to launch the image, see [Install → Method 3: Using Docker](../../../docs/get-started/install#method-3-using-docker). Substitute the inner `sglang serve ...` with what the command generator below produces. @@ -40,14 +40,16 @@ For how to launch the image, see [Install → Method 3: Using Docker](../../../d ```bash Command -docker pull quay.io/ascend/sglang:main-cann9.0.0-a3 +docker pull quay.io/ascend/sglang:main-cann9.0.0-a3 # Ascend A3 Series +docker pull swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:cann9.1.0-950-B070 # Ascend 950PR/DT Series ``` For host and platform setup, see the [NPU installation guide](../../../docs/hardware-platforms/ascend-npus/getting-started/installation) and the [quick start guide](../../../docs/hardware-platforms/ascend-npus/getting-started/quick_start). -**Weights (NPU):** [Kimi-K3-W4A8](https://www.modelscope.cn/models/sgl-npu/Kimi-K3-W4A8) (W4A8, 1.49 TB) · +**Weights (NPU):** [Kimi-K3](https://modelscope.cn/models/moonshotai/Kimi-K3) (950PR/DT Series, MXFP4) · +[Kimi-K3-W4A8](https://www.modelscope.cn/models/sgl-npu/Kimi-K3-W4A8) (A3 Series, W4A8, 1.49 TB) · [Kimi-K3-DSpark](https://www.modelscope.cn/models/RadixArk/Kimi-K3-DSpark) (DSPARK draft, 4.5 GB) @@ -56,14 +58,14 @@ For host and platform setup, see the -Pick your hardware, then the deployment shape and operating point. Node count follows the hardware recipe (B200 2×8, GB200 4×4, H100 4×8, B300 1×8, H200 2×8 — 4×8 on Unified High-Throughput, GB300 2×4, MI350X/MI355X 1×8, Ascend A3 Series 4×8 — 32 cards / 64 dies), so it is not a separate choice. If you serve the NVFP4 checkpoint (`nvidia/Kimi-K3-NVFP4`, the **Quantization** row in the panel below), use the `lmsysorg/sglang:dev-dev-kimi-k3-nvfp4` image. +Pick your hardware, then the deployment shape and operating point. Node count follows the hardware recipe (B200 2×8, GB200 4×4, H100 4×8, B300 1×8, H200 2×8 — 4×8 on Unified High-Throughput, GB300 2×4, MI350X/MI355X 1×8, Ascend A3 Series 4×8 — 32 cards / 64 dies, Ascend 950PR/DT Series 4×8), so it is not a separate choice. If you serve the NVFP4 checkpoint (`nvidia/Kimi-K3-NVFP4`, the **Quantization** row in the panel below), use the `lmsysorg/sglang:dev-dev-kimi-k3-nvfp4` image. **PD Mode** — `Unified` serves prefill and decode together. `Prefill` / `Decode` split them into dedicated pools (see [PD disaggregation](#3-4-pd-disaggregation)); `Prefill` ships two strategies, both chunked at 16k. On the 8-GPU platforms (B300 1×8, GB300 2×4), `Default` is TP8 and `Long-Context` is `--pp-size 8 --tp-size 1`. On the 16-GPU platforms (B200 2×8, GB200 4×4), both are `--pp-size 16 --tp-size 1` and differ only in `--mem-fraction-static` (0.85 vs 0.90) — deep PP is the throughput shape there, not just the long-context one (see [Deep PP](#deep-pp-for-prefill)). **Strategy** — the operating point within that shape: - **Low-Latency** — no DCP, so the MLA KV stays TP-replicated. For chat. B200 splits its two nodes into PP2 × TP8; every other platform is flat TP. -- **Balanced** — the accuracy-preserving default: PP2 × DCPEP8 on B200 (the two pipeline stages and DCP8 split KV and KDA state), TP16/DCP16 on GB200, TP8/DCP8 on B300/GB300, TP8 ROCm/AITER on MI35x. +- **Balanced** — the accuracy-preserving default: PP2 × DCPEP8 on B200 (the two pipeline stages and DCP8 split KV and KDA state), TP16/DCP16 on GB200, TP8/DCP8 on B300/GB300, TP8/DCP8 ROCm/AITER on MI35x. - **High-Throughput** — the large-scale lane: pick a **Cluster Size** and **Large-Scale Preset** in the Playground ([details](#large-scale-presets)). The cell itself is Balanced, except on H100 (plus `extra_buffer_lazy`) and H200 (widens to 4×8 TP32/EP32 at `--mem-fraction-static 0.90`). `Long-Context` appears only under the `Prefill` PD mode; for long-context unified serving on B200, start from High-Throughput and raise `--context-length`. @@ -71,7 +73,7 @@ Pick your hardware, then the deployment shape and operating point. Node count fo **Spec Decode** — layers onto the strategy without changing it, on every platform except B200. DSPARK proposes 7 draft tokens per step (tune in the Playground) and requires `pp_size == 1`, so on B200 it also drops the pipeline and re-lays the same 16 GPUs flat: PP2 × TP8 → TP16, PP2 × DCPEP8 → DCPEP16. DFLASH has no published draft checkpoint. The win is largest on short interactive traffic and fades as the prompt grows. -`--mamba-full-memory-ratio` is the one sizing flag, computed live: set your average request length in the [Mamba ratio calculator](#mamba-ratio-calculator); everything else follows the panels, and the result is pinned into the command. (The Ascend A3 Series uses `--max-mamba-cache-size` instead.) +`--mamba-full-memory-ratio` is the one sizing flag, computed live: set your average request length in the [Mamba ratio calculator](#mamba-ratio-calculator); everything else follows the panels, and the result is pinned into the command. (The Ascend NPU recipes don't use the calculator and set no ratio at all — the Ascend path sizes both pools itself: the A3 Series pins `--max-mamba-cache-size` instead, and the A5 recipe serves with the radix cache off.) import { Deployment } from "/src/snippets/_deployment.jsx"; @@ -91,6 +93,22 @@ import { KimiK3MambaRatioCalculator } from "/src/snippets/_kimi_k3_mamba_ratio_c NVFP4 NOSPEC / NVFP4 DSPARK), which is why no point past concurrency 64 is published for Balanced. +### AMD AITER with DCP8 + +The MI350X/MI355X unified Balanced recipe uses TP8/DCP8 with AITER prefill and +decode attention. DCP shards the target MLA KV cache; RadixArk DSPARK's draft KV +remains replicated. The pinned `v0.5.19-rocm720-mi35x-20260916` image records +SGLang revision `e7f7447333`, which includes +[AITER DCP support (#34432)](https://github.com/sgl-project/sglang/pull/34432) and +the [DCP KV-free fix (#38941)](https://github.com/sgl-project/sglang/pull/38941). +No source overlay is required for DCP. + +Keep `SGLANG_K3_KDA_FUSED_BACKEND` unset with this image. The separate fused-KDA +opt-in requires the [deferred-gate fix (#39066)](https://github.com/sgl-project/sglang/pull/39066), +which is not included in this image. This updated recipe remains **Final +Verification In Progress**; the recorded speed numbers use their original +configurations and do not validate the new image or DCP8 recipe. + ### Mamba ratio calculator @@ -146,7 +164,7 @@ not been re-measured on any cell — re-measure before you rely on one. ## 2. Configuration Tips -**Memory: two pools, one flag.** K3 splits static memory into a worst-case-reserved **KDA state pool** (it sets the concurrency ceiling) and a paged **MLA KV pool**, divided by `--mamba-full-memory-ratio`. The command panel pins that flag to the [calculator](#mamba-ratio-calculator)'s output — set your average request length there; every other calculator input follows the panels. (On the Ascend A3 Series: `--max-mamba-cache-size`, no calculator.) After boot, read back `max_total_num_tokens` (the KV side) and the admitted-request cap (the state side). +**Memory: two pools, one flag.** K3 splits static memory into a worst-case-reserved **KDA state pool** (it sets the concurrency ceiling) and a paged **MLA KV pool**, divided by `--mamba-full-memory-ratio`. The command panel pins that flag to the [calculator](#mamba-ratio-calculator)'s output — set your average request length there; every other calculator input follows the panels. (On the Ascend NPU recipes there is no calculator and no ratio flag: the Ascend path sizes both pools itself — the A3 Series pins `--max-mamba-cache-size`, and the A5 recipe runs the radix cache off.) After boot, read back `max_total_num_tokens` (the KV side) and the admitted-request cap (the state side). Capacity levers, all in the Playground. Each trades precision or cache behavior for capacity — re-verify accuracy on your workload: @@ -178,10 +196,11 @@ Speculation: DSPARK holds block size + 1 (= 8) intermediate states per request | GB200 4×4 | TP16/DCP16 | MNNVL auto-detected | | H200 2×8 (4×8 on Unified High-Throughput) | TP16/EP16 + symm-mem, Marlin + FlashMLA; High-Throughput widens to TP32/EP32 over 4 nodes at mem-frac 0.90 with `extra_buffer_lazy` | same block on every node; export the cross-node NIC (`GLOO_SOCKET_IFNAME` / `NCCL_SOCKET_IFNAME`, `SGLANG_HOST_IP`); keep `NCCL_MNNVL_ENABLE=1 NCCL_CUMEM_ENABLE=1` | | H100 4×8 | TP32/EP32, Marlin + FlashMLA | SM90a build of the K3 image; pin NCCL/Gloo to the same NIC on all nodes; least post-weight headroom (80 GB) | -| MI350X/MI355X 1×8 | TP8 ROCm/AITER | AITER A8W4 FlyDSL MoE, Triton attention (`SGLANG_MLA_DECODE_TUNE=1` for gfx950 MLA decode geometry), graph bs up to 256, fp8 kvcache; DSPARK supported. Activation-quant and fused-KDA-decode knobs: [AMD ROCm/AITER environment](#amd-env) | +| MI350X/MI355X 1×8 | TP8/DCP8 ROCm/AITER (Unified Balanced) | AITER A8W4 FlyDSL MoE, AITER prefill/decode attention with sharded target MLA KV, graph bs up to 256, fp8 kvcache; DSPARK supported. Activation-quant and fused-KDA-decode knobs: [AMD ROCm/AITER environment](#amd-env) | | Ascend A3 Series 4×8 (32 cards / 64 dies) | TP64/DP4 + DeepEP | PD-mixed `Unified` only; DSPARK baked in; pin `GLOO`/`HCCL_SOCKET_IFNAME` on every node | +| Ascend 950PR/DT Series 4×8 | TP32/dp1 + DeepEP | PD-mixed `Unified` only; DSPARK baked in; shared experts / dense MLP shard over attention-TP (`--shared-experts-tp-size 4`); radix cache off; pin `GLOO`/`HCCL_SOCKET_IFNAME` on every node | -**DCP notes** — the DCP cells are Balanced and High-Throughput on every Blackwell platform, in both the `Unified` and `Decode` roles: +**Blackwell DCP notes** — the DCP cells are Balanced and High-Throughput on every Blackwell platform, in both the `Unified` and `Decode` roles: - DCP is the only axis that shards the TP-replicated MLA KV; Low-Latency skips it. - Leave `--dcp-comm-backend` unset (fabric-resolved: `fi_a2a` on GB200/GB300, `a2a` on B200/B300). @@ -239,7 +258,7 @@ Pending update... ### 3.2 Tool Calling -Enable the `kimi_k3` tool-call parser (toggle **Tool Call Parser** in the **Parsers** card of the [Playground above](#playground)) to surface structured tool calls via `message.tool_calls`. Because K3 is a thinking model, the follow-up turn may put text in `reasoning_content` as well as `content` — print both. (Not yet supported on the Ascend A3 Series.) +Enable the `kimi_k3` tool-call parser (toggle **Tool Call Parser** in the **Parsers** card of the [Playground above](#playground)) to surface structured tool calls via `message.tool_calls`. Because K3 is a thinking model, the follow-up turn may put text in `reasoning_content` as well as `content` — print both. (Not yet supported on the Ascend NPU recipes — A3 and A5.) diff --git a/docs/cookbook/autoregressive/NVIDIA/Nemotron3-Nano.mdx b/docs/cookbook/autoregressive/NVIDIA/Nemotron3-Nano.mdx index ee60b6c92..7ec09ef40 100644 --- a/docs/cookbook/autoregressive/NVIDIA/Nemotron3-Nano.mdx +++ b/docs/cookbook/autoregressive/NVIDIA/Nemotron3-Nano.mdx @@ -31,6 +31,8 @@ This section provides a progressive guide from quick deployment to performance t ### 3.1 Basic Configuration +The Nemotron3-Nano series offers models in various sizes and architectures, optimized for different hardware platforms including NVIDIA GPUs and Intel Arc Pro B-Series GPUs(codename: BMG (Battlemage)). + **Interactive Command Generator**: select hardware, model variant, and common knobs to generate a launch command. diff --git a/docs/cookbook/autoregressive/Qwen/Qwen3.5.mdx b/docs/cookbook/autoregressive/Qwen/Qwen3.5.mdx index f61d0513d..8750c7d1e 100644 --- a/docs/cookbook/autoregressive/Qwen/Qwen3.5.mdx +++ b/docs/cookbook/autoregressive/Qwen/Qwen3.5.mdx @@ -111,11 +111,9 @@ docker pull lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260911 For the full Docker setup and other installation methods, please refer to the [official SGLang installation guide](../../../docs/get-started/install). -For SGLang CPU installation, please refer to the [CPU version installation guide](../../../docs/hardware-platforms/cpu_server#installation). - ## 3. Model Deployment -This section provides deployment configurations optimized for different hardware platforms and use cases. +This section provides deployment configurations optimized for different hardware platforms including NVIDIA GPUs, AMD GPUs, Intel Arc Pro B-Series GPUs(codename: BMG (Battlemage)), and Intel Xeon CPUs. ### 3.1 Basic Configuration diff --git a/docs/cookbook/autoregressive/Qwen/Qwen3.mdx b/docs/cookbook/autoregressive/Qwen/Qwen3.mdx index f58397e7d..4e5d64e5e 100644 --- a/docs/cookbook/autoregressive/Qwen/Qwen3.mdx +++ b/docs/cookbook/autoregressive/Qwen/Qwen3.mdx @@ -26,15 +26,13 @@ SGLang offers multiple installation methods. You can choose the most suitable in Please refer to the [official SGLang installation guide](../../../docs/get-started/install) for installation instructions. -For SGLang CPU installation, please refer to the [CPU version installation guide](../../../docs/hardware-platforms/cpu_server#installation). - ## 3. Model Deployment This section provides deployment configurations optimized for different hardware platforms and use cases. ### 3.1 Basic Configuration -The Qwen3 series offers models in various sizes and architectures, optimized for different hardware platforms including NVIDIA GPUs, AMD GPUs, and Intel Xeon CPUs. The recommended launch configurations vary by hardware and model size. +The Qwen3 series offers models in various sizes and architectures, optimized for different hardware platforms including NVIDIA GPUs, AMD GPUs, Intel Arc Pro B-Series GPUs(codename: BMG (Battlemage)), and Intel Xeon CPUs. The recommended launch configurations vary by hardware and model size. **Interactive Command Generator**: Use the configuration selector below to automatically generate the appropriate deployment command for your hardware platform, model size, quantization method, and thinking capabilities. diff --git a/docs/cookbook/autoregressive/Xiaomi/MiMo-V2.5.mdx b/docs/cookbook/autoregressive/Xiaomi/MiMo-V2.5.mdx index 70e5799fc..22f34259d 100644 --- a/docs/cookbook/autoregressive/Xiaomi/MiMo-V2.5.mdx +++ b/docs/cookbook/autoregressive/Xiaomi/MiMo-V2.5.mdx @@ -2,7 +2,6 @@ title: MiMo-V2.5 metatags: description: "Deploy XiaomiMiMo MiMo-V2.5-Pro (1.02T MoE, text) and MiMo-V2.5 (310B MoE, multimodal) with SGLang — EAGLE speculative decoding, hybrid attention, and 1M-token context." -tag: NEW --- ## 1. Model Introduction diff --git a/docs/cookbook/autoregressive/Xiaomi/MiMo-V2.6.mdx b/docs/cookbook/autoregressive/Xiaomi/MiMo-V2.6.mdx new file mode 100644 index 000000000..42709d497 --- /dev/null +++ b/docs/cookbook/autoregressive/Xiaomi/MiMo-V2.6.mdx @@ -0,0 +1,124 @@ +--- +title: MiMo-V2.6 +description: "Deploy MiMo-V2.6-Flash and Pro with SGLang on NVIDIA B300 GPUs, using MXFP4 MoE weights, BF16 routing, DFlash decoding, and a 1M-token context window." +tag: NEW +--- + +## Deployment + +The recipes below target **MiMo-V2.6-Flash on 4× B300** and **MiMo-V2.6-Pro on 8× B300**, each on a single node. Both use MXFP4 MoE weights, a BF16 MoE router, and DFlash speculative decoding. B300 validation was reported by the model team; the launch settings come from [SGLang PR #40448](https://github.com/sgl-project/sglang/pull/40448). + + + + + +BF16 MoE routing and MXFP4 expert loading for MiMo V2.6 landed in [SGLang PR #40448](https://github.com/sgl-project/sglang/pull/40448) and are on `main`, so a recent nightly already carries them. Prepare an NVIDIA CUDA environment with FlashAttention 4, DeepGEMM, and DeepEP available. See the [installation guide](/docs/get-started/install) for platform prerequisites. + + + + +Install from source. The commit below is the revision these recipes were captured at; build from `main` instead if you want later fixes. + +```bash Command +git clone https://github.com/sgl-project/sglang.git sglang-mimo-v2.6 +cd sglang-mimo-v2.6 +git checkout 983e643854f15cf9ef4370a49dfd74b6af54c3e3 +python3 -m pip install -e ./python +``` + +Then run the **Python** command from the panel below. Checkpoint paths refer to directories on this host. + + + + +Pull the official nightly image, which already includes the MiMo-V2.6 support: + +```bash Command +docker pull lmsysorg/sglang:dev +``` + + + + + + +Select **Flash** or **Pro** to generate its command. Each variant has one base recipe; **Balanced** identifies that recipe and does not imply a measured throughput or latency optimum. The `/model/...` checkpoint and DFlash paths in the generated command are editable defaults, not download locations — set them to your own paths under **⚙ Env** before launching. Use the panel's cURL example to send a request to port **30000** after the server is ready. + +import { Deployment } from "/src/snippets/_deployment.jsx"; +import { config } from "/src/snippets/configs/XiaomiMiMo/mimo-v2.6.jsx"; + + + +## Playground + +Experiment with reasoning and tool-call parsers, or disable speculative decoding for comparison. These overrides inherit the selected Flash or Pro recipe; they require separate validation. Parallelism and attention backends remain at the PR settings. + +import { Playground } from "/src/snippets/_playground.jsx"; + + + +## 1. Model introduction + +**MiMo-V2.6-Flash** is an open-source foundation model developed by Xiaomi. Its Mixture-of-Experts architecture has **309B total parameters and 15B activated per token**, with hybrid attention for computational efficiency. It supports a **1M-token context window** and **native multimodal capabilities**, and is designed for coding, visual understanding, general-purpose assistance, and research in agentic workflows, including complex tasks that require many steps. + +**MiMo-V2.6-Pro** is Xiaomi's flagship foundation model, with **1.02T total parameters and 42B activated per token**, designed for demanding workloads. It also supports a **1M-token context window** and **native multimodal capabilities**, with an emphasis on coding, visual understanding, general-purpose assistance, research, and long-horizon agentic tasks. + + + + + + + + + + + + + + + + + + + + + + + + +
VariantTotal parametersContext windowB300 recipe
MiMo-V2.6-Flash309B (15B active)1M tokens4 GPUs, TP=EP=4
MiMo-V2.6-Pro1.02T (42B active)1M tokens8 GPUs, TP=EP=8
+ +**License:** MIT, continuing the MiMo-V2.5 licensing as confirmed by the model team. + +**Architecture references:** The [MiMo-V2.6-Flash-RL](https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL) and [MiMo-V2.6-Pro-RL](https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Pro-RL) model cards document this generation directly — the hybrid sliding-window/global attention backbone, the omnimodal encoders, and the published evaluation scores — and each repository also ships the MiMo-V2.6 technical report. Both checkpoints declare `store_dtype: mxfp4` alongside `moe_router_dtype: bfloat16` in `config.json`, which is the expert-only MXFP4 quantization and BF16 routing these recipes depend on, and both carry the paired DFlash drafter in a `dflash/` subdirectory. + +## 2. Configuration tips + +### Parallelism and precision + +- **Flash:** `--tp 4 --ep 4`; **Pro:** `--tp 8 --ep 8`. Both keep `--dp 1 --pp-size 1`. TP and EP use the same GPU ranks; do not multiply them to determine the GPU count. +- **MoE compute and communication:** Keep `--moe-runner-backend deep_gemm --moe-a2a-backend deepep --deepep-mode auto`, together with `--moe-dense-tp-size 1` and `--enable-dp-lm-head`, as in the PR. +- **Precision:** MXFP4 describes the MoE expert weights, not the entire model. BF16 router weights are selected by the checkpoint's `moe_router_dtype`; the support commit computes router output logits in FP32. The PR relies on checkpoint quantization metadata and does not add a `--quantization` override. +- **Attention:** Keep `--attention-backend fa4` and `--mm-attention-backend fa4` on B300. The recipe also enables `--mm-enable-dp-encoder` for multimodal encoding. + +### Context length and memory + +Both commands set `--context-length 1048576`. This is the configured per-request context limit; it does not establish that 64 simultaneous 1M-token requests fit in memory. + +Flash uses `--mem-fraction-static 0.6 --swa-full-tokens-ratio 0.03`; Pro uses `--mem-fraction-static 0.7 --swa-full-tokens-ratio 0.08`. The memory fraction budgets weights and KV cache, while the SWA ratio controls sliding-window versus full-attention KV allocation. Preserve these model-specific values for the base recipe. + +The shared prefill settings are `--chunked-prefill-size 49152 --max-prefill-tokens 65536`. Both commands cap running requests and decode CUDA-graph batch size at **64**, disable prefill CUDA graphs, and use page size **1**. `--enable-cache-report` and `--log-level-http warning` retain the PR's reporting settings. + +### DFlash and caching + +The base recipe uses `--speculative-algorithm DFLASH --speculative-num-draft-tokens 8`. Supply the DFlash checkpoint paired with the selected target model; the draft path is independently editable and does not have to be a subdirectory of the target checkpoint. + +At the pinned commit, DFlash on NVIDIA GPUs requires PP=1 and rejects DP-attention. Do not add `--enable-dp-attention` or copy EAGLE-specific settings from the V2.5 cookbook. + +## 3. Reasoning and tool calling + +Both launch commands enable `--reasoning-parser mimo --tool-call-parser mimo`. The MiMo reasoning parser uses the request's `chat_template_kwargs.enable_thinking` setting: set it to `true` for reasoning or `false` for a direct answer. The command panel's cURL example enables reasoning explicitly. + +For OpenAI-compatible clients, read reasoning from `choices[0].message.reasoning_content`, the final answer from `choices[0].message.content`, and structured tool requests from `choices[0].message.tool_calls`. Preserve both reasoning and answer fields when inspecting responses. Use the checkpoint's generation defaults unless your application needs explicit sampling settings. + +Runtime throughput, latency, accuracy, and example responses for V2.6 will be added when measurements from these checkpoints are available. diff --git a/docs/cookbook/autoregressive/intro.mdx b/docs/cookbook/autoregressive/intro.mdx index e9eb9215b..17b8eb4a8 100644 --- a/docs/cookbook/autoregressive/intro.mdx +++ b/docs/cookbook/autoregressive/intro.mdx @@ -154,7 +154,7 @@ metatags: diff --git a/docs/cookbook/diffusion/Ernie-Image/Ernie-Image.mdx b/docs/cookbook/diffusion/Ernie-Image/Ernie-Image.mdx index a4c420a07..e8c666809 100644 --- a/docs/cookbook/diffusion/Ernie-Image/Ernie-Image.mdx +++ b/docs/cookbook/diffusion/Ernie-Image/Ernie-Image.mdx @@ -81,3 +81,9 @@ with open("ernie_image.png", "wb") as f: - `--performance-mode auto` keeps conservative defaults while preserving explicit user flags. - If the checkpoint includes a PE component, SGLang loads it automatically with the native Ministral3 runtime. Use `--layerwise-offload-components pe` when the local PE decoder needs to trade latency for lower GPU memory usage. - Treat FSDP, SP/Ulysses/Ring, and TP as explicit benchmark knobs. Measure the target resolution, step count, and GPU type before making them production defaults. + +## 6. Run in ComfyUI + +import { ComfyUISupport } from '/src/snippets/diffusion/comfyui-support.jsx'; + + diff --git a/docs/cookbook/diffusion/FLUX/FLUX.mdx b/docs/cookbook/diffusion/FLUX/FLUX.mdx index 3c14f419f..4cb84c7d8 100644 --- a/docs/cookbook/diffusion/FLUX/FLUX.mdx +++ b/docs/cookbook/diffusion/FLUX/FLUX.mdx @@ -34,7 +34,7 @@ This section provides deployment configurations optimized for different hardware FLUX models are optimized for high-quality image generation. The recommended launch configurations vary by hardware and model version. -**Interactive Command Generator**: Use the configuration selector below to automatically generate the appropriate deployment command for your hardware platform and model version. SGLang supports serving FLUX on NVIDIA B200, H200, H100, and AMD MI355X, MI325X, MI300X GPUs and Ascend A2/A3 Series NPUs. +**Interactive Command Generator**: Use the configuration selector below to automatically generate the appropriate deployment command for your hardware platform and model version. SGLang supports serving FLUX on NVIDIA B200, H200, H100, and AMD MI355X, MI325X, MI300X GPUs, Ascend A2, A3 NPUs Series NPUs and Intel Arc B-series graphics(codename: BMG (Battlemage)). @@ -407,3 +407,9 @@ Test Environment: ``` + +## 6. Run in ComfyUI + +import { ComfyUISupport } from '/src/snippets/diffusion/comfyui-support.jsx'; + + diff --git a/docs/cookbook/diffusion/Ideogram/Ideogram4.mdx b/docs/cookbook/diffusion/Ideogram/Ideogram4.mdx index c02530e6a..9e17b3b27 100644 --- a/docs/cookbook/diffusion/Ideogram/Ideogram4.mdx +++ b/docs/cookbook/diffusion/Ideogram/Ideogram4.mdx @@ -187,3 +187,9 @@ response = client.images.generate( ``` Base Ideogram 4 presets are `V4_DEFAULT_20`, `V4_QUALITY_48`, and `V4_TURBO_12`. The fal variants automatically select `V4_FAST_20` and `V4_INSTANT_8`, respectively. A preset controls both `num_inference_steps` and guidance, so do not set those fields directly. + +## 5. Run in ComfyUI + +import { ComfyUISupport } from '/src/snippets/diffusion/comfyui-support.jsx'; + + diff --git a/docs/cookbook/diffusion/JoyEcho/JoyEcho.mdx b/docs/cookbook/diffusion/JoyEcho/JoyEcho.mdx index 2c17476fa..f41d7b881 100644 --- a/docs/cookbook/diffusion/JoyEcho/JoyEcho.mdx +++ b/docs/cookbook/diffusion/JoyEcho/JoyEcho.mdx @@ -238,3 +238,9 @@ Use `quality=lossless` for this recipe. High-mode output did not pass the separa - For **2-GPU latency**, try **Ulysses SP** (`--num-gpus 2 --ulysses-degree 2`) on both single-shot and multi-shot runs. Use **TP** when you need a different sharding strategy or more than two GPUs. - Set `PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True` for long multi-shot SP sessions. - JoyEcho outputs per-shot mp4 files with synchronized audio. There is no built-in two-stage HQ upscaling path like LTX-2.3 HQ. + +## 6. Run in ComfyUI + +import { ComfyUISupport } from '/src/snippets/diffusion/comfyui-support.jsx'; + + diff --git a/docs/cookbook/diffusion/Krea/Krea-2.mdx b/docs/cookbook/diffusion/Krea/Krea-2.mdx index 1048c80ec..afc40cfea 100644 --- a/docs/cookbook/diffusion/Krea/Krea-2.mdx +++ b/docs/cookbook/diffusion/Krea/Krea-2.mdx @@ -334,3 +334,9 @@ Peak Memory Mean (MB): 37466.40 Peak Memory Median (MB): 37466.00 ============================================================ ``` + +## 6. Run in ComfyUI + +import { ComfyUISupport } from '/src/snippets/diffusion/comfyui-support.jsx'; + + diff --git a/docs/cookbook/diffusion/LTX/LTX2 & LTX2.3.mdx b/docs/cookbook/diffusion/LTX/LTX2 & LTX2.3.mdx index 56f22bafe..99f4f11c8 100644 --- a/docs/cookbook/diffusion/LTX/LTX2 & LTX2.3.mdx +++ b/docs/cookbook/diffusion/LTX/LTX2 & LTX2.3.mdx @@ -248,3 +248,9 @@ Some community LoRAs only include weights for transformer blocks. In that case, - Use `--ltx2-two-stage-device-mode resident` on high-VRAM GPUs if latency matters more than memory usage. - Use `--ltx2-two-stage-device-mode original` when comparing against official two-stage behavior. - Keep `--width` and `--height` aligned with the target model resolution; for LTX models, these are output video dimensions. + +## 6. Run in ComfyUI + +import { ComfyUISupport } from '/src/snippets/diffusion/comfyui-support.jsx'; + + diff --git a/docs/cookbook/diffusion/LingBot-World/LingBot-World-2.0.mdx b/docs/cookbook/diffusion/LingBot-World/LingBot-World-2.0.mdx index fae0cc941..384919e1a 100644 --- a/docs/cookbook/diffusion/LingBot-World/LingBot-World-2.0.mdx +++ b/docs/cookbook/diffusion/LingBot-World/LingBot-World-2.0.mdx @@ -184,3 +184,9 @@ LingBot World 2.0 uses raw-frame websocket GT plus per-chunk latency guards for - Use the realtime endpoint for interactive sessions: `/v1/realtime_video/generate`. - Prefer WebP preview transport for interactive testing; use raw-frame transport for consistency checks. - Long-running sessions should be validated with raw-frame consistency before changing causal cache, condition sampling, or VAE decode behavior. + +## 7. Run in ComfyUI + +import { ComfyUISupport } from '/src/snippets/diffusion/comfyui-support.jsx'; + + diff --git a/docs/cookbook/diffusion/LingBot-World/LingBot-World.mdx b/docs/cookbook/diffusion/LingBot-World/LingBot-World.mdx index b6927bc38..a3dea5325 100644 --- a/docs/cookbook/diffusion/LingBot-World/LingBot-World.mdx +++ b/docs/cookbook/diffusion/LingBot-World/LingBot-World.mdx @@ -159,3 +159,9 @@ LingBot World uses raw-frame websocket GT plus per-chunk latency guards for cons - Use the realtime endpoint for interactive sessions: `/v1/realtime_video/generate`. - Prefer WebP preview transport for interactive testing; use raw-frame transport for consistency checks. - Long-running sessions should be validated with raw-frame consistency before changing causal cache, condition sampling, or VAE decode behavior. + +## 7. Run in ComfyUI + +import { ComfyUISupport } from '/src/snippets/diffusion/comfyui-support.jsx'; + + diff --git a/docs/cookbook/diffusion/LongLive/LongLive-2.0.mdx b/docs/cookbook/diffusion/LongLive/LongLive-2.0.mdx index 4da75d45c..0d9a8363d 100644 --- a/docs/cookbook/diffusion/LongLive/LongLive-2.0.mdx +++ b/docs/cookbook/diffusion/LongLive/LongLive-2.0.mdx @@ -117,3 +117,9 @@ The image is used as the first-frame condition. - SGLang supports T2V sizes 1280x704, 704x1280, 832x480, and 480x832. - I2V request images follow the Wan TI2V preprocessing path in SGLang. This is different from the original LongLive dataset resize path. - For multi-shot runs, set `num_frames` to match `len(shot_prompts) * chunks_per_shot * 8` latent frames, that is `num_frames = (len(shot_prompts) * chunks_per_shot * 8 - 1) * 4 + 1`. + +## 6. Run in ComfyUI + +import { ComfyUISupport } from '/src/snippets/diffusion/comfyui-support.jsx'; + + diff --git a/docs/cookbook/diffusion/MOVA/MOVA.mdx b/docs/cookbook/diffusion/MOVA/MOVA.mdx index d6d7779ee..2f9c9aa74 100644 --- a/docs/cookbook/diffusion/MOVA/MOVA.mdx +++ b/docs/cookbook/diffusion/MOVA/MOVA.mdx @@ -266,3 +266,9 @@ python3 -m sglang.multimodal_gen.benchmarks.bench_serving \ --task image-to-video --dataset vbench --num-prompts 20 --max-concurrency 20 \ --port 30002 ``` + +## 6. Run in ComfyUI + +import { ComfyUISupport } from '/src/snippets/diffusion/comfyui-support.jsx'; + + diff --git a/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx b/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx index 38ef972e0..296cc7e81 100644 --- a/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx +++ b/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx @@ -2055,3 +2055,9 @@ For a measured lower-count AMD deployment, set both `--num-gpus` and `--ulysses-degree` to 4, 2, or 1. AITER packed attention matched segment-wise BF16 SDPA at cosine similarity `0.9999991655` on MI355X and `0.9999991059` on MI300X. + +## 10. Run in ComfyUI + +import { ComfyUISupport } from '/src/snippets/diffusion/comfyui-support.jsx'; + + diff --git a/docs/cookbook/diffusion/Qwen-Image/Qwen-Image-2.1.mdx b/docs/cookbook/diffusion/Qwen-Image/Qwen-Image-2.1.mdx index 33d1be645..46cc9ad90 100644 --- a/docs/cookbook/diffusion/Qwen-Image/Qwen-Image-2.1.mdx +++ b/docs/cookbook/diffusion/Qwen-Image/Qwen-Image-2.1.mdx @@ -14,9 +14,10 @@ import { config } from '/src/snippets/configs/Qwen/qwen-image-2.1.jsx'; Install the runtime dependencies with `uv pip install "sglang[diffusion]" --prerelease=allow`, then install this integration from its source checkout with -`uv pip install -e "python[diffusion]"`. Use an authorized checkpoint directory in -place of `/models/qwen-image-2.1`. The recipes below target NVIDIA CUDA on Linux; -the hardware picker selects a tested single-GPU recipe for the full checkpoint. +`uv pip install -e "python[diffusion]"`. The picker uses `Qwen/Qwen-Image-2.1`; +you can also set a local checkpoint directory under **Variables**. The recipes +target NVIDIA CUDA on Linux; the picker marks which single-GPU workloads have +been verified with the full checkpoint. @@ -27,7 +28,7 @@ steps, and output count. Set reference PNG paths under **Variables**; edits upload files from the machine running cURL, so they need not exist on the server. Hardware selection applies the recommended placement for that GPU. H200, -B200, and RTX PRO 6000 96GB keep weights resident; RTX 5090 and RTX 4090 +B200, RTX PRO 6000 96GB, and DGX Spark keep weights resident; RTX 5090 and RTX 4090 offload selected components to fit the full pipeline. Untested topologies and feature combinations remain selectable and are labeled **Unverified**. Invalid topology combinations disable Copy. This integration @@ -51,6 +52,8 @@ PY The picker defaults to native BF16/FP32 precision, exact attention, eager execution, and full-image VAE decoding. +Commands omit default values, including one GPU, encoder auto scheduling, and +batch size one. Explicit placement and attention overrides preserve each recipe. | GPU | Placement / attention | Generation | Edit | Peak VRAM | | --- | --- | --- | --- | --- | @@ -58,6 +61,7 @@ execution, and full-image VAE decoding. | B200 192GB | Resident / FlashAttention | 2.46 s | 3.02 s | 38.5 GiB | | RTX PRO 6000 96GB | Resident / Torch SDPA | 8.03 s | 9.63 s | 38.4 GiB | | RTX 4090 24GB | DiT and VAE resident, encoder layerwise offload / FlashAttention | 18.68 s | 21.68 s | 22.7 GiB | +| DGX Spark 128GB unified | Resident / Torch SDPA | 35.36 s | 42.23 s | — (unified) | Measured on 2026-09-20 at 1024×1024, 40 steps, CFG 1, and one RGBA PNG per request. Times are median HTTP latency after warmup, including PNG serialization @@ -68,6 +72,25 @@ RTX 5090 uses DiT layerwise offload and Torch SDPA; its recipe has not been retested with the updated checkpoint. Both RTX 5090 and RTX PRO 6000 use SDPA when FlashAttention is selected in this runtime. CPU offload requires host RAM. +### DGX Spark + +Select **DGX Spark** for one GB10 GPU on Linux ARM64 with CUDA 13. Use the +source installation above. The recommended configuration keeps all components +resident, uses native BF16/FP32 precision, and lets the runtime select Torch SDPA: + +```bash Command +sglang serve \ + --model-path Qwen/Qwen-Image-2.1 \ + --performance-mode speed +``` + +The [128 GB unified memory](https://docs.nvidia.com/dgx/dgx-spark/hardware.html) +is shared by the CPU and GPU. CPU offload is unnecessary for the verified +single-image 1024×1024 workload. Keep full-image VAE decoding and eager execution. +Generation, editing, transparent generation, and transparent editing were +verified with PyTorch 2.13.0+cu130. Spark reports no separate VRAM usage in `nvidia-smi`. +This recipe covers one Spark; multi-node deployment and batching remain unverified. + ### Batching Keep **Request batching → Off** and **Outputs → 1** for interactive use. @@ -119,29 +142,24 @@ including partly transparent edges, without thresholding or background removal. ## 4. Offline requests +Defaults are 1024×1024, 40 steps, CFG 1, and seed 42; output saving is enabled. +For GPUs that need offload, also pass the placement flags from the picker. + ### Text-to-image ```bash Command sglang generate \ - --model-path /models/qwen-image-2.1 \ - --model-id Qwen-Image-2.1 \ - --prompt "A capybara reading a book by candlelight" \ - --width 1024 --height 1024 \ - --num-inference-steps 40 --guidance-scale 1 \ - --seed 0 --save-output + --model-path Qwen/Qwen-Image-2.1 \ + --prompt "A capybara reading a book by candlelight" ``` ### Image-conditioned editing ```bash Command sglang generate \ - --model-path /models/qwen-image-2.1 \ - --model-id Qwen-Image-2.1 \ + --model-path Qwen/Qwen-Image-2.1 \ --image-path /path/to/input.png \ - --prompt "Move the scene to a snowy mountain at sunrise" \ - --width 1024 --height 1024 \ - --num-inference-steps 40 --guidance-scale 1 \ - --seed 0 --save-output + --prompt "Move the scene to a snowy mountain at sunrise" ``` Height and width must be positive multiples of 32. Reference images preserve @@ -203,7 +221,7 @@ See the [GGUF guide](/docs/sglang-diffusion/quantization#gguf). NVFP4 requires Blackwell and compatible ModelOpt exports. Select the component directories using `--component-paths.transformer` and/or `--component-paths.text_encoder`. Keep the FlashInfer backend at `auto` on -RTX 5090 and RTX PRO 6000: TensorRT-LLM FP4 GEMM does not support SM120. +RTX 5090, RTX PRO 6000, and DGX Spark: TensorRT-LLM FP4 GEMM does not support SM12.x. These GPUs remain unverified for this model's NVFP4 exports. See the [NVFP4 guide](/docs/sglang-diffusion/quantization#modelopt-nvfp4). @@ -216,3 +234,11 @@ Keep eager execution as the default. Breakable CUDA Graph replay requires matching resolution and condition-prefix length; unseen shapes run eagerly. Text buckets alone do not guarantee replay. SageAttention and Cache-DiT can change numerical results and require quality checks for your workload. + +### Cache-DiT + +Enable `--enable-cache-dit true` or `SGLANG_CACHE_DIT_ENABLED=true`. 2.1 prefix +KV is per layer: each block slices caches by `_layer_id`. Cache-DiT wraps +`transformer_blocks` and forwards the same extras to every layer; without that +slice, later layers reuse layer 0 and the image collapses to color noise. +See the [Cache-DiT guide](/docs/sglang-diffusion/cache_dit). diff --git a/docs/cookbook/diffusion/Qwen-Image/Qwen-Image-Edit.mdx b/docs/cookbook/diffusion/Qwen-Image/Qwen-Image-Edit.mdx index 5479fa733..5b7c92e1e 100644 --- a/docs/cookbook/diffusion/Qwen-Image/Qwen-Image-Edit.mdx +++ b/docs/cookbook/diffusion/Qwen-Image/Qwen-Image-Edit.mdx @@ -315,3 +315,9 @@ Peak Memory Mean (MB): 47971.49 Peak Memory Median (MB): 47971.29 ============================================================ ``` + +## 6. Run in ComfyUI + +import { ComfyUISupport } from '/src/snippets/diffusion/comfyui-support.jsx'; + + diff --git a/docs/cookbook/diffusion/Qwen-Image/Qwen-Image.mdx b/docs/cookbook/diffusion/Qwen-Image/Qwen-Image.mdx index 87d6e1320..881dfa9c5 100644 --- a/docs/cookbook/diffusion/Qwen-Image/Qwen-Image.mdx +++ b/docs/cookbook/diffusion/Qwen-Image/Qwen-Image.mdx @@ -428,3 +428,9 @@ Test Environment: ``` + +## 6. Run in ComfyUI + +import { ComfyUISupport } from '/src/snippets/diffusion/comfyui-support.jsx'; + + diff --git a/docs/cookbook/diffusion/SANA-WM/SANA-WM.mdx b/docs/cookbook/diffusion/SANA-WM/SANA-WM.mdx index 811e13b6a..4983ea663 100644 --- a/docs/cookbook/diffusion/SANA-WM/SANA-WM.mdx +++ b/docs/cookbook/diffusion/SANA-WM/SANA-WM.mdx @@ -487,3 +487,9 @@ At WebSocket `init` the realtime adapter fills SANA-WM defaults that differ from `guidance_scale` applies to the dense path (§4) only; the distilled streaming path uses `streaming_cfg_scale` (default `1.0`, i.e. no CFG) so a `guidance_scale` override never accidentally enables CFG on the streaming stage. `denoising_step_list = (1000, 960, 889, 727, 0)` is the official 4-step streaming schedule (it must end in 0). + +## 10. Run in ComfyUI + +import { ComfyUISupport } from '/src/snippets/diffusion/comfyui-support.jsx'; + + diff --git a/docs/cookbook/diffusion/Wan/Wan2.1.mdx b/docs/cookbook/diffusion/Wan/Wan2.1.mdx index a517902c9..2c5640526 100644 --- a/docs/cookbook/diffusion/Wan/Wan2.1.mdx +++ b/docs/cookbook/diffusion/Wan/Wan2.1.mdx @@ -381,3 +381,9 @@ You can use the built-in SGLang diffusion benchmark script to evaluate Wan2.1 pe ``` + +## 6. Run in ComfyUI + +import { ComfyUISupport } from '/src/snippets/diffusion/comfyui-support.jsx'; + + diff --git a/docs/cookbook/diffusion/Wan/Wan2.2.mdx b/docs/cookbook/diffusion/Wan/Wan2.2.mdx index d2b8f6c9f..a24355ed3 100644 --- a/docs/cookbook/diffusion/Wan/Wan2.2.mdx +++ b/docs/cookbook/diffusion/Wan/Wan2.2.mdx @@ -459,3 +459,9 @@ Test Environment: ``` + +## 6. Run in ComfyUI + +import { ComfyUISupport } from '/src/snippets/diffusion/comfyui-support.jsx'; + + diff --git a/docs/cookbook/diffusion/Z-Image/Z-Image-Turbo.mdx b/docs/cookbook/diffusion/Z-Image/Z-Image-Turbo.mdx index 91cdba231..b5dc01e4a 100644 --- a/docs/cookbook/diffusion/Z-Image/Z-Image-Turbo.mdx +++ b/docs/cookbook/diffusion/Z-Image/Z-Image-Turbo.mdx @@ -29,7 +29,7 @@ This section provides deployment configurations optimized for different hardware Z-Image-Turbo is optimized for high-quality image generation with only 8 inference steps. The recommended launch configurations vary by hardware. -**Interactive Command Generator**: Use the configuration selector below to automatically generate the appropriate deployment command for your hardware platform. +**Interactive Command Generator**: Use the configuration selector below to automatically generate the appropriate deployment command for your hardware platform and model version. SGLang supports serving Z-Image-Turbo on NVIDIA B200, H200, H100, and AMD MI355X, MI325X, MI300X GPUs, Ascend A2, A3 NPUs and Intel Arc Pro B-Series GPUs(codename: BMG (Battlemage)). @@ -366,3 +366,9 @@ Test Environment: ``` + +## 6. Run in ComfyUI + +import { ComfyUISupport } from '/src/snippets/diffusion/comfyui-support.jsx'; + + diff --git a/docs/docs.json b/docs/docs.json index 5418402c3..0cf88a03b 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -649,6 +649,10 @@ "source": "/docs/hardware-platforms/ascend-npus/best_practice/mimo_v2_flash", "destination": "/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/mimo_v2_flash" }, + { + "source": "/docs/hardware-platforms/ascend-npus/best_practice/mimo_v2_5_pro", + "destination": "/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/mimo_v2_5_pro" + }, { "source": "/docs/hardware-platforms/ascend-npus/best_practice/qwen3-8b", "destination": "/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_8b" @@ -1142,6 +1146,7 @@ "docs/hardware-platforms/ascend-npus/model-deployment/best-practices/kimi_k2_6", "docs/hardware-platforms/ascend-npus/model-deployment/best-practices/minimax_m2_5", "docs/hardware-platforms/ascend-npus/model-deployment/best-practices/mimo_v2_flash", + "docs/hardware-platforms/ascend-npus/model-deployment/best-practices/mimo_v2_5_pro", "docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_8b", "docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_32b", "docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_30b_a3b", @@ -1428,6 +1433,7 @@ { "group": "Xiaomi", "pages": [ + "cookbook/autoregressive/Xiaomi/MiMo-V2.6", "cookbook/autoregressive/Xiaomi/MiMo-V2.5", "cookbook/autoregressive/Xiaomi/MiMo-V2-Flash" ] diff --git a/docs/docs/advanced_features/hisparse_guide.mdx b/docs/docs/advanced_features/hisparse_guide.mdx index 294ecba6d..54c412baf 100644 --- a/docs/docs/advanced_features/hisparse_guide.mdx +++ b/docs/docs/advanced_features/hisparse_guide.mdx @@ -6,7 +6,7 @@ metatags: HiSparse reduces per-request GPU memory consumption during the decode phase by maintaining only a small "hot" KV buffer on GPU while keeping complete KV data in CPU pinned memory. Combined with PD disaggregation, it enables significantly higher decode concurrency. -> **Prerequisites**: HiSparse works with models that use **DeepSeek Sparse Attention (DSA)** architectures (e.g., DeepSeek-V3.2, GLM-5.1) and **DeepSeek V4**. These models natively select a subset of tokens for attention, making it possible to keep only the top-k KV on GPU while storing the full KV in host memory — without accuracy loss. Additionally, HiSparse currently requires **PD disaggregation mode** and is enabled on the **decode instance** only. +> **Prerequisites**: HiSparse works with models that use **DeepSeek Sparse Attention (DSA)** architectures (e.g., DeepSeek-V3.2, GLM-5.1), **DeepSeek V4**, and **MiniMax M3**. These models natively select a subset of tokens for attention, making it possible to keep only the top-k KV on GPU while storing the full KV in host memory — without accuracy loss. Additionally, HiSparse currently requires **PD disaggregation mode** and is enabled on the **decode instance** only. ## Why HiSparse? @@ -165,6 +165,15 @@ python3 -m sglang.launch_server \ > **Note**: For DSA models, `--kv-cache-dtype` defaults to `auto`, which resolves to `fp8_e4m3` on SM100+ (Blackwell) and `bfloat16` on older architectures. The DSA decode backend is automatically selected based on KV dtype (`bfloat16` → `flashmla_sparse`, `fp8_e4m3` → `flashmla_kv`), except for GLM DSA models on SM120/SM121 with `fp8_e4m3`, which use `flashinfer_sparse_mla`. DSA backend flags apply only to DSA models; DeepSeek V4 uses its own `dsv4` attention backend. +### MiniMax M3 + +Dense-layer K/V and index K stay on GPU; sparse-layer K/V use host memory plus a GPU working set. + +- Use TP 4 or greater, with the same TP size and PP 1 on both PD instances. +- Use `--attention-backend triton`, `--mm-attention-backend triton_attn`, `--disable-prefill-cuda-graph`, and `--disable-radix-cache`. +- Set `device_buffer_size` to at least 2048 in `--hisparse-config`; `top_k` does not override the model's selection width. +- PD retraction backup is not supported. Use `--num-reserved-decode-tokens` to reserve capacity for the expected output length. + ### Benchmark ```bash Command diff --git a/docs/docs/advanced_features/server_arguments.mdx b/docs/docs/advanced_features/server_arguments.mdx index c386e6d30..608fe3166 100644 --- a/docs/docs/advanced_features/server_arguments.mdx +++ b/docs/docs/advanced_features/server_arguments.mdx @@ -1154,13 +1154,13 @@ Combining `--enable-response-store` with `--disaggregation-mode=prefill` or `dec `--reasoning-parser` Specify the parser for reasoning models. Use `auto` to detect the parser from the model's chat template. `None` - auto, apertus2509, deepseek-r1, deepseek-v3, deepseek-v4, dots, glm45, ling3, hunyuan, gpt-oss, k2_horizon, kimi, kimi_k2, kimi_k3, mimo, muse, poolside_v1, qwen3, qwen3-thinking, minimax, minimax-append-think, minimax-m3, step3, step3p5, mistral, nemotron_3, interns1, gemma4, inkling, cohere_command4 + auto, apertus2509, deepseek-r1, deepseek-v3, deepseek-v4, dots, glm45, ling3, hunyuan, gpt-oss, k2_horizon, kimi, kimi_k2, kimi_k3, mimo, muse, poolside_v1, qwen3, qwen3-thinking, minimax, minimax-append-think, minimax-m3, step3, step3p5, mistral, nemotron_3, interns1, gemma4, gigachat35, inkling, cohere_command4 `--tool-call-parser` Specify the parser for handling tool-call interactions. Use `auto` to detect the parser from the model's chat template. `None` - auto, apertus2509, cohere_command4, deepseekv3, deepseekv31, deepseekv32, deepseekv4, dots, glm, glm45, glm47, gpt-oss, k2_horizon, kimi_k2, kimi_k3, lfm2, ling3, llama3, mimo, minicpm5, mistral, muse, poolside_v1, pythonic, qwen, qwen25, qwen3_coder, spark25, step3, step3p5, minimax-m2, minimax-m3, trinity, interns1, hermes, hunyuan, gigachat3, gemma4, inkling + auto, apertus2509, cohere_command4, deepseekv3, deepseekv31, deepseekv32, deepseekv4, dots, glm, glm45, glm47, gpt-oss, k2_horizon, kimi_k2, kimi_k3, lfm2, ling3, llama3, mimo, minicpm5, mistral, muse, poolside_v1, pythonic, qwen, qwen25, qwen3_coder, spark25, step3, step3p5, minimax-m2, minimax-m3, trinity, interns1, hermes, hunyuan, gigachat3, gigachat35, gemma4, inkling `--tool-server` diff --git a/docs/docs/developer_guide/contribution_guide.mdx b/docs/docs/developer_guide/contribution_guide.mdx index dc1ef9f67..08b212d31 100644 --- a/docs/docs/developer_guide/contribution_guide.mdx +++ b/docs/docs/developer_guide/contribution_guide.mdx @@ -147,6 +147,21 @@ To avoid spamming a PR with too many `/rerun-failed-ci` comments, you can also t If you don’t have permission and you’re not the PR author, please ask maintainers to trigger CI for you. +### CI control labels + +Four labels change how a PR's CI runs. Label before you trigger: +`max-concurrency` fixes the shard fan-out when the run is created. + +| Label | Effect | +| --- | --- | +| `bypass-fail-fast` | A job failing no longer aborts its siblings; the run continues instead of stopping at the first root-cause failure. A failing `lint` still stops everything. | +| `parallel-stages` | Stages stop waiting on each other. `base-a`, `base-b` and `base-c` dispatch together, the way a scheduled run does. | +| `max-concurrency` | A suite fans out to all of its shards at once rather than a third of them. | +| `highest-priority` | All three of the above, and the PR is also skipped by the batch-cancel workflow, sorted first in the active-runs report, and never closed as stale. | + +Each spends extra runner capacity, so they unblock a specific PR rather than +serve as a default. + ### CI rate limits Due to CI scheduling and limited resources, higher-priority PRs may preempt running jobs. In such cases, you may need to rerun the tests. diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/mimo_v2_5_pro.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/mimo_v2_5_pro.mdx new file mode 100644 index 000000000..148749b5b --- /dev/null +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/mimo_v2_5_pro.mdx @@ -0,0 +1,170 @@ +--- +title: "MiMo-V2.5-Pro" +metatags: + description: "Best Practice for MiMo-V2.5-Pro on Ascend NPU" +--- + + +This page focuses on the deployment of MiMo-V2.5-Pro (FP4) with DFlash speculative decoding in PD disaggregation mode on the Ascend NPU. + +On the A3 Series, each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware) for details. + + +### Model Deployment + +MiMo-V2.5-Pro-FP4-DFlash is an MXFP4-quantized checkpoint with a built-in DFlash draft model (located in the `dflash/` subdirectory of the weights). The following example deploys it in 1P1D mode (1 prefill node + 1 decode node, TP8 + DP2 per node). + +#### Common environment setup (both nodes) + +```bash Command +# ============================================================ +# Before running, update the following variables: +# ASCEND_MF_STORE_URL: prefill node IP with port +# HCCL_SOCKET_IFNAME / GLOO_SOCKET_IFNAME: network interface name +# ============================================================ + +echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor +sysctl -w vm.swappiness=0 +sysctl -w kernel.numa_balancing=0 +sysctl -w kernel.sched_migration_cost_ns=50000 + +export SGLANG_SET_CPU_AFFINITY=1 +unset https_proxy +unset http_proxy +unset HTTPS_PROXY +unset HTTP_PROXY +unset ASCEND_LAUNCH_BLOCKING + +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh + +export HCCL_BUFFSIZE=300 +export HCCL_OP_EXPANSION_MODE=AIV +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True +export STREAMS_PER_DEVICE=32 +export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 + +# Use the AscendC flash attention +export ASCEND_USE_FIA=1 + +# PD disaggregation transfer config +export ASCEND_MF_STORE_URL="tcp://:24669" +export ASCEND_MF_TRANSFER_PROTOCOL="device_urma" + +# DeepEP +export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=32 +export HCCL_SOCKET_IFNAME= +export GLOO_SOCKET_IFNAME= +export HCCL_HOST_SOCKET_PORT_RANGE=auto + +MODEL_PATH=/path/to/MiMo-V2.5-Pro-FP4-DFlash +``` + +#### Prefill node + +```bash Command +export DEEPEP_HCCL_BUFFSIZE=2500 +# Enable chunked dispatch for long sequences +export DEEPEP_NORMAL_LONG_SEQ_ROUND=10 +export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=4096 +export DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ=0 + +python3 -m sglang.launch_server \ + --model-path $MODEL_PATH \ + --attention-backend ascend \ + --device npu \ + --tp-size 8 --nnodes 1 --node-rank 0 \ + --chunked-prefill-size 8192 \ + --trust-remote-code --port 10001 \ + --host --max-running-requests 32 \ + --mem-fraction-static 0.90 \ + --swa-full-tokens-ratio 0.3 \ + --disaggregation-mode prefill --disaggregation-transfer-backend ascend \ + --disaggregation-bootstrap-port 8996 \ + --disable-piecewise-cuda-graph \ + --dp-size 2 --enable-dp-attention --enable-dp-lm-head \ + --moe-a2a-backend deepep --deepep-mode normal +``` + +#### Decode node + +```bash Command +# Use eagle_worker_v2 and overlap plan stream to hide the draft/target preparation +export SGLANG_ENABLE_SPEC_V2=1 +export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 +# DFlash draft model has a longer context length than the derived value +export SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1 + +export DEEPEP_HCCL_BUFFSIZE=1200 + +python3 -m sglang.launch_server \ + --model-path $MODEL_PATH \ + --speculative-draft-model-path $MODEL_PATH/dflash \ + --attention-backend ascend \ + --device npu \ + --tp-size 8 --nnodes 1 --node-rank 0 \ + --trust-remote-code --port 20001 \ + --host --max-running-requests 32 \ + --mem-fraction-static 0.88 \ + --swa-full-tokens-ratio 0.3 \ + --cuda-graph-bs 1 2 4 8 12 16 \ + --disaggregation-mode decode --disaggregation-transfer-backend ascend \ + --disaggregation-bootstrap-port 8996 \ + --moe-a2a-backend deepep --deepep-mode low_latency \ + --dp-size 2 --enable-dp-attention --enable-dp-lm-head \ + --speculative-algorithm DFLASH \ + --speculative-num-draft-tokens 8 +``` + +#### Router + +```bash Command +python -m sglang_router.launch_router \ + --pd-disaggregation \ + --policy cache_aware \ + --prefill http://:10001 \ + --decode http://:20001 \ + --host 127.0.0.1 \ + --port 6688 \ + --health-check-interval-secs 3600 --mini-lb +``` + +### Benchmark + +We tested it based on the `RANDOM` dataset. + +#### Benchmark Prefill Node (TTFT) + +```bash Command +python3 -m sglang.bench_serving \ + --backend sglang \ + --host 127.0.0.1 \ + --port 6688 \ + --model /path/to/MiMo-V2.5-Pro-FP4-DFlash \ + --dataset-name random \ + --tokenize-prompt \ + --random-input-len 16000 \ + --random-output-len 1 \ + --request-rate 0.4 \ + --random-range-ratio 1 \ + --num-prompts 128 \ + --max-concurrency 32 +``` + +#### Benchmark Decode Node (TPOT) + +```bash Command +python3 -m sglang.bench_serving \ + --backend sglang \ + --host 127.0.0.1 \ + --port 6688 \ + --model /path/to/MiMo-V2.5-Pro-FP4-DFlash \ + --dataset-name random \ + --tokenize-prompt \ + --random-input-len 16000 \ + --random-output-len 1000 \ + --request-rate inf \ + --random-range-ratio 1 \ + --num-prompts 128 \ + --max-concurrency 32 +``` diff --git a/docs/docs/hardware-platforms/xpu.mdx b/docs/docs/hardware-platforms/xpu.mdx index 12bd42346..336e22a5a 100644 --- a/docs/docs/hardware-platforms/xpu.mdx +++ b/docs/docs/hardware-platforms/xpu.mdx @@ -7,39 +7,8 @@ The document addresses how to set up the [SGLang](https://github.com/sgl-project Specifically, SGLang is optimized for [Intel® Arc™ Pro B-Series Graphics](https://www.intel.com/content/www/us/en/ark/products/series/242616/intel-arc-pro-b-series-graphics.html) and [ Intel® Arc™ B-Series Graphics](https://www.intel.com/content/www/us/en/ark/products/series/240391/intel-arc-b-series-graphics.html). -## Optimized Model List - -A list of LLMs have been optimized on Intel GPU, and more are on the way: - - - - - - - - - - - - - - - - - - - - - - - - - - -
Model NameBF16
Llama-3.2-3B[meta-llama/Llama-3.2-3B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct)
Llama-3.1-8B[meta-llama/Llama-3.1-8B-Instruct](https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct)
Qwen2.5-1.5B[Qwen/Qwen2.5-1.5B](https://huggingface.co/Qwen/Qwen2.5-1.5B)
- -**Note:** The model identifiers listed in the table above -have been verified on [Intel® Arc™ B580 Graphics](https://www.intel.com/content/www/us/en/products/sku/241598/intel-arc-b580-graphics/specifications.html). +A number of popular LLMs are optimized and run efficiently on XPU, including the most notable open-source models like Llama series, Qwen series, and Diffusion model series like FLUX and Z-Image. +Please check the [SGLang Cookbook pages](https://docs.sglang.io/cookbook/intro) in which the support status and example commands can be found. Quantized MoE models are covered separately in [MXFP4 MoE Quantization](#mxfp4-moe-quantization) below. @@ -85,6 +54,20 @@ pip install -v . --extra-index-url https://download.pytorch.org/whl/xpu ### Install Using Docker +It is recommended to use Docker for setting up the SGLang environment. + +#### Pull from Docker Hub + +Pull the prebuilt docker image of SGLang package releases from `lmsysorg/sglang` repository. +The [XPU image tags](https://hub.docker.com/r/lmsysorg/sglang/tags?name=xpu) end with `xpu` suffix. +The image pulling command is like: + +```bash Command +docker pull lmsysorg/sglang:v0.5.20-xpu +``` + +#### Build from Dockerfile + [The SGLang XPU Dockerfile](https://github.com/sgl-project/sglang/blob/main/docker/xpu.Dockerfile) is provided to facilitate the installation. Replace `` below with your [HuggingFace access token](https://huggingface.co/docs/hub/en/security-tokens). @@ -121,7 +104,6 @@ Example command to launch SGLang serving: sglang serve \ --model-path \ --trust-remote-code \ - --disable-overlap-schedule \ --device xpu \ --host 0.0.0.0 \ --tp 2 \ # using multi GPUs diff --git a/docs/docs/sglang-diffusion/cache_dit.mdx b/docs/docs/sglang-diffusion/cache_dit.mdx index d26ac88b9..9ca452450 100644 --- a/docs/docs/sglang-diffusion/cache_dit.mdx +++ b/docs/docs/sglang-diffusion/cache_dit.mdx @@ -783,7 +783,7 @@ SGLang Diffusion x Cache-DiT supports almost all models originally supported in Qwen - Qwen-Image, Qwen-Image-Edit + Qwen-Image, Qwen-Image-Edit, Qwen-Image 2.1 Hunyuan diff --git a/docs/docs/sglang-diffusion/compatibility_matrix.mdx b/docs/docs/sglang-diffusion/compatibility_matrix.mdx index daca9f373..5b9d6a61e 100644 --- a/docs/docs/sglang-diffusion/compatibility_matrix.mdx +++ b/docs/docs/sglang-diffusion/compatibility_matrix.mdx @@ -105,12 +105,10 @@ replay only matching warmup signatures, including condition-prefix length; other shapes run eagerly. VAE tiling is opt-in and can change numerical results. Do not infer compatibility from the older Qwen-Image row. -Use an authorized local checkpoint with `--model-id Qwen-Image-2.1`. +Use the public `Qwen/Qwen-Image-2.1` checkpoint; no Hugging Face token is required. See the [Qwen-Image 2.1 cookbook](/cookbook/diffusion/Qwen-Image/Qwen-Image-2.1) -for checkpoint layout and usage. This entry does not assert public weight -availability. The standard two-GPU E2E suite includes `qwen_image21_t2i_tp2` +for checkpoint layout and usage. The standard two-GPU E2E suite includes `qwen_image21_t2i_tp2` with TP2, 1024px/40-step generation, two requests, and image consistency checks. -Its runners need access to `Qwen/Qwen-Image-2.1`. The additional opt-in HTTP cases check repeated generation, editing, and real RGBA alpha output from a local checkpoint: diff --git a/docs/docs/supported-models/generative_models.mdx b/docs/docs/supported-models/generative_models.mdx index b3fdff683..6811d2954 100644 --- a/docs/docs/supported-models/generative_models.mdx +++ b/docs/docs/supported-models/generative_models.mdx @@ -308,5 +308,10 @@ in the GitHub search bar. JetBrains/Mellum2-12B-A2.5B-Base, JetBrains/Mellum2-12B-A2.5B-Thinking JetBrains' Qwen3-MoE-based code generation model with interleaved sliding-window/full attention, per-layer-type RoPE, and per-layer dense/sparse MLP routing. + + GigaChat 3.5 + ai-sage/GigaChat3.5-432B-A28B + GigaChat's hybrid MoE model: per-layer mix of DeepSeek-style MLA full attention and Qwen3 Gated-Delta-Net (GDN) linear attention, DeepSeek-style routed + shared experts, and multiple multi-token-prediction (MTP) heads for speculative decoding. Emits tool calls in GCML format. + diff --git a/docs/src/snippets/_deployment.jsx b/docs/src/snippets/_deployment.jsx index 42c127978..b86f83774 100644 --- a/docs/src/snippets/_deployment.jsx +++ b/docs/src/snippets/_deployment.jsx @@ -147,10 +147,14 @@ export const Deployment = ({ config, benchmarks }) => { { id: "mi355x", label: "MI355X", vram: "288GB", multiNodeDockerFlags: [...AMD_RDMA_DOCKER_FLAGS] }, ], - // Ascend A3 Series: 1 card = 2 dies, so --tp-size is 2× the card - // count (32 cards -> --tp-size 64). + // Ascend device layout: one /dev/davinciN per core. An A3 Series card is + // the exception — 2 dies per card, so an 8-card node exposes 16 devices + // and --tp-size is twice the card count. A 950PR/DT Series card is a + // single core, so the device count and --tp-size follow the cards. Both + // counts feed the docker `--device` list (`npuDevices`). npu: [ - { id: "a3", label: "Ascend A3 Series", vram: "64GB/die" }, + { id: "a3", label: "A3 Series", vram: "64GB/die", npuDevices: 16 }, + { id: "a5", label: "950PR/DT Series", vram: "128GB", npuDevices: 8 }, ], }; @@ -819,14 +823,31 @@ export const Deployment = ({ config, benchmarks }) => { return (extra && extra.vendor) || "nvidia"; }; // `config.hardware` overrides by id, as in buildHardwareGroups. - const fabricFlagsOf = (hwId) => { + const catalogEntryOf = (hwId) => { const extra = (config.hardware || []).find((h) => h.id === hwId); - if (extra) return extra.multiNodeDockerFlags || []; + if (extra) return extra; for (const list of Object.values(HARDWARE_CATALOG)) { const hit = list.find((h) => h.id === hwId); - if (hit) return hit.multiNodeDockerFlags || []; + if (hit) return hit; } - return []; + return null; + }; + const fabricFlagsOf = (hwId) => + (catalogEntryOf(hwId) || {}).multiNodeDockerFlags || []; + // NPU cards are reached with --device, one per /dev/davinciN core; + // `npuDevices` carries the per-product-line count (16 on an A3 Series + // node, 8 on a 950PR/DT Series node), four devices per line as the host + // docs show. + const davinciLines = (devices) => { + const lines = []; + for (let i = 0; i < devices; i += 4) { + const group = []; + for (let k = i; k < Math.min(i + 4, devices); k++) { + group.push(`--device=/dev/davinci${k}`); + } + lines.push(" " + group.join(" ")); + } + return lines; }; const gpuAccessLines = vendorOf(sel.hw) === "amd" ? [ @@ -838,14 +859,10 @@ export const Deployment = ({ config, benchmarks }) => { ] : vendorOf(sel.hw) === "npu" ? [ - // NPU: --privileged grants the davinci devices (16 dies on an - // 8-card Ascend A3 Series node); the host CANN driver/firmware/state - // must be mounted in. + // NPU: --privileged grants the davinci devices; the host CANN + // driver/firmware/state must be mounted in. "docker run --privileged --shm-size=16g", - " --device=/dev/davinci0 --device=/dev/davinci1 --device=/dev/davinci2 --device=/dev/davinci3", - " --device=/dev/davinci4 --device=/dev/davinci5 --device=/dev/davinci6 --device=/dev/davinci7", - " --device=/dev/davinci8 --device=/dev/davinci9 --device=/dev/davinci10 --device=/dev/davinci11", - " --device=/dev/davinci12 --device=/dev/davinci13 --device=/dev/davinci14 --device=/dev/davinci15", + ...davinciLines((catalogEntryOf(sel.hw) || {}).npuDevices || 16), " --device=/dev/davinci_manager", " --device=/dev/hisi_hdc", " -v /usr/local/sbin:/usr/local/sbin", diff --git a/docs/src/snippets/_kimi_k3_mamba_ratio_calculator.jsx b/docs/src/snippets/_kimi_k3_mamba_ratio_calculator.jsx index 3a996b2ae..97a0b2baf 100644 --- a/docs/src/snippets/_kimi_k3_mamba_ratio_calculator.jsx +++ b/docs/src/snippets/_kimi_k3_mamba_ratio_calculator.jsx @@ -135,10 +135,18 @@ export const KimiK3MambaRatioCalculator = () => { const eff = derive(cfg.flags, cfg.env); const bs = derive(cfg.baseFlags.length ? cfg.baseFlags : cfg.flags, cfg.baseFlags.length ? cfg.baseEnv : cfg.env); - // A --max-mamba-cache-size cell sizes the pool explicitly — no ratio to - // compute or broadcast. - const explicitSizing = (cfg.baseFlags.length ? cfg.baseFlags : cfg.flags) - .some((f) => f.startsWith("--max-mamba-cache-size")); + // Recipes that size the dual pool without the ratio neither render one nor + // broadcast one: + // - a --max-mamba-cache-size cell pins the KDA slot count explicitly; + // - the Ascend NPU recipes size both pools internally and never set + // --mamba-full-memory-ratio (every NPU recipe carries --device npu). + const baseFlagList = cfg.baseFlags.length ? cfg.baseFlags : cfg.flags; + const explicitSizing = baseFlagList.some((f) => f.startsWith("--max-mamba-cache-size")); + const npuRecipe = baseFlagList.some((f) => { + const [head, ...rest] = f.trim().split(/[\s=]+/); + return head === "--device" && rest[0] === "npu"; + }); + const ratioNotApplicable = explicitSizing || npuRecipe; const { ratio, tp, dp, attnTp, dcp, kvDtype, ssmDtype, radixOff, strategy, skipLock, slots, specOn, replaySpec, block, pdRole } = eff; const valid = Number.isFinite(ratio) && ratio > 0 && length > 0 && 96 % attnTp === 0; const baseValid = Number.isFinite(bs.ratio) && bs.ratio > 0 && length > 0; @@ -155,18 +163,22 @@ export const KimiK3MambaRatioCalculator = () => { const cliFlag = valid ? `--mamba-full-memory-ratio ${result}` : ""; // Broadcast both results: the Deploy command takes the base-config value, - // the Playground's composed command takes the effective one. + // the Playground's composed command takes the effective one. On a recipe + // that does not use the ratio, broadcast nulls instead of skipping the + // dispatch — the panels must drop a ratio pinned for a recipe the reader + // has since left rather than keep injecting it. useEffect(() => { - if (explicitSizing) return; window.dispatchEvent( new CustomEvent("sglang-k3-mamba-ratio", { - detail: { - ratio: valid ? result : null, - baseRatio: baseValid ? baseResult : null, - }, + detail: ratioNotApplicable + ? { ratio: null, baseRatio: null } + : { + ratio: valid ? result : null, + baseRatio: baseValid ? baseResult : null, + }, }) ); - }, [result, valid, baseResult, baseValid, explicitSizing]); + }, [result, valid, baseResult, baseValid, ratioNotApplicable]); const copyFlag = () => { if (!cliFlag || typeof navigator === "undefined" || !navigator.clipboard) return; @@ -235,10 +247,12 @@ export const KimiK3MambaRatioCalculator = () => { specLabel, ].filter(Boolean); - if (explicitSizing) { + if (ratioNotApplicable) { return (
- This recipe sizes the KDA state pool explicitly with --max-mamba-cache-size, so the ratio calculator does not apply. + {explicitSizing + ? <>This recipe sizes the KDA state pool explicitly with --max-mamba-cache-size, so the ratio calculator does not apply. + : <>This recipe runs on Ascend NPUs, which size the KDA state pool and the MLA KV pool internally and never set --mamba-full-memory-ratio, so the ratio calculator does not apply.}
); } diff --git a/docs/src/snippets/_playground.jsx b/docs/src/snippets/_playground.jsx index 2fcf21377..11a35d14e 100644 --- a/docs/src/snippets/_playground.jsx +++ b/docs/src/snippets/_playground.jsx @@ -1723,9 +1723,25 @@ export const Playground = ({ config }) => { mi355x: AMD_RDMA_DOCKER_FLAGS, }; const fabricFlags = HW_MULTINODE_DOCKER_FLAGS[sel.hw] || []; - // Mirrors the vendor branch in _deployment.jsx: ROCm reaches its GPUs - // through /dev/kfd + /dev/dri and the video group, not --gpus all. + // Mirrors the vendor branches in _deployment.jsx: ROCm reaches its GPUs + // through /dev/kfd + /dev/dri and the video group (not --gpus all), and + // Ascend NPUs are reached with --device, one per /dev/davinciN core (16 + // on an A3 Series node, 8 on a 950PR/DT Series node — the catalog's + // `npuDevices`). const isAmdHw = /^mi\d/.test(sel.hw || ""); + const HW_NPU_DEVICES = { a3: 16, a5: 8 }; + const npuDevices = HW_NPU_DEVICES[sel.hw]; + const davinciLines = (devices) => { + const lines = []; + for (let i = 0; i < devices; i += 4) { + const group = []; + for (let k = i; k < Math.min(i + 4, devices); k++) { + group.push(`--device=/dev/davinci${k}`); + } + lines.push(" " + group.join(" ")); + } + return lines; + }; const dockerLines = [ ...(isAmdHw ? [ @@ -1735,6 +1751,21 @@ export const Playground = ({ config }) => { " --cap-add=SYS_PTRACE --security-opt seccomp=unconfined", " --shm-size 32g", ] + : npuDevices + ? [ + // NPU: --privileged grants the davinci devices; the host CANN + // driver/firmware/state must be mounted in. + "docker run --privileged --shm-size=16g", + ...davinciLines(npuDevices), + " --device=/dev/davinci_manager", + " --device=/dev/hisi_hdc", + " -v /usr/local/sbin:/usr/local/sbin", + " -v /usr/local/Ascend/driver:/usr/local/Ascend/driver", + " -v /usr/local/Ascend/firmware:/usr/local/Ascend/firmware", + " -v /etc/ascend_install.info:/etc/ascend_install.info", + " -v /var/queue_schedule:/var/queue_schedule", + " -v ~/.cache/:/root/.cache/", + ] : [ "docker run --gpus all", " --shm-size 32g", @@ -1743,7 +1774,8 @@ export const Playground = ({ config }) => { // A PD pair is cross-host even when each role is a single-node cell, so // the RDMA fabric flags are needed for `pdMode` too, not just multinode. ...((multinode || pdMode) ? fabricFlags.map((x) => " " + x) : []), - " -v ~/.cache/huggingface:/root/.cache/huggingface", + // The NPU device block already mounts ~/.cache/. + ...(npuDevices ? [] : [" -v ~/.cache/huggingface:/root/.cache/huggingface"]), ...(config.dockerMounts || []).map((mount) => ` -v ${mount}`), ` --env "HF_TOKEN={{HF_TOKEN}}"`, ...cellEnv.map((e) => ` --env ${e}`), diff --git a/docs/src/snippets/autoregressive/deepseek-ocr-v2-deployment.jsx b/docs/src/snippets/autoregressive/deepseek-ocr-v2-deployment.jsx index a8bfbec9f..2e7cb0670 100644 --- a/docs/src/snippets/autoregressive/deepseek-ocr-v2-deployment.jsx +++ b/docs/src/snippets/autoregressive/deepseek-ocr-v2-deployment.jsx @@ -10,6 +10,7 @@ export const DeepSeekOCR2Deployment = () => { { id: 'mi325x', label: 'MI325X', default: false }, { id: 'mi355x', label: 'MI355X', default: false }, { id: 'xeon', label: 'XEON', default: false }, + { id: 'arc_b', label: 'BMG', default: false }, ] }, quantization: { @@ -25,8 +26,8 @@ export const DeepSeekOCR2Deployment = () => { type: 'checkbox', items: [ { id: 'tp', label: 'TP', subtitle: 'Tensor Parallel', default: true, required: true }, - { id: 'dp', label: 'DP', subtitle: 'Data Parallel', default: false, disabledWhen: (v) => v.hardware === 'xeon', disabledReason: 'Intel Xeon CPUs only support Tensor Parallel (TP)' }, - { id: 'ep', label: 'EP', subtitle: 'Expert Parallel', default: false, disabledWhen: (v) => v.hardware === 'xeon', disabledReason: 'Intel Xeon CPUs only support Tensor Parallel (TP)' } + { id: 'dp', label: 'DP', subtitle: 'Data Parallel', default: false, disabledWhen: (v) => v.hardware === 'xeon' || v.hardware === 'arc_b', disabledReason: 'Only Tensor Parallel (TP) is supported on this hardware' }, + { id: 'ep', label: 'EP', subtitle: 'Expert Parallel', default: false, disabledWhen: (v) => v.hardware === 'xeon' || v.hardware === 'arc_b', disabledReason: 'Only Tensor Parallel (TP) is supported on this hardware' } ] }, }; @@ -42,6 +43,8 @@ export const DeepSeekOCR2Deployment = () => { cmd += ` --model-path ${modelPath}`; if (hardware === 'xeon') { cmd += ` \\\n --device cpu \\\n --disable-overlap-schedule \\\n --trust-remote-code`; + } else if (hardware === 'arc_b') { + cmd += ` \\\n --device xpu`; } cmd += ` \\\n --enable-multimodal`; @@ -272,9 +275,8 @@ export const DeepSeekOCR2Deployment = () => { ) : option.type === 'checkbox' ? ( (option.items || []).map((item) => { const isChecked = (values[option.name] || []).includes(item.id); - const isDisabled = - item.required || - (typeof item.disabledWhen === 'function' && item.disabledWhen(values)); + const dynDisabled = typeof item.disabledWhen === 'function' && item.disabledWhen(values); + const isDisabled = item.required || dynDisabled; return (
{items.map(item => { const isChecked = values[option.name] === item.id; - const isDisabled = !!item.disabled; + const isArcBModelLocked = + values.hardware === 'arc_b' && + option.name === 'model' && + !['35b', '9b', '4b'].includes(item.id); + const isDisabled = !!item.disabled || isArcBModelLocked; return (
+

+ Run this model from ComfyUI with the{" "} + SGLDiffusion plugin, which ships in the SGLang + repository at {PLUGIN_PATH}. +

+ +

+ Server mode — SGLang runs the pipeline and ComfyUI + sends the request. Start a server as shown above, point the{" "} + SGLDiffusion Server Model node at it, then generate with{" "} + {spec.serverNode}. + {spec.verified + ? " This path has been run end to end against a live server." + : ""} +

+ + {spec.integratedKey ? ( +

+ Integrated mode — ComfyUI's own sampler, CLIP, and + VAE drive the loop while SGLang replaces the model forward. Load the + checkpoint with SGLDiffusion UNET Loader and set{" "} + model_type to {spec.integratedKey} on the{" "} + SGLDiffusion Options node. + {spec.workflow ? ( + <> + {" "} + A reference workflow is included at{" "} + {`${PLUGIN_PATH}/workflows/${spec.workflow}`}. + + ) : null} +

+ ) : ( +

+ Integrated mode — not available for this model + {spec.integratedBlockedBecause + ? `: ${spec.integratedBlockedBecause}` + : ", which has no executor in the plugin"} + . Use server mode. +

+ )} + + {extraNote ?

{extraNote}

: null} +
+ ); +}; diff --git a/docs/src/snippets/diffusion/flux-deployment.jsx b/docs/src/snippets/diffusion/flux-deployment.jsx index 24e4b7e00..2dac7cd37 100644 --- a/docs/src/snippets/diffusion/flux-deployment.jsx +++ b/docs/src/snippets/diffusion/flux-deployment.jsx @@ -15,7 +15,8 @@ export const FluxDeployment = () => { { id: 'mi325x', label: 'MI325X', default: false }, { id: 'mi300x', label: 'MI300X', default: false }, { id: 'a2', label: 'A2 Series', default: false }, - { id: 'a3', label: 'A3 Series', default: false } + { id: 'a3', label: 'A3 Series', default: false }, + { id: 'arc_b', label: 'BMG', default: false }, ] }, version: { @@ -58,6 +59,15 @@ sglang serve \\ --num-gpus 2`; } + if (hardware === 'arc_b') { + return `sglang serve \\ + --model-path ${config.repoId} \\ + --num-gpus 4 \\ + --tp-size 4 \\ + --component-residency dit=resident,text_encoder=layerwise-offload \\ + --dit-cpu-offload False`; + } + return `sglang serve \\ --model-path ${config.repoId} \\ --ulysses-degree=1 \\ @@ -145,7 +155,17 @@ sglang serve \\ }, [values.hardware]); const handleRadioChange = (optionName, value) => { - setValues((prev) => ({ ...prev, [optionName]: value })); + setValues((prev) => { + if (prev.hardware === 'arc_b' && optionName === 'version' && value === 'flux1-dev') { + return prev; + } + + const nextValues = { ...prev, [optionName]: value }; + if (optionName === 'hardware' && value === 'arc_b' && nextValues.version === 'flux1-dev') { + nextValues.version = 'flux2-dev'; + } + return nextValues; + }); }; const handleCheckboxChange = (optionName, itemId, isChecked) => { @@ -322,7 +342,11 @@ sglang serve \\ ) : ( items.map((item) => { const isChecked = values[option.name] === item.id; - const isDisabled = Boolean(item.disabled); + const isArcBVersionLocked = + values.hardware === 'arc_b' && + option.name === 'version' && + item.id === 'flux1-dev'; + const isDisabled = Boolean(item.disabled || isArcBVersionLocked); return (