diff --git a/docs/docs.json b/docs/docs.json index bd7e17e654..e0eae7aad9 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -153,10 +153,6 @@ "source": "/advanced_features/pd_disaggregation.html", "destination": "/docs/advanced_features/pd_disaggregation" }, - { - "source": "/advanced_features/piecewise_cuda_graph.html", - "destination": "/docs/advanced_features/piecewise_cuda_graph" - }, { "source": "/advanced_features/pipeline_parallelism.html", "destination": "/docs/advanced_features/pipeline_parallelism" @@ -970,7 +966,6 @@ "docs/advanced_features/dp_for_multi_modal_encoder", "docs/advanced_features/cuda_graph_for_multi_modal_encoder", "docs/advanced_features/breakable_cuda_graph", - "docs/advanced_features/piecewise_cuda_graph", "docs/advanced_features/sgl_model_gateway", "docs/advanced_features/llm-d", "docs/advanced_features/deterministic_inference", diff --git a/docs/docs/advanced_features/breakable_cuda_graph.mdx b/docs/docs/advanced_features/breakable_cuda_graph.mdx index 546af64844..5bc0589e8e 100644 --- a/docs/docs/advanced_features/breakable_cuda_graph.mdx +++ b/docs/docs/advanced_features/breakable_cuda_graph.mdx @@ -35,7 +35,7 @@ For production use, you can mark specific functions as "non-graphable" using the ```python from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph import eager_on_graph -@eager_on_graph(enable=True) +@eager_on_graph def my_dynamic_op(x): # This op is incompatible with CUDA graph capture return some_dynamic_operation(x) diff --git a/docs/docs/advanced_features/cuda_graph_for_multi_modal_encoder.mdx b/docs/docs/advanced_features/cuda_graph_for_multi_modal_encoder.mdx index b1c1aed289..3476d83e99 100644 --- a/docs/docs/advanced_features/cuda_graph_for_multi_modal_encoder.mdx +++ b/docs/docs/advanced_features/cuda_graph_for_multi_modal_encoder.mdx @@ -69,7 +69,7 @@ it; naming a backend locks the choice and skips that rule: SGLANG_VIT_ENABLE_CUDA_GRAPH=1 \ python3 -m sglang.launch_server \ --model Qwen/Qwen3-VL-8B-Instruct \ - --cuda-graph-backend-prefill tc_piecewise \ + --cuda-graph-backend-prefill breakable \ --cuda-graph-max-bs-prefill 4096 \ --cuda-graph-tc-compiler eager ``` diff --git a/docs/docs/advanced_features/piecewise_cuda_graph.mdx b/docs/docs/advanced_features/piecewise_cuda_graph.mdx deleted file mode 100644 index 4f03665e82..0000000000 --- a/docs/docs/advanced_features/piecewise_cuda_graph.mdx +++ /dev/null @@ -1,289 +0,0 @@ ---- -title: "Piecewise CUDA Graph" -metatags: - description: "Use Piecewise CUDA Graph to reduce prefill and extend kernel launch overhead while supporting dynamic token shapes." ---- - -## Motivation - -Standard CUDA graphs capture the entire model forward pass as a single graph. This works well for decode (fixed batch size), but not for extend/prefill where the number of tokens varies across iterations. - -Piecewise CUDA Graph (PCG) solves this by splitting the model's computation graph into pieces (roughly one per layer) at "split points" (e.g., MoE dispatch ops). Each piece is captured as a separate CUDA graph for a set of pre-defined token lengths. At runtime, the input is padded to the nearest captured size, and each piece is replayed. This eliminates kernel launch overhead for prefill/extend while still supporting dynamic shapes. - -PCG is **enabled by default**. Pass `--cuda-graph-backend-prefill=disabled` to turn it off. - -## Usage - -PCG is enabled by default for supported configurations. No extra flags needed: - -```bash -python3 -m sglang.launch_server \ - --model-path meta-llama/Llama-3.1-8B-Instruct -``` - -### Disable PCG - -```bash -python3 -m sglang.launch_server \ - --model-path meta-llama/Llama-3.1-8B-Instruct \ - --cuda-graph-backend-prefill=disabled -``` - -### Custom capture sizes - -```bash -python3 -m sglang.launch_server \ - --model-path meta-llama/Llama-3.1-8B-Instruct \ - --cuda-graph-max-bs-prefill 2048 -``` - -### Server Args - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
ArgumentDefaultDescription
--cuda-graph-backend-prefillNone (auto)Backend for the prefill phase. Choices: full, breakable, tc_piecewise, disabled. Pass disabled to turn PCG off for extend/prefill, or tc_piecewise to force it on, skipping all auto-disable conditions (testing only).
--cuda-graph-max-bs-prefillNone (auto)Maximum token count to capture. Defaults to chunked_prefill_size (non-MLA) or 2048 (MLA).
--cuda-graph-bs-prefillNone (auto)Explicit list of token lengths to capture. Auto-generated if not set.
--cuda-graph-tc-compiler"eager"Compiler backend for the captured subgraphs. Choices: eager, inductor.
- -## Bug Report - -PCG is enabled by default but is still in an experimental stage. Since PCG relies on `torch.compile` to trace the model's forward pass, most bugs are introduced by torch compile tracing failures (e.g., untraceable ops, dynamic control flow, or graph breaks). If you encounter any issues related to PCG, please disable it by adding `--cuda-graph-backend-prefill=disabled` to your launch command and report the bug at [GitHub Issues](https://github.com/sgl-project/sglang/issues/new/choose). We greatly appreciate your help in improving this feature. - -### For Users - -If you see an error message like the following during server startup, it is a PCG bug: - -``` -Piecewise CUDA Graph is enabled by default as an experimental feature. -To work around this error, add --cuda-graph-backend-prefill=disabled to your launch command. -Please report this issue at https://github.com/sgl-project/sglang/issues/new/choose -``` - -To work around it, add `--cuda-graph-backend-prefill=disabled` to your launch command. When filing a bug report, please include: -1. The full error traceback -2. Model name and quantization method -3. Launch command with all arguments -4. GPU type and driver version - -### For Developers - -Since PCG relies on `torch.compile` to trace the model's forward pass, newly developed CUDA kernels (both JIT kernels and sgl-kernels) are typically not compatible with `torch.compile` out of the box. The tracing will fail on untraceable operations such as JIT compilation, file I/O, or dynamic module loading inside the kernel. - -To make a kernel compatible with PCG, you need to register it as a custom op using `register_custom_op` from `sglang.srt.utils.custom_op`. This wraps the kernel as an opaque node in the compiled graph so that `torch.compile` will not trace inside it. - -**Example usage (JIT kernel):** - -```python -from sglang.srt.utils.custom_op import register_custom_op - -# Inplace operator (no return value) -@register_custom_op(mutates_args=["output_q", "output_s"]) -def per_token_group_quant_8bit( - input: torch.Tensor, - output_q: torch.Tensor, - output_s: torch.Tensor, -) -> None: - # kernel implementation ... -``` - -**Example usage (operator with output):** - -```python -# out_shape indicates which argument has the same shape as the output -@register_custom_op(mutates_args=["x"], out_shape=0) -def add(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor: - return x.add_(y) -``` - -For wrapping external library functions (e.g., FlashInfer kernels), use `register_custom_op_from_extern` instead. See `python/sglang/srt/utils/custom_op.py` for full API documentation. - -## How it works - -### Torch compile backend - -PCG uses `torch.compile` with a custom backend (`SGLangBackend`) to split and compile the model's forward pass. The flow is: - -``` -model.forward wrapper -→ torch.compile(..., backend=SGLangBackend) -→ FX graph -→ split_graph() at registered split ops -→ split_gm (top-level graph that chains the pieces) -→ replace capturable submodules with CUDAPiecewiseBackend -→ runtime dispatch: eager split ops + per-piece capture/replay -``` - -- **Install**: `install_torch_compiled()` replaces `model.forward` with a wrapper function. When `is_in_piecewise_cuda_graph()` returns True, the wrapper dispatches to the compiled callable; otherwise it falls back to the original forward. The first invocation through this path triggers Dynamo tracing and graph compilation — CUDA graph replay only happens after the capture phase completes. - -- **Split**: When `torch.compile` traces the model, `SGLangBackend` receives the FX graph and calls `split_graph()`. Ops listed in `CompilationConfig.split_ops` are treated as split points, so the graph is cut at each one. These split-op submodules are left to run eagerly at runtime, while the surrounding submodules are compiled and wrapped by `CUDAPiecewiseBackend`. The result is a top-level "stitching graph" (`split_gm`) with children such as `submod_0`, `submod_1`, … interleaving capturable subgraphs and eager split-op submodules. - -- **Replace**: `PiecewiseCompileInterpreter` iterates over each capturable submodule in `split_gm`, compiles it for general (dynamic) shapes, and replaces it in-place with a `CUDAPiecewiseBackend` instance. Split-op submodules (e.g., attention, all-reduce) are left as-is and run eagerly at runtime. - -- **Dispatch**: At runtime, calling `split_gm` executes the stitching graph, which calls each submodule in order. Split-op submodules run eagerly. Each `CUDAPiecewiseBackend` submodule goes through three phases: - - **Compile warmup** — runs the general-shape compiled path. - - **Capture** — for each capture size, runs one warmup pass then records a CUDA graph. - - **Steady-state replay** — replays the captured CUDA graph for each forward pass. - -### Piecewise cuda graph runner - -`PiecewiseCudaGraphRunner` orchestrates the full lifecycle through three phases: - -- **Compile** — Warms up JIT kernels with a dummy forward pass, then wraps the model with `torch.compile`, triggering Dynamo tracing to split the FX graph and create `CUDAPiecewiseBackend` instances for each subgraph piece. - -- **Capture** — Iterates over capture sizes in reverse order (largest first). For each size, runs the forward pass twice (one warmup, one CUDA graph capture). - -- **Replay** — At runtime, finds the smallest captured size >= actual token count via binary search, copies inputs into static buffers with zero-padding, replays the captured CUDA graphs, and slices outputs back to the actual token count. - -### Memory optimization - -The memory cost of PCG comes from two parts: **torch memory allocator** and **non-torch memory**. - -The torch memory allocator overhead is trivial thanks to several optimizations: a global shared memory pool is reused across all CUDA graph runners and capture sizes, capture is done in reverse order (large to small) so smaller graphs reuse memory allocated by larger ones, and output tensors of the last subgraph are stored as weak references to maximize memory reuse. - -The main memory overhead comes from non-torch memory — the CUDA graph objects themselves require GPU memory to store the recorded kernel launch parameters and internal state. This overhead scales with the number of captured sizes, which is why `piecewise_cuda_graph_max_tokens` is capped conservatively by default. - -### Shape configuration - -Piecewise CUDA graph pre-captures graphs for a set of token counts. At runtime, the actual token count is rounded up to the nearest captured size (via binary search), and the corresponding graph is replayed. If the token count exceeds the largest captured size, the runtime falls back to the normal (non-graph) forward path. - -The default capture schedule is auto-generated with increasing granularity: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Token rangeStep size
4 – 324
48 – 25616
288 – 51232
576 – 102464
1280 – 4096256
4096+512
- -For the auto-generated schedule, sizes are capped at `--cuda-graph-max-bs-prefill`. The default cap is `chunked_prefill_size` for non-MLA models and `2048` for MLA backend models. If `--max-total-tokens` is set, the cap is further limited to not exceed it. Additionally, Llama-2 models are auto-capped at 4096 tokens as a temporary workaround. - -## Compatibility - -PCG is auto-disabled in the following scenarios. We are actively working on expanding compatibility — support for many of these will be coming soon. - -- Disabled model architectures (e.g., `DeepseekV32ForCausalLM`) -- Speculative decoding -- DP attention -- Pipeline parallelism (`pp_size > 1`) -- Non-CUDA hardware (AMD ROCm, Ascend NPU) -- MoE A2A backend -- LoRA -- Multimodal / VLM models -- DLLM (diffusion LLM) -- Deterministic inference -- PD disaggregation -- Expert distribution recorder / EPLB - -Use `--cuda-graph-backend-prefill=tc_piecewise` to skip all auto-disable checks (for testing/debugging only). - -## Code Reference - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
FileDescription
python/sglang/srt/model_executor/runner_backend/tc_piecewise_cuda_graph_backend.pyBackend implementation: compile, capture, replay
python/sglang/srt/compilation/compile.pyinstall_torch_compiled trampoline
python/sglang/srt/compilation/backend.pySGLangBackend, graph splitting, piecewise compilation
python/sglang/srt/compilation/cuda_piecewise_backend.pyPer-subgraph CUDA graph capture/replay
python/sglang/srt/compilation/piecewise_context_manager.pyGlobal context flags and ForwardContext
python/sglang/srt/compilation/compilation_config.pyCapture sizes, split ops, compiler config
python/sglang/srt/utils/custom_op.pyregister_custom_op for torch.compile compatibility
python/sglang/srt/server_args.pyServer arguments and auto-disable logic
diff --git a/docs/docs/advanced_features/server_arguments.mdx b/docs/docs/advanced_features/server_arguments.mdx index ea5f78d4e7..88f264db5a 100644 --- a/docs/docs/advanced_features/server_arguments.mdx +++ b/docs/docs/advanced_features/server_arguments.mdx @@ -2426,7 +2426,7 @@ Combining `--enable-response-store` with `--disaggregation-mode=prefill` or `dec `--cuda-graph-config` - Canonical per-phase CUDA graph settings as JSON, e.g. {`{"decode":{"backend":"full","max_bs":256},"prefill":{"backend":"tc_piecewise","tc_compiler":"eager"}}`}. JSON wins over the per-phase --cuda-graph-* convenience flags and over the legacy flags. Allowed backends: full, breakable, tc_piecewise, disabled (full is decode-only). + Canonical per-phase CUDA graph settings as JSON, e.g. {`{"decode":{"backend":"full","max_bs":256},"prefill":{"backend":"breakable"}}`}. JSON wins over the per-phase --cuda-graph-* convenience flags and over the legacy flags. Allowed backends: full, breakable, disabled. `None` Type: JSON (dict-of-dicts) @@ -2434,13 +2434,13 @@ Combining `--enable-response-store` with `--disaggregation-mode=prefill` or `dec `--cuda-graph-backend-decode` Backend for the decode phase. Folds into cuda_graph_config[decode].backend. `None` - full, breakable, tc_piecewise, disabled + full, breakable, disabled `--cuda-graph-backend-prefill` Backend for the prefill phase. Folds into cuda_graph_config[prefill].backend. `None` - breakable, tc_piecewise, disabled + breakable, disabled `--cuda-graph-max-bs-decode` @@ -2472,12 +2472,6 @@ Combining `--enable-response-store` with `--disaggregation-mode=prefill` or `dec `None` List[int] - - `--cuda-graph-tc-compiler` - Compiler used by the tc_piecewise backend (only the prefill phase consumes it today). - `None` - eager, inductor - `--disable-cuda-graph-padding` Disable cuda graph when padding is needed. Still uses cuda graph when padding is not needed. diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/mimo_v2_5_pro.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/mimo_v2_5_pro.mdx index 7204747970..d51aa3e808 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/mimo_v2_5_pro.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/mimo_v2_5_pro.mdx @@ -81,7 +81,7 @@ python3 -m sglang.launch_server \ --swa-full-tokens-ratio 0.3 \ --disaggregation-mode prefill --disaggregation-transfer-backend ascend \ --disaggregation-bootstrap-port 8996 \ - --disable-piecewise-cuda-graph \ + --cuda-graph-backend-prefill disabled \ --attn-dp-size 2 --enable-dp-lm-head \ --moe-a2a-backend deepep --deepep-mode normal ``` diff --git a/docs/docs/hardware-platforms/ascend-npus/reference/support_features.mdx b/docs/docs/hardware-platforms/ascend-npus/reference/support_features.mdx index 6180d4d068..0a79e367be 100644 --- a/docs/docs/hardware-platforms/ascend-npus/reference/support_features.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/reference/support_features.mdx @@ -2067,7 +2067,7 @@ If the value is int8, you must also set the environment variable:DEEP_NORMAL_MOD `--cuda-graph-backend-prefill` `None` - `disabled`, `tc_piecewise`
(`tc_piecewise` currently supports Llama-3.1-8B-Instruct and Qwen2.5-7B-Instruct) + `disabled`
(the former `tc_piecewise` prefill backend has been removed) A2/A3 Series diff --git a/docs/docs/hardware-platforms/plugin.mdx b/docs/docs/hardware-platforms/plugin.mdx index b6423b9f25..0727b1b525 100644 --- a/docs/docs/hardware-platforms/plugin.mdx +++ b/docs/docs/hardware-platforms/plugin.mdx @@ -571,11 +571,6 @@ field defaults to the conservative answer. False Runs the decode graph runner (get_graph_runner_cls()). - - piecewise_graph - False - Runs the prefill piecewise compilation backend. - hicache_device_kernels False @@ -625,11 +620,6 @@ field defaults to the conservative answer. return None Return hardware-specific quantization config for the specific quantization scheme, raise an error if not supported or return None to use the default config. - - get_piecewise_backend_cls() - return None - Piecewise compilation backend class - get_compile_backend(mode) "inductor" diff --git a/docs/docs/hardware-platforms/xpu.mdx b/docs/docs/hardware-platforms/xpu.mdx index 336e22a5ac..f34d3ecb4e 100644 --- a/docs/docs/hardware-platforms/xpu.mdx +++ b/docs/docs/hardware-platforms/xpu.mdx @@ -177,7 +177,6 @@ SGLang enables XPU graph capture to reduce per-step kernel-launch overhead. | Phase | Backend | Mechanism | Default | |---|---|---|---| | Decode | `full` | One `torch.xpu.XPUGraph` per batch size, captured on startup | **Off** (opt-in) | -| Prefill | `tc_piecewise` | `torch.compile` + XPU graph, one graph segment per token-length bucket | **Off** (opt-in) | | Prefill | `breakable` | Segmented `torch.xpu.XPUGraph` capture/replay (no `torch.compile`); eager break points at attention / MoE boundaries | **Off** (opt-in) | ### Enable Decode Graph @@ -192,26 +191,7 @@ python -m sglang.launch_server --model-path --device xpu \ ### Enable Prefill Graph Prefill graph capture is **opt-in** on XPU and must be enabled explicitly. -Two backends are available: `tc_piecewise` and `breakable`. - -#### tc_piecewise - -Uses `torch.compile` plus an XPU graph, one graph segment per token-length -bucket: - -```bash -python -m sglang.launch_server --model-path --device xpu \ - --cuda-graph-backend-prefill tc_piecewise -``` - -By default the prefill subgraphs are compiled with `eager` mode. Switch to -`inductor` for higher-quality generated code at the cost of longer startup: - -```bash -python -m sglang.launch_server --model-path --device xpu \ - --cuda-graph-backend-prefill tc_piecewise \ - --cuda-graph-tc-compiler inductor -``` +Use the `breakable` backend. The former `tc_piecewise` backend has been removed. #### breakable @@ -227,7 +207,7 @@ You can also configure both phases together with a single `--cuda-graph-config` ```bash python -m sglang.launch_server --model-path --device xpu \ - --cuda-graph-config '{"decode":{"backend":"full"},"prefill":{"backend":"tc_piecewise","tc_compiler":"eager"}}' + --cuda-graph-config '{"decode":{"backend":"full"},"prefill":{"backend":"breakable"}}' ``` ### Enable torch.compile for Decode @@ -242,11 +222,6 @@ python -m sglang.launch_server --model-path --device xpu \ --enable-torch-compile ``` -> **Note:** `--enable-torch-compile` is mutually exclusive with the prefill -> `tc_piecewise` graph (the compatibility rules auto-disable it). Use them -> separately or lock the prefill backend explicitly via `--cuda-graph-config` -> if you need both. - ### Disable XPU Graph Both phases are disabled by default. To explicitly disable them anyway: @@ -274,7 +249,7 @@ To specify explicit token-length buckets: ```bash python -m sglang.launch_server \ --model-path --device xpu \ - --cuda-graph-backend-prefill tc_piecewise \ + --cuda-graph-backend-prefill breakable \ --cuda-graph-bs-prefill 64 128 256 512 ``` @@ -291,11 +266,10 @@ python -m sglang.launch_server \ | Argument | XPU allowed values | Default | Description | |---|---|---|---| | `--cuda-graph-backend-decode` | `full`, `disabled` | `disabled` | Backend for the decode phase. Only `full` is supported on XPU. Set to `full` to enable. | -| `--cuda-graph-backend-prefill` | `tc_piecewise`, `breakable`, `disabled` | `disabled`* | Backend for the prefill phase. Set to `tc_piecewise` or `breakable` explicitly to enable. | -| `--cuda-graph-tc-compiler` | `eager`, `inductor` | `eager` | Compiler for `tc_piecewise` prefill subgraphs. `inductor` produces more optimized code but has longer startup. | +| `--cuda-graph-backend-prefill` | `breakable`, `disabled` | `disabled`* | Backend for the prefill phase. Set to `breakable` explicitly to enable. | | `--cuda-graph-bs-prefill` | list of ints | auto | Explicit token-length buckets to capture for prefill. | | `--cuda-graph-bs-decode` | list of ints | auto | Explicit batch sizes to capture for decode. | -| `--cuda-graph-config` | JSON string | — | One-shot JSON config for both phases, e.g. `'{"decode":{"backend":"full"},"prefill":{"backend":"tc_piecewise","tc_compiler":"eager"}}'`. Overrides all per-phase flags. | +| `--cuda-graph-config` | JSON string | — | One-shot JSON config for both phases, e.g. `'{"decode":{"backend":"full"},"prefill":{"backend":"breakable"}}'`. Overrides all per-phase flags. | | `--disable-decode-cuda-graph` | — | `False` | Shorthand for `--cuda-graph-backend-decode=disabled`. | | `--disable-prefill-cuda-graph` | — | `False` | Shorthand for `--cuda-graph-backend-prefill=disabled`. | | `--enable-torch-compile` | — | `False` | Apply `torch.compile` on top of the decode XPU graph for further kernel optimization. | diff --git a/python/sglang/srt/arg_groups/cuda_graph_hook.py b/python/sglang/srt/arg_groups/cuda_graph_hook.py index 5dc1868f33..2c325b6c3d 100644 --- a/python/sglang/srt/arg_groups/cuda_graph_hook.py +++ b/python/sglang/srt/arg_groups/cuda_graph_hook.py @@ -22,14 +22,9 @@ from sglang.srt.model_executor.cuda_graph_config import ( default_cuda_graph_config, with_phase, ) -from sglang.srt.platforms import current_platform -from sglang.srt.runtime_context import attn_dp_enabled_of, get_platform from sglang.srt.utils.common import ( - is_cpu, - is_mps, parse_connector_type, ) -from sglang.srt.utils.hf_transformers_utils import check_gguf_file logger = logging.getLogger(__name__) @@ -84,11 +79,6 @@ def parse_cuda_graph_config(server_args: Any): _set(Phase.DECODE, "bs", cfg.cuda_graph_bs_decode) if cfg.cuda_graph_bs_prefill is not None: _set(Phase.PREFILL, "bs", cfg.cuda_graph_bs_prefill) - if cfg.cuda_graph_tc_compiler is not None: - # Written to both phases so the value is in place when TC_PIECEWISE - # decode is implemented; today decode ignores it. - _set(Phase.DECODE, "tc_compiler", cfg.cuda_graph_tc_compiler) - _set(Phase.PREFILL, "tc_compiler", cfg.cuda_graph_tc_compiler) if cfg.cuda_graph_prefill_max_context is not None: _set( Phase.PREFILL, @@ -113,10 +103,8 @@ def parse_cuda_graph_config(server_args: Any): def apply_cuda_graph_compatibility(server_args: Any): """Auto-disable prefill cuda graph for incompatible configs. - Rules are split per backend — TcPiecewise and Breakable have - different constraints. Skipped when the user explicitly set the - prefill backend, whichever value they chose (the contract the removed - --enforce-piecewise-cuda-graph used to spell). + Breakable and Full have different constraints. Skip automatic selection + when the user explicitly sets the prefill backend. """ cfg = resolving_view(server_args) @@ -141,137 +129,12 @@ def apply_cuda_graph_compatibility(server_args: Any): ) return - # Breakable is the CUDA default but not multimodal-compatible; - # piecewise-allowlisted archs run their validated decoder prefill - # there instead. Archs also on the breakable allowlist keep it -- - # this runs first, so piecewise would otherwise silently win. - if ( - cfg.cuda_graph_config.prefill.backend == Backend.BREAKABLE - and model_config_of(server_args).is_multimodal_piecewise_cuda_graph_supported - and not model_config_of( - server_args - ).is_multimodal_breakable_cuda_graph_supported - # Keep trtllm_mla on the preferred breakable path, which now serves - # MLA by falling back to the flashinfer MLA impl for extend. - and attention_backends_of(resolved_view(server_args))[0] != "trtllm_mla" - ): - logger.info( - "Using tc_piecewise CUDA graph for validated multimodal decoder prefill." - ) - declare_resolution( - server_args, - "_apply_cuda_graph_compatibility", - cuda_graph_config=with_phase( - cfg.cuda_graph_config, Phase.PREFILL, backend=Backend.TC_PIECEWISE - ), - ) - - if cfg.cuda_graph_config.prefill.backend == Backend.TC_PIECEWISE: - disable_tc_piecewise_cudagraph_if_incompatible(server_args) - elif cfg.cuda_graph_config.prefill.backend == Backend.BREAKABLE: + if cfg.cuda_graph_config.prefill.backend == Backend.BREAKABLE: disable_breakable_cudagraph_if_incompatible(server_args) elif cfg.cuda_graph_config.prefill.backend == Backend.FULL: disable_full_prefill_cudagraph_if_incompatible(server_args) -def disable_tc_piecewise_cudagraph_if_incompatible(server_args: Any): - """TcPiecewise (torch.compile + piecewise) is incompatible with - these configurations. Most are torch.compile / dynamo limitations. - """ - - cfg = resolving_view(server_args) - - rules = [ - ( - "model-arch blacklist", - lambda: model_config_of(server_args).is_piecewise_cuda_graph_disabled_model, - ), - ("DP attention", lambda: attn_dp_enabled_of(resolved_view(server_args))), - ("full torch.compile mode", lambda: cfg.enable_torch_compile), - ("pipeline parallelism (pp_size > 1)", lambda: cfg.pp_size > 1), - ( - "non-CUDA hardware (HIP/NPU/CPU/MPS/XPU)", - lambda: ( - get_platform().is_hip - or get_platform().is_npu - or is_cpu() - or is_mps() - or get_platform().is_xpu - ), - ), - ( - "OOT platform without piecewise support", - lambda: ( - current_platform.is_out_of_tree() - and not current_platform.capabilities.piecewise_graph - ), - ), - ( - "MoE A2A backend", - lambda: resolved_view(server_args).moe_a2a_backend != "none", - ), - # Dynamo blocks LoRA under tc_piecewise (per-batch LoRABatchInfo - # rebinds break guards); breakable/full support LoRA. - ("LoRA", lambda: bool(cfg.lora_paths) or cfg.enable_lora), - ( - "multimodal model", - lambda: ( - model_config_of(server_args).is_multimodal - and not model_config_of( - server_args - ).is_multimodal_piecewise_cuda_graph_supported - ), - ), - ( - "GGUF quantization", - lambda: ( - cfg.load_format == "gguf" - or resolved_view(server_args).quantization == "gguf" - or check_gguf_file(cfg.model_path) - ), - ), - ("DLLM (diffusion LLM)", lambda: cfg.dllm_algorithm is not None), - ( - "CPU offload / hierarchical cache", - lambda: cfg.cpu_offload_gb > 0 or cfg.enable_hierarchical_cache, - ), - ( - "deterministic inference", - lambda: cfg.enable_deterministic_inference, - ), - ("PD disaggregation", lambda: cfg.disaggregation_mode != "null"), - ("symmetric memory", lambda: cfg.enable_symm_mem), - ( - "expert distribution recorder", - lambda: ( - cfg.enable_eplb or cfg.expert_distribution_recorder_mode is not None - ), - ), - ( - "context parallel (attn_cp_size > 1)", - lambda: resolved_view(server_args).attn_cp_size > 1, - ), - ("CUDA graph debug mode", lambda: cfg.debug_cuda_graph), - # Capture builds a dummy extend forward with attn_dcp_metadata=None. - ( - "decode context parallel (dcp_size > 1)", - lambda: cfg.dcp_size > 1, - ), - ] - for _name, predicate in rules: - if predicate(): - declare_resolution( - server_args, - "_disable_tc_piecewise_cudagraph_if_incompatible", - cuda_graph_config=with_phase( - cfg.cuda_graph_config, Phase.PREFILL, backend=Backend.DISABLED - ), - ) - # One decision, one declaration: every rule declares the same - # value, so a later match would only append a duplicate entry. - break - - def disable_breakable_cudagraph_if_incompatible(server_args: Any): """Breakable (segmented capture, no torch.compile). Breakable enforces memory-saver rejection in its own __init__; config-time rules can be @@ -377,8 +240,7 @@ def disable_full_prefill_cudagraph_if_incompatible(server_args: Any): def disable_prefill_cuda_graph_for_deepseek_trtllm_mla(server_args: Any): """Disable prefill CUDA graph for dsr1 by default when using the trtllm_mla - attention backend. Under any captured prefill CUDA graph (tc_piecewise or - breakable) trtllm_mla falls back to FlashAttention for prefill and regresses + attention backend. Under any captured prefill CUDA graph trtllm_mla falls back to FlashAttention for prefill and regresses performance, so disable whichever prefill graph backend is in effect. """ @@ -400,7 +262,7 @@ def disable_prefill_cuda_graph_for_deepseek_trtllm_mla(server_args: Any): "Disabling prefill CUDA graph (%s) by default for the DeepSeek-V3 arch on " "the trtllm_mla attention backend (a captured prefill graph forces a " "FlashAttention fallback that regresses prefill). Set the prefill cuda graph " - "backend explicitly (e.g. --cuda-graph-backend-prefill tc_piecewise) to override.", + "backend explicitly (e.g. --cuda-graph-backend-prefill breakable) to override.", cfg.cuda_graph_config.prefill.backend, ) declare_resolution( @@ -503,7 +365,7 @@ def handle_cuda_graph_config(server_args: Any): if cfg.cuda_graph_config.prefill.backend == Backend.FULL: logger.warning( "cuda_graph_config[prefill].backend='full' is experimental. " - "Use breakable or tc_piecewise for production workloads." + "Use breakable for production workloads." ) @@ -580,7 +442,7 @@ def finalize_cuda_graph_prefill_max_context(server_args: Any) -> None: def generate_prefill_cuda_graph_batch_sizes(max_bs: int): """ Generate the list of batch sizes for prefill CUDA graph capture - based on max_bs. For tc_piecewise prefill, bs carries the + based on max_bs. For prefill, bs carries the captured token count (one shape knob per phase). """ capture_sizes = ( diff --git a/python/sglang/srt/arg_groups/field_order.py b/python/sglang/srt/arg_groups/field_order.py index cd77b9ebb3..1611e714b7 100644 --- a/python/sglang/srt/arg_groups/field_order.py +++ b/python/sglang/srt/arg_groups/field_order.py @@ -227,7 +227,6 @@ POSITIONAL_FIELD_ORDER = ( "cuda_graph_bs_decode", "cuda_graph_bs_prefill", "cuda_graph_prefill_max_context", - "cuda_graph_tc_compiler", "disable_prefill_cuda_graph", "disable_decode_cuda_graph", "disable_cuda_graph", diff --git a/python/sglang/srt/arg_groups/fields/exec_.py b/python/sglang/srt/arg_groups/fields/exec_.py index 7bfcaf0f2a..e8a0e52093 100644 --- a/python/sglang/srt/arg_groups/fields/exec_.py +++ b/python/sglang/srt/arg_groups/fields/exec_.py @@ -28,6 +28,7 @@ from sglang.srt.arg_groups.choices import ( from sglang.srt.model_executor.cuda_graph_config import ( Backend, CudaGraphConfig, + parse_cuda_graph_backend_arg, parse_cuda_graph_config_arg, ) from sglang.srt.utils.common import human_readable_int @@ -505,22 +506,24 @@ class ExecGraph(msgspec.Struct): cuda_graph_config: A[ Optional[CudaGraphConfig], Arg( - help='Per-phase CUDA graph settings as JSON, e.g. \'{"decode":{"backend":"full","max_bs":256},"prefill":{"backend":"tc_piecewise","tc_compiler":"eager"}}\'. Allowed backends per phase: full, breakable, tc_piecewise, disabled (full is decode-only). JSON wins over the per-phase --cuda-graph-* convenience flags and over legacy flags.', + help='Per-phase CUDA graph settings as JSON, e.g. \'{"decode":{"backend":"full","max_bs":256},"prefill":{"backend":"breakable"}}\'. Allowed backends per phase: full, breakable, disabled. JSON wins over the per-phase --cuda-graph-* convenience flags and over legacy flags.', type_parser=parse_cuda_graph_config_arg, ), ] = None cuda_graph_backend_decode: A[ - Optional[Literal["full", "breakable", "tc_piecewise", "disabled"]], + Optional[Literal["full", "breakable", "disabled"]], Arg( help="Backend for the decode phase. Folds into cuda_graph_config[decode].backend.", choices=Backend.ALL, + type_parser=parse_cuda_graph_backend_arg, ), ] = None cuda_graph_backend_prefill: A[ - Optional[Literal["full", "breakable", "tc_piecewise", "disabled"]], + Optional[Literal["full", "breakable", "disabled"]], Arg( help="Backend for the prefill phase. Folds into cuda_graph_config[prefill].backend.", choices=Backend.ALL, + type_parser=parse_cuda_graph_backend_arg, ), ] = None cuda_graph_max_bs_decode: A[ @@ -556,10 +559,6 @@ class ExecGraph(msgspec.Struct): aliases=["--context-bucket"], ), ] = None - cuda_graph_tc_compiler: A[ - Optional[Literal["eager", "inductor"]], - "Compiler used by the tc_piecewise backend (currently only the prefill phase consumes it).", - ] = None disable_prefill_cuda_graph: A[ bool, "Disable the prefill-phase CUDA graph. Convenience for --cuda-graph-backend-prefill=disabled.", diff --git a/python/sglang/srt/arg_groups/platform_hook.py b/python/sglang/srt/arg_groups/platform_hook.py index 7fdffb6c2a..3b51e9268b 100644 --- a/python/sglang/srt/arg_groups/platform_hook.py +++ b/python/sglang/srt/arg_groups/platform_hook.py @@ -37,20 +37,6 @@ def handle_npu_backends(server_args: Any): set_default_server_args(server_args) - current = cfg.cuda_graph_config.prefill.tc_compiler - if current is not None and current != "eager": - logger.warning( - "At this moment Ascend platform only support prefill graph compilation with " - "cuda_graph_config[prefill].tc_compiler='eager'." - ) - declare_resolution( - server_args, - "_handle_npu_backends", - cuda_graph_config=with_phase( - cfg.cuda_graph_config, Phase.PREFILL, tc_compiler="eager" - ), - ) - def handle_mps_backends(server_args: Any): cfg = resolving_view(server_args) diff --git a/python/sglang/srt/model_executor/cuda_graph_config.py b/python/sglang/srt/model_executor/cuda_graph_config.py index 760659ede2..ed481b3c70 100644 --- a/python/sglang/srt/model_executor/cuda_graph_config.py +++ b/python/sglang/srt/model_executor/cuda_graph_config.py @@ -40,18 +40,18 @@ class Phase: class Backend: """CUDA graph capture backends a phase can use.""" + # Temporary internal name while unreachable legacy branches are removed. + TC_PIECEWISE = "tc_piecewise" FULL = "full" BREAKABLE = "breakable" - TC_PIECEWISE = "tc_piecewise" DISABLED = "disabled" - ALL = (FULL, BREAKABLE, TC_PIECEWISE, DISABLED) + ALL = (FULL, BREAKABLE, DISABLED) ALLOWED_BACKENDS_PER_PHASE = { Phase.DECODE: ( Backend.FULL, Backend.BREAKABLE, - Backend.TC_PIECEWISE, Backend.DISABLED, ), # full for prefill captures one whole-forward graph per num_tokens @@ -61,26 +61,23 @@ ALLOWED_BACKENDS_PER_PHASE = { Phase.PREFILL: ( Backend.FULL, Backend.BREAKABLE, - Backend.TC_PIECEWISE, Backend.DISABLED, ), } # Per-phase settings schema. Keys other than backend are runner-level -# (read by any backend in that phase); tc_compiler is the lone -# backend-specific knob (only meaningful when backend == tc_piecewise). +# (read by any backend in that phase). # For prefill, bs carries aggregate-token capture buckets for every backend; # full_prefill_max_req separately controls Full's fixed request-slot count. # full_prefill_max_req and full_prefill_prefix_chunk_tokens are prefill-only and # only meaningful when backend == full. max_context_size is shared by the # breakable and full prefill body-capture backends. ALLOWED_KEYS_PER_PHASE = { - Phase.DECODE: ("backend", "max_bs", "bs", "tc_compiler"), + Phase.DECODE: ("backend", "max_bs", "bs"), Phase.PREFILL: ( "backend", "max_bs", "bs", - "tc_compiler", "max_context_size", "full_prefill_max_req", "full_prefill_prefix_chunk_tokens", @@ -96,7 +93,7 @@ class PhaseConfig: backend: str = Backend.DISABLED max_bs: Optional[int] = None bs: Optional[List[int]] = None - # Only meaningful when backend == tc_piecewise; ignored otherwise. + # Internal compatibility only; rejected by both configuration parsers. tc_compiler: str = "eager" # Effective for both full and breakable backends and currently only DSV4: # fixed maximum context length used by context-shaped prefill graph metadata. @@ -105,7 +102,7 @@ class PhaseConfig: # Only meaningful for the prefill phase with backend == full: max number of # request slots baked into each captured graph. Real bs <= full_prefill_max_req # reuses the graph (unused slots become zero-length sentinels); larger - # batches fall back to eager. Ignored by BCG and TC_PIECEWISE. None + # batches fall back to eager. Ignored by BCG. None # auto-derives chunked_prefill_size // 512. full_prefill_max_req: Optional[int] = None # Only meaningful for Full prefill CUDA graphs that capture a distinct @@ -120,15 +117,10 @@ class PhaseConfig: def default_prefill_backend() -> str: - """BCG (breakable) is the prefill default on CUDA only; other platforms - (HIP/NPU/...) keep tc_piecewise until BCG is validated there. Full-graph - prefill capture is opt-in per model architecture via the declarative - registry (see _inkling_overrides in arg_groups/overrides.py), not a global - default. Lazy import keeps this module's stdlib-only import invariant (see - module docstring).""" + """Enable BCG by default on CUDA; other platforms remain opt-in.""" from sglang.srt.utils import is_cuda - return Backend.BREAKABLE if is_cuda() else Backend.TC_PIECEWISE + return Backend.BREAKABLE if is_cuda() else Backend.DISABLED def with_phase(config: "CudaGraphConfig", phase: str, **changes) -> "CudaGraphConfig": @@ -188,6 +180,12 @@ class CudaGraphConfig: if phase not in Phase.ALL or not isinstance(phase_settings, dict): continue phase_cfg = getattr(cfg, phase) + if phase_settings.get("backend") == "tc_piecewise": + raise ValueError( + "tc_piecewise was removed; select breakable, full, or disabled" + ) + if "tc_compiler" in phase_settings: + raise ValueError("tc_compiler was removed with tc_piecewise") allowed = ALLOWED_KEYS_PER_PHASE[phase] for key, value in phase_settings.items(): if key in allowed: @@ -236,6 +234,15 @@ def cuda_graph_fully_disabled() -> bool: ) and check_cuda_graph_backend(Phase.PREFILL, Backend.DISABLED) +def parse_cuda_graph_backend_arg(raw: str) -> str: + """Reject the retired backend before argparse checks supported choices.""" + if raw == "tc_piecewise": + raise argparse.ArgumentTypeError( + "tc_piecewise was removed; select breakable, full, or disabled" + ) + return raw + + def parse_cuda_graph_config_arg(raw: str) -> Dict[str, Dict[str, Any]]: """argparse type for --cuda-graph-config: parse JSON dict of phase → settings dict. Each phase's settings dict is itself validated @@ -263,6 +270,8 @@ def parse_cuda_graph_config_arg(raw: str) -> Dict[str, Dict[str, Any]]: f"--cuda-graph-config['{phase}'] must be a JSON object, got " f"{type(phase_settings).__name__}" ) + if "backend" in phase_settings: + parse_cuda_graph_backend_arg(phase_settings["backend"]) allowed = ALLOWED_KEYS_PER_PHASE[phase] result[phase] = {} for key, value in phase_settings.items(): diff --git a/python/sglang/srt/model_executor/runner/eager_runner.py b/python/sglang/srt/model_executor/runner/eager_runner.py index 5115e4c90c..7b406aafd5 100644 --- a/python/sglang/srt/model_executor/runner/eager_runner.py +++ b/python/sglang/srt/model_executor/runner/eager_runner.py @@ -50,10 +50,6 @@ from sglang.srt.model_executor.forward_context import ( get_token_to_kv_pool, ) from sglang.srt.model_executor.runner.base_runner import BaseRunner -from sglang.srt.model_executor.runner_backend_utils.tc_piecewise_cuda_graph import ( - enable_tc_piecewise_cuda_graph, - set_tc_piecewise_forward_context, -) from sglang.srt.model_executor.runner_utils import ( maybe_publish_prefill_shared_read_done, ) @@ -343,34 +339,7 @@ class EagerRunner(BaseRunner): else "extend" ) with device_timer_ctx(model_runner.device_timer, category): - pcg_runner = model_runner.prefill_cuda_graph_runner - if ( - _is_hip - and pcg_runner is not None - and not isinstance(pcg_runner, EagerRunner) - and not cp_active - ): - # HIP PCG eager fallback: enter the PCG context so Dynamo guards - # and PCG-specific MoE/attention paths stay consistent. - with ( - enable_tc_piecewise_cuda_graph(), - set_tc_piecewise_forward_context( - forward_batch, - model_runner.attention_layers, - getattr(model_runner.model, "quant_config", None), - model_runner.moe_layers, - model_runner.moe_fusions, - dsa_indexers=model_runner.dsa_indexers, - mha_companion_layers=model_runner.mha_companion_layers, - ), - ): - ret = model_runner.model.forward( - forward_batch.input_ids, - forward_batch.positions, - forward_batch, - **kwargs, - ) - elif cp_active and ( + if cp_active and ( not get_parallel().enable_cp_tp_group_sharing or getattr(model_runner.model, "prepare_cp_inputs", None) is not None ): diff --git a/python/sglang/srt/model_executor/runner_backend/utils.py b/python/sglang/srt/model_executor/runner_backend/utils.py index cb9449f417..9857fe3d96 100644 --- a/python/sglang/srt/model_executor/runner_backend/utils.py +++ b/python/sglang/srt/model_executor/runner_backend/utils.py @@ -34,9 +34,6 @@ from sglang.srt.model_executor.runner_backend.breakable_cuda_graph_backend impor from sglang.srt.model_executor.runner_backend.full_cuda_graph_backend import ( FullCudaGraphBackend, ) -from sglang.srt.model_executor.runner_backend.tc_piecewise_cuda_graph_backend import ( - TcPiecewiseCudaGraphBackend, -) from sglang.srt.platforms import current_platform from sglang.srt.runtime_context import get_exec @@ -47,9 +44,6 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -# Track first occurrence of each fallback warning to avoid log spam. -_TC_PIECEWISE_DECODE_FALLBACK_LOGGED = False - def resolve_decode_backend( cuda_graph_runner: BaseCudaGraphRunner, @@ -90,14 +84,12 @@ def resolve_decode_backend( enable_memory_saver=enable_memory_saver, debug_eager=get_exec().graph.debug_cuda_graph, ) - if backend_name == Backend.TC_PIECEWISE: - global _TC_PIECEWISE_DECODE_FALLBACK_LOGGED - if not _TC_PIECEWISE_DECODE_FALLBACK_LOGGED: - logger.warning( - "cuda_graph_config decode='tc_piecewise' is not yet implemented; " - "falling back to 'full'." - ) - _TC_PIECEWISE_DECODE_FALLBACK_LOGGED = True + + # Callers decide whether to construct a graph runner. Explicit construction + # with a disabled config (e.g. the draft-runner test harness) historically + # uses the full backend; retiring TCPCG must preserve that behavior. + if backend_name not in (Backend.FULL, Backend.DISABLED): + raise ValueError(f"Unsupported decode graph backend: {backend_name}") full_backend_cls = None if current_platform.is_out_of_tree(): @@ -116,9 +108,8 @@ def resolve_prefill_backend( cuda_graph_runner: BaseCudaGraphRunner, ) -> BaseCudaGraphBackend: """Pick a backend instance from cuda_graph_config['prefill']['backend'].""" - model_runner = cuda_graph_runner.model_runner cfg = get_exec().graph.cuda_graph_config - backend_name = cfg.prefill.backend if cfg is not None else Backend.TC_PIECEWISE + backend_name = cfg.prefill.backend if cfg is not None else Backend.BREAKABLE if backend_name == Backend.BREAKABLE: return BreakableCudaGraphBackend( @@ -132,5 +123,4 @@ def resolve_prefill_backend( enable_memory_saver=get_exec().features.enable_memory_saver, reuse_output_buffer=True, ) - # Default: tc_piecewise. - return TcPiecewiseCudaGraphBackend(cuda_graph_runner) + raise ValueError(f"Unsupported prefill graph backend: {backend_name}") diff --git a/python/sglang/srt/model_executor/runner_backend_utils/breakable_cuda_graph/context.py b/python/sglang/srt/model_executor/runner_backend_utils/breakable_cuda_graph/context.py index 16f80ee445..ffafbed344 100644 --- a/python/sglang/srt/model_executor/runner_backend_utils/breakable_cuda_graph/context.py +++ b/python/sglang/srt/model_executor/runner_backend_utils/breakable_cuda_graph/context.py @@ -52,9 +52,8 @@ def enable_breakable_cuda_graph(): BCG_FAILURE_HINT = ( - "1. change to tc_piecewise by --cuda-graph-backend-prefill=tc_piecewise\n" - "2. disable the prefill CUDA graph by --cuda-graph-backend-prefill=disabled\n" - "3. if it is an OOM problem, set --mem-fraction-static to a smaller value " + "1. disable the prefill CUDA graph by --cuda-graph-backend-prefill=disabled\n" + "2. if it is an OOM problem, set --mem-fraction-static to a smaller value " "(e.g., 0.8 or 0.7) or set --cuda-graph-max-bs-prefill to a smaller value " "(e.g., 2048)\n" ) diff --git a/test/manual/piecewise_cuda_graph/test_piecewise_cuda_graph_support_1_gpu_archived.py b/test/manual/piecewise_cuda_graph/test_piecewise_cuda_graph_support_1_gpu_archived.py deleted file mode 100644 index 66e8ae784d..0000000000 --- a/test/manual/piecewise_cuda_graph/test_piecewise_cuda_graph_support_1_gpu_archived.py +++ /dev/null @@ -1,63 +0,0 @@ -"""Archived test classes split out of test/registered/piecewise_cuda_graph/test_piecewise_cuda_graph_support_1_gpu.py. - -Originally registered with `register_cuda_ci(...)`. Moved here as part of -the per-commit pruning effort to keep the code reachable manually. -Run with `python3 test/manual/piecewise_cuda_graph/test_piecewise_cuda_graph_support_1_gpu_archived.py`. -""" - -import unittest - -from sglang.srt.utils import kill_process_tree -from sglang.test.run_eval import run_eval -from sglang.test.test_utils import ( - DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - DEFAULT_URL_FOR_TEST, - CustomTestCase, - SimpleNamespace, - popen_launch_server, -) - - -# CI Registration -class TestPiecewiseCudaGraphInternVL25(CustomTestCase): - """Test piecewise CUDA graph with InternVL2.5-8B model""" - - @classmethod - def setUpClass(cls): - cls.model = "OpenGVLab/InternVL2_5-8B" - cls.base_url = DEFAULT_URL_FOR_TEST - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=[ - "--cuda-graph-backend-prefill=tc_piecewise", - "--disable-radix-cache", - ], - ) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - def test_gsm8k_accuracy(self): - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="gsm8k", - num_examples=None, - num_threads=1024, - ) - - metrics = run_eval(args) - print(f"GSM8K Accuracy: {metrics['score']:.3f}") - - # Baseline (no piecewise CUDA graph): 0.571 — this eval uses 5-shot - # concatenated text via chat API, which scores lower than reported - # benchmarks (~77.8%) that use proper CoT chat format. The threshold - # is set 5% below observed to catch catastrophic regressions. - self.assertGreaterEqual(metrics["score"], 0.54) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/manual/piecewise_cudagraph/test_disaggregation_piecewise_cuda_graph.py b/test/manual/piecewise_cudagraph/test_disaggregation_piecewise_cuda_graph.py deleted file mode 100644 index 9b7b2ca245..0000000000 --- a/test/manual/piecewise_cudagraph/test_disaggregation_piecewise_cuda_graph.py +++ /dev/null @@ -1,87 +0,0 @@ -import unittest -from types import SimpleNamespace - -from sglang.test.server_fixtures.disaggregation_fixture import ( - PDDisaggregationServerBase, -) -from sglang.test.sgl_eval_utils import run_sgl_eval -from sglang.test.test_utils import ( - DEFAULT_MODEL_NAME_FOR_TEST, - DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - popen_launch_pd_server, -) - - -class TestDisaggregationPiecewiseCudaGraph(PDDisaggregationServerBase): - """Test piecewise CUDA graph support in disaggregation prefill server""" - - @classmethod - def setUpClass(cls): - super().setUpClass() - cls.model = DEFAULT_MODEL_NAME_FOR_TEST - - # Start servers - cls.start_prefill() - cls.start_decode() - - # Wait for both to be ready - cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill) - cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode) - - cls.launch_lb() - - @classmethod - def start_prefill(cls): - prefill_args = [ - "--trust-remote-code", - "--disaggregation-mode", - "prefill", - "--tp", - "1", - "--cuda-graph-backend-prefill=tc_piecewise", - ] - prefill_args += cls.transfer_backend + cls.rdma_devices - cls.process_prefill = popen_launch_pd_server( - cls.model, - cls.prefill_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=prefill_args, - ) - - @classmethod - def start_decode(cls): - decode_args = [ - "--trust-remote-code", - "--disaggregation-mode", - "decode", - "--tp", - "1", - "--base-gpu-id", - "1", - ] - decode_args += cls.transfer_backend + cls.rdma_devices - cls.process_decode = popen_launch_pd_server( - cls.model, - cls.decode_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=decode_args, - ) - - def test_gsm8k_accuracy(self): - """Verify that piecewise cuda graph works correctly in prefill server""" - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="gsm8k", - max_tokens=512, - num_examples=200, - num_threads=128, - ) - metrics = run_sgl_eval(args) - print(f"GSM8K accuracy with piecewise cuda graph: {metrics['score']:.3f}") - - self.assertGreater(metrics["score"], 0.62) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/amd/test_deepseek_r1_mxfp4_8gpu.py b/test/registered/amd/test_deepseek_r1_mxfp4_8gpu.py deleted file mode 100644 index 7a4532be9a..0000000000 --- a/test/registered/amd/test_deepseek_r1_mxfp4_8gpu.py +++ /dev/null @@ -1,182 +0,0 @@ -import unittest -from types import SimpleNamespace - -import requests - -from sglang.srt.environ import envs -from sglang.srt.utils import kill_process_tree -from sglang.test.ci.ci_register import register_amd_ci -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k -from sglang.test.send_one import BenchArgs, send_one_prompt -from sglang.test.test_utils import ( - DEFAULT_URL_FOR_TEST, - CustomTestCase, - is_in_ci, - popen_launch_server, - write_github_step_summary, -) - -register_amd_ci(est_time=1300, suite="stage-c-test-large-8-gpu-amd-mi35x") - -DEEPSEEK_R1_MODEL_PATH = "amd/DeepSeek-R1-MXFP4-Preview" -SERVER_LAUNCH_TIMEOUT = 1800 - - -class TestDeepseekR1MXFP4(CustomTestCase): - @classmethod - def setUpClass(cls): - cls.model = DEEPSEEK_R1_MODEL_PATH - cls.base_url = DEFAULT_URL_FOR_TEST - - # Workaround: AITER custom all-gather corrupts CUDA-graph IPC buffer - # registration and triggers a decode-time "Memory access fault" on - # MI35x TP=8. Disable until the AITER-side fix lands (see PR body). - envs.SGLANG_USE_AITER_AG.set(False) - - other_args = [ - "--tp", - "8", - "--chunked-prefill-size", - "131072", - "--model-loader-extra-config", - '{"enable_multithread_load": true}', - "--cuda-graph-backend-prefill=tc_piecewise", - "--cuda-graph-tc-compiler", - "eager", - "--cuda-graph-max-bs-prefill", - "8192", - ] - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=SERVER_LAUNCH_TIMEOUT, - other_args=other_args, - ) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - def test_a_gsm8k( - self, - ): # Append an "a" to make this test run first (alphabetically) to warm up the server - requests.get(self.base_url + "/flush_cache") - - args = SimpleNamespace( - num_shots=8, - data_path=None, - num_questions=1319, - parallel=1319, - max_new_tokens=512, - host="127.0.0.1", - port=int(self.base_url.split(":")[-1]), - ) - metrics = run_eval_few_shot_gsm8k(args) - print(f"{metrics=}") - - if is_in_ci(): - write_github_step_summary( - f'### test_gsm8k (deepseek-r1-mxfp4)\n{metrics["accuracy"]=:.3f}\n' - ) - self.assertGreater(metrics["accuracy"], 0.94) - - def test_bs_1_speed(self): - args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048) - _, speed = send_one_prompt(args) - - print(f"{speed=:.2f}") - - if is_in_ci(): - write_github_step_summary( - f"### test_bs_1_speed (deepseek-r1-mxfp4)\n{speed=:.2f} token/s\n" - ) - self.assertGreater(speed, 75) - - -class TestDeepseekR1MXFP4MTP(CustomTestCase): - @classmethod - def setUpClass(cls): - cls.model = DEEPSEEK_R1_MODEL_PATH - cls.base_url = DEFAULT_URL_FOR_TEST - - envs.SGLANG_ENABLE_OVERLAP_PLAN_STREAM.set(True) - # Same AITER custom all-gather workaround as TestDeepseekR1MXFP4 above. - envs.SGLANG_USE_AITER_AG.set(False) - - other_args = [ - "--tp", - "8", - "--chunked-prefill-size", - "131072", - "--speculative-algorithm", - "EAGLE", - "--speculative-num-steps", - "3", - "--speculative-eagle-topk", - "1", - "--speculative-num-draft-tokens", - "4", - "--model-loader-extra-config", - '{"enable_multithread_load": true}', - ] - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=SERVER_LAUNCH_TIMEOUT, - other_args=other_args, - ) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - def test_a_gsm8k( - self, - ): # Append an "a" to make this test run first (alphabetically) to warm up the server - requests.get(self.base_url + "/flush_cache") - - args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="127.0.0.1", - port=int(self.base_url.split(":")[-1]), - ) - metrics = run_eval_few_shot_gsm8k(args) - print(f"{metrics=}") - - server_info = requests.get(self.base_url + "/server_info") - avg_spec_accept_length = server_info.json()["internal_states"][0][ - "avg_spec_accept_length" - ] - print(f"{avg_spec_accept_length=}") - - if is_in_ci(): - write_github_step_summary( - f"### test_gsm8k (deepseek-r1-mxfp4 mtp)\n" - f'{metrics["accuracy"]=:.3f}\n' - f"{avg_spec_accept_length=:.2f}\n" - ) - self.assertGreater(metrics["accuracy"], 0.94) - self.assertGreater(avg_spec_accept_length, 2.04) - - def test_bs_1_speed(self): - args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048) - acc_length, speed = send_one_prompt(args) - - print(f"{acc_length=:.2f} {speed=:.2f}") - - if is_in_ci(): - write_github_step_summary( - f"### test_bs_1_speed (deepseek-r1-mxfp4 mtp)\n" - f"{acc_length=:.2f}\n" - f"{speed=:.2f} token/s\n" - ) - self.assertGreater(acc_length, 2.04) - self.assertGreater(speed, 150) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/cuda_graph/piecewise/test_pcg_with_speculative_decoding.py b/test/registered/cuda_graph/piecewise/test_pcg_with_speculative_decoding.py deleted file mode 100644 index 99caae4333..0000000000 --- a/test/registered/cuda_graph/piecewise/test_pcg_with_speculative_decoding.py +++ /dev/null @@ -1,45 +0,0 @@ -"""Test piecewise CUDA graph coexisting with speculative decoding (EAGLE3). - -PCG handles prefill/extend path while speculative decoding (EAGLE3) uses -decode CUDA graphs. This test verifies they don't interfere with each -other. The MTP variant runs under breakable CUDA graph in -../breakable/test_bcg_with_speculative_decoding_extra.py. -""" - -import unittest - -from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.server_fixtures.pcg_spec_fixture import PCGSpecBase - -register_cuda_ci(est_time=130, stage="weekly", runner_config="4-gpu-h100") - - -class TestPCGWithEAGLE3(PCGSpecBase, unittest.TestCase): - """PCG + EAGLE3 on Qwen3-30B-A3B-Instruct-2507.""" - - model = "Qwen/Qwen3-30B-A3B-Instruct-2507" - server_args = [ - "--tp", - "2", - "--trust-remote-code", - "--cuda-graph-backend-prefill=tc_piecewise", - "--mem-fraction-static", - "0.6", - "--speculative-algorithm", - "EAGLE3", - "--speculative-draft-model-path", - "lmsys/SGLang-EAGLE3-Qwen3-30B-A3B-Instruct-2507-SpecForge-Nex", - "--speculative-num-steps", - "5", - "--speculative-eagle-topk", - "4", - "--speculative-num-draft-tokens", - "8", - ] - timeout_mult = 3 - server_env = {"SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN": "1"} - accuracy_threshold = 0.75 - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/cuda_graph/test_cuda_piecewise_backend.py b/test/registered/cuda_graph/test_cuda_piecewise_backend.py deleted file mode 100644 index 7efb5f78de..0000000000 --- a/test/registered/cuda_graph/test_cuda_piecewise_backend.py +++ /dev/null @@ -1,56 +0,0 @@ -from unittest.mock import MagicMock - -import pytest -import torch - -from sglang.test.ci.ci_register import register_cuda_ci - -register_cuda_ci(est_time=11, stage="base-b", runner_config="1-gpu-small") - -if not torch.cuda.is_available(): - pytest.skip("CUDA piecewise backend requires CUDA", allow_module_level=True) - -import sglang.srt.compilation.cuda_piecewise_backend as cuda_backend -from sglang.srt.compilation.cuda_piecewise_backend import ( - ConcreteSizeEntry, - CUDAPiecewiseBackend, -) - - -def test_runtime_recompile_without_capture_stream_falls_back(monkeypatch): - """A Dynamo replacement backend cannot capture outside a PCG session.""" - compile_config = MagicMock() - compile_config.get_capture_sizes.return_value = [] - backend = CUDAPiecewiseBackend( - graph=MagicMock(), - compile_config=compile_config, - inductor_config={}, - graph_pool=None, - piecewise_compile_index=0, - total_piecewise_compiles=1, - sym_shape_indices=[0], - compiled_graph_for_general_shape=MagicMock(), - sglang_backend=MagicMock(), - ) - backend.first_run_finished = True - fallback = MagicMock(return_value="fallback-result") - backend.concrete_size_entries = { - 4: ConcreteSizeEntry( - runtime_shape=4, - need_to_compile=False, - use_cudagraph=True, - runnable=fallback, - num_finished_warmup=1, - ) - } - - monkeypatch.setattr(cuda_backend, "get_pcg_capture_stream", lambda: None) - monkeypatch.setattr(cuda_backend, "is_in_torch_compile_warmup", lambda: False) - monkeypatch.setattr(cuda_backend, "print_warning_once", lambda _message: None) - - assert backend(4) == "fallback-result" - fallback.assert_called_once_with(4) - - -if __name__ == "__main__": - raise SystemExit(pytest.main([__file__, "-v"])) diff --git a/test/registered/npu/basic_function/optimization_debug/test_npu_piecewise_graph_prefill.py b/test/registered/npu/basic_function/optimization_debug/test_npu_piecewise_graph_prefill.py deleted file mode 100644 index 1db72e5fb4..0000000000 --- a/test/registered/npu/basic_function/optimization_debug/test_npu_piecewise_graph_prefill.py +++ /dev/null @@ -1,77 +0,0 @@ -import subprocess -import unittest - -from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin -from sglang.test.ascend.test_ascend_utils import ( - QWEN2_5_7B_INSTRUCT_WEIGHTS_PATH, - write_results_to_github_step_summary, -) -from sglang.test.ci.ci_register import register_npu_ci -from sglang.test.test_utils import ( - CustomTestCase, - run_bench_one_batch, -) - -register_npu_ci(est_time=400, suite="base-b-test-1-npu-a3") -register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True) - - -TOKENS_TO_CAPTURE = [i for i in range(128, 4096, 128)] - - -class TestPiecewiseGraphPrefillCorrectness(GSM8KAscendMixin, CustomTestCase): - model = QWEN2_5_7B_INSTRUCT_WEIGHTS_PATH - other_args = [ - "--trust-remote-code", - "--mem-fraction-static", - 0.8, - "--attention-backend", - "ascend", - "--cuda-graph-bs-decode", - 128, - "--cuda-graph-backend-prefill=tc_piecewise", - "--cuda-graph-bs-prefill", - *TOKENS_TO_CAPTURE, - ] - accuracy = 0.84 - num_questions = 1319 - - -class TestPiecewiseGraphPrefillBenchmark(CustomTestCase): - model = QWEN2_5_7B_INSTRUCT_WEIGHTS_PATH - other_args = [ - "--trust-remote-code", - "--mem-fraction-static", - 0.8, - "--attention-backend", - "ascend", - "--cuda-graph-backend-prefill=tc_piecewise", - "--cuda-graph-bs-prefill", - ] + TOKENS_TO_CAPTURE - - latency = 0.045 - - def test_latency(self): - print(f"##=== Testing prefill latency: {self.model} ===##") - model_metrics = { - "server": subprocess.list2cmdline(map(str, self.other_args)), - "client": "bench_one_batch", - "latency_threshold": self.latency, - } - try: - prefill_latency, _, _ = run_bench_one_batch( - self.model, - other_args=self.other_args, - ) - model_metrics["latency"] = float(prefill_latency) - self.assertLess(prefill_latency, self.latency) - except Exception as e: - model_metrics["error"] = e - print(f"Error testing {self.model}: {e}") - self.fail(f"Test failed for {self.model}: {e}") - finally: - write_results_to_github_step_summary({self.model: model_metrics}) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/unit/configs/test_multimodal_piecewise_cuda_graph.py b/test/registered/unit/configs/test_multimodal_piecewise_cuda_graph.py index 5e9c92d103..a3f4bd79ab 100644 --- a/test/registered/unit/configs/test_multimodal_piecewise_cuda_graph.py +++ b/test/registered/unit/configs/test_multimodal_piecewise_cuda_graph.py @@ -15,7 +15,6 @@ from sglang.srt.configs.model_config import ( from sglang.srt.model_executor.cuda_graph_config import ( Backend, CudaGraphConfig, - Phase, PhaseConfig, ) from sglang.srt.model_executor.forward_batch_info import ( @@ -78,34 +77,9 @@ class TestMultimodalPiecewiseCudaGraph(CustomTestCase): ) ) - def test_supported_multimodal_model_upgrades_default_to_tc_piecewise(self): - args = ServerArgs(model_path="dummy") - args._model_config = SimpleNamespace( - is_multimodal_piecewise_cuda_graph_supported=True, - is_multimodal_breakable_cuda_graph_supported=False, - ) - args.cuda_graph_config = CudaGraphConfig( - prefill=PhaseConfig(backend=Backend.BREAKABLE) - ) - args._cuda_graph_config_locked = set() - - args.attention_backend = "fa3" - - with patch( - "sglang.srt.arg_groups.cuda_graph_hook" - ".disable_tc_piecewise_cudagraph_if_incompatible" - ) as disable_if_incompatible: - apply_cuda_graph_compatibility(args) - - self.assertEqual( - resolution_result(args, "cuda_graph_config").prefill.backend, - Backend.TC_PIECEWISE, - ) - disable_if_incompatible.assert_called_once() - def test_trtllm_mla_stays_on_breakable(self): args = ServerArgs(model_path="dummy") - # trtllm_mla skips the tc_piecewise upgrade and keeps breakable, which + # trtllm_mla keeps breakable, which # now serves MLA by falling back to the flashinfer MLA impl for extend. args._model_config = SimpleNamespace( is_multimodal_piecewise_cuda_graph_supported=True, @@ -128,27 +102,6 @@ class TestMultimodalPiecewiseCudaGraph(CustomTestCase): Backend.BREAKABLE, ) - def test_explicit_tc_piecewise_overrides_trtllm_mla_default(self): - args = ServerArgs(model_path="dummy") - args.cuda_graph_config = CudaGraphConfig( - prefill=PhaseConfig(backend=Backend.TC_PIECEWISE) - ) - args._cuda_graph_config_locked = {(Phase.PREFILL, "backend")} - - args.attention_backend = "trtllm_mla" - - apply_cuda_graph_compatibility(args) - - self.assertEqual( - resolution_result(args, "cuda_graph_config").prefill.backend, - Backend.TC_PIECEWISE, - ) - - def test_multimodal_inputs_keep_tc_piecewise_prefill_enabled(self): - runner = self._make_prefill_runner(Backend.TC_PIECEWISE) - - self.assertTrue(runner.can_run_graph(self._make_multimodal_forward_batch())) - def test_multimodal_inputs_keep_breakable_prefill_enabled(self): runner = self._make_prefill_runner(Backend.BREAKABLE) @@ -176,7 +129,7 @@ class TestMultimodalPiecewiseCudaGraph(CustomTestCase): ) args.cuda_graph_config = CudaGraphConfig( decode=PhaseConfig(backend=Backend.FULL), - prefill=PhaseConfig(backend=Backend.TC_PIECEWISE), + prefill=PhaseConfig(backend=Backend.FULL), ) args.disable_radix_cache = False args.chunked_prefill_size = 2048 diff --git a/test/registered/unit/model_executor/runner_backend/test_backend_resolution.py b/test/registered/unit/model_executor/runner_backend/test_backend_resolution.py index a78725a5e6..d27f4dc861 100644 --- a/test/registered/unit/model_executor/runner_backend/test_backend_resolution.py +++ b/test/registered/unit/model_executor/runner_backend/test_backend_resolution.py @@ -146,6 +146,31 @@ class TestResolveDecodeBackend(CustomTestCase): enable_memory_saver=False, ) + def test_disabled_config_still_supports_explicit_graph_runner(self): + runner = _make_graph_runner(device="cuda") + exec_config = _make_exec_config(backend=Backend.DISABLED) + platform = _make_platform(is_out_of_tree=False) + with ( + mock.patch.object(backend_utils, "get_exec", return_value=exec_config), + mock.patch.object(backend_utils, "current_platform", platform), + mock.patch.object(backend_utils, "FullCudaGraphBackend") as full_backend, + ): + self.assertIs( + backend_utils.resolve_decode_backend(runner), full_backend.return_value + ) + full_backend.assert_called_once_with(runner, enable_memory_saver=False) + + def test_retired_backend_is_not_replaced_with_full(self): + runner = _make_graph_runner(device="cuda") + exec_config = _make_exec_config(backend=Backend.TC_PIECEWISE) + with ( + mock.patch.object(backend_utils, "get_exec", return_value=exec_config), + mock.patch.object(backend_utils, "FullCudaGraphBackend") as full_backend, + ): + with self.assertRaisesRegex(ValueError, "Unsupported decode graph backend"): + backend_utils.resolve_decode_backend(runner) + full_backend.assert_not_called() + def test_breakable_backend_does_not_consult_full_backend_factory(self): runner = _make_graph_runner() exec_config = _make_exec_config( diff --git a/test/registered/unit/model_executor/test_cuda_graph_config_backends.py b/test/registered/unit/model_executor/test_cuda_graph_config_backends.py new file mode 100644 index 0000000000..b3a73d8302 --- /dev/null +++ b/test/registered/unit/model_executor/test_cuda_graph_config_backends.py @@ -0,0 +1,70 @@ +"""Validate backend retirement without importing accelerator backends.""" + +import argparse +import sys +from unittest.mock import patch + +import pytest + +from sglang.srt.model_executor.cuda_graph_config import ( + Backend, + CudaGraphConfig, + default_prefill_backend, + parse_cuda_graph_backend_arg, + parse_cuda_graph_config_arg, +) +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=2, suite="base-a-test-cpu") + + +@pytest.mark.parametrize("backend", [Backend.FULL, Backend.BREAKABLE, Backend.DISABLED]) +def test_supported_prefill_backends_round_trip(backend): + raw = parse_cuda_graph_config_arg('{"prefill":{"backend":"' + backend + '"}}') + config = CudaGraphConfig.from_dict(raw) + assert config.prefill.backend == backend + + +def test_removed_backend_is_rejected(): + with pytest.raises(ValueError, match="tc_piecewise was removed"): + CudaGraphConfig.from_dict({"prefill": {"backend": "tc_piecewise"}}) + + +@pytest.mark.parametrize("phase", ["prefill", "decode"]) +@pytest.mark.parametrize("json_config", [False, True]) +def test_cli_reports_backend_removal(phase, json_config, capsys): + parser = argparse.ArgumentParser() + flag = f"--cuda-graph-backend-{phase}" + parser.add_argument(flag, type=parse_cuda_graph_backend_arg, choices=Backend.ALL) + parser.add_argument("--cuda-graph-config", type=parse_cuda_graph_config_arg) + argv = ( + ["--cuda-graph-config", '{"' + phase + '":{"backend":"tc_piecewise"}}'] + if json_config + else [flag, "tc_piecewise"] + ) + with pytest.raises(SystemExit) as error: + parser.parse_args(argv) + assert error.value.code == 2 + assert ( + "tc_piecewise was removed; select breakable, full, or disabled" + in capsys.readouterr().err + ) + + +def test_removed_compiler_option_is_rejected(): + with pytest.raises(argparse.ArgumentTypeError, match="tc_compiler"): + parse_cuda_graph_config_arg('{"prefill":{"tc_compiler":"eager"}}') + with pytest.raises(ValueError, match="tc_compiler was removed"): + CudaGraphConfig.from_dict({"prefill": {"tc_compiler": "inductor"}}) + + +@pytest.mark.parametrize( + "cuda, expected", [(True, Backend.BREAKABLE), (False, Backend.DISABLED)] +) +def test_prefill_defaults_do_not_enable_unvalidated_platforms(cuda, expected): + with patch("sglang.srt.utils.is_cuda", return_value=cuda): + assert default_prefill_backend() == expected + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"])) diff --git a/test/registered/unit/server_args/test_server_args.py b/test/registered/unit/server_args/test_server_args.py index 542a6f43c2..bfc3105080 100644 --- a/test/registered/unit/server_args/test_server_args.py +++ b/test/registered/unit/server_args/test_server_args.py @@ -25,7 +25,6 @@ from sglang.srt.arg_groups.attention_hook import ( ) from sglang.srt.arg_groups.cuda_graph_hook import ( apply_cuda_graph_compatibility, - disable_tc_piecewise_cudagraph_if_incompatible, finalize_cuda_graph_prefill_max_context, handle_cuda_graph_config, ) @@ -2533,44 +2532,6 @@ class TestPrefillOnlyDisableKvCache(unittest.TestCase): self._validate_prefill_only_args(kv_cache_dtype=kv_cache_dtype) -class TestCudaGraphConfigDataclassAccess(CustomTestCase): - @patch( - "sglang.srt.model_executor.runner_backend." - "tc_piecewise_cuda_graph_backend.get_moe_a2a_backend" - ) - def test_tc_piecewise_build_config_reads_phase_config_dataclass( - self, mock_get_moe_a2a_backend - ): - from sglang.srt.model_executor.runner_backend.tc_piecewise_cuda_graph_backend import ( - TcPiecewiseCudaGraphBackend, - ) - - mock_backend = mock_get_moe_a2a_backend.return_value - mock_backend.is_deepep.return_value = False - mock_backend.is_mooncake.return_value = False - from sglang.srt.runtime_context import get_context - - # The graph configuration is a bag leaf; the debug switch is raw input - # and stays on the argument. - override = get_context().override_server_args( - cuda_graph_config=CudaGraphConfig( - prefill=PhaseConfig( - backend=Backend.TC_PIECEWISE, - bs=[32, 64], - tc_compiler="eager", - ) - ) - ) - override.install() - self.addCleanup(override.restore) - server_args = SimpleNamespace(enable_torch_compile_debug_mode=False) - - config = TcPiecewiseCudaGraphBackend.build_compilation_config(server_args) - - self.assertEqual(config.get_capture_sizes(), [32, 64]) - self.assertEqual(config.compiler, "eager") - - class TestPipelineParallelCompat(CustomTestCase): """Features supported with `pipeline-parallel-size > 1`.""" @@ -2854,33 +2815,6 @@ class TestPrefillCudaGraphLoRACompatibility(CustomTestCase): Backend.BREAKABLE, ) - def test_lora_still_disables_tc_piecewise_prefill_graph(self): - # Pin the tc_piecewise LoRA rule itself, with the hardware rule - # neutralized so this runs on CPU-only CI. - args = ServerArgs(model_path="dummy", enable_lora=True) - args._model_config = SimpleNamespace( - hf_config=SimpleNamespace(architectures=["LlamaForCausalLM"]), - is_piecewise_cuda_graph_disabled_model=False, - is_multimodal=False, - is_multimodal_piecewise_cuda_graph_supported=False, - ) - args.cuda_graph_config = CudaGraphConfig( - prefill=PhaseConfig(backend=Backend.TC_PIECEWISE) - ) - with ( - override_platform(is_hip=False), - override_platform(is_npu=False), - patch("sglang.srt.arg_groups.cuda_graph_hook.is_cpu", return_value=False), - patch("sglang.srt.arg_groups.cuda_graph_hook.is_mps", return_value=False), - override_platform(is_xpu=False), - ): - disable_tc_piecewise_cudagraph_if_incompatible(args) - - self.assertEqual( - resolution_result(args, "cuda_graph_config").prefill.backend, - Backend.DISABLED, - ) - class TestBreakableCudaGraphMultimodalAllowlist(CustomTestCase): """The BCG "multimodal model" rule exempts archs on the BCG multimodal diff --git a/test/registered/unit/server_args/test_unified_prefill_cuda_graph_gate.py b/test/registered/unit/server_args/test_unified_prefill_cuda_graph_gate.py index bb20ded244..3c86460f20 100644 --- a/test/registered/unit/server_args/test_unified_prefill_cuda_graph_gate.py +++ b/test/registered/unit/server_args/test_unified_prefill_cuda_graph_gate.py @@ -16,7 +16,7 @@ Capture is wired: the captured batch reads `out_cache_loc` out of the registry slot, refilled before each replay from the already-rebound physical loc, and the read tables are refilled out-of-graph from the live v2p. So BREAKABLE -(the CUDA default) and TC_PIECEWISE must be left alone -- an earlier gate +(the CUDA default) must be left alone -- an earlier gate disabled every prefill backend outright, which cost every unified run its prefill graph. @@ -83,7 +83,7 @@ class TestUnifiedPrefillCudaGraphGate(unittest.TestCase): def test_non_full_prefill_backends_are_left_enabled(self): """BUG REGRESSION. Unified used to disable prefill capture outright, so the default BREAKABLE graph silently never ran.""" - for backend in (Backend.BREAKABLE, Backend.TC_PIECEWISE): + for backend in (Backend.BREAKABLE,): for attn in (("fa4", "fa4"), ("triton", "triton")): with self.subTest(prefill=backend, attn=attn): cg = _run_handler(prefill_backend=backend, attention_backends=attn) diff --git a/test/registered/unit/test_server_args_migration.py b/test/registered/unit/test_server_args_migration.py index 4a5270c081..478090f271 100644 --- a/test/registered/unit/test_server_args_migration.py +++ b/test/registered/unit/test_server_args_migration.py @@ -168,7 +168,7 @@ class TestServerArgsAnnotatedCli(CustomTestCase): return sa.disable_cuda_graph, config.decode.backend, config.prefill.backend # Not a literal: the prefill default is BREAKABLE on CUDA and - # TC_PIECEWISE elsewhere, and this file runs on the CPU runner. + # disabled elsewhere, and this file runs on the CPU runner. self.assertEqual(backends([]), (False, Backend.FULL, default_prefill_backend())) self.assertEqual( backends(["--disable-cuda-graph"]), diff --git a/test/registered/xpu/test_xpu_graph.py b/test/registered/xpu/test_xpu_graph.py deleted file mode 100644 index 943beebdc0..0000000000 --- a/test/registered/xpu/test_xpu_graph.py +++ /dev/null @@ -1,79 +0,0 @@ -""" -XPU graph tests: verifies decode full-graph and prefill tc_piecewise graph -on Intel XPU produce valid outputs. - - - TestXPUGraph : decode full-graph and prefill tc_piecewise graph enabled - together in a single bench_one_batch invocation. - -Usage: - python3 -m unittest test_xpu_graph.TestXPUGraph -""" - -import unittest - -from sglang.test.ci.ci_register import register_xpu_ci -from sglang.test.test_utils import ( - DEFAULT_SMALL_MODEL_NAME_FOR_TEST_QWEN, - CustomTestCase, - is_in_ci, - run_bench_one_batch, -) - -register_xpu_ci( - est_time=600, - suite="stage-b-test-1-gpu-xpu", - disabled=( - "tc_piecewise prefill graph breaks since #42301: Dynamo cannot build the " - "msgspec.Struct layer-boundary values under fullgraph=True. tc_piecewise " - "is being removed in #41634; refactor this test once that lands." - ), -) - -_COMMON_ARGS = [ - "--device", - "xpu", - "--attention-backend", - "triton", - "--disable-radix-cache", - "--mem-fraction-static", - "0.6", - "--batch-size", - "1", -] - -_CI_IO_ARGS = ["--input", "64", "--output", "4"] -_FULL_IO_ARGS = ["--input", "128", "--output", "16"] - - -class TestXPUGraph(CustomTestCase): - """Decode full-graph + prefill tc_piecewise together.""" - - def test_full_graph_runs(self): - args = [ - *_COMMON_ARGS, - "--cuda-graph-config", - '{"decode":{"backend":"full"},"prefill":{"backend":"tc_piecewise","tc_compiler":"eager"}}', - "--cuda-graph-bs-prefill", - "64", - "128", - ] - if is_in_ci(): - args += _CI_IO_ARGS - else: - args += _FULL_IO_ARGS - - prefill_latency, decode_throughput, _ = run_bench_one_batch( - DEFAULT_SMALL_MODEL_NAME_FOR_TEST_QWEN, args - ) - self.assertGreater( - prefill_latency, - 0, - "prefill latency must be > 0 with tc_piecewise XPU graph", - ) - self.assertGreater( - decode_throughput, 0, "decode throughput must be > 0 with full XPU graph" - ) - - -if __name__ == "__main__": - unittest.main()