[Refactor][TCPCG] Retire backend selection and configuration (3/9) (#42287)

This commit is contained in:
Yuwei An
2026-10-09 22:42:35 -07:00
committed by GitHub
parent 0aa580f904
commit 8b2aaf7060
30 changed files with 162 additions and 1292 deletions
+7 -145
View File
@@ -22,14 +22,9 @@ from sglang.srt.model_executor.cuda_graph_config import (
default_cuda_graph_config,
with_phase,
)
from sglang.srt.platforms import current_platform
from sglang.srt.runtime_context import attn_dp_enabled_of, get_platform
from sglang.srt.utils.common import (
is_cpu,
is_mps,
parse_connector_type,
)
from sglang.srt.utils.hf_transformers_utils import check_gguf_file
logger = logging.getLogger(__name__)
@@ -84,11 +79,6 @@ def parse_cuda_graph_config(server_args: Any):
_set(Phase.DECODE, "bs", cfg.cuda_graph_bs_decode)
if cfg.cuda_graph_bs_prefill is not None:
_set(Phase.PREFILL, "bs", cfg.cuda_graph_bs_prefill)
if cfg.cuda_graph_tc_compiler is not None:
# Written to both phases so the value is in place when TC_PIECEWISE
# decode is implemented; today decode ignores it.
_set(Phase.DECODE, "tc_compiler", cfg.cuda_graph_tc_compiler)
_set(Phase.PREFILL, "tc_compiler", cfg.cuda_graph_tc_compiler)
if cfg.cuda_graph_prefill_max_context is not None:
_set(
Phase.PREFILL,
@@ -113,10 +103,8 @@ def parse_cuda_graph_config(server_args: Any):
def apply_cuda_graph_compatibility(server_args: Any):
"""Auto-disable prefill cuda graph for incompatible configs.
Rules are split per backend — TcPiecewise and Breakable have
different constraints. Skipped when the user explicitly set the
prefill backend, whichever value they chose (the contract the removed
--enforce-piecewise-cuda-graph used to spell).
Breakable and Full have different constraints. Skip automatic selection
when the user explicitly sets the prefill backend.
"""
cfg = resolving_view(server_args)
@@ -141,137 +129,12 @@ def apply_cuda_graph_compatibility(server_args: Any):
)
return
# Breakable is the CUDA default but not multimodal-compatible;
# piecewise-allowlisted archs run their validated decoder prefill
# there instead. Archs also on the breakable allowlist keep it --
# this runs first, so piecewise would otherwise silently win.
if (
cfg.cuda_graph_config.prefill.backend == Backend.BREAKABLE
and model_config_of(server_args).is_multimodal_piecewise_cuda_graph_supported
and not model_config_of(
server_args
).is_multimodal_breakable_cuda_graph_supported
# Keep trtllm_mla on the preferred breakable path, which now serves
# MLA by falling back to the flashinfer MLA impl for extend.
and attention_backends_of(resolved_view(server_args))[0] != "trtllm_mla"
):
logger.info(
"Using tc_piecewise CUDA graph for validated multimodal decoder prefill."
)
declare_resolution(
server_args,
"_apply_cuda_graph_compatibility",
cuda_graph_config=with_phase(
cfg.cuda_graph_config, Phase.PREFILL, backend=Backend.TC_PIECEWISE
),
)
if cfg.cuda_graph_config.prefill.backend == Backend.TC_PIECEWISE:
disable_tc_piecewise_cudagraph_if_incompatible(server_args)
elif cfg.cuda_graph_config.prefill.backend == Backend.BREAKABLE:
if cfg.cuda_graph_config.prefill.backend == Backend.BREAKABLE:
disable_breakable_cudagraph_if_incompatible(server_args)
elif cfg.cuda_graph_config.prefill.backend == Backend.FULL:
disable_full_prefill_cudagraph_if_incompatible(server_args)
def disable_tc_piecewise_cudagraph_if_incompatible(server_args: Any):
"""TcPiecewise (torch.compile + piecewise) is incompatible with
these configurations. Most are torch.compile / dynamo limitations.
"""
cfg = resolving_view(server_args)
rules = [
(
"model-arch blacklist",
lambda: model_config_of(server_args).is_piecewise_cuda_graph_disabled_model,
),
("DP attention", lambda: attn_dp_enabled_of(resolved_view(server_args))),
("full torch.compile mode", lambda: cfg.enable_torch_compile),
("pipeline parallelism (pp_size > 1)", lambda: cfg.pp_size > 1),
(
"non-CUDA hardware (HIP/NPU/CPU/MPS/XPU)",
lambda: (
get_platform().is_hip
or get_platform().is_npu
or is_cpu()
or is_mps()
or get_platform().is_xpu
),
),
(
"OOT platform without piecewise support",
lambda: (
current_platform.is_out_of_tree()
and not current_platform.capabilities.piecewise_graph
),
),
(
"MoE A2A backend",
lambda: resolved_view(server_args).moe_a2a_backend != "none",
),
# Dynamo blocks LoRA under tc_piecewise (per-batch LoRABatchInfo
# rebinds break guards); breakable/full support LoRA.
("LoRA", lambda: bool(cfg.lora_paths) or cfg.enable_lora),
(
"multimodal model",
lambda: (
model_config_of(server_args).is_multimodal
and not model_config_of(
server_args
).is_multimodal_piecewise_cuda_graph_supported
),
),
(
"GGUF quantization",
lambda: (
cfg.load_format == "gguf"
or resolved_view(server_args).quantization == "gguf"
or check_gguf_file(cfg.model_path)
),
),
("DLLM (diffusion LLM)", lambda: cfg.dllm_algorithm is not None),
(
"CPU offload / hierarchical cache",
lambda: cfg.cpu_offload_gb > 0 or cfg.enable_hierarchical_cache,
),
(
"deterministic inference",
lambda: cfg.enable_deterministic_inference,
),
("PD disaggregation", lambda: cfg.disaggregation_mode != "null"),
("symmetric memory", lambda: cfg.enable_symm_mem),
(
"expert distribution recorder",
lambda: (
cfg.enable_eplb or cfg.expert_distribution_recorder_mode is not None
),
),
(
"context parallel (attn_cp_size > 1)",
lambda: resolved_view(server_args).attn_cp_size > 1,
),
("CUDA graph debug mode", lambda: cfg.debug_cuda_graph),
# Capture builds a dummy extend forward with attn_dcp_metadata=None.
(
"decode context parallel (dcp_size > 1)",
lambda: cfg.dcp_size > 1,
),
]
for _name, predicate in rules:
if predicate():
declare_resolution(
server_args,
"_disable_tc_piecewise_cudagraph_if_incompatible",
cuda_graph_config=with_phase(
cfg.cuda_graph_config, Phase.PREFILL, backend=Backend.DISABLED
),
)
# One decision, one declaration: every rule declares the same
# value, so a later match would only append a duplicate entry.
break
def disable_breakable_cudagraph_if_incompatible(server_args: Any):
"""Breakable (segmented capture, no torch.compile). Breakable enforces
memory-saver rejection in its own __init__; config-time rules can be
@@ -377,8 +240,7 @@ def disable_full_prefill_cudagraph_if_incompatible(server_args: Any):
def disable_prefill_cuda_graph_for_deepseek_trtllm_mla(server_args: Any):
"""Disable prefill CUDA graph for dsr1 by default when using the trtllm_mla
attention backend. Under any captured prefill CUDA graph (tc_piecewise or
breakable) trtllm_mla falls back to FlashAttention for prefill and regresses
attention backend. Under any captured prefill CUDA graph trtllm_mla falls back to FlashAttention for prefill and regresses
performance, so disable whichever prefill graph backend is in effect.
"""
@@ -400,7 +262,7 @@ def disable_prefill_cuda_graph_for_deepseek_trtllm_mla(server_args: Any):
"Disabling prefill CUDA graph (%s) by default for the DeepSeek-V3 arch on "
"the trtllm_mla attention backend (a captured prefill graph forces a "
"FlashAttention fallback that regresses prefill). Set the prefill cuda graph "
"backend explicitly (e.g. --cuda-graph-backend-prefill tc_piecewise) to override.",
"backend explicitly (e.g. --cuda-graph-backend-prefill breakable) to override.",
cfg.cuda_graph_config.prefill.backend,
)
declare_resolution(
@@ -503,7 +365,7 @@ def handle_cuda_graph_config(server_args: Any):
if cfg.cuda_graph_config.prefill.backend == Backend.FULL:
logger.warning(
"cuda_graph_config[prefill].backend='full' is experimental. "
"Use breakable or tc_piecewise for production workloads."
"Use breakable for production workloads."
)
@@ -580,7 +442,7 @@ def finalize_cuda_graph_prefill_max_context(server_args: Any) -> None:
def generate_prefill_cuda_graph_batch_sizes(max_bs: int):
"""
Generate the list of batch sizes for prefill CUDA graph capture
based on max_bs. For tc_piecewise prefill, bs carries the
based on max_bs. For prefill, bs carries the
captured token count (one shape knob per phase).
"""
capture_sizes = (
@@ -227,7 +227,6 @@ POSITIONAL_FIELD_ORDER = (
"cuda_graph_bs_decode",
"cuda_graph_bs_prefill",
"cuda_graph_prefill_max_context",
"cuda_graph_tc_compiler",
"disable_prefill_cuda_graph",
"disable_decode_cuda_graph",
"disable_cuda_graph",
+6 -7
View File
@@ -28,6 +28,7 @@ from sglang.srt.arg_groups.choices import (
from sglang.srt.model_executor.cuda_graph_config import (
Backend,
CudaGraphConfig,
parse_cuda_graph_backend_arg,
parse_cuda_graph_config_arg,
)
from sglang.srt.utils.common import human_readable_int
@@ -505,22 +506,24 @@ class ExecGraph(msgspec.Struct):
cuda_graph_config: A[
Optional[CudaGraphConfig],
Arg(
help='Per-phase CUDA graph settings as JSON, e.g. \'{"decode":{"backend":"full","max_bs":256},"prefill":{"backend":"tc_piecewise","tc_compiler":"eager"}}\'. Allowed backends per phase: full, breakable, tc_piecewise, disabled (full is decode-only). JSON wins over the per-phase --cuda-graph-* convenience flags and over legacy flags.',
help='Per-phase CUDA graph settings as JSON, e.g. \'{"decode":{"backend":"full","max_bs":256},"prefill":{"backend":"breakable"}}\'. Allowed backends per phase: full, breakable, disabled. JSON wins over the per-phase --cuda-graph-* convenience flags and over legacy flags.',
type_parser=parse_cuda_graph_config_arg,
),
] = None
cuda_graph_backend_decode: A[
Optional[Literal["full", "breakable", "tc_piecewise", "disabled"]],
Optional[Literal["full", "breakable", "disabled"]],
Arg(
help="Backend for the decode phase. Folds into cuda_graph_config[decode].backend.",
choices=Backend.ALL,
type_parser=parse_cuda_graph_backend_arg,
),
] = None
cuda_graph_backend_prefill: A[
Optional[Literal["full", "breakable", "tc_piecewise", "disabled"]],
Optional[Literal["full", "breakable", "disabled"]],
Arg(
help="Backend for the prefill phase. Folds into cuda_graph_config[prefill].backend.",
choices=Backend.ALL,
type_parser=parse_cuda_graph_backend_arg,
),
] = None
cuda_graph_max_bs_decode: A[
@@ -556,10 +559,6 @@ class ExecGraph(msgspec.Struct):
aliases=["--context-bucket"],
),
] = None
cuda_graph_tc_compiler: A[
Optional[Literal["eager", "inductor"]],
"Compiler used by the tc_piecewise backend (currently only the prefill phase consumes it).",
] = None
disable_prefill_cuda_graph: A[
bool,
"Disable the prefill-phase CUDA graph. Convenience for --cuda-graph-backend-prefill=disabled.",
@@ -37,20 +37,6 @@ def handle_npu_backends(server_args: Any):
set_default_server_args(server_args)
current = cfg.cuda_graph_config.prefill.tc_compiler
if current is not None and current != "eager":
logger.warning(
"At this moment Ascend platform only support prefill graph compilation with "
"cuda_graph_config[prefill].tc_compiler='eager'."
)
declare_resolution(
server_args,
"_handle_npu_backends",
cuda_graph_config=with_phase(
cfg.cuda_graph_config, Phase.PREFILL, tc_compiler="eager"
),
)
def handle_mps_backends(server_args: Any):
cfg = resolving_view(server_args)
@@ -40,18 +40,18 @@ class Phase:
class Backend:
"""CUDA graph capture backends a phase can use."""
# Temporary internal name while unreachable legacy branches are removed.
TC_PIECEWISE = "tc_piecewise"
FULL = "full"
BREAKABLE = "breakable"
TC_PIECEWISE = "tc_piecewise"
DISABLED = "disabled"
ALL = (FULL, BREAKABLE, TC_PIECEWISE, DISABLED)
ALL = (FULL, BREAKABLE, DISABLED)
ALLOWED_BACKENDS_PER_PHASE = {
Phase.DECODE: (
Backend.FULL,
Backend.BREAKABLE,
Backend.TC_PIECEWISE,
Backend.DISABLED,
),
# full for prefill captures one whole-forward graph per num_tokens
@@ -61,26 +61,23 @@ ALLOWED_BACKENDS_PER_PHASE = {
Phase.PREFILL: (
Backend.FULL,
Backend.BREAKABLE,
Backend.TC_PIECEWISE,
Backend.DISABLED,
),
}
# Per-phase settings schema. Keys other than backend are runner-level
# (read by any backend in that phase); tc_compiler is the lone
# backend-specific knob (only meaningful when backend == tc_piecewise).
# (read by any backend in that phase).
# For prefill, bs carries aggregate-token capture buckets for every backend;
# full_prefill_max_req separately controls Full's fixed request-slot count.
# full_prefill_max_req and full_prefill_prefix_chunk_tokens are prefill-only and
# only meaningful when backend == full. max_context_size is shared by the
# breakable and full prefill body-capture backends.
ALLOWED_KEYS_PER_PHASE = {
Phase.DECODE: ("backend", "max_bs", "bs", "tc_compiler"),
Phase.DECODE: ("backend", "max_bs", "bs"),
Phase.PREFILL: (
"backend",
"max_bs",
"bs",
"tc_compiler",
"max_context_size",
"full_prefill_max_req",
"full_prefill_prefix_chunk_tokens",
@@ -96,7 +93,7 @@ class PhaseConfig:
backend: str = Backend.DISABLED
max_bs: Optional[int] = None
bs: Optional[List[int]] = None
# Only meaningful when backend == tc_piecewise; ignored otherwise.
# Internal compatibility only; rejected by both configuration parsers.
tc_compiler: str = "eager"
# Effective for both full and breakable backends and currently only DSV4:
# fixed maximum context length used by context-shaped prefill graph metadata.
@@ -105,7 +102,7 @@ class PhaseConfig:
# Only meaningful for the prefill phase with backend == full: max number of
# request slots baked into each captured graph. Real bs <= full_prefill_max_req
# reuses the graph (unused slots become zero-length sentinels); larger
# batches fall back to eager. Ignored by BCG and TC_PIECEWISE. None
# batches fall back to eager. Ignored by BCG. None
# auto-derives chunked_prefill_size // 512.
full_prefill_max_req: Optional[int] = None
# Only meaningful for Full prefill CUDA graphs that capture a distinct
@@ -120,15 +117,10 @@ class PhaseConfig:
def default_prefill_backend() -> str:
"""BCG (breakable) is the prefill default on CUDA only; other platforms
(HIP/NPU/...) keep tc_piecewise until BCG is validated there. Full-graph
prefill capture is opt-in per model architecture via the declarative
registry (see _inkling_overrides in arg_groups/overrides.py), not a global
default. Lazy import keeps this module's stdlib-only import invariant (see
module docstring)."""
"""Enable BCG by default on CUDA; other platforms remain opt-in."""
from sglang.srt.utils import is_cuda
return Backend.BREAKABLE if is_cuda() else Backend.TC_PIECEWISE
return Backend.BREAKABLE if is_cuda() else Backend.DISABLED
def with_phase(config: "CudaGraphConfig", phase: str, **changes) -> "CudaGraphConfig":
@@ -188,6 +180,12 @@ class CudaGraphConfig:
if phase not in Phase.ALL or not isinstance(phase_settings, dict):
continue
phase_cfg = getattr(cfg, phase)
if phase_settings.get("backend") == "tc_piecewise":
raise ValueError(
"tc_piecewise was removed; select breakable, full, or disabled"
)
if "tc_compiler" in phase_settings:
raise ValueError("tc_compiler was removed with tc_piecewise")
allowed = ALLOWED_KEYS_PER_PHASE[phase]
for key, value in phase_settings.items():
if key in allowed:
@@ -236,6 +234,15 @@ def cuda_graph_fully_disabled() -> bool:
) and check_cuda_graph_backend(Phase.PREFILL, Backend.DISABLED)
def parse_cuda_graph_backend_arg(raw: str) -> str:
"""Reject the retired backend before argparse checks supported choices."""
if raw == "tc_piecewise":
raise argparse.ArgumentTypeError(
"tc_piecewise was removed; select breakable, full, or disabled"
)
return raw
def parse_cuda_graph_config_arg(raw: str) -> Dict[str, Dict[str, Any]]:
"""argparse type for --cuda-graph-config: parse JSON dict of
phase → settings dict. Each phase's settings dict is itself validated
@@ -263,6 +270,8 @@ def parse_cuda_graph_config_arg(raw: str) -> Dict[str, Dict[str, Any]]:
f"--cuda-graph-config['{phase}'] must be a JSON object, got "
f"{type(phase_settings).__name__}"
)
if "backend" in phase_settings:
parse_cuda_graph_backend_arg(phase_settings["backend"])
allowed = ALLOWED_KEYS_PER_PHASE[phase]
result[phase] = {}
for key, value in phase_settings.items():
@@ -50,10 +50,6 @@ from sglang.srt.model_executor.forward_context import (
get_token_to_kv_pool,
)
from sglang.srt.model_executor.runner.base_runner import BaseRunner
from sglang.srt.model_executor.runner_backend_utils.tc_piecewise_cuda_graph import (
enable_tc_piecewise_cuda_graph,
set_tc_piecewise_forward_context,
)
from sglang.srt.model_executor.runner_utils import (
maybe_publish_prefill_shared_read_done,
)
@@ -343,34 +339,7 @@ class EagerRunner(BaseRunner):
else "extend"
)
with device_timer_ctx(model_runner.device_timer, category):
pcg_runner = model_runner.prefill_cuda_graph_runner
if (
_is_hip
and pcg_runner is not None
and not isinstance(pcg_runner, EagerRunner)
and not cp_active
):
# HIP PCG eager fallback: enter the PCG context so Dynamo guards
# and PCG-specific MoE/attention paths stay consistent.
with (
enable_tc_piecewise_cuda_graph(),
set_tc_piecewise_forward_context(
forward_batch,
model_runner.attention_layers,
getattr(model_runner.model, "quant_config", None),
model_runner.moe_layers,
model_runner.moe_fusions,
dsa_indexers=model_runner.dsa_indexers,
mha_companion_layers=model_runner.mha_companion_layers,
),
):
ret = model_runner.model.forward(
forward_batch.input_ids,
forward_batch.positions,
forward_batch,
**kwargs,
)
elif cp_active and (
if cp_active and (
not get_parallel().enable_cp_tp_group_sharing
or getattr(model_runner.model, "prepare_cp_inputs", None) is not None
):
@@ -34,9 +34,6 @@ from sglang.srt.model_executor.runner_backend.breakable_cuda_graph_backend impor
from sglang.srt.model_executor.runner_backend.full_cuda_graph_backend import (
FullCudaGraphBackend,
)
from sglang.srt.model_executor.runner_backend.tc_piecewise_cuda_graph_backend import (
TcPiecewiseCudaGraphBackend,
)
from sglang.srt.platforms import current_platform
from sglang.srt.runtime_context import get_exec
@@ -47,9 +44,6 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
# Track first occurrence of each fallback warning to avoid log spam.
_TC_PIECEWISE_DECODE_FALLBACK_LOGGED = False
def resolve_decode_backend(
cuda_graph_runner: BaseCudaGraphRunner,
@@ -90,14 +84,12 @@ def resolve_decode_backend(
enable_memory_saver=enable_memory_saver,
debug_eager=get_exec().graph.debug_cuda_graph,
)
if backend_name == Backend.TC_PIECEWISE:
global _TC_PIECEWISE_DECODE_FALLBACK_LOGGED
if not _TC_PIECEWISE_DECODE_FALLBACK_LOGGED:
logger.warning(
"cuda_graph_config decode='tc_piecewise' is not yet implemented; "
"falling back to 'full'."
)
_TC_PIECEWISE_DECODE_FALLBACK_LOGGED = True
# Callers decide whether to construct a graph runner. Explicit construction
# with a disabled config (e.g. the draft-runner test harness) historically
# uses the full backend; retiring TCPCG must preserve that behavior.
if backend_name not in (Backend.FULL, Backend.DISABLED):
raise ValueError(f"Unsupported decode graph backend: {backend_name}")
full_backend_cls = None
if current_platform.is_out_of_tree():
@@ -116,9 +108,8 @@ def resolve_prefill_backend(
cuda_graph_runner: BaseCudaGraphRunner,
) -> BaseCudaGraphBackend:
"""Pick a backend instance from cuda_graph_config['prefill']['backend']."""
model_runner = cuda_graph_runner.model_runner
cfg = get_exec().graph.cuda_graph_config
backend_name = cfg.prefill.backend if cfg is not None else Backend.TC_PIECEWISE
backend_name = cfg.prefill.backend if cfg is not None else Backend.BREAKABLE
if backend_name == Backend.BREAKABLE:
return BreakableCudaGraphBackend(
@@ -132,5 +123,4 @@ def resolve_prefill_backend(
enable_memory_saver=get_exec().features.enable_memory_saver,
reuse_output_buffer=True,
)
# Default: tc_piecewise.
return TcPiecewiseCudaGraphBackend(cuda_graph_runner)
raise ValueError(f"Unsupported prefill graph backend: {backend_name}")
@@ -52,9 +52,8 @@ def enable_breakable_cuda_graph():
BCG_FAILURE_HINT = (
"1. change to tc_piecewise by --cuda-graph-backend-prefill=tc_piecewise\n"
"2. disable the prefill CUDA graph by --cuda-graph-backend-prefill=disabled\n"
"3. if it is an OOM problem, set --mem-fraction-static to a smaller value "
"1. disable the prefill CUDA graph by --cuda-graph-backend-prefill=disabled\n"
"2. if it is an OOM problem, set --mem-fraction-static to a smaller value "
"(e.g., 0.8 or 0.7) or set --cuda-graph-max-bs-prefill to a smaller value "
"(e.g., 2048)\n"
)