[Refactor][TCPCG] Retire backend selection and configuration (3/9) (#42287)
This commit is contained in:
@@ -22,14 +22,9 @@ from sglang.srt.model_executor.cuda_graph_config import (
|
||||
default_cuda_graph_config,
|
||||
with_phase,
|
||||
)
|
||||
from sglang.srt.platforms import current_platform
|
||||
from sglang.srt.runtime_context import attn_dp_enabled_of, get_platform
|
||||
from sglang.srt.utils.common import (
|
||||
is_cpu,
|
||||
is_mps,
|
||||
parse_connector_type,
|
||||
)
|
||||
from sglang.srt.utils.hf_transformers_utils import check_gguf_file
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -84,11 +79,6 @@ def parse_cuda_graph_config(server_args: Any):
|
||||
_set(Phase.DECODE, "bs", cfg.cuda_graph_bs_decode)
|
||||
if cfg.cuda_graph_bs_prefill is not None:
|
||||
_set(Phase.PREFILL, "bs", cfg.cuda_graph_bs_prefill)
|
||||
if cfg.cuda_graph_tc_compiler is not None:
|
||||
# Written to both phases so the value is in place when TC_PIECEWISE
|
||||
# decode is implemented; today decode ignores it.
|
||||
_set(Phase.DECODE, "tc_compiler", cfg.cuda_graph_tc_compiler)
|
||||
_set(Phase.PREFILL, "tc_compiler", cfg.cuda_graph_tc_compiler)
|
||||
if cfg.cuda_graph_prefill_max_context is not None:
|
||||
_set(
|
||||
Phase.PREFILL,
|
||||
@@ -113,10 +103,8 @@ def parse_cuda_graph_config(server_args: Any):
|
||||
|
||||
def apply_cuda_graph_compatibility(server_args: Any):
|
||||
"""Auto-disable prefill cuda graph for incompatible configs.
|
||||
Rules are split per backend — TcPiecewise and Breakable have
|
||||
different constraints. Skipped when the user explicitly set the
|
||||
prefill backend, whichever value they chose (the contract the removed
|
||||
--enforce-piecewise-cuda-graph used to spell).
|
||||
Breakable and Full have different constraints. Skip automatic selection
|
||||
when the user explicitly sets the prefill backend.
|
||||
"""
|
||||
|
||||
cfg = resolving_view(server_args)
|
||||
@@ -141,137 +129,12 @@ def apply_cuda_graph_compatibility(server_args: Any):
|
||||
)
|
||||
return
|
||||
|
||||
# Breakable is the CUDA default but not multimodal-compatible;
|
||||
# piecewise-allowlisted archs run their validated decoder prefill
|
||||
# there instead. Archs also on the breakable allowlist keep it --
|
||||
# this runs first, so piecewise would otherwise silently win.
|
||||
if (
|
||||
cfg.cuda_graph_config.prefill.backend == Backend.BREAKABLE
|
||||
and model_config_of(server_args).is_multimodal_piecewise_cuda_graph_supported
|
||||
and not model_config_of(
|
||||
server_args
|
||||
).is_multimodal_breakable_cuda_graph_supported
|
||||
# Keep trtllm_mla on the preferred breakable path, which now serves
|
||||
# MLA by falling back to the flashinfer MLA impl for extend.
|
||||
and attention_backends_of(resolved_view(server_args))[0] != "trtllm_mla"
|
||||
):
|
||||
logger.info(
|
||||
"Using tc_piecewise CUDA graph for validated multimodal decoder prefill."
|
||||
)
|
||||
declare_resolution(
|
||||
server_args,
|
||||
"_apply_cuda_graph_compatibility",
|
||||
cuda_graph_config=with_phase(
|
||||
cfg.cuda_graph_config, Phase.PREFILL, backend=Backend.TC_PIECEWISE
|
||||
),
|
||||
)
|
||||
|
||||
if cfg.cuda_graph_config.prefill.backend == Backend.TC_PIECEWISE:
|
||||
disable_tc_piecewise_cudagraph_if_incompatible(server_args)
|
||||
elif cfg.cuda_graph_config.prefill.backend == Backend.BREAKABLE:
|
||||
if cfg.cuda_graph_config.prefill.backend == Backend.BREAKABLE:
|
||||
disable_breakable_cudagraph_if_incompatible(server_args)
|
||||
elif cfg.cuda_graph_config.prefill.backend == Backend.FULL:
|
||||
disable_full_prefill_cudagraph_if_incompatible(server_args)
|
||||
|
||||
|
||||
def disable_tc_piecewise_cudagraph_if_incompatible(server_args: Any):
|
||||
"""TcPiecewise (torch.compile + piecewise) is incompatible with
|
||||
these configurations. Most are torch.compile / dynamo limitations.
|
||||
"""
|
||||
|
||||
cfg = resolving_view(server_args)
|
||||
|
||||
rules = [
|
||||
(
|
||||
"model-arch blacklist",
|
||||
lambda: model_config_of(server_args).is_piecewise_cuda_graph_disabled_model,
|
||||
),
|
||||
("DP attention", lambda: attn_dp_enabled_of(resolved_view(server_args))),
|
||||
("full torch.compile mode", lambda: cfg.enable_torch_compile),
|
||||
("pipeline parallelism (pp_size > 1)", lambda: cfg.pp_size > 1),
|
||||
(
|
||||
"non-CUDA hardware (HIP/NPU/CPU/MPS/XPU)",
|
||||
lambda: (
|
||||
get_platform().is_hip
|
||||
or get_platform().is_npu
|
||||
or is_cpu()
|
||||
or is_mps()
|
||||
or get_platform().is_xpu
|
||||
),
|
||||
),
|
||||
(
|
||||
"OOT platform without piecewise support",
|
||||
lambda: (
|
||||
current_platform.is_out_of_tree()
|
||||
and not current_platform.capabilities.piecewise_graph
|
||||
),
|
||||
),
|
||||
(
|
||||
"MoE A2A backend",
|
||||
lambda: resolved_view(server_args).moe_a2a_backend != "none",
|
||||
),
|
||||
# Dynamo blocks LoRA under tc_piecewise (per-batch LoRABatchInfo
|
||||
# rebinds break guards); breakable/full support LoRA.
|
||||
("LoRA", lambda: bool(cfg.lora_paths) or cfg.enable_lora),
|
||||
(
|
||||
"multimodal model",
|
||||
lambda: (
|
||||
model_config_of(server_args).is_multimodal
|
||||
and not model_config_of(
|
||||
server_args
|
||||
).is_multimodal_piecewise_cuda_graph_supported
|
||||
),
|
||||
),
|
||||
(
|
||||
"GGUF quantization",
|
||||
lambda: (
|
||||
cfg.load_format == "gguf"
|
||||
or resolved_view(server_args).quantization == "gguf"
|
||||
or check_gguf_file(cfg.model_path)
|
||||
),
|
||||
),
|
||||
("DLLM (diffusion LLM)", lambda: cfg.dllm_algorithm is not None),
|
||||
(
|
||||
"CPU offload / hierarchical cache",
|
||||
lambda: cfg.cpu_offload_gb > 0 or cfg.enable_hierarchical_cache,
|
||||
),
|
||||
(
|
||||
"deterministic inference",
|
||||
lambda: cfg.enable_deterministic_inference,
|
||||
),
|
||||
("PD disaggregation", lambda: cfg.disaggregation_mode != "null"),
|
||||
("symmetric memory", lambda: cfg.enable_symm_mem),
|
||||
(
|
||||
"expert distribution recorder",
|
||||
lambda: (
|
||||
cfg.enable_eplb or cfg.expert_distribution_recorder_mode is not None
|
||||
),
|
||||
),
|
||||
(
|
||||
"context parallel (attn_cp_size > 1)",
|
||||
lambda: resolved_view(server_args).attn_cp_size > 1,
|
||||
),
|
||||
("CUDA graph debug mode", lambda: cfg.debug_cuda_graph),
|
||||
# Capture builds a dummy extend forward with attn_dcp_metadata=None.
|
||||
(
|
||||
"decode context parallel (dcp_size > 1)",
|
||||
lambda: cfg.dcp_size > 1,
|
||||
),
|
||||
]
|
||||
for _name, predicate in rules:
|
||||
if predicate():
|
||||
declare_resolution(
|
||||
server_args,
|
||||
"_disable_tc_piecewise_cudagraph_if_incompatible",
|
||||
cuda_graph_config=with_phase(
|
||||
cfg.cuda_graph_config, Phase.PREFILL, backend=Backend.DISABLED
|
||||
),
|
||||
)
|
||||
# One decision, one declaration: every rule declares the same
|
||||
# value, so a later match would only append a duplicate entry.
|
||||
break
|
||||
|
||||
|
||||
def disable_breakable_cudagraph_if_incompatible(server_args: Any):
|
||||
"""Breakable (segmented capture, no torch.compile). Breakable enforces
|
||||
memory-saver rejection in its own __init__; config-time rules can be
|
||||
@@ -377,8 +240,7 @@ def disable_full_prefill_cudagraph_if_incompatible(server_args: Any):
|
||||
|
||||
def disable_prefill_cuda_graph_for_deepseek_trtllm_mla(server_args: Any):
|
||||
"""Disable prefill CUDA graph for dsr1 by default when using the trtllm_mla
|
||||
attention backend. Under any captured prefill CUDA graph (tc_piecewise or
|
||||
breakable) trtllm_mla falls back to FlashAttention for prefill and regresses
|
||||
attention backend. Under any captured prefill CUDA graph trtllm_mla falls back to FlashAttention for prefill and regresses
|
||||
performance, so disable whichever prefill graph backend is in effect.
|
||||
"""
|
||||
|
||||
@@ -400,7 +262,7 @@ def disable_prefill_cuda_graph_for_deepseek_trtllm_mla(server_args: Any):
|
||||
"Disabling prefill CUDA graph (%s) by default for the DeepSeek-V3 arch on "
|
||||
"the trtllm_mla attention backend (a captured prefill graph forces a "
|
||||
"FlashAttention fallback that regresses prefill). Set the prefill cuda graph "
|
||||
"backend explicitly (e.g. --cuda-graph-backend-prefill tc_piecewise) to override.",
|
||||
"backend explicitly (e.g. --cuda-graph-backend-prefill breakable) to override.",
|
||||
cfg.cuda_graph_config.prefill.backend,
|
||||
)
|
||||
declare_resolution(
|
||||
@@ -503,7 +365,7 @@ def handle_cuda_graph_config(server_args: Any):
|
||||
if cfg.cuda_graph_config.prefill.backend == Backend.FULL:
|
||||
logger.warning(
|
||||
"cuda_graph_config[prefill].backend='full' is experimental. "
|
||||
"Use breakable or tc_piecewise for production workloads."
|
||||
"Use breakable for production workloads."
|
||||
)
|
||||
|
||||
|
||||
@@ -580,7 +442,7 @@ def finalize_cuda_graph_prefill_max_context(server_args: Any) -> None:
|
||||
def generate_prefill_cuda_graph_batch_sizes(max_bs: int):
|
||||
"""
|
||||
Generate the list of batch sizes for prefill CUDA graph capture
|
||||
based on max_bs. For tc_piecewise prefill, bs carries the
|
||||
based on max_bs. For prefill, bs carries the
|
||||
captured token count (one shape knob per phase).
|
||||
"""
|
||||
capture_sizes = (
|
||||
|
||||
@@ -227,7 +227,6 @@ POSITIONAL_FIELD_ORDER = (
|
||||
"cuda_graph_bs_decode",
|
||||
"cuda_graph_bs_prefill",
|
||||
"cuda_graph_prefill_max_context",
|
||||
"cuda_graph_tc_compiler",
|
||||
"disable_prefill_cuda_graph",
|
||||
"disable_decode_cuda_graph",
|
||||
"disable_cuda_graph",
|
||||
|
||||
@@ -28,6 +28,7 @@ from sglang.srt.arg_groups.choices import (
|
||||
from sglang.srt.model_executor.cuda_graph_config import (
|
||||
Backend,
|
||||
CudaGraphConfig,
|
||||
parse_cuda_graph_backend_arg,
|
||||
parse_cuda_graph_config_arg,
|
||||
)
|
||||
from sglang.srt.utils.common import human_readable_int
|
||||
@@ -505,22 +506,24 @@ class ExecGraph(msgspec.Struct):
|
||||
cuda_graph_config: A[
|
||||
Optional[CudaGraphConfig],
|
||||
Arg(
|
||||
help='Per-phase CUDA graph settings as JSON, e.g. \'{"decode":{"backend":"full","max_bs":256},"prefill":{"backend":"tc_piecewise","tc_compiler":"eager"}}\'. Allowed backends per phase: full, breakable, tc_piecewise, disabled (full is decode-only). JSON wins over the per-phase --cuda-graph-* convenience flags and over legacy flags.',
|
||||
help='Per-phase CUDA graph settings as JSON, e.g. \'{"decode":{"backend":"full","max_bs":256},"prefill":{"backend":"breakable"}}\'. Allowed backends per phase: full, breakable, disabled. JSON wins over the per-phase --cuda-graph-* convenience flags and over legacy flags.',
|
||||
type_parser=parse_cuda_graph_config_arg,
|
||||
),
|
||||
] = None
|
||||
cuda_graph_backend_decode: A[
|
||||
Optional[Literal["full", "breakable", "tc_piecewise", "disabled"]],
|
||||
Optional[Literal["full", "breakable", "disabled"]],
|
||||
Arg(
|
||||
help="Backend for the decode phase. Folds into cuda_graph_config[decode].backend.",
|
||||
choices=Backend.ALL,
|
||||
type_parser=parse_cuda_graph_backend_arg,
|
||||
),
|
||||
] = None
|
||||
cuda_graph_backend_prefill: A[
|
||||
Optional[Literal["full", "breakable", "tc_piecewise", "disabled"]],
|
||||
Optional[Literal["full", "breakable", "disabled"]],
|
||||
Arg(
|
||||
help="Backend for the prefill phase. Folds into cuda_graph_config[prefill].backend.",
|
||||
choices=Backend.ALL,
|
||||
type_parser=parse_cuda_graph_backend_arg,
|
||||
),
|
||||
] = None
|
||||
cuda_graph_max_bs_decode: A[
|
||||
@@ -556,10 +559,6 @@ class ExecGraph(msgspec.Struct):
|
||||
aliases=["--context-bucket"],
|
||||
),
|
||||
] = None
|
||||
cuda_graph_tc_compiler: A[
|
||||
Optional[Literal["eager", "inductor"]],
|
||||
"Compiler used by the tc_piecewise backend (currently only the prefill phase consumes it).",
|
||||
] = None
|
||||
disable_prefill_cuda_graph: A[
|
||||
bool,
|
||||
"Disable the prefill-phase CUDA graph. Convenience for --cuda-graph-backend-prefill=disabled.",
|
||||
|
||||
@@ -37,20 +37,6 @@ def handle_npu_backends(server_args: Any):
|
||||
|
||||
set_default_server_args(server_args)
|
||||
|
||||
current = cfg.cuda_graph_config.prefill.tc_compiler
|
||||
if current is not None and current != "eager":
|
||||
logger.warning(
|
||||
"At this moment Ascend platform only support prefill graph compilation with "
|
||||
"cuda_graph_config[prefill].tc_compiler='eager'."
|
||||
)
|
||||
declare_resolution(
|
||||
server_args,
|
||||
"_handle_npu_backends",
|
||||
cuda_graph_config=with_phase(
|
||||
cfg.cuda_graph_config, Phase.PREFILL, tc_compiler="eager"
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def handle_mps_backends(server_args: Any):
|
||||
cfg = resolving_view(server_args)
|
||||
|
||||
@@ -40,18 +40,18 @@ class Phase:
|
||||
class Backend:
|
||||
"""CUDA graph capture backends a phase can use."""
|
||||
|
||||
# Temporary internal name while unreachable legacy branches are removed.
|
||||
TC_PIECEWISE = "tc_piecewise"
|
||||
FULL = "full"
|
||||
BREAKABLE = "breakable"
|
||||
TC_PIECEWISE = "tc_piecewise"
|
||||
DISABLED = "disabled"
|
||||
ALL = (FULL, BREAKABLE, TC_PIECEWISE, DISABLED)
|
||||
ALL = (FULL, BREAKABLE, DISABLED)
|
||||
|
||||
|
||||
ALLOWED_BACKENDS_PER_PHASE = {
|
||||
Phase.DECODE: (
|
||||
Backend.FULL,
|
||||
Backend.BREAKABLE,
|
||||
Backend.TC_PIECEWISE,
|
||||
Backend.DISABLED,
|
||||
),
|
||||
# full for prefill captures one whole-forward graph per num_tokens
|
||||
@@ -61,26 +61,23 @@ ALLOWED_BACKENDS_PER_PHASE = {
|
||||
Phase.PREFILL: (
|
||||
Backend.FULL,
|
||||
Backend.BREAKABLE,
|
||||
Backend.TC_PIECEWISE,
|
||||
Backend.DISABLED,
|
||||
),
|
||||
}
|
||||
|
||||
# Per-phase settings schema. Keys other than backend are runner-level
|
||||
# (read by any backend in that phase); tc_compiler is the lone
|
||||
# backend-specific knob (only meaningful when backend == tc_piecewise).
|
||||
# (read by any backend in that phase).
|
||||
# For prefill, bs carries aggregate-token capture buckets for every backend;
|
||||
# full_prefill_max_req separately controls Full's fixed request-slot count.
|
||||
# full_prefill_max_req and full_prefill_prefix_chunk_tokens are prefill-only and
|
||||
# only meaningful when backend == full. max_context_size is shared by the
|
||||
# breakable and full prefill body-capture backends.
|
||||
ALLOWED_KEYS_PER_PHASE = {
|
||||
Phase.DECODE: ("backend", "max_bs", "bs", "tc_compiler"),
|
||||
Phase.DECODE: ("backend", "max_bs", "bs"),
|
||||
Phase.PREFILL: (
|
||||
"backend",
|
||||
"max_bs",
|
||||
"bs",
|
||||
"tc_compiler",
|
||||
"max_context_size",
|
||||
"full_prefill_max_req",
|
||||
"full_prefill_prefix_chunk_tokens",
|
||||
@@ -96,7 +93,7 @@ class PhaseConfig:
|
||||
backend: str = Backend.DISABLED
|
||||
max_bs: Optional[int] = None
|
||||
bs: Optional[List[int]] = None
|
||||
# Only meaningful when backend == tc_piecewise; ignored otherwise.
|
||||
# Internal compatibility only; rejected by both configuration parsers.
|
||||
tc_compiler: str = "eager"
|
||||
# Effective for both full and breakable backends and currently only DSV4:
|
||||
# fixed maximum context length used by context-shaped prefill graph metadata.
|
||||
@@ -105,7 +102,7 @@ class PhaseConfig:
|
||||
# Only meaningful for the prefill phase with backend == full: max number of
|
||||
# request slots baked into each captured graph. Real bs <= full_prefill_max_req
|
||||
# reuses the graph (unused slots become zero-length sentinels); larger
|
||||
# batches fall back to eager. Ignored by BCG and TC_PIECEWISE. None
|
||||
# batches fall back to eager. Ignored by BCG. None
|
||||
# auto-derives chunked_prefill_size // 512.
|
||||
full_prefill_max_req: Optional[int] = None
|
||||
# Only meaningful for Full prefill CUDA graphs that capture a distinct
|
||||
@@ -120,15 +117,10 @@ class PhaseConfig:
|
||||
|
||||
|
||||
def default_prefill_backend() -> str:
|
||||
"""BCG (breakable) is the prefill default on CUDA only; other platforms
|
||||
(HIP/NPU/...) keep tc_piecewise until BCG is validated there. Full-graph
|
||||
prefill capture is opt-in per model architecture via the declarative
|
||||
registry (see _inkling_overrides in arg_groups/overrides.py), not a global
|
||||
default. Lazy import keeps this module's stdlib-only import invariant (see
|
||||
module docstring)."""
|
||||
"""Enable BCG by default on CUDA; other platforms remain opt-in."""
|
||||
from sglang.srt.utils import is_cuda
|
||||
|
||||
return Backend.BREAKABLE if is_cuda() else Backend.TC_PIECEWISE
|
||||
return Backend.BREAKABLE if is_cuda() else Backend.DISABLED
|
||||
|
||||
|
||||
def with_phase(config: "CudaGraphConfig", phase: str, **changes) -> "CudaGraphConfig":
|
||||
@@ -188,6 +180,12 @@ class CudaGraphConfig:
|
||||
if phase not in Phase.ALL or not isinstance(phase_settings, dict):
|
||||
continue
|
||||
phase_cfg = getattr(cfg, phase)
|
||||
if phase_settings.get("backend") == "tc_piecewise":
|
||||
raise ValueError(
|
||||
"tc_piecewise was removed; select breakable, full, or disabled"
|
||||
)
|
||||
if "tc_compiler" in phase_settings:
|
||||
raise ValueError("tc_compiler was removed with tc_piecewise")
|
||||
allowed = ALLOWED_KEYS_PER_PHASE[phase]
|
||||
for key, value in phase_settings.items():
|
||||
if key in allowed:
|
||||
@@ -236,6 +234,15 @@ def cuda_graph_fully_disabled() -> bool:
|
||||
) and check_cuda_graph_backend(Phase.PREFILL, Backend.DISABLED)
|
||||
|
||||
|
||||
def parse_cuda_graph_backend_arg(raw: str) -> str:
|
||||
"""Reject the retired backend before argparse checks supported choices."""
|
||||
if raw == "tc_piecewise":
|
||||
raise argparse.ArgumentTypeError(
|
||||
"tc_piecewise was removed; select breakable, full, or disabled"
|
||||
)
|
||||
return raw
|
||||
|
||||
|
||||
def parse_cuda_graph_config_arg(raw: str) -> Dict[str, Dict[str, Any]]:
|
||||
"""argparse type for --cuda-graph-config: parse JSON dict of
|
||||
phase → settings dict. Each phase's settings dict is itself validated
|
||||
@@ -263,6 +270,8 @@ def parse_cuda_graph_config_arg(raw: str) -> Dict[str, Dict[str, Any]]:
|
||||
f"--cuda-graph-config['{phase}'] must be a JSON object, got "
|
||||
f"{type(phase_settings).__name__}"
|
||||
)
|
||||
if "backend" in phase_settings:
|
||||
parse_cuda_graph_backend_arg(phase_settings["backend"])
|
||||
allowed = ALLOWED_KEYS_PER_PHASE[phase]
|
||||
result[phase] = {}
|
||||
for key, value in phase_settings.items():
|
||||
|
||||
@@ -50,10 +50,6 @@ from sglang.srt.model_executor.forward_context import (
|
||||
get_token_to_kv_pool,
|
||||
)
|
||||
from sglang.srt.model_executor.runner.base_runner import BaseRunner
|
||||
from sglang.srt.model_executor.runner_backend_utils.tc_piecewise_cuda_graph import (
|
||||
enable_tc_piecewise_cuda_graph,
|
||||
set_tc_piecewise_forward_context,
|
||||
)
|
||||
from sglang.srt.model_executor.runner_utils import (
|
||||
maybe_publish_prefill_shared_read_done,
|
||||
)
|
||||
@@ -343,34 +339,7 @@ class EagerRunner(BaseRunner):
|
||||
else "extend"
|
||||
)
|
||||
with device_timer_ctx(model_runner.device_timer, category):
|
||||
pcg_runner = model_runner.prefill_cuda_graph_runner
|
||||
if (
|
||||
_is_hip
|
||||
and pcg_runner is not None
|
||||
and not isinstance(pcg_runner, EagerRunner)
|
||||
and not cp_active
|
||||
):
|
||||
# HIP PCG eager fallback: enter the PCG context so Dynamo guards
|
||||
# and PCG-specific MoE/attention paths stay consistent.
|
||||
with (
|
||||
enable_tc_piecewise_cuda_graph(),
|
||||
set_tc_piecewise_forward_context(
|
||||
forward_batch,
|
||||
model_runner.attention_layers,
|
||||
getattr(model_runner.model, "quant_config", None),
|
||||
model_runner.moe_layers,
|
||||
model_runner.moe_fusions,
|
||||
dsa_indexers=model_runner.dsa_indexers,
|
||||
mha_companion_layers=model_runner.mha_companion_layers,
|
||||
),
|
||||
):
|
||||
ret = model_runner.model.forward(
|
||||
forward_batch.input_ids,
|
||||
forward_batch.positions,
|
||||
forward_batch,
|
||||
**kwargs,
|
||||
)
|
||||
elif cp_active and (
|
||||
if cp_active and (
|
||||
not get_parallel().enable_cp_tp_group_sharing
|
||||
or getattr(model_runner.model, "prepare_cp_inputs", None) is not None
|
||||
):
|
||||
|
||||
@@ -34,9 +34,6 @@ from sglang.srt.model_executor.runner_backend.breakable_cuda_graph_backend impor
|
||||
from sglang.srt.model_executor.runner_backend.full_cuda_graph_backend import (
|
||||
FullCudaGraphBackend,
|
||||
)
|
||||
from sglang.srt.model_executor.runner_backend.tc_piecewise_cuda_graph_backend import (
|
||||
TcPiecewiseCudaGraphBackend,
|
||||
)
|
||||
from sglang.srt.platforms import current_platform
|
||||
from sglang.srt.runtime_context import get_exec
|
||||
|
||||
@@ -47,9 +44,6 @@ if TYPE_CHECKING:
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Track first occurrence of each fallback warning to avoid log spam.
|
||||
_TC_PIECEWISE_DECODE_FALLBACK_LOGGED = False
|
||||
|
||||
|
||||
def resolve_decode_backend(
|
||||
cuda_graph_runner: BaseCudaGraphRunner,
|
||||
@@ -90,14 +84,12 @@ def resolve_decode_backend(
|
||||
enable_memory_saver=enable_memory_saver,
|
||||
debug_eager=get_exec().graph.debug_cuda_graph,
|
||||
)
|
||||
if backend_name == Backend.TC_PIECEWISE:
|
||||
global _TC_PIECEWISE_DECODE_FALLBACK_LOGGED
|
||||
if not _TC_PIECEWISE_DECODE_FALLBACK_LOGGED:
|
||||
logger.warning(
|
||||
"cuda_graph_config decode='tc_piecewise' is not yet implemented; "
|
||||
"falling back to 'full'."
|
||||
)
|
||||
_TC_PIECEWISE_DECODE_FALLBACK_LOGGED = True
|
||||
|
||||
# Callers decide whether to construct a graph runner. Explicit construction
|
||||
# with a disabled config (e.g. the draft-runner test harness) historically
|
||||
# uses the full backend; retiring TCPCG must preserve that behavior.
|
||||
if backend_name not in (Backend.FULL, Backend.DISABLED):
|
||||
raise ValueError(f"Unsupported decode graph backend: {backend_name}")
|
||||
|
||||
full_backend_cls = None
|
||||
if current_platform.is_out_of_tree():
|
||||
@@ -116,9 +108,8 @@ def resolve_prefill_backend(
|
||||
cuda_graph_runner: BaseCudaGraphRunner,
|
||||
) -> BaseCudaGraphBackend:
|
||||
"""Pick a backend instance from cuda_graph_config['prefill']['backend']."""
|
||||
model_runner = cuda_graph_runner.model_runner
|
||||
cfg = get_exec().graph.cuda_graph_config
|
||||
backend_name = cfg.prefill.backend if cfg is not None else Backend.TC_PIECEWISE
|
||||
backend_name = cfg.prefill.backend if cfg is not None else Backend.BREAKABLE
|
||||
|
||||
if backend_name == Backend.BREAKABLE:
|
||||
return BreakableCudaGraphBackend(
|
||||
@@ -132,5 +123,4 @@ def resolve_prefill_backend(
|
||||
enable_memory_saver=get_exec().features.enable_memory_saver,
|
||||
reuse_output_buffer=True,
|
||||
)
|
||||
# Default: tc_piecewise.
|
||||
return TcPiecewiseCudaGraphBackend(cuda_graph_runner)
|
||||
raise ValueError(f"Unsupported prefill graph backend: {backend_name}")
|
||||
|
||||
+2
-3
@@ -52,9 +52,8 @@ def enable_breakable_cuda_graph():
|
||||
|
||||
|
||||
BCG_FAILURE_HINT = (
|
||||
"1. change to tc_piecewise by --cuda-graph-backend-prefill=tc_piecewise\n"
|
||||
"2. disable the prefill CUDA graph by --cuda-graph-backend-prefill=disabled\n"
|
||||
"3. if it is an OOM problem, set --mem-fraction-static to a smaller value "
|
||||
"1. disable the prefill CUDA graph by --cuda-graph-backend-prefill=disabled\n"
|
||||
"2. if it is an OOM problem, set --mem-fraction-static to a smaller value "
|
||||
"(e.g., 0.8 or 0.7) or set --cuda-graph-max-bs-prefill to a smaller value "
|
||||
"(e.g., 2048)\n"
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user