[3/N] elastic-ep: Recapture decode CUDA graphs after scale-up (#33723)

This commit is contained in:
Yoray Zack
2026-09-23 01:51:30 -07:00
committed by GitHub
parent aa0b65c7c7
commit de123f38bb
18 changed files with 561 additions and 52 deletions
+29 -8
View File
@@ -523,16 +523,37 @@ def handle_elastic_ep(server_args: Any):
f"(got pp_size={cfg.pp_size}); WORLD must not span PP stages."
)
decode_cuda_graph_disabled = (
cfg.cuda_graph_config.decode.backend == Backend.DISABLED
decode_backend = cfg.cuda_graph_config.decode.backend
assert decode_backend in (Backend.DISABLED, Backend.FULL), (
"Elastic EP runtime scale-up supports decode CUDA graph backend "
f"'full' or 'disabled' (got {decode_backend!r})."
)
prefill_cuda_graph_disabled = (
cfg.cuda_graph_config.prefill.backend == Backend.DISABLED
)
assert decode_cuda_graph_disabled and prefill_cuda_graph_disabled, (
"Elastic EP runtime scale-up requires decode and prefill CUDA "
"graphs to be disabled."
assert cfg.cuda_graph_config.prefill.backend == Backend.DISABLED, (
"Elastic EP runtime scale-up requires prefill CUDA graph to be disabled."
)
if decode_backend == Backend.FULL:
assert cfg.device == "cuda", (
"Elastic EP CUDA graph recapture requires CUDA "
f"(got device={cfg.device!r})."
)
assert cfg.speculative_algorithm is None, (
"Elastic EP CUDA graph recapture does not support speculative decoding."
)
assert not cfg.is_embedding, (
"Elastic EP CUDA graph recapture does not support embedding models."
)
assert cfg.dllm_algorithm is None, (
"Elastic EP CUDA graph recapture does not support diffusion models."
)
assert not cfg.encoder_only, (
"Elastic EP CUDA graph recapture does not support encoder-only models."
)
assert not cfg.forward_hooks, (
"Elastic EP CUDA graph recapture does not support forward hooks."
)
assert not cfg.enable_pdmux, (
"Elastic EP CUDA graph recapture does not support PDMux."
)
assert resolved.enable_dp_attention, (
"Elastic EP scale-up requires --enable-dp-attention; without it "
"the TP group is not equivalent to WORLD and the post-scale "
+30 -12
View File
@@ -5,6 +5,7 @@ import time
from dataclasses import dataclass
from typing import TYPE_CHECKING, Callable, Iterator, List, Optional
import msgspec
import torch
from sglang.srt.distributed import parallel_state
@@ -26,21 +27,34 @@ logger = logging.getLogger(__name__)
_SCALE_COHORT_KEY_PREFIX = "elastic_ep/scale_cohort"
def register_scale_cohort(rank_offset: int, target_ep_size: int) -> None:
class ScaleCohort(msgspec.Struct, frozen=True, kw_only=True):
target_ep_size: int
cuda_graph_enabled: bool
def register_scale_cohort(
rank_offset: int, target_ep_size: int, cuda_graph_enabled: bool
) -> None:
store = get_global_tcp_store()
if store is None:
raise RuntimeError("Elastic EP scale-up requires the global TCPStore.")
store.set(f"{_SCALE_COHORT_KEY_PREFIX}/{rank_offset}", str(target_ep_size).encode())
payload = msgspec.json.encode(
ScaleCohort(
target_ep_size=target_ep_size,
cuda_graph_enabled=cuda_graph_enabled,
)
)
store.set(f"{_SCALE_COHORT_KEY_PREFIX}/{rank_offset}", payload)
def get_scale_cohort_target(rank_offset: int) -> Optional[int]:
def get_scale_cohort(rank_offset: int) -> Optional[ScaleCohort]:
store = get_global_tcp_store()
if store is None:
return None
key = f"{_SCALE_COHORT_KEY_PREFIX}/{rank_offset}"
if not store.check([key]):
return None
return int(store.get(key).decode())
return msgspec.json.decode(store.get(key), type=ScaleCohort)
@dataclass
@@ -259,6 +273,17 @@ class ElasticEPStateManager:
return None
return inst.pending_ep_size
@classmethod
def get_data_plane_ep_size(cls) -> int:
inst = cls._instance
assert inst is not None, "Elastic EP state is not initialized."
if inst.pending_ep_size is not None and inst.scale_phase in (
"configuring_data_plane",
"syncing_new_world",
):
return inst.pending_ep_size
return inst.effective_ep_size
@classmethod
def get_scale_phase(cls) -> str:
inst = cls._instance
@@ -318,14 +343,7 @@ def elastic_expanded_world_enabled() -> bool:
return False
if get_parallel().max_ep_size is None:
return False
active_target_size = inst.effective_ep_size
if inst.pending_ep_size is not None and inst.scale_phase in (
"configuring_data_plane",
"syncing_new_world",
):
active_target_size = inst.pending_ep_size
return active_target_size > inst.original_ep_size
return ElasticEPStateManager.get_data_plane_ep_size() > inst.original_ep_size
def _refresh_ep_members() -> None:
@@ -202,6 +202,9 @@ class AttentionBackend(ABC):
"""Init the global shared states for cuda graph."""
raise NotImplementedError()
def validate_elastic_cuda_graph_recapture(self) -> None:
return
def init_forward_metadata_for_breakable_cuda_graph_capture(
self,
forward_batch: ForwardBatch,
@@ -331,12 +331,13 @@ class FlashAttentionBackend(AttentionBackend):
# Store head info for precomputing FA3 scheduler metadata
self.head_dim = model_runner.model_config.head_dim
attention_tp_size = get_parallel().attn_tp_size
self.num_attention_heads = (
model_runner.model_config.hf_text_config.num_attention_heads
// model_runner.tp_size
// attention_tp_size
)
self.num_kv_heads = model_runner.model_config.get_num_kv_heads(
model_runner.tp_size
attention_tp_size
)
_softcapping = getattr(
model_runner.model_config.hf_text_config, "attn_logit_softcapping", None
@@ -417,6 +418,12 @@ class FlashAttentionBackend(AttentionBackend):
num_splits=self.num_splits,
)
def validate_elastic_cuda_graph_recapture(self) -> None:
if self.use_mla and self.fa_impl_ver != 3:
raise ValueError(
"Elastic EP CUDA graph recapture with MLA requires FlashAttention 3."
)
def _mxfp8_sf_kwargs(self, layer, forward_batch, q_descale=None):
"""Block-scaled UE8M0 scale factors for the FA4 MXFP8 attention path.
@@ -2154,7 +2161,6 @@ class FlashAttentionBackend(AttentionBackend):
q_nope = q_all[:, :, : layer.v_head_dim]
q_rope = q_all[:, :, layer.v_head_dim :]
max_seqlen_q = metadata.max_seq_len_q
result = flash_attn_with_kvcache(
q=q_rope,
k_cache=k_rope_cache,
@@ -158,6 +158,9 @@ class HybridAttnBackend(AttentionBackend):
ForwardMode.TARGET_VERIFY
).update_verify_buffers_to_fill_after_draft(spec_info, cuda_graph_bs)
def validate_elastic_cuda_graph_recapture(self) -> None:
self.decode_backend.validate_elastic_cuda_graph_recapture()
def forward(
self,
q: Optional[torch.Tensor] = None, # For full attention
@@ -168,6 +168,10 @@ class TboAttnBackend(AttentionBackend):
for child in self.children:
child.on_after_cuda_graph_warmup()
def validate_elastic_cuda_graph_recapture(self) -> None:
for backend in (self.primary, *self.children):
backend.validate_elastic_cuda_graph_recapture()
def get_cuda_graph_seq_len_fill_value(self):
ans = self.primary.get_cuda_graph_seq_len_fill_value()
if not self._children_use_cuda_graph():
+10 -2
View File
@@ -42,6 +42,7 @@ from sglang.srt.layers.cp.utils import (
from sglang.srt.layers.dp_attention import (
attn_tp_all_gather_into_tensor,
attn_tp_reduce_scatter_tensor,
can_use_dp_reduce_scatter,
dp_gather_partial,
dp_gather_replicate,
dp_reduce_scatter_tensor,
@@ -932,7 +933,10 @@ class LayerCommunicator:
):
if should_use_dp_reduce_scatterv():
return True
if forward_batch.dp_padding_mode.is_max_len():
if (
forward_batch.dp_padding_mode.is_max_len()
and can_use_dp_reduce_scatter()
):
return True
# Prefill CP predicates must stay out of decode graph capture.
if forward_batch.forward_mode.is_context_parallel_extend() and (
@@ -1539,7 +1543,11 @@ class CommunicateSummableTensorPairFn:
output=hidden_states,
sizes=get_dp_global_num_tokens(),
)
elif allow_reduce_scatter and forward_batch.dp_padding_mode.is_max_len():
elif (
allow_reduce_scatter
and forward_batch.dp_padding_mode.is_max_len()
and can_use_dp_reduce_scatter()
):
dp_reduce_scatter_tensor(hidden_states, global_hidden_states)
else:
dp_scatter(hidden_states, global_hidden_states, forward_batch)
+9
View File
@@ -897,6 +897,15 @@ def dp_scatter(
memcpy(local_tokens, global_tokens, 0, local_start_pos, local_num_tokens, True)
def can_use_dp_reduce_scatter() -> bool:
"""Whether the fixed TP group tiles the current attention DP x TP layout."""
if not world_dp_gather_enabled():
return True
parallel = get_parallel()
return parallel.tp_size == parallel.dp_size * parallel.attn_tp_size
def dp_reduce_scatter_tensor(output: torch.Tensor, input: torch.Tensor):
_note_dp_gather_in_prefill_graph()
if is_dp_gatherv_active():
@@ -419,7 +419,7 @@ class _NixlEPDispatcherImpl(_NixlEPDispatcherImplBase):
if self._mask_buffer is not None:
buffer.query_mask_buffer(self._mask_buffer)
n = ElasticEPStateManager.get_effective_ep_size()
n = ElasticEPStateManager.get_data_plane_ep_size()
self.active_ranks[:n].copy_(1 - self._mask_buffer[:n])
self.packed_recv_count = self.handle = None
+7
View File
@@ -627,6 +627,10 @@ class TpModelWorker(BaseTpWorker):
can_run_cuda_graph=can_run_cuda_graph,
)
def _maybe_finalize_elastic_cuda_graph_scale(self) -> None:
if self.model_runner._elastic_cuda_graph_enabled():
self.model_runner.maybe_join_ep_ranks()
def forward_batch_generation(
self,
batch: Optional[ScheduleBatch],
@@ -707,6 +711,7 @@ class TpModelWorker(BaseTpWorker):
batch_result.next_token_ids = self.model_runner.sample(
logits_output, forward_batch
)
self._maybe_finalize_elastic_cuda_graph_scale()
return batch_result
batch_result.delay_sample_func = sample_batch_func
@@ -734,6 +739,7 @@ class TpModelWorker(BaseTpWorker):
logits_output, forward_batch
)
self._maybe_finalize_elastic_cuda_graph_scale()
return batch_result
else:
out = self.model_runner.forward(
@@ -766,6 +772,7 @@ class TpModelWorker(BaseTpWorker):
)
else:
next_token_ids = None
self._maybe_finalize_elastic_cuda_graph_scale()
batch_result = GenerationBatchResult(
logits_output=logits_output,
can_run_cuda_graph=can_run_cuda_graph,
@@ -1979,7 +1979,8 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
def enable_num_token_non_padded():
return get_parallel().moe_ep_size > 1
# Elastic joiners also need graph padding masked after joining WORLD.
return get_parallel().moe_ep_size > 1 or world_dp_gather_enabled()
def build_inner_fb_view(
+116 -14
View File
@@ -38,7 +38,7 @@ from sglang.srt.dllm.config import DllmConfig
from sglang.srt.elastic_ep.elastic_ep import (
ElasticEPStateManager,
get_healthy_expert_location_src_rank,
get_scale_cohort_target,
get_scale_cohort,
join_process_groups,
join_scale_process_group,
maybe_rebalance_after_rank_fault,
@@ -89,6 +89,8 @@ from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator
from sglang.srt.mem_cache.memory_pool import HybridReqToTokenPool, ReqToTokenPool
from sglang.srt.model_executor.cuda_graph_config import (
Backend,
Phase,
check_cuda_graph_backend,
cuda_graph_fully_disabled,
)
from sglang.srt.model_executor.forward_batch_info import (
@@ -116,6 +118,12 @@ from sglang.srt.model_executor.model_runner_components.cuda_graph_setup import (
capture_cuda_graphs,
capture_decode_graph,
capture_prefill_graph,
drop_elastic_cuda_graph_state,
finalize_cuda_graph_capture,
recapture_elastic_cuda_graph,
resolve_elastic_cuda_graph_config,
sync_elastic_cuda_graph_config,
validate_elastic_cuda_graph_recapture,
)
from sglang.srt.model_executor.model_runner_components.kv_pool_runtime import (
compute_post_capture_kv_resize,
@@ -493,6 +501,7 @@ class ModelRunner:
register_scale_cohort(
get_parallel().ep_join_rank_offset,
join_effective_ep_size,
self._elastic_cuda_graph_enabled(),
)
join_scale_process_group()
get_context().override("elastic_ep.scale_join", ep_size=join_effective_ep_size)
@@ -534,13 +543,6 @@ class ModelRunner:
state.snapshot_active_to_last()
state.sync_active_to_cpu()
state.scale_phase = "syncing_new_world"
self._elastic_scale_ready_barrier(
target_size=join_effective_ep_size,
log_tag="JOINER",
)
if state is not None:
state.scale_phase = "serving_expanded"
self._rearm_eplb_after_elastic_scale()
def init_msprobe(self):
self.msprobe_debugger = misc_utils.create_msprobe_debugger()
@@ -1105,8 +1107,21 @@ class ModelRunner:
# )
# warmup_all_flashinfer_megamoe_layers(self.model)
is_scale_joiner = get_exec().moe.is_ep_scale_joiner and not self.is_draft_worker
defer_decode_capture = (
is_scale_joiner
and capture_decode_cuda_graph
and self._elastic_cuda_graph_enabled()
)
if is_scale_joiner:
self._validate_elastic_cuda_graph_recapture()
capture = capture_cuda_graphs(
model_runner=self, capture_decode_cuda_graph=capture_decode_cuda_graph
model_runner=self,
capture_decode_cuda_graph=(
capture_decode_cuda_graph and not defer_decode_capture
),
finalize=not defer_decode_capture,
defer_distributed_setup=defer_decode_capture,
)
self.eager_runner = capture.eager_runner
self.prefill_cuda_graph_runner = capture.prefill.runner
@@ -1114,6 +1129,15 @@ class ModelRunner:
self.graph_memory_usage = capture.memory_usage
self.graph_time_usage = capture.time_usage
if is_scale_joiner:
if self._elastic_cuda_graph_enabled():
if defer_decode_capture:
self._recapture_elastic_cuda_graphs()
finalize_cuda_graph_capture(self)
# Scheduler startup calls this path even when CUDA graphs are disabled.
target_size = get_parallel().ep_join_rank_offset + self.tp_size
self._finalize_elastic_ep_joiner(target_size)
def init_routed_experts_capturer(self):
if self.is_draft_worker:
# Capture is target-only. The draft worker runs in the same process
@@ -1772,7 +1796,10 @@ class ModelRunner:
self.msprobe_debugger.stop()
self.msprobe_debugger.step()
if get_exec().moe.elastic_ep_backend is not None:
if (
get_exec().moe.elastic_ep_backend is not None
and not self._elastic_cuda_graph_enabled()
):
self.maybe_join_ep_ranks()
return output
@@ -2126,6 +2153,13 @@ class ModelRunner:
target_size,
)
def _finalize_elastic_ep_joiner(self, target_size: int) -> None:
self._elastic_scale_ready_barrier(target_size=target_size, log_tag="JOINER")
state = ElasticEPStateManager.instance()
if state is not None:
state.scale_phase = "serving_expanded"
self._rearm_eplb_after_elastic_scale()
def _finalize_scale_up(
self,
ranks_to_join: list[int],
@@ -2175,6 +2209,9 @@ class ModelRunner:
)
get_context().override("elastic_ep.scale", dp_size=target_size)
recapture_cuda_graph = self._elastic_cuda_graph_enabled()
if recapture_cuda_graph:
self._recapture_elastic_cuda_graphs()
ElasticEPStateManager.mark_syncing_new_world()
self._elastic_scale_ready_barrier(
target_size=target_size,
@@ -2200,6 +2237,49 @@ class ModelRunner:
ranks_to_join,
)
def _elastic_cuda_graph_enabled(self) -> bool:
parallel = get_parallel()
return (
get_exec().moe.elastic_ep_backend is not None
and parallel.max_ep_size is not None
and parallel.max_ep_size > parallel.tp_size
and check_cuda_graph_backend(Phase.DECODE, Backend.FULL)
)
def _validate_elastic_cuda_graph_recapture(self) -> None:
if not self._elastic_cuda_graph_enabled():
return
validate_elastic_cuda_graph_recapture()
self.attn_backend.validate_elastic_cuda_graph_recapture()
def _recapture_elastic_cuda_graphs(self) -> None:
if not self._elastic_cuda_graph_enabled():
return
sync_elastic_cuda_graph_config(
config=resolve_elastic_cuda_graph_config(self),
device=self.device,
world_group=dist.group.WORLD,
)
drop_elastic_cuda_graph_state(
decode_runner=self.decode_cuda_graph_runner,
eager_runner=self.eager_runner,
)
self.decode_cuda_graph_runner = None
capture = recapture_elastic_cuda_graph(model_runner=self)
self.decode_cuda_graph_runner = capture.runner
self.graph_memory_usage = replace_graph_memory_usage(
self.graph_memory_usage,
capture.memory_usage,
phases=("decode", "target_verify", "draft_decode"),
)
self.graph_time_usage = replace_graph_time_usage(
self.graph_time_usage,
capture.time_usage,
phases=("decode", "target_verify", "draft_decode"),
)
def maybe_join_ep_ranks(self) -> None:
if not ElasticEPStateManager.is_scaling():
return
@@ -2247,13 +2327,13 @@ class ModelRunner:
return
if state.scale_phase == "waiting_for_cohort":
cohort_target = get_scale_cohort_target(effective_size)
if cohort_target is None:
cohort = get_scale_cohort(effective_size)
if cohort is None:
return
if cohort_target != pending_size:
if cohort.target_ep_size != pending_size:
error = (
f"Requested target EP size {pending_size} does not match "
f"joining cohort target {cohort_target}"
f"joining cohort target {cohort.target_ep_size}"
)
ElasticEPStateManager.fail_scale(error)
self._reset_eplb_after_elastic_scale_failure()
@@ -2261,6 +2341,28 @@ class ModelRunner:
if self.tp_rank == 0 and not get_exec().moe.is_ep_scale_joiner:
logger.error("[Elastic EP] %s", error)
return
cuda_graph_enabled = self._elastic_cuda_graph_enabled()
if cohort.cuda_graph_enabled != cuda_graph_enabled:
error = (
"Primary and joining cohort must use the same CUDA graph "
"configuration for Elastic EP scale-up"
)
ElasticEPStateManager.fail_scale(error)
self._reset_eplb_after_elastic_scale_failure()
self._report_elastic_scale_failure(error, effective_size)
if self.tp_rank == 0 and not get_exec().moe.is_ep_scale_joiner:
logger.error("[Elastic EP] %s", error)
return
try:
self._validate_elastic_cuda_graph_recapture()
except ValueError as exc:
error = str(exc)
ElasticEPStateManager.fail_scale(error)
self._reset_eplb_after_elastic_scale_failure()
self._report_elastic_scale_failure(error, effective_size)
if self.tp_rank == 0 and not get_exec().moe.is_ep_scale_joiner:
logger.error("[Elastic EP] %s", error)
return
if not ElasticEPStateManager.begin_scale():
return
@@ -1,11 +1,15 @@
from __future__ import annotations
import gc
import hashlib
import logging
import time
from collections import defaultdict
from typing import TYPE_CHECKING, Any, Optional
import msgspec
import torch
import torch.distributed as dist
from sglang.srt.configs.model_config import ModelImpl
from sglang.srt.distributed.device_communicators.pynccl_allocator import (
@@ -34,6 +38,7 @@ from sglang.srt.model_executor.runner import (
EagerRunner,
PrefillCudaGraphRunner,
get_batch_sizes_to_capture,
set_global_graph_memory_pool,
)
from sglang.srt.model_loader.utils import resolve_language_model
from sglang.srt.platforms import current_platform
@@ -172,6 +177,91 @@ class CudaGraphsCapture(msgspec.Struct, frozen=True, kw_only=True):
)
class ElasticCudaGraphConfig(msgspec.Struct, frozen=True, kw_only=True):
decode_backend: Backend
capture_bs: tuple[int, ...] = ()
compile_bs: tuple[int, ...] = ()
attn_tp_size: int = 1
attn_cp_size: int = 1
attention_backend: str = ""
disable_padding: bool = False
enable_two_batch_overlap: bool = False
capture_hidden_mode: CaptureHiddenMode = CaptureHiddenMode.NULL
def signature(self) -> tuple:
if self.decode_backend != Backend.FULL:
return (self.decode_backend,)
return (
self.decode_backend,
self.capture_bs,
self.compile_bs,
self.attn_tp_size,
self.attn_cp_size,
self.attention_backend,
self.disable_padding,
self.enable_two_batch_overlap,
self.capture_hidden_mode,
)
def validate_elastic_cuda_graph_recapture() -> None:
if not check_cuda_graph_backend(Phase.PREFILL, Backend.DISABLED):
raise ValueError(
"Elastic EP CUDA graph recapture supports decode graphs only; "
"prefill CUDA graphs must be disabled."
)
def resolve_elastic_cuda_graph_config(
model_runner: ModelRunner,
) -> ElasticCudaGraphConfig:
decode_backend = (
Backend.FULL
if check_cuda_graph_backend(Phase.DECODE, Backend.FULL)
else Backend.DISABLED
)
capture_bs: tuple[int, ...] = ()
compile_bs: tuple[int, ...] = ()
if decode_backend == Backend.FULL:
resolved_capture_bs, resolved_compile_bs = get_batch_sizes_to_capture(
model_runner, model_runner.decode_num_tokens_per_req()
)
capture_bs = tuple(resolved_capture_bs)
compile_bs = tuple(resolved_compile_bs)
return ElasticCudaGraphConfig(
decode_backend=decode_backend,
capture_bs=capture_bs,
compile_bs=compile_bs,
attn_tp_size=get_parallel().attn_tp_size,
attn_cp_size=get_parallel().attn_cp_size,
attention_backend=model_runner.decode_attention_backend_str,
disable_padding=get_exec().graph.disable_cuda_graph_padding,
enable_two_batch_overlap=get_exec().overlap.enable_two_batch_overlap,
capture_hidden_mode=get_server_return_hidden_states_mode(),
)
def sync_elastic_cuda_graph_config(
*, config: ElasticCudaGraphConfig, device: torch.device, world_group: Any
) -> None:
signature = config.signature()
digest = hashlib.sha256(repr(signature).encode()).digest()
signature_hash = int.from_bytes(digest[:8], "little") & ((1 << 63) - 1)
local_hash = torch.tensor([signature_hash], dtype=torch.int64, device=device)
gathered_hashes = torch.empty(
dist.get_world_size(world_group), dtype=torch.int64, device=device
)
dist.all_gather_into_tensor(gathered_hashes, local_hash, group=world_group)
hashes = gathered_hashes.cpu().tolist()
if any(value != hashes[0] for value in hashes[1:]):
raise RuntimeError(
"Elastic EP scale-up requires matching CUDA graph configuration "
"and effective capture sizes on every rank; "
f"local_signature={signature}, gathered_hashes={hashes}."
)
def refresh_deep_gemm_layout_memory_budget(
model_runner: ModelRunner, *, only_if_initialized: bool = False
) -> None:
@@ -184,11 +274,14 @@ def refresh_deep_gemm_layout_memory_budget(
return
if only_if_initialized:
if not _deep_gemm_layout_memory_budget_initialized:
return
# Scale joiners refresh with all expanded ranks during recapture.
if get_exec().moe.ep_join_mode == "scale":
return
# Target and draft share the budget. Its pre-capture initialization
# already used a world-wide collective, so this guard is rank-uniform
# and also covers a draft-only DeepGEMM backend outside draft context.
if not _deep_gemm_layout_memory_budget_initialized:
return
else:
if model_runner.is_draft_worker:
moe_runner_backend = (
@@ -241,8 +334,40 @@ def refresh_deep_gemm_layout_memory_budget(
)
def recapture_elastic_cuda_graph(
*,
model_runner: ModelRunner,
) -> GraphCapture:
refresh_deep_gemm_layout_memory_budget(model_runner)
# Warmup may communicate, so every rank repeats it after the rendezvous.
model_runner._kernel_warmed_up = False
capture = capture_decode_graph(model_runner=model_runner)
if capture.runner is None:
raise RuntimeError(
"Elastic EP CUDA graph recapture did not create a decode graph runner."
)
current_platform.synchronize()
return capture
def drop_elastic_cuda_graph_state(
*, decode_runner: Optional[BaseRunner], eager_runner: EagerRunner
) -> None:
current_platform.synchronize()
if decode_runner is not None and decode_runner is not eager_runner:
decode_runner.backend.cleanup()
current_platform.synchronize()
set_global_graph_memory_pool(None)
gc.collect()
current_platform.empty_cache()
def capture_cuda_graphs(
*, model_runner: ModelRunner, capture_decode_cuda_graph: bool = True
*,
model_runner: ModelRunner,
capture_decode_cuda_graph: bool = True,
finalize: bool = True,
defer_distributed_setup: bool = False,
) -> CudaGraphsCapture:
"""Capture cuda graphs. Requires init_attention_backends() to have run.
@@ -262,8 +387,10 @@ def capture_cuda_graphs(
# it. Always built: it serves both the fully-disabled case (decode/prefill
# runners point at it) and the eager fallback when a cg runner can't run a
# batch.
eager_runner = EagerRunner(model_runner)
refresh_deep_gemm_layout_memory_budget(model_runner)
eager_runner = EagerRunner(model_runner, run_warmup=not defer_distributed_setup)
if not defer_distributed_setup:
refresh_deep_gemm_layout_memory_budget(model_runner)
# cuda-graph capture: prefill before decode, so both coalesce onto the
# eager buffer allocated above. (capture_prefill_graph routes prefill
@@ -294,6 +421,14 @@ def capture_cuda_graphs(
capture_time=0,
)
if finalize:
finalize_cuda_graph_capture(model_runner)
return CudaGraphsCapture(eager_runner=eager_runner, prefill=prefill, decode=decode)
def finalize_cuda_graph_capture(model_runner: ModelRunner) -> None:
"""Run setup that must happen after the final graph capture."""
# Register forward hooks AFTER cuda-graph capture so their tensor ops are
# not traced into any captured graph — capture stays hook-free and hooks
# fire only on the eager forward path (capture replay never runs Python
@@ -311,8 +446,6 @@ def capture_cuda_graphs(
if model_runner.canary_manager is not None and not model_runner.is_draft_worker:
model_runner.canary_manager.mark_init_finished()
return CudaGraphsCapture(eager_runner=eager_runner, prefill=prefill, decode=decode)
def capture_prefill_graph(
*,
@@ -82,7 +82,7 @@ if TYPE_CHECKING:
class EagerRunner(BaseRunner):
def __init__(self, model_runner: ModelRunner) -> None:
def __init__(self, model_runner: ModelRunner, *, run_warmup: bool = True) -> None:
super().__init__(model_runner)
mr = model_runner
sa = mr.server_args
@@ -153,7 +153,8 @@ class EagerRunner(BaseRunner):
dp_size=get_parallel().dp_size,
)
# Eager has no capture step, so warm up here (run-once via mr._kernel_warmed_up).
self.warmup()
if run_warmup:
self.warmup()
def _autotune_buffers(self) -> Tuple[Any, int]:
"""Decode-shaped dummy buffers (bs * num_tokens_per_req) for the warmup
@@ -209,6 +209,8 @@ class FullCudaGraphBackend(BaseCudaGraphBackend):
return self._outputs[shape_key]
def cleanup(self) -> None:
for graph in self._graphs.values():
graph.reset()
self._graphs.clear()
self._outputs.clear()
self._output_buffer = None