From de123f38bbc95ea9b14c5c01bab2cac89832b9e1 Mon Sep 17 00:00:00 2001 From: Yoray Zack Date: Wed, 23 Sep 2026 11:51:30 +0300 Subject: [PATCH] [3/N] elastic-ep: Recapture decode CUDA graphs after scale-up (#33723) --- python/sglang/srt/arg_groups/parallel_hook.py | 37 ++++- python/sglang/srt/elastic_ep/elastic_ep.py | 42 +++-- .../srt/layers/attention/base_attn_backend.py | 3 + .../attention/flashattention_backend.py | 12 +- .../layers/attention/hybrid_attn_backend.py | 3 + .../srt/layers/attention/tbo_backend.py | 4 + python/sglang/srt/layers/communicator.py | 12 +- python/sglang/srt/layers/dp_attention.py | 9 ++ .../srt/layers/moe/token_dispatcher/nixl.py | 2 +- python/sglang/srt/managers/tp_worker.py | 7 + .../srt/model_executor/forward_batch_info.py | 3 +- .../sglang/srt/model_executor/model_runner.py | 130 ++++++++++++++-- .../cuda_graph_setup.py | 147 +++++++++++++++++- .../srt/model_executor/runner/eager_runner.py | 5 +- .../runner_backend/full_cuda_graph_backend.py | 2 + test/manual/ep/test_elastic_scale.py | 68 +++++++- .../test_full_cuda_graph_backend.py | 19 +++ .../test_elastic_cuda_graph_setup.py | 108 +++++++++++++ 18 files changed, 561 insertions(+), 52 deletions(-) create mode 100644 test/registered/unit/model_executor/test_elastic_cuda_graph_setup.py diff --git a/python/sglang/srt/arg_groups/parallel_hook.py b/python/sglang/srt/arg_groups/parallel_hook.py index 20cb28fc8c..7cd2fb3bb1 100644 --- a/python/sglang/srt/arg_groups/parallel_hook.py +++ b/python/sglang/srt/arg_groups/parallel_hook.py @@ -523,16 +523,37 @@ def handle_elastic_ep(server_args: Any): f"(got pp_size={cfg.pp_size}); WORLD must not span PP stages." ) - decode_cuda_graph_disabled = ( - cfg.cuda_graph_config.decode.backend == Backend.DISABLED + decode_backend = cfg.cuda_graph_config.decode.backend + assert decode_backend in (Backend.DISABLED, Backend.FULL), ( + "Elastic EP runtime scale-up supports decode CUDA graph backend " + f"'full' or 'disabled' (got {decode_backend!r})." ) - prefill_cuda_graph_disabled = ( - cfg.cuda_graph_config.prefill.backend == Backend.DISABLED - ) - assert decode_cuda_graph_disabled and prefill_cuda_graph_disabled, ( - "Elastic EP runtime scale-up requires decode and prefill CUDA " - "graphs to be disabled." + assert cfg.cuda_graph_config.prefill.backend == Backend.DISABLED, ( + "Elastic EP runtime scale-up requires prefill CUDA graph to be disabled." ) + if decode_backend == Backend.FULL: + assert cfg.device == "cuda", ( + "Elastic EP CUDA graph recapture requires CUDA " + f"(got device={cfg.device!r})." + ) + assert cfg.speculative_algorithm is None, ( + "Elastic EP CUDA graph recapture does not support speculative decoding." + ) + assert not cfg.is_embedding, ( + "Elastic EP CUDA graph recapture does not support embedding models." + ) + assert cfg.dllm_algorithm is None, ( + "Elastic EP CUDA graph recapture does not support diffusion models." + ) + assert not cfg.encoder_only, ( + "Elastic EP CUDA graph recapture does not support encoder-only models." + ) + assert not cfg.forward_hooks, ( + "Elastic EP CUDA graph recapture does not support forward hooks." + ) + assert not cfg.enable_pdmux, ( + "Elastic EP CUDA graph recapture does not support PDMux." + ) assert resolved.enable_dp_attention, ( "Elastic EP scale-up requires --enable-dp-attention; without it " "the TP group is not equivalent to WORLD and the post-scale " diff --git a/python/sglang/srt/elastic_ep/elastic_ep.py b/python/sglang/srt/elastic_ep/elastic_ep.py index da33db8797..fc996713de 100644 --- a/python/sglang/srt/elastic_ep/elastic_ep.py +++ b/python/sglang/srt/elastic_ep/elastic_ep.py @@ -5,6 +5,7 @@ import time from dataclasses import dataclass from typing import TYPE_CHECKING, Callable, Iterator, List, Optional +import msgspec import torch from sglang.srt.distributed import parallel_state @@ -26,21 +27,34 @@ logger = logging.getLogger(__name__) _SCALE_COHORT_KEY_PREFIX = "elastic_ep/scale_cohort" -def register_scale_cohort(rank_offset: int, target_ep_size: int) -> None: +class ScaleCohort(msgspec.Struct, frozen=True, kw_only=True): + target_ep_size: int + cuda_graph_enabled: bool + + +def register_scale_cohort( + rank_offset: int, target_ep_size: int, cuda_graph_enabled: bool +) -> None: store = get_global_tcp_store() if store is None: raise RuntimeError("Elastic EP scale-up requires the global TCPStore.") - store.set(f"{_SCALE_COHORT_KEY_PREFIX}/{rank_offset}", str(target_ep_size).encode()) + payload = msgspec.json.encode( + ScaleCohort( + target_ep_size=target_ep_size, + cuda_graph_enabled=cuda_graph_enabled, + ) + ) + store.set(f"{_SCALE_COHORT_KEY_PREFIX}/{rank_offset}", payload) -def get_scale_cohort_target(rank_offset: int) -> Optional[int]: +def get_scale_cohort(rank_offset: int) -> Optional[ScaleCohort]: store = get_global_tcp_store() if store is None: return None key = f"{_SCALE_COHORT_KEY_PREFIX}/{rank_offset}" if not store.check([key]): return None - return int(store.get(key).decode()) + return msgspec.json.decode(store.get(key), type=ScaleCohort) @dataclass @@ -259,6 +273,17 @@ class ElasticEPStateManager: return None return inst.pending_ep_size + @classmethod + def get_data_plane_ep_size(cls) -> int: + inst = cls._instance + assert inst is not None, "Elastic EP state is not initialized." + if inst.pending_ep_size is not None and inst.scale_phase in ( + "configuring_data_plane", + "syncing_new_world", + ): + return inst.pending_ep_size + return inst.effective_ep_size + @classmethod def get_scale_phase(cls) -> str: inst = cls._instance @@ -318,14 +343,7 @@ def elastic_expanded_world_enabled() -> bool: return False if get_parallel().max_ep_size is None: return False - active_target_size = inst.effective_ep_size - if inst.pending_ep_size is not None and inst.scale_phase in ( - "configuring_data_plane", - "syncing_new_world", - ): - active_target_size = inst.pending_ep_size - - return active_target_size > inst.original_ep_size + return ElasticEPStateManager.get_data_plane_ep_size() > inst.original_ep_size def _refresh_ep_members() -> None: diff --git a/python/sglang/srt/layers/attention/base_attn_backend.py b/python/sglang/srt/layers/attention/base_attn_backend.py index 6279ce7db8..00da62095d 100644 --- a/python/sglang/srt/layers/attention/base_attn_backend.py +++ b/python/sglang/srt/layers/attention/base_attn_backend.py @@ -202,6 +202,9 @@ class AttentionBackend(ABC): """Init the global shared states for cuda graph.""" raise NotImplementedError() + def validate_elastic_cuda_graph_recapture(self) -> None: + return + def init_forward_metadata_for_breakable_cuda_graph_capture( self, forward_batch: ForwardBatch, diff --git a/python/sglang/srt/layers/attention/flashattention_backend.py b/python/sglang/srt/layers/attention/flashattention_backend.py index ee187bade0..a53ff0af39 100644 --- a/python/sglang/srt/layers/attention/flashattention_backend.py +++ b/python/sglang/srt/layers/attention/flashattention_backend.py @@ -331,12 +331,13 @@ class FlashAttentionBackend(AttentionBackend): # Store head info for precomputing FA3 scheduler metadata self.head_dim = model_runner.model_config.head_dim + attention_tp_size = get_parallel().attn_tp_size self.num_attention_heads = ( model_runner.model_config.hf_text_config.num_attention_heads - // model_runner.tp_size + // attention_tp_size ) self.num_kv_heads = model_runner.model_config.get_num_kv_heads( - model_runner.tp_size + attention_tp_size ) _softcapping = getattr( model_runner.model_config.hf_text_config, "attn_logit_softcapping", None @@ -417,6 +418,12 @@ class FlashAttentionBackend(AttentionBackend): num_splits=self.num_splits, ) + def validate_elastic_cuda_graph_recapture(self) -> None: + if self.use_mla and self.fa_impl_ver != 3: + raise ValueError( + "Elastic EP CUDA graph recapture with MLA requires FlashAttention 3." + ) + def _mxfp8_sf_kwargs(self, layer, forward_batch, q_descale=None): """Block-scaled UE8M0 scale factors for the FA4 MXFP8 attention path. @@ -2154,7 +2161,6 @@ class FlashAttentionBackend(AttentionBackend): q_nope = q_all[:, :, : layer.v_head_dim] q_rope = q_all[:, :, layer.v_head_dim :] max_seqlen_q = metadata.max_seq_len_q - result = flash_attn_with_kvcache( q=q_rope, k_cache=k_rope_cache, diff --git a/python/sglang/srt/layers/attention/hybrid_attn_backend.py b/python/sglang/srt/layers/attention/hybrid_attn_backend.py index 4d7f93b6c9..1ec7628b5a 100644 --- a/python/sglang/srt/layers/attention/hybrid_attn_backend.py +++ b/python/sglang/srt/layers/attention/hybrid_attn_backend.py @@ -158,6 +158,9 @@ class HybridAttnBackend(AttentionBackend): ForwardMode.TARGET_VERIFY ).update_verify_buffers_to_fill_after_draft(spec_info, cuda_graph_bs) + def validate_elastic_cuda_graph_recapture(self) -> None: + self.decode_backend.validate_elastic_cuda_graph_recapture() + def forward( self, q: Optional[torch.Tensor] = None, # For full attention diff --git a/python/sglang/srt/layers/attention/tbo_backend.py b/python/sglang/srt/layers/attention/tbo_backend.py index 64c65856ed..d48ca563fb 100644 --- a/python/sglang/srt/layers/attention/tbo_backend.py +++ b/python/sglang/srt/layers/attention/tbo_backend.py @@ -168,6 +168,10 @@ class TboAttnBackend(AttentionBackend): for child in self.children: child.on_after_cuda_graph_warmup() + def validate_elastic_cuda_graph_recapture(self) -> None: + for backend in (self.primary, *self.children): + backend.validate_elastic_cuda_graph_recapture() + def get_cuda_graph_seq_len_fill_value(self): ans = self.primary.get_cuda_graph_seq_len_fill_value() if not self._children_use_cuda_graph(): diff --git a/python/sglang/srt/layers/communicator.py b/python/sglang/srt/layers/communicator.py index 5a7d1e1c9f..eeb6a4604f 100644 --- a/python/sglang/srt/layers/communicator.py +++ b/python/sglang/srt/layers/communicator.py @@ -42,6 +42,7 @@ from sglang.srt.layers.cp.utils import ( from sglang.srt.layers.dp_attention import ( attn_tp_all_gather_into_tensor, attn_tp_reduce_scatter_tensor, + can_use_dp_reduce_scatter, dp_gather_partial, dp_gather_replicate, dp_reduce_scatter_tensor, @@ -932,7 +933,10 @@ class LayerCommunicator: ): if should_use_dp_reduce_scatterv(): return True - if forward_batch.dp_padding_mode.is_max_len(): + if ( + forward_batch.dp_padding_mode.is_max_len() + and can_use_dp_reduce_scatter() + ): return True # Prefill CP predicates must stay out of decode graph capture. if forward_batch.forward_mode.is_context_parallel_extend() and ( @@ -1539,7 +1543,11 @@ class CommunicateSummableTensorPairFn: output=hidden_states, sizes=get_dp_global_num_tokens(), ) - elif allow_reduce_scatter and forward_batch.dp_padding_mode.is_max_len(): + elif ( + allow_reduce_scatter + and forward_batch.dp_padding_mode.is_max_len() + and can_use_dp_reduce_scatter() + ): dp_reduce_scatter_tensor(hidden_states, global_hidden_states) else: dp_scatter(hidden_states, global_hidden_states, forward_batch) diff --git a/python/sglang/srt/layers/dp_attention.py b/python/sglang/srt/layers/dp_attention.py index 8c4f1eead3..39bbaad4fb 100644 --- a/python/sglang/srt/layers/dp_attention.py +++ b/python/sglang/srt/layers/dp_attention.py @@ -897,6 +897,15 @@ def dp_scatter( memcpy(local_tokens, global_tokens, 0, local_start_pos, local_num_tokens, True) +def can_use_dp_reduce_scatter() -> bool: + """Whether the fixed TP group tiles the current attention DP x TP layout.""" + if not world_dp_gather_enabled(): + return True + + parallel = get_parallel() + return parallel.tp_size == parallel.dp_size * parallel.attn_tp_size + + def dp_reduce_scatter_tensor(output: torch.Tensor, input: torch.Tensor): _note_dp_gather_in_prefill_graph() if is_dp_gatherv_active(): diff --git a/python/sglang/srt/layers/moe/token_dispatcher/nixl.py b/python/sglang/srt/layers/moe/token_dispatcher/nixl.py index 7b64b33818..6f71d4a387 100644 --- a/python/sglang/srt/layers/moe/token_dispatcher/nixl.py +++ b/python/sglang/srt/layers/moe/token_dispatcher/nixl.py @@ -419,7 +419,7 @@ class _NixlEPDispatcherImpl(_NixlEPDispatcherImplBase): if self._mask_buffer is not None: buffer.query_mask_buffer(self._mask_buffer) - n = ElasticEPStateManager.get_effective_ep_size() + n = ElasticEPStateManager.get_data_plane_ep_size() self.active_ranks[:n].copy_(1 - self._mask_buffer[:n]) self.packed_recv_count = self.handle = None diff --git a/python/sglang/srt/managers/tp_worker.py b/python/sglang/srt/managers/tp_worker.py index 67d1546d81..9fa3a10993 100644 --- a/python/sglang/srt/managers/tp_worker.py +++ b/python/sglang/srt/managers/tp_worker.py @@ -627,6 +627,10 @@ class TpModelWorker(BaseTpWorker): can_run_cuda_graph=can_run_cuda_graph, ) + def _maybe_finalize_elastic_cuda_graph_scale(self) -> None: + if self.model_runner._elastic_cuda_graph_enabled(): + self.model_runner.maybe_join_ep_ranks() + def forward_batch_generation( self, batch: Optional[ScheduleBatch], @@ -707,6 +711,7 @@ class TpModelWorker(BaseTpWorker): batch_result.next_token_ids = self.model_runner.sample( logits_output, forward_batch ) + self._maybe_finalize_elastic_cuda_graph_scale() return batch_result batch_result.delay_sample_func = sample_batch_func @@ -734,6 +739,7 @@ class TpModelWorker(BaseTpWorker): logits_output, forward_batch ) + self._maybe_finalize_elastic_cuda_graph_scale() return batch_result else: out = self.model_runner.forward( @@ -766,6 +772,7 @@ class TpModelWorker(BaseTpWorker): ) else: next_token_ids = None + self._maybe_finalize_elastic_cuda_graph_scale() batch_result = GenerationBatchResult( logits_output=logits_output, can_run_cuda_graph=can_run_cuda_graph, diff --git a/python/sglang/srt/model_executor/forward_batch_info.py b/python/sglang/srt/model_executor/forward_batch_info.py index a0d6c9444d..028c4fcb96 100644 --- a/python/sglang/srt/model_executor/forward_batch_info.py +++ b/python/sglang/srt/model_executor/forward_batch_info.py @@ -1979,7 +1979,8 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): def enable_num_token_non_padded(): - return get_parallel().moe_ep_size > 1 + # Elastic joiners also need graph padding masked after joining WORLD. + return get_parallel().moe_ep_size > 1 or world_dp_gather_enabled() def build_inner_fb_view( diff --git a/python/sglang/srt/model_executor/model_runner.py b/python/sglang/srt/model_executor/model_runner.py index 9f7a69c7b8..bd0da74046 100644 --- a/python/sglang/srt/model_executor/model_runner.py +++ b/python/sglang/srt/model_executor/model_runner.py @@ -38,7 +38,7 @@ from sglang.srt.dllm.config import DllmConfig from sglang.srt.elastic_ep.elastic_ep import ( ElasticEPStateManager, get_healthy_expert_location_src_rank, - get_scale_cohort_target, + get_scale_cohort, join_process_groups, join_scale_process_group, maybe_rebalance_after_rank_fault, @@ -89,6 +89,8 @@ from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator from sglang.srt.mem_cache.memory_pool import HybridReqToTokenPool, ReqToTokenPool from sglang.srt.model_executor.cuda_graph_config import ( Backend, + Phase, + check_cuda_graph_backend, cuda_graph_fully_disabled, ) from sglang.srt.model_executor.forward_batch_info import ( @@ -116,6 +118,12 @@ from sglang.srt.model_executor.model_runner_components.cuda_graph_setup import ( capture_cuda_graphs, capture_decode_graph, capture_prefill_graph, + drop_elastic_cuda_graph_state, + finalize_cuda_graph_capture, + recapture_elastic_cuda_graph, + resolve_elastic_cuda_graph_config, + sync_elastic_cuda_graph_config, + validate_elastic_cuda_graph_recapture, ) from sglang.srt.model_executor.model_runner_components.kv_pool_runtime import ( compute_post_capture_kv_resize, @@ -493,6 +501,7 @@ class ModelRunner: register_scale_cohort( get_parallel().ep_join_rank_offset, join_effective_ep_size, + self._elastic_cuda_graph_enabled(), ) join_scale_process_group() get_context().override("elastic_ep.scale_join", ep_size=join_effective_ep_size) @@ -534,13 +543,6 @@ class ModelRunner: state.snapshot_active_to_last() state.sync_active_to_cpu() state.scale_phase = "syncing_new_world" - self._elastic_scale_ready_barrier( - target_size=join_effective_ep_size, - log_tag="JOINER", - ) - if state is not None: - state.scale_phase = "serving_expanded" - self._rearm_eplb_after_elastic_scale() def init_msprobe(self): self.msprobe_debugger = misc_utils.create_msprobe_debugger() @@ -1105,8 +1107,21 @@ class ModelRunner: # ) # warmup_all_flashinfer_megamoe_layers(self.model) + is_scale_joiner = get_exec().moe.is_ep_scale_joiner and not self.is_draft_worker + defer_decode_capture = ( + is_scale_joiner + and capture_decode_cuda_graph + and self._elastic_cuda_graph_enabled() + ) + if is_scale_joiner: + self._validate_elastic_cuda_graph_recapture() capture = capture_cuda_graphs( - model_runner=self, capture_decode_cuda_graph=capture_decode_cuda_graph + model_runner=self, + capture_decode_cuda_graph=( + capture_decode_cuda_graph and not defer_decode_capture + ), + finalize=not defer_decode_capture, + defer_distributed_setup=defer_decode_capture, ) self.eager_runner = capture.eager_runner self.prefill_cuda_graph_runner = capture.prefill.runner @@ -1114,6 +1129,15 @@ class ModelRunner: self.graph_memory_usage = capture.memory_usage self.graph_time_usage = capture.time_usage + if is_scale_joiner: + if self._elastic_cuda_graph_enabled(): + if defer_decode_capture: + self._recapture_elastic_cuda_graphs() + finalize_cuda_graph_capture(self) + # Scheduler startup calls this path even when CUDA graphs are disabled. + target_size = get_parallel().ep_join_rank_offset + self.tp_size + self._finalize_elastic_ep_joiner(target_size) + def init_routed_experts_capturer(self): if self.is_draft_worker: # Capture is target-only. The draft worker runs in the same process @@ -1772,7 +1796,10 @@ class ModelRunner: self.msprobe_debugger.stop() self.msprobe_debugger.step() - if get_exec().moe.elastic_ep_backend is not None: + if ( + get_exec().moe.elastic_ep_backend is not None + and not self._elastic_cuda_graph_enabled() + ): self.maybe_join_ep_ranks() return output @@ -2126,6 +2153,13 @@ class ModelRunner: target_size, ) + def _finalize_elastic_ep_joiner(self, target_size: int) -> None: + self._elastic_scale_ready_barrier(target_size=target_size, log_tag="JOINER") + state = ElasticEPStateManager.instance() + if state is not None: + state.scale_phase = "serving_expanded" + self._rearm_eplb_after_elastic_scale() + def _finalize_scale_up( self, ranks_to_join: list[int], @@ -2175,6 +2209,9 @@ class ModelRunner: ) get_context().override("elastic_ep.scale", dp_size=target_size) + recapture_cuda_graph = self._elastic_cuda_graph_enabled() + if recapture_cuda_graph: + self._recapture_elastic_cuda_graphs() ElasticEPStateManager.mark_syncing_new_world() self._elastic_scale_ready_barrier( target_size=target_size, @@ -2200,6 +2237,49 @@ class ModelRunner: ranks_to_join, ) + def _elastic_cuda_graph_enabled(self) -> bool: + parallel = get_parallel() + return ( + get_exec().moe.elastic_ep_backend is not None + and parallel.max_ep_size is not None + and parallel.max_ep_size > parallel.tp_size + and check_cuda_graph_backend(Phase.DECODE, Backend.FULL) + ) + + def _validate_elastic_cuda_graph_recapture(self) -> None: + if not self._elastic_cuda_graph_enabled(): + return + + validate_elastic_cuda_graph_recapture() + self.attn_backend.validate_elastic_cuda_graph_recapture() + + def _recapture_elastic_cuda_graphs(self) -> None: + if not self._elastic_cuda_graph_enabled(): + return + + sync_elastic_cuda_graph_config( + config=resolve_elastic_cuda_graph_config(self), + device=self.device, + world_group=dist.group.WORLD, + ) + drop_elastic_cuda_graph_state( + decode_runner=self.decode_cuda_graph_runner, + eager_runner=self.eager_runner, + ) + self.decode_cuda_graph_runner = None + capture = recapture_elastic_cuda_graph(model_runner=self) + self.decode_cuda_graph_runner = capture.runner + self.graph_memory_usage = replace_graph_memory_usage( + self.graph_memory_usage, + capture.memory_usage, + phases=("decode", "target_verify", "draft_decode"), + ) + self.graph_time_usage = replace_graph_time_usage( + self.graph_time_usage, + capture.time_usage, + phases=("decode", "target_verify", "draft_decode"), + ) + def maybe_join_ep_ranks(self) -> None: if not ElasticEPStateManager.is_scaling(): return @@ -2247,13 +2327,13 @@ class ModelRunner: return if state.scale_phase == "waiting_for_cohort": - cohort_target = get_scale_cohort_target(effective_size) - if cohort_target is None: + cohort = get_scale_cohort(effective_size) + if cohort is None: return - if cohort_target != pending_size: + if cohort.target_ep_size != pending_size: error = ( f"Requested target EP size {pending_size} does not match " - f"joining cohort target {cohort_target}" + f"joining cohort target {cohort.target_ep_size}" ) ElasticEPStateManager.fail_scale(error) self._reset_eplb_after_elastic_scale_failure() @@ -2261,6 +2341,28 @@ class ModelRunner: if self.tp_rank == 0 and not get_exec().moe.is_ep_scale_joiner: logger.error("[Elastic EP] %s", error) return + cuda_graph_enabled = self._elastic_cuda_graph_enabled() + if cohort.cuda_graph_enabled != cuda_graph_enabled: + error = ( + "Primary and joining cohort must use the same CUDA graph " + "configuration for Elastic EP scale-up" + ) + ElasticEPStateManager.fail_scale(error) + self._reset_eplb_after_elastic_scale_failure() + self._report_elastic_scale_failure(error, effective_size) + if self.tp_rank == 0 and not get_exec().moe.is_ep_scale_joiner: + logger.error("[Elastic EP] %s", error) + return + try: + self._validate_elastic_cuda_graph_recapture() + except ValueError as exc: + error = str(exc) + ElasticEPStateManager.fail_scale(error) + self._reset_eplb_after_elastic_scale_failure() + self._report_elastic_scale_failure(error, effective_size) + if self.tp_rank == 0 and not get_exec().moe.is_ep_scale_joiner: + logger.error("[Elastic EP] %s", error) + return if not ElasticEPStateManager.begin_scale(): return diff --git a/python/sglang/srt/model_executor/model_runner_components/cuda_graph_setup.py b/python/sglang/srt/model_executor/model_runner_components/cuda_graph_setup.py index c47229d91d..50e1ad2ed8 100644 --- a/python/sglang/srt/model_executor/model_runner_components/cuda_graph_setup.py +++ b/python/sglang/srt/model_executor/model_runner_components/cuda_graph_setup.py @@ -1,11 +1,15 @@ from __future__ import annotations +import gc +import hashlib import logging import time from collections import defaultdict from typing import TYPE_CHECKING, Any, Optional import msgspec +import torch +import torch.distributed as dist from sglang.srt.configs.model_config import ModelImpl from sglang.srt.distributed.device_communicators.pynccl_allocator import ( @@ -34,6 +38,7 @@ from sglang.srt.model_executor.runner import ( EagerRunner, PrefillCudaGraphRunner, get_batch_sizes_to_capture, + set_global_graph_memory_pool, ) from sglang.srt.model_loader.utils import resolve_language_model from sglang.srt.platforms import current_platform @@ -172,6 +177,91 @@ class CudaGraphsCapture(msgspec.Struct, frozen=True, kw_only=True): ) +class ElasticCudaGraphConfig(msgspec.Struct, frozen=True, kw_only=True): + decode_backend: Backend + capture_bs: tuple[int, ...] = () + compile_bs: tuple[int, ...] = () + attn_tp_size: int = 1 + attn_cp_size: int = 1 + attention_backend: str = "" + disable_padding: bool = False + enable_two_batch_overlap: bool = False + capture_hidden_mode: CaptureHiddenMode = CaptureHiddenMode.NULL + + def signature(self) -> tuple: + if self.decode_backend != Backend.FULL: + return (self.decode_backend,) + return ( + self.decode_backend, + self.capture_bs, + self.compile_bs, + self.attn_tp_size, + self.attn_cp_size, + self.attention_backend, + self.disable_padding, + self.enable_two_batch_overlap, + self.capture_hidden_mode, + ) + + +def validate_elastic_cuda_graph_recapture() -> None: + if not check_cuda_graph_backend(Phase.PREFILL, Backend.DISABLED): + raise ValueError( + "Elastic EP CUDA graph recapture supports decode graphs only; " + "prefill CUDA graphs must be disabled." + ) + + +def resolve_elastic_cuda_graph_config( + model_runner: ModelRunner, +) -> ElasticCudaGraphConfig: + decode_backend = ( + Backend.FULL + if check_cuda_graph_backend(Phase.DECODE, Backend.FULL) + else Backend.DISABLED + ) + capture_bs: tuple[int, ...] = () + compile_bs: tuple[int, ...] = () + if decode_backend == Backend.FULL: + resolved_capture_bs, resolved_compile_bs = get_batch_sizes_to_capture( + model_runner, model_runner.decode_num_tokens_per_req() + ) + capture_bs = tuple(resolved_capture_bs) + compile_bs = tuple(resolved_compile_bs) + + return ElasticCudaGraphConfig( + decode_backend=decode_backend, + capture_bs=capture_bs, + compile_bs=compile_bs, + attn_tp_size=get_parallel().attn_tp_size, + attn_cp_size=get_parallel().attn_cp_size, + attention_backend=model_runner.decode_attention_backend_str, + disable_padding=get_exec().graph.disable_cuda_graph_padding, + enable_two_batch_overlap=get_exec().overlap.enable_two_batch_overlap, + capture_hidden_mode=get_server_return_hidden_states_mode(), + ) + + +def sync_elastic_cuda_graph_config( + *, config: ElasticCudaGraphConfig, device: torch.device, world_group: Any +) -> None: + signature = config.signature() + digest = hashlib.sha256(repr(signature).encode()).digest() + signature_hash = int.from_bytes(digest[:8], "little") & ((1 << 63) - 1) + local_hash = torch.tensor([signature_hash], dtype=torch.int64, device=device) + gathered_hashes = torch.empty( + dist.get_world_size(world_group), dtype=torch.int64, device=device + ) + dist.all_gather_into_tensor(gathered_hashes, local_hash, group=world_group) + hashes = gathered_hashes.cpu().tolist() + if any(value != hashes[0] for value in hashes[1:]): + raise RuntimeError( + "Elastic EP scale-up requires matching CUDA graph configuration " + "and effective capture sizes on every rank; " + f"local_signature={signature}, gathered_hashes={hashes}." + ) + + def refresh_deep_gemm_layout_memory_budget( model_runner: ModelRunner, *, only_if_initialized: bool = False ) -> None: @@ -184,11 +274,14 @@ def refresh_deep_gemm_layout_memory_budget( return if only_if_initialized: + if not _deep_gemm_layout_memory_budget_initialized: + return + # Scale joiners refresh with all expanded ranks during recapture. + if get_exec().moe.ep_join_mode == "scale": + return # Target and draft share the budget. Its pre-capture initialization # already used a world-wide collective, so this guard is rank-uniform # and also covers a draft-only DeepGEMM backend outside draft context. - if not _deep_gemm_layout_memory_budget_initialized: - return else: if model_runner.is_draft_worker: moe_runner_backend = ( @@ -241,8 +334,40 @@ def refresh_deep_gemm_layout_memory_budget( ) +def recapture_elastic_cuda_graph( + *, + model_runner: ModelRunner, +) -> GraphCapture: + refresh_deep_gemm_layout_memory_budget(model_runner) + # Warmup may communicate, so every rank repeats it after the rendezvous. + model_runner._kernel_warmed_up = False + capture = capture_decode_graph(model_runner=model_runner) + if capture.runner is None: + raise RuntimeError( + "Elastic EP CUDA graph recapture did not create a decode graph runner." + ) + current_platform.synchronize() + return capture + + +def drop_elastic_cuda_graph_state( + *, decode_runner: Optional[BaseRunner], eager_runner: EagerRunner +) -> None: + current_platform.synchronize() + if decode_runner is not None and decode_runner is not eager_runner: + decode_runner.backend.cleanup() + current_platform.synchronize() + set_global_graph_memory_pool(None) + gc.collect() + current_platform.empty_cache() + + def capture_cuda_graphs( - *, model_runner: ModelRunner, capture_decode_cuda_graph: bool = True + *, + model_runner: ModelRunner, + capture_decode_cuda_graph: bool = True, + finalize: bool = True, + defer_distributed_setup: bool = False, ) -> CudaGraphsCapture: """Capture cuda graphs. Requires init_attention_backends() to have run. @@ -262,8 +387,10 @@ def capture_cuda_graphs( # it. Always built: it serves both the fully-disabled case (decode/prefill # runners point at it) and the eager fallback when a cg runner can't run a # batch. - eager_runner = EagerRunner(model_runner) - refresh_deep_gemm_layout_memory_budget(model_runner) + eager_runner = EagerRunner(model_runner, run_warmup=not defer_distributed_setup) + + if not defer_distributed_setup: + refresh_deep_gemm_layout_memory_budget(model_runner) # cuda-graph capture: prefill before decode, so both coalesce onto the # eager buffer allocated above. (capture_prefill_graph routes prefill @@ -294,6 +421,14 @@ def capture_cuda_graphs( capture_time=0, ) + if finalize: + finalize_cuda_graph_capture(model_runner) + + return CudaGraphsCapture(eager_runner=eager_runner, prefill=prefill, decode=decode) + + +def finalize_cuda_graph_capture(model_runner: ModelRunner) -> None: + """Run setup that must happen after the final graph capture.""" # Register forward hooks AFTER cuda-graph capture so their tensor ops are # not traced into any captured graph — capture stays hook-free and hooks # fire only on the eager forward path (capture replay never runs Python @@ -311,8 +446,6 @@ def capture_cuda_graphs( if model_runner.canary_manager is not None and not model_runner.is_draft_worker: model_runner.canary_manager.mark_init_finished() - return CudaGraphsCapture(eager_runner=eager_runner, prefill=prefill, decode=decode) - def capture_prefill_graph( *, diff --git a/python/sglang/srt/model_executor/runner/eager_runner.py b/python/sglang/srt/model_executor/runner/eager_runner.py index 5221ab90e9..11a3df8016 100644 --- a/python/sglang/srt/model_executor/runner/eager_runner.py +++ b/python/sglang/srt/model_executor/runner/eager_runner.py @@ -82,7 +82,7 @@ if TYPE_CHECKING: class EagerRunner(BaseRunner): - def __init__(self, model_runner: ModelRunner) -> None: + def __init__(self, model_runner: ModelRunner, *, run_warmup: bool = True) -> None: super().__init__(model_runner) mr = model_runner sa = mr.server_args @@ -153,7 +153,8 @@ class EagerRunner(BaseRunner): dp_size=get_parallel().dp_size, ) # Eager has no capture step, so warm up here (run-once via mr._kernel_warmed_up). - self.warmup() + if run_warmup: + self.warmup() def _autotune_buffers(self) -> Tuple[Any, int]: """Decode-shaped dummy buffers (bs * num_tokens_per_req) for the warmup diff --git a/python/sglang/srt/model_executor/runner_backend/full_cuda_graph_backend.py b/python/sglang/srt/model_executor/runner_backend/full_cuda_graph_backend.py index 3a047c8ffa..f7829bcd6c 100644 --- a/python/sglang/srt/model_executor/runner_backend/full_cuda_graph_backend.py +++ b/python/sglang/srt/model_executor/runner_backend/full_cuda_graph_backend.py @@ -209,6 +209,8 @@ class FullCudaGraphBackend(BaseCudaGraphBackend): return self._outputs[shape_key] def cleanup(self) -> None: + for graph in self._graphs.values(): + graph.reset() self._graphs.clear() self._outputs.clear() self._output_buffer = None diff --git a/test/manual/ep/test_elastic_scale.py b/test/manual/ep/test_elastic_scale.py index cc50412870..6262be1e04 100644 --- a/test/manual/ep/test_elastic_scale.py +++ b/test/manual/ep/test_elastic_scale.py @@ -1,14 +1,18 @@ """Manual tests for elastic EP scale-up. Test classes: + TestElasticScaleUp4To5CudaGraph single-rank decode graph recapture TestElasticScaleUp4To6 primary + joiner scale-up (6 GPUs) TestElasticScaleUp4To5To6 two consecutive single-rank scale-ups TestElasticScaleUp4To8 full primary + joiner scale-up (8 GPUs) + TestElasticScaleUp4To6CudaGraph mixed-TP decode graph recapture + TestElasticScaleUp4To5To6CudaGraph repeated decode graph recapture + TestElasticScaleUp4To8CudaGraph scale-up with decode graph recapture -Run (8-GPU full scale-up): +Run (8-GPU CUDA graph scale-up): CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 python -m pytest \\ - test/manual/ep/test_elastic_scale.py::TestElasticScaleUp4To8 \\ + test/manual/ep/test_elastic_scale.py::TestElasticScaleUp4To8CudaGraph \\ -v -s """ @@ -52,6 +56,18 @@ DISABLED_CUDA_GRAPH_ARGS = [ "--cuda-graph-backend-prefill", "disabled", ] +ELASTIC_CUDA_GRAPH_ARGS = [ + "--cuda-graph-backend-decode", + "full", + "--cuda-graph-backend-prefill", + "disabled", + "--cuda-graph-bs-decode", + "1", + "2", + "4", + "8", + "16", +] def _assert_generate_logprob_ok(testcase: unittest.TestCase, base_url: str) -> None: @@ -476,5 +492,53 @@ class TestElasticScaleUp4To8(_ElasticScaleUpEndToEndBase): MOE_DENSE_TP_SIZE = None +@unittest.skipUnless( + _count_visible_gpus() >= 5, + "4-to-5 CUDA graph scale-up E2E needs 5 GPUs.", +) +class TestElasticScaleUp4To5CudaGraph(_ElasticScaleUpEndToEndBase): + """Scale from four to five ranks and recapture decode CUDA graphs.""" + + JOIN_TP = 1 + JOIN_NNODES = 2 + JOIN_NODE_RANK = 1 + TARGET_EP_SIZE = 5 + CUDA_GRAPH_ARGS = ELASTIC_CUDA_GRAPH_ARGS + + +@unittest.skipUnless( + _count_visible_gpus() >= MAX_EP_SIZE, + f"Full scale-up E2E needs {MAX_EP_SIZE} GPUs.", +) +class TestElasticScaleUp4To8CudaGraph(_ElasticScaleUpEndToEndBase): + """Scale from four to eight ranks and recapture decode CUDA graphs.""" + + JOIN_TP = LAUNCH_EP_SIZE + JOIN_NNODES = 2 + JOIN_NODE_RANK = 1 + TARGET_EP_SIZE = MAX_EP_SIZE + CUDA_GRAPH_ARGS = ELASTIC_CUDA_GRAPH_ARGS + + +@unittest.skipUnless( + _count_visible_gpus() >= 6, + "4-to-6 CUDA graph scale-up E2E needs 6 GPUs.", +) +class TestElasticScaleUp4To6CudaGraph(TestElasticScaleUp4To6): + """Recapture decode CUDA graphs across different local TP sizes.""" + + CUDA_GRAPH_ARGS = ELASTIC_CUDA_GRAPH_ARGS + + +@unittest.skipUnless( + _count_visible_gpus() >= 6, + "4-to-5-to-6 CUDA graph scale-up E2E needs 6 GPUs.", +) +class TestElasticScaleUp4To5To6CudaGraph(TestElasticScaleUp4To5To6): + """Recapture decode CUDA graphs across two scale operations.""" + + CUDA_GRAPH_ARGS = ELASTIC_CUDA_GRAPH_ARGS + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/model_executor/runner_backend/test_full_cuda_graph_backend.py b/test/registered/unit/model_executor/runner_backend/test_full_cuda_graph_backend.py index f653c8d9b0..8b9fb595a7 100644 --- a/test/registered/unit/model_executor/runner_backend/test_full_cuda_graph_backend.py +++ b/test/registered/unit/model_executor/runner_backend/test_full_cuda_graph_backend.py @@ -194,5 +194,24 @@ class TestCaptureOneWithProfiling(CustomTestCase): self.assertEqual(rf_names, []) +class TestCleanup(CustomTestCase): + def test_resets_graphs_before_releasing_references(self): + backend = _make_backend(_make_runner(enable_profile=False, profiler=None)) + graphs = [mock.Mock(), mock.Mock()] + backend._graphs = { + ShapeKey(size=i + 1): graph for i, graph in enumerate(graphs) + } + backend._outputs = {ShapeKey(size=1): object()} + backend._pool = object() + + backend.cleanup() + + for graph in graphs: + graph.reset.assert_called_once_with() + self.assertEqual(backend._graphs, {}) + self.assertEqual(backend._outputs, {}) + self.assertIsNone(backend._pool) + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/model_executor/test_elastic_cuda_graph_setup.py b/test/registered/unit/model_executor/test_elastic_cuda_graph_setup.py new file mode 100644 index 0000000000..9bd9bc93e3 --- /dev/null +++ b/test/registered/unit/model_executor/test_elastic_cuda_graph_setup.py @@ -0,0 +1,108 @@ +import unittest +from types import SimpleNamespace +from unittest.mock import Mock, patch + +import sglang.srt.model_executor.model_runner_components.cuda_graph_setup as graph_setup +from sglang.srt.model_executor.model_runner_components.cuda_graph_setup import ( + GraphCapture, +) +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=1, suite="base-a-test-cpu") + + +class TestElasticCudaGraphSetup(unittest.TestCase): + def test_deferred_capture_skips_distributed_setup(self): + model_runner = SimpleNamespace(is_draft_worker=False) + eager_runner = Mock() + prefill = GraphCapture( + runner=eager_runner, + memory_phase="prefill", + memory_usage_gb=0, + capture_time=0, + ) + + with ( + patch.object(graph_setup.GraphSharedOutput, "create_for_model_runner"), + patch.object( + graph_setup, "EagerRunner", return_value=eager_runner + ) as eager_cls, + patch.object(graph_setup, "capture_prefill_graph", return_value=prefill), + patch.object( + graph_setup, "refresh_deep_gemm_layout_memory_budget" + ) as setup, + patch.object(graph_setup, "finalize_cuda_graph_capture") as finalize, + ): + capture = graph_setup.capture_cuda_graphs( + model_runner=model_runner, + capture_decode_cuda_graph=False, + finalize=False, + defer_distributed_setup=True, + ) + + eager_cls.assert_called_once_with(model_runner, run_warmup=False) + setup.assert_not_called() + finalize.assert_not_called() + self.assertIs(capture.decode.runner, eager_runner) + + def test_recapture_repeats_warmup_after_rendezvous(self): + model_runner = SimpleNamespace(_kernel_warmed_up=True) + eager_runner = Mock() + decode_runner = Mock() + replacement = Mock() + capture = GraphCapture( + runner=replacement, + memory_phase="decode", + memory_usage_gb=1, + capture_time=2, + ) + + def capture_decode_graph(*, model_runner): + self.assertFalse(model_runner._kernel_warmed_up) + return capture + + with ( + patch.object(graph_setup.current_platform, "synchronize"), + patch.object(graph_setup.current_platform, "empty_cache"), + patch.object(graph_setup, "set_global_graph_memory_pool"), + patch.object(graph_setup.gc, "collect"), + patch.object(graph_setup, "refresh_deep_gemm_layout_memory_budget"), + patch.object( + graph_setup, + "capture_decode_graph", + side_effect=capture_decode_graph, + ), + ): + graph_setup.drop_elastic_cuda_graph_state( + decode_runner=decode_runner, + eager_runner=eager_runner, + ) + result = graph_setup.recapture_elastic_cuda_graph( + model_runner=model_runner, + ) + + decode_runner.backend.cleanup.assert_called_once_with() + self.assertIs(result, capture) + + def test_post_start_budget_refresh_skips_scale_joiner(self): + model_runner = SimpleNamespace(device="cuda") + exec_config = SimpleNamespace(moe=SimpleNamespace(ep_join_mode="scale")) + + with ( + patch.object( + graph_setup.envs.SGLANG_DEEPGEMM_STANDARD_LAYOUT, + "get", + return_value="auto", + ), + patch.object(graph_setup, "get_exec", return_value=exec_config), + patch.object(graph_setup, "get_available_gpu_memory") as get_memory, + ): + graph_setup.refresh_deep_gemm_layout_memory_budget( + model_runner, only_if_initialized=True + ) + + get_memory.assert_not_called() + + +if __name__ == "__main__": + unittest.main()