[3/N] elastic-ep: Recapture decode CUDA graphs after scale-up (#33723)
This commit is contained in:
@@ -523,16 +523,37 @@ def handle_elastic_ep(server_args: Any):
|
||||
f"(got pp_size={cfg.pp_size}); WORLD must not span PP stages."
|
||||
)
|
||||
|
||||
decode_cuda_graph_disabled = (
|
||||
cfg.cuda_graph_config.decode.backend == Backend.DISABLED
|
||||
decode_backend = cfg.cuda_graph_config.decode.backend
|
||||
assert decode_backend in (Backend.DISABLED, Backend.FULL), (
|
||||
"Elastic EP runtime scale-up supports decode CUDA graph backend "
|
||||
f"'full' or 'disabled' (got {decode_backend!r})."
|
||||
)
|
||||
prefill_cuda_graph_disabled = (
|
||||
cfg.cuda_graph_config.prefill.backend == Backend.DISABLED
|
||||
)
|
||||
assert decode_cuda_graph_disabled and prefill_cuda_graph_disabled, (
|
||||
"Elastic EP runtime scale-up requires decode and prefill CUDA "
|
||||
"graphs to be disabled."
|
||||
assert cfg.cuda_graph_config.prefill.backend == Backend.DISABLED, (
|
||||
"Elastic EP runtime scale-up requires prefill CUDA graph to be disabled."
|
||||
)
|
||||
if decode_backend == Backend.FULL:
|
||||
assert cfg.device == "cuda", (
|
||||
"Elastic EP CUDA graph recapture requires CUDA "
|
||||
f"(got device={cfg.device!r})."
|
||||
)
|
||||
assert cfg.speculative_algorithm is None, (
|
||||
"Elastic EP CUDA graph recapture does not support speculative decoding."
|
||||
)
|
||||
assert not cfg.is_embedding, (
|
||||
"Elastic EP CUDA graph recapture does not support embedding models."
|
||||
)
|
||||
assert cfg.dllm_algorithm is None, (
|
||||
"Elastic EP CUDA graph recapture does not support diffusion models."
|
||||
)
|
||||
assert not cfg.encoder_only, (
|
||||
"Elastic EP CUDA graph recapture does not support encoder-only models."
|
||||
)
|
||||
assert not cfg.forward_hooks, (
|
||||
"Elastic EP CUDA graph recapture does not support forward hooks."
|
||||
)
|
||||
assert not cfg.enable_pdmux, (
|
||||
"Elastic EP CUDA graph recapture does not support PDMux."
|
||||
)
|
||||
assert resolved.enable_dp_attention, (
|
||||
"Elastic EP scale-up requires --enable-dp-attention; without it "
|
||||
"the TP group is not equivalent to WORLD and the post-scale "
|
||||
|
||||
@@ -5,6 +5,7 @@ import time
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Callable, Iterator, List, Optional
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
from sglang.srt.distributed import parallel_state
|
||||
@@ -26,21 +27,34 @@ logger = logging.getLogger(__name__)
|
||||
_SCALE_COHORT_KEY_PREFIX = "elastic_ep/scale_cohort"
|
||||
|
||||
|
||||
def register_scale_cohort(rank_offset: int, target_ep_size: int) -> None:
|
||||
class ScaleCohort(msgspec.Struct, frozen=True, kw_only=True):
|
||||
target_ep_size: int
|
||||
cuda_graph_enabled: bool
|
||||
|
||||
|
||||
def register_scale_cohort(
|
||||
rank_offset: int, target_ep_size: int, cuda_graph_enabled: bool
|
||||
) -> None:
|
||||
store = get_global_tcp_store()
|
||||
if store is None:
|
||||
raise RuntimeError("Elastic EP scale-up requires the global TCPStore.")
|
||||
store.set(f"{_SCALE_COHORT_KEY_PREFIX}/{rank_offset}", str(target_ep_size).encode())
|
||||
payload = msgspec.json.encode(
|
||||
ScaleCohort(
|
||||
target_ep_size=target_ep_size,
|
||||
cuda_graph_enabled=cuda_graph_enabled,
|
||||
)
|
||||
)
|
||||
store.set(f"{_SCALE_COHORT_KEY_PREFIX}/{rank_offset}", payload)
|
||||
|
||||
|
||||
def get_scale_cohort_target(rank_offset: int) -> Optional[int]:
|
||||
def get_scale_cohort(rank_offset: int) -> Optional[ScaleCohort]:
|
||||
store = get_global_tcp_store()
|
||||
if store is None:
|
||||
return None
|
||||
key = f"{_SCALE_COHORT_KEY_PREFIX}/{rank_offset}"
|
||||
if not store.check([key]):
|
||||
return None
|
||||
return int(store.get(key).decode())
|
||||
return msgspec.json.decode(store.get(key), type=ScaleCohort)
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -259,6 +273,17 @@ class ElasticEPStateManager:
|
||||
return None
|
||||
return inst.pending_ep_size
|
||||
|
||||
@classmethod
|
||||
def get_data_plane_ep_size(cls) -> int:
|
||||
inst = cls._instance
|
||||
assert inst is not None, "Elastic EP state is not initialized."
|
||||
if inst.pending_ep_size is not None and inst.scale_phase in (
|
||||
"configuring_data_plane",
|
||||
"syncing_new_world",
|
||||
):
|
||||
return inst.pending_ep_size
|
||||
return inst.effective_ep_size
|
||||
|
||||
@classmethod
|
||||
def get_scale_phase(cls) -> str:
|
||||
inst = cls._instance
|
||||
@@ -318,14 +343,7 @@ def elastic_expanded_world_enabled() -> bool:
|
||||
return False
|
||||
if get_parallel().max_ep_size is None:
|
||||
return False
|
||||
active_target_size = inst.effective_ep_size
|
||||
if inst.pending_ep_size is not None and inst.scale_phase in (
|
||||
"configuring_data_plane",
|
||||
"syncing_new_world",
|
||||
):
|
||||
active_target_size = inst.pending_ep_size
|
||||
|
||||
return active_target_size > inst.original_ep_size
|
||||
return ElasticEPStateManager.get_data_plane_ep_size() > inst.original_ep_size
|
||||
|
||||
|
||||
def _refresh_ep_members() -> None:
|
||||
|
||||
@@ -202,6 +202,9 @@ class AttentionBackend(ABC):
|
||||
"""Init the global shared states for cuda graph."""
|
||||
raise NotImplementedError()
|
||||
|
||||
def validate_elastic_cuda_graph_recapture(self) -> None:
|
||||
return
|
||||
|
||||
def init_forward_metadata_for_breakable_cuda_graph_capture(
|
||||
self,
|
||||
forward_batch: ForwardBatch,
|
||||
|
||||
@@ -331,12 +331,13 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
|
||||
# Store head info for precomputing FA3 scheduler metadata
|
||||
self.head_dim = model_runner.model_config.head_dim
|
||||
attention_tp_size = get_parallel().attn_tp_size
|
||||
self.num_attention_heads = (
|
||||
model_runner.model_config.hf_text_config.num_attention_heads
|
||||
// model_runner.tp_size
|
||||
// attention_tp_size
|
||||
)
|
||||
self.num_kv_heads = model_runner.model_config.get_num_kv_heads(
|
||||
model_runner.tp_size
|
||||
attention_tp_size
|
||||
)
|
||||
_softcapping = getattr(
|
||||
model_runner.model_config.hf_text_config, "attn_logit_softcapping", None
|
||||
@@ -417,6 +418,12 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
num_splits=self.num_splits,
|
||||
)
|
||||
|
||||
def validate_elastic_cuda_graph_recapture(self) -> None:
|
||||
if self.use_mla and self.fa_impl_ver != 3:
|
||||
raise ValueError(
|
||||
"Elastic EP CUDA graph recapture with MLA requires FlashAttention 3."
|
||||
)
|
||||
|
||||
def _mxfp8_sf_kwargs(self, layer, forward_batch, q_descale=None):
|
||||
"""Block-scaled UE8M0 scale factors for the FA4 MXFP8 attention path.
|
||||
|
||||
@@ -2154,7 +2161,6 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
q_nope = q_all[:, :, : layer.v_head_dim]
|
||||
q_rope = q_all[:, :, layer.v_head_dim :]
|
||||
max_seqlen_q = metadata.max_seq_len_q
|
||||
|
||||
result = flash_attn_with_kvcache(
|
||||
q=q_rope,
|
||||
k_cache=k_rope_cache,
|
||||
|
||||
@@ -158,6 +158,9 @@ class HybridAttnBackend(AttentionBackend):
|
||||
ForwardMode.TARGET_VERIFY
|
||||
).update_verify_buffers_to_fill_after_draft(spec_info, cuda_graph_bs)
|
||||
|
||||
def validate_elastic_cuda_graph_recapture(self) -> None:
|
||||
self.decode_backend.validate_elastic_cuda_graph_recapture()
|
||||
|
||||
def forward(
|
||||
self,
|
||||
q: Optional[torch.Tensor] = None, # For full attention
|
||||
|
||||
@@ -168,6 +168,10 @@ class TboAttnBackend(AttentionBackend):
|
||||
for child in self.children:
|
||||
child.on_after_cuda_graph_warmup()
|
||||
|
||||
def validate_elastic_cuda_graph_recapture(self) -> None:
|
||||
for backend in (self.primary, *self.children):
|
||||
backend.validate_elastic_cuda_graph_recapture()
|
||||
|
||||
def get_cuda_graph_seq_len_fill_value(self):
|
||||
ans = self.primary.get_cuda_graph_seq_len_fill_value()
|
||||
if not self._children_use_cuda_graph():
|
||||
|
||||
@@ -42,6 +42,7 @@ from sglang.srt.layers.cp.utils import (
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
attn_tp_all_gather_into_tensor,
|
||||
attn_tp_reduce_scatter_tensor,
|
||||
can_use_dp_reduce_scatter,
|
||||
dp_gather_partial,
|
||||
dp_gather_replicate,
|
||||
dp_reduce_scatter_tensor,
|
||||
@@ -932,7 +933,10 @@ class LayerCommunicator:
|
||||
):
|
||||
if should_use_dp_reduce_scatterv():
|
||||
return True
|
||||
if forward_batch.dp_padding_mode.is_max_len():
|
||||
if (
|
||||
forward_batch.dp_padding_mode.is_max_len()
|
||||
and can_use_dp_reduce_scatter()
|
||||
):
|
||||
return True
|
||||
# Prefill CP predicates must stay out of decode graph capture.
|
||||
if forward_batch.forward_mode.is_context_parallel_extend() and (
|
||||
@@ -1539,7 +1543,11 @@ class CommunicateSummableTensorPairFn:
|
||||
output=hidden_states,
|
||||
sizes=get_dp_global_num_tokens(),
|
||||
)
|
||||
elif allow_reduce_scatter and forward_batch.dp_padding_mode.is_max_len():
|
||||
elif (
|
||||
allow_reduce_scatter
|
||||
and forward_batch.dp_padding_mode.is_max_len()
|
||||
and can_use_dp_reduce_scatter()
|
||||
):
|
||||
dp_reduce_scatter_tensor(hidden_states, global_hidden_states)
|
||||
else:
|
||||
dp_scatter(hidden_states, global_hidden_states, forward_batch)
|
||||
|
||||
@@ -897,6 +897,15 @@ def dp_scatter(
|
||||
memcpy(local_tokens, global_tokens, 0, local_start_pos, local_num_tokens, True)
|
||||
|
||||
|
||||
def can_use_dp_reduce_scatter() -> bool:
|
||||
"""Whether the fixed TP group tiles the current attention DP x TP layout."""
|
||||
if not world_dp_gather_enabled():
|
||||
return True
|
||||
|
||||
parallel = get_parallel()
|
||||
return parallel.tp_size == parallel.dp_size * parallel.attn_tp_size
|
||||
|
||||
|
||||
def dp_reduce_scatter_tensor(output: torch.Tensor, input: torch.Tensor):
|
||||
_note_dp_gather_in_prefill_graph()
|
||||
if is_dp_gatherv_active():
|
||||
|
||||
@@ -419,7 +419,7 @@ class _NixlEPDispatcherImpl(_NixlEPDispatcherImplBase):
|
||||
if self._mask_buffer is not None:
|
||||
buffer.query_mask_buffer(self._mask_buffer)
|
||||
|
||||
n = ElasticEPStateManager.get_effective_ep_size()
|
||||
n = ElasticEPStateManager.get_data_plane_ep_size()
|
||||
self.active_ranks[:n].copy_(1 - self._mask_buffer[:n])
|
||||
|
||||
self.packed_recv_count = self.handle = None
|
||||
|
||||
@@ -627,6 +627,10 @@ class TpModelWorker(BaseTpWorker):
|
||||
can_run_cuda_graph=can_run_cuda_graph,
|
||||
)
|
||||
|
||||
def _maybe_finalize_elastic_cuda_graph_scale(self) -> None:
|
||||
if self.model_runner._elastic_cuda_graph_enabled():
|
||||
self.model_runner.maybe_join_ep_ranks()
|
||||
|
||||
def forward_batch_generation(
|
||||
self,
|
||||
batch: Optional[ScheduleBatch],
|
||||
@@ -707,6 +711,7 @@ class TpModelWorker(BaseTpWorker):
|
||||
batch_result.next_token_ids = self.model_runner.sample(
|
||||
logits_output, forward_batch
|
||||
)
|
||||
self._maybe_finalize_elastic_cuda_graph_scale()
|
||||
return batch_result
|
||||
|
||||
batch_result.delay_sample_func = sample_batch_func
|
||||
@@ -734,6 +739,7 @@ class TpModelWorker(BaseTpWorker):
|
||||
logits_output, forward_batch
|
||||
)
|
||||
|
||||
self._maybe_finalize_elastic_cuda_graph_scale()
|
||||
return batch_result
|
||||
else:
|
||||
out = self.model_runner.forward(
|
||||
@@ -766,6 +772,7 @@ class TpModelWorker(BaseTpWorker):
|
||||
)
|
||||
else:
|
||||
next_token_ids = None
|
||||
self._maybe_finalize_elastic_cuda_graph_scale()
|
||||
batch_result = GenerationBatchResult(
|
||||
logits_output=logits_output,
|
||||
can_run_cuda_graph=can_run_cuda_graph,
|
||||
|
||||
@@ -1979,7 +1979,8 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
|
||||
|
||||
def enable_num_token_non_padded():
|
||||
return get_parallel().moe_ep_size > 1
|
||||
# Elastic joiners also need graph padding masked after joining WORLD.
|
||||
return get_parallel().moe_ep_size > 1 or world_dp_gather_enabled()
|
||||
|
||||
|
||||
def build_inner_fb_view(
|
||||
|
||||
@@ -38,7 +38,7 @@ from sglang.srt.dllm.config import DllmConfig
|
||||
from sglang.srt.elastic_ep.elastic_ep import (
|
||||
ElasticEPStateManager,
|
||||
get_healthy_expert_location_src_rank,
|
||||
get_scale_cohort_target,
|
||||
get_scale_cohort,
|
||||
join_process_groups,
|
||||
join_scale_process_group,
|
||||
maybe_rebalance_after_rank_fault,
|
||||
@@ -89,6 +89,8 @@ from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator
|
||||
from sglang.srt.mem_cache.memory_pool import HybridReqToTokenPool, ReqToTokenPool
|
||||
from sglang.srt.model_executor.cuda_graph_config import (
|
||||
Backend,
|
||||
Phase,
|
||||
check_cuda_graph_backend,
|
||||
cuda_graph_fully_disabled,
|
||||
)
|
||||
from sglang.srt.model_executor.forward_batch_info import (
|
||||
@@ -116,6 +118,12 @@ from sglang.srt.model_executor.model_runner_components.cuda_graph_setup import (
|
||||
capture_cuda_graphs,
|
||||
capture_decode_graph,
|
||||
capture_prefill_graph,
|
||||
drop_elastic_cuda_graph_state,
|
||||
finalize_cuda_graph_capture,
|
||||
recapture_elastic_cuda_graph,
|
||||
resolve_elastic_cuda_graph_config,
|
||||
sync_elastic_cuda_graph_config,
|
||||
validate_elastic_cuda_graph_recapture,
|
||||
)
|
||||
from sglang.srt.model_executor.model_runner_components.kv_pool_runtime import (
|
||||
compute_post_capture_kv_resize,
|
||||
@@ -493,6 +501,7 @@ class ModelRunner:
|
||||
register_scale_cohort(
|
||||
get_parallel().ep_join_rank_offset,
|
||||
join_effective_ep_size,
|
||||
self._elastic_cuda_graph_enabled(),
|
||||
)
|
||||
join_scale_process_group()
|
||||
get_context().override("elastic_ep.scale_join", ep_size=join_effective_ep_size)
|
||||
@@ -534,13 +543,6 @@ class ModelRunner:
|
||||
state.snapshot_active_to_last()
|
||||
state.sync_active_to_cpu()
|
||||
state.scale_phase = "syncing_new_world"
|
||||
self._elastic_scale_ready_barrier(
|
||||
target_size=join_effective_ep_size,
|
||||
log_tag="JOINER",
|
||||
)
|
||||
if state is not None:
|
||||
state.scale_phase = "serving_expanded"
|
||||
self._rearm_eplb_after_elastic_scale()
|
||||
|
||||
def init_msprobe(self):
|
||||
self.msprobe_debugger = misc_utils.create_msprobe_debugger()
|
||||
@@ -1105,8 +1107,21 @@ class ModelRunner:
|
||||
# )
|
||||
|
||||
# warmup_all_flashinfer_megamoe_layers(self.model)
|
||||
is_scale_joiner = get_exec().moe.is_ep_scale_joiner and not self.is_draft_worker
|
||||
defer_decode_capture = (
|
||||
is_scale_joiner
|
||||
and capture_decode_cuda_graph
|
||||
and self._elastic_cuda_graph_enabled()
|
||||
)
|
||||
if is_scale_joiner:
|
||||
self._validate_elastic_cuda_graph_recapture()
|
||||
capture = capture_cuda_graphs(
|
||||
model_runner=self, capture_decode_cuda_graph=capture_decode_cuda_graph
|
||||
model_runner=self,
|
||||
capture_decode_cuda_graph=(
|
||||
capture_decode_cuda_graph and not defer_decode_capture
|
||||
),
|
||||
finalize=not defer_decode_capture,
|
||||
defer_distributed_setup=defer_decode_capture,
|
||||
)
|
||||
self.eager_runner = capture.eager_runner
|
||||
self.prefill_cuda_graph_runner = capture.prefill.runner
|
||||
@@ -1114,6 +1129,15 @@ class ModelRunner:
|
||||
self.graph_memory_usage = capture.memory_usage
|
||||
self.graph_time_usage = capture.time_usage
|
||||
|
||||
if is_scale_joiner:
|
||||
if self._elastic_cuda_graph_enabled():
|
||||
if defer_decode_capture:
|
||||
self._recapture_elastic_cuda_graphs()
|
||||
finalize_cuda_graph_capture(self)
|
||||
# Scheduler startup calls this path even when CUDA graphs are disabled.
|
||||
target_size = get_parallel().ep_join_rank_offset + self.tp_size
|
||||
self._finalize_elastic_ep_joiner(target_size)
|
||||
|
||||
def init_routed_experts_capturer(self):
|
||||
if self.is_draft_worker:
|
||||
# Capture is target-only. The draft worker runs in the same process
|
||||
@@ -1772,7 +1796,10 @@ class ModelRunner:
|
||||
self.msprobe_debugger.stop()
|
||||
self.msprobe_debugger.step()
|
||||
|
||||
if get_exec().moe.elastic_ep_backend is not None:
|
||||
if (
|
||||
get_exec().moe.elastic_ep_backend is not None
|
||||
and not self._elastic_cuda_graph_enabled()
|
||||
):
|
||||
self.maybe_join_ep_ranks()
|
||||
|
||||
return output
|
||||
@@ -2126,6 +2153,13 @@ class ModelRunner:
|
||||
target_size,
|
||||
)
|
||||
|
||||
def _finalize_elastic_ep_joiner(self, target_size: int) -> None:
|
||||
self._elastic_scale_ready_barrier(target_size=target_size, log_tag="JOINER")
|
||||
state = ElasticEPStateManager.instance()
|
||||
if state is not None:
|
||||
state.scale_phase = "serving_expanded"
|
||||
self._rearm_eplb_after_elastic_scale()
|
||||
|
||||
def _finalize_scale_up(
|
||||
self,
|
||||
ranks_to_join: list[int],
|
||||
@@ -2175,6 +2209,9 @@ class ModelRunner:
|
||||
)
|
||||
get_context().override("elastic_ep.scale", dp_size=target_size)
|
||||
|
||||
recapture_cuda_graph = self._elastic_cuda_graph_enabled()
|
||||
if recapture_cuda_graph:
|
||||
self._recapture_elastic_cuda_graphs()
|
||||
ElasticEPStateManager.mark_syncing_new_world()
|
||||
self._elastic_scale_ready_barrier(
|
||||
target_size=target_size,
|
||||
@@ -2200,6 +2237,49 @@ class ModelRunner:
|
||||
ranks_to_join,
|
||||
)
|
||||
|
||||
def _elastic_cuda_graph_enabled(self) -> bool:
|
||||
parallel = get_parallel()
|
||||
return (
|
||||
get_exec().moe.elastic_ep_backend is not None
|
||||
and parallel.max_ep_size is not None
|
||||
and parallel.max_ep_size > parallel.tp_size
|
||||
and check_cuda_graph_backend(Phase.DECODE, Backend.FULL)
|
||||
)
|
||||
|
||||
def _validate_elastic_cuda_graph_recapture(self) -> None:
|
||||
if not self._elastic_cuda_graph_enabled():
|
||||
return
|
||||
|
||||
validate_elastic_cuda_graph_recapture()
|
||||
self.attn_backend.validate_elastic_cuda_graph_recapture()
|
||||
|
||||
def _recapture_elastic_cuda_graphs(self) -> None:
|
||||
if not self._elastic_cuda_graph_enabled():
|
||||
return
|
||||
|
||||
sync_elastic_cuda_graph_config(
|
||||
config=resolve_elastic_cuda_graph_config(self),
|
||||
device=self.device,
|
||||
world_group=dist.group.WORLD,
|
||||
)
|
||||
drop_elastic_cuda_graph_state(
|
||||
decode_runner=self.decode_cuda_graph_runner,
|
||||
eager_runner=self.eager_runner,
|
||||
)
|
||||
self.decode_cuda_graph_runner = None
|
||||
capture = recapture_elastic_cuda_graph(model_runner=self)
|
||||
self.decode_cuda_graph_runner = capture.runner
|
||||
self.graph_memory_usage = replace_graph_memory_usage(
|
||||
self.graph_memory_usage,
|
||||
capture.memory_usage,
|
||||
phases=("decode", "target_verify", "draft_decode"),
|
||||
)
|
||||
self.graph_time_usage = replace_graph_time_usage(
|
||||
self.graph_time_usage,
|
||||
capture.time_usage,
|
||||
phases=("decode", "target_verify", "draft_decode"),
|
||||
)
|
||||
|
||||
def maybe_join_ep_ranks(self) -> None:
|
||||
if not ElasticEPStateManager.is_scaling():
|
||||
return
|
||||
@@ -2247,13 +2327,13 @@ class ModelRunner:
|
||||
return
|
||||
|
||||
if state.scale_phase == "waiting_for_cohort":
|
||||
cohort_target = get_scale_cohort_target(effective_size)
|
||||
if cohort_target is None:
|
||||
cohort = get_scale_cohort(effective_size)
|
||||
if cohort is None:
|
||||
return
|
||||
if cohort_target != pending_size:
|
||||
if cohort.target_ep_size != pending_size:
|
||||
error = (
|
||||
f"Requested target EP size {pending_size} does not match "
|
||||
f"joining cohort target {cohort_target}"
|
||||
f"joining cohort target {cohort.target_ep_size}"
|
||||
)
|
||||
ElasticEPStateManager.fail_scale(error)
|
||||
self._reset_eplb_after_elastic_scale_failure()
|
||||
@@ -2261,6 +2341,28 @@ class ModelRunner:
|
||||
if self.tp_rank == 0 and not get_exec().moe.is_ep_scale_joiner:
|
||||
logger.error("[Elastic EP] %s", error)
|
||||
return
|
||||
cuda_graph_enabled = self._elastic_cuda_graph_enabled()
|
||||
if cohort.cuda_graph_enabled != cuda_graph_enabled:
|
||||
error = (
|
||||
"Primary and joining cohort must use the same CUDA graph "
|
||||
"configuration for Elastic EP scale-up"
|
||||
)
|
||||
ElasticEPStateManager.fail_scale(error)
|
||||
self._reset_eplb_after_elastic_scale_failure()
|
||||
self._report_elastic_scale_failure(error, effective_size)
|
||||
if self.tp_rank == 0 and not get_exec().moe.is_ep_scale_joiner:
|
||||
logger.error("[Elastic EP] %s", error)
|
||||
return
|
||||
try:
|
||||
self._validate_elastic_cuda_graph_recapture()
|
||||
except ValueError as exc:
|
||||
error = str(exc)
|
||||
ElasticEPStateManager.fail_scale(error)
|
||||
self._reset_eplb_after_elastic_scale_failure()
|
||||
self._report_elastic_scale_failure(error, effective_size)
|
||||
if self.tp_rank == 0 and not get_exec().moe.is_ep_scale_joiner:
|
||||
logger.error("[Elastic EP] %s", error)
|
||||
return
|
||||
if not ElasticEPStateManager.begin_scale():
|
||||
return
|
||||
|
||||
|
||||
@@ -1,11 +1,15 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import gc
|
||||
import hashlib
|
||||
import logging
|
||||
import time
|
||||
from collections import defaultdict
|
||||
from typing import TYPE_CHECKING, Any, Optional
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
|
||||
from sglang.srt.configs.model_config import ModelImpl
|
||||
from sglang.srt.distributed.device_communicators.pynccl_allocator import (
|
||||
@@ -34,6 +38,7 @@ from sglang.srt.model_executor.runner import (
|
||||
EagerRunner,
|
||||
PrefillCudaGraphRunner,
|
||||
get_batch_sizes_to_capture,
|
||||
set_global_graph_memory_pool,
|
||||
)
|
||||
from sglang.srt.model_loader.utils import resolve_language_model
|
||||
from sglang.srt.platforms import current_platform
|
||||
@@ -172,6 +177,91 @@ class CudaGraphsCapture(msgspec.Struct, frozen=True, kw_only=True):
|
||||
)
|
||||
|
||||
|
||||
class ElasticCudaGraphConfig(msgspec.Struct, frozen=True, kw_only=True):
|
||||
decode_backend: Backend
|
||||
capture_bs: tuple[int, ...] = ()
|
||||
compile_bs: tuple[int, ...] = ()
|
||||
attn_tp_size: int = 1
|
||||
attn_cp_size: int = 1
|
||||
attention_backend: str = ""
|
||||
disable_padding: bool = False
|
||||
enable_two_batch_overlap: bool = False
|
||||
capture_hidden_mode: CaptureHiddenMode = CaptureHiddenMode.NULL
|
||||
|
||||
def signature(self) -> tuple:
|
||||
if self.decode_backend != Backend.FULL:
|
||||
return (self.decode_backend,)
|
||||
return (
|
||||
self.decode_backend,
|
||||
self.capture_bs,
|
||||
self.compile_bs,
|
||||
self.attn_tp_size,
|
||||
self.attn_cp_size,
|
||||
self.attention_backend,
|
||||
self.disable_padding,
|
||||
self.enable_two_batch_overlap,
|
||||
self.capture_hidden_mode,
|
||||
)
|
||||
|
||||
|
||||
def validate_elastic_cuda_graph_recapture() -> None:
|
||||
if not check_cuda_graph_backend(Phase.PREFILL, Backend.DISABLED):
|
||||
raise ValueError(
|
||||
"Elastic EP CUDA graph recapture supports decode graphs only; "
|
||||
"prefill CUDA graphs must be disabled."
|
||||
)
|
||||
|
||||
|
||||
def resolve_elastic_cuda_graph_config(
|
||||
model_runner: ModelRunner,
|
||||
) -> ElasticCudaGraphConfig:
|
||||
decode_backend = (
|
||||
Backend.FULL
|
||||
if check_cuda_graph_backend(Phase.DECODE, Backend.FULL)
|
||||
else Backend.DISABLED
|
||||
)
|
||||
capture_bs: tuple[int, ...] = ()
|
||||
compile_bs: tuple[int, ...] = ()
|
||||
if decode_backend == Backend.FULL:
|
||||
resolved_capture_bs, resolved_compile_bs = get_batch_sizes_to_capture(
|
||||
model_runner, model_runner.decode_num_tokens_per_req()
|
||||
)
|
||||
capture_bs = tuple(resolved_capture_bs)
|
||||
compile_bs = tuple(resolved_compile_bs)
|
||||
|
||||
return ElasticCudaGraphConfig(
|
||||
decode_backend=decode_backend,
|
||||
capture_bs=capture_bs,
|
||||
compile_bs=compile_bs,
|
||||
attn_tp_size=get_parallel().attn_tp_size,
|
||||
attn_cp_size=get_parallel().attn_cp_size,
|
||||
attention_backend=model_runner.decode_attention_backend_str,
|
||||
disable_padding=get_exec().graph.disable_cuda_graph_padding,
|
||||
enable_two_batch_overlap=get_exec().overlap.enable_two_batch_overlap,
|
||||
capture_hidden_mode=get_server_return_hidden_states_mode(),
|
||||
)
|
||||
|
||||
|
||||
def sync_elastic_cuda_graph_config(
|
||||
*, config: ElasticCudaGraphConfig, device: torch.device, world_group: Any
|
||||
) -> None:
|
||||
signature = config.signature()
|
||||
digest = hashlib.sha256(repr(signature).encode()).digest()
|
||||
signature_hash = int.from_bytes(digest[:8], "little") & ((1 << 63) - 1)
|
||||
local_hash = torch.tensor([signature_hash], dtype=torch.int64, device=device)
|
||||
gathered_hashes = torch.empty(
|
||||
dist.get_world_size(world_group), dtype=torch.int64, device=device
|
||||
)
|
||||
dist.all_gather_into_tensor(gathered_hashes, local_hash, group=world_group)
|
||||
hashes = gathered_hashes.cpu().tolist()
|
||||
if any(value != hashes[0] for value in hashes[1:]):
|
||||
raise RuntimeError(
|
||||
"Elastic EP scale-up requires matching CUDA graph configuration "
|
||||
"and effective capture sizes on every rank; "
|
||||
f"local_signature={signature}, gathered_hashes={hashes}."
|
||||
)
|
||||
|
||||
|
||||
def refresh_deep_gemm_layout_memory_budget(
|
||||
model_runner: ModelRunner, *, only_if_initialized: bool = False
|
||||
) -> None:
|
||||
@@ -184,11 +274,14 @@ def refresh_deep_gemm_layout_memory_budget(
|
||||
return
|
||||
|
||||
if only_if_initialized:
|
||||
if not _deep_gemm_layout_memory_budget_initialized:
|
||||
return
|
||||
# Scale joiners refresh with all expanded ranks during recapture.
|
||||
if get_exec().moe.ep_join_mode == "scale":
|
||||
return
|
||||
# Target and draft share the budget. Its pre-capture initialization
|
||||
# already used a world-wide collective, so this guard is rank-uniform
|
||||
# and also covers a draft-only DeepGEMM backend outside draft context.
|
||||
if not _deep_gemm_layout_memory_budget_initialized:
|
||||
return
|
||||
else:
|
||||
if model_runner.is_draft_worker:
|
||||
moe_runner_backend = (
|
||||
@@ -241,8 +334,40 @@ def refresh_deep_gemm_layout_memory_budget(
|
||||
)
|
||||
|
||||
|
||||
def recapture_elastic_cuda_graph(
|
||||
*,
|
||||
model_runner: ModelRunner,
|
||||
) -> GraphCapture:
|
||||
refresh_deep_gemm_layout_memory_budget(model_runner)
|
||||
# Warmup may communicate, so every rank repeats it after the rendezvous.
|
||||
model_runner._kernel_warmed_up = False
|
||||
capture = capture_decode_graph(model_runner=model_runner)
|
||||
if capture.runner is None:
|
||||
raise RuntimeError(
|
||||
"Elastic EP CUDA graph recapture did not create a decode graph runner."
|
||||
)
|
||||
current_platform.synchronize()
|
||||
return capture
|
||||
|
||||
|
||||
def drop_elastic_cuda_graph_state(
|
||||
*, decode_runner: Optional[BaseRunner], eager_runner: EagerRunner
|
||||
) -> None:
|
||||
current_platform.synchronize()
|
||||
if decode_runner is not None and decode_runner is not eager_runner:
|
||||
decode_runner.backend.cleanup()
|
||||
current_platform.synchronize()
|
||||
set_global_graph_memory_pool(None)
|
||||
gc.collect()
|
||||
current_platform.empty_cache()
|
||||
|
||||
|
||||
def capture_cuda_graphs(
|
||||
*, model_runner: ModelRunner, capture_decode_cuda_graph: bool = True
|
||||
*,
|
||||
model_runner: ModelRunner,
|
||||
capture_decode_cuda_graph: bool = True,
|
||||
finalize: bool = True,
|
||||
defer_distributed_setup: bool = False,
|
||||
) -> CudaGraphsCapture:
|
||||
"""Capture cuda graphs. Requires init_attention_backends() to have run.
|
||||
|
||||
@@ -262,8 +387,10 @@ def capture_cuda_graphs(
|
||||
# it. Always built: it serves both the fully-disabled case (decode/prefill
|
||||
# runners point at it) and the eager fallback when a cg runner can't run a
|
||||
# batch.
|
||||
eager_runner = EagerRunner(model_runner)
|
||||
refresh_deep_gemm_layout_memory_budget(model_runner)
|
||||
eager_runner = EagerRunner(model_runner, run_warmup=not defer_distributed_setup)
|
||||
|
||||
if not defer_distributed_setup:
|
||||
refresh_deep_gemm_layout_memory_budget(model_runner)
|
||||
|
||||
# cuda-graph capture: prefill before decode, so both coalesce onto the
|
||||
# eager buffer allocated above. (capture_prefill_graph routes prefill
|
||||
@@ -294,6 +421,14 @@ def capture_cuda_graphs(
|
||||
capture_time=0,
|
||||
)
|
||||
|
||||
if finalize:
|
||||
finalize_cuda_graph_capture(model_runner)
|
||||
|
||||
return CudaGraphsCapture(eager_runner=eager_runner, prefill=prefill, decode=decode)
|
||||
|
||||
|
||||
def finalize_cuda_graph_capture(model_runner: ModelRunner) -> None:
|
||||
"""Run setup that must happen after the final graph capture."""
|
||||
# Register forward hooks AFTER cuda-graph capture so their tensor ops are
|
||||
# not traced into any captured graph — capture stays hook-free and hooks
|
||||
# fire only on the eager forward path (capture replay never runs Python
|
||||
@@ -311,8 +446,6 @@ def capture_cuda_graphs(
|
||||
if model_runner.canary_manager is not None and not model_runner.is_draft_worker:
|
||||
model_runner.canary_manager.mark_init_finished()
|
||||
|
||||
return CudaGraphsCapture(eager_runner=eager_runner, prefill=prefill, decode=decode)
|
||||
|
||||
|
||||
def capture_prefill_graph(
|
||||
*,
|
||||
|
||||
@@ -82,7 +82,7 @@ if TYPE_CHECKING:
|
||||
|
||||
|
||||
class EagerRunner(BaseRunner):
|
||||
def __init__(self, model_runner: ModelRunner) -> None:
|
||||
def __init__(self, model_runner: ModelRunner, *, run_warmup: bool = True) -> None:
|
||||
super().__init__(model_runner)
|
||||
mr = model_runner
|
||||
sa = mr.server_args
|
||||
@@ -153,7 +153,8 @@ class EagerRunner(BaseRunner):
|
||||
dp_size=get_parallel().dp_size,
|
||||
)
|
||||
# Eager has no capture step, so warm up here (run-once via mr._kernel_warmed_up).
|
||||
self.warmup()
|
||||
if run_warmup:
|
||||
self.warmup()
|
||||
|
||||
def _autotune_buffers(self) -> Tuple[Any, int]:
|
||||
"""Decode-shaped dummy buffers (bs * num_tokens_per_req) for the warmup
|
||||
|
||||
@@ -209,6 +209,8 @@ class FullCudaGraphBackend(BaseCudaGraphBackend):
|
||||
return self._outputs[shape_key]
|
||||
|
||||
def cleanup(self) -> None:
|
||||
for graph in self._graphs.values():
|
||||
graph.reset()
|
||||
self._graphs.clear()
|
||||
self._outputs.clear()
|
||||
self._output_buffer = None
|
||||
|
||||
Reference in New Issue
Block a user