[PD] Validate Mooncake EFA allocator compatibility (#39973)
This commit is contained in:
@@ -1832,6 +1832,13 @@ def post_capture_kv_sizing_planned(server_args: Any) -> bool:
|
||||
return False
|
||||
if envs.SGLANG_MOONCAKE_CUSTOM_MEM_POOL.get() is not None:
|
||||
return False
|
||||
# Mooncake over EFA cannot register the CUDA VMM allocation used by
|
||||
# post-capture KV sizing. Fall back to the regular cudaMalloc-backed pool.
|
||||
if (
|
||||
cfg.disaggregation_transfer_backend == "mooncake"
|
||||
and envs.MOONCAKE_PROTOCOL.get().lower() == "efa"
|
||||
):
|
||||
return False
|
||||
|
||||
if (
|
||||
cfg.disaggregation_mode != "prefill"
|
||||
|
||||
@@ -43,6 +43,7 @@ from sglang.srt.disaggregation.common.utils import (
|
||||
unpack_int_lists,
|
||||
)
|
||||
from sglang.srt.disaggregation.mooncake.utils import (
|
||||
_validate_efa_allocator_compatibility,
|
||||
check_mooncake_custom_mem_pool_enabled,
|
||||
)
|
||||
from sglang.srt.disaggregation.utils import (
|
||||
@@ -224,6 +225,12 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
server_args: ServerArgs,
|
||||
is_mla_backend: Optional[bool] = False,
|
||||
):
|
||||
enable_custom_mem_pool, custom_mem_pool_type = (
|
||||
check_mooncake_custom_mem_pool_enabled()
|
||||
)
|
||||
_validate_efa_allocator_compatibility(
|
||||
enable_custom_mem_pool, custom_mem_pool_type
|
||||
)
|
||||
super().__init__(args, disaggregation_mode, server_args, is_mla_backend)
|
||||
self.init_engine()
|
||||
self.register_buffer_to_engine()
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
"""Mooncake-specific utilities for custom memory pool management."""
|
||||
|
||||
import logging
|
||||
import os
|
||||
from typing import Any, Optional, Tuple
|
||||
|
||||
import torch
|
||||
@@ -26,6 +27,41 @@ logger = logging.getLogger(__name__)
|
||||
SUPPORTED_MOONCAKE_CUSTOM_MEM_POOL_TYPES = ["NVLINK", "BAREX", "INTRA_NODE_NVLINK"]
|
||||
|
||||
|
||||
def _cuda_expandable_segments_enabled() -> Optional[str]:
|
||||
"""Return the allocator env var that enables CUDA expandable segments."""
|
||||
for var in ("PYTORCH_CUDA_ALLOC_CONF", "PYTORCH_ALLOC_CONF"):
|
||||
for field in os.environ.get(var, "").split(","):
|
||||
key, _, value = field.partition(":")
|
||||
if key.strip() == "expandable_segments" and value.strip().lower() == "true":
|
||||
return var
|
||||
return None
|
||||
|
||||
|
||||
def _validate_efa_allocator_compatibility(
|
||||
enable_custom_mem_pool: bool, custom_mem_pool_type: Optional[str]
|
||||
) -> None:
|
||||
"""Reject CUDA VMM allocators unsupported by the current libfabric EFA path."""
|
||||
if envs.MOONCAKE_PROTOCOL.get().lower() != "efa":
|
||||
return
|
||||
|
||||
if enable_custom_mem_pool and custom_mem_pool_type in ("NVLINK", "BAREX"):
|
||||
raise ValueError(
|
||||
f"SGLANG_MOONCAKE_CUSTOM_MEM_POOL={custom_mem_pool_type} is "
|
||||
"incompatible with MOONCAKE_PROTOCOL=efa. Mooncake custom memory "
|
||||
"pools use CUDA VMM allocations, which the current libfabric EFA "
|
||||
"provider cannot transfer. Unset SGLANG_MOONCAKE_CUSTOM_MEM_POOL."
|
||||
)
|
||||
|
||||
expandable_segments_var = _cuda_expandable_segments_enabled()
|
||||
if expandable_segments_var is not None:
|
||||
raise ValueError(
|
||||
f"{expandable_segments_var} enables expandable_segments, which is "
|
||||
"incompatible with MOONCAKE_PROTOCOL=efa because the current "
|
||||
"libfabric EFA provider cannot transfer CUDA VMM allocations. "
|
||||
"Disable expandable_segments."
|
||||
)
|
||||
|
||||
|
||||
def init_mooncake_custom_mem_pool(
|
||||
device: str,
|
||||
) -> Tuple[bool, Optional[Any], Optional[str]]:
|
||||
|
||||
Reference in New Issue
Block a user