[PD] Validate Mooncake EFA allocator compatibility (#39973)

This commit is contained in:
Po-Han Huang (NVIDIA)
2026-09-22 15:12:29 +08:00
committed by GitHub
parent 1815e490de
commit 771c9d782d
5 changed files with 186 additions and 0 deletions
@@ -1832,6 +1832,13 @@ def post_capture_kv_sizing_planned(server_args: Any) -> bool:
return False
if envs.SGLANG_MOONCAKE_CUSTOM_MEM_POOL.get() is not None:
return False
# Mooncake over EFA cannot register the CUDA VMM allocation used by
# post-capture KV sizing. Fall back to the regular cudaMalloc-backed pool.
if (
cfg.disaggregation_transfer_backend == "mooncake"
and envs.MOONCAKE_PROTOCOL.get().lower() == "efa"
):
return False
if (
cfg.disaggregation_mode != "prefill"
@@ -43,6 +43,7 @@ from sglang.srt.disaggregation.common.utils import (
unpack_int_lists,
)
from sglang.srt.disaggregation.mooncake.utils import (
_validate_efa_allocator_compatibility,
check_mooncake_custom_mem_pool_enabled,
)
from sglang.srt.disaggregation.utils import (
@@ -224,6 +225,12 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
server_args: ServerArgs,
is_mla_backend: Optional[bool] = False,
):
enable_custom_mem_pool, custom_mem_pool_type = (
check_mooncake_custom_mem_pool_enabled()
)
_validate_efa_allocator_compatibility(
enable_custom_mem_pool, custom_mem_pool_type
)
super().__init__(args, disaggregation_mode, server_args, is_mla_backend)
self.init_engine()
self.register_buffer_to_engine()
@@ -14,6 +14,7 @@
"""Mooncake-specific utilities for custom memory pool management."""
import logging
import os
from typing import Any, Optional, Tuple
import torch
@@ -26,6 +27,41 @@ logger = logging.getLogger(__name__)
SUPPORTED_MOONCAKE_CUSTOM_MEM_POOL_TYPES = ["NVLINK", "BAREX", "INTRA_NODE_NVLINK"]
def _cuda_expandable_segments_enabled() -> Optional[str]:
"""Return the allocator env var that enables CUDA expandable segments."""
for var in ("PYTORCH_CUDA_ALLOC_CONF", "PYTORCH_ALLOC_CONF"):
for field in os.environ.get(var, "").split(","):
key, _, value = field.partition(":")
if key.strip() == "expandable_segments" and value.strip().lower() == "true":
return var
return None
def _validate_efa_allocator_compatibility(
enable_custom_mem_pool: bool, custom_mem_pool_type: Optional[str]
) -> None:
"""Reject CUDA VMM allocators unsupported by the current libfabric EFA path."""
if envs.MOONCAKE_PROTOCOL.get().lower() != "efa":
return
if enable_custom_mem_pool and custom_mem_pool_type in ("NVLINK", "BAREX"):
raise ValueError(
f"SGLANG_MOONCAKE_CUSTOM_MEM_POOL={custom_mem_pool_type} is "
"incompatible with MOONCAKE_PROTOCOL=efa. Mooncake custom memory "
"pools use CUDA VMM allocations, which the current libfabric EFA "
"provider cannot transfer. Unset SGLANG_MOONCAKE_CUSTOM_MEM_POOL."
)
expandable_segments_var = _cuda_expandable_segments_enabled()
if expandable_segments_var is not None:
raise ValueError(
f"{expandable_segments_var} enables expandable_segments, which is "
"incompatible with MOONCAKE_PROTOCOL=efa because the current "
"libfabric EFA provider cannot transfer CUDA VMM allocations. "
"Disable expandable_segments."
)
def init_mooncake_custom_mem_pool(
device: str,
) -> Tuple[bool, Optional[Any], Optional[str]]: