[Perf] Bind scheduler CPUs to the GPU's NUMA node when memory policy is not permitted (#44023)

Co-authored-by: Mohammad Angkad <mohammad.angkad@radixark.ai>
This commit is contained in:
Mohammad Miadh Angkad
2026-10-14 00:01:32 -07:00
committed by GitHub
co-authored by Mohammad Angkad
parent 58f2c923ed
commit 6daae9c98e
5 changed files with 73 additions and 41 deletions
@@ -178,7 +178,7 @@ def _worker_cpu_intra_op_threads(num_gpus: int) -> int | None:
def _bind_worker_to_gpu_numa_node(
server_args: ServerArgs, local_rank: int
) -> int | None:
"""Bind this worker's CPUs and host memory to its GPU's NUMA node, as srt does.
"""Bind this worker's CPUs, and host memory when permitted, to its GPU's NUMA node.
Must run before CUDA init and model load so first-touch host memory lands
on that node; an unbound worker drifts across sockets and every
@@ -189,9 +189,7 @@ def _bind_worker_to_gpu_numa_node(
numa_bind_to_node,
)
# Pods without SYS_NICE cannot set memory policy; CPU affinity alone still
# removes the cross-socket drift (FLUX.1 TP2 on GB300: spread 26-60% -> 6-8%).
numa_node = get_numa_node_if_available(server_args, local_rank, cpu_only_ok=True)
numa_node = get_numa_node_if_available(server_args=server_args, gpu_id=local_rank)
if numa_node is not None:
numa_bind_to_node(numa_node)
logger.info("GPU %d worker bound to NUMA node %d", local_rank, numa_node)
+19 -24
View File
@@ -31,6 +31,9 @@ def configure_subprocess(server_args: ServerArgs, gpu_id: int):
if numa_node is not None:
# _numactl_cpu_mem_args returns None (warn/raise) on empty CPU intersection (#26983).
numactl_args = _numactl_cpu_mem_args(numa_node, gpu_id)
if numactl_args is not None and not _can_set_mempolicy():
numactl_args = _strip_memory_args(numactl_args)
_log_cpu_only_binding(numa_node)
if numactl_args is not None:
# Verify numactl can actually apply the binding before we exec it
# in front of the interpreter; relax the memory policy if not.
@@ -122,18 +125,14 @@ def _get_nvml_device_index(device_id: int) -> int:
return get_nvml_device_index(device_id)
def get_numa_node_if_available(
server_args: ServerArgs, gpu_id: int, cpu_only_ok: bool = False
) -> Optional[int]:
def get_numa_node_if_available(server_args: ServerArgs, gpu_id: int) -> Optional[int]:
"""
Returns the NUMA node for the given GPU id. If it is not set in the server_args, it will try to query the NUMA node for the GPU.
If the NUMA node is not available, has already been configured externally, or the user lacks permission to set NUMA affinity, it will return None.
If the NUMA node is not available or has already been configured externally, it will return None.
Args:
server_args: The server arguments.
gpu_id: The GPU id.
cpu_only_ok: Also return the node when NUMA memory policy is not
permitted, so the caller can still bind CPUs (no privilege needed).
Returns:
The NUMA node for the given GPU id or None if it is not available.
@@ -142,7 +141,7 @@ def get_numa_node_if_available(
return server_args.numa_node[gpu_id]
if not envs.SGLANG_AUTO_NUMA_BIND.get():
return None
if _is_numa_available(require_mempolicy=not cpu_only_ok):
if _is_numa_available():
queried_numa_node = _query_numa_node_for_gpu(gpu_id)
if len(queried_numa_node) == 0:
return None
@@ -187,10 +186,19 @@ def numa_bind_to_node(node: int):
os.sched_setaffinity(0, target_cpus)
else:
libnuma.numa_run_on_node(ctypes.c_int(node))
# Without memory-policy permission, CPU affinity alone keeps first-touch
# allocations on the node.
# Without memory policy, CPU affinity alone keeps first-touch allocations local;
# measured on FLUX.1 TP2 on GB300, run-to-run spread fell from 26-60% to 6-8%.
if _can_set_mempolicy():
libnuma.numa_set_preferred(ctypes.c_int(node))
else:
_log_cpu_only_binding(node)
def _log_cpu_only_binding(node: int) -> None:
logger.info(
f"NUMA memory policy not permitted; binding CPUs only to NUMA node {node}. "
"If using docker, --cap-add SYS_NICE also binds host memory."
)
class _Bitmask(ctypes.Structure):
@@ -366,11 +374,10 @@ def _can_set_mempolicy() -> bool:
return False
def _is_numa_available(require_mempolicy: bool = True) -> bool:
def _is_numa_available() -> bool:
"""
Check if NUMA is available and not already configured externally.
``require_mempolicy=False`` accepts a host where only CPU affinity can be set
(a container without SYS_NICE).
A host where only CPU affinity can be set (a container without SYS_NICE) counts.
"""
if not (_is_cuda or _is_xpu):
return False
@@ -385,18 +392,6 @@ def _is_numa_available(require_mempolicy: bool = True) -> bool:
)
return False
if not _can_set_mempolicy():
if not require_mempolicy:
logger.info(
"User lacks permission to set NUMA memory policy; binding CPUs only. "
"If using docker, --cap-add SYS_NICE also binds host memory."
)
return True
logger.warning(
"User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command."
)
return False
return True