diff --git a/docs/docs/references/environment_variables.mdx b/docs/docs/references/environment_variables.mdx
index 7bf0292142..4e25027e80 100644
--- a/docs/docs/references/environment_variables.mdx
+++ b/docs/docs/references/environment_variables.mdx
@@ -2251,7 +2251,7 @@ MPS and does not read them.
SGLANG_AUTO_NUMA_BIND |
- Automatically detect each NVIDIA GPU's local NUMA node and bind its worker when --numa-node is not specified. Set to false to leave workers unbound; an explicit --numa-node takes precedence. |
+ Automatically detect each NVIDIA GPU's local NUMA node and bind its worker when --numa-node is not specified. Without CAP_SYS_NICE, only CPU affinity is set. Set to false to leave workers unbound; an explicit --numa-node takes precedence. |
true |
diff --git a/docs/docs/sglang-diffusion/api/cli.mdx b/docs/docs/sglang-diffusion/api/cli.mdx
index 350a77006d..2609532a34 100644
--- a/docs/docs/sglang-diffusion/api/cli.mdx
+++ b/docs/docs/sglang-diffusion/api/cli.mdx
@@ -88,7 +88,7 @@ Use `sglang generate --help` and `sglang serve --help` for the full argument lis
- PEFT `adapter_config.json` semantics are applied automatically for named adapter slots, RSLoRA, and per-layer `alpha_pattern`. Adapters that require unsupported auxiliary parameters or runtime behavior, such as DoRA, fail before weight injection instead of silently using ordinary LoRA math.
- `--lora-merge-mode {auto|merge|dynamic}`: choose how LoRA is applied. `auto` statically merges regular weights and uses dynamic LoRA for FSDP-sharded weights to avoid full-gather peaks.
- `--num-gpus {N}`: number of GPUs to use
-- `--numa-node {N} [N ...]`: NUMA node for each GPU worker, the i-th value for the worker on local GPU i. When unset, each worker binds its CPUs and host memory to its GPU's NUMA node on multi-socket hosts, as srt does; set `SGLANG_AUTO_NUMA_BIND=0` to turn that off. Unbound workers on a two-socket host (e.g. GB200/GB300) can drift to the other socket, which slows host-bound steps and makes latency vary run to run.
+- `--numa-node {N} [N ...]`: NUMA node for each GPU worker, the i-th value for the worker on local GPU i. When unset, each worker binds its CPUs and host memory to its GPU's NUMA node on multi-socket hosts, as srt does (CPUs only without `CAP_SYS_NICE`); set `SGLANG_AUTO_NUMA_BIND=0` to turn that off. Unbound workers on a two-socket host (e.g. GB200/GB300) can drift to the other socket, which slows host-bound steps and makes latency vary run to run.
- `--performance-mode {manual|auto|speed|memory}` / `--mode`: preset for latency/throughput and memory defaults. `auto` is the default and dispatches residency from selected-GPU headroom and workload type: image DiTs stay resident above the 45 GiB threshold, while video DiT placement remains model-specific. It uses FSDP only for validated DiT-offload replacement paths. `speed` keeps `torch.compile` disabled unless a model-specific deployment config opts in after validation; pass `--enable-torch-compile true` to enable it explicitly. Use `manual` to keep performance-related server args under explicit user control. Explicit offload, FSDP, and parallelism flags take precedence in all modes.
- `--direct-gpu-weight-loading {true|false}`: opt into direct GPU loading for an unquantized, GPU-resident, TP=1 DiT by materializing its complete checkpoint state dict on GPU. Compatible tensors become parameter storage without another GPU copy, while tensors requiring transforms can still need temporary allocations. It is disabled by default; startup time and peak GPU memory remain model-dependent, so benchmark the target model before deployment. It is incompatible with DiT CPU/layerwise offload and FSDP.
- `--tp-size {N}`: tensor parallelism size. Depending on the pipeline, it can shard the DiT, one or more encoders, or both.
diff --git a/python/sglang/multimodal_gen/runtime/managers/gpu_worker.py b/python/sglang/multimodal_gen/runtime/managers/gpu_worker.py
index 8185017273..fe4c366f63 100644
--- a/python/sglang/multimodal_gen/runtime/managers/gpu_worker.py
+++ b/python/sglang/multimodal_gen/runtime/managers/gpu_worker.py
@@ -178,7 +178,7 @@ def _worker_cpu_intra_op_threads(num_gpus: int) -> int | None:
def _bind_worker_to_gpu_numa_node(
server_args: ServerArgs, local_rank: int
) -> int | None:
- """Bind this worker's CPUs and host memory to its GPU's NUMA node, as srt does.
+ """Bind this worker's CPUs, and host memory when permitted, to its GPU's NUMA node.
Must run before CUDA init and model load so first-touch host memory lands
on that node; an unbound worker drifts across sockets and every
@@ -189,9 +189,7 @@ def _bind_worker_to_gpu_numa_node(
numa_bind_to_node,
)
- # Pods without SYS_NICE cannot set memory policy; CPU affinity alone still
- # removes the cross-socket drift (FLUX.1 TP2 on GB300: spread 26-60% -> 6-8%).
- numa_node = get_numa_node_if_available(server_args, local_rank, cpu_only_ok=True)
+ numa_node = get_numa_node_if_available(server_args=server_args, gpu_id=local_rank)
if numa_node is not None:
numa_bind_to_node(numa_node)
logger.info("GPU %d worker bound to NUMA node %d", local_rank, numa_node)
diff --git a/python/sglang/srt/utils/numa_utils.py b/python/sglang/srt/utils/numa_utils.py
index 51814ec862..1f8817ae6c 100644
--- a/python/sglang/srt/utils/numa_utils.py
+++ b/python/sglang/srt/utils/numa_utils.py
@@ -31,6 +31,9 @@ def configure_subprocess(server_args: ServerArgs, gpu_id: int):
if numa_node is not None:
# _numactl_cpu_mem_args returns None (warn/raise) on empty CPU intersection (#26983).
numactl_args = _numactl_cpu_mem_args(numa_node, gpu_id)
+ if numactl_args is not None and not _can_set_mempolicy():
+ numactl_args = _strip_memory_args(numactl_args)
+ _log_cpu_only_binding(numa_node)
if numactl_args is not None:
# Verify numactl can actually apply the binding before we exec it
# in front of the interpreter; relax the memory policy if not.
@@ -122,18 +125,14 @@ def _get_nvml_device_index(device_id: int) -> int:
return get_nvml_device_index(device_id)
-def get_numa_node_if_available(
- server_args: ServerArgs, gpu_id: int, cpu_only_ok: bool = False
-) -> Optional[int]:
+def get_numa_node_if_available(server_args: ServerArgs, gpu_id: int) -> Optional[int]:
"""
Returns the NUMA node for the given GPU id. If it is not set in the server_args, it will try to query the NUMA node for the GPU.
- If the NUMA node is not available, has already been configured externally, or the user lacks permission to set NUMA affinity, it will return None.
+ If the NUMA node is not available or has already been configured externally, it will return None.
Args:
server_args: The server arguments.
gpu_id: The GPU id.
- cpu_only_ok: Also return the node when NUMA memory policy is not
- permitted, so the caller can still bind CPUs (no privilege needed).
Returns:
The NUMA node for the given GPU id or None if it is not available.
@@ -142,7 +141,7 @@ def get_numa_node_if_available(
return server_args.numa_node[gpu_id]
if not envs.SGLANG_AUTO_NUMA_BIND.get():
return None
- if _is_numa_available(require_mempolicy=not cpu_only_ok):
+ if _is_numa_available():
queried_numa_node = _query_numa_node_for_gpu(gpu_id)
if len(queried_numa_node) == 0:
return None
@@ -187,10 +186,19 @@ def numa_bind_to_node(node: int):
os.sched_setaffinity(0, target_cpus)
else:
libnuma.numa_run_on_node(ctypes.c_int(node))
- # Without memory-policy permission, CPU affinity alone keeps first-touch
- # allocations on the node.
+ # Without memory policy, CPU affinity alone keeps first-touch allocations local;
+ # measured on FLUX.1 TP2 on GB300, run-to-run spread fell from 26-60% to 6-8%.
if _can_set_mempolicy():
libnuma.numa_set_preferred(ctypes.c_int(node))
+ else:
+ _log_cpu_only_binding(node)
+
+
+def _log_cpu_only_binding(node: int) -> None:
+ logger.info(
+ f"NUMA memory policy not permitted; binding CPUs only to NUMA node {node}. "
+ "If using docker, --cap-add SYS_NICE also binds host memory."
+ )
class _Bitmask(ctypes.Structure):
@@ -366,11 +374,10 @@ def _can_set_mempolicy() -> bool:
return False
-def _is_numa_available(require_mempolicy: bool = True) -> bool:
+def _is_numa_available() -> bool:
"""
Check if NUMA is available and not already configured externally.
- ``require_mempolicy=False`` accepts a host where only CPU affinity can be set
- (a container without SYS_NICE).
+ A host where only CPU affinity can be set (a container without SYS_NICE) counts.
"""
if not (_is_cuda or _is_xpu):
return False
@@ -385,18 +392,6 @@ def _is_numa_available(require_mempolicy: bool = True) -> bool:
)
return False
- if not _can_set_mempolicy():
- if not require_mempolicy:
- logger.info(
- "User lacks permission to set NUMA memory policy; binding CPUs only. "
- "If using docker, --cap-add SYS_NICE also binds host memory."
- )
- return True
- logger.warning(
- "User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command."
- )
- return False
-
return True
diff --git a/test/registered/utils/test_numa_utils.py b/test/registered/utils/test_numa_utils.py
index 3db9d8abd3..30bc3d7782 100644
--- a/test/registered/utils/test_numa_utils.py
+++ b/test/registered/utils/test_numa_utils.py
@@ -6,6 +6,7 @@ from unittest.mock import MagicMock, patch
import torch
+from sglang.srt.environ import envs
from sglang.srt.utils.numa_utils import (
_handle_numa_bind_failure,
_is_numa_available,
@@ -48,15 +49,6 @@ class TestIsNumaAvailable(unittest.TestCase):
):
self.assertTrue(_is_numa_available())
- @patch("sglang.srt.utils.numa_utils._can_set_mempolicy", return_value=False)
- @patch("sglang.srt.utils.numa_utils.shutil.which", return_value="/usr/bin/numactl")
- @patch("sglang.srt.utils.numa_utils._is_cuda", True)
- @patch("os.path.isdir", return_value=True)
- def test_returns_false_when_mempolicy_not_permitted(
- self, _mock_isdir, _mock_which, _mock_mempolicy
- ):
- self.assertFalse(_is_numa_available())
-
@patch("sglang.srt.utils.numa_utils._can_set_mempolicy", return_value=False)
@patch("sglang.srt.utils.numa_utils.shutil.which", return_value="/usr/bin/numactl")
@patch("sglang.srt.utils.numa_utils._is_cuda", True)
@@ -64,7 +56,7 @@ class TestIsNumaAvailable(unittest.TestCase):
def test_cpu_only_accepted_when_mempolicy_not_permitted(
self, _mock_isdir, _mock_which, _mock_mempolicy
):
- self.assertTrue(_is_numa_available(require_mempolicy=False))
+ self.assertTrue(_is_numa_available())
class TestNumaBindWithoutMempolicy(unittest.TestCase):
@@ -88,12 +80,59 @@ class TestNumaBindWithoutMempolicy(unittest.TestCase):
return libnuma
def test_cpus_bound_and_memory_policy_skipped(self):
- self._bind(mempolicy_ok=False).numa_set_preferred.assert_not_called()
+ with self.assertLogs("sglang.srt.utils.numa_utils", level="INFO") as cm:
+ libnuma = self._bind(mempolicy_ok=False)
+ libnuma.numa_set_preferred.assert_not_called()
+ self.assertTrue(any("binding CPUs only" in msg for msg in cm.output))
def test_memory_policy_set_when_permitted(self):
self._bind(mempolicy_ok=True).numa_set_preferred.assert_called_once()
+class TestConfigureSubprocessCpuOnly(unittest.TestCase):
+ """Without memory-policy permission the numactl wrapper drops --membind and says so,
+ including for an explicit --numa-node, which skips the auto-detection path."""
+
+ def _configure(self, mempolicy_ok: bool) -> MagicMock:
+ server_args = MagicMock()
+ server_args.numa_node = [0]
+ with (
+ envs.SGLANG_NUMA_BIND_V2.override(True),
+ patch(
+ "sglang.srt.utils.numa_utils._numactl_cpu_mem_args",
+ return_value="--cpunodebind=0 --membind=0",
+ ),
+ patch(
+ "sglang.srt.utils.numa_utils._can_set_mempolicy",
+ return_value=mempolicy_ok,
+ ),
+ patch(
+ "sglang.srt.utils.numa_utils._probe_numactl_args",
+ side_effect=lambda args: (args, ""),
+ ) as probe,
+ patch(
+ "sglang.srt.utils.numa_utils._create_numactl_executable",
+ return_value=("numactl-exe", ""),
+ ),
+ patch("sglang.srt.utils.numa_utils._mp_set_executable"),
+ patch("sglang.srt.utils.numa_utils._get_nvml_device_index", return_value=0),
+ ):
+ with configure_subprocess(server_args=server_args, gpu_id=0):
+ pass
+ return probe
+
+ def test_memory_segment_stripped_and_logged(self):
+ with self.assertLogs("sglang.srt.utils.numa_utils", level="INFO") as cm:
+ probe = self._configure(mempolicy_ok=False)
+ probe.assert_called_once_with("--cpunodebind=0")
+ self.assertTrue(any("binding CPUs only" in msg for msg in cm.output))
+
+ def test_memory_segment_kept_when_permitted(self):
+ self._configure(mempolicy_ok=True).assert_called_once_with(
+ "--cpunodebind=0 --membind=0"
+ )
+
+
# Pin _is_xpu=False so these cases still reach the mocked pynvml on a real XPU
# host, where _query_numa_node_for_gpu short-circuits into the XPU sysfs branch.
@patch("sglang.srt.utils.numa_utils._is_xpu", False)