fix: make automatic NUMA binding configurable (#30394)
This commit is contained in:
@@ -2158,13 +2158,13 @@ SGLang supports various environment variables that can be used to configure its
|
||||
<tbody>
|
||||
<tr>
|
||||
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}><code>SGLANG_NUMA_BIND_V2</code></td>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Use the v2 NUMA binding implementation.</td>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Select the NUMA binding implementation. <code>true</code> uses a pre-launch <code>numactl</code> wrapper; <code>false</code> binds in the worker process. This variable does not enable or disable NUMA binding.</td>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}><code>true</code></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}><code>SGLANG_AUTO_NUMA_BIND</code></td>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Automatically bind processes to NUMA nodes.</td>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}><code>false</code></td>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Automatically detect each NVIDIA GPU's local NUMA node and bind its worker when <code>--numa-node</code> is not specified. Set to <code>false</code> to leave workers unbound; an explicit <code>--numa-node</code> takes precedence.</td>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}><code>true</code></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}><code>SGLANG_CRASH_ON_NUMA_BIND_FAILURE</code></td>
|
||||
|
||||
@@ -1053,7 +1053,7 @@ class Envs:
|
||||
|
||||
# Numa
|
||||
SGLANG_NUMA_BIND_V2 = EnvBool(True)
|
||||
SGLANG_AUTO_NUMA_BIND = EnvBool(False)
|
||||
SGLANG_AUTO_NUMA_BIND = EnvBool(True)
|
||||
SGLANG_CRASH_ON_NUMA_BIND_FAILURE = EnvBool(False)
|
||||
|
||||
# Metrics
|
||||
|
||||
@@ -129,6 +129,8 @@ def get_numa_node_if_available(server_args: ServerArgs, gpu_id: int) -> Optional
|
||||
"""
|
||||
if server_args.numa_node is not None:
|
||||
return server_args.numa_node[gpu_id]
|
||||
if not envs.SGLANG_AUTO_NUMA_BIND.get():
|
||||
return None
|
||||
if _is_numa_available():
|
||||
queried_numa_node = _query_numa_node_for_gpu(gpu_id)
|
||||
if len(queried_numa_node) == 0:
|
||||
|
||||
@@ -4,6 +4,7 @@ import unittest
|
||||
from contextlib import ExitStack
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.utils.numa_utils import (
|
||||
_handle_numa_bind_failure,
|
||||
_is_numa_available,
|
||||
@@ -178,13 +179,34 @@ class TestGetNumaNodeIfAvailable(unittest.TestCase):
|
||||
args.numa_node = numa_node
|
||||
return args
|
||||
|
||||
def test_returns_explicit_numa_node_from_server_args(self):
|
||||
def test_auto_numa_bind_enabled_by_default(self):
|
||||
with patch.dict(os.environ, {}, clear=True):
|
||||
self.assertTrue(envs.SGLANG_AUTO_NUMA_BIND.get())
|
||||
|
||||
@patch.dict(os.environ, {"SGLANG_AUTO_NUMA_BIND": "0"})
|
||||
def test_returns_explicit_numa_node_when_auto_bind_disabled(self):
|
||||
args = self._make_server_args(numa_node=[2, 3, 0, 1])
|
||||
self.assertEqual(get_numa_node_if_available(args, 0), 2)
|
||||
self.assertEqual(get_numa_node_if_available(args, 1), 3)
|
||||
self.assertEqual(get_numa_node_if_available(args, 2), 0)
|
||||
self.assertEqual(get_numa_node_if_available(args, 3), 1)
|
||||
|
||||
@patch("sglang.srt.utils.numa_utils._query_numa_node_for_gpu")
|
||||
@patch("sglang.srt.utils.numa_utils._is_numa_available")
|
||||
def test_auto_bind_disabled_skips_numa_detection(self, mock_avail, mock_query):
|
||||
args = self._make_server_args(numa_node=None)
|
||||
for bind_v2 in ("0", "1"):
|
||||
with self.subTest(bind_v2=bind_v2), patch.dict(
|
||||
os.environ,
|
||||
{
|
||||
"SGLANG_AUTO_NUMA_BIND": "0",
|
||||
"SGLANG_NUMA_BIND_V2": bind_v2,
|
||||
},
|
||||
):
|
||||
self.assertIsNone(get_numa_node_if_available(args, 0))
|
||||
mock_avail.assert_not_called()
|
||||
mock_query.assert_not_called()
|
||||
|
||||
@patch("sglang.srt.utils.numa_utils._is_numa_available", return_value=False)
|
||||
def test_returns_none_when_numa_not_available(self, _mock_avail):
|
||||
args = self._make_server_args(numa_node=None)
|
||||
@@ -196,6 +218,7 @@ class TestGetNumaNodeIfAvailable(unittest.TestCase):
|
||||
args = self._make_server_args(numa_node=None)
|
||||
self.assertIsNone(get_numa_node_if_available(args, 0))
|
||||
|
||||
@patch.dict(os.environ, {"SGLANG_AUTO_NUMA_BIND": "1"})
|
||||
@patch("sglang.srt.utils.numa_utils._query_numa_node_for_gpu", return_value=[1])
|
||||
@patch("sglang.srt.utils.numa_utils._is_numa_available", return_value=True)
|
||||
def test_returns_queried_single_node(self, _mock_avail, _mock_gpu):
|
||||
|
||||
Reference in New Issue
Block a user