fix: make automatic NUMA binding configurable (#30394)
This commit is contained in:
@@ -2158,13 +2158,13 @@ SGLang supports various environment variables that can be used to configure its
|
|||||||
<tbody>
|
<tbody>
|
||||||
<tr>
|
<tr>
|
||||||
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}><code>SGLANG_NUMA_BIND_V2</code></td>
|
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}><code>SGLANG_NUMA_BIND_V2</code></td>
|
||||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Use the v2 NUMA binding implementation.</td>
|
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Select the NUMA binding implementation. <code>true</code> uses a pre-launch <code>numactl</code> wrapper; <code>false</code> binds in the worker process. This variable does not enable or disable NUMA binding.</td>
|
||||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}><code>true</code></td>
|
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}><code>true</code></td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr>
|
<tr>
|
||||||
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}><code>SGLANG_AUTO_NUMA_BIND</code></td>
|
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}><code>SGLANG_AUTO_NUMA_BIND</code></td>
|
||||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Automatically bind processes to NUMA nodes.</td>
|
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Automatically detect each NVIDIA GPU's local NUMA node and bind its worker when <code>--numa-node</code> is not specified. Set to <code>false</code> to leave workers unbound; an explicit <code>--numa-node</code> takes precedence.</td>
|
||||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}><code>false</code></td>
|
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}><code>true</code></td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr>
|
<tr>
|
||||||
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}><code>SGLANG_CRASH_ON_NUMA_BIND_FAILURE</code></td>
|
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}><code>SGLANG_CRASH_ON_NUMA_BIND_FAILURE</code></td>
|
||||||
|
|||||||
@@ -1053,7 +1053,7 @@ class Envs:
|
|||||||
|
|
||||||
# Numa
|
# Numa
|
||||||
SGLANG_NUMA_BIND_V2 = EnvBool(True)
|
SGLANG_NUMA_BIND_V2 = EnvBool(True)
|
||||||
SGLANG_AUTO_NUMA_BIND = EnvBool(False)
|
SGLANG_AUTO_NUMA_BIND = EnvBool(True)
|
||||||
SGLANG_CRASH_ON_NUMA_BIND_FAILURE = EnvBool(False)
|
SGLANG_CRASH_ON_NUMA_BIND_FAILURE = EnvBool(False)
|
||||||
|
|
||||||
# Metrics
|
# Metrics
|
||||||
|
|||||||
@@ -129,6 +129,8 @@ def get_numa_node_if_available(server_args: ServerArgs, gpu_id: int) -> Optional
|
|||||||
"""
|
"""
|
||||||
if server_args.numa_node is not None:
|
if server_args.numa_node is not None:
|
||||||
return server_args.numa_node[gpu_id]
|
return server_args.numa_node[gpu_id]
|
||||||
|
if not envs.SGLANG_AUTO_NUMA_BIND.get():
|
||||||
|
return None
|
||||||
if _is_numa_available():
|
if _is_numa_available():
|
||||||
queried_numa_node = _query_numa_node_for_gpu(gpu_id)
|
queried_numa_node = _query_numa_node_for_gpu(gpu_id)
|
||||||
if len(queried_numa_node) == 0:
|
if len(queried_numa_node) == 0:
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import unittest
|
|||||||
from contextlib import ExitStack
|
from contextlib import ExitStack
|
||||||
from unittest.mock import MagicMock, patch
|
from unittest.mock import MagicMock, patch
|
||||||
|
|
||||||
|
from sglang.srt.environ import envs
|
||||||
from sglang.srt.utils.numa_utils import (
|
from sglang.srt.utils.numa_utils import (
|
||||||
_handle_numa_bind_failure,
|
_handle_numa_bind_failure,
|
||||||
_is_numa_available,
|
_is_numa_available,
|
||||||
@@ -178,13 +179,34 @@ class TestGetNumaNodeIfAvailable(unittest.TestCase):
|
|||||||
args.numa_node = numa_node
|
args.numa_node = numa_node
|
||||||
return args
|
return args
|
||||||
|
|
||||||
def test_returns_explicit_numa_node_from_server_args(self):
|
def test_auto_numa_bind_enabled_by_default(self):
|
||||||
|
with patch.dict(os.environ, {}, clear=True):
|
||||||
|
self.assertTrue(envs.SGLANG_AUTO_NUMA_BIND.get())
|
||||||
|
|
||||||
|
@patch.dict(os.environ, {"SGLANG_AUTO_NUMA_BIND": "0"})
|
||||||
|
def test_returns_explicit_numa_node_when_auto_bind_disabled(self):
|
||||||
args = self._make_server_args(numa_node=[2, 3, 0, 1])
|
args = self._make_server_args(numa_node=[2, 3, 0, 1])
|
||||||
self.assertEqual(get_numa_node_if_available(args, 0), 2)
|
self.assertEqual(get_numa_node_if_available(args, 0), 2)
|
||||||
self.assertEqual(get_numa_node_if_available(args, 1), 3)
|
self.assertEqual(get_numa_node_if_available(args, 1), 3)
|
||||||
self.assertEqual(get_numa_node_if_available(args, 2), 0)
|
self.assertEqual(get_numa_node_if_available(args, 2), 0)
|
||||||
self.assertEqual(get_numa_node_if_available(args, 3), 1)
|
self.assertEqual(get_numa_node_if_available(args, 3), 1)
|
||||||
|
|
||||||
|
@patch("sglang.srt.utils.numa_utils._query_numa_node_for_gpu")
|
||||||
|
@patch("sglang.srt.utils.numa_utils._is_numa_available")
|
||||||
|
def test_auto_bind_disabled_skips_numa_detection(self, mock_avail, mock_query):
|
||||||
|
args = self._make_server_args(numa_node=None)
|
||||||
|
for bind_v2 in ("0", "1"):
|
||||||
|
with self.subTest(bind_v2=bind_v2), patch.dict(
|
||||||
|
os.environ,
|
||||||
|
{
|
||||||
|
"SGLANG_AUTO_NUMA_BIND": "0",
|
||||||
|
"SGLANG_NUMA_BIND_V2": bind_v2,
|
||||||
|
},
|
||||||
|
):
|
||||||
|
self.assertIsNone(get_numa_node_if_available(args, 0))
|
||||||
|
mock_avail.assert_not_called()
|
||||||
|
mock_query.assert_not_called()
|
||||||
|
|
||||||
@patch("sglang.srt.utils.numa_utils._is_numa_available", return_value=False)
|
@patch("sglang.srt.utils.numa_utils._is_numa_available", return_value=False)
|
||||||
def test_returns_none_when_numa_not_available(self, _mock_avail):
|
def test_returns_none_when_numa_not_available(self, _mock_avail):
|
||||||
args = self._make_server_args(numa_node=None)
|
args = self._make_server_args(numa_node=None)
|
||||||
@@ -196,6 +218,7 @@ class TestGetNumaNodeIfAvailable(unittest.TestCase):
|
|||||||
args = self._make_server_args(numa_node=None)
|
args = self._make_server_args(numa_node=None)
|
||||||
self.assertIsNone(get_numa_node_if_available(args, 0))
|
self.assertIsNone(get_numa_node_if_available(args, 0))
|
||||||
|
|
||||||
|
@patch.dict(os.environ, {"SGLANG_AUTO_NUMA_BIND": "1"})
|
||||||
@patch("sglang.srt.utils.numa_utils._query_numa_node_for_gpu", return_value=[1])
|
@patch("sglang.srt.utils.numa_utils._query_numa_node_for_gpu", return_value=[1])
|
||||||
@patch("sglang.srt.utils.numa_utils._is_numa_available", return_value=True)
|
@patch("sglang.srt.utils.numa_utils._is_numa_available", return_value=True)
|
||||||
def test_returns_queried_single_node(self, _mock_avail, _mock_gpu):
|
def test_returns_queried_single_node(self, _mock_avail, _mock_gpu):
|
||||||
|
|||||||
Reference in New Issue
Block a user