diff --git a/.github/workflows/pr-test-xeon.yml b/.github/workflows/pr-test-xeon.yml index 558d2cd655..b5cca6659d 100644 --- a/.github/workflows/pr-test-xeon.yml +++ b/.github/workflows/pr-test-xeon.yml @@ -93,31 +93,27 @@ jobs: suite: "stage-a-tp-test-cpu-intel" name: gnr - # Three SPR boxes each run ONE full-machine container taking one - # partition of a 3-way split over the merged non-TP pool - # (base-b non-TP + base-c). Single container per box (no socket - # split): sglang auto-computes mem_fraction_static from host memory - # and does not honor cgroup limits, so two parallel containers on - # one box exhaust host RAM and OOM (tracked upstream). One container - # per box keeps each server within the box's memory budget. + # Each SPR box (2 sockets / 2 NUMA, 32 physical cores each) runs TWO + # socket-pinned containers in parallel, each taking one partition of a + # 4-way split over the merged non-TP pool. Each container is capped + # with --memory so sglang (which reads cgroup memory.max, PR #38260 + + # the empty-NUMA-node fix) sizes its KV pool against one socket's + # budget, not the whole host -- so two parallel servers no longer + # exhaust host RAM. 2 boxes x 2 sockets = 4 partitions. - runner: xeon-spr role: spr suite: "stage-a-test-cpu-intel,stage-b-test-cpu-intel" name: spr1 - part_id: 0 - part_size: 3 + part_lo: 0 + part_hi: 1 + part_size: 4 - runner: xeon-spr role: spr suite: "stage-a-test-cpu-intel,stage-b-test-cpu-intel" name: spr2 - part_id: 1 - part_size: 3 - - runner: xeon-spr - role: spr - suite: "stage-a-test-cpu-intel,stage-b-test-cpu-intel" - name: spr3 - part_id: 2 - part_size: 3 + part_lo: 2 + part_hi: 3 + part_size: 4 steps: - name: Cleanup workspace run: | @@ -169,43 +165,70 @@ jobs: timeout-minutes: 120 run: | docker exec -w /sglang-checkout/ ci_sglang_${{ matrix.name }} \ - bash -c "source /opt/.venv/bin/activate && cd ./test && python3 run_suite.py --hw cpu --suite ${{ matrix.suite }} --timeout-from-est-time" + bash -c "source /opt/.venv/bin/activate && hf auth login --token \$HF_TOKEN && cd ./test && python3 run_suite.py --hw cpu --suite ${{ matrix.suite }} --timeout-from-est-time" - # ============ SPR: single full-machine container, one partition ====== # - - name: Run container (SPR) - if: matrix.role == 'spr' - run: | - docker rm -f ci_sglang_${{ matrix.name }} || true - docker run -dt \ - -v ${{ github.workspace }}:/sglang-checkout/ --shm-size=64g \ - -v $HOME/.cache/huggingface:/root/.cache/huggingface \ - -v $HOME/.cache/sgl_eval:/root/.cache/sgl_eval \ - -e HF_TOKEN="$(cat ~/huggingface_token.txt)" \ - --name ci_sglang_${{ matrix.name }} \ - sglang_xeon - - - name: Check AMX support (SPR) - if: matrix.role == 'spr' - timeout-minutes: 5 - run: | - docker exec -w /sglang-checkout/ ci_sglang_${{ matrix.name }} \ - bash -c "source /opt/.venv/bin/activate && python3 -c 'import torch; import sgl_kernel; assert torch.cpu._is_amx_tile_supported(); assert hasattr(torch.ops.sgl_kernel, \"convert_weight_packed\"); '" - - - name: Run unit tests (SPR) + # ========= SPR: two socket-pinned containers, run in parallel ======== # + - name: Run unit tests (SPR, two sockets in parallel) if: matrix.role == 'spr' timeout-minutes: 120 run: | - docker exec -w /sglang-checkout/ ci_sglang_${{ matrix.name }} \ - bash -c "source /opt/.venv/bin/activate && cd ./test && python3 run_suite.py --hw cpu --suite ${{ matrix.suite }} --auto-partition-id ${{ matrix.part_id }} --auto-partition-size ${{ matrix.part_size }} --timeout-from-est-time" + set -uo pipefail + HF_TOKEN_VAL="$(cat ~/huggingface_token.txt)" + + # One socket-pinned container. Physical cores only (0-31 = socket0, + # 32-63 = socket1); logical/HT siblings (64-127) are never bound. + # --memory caps the cgroup so sglang sizes KV against one socket, and + # --cpuset-mems keeps allocations on the local memory bank. + # $1 socket id, $2 cpuset-cpus, $3 cpuset-mems, $4 partition id + run_socket () { + local sid=$1 cpus=$2 mems=$3 pid=$4 + local cname=ci_sglang_${{ matrix.name }}_s${sid} + docker rm -f "$cname" >/dev/null 2>&1 || true + docker run -dt \ + --cpuset-cpus="$cpus" --cpuset-mems="$mems" --memory=240g \ + -v ${{ github.workspace }}:/sglang-checkout/ --shm-size=32g \ + -v $HOME/.cache/huggingface:/root/.cache/huggingface \ + -v $HOME/.cache/sgl_eval:/root/.cache/sgl_eval \ + -e HF_TOKEN="$HF_TOKEN_VAL" \ + -e SGLANG_CPU_OMP_THREADS_BIND="$cpus" \ + --name "$cname" \ + sglang_xeon + docker exec -w /sglang-checkout/ "$cname" bash -c \ + "source /opt/.venv/bin/activate && python3 -c 'import torch; import sgl_kernel; assert torch.cpu._is_amx_tile_supported(); assert hasattr(torch.ops.sgl_kernel, \"convert_weight_packed\")' && \ + hf auth login --token \$HF_TOKEN && \ + cd ./test && python3 run_suite.py --hw cpu --suite ${{ matrix.suite }} --auto-partition-id $pid --auto-partition-size ${{ matrix.part_size }} --timeout-from-est-time" + } + + # socket0 -> part_lo, socket1 -> part_hi; run concurrently, wait both. + run_socket 0 "0-31" 0 ${{ matrix.part_lo }} > s0.log 2>&1 & + P0=$! + run_socket 1 "32-63" 1 ${{ matrix.part_hi }} > s1.log 2>&1 & + P1=$! + + rc=0 + wait $P0 || rc=1 + wait $P1 || rc=1 + + echo "==================== socket0 (partition ${{ matrix.part_lo }}) ====================" + cat s0.log || true + echo "==================== socket1 (partition ${{ matrix.part_hi }}) ====================" + cat s1.log || true + exit $rc # ============================ Common ================================= # - name: Change permission if: always() timeout-minutes: 2 run: | - docker exec -u root ci_sglang_${{ matrix.name }} bash -c "rm -rf /tmp/ci-home && chown -R $(id -u):$(id -g) /sglang-checkout/ 2>/dev/null || true" 2>/dev/null || true + # GNR has one container (ci_sglang_); SPR has two socket + # containers (ci_sglang__s0/_s1). Chown from whichever exist. + for c in ci_sglang_${{ matrix.name }} ci_sglang_${{ matrix.name }}_s0 ci_sglang_${{ matrix.name }}_s1; do + docker exec -u root "$c" bash -c "rm -rf /tmp/ci-home && chown -R $(id -u):$(id -g) /sglang-checkout/ 2>/dev/null || true" 2>/dev/null || true + done - name: Cleanup container if: always() run: | - docker rm -f ci_sglang_${{ matrix.name }} || true + for c in ci_sglang_${{ matrix.name }} ci_sglang_${{ matrix.name }}_s0 ci_sglang_${{ matrix.name }}_s1; do + docker rm -f "$c" || true + done diff --git a/python/sglang/srt/model_executor/cpu_graph_runner.py b/python/sglang/srt/model_executor/cpu_graph_runner.py index ad8a0e4616..f4a49b4fe0 100644 --- a/python/sglang/srt/model_executor/cpu_graph_runner.py +++ b/python/sglang/srt/model_executor/cpu_graph_runner.py @@ -22,7 +22,6 @@ import logging from contextlib import contextmanager from typing import TYPE_CHECKING, Callable, Optional, Union -import psutil import torch import tqdm @@ -48,6 +47,7 @@ from sglang.srt.runtime_context import ( ) from sglang.srt.utils import ( empty_context, + get_free_cpu_memory, log_info_on_rank0, require_attn_tp_gather, require_gathered_buffer, @@ -741,7 +741,7 @@ class CPUGraphRunner: ) for bs in capture_range: if get_parallel().tp_rank == 0: - avail_mem = psutil.virtual_memory().available / (1 << 30) + avail_mem = get_free_cpu_memory() / (1 << 30) capture_range.set_description( f"Capturing batches ({bs=} {avail_mem=:.2f} GB)" ) diff --git a/python/sglang/srt/utils/common.py b/python/sglang/srt/utils/common.py index 7a471f337a..ca7fc1963b 100644 --- a/python/sglang/srt/utils/common.py +++ b/python/sglang/srt/utils/common.py @@ -495,7 +495,7 @@ def get_available_gpu_memory( elif device == "cpu": # TODO: rename the variables in the current function to be not GPU specific - total_free_memory = psutil.virtual_memory().available + total_free_memory = get_free_cpu_memory() n_numa_node: int = len(get_cpu_ids_by_node()) free_gpu_memory = round(total_free_memory / n_numa_node, 3) elif device == "npu": @@ -880,35 +880,51 @@ def get_npu_memory_capacity(): raise ImportError("torch_npu is required when run on npu device.") +def get_instance_memory_status(cgroup_file: str) -> Optional[int]: + try: + with open(cgroup_file, "r") as f: + content = f.read().strip() + # "max" or other non memory size content will be skipped + if not content.isdigit(): + raise ValueError + + return int(content) + except (PermissionError, FileNotFoundError, ValueError): + return None + + +def get_free_cpu_memory() -> int: + free_cpu_memory = psutil.virtual_memory().available + instance_max_cpu_memory = get_instance_memory_status("/sys/fs/cgroup/memory.max") + instance_current_cpu_memory = get_instance_memory_status( + "/sys/fs/cgroup/memory.current" + ) + if instance_max_cpu_memory is not None and instance_current_cpu_memory is not None: + instance_free_cpu_memory = max( + 0, instance_max_cpu_memory - instance_current_cpu_memory + ) + free_cpu_memory = min(free_cpu_memory, instance_free_cpu_memory) + + return free_cpu_memory + + def get_cpu_memory_capacity(): # Per-rank memory capacity cannot be determined for customized core settings if os.environ.get("SGLANG_CPU_OMP_THREADS_BIND", ""): + logger.warning( + "SGLANG_CPU_OMP_THREADS_BIND is set for explicit control of CPU cores for each TP rank." + "Please beware that in this case the available memory amounts of the ranks cannot be determined in prior." + "You need to set proper --max-total-tokens to avoid the out-of-memory error." + ) return None - n_numa_node: int = len(get_cpu_ids_by_node()) - if n_numa_node == 0: - # Cannot determine NUMA config, fallback to total memory and avoid ZeroDivisionError. - return float(psutil.virtual_memory().total // (1 << 20)) - try: - numa_mem_list = list() - file_prefix = "/sys/devices/system/node/" - for numa_id in range(n_numa_node): - file_meminfo = f"node{numa_id}/meminfo" - with open(os.path.join(file_prefix, file_meminfo), "r") as f: - # MemTotal info is at the 1st line - line = f.readline() - # Expected format: "Node 0 MemTotal: 100000000 kB" - parts = line.split() - if len(parts) >= 4 and parts[2] == "MemTotal:": - numa_mem_list.append(int(parts[3])) - else: - raise ValueError(f"Unexpected format in {file_meminfo}: {line}") - # Retrieved value in KB, need MB - numa_mem = float(min(numa_mem_list) // 1024) - return numa_mem - except (FileNotFoundError, ValueError, IndexError): - numa_mem = psutil.virtual_memory().total / n_numa_node - # Retrieved value in Byte, need MB - return float(numa_mem // (1 << 20)) + n_numa_node = len(get_cpu_ids_by_node()) or 1 + total_cpu_memory = psutil.virtual_memory().total + instance_max_cpu_memory = get_instance_memory_status("/sys/fs/cgroup/memory.max") + if instance_max_cpu_memory is not None: + total_cpu_memory = min(total_cpu_memory, instance_max_cpu_memory) + per_numa_mem = total_cpu_memory / n_numa_node + # Retrieved value in Byte, need MB + return float(per_numa_mem // (1 << 20)) def get_xpu_memory_capacity(): @@ -4594,12 +4610,12 @@ def get_physical_cpus_by_numa(): def get_cpu_ids_by_node(): node_to_cpus = get_physical_cpus_by_numa() # Sort by NUMA node index - cpu_ids = [ - ",".join(map(str, sorted(node_to_cpus[node]))) for node in sorted(node_to_cpus) - ] - # ['0,1,2,3', '4,5,6,7', '8,9,10,11', '12,13,14,15', '16,17,18,19', '20,21,22,23'] - return cpu_ids + return [ + ",".join(map(str, sorted(cpus))) + for _, cpus in sorted(node_to_cpus.items()) + if cpus + ] def is_shm_available(dtype, world_size, local_size):