[JIT Kernel][Feature] Support JIT custom all reduce (rewrite as v2) (#19880)

Co-authored-by: Xiaoyu Zhang <35585791+BBuf@users.noreply.github.com>
This commit is contained in:
DarkSharpness
2026-03-20 18:24:07 +08:00
committed by GitHub
co-authored by Xiaoyu Zhang
parent 2099943a49
commit 2dd9196079
16 changed files with 2154 additions and 96 deletions
@@ -0,0 +1,227 @@
"""
Correctness test for the JIT custom all-reduce (v2) kernel.
The test compares the JIT custom all-reduce output against NCCL all-reduce
for various tensor sizes and dtypes, in both eager and CUDA-graph modes.
Usage:
python -m pytest test_jit_custom_all_reduce.py -v
This file doubles as the torchrun worker script. The test class launches
torchrun --nproc_per_node=N <this_file>
and asserts that all worker processes exit successfully.
"""
from __future__ import annotations
import itertools
import logging
import os
import subprocess
from typing import Optional
import pytest
import torch
import torch.distributed as dist
from tqdm import tqdm
import sglang.srt.distributed.parallel_state as ps
from sglang.jit_kernel.all_reduce import AllReduceAlgo
from sglang.srt.distributed.device_communicators.custom_all_reduce_v2 import (
CustomAllReduceV2,
)
# ---------------------------------------------------------------------------
# Test parameters (shared between test class and worker)
# ---------------------------------------------------------------------------
TEST_SIZES = [
16,
32,
512,
1024,
1024 + 16, # weird case
4 * 1024,
32 * 1024,
256 * 1024,
2 * 1024 * 1024, # 2M elements
4 * 1024 * 1024, # 4M elements
]
TEST_DTYPES = [torch.float16, torch.bfloat16, torch.float32]
SHOTS = [
AllReduceAlgo.ONE_SHOT_PULL,
AllReduceAlgo.ONE_SHOT_PUSH,
AllReduceAlgo.TWO_SHOT_PULL,
]
USE_GRAPH_OPTIONS = [True, False]
TEST_CONFIG = itertools.product(TEST_SIZES, TEST_DTYPES, SHOTS, USE_GRAPH_OPTIONS)
TEST_LAYERS = 2
TEST_LOOP = 16
# ---------------------------------------------------------------------------
# Test class (runs via pytest, launches torchrun subprocesses)
# ---------------------------------------------------------------------------
def run_torchrun(nproc: int, timeout: int = 300) -> None:
"""Launch this script as a torchrun worker and assert success."""
cmd = [
"torchrun",
f"--nproc_per_node={nproc}",
__file__,
]
os.environ["DISABLE_PBAR"] = "1"
result = subprocess.run(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
timeout=timeout,
)
assert result.returncode == 0, (
f"torchrun (nproc={nproc}) failed with rc={result.returncode}\n"
f"{result.stdout}"
)
@pytest.mark.parametrize("nproc", [2, 3, 4, 5, 6, 7, 8])
def test_custom_allreduce(nproc: int) -> None:
device_count = torch.cuda.device_count()
if device_count < nproc:
pytest.skip(
f"Requires at least {nproc} GPUs, but only {device_count} available"
)
run_torchrun(nproc)
# ---------------------------------------------------------------------------
# Worker logic (executed by each torchrun process)
# ---------------------------------------------------------------------------
def init_distributed():
"""Initialize distributed groups via torchrun env vars.
Returns (rank, device, cpu_group, nccl_group, comm).
"""
local_rank = int(os.environ["LOCAL_RANK"])
world_size = int(os.environ["WORLD_SIZE"])
rank = local_rank
device = torch.device(f"cuda:{rank}")
torch.cuda.set_device(device)
dist.init_process_group(backend="gloo")
ps._WORLD = coord = ps.init_world_group(
ranks=list(range(world_size)),
local_rank=local_rank,
backend="nccl",
)
cpu_group = coord.cpu_group
nccl_group = coord.device_group
assert nccl_group is not None
max_size = max(TEST_SIZES) * 4
comm = CustomAllReduceV2(cpu_group, device, max_size, max_size)
if comm.disabled:
raise RuntimeError("JIT CustomAllReduceV2 is disabled on this system")
return rank, device, cpu_group, nccl_group, comm
@torch.inference_mode()
def worker_test(
device: torch.device,
nccl_group: dist.ProcessGroup,
comm: CustomAllReduceV2,
size: int,
dtype: torch.dtype,
use_graph: bool,
algo: AllReduceAlgo,
) -> Optional[RuntimeError]:
comm.override_algo = algo
def get_run_graph_fn():
graph = torch.cuda.CUDAGraph()
graph_inp = torch.zeros((TEST_LAYERS, size), dtype=dtype, device=device)
out_jits = []
with comm.capture():
with torch.cuda.graph(graph):
for i in range(TEST_LAYERS):
out_jits.append(comm.custom_all_reduce(graph_inp[i]))
out_jit = torch.stack(out_jits)
torch.cuda.synchronize()
def run_graph(x: torch.Tensor) -> torch.Tensor:
graph_inp.copy_(x)
graph.replay()
return out_jit.clone()
return run_graph
def get_run_eager_fn():
def run_eager(x: torch.Tensor) -> torch.Tensor:
eager_inp = x.clone()
out_eagers = []
for i in range(TEST_LAYERS):
out_eagers.append(comm.custom_all_reduce(eager_inp[i]))
torch.cuda.synchronize()
return torch.stack(out_eagers)
return run_eager
run_fn = get_run_graph_fn() if use_graph else get_run_eager_fn()
num_errors = 0
for _ in range(TEST_LOOP):
# NOTE: 15 * 8 < 128, which is the precision limit for bf16
inp = torch.randint(0, 16, (TEST_LAYERS, size), dtype=dtype, device=device)
assert comm.should_custom_ar(inp[0])
out_ref = inp.clone()
dist.all_reduce(out_ref, group=nccl_group)
out_jit = run_fn(inp)
num_errors += not torch.all(out_jit == out_ref)
torch.cuda.synchronize()
nccl_group.barrier().wait()
if num_errors > 0:
return RuntimeError(
f"Test failed for {size=}, {dtype=}, {algo=}, "
f"{use_graph=} with {num_errors} errors. "
)
return None
def worker_main() -> None:
"""Entry point for each torchrun worker process."""
rank, device, cpu_group, nccl_group, comm = init_distributed()
world_size = dist.get_world_size()
torch.cuda.set_stream(torch.cuda.Stream())
logging.disable(logging.INFO) # Suppress internal logging for cleaner test output
items = list(enumerate(TEST_CONFIG))
disable_pbar = os.environ.get("DISABLE_PBAR", "0") == "1" or rank != 0
pbar = tqdm(items, desc=f"Testing {world_size} GPUs", disable=disable_pbar)
for i, (size, dtype, algo, use_graph) in pbar:
error = worker_test(device, nccl_group, comm, size, dtype, use_graph, algo)
if error is not None:
print(
f"Worker {rank} failed for {size=}, {dtype=}, "
f"{algo=}, {use_graph=}, iteration={i}\n"
f"Error: {error}"
)
# communicate the result to rank 0 for logging
result = torch.tensor([int(error is not None)], device=device)
dist.all_reduce(result, group=cpu_group)
failed = bool(result.item())
if failed:
raise RuntimeError(
f"Test failed on rank {rank} for config: "
f"{size=}, {dtype=}, {algo=}, {use_graph=}"
)
comm.close()
dist.destroy_process_group()
if __name__ == "__main__":
worker_main()