[CPU]: Add group-aware CPU SHM collective kernels (#35435)

This commit is contained in:
Yuxingwang-intel
2026-10-15 13:22:20 +08:00
committed by GitHub
parent 4dc9b5c68c
commit 93842f52fe
8 changed files with 538 additions and 195 deletions
@@ -1,4 +1,3 @@
#include <ATen/record_function.h>
#include <torch/all.h>
#include "shm.h"
@@ -43,50 +42,50 @@ void initialize(int64_t size, int64_t rank) {
}
if (all_ranks_local_p) {
shm_initialize(size, rank, addr_string, port_string);
shm_default_initialize(size, rank, addr_string, port_string);
}
}
void shm_allreduce(torch::Tensor& data, int64_t op) {
void shm_allreduce(torch::Tensor& data, int64_t op, int64_t handle) {
TORCH_CHECK(op == c10d::ReduceOp::SUM, "Only torch.distributed.ReduceOp.SUM is supported");
auto numel = data.numel();
int data_size = numel * data.element_size();
all_reduce_outer_loop(data, numel, data_size);
return;
all_reduce_outer_loop(data, numel, data_size, handle);
}
torch::Tensor shm_allgather(torch::Tensor& data, int64_t dim) {
torch::Tensor shm_allgather(torch::Tensor& data, int64_t dim, int64_t handle) {
auto numel = data.numel();
int data_size = numel * data.element_size();
if (dim < 0) {
dim += data.dim();
}
int64_t group_size = handle == -1 ? world_size : shm_get_group_size(handle);
std::vector<int64_t> result_shape = data.sizes().vec();
result_shape[dim] *= world_size;
result_shape[dim] *= group_size;
torch::Tensor result_tensor = torch::empty(result_shape, data.options());
return all_gather<STATE_GROUP_ALL_GATHER>(result_tensor, data, dim, numel, data_size);
return all_gather<STATE_GROUP_ALL_GATHER>(result_tensor, data, dim, numel, data_size, handle);
}
void shm_allgather_into_tensor(torch::Tensor& output_tensor, torch::Tensor& data) {
RECORD_FUNCTION("sgl-kernel::shm_allgather_into_tensor", std::vector<c10::IValue>({data}));
void shm_allgather_into_tensor(torch::Tensor& output_tensor, torch::Tensor& data, int64_t handle) {
auto numel = data.numel();
int data_size = numel * data.element_size();
int64_t dim = 0;
all_gather<STATE_GROUP_ALL_GATHER_INTO_TENSOR>(output_tensor, data, dim, numel, data_size);
all_gather<STATE_GROUP_ALL_GATHER_INTO_TENSOR>(output_tensor, data, dim, numel, data_size, handle);
}
void shm_reduce_scatter_tensor(at::Tensor& output_tensor, at::Tensor& data, int64_t op) {
RECORD_FUNCTION("sgl-kernel::shm_reduce_scatter_tensor", std::vector<c10::IValue>({data}));
void shm_reduce_scatter_tensor(at::Tensor& output_tensor, at::Tensor& data, int64_t op, int64_t handle) {
TORCH_CHECK(op == c10d::ReduceOp::SUM, "Only torch.distributed.ReduceOp.SUM is supported");
auto numel = data.numel();
int data_size = numel * data.element_size();
reduce_scatter_outer_loop(output_tensor, data, numel, data_size);
reduce_scatter_outer_loop(output_tensor, data, numel, data_size, handle);
}
return;
void shm_alltoall(torch::Tensor& output_tensor, torch::Tensor& input_tensor, int64_t handle) {
size_t data_size = input_tensor.numel() * input_tensor.element_size();
all_to_all(
static_cast<char*>(output_tensor.data_ptr()), static_cast<char*>(input_tensor.data_ptr()), data_size, handle);
}
+295 -139
View File
@@ -73,20 +73,21 @@ void shared_create(SharedData* data, const char* name, void* bytes, size_t nbyte
}
}
static int world_size;
// SHM based allreduce helper functions
// buffer that holds shm name
#define NAME_BUF_SIZE 1000
#define MAX_BUF_SIZE 1048576 * 32
#define NAIVE_ALLREDUCE_THRESHOLD 1048576
#define SHM_BUFFER_NAME "deepspeed_allreduce_buffer"
constexpr int kNumCollectiveStates = STATE_GROUP_ALL_TO_ALL + 1;
// idx=0 -- state for symmetric_naive_all_reduce
// idx=1 -- state for distributed_naive_all_reduce
// idx=2 -- state for all_gather
// idx=3 -- state for all_gather_into_tensor
// idx=4 -- state for reduce_scatter
// idx=5 -- state for all_to_all
struct allreduce_workspace {
enum coll_state states[5]; // idx=0 -- state for symmetric_naive_all_reduce
// idx=1 -- state for distributed_naive_all_reduce
// idx=2 -- state for all_gather
// idx=3 -- state for all_gather_into_tensor
// idx=4 -- state for reduce_scatter
enum coll_state states[kNumCollectiveStates];
// double buffer to avoid syncing between rounds
// offset=0 -- 2*NAIVE_ALLREDUCE_THRESHOLD : buffer for
// symmetric_naive_all_reduce after that : buffer for
@@ -96,7 +97,9 @@ struct allreduce_workspace {
2 * MAX_BUF_SIZE + // distributed naive reduce
2 * MAX_BUF_SIZE + // allgather
2 * MAX_BUF_SIZE + // allgather_into_tensor
2 * MAX_BUF_SIZE // reduce_scatter
2 * MAX_BUF_SIZE + // reduce_scatter
2 * MAX_BUF_SIZE // all_to_all
];
};
@@ -108,20 +111,63 @@ struct allreduce_workspace {
(2 * NAIVE_ALLREDUCE_THRESHOLD + 4 * MAX_BUF_SIZE + current_buffer * MAX_BUF_SIZE) // allgather_into_tensor
#define BUFFER4_OFFSET(current_buffer) \
(2 * NAIVE_ALLREDUCE_THRESHOLD + 6 * MAX_BUF_SIZE + current_buffer * MAX_BUF_SIZE) // reduce_scatter
#define BUFFER5_OFFSET(current_buffer) \
(2 * NAIVE_ALLREDUCE_THRESHOLD + 8 * MAX_BUF_SIZE + current_buffer * MAX_BUF_SIZE) // all_to_all
struct allreduce_workspace** workspace;
struct shm_context {
int group_size;
int group_rank;
// buffer for small messages, double buffer
char** symmetric_buffer[2];
// buffer for large messages, double buffer
char** distributed_buffer[2];
struct allreduce_workspace** workspace;
char** allgather_buffer[2];
char** allgather_into_tensor_buffer[2];
char** reduce_scatter_buffer[2];
char** symmetric_buffer[2];
char** distributed_buffer[2];
char** allgather_buffer[2];
char** allgather_into_tensor_buffer[2];
char** reduce_scatter_buffer[2];
char** alltoall_buffer[2];
void wait_buffer_state_until_2(int index, enum coll_state state0, enum coll_state state1, int state_group) {
volatile enum coll_state* state_ptr = &(workspace[index]->states[state_group]);
int symmetric_current_buffer;
int symmetric_state_idx;
int distributed_current_buffer;
int distributed_state_idx;
int allgather_current_buffer;
int allgather_state_idx;
int allgather_into_tensor_current_buffer;
int allgather_into_tensor_state_idx;
int reduce_scatter_current_buffer;
int reduce_scatter_state_idx;
int alltoall_current_buffer;
int alltoall_state_idx;
};
static struct shm_context* default_shm_context = nullptr;
static std::vector<struct shm_context*> shm_contexts;
static struct shm_context* get_shm_context(int64_t handle) {
if (handle == -1) {
TORCH_CHECK(default_shm_context != nullptr, "Default SHM context is not initialized");
return default_shm_context;
}
TORCH_CHECK(handle >= 0 && handle < static_cast<int64_t>(shm_contexts.size()), "Invalid SHM context handle: ");
auto* ctx = shm_contexts[handle];
TORCH_CHECK(ctx != nullptr, "SHM context is null");
return ctx;
}
int64_t shm_get_group_size(int64_t handle) {
return get_shm_context(handle)->group_size;
}
void wait_buffer_state_until_2(
struct shm_context* ctx, int index, enum coll_state state0, enum coll_state state1, int state_group) {
volatile enum coll_state* state_ptr = &(ctx->workspace[index]->states[state_group]);
while (1) {
volatile enum coll_state cur_state = *state_ptr;
@@ -133,40 +179,53 @@ void reduce_all_buffers(
int start_elements,
int num_elements,
c10::ScalarType scalar_type,
int to_buffer_idx,
char* to_buffer,
char** buffers) {
char** buffers,
int group_size) {
switch (scalar_type) {
case c10::ScalarType::BFloat16:
reduce_bf16_buffers(start_elements, num_elements, to_buffer, buffers, world_size);
reduce_bf16_buffers(start_elements, num_elements, to_buffer, buffers, group_size);
break;
case c10::ScalarType::Half:
reduce_fp16_buffers(start_elements, num_elements, to_buffer, buffers, world_size);
reduce_fp16_buffers(start_elements, num_elements, to_buffer, buffers, group_size);
break;
case c10::ScalarType::Float:
reduce_fp32_buffers(start_elements, num_elements, to_buffer, buffers, world_size);
reduce_fp32_buffers(start_elements, num_elements, to_buffer, buffers, group_size);
break;
default:
assert(!"Should not get here");
}
}
static bool is_initialized = false;
static int world_rank;
void shm_initialize(int size, int rank, const char* addr_string, const char* port_string) {
if (is_initialized) {
return;
}
is_initialized = true;
world_size = size;
world_rank = rank;
int64_t shm_initialize(int size, int rank, const char* addr_string, const char* port_string, const char* group_name) {
auto* ctx = (struct shm_context*)malloc(sizeof(struct shm_context));
ctx->group_size = size;
ctx->group_rank = rank;
ctx->symmetric_current_buffer = 0;
ctx->symmetric_state_idx = 0;
ctx->distributed_current_buffer = 0;
ctx->distributed_state_idx = 0;
ctx->allgather_current_buffer = 0;
ctx->allgather_state_idx = 0;
ctx->allgather_into_tensor_current_buffer = 0;
ctx->allgather_into_tensor_state_idx = 0;
ctx->reduce_scatter_current_buffer = 0;
ctx->reduce_scatter_state_idx = 0;
ctx->alltoall_current_buffer = 0;
ctx->alltoall_state_idx = 0;
char shm_name_prefix[NAME_BUF_SIZE];
char shm_name[NAME_BUF_SIZE];
snprintf(shm_name_prefix, NAME_BUF_SIZE, "%s_%d_%s_%s", SHM_BUFFER_NAME, getuid(), addr_string, port_string);
// create shared workspace for SHM based allreduce
snprintf(
shm_name_prefix,
NAME_BUF_SIZE,
"%s_%d_%s_%s_%s",
SHM_BUFFER_NAME,
getuid(),
addr_string,
port_string,
group_name);
// create shared workspace for SHM based collectives
SharedData allreduce_buffer;
// allocate workspace_buf for current rank
struct allreduce_workspace* workspace_buf;
@@ -174,30 +233,34 @@ void shm_initialize(int size, int rank, const char* addr_string, const char* por
workspace_buf = (struct allreduce_workspace*)malloc(sizeof(struct allreduce_workspace));
snprintf(shm_name, NAME_BUF_SIZE, "%.900s_%d", shm_name_prefix, rank);
shared_create(&allreduce_buffer, shm_name, workspace_buf, sizeof(struct allreduce_workspace));
free(workspace_buf);
workspace_buf = (struct allreduce_workspace*)allreduce_buffer.bytes;
workspace_buf->states[STATE_GROUP_SYMMETRIC_ALLREDUCE] =
coll_alt2_allreduce_naive__copy_in_done; // symmetric_naive_all_reduce
workspace_buf->states[STATE_GROUP_DISTRIBUTED_ALLREDUCE] = coll_begin; // distributed_naive_reduce
workspace_buf->states[STATE_GROUP_ALL_GATHER] = coll_alt2_allgather_naive__copy_in_done; // all_gather
workspace_buf->states[STATE_GROUP_ALL_GATHER_INTO_TENSOR] =
coll_alt2_allgather_naive__copy_in_done; // all_gather_into_tensor
workspace_buf->states[STATE_GROUP_REDUCE_SCATTER] = coll_begin; // reduce_scatter
coll_alt2_allgather_naive__copy_in_done; // all_gather_into_tensor
workspace_buf->states[STATE_GROUP_REDUCE_SCATTER] = coll_begin; // reduce_scatter
workspace_buf->states[STATE_GROUP_ALL_TO_ALL] = coll_alt2_allgather_naive__copy_in_done; // all_to_all
// create the workspace pointer list
workspace = (struct allreduce_workspace**)malloc(size * sizeof(struct allreduce_workspace*));
symmetric_buffer[0] = (char**)malloc(size * sizeof(char**));
symmetric_buffer[1] = (char**)malloc(size * sizeof(char**));
distributed_buffer[0] = (char**)malloc(size * sizeof(char**));
distributed_buffer[1] = (char**)malloc(size * sizeof(char**));
ctx->workspace = (struct allreduce_workspace**)malloc(size * sizeof(struct allreduce_workspace*));
ctx->symmetric_buffer[0] = (char**)malloc(size * sizeof(char*));
ctx->symmetric_buffer[1] = (char**)malloc(size * sizeof(char*));
ctx->distributed_buffer[0] = (char**)malloc(size * sizeof(char*));
ctx->distributed_buffer[1] = (char**)malloc(size * sizeof(char*));
ctx->allgather_buffer[0] = (char**)malloc(size * sizeof(char*));
ctx->allgather_buffer[1] = (char**)malloc(size * sizeof(char*));
allgather_buffer[0] = (char**)malloc(size * sizeof(char*));
allgather_buffer[1] = (char**)malloc(size * sizeof(char*));
ctx->allgather_into_tensor_buffer[0] = (char**)malloc(size * sizeof(char*));
ctx->allgather_into_tensor_buffer[1] = (char**)malloc(size * sizeof(char*));
allgather_into_tensor_buffer[0] = (char**)malloc(size * sizeof(char*));
allgather_into_tensor_buffer[1] = (char**)malloc(size * sizeof(char*));
ctx->reduce_scatter_buffer[0] = (char**)malloc(size * sizeof(char*));
ctx->reduce_scatter_buffer[1] = (char**)malloc(size * sizeof(char*));
reduce_scatter_buffer[0] = (char**)malloc(size * sizeof(char*));
reduce_scatter_buffer[1] = (char**)malloc(size * sizeof(char*));
ctx->alltoall_buffer[0] = (char**)malloc(size * sizeof(char*));
ctx->alltoall_buffer[1] = (char**)malloc(size * sizeof(char*));
// map shm of all ranks
for (int i = 0; i < size; i++) {
@@ -208,48 +271,71 @@ void shm_initialize(int size, int rank, const char* addr_string, const char* por
shared_open(&allreduce_buffer, shm_name, sizeof(struct allreduce_workspace));
} while (allreduce_buffer.descriptor == -1 && errno == ENOENT);
workspace_buf_other = (struct allreduce_workspace*)allreduce_buffer.bytes;
workspace[i] = workspace_buf_other;
ctx->workspace[i] = workspace_buf_other;
} else {
workspace[i] = workspace_buf;
ctx->workspace[i] = workspace_buf;
}
symmetric_buffer[0][i] = workspace[i]->buffer + BUFFER0_OFFSET(0);
symmetric_buffer[1][i] = workspace[i]->buffer + BUFFER0_OFFSET(1);
distributed_buffer[0][i] = workspace[i]->buffer + BUFFER1_OFFSET(0);
distributed_buffer[1][i] = workspace[i]->buffer + BUFFER1_OFFSET(1);
allgather_buffer[0][i] = workspace[i]->buffer + BUFFER2_OFFSET(0);
allgather_buffer[1][i] = workspace[i]->buffer + BUFFER2_OFFSET(1);
ctx->symmetric_buffer[0][i] = ctx->workspace[i]->buffer + BUFFER0_OFFSET(0);
ctx->symmetric_buffer[1][i] = ctx->workspace[i]->buffer + BUFFER0_OFFSET(1);
allgather_into_tensor_buffer[0][i] = workspace[i]->buffer + BUFFER3_OFFSET(0);
allgather_into_tensor_buffer[1][i] = workspace[i]->buffer + BUFFER3_OFFSET(1);
ctx->distributed_buffer[0][i] = ctx->workspace[i]->buffer + BUFFER1_OFFSET(0);
ctx->distributed_buffer[1][i] = ctx->workspace[i]->buffer + BUFFER1_OFFSET(1);
reduce_scatter_buffer[0][i] = workspace[i]->buffer + BUFFER4_OFFSET(0);
reduce_scatter_buffer[1][i] = workspace[i]->buffer + BUFFER4_OFFSET(1);
ctx->allgather_buffer[0][i] = ctx->workspace[i]->buffer + BUFFER2_OFFSET(0);
ctx->allgather_buffer[1][i] = ctx->workspace[i]->buffer + BUFFER2_OFFSET(1);
ctx->allgather_into_tensor_buffer[0][i] = ctx->workspace[i]->buffer + BUFFER3_OFFSET(0);
ctx->allgather_into_tensor_buffer[1][i] = ctx->workspace[i]->buffer + BUFFER3_OFFSET(1);
ctx->reduce_scatter_buffer[0][i] = ctx->workspace[i]->buffer + BUFFER4_OFFSET(0);
ctx->reduce_scatter_buffer[1][i] = ctx->workspace[i]->buffer + BUFFER4_OFFSET(1);
ctx->alltoall_buffer[0][i] = ctx->workspace[i]->buffer + BUFFER5_OFFSET(0);
ctx->alltoall_buffer[1][i] = ctx->workspace[i]->buffer + BUFFER5_OFFSET(1);
}
const int64_t handle = static_cast<int64_t>(shm_contexts.size());
shm_contexts.push_back(ctx);
return handle;
}
void shm_default_initialize(int size, int rank, const char* addr_string, const char* port_string) {
int64_t handle = shm_initialize(size, rank, addr_string, port_string, "default");
default_shm_context = shm_contexts[handle];
}
#define positive_mod(num, mod) ((((num) % (mod)) + (mod)) % (mod))
#define rank_mod(rank) positive_mod(rank, world_size)
size_t slice_size(size_t chunk_el, int slice_idx) {
size_t slice_size = chunk_el / world_size;
return slice_idx == world_size - 1 ? slice_size + (chunk_el % world_size) : slice_size;
int64_t shm_group_initialize(const std::string& group_name, int64_t group_size, int64_t group_rank) {
const char* addr_string = getenv("MASTER_ADDR");
const char* port_string = getenv("MASTER_PORT");
TORCH_CHECK(addr_string != nullptr, "MASTER_ADDR is not set");
TORCH_CHECK(port_string != nullptr, "MASTER_PORT is not set");
return shm_initialize(group_size, group_rank, addr_string, port_string, group_name.c_str());
}
char* slice_data(char* data_ptr, size_t chunk_el, int el_size, int slice_idx) {
size_t slice_size = chunk_el / world_size;
size_t slice_size(size_t chunk_el, int slice_idx, int group_size) {
size_t slice_size = chunk_el / group_size;
return slice_idx == group_size - 1 ? slice_size + (chunk_el % group_size) : slice_size;
}
char* slice_data(char* data_ptr, size_t chunk_el, int el_size, int slice_idx, int group_size) {
size_t slice_size = chunk_el / group_size;
size_t el_offset = slice_size * slice_idx;
return data_ptr + el_offset * el_size;
}
size_t slice_el_start(size_t chunk_el, int slice_idx) {
size_t slice_size = chunk_el / world_size;
size_t slice_el_start(size_t chunk_el, int slice_idx, int group_size) {
size_t slice_size = chunk_el / group_size;
return slice_size * slice_idx;
}
void symmetric_naive_all_reduce(char* data_ptr, c10::ScalarType scalar_type, size_t chunk_size, size_t chunk_el) {
void symmetric_naive_all_reduce(
struct shm_context* ctx, char* data_ptr, c10::ScalarType scalar_type, size_t chunk_size, size_t chunk_el) {
const int state_group = STATE_GROUP_SYMMETRIC_ALLREDUCE;
static int current_buffer = 0;
static int state_idx = 0;
int& current_buffer = ctx->symmetric_current_buffer;
int& state_idx = ctx->symmetric_state_idx;
// init states to case 0 to get rid of "maybe-uninitialized" warning.
enum coll_state copy_current = coll_allreduce_naive__copy_in_done;
@@ -273,31 +359,30 @@ void symmetric_naive_all_reduce(char* data_ptr, c10::ScalarType scalar_type, siz
}
state_idx = (state_idx + 1) % 3;
parallel_memcpy(symmetric_buffer[current_buffer][world_rank], data_ptr, chunk_size);
parallel_memcpy(ctx->symmetric_buffer[current_buffer][ctx->group_rank], data_ptr, chunk_size);
std::atomic_thread_fence(std::memory_order_release);
workspace[world_rank]->states[state_group] = copy_current;
ctx->workspace[ctx->group_rank]->states[state_group] = copy_current;
for (int i = 0; i < world_size; i++) {
for (int i = 0; i < ctx->group_size; i++) {
// wait until the other rank copy the buffer
if (i != world_rank) {
wait_buffer_state_until_2(i, copy_current, copy_next, state_group);
if (i != ctx->group_rank) {
wait_buffer_state_until_2(ctx, i, copy_current, copy_next, state_group);
}
}
// each rank reduce the buffer independently so therre is no need for
// each rank reduce the buffer independently so there is no need for
// synchronization afterward
reduce_all_buffers(0, chunk_el, scalar_type, world_rank, data_ptr, symmetric_buffer[current_buffer]);
reduce_all_buffers(0, chunk_el, scalar_type, data_ptr, ctx->symmetric_buffer[current_buffer], ctx->group_size);
// switch buffer
current_buffer = 1 - current_buffer;
}
// naive allreduce distributed, each rank do naive reduce on its slice
void distributed_naive_reduce(char* data_ptr, c10::ScalarType scalar_type, size_t chunk_size, size_t chunk_el) {
void distributed_naive_reduce(
struct shm_context* ctx, char* data_ptr, c10::ScalarType scalar_type, size_t chunk_size, size_t chunk_el) {
const int state_group = STATE_GROUP_DISTRIBUTED_ALLREDUCE;
static int current_buffer = 0;
static int state_idx = 0;
int& current_buffer = ctx->distributed_current_buffer;
int& state_idx = ctx->distributed_state_idx;
// init states to case 0 to get rid of "maybe-uninitialized" warning.
enum coll_state copy_current = coll_allreduce_naive__copy_in_done;
enum coll_state reduce_current = coll_allreduce_naive__reduce_done;
@@ -322,65 +407,75 @@ void distributed_naive_reduce(char* data_ptr, c10::ScalarType scalar_type, size_
state_idx = (state_idx + 1) % 2;
int data_size = chunk_size / chunk_el;
parallel_memcpy(distributed_buffer[current_buffer][world_rank], data_ptr, chunk_size);
parallel_memcpy(ctx->distributed_buffer[current_buffer][ctx->group_rank], data_ptr, chunk_size);
std::atomic_thread_fence(std::memory_order_release);
workspace[world_rank]->states[state_group] = copy_current;
ctx->workspace[ctx->group_rank]->states[state_group] = copy_current;
for (int i = 0; i < world_size; i++) {
for (int i = 0; i < ctx->group_size; i++) {
// wait until all the other ranks copy the buffer
if (i != world_rank) wait_buffer_state_until_2(i, copy_current, reduce_current, state_group);
if (i != ctx->group_rank) {
wait_buffer_state_until_2(ctx, i, copy_current, reduce_current, state_group);
}
}
// reduce scatter
reduce_all_buffers(
slice_el_start(chunk_el, world_rank),
slice_size(chunk_el, world_rank),
slice_el_start(chunk_el, ctx->group_rank, ctx->group_size),
slice_size(chunk_el, ctx->group_rank, ctx->group_size),
scalar_type,
world_rank,
distributed_buffer[current_buffer][world_rank],
distributed_buffer[current_buffer]);
std::atomic_thread_fence(std::memory_order_release);
workspace[world_rank]->states[state_group] = reduce_current;
ctx->distributed_buffer[current_buffer][ctx->group_rank],
ctx->distributed_buffer[current_buffer],
ctx->group_size);
for (int i = 0; i < world_size; i++) {
std::atomic_thread_fence(std::memory_order_release);
ctx->workspace[ctx->group_rank]->states[state_group] = reduce_current;
for (int i = 0; i < ctx->group_size; i++) {
// wait until all the other ranks reduce the buffer
if (i != world_rank) wait_buffer_state_until_2(i, reduce_current, copy_next, state_group);
if (i != ctx->group_rank) {
wait_buffer_state_until_2(ctx, i, reduce_current, copy_next, state_group);
}
}
for (int i = 0; i < world_size; i++) {
int rank = (i + world_rank) % world_size;
for (int i = 0; i < ctx->group_size; i++) {
int rank = (i + ctx->group_rank) % ctx->group_size;
parallel_memcpy(
slice_data(data_ptr, chunk_el, data_size, rank),
slice_data(distributed_buffer[current_buffer][rank], chunk_el, chunk_size / chunk_el, rank),
slice_size(chunk_el, rank) * data_size);
slice_data(data_ptr, chunk_el, data_size, rank, ctx->group_size),
slice_data(
ctx->distributed_buffer[current_buffer][rank], chunk_el, chunk_size / chunk_el, rank, ctx->group_size),
slice_size(chunk_el, rank, ctx->group_size) * data_size);
}
current_buffer = 1 - current_buffer;
}
void all_reduce_outer_loop(torch::Tensor& data, size_t numel, int data_size) {
void all_reduce_outer_loop(torch::Tensor& data, size_t numel, int data_size, int64_t handle) {
auto* ctx = get_shm_context(handle);
for (int offset = 0; offset < data_size; offset += MAX_BUF_SIZE) {
auto data_ptr = ((char*)(data.data_ptr()) + offset);
size_t chunk_size = data_size - offset > MAX_BUF_SIZE ? MAX_BUF_SIZE : data_size - offset;
size_t chunk_el = chunk_size / (data_size / numel);
if (chunk_size < NAIVE_ALLREDUCE_THRESHOLD) {
symmetric_naive_all_reduce(data_ptr, data.scalar_type(), chunk_size, chunk_el);
symmetric_naive_all_reduce(ctx, data_ptr, data.scalar_type(), chunk_size, chunk_el);
} else {
distributed_naive_reduce(data_ptr, data.scalar_type(), chunk_size, chunk_el);
distributed_naive_reduce(ctx, data_ptr, data.scalar_type(), chunk_size, chunk_el);
}
}
}
template <int STATE_GROUP>
void naive_all_gather(char* result_ptr, char* data_ptr, size_t res_stride, size_t chunk_size, size_t chunk_el) {
static int current_buffer = 0;
static int state_idx = 0;
void naive_all_gather(
struct shm_context* ctx, char* result_ptr, char* data_ptr, size_t res_stride, size_t chunk_size, size_t chunk_el) {
int& current_buffer =
STATE_GROUP == STATE_GROUP_ALL_GATHER ? ctx->allgather_current_buffer : ctx->allgather_into_tensor_current_buffer;
int& state_idx =
STATE_GROUP == STATE_GROUP_ALL_GATHER ? ctx->allgather_state_idx : ctx->allgather_into_tensor_state_idx;
char*** buffer = nullptr;
if constexpr (STATE_GROUP == STATE_GROUP_ALL_GATHER) {
buffer = allgather_buffer;
buffer = ctx->allgather_buffer;
} else if constexpr (STATE_GROUP == STATE_GROUP_ALL_GATHER_INTO_TENSOR) {
buffer = allgather_into_tensor_buffer;
buffer = ctx->allgather_into_tensor_buffer;
} else {
static_assert(
STATE_GROUP == STATE_GROUP_ALL_GATHER || STATE_GROUP == STATE_GROUP_ALL_GATHER_INTO_TENSOR,
@@ -409,22 +504,26 @@ void naive_all_gather(char* result_ptr, char* data_ptr, size_t res_stride, size_
}
state_idx = (state_idx + 1) % 3;
parallel_memcpy(buffer[current_buffer][world_rank], data_ptr, chunk_size);
parallel_memcpy(buffer[current_buffer][ctx->group_rank], data_ptr, chunk_size);
std::atomic_thread_fence(std::memory_order_release);
workspace[world_rank]->states[STATE_GROUP] = copy_current;
ctx->workspace[ctx->group_rank]->states[STATE_GROUP] = copy_current;
for (int i = 0; i < world_size; i++) {
for (int i = 0; i < ctx->group_size; i++) {
// wait until all the other ranks copy the buffer
if (i != world_rank) wait_buffer_state_until_2(i, copy_current, copy_next, STATE_GROUP);
if (i != ctx->group_rank) {
wait_buffer_state_until_2(ctx, i, copy_current, copy_next, STATE_GROUP);
}
}
for (int i = 0; i < world_size; i++) {
for (int i = 0; i < ctx->group_size; i++) {
parallel_memcpy(result_ptr + i * res_stride, buffer[current_buffer][i], chunk_size);
}
current_buffer = 1 - current_buffer;
}
template <int STATE_GROUP>
torch::Tensor& all_gather(torch::Tensor& result, torch::Tensor& data, int dim, size_t numel, int data_size) {
torch::Tensor&
all_gather(torch::Tensor& result, torch::Tensor& data, int dim, size_t numel, int data_size, int64_t handle) {
auto* ctx = get_shm_context(handle);
size_t dim_el = data.stride(dim) * data.size(dim);
int dtype_size = data_size / numel;
size_t dim_size = dim_el * dtype_size;
@@ -436,7 +535,8 @@ torch::Tensor& all_gather(torch::Tensor& result, torch::Tensor& data, int dim, s
size_t chunk_size = dim_size - offset > MAX_BUF_SIZE ? MAX_BUF_SIZE : dim_size - offset;
size_t chunk_el = chunk_size / dtype_size;
naive_all_gather<STATE_GROUP>(
result_ptr + i * dim_size * world_size + offset,
ctx,
result_ptr + i * dim_size * ctx->group_size + offset,
data_ptr + i * dim_size + offset,
dim_size,
chunk_size,
@@ -445,12 +545,11 @@ torch::Tensor& all_gather(torch::Tensor& result, torch::Tensor& data, int dim, s
}
return result;
}
template torch::Tensor& all_gather<STATE_GROUP_ALL_GATHER>(torch::Tensor&, torch::Tensor&, int, size_t, int);
template torch::Tensor& all_gather<STATE_GROUP_ALL_GATHER>(torch::Tensor&, torch::Tensor&, int, size_t, int, int64_t);
template torch::Tensor&
all_gather<STATE_GROUP_ALL_GATHER_INTO_TENSOR>(torch::Tensor&, torch::Tensor&, int, size_t, int);
all_gather<STATE_GROUP_ALL_GATHER_INTO_TENSOR>(torch::Tensor&, torch::Tensor&, int, size_t, int, int64_t);
void naive_reduce_scatter(
struct shm_context* ctx,
char* output_ptr,
char* data_ptr,
c10::ScalarType scalar_type,
@@ -458,8 +557,8 @@ void naive_reduce_scatter(
size_t chunk_el,
int element_size) {
const int state_group = STATE_GROUP_REDUCE_SCATTER;
static int current_buffer = 0;
static int state_idx = 0;
int& current_buffer = ctx->reduce_scatter_current_buffer;
int& state_idx = ctx->reduce_scatter_state_idx;
enum coll_state copy_current = coll_reduce_scatter_naive__copy_in_done;
enum coll_state copy_next = coll_alt1_reduce_scatter_naive__copy_in_done;
@@ -483,40 +582,97 @@ void naive_reduce_scatter(
state_idx = (state_idx + 1) % 3;
// Step 1: copy local data to shared buffer
parallel_memcpy(reduce_scatter_buffer[current_buffer][world_rank], data_ptr, chunk_size);
parallel_memcpy(ctx->reduce_scatter_buffer[current_buffer][ctx->group_rank], data_ptr, chunk_size);
std::atomic_thread_fence(std::memory_order_release);
workspace[world_rank]->states[state_group] = copy_current;
ctx->workspace[ctx->group_rank]->states[state_group] = copy_current;
// Step 2: wait for all ranks to copy in
for (int i = 0; i < world_size; i++) {
if (i != world_rank) wait_buffer_state_until_2(i, copy_current, copy_next, state_group);
for (int i = 0; i < ctx->group_size; i++) {
if (i != ctx->group_rank) {
wait_buffer_state_until_2(ctx, i, copy_current, copy_next, state_group);
}
}
// // Step 3: do local reduce on this rank’s slice only
int start_el = slice_el_start(chunk_el, world_rank);
// each rank reduce its slice of buffer independently so therre is no need for
// Step 3: do local reduce on this rank's slice only
int start_el = slice_el_start(chunk_el, ctx->group_rank, ctx->group_size);
// each rank reduce its slice of buffer independently so there is no need for
// synchronization afterward
reduce_all_buffers(
start_el,
slice_size(chunk_el, world_rank),
slice_size(chunk_el, ctx->group_rank, ctx->group_size),
scalar_type,
world_rank,
output_ptr -
start_el * element_size, // in reduce_all_buffers, the output_ptr is the buffer for all ranks, but here
// output_ptr is already the local buffer for one rank. Adjust it here.
reduce_scatter_buffer[current_buffer]);
output_ptr - start_el * element_size,
ctx->reduce_scatter_buffer[current_buffer],
ctx->group_size);
// done
current_buffer = 1 - current_buffer;
}
void reduce_scatter_outer_loop(torch::Tensor& output, torch::Tensor& data, size_t numel, int data_size) {
void reduce_scatter_outer_loop(
torch::Tensor& output, torch::Tensor& data, size_t numel, int data_size, int64_t handle) {
auto* ctx = get_shm_context(handle);
for (int offset = 0; offset < data_size; offset += MAX_BUF_SIZE) {
auto data_ptr = ((char*)(data.data_ptr()) + offset);
auto output_ptr = ((char*)(output.data_ptr()) + offset);
size_t chunk_size = std::min((size_t)MAX_BUF_SIZE, (size_t)(data_size - offset));
size_t chunk_el = chunk_size / (data_size / numel);
naive_reduce_scatter(output_ptr, data_ptr, data.scalar_type(), chunk_size, chunk_el, data.element_size());
naive_reduce_scatter(ctx, output_ptr, data_ptr, data.scalar_type(), chunk_size, chunk_el, data.element_size());
}
}
void all_to_all(char* output_ptr, char* input_ptr, size_t data_size, int64_t handle) {
auto* ctx = get_shm_context(handle);
TORCH_CHECK(data_size <= MAX_BUF_SIZE, "SHM all-to-all input size exceeds maximum buffer size");
TORCH_CHECK(data_size % ctx->group_size == 0, "SHM all-to-all input size must be divisible by group size");
int& current_buffer = ctx->alltoall_current_buffer;
int& state_idx = ctx->alltoall_state_idx;
enum coll_state copy_current = coll_allgather_naive__copy_in_done;
enum coll_state copy_next = coll_alt1_allgather_naive__copy_in_done;
switch (state_idx) {
case 0:
copy_current = coll_allgather_naive__copy_in_done;
copy_next = coll_alt1_allgather_naive__copy_in_done;
break;
case 1:
copy_current = coll_alt1_allgather_naive__copy_in_done;
copy_next = coll_alt2_allgather_naive__copy_in_done;
break;
case 2:
copy_current = coll_alt2_allgather_naive__copy_in_done;
copy_next = coll_allgather_naive__copy_in_done;
break;
default:
assert(!"Should not get here.");
}
state_idx = (state_idx + 1) % 3;
// copy local input to shared buffer
parallel_memcpy(ctx->alltoall_buffer[current_buffer][ctx->group_rank], input_ptr, data_size);
std::atomic_thread_fence(std::memory_order_release);
ctx->workspace[ctx->group_rank]->states[STATE_GROUP_ALL_TO_ALL] = copy_current;
// wait until all ranks copy the buffer
for (int i = 0; i < ctx->group_size; i++) {
if (i != ctx->group_rank) {
wait_buffer_state_until_2(ctx, i, copy_current, copy_next, STATE_GROUP_ALL_TO_ALL);
}
}
size_t chunk_size = data_size / ctx->group_size;
// read the chunk addressed to this rank from every source rank
for (int i = 0; i < ctx->group_size; i++) {
parallel_memcpy(
output_ptr + i * chunk_size,
ctx->alltoall_buffer[current_buffer][i] + ctx->group_rank * chunk_size,
chunk_size);
}
current_buffer = 1 - current_buffer;
}
+19 -5
View File
@@ -10,10 +10,24 @@ constexpr int STATE_GROUP_DISTRIBUTED_ALLREDUCE = 1;
constexpr int STATE_GROUP_ALL_GATHER = 2;
constexpr int STATE_GROUP_ALL_GATHER_INTO_TENSOR = 3;
constexpr int STATE_GROUP_REDUCE_SCATTER = 4;
constexpr int STATE_GROUP_ALL_TO_ALL = 5;
int64_t shm_initialize(int size, int rank, const char* addr_string, const char* port_string, const char* group_name);
void shm_default_initialize(int size, int rank, const char* addr_string, const char* port_string);
int64_t shm_group_initialize(const std::string& group_name, int64_t group_size, int64_t group_rank);
int64_t shm_get_group_size(int64_t handle);
void all_reduce_outer_loop(torch::Tensor& data, size_t numel, int data_size, int64_t handle);
void shm_initialize(int size, int rank, const char* addr_string, const char* port_string);
void all_reduce_outer_loop(torch::Tensor& data, size_t numel, int data_size);
template <int STATE_GROUP>
torch::Tensor& all_gather(torch::Tensor& result, torch::Tensor& data, int dim, size_t numel, int data_size);
void reduce_scatter_outer_loop(torch::Tensor& output, torch::Tensor& data, size_t numel, int data_size);
#endif
torch::Tensor&
all_gather(torch::Tensor& result, torch::Tensor& data, int dim, size_t numel, int data_size, int64_t handle);
void reduce_scatter_outer_loop(torch::Tensor& output, torch::Tensor& data, size_t numel, int data_size, int64_t handle);
void all_to_all(char* output_ptr, char* input_ptr, size_t data_size, int64_t handle);
#endif // __SHM_COLLECTIVES__
@@ -127,7 +127,6 @@ std::tuple<at::Tensor, at::Tensor, at::Tensor> fused_qk_gemma_rmsnorm_with_gate_
double eps,
int64_t head_dim,
int64_t num_head);
// speculative decoding
void verify_tree_greedy_cpu(
at::Tensor predicts,
@@ -509,17 +508,23 @@ at::Tensor conv3d_embed_cpu(const at::Tensor& input, const at::Tensor& weight, c
// shared memory init
void initialize(int64_t size, int64_t rank);
// shared mmeory all_reduce
void shm_allreduce(at::Tensor& data, int64_t op);
// group shared memory init
int64_t shm_group_initialize(const std::string& group_name, int64_t group_size, int64_t group_rank);
// shared memory all_reduce
void shm_allreduce(at::Tensor& data, int64_t op, int64_t handle = -1);
// shared memory all_gather
at::Tensor shm_allgather(at::Tensor& data, int64_t dim);
at::Tensor shm_allgather(at::Tensor& data, int64_t dim, int64_t handle = -1);
// group shared memory all_to_all
void shm_alltoall(at::Tensor& output_tensor, at::Tensor& data, int64_t handle);
// shared memory all_gather_into_tensor
void shm_allgather_into_tensor(at::Tensor& output_tensor, at::Tensor& data);
void shm_allgather_into_tensor(at::Tensor& output_tensor, at::Tensor& data, int64_t handle = -1);
// shared memory reduce_scatter_tensor
void shm_reduce_scatter_tensor(at::Tensor& output_tensor, at::Tensor& data, int64_t op);
void shm_reduce_scatter_tensor(at::Tensor& output_tensor, at::Tensor& data, int64_t op, int64_t handle = -1);
// rope
std::tuple<at::Tensor, at::Tensor> rotary_embedding_cpu(
@@ -943,17 +948,32 @@ TORCH_LIBRARY_FRAGMENT(sgl_kernel, m) {
m.def("conv3d_embed_cpu(Tensor input, Tensor weight, Tensor bias, bool is_vnni) -> Tensor");
m.impl("conv3d_embed_cpu", torch::kCPU, &conv3d_embed_cpu);
// all reduce
// shared memory
m.def("initialize(int size, int rank) -> ()");
m.def("shm_allreduce(Tensor(a!) data, int reduce_op) -> ()");
// all_reduce
m.def("shm_allreduce(Tensor(a!) data, int reduce_op, int handle=-1) -> ()");
m.impl("shm_allreduce", torch::kCPU, &shm_allreduce);
m.def("shm_allgather(Tensor data, int dim) -> Tensor");
// all_gather
m.def("shm_allgather(Tensor data, int dim, int handle=-1) -> Tensor");
m.impl("shm_allgather", torch::kCPU, &shm_allgather);
m.def("shm_allgather_into_tensor(Tensor(a!) output_tensor, Tensor data) -> ()");
// all_gather_into_tensor
m.def("shm_allgather_into_tensor(Tensor(a!) output_tensor, Tensor data, int handle=-1) -> ()");
m.impl("shm_allgather_into_tensor", torch::kCPU, &shm_allgather_into_tensor);
m.def("shm_reduce_scatter_tensor(Tensor(a!) output_tensor, Tensor data, int reduce_op) -> ()");
// reduce_scatter_tensor
m.def("shm_reduce_scatter_tensor(Tensor(a!) output_tensor, Tensor data, int reduce_op, int handle=-1) -> ()");
m.impl("shm_reduce_scatter_tensor", torch::kCPU, &shm_reduce_scatter_tensor);
// explicit group context initialization
m.def("shm_group_initialize(str group_name, int group_size, int group_rank) -> int");
// all_to_all
m.def("shm_alltoall(Tensor(a!) output_tensor, Tensor data, int handle) -> ()");
m.impl("shm_alltoall", torch::kCPU, &shm_alltoall);
// rope
m.def(
"rotary_embedding_cpu(Tensor positions, Tensor query, Tensor key, int head_size, Tensor cos_sin_cache, "
@@ -1028,6 +1048,7 @@ TORCH_LIBRARY_FRAGMENT(sgl_kernel, m) {
TORCH_LIBRARY_IMPL(sgl_kernel, CatchAll, m) {
m.impl("init_cpu_threads_env", init_cpu_threads_env);
m.impl("initialize", &initialize);
m.impl("shm_group_initialize", &shm_group_initialize);
}
REGISTER_EXTENSION(common_ops)
@@ -2,6 +2,8 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo
import os
# Copyright 2024 xDiT team.
# Adapted from
# https://github.com/vllm-project/vllm/blob/main/vllm/distributed/parallel_state.py
@@ -219,6 +221,28 @@ class GroupCoordinator:
# TODO: fix it for other platforms
self.device = get_local_torch_device()
self.shm_handle = None
global_world_size = torch.distributed.get_world_size()
local_size = int(os.environ.get("LOCAL_SIZE", "0"))
self.is_single_node = local_size > 0 and global_world_size == local_size
if (
current_platform.is_cpu()
and self.is_single_node
and self.world_size > 1
and hasattr(torch.ops.sgl_kernel, "shm_group_initialize")
and self.unique_name.startswith(("tp", "sp"))
):
group_ranks = "-".join(str(rank) for rank in self.ranks)
group_name = f"{self.unique_name}-{group_ranks}"
self.shm_handle = int(
torch.ops.sgl_kernel.shm_group_initialize(
group_name,
self.world_size,
self.rank_in_group,
)
)
torch.distributed.barrier(group=self.cpu_group)
self.device_communicator: DeviceCommunicatorBase = None # type: ignore
if use_device_communicator and self.world_size > 1:
@@ -396,17 +420,20 @@ class GroupCoordinator:
return output
if (
current_platform.is_cpu()
and not async_op
and self.shm_handle is not None
and is_shm_available(input_.dtype, self.world_size, len(self.ranks))
and op is torch.distributed.ReduceOp.SUM
):
# for CPU platform, intra-node case we could speedup with shared memory based comm ops
torch.ops.sgl_kernel.shm_allreduce(
input_, int(torch.distributed.ReduceOp.SUM)
input_, int(torch.distributed.ReduceOp.SUM), self.shm_handle
)
else:
torch.distributed.all_reduce(
input_, op=op, group=self.device_group, async_op=async_op
)
return input_
def _all_reduce_out_of_place(self, input_: torch.Tensor) -> torch.Tensor:
@@ -440,18 +467,24 @@ class GroupCoordinator:
input_size, dtype=input_.dtype, device=input_.device
)
# All-gather.
if current_platform.is_cpu() and is_shm_available(
input_.dtype, self.world_size, len(self.ranks)
# Group-aware CPU SHM all-gather.
if (
current_platform.is_cpu()
and self.shm_handle is not None
and is_shm_available(input_.dtype, self.world_size, len(self.ranks))
):
output_tensor = torch.ops.sgl_kernel.shm_allgather(input_, dim)
output_tensor = torch.ops.sgl_kernel.shm_allgather(
input_, dim, self.shm_handle
)
if separate_tensors:
return list(output_tensor.chunk(world_size, dim=dim))
return output_tensor
else:
if not input_.is_contiguous():
input_ = input_.contiguous()
all_gather_single(output_tensor, input_, group=self.device_group)
# Fallback.
if not input_.is_contiguous():
input_ = input_.contiguous()
all_gather_single(output_tensor, input_, group=self.device_group)
if dim != 0:
input_size[0] //= world_size
@@ -889,3 +922,13 @@ class SequenceParallelGroupCoordinator(GroupCoordinator):
self.ulysses_rank = torch.distributed.get_rank(self.ulysses_group)
self.ring_world_size = torch.distributed.get_world_size(self.ring_group)
self.ring_rank = torch.distributed.get_rank(self.ring_group)
self.ulysses_shm_handle = None
if (
current_platform.is_cpu()
and self.ulysses_world_size > 1
and hasattr(torch.ops.sgl_kernel, "shm_group_initialize")
):
# TODO: Initialize a dedicated SHM handle for the Ulysses subgroup
# when Ring Attention is supported on CPU. The current SP handle
# is only valid when ring_degree == 1.
self.ulysses_shm_handle = self.shm_handle
@@ -16,9 +16,11 @@ from sglang.multimodal_gen.runtime.distributed.parallel_state import (
get_ulysses_parallel_rank,
get_ulysses_parallel_world_size,
)
from sglang.multimodal_gen.runtime.platforms import current_platform
from sglang.srt.utils.common import torch_release
_cp_options.enable_load_balance = False
SHM_ALLTOALL_MAX_BYTES = 32 * 1024 * 1024
if TYPE_CHECKING:
from sglang.multimodal_gen.runtime.layers.attention.backends.attention_backend import (
@@ -105,7 +107,8 @@ def _a2a_staging_buffer(
def _usp_all_to_all_single(x: torch.Tensor, role: str | None = None) -> torch.Tensor:
ulysses_pg = get_sp_group().ulysses_group
sp_group = get_sp_group()
ulysses_pg = sp_group.ulysses_group
assert ulysses_pg is not None, "Ulysses process group is not initialized."
x_shape = x.shape
x = x.flatten().contiguous()
@@ -124,9 +127,21 @@ def _usp_all_to_all_single(x: torch.Tensor, role: str | None = None) -> torch.Te
output = torch.empty_like(x)
else:
output = _a2a_staging_buffer(role, x.shape, x.dtype, x.device)
# USP calls this collective many times per denoising step and waits
# immediately, so avoid the extra wrapper overhead of functional collectives.
torch.distributed.all_to_all_single(output, x, group=ulysses_pg)
handle = sp_group.ulysses_shm_handle
input_bytes = x.numel() * x.element_size()
if (
current_platform.is_cpu()
and handle is not None
and handle >= 0
and input_bytes <= SHM_ALLTOALL_MAX_BYTES
):
torch.ops.sgl_kernel.shm_alltoall(output, x, sp_group.ulysses_shm_handle)
else:
torch.distributed.all_to_all_single(output, x, group=ulysses_pg)
return output.reshape(x_shape)
@@ -30,6 +30,7 @@ class CPUWorker(GPUWorker):
master_port: int,
server_args: ServerArgs,
):
os.environ["LOCAL_SIZE"] = str(server_args.num_gpus // server_args.nnodes)
super().__init__(local_rank, rank, master_port, server_args)
if _is_cpu_amx_available:
self.init_cpu_threads_binding()
@@ -70,7 +71,3 @@ class CPUWorker(GPUWorker):
# Bind OpenMP threads to CPU cores
torch.ops.sgl_kernel.init_cpu_threads_env(self.local_omp_cpuid)
# Set local size to hint SGLang to use shared memory based AllReduce
os.environ["LOCAL_SIZE"] = str(self.server_args.tp_size)
torch.ops.sgl_kernel.initialize(self.server_args.tp_size, self.rank)