[CPU]: Add group-aware CPU SHM collective kernels (#35435)
This commit is contained in:
@@ -1,4 +1,3 @@
|
||||
#include <ATen/record_function.h>
|
||||
#include <torch/all.h>
|
||||
|
||||
#include "shm.h"
|
||||
@@ -43,50 +42,50 @@ void initialize(int64_t size, int64_t rank) {
|
||||
}
|
||||
|
||||
if (all_ranks_local_p) {
|
||||
shm_initialize(size, rank, addr_string, port_string);
|
||||
shm_default_initialize(size, rank, addr_string, port_string);
|
||||
}
|
||||
}
|
||||
|
||||
void shm_allreduce(torch::Tensor& data, int64_t op) {
|
||||
void shm_allreduce(torch::Tensor& data, int64_t op, int64_t handle) {
|
||||
TORCH_CHECK(op == c10d::ReduceOp::SUM, "Only torch.distributed.ReduceOp.SUM is supported");
|
||||
|
||||
auto numel = data.numel();
|
||||
int data_size = numel * data.element_size();
|
||||
all_reduce_outer_loop(data, numel, data_size);
|
||||
|
||||
return;
|
||||
all_reduce_outer_loop(data, numel, data_size, handle);
|
||||
}
|
||||
|
||||
torch::Tensor shm_allgather(torch::Tensor& data, int64_t dim) {
|
||||
torch::Tensor shm_allgather(torch::Tensor& data, int64_t dim, int64_t handle) {
|
||||
auto numel = data.numel();
|
||||
int data_size = numel * data.element_size();
|
||||
if (dim < 0) {
|
||||
dim += data.dim();
|
||||
}
|
||||
int64_t group_size = handle == -1 ? world_size : shm_get_group_size(handle);
|
||||
std::vector<int64_t> result_shape = data.sizes().vec();
|
||||
result_shape[dim] *= world_size;
|
||||
result_shape[dim] *= group_size;
|
||||
torch::Tensor result_tensor = torch::empty(result_shape, data.options());
|
||||
return all_gather<STATE_GROUP_ALL_GATHER>(result_tensor, data, dim, numel, data_size);
|
||||
return all_gather<STATE_GROUP_ALL_GATHER>(result_tensor, data, dim, numel, data_size, handle);
|
||||
}
|
||||
|
||||
void shm_allgather_into_tensor(torch::Tensor& output_tensor, torch::Tensor& data) {
|
||||
RECORD_FUNCTION("sgl-kernel::shm_allgather_into_tensor", std::vector<c10::IValue>({data}));
|
||||
|
||||
void shm_allgather_into_tensor(torch::Tensor& output_tensor, torch::Tensor& data, int64_t handle) {
|
||||
auto numel = data.numel();
|
||||
int data_size = numel * data.element_size();
|
||||
int64_t dim = 0;
|
||||
all_gather<STATE_GROUP_ALL_GATHER_INTO_TENSOR>(output_tensor, data, dim, numel, data_size);
|
||||
all_gather<STATE_GROUP_ALL_GATHER_INTO_TENSOR>(output_tensor, data, dim, numel, data_size, handle);
|
||||
}
|
||||
|
||||
void shm_reduce_scatter_tensor(at::Tensor& output_tensor, at::Tensor& data, int64_t op) {
|
||||
RECORD_FUNCTION("sgl-kernel::shm_reduce_scatter_tensor", std::vector<c10::IValue>({data}));
|
||||
|
||||
void shm_reduce_scatter_tensor(at::Tensor& output_tensor, at::Tensor& data, int64_t op, int64_t handle) {
|
||||
TORCH_CHECK(op == c10d::ReduceOp::SUM, "Only torch.distributed.ReduceOp.SUM is supported");
|
||||
|
||||
auto numel = data.numel();
|
||||
int data_size = numel * data.element_size();
|
||||
|
||||
reduce_scatter_outer_loop(output_tensor, data, numel, data_size);
|
||||
reduce_scatter_outer_loop(output_tensor, data, numel, data_size, handle);
|
||||
}
|
||||
|
||||
return;
|
||||
void shm_alltoall(torch::Tensor& output_tensor, torch::Tensor& input_tensor, int64_t handle) {
|
||||
size_t data_size = input_tensor.numel() * input_tensor.element_size();
|
||||
all_to_all(
|
||||
static_cast<char*>(output_tensor.data_ptr()), static_cast<char*>(input_tensor.data_ptr()), data_size, handle);
|
||||
}
|
||||
|
||||
@@ -73,20 +73,21 @@ void shared_create(SharedData* data, const char* name, void* bytes, size_t nbyte
|
||||
}
|
||||
}
|
||||
|
||||
static int world_size;
|
||||
|
||||
// SHM based allreduce helper functions
|
||||
// buffer that holds shm name
|
||||
#define NAME_BUF_SIZE 1000
|
||||
#define MAX_BUF_SIZE 1048576 * 32
|
||||
#define NAIVE_ALLREDUCE_THRESHOLD 1048576
|
||||
#define SHM_BUFFER_NAME "deepspeed_allreduce_buffer"
|
||||
constexpr int kNumCollectiveStates = STATE_GROUP_ALL_TO_ALL + 1;
|
||||
// idx=0 -- state for symmetric_naive_all_reduce
|
||||
// idx=1 -- state for distributed_naive_all_reduce
|
||||
// idx=2 -- state for all_gather
|
||||
// idx=3 -- state for all_gather_into_tensor
|
||||
// idx=4 -- state for reduce_scatter
|
||||
// idx=5 -- state for all_to_all
|
||||
struct allreduce_workspace {
|
||||
enum coll_state states[5]; // idx=0 -- state for symmetric_naive_all_reduce
|
||||
// idx=1 -- state for distributed_naive_all_reduce
|
||||
// idx=2 -- state for all_gather
|
||||
// idx=3 -- state for all_gather_into_tensor
|
||||
// idx=4 -- state for reduce_scatter
|
||||
enum coll_state states[kNumCollectiveStates];
|
||||
// double buffer to avoid syncing between rounds
|
||||
// offset=0 -- 2*NAIVE_ALLREDUCE_THRESHOLD : buffer for
|
||||
// symmetric_naive_all_reduce after that : buffer for
|
||||
@@ -96,7 +97,9 @@ struct allreduce_workspace {
|
||||
2 * MAX_BUF_SIZE + // distributed naive reduce
|
||||
2 * MAX_BUF_SIZE + // allgather
|
||||
2 * MAX_BUF_SIZE + // allgather_into_tensor
|
||||
2 * MAX_BUF_SIZE // reduce_scatter
|
||||
2 * MAX_BUF_SIZE + // reduce_scatter
|
||||
2 * MAX_BUF_SIZE // all_to_all
|
||||
|
||||
];
|
||||
};
|
||||
|
||||
@@ -108,20 +111,63 @@ struct allreduce_workspace {
|
||||
(2 * NAIVE_ALLREDUCE_THRESHOLD + 4 * MAX_BUF_SIZE + current_buffer * MAX_BUF_SIZE) // allgather_into_tensor
|
||||
#define BUFFER4_OFFSET(current_buffer) \
|
||||
(2 * NAIVE_ALLREDUCE_THRESHOLD + 6 * MAX_BUF_SIZE + current_buffer * MAX_BUF_SIZE) // reduce_scatter
|
||||
#define BUFFER5_OFFSET(current_buffer) \
|
||||
(2 * NAIVE_ALLREDUCE_THRESHOLD + 8 * MAX_BUF_SIZE + current_buffer * MAX_BUF_SIZE) // all_to_all
|
||||
|
||||
struct allreduce_workspace** workspace;
|
||||
struct shm_context {
|
||||
int group_size;
|
||||
int group_rank;
|
||||
|
||||
// buffer for small messages, double buffer
|
||||
char** symmetric_buffer[2];
|
||||
// buffer for large messages, double buffer
|
||||
char** distributed_buffer[2];
|
||||
struct allreduce_workspace** workspace;
|
||||
|
||||
char** allgather_buffer[2];
|
||||
char** allgather_into_tensor_buffer[2];
|
||||
char** reduce_scatter_buffer[2];
|
||||
char** symmetric_buffer[2];
|
||||
char** distributed_buffer[2];
|
||||
char** allgather_buffer[2];
|
||||
char** allgather_into_tensor_buffer[2];
|
||||
char** reduce_scatter_buffer[2];
|
||||
char** alltoall_buffer[2];
|
||||
|
||||
void wait_buffer_state_until_2(int index, enum coll_state state0, enum coll_state state1, int state_group) {
|
||||
volatile enum coll_state* state_ptr = &(workspace[index]->states[state_group]);
|
||||
int symmetric_current_buffer;
|
||||
int symmetric_state_idx;
|
||||
|
||||
int distributed_current_buffer;
|
||||
int distributed_state_idx;
|
||||
|
||||
int allgather_current_buffer;
|
||||
int allgather_state_idx;
|
||||
|
||||
int allgather_into_tensor_current_buffer;
|
||||
int allgather_into_tensor_state_idx;
|
||||
|
||||
int reduce_scatter_current_buffer;
|
||||
int reduce_scatter_state_idx;
|
||||
|
||||
int alltoall_current_buffer;
|
||||
int alltoall_state_idx;
|
||||
};
|
||||
|
||||
static struct shm_context* default_shm_context = nullptr;
|
||||
static std::vector<struct shm_context*> shm_contexts;
|
||||
|
||||
static struct shm_context* get_shm_context(int64_t handle) {
|
||||
if (handle == -1) {
|
||||
TORCH_CHECK(default_shm_context != nullptr, "Default SHM context is not initialized");
|
||||
return default_shm_context;
|
||||
}
|
||||
TORCH_CHECK(handle >= 0 && handle < static_cast<int64_t>(shm_contexts.size()), "Invalid SHM context handle: ");
|
||||
|
||||
auto* ctx = shm_contexts[handle];
|
||||
TORCH_CHECK(ctx != nullptr, "SHM context is null");
|
||||
return ctx;
|
||||
}
|
||||
|
||||
int64_t shm_get_group_size(int64_t handle) {
|
||||
return get_shm_context(handle)->group_size;
|
||||
}
|
||||
|
||||
void wait_buffer_state_until_2(
|
||||
struct shm_context* ctx, int index, enum coll_state state0, enum coll_state state1, int state_group) {
|
||||
volatile enum coll_state* state_ptr = &(ctx->workspace[index]->states[state_group]);
|
||||
|
||||
while (1) {
|
||||
volatile enum coll_state cur_state = *state_ptr;
|
||||
@@ -133,40 +179,53 @@ void reduce_all_buffers(
|
||||
int start_elements,
|
||||
int num_elements,
|
||||
c10::ScalarType scalar_type,
|
||||
int to_buffer_idx,
|
||||
char* to_buffer,
|
||||
char** buffers) {
|
||||
char** buffers,
|
||||
int group_size) {
|
||||
switch (scalar_type) {
|
||||
case c10::ScalarType::BFloat16:
|
||||
reduce_bf16_buffers(start_elements, num_elements, to_buffer, buffers, world_size);
|
||||
reduce_bf16_buffers(start_elements, num_elements, to_buffer, buffers, group_size);
|
||||
break;
|
||||
case c10::ScalarType::Half:
|
||||
reduce_fp16_buffers(start_elements, num_elements, to_buffer, buffers, world_size);
|
||||
reduce_fp16_buffers(start_elements, num_elements, to_buffer, buffers, group_size);
|
||||
break;
|
||||
case c10::ScalarType::Float:
|
||||
reduce_fp32_buffers(start_elements, num_elements, to_buffer, buffers, world_size);
|
||||
reduce_fp32_buffers(start_elements, num_elements, to_buffer, buffers, group_size);
|
||||
break;
|
||||
default:
|
||||
assert(!"Should not get here");
|
||||
}
|
||||
}
|
||||
|
||||
static bool is_initialized = false;
|
||||
static int world_rank;
|
||||
|
||||
void shm_initialize(int size, int rank, const char* addr_string, const char* port_string) {
|
||||
if (is_initialized) {
|
||||
return;
|
||||
}
|
||||
is_initialized = true;
|
||||
|
||||
world_size = size;
|
||||
world_rank = rank;
|
||||
int64_t shm_initialize(int size, int rank, const char* addr_string, const char* port_string, const char* group_name) {
|
||||
auto* ctx = (struct shm_context*)malloc(sizeof(struct shm_context));
|
||||
ctx->group_size = size;
|
||||
ctx->group_rank = rank;
|
||||
ctx->symmetric_current_buffer = 0;
|
||||
ctx->symmetric_state_idx = 0;
|
||||
ctx->distributed_current_buffer = 0;
|
||||
ctx->distributed_state_idx = 0;
|
||||
ctx->allgather_current_buffer = 0;
|
||||
ctx->allgather_state_idx = 0;
|
||||
ctx->allgather_into_tensor_current_buffer = 0;
|
||||
ctx->allgather_into_tensor_state_idx = 0;
|
||||
ctx->reduce_scatter_current_buffer = 0;
|
||||
ctx->reduce_scatter_state_idx = 0;
|
||||
ctx->alltoall_current_buffer = 0;
|
||||
ctx->alltoall_state_idx = 0;
|
||||
|
||||
char shm_name_prefix[NAME_BUF_SIZE];
|
||||
char shm_name[NAME_BUF_SIZE];
|
||||
snprintf(shm_name_prefix, NAME_BUF_SIZE, "%s_%d_%s_%s", SHM_BUFFER_NAME, getuid(), addr_string, port_string);
|
||||
// create shared workspace for SHM based allreduce
|
||||
snprintf(
|
||||
shm_name_prefix,
|
||||
NAME_BUF_SIZE,
|
||||
"%s_%d_%s_%s_%s",
|
||||
SHM_BUFFER_NAME,
|
||||
getuid(),
|
||||
addr_string,
|
||||
port_string,
|
||||
group_name);
|
||||
// create shared workspace for SHM based collectives
|
||||
SharedData allreduce_buffer;
|
||||
// allocate workspace_buf for current rank
|
||||
struct allreduce_workspace* workspace_buf;
|
||||
@@ -174,30 +233,34 @@ void shm_initialize(int size, int rank, const char* addr_string, const char* por
|
||||
workspace_buf = (struct allreduce_workspace*)malloc(sizeof(struct allreduce_workspace));
|
||||
snprintf(shm_name, NAME_BUF_SIZE, "%.900s_%d", shm_name_prefix, rank);
|
||||
shared_create(&allreduce_buffer, shm_name, workspace_buf, sizeof(struct allreduce_workspace));
|
||||
free(workspace_buf);
|
||||
workspace_buf = (struct allreduce_workspace*)allreduce_buffer.bytes;
|
||||
workspace_buf->states[STATE_GROUP_SYMMETRIC_ALLREDUCE] =
|
||||
coll_alt2_allreduce_naive__copy_in_done; // symmetric_naive_all_reduce
|
||||
workspace_buf->states[STATE_GROUP_DISTRIBUTED_ALLREDUCE] = coll_begin; // distributed_naive_reduce
|
||||
workspace_buf->states[STATE_GROUP_ALL_GATHER] = coll_alt2_allgather_naive__copy_in_done; // all_gather
|
||||
workspace_buf->states[STATE_GROUP_ALL_GATHER_INTO_TENSOR] =
|
||||
coll_alt2_allgather_naive__copy_in_done; // all_gather_into_tensor
|
||||
workspace_buf->states[STATE_GROUP_REDUCE_SCATTER] = coll_begin; // reduce_scatter
|
||||
coll_alt2_allgather_naive__copy_in_done; // all_gather_into_tensor
|
||||
workspace_buf->states[STATE_GROUP_REDUCE_SCATTER] = coll_begin; // reduce_scatter
|
||||
workspace_buf->states[STATE_GROUP_ALL_TO_ALL] = coll_alt2_allgather_naive__copy_in_done; // all_to_all
|
||||
|
||||
// create the workspace pointer list
|
||||
workspace = (struct allreduce_workspace**)malloc(size * sizeof(struct allreduce_workspace*));
|
||||
symmetric_buffer[0] = (char**)malloc(size * sizeof(char**));
|
||||
symmetric_buffer[1] = (char**)malloc(size * sizeof(char**));
|
||||
distributed_buffer[0] = (char**)malloc(size * sizeof(char**));
|
||||
distributed_buffer[1] = (char**)malloc(size * sizeof(char**));
|
||||
ctx->workspace = (struct allreduce_workspace**)malloc(size * sizeof(struct allreduce_workspace*));
|
||||
ctx->symmetric_buffer[0] = (char**)malloc(size * sizeof(char*));
|
||||
ctx->symmetric_buffer[1] = (char**)malloc(size * sizeof(char*));
|
||||
ctx->distributed_buffer[0] = (char**)malloc(size * sizeof(char*));
|
||||
ctx->distributed_buffer[1] = (char**)malloc(size * sizeof(char*));
|
||||
ctx->allgather_buffer[0] = (char**)malloc(size * sizeof(char*));
|
||||
ctx->allgather_buffer[1] = (char**)malloc(size * sizeof(char*));
|
||||
|
||||
allgather_buffer[0] = (char**)malloc(size * sizeof(char*));
|
||||
allgather_buffer[1] = (char**)malloc(size * sizeof(char*));
|
||||
ctx->allgather_into_tensor_buffer[0] = (char**)malloc(size * sizeof(char*));
|
||||
ctx->allgather_into_tensor_buffer[1] = (char**)malloc(size * sizeof(char*));
|
||||
|
||||
allgather_into_tensor_buffer[0] = (char**)malloc(size * sizeof(char*));
|
||||
allgather_into_tensor_buffer[1] = (char**)malloc(size * sizeof(char*));
|
||||
ctx->reduce_scatter_buffer[0] = (char**)malloc(size * sizeof(char*));
|
||||
ctx->reduce_scatter_buffer[1] = (char**)malloc(size * sizeof(char*));
|
||||
|
||||
reduce_scatter_buffer[0] = (char**)malloc(size * sizeof(char*));
|
||||
reduce_scatter_buffer[1] = (char**)malloc(size * sizeof(char*));
|
||||
ctx->alltoall_buffer[0] = (char**)malloc(size * sizeof(char*));
|
||||
ctx->alltoall_buffer[1] = (char**)malloc(size * sizeof(char*));
|
||||
|
||||
// map shm of all ranks
|
||||
for (int i = 0; i < size; i++) {
|
||||
@@ -208,48 +271,71 @@ void shm_initialize(int size, int rank, const char* addr_string, const char* por
|
||||
shared_open(&allreduce_buffer, shm_name, sizeof(struct allreduce_workspace));
|
||||
} while (allreduce_buffer.descriptor == -1 && errno == ENOENT);
|
||||
workspace_buf_other = (struct allreduce_workspace*)allreduce_buffer.bytes;
|
||||
workspace[i] = workspace_buf_other;
|
||||
ctx->workspace[i] = workspace_buf_other;
|
||||
} else {
|
||||
workspace[i] = workspace_buf;
|
||||
ctx->workspace[i] = workspace_buf;
|
||||
}
|
||||
symmetric_buffer[0][i] = workspace[i]->buffer + BUFFER0_OFFSET(0);
|
||||
symmetric_buffer[1][i] = workspace[i]->buffer + BUFFER0_OFFSET(1);
|
||||
distributed_buffer[0][i] = workspace[i]->buffer + BUFFER1_OFFSET(0);
|
||||
distributed_buffer[1][i] = workspace[i]->buffer + BUFFER1_OFFSET(1);
|
||||
|
||||
allgather_buffer[0][i] = workspace[i]->buffer + BUFFER2_OFFSET(0);
|
||||
allgather_buffer[1][i] = workspace[i]->buffer + BUFFER2_OFFSET(1);
|
||||
ctx->symmetric_buffer[0][i] = ctx->workspace[i]->buffer + BUFFER0_OFFSET(0);
|
||||
ctx->symmetric_buffer[1][i] = ctx->workspace[i]->buffer + BUFFER0_OFFSET(1);
|
||||
|
||||
allgather_into_tensor_buffer[0][i] = workspace[i]->buffer + BUFFER3_OFFSET(0);
|
||||
allgather_into_tensor_buffer[1][i] = workspace[i]->buffer + BUFFER3_OFFSET(1);
|
||||
ctx->distributed_buffer[0][i] = ctx->workspace[i]->buffer + BUFFER1_OFFSET(0);
|
||||
ctx->distributed_buffer[1][i] = ctx->workspace[i]->buffer + BUFFER1_OFFSET(1);
|
||||
|
||||
reduce_scatter_buffer[0][i] = workspace[i]->buffer + BUFFER4_OFFSET(0);
|
||||
reduce_scatter_buffer[1][i] = workspace[i]->buffer + BUFFER4_OFFSET(1);
|
||||
ctx->allgather_buffer[0][i] = ctx->workspace[i]->buffer + BUFFER2_OFFSET(0);
|
||||
ctx->allgather_buffer[1][i] = ctx->workspace[i]->buffer + BUFFER2_OFFSET(1);
|
||||
|
||||
ctx->allgather_into_tensor_buffer[0][i] = ctx->workspace[i]->buffer + BUFFER3_OFFSET(0);
|
||||
ctx->allgather_into_tensor_buffer[1][i] = ctx->workspace[i]->buffer + BUFFER3_OFFSET(1);
|
||||
|
||||
ctx->reduce_scatter_buffer[0][i] = ctx->workspace[i]->buffer + BUFFER4_OFFSET(0);
|
||||
ctx->reduce_scatter_buffer[1][i] = ctx->workspace[i]->buffer + BUFFER4_OFFSET(1);
|
||||
|
||||
ctx->alltoall_buffer[0][i] = ctx->workspace[i]->buffer + BUFFER5_OFFSET(0);
|
||||
ctx->alltoall_buffer[1][i] = ctx->workspace[i]->buffer + BUFFER5_OFFSET(1);
|
||||
}
|
||||
|
||||
const int64_t handle = static_cast<int64_t>(shm_contexts.size());
|
||||
shm_contexts.push_back(ctx);
|
||||
return handle;
|
||||
}
|
||||
void shm_default_initialize(int size, int rank, const char* addr_string, const char* port_string) {
|
||||
int64_t handle = shm_initialize(size, rank, addr_string, port_string, "default");
|
||||
|
||||
default_shm_context = shm_contexts[handle];
|
||||
}
|
||||
|
||||
#define positive_mod(num, mod) ((((num) % (mod)) + (mod)) % (mod))
|
||||
#define rank_mod(rank) positive_mod(rank, world_size)
|
||||
size_t slice_size(size_t chunk_el, int slice_idx) {
|
||||
size_t slice_size = chunk_el / world_size;
|
||||
return slice_idx == world_size - 1 ? slice_size + (chunk_el % world_size) : slice_size;
|
||||
int64_t shm_group_initialize(const std::string& group_name, int64_t group_size, int64_t group_rank) {
|
||||
const char* addr_string = getenv("MASTER_ADDR");
|
||||
const char* port_string = getenv("MASTER_PORT");
|
||||
|
||||
TORCH_CHECK(addr_string != nullptr, "MASTER_ADDR is not set");
|
||||
TORCH_CHECK(port_string != nullptr, "MASTER_PORT is not set");
|
||||
|
||||
return shm_initialize(group_size, group_rank, addr_string, port_string, group_name.c_str());
|
||||
}
|
||||
|
||||
char* slice_data(char* data_ptr, size_t chunk_el, int el_size, int slice_idx) {
|
||||
size_t slice_size = chunk_el / world_size;
|
||||
size_t slice_size(size_t chunk_el, int slice_idx, int group_size) {
|
||||
size_t slice_size = chunk_el / group_size;
|
||||
return slice_idx == group_size - 1 ? slice_size + (chunk_el % group_size) : slice_size;
|
||||
}
|
||||
|
||||
char* slice_data(char* data_ptr, size_t chunk_el, int el_size, int slice_idx, int group_size) {
|
||||
size_t slice_size = chunk_el / group_size;
|
||||
size_t el_offset = slice_size * slice_idx;
|
||||
return data_ptr + el_offset * el_size;
|
||||
}
|
||||
|
||||
size_t slice_el_start(size_t chunk_el, int slice_idx) {
|
||||
size_t slice_size = chunk_el / world_size;
|
||||
size_t slice_el_start(size_t chunk_el, int slice_idx, int group_size) {
|
||||
size_t slice_size = chunk_el / group_size;
|
||||
return slice_size * slice_idx;
|
||||
}
|
||||
|
||||
void symmetric_naive_all_reduce(char* data_ptr, c10::ScalarType scalar_type, size_t chunk_size, size_t chunk_el) {
|
||||
void symmetric_naive_all_reduce(
|
||||
struct shm_context* ctx, char* data_ptr, c10::ScalarType scalar_type, size_t chunk_size, size_t chunk_el) {
|
||||
const int state_group = STATE_GROUP_SYMMETRIC_ALLREDUCE;
|
||||
static int current_buffer = 0;
|
||||
static int state_idx = 0;
|
||||
int& current_buffer = ctx->symmetric_current_buffer;
|
||||
int& state_idx = ctx->symmetric_state_idx;
|
||||
|
||||
// init states to case 0 to get rid of "maybe-uninitialized" warning.
|
||||
enum coll_state copy_current = coll_allreduce_naive__copy_in_done;
|
||||
@@ -273,31 +359,30 @@ void symmetric_naive_all_reduce(char* data_ptr, c10::ScalarType scalar_type, siz
|
||||
}
|
||||
state_idx = (state_idx + 1) % 3;
|
||||
|
||||
parallel_memcpy(symmetric_buffer[current_buffer][world_rank], data_ptr, chunk_size);
|
||||
parallel_memcpy(ctx->symmetric_buffer[current_buffer][ctx->group_rank], data_ptr, chunk_size);
|
||||
std::atomic_thread_fence(std::memory_order_release);
|
||||
workspace[world_rank]->states[state_group] = copy_current;
|
||||
ctx->workspace[ctx->group_rank]->states[state_group] = copy_current;
|
||||
|
||||
for (int i = 0; i < world_size; i++) {
|
||||
for (int i = 0; i < ctx->group_size; i++) {
|
||||
// wait until the other rank copy the buffer
|
||||
if (i != world_rank) {
|
||||
wait_buffer_state_until_2(i, copy_current, copy_next, state_group);
|
||||
if (i != ctx->group_rank) {
|
||||
wait_buffer_state_until_2(ctx, i, copy_current, copy_next, state_group);
|
||||
}
|
||||
}
|
||||
|
||||
// each rank reduce the buffer independently so therre is no need for
|
||||
// each rank reduce the buffer independently so there is no need for
|
||||
// synchronization afterward
|
||||
reduce_all_buffers(0, chunk_el, scalar_type, world_rank, data_ptr, symmetric_buffer[current_buffer]);
|
||||
reduce_all_buffers(0, chunk_el, scalar_type, data_ptr, ctx->symmetric_buffer[current_buffer], ctx->group_size);
|
||||
|
||||
// switch buffer
|
||||
current_buffer = 1 - current_buffer;
|
||||
}
|
||||
|
||||
// naive allreduce distributed, each rank do naive reduce on its slice
|
||||
void distributed_naive_reduce(char* data_ptr, c10::ScalarType scalar_type, size_t chunk_size, size_t chunk_el) {
|
||||
void distributed_naive_reduce(
|
||||
struct shm_context* ctx, char* data_ptr, c10::ScalarType scalar_type, size_t chunk_size, size_t chunk_el) {
|
||||
const int state_group = STATE_GROUP_DISTRIBUTED_ALLREDUCE;
|
||||
static int current_buffer = 0;
|
||||
static int state_idx = 0;
|
||||
|
||||
int& current_buffer = ctx->distributed_current_buffer;
|
||||
int& state_idx = ctx->distributed_state_idx;
|
||||
// init states to case 0 to get rid of "maybe-uninitialized" warning.
|
||||
enum coll_state copy_current = coll_allreduce_naive__copy_in_done;
|
||||
enum coll_state reduce_current = coll_allreduce_naive__reduce_done;
|
||||
@@ -322,65 +407,75 @@ void distributed_naive_reduce(char* data_ptr, c10::ScalarType scalar_type, size_
|
||||
state_idx = (state_idx + 1) % 2;
|
||||
|
||||
int data_size = chunk_size / chunk_el;
|
||||
parallel_memcpy(distributed_buffer[current_buffer][world_rank], data_ptr, chunk_size);
|
||||
parallel_memcpy(ctx->distributed_buffer[current_buffer][ctx->group_rank], data_ptr, chunk_size);
|
||||
std::atomic_thread_fence(std::memory_order_release);
|
||||
workspace[world_rank]->states[state_group] = copy_current;
|
||||
ctx->workspace[ctx->group_rank]->states[state_group] = copy_current;
|
||||
|
||||
for (int i = 0; i < world_size; i++) {
|
||||
for (int i = 0; i < ctx->group_size; i++) {
|
||||
// wait until all the other ranks copy the buffer
|
||||
if (i != world_rank) wait_buffer_state_until_2(i, copy_current, reduce_current, state_group);
|
||||
if (i != ctx->group_rank) {
|
||||
wait_buffer_state_until_2(ctx, i, copy_current, reduce_current, state_group);
|
||||
}
|
||||
}
|
||||
|
||||
// reduce scatter
|
||||
reduce_all_buffers(
|
||||
slice_el_start(chunk_el, world_rank),
|
||||
slice_size(chunk_el, world_rank),
|
||||
slice_el_start(chunk_el, ctx->group_rank, ctx->group_size),
|
||||
slice_size(chunk_el, ctx->group_rank, ctx->group_size),
|
||||
scalar_type,
|
||||
world_rank,
|
||||
distributed_buffer[current_buffer][world_rank],
|
||||
distributed_buffer[current_buffer]);
|
||||
std::atomic_thread_fence(std::memory_order_release);
|
||||
workspace[world_rank]->states[state_group] = reduce_current;
|
||||
ctx->distributed_buffer[current_buffer][ctx->group_rank],
|
||||
ctx->distributed_buffer[current_buffer],
|
||||
ctx->group_size);
|
||||
|
||||
for (int i = 0; i < world_size; i++) {
|
||||
std::atomic_thread_fence(std::memory_order_release);
|
||||
ctx->workspace[ctx->group_rank]->states[state_group] = reduce_current;
|
||||
|
||||
for (int i = 0; i < ctx->group_size; i++) {
|
||||
// wait until all the other ranks reduce the buffer
|
||||
if (i != world_rank) wait_buffer_state_until_2(i, reduce_current, copy_next, state_group);
|
||||
if (i != ctx->group_rank) {
|
||||
wait_buffer_state_until_2(ctx, i, reduce_current, copy_next, state_group);
|
||||
}
|
||||
}
|
||||
|
||||
for (int i = 0; i < world_size; i++) {
|
||||
int rank = (i + world_rank) % world_size;
|
||||
for (int i = 0; i < ctx->group_size; i++) {
|
||||
int rank = (i + ctx->group_rank) % ctx->group_size;
|
||||
parallel_memcpy(
|
||||
slice_data(data_ptr, chunk_el, data_size, rank),
|
||||
slice_data(distributed_buffer[current_buffer][rank], chunk_el, chunk_size / chunk_el, rank),
|
||||
slice_size(chunk_el, rank) * data_size);
|
||||
slice_data(data_ptr, chunk_el, data_size, rank, ctx->group_size),
|
||||
slice_data(
|
||||
ctx->distributed_buffer[current_buffer][rank], chunk_el, chunk_size / chunk_el, rank, ctx->group_size),
|
||||
slice_size(chunk_el, rank, ctx->group_size) * data_size);
|
||||
}
|
||||
|
||||
current_buffer = 1 - current_buffer;
|
||||
}
|
||||
|
||||
void all_reduce_outer_loop(torch::Tensor& data, size_t numel, int data_size) {
|
||||
void all_reduce_outer_loop(torch::Tensor& data, size_t numel, int data_size, int64_t handle) {
|
||||
auto* ctx = get_shm_context(handle);
|
||||
for (int offset = 0; offset < data_size; offset += MAX_BUF_SIZE) {
|
||||
auto data_ptr = ((char*)(data.data_ptr()) + offset);
|
||||
size_t chunk_size = data_size - offset > MAX_BUF_SIZE ? MAX_BUF_SIZE : data_size - offset;
|
||||
size_t chunk_el = chunk_size / (data_size / numel);
|
||||
if (chunk_size < NAIVE_ALLREDUCE_THRESHOLD) {
|
||||
symmetric_naive_all_reduce(data_ptr, data.scalar_type(), chunk_size, chunk_el);
|
||||
symmetric_naive_all_reduce(ctx, data_ptr, data.scalar_type(), chunk_size, chunk_el);
|
||||
} else {
|
||||
distributed_naive_reduce(data_ptr, data.scalar_type(), chunk_size, chunk_el);
|
||||
distributed_naive_reduce(ctx, data_ptr, data.scalar_type(), chunk_size, chunk_el);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <int STATE_GROUP>
|
||||
void naive_all_gather(char* result_ptr, char* data_ptr, size_t res_stride, size_t chunk_size, size_t chunk_el) {
|
||||
static int current_buffer = 0;
|
||||
static int state_idx = 0;
|
||||
void naive_all_gather(
|
||||
struct shm_context* ctx, char* result_ptr, char* data_ptr, size_t res_stride, size_t chunk_size, size_t chunk_el) {
|
||||
int& current_buffer =
|
||||
STATE_GROUP == STATE_GROUP_ALL_GATHER ? ctx->allgather_current_buffer : ctx->allgather_into_tensor_current_buffer;
|
||||
int& state_idx =
|
||||
STATE_GROUP == STATE_GROUP_ALL_GATHER ? ctx->allgather_state_idx : ctx->allgather_into_tensor_state_idx;
|
||||
|
||||
char*** buffer = nullptr;
|
||||
if constexpr (STATE_GROUP == STATE_GROUP_ALL_GATHER) {
|
||||
buffer = allgather_buffer;
|
||||
buffer = ctx->allgather_buffer;
|
||||
} else if constexpr (STATE_GROUP == STATE_GROUP_ALL_GATHER_INTO_TENSOR) {
|
||||
buffer = allgather_into_tensor_buffer;
|
||||
buffer = ctx->allgather_into_tensor_buffer;
|
||||
} else {
|
||||
static_assert(
|
||||
STATE_GROUP == STATE_GROUP_ALL_GATHER || STATE_GROUP == STATE_GROUP_ALL_GATHER_INTO_TENSOR,
|
||||
@@ -409,22 +504,26 @@ void naive_all_gather(char* result_ptr, char* data_ptr, size_t res_stride, size_
|
||||
}
|
||||
state_idx = (state_idx + 1) % 3;
|
||||
|
||||
parallel_memcpy(buffer[current_buffer][world_rank], data_ptr, chunk_size);
|
||||
parallel_memcpy(buffer[current_buffer][ctx->group_rank], data_ptr, chunk_size);
|
||||
std::atomic_thread_fence(std::memory_order_release);
|
||||
workspace[world_rank]->states[STATE_GROUP] = copy_current;
|
||||
ctx->workspace[ctx->group_rank]->states[STATE_GROUP] = copy_current;
|
||||
|
||||
for (int i = 0; i < world_size; i++) {
|
||||
for (int i = 0; i < ctx->group_size; i++) {
|
||||
// wait until all the other ranks copy the buffer
|
||||
if (i != world_rank) wait_buffer_state_until_2(i, copy_current, copy_next, STATE_GROUP);
|
||||
if (i != ctx->group_rank) {
|
||||
wait_buffer_state_until_2(ctx, i, copy_current, copy_next, STATE_GROUP);
|
||||
}
|
||||
}
|
||||
for (int i = 0; i < world_size; i++) {
|
||||
for (int i = 0; i < ctx->group_size; i++) {
|
||||
parallel_memcpy(result_ptr + i * res_stride, buffer[current_buffer][i], chunk_size);
|
||||
}
|
||||
current_buffer = 1 - current_buffer;
|
||||
}
|
||||
|
||||
template <int STATE_GROUP>
|
||||
torch::Tensor& all_gather(torch::Tensor& result, torch::Tensor& data, int dim, size_t numel, int data_size) {
|
||||
torch::Tensor&
|
||||
all_gather(torch::Tensor& result, torch::Tensor& data, int dim, size_t numel, int data_size, int64_t handle) {
|
||||
auto* ctx = get_shm_context(handle);
|
||||
size_t dim_el = data.stride(dim) * data.size(dim);
|
||||
int dtype_size = data_size / numel;
|
||||
size_t dim_size = dim_el * dtype_size;
|
||||
@@ -436,7 +535,8 @@ torch::Tensor& all_gather(torch::Tensor& result, torch::Tensor& data, int dim, s
|
||||
size_t chunk_size = dim_size - offset > MAX_BUF_SIZE ? MAX_BUF_SIZE : dim_size - offset;
|
||||
size_t chunk_el = chunk_size / dtype_size;
|
||||
naive_all_gather<STATE_GROUP>(
|
||||
result_ptr + i * dim_size * world_size + offset,
|
||||
ctx,
|
||||
result_ptr + i * dim_size * ctx->group_size + offset,
|
||||
data_ptr + i * dim_size + offset,
|
||||
dim_size,
|
||||
chunk_size,
|
||||
@@ -445,12 +545,11 @@ torch::Tensor& all_gather(torch::Tensor& result, torch::Tensor& data, int dim, s
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
template torch::Tensor& all_gather<STATE_GROUP_ALL_GATHER>(torch::Tensor&, torch::Tensor&, int, size_t, int);
|
||||
template torch::Tensor& all_gather<STATE_GROUP_ALL_GATHER>(torch::Tensor&, torch::Tensor&, int, size_t, int, int64_t);
|
||||
template torch::Tensor&
|
||||
all_gather<STATE_GROUP_ALL_GATHER_INTO_TENSOR>(torch::Tensor&, torch::Tensor&, int, size_t, int);
|
||||
|
||||
all_gather<STATE_GROUP_ALL_GATHER_INTO_TENSOR>(torch::Tensor&, torch::Tensor&, int, size_t, int, int64_t);
|
||||
void naive_reduce_scatter(
|
||||
struct shm_context* ctx,
|
||||
char* output_ptr,
|
||||
char* data_ptr,
|
||||
c10::ScalarType scalar_type,
|
||||
@@ -458,8 +557,8 @@ void naive_reduce_scatter(
|
||||
size_t chunk_el,
|
||||
int element_size) {
|
||||
const int state_group = STATE_GROUP_REDUCE_SCATTER;
|
||||
static int current_buffer = 0;
|
||||
static int state_idx = 0;
|
||||
int& current_buffer = ctx->reduce_scatter_current_buffer;
|
||||
int& state_idx = ctx->reduce_scatter_state_idx;
|
||||
|
||||
enum coll_state copy_current = coll_reduce_scatter_naive__copy_in_done;
|
||||
enum coll_state copy_next = coll_alt1_reduce_scatter_naive__copy_in_done;
|
||||
@@ -483,40 +582,97 @@ void naive_reduce_scatter(
|
||||
state_idx = (state_idx + 1) % 3;
|
||||
|
||||
// Step 1: copy local data to shared buffer
|
||||
parallel_memcpy(reduce_scatter_buffer[current_buffer][world_rank], data_ptr, chunk_size);
|
||||
parallel_memcpy(ctx->reduce_scatter_buffer[current_buffer][ctx->group_rank], data_ptr, chunk_size);
|
||||
std::atomic_thread_fence(std::memory_order_release);
|
||||
workspace[world_rank]->states[state_group] = copy_current;
|
||||
ctx->workspace[ctx->group_rank]->states[state_group] = copy_current;
|
||||
|
||||
// Step 2: wait for all ranks to copy in
|
||||
for (int i = 0; i < world_size; i++) {
|
||||
if (i != world_rank) wait_buffer_state_until_2(i, copy_current, copy_next, state_group);
|
||||
for (int i = 0; i < ctx->group_size; i++) {
|
||||
if (i != ctx->group_rank) {
|
||||
wait_buffer_state_until_2(ctx, i, copy_current, copy_next, state_group);
|
||||
}
|
||||
}
|
||||
|
||||
// // Step 3: do local reduce on this rank’s slice only
|
||||
int start_el = slice_el_start(chunk_el, world_rank);
|
||||
// each rank reduce its slice of buffer independently so therre is no need for
|
||||
// Step 3: do local reduce on this rank's slice only
|
||||
int start_el = slice_el_start(chunk_el, ctx->group_rank, ctx->group_size);
|
||||
|
||||
// each rank reduce its slice of buffer independently so there is no need for
|
||||
// synchronization afterward
|
||||
reduce_all_buffers(
|
||||
start_el,
|
||||
slice_size(chunk_el, world_rank),
|
||||
slice_size(chunk_el, ctx->group_rank, ctx->group_size),
|
||||
scalar_type,
|
||||
world_rank,
|
||||
output_ptr -
|
||||
start_el * element_size, // in reduce_all_buffers, the output_ptr is the buffer for all ranks, but here
|
||||
// output_ptr is already the local buffer for one rank. Adjust it here.
|
||||
reduce_scatter_buffer[current_buffer]);
|
||||
output_ptr - start_el * element_size,
|
||||
ctx->reduce_scatter_buffer[current_buffer],
|
||||
ctx->group_size);
|
||||
|
||||
// done
|
||||
current_buffer = 1 - current_buffer;
|
||||
}
|
||||
|
||||
void reduce_scatter_outer_loop(torch::Tensor& output, torch::Tensor& data, size_t numel, int data_size) {
|
||||
void reduce_scatter_outer_loop(
|
||||
torch::Tensor& output, torch::Tensor& data, size_t numel, int data_size, int64_t handle) {
|
||||
auto* ctx = get_shm_context(handle);
|
||||
for (int offset = 0; offset < data_size; offset += MAX_BUF_SIZE) {
|
||||
auto data_ptr = ((char*)(data.data_ptr()) + offset);
|
||||
auto output_ptr = ((char*)(output.data_ptr()) + offset);
|
||||
size_t chunk_size = std::min((size_t)MAX_BUF_SIZE, (size_t)(data_size - offset));
|
||||
size_t chunk_el = chunk_size / (data_size / numel);
|
||||
|
||||
naive_reduce_scatter(output_ptr, data_ptr, data.scalar_type(), chunk_size, chunk_el, data.element_size());
|
||||
naive_reduce_scatter(ctx, output_ptr, data_ptr, data.scalar_type(), chunk_size, chunk_el, data.element_size());
|
||||
}
|
||||
}
|
||||
|
||||
void all_to_all(char* output_ptr, char* input_ptr, size_t data_size, int64_t handle) {
|
||||
auto* ctx = get_shm_context(handle);
|
||||
|
||||
TORCH_CHECK(data_size <= MAX_BUF_SIZE, "SHM all-to-all input size exceeds maximum buffer size");
|
||||
TORCH_CHECK(data_size % ctx->group_size == 0, "SHM all-to-all input size must be divisible by group size");
|
||||
|
||||
int& current_buffer = ctx->alltoall_current_buffer;
|
||||
int& state_idx = ctx->alltoall_state_idx;
|
||||
|
||||
enum coll_state copy_current = coll_allgather_naive__copy_in_done;
|
||||
enum coll_state copy_next = coll_alt1_allgather_naive__copy_in_done;
|
||||
|
||||
switch (state_idx) {
|
||||
case 0:
|
||||
copy_current = coll_allgather_naive__copy_in_done;
|
||||
copy_next = coll_alt1_allgather_naive__copy_in_done;
|
||||
break;
|
||||
case 1:
|
||||
copy_current = coll_alt1_allgather_naive__copy_in_done;
|
||||
copy_next = coll_alt2_allgather_naive__copy_in_done;
|
||||
break;
|
||||
case 2:
|
||||
copy_current = coll_alt2_allgather_naive__copy_in_done;
|
||||
copy_next = coll_allgather_naive__copy_in_done;
|
||||
break;
|
||||
default:
|
||||
assert(!"Should not get here.");
|
||||
}
|
||||
|
||||
state_idx = (state_idx + 1) % 3;
|
||||
// copy local input to shared buffer
|
||||
parallel_memcpy(ctx->alltoall_buffer[current_buffer][ctx->group_rank], input_ptr, data_size);
|
||||
std::atomic_thread_fence(std::memory_order_release);
|
||||
ctx->workspace[ctx->group_rank]->states[STATE_GROUP_ALL_TO_ALL] = copy_current;
|
||||
|
||||
// wait until all ranks copy the buffer
|
||||
for (int i = 0; i < ctx->group_size; i++) {
|
||||
if (i != ctx->group_rank) {
|
||||
wait_buffer_state_until_2(ctx, i, copy_current, copy_next, STATE_GROUP_ALL_TO_ALL);
|
||||
}
|
||||
}
|
||||
|
||||
size_t chunk_size = data_size / ctx->group_size;
|
||||
|
||||
// read the chunk addressed to this rank from every source rank
|
||||
for (int i = 0; i < ctx->group_size; i++) {
|
||||
parallel_memcpy(
|
||||
output_ptr + i * chunk_size,
|
||||
ctx->alltoall_buffer[current_buffer][i] + ctx->group_rank * chunk_size,
|
||||
chunk_size);
|
||||
}
|
||||
current_buffer = 1 - current_buffer;
|
||||
}
|
||||
|
||||
@@ -10,10 +10,24 @@ constexpr int STATE_GROUP_DISTRIBUTED_ALLREDUCE = 1;
|
||||
constexpr int STATE_GROUP_ALL_GATHER = 2;
|
||||
constexpr int STATE_GROUP_ALL_GATHER_INTO_TENSOR = 3;
|
||||
constexpr int STATE_GROUP_REDUCE_SCATTER = 4;
|
||||
constexpr int STATE_GROUP_ALL_TO_ALL = 5;
|
||||
|
||||
int64_t shm_initialize(int size, int rank, const char* addr_string, const char* port_string, const char* group_name);
|
||||
|
||||
void shm_default_initialize(int size, int rank, const char* addr_string, const char* port_string);
|
||||
|
||||
int64_t shm_group_initialize(const std::string& group_name, int64_t group_size, int64_t group_rank);
|
||||
|
||||
int64_t shm_get_group_size(int64_t handle);
|
||||
|
||||
void all_reduce_outer_loop(torch::Tensor& data, size_t numel, int data_size, int64_t handle);
|
||||
|
||||
void shm_initialize(int size, int rank, const char* addr_string, const char* port_string);
|
||||
void all_reduce_outer_loop(torch::Tensor& data, size_t numel, int data_size);
|
||||
template <int STATE_GROUP>
|
||||
torch::Tensor& all_gather(torch::Tensor& result, torch::Tensor& data, int dim, size_t numel, int data_size);
|
||||
void reduce_scatter_outer_loop(torch::Tensor& output, torch::Tensor& data, size_t numel, int data_size);
|
||||
#endif
|
||||
torch::Tensor&
|
||||
all_gather(torch::Tensor& result, torch::Tensor& data, int dim, size_t numel, int data_size, int64_t handle);
|
||||
|
||||
void reduce_scatter_outer_loop(torch::Tensor& output, torch::Tensor& data, size_t numel, int data_size, int64_t handle);
|
||||
|
||||
void all_to_all(char* output_ptr, char* input_ptr, size_t data_size, int64_t handle);
|
||||
|
||||
#endif // __SHM_COLLECTIVES__
|
||||
|
||||
@@ -127,7 +127,6 @@ std::tuple<at::Tensor, at::Tensor, at::Tensor> fused_qk_gemma_rmsnorm_with_gate_
|
||||
double eps,
|
||||
int64_t head_dim,
|
||||
int64_t num_head);
|
||||
|
||||
// speculative decoding
|
||||
void verify_tree_greedy_cpu(
|
||||
at::Tensor predicts,
|
||||
@@ -509,17 +508,23 @@ at::Tensor conv3d_embed_cpu(const at::Tensor& input, const at::Tensor& weight, c
|
||||
// shared memory init
|
||||
void initialize(int64_t size, int64_t rank);
|
||||
|
||||
// shared mmeory all_reduce
|
||||
void shm_allreduce(at::Tensor& data, int64_t op);
|
||||
// group shared memory init
|
||||
int64_t shm_group_initialize(const std::string& group_name, int64_t group_size, int64_t group_rank);
|
||||
|
||||
// shared memory all_reduce
|
||||
void shm_allreduce(at::Tensor& data, int64_t op, int64_t handle = -1);
|
||||
|
||||
// shared memory all_gather
|
||||
at::Tensor shm_allgather(at::Tensor& data, int64_t dim);
|
||||
at::Tensor shm_allgather(at::Tensor& data, int64_t dim, int64_t handle = -1);
|
||||
|
||||
// group shared memory all_to_all
|
||||
void shm_alltoall(at::Tensor& output_tensor, at::Tensor& data, int64_t handle);
|
||||
|
||||
// shared memory all_gather_into_tensor
|
||||
void shm_allgather_into_tensor(at::Tensor& output_tensor, at::Tensor& data);
|
||||
void shm_allgather_into_tensor(at::Tensor& output_tensor, at::Tensor& data, int64_t handle = -1);
|
||||
|
||||
// shared memory reduce_scatter_tensor
|
||||
void shm_reduce_scatter_tensor(at::Tensor& output_tensor, at::Tensor& data, int64_t op);
|
||||
void shm_reduce_scatter_tensor(at::Tensor& output_tensor, at::Tensor& data, int64_t op, int64_t handle = -1);
|
||||
|
||||
// rope
|
||||
std::tuple<at::Tensor, at::Tensor> rotary_embedding_cpu(
|
||||
@@ -943,17 +948,32 @@ TORCH_LIBRARY_FRAGMENT(sgl_kernel, m) {
|
||||
m.def("conv3d_embed_cpu(Tensor input, Tensor weight, Tensor bias, bool is_vnni) -> Tensor");
|
||||
m.impl("conv3d_embed_cpu", torch::kCPU, &conv3d_embed_cpu);
|
||||
|
||||
// all reduce
|
||||
// shared memory
|
||||
m.def("initialize(int size, int rank) -> ()");
|
||||
m.def("shm_allreduce(Tensor(a!) data, int reduce_op) -> ()");
|
||||
|
||||
// all_reduce
|
||||
m.def("shm_allreduce(Tensor(a!) data, int reduce_op, int handle=-1) -> ()");
|
||||
m.impl("shm_allreduce", torch::kCPU, &shm_allreduce);
|
||||
m.def("shm_allgather(Tensor data, int dim) -> Tensor");
|
||||
|
||||
// all_gather
|
||||
m.def("shm_allgather(Tensor data, int dim, int handle=-1) -> Tensor");
|
||||
m.impl("shm_allgather", torch::kCPU, &shm_allgather);
|
||||
m.def("shm_allgather_into_tensor(Tensor(a!) output_tensor, Tensor data) -> ()");
|
||||
|
||||
// all_gather_into_tensor
|
||||
m.def("shm_allgather_into_tensor(Tensor(a!) output_tensor, Tensor data, int handle=-1) -> ()");
|
||||
m.impl("shm_allgather_into_tensor", torch::kCPU, &shm_allgather_into_tensor);
|
||||
m.def("shm_reduce_scatter_tensor(Tensor(a!) output_tensor, Tensor data, int reduce_op) -> ()");
|
||||
|
||||
// reduce_scatter_tensor
|
||||
m.def("shm_reduce_scatter_tensor(Tensor(a!) output_tensor, Tensor data, int reduce_op, int handle=-1) -> ()");
|
||||
m.impl("shm_reduce_scatter_tensor", torch::kCPU, &shm_reduce_scatter_tensor);
|
||||
|
||||
// explicit group context initialization
|
||||
m.def("shm_group_initialize(str group_name, int group_size, int group_rank) -> int");
|
||||
|
||||
// all_to_all
|
||||
m.def("shm_alltoall(Tensor(a!) output_tensor, Tensor data, int handle) -> ()");
|
||||
m.impl("shm_alltoall", torch::kCPU, &shm_alltoall);
|
||||
|
||||
// rope
|
||||
m.def(
|
||||
"rotary_embedding_cpu(Tensor positions, Tensor query, Tensor key, int head_size, Tensor cos_sin_cache, "
|
||||
@@ -1028,6 +1048,7 @@ TORCH_LIBRARY_FRAGMENT(sgl_kernel, m) {
|
||||
TORCH_LIBRARY_IMPL(sgl_kernel, CatchAll, m) {
|
||||
m.impl("init_cpu_threads_env", init_cpu_threads_env);
|
||||
m.impl("initialize", &initialize);
|
||||
m.impl("shm_group_initialize", &shm_group_initialize);
|
||||
}
|
||||
|
||||
REGISTER_EXTENSION(common_ops)
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo
|
||||
|
||||
import os
|
||||
|
||||
# Copyright 2024 xDiT team.
|
||||
# Adapted from
|
||||
# https://github.com/vllm-project/vllm/blob/main/vllm/distributed/parallel_state.py
|
||||
@@ -219,6 +221,28 @@ class GroupCoordinator:
|
||||
|
||||
# TODO: fix it for other platforms
|
||||
self.device = get_local_torch_device()
|
||||
self.shm_handle = None
|
||||
global_world_size = torch.distributed.get_world_size()
|
||||
local_size = int(os.environ.get("LOCAL_SIZE", "0"))
|
||||
self.is_single_node = local_size > 0 and global_world_size == local_size
|
||||
if (
|
||||
current_platform.is_cpu()
|
||||
and self.is_single_node
|
||||
and self.world_size > 1
|
||||
and hasattr(torch.ops.sgl_kernel, "shm_group_initialize")
|
||||
and self.unique_name.startswith(("tp", "sp"))
|
||||
):
|
||||
group_ranks = "-".join(str(rank) for rank in self.ranks)
|
||||
group_name = f"{self.unique_name}-{group_ranks}"
|
||||
|
||||
self.shm_handle = int(
|
||||
torch.ops.sgl_kernel.shm_group_initialize(
|
||||
group_name,
|
||||
self.world_size,
|
||||
self.rank_in_group,
|
||||
)
|
||||
)
|
||||
torch.distributed.barrier(group=self.cpu_group)
|
||||
|
||||
self.device_communicator: DeviceCommunicatorBase = None # type: ignore
|
||||
if use_device_communicator and self.world_size > 1:
|
||||
@@ -396,17 +420,20 @@ class GroupCoordinator:
|
||||
return output
|
||||
if (
|
||||
current_platform.is_cpu()
|
||||
and not async_op
|
||||
and self.shm_handle is not None
|
||||
and is_shm_available(input_.dtype, self.world_size, len(self.ranks))
|
||||
and op is torch.distributed.ReduceOp.SUM
|
||||
):
|
||||
# for CPU platform, intra-node case we could speedup with shared memory based comm ops
|
||||
torch.ops.sgl_kernel.shm_allreduce(
|
||||
input_, int(torch.distributed.ReduceOp.SUM)
|
||||
input_, int(torch.distributed.ReduceOp.SUM), self.shm_handle
|
||||
)
|
||||
else:
|
||||
torch.distributed.all_reduce(
|
||||
input_, op=op, group=self.device_group, async_op=async_op
|
||||
)
|
||||
|
||||
return input_
|
||||
|
||||
def _all_reduce_out_of_place(self, input_: torch.Tensor) -> torch.Tensor:
|
||||
@@ -440,18 +467,24 @@ class GroupCoordinator:
|
||||
input_size, dtype=input_.dtype, device=input_.device
|
||||
)
|
||||
|
||||
# All-gather.
|
||||
if current_platform.is_cpu() and is_shm_available(
|
||||
input_.dtype, self.world_size, len(self.ranks)
|
||||
# Group-aware CPU SHM all-gather.
|
||||
if (
|
||||
current_platform.is_cpu()
|
||||
and self.shm_handle is not None
|
||||
and is_shm_available(input_.dtype, self.world_size, len(self.ranks))
|
||||
):
|
||||
output_tensor = torch.ops.sgl_kernel.shm_allgather(input_, dim)
|
||||
output_tensor = torch.ops.sgl_kernel.shm_allgather(
|
||||
input_, dim, self.shm_handle
|
||||
)
|
||||
|
||||
if separate_tensors:
|
||||
return list(output_tensor.chunk(world_size, dim=dim))
|
||||
return output_tensor
|
||||
else:
|
||||
if not input_.is_contiguous():
|
||||
input_ = input_.contiguous()
|
||||
all_gather_single(output_tensor, input_, group=self.device_group)
|
||||
|
||||
# Fallback.
|
||||
if not input_.is_contiguous():
|
||||
input_ = input_.contiguous()
|
||||
all_gather_single(output_tensor, input_, group=self.device_group)
|
||||
|
||||
if dim != 0:
|
||||
input_size[0] //= world_size
|
||||
@@ -889,3 +922,13 @@ class SequenceParallelGroupCoordinator(GroupCoordinator):
|
||||
self.ulysses_rank = torch.distributed.get_rank(self.ulysses_group)
|
||||
self.ring_world_size = torch.distributed.get_world_size(self.ring_group)
|
||||
self.ring_rank = torch.distributed.get_rank(self.ring_group)
|
||||
self.ulysses_shm_handle = None
|
||||
if (
|
||||
current_platform.is_cpu()
|
||||
and self.ulysses_world_size > 1
|
||||
and hasattr(torch.ops.sgl_kernel, "shm_group_initialize")
|
||||
):
|
||||
# TODO: Initialize a dedicated SHM handle for the Ulysses subgroup
|
||||
# when Ring Attention is supported on CPU. The current SP handle
|
||||
# is only valid when ring_degree == 1.
|
||||
self.ulysses_shm_handle = self.shm_handle
|
||||
|
||||
@@ -16,9 +16,11 @@ from sglang.multimodal_gen.runtime.distributed.parallel_state import (
|
||||
get_ulysses_parallel_rank,
|
||||
get_ulysses_parallel_world_size,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.platforms import current_platform
|
||||
from sglang.srt.utils.common import torch_release
|
||||
|
||||
_cp_options.enable_load_balance = False
|
||||
SHM_ALLTOALL_MAX_BYTES = 32 * 1024 * 1024
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.multimodal_gen.runtime.layers.attention.backends.attention_backend import (
|
||||
@@ -105,7 +107,8 @@ def _a2a_staging_buffer(
|
||||
|
||||
|
||||
def _usp_all_to_all_single(x: torch.Tensor, role: str | None = None) -> torch.Tensor:
|
||||
ulysses_pg = get_sp_group().ulysses_group
|
||||
sp_group = get_sp_group()
|
||||
ulysses_pg = sp_group.ulysses_group
|
||||
assert ulysses_pg is not None, "Ulysses process group is not initialized."
|
||||
x_shape = x.shape
|
||||
x = x.flatten().contiguous()
|
||||
@@ -124,9 +127,21 @@ def _usp_all_to_all_single(x: torch.Tensor, role: str | None = None) -> torch.Te
|
||||
output = torch.empty_like(x)
|
||||
else:
|
||||
output = _a2a_staging_buffer(role, x.shape, x.dtype, x.device)
|
||||
|
||||
# USP calls this collective many times per denoising step and waits
|
||||
# immediately, so avoid the extra wrapper overhead of functional collectives.
|
||||
torch.distributed.all_to_all_single(output, x, group=ulysses_pg)
|
||||
handle = sp_group.ulysses_shm_handle
|
||||
input_bytes = x.numel() * x.element_size()
|
||||
if (
|
||||
current_platform.is_cpu()
|
||||
and handle is not None
|
||||
and handle >= 0
|
||||
and input_bytes <= SHM_ALLTOALL_MAX_BYTES
|
||||
):
|
||||
torch.ops.sgl_kernel.shm_alltoall(output, x, sp_group.ulysses_shm_handle)
|
||||
else:
|
||||
torch.distributed.all_to_all_single(output, x, group=ulysses_pg)
|
||||
|
||||
return output.reshape(x_shape)
|
||||
|
||||
|
||||
|
||||
@@ -30,6 +30,7 @@ class CPUWorker(GPUWorker):
|
||||
master_port: int,
|
||||
server_args: ServerArgs,
|
||||
):
|
||||
os.environ["LOCAL_SIZE"] = str(server_args.num_gpus // server_args.nnodes)
|
||||
super().__init__(local_rank, rank, master_port, server_args)
|
||||
if _is_cpu_amx_available:
|
||||
self.init_cpu_threads_binding()
|
||||
@@ -70,7 +71,3 @@ class CPUWorker(GPUWorker):
|
||||
|
||||
# Bind OpenMP threads to CPU cores
|
||||
torch.ops.sgl_kernel.init_cpu_threads_env(self.local_omp_cpuid)
|
||||
|
||||
# Set local size to hint SGLang to use shared memory based AllReduce
|
||||
os.environ["LOCAL_SIZE"] = str(self.server_args.tp_size)
|
||||
torch.ops.sgl_kernel.initialize(self.server_args.tp_size, self.rank)
|
||||
|
||||
Reference in New Issue
Block a user