From 93842f52fee72a47bd430e8f6480bb274a101ddd Mon Sep 17 00:00:00 2001 From: Yuxingwang-intel Date: Thu, 15 Oct 2026 13:22:20 +0800 Subject: [PATCH] [CPU]: Add group-aware CPU SHM collective kernels (#35435) --- .../sglang/kernels/aot/csrc/cpu/interface.cpp | 33 +- python/sglang/kernels/aot/csrc/cpu/shm.cpp | 434 ++++++++++++------ python/sglang/kernels/aot/csrc/cpu/shm.h | 24 +- .../aot/csrc/cpu/torch_extension_cpu.cpp | 43 +- .../runtime/distributed/group_coordinator.py | 61 ++- .../multimodal_gen/runtime/layers/usp.py | 19 +- .../runtime/managers/cpu_worker.py | 5 +- test/registered/cpu/test_comm.py | 114 ++++- 8 files changed, 538 insertions(+), 195 deletions(-) diff --git a/python/sglang/kernels/aot/csrc/cpu/interface.cpp b/python/sglang/kernels/aot/csrc/cpu/interface.cpp index 0c4e8cd5b9..2fba457ddd 100644 --- a/python/sglang/kernels/aot/csrc/cpu/interface.cpp +++ b/python/sglang/kernels/aot/csrc/cpu/interface.cpp @@ -1,4 +1,3 @@ -#include #include #include "shm.h" @@ -43,50 +42,50 @@ void initialize(int64_t size, int64_t rank) { } if (all_ranks_local_p) { - shm_initialize(size, rank, addr_string, port_string); + shm_default_initialize(size, rank, addr_string, port_string); } } -void shm_allreduce(torch::Tensor& data, int64_t op) { +void shm_allreduce(torch::Tensor& data, int64_t op, int64_t handle) { TORCH_CHECK(op == c10d::ReduceOp::SUM, "Only torch.distributed.ReduceOp.SUM is supported"); auto numel = data.numel(); int data_size = numel * data.element_size(); - all_reduce_outer_loop(data, numel, data_size); - return; + all_reduce_outer_loop(data, numel, data_size, handle); } -torch::Tensor shm_allgather(torch::Tensor& data, int64_t dim) { +torch::Tensor shm_allgather(torch::Tensor& data, int64_t dim, int64_t handle) { auto numel = data.numel(); int data_size = numel * data.element_size(); if (dim < 0) { dim += data.dim(); } + int64_t group_size = handle == -1 ? world_size : shm_get_group_size(handle); std::vector result_shape = data.sizes().vec(); - result_shape[dim] *= world_size; + result_shape[dim] *= group_size; torch::Tensor result_tensor = torch::empty(result_shape, data.options()); - return all_gather(result_tensor, data, dim, numel, data_size); + return all_gather(result_tensor, data, dim, numel, data_size, handle); } -void shm_allgather_into_tensor(torch::Tensor& output_tensor, torch::Tensor& data) { - RECORD_FUNCTION("sgl-kernel::shm_allgather_into_tensor", std::vector({data})); - +void shm_allgather_into_tensor(torch::Tensor& output_tensor, torch::Tensor& data, int64_t handle) { auto numel = data.numel(); int data_size = numel * data.element_size(); int64_t dim = 0; - all_gather(output_tensor, data, dim, numel, data_size); + all_gather(output_tensor, data, dim, numel, data_size, handle); } -void shm_reduce_scatter_tensor(at::Tensor& output_tensor, at::Tensor& data, int64_t op) { - RECORD_FUNCTION("sgl-kernel::shm_reduce_scatter_tensor", std::vector({data})); - +void shm_reduce_scatter_tensor(at::Tensor& output_tensor, at::Tensor& data, int64_t op, int64_t handle) { TORCH_CHECK(op == c10d::ReduceOp::SUM, "Only torch.distributed.ReduceOp.SUM is supported"); auto numel = data.numel(); int data_size = numel * data.element_size(); - reduce_scatter_outer_loop(output_tensor, data, numel, data_size); + reduce_scatter_outer_loop(output_tensor, data, numel, data_size, handle); +} - return; +void shm_alltoall(torch::Tensor& output_tensor, torch::Tensor& input_tensor, int64_t handle) { + size_t data_size = input_tensor.numel() * input_tensor.element_size(); + all_to_all( + static_cast(output_tensor.data_ptr()), static_cast(input_tensor.data_ptr()), data_size, handle); } diff --git a/python/sglang/kernels/aot/csrc/cpu/shm.cpp b/python/sglang/kernels/aot/csrc/cpu/shm.cpp index 83b0b6bea6..6a0fa41cc2 100644 --- a/python/sglang/kernels/aot/csrc/cpu/shm.cpp +++ b/python/sglang/kernels/aot/csrc/cpu/shm.cpp @@ -73,20 +73,21 @@ void shared_create(SharedData* data, const char* name, void* bytes, size_t nbyte } } -static int world_size; - // SHM based allreduce helper functions // buffer that holds shm name #define NAME_BUF_SIZE 1000 #define MAX_BUF_SIZE 1048576 * 32 #define NAIVE_ALLREDUCE_THRESHOLD 1048576 #define SHM_BUFFER_NAME "deepspeed_allreduce_buffer" +constexpr int kNumCollectiveStates = STATE_GROUP_ALL_TO_ALL + 1; +// idx=0 -- state for symmetric_naive_all_reduce +// idx=1 -- state for distributed_naive_all_reduce +// idx=2 -- state for all_gather +// idx=3 -- state for all_gather_into_tensor +// idx=4 -- state for reduce_scatter +// idx=5 -- state for all_to_all struct allreduce_workspace { - enum coll_state states[5]; // idx=0 -- state for symmetric_naive_all_reduce - // idx=1 -- state for distributed_naive_all_reduce - // idx=2 -- state for all_gather - // idx=3 -- state for all_gather_into_tensor - // idx=4 -- state for reduce_scatter + enum coll_state states[kNumCollectiveStates]; // double buffer to avoid syncing between rounds // offset=0 -- 2*NAIVE_ALLREDUCE_THRESHOLD : buffer for // symmetric_naive_all_reduce after that : buffer for @@ -96,7 +97,9 @@ struct allreduce_workspace { 2 * MAX_BUF_SIZE + // distributed naive reduce 2 * MAX_BUF_SIZE + // allgather 2 * MAX_BUF_SIZE + // allgather_into_tensor - 2 * MAX_BUF_SIZE // reduce_scatter + 2 * MAX_BUF_SIZE + // reduce_scatter + 2 * MAX_BUF_SIZE // all_to_all + ]; }; @@ -108,20 +111,63 @@ struct allreduce_workspace { (2 * NAIVE_ALLREDUCE_THRESHOLD + 4 * MAX_BUF_SIZE + current_buffer * MAX_BUF_SIZE) // allgather_into_tensor #define BUFFER4_OFFSET(current_buffer) \ (2 * NAIVE_ALLREDUCE_THRESHOLD + 6 * MAX_BUF_SIZE + current_buffer * MAX_BUF_SIZE) // reduce_scatter +#define BUFFER5_OFFSET(current_buffer) \ + (2 * NAIVE_ALLREDUCE_THRESHOLD + 8 * MAX_BUF_SIZE + current_buffer * MAX_BUF_SIZE) // all_to_all -struct allreduce_workspace** workspace; +struct shm_context { + int group_size; + int group_rank; -// buffer for small messages, double buffer -char** symmetric_buffer[2]; -// buffer for large messages, double buffer -char** distributed_buffer[2]; + struct allreduce_workspace** workspace; -char** allgather_buffer[2]; -char** allgather_into_tensor_buffer[2]; -char** reduce_scatter_buffer[2]; + char** symmetric_buffer[2]; + char** distributed_buffer[2]; + char** allgather_buffer[2]; + char** allgather_into_tensor_buffer[2]; + char** reduce_scatter_buffer[2]; + char** alltoall_buffer[2]; -void wait_buffer_state_until_2(int index, enum coll_state state0, enum coll_state state1, int state_group) { - volatile enum coll_state* state_ptr = &(workspace[index]->states[state_group]); + int symmetric_current_buffer; + int symmetric_state_idx; + + int distributed_current_buffer; + int distributed_state_idx; + + int allgather_current_buffer; + int allgather_state_idx; + + int allgather_into_tensor_current_buffer; + int allgather_into_tensor_state_idx; + + int reduce_scatter_current_buffer; + int reduce_scatter_state_idx; + + int alltoall_current_buffer; + int alltoall_state_idx; +}; + +static struct shm_context* default_shm_context = nullptr; +static std::vector shm_contexts; + +static struct shm_context* get_shm_context(int64_t handle) { + if (handle == -1) { + TORCH_CHECK(default_shm_context != nullptr, "Default SHM context is not initialized"); + return default_shm_context; + } + TORCH_CHECK(handle >= 0 && handle < static_cast(shm_contexts.size()), "Invalid SHM context handle: "); + + auto* ctx = shm_contexts[handle]; + TORCH_CHECK(ctx != nullptr, "SHM context is null"); + return ctx; +} + +int64_t shm_get_group_size(int64_t handle) { + return get_shm_context(handle)->group_size; +} + +void wait_buffer_state_until_2( + struct shm_context* ctx, int index, enum coll_state state0, enum coll_state state1, int state_group) { + volatile enum coll_state* state_ptr = &(ctx->workspace[index]->states[state_group]); while (1) { volatile enum coll_state cur_state = *state_ptr; @@ -133,40 +179,53 @@ void reduce_all_buffers( int start_elements, int num_elements, c10::ScalarType scalar_type, - int to_buffer_idx, char* to_buffer, - char** buffers) { + char** buffers, + int group_size) { switch (scalar_type) { case c10::ScalarType::BFloat16: - reduce_bf16_buffers(start_elements, num_elements, to_buffer, buffers, world_size); + reduce_bf16_buffers(start_elements, num_elements, to_buffer, buffers, group_size); break; case c10::ScalarType::Half: - reduce_fp16_buffers(start_elements, num_elements, to_buffer, buffers, world_size); + reduce_fp16_buffers(start_elements, num_elements, to_buffer, buffers, group_size); break; case c10::ScalarType::Float: - reduce_fp32_buffers(start_elements, num_elements, to_buffer, buffers, world_size); + reduce_fp32_buffers(start_elements, num_elements, to_buffer, buffers, group_size); break; default: assert(!"Should not get here"); } } -static bool is_initialized = false; -static int world_rank; - -void shm_initialize(int size, int rank, const char* addr_string, const char* port_string) { - if (is_initialized) { - return; - } - is_initialized = true; - - world_size = size; - world_rank = rank; +int64_t shm_initialize(int size, int rank, const char* addr_string, const char* port_string, const char* group_name) { + auto* ctx = (struct shm_context*)malloc(sizeof(struct shm_context)); + ctx->group_size = size; + ctx->group_rank = rank; + ctx->symmetric_current_buffer = 0; + ctx->symmetric_state_idx = 0; + ctx->distributed_current_buffer = 0; + ctx->distributed_state_idx = 0; + ctx->allgather_current_buffer = 0; + ctx->allgather_state_idx = 0; + ctx->allgather_into_tensor_current_buffer = 0; + ctx->allgather_into_tensor_state_idx = 0; + ctx->reduce_scatter_current_buffer = 0; + ctx->reduce_scatter_state_idx = 0; + ctx->alltoall_current_buffer = 0; + ctx->alltoall_state_idx = 0; char shm_name_prefix[NAME_BUF_SIZE]; char shm_name[NAME_BUF_SIZE]; - snprintf(shm_name_prefix, NAME_BUF_SIZE, "%s_%d_%s_%s", SHM_BUFFER_NAME, getuid(), addr_string, port_string); - // create shared workspace for SHM based allreduce + snprintf( + shm_name_prefix, + NAME_BUF_SIZE, + "%s_%d_%s_%s_%s", + SHM_BUFFER_NAME, + getuid(), + addr_string, + port_string, + group_name); + // create shared workspace for SHM based collectives SharedData allreduce_buffer; // allocate workspace_buf for current rank struct allreduce_workspace* workspace_buf; @@ -174,30 +233,34 @@ void shm_initialize(int size, int rank, const char* addr_string, const char* por workspace_buf = (struct allreduce_workspace*)malloc(sizeof(struct allreduce_workspace)); snprintf(shm_name, NAME_BUF_SIZE, "%.900s_%d", shm_name_prefix, rank); shared_create(&allreduce_buffer, shm_name, workspace_buf, sizeof(struct allreduce_workspace)); + free(workspace_buf); workspace_buf = (struct allreduce_workspace*)allreduce_buffer.bytes; workspace_buf->states[STATE_GROUP_SYMMETRIC_ALLREDUCE] = coll_alt2_allreduce_naive__copy_in_done; // symmetric_naive_all_reduce workspace_buf->states[STATE_GROUP_DISTRIBUTED_ALLREDUCE] = coll_begin; // distributed_naive_reduce workspace_buf->states[STATE_GROUP_ALL_GATHER] = coll_alt2_allgather_naive__copy_in_done; // all_gather workspace_buf->states[STATE_GROUP_ALL_GATHER_INTO_TENSOR] = - coll_alt2_allgather_naive__copy_in_done; // all_gather_into_tensor - workspace_buf->states[STATE_GROUP_REDUCE_SCATTER] = coll_begin; // reduce_scatter + coll_alt2_allgather_naive__copy_in_done; // all_gather_into_tensor + workspace_buf->states[STATE_GROUP_REDUCE_SCATTER] = coll_begin; // reduce_scatter + workspace_buf->states[STATE_GROUP_ALL_TO_ALL] = coll_alt2_allgather_naive__copy_in_done; // all_to_all // create the workspace pointer list - workspace = (struct allreduce_workspace**)malloc(size * sizeof(struct allreduce_workspace*)); - symmetric_buffer[0] = (char**)malloc(size * sizeof(char**)); - symmetric_buffer[1] = (char**)malloc(size * sizeof(char**)); - distributed_buffer[0] = (char**)malloc(size * sizeof(char**)); - distributed_buffer[1] = (char**)malloc(size * sizeof(char**)); + ctx->workspace = (struct allreduce_workspace**)malloc(size * sizeof(struct allreduce_workspace*)); + ctx->symmetric_buffer[0] = (char**)malloc(size * sizeof(char*)); + ctx->symmetric_buffer[1] = (char**)malloc(size * sizeof(char*)); + ctx->distributed_buffer[0] = (char**)malloc(size * sizeof(char*)); + ctx->distributed_buffer[1] = (char**)malloc(size * sizeof(char*)); + ctx->allgather_buffer[0] = (char**)malloc(size * sizeof(char*)); + ctx->allgather_buffer[1] = (char**)malloc(size * sizeof(char*)); - allgather_buffer[0] = (char**)malloc(size * sizeof(char*)); - allgather_buffer[1] = (char**)malloc(size * sizeof(char*)); + ctx->allgather_into_tensor_buffer[0] = (char**)malloc(size * sizeof(char*)); + ctx->allgather_into_tensor_buffer[1] = (char**)malloc(size * sizeof(char*)); - allgather_into_tensor_buffer[0] = (char**)malloc(size * sizeof(char*)); - allgather_into_tensor_buffer[1] = (char**)malloc(size * sizeof(char*)); + ctx->reduce_scatter_buffer[0] = (char**)malloc(size * sizeof(char*)); + ctx->reduce_scatter_buffer[1] = (char**)malloc(size * sizeof(char*)); - reduce_scatter_buffer[0] = (char**)malloc(size * sizeof(char*)); - reduce_scatter_buffer[1] = (char**)malloc(size * sizeof(char*)); + ctx->alltoall_buffer[0] = (char**)malloc(size * sizeof(char*)); + ctx->alltoall_buffer[1] = (char**)malloc(size * sizeof(char*)); // map shm of all ranks for (int i = 0; i < size; i++) { @@ -208,48 +271,71 @@ void shm_initialize(int size, int rank, const char* addr_string, const char* por shared_open(&allreduce_buffer, shm_name, sizeof(struct allreduce_workspace)); } while (allreduce_buffer.descriptor == -1 && errno == ENOENT); workspace_buf_other = (struct allreduce_workspace*)allreduce_buffer.bytes; - workspace[i] = workspace_buf_other; + ctx->workspace[i] = workspace_buf_other; } else { - workspace[i] = workspace_buf; + ctx->workspace[i] = workspace_buf; } - symmetric_buffer[0][i] = workspace[i]->buffer + BUFFER0_OFFSET(0); - symmetric_buffer[1][i] = workspace[i]->buffer + BUFFER0_OFFSET(1); - distributed_buffer[0][i] = workspace[i]->buffer + BUFFER1_OFFSET(0); - distributed_buffer[1][i] = workspace[i]->buffer + BUFFER1_OFFSET(1); - allgather_buffer[0][i] = workspace[i]->buffer + BUFFER2_OFFSET(0); - allgather_buffer[1][i] = workspace[i]->buffer + BUFFER2_OFFSET(1); + ctx->symmetric_buffer[0][i] = ctx->workspace[i]->buffer + BUFFER0_OFFSET(0); + ctx->symmetric_buffer[1][i] = ctx->workspace[i]->buffer + BUFFER0_OFFSET(1); - allgather_into_tensor_buffer[0][i] = workspace[i]->buffer + BUFFER3_OFFSET(0); - allgather_into_tensor_buffer[1][i] = workspace[i]->buffer + BUFFER3_OFFSET(1); + ctx->distributed_buffer[0][i] = ctx->workspace[i]->buffer + BUFFER1_OFFSET(0); + ctx->distributed_buffer[1][i] = ctx->workspace[i]->buffer + BUFFER1_OFFSET(1); - reduce_scatter_buffer[0][i] = workspace[i]->buffer + BUFFER4_OFFSET(0); - reduce_scatter_buffer[1][i] = workspace[i]->buffer + BUFFER4_OFFSET(1); + ctx->allgather_buffer[0][i] = ctx->workspace[i]->buffer + BUFFER2_OFFSET(0); + ctx->allgather_buffer[1][i] = ctx->workspace[i]->buffer + BUFFER2_OFFSET(1); + + ctx->allgather_into_tensor_buffer[0][i] = ctx->workspace[i]->buffer + BUFFER3_OFFSET(0); + ctx->allgather_into_tensor_buffer[1][i] = ctx->workspace[i]->buffer + BUFFER3_OFFSET(1); + + ctx->reduce_scatter_buffer[0][i] = ctx->workspace[i]->buffer + BUFFER4_OFFSET(0); + ctx->reduce_scatter_buffer[1][i] = ctx->workspace[i]->buffer + BUFFER4_OFFSET(1); + + ctx->alltoall_buffer[0][i] = ctx->workspace[i]->buffer + BUFFER5_OFFSET(0); + ctx->alltoall_buffer[1][i] = ctx->workspace[i]->buffer + BUFFER5_OFFSET(1); } + + const int64_t handle = static_cast(shm_contexts.size()); + shm_contexts.push_back(ctx); + return handle; +} +void shm_default_initialize(int size, int rank, const char* addr_string, const char* port_string) { + int64_t handle = shm_initialize(size, rank, addr_string, port_string, "default"); + + default_shm_context = shm_contexts[handle]; } -#define positive_mod(num, mod) ((((num) % (mod)) + (mod)) % (mod)) -#define rank_mod(rank) positive_mod(rank, world_size) -size_t slice_size(size_t chunk_el, int slice_idx) { - size_t slice_size = chunk_el / world_size; - return slice_idx == world_size - 1 ? slice_size + (chunk_el % world_size) : slice_size; +int64_t shm_group_initialize(const std::string& group_name, int64_t group_size, int64_t group_rank) { + const char* addr_string = getenv("MASTER_ADDR"); + const char* port_string = getenv("MASTER_PORT"); + + TORCH_CHECK(addr_string != nullptr, "MASTER_ADDR is not set"); + TORCH_CHECK(port_string != nullptr, "MASTER_PORT is not set"); + + return shm_initialize(group_size, group_rank, addr_string, port_string, group_name.c_str()); } -char* slice_data(char* data_ptr, size_t chunk_el, int el_size, int slice_idx) { - size_t slice_size = chunk_el / world_size; +size_t slice_size(size_t chunk_el, int slice_idx, int group_size) { + size_t slice_size = chunk_el / group_size; + return slice_idx == group_size - 1 ? slice_size + (chunk_el % group_size) : slice_size; +} + +char* slice_data(char* data_ptr, size_t chunk_el, int el_size, int slice_idx, int group_size) { + size_t slice_size = chunk_el / group_size; size_t el_offset = slice_size * slice_idx; return data_ptr + el_offset * el_size; } -size_t slice_el_start(size_t chunk_el, int slice_idx) { - size_t slice_size = chunk_el / world_size; +size_t slice_el_start(size_t chunk_el, int slice_idx, int group_size) { + size_t slice_size = chunk_el / group_size; return slice_size * slice_idx; } -void symmetric_naive_all_reduce(char* data_ptr, c10::ScalarType scalar_type, size_t chunk_size, size_t chunk_el) { +void symmetric_naive_all_reduce( + struct shm_context* ctx, char* data_ptr, c10::ScalarType scalar_type, size_t chunk_size, size_t chunk_el) { const int state_group = STATE_GROUP_SYMMETRIC_ALLREDUCE; - static int current_buffer = 0; - static int state_idx = 0; + int& current_buffer = ctx->symmetric_current_buffer; + int& state_idx = ctx->symmetric_state_idx; // init states to case 0 to get rid of "maybe-uninitialized" warning. enum coll_state copy_current = coll_allreduce_naive__copy_in_done; @@ -273,31 +359,30 @@ void symmetric_naive_all_reduce(char* data_ptr, c10::ScalarType scalar_type, siz } state_idx = (state_idx + 1) % 3; - parallel_memcpy(symmetric_buffer[current_buffer][world_rank], data_ptr, chunk_size); + parallel_memcpy(ctx->symmetric_buffer[current_buffer][ctx->group_rank], data_ptr, chunk_size); std::atomic_thread_fence(std::memory_order_release); - workspace[world_rank]->states[state_group] = copy_current; + ctx->workspace[ctx->group_rank]->states[state_group] = copy_current; - for (int i = 0; i < world_size; i++) { + for (int i = 0; i < ctx->group_size; i++) { // wait until the other rank copy the buffer - if (i != world_rank) { - wait_buffer_state_until_2(i, copy_current, copy_next, state_group); + if (i != ctx->group_rank) { + wait_buffer_state_until_2(ctx, i, copy_current, copy_next, state_group); } } - // each rank reduce the buffer independently so therre is no need for + // each rank reduce the buffer independently so there is no need for // synchronization afterward - reduce_all_buffers(0, chunk_el, scalar_type, world_rank, data_ptr, symmetric_buffer[current_buffer]); + reduce_all_buffers(0, chunk_el, scalar_type, data_ptr, ctx->symmetric_buffer[current_buffer], ctx->group_size); // switch buffer current_buffer = 1 - current_buffer; } -// naive allreduce distributed, each rank do naive reduce on its slice -void distributed_naive_reduce(char* data_ptr, c10::ScalarType scalar_type, size_t chunk_size, size_t chunk_el) { +void distributed_naive_reduce( + struct shm_context* ctx, char* data_ptr, c10::ScalarType scalar_type, size_t chunk_size, size_t chunk_el) { const int state_group = STATE_GROUP_DISTRIBUTED_ALLREDUCE; - static int current_buffer = 0; - static int state_idx = 0; - + int& current_buffer = ctx->distributed_current_buffer; + int& state_idx = ctx->distributed_state_idx; // init states to case 0 to get rid of "maybe-uninitialized" warning. enum coll_state copy_current = coll_allreduce_naive__copy_in_done; enum coll_state reduce_current = coll_allreduce_naive__reduce_done; @@ -322,65 +407,75 @@ void distributed_naive_reduce(char* data_ptr, c10::ScalarType scalar_type, size_ state_idx = (state_idx + 1) % 2; int data_size = chunk_size / chunk_el; - parallel_memcpy(distributed_buffer[current_buffer][world_rank], data_ptr, chunk_size); + parallel_memcpy(ctx->distributed_buffer[current_buffer][ctx->group_rank], data_ptr, chunk_size); std::atomic_thread_fence(std::memory_order_release); - workspace[world_rank]->states[state_group] = copy_current; + ctx->workspace[ctx->group_rank]->states[state_group] = copy_current; - for (int i = 0; i < world_size; i++) { + for (int i = 0; i < ctx->group_size; i++) { // wait until all the other ranks copy the buffer - if (i != world_rank) wait_buffer_state_until_2(i, copy_current, reduce_current, state_group); + if (i != ctx->group_rank) { + wait_buffer_state_until_2(ctx, i, copy_current, reduce_current, state_group); + } } // reduce scatter reduce_all_buffers( - slice_el_start(chunk_el, world_rank), - slice_size(chunk_el, world_rank), + slice_el_start(chunk_el, ctx->group_rank, ctx->group_size), + slice_size(chunk_el, ctx->group_rank, ctx->group_size), scalar_type, - world_rank, - distributed_buffer[current_buffer][world_rank], - distributed_buffer[current_buffer]); - std::atomic_thread_fence(std::memory_order_release); - workspace[world_rank]->states[state_group] = reduce_current; + ctx->distributed_buffer[current_buffer][ctx->group_rank], + ctx->distributed_buffer[current_buffer], + ctx->group_size); - for (int i = 0; i < world_size; i++) { + std::atomic_thread_fence(std::memory_order_release); + ctx->workspace[ctx->group_rank]->states[state_group] = reduce_current; + + for (int i = 0; i < ctx->group_size; i++) { // wait until all the other ranks reduce the buffer - if (i != world_rank) wait_buffer_state_until_2(i, reduce_current, copy_next, state_group); + if (i != ctx->group_rank) { + wait_buffer_state_until_2(ctx, i, reduce_current, copy_next, state_group); + } } - for (int i = 0; i < world_size; i++) { - int rank = (i + world_rank) % world_size; + for (int i = 0; i < ctx->group_size; i++) { + int rank = (i + ctx->group_rank) % ctx->group_size; parallel_memcpy( - slice_data(data_ptr, chunk_el, data_size, rank), - slice_data(distributed_buffer[current_buffer][rank], chunk_el, chunk_size / chunk_el, rank), - slice_size(chunk_el, rank) * data_size); + slice_data(data_ptr, chunk_el, data_size, rank, ctx->group_size), + slice_data( + ctx->distributed_buffer[current_buffer][rank], chunk_el, chunk_size / chunk_el, rank, ctx->group_size), + slice_size(chunk_el, rank, ctx->group_size) * data_size); } current_buffer = 1 - current_buffer; } -void all_reduce_outer_loop(torch::Tensor& data, size_t numel, int data_size) { +void all_reduce_outer_loop(torch::Tensor& data, size_t numel, int data_size, int64_t handle) { + auto* ctx = get_shm_context(handle); for (int offset = 0; offset < data_size; offset += MAX_BUF_SIZE) { auto data_ptr = ((char*)(data.data_ptr()) + offset); size_t chunk_size = data_size - offset > MAX_BUF_SIZE ? MAX_BUF_SIZE : data_size - offset; size_t chunk_el = chunk_size / (data_size / numel); if (chunk_size < NAIVE_ALLREDUCE_THRESHOLD) { - symmetric_naive_all_reduce(data_ptr, data.scalar_type(), chunk_size, chunk_el); + symmetric_naive_all_reduce(ctx, data_ptr, data.scalar_type(), chunk_size, chunk_el); } else { - distributed_naive_reduce(data_ptr, data.scalar_type(), chunk_size, chunk_el); + distributed_naive_reduce(ctx, data_ptr, data.scalar_type(), chunk_size, chunk_el); } } } template -void naive_all_gather(char* result_ptr, char* data_ptr, size_t res_stride, size_t chunk_size, size_t chunk_el) { - static int current_buffer = 0; - static int state_idx = 0; +void naive_all_gather( + struct shm_context* ctx, char* result_ptr, char* data_ptr, size_t res_stride, size_t chunk_size, size_t chunk_el) { + int& current_buffer = + STATE_GROUP == STATE_GROUP_ALL_GATHER ? ctx->allgather_current_buffer : ctx->allgather_into_tensor_current_buffer; + int& state_idx = + STATE_GROUP == STATE_GROUP_ALL_GATHER ? ctx->allgather_state_idx : ctx->allgather_into_tensor_state_idx; char*** buffer = nullptr; if constexpr (STATE_GROUP == STATE_GROUP_ALL_GATHER) { - buffer = allgather_buffer; + buffer = ctx->allgather_buffer; } else if constexpr (STATE_GROUP == STATE_GROUP_ALL_GATHER_INTO_TENSOR) { - buffer = allgather_into_tensor_buffer; + buffer = ctx->allgather_into_tensor_buffer; } else { static_assert( STATE_GROUP == STATE_GROUP_ALL_GATHER || STATE_GROUP == STATE_GROUP_ALL_GATHER_INTO_TENSOR, @@ -409,22 +504,26 @@ void naive_all_gather(char* result_ptr, char* data_ptr, size_t res_stride, size_ } state_idx = (state_idx + 1) % 3; - parallel_memcpy(buffer[current_buffer][world_rank], data_ptr, chunk_size); + parallel_memcpy(buffer[current_buffer][ctx->group_rank], data_ptr, chunk_size); std::atomic_thread_fence(std::memory_order_release); - workspace[world_rank]->states[STATE_GROUP] = copy_current; + ctx->workspace[ctx->group_rank]->states[STATE_GROUP] = copy_current; - for (int i = 0; i < world_size; i++) { + for (int i = 0; i < ctx->group_size; i++) { // wait until all the other ranks copy the buffer - if (i != world_rank) wait_buffer_state_until_2(i, copy_current, copy_next, STATE_GROUP); + if (i != ctx->group_rank) { + wait_buffer_state_until_2(ctx, i, copy_current, copy_next, STATE_GROUP); + } } - for (int i = 0; i < world_size; i++) { + for (int i = 0; i < ctx->group_size; i++) { parallel_memcpy(result_ptr + i * res_stride, buffer[current_buffer][i], chunk_size); } current_buffer = 1 - current_buffer; } template -torch::Tensor& all_gather(torch::Tensor& result, torch::Tensor& data, int dim, size_t numel, int data_size) { +torch::Tensor& +all_gather(torch::Tensor& result, torch::Tensor& data, int dim, size_t numel, int data_size, int64_t handle) { + auto* ctx = get_shm_context(handle); size_t dim_el = data.stride(dim) * data.size(dim); int dtype_size = data_size / numel; size_t dim_size = dim_el * dtype_size; @@ -436,7 +535,8 @@ torch::Tensor& all_gather(torch::Tensor& result, torch::Tensor& data, int dim, s size_t chunk_size = dim_size - offset > MAX_BUF_SIZE ? MAX_BUF_SIZE : dim_size - offset; size_t chunk_el = chunk_size / dtype_size; naive_all_gather( - result_ptr + i * dim_size * world_size + offset, + ctx, + result_ptr + i * dim_size * ctx->group_size + offset, data_ptr + i * dim_size + offset, dim_size, chunk_size, @@ -445,12 +545,11 @@ torch::Tensor& all_gather(torch::Tensor& result, torch::Tensor& data, int dim, s } return result; } - -template torch::Tensor& all_gather(torch::Tensor&, torch::Tensor&, int, size_t, int); +template torch::Tensor& all_gather(torch::Tensor&, torch::Tensor&, int, size_t, int, int64_t); template torch::Tensor& -all_gather(torch::Tensor&, torch::Tensor&, int, size_t, int); - +all_gather(torch::Tensor&, torch::Tensor&, int, size_t, int, int64_t); void naive_reduce_scatter( + struct shm_context* ctx, char* output_ptr, char* data_ptr, c10::ScalarType scalar_type, @@ -458,8 +557,8 @@ void naive_reduce_scatter( size_t chunk_el, int element_size) { const int state_group = STATE_GROUP_REDUCE_SCATTER; - static int current_buffer = 0; - static int state_idx = 0; + int& current_buffer = ctx->reduce_scatter_current_buffer; + int& state_idx = ctx->reduce_scatter_state_idx; enum coll_state copy_current = coll_reduce_scatter_naive__copy_in_done; enum coll_state copy_next = coll_alt1_reduce_scatter_naive__copy_in_done; @@ -483,40 +582,97 @@ void naive_reduce_scatter( state_idx = (state_idx + 1) % 3; // Step 1: copy local data to shared buffer - parallel_memcpy(reduce_scatter_buffer[current_buffer][world_rank], data_ptr, chunk_size); + parallel_memcpy(ctx->reduce_scatter_buffer[current_buffer][ctx->group_rank], data_ptr, chunk_size); std::atomic_thread_fence(std::memory_order_release); - workspace[world_rank]->states[state_group] = copy_current; + ctx->workspace[ctx->group_rank]->states[state_group] = copy_current; // Step 2: wait for all ranks to copy in - for (int i = 0; i < world_size; i++) { - if (i != world_rank) wait_buffer_state_until_2(i, copy_current, copy_next, state_group); + for (int i = 0; i < ctx->group_size; i++) { + if (i != ctx->group_rank) { + wait_buffer_state_until_2(ctx, i, copy_current, copy_next, state_group); + } } - // // Step 3: do local reduce on this rank’s slice only - int start_el = slice_el_start(chunk_el, world_rank); - // each rank reduce its slice of buffer independently so therre is no need for + // Step 3: do local reduce on this rank's slice only + int start_el = slice_el_start(chunk_el, ctx->group_rank, ctx->group_size); + + // each rank reduce its slice of buffer independently so there is no need for // synchronization afterward reduce_all_buffers( start_el, - slice_size(chunk_el, world_rank), + slice_size(chunk_el, ctx->group_rank, ctx->group_size), scalar_type, - world_rank, - output_ptr - - start_el * element_size, // in reduce_all_buffers, the output_ptr is the buffer for all ranks, but here - // output_ptr is already the local buffer for one rank. Adjust it here. - reduce_scatter_buffer[current_buffer]); + output_ptr - start_el * element_size, + ctx->reduce_scatter_buffer[current_buffer], + ctx->group_size); // done current_buffer = 1 - current_buffer; } -void reduce_scatter_outer_loop(torch::Tensor& output, torch::Tensor& data, size_t numel, int data_size) { +void reduce_scatter_outer_loop( + torch::Tensor& output, torch::Tensor& data, size_t numel, int data_size, int64_t handle) { + auto* ctx = get_shm_context(handle); for (int offset = 0; offset < data_size; offset += MAX_BUF_SIZE) { auto data_ptr = ((char*)(data.data_ptr()) + offset); auto output_ptr = ((char*)(output.data_ptr()) + offset); size_t chunk_size = std::min((size_t)MAX_BUF_SIZE, (size_t)(data_size - offset)); size_t chunk_el = chunk_size / (data_size / numel); - naive_reduce_scatter(output_ptr, data_ptr, data.scalar_type(), chunk_size, chunk_el, data.element_size()); + naive_reduce_scatter(ctx, output_ptr, data_ptr, data.scalar_type(), chunk_size, chunk_el, data.element_size()); } } + +void all_to_all(char* output_ptr, char* input_ptr, size_t data_size, int64_t handle) { + auto* ctx = get_shm_context(handle); + + TORCH_CHECK(data_size <= MAX_BUF_SIZE, "SHM all-to-all input size exceeds maximum buffer size"); + TORCH_CHECK(data_size % ctx->group_size == 0, "SHM all-to-all input size must be divisible by group size"); + + int& current_buffer = ctx->alltoall_current_buffer; + int& state_idx = ctx->alltoall_state_idx; + + enum coll_state copy_current = coll_allgather_naive__copy_in_done; + enum coll_state copy_next = coll_alt1_allgather_naive__copy_in_done; + + switch (state_idx) { + case 0: + copy_current = coll_allgather_naive__copy_in_done; + copy_next = coll_alt1_allgather_naive__copy_in_done; + break; + case 1: + copy_current = coll_alt1_allgather_naive__copy_in_done; + copy_next = coll_alt2_allgather_naive__copy_in_done; + break; + case 2: + copy_current = coll_alt2_allgather_naive__copy_in_done; + copy_next = coll_allgather_naive__copy_in_done; + break; + default: + assert(!"Should not get here."); + } + + state_idx = (state_idx + 1) % 3; + // copy local input to shared buffer + parallel_memcpy(ctx->alltoall_buffer[current_buffer][ctx->group_rank], input_ptr, data_size); + std::atomic_thread_fence(std::memory_order_release); + ctx->workspace[ctx->group_rank]->states[STATE_GROUP_ALL_TO_ALL] = copy_current; + + // wait until all ranks copy the buffer + for (int i = 0; i < ctx->group_size; i++) { + if (i != ctx->group_rank) { + wait_buffer_state_until_2(ctx, i, copy_current, copy_next, STATE_GROUP_ALL_TO_ALL); + } + } + + size_t chunk_size = data_size / ctx->group_size; + + // read the chunk addressed to this rank from every source rank + for (int i = 0; i < ctx->group_size; i++) { + parallel_memcpy( + output_ptr + i * chunk_size, + ctx->alltoall_buffer[current_buffer][i] + ctx->group_rank * chunk_size, + chunk_size); + } + current_buffer = 1 - current_buffer; +} diff --git a/python/sglang/kernels/aot/csrc/cpu/shm.h b/python/sglang/kernels/aot/csrc/cpu/shm.h index 3d5ec02040..827adbb2ba 100644 --- a/python/sglang/kernels/aot/csrc/cpu/shm.h +++ b/python/sglang/kernels/aot/csrc/cpu/shm.h @@ -10,10 +10,24 @@ constexpr int STATE_GROUP_DISTRIBUTED_ALLREDUCE = 1; constexpr int STATE_GROUP_ALL_GATHER = 2; constexpr int STATE_GROUP_ALL_GATHER_INTO_TENSOR = 3; constexpr int STATE_GROUP_REDUCE_SCATTER = 4; +constexpr int STATE_GROUP_ALL_TO_ALL = 5; + +int64_t shm_initialize(int size, int rank, const char* addr_string, const char* port_string, const char* group_name); + +void shm_default_initialize(int size, int rank, const char* addr_string, const char* port_string); + +int64_t shm_group_initialize(const std::string& group_name, int64_t group_size, int64_t group_rank); + +int64_t shm_get_group_size(int64_t handle); + +void all_reduce_outer_loop(torch::Tensor& data, size_t numel, int data_size, int64_t handle); -void shm_initialize(int size, int rank, const char* addr_string, const char* port_string); -void all_reduce_outer_loop(torch::Tensor& data, size_t numel, int data_size); template -torch::Tensor& all_gather(torch::Tensor& result, torch::Tensor& data, int dim, size_t numel, int data_size); -void reduce_scatter_outer_loop(torch::Tensor& output, torch::Tensor& data, size_t numel, int data_size); -#endif +torch::Tensor& +all_gather(torch::Tensor& result, torch::Tensor& data, int dim, size_t numel, int data_size, int64_t handle); + +void reduce_scatter_outer_loop(torch::Tensor& output, torch::Tensor& data, size_t numel, int data_size, int64_t handle); + +void all_to_all(char* output_ptr, char* input_ptr, size_t data_size, int64_t handle); + +#endif // __SHM_COLLECTIVES__ diff --git a/python/sglang/kernels/aot/csrc/cpu/torch_extension_cpu.cpp b/python/sglang/kernels/aot/csrc/cpu/torch_extension_cpu.cpp index 59254ad7da..521cbc7f0b 100644 --- a/python/sglang/kernels/aot/csrc/cpu/torch_extension_cpu.cpp +++ b/python/sglang/kernels/aot/csrc/cpu/torch_extension_cpu.cpp @@ -127,7 +127,6 @@ std::tuple fused_qk_gemma_rmsnorm_with_gate_ double eps, int64_t head_dim, int64_t num_head); - // speculative decoding void verify_tree_greedy_cpu( at::Tensor predicts, @@ -509,17 +508,23 @@ at::Tensor conv3d_embed_cpu(const at::Tensor& input, const at::Tensor& weight, c // shared memory init void initialize(int64_t size, int64_t rank); -// shared mmeory all_reduce -void shm_allreduce(at::Tensor& data, int64_t op); +// group shared memory init +int64_t shm_group_initialize(const std::string& group_name, int64_t group_size, int64_t group_rank); + +// shared memory all_reduce +void shm_allreduce(at::Tensor& data, int64_t op, int64_t handle = -1); // shared memory all_gather -at::Tensor shm_allgather(at::Tensor& data, int64_t dim); +at::Tensor shm_allgather(at::Tensor& data, int64_t dim, int64_t handle = -1); + +// group shared memory all_to_all +void shm_alltoall(at::Tensor& output_tensor, at::Tensor& data, int64_t handle); // shared memory all_gather_into_tensor -void shm_allgather_into_tensor(at::Tensor& output_tensor, at::Tensor& data); +void shm_allgather_into_tensor(at::Tensor& output_tensor, at::Tensor& data, int64_t handle = -1); // shared memory reduce_scatter_tensor -void shm_reduce_scatter_tensor(at::Tensor& output_tensor, at::Tensor& data, int64_t op); +void shm_reduce_scatter_tensor(at::Tensor& output_tensor, at::Tensor& data, int64_t op, int64_t handle = -1); // rope std::tuple rotary_embedding_cpu( @@ -943,17 +948,32 @@ TORCH_LIBRARY_FRAGMENT(sgl_kernel, m) { m.def("conv3d_embed_cpu(Tensor input, Tensor weight, Tensor bias, bool is_vnni) -> Tensor"); m.impl("conv3d_embed_cpu", torch::kCPU, &conv3d_embed_cpu); - // all reduce + // shared memory m.def("initialize(int size, int rank) -> ()"); - m.def("shm_allreduce(Tensor(a!) data, int reduce_op) -> ()"); + + // all_reduce + m.def("shm_allreduce(Tensor(a!) data, int reduce_op, int handle=-1) -> ()"); m.impl("shm_allreduce", torch::kCPU, &shm_allreduce); - m.def("shm_allgather(Tensor data, int dim) -> Tensor"); + + // all_gather + m.def("shm_allgather(Tensor data, int dim, int handle=-1) -> Tensor"); m.impl("shm_allgather", torch::kCPU, &shm_allgather); - m.def("shm_allgather_into_tensor(Tensor(a!) output_tensor, Tensor data) -> ()"); + + // all_gather_into_tensor + m.def("shm_allgather_into_tensor(Tensor(a!) output_tensor, Tensor data, int handle=-1) -> ()"); m.impl("shm_allgather_into_tensor", torch::kCPU, &shm_allgather_into_tensor); - m.def("shm_reduce_scatter_tensor(Tensor(a!) output_tensor, Tensor data, int reduce_op) -> ()"); + + // reduce_scatter_tensor + m.def("shm_reduce_scatter_tensor(Tensor(a!) output_tensor, Tensor data, int reduce_op, int handle=-1) -> ()"); m.impl("shm_reduce_scatter_tensor", torch::kCPU, &shm_reduce_scatter_tensor); + // explicit group context initialization + m.def("shm_group_initialize(str group_name, int group_size, int group_rank) -> int"); + + // all_to_all + m.def("shm_alltoall(Tensor(a!) output_tensor, Tensor data, int handle) -> ()"); + m.impl("shm_alltoall", torch::kCPU, &shm_alltoall); + // rope m.def( "rotary_embedding_cpu(Tensor positions, Tensor query, Tensor key, int head_size, Tensor cos_sin_cache, " @@ -1028,6 +1048,7 @@ TORCH_LIBRARY_FRAGMENT(sgl_kernel, m) { TORCH_LIBRARY_IMPL(sgl_kernel, CatchAll, m) { m.impl("init_cpu_threads_env", init_cpu_threads_env); m.impl("initialize", &initialize); + m.impl("shm_group_initialize", &shm_group_initialize); } REGISTER_EXTENSION(common_ops) diff --git a/python/sglang/multimodal_gen/runtime/distributed/group_coordinator.py b/python/sglang/multimodal_gen/runtime/distributed/group_coordinator.py index 3d32e83b1d..471317d49e 100644 --- a/python/sglang/multimodal_gen/runtime/distributed/group_coordinator.py +++ b/python/sglang/multimodal_gen/runtime/distributed/group_coordinator.py @@ -2,6 +2,8 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project # Copied and adapted from: https://github.com/hao-ai-lab/FastVideo +import os + # Copyright 2024 xDiT team. # Adapted from # https://github.com/vllm-project/vllm/blob/main/vllm/distributed/parallel_state.py @@ -219,6 +221,28 @@ class GroupCoordinator: # TODO: fix it for other platforms self.device = get_local_torch_device() + self.shm_handle = None + global_world_size = torch.distributed.get_world_size() + local_size = int(os.environ.get("LOCAL_SIZE", "0")) + self.is_single_node = local_size > 0 and global_world_size == local_size + if ( + current_platform.is_cpu() + and self.is_single_node + and self.world_size > 1 + and hasattr(torch.ops.sgl_kernel, "shm_group_initialize") + and self.unique_name.startswith(("tp", "sp")) + ): + group_ranks = "-".join(str(rank) for rank in self.ranks) + group_name = f"{self.unique_name}-{group_ranks}" + + self.shm_handle = int( + torch.ops.sgl_kernel.shm_group_initialize( + group_name, + self.world_size, + self.rank_in_group, + ) + ) + torch.distributed.barrier(group=self.cpu_group) self.device_communicator: DeviceCommunicatorBase = None # type: ignore if use_device_communicator and self.world_size > 1: @@ -396,17 +420,20 @@ class GroupCoordinator: return output if ( current_platform.is_cpu() + and not async_op + and self.shm_handle is not None and is_shm_available(input_.dtype, self.world_size, len(self.ranks)) and op is torch.distributed.ReduceOp.SUM ): # for CPU platform, intra-node case we could speedup with shared memory based comm ops torch.ops.sgl_kernel.shm_allreduce( - input_, int(torch.distributed.ReduceOp.SUM) + input_, int(torch.distributed.ReduceOp.SUM), self.shm_handle ) else: torch.distributed.all_reduce( input_, op=op, group=self.device_group, async_op=async_op ) + return input_ def _all_reduce_out_of_place(self, input_: torch.Tensor) -> torch.Tensor: @@ -440,18 +467,24 @@ class GroupCoordinator: input_size, dtype=input_.dtype, device=input_.device ) - # All-gather. - if current_platform.is_cpu() and is_shm_available( - input_.dtype, self.world_size, len(self.ranks) + # Group-aware CPU SHM all-gather. + if ( + current_platform.is_cpu() + and self.shm_handle is not None + and is_shm_available(input_.dtype, self.world_size, len(self.ranks)) ): - output_tensor = torch.ops.sgl_kernel.shm_allgather(input_, dim) + output_tensor = torch.ops.sgl_kernel.shm_allgather( + input_, dim, self.shm_handle + ) + if separate_tensors: return list(output_tensor.chunk(world_size, dim=dim)) return output_tensor - else: - if not input_.is_contiguous(): - input_ = input_.contiguous() - all_gather_single(output_tensor, input_, group=self.device_group) + + # Fallback. + if not input_.is_contiguous(): + input_ = input_.contiguous() + all_gather_single(output_tensor, input_, group=self.device_group) if dim != 0: input_size[0] //= world_size @@ -889,3 +922,13 @@ class SequenceParallelGroupCoordinator(GroupCoordinator): self.ulysses_rank = torch.distributed.get_rank(self.ulysses_group) self.ring_world_size = torch.distributed.get_world_size(self.ring_group) self.ring_rank = torch.distributed.get_rank(self.ring_group) + self.ulysses_shm_handle = None + if ( + current_platform.is_cpu() + and self.ulysses_world_size > 1 + and hasattr(torch.ops.sgl_kernel, "shm_group_initialize") + ): + # TODO: Initialize a dedicated SHM handle for the Ulysses subgroup + # when Ring Attention is supported on CPU. The current SP handle + # is only valid when ring_degree == 1. + self.ulysses_shm_handle = self.shm_handle diff --git a/python/sglang/multimodal_gen/runtime/layers/usp.py b/python/sglang/multimodal_gen/runtime/layers/usp.py index 9afe051928..151fe0d2dc 100644 --- a/python/sglang/multimodal_gen/runtime/layers/usp.py +++ b/python/sglang/multimodal_gen/runtime/layers/usp.py @@ -16,9 +16,11 @@ from sglang.multimodal_gen.runtime.distributed.parallel_state import ( get_ulysses_parallel_rank, get_ulysses_parallel_world_size, ) +from sglang.multimodal_gen.runtime.platforms import current_platform from sglang.srt.utils.common import torch_release _cp_options.enable_load_balance = False +SHM_ALLTOALL_MAX_BYTES = 32 * 1024 * 1024 if TYPE_CHECKING: from sglang.multimodal_gen.runtime.layers.attention.backends.attention_backend import ( @@ -105,7 +107,8 @@ def _a2a_staging_buffer( def _usp_all_to_all_single(x: torch.Tensor, role: str | None = None) -> torch.Tensor: - ulysses_pg = get_sp_group().ulysses_group + sp_group = get_sp_group() + ulysses_pg = sp_group.ulysses_group assert ulysses_pg is not None, "Ulysses process group is not initialized." x_shape = x.shape x = x.flatten().contiguous() @@ -124,9 +127,21 @@ def _usp_all_to_all_single(x: torch.Tensor, role: str | None = None) -> torch.Te output = torch.empty_like(x) else: output = _a2a_staging_buffer(role, x.shape, x.dtype, x.device) + # USP calls this collective many times per denoising step and waits # immediately, so avoid the extra wrapper overhead of functional collectives. - torch.distributed.all_to_all_single(output, x, group=ulysses_pg) + handle = sp_group.ulysses_shm_handle + input_bytes = x.numel() * x.element_size() + if ( + current_platform.is_cpu() + and handle is not None + and handle >= 0 + and input_bytes <= SHM_ALLTOALL_MAX_BYTES + ): + torch.ops.sgl_kernel.shm_alltoall(output, x, sp_group.ulysses_shm_handle) + else: + torch.distributed.all_to_all_single(output, x, group=ulysses_pg) + return output.reshape(x_shape) diff --git a/python/sglang/multimodal_gen/runtime/managers/cpu_worker.py b/python/sglang/multimodal_gen/runtime/managers/cpu_worker.py index 69cf9af239..68d23fa219 100644 --- a/python/sglang/multimodal_gen/runtime/managers/cpu_worker.py +++ b/python/sglang/multimodal_gen/runtime/managers/cpu_worker.py @@ -30,6 +30,7 @@ class CPUWorker(GPUWorker): master_port: int, server_args: ServerArgs, ): + os.environ["LOCAL_SIZE"] = str(server_args.num_gpus // server_args.nnodes) super().__init__(local_rank, rank, master_port, server_args) if _is_cpu_amx_available: self.init_cpu_threads_binding() @@ -70,7 +71,3 @@ class CPUWorker(GPUWorker): # Bind OpenMP threads to CPU cores torch.ops.sgl_kernel.init_cpu_threads_env(self.local_omp_cpuid) - - # Set local size to hint SGLang to use shared memory based AllReduce - os.environ["LOCAL_SIZE"] = str(self.server_args.tp_size) - torch.ops.sgl_kernel.initialize(self.server_args.tp_size, self.rank) diff --git a/test/registered/cpu/test_comm.py b/test/registered/cpu/test_comm.py index dc08bb92d9..2a9a6f47c0 100644 --- a/test/registered/cpu/test_comm.py +++ b/test/registered/cpu/test_comm.py @@ -16,7 +16,9 @@ from sglang.test.test_utils import CustomTestCase, find_available_port register_cpu_ci(est_time=43, suite="stage-a-test-cpu-intel") -def run_distributed_test(rank, world_size, master_port, output_writer, fn): +def run_distributed_test( + rank, world_size, master_port, output_writer, fn, init_legacy_shm=True +): try: os.environ["RANK"] = str(rank) os.environ["WORLD_SIZE"] = str(world_size) @@ -25,10 +27,10 @@ def run_distributed_test(rank, world_size, master_port, output_writer, fn): os.environ["LOCAL_SIZE"] = str(world_size) dist.init_process_group("gloo", rank=rank, world_size=world_size) - torch.ops.sgl_kernel.initialize(world_size, rank) + if init_legacy_shm: + torch.ops.sgl_kernel.initialize(world_size, rank) fn(rank, world_size) - execution_ok = True except Exception as e: print(f"subprocess[{rank=}] has error: {e}", flush=True) @@ -117,8 +119,94 @@ def reduce_scatter_tensor_fn(rank, world_size): torch.testing.assert_close(output_tensor, output_shm) +def init_group(rank): + group_ranks_list = [[0, 2], [1, 3]] + + groups = [dist.new_group(ranks=ranks, backend="gloo") for ranks in group_ranks_list] + + group_idx = rank % 2 + group = groups[group_idx] + group_ranks = group_ranks_list[group_idx] + + group_size = dist.get_world_size(group) + group_rank = dist.get_rank(group) + + group_name = f"test_group_{os.environ['MASTER_PORT']}_" + "_".join( + map(str, group_ranks) + ) + + handle = torch.ops.sgl_kernel.shm_group_initialize( + group_name, group_size, group_rank + ) + + return group, handle + + +def group_all_reduce_fn(rank, world_size): + group, handle = init_group(rank) + + op = dist.ReduceOp.SUM + + for dtype in [torch.float32, torch.bfloat16, torch.float16]: + for size in [(2, 10), (600000,)]: + tensor = torch.randn(size, dtype=dtype) + tensor_shm = copy.deepcopy(tensor) + + dist.all_reduce(tensor, op=op, group=group) + + torch.ops.sgl_kernel.shm_allreduce(tensor_shm, op, handle) + + torch.testing.assert_close(tensor, tensor_shm) + + +def group_all_gather_fn(rank, world_size): + group, handle = init_group(rank) + + group_size = dist.get_world_size(group) + + for dtype in [torch.float32, torch.bfloat16, torch.float16]: + tensor = torch.randn(2, 10, dtype=dtype) + + input_size = tensor.size() + output_size = (input_size[0] * group_size,) + input_size[1:] + + output_tensor = torch.empty( + output_size, + dtype=tensor.dtype, + device=tensor.device, + ) + output_shm = torch.empty( + output_size, + dtype=tensor.dtype, + device=tensor.device, + ) + + dist.all_gather_into_tensor(output_tensor, tensor, group=group) + + torch.ops.sgl_kernel.shm_allgather_into_tensor(output_shm, tensor, handle) + + torch.testing.assert_close(output_tensor, output_shm) + + +def group_all_to_all_fn(rank, world_size): + group, handle = init_group(rank) + + group_size = dist.get_world_size(group) + + for dtype in [torch.float32, torch.bfloat16, torch.float16]: + tensor = torch.randn(group_size * 2, 10, dtype=dtype) + output_tensor = torch.empty_like(tensor) + output_shm = torch.empty_like(tensor) + + dist.all_to_all_single(output_tensor, tensor, group=group) + + torch.ops.sgl_kernel.shm_alltoall(output_shm, tensor, handle) + + torch.testing.assert_close(output_tensor, output_shm) + + class TestComm(CustomTestCase): - def _spawn_and_check(self, fn, world_size=2): + def _spawn_and_check(self, fn, world_size=2, init_legacy_shm=True): mp.set_start_method("spawn", force=True) master_port = find_available_port(23456) @@ -134,6 +222,7 @@ class TestComm(CustomTestCase): master_port=master_port, output_writer=output_writer, fn=fn, + init_legacy_shm=init_legacy_shm, ), ) p.start() @@ -146,16 +235,25 @@ class TestComm(CustomTestCase): p.join() def test_all_reduce(self): - self._spawn_and_check(all_reduce_fn) + self._spawn_and_check(all_reduce_fn, init_legacy_shm=True) def test_all_gather(self): - self._spawn_and_check(all_gather_fn) + self._spawn_and_check(all_gather_fn, init_legacy_shm=True) def test_all_gather_into_tensor(self): - self._spawn_and_check(all_gather_into_tensor_fn) + self._spawn_and_check(all_gather_into_tensor_fn, init_legacy_shm=True) def test_reduce_scatter_tensor(self): - self._spawn_and_check(reduce_scatter_tensor_fn) + self._spawn_and_check(reduce_scatter_tensor_fn, init_legacy_shm=True) + + def test_group_all_reduce(self): + self._spawn_and_check(group_all_reduce_fn, world_size=4, init_legacy_shm=False) + + def test_group_all_gather(self): + self._spawn_and_check(group_all_gather_fn, world_size=4, init_legacy_shm=False) + + def test_group_all_to_all(self): + self._spawn_and_check(group_all_to_all_fn, world_size=4, init_legacy_shm=False) if __name__ == "__main__":