[eplb] Warm default-group NCCL P2P transports before KV-cache sizing (#42026)
Co-authored-by: merrymercy <merrymercy@users.noreply.github.com> Co-authored-by: luccafong <luccafong@users.noreply.github.com>
This commit is contained in:
co-authored by
merrymercy
luccafong
parent
ebf161cea4
commit
2ec9e3b11d
@@ -168,6 +168,19 @@ def init_parallel_runtime(
|
||||
):
|
||||
_prewarm_tp_lm_head_all_to_all()
|
||||
|
||||
# EPLB uses the default process group, not the TP/EP coordinators.
|
||||
# Limit warmup to static, single-node full-EP layouts for now.
|
||||
if (
|
||||
device == "cuda"
|
||||
and backend == "nccl"
|
||||
and get_exec().moe.enable_eplb
|
||||
and get_exec().moe.elastic_ep_backend is None
|
||||
and parallel.nnodes == 1
|
||||
and parallel.pp_size == 1
|
||||
and parallel.moe_ep_size == parallel.tp_size > 1
|
||||
):
|
||||
_prewarm_eplb_p2p()
|
||||
|
||||
logger.info(f"Init parallel ends. elapsed={time.perf_counter() - tic:.2f} s")
|
||||
|
||||
|
||||
@@ -350,6 +363,41 @@ def _prewarm_nccl() -> None:
|
||||
)
|
||||
|
||||
|
||||
def _prewarm_eplb_p2p() -> None:
|
||||
"""Materialize EPLB's default-group NCCL transports before KV sizing."""
|
||||
warmup_start = time.perf_counter()
|
||||
rank = dist.get_rank()
|
||||
world_size = dist.get_world_size()
|
||||
free_before, _ = torch.cuda.mem_get_info()
|
||||
# Exercise bulk transfers as well as communicator creation.
|
||||
bytes_per_peer = 4 << 20
|
||||
send = torch.empty(bytes_per_peer, dtype=torch.uint8, device="cuda")
|
||||
recv = torch.empty(
|
||||
(world_size - 1, bytes_per_peer), dtype=torch.uint8, device="cuda"
|
||||
)
|
||||
ops = []
|
||||
for slot, peer in enumerate(p for p in range(world_size) if p != rank):
|
||||
ops.extend(
|
||||
[
|
||||
dist.P2POp(dist.isend, send, peer),
|
||||
dist.P2POp(dist.irecv, recv[slot], peer),
|
||||
]
|
||||
)
|
||||
for work in dist.batch_isend_irecv(ops):
|
||||
work.wait()
|
||||
current_platform.synchronize()
|
||||
del ops, send, recv
|
||||
current_platform.empty_cache()
|
||||
free_after, _ = torch.cuda.mem_get_info()
|
||||
logger.info(
|
||||
"EPLB default-group P2P warmup completed in %.3fs "
|
||||
"(world_size=%d, free memory delta=%.2f MiB)",
|
||||
time.perf_counter() - warmup_start,
|
||||
world_size,
|
||||
(free_before - free_after) / (1 << 20),
|
||||
)
|
||||
|
||||
|
||||
def _prewarm_tp_lm_head_all_to_all() -> None:
|
||||
"""Materialize PyNCCL P2P resources before model-memory accounting."""
|
||||
warmup_start = time.perf_counter()
|
||||
|
||||
Reference in New Issue
Block a user