Files
sglang/test/registered/e2e/layers/test_logprob_memory_distributed.py
T

164 lines
6.7 KiB
Python

"""Logprob memory budgets must preserve TP collectives and graph replay safety."""
import sys
from datetime import timedelta
from types import SimpleNamespace
from unittest.mock import patch
import pytest
import torch
import torch.distributed as dist
import torch.multiprocessing as mp
from sglang.srt.environ import envs
from sglang.srt.layers.logprob_processor import InputLogprobProcessor
from sglang.srt.model_executor.runner_utils import pool
from sglang.test.ci.ci_register import register_cuda_ci
register_cuda_ci(est_time=60, stage="base-b", runner_config="2-gpu-large")
def _run_rank(rank, rendezvous):
torch.cuda.set_device(rank)
dist.init_process_group(
"gloo",
init_method=f"file://{rendezvous}",
rank=rank,
world_size=2,
timeout=timedelta(seconds=60),
)
nccl_group = dist.new_group(backend="nccl", timeout=timedelta(seconds=60))
try:
torch.manual_seed(0)
device = torch.device("cuda", rank)
rows, vocab = 2048, 202752
states = torch.randint(-2, 3, (rows, 64), device=device).to(torch.bfloat16)
weight = torch.randint(-2, 3, (vocab, 64), device=device).to(torch.bfloat16)
local_weight = weight.chunk(2)[rank].contiguous()
token_ids = torch.randint(vocab, (rows,), device=device)
sample_rows = [rows // 2 - 1, rows - 1]
metadata = SimpleNamespace(
sample_indices_cpu=sample_rows,
input_logprob_indices_cpu=list(range(rows)),
extend_return_top_logprob=False,
extend_token_ids_logprob=False,
top_logprobs_nums=None,
extend_logprob_pruned_lens_cpu=[rows // 2, rows // 2],
extend_input_logprob_token_ids_gpu=token_ids,
token_ids_logprobs=None,
)
chunk_rows = []
allocations_borrowed = []
runs = []
def get_logits(chunk, *_args, **_kwargs):
chunk_rows.append(chunk.shape[0])
local = torch.mm(chunk, local_weight.T)
gathered = torch.empty(
(2 * chunk.shape[0], vocab // 2), device=device, dtype=local.dtype
)
dist.all_gather_into_tensor(gathered, local, group=nccl_group)
reshaped = (
gathered.reshape(2, chunk.shape[0], vocab // 2)
.movedim(0, 1)
.reshape(chunk.shape[0], vocab)
)
converted = reshaped.float()
allocations_borrowed.extend(
any(
lo <= tensor.data_ptr()
and tensor.data_ptr() + tensor.nbytes <= lo + size
for lo, size in runs
)
for tensor in (local, gathered, reshaped, converted)
)
return converted
processor = InputLogprobProcessor(vocab, chunking_group=dist.group.WORLD)
args = dict(
pruned_states=states,
sample_indices=torch.tensor(sample_rows, device=device),
input_logprob_indices=torch.arange(rows, device=device),
token_to_seq_idx=[0] * (rows // 2) + [1] * (rows // 2),
lm_head=None,
get_logits_fn=get_logits,
logits_metadata=metadata,
)
for fast in (False, True):
processor.enable_fast_input_logprobs = fast
processor.enable_logprobs_chunk = False
reference, reference_sampled = processor.forward(**args)
reference_logprobs = reference.token_logprobs.cpu()
processor.enable_logprobs_chunk = True
for chunk_size, disabled_rank, borrowing in (
(128, None, True),
(256, None, False),
(128, 1, False),
):
processor.logprobs_chunk_size = chunk_size
state = pool.GraphPoolBorrowState()
handle = torch.cuda.graph_pool_handle()
graph = torch.cuda.CUDAGraph()
seed = torch.zeros(8, device=device)
stream = torch.cuda.Stream()
with (
torch.cuda.stream(stream),
torch.cuda.graph(graph, pool=handle, stream=stream),
):
transient = torch.empty(
(512 + rank * 256) << 20, dtype=torch.uint8, device=device
)
transient.fill_(7)
keep = seed + 1
del transient
torch.cuda.synchronize()
state.disabled_reason = (
"test fallback" if rank == disabled_rank else None
)
with (
pool.get_resources().override(graph_pool_borrow=state),
envs.SGLANG_ENABLE_GRAPH_POOL_BORROW.override(True),
patch.object(
pool, "get_global_graph_memory_pool", return_value=handle
),
patch.object(
torch.cuda,
"mem_get_info",
side_effect=AssertionError(
"Chunk sizing must not query heap headroom"
),
),
):
chunk_rows.clear()
allocations_borrowed.clear()
runs = pool.find_free_graph_pool_runs(handle)
result, sampled = processor.forward(**args)
with pool.graph_pool_replay_scope():
graph.replay()
if result.input_copy_done is not None:
result.input_copy_done.synchronize()
assert torch.equal(result.token_logprobs.cpu(), reference_logprobs)
assert torch.equal(sampled, reference_sampled)
assert (result.input_copy_done is not None) == borrowing
assert all(
borrowed == borrowing for borrowed in allocations_borrowed
)
all_chunk_rows = [None, None]
dist.all_gather_object(all_chunk_rows, chunk_rows)
assert all_chunk_rows[0] == all_chunk_rows[1]
assert chunk_rows == [chunk_size] * (rows // chunk_size)
pool._teardown_borrow_pool()
del graph, keep
finally:
dist.destroy_process_group(nccl_group)
dist.destroy_process_group()
@pytest.mark.skipif(torch.cuda.device_count() < 2, reason="requires two CUDA devices")
def test_logprob_chunks_share_tp_budget_and_survive_replay(tmp_path):
mp.spawn(_run_rank, args=(str(tmp_path / "rendezvous"),), nprocs=2, join=True)
if __name__ == "__main__":
sys.exit(pytest.main([__file__, "-v"]))