Deprecate the parallel getters the context answers, and ratchet them shut (#40342)
This commit is contained in:
@@ -134,7 +134,9 @@ class TestFilterDcpLocalChunkKvIndices(CustomTestCase):
|
||||
|
||||
def test_identity_without_dcp(self):
|
||||
kv = torch.arange(37)
|
||||
with rc.get_parallel().override(dcp_enabled=False, dcp_size=1, dcp_rank=0):
|
||||
with rc.get_parallel().override(
|
||||
dcp_enabled=False, dcp_size=1, dcp_rank=0, attn_dcp_rank=0
|
||||
):
|
||||
self.assertIs(
|
||||
filter_dcp_local_chunk_kv_indices(
|
||||
kv, torch.tensor([0]), torch.tensor([37])
|
||||
|
||||
@@ -25,6 +25,7 @@ import torch
|
||||
import torch.distributed as dist
|
||||
|
||||
import sglang.srt.distributed.parallel_state as ps
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(
|
||||
@@ -130,6 +131,9 @@ def init_distributed():
|
||||
local_rank=local_rank,
|
||||
backend="nccl",
|
||||
)
|
||||
# The context answers a handle from what was stated on it, not from
|
||||
# this module global, so a rank stood up by hand says so itself.
|
||||
get_parallel().override_permanently(world_group=coord)
|
||||
|
||||
cpu_group = coord.cpu_group
|
||||
nccl_group = coord.device_group
|
||||
|
||||
@@ -14,6 +14,7 @@ from sglang.kernels.jit.benchmark import marker
|
||||
from sglang.kernels.jit.benchmark.utils import get_benchmark_range, multigpu_bench_main
|
||||
from sglang.kernels.jit.utils import cache_once, is_arch_support_pdl
|
||||
from sglang.kernels.ops.communication.mp import register_comm_cleanup
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
|
||||
register_cuda_ci(
|
||||
@@ -61,6 +62,9 @@ def _init_cpu_group() -> dist.ProcessGroup:
|
||||
local_rank=local_rank,
|
||||
backend="nccl",
|
||||
)
|
||||
# The context answers a handle from what was stated on it, not from
|
||||
# this module global, so a rank stood up by hand says so itself.
|
||||
get_parallel().override_permanently(world_group=coord)
|
||||
atexit.register(dist.destroy_process_group)
|
||||
torch.cuda.set_stream(torch.cuda.Stream())
|
||||
return coord.cpu_group
|
||||
|
||||
@@ -33,6 +33,7 @@ from sglang.srt.distributed.device_communicators.triton_symm_mem_ag import (
|
||||
all_gather_inner,
|
||||
create_state,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
|
||||
register_cuda_ci(
|
||||
@@ -77,6 +78,9 @@ def _init_cpu_group() -> dist.ProcessGroup:
|
||||
local_rank=local_rank,
|
||||
backend="nccl",
|
||||
)
|
||||
# The context answers a handle from what was stated on it, not from
|
||||
# this module global, so a rank stood up by hand says so itself.
|
||||
get_parallel().override_permanently(world_group=ps._WORLD)
|
||||
atexit.register(dist.destroy_process_group)
|
||||
logging.disable(logging.INFO)
|
||||
torch.cuda.set_stream(torch.cuda.Stream())
|
||||
|
||||
@@ -36,6 +36,7 @@ from sglang.kernels.ops.communication.mp import register_comm_cleanup
|
||||
from sglang.srt.distributed.device_communicators.custom_all_reduce_v2 import (
|
||||
CustomAllReduceV2,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
|
||||
register_cuda_ci(
|
||||
@@ -108,6 +109,9 @@ def _init_cpu_group() -> dist.ProcessGroup:
|
||||
local_rank=local_rank,
|
||||
backend="nccl",
|
||||
)
|
||||
# The context answers a handle from what was stated on it, not from
|
||||
# this module global, so a rank stood up by hand says so itself.
|
||||
get_parallel().override_permanently(world_group=coord)
|
||||
atexit.register(dist.destroy_process_group)
|
||||
logging.disable(logging.INFO)
|
||||
torch.cuda.set_stream(torch.cuda.Stream())
|
||||
|
||||
@@ -37,6 +37,7 @@ from sglang.kernels.ops.communication.mp import register_comm_cleanup
|
||||
from sglang.srt.distributed.device_communicators.custom_all_reduce_v2 import (
|
||||
CustomAllReduceV2,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.kernels.utils import multigpu_pytest_main
|
||||
|
||||
@@ -129,6 +130,9 @@ def _init_cpu_group_once() -> dist.ProcessGroup:
|
||||
local_rank=local_rank,
|
||||
backend="nccl",
|
||||
)
|
||||
# The context answers a handle from what was stated on it, not from
|
||||
# this module global, so a rank stood up by hand says so itself.
|
||||
get_parallel().override_permanently(world_group=coord)
|
||||
atexit.register(dist.destroy_process_group)
|
||||
cpu_group = coord.cpu_group
|
||||
assert isinstance(cpu_group, dist.ProcessGroup)
|
||||
|
||||
@@ -31,6 +31,7 @@ from sglang.srt.distributed.device_communicators.triton_symm_mem_ag import (
|
||||
all_gather_inner,
|
||||
create_state,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.kernels.utils import multigpu_pytest_main
|
||||
|
||||
@@ -70,6 +71,9 @@ def _init_cpu_group_once() -> dist.ProcessGroup:
|
||||
local_rank=local_rank,
|
||||
backend="nccl",
|
||||
)
|
||||
# The context answers a handle from what was stated on it, not from
|
||||
# this module global, so a rank stood up by hand says so itself.
|
||||
get_parallel().override_permanently(world_group=ps._WORLD)
|
||||
atexit.register(dist.destroy_process_group)
|
||||
logging.disable(logging.INFO)
|
||||
torch.cuda.set_stream(torch.cuda.Stream())
|
||||
|
||||
@@ -24,6 +24,7 @@ from sglang.kernels.ops.communication.mp import register_comm_cleanup
|
||||
from sglang.srt.distributed.device_communicators.custom_all_reduce_v2 import (
|
||||
CustomAllReduceV2,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.kernels.utils import multigpu_pytest_main
|
||||
|
||||
@@ -92,6 +93,9 @@ def _init_cpu_group_once() -> dist.ProcessGroup:
|
||||
local_rank=local_rank,
|
||||
backend="nccl",
|
||||
)
|
||||
# The context answers a handle from what was stated on it, not from
|
||||
# this module global, so a rank stood up by hand says so itself.
|
||||
get_parallel().override_permanently(world_group=coord)
|
||||
atexit.register(dist.destroy_process_group)
|
||||
cpu_group = coord.cpu_group
|
||||
assert isinstance(cpu_group, dist.ProcessGroup)
|
||||
|
||||
@@ -30,6 +30,7 @@ from sglang.kernels.ops.kimi_k3 import all_reduce
|
||||
from sglang.srt.distributed.device_communicators.custom_all_reduce_v2 import (
|
||||
CustomAllReduceV2,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.kernels.utils import multigpu_pytest_main
|
||||
|
||||
@@ -65,6 +66,9 @@ def _init_world():
|
||||
local_rank=local_rank,
|
||||
backend="nccl",
|
||||
)
|
||||
# The context answers a handle from what was stated on it, not from
|
||||
# this module global, so a rank stood up by hand says so itself.
|
||||
get_parallel().override_permanently(world_group=coord)
|
||||
atexit.register(dist.destroy_process_group)
|
||||
logging.disable(logging.INFO)
|
||||
torch.cuda.set_stream(torch.cuda.Stream())
|
||||
|
||||
@@ -20,6 +20,7 @@ from sglang.kernels.ops.kimi_k3 import (
|
||||
from sglang.srt.distributed.device_communicators.custom_all_reduce_v2 import (
|
||||
CustomAllReduceV2,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.kernels.utils import multigpu_pytest_main
|
||||
|
||||
@@ -53,6 +54,9 @@ def _init_world():
|
||||
local_rank=local_rank,
|
||||
backend="nccl",
|
||||
)
|
||||
# The context answers a handle from what was stated on it, not from
|
||||
# this module global, so a rank stood up by hand says so itself.
|
||||
get_parallel().override_permanently(world_group=coord)
|
||||
atexit.register(dist.destroy_process_group)
|
||||
cpu_group = coord.cpu_group
|
||||
assert isinstance(cpu_group, dist.ProcessGroup)
|
||||
|
||||
@@ -48,6 +48,7 @@ from sglang.srt.mem_cache.multimodal_cache import (
|
||||
EmbeddingResult,
|
||||
MultiModalStaticCache,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils.common import safe_pickle_loads
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
@@ -1011,10 +1012,7 @@ class TestEncoderDelivery(CustomTestCase):
|
||||
statuses[1].copy_(torch.tensor([400, 1, 0, 0]))
|
||||
|
||||
with (
|
||||
patch(
|
||||
"sglang.srt.distributed.parallel_state.get_tp_group",
|
||||
return_value=TPGroup(),
|
||||
),
|
||||
get_parallel().override(tp_group=TPGroup()),
|
||||
patch(
|
||||
"sglang.srt.disaggregation.encoder.server.torch.distributed.all_gather",
|
||||
side_effect=all_gather,
|
||||
@@ -1051,10 +1049,7 @@ class TestEncoderDelivery(CustomTestCase):
|
||||
statuses[1][2] += 1
|
||||
|
||||
with (
|
||||
patch(
|
||||
"sglang.srt.distributed.parallel_state.get_tp_group",
|
||||
return_value=TPGroup(),
|
||||
),
|
||||
get_parallel().override(tp_group=TPGroup()),
|
||||
patch(
|
||||
"sglang.srt.disaggregation.encoder.server.torch.distributed.all_gather",
|
||||
side_effect=all_gather,
|
||||
@@ -1095,10 +1090,7 @@ class TestEncoderDelivery(CustomTestCase):
|
||||
statuses[1].copy_(local_status)
|
||||
|
||||
with (
|
||||
patch(
|
||||
"sglang.srt.distributed.parallel_state.get_tp_group",
|
||||
return_value=TPGroup(),
|
||||
),
|
||||
get_parallel().override(tp_group=TPGroup()),
|
||||
patch(
|
||||
"sglang.srt.disaggregation.encoder.server.torch.distributed.all_gather",
|
||||
side_effect=all_gather,
|
||||
|
||||
@@ -9,7 +9,7 @@ import unittest
|
||||
from unittest.mock import MagicMock, call, patch
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.runtime_context import get_context
|
||||
from sglang.srt.runtime_context import get_context, get_parallel
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
|
||||
@@ -201,12 +201,12 @@ class TestRegisterToBootstrap(CustomTestCase):
|
||||
self.assertIn("10.0.0.1", url_used)
|
||||
|
||||
@patch("sglang.srt.disaggregation.common.conn.requests.put")
|
||||
# The consumer reads the group through `get_parallel()`, which reads
|
||||
# through to the canonical getter, so that is where the stub belongs.
|
||||
@patch("sglang.srt.distributed.parallel_state.get_world_group")
|
||||
def test_rust_attention_dp_replicates_complete_topology_across_hosts(
|
||||
self, mock_world_group, mock_put
|
||||
self, mock_put
|
||||
):
|
||||
# The consumer reads the group through `get_parallel()`, so the
|
||||
# stub is stated there rather than in the module the build writes.
|
||||
mock_world_group = MagicMock()
|
||||
success_resp = MagicMock()
|
||||
success_resp.status_code = 200
|
||||
mock_put.return_value = success_resp
|
||||
@@ -230,9 +230,12 @@ class TestRegisterToBootstrap(CustomTestCase):
|
||||
for dp_rank, tp_rank, host, rank_port, _ in schedulers
|
||||
]
|
||||
|
||||
mock_world_group.return_value.all_gather_object.side_effect = gather_topology
|
||||
mock_world_group.all_gather_object.side_effect = gather_topology
|
||||
|
||||
with envs.SGLANG_RUST_SERVER.override(True):
|
||||
with (
|
||||
get_parallel().override(world_group=mock_world_group),
|
||||
envs.SGLANG_RUST_SERVER.override(True),
|
||||
):
|
||||
for dp_rank, tp_rank, local_ip, _, rust_http_port in schedulers:
|
||||
manager = self._make_manager()
|
||||
manager.attn_dp_size = 2
|
||||
@@ -277,7 +280,7 @@ class TestRegisterToBootstrap(CustomTestCase):
|
||||
gather_call.args[0]["attn_dp_rank"],
|
||||
gather_call.args[0]["attn_tp_rank"],
|
||||
)
|
||||
for gather_call in mock_world_group.return_value.all_gather_object.call_args_list
|
||||
for gather_call in mock_world_group.all_gather_object.call_args_list
|
||||
],
|
||||
[(dp, tp) for dp, tp, _, _, _ in schedulers],
|
||||
)
|
||||
|
||||
@@ -93,6 +93,9 @@ def _make_prefill_aware_swa_runner(
|
||||
page_size=1,
|
||||
attn_cp_size=1,
|
||||
tp_size=1,
|
||||
# The backend still reads the runner's frozen record for these two;
|
||||
# same single-rank placement, stated where it looks for it.
|
||||
ps=SimpleNamespace(attn_cp_size=1, tp_size=1),
|
||||
is_draft_worker=False,
|
||||
server_args=server_args,
|
||||
attention_chunk_size=None,
|
||||
|
||||
@@ -14,6 +14,7 @@ from sglang.test.test_utils import CustomTestCase, maybe_stub_sgl_kernel
|
||||
maybe_stub_sgl_kernel()
|
||||
|
||||
from sglang.srt.layers.moe import mega_moe
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
register_cpu_ci(est_time=8, suite="base-a-test-cpu")
|
||||
|
||||
@@ -271,11 +272,8 @@ class TestDeepGemmMegaMoeApi(CustomTestCase):
|
||||
"init_new",
|
||||
return_value=object(),
|
||||
),
|
||||
patch(
|
||||
"sglang.srt.runtime_context.get_parallel",
|
||||
return_value=SimpleNamespace(
|
||||
moe_ep_group=SimpleNamespace(device_group=object())
|
||||
),
|
||||
get_parallel().override(
|
||||
moe_ep_group=SimpleNamespace(device_group=object())
|
||||
),
|
||||
):
|
||||
mega_moe._run_mega_routed(
|
||||
|
||||
@@ -11,6 +11,7 @@ from unittest.mock import Mock, patch
|
||||
|
||||
from parameterized import parameterized
|
||||
|
||||
from sglang.srt.distributed import parallel_state
|
||||
from sglang.srt.managers import scheduler as scheduler_module
|
||||
from sglang.srt.managers.schedule_batch import ScheduleBatch
|
||||
from sglang.srt.managers.scheduler import Scheduler
|
||||
@@ -168,12 +169,21 @@ class TestSchedulerIdleStepCounters(CustomTestCase):
|
||||
)
|
||||
with (
|
||||
patch(f"{PDMUX_MODULE}.get_current_stream_idx", return_value=0),
|
||||
patch(f"{PDMUX_MODULE}.set_pdmux_status"),
|
||||
patch(f"{PDMUX_MODULE}.torch.cuda.empty_cache"),
|
||||
patch(
|
||||
f"{PDMUX_MODULE}.torch.cuda.stream",
|
||||
side_effect=lambda stream: nullcontext(),
|
||||
),
|
||||
# The prefill section runs under the duplicate communicator
|
||||
# `--enable-pdmux` builds, in place of the module flag this
|
||||
# replaces. The loop has no process groups at all, so stand
|
||||
# one in: the scope refuses to open without it rather than
|
||||
# letting prefill quietly share the decode communicator.
|
||||
patch.object(
|
||||
parallel_state,
|
||||
"_PDMUX_PREFILL_TP_GROUP",
|
||||
SimpleNamespace(world_size=1, rank_in_group=0),
|
||||
),
|
||||
):
|
||||
self.run_and_check(
|
||||
scheduler,
|
||||
|
||||
@@ -297,7 +297,9 @@ class TestPPMambaPoolSizing(unittest.TestCase):
|
||||
server_args=SimpleNamespace(),
|
||||
spec_algorithm=SimpleNamespace(is_none=lambda: True),
|
||||
layer_info=SimpleNamespace(start_layer=start, end_layer=end),
|
||||
ps=SimpleNamespace(attn_dp_size=1, pp_size=pp_size),
|
||||
# The runner carries its placement as plain attributes.
|
||||
attn_dp_size=1,
|
||||
pp_size=pp_size,
|
||||
hybrid_gdn_config=None,
|
||||
model_config=SimpleNamespace(
|
||||
hf_config=SimpleNamespace(), num_hidden_layers=cls.TOTAL_LAYERS
|
||||
|
||||
@@ -304,16 +304,13 @@ class TestHostMemoryBudget(CustomTestCase):
|
||||
def test_ranks_per_host_divides_world_size_by_nodes(self):
|
||||
# The launcher slices ranks uniformly across nodes, so the co-located
|
||||
# rank count is world_size // nnodes — no hostname collective.
|
||||
fake_group = unittest.mock.Mock(world_size=16)
|
||||
# tp_size=16 states the launch width the count divides -- the
|
||||
# published configuration is where ranks_per_host reads it from.
|
||||
with (
|
||||
get_context().override_server_args(nnodes=2),
|
||||
get_context().override_server_args(nnodes=2, tp_size=16),
|
||||
unittest.mock.patch.object(
|
||||
torch.distributed, "is_initialized", return_value=True
|
||||
),
|
||||
unittest.mock.patch(
|
||||
"sglang.srt.distributed.parallel_state.get_world_group",
|
||||
return_value=fake_group,
|
||||
),
|
||||
):
|
||||
self.assertEqual(base.ranks_per_host(), 8)
|
||||
|
||||
|
||||
@@ -248,7 +248,7 @@ class TestUnifiedMLATokenToKVPool(unittest.TestCase):
|
||||
] = float(layer + 1)
|
||||
|
||||
with (
|
||||
get_parallel().override(dcp_enabled=False),
|
||||
get_parallel().override(dcp_enabled=False, attn_dcp_rank=0),
|
||||
mock.patch(
|
||||
"sglang.srt.mem_cache.memory_pool.current_platform.synchronize"
|
||||
),
|
||||
@@ -411,7 +411,9 @@ class TestMambaAllocatorCpuCopyIsPhysical(unittest.TestCase):
|
||||
_FakeKVCache(pool.max_slots("full")),
|
||||
_FakeKVCache(pool.max_slots("mamba")),
|
||||
)
|
||||
with get_parallel().override(dcp_enabled=False, attn_dcp_size=1):
|
||||
with get_parallel().override(
|
||||
dcp_enabled=False, attn_dcp_size=1, attn_dcp_rank=0
|
||||
):
|
||||
allocator = UnifiedMambaTokenToKVPoolAllocator(
|
||||
unified_buffer=pool, kvcache=kvcache, device=_DEV, page_size=ps
|
||||
)
|
||||
|
||||
@@ -18,6 +18,7 @@ from sglang.srt.model_executor.runner_backend.full_cuda_graph_backend import (
|
||||
FullCudaGraphBackend,
|
||||
)
|
||||
from sglang.srt.model_executor.runner_utils import pool
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.speculative import dflash_utils, dflash_worker_v2, eagle_utils
|
||||
from sglang.srt.speculative.dflash_worker_v2 import DFlashWorkerV2
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
@@ -222,12 +223,7 @@ class TestGraphPoolBorrow(CustomTestCase):
|
||||
"sglang.srt.layers.dp_attention.is_dp_attention_enabled",
|
||||
return_value=False,
|
||||
),
|
||||
# `parallel_state`, not the package re-export: a stub on the
|
||||
# re-export is never consulted.
|
||||
patch(
|
||||
"sglang.srt.distributed.parallel_state.get_tp_group",
|
||||
return_value=tp_group,
|
||||
),
|
||||
get_parallel().override(tp_group=tp_group),
|
||||
patch(
|
||||
"sglang.kernels.ops.speculative.sampling.tree_speculative_sampling_target_only",
|
||||
side_effect=fake_sampling,
|
||||
|
||||
@@ -11,7 +11,7 @@ from sglang.srt.model_executor.cuda_graph_config import (
|
||||
)
|
||||
from sglang.srt.model_executor.model_runner_components import kv_pool_runtime
|
||||
from sglang.srt.model_executor.pool_configurator import MemoryPoolConfig
|
||||
from sglang.srt.runtime_context import get_context
|
||||
from sglang.srt.runtime_context import get_context, get_parallel
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
@@ -60,9 +60,8 @@ class TestCanaryHeadroom(CustomTestCase):
|
||||
),
|
||||
),
|
||||
patch.object(kv_pool_runtime.torch.cuda, "synchronize"),
|
||||
patch(
|
||||
"sglang.srt.distributed.parallel_state.get_world_group",
|
||||
return_value=SimpleNamespace(world_size=1, cpu_group=None),
|
||||
get_parallel().override(
|
||||
world_group=SimpleNamespace(world_size=1, cpu_group=None)
|
||||
),
|
||||
patch.object(kv_pool_runtime, "get_available_gpu_memory", return_value=20),
|
||||
patch.object(kv_pool_runtime, "mambaish_config", return_value=None),
|
||||
|
||||
@@ -36,7 +36,11 @@ def mock_cpu_env(kv_size=2, tp_size=1, swa_eviction_interval=4):
|
||||
|
||||
with (
|
||||
patch("torch._utils._element_size", return_value=kv_size),
|
||||
get_parallel().override(attn_tp_size=tp_size),
|
||||
# A width is a whole topology: state the TP siblings the identities
|
||||
# relate it to, not the attention share alone.
|
||||
get_parallel().override(
|
||||
tp_size=tp_size, attn_tp_size=tp_size, moe_tp_size=tp_size
|
||||
),
|
||||
envs.SGLANG_SWA_EVICTION_INTERVAL.override(swa_eviction_interval),
|
||||
):
|
||||
yield
|
||||
|
||||
@@ -53,9 +53,7 @@ class TestModelOptExport(unittest.TestCase):
|
||||
self.mock_logger.start()
|
||||
|
||||
# Mock all distributed functions that might be called
|
||||
self.mock_get_tp_group = patch(
|
||||
"sglang.srt.distributed.parallel_state.get_tp_group"
|
||||
)
|
||||
self.mock_get_tp_group = patch("sglang.srt.distributed.parallel_state._TP")
|
||||
self.mock_get_tp_group.start()
|
||||
|
||||
# Mock model parallel initialization check
|
||||
|
||||
@@ -82,9 +82,7 @@ class TestModelOptModelLoader(CustomTestCase):
|
||||
self.mock_logger.start()
|
||||
|
||||
# Mock all distributed functions that might be called
|
||||
self.mock_get_tp_group = patch(
|
||||
"sglang.srt.distributed.parallel_state.get_tp_group"
|
||||
)
|
||||
self.mock_get_tp_group = patch("sglang.srt.distributed.parallel_state._TP")
|
||||
self.mock_get_tp_group.start()
|
||||
|
||||
# Mock model parallel initialization check
|
||||
|
||||
@@ -25,6 +25,7 @@ from sglang.srt.model_loader.weight_utils import (
|
||||
fastsafetensors_weights_iterator,
|
||||
safetensors_weights_iterator,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
@@ -237,10 +238,7 @@ class TestPrefetchCheckpoints(CustomTestCase):
|
||||
patch("threading.Thread", _InlineThread),
|
||||
patch("concurrent.futures.ThreadPoolExecutor", _InlineExecutor),
|
||||
patch("concurrent.futures.wait", side_effect=_wait_all),
|
||||
patch(
|
||||
"sglang.srt.distributed.parallel_state.get_world_group",
|
||||
return_value=FakeWorldGroup(),
|
||||
),
|
||||
get_parallel().override(world_group=FakeWorldGroup()),
|
||||
patch(
|
||||
"sglang.srt.model_loader.weight_utils._prefetch_checkpoint_file",
|
||||
side_effect=lambda path, cancel_event: loaded_paths.append(path),
|
||||
|
||||
@@ -652,10 +652,7 @@ class TestStructuralSignature(unittest.TestCase):
|
||||
# must see the same gathered list and thus the same aggregate.
|
||||
fake_group.all_gather_object.side_effect = lambda local: ["sig-pp0", "sig-pp1"]
|
||||
|
||||
with mock.patch(
|
||||
"sglang.srt.distributed.parallel_state.get_world_group",
|
||||
return_value=fake_group,
|
||||
):
|
||||
with get_parallel().override(world_group=fake_group):
|
||||
agg_from_rank0 = (
|
||||
PreshardedModelLoader._make_rank_invariant_structural_signature(
|
||||
"sig-pp0"
|
||||
@@ -673,10 +670,7 @@ class TestStructuralSignature(unittest.TestCase):
|
||||
"sig-pp0",
|
||||
"sig-pp1-changed",
|
||||
]
|
||||
with mock.patch(
|
||||
"sglang.srt.distributed.parallel_state.get_world_group",
|
||||
return_value=fake_group,
|
||||
):
|
||||
with get_parallel().override(world_group=fake_group):
|
||||
agg_changed = (
|
||||
PreshardedModelLoader._make_rank_invariant_structural_signature(
|
||||
"sig-pp0"
|
||||
|
||||
@@ -5,6 +5,7 @@ from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
from sglang.srt.models.transformers import TransformersBase
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
@@ -54,10 +55,9 @@ class TestTransformersFallbackSkipSubstrs(CustomTestCase):
|
||||
pass
|
||||
|
||||
with (
|
||||
patch(
|
||||
"sglang.srt.distributed.parallel_state.get_pp_group",
|
||||
return_value=SimpleNamespace(),
|
||||
),
|
||||
# `__init__` only stashes the pipeline group, so an empty
|
||||
# stand-in carries it past the read.
|
||||
get_parallel().override(pp_group=SimpleNamespace()),
|
||||
patch(
|
||||
"sglang.srt.models.transformers.get_hf_text_config",
|
||||
return_value=SimpleNamespace(),
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import sys
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -13,6 +12,7 @@ from sglang.srt.multimodal.internvl_vit_cuda_graph_runner import (
|
||||
InternViTCudaGraphRunner,
|
||||
)
|
||||
from sglang.srt.multimodal.vit_cuda_graph_runner import ViTCudaGraphRunner
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
|
||||
class _Block:
|
||||
@@ -32,12 +32,10 @@ def _runner(*, use_data_parallel: bool) -> ViTCudaGraphRunner:
|
||||
|
||||
def test_dp_vit_graph_capture_does_not_enter_tp_communication_capture():
|
||||
runner = _runner(use_data_parallel=True)
|
||||
with patch(
|
||||
"sglang.srt.distributed.parallel_state.get_tp_group",
|
||||
side_effect=AssertionError("DP capture must be rank-local"),
|
||||
):
|
||||
with runner._capture_context():
|
||||
pass
|
||||
# No tp_group is stated, so reading one would raise: the DP path must not
|
||||
# ask for the TP group at all.
|
||||
with runner._capture_context():
|
||||
pass
|
||||
|
||||
|
||||
def test_non_dp_vit_graph_capture_uses_tp_communication_capture():
|
||||
@@ -52,9 +50,7 @@ def test_non_dp_vit_graph_capture_uses_tp_communication_capture():
|
||||
|
||||
group = SimpleNamespace(ca_comm=SimpleNamespace(capture=lambda: Capture()))
|
||||
runner = _runner(use_data_parallel=False)
|
||||
with patch(
|
||||
"sglang.srt.distributed.parallel_state.get_tp_group", return_value=group
|
||||
):
|
||||
with get_parallel().override(tp_group=group):
|
||||
with runner._capture_context():
|
||||
pass
|
||||
assert entered == [True]
|
||||
|
||||
@@ -132,18 +132,18 @@ def _stash_overlay(server_args):
|
||||
|
||||
|
||||
def _live_topology_leaves():
|
||||
"""Names `ParallelContext` serves from the live topology, not the config.
|
||||
"""Names `ParallelContext` answers from a runtime write, not the config.
|
||||
|
||||
Read out of `_LIVE_READS`, which is where those names are declared.
|
||||
Inferring them from "did the read raise" is wrong -- it only raises while
|
||||
the process groups are missing, so in a process where an earlier test built
|
||||
them the property answers the *live* size and a leaf check reads it as a
|
||||
config mismatch (`parallel.tp_size: bag=1 resolution=2`). Whether a name is
|
||||
Read off the declarations that carry no `fn`, which is what those are.
|
||||
Inferring them from "did the read raise" is wrong -- it raises only while
|
||||
nothing has written the name, so in a process where an earlier test stated
|
||||
one the property answers that value and a leaf check reads it as a config
|
||||
mismatch (`parallel.tp_size: bag=1 resolution=2`). Whether a name is
|
||||
shadowed is a property of the declaration, not of the process.
|
||||
"""
|
||||
from sglang.srt.runtime_context import _LIVE_READS
|
||||
from sglang.srt.runtime_context import _derived_widths
|
||||
|
||||
return frozenset(_LIVE_READS)
|
||||
return frozenset(n for n, d in _derived_widths().items() if not d.fn)
|
||||
|
||||
|
||||
class TestResolutionDeclarations(CustomTestCase):
|
||||
|
||||
@@ -249,6 +249,8 @@ def test_worker_folds_a_gate_admitted_quantized_selector_head(monkeypatch):
|
||||
block_size=8,
|
||||
selector=object(),
|
||||
model_runner=SimpleNamespace(tp_rank=0),
|
||||
# The worker rank-gates its logging on its own frozen record.
|
||||
ps=SimpleNamespace(tp_rank=0),
|
||||
draft_model=SimpleNamespace(lm_head=None),
|
||||
device="cpu",
|
||||
_selector_sampling_enabled=True,
|
||||
@@ -283,6 +285,8 @@ def test_worker_warns_once_when_selector_sampling_is_disabled(monkeypatch):
|
||||
_selector_sampling_enabled=False,
|
||||
_warned_sampling_fallback=False,
|
||||
model_runner=SimpleNamespace(tp_rank=0),
|
||||
# The worker rank-gates its logging on its own frozen record.
|
||||
ps=SimpleNamespace(tp_rank=0),
|
||||
)
|
||||
batch = SimpleNamespace(sampling_info=SimpleNamespace(is_all_greedy=False))
|
||||
|
||||
|
||||
@@ -10,8 +10,10 @@ import os
|
||||
import pathlib as _pathlib
|
||||
import shutil
|
||||
import tempfile
|
||||
import types
|
||||
import unittest
|
||||
import warnings
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import msgspec
|
||||
@@ -41,7 +43,6 @@ from sglang.srt.runtime_context import (
|
||||
RuntimeContext,
|
||||
SpawnRanks,
|
||||
_FlagGroupBase,
|
||||
_validate_parallel,
|
||||
assert_published,
|
||||
derive_parallel_widths,
|
||||
get_context,
|
||||
@@ -111,39 +112,58 @@ def _scope_entries_that_say_nothing(paths):
|
||||
|
||||
|
||||
_PS = "sglang.srt.distributed.parallel_state"
|
||||
|
||||
|
||||
def _parallel_state():
|
||||
from sglang.srt.distributed import parallel_state
|
||||
|
||||
return parallel_state
|
||||
|
||||
|
||||
_DP = "sglang.srt.layers.dp_attention"
|
||||
|
||||
# Ranks and the launch width are asked of the group: they are not implied by
|
||||
# anything, so there is nothing to derive them from. The quotients are not
|
||||
# here -- `attn_tp_size` and its siblings are functions of the configured
|
||||
# leaves, and `TestDerivedWidths` pins them. `attn_dp_rank` is not here either: no group coordinator
|
||||
# knows it, so it is stamped when the attention topology is initialized and
|
||||
# `TestStampedRanks` is what pins it. The other world width is not here
|
||||
# because the group does not know it; `TestTheTwoWorldWidths` pins it.
|
||||
SIZE_RANK_DELEGATIONS = [
|
||||
("launch_world_size", f"{_PS}.get_world_size"),
|
||||
("launch_world_rank", f"{_PS}.get_world_rank"),
|
||||
("tp_rank", f"{_PS}.get_tensor_model_parallel_rank"),
|
||||
("dcp_rank", f"{_PS}.get_dcp_rank"),
|
||||
("pp_rank", f"{_PS}.get_pipeline_model_parallel_rank"),
|
||||
("moe_ep_rank", f"{_PS}.get_moe_expert_parallel_rank"),
|
||||
("moe_dp_rank", f"{_PS}.get_moe_data_parallel_rank"),
|
||||
("moe_tp_rank", f"{_PS}.get_moe_tensor_parallel_rank"),
|
||||
("attn_tp_rank", f"{_PS}.get_attn_tensor_model_parallel_rank"),
|
||||
("attn_cp_rank", f"{_PS}.get_attn_context_model_parallel_rank"),
|
||||
]
|
||||
#: The groups `initialize_model_parallel` states on the context, by the module
|
||||
#: global it builds each one into. WORLD is not among them: it is built and
|
||||
#: stated by `init_distributed_environment`, one call earlier.
|
||||
GROUP_STAMPS = {
|
||||
"tp_group": "_TP",
|
||||
"dcp_group": "_DCP",
|
||||
"pp_group": "_PP",
|
||||
"moe_ep_group": "_MOE_EP",
|
||||
"moe_dp_group": "_MOE_DP",
|
||||
"moe_tp_group": "_MOE_TP",
|
||||
"attn_tp_group": "_ATTN_TP",
|
||||
"attn_cp_group": "_ATTN_CP",
|
||||
"shared_experts_tp_group": "_SHARED_EXPERTS_TP",
|
||||
}
|
||||
|
||||
GROUP_DELEGATIONS = [
|
||||
("world_group", f"{_PS}.get_world_group"),
|
||||
("tp_group", f"{_PS}.get_tp_group"),
|
||||
("dcp_group", f"{_PS}.get_dcp_group"),
|
||||
("pp_group", f"{_PS}.get_pp_group"),
|
||||
("moe_ep_group", f"{_PS}.get_moe_ep_group"),
|
||||
("moe_dp_group", f"{_PS}.get_moe_dp_group"),
|
||||
("moe_tp_group", f"{_PS}.get_moe_tp_group"),
|
||||
("attn_tp_group", f"{_PS}.get_attn_tp_group"),
|
||||
("attn_cp_group", f"{_PS}.get_attn_cp_group"),
|
||||
]
|
||||
|
||||
def _groups_the_build_states() -> dict:
|
||||
"""What `initialize_model_parallel` hands the context, read out of its
|
||||
source: `{context name: the module global it passes}`.
|
||||
|
||||
Out of the source because that is the only place the whole set appears at
|
||||
once -- calling the function needs ten live process groups.
|
||||
"""
|
||||
import ast
|
||||
import inspect
|
||||
import textwrap
|
||||
|
||||
from sglang.srt.distributed import parallel_state
|
||||
|
||||
body = textwrap.dedent(inspect.getsource(parallel_state.initialize_model_parallel))
|
||||
for node in ast.walk(ast.parse(body)):
|
||||
keys = getattr(node, "keys", None)
|
||||
if (
|
||||
isinstance(node, ast.Dict)
|
||||
and keys
|
||||
and all(
|
||||
isinstance(k, ast.Constant) and str(k.value).endswith("_group")
|
||||
for k in keys
|
||||
)
|
||||
):
|
||||
return {k.value: v.id for k, v in zip(node.keys, node.values)}
|
||||
raise AssertionError("initialize_model_parallel states no group at all")
|
||||
|
||||
|
||||
class TestRuntimeContextSingletons(CustomTestCase):
|
||||
@@ -171,29 +191,91 @@ class _IsolatedOverrides(CustomTestCase):
|
||||
super().tearDown()
|
||||
|
||||
|
||||
class TestTheBuildStatesEveryGroup(_IsolatedOverrides):
|
||||
"""Nothing derives a group, so one the build forgets to state is a name
|
||||
that answers "not written" for the rest of the process -- and the reader
|
||||
that finds out is a model layer, a long way from here."""
|
||||
|
||||
def test_the_build_states_every_group_the_context_declares(self):
|
||||
from sglang.srt.runtime_context import _parallel_fields
|
||||
|
||||
declared = {name for name in _parallel_fields() if name.endswith("_group")}
|
||||
self.assertEqual(declared, set(GROUP_STAMPS) | {"world_group"})
|
||||
|
||||
def test_the_world_group_is_stated_where_it_is_built(self):
|
||||
"""`initialize_model_parallel` places every group it builds by reading
|
||||
`get_world_group().local_rank`, so WORLD has to be answerable before it
|
||||
runs -- one function earlier, where it is constructed."""
|
||||
import ast
|
||||
import inspect
|
||||
import textwrap
|
||||
|
||||
from sglang.srt.distributed import parallel_state
|
||||
|
||||
body = textwrap.dedent(
|
||||
inspect.getsource(parallel_state.init_distributed_environment)
|
||||
)
|
||||
stated = {
|
||||
kw.arg
|
||||
for node in ast.walk(ast.parse(body))
|
||||
if isinstance(node, ast.Call)
|
||||
and isinstance(node.func, ast.Attribute)
|
||||
and node.func.attr == "override_permanently"
|
||||
for kw in node.keywords
|
||||
}
|
||||
self.assertIn("world_group", stated)
|
||||
|
||||
def test_nothing_reads_a_name_this_build_has_not_stated_yet(self):
|
||||
"""The stamp is at the end, so a getter called before it answers a name
|
||||
nothing has written -- a crash at startup, in a process no unit test
|
||||
runs. What the function may read is what a *previous* call stated, and
|
||||
that is WORLD alone: a rank belongs to the spawn, and this build does
|
||||
not get to assume the spawn ran first."""
|
||||
import ast
|
||||
import inspect
|
||||
import textwrap
|
||||
|
||||
from sglang.srt.distributed import parallel_state
|
||||
|
||||
body = textwrap.dedent(
|
||||
inspect.getsource(parallel_state.initialize_model_parallel)
|
||||
)
|
||||
read = set()
|
||||
for node in ast.walk(ast.parse(body)):
|
||||
if isinstance(node, ast.Call) and isinstance(node.func, ast.Name):
|
||||
name = parallel_state._CONTEXT_NAME_OF.get(node.func.id)
|
||||
if name is not None:
|
||||
read.add(name)
|
||||
self.assertTrue(read, "no getter is called here; this proves nothing")
|
||||
self.assertEqual(read - {"world_group"}, set())
|
||||
|
||||
def test_each_group_is_stated_from_the_global_it_was_built_into(self):
|
||||
self.assertEqual(_groups_the_build_states(), GROUP_STAMPS)
|
||||
|
||||
def test_a_dimension_the_configuration_has_not_got_is_left_unstated(self):
|
||||
"""`_DCP` is None without decode context parallelism, and every one of
|
||||
these getters has always refused to answer for a group that was never
|
||||
built rather than handing back a None to fail on at the collective."""
|
||||
import ast
|
||||
import inspect
|
||||
import textwrap
|
||||
|
||||
from sglang.srt.distributed import parallel_state
|
||||
|
||||
body = textwrap.dedent(
|
||||
inspect.getsource(parallel_state.initialize_model_parallel)
|
||||
)
|
||||
stamp = next(
|
||||
node
|
||||
for node in ast.walk(ast.parse(body))
|
||||
if isinstance(node, ast.Call)
|
||||
and isinstance(node.func, ast.Attribute)
|
||||
and node.func.attr == "override_permanently"
|
||||
)
|
||||
self.assertIn("is not None", ast.unparse(stamp))
|
||||
|
||||
|
||||
class TestParallelDelegation(_IsolatedOverrides):
|
||||
def test_size_rank_delegate_to_canonical_getters(self):
|
||||
# Patch each getter to a distinct sentinel: a miswired attribute would read
|
||||
# a different (unpatched) getter and fail.
|
||||
for i, (attr, target) in enumerate(SIZE_RANK_DELEGATIONS):
|
||||
sentinel = 1000 + i
|
||||
with patch(target, return_value=sentinel):
|
||||
self.assertEqual(
|
||||
getattr(get_parallel(), attr),
|
||||
sentinel,
|
||||
msg=f"{attr} must delegate to {target}",
|
||||
)
|
||||
|
||||
def test_groups_delegate_to_canonical_getters(self):
|
||||
for attr, target in GROUP_DELEGATIONS:
|
||||
sentinel = object()
|
||||
with patch(target, return_value=sentinel):
|
||||
self.assertIs(
|
||||
getattr(get_parallel(), attr),
|
||||
sentinel,
|
||||
msg=f"{attr} must delegate to {target}",
|
||||
)
|
||||
|
||||
def test_wrapper_holds_no_resolved_state(self):
|
||||
# __slots__: no __dict__; the only instance state is the override hook.
|
||||
self.assertFalse(hasattr(get_parallel(), "__dict__"))
|
||||
@@ -204,50 +286,46 @@ class TestParallelDelegation(_IsolatedOverrides):
|
||||
|
||||
|
||||
class TestTheTwoWorldWidths(_IsolatedOverrides):
|
||||
"""Two questions about the WORLD group: what it was built at, and what it
|
||||
has room for.
|
||||
"""Two questions about the WORLD group: what it was launched at, and what
|
||||
it has room for.
|
||||
|
||||
Neither is stored here. How much of that room is serving after a scale-up
|
||||
is elastic-EP state, and is asked of the manager that owns it rather than
|
||||
mirrored onto this namespace.
|
||||
Both are arithmetic over the configured leaves and are worked out at
|
||||
publish, so they answer in a process that never builds the group -- which
|
||||
is where a good half of the readers are. How much of that room is serving
|
||||
after a scale-up is neither of them: that is elastic-EP state, asked of the
|
||||
manager that owns it rather than mirrored onto this namespace.
|
||||
"""
|
||||
|
||||
def test_the_launch_width_is_what_the_group_was_built_at(self):
|
||||
with patch(f"{_PS}.get_world_size", return_value=4):
|
||||
self.assertEqual(get_parallel().launch_world_size, 4)
|
||||
def _published(self, **fields):
|
||||
reset_context()
|
||||
self.addCleanup(reset_context)
|
||||
publish(ServerArgs(model_path="dummy", **fields), role="test")
|
||||
return get_parallel()
|
||||
|
||||
def test_the_launch_width_is_a_rank_per_stage_of_each_group(self):
|
||||
self.assertEqual(self._published(tp_size=4, pp_size=2).launch_world_size, 8)
|
||||
|
||||
def test_the_launch_width_spans_the_ranks_a_joiner_came_in_above(self):
|
||||
"""A scale joiner lays its own groups out at `tp * pp`, while its WORLD
|
||||
spans the cohort already running underneath it as well."""
|
||||
parallel = self._published(tp_size=4, pp_size=1, ep_join_rank_offset=8)
|
||||
self.assertEqual(parallel.launch_world_size, 12)
|
||||
|
||||
def test_the_ceiling_is_the_configured_one_when_there_is_one(self):
|
||||
parallel = get_parallel()
|
||||
with (
|
||||
parallel.override(max_ep_size=32),
|
||||
patch(
|
||||
f"{_PS}.get_world_size",
|
||||
side_effect=AssertionError("the built group must not be asked"),
|
||||
),
|
||||
):
|
||||
self.assertEqual(parallel.max_world_size, 32)
|
||||
self.assertEqual(self._published(tp_size=4, max_ep_size=32).max_world_size, 32)
|
||||
|
||||
def test_without_a_configured_ceiling_the_room_is_the_launch_width(self):
|
||||
parallel = get_parallel()
|
||||
with (
|
||||
parallel.override(max_ep_size=None),
|
||||
patch(f"{_PS}.get_world_size", return_value=8),
|
||||
):
|
||||
self.assertEqual(parallel.max_world_size, 8)
|
||||
parallel = self._published(tp_size=8)
|
||||
self.assertEqual(parallel.launch_world_size, 8)
|
||||
self.assertEqual(parallel.max_world_size, 8)
|
||||
|
||||
def test_each_width_can_be_stated_on_its_own(self):
|
||||
"""Stating one must not answer for the other: they are two names."""
|
||||
parallel = get_parallel()
|
||||
with (
|
||||
parallel.override(launch_world_size=2, max_ep_size=None),
|
||||
patch(
|
||||
f"{_PS}.get_world_size",
|
||||
side_effect=AssertionError("the built group must not be asked"),
|
||||
),
|
||||
):
|
||||
parallel = self._published(tp_size=8)
|
||||
with parallel.override(launch_world_size=2):
|
||||
self.assertEqual(parallel.launch_world_size, 2)
|
||||
self.assertEqual(parallel.max_world_size, 2)
|
||||
with parallel.override(max_ep_size=6):
|
||||
self.assertEqual(parallel.max_world_size, 8)
|
||||
with parallel.override(max_world_size=6):
|
||||
self.assertEqual(parallel.max_world_size, 6)
|
||||
self.assertEqual(parallel.launch_world_size, 2)
|
||||
|
||||
@@ -380,18 +458,20 @@ class TestAttentionRanksComeFromPublish(_IsolatedOverrides):
|
||||
role="test",
|
||||
ranks=SpawnRanks(world_rank=5),
|
||||
)
|
||||
with patch(
|
||||
f"{_PS}.get_attn_tensor_model_parallel_rank",
|
||||
side_effect=AssertionError("no group must be consulted"),
|
||||
):
|
||||
with patch.object(_parallel_state(), "_ATTN_TP", None):
|
||||
self.assertEqual(get_parallel().attn_tp_rank, 1)
|
||||
self.assertEqual(get_parallel().attn_dp_rank, 1)
|
||||
|
||||
def test_without_a_bundle_it_still_asks_the_group(self):
|
||||
"""Unchanged for every process that publishes without a placement."""
|
||||
def test_without_a_bundle_a_rank_read_says_what_is_missing(self):
|
||||
"""There is nothing to fall back to. Deriving one from whatever group
|
||||
happens to be installed would answer a different question -- where this
|
||||
process sits in that group, not where the launcher put it."""
|
||||
publish(ServerArgs(model_path="dummy", tp_size=8), role="test")
|
||||
with patch(f"{_PS}.get_attn_tensor_model_parallel_rank", return_value=3):
|
||||
self.assertEqual(get_parallel().attn_tp_rank, 3)
|
||||
with self.assertRaises(RuntimeError) as caught:
|
||||
get_parallel().attn_tp_rank
|
||||
message = str(caught.exception)
|
||||
self.assertIn("has not been written in this process", message)
|
||||
self.assertIn("override(attn_tp_rank=...)", message)
|
||||
|
||||
|
||||
class TestStampedRanks(_IsolatedOverrides):
|
||||
@@ -568,16 +648,14 @@ class TestEveryDeclaredParallelNameIsStatable(_IsolatedOverrides):
|
||||
|
||||
The bag carries the declared quotients as well as the operator's
|
||||
leaves, and both are ahead of the live getter once a configuration is
|
||||
published: a name in `_LIVE_READS` and in either of them would answer
|
||||
published: a declared name that is also a leaf would answer
|
||||
from the getter before publish and from the bag after."""
|
||||
from sglang.srt.runtime_context import (
|
||||
_LIVE_READS,
|
||||
_derived_widths,
|
||||
_parallel_config_leaves,
|
||||
)
|
||||
|
||||
self.assertEqual(set(_LIVE_READS) & _parallel_config_leaves(), set())
|
||||
self.assertEqual(set(_LIVE_READS) & set(_derived_widths()), set())
|
||||
self.assertEqual(set(_derived_widths()) & _parallel_config_leaves(), set())
|
||||
|
||||
def test_an_undeclared_name_is_refused(self):
|
||||
with self.assertRaises(ValueError):
|
||||
@@ -741,19 +819,43 @@ class TestParallelDCP(_IsolatedOverrides):
|
||||
self.assertTrue(parallel.dcp_enabled)
|
||||
self.assertEqual(parallel.attn_dcp_size, 8)
|
||||
|
||||
def test_the_dcp_rank_still_reads_the_group(self):
|
||||
"""A rank is not implied by the configuration, so it reads the group --
|
||||
gated on a width that is."""
|
||||
with (
|
||||
get_parallel().override(tp_size=8, dcp_size=8, dcp_enabled=False),
|
||||
patch(f"{_PS}.get_dcp_rank", side_effect=AssertionError),
|
||||
):
|
||||
self.assertEqual(get_parallel().attn_dcp_rank, 0)
|
||||
with (
|
||||
get_parallel().override(tp_size=8, dcp_size=8, dcp_enabled=True),
|
||||
patch(f"{_PS}.get_dcp_rank", return_value=3),
|
||||
):
|
||||
self.assertEqual(get_parallel().attn_dcp_rank, 3)
|
||||
def _placed(self, world_rank, **fields):
|
||||
reset_context()
|
||||
self.addCleanup(reset_context)
|
||||
publish(
|
||||
ServerArgs(model_path="dummy", **fields),
|
||||
role="test",
|
||||
ranks=SpawnRanks(world_rank=world_rank),
|
||||
)
|
||||
return get_parallel()
|
||||
|
||||
def test_the_dcp_rank_is_where_the_tp_rank_falls_in_its_slice(self):
|
||||
"""A DCP group is a contiguous slice of the TP group, so the place in
|
||||
one is the TP rank folded by the width."""
|
||||
parallel = self._placed(5, tp_size=8, dcp_size=4)
|
||||
self.assertEqual(parallel.dcp_rank, 1)
|
||||
self.assertEqual(parallel.attn_dcp_rank, 1)
|
||||
|
||||
def test_the_gated_off_rank_answers_without_a_spawn_bundle(self):
|
||||
"""Zero for every process when decode context parallelism is off, so a
|
||||
reader on a path that never publishes a bundle -- a memory pool, an
|
||||
attention backend built in a unit test -- still gets an answer. It
|
||||
stood for a group that was never built before, and it has to keep
|
||||
answering the same way."""
|
||||
reset_context()
|
||||
self.addCleanup(reset_context)
|
||||
publish(ServerArgs(model_path="dummy", tp_size=8), role="test")
|
||||
self.assertEqual(get_parallel().attn_dcp_rank, 0)
|
||||
|
||||
def test_the_dcp_rank_is_gated_on_a_width_the_configuration_carries(self):
|
||||
"""Zero where decode context parallelism is off, so a reader does not
|
||||
have to ask whether it is on before asking where it sits -- and the
|
||||
gated-off name is not answered at all, because no group holds it."""
|
||||
parallel = self._placed(5, tp_size=8, dcp_size=1)
|
||||
self.assertFalse(parallel.dcp_enabled)
|
||||
self.assertEqual(parallel.attn_dcp_rank, 0)
|
||||
with self.assertRaises(RuntimeError):
|
||||
parallel.dcp_rank
|
||||
|
||||
def test_the_width_does_not_consult_the_platform(self):
|
||||
with patch("sglang.srt.utils.is_cuda", return_value=False) as is_cuda:
|
||||
@@ -2074,10 +2176,16 @@ class TestDerivedWidths(_IsolatedOverrides):
|
||||
{name for name in widths if "world" in name},
|
||||
set(),
|
||||
)
|
||||
parallel = get_parallel()
|
||||
parallel.override_permanently(attn_tp_size=4)
|
||||
with patch(f"{_PS}.get_world_size", return_value=9):
|
||||
self.assertEqual(parallel.launch_world_size, 9)
|
||||
# Its own arithmetic spans the offset, which is the leaf the quotients
|
||||
# above are not given and could not account for.
|
||||
from sglang.srt.runtime_context import launch_world_size_of
|
||||
|
||||
self.assertEqual(
|
||||
launch_world_size_of(
|
||||
SimpleNamespace(ep_join_rank_offset=8, tp_size=4, pp_size=1)
|
||||
),
|
||||
12,
|
||||
)
|
||||
|
||||
def test_the_bare_name_is_gone(self):
|
||||
"""It answered two questions, so every reader had to remember which.
|
||||
@@ -2315,15 +2423,27 @@ class TestTheDerivedHalfIsDeclared(CustomTestCase):
|
||||
f"{name} is declared but no property was installed",
|
||||
)
|
||||
|
||||
def test_the_declared_set_is_what_derive_parallel_widths_produces(self):
|
||||
"""The declaration is not a second list to keep in step: it names
|
||||
exactly the quotients the derivation returns."""
|
||||
from sglang.srt.arg_groups.arg_utils import Derived
|
||||
from sglang.srt.arg_groups.fields.parallel import Parallel
|
||||
def test_a_computed_name_names_the_function_that_computes_it(self):
|
||||
"""The declaration is not a second list to keep in step: a name that
|
||||
is a function of the leaves points at a function called after it, and
|
||||
that function existing is the whole of what publish needs."""
|
||||
import importlib
|
||||
|
||||
from sglang.srt.runtime_context import _derived_widths
|
||||
|
||||
computed = {n: d.fn for n, d in _derived_widths().items() if d.fn}
|
||||
self.assertTrue(computed, "nothing is computed from the leaves")
|
||||
for name, fn in computed.items():
|
||||
module, _, attr = fn.rpartition(".")
|
||||
self.assertEqual(attr, f"{name}_of", f"{name} is computed by {attr}")
|
||||
self.assertTrue(callable(getattr(importlib.import_module(module), attr)))
|
||||
|
||||
def test_the_arithmetic_produces_nothing_that_is_not_declared(self):
|
||||
"""The other side of it: a key the derivation returns and no
|
||||
declaration names is a width the namespace never answers with, and the
|
||||
group build re-states it into a name nobody can read."""
|
||||
from sglang.srt.runtime_context import _derived_widths
|
||||
|
||||
declared = {
|
||||
name for name, value in vars(Parallel).items() if isinstance(value, Derived)
|
||||
}
|
||||
produced = set(
|
||||
derive_parallel_widths(
|
||||
tp_size=8,
|
||||
@@ -2335,7 +2455,7 @@ class TestTheDerivedHalfIsDeclared(CustomTestCase):
|
||||
dcp_enabled=False,
|
||||
)
|
||||
)
|
||||
self.assertEqual(declared, produced)
|
||||
self.assertEqual(produced - set(_derived_widths()), set())
|
||||
|
||||
def test_a_declared_quotient_is_not_a_record_field(self):
|
||||
"""It has no operator input to preserve, and the record is what crosses
|
||||
@@ -2405,15 +2525,34 @@ class TestTheAccessorsHaveNoCallersOutsideTheirPackage(CustomTestCase):
|
||||
has its own parallel state.
|
||||
"""
|
||||
|
||||
#: Not topology. `get_self_pp_group` builds the single-rank group a draft
|
||||
#: pipeline scope installs, so there is nothing for the context to answer
|
||||
#: with until the scope has installed it.
|
||||
#: May have callers. `get_self_pp_group` builds the single-rank group a
|
||||
#: draft pipeline scope installs, so there is nothing for the context to
|
||||
#: answer with until the scope has installed it; the other two are not
|
||||
#: topology at all.
|
||||
ALLOWED = {
|
||||
"get_self_pp_group",
|
||||
"get_default_distributed_backend",
|
||||
"get_mooncake_transfer_engine",
|
||||
}
|
||||
|
||||
#: Zero callers required, but not deprecated either: the context has no
|
||||
#: name that answers the same question.
|
||||
#:
|
||||
#: The three widths read a group the build does not check against the
|
||||
#: configuration, so "the group's width" and "the configured width" are two
|
||||
#: facts -- the MoE-DP group is the attention-CP group when the latter is
|
||||
#: wider, and the other two are simply not pinned yet. Pinning them in
|
||||
#: `_WIDTH_AND_GROUP` is what would let them move.
|
||||
NOT_ANSWERED_BY_THE_CONTEXT = {
|
||||
"get_moe_data_parallel_world_size",
|
||||
"get_moe_tensor_parallel_world_size",
|
||||
"get_dcp_world_size",
|
||||
# Answers `None` where the context asserts, which is the whole point of
|
||||
# the caller that wants it.
|
||||
"get_dcp_group_no_assert",
|
||||
"get_torch_distributed_pg_options",
|
||||
}
|
||||
|
||||
def _accessors(self):
|
||||
"""Derived from the source, not listed here: a guard whose subject set
|
||||
is written by hand stops watching whatever gets added next."""
|
||||
@@ -2458,6 +2597,139 @@ class TestTheAccessorsHaveNoCallersOutsideTheirPackage(CustomTestCase):
|
||||
"context cannot answer them",
|
||||
)
|
||||
|
||||
#: How many callers each exempt accessor has outside the defining package.
|
||||
#: A ratchet, not a description: these may go down and never up, and a name
|
||||
#: that reaches zero comes off the list. Anything not here must have none.
|
||||
ALLOWED_CALLERS = {
|
||||
"get_self_pp_group": 1,
|
||||
"get_default_distributed_backend": 1,
|
||||
"get_mooncake_transfer_engine": 6,
|
||||
}
|
||||
|
||||
def test_the_exempt_accessors_do_not_grow_new_callers(self):
|
||||
"""The zero-caller rule above cannot cover the three that are not
|
||||
topology, so they get a count instead. Ratchets only turn one way: a
|
||||
number that has to go up means a new business-code reader of a name the
|
||||
context should be answering."""
|
||||
for name, allowed in sorted(self.ALLOWED_CALLERS.items()):
|
||||
callers = self._callers(name)
|
||||
self.assertLessEqual(
|
||||
len(callers),
|
||||
allowed,
|
||||
f"{name} grew a caller: {callers}. Read it through "
|
||||
f"get_parallel() if the context can answer it; if it truly "
|
||||
f"cannot, lower this number only when one goes away.",
|
||||
)
|
||||
|
||||
def test_every_getter_the_context_answers_is_deprecated(self):
|
||||
"""The other half of the ratchet: the deprecation set is derived from
|
||||
the table that maps a context name to the getter behind it, so dropping
|
||||
a getter out of that table would quietly take it off the list. This
|
||||
fails if one of them stops being marked."""
|
||||
from sglang.srt.distributed import parallel_state
|
||||
|
||||
marked = set(parallel_state._CONTEXT_NAME_OF)
|
||||
unclassified = (
|
||||
self._accessors() - self.ALLOWED - self.NOT_ANSWERED_BY_THE_CONTEXT
|
||||
)
|
||||
for name in sorted(unclassified):
|
||||
if name in marked:
|
||||
continue
|
||||
# Not answered by the context and not exempt: a getter that is
|
||||
# neither is a name with no home, which is what this module exists
|
||||
# to prevent.
|
||||
self.assertIn(
|
||||
name,
|
||||
marked,
|
||||
f"{name} is neither deprecated nor listed as exempt -- give it "
|
||||
"a context name or say here why it has none",
|
||||
)
|
||||
|
||||
def test_calling_one_from_outside_the_package_is_deprecated(self):
|
||||
"""The getters stay -- they are the definition -- but a call that comes
|
||||
from outside the package that defines them cannot be redirected by a
|
||||
scope, so it says what to read instead."""
|
||||
import warnings
|
||||
|
||||
from sglang.srt.distributed import parallel_state
|
||||
|
||||
parallel_state._ALREADY_WARNED.discard("get_tensor_model_parallel_rank")
|
||||
self.addCleanup(
|
||||
parallel_state._ALREADY_WARNED.discard, "get_tensor_model_parallel_rank"
|
||||
)
|
||||
with warnings.catch_warnings(record=True) as seen:
|
||||
warnings.simplefilter("always")
|
||||
try:
|
||||
parallel_state.get_tensor_model_parallel_rank()
|
||||
except Exception:
|
||||
pass
|
||||
messages = [str(w.message) for w in seen]
|
||||
self.assertTrue(
|
||||
any("get_parallel().tp_rank" in m for m in messages),
|
||||
f"expected the replacement to be named, got {messages}",
|
||||
)
|
||||
|
||||
def test_nothing_the_context_answers_with_calls_back_into_the_package(self):
|
||||
"""The read path reaches the stored group, not the getter that used to
|
||||
wrap it -- which is what lets the getters be deprecated without the
|
||||
replacement tripping the warning meant for people who bypass it."""
|
||||
|
||||
import inspect
|
||||
|
||||
from sglang.srt.distributed import parallel_state
|
||||
from sglang.srt.runtime_context import ParallelContext, _derived_widths
|
||||
|
||||
written = {n for n, d in _derived_widths().items() if not d.fn}
|
||||
self.assertTrue(written, "no written-at-runtime names; this proves nothing")
|
||||
self.assertIn("tp_group", written)
|
||||
|
||||
# The read path is lookups only -- override, stamp, bag. Nothing in it
|
||||
# can reach a getter, which is what lets them be deprecated without the
|
||||
# replacement tripping the warning meant for people who bypass it.
|
||||
body = inspect.getsource(ParallelContext._read)
|
||||
self.assertNotIn("_ps()", body)
|
||||
self.assertNotIn("parallel_state", body)
|
||||
self.assertNotIn("sglang.srt.runtime_context", parallel_state._EXEMPT_CALLERS)
|
||||
|
||||
def test_a_scope_reaches_callers_that_went_straight_to_the_getter(self):
|
||||
"""The getters read the context, so redirecting a group redirects them
|
||||
too. PD multiplexing needs exactly this: the in-package readers have to
|
||||
follow the prefill communicator, not just the ones asking the context."""
|
||||
from sglang.srt.distributed import parallel_state
|
||||
|
||||
stand_in = SimpleNamespace(world_size=1, rank_in_group=0)
|
||||
with get_parallel().override(tp_group=stand_in):
|
||||
self.assertIs(get_parallel().tp_group, stand_in)
|
||||
self.assertIs(parallel_state.get_tp_group(), stand_in)
|
||||
|
||||
def test_the_package_that_defines_them_is_not_warned_at(self):
|
||||
"""`srt/distributed/` keeps calling them: a read there would go through
|
||||
the context back into itself."""
|
||||
import warnings
|
||||
|
||||
from sglang.srt.distributed import parallel_state
|
||||
|
||||
parallel_state._ALREADY_WARNED.discard("get_tensor_model_parallel_rank")
|
||||
self.addCleanup(
|
||||
parallel_state._ALREADY_WARNED.discard, "get_tensor_model_parallel_rank"
|
||||
)
|
||||
caller = types.ModuleType("sglang.srt.distributed.pretend_internal")
|
||||
caller.__dict__["call"] = lambda: (
|
||||
parallel_state.get_tensor_model_parallel_rank()
|
||||
)
|
||||
exec(
|
||||
"def call():\n from sglang.srt.distributed import parallel_state\n"
|
||||
" return parallel_state.get_tp_group()",
|
||||
caller.__dict__,
|
||||
)
|
||||
with warnings.catch_warnings(record=True) as seen:
|
||||
warnings.simplefilter("always")
|
||||
try:
|
||||
caller.call()
|
||||
except Exception:
|
||||
pass
|
||||
self.assertEqual([str(w.message) for w in seen], [])
|
||||
|
||||
def test_the_guard_would_notice_a_caller(self):
|
||||
"""The subject set is derived, so this checks the search finds a real
|
||||
call rather than that the list happens to be empty: `get_self_pp_group`
|
||||
@@ -2571,37 +2843,39 @@ class TestTheTopologyIdentities(CustomTestCase):
|
||||
|
||||
def test_a_group_built_at_another_width_is_refused(self):
|
||||
"""The other end of the same identity: what the configuration says and
|
||||
what the coordinators were actually built at, checked where the
|
||||
disagreement is still attributable to the build."""
|
||||
from sglang.srt.distributed import parallel_state
|
||||
what the coordinators were actually built at. Stating a group is how
|
||||
the build hands it over, so that write is where the disagreement
|
||||
surfaces -- still attributable to the build, and before a collective
|
||||
runs on the wrong peers."""
|
||||
from sglang.srt.distributed.parallel_state import GroupCoordinator
|
||||
|
||||
self._publish_square()
|
||||
wrong = GroupCoordinator.__new__(GroupCoordinator)
|
||||
wrong.world_size = 8
|
||||
wrong.rank_in_group = 0
|
||||
with patch.object(parallel_state, "_TP", wrong):
|
||||
with self.assertRaises(ValueError) as caught:
|
||||
_validate_parallel(get_parallel(), "group build")
|
||||
with self.assertRaises(ValueError) as caught:
|
||||
get_parallel().override_permanently(tp_group=wrong)
|
||||
message = str(caught.exception)
|
||||
self.assertIn("set by group build", message)
|
||||
self.assertIn("tp_group.world_size == tp_size", message)
|
||||
self.assertIn("built 8, configured 4", message)
|
||||
# The refused write left nothing behind: the name is unwritten, not
|
||||
# holding a group no identity accepts.
|
||||
with self.assertRaises(RuntimeError):
|
||||
get_parallel().tp_group
|
||||
|
||||
def test_a_group_built_at_the_configured_width_is_quiet(self):
|
||||
from sglang.srt.distributed import parallel_state
|
||||
from sglang.srt.distributed.parallel_state import GroupCoordinator
|
||||
|
||||
self._publish_square()
|
||||
right = GroupCoordinator.__new__(GroupCoordinator)
|
||||
right.world_size = 4
|
||||
right.rank_in_group = 3
|
||||
with patch.object(parallel_state, "_TP", right):
|
||||
_validate_parallel(get_parallel(), "group build")
|
||||
get_parallel().override_permanently(tp_group=right)
|
||||
self.assertIs(get_parallel().tp_group, right)
|
||||
|
||||
def test_a_draft_scope_states_a_consistent_topology(self):
|
||||
"""The scope narrows four names at once, so the identity applies to it
|
||||
-- and holds, which is what step lets the guard stay on."""
|
||||
-- and holds, which is what lets the guard stay on."""
|
||||
from sglang.srt.distributed import parallel_state
|
||||
from sglang.srt.distributed.parallel_state import GroupCoordinator
|
||||
|
||||
@@ -2609,9 +2883,8 @@ class TestTheTopologyIdentities(CustomTestCase):
|
||||
group = GroupCoordinator.__new__(GroupCoordinator)
|
||||
group.world_size = 2
|
||||
group.rank_in_group = 1
|
||||
with patch.object(parallel_state, "_TP", group):
|
||||
with parallel_state.patch_tensor_parallel_group(group, owns_attention=True):
|
||||
self.assertEqual(get_parallel().attn_tp_size, 2)
|
||||
with parallel_state.patch_tensor_parallel_group(group, owns_attention=True):
|
||||
self.assertEqual(get_parallel().attn_tp_size, 2)
|
||||
|
||||
|
||||
class TestWhoAnswersDuringADraftScope(CustomTestCase):
|
||||
@@ -2650,11 +2923,10 @@ class TestWhoAnswersDuringADraftScope(CustomTestCase):
|
||||
group = self._single_member_group()
|
||||
self._two_stage_pipeline()
|
||||
self.assertEqual(get_parallel().pp_size, 2)
|
||||
with patch.object(parallel_state, "_PP", group):
|
||||
with parallel_state.patch_pipeline_parallel_group(group):
|
||||
self.assertEqual(get_parallel().pp_size, 1)
|
||||
self.assertEqual(get_parallel().pp_rank, 0)
|
||||
self.assertIs(get_parallel().pp_group, group)
|
||||
with parallel_state.patch_pipeline_parallel_group(group):
|
||||
self.assertEqual(get_parallel().pp_size, 1)
|
||||
self.assertEqual(get_parallel().pp_rank, 0)
|
||||
self.assertIs(get_parallel().pp_group, group)
|
||||
self.assertEqual(get_parallel().pp_size, 2)
|
||||
self.assertEqual(get_parallel().pp_rank, 1)
|
||||
|
||||
@@ -2685,28 +2957,25 @@ class TestWhoAnswersDuringADraftScope(CustomTestCase):
|
||||
self.assertEqual(get_parallel().attn_tp_size, 2)
|
||||
|
||||
group = self._group(world_size=2, rank=1)
|
||||
with patch.object(parallel_state, "_TP", group):
|
||||
with parallel_state.patch_tensor_parallel_group(group, owns_attention=True):
|
||||
parallel = get_parallel()
|
||||
self.assertEqual(parallel.tp_size, 2)
|
||||
self.assertEqual(parallel.attn_tp_size, 2)
|
||||
self.assertEqual(parallel.attn_tp_rank, 1)
|
||||
self.assertEqual(parallel.attn_dp_size, 1)
|
||||
self.assertEqual(parallel.attn_dp_rank, 0)
|
||||
self.assertEqual(parallel.attn_cp_size, 1)
|
||||
self.assertEqual(parallel.attn_cp_rank, 0)
|
||||
# `dp_size` is the deployment's replica count, not a property
|
||||
# of the group being installed, so the scope leaves it alone --
|
||||
# `require_mlp_tp_gather` asserts on it under dp attention.
|
||||
self.assertEqual(parallel.dp_size, 2)
|
||||
# The whole point of stating the rest: the identity the
|
||||
# override path and the group build both check holds in here.
|
||||
self.assertEqual(
|
||||
parallel.tp_size,
|
||||
parallel.attn_tp_size
|
||||
* parallel.attn_dp_size
|
||||
* parallel.attn_cp_size,
|
||||
)
|
||||
with parallel_state.patch_tensor_parallel_group(group, owns_attention=True):
|
||||
parallel = get_parallel()
|
||||
self.assertEqual(parallel.tp_size, 2)
|
||||
self.assertEqual(parallel.attn_tp_size, 2)
|
||||
self.assertEqual(parallel.attn_tp_rank, 1)
|
||||
self.assertEqual(parallel.attn_dp_size, 1)
|
||||
self.assertEqual(parallel.attn_dp_rank, 0)
|
||||
self.assertEqual(parallel.attn_cp_size, 1)
|
||||
self.assertEqual(parallel.attn_cp_rank, 0)
|
||||
# `dp_size` is the deployment's replica count, not a property of
|
||||
# the group being installed, so the scope leaves it alone --
|
||||
# `require_mlp_tp_gather` asserts on it under dp attention.
|
||||
self.assertEqual(parallel.dp_size, 2)
|
||||
# The whole point of stating the rest: the identity the override
|
||||
# path and the group build both check holds in here.
|
||||
self.assertEqual(
|
||||
parallel.tp_size,
|
||||
parallel.attn_tp_size * parallel.attn_dp_size * parallel.attn_cp_size,
|
||||
)
|
||||
self.assertEqual(get_parallel().attn_dp_size, 2)
|
||||
self.assertEqual(get_parallel().dp_size, 2)
|
||||
|
||||
@@ -2790,9 +3059,8 @@ class TestWhoAnswersDuringADraftScope(CustomTestCase):
|
||||
self._two_stage_pipeline()
|
||||
self.assertEqual(get_parallel().pp_size, 2)
|
||||
group = self._single_member_group()
|
||||
with patch.object(parallel_state, "_PP", group):
|
||||
with parallel_state.patch_pipeline_parallel_group(group):
|
||||
checker = WeightChecker(get_model=lambda: None)
|
||||
with parallel_state.patch_pipeline_parallel_group(group):
|
||||
checker = WeightChecker(get_model=lambda: None)
|
||||
|
||||
# The scope has closed and the context answers the target's shape again.
|
||||
self.assertEqual(get_parallel().pp_size, 2)
|
||||
|
||||
Reference in New Issue
Block a user