Deprecate the parallel getters the context answers, and ratchet them shut (#40342)

This commit is contained in:
Cheng Wan
2026-09-21 12:25:32 -07:00
committed by GitHub
parent 65be3fa71a
commit 73f071db52
44 changed files with 933 additions and 501 deletions
+3 -1
View File
@@ -134,7 +134,9 @@ class TestFilterDcpLocalChunkKvIndices(CustomTestCase):
def test_identity_without_dcp(self):
kv = torch.arange(37)
with rc.get_parallel().override(dcp_enabled=False, dcp_size=1, dcp_rank=0):
with rc.get_parallel().override(
dcp_enabled=False, dcp_size=1, dcp_rank=0, attn_dcp_rank=0
):
self.assertIs(
filter_dcp_local_chunk_kv_indices(
kv, torch.tensor([0]), torch.tensor([37])
@@ -25,6 +25,7 @@ import torch
import torch.distributed as dist
import sglang.srt.distributed.parallel_state as ps
from sglang.srt.runtime_context import get_parallel
from sglang.test.ci.ci_register import register_cuda_ci
register_cuda_ci(
@@ -130,6 +131,9 @@ def init_distributed():
local_rank=local_rank,
backend="nccl",
)
# The context answers a handle from what was stated on it, not from
# this module global, so a rank stood up by hand says so itself.
get_parallel().override_permanently(world_group=coord)
cpu_group = coord.cpu_group
nccl_group = coord.device_group
@@ -14,6 +14,7 @@ from sglang.kernels.jit.benchmark import marker
from sglang.kernels.jit.benchmark.utils import get_benchmark_range, multigpu_bench_main
from sglang.kernels.jit.utils import cache_once, is_arch_support_pdl
from sglang.kernels.ops.communication.mp import register_comm_cleanup
from sglang.srt.runtime_context import get_parallel
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
register_cuda_ci(
@@ -61,6 +62,9 @@ def _init_cpu_group() -> dist.ProcessGroup:
local_rank=local_rank,
backend="nccl",
)
# The context answers a handle from what was stated on it, not from
# this module global, so a rank stood up by hand says so itself.
get_parallel().override_permanently(world_group=coord)
atexit.register(dist.destroy_process_group)
torch.cuda.set_stream(torch.cuda.Stream())
return coord.cpu_group
@@ -33,6 +33,7 @@ from sglang.srt.distributed.device_communicators.triton_symm_mem_ag import (
all_gather_inner,
create_state,
)
from sglang.srt.runtime_context import get_parallel
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
register_cuda_ci(
@@ -77,6 +78,9 @@ def _init_cpu_group() -> dist.ProcessGroup:
local_rank=local_rank,
backend="nccl",
)
# The context answers a handle from what was stated on it, not from
# this module global, so a rank stood up by hand says so itself.
get_parallel().override_permanently(world_group=ps._WORLD)
atexit.register(dist.destroy_process_group)
logging.disable(logging.INFO)
torch.cuda.set_stream(torch.cuda.Stream())
@@ -36,6 +36,7 @@ from sglang.kernels.ops.communication.mp import register_comm_cleanup
from sglang.srt.distributed.device_communicators.custom_all_reduce_v2 import (
CustomAllReduceV2,
)
from sglang.srt.runtime_context import get_parallel
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
register_cuda_ci(
@@ -108,6 +109,9 @@ def _init_cpu_group() -> dist.ProcessGroup:
local_rank=local_rank,
backend="nccl",
)
# The context answers a handle from what was stated on it, not from
# this module global, so a rank stood up by hand says so itself.
get_parallel().override_permanently(world_group=coord)
atexit.register(dist.destroy_process_group)
logging.disable(logging.INFO)
torch.cuda.set_stream(torch.cuda.Stream())
@@ -37,6 +37,7 @@ from sglang.kernels.ops.communication.mp import register_comm_cleanup
from sglang.srt.distributed.device_communicators.custom_all_reduce_v2 import (
CustomAllReduceV2,
)
from sglang.srt.runtime_context import get_parallel
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kernels.utils import multigpu_pytest_main
@@ -129,6 +130,9 @@ def _init_cpu_group_once() -> dist.ProcessGroup:
local_rank=local_rank,
backend="nccl",
)
# The context answers a handle from what was stated on it, not from
# this module global, so a rank stood up by hand says so itself.
get_parallel().override_permanently(world_group=coord)
atexit.register(dist.destroy_process_group)
cpu_group = coord.cpu_group
assert isinstance(cpu_group, dist.ProcessGroup)
@@ -31,6 +31,7 @@ from sglang.srt.distributed.device_communicators.triton_symm_mem_ag import (
all_gather_inner,
create_state,
)
from sglang.srt.runtime_context import get_parallel
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kernels.utils import multigpu_pytest_main
@@ -70,6 +71,9 @@ def _init_cpu_group_once() -> dist.ProcessGroup:
local_rank=local_rank,
backend="nccl",
)
# The context answers a handle from what was stated on it, not from
# this module global, so a rank stood up by hand says so itself.
get_parallel().override_permanently(world_group=ps._WORLD)
atexit.register(dist.destroy_process_group)
logging.disable(logging.INFO)
torch.cuda.set_stream(torch.cuda.Stream())
@@ -24,6 +24,7 @@ from sglang.kernels.ops.communication.mp import register_comm_cleanup
from sglang.srt.distributed.device_communicators.custom_all_reduce_v2 import (
CustomAllReduceV2,
)
from sglang.srt.runtime_context import get_parallel
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kernels.utils import multigpu_pytest_main
@@ -92,6 +93,9 @@ def _init_cpu_group_once() -> dist.ProcessGroup:
local_rank=local_rank,
backend="nccl",
)
# The context answers a handle from what was stated on it, not from
# this module global, so a rank stood up by hand says so itself.
get_parallel().override_permanently(world_group=coord)
atexit.register(dist.destroy_process_group)
cpu_group = coord.cpu_group
assert isinstance(cpu_group, dist.ProcessGroup)
@@ -30,6 +30,7 @@ from sglang.kernels.ops.kimi_k3 import all_reduce
from sglang.srt.distributed.device_communicators.custom_all_reduce_v2 import (
CustomAllReduceV2,
)
from sglang.srt.runtime_context import get_parallel
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kernels.utils import multigpu_pytest_main
@@ -65,6 +66,9 @@ def _init_world():
local_rank=local_rank,
backend="nccl",
)
# The context answers a handle from what was stated on it, not from
# this module global, so a rank stood up by hand says so itself.
get_parallel().override_permanently(world_group=coord)
atexit.register(dist.destroy_process_group)
logging.disable(logging.INFO)
torch.cuda.set_stream(torch.cuda.Stream())
@@ -20,6 +20,7 @@ from sglang.kernels.ops.kimi_k3 import (
from sglang.srt.distributed.device_communicators.custom_all_reduce_v2 import (
CustomAllReduceV2,
)
from sglang.srt.runtime_context import get_parallel
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kernels.utils import multigpu_pytest_main
@@ -53,6 +54,9 @@ def _init_world():
local_rank=local_rank,
backend="nccl",
)
# The context answers a handle from what was stated on it, not from
# this module global, so a rank stood up by hand says so itself.
get_parallel().override_permanently(world_group=coord)
atexit.register(dist.destroy_process_group)
cpu_group = coord.cpu_group
assert isinstance(cpu_group, dist.ProcessGroup)
@@ -48,6 +48,7 @@ from sglang.srt.mem_cache.multimodal_cache import (
EmbeddingResult,
MultiModalStaticCache,
)
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils.common import safe_pickle_loads
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
@@ -1011,10 +1012,7 @@ class TestEncoderDelivery(CustomTestCase):
statuses[1].copy_(torch.tensor([400, 1, 0, 0]))
with (
patch(
"sglang.srt.distributed.parallel_state.get_tp_group",
return_value=TPGroup(),
),
get_parallel().override(tp_group=TPGroup()),
patch(
"sglang.srt.disaggregation.encoder.server.torch.distributed.all_gather",
side_effect=all_gather,
@@ -1051,10 +1049,7 @@ class TestEncoderDelivery(CustomTestCase):
statuses[1][2] += 1
with (
patch(
"sglang.srt.distributed.parallel_state.get_tp_group",
return_value=TPGroup(),
),
get_parallel().override(tp_group=TPGroup()),
patch(
"sglang.srt.disaggregation.encoder.server.torch.distributed.all_gather",
side_effect=all_gather,
@@ -1095,10 +1090,7 @@ class TestEncoderDelivery(CustomTestCase):
statuses[1].copy_(local_status)
with (
patch(
"sglang.srt.distributed.parallel_state.get_tp_group",
return_value=TPGroup(),
),
get_parallel().override(tp_group=TPGroup()),
patch(
"sglang.srt.disaggregation.encoder.server.torch.distributed.all_gather",
side_effect=all_gather,
@@ -9,7 +9,7 @@ import unittest
from unittest.mock import MagicMock, call, patch
from sglang.srt.environ import envs
from sglang.srt.runtime_context import get_context
from sglang.srt.runtime_context import get_context, get_parallel
from sglang.test.test_utils import CustomTestCase
@@ -201,12 +201,12 @@ class TestRegisterToBootstrap(CustomTestCase):
self.assertIn("10.0.0.1", url_used)
@patch("sglang.srt.disaggregation.common.conn.requests.put")
# The consumer reads the group through `get_parallel()`, which reads
# through to the canonical getter, so that is where the stub belongs.
@patch("sglang.srt.distributed.parallel_state.get_world_group")
def test_rust_attention_dp_replicates_complete_topology_across_hosts(
self, mock_world_group, mock_put
self, mock_put
):
# The consumer reads the group through `get_parallel()`, so the
# stub is stated there rather than in the module the build writes.
mock_world_group = MagicMock()
success_resp = MagicMock()
success_resp.status_code = 200
mock_put.return_value = success_resp
@@ -230,9 +230,12 @@ class TestRegisterToBootstrap(CustomTestCase):
for dp_rank, tp_rank, host, rank_port, _ in schedulers
]
mock_world_group.return_value.all_gather_object.side_effect = gather_topology
mock_world_group.all_gather_object.side_effect = gather_topology
with envs.SGLANG_RUST_SERVER.override(True):
with (
get_parallel().override(world_group=mock_world_group),
envs.SGLANG_RUST_SERVER.override(True),
):
for dp_rank, tp_rank, local_ip, _, rust_http_port in schedulers:
manager = self._make_manager()
manager.attn_dp_size = 2
@@ -277,7 +280,7 @@ class TestRegisterToBootstrap(CustomTestCase):
gather_call.args[0]["attn_dp_rank"],
gather_call.args[0]["attn_tp_rank"],
)
for gather_call in mock_world_group.return_value.all_gather_object.call_args_list
for gather_call in mock_world_group.all_gather_object.call_args_list
],
[(dp, tp) for dp, tp, _, _, _ in schedulers],
)
@@ -93,6 +93,9 @@ def _make_prefill_aware_swa_runner(
page_size=1,
attn_cp_size=1,
tp_size=1,
# The backend still reads the runner's frozen record for these two;
# same single-rank placement, stated where it looks for it.
ps=SimpleNamespace(attn_cp_size=1, tp_size=1),
is_draft_worker=False,
server_args=server_args,
attention_chunk_size=None,
@@ -14,6 +14,7 @@ from sglang.test.test_utils import CustomTestCase, maybe_stub_sgl_kernel
maybe_stub_sgl_kernel()
from sglang.srt.layers.moe import mega_moe
from sglang.srt.runtime_context import get_parallel
register_cpu_ci(est_time=8, suite="base-a-test-cpu")
@@ -271,11 +272,8 @@ class TestDeepGemmMegaMoeApi(CustomTestCase):
"init_new",
return_value=object(),
),
patch(
"sglang.srt.runtime_context.get_parallel",
return_value=SimpleNamespace(
moe_ep_group=SimpleNamespace(device_group=object())
),
get_parallel().override(
moe_ep_group=SimpleNamespace(device_group=object())
),
):
mega_moe._run_mega_routed(
@@ -11,6 +11,7 @@ from unittest.mock import Mock, patch
from parameterized import parameterized
from sglang.srt.distributed import parallel_state
from sglang.srt.managers import scheduler as scheduler_module
from sglang.srt.managers.schedule_batch import ScheduleBatch
from sglang.srt.managers.scheduler import Scheduler
@@ -168,12 +169,21 @@ class TestSchedulerIdleStepCounters(CustomTestCase):
)
with (
patch(f"{PDMUX_MODULE}.get_current_stream_idx", return_value=0),
patch(f"{PDMUX_MODULE}.set_pdmux_status"),
patch(f"{PDMUX_MODULE}.torch.cuda.empty_cache"),
patch(
f"{PDMUX_MODULE}.torch.cuda.stream",
side_effect=lambda stream: nullcontext(),
),
# The prefill section runs under the duplicate communicator
# `--enable-pdmux` builds, in place of the module flag this
# replaces. The loop has no process groups at all, so stand
# one in: the scope refuses to open without it rather than
# letting prefill quietly share the decode communicator.
patch.object(
parallel_state,
"_PDMUX_PREFILL_TP_GROUP",
SimpleNamespace(world_size=1, rank_in_group=0),
),
):
self.run_and_check(
scheduler,
@@ -297,7 +297,9 @@ class TestPPMambaPoolSizing(unittest.TestCase):
server_args=SimpleNamespace(),
spec_algorithm=SimpleNamespace(is_none=lambda: True),
layer_info=SimpleNamespace(start_layer=start, end_layer=end),
ps=SimpleNamespace(attn_dp_size=1, pp_size=pp_size),
# The runner carries its placement as plain attributes.
attn_dp_size=1,
pp_size=pp_size,
hybrid_gdn_config=None,
model_config=SimpleNamespace(
hf_config=SimpleNamespace(), num_hidden_layers=cls.TOTAL_LAYERS
@@ -304,16 +304,13 @@ class TestHostMemoryBudget(CustomTestCase):
def test_ranks_per_host_divides_world_size_by_nodes(self):
# The launcher slices ranks uniformly across nodes, so the co-located
# rank count is world_size // nnodes — no hostname collective.
fake_group = unittest.mock.Mock(world_size=16)
# tp_size=16 states the launch width the count divides -- the
# published configuration is where ranks_per_host reads it from.
with (
get_context().override_server_args(nnodes=2),
get_context().override_server_args(nnodes=2, tp_size=16),
unittest.mock.patch.object(
torch.distributed, "is_initialized", return_value=True
),
unittest.mock.patch(
"sglang.srt.distributed.parallel_state.get_world_group",
return_value=fake_group,
),
):
self.assertEqual(base.ranks_per_host(), 8)
@@ -248,7 +248,7 @@ class TestUnifiedMLATokenToKVPool(unittest.TestCase):
] = float(layer + 1)
with (
get_parallel().override(dcp_enabled=False),
get_parallel().override(dcp_enabled=False, attn_dcp_rank=0),
mock.patch(
"sglang.srt.mem_cache.memory_pool.current_platform.synchronize"
),
@@ -411,7 +411,9 @@ class TestMambaAllocatorCpuCopyIsPhysical(unittest.TestCase):
_FakeKVCache(pool.max_slots("full")),
_FakeKVCache(pool.max_slots("mamba")),
)
with get_parallel().override(dcp_enabled=False, attn_dcp_size=1):
with get_parallel().override(
dcp_enabled=False, attn_dcp_size=1, attn_dcp_rank=0
):
allocator = UnifiedMambaTokenToKVPoolAllocator(
unified_buffer=pool, kvcache=kvcache, device=_DEV, page_size=ps
)
@@ -18,6 +18,7 @@ from sglang.srt.model_executor.runner_backend.full_cuda_graph_backend import (
FullCudaGraphBackend,
)
from sglang.srt.model_executor.runner_utils import pool
from sglang.srt.runtime_context import get_parallel
from sglang.srt.speculative import dflash_utils, dflash_worker_v2, eagle_utils
from sglang.srt.speculative.dflash_worker_v2 import DFlashWorkerV2
from sglang.test.ci.ci_register import register_cuda_ci
@@ -222,12 +223,7 @@ class TestGraphPoolBorrow(CustomTestCase):
"sglang.srt.layers.dp_attention.is_dp_attention_enabled",
return_value=False,
),
# `parallel_state`, not the package re-export: a stub on the
# re-export is never consulted.
patch(
"sglang.srt.distributed.parallel_state.get_tp_group",
return_value=tp_group,
),
get_parallel().override(tp_group=tp_group),
patch(
"sglang.kernels.ops.speculative.sampling.tree_speculative_sampling_target_only",
side_effect=fake_sampling,
@@ -11,7 +11,7 @@ from sglang.srt.model_executor.cuda_graph_config import (
)
from sglang.srt.model_executor.model_runner_components import kv_pool_runtime
from sglang.srt.model_executor.pool_configurator import MemoryPoolConfig
from sglang.srt.runtime_context import get_context
from sglang.srt.runtime_context import get_context, get_parallel
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
@@ -60,9 +60,8 @@ class TestCanaryHeadroom(CustomTestCase):
),
),
patch.object(kv_pool_runtime.torch.cuda, "synchronize"),
patch(
"sglang.srt.distributed.parallel_state.get_world_group",
return_value=SimpleNamespace(world_size=1, cpu_group=None),
get_parallel().override(
world_group=SimpleNamespace(world_size=1, cpu_group=None)
),
patch.object(kv_pool_runtime, "get_available_gpu_memory", return_value=20),
patch.object(kv_pool_runtime, "mambaish_config", return_value=None),
@@ -36,7 +36,11 @@ def mock_cpu_env(kv_size=2, tp_size=1, swa_eviction_interval=4):
with (
patch("torch._utils._element_size", return_value=kv_size),
get_parallel().override(attn_tp_size=tp_size),
# A width is a whole topology: state the TP siblings the identities
# relate it to, not the attention share alone.
get_parallel().override(
tp_size=tp_size, attn_tp_size=tp_size, moe_tp_size=tp_size
),
envs.SGLANG_SWA_EVICTION_INTERVAL.override(swa_eviction_interval),
):
yield
@@ -53,9 +53,7 @@ class TestModelOptExport(unittest.TestCase):
self.mock_logger.start()
# Mock all distributed functions that might be called
self.mock_get_tp_group = patch(
"sglang.srt.distributed.parallel_state.get_tp_group"
)
self.mock_get_tp_group = patch("sglang.srt.distributed.parallel_state._TP")
self.mock_get_tp_group.start()
# Mock model parallel initialization check
@@ -82,9 +82,7 @@ class TestModelOptModelLoader(CustomTestCase):
self.mock_logger.start()
# Mock all distributed functions that might be called
self.mock_get_tp_group = patch(
"sglang.srt.distributed.parallel_state.get_tp_group"
)
self.mock_get_tp_group = patch("sglang.srt.distributed.parallel_state._TP")
self.mock_get_tp_group.start()
# Mock model parallel initialization check
@@ -25,6 +25,7 @@ from sglang.srt.model_loader.weight_utils import (
fastsafetensors_weights_iterator,
safetensors_weights_iterator,
)
from sglang.srt.runtime_context import get_parallel
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
@@ -237,10 +238,7 @@ class TestPrefetchCheckpoints(CustomTestCase):
patch("threading.Thread", _InlineThread),
patch("concurrent.futures.ThreadPoolExecutor", _InlineExecutor),
patch("concurrent.futures.wait", side_effect=_wait_all),
patch(
"sglang.srt.distributed.parallel_state.get_world_group",
return_value=FakeWorldGroup(),
),
get_parallel().override(world_group=FakeWorldGroup()),
patch(
"sglang.srt.model_loader.weight_utils._prefetch_checkpoint_file",
side_effect=lambda path, cancel_event: loaded_paths.append(path),
@@ -652,10 +652,7 @@ class TestStructuralSignature(unittest.TestCase):
# must see the same gathered list and thus the same aggregate.
fake_group.all_gather_object.side_effect = lambda local: ["sig-pp0", "sig-pp1"]
with mock.patch(
"sglang.srt.distributed.parallel_state.get_world_group",
return_value=fake_group,
):
with get_parallel().override(world_group=fake_group):
agg_from_rank0 = (
PreshardedModelLoader._make_rank_invariant_structural_signature(
"sig-pp0"
@@ -673,10 +670,7 @@ class TestStructuralSignature(unittest.TestCase):
"sig-pp0",
"sig-pp1-changed",
]
with mock.patch(
"sglang.srt.distributed.parallel_state.get_world_group",
return_value=fake_group,
):
with get_parallel().override(world_group=fake_group):
agg_changed = (
PreshardedModelLoader._make_rank_invariant_structural_signature(
"sig-pp0"
@@ -5,6 +5,7 @@ from types import SimpleNamespace
from unittest.mock import patch
from sglang.srt.models.transformers import TransformersBase
from sglang.srt.runtime_context import get_parallel
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
@@ -54,10 +55,9 @@ class TestTransformersFallbackSkipSubstrs(CustomTestCase):
pass
with (
patch(
"sglang.srt.distributed.parallel_state.get_pp_group",
return_value=SimpleNamespace(),
),
# `__init__` only stashes the pipeline group, so an empty
# stand-in carries it past the read.
get_parallel().override(pp_group=SimpleNamespace()),
patch(
"sglang.srt.models.transformers.get_hf_text_config",
return_value=SimpleNamespace(),
@@ -1,6 +1,5 @@
import sys
from types import SimpleNamespace
from unittest.mock import patch
import pytest
import torch
@@ -13,6 +12,7 @@ from sglang.srt.multimodal.internvl_vit_cuda_graph_runner import (
InternViTCudaGraphRunner,
)
from sglang.srt.multimodal.vit_cuda_graph_runner import ViTCudaGraphRunner
from sglang.srt.runtime_context import get_parallel
class _Block:
@@ -32,12 +32,10 @@ def _runner(*, use_data_parallel: bool) -> ViTCudaGraphRunner:
def test_dp_vit_graph_capture_does_not_enter_tp_communication_capture():
runner = _runner(use_data_parallel=True)
with patch(
"sglang.srt.distributed.parallel_state.get_tp_group",
side_effect=AssertionError("DP capture must be rank-local"),
):
with runner._capture_context():
pass
# No tp_group is stated, so reading one would raise: the DP path must not
# ask for the TP group at all.
with runner._capture_context():
pass
def test_non_dp_vit_graph_capture_uses_tp_communication_capture():
@@ -52,9 +50,7 @@ def test_non_dp_vit_graph_capture_uses_tp_communication_capture():
group = SimpleNamespace(ca_comm=SimpleNamespace(capture=lambda: Capture()))
runner = _runner(use_data_parallel=False)
with patch(
"sglang.srt.distributed.parallel_state.get_tp_group", return_value=group
):
with get_parallel().override(tp_group=group):
with runner._capture_context():
pass
assert entered == [True]
@@ -132,18 +132,18 @@ def _stash_overlay(server_args):
def _live_topology_leaves():
"""Names `ParallelContext` serves from the live topology, not the config.
"""Names `ParallelContext` answers from a runtime write, not the config.
Read out of `_LIVE_READS`, which is where those names are declared.
Inferring them from "did the read raise" is wrong -- it only raises while
the process groups are missing, so in a process where an earlier test built
them the property answers the *live* size and a leaf check reads it as a
config mismatch (`parallel.tp_size: bag=1 resolution=2`). Whether a name is
Read off the declarations that carry no `fn`, which is what those are.
Inferring them from "did the read raise" is wrong -- it raises only while
nothing has written the name, so in a process where an earlier test stated
one the property answers that value and a leaf check reads it as a config
mismatch (`parallel.tp_size: bag=1 resolution=2`). Whether a name is
shadowed is a property of the declaration, not of the process.
"""
from sglang.srt.runtime_context import _LIVE_READS
from sglang.srt.runtime_context import _derived_widths
return frozenset(_LIVE_READS)
return frozenset(n for n, d in _derived_widths().items() if not d.fn)
class TestResolutionDeclarations(CustomTestCase):
@@ -249,6 +249,8 @@ def test_worker_folds_a_gate_admitted_quantized_selector_head(monkeypatch):
block_size=8,
selector=object(),
model_runner=SimpleNamespace(tp_rank=0),
# The worker rank-gates its logging on its own frozen record.
ps=SimpleNamespace(tp_rank=0),
draft_model=SimpleNamespace(lm_head=None),
device="cpu",
_selector_sampling_enabled=True,
@@ -283,6 +285,8 @@ def test_worker_warns_once_when_selector_sampling_is_disabled(monkeypatch):
_selector_sampling_enabled=False,
_warned_sampling_fallback=False,
model_runner=SimpleNamespace(tp_rank=0),
# The worker rank-gates its logging on its own frozen record.
ps=SimpleNamespace(tp_rank=0),
)
batch = SimpleNamespace(sampling_info=SimpleNamespace(is_all_greedy=False))
+439 -171
View File
@@ -10,8 +10,10 @@ import os
import pathlib as _pathlib
import shutil
import tempfile
import types
import unittest
import warnings
from types import SimpleNamespace
from unittest.mock import patch
import msgspec
@@ -41,7 +43,6 @@ from sglang.srt.runtime_context import (
RuntimeContext,
SpawnRanks,
_FlagGroupBase,
_validate_parallel,
assert_published,
derive_parallel_widths,
get_context,
@@ -111,39 +112,58 @@ def _scope_entries_that_say_nothing(paths):
_PS = "sglang.srt.distributed.parallel_state"
def _parallel_state():
from sglang.srt.distributed import parallel_state
return parallel_state
_DP = "sglang.srt.layers.dp_attention"
# Ranks and the launch width are asked of the group: they are not implied by
# anything, so there is nothing to derive them from. The quotients are not
# here -- `attn_tp_size` and its siblings are functions of the configured
# leaves, and `TestDerivedWidths` pins them. `attn_dp_rank` is not here either: no group coordinator
# knows it, so it is stamped when the attention topology is initialized and
# `TestStampedRanks` is what pins it. The other world width is not here
# because the group does not know it; `TestTheTwoWorldWidths` pins it.
SIZE_RANK_DELEGATIONS = [
("launch_world_size", f"{_PS}.get_world_size"),
("launch_world_rank", f"{_PS}.get_world_rank"),
("tp_rank", f"{_PS}.get_tensor_model_parallel_rank"),
("dcp_rank", f"{_PS}.get_dcp_rank"),
("pp_rank", f"{_PS}.get_pipeline_model_parallel_rank"),
("moe_ep_rank", f"{_PS}.get_moe_expert_parallel_rank"),
("moe_dp_rank", f"{_PS}.get_moe_data_parallel_rank"),
("moe_tp_rank", f"{_PS}.get_moe_tensor_parallel_rank"),
("attn_tp_rank", f"{_PS}.get_attn_tensor_model_parallel_rank"),
("attn_cp_rank", f"{_PS}.get_attn_context_model_parallel_rank"),
]
#: The groups `initialize_model_parallel` states on the context, by the module
#: global it builds each one into. WORLD is not among them: it is built and
#: stated by `init_distributed_environment`, one call earlier.
GROUP_STAMPS = {
"tp_group": "_TP",
"dcp_group": "_DCP",
"pp_group": "_PP",
"moe_ep_group": "_MOE_EP",
"moe_dp_group": "_MOE_DP",
"moe_tp_group": "_MOE_TP",
"attn_tp_group": "_ATTN_TP",
"attn_cp_group": "_ATTN_CP",
"shared_experts_tp_group": "_SHARED_EXPERTS_TP",
}
GROUP_DELEGATIONS = [
("world_group", f"{_PS}.get_world_group"),
("tp_group", f"{_PS}.get_tp_group"),
("dcp_group", f"{_PS}.get_dcp_group"),
("pp_group", f"{_PS}.get_pp_group"),
("moe_ep_group", f"{_PS}.get_moe_ep_group"),
("moe_dp_group", f"{_PS}.get_moe_dp_group"),
("moe_tp_group", f"{_PS}.get_moe_tp_group"),
("attn_tp_group", f"{_PS}.get_attn_tp_group"),
("attn_cp_group", f"{_PS}.get_attn_cp_group"),
]
def _groups_the_build_states() -> dict:
"""What `initialize_model_parallel` hands the context, read out of its
source: `{context name: the module global it passes}`.
Out of the source because that is the only place the whole set appears at
once -- calling the function needs ten live process groups.
"""
import ast
import inspect
import textwrap
from sglang.srt.distributed import parallel_state
body = textwrap.dedent(inspect.getsource(parallel_state.initialize_model_parallel))
for node in ast.walk(ast.parse(body)):
keys = getattr(node, "keys", None)
if (
isinstance(node, ast.Dict)
and keys
and all(
isinstance(k, ast.Constant) and str(k.value).endswith("_group")
for k in keys
)
):
return {k.value: v.id for k, v in zip(node.keys, node.values)}
raise AssertionError("initialize_model_parallel states no group at all")
class TestRuntimeContextSingletons(CustomTestCase):
@@ -171,29 +191,91 @@ class _IsolatedOverrides(CustomTestCase):
super().tearDown()
class TestTheBuildStatesEveryGroup(_IsolatedOverrides):
"""Nothing derives a group, so one the build forgets to state is a name
that answers "not written" for the rest of the process -- and the reader
that finds out is a model layer, a long way from here."""
def test_the_build_states_every_group_the_context_declares(self):
from sglang.srt.runtime_context import _parallel_fields
declared = {name for name in _parallel_fields() if name.endswith("_group")}
self.assertEqual(declared, set(GROUP_STAMPS) | {"world_group"})
def test_the_world_group_is_stated_where_it_is_built(self):
"""`initialize_model_parallel` places every group it builds by reading
`get_world_group().local_rank`, so WORLD has to be answerable before it
runs -- one function earlier, where it is constructed."""
import ast
import inspect
import textwrap
from sglang.srt.distributed import parallel_state
body = textwrap.dedent(
inspect.getsource(parallel_state.init_distributed_environment)
)
stated = {
kw.arg
for node in ast.walk(ast.parse(body))
if isinstance(node, ast.Call)
and isinstance(node.func, ast.Attribute)
and node.func.attr == "override_permanently"
for kw in node.keywords
}
self.assertIn("world_group", stated)
def test_nothing_reads_a_name_this_build_has_not_stated_yet(self):
"""The stamp is at the end, so a getter called before it answers a name
nothing has written -- a crash at startup, in a process no unit test
runs. What the function may read is what a *previous* call stated, and
that is WORLD alone: a rank belongs to the spawn, and this build does
not get to assume the spawn ran first."""
import ast
import inspect
import textwrap
from sglang.srt.distributed import parallel_state
body = textwrap.dedent(
inspect.getsource(parallel_state.initialize_model_parallel)
)
read = set()
for node in ast.walk(ast.parse(body)):
if isinstance(node, ast.Call) and isinstance(node.func, ast.Name):
name = parallel_state._CONTEXT_NAME_OF.get(node.func.id)
if name is not None:
read.add(name)
self.assertTrue(read, "no getter is called here; this proves nothing")
self.assertEqual(read - {"world_group"}, set())
def test_each_group_is_stated_from_the_global_it_was_built_into(self):
self.assertEqual(_groups_the_build_states(), GROUP_STAMPS)
def test_a_dimension_the_configuration_has_not_got_is_left_unstated(self):
"""`_DCP` is None without decode context parallelism, and every one of
these getters has always refused to answer for a group that was never
built rather than handing back a None to fail on at the collective."""
import ast
import inspect
import textwrap
from sglang.srt.distributed import parallel_state
body = textwrap.dedent(
inspect.getsource(parallel_state.initialize_model_parallel)
)
stamp = next(
node
for node in ast.walk(ast.parse(body))
if isinstance(node, ast.Call)
and isinstance(node.func, ast.Attribute)
and node.func.attr == "override_permanently"
)
self.assertIn("is not None", ast.unparse(stamp))
class TestParallelDelegation(_IsolatedOverrides):
def test_size_rank_delegate_to_canonical_getters(self):
# Patch each getter to a distinct sentinel: a miswired attribute would read
# a different (unpatched) getter and fail.
for i, (attr, target) in enumerate(SIZE_RANK_DELEGATIONS):
sentinel = 1000 + i
with patch(target, return_value=sentinel):
self.assertEqual(
getattr(get_parallel(), attr),
sentinel,
msg=f"{attr} must delegate to {target}",
)
def test_groups_delegate_to_canonical_getters(self):
for attr, target in GROUP_DELEGATIONS:
sentinel = object()
with patch(target, return_value=sentinel):
self.assertIs(
getattr(get_parallel(), attr),
sentinel,
msg=f"{attr} must delegate to {target}",
)
def test_wrapper_holds_no_resolved_state(self):
# __slots__: no __dict__; the only instance state is the override hook.
self.assertFalse(hasattr(get_parallel(), "__dict__"))
@@ -204,50 +286,46 @@ class TestParallelDelegation(_IsolatedOverrides):
class TestTheTwoWorldWidths(_IsolatedOverrides):
"""Two questions about the WORLD group: what it was built at, and what it
has room for.
"""Two questions about the WORLD group: what it was launched at, and what
it has room for.
Neither is stored here. How much of that room is serving after a scale-up
is elastic-EP state, and is asked of the manager that owns it rather than
mirrored onto this namespace.
Both are arithmetic over the configured leaves and are worked out at
publish, so they answer in a process that never builds the group -- which
is where a good half of the readers are. How much of that room is serving
after a scale-up is neither of them: that is elastic-EP state, asked of the
manager that owns it rather than mirrored onto this namespace.
"""
def test_the_launch_width_is_what_the_group_was_built_at(self):
with patch(f"{_PS}.get_world_size", return_value=4):
self.assertEqual(get_parallel().launch_world_size, 4)
def _published(self, **fields):
reset_context()
self.addCleanup(reset_context)
publish(ServerArgs(model_path="dummy", **fields), role="test")
return get_parallel()
def test_the_launch_width_is_a_rank_per_stage_of_each_group(self):
self.assertEqual(self._published(tp_size=4, pp_size=2).launch_world_size, 8)
def test_the_launch_width_spans_the_ranks_a_joiner_came_in_above(self):
"""A scale joiner lays its own groups out at `tp * pp`, while its WORLD
spans the cohort already running underneath it as well."""
parallel = self._published(tp_size=4, pp_size=1, ep_join_rank_offset=8)
self.assertEqual(parallel.launch_world_size, 12)
def test_the_ceiling_is_the_configured_one_when_there_is_one(self):
parallel = get_parallel()
with (
parallel.override(max_ep_size=32),
patch(
f"{_PS}.get_world_size",
side_effect=AssertionError("the built group must not be asked"),
),
):
self.assertEqual(parallel.max_world_size, 32)
self.assertEqual(self._published(tp_size=4, max_ep_size=32).max_world_size, 32)
def test_without_a_configured_ceiling_the_room_is_the_launch_width(self):
parallel = get_parallel()
with (
parallel.override(max_ep_size=None),
patch(f"{_PS}.get_world_size", return_value=8),
):
self.assertEqual(parallel.max_world_size, 8)
parallel = self._published(tp_size=8)
self.assertEqual(parallel.launch_world_size, 8)
self.assertEqual(parallel.max_world_size, 8)
def test_each_width_can_be_stated_on_its_own(self):
"""Stating one must not answer for the other: they are two names."""
parallel = get_parallel()
with (
parallel.override(launch_world_size=2, max_ep_size=None),
patch(
f"{_PS}.get_world_size",
side_effect=AssertionError("the built group must not be asked"),
),
):
parallel = self._published(tp_size=8)
with parallel.override(launch_world_size=2):
self.assertEqual(parallel.launch_world_size, 2)
self.assertEqual(parallel.max_world_size, 2)
with parallel.override(max_ep_size=6):
self.assertEqual(parallel.max_world_size, 8)
with parallel.override(max_world_size=6):
self.assertEqual(parallel.max_world_size, 6)
self.assertEqual(parallel.launch_world_size, 2)
@@ -380,18 +458,20 @@ class TestAttentionRanksComeFromPublish(_IsolatedOverrides):
role="test",
ranks=SpawnRanks(world_rank=5),
)
with patch(
f"{_PS}.get_attn_tensor_model_parallel_rank",
side_effect=AssertionError("no group must be consulted"),
):
with patch.object(_parallel_state(), "_ATTN_TP", None):
self.assertEqual(get_parallel().attn_tp_rank, 1)
self.assertEqual(get_parallel().attn_dp_rank, 1)
def test_without_a_bundle_it_still_asks_the_group(self):
"""Unchanged for every process that publishes without a placement."""
def test_without_a_bundle_a_rank_read_says_what_is_missing(self):
"""There is nothing to fall back to. Deriving one from whatever group
happens to be installed would answer a different question -- where this
process sits in that group, not where the launcher put it."""
publish(ServerArgs(model_path="dummy", tp_size=8), role="test")
with patch(f"{_PS}.get_attn_tensor_model_parallel_rank", return_value=3):
self.assertEqual(get_parallel().attn_tp_rank, 3)
with self.assertRaises(RuntimeError) as caught:
get_parallel().attn_tp_rank
message = str(caught.exception)
self.assertIn("has not been written in this process", message)
self.assertIn("override(attn_tp_rank=...)", message)
class TestStampedRanks(_IsolatedOverrides):
@@ -568,16 +648,14 @@ class TestEveryDeclaredParallelNameIsStatable(_IsolatedOverrides):
The bag carries the declared quotients as well as the operator's
leaves, and both are ahead of the live getter once a configuration is
published: a name in `_LIVE_READS` and in either of them would answer
published: a declared name that is also a leaf would answer
from the getter before publish and from the bag after."""
from sglang.srt.runtime_context import (
_LIVE_READS,
_derived_widths,
_parallel_config_leaves,
)
self.assertEqual(set(_LIVE_READS) & _parallel_config_leaves(), set())
self.assertEqual(set(_LIVE_READS) & set(_derived_widths()), set())
self.assertEqual(set(_derived_widths()) & _parallel_config_leaves(), set())
def test_an_undeclared_name_is_refused(self):
with self.assertRaises(ValueError):
@@ -741,19 +819,43 @@ class TestParallelDCP(_IsolatedOverrides):
self.assertTrue(parallel.dcp_enabled)
self.assertEqual(parallel.attn_dcp_size, 8)
def test_the_dcp_rank_still_reads_the_group(self):
"""A rank is not implied by the configuration, so it reads the group --
gated on a width that is."""
with (
get_parallel().override(tp_size=8, dcp_size=8, dcp_enabled=False),
patch(f"{_PS}.get_dcp_rank", side_effect=AssertionError),
):
self.assertEqual(get_parallel().attn_dcp_rank, 0)
with (
get_parallel().override(tp_size=8, dcp_size=8, dcp_enabled=True),
patch(f"{_PS}.get_dcp_rank", return_value=3),
):
self.assertEqual(get_parallel().attn_dcp_rank, 3)
def _placed(self, world_rank, **fields):
reset_context()
self.addCleanup(reset_context)
publish(
ServerArgs(model_path="dummy", **fields),
role="test",
ranks=SpawnRanks(world_rank=world_rank),
)
return get_parallel()
def test_the_dcp_rank_is_where_the_tp_rank_falls_in_its_slice(self):
"""A DCP group is a contiguous slice of the TP group, so the place in
one is the TP rank folded by the width."""
parallel = self._placed(5, tp_size=8, dcp_size=4)
self.assertEqual(parallel.dcp_rank, 1)
self.assertEqual(parallel.attn_dcp_rank, 1)
def test_the_gated_off_rank_answers_without_a_spawn_bundle(self):
"""Zero for every process when decode context parallelism is off, so a
reader on a path that never publishes a bundle -- a memory pool, an
attention backend built in a unit test -- still gets an answer. It
stood for a group that was never built before, and it has to keep
answering the same way."""
reset_context()
self.addCleanup(reset_context)
publish(ServerArgs(model_path="dummy", tp_size=8), role="test")
self.assertEqual(get_parallel().attn_dcp_rank, 0)
def test_the_dcp_rank_is_gated_on_a_width_the_configuration_carries(self):
"""Zero where decode context parallelism is off, so a reader does not
have to ask whether it is on before asking where it sits -- and the
gated-off name is not answered at all, because no group holds it."""
parallel = self._placed(5, tp_size=8, dcp_size=1)
self.assertFalse(parallel.dcp_enabled)
self.assertEqual(parallel.attn_dcp_rank, 0)
with self.assertRaises(RuntimeError):
parallel.dcp_rank
def test_the_width_does_not_consult_the_platform(self):
with patch("sglang.srt.utils.is_cuda", return_value=False) as is_cuda:
@@ -2074,10 +2176,16 @@ class TestDerivedWidths(_IsolatedOverrides):
{name for name in widths if "world" in name},
set(),
)
parallel = get_parallel()
parallel.override_permanently(attn_tp_size=4)
with patch(f"{_PS}.get_world_size", return_value=9):
self.assertEqual(parallel.launch_world_size, 9)
# Its own arithmetic spans the offset, which is the leaf the quotients
# above are not given and could not account for.
from sglang.srt.runtime_context import launch_world_size_of
self.assertEqual(
launch_world_size_of(
SimpleNamespace(ep_join_rank_offset=8, tp_size=4, pp_size=1)
),
12,
)
def test_the_bare_name_is_gone(self):
"""It answered two questions, so every reader had to remember which.
@@ -2315,15 +2423,27 @@ class TestTheDerivedHalfIsDeclared(CustomTestCase):
f"{name} is declared but no property was installed",
)
def test_the_declared_set_is_what_derive_parallel_widths_produces(self):
"""The declaration is not a second list to keep in step: it names
exactly the quotients the derivation returns."""
from sglang.srt.arg_groups.arg_utils import Derived
from sglang.srt.arg_groups.fields.parallel import Parallel
def test_a_computed_name_names_the_function_that_computes_it(self):
"""The declaration is not a second list to keep in step: a name that
is a function of the leaves points at a function called after it, and
that function existing is the whole of what publish needs."""
import importlib
from sglang.srt.runtime_context import _derived_widths
computed = {n: d.fn for n, d in _derived_widths().items() if d.fn}
self.assertTrue(computed, "nothing is computed from the leaves")
for name, fn in computed.items():
module, _, attr = fn.rpartition(".")
self.assertEqual(attr, f"{name}_of", f"{name} is computed by {attr}")
self.assertTrue(callable(getattr(importlib.import_module(module), attr)))
def test_the_arithmetic_produces_nothing_that_is_not_declared(self):
"""The other side of it: a key the derivation returns and no
declaration names is a width the namespace never answers with, and the
group build re-states it into a name nobody can read."""
from sglang.srt.runtime_context import _derived_widths
declared = {
name for name, value in vars(Parallel).items() if isinstance(value, Derived)
}
produced = set(
derive_parallel_widths(
tp_size=8,
@@ -2335,7 +2455,7 @@ class TestTheDerivedHalfIsDeclared(CustomTestCase):
dcp_enabled=False,
)
)
self.assertEqual(declared, produced)
self.assertEqual(produced - set(_derived_widths()), set())
def test_a_declared_quotient_is_not_a_record_field(self):
"""It has no operator input to preserve, and the record is what crosses
@@ -2405,15 +2525,34 @@ class TestTheAccessorsHaveNoCallersOutsideTheirPackage(CustomTestCase):
has its own parallel state.
"""
#: Not topology. `get_self_pp_group` builds the single-rank group a draft
#: pipeline scope installs, so there is nothing for the context to answer
#: with until the scope has installed it.
#: May have callers. `get_self_pp_group` builds the single-rank group a
#: draft pipeline scope installs, so there is nothing for the context to
#: answer with until the scope has installed it; the other two are not
#: topology at all.
ALLOWED = {
"get_self_pp_group",
"get_default_distributed_backend",
"get_mooncake_transfer_engine",
}
#: Zero callers required, but not deprecated either: the context has no
#: name that answers the same question.
#:
#: The three widths read a group the build does not check against the
#: configuration, so "the group's width" and "the configured width" are two
#: facts -- the MoE-DP group is the attention-CP group when the latter is
#: wider, and the other two are simply not pinned yet. Pinning them in
#: `_WIDTH_AND_GROUP` is what would let them move.
NOT_ANSWERED_BY_THE_CONTEXT = {
"get_moe_data_parallel_world_size",
"get_moe_tensor_parallel_world_size",
"get_dcp_world_size",
# Answers `None` where the context asserts, which is the whole point of
# the caller that wants it.
"get_dcp_group_no_assert",
"get_torch_distributed_pg_options",
}
def _accessors(self):
"""Derived from the source, not listed here: a guard whose subject set
is written by hand stops watching whatever gets added next."""
@@ -2458,6 +2597,139 @@ class TestTheAccessorsHaveNoCallersOutsideTheirPackage(CustomTestCase):
"context cannot answer them",
)
#: How many callers each exempt accessor has outside the defining package.
#: A ratchet, not a description: these may go down and never up, and a name
#: that reaches zero comes off the list. Anything not here must have none.
ALLOWED_CALLERS = {
"get_self_pp_group": 1,
"get_default_distributed_backend": 1,
"get_mooncake_transfer_engine": 6,
}
def test_the_exempt_accessors_do_not_grow_new_callers(self):
"""The zero-caller rule above cannot cover the three that are not
topology, so they get a count instead. Ratchets only turn one way: a
number that has to go up means a new business-code reader of a name the
context should be answering."""
for name, allowed in sorted(self.ALLOWED_CALLERS.items()):
callers = self._callers(name)
self.assertLessEqual(
len(callers),
allowed,
f"{name} grew a caller: {callers}. Read it through "
f"get_parallel() if the context can answer it; if it truly "
f"cannot, lower this number only when one goes away.",
)
def test_every_getter_the_context_answers_is_deprecated(self):
"""The other half of the ratchet: the deprecation set is derived from
the table that maps a context name to the getter behind it, so dropping
a getter out of that table would quietly take it off the list. This
fails if one of them stops being marked."""
from sglang.srt.distributed import parallel_state
marked = set(parallel_state._CONTEXT_NAME_OF)
unclassified = (
self._accessors() - self.ALLOWED - self.NOT_ANSWERED_BY_THE_CONTEXT
)
for name in sorted(unclassified):
if name in marked:
continue
# Not answered by the context and not exempt: a getter that is
# neither is a name with no home, which is what this module exists
# to prevent.
self.assertIn(
name,
marked,
f"{name} is neither deprecated nor listed as exempt -- give it "
"a context name or say here why it has none",
)
def test_calling_one_from_outside_the_package_is_deprecated(self):
"""The getters stay -- they are the definition -- but a call that comes
from outside the package that defines them cannot be redirected by a
scope, so it says what to read instead."""
import warnings
from sglang.srt.distributed import parallel_state
parallel_state._ALREADY_WARNED.discard("get_tensor_model_parallel_rank")
self.addCleanup(
parallel_state._ALREADY_WARNED.discard, "get_tensor_model_parallel_rank"
)
with warnings.catch_warnings(record=True) as seen:
warnings.simplefilter("always")
try:
parallel_state.get_tensor_model_parallel_rank()
except Exception:
pass
messages = [str(w.message) for w in seen]
self.assertTrue(
any("get_parallel().tp_rank" in m for m in messages),
f"expected the replacement to be named, got {messages}",
)
def test_nothing_the_context_answers_with_calls_back_into_the_package(self):
"""The read path reaches the stored group, not the getter that used to
wrap it -- which is what lets the getters be deprecated without the
replacement tripping the warning meant for people who bypass it."""
import inspect
from sglang.srt.distributed import parallel_state
from sglang.srt.runtime_context import ParallelContext, _derived_widths
written = {n for n, d in _derived_widths().items() if not d.fn}
self.assertTrue(written, "no written-at-runtime names; this proves nothing")
self.assertIn("tp_group", written)
# The read path is lookups only -- override, stamp, bag. Nothing in it
# can reach a getter, which is what lets them be deprecated without the
# replacement tripping the warning meant for people who bypass it.
body = inspect.getsource(ParallelContext._read)
self.assertNotIn("_ps()", body)
self.assertNotIn("parallel_state", body)
self.assertNotIn("sglang.srt.runtime_context", parallel_state._EXEMPT_CALLERS)
def test_a_scope_reaches_callers_that_went_straight_to_the_getter(self):
"""The getters read the context, so redirecting a group redirects them
too. PD multiplexing needs exactly this: the in-package readers have to
follow the prefill communicator, not just the ones asking the context."""
from sglang.srt.distributed import parallel_state
stand_in = SimpleNamespace(world_size=1, rank_in_group=0)
with get_parallel().override(tp_group=stand_in):
self.assertIs(get_parallel().tp_group, stand_in)
self.assertIs(parallel_state.get_tp_group(), stand_in)
def test_the_package_that_defines_them_is_not_warned_at(self):
"""`srt/distributed/` keeps calling them: a read there would go through
the context back into itself."""
import warnings
from sglang.srt.distributed import parallel_state
parallel_state._ALREADY_WARNED.discard("get_tensor_model_parallel_rank")
self.addCleanup(
parallel_state._ALREADY_WARNED.discard, "get_tensor_model_parallel_rank"
)
caller = types.ModuleType("sglang.srt.distributed.pretend_internal")
caller.__dict__["call"] = lambda: (
parallel_state.get_tensor_model_parallel_rank()
)
exec(
"def call():\n from sglang.srt.distributed import parallel_state\n"
" return parallel_state.get_tp_group()",
caller.__dict__,
)
with warnings.catch_warnings(record=True) as seen:
warnings.simplefilter("always")
try:
caller.call()
except Exception:
pass
self.assertEqual([str(w.message) for w in seen], [])
def test_the_guard_would_notice_a_caller(self):
"""The subject set is derived, so this checks the search finds a real
call rather than that the list happens to be empty: `get_self_pp_group`
@@ -2571,37 +2843,39 @@ class TestTheTopologyIdentities(CustomTestCase):
def test_a_group_built_at_another_width_is_refused(self):
"""The other end of the same identity: what the configuration says and
what the coordinators were actually built at, checked where the
disagreement is still attributable to the build."""
from sglang.srt.distributed import parallel_state
what the coordinators were actually built at. Stating a group is how
the build hands it over, so that write is where the disagreement
surfaces -- still attributable to the build, and before a collective
runs on the wrong peers."""
from sglang.srt.distributed.parallel_state import GroupCoordinator
self._publish_square()
wrong = GroupCoordinator.__new__(GroupCoordinator)
wrong.world_size = 8
wrong.rank_in_group = 0
with patch.object(parallel_state, "_TP", wrong):
with self.assertRaises(ValueError) as caught:
_validate_parallel(get_parallel(), "group build")
with self.assertRaises(ValueError) as caught:
get_parallel().override_permanently(tp_group=wrong)
message = str(caught.exception)
self.assertIn("set by group build", message)
self.assertIn("tp_group.world_size == tp_size", message)
self.assertIn("built 8, configured 4", message)
# The refused write left nothing behind: the name is unwritten, not
# holding a group no identity accepts.
with self.assertRaises(RuntimeError):
get_parallel().tp_group
def test_a_group_built_at_the_configured_width_is_quiet(self):
from sglang.srt.distributed import parallel_state
from sglang.srt.distributed.parallel_state import GroupCoordinator
self._publish_square()
right = GroupCoordinator.__new__(GroupCoordinator)
right.world_size = 4
right.rank_in_group = 3
with patch.object(parallel_state, "_TP", right):
_validate_parallel(get_parallel(), "group build")
get_parallel().override_permanently(tp_group=right)
self.assertIs(get_parallel().tp_group, right)
def test_a_draft_scope_states_a_consistent_topology(self):
"""The scope narrows four names at once, so the identity applies to it
-- and holds, which is what step lets the guard stay on."""
-- and holds, which is what lets the guard stay on."""
from sglang.srt.distributed import parallel_state
from sglang.srt.distributed.parallel_state import GroupCoordinator
@@ -2609,9 +2883,8 @@ class TestTheTopologyIdentities(CustomTestCase):
group = GroupCoordinator.__new__(GroupCoordinator)
group.world_size = 2
group.rank_in_group = 1
with patch.object(parallel_state, "_TP", group):
with parallel_state.patch_tensor_parallel_group(group, owns_attention=True):
self.assertEqual(get_parallel().attn_tp_size, 2)
with parallel_state.patch_tensor_parallel_group(group, owns_attention=True):
self.assertEqual(get_parallel().attn_tp_size, 2)
class TestWhoAnswersDuringADraftScope(CustomTestCase):
@@ -2650,11 +2923,10 @@ class TestWhoAnswersDuringADraftScope(CustomTestCase):
group = self._single_member_group()
self._two_stage_pipeline()
self.assertEqual(get_parallel().pp_size, 2)
with patch.object(parallel_state, "_PP", group):
with parallel_state.patch_pipeline_parallel_group(group):
self.assertEqual(get_parallel().pp_size, 1)
self.assertEqual(get_parallel().pp_rank, 0)
self.assertIs(get_parallel().pp_group, group)
with parallel_state.patch_pipeline_parallel_group(group):
self.assertEqual(get_parallel().pp_size, 1)
self.assertEqual(get_parallel().pp_rank, 0)
self.assertIs(get_parallel().pp_group, group)
self.assertEqual(get_parallel().pp_size, 2)
self.assertEqual(get_parallel().pp_rank, 1)
@@ -2685,28 +2957,25 @@ class TestWhoAnswersDuringADraftScope(CustomTestCase):
self.assertEqual(get_parallel().attn_tp_size, 2)
group = self._group(world_size=2, rank=1)
with patch.object(parallel_state, "_TP", group):
with parallel_state.patch_tensor_parallel_group(group, owns_attention=True):
parallel = get_parallel()
self.assertEqual(parallel.tp_size, 2)
self.assertEqual(parallel.attn_tp_size, 2)
self.assertEqual(parallel.attn_tp_rank, 1)
self.assertEqual(parallel.attn_dp_size, 1)
self.assertEqual(parallel.attn_dp_rank, 0)
self.assertEqual(parallel.attn_cp_size, 1)
self.assertEqual(parallel.attn_cp_rank, 0)
# `dp_size` is the deployment's replica count, not a property
# of the group being installed, so the scope leaves it alone --
# `require_mlp_tp_gather` asserts on it under dp attention.
self.assertEqual(parallel.dp_size, 2)
# The whole point of stating the rest: the identity the
# override path and the group build both check holds in here.
self.assertEqual(
parallel.tp_size,
parallel.attn_tp_size
* parallel.attn_dp_size
* parallel.attn_cp_size,
)
with parallel_state.patch_tensor_parallel_group(group, owns_attention=True):
parallel = get_parallel()
self.assertEqual(parallel.tp_size, 2)
self.assertEqual(parallel.attn_tp_size, 2)
self.assertEqual(parallel.attn_tp_rank, 1)
self.assertEqual(parallel.attn_dp_size, 1)
self.assertEqual(parallel.attn_dp_rank, 0)
self.assertEqual(parallel.attn_cp_size, 1)
self.assertEqual(parallel.attn_cp_rank, 0)
# `dp_size` is the deployment's replica count, not a property of
# the group being installed, so the scope leaves it alone --
# `require_mlp_tp_gather` asserts on it under dp attention.
self.assertEqual(parallel.dp_size, 2)
# The whole point of stating the rest: the identity the override
# path and the group build both check holds in here.
self.assertEqual(
parallel.tp_size,
parallel.attn_tp_size * parallel.attn_dp_size * parallel.attn_cp_size,
)
self.assertEqual(get_parallel().attn_dp_size, 2)
self.assertEqual(get_parallel().dp_size, 2)
@@ -2790,9 +3059,8 @@ class TestWhoAnswersDuringADraftScope(CustomTestCase):
self._two_stage_pipeline()
self.assertEqual(get_parallel().pp_size, 2)
group = self._single_member_group()
with patch.object(parallel_state, "_PP", group):
with parallel_state.patch_pipeline_parallel_group(group):
checker = WeightChecker(get_model=lambda: None)
with parallel_state.patch_pipeline_parallel_group(group):
checker = WeightChecker(get_model=lambda: None)
# The scope has closed and the context answers the target's shape again.
self.assertEqual(get_parallel().pp_size, 2)