From 986cb8268b8bfdc1f937251249943eb5507adbfa Mon Sep 17 00:00:00 2001 From: Cheng Wan <54331508+ch-wan@users.noreply.github.com> Date: Wed, 30 Sep 2026 13:28:50 -0700 Subject: [PATCH] [Refactor] Enter draft TP scopes by attention ownership and drop ModelRunner.tp_group (#41817) --- .../eagle_draft_npu_graph_runner.py | 3 +- .../npu/graph_runner/npu_cudagraph_backend.py | 3 +- .../graph_runner/xpu_full_graph_backend.py | 3 +- python/sglang/srt/managers/tp_worker.py | 2 +- .../srt/model_executor/cpu_graph_runner.py | 6 +- .../sglang/srt/model_executor/model_runner.py | 17 +++-- .../srt/model_executor/runner/base_runner.py | 2 +- .../runner/decode_cuda_graph_runner.py | 2 +- .../runner/flashinfer_autotune.py | 4 +- .../runner/prefill_cuda_graph_runner.py | 2 +- .../breakable_cuda_graph_backend.py | 3 +- .../runner_backend/full_cuda_graph_backend.py | 3 +- .../tc_piecewise_cuda_graph_backend.py | 2 +- .../srt/speculative/dflash_worker_v2.py | 63 +++++-------------- .../dspark_components/dspark_draft.py | 6 +- .../dspark_components/dspark_worker_v2.py | 9 ++- .../sglang/srt/speculative/eagle_worker_v2.py | 55 +++------------- .../speculative/frozen_kv_mtp_worker_v2.py | 31 ++------- .../multi_layer_eagle_worker_v2.py | 16 +---- python/sglang/srt/speculative/spec_utils.py | 21 ++++--- .../srt/speculative/standalone_worker_v2.py | 37 +---------- .../speculative_draft_extend_runner.py | 3 +- .../runner_modes/speculative_draft_runner.py | 5 +- .../runner/test_flashinfer_autotune_sync.py | 8 +-- .../test_full_cuda_graph_backend.py | 4 +- .../unit/spec/test_dflash_logits.py | 2 +- .../spec/test_dp_spec_prefill_coordination.py | 11 +++- .../unit/spec/test_draft_tp_context.py | 58 +++++++++++++++++ test/registered/unit/test_runtime_context.py | 4 +- 29 files changed, 161 insertions(+), 224 deletions(-) create mode 100644 test/registered/unit/spec/test_draft_tp_context.py diff --git a/python/sglang/srt/hardware_backend/npu/graph_runner/eagle_draft_npu_graph_runner.py b/python/sglang/srt/hardware_backend/npu/graph_runner/eagle_draft_npu_graph_runner.py index 113eed0f50..82531343e8 100644 --- a/python/sglang/srt/hardware_backend/npu/graph_runner/eagle_draft_npu_graph_runner.py +++ b/python/sglang/srt/hardware_backend/npu/graph_runner/eagle_draft_npu_graph_runner.py @@ -23,6 +23,7 @@ from sglang.srt.configs.model_config import ( is_deepseek_dsa, is_deepseek_v4, ) +from sglang.srt.runtime_context import get_parallel from sglang.srt.speculative.eagle_draft_cuda_graph_runner import ( EAGLEDraftCudaGraphRunner, ) @@ -82,7 +83,7 @@ class EAGLEDraftNpuGraphRunner(EAGLEDraftCudaGraphRunner): torch.distributed.all_reduce( decision, op=torch.distributed.ReduceOp.MIN, - group=self.model_runner.tp_group.device_group, + group=get_parallel().tp_group.device_group, ) return bool(decision.item()) diff --git a/python/sglang/srt/hardware_backend/npu/graph_runner/npu_cudagraph_backend.py b/python/sglang/srt/hardware_backend/npu/graph_runner/npu_cudagraph_backend.py index 26158b08be..f666c8dfe5 100644 --- a/python/sglang/srt/hardware_backend/npu/graph_runner/npu_cudagraph_backend.py +++ b/python/sglang/srt/hardware_backend/npu/graph_runner/npu_cudagraph_backend.py @@ -28,6 +28,7 @@ from sglang.srt.model_executor.runner.shape_key import ShapeKey from sglang.srt.model_executor.runner_backend.base_cuda_graph_backend import ( BaseCudaGraphBackend, ) +from sglang.srt.runtime_context import get_parallel from sglang.srt.utils import empty_context, get_bool_env_var from sglang.srt.utils.torch_memory_saver_adapter import TorchMemorySaverAdapter @@ -54,7 +55,7 @@ class NPUCudaGraphBackend(BaseCudaGraphBackend): self._pool = None self._device_module = cuda_graph_runner.device_module self._device_id = self._device_module.current_device() - self._tp_group = cuda_graph_runner.model_runner.tp_group + self._tp_group = get_parallel().tp_group self._capture_stream = None self._memory_saver_adapter: Optional[Any] = TorchMemorySaverAdapter.create( enable=enable_memory_saver diff --git a/python/sglang/srt/hardware_backend/xpu/graph_runner/xpu_full_graph_backend.py b/python/sglang/srt/hardware_backend/xpu/graph_runner/xpu_full_graph_backend.py index 9c59fccf73..d3e08f76ea 100644 --- a/python/sglang/srt/hardware_backend/xpu/graph_runner/xpu_full_graph_backend.py +++ b/python/sglang/srt/hardware_backend/xpu/graph_runner/xpu_full_graph_backend.py @@ -22,6 +22,7 @@ from sglang.srt.model_executor.runner_backend.base_cuda_graph_backend import ( from sglang.srt.model_executor.runner_utils.pool import ( get_or_create_global_graph_memory_pool, ) +from sglang.srt.runtime_context import get_parallel if TYPE_CHECKING: from sglang.srt.model_executor.forward_batch_info import ForwardBatch @@ -41,7 +42,7 @@ class FullXPUGraphBackend(BaseCudaGraphBackend): self._outputs: Dict[Any, Any] = {} self._pool = None self._device_module = cuda_graph_runner.device_module - self._tp_group = cuda_graph_runner.model_runner.tp_group + self._tp_group = get_parallel().tp_group self._capture_stream: Optional[torch.xpu.Stream] = None @contextmanager diff --git a/python/sglang/srt/managers/tp_worker.py b/python/sglang/srt/managers/tp_worker.py index b11ee1f995..e5c3bc36a8 100644 --- a/python/sglang/srt/managers/tp_worker.py +++ b/python/sglang/srt/managers/tp_worker.py @@ -366,7 +366,7 @@ class TpModelWorker(BaseTpWorker): # the set of ranks holding a draft worker. The draft worker is # constructed with pp_rank=0, so derive the caller's global rank # from the TP group rather than tp_size * pp_rank + tp_rank. - tp_group = self.model_runner.tp_group + tp_group = get_parallel().tp_group self.random_seed = broadcast_pyobj( [get_device().random_seed], tp_group.ranks[tp_group.rank_in_group], diff --git a/python/sglang/srt/model_executor/cpu_graph_runner.py b/python/sglang/srt/model_executor/cpu_graph_runner.py index 7abe4da166..3f055ee49e 100644 --- a/python/sglang/srt/model_executor/cpu_graph_runner.py +++ b/python/sglang/srt/model_executor/cpu_graph_runner.py @@ -750,7 +750,7 @@ class CPUGraphRunner: self.model_runner.model, bs in self.capture_bs, num_tokens=bs * self.captured_req_width, - tp_group=self.model_runner.tp_group, + tp_group=get_parallel().tp_group, ) as forward: graph, output_buffers = self.capture_one_batch_size( bs, forward, skip_cross_attention=True @@ -853,7 +853,7 @@ class CPUGraphRunner: forward_batch.spec_info, ) with torch.no_grad(): - self.model_runner.tp_group.barrier() + get_parallel().tp_group.barrier() self.model_runner.model.forward( forward_batch.input_ids, forward_batch.positions, @@ -875,7 +875,7 @@ class CPUGraphRunner: with torch.no_grad(): for _ in range(2): - self.model_runner.tp_group.barrier() + get_parallel().tp_group.barrier() out = run_once() # Save the captured forward_batch in the appropriate dict if skip_cross_attention: diff --git a/python/sglang/srt/model_executor/model_runner.py b/python/sglang/srt/model_executor/model_runner.py index 1fc285c697..b66e633704 100644 --- a/python/sglang/srt/model_executor/model_runner.py +++ b/python/sglang/srt/model_executor/model_runner.py @@ -497,7 +497,7 @@ class ModelRunner: parallel = get_parallel() join_effective_ep_size = parallel.ep_join_rank_offset + parallel.tp_size - dist.barrier(group=self.tp_group.cpu_group) + dist.barrier(group=parallel.tp_group.cpu_group) if parallel.tp_rank == 0: register_scale_cohort( parallel.ep_join_rank_offset, @@ -966,7 +966,7 @@ class ModelRunner: tp_group=( get_parallel().attn_tp_group.cpu_group if get_parallel().enable_dp_attention - else self.tp_group.cpu_group + else get_parallel().tp_group.cpu_group ), host_to_device_ratio=hisparse_cfg.host_to_device_ratio, swap_in_block_size=hisparse_cfg.swap_in_block_size, @@ -1178,13 +1178,10 @@ class ModelRunner: self.pre_model_load_memory = bootstrap.measure_pre_model_load_memory( device=self.device, is_draft_worker=self.is_draft_worker ) - # A draft runner is used outside the scope it is built in: speculative - # workers re-enter that scope through its TP group, and draft forwards - # read its PP group without the pipeline scope. Keep both groups; read - # every other placement value from the context where it is used. - parallel = get_parallel() - self.tp_group = parallel.tp_group - self.pp_group = parallel.pp_group + # Draft forwards run without the pipeline scope the draft is built in, + # so keep the PP group it was built with; read every other placement + # value from the context where it is used. + self.pp_group = get_parallel().pp_group def load_model(self): tic_total = time.perf_counter() @@ -2317,7 +2314,7 @@ class ModelRunner: return recovered = maybe_recover_ep_ranks( - tp_group=self.tp_group, + tp_group=get_parallel().tp_group, eplb_manager=self.eplb_manager, model_config=self.model_config, moe_ep_rank=self._elastic_global_rank(), diff --git a/python/sglang/srt/model_executor/runner/base_runner.py b/python/sglang/srt/model_executor/runner/base_runner.py index 03fcf09170..b7d5ebdefd 100644 --- a/python/sglang/srt/model_executor/runner/base_runner.py +++ b/python/sglang/srt/model_executor/runner/base_runner.py @@ -741,7 +741,7 @@ class BaseRunner(ABC): return logits_output_or_pp_proxy_tensors torch.get_device_module(mr.device).synchronize() - mr.tp_group.barrier() + get_parallel().tp_group.barrier() with forward_context(ForwardContext(attn_backend=mr.attn_backend)): with run_ctx or empty_context(): run_once() diff --git a/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py b/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py index 88c73d5370..1846c7b3b2 100644 --- a/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py @@ -1135,7 +1135,7 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner): self.model_runner.model, bs in self.compile_bs, num_tokens=bs * self.captured_req_width, - tp_group=self.model_runner.tp_group, + tp_group=get_parallel().tp_group, ) as forward: self.capture_one_shape( bs, diff --git a/python/sglang/srt/model_executor/runner/flashinfer_autotune.py b/python/sglang/srt/model_executor/runner/flashinfer_autotune.py index 70e4fdf4cb..794fdaffd1 100644 --- a/python/sglang/srt/model_executor/runner/flashinfer_autotune.py +++ b/python/sglang/srt/model_executor/runner/flashinfer_autotune.py @@ -254,7 +254,7 @@ def flashinfer_autotune_context(model_runner: ModelRunner, *, run_lm_head: bool) mr = model_runner cache_path = flashinfer_autotune_cache_path(mr) - sync_group = _autotune_tactic_sync_group(mr.tp_group) + sync_group = _autotune_tactic_sync_group(get_parallel().tp_group) reuse_cache = envs.SGLANG_FLASHINFER_AUTOTUNE_CACHE.get() if reuse_cache: autotune_cache = cache_path @@ -425,7 +425,7 @@ def maybe_flashinfer_autotune_extend( try: run_flashinfer_autotune_forward(mr, forward_fn, run_lm_head=False) except torch.OutOfMemoryError: - if _autotune_tactic_sync_group(mr.tp_group) is not None: + if _autotune_tactic_sync_group(get_parallel().tp_group) is not None: # Tuning is collective: this rank has stopped reducing while its # peers wait on the next tactic, so skipping the pass would hang # them. Fail instead of degrading alone. diff --git a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py index 70f3a0e8d6..2def2fd4aa 100644 --- a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py @@ -616,7 +616,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): # --- capture -------------------------------------------------- self.device_module.synchronize() - self.model_runner.tp_group.barrier() + get_parallel().tp_group.barrier() self.capture() self.raw_num_tokens = 0 diff --git a/python/sglang/srt/model_executor/runner_backend/breakable_cuda_graph_backend.py b/python/sglang/srt/model_executor/runner_backend/breakable_cuda_graph_backend.py index b36c6373d2..2f446cbe5e 100644 --- a/python/sglang/srt/model_executor/runner_backend/breakable_cuda_graph_backend.py +++ b/python/sglang/srt/model_executor/runner_backend/breakable_cuda_graph_backend.py @@ -45,6 +45,7 @@ from sglang.srt.model_executor.runner_utils.pool import ( graph_pool_capture_scope, graph_pool_replay_scope, ) +from sglang.srt.runtime_context import get_parallel from sglang.srt.utils import get_bool_env_var from sglang.srt.utils.torch_memory_saver_adapter import TorchMemorySaverAdapter @@ -75,7 +76,7 @@ class BreakableCudaGraphBackend(DedupedCudaGraphMixin, BaseCudaGraphBackend): self._capture_inputs: Dict[Any, Any] = {} self._pool = None self._device_module = cuda_graph_runner.device_module - self._tp_group = cuda_graph_runner.model_runner.tp_group + self._tp_group = get_parallel().tp_group self._capture_stream: Optional[torch.cuda.Stream] = None self._debug_eager = debug_eager self._shared_output_buffer: Optional[Any] = None diff --git a/python/sglang/srt/model_executor/runner_backend/full_cuda_graph_backend.py b/python/sglang/srt/model_executor/runner_backend/full_cuda_graph_backend.py index a86a74722d..a72528c612 100644 --- a/python/sglang/srt/model_executor/runner_backend/full_cuda_graph_backend.py +++ b/python/sglang/srt/model_executor/runner_backend/full_cuda_graph_backend.py @@ -36,6 +36,7 @@ from sglang.srt.model_executor.runner_utils.pool import ( graph_pool_capture_scope, graph_pool_replay_scope, ) +from sglang.srt.runtime_context import get_parallel from sglang.srt.utils import get_bool_env_var from sglang.srt.utils.torch_memory_saver_adapter import TorchMemorySaverAdapter @@ -91,7 +92,7 @@ class FullCudaGraphBackend(BaseCudaGraphBackend): self._pool = None self._cuda_graph_runner = cuda_graph_runner self._device_module = cuda_graph_runner.device_module - self._tp_group = cuda_graph_runner.model_runner.tp_group + self._tp_group = get_parallel().tp_group self._capture_stream: Optional[torch.cuda.Stream] = None self._precarve = GraphPoolPrecarve() self._reuse_output_buffer = reuse_output_buffer diff --git a/python/sglang/srt/model_executor/runner_backend/tc_piecewise_cuda_graph_backend.py b/python/sglang/srt/model_executor/runner_backend/tc_piecewise_cuda_graph_backend.py index 130a00e7d1..e48364cb07 100644 --- a/python/sglang/srt/model_executor/runner_backend/tc_piecewise_cuda_graph_backend.py +++ b/python/sglang/srt/model_executor/runner_backend/tc_piecewise_cuda_graph_backend.py @@ -95,7 +95,7 @@ class TcPiecewiseCudaGraphBackend(BaseCudaGraphBackend): model_runner = cuda_graph_runner.model_runner self._pool = None self._device_module = cuda_graph_runner.device_module - self._tp_group = model_runner.tp_group + self._tp_group = get_parallel().tp_group self._capture_stream: Optional[torch.cuda.Stream] = None self._compile_config: CompilationConfig = self.build_compilation_config( model_runner.server_args diff --git a/python/sglang/srt/speculative/dflash_worker_v2.py b/python/sglang/srt/speculative/dflash_worker_v2.py index 7604f0207c..50edb5bf5b 100644 --- a/python/sglang/srt/speculative/dflash_worker_v2.py +++ b/python/sglang/srt/speculative/dflash_worker_v2.py @@ -94,7 +94,6 @@ from sglang.srt.speculative.spec_utils import ( draft_tp_context, ) from sglang.srt.utils import is_cuda, is_hip, is_npu, is_xpu -from sglang.srt.utils.common import empty_context _is_npu = is_npu() @@ -406,19 +405,12 @@ class DFlashWorkerV2(BaseSpecWorker): ) # Under dp attention, the draft worker runs on the per-DP attn-TP - # group, independent of idle peer DP ranks. - self.draft_tp_context = ( - draft_tp_context if get_parallel().enable_dp_attention else empty_context - ) - # Use the same attention topology during draft construction and execution. + # group, independent of idle peer DP ranks; it is built and run under + # the same placement. self.draft_owns_attention = get_parallel().enable_dp_attention - if self.draft_owns_attention: - draft_init_ctx = draft_tp_context( - get_parallel().attn_tp_group, owns_attention=True - ) - else: - draft_init_ctx = empty_context() - with draft_pp_context(), draft_init_ctx: + # Inside the draft scope the context answers the draft's narrowed rank. + self._target_tp_rank = get_parallel().tp_rank + with draft_pp_context(), draft_tp_context(self.draft_owns_attention): bundle = build_draft_tp_worker( server_args=server_args, gpu_id=gpu_id, @@ -621,10 +613,7 @@ class DFlashWorkerV2(BaseSpecWorker): def init_attention_backends(self): with ( draft_pp_context(), - self.draft_tp_context( - self.draft_model_runner.tp_group, - owns_attention=self.draft_owns_attention, - ), + draft_tp_context(self.draft_owns_attention), ): self._draft_worker.init_attention_backends() self._need_mamba_verify_commit = mambaish_config( @@ -637,10 +626,7 @@ class DFlashWorkerV2(BaseSpecWorker): def init_cuda_graphs(self): with ( draft_pp_context(), - self.draft_tp_context( - self.draft_model_runner.tp_group, - owns_attention=self.draft_owns_attention, - ), + draft_tp_context(self.draft_owns_attention), ): capture_decode_cuda_graph = ( get_exec().graph.cuda_graph_config.decode.backend != Backend.DISABLED @@ -661,7 +647,7 @@ class DFlashWorkerV2(BaseSpecWorker): # shared graph capture/replay; keep the draft eager under dp # attention. capture_decode_cuda_graph = False - if self.model_runner.tp_group.rank_in_group == 0: + if self._target_tp_rank == 0: logger.warning( "Disable DFLASH draft cuda graph because dp attention " "is enabled (draft runs eager)." @@ -806,7 +792,7 @@ class DFlashWorkerV2(BaseSpecWorker): def _maybe_build_draft_sampler(self): def _eager(reason): - if self.model_runner.tp_group.rank_in_group == 0: + if self._target_tp_rank == 0: logger.info("DFLASH draft greedy head kept eager (reason=%s).", reason) return None @@ -832,7 +818,7 @@ class DFlashWorkerV2(BaseSpecWorker): if not head_supported: return _eager("unsupported quantized lm_head") self.draft_model.lm_head = lm_head - if self.model_runner.tp_group.rank_in_group == 0: + if self._target_tp_rank == 0: logger.info( "DFLASH selector decode folded into the draft cuda graph " "(sampling_enabled=%s).", @@ -867,7 +853,7 @@ class DFlashWorkerV2(BaseSpecWorker): embed_proj = self.draft_model.embed_proj if prefix_gru is None or embed_proj is None: return _eager("Domino projector modules are unavailable") - if self.model_runner.tp_group.rank_in_group == 0: + if self._target_tp_rank == 0: logger.info( "DFLASH Domino rollout folded into the draft cuda graph (tp=%s).", int(tp_group.world_size), @@ -906,7 +892,7 @@ class DFlashWorkerV2(BaseSpecWorker): return _eager("added vocab") num_org = int(shard.num_org_elements) org_vocab_start = int(shard.org_vocab_start_index) - if self.model_runner.tp_group.rank_in_group == 0: + if self._target_tp_rank == 0: logger.info( "DFLASH draft greedy head folded into the draft cuda graph (tp=%d).", tp_group.world_size, @@ -1857,10 +1843,7 @@ class DFlashWorkerV2(BaseSpecWorker): with ( torch.inference_mode(), - self.draft_tp_context( - self.draft_model_runner.tp_group, - owns_attention=self.draft_owns_attention, - ), + draft_tp_context(self.draft_owns_attention), ): ctx_hidden = self.draft_model.project_target_hidden(target_hidden) if self.lilicorr is not None: @@ -2567,10 +2550,7 @@ class DFlashWorkerV2(BaseSpecWorker): with ( torch.inference_mode(), - self.draft_tp_context( - self.draft_model_runner.tp_group, - owns_attention=self.draft_owns_attention, - ), + draft_tp_context(self.draft_owns_attention), ): draft_out = self.draft_model_runner.forward(forward_batch) draft_logits_output = draft_out.logits_output @@ -2638,10 +2618,7 @@ class DFlashWorkerV2(BaseSpecWorker): self._draft_sampler.q_out[:bs], ) elif self.selector is not None: - with self.draft_tp_context( - self.draft_model_runner.tp_group, - owns_attention=self.draft_owns_attention, - ): + with draft_tp_context(self.draft_owns_attention): draft_next = self._propose_selector_block( draft_logits_output=draft_logits_output, bs=bs, @@ -2667,10 +2644,7 @@ class DFlashWorkerV2(BaseSpecWorker): draft_hidden = draft_logits_output.hidden_states if draft_hidden is None: raise RuntimeError("DFLASH draft model returned no hidden states.") - with self.draft_tp_context( - self.draft_model_runner.tp_group, - owns_attention=self.draft_owns_attention, - ): + with draft_tp_context(self.draft_owns_attention): draft_next, lilicorr_candidate_ids, lilicorr_q_rows = ( propose_lilicorr_block( head=self.lilicorr, @@ -2691,10 +2665,7 @@ class DFlashWorkerV2(BaseSpecWorker): if draft_hidden is None: raise RuntimeError("DFLASH draft model returned no hidden states.") draft_hidden = draft_hidden.view(bs, int(self.block_size), -1) - with self.draft_tp_context( - self.draft_model_runner.tp_group, - owns_attention=self.draft_owns_attention, - ): + with draft_tp_context(self.draft_owns_attention): draft_next = self._greedy_sample_from_vocab_parallel_head( hidden_states=draft_hidden[:, 1:, :].reshape( -1, draft_hidden.shape[-1] diff --git a/python/sglang/srt/speculative/dspark_components/dspark_draft.py b/python/sglang/srt/speculative/dspark_components/dspark_draft.py index 872eb33117..1afcac4879 100644 --- a/python/sglang/srt/speculative/dspark_components/dspark_draft.py +++ b/python/sglang/srt/speculative/dspark_components/dspark_draft.py @@ -1,7 +1,6 @@ from __future__ import annotations import logging -from contextlib import nullcontext from typing import Optional import msgspec @@ -19,7 +18,6 @@ from sglang.srt.model_executor.forward_batch_info import ( ForwardMode, enable_num_token_non_padded, ) -from sglang.srt.runtime_context import get_parallel from sglang.srt.speculative.dflash_info_v2 import DFlashDraftInputV2 from sglang.srt.speculative.draft_worker_common import make_draft_input_v2 from sglang.srt.speculative.dspark_components.dspark_planner import VerifyWindow @@ -235,9 +233,7 @@ class DraftBlockProposer: self._draft_sampler = draft_sampler def _base_logits_context(self): - if self._dp_moe_sync: - return draft_tp_context(get_parallel().attn_tp_group, owns_attention=True) - return nullcontext() + return draft_tp_context(self._dp_moe_sync) def propose( self, diff --git a/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py b/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py index 1751d53c80..ec34bf5789 100644 --- a/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py +++ b/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py @@ -1,5 +1,4 @@ import logging -from contextlib import nullcontext from typing import TYPE_CHECKING, Callable, Optional, Protocol, runtime_checkable import torch @@ -181,6 +180,8 @@ class DSparkWorkerV2(BaseSpecWorker): "MoE-under-DP all-reduce." ) + # Inside the draft scope the context answers the draft's narrowed rank. + self._target_tp_rank = get_parallel().tp_rank with draft_pp_context(), self._draft_context(): bundle = build_draft_tp_worker( server_args=server_args, @@ -431,9 +432,7 @@ class DSparkWorkerV2(BaseSpecWorker): return getattr(self.target_worker, name) def _draft_context(self): - if self._draft_dp_context_enabled: - return draft_tp_context(get_parallel().attn_tp_group, owns_attention=True) - return nullcontext() + return draft_tp_context(self._draft_dp_context_enabled) def alloc_memory_pool( self, @@ -516,7 +515,7 @@ class DSparkWorkerV2(BaseSpecWorker): gamma=self.gamma, max_bs=max(get_exec().graph.cuda_graph_config.decode.bs), device=self.device, - tp_rank=self.model_runner.tp_group.rank_in_group, + tp_rank=self._target_tp_rank, tp_sync=self._tp_sync, available_memory_gb=available_memory_gb, confidence_fn=( diff --git a/python/sglang/srt/speculative/eagle_worker_v2.py b/python/sglang/srt/speculative/eagle_worker_v2.py index 993b53f637..4a8973bffc 100644 --- a/python/sglang/srt/speculative/eagle_worker_v2.py +++ b/python/sglang/srt/speculative/eagle_worker_v2.py @@ -120,7 +120,6 @@ from sglang.srt.utils.async_probe import ( maybe_detect_oob, ) from sglang.srt.utils.common import ( - empty_context, fast_topk, get_available_gpu_memory, is_cpu, @@ -190,12 +189,8 @@ class EagleDraftWorker(EagleDraftWorkerBase): get_parallel().enable_dp_attention and self.speculative_algorithm.is_eagle3() ) - if self.draft_owns_attention: - ctx = draft_tp_context(get_parallel().attn_tp_group, owns_attention=True) - else: - ctx = empty_context() with ( - ctx, + draft_tp_context(self.draft_owns_attention), draft_pp_context(), speculative_moe_backend_context(), speculative_moe_a2a_backend_context(), @@ -216,9 +211,6 @@ class EagleDraftWorker(EagleDraftWorkerBase): self._init_dsa_index_share_state() # Eager draft-extend seed buffer (graph paths use their own static ones). self.dsa_extend_topk_buf: Optional[torch.Tensor] = None - self.draft_tp_context = ( - draft_tp_context if get_parallel().enable_dp_attention else empty_context - ) self.tree_mask_mode = default_tree_mask_mode() self.plan_stream, self.plan_stream_ctx = get_plan_stream(self.device) @@ -259,10 +251,7 @@ class EagleDraftWorker(EagleDraftWorkerBase): def init_attention_backends(self): with ( draft_pp_context(), - self.draft_tp_context( - self.draft_runner.tp_group, - owns_attention=self.draft_owns_attention, - ), + draft_tp_context(self.draft_owns_attention), speculative_moe_backend_context(), speculative_moe_a2a_backend_context(), ): @@ -272,10 +261,7 @@ class EagleDraftWorker(EagleDraftWorkerBase): def init_cuda_graphs(self): with ( draft_pp_context(), - self.draft_tp_context( - self.draft_runner.tp_group, - owns_attention=self.draft_owns_attention, - ), + draft_tp_context(self.draft_owns_attention), speculative_moe_backend_context(), speculative_moe_a2a_backend_context(), ): @@ -1283,10 +1269,7 @@ class EAGLEWorkerV2(BaseSpecWorker): # Build adaptive runtime states after target and draft backends exist. if self.adaptive_controller is not None: with ( - self._draft_worker.draft_tp_context( - self._draft_worker.draft_runner.tp_group, - owns_attention=self._draft_worker.draft_owns_attention, - ), + draft_tp_context(self._draft_worker.draft_owns_attention), speculative_moe_backend_context(), speculative_moe_a2a_backend_context(), ): @@ -1369,10 +1352,7 @@ class EAGLEWorkerV2(BaseSpecWorker): verify_input = self._build_trivial_verify_input(batch) else: with ( - self.draft_worker.draft_tp_context( - self.draft_worker.draft_runner.tp_group, - owns_attention=self.draft_worker.draft_owns_attention, - ), + draft_tp_context(self.draft_worker.draft_owns_attention), speculative_moe_backend_context(), speculative_moe_a2a_backend_context(), spec_stage_span("draft"), @@ -1395,10 +1375,7 @@ class EAGLEWorkerV2(BaseSpecWorker): self._stub_skipped_draft_extend(batch, batch_output) else: with ( - self.draft_worker.draft_tp_context( - self.draft_worker.draft_runner.tp_group, - owns_attention=self.draft_worker.draft_owns_attention, - ), + draft_tp_context(self.draft_worker.draft_owns_attention), speculative_moe_backend_context(), speculative_moe_a2a_backend_context(), spec_stage_span("draft_extend"), @@ -1428,10 +1405,7 @@ class EAGLEWorkerV2(BaseSpecWorker): batch.seq_lens_cpu = batch_output.new_seq_lens.to("cpu") batch.seq_lens_sum = int(batch.seq_lens_cpu.sum()) with ( - self.draft_worker.draft_tp_context( - self.draft_worker.draft_runner.tp_group, - owns_attention=self.draft_worker.draft_owns_attention, - ), + draft_tp_context(self.draft_worker.draft_owns_attention), speculative_moe_backend_context(), speculative_moe_a2a_backend_context(), spec_stage_span("draft"), @@ -1489,10 +1463,7 @@ class EAGLEWorkerV2(BaseSpecWorker): # Draft prefill with ( - self.draft_worker.draft_tp_context( - self.draft_worker.draft_runner.tp_group, - owns_attention=self.draft_worker.draft_owns_attention, - ), + draft_tp_context(self.draft_worker.draft_owns_attention), speculative_moe_backend_context(), speculative_moe_a2a_backend_context(), spec_stage_span("draft_extend"), @@ -1547,10 +1518,7 @@ class EAGLEWorkerV2(BaseSpecWorker): ) plan.apply(draft_batch, "draft", rank, local_only=draft_local_only) with ( - self.draft_worker.draft_tp_context( - self.draft_worker.draft_runner.tp_group, - owns_attention=self.draft_worker.draft_owns_attention, - ), + draft_tp_context(self.draft_worker.draft_owns_attention), speculative_moe_backend_context(), speculative_moe_a2a_backend_context(), spec_stage_span("draft"), @@ -1574,10 +1542,7 @@ class EAGLEWorkerV2(BaseSpecWorker): on_publish(result.new_seq_lens) plan.apply(batch, "draft_extend", rank, local_only=draft_local_only) with ( - self.draft_worker.draft_tp_context( - self.draft_worker.draft_runner.tp_group, - owns_attention=self.draft_worker.draft_owns_attention, - ), + draft_tp_context(self.draft_worker.draft_owns_attention), speculative_moe_backend_context(), speculative_moe_a2a_backend_context(), spec_stage_span("draft_extend"), diff --git a/python/sglang/srt/speculative/frozen_kv_mtp_worker_v2.py b/python/sglang/srt/speculative/frozen_kv_mtp_worker_v2.py index fa73bf2071..69021e7e18 100644 --- a/python/sglang/srt/speculative/frozen_kv_mtp_worker_v2.py +++ b/python/sglang/srt/speculative/frozen_kv_mtp_worker_v2.py @@ -45,7 +45,6 @@ from sglang.srt.model_executor.pool_configurator import MemoryPoolConfig from sglang.srt.runtime_context import ( attention_backends, get_device, - get_parallel, get_schedule, get_spec, ) @@ -79,7 +78,7 @@ from sglang.srt.speculative.spec_utils import ( select_top_k_tokens, spec_stage_span, ) -from sglang.srt.utils import empty_context, get_available_gpu_memory +from sglang.srt.utils import get_available_gpu_memory from sglang.srt.utils.async_probe import ( maybe_detect_inf, maybe_detect_nan, @@ -170,9 +169,6 @@ class FrozenKVMTPDraftWorker(EagleDraftWorkerBase, TpModelWorker): # Retain the target's attention topology when swapping TP groups. self.draft_owns_attention = False - self.draft_tp_context = ( - draft_tp_context if get_parallel().enable_dp_attention else empty_context - ) self.draft_attn_backend = None self.cuda_graph_runner = None @@ -210,10 +206,7 @@ class FrozenKVMTPDraftWorker(EagleDraftWorkerBase, TpModelWorker): def init_attention_backends(self): with ( draft_pp_context(), - self.draft_tp_context( - self.draft_model_runner.tp_group, - owns_attention=self.draft_owns_attention, - ), + draft_tp_context(self.draft_owns_attention), speculative_moe_backend_context(), speculative_moe_a2a_backend_context(), ): @@ -224,10 +217,7 @@ class FrozenKVMTPDraftWorker(EagleDraftWorkerBase, TpModelWorker): def init_cuda_graphs(self): with ( draft_pp_context(), - self.draft_tp_context( - self.draft_model_runner.tp_group, - owns_attention=self.draft_owns_attention, - ), + draft_tp_context(self.draft_owns_attention), speculative_moe_backend_context(), speculative_moe_a2a_backend_context(), ): @@ -774,10 +764,7 @@ class FrozenKVMTPWorkerV2(EAGLEWorkerV2): # Draft prefill seed (no forward). with ( - self.draft_worker.draft_tp_context( - self.draft_worker.draft_runner.tp_group, - owns_attention=self.draft_worker.draft_owns_attention, - ), + draft_tp_context(self.draft_worker.draft_owns_attention), speculative_moe_backend_context(), speculative_moe_a2a_backend_context(), spec_stage_span("draft_extend"), @@ -797,10 +784,7 @@ class FrozenKVMTPWorkerV2(EAGLEWorkerV2): if batch.spec_info is None: batch.spec_info = self.draft_worker._idle_seed() with ( - self.draft_worker.draft_tp_context( - self.draft_worker.draft_runner.tp_group, - owns_attention=self.draft_worker.draft_owns_attention, - ), + draft_tp_context(self.draft_worker.draft_owns_attention), speculative_moe_backend_context(), speculative_moe_a2a_backend_context(), spec_stage_span("draft"), @@ -813,10 +797,7 @@ class FrozenKVMTPWorkerV2(EAGLEWorkerV2): if on_publish is not None: on_publish(batch_output.new_seq_lens) with ( - self.draft_worker.draft_tp_context( - self.draft_worker.draft_runner.tp_group, - owns_attention=self.draft_worker.draft_owns_attention, - ), + draft_tp_context(self.draft_worker.draft_owns_attention), speculative_moe_backend_context(), speculative_moe_a2a_backend_context(), spec_stage_span("draft_extend"), diff --git a/python/sglang/srt/speculative/multi_layer_eagle_worker_v2.py b/python/sglang/srt/speculative/multi_layer_eagle_worker_v2.py index 504f994247..cd23867273 100644 --- a/python/sglang/srt/speculative/multi_layer_eagle_worker_v2.py +++ b/python/sglang/srt/speculative/multi_layer_eagle_worker_v2.py @@ -45,7 +45,6 @@ from sglang.srt.model_executor.forward_batch_info import ( ) from sglang.srt.runtime_context import ( get_device, - get_parallel, get_schedule, get_spec, ) @@ -99,7 +98,7 @@ from sglang.srt.utils.async_probe import ( maybe_detect_nan, maybe_detect_oob, ) -from sglang.srt.utils.common import empty_context, fast_topk +from sglang.srt.utils.common import fast_topk from sglang.srt.utils.nvtx_utils import profile_range from sglang.srt.utils.profile_utils import build_step_span_name @@ -191,9 +190,6 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase): ] # Retain the target's attention topology when swapping TP groups. self.draft_owns_attention = False - self.draft_tp_context = ( - draft_tp_context if get_parallel().enable_dp_attention else empty_context - ) self.tree_mask_mode = default_tree_mask_mode() self.plan_stream, self.plan_stream_ctx = get_plan_stream(self.device) @@ -222,10 +218,7 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase): def init_attention_backends(self): with ( draft_pp_context(), - self.draft_tp_context( - self.draft_runner_list[0].tp_group, - owns_attention=self.draft_owns_attention, - ), + draft_tp_context(self.draft_owns_attention), speculative_moe_backend_context(), ): super().init_attention_backends() @@ -233,10 +226,7 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase): def init_cuda_graphs(self): with ( draft_pp_context(), - self.draft_tp_context( - self.draft_runner_list[0].tp_group, - owns_attention=self.draft_owns_attention, - ), + draft_tp_context(self.draft_owns_attention), speculative_moe_backend_context(), ): super().init_cuda_graphs() diff --git a/python/sglang/srt/speculative/spec_utils.py b/python/sglang/srt/speculative/spec_utils.py index 3042d78d50..d6ef5cb441 100644 --- a/python/sglang/srt/speculative/spec_utils.py +++ b/python/sglang/srt/speculative/spec_utils.py @@ -35,7 +35,6 @@ from sglang.srt.arg_groups.overrides import resolving_view from sglang.srt.configs.hybrid_arch import mambaish_config from sglang.srt.constrained.base_grammar_backend import GrammarMask from sglang.srt.distributed.parallel_state import ( - GroupCoordinator, get_self_pp_group, patch_pipeline_parallel_group, patch_tensor_parallel_group, @@ -52,6 +51,7 @@ from sglang.srt.mem_cache.allocation import ( ) from sglang.srt.runtime_context import ( get_exec, + get_parallel, get_spec, mamba_track_grid, max_speculative_num_draft_tokens, @@ -747,12 +747,19 @@ def draft_pp_context(): yield -@contextmanager -def draft_tp_context(tp_group: GroupCoordinator, *, owns_attention: bool): - # Draft model doesn't use dp and has its own tp group. - # We disable mscclpp now because it doesn't support 2 comm groups. - with patch_tensor_parallel_group(tp_group, owns_attention=owns_attention): - yield +def draft_tp_context(owns_attention: bool): + """Enter the TP placement that draft work runs under. + + Work that owns attention (an attention-owning draft, or DSpark's DP-MoE + sync) runs on the target's attention-TP group; other draft work keeps the + target's TP placement. Enter it from outside any draft scope, where + ``attn_tp_group`` is still the target's. + """ + if not owns_attention: + return contextlib.nullcontext() + return patch_tensor_parallel_group( + get_parallel().attn_tp_group, owns_attention=True + ) def spec_stage_span(name: str): diff --git a/python/sglang/srt/speculative/standalone_worker_v2.py b/python/sglang/srt/speculative/standalone_worker_v2.py index 3122d98aca..5a232f5326 100644 --- a/python/sglang/srt/speculative/standalone_worker_v2.py +++ b/python/sglang/srt/speculative/standalone_worker_v2.py @@ -3,14 +3,10 @@ from typing import Optional import torch -from sglang.srt.layers.moe.utils import ( - draft_model_build_scope, - speculative_moe_backend_context, -) +from sglang.srt.layers.moe.utils import draft_model_build_scope from sglang.srt.managers.tp_worker import TpModelWorker from sglang.srt.runtime_context import ( get_device, - get_parallel, get_schedule, get_spec, ) @@ -25,12 +21,8 @@ from sglang.srt.speculative.base_spec_worker import ( from sglang.srt.speculative.eagle_utils import default_tree_mask_mode from sglang.srt.speculative.eagle_worker_v2 import EagleDraftWorker, EAGLEWorkerV2 from sglang.srt.speculative.spec_info import SpeculativeAlgorithm -from sglang.srt.speculative.spec_utils import ( - draft_pp_context, - draft_tp_context, - get_plan_stream, -) -from sglang.srt.utils import empty_context, get_bool_env_var +from sglang.srt.speculative.spec_utils import draft_pp_context, get_plan_stream +from sglang.srt.utils import get_bool_env_var logger = logging.getLogger(__name__) SGLANG_RETURN_ORIGINAL_LOGPROB = get_bool_env_var("SGLANG_RETURN_ORIGINAL_LOGPROB") @@ -90,9 +82,6 @@ class StandaloneDraftWorker(EagleDraftWorker): self.draft_runner = self.draft_worker.model_runner # Retain the target's attention topology when swapping TP groups. self.draft_owns_attention = False - self.draft_tp_context = ( - draft_tp_context if get_parallel().enable_dp_attention else empty_context - ) self.tree_mask_mode = default_tree_mask_mode() self.plan_stream, self.plan_stream_ctx = get_plan_stream(self.device) # draft_forward reads this (set in EagleDraftWorker.__init__, skipped here). @@ -126,26 +115,6 @@ class StandaloneDraftWorker(EagleDraftWorker): self.init_token_map() self.init_lm_head() - def init_attention_backends(self): - with ( - self.draft_tp_context( - self.draft_runner.tp_group, - owns_attention=self.draft_owns_attention, - ), - speculative_moe_backend_context(), - ): - super().init_attention_backends() - - def init_cuda_graphs(self): - with ( - self.draft_tp_context( - self.draft_runner.tp_group, - owns_attention=self.draft_owns_attention, - ), - speculative_moe_backend_context(), - ): - super().init_cuda_graphs() - def init_lm_head(self): """Override to prevent sharing embeddings and lm_head with target model.""" # For standalone worker, we don't share embeddings and lm_head diff --git a/python/sglang/test/kits/attention_unittest/runner_modes/speculative_draft_extend_runner.py b/python/sglang/test/kits/attention_unittest/runner_modes/speculative_draft_extend_runner.py index bc4f6003a6..2f562fd907 100644 --- a/python/sglang/test/kits/attention_unittest/runner_modes/speculative_draft_extend_runner.py +++ b/python/sglang/test/kits/attention_unittest/runner_modes/speculative_draft_extend_runner.py @@ -89,6 +89,7 @@ from .speculative_cuda_graph_runner import ( from .speculative_draft_runner import ( EagleDraftRunnerSettings, _configure_runner_for_eagle_draft, + _DummyTpGroup, _reset_cuda_graph_test_buffers, _seeded_rng, _single_rank_graph_capture, @@ -531,7 +532,7 @@ def _capture_eagle_draft_extend_graph_runner( "sglang.srt.model_executor.runner.decode_cuda_graph_runner.get_available_gpu_memory", lambda *args, **kwargs: 0.0, ), - get_parallel().override(attn_cp_size=1, tp_rank=0), + get_parallel().override(attn_cp_size=1, tp_rank=0, tp_group=_DummyTpGroup()), ): _reset_cuda_graph_test_buffers() return EAGLEDraftExtendCudaGraphRunner( diff --git a/python/sglang/test/kits/attention_unittest/runner_modes/speculative_draft_runner.py b/python/sglang/test/kits/attention_unittest/runner_modes/speculative_draft_runner.py index 79ba07b03b..802a677dfe 100644 --- a/python/sglang/test/kits/attention_unittest/runner_modes/speculative_draft_runner.py +++ b/python/sglang/test/kits/attention_unittest/runner_modes/speculative_draft_runner.py @@ -335,7 +335,6 @@ def _configure_runner_for_eagle_draft( runner.spec_algorithm = SpeculativeAlgorithm.EAGLE runner.is_draft_worker = True runner.model = _TinyDraftModel() - runner.tp_group = _DummyTpGroup() runner.device_timer = None runner.model_config.spec_hidden_size = settings.hidden_size runner.model_config.dtype = runner.dtype @@ -445,7 +444,7 @@ def _capture_eagle_draft_graph_runner( "sglang.srt.model_executor.runner.decode_cuda_graph_runner.get_available_gpu_memory", lambda *args, **kwargs: 0.0, ), - get_parallel().override(attn_cp_size=1, tp_rank=0), + get_parallel().override(attn_cp_size=1, tp_rank=0, tp_group=_DummyTpGroup()), ): _reset_cuda_graph_test_buffers() return EAGLEDraftCudaGraphRunner( @@ -467,7 +466,7 @@ def _capture_frozen_kv_mtp_graph_runner( "sglang.srt.model_executor.runner.decode_cuda_graph_runner.get_available_gpu_memory", lambda *args, **kwargs: 0.0, ), - get_parallel().override(attn_cp_size=1, tp_rank=0), + get_parallel().override(attn_cp_size=1, tp_rank=0, tp_group=_DummyTpGroup()), ): _reset_cuda_graph_test_buffers() return FrozenKVMTPCudaGraphRunner(worker) diff --git a/test/registered/unit/model_executor/runner/test_flashinfer_autotune_sync.py b/test/registered/unit/model_executor/runner/test_flashinfer_autotune_sync.py index 930450a831..c8151e4455 100644 --- a/test/registered/unit/model_executor/runner/test_flashinfer_autotune_sync.py +++ b/test/registered/unit/model_executor/runner/test_flashinfer_autotune_sync.py @@ -31,6 +31,7 @@ from sglang.srt.model_executor.runner.flashinfer_autotune import ( _autotune_tactic_sync_group, _drop_diverged_autotune_cache, ) +from sglang.srt.runtime_context import get_parallel from sglang.test.test_utils import CustomTestCase, find_available_port ENV = {"flashinfer_version": "0.6.17", "gpu": "NVIDIA GB300"} @@ -225,11 +226,7 @@ class TestAutotuneCachePhases(CustomTestCase): tuner = AutoTuner.get() tuner.clear_cache() self.addCleanup(tuner.clear_cache) - runner = SimpleNamespace( - device="cuda", - forward_stream=torch.cuda.Stream(), - tp_group=SimpleNamespace(world_size=1), - ) + runner = SimpleNamespace(device="cuda", forward_stream=torch.cuda.Stream()) with tempfile.TemporaryDirectory() as directory: target, draft = ( Path(directory) / name for name in ("target.json", "draft.json") @@ -253,6 +250,7 @@ class TestAutotuneCachePhases(CustomTestCase): autotune, "get_flashinfer_autotune_skip_ops", return_value=set() ), autotune.envs.SGLANG_FLASHINFER_AUTOTUNE_CACHE.override(True), + get_parallel().override(tp_group=SimpleNamespace(world_size=1)), ): with autotune.flashinfer_autotune_context(runner, run_lm_head=False): self.assertEqual( diff --git a/test/registered/unit/model_executor/runner_backend/test_full_cuda_graph_backend.py b/test/registered/unit/model_executor/runner_backend/test_full_cuda_graph_backend.py index 55e79ce4fd..6820ef21bd 100644 --- a/test/registered/unit/model_executor/runner_backend/test_full_cuda_graph_backend.py +++ b/test/registered/unit/model_executor/runner_backend/test_full_cuda_graph_backend.py @@ -66,7 +66,7 @@ def _make_backend(runner): backend._memory_saver_adapter = None backend._cuda_graph_runner = runner backend._device_module = runner.device_module - backend._tp_group = runner.model_runner.tp_group + backend._tp_group = SimpleNamespace(barrier=mock.Mock(name="barrier")) return backend @@ -75,10 +75,8 @@ def _make_runner(*, enable_profile, profiler, num_tokens_per_bs=1, mode_name="DE synchronize=mock.Mock(name="synchronize"), graph=mock.Mock(name="graph", side_effect=lambda **kw: _FakeGraphCtx()), ) - tp_group = SimpleNamespace(barrier=mock.Mock(name="barrier")) runner = SimpleNamespace( device_module=device_module, - model_runner=SimpleNamespace(tp_group=tp_group), num_tokens_per_bs=num_tokens_per_bs, capture_forward_mode=SimpleNamespace(name=mode_name), enable_profile_cuda_graph=enable_profile, diff --git a/test/registered/unit/spec/test_dflash_logits.py b/test/registered/unit/spec/test_dflash_logits.py index acfc8a5858..3d30448b21 100644 --- a/test/registered/unit/spec/test_dflash_logits.py +++ b/test/registered/unit/spec/test_dflash_logits.py @@ -256,7 +256,7 @@ def test_worker_folds_a_gate_admitted_quantized_selector_head(monkeypatch): worker = SimpleNamespace( block_size=8, selector=object(), - model_runner=SimpleNamespace(tp_group=SimpleNamespace(rank_in_group=0)), + _target_tp_rank=0, ps=SimpleNamespace(tp_rank=0), draft_model=SimpleNamespace(lm_head=None), device="cpu", diff --git a/test/registered/unit/spec/test_dp_spec_prefill_coordination.py b/test/registered/unit/spec/test_dp_spec_prefill_coordination.py index 9f7d8958aa..4e5051f9e1 100644 --- a/test/registered/unit/spec/test_dp_spec_prefill_coordination.py +++ b/test/registered/unit/spec/test_dp_spec_prefill_coordination.py @@ -310,7 +310,7 @@ class TestDPSpecPrefillCoordinationWorker(CustomTestCase): record("draft_extend", current) return object() - def draft_context(group, *, owns_attention): + def draft_context(owns_attention): self.assertEqual(owns_attention, bool(rank % 2)) return contextlib.nullcontext() @@ -320,9 +320,8 @@ class TestDPSpecPrefillCoordinationWorker(CustomTestCase): worker.speculative_algorithm = SpeculativeAlgorithm.EAGLE worker._target_worker = SimpleNamespace(forward_batch_generation=target) worker._draft_worker = SimpleNamespace( - draft_runner=SimpleNamespace(tp_group=None), + draft_runner=SimpleNamespace(), draft_owns_attention=bool(rank % 2), - draft_tp_context=draft_context, draft=draft, _draft_extend_for_prefill=extend, _draft_extend_for_decode=extend, @@ -340,6 +339,12 @@ class TestDPSpecPrefillCoordinationWorker(CustomTestCase): f"{WORKER_MODULE}.ScheduleBatch.init_new", return_value=idle ) ) + stack.enter_context( + patch( + f"{WORKER_MODULE}.draft_tp_context", + side_effect=draft_context, + ) + ) stack.enter_context( patch( f"{WORKER_MODULE}.get_draft_recurrent_hidden_state_spec", diff --git a/test/registered/unit/spec/test_draft_tp_context.py b/test/registered/unit/spec/test_draft_tp_context.py new file mode 100644 index 0000000000..d7ef2c3359 --- /dev/null +++ b/test/registered/unit/spec/test_draft_tp_context.py @@ -0,0 +1,58 @@ +import unittest + +from sglang.srt.distributed import parallel_state +from sglang.srt.distributed.parallel_state import GroupCoordinator +from sglang.srt.runtime_context import get_parallel +from sglang.srt.speculative.spec_utils import draft_tp_context +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase, published_topology + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + + +def _group(*, world_size, rank): + group = GroupCoordinator.__new__(GroupCoordinator) + group.world_size = world_size + group.rank_in_group = rank + return group + + +class TestDraftTpContext(CustomTestCase): + """Draft work enters the attention-TP placement only when it owns attention.""" + + def setUp(self): + topology = published_topology( + tp_size=4, dp_size=2, enable_dp_attention=True, ranks=dict(dp_rank=0) + ) + topology.__enter__() + self.addCleanup(topology.__exit__, None, None, None) + self.whole_tp = _group(world_size=4, rank=0) + self.attn_tp = _group(world_size=2, rank=0) + + def test_owning_attention_runs_on_the_attention_tp_group(self): + with get_parallel().override( + tp_group=self.whole_tp, attn_tp_group=self.attn_tp + ): + with draft_tp_context(True): + parallel = get_parallel() + self.assertIs(parallel.tp_group, self.attn_tp) + self.assertEqual(parallel.tp_size, 2) + self.assertEqual(parallel.attn_dp_size, 1) + self.assertIs(get_parallel().tp_group, self.whole_tp) + self.assertEqual(get_parallel().tp_size, 4) + self.assertEqual(get_parallel().attn_dp_size, 2) + + def test_otherwise_the_target_placement_stays(self): + with get_parallel().override( + tp_group=self.whole_tp, attn_tp_group=self.attn_tp + ): + with draft_tp_context(False): + parallel = get_parallel() + self.assertIs(parallel.tp_group, self.whole_tp) + self.assertEqual(parallel.tp_size, 4) + self.assertEqual(parallel.attn_dp_size, 2) + self.assertFalse(parallel_state._TP_STATE_PATCHED) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/test_runtime_context.py b/test/registered/unit/test_runtime_context.py index b089fe91ff..5aedb9dc6f 100644 --- a/test/registered/unit/test_runtime_context.py +++ b/test/registered/unit/test_runtime_context.py @@ -3262,9 +3262,7 @@ class TestNothingReadsThePlacementBeforeItIsFrozen(CustomTestCase): def test_no_method_called_before_the_freeze_reads_what_it_freezes(self): methods = self._model_runner() frozen = self._frozen_names(methods) - self.assertLessEqual( - {"tp_group", "pp_group"}, frozen, "found no frozen groups; census is broken" - ) + self.assertIn("pp_group", frozen, "found no frozen group; census is broken") offenders = [] for lineno, name in self._calls_before_the_freeze(methods): fn = methods.get(name)