[Refactor] Enter draft TP scopes by attention ownership and drop ModelRunner.tp_group (#41817)

This commit is contained in:
Cheng Wan
2026-09-30 13:28:50 -07:00
committed by GitHub
parent e7f6a99313
commit 986cb8268b
29 changed files with 161 additions and 224 deletions
@@ -23,6 +23,7 @@ from sglang.srt.configs.model_config import (
is_deepseek_dsa,
is_deepseek_v4,
)
from sglang.srt.runtime_context import get_parallel
from sglang.srt.speculative.eagle_draft_cuda_graph_runner import (
EAGLEDraftCudaGraphRunner,
)
@@ -82,7 +83,7 @@ class EAGLEDraftNpuGraphRunner(EAGLEDraftCudaGraphRunner):
torch.distributed.all_reduce(
decision,
op=torch.distributed.ReduceOp.MIN,
group=self.model_runner.tp_group.device_group,
group=get_parallel().tp_group.device_group,
)
return bool(decision.item())
@@ -28,6 +28,7 @@ from sglang.srt.model_executor.runner.shape_key import ShapeKey
from sglang.srt.model_executor.runner_backend.base_cuda_graph_backend import (
BaseCudaGraphBackend,
)
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import empty_context, get_bool_env_var
from sglang.srt.utils.torch_memory_saver_adapter import TorchMemorySaverAdapter
@@ -54,7 +55,7 @@ class NPUCudaGraphBackend(BaseCudaGraphBackend):
self._pool = None
self._device_module = cuda_graph_runner.device_module
self._device_id = self._device_module.current_device()
self._tp_group = cuda_graph_runner.model_runner.tp_group
self._tp_group = get_parallel().tp_group
self._capture_stream = None
self._memory_saver_adapter: Optional[Any] = TorchMemorySaverAdapter.create(
enable=enable_memory_saver
@@ -22,6 +22,7 @@ from sglang.srt.model_executor.runner_backend.base_cuda_graph_backend import (
from sglang.srt.model_executor.runner_utils.pool import (
get_or_create_global_graph_memory_pool,
)
from sglang.srt.runtime_context import get_parallel
if TYPE_CHECKING:
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
@@ -41,7 +42,7 @@ class FullXPUGraphBackend(BaseCudaGraphBackend):
self._outputs: Dict[Any, Any] = {}
self._pool = None
self._device_module = cuda_graph_runner.device_module
self._tp_group = cuda_graph_runner.model_runner.tp_group
self._tp_group = get_parallel().tp_group
self._capture_stream: Optional[torch.xpu.Stream] = None
@contextmanager
+1 -1
View File
@@ -366,7 +366,7 @@ class TpModelWorker(BaseTpWorker):
# the set of ranks holding a draft worker. The draft worker is
# constructed with pp_rank=0, so derive the caller's global rank
# from the TP group rather than tp_size * pp_rank + tp_rank.
tp_group = self.model_runner.tp_group
tp_group = get_parallel().tp_group
self.random_seed = broadcast_pyobj(
[get_device().random_seed],
tp_group.ranks[tp_group.rank_in_group],
@@ -750,7 +750,7 @@ class CPUGraphRunner:
self.model_runner.model,
bs in self.capture_bs,
num_tokens=bs * self.captured_req_width,
tp_group=self.model_runner.tp_group,
tp_group=get_parallel().tp_group,
) as forward:
graph, output_buffers = self.capture_one_batch_size(
bs, forward, skip_cross_attention=True
@@ -853,7 +853,7 @@ class CPUGraphRunner:
forward_batch.spec_info,
)
with torch.no_grad():
self.model_runner.tp_group.barrier()
get_parallel().tp_group.barrier()
self.model_runner.model.forward(
forward_batch.input_ids,
forward_batch.positions,
@@ -875,7 +875,7 @@ class CPUGraphRunner:
with torch.no_grad():
for _ in range(2):
self.model_runner.tp_group.barrier()
get_parallel().tp_group.barrier()
out = run_once()
# Save the captured forward_batch in the appropriate dict
if skip_cross_attention:
@@ -497,7 +497,7 @@ class ModelRunner:
parallel = get_parallel()
join_effective_ep_size = parallel.ep_join_rank_offset + parallel.tp_size
dist.barrier(group=self.tp_group.cpu_group)
dist.barrier(group=parallel.tp_group.cpu_group)
if parallel.tp_rank == 0:
register_scale_cohort(
parallel.ep_join_rank_offset,
@@ -966,7 +966,7 @@ class ModelRunner:
tp_group=(
get_parallel().attn_tp_group.cpu_group
if get_parallel().enable_dp_attention
else self.tp_group.cpu_group
else get_parallel().tp_group.cpu_group
),
host_to_device_ratio=hisparse_cfg.host_to_device_ratio,
swap_in_block_size=hisparse_cfg.swap_in_block_size,
@@ -1178,13 +1178,10 @@ class ModelRunner:
self.pre_model_load_memory = bootstrap.measure_pre_model_load_memory(
device=self.device, is_draft_worker=self.is_draft_worker
)
# A draft runner is used outside the scope it is built in: speculative
# workers re-enter that scope through its TP group, and draft forwards
# read its PP group without the pipeline scope. Keep both groups; read
# every other placement value from the context where it is used.
parallel = get_parallel()
self.tp_group = parallel.tp_group
self.pp_group = parallel.pp_group
# Draft forwards run without the pipeline scope the draft is built in,
# so keep the PP group it was built with; read every other placement
# value from the context where it is used.
self.pp_group = get_parallel().pp_group
def load_model(self):
tic_total = time.perf_counter()
@@ -2317,7 +2314,7 @@ class ModelRunner:
return
recovered = maybe_recover_ep_ranks(
tp_group=self.tp_group,
tp_group=get_parallel().tp_group,
eplb_manager=self.eplb_manager,
model_config=self.model_config,
moe_ep_rank=self._elastic_global_rank(),
@@ -741,7 +741,7 @@ class BaseRunner(ABC):
return logits_output_or_pp_proxy_tensors
torch.get_device_module(mr.device).synchronize()
mr.tp_group.barrier()
get_parallel().tp_group.barrier()
with forward_context(ForwardContext(attn_backend=mr.attn_backend)):
with run_ctx or empty_context():
run_once()
@@ -1135,7 +1135,7 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
self.model_runner.model,
bs in self.compile_bs,
num_tokens=bs * self.captured_req_width,
tp_group=self.model_runner.tp_group,
tp_group=get_parallel().tp_group,
) as forward:
self.capture_one_shape(
bs,
@@ -254,7 +254,7 @@ def flashinfer_autotune_context(model_runner: ModelRunner, *, run_lm_head: bool)
mr = model_runner
cache_path = flashinfer_autotune_cache_path(mr)
sync_group = _autotune_tactic_sync_group(mr.tp_group)
sync_group = _autotune_tactic_sync_group(get_parallel().tp_group)
reuse_cache = envs.SGLANG_FLASHINFER_AUTOTUNE_CACHE.get()
if reuse_cache:
autotune_cache = cache_path
@@ -425,7 +425,7 @@ def maybe_flashinfer_autotune_extend(
try:
run_flashinfer_autotune_forward(mr, forward_fn, run_lm_head=False)
except torch.OutOfMemoryError:
if _autotune_tactic_sync_group(mr.tp_group) is not None:
if _autotune_tactic_sync_group(get_parallel().tp_group) is not None:
# Tuning is collective: this rank has stopped reducing while its
# peers wait on the next tactic, so skipping the pass would hang
# them. Fail instead of degrading alone.
@@ -616,7 +616,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
# --- capture --------------------------------------------------
self.device_module.synchronize()
self.model_runner.tp_group.barrier()
get_parallel().tp_group.barrier()
self.capture()
self.raw_num_tokens = 0
@@ -45,6 +45,7 @@ from sglang.srt.model_executor.runner_utils.pool import (
graph_pool_capture_scope,
graph_pool_replay_scope,
)
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import get_bool_env_var
from sglang.srt.utils.torch_memory_saver_adapter import TorchMemorySaverAdapter
@@ -75,7 +76,7 @@ class BreakableCudaGraphBackend(DedupedCudaGraphMixin, BaseCudaGraphBackend):
self._capture_inputs: Dict[Any, Any] = {}
self._pool = None
self._device_module = cuda_graph_runner.device_module
self._tp_group = cuda_graph_runner.model_runner.tp_group
self._tp_group = get_parallel().tp_group
self._capture_stream: Optional[torch.cuda.Stream] = None
self._debug_eager = debug_eager
self._shared_output_buffer: Optional[Any] = None
@@ -36,6 +36,7 @@ from sglang.srt.model_executor.runner_utils.pool import (
graph_pool_capture_scope,
graph_pool_replay_scope,
)
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import get_bool_env_var
from sglang.srt.utils.torch_memory_saver_adapter import TorchMemorySaverAdapter
@@ -91,7 +92,7 @@ class FullCudaGraphBackend(BaseCudaGraphBackend):
self._pool = None
self._cuda_graph_runner = cuda_graph_runner
self._device_module = cuda_graph_runner.device_module
self._tp_group = cuda_graph_runner.model_runner.tp_group
self._tp_group = get_parallel().tp_group
self._capture_stream: Optional[torch.cuda.Stream] = None
self._precarve = GraphPoolPrecarve()
self._reuse_output_buffer = reuse_output_buffer
@@ -95,7 +95,7 @@ class TcPiecewiseCudaGraphBackend(BaseCudaGraphBackend):
model_runner = cuda_graph_runner.model_runner
self._pool = None
self._device_module = cuda_graph_runner.device_module
self._tp_group = model_runner.tp_group
self._tp_group = get_parallel().tp_group
self._capture_stream: Optional[torch.cuda.Stream] = None
self._compile_config: CompilationConfig = self.build_compilation_config(
model_runner.server_args
@@ -94,7 +94,6 @@ from sglang.srt.speculative.spec_utils import (
draft_tp_context,
)
from sglang.srt.utils import is_cuda, is_hip, is_npu, is_xpu
from sglang.srt.utils.common import empty_context
_is_npu = is_npu()
@@ -406,19 +405,12 @@ class DFlashWorkerV2(BaseSpecWorker):
)
# Under dp attention, the draft worker runs on the per-DP attn-TP
# group, independent of idle peer DP ranks.
self.draft_tp_context = (
draft_tp_context if get_parallel().enable_dp_attention else empty_context
)
# Use the same attention topology during draft construction and execution.
# group, independent of idle peer DP ranks; it is built and run under
# the same placement.
self.draft_owns_attention = get_parallel().enable_dp_attention
if self.draft_owns_attention:
draft_init_ctx = draft_tp_context(
get_parallel().attn_tp_group, owns_attention=True
)
else:
draft_init_ctx = empty_context()
with draft_pp_context(), draft_init_ctx:
# Inside the draft scope the context answers the draft's narrowed rank.
self._target_tp_rank = get_parallel().tp_rank
with draft_pp_context(), draft_tp_context(self.draft_owns_attention):
bundle = build_draft_tp_worker(
server_args=server_args,
gpu_id=gpu_id,
@@ -621,10 +613,7 @@ class DFlashWorkerV2(BaseSpecWorker):
def init_attention_backends(self):
with (
draft_pp_context(),
self.draft_tp_context(
self.draft_model_runner.tp_group,
owns_attention=self.draft_owns_attention,
),
draft_tp_context(self.draft_owns_attention),
):
self._draft_worker.init_attention_backends()
self._need_mamba_verify_commit = mambaish_config(
@@ -637,10 +626,7 @@ class DFlashWorkerV2(BaseSpecWorker):
def init_cuda_graphs(self):
with (
draft_pp_context(),
self.draft_tp_context(
self.draft_model_runner.tp_group,
owns_attention=self.draft_owns_attention,
),
draft_tp_context(self.draft_owns_attention),
):
capture_decode_cuda_graph = (
get_exec().graph.cuda_graph_config.decode.backend != Backend.DISABLED
@@ -661,7 +647,7 @@ class DFlashWorkerV2(BaseSpecWorker):
# shared graph capture/replay; keep the draft eager under dp
# attention.
capture_decode_cuda_graph = False
if self.model_runner.tp_group.rank_in_group == 0:
if self._target_tp_rank == 0:
logger.warning(
"Disable DFLASH draft cuda graph because dp attention "
"is enabled (draft runs eager)."
@@ -806,7 +792,7 @@ class DFlashWorkerV2(BaseSpecWorker):
def _maybe_build_draft_sampler(self):
def _eager(reason):
if self.model_runner.tp_group.rank_in_group == 0:
if self._target_tp_rank == 0:
logger.info("DFLASH draft greedy head kept eager (reason=%s).", reason)
return None
@@ -832,7 +818,7 @@ class DFlashWorkerV2(BaseSpecWorker):
if not head_supported:
return _eager("unsupported quantized lm_head")
self.draft_model.lm_head = lm_head
if self.model_runner.tp_group.rank_in_group == 0:
if self._target_tp_rank == 0:
logger.info(
"DFLASH selector decode folded into the draft cuda graph "
"(sampling_enabled=%s).",
@@ -867,7 +853,7 @@ class DFlashWorkerV2(BaseSpecWorker):
embed_proj = self.draft_model.embed_proj
if prefix_gru is None or embed_proj is None:
return _eager("Domino projector modules are unavailable")
if self.model_runner.tp_group.rank_in_group == 0:
if self._target_tp_rank == 0:
logger.info(
"DFLASH Domino rollout folded into the draft cuda graph (tp=%s).",
int(tp_group.world_size),
@@ -906,7 +892,7 @@ class DFlashWorkerV2(BaseSpecWorker):
return _eager("added vocab")
num_org = int(shard.num_org_elements)
org_vocab_start = int(shard.org_vocab_start_index)
if self.model_runner.tp_group.rank_in_group == 0:
if self._target_tp_rank == 0:
logger.info(
"DFLASH draft greedy head folded into the draft cuda graph (tp=%d).",
tp_group.world_size,
@@ -1857,10 +1843,7 @@ class DFlashWorkerV2(BaseSpecWorker):
with (
torch.inference_mode(),
self.draft_tp_context(
self.draft_model_runner.tp_group,
owns_attention=self.draft_owns_attention,
),
draft_tp_context(self.draft_owns_attention),
):
ctx_hidden = self.draft_model.project_target_hidden(target_hidden)
if self.lilicorr is not None:
@@ -2567,10 +2550,7 @@ class DFlashWorkerV2(BaseSpecWorker):
with (
torch.inference_mode(),
self.draft_tp_context(
self.draft_model_runner.tp_group,
owns_attention=self.draft_owns_attention,
),
draft_tp_context(self.draft_owns_attention),
):
draft_out = self.draft_model_runner.forward(forward_batch)
draft_logits_output = draft_out.logits_output
@@ -2638,10 +2618,7 @@ class DFlashWorkerV2(BaseSpecWorker):
self._draft_sampler.q_out[:bs],
)
elif self.selector is not None:
with self.draft_tp_context(
self.draft_model_runner.tp_group,
owns_attention=self.draft_owns_attention,
):
with draft_tp_context(self.draft_owns_attention):
draft_next = self._propose_selector_block(
draft_logits_output=draft_logits_output,
bs=bs,
@@ -2667,10 +2644,7 @@ class DFlashWorkerV2(BaseSpecWorker):
draft_hidden = draft_logits_output.hidden_states
if draft_hidden is None:
raise RuntimeError("DFLASH draft model returned no hidden states.")
with self.draft_tp_context(
self.draft_model_runner.tp_group,
owns_attention=self.draft_owns_attention,
):
with draft_tp_context(self.draft_owns_attention):
draft_next, lilicorr_candidate_ids, lilicorr_q_rows = (
propose_lilicorr_block(
head=self.lilicorr,
@@ -2691,10 +2665,7 @@ class DFlashWorkerV2(BaseSpecWorker):
if draft_hidden is None:
raise RuntimeError("DFLASH draft model returned no hidden states.")
draft_hidden = draft_hidden.view(bs, int(self.block_size), -1)
with self.draft_tp_context(
self.draft_model_runner.tp_group,
owns_attention=self.draft_owns_attention,
):
with draft_tp_context(self.draft_owns_attention):
draft_next = self._greedy_sample_from_vocab_parallel_head(
hidden_states=draft_hidden[:, 1:, :].reshape(
-1, draft_hidden.shape[-1]
@@ -1,7 +1,6 @@
from __future__ import annotations
import logging
from contextlib import nullcontext
from typing import Optional
import msgspec
@@ -19,7 +18,6 @@ from sglang.srt.model_executor.forward_batch_info import (
ForwardMode,
enable_num_token_non_padded,
)
from sglang.srt.runtime_context import get_parallel
from sglang.srt.speculative.dflash_info_v2 import DFlashDraftInputV2
from sglang.srt.speculative.draft_worker_common import make_draft_input_v2
from sglang.srt.speculative.dspark_components.dspark_planner import VerifyWindow
@@ -235,9 +233,7 @@ class DraftBlockProposer:
self._draft_sampler = draft_sampler
def _base_logits_context(self):
if self._dp_moe_sync:
return draft_tp_context(get_parallel().attn_tp_group, owns_attention=True)
return nullcontext()
return draft_tp_context(self._dp_moe_sync)
def propose(
self,
@@ -1,5 +1,4 @@
import logging
from contextlib import nullcontext
from typing import TYPE_CHECKING, Callable, Optional, Protocol, runtime_checkable
import torch
@@ -181,6 +180,8 @@ class DSparkWorkerV2(BaseSpecWorker):
"MoE-under-DP all-reduce."
)
# Inside the draft scope the context answers the draft's narrowed rank.
self._target_tp_rank = get_parallel().tp_rank
with draft_pp_context(), self._draft_context():
bundle = build_draft_tp_worker(
server_args=server_args,
@@ -431,9 +432,7 @@ class DSparkWorkerV2(BaseSpecWorker):
return getattr(self.target_worker, name)
def _draft_context(self):
if self._draft_dp_context_enabled:
return draft_tp_context(get_parallel().attn_tp_group, owns_attention=True)
return nullcontext()
return draft_tp_context(self._draft_dp_context_enabled)
def alloc_memory_pool(
self,
@@ -516,7 +515,7 @@ class DSparkWorkerV2(BaseSpecWorker):
gamma=self.gamma,
max_bs=max(get_exec().graph.cuda_graph_config.decode.bs),
device=self.device,
tp_rank=self.model_runner.tp_group.rank_in_group,
tp_rank=self._target_tp_rank,
tp_sync=self._tp_sync,
available_memory_gb=available_memory_gb,
confidence_fn=(
@@ -120,7 +120,6 @@ from sglang.srt.utils.async_probe import (
maybe_detect_oob,
)
from sglang.srt.utils.common import (
empty_context,
fast_topk,
get_available_gpu_memory,
is_cpu,
@@ -190,12 +189,8 @@ class EagleDraftWorker(EagleDraftWorkerBase):
get_parallel().enable_dp_attention
and self.speculative_algorithm.is_eagle3()
)
if self.draft_owns_attention:
ctx = draft_tp_context(get_parallel().attn_tp_group, owns_attention=True)
else:
ctx = empty_context()
with (
ctx,
draft_tp_context(self.draft_owns_attention),
draft_pp_context(),
speculative_moe_backend_context(),
speculative_moe_a2a_backend_context(),
@@ -216,9 +211,6 @@ class EagleDraftWorker(EagleDraftWorkerBase):
self._init_dsa_index_share_state()
# Eager draft-extend seed buffer (graph paths use their own static ones).
self.dsa_extend_topk_buf: Optional[torch.Tensor] = None
self.draft_tp_context = (
draft_tp_context if get_parallel().enable_dp_attention else empty_context
)
self.tree_mask_mode = default_tree_mask_mode()
self.plan_stream, self.plan_stream_ctx = get_plan_stream(self.device)
@@ -259,10 +251,7 @@ class EagleDraftWorker(EagleDraftWorkerBase):
def init_attention_backends(self):
with (
draft_pp_context(),
self.draft_tp_context(
self.draft_runner.tp_group,
owns_attention=self.draft_owns_attention,
),
draft_tp_context(self.draft_owns_attention),
speculative_moe_backend_context(),
speculative_moe_a2a_backend_context(),
):
@@ -272,10 +261,7 @@ class EagleDraftWorker(EagleDraftWorkerBase):
def init_cuda_graphs(self):
with (
draft_pp_context(),
self.draft_tp_context(
self.draft_runner.tp_group,
owns_attention=self.draft_owns_attention,
),
draft_tp_context(self.draft_owns_attention),
speculative_moe_backend_context(),
speculative_moe_a2a_backend_context(),
):
@@ -1283,10 +1269,7 @@ class EAGLEWorkerV2(BaseSpecWorker):
# Build adaptive runtime states after target and draft backends exist.
if self.adaptive_controller is not None:
with (
self._draft_worker.draft_tp_context(
self._draft_worker.draft_runner.tp_group,
owns_attention=self._draft_worker.draft_owns_attention,
),
draft_tp_context(self._draft_worker.draft_owns_attention),
speculative_moe_backend_context(),
speculative_moe_a2a_backend_context(),
):
@@ -1369,10 +1352,7 @@ class EAGLEWorkerV2(BaseSpecWorker):
verify_input = self._build_trivial_verify_input(batch)
else:
with (
self.draft_worker.draft_tp_context(
self.draft_worker.draft_runner.tp_group,
owns_attention=self.draft_worker.draft_owns_attention,
),
draft_tp_context(self.draft_worker.draft_owns_attention),
speculative_moe_backend_context(),
speculative_moe_a2a_backend_context(),
spec_stage_span("draft"),
@@ -1395,10 +1375,7 @@ class EAGLEWorkerV2(BaseSpecWorker):
self._stub_skipped_draft_extend(batch, batch_output)
else:
with (
self.draft_worker.draft_tp_context(
self.draft_worker.draft_runner.tp_group,
owns_attention=self.draft_worker.draft_owns_attention,
),
draft_tp_context(self.draft_worker.draft_owns_attention),
speculative_moe_backend_context(),
speculative_moe_a2a_backend_context(),
spec_stage_span("draft_extend"),
@@ -1428,10 +1405,7 @@ class EAGLEWorkerV2(BaseSpecWorker):
batch.seq_lens_cpu = batch_output.new_seq_lens.to("cpu")
batch.seq_lens_sum = int(batch.seq_lens_cpu.sum())
with (
self.draft_worker.draft_tp_context(
self.draft_worker.draft_runner.tp_group,
owns_attention=self.draft_worker.draft_owns_attention,
),
draft_tp_context(self.draft_worker.draft_owns_attention),
speculative_moe_backend_context(),
speculative_moe_a2a_backend_context(),
spec_stage_span("draft"),
@@ -1489,10 +1463,7 @@ class EAGLEWorkerV2(BaseSpecWorker):
# Draft prefill
with (
self.draft_worker.draft_tp_context(
self.draft_worker.draft_runner.tp_group,
owns_attention=self.draft_worker.draft_owns_attention,
),
draft_tp_context(self.draft_worker.draft_owns_attention),
speculative_moe_backend_context(),
speculative_moe_a2a_backend_context(),
spec_stage_span("draft_extend"),
@@ -1547,10 +1518,7 @@ class EAGLEWorkerV2(BaseSpecWorker):
)
plan.apply(draft_batch, "draft", rank, local_only=draft_local_only)
with (
self.draft_worker.draft_tp_context(
self.draft_worker.draft_runner.tp_group,
owns_attention=self.draft_worker.draft_owns_attention,
),
draft_tp_context(self.draft_worker.draft_owns_attention),
speculative_moe_backend_context(),
speculative_moe_a2a_backend_context(),
spec_stage_span("draft"),
@@ -1574,10 +1542,7 @@ class EAGLEWorkerV2(BaseSpecWorker):
on_publish(result.new_seq_lens)
plan.apply(batch, "draft_extend", rank, local_only=draft_local_only)
with (
self.draft_worker.draft_tp_context(
self.draft_worker.draft_runner.tp_group,
owns_attention=self.draft_worker.draft_owns_attention,
),
draft_tp_context(self.draft_worker.draft_owns_attention),
speculative_moe_backend_context(),
speculative_moe_a2a_backend_context(),
spec_stage_span("draft_extend"),
@@ -45,7 +45,6 @@ from sglang.srt.model_executor.pool_configurator import MemoryPoolConfig
from sglang.srt.runtime_context import (
attention_backends,
get_device,
get_parallel,
get_schedule,
get_spec,
)
@@ -79,7 +78,7 @@ from sglang.srt.speculative.spec_utils import (
select_top_k_tokens,
spec_stage_span,
)
from sglang.srt.utils import empty_context, get_available_gpu_memory
from sglang.srt.utils import get_available_gpu_memory
from sglang.srt.utils.async_probe import (
maybe_detect_inf,
maybe_detect_nan,
@@ -170,9 +169,6 @@ class FrozenKVMTPDraftWorker(EagleDraftWorkerBase, TpModelWorker):
# Retain the target's attention topology when swapping TP groups.
self.draft_owns_attention = False
self.draft_tp_context = (
draft_tp_context if get_parallel().enable_dp_attention else empty_context
)
self.draft_attn_backend = None
self.cuda_graph_runner = None
@@ -210,10 +206,7 @@ class FrozenKVMTPDraftWorker(EagleDraftWorkerBase, TpModelWorker):
def init_attention_backends(self):
with (
draft_pp_context(),
self.draft_tp_context(
self.draft_model_runner.tp_group,
owns_attention=self.draft_owns_attention,
),
draft_tp_context(self.draft_owns_attention),
speculative_moe_backend_context(),
speculative_moe_a2a_backend_context(),
):
@@ -224,10 +217,7 @@ class FrozenKVMTPDraftWorker(EagleDraftWorkerBase, TpModelWorker):
def init_cuda_graphs(self):
with (
draft_pp_context(),
self.draft_tp_context(
self.draft_model_runner.tp_group,
owns_attention=self.draft_owns_attention,
),
draft_tp_context(self.draft_owns_attention),
speculative_moe_backend_context(),
speculative_moe_a2a_backend_context(),
):
@@ -774,10 +764,7 @@ class FrozenKVMTPWorkerV2(EAGLEWorkerV2):
# Draft prefill seed (no forward).
with (
self.draft_worker.draft_tp_context(
self.draft_worker.draft_runner.tp_group,
owns_attention=self.draft_worker.draft_owns_attention,
),
draft_tp_context(self.draft_worker.draft_owns_attention),
speculative_moe_backend_context(),
speculative_moe_a2a_backend_context(),
spec_stage_span("draft_extend"),
@@ -797,10 +784,7 @@ class FrozenKVMTPWorkerV2(EAGLEWorkerV2):
if batch.spec_info is None:
batch.spec_info = self.draft_worker._idle_seed()
with (
self.draft_worker.draft_tp_context(
self.draft_worker.draft_runner.tp_group,
owns_attention=self.draft_worker.draft_owns_attention,
),
draft_tp_context(self.draft_worker.draft_owns_attention),
speculative_moe_backend_context(),
speculative_moe_a2a_backend_context(),
spec_stage_span("draft"),
@@ -813,10 +797,7 @@ class FrozenKVMTPWorkerV2(EAGLEWorkerV2):
if on_publish is not None:
on_publish(batch_output.new_seq_lens)
with (
self.draft_worker.draft_tp_context(
self.draft_worker.draft_runner.tp_group,
owns_attention=self.draft_worker.draft_owns_attention,
),
draft_tp_context(self.draft_worker.draft_owns_attention),
speculative_moe_backend_context(),
speculative_moe_a2a_backend_context(),
spec_stage_span("draft_extend"),
@@ -45,7 +45,6 @@ from sglang.srt.model_executor.forward_batch_info import (
)
from sglang.srt.runtime_context import (
get_device,
get_parallel,
get_schedule,
get_spec,
)
@@ -99,7 +98,7 @@ from sglang.srt.utils.async_probe import (
maybe_detect_nan,
maybe_detect_oob,
)
from sglang.srt.utils.common import empty_context, fast_topk
from sglang.srt.utils.common import fast_topk
from sglang.srt.utils.nvtx_utils import profile_range
from sglang.srt.utils.profile_utils import build_step_span_name
@@ -191,9 +190,6 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase):
]
# Retain the target's attention topology when swapping TP groups.
self.draft_owns_attention = False
self.draft_tp_context = (
draft_tp_context if get_parallel().enable_dp_attention else empty_context
)
self.tree_mask_mode = default_tree_mask_mode()
self.plan_stream, self.plan_stream_ctx = get_plan_stream(self.device)
@@ -222,10 +218,7 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase):
def init_attention_backends(self):
with (
draft_pp_context(),
self.draft_tp_context(
self.draft_runner_list[0].tp_group,
owns_attention=self.draft_owns_attention,
),
draft_tp_context(self.draft_owns_attention),
speculative_moe_backend_context(),
):
super().init_attention_backends()
@@ -233,10 +226,7 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase):
def init_cuda_graphs(self):
with (
draft_pp_context(),
self.draft_tp_context(
self.draft_runner_list[0].tp_group,
owns_attention=self.draft_owns_attention,
),
draft_tp_context(self.draft_owns_attention),
speculative_moe_backend_context(),
):
super().init_cuda_graphs()
+14 -7
View File
@@ -35,7 +35,6 @@ from sglang.srt.arg_groups.overrides import resolving_view
from sglang.srt.configs.hybrid_arch import mambaish_config
from sglang.srt.constrained.base_grammar_backend import GrammarMask
from sglang.srt.distributed.parallel_state import (
GroupCoordinator,
get_self_pp_group,
patch_pipeline_parallel_group,
patch_tensor_parallel_group,
@@ -52,6 +51,7 @@ from sglang.srt.mem_cache.allocation import (
)
from sglang.srt.runtime_context import (
get_exec,
get_parallel,
get_spec,
mamba_track_grid,
max_speculative_num_draft_tokens,
@@ -747,12 +747,19 @@ def draft_pp_context():
yield
@contextmanager
def draft_tp_context(tp_group: GroupCoordinator, *, owns_attention: bool):
# Draft model doesn't use dp and has its own tp group.
# We disable mscclpp now because it doesn't support 2 comm groups.
with patch_tensor_parallel_group(tp_group, owns_attention=owns_attention):
yield
def draft_tp_context(owns_attention: bool):
"""Enter the TP placement that draft work runs under.
Work that owns attention (an attention-owning draft, or DSpark's DP-MoE
sync) runs on the target's attention-TP group; other draft work keeps the
target's TP placement. Enter it from outside any draft scope, where
``attn_tp_group`` is still the target's.
"""
if not owns_attention:
return contextlib.nullcontext()
return patch_tensor_parallel_group(
get_parallel().attn_tp_group, owns_attention=True
)
def spec_stage_span(name: str):
@@ -3,14 +3,10 @@ from typing import Optional
import torch
from sglang.srt.layers.moe.utils import (
draft_model_build_scope,
speculative_moe_backend_context,
)
from sglang.srt.layers.moe.utils import draft_model_build_scope
from sglang.srt.managers.tp_worker import TpModelWorker
from sglang.srt.runtime_context import (
get_device,
get_parallel,
get_schedule,
get_spec,
)
@@ -25,12 +21,8 @@ from sglang.srt.speculative.base_spec_worker import (
from sglang.srt.speculative.eagle_utils import default_tree_mask_mode
from sglang.srt.speculative.eagle_worker_v2 import EagleDraftWorker, EAGLEWorkerV2
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
from sglang.srt.speculative.spec_utils import (
draft_pp_context,
draft_tp_context,
get_plan_stream,
)
from sglang.srt.utils import empty_context, get_bool_env_var
from sglang.srt.speculative.spec_utils import draft_pp_context, get_plan_stream
from sglang.srt.utils import get_bool_env_var
logger = logging.getLogger(__name__)
SGLANG_RETURN_ORIGINAL_LOGPROB = get_bool_env_var("SGLANG_RETURN_ORIGINAL_LOGPROB")
@@ -90,9 +82,6 @@ class StandaloneDraftWorker(EagleDraftWorker):
self.draft_runner = self.draft_worker.model_runner
# Retain the target's attention topology when swapping TP groups.
self.draft_owns_attention = False
self.draft_tp_context = (
draft_tp_context if get_parallel().enable_dp_attention else empty_context
)
self.tree_mask_mode = default_tree_mask_mode()
self.plan_stream, self.plan_stream_ctx = get_plan_stream(self.device)
# draft_forward reads this (set in EagleDraftWorker.__init__, skipped here).
@@ -126,26 +115,6 @@ class StandaloneDraftWorker(EagleDraftWorker):
self.init_token_map()
self.init_lm_head()
def init_attention_backends(self):
with (
self.draft_tp_context(
self.draft_runner.tp_group,
owns_attention=self.draft_owns_attention,
),
speculative_moe_backend_context(),
):
super().init_attention_backends()
def init_cuda_graphs(self):
with (
self.draft_tp_context(
self.draft_runner.tp_group,
owns_attention=self.draft_owns_attention,
),
speculative_moe_backend_context(),
):
super().init_cuda_graphs()
def init_lm_head(self):
"""Override to prevent sharing embeddings and lm_head with target model."""
# For standalone worker, we don't share embeddings and lm_head
@@ -89,6 +89,7 @@ from .speculative_cuda_graph_runner import (
from .speculative_draft_runner import (
EagleDraftRunnerSettings,
_configure_runner_for_eagle_draft,
_DummyTpGroup,
_reset_cuda_graph_test_buffers,
_seeded_rng,
_single_rank_graph_capture,
@@ -531,7 +532,7 @@ def _capture_eagle_draft_extend_graph_runner(
"sglang.srt.model_executor.runner.decode_cuda_graph_runner.get_available_gpu_memory",
lambda *args, **kwargs: 0.0,
),
get_parallel().override(attn_cp_size=1, tp_rank=0),
get_parallel().override(attn_cp_size=1, tp_rank=0, tp_group=_DummyTpGroup()),
):
_reset_cuda_graph_test_buffers()
return EAGLEDraftExtendCudaGraphRunner(
@@ -335,7 +335,6 @@ def _configure_runner_for_eagle_draft(
runner.spec_algorithm = SpeculativeAlgorithm.EAGLE
runner.is_draft_worker = True
runner.model = _TinyDraftModel()
runner.tp_group = _DummyTpGroup()
runner.device_timer = None
runner.model_config.spec_hidden_size = settings.hidden_size
runner.model_config.dtype = runner.dtype
@@ -445,7 +444,7 @@ def _capture_eagle_draft_graph_runner(
"sglang.srt.model_executor.runner.decode_cuda_graph_runner.get_available_gpu_memory",
lambda *args, **kwargs: 0.0,
),
get_parallel().override(attn_cp_size=1, tp_rank=0),
get_parallel().override(attn_cp_size=1, tp_rank=0, tp_group=_DummyTpGroup()),
):
_reset_cuda_graph_test_buffers()
return EAGLEDraftCudaGraphRunner(
@@ -467,7 +466,7 @@ def _capture_frozen_kv_mtp_graph_runner(
"sglang.srt.model_executor.runner.decode_cuda_graph_runner.get_available_gpu_memory",
lambda *args, **kwargs: 0.0,
),
get_parallel().override(attn_cp_size=1, tp_rank=0),
get_parallel().override(attn_cp_size=1, tp_rank=0, tp_group=_DummyTpGroup()),
):
_reset_cuda_graph_test_buffers()
return FrozenKVMTPCudaGraphRunner(worker)