[Refactor] Enter draft TP scopes by attention ownership and drop ModelRunner.tp_group (#41817)
This commit is contained in:
@@ -23,6 +23,7 @@ from sglang.srt.configs.model_config import (
|
||||
is_deepseek_dsa,
|
||||
is_deepseek_v4,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.speculative.eagle_draft_cuda_graph_runner import (
|
||||
EAGLEDraftCudaGraphRunner,
|
||||
)
|
||||
@@ -82,7 +83,7 @@ class EAGLEDraftNpuGraphRunner(EAGLEDraftCudaGraphRunner):
|
||||
torch.distributed.all_reduce(
|
||||
decision,
|
||||
op=torch.distributed.ReduceOp.MIN,
|
||||
group=self.model_runner.tp_group.device_group,
|
||||
group=get_parallel().tp_group.device_group,
|
||||
)
|
||||
return bool(decision.item())
|
||||
|
||||
|
||||
@@ -28,6 +28,7 @@ from sglang.srt.model_executor.runner.shape_key import ShapeKey
|
||||
from sglang.srt.model_executor.runner_backend.base_cuda_graph_backend import (
|
||||
BaseCudaGraphBackend,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import empty_context, get_bool_env_var
|
||||
from sglang.srt.utils.torch_memory_saver_adapter import TorchMemorySaverAdapter
|
||||
|
||||
@@ -54,7 +55,7 @@ class NPUCudaGraphBackend(BaseCudaGraphBackend):
|
||||
self._pool = None
|
||||
self._device_module = cuda_graph_runner.device_module
|
||||
self._device_id = self._device_module.current_device()
|
||||
self._tp_group = cuda_graph_runner.model_runner.tp_group
|
||||
self._tp_group = get_parallel().tp_group
|
||||
self._capture_stream = None
|
||||
self._memory_saver_adapter: Optional[Any] = TorchMemorySaverAdapter.create(
|
||||
enable=enable_memory_saver
|
||||
|
||||
@@ -22,6 +22,7 @@ from sglang.srt.model_executor.runner_backend.base_cuda_graph_backend import (
|
||||
from sglang.srt.model_executor.runner_utils.pool import (
|
||||
get_or_create_global_graph_memory_pool,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
@@ -41,7 +42,7 @@ class FullXPUGraphBackend(BaseCudaGraphBackend):
|
||||
self._outputs: Dict[Any, Any] = {}
|
||||
self._pool = None
|
||||
self._device_module = cuda_graph_runner.device_module
|
||||
self._tp_group = cuda_graph_runner.model_runner.tp_group
|
||||
self._tp_group = get_parallel().tp_group
|
||||
self._capture_stream: Optional[torch.xpu.Stream] = None
|
||||
|
||||
@contextmanager
|
||||
|
||||
@@ -366,7 +366,7 @@ class TpModelWorker(BaseTpWorker):
|
||||
# the set of ranks holding a draft worker. The draft worker is
|
||||
# constructed with pp_rank=0, so derive the caller's global rank
|
||||
# from the TP group rather than tp_size * pp_rank + tp_rank.
|
||||
tp_group = self.model_runner.tp_group
|
||||
tp_group = get_parallel().tp_group
|
||||
self.random_seed = broadcast_pyobj(
|
||||
[get_device().random_seed],
|
||||
tp_group.ranks[tp_group.rank_in_group],
|
||||
|
||||
@@ -750,7 +750,7 @@ class CPUGraphRunner:
|
||||
self.model_runner.model,
|
||||
bs in self.capture_bs,
|
||||
num_tokens=bs * self.captured_req_width,
|
||||
tp_group=self.model_runner.tp_group,
|
||||
tp_group=get_parallel().tp_group,
|
||||
) as forward:
|
||||
graph, output_buffers = self.capture_one_batch_size(
|
||||
bs, forward, skip_cross_attention=True
|
||||
@@ -853,7 +853,7 @@ class CPUGraphRunner:
|
||||
forward_batch.spec_info,
|
||||
)
|
||||
with torch.no_grad():
|
||||
self.model_runner.tp_group.barrier()
|
||||
get_parallel().tp_group.barrier()
|
||||
self.model_runner.model.forward(
|
||||
forward_batch.input_ids,
|
||||
forward_batch.positions,
|
||||
@@ -875,7 +875,7 @@ class CPUGraphRunner:
|
||||
|
||||
with torch.no_grad():
|
||||
for _ in range(2):
|
||||
self.model_runner.tp_group.barrier()
|
||||
get_parallel().tp_group.barrier()
|
||||
out = run_once()
|
||||
# Save the captured forward_batch in the appropriate dict
|
||||
if skip_cross_attention:
|
||||
|
||||
@@ -497,7 +497,7 @@ class ModelRunner:
|
||||
|
||||
parallel = get_parallel()
|
||||
join_effective_ep_size = parallel.ep_join_rank_offset + parallel.tp_size
|
||||
dist.barrier(group=self.tp_group.cpu_group)
|
||||
dist.barrier(group=parallel.tp_group.cpu_group)
|
||||
if parallel.tp_rank == 0:
|
||||
register_scale_cohort(
|
||||
parallel.ep_join_rank_offset,
|
||||
@@ -966,7 +966,7 @@ class ModelRunner:
|
||||
tp_group=(
|
||||
get_parallel().attn_tp_group.cpu_group
|
||||
if get_parallel().enable_dp_attention
|
||||
else self.tp_group.cpu_group
|
||||
else get_parallel().tp_group.cpu_group
|
||||
),
|
||||
host_to_device_ratio=hisparse_cfg.host_to_device_ratio,
|
||||
swap_in_block_size=hisparse_cfg.swap_in_block_size,
|
||||
@@ -1178,13 +1178,10 @@ class ModelRunner:
|
||||
self.pre_model_load_memory = bootstrap.measure_pre_model_load_memory(
|
||||
device=self.device, is_draft_worker=self.is_draft_worker
|
||||
)
|
||||
# A draft runner is used outside the scope it is built in: speculative
|
||||
# workers re-enter that scope through its TP group, and draft forwards
|
||||
# read its PP group without the pipeline scope. Keep both groups; read
|
||||
# every other placement value from the context where it is used.
|
||||
parallel = get_parallel()
|
||||
self.tp_group = parallel.tp_group
|
||||
self.pp_group = parallel.pp_group
|
||||
# Draft forwards run without the pipeline scope the draft is built in,
|
||||
# so keep the PP group it was built with; read every other placement
|
||||
# value from the context where it is used.
|
||||
self.pp_group = get_parallel().pp_group
|
||||
|
||||
def load_model(self):
|
||||
tic_total = time.perf_counter()
|
||||
@@ -2317,7 +2314,7 @@ class ModelRunner:
|
||||
return
|
||||
|
||||
recovered = maybe_recover_ep_ranks(
|
||||
tp_group=self.tp_group,
|
||||
tp_group=get_parallel().tp_group,
|
||||
eplb_manager=self.eplb_manager,
|
||||
model_config=self.model_config,
|
||||
moe_ep_rank=self._elastic_global_rank(),
|
||||
|
||||
@@ -741,7 +741,7 @@ class BaseRunner(ABC):
|
||||
return logits_output_or_pp_proxy_tensors
|
||||
|
||||
torch.get_device_module(mr.device).synchronize()
|
||||
mr.tp_group.barrier()
|
||||
get_parallel().tp_group.barrier()
|
||||
with forward_context(ForwardContext(attn_backend=mr.attn_backend)):
|
||||
with run_ctx or empty_context():
|
||||
run_once()
|
||||
|
||||
@@ -1135,7 +1135,7 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
|
||||
self.model_runner.model,
|
||||
bs in self.compile_bs,
|
||||
num_tokens=bs * self.captured_req_width,
|
||||
tp_group=self.model_runner.tp_group,
|
||||
tp_group=get_parallel().tp_group,
|
||||
) as forward:
|
||||
self.capture_one_shape(
|
||||
bs,
|
||||
|
||||
@@ -254,7 +254,7 @@ def flashinfer_autotune_context(model_runner: ModelRunner, *, run_lm_head: bool)
|
||||
|
||||
mr = model_runner
|
||||
cache_path = flashinfer_autotune_cache_path(mr)
|
||||
sync_group = _autotune_tactic_sync_group(mr.tp_group)
|
||||
sync_group = _autotune_tactic_sync_group(get_parallel().tp_group)
|
||||
reuse_cache = envs.SGLANG_FLASHINFER_AUTOTUNE_CACHE.get()
|
||||
if reuse_cache:
|
||||
autotune_cache = cache_path
|
||||
@@ -425,7 +425,7 @@ def maybe_flashinfer_autotune_extend(
|
||||
try:
|
||||
run_flashinfer_autotune_forward(mr, forward_fn, run_lm_head=False)
|
||||
except torch.OutOfMemoryError:
|
||||
if _autotune_tactic_sync_group(mr.tp_group) is not None:
|
||||
if _autotune_tactic_sync_group(get_parallel().tp_group) is not None:
|
||||
# Tuning is collective: this rank has stopped reducing while its
|
||||
# peers wait on the next tactic, so skipping the pass would hang
|
||||
# them. Fail instead of degrading alone.
|
||||
|
||||
@@ -616,7 +616,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
|
||||
# --- capture --------------------------------------------------
|
||||
self.device_module.synchronize()
|
||||
self.model_runner.tp_group.barrier()
|
||||
get_parallel().tp_group.barrier()
|
||||
self.capture()
|
||||
|
||||
self.raw_num_tokens = 0
|
||||
|
||||
@@ -45,6 +45,7 @@ from sglang.srt.model_executor.runner_utils.pool import (
|
||||
graph_pool_capture_scope,
|
||||
graph_pool_replay_scope,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import get_bool_env_var
|
||||
from sglang.srt.utils.torch_memory_saver_adapter import TorchMemorySaverAdapter
|
||||
|
||||
@@ -75,7 +76,7 @@ class BreakableCudaGraphBackend(DedupedCudaGraphMixin, BaseCudaGraphBackend):
|
||||
self._capture_inputs: Dict[Any, Any] = {}
|
||||
self._pool = None
|
||||
self._device_module = cuda_graph_runner.device_module
|
||||
self._tp_group = cuda_graph_runner.model_runner.tp_group
|
||||
self._tp_group = get_parallel().tp_group
|
||||
self._capture_stream: Optional[torch.cuda.Stream] = None
|
||||
self._debug_eager = debug_eager
|
||||
self._shared_output_buffer: Optional[Any] = None
|
||||
|
||||
@@ -36,6 +36,7 @@ from sglang.srt.model_executor.runner_utils.pool import (
|
||||
graph_pool_capture_scope,
|
||||
graph_pool_replay_scope,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import get_bool_env_var
|
||||
from sglang.srt.utils.torch_memory_saver_adapter import TorchMemorySaverAdapter
|
||||
|
||||
@@ -91,7 +92,7 @@ class FullCudaGraphBackend(BaseCudaGraphBackend):
|
||||
self._pool = None
|
||||
self._cuda_graph_runner = cuda_graph_runner
|
||||
self._device_module = cuda_graph_runner.device_module
|
||||
self._tp_group = cuda_graph_runner.model_runner.tp_group
|
||||
self._tp_group = get_parallel().tp_group
|
||||
self._capture_stream: Optional[torch.cuda.Stream] = None
|
||||
self._precarve = GraphPoolPrecarve()
|
||||
self._reuse_output_buffer = reuse_output_buffer
|
||||
|
||||
@@ -95,7 +95,7 @@ class TcPiecewiseCudaGraphBackend(BaseCudaGraphBackend):
|
||||
model_runner = cuda_graph_runner.model_runner
|
||||
self._pool = None
|
||||
self._device_module = cuda_graph_runner.device_module
|
||||
self._tp_group = model_runner.tp_group
|
||||
self._tp_group = get_parallel().tp_group
|
||||
self._capture_stream: Optional[torch.cuda.Stream] = None
|
||||
self._compile_config: CompilationConfig = self.build_compilation_config(
|
||||
model_runner.server_args
|
||||
|
||||
@@ -94,7 +94,6 @@ from sglang.srt.speculative.spec_utils import (
|
||||
draft_tp_context,
|
||||
)
|
||||
from sglang.srt.utils import is_cuda, is_hip, is_npu, is_xpu
|
||||
from sglang.srt.utils.common import empty_context
|
||||
|
||||
_is_npu = is_npu()
|
||||
|
||||
@@ -406,19 +405,12 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
)
|
||||
|
||||
# Under dp attention, the draft worker runs on the per-DP attn-TP
|
||||
# group, independent of idle peer DP ranks.
|
||||
self.draft_tp_context = (
|
||||
draft_tp_context if get_parallel().enable_dp_attention else empty_context
|
||||
)
|
||||
# Use the same attention topology during draft construction and execution.
|
||||
# group, independent of idle peer DP ranks; it is built and run under
|
||||
# the same placement.
|
||||
self.draft_owns_attention = get_parallel().enable_dp_attention
|
||||
if self.draft_owns_attention:
|
||||
draft_init_ctx = draft_tp_context(
|
||||
get_parallel().attn_tp_group, owns_attention=True
|
||||
)
|
||||
else:
|
||||
draft_init_ctx = empty_context()
|
||||
with draft_pp_context(), draft_init_ctx:
|
||||
# Inside the draft scope the context answers the draft's narrowed rank.
|
||||
self._target_tp_rank = get_parallel().tp_rank
|
||||
with draft_pp_context(), draft_tp_context(self.draft_owns_attention):
|
||||
bundle = build_draft_tp_worker(
|
||||
server_args=server_args,
|
||||
gpu_id=gpu_id,
|
||||
@@ -621,10 +613,7 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
def init_attention_backends(self):
|
||||
with (
|
||||
draft_pp_context(),
|
||||
self.draft_tp_context(
|
||||
self.draft_model_runner.tp_group,
|
||||
owns_attention=self.draft_owns_attention,
|
||||
),
|
||||
draft_tp_context(self.draft_owns_attention),
|
||||
):
|
||||
self._draft_worker.init_attention_backends()
|
||||
self._need_mamba_verify_commit = mambaish_config(
|
||||
@@ -637,10 +626,7 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
def init_cuda_graphs(self):
|
||||
with (
|
||||
draft_pp_context(),
|
||||
self.draft_tp_context(
|
||||
self.draft_model_runner.tp_group,
|
||||
owns_attention=self.draft_owns_attention,
|
||||
),
|
||||
draft_tp_context(self.draft_owns_attention),
|
||||
):
|
||||
capture_decode_cuda_graph = (
|
||||
get_exec().graph.cuda_graph_config.decode.backend != Backend.DISABLED
|
||||
@@ -661,7 +647,7 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
# shared graph capture/replay; keep the draft eager under dp
|
||||
# attention.
|
||||
capture_decode_cuda_graph = False
|
||||
if self.model_runner.tp_group.rank_in_group == 0:
|
||||
if self._target_tp_rank == 0:
|
||||
logger.warning(
|
||||
"Disable DFLASH draft cuda graph because dp attention "
|
||||
"is enabled (draft runs eager)."
|
||||
@@ -806,7 +792,7 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
|
||||
def _maybe_build_draft_sampler(self):
|
||||
def _eager(reason):
|
||||
if self.model_runner.tp_group.rank_in_group == 0:
|
||||
if self._target_tp_rank == 0:
|
||||
logger.info("DFLASH draft greedy head kept eager (reason=%s).", reason)
|
||||
return None
|
||||
|
||||
@@ -832,7 +818,7 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
if not head_supported:
|
||||
return _eager("unsupported quantized lm_head")
|
||||
self.draft_model.lm_head = lm_head
|
||||
if self.model_runner.tp_group.rank_in_group == 0:
|
||||
if self._target_tp_rank == 0:
|
||||
logger.info(
|
||||
"DFLASH selector decode folded into the draft cuda graph "
|
||||
"(sampling_enabled=%s).",
|
||||
@@ -867,7 +853,7 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
embed_proj = self.draft_model.embed_proj
|
||||
if prefix_gru is None or embed_proj is None:
|
||||
return _eager("Domino projector modules are unavailable")
|
||||
if self.model_runner.tp_group.rank_in_group == 0:
|
||||
if self._target_tp_rank == 0:
|
||||
logger.info(
|
||||
"DFLASH Domino rollout folded into the draft cuda graph (tp=%s).",
|
||||
int(tp_group.world_size),
|
||||
@@ -906,7 +892,7 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
return _eager("added vocab")
|
||||
num_org = int(shard.num_org_elements)
|
||||
org_vocab_start = int(shard.org_vocab_start_index)
|
||||
if self.model_runner.tp_group.rank_in_group == 0:
|
||||
if self._target_tp_rank == 0:
|
||||
logger.info(
|
||||
"DFLASH draft greedy head folded into the draft cuda graph (tp=%d).",
|
||||
tp_group.world_size,
|
||||
@@ -1857,10 +1843,7 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
|
||||
with (
|
||||
torch.inference_mode(),
|
||||
self.draft_tp_context(
|
||||
self.draft_model_runner.tp_group,
|
||||
owns_attention=self.draft_owns_attention,
|
||||
),
|
||||
draft_tp_context(self.draft_owns_attention),
|
||||
):
|
||||
ctx_hidden = self.draft_model.project_target_hidden(target_hidden)
|
||||
if self.lilicorr is not None:
|
||||
@@ -2567,10 +2550,7 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
|
||||
with (
|
||||
torch.inference_mode(),
|
||||
self.draft_tp_context(
|
||||
self.draft_model_runner.tp_group,
|
||||
owns_attention=self.draft_owns_attention,
|
||||
),
|
||||
draft_tp_context(self.draft_owns_attention),
|
||||
):
|
||||
draft_out = self.draft_model_runner.forward(forward_batch)
|
||||
draft_logits_output = draft_out.logits_output
|
||||
@@ -2638,10 +2618,7 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
self._draft_sampler.q_out[:bs],
|
||||
)
|
||||
elif self.selector is not None:
|
||||
with self.draft_tp_context(
|
||||
self.draft_model_runner.tp_group,
|
||||
owns_attention=self.draft_owns_attention,
|
||||
):
|
||||
with draft_tp_context(self.draft_owns_attention):
|
||||
draft_next = self._propose_selector_block(
|
||||
draft_logits_output=draft_logits_output,
|
||||
bs=bs,
|
||||
@@ -2667,10 +2644,7 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
draft_hidden = draft_logits_output.hidden_states
|
||||
if draft_hidden is None:
|
||||
raise RuntimeError("DFLASH draft model returned no hidden states.")
|
||||
with self.draft_tp_context(
|
||||
self.draft_model_runner.tp_group,
|
||||
owns_attention=self.draft_owns_attention,
|
||||
):
|
||||
with draft_tp_context(self.draft_owns_attention):
|
||||
draft_next, lilicorr_candidate_ids, lilicorr_q_rows = (
|
||||
propose_lilicorr_block(
|
||||
head=self.lilicorr,
|
||||
@@ -2691,10 +2665,7 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
if draft_hidden is None:
|
||||
raise RuntimeError("DFLASH draft model returned no hidden states.")
|
||||
draft_hidden = draft_hidden.view(bs, int(self.block_size), -1)
|
||||
with self.draft_tp_context(
|
||||
self.draft_model_runner.tp_group,
|
||||
owns_attention=self.draft_owns_attention,
|
||||
):
|
||||
with draft_tp_context(self.draft_owns_attention):
|
||||
draft_next = self._greedy_sample_from_vocab_parallel_head(
|
||||
hidden_states=draft_hidden[:, 1:, :].reshape(
|
||||
-1, draft_hidden.shape[-1]
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from contextlib import nullcontext
|
||||
from typing import Optional
|
||||
|
||||
import msgspec
|
||||
@@ -19,7 +18,6 @@ from sglang.srt.model_executor.forward_batch_info import (
|
||||
ForwardMode,
|
||||
enable_num_token_non_padded,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.speculative.dflash_info_v2 import DFlashDraftInputV2
|
||||
from sglang.srt.speculative.draft_worker_common import make_draft_input_v2
|
||||
from sglang.srt.speculative.dspark_components.dspark_planner import VerifyWindow
|
||||
@@ -235,9 +233,7 @@ class DraftBlockProposer:
|
||||
self._draft_sampler = draft_sampler
|
||||
|
||||
def _base_logits_context(self):
|
||||
if self._dp_moe_sync:
|
||||
return draft_tp_context(get_parallel().attn_tp_group, owns_attention=True)
|
||||
return nullcontext()
|
||||
return draft_tp_context(self._dp_moe_sync)
|
||||
|
||||
def propose(
|
||||
self,
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import logging
|
||||
from contextlib import nullcontext
|
||||
from typing import TYPE_CHECKING, Callable, Optional, Protocol, runtime_checkable
|
||||
|
||||
import torch
|
||||
@@ -181,6 +180,8 @@ class DSparkWorkerV2(BaseSpecWorker):
|
||||
"MoE-under-DP all-reduce."
|
||||
)
|
||||
|
||||
# Inside the draft scope the context answers the draft's narrowed rank.
|
||||
self._target_tp_rank = get_parallel().tp_rank
|
||||
with draft_pp_context(), self._draft_context():
|
||||
bundle = build_draft_tp_worker(
|
||||
server_args=server_args,
|
||||
@@ -431,9 +432,7 @@ class DSparkWorkerV2(BaseSpecWorker):
|
||||
return getattr(self.target_worker, name)
|
||||
|
||||
def _draft_context(self):
|
||||
if self._draft_dp_context_enabled:
|
||||
return draft_tp_context(get_parallel().attn_tp_group, owns_attention=True)
|
||||
return nullcontext()
|
||||
return draft_tp_context(self._draft_dp_context_enabled)
|
||||
|
||||
def alloc_memory_pool(
|
||||
self,
|
||||
@@ -516,7 +515,7 @@ class DSparkWorkerV2(BaseSpecWorker):
|
||||
gamma=self.gamma,
|
||||
max_bs=max(get_exec().graph.cuda_graph_config.decode.bs),
|
||||
device=self.device,
|
||||
tp_rank=self.model_runner.tp_group.rank_in_group,
|
||||
tp_rank=self._target_tp_rank,
|
||||
tp_sync=self._tp_sync,
|
||||
available_memory_gb=available_memory_gb,
|
||||
confidence_fn=(
|
||||
|
||||
@@ -120,7 +120,6 @@ from sglang.srt.utils.async_probe import (
|
||||
maybe_detect_oob,
|
||||
)
|
||||
from sglang.srt.utils.common import (
|
||||
empty_context,
|
||||
fast_topk,
|
||||
get_available_gpu_memory,
|
||||
is_cpu,
|
||||
@@ -190,12 +189,8 @@ class EagleDraftWorker(EagleDraftWorkerBase):
|
||||
get_parallel().enable_dp_attention
|
||||
and self.speculative_algorithm.is_eagle3()
|
||||
)
|
||||
if self.draft_owns_attention:
|
||||
ctx = draft_tp_context(get_parallel().attn_tp_group, owns_attention=True)
|
||||
else:
|
||||
ctx = empty_context()
|
||||
with (
|
||||
ctx,
|
||||
draft_tp_context(self.draft_owns_attention),
|
||||
draft_pp_context(),
|
||||
speculative_moe_backend_context(),
|
||||
speculative_moe_a2a_backend_context(),
|
||||
@@ -216,9 +211,6 @@ class EagleDraftWorker(EagleDraftWorkerBase):
|
||||
self._init_dsa_index_share_state()
|
||||
# Eager draft-extend seed buffer (graph paths use their own static ones).
|
||||
self.dsa_extend_topk_buf: Optional[torch.Tensor] = None
|
||||
self.draft_tp_context = (
|
||||
draft_tp_context if get_parallel().enable_dp_attention else empty_context
|
||||
)
|
||||
self.tree_mask_mode = default_tree_mask_mode()
|
||||
|
||||
self.plan_stream, self.plan_stream_ctx = get_plan_stream(self.device)
|
||||
@@ -259,10 +251,7 @@ class EagleDraftWorker(EagleDraftWorkerBase):
|
||||
def init_attention_backends(self):
|
||||
with (
|
||||
draft_pp_context(),
|
||||
self.draft_tp_context(
|
||||
self.draft_runner.tp_group,
|
||||
owns_attention=self.draft_owns_attention,
|
||||
),
|
||||
draft_tp_context(self.draft_owns_attention),
|
||||
speculative_moe_backend_context(),
|
||||
speculative_moe_a2a_backend_context(),
|
||||
):
|
||||
@@ -272,10 +261,7 @@ class EagleDraftWorker(EagleDraftWorkerBase):
|
||||
def init_cuda_graphs(self):
|
||||
with (
|
||||
draft_pp_context(),
|
||||
self.draft_tp_context(
|
||||
self.draft_runner.tp_group,
|
||||
owns_attention=self.draft_owns_attention,
|
||||
),
|
||||
draft_tp_context(self.draft_owns_attention),
|
||||
speculative_moe_backend_context(),
|
||||
speculative_moe_a2a_backend_context(),
|
||||
):
|
||||
@@ -1283,10 +1269,7 @@ class EAGLEWorkerV2(BaseSpecWorker):
|
||||
# Build adaptive runtime states after target and draft backends exist.
|
||||
if self.adaptive_controller is not None:
|
||||
with (
|
||||
self._draft_worker.draft_tp_context(
|
||||
self._draft_worker.draft_runner.tp_group,
|
||||
owns_attention=self._draft_worker.draft_owns_attention,
|
||||
),
|
||||
draft_tp_context(self._draft_worker.draft_owns_attention),
|
||||
speculative_moe_backend_context(),
|
||||
speculative_moe_a2a_backend_context(),
|
||||
):
|
||||
@@ -1369,10 +1352,7 @@ class EAGLEWorkerV2(BaseSpecWorker):
|
||||
verify_input = self._build_trivial_verify_input(batch)
|
||||
else:
|
||||
with (
|
||||
self.draft_worker.draft_tp_context(
|
||||
self.draft_worker.draft_runner.tp_group,
|
||||
owns_attention=self.draft_worker.draft_owns_attention,
|
||||
),
|
||||
draft_tp_context(self.draft_worker.draft_owns_attention),
|
||||
speculative_moe_backend_context(),
|
||||
speculative_moe_a2a_backend_context(),
|
||||
spec_stage_span("draft"),
|
||||
@@ -1395,10 +1375,7 @@ class EAGLEWorkerV2(BaseSpecWorker):
|
||||
self._stub_skipped_draft_extend(batch, batch_output)
|
||||
else:
|
||||
with (
|
||||
self.draft_worker.draft_tp_context(
|
||||
self.draft_worker.draft_runner.tp_group,
|
||||
owns_attention=self.draft_worker.draft_owns_attention,
|
||||
),
|
||||
draft_tp_context(self.draft_worker.draft_owns_attention),
|
||||
speculative_moe_backend_context(),
|
||||
speculative_moe_a2a_backend_context(),
|
||||
spec_stage_span("draft_extend"),
|
||||
@@ -1428,10 +1405,7 @@ class EAGLEWorkerV2(BaseSpecWorker):
|
||||
batch.seq_lens_cpu = batch_output.new_seq_lens.to("cpu")
|
||||
batch.seq_lens_sum = int(batch.seq_lens_cpu.sum())
|
||||
with (
|
||||
self.draft_worker.draft_tp_context(
|
||||
self.draft_worker.draft_runner.tp_group,
|
||||
owns_attention=self.draft_worker.draft_owns_attention,
|
||||
),
|
||||
draft_tp_context(self.draft_worker.draft_owns_attention),
|
||||
speculative_moe_backend_context(),
|
||||
speculative_moe_a2a_backend_context(),
|
||||
spec_stage_span("draft"),
|
||||
@@ -1489,10 +1463,7 @@ class EAGLEWorkerV2(BaseSpecWorker):
|
||||
|
||||
# Draft prefill
|
||||
with (
|
||||
self.draft_worker.draft_tp_context(
|
||||
self.draft_worker.draft_runner.tp_group,
|
||||
owns_attention=self.draft_worker.draft_owns_attention,
|
||||
),
|
||||
draft_tp_context(self.draft_worker.draft_owns_attention),
|
||||
speculative_moe_backend_context(),
|
||||
speculative_moe_a2a_backend_context(),
|
||||
spec_stage_span("draft_extend"),
|
||||
@@ -1547,10 +1518,7 @@ class EAGLEWorkerV2(BaseSpecWorker):
|
||||
)
|
||||
plan.apply(draft_batch, "draft", rank, local_only=draft_local_only)
|
||||
with (
|
||||
self.draft_worker.draft_tp_context(
|
||||
self.draft_worker.draft_runner.tp_group,
|
||||
owns_attention=self.draft_worker.draft_owns_attention,
|
||||
),
|
||||
draft_tp_context(self.draft_worker.draft_owns_attention),
|
||||
speculative_moe_backend_context(),
|
||||
speculative_moe_a2a_backend_context(),
|
||||
spec_stage_span("draft"),
|
||||
@@ -1574,10 +1542,7 @@ class EAGLEWorkerV2(BaseSpecWorker):
|
||||
on_publish(result.new_seq_lens)
|
||||
plan.apply(batch, "draft_extend", rank, local_only=draft_local_only)
|
||||
with (
|
||||
self.draft_worker.draft_tp_context(
|
||||
self.draft_worker.draft_runner.tp_group,
|
||||
owns_attention=self.draft_worker.draft_owns_attention,
|
||||
),
|
||||
draft_tp_context(self.draft_worker.draft_owns_attention),
|
||||
speculative_moe_backend_context(),
|
||||
speculative_moe_a2a_backend_context(),
|
||||
spec_stage_span("draft_extend"),
|
||||
|
||||
@@ -45,7 +45,6 @@ from sglang.srt.model_executor.pool_configurator import MemoryPoolConfig
|
||||
from sglang.srt.runtime_context import (
|
||||
attention_backends,
|
||||
get_device,
|
||||
get_parallel,
|
||||
get_schedule,
|
||||
get_spec,
|
||||
)
|
||||
@@ -79,7 +78,7 @@ from sglang.srt.speculative.spec_utils import (
|
||||
select_top_k_tokens,
|
||||
spec_stage_span,
|
||||
)
|
||||
from sglang.srt.utils import empty_context, get_available_gpu_memory
|
||||
from sglang.srt.utils import get_available_gpu_memory
|
||||
from sglang.srt.utils.async_probe import (
|
||||
maybe_detect_inf,
|
||||
maybe_detect_nan,
|
||||
@@ -170,9 +169,6 @@ class FrozenKVMTPDraftWorker(EagleDraftWorkerBase, TpModelWorker):
|
||||
|
||||
# Retain the target's attention topology when swapping TP groups.
|
||||
self.draft_owns_attention = False
|
||||
self.draft_tp_context = (
|
||||
draft_tp_context if get_parallel().enable_dp_attention else empty_context
|
||||
)
|
||||
|
||||
self.draft_attn_backend = None
|
||||
self.cuda_graph_runner = None
|
||||
@@ -210,10 +206,7 @@ class FrozenKVMTPDraftWorker(EagleDraftWorkerBase, TpModelWorker):
|
||||
def init_attention_backends(self):
|
||||
with (
|
||||
draft_pp_context(),
|
||||
self.draft_tp_context(
|
||||
self.draft_model_runner.tp_group,
|
||||
owns_attention=self.draft_owns_attention,
|
||||
),
|
||||
draft_tp_context(self.draft_owns_attention),
|
||||
speculative_moe_backend_context(),
|
||||
speculative_moe_a2a_backend_context(),
|
||||
):
|
||||
@@ -224,10 +217,7 @@ class FrozenKVMTPDraftWorker(EagleDraftWorkerBase, TpModelWorker):
|
||||
def init_cuda_graphs(self):
|
||||
with (
|
||||
draft_pp_context(),
|
||||
self.draft_tp_context(
|
||||
self.draft_model_runner.tp_group,
|
||||
owns_attention=self.draft_owns_attention,
|
||||
),
|
||||
draft_tp_context(self.draft_owns_attention),
|
||||
speculative_moe_backend_context(),
|
||||
speculative_moe_a2a_backend_context(),
|
||||
):
|
||||
@@ -774,10 +764,7 @@ class FrozenKVMTPWorkerV2(EAGLEWorkerV2):
|
||||
|
||||
# Draft prefill seed (no forward).
|
||||
with (
|
||||
self.draft_worker.draft_tp_context(
|
||||
self.draft_worker.draft_runner.tp_group,
|
||||
owns_attention=self.draft_worker.draft_owns_attention,
|
||||
),
|
||||
draft_tp_context(self.draft_worker.draft_owns_attention),
|
||||
speculative_moe_backend_context(),
|
||||
speculative_moe_a2a_backend_context(),
|
||||
spec_stage_span("draft_extend"),
|
||||
@@ -797,10 +784,7 @@ class FrozenKVMTPWorkerV2(EAGLEWorkerV2):
|
||||
if batch.spec_info is None:
|
||||
batch.spec_info = self.draft_worker._idle_seed()
|
||||
with (
|
||||
self.draft_worker.draft_tp_context(
|
||||
self.draft_worker.draft_runner.tp_group,
|
||||
owns_attention=self.draft_worker.draft_owns_attention,
|
||||
),
|
||||
draft_tp_context(self.draft_worker.draft_owns_attention),
|
||||
speculative_moe_backend_context(),
|
||||
speculative_moe_a2a_backend_context(),
|
||||
spec_stage_span("draft"),
|
||||
@@ -813,10 +797,7 @@ class FrozenKVMTPWorkerV2(EAGLEWorkerV2):
|
||||
if on_publish is not None:
|
||||
on_publish(batch_output.new_seq_lens)
|
||||
with (
|
||||
self.draft_worker.draft_tp_context(
|
||||
self.draft_worker.draft_runner.tp_group,
|
||||
owns_attention=self.draft_worker.draft_owns_attention,
|
||||
),
|
||||
draft_tp_context(self.draft_worker.draft_owns_attention),
|
||||
speculative_moe_backend_context(),
|
||||
speculative_moe_a2a_backend_context(),
|
||||
spec_stage_span("draft_extend"),
|
||||
|
||||
@@ -45,7 +45,6 @@ from sglang.srt.model_executor.forward_batch_info import (
|
||||
)
|
||||
from sglang.srt.runtime_context import (
|
||||
get_device,
|
||||
get_parallel,
|
||||
get_schedule,
|
||||
get_spec,
|
||||
)
|
||||
@@ -99,7 +98,7 @@ from sglang.srt.utils.async_probe import (
|
||||
maybe_detect_nan,
|
||||
maybe_detect_oob,
|
||||
)
|
||||
from sglang.srt.utils.common import empty_context, fast_topk
|
||||
from sglang.srt.utils.common import fast_topk
|
||||
from sglang.srt.utils.nvtx_utils import profile_range
|
||||
from sglang.srt.utils.profile_utils import build_step_span_name
|
||||
|
||||
@@ -191,9 +190,6 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase):
|
||||
]
|
||||
# Retain the target's attention topology when swapping TP groups.
|
||||
self.draft_owns_attention = False
|
||||
self.draft_tp_context = (
|
||||
draft_tp_context if get_parallel().enable_dp_attention else empty_context
|
||||
)
|
||||
self.tree_mask_mode = default_tree_mask_mode()
|
||||
self.plan_stream, self.plan_stream_ctx = get_plan_stream(self.device)
|
||||
|
||||
@@ -222,10 +218,7 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase):
|
||||
def init_attention_backends(self):
|
||||
with (
|
||||
draft_pp_context(),
|
||||
self.draft_tp_context(
|
||||
self.draft_runner_list[0].tp_group,
|
||||
owns_attention=self.draft_owns_attention,
|
||||
),
|
||||
draft_tp_context(self.draft_owns_attention),
|
||||
speculative_moe_backend_context(),
|
||||
):
|
||||
super().init_attention_backends()
|
||||
@@ -233,10 +226,7 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase):
|
||||
def init_cuda_graphs(self):
|
||||
with (
|
||||
draft_pp_context(),
|
||||
self.draft_tp_context(
|
||||
self.draft_runner_list[0].tp_group,
|
||||
owns_attention=self.draft_owns_attention,
|
||||
),
|
||||
draft_tp_context(self.draft_owns_attention),
|
||||
speculative_moe_backend_context(),
|
||||
):
|
||||
super().init_cuda_graphs()
|
||||
|
||||
@@ -35,7 +35,6 @@ from sglang.srt.arg_groups.overrides import resolving_view
|
||||
from sglang.srt.configs.hybrid_arch import mambaish_config
|
||||
from sglang.srt.constrained.base_grammar_backend import GrammarMask
|
||||
from sglang.srt.distributed.parallel_state import (
|
||||
GroupCoordinator,
|
||||
get_self_pp_group,
|
||||
patch_pipeline_parallel_group,
|
||||
patch_tensor_parallel_group,
|
||||
@@ -52,6 +51,7 @@ from sglang.srt.mem_cache.allocation import (
|
||||
)
|
||||
from sglang.srt.runtime_context import (
|
||||
get_exec,
|
||||
get_parallel,
|
||||
get_spec,
|
||||
mamba_track_grid,
|
||||
max_speculative_num_draft_tokens,
|
||||
@@ -747,12 +747,19 @@ def draft_pp_context():
|
||||
yield
|
||||
|
||||
|
||||
@contextmanager
|
||||
def draft_tp_context(tp_group: GroupCoordinator, *, owns_attention: bool):
|
||||
# Draft model doesn't use dp and has its own tp group.
|
||||
# We disable mscclpp now because it doesn't support 2 comm groups.
|
||||
with patch_tensor_parallel_group(tp_group, owns_attention=owns_attention):
|
||||
yield
|
||||
def draft_tp_context(owns_attention: bool):
|
||||
"""Enter the TP placement that draft work runs under.
|
||||
|
||||
Work that owns attention (an attention-owning draft, or DSpark's DP-MoE
|
||||
sync) runs on the target's attention-TP group; other draft work keeps the
|
||||
target's TP placement. Enter it from outside any draft scope, where
|
||||
``attn_tp_group`` is still the target's.
|
||||
"""
|
||||
if not owns_attention:
|
||||
return contextlib.nullcontext()
|
||||
return patch_tensor_parallel_group(
|
||||
get_parallel().attn_tp_group, owns_attention=True
|
||||
)
|
||||
|
||||
|
||||
def spec_stage_span(name: str):
|
||||
|
||||
@@ -3,14 +3,10 @@ from typing import Optional
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers.moe.utils import (
|
||||
draft_model_build_scope,
|
||||
speculative_moe_backend_context,
|
||||
)
|
||||
from sglang.srt.layers.moe.utils import draft_model_build_scope
|
||||
from sglang.srt.managers.tp_worker import TpModelWorker
|
||||
from sglang.srt.runtime_context import (
|
||||
get_device,
|
||||
get_parallel,
|
||||
get_schedule,
|
||||
get_spec,
|
||||
)
|
||||
@@ -25,12 +21,8 @@ from sglang.srt.speculative.base_spec_worker import (
|
||||
from sglang.srt.speculative.eagle_utils import default_tree_mask_mode
|
||||
from sglang.srt.speculative.eagle_worker_v2 import EagleDraftWorker, EAGLEWorkerV2
|
||||
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
|
||||
from sglang.srt.speculative.spec_utils import (
|
||||
draft_pp_context,
|
||||
draft_tp_context,
|
||||
get_plan_stream,
|
||||
)
|
||||
from sglang.srt.utils import empty_context, get_bool_env_var
|
||||
from sglang.srt.speculative.spec_utils import draft_pp_context, get_plan_stream
|
||||
from sglang.srt.utils import get_bool_env_var
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
SGLANG_RETURN_ORIGINAL_LOGPROB = get_bool_env_var("SGLANG_RETURN_ORIGINAL_LOGPROB")
|
||||
@@ -90,9 +82,6 @@ class StandaloneDraftWorker(EagleDraftWorker):
|
||||
self.draft_runner = self.draft_worker.model_runner
|
||||
# Retain the target's attention topology when swapping TP groups.
|
||||
self.draft_owns_attention = False
|
||||
self.draft_tp_context = (
|
||||
draft_tp_context if get_parallel().enable_dp_attention else empty_context
|
||||
)
|
||||
self.tree_mask_mode = default_tree_mask_mode()
|
||||
self.plan_stream, self.plan_stream_ctx = get_plan_stream(self.device)
|
||||
# draft_forward reads this (set in EagleDraftWorker.__init__, skipped here).
|
||||
@@ -126,26 +115,6 @@ class StandaloneDraftWorker(EagleDraftWorker):
|
||||
self.init_token_map()
|
||||
self.init_lm_head()
|
||||
|
||||
def init_attention_backends(self):
|
||||
with (
|
||||
self.draft_tp_context(
|
||||
self.draft_runner.tp_group,
|
||||
owns_attention=self.draft_owns_attention,
|
||||
),
|
||||
speculative_moe_backend_context(),
|
||||
):
|
||||
super().init_attention_backends()
|
||||
|
||||
def init_cuda_graphs(self):
|
||||
with (
|
||||
self.draft_tp_context(
|
||||
self.draft_runner.tp_group,
|
||||
owns_attention=self.draft_owns_attention,
|
||||
),
|
||||
speculative_moe_backend_context(),
|
||||
):
|
||||
super().init_cuda_graphs()
|
||||
|
||||
def init_lm_head(self):
|
||||
"""Override to prevent sharing embeddings and lm_head with target model."""
|
||||
# For standalone worker, we don't share embeddings and lm_head
|
||||
|
||||
+2
-1
@@ -89,6 +89,7 @@ from .speculative_cuda_graph_runner import (
|
||||
from .speculative_draft_runner import (
|
||||
EagleDraftRunnerSettings,
|
||||
_configure_runner_for_eagle_draft,
|
||||
_DummyTpGroup,
|
||||
_reset_cuda_graph_test_buffers,
|
||||
_seeded_rng,
|
||||
_single_rank_graph_capture,
|
||||
@@ -531,7 +532,7 @@ def _capture_eagle_draft_extend_graph_runner(
|
||||
"sglang.srt.model_executor.runner.decode_cuda_graph_runner.get_available_gpu_memory",
|
||||
lambda *args, **kwargs: 0.0,
|
||||
),
|
||||
get_parallel().override(attn_cp_size=1, tp_rank=0),
|
||||
get_parallel().override(attn_cp_size=1, tp_rank=0, tp_group=_DummyTpGroup()),
|
||||
):
|
||||
_reset_cuda_graph_test_buffers()
|
||||
return EAGLEDraftExtendCudaGraphRunner(
|
||||
|
||||
@@ -335,7 +335,6 @@ def _configure_runner_for_eagle_draft(
|
||||
runner.spec_algorithm = SpeculativeAlgorithm.EAGLE
|
||||
runner.is_draft_worker = True
|
||||
runner.model = _TinyDraftModel()
|
||||
runner.tp_group = _DummyTpGroup()
|
||||
runner.device_timer = None
|
||||
runner.model_config.spec_hidden_size = settings.hidden_size
|
||||
runner.model_config.dtype = runner.dtype
|
||||
@@ -445,7 +444,7 @@ def _capture_eagle_draft_graph_runner(
|
||||
"sglang.srt.model_executor.runner.decode_cuda_graph_runner.get_available_gpu_memory",
|
||||
lambda *args, **kwargs: 0.0,
|
||||
),
|
||||
get_parallel().override(attn_cp_size=1, tp_rank=0),
|
||||
get_parallel().override(attn_cp_size=1, tp_rank=0, tp_group=_DummyTpGroup()),
|
||||
):
|
||||
_reset_cuda_graph_test_buffers()
|
||||
return EAGLEDraftCudaGraphRunner(
|
||||
@@ -467,7 +466,7 @@ def _capture_frozen_kv_mtp_graph_runner(
|
||||
"sglang.srt.model_executor.runner.decode_cuda_graph_runner.get_available_gpu_memory",
|
||||
lambda *args, **kwargs: 0.0,
|
||||
),
|
||||
get_parallel().override(attn_cp_size=1, tp_rank=0),
|
||||
get_parallel().override(attn_cp_size=1, tp_rank=0, tp_group=_DummyTpGroup()),
|
||||
):
|
||||
_reset_cuda_graph_test_buffers()
|
||||
return FrozenKVMTPCudaGraphRunner(worker)
|
||||
|
||||
Reference in New Issue
Block a user