[Refactor] Drop placement values nothing reads (#41811)
This commit is contained in:
@@ -405,8 +405,6 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
# Destinations visible to prefill but not yet on the transfer queue.
|
||||
self._num_published_destinations = 0
|
||||
self.tp_rank = parallel.tp_rank
|
||||
self.tp_size = parallel.tp_size
|
||||
self.dp_size = parallel.dp_size
|
||||
self.gpu_id = gpu_id
|
||||
self.bootstrap_port = bootstrap_port
|
||||
self.max_total_num_tokens = max_total_num_tokens
|
||||
|
||||
@@ -97,8 +97,6 @@ from sglang.srt.runtime_context import (
|
||||
from sglang.srt.utils import is_npu
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from torch.distributed import ProcessGroup
|
||||
|
||||
from sglang.srt.managers.scheduler import GenerationBatchResult, Scheduler
|
||||
from sglang.srt.mem_cache.memory_pool import KVCache
|
||||
|
||||
@@ -166,7 +164,6 @@ class PrefillBootstrapQueue:
|
||||
metadata_buffers: MetadataBuffers,
|
||||
gpu_id: int,
|
||||
bootstrap_port: int,
|
||||
gloo_group: ProcessGroup,
|
||||
max_total_num_tokens: int,
|
||||
scheduler: Scheduler,
|
||||
scheduler_stage_metrics: SchedulerStageMetricsRecorder,
|
||||
@@ -179,13 +176,11 @@ class PrefillBootstrapQueue:
|
||||
self.metadata_buffers = metadata_buffers
|
||||
self.req_to_metadata_buffer_idx_allocator = req_to_metadata_buffer_idx_allocator
|
||||
self.tp_rank = parallel.tp_rank
|
||||
self.tp_size = parallel.tp_size
|
||||
self.pp_rank = parallel.pp_rank
|
||||
self.pp_size = parallel.pp_size
|
||||
self.gpu_id = gpu_id
|
||||
self.bootstrap_port = bootstrap_port
|
||||
self.queue: List[Req] = []
|
||||
self.gloo_group = gloo_group
|
||||
self.scheduler = scheduler
|
||||
self.scheduler_stage_metrics = scheduler_stage_metrics
|
||||
self.max_total_num_tokens = (
|
||||
|
||||
@@ -314,10 +314,6 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp):
|
||||
)
|
||||
self.alt_stream = alt_stream
|
||||
self.dsa_enable_prefill_cp = is_dsa_enable_prefill_cp()
|
||||
if self.dsa_enable_prefill_cp:
|
||||
self.cp_size = get_parallel().attn_cp_size
|
||||
else:
|
||||
self.cp_size = None
|
||||
if _is_cuda:
|
||||
self.sm_count = deep_gemm.get_num_sms()
|
||||
self.half_device_sm_count = ceil_align(self.sm_count // 2, 8)
|
||||
|
||||
@@ -110,7 +110,6 @@ class FlashMLABackend(FlashInferMLAAttnBackend):
|
||||
|
||||
# get dcp info
|
||||
self.dcp_world_size = get_parallel().attn_dcp_size
|
||||
self.dcp_rank = get_parallel().attn_dcp_rank
|
||||
|
||||
def init_forward_metadata_out_graph(
|
||||
self,
|
||||
|
||||
@@ -454,10 +454,6 @@ class VisionTritonAttention(nn.Module):
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__()
|
||||
use_data_parallel = (
|
||||
kwargs["use_data_parallel"] if "use_data_parallel" in kwargs else False
|
||||
)
|
||||
self.tp_size = 1 if use_data_parallel else get_parallel().attn_tp_size
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -528,10 +524,6 @@ class VisionFlash3Attention(nn.Module):
|
||||
if not (_is_cuda or _is_musa):
|
||||
raise Exception("VisionFlash3Attention is only available for cuda or musa")
|
||||
super().__init__()
|
||||
use_data_parallel = (
|
||||
kwargs["use_data_parallel"] if "use_data_parallel" in kwargs else False
|
||||
)
|
||||
self.tp_size = 1 if use_data_parallel else get_parallel().attn_tp_size
|
||||
|
||||
def forward(
|
||||
self,
|
||||
|
||||
@@ -863,7 +863,6 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
|
||||
shard_offset=rank_shard_offset,
|
||||
shard_size=rank_shard_size,
|
||||
tp_rank=self.tp_rank,
|
||||
tp_size=self.tp_size,
|
||||
use_presharded_weights=self.use_presharded_weights,
|
||||
)
|
||||
|
||||
@@ -895,7 +894,6 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
|
||||
loaded_weight=loaded_weight,
|
||||
shard_id=shard_id,
|
||||
tp_rank=self.tp_rank,
|
||||
tp_size=self.tp_size,
|
||||
)
|
||||
return
|
||||
elif isinstance(param, BlockQuantScaleParameter):
|
||||
@@ -905,7 +903,6 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
|
||||
param.load_merged_column_weight(
|
||||
loaded_weight=loaded_weight,
|
||||
tp_rank=self.tp_rank,
|
||||
tp_size=self.tp_size,
|
||||
)
|
||||
return
|
||||
output_sizes = (
|
||||
@@ -944,7 +941,6 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
|
||||
shard_size=shard_size,
|
||||
use_presharded_weights=self.use_presharded_weights,
|
||||
tp_rank=self.tp_rank,
|
||||
tp_size=self.tp_size,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -788,8 +788,6 @@ class HummingLinearMethod(LinearMethodBase):
|
||||
output_dim=0,
|
||||
weight_loader=orig_weight_loader,
|
||||
)
|
||||
layer.weight.tp_size = layer.tp_size
|
||||
layer.weight.tp_rank = layer.tp_rank
|
||||
set_weight_attrs(layer.weight, extra_weight_attrs)
|
||||
|
||||
param = layer.weight
|
||||
|
||||
@@ -23,7 +23,6 @@ from sglang.srt.layers.quantization.base_config import (
|
||||
)
|
||||
from sglang.srt.layers.quantization.compressed_tensors.utils import should_ignore_layer
|
||||
from sglang.srt.layers.quantization.unquant import UnquantizedLinearMethod
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import (
|
||||
cpu_has_amx_support,
|
||||
is_cpu,
|
||||
@@ -259,8 +258,6 @@ class W8A8Int8MoEMethod(FusedMoEMethodBase):
|
||||
):
|
||||
from sglang.srt.layers.moe.fused_moe_triton import FusedMoeWeightScaleSupported
|
||||
|
||||
tp_size = get_parallel().tp_size
|
||||
|
||||
# WEIGHTS
|
||||
w13_weight = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from typing import List, Optional, Sequence, Tuple
|
||||
from typing import Optional, Sequence, Tuple
|
||||
|
||||
import torch
|
||||
from torch.nn.parameter import Parameter, UninitializedParameter
|
||||
@@ -384,71 +384,6 @@ class VocabParallelEmbedding(torch.nn.Module):
|
||||
added_vocab_end_index,
|
||||
)
|
||||
|
||||
def get_sharded_to_full_mapping(self) -> Optional[List[int]]:
|
||||
"""Get a mapping that can be used to reindex the gathered
|
||||
logits for sampling.
|
||||
|
||||
During sampling, we gather logits from all ranks. The relationship
|
||||
of index->token_id will follow the same format as outlined in the class
|
||||
docstring. However, after the gather, we want to reindex the final
|
||||
logits tensor to map index->token_id one-to-one (the index is always
|
||||
equal the token_id it corresponds to). The indices returned by this
|
||||
method allow us to do that.
|
||||
"""
|
||||
if self.tp_size < 2:
|
||||
return None
|
||||
|
||||
base_embeddings: List[int] = []
|
||||
added_embeddings: List[int] = []
|
||||
padding: List[int] = []
|
||||
for tp_rank in range(self.tp_size):
|
||||
shard_indices = self._get_indices(
|
||||
self.num_embeddings_padded,
|
||||
self.org_vocab_size_padded,
|
||||
self.num_embeddings,
|
||||
self.org_vocab_size,
|
||||
tp_rank,
|
||||
self.tp_size,
|
||||
)
|
||||
range_start = self.num_embeddings_per_partition * tp_rank
|
||||
range_end = self.num_embeddings_per_partition * (tp_rank + 1)
|
||||
base_embeddings.extend(
|
||||
range(range_start, range_start + shard_indices.num_org_elements)
|
||||
)
|
||||
padding.extend(
|
||||
range(
|
||||
range_start + shard_indices.num_org_elements,
|
||||
range_start + shard_indices.num_org_elements_padded,
|
||||
)
|
||||
)
|
||||
added_embeddings.extend(
|
||||
range(
|
||||
range_start + shard_indices.num_org_elements_padded,
|
||||
range_start
|
||||
+ shard_indices.num_org_elements_padded
|
||||
+ shard_indices.num_added_elements,
|
||||
)
|
||||
)
|
||||
padding.extend(
|
||||
range(
|
||||
range_start
|
||||
+ shard_indices.num_org_elements_padded
|
||||
+ shard_indices.num_added_elements,
|
||||
range_start
|
||||
+ shard_indices.num_org_elements_padded
|
||||
+ shard_indices.num_added_elements_padded,
|
||||
)
|
||||
)
|
||||
assert (
|
||||
range_start
|
||||
+ shard_indices.num_org_elements_padded
|
||||
+ shard_indices.num_added_elements_padded
|
||||
== range_end
|
||||
)
|
||||
ret = base_embeddings + added_embeddings + padding
|
||||
assert len(ret) == self.num_embeddings_padded
|
||||
return ret
|
||||
|
||||
def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor):
|
||||
output_dim = getattr(param, "output_dim", None)
|
||||
packed_dim = getattr(param, "packed_dim", None)
|
||||
|
||||
@@ -1576,7 +1576,6 @@ class Scheduler(
|
||||
metadata_buffers=self.disagg_metadata_buffers,
|
||||
gpu_id=get_device().gpu_id,
|
||||
bootstrap_port=get_disagg().disaggregation_bootstrap_port,
|
||||
gloo_group=self.attn_tp_cpu_group,
|
||||
max_total_num_tokens=self.max_total_num_tokens,
|
||||
scheduler=self,
|
||||
scheduler_stage_metrics=self.scheduler_stage_metrics,
|
||||
@@ -2308,9 +2307,7 @@ class Scheduler(
|
||||
mm_receiver=self.mm_receiver,
|
||||
tp_group=self.tp_group,
|
||||
tp_cpu_group=self.tp_cpu_group,
|
||||
attn_tp_group=self.attn_tp_group,
|
||||
attn_tp_cpu_group=self.attn_tp_cpu_group,
|
||||
attn_cp_group=self.attn_cp_group,
|
||||
attn_cp_cpu_group=self.attn_cp_cpu_group,
|
||||
world_group=self.world_group,
|
||||
server_args=self.server_args,
|
||||
|
||||
@@ -65,9 +65,7 @@ class SchedulerRequestReceiver:
|
||||
mm_receiver: Any
|
||||
tp_group: Any
|
||||
tp_cpu_group: Any
|
||||
attn_tp_group: Any
|
||||
attn_tp_cpu_group: Any
|
||||
attn_cp_group: Any
|
||||
attn_cp_cpu_group: Any
|
||||
world_group: Any
|
||||
server_args: ServerArgs
|
||||
|
||||
@@ -226,7 +226,6 @@ class BaseRunner(ABC):
|
||||
self.model_runner = model_runner
|
||||
self.device = model_runner.device
|
||||
self.device_module = torch.get_device_module(self.device)
|
||||
self.tp_size = get_parallel().tp_size
|
||||
# elastic-EP scale-up rewrites dp_size on the published config
|
||||
self.dp_size = get_parallel().dp_size
|
||||
self.pp_size = get_parallel().pp_size
|
||||
@@ -238,7 +237,6 @@ class BaseRunner(ABC):
|
||||
)
|
||||
self.enable_return_hidden_states = self.return_hidden_states_mode.need_capture()
|
||||
self.attn_tp_size = get_parallel().attn_tp_size
|
||||
self.attn_tp_rank = get_parallel().attn_tp_rank
|
||||
self.tbo_plugin = TboCudaGraphRunnerPlugin()
|
||||
|
||||
def warmup(self) -> None:
|
||||
|
||||
@@ -267,8 +267,6 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
|
||||
self.speculative_algorithm = get_spec().speculative_algorithm
|
||||
self.enable_profile_cuda_graph = get_exec().graph.enable_profile_cuda_graph
|
||||
|
||||
self.attn_tp_size = get_parallel().attn_tp_size
|
||||
self.attn_tp_rank = get_parallel().attn_tp_rank
|
||||
# True if the DSA or MLA prefill-CP flavor is active. These flavors
|
||||
# feed a zigzag-split rank-local layout into the runner; MHA-arch
|
||||
# prefill CP (Qwen3/Qwen2 MoE via PR #18233) keeps an
|
||||
|
||||
@@ -427,7 +427,6 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
self.moe_fusions = self.model_runner.moe_fusions
|
||||
self.dsa_indexers = getattr(self.model_runner, "dsa_indexers", None)
|
||||
|
||||
self.dp_size = get_parallel().dp_size
|
||||
self.require_mlp_tp_gather = require_mlp_tp_gather()
|
||||
self.require_attn_tp_gather = require_attn_tp_gather()
|
||||
|
||||
|
||||
@@ -115,7 +115,6 @@ class EAGLEDraftCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
# Fields the parent's capture() reads:
|
||||
self.device = model_runner.device
|
||||
self.device_module = torch.get_device_module(self.device)
|
||||
self.tp_size = model_runner.tp_size
|
||||
self.attn_dp_size = deployment_attn_dp_size()
|
||||
self.pp_size = get_parallel().pp_size
|
||||
self.enable_torch_compile = get_flags().capture.enable_torch_compile
|
||||
|
||||
@@ -112,7 +112,6 @@ class EAGLEDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
# Fields the parent's capture() reads:
|
||||
self.device = model_runner.device
|
||||
self.device_module = torch.get_device_module(self.device)
|
||||
self.tp_size = model_runner.tp_size
|
||||
self.attn_dp_size = deployment_attn_dp_size()
|
||||
self.pp_size = get_parallel().pp_size
|
||||
self.enable_torch_compile = get_flags().capture.enable_torch_compile
|
||||
|
||||
@@ -99,7 +99,6 @@ class FrozenKVMTPCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
self.require_mlp_tp_gather = require_mlp_tp_gather()
|
||||
self.require_mlp_sync = require_mlp_sync()
|
||||
self.require_attn_tp_gather = require_attn_tp_gather()
|
||||
self.tp_size = self.model_runner.tp_size
|
||||
self.attn_dp_size = deployment_attn_dp_size()
|
||||
self.pp_size = get_parallel().pp_size
|
||||
self.speculative_num_steps = get_spec().speculative_num_steps
|
||||
|
||||
@@ -155,7 +155,6 @@ class MultiLayerEagleDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
# Fields the parent's capture() reads:
|
||||
self.device = model_runner.device
|
||||
self.device_module = torch.get_device_module(self.device)
|
||||
self.tp_size = model_runner.tp_size
|
||||
self.dp_size = get_parallel().dp_size
|
||||
self.pp_size = get_parallel().pp_size
|
||||
self.enable_torch_compile = get_flags().capture.enable_torch_compile
|
||||
|
||||
@@ -101,7 +101,6 @@ class NGRAMWorker(BaseSpecWorker):
|
||||
self.enable_overlap = not get_schedule().disable_overlap_schedule
|
||||
self._target_worker = target_worker
|
||||
self.model_runner = target_worker.model_runner
|
||||
self.tp_rank = self.model_runner.tp_rank
|
||||
self.page_size = get_schedule().page_size
|
||||
self.draft_token_num: int = get_spec().speculative_num_draft_tokens
|
||||
self.max_trie_depth: int = get_spec().speculative_ngram_max_trie_depth
|
||||
|
||||
Reference in New Issue
Block a user