[Fix] Load a draft's tensor weight update from its deployment TP rank (#42346)
This commit is contained in:
@@ -16,7 +16,7 @@ from sglang.srt.model_loader.loader import (
|
||||
from sglang.srt.model_loader.utils import set_default_torch_dtype
|
||||
from sglang.srt.model_loader.weight_utils import default_weight_loader
|
||||
from sglang.srt.platforms import current_platform
|
||||
from sglang.srt.runtime_context import get_model
|
||||
from sglang.srt.runtime_context import get_model, get_parallel
|
||||
from sglang.srt.utils import (
|
||||
MultiprocessingSerializer,
|
||||
dynamic_import,
|
||||
@@ -361,8 +361,12 @@ class WeightUpdater:
|
||||
device_module = torch.get_device_module(self.device)
|
||||
infered_device = device_module.current_device()
|
||||
|
||||
# The payload lists one entry per deployment TP rank. An
|
||||
# attention-owning draft records its narrowed rank at init, so read the
|
||||
# deployment rank here, on the scheduler thread outside any draft scope.
|
||||
tp_rank = get_parallel().tp_rank
|
||||
named_tensors = [
|
||||
(name, _unwrap_tensor(tensor, tp_rank=self.tp_rank, device=infered_device))
|
||||
(name, _unwrap_tensor(tensor, tp_rank=tp_rank, device=infered_device))
|
||||
for name, tensor in named_tensors
|
||||
]
|
||||
if load_format == "direct":
|
||||
|
||||
Reference in New Issue
Block a user