[Fix] Load a draft's tensor weight update from its deployment TP rank (#42346)

This commit is contained in:
Cheng Wan
2026-10-03 14:52:25 -07:00
committed by GitHub
parent 061e712bab
commit 1a18de4b29
2 changed files with 60 additions and 2 deletions
@@ -16,7 +16,7 @@ from sglang.srt.model_loader.loader import (
from sglang.srt.model_loader.utils import set_default_torch_dtype
from sglang.srt.model_loader.weight_utils import default_weight_loader
from sglang.srt.platforms import current_platform
from sglang.srt.runtime_context import get_model
from sglang.srt.runtime_context import get_model, get_parallel
from sglang.srt.utils import (
MultiprocessingSerializer,
dynamic_import,
@@ -361,8 +361,12 @@ class WeightUpdater:
device_module = torch.get_device_module(self.device)
infered_device = device_module.current_device()
# The payload lists one entry per deployment TP rank. An
# attention-owning draft records its narrowed rank at init, so read the
# deployment rank here, on the scheduler thread outside any draft scope.
tp_rank = get_parallel().tp_rank
named_tensors = [
(name, _unwrap_tensor(tensor, tp_rank=self.tp_rank, device=infered_device))
(name, _unwrap_tensor(tensor, tp_rank=tp_rank, device=infered_device))
for name, tensor in named_tensors
]
if load_format == "direct":