diff --git a/python/sglang/srt/model_executor/model_runner_components/weight_updater.py b/python/sglang/srt/model_executor/model_runner_components/weight_updater.py index 2f418849b1..5f164c7904 100644 --- a/python/sglang/srt/model_executor/model_runner_components/weight_updater.py +++ b/python/sglang/srt/model_executor/model_runner_components/weight_updater.py @@ -16,7 +16,7 @@ from sglang.srt.model_loader.loader import ( from sglang.srt.model_loader.utils import set_default_torch_dtype from sglang.srt.model_loader.weight_utils import default_weight_loader from sglang.srt.platforms import current_platform -from sglang.srt.runtime_context import get_model +from sglang.srt.runtime_context import get_model, get_parallel from sglang.srt.utils import ( MultiprocessingSerializer, dynamic_import, @@ -361,8 +361,12 @@ class WeightUpdater: device_module = torch.get_device_module(self.device) infered_device = device_module.current_device() + # The payload lists one entry per deployment TP rank. An + # attention-owning draft records its narrowed rank at init, so read the + # deployment rank here, on the scheduler thread outside any draft scope. + tp_rank = get_parallel().tp_rank named_tensors = [ - (name, _unwrap_tensor(tensor, tp_rank=self.tp_rank, device=infered_device)) + (name, _unwrap_tensor(tensor, tp_rank=tp_rank, device=infered_device)) for name, tensor in named_tensors ] if load_format == "direct": diff --git a/test/registered/unit/model_executor/test_weight_update_tensor_rank.py b/test/registered/unit/model_executor/test_weight_update_tensor_rank.py new file mode 100644 index 0000000000..587cccd1f4 --- /dev/null +++ b/test/registered/unit/model_executor/test_weight_update_tensor_rank.py @@ -0,0 +1,54 @@ +"""A tensor weight update loads the entry of the process's deployment TP rank.""" + +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +import torch + +from sglang.srt.model_executor.model_runner_components.weight_updater import ( + LocalSerializedTensor, + WeightUpdater, +) +from sglang.srt.runtime_context import get_parallel +from sglang.srt.utils import MultiprocessingSerializer +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=10, suite="base-a-test-cpu") + + +class TestTensorUpdateRank(unittest.TestCase): + def test_a_narrowed_updater_reads_the_deployment_rank_entry(self): + model = torch.nn.Linear(2, 2, bias=False) + # An attention-owning draft records its attention-TP rank at init. + updater = WeightUpdater( + tp_rank=0, + device="cpu", + gpu_id=0, + model_config=None, + custom_weight_loaders={}, + get_model=lambda: model, + update_model_fields=lambda *args, **kwargs: None, + recapture_cuda_graph=lambda: None, + get_model_runner=lambda: None, + ) + per_rank = [torch.full((2, 2), float(rank)) for rank in range(2)] + payload = LocalSerializedTensor( + values=[MultiprocessingSerializer.serialize(t) for t in per_rank] + ) + with ( + patch( + "sglang.srt.model_executor.model_runner_components.weight_updater.get_model", + return_value=SimpleNamespace(weight_cache_mode="off"), + ), + get_parallel().override(tp_rank=1), + ): + success, message = updater.update_weights_from_tensor( + [("weight", payload)], load_format="direct" + ) + self.assertTrue(success, message) + self.assertTrue(torch.equal(model.weight.detach(), per_rank[1])) + + +if __name__ == "__main__": + unittest.main()