[RL] Add RL weight-update sessions and support updating spec draft runners (#40777)
Co-authored-by: Jiajun Li <guapisolo@gmail.com> Co-authored-by: Nan Jiang <59716405+nanjiangwill@users.noreply.github.com> Co-authored-by: maocheng23 <maocheng@berkeley.edu> Co-authored-by: Byron Hsu <byron+per@periodiclabs.ai> Co-authored-by: JD-ETH <jaedon.guo@gmail.com> Co-authored-by: Letian Ruan <ruanlt420@gmail.com> Co-authored-by: hnyls2002 <lsyincs@gmail.com>
This commit is contained in:
co-authored by
Jiajun Li
Nan Jiang
maocheng23
Byron Hsu
JD-ETH
Letian Ruan
hnyls2002
parent
f4b90382a2
commit
f4d9d2e670
@@ -45,6 +45,16 @@ class EngineBase(ABC):
|
||||
"""Flush the cache of the engine."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def begin_weight_update(self, selector: str = "all"):
|
||||
"""Open a weight-update session; update_weights_from_* must run inside one."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def end_weight_update(self):
|
||||
"""Close the weight-update session and finalize quantized weights."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def update_weights_from_tensor(
|
||||
self,
|
||||
|
||||
@@ -69,9 +69,11 @@ from sglang.srt.managers.data_parallel_controller import (
|
||||
)
|
||||
from sglang.srt.managers.detokenizer_manager import run_detokenizer_process
|
||||
from sglang.srt.managers.io_struct import (
|
||||
BeginWeightUpdateReqInput,
|
||||
CloseSessionReqInput,
|
||||
DestroyWeightsUpdateGroupReqInput,
|
||||
EmbeddingReqInput,
|
||||
EndWeightUpdateReqInput,
|
||||
GenerateReqInput,
|
||||
GetWeightsByNameReqInput,
|
||||
InitWeightsUpdateGroupReqInput,
|
||||
@@ -1537,6 +1539,20 @@ class Engine(EngineScoreMixin, EngineBase):
|
||||
self.tokenizer_manager.destroy_weights_update_group(obj, None)
|
||||
)
|
||||
|
||||
def begin_weight_update(self, selector: str = "all"):
|
||||
"""Open a weight-update session; close it with end_weight_update()."""
|
||||
obj = BeginWeightUpdateReqInput(selector=selector)
|
||||
return self.loop.run_until_complete(
|
||||
self.tokenizer_manager.begin_weight_update(obj, None)
|
||||
)
|
||||
|
||||
def end_weight_update(self):
|
||||
"""Close the session and finalize quantized weights into kernel layout."""
|
||||
obj = EndWeightUpdateReqInput()
|
||||
return self.loop.run_until_complete(
|
||||
self.tokenizer_manager.end_weight_update(obj, None)
|
||||
)
|
||||
|
||||
def update_weights_from_distributed(
|
||||
self,
|
||||
names: list[str],
|
||||
|
||||
@@ -117,6 +117,7 @@ from sglang.srt.function_call.function_call_parser import FunctionCallParser
|
||||
from sglang.srt.managers.io_struct import (
|
||||
AbortReq,
|
||||
AttachHiCacheStorageReqInput,
|
||||
BeginWeightUpdateReqInput,
|
||||
CheckWeightsReqInput,
|
||||
CloseSessionReqInput,
|
||||
ConfigureLoggingReq,
|
||||
@@ -124,6 +125,7 @@ from sglang.srt.managers.io_struct import (
|
||||
DestroyWeightsUpdateGroupReqInput,
|
||||
DumperControlReqInput,
|
||||
EmbeddingReqInput,
|
||||
EndWeightUpdateReqInput,
|
||||
GenerateReqInput,
|
||||
GetWeightsByNameReqInput,
|
||||
InitWeightsSendGroupForRemoteInstanceReqInput,
|
||||
@@ -1368,6 +1370,36 @@ async def update_weights_from_tensor(
|
||||
)
|
||||
|
||||
|
||||
@app.post("/begin_weight_update")
|
||||
@auth_level(AuthLevel.ADMIN_OPTIONAL)
|
||||
async def begin_weight_update(
|
||||
obj: Annotated[BeginWeightUpdateReqInput, Body()], request: Request
|
||||
):
|
||||
"""Open a weight-update session so in-place-quantized weights become loadable."""
|
||||
success, message = await _global_state.tokenizer_manager.begin_weight_update(
|
||||
obj, request
|
||||
)
|
||||
return ORJSONResponse(
|
||||
{"success": success, "message": message},
|
||||
status_code=HTTPStatus.OK if success else HTTPStatus.BAD_REQUEST,
|
||||
)
|
||||
|
||||
|
||||
@app.post("/end_weight_update")
|
||||
@auth_level(AuthLevel.ADMIN_OPTIONAL)
|
||||
async def end_weight_update(
|
||||
obj: Annotated[EndWeightUpdateReqInput, Body()], request: Request
|
||||
):
|
||||
"""Close the weight-update session and finalize quantized weights."""
|
||||
success, message = await _global_state.tokenizer_manager.end_weight_update(
|
||||
obj, request
|
||||
)
|
||||
return ORJSONResponse(
|
||||
{"success": success, "message": message},
|
||||
status_code=HTTPStatus.OK if success else HTTPStatus.BAD_REQUEST,
|
||||
)
|
||||
|
||||
|
||||
@app.post("/update_weights_from_distributed")
|
||||
@auth_level(AuthLevel.ADMIN_OPTIONAL)
|
||||
async def update_weights_from_distributed(
|
||||
|
||||
@@ -86,6 +86,12 @@ class HttpServerEngineAdapter(EngineBase):
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
|
||||
def begin_weight_update(self, selector: str = "all"):
|
||||
return self._make_request("begin_weight_update", {"selector": selector})
|
||||
|
||||
def end_weight_update(self):
|
||||
return self._make_request("end_weight_update")
|
||||
|
||||
def update_weights_from_tensor(
|
||||
self,
|
||||
named_tensors: List[Tuple[str, torch.Tensor]],
|
||||
|
||||
@@ -46,6 +46,13 @@ class QuantizeMethodBase(ABC):
|
||||
"""
|
||||
return
|
||||
|
||||
def restore_weights_before_loading(self, layer: nn.Module) -> None:
|
||||
"""Undo an in-place repack so checkpoint-format weights can be loaded again.
|
||||
|
||||
Needed only when `process_weights_after_loading` changes parameter shapes.
|
||||
"""
|
||||
return
|
||||
|
||||
|
||||
class LinearMethodBase(QuantizeMethodBase):
|
||||
"""Base class for different (maybe quantized) linear methods."""
|
||||
|
||||
@@ -1252,6 +1252,9 @@ class CompressedTensorsFusedMoEMethod(FusedMoEMethodBase):
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
layer.scheme.process_weights_after_loading(layer)
|
||||
|
||||
def restore_weights_before_loading(self, layer: torch.nn.Module) -> None:
|
||||
layer.scheme.restore_weights_before_loading(layer)
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
|
||||
+3
@@ -99,6 +99,9 @@ class CompressedTensorsMoEScheme(BaseMoEScheme):
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def restore_weights_before_loading(self, layer: torch.nn.Module) -> None:
|
||||
return
|
||||
|
||||
@abstractmethod
|
||||
def apply_weights(
|
||||
self,
|
||||
|
||||
@@ -1875,6 +1875,8 @@ class UpdateWeightsFromDistributedReqInput(BaseReq, kw_only=True):
|
||||
weight_version: Optional[str] = None
|
||||
# Optional format specification for loading
|
||||
load_format: Optional[str] = None
|
||||
# which runners the op applies to
|
||||
selector: Literal["target", "draft", "all"] = "all"
|
||||
# Whether to call torch.cuda.empty_cache() during flush
|
||||
torch_empty_cache: bool = False
|
||||
|
||||
@@ -1901,8 +1903,8 @@ class UpdateWeightsFromTensorReqInput(BaseReq, kw_only=True):
|
||||
abort_all_requests: bool = False
|
||||
# Optional: Update weight version along with weights
|
||||
weight_version: Optional[str] = None
|
||||
# Optional: Determine whether to disable updating the draft model
|
||||
disable_draft_model: Optional[bool] = None
|
||||
# which runners the op applies to
|
||||
selector: Literal["target", "draft", "all"] = "all"
|
||||
# Whether to call torch.cuda.empty_cache() during flush
|
||||
torch_empty_cache: bool = False
|
||||
|
||||
@@ -2054,15 +2056,41 @@ class ResumeMemoryOccupationReqOutput(BaseReq, kw_only=True):
|
||||
pass
|
||||
|
||||
|
||||
class BeginWeightUpdateReqInput(BaseReq, kw_only=True):
|
||||
"""Open a weight-update session: restore in-place-packed weights so new ones can load."""
|
||||
|
||||
selector: Literal["target", "draft", "all"] = "all"
|
||||
|
||||
|
||||
class BeginWeightUpdateReqOutput(BaseReq, kw_only=True):
|
||||
success: bool
|
||||
message: str
|
||||
|
||||
|
||||
class EndWeightUpdateReqInput(BaseReq, kw_only=True):
|
||||
"""Close the weight-update session opened by BeginWeightUpdateReqInput."""
|
||||
|
||||
|
||||
class EndWeightUpdateReqOutput(BaseReq, kw_only=True):
|
||||
success: bool
|
||||
message: str
|
||||
|
||||
|
||||
class CheckWeightsReqInput(BaseReq, kw_only=True):
|
||||
action: str = "checksum"
|
||||
allow_quant_error: bool = False
|
||||
# Substrings of tensor names to exclude from reset/compare/checksum.
|
||||
skip_tensor_list: Optional[List[str]] = None
|
||||
# which runners the op applies to
|
||||
selector: Literal["target", "draft", "all"] = "all"
|
||||
|
||||
|
||||
# Wire versions of the pydantic ParallelismInfo/ChecksumInfo in
|
||||
# sglang.srt.utils.weight_checker. Not array_like: the payload is read by field
|
||||
# name and re-serialized to JSON, so it must stay a {field: value} map.
|
||||
class ParallelismInfo(msgspec.Struct, kw_only=True):
|
||||
# "target", or a draft role such as "draft" / "draft_step_0"
|
||||
role: str
|
||||
tp_rank: int
|
||||
tp_size: int
|
||||
dp_rank: int
|
||||
@@ -2076,7 +2104,8 @@ class ParallelismInfo(msgspec.Struct, kw_only=True):
|
||||
class ChecksumInfo(msgspec.Struct, kw_only=True):
|
||||
checksums: Dict[str, str]
|
||||
per_gpu_checksum: str
|
||||
parallelism_info: ParallelismInfo
|
||||
# one entry per role (target plus each draft runner); all share the GPU rank
|
||||
parallelism_info: List[ParallelismInfo]
|
||||
|
||||
|
||||
class CheckWeightsReqOutput(BaseReq, kw_only=True):
|
||||
|
||||
@@ -126,6 +126,7 @@ from sglang.srt.managers.io_struct import (
|
||||
AttachHiCacheStorageReqOutput,
|
||||
BatchTokenizedEmbeddingReqInput,
|
||||
BatchTokenizedGenerateReqInput,
|
||||
BeginWeightUpdateReqInput,
|
||||
CheckWeightsReqInput,
|
||||
ClearHiCacheReqInput,
|
||||
ClearHiCacheReqOutput,
|
||||
@@ -137,6 +138,7 @@ from sglang.srt.managers.io_struct import (
|
||||
DetachHiCacheStorageReqOutput,
|
||||
DumperControlReqInput,
|
||||
DumperControlReqOutput,
|
||||
EndWeightUpdateReqInput,
|
||||
ExpertDistributionReq,
|
||||
ExpertDistributionReqOutput,
|
||||
ExpertDistributionReqType,
|
||||
@@ -1738,6 +1740,14 @@ class Scheduler(
|
||||
SendWeightsToRemoteInstanceReqInput,
|
||||
self.send_weights_to_remote_instance,
|
||||
),
|
||||
(
|
||||
BeginWeightUpdateReqInput,
|
||||
self.weight_updater.begin_weight_update,
|
||||
),
|
||||
(
|
||||
EndWeightUpdateReqInput,
|
||||
self.weight_updater.end_weight_update,
|
||||
),
|
||||
(
|
||||
UpdateWeightsFromDistributedReqInput,
|
||||
self.weight_updater.update_weights_from_distributed,
|
||||
|
||||
@@ -1,12 +1,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import time
|
||||
import traceback
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Callable, Dict, Iterator, Optional, Tuple
|
||||
from typing import Any, Callable, Dict, Iterator, List, Optional, Set, Tuple
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
@@ -19,11 +18,15 @@ from sglang.srt.constants import (
|
||||
)
|
||||
from sglang.srt.disaggregation.utils import DisaggregationMode
|
||||
from sglang.srt.managers.io_struct import (
|
||||
BeginWeightUpdateReqInput,
|
||||
BeginWeightUpdateReqOutput,
|
||||
ChecksumInfo,
|
||||
CheckWeightsReqInput,
|
||||
CheckWeightsReqOutput,
|
||||
DestroyWeightsUpdateGroupReqInput,
|
||||
DestroyWeightsUpdateGroupReqOutput,
|
||||
EndWeightUpdateReqInput,
|
||||
EndWeightUpdateReqOutput,
|
||||
GetWeightsByNameReqInput,
|
||||
GetWeightsByNameReqOutput,
|
||||
InitWeightsUpdateGroupReqInput,
|
||||
@@ -42,35 +45,43 @@ from sglang.srt.managers.io_struct import (
|
||||
UpdateWeightsFromTensorReqOutput,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_model
|
||||
from sglang.srt.utils.weight_checker import overall_checksum
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_draft_model_runner(draft_worker):
|
||||
# DFlash / FrozenKVMTP workers expose draft_model_runner directly
|
||||
runner = getattr(draft_worker, "draft_model_runner", None)
|
||||
if runner is not None:
|
||||
return runner
|
||||
# EAGLEWorkerV2: _draft_worker.draft_runner
|
||||
inner = getattr(draft_worker, "_draft_worker", None)
|
||||
if inner is not None:
|
||||
runner = getattr(inner, "draft_runner", None)
|
||||
if runner is not None:
|
||||
return runner
|
||||
return None
|
||||
def _merge_checksum_payloads(role_payloads: List[Tuple[str, Dict]]) -> Dict:
|
||||
merged: Dict[str, str] = {}
|
||||
parallelism_infos = []
|
||||
for role, p in role_payloads:
|
||||
for name, chk in p["checksums"].items():
|
||||
# only draft roles are prefixed, so target keys stay stable
|
||||
key = name if role == "target" else f"{role}.{name}"
|
||||
if key in merged:
|
||||
raise ValueError(f"checksum key collision: {key}")
|
||||
merged[key] = chk
|
||||
parallelism_infos.append(p["parallelism_info"])
|
||||
return {
|
||||
"checksums": merged,
|
||||
"per_gpu_checksum": overall_checksum(merged),
|
||||
"parallelism_info": parallelism_infos,
|
||||
}
|
||||
|
||||
|
||||
def _merge_checksum_payloads(target: Dict, draft: Dict) -> Dict:
|
||||
merged_checksums = dict(target["checksums"])
|
||||
for name, chk in draft["checksums"].items():
|
||||
merged_checksums[f"draft.{name}"] = chk
|
||||
h = hashlib.sha256()
|
||||
for name in sorted(merged_checksums):
|
||||
h.update(name.encode())
|
||||
h.update(merged_checksums[name].encode())
|
||||
target["checksums"] = merged_checksums
|
||||
target["per_gpu_checksum"] = h.hexdigest()
|
||||
return target
|
||||
def _parse_runner_selector(selector: str) -> Set[str]:
|
||||
if selector == "all":
|
||||
return {"target", "draft"}
|
||||
if selector in ("target", "draft"):
|
||||
return {selector}
|
||||
raise ValueError(
|
||||
f"invalid selector {selector!r}; expected 'target', 'draft', or 'all'"
|
||||
)
|
||||
|
||||
|
||||
class _WeightUpdateSession(msgspec.Struct, frozen=True):
|
||||
# recorded at begin so end finalizes the same runners
|
||||
selector: str
|
||||
loaded_weights: bool = False
|
||||
|
||||
|
||||
@dataclass(kw_only=True, slots=True)
|
||||
@@ -85,6 +96,8 @@ class SchedulerWeightUpdaterManager:
|
||||
metrics_collector: Optional[Any] = None
|
||||
offload_tags: set = field(default_factory=set)
|
||||
stashed_model_static_state: Any = None
|
||||
# replicated on every TP rank, so a rejected call returns on all ranks before any barrier
|
||||
_session: Optional[_WeightUpdateSession] = None
|
||||
|
||||
@contextmanager
|
||||
def _observe_weight_load(self, source: str) -> Iterator[None]:
|
||||
@@ -114,11 +127,19 @@ class SchedulerWeightUpdaterManager:
|
||||
def update_weights_from_disk(self, recv_req: UpdateWeightFromDiskReqInput):
|
||||
"""In-place update of the weights from disk."""
|
||||
with self._observe_weight_load("disk"):
|
||||
success, message = self.tp_worker.update_weights_from_disk(recv_req)
|
||||
tp_success = success
|
||||
if success and self.draft_worker is not None:
|
||||
success, message = self.draft_worker.update_weights_from_disk(recv_req)
|
||||
if tp_success:
|
||||
success, message = True, "Succeeded to update model weights."
|
||||
target_updated = False
|
||||
for role, runner in self._select_runners():
|
||||
success, message = runner.weight_updater.update_weights_from_disk(
|
||||
recv_req.model_path,
|
||||
recv_req.load_format,
|
||||
recapture_cuda_graph=recv_req.recapture_cuda_graph,
|
||||
)
|
||||
if not success:
|
||||
break
|
||||
target_updated |= role == "target"
|
||||
# the served weights changed even if a draft runner failed afterwards
|
||||
if target_updated:
|
||||
self.flush_cache_after_weight_update(recv_req)
|
||||
if success:
|
||||
self.record_weight_version_after_update(recv_req.weight_version)
|
||||
@@ -141,31 +162,82 @@ class SchedulerWeightUpdaterManager:
|
||||
success, message = self.tp_worker.destroy_weights_update_group(recv_req)
|
||||
return DestroyWeightsUpdateGroupReqOutput(success=success, message=message)
|
||||
|
||||
def _select_runners(self, selector: str = "all") -> List[Tuple[str, Any]]:
|
||||
roles = _parse_runner_selector(selector)
|
||||
runners: List[Tuple[str, Any]] = []
|
||||
if "target" in roles:
|
||||
runners += self.tp_worker.weight_update_runners()
|
||||
if "draft" in roles and self.draft_worker is not None:
|
||||
runners += self.draft_worker.weight_update_runners()
|
||||
return runners
|
||||
|
||||
def update_weights_from_distributed(
|
||||
self,
|
||||
recv_req: UpdateWeightsFromDistributedReqInput,
|
||||
) -> Tuple[bool, str]:
|
||||
"""Update the online model parameter."""
|
||||
"""Update the online model parameter, fanning out to the selected runners."""
|
||||
if self._session is None:
|
||||
return UpdateWeightsFromDistributedReqOutput(
|
||||
success=False,
|
||||
message="update_weights_from_distributed must run between "
|
||||
"begin_weight_update() and end_weight_update()",
|
||||
)
|
||||
with self._observe_weight_load("distributed"):
|
||||
success, message = self.tp_worker.update_weights_from_distributed(recv_req)
|
||||
# only the target runner joined the update group; drafts load its receive
|
||||
target = self.tp_worker.model_runner.weight_updater
|
||||
try:
|
||||
weights = target.receive_weights_from_distributed(
|
||||
names=recv_req.names,
|
||||
dtypes=recv_req.dtypes,
|
||||
shapes=recv_req.shapes,
|
||||
group_name=recv_req.group_name,
|
||||
load_format=recv_req.load_format,
|
||||
)
|
||||
except Exception as e:
|
||||
success, message = False, f"Failed to receive weights: {e}"
|
||||
logger.error(message)
|
||||
else:
|
||||
success, message = True, "Succeeded to update parameter online."
|
||||
for _, runner in self._select_runners(recv_req.selector):
|
||||
success, message = (
|
||||
runner.weight_updater.load_weights_from_distributed(weights)
|
||||
)
|
||||
if not success:
|
||||
break
|
||||
if success:
|
||||
self._session = msgspec.structs.replace(
|
||||
self._session, loaded_weights=True
|
||||
)
|
||||
self.flush_cache_after_weight_update(recv_req)
|
||||
self.record_weight_version_after_update(recv_req.weight_version)
|
||||
else:
|
||||
logger.error(message)
|
||||
return UpdateWeightsFromDistributedReqOutput(
|
||||
success=success, message=message
|
||||
)
|
||||
|
||||
def update_weights_from_tensor(self, recv_req: UpdateWeightsFromTensorReqInput):
|
||||
"""Update the online model parameter from tensors."""
|
||||
"""Update the online model parameter from tensors on the selected runners."""
|
||||
if self._session is None:
|
||||
return UpdateWeightsFromTensorReqOutput(
|
||||
success=False,
|
||||
message="update_weights_from_tensor must run between "
|
||||
"begin_weight_update() and end_weight_update()",
|
||||
)
|
||||
with self._observe_weight_load("tensor"):
|
||||
if recv_req.disable_draft_model:
|
||||
worker = self.tp_worker
|
||||
else:
|
||||
worker = self.draft_worker or self.tp_worker
|
||||
success, message = worker.update_weights_from_tensor(recv_req)
|
||||
named_tensors = self.tp_worker.deserialize_own_rank(
|
||||
recv_req.serialized_named_tensors
|
||||
)
|
||||
success, message = True, "Success"
|
||||
for _, runner in self._select_runners(recv_req.selector):
|
||||
success, message = runner.weight_updater.update_weights_from_tensor(
|
||||
named_tensors=named_tensors,
|
||||
load_format=recv_req.load_format,
|
||||
)
|
||||
if not success:
|
||||
break
|
||||
if success:
|
||||
self._session = msgspec.structs.replace(
|
||||
self._session, loaded_weights=True
|
||||
)
|
||||
self.flush_cache_after_weight_update(recv_req)
|
||||
self.record_weight_version_after_update(recv_req.weight_version)
|
||||
else:
|
||||
@@ -176,11 +248,17 @@ class SchedulerWeightUpdaterManager:
|
||||
def update_weights_from_ipc(self, recv_req: UpdateWeightsFromIPCReqInput):
|
||||
"""Update the online model parameter from IPC for checkpoint-engine integration."""
|
||||
with self._observe_weight_load("ipc"):
|
||||
success, message = self.tp_worker.update_weights_from_ipc(recv_req)
|
||||
tp_success = success
|
||||
if success and self.draft_worker is not None:
|
||||
success, message = self.draft_worker.update_weights_from_ipc(recv_req)
|
||||
if tp_success:
|
||||
success, message = True, "Succeeded to update model weights."
|
||||
target_updated = False
|
||||
for role, runner in self._select_runners():
|
||||
success, message = runner.weight_updater.update_weights_from_ipc(
|
||||
recv_req
|
||||
)
|
||||
if not success:
|
||||
break
|
||||
target_updated |= role == "target"
|
||||
# the served weights changed even if a draft runner failed afterwards
|
||||
if target_updated:
|
||||
self.flush_cache_after_weight_update(recv_req)
|
||||
if success:
|
||||
self.record_weight_version_after_update(recv_req.weight_version)
|
||||
@@ -209,6 +287,34 @@ class SchedulerWeightUpdaterManager:
|
||||
f"Restart with --weight-cache-mode off to use this operation."
|
||||
)
|
||||
|
||||
def begin_weight_update(self, recv_req: BeginWeightUpdateReqInput):
|
||||
"""Open the session: restore in-place-packed weights on the selected runners."""
|
||||
if self._session is not None:
|
||||
return BeginWeightUpdateReqOutput(
|
||||
success=False,
|
||||
message="a weight-update session is already open; "
|
||||
"call end_weight_update() first",
|
||||
)
|
||||
for _, runner in self._select_runners(recv_req.selector):
|
||||
runner.weight_updater.begin_weight_update()
|
||||
self._session = _WeightUpdateSession(selector=recv_req.selector)
|
||||
torch.distributed.barrier(group=self.tp_cpu_group)
|
||||
return BeginWeightUpdateReqOutput(success=True, message="Success")
|
||||
|
||||
def end_weight_update(self, recv_req: EndWeightUpdateReqInput):
|
||||
"""Finalize the runners begin opened; post_load_weights only if no load ran (P2P/RDMA)."""
|
||||
if self._session is None:
|
||||
return EndWeightUpdateReqOutput(
|
||||
success=False,
|
||||
message="no weight-update session is open; call begin_weight_update() first",
|
||||
)
|
||||
run_post_load = not self._session.loaded_weights
|
||||
for _, runner in self._select_runners(self._session.selector):
|
||||
runner.weight_updater.end_weight_update(run_post_load=run_post_load)
|
||||
self._session = None
|
||||
torch.distributed.barrier(group=self.tp_cpu_group)
|
||||
return EndWeightUpdateReqOutput(success=True, message="Success")
|
||||
|
||||
def release_memory_occupation(self, recv_req: ReleaseMemoryOccupationReqInput):
|
||||
scheduler = self.scheduler
|
||||
assert self.is_fully_idle(
|
||||
@@ -298,19 +404,17 @@ class SchedulerWeightUpdaterManager:
|
||||
|
||||
def check_weights(self, recv_req: CheckWeightsReqInput):
|
||||
try:
|
||||
payload = self.tp_worker.model_runner.check_weights(
|
||||
action=recv_req.action, allow_quant_error=recv_req.allow_quant_error
|
||||
)
|
||||
|
||||
if self.draft_worker is not None:
|
||||
draft_runner = _get_draft_model_runner(self.draft_worker)
|
||||
if draft_runner is not None:
|
||||
draft_payload = draft_runner.check_weights(
|
||||
action=recv_req.action,
|
||||
allow_quant_error=recv_req.allow_quant_error,
|
||||
)
|
||||
if payload is not None and draft_payload is not None:
|
||||
payload = _merge_checksum_payloads(payload, draft_payload)
|
||||
role_payloads = []
|
||||
for role, runner in self._select_runners(recv_req.selector):
|
||||
p = runner.check_weights(
|
||||
action=recv_req.action,
|
||||
allow_quant_error=recv_req.allow_quant_error,
|
||||
skip_tensor_list=recv_req.skip_tensor_list,
|
||||
role=role,
|
||||
)
|
||||
if p is not None:
|
||||
role_payloads.append((role, p))
|
||||
payload = _merge_checksum_payloads(role_payloads) if role_payloads else None
|
||||
|
||||
tp_size = torch.distributed.get_world_size(group=self.tp_cpu_group)
|
||||
if tp_size > 1 and payload is not None:
|
||||
|
||||
@@ -16,6 +16,8 @@ from sglang.srt.managers.io_struct import (
|
||||
AddExternalCorpusReqOutput,
|
||||
AttachHiCacheStorageReqInput,
|
||||
AttachHiCacheStorageReqOutput,
|
||||
BeginWeightUpdateReqInput,
|
||||
BeginWeightUpdateReqOutput,
|
||||
ChecksumInfo,
|
||||
CheckWeightsReqInput,
|
||||
CheckWeightsReqOutput,
|
||||
@@ -28,6 +30,8 @@ from sglang.srt.managers.io_struct import (
|
||||
DetachHiCacheStorageReqOutput,
|
||||
DumperControlReqInput,
|
||||
DumperControlReqOutput,
|
||||
EndWeightUpdateReqInput,
|
||||
EndWeightUpdateReqOutput,
|
||||
ExpertDistributionReq,
|
||||
ExpertDistributionReqOutput,
|
||||
ExpertDistributionReqType,
|
||||
@@ -131,6 +135,8 @@ _COMMUNICATOR_SPECS = [
|
||||
("get_internal_state", GetInternalStateReqOutput),
|
||||
("set_internal_state", SetInternalStateReqOutput),
|
||||
("expert_distribution", ExpertDistributionReqOutput),
|
||||
("begin_weight_update", BeginWeightUpdateReqOutput),
|
||||
("end_weight_update", EndWeightUpdateReqOutput),
|
||||
("update_lora_adapter", LoRAUpdateOutput),
|
||||
("dumper_control", DumperControlReqOutput),
|
||||
("scale_elastic_ep", ScaleElasticEPReqOutput),
|
||||
@@ -449,6 +455,38 @@ class TokenizerControlMixin:
|
||||
results = await self.destroy_weights_update_group_communicator(obj)
|
||||
return FanOutCommunicator.merge_results(results)
|
||||
|
||||
async def _weight_update_session_call(
|
||||
self: TokenizerManager, communicator, obj
|
||||
) -> Tuple[bool, str]:
|
||||
self.auto_create_handle_loop()
|
||||
async with self.is_pause_cond:
|
||||
is_paused = self.is_pause
|
||||
# whoever paused the engine holds the writer lock; taking it again deadlocks
|
||||
if is_paused:
|
||||
results = await communicator(obj)
|
||||
if not is_paused:
|
||||
async with self.model_update_lock.writer_lock:
|
||||
results = await communicator(obj)
|
||||
return FanOutCommunicator.merge_results(results)
|
||||
|
||||
async def begin_weight_update(
|
||||
self: TokenizerManager,
|
||||
obj: BeginWeightUpdateReqInput,
|
||||
request: Optional[fastapi.Request] = None,
|
||||
) -> Tuple[bool, str]:
|
||||
return await self._weight_update_session_call(
|
||||
self.begin_weight_update_communicator, obj
|
||||
)
|
||||
|
||||
async def end_weight_update(
|
||||
self: TokenizerManager,
|
||||
obj: EndWeightUpdateReqInput,
|
||||
request: Optional[fastapi.Request] = None,
|
||||
) -> Tuple[bool, str]:
|
||||
return await self._weight_update_session_call(
|
||||
self.end_weight_update_communicator, obj
|
||||
)
|
||||
|
||||
async def update_weights_from_distributed(
|
||||
self: TokenizerManager,
|
||||
obj: UpdateWeightsFromDistributedReqInput,
|
||||
|
||||
@@ -32,10 +32,6 @@ from sglang.srt.managers.io_struct import (
|
||||
LoadLoRAAdapterReqInput,
|
||||
SendWeightsToRemoteInstanceReqInput,
|
||||
UnloadLoRAAdapterReqInput,
|
||||
UpdateWeightFromDiskReqInput,
|
||||
UpdateWeightsFromDistributedReqInput,
|
||||
UpdateWeightsFromIPCReqInput,
|
||||
UpdateWeightsFromTensorReqInput,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import ScheduleBatch
|
||||
from sglang.srt.managers.scheduler import GenerationBatchResult
|
||||
@@ -146,14 +142,6 @@ class BaseTpWorker(ABC):
|
||||
self.model_runner.token_to_kv_pool_allocator,
|
||||
)
|
||||
|
||||
def update_weights_from_disk(self, recv_req: UpdateWeightFromDiskReqInput):
|
||||
success, message = self.model_runner.weight_updater.update_weights_from_disk(
|
||||
recv_req.model_path,
|
||||
recv_req.load_format,
|
||||
recapture_cuda_graph=recv_req.recapture_cuda_graph,
|
||||
)
|
||||
return success, message
|
||||
|
||||
def init_weights_update_group(self, recv_req: InitWeightsUpdateGroupReqInput):
|
||||
success, message = self.model_runner.weight_updater.init_weights_update_group(
|
||||
recv_req.master_address,
|
||||
@@ -200,21 +188,7 @@ class BaseTpWorker(ABC):
|
||||
)
|
||||
return success, message
|
||||
|
||||
def update_weights_from_distributed(
|
||||
self, recv_req: UpdateWeightsFromDistributedReqInput
|
||||
):
|
||||
success, message = (
|
||||
self.model_runner.weight_updater.update_weights_from_distributed(
|
||||
recv_req.names,
|
||||
recv_req.dtypes,
|
||||
recv_req.shapes,
|
||||
recv_req.group_name,
|
||||
recv_req.load_format,
|
||||
)
|
||||
)
|
||||
return success, message
|
||||
|
||||
def _deserialize_own_rank(self, serialized_named_tensors):
|
||||
def deserialize_own_rank(self, serialized_named_tensors):
|
||||
"""Each rank deserializes only its own payload (index tp_rank);
|
||||
deserializing another rank's copy would break producer-side CUDA-IPC
|
||||
refcounting."""
|
||||
@@ -223,20 +197,6 @@ class BaseTpWorker(ABC):
|
||||
serialized_named_tensors[self.model_runner.tp_rank]
|
||||
)
|
||||
|
||||
def update_weights_from_tensor(self, recv_req: UpdateWeightsFromTensorReqInput):
|
||||
success, message = self.model_runner.weight_updater.update_weights_from_tensor(
|
||||
named_tensors=self._deserialize_own_rank(recv_req.serialized_named_tensors),
|
||||
load_format=recv_req.load_format,
|
||||
)
|
||||
return success, message
|
||||
|
||||
def update_weights_from_ipc(self, recv_req: UpdateWeightsFromIPCReqInput):
|
||||
"""Update weights from IPC for checkpoint-engine integration."""
|
||||
success, message = self.model_runner.weight_updater.update_weights_from_ipc(
|
||||
recv_req
|
||||
)
|
||||
return success, message
|
||||
|
||||
def get_weights_by_name(self, recv_req: GetWeightsByNameReqInput):
|
||||
parameter = self.model_runner.weight_exporter.get_weights_by_name(
|
||||
recv_req.name, recv_req.truncate_size
|
||||
@@ -256,7 +216,7 @@ class BaseTpWorker(ABC):
|
||||
):
|
||||
# The LoRA code handles TP sharding internally using slice_lora_a_weights
|
||||
# and slice_lora_b_weights methods (see lora/layers.py and mem_pool.py).
|
||||
data = self._deserialize_own_rank(recv_req.serialized_named_tensors)
|
||||
data = self.deserialize_own_rank(recv_req.serialized_named_tensors)
|
||||
if recv_req.load_format == "flattened_bucket":
|
||||
bucket = FlattenedTensorBucket(
|
||||
flattened_tensor=data["flattened_tensor"],
|
||||
@@ -562,6 +522,10 @@ class TpModelWorker(BaseTpWorker):
|
||||
def model_runner(self) -> ModelRunner:
|
||||
return self._model_runner
|
||||
|
||||
def weight_update_runners(self) -> List[Tuple[str, ModelRunner]]:
|
||||
"""(role, runner) pairs weight ops apply to; the target worker owns one."""
|
||||
return [("target", self._model_runner)]
|
||||
|
||||
def register_hicache_layer_transfer_counter(self, counter: LayerDoneCounter):
|
||||
self.hicache_layer_transfer_counter = counter
|
||||
|
||||
|
||||
@@ -20,7 +20,7 @@ import inspect
|
||||
import logging
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Callable, Optional, Union
|
||||
from typing import Callable, List, Optional, Union
|
||||
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
@@ -2048,9 +2048,19 @@ class ModelRunner:
|
||||
forward_batch.token_ids_logprobs,
|
||||
)
|
||||
|
||||
def check_weights(self, action: str, allow_quant_error: bool = False):
|
||||
def check_weights(
|
||||
self,
|
||||
action: str,
|
||||
allow_quant_error: bool = False,
|
||||
skip_tensor_list: Optional[List[str]] = None,
|
||||
*,
|
||||
role: str,
|
||||
):
|
||||
return self._weight_checker.handle(
|
||||
action=action, allow_quant_error=allow_quant_error
|
||||
action=action,
|
||||
allow_quant_error=allow_quant_error,
|
||||
skip_tensor_list=skip_tensor_list,
|
||||
role=role,
|
||||
)
|
||||
|
||||
def _expand_eplb_metadata_for_scale(
|
||||
|
||||
@@ -8,7 +8,11 @@ from typing import TYPE_CHECKING, Any, Callable, List, Optional, Tuple, Union
|
||||
import torch
|
||||
|
||||
from sglang.srt.configs.load_config import LoadConfig
|
||||
from sglang.srt.model_loader.loader import DefaultModelLoader, get_model_loader
|
||||
from sglang.srt.model_loader.loader import (
|
||||
DefaultModelLoader,
|
||||
get_model_loader,
|
||||
post_load_weights,
|
||||
)
|
||||
from sglang.srt.model_loader.utils import set_default_torch_dtype
|
||||
from sglang.srt.model_loader.weight_utils import default_weight_loader
|
||||
from sglang.srt.platforms import current_platform
|
||||
@@ -241,94 +245,87 @@ class WeightUpdater:
|
||||
logger.info("Update weights end.")
|
||||
return True, "Succeeded to update model weights."
|
||||
|
||||
def update_weights_from_distributed(
|
||||
def receive_weights_from_distributed(
|
||||
self: WeightUpdater,
|
||||
*,
|
||||
names,
|
||||
dtypes,
|
||||
shapes,
|
||||
group_name,
|
||||
load_format: Optional[str] = None,
|
||||
):
|
||||
"""
|
||||
Update specific parameter in the model weights online
|
||||
through `_model_update_group` process group.
|
||||
|
||||
Args:
|
||||
name: the name of the parameter to be updated.
|
||||
dtype: the data type of the parameter to be updated.
|
||||
shape: the shape of the parameter to be updated.
|
||||
"""
|
||||
self._assert_weight_cache_inactive("update_weights_from_distributed")
|
||||
error = _unsupported_derived_weight_cache_error(self.get_model())
|
||||
if error is not None:
|
||||
return False, error
|
||||
|
||||
"""Receive one broadcast without loading it; only the target runner joined the group."""
|
||||
assert group_name in self._model_update_group, (
|
||||
f"Group {group_name} not in {list(self._model_update_group.keys())}. "
|
||||
"Please call `init_weights_update_group` first."
|
||||
)
|
||||
|
||||
if load_format == "flattened_bucket":
|
||||
return self._update_bucketed_weights_from_distributed(
|
||||
names, dtypes, shapes, group_name
|
||||
return self._receive_bucketed_weights_from_distributed(
|
||||
names=names, dtypes=dtypes, shapes=shapes, group_name=group_name
|
||||
)
|
||||
try:
|
||||
weights = []
|
||||
handles = []
|
||||
for name, dtype, shape in zip(names, dtypes, shapes):
|
||||
target_dtype = (
|
||||
dtype if isinstance(dtype, torch.dtype) else getattr(torch, dtype)
|
||||
)
|
||||
weight = torch.empty(shape, dtype=target_dtype, device=self.device)
|
||||
handles.append(
|
||||
torch.distributed.broadcast(
|
||||
weight,
|
||||
src=0,
|
||||
group=self._model_update_group[group_name],
|
||||
async_op=True,
|
||||
)
|
||||
)
|
||||
weights.append((name, weight))
|
||||
for handle in handles:
|
||||
handle.wait()
|
||||
|
||||
self.get_model().load_weights(weights)
|
||||
return True, "Succeeded to update parameter online."
|
||||
|
||||
except Exception as e:
|
||||
error_msg = (
|
||||
f"Failed to update parameter online: {e}. "
|
||||
f"The full weights of the ModelRunner are partially updated. "
|
||||
f"Please discard the whole weights."
|
||||
weights = []
|
||||
handles = []
|
||||
for name, dtype, shape in zip(names, dtypes, shapes):
|
||||
target_dtype = (
|
||||
dtype if isinstance(dtype, torch.dtype) else getattr(torch, dtype)
|
||||
)
|
||||
logger.error(error_msg)
|
||||
return False, error_msg
|
||||
weight = torch.empty(shape, dtype=target_dtype, device=self.device)
|
||||
handles.append(
|
||||
torch.distributed.broadcast(
|
||||
weight,
|
||||
src=0,
|
||||
group=self._model_update_group[group_name],
|
||||
async_op=True,
|
||||
)
|
||||
)
|
||||
weights.append((name, weight))
|
||||
for handle in handles:
|
||||
handle.wait()
|
||||
return weights
|
||||
|
||||
def _update_bucketed_weights_from_distributed(
|
||||
self: WeightUpdater, names, dtypes, shapes, group_name
|
||||
def _receive_bucketed_weights_from_distributed(
|
||||
self: WeightUpdater, *, names, dtypes, shapes, group_name
|
||||
):
|
||||
try:
|
||||
named_tensors = []
|
||||
for name, dtype, shape in zip(names, dtypes, shapes):
|
||||
target_dtype = (
|
||||
dtype if isinstance(dtype, torch.dtype) else getattr(torch, dtype)
|
||||
)
|
||||
named_tensors.append(
|
||||
(
|
||||
name,
|
||||
torch.empty(shape, dtype=target_dtype, device=self.device),
|
||||
)
|
||||
)
|
||||
bucket = FlattenedTensorBucket(named_tensors=named_tensors)
|
||||
flattened_tensor = bucket.get_flattened_tensor()
|
||||
torch.distributed.broadcast(
|
||||
flattened_tensor,
|
||||
src=0,
|
||||
group=self._model_update_group[group_name],
|
||||
named_tensors = []
|
||||
for name, dtype, shape in zip(names, dtypes, shapes):
|
||||
target_dtype = (
|
||||
dtype if isinstance(dtype, torch.dtype) else getattr(torch, dtype)
|
||||
)
|
||||
reconstructed_tensors = bucket.reconstruct_tensors()
|
||||
self.get_model().load_weights(reconstructed_tensors)
|
||||
return True, "Succeeded to update parameter online."
|
||||
named_tensors.append(
|
||||
(name, torch.empty(shape, dtype=target_dtype, device=self.device))
|
||||
)
|
||||
bucket = FlattenedTensorBucket(named_tensors=named_tensors)
|
||||
flattened_tensor = bucket.get_flattened_tensor()
|
||||
torch.distributed.broadcast(
|
||||
flattened_tensor,
|
||||
src=0,
|
||||
group=self._model_update_group[group_name],
|
||||
)
|
||||
return bucket.reconstruct_tensors()
|
||||
|
||||
def begin_weight_update(self: WeightUpdater) -> None:
|
||||
DefaultModelLoader.restore_weights_before_loading(
|
||||
self.get_model(), torch.device(self.device)
|
||||
)
|
||||
|
||||
def end_weight_update(self: WeightUpdater, *, run_post_load: bool) -> None:
|
||||
if run_post_load:
|
||||
post_load_weights(self.get_model())
|
||||
DefaultModelLoader.postprocess_weights(
|
||||
self.get_model(), torch.device(self.device)
|
||||
)
|
||||
|
||||
def load_weights_from_distributed(
|
||||
self: WeightUpdater, named_tensors: List[Tuple[str, torch.Tensor]]
|
||||
) -> Tuple[bool, str]:
|
||||
self._assert_weight_cache_inactive("update_weights_from_distributed")
|
||||
error = _unsupported_derived_weight_cache_error(self.get_model())
|
||||
if error is not None:
|
||||
return False, error
|
||||
try:
|
||||
self.get_model().load_weights(named_tensors)
|
||||
except Exception as e:
|
||||
error_msg = (
|
||||
f"Failed to update parameter online: {e}. "
|
||||
@@ -337,6 +334,7 @@ class WeightUpdater:
|
||||
)
|
||||
logger.error(error_msg)
|
||||
return False, error_msg
|
||||
return True, "Succeeded to update parameter online."
|
||||
|
||||
def update_weights_from_tensor(
|
||||
self: WeightUpdater,
|
||||
|
||||
@@ -87,7 +87,10 @@ from sglang.srt.layers.modelopt_utils import QUANT_CFG_CHOICES
|
||||
from sglang.srt.layers.moe.utils import (
|
||||
install_shared_experts_fusion_decision,
|
||||
)
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.quantization.base_config import (
|
||||
QuantizationConfig,
|
||||
QuantizeMethodBase,
|
||||
)
|
||||
from sglang.srt.model_loader.remote_instance_weight_loader_utils import (
|
||||
trigger_transferring_weights_request,
|
||||
)
|
||||
@@ -304,7 +307,7 @@ def _initialize_model(
|
||||
return model_class(**kwargs)
|
||||
|
||||
|
||||
def _post_load_weights(model: nn.Module) -> None:
|
||||
def post_load_weights(model: nn.Module) -> None:
|
||||
# Loaders that bypass `model.load_weights()` (dummy / sharded state / remote instance /
|
||||
# remote fs) must trigger the model's post-load fixup explicitly; `model.load_weights()`
|
||||
# would normally do it internally. NextN subclasses override the method to fill in
|
||||
@@ -313,6 +316,18 @@ def _post_load_weights(model: nn.Module) -> None:
|
||||
model.post_load_weights()
|
||||
|
||||
|
||||
def _modules_with_quant_method(model: nn.Module):
|
||||
from sglang.srt.lora.layers import BaseLayerWithLoRA
|
||||
|
||||
for _, module in model.named_modules():
|
||||
# LoRA wrappers forward quant_method but do not own the packed params
|
||||
if isinstance(module, BaseLayerWithLoRA):
|
||||
continue
|
||||
quant_method = getattr(module, "quant_method", None)
|
||||
if quant_method is not None:
|
||||
yield module, quant_method
|
||||
|
||||
|
||||
class BaseModelLoader(ABC):
|
||||
"""Base class for model loaders."""
|
||||
|
||||
@@ -836,7 +851,7 @@ class DefaultModelLoader(BaseModelLoader):
|
||||
"""
|
||||
with set_default_torch_dtype(model_config.dtype):
|
||||
initialize_capture_safe_weights(model)
|
||||
_post_load_weights(model)
|
||||
post_load_weights(model)
|
||||
for _, module in model.named_modules():
|
||||
quant_method = getattr(module, "quant_method", None)
|
||||
if quant_method is None:
|
||||
@@ -1056,16 +1071,23 @@ class DefaultModelLoader(BaseModelLoader):
|
||||
|
||||
@staticmethod
|
||||
def postprocess_weights(model, target_device):
|
||||
for _, module in model.named_modules():
|
||||
quant_method = getattr(module, "quant_method", None)
|
||||
if quant_method is not None:
|
||||
# When quant methods need to process weights after loading
|
||||
# (for repacking, quantizing, etc), they expect parameters
|
||||
# to be on the global target device. This scope is for the
|
||||
# case where cpu offloading is used, where we will move the
|
||||
# parameters onto device for processing and back off after.
|
||||
for module, quant_method in _modules_with_quant_method(model):
|
||||
# When quant methods need to process weights after loading
|
||||
# (for repacking, quantizing, etc), they expect parameters
|
||||
# to be on the global target device. This scope is for the
|
||||
# case where cpu offloading is used, where we will move the
|
||||
# parameters onto device for processing and back off after.
|
||||
with device_loading_context(module, target_device):
|
||||
quant_method.process_weights_after_loading(module)
|
||||
|
||||
@staticmethod
|
||||
def restore_weights_before_loading(model, target_device):
|
||||
"""Undo in-place quant packing so fresh weights can be loaded."""
|
||||
for module, quant_method in _modules_with_quant_method(model):
|
||||
# AMX packing and the MXFP4 backend wrappers are duck-typed and cannot restore
|
||||
if isinstance(quant_method, QuantizeMethodBase):
|
||||
with device_loading_context(module, target_device):
|
||||
quant_method.process_weights_after_loading(module)
|
||||
quant_method.restore_weights_before_loading(module)
|
||||
|
||||
|
||||
class LayeredModelLoader(DefaultModelLoader):
|
||||
@@ -1643,7 +1665,7 @@ class DummyModelLoader(BaseModelLoader):
|
||||
# random values to the weights.
|
||||
initialize_dummy_weights(model)
|
||||
|
||||
_post_load_weights(model)
|
||||
post_load_weights(model)
|
||||
|
||||
for _, module in model.named_modules():
|
||||
quant_method = getattr(module, "quant_method", None)
|
||||
@@ -1804,7 +1826,7 @@ class ShardedStateLoader(BaseModelLoader):
|
||||
if state_dict:
|
||||
raise ValueError(f"Missing keys {tuple(state_dict)} in loaded state!")
|
||||
|
||||
_post_load_weights(model)
|
||||
post_load_weights(model)
|
||||
|
||||
return model.eval()
|
||||
|
||||
@@ -3426,7 +3448,7 @@ class RemoteInstanceModelLoader(BaseModelLoader):
|
||||
)
|
||||
current_platform.synchronize()
|
||||
|
||||
_post_load_weights(model)
|
||||
post_load_weights(model)
|
||||
end_get_weights_tic = time.time()
|
||||
logger.debug(
|
||||
f"finish getting all weights from remote instance, time used: {(end_get_weights_tic - start_get_weights_tic):.4f}s"
|
||||
@@ -3489,7 +3511,7 @@ class RemoteInstanceModelLoader(BaseModelLoader):
|
||||
logger.error(f"batch transfer failed, error: {ret}")
|
||||
return False
|
||||
|
||||
_post_load_weights(model)
|
||||
post_load_weights(model)
|
||||
|
||||
return True
|
||||
|
||||
@@ -3579,7 +3601,7 @@ class RemoteModelLoader(BaseModelLoader):
|
||||
if state_dict:
|
||||
raise ValueError(f"Missing keys {tuple(state_dict)} in loaded state!")
|
||||
|
||||
_post_load_weights(model)
|
||||
post_load_weights(model)
|
||||
|
||||
def _load_model_from_remote_fs(
|
||||
self, model, client, model_config: ModelConfig, device_config: DeviceConfig
|
||||
|
||||
@@ -14,10 +14,6 @@ from sglang.srt.model_executor.graph_memory_usage import (
|
||||
from sglang.srt.runtime_context import get_disagg, get_exec, get_memory, get_schedule
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.managers.io_struct import (
|
||||
UpdateWeightFromDiskReqInput,
|
||||
UpdateWeightsFromIPCReqInput,
|
||||
)
|
||||
from sglang.srt.managers.tp_worker import TpModelWorker
|
||||
from sglang.srt.model_executor.model_runner import (
|
||||
ModelRunner,
|
||||
@@ -162,6 +158,13 @@ class BaseSpecWorker(ABC):
|
||||
def hicache_draft_plan(self) -> HiCacheDraftPlan:
|
||||
return self._hicache_draft_plan
|
||||
|
||||
def weight_update_runners(self) -> list[tuple[str, ModelRunner]]:
|
||||
"""(role, runner) pairs weight ops apply to; [] when the draft owns no weights."""
|
||||
# NGRAM and UNO have no draft model
|
||||
if self.draft_worker is None:
|
||||
return []
|
||||
return [("draft", runner) for runner in self.draft_worker.draft_runners]
|
||||
|
||||
def _draft_model_runners(self) -> tuple[ModelRunner, ...]:
|
||||
spec_algorithm = self.target_worker.model_runner.spec_algorithm
|
||||
draft_worker = self.draft_worker
|
||||
@@ -315,24 +318,6 @@ class BaseSpecWorker(ABC):
|
||||
if self.draft_worker is not None:
|
||||
self.draft_worker.init_cuda_graphs()
|
||||
|
||||
def update_weights_from_disk(self, recv_req: UpdateWeightFromDiskReqInput):
|
||||
for runner in self.draft_worker.draft_runners:
|
||||
success, message = runner.weight_updater.update_weights_from_disk(
|
||||
recv_req.model_path,
|
||||
recv_req.load_format,
|
||||
recapture_cuda_graph=recv_req.recapture_cuda_graph,
|
||||
)
|
||||
if not success:
|
||||
return success, message
|
||||
return True, "Succeeded to update model weights."
|
||||
|
||||
def update_weights_from_ipc(self, recv_req: UpdateWeightsFromIPCReqInput):
|
||||
for runner in self.draft_worker.draft_runners:
|
||||
success, message = runner.weight_updater.update_weights_from_ipc(recv_req)
|
||||
if not success:
|
||||
return success, message
|
||||
return True, "Succeeded to update model weights."
|
||||
|
||||
def on_verify_complete_cpu(
|
||||
self, num_correct_drafts_per_req: list[int], batch_size: int = 0
|
||||
) -> None:
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
from typing import List, Optional, Tuple
|
||||
from typing import TYPE_CHECKING, List, Optional, Tuple
|
||||
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
@@ -92,6 +92,9 @@ from sglang.srt.utils.common import empty_context
|
||||
_is_npu = is_npu()
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.model_executor.model_runner import ModelRunner
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_FusedKVMaterializeHelper = None
|
||||
@@ -358,6 +361,9 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
scheduler runs it synchronously when overlap is disabled.
|
||||
"""
|
||||
|
||||
def weight_update_runners(self) -> List[Tuple[str, "ModelRunner"]]:
|
||||
return [("draft", self.draft_model_runner)]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
server_args: ServerArgs,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import logging
|
||||
from contextlib import nullcontext
|
||||
from typing import Callable, Optional, Protocol, runtime_checkable
|
||||
from typing import TYPE_CHECKING, Callable, Optional, Protocol, runtime_checkable
|
||||
|
||||
import torch
|
||||
|
||||
@@ -84,6 +84,9 @@ from sglang.srt.utils import (
|
||||
is_pin_memory_available,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.model_executor.model_runner import ModelRunner
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_is_npu = is_npu()
|
||||
@@ -127,6 +130,9 @@ def _configure_target_hidden_projection(
|
||||
class DSparkWorkerV2(BaseSpecWorker):
|
||||
"""Non-last PP stages run only the target; draft state belongs to the last stage."""
|
||||
|
||||
def weight_update_runners(self) -> list[tuple[str, "ModelRunner"]]:
|
||||
return [("draft", self.draft_model_runner)] if self._hosts_draft else []
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
server_args: ServerArgs,
|
||||
|
||||
@@ -38,7 +38,6 @@ from sglang.srt.layers.moe.utils import (
|
||||
speculative_moe_a2a_backend_context,
|
||||
speculative_moe_backend_context,
|
||||
)
|
||||
from sglang.srt.managers.io_struct import UpdateWeightsFromTensorReqInput
|
||||
from sglang.srt.managers.schedule_batch import ScheduleBatch
|
||||
from sglang.srt.managers.scheduler import GenerationBatchResult
|
||||
from sglang.srt.managers.tp_worker import TpModelWorker
|
||||
@@ -117,7 +116,6 @@ from sglang.srt.utils.async_probe import (
|
||||
maybe_detect_oob,
|
||||
)
|
||||
from sglang.srt.utils.common import (
|
||||
MultiprocessingSerializer,
|
||||
empty_context,
|
||||
fast_topk,
|
||||
get_available_gpu_memory,
|
||||
@@ -129,7 +127,6 @@ from sglang.srt.utils.common import (
|
||||
is_xpu,
|
||||
log_info_on_rank0,
|
||||
)
|
||||
from sglang.srt.utils.patch_torch import monkey_patch_torch_reductions
|
||||
|
||||
_is_cpu = is_cpu()
|
||||
_is_npu = is_npu()
|
||||
@@ -1868,25 +1865,3 @@ class EAGLEWorkerV2(BaseSpecWorker):
|
||||
finalize_tree_path=True,
|
||||
grammar_barrier=grammar_barrier,
|
||||
)
|
||||
|
||||
def update_weights_from_tensor(self, recv_req: UpdateWeightsFromTensorReqInput):
|
||||
monkey_patch_torch_reductions()
|
||||
named_tensors = MultiprocessingSerializer.deserialize(
|
||||
recv_req.serialized_named_tensors[self.model_runner.tp_rank]
|
||||
)
|
||||
success, message = (
|
||||
self.draft_worker.draft_runner.weight_updater.update_weights_from_tensor(
|
||||
named_tensors=named_tensors,
|
||||
load_format=recv_req.load_format,
|
||||
)
|
||||
)
|
||||
if not success:
|
||||
return success, message
|
||||
|
||||
success, message = (
|
||||
self.target_worker.model_runner.weight_updater.update_weights_from_tensor(
|
||||
named_tensors=named_tensors,
|
||||
load_format=recv_req.load_format,
|
||||
)
|
||||
)
|
||||
return success, message
|
||||
|
||||
@@ -17,7 +17,7 @@ from __future__ import annotations
|
||||
import logging
|
||||
import time
|
||||
from dataclasses import replace
|
||||
from typing import TYPE_CHECKING, List
|
||||
from typing import TYPE_CHECKING, List, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
@@ -1001,6 +1001,12 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase):
|
||||
|
||||
|
||||
class MultiLayerEagleWorkerV2(BaseSpecWorker):
|
||||
def weight_update_runners(self) -> List[Tuple[str, ModelRunner]]:
|
||||
return [
|
||||
(f"draft_step_{i}", r)
|
||||
for i, r in enumerate(self.draft_worker.draft_runners)
|
||||
]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
server_args: ServerArgs,
|
||||
|
||||
@@ -158,15 +158,6 @@ class NGRAMWorker(BaseSpecWorker):
|
||||
self.ngram_corpus.reset()
|
||||
self._prev_decode_rids = set()
|
||||
|
||||
def update_weights_from_tensor(self, recv_req):
|
||||
# NGRAM has no draft weights of its own — the n-gram corpus is a CPU
|
||||
# lookup structure built from request token streams — and its
|
||||
# `model_runner` is shared with the target worker. The scheduler
|
||||
# mixin dispatches via `self.draft_worker or self.tp_worker`, so
|
||||
# without this method any caller of `update_weights_from_tensor`
|
||||
# under `--speculative-algorithm NGRAM` raises AttributeError.
|
||||
return self.target_worker.update_weights_from_tensor(recv_req)
|
||||
|
||||
def add_external_corpus(self, corpus_id: str, token_chunks: list[list[int]]) -> int:
|
||||
return self.ngram_corpus.load_external_corpus_named(corpus_id, token_chunks)
|
||||
|
||||
|
||||
@@ -786,18 +786,6 @@ class UnoWorkerV2(BaseSpecWorker):
|
||||
f"UNO expected an EXTEND or DECODE batch, got {batch.forward_mode}."
|
||||
)
|
||||
|
||||
def update_weights_from_disk(self, recv_req):
|
||||
# The scheduler updates the target worker before calling the spec worker.
|
||||
return True, "UNO has no separate draft weights."
|
||||
|
||||
def update_weights_from_ipc(self, recv_req):
|
||||
# The scheduler updates the target worker before calling the spec worker.
|
||||
return True, "UNO has no separate draft weights."
|
||||
|
||||
def update_weights_from_tensor(self, recv_req):
|
||||
# This update route selects the spec worker instead of updating both.
|
||||
return self.target_worker.update_weights_from_tensor(recv_req)
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _bind_uno_draft_runtime(self):
|
||||
target_attn_backend = self.model_runner.attn_backend
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import hashlib
|
||||
import logging
|
||||
import time
|
||||
from typing import Any, Callable, Dict, Iterable, NamedTuple, Optional, Set
|
||||
from typing import Any, Callable, Dict, Iterable, List, NamedTuple, Optional, Set
|
||||
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
@@ -26,6 +26,8 @@ class _StrictBaseModel(BaseModel):
|
||||
|
||||
|
||||
class ParallelismInfo(_StrictBaseModel):
|
||||
# "target", or a draft role such as "draft" / "draft_step_0"
|
||||
role: str
|
||||
tp_rank: int
|
||||
tp_size: int
|
||||
dp_rank: int
|
||||
@@ -68,6 +70,23 @@ def _is_non_persistent_buffer_name(name: str) -> bool:
|
||||
return any(pat in name for pat in _NON_PERSISTENT_BUFFER_PATTERNS)
|
||||
|
||||
|
||||
def _is_skip_weight_check(name, param, skip_tensor_list=None) -> bool:
|
||||
# one skip set shared by reset / compare / checksum
|
||||
return (
|
||||
_is_non_persistent_buffer_name(name)
|
||||
or getattr(param, "_skip_weight_check", False)
|
||||
or any(pat in name for pat in (skip_tensor_list or ()))
|
||||
)
|
||||
|
||||
|
||||
def overall_checksum(checksums: Dict[str, str]) -> str:
|
||||
h = hashlib.sha256()
|
||||
for name in sorted(checksums):
|
||||
h.update(name.encode())
|
||||
h.update(checksums[name].encode())
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def _padded(nbytes: int, align: int) -> int:
|
||||
return (nbytes + align - 1) // align * align
|
||||
|
||||
@@ -92,31 +111,39 @@ class WeightChecker:
|
||||
self._get_model = get_model
|
||||
# Capture the runner placement before its draft scope exits.
|
||||
parallel = get_parallel()
|
||||
self._placement = ParallelismInfo(
|
||||
self._placement = dict(
|
||||
tp_rank=parallel.tp_rank,
|
||||
tp_size=parallel.tp_size,
|
||||
dp_rank=parallel.dp_rank if parallel.dp_rank is not None else 0,
|
||||
dp_size=parallel.attn_dp_size,
|
||||
pp_rank=parallel.pp_rank,
|
||||
pp_size=parallel.pp_size,
|
||||
rank=0,
|
||||
size=1,
|
||||
)
|
||||
self._snapshot_tensors = None
|
||||
self._snapshot_arena = None
|
||||
|
||||
def handle(self, action: str, allow_quant_error: bool = False) -> Optional[Dict]:
|
||||
def handle(
|
||||
self,
|
||||
action: str,
|
||||
allow_quant_error: bool = False,
|
||||
skip_tensor_list: Optional[List[str]] = None,
|
||||
*,
|
||||
role: str,
|
||||
) -> Optional[Dict]:
|
||||
logger.info(
|
||||
f"[WeightChecker] handle action={action} allow_quant_error={allow_quant_error}"
|
||||
f"[WeightChecker] handle action={action} "
|
||||
f"allow_quant_error={allow_quant_error} skip_tensor_list={skip_tensor_list}"
|
||||
)
|
||||
if action == "snapshot":
|
||||
return self._snapshot()
|
||||
elif action == "reset_tensors":
|
||||
return self._reset_tensors()
|
||||
return self._reset_tensors(skip_tensor_list)
|
||||
elif action == "compare":
|
||||
return self._compare(allow_quant_error=allow_quant_error)
|
||||
return self._compare(
|
||||
allow_quant_error=allow_quant_error, skip_tensor_list=skip_tensor_list
|
||||
)
|
||||
elif action == "checksum":
|
||||
return self._compute_checksum()
|
||||
return self._compute_checksum(skip_tensor_list, role=role)
|
||||
else:
|
||||
raise Exception(f"Unsupported {action=}")
|
||||
|
||||
@@ -138,21 +165,31 @@ class WeightChecker:
|
||||
self._snapshot_arena = allocator.arena
|
||||
self._snapshot_tensors = snapshot_tensors
|
||||
|
||||
def _reset_tensors(self):
|
||||
def _skip_compare_names(
|
||||
self, skip_tensor_list: Optional[List[str]] = None
|
||||
) -> Set[str]:
|
||||
return {
|
||||
name
|
||||
for name, param in self._model_state()
|
||||
if _is_skip_weight_check(name, param, skip_tensor_list)
|
||||
}
|
||||
|
||||
def _reset_tensors(self, skip_tensor_list: Optional[List[str]] = None):
|
||||
for name, param in self._model_state():
|
||||
if _is_non_persistent_buffer_name(name):
|
||||
# reset must skip exactly what compare skips
|
||||
if _is_skip_weight_check(name, param, skip_tensor_list):
|
||||
continue
|
||||
param.copy_(_random_like(param))
|
||||
|
||||
def _compare(self, allow_quant_error: bool = False):
|
||||
def _compare(
|
||||
self,
|
||||
allow_quant_error: bool = False,
|
||||
skip_tensor_list: Optional[List[str]] = None,
|
||||
):
|
||||
assert self._snapshot_tensors is not None
|
||||
|
||||
quantized_set = _build_quantized_set(self._get_model())
|
||||
skip_compare_names = {
|
||||
name
|
||||
for name, param in self._model_state()
|
||||
if getattr(param, "_skip_weight_check", False)
|
||||
}
|
||||
skip_compare_names = self._skip_compare_names(skip_tensor_list)
|
||||
_check_tensors(
|
||||
expect_tensors=_build_check_entries(
|
||||
self._snapshot_tensors, skip_compare_names, quantized_set
|
||||
@@ -165,16 +202,14 @@ class WeightChecker:
|
||||
self._snapshot_tensors = None
|
||||
self._snapshot_arena = None
|
||||
|
||||
def _compute_checksum(self) -> Dict:
|
||||
def _compute_checksum(
|
||||
self, skip_tensor_list: Optional[List[str]] = None, *, role: str
|
||||
) -> Dict:
|
||||
torch.cuda.synchronize()
|
||||
start = time.perf_counter()
|
||||
|
||||
quantized_set = _build_quantized_set(self._get_model())
|
||||
skip_compare_names = {
|
||||
name
|
||||
for name, param in self._model_state()
|
||||
if getattr(param, "_skip_weight_check", False)
|
||||
}
|
||||
skip_compare_names = self._skip_compare_names(skip_tensor_list)
|
||||
|
||||
# Hash the dequantized weight so two (qweight, scale) pairs with the same
|
||||
# bf16 hash equal.
|
||||
@@ -185,11 +220,7 @@ class WeightChecker:
|
||||
if should_compare:
|
||||
checksums[name] = _hash_tensor(comparable.dequantize().data)
|
||||
|
||||
h = hashlib.sha256()
|
||||
for name in sorted(checksums):
|
||||
h.update(name.encode())
|
||||
h.update(checksums[name].encode())
|
||||
overall = h.hexdigest()
|
||||
overall = overall_checksum(checksums)
|
||||
|
||||
torch.cuda.synchronize()
|
||||
elapsed = time.perf_counter() - start
|
||||
@@ -200,17 +231,17 @@ class WeightChecker:
|
||||
info = ChecksumInfo(
|
||||
checksums=checksums,
|
||||
per_gpu_checksum=overall,
|
||||
parallelism_info=self._parallelism_info(),
|
||||
parallelism_info=self._parallelism_info(role=role),
|
||||
)
|
||||
return info.model_dump()
|
||||
|
||||
def _parallelism_info(self) -> ParallelismInfo:
|
||||
def _parallelism_info(self, *, role: str) -> ParallelismInfo:
|
||||
# Read the current WORLD rank because elastic scale-up can change it.
|
||||
return self._placement.model_copy(
|
||||
update={
|
||||
"rank": dist.get_rank() if dist.is_initialized() else 0,
|
||||
"size": dist.get_world_size() if dist.is_initialized() else 1,
|
||||
}
|
||||
return ParallelismInfo(
|
||||
role=role,
|
||||
rank=dist.get_rank() if dist.is_initialized() else 0,
|
||||
size=dist.get_world_size() if dist.is_initialized() else 1,
|
||||
**self._placement,
|
||||
)
|
||||
|
||||
def _model_state(self):
|
||||
|
||||
@@ -23,6 +23,8 @@ async def update_weights(
|
||||
"""
|
||||
Update weights for the inference engine.
|
||||
This function is designed to be stateless, so that the caller process could keep the stateful engine.
|
||||
The caller must hold an open weight-update session (engine.begin_weight_update() ...
|
||||
engine.end_weight_update()) around one or more calls.
|
||||
Example Use Case:
|
||||
- Multiple Producer Process will call this function in a SPMD style
|
||||
|
||||
|
||||
Reference in New Issue
Block a user