[RL] Add RL weight-update sessions and support updating spec draft runners (#40777)

Co-authored-by: Jiajun Li <guapisolo@gmail.com>
Co-authored-by: Nan Jiang <59716405+nanjiangwill@users.noreply.github.com>
Co-authored-by: maocheng23 <maocheng@berkeley.edu>
Co-authored-by: Byron Hsu <byron+per@periodiclabs.ai>
Co-authored-by: JD-ETH <jaedon.guo@gmail.com>
Co-authored-by: Letian Ruan <ruanlt420@gmail.com>
Co-authored-by: hnyls2002 <lsyincs@gmail.com>
This commit is contained in:
Yueming Yuan
2026-09-23 20:04:57 -07:00
committed by GitHub
co-authored by Jiajun Li Nan Jiang maocheng23 Byron Hsu JD-ETH Letian Ruan hnyls2002
parent f4b90382a2
commit f4d9d2e670
37 changed files with 1157 additions and 398 deletions
@@ -45,6 +45,16 @@ class EngineBase(ABC):
"""Flush the cache of the engine."""
pass
@abstractmethod
def begin_weight_update(self, selector: str = "all"):
"""Open a weight-update session; update_weights_from_* must run inside one."""
pass
@abstractmethod
def end_weight_update(self):
"""Close the weight-update session and finalize quantized weights."""
pass
@abstractmethod
def update_weights_from_tensor(
self,
+16
View File
@@ -69,9 +69,11 @@ from sglang.srt.managers.data_parallel_controller import (
)
from sglang.srt.managers.detokenizer_manager import run_detokenizer_process
from sglang.srt.managers.io_struct import (
BeginWeightUpdateReqInput,
CloseSessionReqInput,
DestroyWeightsUpdateGroupReqInput,
EmbeddingReqInput,
EndWeightUpdateReqInput,
GenerateReqInput,
GetWeightsByNameReqInput,
InitWeightsUpdateGroupReqInput,
@@ -1537,6 +1539,20 @@ class Engine(EngineScoreMixin, EngineBase):
self.tokenizer_manager.destroy_weights_update_group(obj, None)
)
def begin_weight_update(self, selector: str = "all"):
"""Open a weight-update session; close it with end_weight_update()."""
obj = BeginWeightUpdateReqInput(selector=selector)
return self.loop.run_until_complete(
self.tokenizer_manager.begin_weight_update(obj, None)
)
def end_weight_update(self):
"""Close the session and finalize quantized weights into kernel layout."""
obj = EndWeightUpdateReqInput()
return self.loop.run_until_complete(
self.tokenizer_manager.end_weight_update(obj, None)
)
def update_weights_from_distributed(
self,
names: list[str],
@@ -117,6 +117,7 @@ from sglang.srt.function_call.function_call_parser import FunctionCallParser
from sglang.srt.managers.io_struct import (
AbortReq,
AttachHiCacheStorageReqInput,
BeginWeightUpdateReqInput,
CheckWeightsReqInput,
CloseSessionReqInput,
ConfigureLoggingReq,
@@ -124,6 +125,7 @@ from sglang.srt.managers.io_struct import (
DestroyWeightsUpdateGroupReqInput,
DumperControlReqInput,
EmbeddingReqInput,
EndWeightUpdateReqInput,
GenerateReqInput,
GetWeightsByNameReqInput,
InitWeightsSendGroupForRemoteInstanceReqInput,
@@ -1368,6 +1370,36 @@ async def update_weights_from_tensor(
)
@app.post("/begin_weight_update")
@auth_level(AuthLevel.ADMIN_OPTIONAL)
async def begin_weight_update(
obj: Annotated[BeginWeightUpdateReqInput, Body()], request: Request
):
"""Open a weight-update session so in-place-quantized weights become loadable."""
success, message = await _global_state.tokenizer_manager.begin_weight_update(
obj, request
)
return ORJSONResponse(
{"success": success, "message": message},
status_code=HTTPStatus.OK if success else HTTPStatus.BAD_REQUEST,
)
@app.post("/end_weight_update")
@auth_level(AuthLevel.ADMIN_OPTIONAL)
async def end_weight_update(
obj: Annotated[EndWeightUpdateReqInput, Body()], request: Request
):
"""Close the weight-update session and finalize quantized weights."""
success, message = await _global_state.tokenizer_manager.end_weight_update(
obj, request
)
return ORJSONResponse(
{"success": success, "message": message},
status_code=HTTPStatus.OK if success else HTTPStatus.BAD_REQUEST,
)
@app.post("/update_weights_from_distributed")
@auth_level(AuthLevel.ADMIN_OPTIONAL)
async def update_weights_from_distributed(
@@ -86,6 +86,12 @@ class HttpServerEngineAdapter(EngineBase):
response.raise_for_status()
return response.json()
def begin_weight_update(self, selector: str = "all"):
return self._make_request("begin_weight_update", {"selector": selector})
def end_weight_update(self):
return self._make_request("end_weight_update")
def update_weights_from_tensor(
self,
named_tensors: List[Tuple[str, torch.Tensor]],
@@ -46,6 +46,13 @@ class QuantizeMethodBase(ABC):
"""
return
def restore_weights_before_loading(self, layer: nn.Module) -> None:
"""Undo an in-place repack so checkpoint-format weights can be loaded again.
Needed only when `process_weights_after_loading` changes parameter shapes.
"""
return
class LinearMethodBase(QuantizeMethodBase):
"""Base class for different (maybe quantized) linear methods."""
@@ -1252,6 +1252,9 @@ class CompressedTensorsFusedMoEMethod(FusedMoEMethodBase):
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
layer.scheme.process_weights_after_loading(layer)
def restore_weights_before_loading(self, layer: torch.nn.Module) -> None:
layer.scheme.restore_weights_before_loading(layer)
def create_weights(
self,
layer: torch.nn.Module,
@@ -99,6 +99,9 @@ class CompressedTensorsMoEScheme(BaseMoEScheme):
"""
raise NotImplementedError
def restore_weights_before_loading(self, layer: torch.nn.Module) -> None:
return
@abstractmethod
def apply_weights(
self,
+32 -3
View File
@@ -1875,6 +1875,8 @@ class UpdateWeightsFromDistributedReqInput(BaseReq, kw_only=True):
weight_version: Optional[str] = None
# Optional format specification for loading
load_format: Optional[str] = None
# which runners the op applies to
selector: Literal["target", "draft", "all"] = "all"
# Whether to call torch.cuda.empty_cache() during flush
torch_empty_cache: bool = False
@@ -1901,8 +1903,8 @@ class UpdateWeightsFromTensorReqInput(BaseReq, kw_only=True):
abort_all_requests: bool = False
# Optional: Update weight version along with weights
weight_version: Optional[str] = None
# Optional: Determine whether to disable updating the draft model
disable_draft_model: Optional[bool] = None
# which runners the op applies to
selector: Literal["target", "draft", "all"] = "all"
# Whether to call torch.cuda.empty_cache() during flush
torch_empty_cache: bool = False
@@ -2054,15 +2056,41 @@ class ResumeMemoryOccupationReqOutput(BaseReq, kw_only=True):
pass
class BeginWeightUpdateReqInput(BaseReq, kw_only=True):
"""Open a weight-update session: restore in-place-packed weights so new ones can load."""
selector: Literal["target", "draft", "all"] = "all"
class BeginWeightUpdateReqOutput(BaseReq, kw_only=True):
success: bool
message: str
class EndWeightUpdateReqInput(BaseReq, kw_only=True):
"""Close the weight-update session opened by BeginWeightUpdateReqInput."""
class EndWeightUpdateReqOutput(BaseReq, kw_only=True):
success: bool
message: str
class CheckWeightsReqInput(BaseReq, kw_only=True):
action: str = "checksum"
allow_quant_error: bool = False
# Substrings of tensor names to exclude from reset/compare/checksum.
skip_tensor_list: Optional[List[str]] = None
# which runners the op applies to
selector: Literal["target", "draft", "all"] = "all"
# Wire versions of the pydantic ParallelismInfo/ChecksumInfo in
# sglang.srt.utils.weight_checker. Not array_like: the payload is read by field
# name and re-serialized to JSON, so it must stay a {field: value} map.
class ParallelismInfo(msgspec.Struct, kw_only=True):
# "target", or a draft role such as "draft" / "draft_step_0"
role: str
tp_rank: int
tp_size: int
dp_rank: int
@@ -2076,7 +2104,8 @@ class ParallelismInfo(msgspec.Struct, kw_only=True):
class ChecksumInfo(msgspec.Struct, kw_only=True):
checksums: Dict[str, str]
per_gpu_checksum: str
parallelism_info: ParallelismInfo
# one entry per role (target plus each draft runner); all share the GPU rank
parallelism_info: List[ParallelismInfo]
class CheckWeightsReqOutput(BaseReq, kw_only=True):
+10
View File
@@ -126,6 +126,7 @@ from sglang.srt.managers.io_struct import (
AttachHiCacheStorageReqOutput,
BatchTokenizedEmbeddingReqInput,
BatchTokenizedGenerateReqInput,
BeginWeightUpdateReqInput,
CheckWeightsReqInput,
ClearHiCacheReqInput,
ClearHiCacheReqOutput,
@@ -137,6 +138,7 @@ from sglang.srt.managers.io_struct import (
DetachHiCacheStorageReqOutput,
DumperControlReqInput,
DumperControlReqOutput,
EndWeightUpdateReqInput,
ExpertDistributionReq,
ExpertDistributionReqOutput,
ExpertDistributionReqType,
@@ -1738,6 +1740,14 @@ class Scheduler(
SendWeightsToRemoteInstanceReqInput,
self.send_weights_to_remote_instance,
),
(
BeginWeightUpdateReqInput,
self.weight_updater.begin_weight_update,
),
(
EndWeightUpdateReqInput,
self.weight_updater.end_weight_update,
),
(
UpdateWeightsFromDistributedReqInput,
self.weight_updater.update_weights_from_distributed,
@@ -1,12 +1,11 @@
from __future__ import annotations
import hashlib
import logging
import time
import traceback
from contextlib import contextmanager
from dataclasses import dataclass, field
from typing import Any, Callable, Dict, Iterator, Optional, Tuple
from typing import Any, Callable, Dict, Iterator, List, Optional, Set, Tuple
import msgspec
import torch
@@ -19,11 +18,15 @@ from sglang.srt.constants import (
)
from sglang.srt.disaggregation.utils import DisaggregationMode
from sglang.srt.managers.io_struct import (
BeginWeightUpdateReqInput,
BeginWeightUpdateReqOutput,
ChecksumInfo,
CheckWeightsReqInput,
CheckWeightsReqOutput,
DestroyWeightsUpdateGroupReqInput,
DestroyWeightsUpdateGroupReqOutput,
EndWeightUpdateReqInput,
EndWeightUpdateReqOutput,
GetWeightsByNameReqInput,
GetWeightsByNameReqOutput,
InitWeightsUpdateGroupReqInput,
@@ -42,35 +45,43 @@ from sglang.srt.managers.io_struct import (
UpdateWeightsFromTensorReqOutput,
)
from sglang.srt.runtime_context import get_model
from sglang.srt.utils.weight_checker import overall_checksum
logger = logging.getLogger(__name__)
def _get_draft_model_runner(draft_worker):
# DFlash / FrozenKVMTP workers expose draft_model_runner directly
runner = getattr(draft_worker, "draft_model_runner", None)
if runner is not None:
return runner
# EAGLEWorkerV2: _draft_worker.draft_runner
inner = getattr(draft_worker, "_draft_worker", None)
if inner is not None:
runner = getattr(inner, "draft_runner", None)
if runner is not None:
return runner
return None
def _merge_checksum_payloads(role_payloads: List[Tuple[str, Dict]]) -> Dict:
merged: Dict[str, str] = {}
parallelism_infos = []
for role, p in role_payloads:
for name, chk in p["checksums"].items():
# only draft roles are prefixed, so target keys stay stable
key = name if role == "target" else f"{role}.{name}"
if key in merged:
raise ValueError(f"checksum key collision: {key}")
merged[key] = chk
parallelism_infos.append(p["parallelism_info"])
return {
"checksums": merged,
"per_gpu_checksum": overall_checksum(merged),
"parallelism_info": parallelism_infos,
}
def _merge_checksum_payloads(target: Dict, draft: Dict) -> Dict:
merged_checksums = dict(target["checksums"])
for name, chk in draft["checksums"].items():
merged_checksums[f"draft.{name}"] = chk
h = hashlib.sha256()
for name in sorted(merged_checksums):
h.update(name.encode())
h.update(merged_checksums[name].encode())
target["checksums"] = merged_checksums
target["per_gpu_checksum"] = h.hexdigest()
return target
def _parse_runner_selector(selector: str) -> Set[str]:
if selector == "all":
return {"target", "draft"}
if selector in ("target", "draft"):
return {selector}
raise ValueError(
f"invalid selector {selector!r}; expected 'target', 'draft', or 'all'"
)
class _WeightUpdateSession(msgspec.Struct, frozen=True):
# recorded at begin so end finalizes the same runners
selector: str
loaded_weights: bool = False
@dataclass(kw_only=True, slots=True)
@@ -85,6 +96,8 @@ class SchedulerWeightUpdaterManager:
metrics_collector: Optional[Any] = None
offload_tags: set = field(default_factory=set)
stashed_model_static_state: Any = None
# replicated on every TP rank, so a rejected call returns on all ranks before any barrier
_session: Optional[_WeightUpdateSession] = None
@contextmanager
def _observe_weight_load(self, source: str) -> Iterator[None]:
@@ -114,11 +127,19 @@ class SchedulerWeightUpdaterManager:
def update_weights_from_disk(self, recv_req: UpdateWeightFromDiskReqInput):
"""In-place update of the weights from disk."""
with self._observe_weight_load("disk"):
success, message = self.tp_worker.update_weights_from_disk(recv_req)
tp_success = success
if success and self.draft_worker is not None:
success, message = self.draft_worker.update_weights_from_disk(recv_req)
if tp_success:
success, message = True, "Succeeded to update model weights."
target_updated = False
for role, runner in self._select_runners():
success, message = runner.weight_updater.update_weights_from_disk(
recv_req.model_path,
recv_req.load_format,
recapture_cuda_graph=recv_req.recapture_cuda_graph,
)
if not success:
break
target_updated |= role == "target"
# the served weights changed even if a draft runner failed afterwards
if target_updated:
self.flush_cache_after_weight_update(recv_req)
if success:
self.record_weight_version_after_update(recv_req.weight_version)
@@ -141,31 +162,82 @@ class SchedulerWeightUpdaterManager:
success, message = self.tp_worker.destroy_weights_update_group(recv_req)
return DestroyWeightsUpdateGroupReqOutput(success=success, message=message)
def _select_runners(self, selector: str = "all") -> List[Tuple[str, Any]]:
roles = _parse_runner_selector(selector)
runners: List[Tuple[str, Any]] = []
if "target" in roles:
runners += self.tp_worker.weight_update_runners()
if "draft" in roles and self.draft_worker is not None:
runners += self.draft_worker.weight_update_runners()
return runners
def update_weights_from_distributed(
self,
recv_req: UpdateWeightsFromDistributedReqInput,
) -> Tuple[bool, str]:
"""Update the online model parameter."""
"""Update the online model parameter, fanning out to the selected runners."""
if self._session is None:
return UpdateWeightsFromDistributedReqOutput(
success=False,
message="update_weights_from_distributed must run between "
"begin_weight_update() and end_weight_update()",
)
with self._observe_weight_load("distributed"):
success, message = self.tp_worker.update_weights_from_distributed(recv_req)
# only the target runner joined the update group; drafts load its receive
target = self.tp_worker.model_runner.weight_updater
try:
weights = target.receive_weights_from_distributed(
names=recv_req.names,
dtypes=recv_req.dtypes,
shapes=recv_req.shapes,
group_name=recv_req.group_name,
load_format=recv_req.load_format,
)
except Exception as e:
success, message = False, f"Failed to receive weights: {e}"
logger.error(message)
else:
success, message = True, "Succeeded to update parameter online."
for _, runner in self._select_runners(recv_req.selector):
success, message = (
runner.weight_updater.load_weights_from_distributed(weights)
)
if not success:
break
if success:
self._session = msgspec.structs.replace(
self._session, loaded_weights=True
)
self.flush_cache_after_weight_update(recv_req)
self.record_weight_version_after_update(recv_req.weight_version)
else:
logger.error(message)
return UpdateWeightsFromDistributedReqOutput(
success=success, message=message
)
def update_weights_from_tensor(self, recv_req: UpdateWeightsFromTensorReqInput):
"""Update the online model parameter from tensors."""
"""Update the online model parameter from tensors on the selected runners."""
if self._session is None:
return UpdateWeightsFromTensorReqOutput(
success=False,
message="update_weights_from_tensor must run between "
"begin_weight_update() and end_weight_update()",
)
with self._observe_weight_load("tensor"):
if recv_req.disable_draft_model:
worker = self.tp_worker
else:
worker = self.draft_worker or self.tp_worker
success, message = worker.update_weights_from_tensor(recv_req)
named_tensors = self.tp_worker.deserialize_own_rank(
recv_req.serialized_named_tensors
)
success, message = True, "Success"
for _, runner in self._select_runners(recv_req.selector):
success, message = runner.weight_updater.update_weights_from_tensor(
named_tensors=named_tensors,
load_format=recv_req.load_format,
)
if not success:
break
if success:
self._session = msgspec.structs.replace(
self._session, loaded_weights=True
)
self.flush_cache_after_weight_update(recv_req)
self.record_weight_version_after_update(recv_req.weight_version)
else:
@@ -176,11 +248,17 @@ class SchedulerWeightUpdaterManager:
def update_weights_from_ipc(self, recv_req: UpdateWeightsFromIPCReqInput):
"""Update the online model parameter from IPC for checkpoint-engine integration."""
with self._observe_weight_load("ipc"):
success, message = self.tp_worker.update_weights_from_ipc(recv_req)
tp_success = success
if success and self.draft_worker is not None:
success, message = self.draft_worker.update_weights_from_ipc(recv_req)
if tp_success:
success, message = True, "Succeeded to update model weights."
target_updated = False
for role, runner in self._select_runners():
success, message = runner.weight_updater.update_weights_from_ipc(
recv_req
)
if not success:
break
target_updated |= role == "target"
# the served weights changed even if a draft runner failed afterwards
if target_updated:
self.flush_cache_after_weight_update(recv_req)
if success:
self.record_weight_version_after_update(recv_req.weight_version)
@@ -209,6 +287,34 @@ class SchedulerWeightUpdaterManager:
f"Restart with --weight-cache-mode off to use this operation."
)
def begin_weight_update(self, recv_req: BeginWeightUpdateReqInput):
"""Open the session: restore in-place-packed weights on the selected runners."""
if self._session is not None:
return BeginWeightUpdateReqOutput(
success=False,
message="a weight-update session is already open; "
"call end_weight_update() first",
)
for _, runner in self._select_runners(recv_req.selector):
runner.weight_updater.begin_weight_update()
self._session = _WeightUpdateSession(selector=recv_req.selector)
torch.distributed.barrier(group=self.tp_cpu_group)
return BeginWeightUpdateReqOutput(success=True, message="Success")
def end_weight_update(self, recv_req: EndWeightUpdateReqInput):
"""Finalize the runners begin opened; post_load_weights only if no load ran (P2P/RDMA)."""
if self._session is None:
return EndWeightUpdateReqOutput(
success=False,
message="no weight-update session is open; call begin_weight_update() first",
)
run_post_load = not self._session.loaded_weights
for _, runner in self._select_runners(self._session.selector):
runner.weight_updater.end_weight_update(run_post_load=run_post_load)
self._session = None
torch.distributed.barrier(group=self.tp_cpu_group)
return EndWeightUpdateReqOutput(success=True, message="Success")
def release_memory_occupation(self, recv_req: ReleaseMemoryOccupationReqInput):
scheduler = self.scheduler
assert self.is_fully_idle(
@@ -298,19 +404,17 @@ class SchedulerWeightUpdaterManager:
def check_weights(self, recv_req: CheckWeightsReqInput):
try:
payload = self.tp_worker.model_runner.check_weights(
action=recv_req.action, allow_quant_error=recv_req.allow_quant_error
)
if self.draft_worker is not None:
draft_runner = _get_draft_model_runner(self.draft_worker)
if draft_runner is not None:
draft_payload = draft_runner.check_weights(
action=recv_req.action,
allow_quant_error=recv_req.allow_quant_error,
)
if payload is not None and draft_payload is not None:
payload = _merge_checksum_payloads(payload, draft_payload)
role_payloads = []
for role, runner in self._select_runners(recv_req.selector):
p = runner.check_weights(
action=recv_req.action,
allow_quant_error=recv_req.allow_quant_error,
skip_tensor_list=recv_req.skip_tensor_list,
role=role,
)
if p is not None:
role_payloads.append((role, p))
payload = _merge_checksum_payloads(role_payloads) if role_payloads else None
tp_size = torch.distributed.get_world_size(group=self.tp_cpu_group)
if tp_size > 1 and payload is not None:
@@ -16,6 +16,8 @@ from sglang.srt.managers.io_struct import (
AddExternalCorpusReqOutput,
AttachHiCacheStorageReqInput,
AttachHiCacheStorageReqOutput,
BeginWeightUpdateReqInput,
BeginWeightUpdateReqOutput,
ChecksumInfo,
CheckWeightsReqInput,
CheckWeightsReqOutput,
@@ -28,6 +30,8 @@ from sglang.srt.managers.io_struct import (
DetachHiCacheStorageReqOutput,
DumperControlReqInput,
DumperControlReqOutput,
EndWeightUpdateReqInput,
EndWeightUpdateReqOutput,
ExpertDistributionReq,
ExpertDistributionReqOutput,
ExpertDistributionReqType,
@@ -131,6 +135,8 @@ _COMMUNICATOR_SPECS = [
("get_internal_state", GetInternalStateReqOutput),
("set_internal_state", SetInternalStateReqOutput),
("expert_distribution", ExpertDistributionReqOutput),
("begin_weight_update", BeginWeightUpdateReqOutput),
("end_weight_update", EndWeightUpdateReqOutput),
("update_lora_adapter", LoRAUpdateOutput),
("dumper_control", DumperControlReqOutput),
("scale_elastic_ep", ScaleElasticEPReqOutput),
@@ -449,6 +455,38 @@ class TokenizerControlMixin:
results = await self.destroy_weights_update_group_communicator(obj)
return FanOutCommunicator.merge_results(results)
async def _weight_update_session_call(
self: TokenizerManager, communicator, obj
) -> Tuple[bool, str]:
self.auto_create_handle_loop()
async with self.is_pause_cond:
is_paused = self.is_pause
# whoever paused the engine holds the writer lock; taking it again deadlocks
if is_paused:
results = await communicator(obj)
if not is_paused:
async with self.model_update_lock.writer_lock:
results = await communicator(obj)
return FanOutCommunicator.merge_results(results)
async def begin_weight_update(
self: TokenizerManager,
obj: BeginWeightUpdateReqInput,
request: Optional[fastapi.Request] = None,
) -> Tuple[bool, str]:
return await self._weight_update_session_call(
self.begin_weight_update_communicator, obj
)
async def end_weight_update(
self: TokenizerManager,
obj: EndWeightUpdateReqInput,
request: Optional[fastapi.Request] = None,
) -> Tuple[bool, str]:
return await self._weight_update_session_call(
self.end_weight_update_communicator, obj
)
async def update_weights_from_distributed(
self: TokenizerManager,
obj: UpdateWeightsFromDistributedReqInput,
+6 -42
View File
@@ -32,10 +32,6 @@ from sglang.srt.managers.io_struct import (
LoadLoRAAdapterReqInput,
SendWeightsToRemoteInstanceReqInput,
UnloadLoRAAdapterReqInput,
UpdateWeightFromDiskReqInput,
UpdateWeightsFromDistributedReqInput,
UpdateWeightsFromIPCReqInput,
UpdateWeightsFromTensorReqInput,
)
from sglang.srt.managers.schedule_batch import ScheduleBatch
from sglang.srt.managers.scheduler import GenerationBatchResult
@@ -146,14 +142,6 @@ class BaseTpWorker(ABC):
self.model_runner.token_to_kv_pool_allocator,
)
def update_weights_from_disk(self, recv_req: UpdateWeightFromDiskReqInput):
success, message = self.model_runner.weight_updater.update_weights_from_disk(
recv_req.model_path,
recv_req.load_format,
recapture_cuda_graph=recv_req.recapture_cuda_graph,
)
return success, message
def init_weights_update_group(self, recv_req: InitWeightsUpdateGroupReqInput):
success, message = self.model_runner.weight_updater.init_weights_update_group(
recv_req.master_address,
@@ -200,21 +188,7 @@ class BaseTpWorker(ABC):
)
return success, message
def update_weights_from_distributed(
self, recv_req: UpdateWeightsFromDistributedReqInput
):
success, message = (
self.model_runner.weight_updater.update_weights_from_distributed(
recv_req.names,
recv_req.dtypes,
recv_req.shapes,
recv_req.group_name,
recv_req.load_format,
)
)
return success, message
def _deserialize_own_rank(self, serialized_named_tensors):
def deserialize_own_rank(self, serialized_named_tensors):
"""Each rank deserializes only its own payload (index tp_rank);
deserializing another rank's copy would break producer-side CUDA-IPC
refcounting."""
@@ -223,20 +197,6 @@ class BaseTpWorker(ABC):
serialized_named_tensors[self.model_runner.tp_rank]
)
def update_weights_from_tensor(self, recv_req: UpdateWeightsFromTensorReqInput):
success, message = self.model_runner.weight_updater.update_weights_from_tensor(
named_tensors=self._deserialize_own_rank(recv_req.serialized_named_tensors),
load_format=recv_req.load_format,
)
return success, message
def update_weights_from_ipc(self, recv_req: UpdateWeightsFromIPCReqInput):
"""Update weights from IPC for checkpoint-engine integration."""
success, message = self.model_runner.weight_updater.update_weights_from_ipc(
recv_req
)
return success, message
def get_weights_by_name(self, recv_req: GetWeightsByNameReqInput):
parameter = self.model_runner.weight_exporter.get_weights_by_name(
recv_req.name, recv_req.truncate_size
@@ -256,7 +216,7 @@ class BaseTpWorker(ABC):
):
# The LoRA code handles TP sharding internally using slice_lora_a_weights
# and slice_lora_b_weights methods (see lora/layers.py and mem_pool.py).
data = self._deserialize_own_rank(recv_req.serialized_named_tensors)
data = self.deserialize_own_rank(recv_req.serialized_named_tensors)
if recv_req.load_format == "flattened_bucket":
bucket = FlattenedTensorBucket(
flattened_tensor=data["flattened_tensor"],
@@ -562,6 +522,10 @@ class TpModelWorker(BaseTpWorker):
def model_runner(self) -> ModelRunner:
return self._model_runner
def weight_update_runners(self) -> List[Tuple[str, ModelRunner]]:
"""(role, runner) pairs weight ops apply to; the target worker owns one."""
return [("target", self._model_runner)]
def register_hicache_layer_transfer_counter(self, counter: LayerDoneCounter):
self.hicache_layer_transfer_counter = counter
@@ -20,7 +20,7 @@ import inspect
import logging
import time
from dataclasses import dataclass
from typing import Callable, Optional, Union
from typing import Callable, List, Optional, Union
import torch
import torch.distributed as dist
@@ -2048,9 +2048,19 @@ class ModelRunner:
forward_batch.token_ids_logprobs,
)
def check_weights(self, action: str, allow_quant_error: bool = False):
def check_weights(
self,
action: str,
allow_quant_error: bool = False,
skip_tensor_list: Optional[List[str]] = None,
*,
role: str,
):
return self._weight_checker.handle(
action=action, allow_quant_error=allow_quant_error
action=action,
allow_quant_error=allow_quant_error,
skip_tensor_list=skip_tensor_list,
role=role,
)
def _expand_eplb_metadata_for_scale(
@@ -8,7 +8,11 @@ from typing import TYPE_CHECKING, Any, Callable, List, Optional, Tuple, Union
import torch
from sglang.srt.configs.load_config import LoadConfig
from sglang.srt.model_loader.loader import DefaultModelLoader, get_model_loader
from sglang.srt.model_loader.loader import (
DefaultModelLoader,
get_model_loader,
post_load_weights,
)
from sglang.srt.model_loader.utils import set_default_torch_dtype
from sglang.srt.model_loader.weight_utils import default_weight_loader
from sglang.srt.platforms import current_platform
@@ -241,94 +245,87 @@ class WeightUpdater:
logger.info("Update weights end.")
return True, "Succeeded to update model weights."
def update_weights_from_distributed(
def receive_weights_from_distributed(
self: WeightUpdater,
*,
names,
dtypes,
shapes,
group_name,
load_format: Optional[str] = None,
):
"""
Update specific parameter in the model weights online
through `_model_update_group` process group.
Args:
name: the name of the parameter to be updated.
dtype: the data type of the parameter to be updated.
shape: the shape of the parameter to be updated.
"""
self._assert_weight_cache_inactive("update_weights_from_distributed")
error = _unsupported_derived_weight_cache_error(self.get_model())
if error is not None:
return False, error
"""Receive one broadcast without loading it; only the target runner joined the group."""
assert group_name in self._model_update_group, (
f"Group {group_name} not in {list(self._model_update_group.keys())}. "
"Please call `init_weights_update_group` first."
)
if load_format == "flattened_bucket":
return self._update_bucketed_weights_from_distributed(
names, dtypes, shapes, group_name
return self._receive_bucketed_weights_from_distributed(
names=names, dtypes=dtypes, shapes=shapes, group_name=group_name
)
try:
weights = []
handles = []
for name, dtype, shape in zip(names, dtypes, shapes):
target_dtype = (
dtype if isinstance(dtype, torch.dtype) else getattr(torch, dtype)
)
weight = torch.empty(shape, dtype=target_dtype, device=self.device)
handles.append(
torch.distributed.broadcast(
weight,
src=0,
group=self._model_update_group[group_name],
async_op=True,
)
)
weights.append((name, weight))
for handle in handles:
handle.wait()
self.get_model().load_weights(weights)
return True, "Succeeded to update parameter online."
except Exception as e:
error_msg = (
f"Failed to update parameter online: {e}. "
f"The full weights of the ModelRunner are partially updated. "
f"Please discard the whole weights."
weights = []
handles = []
for name, dtype, shape in zip(names, dtypes, shapes):
target_dtype = (
dtype if isinstance(dtype, torch.dtype) else getattr(torch, dtype)
)
logger.error(error_msg)
return False, error_msg
weight = torch.empty(shape, dtype=target_dtype, device=self.device)
handles.append(
torch.distributed.broadcast(
weight,
src=0,
group=self._model_update_group[group_name],
async_op=True,
)
)
weights.append((name, weight))
for handle in handles:
handle.wait()
return weights
def _update_bucketed_weights_from_distributed(
self: WeightUpdater, names, dtypes, shapes, group_name
def _receive_bucketed_weights_from_distributed(
self: WeightUpdater, *, names, dtypes, shapes, group_name
):
try:
named_tensors = []
for name, dtype, shape in zip(names, dtypes, shapes):
target_dtype = (
dtype if isinstance(dtype, torch.dtype) else getattr(torch, dtype)
)
named_tensors.append(
(
name,
torch.empty(shape, dtype=target_dtype, device=self.device),
)
)
bucket = FlattenedTensorBucket(named_tensors=named_tensors)
flattened_tensor = bucket.get_flattened_tensor()
torch.distributed.broadcast(
flattened_tensor,
src=0,
group=self._model_update_group[group_name],
named_tensors = []
for name, dtype, shape in zip(names, dtypes, shapes):
target_dtype = (
dtype if isinstance(dtype, torch.dtype) else getattr(torch, dtype)
)
reconstructed_tensors = bucket.reconstruct_tensors()
self.get_model().load_weights(reconstructed_tensors)
return True, "Succeeded to update parameter online."
named_tensors.append(
(name, torch.empty(shape, dtype=target_dtype, device=self.device))
)
bucket = FlattenedTensorBucket(named_tensors=named_tensors)
flattened_tensor = bucket.get_flattened_tensor()
torch.distributed.broadcast(
flattened_tensor,
src=0,
group=self._model_update_group[group_name],
)
return bucket.reconstruct_tensors()
def begin_weight_update(self: WeightUpdater) -> None:
DefaultModelLoader.restore_weights_before_loading(
self.get_model(), torch.device(self.device)
)
def end_weight_update(self: WeightUpdater, *, run_post_load: bool) -> None:
if run_post_load:
post_load_weights(self.get_model())
DefaultModelLoader.postprocess_weights(
self.get_model(), torch.device(self.device)
)
def load_weights_from_distributed(
self: WeightUpdater, named_tensors: List[Tuple[str, torch.Tensor]]
) -> Tuple[bool, str]:
self._assert_weight_cache_inactive("update_weights_from_distributed")
error = _unsupported_derived_weight_cache_error(self.get_model())
if error is not None:
return False, error
try:
self.get_model().load_weights(named_tensors)
except Exception as e:
error_msg = (
f"Failed to update parameter online: {e}. "
@@ -337,6 +334,7 @@ class WeightUpdater:
)
logger.error(error_msg)
return False, error_msg
return True, "Succeeded to update parameter online."
def update_weights_from_tensor(
self: WeightUpdater,
+39 -17
View File
@@ -87,7 +87,10 @@ from sglang.srt.layers.modelopt_utils import QUANT_CFG_CHOICES
from sglang.srt.layers.moe.utils import (
install_shared_experts_fusion_decision,
)
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.quantization.base_config import (
QuantizationConfig,
QuantizeMethodBase,
)
from sglang.srt.model_loader.remote_instance_weight_loader_utils import (
trigger_transferring_weights_request,
)
@@ -304,7 +307,7 @@ def _initialize_model(
return model_class(**kwargs)
def _post_load_weights(model: nn.Module) -> None:
def post_load_weights(model: nn.Module) -> None:
# Loaders that bypass `model.load_weights()` (dummy / sharded state / remote instance /
# remote fs) must trigger the model's post-load fixup explicitly; `model.load_weights()`
# would normally do it internally. NextN subclasses override the method to fill in
@@ -313,6 +316,18 @@ def _post_load_weights(model: nn.Module) -> None:
model.post_load_weights()
def _modules_with_quant_method(model: nn.Module):
from sglang.srt.lora.layers import BaseLayerWithLoRA
for _, module in model.named_modules():
# LoRA wrappers forward quant_method but do not own the packed params
if isinstance(module, BaseLayerWithLoRA):
continue
quant_method = getattr(module, "quant_method", None)
if quant_method is not None:
yield module, quant_method
class BaseModelLoader(ABC):
"""Base class for model loaders."""
@@ -836,7 +851,7 @@ class DefaultModelLoader(BaseModelLoader):
"""
with set_default_torch_dtype(model_config.dtype):
initialize_capture_safe_weights(model)
_post_load_weights(model)
post_load_weights(model)
for _, module in model.named_modules():
quant_method = getattr(module, "quant_method", None)
if quant_method is None:
@@ -1056,16 +1071,23 @@ class DefaultModelLoader(BaseModelLoader):
@staticmethod
def postprocess_weights(model, target_device):
for _, module in model.named_modules():
quant_method = getattr(module, "quant_method", None)
if quant_method is not None:
# When quant methods need to process weights after loading
# (for repacking, quantizing, etc), they expect parameters
# to be on the global target device. This scope is for the
# case where cpu offloading is used, where we will move the
# parameters onto device for processing and back off after.
for module, quant_method in _modules_with_quant_method(model):
# When quant methods need to process weights after loading
# (for repacking, quantizing, etc), they expect parameters
# to be on the global target device. This scope is for the
# case where cpu offloading is used, where we will move the
# parameters onto device for processing and back off after.
with device_loading_context(module, target_device):
quant_method.process_weights_after_loading(module)
@staticmethod
def restore_weights_before_loading(model, target_device):
"""Undo in-place quant packing so fresh weights can be loaded."""
for module, quant_method in _modules_with_quant_method(model):
# AMX packing and the MXFP4 backend wrappers are duck-typed and cannot restore
if isinstance(quant_method, QuantizeMethodBase):
with device_loading_context(module, target_device):
quant_method.process_weights_after_loading(module)
quant_method.restore_weights_before_loading(module)
class LayeredModelLoader(DefaultModelLoader):
@@ -1643,7 +1665,7 @@ class DummyModelLoader(BaseModelLoader):
# random values to the weights.
initialize_dummy_weights(model)
_post_load_weights(model)
post_load_weights(model)
for _, module in model.named_modules():
quant_method = getattr(module, "quant_method", None)
@@ -1804,7 +1826,7 @@ class ShardedStateLoader(BaseModelLoader):
if state_dict:
raise ValueError(f"Missing keys {tuple(state_dict)} in loaded state!")
_post_load_weights(model)
post_load_weights(model)
return model.eval()
@@ -3426,7 +3448,7 @@ class RemoteInstanceModelLoader(BaseModelLoader):
)
current_platform.synchronize()
_post_load_weights(model)
post_load_weights(model)
end_get_weights_tic = time.time()
logger.debug(
f"finish getting all weights from remote instance, time used: {(end_get_weights_tic - start_get_weights_tic):.4f}s"
@@ -3489,7 +3511,7 @@ class RemoteInstanceModelLoader(BaseModelLoader):
logger.error(f"batch transfer failed, error: {ret}")
return False
_post_load_weights(model)
post_load_weights(model)
return True
@@ -3579,7 +3601,7 @@ class RemoteModelLoader(BaseModelLoader):
if state_dict:
raise ValueError(f"Missing keys {tuple(state_dict)} in loaded state!")
_post_load_weights(model)
post_load_weights(model)
def _load_model_from_remote_fs(
self, model, client, model_config: ModelConfig, device_config: DeviceConfig
@@ -14,10 +14,6 @@ from sglang.srt.model_executor.graph_memory_usage import (
from sglang.srt.runtime_context import get_disagg, get_exec, get_memory, get_schedule
if TYPE_CHECKING:
from sglang.srt.managers.io_struct import (
UpdateWeightFromDiskReqInput,
UpdateWeightsFromIPCReqInput,
)
from sglang.srt.managers.tp_worker import TpModelWorker
from sglang.srt.model_executor.model_runner import (
ModelRunner,
@@ -162,6 +158,13 @@ class BaseSpecWorker(ABC):
def hicache_draft_plan(self) -> HiCacheDraftPlan:
return self._hicache_draft_plan
def weight_update_runners(self) -> list[tuple[str, ModelRunner]]:
"""(role, runner) pairs weight ops apply to; [] when the draft owns no weights."""
# NGRAM and UNO have no draft model
if self.draft_worker is None:
return []
return [("draft", runner) for runner in self.draft_worker.draft_runners]
def _draft_model_runners(self) -> tuple[ModelRunner, ...]:
spec_algorithm = self.target_worker.model_runner.spec_algorithm
draft_worker = self.draft_worker
@@ -315,24 +318,6 @@ class BaseSpecWorker(ABC):
if self.draft_worker is not None:
self.draft_worker.init_cuda_graphs()
def update_weights_from_disk(self, recv_req: UpdateWeightFromDiskReqInput):
for runner in self.draft_worker.draft_runners:
success, message = runner.weight_updater.update_weights_from_disk(
recv_req.model_path,
recv_req.load_format,
recapture_cuda_graph=recv_req.recapture_cuda_graph,
)
if not success:
return success, message
return True, "Succeeded to update model weights."
def update_weights_from_ipc(self, recv_req: UpdateWeightsFromIPCReqInput):
for runner in self.draft_worker.draft_runners:
success, message = runner.weight_updater.update_weights_from_ipc(recv_req)
if not success:
return success, message
return True, "Succeeded to update model weights."
def on_verify_complete_cpu(
self, num_correct_drafts_per_req: list[int], batch_size: int = 0
) -> None:
@@ -1,7 +1,7 @@
import logging
import math
import os
from typing import List, Optional, Tuple
from typing import TYPE_CHECKING, List, Optional, Tuple
import torch
import torch.distributed as dist
@@ -92,6 +92,9 @@ from sglang.srt.utils.common import empty_context
_is_npu = is_npu()
if TYPE_CHECKING:
from sglang.srt.model_executor.model_runner import ModelRunner
logger = logging.getLogger(__name__)
_FusedKVMaterializeHelper = None
@@ -358,6 +361,9 @@ class DFlashWorkerV2(BaseSpecWorker):
scheduler runs it synchronously when overlap is disabled.
"""
def weight_update_runners(self) -> List[Tuple[str, "ModelRunner"]]:
return [("draft", self.draft_model_runner)]
def __init__(
self,
server_args: ServerArgs,
@@ -1,6 +1,6 @@
import logging
from contextlib import nullcontext
from typing import Callable, Optional, Protocol, runtime_checkable
from typing import TYPE_CHECKING, Callable, Optional, Protocol, runtime_checkable
import torch
@@ -84,6 +84,9 @@ from sglang.srt.utils import (
is_pin_memory_available,
)
if TYPE_CHECKING:
from sglang.srt.model_executor.model_runner import ModelRunner
logger = logging.getLogger(__name__)
_is_npu = is_npu()
@@ -127,6 +130,9 @@ def _configure_target_hidden_projection(
class DSparkWorkerV2(BaseSpecWorker):
"""Non-last PP stages run only the target; draft state belongs to the last stage."""
def weight_update_runners(self) -> list[tuple[str, "ModelRunner"]]:
return [("draft", self.draft_model_runner)] if self._hosts_draft else []
def __init__(
self,
server_args: ServerArgs,
@@ -38,7 +38,6 @@ from sglang.srt.layers.moe.utils import (
speculative_moe_a2a_backend_context,
speculative_moe_backend_context,
)
from sglang.srt.managers.io_struct import UpdateWeightsFromTensorReqInput
from sglang.srt.managers.schedule_batch import ScheduleBatch
from sglang.srt.managers.scheduler import GenerationBatchResult
from sglang.srt.managers.tp_worker import TpModelWorker
@@ -117,7 +116,6 @@ from sglang.srt.utils.async_probe import (
maybe_detect_oob,
)
from sglang.srt.utils.common import (
MultiprocessingSerializer,
empty_context,
fast_topk,
get_available_gpu_memory,
@@ -129,7 +127,6 @@ from sglang.srt.utils.common import (
is_xpu,
log_info_on_rank0,
)
from sglang.srt.utils.patch_torch import monkey_patch_torch_reductions
_is_cpu = is_cpu()
_is_npu = is_npu()
@@ -1868,25 +1865,3 @@ class EAGLEWorkerV2(BaseSpecWorker):
finalize_tree_path=True,
grammar_barrier=grammar_barrier,
)
def update_weights_from_tensor(self, recv_req: UpdateWeightsFromTensorReqInput):
monkey_patch_torch_reductions()
named_tensors = MultiprocessingSerializer.deserialize(
recv_req.serialized_named_tensors[self.model_runner.tp_rank]
)
success, message = (
self.draft_worker.draft_runner.weight_updater.update_weights_from_tensor(
named_tensors=named_tensors,
load_format=recv_req.load_format,
)
)
if not success:
return success, message
success, message = (
self.target_worker.model_runner.weight_updater.update_weights_from_tensor(
named_tensors=named_tensors,
load_format=recv_req.load_format,
)
)
return success, message
@@ -17,7 +17,7 @@ from __future__ import annotations
import logging
import time
from dataclasses import replace
from typing import TYPE_CHECKING, List
from typing import TYPE_CHECKING, List, Tuple
import torch
@@ -1001,6 +1001,12 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase):
class MultiLayerEagleWorkerV2(BaseSpecWorker):
def weight_update_runners(self) -> List[Tuple[str, ModelRunner]]:
return [
(f"draft_step_{i}", r)
for i, r in enumerate(self.draft_worker.draft_runners)
]
def __init__(
self,
server_args: ServerArgs,
@@ -158,15 +158,6 @@ class NGRAMWorker(BaseSpecWorker):
self.ngram_corpus.reset()
self._prev_decode_rids = set()
def update_weights_from_tensor(self, recv_req):
# NGRAM has no draft weights of its own — the n-gram corpus is a CPU
# lookup structure built from request token streams — and its
# `model_runner` is shared with the target worker. The scheduler
# mixin dispatches via `self.draft_worker or self.tp_worker`, so
# without this method any caller of `update_weights_from_tensor`
# under `--speculative-algorithm NGRAM` raises AttributeError.
return self.target_worker.update_weights_from_tensor(recv_req)
def add_external_corpus(self, corpus_id: str, token_chunks: list[list[int]]) -> int:
return self.ngram_corpus.load_external_corpus_named(corpus_id, token_chunks)
@@ -786,18 +786,6 @@ class UnoWorkerV2(BaseSpecWorker):
f"UNO expected an EXTEND or DECODE batch, got {batch.forward_mode}."
)
def update_weights_from_disk(self, recv_req):
# The scheduler updates the target worker before calling the spec worker.
return True, "UNO has no separate draft weights."
def update_weights_from_ipc(self, recv_req):
# The scheduler updates the target worker before calling the spec worker.
return True, "UNO has no separate draft weights."
def update_weights_from_tensor(self, recv_req):
# This update route selects the spec worker instead of updating both.
return self.target_worker.update_weights_from_tensor(recv_req)
@contextlib.contextmanager
def _bind_uno_draft_runtime(self):
target_attn_backend = self.model_runner.attn_backend
+66 -35
View File
@@ -1,7 +1,7 @@
import hashlib
import logging
import time
from typing import Any, Callable, Dict, Iterable, NamedTuple, Optional, Set
from typing import Any, Callable, Dict, Iterable, List, NamedTuple, Optional, Set
import torch
import torch.distributed as dist
@@ -26,6 +26,8 @@ class _StrictBaseModel(BaseModel):
class ParallelismInfo(_StrictBaseModel):
# "target", or a draft role such as "draft" / "draft_step_0"
role: str
tp_rank: int
tp_size: int
dp_rank: int
@@ -68,6 +70,23 @@ def _is_non_persistent_buffer_name(name: str) -> bool:
return any(pat in name for pat in _NON_PERSISTENT_BUFFER_PATTERNS)
def _is_skip_weight_check(name, param, skip_tensor_list=None) -> bool:
# one skip set shared by reset / compare / checksum
return (
_is_non_persistent_buffer_name(name)
or getattr(param, "_skip_weight_check", False)
or any(pat in name for pat in (skip_tensor_list or ()))
)
def overall_checksum(checksums: Dict[str, str]) -> str:
h = hashlib.sha256()
for name in sorted(checksums):
h.update(name.encode())
h.update(checksums[name].encode())
return h.hexdigest()
def _padded(nbytes: int, align: int) -> int:
return (nbytes + align - 1) // align * align
@@ -92,31 +111,39 @@ class WeightChecker:
self._get_model = get_model
# Capture the runner placement before its draft scope exits.
parallel = get_parallel()
self._placement = ParallelismInfo(
self._placement = dict(
tp_rank=parallel.tp_rank,
tp_size=parallel.tp_size,
dp_rank=parallel.dp_rank if parallel.dp_rank is not None else 0,
dp_size=parallel.attn_dp_size,
pp_rank=parallel.pp_rank,
pp_size=parallel.pp_size,
rank=0,
size=1,
)
self._snapshot_tensors = None
self._snapshot_arena = None
def handle(self, action: str, allow_quant_error: bool = False) -> Optional[Dict]:
def handle(
self,
action: str,
allow_quant_error: bool = False,
skip_tensor_list: Optional[List[str]] = None,
*,
role: str,
) -> Optional[Dict]:
logger.info(
f"[WeightChecker] handle action={action} allow_quant_error={allow_quant_error}"
f"[WeightChecker] handle action={action} "
f"allow_quant_error={allow_quant_error} skip_tensor_list={skip_tensor_list}"
)
if action == "snapshot":
return self._snapshot()
elif action == "reset_tensors":
return self._reset_tensors()
return self._reset_tensors(skip_tensor_list)
elif action == "compare":
return self._compare(allow_quant_error=allow_quant_error)
return self._compare(
allow_quant_error=allow_quant_error, skip_tensor_list=skip_tensor_list
)
elif action == "checksum":
return self._compute_checksum()
return self._compute_checksum(skip_tensor_list, role=role)
else:
raise Exception(f"Unsupported {action=}")
@@ -138,21 +165,31 @@ class WeightChecker:
self._snapshot_arena = allocator.arena
self._snapshot_tensors = snapshot_tensors
def _reset_tensors(self):
def _skip_compare_names(
self, skip_tensor_list: Optional[List[str]] = None
) -> Set[str]:
return {
name
for name, param in self._model_state()
if _is_skip_weight_check(name, param, skip_tensor_list)
}
def _reset_tensors(self, skip_tensor_list: Optional[List[str]] = None):
for name, param in self._model_state():
if _is_non_persistent_buffer_name(name):
# reset must skip exactly what compare skips
if _is_skip_weight_check(name, param, skip_tensor_list):
continue
param.copy_(_random_like(param))
def _compare(self, allow_quant_error: bool = False):
def _compare(
self,
allow_quant_error: bool = False,
skip_tensor_list: Optional[List[str]] = None,
):
assert self._snapshot_tensors is not None
quantized_set = _build_quantized_set(self._get_model())
skip_compare_names = {
name
for name, param in self._model_state()
if getattr(param, "_skip_weight_check", False)
}
skip_compare_names = self._skip_compare_names(skip_tensor_list)
_check_tensors(
expect_tensors=_build_check_entries(
self._snapshot_tensors, skip_compare_names, quantized_set
@@ -165,16 +202,14 @@ class WeightChecker:
self._snapshot_tensors = None
self._snapshot_arena = None
def _compute_checksum(self) -> Dict:
def _compute_checksum(
self, skip_tensor_list: Optional[List[str]] = None, *, role: str
) -> Dict:
torch.cuda.synchronize()
start = time.perf_counter()
quantized_set = _build_quantized_set(self._get_model())
skip_compare_names = {
name
for name, param in self._model_state()
if getattr(param, "_skip_weight_check", False)
}
skip_compare_names = self._skip_compare_names(skip_tensor_list)
# Hash the dequantized weight so two (qweight, scale) pairs with the same
# bf16 hash equal.
@@ -185,11 +220,7 @@ class WeightChecker:
if should_compare:
checksums[name] = _hash_tensor(comparable.dequantize().data)
h = hashlib.sha256()
for name in sorted(checksums):
h.update(name.encode())
h.update(checksums[name].encode())
overall = h.hexdigest()
overall = overall_checksum(checksums)
torch.cuda.synchronize()
elapsed = time.perf_counter() - start
@@ -200,17 +231,17 @@ class WeightChecker:
info = ChecksumInfo(
checksums=checksums,
per_gpu_checksum=overall,
parallelism_info=self._parallelism_info(),
parallelism_info=self._parallelism_info(role=role),
)
return info.model_dump()
def _parallelism_info(self) -> ParallelismInfo:
def _parallelism_info(self, *, role: str) -> ParallelismInfo:
# Read the current WORLD rank because elastic scale-up can change it.
return self._placement.model_copy(
update={
"rank": dist.get_rank() if dist.is_initialized() else 0,
"size": dist.get_world_size() if dist.is_initialized() else 1,
}
return ParallelismInfo(
role=role,
rank=dist.get_rank() if dist.is_initialized() else 0,
size=dist.get_world_size() if dist.is_initialized() else 1,
**self._placement,
)
def _model_state(self):
+2
View File
@@ -23,6 +23,8 @@ async def update_weights(
"""
Update weights for the inference engine.
This function is designed to be stateless, so that the caller process could keep the stateful engine.
The caller must hold an open weight-update session (engine.begin_weight_update() ...
engine.end_weight_update()) around one or more calls.
Example Use Case:
- Multiple Producer Process will call this function in a SPMD style