diff --git a/docs/docs/advanced_features/sglang_for_rl.mdx b/docs/docs/advanced_features/sglang_for_rl.mdx index 37f242a8e4..1c2c663dd6 100644 --- a/docs/docs/advanced_features/sglang_for_rl.mdx +++ b/docs/docs/advanced_features/sglang_for_rl.mdx @@ -328,6 +328,21 @@ This path trades some I/O overhead for simplicity and flexibility. It integrates > **Note:** The diffusion engine (SGLang-Diffusion) does not currently support hot refit (updating weights while inference is in progress). The diffusion scheduler processes one request at a time and completes the entire inference before handling the next request, so weight updates and inference never run concurrently. +### Weight Update Session + +`update_weights_from_tensor` and `update_weights_from_distributed` must run inside a weight-update session. The update calls only write checkpoint-format values; `end_weight_update` runs every quantized layer's `process_weights_after_loading` once, turning them into kernel layout (Marlin repack, MXFP8 shuffles, ...), and runs `post_load_weights` if no weights were loaded through the engine during the session (P2P/RDMA writes). `begin_weight_update` first restores the few schemes whose kernel layout changes parameter shapes (today compressed-tensors W4A16 MoE on Marlin) so the checkpoint-format values fit. One session can span many update calls, so the finalize cost is paid once per refit rather than once per bucket. A call outside a session, a second `begin_weight_update`, or an `end_weight_update` without one is rejected with `success: false` (HTTP 400) and leaves the weights untouched. + +**Endpoints:** `POST /begin_weight_update`, `POST /end_weight_update` + +`begin_weight_update` accepts `selector`: `"target"`, `"draft"`, or `"all"` (default). Under speculative decoding the same selector on the update endpoints picks which model runners receive the weights; `end_weight_update` finalizes the runners the session opened. + +```python +engine.begin_weight_update() +for bucket in buckets: + engine.update_weights_from_tensor(bucket) +engine.end_weight_update() +``` + ### Update Weights from Tensor **When to use:** @@ -394,7 +409,7 @@ This strategy requires the training process and rollout engine to share access t **Note:** The serialized tensor payloads must be created with `MultiprocessingSerializer.serialize(...)` and should be base64-safe strings. -**Python Engine API:** `engine.update_weights_from_tensor(named_tensors, load_format=None, flush_cache=True)` +**Python Engine API:** `engine.update_weights_from_tensor(named_tensors, load_format=None, flush_cache=True)`, inside `begin_weight_update()` / `end_weight_update()`. ### Update Weights from Distributed Group @@ -575,7 +590,9 @@ Training workers gather weights (typically on TP rank 0), broadcast them to the **Python Engine APIs:** - `engine.init_weights_update_group(...)` +- `engine.begin_weight_update(selector="all")` - `engine.update_weights_from_distributed(names, dtypes, shapes, ...)` +- `engine.end_weight_update()` - `engine.destroy_weights_update_group(group_name)` ### Per-Token Weight Version Attribution diff --git a/python/sglang/srt/entrypoints/EngineBase.py b/python/sglang/srt/entrypoints/EngineBase.py index 06c4f1d240..f4af4df5ea 100644 --- a/python/sglang/srt/entrypoints/EngineBase.py +++ b/python/sglang/srt/entrypoints/EngineBase.py @@ -45,6 +45,16 @@ class EngineBase(ABC): """Flush the cache of the engine.""" pass + @abstractmethod + def begin_weight_update(self, selector: str = "all"): + """Open a weight-update session; update_weights_from_* must run inside one.""" + pass + + @abstractmethod + def end_weight_update(self): + """Close the weight-update session and finalize quantized weights.""" + pass + @abstractmethod def update_weights_from_tensor( self, diff --git a/python/sglang/srt/entrypoints/engine.py b/python/sglang/srt/entrypoints/engine.py index 9b2454f9a8..665eac6f0e 100644 --- a/python/sglang/srt/entrypoints/engine.py +++ b/python/sglang/srt/entrypoints/engine.py @@ -69,9 +69,11 @@ from sglang.srt.managers.data_parallel_controller import ( ) from sglang.srt.managers.detokenizer_manager import run_detokenizer_process from sglang.srt.managers.io_struct import ( + BeginWeightUpdateReqInput, CloseSessionReqInput, DestroyWeightsUpdateGroupReqInput, EmbeddingReqInput, + EndWeightUpdateReqInput, GenerateReqInput, GetWeightsByNameReqInput, InitWeightsUpdateGroupReqInput, @@ -1537,6 +1539,20 @@ class Engine(EngineScoreMixin, EngineBase): self.tokenizer_manager.destroy_weights_update_group(obj, None) ) + def begin_weight_update(self, selector: str = "all"): + """Open a weight-update session; close it with end_weight_update().""" + obj = BeginWeightUpdateReqInput(selector=selector) + return self.loop.run_until_complete( + self.tokenizer_manager.begin_weight_update(obj, None) + ) + + def end_weight_update(self): + """Close the session and finalize quantized weights into kernel layout.""" + obj = EndWeightUpdateReqInput() + return self.loop.run_until_complete( + self.tokenizer_manager.end_weight_update(obj, None) + ) + def update_weights_from_distributed( self, names: list[str], diff --git a/python/sglang/srt/entrypoints/http_server.py b/python/sglang/srt/entrypoints/http_server.py index 0cd1338241..ec0cacf01a 100644 --- a/python/sglang/srt/entrypoints/http_server.py +++ b/python/sglang/srt/entrypoints/http_server.py @@ -117,6 +117,7 @@ from sglang.srt.function_call.function_call_parser import FunctionCallParser from sglang.srt.managers.io_struct import ( AbortReq, AttachHiCacheStorageReqInput, + BeginWeightUpdateReqInput, CheckWeightsReqInput, CloseSessionReqInput, ConfigureLoggingReq, @@ -124,6 +125,7 @@ from sglang.srt.managers.io_struct import ( DestroyWeightsUpdateGroupReqInput, DumperControlReqInput, EmbeddingReqInput, + EndWeightUpdateReqInput, GenerateReqInput, GetWeightsByNameReqInput, InitWeightsSendGroupForRemoteInstanceReqInput, @@ -1368,6 +1370,36 @@ async def update_weights_from_tensor( ) +@app.post("/begin_weight_update") +@auth_level(AuthLevel.ADMIN_OPTIONAL) +async def begin_weight_update( + obj: Annotated[BeginWeightUpdateReqInput, Body()], request: Request +): + """Open a weight-update session so in-place-quantized weights become loadable.""" + success, message = await _global_state.tokenizer_manager.begin_weight_update( + obj, request + ) + return ORJSONResponse( + {"success": success, "message": message}, + status_code=HTTPStatus.OK if success else HTTPStatus.BAD_REQUEST, + ) + + +@app.post("/end_weight_update") +@auth_level(AuthLevel.ADMIN_OPTIONAL) +async def end_weight_update( + obj: Annotated[EndWeightUpdateReqInput, Body()], request: Request +): + """Close the weight-update session and finalize quantized weights.""" + success, message = await _global_state.tokenizer_manager.end_weight_update( + obj, request + ) + return ORJSONResponse( + {"success": success, "message": message}, + status_code=HTTPStatus.OK if success else HTTPStatus.BAD_REQUEST, + ) + + @app.post("/update_weights_from_distributed") @auth_level(AuthLevel.ADMIN_OPTIONAL) async def update_weights_from_distributed( diff --git a/python/sglang/srt/entrypoints/http_server_engine.py b/python/sglang/srt/entrypoints/http_server_engine.py index bc2c3d68b0..1d43344513 100644 --- a/python/sglang/srt/entrypoints/http_server_engine.py +++ b/python/sglang/srt/entrypoints/http_server_engine.py @@ -86,6 +86,12 @@ class HttpServerEngineAdapter(EngineBase): response.raise_for_status() return response.json() + def begin_weight_update(self, selector: str = "all"): + return self._make_request("begin_weight_update", {"selector": selector}) + + def end_weight_update(self): + return self._make_request("end_weight_update") + def update_weights_from_tensor( self, named_tensors: List[Tuple[str, torch.Tensor]], diff --git a/python/sglang/srt/layers/quantization/base_config.py b/python/sglang/srt/layers/quantization/base_config.py index 2877c8eb4d..6ab29d05d6 100644 --- a/python/sglang/srt/layers/quantization/base_config.py +++ b/python/sglang/srt/layers/quantization/base_config.py @@ -46,6 +46,13 @@ class QuantizeMethodBase(ABC): """ return + def restore_weights_before_loading(self, layer: nn.Module) -> None: + """Undo an in-place repack so checkpoint-format weights can be loaded again. + + Needed only when `process_weights_after_loading` changes parameter shapes. + """ + return + class LinearMethodBase(QuantizeMethodBase): """Base class for different (maybe quantized) linear methods.""" diff --git a/python/sglang/srt/layers/quantization/compressed_tensors/compressed_tensors.py b/python/sglang/srt/layers/quantization/compressed_tensors/compressed_tensors.py index 68e85fe3bf..3051ecdfd1 100644 --- a/python/sglang/srt/layers/quantization/compressed_tensors/compressed_tensors.py +++ b/python/sglang/srt/layers/quantization/compressed_tensors/compressed_tensors.py @@ -1252,6 +1252,9 @@ class CompressedTensorsFusedMoEMethod(FusedMoEMethodBase): def process_weights_after_loading(self, layer: torch.nn.Module) -> None: layer.scheme.process_weights_after_loading(layer) + def restore_weights_before_loading(self, layer: torch.nn.Module) -> None: + layer.scheme.restore_weights_before_loading(layer) + def create_weights( self, layer: torch.nn.Module, diff --git a/python/sglang/srt/layers/quantization/compressed_tensors/schemes/compressed_tensors_scheme.py b/python/sglang/srt/layers/quantization/compressed_tensors/schemes/compressed_tensors_scheme.py index 6f9ef04988..4fe32c4fd9 100644 --- a/python/sglang/srt/layers/quantization/compressed_tensors/schemes/compressed_tensors_scheme.py +++ b/python/sglang/srt/layers/quantization/compressed_tensors/schemes/compressed_tensors_scheme.py @@ -99,6 +99,9 @@ class CompressedTensorsMoEScheme(BaseMoEScheme): """ raise NotImplementedError + def restore_weights_before_loading(self, layer: torch.nn.Module) -> None: + return + @abstractmethod def apply_weights( self, diff --git a/python/sglang/srt/managers/io_struct.py b/python/sglang/srt/managers/io_struct.py index 8f5309db2c..c8a4f01a5e 100644 --- a/python/sglang/srt/managers/io_struct.py +++ b/python/sglang/srt/managers/io_struct.py @@ -1875,6 +1875,8 @@ class UpdateWeightsFromDistributedReqInput(BaseReq, kw_only=True): weight_version: Optional[str] = None # Optional format specification for loading load_format: Optional[str] = None + # which runners the op applies to + selector: Literal["target", "draft", "all"] = "all" # Whether to call torch.cuda.empty_cache() during flush torch_empty_cache: bool = False @@ -1901,8 +1903,8 @@ class UpdateWeightsFromTensorReqInput(BaseReq, kw_only=True): abort_all_requests: bool = False # Optional: Update weight version along with weights weight_version: Optional[str] = None - # Optional: Determine whether to disable updating the draft model - disable_draft_model: Optional[bool] = None + # which runners the op applies to + selector: Literal["target", "draft", "all"] = "all" # Whether to call torch.cuda.empty_cache() during flush torch_empty_cache: bool = False @@ -2054,15 +2056,41 @@ class ResumeMemoryOccupationReqOutput(BaseReq, kw_only=True): pass +class BeginWeightUpdateReqInput(BaseReq, kw_only=True): + """Open a weight-update session: restore in-place-packed weights so new ones can load.""" + + selector: Literal["target", "draft", "all"] = "all" + + +class BeginWeightUpdateReqOutput(BaseReq, kw_only=True): + success: bool + message: str + + +class EndWeightUpdateReqInput(BaseReq, kw_only=True): + """Close the weight-update session opened by BeginWeightUpdateReqInput.""" + + +class EndWeightUpdateReqOutput(BaseReq, kw_only=True): + success: bool + message: str + + class CheckWeightsReqInput(BaseReq, kw_only=True): action: str = "checksum" allow_quant_error: bool = False + # Substrings of tensor names to exclude from reset/compare/checksum. + skip_tensor_list: Optional[List[str]] = None + # which runners the op applies to + selector: Literal["target", "draft", "all"] = "all" # Wire versions of the pydantic ParallelismInfo/ChecksumInfo in # sglang.srt.utils.weight_checker. Not array_like: the payload is read by field # name and re-serialized to JSON, so it must stay a {field: value} map. class ParallelismInfo(msgspec.Struct, kw_only=True): + # "target", or a draft role such as "draft" / "draft_step_0" + role: str tp_rank: int tp_size: int dp_rank: int @@ -2076,7 +2104,8 @@ class ParallelismInfo(msgspec.Struct, kw_only=True): class ChecksumInfo(msgspec.Struct, kw_only=True): checksums: Dict[str, str] per_gpu_checksum: str - parallelism_info: ParallelismInfo + # one entry per role (target plus each draft runner); all share the GPU rank + parallelism_info: List[ParallelismInfo] class CheckWeightsReqOutput(BaseReq, kw_only=True): diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index f0d8533e03..0fe158157e 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -126,6 +126,7 @@ from sglang.srt.managers.io_struct import ( AttachHiCacheStorageReqOutput, BatchTokenizedEmbeddingReqInput, BatchTokenizedGenerateReqInput, + BeginWeightUpdateReqInput, CheckWeightsReqInput, ClearHiCacheReqInput, ClearHiCacheReqOutput, @@ -137,6 +138,7 @@ from sglang.srt.managers.io_struct import ( DetachHiCacheStorageReqOutput, DumperControlReqInput, DumperControlReqOutput, + EndWeightUpdateReqInput, ExpertDistributionReq, ExpertDistributionReqOutput, ExpertDistributionReqType, @@ -1738,6 +1740,14 @@ class Scheduler( SendWeightsToRemoteInstanceReqInput, self.send_weights_to_remote_instance, ), + ( + BeginWeightUpdateReqInput, + self.weight_updater.begin_weight_update, + ), + ( + EndWeightUpdateReqInput, + self.weight_updater.end_weight_update, + ), ( UpdateWeightsFromDistributedReqInput, self.weight_updater.update_weights_from_distributed, diff --git a/python/sglang/srt/managers/scheduler_components/weight_updater.py b/python/sglang/srt/managers/scheduler_components/weight_updater.py index adc971baee..d9a1173732 100644 --- a/python/sglang/srt/managers/scheduler_components/weight_updater.py +++ b/python/sglang/srt/managers/scheduler_components/weight_updater.py @@ -1,12 +1,11 @@ from __future__ import annotations -import hashlib import logging import time import traceback from contextlib import contextmanager from dataclasses import dataclass, field -from typing import Any, Callable, Dict, Iterator, Optional, Tuple +from typing import Any, Callable, Dict, Iterator, List, Optional, Set, Tuple import msgspec import torch @@ -19,11 +18,15 @@ from sglang.srt.constants import ( ) from sglang.srt.disaggregation.utils import DisaggregationMode from sglang.srt.managers.io_struct import ( + BeginWeightUpdateReqInput, + BeginWeightUpdateReqOutput, ChecksumInfo, CheckWeightsReqInput, CheckWeightsReqOutput, DestroyWeightsUpdateGroupReqInput, DestroyWeightsUpdateGroupReqOutput, + EndWeightUpdateReqInput, + EndWeightUpdateReqOutput, GetWeightsByNameReqInput, GetWeightsByNameReqOutput, InitWeightsUpdateGroupReqInput, @@ -42,35 +45,43 @@ from sglang.srt.managers.io_struct import ( UpdateWeightsFromTensorReqOutput, ) from sglang.srt.runtime_context import get_model +from sglang.srt.utils.weight_checker import overall_checksum logger = logging.getLogger(__name__) -def _get_draft_model_runner(draft_worker): - # DFlash / FrozenKVMTP workers expose draft_model_runner directly - runner = getattr(draft_worker, "draft_model_runner", None) - if runner is not None: - return runner - # EAGLEWorkerV2: _draft_worker.draft_runner - inner = getattr(draft_worker, "_draft_worker", None) - if inner is not None: - runner = getattr(inner, "draft_runner", None) - if runner is not None: - return runner - return None +def _merge_checksum_payloads(role_payloads: List[Tuple[str, Dict]]) -> Dict: + merged: Dict[str, str] = {} + parallelism_infos = [] + for role, p in role_payloads: + for name, chk in p["checksums"].items(): + # only draft roles are prefixed, so target keys stay stable + key = name if role == "target" else f"{role}.{name}" + if key in merged: + raise ValueError(f"checksum key collision: {key}") + merged[key] = chk + parallelism_infos.append(p["parallelism_info"]) + return { + "checksums": merged, + "per_gpu_checksum": overall_checksum(merged), + "parallelism_info": parallelism_infos, + } -def _merge_checksum_payloads(target: Dict, draft: Dict) -> Dict: - merged_checksums = dict(target["checksums"]) - for name, chk in draft["checksums"].items(): - merged_checksums[f"draft.{name}"] = chk - h = hashlib.sha256() - for name in sorted(merged_checksums): - h.update(name.encode()) - h.update(merged_checksums[name].encode()) - target["checksums"] = merged_checksums - target["per_gpu_checksum"] = h.hexdigest() - return target +def _parse_runner_selector(selector: str) -> Set[str]: + if selector == "all": + return {"target", "draft"} + if selector in ("target", "draft"): + return {selector} + raise ValueError( + f"invalid selector {selector!r}; expected 'target', 'draft', or 'all'" + ) + + +class _WeightUpdateSession(msgspec.Struct, frozen=True): + # recorded at begin so end finalizes the same runners + selector: str + loaded_weights: bool = False @dataclass(kw_only=True, slots=True) @@ -85,6 +96,8 @@ class SchedulerWeightUpdaterManager: metrics_collector: Optional[Any] = None offload_tags: set = field(default_factory=set) stashed_model_static_state: Any = None + # replicated on every TP rank, so a rejected call returns on all ranks before any barrier + _session: Optional[_WeightUpdateSession] = None @contextmanager def _observe_weight_load(self, source: str) -> Iterator[None]: @@ -114,11 +127,19 @@ class SchedulerWeightUpdaterManager: def update_weights_from_disk(self, recv_req: UpdateWeightFromDiskReqInput): """In-place update of the weights from disk.""" with self._observe_weight_load("disk"): - success, message = self.tp_worker.update_weights_from_disk(recv_req) - tp_success = success - if success and self.draft_worker is not None: - success, message = self.draft_worker.update_weights_from_disk(recv_req) - if tp_success: + success, message = True, "Succeeded to update model weights." + target_updated = False + for role, runner in self._select_runners(): + success, message = runner.weight_updater.update_weights_from_disk( + recv_req.model_path, + recv_req.load_format, + recapture_cuda_graph=recv_req.recapture_cuda_graph, + ) + if not success: + break + target_updated |= role == "target" + # the served weights changed even if a draft runner failed afterwards + if target_updated: self.flush_cache_after_weight_update(recv_req) if success: self.record_weight_version_after_update(recv_req.weight_version) @@ -141,31 +162,82 @@ class SchedulerWeightUpdaterManager: success, message = self.tp_worker.destroy_weights_update_group(recv_req) return DestroyWeightsUpdateGroupReqOutput(success=success, message=message) + def _select_runners(self, selector: str = "all") -> List[Tuple[str, Any]]: + roles = _parse_runner_selector(selector) + runners: List[Tuple[str, Any]] = [] + if "target" in roles: + runners += self.tp_worker.weight_update_runners() + if "draft" in roles and self.draft_worker is not None: + runners += self.draft_worker.weight_update_runners() + return runners + def update_weights_from_distributed( self, recv_req: UpdateWeightsFromDistributedReqInput, ) -> Tuple[bool, str]: - """Update the online model parameter.""" + """Update the online model parameter, fanning out to the selected runners.""" + if self._session is None: + return UpdateWeightsFromDistributedReqOutput( + success=False, + message="update_weights_from_distributed must run between " + "begin_weight_update() and end_weight_update()", + ) with self._observe_weight_load("distributed"): - success, message = self.tp_worker.update_weights_from_distributed(recv_req) + # only the target runner joined the update group; drafts load its receive + target = self.tp_worker.model_runner.weight_updater + try: + weights = target.receive_weights_from_distributed( + names=recv_req.names, + dtypes=recv_req.dtypes, + shapes=recv_req.shapes, + group_name=recv_req.group_name, + load_format=recv_req.load_format, + ) + except Exception as e: + success, message = False, f"Failed to receive weights: {e}" + logger.error(message) + else: + success, message = True, "Succeeded to update parameter online." + for _, runner in self._select_runners(recv_req.selector): + success, message = ( + runner.weight_updater.load_weights_from_distributed(weights) + ) + if not success: + break if success: + self._session = msgspec.structs.replace( + self._session, loaded_weights=True + ) self.flush_cache_after_weight_update(recv_req) self.record_weight_version_after_update(recv_req.weight_version) - else: - logger.error(message) return UpdateWeightsFromDistributedReqOutput( success=success, message=message ) def update_weights_from_tensor(self, recv_req: UpdateWeightsFromTensorReqInput): - """Update the online model parameter from tensors.""" + """Update the online model parameter from tensors on the selected runners.""" + if self._session is None: + return UpdateWeightsFromTensorReqOutput( + success=False, + message="update_weights_from_tensor must run between " + "begin_weight_update() and end_weight_update()", + ) with self._observe_weight_load("tensor"): - if recv_req.disable_draft_model: - worker = self.tp_worker - else: - worker = self.draft_worker or self.tp_worker - success, message = worker.update_weights_from_tensor(recv_req) + named_tensors = self.tp_worker.deserialize_own_rank( + recv_req.serialized_named_tensors + ) + success, message = True, "Success" + for _, runner in self._select_runners(recv_req.selector): + success, message = runner.weight_updater.update_weights_from_tensor( + named_tensors=named_tensors, + load_format=recv_req.load_format, + ) + if not success: + break if success: + self._session = msgspec.structs.replace( + self._session, loaded_weights=True + ) self.flush_cache_after_weight_update(recv_req) self.record_weight_version_after_update(recv_req.weight_version) else: @@ -176,11 +248,17 @@ class SchedulerWeightUpdaterManager: def update_weights_from_ipc(self, recv_req: UpdateWeightsFromIPCReqInput): """Update the online model parameter from IPC for checkpoint-engine integration.""" with self._observe_weight_load("ipc"): - success, message = self.tp_worker.update_weights_from_ipc(recv_req) - tp_success = success - if success and self.draft_worker is not None: - success, message = self.draft_worker.update_weights_from_ipc(recv_req) - if tp_success: + success, message = True, "Succeeded to update model weights." + target_updated = False + for role, runner in self._select_runners(): + success, message = runner.weight_updater.update_weights_from_ipc( + recv_req + ) + if not success: + break + target_updated |= role == "target" + # the served weights changed even if a draft runner failed afterwards + if target_updated: self.flush_cache_after_weight_update(recv_req) if success: self.record_weight_version_after_update(recv_req.weight_version) @@ -209,6 +287,34 @@ class SchedulerWeightUpdaterManager: f"Restart with --weight-cache-mode off to use this operation." ) + def begin_weight_update(self, recv_req: BeginWeightUpdateReqInput): + """Open the session: restore in-place-packed weights on the selected runners.""" + if self._session is not None: + return BeginWeightUpdateReqOutput( + success=False, + message="a weight-update session is already open; " + "call end_weight_update() first", + ) + for _, runner in self._select_runners(recv_req.selector): + runner.weight_updater.begin_weight_update() + self._session = _WeightUpdateSession(selector=recv_req.selector) + torch.distributed.barrier(group=self.tp_cpu_group) + return BeginWeightUpdateReqOutput(success=True, message="Success") + + def end_weight_update(self, recv_req: EndWeightUpdateReqInput): + """Finalize the runners begin opened; post_load_weights only if no load ran (P2P/RDMA).""" + if self._session is None: + return EndWeightUpdateReqOutput( + success=False, + message="no weight-update session is open; call begin_weight_update() first", + ) + run_post_load = not self._session.loaded_weights + for _, runner in self._select_runners(self._session.selector): + runner.weight_updater.end_weight_update(run_post_load=run_post_load) + self._session = None + torch.distributed.barrier(group=self.tp_cpu_group) + return EndWeightUpdateReqOutput(success=True, message="Success") + def release_memory_occupation(self, recv_req: ReleaseMemoryOccupationReqInput): scheduler = self.scheduler assert self.is_fully_idle( @@ -298,19 +404,17 @@ class SchedulerWeightUpdaterManager: def check_weights(self, recv_req: CheckWeightsReqInput): try: - payload = self.tp_worker.model_runner.check_weights( - action=recv_req.action, allow_quant_error=recv_req.allow_quant_error - ) - - if self.draft_worker is not None: - draft_runner = _get_draft_model_runner(self.draft_worker) - if draft_runner is not None: - draft_payload = draft_runner.check_weights( - action=recv_req.action, - allow_quant_error=recv_req.allow_quant_error, - ) - if payload is not None and draft_payload is not None: - payload = _merge_checksum_payloads(payload, draft_payload) + role_payloads = [] + for role, runner in self._select_runners(recv_req.selector): + p = runner.check_weights( + action=recv_req.action, + allow_quant_error=recv_req.allow_quant_error, + skip_tensor_list=recv_req.skip_tensor_list, + role=role, + ) + if p is not None: + role_payloads.append((role, p)) + payload = _merge_checksum_payloads(role_payloads) if role_payloads else None tp_size = torch.distributed.get_world_size(group=self.tp_cpu_group) if tp_size > 1 and payload is not None: diff --git a/python/sglang/srt/managers/tokenizer_control_mixin.py b/python/sglang/srt/managers/tokenizer_control_mixin.py index 0170ac34ea..3d27e17fe3 100644 --- a/python/sglang/srt/managers/tokenizer_control_mixin.py +++ b/python/sglang/srt/managers/tokenizer_control_mixin.py @@ -16,6 +16,8 @@ from sglang.srt.managers.io_struct import ( AddExternalCorpusReqOutput, AttachHiCacheStorageReqInput, AttachHiCacheStorageReqOutput, + BeginWeightUpdateReqInput, + BeginWeightUpdateReqOutput, ChecksumInfo, CheckWeightsReqInput, CheckWeightsReqOutput, @@ -28,6 +30,8 @@ from sglang.srt.managers.io_struct import ( DetachHiCacheStorageReqOutput, DumperControlReqInput, DumperControlReqOutput, + EndWeightUpdateReqInput, + EndWeightUpdateReqOutput, ExpertDistributionReq, ExpertDistributionReqOutput, ExpertDistributionReqType, @@ -131,6 +135,8 @@ _COMMUNICATOR_SPECS = [ ("get_internal_state", GetInternalStateReqOutput), ("set_internal_state", SetInternalStateReqOutput), ("expert_distribution", ExpertDistributionReqOutput), + ("begin_weight_update", BeginWeightUpdateReqOutput), + ("end_weight_update", EndWeightUpdateReqOutput), ("update_lora_adapter", LoRAUpdateOutput), ("dumper_control", DumperControlReqOutput), ("scale_elastic_ep", ScaleElasticEPReqOutput), @@ -449,6 +455,38 @@ class TokenizerControlMixin: results = await self.destroy_weights_update_group_communicator(obj) return FanOutCommunicator.merge_results(results) + async def _weight_update_session_call( + self: TokenizerManager, communicator, obj + ) -> Tuple[bool, str]: + self.auto_create_handle_loop() + async with self.is_pause_cond: + is_paused = self.is_pause + # whoever paused the engine holds the writer lock; taking it again deadlocks + if is_paused: + results = await communicator(obj) + if not is_paused: + async with self.model_update_lock.writer_lock: + results = await communicator(obj) + return FanOutCommunicator.merge_results(results) + + async def begin_weight_update( + self: TokenizerManager, + obj: BeginWeightUpdateReqInput, + request: Optional[fastapi.Request] = None, + ) -> Tuple[bool, str]: + return await self._weight_update_session_call( + self.begin_weight_update_communicator, obj + ) + + async def end_weight_update( + self: TokenizerManager, + obj: EndWeightUpdateReqInput, + request: Optional[fastapi.Request] = None, + ) -> Tuple[bool, str]: + return await self._weight_update_session_call( + self.end_weight_update_communicator, obj + ) + async def update_weights_from_distributed( self: TokenizerManager, obj: UpdateWeightsFromDistributedReqInput, diff --git a/python/sglang/srt/managers/tp_worker.py b/python/sglang/srt/managers/tp_worker.py index 9fa3a10993..477ae90576 100644 --- a/python/sglang/srt/managers/tp_worker.py +++ b/python/sglang/srt/managers/tp_worker.py @@ -32,10 +32,6 @@ from sglang.srt.managers.io_struct import ( LoadLoRAAdapterReqInput, SendWeightsToRemoteInstanceReqInput, UnloadLoRAAdapterReqInput, - UpdateWeightFromDiskReqInput, - UpdateWeightsFromDistributedReqInput, - UpdateWeightsFromIPCReqInput, - UpdateWeightsFromTensorReqInput, ) from sglang.srt.managers.schedule_batch import ScheduleBatch from sglang.srt.managers.scheduler import GenerationBatchResult @@ -146,14 +142,6 @@ class BaseTpWorker(ABC): self.model_runner.token_to_kv_pool_allocator, ) - def update_weights_from_disk(self, recv_req: UpdateWeightFromDiskReqInput): - success, message = self.model_runner.weight_updater.update_weights_from_disk( - recv_req.model_path, - recv_req.load_format, - recapture_cuda_graph=recv_req.recapture_cuda_graph, - ) - return success, message - def init_weights_update_group(self, recv_req: InitWeightsUpdateGroupReqInput): success, message = self.model_runner.weight_updater.init_weights_update_group( recv_req.master_address, @@ -200,21 +188,7 @@ class BaseTpWorker(ABC): ) return success, message - def update_weights_from_distributed( - self, recv_req: UpdateWeightsFromDistributedReqInput - ): - success, message = ( - self.model_runner.weight_updater.update_weights_from_distributed( - recv_req.names, - recv_req.dtypes, - recv_req.shapes, - recv_req.group_name, - recv_req.load_format, - ) - ) - return success, message - - def _deserialize_own_rank(self, serialized_named_tensors): + def deserialize_own_rank(self, serialized_named_tensors): """Each rank deserializes only its own payload (index tp_rank); deserializing another rank's copy would break producer-side CUDA-IPC refcounting.""" @@ -223,20 +197,6 @@ class BaseTpWorker(ABC): serialized_named_tensors[self.model_runner.tp_rank] ) - def update_weights_from_tensor(self, recv_req: UpdateWeightsFromTensorReqInput): - success, message = self.model_runner.weight_updater.update_weights_from_tensor( - named_tensors=self._deserialize_own_rank(recv_req.serialized_named_tensors), - load_format=recv_req.load_format, - ) - return success, message - - def update_weights_from_ipc(self, recv_req: UpdateWeightsFromIPCReqInput): - """Update weights from IPC for checkpoint-engine integration.""" - success, message = self.model_runner.weight_updater.update_weights_from_ipc( - recv_req - ) - return success, message - def get_weights_by_name(self, recv_req: GetWeightsByNameReqInput): parameter = self.model_runner.weight_exporter.get_weights_by_name( recv_req.name, recv_req.truncate_size @@ -256,7 +216,7 @@ class BaseTpWorker(ABC): ): # The LoRA code handles TP sharding internally using slice_lora_a_weights # and slice_lora_b_weights methods (see lora/layers.py and mem_pool.py). - data = self._deserialize_own_rank(recv_req.serialized_named_tensors) + data = self.deserialize_own_rank(recv_req.serialized_named_tensors) if recv_req.load_format == "flattened_bucket": bucket = FlattenedTensorBucket( flattened_tensor=data["flattened_tensor"], @@ -562,6 +522,10 @@ class TpModelWorker(BaseTpWorker): def model_runner(self) -> ModelRunner: return self._model_runner + def weight_update_runners(self) -> List[Tuple[str, ModelRunner]]: + """(role, runner) pairs weight ops apply to; the target worker owns one.""" + return [("target", self._model_runner)] + def register_hicache_layer_transfer_counter(self, counter: LayerDoneCounter): self.hicache_layer_transfer_counter = counter diff --git a/python/sglang/srt/model_executor/model_runner.py b/python/sglang/srt/model_executor/model_runner.py index c3085ee01a..64f92832a2 100644 --- a/python/sglang/srt/model_executor/model_runner.py +++ b/python/sglang/srt/model_executor/model_runner.py @@ -20,7 +20,7 @@ import inspect import logging import time from dataclasses import dataclass -from typing import Callable, Optional, Union +from typing import Callable, List, Optional, Union import torch import torch.distributed as dist @@ -2048,9 +2048,19 @@ class ModelRunner: forward_batch.token_ids_logprobs, ) - def check_weights(self, action: str, allow_quant_error: bool = False): + def check_weights( + self, + action: str, + allow_quant_error: bool = False, + skip_tensor_list: Optional[List[str]] = None, + *, + role: str, + ): return self._weight_checker.handle( - action=action, allow_quant_error=allow_quant_error + action=action, + allow_quant_error=allow_quant_error, + skip_tensor_list=skip_tensor_list, + role=role, ) def _expand_eplb_metadata_for_scale( diff --git a/python/sglang/srt/model_executor/model_runner_components/weight_updater.py b/python/sglang/srt/model_executor/model_runner_components/weight_updater.py index 37c38ad5b8..0e0d8db22e 100644 --- a/python/sglang/srt/model_executor/model_runner_components/weight_updater.py +++ b/python/sglang/srt/model_executor/model_runner_components/weight_updater.py @@ -8,7 +8,11 @@ from typing import TYPE_CHECKING, Any, Callable, List, Optional, Tuple, Union import torch from sglang.srt.configs.load_config import LoadConfig -from sglang.srt.model_loader.loader import DefaultModelLoader, get_model_loader +from sglang.srt.model_loader.loader import ( + DefaultModelLoader, + get_model_loader, + post_load_weights, +) from sglang.srt.model_loader.utils import set_default_torch_dtype from sglang.srt.model_loader.weight_utils import default_weight_loader from sglang.srt.platforms import current_platform @@ -241,94 +245,87 @@ class WeightUpdater: logger.info("Update weights end.") return True, "Succeeded to update model weights." - def update_weights_from_distributed( + def receive_weights_from_distributed( self: WeightUpdater, + *, names, dtypes, shapes, group_name, load_format: Optional[str] = None, ): - """ - Update specific parameter in the model weights online - through `_model_update_group` process group. - - Args: - name: the name of the parameter to be updated. - dtype: the data type of the parameter to be updated. - shape: the shape of the parameter to be updated. - """ - self._assert_weight_cache_inactive("update_weights_from_distributed") - error = _unsupported_derived_weight_cache_error(self.get_model()) - if error is not None: - return False, error - + """Receive one broadcast without loading it; only the target runner joined the group.""" assert group_name in self._model_update_group, ( f"Group {group_name} not in {list(self._model_update_group.keys())}. " "Please call `init_weights_update_group` first." ) if load_format == "flattened_bucket": - return self._update_bucketed_weights_from_distributed( - names, dtypes, shapes, group_name + return self._receive_bucketed_weights_from_distributed( + names=names, dtypes=dtypes, shapes=shapes, group_name=group_name ) - try: - weights = [] - handles = [] - for name, dtype, shape in zip(names, dtypes, shapes): - target_dtype = ( - dtype if isinstance(dtype, torch.dtype) else getattr(torch, dtype) - ) - weight = torch.empty(shape, dtype=target_dtype, device=self.device) - handles.append( - torch.distributed.broadcast( - weight, - src=0, - group=self._model_update_group[group_name], - async_op=True, - ) - ) - weights.append((name, weight)) - for handle in handles: - handle.wait() - self.get_model().load_weights(weights) - return True, "Succeeded to update parameter online." - - except Exception as e: - error_msg = ( - f"Failed to update parameter online: {e}. " - f"The full weights of the ModelRunner are partially updated. " - f"Please discard the whole weights." + weights = [] + handles = [] + for name, dtype, shape in zip(names, dtypes, shapes): + target_dtype = ( + dtype if isinstance(dtype, torch.dtype) else getattr(torch, dtype) ) - logger.error(error_msg) - return False, error_msg + weight = torch.empty(shape, dtype=target_dtype, device=self.device) + handles.append( + torch.distributed.broadcast( + weight, + src=0, + group=self._model_update_group[group_name], + async_op=True, + ) + ) + weights.append((name, weight)) + for handle in handles: + handle.wait() + return weights - def _update_bucketed_weights_from_distributed( - self: WeightUpdater, names, dtypes, shapes, group_name + def _receive_bucketed_weights_from_distributed( + self: WeightUpdater, *, names, dtypes, shapes, group_name ): - try: - named_tensors = [] - for name, dtype, shape in zip(names, dtypes, shapes): - target_dtype = ( - dtype if isinstance(dtype, torch.dtype) else getattr(torch, dtype) - ) - named_tensors.append( - ( - name, - torch.empty(shape, dtype=target_dtype, device=self.device), - ) - ) - bucket = FlattenedTensorBucket(named_tensors=named_tensors) - flattened_tensor = bucket.get_flattened_tensor() - torch.distributed.broadcast( - flattened_tensor, - src=0, - group=self._model_update_group[group_name], + named_tensors = [] + for name, dtype, shape in zip(names, dtypes, shapes): + target_dtype = ( + dtype if isinstance(dtype, torch.dtype) else getattr(torch, dtype) ) - reconstructed_tensors = bucket.reconstruct_tensors() - self.get_model().load_weights(reconstructed_tensors) - return True, "Succeeded to update parameter online." + named_tensors.append( + (name, torch.empty(shape, dtype=target_dtype, device=self.device)) + ) + bucket = FlattenedTensorBucket(named_tensors=named_tensors) + flattened_tensor = bucket.get_flattened_tensor() + torch.distributed.broadcast( + flattened_tensor, + src=0, + group=self._model_update_group[group_name], + ) + return bucket.reconstruct_tensors() + + def begin_weight_update(self: WeightUpdater) -> None: + DefaultModelLoader.restore_weights_before_loading( + self.get_model(), torch.device(self.device) + ) + + def end_weight_update(self: WeightUpdater, *, run_post_load: bool) -> None: + if run_post_load: + post_load_weights(self.get_model()) + DefaultModelLoader.postprocess_weights( + self.get_model(), torch.device(self.device) + ) + + def load_weights_from_distributed( + self: WeightUpdater, named_tensors: List[Tuple[str, torch.Tensor]] + ) -> Tuple[bool, str]: + self._assert_weight_cache_inactive("update_weights_from_distributed") + error = _unsupported_derived_weight_cache_error(self.get_model()) + if error is not None: + return False, error + try: + self.get_model().load_weights(named_tensors) except Exception as e: error_msg = ( f"Failed to update parameter online: {e}. " @@ -337,6 +334,7 @@ class WeightUpdater: ) logger.error(error_msg) return False, error_msg + return True, "Succeeded to update parameter online." def update_weights_from_tensor( self: WeightUpdater, diff --git a/python/sglang/srt/model_loader/loader.py b/python/sglang/srt/model_loader/loader.py index 31f26a33b1..788ac19b8c 100644 --- a/python/sglang/srt/model_loader/loader.py +++ b/python/sglang/srt/model_loader/loader.py @@ -87,7 +87,10 @@ from sglang.srt.layers.modelopt_utils import QUANT_CFG_CHOICES from sglang.srt.layers.moe.utils import ( install_shared_experts_fusion_decision, ) -from sglang.srt.layers.quantization.base_config import QuantizationConfig +from sglang.srt.layers.quantization.base_config import ( + QuantizationConfig, + QuantizeMethodBase, +) from sglang.srt.model_loader.remote_instance_weight_loader_utils import ( trigger_transferring_weights_request, ) @@ -304,7 +307,7 @@ def _initialize_model( return model_class(**kwargs) -def _post_load_weights(model: nn.Module) -> None: +def post_load_weights(model: nn.Module) -> None: # Loaders that bypass `model.load_weights()` (dummy / sharded state / remote instance / # remote fs) must trigger the model's post-load fixup explicitly; `model.load_weights()` # would normally do it internally. NextN subclasses override the method to fill in @@ -313,6 +316,18 @@ def _post_load_weights(model: nn.Module) -> None: model.post_load_weights() +def _modules_with_quant_method(model: nn.Module): + from sglang.srt.lora.layers import BaseLayerWithLoRA + + for _, module in model.named_modules(): + # LoRA wrappers forward quant_method but do not own the packed params + if isinstance(module, BaseLayerWithLoRA): + continue + quant_method = getattr(module, "quant_method", None) + if quant_method is not None: + yield module, quant_method + + class BaseModelLoader(ABC): """Base class for model loaders.""" @@ -836,7 +851,7 @@ class DefaultModelLoader(BaseModelLoader): """ with set_default_torch_dtype(model_config.dtype): initialize_capture_safe_weights(model) - _post_load_weights(model) + post_load_weights(model) for _, module in model.named_modules(): quant_method = getattr(module, "quant_method", None) if quant_method is None: @@ -1056,16 +1071,23 @@ class DefaultModelLoader(BaseModelLoader): @staticmethod def postprocess_weights(model, target_device): - for _, module in model.named_modules(): - quant_method = getattr(module, "quant_method", None) - if quant_method is not None: - # When quant methods need to process weights after loading - # (for repacking, quantizing, etc), they expect parameters - # to be on the global target device. This scope is for the - # case where cpu offloading is used, where we will move the - # parameters onto device for processing and back off after. + for module, quant_method in _modules_with_quant_method(model): + # When quant methods need to process weights after loading + # (for repacking, quantizing, etc), they expect parameters + # to be on the global target device. This scope is for the + # case where cpu offloading is used, where we will move the + # parameters onto device for processing and back off after. + with device_loading_context(module, target_device): + quant_method.process_weights_after_loading(module) + + @staticmethod + def restore_weights_before_loading(model, target_device): + """Undo in-place quant packing so fresh weights can be loaded.""" + for module, quant_method in _modules_with_quant_method(model): + # AMX packing and the MXFP4 backend wrappers are duck-typed and cannot restore + if isinstance(quant_method, QuantizeMethodBase): with device_loading_context(module, target_device): - quant_method.process_weights_after_loading(module) + quant_method.restore_weights_before_loading(module) class LayeredModelLoader(DefaultModelLoader): @@ -1643,7 +1665,7 @@ class DummyModelLoader(BaseModelLoader): # random values to the weights. initialize_dummy_weights(model) - _post_load_weights(model) + post_load_weights(model) for _, module in model.named_modules(): quant_method = getattr(module, "quant_method", None) @@ -1804,7 +1826,7 @@ class ShardedStateLoader(BaseModelLoader): if state_dict: raise ValueError(f"Missing keys {tuple(state_dict)} in loaded state!") - _post_load_weights(model) + post_load_weights(model) return model.eval() @@ -3426,7 +3448,7 @@ class RemoteInstanceModelLoader(BaseModelLoader): ) current_platform.synchronize() - _post_load_weights(model) + post_load_weights(model) end_get_weights_tic = time.time() logger.debug( f"finish getting all weights from remote instance, time used: {(end_get_weights_tic - start_get_weights_tic):.4f}s" @@ -3489,7 +3511,7 @@ class RemoteInstanceModelLoader(BaseModelLoader): logger.error(f"batch transfer failed, error: {ret}") return False - _post_load_weights(model) + post_load_weights(model) return True @@ -3579,7 +3601,7 @@ class RemoteModelLoader(BaseModelLoader): if state_dict: raise ValueError(f"Missing keys {tuple(state_dict)} in loaded state!") - _post_load_weights(model) + post_load_weights(model) def _load_model_from_remote_fs( self, model, client, model_config: ModelConfig, device_config: DeviceConfig diff --git a/python/sglang/srt/speculative/base_spec_worker.py b/python/sglang/srt/speculative/base_spec_worker.py index 2e9a9fafc5..d8bab5c428 100644 --- a/python/sglang/srt/speculative/base_spec_worker.py +++ b/python/sglang/srt/speculative/base_spec_worker.py @@ -14,10 +14,6 @@ from sglang.srt.model_executor.graph_memory_usage import ( from sglang.srt.runtime_context import get_disagg, get_exec, get_memory, get_schedule if TYPE_CHECKING: - from sglang.srt.managers.io_struct import ( - UpdateWeightFromDiskReqInput, - UpdateWeightsFromIPCReqInput, - ) from sglang.srt.managers.tp_worker import TpModelWorker from sglang.srt.model_executor.model_runner import ( ModelRunner, @@ -162,6 +158,13 @@ class BaseSpecWorker(ABC): def hicache_draft_plan(self) -> HiCacheDraftPlan: return self._hicache_draft_plan + def weight_update_runners(self) -> list[tuple[str, ModelRunner]]: + """(role, runner) pairs weight ops apply to; [] when the draft owns no weights.""" + # NGRAM and UNO have no draft model + if self.draft_worker is None: + return [] + return [("draft", runner) for runner in self.draft_worker.draft_runners] + def _draft_model_runners(self) -> tuple[ModelRunner, ...]: spec_algorithm = self.target_worker.model_runner.spec_algorithm draft_worker = self.draft_worker @@ -315,24 +318,6 @@ class BaseSpecWorker(ABC): if self.draft_worker is not None: self.draft_worker.init_cuda_graphs() - def update_weights_from_disk(self, recv_req: UpdateWeightFromDiskReqInput): - for runner in self.draft_worker.draft_runners: - success, message = runner.weight_updater.update_weights_from_disk( - recv_req.model_path, - recv_req.load_format, - recapture_cuda_graph=recv_req.recapture_cuda_graph, - ) - if not success: - return success, message - return True, "Succeeded to update model weights." - - def update_weights_from_ipc(self, recv_req: UpdateWeightsFromIPCReqInput): - for runner in self.draft_worker.draft_runners: - success, message = runner.weight_updater.update_weights_from_ipc(recv_req) - if not success: - return success, message - return True, "Succeeded to update model weights." - def on_verify_complete_cpu( self, num_correct_drafts_per_req: list[int], batch_size: int = 0 ) -> None: diff --git a/python/sglang/srt/speculative/dflash_worker_v2.py b/python/sglang/srt/speculative/dflash_worker_v2.py index c55007f3df..b16f024d1b 100644 --- a/python/sglang/srt/speculative/dflash_worker_v2.py +++ b/python/sglang/srt/speculative/dflash_worker_v2.py @@ -1,7 +1,7 @@ import logging import math import os -from typing import List, Optional, Tuple +from typing import TYPE_CHECKING, List, Optional, Tuple import torch import torch.distributed as dist @@ -92,6 +92,9 @@ from sglang.srt.utils.common import empty_context _is_npu = is_npu() +if TYPE_CHECKING: + from sglang.srt.model_executor.model_runner import ModelRunner + logger = logging.getLogger(__name__) _FusedKVMaterializeHelper = None @@ -358,6 +361,9 @@ class DFlashWorkerV2(BaseSpecWorker): scheduler runs it synchronously when overlap is disabled. """ + def weight_update_runners(self) -> List[Tuple[str, "ModelRunner"]]: + return [("draft", self.draft_model_runner)] + def __init__( self, server_args: ServerArgs, diff --git a/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py b/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py index f359a900e5..e40b4afaf1 100644 --- a/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py +++ b/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py @@ -1,6 +1,6 @@ import logging from contextlib import nullcontext -from typing import Callable, Optional, Protocol, runtime_checkable +from typing import TYPE_CHECKING, Callable, Optional, Protocol, runtime_checkable import torch @@ -84,6 +84,9 @@ from sglang.srt.utils import ( is_pin_memory_available, ) +if TYPE_CHECKING: + from sglang.srt.model_executor.model_runner import ModelRunner + logger = logging.getLogger(__name__) _is_npu = is_npu() @@ -127,6 +130,9 @@ def _configure_target_hidden_projection( class DSparkWorkerV2(BaseSpecWorker): """Non-last PP stages run only the target; draft state belongs to the last stage.""" + def weight_update_runners(self) -> list[tuple[str, "ModelRunner"]]: + return [("draft", self.draft_model_runner)] if self._hosts_draft else [] + def __init__( self, server_args: ServerArgs, diff --git a/python/sglang/srt/speculative/eagle_worker_v2.py b/python/sglang/srt/speculative/eagle_worker_v2.py index 8fe05442c3..bbe5e4733a 100644 --- a/python/sglang/srt/speculative/eagle_worker_v2.py +++ b/python/sglang/srt/speculative/eagle_worker_v2.py @@ -38,7 +38,6 @@ from sglang.srt.layers.moe.utils import ( speculative_moe_a2a_backend_context, speculative_moe_backend_context, ) -from sglang.srt.managers.io_struct import UpdateWeightsFromTensorReqInput from sglang.srt.managers.schedule_batch import ScheduleBatch from sglang.srt.managers.scheduler import GenerationBatchResult from sglang.srt.managers.tp_worker import TpModelWorker @@ -117,7 +116,6 @@ from sglang.srt.utils.async_probe import ( maybe_detect_oob, ) from sglang.srt.utils.common import ( - MultiprocessingSerializer, empty_context, fast_topk, get_available_gpu_memory, @@ -129,7 +127,6 @@ from sglang.srt.utils.common import ( is_xpu, log_info_on_rank0, ) -from sglang.srt.utils.patch_torch import monkey_patch_torch_reductions _is_cpu = is_cpu() _is_npu = is_npu() @@ -1868,25 +1865,3 @@ class EAGLEWorkerV2(BaseSpecWorker): finalize_tree_path=True, grammar_barrier=grammar_barrier, ) - - def update_weights_from_tensor(self, recv_req: UpdateWeightsFromTensorReqInput): - monkey_patch_torch_reductions() - named_tensors = MultiprocessingSerializer.deserialize( - recv_req.serialized_named_tensors[self.model_runner.tp_rank] - ) - success, message = ( - self.draft_worker.draft_runner.weight_updater.update_weights_from_tensor( - named_tensors=named_tensors, - load_format=recv_req.load_format, - ) - ) - if not success: - return success, message - - success, message = ( - self.target_worker.model_runner.weight_updater.update_weights_from_tensor( - named_tensors=named_tensors, - load_format=recv_req.load_format, - ) - ) - return success, message diff --git a/python/sglang/srt/speculative/multi_layer_eagle_worker_v2.py b/python/sglang/srt/speculative/multi_layer_eagle_worker_v2.py index 1a9d74be50..79a24f79c7 100644 --- a/python/sglang/srt/speculative/multi_layer_eagle_worker_v2.py +++ b/python/sglang/srt/speculative/multi_layer_eagle_worker_v2.py @@ -17,7 +17,7 @@ from __future__ import annotations import logging import time from dataclasses import replace -from typing import TYPE_CHECKING, List +from typing import TYPE_CHECKING, List, Tuple import torch @@ -1001,6 +1001,12 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase): class MultiLayerEagleWorkerV2(BaseSpecWorker): + def weight_update_runners(self) -> List[Tuple[str, ModelRunner]]: + return [ + (f"draft_step_{i}", r) + for i, r in enumerate(self.draft_worker.draft_runners) + ] + def __init__( self, server_args: ServerArgs, diff --git a/python/sglang/srt/speculative/ngram_worker.py b/python/sglang/srt/speculative/ngram_worker.py index 109d5cfbf7..c31157acb5 100644 --- a/python/sglang/srt/speculative/ngram_worker.py +++ b/python/sglang/srt/speculative/ngram_worker.py @@ -158,15 +158,6 @@ class NGRAMWorker(BaseSpecWorker): self.ngram_corpus.reset() self._prev_decode_rids = set() - def update_weights_from_tensor(self, recv_req): - # NGRAM has no draft weights of its own — the n-gram corpus is a CPU - # lookup structure built from request token streams — and its - # `model_runner` is shared with the target worker. The scheduler - # mixin dispatches via `self.draft_worker or self.tp_worker`, so - # without this method any caller of `update_weights_from_tensor` - # under `--speculative-algorithm NGRAM` raises AttributeError. - return self.target_worker.update_weights_from_tensor(recv_req) - def add_external_corpus(self, corpus_id: str, token_chunks: list[list[int]]) -> int: return self.ngram_corpus.load_external_corpus_named(corpus_id, token_chunks) diff --git a/python/sglang/srt/speculative/uno_worker_v2.py b/python/sglang/srt/speculative/uno_worker_v2.py index ebd4deba36..472de85cec 100644 --- a/python/sglang/srt/speculative/uno_worker_v2.py +++ b/python/sglang/srt/speculative/uno_worker_v2.py @@ -786,18 +786,6 @@ class UnoWorkerV2(BaseSpecWorker): f"UNO expected an EXTEND or DECODE batch, got {batch.forward_mode}." ) - def update_weights_from_disk(self, recv_req): - # The scheduler updates the target worker before calling the spec worker. - return True, "UNO has no separate draft weights." - - def update_weights_from_ipc(self, recv_req): - # The scheduler updates the target worker before calling the spec worker. - return True, "UNO has no separate draft weights." - - def update_weights_from_tensor(self, recv_req): - # This update route selects the spec worker instead of updating both. - return self.target_worker.update_weights_from_tensor(recv_req) - @contextlib.contextmanager def _bind_uno_draft_runtime(self): target_attn_backend = self.model_runner.attn_backend diff --git a/python/sglang/srt/utils/weight_checker.py b/python/sglang/srt/utils/weight_checker.py index 3755c1e725..0fdacbca0a 100644 --- a/python/sglang/srt/utils/weight_checker.py +++ b/python/sglang/srt/utils/weight_checker.py @@ -1,7 +1,7 @@ import hashlib import logging import time -from typing import Any, Callable, Dict, Iterable, NamedTuple, Optional, Set +from typing import Any, Callable, Dict, Iterable, List, NamedTuple, Optional, Set import torch import torch.distributed as dist @@ -26,6 +26,8 @@ class _StrictBaseModel(BaseModel): class ParallelismInfo(_StrictBaseModel): + # "target", or a draft role such as "draft" / "draft_step_0" + role: str tp_rank: int tp_size: int dp_rank: int @@ -68,6 +70,23 @@ def _is_non_persistent_buffer_name(name: str) -> bool: return any(pat in name for pat in _NON_PERSISTENT_BUFFER_PATTERNS) +def _is_skip_weight_check(name, param, skip_tensor_list=None) -> bool: + # one skip set shared by reset / compare / checksum + return ( + _is_non_persistent_buffer_name(name) + or getattr(param, "_skip_weight_check", False) + or any(pat in name for pat in (skip_tensor_list or ())) + ) + + +def overall_checksum(checksums: Dict[str, str]) -> str: + h = hashlib.sha256() + for name in sorted(checksums): + h.update(name.encode()) + h.update(checksums[name].encode()) + return h.hexdigest() + + def _padded(nbytes: int, align: int) -> int: return (nbytes + align - 1) // align * align @@ -92,31 +111,39 @@ class WeightChecker: self._get_model = get_model # Capture the runner placement before its draft scope exits. parallel = get_parallel() - self._placement = ParallelismInfo( + self._placement = dict( tp_rank=parallel.tp_rank, tp_size=parallel.tp_size, dp_rank=parallel.dp_rank if parallel.dp_rank is not None else 0, dp_size=parallel.attn_dp_size, pp_rank=parallel.pp_rank, pp_size=parallel.pp_size, - rank=0, - size=1, ) self._snapshot_tensors = None self._snapshot_arena = None - def handle(self, action: str, allow_quant_error: bool = False) -> Optional[Dict]: + def handle( + self, + action: str, + allow_quant_error: bool = False, + skip_tensor_list: Optional[List[str]] = None, + *, + role: str, + ) -> Optional[Dict]: logger.info( - f"[WeightChecker] handle action={action} allow_quant_error={allow_quant_error}" + f"[WeightChecker] handle action={action} " + f"allow_quant_error={allow_quant_error} skip_tensor_list={skip_tensor_list}" ) if action == "snapshot": return self._snapshot() elif action == "reset_tensors": - return self._reset_tensors() + return self._reset_tensors(skip_tensor_list) elif action == "compare": - return self._compare(allow_quant_error=allow_quant_error) + return self._compare( + allow_quant_error=allow_quant_error, skip_tensor_list=skip_tensor_list + ) elif action == "checksum": - return self._compute_checksum() + return self._compute_checksum(skip_tensor_list, role=role) else: raise Exception(f"Unsupported {action=}") @@ -138,21 +165,31 @@ class WeightChecker: self._snapshot_arena = allocator.arena self._snapshot_tensors = snapshot_tensors - def _reset_tensors(self): + def _skip_compare_names( + self, skip_tensor_list: Optional[List[str]] = None + ) -> Set[str]: + return { + name + for name, param in self._model_state() + if _is_skip_weight_check(name, param, skip_tensor_list) + } + + def _reset_tensors(self, skip_tensor_list: Optional[List[str]] = None): for name, param in self._model_state(): - if _is_non_persistent_buffer_name(name): + # reset must skip exactly what compare skips + if _is_skip_weight_check(name, param, skip_tensor_list): continue param.copy_(_random_like(param)) - def _compare(self, allow_quant_error: bool = False): + def _compare( + self, + allow_quant_error: bool = False, + skip_tensor_list: Optional[List[str]] = None, + ): assert self._snapshot_tensors is not None quantized_set = _build_quantized_set(self._get_model()) - skip_compare_names = { - name - for name, param in self._model_state() - if getattr(param, "_skip_weight_check", False) - } + skip_compare_names = self._skip_compare_names(skip_tensor_list) _check_tensors( expect_tensors=_build_check_entries( self._snapshot_tensors, skip_compare_names, quantized_set @@ -165,16 +202,14 @@ class WeightChecker: self._snapshot_tensors = None self._snapshot_arena = None - def _compute_checksum(self) -> Dict: + def _compute_checksum( + self, skip_tensor_list: Optional[List[str]] = None, *, role: str + ) -> Dict: torch.cuda.synchronize() start = time.perf_counter() quantized_set = _build_quantized_set(self._get_model()) - skip_compare_names = { - name - for name, param in self._model_state() - if getattr(param, "_skip_weight_check", False) - } + skip_compare_names = self._skip_compare_names(skip_tensor_list) # Hash the dequantized weight so two (qweight, scale) pairs with the same # bf16 hash equal. @@ -185,11 +220,7 @@ class WeightChecker: if should_compare: checksums[name] = _hash_tensor(comparable.dequantize().data) - h = hashlib.sha256() - for name in sorted(checksums): - h.update(name.encode()) - h.update(checksums[name].encode()) - overall = h.hexdigest() + overall = overall_checksum(checksums) torch.cuda.synchronize() elapsed = time.perf_counter() - start @@ -200,17 +231,17 @@ class WeightChecker: info = ChecksumInfo( checksums=checksums, per_gpu_checksum=overall, - parallelism_info=self._parallelism_info(), + parallelism_info=self._parallelism_info(role=role), ) return info.model_dump() - def _parallelism_info(self) -> ParallelismInfo: + def _parallelism_info(self, *, role: str) -> ParallelismInfo: # Read the current WORLD rank because elastic scale-up can change it. - return self._placement.model_copy( - update={ - "rank": dist.get_rank() if dist.is_initialized() else 0, - "size": dist.get_world_size() if dist.is_initialized() else 1, - } + return ParallelismInfo( + role=role, + rank=dist.get_rank() if dist.is_initialized() else 0, + size=dist.get_world_size() if dist.is_initialized() else 1, + **self._placement, ) def _model_state(self): diff --git a/python/sglang/srt/weight_sync/utils.py b/python/sglang/srt/weight_sync/utils.py index af199ad297..b7a969c6d1 100644 --- a/python/sglang/srt/weight_sync/utils.py +++ b/python/sglang/srt/weight_sync/utils.py @@ -23,6 +23,8 @@ async def update_weights( """ Update weights for the inference engine. This function is designed to be stateless, so that the caller process could keep the stateful engine. + The caller must hold an open weight-update session (engine.begin_weight_update() ... + engine.end_weight_update()) around one or more calls. Example Use Case: - Multiple Producer Process will call this function in a SPMD style diff --git a/test/registered/model_loading/test_utils_update_weights.py b/test/registered/model_loading/test_utils_update_weights.py index 36dd613c8a..a995e8b1f2 100644 --- a/test/registered/model_loading/test_utils_update_weights.py +++ b/test/registered/model_loading/test_utils_update_weights.py @@ -8,6 +8,10 @@ from torch.distributed.device_mesh import init_device_mesh from transformers import AutoModelForCausalLM from sglang.srt.entrypoints.engine import Engine +from sglang.srt.managers.io_struct import ( + BeginWeightUpdateReqInput, + EndWeightUpdateReqInput, +) from sglang.srt.weight_sync.utils import update_weights from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.test_utils import DEFAULT_SMALL_MODEL_NAME_FOR_TEST @@ -24,6 +28,16 @@ class AsyncEngine(Engine): update_weights_request, None ) + async def begin_weight_update(self): + return await self.tokenizer_manager.begin_weight_update( + BeginWeightUpdateReqInput(), None + ) + + async def end_weight_update(self): + return await self.tokenizer_manager.end_weight_update( + EndWeightUpdateReqInput(), None + ) + def is_distributed_available(): """Check if distributed training environment is available""" @@ -155,6 +169,7 @@ class TestUtilsUpdateWeights(unittest.TestCase): params_batch = self.create_test_params_batch(self.model, num_params=2) # Test the utils.update_weights function + await self.engine.begin_weight_update() result = await update_weights( engine=self.engine, params_batch=params_batch, @@ -162,6 +177,7 @@ class TestUtilsUpdateWeights(unittest.TestCase): device_mesh=self.mesh, load_format=None, ) + await self.engine.end_weight_update() self.assertIn("Success", result) diff --git a/test/registered/rl/test_multi_instance_release_memory_occupation.py b/test/registered/rl/test_multi_instance_release_memory_occupation.py index fcb3b25235..a736839033 100644 --- a/test/registered/rl/test_multi_instance_release_memory_occupation.py +++ b/test/registered/rl/test_multi_instance_release_memory_occupation.py @@ -77,7 +77,9 @@ class EngineWrapper: self, named_tensors: Iterable[Tuple[str, torch.Tensor]] ): if self._tp_rank == 0: + self._engine.begin_weight_update() self._engine.update_weights_from_tensor(list(named_tensors)) + self._engine.end_weight_update() self._engine.flush_cache() dist.barrier(group=self._device_mesh_cpu.get_group()) diff --git a/test/registered/rl/test_update_weights_from_distributed.py b/test/registered/rl/test_update_weights_from_distributed.py index ec47a36885..03d14bfb01 100644 --- a/test/registered/rl/test_update_weights_from_distributed.py +++ b/test/registered/rl/test_update_weights_from_distributed.py @@ -98,6 +98,7 @@ def _warmup_update( ): """Run one update round to warm up RCCL before timing.""" if backend == "Engine": + engine.begin_weight_update() engine.update_weights_from_distributed( names, dtypes=dtypes, @@ -105,7 +106,9 @@ def _warmup_update( group_name="test_parameter_update_group", load_format=load_format, ) + engine.end_weight_update() else: + requests.post(f"{url}/begin_weight_update", json={}) requests.post( f"{url}/update_weights_from_distributed", json={ @@ -117,6 +120,7 @@ def _warmup_update( "flush_cache": not (pause_generation_mode == "in_place"), }, ) + requests.post(f"{url}/end_weight_update", json={}) def init_process( @@ -439,6 +443,7 @@ def init_process_sgl( time_begin_update = time.perf_counter() if backend == "Engine": + engine.begin_weight_update() engine.update_weights_from_distributed( names, dtypes=dtypes, @@ -446,7 +451,9 @@ def init_process_sgl( group_name="test_parameter_update_group", load_format=load_format, ) + engine.end_weight_update() else: + requests.post(f"{url}/begin_weight_update", json={}) requests.post( f"{url}/update_weights_from_distributed", json={ @@ -458,6 +465,7 @@ def init_process_sgl( "flush_cache": not (pause_generation_mode == "in_place"), }, ) + requests.post(f"{url}/end_weight_update", json={}) torch.cuda.synchronize() time_end_update = time.perf_counter() if pause_generation_mode in ["in_place", "retract"]: diff --git a/test/registered/rl/test_update_weights_from_tensor.py b/test/registered/rl/test_update_weights_from_tensor.py index d9dd41a604..38d57f95f7 100644 --- a/test/registered/rl/test_update_weights_from_tensor.py +++ b/test/registered/rl/test_update_weights_from_tensor.py @@ -39,7 +39,9 @@ def test_update_weights_from_tensor(tp_size): new_tensor = torch.full((16384, 2048), 1.5, device="cuda") time_start = time.perf_counter() + engine.begin_weight_update() engine.update_weights_from_tensor([(x, new_tensor) for x in param_names]) + engine.end_weight_update() print(f"Time delta: {time.perf_counter() - time_start:.03f}") for param_name in param_names[:3]: @@ -82,6 +84,7 @@ class TestUpdateWeightsFromTensor(CustomTestCase): ) new_tensor = torch.full((3072, 2048), 1.5) + engine.begin_weight_update() engine.update_weights_from_tensor( [ (write_param_name, new_tensor.clone()) @@ -89,6 +92,7 @@ class TestUpdateWeightsFromTensor(CustomTestCase): ], load_format="direct", ) + engine.end_weight_update() for read_param_name in read_param_names[:3]: _check_param(engine, read_param_name, [1.5] * 5) @@ -114,6 +118,7 @@ class TestUpdateWeightsFromTensor(CustomTestCase): ) new_tensor = torch.full((3072, 2048), 1.5) + engine.begin_weight_update() engine.update_weights_from_tensor( [ (write_param_name, new_tensor.clone()) @@ -121,6 +126,7 @@ class TestUpdateWeightsFromTensor(CustomTestCase): ], load_format=custom_loader_name, ) + engine.end_weight_update() for read_param_name in read_param_names[:3]: _check_param(engine, read_param_name, [1.5] * 5) @@ -168,9 +174,11 @@ class TestUpdateWeightsFromTensor(CustomTestCase): # Update weights using flattened_bucket format time_start = time.perf_counter() + engine.begin_weight_update() engine.update_weights_from_tensor( named_tensors=serialized_bucket_list, load_format="flattened_bucket" ) + engine.end_weight_update() update_time = time.perf_counter() - time_start print(f"Flattened bucket update time: {update_time:.03f}") @@ -234,6 +242,7 @@ class TestServerUpdateWeightsFromTensorNonBlocking(CustomTestCase): return ret def run_update_weights(self, named_tensors, flush_cache=True): + requests.post(self.base_url + "/begin_weight_update", json={}) response = requests.post( self.base_url + "/update_weights_from_tensor", json={ @@ -243,6 +252,7 @@ class TestServerUpdateWeightsFromTensorNonBlocking(CustomTestCase): "flush_cache": flush_cache, }, ) + requests.post(self.base_url + "/end_weight_update", json={}) ret = response.json() return ret diff --git a/test/registered/rl/test_weight_checker_e2e.py b/test/registered/rl/test_weight_checker_e2e.py index bf1de73642..67754fcb92 100644 --- a/test/registered/rl/test_weight_checker_e2e.py +++ b/test/registered/rl/test_weight_checker_e2e.py @@ -80,7 +80,8 @@ class TestWeightCheckerE2E(CustomTestCase): def _update_weights( self, named_tensors: List[Tuple[str, torch.Tensor]] ) -> requests.Response: - return requests.post( + requests.post(f"{self.url}/begin_weight_update", json={}, timeout=120) + resp = requests.post( f"{self.url}/update_weights_from_tensor", json={ "serialized_named_tensors": [ @@ -90,6 +91,8 @@ class TestWeightCheckerE2E(CustomTestCase): }, timeout=120, ) + requests.post(f"{self.url}/end_weight_update", json={}, timeout=120) + return resp def test_a_snapshot_then_compare_unchanged_succeeds(self): resp = self._post("snapshot") @@ -160,7 +163,10 @@ class TestWeightCheckerE2E(CustomTestCase): self.assertIn("checksums", first) self.assertIn("parallelism_info", first) - info = first["parallelism_info"] + infos = first["parallelism_info"] + # one entry per runner; without speculative decoding that is the target + self.assertEqual([info["role"] for info in infos], ["target"]) + info = infos[0] for key in ( "tp_rank", "tp_size", diff --git a/test/registered/unit/layers/quantization/test_compressed_tensors_wna16_moe_reload.py b/test/registered/unit/layers/quantization/test_compressed_tensors_wna16_moe_reload.py new file mode 100644 index 0000000000..2c4020600e --- /dev/null +++ b/test/registered/unit/layers/quantization/test_compressed_tensors_wna16_moe_reload.py @@ -0,0 +1,205 @@ +"""Restore, reload and finalize of a Marlin W4A16 MoE must reproduce a fresh load.""" + +import unittest +from contextlib import contextmanager + +import torch + +from sglang.srt.layers.moe.utils import MoeRunnerBackend +from sglang.srt.layers.quantization.compressed_tensors.compressed_tensors import ( + CompressedTensorsConfig, +) +from sglang.srt.layers.quantization.compressed_tensors.schemes import ( + CompressedTensorsWNA16MoE, +) +from sglang.srt.model_executor.model_runner_components.weight_updater import ( + WeightUpdater, +) +from sglang.srt.model_loader.loader import DefaultModelLoader +from sglang.srt.runtime_context import get_context, get_flags, get_parallel +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.layer_ut_utils import assert_output_close, init_single_process_dist +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=120, stage="base-b", runner_config="1-gpu-small") + +E, H, I, TOPK, M = 8, 1024, 1024, 2, 32 +GROUP_SIZE, PACK_FACTOR = 128, 8 +DEVICE = torch.device("cuda") +QUANT_CONFIG = { + "quant_method": "compressed-tensors", + "format": "pack-quantized", + "config_groups": { + "group_0": { + "targets": ["re:.*mlp\\.experts.*"], + "weights": { + "num_bits": 4, + "type": "int", + "symmetric": True, + "strategy": "group", + "group_size": GROUP_SIZE, + }, + "input_activations": None, + } + }, + "ignore": ["lm_head"], +} + + +def _random_checkpoint(seed): + gen = torch.Generator(device=DEVICE).manual_seed(seed) + shards = {} + for expert_id in range(E): + for shard_id, (rows, cols) in (("w1", (I, H)), ("w3", (I, H)), ("w2", (H, I))): + shards[expert_id, shard_id] = { + # any int32 packs eight valid int4 values + "weight_packed": torch.randint( + -(2**31), + 2**31 - 1, + (rows, cols // PACK_FACTOR), + dtype=torch.int32, + device=DEVICE, + generator=gen, + ), + "weight_scale": ( + torch.rand(rows, cols // GROUP_SIZE, device=DEVICE, generator=gen) + / 100 + + 1e-3 + ).to(torch.bfloat16), + "weight_shape": torch.tensor( + [rows, cols], dtype=torch.float32, device=DEVICE + ), + } + return shards + + +def _load(layer, shards): + for (expert_id, shard_id), tensors in shards.items(): + prefix = "w2" if shard_id == "w2" else "w13" + for suffix, tensor in tensors.items(): + name = f"{prefix}_{suffix}" + layer.weight_loader( + getattr(layer, name), + tensor, + name, + shard_id=shard_id, + expert_id=expert_id, + ) + + +class TestWNA16MoEMarlinReload(CustomTestCase): + @classmethod + def setUpClass(cls): + init_single_process_dist(master_port=29633) + torch.set_default_device("cuda") + + @contextmanager + def _runtime(self): + with ( + get_context().override_server_args(model_path="dummy"), + get_flags().moe.override(runner_backend=MoeRunnerBackend.MARLIN), + get_parallel().override( + moe_ep_size=1, + moe_ep_rank=0, + moe_tp_size=1, + moe_tp_rank=0, + tp_size=1, + tp_rank=0, + ), + ): + yield + + def _fresh_model(self, shards): + from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE + + model = torch.nn.Module() + model.experts = FusedMoE( + num_experts=E, + hidden_size=H, + intermediate_size=I, + layer_id=0, + top_k=TOPK, + params_dtype=torch.bfloat16, + quant_config=CompressedTensorsConfig.from_config(QUANT_CONFIG), + prefix="model.layers.0.mlp.experts", + ).cuda() + _load(model.experts, shards) + DefaultModelLoader.postprocess_weights(model, DEVICE) + return model + + @staticmethod + def _refit(model, shards): + # the component the scheduler drives for each runner in a session + updater = WeightUpdater( + tp_rank=0, + device="cuda", + gpu_id=0, + model_config=None, + custom_weight_loaders={}, + get_model=lambda: model, + update_model_fields=None, + recapture_cuda_graph=None, + get_model_runner=None, + ) + updater.begin_weight_update() + _load(model.experts, shards) + updater.end_weight_update(run_post_load=False) + + def _assert_same_weights(self, model, expected): + expected_params = dict(expected.named_parameters()) + for name, param in model.named_parameters(): + self.assertTrue( + torch.equal(param, expected_params[name]), + f"{name} differs from a fresh load", + ) + + @staticmethod + def _forward(model, x, topk_output): + out = model.experts.forward(x, topk_output) + if isinstance(out, torch.Tensor): + return out + return out[0] if isinstance(out, tuple) else out.hidden_states + + def test_refit_reproduces_a_fresh_load(self): + """Without restore the checkpoint shards do not fit the Marlin layout; without finalize the kernel reads raw shards.""" + from sglang.srt.layers.moe.topk import TopKConfig, select_experts + + with self._runtime(): + ckpt_a, ckpt_b = _random_checkpoint(1), _random_checkpoint(2) + fresh_a, fresh_b = self._fresh_model(ckpt_a), self._fresh_model(ckpt_b) + model = self._fresh_model(ckpt_a) + # SM100 auto picks the Triton subclass, which cannot be refit: its loads fail on shape + self.assertIs(type(model.experts.scheme), CompressedTensorsWNA16MoE) + ptrs = {name: p.data_ptr() for name, p in model.named_parameters()} + + self._refit(model, ckpt_b) + self._assert_same_weights(model, fresh_b) + self._refit(model, ckpt_a) + self._assert_same_weights(model, fresh_a) + # captured CUDA graphs keep reading the original buffers + self.assertEqual( + {name: p.data_ptr() for name, p in model.named_parameters()}, ptrs + ) + + torch.manual_seed(0) + x = torch.randn(M, H, dtype=torch.bfloat16) / 10 + topk_output = select_experts( + hidden_states=x, + router_logits=torch.randn(M, E, dtype=torch.float32), + topk_config=TopKConfig(top_k=TOPK, renormalize=True), + ) + out_a = self._forward(fresh_a, x, topk_output).float() + out_b = self._forward(fresh_b, x, topk_output).float() + assert_output_close( + self, self._forward(model, x, topk_output), out_a, cos_threshold=0.9999 + ) + cos_ab = torch.nn.functional.cosine_similarity( + out_a.flatten(), out_b.flatten(), dim=0 + ).item() + self.assertLess( + cos_ab, 0.9, "the two checkpoints must give different outputs" + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/managers/test_msgpack_ipc_roundtrip.py b/test/registered/unit/managers/test_msgpack_ipc_roundtrip.py index a206c27fbc..f718711a4b 100644 --- a/test/registered/unit/managers/test_msgpack_ipc_roundtrip.py +++ b/test/registered/unit/managers/test_msgpack_ipc_roundtrip.py @@ -34,6 +34,9 @@ from sglang.srt.managers.io_struct import ( msgpack_decode, msgpack_encode, ) +from sglang.srt.managers.scheduler_components.weight_updater import ( + _merge_checksum_payloads, +) from sglang.srt.model_executor.cuda_graph_config import CudaGraphConfig from sglang.srt.utils.msgspec_utils import msgspec_to_builtins from sglang.srt.utils.weight_checker import ChecksumInfo as PydanticChecksumInfo @@ -65,9 +68,17 @@ def _contains_dataclass(obj) -> bool: return False -def _parallelism_info() -> ParallelismInfo: +def _parallelism_info(role: str = "target") -> ParallelismInfo: return ParallelismInfo( - tp_rank=0, tp_size=2, dp_rank=0, dp_size=1, pp_rank=0, pp_size=1, rank=0, size=2 + role=role, + tp_rank=0, + tp_size=2, + dp_rank=0, + dp_size=1, + pp_rank=0, + pp_size=1, + rank=0, + size=2, ) @@ -75,7 +86,7 @@ def _checksum_info(tag: str) -> ChecksumInfo: return ChecksumInfo( checksums={f"model.layers.{tag}": "deadbeef"}, per_gpu_checksum="cafef00d", - parallelism_info=_parallelism_info(), + parallelism_info=[_parallelism_info("target"), _parallelism_info("draft")], ) @@ -194,29 +205,41 @@ class TestMsgpackIpcRoundtrip(CustomTestCase): self.assertEqual(len(decoded.payload), 2) as_dict = msgspec_to_builtins(decoded.payload[0]) self.assertEqual(as_dict["per_gpu_checksum"], "cafef00d") - self.assertIn("tp_rank", as_dict["parallelism_info"]) + self.assertEqual( + [pi["role"] for pi in as_dict["parallelism_info"]], ["target", "draft"] + ) + self.assertIn("tp_rank", as_dict["parallelism_info"][0]) def test_check_weights_producer_conversion(self): - # Mirrors weight_updater.check_weights: WeightChecker returns - # ChecksumInfo.model_dump() (a dict), converted to the msgspec struct via - # msgspec.convert, and the result round-trips as the payload. - pydantic_checksum = PydanticChecksumInfo( - checksums={"model.layers.0": "deadbeef"}, - per_gpu_checksum="cafef00d", - parallelism_info=PydanticParallelismInfo( - tp_rank=0, - tp_size=2, - dp_rank=0, - dp_size=1, - pp_rank=0, - pp_size=1, - rank=0, - size=2, - ), + # the merged per-role payload is what msgspec.convert has to accept + def _dump(role): + return PydanticChecksumInfo( + checksums={"model.layers.0": "deadbeef"}, + per_gpu_checksum="cafef00d", + parallelism_info=PydanticParallelismInfo( + role=role, + tp_rank=0, + tp_size=2, + dp_rank=0, + dp_size=1, + pp_rank=0, + pp_size=1, + rank=0, + size=2, + ), + ).model_dump() + + merged = _merge_checksum_payloads( + [("target", _dump("target")), ("draft", _dump("draft"))] ) - converted = msgspec.convert(pydantic_checksum.model_dump(), ChecksumInfo) - self.assertEqual(converted.per_gpu_checksum, "cafef00d") - self.assertEqual(converted.parallelism_info.tp_rank, 0) + converted = msgspec.convert(merged, ChecksumInfo) + self.assertEqual( + sorted(converted.checksums), ["draft.model.layers.0", "model.layers.0"] + ) + self.assertEqual( + [pi.role for pi in converted.parallelism_info], ["target", "draft"] + ) + self.assertEqual(converted.parallelism_info[0].tp_rank, 0) output = CheckWeightsReqOutput(success=True, message="ok", payload=[converted]) self.assertEqual(_round_trip(output), output) diff --git a/test/registered/unit/managers/test_scheduler_weight_version_tracking.py b/test/registered/unit/managers/test_scheduler_weight_version_tracking.py index a3d1c61da3..8eb58463dd 100644 --- a/test/registered/unit/managers/test_scheduler_weight_version_tracking.py +++ b/test/registered/unit/managers/test_scheduler_weight_version_tracking.py @@ -1,10 +1,15 @@ import unittest from types import SimpleNamespace -from unittest.mock import patch +from unittest.mock import Mock, patch +from sglang.srt.managers.io_struct import ( + BeginWeightUpdateReqInput, + EndWeightUpdateReqInput, +) from sglang.srt.managers.scheduler import Scheduler from sglang.srt.managers.scheduler_components.weight_updater import ( SchedulerWeightUpdaterManager, + _WeightUpdateSession, ) from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -96,17 +101,56 @@ class TestSchedulerRecordWeightVersionChange(CustomTestCase): ) -class TestRecordWeightVersionAfterUpdate(CustomTestCase): - def _updater( - self, target_result, draft_result=None, method="update_weights_from_disk" +def _runner(result=(True, "ok")): + runner = Mock() + for method in ( + "update_weights_from_disk", + "update_weights_from_tensor", + "update_weights_from_ipc", + "load_weights_from_distributed", ): + getattr(runner.weight_updater, method).return_value = result + return runner + + +def _request(**fields): + return SimpleNamespace( + **{ + "weight_version": "v2", + "flush_cache": True, + "torch_empty_cache": False, + "model_path": "m", + "recapture_cuda_graph": False, + "load_format": None, + "selector": "all", + "names": ["model.layers.0.weight"], + "dtypes": ["float32"], + "shapes": [[1]], + "group_name": "g", + "serialized_named_tensors": [b""], + **fields, + } + ) + + +class _WeightUpdaterManagerTestBase(CustomTestCase): + def setUp(self): + patcher = patch("torch.distributed.barrier") + patcher.start() + self.addCleanup(patcher.stop) self.recorded = [] - return SchedulerWeightUpdaterManager( - tp_worker=SimpleNamespace(**{method: lambda recv_req: target_result}), + + def _manager(self, target, draft=None, *, session=True): + manager = SchedulerWeightUpdaterManager( + tp_worker=SimpleNamespace( + model_runner=target, + weight_update_runners=lambda: [("target", target)], + deserialize_own_rank=lambda payloads: [], + ), draft_worker=( None - if draft_result is None - else SimpleNamespace(**{method: lambda recv_req: draft_result}) + if draft is None + else SimpleNamespace(weight_update_runners=lambda: [("draft", draft)]) ), tp_cpu_group=None, memory_saver_adapter=None, @@ -118,104 +162,189 @@ class TestRecordWeightVersionAfterUpdate(CustomTestCase): ) ), ) + if session: + # update_weights_from_* require an open session + manager._session = _WeightUpdateSession(selector="all") + return manager - def _request(self, **fields): - return SimpleNamespace( - weight_version="v2", - flush_cache=True, - torch_empty_cache=False, - **fields, - ) +class TestRecordWeightVersionAfterUpdate(_WeightUpdaterManagerTestBase): def test_successful_update_records_the_version(self): """A refit that reports success advances the scheduler-side version.""" - updater = self._updater(target_result=(True, "ok")) - - output = updater.update_weights_from_disk(self._request()) + output = self._manager(_runner()).update_weights_from_disk(_request()) self.assertTrue(output.success) self.assertEqual(self.recorded, ["v2"]) def test_failed_update_does_not_record_the_version(self): """A refit that fails must leave the version alone, or later tokens are mislabelled.""" - updater = self._updater(target_result=(False, "boom")) - - output = updater.update_weights_from_disk(self._request()) + output = self._manager(_runner((False, "boom"))).update_weights_from_disk( + _request() + ) self.assertFalse(output.success) self.assertEqual(self.recorded, []) def test_draft_failure_does_not_record_the_version(self): """The target succeeding is not enough: a failed draft refit leaves the engine mixed.""" - updater = self._updater( - target_result=(True, "ok"), draft_result=(False, "draft boom") - ) + manager = self._manager(_runner(), draft=_runner((False, "draft boom"))) - output = updater.update_weights_from_disk(self._request()) + output = manager.update_weights_from_disk(_request()) self.assertFalse(output.success) self.assertEqual(self.recorded, []) def test_successful_distributed_update_records_the_version(self): """The distributed refit is the path an RL trainer actually drives, so it must record too.""" - updater = self._updater( - target_result=(True, "ok"), method="update_weights_from_distributed" - ) - - output = updater.update_weights_from_distributed(self._request()) + output = self._manager(_runner()).update_weights_from_distributed(_request()) self.assertTrue(output.success) self.assertEqual(self.recorded, ["v2"]) def test_failed_distributed_update_does_not_record_the_version(self): """A failed distributed refit leaves the version alone, exactly like the disk path.""" - updater = self._updater( - target_result=(False, "boom"), method="update_weights_from_distributed" - ) - - output = updater.update_weights_from_distributed(self._request()) + output = self._manager( + _runner((False, "boom")) + ).update_weights_from_distributed(_request()) self.assertFalse(output.success) self.assertEqual(self.recorded, []) def test_successful_tensor_update_records_the_version(self): """The tensor refit records the version once the load reports success.""" - updater = self._updater( - target_result=(True, "ok"), method="update_weights_from_tensor" - ) - - with patch("torch.distributed.barrier"): - output = updater.update_weights_from_tensor( - self._request(disable_draft_model=True) - ) + output = self._manager(_runner()).update_weights_from_tensor(_request()) self.assertTrue(output.success) self.assertEqual(self.recorded, ["v2"]) def test_successful_ipc_update_records_the_version(self): """The checkpoint-engine IPC refit records the version like every other path.""" - updater = self._updater( - target_result=(True, "ok"), method="update_weights_from_ipc" - ) - - with patch("torch.distributed.barrier"): - output = updater.update_weights_from_ipc(self._request()) + output = self._manager(_runner()).update_weights_from_ipc(_request()) self.assertTrue(output.success) self.assertEqual(self.recorded, ["v2"]) def test_failed_ipc_update_does_not_record_the_version(self): """The IPC path branches on success separately from the cache flush, so failure must record nothing.""" - updater = self._updater( - target_result=(False, "boom"), method="update_weights_from_ipc" + output = self._manager(_runner((False, "boom"))).update_weights_from_ipc( + _request() ) - with patch("torch.distributed.barrier"): - output = updater.update_weights_from_ipc(self._request()) - self.assertFalse(output.success) self.assertEqual(self.recorded, []) +class TestWeightUpdateSession(_WeightUpdaterManagerTestBase): + def test_distributed_update_receives_once_on_target_loads_into_each(self): + """A draft runner that received its own broadcast would deadlock the update group.""" + target, draft = _runner(), _runner() + weights = target.weight_updater.receive_weights_from_distributed.return_value + req = _request() + + output = self._manager(target, draft).update_weights_from_distributed(req) + + self.assertTrue(output.success) + target.weight_updater.receive_weights_from_distributed.assert_called_once_with( + names=req.names, + dtypes=req.dtypes, + shapes=req.shapes, + group_name=req.group_name, + load_format=req.load_format, + ) + target.weight_updater.load_weights_from_distributed.assert_called_once_with( + weights + ) + draft.weight_updater.load_weights_from_distributed.assert_called_once_with( + weights + ) + draft.weight_updater.receive_weights_from_distributed.assert_not_called() + + def test_distributed_update_target_only_selector_skips_draft(self): + """selector="target" must not load into the draft.""" + target, draft = _runner(), _runner() + + output = self._manager(target, draft).update_weights_from_distributed( + _request(selector="target") + ) + + self.assertTrue(output.success) + target.weight_updater.load_weights_from_distributed.assert_called_once() + draft.weight_updater.load_weights_from_distributed.assert_not_called() + + def test_end_runs_post_load_on_both_when_load_was_bypassed(self): + """A P2P/RDMA session never calls load_weights, so end must run post_load_weights.""" + target, draft = _runner(), _runner() + manager = self._manager(target, draft) + + output = manager.end_weight_update(EndWeightUpdateReqInput()) + + self.assertTrue(output.success) + for runner in (target, draft): + runner.weight_updater.end_weight_update.assert_called_once_with( + run_post_load=True + ) + self.assertIsNone(manager._session) + + def test_end_skips_post_load_on_both_when_weights_loaded(self): + """load_weights already ran post_load_weights; running it twice would double-apply.""" + target, draft = _runner(), _runner() + manager = self._manager(target, draft) + manager._session = _WeightUpdateSession(selector="all", loaded_weights=True) + + manager.end_weight_update(EndWeightUpdateReqInput()) + + for runner in (target, draft): + runner.weight_updater.end_weight_update.assert_called_once_with( + run_post_load=False + ) + + def test_session_selector_confines_begin_and_end_to_selected_runners(self): + """end finalizing a runner begin never restored would repack unrestored weights.""" + target, draft = _runner(), _runner() + manager = self._manager(target, draft, session=False) + + manager.begin_weight_update(BeginWeightUpdateReqInput(selector="draft")) + manager.end_weight_update(EndWeightUpdateReqInput()) + + target.weight_updater.begin_weight_update.assert_not_called() + target.weight_updater.end_weight_update.assert_not_called() + draft.weight_updater.begin_weight_update.assert_called_once_with() + draft.weight_updater.end_weight_update.assert_called_once() + + def test_begin_rejects_reentry(self): + """A second begin would leave the first session's runners unfinalized.""" + target = _runner() + + output = self._manager(target).begin_weight_update(BeginWeightUpdateReqInput()) + + self.assertFalse(output.success) + self.assertIn("already open", output.message) + target.weight_updater.begin_weight_update.assert_not_called() + + def test_end_without_session_is_rejected(self): + """Finalizing runners begin never restored would repack weights twice.""" + target = _runner() + + output = self._manager(target, session=False).end_weight_update( + EndWeightUpdateReqInput() + ) + + self.assertFalse(output.success) + self.assertIn("begin_weight_update", output.message) + target.weight_updater.end_weight_update.assert_not_called() + + def test_update_without_session_is_rejected_without_loading(self): + """A caller that skips begin gets an error back instead of crashing the scheduler.""" + target = _runner() + + output = self._manager(target, session=False).update_weights_from_distributed( + _request() + ) + + self.assertFalse(output.success) + self.assertIn("begin_weight_update", output.message) + target.weight_updater.receive_weights_from_distributed.assert_not_called() + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/model_executor/test_compensated_mhc_update_guard.py b/test/registered/unit/model_executor/test_compensated_mhc_update_guard.py index 82e27cca44..4fd8864cd3 100644 --- a/test/registered/unit/model_executor/test_compensated_mhc_update_guard.py +++ b/test/registered/unit/model_executor/test_compensated_mhc_update_guard.py @@ -22,7 +22,7 @@ class TestCompensatedMhcUpdateGuard(CustomTestCase): for field in ("_hc_attn_tf32_parts", "_hc_ffn_tf32_parts"): for method, args in ( ("update_weights_from_tensor", ([], "direct")), - ("update_weights_from_distributed", ([], [], [], "unused")), + ("load_weights_from_distributed", ([],)), ("update_weights_from_disk", ("unused", "auto")), ("update_weights_from_ipc", (SimpleNamespace(),)), ): diff --git a/test/registered/unit/spec/test_weight_update_runners.py b/test/registered/unit/spec/test_weight_update_runners.py new file mode 100644 index 0000000000..aba2b0b41b --- /dev/null +++ b/test/registered/unit/spec/test_weight_update_runners.py @@ -0,0 +1,101 @@ +"""Every spec family must name exactly the draft runners that own weights.""" + +import unittest +from types import SimpleNamespace + +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase, maybe_stub_sgl_kernel + +register_cpu_ci(est_time=10, suite="base-a-test-cpu") +maybe_stub_sgl_kernel() + +from sglang.srt.speculative.dflash_worker_v2 import DFlashWorkerV2 +from sglang.srt.speculative.dspark_components.dspark_worker_v2 import DSparkWorkerV2 +from sglang.srt.speculative.eagle_worker_v2 import EAGLEWorkerV2 +from sglang.srt.speculative.frozen_kv_mtp_worker_v2 import ( + FrozenKVMTPDraftWorker, + FrozenKVMTPWorkerV2, +) +from sglang.srt.speculative.multi_layer_eagle_worker_v2 import ( + MultiLayerEagleDraftWorker, + MultiLayerEagleWorkerV2, +) +from sglang.srt.speculative.ngram_worker import NGRAMWorker +from sglang.srt.speculative.standalone_worker_v2 import StandaloneWorkerV2 +from sglang.srt.speculative.uno_worker_v2 import UnoWorkerV2 + + +def _bare(cls, **attrs): + # skip __init__: only the attributes the enumeration reads are set + obj = object.__new__(cls) + for name, value in attrs.items(): + setattr(obj, name, value) + return obj + + +def _eagle_draft(runners): + return SimpleNamespace(draft_runners=runners[:1]) + + +CASES = { + "eagle": ( + EAGLEWorkerV2, + lambda rs: {"_draft_worker": _eagle_draft(rs)}, + [("draft", 0)], + ), + "standalone": ( + StandaloneWorkerV2, + lambda rs: {"_draft_worker": _eagle_draft(rs)}, + [("draft", 0)], + ), + # reuses the target KV but owns its weights, unlike _draft_model_runners() + "frozen_kv_mtp": ( + FrozenKVMTPWorkerV2, + lambda rs: { + "_draft_worker": _bare(FrozenKVMTPDraftWorker, _model_runner=rs[0]) + }, + [("draft", 0)], + ), + "multi_layer_eagle": ( + MultiLayerEagleWorkerV2, + lambda rs: { + "_draft_worker": _bare(MultiLayerEagleDraftWorker, draft_runner_list=rs) + }, + [("draft_step_0", 0), ("draft_step_1", 1)], + ), + "dflash": ( + DFlashWorkerV2, + lambda rs: {"draft_model_runner": rs[0]}, + [("draft", 0)], + ), + "dspark_last_pp_stage": ( + DSparkWorkerV2, + lambda rs: {"_hosts_draft": True, "draft_model_runner": rs[0]}, + [("draft", 0)], + ), + "dspark_non_last_pp_stage": ( + DSparkWorkerV2, + lambda rs: {"_hosts_draft": False}, + [], + ), + "ngram": (NGRAMWorker, lambda rs: {}, []), + "uno": (UnoWorkerV2, lambda rs: {}, []), +} + + +class TestWeightUpdateRunners(CustomTestCase): + def test_weight_update_runners_cover_each_spec_family(self): + for case, (cls, attrs, expected) in CASES.items(): + with self.subTest(case): + runners = [object(), object()] + worker = _bare(cls, **attrs(runners)) + + got = worker.weight_update_runners() + + self.assertEqual( + [(role, runners.index(r)) for role, r in got], expected + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/test_runtime_context.py b/test/registered/unit/test_runtime_context.py index 957db33d24..83d5f207a8 100644 --- a/test/registered/unit/test_runtime_context.py +++ b/test/registered/unit/test_runtime_context.py @@ -2841,7 +2841,7 @@ class TestWhoAnswersDuringADraftScope(CustomTestCase): checker = WeightChecker(get_model=lambda: None) self.assertEqual(get_parallel().pp_size, 2) - info = checker._parallelism_info() + info = checker._parallelism_info(role="target") self.assertEqual((info.pp_rank, info.pp_size), (0, 1)) diff --git a/test/registered/unit/utils/test_weight_checker.py b/test/registered/unit/utils/test_weight_checker.py index 0de2649ed3..6bd689dfa5 100644 --- a/test/registered/unit/utils/test_weight_checker.py +++ b/test/registered/unit/utils/test_weight_checker.py @@ -681,21 +681,21 @@ class TestHandle(_WeightCheckerTestBase): self.checker, "_compute_checksum", return_value={"checksums": {}} ) as m_checksum, ): - self.checker.handle("snapshot") - self.checker.handle("reset_tensors") - self.checker.handle("compare") - self.checker.handle("checksum") + self.checker.handle("snapshot", role="target") + self.checker.handle("reset_tensors", role="target") + self.checker.handle("compare", role="target") + self.checker.handle("checksum", role="target") m_snap.assert_called_once() m_reset.assert_called_once() m_compare.assert_called_once() m_checksum.assert_called_once() def test_returns_none_for_non_checksum_actions(self): - self.assertIsNone(self.checker.handle("snapshot")) - self.assertIsNone(self.checker.handle("compare")) + self.assertIsNone(self.checker.handle("snapshot", role="target")) + self.assertIsNone(self.checker.handle("compare", role="target")) def test_returns_dict_for_checksum_action(self): - out = self.checker.handle("checksum") + out = self.checker.handle("checksum", role="target") self.assertIsInstance(out, dict) self.assertIn("checksums", out) self.assertIn("per_gpu_checksum", out) @@ -703,7 +703,7 @@ class TestHandle(_WeightCheckerTestBase): def test_unknown_action_raises(self): with self.assertRaises(Exception) as ctx: - self.checker.handle("nonsense_action") + self.checker.handle("nonsense_action", role="target") self.assertIn("Unsupported", str(ctx.exception)) @@ -789,13 +789,13 @@ class _ChecksumTestBase(CustomTestCase): class TestComputeChecksum(_ChecksumTestBase): def test_returns_dict_with_expected_top_level_keys(self): - out = self.checker._compute_checksum() + out = self.checker._compute_checksum(role="target") self.assertEqual( set(out.keys()), {"checksums", "per_gpu_checksum", "parallelism_info"} ) def test_skips_non_persistent_buffers(self): - out = self.checker._compute_checksum() + out = self.checker._compute_checksum(role="target") names = set(out["checksums"].keys()) # Normal params and buffers are present. self.assertIn("w", names) @@ -807,13 +807,13 @@ class TestComputeChecksum(_ChecksumTestBase): self.assertNotIn("rotary_emb_freqs_cis", names) def test_hashes_are_hex_strings(self): - out = self.checker._compute_checksum() + out = self.checker._compute_checksum(role="target") for name, h in out["checksums"].items(): self.assertEqual(len(h), 16, f"unexpected hash length for {name!r}") int(h, 16) def test_parallelism_info_reflects_runner_state(self): - info = self.checker._compute_checksum()["parallelism_info"] + info = self.checker._compute_checksum(role="target")["parallelism_info"] self.assertEqual(info["tp_rank"], 2) self.assertEqual(info["tp_size"], 4) self.assertEqual(info["dp_rank"], 1) @@ -825,19 +825,19 @@ class TestComputeChecksum(_ChecksumTestBase): self.assertIn("size", info) def test_checksum_is_stable_for_unchanged_weights(self): - first = self.checker._compute_checksum() - second = self.checker._compute_checksum() + first = self.checker._compute_checksum(role="target") + second = self.checker._compute_checksum(role="target") self.assertEqual(first, second) def test_checksum_changes_after_param_mutation(self): - first = self.checker._compute_checksum()["checksums"]["w"] + first = self.checker._compute_checksum(role="target")["checksums"]["w"] with torch.no_grad(): self.model.w.data.fill_(99.0) - second = self.checker._compute_checksum()["checksums"]["w"] + second = self.checker._compute_checksum(role="target")["checksums"]["w"] self.assertNotEqual(first, second) def test_validates_against_pydantic_schema(self): - out = self.checker._compute_checksum() + out = self.checker._compute_checksum(role="target") info = ChecksumInfo.model_validate(out) self.assertIsInstance(info.parallelism_info, ParallelismInfo)