From b83827ea02618332f4bc8b60dec92baec80a3c37 Mon Sep 17 00:00:00 2001 From: Cheng Wan <54331508+ch-wan@users.noreply.github.com> Date: Wed, 7 Oct 2026 16:20:47 -0700 Subject: [PATCH] [Fix] Quantize reloaded FP8 weights after native sharding (#42585) --- .../kernels/ops/quantization/fp8_kernel.py | 87 ++- .../srt/layers/quantization/fp8_utils.py | 11 +- python/sglang/srt/model_loader/loader.py | 478 ++++++---------- .../sglang/srt/model_loader/weight_utils.py | 12 + python/sglang/srt/models/qwen2.py | 2 + python/sglang/srt/models/qwen3.py | 2 + .../model_loader/test_default_model_loader.py | 14 + .../test_quantized_reload_scale_layout.py | 518 ++++++++++++++++++ 8 files changed, 783 insertions(+), 341 deletions(-) create mode 100644 test/registered/unit/model_loader/test_quantized_reload_scale_layout.py diff --git a/python/sglang/kernels/ops/quantization/fp8_kernel.py b/python/sglang/kernels/ops/quantization/fp8_kernel.py index 67d67a33eb..3706819237 100644 --- a/python/sglang/kernels/ops/quantization/fp8_kernel.py +++ b/python/sglang/kernels/ops/quantization/fp8_kernel.py @@ -214,6 +214,9 @@ def _per_token_group_quant_8bit_raw( column_major_scales: bool = False, scale_tma_aligned: bool = False, scale_ue8m0: bool = False, + *, + output_q: Optional[torch.Tensor] = None, + output_s: Optional[torch.Tensor] = None, ) -> Tuple[torch.Tensor, torch.Tensor]: """Function to perform per-token-group quantization on an input tensor `x`. @@ -251,14 +254,22 @@ def _per_token_group_quant_8bit_raw( bit8_max = info.max bit8_min = info.min - x_q = torch.empty_like(x, device=x.device, dtype=dtype) - x_s = create_per_token_group_quant_fp8_output_scale( - x_shape=x.shape, - device=x.device, - group_size=group_size, - column_major_scales=column_major_scales, - scale_tma_aligned=scale_tma_aligned, - scale_ue8m0=False, + x_q = ( + output_q + if output_q is not None + else torch.empty_like(x, device=x.device, dtype=dtype) + ) + x_s = ( + output_s + if output_s is not None + else create_per_token_group_quant_fp8_output_scale( + x_shape=x.shape, + device=x.device, + group_size=group_size, + column_major_scales=column_major_scales, + scale_tma_aligned=scale_tma_aligned, + scale_ue8m0=False, + ) ) M = x.numel() // group_size @@ -573,6 +584,9 @@ def sglang_per_token_group_quant_fp8( scale_ue8m0: bool = False, fuse_silu_and_mul: bool = False, masked_m: Optional[torch.Tensor] = None, + *, + output_q: Optional[torch.Tensor] = None, + output_s: Optional[torch.Tensor] = None, ): assert x.shape[-1] % group_size == 0, ( "the last dimension of `x` cannot be divisible by `group_size`" @@ -588,18 +602,34 @@ def sglang_per_token_group_quant_fp8( # Whole-row group quant is per-token quant; route to the dedicated # kernel (same [T, 1] scale shape) instead of a group kernel that # would need arbitrary group sizes. - return sglang_per_token_quant_fp8(x) + if output_q is not None and output_q.dtype != fp8_dtype: + return _per_token_group_quant_8bit_raw( + x, + group_size, + dtype=output_q.dtype, + output_q=output_q, + output_s=output_s, + ) + return sglang_per_token_quant_fp8(x, output_q=output_q, output_s=output_s) out_shape = (*x.shape[:-1], x.shape[-1] // (2 if fuse_silu_and_mul else 1)) - x_q = torch.empty(out_shape, device=x.device, dtype=fp8_dtype) - x_s = create_per_token_group_quant_fp8_output_scale( - x_shape=out_shape, - device=x.device, - group_size=group_size, - column_major_scales=column_major_scales, - scale_tma_aligned=scale_tma_aligned, - scale_ue8m0=scale_ue8m0, + x_q = ( + output_q + if output_q is not None + else torch.empty(out_shape, device=x.device, dtype=fp8_dtype) + ) + x_s = ( + output_s + if output_s is not None + else create_per_token_group_quant_fp8_output_scale( + x_shape=out_shape, + device=x.device, + group_size=group_size, + column_major_scales=column_major_scales, + scale_tma_aligned=scale_tma_aligned, + scale_ue8m0=scale_ue8m0, + ) ) if x.shape[0] > 0: @@ -764,15 +794,26 @@ def sglang_per_token_group_quant_8bit( def sglang_per_token_quant_fp8( x: torch.Tensor, dtype: torch.dtype = fp8_dtype, + *, + output_q: Optional[torch.Tensor] = None, + output_s: Optional[torch.Tensor] = None, ): assert x.is_contiguous(), "`x` is not contiguous" - x_q = torch.empty_like(x, device=x.device, dtype=dtype) - x_s = torch.empty( - x.shape[0], - 1, - device=x.device, - dtype=torch.float32, + x_q = ( + output_q + if output_q is not None + else torch.empty_like(x, device=x.device, dtype=dtype) + ) + x_s = ( + output_s + if output_s is not None + else torch.empty( + x.shape[0], + 1, + device=x.device, + dtype=torch.float32, + ) ) sgl_per_token_quant_fp8(x, x_q, x_s) diff --git a/python/sglang/srt/layers/quantization/fp8_utils.py b/python/sglang/srt/layers/quantization/fp8_utils.py index 883af39a4c..bf4f10134a 100755 --- a/python/sglang/srt/layers/quantization/fp8_utils.py +++ b/python/sglang/srt/layers/quantization/fp8_utils.py @@ -1663,7 +1663,10 @@ def dequant_mxfp4( def input_to_float8( - x: torch.Tensor, dtype: torch.dtype = fp8_dtype + x: torch.Tensor, + dtype: torch.dtype = fp8_dtype, + *, + out: Optional[torch.Tensor] = None, ) -> Tuple[torch.Tensor, torch.Tensor]: """This function quantizes input values to float8 values with tensor-wise quantization.""" min_val, max_val = x.aminmax() @@ -1677,6 +1680,12 @@ def input_to_float8( fp_max = finfo.max scale = fp_max / amax + if out is not None: + assert out.shape == x.shape and out.dtype == dtype + assert x.dtype in (torch.float16, torch.bfloat16, torch.float32) + # FP32 broadcast avoids a full input copy; absmax bounds the FP8 range. + torch.mul(x, scale.reshape((1,) * x.ndim), out=out) + return out, scale.float().reciprocal() x_scl_sat = (x.float() * scale).clamp(min=-fp_max, max=fp_max) return x_scl_sat.to(dtype).contiguous(), scale.float().reciprocal() diff --git a/python/sglang/srt/model_loader/loader.py b/python/sglang/srt/model_loader/loader.py index d60efacb38..b0f8813b87 100644 --- a/python/sglang/srt/model_loader/loader.py +++ b/python/sglang/srt/model_loader/loader.py @@ -33,7 +33,6 @@ from typing import ( List, Optional, Tuple, - Union, cast, ) @@ -1074,6 +1073,8 @@ class DefaultModelLoader(BaseModelLoader): @staticmethod def postprocess_weights(model, target_device): + if QuantizedRLModelLoader.is_reload_scenario(model): + return for module, quant_method in _modules_with_quant_method(model): # When quant methods need to process weights after loading # (for repacking, quantizing, etc), they expect parameters @@ -1086,6 +1087,8 @@ class DefaultModelLoader(BaseModelLoader): @staticmethod def restore_weights_before_loading(model, target_device): """Undo in-place quant packing so fresh weights can be loaded.""" + if QuantizedRLModelLoader.is_reload_scenario(model): + return for module, quant_method in _modules_with_quant_method(model): # AMX packing and the MXFP4 backend wrappers are duck-typed and cannot restore if isinstance(quant_method, QuantizeMethodBase): @@ -1156,14 +1159,12 @@ class LayeredModelLoader(DefaultModelLoader): class QuantizedRLModelLoader(DefaultModelLoader): - """ - Model loader for RL training with FP8 quantization (profile-free, native SGLang). + """FP8 RL loader for audited Qwen2 (legacy/v2) and Qwen3 native loaders. - Workflow: - 1. Initial load: Load base model → Record state → Apply FP8 quantization - 2. Training Actor in full precision - 3. Reload: Trainer sends full precision weights → Quantize to FP8 → Copy to original memory - 4. Use torch.as_strided to preserve memory locations across reloads + Initial loads record native parameter loaders before quantization. Reloads + invoke the native loader once, deferring FP8 writes; each local destination + is staged and quantized into its existing FP8 weight and scale buffers. + Unsupported native loaders are rejected before initialization loads weights. Usage: --model-path Qwen/Qwen2.5-7B --quantization fp8 --load-format flash_rl @@ -1181,23 +1182,6 @@ class QuantizedRLModelLoader(DefaultModelLoader): "_assert_and_load", ] - # Parameters to skip during FP8 quantization (matches FlashRL's exclude_list) - SKIP_QUANTIZATION_PARAMS = [ - "weight_scale", - "input_scale", - "output_scale", - ".bias", - "lm_head.weight", - "model.norm.weight", - "embed_tokens", # BF16 params - "rotary_emb.inv_freq", - "rotary_emb.cos_cached", - "rotary_emb.sin_cached", - "projector", - "input_layernorm.weight", - "post_attention_layernorm.weight", # LayerNorms - ] - # Stacked parameters (Qwen2): shards loaded separately, then combined STACKED_PARAMS_MAPPING = [ ("qkv_proj", ["q_proj", "k_proj", "v_proj"]), @@ -1215,14 +1199,18 @@ class QuantizedRLModelLoader(DefaultModelLoader): self._initial_load_complete = False def _prepare_weights( - self, model_name_or_path: str, revision: Optional[str], fall_back_to_pt: bool + self, + model_name_or_path: str, + revision: Optional[str], + fall_back_to_pt: bool, + allow_patterns_overrides: Optional[list[str]] = None, ): """Standard weight preparation using base model path.""" logger.info(f"[QuantizedRL] Loading from base model: {model_name_or_path}") temp_config = LoadConfig(load_format=LoadFormat.AUTO) temp_loader = DefaultModelLoader(temp_config) return temp_loader._prepare_weights( - model_name_or_path, revision, fall_back_to_pt + model_name_or_path, revision, fall_back_to_pt, allow_patterns_overrides ) @staticmethod @@ -1242,6 +1230,14 @@ class QuantizedRLModelLoader(DefaultModelLoader): logger.info("[QuantizedRL] Initial load with FP8 quantization") original_load_weights = model.load_weights + native = getattr(original_load_weights, "__func__", original_load_weights) + if getattr(native, "_supports_quantized_rl_reload", None) is not native: + raise ValueError( + f"FlashRL does not support {type(model).__name__}.load_weights: " + "the native loader has not declared the deferred FP8-write contract. " + "Supported native loaders are Qwen2ForCausalLM (legacy/v2) and " + "Qwen3ForCausalLM; use a regular load format for other models." + ) def load_weights_proxy(weights): if QuantizedRLModelLoader.is_reload_scenario(model): @@ -1301,331 +1297,179 @@ class QuantizedRLModelLoader(DefaultModelLoader): return ( hasattr(model, "original_weights_rebuild_keys") and hasattr(model, "recorded_loader") - and getattr(model, "flash_rl_initial_load_complete", False) + and getattr(model, "flash_rl_initial_load_complete", False) is True ) - @staticmethod - def _is_stacked_param(name): - """Check if parameter is stacked (qkv_proj, gate_up_proj).""" - for stacked_name, _ in QuantizedRLModelLoader.STACKED_PARAMS_MAPPING: - if stacked_name in name: - return True - return False - @staticmethod def _resolve_stacked_info(name: str) -> Tuple[str, Optional[str], Optional[Any]]: for target, shard_names in QuantizedRLModelLoader.STACKED_PARAMS_MAPPING: for idx, shard in enumerate(shard_names): - if shard in name: + components = name.split(".") + if shard in components: + components[components.index(shard)] = target shard_id = ( - QuantizedRLModelLoader._QKV_SHARD_ALIASES.get(shard, shard) + QuantizedRLModelLoader._QKV_SHARD_ALIASES[shard] if target == "qkv_proj" else idx ) - return name.replace(shard, target), target, shard_id + return ".".join(components), target, shard_id return name, None, None - @staticmethod - def _store_quantized_scale( - scale_store: Dict[str, Union[torch.Tensor, Dict[Any, torch.Tensor]]], - name: str, - scale: torch.Tensor, - ) -> None: - param_name, stacked_key, shard_id = ( - QuantizedRLModelLoader._resolve_stacked_info(name) - ) - if stacked_key is None: - scale_store[param_name] = scale - else: - shard_dict = scale_store.setdefault(param_name, {}) - assert isinstance(shard_dict, dict) - shard_dict[shard_id] = scale - - @staticmethod - def _apply_scale_update( - all_params: Dict[str, torch.nn.Parameter], - param_name: str, - scale_info: Union[torch.Tensor, Dict[Any, torch.Tensor], None], - ) -> None: - if scale_info is None: - return - # Get tp rank and size - tp_rank = get_parallel().tp_rank - tp_size = get_parallel().tp_size - - def _get_tp_sharded_scale(full_scale_tensor): - """Get tp sharded scale from full scale tensor""" - if tp_size == 1: - return full_scale_tensor - - full_dim = full_scale_tensor.shape[0] - shard_dim = full_dim // tp_size - start_idx = tp_rank * shard_dim - end_idx = start_idx + shard_dim - return full_scale_tensor[start_idx:end_idx] - - if param_name.endswith(".weight"): - scale_param_name = f"{param_name[:-7]}.weight_scale" - else: - scale_param_name = f"{param_name}.weight_scale" - - scale_param = all_params.get(scale_param_name) - if scale_param is None: - logger.warning( - "[QuantizedRL] Scale parameter not found: %s", scale_param_name - ) - return - if isinstance(scale_info, torch.Tensor): - new_scale = scale_info.t().contiguous() - if scale_param.data.shape == new_scale.shape: - scale_param.data.copy_(new_scale) - else: - logger.warning( - "[QuantizedRL] Scale shape mismatch for %s: expected %s, got %s", - scale_param_name, - scale_param.data.shape, - new_scale.shape, - ) - else: - stacked_key = next( - ( - target - for target, _ in QuantizedRLModelLoader.STACKED_PARAMS_MAPPING - if target in param_name - ), - None, - ) - shard_names = next( - ( - names - for target, names in QuantizedRLModelLoader.STACKED_PARAMS_MAPPING - if target == stacked_key - ), - [], - ) - rows_per_shard = scale_param.data.shape[-1] // max(len(shard_names), 1) - if rows_per_shard * len(shard_names) != scale_param.data.shape[-1]: - logger.warning( - f"Scale param shape {scale_param.data.shape[-1]} not divisible by {len(shard_names)}" - ) - offset = 0 - for idx, shard in enumerate(shard_names): - shard_id = ( - QuantizedRLModelLoader._QKV_SHARD_ALIASES.get(shard, shard) - if stacked_key == "qkv_proj" - else idx - ) - shard_scale = scale_info.get(shard_id) - shard_scale = _get_tp_sharded_scale(shard_scale) - if shard_scale is None: - offset += rows_per_shard - continue - shard_rows = shard_scale.shape[0] - start = offset - end = start + shard_rows - scale_param.data[..., start:end] = shard_scale.t().contiguous() - offset = end - @staticmethod def rebinding_and_load_weights(model, first_time_load_weights, weights): - """ - Reload: VERL sends BF16 → Quantize to FP8 → Copy to original memory. - - Flow: Reset params → Restore attributes → Quantize in iterator → Load → Copy back - """ - logger.info("[QuantizedRL] Reload: Updating weights with FP8 quantization") + """Shard new BF16 weights natively, then quantize into their original buffers.""" + from sglang.kernels.ops.quantization.fp8_kernel import ( + per_token_group_quant_fp8, + ) + from sglang.srt.layers.quantization.fp8_utils import input_to_float8 weights_list = list(weights) + for name, _ in weights_list: + if name.startswith(("layers.", "embed_tokens.", "norm.")): + raise ValueError( + f"FlashRL reload requires canonical checkpoint names: {name!r} " + "must include the 'model.' prefix." + ) updated_param_names, is_last_update = ( QuantizedRLModelLoader._get_updated_params(weights_list, model) ) - - # Save current FP8 parameter data pointers - existing_params = dict(model.named_parameters()) - current_param_data = {} - for name in updated_param_names: - if name in existing_params: - current_param_data[name] = existing_params[name].data - - # Reset to pre-quantization shape using torch.as_strided - # Keeps same storage, just changes view - critical for memory preservation - for name, rebuild_info in model.original_weights_rebuild_keys.items(): - if name in updated_param_names and name in existing_params: - existing_params[name].data = torch.as_strided( - # Note: avoid clone here - existing_params[name].data.clone(), - rebuild_info["shape"], - rebuild_info["stride"], - ) - - # Restore weight loader attributes (only if missing) - for k, loader_dict in model.recorded_loader.items(): - for param_name, loader in loader_dict.items(): - if param_name in updated_param_names and param_name in existing_params: - param = existing_params[param_name] - if not hasattr(param, k): - if callable(loader): - if hasattr(loader, "__self__"): - setattr(param, k, loader) - else: - setattr( - param, - k, - QuantizedRLModelLoader._bind_method_to_cls( - loader, param - ), - ) - else: - setattr(param, k, loader) - - del existing_params - - # Quantize BF16 weights to FP8 in iterator (before weight_loader) - # Store scales for later update - quantized_scales: Dict[str, Union[torch.Tensor, Dict[Any, torch.Tensor]]] = {} - - def quantize_weights_iterator(weights_iter): - """Quantize individual shards before weight_loader stacks them.""" - from sglang.kernels.ops.quantization.fp8_kernel import ( - per_token_group_quant_fp8, + params = dict(model.named_parameters()) + grouped = collections.defaultdict(list) + complete_params = set() + for source_name, weight in weights_list: + name, _, shard_id = QuantizedRLModelLoader._resolve_stacked_info( + source_name ) + grouped[name].append((source_name, weight)) + if shard_id is None: + complete_params.add(name) - for name, weight in weights_iter: - if any( - skip in name - for skip in QuantizedRLModelLoader.SKIP_QUANTIZATION_PARAMS - ): - logger.info(f"[QuantizedRL] Skip: {name} ({weight.dtype})") - yield (name, weight) - elif weight.dtype in [torch.bfloat16, torch.float32, torch.float16]: - qweight, scale = per_token_group_quant_fp8(weight, weight.shape[-1]) - logger.info(f"[QuantizedRL] Quantize: {name} {weight.dtype}→FP8") - QuantizedRLModelLoader._store_quantized_scale( - quantized_scales, name, scale + row_masks = {} + # Check coverage before modifying any model parameter. Per-tensor + # quantization needs the original BF16 values for every destination row. + for name, sources in grouped.items(): + if name not in params or params[name].dtype not in ( + torch.float8_e4m3fn, + torch.float8_e4m3fnuz, + ): + continue + rebuild = model.original_weights_rebuild_keys[name] + layer = model.get_submodule(name.rpartition(".")[0]) + per_channel = layer.weight_scale.shape == (1, rebuild["shape"][0]) + if name in complete_params: + row_masks[name] = (None, per_channel) + continue + marker = nn.Parameter( + torch.zeros((rebuild["shape"][0], 1), device="cpu"), requires_grad=False + ) + marker.output_dim = 0 + for source_name, weight in sources: + _, _, shard_id = QuantizedRLModelLoader._resolve_stacked_info( + source_name + ) + args = () if shard_id is None else (shard_id,) + layer.weight_loader( + marker, torch.ones((weight.shape[0], 1), device="cpu"), *args + ) + rows = marker.data[:, 0].bool() + if not per_channel and not rows.all(): + raise ValueError( + f"Partial per-tensor FP8 update for {name}: supply all destination " + "rows in the same update call, including inside a weight-update session." + ) + row_masks[name] = (rows, per_channel) + + # Defer only FP8 writes so one native call preserves cross-key lookups + # and the order of non-FP8 loads, including tied embeddings. + writes = collections.defaultdict(list) + native_loaders = {} + for name in row_masks: + param = params[name] + for key, loaders in model.recorded_loader.items(): + if name in loaders and not hasattr(param, key): + loader = loaders[name] + setattr( + param, + key, + QuantizedRLModelLoader._bind_method_to_cls(loader, param) + if callable(loader) + else loader, ) - yield (name, qweight) + native_loaders[name] = param.weight_loader + + def record_write(*args, _name=name, **kwargs): + writes[_name].append((args, kwargs)) + + param.weight_loader = record_write + try: + first_time_load_weights(weights_list) + finally: + for name, loader in native_loaders.items(): + params[name].weight_loader = loader + + for name, calls in writes.items(): + param = params[name] + data = param.data + rebuild = model.original_weights_rebuild_keys[name] + layer = model.get_submodule(name.rpartition(".")[0]) + rows, per_channel = row_masks[name] + complete = rows is None or bool(rows.all()) + staging = torch.empty_strided( + rebuild["shape"], + rebuild["stride"], + dtype=rebuild["dtype"], + device=data.device, + ) + param.data = staging + try: + for args, kwargs in calls: + native_loaders[name](*args, **kwargs) + local_weight = torch.as_strided( + data, rebuild["shape"], rebuild["stride"] + ) + if per_channel: + # Untouched rows never pass through dequantization. + if complete: + ranges = [(0, staging.shape[0])] + else: + padded = torch.cat((rows.new_zeros(1), rows, rows.new_zeros(1))) + edges = (padded[1:] != padded[:-1]).nonzero().flatten().tolist() + ranges = zip(edges[::2], edges[1::2]) + for begin, end in ranges: + per_token_group_quant_fp8( + staging[begin:end], + staging.shape[-1], + output_q=local_weight[begin:end], + output_s=layer.weight_scale.data[:, begin:end].t(), + ) else: - logger.info(f"[QuantizedRL] Keep: {name} ({weight.dtype})") - yield (name, weight) + _, scale = input_to_float8( + staging, dtype=data.dtype, out=local_weight + ) + layer.weight_scale.data.copy_(scale) + finally: + param.data = data + # No BF16 destination survives to the next destination or call. + del staging - # Load quantized weights (weight_loader stacks FP8 shards) - first_time_load_weights(quantize_weights_iterator(iter(weights_list))) - - # Copy back to original FP8 memory locations and update scales - all_params = dict(model.named_parameters()) - - for name in updated_param_names: - if name not in all_params or name not in current_param_data: - continue - if any( - skip in name for skip in QuantizedRLModelLoader.SKIP_QUANTIZATION_PARAMS - ): - continue - - new_param = all_params[name] - old_fp8_data = current_param_data[name] - - # Handle embeddings/lm_head (BF16) and quantized weights (FP8) - if "embed_tokens" in name or "lm_head" in name: - old_fp8_data.copy_(new_param.data) - new_param.data = old_fp8_data - elif ( - new_param.dtype == torch.float8_e4m3fn - and old_fp8_data.dtype == torch.float8_e4m3fn - ): - # FP8: Use strided view for transposed storage - strided_data = torch.as_strided( - new_param.data, old_fp8_data.shape, old_fp8_data.stride() - ) - old_fp8_data.copy_(strided_data) - new_param.data = old_fp8_data - QuantizedRLModelLoader._apply_scale_update( - all_params, - name, - quantized_scales.get(name), - ) - elif new_param.dtype == old_fp8_data.dtype: - # Same dtype (LayerNorm, etc.): Direct copy - old_fp8_data.copy_(new_param.data) - new_param.data = old_fp8_data - else: - raise RuntimeError( - f"Unexpected dtype mismatch for {name}: " - f"new={new_param.dtype}, old={old_fp8_data.dtype}" - ) - - # Cleanup - del current_param_data if is_last_update: gc.collect() current_platform.empty_cache() - - logger.info("[QuantizedRL] Reload complete") return updated_param_names, is_last_update @staticmethod def _get_updated_params(weights_list, model): - """Identify which parameters need updating from incoming weights.""" - stacked_params_mapping = [ - ("qkv_proj", "q_proj", "q"), - ("qkv_proj", "k_proj", "k"), - ("qkv_proj", "v_proj", "v"), - ("gate_up_proj", "gate_proj", 0), - ("gate_up_proj", "up_proj", 1), - ] - - params_dict = dict(model.named_parameters()) - updated_params = set() + """Resolve checkpoint names to parameters owned by this model shard.""" + params = dict(model.named_parameters()) + updated = set() is_last_update = False - for name, _ in weights_list: if name == "lm_head.weight": is_last_update = True - - if any( - skip in name for skip in QuantizedRLModelLoader.SKIP_QUANTIZATION_PARAMS - ): - continue - - from sglang.srt.layers.utils import get_layer_id - - # Skip params outside layer range (for pipeline parallelism) - layer_id = get_layer_id(name) - if ( - layer_id is not None - and hasattr(model, "start_layer") - and (layer_id < model.start_layer or layer_id >= model.end_layer) - ): - continue - - # Skip tied embeddings and vision tower params - if ( - hasattr(model, "config") - and model.config.tie_word_embeddings - and "lm_head.weight" in name - ): - continue - if name.startswith("model.vision_tower") and name not in params_dict: - continue - - # Map stacked param shards (q/k/v_proj → qkv_proj) - mapped = False - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name in name: - name = name.replace(weight_name, param_name) - if name.endswith(".bias") and name not in params_dict: - continue - updated_params.add(name) - mapped = True - break - - if not mapped: - if name.endswith(".bias") and name not in params_dict: - continue - if name in params_dict: - updated_params.add(name) - - return list(updated_params), is_last_update + name, _, _ = QuantizedRLModelLoader._resolve_stacked_info(name) + if name in params: + updated.add(name) + return sorted(updated), is_last_update class DummyModelLoader(BaseModelLoader): diff --git a/python/sglang/srt/model_loader/weight_utils.py b/python/sglang/srt/model_loader/weight_utils.py index 82951f9f19..3a9f2aa127 100644 --- a/python/sglang/srt/model_loader/weight_utils.py +++ b/python/sglang/srt/model_loader/weight_utils.py @@ -1474,6 +1474,18 @@ def gguf_quant_weights_iterator( yield name, param +def supports_quantized_rl_reload(func: Callable) -> Callable: + """Allow FlashRL to defer FP8 parameter writes until this method returns. + + Checkpoint names must use the parameter names or QKV/gate-up aliases supported + by QuantizedRLModelLoader. FP8 writes must go through param.weight_loader; + the method must not read their values afterward, and source tensors must + remain unchanged until return. Non-FP8 parameters are loaded immediately. + """ + func._supports_quantized_rl_reload = func + return func + + def default_weight_loader(param: torch.Tensor, loaded_weight: torch.Tensor) -> None: """Default weight loader.""" if param.numel() == 1 and loaded_weight.numel() == 1: diff --git a/python/sglang/srt/models/qwen2.py b/python/sglang/srt/models/qwen2.py index 545b6618df..9f89bd21f9 100644 --- a/python/sglang/srt/models/qwen2.py +++ b/python/sglang/srt/models/qwen2.py @@ -55,6 +55,7 @@ from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTe from sglang.srt.model_loader.weight_utils import ( default_weight_loader, kv_cache_scales_loader, + supports_quantized_rl_reload, ) from sglang.srt.platforms import current_platform from sglang.srt.runtime_context import get_exec, get_parallel @@ -616,6 +617,7 @@ class Qwen2ForCausalLM(nn.Module): def end_layer(self): return self.model.end_layer + @supports_quantized_rl_reload def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): from sglang.srt.environ import envs diff --git a/python/sglang/srt/models/qwen3.py b/python/sglang/srt/models/qwen3.py index f793c90bb5..78574b6eef 100644 --- a/python/sglang/srt/models/qwen3.py +++ b/python/sglang/srt/models/qwen3.py @@ -32,6 +32,7 @@ from sglang.srt.model_executor.forward_context import get_token_to_kv_pool from sglang.srt.model_loader.weight_utils import ( default_weight_loader, maybe_remap_kv_scale_name, + supports_quantized_rl_reload, ) from sglang.srt.models.qwen2 import Qwen2MLP as Qwen3MLP from sglang.srt.models.qwen2 import Qwen2Model @@ -620,6 +621,7 @@ class Qwen3ForCausalLM(nn.Module): def end_layer(self): return self.model.end_layer + @supports_quantized_rl_reload def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): stacked_params_mapping = [ # (param_name, shard_name, shard_id) diff --git a/test/registered/unit/model_loader/test_default_model_loader.py b/test/registered/unit/model_loader/test_default_model_loader.py index da5b3676ae..c70c0da948 100644 --- a/test/registered/unit/model_loader/test_default_model_loader.py +++ b/test/registered/unit/model_loader/test_default_model_loader.py @@ -41,6 +41,20 @@ class TestDefaultModelLoader(CustomTestCase): self.assertEqual(events, ["load", "postprocess"]) + def test_reload_requires_explicit_initial_load_marker(self): + self.assertFalse(loader_mod.QuantizedRLModelLoader.is_reload_scenario(Mock())) + model = torch.nn.Module() + model.original_weights_rebuild_keys = set() + model.recorded_loader = {} + for marker in (None, False, 1, Mock()): + with self.subTest(marker=marker): + model.flash_rl_initial_load_complete = marker + self.assertFalse( + loader_mod.QuantizedRLModelLoader.is_reload_scenario(model) + ) + model.flash_rl_initial_load_complete = True + self.assertTrue(loader_mod.QuantizedRLModelLoader.is_reload_scenario(model)) + def test_boot_paths_preserve_load_order_and_custom_override(self): class CustomModelLoader(DefaultModelLoader): def load_weights_and_postprocess(self, model, weights, target_device): diff --git a/test/registered/unit/model_loader/test_quantized_reload_scale_layout.py b/test/registered/unit/model_loader/test_quantized_reload_scale_layout.py new file mode 100644 index 0000000000..772f924e7d --- /dev/null +++ b/test/registered/unit/model_loader/test_quantized_reload_scale_layout.py @@ -0,0 +1,518 @@ +"""Quantized reload scales follow their destination projection's layout.""" + +import math +import socket +import unittest +from contextlib import nullcontext +from unittest.mock import patch + +import torch +from transformers import Qwen2Config, Qwen3Config + +from sglang.srt.configs.load_config import LoadConfig, LoadFormat +from sglang.srt.distributed.parallel_state import ( + destroy_distributed_environment, + destroy_model_parallel, + init_distributed_environment, + initialize_model_parallel, +) +from sglang.srt.layers.dp_attention import initialize_dp_attention +from sglang.srt.layers.quantization.fp8 import Fp8Config +from sglang.srt.model_loader.loader import DefaultModelLoader, QuantizedRLModelLoader +from sglang.srt.models.qwen2 import Qwen2ForCausalLM +from sglang.srt.models.qwen3 import Qwen3ForCausalLM +from sglang.srt.runtime_context import SpawnRanks, get_parallel, publish, reset_context +from sglang.srt.server_args import ServerArgs +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=20, stage="base-b", runner_config="1-gpu-small") + + +def values(shape, version=0): + base = ( + torch.arange(math.prod(shape), device="cuda").reshape(shape) % 31 - 15 + ).float() / 64 + if len(shape) == 2: + base *= ((torch.arange(shape[0], device="cuda") + version) % 17 + 1)[:, None] + base *= (torch.arange(shape[1], device="cuda") // max(shape[1] // 4, 1) + 1)[ + None, : + ] + return (base + version / 128).to(torch.bfloat16) + + +def checkpoint(version=0, kind="qwen2", kv_heads=8, num_layers=1): + weights = [] + for name, shape in { + "model.embed_tokens.weight": (512, 512), + "lm_head.weight": (512, 512), + "model.norm.weight": (512,), + "model.layers.0.input_layernorm.weight": (512,), + "model.layers.0.post_attention_layernorm.weight": (512,), + **{ + f"model.layers.0.self_attn.{p}.weight": ( + kv_heads * 64 if p in ("k_proj", "v_proj") else 512, + 512, + ) + for p in ("q_proj", "k_proj", "v_proj", "o_proj") + }, + **{ + f"model.layers.0.self_attn.{p}.bias": ( + kv_heads * 64 if p in ("k_proj", "v_proj") else 512, + ) + for p in ("q_proj", "k_proj", "v_proj") + if kind == "qwen2" + }, + **{ + f"model.layers.0.self_attn.{p}.weight": (64,) + for p in ("q_norm", "k_norm") + if kind == "qwen3" + }, + **{ + f"model.layers.0.mlp.{p}.weight": (1024, 512) + for p in ("gate_proj", "up_proj") + }, + "model.layers.0.mlp.down_proj.weight": (512, 1024), + }.items(): + weights.append((name, values(shape, version))) + first_layer = [(n, t) for n, t in weights if n.startswith("model.layers.0.")] + for layer in range(1, num_layers): + weights.extend( + (n.replace("model.layers.0.", f"model.layers.{layer}."), t) + for n, t in first_layer + ) + return weights + + +def build_model( + kind="qwen2", *, kv_heads=8, version=0, weights=None, per_tensor=False, num_layers=1 +): + config_class, model_class = { + "qwen2": (Qwen2Config, Qwen2ForCausalLM), + "qwen3": (Qwen3Config, Qwen3ForCausalLM), + }[kind] + config = config_class( + hidden_size=512, + intermediate_size=1024, + num_hidden_layers=num_layers, + num_attention_heads=8, + num_key_value_heads=kv_heads, + head_dim=64, + vocab_size=512, + max_position_embeddings=32, + tie_word_embeddings=False, + pad_token_id=0, + bos_token_id=1, + eos_token_id=2, + ) + with torch.device("cuda"): + model = model_class(config, Fp8Config(is_checkpoint_fp8_serialized=False)) + if per_tensor: + for module in model.modules(): + method = getattr(module, "quant_method", None) + if hasattr(method, "cutlass_fp8_supported"): + method.cutlass_fp8_supported = False + method.use_marlin = False + loader = QuantizedRLModelLoader(LoadConfig(load_format=LoadFormat.FLASH_RL)) + loader.load_weights_and_postprocess( + model, + checkpoint(version, kind, kv_heads, num_layers) if weights is None else weights, + torch.device("cuda"), + ) + return model + + +def loading_scope(changed): + if not changed: + return nullcontext() + return get_parallel().override( + tp_size=4, + tp_rank=3, + tp_group=None, + attn_tp_size=4, + attn_tp_rank=3, + attn_tp_group=None, + attn_dp_size=1, + attn_dp_rank=0, + attn_cp_size=1, + attn_cp_rank=0, + moe_tp_size=4, + moe_tp_rank=3, + moe_ep_size=1, + moe_ep_rank=0, + moe_ep_group=None, + moe_dp_size=1, + moe_dp_rank=0, + ) + + +def assert_same_parameters(actual, expected): + expected_params = dict(expected.named_parameters()) + for name, param in actual.named_parameters(): + reference = expected_params[name] + assert param.shape == reference.shape, name + assert param.stride() == reference.stride(), name + assert param.dtype == reference.dtype, name + torch.testing.assert_close( + param.contiguous().reshape(-1).view(torch.uint8), + reference.contiguous().reshape(-1).view(torch.uint8), + rtol=0, + atol=0, + msg=name, + ) + + +def check_reload(model, version=13, changed=False, per_tensor=False): + kind, kv_heads = model.config.model_type, model.config.num_key_value_heads + source = checkpoint(version, kind, kv_heads) + pointers = {n: p.data_ptr() for n, p in model.named_parameters()} + DefaultModelLoader.restore_weights_before_loading(model, torch.device("cuda")) + with loading_scope(changed): + model.load_weights(source) + DefaultModelLoader.postprocess_weights(model, torch.device("cuda")) + for name, param in model.named_parameters(): + assert param.data_ptr() == pointers[name], name + reference = build_model( + kind, kv_heads=kv_heads, version=version, per_tensor=per_tensor + ) + assert_same_parameters(model, reference) + return model + + +@unittest.skipUnless(torch.cuda.is_available(), "needs a GPU") +class TestQuantizedReloadScaleLayout(CustomTestCase): + def setUp(self): + if torch.distributed.is_initialized(): + self.skipTest("requires an isolated distributed test process") + reset_context() + self.addCleanup(reset_context) + original = torch.get_default_dtype() + self.addCleanup(torch.set_default_dtype, original) + torch.set_default_dtype(torch.bfloat16) + server = ServerArgs(model_path="dummy", device="cuda", tp_size=1) + publish(server, role="test", ranks=SpawnRanks(world_rank=0, gpu_id=0)) + torch.cuda.set_device(0) + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + init_distributed_environment( + world_size=1, + rank=0, + local_rank=0, + distributed_init_method=f"tcp://127.0.0.1:{port}", + ) + self.addCleanup(destroy_distributed_environment) + initialize_model_parallel() + self.addCleanup(destroy_model_parallel) + initialize_dp_attention(server) + + def test_native_model_reload_after_scope_exit(self): + model = build_model() + check_reload(model, changed=True) + check_reload(model, version=29, changed=True) + + def test_native_attention_group_projection_reload_after_scope_exit(self): + model = build_model("qwen3") + check_reload(model, changed=True) + check_reload(model, version=29, changed=True) + + def test_unequal_qkv_and_replicated_kv_match_cold_load(self): + for kv_heads in (1, 2, 4): + with self.subTest(kv_heads=kv_heads): + check_reload(build_model("qwen3", kv_heads=kv_heads), changed=True) + + def test_partial_qkv_and_mlp_updates_preserve_other_rows(self): + for kind in ("qwen2", "qwen3"): + for shard in ("q_proj", "k_proj", "v_proj", "gate_proj", "up_proj"): + with self.subTest(kind=kind, shard=shard): + model = build_model(kind, kv_heads=1) + before = checkpoint(kind=kind, kv_heads=1) + update = { + n: t for n, t in checkpoint(13, kind, 1) if f".{shard}." in n + } + pointers = {n: p.data_ptr() for n, p in model.named_parameters()} + with loading_scope(True): + model.load_weights(list(update.items())) + expected = [(n, update.get(n, t)) for n, t in before] + reference = build_model(kind, kv_heads=1, weights=expected) + assert_same_parameters(model, reference) + for name, param in model.named_parameters(): + self.assertEqual(param.data_ptr(), pointers[name], name) + + def test_per_tensor_quantization_matches_native_cold_load(self): + model = build_model("qwen3", kv_heads=1, per_tensor=True) + check_reload(model, version=0, changed=True, per_tensor=True) + check_reload(model, version=13, changed=True, per_tensor=True) + + def test_partial_per_tensor_update_rejected_before_any_parameter_changes(self): + for shard in ("q_proj", "k_proj", "v_proj", "gate_proj", "up_proj"): + with self.subTest(shard=shard): + model = build_model("qwen3", kv_heads=1, per_tensor=True) + pointers = {n: p.data_ptr() for n, p in model.named_parameters()} + source = checkpoint(13, "qwen3", 1) + update = [ + (n, t) + for n, t in source + if n == "model.norm.weight" or f".{shard}." in n + ] + with self.assertRaisesRegex( + ValueError, "Partial per-tensor FP8 update" + ): + model.load_weights(update) + assert_same_parameters( + model, build_model("qwen3", kv_heads=1, per_tensor=True) + ) + for name, param in model.named_parameters(): + self.assertEqual(param.data_ptr(), pointers[name], name) + + def test_per_tensor_session_accepts_complete_destinations(self): + model = build_model("qwen3", kv_heads=1, per_tensor=True) + pointers = {n: p.data_ptr() for n, p in model.named_parameters()} + DefaultModelLoader.restore_weights_before_loading(model, torch.device("cuda")) + grouped = {} + for name, tensor in checkpoint(13, "qwen3", 1): + target, _, _ = QuantizedRLModelLoader._resolve_stacked_info(name) + grouped.setdefault(target, []).append((name, tensor)) + for source in reversed(list(grouped.values())): + with loading_scope(True): + model.load_weights(source) + self.assertFalse(hasattr(model, "_quantized_rl_pending")) + DefaultModelLoader.postprocess_weights(model, torch.device("cuda")) + assert_same_parameters( + model, build_model("qwen3", kv_heads=1, version=13, per_tensor=True) + ) + for name, param in model.named_parameters(): + self.assertEqual(param.data_ptr(), pointers[name], name) + + def test_per_tensor_session_rejects_partial_chunks_without_host_cache(self): + model = build_model("qwen3", kv_heads=1, per_tensor=True) + original = build_model("qwen3", kv_heads=1, per_tensor=True) + DefaultModelLoader.restore_weights_before_loading(model, torch.device("cuda")) + for shard in ("q_proj", "k_proj", "v_proj", "gate_proj", "up_proj"): + update = [ + (n, t) + for n, t in checkpoint(13, "qwen3", 1) + if n == "model.norm.weight" or f".{shard}." in n + ] + with self.assertRaisesRegex(ValueError, "Partial per-tensor FP8 update"): + model.load_weights(update) + assert_same_parameters(model, original) + self.assertFalse(hasattr(model, "_quantized_rl_pending")) + model.load_weights(checkpoint(13, "qwen3", 1)) + DefaultModelLoader.postprocess_weights(model, torch.device("cuda")) + assert_same_parameters( + model, build_model("qwen3", kv_heads=1, version=13, per_tensor=True) + ) + + def test_interleaved_layers_cannot_accumulate_pending_bf16_matrices(self): + model = build_model("qwen3", kv_heads=1, per_tensor=True, num_layers=16) + snapshot = { + n: p.contiguous().reshape(-1).view(torch.uint8).clone() + for n, p in model.named_parameters() + } + pointers = {n: p.data_ptr() for n, p in model.named_parameters()} + DefaultModelLoader.restore_weights_before_loading(model, torch.device("cuda")) + source = [ + (n, t) + for n, t in checkpoint(13, "qwen3", 1, 16) + if ".q_proj." in n or n == "model.norm.weight" + ] + with self.assertRaisesRegex(ValueError, "Partial per-tensor FP8 update"): + model.load_weights(source) + self.assertFalse(hasattr(model, "_quantized_rl_pending")) + for name, param in model.named_parameters(): + self.assertEqual(param.data_ptr(), pointers[name], name) + torch.testing.assert_close( + param.contiguous().reshape(-1).view(torch.uint8), + snapshot[name], + rtol=0, + atol=0, + ) + DefaultModelLoader.postprocess_weights(model, torch.device("cuda")) + + def test_noncanonical_reload_names_are_rejected_before_mutation(self): + model = build_model("qwen3", kv_heads=1) + sources = dict(checkpoint(13, "qwen3", 1)) + snapshot = { + n: (p.data_ptr(), p.contiguous().view(torch.uint8).clone()) + for n, p in model.named_parameters() + } + for canonical in ( + "model.layers.0.self_attn.q_proj.weight", + "model.embed_tokens.weight", + "model.norm.weight", + ): + with self.subTest(name=canonical): + update = [ + ("model.norm.weight", sources["model.norm.weight"]), + (canonical.removeprefix("model."), sources[canonical]), + ] + with self.assertRaisesRegex(ValueError, "canonical checkpoint names"): + model.load_weights(update) + for name, param in model.named_parameters(): + pointer, data = snapshot[name] + self.assertEqual(param.data_ptr(), pointer) + torch.testing.assert_close( + param.contiguous().view(torch.uint8), data, rtol=0, atol=0 + ) + + def test_qwen2_walker_reload_matches_cold_load(self): + from sglang.srt.environ import envs + + with patch.object( + envs.SGLANG_ENABLE_WEIGHT_LOADER_V2, "get", return_value=True + ): + for per_tensor in (False, True): + with self.subTest(per_tensor=per_tensor): + check_reload( + build_model("qwen2", kv_heads=1, per_tensor=per_tensor), + changed=True, + per_tensor=per_tensor, + ) + + def test_qwen2_walker_tied_head_post_copy_occurs_once(self): + from sglang.srt.environ import envs + + model = build_model("qwen2", kv_heads=1) + model.config.tie_word_embeddings = True + head = model.lm_head.weight + original = head.weight_loader + copies = [] + + def load_head(*args, **kwargs): + copies.append(True) + return original(*args, **kwargs) + + head.weight_loader = load_head + source = checkpoint(13, "qwen2", 1) + embedding = dict(source)["model.embed_tokens.weight"] + with patch.object( + envs.SGLANG_ENABLE_WEIGHT_LOADER_V2, "get", return_value=True + ): + model.load_weights(source) + self.assertEqual(len(copies), 1) + torch.testing.assert_close(head, embedding, rtol=0, atol=0) + expected = [(n, embedding if n == "lm_head.weight" else t) for n, t in source] + assert_same_parameters( + model, build_model("qwen2", kv_heads=1, weights=expected) + ) + + def test_unsupported_native_loaders_fail_before_initialization(self): + from sglang.srt.models.mimo_v2 import MiMoV2ForCausalLM + from sglang.srt.models.whisper import WhisperForConditionalGeneration + + for native in ( + WhisperForConditionalGeneration.load_weights, + MiMoV2ForCausalLM.load_weights, + ): + with self.subTest(native=native.__qualname__): + model = torch.nn.Linear(4, 4, device="cuda") + before = model.weight.clone() + model.load_weights = native.__get__(model) + original = model.load_weights + loader = QuantizedRLModelLoader( + LoadConfig(load_format=LoadFormat.FLASH_RL) + ) + with self.assertRaisesRegex(ValueError, "deferred FP8-write contract"): + loader.load_weights_and_postprocess(model, [], torch.device("cuda")) + self.assertIs(model.load_weights, original) + self.assertFalse(hasattr(model, "original_weights_rebuild_keys")) + torch.testing.assert_close(model.weight, before, rtol=0, atol=0) + + def test_failed_native_load_restores_fp8_storage(self): + model = build_model("qwen3", kv_heads=1) + pointers = {n: p.data_ptr() for n, p in model.named_parameters()} + + def fail(weights): + Qwen3ForCausalLM.load_weights(model, weights) + self.assertEqual( + model.model.layers[0].self_attn.qkv_proj.weight.dtype, + torch.float8_e4m3fn, + ) + raise RuntimeError("native loader failed") + + with self.assertRaisesRegex(RuntimeError, "native loader failed"): + QuantizedRLModelLoader.rebinding_and_load_weights( + model, + fail, + [ + (n, t) + for n, t in checkpoint(13, "qwen3", 1) + if any(f".{p}." in n for p in ("q_proj", "k_proj", "v_proj")) + ], + ) + assert_same_parameters(model, build_model("qwen3", kv_heads=1)) + for name, param in model.named_parameters(): + self.assertEqual(param.data_ptr(), pointers[name], name) + + def test_failed_deferred_parameter_write_restores_storage_and_loader(self): + model = build_model("qwen3", kv_heads=1) + param = model.model.layers[0].self_attn.qkv_proj.weight + pointer = param.data_ptr() + snapshot = param.contiguous().view(torch.uint8).clone() + + def fail(target, weight, shard): + self.assertEqual(target.dtype, torch.bfloat16) + raise RuntimeError("deferred write failed") + + param.weight_loader = fail + source = [ + (n, t) + for n, t in checkpoint(13, "qwen3", 1) + if any(f".{shard}_proj." in n for shard in ("q", "k", "v")) + ] + with self.assertRaisesRegex(RuntimeError, "deferred write failed"): + model.load_weights(source) + self.assertEqual(param.data_ptr(), pointer) + self.assertIs(param.weight_loader, fail) + torch.testing.assert_close( + param.contiguous().view(torch.uint8), snapshot, rtol=0, atol=0 + ) + + def test_fnuz_per_tensor_reload_matches_cold_load(self): + from sglang.srt.layers.quantization import fp8 + from sglang.srt.layers.quantization.fp8_utils import input_to_float8 + + with patch.object( + fp8, + "input_to_float8", + lambda x: input_to_float8(x, dtype=torch.float8_e4m3fnuz), + ): + model = build_model("qwen3", kv_heads=1, per_tensor=True) + check_reload(model, version=13, per_tensor=True) + + def test_per_tensor_output_buffer_matches_reference_without_weight_temporary(self): + from sglang.srt.layers.quantization.fp8_utils import input_to_float8 + + x = values((2048, 2048), 13) + for dtype in (torch.float8_e4m3fn, torch.float8_e4m3fnuz): + with self.subTest(dtype=dtype): + expected, expected_scale = input_to_float8(x, dtype=dtype) + output = torch.empty_like(x, dtype=dtype) + input_to_float8(x, dtype=dtype, out=output) + torch.cuda.synchronize() + baseline = torch.cuda.memory_allocated() + torch.cuda.reset_peak_memory_stats() + actual, scale = input_to_float8(x, dtype=dtype, out=output) + torch.cuda.synchronize() + extra = torch.cuda.max_memory_allocated() - baseline + self.assertLess(extra, 64 * 1024) + self.assertEqual(actual.data_ptr(), output.data_ptr()) + torch.testing.assert_close( + actual.view(torch.uint8), expected.view(torch.uint8), rtol=0, atol=0 + ) + torch.testing.assert_close(scale, expected_scale, rtol=0, atol=0) + + def test_empty_session_keeps_native_layout_and_memory(self): + model = build_model("qwen3", kv_heads=1) + pointers = {n: p.data_ptr() for n, p in model.named_parameters()} + DefaultModelLoader.restore_weights_before_loading(model, torch.device("cuda")) + model.load_weights([]) + DefaultModelLoader.postprocess_weights(model, torch.device("cuda")) + assert_same_parameters(model, build_model("qwen3", kv_heads=1)) + for name, param in model.named_parameters(): + self.assertEqual(param.data_ptr(), pointers[name], name) + + +if __name__ == "__main__": + unittest.main()