[Fix] Quantize reloaded FP8 weights after native sharding (#42585)
This commit is contained in:
@@ -214,6 +214,9 @@ def _per_token_group_quant_8bit_raw(
|
||||
column_major_scales: bool = False,
|
||||
scale_tma_aligned: bool = False,
|
||||
scale_ue8m0: bool = False,
|
||||
*,
|
||||
output_q: Optional[torch.Tensor] = None,
|
||||
output_s: Optional[torch.Tensor] = None,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Function to perform per-token-group quantization on an input tensor `x`.
|
||||
|
||||
@@ -251,14 +254,22 @@ def _per_token_group_quant_8bit_raw(
|
||||
bit8_max = info.max
|
||||
bit8_min = info.min
|
||||
|
||||
x_q = torch.empty_like(x, device=x.device, dtype=dtype)
|
||||
x_s = create_per_token_group_quant_fp8_output_scale(
|
||||
x_shape=x.shape,
|
||||
device=x.device,
|
||||
group_size=group_size,
|
||||
column_major_scales=column_major_scales,
|
||||
scale_tma_aligned=scale_tma_aligned,
|
||||
scale_ue8m0=False,
|
||||
x_q = (
|
||||
output_q
|
||||
if output_q is not None
|
||||
else torch.empty_like(x, device=x.device, dtype=dtype)
|
||||
)
|
||||
x_s = (
|
||||
output_s
|
||||
if output_s is not None
|
||||
else create_per_token_group_quant_fp8_output_scale(
|
||||
x_shape=x.shape,
|
||||
device=x.device,
|
||||
group_size=group_size,
|
||||
column_major_scales=column_major_scales,
|
||||
scale_tma_aligned=scale_tma_aligned,
|
||||
scale_ue8m0=False,
|
||||
)
|
||||
)
|
||||
|
||||
M = x.numel() // group_size
|
||||
@@ -573,6 +584,9 @@ def sglang_per_token_group_quant_fp8(
|
||||
scale_ue8m0: bool = False,
|
||||
fuse_silu_and_mul: bool = False,
|
||||
masked_m: Optional[torch.Tensor] = None,
|
||||
*,
|
||||
output_q: Optional[torch.Tensor] = None,
|
||||
output_s: Optional[torch.Tensor] = None,
|
||||
):
|
||||
assert x.shape[-1] % group_size == 0, (
|
||||
"the last dimension of `x` cannot be divisible by `group_size`"
|
||||
@@ -588,18 +602,34 @@ def sglang_per_token_group_quant_fp8(
|
||||
# Whole-row group quant is per-token quant; route to the dedicated
|
||||
# kernel (same [T, 1] scale shape) instead of a group kernel that
|
||||
# would need arbitrary group sizes.
|
||||
return sglang_per_token_quant_fp8(x)
|
||||
if output_q is not None and output_q.dtype != fp8_dtype:
|
||||
return _per_token_group_quant_8bit_raw(
|
||||
x,
|
||||
group_size,
|
||||
dtype=output_q.dtype,
|
||||
output_q=output_q,
|
||||
output_s=output_s,
|
||||
)
|
||||
return sglang_per_token_quant_fp8(x, output_q=output_q, output_s=output_s)
|
||||
|
||||
out_shape = (*x.shape[:-1], x.shape[-1] // (2 if fuse_silu_and_mul else 1))
|
||||
|
||||
x_q = torch.empty(out_shape, device=x.device, dtype=fp8_dtype)
|
||||
x_s = create_per_token_group_quant_fp8_output_scale(
|
||||
x_shape=out_shape,
|
||||
device=x.device,
|
||||
group_size=group_size,
|
||||
column_major_scales=column_major_scales,
|
||||
scale_tma_aligned=scale_tma_aligned,
|
||||
scale_ue8m0=scale_ue8m0,
|
||||
x_q = (
|
||||
output_q
|
||||
if output_q is not None
|
||||
else torch.empty(out_shape, device=x.device, dtype=fp8_dtype)
|
||||
)
|
||||
x_s = (
|
||||
output_s
|
||||
if output_s is not None
|
||||
else create_per_token_group_quant_fp8_output_scale(
|
||||
x_shape=out_shape,
|
||||
device=x.device,
|
||||
group_size=group_size,
|
||||
column_major_scales=column_major_scales,
|
||||
scale_tma_aligned=scale_tma_aligned,
|
||||
scale_ue8m0=scale_ue8m0,
|
||||
)
|
||||
)
|
||||
|
||||
if x.shape[0] > 0:
|
||||
@@ -764,15 +794,26 @@ def sglang_per_token_group_quant_8bit(
|
||||
def sglang_per_token_quant_fp8(
|
||||
x: torch.Tensor,
|
||||
dtype: torch.dtype = fp8_dtype,
|
||||
*,
|
||||
output_q: Optional[torch.Tensor] = None,
|
||||
output_s: Optional[torch.Tensor] = None,
|
||||
):
|
||||
assert x.is_contiguous(), "`x` is not contiguous"
|
||||
|
||||
x_q = torch.empty_like(x, device=x.device, dtype=dtype)
|
||||
x_s = torch.empty(
|
||||
x.shape[0],
|
||||
1,
|
||||
device=x.device,
|
||||
dtype=torch.float32,
|
||||
x_q = (
|
||||
output_q
|
||||
if output_q is not None
|
||||
else torch.empty_like(x, device=x.device, dtype=dtype)
|
||||
)
|
||||
x_s = (
|
||||
output_s
|
||||
if output_s is not None
|
||||
else torch.empty(
|
||||
x.shape[0],
|
||||
1,
|
||||
device=x.device,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
)
|
||||
|
||||
sgl_per_token_quant_fp8(x, x_q, x_s)
|
||||
|
||||
@@ -1663,7 +1663,10 @@ def dequant_mxfp4(
|
||||
|
||||
|
||||
def input_to_float8(
|
||||
x: torch.Tensor, dtype: torch.dtype = fp8_dtype
|
||||
x: torch.Tensor,
|
||||
dtype: torch.dtype = fp8_dtype,
|
||||
*,
|
||||
out: Optional[torch.Tensor] = None,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
"""This function quantizes input values to float8 values with tensor-wise quantization."""
|
||||
min_val, max_val = x.aminmax()
|
||||
@@ -1677,6 +1680,12 @@ def input_to_float8(
|
||||
fp_max = finfo.max
|
||||
|
||||
scale = fp_max / amax
|
||||
if out is not None:
|
||||
assert out.shape == x.shape and out.dtype == dtype
|
||||
assert x.dtype in (torch.float16, torch.bfloat16, torch.float32)
|
||||
# FP32 broadcast avoids a full input copy; absmax bounds the FP8 range.
|
||||
torch.mul(x, scale.reshape((1,) * x.ndim), out=out)
|
||||
return out, scale.float().reciprocal()
|
||||
x_scl_sat = (x.float() * scale).clamp(min=-fp_max, max=fp_max)
|
||||
return x_scl_sat.to(dtype).contiguous(), scale.float().reciprocal()
|
||||
|
||||
|
||||
@@ -33,7 +33,6 @@ from typing import (
|
||||
List,
|
||||
Optional,
|
||||
Tuple,
|
||||
Union,
|
||||
cast,
|
||||
)
|
||||
|
||||
@@ -1074,6 +1073,8 @@ class DefaultModelLoader(BaseModelLoader):
|
||||
|
||||
@staticmethod
|
||||
def postprocess_weights(model, target_device):
|
||||
if QuantizedRLModelLoader.is_reload_scenario(model):
|
||||
return
|
||||
for module, quant_method in _modules_with_quant_method(model):
|
||||
# When quant methods need to process weights after loading
|
||||
# (for repacking, quantizing, etc), they expect parameters
|
||||
@@ -1086,6 +1087,8 @@ class DefaultModelLoader(BaseModelLoader):
|
||||
@staticmethod
|
||||
def restore_weights_before_loading(model, target_device):
|
||||
"""Undo in-place quant packing so fresh weights can be loaded."""
|
||||
if QuantizedRLModelLoader.is_reload_scenario(model):
|
||||
return
|
||||
for module, quant_method in _modules_with_quant_method(model):
|
||||
# AMX packing and the MXFP4 backend wrappers are duck-typed and cannot restore
|
||||
if isinstance(quant_method, QuantizeMethodBase):
|
||||
@@ -1156,14 +1159,12 @@ class LayeredModelLoader(DefaultModelLoader):
|
||||
|
||||
|
||||
class QuantizedRLModelLoader(DefaultModelLoader):
|
||||
"""
|
||||
Model loader for RL training with FP8 quantization (profile-free, native SGLang).
|
||||
"""FP8 RL loader for audited Qwen2 (legacy/v2) and Qwen3 native loaders.
|
||||
|
||||
Workflow:
|
||||
1. Initial load: Load base model → Record state → Apply FP8 quantization
|
||||
2. Training Actor in full precision
|
||||
3. Reload: Trainer sends full precision weights → Quantize to FP8 → Copy to original memory
|
||||
4. Use torch.as_strided to preserve memory locations across reloads
|
||||
Initial loads record native parameter loaders before quantization. Reloads
|
||||
invoke the native loader once, deferring FP8 writes; each local destination
|
||||
is staged and quantized into its existing FP8 weight and scale buffers.
|
||||
Unsupported native loaders are rejected before initialization loads weights.
|
||||
|
||||
Usage:
|
||||
--model-path Qwen/Qwen2.5-7B --quantization fp8 --load-format flash_rl
|
||||
@@ -1181,23 +1182,6 @@ class QuantizedRLModelLoader(DefaultModelLoader):
|
||||
"_assert_and_load",
|
||||
]
|
||||
|
||||
# Parameters to skip during FP8 quantization (matches FlashRL's exclude_list)
|
||||
SKIP_QUANTIZATION_PARAMS = [
|
||||
"weight_scale",
|
||||
"input_scale",
|
||||
"output_scale",
|
||||
".bias",
|
||||
"lm_head.weight",
|
||||
"model.norm.weight",
|
||||
"embed_tokens", # BF16 params
|
||||
"rotary_emb.inv_freq",
|
||||
"rotary_emb.cos_cached",
|
||||
"rotary_emb.sin_cached",
|
||||
"projector",
|
||||
"input_layernorm.weight",
|
||||
"post_attention_layernorm.weight", # LayerNorms
|
||||
]
|
||||
|
||||
# Stacked parameters (Qwen2): shards loaded separately, then combined
|
||||
STACKED_PARAMS_MAPPING = [
|
||||
("qkv_proj", ["q_proj", "k_proj", "v_proj"]),
|
||||
@@ -1215,14 +1199,18 @@ class QuantizedRLModelLoader(DefaultModelLoader):
|
||||
self._initial_load_complete = False
|
||||
|
||||
def _prepare_weights(
|
||||
self, model_name_or_path: str, revision: Optional[str], fall_back_to_pt: bool
|
||||
self,
|
||||
model_name_or_path: str,
|
||||
revision: Optional[str],
|
||||
fall_back_to_pt: bool,
|
||||
allow_patterns_overrides: Optional[list[str]] = None,
|
||||
):
|
||||
"""Standard weight preparation using base model path."""
|
||||
logger.info(f"[QuantizedRL] Loading from base model: {model_name_or_path}")
|
||||
temp_config = LoadConfig(load_format=LoadFormat.AUTO)
|
||||
temp_loader = DefaultModelLoader(temp_config)
|
||||
return temp_loader._prepare_weights(
|
||||
model_name_or_path, revision, fall_back_to_pt
|
||||
model_name_or_path, revision, fall_back_to_pt, allow_patterns_overrides
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
@@ -1242,6 +1230,14 @@ class QuantizedRLModelLoader(DefaultModelLoader):
|
||||
logger.info("[QuantizedRL] Initial load with FP8 quantization")
|
||||
|
||||
original_load_weights = model.load_weights
|
||||
native = getattr(original_load_weights, "__func__", original_load_weights)
|
||||
if getattr(native, "_supports_quantized_rl_reload", None) is not native:
|
||||
raise ValueError(
|
||||
f"FlashRL does not support {type(model).__name__}.load_weights: "
|
||||
"the native loader has not declared the deferred FP8-write contract. "
|
||||
"Supported native loaders are Qwen2ForCausalLM (legacy/v2) and "
|
||||
"Qwen3ForCausalLM; use a regular load format for other models."
|
||||
)
|
||||
|
||||
def load_weights_proxy(weights):
|
||||
if QuantizedRLModelLoader.is_reload_scenario(model):
|
||||
@@ -1301,331 +1297,179 @@ class QuantizedRLModelLoader(DefaultModelLoader):
|
||||
return (
|
||||
hasattr(model, "original_weights_rebuild_keys")
|
||||
and hasattr(model, "recorded_loader")
|
||||
and getattr(model, "flash_rl_initial_load_complete", False)
|
||||
and getattr(model, "flash_rl_initial_load_complete", False) is True
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _is_stacked_param(name):
|
||||
"""Check if parameter is stacked (qkv_proj, gate_up_proj)."""
|
||||
for stacked_name, _ in QuantizedRLModelLoader.STACKED_PARAMS_MAPPING:
|
||||
if stacked_name in name:
|
||||
return True
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _resolve_stacked_info(name: str) -> Tuple[str, Optional[str], Optional[Any]]:
|
||||
for target, shard_names in QuantizedRLModelLoader.STACKED_PARAMS_MAPPING:
|
||||
for idx, shard in enumerate(shard_names):
|
||||
if shard in name:
|
||||
components = name.split(".")
|
||||
if shard in components:
|
||||
components[components.index(shard)] = target
|
||||
shard_id = (
|
||||
QuantizedRLModelLoader._QKV_SHARD_ALIASES.get(shard, shard)
|
||||
QuantizedRLModelLoader._QKV_SHARD_ALIASES[shard]
|
||||
if target == "qkv_proj"
|
||||
else idx
|
||||
)
|
||||
return name.replace(shard, target), target, shard_id
|
||||
return ".".join(components), target, shard_id
|
||||
return name, None, None
|
||||
|
||||
@staticmethod
|
||||
def _store_quantized_scale(
|
||||
scale_store: Dict[str, Union[torch.Tensor, Dict[Any, torch.Tensor]]],
|
||||
name: str,
|
||||
scale: torch.Tensor,
|
||||
) -> None:
|
||||
param_name, stacked_key, shard_id = (
|
||||
QuantizedRLModelLoader._resolve_stacked_info(name)
|
||||
)
|
||||
if stacked_key is None:
|
||||
scale_store[param_name] = scale
|
||||
else:
|
||||
shard_dict = scale_store.setdefault(param_name, {})
|
||||
assert isinstance(shard_dict, dict)
|
||||
shard_dict[shard_id] = scale
|
||||
|
||||
@staticmethod
|
||||
def _apply_scale_update(
|
||||
all_params: Dict[str, torch.nn.Parameter],
|
||||
param_name: str,
|
||||
scale_info: Union[torch.Tensor, Dict[Any, torch.Tensor], None],
|
||||
) -> None:
|
||||
if scale_info is None:
|
||||
return
|
||||
# Get tp rank and size
|
||||
tp_rank = get_parallel().tp_rank
|
||||
tp_size = get_parallel().tp_size
|
||||
|
||||
def _get_tp_sharded_scale(full_scale_tensor):
|
||||
"""Get tp sharded scale from full scale tensor"""
|
||||
if tp_size == 1:
|
||||
return full_scale_tensor
|
||||
|
||||
full_dim = full_scale_tensor.shape[0]
|
||||
shard_dim = full_dim // tp_size
|
||||
start_idx = tp_rank * shard_dim
|
||||
end_idx = start_idx + shard_dim
|
||||
return full_scale_tensor[start_idx:end_idx]
|
||||
|
||||
if param_name.endswith(".weight"):
|
||||
scale_param_name = f"{param_name[:-7]}.weight_scale"
|
||||
else:
|
||||
scale_param_name = f"{param_name}.weight_scale"
|
||||
|
||||
scale_param = all_params.get(scale_param_name)
|
||||
if scale_param is None:
|
||||
logger.warning(
|
||||
"[QuantizedRL] Scale parameter not found: %s", scale_param_name
|
||||
)
|
||||
return
|
||||
if isinstance(scale_info, torch.Tensor):
|
||||
new_scale = scale_info.t().contiguous()
|
||||
if scale_param.data.shape == new_scale.shape:
|
||||
scale_param.data.copy_(new_scale)
|
||||
else:
|
||||
logger.warning(
|
||||
"[QuantizedRL] Scale shape mismatch for %s: expected %s, got %s",
|
||||
scale_param_name,
|
||||
scale_param.data.shape,
|
||||
new_scale.shape,
|
||||
)
|
||||
else:
|
||||
stacked_key = next(
|
||||
(
|
||||
target
|
||||
for target, _ in QuantizedRLModelLoader.STACKED_PARAMS_MAPPING
|
||||
if target in param_name
|
||||
),
|
||||
None,
|
||||
)
|
||||
shard_names = next(
|
||||
(
|
||||
names
|
||||
for target, names in QuantizedRLModelLoader.STACKED_PARAMS_MAPPING
|
||||
if target == stacked_key
|
||||
),
|
||||
[],
|
||||
)
|
||||
rows_per_shard = scale_param.data.shape[-1] // max(len(shard_names), 1)
|
||||
if rows_per_shard * len(shard_names) != scale_param.data.shape[-1]:
|
||||
logger.warning(
|
||||
f"Scale param shape {scale_param.data.shape[-1]} not divisible by {len(shard_names)}"
|
||||
)
|
||||
offset = 0
|
||||
for idx, shard in enumerate(shard_names):
|
||||
shard_id = (
|
||||
QuantizedRLModelLoader._QKV_SHARD_ALIASES.get(shard, shard)
|
||||
if stacked_key == "qkv_proj"
|
||||
else idx
|
||||
)
|
||||
shard_scale = scale_info.get(shard_id)
|
||||
shard_scale = _get_tp_sharded_scale(shard_scale)
|
||||
if shard_scale is None:
|
||||
offset += rows_per_shard
|
||||
continue
|
||||
shard_rows = shard_scale.shape[0]
|
||||
start = offset
|
||||
end = start + shard_rows
|
||||
scale_param.data[..., start:end] = shard_scale.t().contiguous()
|
||||
offset = end
|
||||
|
||||
@staticmethod
|
||||
def rebinding_and_load_weights(model, first_time_load_weights, weights):
|
||||
"""
|
||||
Reload: VERL sends BF16 → Quantize to FP8 → Copy to original memory.
|
||||
|
||||
Flow: Reset params → Restore attributes → Quantize in iterator → Load → Copy back
|
||||
"""
|
||||
logger.info("[QuantizedRL] Reload: Updating weights with FP8 quantization")
|
||||
"""Shard new BF16 weights natively, then quantize into their original buffers."""
|
||||
from sglang.kernels.ops.quantization.fp8_kernel import (
|
||||
per_token_group_quant_fp8,
|
||||
)
|
||||
from sglang.srt.layers.quantization.fp8_utils import input_to_float8
|
||||
|
||||
weights_list = list(weights)
|
||||
for name, _ in weights_list:
|
||||
if name.startswith(("layers.", "embed_tokens.", "norm.")):
|
||||
raise ValueError(
|
||||
f"FlashRL reload requires canonical checkpoint names: {name!r} "
|
||||
"must include the 'model.' prefix."
|
||||
)
|
||||
updated_param_names, is_last_update = (
|
||||
QuantizedRLModelLoader._get_updated_params(weights_list, model)
|
||||
)
|
||||
|
||||
# Save current FP8 parameter data pointers
|
||||
existing_params = dict(model.named_parameters())
|
||||
current_param_data = {}
|
||||
for name in updated_param_names:
|
||||
if name in existing_params:
|
||||
current_param_data[name] = existing_params[name].data
|
||||
|
||||
# Reset to pre-quantization shape using torch.as_strided
|
||||
# Keeps same storage, just changes view - critical for memory preservation
|
||||
for name, rebuild_info in model.original_weights_rebuild_keys.items():
|
||||
if name in updated_param_names and name in existing_params:
|
||||
existing_params[name].data = torch.as_strided(
|
||||
# Note: avoid clone here
|
||||
existing_params[name].data.clone(),
|
||||
rebuild_info["shape"],
|
||||
rebuild_info["stride"],
|
||||
)
|
||||
|
||||
# Restore weight loader attributes (only if missing)
|
||||
for k, loader_dict in model.recorded_loader.items():
|
||||
for param_name, loader in loader_dict.items():
|
||||
if param_name in updated_param_names and param_name in existing_params:
|
||||
param = existing_params[param_name]
|
||||
if not hasattr(param, k):
|
||||
if callable(loader):
|
||||
if hasattr(loader, "__self__"):
|
||||
setattr(param, k, loader)
|
||||
else:
|
||||
setattr(
|
||||
param,
|
||||
k,
|
||||
QuantizedRLModelLoader._bind_method_to_cls(
|
||||
loader, param
|
||||
),
|
||||
)
|
||||
else:
|
||||
setattr(param, k, loader)
|
||||
|
||||
del existing_params
|
||||
|
||||
# Quantize BF16 weights to FP8 in iterator (before weight_loader)
|
||||
# Store scales for later update
|
||||
quantized_scales: Dict[str, Union[torch.Tensor, Dict[Any, torch.Tensor]]] = {}
|
||||
|
||||
def quantize_weights_iterator(weights_iter):
|
||||
"""Quantize individual shards before weight_loader stacks them."""
|
||||
from sglang.kernels.ops.quantization.fp8_kernel import (
|
||||
per_token_group_quant_fp8,
|
||||
params = dict(model.named_parameters())
|
||||
grouped = collections.defaultdict(list)
|
||||
complete_params = set()
|
||||
for source_name, weight in weights_list:
|
||||
name, _, shard_id = QuantizedRLModelLoader._resolve_stacked_info(
|
||||
source_name
|
||||
)
|
||||
grouped[name].append((source_name, weight))
|
||||
if shard_id is None:
|
||||
complete_params.add(name)
|
||||
|
||||
for name, weight in weights_iter:
|
||||
if any(
|
||||
skip in name
|
||||
for skip in QuantizedRLModelLoader.SKIP_QUANTIZATION_PARAMS
|
||||
):
|
||||
logger.info(f"[QuantizedRL] Skip: {name} ({weight.dtype})")
|
||||
yield (name, weight)
|
||||
elif weight.dtype in [torch.bfloat16, torch.float32, torch.float16]:
|
||||
qweight, scale = per_token_group_quant_fp8(weight, weight.shape[-1])
|
||||
logger.info(f"[QuantizedRL] Quantize: {name} {weight.dtype}→FP8")
|
||||
QuantizedRLModelLoader._store_quantized_scale(
|
||||
quantized_scales, name, scale
|
||||
row_masks = {}
|
||||
# Check coverage before modifying any model parameter. Per-tensor
|
||||
# quantization needs the original BF16 values for every destination row.
|
||||
for name, sources in grouped.items():
|
||||
if name not in params or params[name].dtype not in (
|
||||
torch.float8_e4m3fn,
|
||||
torch.float8_e4m3fnuz,
|
||||
):
|
||||
continue
|
||||
rebuild = model.original_weights_rebuild_keys[name]
|
||||
layer = model.get_submodule(name.rpartition(".")[0])
|
||||
per_channel = layer.weight_scale.shape == (1, rebuild["shape"][0])
|
||||
if name in complete_params:
|
||||
row_masks[name] = (None, per_channel)
|
||||
continue
|
||||
marker = nn.Parameter(
|
||||
torch.zeros((rebuild["shape"][0], 1), device="cpu"), requires_grad=False
|
||||
)
|
||||
marker.output_dim = 0
|
||||
for source_name, weight in sources:
|
||||
_, _, shard_id = QuantizedRLModelLoader._resolve_stacked_info(
|
||||
source_name
|
||||
)
|
||||
args = () if shard_id is None else (shard_id,)
|
||||
layer.weight_loader(
|
||||
marker, torch.ones((weight.shape[0], 1), device="cpu"), *args
|
||||
)
|
||||
rows = marker.data[:, 0].bool()
|
||||
if not per_channel and not rows.all():
|
||||
raise ValueError(
|
||||
f"Partial per-tensor FP8 update for {name}: supply all destination "
|
||||
"rows in the same update call, including inside a weight-update session."
|
||||
)
|
||||
row_masks[name] = (rows, per_channel)
|
||||
|
||||
# Defer only FP8 writes so one native call preserves cross-key lookups
|
||||
# and the order of non-FP8 loads, including tied embeddings.
|
||||
writes = collections.defaultdict(list)
|
||||
native_loaders = {}
|
||||
for name in row_masks:
|
||||
param = params[name]
|
||||
for key, loaders in model.recorded_loader.items():
|
||||
if name in loaders and not hasattr(param, key):
|
||||
loader = loaders[name]
|
||||
setattr(
|
||||
param,
|
||||
key,
|
||||
QuantizedRLModelLoader._bind_method_to_cls(loader, param)
|
||||
if callable(loader)
|
||||
else loader,
|
||||
)
|
||||
yield (name, qweight)
|
||||
native_loaders[name] = param.weight_loader
|
||||
|
||||
def record_write(*args, _name=name, **kwargs):
|
||||
writes[_name].append((args, kwargs))
|
||||
|
||||
param.weight_loader = record_write
|
||||
try:
|
||||
first_time_load_weights(weights_list)
|
||||
finally:
|
||||
for name, loader in native_loaders.items():
|
||||
params[name].weight_loader = loader
|
||||
|
||||
for name, calls in writes.items():
|
||||
param = params[name]
|
||||
data = param.data
|
||||
rebuild = model.original_weights_rebuild_keys[name]
|
||||
layer = model.get_submodule(name.rpartition(".")[0])
|
||||
rows, per_channel = row_masks[name]
|
||||
complete = rows is None or bool(rows.all())
|
||||
staging = torch.empty_strided(
|
||||
rebuild["shape"],
|
||||
rebuild["stride"],
|
||||
dtype=rebuild["dtype"],
|
||||
device=data.device,
|
||||
)
|
||||
param.data = staging
|
||||
try:
|
||||
for args, kwargs in calls:
|
||||
native_loaders[name](*args, **kwargs)
|
||||
local_weight = torch.as_strided(
|
||||
data, rebuild["shape"], rebuild["stride"]
|
||||
)
|
||||
if per_channel:
|
||||
# Untouched rows never pass through dequantization.
|
||||
if complete:
|
||||
ranges = [(0, staging.shape[0])]
|
||||
else:
|
||||
padded = torch.cat((rows.new_zeros(1), rows, rows.new_zeros(1)))
|
||||
edges = (padded[1:] != padded[:-1]).nonzero().flatten().tolist()
|
||||
ranges = zip(edges[::2], edges[1::2])
|
||||
for begin, end in ranges:
|
||||
per_token_group_quant_fp8(
|
||||
staging[begin:end],
|
||||
staging.shape[-1],
|
||||
output_q=local_weight[begin:end],
|
||||
output_s=layer.weight_scale.data[:, begin:end].t(),
|
||||
)
|
||||
else:
|
||||
logger.info(f"[QuantizedRL] Keep: {name} ({weight.dtype})")
|
||||
yield (name, weight)
|
||||
_, scale = input_to_float8(
|
||||
staging, dtype=data.dtype, out=local_weight
|
||||
)
|
||||
layer.weight_scale.data.copy_(scale)
|
||||
finally:
|
||||
param.data = data
|
||||
# No BF16 destination survives to the next destination or call.
|
||||
del staging
|
||||
|
||||
# Load quantized weights (weight_loader stacks FP8 shards)
|
||||
first_time_load_weights(quantize_weights_iterator(iter(weights_list)))
|
||||
|
||||
# Copy back to original FP8 memory locations and update scales
|
||||
all_params = dict(model.named_parameters())
|
||||
|
||||
for name in updated_param_names:
|
||||
if name not in all_params or name not in current_param_data:
|
||||
continue
|
||||
if any(
|
||||
skip in name for skip in QuantizedRLModelLoader.SKIP_QUANTIZATION_PARAMS
|
||||
):
|
||||
continue
|
||||
|
||||
new_param = all_params[name]
|
||||
old_fp8_data = current_param_data[name]
|
||||
|
||||
# Handle embeddings/lm_head (BF16) and quantized weights (FP8)
|
||||
if "embed_tokens" in name or "lm_head" in name:
|
||||
old_fp8_data.copy_(new_param.data)
|
||||
new_param.data = old_fp8_data
|
||||
elif (
|
||||
new_param.dtype == torch.float8_e4m3fn
|
||||
and old_fp8_data.dtype == torch.float8_e4m3fn
|
||||
):
|
||||
# FP8: Use strided view for transposed storage
|
||||
strided_data = torch.as_strided(
|
||||
new_param.data, old_fp8_data.shape, old_fp8_data.stride()
|
||||
)
|
||||
old_fp8_data.copy_(strided_data)
|
||||
new_param.data = old_fp8_data
|
||||
QuantizedRLModelLoader._apply_scale_update(
|
||||
all_params,
|
||||
name,
|
||||
quantized_scales.get(name),
|
||||
)
|
||||
elif new_param.dtype == old_fp8_data.dtype:
|
||||
# Same dtype (LayerNorm, etc.): Direct copy
|
||||
old_fp8_data.copy_(new_param.data)
|
||||
new_param.data = old_fp8_data
|
||||
else:
|
||||
raise RuntimeError(
|
||||
f"Unexpected dtype mismatch for {name}: "
|
||||
f"new={new_param.dtype}, old={old_fp8_data.dtype}"
|
||||
)
|
||||
|
||||
# Cleanup
|
||||
del current_param_data
|
||||
if is_last_update:
|
||||
gc.collect()
|
||||
current_platform.empty_cache()
|
||||
|
||||
logger.info("[QuantizedRL] Reload complete")
|
||||
return updated_param_names, is_last_update
|
||||
|
||||
@staticmethod
|
||||
def _get_updated_params(weights_list, model):
|
||||
"""Identify which parameters need updating from incoming weights."""
|
||||
stacked_params_mapping = [
|
||||
("qkv_proj", "q_proj", "q"),
|
||||
("qkv_proj", "k_proj", "k"),
|
||||
("qkv_proj", "v_proj", "v"),
|
||||
("gate_up_proj", "gate_proj", 0),
|
||||
("gate_up_proj", "up_proj", 1),
|
||||
]
|
||||
|
||||
params_dict = dict(model.named_parameters())
|
||||
updated_params = set()
|
||||
"""Resolve checkpoint names to parameters owned by this model shard."""
|
||||
params = dict(model.named_parameters())
|
||||
updated = set()
|
||||
is_last_update = False
|
||||
|
||||
for name, _ in weights_list:
|
||||
if name == "lm_head.weight":
|
||||
is_last_update = True
|
||||
|
||||
if any(
|
||||
skip in name for skip in QuantizedRLModelLoader.SKIP_QUANTIZATION_PARAMS
|
||||
):
|
||||
continue
|
||||
|
||||
from sglang.srt.layers.utils import get_layer_id
|
||||
|
||||
# Skip params outside layer range (for pipeline parallelism)
|
||||
layer_id = get_layer_id(name)
|
||||
if (
|
||||
layer_id is not None
|
||||
and hasattr(model, "start_layer")
|
||||
and (layer_id < model.start_layer or layer_id >= model.end_layer)
|
||||
):
|
||||
continue
|
||||
|
||||
# Skip tied embeddings and vision tower params
|
||||
if (
|
||||
hasattr(model, "config")
|
||||
and model.config.tie_word_embeddings
|
||||
and "lm_head.weight" in name
|
||||
):
|
||||
continue
|
||||
if name.startswith("model.vision_tower") and name not in params_dict:
|
||||
continue
|
||||
|
||||
# Map stacked param shards (q/k/v_proj → qkv_proj)
|
||||
mapped = False
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name in name:
|
||||
name = name.replace(weight_name, param_name)
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
updated_params.add(name)
|
||||
mapped = True
|
||||
break
|
||||
|
||||
if not mapped:
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
if name in params_dict:
|
||||
updated_params.add(name)
|
||||
|
||||
return list(updated_params), is_last_update
|
||||
name, _, _ = QuantizedRLModelLoader._resolve_stacked_info(name)
|
||||
if name in params:
|
||||
updated.add(name)
|
||||
return sorted(updated), is_last_update
|
||||
|
||||
|
||||
class DummyModelLoader(BaseModelLoader):
|
||||
|
||||
@@ -1474,6 +1474,18 @@ def gguf_quant_weights_iterator(
|
||||
yield name, param
|
||||
|
||||
|
||||
def supports_quantized_rl_reload(func: Callable) -> Callable:
|
||||
"""Allow FlashRL to defer FP8 parameter writes until this method returns.
|
||||
|
||||
Checkpoint names must use the parameter names or QKV/gate-up aliases supported
|
||||
by QuantizedRLModelLoader. FP8 writes must go through param.weight_loader;
|
||||
the method must not read their values afterward, and source tensors must
|
||||
remain unchanged until return. Non-FP8 parameters are loaded immediately.
|
||||
"""
|
||||
func._supports_quantized_rl_reload = func
|
||||
return func
|
||||
|
||||
|
||||
def default_weight_loader(param: torch.Tensor, loaded_weight: torch.Tensor) -> None:
|
||||
"""Default weight loader."""
|
||||
if param.numel() == 1 and loaded_weight.numel() == 1:
|
||||
|
||||
@@ -55,6 +55,7 @@ from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTe
|
||||
from sglang.srt.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
kv_cache_scales_loader,
|
||||
supports_quantized_rl_reload,
|
||||
)
|
||||
from sglang.srt.platforms import current_platform
|
||||
from sglang.srt.runtime_context import get_exec, get_parallel
|
||||
@@ -616,6 +617,7 @@ class Qwen2ForCausalLM(nn.Module):
|
||||
def end_layer(self):
|
||||
return self.model.end_layer
|
||||
|
||||
@supports_quantized_rl_reload
|
||||
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
|
||||
from sglang.srt.environ import envs
|
||||
|
||||
|
||||
@@ -32,6 +32,7 @@ from sglang.srt.model_executor.forward_context import get_token_to_kv_pool
|
||||
from sglang.srt.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
maybe_remap_kv_scale_name,
|
||||
supports_quantized_rl_reload,
|
||||
)
|
||||
from sglang.srt.models.qwen2 import Qwen2MLP as Qwen3MLP
|
||||
from sglang.srt.models.qwen2 import Qwen2Model
|
||||
@@ -620,6 +621,7 @@ class Qwen3ForCausalLM(nn.Module):
|
||||
def end_layer(self):
|
||||
return self.model.end_layer
|
||||
|
||||
@supports_quantized_rl_reload
|
||||
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
|
||||
@@ -41,6 +41,20 @@ class TestDefaultModelLoader(CustomTestCase):
|
||||
|
||||
self.assertEqual(events, ["load", "postprocess"])
|
||||
|
||||
def test_reload_requires_explicit_initial_load_marker(self):
|
||||
self.assertFalse(loader_mod.QuantizedRLModelLoader.is_reload_scenario(Mock()))
|
||||
model = torch.nn.Module()
|
||||
model.original_weights_rebuild_keys = set()
|
||||
model.recorded_loader = {}
|
||||
for marker in (None, False, 1, Mock()):
|
||||
with self.subTest(marker=marker):
|
||||
model.flash_rl_initial_load_complete = marker
|
||||
self.assertFalse(
|
||||
loader_mod.QuantizedRLModelLoader.is_reload_scenario(model)
|
||||
)
|
||||
model.flash_rl_initial_load_complete = True
|
||||
self.assertTrue(loader_mod.QuantizedRLModelLoader.is_reload_scenario(model))
|
||||
|
||||
def test_boot_paths_preserve_load_order_and_custom_override(self):
|
||||
class CustomModelLoader(DefaultModelLoader):
|
||||
def load_weights_and_postprocess(self, model, weights, target_device):
|
||||
|
||||
@@ -0,0 +1,518 @@
|
||||
"""Quantized reload scales follow their destination projection's layout."""
|
||||
|
||||
import math
|
||||
import socket
|
||||
import unittest
|
||||
from contextlib import nullcontext
|
||||
from unittest.mock import patch
|
||||
|
||||
import torch
|
||||
from transformers import Qwen2Config, Qwen3Config
|
||||
|
||||
from sglang.srt.configs.load_config import LoadConfig, LoadFormat
|
||||
from sglang.srt.distributed.parallel_state import (
|
||||
destroy_distributed_environment,
|
||||
destroy_model_parallel,
|
||||
init_distributed_environment,
|
||||
initialize_model_parallel,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import initialize_dp_attention
|
||||
from sglang.srt.layers.quantization.fp8 import Fp8Config
|
||||
from sglang.srt.model_loader.loader import DefaultModelLoader, QuantizedRLModelLoader
|
||||
from sglang.srt.models.qwen2 import Qwen2ForCausalLM
|
||||
from sglang.srt.models.qwen3 import Qwen3ForCausalLM
|
||||
from sglang.srt.runtime_context import SpawnRanks, get_parallel, publish, reset_context
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cuda_ci(est_time=20, stage="base-b", runner_config="1-gpu-small")
|
||||
|
||||
|
||||
def values(shape, version=0):
|
||||
base = (
|
||||
torch.arange(math.prod(shape), device="cuda").reshape(shape) % 31 - 15
|
||||
).float() / 64
|
||||
if len(shape) == 2:
|
||||
base *= ((torch.arange(shape[0], device="cuda") + version) % 17 + 1)[:, None]
|
||||
base *= (torch.arange(shape[1], device="cuda") // max(shape[1] // 4, 1) + 1)[
|
||||
None, :
|
||||
]
|
||||
return (base + version / 128).to(torch.bfloat16)
|
||||
|
||||
|
||||
def checkpoint(version=0, kind="qwen2", kv_heads=8, num_layers=1):
|
||||
weights = []
|
||||
for name, shape in {
|
||||
"model.embed_tokens.weight": (512, 512),
|
||||
"lm_head.weight": (512, 512),
|
||||
"model.norm.weight": (512,),
|
||||
"model.layers.0.input_layernorm.weight": (512,),
|
||||
"model.layers.0.post_attention_layernorm.weight": (512,),
|
||||
**{
|
||||
f"model.layers.0.self_attn.{p}.weight": (
|
||||
kv_heads * 64 if p in ("k_proj", "v_proj") else 512,
|
||||
512,
|
||||
)
|
||||
for p in ("q_proj", "k_proj", "v_proj", "o_proj")
|
||||
},
|
||||
**{
|
||||
f"model.layers.0.self_attn.{p}.bias": (
|
||||
kv_heads * 64 if p in ("k_proj", "v_proj") else 512,
|
||||
)
|
||||
for p in ("q_proj", "k_proj", "v_proj")
|
||||
if kind == "qwen2"
|
||||
},
|
||||
**{
|
||||
f"model.layers.0.self_attn.{p}.weight": (64,)
|
||||
for p in ("q_norm", "k_norm")
|
||||
if kind == "qwen3"
|
||||
},
|
||||
**{
|
||||
f"model.layers.0.mlp.{p}.weight": (1024, 512)
|
||||
for p in ("gate_proj", "up_proj")
|
||||
},
|
||||
"model.layers.0.mlp.down_proj.weight": (512, 1024),
|
||||
}.items():
|
||||
weights.append((name, values(shape, version)))
|
||||
first_layer = [(n, t) for n, t in weights if n.startswith("model.layers.0.")]
|
||||
for layer in range(1, num_layers):
|
||||
weights.extend(
|
||||
(n.replace("model.layers.0.", f"model.layers.{layer}."), t)
|
||||
for n, t in first_layer
|
||||
)
|
||||
return weights
|
||||
|
||||
|
||||
def build_model(
|
||||
kind="qwen2", *, kv_heads=8, version=0, weights=None, per_tensor=False, num_layers=1
|
||||
):
|
||||
config_class, model_class = {
|
||||
"qwen2": (Qwen2Config, Qwen2ForCausalLM),
|
||||
"qwen3": (Qwen3Config, Qwen3ForCausalLM),
|
||||
}[kind]
|
||||
config = config_class(
|
||||
hidden_size=512,
|
||||
intermediate_size=1024,
|
||||
num_hidden_layers=num_layers,
|
||||
num_attention_heads=8,
|
||||
num_key_value_heads=kv_heads,
|
||||
head_dim=64,
|
||||
vocab_size=512,
|
||||
max_position_embeddings=32,
|
||||
tie_word_embeddings=False,
|
||||
pad_token_id=0,
|
||||
bos_token_id=1,
|
||||
eos_token_id=2,
|
||||
)
|
||||
with torch.device("cuda"):
|
||||
model = model_class(config, Fp8Config(is_checkpoint_fp8_serialized=False))
|
||||
if per_tensor:
|
||||
for module in model.modules():
|
||||
method = getattr(module, "quant_method", None)
|
||||
if hasattr(method, "cutlass_fp8_supported"):
|
||||
method.cutlass_fp8_supported = False
|
||||
method.use_marlin = False
|
||||
loader = QuantizedRLModelLoader(LoadConfig(load_format=LoadFormat.FLASH_RL))
|
||||
loader.load_weights_and_postprocess(
|
||||
model,
|
||||
checkpoint(version, kind, kv_heads, num_layers) if weights is None else weights,
|
||||
torch.device("cuda"),
|
||||
)
|
||||
return model
|
||||
|
||||
|
||||
def loading_scope(changed):
|
||||
if not changed:
|
||||
return nullcontext()
|
||||
return get_parallel().override(
|
||||
tp_size=4,
|
||||
tp_rank=3,
|
||||
tp_group=None,
|
||||
attn_tp_size=4,
|
||||
attn_tp_rank=3,
|
||||
attn_tp_group=None,
|
||||
attn_dp_size=1,
|
||||
attn_dp_rank=0,
|
||||
attn_cp_size=1,
|
||||
attn_cp_rank=0,
|
||||
moe_tp_size=4,
|
||||
moe_tp_rank=3,
|
||||
moe_ep_size=1,
|
||||
moe_ep_rank=0,
|
||||
moe_ep_group=None,
|
||||
moe_dp_size=1,
|
||||
moe_dp_rank=0,
|
||||
)
|
||||
|
||||
|
||||
def assert_same_parameters(actual, expected):
|
||||
expected_params = dict(expected.named_parameters())
|
||||
for name, param in actual.named_parameters():
|
||||
reference = expected_params[name]
|
||||
assert param.shape == reference.shape, name
|
||||
assert param.stride() == reference.stride(), name
|
||||
assert param.dtype == reference.dtype, name
|
||||
torch.testing.assert_close(
|
||||
param.contiguous().reshape(-1).view(torch.uint8),
|
||||
reference.contiguous().reshape(-1).view(torch.uint8),
|
||||
rtol=0,
|
||||
atol=0,
|
||||
msg=name,
|
||||
)
|
||||
|
||||
|
||||
def check_reload(model, version=13, changed=False, per_tensor=False):
|
||||
kind, kv_heads = model.config.model_type, model.config.num_key_value_heads
|
||||
source = checkpoint(version, kind, kv_heads)
|
||||
pointers = {n: p.data_ptr() for n, p in model.named_parameters()}
|
||||
DefaultModelLoader.restore_weights_before_loading(model, torch.device("cuda"))
|
||||
with loading_scope(changed):
|
||||
model.load_weights(source)
|
||||
DefaultModelLoader.postprocess_weights(model, torch.device("cuda"))
|
||||
for name, param in model.named_parameters():
|
||||
assert param.data_ptr() == pointers[name], name
|
||||
reference = build_model(
|
||||
kind, kv_heads=kv_heads, version=version, per_tensor=per_tensor
|
||||
)
|
||||
assert_same_parameters(model, reference)
|
||||
return model
|
||||
|
||||
|
||||
@unittest.skipUnless(torch.cuda.is_available(), "needs a GPU")
|
||||
class TestQuantizedReloadScaleLayout(CustomTestCase):
|
||||
def setUp(self):
|
||||
if torch.distributed.is_initialized():
|
||||
self.skipTest("requires an isolated distributed test process")
|
||||
reset_context()
|
||||
self.addCleanup(reset_context)
|
||||
original = torch.get_default_dtype()
|
||||
self.addCleanup(torch.set_default_dtype, original)
|
||||
torch.set_default_dtype(torch.bfloat16)
|
||||
server = ServerArgs(model_path="dummy", device="cuda", tp_size=1)
|
||||
publish(server, role="test", ranks=SpawnRanks(world_rank=0, gpu_id=0))
|
||||
torch.cuda.set_device(0)
|
||||
with socket.socket() as sock:
|
||||
sock.bind(("127.0.0.1", 0))
|
||||
port = sock.getsockname()[1]
|
||||
init_distributed_environment(
|
||||
world_size=1,
|
||||
rank=0,
|
||||
local_rank=0,
|
||||
distributed_init_method=f"tcp://127.0.0.1:{port}",
|
||||
)
|
||||
self.addCleanup(destroy_distributed_environment)
|
||||
initialize_model_parallel()
|
||||
self.addCleanup(destroy_model_parallel)
|
||||
initialize_dp_attention(server)
|
||||
|
||||
def test_native_model_reload_after_scope_exit(self):
|
||||
model = build_model()
|
||||
check_reload(model, changed=True)
|
||||
check_reload(model, version=29, changed=True)
|
||||
|
||||
def test_native_attention_group_projection_reload_after_scope_exit(self):
|
||||
model = build_model("qwen3")
|
||||
check_reload(model, changed=True)
|
||||
check_reload(model, version=29, changed=True)
|
||||
|
||||
def test_unequal_qkv_and_replicated_kv_match_cold_load(self):
|
||||
for kv_heads in (1, 2, 4):
|
||||
with self.subTest(kv_heads=kv_heads):
|
||||
check_reload(build_model("qwen3", kv_heads=kv_heads), changed=True)
|
||||
|
||||
def test_partial_qkv_and_mlp_updates_preserve_other_rows(self):
|
||||
for kind in ("qwen2", "qwen3"):
|
||||
for shard in ("q_proj", "k_proj", "v_proj", "gate_proj", "up_proj"):
|
||||
with self.subTest(kind=kind, shard=shard):
|
||||
model = build_model(kind, kv_heads=1)
|
||||
before = checkpoint(kind=kind, kv_heads=1)
|
||||
update = {
|
||||
n: t for n, t in checkpoint(13, kind, 1) if f".{shard}." in n
|
||||
}
|
||||
pointers = {n: p.data_ptr() for n, p in model.named_parameters()}
|
||||
with loading_scope(True):
|
||||
model.load_weights(list(update.items()))
|
||||
expected = [(n, update.get(n, t)) for n, t in before]
|
||||
reference = build_model(kind, kv_heads=1, weights=expected)
|
||||
assert_same_parameters(model, reference)
|
||||
for name, param in model.named_parameters():
|
||||
self.assertEqual(param.data_ptr(), pointers[name], name)
|
||||
|
||||
def test_per_tensor_quantization_matches_native_cold_load(self):
|
||||
model = build_model("qwen3", kv_heads=1, per_tensor=True)
|
||||
check_reload(model, version=0, changed=True, per_tensor=True)
|
||||
check_reload(model, version=13, changed=True, per_tensor=True)
|
||||
|
||||
def test_partial_per_tensor_update_rejected_before_any_parameter_changes(self):
|
||||
for shard in ("q_proj", "k_proj", "v_proj", "gate_proj", "up_proj"):
|
||||
with self.subTest(shard=shard):
|
||||
model = build_model("qwen3", kv_heads=1, per_tensor=True)
|
||||
pointers = {n: p.data_ptr() for n, p in model.named_parameters()}
|
||||
source = checkpoint(13, "qwen3", 1)
|
||||
update = [
|
||||
(n, t)
|
||||
for n, t in source
|
||||
if n == "model.norm.weight" or f".{shard}." in n
|
||||
]
|
||||
with self.assertRaisesRegex(
|
||||
ValueError, "Partial per-tensor FP8 update"
|
||||
):
|
||||
model.load_weights(update)
|
||||
assert_same_parameters(
|
||||
model, build_model("qwen3", kv_heads=1, per_tensor=True)
|
||||
)
|
||||
for name, param in model.named_parameters():
|
||||
self.assertEqual(param.data_ptr(), pointers[name], name)
|
||||
|
||||
def test_per_tensor_session_accepts_complete_destinations(self):
|
||||
model = build_model("qwen3", kv_heads=1, per_tensor=True)
|
||||
pointers = {n: p.data_ptr() for n, p in model.named_parameters()}
|
||||
DefaultModelLoader.restore_weights_before_loading(model, torch.device("cuda"))
|
||||
grouped = {}
|
||||
for name, tensor in checkpoint(13, "qwen3", 1):
|
||||
target, _, _ = QuantizedRLModelLoader._resolve_stacked_info(name)
|
||||
grouped.setdefault(target, []).append((name, tensor))
|
||||
for source in reversed(list(grouped.values())):
|
||||
with loading_scope(True):
|
||||
model.load_weights(source)
|
||||
self.assertFalse(hasattr(model, "_quantized_rl_pending"))
|
||||
DefaultModelLoader.postprocess_weights(model, torch.device("cuda"))
|
||||
assert_same_parameters(
|
||||
model, build_model("qwen3", kv_heads=1, version=13, per_tensor=True)
|
||||
)
|
||||
for name, param in model.named_parameters():
|
||||
self.assertEqual(param.data_ptr(), pointers[name], name)
|
||||
|
||||
def test_per_tensor_session_rejects_partial_chunks_without_host_cache(self):
|
||||
model = build_model("qwen3", kv_heads=1, per_tensor=True)
|
||||
original = build_model("qwen3", kv_heads=1, per_tensor=True)
|
||||
DefaultModelLoader.restore_weights_before_loading(model, torch.device("cuda"))
|
||||
for shard in ("q_proj", "k_proj", "v_proj", "gate_proj", "up_proj"):
|
||||
update = [
|
||||
(n, t)
|
||||
for n, t in checkpoint(13, "qwen3", 1)
|
||||
if n == "model.norm.weight" or f".{shard}." in n
|
||||
]
|
||||
with self.assertRaisesRegex(ValueError, "Partial per-tensor FP8 update"):
|
||||
model.load_weights(update)
|
||||
assert_same_parameters(model, original)
|
||||
self.assertFalse(hasattr(model, "_quantized_rl_pending"))
|
||||
model.load_weights(checkpoint(13, "qwen3", 1))
|
||||
DefaultModelLoader.postprocess_weights(model, torch.device("cuda"))
|
||||
assert_same_parameters(
|
||||
model, build_model("qwen3", kv_heads=1, version=13, per_tensor=True)
|
||||
)
|
||||
|
||||
def test_interleaved_layers_cannot_accumulate_pending_bf16_matrices(self):
|
||||
model = build_model("qwen3", kv_heads=1, per_tensor=True, num_layers=16)
|
||||
snapshot = {
|
||||
n: p.contiguous().reshape(-1).view(torch.uint8).clone()
|
||||
for n, p in model.named_parameters()
|
||||
}
|
||||
pointers = {n: p.data_ptr() for n, p in model.named_parameters()}
|
||||
DefaultModelLoader.restore_weights_before_loading(model, torch.device("cuda"))
|
||||
source = [
|
||||
(n, t)
|
||||
for n, t in checkpoint(13, "qwen3", 1, 16)
|
||||
if ".q_proj." in n or n == "model.norm.weight"
|
||||
]
|
||||
with self.assertRaisesRegex(ValueError, "Partial per-tensor FP8 update"):
|
||||
model.load_weights(source)
|
||||
self.assertFalse(hasattr(model, "_quantized_rl_pending"))
|
||||
for name, param in model.named_parameters():
|
||||
self.assertEqual(param.data_ptr(), pointers[name], name)
|
||||
torch.testing.assert_close(
|
||||
param.contiguous().reshape(-1).view(torch.uint8),
|
||||
snapshot[name],
|
||||
rtol=0,
|
||||
atol=0,
|
||||
)
|
||||
DefaultModelLoader.postprocess_weights(model, torch.device("cuda"))
|
||||
|
||||
def test_noncanonical_reload_names_are_rejected_before_mutation(self):
|
||||
model = build_model("qwen3", kv_heads=1)
|
||||
sources = dict(checkpoint(13, "qwen3", 1))
|
||||
snapshot = {
|
||||
n: (p.data_ptr(), p.contiguous().view(torch.uint8).clone())
|
||||
for n, p in model.named_parameters()
|
||||
}
|
||||
for canonical in (
|
||||
"model.layers.0.self_attn.q_proj.weight",
|
||||
"model.embed_tokens.weight",
|
||||
"model.norm.weight",
|
||||
):
|
||||
with self.subTest(name=canonical):
|
||||
update = [
|
||||
("model.norm.weight", sources["model.norm.weight"]),
|
||||
(canonical.removeprefix("model."), sources[canonical]),
|
||||
]
|
||||
with self.assertRaisesRegex(ValueError, "canonical checkpoint names"):
|
||||
model.load_weights(update)
|
||||
for name, param in model.named_parameters():
|
||||
pointer, data = snapshot[name]
|
||||
self.assertEqual(param.data_ptr(), pointer)
|
||||
torch.testing.assert_close(
|
||||
param.contiguous().view(torch.uint8), data, rtol=0, atol=0
|
||||
)
|
||||
|
||||
def test_qwen2_walker_reload_matches_cold_load(self):
|
||||
from sglang.srt.environ import envs
|
||||
|
||||
with patch.object(
|
||||
envs.SGLANG_ENABLE_WEIGHT_LOADER_V2, "get", return_value=True
|
||||
):
|
||||
for per_tensor in (False, True):
|
||||
with self.subTest(per_tensor=per_tensor):
|
||||
check_reload(
|
||||
build_model("qwen2", kv_heads=1, per_tensor=per_tensor),
|
||||
changed=True,
|
||||
per_tensor=per_tensor,
|
||||
)
|
||||
|
||||
def test_qwen2_walker_tied_head_post_copy_occurs_once(self):
|
||||
from sglang.srt.environ import envs
|
||||
|
||||
model = build_model("qwen2", kv_heads=1)
|
||||
model.config.tie_word_embeddings = True
|
||||
head = model.lm_head.weight
|
||||
original = head.weight_loader
|
||||
copies = []
|
||||
|
||||
def load_head(*args, **kwargs):
|
||||
copies.append(True)
|
||||
return original(*args, **kwargs)
|
||||
|
||||
head.weight_loader = load_head
|
||||
source = checkpoint(13, "qwen2", 1)
|
||||
embedding = dict(source)["model.embed_tokens.weight"]
|
||||
with patch.object(
|
||||
envs.SGLANG_ENABLE_WEIGHT_LOADER_V2, "get", return_value=True
|
||||
):
|
||||
model.load_weights(source)
|
||||
self.assertEqual(len(copies), 1)
|
||||
torch.testing.assert_close(head, embedding, rtol=0, atol=0)
|
||||
expected = [(n, embedding if n == "lm_head.weight" else t) for n, t in source]
|
||||
assert_same_parameters(
|
||||
model, build_model("qwen2", kv_heads=1, weights=expected)
|
||||
)
|
||||
|
||||
def test_unsupported_native_loaders_fail_before_initialization(self):
|
||||
from sglang.srt.models.mimo_v2 import MiMoV2ForCausalLM
|
||||
from sglang.srt.models.whisper import WhisperForConditionalGeneration
|
||||
|
||||
for native in (
|
||||
WhisperForConditionalGeneration.load_weights,
|
||||
MiMoV2ForCausalLM.load_weights,
|
||||
):
|
||||
with self.subTest(native=native.__qualname__):
|
||||
model = torch.nn.Linear(4, 4, device="cuda")
|
||||
before = model.weight.clone()
|
||||
model.load_weights = native.__get__(model)
|
||||
original = model.load_weights
|
||||
loader = QuantizedRLModelLoader(
|
||||
LoadConfig(load_format=LoadFormat.FLASH_RL)
|
||||
)
|
||||
with self.assertRaisesRegex(ValueError, "deferred FP8-write contract"):
|
||||
loader.load_weights_and_postprocess(model, [], torch.device("cuda"))
|
||||
self.assertIs(model.load_weights, original)
|
||||
self.assertFalse(hasattr(model, "original_weights_rebuild_keys"))
|
||||
torch.testing.assert_close(model.weight, before, rtol=0, atol=0)
|
||||
|
||||
def test_failed_native_load_restores_fp8_storage(self):
|
||||
model = build_model("qwen3", kv_heads=1)
|
||||
pointers = {n: p.data_ptr() for n, p in model.named_parameters()}
|
||||
|
||||
def fail(weights):
|
||||
Qwen3ForCausalLM.load_weights(model, weights)
|
||||
self.assertEqual(
|
||||
model.model.layers[0].self_attn.qkv_proj.weight.dtype,
|
||||
torch.float8_e4m3fn,
|
||||
)
|
||||
raise RuntimeError("native loader failed")
|
||||
|
||||
with self.assertRaisesRegex(RuntimeError, "native loader failed"):
|
||||
QuantizedRLModelLoader.rebinding_and_load_weights(
|
||||
model,
|
||||
fail,
|
||||
[
|
||||
(n, t)
|
||||
for n, t in checkpoint(13, "qwen3", 1)
|
||||
if any(f".{p}." in n for p in ("q_proj", "k_proj", "v_proj"))
|
||||
],
|
||||
)
|
||||
assert_same_parameters(model, build_model("qwen3", kv_heads=1))
|
||||
for name, param in model.named_parameters():
|
||||
self.assertEqual(param.data_ptr(), pointers[name], name)
|
||||
|
||||
def test_failed_deferred_parameter_write_restores_storage_and_loader(self):
|
||||
model = build_model("qwen3", kv_heads=1)
|
||||
param = model.model.layers[0].self_attn.qkv_proj.weight
|
||||
pointer = param.data_ptr()
|
||||
snapshot = param.contiguous().view(torch.uint8).clone()
|
||||
|
||||
def fail(target, weight, shard):
|
||||
self.assertEqual(target.dtype, torch.bfloat16)
|
||||
raise RuntimeError("deferred write failed")
|
||||
|
||||
param.weight_loader = fail
|
||||
source = [
|
||||
(n, t)
|
||||
for n, t in checkpoint(13, "qwen3", 1)
|
||||
if any(f".{shard}_proj." in n for shard in ("q", "k", "v"))
|
||||
]
|
||||
with self.assertRaisesRegex(RuntimeError, "deferred write failed"):
|
||||
model.load_weights(source)
|
||||
self.assertEqual(param.data_ptr(), pointer)
|
||||
self.assertIs(param.weight_loader, fail)
|
||||
torch.testing.assert_close(
|
||||
param.contiguous().view(torch.uint8), snapshot, rtol=0, atol=0
|
||||
)
|
||||
|
||||
def test_fnuz_per_tensor_reload_matches_cold_load(self):
|
||||
from sglang.srt.layers.quantization import fp8
|
||||
from sglang.srt.layers.quantization.fp8_utils import input_to_float8
|
||||
|
||||
with patch.object(
|
||||
fp8,
|
||||
"input_to_float8",
|
||||
lambda x: input_to_float8(x, dtype=torch.float8_e4m3fnuz),
|
||||
):
|
||||
model = build_model("qwen3", kv_heads=1, per_tensor=True)
|
||||
check_reload(model, version=13, per_tensor=True)
|
||||
|
||||
def test_per_tensor_output_buffer_matches_reference_without_weight_temporary(self):
|
||||
from sglang.srt.layers.quantization.fp8_utils import input_to_float8
|
||||
|
||||
x = values((2048, 2048), 13)
|
||||
for dtype in (torch.float8_e4m3fn, torch.float8_e4m3fnuz):
|
||||
with self.subTest(dtype=dtype):
|
||||
expected, expected_scale = input_to_float8(x, dtype=dtype)
|
||||
output = torch.empty_like(x, dtype=dtype)
|
||||
input_to_float8(x, dtype=dtype, out=output)
|
||||
torch.cuda.synchronize()
|
||||
baseline = torch.cuda.memory_allocated()
|
||||
torch.cuda.reset_peak_memory_stats()
|
||||
actual, scale = input_to_float8(x, dtype=dtype, out=output)
|
||||
torch.cuda.synchronize()
|
||||
extra = torch.cuda.max_memory_allocated() - baseline
|
||||
self.assertLess(extra, 64 * 1024)
|
||||
self.assertEqual(actual.data_ptr(), output.data_ptr())
|
||||
torch.testing.assert_close(
|
||||
actual.view(torch.uint8), expected.view(torch.uint8), rtol=0, atol=0
|
||||
)
|
||||
torch.testing.assert_close(scale, expected_scale, rtol=0, atol=0)
|
||||
|
||||
def test_empty_session_keeps_native_layout_and_memory(self):
|
||||
model = build_model("qwen3", kv_heads=1)
|
||||
pointers = {n: p.data_ptr() for n, p in model.named_parameters()}
|
||||
DefaultModelLoader.restore_weights_before_loading(model, torch.device("cuda"))
|
||||
model.load_weights([])
|
||||
DefaultModelLoader.postprocess_weights(model, torch.device("cuda"))
|
||||
assert_same_parameters(model, build_model("qwen3", kv_heads=1))
|
||||
for name, param in model.named_parameters():
|
||||
self.assertEqual(param.data_ptr(), pointers[name], name)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user