[Fix] Quantize reloaded FP8 weights after native sharding (#42585)
This commit is contained in:
@@ -214,6 +214,9 @@ def _per_token_group_quant_8bit_raw(
|
||||
column_major_scales: bool = False,
|
||||
scale_tma_aligned: bool = False,
|
||||
scale_ue8m0: bool = False,
|
||||
*,
|
||||
output_q: Optional[torch.Tensor] = None,
|
||||
output_s: Optional[torch.Tensor] = None,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Function to perform per-token-group quantization on an input tensor `x`.
|
||||
|
||||
@@ -251,14 +254,22 @@ def _per_token_group_quant_8bit_raw(
|
||||
bit8_max = info.max
|
||||
bit8_min = info.min
|
||||
|
||||
x_q = torch.empty_like(x, device=x.device, dtype=dtype)
|
||||
x_s = create_per_token_group_quant_fp8_output_scale(
|
||||
x_shape=x.shape,
|
||||
device=x.device,
|
||||
group_size=group_size,
|
||||
column_major_scales=column_major_scales,
|
||||
scale_tma_aligned=scale_tma_aligned,
|
||||
scale_ue8m0=False,
|
||||
x_q = (
|
||||
output_q
|
||||
if output_q is not None
|
||||
else torch.empty_like(x, device=x.device, dtype=dtype)
|
||||
)
|
||||
x_s = (
|
||||
output_s
|
||||
if output_s is not None
|
||||
else create_per_token_group_quant_fp8_output_scale(
|
||||
x_shape=x.shape,
|
||||
device=x.device,
|
||||
group_size=group_size,
|
||||
column_major_scales=column_major_scales,
|
||||
scale_tma_aligned=scale_tma_aligned,
|
||||
scale_ue8m0=False,
|
||||
)
|
||||
)
|
||||
|
||||
M = x.numel() // group_size
|
||||
@@ -573,6 +584,9 @@ def sglang_per_token_group_quant_fp8(
|
||||
scale_ue8m0: bool = False,
|
||||
fuse_silu_and_mul: bool = False,
|
||||
masked_m: Optional[torch.Tensor] = None,
|
||||
*,
|
||||
output_q: Optional[torch.Tensor] = None,
|
||||
output_s: Optional[torch.Tensor] = None,
|
||||
):
|
||||
assert x.shape[-1] % group_size == 0, (
|
||||
"the last dimension of `x` cannot be divisible by `group_size`"
|
||||
@@ -588,18 +602,34 @@ def sglang_per_token_group_quant_fp8(
|
||||
# Whole-row group quant is per-token quant; route to the dedicated
|
||||
# kernel (same [T, 1] scale shape) instead of a group kernel that
|
||||
# would need arbitrary group sizes.
|
||||
return sglang_per_token_quant_fp8(x)
|
||||
if output_q is not None and output_q.dtype != fp8_dtype:
|
||||
return _per_token_group_quant_8bit_raw(
|
||||
x,
|
||||
group_size,
|
||||
dtype=output_q.dtype,
|
||||
output_q=output_q,
|
||||
output_s=output_s,
|
||||
)
|
||||
return sglang_per_token_quant_fp8(x, output_q=output_q, output_s=output_s)
|
||||
|
||||
out_shape = (*x.shape[:-1], x.shape[-1] // (2 if fuse_silu_and_mul else 1))
|
||||
|
||||
x_q = torch.empty(out_shape, device=x.device, dtype=fp8_dtype)
|
||||
x_s = create_per_token_group_quant_fp8_output_scale(
|
||||
x_shape=out_shape,
|
||||
device=x.device,
|
||||
group_size=group_size,
|
||||
column_major_scales=column_major_scales,
|
||||
scale_tma_aligned=scale_tma_aligned,
|
||||
scale_ue8m0=scale_ue8m0,
|
||||
x_q = (
|
||||
output_q
|
||||
if output_q is not None
|
||||
else torch.empty(out_shape, device=x.device, dtype=fp8_dtype)
|
||||
)
|
||||
x_s = (
|
||||
output_s
|
||||
if output_s is not None
|
||||
else create_per_token_group_quant_fp8_output_scale(
|
||||
x_shape=out_shape,
|
||||
device=x.device,
|
||||
group_size=group_size,
|
||||
column_major_scales=column_major_scales,
|
||||
scale_tma_aligned=scale_tma_aligned,
|
||||
scale_ue8m0=scale_ue8m0,
|
||||
)
|
||||
)
|
||||
|
||||
if x.shape[0] > 0:
|
||||
@@ -764,15 +794,26 @@ def sglang_per_token_group_quant_8bit(
|
||||
def sglang_per_token_quant_fp8(
|
||||
x: torch.Tensor,
|
||||
dtype: torch.dtype = fp8_dtype,
|
||||
*,
|
||||
output_q: Optional[torch.Tensor] = None,
|
||||
output_s: Optional[torch.Tensor] = None,
|
||||
):
|
||||
assert x.is_contiguous(), "`x` is not contiguous"
|
||||
|
||||
x_q = torch.empty_like(x, device=x.device, dtype=dtype)
|
||||
x_s = torch.empty(
|
||||
x.shape[0],
|
||||
1,
|
||||
device=x.device,
|
||||
dtype=torch.float32,
|
||||
x_q = (
|
||||
output_q
|
||||
if output_q is not None
|
||||
else torch.empty_like(x, device=x.device, dtype=dtype)
|
||||
)
|
||||
x_s = (
|
||||
output_s
|
||||
if output_s is not None
|
||||
else torch.empty(
|
||||
x.shape[0],
|
||||
1,
|
||||
device=x.device,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
)
|
||||
|
||||
sgl_per_token_quant_fp8(x, x_q, x_s)
|
||||
|
||||
@@ -1663,7 +1663,10 @@ def dequant_mxfp4(
|
||||
|
||||
|
||||
def input_to_float8(
|
||||
x: torch.Tensor, dtype: torch.dtype = fp8_dtype
|
||||
x: torch.Tensor,
|
||||
dtype: torch.dtype = fp8_dtype,
|
||||
*,
|
||||
out: Optional[torch.Tensor] = None,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
"""This function quantizes input values to float8 values with tensor-wise quantization."""
|
||||
min_val, max_val = x.aminmax()
|
||||
@@ -1677,6 +1680,12 @@ def input_to_float8(
|
||||
fp_max = finfo.max
|
||||
|
||||
scale = fp_max / amax
|
||||
if out is not None:
|
||||
assert out.shape == x.shape and out.dtype == dtype
|
||||
assert x.dtype in (torch.float16, torch.bfloat16, torch.float32)
|
||||
# FP32 broadcast avoids a full input copy; absmax bounds the FP8 range.
|
||||
torch.mul(x, scale.reshape((1,) * x.ndim), out=out)
|
||||
return out, scale.float().reciprocal()
|
||||
x_scl_sat = (x.float() * scale).clamp(min=-fp_max, max=fp_max)
|
||||
return x_scl_sat.to(dtype).contiguous(), scale.float().reciprocal()
|
||||
|
||||
|
||||
@@ -33,7 +33,6 @@ from typing import (
|
||||
List,
|
||||
Optional,
|
||||
Tuple,
|
||||
Union,
|
||||
cast,
|
||||
)
|
||||
|
||||
@@ -1074,6 +1073,8 @@ class DefaultModelLoader(BaseModelLoader):
|
||||
|
||||
@staticmethod
|
||||
def postprocess_weights(model, target_device):
|
||||
if QuantizedRLModelLoader.is_reload_scenario(model):
|
||||
return
|
||||
for module, quant_method in _modules_with_quant_method(model):
|
||||
# When quant methods need to process weights after loading
|
||||
# (for repacking, quantizing, etc), they expect parameters
|
||||
@@ -1086,6 +1087,8 @@ class DefaultModelLoader(BaseModelLoader):
|
||||
@staticmethod
|
||||
def restore_weights_before_loading(model, target_device):
|
||||
"""Undo in-place quant packing so fresh weights can be loaded."""
|
||||
if QuantizedRLModelLoader.is_reload_scenario(model):
|
||||
return
|
||||
for module, quant_method in _modules_with_quant_method(model):
|
||||
# AMX packing and the MXFP4 backend wrappers are duck-typed and cannot restore
|
||||
if isinstance(quant_method, QuantizeMethodBase):
|
||||
@@ -1156,14 +1159,12 @@ class LayeredModelLoader(DefaultModelLoader):
|
||||
|
||||
|
||||
class QuantizedRLModelLoader(DefaultModelLoader):
|
||||
"""
|
||||
Model loader for RL training with FP8 quantization (profile-free, native SGLang).
|
||||
"""FP8 RL loader for audited Qwen2 (legacy/v2) and Qwen3 native loaders.
|
||||
|
||||
Workflow:
|
||||
1. Initial load: Load base model → Record state → Apply FP8 quantization
|
||||
2. Training Actor in full precision
|
||||
3. Reload: Trainer sends full precision weights → Quantize to FP8 → Copy to original memory
|
||||
4. Use torch.as_strided to preserve memory locations across reloads
|
||||
Initial loads record native parameter loaders before quantization. Reloads
|
||||
invoke the native loader once, deferring FP8 writes; each local destination
|
||||
is staged and quantized into its existing FP8 weight and scale buffers.
|
||||
Unsupported native loaders are rejected before initialization loads weights.
|
||||
|
||||
Usage:
|
||||
--model-path Qwen/Qwen2.5-7B --quantization fp8 --load-format flash_rl
|
||||
@@ -1181,23 +1182,6 @@ class QuantizedRLModelLoader(DefaultModelLoader):
|
||||
"_assert_and_load",
|
||||
]
|
||||
|
||||
# Parameters to skip during FP8 quantization (matches FlashRL's exclude_list)
|
||||
SKIP_QUANTIZATION_PARAMS = [
|
||||
"weight_scale",
|
||||
"input_scale",
|
||||
"output_scale",
|
||||
".bias",
|
||||
"lm_head.weight",
|
||||
"model.norm.weight",
|
||||
"embed_tokens", # BF16 params
|
||||
"rotary_emb.inv_freq",
|
||||
"rotary_emb.cos_cached",
|
||||
"rotary_emb.sin_cached",
|
||||
"projector",
|
||||
"input_layernorm.weight",
|
||||
"post_attention_layernorm.weight", # LayerNorms
|
||||
]
|
||||
|
||||
# Stacked parameters (Qwen2): shards loaded separately, then combined
|
||||
STACKED_PARAMS_MAPPING = [
|
||||
("qkv_proj", ["q_proj", "k_proj", "v_proj"]),
|
||||
@@ -1215,14 +1199,18 @@ class QuantizedRLModelLoader(DefaultModelLoader):
|
||||
self._initial_load_complete = False
|
||||
|
||||
def _prepare_weights(
|
||||
self, model_name_or_path: str, revision: Optional[str], fall_back_to_pt: bool
|
||||
self,
|
||||
model_name_or_path: str,
|
||||
revision: Optional[str],
|
||||
fall_back_to_pt: bool,
|
||||
allow_patterns_overrides: Optional[list[str]] = None,
|
||||
):
|
||||
"""Standard weight preparation using base model path."""
|
||||
logger.info(f"[QuantizedRL] Loading from base model: {model_name_or_path}")
|
||||
temp_config = LoadConfig(load_format=LoadFormat.AUTO)
|
||||
temp_loader = DefaultModelLoader(temp_config)
|
||||
return temp_loader._prepare_weights(
|
||||
model_name_or_path, revision, fall_back_to_pt
|
||||
model_name_or_path, revision, fall_back_to_pt, allow_patterns_overrides
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
@@ -1242,6 +1230,14 @@ class QuantizedRLModelLoader(DefaultModelLoader):
|
||||
logger.info("[QuantizedRL] Initial load with FP8 quantization")
|
||||
|
||||
original_load_weights = model.load_weights
|
||||
native = getattr(original_load_weights, "__func__", original_load_weights)
|
||||
if getattr(native, "_supports_quantized_rl_reload", None) is not native:
|
||||
raise ValueError(
|
||||
f"FlashRL does not support {type(model).__name__}.load_weights: "
|
||||
"the native loader has not declared the deferred FP8-write contract. "
|
||||
"Supported native loaders are Qwen2ForCausalLM (legacy/v2) and "
|
||||
"Qwen3ForCausalLM; use a regular load format for other models."
|
||||
)
|
||||
|
||||
def load_weights_proxy(weights):
|
||||
if QuantizedRLModelLoader.is_reload_scenario(model):
|
||||
@@ -1301,331 +1297,179 @@ class QuantizedRLModelLoader(DefaultModelLoader):
|
||||
return (
|
||||
hasattr(model, "original_weights_rebuild_keys")
|
||||
and hasattr(model, "recorded_loader")
|
||||
and getattr(model, "flash_rl_initial_load_complete", False)
|
||||
and getattr(model, "flash_rl_initial_load_complete", False) is True
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _is_stacked_param(name):
|
||||
"""Check if parameter is stacked (qkv_proj, gate_up_proj)."""
|
||||
for stacked_name, _ in QuantizedRLModelLoader.STACKED_PARAMS_MAPPING:
|
||||
if stacked_name in name:
|
||||
return True
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _resolve_stacked_info(name: str) -> Tuple[str, Optional[str], Optional[Any]]:
|
||||
for target, shard_names in QuantizedRLModelLoader.STACKED_PARAMS_MAPPING:
|
||||
for idx, shard in enumerate(shard_names):
|
||||
if shard in name:
|
||||
components = name.split(".")
|
||||
if shard in components:
|
||||
components[components.index(shard)] = target
|
||||
shard_id = (
|
||||
QuantizedRLModelLoader._QKV_SHARD_ALIASES.get(shard, shard)
|
||||
QuantizedRLModelLoader._QKV_SHARD_ALIASES[shard]
|
||||
if target == "qkv_proj"
|
||||
else idx
|
||||
)
|
||||
return name.replace(shard, target), target, shard_id
|
||||
return ".".join(components), target, shard_id
|
||||
return name, None, None
|
||||
|
||||
@staticmethod
|
||||
def _store_quantized_scale(
|
||||
scale_store: Dict[str, Union[torch.Tensor, Dict[Any, torch.Tensor]]],
|
||||
name: str,
|
||||
scale: torch.Tensor,
|
||||
) -> None:
|
||||
param_name, stacked_key, shard_id = (
|
||||
QuantizedRLModelLoader._resolve_stacked_info(name)
|
||||
)
|
||||
if stacked_key is None:
|
||||
scale_store[param_name] = scale
|
||||
else:
|
||||
shard_dict = scale_store.setdefault(param_name, {})
|
||||
assert isinstance(shard_dict, dict)
|
||||
shard_dict[shard_id] = scale
|
||||
|
||||
@staticmethod
|
||||
def _apply_scale_update(
|
||||
all_params: Dict[str, torch.nn.Parameter],
|
||||
param_name: str,
|
||||
scale_info: Union[torch.Tensor, Dict[Any, torch.Tensor], None],
|
||||
) -> None:
|
||||
if scale_info is None:
|
||||
return
|
||||
# Get tp rank and size
|
||||
tp_rank = get_parallel().tp_rank
|
||||
tp_size = get_parallel().tp_size
|
||||
|
||||
def _get_tp_sharded_scale(full_scale_tensor):
|
||||
"""Get tp sharded scale from full scale tensor"""
|
||||
if tp_size == 1:
|
||||
return full_scale_tensor
|
||||
|
||||
full_dim = full_scale_tensor.shape[0]
|
||||
shard_dim = full_dim // tp_size
|
||||
start_idx = tp_rank * shard_dim
|
||||
end_idx = start_idx + shard_dim
|
||||
return full_scale_tensor[start_idx:end_idx]
|
||||
|
||||
if param_name.endswith(".weight"):
|
||||
scale_param_name = f"{param_name[:-7]}.weight_scale"
|
||||
else:
|
||||
scale_param_name = f"{param_name}.weight_scale"
|
||||
|
||||
scale_param = all_params.get(scale_param_name)
|
||||
if scale_param is None:
|
||||
logger.warning(
|
||||
"[QuantizedRL] Scale parameter not found: %s", scale_param_name
|
||||
)
|
||||
return
|
||||
if isinstance(scale_info, torch.Tensor):
|
||||
new_scale = scale_info.t().contiguous()
|
||||
if scale_param.data.shape == new_scale.shape:
|
||||
scale_param.data.copy_(new_scale)
|
||||
else:
|
||||
logger.warning(
|
||||
"[QuantizedRL] Scale shape mismatch for %s: expected %s, got %s",
|
||||
scale_param_name,
|
||||
scale_param.data.shape,
|
||||
new_scale.shape,
|
||||
)
|
||||
else:
|
||||
stacked_key = next(
|
||||
(
|
||||
target
|
||||
for target, _ in QuantizedRLModelLoader.STACKED_PARAMS_MAPPING
|
||||
if target in param_name
|
||||
),
|
||||
None,
|
||||
)
|
||||
shard_names = next(
|
||||
(
|
||||
names
|
||||
for target, names in QuantizedRLModelLoader.STACKED_PARAMS_MAPPING
|
||||
if target == stacked_key
|
||||
),
|
||||
[],
|
||||
)
|
||||
rows_per_shard = scale_param.data.shape[-1] // max(len(shard_names), 1)
|
||||
if rows_per_shard * len(shard_names) != scale_param.data.shape[-1]:
|
||||
logger.warning(
|
||||
f"Scale param shape {scale_param.data.shape[-1]} not divisible by {len(shard_names)}"
|
||||
)
|
||||
offset = 0
|
||||
for idx, shard in enumerate(shard_names):
|
||||
shard_id = (
|
||||
QuantizedRLModelLoader._QKV_SHARD_ALIASES.get(shard, shard)
|
||||
if stacked_key == "qkv_proj"
|
||||
else idx
|
||||
)
|
||||
shard_scale = scale_info.get(shard_id)
|
||||
shard_scale = _get_tp_sharded_scale(shard_scale)
|
||||
if shard_scale is None:
|
||||
offset += rows_per_shard
|
||||
continue
|
||||
shard_rows = shard_scale.shape[0]
|
||||
start = offset
|
||||
end = start + shard_rows
|
||||
scale_param.data[..., start:end] = shard_scale.t().contiguous()
|
||||
offset = end
|
||||
|
||||
@staticmethod
|
||||
def rebinding_and_load_weights(model, first_time_load_weights, weights):
|
||||
"""
|
||||
Reload: VERL sends BF16 → Quantize to FP8 → Copy to original memory.
|
||||
|
||||
Flow: Reset params → Restore attributes → Quantize in iterator → Load → Copy back
|
||||
"""
|
||||
logger.info("[QuantizedRL] Reload: Updating weights with FP8 quantization")
|
||||
"""Shard new BF16 weights natively, then quantize into their original buffers."""
|
||||
from sglang.kernels.ops.quantization.fp8_kernel import (
|
||||
per_token_group_quant_fp8,
|
||||
)
|
||||
from sglang.srt.layers.quantization.fp8_utils import input_to_float8
|
||||
|
||||
weights_list = list(weights)
|
||||
for name, _ in weights_list:
|
||||
if name.startswith(("layers.", "embed_tokens.", "norm.")):
|
||||
raise ValueError(
|
||||
f"FlashRL reload requires canonical checkpoint names: {name!r} "
|
||||
"must include the 'model.' prefix."
|
||||
)
|
||||
updated_param_names, is_last_update = (
|
||||
QuantizedRLModelLoader._get_updated_params(weights_list, model)
|
||||
)
|
||||
|
||||
# Save current FP8 parameter data pointers
|
||||
existing_params = dict(model.named_parameters())
|
||||
current_param_data = {}
|
||||
for name in updated_param_names:
|
||||
if name in existing_params:
|
||||
current_param_data[name] = existing_params[name].data
|
||||
|
||||
# Reset to pre-quantization shape using torch.as_strided
|
||||
# Keeps same storage, just changes view - critical for memory preservation
|
||||
for name, rebuild_info in model.original_weights_rebuild_keys.items():
|
||||
if name in updated_param_names and name in existing_params:
|
||||
existing_params[name].data = torch.as_strided(
|
||||
# Note: avoid clone here
|
||||
existing_params[name].data.clone(),
|
||||
rebuild_info["shape"],
|
||||
rebuild_info["stride"],
|
||||
)
|
||||
|
||||
# Restore weight loader attributes (only if missing)
|
||||
for k, loader_dict in model.recorded_loader.items():
|
||||
for param_name, loader in loader_dict.items():
|
||||
if param_name in updated_param_names and param_name in existing_params:
|
||||
param = existing_params[param_name]
|
||||
if not hasattr(param, k):
|
||||
if callable(loader):
|
||||
if hasattr(loader, "__self__"):
|
||||
setattr(param, k, loader)
|
||||
else:
|
||||
setattr(
|
||||
param,
|
||||
k,
|
||||
QuantizedRLModelLoader._bind_method_to_cls(
|
||||
loader, param
|
||||
),
|
||||
)
|
||||
else:
|
||||
setattr(param, k, loader)
|
||||
|
||||
del existing_params
|
||||
|
||||
# Quantize BF16 weights to FP8 in iterator (before weight_loader)
|
||||
# Store scales for later update
|
||||
quantized_scales: Dict[str, Union[torch.Tensor, Dict[Any, torch.Tensor]]] = {}
|
||||
|
||||
def quantize_weights_iterator(weights_iter):
|
||||
"""Quantize individual shards before weight_loader stacks them."""
|
||||
from sglang.kernels.ops.quantization.fp8_kernel import (
|
||||
per_token_group_quant_fp8,
|
||||
params = dict(model.named_parameters())
|
||||
grouped = collections.defaultdict(list)
|
||||
complete_params = set()
|
||||
for source_name, weight in weights_list:
|
||||
name, _, shard_id = QuantizedRLModelLoader._resolve_stacked_info(
|
||||
source_name
|
||||
)
|
||||
grouped[name].append((source_name, weight))
|
||||
if shard_id is None:
|
||||
complete_params.add(name)
|
||||
|
||||
for name, weight in weights_iter:
|
||||
if any(
|
||||
skip in name
|
||||
for skip in QuantizedRLModelLoader.SKIP_QUANTIZATION_PARAMS
|
||||
):
|
||||
logger.info(f"[QuantizedRL] Skip: {name} ({weight.dtype})")
|
||||
yield (name, weight)
|
||||
elif weight.dtype in [torch.bfloat16, torch.float32, torch.float16]:
|
||||
qweight, scale = per_token_group_quant_fp8(weight, weight.shape[-1])
|
||||
logger.info(f"[QuantizedRL] Quantize: {name} {weight.dtype}→FP8")
|
||||
QuantizedRLModelLoader._store_quantized_scale(
|
||||
quantized_scales, name, scale
|
||||
row_masks = {}
|
||||
# Check coverage before modifying any model parameter. Per-tensor
|
||||
# quantization needs the original BF16 values for every destination row.
|
||||
for name, sources in grouped.items():
|
||||
if name not in params or params[name].dtype not in (
|
||||
torch.float8_e4m3fn,
|
||||
torch.float8_e4m3fnuz,
|
||||
):
|
||||
continue
|
||||
rebuild = model.original_weights_rebuild_keys[name]
|
||||
layer = model.get_submodule(name.rpartition(".")[0])
|
||||
per_channel = layer.weight_scale.shape == (1, rebuild["shape"][0])
|
||||
if name in complete_params:
|
||||
row_masks[name] = (None, per_channel)
|
||||
continue
|
||||
marker = nn.Parameter(
|
||||
torch.zeros((rebuild["shape"][0], 1), device="cpu"), requires_grad=False
|
||||
)
|
||||
marker.output_dim = 0
|
||||
for source_name, weight in sources:
|
||||
_, _, shard_id = QuantizedRLModelLoader._resolve_stacked_info(
|
||||
source_name
|
||||
)
|
||||
args = () if shard_id is None else (shard_id,)
|
||||
layer.weight_loader(
|
||||
marker, torch.ones((weight.shape[0], 1), device="cpu"), *args
|
||||
)
|
||||
rows = marker.data[:, 0].bool()
|
||||
if not per_channel and not rows.all():
|
||||
raise ValueError(
|
||||
f"Partial per-tensor FP8 update for {name}: supply all destination "
|
||||
"rows in the same update call, including inside a weight-update session."
|
||||
)
|
||||
row_masks[name] = (rows, per_channel)
|
||||
|
||||
# Defer only FP8 writes so one native call preserves cross-key lookups
|
||||
# and the order of non-FP8 loads, including tied embeddings.
|
||||
writes = collections.defaultdict(list)
|
||||
native_loaders = {}
|
||||
for name in row_masks:
|
||||
param = params[name]
|
||||
for key, loaders in model.recorded_loader.items():
|
||||
if name in loaders and not hasattr(param, key):
|
||||
loader = loaders[name]
|
||||
setattr(
|
||||
param,
|
||||
key,
|
||||
QuantizedRLModelLoader._bind_method_to_cls(loader, param)
|
||||
if callable(loader)
|
||||
else loader,
|
||||
)
|
||||
yield (name, qweight)
|
||||
native_loaders[name] = param.weight_loader
|
||||
|
||||
def record_write(*args, _name=name, **kwargs):
|
||||
writes[_name].append((args, kwargs))
|
||||
|
||||
param.weight_loader = record_write
|
||||
try:
|
||||
first_time_load_weights(weights_list)
|
||||
finally:
|
||||
for name, loader in native_loaders.items():
|
||||
params[name].weight_loader = loader
|
||||
|
||||
for name, calls in writes.items():
|
||||
param = params[name]
|
||||
data = param.data
|
||||
rebuild = model.original_weights_rebuild_keys[name]
|
||||
layer = model.get_submodule(name.rpartition(".")[0])
|
||||
rows, per_channel = row_masks[name]
|
||||
complete = rows is None or bool(rows.all())
|
||||
staging = torch.empty_strided(
|
||||
rebuild["shape"],
|
||||
rebuild["stride"],
|
||||
dtype=rebuild["dtype"],
|
||||
device=data.device,
|
||||
)
|
||||
param.data = staging
|
||||
try:
|
||||
for args, kwargs in calls:
|
||||
native_loaders[name](*args, **kwargs)
|
||||
local_weight = torch.as_strided(
|
||||
data, rebuild["shape"], rebuild["stride"]
|
||||
)
|
||||
if per_channel:
|
||||
# Untouched rows never pass through dequantization.
|
||||
if complete:
|
||||
ranges = [(0, staging.shape[0])]
|
||||
else:
|
||||
padded = torch.cat((rows.new_zeros(1), rows, rows.new_zeros(1)))
|
||||
edges = (padded[1:] != padded[:-1]).nonzero().flatten().tolist()
|
||||
ranges = zip(edges[::2], edges[1::2])
|
||||
for begin, end in ranges:
|
||||
per_token_group_quant_fp8(
|
||||
staging[begin:end],
|
||||
staging.shape[-1],
|
||||
output_q=local_weight[begin:end],
|
||||
output_s=layer.weight_scale.data[:, begin:end].t(),
|
||||
)
|
||||
else:
|
||||
logger.info(f"[QuantizedRL] Keep: {name} ({weight.dtype})")
|
||||
yield (name, weight)
|
||||
_, scale = input_to_float8(
|
||||
staging, dtype=data.dtype, out=local_weight
|
||||
)
|
||||
layer.weight_scale.data.copy_(scale)
|
||||
finally:
|
||||
param.data = data
|
||||
# No BF16 destination survives to the next destination or call.
|
||||
del staging
|
||||
|
||||
# Load quantized weights (weight_loader stacks FP8 shards)
|
||||
first_time_load_weights(quantize_weights_iterator(iter(weights_list)))
|
||||
|
||||
# Copy back to original FP8 memory locations and update scales
|
||||
all_params = dict(model.named_parameters())
|
||||
|
||||
for name in updated_param_names:
|
||||
if name not in all_params or name not in current_param_data:
|
||||
continue
|
||||
if any(
|
||||
skip in name for skip in QuantizedRLModelLoader.SKIP_QUANTIZATION_PARAMS
|
||||
):
|
||||
continue
|
||||
|
||||
new_param = all_params[name]
|
||||
old_fp8_data = current_param_data[name]
|
||||
|
||||
# Handle embeddings/lm_head (BF16) and quantized weights (FP8)
|
||||
if "embed_tokens" in name or "lm_head" in name:
|
||||
old_fp8_data.copy_(new_param.data)
|
||||
new_param.data = old_fp8_data
|
||||
elif (
|
||||
new_param.dtype == torch.float8_e4m3fn
|
||||
and old_fp8_data.dtype == torch.float8_e4m3fn
|
||||
):
|
||||
# FP8: Use strided view for transposed storage
|
||||
strided_data = torch.as_strided(
|
||||
new_param.data, old_fp8_data.shape, old_fp8_data.stride()
|
||||
)
|
||||
old_fp8_data.copy_(strided_data)
|
||||
new_param.data = old_fp8_data
|
||||
QuantizedRLModelLoader._apply_scale_update(
|
||||
all_params,
|
||||
name,
|
||||
quantized_scales.get(name),
|
||||
)
|
||||
elif new_param.dtype == old_fp8_data.dtype:
|
||||
# Same dtype (LayerNorm, etc.): Direct copy
|
||||
old_fp8_data.copy_(new_param.data)
|
||||
new_param.data = old_fp8_data
|
||||
else:
|
||||
raise RuntimeError(
|
||||
f"Unexpected dtype mismatch for {name}: "
|
||||
f"new={new_param.dtype}, old={old_fp8_data.dtype}"
|
||||
)
|
||||
|
||||
# Cleanup
|
||||
del current_param_data
|
||||
if is_last_update:
|
||||
gc.collect()
|
||||
current_platform.empty_cache()
|
||||
|
||||
logger.info("[QuantizedRL] Reload complete")
|
||||
return updated_param_names, is_last_update
|
||||
|
||||
@staticmethod
|
||||
def _get_updated_params(weights_list, model):
|
||||
"""Identify which parameters need updating from incoming weights."""
|
||||
stacked_params_mapping = [
|
||||
("qkv_proj", "q_proj", "q"),
|
||||
("qkv_proj", "k_proj", "k"),
|
||||
("qkv_proj", "v_proj", "v"),
|
||||
("gate_up_proj", "gate_proj", 0),
|
||||
("gate_up_proj", "up_proj", 1),
|
||||
]
|
||||
|
||||
params_dict = dict(model.named_parameters())
|
||||
updated_params = set()
|
||||
"""Resolve checkpoint names to parameters owned by this model shard."""
|
||||
params = dict(model.named_parameters())
|
||||
updated = set()
|
||||
is_last_update = False
|
||||
|
||||
for name, _ in weights_list:
|
||||
if name == "lm_head.weight":
|
||||
is_last_update = True
|
||||
|
||||
if any(
|
||||
skip in name for skip in QuantizedRLModelLoader.SKIP_QUANTIZATION_PARAMS
|
||||
):
|
||||
continue
|
||||
|
||||
from sglang.srt.layers.utils import get_layer_id
|
||||
|
||||
# Skip params outside layer range (for pipeline parallelism)
|
||||
layer_id = get_layer_id(name)
|
||||
if (
|
||||
layer_id is not None
|
||||
and hasattr(model, "start_layer")
|
||||
and (layer_id < model.start_layer or layer_id >= model.end_layer)
|
||||
):
|
||||
continue
|
||||
|
||||
# Skip tied embeddings and vision tower params
|
||||
if (
|
||||
hasattr(model, "config")
|
||||
and model.config.tie_word_embeddings
|
||||
and "lm_head.weight" in name
|
||||
):
|
||||
continue
|
||||
if name.startswith("model.vision_tower") and name not in params_dict:
|
||||
continue
|
||||
|
||||
# Map stacked param shards (q/k/v_proj → qkv_proj)
|
||||
mapped = False
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name in name:
|
||||
name = name.replace(weight_name, param_name)
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
updated_params.add(name)
|
||||
mapped = True
|
||||
break
|
||||
|
||||
if not mapped:
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
if name in params_dict:
|
||||
updated_params.add(name)
|
||||
|
||||
return list(updated_params), is_last_update
|
||||
name, _, _ = QuantizedRLModelLoader._resolve_stacked_info(name)
|
||||
if name in params:
|
||||
updated.add(name)
|
||||
return sorted(updated), is_last_update
|
||||
|
||||
|
||||
class DummyModelLoader(BaseModelLoader):
|
||||
|
||||
@@ -1474,6 +1474,18 @@ def gguf_quant_weights_iterator(
|
||||
yield name, param
|
||||
|
||||
|
||||
def supports_quantized_rl_reload(func: Callable) -> Callable:
|
||||
"""Allow FlashRL to defer FP8 parameter writes until this method returns.
|
||||
|
||||
Checkpoint names must use the parameter names or QKV/gate-up aliases supported
|
||||
by QuantizedRLModelLoader. FP8 writes must go through param.weight_loader;
|
||||
the method must not read their values afterward, and source tensors must
|
||||
remain unchanged until return. Non-FP8 parameters are loaded immediately.
|
||||
"""
|
||||
func._supports_quantized_rl_reload = func
|
||||
return func
|
||||
|
||||
|
||||
def default_weight_loader(param: torch.Tensor, loaded_weight: torch.Tensor) -> None:
|
||||
"""Default weight loader."""
|
||||
if param.numel() == 1 and loaded_weight.numel() == 1:
|
||||
|
||||
@@ -55,6 +55,7 @@ from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTe
|
||||
from sglang.srt.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
kv_cache_scales_loader,
|
||||
supports_quantized_rl_reload,
|
||||
)
|
||||
from sglang.srt.platforms import current_platform
|
||||
from sglang.srt.runtime_context import get_exec, get_parallel
|
||||
@@ -616,6 +617,7 @@ class Qwen2ForCausalLM(nn.Module):
|
||||
def end_layer(self):
|
||||
return self.model.end_layer
|
||||
|
||||
@supports_quantized_rl_reload
|
||||
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
|
||||
from sglang.srt.environ import envs
|
||||
|
||||
|
||||
@@ -32,6 +32,7 @@ from sglang.srt.model_executor.forward_context import get_token_to_kv_pool
|
||||
from sglang.srt.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
maybe_remap_kv_scale_name,
|
||||
supports_quantized_rl_reload,
|
||||
)
|
||||
from sglang.srt.models.qwen2 import Qwen2MLP as Qwen3MLP
|
||||
from sglang.srt.models.qwen2 import Qwen2Model
|
||||
@@ -620,6 +621,7 @@ class Qwen3ForCausalLM(nn.Module):
|
||||
def end_layer(self):
|
||||
return self.model.end_layer
|
||||
|
||||
@supports_quantized_rl_reload
|
||||
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
|
||||
stacked_params_mapping = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
|
||||
Reference in New Issue
Block a user