[Fix] Quantize reloaded FP8 weights after native sharding (#42585)

This commit is contained in:
Cheng Wan
2026-10-07 16:20:47 -07:00
committed by GitHub
parent 59740a728f
commit b83827ea02
8 changed files with 783 additions and 341 deletions
@@ -214,6 +214,9 @@ def _per_token_group_quant_8bit_raw(
column_major_scales: bool = False,
scale_tma_aligned: bool = False,
scale_ue8m0: bool = False,
*,
output_q: Optional[torch.Tensor] = None,
output_s: Optional[torch.Tensor] = None,
) -> Tuple[torch.Tensor, torch.Tensor]:
"""Function to perform per-token-group quantization on an input tensor `x`.
@@ -251,14 +254,22 @@ def _per_token_group_quant_8bit_raw(
bit8_max = info.max
bit8_min = info.min
x_q = torch.empty_like(x, device=x.device, dtype=dtype)
x_s = create_per_token_group_quant_fp8_output_scale(
x_shape=x.shape,
device=x.device,
group_size=group_size,
column_major_scales=column_major_scales,
scale_tma_aligned=scale_tma_aligned,
scale_ue8m0=False,
x_q = (
output_q
if output_q is not None
else torch.empty_like(x, device=x.device, dtype=dtype)
)
x_s = (
output_s
if output_s is not None
else create_per_token_group_quant_fp8_output_scale(
x_shape=x.shape,
device=x.device,
group_size=group_size,
column_major_scales=column_major_scales,
scale_tma_aligned=scale_tma_aligned,
scale_ue8m0=False,
)
)
M = x.numel() // group_size
@@ -573,6 +584,9 @@ def sglang_per_token_group_quant_fp8(
scale_ue8m0: bool = False,
fuse_silu_and_mul: bool = False,
masked_m: Optional[torch.Tensor] = None,
*,
output_q: Optional[torch.Tensor] = None,
output_s: Optional[torch.Tensor] = None,
):
assert x.shape[-1] % group_size == 0, (
"the last dimension of `x` cannot be divisible by `group_size`"
@@ -588,18 +602,34 @@ def sglang_per_token_group_quant_fp8(
# Whole-row group quant is per-token quant; route to the dedicated
# kernel (same [T, 1] scale shape) instead of a group kernel that
# would need arbitrary group sizes.
return sglang_per_token_quant_fp8(x)
if output_q is not None and output_q.dtype != fp8_dtype:
return _per_token_group_quant_8bit_raw(
x,
group_size,
dtype=output_q.dtype,
output_q=output_q,
output_s=output_s,
)
return sglang_per_token_quant_fp8(x, output_q=output_q, output_s=output_s)
out_shape = (*x.shape[:-1], x.shape[-1] // (2 if fuse_silu_and_mul else 1))
x_q = torch.empty(out_shape, device=x.device, dtype=fp8_dtype)
x_s = create_per_token_group_quant_fp8_output_scale(
x_shape=out_shape,
device=x.device,
group_size=group_size,
column_major_scales=column_major_scales,
scale_tma_aligned=scale_tma_aligned,
scale_ue8m0=scale_ue8m0,
x_q = (
output_q
if output_q is not None
else torch.empty(out_shape, device=x.device, dtype=fp8_dtype)
)
x_s = (
output_s
if output_s is not None
else create_per_token_group_quant_fp8_output_scale(
x_shape=out_shape,
device=x.device,
group_size=group_size,
column_major_scales=column_major_scales,
scale_tma_aligned=scale_tma_aligned,
scale_ue8m0=scale_ue8m0,
)
)
if x.shape[0] > 0:
@@ -764,15 +794,26 @@ def sglang_per_token_group_quant_8bit(
def sglang_per_token_quant_fp8(
x: torch.Tensor,
dtype: torch.dtype = fp8_dtype,
*,
output_q: Optional[torch.Tensor] = None,
output_s: Optional[torch.Tensor] = None,
):
assert x.is_contiguous(), "`x` is not contiguous"
x_q = torch.empty_like(x, device=x.device, dtype=dtype)
x_s = torch.empty(
x.shape[0],
1,
device=x.device,
dtype=torch.float32,
x_q = (
output_q
if output_q is not None
else torch.empty_like(x, device=x.device, dtype=dtype)
)
x_s = (
output_s
if output_s is not None
else torch.empty(
x.shape[0],
1,
device=x.device,
dtype=torch.float32,
)
)
sgl_per_token_quant_fp8(x, x_q, x_s)
@@ -1663,7 +1663,10 @@ def dequant_mxfp4(
def input_to_float8(
x: torch.Tensor, dtype: torch.dtype = fp8_dtype
x: torch.Tensor,
dtype: torch.dtype = fp8_dtype,
*,
out: Optional[torch.Tensor] = None,
) -> Tuple[torch.Tensor, torch.Tensor]:
"""This function quantizes input values to float8 values with tensor-wise quantization."""
min_val, max_val = x.aminmax()
@@ -1677,6 +1680,12 @@ def input_to_float8(
fp_max = finfo.max
scale = fp_max / amax
if out is not None:
assert out.shape == x.shape and out.dtype == dtype
assert x.dtype in (torch.float16, torch.bfloat16, torch.float32)
# FP32 broadcast avoids a full input copy; absmax bounds the FP8 range.
torch.mul(x, scale.reshape((1,) * x.ndim), out=out)
return out, scale.float().reciprocal()
x_scl_sat = (x.float() * scale).clamp(min=-fp_max, max=fp_max)
return x_scl_sat.to(dtype).contiguous(), scale.float().reciprocal()
+161 -317
View File
@@ -33,7 +33,6 @@ from typing import (
List,
Optional,
Tuple,
Union,
cast,
)
@@ -1074,6 +1073,8 @@ class DefaultModelLoader(BaseModelLoader):
@staticmethod
def postprocess_weights(model, target_device):
if QuantizedRLModelLoader.is_reload_scenario(model):
return
for module, quant_method in _modules_with_quant_method(model):
# When quant methods need to process weights after loading
# (for repacking, quantizing, etc), they expect parameters
@@ -1086,6 +1087,8 @@ class DefaultModelLoader(BaseModelLoader):
@staticmethod
def restore_weights_before_loading(model, target_device):
"""Undo in-place quant packing so fresh weights can be loaded."""
if QuantizedRLModelLoader.is_reload_scenario(model):
return
for module, quant_method in _modules_with_quant_method(model):
# AMX packing and the MXFP4 backend wrappers are duck-typed and cannot restore
if isinstance(quant_method, QuantizeMethodBase):
@@ -1156,14 +1159,12 @@ class LayeredModelLoader(DefaultModelLoader):
class QuantizedRLModelLoader(DefaultModelLoader):
"""
Model loader for RL training with FP8 quantization (profile-free, native SGLang).
"""FP8 RL loader for audited Qwen2 (legacy/v2) and Qwen3 native loaders.
Workflow:
1. Initial load: Load base model → Record state → Apply FP8 quantization
2. Training Actor in full precision
3. Reload: Trainer sends full precision weights → Quantize to FP8 → Copy to original memory
4. Use torch.as_strided to preserve memory locations across reloads
Initial loads record native parameter loaders before quantization. Reloads
invoke the native loader once, deferring FP8 writes; each local destination
is staged and quantized into its existing FP8 weight and scale buffers.
Unsupported native loaders are rejected before initialization loads weights.
Usage:
--model-path Qwen/Qwen2.5-7B --quantization fp8 --load-format flash_rl
@@ -1181,23 +1182,6 @@ class QuantizedRLModelLoader(DefaultModelLoader):
"_assert_and_load",
]
# Parameters to skip during FP8 quantization (matches FlashRL's exclude_list)
SKIP_QUANTIZATION_PARAMS = [
"weight_scale",
"input_scale",
"output_scale",
".bias",
"lm_head.weight",
"model.norm.weight",
"embed_tokens", # BF16 params
"rotary_emb.inv_freq",
"rotary_emb.cos_cached",
"rotary_emb.sin_cached",
"projector",
"input_layernorm.weight",
"post_attention_layernorm.weight", # LayerNorms
]
# Stacked parameters (Qwen2): shards loaded separately, then combined
STACKED_PARAMS_MAPPING = [
("qkv_proj", ["q_proj", "k_proj", "v_proj"]),
@@ -1215,14 +1199,18 @@ class QuantizedRLModelLoader(DefaultModelLoader):
self._initial_load_complete = False
def _prepare_weights(
self, model_name_or_path: str, revision: Optional[str], fall_back_to_pt: bool
self,
model_name_or_path: str,
revision: Optional[str],
fall_back_to_pt: bool,
allow_patterns_overrides: Optional[list[str]] = None,
):
"""Standard weight preparation using base model path."""
logger.info(f"[QuantizedRL] Loading from base model: {model_name_or_path}")
temp_config = LoadConfig(load_format=LoadFormat.AUTO)
temp_loader = DefaultModelLoader(temp_config)
return temp_loader._prepare_weights(
model_name_or_path, revision, fall_back_to_pt
model_name_or_path, revision, fall_back_to_pt, allow_patterns_overrides
)
@staticmethod
@@ -1242,6 +1230,14 @@ class QuantizedRLModelLoader(DefaultModelLoader):
logger.info("[QuantizedRL] Initial load with FP8 quantization")
original_load_weights = model.load_weights
native = getattr(original_load_weights, "__func__", original_load_weights)
if getattr(native, "_supports_quantized_rl_reload", None) is not native:
raise ValueError(
f"FlashRL does not support {type(model).__name__}.load_weights: "
"the native loader has not declared the deferred FP8-write contract. "
"Supported native loaders are Qwen2ForCausalLM (legacy/v2) and "
"Qwen3ForCausalLM; use a regular load format for other models."
)
def load_weights_proxy(weights):
if QuantizedRLModelLoader.is_reload_scenario(model):
@@ -1301,331 +1297,179 @@ class QuantizedRLModelLoader(DefaultModelLoader):
return (
hasattr(model, "original_weights_rebuild_keys")
and hasattr(model, "recorded_loader")
and getattr(model, "flash_rl_initial_load_complete", False)
and getattr(model, "flash_rl_initial_load_complete", False) is True
)
@staticmethod
def _is_stacked_param(name):
"""Check if parameter is stacked (qkv_proj, gate_up_proj)."""
for stacked_name, _ in QuantizedRLModelLoader.STACKED_PARAMS_MAPPING:
if stacked_name in name:
return True
return False
@staticmethod
def _resolve_stacked_info(name: str) -> Tuple[str, Optional[str], Optional[Any]]:
for target, shard_names in QuantizedRLModelLoader.STACKED_PARAMS_MAPPING:
for idx, shard in enumerate(shard_names):
if shard in name:
components = name.split(".")
if shard in components:
components[components.index(shard)] = target
shard_id = (
QuantizedRLModelLoader._QKV_SHARD_ALIASES.get(shard, shard)
QuantizedRLModelLoader._QKV_SHARD_ALIASES[shard]
if target == "qkv_proj"
else idx
)
return name.replace(shard, target), target, shard_id
return ".".join(components), target, shard_id
return name, None, None
@staticmethod
def _store_quantized_scale(
scale_store: Dict[str, Union[torch.Tensor, Dict[Any, torch.Tensor]]],
name: str,
scale: torch.Tensor,
) -> None:
param_name, stacked_key, shard_id = (
QuantizedRLModelLoader._resolve_stacked_info(name)
)
if stacked_key is None:
scale_store[param_name] = scale
else:
shard_dict = scale_store.setdefault(param_name, {})
assert isinstance(shard_dict, dict)
shard_dict[shard_id] = scale
@staticmethod
def _apply_scale_update(
all_params: Dict[str, torch.nn.Parameter],
param_name: str,
scale_info: Union[torch.Tensor, Dict[Any, torch.Tensor], None],
) -> None:
if scale_info is None:
return
# Get tp rank and size
tp_rank = get_parallel().tp_rank
tp_size = get_parallel().tp_size
def _get_tp_sharded_scale(full_scale_tensor):
"""Get tp sharded scale from full scale tensor"""
if tp_size == 1:
return full_scale_tensor
full_dim = full_scale_tensor.shape[0]
shard_dim = full_dim // tp_size
start_idx = tp_rank * shard_dim
end_idx = start_idx + shard_dim
return full_scale_tensor[start_idx:end_idx]
if param_name.endswith(".weight"):
scale_param_name = f"{param_name[:-7]}.weight_scale"
else:
scale_param_name = f"{param_name}.weight_scale"
scale_param = all_params.get(scale_param_name)
if scale_param is None:
logger.warning(
"[QuantizedRL] Scale parameter not found: %s", scale_param_name
)
return
if isinstance(scale_info, torch.Tensor):
new_scale = scale_info.t().contiguous()
if scale_param.data.shape == new_scale.shape:
scale_param.data.copy_(new_scale)
else:
logger.warning(
"[QuantizedRL] Scale shape mismatch for %s: expected %s, got %s",
scale_param_name,
scale_param.data.shape,
new_scale.shape,
)
else:
stacked_key = next(
(
target
for target, _ in QuantizedRLModelLoader.STACKED_PARAMS_MAPPING
if target in param_name
),
None,
)
shard_names = next(
(
names
for target, names in QuantizedRLModelLoader.STACKED_PARAMS_MAPPING
if target == stacked_key
),
[],
)
rows_per_shard = scale_param.data.shape[-1] // max(len(shard_names), 1)
if rows_per_shard * len(shard_names) != scale_param.data.shape[-1]:
logger.warning(
f"Scale param shape {scale_param.data.shape[-1]} not divisible by {len(shard_names)}"
)
offset = 0
for idx, shard in enumerate(shard_names):
shard_id = (
QuantizedRLModelLoader._QKV_SHARD_ALIASES.get(shard, shard)
if stacked_key == "qkv_proj"
else idx
)
shard_scale = scale_info.get(shard_id)
shard_scale = _get_tp_sharded_scale(shard_scale)
if shard_scale is None:
offset += rows_per_shard
continue
shard_rows = shard_scale.shape[0]
start = offset
end = start + shard_rows
scale_param.data[..., start:end] = shard_scale.t().contiguous()
offset = end
@staticmethod
def rebinding_and_load_weights(model, first_time_load_weights, weights):
"""
Reload: VERL sends BF16 → Quantize to FP8 → Copy to original memory.
Flow: Reset params → Restore attributes → Quantize in iterator → Load → Copy back
"""
logger.info("[QuantizedRL] Reload: Updating weights with FP8 quantization")
"""Shard new BF16 weights natively, then quantize into their original buffers."""
from sglang.kernels.ops.quantization.fp8_kernel import (
per_token_group_quant_fp8,
)
from sglang.srt.layers.quantization.fp8_utils import input_to_float8
weights_list = list(weights)
for name, _ in weights_list:
if name.startswith(("layers.", "embed_tokens.", "norm.")):
raise ValueError(
f"FlashRL reload requires canonical checkpoint names: {name!r} "
"must include the 'model.' prefix."
)
updated_param_names, is_last_update = (
QuantizedRLModelLoader._get_updated_params(weights_list, model)
)
# Save current FP8 parameter data pointers
existing_params = dict(model.named_parameters())
current_param_data = {}
for name in updated_param_names:
if name in existing_params:
current_param_data[name] = existing_params[name].data
# Reset to pre-quantization shape using torch.as_strided
# Keeps same storage, just changes view - critical for memory preservation
for name, rebuild_info in model.original_weights_rebuild_keys.items():
if name in updated_param_names and name in existing_params:
existing_params[name].data = torch.as_strided(
# Note: avoid clone here
existing_params[name].data.clone(),
rebuild_info["shape"],
rebuild_info["stride"],
)
# Restore weight loader attributes (only if missing)
for k, loader_dict in model.recorded_loader.items():
for param_name, loader in loader_dict.items():
if param_name in updated_param_names and param_name in existing_params:
param = existing_params[param_name]
if not hasattr(param, k):
if callable(loader):
if hasattr(loader, "__self__"):
setattr(param, k, loader)
else:
setattr(
param,
k,
QuantizedRLModelLoader._bind_method_to_cls(
loader, param
),
)
else:
setattr(param, k, loader)
del existing_params
# Quantize BF16 weights to FP8 in iterator (before weight_loader)
# Store scales for later update
quantized_scales: Dict[str, Union[torch.Tensor, Dict[Any, torch.Tensor]]] = {}
def quantize_weights_iterator(weights_iter):
"""Quantize individual shards before weight_loader stacks them."""
from sglang.kernels.ops.quantization.fp8_kernel import (
per_token_group_quant_fp8,
params = dict(model.named_parameters())
grouped = collections.defaultdict(list)
complete_params = set()
for source_name, weight in weights_list:
name, _, shard_id = QuantizedRLModelLoader._resolve_stacked_info(
source_name
)
grouped[name].append((source_name, weight))
if shard_id is None:
complete_params.add(name)
for name, weight in weights_iter:
if any(
skip in name
for skip in QuantizedRLModelLoader.SKIP_QUANTIZATION_PARAMS
):
logger.info(f"[QuantizedRL] Skip: {name} ({weight.dtype})")
yield (name, weight)
elif weight.dtype in [torch.bfloat16, torch.float32, torch.float16]:
qweight, scale = per_token_group_quant_fp8(weight, weight.shape[-1])
logger.info(f"[QuantizedRL] Quantize: {name} {weight.dtype}→FP8")
QuantizedRLModelLoader._store_quantized_scale(
quantized_scales, name, scale
row_masks = {}
# Check coverage before modifying any model parameter. Per-tensor
# quantization needs the original BF16 values for every destination row.
for name, sources in grouped.items():
if name not in params or params[name].dtype not in (
torch.float8_e4m3fn,
torch.float8_e4m3fnuz,
):
continue
rebuild = model.original_weights_rebuild_keys[name]
layer = model.get_submodule(name.rpartition(".")[0])
per_channel = layer.weight_scale.shape == (1, rebuild["shape"][0])
if name in complete_params:
row_masks[name] = (None, per_channel)
continue
marker = nn.Parameter(
torch.zeros((rebuild["shape"][0], 1), device="cpu"), requires_grad=False
)
marker.output_dim = 0
for source_name, weight in sources:
_, _, shard_id = QuantizedRLModelLoader._resolve_stacked_info(
source_name
)
args = () if shard_id is None else (shard_id,)
layer.weight_loader(
marker, torch.ones((weight.shape[0], 1), device="cpu"), *args
)
rows = marker.data[:, 0].bool()
if not per_channel and not rows.all():
raise ValueError(
f"Partial per-tensor FP8 update for {name}: supply all destination "
"rows in the same update call, including inside a weight-update session."
)
row_masks[name] = (rows, per_channel)
# Defer only FP8 writes so one native call preserves cross-key lookups
# and the order of non-FP8 loads, including tied embeddings.
writes = collections.defaultdict(list)
native_loaders = {}
for name in row_masks:
param = params[name]
for key, loaders in model.recorded_loader.items():
if name in loaders and not hasattr(param, key):
loader = loaders[name]
setattr(
param,
key,
QuantizedRLModelLoader._bind_method_to_cls(loader, param)
if callable(loader)
else loader,
)
yield (name, qweight)
native_loaders[name] = param.weight_loader
def record_write(*args, _name=name, **kwargs):
writes[_name].append((args, kwargs))
param.weight_loader = record_write
try:
first_time_load_weights(weights_list)
finally:
for name, loader in native_loaders.items():
params[name].weight_loader = loader
for name, calls in writes.items():
param = params[name]
data = param.data
rebuild = model.original_weights_rebuild_keys[name]
layer = model.get_submodule(name.rpartition(".")[0])
rows, per_channel = row_masks[name]
complete = rows is None or bool(rows.all())
staging = torch.empty_strided(
rebuild["shape"],
rebuild["stride"],
dtype=rebuild["dtype"],
device=data.device,
)
param.data = staging
try:
for args, kwargs in calls:
native_loaders[name](*args, **kwargs)
local_weight = torch.as_strided(
data, rebuild["shape"], rebuild["stride"]
)
if per_channel:
# Untouched rows never pass through dequantization.
if complete:
ranges = [(0, staging.shape[0])]
else:
padded = torch.cat((rows.new_zeros(1), rows, rows.new_zeros(1)))
edges = (padded[1:] != padded[:-1]).nonzero().flatten().tolist()
ranges = zip(edges[::2], edges[1::2])
for begin, end in ranges:
per_token_group_quant_fp8(
staging[begin:end],
staging.shape[-1],
output_q=local_weight[begin:end],
output_s=layer.weight_scale.data[:, begin:end].t(),
)
else:
logger.info(f"[QuantizedRL] Keep: {name} ({weight.dtype})")
yield (name, weight)
_, scale = input_to_float8(
staging, dtype=data.dtype, out=local_weight
)
layer.weight_scale.data.copy_(scale)
finally:
param.data = data
# No BF16 destination survives to the next destination or call.
del staging
# Load quantized weights (weight_loader stacks FP8 shards)
first_time_load_weights(quantize_weights_iterator(iter(weights_list)))
# Copy back to original FP8 memory locations and update scales
all_params = dict(model.named_parameters())
for name in updated_param_names:
if name not in all_params or name not in current_param_data:
continue
if any(
skip in name for skip in QuantizedRLModelLoader.SKIP_QUANTIZATION_PARAMS
):
continue
new_param = all_params[name]
old_fp8_data = current_param_data[name]
# Handle embeddings/lm_head (BF16) and quantized weights (FP8)
if "embed_tokens" in name or "lm_head" in name:
old_fp8_data.copy_(new_param.data)
new_param.data = old_fp8_data
elif (
new_param.dtype == torch.float8_e4m3fn
and old_fp8_data.dtype == torch.float8_e4m3fn
):
# FP8: Use strided view for transposed storage
strided_data = torch.as_strided(
new_param.data, old_fp8_data.shape, old_fp8_data.stride()
)
old_fp8_data.copy_(strided_data)
new_param.data = old_fp8_data
QuantizedRLModelLoader._apply_scale_update(
all_params,
name,
quantized_scales.get(name),
)
elif new_param.dtype == old_fp8_data.dtype:
# Same dtype (LayerNorm, etc.): Direct copy
old_fp8_data.copy_(new_param.data)
new_param.data = old_fp8_data
else:
raise RuntimeError(
f"Unexpected dtype mismatch for {name}: "
f"new={new_param.dtype}, old={old_fp8_data.dtype}"
)
# Cleanup
del current_param_data
if is_last_update:
gc.collect()
current_platform.empty_cache()
logger.info("[QuantizedRL] Reload complete")
return updated_param_names, is_last_update
@staticmethod
def _get_updated_params(weights_list, model):
"""Identify which parameters need updating from incoming weights."""
stacked_params_mapping = [
("qkv_proj", "q_proj", "q"),
("qkv_proj", "k_proj", "k"),
("qkv_proj", "v_proj", "v"),
("gate_up_proj", "gate_proj", 0),
("gate_up_proj", "up_proj", 1),
]
params_dict = dict(model.named_parameters())
updated_params = set()
"""Resolve checkpoint names to parameters owned by this model shard."""
params = dict(model.named_parameters())
updated = set()
is_last_update = False
for name, _ in weights_list:
if name == "lm_head.weight":
is_last_update = True
if any(
skip in name for skip in QuantizedRLModelLoader.SKIP_QUANTIZATION_PARAMS
):
continue
from sglang.srt.layers.utils import get_layer_id
# Skip params outside layer range (for pipeline parallelism)
layer_id = get_layer_id(name)
if (
layer_id is not None
and hasattr(model, "start_layer")
and (layer_id < model.start_layer or layer_id >= model.end_layer)
):
continue
# Skip tied embeddings and vision tower params
if (
hasattr(model, "config")
and model.config.tie_word_embeddings
and "lm_head.weight" in name
):
continue
if name.startswith("model.vision_tower") and name not in params_dict:
continue
# Map stacked param shards (q/k/v_proj → qkv_proj)
mapped = False
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name in name:
name = name.replace(weight_name, param_name)
if name.endswith(".bias") and name not in params_dict:
continue
updated_params.add(name)
mapped = True
break
if not mapped:
if name.endswith(".bias") and name not in params_dict:
continue
if name in params_dict:
updated_params.add(name)
return list(updated_params), is_last_update
name, _, _ = QuantizedRLModelLoader._resolve_stacked_info(name)
if name in params:
updated.add(name)
return sorted(updated), is_last_update
class DummyModelLoader(BaseModelLoader):
@@ -1474,6 +1474,18 @@ def gguf_quant_weights_iterator(
yield name, param
def supports_quantized_rl_reload(func: Callable) -> Callable:
"""Allow FlashRL to defer FP8 parameter writes until this method returns.
Checkpoint names must use the parameter names or QKV/gate-up aliases supported
by QuantizedRLModelLoader. FP8 writes must go through param.weight_loader;
the method must not read their values afterward, and source tensors must
remain unchanged until return. Non-FP8 parameters are loaded immediately.
"""
func._supports_quantized_rl_reload = func
return func
def default_weight_loader(param: torch.Tensor, loaded_weight: torch.Tensor) -> None:
"""Default weight loader."""
if param.numel() == 1 and loaded_weight.numel() == 1:
+2
View File
@@ -55,6 +55,7 @@ from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTe
from sglang.srt.model_loader.weight_utils import (
default_weight_loader,
kv_cache_scales_loader,
supports_quantized_rl_reload,
)
from sglang.srt.platforms import current_platform
from sglang.srt.runtime_context import get_exec, get_parallel
@@ -616,6 +617,7 @@ class Qwen2ForCausalLM(nn.Module):
def end_layer(self):
return self.model.end_layer
@supports_quantized_rl_reload
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
from sglang.srt.environ import envs
+2
View File
@@ -32,6 +32,7 @@ from sglang.srt.model_executor.forward_context import get_token_to_kv_pool
from sglang.srt.model_loader.weight_utils import (
default_weight_loader,
maybe_remap_kv_scale_name,
supports_quantized_rl_reload,
)
from sglang.srt.models.qwen2 import Qwen2MLP as Qwen3MLP
from sglang.srt.models.qwen2 import Qwen2Model
@@ -620,6 +621,7 @@ class Qwen3ForCausalLM(nn.Module):
def end_layer(self):
return self.model.end_layer
@supports_quantized_rl_reload
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
@@ -41,6 +41,20 @@ class TestDefaultModelLoader(CustomTestCase):
self.assertEqual(events, ["load", "postprocess"])
def test_reload_requires_explicit_initial_load_marker(self):
self.assertFalse(loader_mod.QuantizedRLModelLoader.is_reload_scenario(Mock()))
model = torch.nn.Module()
model.original_weights_rebuild_keys = set()
model.recorded_loader = {}
for marker in (None, False, 1, Mock()):
with self.subTest(marker=marker):
model.flash_rl_initial_load_complete = marker
self.assertFalse(
loader_mod.QuantizedRLModelLoader.is_reload_scenario(model)
)
model.flash_rl_initial_load_complete = True
self.assertTrue(loader_mod.QuantizedRLModelLoader.is_reload_scenario(model))
def test_boot_paths_preserve_load_order_and_custom_override(self):
class CustomModelLoader(DefaultModelLoader):
def load_weights_and_postprocess(self, model, weights, target_device):
@@ -0,0 +1,518 @@
"""Quantized reload scales follow their destination projection's layout."""
import math
import socket
import unittest
from contextlib import nullcontext
from unittest.mock import patch
import torch
from transformers import Qwen2Config, Qwen3Config
from sglang.srt.configs.load_config import LoadConfig, LoadFormat
from sglang.srt.distributed.parallel_state import (
destroy_distributed_environment,
destroy_model_parallel,
init_distributed_environment,
initialize_model_parallel,
)
from sglang.srt.layers.dp_attention import initialize_dp_attention
from sglang.srt.layers.quantization.fp8 import Fp8Config
from sglang.srt.model_loader.loader import DefaultModelLoader, QuantizedRLModelLoader
from sglang.srt.models.qwen2 import Qwen2ForCausalLM
from sglang.srt.models.qwen3 import Qwen3ForCausalLM
from sglang.srt.runtime_context import SpawnRanks, get_parallel, publish, reset_context
from sglang.srt.server_args import ServerArgs
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.test_utils import CustomTestCase
register_cuda_ci(est_time=20, stage="base-b", runner_config="1-gpu-small")
def values(shape, version=0):
base = (
torch.arange(math.prod(shape), device="cuda").reshape(shape) % 31 - 15
).float() / 64
if len(shape) == 2:
base *= ((torch.arange(shape[0], device="cuda") + version) % 17 + 1)[:, None]
base *= (torch.arange(shape[1], device="cuda") // max(shape[1] // 4, 1) + 1)[
None, :
]
return (base + version / 128).to(torch.bfloat16)
def checkpoint(version=0, kind="qwen2", kv_heads=8, num_layers=1):
weights = []
for name, shape in {
"model.embed_tokens.weight": (512, 512),
"lm_head.weight": (512, 512),
"model.norm.weight": (512,),
"model.layers.0.input_layernorm.weight": (512,),
"model.layers.0.post_attention_layernorm.weight": (512,),
**{
f"model.layers.0.self_attn.{p}.weight": (
kv_heads * 64 if p in ("k_proj", "v_proj") else 512,
512,
)
for p in ("q_proj", "k_proj", "v_proj", "o_proj")
},
**{
f"model.layers.0.self_attn.{p}.bias": (
kv_heads * 64 if p in ("k_proj", "v_proj") else 512,
)
for p in ("q_proj", "k_proj", "v_proj")
if kind == "qwen2"
},
**{
f"model.layers.0.self_attn.{p}.weight": (64,)
for p in ("q_norm", "k_norm")
if kind == "qwen3"
},
**{
f"model.layers.0.mlp.{p}.weight": (1024, 512)
for p in ("gate_proj", "up_proj")
},
"model.layers.0.mlp.down_proj.weight": (512, 1024),
}.items():
weights.append((name, values(shape, version)))
first_layer = [(n, t) for n, t in weights if n.startswith("model.layers.0.")]
for layer in range(1, num_layers):
weights.extend(
(n.replace("model.layers.0.", f"model.layers.{layer}."), t)
for n, t in first_layer
)
return weights
def build_model(
kind="qwen2", *, kv_heads=8, version=0, weights=None, per_tensor=False, num_layers=1
):
config_class, model_class = {
"qwen2": (Qwen2Config, Qwen2ForCausalLM),
"qwen3": (Qwen3Config, Qwen3ForCausalLM),
}[kind]
config = config_class(
hidden_size=512,
intermediate_size=1024,
num_hidden_layers=num_layers,
num_attention_heads=8,
num_key_value_heads=kv_heads,
head_dim=64,
vocab_size=512,
max_position_embeddings=32,
tie_word_embeddings=False,
pad_token_id=0,
bos_token_id=1,
eos_token_id=2,
)
with torch.device("cuda"):
model = model_class(config, Fp8Config(is_checkpoint_fp8_serialized=False))
if per_tensor:
for module in model.modules():
method = getattr(module, "quant_method", None)
if hasattr(method, "cutlass_fp8_supported"):
method.cutlass_fp8_supported = False
method.use_marlin = False
loader = QuantizedRLModelLoader(LoadConfig(load_format=LoadFormat.FLASH_RL))
loader.load_weights_and_postprocess(
model,
checkpoint(version, kind, kv_heads, num_layers) if weights is None else weights,
torch.device("cuda"),
)
return model
def loading_scope(changed):
if not changed:
return nullcontext()
return get_parallel().override(
tp_size=4,
tp_rank=3,
tp_group=None,
attn_tp_size=4,
attn_tp_rank=3,
attn_tp_group=None,
attn_dp_size=1,
attn_dp_rank=0,
attn_cp_size=1,
attn_cp_rank=0,
moe_tp_size=4,
moe_tp_rank=3,
moe_ep_size=1,
moe_ep_rank=0,
moe_ep_group=None,
moe_dp_size=1,
moe_dp_rank=0,
)
def assert_same_parameters(actual, expected):
expected_params = dict(expected.named_parameters())
for name, param in actual.named_parameters():
reference = expected_params[name]
assert param.shape == reference.shape, name
assert param.stride() == reference.stride(), name
assert param.dtype == reference.dtype, name
torch.testing.assert_close(
param.contiguous().reshape(-1).view(torch.uint8),
reference.contiguous().reshape(-1).view(torch.uint8),
rtol=0,
atol=0,
msg=name,
)
def check_reload(model, version=13, changed=False, per_tensor=False):
kind, kv_heads = model.config.model_type, model.config.num_key_value_heads
source = checkpoint(version, kind, kv_heads)
pointers = {n: p.data_ptr() for n, p in model.named_parameters()}
DefaultModelLoader.restore_weights_before_loading(model, torch.device("cuda"))
with loading_scope(changed):
model.load_weights(source)
DefaultModelLoader.postprocess_weights(model, torch.device("cuda"))
for name, param in model.named_parameters():
assert param.data_ptr() == pointers[name], name
reference = build_model(
kind, kv_heads=kv_heads, version=version, per_tensor=per_tensor
)
assert_same_parameters(model, reference)
return model
@unittest.skipUnless(torch.cuda.is_available(), "needs a GPU")
class TestQuantizedReloadScaleLayout(CustomTestCase):
def setUp(self):
if torch.distributed.is_initialized():
self.skipTest("requires an isolated distributed test process")
reset_context()
self.addCleanup(reset_context)
original = torch.get_default_dtype()
self.addCleanup(torch.set_default_dtype, original)
torch.set_default_dtype(torch.bfloat16)
server = ServerArgs(model_path="dummy", device="cuda", tp_size=1)
publish(server, role="test", ranks=SpawnRanks(world_rank=0, gpu_id=0))
torch.cuda.set_device(0)
with socket.socket() as sock:
sock.bind(("127.0.0.1", 0))
port = sock.getsockname()[1]
init_distributed_environment(
world_size=1,
rank=0,
local_rank=0,
distributed_init_method=f"tcp://127.0.0.1:{port}",
)
self.addCleanup(destroy_distributed_environment)
initialize_model_parallel()
self.addCleanup(destroy_model_parallel)
initialize_dp_attention(server)
def test_native_model_reload_after_scope_exit(self):
model = build_model()
check_reload(model, changed=True)
check_reload(model, version=29, changed=True)
def test_native_attention_group_projection_reload_after_scope_exit(self):
model = build_model("qwen3")
check_reload(model, changed=True)
check_reload(model, version=29, changed=True)
def test_unequal_qkv_and_replicated_kv_match_cold_load(self):
for kv_heads in (1, 2, 4):
with self.subTest(kv_heads=kv_heads):
check_reload(build_model("qwen3", kv_heads=kv_heads), changed=True)
def test_partial_qkv_and_mlp_updates_preserve_other_rows(self):
for kind in ("qwen2", "qwen3"):
for shard in ("q_proj", "k_proj", "v_proj", "gate_proj", "up_proj"):
with self.subTest(kind=kind, shard=shard):
model = build_model(kind, kv_heads=1)
before = checkpoint(kind=kind, kv_heads=1)
update = {
n: t for n, t in checkpoint(13, kind, 1) if f".{shard}." in n
}
pointers = {n: p.data_ptr() for n, p in model.named_parameters()}
with loading_scope(True):
model.load_weights(list(update.items()))
expected = [(n, update.get(n, t)) for n, t in before]
reference = build_model(kind, kv_heads=1, weights=expected)
assert_same_parameters(model, reference)
for name, param in model.named_parameters():
self.assertEqual(param.data_ptr(), pointers[name], name)
def test_per_tensor_quantization_matches_native_cold_load(self):
model = build_model("qwen3", kv_heads=1, per_tensor=True)
check_reload(model, version=0, changed=True, per_tensor=True)
check_reload(model, version=13, changed=True, per_tensor=True)
def test_partial_per_tensor_update_rejected_before_any_parameter_changes(self):
for shard in ("q_proj", "k_proj", "v_proj", "gate_proj", "up_proj"):
with self.subTest(shard=shard):
model = build_model("qwen3", kv_heads=1, per_tensor=True)
pointers = {n: p.data_ptr() for n, p in model.named_parameters()}
source = checkpoint(13, "qwen3", 1)
update = [
(n, t)
for n, t in source
if n == "model.norm.weight" or f".{shard}." in n
]
with self.assertRaisesRegex(
ValueError, "Partial per-tensor FP8 update"
):
model.load_weights(update)
assert_same_parameters(
model, build_model("qwen3", kv_heads=1, per_tensor=True)
)
for name, param in model.named_parameters():
self.assertEqual(param.data_ptr(), pointers[name], name)
def test_per_tensor_session_accepts_complete_destinations(self):
model = build_model("qwen3", kv_heads=1, per_tensor=True)
pointers = {n: p.data_ptr() for n, p in model.named_parameters()}
DefaultModelLoader.restore_weights_before_loading(model, torch.device("cuda"))
grouped = {}
for name, tensor in checkpoint(13, "qwen3", 1):
target, _, _ = QuantizedRLModelLoader._resolve_stacked_info(name)
grouped.setdefault(target, []).append((name, tensor))
for source in reversed(list(grouped.values())):
with loading_scope(True):
model.load_weights(source)
self.assertFalse(hasattr(model, "_quantized_rl_pending"))
DefaultModelLoader.postprocess_weights(model, torch.device("cuda"))
assert_same_parameters(
model, build_model("qwen3", kv_heads=1, version=13, per_tensor=True)
)
for name, param in model.named_parameters():
self.assertEqual(param.data_ptr(), pointers[name], name)
def test_per_tensor_session_rejects_partial_chunks_without_host_cache(self):
model = build_model("qwen3", kv_heads=1, per_tensor=True)
original = build_model("qwen3", kv_heads=1, per_tensor=True)
DefaultModelLoader.restore_weights_before_loading(model, torch.device("cuda"))
for shard in ("q_proj", "k_proj", "v_proj", "gate_proj", "up_proj"):
update = [
(n, t)
for n, t in checkpoint(13, "qwen3", 1)
if n == "model.norm.weight" or f".{shard}." in n
]
with self.assertRaisesRegex(ValueError, "Partial per-tensor FP8 update"):
model.load_weights(update)
assert_same_parameters(model, original)
self.assertFalse(hasattr(model, "_quantized_rl_pending"))
model.load_weights(checkpoint(13, "qwen3", 1))
DefaultModelLoader.postprocess_weights(model, torch.device("cuda"))
assert_same_parameters(
model, build_model("qwen3", kv_heads=1, version=13, per_tensor=True)
)
def test_interleaved_layers_cannot_accumulate_pending_bf16_matrices(self):
model = build_model("qwen3", kv_heads=1, per_tensor=True, num_layers=16)
snapshot = {
n: p.contiguous().reshape(-1).view(torch.uint8).clone()
for n, p in model.named_parameters()
}
pointers = {n: p.data_ptr() for n, p in model.named_parameters()}
DefaultModelLoader.restore_weights_before_loading(model, torch.device("cuda"))
source = [
(n, t)
for n, t in checkpoint(13, "qwen3", 1, 16)
if ".q_proj." in n or n == "model.norm.weight"
]
with self.assertRaisesRegex(ValueError, "Partial per-tensor FP8 update"):
model.load_weights(source)
self.assertFalse(hasattr(model, "_quantized_rl_pending"))
for name, param in model.named_parameters():
self.assertEqual(param.data_ptr(), pointers[name], name)
torch.testing.assert_close(
param.contiguous().reshape(-1).view(torch.uint8),
snapshot[name],
rtol=0,
atol=0,
)
DefaultModelLoader.postprocess_weights(model, torch.device("cuda"))
def test_noncanonical_reload_names_are_rejected_before_mutation(self):
model = build_model("qwen3", kv_heads=1)
sources = dict(checkpoint(13, "qwen3", 1))
snapshot = {
n: (p.data_ptr(), p.contiguous().view(torch.uint8).clone())
for n, p in model.named_parameters()
}
for canonical in (
"model.layers.0.self_attn.q_proj.weight",
"model.embed_tokens.weight",
"model.norm.weight",
):
with self.subTest(name=canonical):
update = [
("model.norm.weight", sources["model.norm.weight"]),
(canonical.removeprefix("model."), sources[canonical]),
]
with self.assertRaisesRegex(ValueError, "canonical checkpoint names"):
model.load_weights(update)
for name, param in model.named_parameters():
pointer, data = snapshot[name]
self.assertEqual(param.data_ptr(), pointer)
torch.testing.assert_close(
param.contiguous().view(torch.uint8), data, rtol=0, atol=0
)
def test_qwen2_walker_reload_matches_cold_load(self):
from sglang.srt.environ import envs
with patch.object(
envs.SGLANG_ENABLE_WEIGHT_LOADER_V2, "get", return_value=True
):
for per_tensor in (False, True):
with self.subTest(per_tensor=per_tensor):
check_reload(
build_model("qwen2", kv_heads=1, per_tensor=per_tensor),
changed=True,
per_tensor=per_tensor,
)
def test_qwen2_walker_tied_head_post_copy_occurs_once(self):
from sglang.srt.environ import envs
model = build_model("qwen2", kv_heads=1)
model.config.tie_word_embeddings = True
head = model.lm_head.weight
original = head.weight_loader
copies = []
def load_head(*args, **kwargs):
copies.append(True)
return original(*args, **kwargs)
head.weight_loader = load_head
source = checkpoint(13, "qwen2", 1)
embedding = dict(source)["model.embed_tokens.weight"]
with patch.object(
envs.SGLANG_ENABLE_WEIGHT_LOADER_V2, "get", return_value=True
):
model.load_weights(source)
self.assertEqual(len(copies), 1)
torch.testing.assert_close(head, embedding, rtol=0, atol=0)
expected = [(n, embedding if n == "lm_head.weight" else t) for n, t in source]
assert_same_parameters(
model, build_model("qwen2", kv_heads=1, weights=expected)
)
def test_unsupported_native_loaders_fail_before_initialization(self):
from sglang.srt.models.mimo_v2 import MiMoV2ForCausalLM
from sglang.srt.models.whisper import WhisperForConditionalGeneration
for native in (
WhisperForConditionalGeneration.load_weights,
MiMoV2ForCausalLM.load_weights,
):
with self.subTest(native=native.__qualname__):
model = torch.nn.Linear(4, 4, device="cuda")
before = model.weight.clone()
model.load_weights = native.__get__(model)
original = model.load_weights
loader = QuantizedRLModelLoader(
LoadConfig(load_format=LoadFormat.FLASH_RL)
)
with self.assertRaisesRegex(ValueError, "deferred FP8-write contract"):
loader.load_weights_and_postprocess(model, [], torch.device("cuda"))
self.assertIs(model.load_weights, original)
self.assertFalse(hasattr(model, "original_weights_rebuild_keys"))
torch.testing.assert_close(model.weight, before, rtol=0, atol=0)
def test_failed_native_load_restores_fp8_storage(self):
model = build_model("qwen3", kv_heads=1)
pointers = {n: p.data_ptr() for n, p in model.named_parameters()}
def fail(weights):
Qwen3ForCausalLM.load_weights(model, weights)
self.assertEqual(
model.model.layers[0].self_attn.qkv_proj.weight.dtype,
torch.float8_e4m3fn,
)
raise RuntimeError("native loader failed")
with self.assertRaisesRegex(RuntimeError, "native loader failed"):
QuantizedRLModelLoader.rebinding_and_load_weights(
model,
fail,
[
(n, t)
for n, t in checkpoint(13, "qwen3", 1)
if any(f".{p}." in n for p in ("q_proj", "k_proj", "v_proj"))
],
)
assert_same_parameters(model, build_model("qwen3", kv_heads=1))
for name, param in model.named_parameters():
self.assertEqual(param.data_ptr(), pointers[name], name)
def test_failed_deferred_parameter_write_restores_storage_and_loader(self):
model = build_model("qwen3", kv_heads=1)
param = model.model.layers[0].self_attn.qkv_proj.weight
pointer = param.data_ptr()
snapshot = param.contiguous().view(torch.uint8).clone()
def fail(target, weight, shard):
self.assertEqual(target.dtype, torch.bfloat16)
raise RuntimeError("deferred write failed")
param.weight_loader = fail
source = [
(n, t)
for n, t in checkpoint(13, "qwen3", 1)
if any(f".{shard}_proj." in n for shard in ("q", "k", "v"))
]
with self.assertRaisesRegex(RuntimeError, "deferred write failed"):
model.load_weights(source)
self.assertEqual(param.data_ptr(), pointer)
self.assertIs(param.weight_loader, fail)
torch.testing.assert_close(
param.contiguous().view(torch.uint8), snapshot, rtol=0, atol=0
)
def test_fnuz_per_tensor_reload_matches_cold_load(self):
from sglang.srt.layers.quantization import fp8
from sglang.srt.layers.quantization.fp8_utils import input_to_float8
with patch.object(
fp8,
"input_to_float8",
lambda x: input_to_float8(x, dtype=torch.float8_e4m3fnuz),
):
model = build_model("qwen3", kv_heads=1, per_tensor=True)
check_reload(model, version=13, per_tensor=True)
def test_per_tensor_output_buffer_matches_reference_without_weight_temporary(self):
from sglang.srt.layers.quantization.fp8_utils import input_to_float8
x = values((2048, 2048), 13)
for dtype in (torch.float8_e4m3fn, torch.float8_e4m3fnuz):
with self.subTest(dtype=dtype):
expected, expected_scale = input_to_float8(x, dtype=dtype)
output = torch.empty_like(x, dtype=dtype)
input_to_float8(x, dtype=dtype, out=output)
torch.cuda.synchronize()
baseline = torch.cuda.memory_allocated()
torch.cuda.reset_peak_memory_stats()
actual, scale = input_to_float8(x, dtype=dtype, out=output)
torch.cuda.synchronize()
extra = torch.cuda.max_memory_allocated() - baseline
self.assertLess(extra, 64 * 1024)
self.assertEqual(actual.data_ptr(), output.data_ptr())
torch.testing.assert_close(
actual.view(torch.uint8), expected.view(torch.uint8), rtol=0, atol=0
)
torch.testing.assert_close(scale, expected_scale, rtol=0, atol=0)
def test_empty_session_keeps_native_layout_and_memory(self):
model = build_model("qwen3", kv_heads=1)
pointers = {n: p.data_ptr() for n, p in model.named_parameters()}
DefaultModelLoader.restore_weights_before_loading(model, torch.device("cuda"))
model.load_weights([])
DefaultModelLoader.postprocess_weights(model, torch.device("cuda"))
assert_same_parameters(model, build_model("qwen3", kv_heads=1))
for name, param in model.named_parameters():
self.assertEqual(param.data_ptr(), pointers[name], name)
if __name__ == "__main__":
unittest.main()