[Fix] Quantize reloaded FP8 weights after native sharding (#42585)

This commit is contained in:
Cheng Wan
2026-10-07 16:20:47 -07:00
committed by GitHub
parent 59740a728f
commit b83827ea02
8 changed files with 783 additions and 341 deletions
@@ -214,6 +214,9 @@ def _per_token_group_quant_8bit_raw(
column_major_scales: bool = False,
scale_tma_aligned: bool = False,
scale_ue8m0: bool = False,
*,
output_q: Optional[torch.Tensor] = None,
output_s: Optional[torch.Tensor] = None,
) -> Tuple[torch.Tensor, torch.Tensor]:
"""Function to perform per-token-group quantization on an input tensor `x`.
@@ -251,14 +254,22 @@ def _per_token_group_quant_8bit_raw(
bit8_max = info.max
bit8_min = info.min
x_q = torch.empty_like(x, device=x.device, dtype=dtype)
x_s = create_per_token_group_quant_fp8_output_scale(
x_shape=x.shape,
device=x.device,
group_size=group_size,
column_major_scales=column_major_scales,
scale_tma_aligned=scale_tma_aligned,
scale_ue8m0=False,
x_q = (
output_q
if output_q is not None
else torch.empty_like(x, device=x.device, dtype=dtype)
)
x_s = (
output_s
if output_s is not None
else create_per_token_group_quant_fp8_output_scale(
x_shape=x.shape,
device=x.device,
group_size=group_size,
column_major_scales=column_major_scales,
scale_tma_aligned=scale_tma_aligned,
scale_ue8m0=False,
)
)
M = x.numel() // group_size
@@ -573,6 +584,9 @@ def sglang_per_token_group_quant_fp8(
scale_ue8m0: bool = False,
fuse_silu_and_mul: bool = False,
masked_m: Optional[torch.Tensor] = None,
*,
output_q: Optional[torch.Tensor] = None,
output_s: Optional[torch.Tensor] = None,
):
assert x.shape[-1] % group_size == 0, (
"the last dimension of `x` cannot be divisible by `group_size`"
@@ -588,18 +602,34 @@ def sglang_per_token_group_quant_fp8(
# Whole-row group quant is per-token quant; route to the dedicated
# kernel (same [T, 1] scale shape) instead of a group kernel that
# would need arbitrary group sizes.
return sglang_per_token_quant_fp8(x)
if output_q is not None and output_q.dtype != fp8_dtype:
return _per_token_group_quant_8bit_raw(
x,
group_size,
dtype=output_q.dtype,
output_q=output_q,
output_s=output_s,
)
return sglang_per_token_quant_fp8(x, output_q=output_q, output_s=output_s)
out_shape = (*x.shape[:-1], x.shape[-1] // (2 if fuse_silu_and_mul else 1))
x_q = torch.empty(out_shape, device=x.device, dtype=fp8_dtype)
x_s = create_per_token_group_quant_fp8_output_scale(
x_shape=out_shape,
device=x.device,
group_size=group_size,
column_major_scales=column_major_scales,
scale_tma_aligned=scale_tma_aligned,
scale_ue8m0=scale_ue8m0,
x_q = (
output_q
if output_q is not None
else torch.empty(out_shape, device=x.device, dtype=fp8_dtype)
)
x_s = (
output_s
if output_s is not None
else create_per_token_group_quant_fp8_output_scale(
x_shape=out_shape,
device=x.device,
group_size=group_size,
column_major_scales=column_major_scales,
scale_tma_aligned=scale_tma_aligned,
scale_ue8m0=scale_ue8m0,
)
)
if x.shape[0] > 0:
@@ -764,15 +794,26 @@ def sglang_per_token_group_quant_8bit(
def sglang_per_token_quant_fp8(
x: torch.Tensor,
dtype: torch.dtype = fp8_dtype,
*,
output_q: Optional[torch.Tensor] = None,
output_s: Optional[torch.Tensor] = None,
):
assert x.is_contiguous(), "`x` is not contiguous"
x_q = torch.empty_like(x, device=x.device, dtype=dtype)
x_s = torch.empty(
x.shape[0],
1,
device=x.device,
dtype=torch.float32,
x_q = (
output_q
if output_q is not None
else torch.empty_like(x, device=x.device, dtype=dtype)
)
x_s = (
output_s
if output_s is not None
else torch.empty(
x.shape[0],
1,
device=x.device,
dtype=torch.float32,
)
)
sgl_per_token_quant_fp8(x, x_q, x_s)
@@ -1663,7 +1663,10 @@ def dequant_mxfp4(
def input_to_float8(
x: torch.Tensor, dtype: torch.dtype = fp8_dtype
x: torch.Tensor,
dtype: torch.dtype = fp8_dtype,
*,
out: Optional[torch.Tensor] = None,
) -> Tuple[torch.Tensor, torch.Tensor]:
"""This function quantizes input values to float8 values with tensor-wise quantization."""
min_val, max_val = x.aminmax()
@@ -1677,6 +1680,12 @@ def input_to_float8(
fp_max = finfo.max
scale = fp_max / amax
if out is not None:
assert out.shape == x.shape and out.dtype == dtype
assert x.dtype in (torch.float16, torch.bfloat16, torch.float32)
# FP32 broadcast avoids a full input copy; absmax bounds the FP8 range.
torch.mul(x, scale.reshape((1,) * x.ndim), out=out)
return out, scale.float().reciprocal()
x_scl_sat = (x.float() * scale).clamp(min=-fp_max, max=fp_max)
return x_scl_sat.to(dtype).contiguous(), scale.float().reciprocal()
+161 -317
View File
@@ -33,7 +33,6 @@ from typing import (
List,
Optional,
Tuple,
Union,
cast,
)
@@ -1074,6 +1073,8 @@ class DefaultModelLoader(BaseModelLoader):
@staticmethod
def postprocess_weights(model, target_device):
if QuantizedRLModelLoader.is_reload_scenario(model):
return
for module, quant_method in _modules_with_quant_method(model):
# When quant methods need to process weights after loading
# (for repacking, quantizing, etc), they expect parameters
@@ -1086,6 +1087,8 @@ class DefaultModelLoader(BaseModelLoader):
@staticmethod
def restore_weights_before_loading(model, target_device):
"""Undo in-place quant packing so fresh weights can be loaded."""
if QuantizedRLModelLoader.is_reload_scenario(model):
return
for module, quant_method in _modules_with_quant_method(model):
# AMX packing and the MXFP4 backend wrappers are duck-typed and cannot restore
if isinstance(quant_method, QuantizeMethodBase):
@@ -1156,14 +1159,12 @@ class LayeredModelLoader(DefaultModelLoader):
class QuantizedRLModelLoader(DefaultModelLoader):
"""
Model loader for RL training with FP8 quantization (profile-free, native SGLang).
"""FP8 RL loader for audited Qwen2 (legacy/v2) and Qwen3 native loaders.
Workflow:
1. Initial load: Load base model → Record state → Apply FP8 quantization
2. Training Actor in full precision
3. Reload: Trainer sends full precision weights → Quantize to FP8 → Copy to original memory
4. Use torch.as_strided to preserve memory locations across reloads
Initial loads record native parameter loaders before quantization. Reloads
invoke the native loader once, deferring FP8 writes; each local destination
is staged and quantized into its existing FP8 weight and scale buffers.
Unsupported native loaders are rejected before initialization loads weights.
Usage:
--model-path Qwen/Qwen2.5-7B --quantization fp8 --load-format flash_rl
@@ -1181,23 +1182,6 @@ class QuantizedRLModelLoader(DefaultModelLoader):
"_assert_and_load",
]
# Parameters to skip during FP8 quantization (matches FlashRL's exclude_list)
SKIP_QUANTIZATION_PARAMS = [
"weight_scale",
"input_scale",
"output_scale",
".bias",
"lm_head.weight",
"model.norm.weight",
"embed_tokens", # BF16 params
"rotary_emb.inv_freq",
"rotary_emb.cos_cached",
"rotary_emb.sin_cached",
"projector",
"input_layernorm.weight",
"post_attention_layernorm.weight", # LayerNorms
]
# Stacked parameters (Qwen2): shards loaded separately, then combined
STACKED_PARAMS_MAPPING = [
("qkv_proj", ["q_proj", "k_proj", "v_proj"]),
@@ -1215,14 +1199,18 @@ class QuantizedRLModelLoader(DefaultModelLoader):
self._initial_load_complete = False
def _prepare_weights(
self, model_name_or_path: str, revision: Optional[str], fall_back_to_pt: bool
self,
model_name_or_path: str,
revision: Optional[str],
fall_back_to_pt: bool,
allow_patterns_overrides: Optional[list[str]] = None,
):
"""Standard weight preparation using base model path."""
logger.info(f"[QuantizedRL] Loading from base model: {model_name_or_path}")
temp_config = LoadConfig(load_format=LoadFormat.AUTO)
temp_loader = DefaultModelLoader(temp_config)
return temp_loader._prepare_weights(
model_name_or_path, revision, fall_back_to_pt
model_name_or_path, revision, fall_back_to_pt, allow_patterns_overrides
)
@staticmethod
@@ -1242,6 +1230,14 @@ class QuantizedRLModelLoader(DefaultModelLoader):
logger.info("[QuantizedRL] Initial load with FP8 quantization")
original_load_weights = model.load_weights
native = getattr(original_load_weights, "__func__", original_load_weights)
if getattr(native, "_supports_quantized_rl_reload", None) is not native:
raise ValueError(
f"FlashRL does not support {type(model).__name__}.load_weights: "
"the native loader has not declared the deferred FP8-write contract. "
"Supported native loaders are Qwen2ForCausalLM (legacy/v2) and "
"Qwen3ForCausalLM; use a regular load format for other models."
)
def load_weights_proxy(weights):
if QuantizedRLModelLoader.is_reload_scenario(model):
@@ -1301,331 +1297,179 @@ class QuantizedRLModelLoader(DefaultModelLoader):
return (
hasattr(model, "original_weights_rebuild_keys")
and hasattr(model, "recorded_loader")
and getattr(model, "flash_rl_initial_load_complete", False)
and getattr(model, "flash_rl_initial_load_complete", False) is True
)
@staticmethod
def _is_stacked_param(name):
"""Check if parameter is stacked (qkv_proj, gate_up_proj)."""
for stacked_name, _ in QuantizedRLModelLoader.STACKED_PARAMS_MAPPING:
if stacked_name in name:
return True
return False
@staticmethod
def _resolve_stacked_info(name: str) -> Tuple[str, Optional[str], Optional[Any]]:
for target, shard_names in QuantizedRLModelLoader.STACKED_PARAMS_MAPPING:
for idx, shard in enumerate(shard_names):
if shard in name:
components = name.split(".")
if shard in components:
components[components.index(shard)] = target
shard_id = (
QuantizedRLModelLoader._QKV_SHARD_ALIASES.get(shard, shard)
QuantizedRLModelLoader._QKV_SHARD_ALIASES[shard]
if target == "qkv_proj"
else idx
)
return name.replace(shard, target), target, shard_id
return ".".join(components), target, shard_id
return name, None, None
@staticmethod
def _store_quantized_scale(
scale_store: Dict[str, Union[torch.Tensor, Dict[Any, torch.Tensor]]],
name: str,
scale: torch.Tensor,
) -> None:
param_name, stacked_key, shard_id = (
QuantizedRLModelLoader._resolve_stacked_info(name)
)
if stacked_key is None:
scale_store[param_name] = scale
else:
shard_dict = scale_store.setdefault(param_name, {})
assert isinstance(shard_dict, dict)
shard_dict[shard_id] = scale
@staticmethod
def _apply_scale_update(
all_params: Dict[str, torch.nn.Parameter],
param_name: str,
scale_info: Union[torch.Tensor, Dict[Any, torch.Tensor], None],
) -> None:
if scale_info is None:
return
# Get tp rank and size
tp_rank = get_parallel().tp_rank
tp_size = get_parallel().tp_size
def _get_tp_sharded_scale(full_scale_tensor):
"""Get tp sharded scale from full scale tensor"""
if tp_size == 1:
return full_scale_tensor
full_dim = full_scale_tensor.shape[0]
shard_dim = full_dim // tp_size
start_idx = tp_rank * shard_dim
end_idx = start_idx + shard_dim
return full_scale_tensor[start_idx:end_idx]
if param_name.endswith(".weight"):
scale_param_name = f"{param_name[:-7]}.weight_scale"
else:
scale_param_name = f"{param_name}.weight_scale"
scale_param = all_params.get(scale_param_name)
if scale_param is None:
logger.warning(
"[QuantizedRL] Scale parameter not found: %s", scale_param_name
)
return
if isinstance(scale_info, torch.Tensor):
new_scale = scale_info.t().contiguous()
if scale_param.data.shape == new_scale.shape:
scale_param.data.copy_(new_scale)
else:
logger.warning(
"[QuantizedRL] Scale shape mismatch for %s: expected %s, got %s",
scale_param_name,
scale_param.data.shape,
new_scale.shape,
)
else:
stacked_key = next(
(
target
for target, _ in QuantizedRLModelLoader.STACKED_PARAMS_MAPPING
if target in param_name
),
None,
)
shard_names = next(
(
names
for target, names in QuantizedRLModelLoader.STACKED_PARAMS_MAPPING
if target == stacked_key
),
[],
)
rows_per_shard = scale_param.data.shape[-1] // max(len(shard_names), 1)
if rows_per_shard * len(shard_names) != scale_param.data.shape[-1]:
logger.warning(
f"Scale param shape {scale_param.data.shape[-1]} not divisible by {len(shard_names)}"
)
offset = 0
for idx, shard in enumerate(shard_names):
shard_id = (
QuantizedRLModelLoader._QKV_SHARD_ALIASES.get(shard, shard)
if stacked_key == "qkv_proj"
else idx
)
shard_scale = scale_info.get(shard_id)
shard_scale = _get_tp_sharded_scale(shard_scale)
if shard_scale is None:
offset += rows_per_shard
continue
shard_rows = shard_scale.shape[0]
start = offset
end = start + shard_rows
scale_param.data[..., start:end] = shard_scale.t().contiguous()
offset = end
@staticmethod
def rebinding_and_load_weights(model, first_time_load_weights, weights):
"""
Reload: VERL sends BF16 → Quantize to FP8 → Copy to original memory.
Flow: Reset params → Restore attributes → Quantize in iterator → Load → Copy back
"""
logger.info("[QuantizedRL] Reload: Updating weights with FP8 quantization")
"""Shard new BF16 weights natively, then quantize into their original buffers."""
from sglang.kernels.ops.quantization.fp8_kernel import (
per_token_group_quant_fp8,
)
from sglang.srt.layers.quantization.fp8_utils import input_to_float8
weights_list = list(weights)
for name, _ in weights_list:
if name.startswith(("layers.", "embed_tokens.", "norm.")):
raise ValueError(
f"FlashRL reload requires canonical checkpoint names: {name!r} "
"must include the 'model.' prefix."
)
updated_param_names, is_last_update = (
QuantizedRLModelLoader._get_updated_params(weights_list, model)
)
# Save current FP8 parameter data pointers
existing_params = dict(model.named_parameters())
current_param_data = {}
for name in updated_param_names:
if name in existing_params:
current_param_data[name] = existing_params[name].data
# Reset to pre-quantization shape using torch.as_strided
# Keeps same storage, just changes view - critical for memory preservation
for name, rebuild_info in model.original_weights_rebuild_keys.items():
if name in updated_param_names and name in existing_params:
existing_params[name].data = torch.as_strided(
# Note: avoid clone here
existing_params[name].data.clone(),
rebuild_info["shape"],
rebuild_info["stride"],
)
# Restore weight loader attributes (only if missing)
for k, loader_dict in model.recorded_loader.items():
for param_name, loader in loader_dict.items():
if param_name in updated_param_names and param_name in existing_params:
param = existing_params[param_name]
if not hasattr(param, k):
if callable(loader):
if hasattr(loader, "__self__"):
setattr(param, k, loader)
else:
setattr(
param,
k,
QuantizedRLModelLoader._bind_method_to_cls(
loader, param
),
)
else:
setattr(param, k, loader)
del existing_params
# Quantize BF16 weights to FP8 in iterator (before weight_loader)
# Store scales for later update
quantized_scales: Dict[str, Union[torch.Tensor, Dict[Any, torch.Tensor]]] = {}
def quantize_weights_iterator(weights_iter):
"""Quantize individual shards before weight_loader stacks them."""
from sglang.kernels.ops.quantization.fp8_kernel import (
per_token_group_quant_fp8,
params = dict(model.named_parameters())
grouped = collections.defaultdict(list)
complete_params = set()
for source_name, weight in weights_list:
name, _, shard_id = QuantizedRLModelLoader._resolve_stacked_info(
source_name
)
grouped[name].append((source_name, weight))
if shard_id is None:
complete_params.add(name)
for name, weight in weights_iter:
if any(
skip in name
for skip in QuantizedRLModelLoader.SKIP_QUANTIZATION_PARAMS
):
logger.info(f"[QuantizedRL] Skip: {name} ({weight.dtype})")
yield (name, weight)
elif weight.dtype in [torch.bfloat16, torch.float32, torch.float16]:
qweight, scale = per_token_group_quant_fp8(weight, weight.shape[-1])
logger.info(f"[QuantizedRL] Quantize: {name} {weight.dtype}→FP8")
QuantizedRLModelLoader._store_quantized_scale(
quantized_scales, name, scale
row_masks = {}
# Check coverage before modifying any model parameter. Per-tensor
# quantization needs the original BF16 values for every destination row.
for name, sources in grouped.items():
if name not in params or params[name].dtype not in (
torch.float8_e4m3fn,
torch.float8_e4m3fnuz,
):
continue
rebuild = model.original_weights_rebuild_keys[name]
layer = model.get_submodule(name.rpartition(".")[0])
per_channel = layer.weight_scale.shape == (1, rebuild["shape"][0])
if name in complete_params:
row_masks[name] = (None, per_channel)
continue
marker = nn.Parameter(
torch.zeros((rebuild["shape"][0], 1), device="cpu"), requires_grad=False
)
marker.output_dim = 0
for source_name, weight in sources:
_, _, shard_id = QuantizedRLModelLoader._resolve_stacked_info(
source_name
)
args = () if shard_id is None else (shard_id,)
layer.weight_loader(
marker, torch.ones((weight.shape[0], 1), device="cpu"), *args
)
rows = marker.data[:, 0].bool()
if not per_channel and not rows.all():
raise ValueError(
f"Partial per-tensor FP8 update for {name}: supply all destination "
"rows in the same update call, including inside a weight-update session."
)
row_masks[name] = (rows, per_channel)
# Defer only FP8 writes so one native call preserves cross-key lookups
# and the order of non-FP8 loads, including tied embeddings.
writes = collections.defaultdict(list)
native_loaders = {}
for name in row_masks:
param = params[name]
for key, loaders in model.recorded_loader.items():
if name in loaders and not hasattr(param, key):
loader = loaders[name]
setattr(
param,
key,
QuantizedRLModelLoader._bind_method_to_cls(loader, param)
if callable(loader)
else loader,
)
yield (name, qweight)
native_loaders[name] = param.weight_loader
def record_write(*args, _name=name, **kwargs):
writes[_name].append((args, kwargs))
param.weight_loader = record_write
try:
first_time_load_weights(weights_list)
finally:
for name, loader in native_loaders.items():
params[name].weight_loader = loader
for name, calls in writes.items():
param = params[name]
data = param.data
rebuild = model.original_weights_rebuild_keys[name]
layer = model.get_submodule(name.rpartition(".")[0])
rows, per_channel = row_masks[name]
complete = rows is None or bool(rows.all())
staging = torch.empty_strided(
rebuild["shape"],
rebuild["stride"],
dtype=rebuild["dtype"],
device=data.device,
)
param.data = staging
try:
for args, kwargs in calls:
native_loaders[name](*args, **kwargs)
local_weight = torch.as_strided(
data, rebuild["shape"], rebuild["stride"]
)
if per_channel:
# Untouched rows never pass through dequantization.
if complete:
ranges = [(0, staging.shape[0])]
else:
padded = torch.cat((rows.new_zeros(1), rows, rows.new_zeros(1)))
edges = (padded[1:] != padded[:-1]).nonzero().flatten().tolist()
ranges = zip(edges[::2], edges[1::2])
for begin, end in ranges:
per_token_group_quant_fp8(
staging[begin:end],
staging.shape[-1],
output_q=local_weight[begin:end],
output_s=layer.weight_scale.data[:, begin:end].t(),
)
else:
logger.info(f"[QuantizedRL] Keep: {name} ({weight.dtype})")
yield (name, weight)
_, scale = input_to_float8(
staging, dtype=data.dtype, out=local_weight
)
layer.weight_scale.data.copy_(scale)
finally:
param.data = data
# No BF16 destination survives to the next destination or call.
del staging
# Load quantized weights (weight_loader stacks FP8 shards)
first_time_load_weights(quantize_weights_iterator(iter(weights_list)))
# Copy back to original FP8 memory locations and update scales
all_params = dict(model.named_parameters())
for name in updated_param_names:
if name not in all_params or name not in current_param_data:
continue
if any(
skip in name for skip in QuantizedRLModelLoader.SKIP_QUANTIZATION_PARAMS
):
continue
new_param = all_params[name]
old_fp8_data = current_param_data[name]
# Handle embeddings/lm_head (BF16) and quantized weights (FP8)
if "embed_tokens" in name or "lm_head" in name:
old_fp8_data.copy_(new_param.data)
new_param.data = old_fp8_data
elif (
new_param.dtype == torch.float8_e4m3fn
and old_fp8_data.dtype == torch.float8_e4m3fn
):
# FP8: Use strided view for transposed storage
strided_data = torch.as_strided(
new_param.data, old_fp8_data.shape, old_fp8_data.stride()
)
old_fp8_data.copy_(strided_data)
new_param.data = old_fp8_data
QuantizedRLModelLoader._apply_scale_update(
all_params,
name,
quantized_scales.get(name),
)
elif new_param.dtype == old_fp8_data.dtype:
# Same dtype (LayerNorm, etc.): Direct copy
old_fp8_data.copy_(new_param.data)
new_param.data = old_fp8_data
else:
raise RuntimeError(
f"Unexpected dtype mismatch for {name}: "
f"new={new_param.dtype}, old={old_fp8_data.dtype}"
)
# Cleanup
del current_param_data
if is_last_update:
gc.collect()
current_platform.empty_cache()
logger.info("[QuantizedRL] Reload complete")
return updated_param_names, is_last_update
@staticmethod
def _get_updated_params(weights_list, model):
"""Identify which parameters need updating from incoming weights."""
stacked_params_mapping = [
("qkv_proj", "q_proj", "q"),
("qkv_proj", "k_proj", "k"),
("qkv_proj", "v_proj", "v"),
("gate_up_proj", "gate_proj", 0),
("gate_up_proj", "up_proj", 1),
]
params_dict = dict(model.named_parameters())
updated_params = set()
"""Resolve checkpoint names to parameters owned by this model shard."""
params = dict(model.named_parameters())
updated = set()
is_last_update = False
for name, _ in weights_list:
if name == "lm_head.weight":
is_last_update = True
if any(
skip in name for skip in QuantizedRLModelLoader.SKIP_QUANTIZATION_PARAMS
):
continue
from sglang.srt.layers.utils import get_layer_id
# Skip params outside layer range (for pipeline parallelism)
layer_id = get_layer_id(name)
if (
layer_id is not None
and hasattr(model, "start_layer")
and (layer_id < model.start_layer or layer_id >= model.end_layer)
):
continue
# Skip tied embeddings and vision tower params
if (
hasattr(model, "config")
and model.config.tie_word_embeddings
and "lm_head.weight" in name
):
continue
if name.startswith("model.vision_tower") and name not in params_dict:
continue
# Map stacked param shards (q/k/v_proj → qkv_proj)
mapped = False
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name in name:
name = name.replace(weight_name, param_name)
if name.endswith(".bias") and name not in params_dict:
continue
updated_params.add(name)
mapped = True
break
if not mapped:
if name.endswith(".bias") and name not in params_dict:
continue
if name in params_dict:
updated_params.add(name)
return list(updated_params), is_last_update
name, _, _ = QuantizedRLModelLoader._resolve_stacked_info(name)
if name in params:
updated.add(name)
return sorted(updated), is_last_update
class DummyModelLoader(BaseModelLoader):
@@ -1474,6 +1474,18 @@ def gguf_quant_weights_iterator(
yield name, param
def supports_quantized_rl_reload(func: Callable) -> Callable:
"""Allow FlashRL to defer FP8 parameter writes until this method returns.
Checkpoint names must use the parameter names or QKV/gate-up aliases supported
by QuantizedRLModelLoader. FP8 writes must go through param.weight_loader;
the method must not read their values afterward, and source tensors must
remain unchanged until return. Non-FP8 parameters are loaded immediately.
"""
func._supports_quantized_rl_reload = func
return func
def default_weight_loader(param: torch.Tensor, loaded_weight: torch.Tensor) -> None:
"""Default weight loader."""
if param.numel() == 1 and loaded_weight.numel() == 1:
+2
View File
@@ -55,6 +55,7 @@ from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTe
from sglang.srt.model_loader.weight_utils import (
default_weight_loader,
kv_cache_scales_loader,
supports_quantized_rl_reload,
)
from sglang.srt.platforms import current_platform
from sglang.srt.runtime_context import get_exec, get_parallel
@@ -616,6 +617,7 @@ class Qwen2ForCausalLM(nn.Module):
def end_layer(self):
return self.model.end_layer
@supports_quantized_rl_reload
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
from sglang.srt.environ import envs
+2
View File
@@ -32,6 +32,7 @@ from sglang.srt.model_executor.forward_context import get_token_to_kv_pool
from sglang.srt.model_loader.weight_utils import (
default_weight_loader,
maybe_remap_kv_scale_name,
supports_quantized_rl_reload,
)
from sglang.srt.models.qwen2 import Qwen2MLP as Qwen3MLP
from sglang.srt.models.qwen2 import Qwen2Model
@@ -620,6 +621,7 @@ class Qwen3ForCausalLM(nn.Module):
def end_layer(self):
return self.model.end_layer
@supports_quantized_rl_reload
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
stacked_params_mapping = [
# (param_name, shard_name, shard_id)