[Fix] inkling: Keep BF16 LM-head operands with FP32 output (#45335)
This commit is contained in:
@@ -68,6 +68,7 @@ class InklingModelConfig(PretrainedConfig):
|
||||
unpadded_vocab_size: Optional[int] = None,
|
||||
padded_vocab_size: Optional[int] = None,
|
||||
logits_mup_width_multiplier: Optional[float] = None,
|
||||
enable_lm_head_output_fp32: bool = True,
|
||||
final_logit_softcapping: Optional[float] = None,
|
||||
num_nextn_predict_layers: int = 8,
|
||||
tie_word_embeddings: bool = False,
|
||||
@@ -184,6 +185,8 @@ class InklingModelConfig(PretrainedConfig):
|
||||
self.log_scaling_alpha = log_scaling_alpha
|
||||
self.unpadded_vocab_size = self.vocab_size
|
||||
self.logits_mup_width_multiplier = logits_mup_width_multiplier
|
||||
# Preserve CUDA projection scores without promoting the inputs or weights.
|
||||
self.enable_lm_head_output_fp32 = enable_lm_head_output_fp32
|
||||
self.final_logit_softcapping = final_logit_softcapping
|
||||
self.num_nextn_predict_layers = num_nextn_predict_layers
|
||||
|
||||
|
||||
@@ -465,6 +465,9 @@ class LogitsProcessor(nn.Module):
|
||||
self.use_fp32_lm_head = get_exec().features.enable_fp32_lm_head or getattr(
|
||||
config, "enable_lm_head_fp32", False
|
||||
)
|
||||
self.use_fp32_lm_head_output = getattr(
|
||||
config, "enable_lm_head_output_fp32", False
|
||||
)
|
||||
if self.use_attn_tp_group:
|
||||
self.attn_tp_size = get_parallel().attn_tp_size
|
||||
self.do_tensor_parallel_all_gather = (
|
||||
@@ -1092,7 +1095,9 @@ class LogitsProcessor(nn.Module):
|
||||
)
|
||||
# A shared FP32 buffer must not change the lm_head's output precision.
|
||||
# Keep the same rounding as the unbuffered path unless FP32 is enabled.
|
||||
if use_mm_out_dtype and self.use_fp32_lm_head:
|
||||
if use_mm_out_dtype and (
|
||||
self.use_fp32_lm_head or self.use_fp32_lm_head_output
|
||||
):
|
||||
logits = torch.mm(
|
||||
hidden_states,
|
||||
lm_head.weight.T,
|
||||
|
||||
Reference in New Issue
Block a user