[Fix] inkling: Keep BF16 LM-head operands with FP32 output (#45335)

This commit is contained in:
Cheng Wan
2026-10-23 15:51:44 -07:00
committed by GitHub
parent 197be2203a
commit 855c092deb
4 changed files with 128 additions and 1 deletions
+3
View File
@@ -68,6 +68,7 @@ class InklingModelConfig(PretrainedConfig):
unpadded_vocab_size: Optional[int] = None,
padded_vocab_size: Optional[int] = None,
logits_mup_width_multiplier: Optional[float] = None,
enable_lm_head_output_fp32: bool = True,
final_logit_softcapping: Optional[float] = None,
num_nextn_predict_layers: int = 8,
tie_word_embeddings: bool = False,
@@ -184,6 +185,8 @@ class InklingModelConfig(PretrainedConfig):
self.log_scaling_alpha = log_scaling_alpha
self.unpadded_vocab_size = self.vocab_size
self.logits_mup_width_multiplier = logits_mup_width_multiplier
# Preserve CUDA projection scores without promoting the inputs or weights.
self.enable_lm_head_output_fp32 = enable_lm_head_output_fp32
self.final_logit_softcapping = final_logit_softcapping
self.num_nextn_predict_layers = num_nextn_predict_layers
+6 -1
View File
@@ -465,6 +465,9 @@ class LogitsProcessor(nn.Module):
self.use_fp32_lm_head = get_exec().features.enable_fp32_lm_head or getattr(
config, "enable_lm_head_fp32", False
)
self.use_fp32_lm_head_output = getattr(
config, "enable_lm_head_output_fp32", False
)
if self.use_attn_tp_group:
self.attn_tp_size = get_parallel().attn_tp_size
self.do_tensor_parallel_all_gather = (
@@ -1092,7 +1095,9 @@ class LogitsProcessor(nn.Module):
)
# A shared FP32 buffer must not change the lm_head's output precision.
# Keep the same rounding as the unbuffered path unless FP32 is enabled.
if use_mm_out_dtype and self.use_fp32_lm_head:
if use_mm_out_dtype and (
self.use_fp32_lm_head or self.use_fp32_lm_head_output
):
logits = torch.mm(
hidden_states,
lm_head.weight.T,