[NPU] Support batch invariant FIA graphs for deterministic inference (#39607)

This commit is contained in:
hanwlax
2026-09-24 09:53:17 +08:00
committed by GitHub
parent 81b81664a6
commit 03fcbe147f
2 changed files with 56 additions and 6 deletions
@@ -1030,6 +1030,7 @@ def enable_batch_invariant_mode(enable_bmm: bool = True):
else:
from sglang.srt.hardware_backend.npu.batch_invariant_ops.npu_batch_invariant_ops import (
npu_add_rms_norm_batch_invariant,
npu_fia_batch_invariant_get_max_workspace,
npu_fused_infer_attention_score_batch_invariant,
npu_log_softmax_batch_invariant,
npu_matmul_batch_invariant,
@@ -1050,6 +1051,12 @@ def enable_batch_invariant_mode(enable_bmm: bool = True):
torch.ops.npu.npu_fused_infer_attention_score = (
npu_fused_infer_attention_score_batch_invariant
)
torch_npu.npu_fused_infer_attention_score = (
npu_fused_infer_attention_score_batch_invariant
)
torch_npu._npu_fused_infer_attention_score_get_max_workspace = (
npu_fia_batch_invariant_get_max_workspace
)
torch_npu.npu_add_rms_norm = npu_add_rms_norm_batch_invariant
@@ -3,6 +3,7 @@
import batch_invariant_ops # noqa: F401
import torch
import torch_npu
from torch_npu.npu import NpuGraphOpHandler, register_npu_graph_handler
def npu_mm_batch_invariant(a, b):
@@ -38,12 +39,11 @@ def npu_log_softmax_batch_invariant(input, dim, _half_to_float):
return torch.ops.batch_invariant_ops.npu_log_softmax_batch_invariant(input, dim=dim)
def npu_fused_infer_attention_score_batch_invariant(*args, **kwargs):
return (
torch.ops.batch_invariant_ops.npu_fused_infer_attention_score_batch_invariant(
*args, **kwargs
)
)
# Preserve the OpOverloadPacket, including its graph-compatible .out overload.
npu_fused_infer_attention_score_batch_invariant = (
torch.ops.batch_invariant_ops.npu_fused_infer_attention_score_batch_invariant
)
npu_fia_batch_invariant_get_max_workspace = torch.ops.batch_invariant_ops._npu_fused_infer_attention_score_batch_invariant_get_max_workspace
def npu_add_rms_norm_batch_invariant(
@@ -60,3 +60,46 @@ def npu_add_rms_norm_batch_invariant(
residual_ = x_
x_, _ = torch_npu.npu_rms_norm(x_, weight, eps)
return x_, None, residual_
@register_npu_graph_handler(
[
"npu_fused_infer_attention_score_batch_invariant.default",
"npu_fused_infer_attention_score_batch_invariant.out",
]
)
class NativeFIAGraphHandler(NpuGraphOpHandler):
"""Rebind the native FIA's keyword-only CPU lengths on graph updates."""
@classmethod
def prepare_capture(cls, func, args, kwargs):
ops = torch.ops.batch_invariant_ops
out_op = npu_fused_infer_attention_score_batch_invariant.out
# graph.update replaces matching kwargs; include optional lengths too.
kwargs = dict(kwargs)
kwargs.setdefault("actual_seq_lengths", None)
kwargs.setdefault("actual_seq_lengths_kv", None)
if func is out_op:
return func, args, kwargs
workspace = npu_fia_batch_invariant_get_max_workspace(*args, **kwargs)
# Match infer_output's keyword-only schema; omit absent kwargs to keep
# its defaults. Sequence lengths are not accepted by this helper;
# they remain in kwargs for FIA execution and graph updates.
keys = [
"input_layout",
"quant_scale2",
"block_table",
"num_heads",
"num_key_value_heads",
"softmax_lse_flag",
"query_rope",
]
# FIA positional args are (query, key, value): args[0] is Q, args[2] is V.
# This helper derives output shapes/dtypes from Q, V and the options
# above, then allocates (attention_output, softmax_lse) for .out.
# It does not compute attention; .out fills these buffers.
output = ops._npu_fused_infer_attention_score_batch_invariant_infer_output(
args[0], args[2], **{k: kwargs[k] for k in keys if k in kwargs}
)
kwargs.update(workspace=workspace, out=list(output))
return out_op, args, kwargs