[NPU] Support batch invariant FIA graphs for deterministic inference (#39607)
This commit is contained in:
@@ -1030,6 +1030,7 @@ def enable_batch_invariant_mode(enable_bmm: bool = True):
|
||||
else:
|
||||
from sglang.srt.hardware_backend.npu.batch_invariant_ops.npu_batch_invariant_ops import (
|
||||
npu_add_rms_norm_batch_invariant,
|
||||
npu_fia_batch_invariant_get_max_workspace,
|
||||
npu_fused_infer_attention_score_batch_invariant,
|
||||
npu_log_softmax_batch_invariant,
|
||||
npu_matmul_batch_invariant,
|
||||
@@ -1050,6 +1051,12 @@ def enable_batch_invariant_mode(enable_bmm: bool = True):
|
||||
torch.ops.npu.npu_fused_infer_attention_score = (
|
||||
npu_fused_infer_attention_score_batch_invariant
|
||||
)
|
||||
torch_npu.npu_fused_infer_attention_score = (
|
||||
npu_fused_infer_attention_score_batch_invariant
|
||||
)
|
||||
torch_npu._npu_fused_infer_attention_score_get_max_workspace = (
|
||||
npu_fia_batch_invariant_get_max_workspace
|
||||
)
|
||||
torch_npu.npu_add_rms_norm = npu_add_rms_norm_batch_invariant
|
||||
|
||||
|
||||
|
||||
+49
-6
@@ -3,6 +3,7 @@
|
||||
import batch_invariant_ops # noqa: F401
|
||||
import torch
|
||||
import torch_npu
|
||||
from torch_npu.npu import NpuGraphOpHandler, register_npu_graph_handler
|
||||
|
||||
|
||||
def npu_mm_batch_invariant(a, b):
|
||||
@@ -38,12 +39,11 @@ def npu_log_softmax_batch_invariant(input, dim, _half_to_float):
|
||||
return torch.ops.batch_invariant_ops.npu_log_softmax_batch_invariant(input, dim=dim)
|
||||
|
||||
|
||||
def npu_fused_infer_attention_score_batch_invariant(*args, **kwargs):
|
||||
return (
|
||||
torch.ops.batch_invariant_ops.npu_fused_infer_attention_score_batch_invariant(
|
||||
*args, **kwargs
|
||||
)
|
||||
)
|
||||
# Preserve the OpOverloadPacket, including its graph-compatible .out overload.
|
||||
npu_fused_infer_attention_score_batch_invariant = (
|
||||
torch.ops.batch_invariant_ops.npu_fused_infer_attention_score_batch_invariant
|
||||
)
|
||||
npu_fia_batch_invariant_get_max_workspace = torch.ops.batch_invariant_ops._npu_fused_infer_attention_score_batch_invariant_get_max_workspace
|
||||
|
||||
|
||||
def npu_add_rms_norm_batch_invariant(
|
||||
@@ -60,3 +60,46 @@ def npu_add_rms_norm_batch_invariant(
|
||||
residual_ = x_
|
||||
x_, _ = torch_npu.npu_rms_norm(x_, weight, eps)
|
||||
return x_, None, residual_
|
||||
|
||||
|
||||
@register_npu_graph_handler(
|
||||
[
|
||||
"npu_fused_infer_attention_score_batch_invariant.default",
|
||||
"npu_fused_infer_attention_score_batch_invariant.out",
|
||||
]
|
||||
)
|
||||
class NativeFIAGraphHandler(NpuGraphOpHandler):
|
||||
"""Rebind the native FIA's keyword-only CPU lengths on graph updates."""
|
||||
|
||||
@classmethod
|
||||
def prepare_capture(cls, func, args, kwargs):
|
||||
ops = torch.ops.batch_invariant_ops
|
||||
out_op = npu_fused_infer_attention_score_batch_invariant.out
|
||||
# graph.update replaces matching kwargs; include optional lengths too.
|
||||
kwargs = dict(kwargs)
|
||||
kwargs.setdefault("actual_seq_lengths", None)
|
||||
kwargs.setdefault("actual_seq_lengths_kv", None)
|
||||
if func is out_op:
|
||||
return func, args, kwargs
|
||||
workspace = npu_fia_batch_invariant_get_max_workspace(*args, **kwargs)
|
||||
# Match infer_output's keyword-only schema; omit absent kwargs to keep
|
||||
# its defaults. Sequence lengths are not accepted by this helper;
|
||||
# they remain in kwargs for FIA execution and graph updates.
|
||||
keys = [
|
||||
"input_layout",
|
||||
"quant_scale2",
|
||||
"block_table",
|
||||
"num_heads",
|
||||
"num_key_value_heads",
|
||||
"softmax_lse_flag",
|
||||
"query_rope",
|
||||
]
|
||||
# FIA positional args are (query, key, value): args[0] is Q, args[2] is V.
|
||||
# This helper derives output shapes/dtypes from Q, V and the options
|
||||
# above, then allocates (attention_output, softmax_lse) for .out.
|
||||
# It does not compute attention; .out fills these buffers.
|
||||
output = ops._npu_fused_infer_attention_score_batch_invariant_infer_output(
|
||||
args[0], args[2], **{k: kwargs[k] for k in keys if k in kwargs}
|
||||
)
|
||||
kwargs.update(workspace=workspace, out=list(output))
|
||||
return out_op, args, kwargs
|
||||
|
||||
Reference in New Issue
Block a user