diff --git a/python/sglang/srt/batch_invariant_ops/batch_invariant_ops.py b/python/sglang/srt/batch_invariant_ops/batch_invariant_ops.py index 882f3535d9..b6a7a5849a 100644 --- a/python/sglang/srt/batch_invariant_ops/batch_invariant_ops.py +++ b/python/sglang/srt/batch_invariant_ops/batch_invariant_ops.py @@ -1030,6 +1030,7 @@ def enable_batch_invariant_mode(enable_bmm: bool = True): else: from sglang.srt.hardware_backend.npu.batch_invariant_ops.npu_batch_invariant_ops import ( npu_add_rms_norm_batch_invariant, + npu_fia_batch_invariant_get_max_workspace, npu_fused_infer_attention_score_batch_invariant, npu_log_softmax_batch_invariant, npu_matmul_batch_invariant, @@ -1050,6 +1051,12 @@ def enable_batch_invariant_mode(enable_bmm: bool = True): torch.ops.npu.npu_fused_infer_attention_score = ( npu_fused_infer_attention_score_batch_invariant ) + torch_npu.npu_fused_infer_attention_score = ( + npu_fused_infer_attention_score_batch_invariant + ) + torch_npu._npu_fused_infer_attention_score_get_max_workspace = ( + npu_fia_batch_invariant_get_max_workspace + ) torch_npu.npu_add_rms_norm = npu_add_rms_norm_batch_invariant diff --git a/python/sglang/srt/hardware_backend/npu/batch_invariant_ops/npu_batch_invariant_ops.py b/python/sglang/srt/hardware_backend/npu/batch_invariant_ops/npu_batch_invariant_ops.py index b947c3c893..d25330c5a3 100644 --- a/python/sglang/srt/hardware_backend/npu/batch_invariant_ops/npu_batch_invariant_ops.py +++ b/python/sglang/srt/hardware_backend/npu/batch_invariant_ops/npu_batch_invariant_ops.py @@ -3,6 +3,7 @@ import batch_invariant_ops # noqa: F401 import torch import torch_npu +from torch_npu.npu import NpuGraphOpHandler, register_npu_graph_handler def npu_mm_batch_invariant(a, b): @@ -38,12 +39,11 @@ def npu_log_softmax_batch_invariant(input, dim, _half_to_float): return torch.ops.batch_invariant_ops.npu_log_softmax_batch_invariant(input, dim=dim) -def npu_fused_infer_attention_score_batch_invariant(*args, **kwargs): - return ( - torch.ops.batch_invariant_ops.npu_fused_infer_attention_score_batch_invariant( - *args, **kwargs - ) - ) +# Preserve the OpOverloadPacket, including its graph-compatible .out overload. +npu_fused_infer_attention_score_batch_invariant = ( + torch.ops.batch_invariant_ops.npu_fused_infer_attention_score_batch_invariant +) +npu_fia_batch_invariant_get_max_workspace = torch.ops.batch_invariant_ops._npu_fused_infer_attention_score_batch_invariant_get_max_workspace def npu_add_rms_norm_batch_invariant( @@ -60,3 +60,46 @@ def npu_add_rms_norm_batch_invariant( residual_ = x_ x_, _ = torch_npu.npu_rms_norm(x_, weight, eps) return x_, None, residual_ + + +@register_npu_graph_handler( + [ + "npu_fused_infer_attention_score_batch_invariant.default", + "npu_fused_infer_attention_score_batch_invariant.out", + ] +) +class NativeFIAGraphHandler(NpuGraphOpHandler): + """Rebind the native FIA's keyword-only CPU lengths on graph updates.""" + + @classmethod + def prepare_capture(cls, func, args, kwargs): + ops = torch.ops.batch_invariant_ops + out_op = npu_fused_infer_attention_score_batch_invariant.out + # graph.update replaces matching kwargs; include optional lengths too. + kwargs = dict(kwargs) + kwargs.setdefault("actual_seq_lengths", None) + kwargs.setdefault("actual_seq_lengths_kv", None) + if func is out_op: + return func, args, kwargs + workspace = npu_fia_batch_invariant_get_max_workspace(*args, **kwargs) + # Match infer_output's keyword-only schema; omit absent kwargs to keep + # its defaults. Sequence lengths are not accepted by this helper; + # they remain in kwargs for FIA execution and graph updates. + keys = [ + "input_layout", + "quant_scale2", + "block_table", + "num_heads", + "num_key_value_heads", + "softmax_lse_flag", + "query_rope", + ] + # FIA positional args are (query, key, value): args[0] is Q, args[2] is V. + # This helper derives output shapes/dtypes from Q, V and the options + # above, then allocates (attention_output, softmax_lse) for .out. + # It does not compute attention; .out fills these buffers. + output = ops._npu_fused_infer_attention_score_batch_invariant_infer_output( + args[0], args[2], **{k: kwargs[k] for k in keys if k in kwargs} + ) + kwargs.update(workspace=workspace, out=list(output)) + return out_op, args, kwargs