diff --git a/python/sglang/srt/hardware_backend/npu/attention/mla_preprocess.py b/python/sglang/srt/hardware_backend/npu/attention/mla_preprocess.py index c2b6249caf..b9b48ffdee 100644 --- a/python/sglang/srt/hardware_backend/npu/attention/mla_preprocess.py +++ b/python/sglang/srt/hardware_backend/npu/attention/mla_preprocess.py @@ -108,6 +108,39 @@ class NPUFusedMLAPreprocess(torch.nn.Module): ) self.is_npu_arch35 = is_npu_arch35() + # Resolve the MLAProlog mode once from the loaded projection methods. + # INT8 dtype alone cannot distinguish static from dynamic W8A8. + from sglang.srt.hardware_backend.npu.quantization.linear_method_npu import ( + NPUMXFP8LinearMethod, + NPUW8A8Int8DynamicLinearMethod, + ) + + kernels = [ + getattr(getattr(layer, "scheme", None), "kernel", layer.quant_method) + for layer in (self.qkv_a_proj, self.q_b_proj) + ] + # Each dynamic scheme also specifies its precision and scale layout. + dynamic_quant_modes = ( + (NPUW8A8Int8DynamicLinearMethod, 2), + (NPUMXFP8LinearMethod, 3), + ) + self.weight_quant_mode = None + for method, mode in dynamic_quant_modes: + if all(isinstance(kernel, method) for kernel in kernels): + self.weight_quant_mode = mode + break + if self.weight_quant_mode is None and self.qkv_a_proj.weight.dtype in ( + torch.float16, + torch.bfloat16, + ): + if self.q_b_proj.weight.dtype in (torch.float16, torch.bfloat16): + self.weight_quant_mode = 0 + elif ( + self.q_b_proj.weight.dtype == torch.int8 + and self.q_b_proj_weight_scale is not None + ): + self.weight_quant_mode = 1 + def preprocess_weights(self, hidden_states): self.dummy = torch.zeros( (hidden_states.shape[-1]), @@ -251,31 +284,53 @@ class NPUFusedMLAPreprocess(torch.nn.Module): # 1: Partial INT8 quantization. Only weight_uq_qr (Q-B projection) # is INT8; weight_dq and weight_dkv_kr (QKV-A projection) remain # FP16/BF16. dequant_scale_w_uq_qr is required. + # 2: Dynamic INT8 activations and per-channel INT8 QKV-A/Q-B weights. # 3: MXFP8 quantization. token_x, weight_dq, weight_uq_qr, and # weight_dkv_kr use MXFP8 with their corresponding dequant scales. # weight_uk remains unquantized. - from sglang.srt.hardware_backend.npu.quantization.linear_method_npu import ( - NPUMXFP8LinearMethod, - ) - projections = (self.qkv_a_proj, self.q_b_proj) - kernels = [ - getattr(getattr(layer, "scheme", None), "kernel", layer.quant_method) - for layer in projections - ] - is_mxfp8 = [isinstance(kernel, NPUMXFP8LinearMethod) for kernel in kernels] - if any(is_mxfp8): - if not all(is_mxfp8): - raise RuntimeError( - "MLAProlog MXFP8 requires both QKV-A and Q-B to use MXFP8" - ) - expected_shapes = ( - ( - self.qkv_a_proj.input_size, - self.q_lora_rank + self.kv_lora_rank + self.qk_rope_head_dim, - ), - (self.q_lora_rank, self.num_local_heads * self.qk_head_dim), + expected_shapes = ( + ( + self.qkv_a_proj.input_size, + self.q_lora_rank + self.kv_lora_rank + self.qk_rope_head_dim, + ), + (self.q_lora_rank, self.num_local_heads * self.qk_head_dim), + ) + if self.weight_quant_mode == 2: + if self.rotary_emb.is_neox_style: + raise RuntimeError("Dynamic INT8 MLAProlog requires interleaved RoPE") + # ModelSlim has already transposed INT8 weights to [K, N]. + for norm in (self.q_a_layernorm, self.kv_a_layernorm): + bias = getattr(norm, "bias", None) + if bias is not None and torch.count_nonzero(bias).item(): + raise RuntimeError( + "Dynamic INT8 MLAProlog does not support an RMSNorm bias" + ) + for layer, shape in zip(projections, expected_shapes): + if ( + layer.weight.dtype != torch.int8 + or tuple(layer.weight.shape) != shape + ): + raise RuntimeError( + "Unsupported ModelSlim dynamic INT8 MLA weight layout" + ) + if layer.weight_scale.numel() != shape[1]: + raise RuntimeError( + "Dynamic INT8 MLAProlog requires per-channel scales" + ) + qkv_weight = self.qkv_a_proj.weight.data + qb_weight = self.q_b_proj.weight.data + qkv_scale = self.qkv_a_proj.weight_scale.float().reshape(-1) + self.qkv_a_proj_scale_q = ( + qkv_scale[: self.q_lora_rank].view(1, -1).contiguous() ) + self.qkv_a_proj_scale_kv = ( + qkv_scale[self.q_lora_rank :].view(1, -1).contiguous() + ) + self.q_b_proj_scale = ( + self.q_b_proj.weight_scale.float().view(1, -1).contiguous() + ) + elif self.weight_quant_mode == 3: checkpoint_scales = [] for layer, (k_dim, n_dim) in zip(projections, expected_shapes): scale = getattr(layer, "weight_scale_inv", None) @@ -303,35 +358,29 @@ class NPUFusedMLAPreprocess(torch.nn.Module): qkv_scale[self.q_lora_rank :].contiguous().view(torch.float8_e8m0fnu) ) self.q_b_proj_scale = qb_scale.contiguous().view(torch.float8_e8m0fnu) - self.q_b_proj_weight = npu_format_cast( - self.q_b_proj.weight.data.contiguous() - ) - self.weight_quant_mode = 3 - else: - if self.qkv_a_proj.weight.dtype not in (torch.float16, torch.bfloat16): - raise RuntimeError("Unsupported MLAProlog QKV-A weight format") + qb_weight = self.q_b_proj.weight.data.contiguous() + elif self.weight_quant_mode in (0, 1): qkv_weight = self.qkv_a_proj.weight.data.transpose(0, 1) - if self.q_b_proj.weight.dtype in (torch.float16, torch.bfloat16): - self.weight_quant_mode = 0 - self.q_b_proj_weight = npu_format_cast( - self.q_b_proj.weight.data.transpose(0, 1).contiguous() - ) - elif ( - self.q_b_proj.weight.dtype == torch.int8 - and self.q_b_proj_weight_scale is not None - ): - self.weight_quant_mode = 1 - self.q_b_proj_weight = self.q_b_proj.weight + if self.weight_quant_mode == 0: + qb_weight = self.q_b_proj.weight.data.transpose(0, 1).contiguous() else: - raise RuntimeError("Unsupported MLAProlog Q-B weight format") - self.q_a_proj_weight = npu_format_cast( - qkv_weight[:, : self.q_lora_rank].contiguous() - ) - self.kv_a_proj_weight = npu_format_cast( - qkv_weight[:, self.q_lora_rank :].contiguous() + qb_weight = self.q_b_proj.weight + else: + raise RuntimeError( + "Unsupported MLAProlog QKV-A/Q-B quantization combination" + ) + + q_weight = qkv_weight[:, : self.q_lora_rank].contiguous() + kv_weight = qkv_weight[:, self.q_lora_rank :].contiguous() + + self.q_a_proj_weight = npu_format_cast(q_weight) + self.kv_a_proj_weight = npu_format_cast(kv_weight) + self.q_b_proj_weight = ( + qb_weight if self.weight_quant_mode == 1 else npu_format_cast(qb_weight) ) - if get_disagg().disaggregation_mode != "null": + # Dynamic INT8 keeps the original projections for extend and indexers. + if self.weight_quant_mode != 2 and get_disagg().disaggregation_mode != "null": qkv_weight.data.untyped_storage().resize_(0) def get_sin_cos(self, positions): @@ -527,7 +576,18 @@ class NPUFusedMLAPreprocess(torch.nn.Module): "use BF16 draft KV for BF16 draft weights" ) token_x = hidden_states - if self.weight_quant_mode == 3: + if self.weight_quant_mode == 2: + if any( + t.dtype != torch.bfloat16 + for t in (hidden_states, k_cache, v_cache, self.w_kc) + ): + raise RuntimeError( + "Dynamic INT8 MLAProlog requires BF16 activations and KV cache" + ) + token_x, token_x_scale = torch.ops.npu.npu_dynamic_quant( + hidden_states.contiguous() + ) + elif self.weight_quant_mode == 3: token_x, token_x_scale = torch.ops.npu.npu_dynamic_mx_quant( hidden_states.reshape(-1, hidden_states.shape[-1]).contiguous(), axis=1, @@ -560,7 +620,16 @@ class NPUFusedMLAPreprocess(torch.nn.Module): kv_cache_quant_mode=3 if packed else 0, query_quant_mode=0, ) - if self.weight_quant_mode == 3: + if self.weight_quant_mode == 2: + mla_prolog_input_args.update( + dequant_scale_x=token_x_scale.float().view(-1, 1), + dequant_scale_w_dq=self.qkv_a_proj_scale_q, + dequant_scale_w_dkv_kr=self.qkv_a_proj_scale_kv, + dequant_scale_w_uq_qr=self.q_b_proj_scale, + kv_cache_quant_mode=0, + query_quant_mode=0, + ) + elif self.weight_quant_mode == 3: mla_prolog_input_args.update( dequant_scale_w_dq=self.qkv_a_proj_scale_q, dequant_scale_w_dkv_kr=self.qkv_a_proj_scale_kv, @@ -583,8 +652,13 @@ class NPUFusedMLAPreprocess(torch.nn.Module): ) if self.weight_quant_mode == 0: dequant_q_norm = None - elif self.weight_quant_mode == 1: + elif self.weight_quant_mode in (1, 2): dequant_q_norm = dequant_q_norm.view(hidden_states.shape[0]) + if self.weight_quant_mode == 2 and q_nope.dtype != torch.bfloat16: + raise RuntimeError( + "This torch_npu/CANN MLAProlog implementation does not return BF16 " + "query for weight_quant_mode=2, kv_cache_quant_mode=0" + ) return ( q_pe, v_cache, @@ -597,12 +671,18 @@ class NPUFusedMLAPreprocess(torch.nn.Module): ) def uses_mlaprolog(self) -> bool: - _is_arch35_dsa = ( - self.is_npu_arch35 and get_token_to_kv_pool().index_head_dim is not None - ) - return _is_arch35_dsa or ( + _is_dsa_mlaprolog = ( + self.is_npu_arch35 or self.weight_quant_mode == 2 + ) and get_token_to_kv_pool().index_head_dim is not None + return _is_dsa_mlaprolog or ( hasattr(self.quant_config, "ignore") and any(re.fullmatch(r".*kv_b_proj", l) for l in self.quant_config.ignore) + # Other ModelSlim projections retain the legacy MLAPO path. + and not ( + hasattr(self.qkv_a_proj.quant_method, "quantization_config") + and self.qkv_a_proj.quant_method.quantization_config.get_name() + == "modelslim" + ) ) def forward(self, positions, hidden_states, forward_batch, zero_allocator): @@ -613,12 +693,9 @@ class NPUFusedMLAPreprocess(torch.nn.Module): and self.qkv_a_proj.quant_method.quantization_config.get_name() == "modelslim" ) - _is_arch35_dsa = ( - self.is_npu_arch35 and get_token_to_kv_pool().index_head_dim is not None - ) # with the mlaprolog enabled, the kv_b_proj layers are unquantized _is_mlaprolog = self.uses_mlaprolog() - if _is_w8a8 and not _is_arch35_dsa: + if _is_w8a8 and not _is_mlaprolog: return self.forward_mlapo( positions, hidden_states, forward_batch, zero_allocator ) diff --git a/python/sglang/srt/hardware_backend/npu/modules/deepseek_v2_attention_mla_npu.py b/python/sglang/srt/hardware_backend/npu/modules/deepseek_v2_attention_mla_npu.py index 0e1fa06d49..8f92e73f3b 100644 --- a/python/sglang/srt/hardware_backend/npu/modules/deepseek_v2_attention_mla_npu.py +++ b/python/sglang/srt/hardware_backend/npu/modules/deepseek_v2_attention_mla_npu.py @@ -631,10 +631,11 @@ def npu_mla_preprocess( if ( get_disagg().disaggregation_mode == "decode" and m.mla_preprocess.uses_mlaprolog() + and m.mla_preprocess.weight_quant_mode != 2 and m.w_kc is not None ): m.w_kc.untyped_storage().resize_(0) - # mlaprolog does not require additional calculation of q_lora + # MLAProlog returns q_lora; BF16 indexers need it recomputed below. if m.mla_preprocess.uses_mlaprolog(): ( q_pe, @@ -648,6 +649,19 @@ def npu_mla_preprocess( ) = m.mla_preprocess.forward( positions, hidden_states, forward_batch, zero_allocator ) + if m.mla_preprocess.weight_quant_mode == 2 and m.indexer is not None: + from sglang.srt.hardware_backend.npu.quantization.linear_method_npu import ( + NPUW8A8Int8DynamicLinearMethod, + ) + + if not isinstance( + getattr(m.indexer.wq_b.scheme, "kernel", None), + NPUW8A8Int8DynamicLinearMethod, + ): + # BF16 indexers need q_lora before INT8 quantization. + q = m.fused_qkv_a_proj_with_mqa(hidden_states)[0][..., : m.q_lora_rank] + q_lora = m.q_a_layernorm(q) + dynamic_scale = None else: if m.alt_stream is not None: mla_event = torch.npu.Event() diff --git a/test/registered/unit/attention/test_npu_mlaprolog.py b/test/registered/unit/attention/test_npu_mlaprolog.py new file mode 100644 index 0000000000..9e735e2ef5 --- /dev/null +++ b/test/registered/unit/attention/test_npu_mlaprolog.py @@ -0,0 +1,336 @@ +"""CPU regressions for dynamic INT8 MLAProlog dispatch and tensor contracts. + +Only vendor operators are replaced; weight preparation and indexer adaptation +run through the production code. These tests do not validate NPU numerics. +""" + +import importlib +import unittest +from contextlib import ExitStack +from types import ModuleType, SimpleNamespace +from unittest.mock import Mock, patch + +import torch + +# Initialize quantization before importing its NPU kernels (avoids a cycle). +import sglang.srt.layers.quantization # noqa: F401 +from sglang.srt.hardware_backend.npu.attention import mla_preprocess as mla +from sglang.srt.hardware_backend.npu.quantization.linear_method_npu import ( + NPUW8A8Int8DynamicLinearMethod, + NPUW8A8Int8LinearMethod, +) +from sglang.srt.layers.layernorm import RMSNorm +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=15, suite="base-a-test-cpu") + + +class _Projection(torch.nn.Module): + """Small CPU projection with the post-load ModelSlim weight layout.""" + + def __init__(self, input_size, output_size, kernel): + super().__init__() + self.input_size = input_size + self.weight = torch.arange(input_size * output_size, dtype=torch.int8).view( + input_size, output_size + ) + self.weight_scale = torch.arange(1, output_size + 1).bfloat16() / 16 + self.scheme = SimpleNamespace(kernel=kernel) + self.quant_method = SimpleNamespace( + quantization_config=SimpleNamespace(get_name=lambda: "modelslim") + ) + + def forward(self, x): + # Fail cleanly if preprocessing incorrectly frees a shared weight buffer. + if self.weight.untyped_storage().nbytes() == 0: + raise AssertionError("Source projection weight was released") + weight = self.weight.float() * self.weight_scale + return (x.float() @ weight).to(x.dtype), None + + +class TestDynamicInt8MLAProlog(CustomTestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + # The attention wrapper imports vendor packages even on a CPU host. + vendor_norm = ModuleType("sgl_kernel_npu.norm.fused_split_qk_norm") + vendor_norm.fused_split_qk_norm = Mock( + side_effect=AssertionError("Unexpected unfused NPU operator") + ) + with patch.dict( + "sys.modules", + { + "torch_npu": ModuleType("torch_npu"), + "sgl_kernel_npu.norm.fused_split_qk_norm": vendor_norm, + }, + ): + cls.attention = importlib.import_module( + "sglang.srt.hardware_backend.npu.modules.deepseek_v2_attention_mla_npu" + ) + + def setUp(self): + self.stack = ExitStack() + self.addCleanup(self.stack.close) + self.k_cache = torch.zeros(1, 4, 1, 3, dtype=torch.bfloat16) + self.kr_cache = torch.zeros(1, 4, 1, 2, dtype=torch.bfloat16) + self.pool = SimpleNamespace( + index_head_dim=4, + dsa_kv_cache_store_fp8=False, + get_kv_buffer=lambda _: (self.k_cache, self.kr_cache), + ) + self.disagg = SimpleNamespace(disaggregation_mode="decode") + for module in (mla, self.attention): + self.stack.enter_context( + patch.object(module, "get_disagg", return_value=self.disagg) + ) + self.stack.enter_context( + patch.object(mla, "get_token_to_kv_pool", return_value=self.pool) + ) + self.stack.enter_context(patch.object(mla, "is_npu_arch35", return_value=False)) + self.stack.enter_context(patch.object(mla, "is_fia_nz", return_value=False)) + self.stack.enter_context( + patch.object(mla, "npu_format_cast", side_effect=lambda x: x.clone()) + ) + self.x = torch.tensor([[1, -2, 3, -4], [2, 1, -1, 3]], dtype=torch.bfloat16) + self.positions = torch.arange(2) + self.batch = SimpleNamespace(out_cache_loc=torch.tensor([0, 1])) + self.quant_x = torch.tensor([[2, -4, 6, -8], [8, 4, -4, 12]], dtype=torch.int8) + self.x_scale = torch.tensor([0.5, 0.25], dtype=torch.float64) + self.query = torch.ones(2, 2, 3, dtype=torch.bfloat16) + self.query_rope = torch.full((2, 2, 2), 2, dtype=torch.bfloat16) + self.latent = torch.tensor([[1, -3], [4, 2]], dtype=torch.int8) + self.latent_scale = torch.tensor([[0.125], [0.25]]) + self.vendor = ModuleType("torch_npu") + self.vendor.npu_mla_prolog_v3 = Mock( + return_value=( + self.query, + self.query_rope, + None, + self.latent, + self.latent_scale, + ) + ) + self.stack.enter_context(patch.dict("sys.modules", {"torch_npu": self.vendor})) + self.stack.enter_context( + patch.object( + torch.ops, + "npu", + SimpleNamespace( + npu_dynamic_quant=Mock(return_value=(self.quant_x, self.x_scale)) + ), + ) + ) + + def make_model(self, kernel_type=NPUW8A8Int8DynamicLinearMethod): + q_norm = RMSNorm(2, weight_dtype=torch.bfloat16, force_native=True) + kv_norm = RMSNorm(3, weight_dtype=torch.bfloat16, force_native=True) + # A present but zero bias is supported. + q_norm.bias = torch.zeros(2, dtype=torch.bfloat16) + return SimpleNamespace( + fused_qkv_a_proj_with_mqa=_Projection(4, 7, kernel_type()), + q_a_layernorm=q_norm, + kv_a_layernorm=kv_norm, + q_b_proj=_Projection(2, 8, kernel_type()), + w_kc=torch.arange(12, dtype=torch.bfloat16).view(2, 2, 3), + rotary_emb=SimpleNamespace( + is_neox_style=False, + cos_sin_cache=torch.tensor([[1, 0]] * 4, dtype=torch.bfloat16), + ), + layer_id=0, + q_lora_rank=2, + num_local_heads=2, + qk_nope_head_dim=2, + qk_rope_head_dim=2, + v_head_dim=3, + quant_config=SimpleNamespace(ignore=["model.layers.0.self_attn.kv_b_proj"]), + indexer=None, + ) + + def make_preprocess(self, model): + return mla.NPUFusedMLAPreprocess( + model.fused_qkv_a_proj_with_mqa, + model.q_a_layernorm, + model.kv_a_layernorm, + model.q_b_proj, + model.w_kc, + model.rotary_emb, + model.layer_id, + model.num_local_heads, + model.qk_nope_head_dim, + model.qk_rope_head_dim, + model.v_head_dim, + model.quant_config, + ) + + def test_dynamic_dispatch_supplies_int8_operator_contract(self): + """Dynamic ModelSlim must not access static input_offset parameters.""" + model = self.make_model() + preprocess = self.make_preprocess(model) + result = preprocess(self.positions, self.x, self.batch, None) + args = self.vendor.npu_mla_prolog_v3.call_args.kwargs + self.assertEqual(args["weight_quant_mode"], 2) + self.assertEqual(args["kv_cache_quant_mode"], 0) + self.assertEqual(args["query_quant_mode"], 0) + self.assertEqual(args["token_x"].dtype, torch.int8) + self.assertEqual(args["dequant_scale_x"].dtype, torch.float32) + self.assertEqual(args["dequant_scale_x"].shape, (2, 1)) + for scale, channels in ( + ("dequant_scale_w_dq", 2), + ("dequant_scale_w_dkv_kr", 5), + ("dequant_scale_w_uq_qr", 8), + ): + self.assertEqual(args[scale].dtype, torch.float32) + self.assertEqual(args[scale].shape, (1, channels)) + for weight, scale, expected in ( + ( + "weight_dq", + "dequant_scale_w_dq", + model.fused_qkv_a_proj_with_mqa(self.x)[0][:, :2], + ), + ( + "weight_dkv_kr", + "dequant_scale_w_dkv_kr", + model.fused_qkv_a_proj_with_mqa(self.x)[0][:, 2:], + ), + ): + actual = ( + (args["token_x"].float() @ args[weight].float()) + * args["dequant_scale_x"] + * args[scale] + ) + torch.testing.assert_close(actual.to(torch.bfloat16), expected) + torch.testing.assert_close( + args["dequant_scale_w_uq_qr"], + model.q_b_proj.weight_scale.float().view(1, -1), + ) + self.assertIs(result[0], self.query_rope) + self.assertIs(result[1], self.kr_cache) + self.assertIs(result[2], self.query) + self.assertIs(result[3], self.k_cache) + self.assertIs(result[4], self.latent) + torch.testing.assert_close(result[-1], self.latent_scale.flatten()) + + def test_static_and_mixed_int8_do_not_select_dynamic_mlaprolog(self): + for mixed in (False, True): + with self.subTest(mixed=mixed): + model = self.make_model(NPUW8A8Int8LinearMethod) + if mixed: + model.q_b_proj.scheme.kernel = NPUW8A8Int8DynamicLinearMethod() + preprocess = self.make_preprocess(model) + self.assertFalse(preprocess.uses_mlaprolog()) + with self.assertRaisesRegex(RuntimeError, "Unsupported MLAProlog"): + preprocess.mlaprolog_preprocess_weight() + + def test_weight_split_preserves_dequantized_projections_and_source_storage(self): + model = self.make_model() + qkv = model.fused_qkv_a_proj_with_mqa + original = qkv.weight.clone() + original_bytes = qkv.weight.untyped_storage().nbytes() + preprocess = self.make_preprocess(model) + preprocess.mlaprolog_preprocess_weight() + self.assertEqual(qkv.weight.untyped_storage().nbytes(), original_bytes) + torch.testing.assert_close(qkv.weight, original) + restored = torch.cat( + ( + preprocess.q_a_proj_weight.float() * preprocess.qkv_a_proj_scale_q, + preprocess.kv_a_proj_weight.float() * preprocess.qkv_a_proj_scale_kv, + ), + dim=1, + ) + torch.testing.assert_close(restored, original.float() * qkv.weight_scale) + torch.testing.assert_close( + preprocess.q_b_proj_weight.float() * preprocess.q_b_proj_scale, + model.q_b_proj.weight.float() * model.q_b_proj.weight_scale, + ) + + def test_unsupported_weight_contracts_fail_before_operator_execution(self): + for invalid in ("neox", "bias", "shape", "dtype", "scale"): + with self.subTest(invalid=invalid): + model = self.make_model() + preprocess = self.make_preprocess(model) + if invalid == "neox": + model.rotary_emb.is_neox_style = True + error = "interleaved RoPE" + elif invalid == "bias": + model.q_a_layernorm.bias[0] = 1 + error = "RMSNorm bias" + elif invalid == "shape": + model.q_b_proj.weight = model.q_b_proj.weight.t() + error = "weight layout" + elif invalid == "dtype": + model.fused_qkv_a_proj_with_mqa.weight = ( + model.fused_qkv_a_proj_with_mqa.weight.float() + ) + error = "weight layout" + else: + model.q_b_proj.weight_scale = model.q_b_proj.weight_scale[:1] + error = "per-channel scales" + with self.assertRaisesRegex(RuntimeError, error): + preprocess(self.positions, self.x, self.batch, None) + + def test_requires_bf16_activations_cache_and_final_query(self): + for invalid in ("input", "kv", "kr", "w_kc", "query"): + with self.subTest(invalid=invalid): + model = self.make_model() + preprocess = self.make_preprocess(model) + x = self.x.float() if invalid == "input" else self.x + k_cache = ( + self.k_cache.to(torch.int8) if invalid == "kv" else self.k_cache + ) + kr_cache = ( + self.kr_cache.to(torch.int8) if invalid == "kr" else self.kr_cache + ) + if invalid == "w_kc": + preprocess.w_kc = preprocess.w_kc.float() + outputs = ( + self.query.to(torch.int8) if invalid == "query" else self.query, + self.query_rope, + None, + self.latent, + self.latent_scale, + ) + with ( + patch.object( + self.pool, "get_kv_buffer", return_value=(k_cache, kr_cache) + ), + patch.object( + self.vendor, "npu_mla_prolog_v3", Mock(return_value=outputs) + ), + self.assertRaisesRegex(RuntimeError, "BF16"), + ): + preprocess(self.positions, x, self.batch, None) + + def test_indexer_latent_and_decode_weight_lifetime(self): + """BF16 indexers need the pre-quantization latent; shared-topk has none.""" + for kind in ("dynamic", "bf16", "none"): + with self.subTest(indexer=kind): + model = self.make_model() + if kind != "none": + scheme = ( + SimpleNamespace(kernel=NPUW8A8Int8DynamicLinearMethod()) + if kind == "dynamic" + else None + ) + model.indexer = SimpleNamespace(wq_b=SimpleNamespace(scheme=scheme)) + expected = model.q_a_layernorm( + model.fused_qkv_a_proj_with_mqa(self.x)[0][:, :2] + ) + original_w_kc = model.w_kc.clone() + original_bytes = model.w_kc.untyped_storage().nbytes() + result = self.attention.npu_mla_preprocess( + model, self.x, self.positions, self.batch, None + ) + self.assertEqual(model.w_kc.untyped_storage().nbytes(), original_bytes) + torch.testing.assert_close(model.w_kc, original_w_kc) + if kind == "bf16": + self.assertEqual(result[4].dtype, torch.bfloat16) + torch.testing.assert_close(result[4], expected) + self.assertIsNone(result[-1]) + else: + self.assertIs(result[4], self.latent) + torch.testing.assert_close(result[-1], self.latent_scale.flatten()) + + +if __name__ == "__main__": + unittest.main()