diff --git a/python/sglang/srt/layers/cp/cp_decode_attn_tp.py b/python/sglang/srt/layers/cp/cp_decode_attn_tp.py index d4b57aafdd..4967d749bc 100644 --- a/python/sglang/srt/layers/cp/cp_decode_attn_tp.py +++ b/python/sglang/srt/layers/cp/cp_decode_attn_tp.py @@ -84,6 +84,17 @@ class CpDecodeAttnTpContext: sliced = tensor.narrow(dim, self.decode_tp_rank * chunk, chunk) return sliced if dim == 0 else sliced.contiguous() + def _slice_aiter_bpreshuffled_weight(self, tensor: torch.Tensor) -> torch.Tensor: + """Slice an AITER B-preshuffled weight along logical K.""" + from aiter.ops.shuffle import shuffle_weight + + from sglang.srt.layers.quantization.fp8_utils import ( + unshuffle_aiter_fp8_weight, + ) + + logical = unshuffle_aiter_fp8_weight(tensor) + return shuffle_weight(self._slice(logical, 1), layout=(16, 16)) + # ==================== Unified activate/restore ==================== def _activate(self, obj, attr_name: str, dim: int): @@ -105,7 +116,16 @@ class CpDecodeAttnTpContext: cache_key = (id(obj), attr_name) cache = self._slice_cache.get(cache_key) if cache is None: - cache = (raw, self._slice(raw, dim), is_param) + if ( + attr_name == "weight" + and dim == 1 + and getattr(obj, "aiter_bpreshuffled", False) + ): + sliced = self._slice_aiter_bpreshuffled_weight(raw) + else: + sliced = self._slice(raw, dim) + # Cache shards to avoid recomputation and keep CUDA graph pointers stable. + cache = (raw, sliced, is_param) self._slice_cache[cache_key] = cache if cache[2]: diff --git a/test/registered/kernels/ops/quantization/test_fp8_utils_aiter.py b/test/registered/kernels/ops/quantization/test_fp8_utils_aiter.py index b35a46d667..74f93099ed 100644 --- a/test/registered/kernels/ops/quantization/test_fp8_utils_aiter.py +++ b/test/registered/kernels/ops/quantization/test_fp8_utils_aiter.py @@ -16,6 +16,7 @@ import unittest import torch +from sglang.srt.layers.cp.cp_decode_attn_tp import CpDecodeAttnTpContext from sglang.srt.layers.quantization.fp8_utils import unshuffle_aiter_fp8_weight from sglang.srt.utils import is_hip from sglang.test.ci.ci_register import register_amd_ci @@ -45,6 +46,31 @@ class TestAiterFp8Utils(CustomTestCase): unshuffle_aiter_fp8_weight(shuffled), logical ) + def test_cp_row_slice_preserves_aiter_layout(self): + from aiter.ops.shuffle import shuffle_weight + + tp_size = 4 + rank = 2 + n, k = 32, 128 + logical_weight = ( + torch.arange(n * k, device="cuda", dtype=torch.float32).remainder(7) - 3 + ).to(torch.float8_e4m3fn) + logical_weight = logical_weight.reshape(n, k) + shuffled_weight = shuffle_weight(logical_weight, layout=(16, 16)) + context = CpDecodeAttnTpContext.__new__(CpDecodeAttnTpContext) + context.decode_tp_rank = rank + context.decode_tp_size = tp_size + + expected = context._slice(logical_weight, dim=1) + naive = context._slice(shuffled_weight, dim=1) + self.assertFalse(torch.equal(unshuffle_aiter_fp8_weight(naive), expected)) + + actual = context._slice_aiter_bpreshuffled_weight(shuffled_weight) + torch.testing.assert_close( + actual, + shuffle_weight(expected, layout=(16, 16)), + ) + if __name__ == "__main__": unittest.main()