[AMD] Fix AITER weight slicing for CP decode TP (#39950)

This commit is contained in:
AMD-yanfeiwang
2026-10-03 12:09:08 -07:00
committed by GitHub
parent f327f92424
commit e83c95b650
2 changed files with 47 additions and 1 deletions
@@ -84,6 +84,17 @@ class CpDecodeAttnTpContext:
sliced = tensor.narrow(dim, self.decode_tp_rank * chunk, chunk)
return sliced if dim == 0 else sliced.contiguous()
def _slice_aiter_bpreshuffled_weight(self, tensor: torch.Tensor) -> torch.Tensor:
"""Slice an AITER B-preshuffled weight along logical K."""
from aiter.ops.shuffle import shuffle_weight
from sglang.srt.layers.quantization.fp8_utils import (
unshuffle_aiter_fp8_weight,
)
logical = unshuffle_aiter_fp8_weight(tensor)
return shuffle_weight(self._slice(logical, 1), layout=(16, 16))
# ==================== Unified activate/restore ====================
def _activate(self, obj, attr_name: str, dim: int):
@@ -105,7 +116,16 @@ class CpDecodeAttnTpContext:
cache_key = (id(obj), attr_name)
cache = self._slice_cache.get(cache_key)
if cache is None:
cache = (raw, self._slice(raw, dim), is_param)
if (
attr_name == "weight"
and dim == 1
and getattr(obj, "aiter_bpreshuffled", False)
):
sliced = self._slice_aiter_bpreshuffled_weight(raw)
else:
sliced = self._slice(raw, dim)
# Cache shards to avoid recomputation and keep CUDA graph pointers stable.
cache = (raw, sliced, is_param)
self._slice_cache[cache_key] = cache
if cache[2]:
@@ -16,6 +16,7 @@ import unittest
import torch
from sglang.srt.layers.cp.cp_decode_attn_tp import CpDecodeAttnTpContext
from sglang.srt.layers.quantization.fp8_utils import unshuffle_aiter_fp8_weight
from sglang.srt.utils import is_hip
from sglang.test.ci.ci_register import register_amd_ci
@@ -45,6 +46,31 @@ class TestAiterFp8Utils(CustomTestCase):
unshuffle_aiter_fp8_weight(shuffled), logical
)
def test_cp_row_slice_preserves_aiter_layout(self):
from aiter.ops.shuffle import shuffle_weight
tp_size = 4
rank = 2
n, k = 32, 128
logical_weight = (
torch.arange(n * k, device="cuda", dtype=torch.float32).remainder(7) - 3
).to(torch.float8_e4m3fn)
logical_weight = logical_weight.reshape(n, k)
shuffled_weight = shuffle_weight(logical_weight, layout=(16, 16))
context = CpDecodeAttnTpContext.__new__(CpDecodeAttnTpContext)
context.decode_tp_rank = rank
context.decode_tp_size = tp_size
expected = context._slice(logical_weight, dim=1)
naive = context._slice(shuffled_weight, dim=1)
self.assertFalse(torch.equal(unshuffle_aiter_fp8_weight(naive), expected))
actual = context._slice_aiter_bpreshuffled_weight(shuffled_weight)
torch.testing.assert_close(
actual,
shuffle_weight(expected, layout=(16, 16)),
)
if __name__ == "__main__":
unittest.main()