From c2f14bfbc4e82ade955c4dacfaefd8e0a19abddf Mon Sep 17 00:00:00 2001 From: chuyeh Date: Tue, 22 Sep 2026 16:49:59 +0800 Subject: [PATCH] [AMD] Use exact CU share for gfx950 segment-plan headroom (#39503) Co-authored-by: jacky.cheng --- .../attention/vattn_asm_gfx950/__init__.py | 21 +++++- .../attention/test_vattn_asm_segment_plan.py | 73 +++++++++++++++++++ 2 files changed, 91 insertions(+), 3 deletions(-) create mode 100644 test/registered/unit/layers/attention/test_vattn_asm_segment_plan.py diff --git a/python/sglang/kernels/ops/attention/vattn_asm_gfx950/__init__.py b/python/sglang/kernels/ops/attention/vattn_asm_gfx950/__init__.py index 56576724d9..29545fbf83 100644 --- a/python/sglang/kernels/ops/attention/vattn_asm_gfx950/__init__.py +++ b/python/sglang/kernels/ops/attention/vattn_asm_gfx950/__init__.py @@ -323,9 +323,24 @@ def _seg_plan_target_wgs() -> int: return _SEG_PLAN_TARGET_WGS -def mtp_verify_attn_seg_max(num_seqs: int, num_kv_heads: int) -> int: - """Static grid.x for the planned split: 2x the legacy per-seq count, clamped to 16..64.""" - return max(16, min(64, 2 * mtp_verify_attn_num_segments(num_seqs, num_kv_heads))) +def mtp_verify_attn_seg_max( + num_seqs: int, num_kv_heads: int, num_cus: int | None = None +) -> int: + """Static grid.x for the planned split: 2x the exact equal-share CU budget, + clamped to 16..64. + + Unlike mtp_verify_attn_num_segments (the legacy fixed split), the share is + not rounded down to a power of two: that quantization costs a long request + in a skewed batch half its headroom at every batch discontinuity, while the + planner's own workgroup budget already prevents oversubscription. + + num_cus defaults to the device count; pass it to keep the cap independent of + the current device (a CPX partition exposes 32 of an MI355X's 256). + """ + if num_cus is None: + num_cus = _seg_plan_target_wgs() + uniform_share = max(1, num_cus // max(1, num_seqs * num_kv_heads)) + return max(16, min(64, 2 * uniform_share)) def _get_plan_kernel(): diff --git a/test/registered/unit/layers/attention/test_vattn_asm_segment_plan.py b/test/registered/unit/layers/attention/test_vattn_asm_segment_plan.py new file mode 100644 index 0000000000..fde2e55b44 --- /dev/null +++ b/test/registered/unit/layers/attention/test_vattn_asm_segment_plan.py @@ -0,0 +1,73 @@ +"""Planner segment cap for the gfx950 assembly attention split. + +The cap is the static grid.x of the planned split, so it bounds how much of the +device one long request may take. Deriving it from the exact equal-share CU +budget rather than a power-of-two floor of that share is what keeps a skewed +batch's long request from losing half its headroom at a batch discontinuity: +at 17 sequences on 256 CUs the pow2 form drops from 32 to 16, the exact form to +30. These tests pin that boundary, the 16/64 clamp, and monotonicity, so +restoring the rounding fails here instead of costing 1.2-1.8x on a skewed +batch. + + python -m pytest test/registered/unit/layers/attention/test_vattn_asm_segment_plan.py -v +""" + +import unittest + +from sglang.kernels.ops.attention.vattn_asm_gfx950 import mtp_verify_attn_seg_max +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + +# MI355X. Passed explicitly so the tests are device-independent and run on CPU. +NUM_CU = 256 + +# GQA 16 (TP2) and GQA 8 (TP4) are the only layouts the asm kernel serves. +KV_HEADS = (1, 2) + + +class TestVattnAsmSegmentPlanCap(CustomTestCase): + def test_cap_is_twice_the_exact_uniform_cu_share(self): + cases = [ + (8, 1, 64), # share 32, clamped to 64 + (9, 1, 56), # 28 is not a power of two + (16, 1, 32), # exact divisor: pow2 and exact agree + (17, 1, 30), # the discontinuity; the pow2 form gives 16 + (24, 1, 20), + (29, 1, 16), # share 8, clamped up to 16 + (9, 2, 28), + (14, 2, 18), + (15, 2, 16), + ] + for num_seqs, num_kv_heads, expected in cases: + with self.subTest(num_seqs=num_seqs, num_kv_heads=num_kv_heads): + self.assertEqual( + mtp_verify_attn_seg_max(num_seqs, num_kv_heads, num_cus=NUM_CU), + expected, + ) + + def test_cap_never_increases_with_more_sequences(self): + """A cap that rose with the batch would hand each sequence more of the + device exactly as the device got busier.""" + for num_kv_heads in KV_HEADS: + with self.subTest(num_kv_heads=num_kv_heads): + caps = [ + mtp_verify_attn_seg_max(num_seqs, num_kv_heads, num_cus=NUM_CU) + for num_seqs in range(1, NUM_CU + 2) + ] + self.assertEqual(caps, sorted(caps, reverse=True)) + + def test_cap_stays_within_the_kernel_bounds(self): + """The partial buffers are allocated for the cap, and the assembly + kernel's segment loop is built for at most 64.""" + for num_kv_heads in KV_HEADS: + for num_seqs in range(1, NUM_CU + 2): + cap = mtp_verify_attn_seg_max(num_seqs, num_kv_heads, num_cus=NUM_CU) + with self.subTest(num_seqs=num_seqs, num_kv_heads=num_kv_heads): + self.assertGreaterEqual(cap, 16) + self.assertLessEqual(cap, 64) + + +if __name__ == "__main__": + unittest.main()