From a90e9c5dd8973cdb84af4ff6154c7b4958083b31 Mon Sep 17 00:00:00 2001 From: Niko Ma Date: Mon, 19 Oct 2026 16:37:50 +0800 Subject: [PATCH] [Fix] [AMD] mhc: Bound the Triton fused post+pre buffer cache (#44443) Co-authored-by: inkcherry --- .../amd/deepseek_v4_fused_mhc.py | 5 +++- .../models/test_deepseek_v4_amd_fused_mhc.py | 30 +++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/python/sglang/srt/models/deepseek_common/amd/deepseek_v4_fused_mhc.py b/python/sglang/srt/models/deepseek_common/amd/deepseek_v4_fused_mhc.py index 8ac5d7a9c5..dd86c5bb7f 100644 --- a/python/sglang/srt/models/deepseek_common/amd/deepseek_v4_fused_mhc.py +++ b/python/sglang/srt/models/deepseek_common/amd/deepseek_v4_fused_mhc.py @@ -168,7 +168,10 @@ def _get_fused_hc_post_pre_buffers( num_ksplit, num_tokens, dtype=torch.float32, device=device ), } - _FUSED_HC_POST_PRE_CACHE[key] = bufs + # Cache only small batches: a caller that runs this path for every prefill + # size (gfx1250) would otherwise keep one buffer set per num_tokens forever. + if num_tokens <= _FUSED_HC_POST_PRE_M_THRESHOLD: + _FUSED_HC_POST_PRE_CACHE[key] = bufs return bufs diff --git a/test/registered/unit/models/test_deepseek_v4_amd_fused_mhc.py b/test/registered/unit/models/test_deepseek_v4_amd_fused_mhc.py index 83d84d8f78..f21a0f54b0 100644 --- a/test/registered/unit/models/test_deepseek_v4_amd_fused_mhc.py +++ b/test/registered/unit/models/test_deepseek_v4_amd_fused_mhc.py @@ -1,10 +1,13 @@ import unittest from unittest import mock +import torch + from sglang.srt.environ import envs from sglang.srt.models.deepseek_common.amd import deepseek_v4_fused_mhc from sglang.srt.runtime_context import override_platform from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase register_cpu_ci(est_time=11, suite="base-a-test-cpu") @@ -324,6 +327,33 @@ class TestAmdFusedMhcNormFusedHandling(unittest.TestCase): layer.input_layernorm.assert_called_once_with(fused_hs) +class TestAmdFusedMhcBufferCache(CustomTestCase): + def test_cache_keeps_only_decode_sized_batches(self): + """Only batches up to the M threshold may stay cached; larger ones are + allocated per call, or every distinct prefill size stays alive forever.""" + threshold = deepseek_v4_fused_mhc._FUSED_HC_POST_PRE_M_THRESHOLD + fake_ops = (None, lambda *args, **kwargs: ({"BLOCK_K": 64}, False)) + cache = deepseek_v4_fused_mhc._FUSED_HC_POST_PRE_CACHE + with ( + mock.patch.object( + deepseek_v4_fused_mhc, + "_get_triton_mhc_post_pre_ops", + return_value=fake_ops, + ), + mock.patch.dict(cache, clear=True), + ): + for num_tokens in (1, threshold, threshold + 1, 2048): + bufs = deepseek_v4_fused_mhc._get_fused_hc_post_pre_buffers( + num_tokens=num_tokens, + hidden_size=64, + hc_mult=4, + dtype=torch.bfloat16, + device=torch.device("cpu"), + ) + self.assertIsNotNone(bufs) + self.assertEqual(sorted(key[0] for key in cache), [1, threshold]) + + def _hardware_available() -> bool: try: import torch