[Fix] Reject online weight updates while the HPC-Ops router GEMM split cache is active (#31943)

Co-authored-by: Halcyon <56064364+VAthree@users.noreply.github.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Xiaoyu Zhang
2026-07-24 19:30:13 +08:00
committed by GitHub
co-authored by Halcyon Claude Fable 5
parent a31542ebd9
commit 841fa293b5
4 changed files with 122 additions and 2 deletions
@@ -13,7 +13,9 @@ import torch
from sglang.kernels.ops.attention.dsv4.gemm import (
_hpc_gemm_bf16xfp32_available,
_linear_bf16_fp32_hpc,
hpc_bf16xfp32_gemm_enabled,
linear_bf16_fp32,
mark_hpc_bf16xfp32_gemm_enabled,
)
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.test_utils import CustomTestCase
@@ -32,6 +34,7 @@ class TestLinearBf16Fp32Hpc(CustomTestCase):
@classmethod
def setUpClass(cls):
mark_hpc_bf16xfp32_gemm_enabled()
torch.manual_seed(0)
def test_matches_fp32_reference(self):
@@ -70,6 +73,48 @@ class TestLinearBf16Fp32Hpc(CustomTestCase):
# The kernel leaves the cached split-K workspace zeroed.
self.assertTrue((cache[3] == 0).all().item())
def test_online_weight_updates_rejected_when_enabled(self):
from sglang.srt.model_executor.model_runner_components.weight_updater import (
_unsupported_derived_weight_cache_error,
)
self.assertTrue(hpc_bf16xfp32_gemm_enabled())
self.assertIsNotNone(_unsupported_derived_weight_cache_error())
def test_split_buffers_stable_for_cuda_graph(self):
# Captured graphs replay the split buffers by address; an in-place
# weight write must never reallocate them.
k, n = _ROUTER_SHAPES[1]
x = torch.randn(16, k, dtype=torch.bfloat16, device="cuda")
w = torch.randn(n, k, dtype=torch.float32, device="cuda")
w_orig = w.clone()
_linear_bf16_fp32_hpc(x, w) # populate the cache outside the graph
cache_before = getattr(w, "_sglang_bf16xfp32_weight_cache")
graph = torch.cuda.CUDAGraph()
stream = torch.cuda.Stream()
stream.wait_stream(torch.cuda.current_stream())
with torch.cuda.stream(stream):
_linear_bf16_fp32_hpc(x, w) # warmup on the side stream
torch.cuda.current_stream().wait_stream(stream)
with torch.cuda.graph(graph):
out = _linear_bf16_fp32_hpc(x, w)
graph.replay()
torch.testing.assert_close(
out, torch.mm(x.float(), w_orig.t()), rtol=0.08, atol=0.01
)
w.data.copy_(torch.randn_like(w))
out_eager = _linear_bf16_fp32_hpc(x, w)
cache_after = getattr(w, "_sglang_bf16xfp32_weight_cache")
self.assertIs(cache_before, cache_after)
graph.replay()
torch.testing.assert_close(
out, torch.mm(x.float(), w_orig.t()), rtol=0.08, atol=0.01
)
torch.testing.assert_close(out_eager, out)
if __name__ == "__main__":
unittest.main()