[Fix] Reject online weight updates while the HPC-Ops router GEMM split cache is active (#31943)
Co-authored-by: Halcyon <56064364+VAthree@users.noreply.github.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Halcyon
Claude Fable 5
parent
a31542ebd9
commit
841fa293b5
@@ -13,7 +13,9 @@ import torch
|
||||
from sglang.kernels.ops.attention.dsv4.gemm import (
|
||||
_hpc_gemm_bf16xfp32_available,
|
||||
_linear_bf16_fp32_hpc,
|
||||
hpc_bf16xfp32_gemm_enabled,
|
||||
linear_bf16_fp32,
|
||||
mark_hpc_bf16xfp32_gemm_enabled,
|
||||
)
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
@@ -32,6 +34,7 @@ class TestLinearBf16Fp32Hpc(CustomTestCase):
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
mark_hpc_bf16xfp32_gemm_enabled()
|
||||
torch.manual_seed(0)
|
||||
|
||||
def test_matches_fp32_reference(self):
|
||||
@@ -70,6 +73,48 @@ class TestLinearBf16Fp32Hpc(CustomTestCase):
|
||||
# The kernel leaves the cached split-K workspace zeroed.
|
||||
self.assertTrue((cache[3] == 0).all().item())
|
||||
|
||||
def test_online_weight_updates_rejected_when_enabled(self):
|
||||
from sglang.srt.model_executor.model_runner_components.weight_updater import (
|
||||
_unsupported_derived_weight_cache_error,
|
||||
)
|
||||
|
||||
self.assertTrue(hpc_bf16xfp32_gemm_enabled())
|
||||
self.assertIsNotNone(_unsupported_derived_weight_cache_error())
|
||||
|
||||
def test_split_buffers_stable_for_cuda_graph(self):
|
||||
# Captured graphs replay the split buffers by address; an in-place
|
||||
# weight write must never reallocate them.
|
||||
k, n = _ROUTER_SHAPES[1]
|
||||
x = torch.randn(16, k, dtype=torch.bfloat16, device="cuda")
|
||||
w = torch.randn(n, k, dtype=torch.float32, device="cuda")
|
||||
w_orig = w.clone()
|
||||
_linear_bf16_fp32_hpc(x, w) # populate the cache outside the graph
|
||||
cache_before = getattr(w, "_sglang_bf16xfp32_weight_cache")
|
||||
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
stream = torch.cuda.Stream()
|
||||
stream.wait_stream(torch.cuda.current_stream())
|
||||
with torch.cuda.stream(stream):
|
||||
_linear_bf16_fp32_hpc(x, w) # warmup on the side stream
|
||||
torch.cuda.current_stream().wait_stream(stream)
|
||||
with torch.cuda.graph(graph):
|
||||
out = _linear_bf16_fp32_hpc(x, w)
|
||||
|
||||
graph.replay()
|
||||
torch.testing.assert_close(
|
||||
out, torch.mm(x.float(), w_orig.t()), rtol=0.08, atol=0.01
|
||||
)
|
||||
|
||||
w.data.copy_(torch.randn_like(w))
|
||||
out_eager = _linear_bf16_fp32_hpc(x, w)
|
||||
cache_after = getattr(w, "_sglang_bf16xfp32_weight_cache")
|
||||
self.assertIs(cache_before, cache_after)
|
||||
graph.replay()
|
||||
torch.testing.assert_close(
|
||||
out, torch.mm(x.float(), w_orig.t()), rtol=0.08, atol=0.01
|
||||
)
|
||||
torch.testing.assert_close(out_eager, out)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user