minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 13 Packages Projects Releases Wiki Activity

Add real-data KV verification to the KV-canary (#26817)

Browse Source
This commit is contained in:
fzyzcjy
2026-05-31 09:58:32 +08:00
committed by GitHub
parent cdee16e144
commit 0ca610a6df
45 changed files with 2695 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/registered/kv_canary/test_self_unit_buffer_alloc.py
+125 -11
View File
@@ -1,26 +1,140 @@
from __future__ import annotations
import sys
import unittest
import torch
from sglang.jit_kernel.kv_canary.verify import CANARY_SLOT_BYTES
from sglang.srt.kv_canary.pool_patcher.buffer_alloc import alloc_canary_buf
from sglang.jit_kernel.kv_canary.consts import RealKvHashMode
from sglang.srt.kv_canary.config import CanaryConfig, CanaryMode
from sglang.srt.kv_canary.pool_patcher.buffer_alloc import (
make_packed_source,
make_row_source,
resolve_real_kv_read_bytes,
)
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kv_canary.fixtures import DEFAULT_DEVICE
from sglang.test.test_utils import CustomTestCase
register_cuda_ci(est_time=10, stage="extra-a", runner_config="1-gpu-small")
class TestAllocCanaryBuf(CustomTestCase):
def test_alloc_canary_buf_shape_and_dtype(self) -> None:
"""Verify alloc_canary_buf returns a zeroed uint8 buffer of [num_slots, CANARY_SLOT_BYTES]."""
buf = alloc_canary_buf(num_slots=8, device=DEFAULT_DEVICE)
self.assertEqual(buf.shape, (8, CANARY_SLOT_BYTES))
self.assertEqual(buf.dtype, torch.uint8)
self.assertEqual(buf.device.type, DEFAULT_DEVICE.type)
self.assertTrue(torch.equal(buf, torch.zeros_like(buf)))
def _config(mode: RealKvHashMode) -> CanaryConfig:
return CanaryConfig(
mode=CanaryMode.RAISE,
ring_capacity=1024,
sweep_interval=0,
real_kv_hash_mode=mode,
enable_write_input_assert=False,
)
class TestResolveRealKvReadBytes(CustomTestCase):
def test_resolve_real_kv_read_bytes_off_returns_zero(self) -> None:
"""Verify NONE mode disables real KV byte reads."""
self.assertEqual(resolve_real_kv_read_bytes(_config(RealKvHashMode.NONE)), 0)
def test_resolve_real_kv_read_bytes_partial_returns_16(self) -> None:
"""Verify PARTIAL mode reads the fixed byte prefix."""
self.assertEqual(
resolve_real_kv_read_bytes(_config(RealKvHashMode.PARTIAL)), 16
)
def test_resolve_real_kv_read_bytes_all_returns_sentinel_so_full_stride_used(
self,
) -> None:
"""Verify ALL mode requests the full token stride."""
self.assertEqual(
resolve_real_kv_read_bytes(_config(RealKvHashMode.ALL)), sys.maxsize
)
class TestMakeRowSource(CustomTestCase):
def test_make_row_source_large_stride(self) -> None:
"""Verify row sources with 128-byte stride return the requested clip / full stride."""
bytes_per_token = 128
layer_buf = torch.zeros(4, bytes_per_token, dtype=torch.uint8)
cases = [
("partial", 32, 32),
("all", sys.maxsize, bytes_per_token),
]
for label, read_bytes, expected_read in cases:
with self.subTest(label=label):
sources = make_row_source(layer_buffer=layer_buf, read_bytes=read_bytes)
self.assertEqual(len(sources), 1)
self.assertEqual(sources[0].read_bytes, expected_read)
self.assertEqual(sources[0].num_bytes_per_token, bytes_per_token)
def test_make_row_source_small_stride_raises(self) -> None:
"""Verify row sources reject 8-byte strides (cannot satisfy 16-byte aligned loads)."""
layer_buf = torch.zeros(4, 8, dtype=torch.uint8)
for label, read_bytes in [("partial", 32), ("all", sys.maxsize)]:
with self.subTest(label=label):
with self.assertRaisesRegex(ValueError, "num_bytes_per_token"):
make_row_source(layer_buffer=layer_buf, read_bytes=read_bytes)
class TestMakePackedSource(CustomTestCase):
def test_make_packed_source_large_stride(self) -> None:
"""Verify packed sources with 128-byte stride return the requested clip / full stride."""
bytes_per_token = 128
page_size = 2
page_buffer = torch.zeros(4, bytes_per_token * page_size, dtype=torch.uint8)
cases = [
("partial", 32, 32),
("all", sys.maxsize, bytes_per_token),
]
for label, read_bytes, expected_read in cases:
with self.subTest(label=label):
sources = make_packed_source(
page_buffer=page_buffer,
page_size=page_size,
bytes_per_token=bytes_per_token,
read_bytes=read_bytes,
)
self.assertEqual(len(sources), 1)
self.assertEqual(sources[0].read_bytes, expected_read)
self.assertEqual(sources[0].num_bytes_per_token, bytes_per_token)
def test_make_packed_source_small_stride_raises(self) -> None:
"""Verify packed sources reject 8-byte strides (cannot satisfy 16-byte aligned loads)."""
bytes_per_token = 8
page_size = 1
page_buffer = torch.zeros(4, bytes_per_token, dtype=torch.uint8)
for label, read_bytes in [("partial", 32), ("all", sys.maxsize)]:
with self.subTest(label=label):
with self.assertRaisesRegex(ValueError, "num_bytes_per_token"):
make_packed_source(
page_buffer=page_buffer,
page_size=page_size,
bytes_per_token=bytes_per_token,
read_bytes=read_bytes,
)
def test_make_packed_source_unaligned_read_bytes_raises(self) -> None:
"""Verify packed sources reject unaligned explicit reads."""
bytes_per_token = 128
page_size = 1
page_buffer = torch.zeros(4, bytes_per_token, dtype=torch.uint8)
with self.assertRaisesRegex(ValueError, "multiple of 16"):
make_packed_source(
page_buffer=page_buffer,
page_size=page_size,
bytes_per_token=bytes_per_token,
read_bytes=24,
)
def test_make_packed_source_oversized_read_bytes_raises(self) -> None:
"""Verify packed sources reject oversized explicit reads."""
bytes_per_token = 128
page_size = 1
page_buffer = torch.zeros(4, bytes_per_token, dtype=torch.uint8)
with self.assertRaisesRegex(ValueError, "<= num_bytes_per_token"):
make_packed_source(
page_buffer=page_buffer,
page_size=page_size,
bytes_per_token=bytes_per_token,
read_bytes=256,
)
if __name__ == "__main__":
test/registered/kv_canary/test_self_unit_endpoint.py
+10
View File
@@ -6,6 +6,7 @@ from unittest.mock import patch
import torch
from sglang.jit_kernel.kv_canary.consts import RealKvHashMode
from sglang.jit_kernel.kv_canary.verify import (
CANARY_SLOT_BYTES,
CanaryLaunchTag,
@@ -36,6 +37,7 @@ def _make_endpoint(*, device, kernel_kind=CanaryLaunchTag.HEAD_K_FULL, swa_lut=N
kernel_kind=kernel_kind,
canary_buf=canary_buf,
full_to_swa_index_mapping=swa_lut,
real_kv_sources=(),
slot_run_counter_view=slot_view,
kernel_run_counter_view=kernel_view,
enable_chain_position_assert=enable_chain_position_assert,
@@ -56,6 +58,7 @@ def _make_kernel_args(device):
enable_write_input_assert=False,
enable_verify_token_assert=False,
expected_inputs=ExpectedInputs.allocate(capacity=1, device=device),
real_kv_hash_mode=RealKvHashMode.NONE,
)
@@ -82,6 +85,7 @@ class TestSelfUnitEndpoint(CustomTestCase):
ep.launch_sweep(
verify_plan=args.verify_plan,
violation_log=args.violation_log,
real_kv_hash_mode=args.real_kv_hash_mode,
)
self.assertEqual(calls, ["verify"])
@@ -111,6 +115,7 @@ class TestSelfUnitEndpoint(CustomTestCase):
enable_verify_token_assert=args.enable_verify_token_assert,
expected_inputs=args.expected_inputs,
violation_log=args.violation_log,
real_kv_hash_mode=args.real_kv_hash_mode,
)
self.assertIn(("verify", CanaryLaunchTag.TAIL_V_SWA), captured)
self.assertIn(("write", CanaryLaunchTag.TAIL_V_SWA), captured)
@@ -141,10 +146,12 @@ class TestSelfUnitEndpoint(CustomTestCase):
ep_a.launch_sweep(
verify_plan=plan,
violation_log=shared_log,
real_kv_hash_mode=RealKvHashMode.NONE,
)
ep_b.launch_sweep(
verify_plan=plan,
violation_log=shared_log,
real_kv_hash_mode=RealKvHashMode.NONE,
)
self.assertEqual(captured_rings[0], captured_rings[1])
self.assertEqual(captured_rings[0], shared_log.violation_ring.data_ptr())
@@ -183,6 +190,7 @@ class TestSelfUnitEndpoint(CustomTestCase):
enable_verify_token_assert=args.enable_verify_token_assert,
expected_inputs=args.expected_inputs,
violation_log=args.violation_log,
real_kv_hash_mode=args.real_kv_hash_mode,
)
full_ep.launch_per_forward(
verify_plan=args.verify_plan,
@@ -194,6 +202,7 @@ class TestSelfUnitEndpoint(CustomTestCase):
enable_verify_token_assert=args.enable_verify_token_assert,
expected_inputs=args.expected_inputs,
violation_log=args.violation_log,
real_kv_hash_mode=args.real_kv_hash_mode,
)
# SWA call: out_cache_loc was rewritten via lut gather (so identity-shifted by +100 here).
expected_swa = lut[args.out_cache_loc]
@@ -235,6 +244,7 @@ class TestSelfUnitEndpoint(CustomTestCase):
enable_verify_token_assert=args.enable_verify_token_assert,
expected_inputs=args.expected_inputs,
violation_log=args.violation_log,
real_kv_hash_mode=args.real_kv_hash_mode,
)
self.assertTrue(
test/registered/kv_canary/test_self_unit_perturb.py
+105
View File
@@ -2,14 +2,26 @@ from __future__ import annotations
import os
import unittest
from typing import cast
from unittest.mock import patch
import torch
from sglang.jit_kernel.kv_canary.verify import RealKvSource
from sglang.srt.kv_canary.buffer_group import PoolKind
from sglang.srt.kv_canary.perturb.config import (
PerturbConfig,
TargetGroupKind,
_parse_target_group_kind,
)
from sglang.srt.kv_canary.perturb.utils import (
flip_first_byte_in_source,
pick_target_group,
)
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kv_canary.fixtures import (
make_buffer_group,
)
from sglang.test.test_utils import CustomTestCase
register_cuda_ci(est_time=10, stage="extra-a", runner_config="1-gpu-small")
@@ -55,5 +67,98 @@ class TestParseTargetGroupKind(CustomTestCase):
self.assertIsNone(config.target_group_kind)
class TestPickTargetGroup(CustomTestCase):
def test_pick_target_group_filters_exact_kind(self) -> None:
"""Verify target group selection returns only the requested pool kind."""
cases = [
(TargetGroupKind.FULL, PoolKind.FULL),
(TargetGroupKind.SWA, PoolKind.SWA),
]
for target_kind, expected_kind in cases:
with self.subTest(target_kind=target_kind):
full_group = make_buffer_group(kind=PoolKind.FULL, has_real_kv=True)
swa_group = make_buffer_group(kind=PoolKind.SWA, has_real_kv=True)
group = pick_target_group(
buffer_groups=(full_group, swa_group),
target_kind=target_kind,
)
self.assertIsNotNone(group)
self.assertEqual(group.kind, expected_kind)
def test_pick_target_group_rejects_unsupported_kind(self) -> None:
"""Verify target group selection rejects unsupported enum values."""
full_group = make_buffer_group(kind=PoolKind.FULL, has_real_kv=True)
with self.assertRaisesRegex(ValueError, "Unsupported target_group_kind"):
pick_target_group(
buffer_groups=(full_group,),
target_kind=cast(TargetGroupKind, 2),
)
def test_pick_target_group_ignores_groups_without_real_kv_sources(self) -> None:
"""Verify target group selection skips groups without real KV sources."""
full_group = make_buffer_group(kind=PoolKind.FULL, has_real_kv=False)
swa_group = make_buffer_group(kind=PoolKind.SWA, has_real_kv=True)
group = pick_target_group(
buffer_groups=(full_group, swa_group),
target_kind=TargetGroupKind.FULL,
)
self.assertIsNone(group)
class TestPerturbWarmupAndUtils(CustomTestCase):
def test_flip_first_byte_in_source_maps_swa_logical_slot_through_lut(
self,
) -> None:
"""Verify SWA groups map logical slots through swa_index_lut before flipping bytes."""
source = RealKvSource(
tensor=torch.arange(64, dtype=torch.uint8).view(2, 32),
page_size=2,
num_bytes_per_token=16,
read_bytes=16,
)
group = make_buffer_group(
kind=PoolKind.SWA,
has_real_kv=True,
real_kv_source=source,
swa_index_lut=torch.tensor([0, 3], dtype=torch.int32),
)
snapshot = source.tensor.clone()
result = flip_first_byte_in_source(group=group, source=source, slot_idx=1)
self.assertEqual(result, (1, 16, int(snapshot[1, 16].item())))
expected = snapshot.clone()
expected[1, 16] = int(snapshot[1, 16].item()) ^ 0xFF
self.assertTrue(torch.equal(source.tensor, expected))
class TestPerturbUtils(CustomTestCase):
def test_flip_first_byte_in_physical_swa_slot_does_not_translate_twice(
self,
) -> None:
"""Verify a physical SWA slot selected from sweep is not LUT-translated again."""
group = make_buffer_group(kind=PoolKind.SWA, has_real_kv=True)
source = group.real_kv_sources_k[0]
source.tensor[2, 0] = 0x12
source.tensor[3, 0] = 0x34
result = flip_first_byte_in_source(
group=group,
source=source,
slot_idx=2,
slot_is_physical=True,
)
self.assertEqual(result, (2, 0, 0x12))
self.assertEqual(int(source.tensor[2, 0].item()), 0xED)
self.assertEqual(int(source.tensor[3, 0].item()), 0x34)
if __name__ == "__main__":
unittest.main()
test/registered/kv_canary/test_self_unit_pool_patcher.py
+52 -1
View File
@@ -2,7 +2,17 @@ from __future__ import annotations
import unittest
from sglang.jit_kernel.kv_canary.verify import CANARY_SLOT_BYTES
import torch
from sglang.jit_kernel.kv_canary.consts import MAX_REAL_KV_SOURCES, RealKvHashMode
from sglang.jit_kernel.kv_canary.verify import (
CANARY_SLOT_BYTES,
CanaryLaunchTag,
RealKvSource,
VerifyOrWriteContext,
VerifyPlan,
launch_canary_verify_kernel,
)
from sglang.srt.kv_canary.buffer_group import PoolKind
from sglang.srt.kv_canary.pool_patcher.api import attach_canary_buffers
from sglang.test.ci.ci_register import register_cuda_ci
@@ -59,6 +69,47 @@ class TestAttachCanaryBuffers(PoolPatcherHelper, CustomTestCase):
self.assertIsNone(groups[PoolKind.FULL].swa_index_lut)
class TestRealKvSources(PoolPatcherHelper, CustomTestCase):
def test_real_kv_sources_above_4_raises(self):
"""Verify too many real KV sources are rejected."""
tensor = torch.zeros(4, 16, dtype=torch.uint8, device=self.device)
sources = tuple(
RealKvSource(
tensor=tensor, page_size=1, num_bytes_per_token=16, read_bytes=16
)
for _ in range(MAX_REAL_KV_SOURCES + 1)
)
canary_buf = torch.zeros(
4, CANARY_SLOT_BYTES, dtype=torch.uint8, device=self.device
)
plan = VerifyPlan.allocate(verify_capacity=1, device=self.device)
violation_ring = torch.zeros(2, 8, dtype=torch.int64, device=self.device)
violation_write_index = torch.zeros(1, dtype=torch.int32, device=self.device)
slot_run_counter = torch.zeros(1, dtype=torch.int64, device=self.device)
kernel_run_counter = torch.zeros(1, dtype=torch.int64, device=self.device)
enable_chain_position_assert = torch.ones(
1, dtype=torch.int32, device=self.device
)
with self.assertRaises(ValueError):
launch_canary_verify_kernel(
context=VerifyOrWriteContext(
canary_buf=canary_buf,
kernel_kind=CanaryLaunchTag.HEAD_K_FULL,
violation_ring=violation_ring,
violation_write_index=violation_write_index,
slot_run_counter=slot_run_counter,
kernel_run_counter=kernel_run_counter,
real_kv_sources=sources,
real_kv_hash_mode=RealKvHashMode.NONE,
enable_chain_position_assert=enable_chain_position_assert,
),
plan=plan,
check_verify_expected_token=True,
)
class TestPoolPatcherBufferInfos(PoolPatcherHelper, CustomTestCase):
def test_get_contiguous_buf_infos_inserts_canary_entries(self):
"""Verify contiguous buffer metadata includes canary entries after patching."""
test/registered/kv_canary/test_self_unit_runner_per_forward.py
+5
View File
@@ -5,6 +5,7 @@ from unittest.mock import patch
import torch
from sglang.jit_kernel.kv_canary.consts import RealKvHashMode
from sglang.jit_kernel.kv_canary.verify import CanaryLaunchTag, VerifyPlan
from sglang.jit_kernel.kv_canary.write import WritePlan
from sglang.srt.kv_canary import endpoint as endpoint_module
@@ -96,6 +97,7 @@ class TestLaunchEndpointsPerForward(CanaryManagerTestCase):
forward_batch=forward_batch,
expected_inputs=ExpectedInputs.allocate(capacity=3, device=self.device),
violation_log=ViolationLog.allocate(ring_capacity=2, device=self.device),
real_kv_hash_mode=RealKvHashMode.NONE,
enable_write_input_assert=False,
enable_verify_token_assert=False,
)
@@ -148,6 +150,7 @@ class TestLaunchEndpointsPerForward(CanaryManagerTestCase):
forward_batch=forward_batch,
expected_inputs=ExpectedInputs.allocate(capacity=1, device=self.device),
violation_log=ViolationLog.allocate(ring_capacity=2, device=self.device),
real_kv_hash_mode=RealKvHashMode.NONE,
enable_write_input_assert=False,
enable_verify_token_assert=False,
)
@@ -185,6 +188,7 @@ class TestLaunchEndpointsPerForward(CanaryManagerTestCase):
forward_batch=forward_batch,
expected_inputs=ExpectedInputs.allocate(capacity=3, device=self.device),
violation_log=ViolationLog.allocate(ring_capacity=2, device=self.device),
real_kv_hash_mode=RealKvHashMode.NONE,
enable_write_input_assert=False,
enable_verify_token_assert=True,
)
@@ -220,6 +224,7 @@ class TestLaunchEndpointsPerForward(CanaryManagerTestCase):
forward_batch=forward_batch,
expected_inputs=ExpectedInputs.allocate(capacity=1, device=self.device),
violation_log=ViolationLog.allocate(ring_capacity=2, device=self.device),
real_kv_hash_mode=RealKvHashMode.NONE,
enable_write_input_assert=False,
enable_verify_token_assert=False,
)
test/registered/kv_canary/test_self_unit_violation.py
+43 -11
View File
@@ -53,6 +53,7 @@ class TestViolationReporter(CustomTestCase):
fail_reason_bits=int(
FailReason.VERIFY_CHAIN_HASH_MISMATCH
| FailReason.VERIFY_POSITION_MISMATCH
| FailReason.VERIFY_REAL_KV_HASH_MISMATCH
),
)
out = _format_violation(
@@ -60,12 +61,12 @@ class TestViolationReporter(CustomTestCase):
)
self.assertEqual(
out,
"kv_canary violation: launch_tag=HEAD_K_FULL fail_reason=verify_chain_hash+verify_position "
"kv_canary violation: launch_tag=HEAD_K_FULL fail_reason=verify_chain_hash+verify_position+verify_real_kv_hash "
"slot_idx=17 position=42 stored_token=111 expected_token=0 stored_chain_hash=0x1111111111111111 "
"expected_aux=0x2222222222222222\n"
"KV cache canary violation detected (kernel_kind=HEAD_K_FULL, slot_idx=17, position=42)\n"
"canary_kind: per_forward_head_k_full\n"
" fail_reasons: verify_chain_hash verify_position\n"
" fail_reasons: verify_chain_hash verify_position verify_real_kv_hash\n"
" stored: token_id=111 position=42 prev_hash=0x1111111111111111\n"
" expected: prev_hash=0x2222222222222222\n"
" total_violations=1 ring_overflow=False step_when_pumped=7",
@@ -199,7 +200,7 @@ class TestLogOrRaiseViolation(CustomTestCase):
self.assertFalse(reporter.is_raised)
def test_log_mode_emits_one_warning_per_violation(self) -> None:
"""Log mode with 2 valid rows emits 2 warnings, each a full _format_violation snapshot for that row."""
"""Log mode with 3 valid rows emits 3 warnings, each a full _format_violation snapshot for that row."""
rows = [
_make_row(
slot_idx=11,
@@ -211,14 +212,19 @@ class TestLogOrRaiseViolation(CustomTestCase):
position=202,
fail_reason_bits=int(FailReason.VERIFY_POSITION_MISMATCH),
),
_make_row(
slot_idx=33,
position=303,
fail_reason_bits=int(FailReason.VERIFY_REAL_KV_HASH_MISMATCH),
),
]
reporter = _make_reporter(
rows=rows, write_index=2, ring_capacity=4, mode=CanaryMode.LOG
rows=rows, write_index=3, ring_capacity=4, mode=CanaryMode.LOG
)
with patch.object(violation_reporter_module.logger, "warning") as mock_warning:
reporter.log_or_raise_violation(outer_step_counter=7)
self.assertEqual(mock_warning.call_count, 2)
self.assertEqual(mock_warning.call_count, 3)
messages: list[str] = [call.args[0] for call in mock_warning.call_args_list]
self.assertEqual(
messages[0],
@@ -230,7 +236,7 @@ class TestLogOrRaiseViolation(CustomTestCase):
" fail_reasons: verify_chain_hash\n"
" stored: token_id=111 position=101 prev_hash=0x0000000000000000\n"
" expected: prev_hash=0x0000000000000000\n"
" total_violations=2 ring_overflow=False step_when_pumped=7",
" total_violations=3 ring_overflow=False step_when_pumped=7",
)
self.assertEqual(
messages[1],
@@ -242,12 +248,24 @@ class TestLogOrRaiseViolation(CustomTestCase):
" fail_reasons: verify_position\n"
" stored: token_id=111 position=202 prev_hash=0x0000000000000000\n"
" expected: prev_hash=0x0000000000000000\n"
" total_violations=2 ring_overflow=False step_when_pumped=7",
" total_violations=3 ring_overflow=False step_when_pumped=7",
)
self.assertEqual(
messages[2],
"kv_canary violation: launch_tag=HEAD_K_FULL fail_reason=verify_real_kv_hash slot_idx=33 position=303 "
"stored_token=111 expected_token=0 stored_chain_hash=0x0000000000000000 "
"expected_aux=0x0000000000000000\n"
"KV cache canary violation detected (kernel_kind=HEAD_K_FULL, slot_idx=33, position=303)\n"
"canary_kind: per_forward_head_k_full\n"
" fail_reasons: verify_real_kv_hash\n"
" stored: token_id=111 position=303 prev_hash=0x0000000000000000\n"
" expected: prev_hash=0x0000000000000000\n"
" total_violations=3 ring_overflow=False step_when_pumped=7",
)
self.assertFalse(reporter.is_raised)
def test_raise_mode_raises_one_error_containing_all_violations(self) -> None:
"""Raise mode raises a single RuntimeError whose text is the 2 formatted rows joined with single newlines."""
"""Raise mode raises a single RuntimeError whose text is the 3 formatted rows joined with single newlines."""
rows = [
_make_row(
slot_idx=11,
@@ -259,9 +277,14 @@ class TestLogOrRaiseViolation(CustomTestCase):
position=202,
fail_reason_bits=int(FailReason.VERIFY_POSITION_MISMATCH),
),
_make_row(
slot_idx=33,
position=303,
fail_reason_bits=int(FailReason.VERIFY_REAL_KV_HASH_MISMATCH),
),
]
reporter = _make_reporter(
rows=rows, write_index=2, ring_capacity=4, mode=CanaryMode.RAISE
rows=rows, write_index=3, ring_capacity=4, mode=CanaryMode.RAISE
)
with self.assertRaises(RuntimeError) as ctx:
reporter.log_or_raise_violation(outer_step_counter=5)
@@ -276,7 +299,7 @@ class TestLogOrRaiseViolation(CustomTestCase):
" fail_reasons: verify_chain_hash\n"
" stored: token_id=111 position=101 prev_hash=0x0000000000000000\n"
" expected: prev_hash=0x0000000000000000\n"
" total_violations=2 ring_overflow=False step_when_pumped=5\n"
" total_violations=3 ring_overflow=False step_when_pumped=5\n"
"kv_canary violation: launch_tag=HEAD_K_FULL fail_reason=verify_position slot_idx=22 position=202 "
"stored_token=111 expected_token=0 stored_chain_hash=0x0000000000000000 "
"expected_aux=0x0000000000000000\n"
@@ -285,7 +308,16 @@ class TestLogOrRaiseViolation(CustomTestCase):
" fail_reasons: verify_position\n"
" stored: token_id=111 position=202 prev_hash=0x0000000000000000\n"
" expected: prev_hash=0x0000000000000000\n"
" total_violations=2 ring_overflow=False step_when_pumped=5",
" total_violations=3 ring_overflow=False step_when_pumped=5\n"
"kv_canary violation: launch_tag=HEAD_K_FULL fail_reason=verify_real_kv_hash slot_idx=33 position=303 "
"stored_token=111 expected_token=0 stored_chain_hash=0x0000000000000000 "
"expected_aux=0x0000000000000000\n"
"KV cache canary violation detected (kernel_kind=HEAD_K_FULL, slot_idx=33, position=303)\n"
"canary_kind: per_forward_head_k_full\n"
" fail_reasons: verify_real_kv_hash\n"
" stored: token_id=111 position=303 prev_hash=0x0000000000000000\n"
" expected: prev_hash=0x0000000000000000\n"
" total_violations=3 ring_overflow=False step_when_pumped=5",
)
self.assertTrue(reporter.is_raised)