[PD] Support --enable-unified-memory with PD disaggregation (kimi-linear MLA hybrid-Mamba) (#33362)
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Cheng Wan
co-authored by
Claude Fable 5
Claude Fable 5
parent
8a7c8a72d6
commit
ceeaec2078
@@ -0,0 +1,152 @@
|
||||
"""PD disaggregation with --enable-unified-memory (MLA hybrid-Mamba).
|
||||
|
||||
Guards the unified-memory PD transfer scheme end to end: whole page-envelope
|
||||
KV registration (`UnifiedMLATokenToKVPool.get_contiguous_buf_infos`), whole
|
||||
slot-envelope KDA/mamba state transfer, virtual->physical index translation at
|
||||
the prefill send / decode prealloc sites, and the compaction move gate. A
|
||||
regression in any of them shifts the decode-side KV/state bytes and breaks
|
||||
logprob parity with the non-PD unified-memory reference.
|
||||
|
||||
`--attention-backend` is deliberately NOT pinned, matching
|
||||
`models_e2e/test_kimi_linear_unified_memory.py`, which documents that pinning
|
||||
hides defects reachable only under the resolved default. The transferred bytes
|
||||
are backend-independent, so the default (fa3 on this suite's H100 runner) covers
|
||||
this file's subject either way. The linear-attn/Mamba backends stay pinned to
|
||||
triton -- the page-major layout requires them.
|
||||
|
||||
`--enable-deterministic-inference` is deliberately NOT set. It would only guard
|
||||
against batch-shape-dependent kernel variation, and the reference and P+D paths
|
||||
run the same shapes: measured, two fresh servers on separate GPUs produce
|
||||
bit-identical logits without it. Setting it would narrow the test to the
|
||||
batch-invariant op set and a non-default sampling backend -- a less
|
||||
representative config -- and couple a PD-transfer test to the deterministic code
|
||||
path, so a defect there would fail this file for an unrelated reason.
|
||||
"""
|
||||
|
||||
import time
|
||||
import unittest
|
||||
|
||||
import requests
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.server_fixtures.disaggregation_fixture import (
|
||||
PDDisaggregationServerBase,
|
||||
assert_process_healthy,
|
||||
)
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=900, stage="base-c", runner_config="4-gpu-h100")
|
||||
|
||||
KIMI_LINEAR_MODEL = "yujiepan/kimi-linear-tiny-random"
|
||||
SERVER_ENV = {"SGLANG_BATCH_INVARIANT_OPS_ENABLE_MM_DEEPGEMM": "0"}
|
||||
SERVER_ARGS = [
|
||||
"--skip-tokenizer-init",
|
||||
"--random-seed",
|
||||
"1",
|
||||
"--enable-unified-memory",
|
||||
"--linear-attn-backend",
|
||||
"triton",
|
||||
"--mamba-backend",
|
||||
"triton",
|
||||
"--max-mamba-cache-size",
|
||||
"32",
|
||||
"--max-total-tokens",
|
||||
"4096",
|
||||
"--cuda-graph-backend-decode",
|
||||
"disabled",
|
||||
"--cuda-graph-backend-prefill",
|
||||
"disabled",
|
||||
]
|
||||
|
||||
|
||||
class TestUnifiedMemoryDisaggregation(PDDisaggregationServerBase):
|
||||
"""1 prefill + 1 decode, both with --enable-unified-memory, vs a non-PD
|
||||
unified-memory reference server."""
|
||||
|
||||
prefill_tp_size = 1
|
||||
decode_tp_size = 1
|
||||
decode_base_gpu_id = 1
|
||||
extra_prefill_args = SERVER_ARGS
|
||||
extra_decode_args = SERVER_ARGS
|
||||
extra_prefill_env = SERVER_ENV
|
||||
extra_decode_env = SERVER_ENV
|
||||
baseline_args = SERVER_ARGS
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
super().setUpClass()
|
||||
cls.model = KIMI_LINEAR_MODEL
|
||||
|
||||
@staticmethod
|
||||
def generate(base_url):
|
||||
response = requests.post(
|
||||
base_url + "/generate",
|
||||
json={
|
||||
"input_ids": [1] + [100 + i % 1000 for i in range(256)],
|
||||
"sampling_params": {
|
||||
"temperature": 0,
|
||||
"max_new_tokens": 4,
|
||||
"ignore_eos": True,
|
||||
},
|
||||
"return_logprob": True,
|
||||
"top_logprobs_num": 5,
|
||||
},
|
||||
timeout=120,
|
||||
)
|
||||
response.raise_for_status()
|
||||
return response.json()["meta_info"]
|
||||
|
||||
def test_logprob_parity(self):
|
||||
baseline = popen_launch_server(
|
||||
self.model,
|
||||
self.lb_url,
|
||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
other_args=["--trust-remote-code"] + self.baseline_args,
|
||||
env=SERVER_ENV,
|
||||
)
|
||||
try:
|
||||
reference = self.generate(self.lb_url)
|
||||
finally:
|
||||
kill_process_tree(baseline.pid, wait_timeout=60)
|
||||
time.sleep(5)
|
||||
|
||||
self.launch_all()
|
||||
disaggregated = self.generate(self.lb_url)
|
||||
|
||||
reference_logprobs = reference["output_token_logprobs"]
|
||||
disaggregated_logprobs = disaggregated["output_token_logprobs"]
|
||||
self.assertEqual(
|
||||
[item[1] for item in reference_logprobs],
|
||||
[item[1] for item in disaggregated_logprobs],
|
||||
)
|
||||
self.assertEqual(len(reference_logprobs), 4)
|
||||
for reference_item, disaggregated_item in zip(
|
||||
reference_logprobs, disaggregated_logprobs
|
||||
):
|
||||
self.assertAlmostEqual(reference_item[0], disaggregated_item[0], delta=0.05)
|
||||
|
||||
assert_process_healthy(self, "load balancer", self.process_lb, self.lb_url)
|
||||
assert_process_healthy(self, "prefill", self.process_prefill, self.prefill_url)
|
||||
assert_process_healthy(self, "decode", self.process_decode, self.decode_url)
|
||||
|
||||
|
||||
class TestUnifiedMemoryDisaggregationChunkedPrefill(TestUnifiedMemoryDisaggregation):
|
||||
"""Multi-chunk prefill (257-token prompt, 64-token chunks): each chunk's KV
|
||||
pages are translated to physical ids and shipped while later chunks still
|
||||
run, exercising the chunked send path and the prefill-side move gate
|
||||
(`chunked_req.start_send_idx > 0`). The reference server uses the same
|
||||
chunk size so any parity break isolates to the PD transfer.
|
||||
"""
|
||||
|
||||
_chunked_args = SERVER_ARGS + ["--chunked-prefill-size", "64"]
|
||||
extra_prefill_args = _chunked_args
|
||||
extra_decode_args = _chunked_args
|
||||
baseline_args = _chunked_args
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,179 @@
|
||||
"""Regression tests for the unified-memory PD compaction move gate.
|
||||
|
||||
The gate decides when lazy compaction may relocate physical pages. A page is
|
||||
exposed to the peer from the moment its address is published until the transfer
|
||||
concludes, and for part of that lifetime the request sits in NEITHER end's
|
||||
queue. Both cases below are exactly those windows: an earlier version of the
|
||||
predicates looked only at `disagg_prefill_inflight_queue` /
|
||||
`disagg_decode_transfer_queue` (plus `scheduler.chunked_req`) and returned True
|
||||
here, letting compaction move pages under in-flight RDMA -- silent KV
|
||||
corruption with no crash.
|
||||
"""
|
||||
|
||||
import unittest
|
||||
from typing import List, Optional, Set
|
||||
|
||||
from sglang.srt.disaggregation.utils import (
|
||||
DisaggregationMode,
|
||||
unified_memory_disagg_move_gate,
|
||||
)
|
||||
from sglang.srt.mem_cache.multi_ended_allocator import MultiEndedAllocator
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cpu_ci(est_time=30, suite="base-a-test-cpu")
|
||||
|
||||
|
||||
class _FakeTransferQueue:
|
||||
def __init__(self):
|
||||
self.queue: List[object] = []
|
||||
|
||||
|
||||
class _FakePreallocQueue:
|
||||
"""Mirrors the real queue's published-destination bookkeeping."""
|
||||
|
||||
def __init__(self):
|
||||
self._num_published_destinations = 0
|
||||
|
||||
@property
|
||||
def has_published_destinations(self) -> bool:
|
||||
return self._num_published_destinations > 0
|
||||
|
||||
def note_destinations_published(self) -> None:
|
||||
self._num_published_destinations += 1
|
||||
|
||||
def note_destinations_queued(self, count: int) -> None:
|
||||
self._num_published_destinations = max(
|
||||
0, self._num_published_destinations - count
|
||||
)
|
||||
|
||||
|
||||
class _FakeScheduler:
|
||||
def __init__(self, mode: DisaggregationMode):
|
||||
self.disaggregation_mode = mode
|
||||
self.chunked_req: Optional[object] = None
|
||||
self.disagg_prefill_inflight_queue: List[object] = []
|
||||
self.disagg_prefill_pending_chunk_rids: Set[str] = set()
|
||||
self.disagg_decode_transfer_queue = _FakeTransferQueue()
|
||||
self.disagg_decode_prealloc_queue = _FakePreallocQueue()
|
||||
|
||||
|
||||
class TestDecodeMoveGate(CustomTestCase):
|
||||
def test_closed_while_destination_published_but_not_queued(self):
|
||||
"""`pop_preallocated` publishes request A's destination addresses via
|
||||
`send_metadata`, then keeps allocating for request B in the same loop;
|
||||
the batch only reaches the transfer queue after the loop returns. B's
|
||||
allocation can urgently flush the peer sub-allocator, so the gate must
|
||||
stay closed across that window even though the transfer queue is empty.
|
||||
"""
|
||||
scheduler = _FakeScheduler(DisaggregationMode.DECODE)
|
||||
gate = unified_memory_disagg_move_gate(scheduler)
|
||||
self.assertTrue(gate(), "idle decode node should allow compaction")
|
||||
|
||||
# A's destination is now visible to prefill; transfer queue still empty.
|
||||
scheduler.disagg_decode_prealloc_queue.note_destinations_published()
|
||||
self.assertFalse(scheduler.disagg_decode_transfer_queue.queue)
|
||||
self.assertFalse(gate())
|
||||
|
||||
# Handing the batch to the transfer queue transfers responsibility.
|
||||
scheduler.disagg_decode_transfer_queue.queue.append(object())
|
||||
scheduler.disagg_decode_prealloc_queue.note_destinations_queued(1)
|
||||
self.assertFalse(gate(), "transfer queue still holds it")
|
||||
|
||||
scheduler.disagg_decode_transfer_queue.queue.clear()
|
||||
self.assertTrue(gate())
|
||||
|
||||
|
||||
class TestPrefillMoveGate(CustomTestCase):
|
||||
def test_closed_after_final_chunk_clears_chunked_req(self):
|
||||
"""Scheduling the final chunk clears `scheduler.chunked_req`, but the
|
||||
request only reaches `disagg_prefill_inflight_queue` later in the result
|
||||
path. Earlier middle chunks may still be draining in that window, so the
|
||||
gate must not key off `chunked_req` alone.
|
||||
"""
|
||||
scheduler = _FakeScheduler(DisaggregationMode.PREFILL)
|
||||
gate = unified_memory_disagg_move_gate(scheduler)
|
||||
self.assertTrue(gate(), "idle prefill node should allow compaction")
|
||||
|
||||
# A middle chunk went out for rid "r0".
|
||||
scheduler.chunked_req = object()
|
||||
scheduler.disagg_prefill_pending_chunk_rids.add("r0")
|
||||
self.assertFalse(gate())
|
||||
|
||||
# Final chunk scheduled: chunked_req cleared, not yet inflight-queued.
|
||||
scheduler.chunked_req = None
|
||||
self.assertFalse(scheduler.disagg_prefill_inflight_queue)
|
||||
self.assertFalse(gate())
|
||||
|
||||
# Last chunk sent: the request is on the inflight queue, which covers it.
|
||||
scheduler.disagg_prefill_inflight_queue.append(object())
|
||||
scheduler.disagg_prefill_pending_chunk_rids.discard("r0")
|
||||
self.assertFalse(gate())
|
||||
|
||||
scheduler.disagg_prefill_inflight_queue.clear()
|
||||
self.assertTrue(gate())
|
||||
|
||||
def test_reopens_when_middle_sent_request_is_retired_without_final_chunk(self):
|
||||
"""A request aborted after a middle chunk never reaches a `last_chunk`
|
||||
send, so its rid is only dropped by the abort/release cleanup. Without
|
||||
that discard the gate stays closed for the process lifetime and lazy
|
||||
compaction never packs the free list again -- a liveness leak that ends
|
||||
in allocation failure despite reclaimable space.
|
||||
"""
|
||||
scheduler = _FakeScheduler(DisaggregationMode.PREFILL)
|
||||
gate = unified_memory_disagg_move_gate(scheduler)
|
||||
|
||||
scheduler.chunked_req = object()
|
||||
scheduler.disagg_prefill_pending_chunk_rids.add("r0")
|
||||
self.assertFalse(gate())
|
||||
|
||||
# Aborted mid-chunking: chunked_req dropped, no final send, never queued.
|
||||
scheduler.chunked_req = None
|
||||
scheduler.disagg_prefill_pending_chunk_rids.discard("r0")
|
||||
self.assertTrue(gate(), "abort cleanup must let compaction resume")
|
||||
|
||||
|
||||
class TestGatedPeerHolesAreNotSchedulable(CustomTestCase):
|
||||
"""`schedulable_available_size` credits holes a peer urgent-flush would
|
||||
release. While the move gate is closed that flush relocates nothing, so
|
||||
crediting them lets the scheduler admit work `_flush_peer_for_alloc` cannot
|
||||
satisfy; the alloc then returns None and the decode prealloc path treats
|
||||
that as a memory-estimation bug and aborts the scheduler.
|
||||
"""
|
||||
|
||||
class _Peer:
|
||||
def __init__(self, gate):
|
||||
self.lazy_compaction = True
|
||||
self._free_phys_pages = [0, 1, 2, 3] # only len() is read
|
||||
self.entry_bytes_per_page = 512
|
||||
self.disagg_move_gate = gate
|
||||
|
||||
class _Owner:
|
||||
def __init__(self, peer):
|
||||
self._peer = peer
|
||||
|
||||
def _credit(self, gate):
|
||||
peer = self._Peer(gate)
|
||||
owner = self._Owner(peer)
|
||||
return MultiEndedAllocator._peer_drainable_hole_bytes(owner)
|
||||
|
||||
def test_credit_follows_the_gate(self):
|
||||
# No PD gate installed (non-disagg): holes are realizable as before.
|
||||
self.assertEqual(self._credit(gate=None), 4 * 512)
|
||||
# Gate open: peer can compact, so the credit stands.
|
||||
self.assertEqual(self._credit(gate=lambda: True), 4 * 512)
|
||||
# Gate closed: an urgent flush would move nothing, so credit nothing.
|
||||
self.assertEqual(self._credit(gate=lambda: False), 0)
|
||||
|
||||
|
||||
class TestMoveGateRejectsNonPdNode(CustomTestCase):
|
||||
def test_null_mode_is_rejected(self):
|
||||
"""The gate is only meaningful on a PD node; a NULL-mode scheduler is a
|
||||
wiring bug and must not silently produce an always-open predicate."""
|
||||
scheduler = _FakeScheduler(DisaggregationMode.NULL)
|
||||
with self.assertRaises(ValueError):
|
||||
unified_memory_disagg_move_gate(scheduler)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -109,6 +109,9 @@ class TestDecodePreallocQueuePriority(unittest.TestCase):
|
||||
queue.pending_reqs = []
|
||||
queue.retracted_queue = []
|
||||
queue.num_reserved_decode_tokens = 0
|
||||
# `pop_preallocated` credits this counter; `__new__` skips the __init__
|
||||
# that seeds it.
|
||||
queue._num_published_destinations = 0
|
||||
queue._resolve_pending_reqs = MagicMock()
|
||||
queue._update_handshake_waiters = MagicMock()
|
||||
queue._allocatable_tokens = MagicMock(return_value=1000)
|
||||
|
||||
@@ -9,6 +9,7 @@ import torch
|
||||
from sglang.srt.mem_cache.allocator.hisparse import (
|
||||
DeepSeekV4HiSparseTokenToKVPoolAllocator,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_context
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
@@ -214,14 +215,18 @@ class TestDeepSeekV4HiSparseAllocator(CustomTestCase):
|
||||
manager._send_kvcache_generic = MagicMock(return_value=0)
|
||||
executor = MagicMock()
|
||||
|
||||
manager.send_kvcache(
|
||||
"session",
|
||||
np.array([1], dtype=np.int32),
|
||||
[10000, 20000, 30000],
|
||||
np.array([7], dtype=np.int32),
|
||||
executor,
|
||||
dst_device_kv_indices=np.array([21], dtype=np.int32),
|
||||
)
|
||||
# send_kvcache reads the memory bag (the unified-memory envelope-layout
|
||||
# check), so the context has to be published. This is the non-unified
|
||||
# path -- pin that explicitly rather than leaning on the default.
|
||||
with get_context().override_server_args(enable_unified_memory=False):
|
||||
manager.send_kvcache(
|
||||
"session",
|
||||
np.array([1], dtype=np.int32),
|
||||
[10000, 20000, 30000],
|
||||
np.array([7], dtype=np.int32),
|
||||
executor,
|
||||
dst_device_kv_indices=np.array([21], dtype=np.int32),
|
||||
)
|
||||
|
||||
kwargs = manager._send_kvcache_generic.call_args.kwargs
|
||||
self.assertEqual(kwargs["dst_device_data_ptrs"], {20000, 30000})
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
"""Derived-property tests for the PD whole-envelope transfer addressing.
|
||||
|
||||
PD disaggregation transfers the unified memory pool as whole envelopes with
|
||||
``addr = raw_ptr + physical_index * item_len`` (see
|
||||
``UnifiedMLATokenToKVPool.get_contiguous_buf_infos`` /
|
||||
``UnifiedMambaPool.get_contiguous_buf_infos`` and mooncake's
|
||||
``_send_kvcache_generic`` / ``_send_mamba_state``). That contract only holds if
|
||||
the page-major view builders keep (a) one page's data for ALL layers inside one
|
||||
contiguous ``page_envelope_bytes`` block, and (b) one mamba slot's conv+temporal
|
||||
state for all layers inside one contiguous ``entry_bytes`` block. A
|
||||
"looks equivalent" reordering of the view layout (e.g. layer-major across
|
||||
pages) would silently corrupt every PD transfer while all kernels keep working,
|
||||
because kernels read through the strided views, not through raw offsets.
|
||||
"""
|
||||
|
||||
import unittest
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.mem_cache.layout.page_major import (
|
||||
build_dense_mla_views,
|
||||
build_page_major_mamba_views,
|
||||
mamba_entry_bytes,
|
||||
mla_entry_bytes,
|
||||
)
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cpu_ci(est_time=60, suite="base-a-test-cpu")
|
||||
|
||||
|
||||
class TestMLAEnvelopeTransferAddressing(CustomTestCase):
|
||||
def test_page_envelope_matches_dense_views(self):
|
||||
"""Every (page, layer, slot) row written through the dense MLA views
|
||||
must land at raw_ptr + page * page_envelope_bytes + layer-block offset,
|
||||
i.e. inside the page's transfer envelope."""
|
||||
layer_num, page_size, kv_dim, num_pages = 3, 4, 8, 6
|
||||
store_dtype = torch.bfloat16
|
||||
row_bytes = kv_dim * store_dtype.itemsize
|
||||
page_bytes = page_size * layer_num * row_bytes
|
||||
self.assertEqual(
|
||||
page_bytes,
|
||||
page_size
|
||||
* mla_entry_bytes(
|
||||
layer_num=layer_num,
|
||||
kv_cache_dim=kv_dim,
|
||||
itemsize=store_dtype.itemsize,
|
||||
),
|
||||
)
|
||||
# +1 page envelope of tail pad, as UnifiedKVPool allocates for MLA.
|
||||
raw = torch.zeros((num_pages + 1) * page_bytes, dtype=torch.uint8)
|
||||
views = build_dense_mla_views(
|
||||
raw,
|
||||
layer_num=layer_num,
|
||||
kv_cache_dim=kv_dim,
|
||||
store_dtype=store_dtype,
|
||||
page_size=page_size,
|
||||
num_pages=num_pages,
|
||||
anchor_bytes=0,
|
||||
)
|
||||
torch.manual_seed(0)
|
||||
for page in range(num_pages):
|
||||
for layer in range(layer_num):
|
||||
for off in range(page_size):
|
||||
dense_id = page * layer_num * page_size + off
|
||||
val = torch.randn(kv_dim, dtype=store_dtype)
|
||||
views[layer][dense_id, 0] = val
|
||||
start = (
|
||||
page * page_bytes
|
||||
+ layer * page_size * row_bytes
|
||||
+ off * row_bytes
|
||||
)
|
||||
got = raw[start : start + row_bytes].view(store_dtype)
|
||||
self.assertTrue(torch.equal(got, val), (page, layer, off))
|
||||
|
||||
|
||||
class TestMambaEnvelopeTransferAddressing(CustomTestCase):
|
||||
def test_slot_envelope_is_self_contained(self):
|
||||
"""A slot's conv+temporal state for all layers must live exactly in
|
||||
raw[slot * entry_bytes : (slot+1) * entry_bytes]: no byte outside the
|
||||
envelope may change, and the payload byte count must fill it."""
|
||||
layer_num, max_slots = 2, 5
|
||||
conv_shapes = ((3, 4), (2, 6))
|
||||
temporal_shape = (2, 3, 4)
|
||||
conv_dtype = torch.bfloat16
|
||||
temporal_dtype = torch.float32
|
||||
entry = mamba_entry_bytes(
|
||||
layer_num=layer_num,
|
||||
conv_state_shapes=conv_shapes,
|
||||
conv_dtype=conv_dtype,
|
||||
temporal_state_shape=temporal_shape,
|
||||
temporal_dtype=temporal_dtype,
|
||||
)
|
||||
raw = torch.zeros(max_slots * entry, dtype=torch.uint8)
|
||||
conv_views, temporal_view = build_page_major_mamba_views(
|
||||
raw,
|
||||
layer_num=layer_num,
|
||||
conv_state_shapes=conv_shapes,
|
||||
conv_dtype=conv_dtype,
|
||||
temporal_state_shape=temporal_shape,
|
||||
temporal_dtype=temporal_dtype,
|
||||
max_slots=max_slots,
|
||||
anchor_bytes=0,
|
||||
)
|
||||
torch.manual_seed(0)
|
||||
for slot in range(max_slots):
|
||||
raw.zero_()
|
||||
n_payload = 0
|
||||
for i, conv_view in enumerate(conv_views):
|
||||
val = torch.randn((layer_num,) + conv_shapes[i], dtype=conv_dtype)
|
||||
conv_view[:, slot] = val
|
||||
n_payload += val.numel() * val.element_size()
|
||||
val = torch.randn((layer_num,) + temporal_shape, dtype=temporal_dtype)
|
||||
temporal_view[:, slot] = val
|
||||
n_payload += val.numel() * val.element_size()
|
||||
|
||||
outside = torch.cat([raw[: slot * entry], raw[(slot + 1) * entry :]])
|
||||
self.assertTrue(
|
||||
bool(outside.eq(0).all()),
|
||||
f"slot {slot} state bled outside its transfer envelope",
|
||||
)
|
||||
self.assertEqual(n_payload, entry)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user