fix(inkling): translate unified-memory checkpoint destinations to physical slots (#38229)
Co-authored-by: Cheng Wan <cheng.wan@radixark.ai>
This commit is contained in:
@@ -0,0 +1,997 @@
|
||||
"""Inkling checkpoint writes use live physical slots, including graph replay."""
|
||||
|
||||
import unittest
|
||||
from contextlib import ExitStack, nullcontext
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers.attention.linear.inkling_sconv_backend import (
|
||||
InklingShortConvAttnBackend,
|
||||
InklingShortConvHybridAttnBackend,
|
||||
)
|
||||
from sglang.srt.mem_cache.allocator.unified_sub_pool import MultiEndedAllocator
|
||||
from sglang.srt.mem_cache.memory_pool import HybridReqToTokenPool
|
||||
from sglang.srt.mem_cache.unified_memory_pool import (
|
||||
MambaSubPoolSpec,
|
||||
MHASubPoolSpec,
|
||||
UnifiedHybridReqToTokenPool,
|
||||
UnifiedKVPool,
|
||||
UnifiedMambaPool,
|
||||
UnifiedMambaSlotAllocator,
|
||||
)
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardMode
|
||||
from sglang.srt.models.inkling_common.sconv import ShortConvolution
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cuda_ci(est_time=30, stage="base-b-kernel-unit", runner_config="1-gpu-large")
|
||||
|
||||
_BACKEND = "sglang.srt.layers.attention.linear.inkling_sconv_backend"
|
||||
|
||||
|
||||
@unittest.skipUnless(torch.cuda.is_available(), "requires CUDA")
|
||||
class TestInklingCheckpointIndices(CustomTestCase):
|
||||
def make_backend(
|
||||
self, *, lazy=False, static=False, slot_count=6, draft_token_num=None
|
||||
):
|
||||
mamba = MambaSubPoolSpec(
|
||||
name="mamba",
|
||||
layer_num=2,
|
||||
grow_direction="up",
|
||||
conv_state_shapes=((3, 64),),
|
||||
conv_dtype=torch.bfloat16,
|
||||
temporal_state_shape=(0,),
|
||||
temporal_dtype=torch.float32,
|
||||
)
|
||||
full = MHASubPoolSpec(
|
||||
name="full",
|
||||
layer_num=1,
|
||||
head_num=1,
|
||||
head_dim=8,
|
||||
store_dtype=torch.bfloat16,
|
||||
grow_direction="down",
|
||||
)
|
||||
shared = UnifiedKVPool(
|
||||
total_bytes=64 * mamba.entry_bytes(),
|
||||
sub_pool_specs=[mamba, full],
|
||||
device="cuda",
|
||||
enable_memory_saver=False,
|
||||
)
|
||||
shared._raw.zero_()
|
||||
pool = UnifiedMambaPool(
|
||||
unified_buffer=shared,
|
||||
sub_pool_name="mamba",
|
||||
spec_state_size=0,
|
||||
mamba_layer_ids=[0, 1],
|
||||
)
|
||||
allocator = MultiEndedAllocator(
|
||||
kvcache=pool,
|
||||
unified_buffer=shared,
|
||||
sub_pool_name="mamba",
|
||||
device="cuda",
|
||||
is_id_owner=True,
|
||||
lazy_compaction=lazy,
|
||||
)
|
||||
slots = allocator.alloc(slot_count)
|
||||
self.assertIsNotNone(slots)
|
||||
req_type = HybridReqToTokenPool if static else UnifiedHybridReqToTokenPool
|
||||
req_pool = req_type.__new__(req_type)
|
||||
req_pool.size = 4
|
||||
req_pool.mamba_pool = pool
|
||||
req_pool.mamba_allocator = UnifiedMambaSlotAllocator(
|
||||
allocator, max_size=pool.size, device="cuda"
|
||||
)
|
||||
req_pool.req_index_to_mamba_index_mapping = slots[1:2].to(torch.int32).repeat(4)
|
||||
backend = InklingShortConvAttnBackend.__new__(InklingShortConvAttnBackend)
|
||||
backend.req_to_token_pool = req_pool
|
||||
backend.device = "cuda"
|
||||
backend.conv_state_len = 3
|
||||
backend._cache_indices_buf = None
|
||||
backend._slot_gather_recordable = static
|
||||
config = SimpleNamespace(
|
||||
decode=SimpleNamespace(bs=[1, 2, 4], max_bs=4),
|
||||
prefill=SimpleNamespace(bs=[8]),
|
||||
)
|
||||
with (
|
||||
patch(
|
||||
_BACKEND + ".get_exec",
|
||||
return_value=SimpleNamespace(
|
||||
graph=SimpleNamespace(cuda_graph_config=config)
|
||||
),
|
||||
),
|
||||
patch(
|
||||
_BACKEND + ".get_spec",
|
||||
return_value=SimpleNamespace(
|
||||
speculative_num_draft_tokens=draft_token_num
|
||||
),
|
||||
),
|
||||
):
|
||||
backend._alloc_graph_buffers()
|
||||
return backend, allocator, pool, slots
|
||||
|
||||
def batch(self, ids, mode=ForwardMode.EXTEND, mask=None):
|
||||
n = len(ids)
|
||||
return SimpleNamespace(
|
||||
forward_mode=mode,
|
||||
batch_size=n,
|
||||
req_pool_indices=torch.arange(n, device="cuda"),
|
||||
mamba_track_indices=ids,
|
||||
mamba_track_mask=torch.ones(n, dtype=torch.bool, device="cuda")
|
||||
if mask is None
|
||||
else mask,
|
||||
)
|
||||
|
||||
def scatter(self, backend, batch, cache):
|
||||
hidden = (
|
||||
torch.arange(3 * 64, device="cuda", dtype=torch.float32)
|
||||
.reshape(3, 64)
|
||||
.to(torch.bfloat16)
|
||||
)
|
||||
rows = torch.arange(3, device="cuda").repeat(batch.batch_size, 1)
|
||||
ShortConvolution._prepare_extend_sconv_cache(
|
||||
SimpleNamespace(_conv_state=lambda _: backend.sconv_metadata),
|
||||
batch,
|
||||
cache,
|
||||
hidden,
|
||||
rows,
|
||||
)
|
||||
return hidden
|
||||
|
||||
def test_prefill_checkpoint_after_real_compaction(self):
|
||||
for lazy in (False, True):
|
||||
with self.subTest(lazy=lazy):
|
||||
backend, allocator, pool, slots = self.make_backend(lazy=lazy)
|
||||
virtual = slots[-1:].clone()
|
||||
allocator.free(slots[:1].clone())
|
||||
if lazy:
|
||||
allocator._flush(urgent=True)
|
||||
physical = backend._translate_mamba_indices(virtual)
|
||||
self.assertFalse(torch.equal(virtual, physical.to(virtual.dtype)))
|
||||
ids = torch.cat([virtual, virtual.new_tensor([-1])])
|
||||
original = ids.clone()
|
||||
batch = self.batch(ids, mask=torch.tensor([True, False], device="cuda"))
|
||||
backend._prepare_slot_indices(batch)
|
||||
cache = pool.mamba_cache.conv[0][0]
|
||||
before = cache.clone()
|
||||
expected = self.scatter(backend, batch, cache)
|
||||
torch.testing.assert_close(cache[physical[0]], expected, rtol=0, atol=0)
|
||||
before[physical[0]] = expected
|
||||
torch.testing.assert_close(cache, before, rtol=0, atol=0)
|
||||
torch.testing.assert_close(ids, original, rtol=0, atol=0)
|
||||
self.assertEqual(
|
||||
backend.sconv_metadata.track_cache_indices.dtype, torch.int64
|
||||
)
|
||||
|
||||
def test_replay_refreshes_captured_destination_after_compaction(self):
|
||||
backend, allocator, pool, slots = self.make_backend(lazy=True)
|
||||
virtual = slots[-1:].clone()
|
||||
batch = self.batch(virtual, mode=ForwardMode.DECODE)
|
||||
backend.init_forward_metadata_out_graph(batch)
|
||||
pointer = backend.sconv_metadata.track_cache_indices.data_ptr()
|
||||
cache = pool.mamba_cache.conv[0][0]
|
||||
self.scatter(backend, batch, cache) # compile before capture
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
self.scatter(backend, batch, cache)
|
||||
allocator.free(slots[:1].clone())
|
||||
allocator._flush(urgent=True)
|
||||
physical = backend._translate_mamba_indices(virtual)
|
||||
cache.zero_()
|
||||
fresh = self.batch(virtual, mode=ForwardMode.DECODE)
|
||||
backend.init_forward_metadata_out_graph(fresh)
|
||||
self.assertEqual(backend.sconv_metadata.track_cache_indices.data_ptr(), pointer)
|
||||
graph.replay()
|
||||
expected = (
|
||||
torch.arange(3 * 64, device="cuda", dtype=torch.float32)
|
||||
.reshape(3, 64)
|
||||
.to(torch.bfloat16)
|
||||
)
|
||||
torch.testing.assert_close(cache[physical[0]], expected, rtol=0, atol=0)
|
||||
self.assertEqual(
|
||||
torch.count_nonzero(cache).item(), torch.count_nonzero(expected).item()
|
||||
)
|
||||
torch.testing.assert_close(virtual, slots[-1:], rtol=0, atol=0)
|
||||
|
||||
def test_decode_tracks_the_physical_checkpoint(self):
|
||||
backend, allocator, pool, slots = self.make_backend()
|
||||
allocator.free(slots[:1].clone())
|
||||
batch = self.batch(slots[-1:].clone(), mode=ForwardMode.DECODE)
|
||||
backend._prepare_slot_indices(batch)
|
||||
cache = pool.mamba_cache.conv[0][0]
|
||||
conv = SimpleNamespace(
|
||||
activation=None,
|
||||
_conv_state=lambda _: backend.sconv_metadata,
|
||||
use_residual=True,
|
||||
_weight_2d=lambda: torch.ones(64, 4, dtype=torch.bfloat16, device="cuda"),
|
||||
)
|
||||
ShortConvolution._apply_decode_sconv_kernel(
|
||||
conv,
|
||||
torch.ones(1, 64, dtype=torch.bfloat16, device="cuda"),
|
||||
cache,
|
||||
backend._cache_indices,
|
||||
{"cache_mask": torch.ones(1, 1, 1, dtype=torch.bool, device="cuda")},
|
||||
batch,
|
||||
)
|
||||
physical = backend._translate_mamba_indices(slots[-1:])
|
||||
torch.testing.assert_close(
|
||||
cache[physical[0]], cache[backend._cache_indices[0]], rtol=0, atol=0
|
||||
)
|
||||
self.assertGreater(torch.count_nonzero(cache[physical[0]]).item(), 0)
|
||||
|
||||
def test_static_identity_path_and_disabled_tracking(self):
|
||||
backend, _, _, slots = self.make_backend(static=True)
|
||||
ids = slots[-1:].clone()
|
||||
batch = self.batch(ids)
|
||||
backend._prepare_slot_indices(batch)
|
||||
self.assertIs(batch.mamba_track_indices, ids)
|
||||
batch.mamba_track_indices = None
|
||||
backend._prepare_slot_indices(batch)
|
||||
self.assertIsNone(batch.mamba_track_indices)
|
||||
backend._slot_gather_recordable = False
|
||||
backend._prepare_slot_indices(batch)
|
||||
self.assertIsNone(batch.mamba_track_indices)
|
||||
|
||||
def test_graph_hooks_translate_once_for_each_mode(self):
|
||||
backend, allocator, _, slots = self.make_backend()
|
||||
allocator.free(slots[:1].clone())
|
||||
for mode in (
|
||||
ForwardMode.DECODE,
|
||||
ForwardMode.TARGET_VERIFY,
|
||||
ForwardMode.DRAFT_EXTEND_V2,
|
||||
):
|
||||
with self.subTest(mode=mode):
|
||||
ids = slots[-1:].clone()
|
||||
batch = self.batch(ids, mode=mode)
|
||||
with (
|
||||
patch.object(backend, "_refresh_sconv_metadata"),
|
||||
patch.object(
|
||||
backend,
|
||||
"_translate_mamba_indices",
|
||||
wraps=backend._translate_mamba_indices,
|
||||
) as translate,
|
||||
):
|
||||
backend.init_forward_metadata_out_graph(batch)
|
||||
calls = translate.call_count
|
||||
self.assertEqual(calls, 2) # active and checkpoint slots
|
||||
backend.init_forward_metadata_in_graph(batch)
|
||||
self.assertEqual(translate.call_count, calls)
|
||||
torch.testing.assert_close(
|
||||
backend.sconv_metadata.track_cache_indices,
|
||||
backend._translate_mamba_indices(ids).to(torch.int64),
|
||||
)
|
||||
torch.testing.assert_close(ids, slots[-1:])
|
||||
|
||||
def test_distinct_strided_destinations_keep_static_storage(self):
|
||||
backend, allocator, pool, slots = self.make_backend()
|
||||
allocator.free(slots[:1].clone())
|
||||
selected = slots[2:].flip(0)
|
||||
storage = torch.stack(
|
||||
[selected, slots[1:2].expand_as(selected)], dim=1
|
||||
).flatten()
|
||||
original = storage.clone()
|
||||
ids = storage[::2]
|
||||
self.assertFalse(ids.is_contiguous())
|
||||
batch = self.batch(ids)
|
||||
backend._prepare_slot_indices(batch)
|
||||
pointer = backend.sconv_metadata.track_cache_indices.data_ptr()
|
||||
physical = backend._translate_mamba_indices(selected).to(torch.int64)
|
||||
torch.testing.assert_close(
|
||||
backend.sconv_metadata.track_cache_indices, physical, rtol=0, atol=0
|
||||
)
|
||||
cache = pool.mamba_cache.conv[0][0]
|
||||
expected = cache.clone()
|
||||
hidden = torch.arange(12 * 64, device="cuda").reshape(12, 64).to(torch.bfloat16)
|
||||
rows = torch.arange(12, device="cuda").reshape(4, 3)
|
||||
ShortConvolution._prepare_extend_sconv_cache(
|
||||
SimpleNamespace(_conv_state=lambda _: backend.sconv_metadata),
|
||||
batch,
|
||||
cache,
|
||||
hidden,
|
||||
rows,
|
||||
)
|
||||
expected[physical] = hidden.reshape(4, 3, 64)
|
||||
torch.testing.assert_close(cache, expected, rtol=0, atol=0)
|
||||
smaller = self.batch(ids[:2])
|
||||
backend._prepare_slot_indices(smaller)
|
||||
self.assertEqual(backend.sconv_metadata.track_cache_indices.data_ptr(), pointer)
|
||||
torch.testing.assert_close(storage, original, rtol=0, atol=0)
|
||||
|
||||
def test_decode_graph_refreshes_distinct_destinations_and_masks_padding(self):
|
||||
backend, allocator, pool, slots = self.make_backend(lazy=True, slot_count=12)
|
||||
req_pool = backend.req_to_token_pool
|
||||
req_pool.req_index_to_mamba_index_mapping = torch.cat(
|
||||
[slots.new_zeros(1), slots[1:5]]
|
||||
).to(torch.int32)
|
||||
ids = slots[-4:].clone()
|
||||
track_mask = torch.ones(4, dtype=torch.bool, device="cuda")
|
||||
batch = self.batch(ids, mode=ForwardMode.DECODE, mask=track_mask)
|
||||
batch.req_pool_indices = torch.arange(1, 5, device="cuda")
|
||||
backend.init_forward_metadata_out_graph(batch)
|
||||
pointer = backend.sconv_metadata.track_cache_indices.data_ptr()
|
||||
cache = pool.mamba_cache.conv[0][0]
|
||||
hidden = torch.zeros(4, 64, dtype=torch.bfloat16, device="cuda")
|
||||
weight = torch.ones(64, 4, dtype=torch.bfloat16, device="cuda")
|
||||
conv = SimpleNamespace(
|
||||
activation=None,
|
||||
use_residual=True,
|
||||
_weight_2d=lambda: weight,
|
||||
_conv_state=lambda _: backend.sconv_metadata,
|
||||
)
|
||||
|
||||
def decode():
|
||||
backend.init_forward_metadata_in_graph(batch)
|
||||
ShortConvolution._apply_decode_sconv_kernel(
|
||||
conv,
|
||||
hidden,
|
||||
cache,
|
||||
backend._cache_indices,
|
||||
backend.sconv_metadata.precomputed,
|
||||
batch,
|
||||
)
|
||||
|
||||
decode()
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
decode()
|
||||
for step, live in enumerate((3, 1, 4)):
|
||||
with self.subTest(live=live):
|
||||
if step < 2:
|
||||
hole = slots[:1] if step == 0 else slots[5:6]
|
||||
allocator.free(hole.clone())
|
||||
allocator._flush(urgent=True)
|
||||
ids.copy_(slots[-4:].roll(step + 1))
|
||||
ids[live:].zero_()
|
||||
original = ids.clone()
|
||||
track_mask.zero_()
|
||||
track_mask[:live] = True
|
||||
if live > 1:
|
||||
track_mask[1] = False
|
||||
fresh = self.batch(ids, mode=ForwardMode.DECODE, mask=track_mask)
|
||||
fresh.req_pool_indices = torch.arange(1, 5, device="cuda")
|
||||
fresh.req_pool_indices[live:] = 0
|
||||
backend.init_forward_metadata_out_graph(fresh)
|
||||
self.assertEqual(
|
||||
backend.sconv_metadata.track_cache_indices.data_ptr(), pointer
|
||||
)
|
||||
physical = backend._translate_mamba_indices(ids).to(torch.int64)
|
||||
active = backend._cache_indices.clone().to(torch.int64)
|
||||
cache.fill_(-7)
|
||||
cache[0].zero_()
|
||||
hidden.copy_(torch.arange(4, device="cuda")[:, None] + step + 1)
|
||||
hidden[live:].zero_()
|
||||
expected = cache.clone()
|
||||
for row in range(live):
|
||||
window = torch.cat([cache[active[row], 1:], hidden[row : row + 1]])
|
||||
expected[active[row]] = window
|
||||
if row != 1:
|
||||
expected[physical[row]] = window
|
||||
graph.replay()
|
||||
torch.testing.assert_close(cache, expected, rtol=0, atol=0)
|
||||
torch.testing.assert_close(ids, original, rtol=0, atol=0)
|
||||
|
||||
def test_oversized_checkpoint_batch_fails_without_reallocating(self):
|
||||
backend, _, _, slots = self.make_backend()
|
||||
pointer = backend._graph_track_indices.data_ptr()
|
||||
batch = self.batch(slots[-1:].repeat(5))
|
||||
with self.assertRaisesRegex(
|
||||
AssertionError, "checkpoint-index buffer too small"
|
||||
):
|
||||
backend._prepare_slot_indices(batch)
|
||||
self.assertEqual(backend._graph_track_indices.data_ptr(), pointer)
|
||||
|
||||
def test_verify_commit_translates_both_destinations(self):
|
||||
backend, allocator, _, slots = self.make_backend()
|
||||
allocator.free(slots[:1].clone())
|
||||
indices = torch.tensor([0], device="cuda")
|
||||
for tracking in (False, True):
|
||||
with self.subTest(tracking=tracking):
|
||||
ids = slots[-1:].clone() if tracking else None
|
||||
with (
|
||||
patch.object(
|
||||
backend.req_to_token_pool,
|
||||
"get_speculative_mamba2_params_all_layers",
|
||||
return_value=object(),
|
||||
),
|
||||
patch(
|
||||
_BACKEND + ".scatter_mamba_states_after_mtp_verify"
|
||||
) as scatter,
|
||||
):
|
||||
backend.commit_conv_state_after_mtp_verify(
|
||||
req_pool_indices=indices,
|
||||
last_correct_step_indices=indices,
|
||||
mamba_track_indices=ids,
|
||||
mamba_steps_to_track=indices if tracking else None,
|
||||
)
|
||||
if tracking:
|
||||
torch.testing.assert_close(
|
||||
scatter.call_args.args[3], backend._translate_mamba_indices(ids)
|
||||
)
|
||||
torch.testing.assert_close(ids, slots[-1:], rtol=0, atol=0)
|
||||
else:
|
||||
self.assertIsNone(scatter.call_args.args[3])
|
||||
self.assertIsNone(scatter.call_args.args[4])
|
||||
torch.testing.assert_close(
|
||||
scatter.call_args.args[1],
|
||||
backend._translate_mamba_indices(slots[1:2].to(torch.int32)),
|
||||
)
|
||||
|
||||
def test_explicit_none_tracking_still_refreshes_active_slots(self):
|
||||
backend, allocator, _, slots = self.make_backend()
|
||||
allocator.free(slots[:1].clone())
|
||||
batch = SimpleNamespace(
|
||||
forward_mode=ForwardMode.DRAFT_EXTEND_V2,
|
||||
req_pool_indices=torch.tensor([0], device="cuda"),
|
||||
mamba_track_indices=None,
|
||||
)
|
||||
backend.init_forward_metadata_out_graph(batch)
|
||||
self.assertIsNone(backend.sconv_metadata.track_cache_indices)
|
||||
torch.testing.assert_close(
|
||||
backend._cache_indices,
|
||||
backend._translate_mamba_indices(slots[1:2]).to(torch.int32),
|
||||
)
|
||||
|
||||
def test_missing_tracking_field_rejects_incomplete_metadata_view(self):
|
||||
for static in (False, True):
|
||||
with self.subTest(static=static):
|
||||
backend, _, _, slots = self.make_backend(static=static)
|
||||
batch = self.batch(slots[-1:].clone())
|
||||
backend._prepare_slot_indices(batch)
|
||||
del batch.mamba_track_indices
|
||||
with self.assertRaisesRegex(AttributeError, "mamba_track_indices"):
|
||||
backend._prepare_slot_indices(batch)
|
||||
|
||||
def test_multilayer_staging_provides_checkpoint_ids_or_none(self):
|
||||
from sglang.srt.speculative.multi_layer_eagle_draft_extend_cuda_graph_runner import (
|
||||
MultiLayerEagleMultiStepDraftExtendCudaGraphRunner,
|
||||
)
|
||||
|
||||
_, backend, _, pool, slots, buffers = self._make_draft_checkpoint_runner(
|
||||
multilayer=True, static=True, tracking=True
|
||||
)
|
||||
wrapper = InklingShortConvHybridAttnBackend.__new__(
|
||||
InklingShortConvHybridAttnBackend
|
||||
)
|
||||
wrapper.short_conv_backend = backend
|
||||
wrapper.full_attn_backend = Mock(supports_draft_extend_metadata_staging=True)
|
||||
composite = MultiLayerEagleMultiStepDraftExtendCudaGraphRunner.__new__(
|
||||
MultiLayerEagleMultiStepDraftExtendCudaGraphRunner
|
||||
)
|
||||
composite.buffers = buffers
|
||||
composite.captured_req_width = 4
|
||||
composite.draft_extend_attn_backend_list = [wrapper]
|
||||
for step, tracking in enumerate((True, False, True)):
|
||||
with self.subTest(tracking=tracking, step=step):
|
||||
ids = slots[-4:].roll(step).clone()
|
||||
original = ids.clone()
|
||||
buffers.mamba_track_indices = ids if tracking else None
|
||||
composite._stage_metadata(bs=3, raw_bs=2)
|
||||
torch.testing.assert_close(
|
||||
backend._cache_indices,
|
||||
torch.cat([slots[1:3], slots.new_zeros(1)]).to(torch.int32),
|
||||
)
|
||||
if tracking:
|
||||
torch.testing.assert_close(
|
||||
backend.sconv_metadata.track_cache_indices, ids[:3]
|
||||
)
|
||||
self.assertEqual(
|
||||
backend.sconv_metadata.track_cache_indices.data_ptr(),
|
||||
ids.data_ptr(),
|
||||
)
|
||||
batch = self.batch(
|
||||
ids[:3], mask=torch.tensor([True, True, False], device="cuda")
|
||||
)
|
||||
cache = pool.mamba_cache.conv[0][0]
|
||||
expected = cache.clone()
|
||||
hidden = self.scatter(backend, batch, cache)
|
||||
expected[ids[:2].long()] = hidden
|
||||
torch.testing.assert_close(cache, expected, rtol=0, atol=0)
|
||||
torch.testing.assert_close(ids, original, rtol=0, atol=0)
|
||||
else:
|
||||
self.assertIsNone(backend.sconv_metadata.track_cache_indices)
|
||||
|
||||
def test_draft_eager_preplan_and_repeated_steps_preserve_virtual_source(self):
|
||||
from sglang.srt.speculative import eagle_worker_common as common
|
||||
from sglang.srt.speculative.eagle_info import EagleDraftExtendInput
|
||||
|
||||
for static in (False, True):
|
||||
for graph_available in (False, True):
|
||||
with self.subTest(static=static, graph_available=graph_available):
|
||||
steps = []
|
||||
for step in range(2):
|
||||
backend, allocator, pool, slots = self.make_backend(
|
||||
static=static, slot_count=12, draft_token_num=4
|
||||
)
|
||||
backend.mamba_cache_chunk_size = 4
|
||||
backend.req_to_token_pool.req_index_to_mamba_index_mapping.fill_(
|
||||
slots[3]
|
||||
)
|
||||
if not static:
|
||||
allocator.free(slots[step : step + 1].clone())
|
||||
wrapper = InklingShortConvHybridAttnBackend.__new__(
|
||||
InklingShortConvHybridAttnBackend
|
||||
)
|
||||
wrapper.full_attn_backend = Mock()
|
||||
wrapper.short_conv_backend = backend
|
||||
wrapper.attn_backend_list = [wrapper.full_attn_backend, backend]
|
||||
steps.append((wrapper, backend, pool))
|
||||
ids = slots[-1:].clone()
|
||||
original = ids.clone()
|
||||
batch = self.batch(ids, mode=ForwardMode.DRAFT_EXTEND_V2)
|
||||
batch.seq_lens = torch.tensor([2], device="cuda")
|
||||
batch.extend_seq_lens = torch.tensor([4], device="cuda")
|
||||
batch.extend_prefix_lens = None
|
||||
batch.extend_num_tokens = 4
|
||||
batch.spec_info = EagleDraftExtendInput(
|
||||
num_accept_tokens=torch.tensor([3], device="cuda")
|
||||
)
|
||||
batch.mark_forward_metadata_ready = Mock()
|
||||
schedule_batch = SimpleNamespace(
|
||||
seq_lens=batch.seq_lens,
|
||||
seq_lens_cpu=None,
|
||||
forward_mode=batch.forward_mode,
|
||||
model_config=SimpleNamespace(vocab_size=32),
|
||||
)
|
||||
runner = SimpleNamespace(
|
||||
spec_algorithm=SimpleNamespace(is_standalone=lambda: False),
|
||||
attn_backend=steps[0][0],
|
||||
model_config=SimpleNamespace(model_is_mrope=False),
|
||||
)
|
||||
graph_runner = (
|
||||
SimpleNamespace(can_run_graph=lambda _: False)
|
||||
if graph_available
|
||||
else None
|
||||
)
|
||||
with (
|
||||
patch.object(
|
||||
common.ForwardBatch, "init_new", return_value=batch
|
||||
),
|
||||
patch.object(common, "maybe_detect_oob"),
|
||||
patch.object(common, "is_npu", return_value=False),
|
||||
):
|
||||
planned = common.prepare_for_draft_extend(
|
||||
batch.spec_info,
|
||||
schedule_batch,
|
||||
torch.zeros(4, dtype=torch.int64, device="cuda"),
|
||||
4,
|
||||
runner,
|
||||
graph_runner,
|
||||
return_hidden_states_before_norm=False,
|
||||
)
|
||||
self.assertIs(planned, batch)
|
||||
destinations = []
|
||||
# Inspect the real pre-plan, then repeat step 0 and enter step 1.
|
||||
for index in (0, 0, 1):
|
||||
wrapper, backend, pool = steps[index]
|
||||
if destinations:
|
||||
wrapper.init_forward_metadata(batch)
|
||||
self.assertIs(batch.mamba_track_indices, ids)
|
||||
torch.testing.assert_close(ids, original, rtol=0, atol=0)
|
||||
physical = backend._translate_mamba_indices(ids).long()
|
||||
torch.testing.assert_close(
|
||||
backend.sconv_metadata.track_cache_indices,
|
||||
physical,
|
||||
rtol=0,
|
||||
atol=0,
|
||||
)
|
||||
destinations.append(physical.clone())
|
||||
cache = pool.mamba_cache.conv[0][0]
|
||||
cache.fill_(-7)
|
||||
hidden = torch.arange(4, device="cuda")[:, None].expand(4, 64)
|
||||
hidden = hidden.to(torch.bfloat16)
|
||||
active = backend._cache_indices.long()
|
||||
joined = torch.cat([cache[active[0]], hidden])
|
||||
expected = cache.clone()
|
||||
expected[physical[0]] = joined[1:4]
|
||||
expected[active[0]] = joined[3:6]
|
||||
with patch(
|
||||
"sglang.srt.models.inkling_common.sconv.get_exec",
|
||||
return_value=SimpleNamespace(
|
||||
mamba=SimpleNamespace(
|
||||
enable_mamba_extra_buffer=True,
|
||||
mamba_track_interval=4,
|
||||
)
|
||||
),
|
||||
):
|
||||
ShortConvolution._update_sconv_cache_for_draft_extend(
|
||||
SimpleNamespace(
|
||||
_conv_state=lambda _: backend.sconv_metadata
|
||||
),
|
||||
batch,
|
||||
cache,
|
||||
backend._cache_indices,
|
||||
hidden,
|
||||
)
|
||||
torch.testing.assert_close(cache, expected, rtol=0, atol=0)
|
||||
if not static:
|
||||
self.assertFalse(torch.equal(destinations[0], destinations[-1]))
|
||||
|
||||
def test_static_draft_staging_keeps_checkpoint_metadata(self):
|
||||
from sglang.srt.speculative.eagle_info import EagleDraftExtendInput
|
||||
|
||||
backend, _, _, slots = self.make_backend(static=True, draft_token_num=4)
|
||||
wrapper = InklingShortConvHybridAttnBackend.__new__(
|
||||
InklingShortConvHybridAttnBackend
|
||||
)
|
||||
wrapper.short_conv_backend = backend
|
||||
wrapper.full_attn_backend = Mock(supports_draft_extend_metadata_staging=True)
|
||||
batch = self.batch(slots[-1:].clone(), mode=ForwardMode.DRAFT_EXTEND_V2)
|
||||
batch.seq_lens = torch.tensor([6], device="cuda")
|
||||
batch.extend_seq_lens = torch.tensor([4], device="cuda")
|
||||
batch.extend_prefix_lens = None
|
||||
batch.extend_num_tokens = 4
|
||||
batch.spec_info = EagleDraftExtendInput(
|
||||
num_accept_tokens=torch.tensor([3], device="cuda")
|
||||
)
|
||||
wrapper.init_forward_metadata_out_graph(batch, in_capture=True)
|
||||
wrapper.init_forward_metadata_in_graph(batch)
|
||||
self.assertIs(
|
||||
backend.sconv_metadata.track_cache_indices, batch.mamba_track_indices
|
||||
)
|
||||
|
||||
def test_prefill_capture_reinitializes_inert_checkpoint_metadata(self):
|
||||
backend, _, pool, _ = self.make_backend()
|
||||
backend.mamba_cache_chunk_size = 4
|
||||
batch = self.batch(torch.zeros(1, dtype=torch.int64, device="cuda"))
|
||||
batch.mamba_track_indices = None
|
||||
batch.mamba_track_mask = None
|
||||
batch.seq_lens = torch.tensor([4], device="cuda")
|
||||
batch.extend_seq_lens = torch.tensor([4], device="cuda")
|
||||
batch.extend_prefix_lens = None
|
||||
batch.extend_num_tokens = 4
|
||||
batch.spec_info = None
|
||||
cache = pool.mamba_cache.conv[0][0]
|
||||
expected = cache.clone()
|
||||
for _ in range(2):
|
||||
backend.init_forward_metadata_out_graph(batch, in_capture=True)
|
||||
torch.testing.assert_close(
|
||||
backend.sconv_metadata.track_cache_indices,
|
||||
torch.zeros(1, dtype=torch.int64, device="cuda"),
|
||||
)
|
||||
self.scatter(backend, batch, cache)
|
||||
torch.testing.assert_close(cache, expected, rtol=0, atol=0)
|
||||
|
||||
def test_draft_runners_capture_and_replay_checkpoint_destinations(self):
|
||||
for multilayer in (False, True):
|
||||
for static in (False, True):
|
||||
for tracking in (False, True):
|
||||
with self.subTest(
|
||||
multilayer=multilayer, static=static, tracking=tracking
|
||||
):
|
||||
self._check_draft_checkpoint_replay(
|
||||
multilayer, static, tracking
|
||||
)
|
||||
|
||||
def _allocate_production_draft_buffers(self, multilayer, tracking):
|
||||
from sglang.srt.speculative import eagle_draft_extend_cuda_graph_runner as eagle
|
||||
from sglang.srt.speculative import (
|
||||
multi_layer_eagle_draft_extend_cuda_graph_runner as multi,
|
||||
)
|
||||
|
||||
model_runner = SimpleNamespace(
|
||||
device="cuda",
|
||||
tp_size=1,
|
||||
model_config=SimpleNamespace(
|
||||
dtype=torch.bfloat16, vocab_size=8, hf_config=SimpleNamespace()
|
||||
),
|
||||
graph_shared_output=SimpleNamespace(
|
||||
get_logits_buffer=lambda vocab, rows: torch.zeros(
|
||||
rows, vocab, device="cuda"
|
||||
)
|
||||
),
|
||||
)
|
||||
worker = SimpleNamespace(
|
||||
draft_runner=model_runner,
|
||||
speculative_algorithm=SimpleNamespace(is_standalone=lambda: False),
|
||||
seed_dsa_topk_from_draft_extend=False,
|
||||
mtp_model_runner=lambda step: model_runner,
|
||||
)
|
||||
runtime = SimpleNamespace(
|
||||
mamba=SimpleNamespace(enable_mamba_extra_buffer=tracking),
|
||||
graph=SimpleNamespace(
|
||||
disable_cuda_graph_padding=False, enable_profile_cuda_graph=False
|
||||
),
|
||||
)
|
||||
if multilayer:
|
||||
composite = (
|
||||
multi.MultiLayerEagleMultiStepDraftExtendCudaGraphRunner.__new__(
|
||||
multi.MultiLayerEagleMultiStepDraftExtendCudaGraphRunner
|
||||
)
|
||||
)
|
||||
composite.runners = [SimpleNamespace(model_runner=model_runner)]
|
||||
composite.eagle_worker = worker
|
||||
composite.max_bs = 4
|
||||
composite.captured_req_width = 4
|
||||
composite.seq_len_fill_value = 4
|
||||
composite.device = "cuda"
|
||||
composite.prune_draft_extend_logits = False
|
||||
composite.rotates_in_graph = False
|
||||
composite.require_gathered_buffer = False
|
||||
with (
|
||||
patch.object(multi, "get_exec", return_value=runtime),
|
||||
patch.object(
|
||||
multi, "get_draft_input_from_target_hidden_dim", return_value=64
|
||||
),
|
||||
):
|
||||
return composite._allocate_buffers()
|
||||
|
||||
worker.draft_extend_attn_backend = SimpleNamespace(
|
||||
init_cuda_graph_state=lambda *args: None,
|
||||
get_cuda_graph_seq_len_fill_value=lambda: 4,
|
||||
)
|
||||
# Only model/distributed capture scaffolding is replaced. __init__ and
|
||||
# its real input-buffer allocation run before the kernel replay test.
|
||||
with ExitStack() as stack:
|
||||
replacements = {
|
||||
"get_exec": runtime,
|
||||
"get_parallel": SimpleNamespace(pp_size=1),
|
||||
"deployment_attn_dp_size": 1,
|
||||
"get_flags": SimpleNamespace(
|
||||
capture=SimpleNamespace(enable_torch_compile=False)
|
||||
),
|
||||
"get_spec": SimpleNamespace(speculative_num_steps=1),
|
||||
"get_batch_sizes_to_capture": ([4], []),
|
||||
"resolve_num_tokens_per_req": 4,
|
||||
"get_draft_input_from_target_hidden_dim": 64,
|
||||
"require_gathered_buffer": False,
|
||||
"require_mlp_tp_gather": False,
|
||||
"require_mlp_sync": False,
|
||||
"require_attn_tp_gather": False,
|
||||
"resolve_decode_backend": Mock(),
|
||||
}
|
||||
for name, result in replacements.items():
|
||||
stack.enter_context(patch.object(eagle, name, return_value=result))
|
||||
stack.enter_context(patch.object(eagle, "model_capture_mode", nullcontext))
|
||||
stack.enter_context(patch.object(eagle, "DeepEPCudaGraphRunnerAdapter"))
|
||||
stack.enter_context(
|
||||
patch.object(eagle.EagleDraftExtendInputBuffers, "share_buffers")
|
||||
)
|
||||
stack.enter_context(
|
||||
patch.object(eagle.EAGLEDraftExtendCudaGraphRunner, "capture")
|
||||
)
|
||||
return eagle.EAGLEDraftExtendCudaGraphRunner(worker).buffers
|
||||
|
||||
def _make_draft_checkpoint_runner(self, multilayer, static, tracking):
|
||||
from sglang.srt.speculative import eagle_draft_extend_cuda_graph_runner as eagle
|
||||
from sglang.srt.speculative import (
|
||||
multi_layer_eagle_draft_extend_cuda_graph_runner as multi,
|
||||
)
|
||||
|
||||
backend, allocator, pool, slots = self.make_backend(
|
||||
lazy=True, static=static, slot_count=12, draft_token_num=4
|
||||
)
|
||||
backend.mamba_cache_chunk_size = 4
|
||||
req_pool = backend.req_to_token_pool
|
||||
req_pool.req_index_to_mamba_index_mapping = torch.cat(
|
||||
[slots.new_zeros(1), slots[1:5]]
|
||||
).to(torch.int32)
|
||||
buffers = self._allocate_production_draft_buffers(multilayer, tracking)
|
||||
buffers.req_pool_indices.copy_(torch.arange(1, 5, device="cuda"))
|
||||
buffers.seq_lens.fill_(6)
|
||||
buffers.seq_lens_cpu.fill_(6)
|
||||
buffers.num_accept_tokens.fill_(3)
|
||||
if tracking:
|
||||
self.assertIsNotNone(buffers.mamba_track_indices)
|
||||
self.assertEqual(buffers.mamba_track_indices.shape, (4,))
|
||||
self.assertEqual(buffers.mamba_track_indices.dtype, torch.int64)
|
||||
self.assertEqual(buffers.mamba_track_indices.device.type, "cuda")
|
||||
torch.testing.assert_close(
|
||||
buffers.mamba_track_indices,
|
||||
torch.zeros(4, dtype=torch.int64, device="cuda"),
|
||||
)
|
||||
buffers.mamba_track_indices.copy_(slots[-4:])
|
||||
else:
|
||||
self.assertIsNone(buffers.mamba_track_indices)
|
||||
cls = (
|
||||
multi.MultiLayerEagleDraftExtendCudaGraphRunner
|
||||
if multilayer
|
||||
else eagle.EAGLEDraftExtendCudaGraphRunner
|
||||
)
|
||||
runner = cls.__new__(cls)
|
||||
runner.buffers = buffers
|
||||
runner.captured_req_width = 4
|
||||
runner.forward_mode = ForwardMode.DRAFT_EXTEND_V2
|
||||
runner.extend_seq_lens_cpu = [4] * 4
|
||||
runner.require_mlp_tp_gather = False
|
||||
runner.require_attn_tp_gather = False
|
||||
runner.require_gathered_buffer = False
|
||||
runner.capture_bs = [4]
|
||||
runner.seq_len_fill_value = 4
|
||||
runner.num_front_tokens = 0
|
||||
runner.prune_draft_extend_logits = False
|
||||
runner.metadata_captured_in_graph = False
|
||||
runner.step = 0
|
||||
runner.deepep_adapter = Mock()
|
||||
runner.device_module = torch.cuda
|
||||
runner.attn_backend = SimpleNamespace(
|
||||
supports_draft_extend_metadata_staging=False
|
||||
)
|
||||
runner.draft_extend_attn_backend = backend
|
||||
runner.eagle_worker = SimpleNamespace(draft_extend_attn_backend_list=[backend])
|
||||
runner.model_runner = SimpleNamespace(
|
||||
model_config=SimpleNamespace(model_is_mrope=False),
|
||||
spec_algorithm=SimpleNamespace(is_standalone=lambda: False),
|
||||
device_timer=None,
|
||||
canary_manager=None,
|
||||
)
|
||||
return runner, backend, allocator, pool, slots, buffers
|
||||
|
||||
def test_static_staged_runner_checkpoint_replay(self):
|
||||
for tracking in (False, True):
|
||||
with self.subTest(tracking=tracking):
|
||||
self._check_draft_checkpoint_replay(True, True, tracking, staging=True)
|
||||
|
||||
def _check_draft_checkpoint_replay(
|
||||
self, multilayer, static, tracking, staging=False
|
||||
):
|
||||
from sglang.srt.speculative import eagle_draft_extend_cuda_graph_runner as eagle
|
||||
from sglang.srt.speculative import (
|
||||
multi_layer_eagle_draft_extend_cuda_graph_runner as multi,
|
||||
)
|
||||
from sglang.srt.speculative.eagle_info import EagleDraftExtendInput
|
||||
|
||||
runner, backend, allocator, pool, slots, buffers = (
|
||||
self._make_draft_checkpoint_runner(multilayer, static, tracking)
|
||||
)
|
||||
hook_backend = backend
|
||||
if staging:
|
||||
wrapper = InklingShortConvHybridAttnBackend.__new__(
|
||||
InklingShortConvHybridAttnBackend
|
||||
)
|
||||
wrapper.short_conv_backend = backend
|
||||
wrapper.full_attn_backend = Mock(
|
||||
supports_draft_extend_metadata_staging=True
|
||||
)
|
||||
wrapper.full_attn_backend.draft_extend_metadata_captured_in_graph.return_value = False
|
||||
hook_backend = wrapper
|
||||
runner.attn_backend = wrapper
|
||||
runner.draft_extend_attn_backend = wrapper
|
||||
runner.eagle_worker.draft_extend_attn_backend_list = [wrapper]
|
||||
cache = pool.mamba_cache.conv[0][0]
|
||||
captured = []
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
output = SimpleNamespace(
|
||||
next_token_logits=buffers.next_token_logits_buffer,
|
||||
hidden_states=buffers.hidden_states,
|
||||
)
|
||||
|
||||
def model_forward(input_ids, positions, batch):
|
||||
if not captured:
|
||||
captured.append(batch)
|
||||
ShortConvolution._update_sconv_cache_for_draft_extend(
|
||||
SimpleNamespace(_conv_state=lambda _: backend.sconv_metadata),
|
||||
batch,
|
||||
cache,
|
||||
backend._cache_indices,
|
||||
buffers.hidden_states,
|
||||
)
|
||||
return output
|
||||
|
||||
def capture_graph(key, run_once, **kwargs):
|
||||
run_once()
|
||||
with torch.cuda.graph(graph):
|
||||
run_once()
|
||||
|
||||
runner.model_runner.model = SimpleNamespace(forward=model_forward)
|
||||
runner.backend = SimpleNamespace(capture_one=capture_graph)
|
||||
runner._replay_graph = lambda *args: (graph.replay(), output)[1]
|
||||
sconv_module = "sglang.srt.models.inkling_common.sconv"
|
||||
with (
|
||||
patch(
|
||||
sconv_module + ".get_exec",
|
||||
return_value=SimpleNamespace(
|
||||
mamba=SimpleNamespace(
|
||||
mamba_track_interval=4, enable_mamba_extra_buffer=True
|
||||
)
|
||||
),
|
||||
),
|
||||
patch.object(eagle, "maybe_flashinfer_autotune_speculative_draft"),
|
||||
patch.object(eagle, "set_dp_buffer_len"),
|
||||
patch.object(eagle, "set_is_extend_in_batch"),
|
||||
):
|
||||
if multilayer:
|
||||
batch = runner.get_forward_batch(4)
|
||||
hook_backend.init_forward_metadata_out_graph(batch, in_capture=True)
|
||||
|
||||
def run_once():
|
||||
hook_backend.init_forward_metadata_in_graph(batch)
|
||||
return model_forward(batch.input_ids, batch.positions, batch)
|
||||
|
||||
capture_graph(None, run_once)
|
||||
else:
|
||||
runner.capture_one_shape(4, None)
|
||||
pointer = (
|
||||
backend.sconv_metadata.track_cache_indices.data_ptr()
|
||||
if tracking
|
||||
else None
|
||||
)
|
||||
self.assertIsNone(backend.sconv_metadata.track_conv_indices)
|
||||
for live in (4, 2, 1, 3, 4):
|
||||
if live == 2 and not static:
|
||||
allocator.free(slots[:1].clone())
|
||||
allocator._flush(urgent=True)
|
||||
ids = slots[-live:].flip(0).clone()
|
||||
spec = EagleDraftExtendInput(
|
||||
hidden_states=buffers.hidden_states[: live * 4],
|
||||
num_correct_drafts=torch.ones(
|
||||
live, dtype=torch.int32, device="cuda"
|
||||
),
|
||||
num_accept_tokens=torch.full(
|
||||
(live,), 3, dtype=torch.int32, device="cuda"
|
||||
),
|
||||
)
|
||||
fresh = SimpleNamespace(
|
||||
batch_size=live,
|
||||
input_ids=buffers.input_ids[: live * 4],
|
||||
positions=buffers.positions[: live * 4],
|
||||
out_cache_loc=buffers.out_cache_loc[: live * 4],
|
||||
req_pool_indices=torch.arange(1, live + 1, device="cuda"),
|
||||
seq_lens=torch.full((live,), 6, dtype=torch.int64, device="cuda"),
|
||||
seq_lens_sum=6 * live,
|
||||
seq_lens_cpu=None,
|
||||
extend_seq_lens=torch.full(
|
||||
(live,), 4, dtype=torch.int32, device="cuda"
|
||||
),
|
||||
extend_seq_lens_cpu=None,
|
||||
mamba_track_indices=ids if tracking and live != 3 else None,
|
||||
spec_info=spec,
|
||||
)
|
||||
cache.fill_(-7)
|
||||
cache[0].zero_()
|
||||
buffers.hidden_states.copy_(
|
||||
torch.arange(16, device="cuda")[:, None] + 1
|
||||
)
|
||||
original = ids.clone()
|
||||
active = backend._translate_mamba_indices(slots[1 : live + 1]).long()
|
||||
physical = backend._translate_mamba_indices(ids).long()
|
||||
expected = cache.clone()
|
||||
for row in range(live):
|
||||
joined = torch.cat(
|
||||
[
|
||||
cache[active[row]],
|
||||
buffers.hidden_states[row * 4 : (row + 1) * 4],
|
||||
]
|
||||
)
|
||||
if tracking and live != 3:
|
||||
expected[physical[row]] = joined[1:4]
|
||||
expected[active[row]] = joined[3:6]
|
||||
if multilayer:
|
||||
composite = multi.MultiLayerEagleMultiStepDraftExtendCudaGraphRunner.__new__(
|
||||
multi.MultiLayerEagleMultiStepDraftExtendCudaGraphRunner
|
||||
)
|
||||
composite.buffers = buffers
|
||||
composite.captured_req_width = 4
|
||||
composite.require_mlp_tp_gather = False
|
||||
composite.require_gathered_buffer = False
|
||||
composite.capture_bs = [4]
|
||||
composite.num_front_tokens = 0
|
||||
composite.seq_len_fill_value = 4
|
||||
composite.runners = [runner]
|
||||
composite.draft_extend_attn_backend_list = [hook_backend]
|
||||
if not staging:
|
||||
composite._stage_metadata = Mock()
|
||||
composite.prepare(fresh)
|
||||
runner.replay(
|
||||
4, composite.seq_lens_sum, composite._replay_spec_info, None
|
||||
)
|
||||
else:
|
||||
runner.execute(fresh, torch.arange(live, device="cuda") * 4 + 1)
|
||||
torch.testing.assert_close(cache[1:], expected[1:], rtol=0, atol=0)
|
||||
torch.testing.assert_close(ids, original, rtol=0, atol=0)
|
||||
if tracking:
|
||||
expected_ids = torch.zeros_like(buffers.mamba_track_indices)
|
||||
if live != 3:
|
||||
expected_ids[:live] = original
|
||||
torch.testing.assert_close(
|
||||
buffers.mamba_track_indices, expected_ids, rtol=0, atol=0
|
||||
)
|
||||
self.assertEqual(
|
||||
backend.sconv_metadata.track_cache_indices.data_ptr(), pointer
|
||||
)
|
||||
else:
|
||||
self.assertIsNone(captured[0].mamba_track_indices)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user