diff --git a/python/sglang/srt/layers/attention/linear/inkling_sconv_backend.py b/python/sglang/srt/layers/attention/linear/inkling_sconv_backend.py index 8af6d78f17..46ae1a6a7b 100644 --- a/python/sglang/srt/layers/attention/linear/inkling_sconv_backend.py +++ b/python/sglang/srt/layers/attention/linear/inkling_sconv_backend.py @@ -155,6 +155,8 @@ class InklingShortConvAttnBackend(ShortConvAttnBackend): self._graph_track_inert_seqlens = torch.zeros( max_bs, dtype=torch.int64, device=dev ) + # Graph-static: captured track scatters read this address. + self._track_indices_buf = torch.zeros(max_bs, dtype=torch.int64, device=dev) # Same address-stability requirement; the base only sizes this from # init_cuda_graph_state, which the prefill graph never calls. self._alloc_cache_indices_buf(max_bs) @@ -269,6 +271,22 @@ class InklingShortConvAttnBackend(ShortConvAttnBackend): return self.forward_metadata = self._forward_metadata(forward_batch) self._refresh_cache_indices() + if not self._slot_gather_recordable: + self._translate_track_indices(forward_batch) + + def _translate_track_indices(self, forward_batch: ForwardBatch): + """Translate virtual track ids to the physical slots the conv kernels write.""" + track_indices = forward_batch.mamba_track_indices + buf = self._track_indices_buf + # Prep may re-run on the same batch; never translate twice. + if track_indices is None or track_indices.data_ptr() == buf.data_ptr(): + return + n = track_indices.shape[0] + assert n <= buf.shape[0], ( + f"track-index buffer too small: rows={n} vs bound {buf.shape[0]}" + ) + buf[:n].copy_(self._translate_mamba_indices(track_indices)) + forward_batch.mamba_track_indices = buf[:n] def _refresh_sconv_metadata( self, forward_batch: ForwardBatch, *, on_graph_path: bool diff --git a/test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_kl_hybrid_bitexact.py b/test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_kl_hybrid_bitexact.py index c7c5358ba8..0365d34f3e 100644 --- a/test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_kl_hybrid_bitexact.py +++ b/test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_kl_hybrid_bitexact.py @@ -21,6 +21,11 @@ Measured avg_kl_div: multiturn branching 0.0 2.01e-07 0.0 lazy test_prefill_cache_hit 5.56e-06 n/a 0.0 #29792 multiturn branching 9.43e-06/1.16e-05 5.14e-04 0.0 + unified test_logprobs_match n/a 1.82e-03 0.0 + memory test_prefill_cache_hit n/a 1.73e-03 0.0 + track test_decode_cache_hit n/a 2.44e-03 0.0 + +The unified-memory rows were measured on SM90 only. The lazy row was measured in the same session as the 1.18e-05 the non-lazy `test_prefill_cache_hit` read on that GPU, so read the pair as "both fire", @@ -73,7 +78,7 @@ from sglang.test.test_utils import ( unified_radix_tree_server_env, ) -register_cuda_ci(est_time=2300, stage="extra-a", runner_config="1-gpu-large") +register_cuda_ci(est_time=2640, stage="extra-a", runner_config="1-gpu-large") _MODEL_PATH = os.environ.get("INKLING_TEST_MODEL_PATH", "thinkingmachines/Inkling") _MODEL_REVISION = os.environ.get("INKLING_TEST_MODEL_REVISION", "test") @@ -233,6 +238,29 @@ class TestUnifiedHybridLazyBitExact(TestUnifiedHybridBitExact): ) +class TestUnifiedMemoryHybridBitExact(TestUnifiedHybridBitExact): + """Same exactness bar on the unified memory pool (virtual mamba slot ids).""" + + @classmethod + def setUpClass(cls): + cls.model = _MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = _base_args() + [ + "--enable-unified-memory", + "--chunked-prefill-size", + "16384", + ] + if _MODEL_REVISION: + other_args += ["--revision", _MODEL_REVISION] + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + env=unified_radix_tree_server_env(cls.tree_core_backend), + ) + + class TestUnifiedHybridHiCacheBitExact(CustomTestCase): """Same exactness bar with the host tier in the loop, over interleaved branches so hits land at many prefix lengths rather than one aligned one.