[Unified Memory] Fix Inkling conv-checkpoint track ids written to virtual slot numbers (#41144)

This commit is contained in:
Shuwen Wang
2026-09-28 05:19:19 -07:00
committed by GitHub
parent bddeb61d94
commit da3eb6db32
2 changed files with 47 additions and 1 deletions
@@ -155,6 +155,8 @@ class InklingShortConvAttnBackend(ShortConvAttnBackend):
self._graph_track_inert_seqlens = torch.zeros(
max_bs, dtype=torch.int64, device=dev
)
# Graph-static: captured track scatters read this address.
self._track_indices_buf = torch.zeros(max_bs, dtype=torch.int64, device=dev)
# Same address-stability requirement; the base only sizes this from
# init_cuda_graph_state, which the prefill graph never calls.
self._alloc_cache_indices_buf(max_bs)
@@ -269,6 +271,22 @@ class InklingShortConvAttnBackend(ShortConvAttnBackend):
return
self.forward_metadata = self._forward_metadata(forward_batch)
self._refresh_cache_indices()
if not self._slot_gather_recordable:
self._translate_track_indices(forward_batch)
def _translate_track_indices(self, forward_batch: ForwardBatch):
"""Translate virtual track ids to the physical slots the conv kernels write."""
track_indices = forward_batch.mamba_track_indices
buf = self._track_indices_buf
# Prep may re-run on the same batch; never translate twice.
if track_indices is None or track_indices.data_ptr() == buf.data_ptr():
return
n = track_indices.shape[0]
assert n <= buf.shape[0], (
f"track-index buffer too small: rows={n} vs bound {buf.shape[0]}"
)
buf[:n].copy_(self._translate_mamba_indices(track_indices))
forward_batch.mamba_track_indices = buf[:n]
def _refresh_sconv_metadata(
self, forward_batch: ForwardBatch, *, on_graph_path: bool
@@ -21,6 +21,11 @@ Measured avg_kl_div:
multiturn branching 0.0 2.01e-07 0.0
lazy test_prefill_cache_hit 5.56e-06 n/a 0.0
#29792 multiturn branching 9.43e-06/1.16e-05 5.14e-04 0.0
unified test_logprobs_match n/a 1.82e-03 0.0
memory test_prefill_cache_hit n/a 1.73e-03 0.0
track test_decode_cache_hit n/a 2.44e-03 0.0
The unified-memory rows were measured on SM90 only.
The lazy row was measured in the same session as the 1.18e-05 the non-lazy
`test_prefill_cache_hit` read on that GPU, so read the pair as "both fire",
@@ -73,7 +78,7 @@ from sglang.test.test_utils import (
unified_radix_tree_server_env,
)
register_cuda_ci(est_time=2300, stage="extra-a", runner_config="1-gpu-large")
register_cuda_ci(est_time=2640, stage="extra-a", runner_config="1-gpu-large")
_MODEL_PATH = os.environ.get("INKLING_TEST_MODEL_PATH", "thinkingmachines/Inkling")
_MODEL_REVISION = os.environ.get("INKLING_TEST_MODEL_REVISION", "test")
@@ -233,6 +238,29 @@ class TestUnifiedHybridLazyBitExact(TestUnifiedHybridBitExact):
)
class TestUnifiedMemoryHybridBitExact(TestUnifiedHybridBitExact):
"""Same exactness bar on the unified memory pool (virtual mamba slot ids)."""
@classmethod
def setUpClass(cls):
cls.model = _MODEL_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
other_args = _base_args() + [
"--enable-unified-memory",
"--chunked-prefill-size",
"16384",
]
if _MODEL_REVISION:
other_args += ["--revision", _MODEL_REVISION]
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=other_args,
env=unified_radix_tree_server_env(cls.tree_core_backend),
)
class TestUnifiedHybridHiCacheBitExact(CustomTestCase):
"""Same exactness bar with the host tier in the loop, over interleaved
branches so hits land at many prefix lengths rather than one aligned one.