[Unified Memory] Fix Inkling conv-checkpoint track ids written to virtual slot numbers (#41144)
This commit is contained in:
@@ -155,6 +155,8 @@ class InklingShortConvAttnBackend(ShortConvAttnBackend):
|
||||
self._graph_track_inert_seqlens = torch.zeros(
|
||||
max_bs, dtype=torch.int64, device=dev
|
||||
)
|
||||
# Graph-static: captured track scatters read this address.
|
||||
self._track_indices_buf = torch.zeros(max_bs, dtype=torch.int64, device=dev)
|
||||
# Same address-stability requirement; the base only sizes this from
|
||||
# init_cuda_graph_state, which the prefill graph never calls.
|
||||
self._alloc_cache_indices_buf(max_bs)
|
||||
@@ -269,6 +271,22 @@ class InklingShortConvAttnBackend(ShortConvAttnBackend):
|
||||
return
|
||||
self.forward_metadata = self._forward_metadata(forward_batch)
|
||||
self._refresh_cache_indices()
|
||||
if not self._slot_gather_recordable:
|
||||
self._translate_track_indices(forward_batch)
|
||||
|
||||
def _translate_track_indices(self, forward_batch: ForwardBatch):
|
||||
"""Translate virtual track ids to the physical slots the conv kernels write."""
|
||||
track_indices = forward_batch.mamba_track_indices
|
||||
buf = self._track_indices_buf
|
||||
# Prep may re-run on the same batch; never translate twice.
|
||||
if track_indices is None or track_indices.data_ptr() == buf.data_ptr():
|
||||
return
|
||||
n = track_indices.shape[0]
|
||||
assert n <= buf.shape[0], (
|
||||
f"track-index buffer too small: rows={n} vs bound {buf.shape[0]}"
|
||||
)
|
||||
buf[:n].copy_(self._translate_mamba_indices(track_indices))
|
||||
forward_batch.mamba_track_indices = buf[:n]
|
||||
|
||||
def _refresh_sconv_metadata(
|
||||
self, forward_batch: ForwardBatch, *, on_graph_path: bool
|
||||
|
||||
+29
-1
@@ -21,6 +21,11 @@ Measured avg_kl_div:
|
||||
multiturn branching 0.0 2.01e-07 0.0
|
||||
lazy test_prefill_cache_hit 5.56e-06 n/a 0.0
|
||||
#29792 multiturn branching 9.43e-06/1.16e-05 5.14e-04 0.0
|
||||
unified test_logprobs_match n/a 1.82e-03 0.0
|
||||
memory test_prefill_cache_hit n/a 1.73e-03 0.0
|
||||
track test_decode_cache_hit n/a 2.44e-03 0.0
|
||||
|
||||
The unified-memory rows were measured on SM90 only.
|
||||
|
||||
The lazy row was measured in the same session as the 1.18e-05 the non-lazy
|
||||
`test_prefill_cache_hit` read on that GPU, so read the pair as "both fire",
|
||||
@@ -73,7 +78,7 @@ from sglang.test.test_utils import (
|
||||
unified_radix_tree_server_env,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=2300, stage="extra-a", runner_config="1-gpu-large")
|
||||
register_cuda_ci(est_time=2640, stage="extra-a", runner_config="1-gpu-large")
|
||||
|
||||
_MODEL_PATH = os.environ.get("INKLING_TEST_MODEL_PATH", "thinkingmachines/Inkling")
|
||||
_MODEL_REVISION = os.environ.get("INKLING_TEST_MODEL_REVISION", "test")
|
||||
@@ -233,6 +238,29 @@ class TestUnifiedHybridLazyBitExact(TestUnifiedHybridBitExact):
|
||||
)
|
||||
|
||||
|
||||
class TestUnifiedMemoryHybridBitExact(TestUnifiedHybridBitExact):
|
||||
"""Same exactness bar on the unified memory pool (virtual mamba slot ids)."""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.model = _MODEL_PATH
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
other_args = _base_args() + [
|
||||
"--enable-unified-memory",
|
||||
"--chunked-prefill-size",
|
||||
"16384",
|
||||
]
|
||||
if _MODEL_REVISION:
|
||||
other_args += ["--revision", _MODEL_REVISION]
|
||||
cls.process = popen_launch_server(
|
||||
cls.model,
|
||||
cls.base_url,
|
||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
other_args=other_args,
|
||||
env=unified_radix_tree_server_env(cls.tree_core_backend),
|
||||
)
|
||||
|
||||
|
||||
class TestUnifiedHybridHiCacheBitExact(CustomTestCase):
|
||||
"""Same exactness bar with the host tier in the loop, over interleaved
|
||||
branches so hits land at many prefix lengths rather than one aligned one.
|
||||
|
||||
Reference in New Issue
Block a user