diff --git a/python/sglang/kernels/ops/kvcache/kv_indices.py b/python/sglang/kernels/ops/kvcache/kv_indices.py index e11c1b7cde..13aa9dc339 100644 --- a/python/sglang/kernels/ops/kvcache/kv_indices.py +++ b/python/sglang/kernels/ops/kvcache/kv_indices.py @@ -39,7 +39,7 @@ def create_flashinfer_kv_indices_triton( ``ENTRY_PAGE_SIZE == 1`` (default): the source table is token-granular and entries are emitted verbatim -- byte-identical to the historical kernel. ``ENTRY_PAGE_SIZE == ps``: the source is the translator's PAGE-granular - read table (entries already kernel-facing page ids); token ids are rebuilt + read table (entries already physical page ids); token ids are rebuilt as ``token = entry * ps + pos % ps``, exact because converting an id keeps its offset inside the page. ``TOKEN_BLOCK_PARALLEL`` (default False): launched on a 2D grid diff --git a/python/sglang/kernels/ops/memory/__init__.py b/python/sglang/kernels/ops/memory/__init__.py index 7d3f2fe8f5..bcb6618dee 100644 --- a/python/sglang/kernels/ops/memory/__init__.py +++ b/python/sglang/kernels/ops/memory/__init__.py @@ -18,7 +18,7 @@ _TRITON_KERNELS = [ ("virtual_slot", "alloc_bind_inplace"), ("virtual_slot", "free_unbind_inplace"), ("virtual_slot", "bind_inplace"), - ("virtual_slot", "write_loc_to_kernel_ids"), + ("virtual_slot", "translate_token_ids"), ] for _mod, _fn in _TRITON_KERNELS: register_kernel( diff --git a/python/sglang/kernels/ops/memory/virtual_slot.py b/python/sglang/kernels/ops/memory/virtual_slot.py index f1c744c751..482f61d7d6 100644 --- a/python/sglang/kernels/ops/memory/virtual_slot.py +++ b/python/sglang/kernels/ops/memory/virtual_slot.py @@ -193,14 +193,14 @@ def bind_inplace( bind_inplace_kernel[grid](v, p, v2p, p2v, N, BLOCK=ALLOC_BIND_BLOCK) -WRITE_LOC_BLOCK = 512 +TRANSLATE_TOKEN_IDS_BLOCK = 512 @triton.jit -def write_loc_to_kernel_id_kernel( - loc_ptr, # in: [N] int64 — WIDENED virtual token ids +def translate_token_ids_kernel( + loc_ptr, # in: [N] or strided (rows, cols), int32/int64 — virtual token ids, widened under DCP v2p_ptr, # in: [num_pages + 1] int64 — virtual->physical page table - out_ptr, # out: [N] int64 — physical token ids + out_ptr, # out: shaped like loc, or packed [W] when W > N; int64 — physical token ids N, # runtime: live element count W, # runtime: lanes to write; [N, W) get 0 num_v_pages, # runtime: rows of v2p; a page at or past it is unmapped @@ -264,13 +264,13 @@ def write_loc_to_kernel_id_kernel( def _row_col_strides(t: torch.Tensor) -> Tuple[int, int]: - """Element strides of `t` seen as ``(rows, cols)``; 0-d and 1-D are one row.""" + """Element strides of a 1-D or 2-D `t` seen as ``(rows, cols)``; 1-D is one row.""" if t.dim() == 2: return t.stride(0), t.stride(1) - return 0, (t.stride(0) if t.dim() == 1 else 1) + return 0, t.stride(0) -def write_loc_to_kernel_ids( +def translate_token_ids( *, loc: torch.Tensor, v2p: torch.Tensor, @@ -280,7 +280,7 @@ def write_loc_to_kernel_ids( out: Optional[torch.Tensor] = None, out_width: Optional[int] = None, ) -> torch.Tensor: - """One launch for the whole write-loc conversion; see the kernel. + """Virtual token ids -> physical token ids in one launch; see the kernel. ``out`` is written in place when given (a captured graph records the gather against a fixed ``data_ptr``), else a fresh int64 tensor is @@ -297,26 +297,25 @@ def write_loc_to_kernel_ids( out = torch.empty(loc.shape, dtype=torch.int64, device=loc.device) width = N if out_width is None else int(out_width) assert out.dtype == torch.int64, ( - f"write_loc_to_kernel_ids: out dtype must be int64 (matches v2p), " - f"got {out.dtype}" + f"translate_token_ids: out dtype must be int64 (matches v2p), got {out.dtype}" ) if out_width is None: # `out` mirrors `loc` whatever its shape; a 2-D page table is legal. assert out.shape == loc.shape, ( - f"write_loc_to_kernel_ids: out shape {tuple(out.shape)} must match " + f"translate_token_ids: out shape {tuple(out.shape)} must match " f"loc shape {tuple(loc.shape)}" ) else: assert loc.is_contiguous(), ( - f"write_loc_to_kernel_ids: out_width writes a packed lane range, " + f"translate_token_ids: out_width writes a packed lane range, " f"so loc must be packed; got stride {tuple(loc.stride())}" ) assert out.dim() == 1 and out.is_contiguous() and out.numel() >= width, ( - f"write_loc_to_kernel_ids: out_width needs a packed 1-D out of at " + f"translate_token_ids: out_width needs a packed 1-D out of at " f"least {width}, got {tuple(out.shape)}" ) assert width >= N, ( - f"write_loc_to_kernel_ids: out_width {width} is under the batch's " + f"translate_token_ids: out_width {width} is under the batch's " f"{N} locs, which would drop live rows" ) if width == 0: @@ -348,14 +347,14 @@ def write_loc_to_kernel_ids( out_row_stride = out_col_stride = 1 else: assert loc.dim() <= 2 and out.dim() <= 2, ( - f"write_loc_to_kernel_ids: a strided loc/out is walked as 2-D, got " + f"translate_token_ids: a strided loc/out is walked as 2-D, got " f"{loc.dim()}-D loc and {out.dim()}-D out; pass a contiguous view" ) cols = loc.shape[-1] if loc.dim() == 2 else N loc_row_stride, loc_col_stride = _row_col_strides(loc) out_row_stride, out_col_stride = _row_col_strides(out) - grid = (triton.cdiv(width, WRITE_LOC_BLOCK),) - write_loc_to_kernel_id_kernel[grid]( + grid = (triton.cdiv(width, TRANSLATE_TOKEN_IDS_BLOCK),) + translate_token_ids_kernel[grid]( loc, v2p, out, @@ -371,6 +370,6 @@ def write_loc_to_kernel_ids( DCP_SIZE=dcp_size, DCP_RANK=dcp_rank, STRIDED=not flat, - BLOCK=WRITE_LOC_BLOCK, + BLOCK=TRANSLATE_TOKEN_IDS_BLOCK, ) return out diff --git a/python/sglang/srt/arg_groups/kv_cache_hook.py b/python/sglang/srt/arg_groups/kv_cache_hook.py index a82fbd1866..a5780bf3a9 100644 --- a/python/sglang/srt/arg_groups/kv_cache_hook.py +++ b/python/sglang/srt/arg_groups/kv_cache_hook.py @@ -545,7 +545,7 @@ def handle_unified_memory_pool(server_args: Any) -> None: assert cfg.speculative_algorithm in (None, "DSPARK"), ( "--enable-unified-memory only supports --speculative-algorithm " "DSPARK (chain draft); other speculative algorithms are not yet " - "audited for the unified pool's virtual/kernel-facing loc translation. Got " + "audited for the unified pool's virtual-to-physical loc translation. Got " f"--speculative-algorithm={cfg.speculative_algorithm!r}." ) if cfg.speculative_algorithm == "DSPARK": @@ -565,7 +565,7 @@ def handle_unified_memory_pool(server_args: Any) -> None: f"attention backends {sorted(spec_allowed)} for both prefill " f"and decode; got {sorted(spec_backends)}. flashinfer / fa3 do " "not translate speculative verify indices to the unified " - "pool's kernel-facing space yet." + "pool's physical ids yet." ) assert not cfg.enable_two_batch_overlap, ( "--enable-unified-memory does not support --enable-two-batch-overlap: " diff --git a/python/sglang/srt/layers/attention/flashattention_backend.py b/python/sglang/srt/layers/attention/flashattention_backend.py index 8b3fba1334..0e276e9b3c 100644 --- a/python/sglang/srt/layers/attention/flashattention_backend.py +++ b/python/sglang/srt/layers/attention/flashattention_backend.py @@ -383,7 +383,7 @@ class FlashAttentionBackend(AttentionBackend): if not model_runner.model_config.is_local_attention_model: return None # Local attention re-translates metadata.page_table through the static - # full->swa map, which is meaningless on the unified pool's kernel tables. + # full->swa map, which is meaningless on the unified pool's physical tables. assert not self.kv_index_translator.is_translating, ( "--enable-unified-memory does not support local-attention models " "on the fa3/fa4 backend." @@ -749,7 +749,7 @@ class FlashAttentionBackend(AttentionBackend): # straight into these capture-stable buffers from the LIVE v2p, so # a page relocated by compaction since capture is picked up. Same # substitution the eager extend branch makes in its `_unified_read` - # fixup; `build_index_table` emits page-granular kernel-facing ids + # fixup; `build_index_table` emits page-granular physical ids # directly, so there is no `// page_size` to undo. self.kv_index_translator.build_index_table( req_pool_indices=forward_batch.req_pool_indices[:bs], @@ -1208,7 +1208,7 @@ class FlashAttentionBackend(AttentionBackend): if swa_out_cache_loc is not None: # The swa write loc was computed from the still-VIRTUAL # loc at ForwardBatch construction; re-running the - # full->swa map on the kernel-facing loc would be garbage. + # full->swa map on the physical loc would be garbage. metadata.swa_out_cache_loc = ( self.kv_index_translator.sliding_window_write_loc_for( swa_out_cache_loc diff --git a/python/sglang/srt/layers/attention/flashinfer_backend.py b/python/sglang/srt/layers/attention/flashinfer_backend.py index 1ecab40524..22d0a9b915 100644 --- a/python/sglang/srt/layers/attention/flashinfer_backend.py +++ b/python/sglang/srt/layers/attention/flashinfer_backend.py @@ -1722,7 +1722,7 @@ class FlashInferIndicesUpdaterDecode: req_pool_indices: torch.Tensor, ): # Unified SWA wrapper-0: gather from the swa canonical directly -- its - # entries are already swa-side kernel-facing ids, so the in-place + # entries are already swa-side physical ids, so the in-place # full->swa translate below must not run on top of them. translator = self.attn_backend.kv_index_translator use_swa_source = use_sliding_window_kv_pool and translator.reads_are_translated @@ -2155,7 +2155,7 @@ class FlashInferIndicesUpdaterPrefill: ): bs = len(seq_lens) # Unified SWA wrapper-0: gather from the swa canonical directly -- its - # entries are already swa-side kernel-facing ids, so the in-place + # entries are already swa-side physical ids, so the in-place # full->swa translate below must not run on top of them. translator = self.attn_backend.kv_index_translator use_swa_source = use_sliding_window_kv_pool and translator.reads_are_translated diff --git a/python/sglang/srt/layers/attention/trtllm_mla_backend.py b/python/sglang/srt/layers/attention/trtllm_mla_backend.py index 36cca3c45a..e3d2f768a7 100755 --- a/python/sglang/srt/layers/attention/trtllm_mla_backend.py +++ b/python/sglang/srt/layers/attention/trtllm_mla_backend.py @@ -480,7 +480,7 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend): self.decode_cuda_graph_kv_indices = torch.full( (max_bs, max_blocks_per_seq), -1, dtype=torch.int32, device=self.device ) - # Unified pool: capture-stable buffer for the kernel-facing KV write loc, filled + # Unified pool: capture-stable buffer for the physical KV write loc, filled # out-of-graph in init_forward_metadata_out_graph so the in-graph # set_mla_kv_buffer captures no translate. if self.kv_index_translator.is_translating: @@ -1151,7 +1151,7 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend): q: torch.Tensor, q_rope: torch.Tensor, ) -> Optional[torch.Tensor]: - """Decode: scatter the KV row at ``loc`` (already kernel-facing) and + """Decode: scatter the KV row at ``loc`` (already physical) and build the [q_nope | q_rope] fmha query in one kernel launch (saves one launch per MLA layer and keeps the PDL chain intact). @@ -1340,7 +1340,7 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend): layer, self._kv_write_loc(forward_batch), k, k_rope ) else: - # eager (or static pool): out_cache_loc is kernel-facing. + # eager (or static pool): out_cache_loc is physical. if ( merge_query and self._fused_set_kv_concat_q diff --git a/python/sglang/srt/mem_cache/allocator/unified_hybrid_swa.py b/python/sglang/srt/mem_cache/allocator/unified_hybrid_swa.py index cd8f39742f..c94f9c56de 100644 --- a/python/sglang/srt/mem_cache/allocator/unified_hybrid_swa.py +++ b/python/sglang/srt/mem_cache/allocator/unified_hybrid_swa.py @@ -152,7 +152,7 @@ class UnifiedSWAAllocatorBase(SWATokenToKVPoolAllocator): full_allocator=self.full_attn_allocator, swa_allocator=self.swa_attn_allocator, ) - # Size host pools in tokens; sub-pool `size` counts kernel-facing rows. + # Size host pools in tokens; sub-pool `size` counts physical rows. kvcache.full_kv_pool.host_capacity_tokens = self._size_full kvcache.swa_kv_pool.host_capacity_tokens = self._size_swa for name, pool in ( @@ -163,7 +163,7 @@ class UnifiedSWAAllocatorBase(SWATokenToKVPoolAllocator): pool.host_capacity_tokens * unified_buffer.spec(name).entry_bytes() ) # Full-attention transfers use virtual IDs. SWA transfers already use - # kernel-facing IDs from translate_loc_from_full_to_swa. + # physical IDs from translate_loc_from_full_to_swa. kvcache.full_kv_pool.host_transfer_translate = ( self.full_attn_allocator.translate_kv_loc ) @@ -306,7 +306,7 @@ class UnifiedSWAAllocatorBase(SWATokenToKVPoolAllocator): """Virtual TOKEN ids -> full-sub-pool PHYSICAL token ids for the PD transfer engine. - PHYSICAL, not kernel-facing: the transfer registers page ENVELOPES (see + The transfer registers page ENVELOPES and addresses them by PHYSICAL id (see `UnifiedMHATokenToKVPool.get_contiguous_buf_infos`). Without this override the base identity would put VIRTUAL ids on the wire, which address real bytes and so corrupt silently rather than fail. @@ -341,7 +341,7 @@ class UnifiedSWAAllocatorBase(SWATokenToKVPoolAllocator): """Bind SWA pages to resident or newly loaded full-attention virtual IDs. Bind before translating: unbound pages translate to the padding sink. - Return kernel-facing IDs, or None if capacity cannot be reclaimed. + Return physical IDs, or None if capacity cannot be reclaimed. """ ids = full_token_ids.to(torch.int64) if ids.numel() == 0: @@ -372,16 +372,16 @@ class UnifiedSWAAllocatorBase(SWATokenToKVPoolAllocator): lazy_compaction=self.lazy_compaction, ) - def translate_write_loc_for_kernel( + def translate_write_loc( self, loc: torch.Tensor, *, out: Optional[torch.Tensor] = None, out_width: Optional[int] = None, ) -> torch.Tensor: - """Widened virtual WRITE loc -> kernel-facing id. DCP is rejected for this + """Widened virtual WRITE loc -> physical id. DCP is rejected for this composite at argument validation, so it coincides with the read translate.""" - return self.full_attn_allocator.translate_write_loc_for_kernel( + return self.full_attn_allocator.translate_write_loc( loc, out=out, out_width=out_width ) diff --git a/python/sglang/srt/mem_cache/allocator/unified_mamba.py b/python/sglang/srt/mem_cache/allocator/unified_mamba.py index 41d25e6f10..71c25cabc6 100644 --- a/python/sglang/srt/mem_cache/allocator/unified_mamba.py +++ b/python/sglang/srt/mem_cache/allocator/unified_mamba.py @@ -127,7 +127,7 @@ class UnifiedMambaTokenToKVPoolAllocator(BaseTokenToKVPoolAllocator): self.mamba_allocator.available_size(), ) - # HiCache indexes the full sub-pool's per-layer views with kernel-facing IDs. + # HiCache indexes the full sub-pool's per-layer views with physical IDs. kvcache.full_kv_pool.host_transfer_translate = ( self.full_attn_allocator.translate_kv_loc ) @@ -273,15 +273,15 @@ class UnifiedMambaTokenToKVPoolAllocator(BaseTokenToKVPoolAllocator): """Page-level physical->virtual table of the full sub-pool.""" return self.full_attn_allocator.physical_to_virtual - def translate_write_loc_for_kernel( + def translate_write_loc( self, loc: torch.Tensor, *, out: Optional[torch.Tensor] = None, out_width: Optional[int] = None, ) -> torch.Tensor: - """Widened virtual WRITE loc -> DENSE id; see the sub-allocator's copy.""" - return self.full_attn_allocator.translate_write_loc_for_kernel( + """Widened virtual WRITE loc -> physical id; see the sub-allocator's copy.""" + return self.full_attn_allocator.translate_write_loc( loc, out=out, out_width=out_width ) @@ -289,7 +289,7 @@ class UnifiedMambaTokenToKVPoolAllocator(BaseTokenToKVPoolAllocator): self, kv_indices: torch.Tensor ) -> torch.Tensor: """Virtual TOKEN ids -> PHYSICAL token ids for the PD transfer engine. - PHYSICAL, not kernel-facing: the transfer registers page ENVELOPES (see + The transfer registers page ENVELOPES and addresses them by PHYSICAL id (see `UnifiedMLATokenToKVPool.get_contiguous_buf_infos`).""" # Defensive: `_validate_unified_memory_dcp` rejects this pairing at # argument validation, so reaching it means a config path got past that. diff --git a/python/sglang/srt/mem_cache/allocator/unified_sub_pool.py b/python/sglang/srt/mem_cache/allocator/unified_sub_pool.py index 01fd789a62..86e4184410 100644 --- a/python/sglang/srt/mem_cache/allocator/unified_sub_pool.py +++ b/python/sglang/srt/mem_cache/allocator/unified_sub_pool.py @@ -44,7 +44,7 @@ from sglang.kernels.ops.memory.virtual_slot import ( alloc_bind_inplace, bind_inplace, free_unbind_inplace, - write_loc_to_kernel_ids, + translate_token_ids, ) from sglang.srt.environ import envs from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator @@ -993,10 +993,10 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator): *, out: Optional[torch.Tensor] = None, ) -> torch.Tensor: - """Virtual token ids -> physical token ids, which under the token-major - views ARE the kernel-facing ids: + """Virtual token ids -> physical token ids, the ids kernels index the + per-layer views with: - kernel_id(t) = v2p[t // ps] * ps + t % ps + physical(t) = v2p[t // ps] * ps + t % ps Under DCP the input is the DCP-collapsed id (`widened // dcp_size`, what `KVIndexTranslator.translate_dcp_read_ids` hands down), so this works on @@ -1020,9 +1020,9 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator): out_width: Optional[int] = None, ) -> torch.Tensor: """One launch for the read and write conversions alike; see - `write_loc_to_kernel_ids`, which owns the `out=` dtype and shape + `translate_token_ids`, which owns the `out=` dtype and shape contract for both.""" - return write_loc_to_kernel_ids( + return translate_token_ids( loc=loc, v2p=self.virtual_to_physical, page_size=self.pool_page_size, @@ -1032,22 +1032,22 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator): out_width=out_width, ) - def translate_write_loc_for_kernel( + def translate_write_loc( self, widened_loc: torch.Tensor, *, out: Optional[torch.Tensor] = None, out_width: Optional[int] = None, ) -> torch.Tensor: - """Widened virtual WRITE loc (`out_cache_loc`) -> kernel-facing id. + """Widened virtual WRITE loc (`out_cache_loc`) -> physical id. Reads arrive already DCP-collapsed, but `out_cache_loc` does not: it still carries the owner rule in `loc % dcp_size`. Ids this rank does not own go - to kernel id 0, the padding sink every write kernel skips. + to id 0, the padding sink every write kernel skips. """ parallel = get_parallel() dcp_size = parallel.attn_dcp_size if self.shards_under_dcp else 1 - with record_function("MultiEndedAlloc.translate_write_loc_for_kernel"): + with record_function("MultiEndedAlloc.translate_write_loc"): return self._translate_loc_fused( widened_loc, dcp_size=dcp_size, diff --git a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py index 89482ce48c..f2556a12a6 100644 --- a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py +++ b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py @@ -1535,7 +1535,7 @@ class HybridCacheController(BaseHiCacheController): continue if entry.device_indices_from_anchor_fn is not None: # Allocate independent pools first: their allocation/eviction - # can compact SWA before its kernel-facing IDs are captured. + # can compact SWA before its physical IDs are captured. anchor_transfers.append((pool, entry)) continue # device_alloc_fn / device_free_fn override entry.device_pool's diff --git a/python/sglang/srt/mem_cache/kv_index_translator.py b/python/sglang/srt/mem_cache/kv_index_translator.py index 999e62fc2a..ebfe6dd003 100644 --- a/python/sglang/srt/mem_cache/kv_index_translator.py +++ b/python/sglang/srt/mem_cache/kv_index_translator.py @@ -42,7 +42,7 @@ uses its rows as-is, and one that wants flat per-token ids rebuilds them as token_id = entry * entry_page_size + pos % entry_page_size -WRITES, IN TWO PHASES. The full-side write loc is rebound to kernel-facing +WRITES, IN TWO PHASES. The full-side write loc is rebound to physical ids at ForwardBatch construction - the earliest consumer can snapshot it right after. The sliding-window write loc is derived at the same moment as read table, into the same index table. @@ -92,7 +92,7 @@ class KVIndexTable(msgspec.Struct, frozen=True): row_ids: torch.Tensor # which row belongs to batch lane b row_stride: int # stride between rows of `ids`, in elements entry_page_size: int # what one entry covers: 1 = a token, N = a page of N - is_translated: bool # entries are already kernel-facing ids + is_translated: bool # entries are already physical ids sliding_window_ids: Optional[torch.Tensor] # SWA models: the parallel swa array @@ -129,7 +129,7 @@ class KVIndexTranslator: # are collapsed by the DCP index kernels, `out_cache_loc` still # carries the owner rule in `loc % dcp_size`. Identity with the read # translate when dcp_size == 1. - self._translate_write_full = alloc.translate_write_loc_for_kernel + self._translate_write_full = alloc.translate_write_loc # DCP read ids stay WIDENED to the consumer: selecting this rank's # share changes the length, so only the production site can do it. self.defer_read_translate = get_parallel().attn_dcp_size > 1 @@ -179,7 +179,7 @@ class KVIndexTranslator: @property def reads_are_translated(self) -> bool: - """Whether a read this translator fills comes out kernel-facing. False + """Whether a read this translator fills comes out physical. False on a non-unified pool, and under DCP, where the ids stay VIRTUAL for ``translate_dcp_read_ids`` to finish.""" return self.is_translating and not self.defer_read_translate @@ -506,7 +506,7 @@ class KVIndexTranslator: return self.is_translating or get_parallel().attn_dcp_size > 1 def translate_dcp_read_ids(self, widened_ids: torch.Tensor) -> torch.Tensor: - """Widened logical READ ids -> kernel-facing ids, for either pool. + """Widened logical READ ids -> physical ids, for either pool. The one hook every DCP read-index production site calls; on a static pool `widened // dcp_size` IS the whole virtual->physical translation. @@ -519,7 +519,7 @@ class KVIndexTranslator: def translate_full_attn_ids( self, kv_indices: torch.Tensor, *, out: Optional[torch.Tensor] = None ) -> torch.Tensor: - """Virtual token ids -> kernel-facing full-attention ids (the identity + """Virtual token ids -> physical full-attention ids (the identity when no translation is needed, so callers never branch).""" if not self.is_translating: assert out is None, "passthrough translate takes no out=" diff --git a/python/sglang/srt/mem_cache/memory_pool.py b/python/sglang/srt/mem_cache/memory_pool.py index 61c441a0d4..00b00400aa 100644 --- a/python/sglang/srt/mem_cache/memory_pool.py +++ b/python/sglang/srt/mem_cache/memory_pool.py @@ -1747,8 +1747,8 @@ class KVWriteLoc: All location info lives here (in the attention metadata), NOT in the pool: - ``loc``: the generic per-token write location (``out_cache_loc``). - KERNEL-FACING on every pool: physical by allocation on non-unified - pools, rebound at ForwardBatch construction (``rebind_write_loc``) on + PHYSICAL on every pool: by allocation on non-unified pools, + rebound at ForwardBatch construction (``rebind_write_loc``) on the unified pool. - ``swa_loc``: the SWA-sub-pool location for hybrid SWA pools (``None`` otherwise); under the unified pool the translator derives it from the @@ -4349,7 +4349,7 @@ class HybridLinearKVPool(KVCache): dst_dtype: Optional[torch.dtype] = None, ): assert self.use_mla, "get_mla_kv_buffer called when use_mla is False" - # Read door -- same kernel-facing contract as the write door: `loc` is + # Read door -- same physical-id contract as the write door: `loc` is # a read-index tensor already translated at its production site # (fetch_mha_one_shot_kv_indices / prepare_chunked_kv_indices); the # pool never translates. diff --git a/python/sglang/srt/mem_cache/unified_memory_pool.py b/python/sglang/srt/mem_cache/unified_memory_pool.py index caf2be3e87..8f2d66ff58 100644 --- a/python/sglang/srt/mem_cache/unified_memory_pool.py +++ b/python/sglang/srt/mem_cache/unified_memory_pool.py @@ -296,11 +296,11 @@ def unified_memory_supported_for_model(model_config, *, use_mla_backend: bool) - return use_mla_backend or not model_config.has_asymmetric_kv -def _assert_kernel_id_bound(*, sub_pool_name: str, n_rows: int) -> None: - """Check if kernel-facing ids can flow through int32 read-index buffers.""" +def _assert_physical_id_bound(*, sub_pool_name: str, n_rows: int) -> None: + """Physical ids must fit the int32 read-index buffers.""" assert n_rows < 2**31, ( - f"sub-pool {sub_pool_name!r}: kernel-facing id space has {n_rows} rows, " - f"exceeding the int32 bound (2^31) that read-index buffers assume. " + f"sub-pool {sub_pool_name!r}: {n_rows} physical ids exceed the int32 " + f"bound (2^31) that read-index buffers assume. " "Reduce max_total_num_tokens." ) @@ -521,7 +521,7 @@ class UnifiedKVPool: page_size: int, ) -> Tuple[List[torch.Tensor], List[torch.Tensor]]: num_slots = max_slots // page_size * page_size - _assert_kernel_id_bound(sub_pool_name=spec.name, n_rows=num_slots) + _assert_physical_id_bound(sub_pool_name=spec.name, n_rows=num_slots) layout = spec.layout() k_views, v_views = ( build_dense_views( @@ -543,7 +543,7 @@ class UnifiedKVPool: page_size: int, ) -> List[torch.Tensor]: num_slots = max_slots // page_size * page_size - _assert_kernel_id_bound(sub_pool_name=spec.name, n_rows=num_slots) + _assert_physical_id_bound(sub_pool_name=spec.name, n_rows=num_slots) layout = spec.layout() return build_dense_views( self._raw, @@ -1424,7 +1424,7 @@ def init_unified_mamba_pools( # Only HybridLinearKVPool's retraction CPU-copy path uses this hook. token_to_kv_pool._mamba_translate = mamba_slot_allocator.translate # No full-KV translate hook is wired: both MLA doors now receive - # KERNEL-FACING ids -- writes from the ForwardBatch rebind, reads + # PHYSICAL ids -- writes from the ForwardBatch rebind, reads # translated at their production sites. logger.info( @@ -1668,7 +1668,7 @@ class UnifiedSWAKVPool(SWAKVPool): layer_id_override=pool_layer_id, ) return - # Full layer: `loc` is already the full-side kernel-facing id, so an + # Full layer: `loc` is already the full-side physical id, so an # explicit full_loc is a same-space alias -- only triton's captured path # passes one (its capture-stable buffer). if full_loc is None: diff --git a/python/sglang/srt/model_executor/forward_batch_info.py b/python/sglang/srt/model_executor/forward_batch_info.py index f19c952d0f..09351a2b80 100644 --- a/python/sglang/srt/model_executor/forward_batch_info.py +++ b/python/sglang/srt/model_executor/forward_batch_info.py @@ -532,7 +532,7 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): # The original sequence length without being chunked. Qwen-1M related. orig_seq_lens: Optional[torch.Tensor] = None - # The write loc before `rebind_write_loc` replaced it with kernel-facing + # The write loc before `rebind_write_loc` replaced it with physical # ids; a backend re-derives from it into its capture-stable buffer. out_cache_loc_virtual: Optional[torch.Tensor] = None # DSV4-NPU only: per-pool slot bundle from DSV4NPUTokenToKVPoolAllocator, diff --git a/python/sglang/srt/state_capturer/base.py b/python/sglang/srt/state_capturer/base.py index f451f7e6f5..d153eaf9c1 100644 --- a/python/sglang/srt/state_capturer/base.py +++ b/python/sglang/srt/state_capturer/base.py @@ -202,7 +202,7 @@ class BaseTopkCapturer: forward_batch, can_run_graph, cuda_graph_batch ) # get_topk reads req_to_token IDs; attention may have rebound the write - # loc to kernel-facing IDs, which are neither stable nor the same space. + # loc to physical IDs, which are neither stable nor the same space. out_cache_loc = forward_batch.out_cache_loc_virtual if out_cache_loc is None: out_cache_loc = forward_batch.out_cache_loc diff --git a/test/registered/dcp/test_trtllm_mla_family_dcp_metadata.py b/test/registered/dcp/test_trtllm_mla_family_dcp_metadata.py index 4bb0e5c271..8f52551df4 100644 --- a/test/registered/dcp/test_trtllm_mla_family_dcp_metadata.py +++ b/test/registered/dcp/test_trtllm_mla_family_dcp_metadata.py @@ -493,7 +493,7 @@ class TestFusedFp8WriteGate(CustomTestCase): @unittest.skipUnless(torch.cuda.is_available(), "the fused translate is Triton") -class TestFusedWriteLocTranslateCuda(CustomTestCase): +class TestTranslateTokenIdsCuda(CustomTestCase): """The fused write-loc translate must agree with its own CPU branch. The two implementations must not drift: Triton truncates division toward @@ -503,7 +503,7 @@ class TestFusedWriteLocTranslateCuda(CustomTestCase): """ def test_cuda_matches_the_cpu_branch(self): - from sglang.kernels.ops.memory.virtual_slot import write_loc_to_kernel_ids + from sglang.kernels.ops.memory.virtual_slot import translate_token_ids for page_size in (1, 64): span = page_size * 4 @@ -526,8 +526,8 @@ class TestFusedWriteLocTranslateCuda(CustomTestCase): dcp_size=dcp_size, dcp_rank=dcp_rank, ) - cpu = write_loc_to_kernel_ids(loc=loc, v2p=v2p, **kw) - gpu = write_loc_to_kernel_ids(loc=loc.cuda(), v2p=v2p.cuda(), **kw) + cpu = translate_token_ids(loc=loc, v2p=v2p, **kw) + gpu = translate_token_ids(loc=loc.cuda(), v2p=v2p.cuda(), **kw) self.assertEqual( cpu.tolist(), gpu.cpu().tolist(), @@ -542,32 +542,30 @@ class TestFusedWriteLocTranslateCuda(CustomTestCase): captured write kernel consumes the full buffer, so an uncleared tail scatters pad rows into live KV pages. """ - from sglang.kernels.ops.memory.virtual_slot import write_loc_to_kernel_ids + from sglang.kernels.ops.memory.virtual_slot import translate_token_ids v2p = torch.tensor([2, 5, 1, 3, 4], dtype=torch.int64, device="cuda") loc = torch.tensor([0, 64, 128], dtype=torch.int64, device="cuda") width = 8 # Poison the whole buffer so an unwritten or uncleared cell is visible. buf = torch.full((width,), -999, dtype=torch.int64, device="cuda") - write_loc_to_kernel_ids( - loc=loc, v2p=v2p, page_size=64, out=buf, out_width=width - ) + translate_token_ids(loc=loc, v2p=v2p, page_size=64, out=buf, out_width=width) self.assertEqual(buf[:3].tolist(), [2 * 64, 5 * 64, 1 * 64]) self.assertEqual(buf[3:].tolist(), [0] * (width - 3)) # And it must agree with the narrow call on the live prefix. - narrow = write_loc_to_kernel_ids(loc=loc, v2p=v2p, page_size=64) + narrow = translate_token_ids(loc=loc, v2p=v2p, page_size=64) self.assertEqual(narrow.tolist(), buf[:3].tolist()) def test_out_is_written_in_place(self): # The captured decode path hands in a capture-stable buffer; rebinding # instead of filling it would leave the graph on a stale pointer. - from sglang.kernels.ops.memory.virtual_slot import write_loc_to_kernel_ids + from sglang.kernels.ops.memory.virtual_slot import translate_token_ids v2p = torch.tensor([2, 5, -1, 3], dtype=torch.int64, device="cuda") loc = torch.tensor([0, 64, 128, 192], dtype=torch.int64, device="cuda") dst = torch.full_like(loc, -7) - ret = write_loc_to_kernel_ids(loc=loc, v2p=v2p, page_size=64, out=dst) + ret = translate_token_ids(loc=loc, v2p=v2p, page_size=64, out=dst) self.assertIs(ret, dst) self.assertEqual(dst.tolist(), [2 * 64, 5 * 64, 0, 3 * 64]) diff --git a/test/registered/e2e/models/test_inkling_unified.py b/test/registered/e2e/models/test_inkling_unified.py index 879b5d73d0..364c689786 100644 --- a/test/registered/e2e/models/test_inkling_unified.py +++ b/test/registered/e2e/models/test_inkling_unified.py @@ -206,8 +206,9 @@ class TestInklingUnifiedFullPrefillGraph(TestInklingUnifiedTriPool): capture outright. With capture on, both the captured block table and the SWA write loc have to come from the translator. Re-running the full->swa map on `out_cache_loc` does not work here -- it is already FULL-side - kernel-facing by then, and indexes far past the swa v2p table (a - device-side "index out of bounds" assert). + physical by then, and a full physical page number is also a valid + virtual page of the swa v2p, so the map would silently resolve it to an + unrelated token's swa slot. `test_input_output_logprobs_match` is the sharp guard: a wrong-slot SWA write moves logprobs at once, and `test_long_decode_slides_past_swa_window` keeps compaction running diff --git a/test/registered/kernels/ops/memory/test_write_loc_to_kernel_ids_strided.py b/test/registered/kernels/ops/memory/test_translate_token_ids_strided.py similarity index 93% rename from test/registered/kernels/ops/memory/test_write_loc_to_kernel_ids_strided.py rename to test/registered/kernels/ops/memory/test_translate_token_ids_strided.py index 1c7e327b1f..96ebca1572 100644 --- a/test/registered/kernels/ops/memory/test_write_loc_to_kernel_ids_strided.py +++ b/test/registered/kernels/ops/memory/test_translate_token_ids_strided.py @@ -11,13 +11,13 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""``write_loc_to_kernel_ids`` must walk `loc` and `out` by their own strides. +"""``translate_token_ids`` must walk `loc` and `out` by their own strides. The SWA read path hands it a column slice of a capture-stable page table, whose row stride is the whole buffer width; flat addressing reads the wrong ids there without any error. Only this test enters the Triton kernel -- the CPU reference never does. - python -m pytest test/registered/kernels/ops/memory/test_write_loc_to_kernel_ids_strided.py -v + python -m pytest test/registered/kernels/ops/memory/test_translate_token_ids_strided.py -v """ import itertools @@ -25,14 +25,14 @@ import unittest import torch -from sglang.kernels.ops.memory.virtual_slot import write_loc_to_kernel_ids +from sglang.kernels.ops.memory.virtual_slot import translate_token_ids from sglang.test.ci.ci_register import register_cuda_ci register_cuda_ci(est_time=10, stage="base-b-kernel-unit", runner_config="1-gpu-large") @unittest.skipUnless(torch.cuda.is_available(), "needs CUDA") -class TestWriteLocToKernelIdsStrided(unittest.TestCase): +class TestTranslateTokenIdsStrided(unittest.TestCase): ROWS, COLS, PAD = 6, 13, 11 def _v2p(self, num_pages): @@ -64,7 +64,7 @@ class TestWriteLocToKernelIdsStrided(unittest.TestCase): return locs.reshape(self.ROWS, self.COLS).to(dtype), num_pages def _run(self, loc, v2p, page_size, out=None): - return write_loc_to_kernel_ids(loc=loc, v2p=v2p, page_size=page_size, out=out) + return translate_token_ids(loc=loc, v2p=v2p, page_size=page_size, out=out) def test_a_column_slice_reads_the_same_ids_as_its_contiguous_copy(self): for page_size, dtype in itertools.product((1, 64), (torch.int64, torch.int32)): diff --git a/test/registered/unit/layers/attention/test_kv_translate_ownership.py b/test/registered/unit/layers/attention/test_kv_translate_ownership.py index 45ef311497..97882643e4 100644 --- a/test/registered/unit/layers/attention/test_kv_translate_ownership.py +++ b/test/registered/unit/layers/attention/test_kv_translate_ownership.py @@ -164,7 +164,7 @@ def _build_wrappers(translator): class TestWrapperBackendsForwardTranslator(CustomTestCase): """Bug regression: `AttentionBackend.kv_index_translator` defaults to None, so a wrapper that does not re-expose its inner's copy makes producers skip - the virtual->kernel-facing translation instead of failing.""" + the virtual->physical translation instead of failing.""" def test_every_wrapper_is_constructed_here(self): self.assertEqual( diff --git a/test/registered/unit/mem_cache/test_full_loc_fast_path.py b/test/registered/unit/mem_cache/test_full_loc_fast_path.py index b3ffb30946..d5097646a6 100644 --- a/test/registered/unit/mem_cache/test_full_loc_fast_path.py +++ b/test/registered/unit/mem_cache/test_full_loc_fast_path.py @@ -96,7 +96,7 @@ class TestUnifiedSWARouting(unittest.TestCase): def test_full_layer_falls_back_to_generic_loc(self): """Bug regression: a 2-arg `KVWriteLoc(loc, swa)` with no explicit `full_loc` must fall back to the rebound `loc` -- which IS the full-side - kernel-facing id -- instead of failing the full-layer door.""" + physical id -- instead of failing the full-layer door.""" pool = self._make_bare_pool() rebound_loc = torch.tensor([10, 11, 12], dtype=torch.int64) swa_phys = torch.tensor([1, 2, 0], dtype=torch.int64) @@ -267,7 +267,7 @@ class _RecordingMLAPool(_RecordingPool): class TestHybridLinearMLARouting(unittest.TestCase): """MLA-side door contract of `HybridLinearKVPool`: `set_mla_kv_buffer` - forwards `loc` untouched -- writes are kernel-facing since the ForwardBatch + forwards `loc` untouched -- writes are physical since the ForwardBatch rebind.""" def _make_bare_pool(self): diff --git a/test/registered/unit/mem_cache/test_kv_index_translator.py b/test/registered/unit/mem_cache/test_kv_index_translator.py index 905fcabdf1..d2db0cad35 100644 --- a/test/registered/unit/mem_cache/test_kv_index_translator.py +++ b/test/registered/unit/mem_cache/test_kv_index_translator.py @@ -339,7 +339,7 @@ class TestBuildInto(unittest.TestCase): class TestPoolOwnership(unittest.TestCase): - """A runner only gets the kernel-facing id space when the pool IT reads and + """A runner only gets the physical id space when the pool IT reads and writes is the one the allocator's ids address. Guarded shape: a runner handed a SHARED allocator while owning a SEPARATE @@ -612,7 +612,7 @@ class TestWriteLoc(CustomTestCase): keep = virt.clone() fb = _FakeForwardBatch(out_cache_loc=virt) src.rebind_write_loc(fb) - # Full side: rebound to a FRESH kernel-facing tensor; the + # Full side: rebound to a FRESH physical tensor; the # ScheduleBatch's aliased virtual tensor is untouched. self.assertIsNot(fb.out_cache_loc, virt) self.assertTrue(torch.equal(fb.out_cache_loc, want_full)) @@ -747,7 +747,7 @@ class TestWriteLoc(CustomTestCase): self.assertTrue(torch.equal(got, want_swa)) def test_pad_lanes_derive_to_sink(self): - """The DP pad appends zeros, and kernel-facing 0 is the reserved + """The DP pad appends zeros, and physical 0 is the reserved padding slot in every id space, so pad lanes derive to swa slot 0 with no `num_live` bookkeeping.""" src, _, rows, seq_lens, _, want_full, want_swa = self._built(n=3) diff --git a/test/registered/unit/mem_cache/test_multi_ended_allocator.py b/test/registered/unit/mem_cache/test_multi_ended_allocator.py index 08ce1fa013..c91f0289e5 100644 --- a/test/registered/unit/mem_cache/test_multi_ended_allocator.py +++ b/test/registered/unit/mem_cache/test_multi_ended_allocator.py @@ -2626,7 +2626,7 @@ class TestPs64MLACompositeFeasibility(unittest.TestCase): v2p = a.full_v2p_page_table want = v2p[v // self.PS] * self.PS + v % self.PS got = a.translate_kv_loc(v) - self.assertTrue(torch.equal(got, want), "kernel-facing formula broke at ps=64") + self.assertTrue(torch.equal(got, want), "physical-id formula broke at ps=64") self.assertTrue(bool((got < 2**31).all().item())) @@ -3249,7 +3249,7 @@ class TestDcpWidening(unittest.TestCase): with self._dcp(dcp_size, rank): a = self._build(page_size=2) ids = a.alloc(2 * dcp_size * 3) - written = a.translate_write_loc_for_kernel(ids) + written = a.translate_write_loc(ids) owned = (ids % dcp_size) == rank # Owned ids agree with the read translate of the collapsed id... self.assertTrue( @@ -3533,8 +3533,8 @@ class TestDcpWidening(unittest.TestCase): allocator.free_group_end() -class TestFusedWriteLocTranslate(unittest.TestCase): - """`write_loc_to_kernel_ids` must equal the arithmetic it stands for. +class TestTranslateTokenIds(unittest.TestCase): + """`translate_token_ids` must equal the arithmetic it stands for. Nothing downstream can tell the two apart except by value, so the reference here is the definition rather than a recorded expectation. Triton truncates @@ -3555,7 +3555,7 @@ class TestFusedWriteLocTranslate(unittest.TestCase): return out def _check(self, *, page_size, dcp_size, dcp_rank, device): - from sglang.kernels.ops.memory.virtual_slot import write_loc_to_kernel_ids + from sglang.kernels.ops.memory.virtual_slot import translate_token_ids span = page_size * dcp_size locs = [0, 1, span - 1, span, 2 * span + 3, 5 * span + dcp_rank, -1] @@ -3571,7 +3571,7 @@ class TestFusedWriteLocTranslate(unittest.TestCase): ) v2p[min(3, num_pages - 1)] = -1 - got = write_loc_to_kernel_ids( + got = translate_token_ids( loc=loc, v2p=v2p, page_size=page_size, @@ -3582,7 +3582,7 @@ class TestFusedWriteLocTranslate(unittest.TestCase): self.assertEqual(got.tolist(), want, f"ps={page_size} dcp={dcp_size}") # `out=` must write in place and agree (the cuda-graph-stable path). dst = torch.full_like(loc, -7) - ret = write_loc_to_kernel_ids( + ret = translate_token_ids( loc=loc, v2p=v2p, page_size=page_size, @@ -3606,7 +3606,7 @@ class TestFusedWriteLocTranslate(unittest.TestCase): def test_matches_reference_on_a_strided_view(self): """A column slice of a wider page table is what the SWA read path hands down under cuda graphs, so its ids must equal the flat reference's.""" - from sglang.kernels.ops.memory.virtual_slot import write_loc_to_kernel_ids + from sglang.kernels.ops.memory.virtual_slot import translate_token_ids rows, cols, page_size = 3, 5, 4 v2p = torch.arange(32, dtype=torch.int64, device=_DEV) @@ -3616,14 +3616,14 @@ class TestFusedWriteLocTranslate(unittest.TestCase): self.assertFalse(view.is_contiguous()) want = self._reference(view.reshape(-1).cpu(), v2p.cpu(), page_size, 1, 0) - got = write_loc_to_kernel_ids(loc=view, v2p=v2p, page_size=page_size) + got = translate_token_ids(loc=view, v2p=v2p, page_size=page_size) self.assertEqual(got.shape, view.shape) self.assertEqual(got.reshape(-1).tolist(), want) # `out=` into a slice of a DIFFERENTLY strided backing: the ids land in # the sliced columns and the rest of that buffer is left alone. dst = torch.full((rows, cols + 7), -9, dtype=torch.int64, device=_DEV) - ret = write_loc_to_kernel_ids( + ret = translate_token_ids( loc=view, v2p=v2p, page_size=page_size, @@ -3636,12 +3636,12 @@ class TestFusedWriteLocTranslate(unittest.TestCase): def test_out_width_needs_a_packed_loc(self): """`out_width` addresses a packed lane range, so a strided loc would write its tail zeros over live ids.""" - from sglang.kernels.ops.memory.virtual_slot import write_loc_to_kernel_ids + from sglang.kernels.ops.memory.virtual_slot import translate_token_ids v2p = torch.arange(16, dtype=torch.int64, device=_DEV) loc = torch.arange(12, dtype=torch.int64, device=_DEV)[::2] with self.assertRaises(AssertionError): - write_loc_to_kernel_ids( + translate_token_ids( loc=loc, v2p=v2p, page_size=1, diff --git a/test/registered/unit/mem_cache/test_unified_mla_block_table.py b/test/registered/unit/mem_cache/test_unified_mla_block_table.py index a180888d60..af3fa742f1 100644 --- a/test/registered/unit/mem_cache/test_unified_mla_block_table.py +++ b/test/registered/unit/mem_cache/test_unified_mla_block_table.py @@ -39,7 +39,7 @@ Covered here: sizes, ragged sequence lengths and a non-identity v2p permutation; - lanes past a row's live prefix keep the backend's -1 sentinel (prefix-only discipline — the trtllm/flashmla tail contract); - - the token-level kernel-facing translate the flashinfer updaters used to apply + - the token-level physical translate the flashinfer updaters used to apply agrees with the canonical page table (page-affinity of the id space); - fa3's fused metadata kernels agree with the same reference, on both the page_size == 1 fast path (which is what Kimi-Linear takes: fa3 imposes no @@ -204,7 +204,7 @@ class TestBlockTable(unittest.TestCase): kernel_tokens = ( v2p[virt_tokens // page_size] * page_size + virt_tokens % page_size ) - # The block-table entry scaled by page_size must be the kernel-facing id of + # The block-table entry scaled by page_size must be the physical id of # each page's first token. first_of_page = kernel_tokens[::page_size] n_pages = (n + page_size - 1) // page_size @@ -221,7 +221,7 @@ class TestFa3MetadataBlockTable(unittest.TestCase): (src_is_read_table=True): the fused kernel copies the canonical rows' live prefixes into the capture-stable buffer. Pinned END-TO-END: build_kv_read_table -> wrapper -> page_table must equal the python - reference of the kernel-facing formula, on both the page_size == 1 / no-SWA fast + reference of the physical-id formula, on both the page_size == 1 / no-SWA fast path (what Kimi-Linear takes) and the general kernel. The static call (no source flag) stays byte-identical to the pre-translator kernel. """ diff --git a/test/registered/unit/mem_cache/test_unified_mla_gpu_parity.py b/test/registered/unit/mem_cache/test_unified_mla_gpu_parity.py index 88cc838799..1de6853832 100644 --- a/test/registered/unit/mem_cache/test_unified_mla_gpu_parity.py +++ b/test/registered/unit/mem_cache/test_unified_mla_gpu_parity.py @@ -14,11 +14,11 @@ """GPU parity of the per-layer-view `UnifiedMLATokenToKVPool` against the stock `MLATokenToKVPool` on real K3 MLA geometry (L=24, D=512+64). -The unified pool receives kernel-facing locs (kernel_id(t) = (t//ps)*(ps*L) + -t%ps) where the reference pool receives raw token ids; every (layer, token) -cell must hold identical bytes afterwards. The TMA JIT fast path (n_loc >= 768) -flattens the buffer via `.view(shape[0], -1)`, which is legal only because the -per-layer views are contiguous. +Both pools receive the same token ids: under the token-major views a token's +physical id is its own id, so the unified pool needs no re-numbering. Every +(layer, token) cell must hold identical bytes afterwards. The TMA JIT fast path +(n_loc >= 768) flattens the buffer via `.view(shape[0], -1)`, which only needs +each row to be contiguous, as it is in a per-layer view. """ import types @@ -43,7 +43,7 @@ _DTYPE = torch.bfloat16 def _kernel_id(t: torch.Tensor, ps: int) -> torch.Tensor: - return t # token-major views: the kernel id is the physical token id + return t # token-major views: the physical id is the token id def _marked(loc: torch.Tensor): diff --git a/test/registered/unit/model_executor/test_unified_out_cache_loc_rebind.py b/test/registered/unit/model_executor/test_unified_out_cache_loc_rebind.py index 3ada5783b5..96ffd8fe4d 100644 --- a/test/registered/unit/model_executor/test_unified_out_cache_loc_rebind.py +++ b/test/registered/unit/model_executor/test_unified_out_cache_loc_rebind.py @@ -162,7 +162,7 @@ class TestPadComposesWithDerivation(CustomTestCase): class TestReadRailTranslatesAtProduction(CustomTestCase): """The model-door READ indices (req_to_token-derived, VIRTUAL under the unified pool) are translated at their PRODUCTION site -- the cache then - holds the kernel-facing result and the pool door never translates.""" + holds the physical result and the pool door never translates.""" def _fb_for_one_shot(self): fb = _make_fb(torch.tensor([1, 2], dtype=torch.int64)) diff --git a/test/registered/unit/server_args/test_page_major_backend_allowlist.py b/test/registered/unit/server_args/test_page_major_backend_allowlist.py index 6d71695b71..60d9e6b5fa 100644 --- a/test/registered/unit/server_args/test_page_major_backend_allowlist.py +++ b/test/registered/unit/server_args/test_page_major_backend_allowlist.py @@ -100,7 +100,7 @@ class TestPageMajorBackendAllowlist(unittest.TestCase): PER_LAYER_VIEW_MHA_BACKENDS = ("fa3", "fa4", "flashinfer", "trtllm_mha") # MLA-family kernels that must never leak into the MHA arm. MLA_ONLY_BACKENDS = ("trtllm_mla", "cutedsl_mla", "tokenspeed_mla", "flashmla") - # No kernel-facing-id wiring anywhere: must stay rejected until they get one. + # No virtual-to-physical id wiring anywhere: must stay rejected until they get one. UNWIRED_BACKENDS = ("aiter",) def test_triton_allowed_on_every_arm(self): @@ -170,7 +170,7 @@ class TestPageMajorBackendAllowlist(unittest.TestCase): for use_mla in (True, False): self.assertFalse( _accepts(backend, use_mla=use_mla), - f"{backend} has no kernel-facing-id wiring and must be rejected", + f"{backend} has no virtual-to-physical id wiring and must be rejected", ) def test_helion_linear_attention_is_kda_only(self): diff --git a/test/registered/unit/server_args/test_unified_prefill_cuda_graph_gate.py b/test/registered/unit/server_args/test_unified_prefill_cuda_graph_gate.py index df2e375270..bb20ded244 100644 --- a/test/registered/unit/server_args/test_unified_prefill_cuda_graph_gate.py +++ b/test/registered/unit/server_args/test_unified_prefill_cuda_graph_gate.py @@ -14,7 +14,7 @@ """`--enable-unified-memory` and PREFILL cuda-graph capture. Capture is wired: the captured batch reads `out_cache_loc` out of the registry -slot, refilled before each replay from the already-rebound kernel-facing loc, +slot, refilled before each replay from the already-rebound physical loc, and the read tables are refilled out-of-graph from the live v2p. So BREAKABLE (the CUDA default) and TC_PIECEWISE must be left alone -- an earlier gate disabled every prefill backend outright, which cost every unified run its