From 8b80808366bb9ea476b1b2d0be09aa36b7fb6cd2 Mon Sep 17 00:00:00 2001 From: caihuali95 <42954765+caihuali95@users.noreply.github.com> Date: Thu, 8 Oct 2026 00:54:19 -0700 Subject: [PATCH] [unified-memory] Fuse the draft model's KV into the target's page envelope (mechanism) (#37628) Co-authored-by: Caihua Li Co-authored-by: Cheng Wan --- .../kernels/ops/speculative/cache_locs.py | 22 ++ .../srt/layers/attention/aiter_backend.py | 4 + .../attention/flashattention_backend.py | 1 + .../layers/attention/flashinfer_backend.py | 4 + .../attention/flashinfer_mla_backend.py | 4 + .../srt/layers/attention/flashmla_backend.py | 1 + .../srt/layers/attention/triton_backend.py | 4 + .../srt/mem_cache/kv_cache_configurator.py | 186 ++++++++- .../srt/mem_cache/kv_index_translator.py | 22 +- .../srt/mem_cache/layout/fused_draft.py | 262 +++++++++++++ .../srt/mem_cache/layout/token_major.py | 5 + .../srt/mem_cache/unified_draft_pool.py | 159 ++++++++ .../srt/mem_cache/unified_memory_pool.py | 133 +++++-- .../spec_aux_hidden_state.py | 45 ++- .../srt/model_executor/pool_configurator.py | 48 ++- .../srt/speculative/base_spec_worker.py | 9 +- .../sglang/srt/speculative/eagle_worker_v2.py | 9 + .../multi_layer_eagle_worker_v2.py | 7 + .../speculative/test_draft_decode_window.py | 1 + ...erate_draft_decode_kv_indices_translate.py | 172 ++++++++ .../speculative/test_spec_kv_indices_grid.py | 1 + .../test_dense_fused_draft_geometry.py | 308 +++++++++++++++ .../mem_cache/test_fused_draft_placement.py | 241 ++++++++++++ .../test_kv_cache_configurator_draft_gate.py | 249 ++++++++++++ ...t_kv_index_translator_draft_disposition.py | 369 ++++++++++++++++++ .../model_executor/test_pool_configurator.py | 104 +++++ .../test_spec_aux_hidden_state_boot_order.py | 191 +++++++++ .../test_eagle_worker_v2_topk1_fastpath.py | 2 + 28 files changed, 2506 insertions(+), 57 deletions(-) create mode 100644 python/sglang/srt/mem_cache/layout/fused_draft.py create mode 100644 python/sglang/srt/mem_cache/unified_draft_pool.py create mode 100644 test/registered/kernels/ops/speculative/test_generate_draft_decode_kv_indices_translate.py create mode 100644 test/registered/unit/mem_cache/test_dense_fused_draft_geometry.py create mode 100644 test/registered/unit/mem_cache/test_fused_draft_placement.py create mode 100644 test/registered/unit/mem_cache/test_kv_cache_configurator_draft_gate.py create mode 100644 test/registered/unit/mem_cache/test_kv_index_translator_draft_disposition.py create mode 100644 test/registered/unit/model_executor/test_spec_aux_hidden_state_boot_order.py diff --git a/python/sglang/kernels/ops/speculative/cache_locs.py b/python/sglang/kernels/ops/speculative/cache_locs.py index abf8bbb612..4d1f93d261 100644 --- a/python/sglang/kernels/ops/speculative/cache_locs.py +++ b/python/sglang/kernels/ops/speculative/cache_locs.py @@ -78,6 +78,17 @@ def assign_draft_cache_locs_contiguous( ) +@triton.jit +def _translate_token_ids(ids, v2p, mask, page_size: tl.constexpr): + """Virtual token ids -> physical ones through the page-level v2p table, + as `translate_kv_loc` does. A negative id maps to the page-0 sink instead + of loading before `v2p`.""" + i64 = ids.to(tl.int64) + vpage = tl.where(i64 < 0, 0, i64 // page_size) + phys = tl.load(v2p + vpage, mask=mask, other=0) + return tl.where(i64 < 0, 0, tl.maximum(phys * page_size + i64 % page_size, 0)) + + @triton.jit def generate_draft_decode_kv_indices( req_pool_indices, @@ -86,6 +97,7 @@ def generate_draft_decode_kv_indices( kv_indices, kv_indptr, positions, + v2p, pool_len: tl.constexpr, kv_indices_stride: tl.constexpr, kv_indptr_stride: tl.constexpr, @@ -96,6 +108,7 @@ def generate_draft_decode_kv_indices( window_size: tl.constexpr = 0, sink_size: tl.constexpr = 0, NUM_STEPS: tl.constexpr = 0, + TRANSLATE: tl.constexpr = False, ): # window_size > 0 restricts the draft (not the target) to sink_size prefix # tokens + the most-recent window_size; window_size == 0 is the identity. @@ -153,6 +166,8 @@ def generate_draft_decode_kv_indices( recent_start + copy_offset - s_eff, ) data = tl.load(token_pool_ptr + src, mask=mask) + if TRANSLATE: + data = _translate_token_ids(data, v2p, mask, page_size) tl.store(kv_ptr + copy_offset, data, mask=mask) copy_offset += BLOCK_SIZE else: @@ -165,6 +180,8 @@ def generate_draft_decode_kv_indices( recent_start + copy_offset - s_eff, ) data = tl.load(token_pool_ptr + src, mask=mask) + if TRANSLATE: + data = _translate_token_ids(data, v2p, mask, page_size) tl.store(kv_ptr + copy_offset, data, mask=mask) # Extension entries and kv_indptr belong to token block 0 alone; other @@ -196,6 +213,11 @@ def generate_draft_decode_kv_indices( mask=extend_offset < iters, ) + if TRANSLATE: + extend_data = _translate_token_ids( + extend_data, v2p, extend_offset < iters, page_size + ) + tl.store( kv_ptr + seq_len_w + extend_offset, extend_data, diff --git a/python/sglang/srt/layers/attention/aiter_backend.py b/python/sglang/srt/layers/attention/aiter_backend.py index f3537946a0..8080934139 100755 --- a/python/sglang/srt/layers/attention/aiter_backend.py +++ b/python/sglang/srt/layers/attention/aiter_backend.py @@ -4647,6 +4647,7 @@ class AiterMultiStepDraftBackend: self.req_to_token_pool = model_runner.req_to_token_pool self.pool_len = model_runner.req_to_token_pool.req_to_token.shape[1] self.page_size = get_schedule().page_size + self.kv_index_translator = model_runner.kv_index_translator def common_template( self, forward_batch: ForwardBatch, kv_indices_buffer: torch.Tensor, call_fn: int @@ -4662,6 +4663,7 @@ class AiterMultiStepDraftBackend: if self.max_context_len >= _KV_INDEX_BLOCKS_MIN_CONTEXT else 1 ) + v2p = self.kv_index_translator.full_flat_v2p() self.generate_draft_decode_kv_indices[ (self.speculative_num_steps * num_token_blocks, num_seqs, self.topk) ]( @@ -4671,6 +4673,7 @@ class AiterMultiStepDraftBackend: kv_indices_buffer, self.kv_indptr, forward_batch.positions, + v2p, self.pool_len, kv_indices_buffer.shape[1], self.kv_indptr.shape[1], @@ -4681,6 +4684,7 @@ class AiterMultiStepDraftBackend: # A single token block is the historical launch; NUM_STEPS=0 keeps # its 128-wide program instead of the token-block specialization. NUM_STEPS=self.speculative_num_steps if num_token_blocks > 1 else 0, + TRANSLATE=v2p is not None, ) for i in range(self.speculative_num_steps - 1): diff --git a/python/sglang/srt/layers/attention/flashattention_backend.py b/python/sglang/srt/layers/attention/flashattention_backend.py index d0c2f40390..05ba0aca40 100644 --- a/python/sglang/srt/layers/attention/flashattention_backend.py +++ b/python/sglang/srt/layers/attention/flashattention_backend.py @@ -3396,6 +3396,7 @@ class FlashAttentionMultiStepBackend: self.model_runner = model_runner self.topk = topk self.speculative_num_steps = speculative_num_steps + self.kv_index_translator = model_runner.kv_index_translator self.attn_backends = [] for i in range(self.speculative_num_steps - 1): self.attn_backends.append( diff --git a/python/sglang/srt/layers/attention/flashinfer_backend.py b/python/sglang/srt/layers/attention/flashinfer_backend.py index d9765566cd..5fb4ca1ae7 100644 --- a/python/sglang/srt/layers/attention/flashinfer_backend.py +++ b/python/sglang/srt/layers/attention/flashinfer_backend.py @@ -2369,6 +2369,7 @@ class FlashInferMultiStepDraftBackend: self.draft_window_size, self.draft_sink_size = resolve_draft_decode_window( model_runner ) + self.kv_index_translator = model_runner.kv_index_translator def common_template( self, @@ -2391,6 +2392,7 @@ class FlashInferMultiStepDraftBackend: seq_lens_sum=seq_lens_sum, ) + v2p = self.kv_index_translator.full_flat_v2p() self.generate_draft_decode_kv_indices[ (self.speculative_num_steps, num_seqs, self.topk) ]( @@ -2400,6 +2402,7 @@ class FlashInferMultiStepDraftBackend: kv_indices_buffer, self.kv_indptr, forward_batch.positions, + v2p, self.pool_len, kv_indices_buffer.shape[1], self.kv_indptr.shape[1], @@ -2409,6 +2412,7 @@ class FlashInferMultiStepDraftBackend: self.page_size, self.draft_window_size, self.draft_sink_size, + TRANSLATE=v2p is not None, ) assert forward_batch.spec_info is not None diff --git a/python/sglang/srt/layers/attention/flashinfer_mla_backend.py b/python/sglang/srt/layers/attention/flashinfer_mla_backend.py index 64023c962d..6a68be22f2 100644 --- a/python/sglang/srt/layers/attention/flashinfer_mla_backend.py +++ b/python/sglang/srt/layers/attention/flashinfer_mla_backend.py @@ -1218,6 +1218,7 @@ class FlashInferMLAMultiStepDraftBackend: self.req_to_token_pool = model_runner.req_to_token_pool self.pool_len = model_runner.req_to_token_pool.req_to_token.shape[1] self.page_size = get_schedule().page_size + self.kv_index_translator = model_runner.kv_index_translator def common_template( self, @@ -1240,6 +1241,7 @@ class FlashInferMLAMultiStepDraftBackend: seq_lens_sum=seq_lens_sum, ) + v2p = self.kv_index_translator.full_flat_v2p() self.generate_draft_decode_kv_indices[ (self.speculative_num_steps, num_seqs, self.topk) ]( @@ -1249,6 +1251,7 @@ class FlashInferMLAMultiStepDraftBackend: kv_indices_buffer, self.kv_indptr, forward_batch.positions, + v2p, self.pool_len, kv_indices_buffer.shape[1], self.kv_indptr.shape[1], @@ -1256,6 +1259,7 @@ class FlashInferMLAMultiStepDraftBackend: next_power_of_2(self.speculative_num_steps), next_power_of_2(bs), self.page_size, + TRANSLATE=v2p is not None, ) assert forward_batch.spec_info is not None diff --git a/python/sglang/srt/layers/attention/flashmla_backend.py b/python/sglang/srt/layers/attention/flashmla_backend.py index 26f99b997b..57a7bc0b02 100644 --- a/python/sglang/srt/layers/attention/flashmla_backend.py +++ b/python/sglang/srt/layers/attention/flashmla_backend.py @@ -612,6 +612,7 @@ class FlashMLAMultiStepDraftBackend: ) self.topk = topk self.speculative_num_steps = speculative_num_steps + self.kv_index_translator = model_runner.kv_index_translator max_bs = model_runner.req_to_token_pool.size * self.topk self.kv_indptr = torch.zeros( ( diff --git a/python/sglang/srt/layers/attention/triton_backend.py b/python/sglang/srt/layers/attention/triton_backend.py index e9b1fd99c6..0f7d8624bc 100644 --- a/python/sglang/srt/layers/attention/triton_backend.py +++ b/python/sglang/srt/layers/attention/triton_backend.py @@ -2449,6 +2449,7 @@ class TritonMultiStepDraftBackend: self.draft_window_size, self.draft_sink_size = resolve_draft_decode_window( model_runner ) + self.kv_index_translator = model_runner.kv_index_translator def common_template( self, @@ -2467,6 +2468,7 @@ class TritonMultiStepDraftBackend: # over-estimate is safe. Use a static UB to skip the per-iter .sum().item() D2H. seq_lens_sum = num_seqs * self.max_context_len + v2p = self.kv_index_translator.full_flat_v2p() generate_draft_decode_kv_indices[ (self.speculative_num_steps, num_seqs, self.topk) ]( @@ -2476,6 +2478,7 @@ class TritonMultiStepDraftBackend: kv_indices_buffer, self.kv_indptr, forward_batch.positions, + v2p, self.pool_len, kv_indices_buffer.shape[1], self.kv_indptr.shape[1], @@ -2485,6 +2488,7 @@ class TritonMultiStepDraftBackend: self.page_size, self.draft_window_size, self.draft_sink_size, + TRANSLATE=v2p is not None, ) if call_fn is None: diff --git a/python/sglang/srt/mem_cache/kv_cache_configurator.py b/python/sglang/srt/mem_cache/kv_cache_configurator.py index 54521816d6..a35f72ee7c 100644 --- a/python/sglang/srt/mem_cache/kv_cache_configurator.py +++ b/python/sglang/srt/mem_cache/kv_cache_configurator.py @@ -570,16 +570,45 @@ class KVCacheConfigurator: UnifiedSWAAllocatorBase, ), ): - draft_virtual_id_space = ( - token_to_kv_pool_allocator.draft_virtual_id_space - ) + alloc = token_to_kv_pool_allocator + placement = self._fused_draft_from_target_buffer(alloc) + if placement is not None: + # FUSED arm + from sglang.srt.mem_cache.layout.fused_draft import ( + draft_swa_layer_ids, + ) + from sglang.srt.mem_cache.unified_draft_pool import ( + bind_fused_draft, + draft_kv_layer_ids, + ) + + assert req_to_token_pool is not None, ( + "a draft worker shares the target's req_to_token_pool" + ) + draft_pool = bind_fused_draft( + unified_buffer=alloc.unified_buffer, + host_allocator=alloc, + placement=placement, + runner=self.draft_model_idx or 0, + kv_layer_ids=draft_kv_layer_ids(self.model), + swa_layer_ids=draft_swa_layer_ids(self.model_config), + page_size=self.page_size, + ) + return _InitializedPools( + req_to_token_pool=req_to_token_pool, + token_to_kv_pool=draft_pool, + token_to_kv_pool_allocator=alloc, + unified_memory_pool=None, + ) + # PRIVATE arm + draft_virtual_id_space = alloc.draft_virtual_id_space assert draft_virtual_id_space >= sizes.max_total_num_tokens, ( "unified allocator virtual space smaller than the token " f"budget: virtual_id_space={draft_virtual_id_space} < " f"max_total_num_tokens={sizes.max_total_num_tokens}" ) # Round UP to page alignment (paged draft backends view the - # pool as (-1, page_size, H, D); the virtual space is not aligned). + # pool as (-1, page_size, H, D); the space is not aligned). page = max(int(self.pool_page_size or 1), 1) draft_virtual_id_space = ( (draft_virtual_id_space + page - 1) // page * page @@ -684,6 +713,33 @@ class KVCacheConfigurator: token_to_kv_pool_allocator=token_to_kv_pool_allocator, ) + def _fused_draft_from_target_buffer(self, alloc): + """The placement this draft binds to, or None for the private-pool + fallback (the draft then owns a raw-virtual-indexed pool of its own). + Target boot declines a placement for legitimate geometry, so the + private arm is the rollback lever, not a boot-order bug.""" + if not self.spec_algorithm.is_eagle(): + return None + from sglang.srt.mem_cache.unified_memory_pool import _store_dtype_for + + placement = alloc.unified_buffer.fused_draft + if placement is None: + logger.info( + "[unified-memory-pool] no fused draft placement on the target's " + "buffer; the draft binds a private pool over the virtual id space." + ) + return None + # The region stores rows in the target's KV dtype; a draft that resolved + # its own would read and write them as something else. + if _store_dtype_for(self.kv_cache_dtype) != placement.region.store_dtype: + raise ValueError( + f"Fused draft KV: the draft resolved its KV cache dtype to " + f"{self.kv_cache_dtype}, but its region inside the target's pages " + f"stores {placement.region.store_dtype}. Set " + "--speculative-draft-kv-cache-dtype to the target's KV cache dtype." + ) + return placement + def _init_unified_mamba_pools( self, *, @@ -879,6 +935,127 @@ class KVCacheConfigurator: ), ) + def _fused_draft_decision(self): + """Whether, and where, the EAGLE draft's layers fuse into the target's + sub-pools. Fusion applies only for: unified memory ON, a two-pool + hybrid-SWA target, an EAGLE-family algorithm whose draft config was + loaded at target boot and has no recurrent state of its own; + `place_fused_draft` then admits or declines the draft's layer kinds.""" + from sglang.srt.mem_cache.layout.fused_draft import ( + FusedDraftDecision, + draft_kv_profile, + place_fused_draft, + ) + from sglang.srt.mem_cache.unified_memory_pool import _store_dtype_for + + aux = self.spec_aux_config + if not ( + get_memory().enable_unified_memory + and self.is_hybrid_swa + and self.mambaish_config is None + and not self.is_draft_worker + and self.spec_algorithm.is_eagle() + and aux.draft_kv_num_layers + and aux.draft_model_config is not None + and mambaish_config(aux.draft_model_config) is None + ): + return FusedDraftDecision() + profile = draft_kv_profile( + aux.draft_model_config, + num_layers=int(aux.draft_kv_num_layers), + attn_tp_size=get_parallel().attn_tp_size, + ) + num_runners = ( + int(get_spec().speculative_num_steps) + if self.model_config.is_multi_layer_eagle + else 1 + ) + # A fused draft stores its rows in the host's KV dtype. An explicit + # draft dtype resolves without the draft model, so a mismatch is + # declined here; `auto` follows the draft's quant config, which only + # the draft runner knows, and is checked when it binds. + draft_kv_cache_dtype = get_spec().speculative_draft_kv_cache_dtype + if draft_kv_cache_dtype not in (None, "auto"): + from sglang.srt.mem_cache.kv_cache_dtype import configure_kv_cache_dtype + + _, draft_kv_dtype = configure_kv_cache_dtype( + server_args_kv_cache_dtype=draft_kv_cache_dtype, + model=None, + model_dtype=aux.draft_model_config.dtype, + is_draft_worker=True, + is_dflash=False, + speculative_draft_attention_backend=None, + speculative_draft_kv_cache_dtype=draft_kv_cache_dtype, + ) + if draft_kv_dtype != self.kv_cache_dtype: + return FusedDraftDecision( + declined=( + f"the draft's KV cache dtype ({draft_kv_dtype}) differs " + f"from the host's ({self.kv_cache_dtype})" + ) + ) + return place_fused_draft( + profile=profile, + num_runners=num_runners, + store_dtype=_store_dtype_for(self.kv_cache_dtype), + ) + + def fused_entry_bytes(self, sub_pool_name: str) -> Optional[int]: + """Per-token bytes of ``sub_pool_name``'s FUSED entry (host + draft + + pad) for the boot solve's cell model. Single source of truth: assembled + through the same spec the pool factory builds, so the priced entry and + the allocated entry cannot drift. None when the draft does not fuse + into that sub-pool.""" + placement = self._fused_draft_decision().placement + if placement is None or sub_pool_name != "full": + return None + return self._full_host_spec(placement.region).entry_bytes() + + def _full_host_spec(self, region): + from sglang.srt.mem_cache.unified_memory_pool import ( + MHASubPoolSpec, + _store_dtype_for, + ) + + return MHASubPoolSpec( + name="full", + layer_num=len(self.layer_info.full_attention_layer_ids), + head_num=self.model_config.get_num_kv_heads( + get_parallel().attn_tp_size, get_parallel().attn_dcp_size + ), + head_dim=self.model_config.head_dim, + store_dtype=_store_dtype_for(self.kv_cache_dtype), + grow_direction="down", + draft_region=region, + ) + + def _fused_draft_for_pool_factory(self): + """Resolve ONCE per factory call (not inline) so the boot log reports + exactly the placement the factory is handed: a declined fusion and an + engaged one otherwise look identical from outside.""" + decision = self._fused_draft_decision() + if decision.placement is None: + if decision.declined is not None: + logger.warning("fused draft KV disabled: %s", decision.declined) + return None + placement = decision.placement + region = placement.region + logger.info( + "[unified-memory-pool] fused draft region in 'full': %d lane(s) x %d " + "kv head(s) x %d/%d k/v head_dim @ %s = %d B/token; runner lanes %s", + region.lane_num, + region.head_num, + region.head_dim, + region.resolved_v_head_dim(), + region.store_dtype, + region.entry_bytes(), + [ + tuple(placement.lanes_for(r)) + for r in range(len(placement.runner_lane_counts)) + ], + ) + return placement + def _init_unified_swa_pools( self, *, @@ -964,6 +1141,7 @@ class KVCacheConfigurator: # charged, see `_check_bs1_feasibility_floor`. model_context_len=self.model_config.context_len, sliding_window_size=self.model_config.sliding_window_size, + fused_draft=self._fused_draft_for_pool_factory(), ) return UnifiedPoolBundle( unified_memory_pool=bundle.unified_memory_pool, diff --git a/python/sglang/srt/mem_cache/kv_index_translator.py b/python/sglang/srt/mem_cache/kv_index_translator.py index 99691e095d..6b8bffd7d2 100644 --- a/python/sglang/srt/mem_cache/kv_index_translator.py +++ b/python/sglang/srt/mem_cache/kv_index_translator.py @@ -71,6 +71,7 @@ from sglang.srt.mem_cache.allocator.unified_mamba import ( ) from sglang.srt.mem_cache.base_swa_memory_pool import BaseSWAKVPool from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool +from sglang.srt.mem_cache.unified_draft_pool import fused_draft_host_allocator from sglang.srt.runtime_context import get_parallel @@ -112,13 +113,18 @@ class KVIndexTranslator: self.page_size = page_size self.device = device - self.is_translating = ( + is_unified_target = ( isinstance( token_to_kv_pool_allocator, (UnifiedMambaTokenToKVPoolAllocator, UnifiedSWAAllocatorBase), ) and token_to_kv_pool_allocator.get_kvcache() is token_to_kv_pool ) + host_allocator = fused_draft_host_allocator(token_to_kv_pool) + is_fused_draft = ( + host_allocator is not None and host_allocator is token_to_kv_pool_allocator + ) + self.is_translating = is_unified_target or is_fused_draft if self.is_translating: alloc = token_to_kv_pool_allocator self._capture_page_size = alloc.page_size @@ -133,13 +139,17 @@ class KVIndexTranslator: # DCP read ids stay WIDENED to the consumer: selecting this rank's # share changes the length, so only the production site can do it. self.defer_read_translate = get_parallel().attn_dcp_size > 1 - if isinstance(alloc, UnifiedSWAAllocatorBase): + routes_window_layers = is_unified_target or isinstance( + token_to_kv_pool, BaseSWAKVPool + ) + if isinstance(alloc, UnifiedSWAAllocatorBase) and routes_window_layers: self._swa_v2p_table = alloc.swa_v2p_page_table self._swa_write_loc_from_full = self._swa_write_loc_unified else: self._swa_v2p_table = None self._swa_write_loc_from_full = None else: + self._capture_page_size = page_size self._full_v2p_table = None self._full_p2v_table = None self._translate_full = None @@ -175,6 +185,14 @@ class KVIndexTranslator: return self._full_v2p_table.numel() * self._capture_page_size return max_token_pool_size + self.page_size + def full_flat_v2p(self) -> Optional[torch.Tensor]: + """The full-side v2p page table for a kernel that translates flat ids + itself, or ``None`` when this runner does not translate (pass-through + and static pools).""" + if not self.is_translating: + return None + return self._full_v2p_table + # -- per-batch view -------------------------------------------------------- @property diff --git a/python/sglang/srt/mem_cache/layout/fused_draft.py b/python/sglang/srt/mem_cache/layout/fused_draft.py new file mode 100644 index 0000000000..0936606c25 --- /dev/null +++ b/python/sglang/srt/mem_cache/layout/fused_draft.py @@ -0,0 +1,262 @@ +"""Draft-model regions nested inside unified sub-pools' slot entries. + +The nesting, outermost first: + + sub-pool a named pool inside the unified pool, with its own grow direction + and frontier. HOST is the role it plays when its entries also + carry draft bytes; only "full" hosts a draft here. + slot one token entry of that sub-pool. The draft's KV becomes extra + parts of every slot, after the host's own parts at + `draft_offset_in_entry()`, so ONE slot id per token covers target + and draft: one allocation, one free, one whole-page relocation + carry both. + region the draft geometry inside the host's entries. A region is not a + `SubPoolSpec`: no grow direction, no frontier logic, only geometry. + lane one position in the region's layer dimension -- one draft layer of + one runner, present in every slot. + +Runners are the draft's execution copies, and the two head shapes place them +oppositely. A replicated head gives EVERY runner all the draft's layers, so +each needs its own lanes or the runners clobber each other's KV (3 runners x +2 layers = 6 lanes). A per-depth head (depth 0 predicts the next token, +depth 1 the one after, and so on) gives each runner ONE depth (8 layers +across 2 runners = 2 lanes). Lane count is therefore not layer count, in +either direction. Each runner owns a contiguous run of lanes, and the runs +tile the region in runner order. + +Three phases, three moments in boot, and what carries each: + + PLAN on the TARGET, before any memory exists + KVCacheConfigurator._fused_draft_decision() + draft_kv_profile(draft_model_config) what the checkpoint asks for + place_fused_draft(profile, runners) admit, or decline with a reason + KVCacheConfigurator.fused_entry_bytes() prices it for the boot solve + + CARVE as the byte buffer is cut + KVCacheConfigurator._fused_draft_for_pool_factory() resolve once + init_unified_*_pools(fused_draft=...) + MHASubPoolSpec.draft_offset_in_entry() / .entry_bytes() + UnifiedKVPool.build_dense_draft_views() + + BIND on each DRAFT runner, built after the target + KVCacheConfigurator._fused_draft_from_target_buffer(alloc) + bind_fused_draft(...) -> UnifiedDraftKVPool + KVIndexTranslator: fused_draft_host_allocator(pool) is alloc -> translate + +A draft whose layers need another sub-pool kind declines to a private pool. +""" + +from typing import List, Optional, Tuple + +import msgspec +import torch + +from sglang.srt.mem_cache.layout.token_major import ROW_ALIGN_BYTES, DensePart + + +class DenseDraftRegion(msgspec.Struct, frozen=True, kw_only=True): + """Geometry of the DRAFT model's K/V rows fused into every slot of a host + sub-pool. The draft is a separate checkpoint, so its head geometry and + layer count differ from the host's; its K and V rows are two more parts of + the host entry, indexed by the host's physical token id.""" + + lane_num: int + head_num: int + head_dim: int + store_dtype: torch.dtype + v_head_dim: Optional[int] = None + + def validate(self) -> None: + assert self.lane_num > 0, f"lane_num must be positive; got {self.lane_num}" + assert self.head_num > 0, f"head_num must be positive; got {self.head_num}" + assert self.head_dim > 0, f"head_dim must be positive; got {self.head_dim}" + v = self.resolved_v_head_dim() + assert v > 0, f"v_head_dim must be positive; got {v}" + + def resolved_v_head_dim(self) -> int: + return self.head_dim if self.v_head_dim is None else self.v_head_dim + + def k_row_bytes(self) -> int: + return self.head_num * self.head_dim * self.store_dtype.itemsize + + def v_row_bytes(self) -> int: + return self.head_num * self.resolved_v_head_dim() * self.store_dtype.itemsize + + def entry_bytes(self) -> int: + """Draft bytes per slot, before the host entry's alignment.""" + return self.lane_num * (self.k_row_bytes() + self.v_row_bytes()) + + def parts(self, offset_bytes: int) -> Tuple[DensePart, DensePart]: + """The draft's K and V parts, laid out from ``offset_bytes`` inside + the host entry.""" + layer_stride = self.k_row_bytes() + self.v_row_bytes() + return ( + DensePart( + name="draft_k", + offset_bytes=offset_bytes, + layer_stride_bytes=layer_stride, + layer_num=self.lane_num, + row_shape=(self.head_num, self.head_dim), + dtype=self.store_dtype, + ), + DensePart( + name="draft_v", + offset_bytes=offset_bytes + self.k_row_bytes(), + layer_stride_bytes=layer_stride, + layer_num=self.lane_num, + row_shape=(self.head_num, self.resolved_v_head_dim()), + dtype=self.store_dtype, + ), + ) + + +class FusedDraftPlacement(msgspec.Struct, frozen=True, kw_only=True): + """Where every draft runner's layers live inside the "full" sub-pool: the + region, and each runner's lane count in runner order. Built once on the + target, stored on the `UnifiedKVPool`, and read back by each draft + runner, so the two sides cannot disagree on a lane. + """ + + region: DenseDraftRegion + runner_lane_counts: Tuple[int, ...] + + def __post_init__(self): + assert self.runner_lane_counts, "a placement needs at least one draft runner" + assert sum(self.runner_lane_counts) == self.region.lane_num, ( + f"runner lane counts {self.runner_lane_counts} must tile " + f"range({self.region.lane_num})" + ) + + def lanes_for(self, runner: int) -> range: + start = sum(self.runner_lane_counts[:runner]) + return range(start, start + self.runner_lane_counts[runner]) + + +class DraftKVGeometry(msgspec.Struct, frozen=True, kw_only=True): + """Per-GPU K/V row geometry of one kind of draft attention layer.""" + + head_num: int + head_dim: int + v_head_dim: int + + +class DraftKVProfile(msgspec.Struct, frozen=True, kw_only=True): + """What the draft checkpoint asks of the host sub-pools. + + ``num_depths`` > 1 marks a per-depth head (one transformer block per MTP + depth, served by one runner each under multi-layer EAGLE); otherwise + every runner serves all ``num_layers`` layers. + """ + + num_layers: int + full: DraftKVGeometry + swa_layer_ids: Tuple[int, ...] = () + num_depths: int = 1 + + +def draft_swa_layer_ids(draft_model_config) -> Tuple[int, ...]: + """The draft layer ids a hybrid-SWA pool routes to its swa side. This is + the pool-routing convention (`SWAKVPool.layers_mapping`), not a per-layer + kernel window: an attention-only draft window stays full-kind.""" + mc = draft_model_config + if mc.is_hybrid_swa and not mc.is_deepseek_v4_arch: + return tuple(int(i) for i in mc.swa_attention_layer_ids) + return () + + +def draft_kv_profile( + draft_model_config, *, num_layers: int, attn_tp_size: int +) -> DraftKVProfile: + """The profile of a draft `ModelConfig`, heads divided by attn_tp the way + the target divides its own (drafts never join the DCP group).""" + mc = draft_model_config + num_depths = mc.num_nextn_predict_layers + return DraftKVProfile( + num_layers=int(num_layers), + full=DraftKVGeometry( + head_num=int(mc.get_num_kv_heads(attn_tp_size)), + head_dim=int(mc.head_dim), + v_head_dim=int(mc.v_head_dim), + ), + swa_layer_ids=draft_swa_layer_ids(mc), + num_depths=1 if num_depths is None else int(num_depths), + ) + + +class FusedDraftDecision(msgspec.Struct, frozen=True, kw_only=True): + """`place_fused_draft`'s answer: the placement, or why the draft keeps a + private pool. Neither means fusion simply does not apply.""" + + placement: Optional[FusedDraftPlacement] = None + declined: Optional[str] = None + + +def _runner_layer_counts( + profile: DraftKVProfile, num_runners: int +) -> Tuple[Optional[List[Tuple[int, int]]], Optional[str]]: + """Per runner, its (full, swa) layer counts; or why no runner layout exists.""" + if profile.num_depths <= 1: + num_swa = len(profile.swa_layer_ids) + return [(profile.num_layers - num_swa, num_swa)] * num_runners, None + if num_runners == 1: + return None, ( + f"a per-depth draft head ({profile.num_depths} depths) needs one " + "runner per depth (multi-layer EAGLE)" + ) + if num_runners > profile.num_depths: + return None, ( + f"{num_runners} draft runners exceed the head's {profile.num_depths} depths" + ) + swa = set(profile.swa_layer_ids) + return [(0, 1) if r in swa else (1, 0) for r in range(num_runners)], None + + +def place_fused_draft( + *, + profile: DraftKVProfile, + num_runners: int, + store_dtype: torch.dtype, +) -> FusedDraftDecision: + """Assign every draft layer of every runner to the host sub-pool whose + lifetime covers what the layer reads: a full-attention layer rides in + ``"full"``. A layer kind no host arm serves declines the whole draft to + its private pool.""" + counts, reason = _runner_layer_counts(profile, num_runners) + if counts is None: + return FusedDraftDecision(declined=reason) + num_swa = sum(swa for _, swa in counts) + if num_swa: + return FusedDraftDecision( + declined=( + f"the draft has {num_swa} SWA layer(s) of its own, which the " + "fused dense pool cannot serve" + ) + ) + geometry = profile.full + if geometry.head_dim != geometry.v_head_dim: + return FusedDraftDecision( + declined=( + "the draft's K/V rows are asymmetric " + f"(head_dim={geometry.head_dim}, v_head_dim={geometry.v_head_dim}), " + "which is not admitted yet" + ) + ) + row_bytes = geometry.head_num * geometry.head_dim * store_dtype.itemsize + if row_bytes % ROW_ALIGN_BYTES: + return FusedDraftDecision( + declined=( + f"the draft's K/V rows are {row_bytes} B, not a multiple of the " + f"{ROW_ALIGN_BYTES}-B row alignment a fused entry part needs" + ) + ) + full_counts = tuple(full for full, _ in counts) + region = DenseDraftRegion( + lane_num=sum(full_counts), + head_num=geometry.head_num, + head_dim=geometry.head_dim, + v_head_dim=geometry.v_head_dim, + store_dtype=store_dtype, + ) + return FusedDraftDecision( + placement=FusedDraftPlacement(region=region, runner_lane_counts=full_counts) + ) diff --git a/python/sglang/srt/mem_cache/layout/token_major.py b/python/sglang/srt/mem_cache/layout/token_major.py index 98322d7f84..351e864436 100644 --- a/python/sglang/srt/mem_cache/layout/token_major.py +++ b/python/sglang/srt/mem_cache/layout/token_major.py @@ -43,6 +43,11 @@ def align_entry_bytes(num_bytes: int) -> int: return -(-num_bytes // ENTRY_ALIGN_BYTES) * ENTRY_ALIGN_BYTES +def align_part_offset(num_bytes: int) -> int: + """Round a part's byte offset inside the entry up to the row alignment.""" + return -(-num_bytes // ROW_ALIGN_BYTES) * ROW_ALIGN_BYTES + + class DensePart(msgspec.Struct, frozen=True, kw_only=True): """One row family inside the entry: ``layer_num`` rows of ``row_shape``, layer ``l`` at ``offset_bytes + l * layer_stride_bytes``.""" diff --git a/python/sglang/srt/mem_cache/unified_draft_pool.py b/python/sglang/srt/mem_cache/unified_draft_pool.py new file mode 100644 index 0000000000..dffbfa9ac7 --- /dev/null +++ b/python/sglang/srt/mem_cache/unified_draft_pool.py @@ -0,0 +1,159 @@ +"""KV pools a DRAFT runner binds over the draft lanes fused into the target's +unified pool: same pages, same slot ids, same v2p table as the target -- one +allocation, one free, one relocation.""" + +from typing import Any, Dict, List, Mapping, Optional, Sequence + +import torch + +from sglang.srt.mem_cache.layout.fused_draft import FusedDraftPlacement +from sglang.srt.mem_cache.memory_pool import MHATokenToKVPool +from sglang.srt.mem_cache.unified_memory_pool import UnifiedKVPool + + +class UnifiedDraftKVPool(MHATokenToKVPool): + """Dense draft KV over the draft parts of one host sub-pool's entries. + + Per-layer `k_buffer` / `v_buffer` are views of the draft parts inside + every slot of the host sub-pool (`UnifiedKVPool.build_dense_draft_views`); + ``layer_lanes`` maps each of this runner's layer ids to its region lane. + Locs arriving through the KVCache API are the target's PHYSICAL token ids, + produced by the allocator's translate (the id-space choke point binds it, + see KVIndexTranslator); the pool exposes `host_allocator` for that + binding. Relocation needs no method here: compaction moves whole page + envelopes on the HOST pool, which carries the draft bytes; `move_kv_cache` + raises so a stray per-slot move fails loudly instead of corrupting the + fused layout. + """ + + requires_physical_write_loc = True + + def __init__( + self, + *, + unified_buffer: UnifiedKVPool, + host_sub_pool_name: str, + host_allocator, + layer_lanes: Mapping[int, int], + page_size: int = 1, + ): + region = unified_buffer.require_draft_host_spec(host_sub_pool_name).draft_region + layer_ids = sorted(layer_lanes) + assert layer_ids, "UnifiedDraftKVPool binds at least one layer" + start_layer = layer_ids[0] + assert layer_ids == list(range(start_layer, start_layer + len(layer_ids))), ( + f"fused draft layer ids must be contiguous; got {layer_ids}" + ) + lanes = [layer_lanes[layer_id] for layer_id in layer_ids] + assert len(set(lanes)) == len(lanes) and all( + 0 <= s < region.lane_num for s in lanes + ), f"draft lanes {lanes} must be distinct and within range({region.lane_num})" + k_views, v_views = unified_buffer.build_dense_draft_views(host_sub_pool_name) + max_slots = unified_buffer.max_slots(host_sub_pool_name) + + self._unified_buffer = unified_buffer + self._host_sub_pool_name = host_sub_pool_name + self.host_allocator = host_allocator + self.layer_lanes: Dict[int, int] = dict(layer_lanes) + self._k_views: List[torch.Tensor] = [k_views[s] for s in lanes] + self._v_views: List[torch.Tensor] = [v_views[s] for s in lanes] + num_pages = max_slots // page_size + + super().__init__( + size=num_pages * page_size - page_size, + page_size=page_size, + dtype=region.store_dtype, + head_num=region.head_num, + head_dim=region.head_dim, + layer_num=len(layer_ids), + device=unified_buffer.device, + enable_memory_saver=False, # buffer owned by UnifiedKVPool + v_head_dim=region.resolved_v_head_dim(), + start_layer=start_layer, + end_layer=start_layer + len(layer_ids) - 1, + enable_kv_cache_copy=False, + kv_cache_layout="page_major", + ) + + def _create_buffers(self): + self.k_buffer = self._k_views + self.v_buffer = self._v_views + + def _clear_buffers(self): + pass # lifetime owned by UnifiedKVPool + + def get_kv_size_bytes(self): + return 0, 0 # fused into the host entries; UnifiedKVPool logs the total + + def move_kv_cache(self, tgt_loc: torch.Tensor, src_loc: torch.Tensor): + raise NotImplementedError( + "fused draft KV relocates with the HOST pool's whole-page move; a " + "draft-side per-slot move would corrupt the fused layout" + ) + + def get_contiguous_buf_infos(self): + raise NotImplementedError( + "fused draft KV has no per-layer contiguous regions; KV transfer / " + "disaggregation is unsupported." + ) + + def get_cpu_copy(self, indices, mamba_indices=None): + raise NotImplementedError("CPU offloading is unsupported for fused draft KV.") + + def load_cpu_copy(self, kv_cache_cpu, indices, mamba_indices=None): + raise NotImplementedError("CPU offloading is unsupported for fused draft KV.") + + +def fused_draft_host_allocator(token_to_kv_pool: Any) -> Optional[Any]: + """The host allocator a fused draft pool translates through, or None for + any other pool.""" + if isinstance(token_to_kv_pool, UnifiedDraftKVPool): + return token_to_kv_pool.host_allocator + return None + + +def draft_kv_layer_ids(model) -> List[int]: + """Layer ids owning attention KV in the BUILT draft model, in layer order. + Window layers count -- a window is a parameter of the attention layer; + linear and recurrent layers have no attention module and are absent.""" + from sglang.srt.layers.radix_attention import RadixAttention + + return sorted( + {m.layer_id for m in model.modules() if isinstance(m, RadixAttention)} + ) + + +def bind_fused_draft( + *, + unified_buffer: UnifiedKVPool, + host_allocator, + placement: FusedDraftPlacement, + runner: int, + kv_layer_ids: Sequence[int], + swa_layer_ids: Sequence[int], + page_size: int, +) -> UnifiedDraftKVPool: + """The KV pool draft runner ``runner`` binds over its fused slots. + + The placement sized the lanes from the draft config; the model's real + layer ids fill them in layer order, so a count mismatch is a loud boot + failure, never a silent alias. + """ + swa = set(swa_layer_ids) + full_ids = [layer_id for layer_id in kv_layer_ids if layer_id not in swa] + assert len(full_ids) == len(kv_layer_ids), ( + f"draft layers {sorted(swa & set(kv_layer_ids))} are SWA-kind; fused " + "SWA KV is not supported yet" + ) + full_lanes = placement.lanes_for(runner) + assert len(full_ids) == len(full_lanes), ( + f"draft runner {runner}: {len(full_ids)} full-attention layer(s) " + f"{full_ids} vs {len(full_lanes)} placed lane(s) {list(full_lanes)}" + ) + return UnifiedDraftKVPool( + unified_buffer=unified_buffer, + host_sub_pool_name="full", + host_allocator=host_allocator, + layer_lanes=dict(zip(full_ids, full_lanes)), + page_size=page_size, + ) diff --git a/python/sglang/srt/mem_cache/unified_memory_pool.py b/python/sglang/srt/mem_cache/unified_memory_pool.py index 1ea0fee0b1..f71ddb128d 100644 --- a/python/sglang/srt/mem_cache/unified_memory_pool.py +++ b/python/sglang/srt/mem_cache/unified_memory_pool.py @@ -38,11 +38,16 @@ from sglang.kernels.ops.kvcache.zero_pages import zero_pages from sglang.srt.constants import GPU_MEMORY_TYPE_KV_CACHE from sglang.srt.environ import envs from sglang.srt.mem_cache.kv_vmm_backing import KvVmmBufferOwner +from sglang.srt.mem_cache.layout.fused_draft import ( + DenseDraftRegion, + FusedDraftPlacement, +) from sglang.srt.mem_cache.layout.token_major import ( ROW_ALIGN_BYTES, DenseEntryLayout, DensePart, align_entry_bytes, + align_part_offset, build_dense_views, build_mamba_entry_views, ) @@ -113,6 +118,8 @@ class SubPoolSpec(ABC): name: str layer_num: int grow_direction: str # "up" | "down" | "float" + # Fused draft region riding in this sub-pool's entries; None = unfused. + draft_region: Optional[DenseDraftRegion] = None def __post_init__(self): assert self.grow_direction in self._allowed_grow_directions, ( @@ -134,7 +141,17 @@ class SubPoolSpec(ABC): @dataclass(frozen=True, kw_only=True) class MHASubPoolSpec(SubPoolSpec): - """Per-slot layout of one MHA-shaped sub-pool. `v_head_dim` defaults to `head_dim`.""" + """Per-slot layout of one MHA-shaped sub-pool. `v_head_dim` defaults to `head_dim`. + + With `draft_region` set, the draft model's K and V rows are two more + parts of every slot's entry, after the host parts: + + [ K_0 | V_0 | ... | K_{Lh-1} | V_{Lh-1} | dK_0 | dV_0 | ... | pad ] + + The slot stride stays the one entry, so host and draft views are indexed + by the same physical token id; only their offsets differ. `draft_region + is None` keeps the layout byte-identical to the unfused one. + """ head_num: int head_dim: int @@ -159,6 +176,8 @@ class MHASubPoolSpec(SubPoolSpec): assert self.v_head_dim > 0, ( f"v_head_dim must be positive; got {self.v_head_dim}" ) + if self.draft_region is not None: + self.draft_region.validate() def k_row_bytes(self) -> int: return self.head_num * self.head_dim * self.store_dtype.itemsize @@ -166,9 +185,20 @@ class MHASubPoolSpec(SubPoolSpec): def v_row_bytes(self) -> int: return self.head_num * self.v_head_dim * self.store_dtype.itemsize + def host_entry_bytes(self) -> int: + """Host (target-only) bytes for one slot, before any draft parts.""" + return self.layer_num * (self.k_row_bytes() + self.v_row_bytes()) + + def draft_offset_in_entry(self) -> int: + """Byte offset of the fused draft parts inside one slot's entry.""" + assert self.draft_region is not None + return align_part_offset(self.host_entry_bytes()) + def entry_bytes(self) -> int: + if self.draft_region is None: + return align_entry_bytes(self.host_entry_bytes()) return align_entry_bytes( - self.layer_num * (self.k_row_bytes() + self.v_row_bytes()) + self.draft_offset_in_entry() + self.draft_region.entry_bytes() ) def page_bytes(self, page_size: int) -> int: @@ -179,27 +209,27 @@ class MHASubPoolSpec(SubPoolSpec): is ``page_size`` such entries back to back.""" _check_row_alignment(self.name, K=self.k_row_bytes(), V=self.v_row_bytes()) layer_stride = self.k_row_bytes() + self.v_row_bytes() - return DenseEntryLayout( - entry_bytes=self.entry_bytes(), - parts=( - DensePart( - name="k", - offset_bytes=0, - layer_stride_bytes=layer_stride, - layer_num=self.layer_num, - row_shape=(self.head_num, self.head_dim), - dtype=self.store_dtype, - ), - DensePart( - name="v", - offset_bytes=self.k_row_bytes(), - layer_stride_bytes=layer_stride, - layer_num=self.layer_num, - row_shape=(self.head_num, self.v_head_dim), - dtype=self.store_dtype, - ), + parts = ( + DensePart( + name="k", + offset_bytes=0, + layer_stride_bytes=layer_stride, + layer_num=self.layer_num, + row_shape=(self.head_num, self.head_dim), + dtype=self.store_dtype, + ), + DensePart( + name="v", + offset_bytes=self.k_row_bytes(), + layer_stride_bytes=layer_stride, + layer_num=self.layer_num, + row_shape=(self.head_num, self.v_head_dim), + dtype=self.store_dtype, ), ) + if self.draft_region is not None: + parts += self.draft_region.parts(self.draft_offset_in_entry()) + return DenseEntryLayout(entry_bytes=self.entry_bytes(), parts=parts) def get_dtype(self) -> torch.dtype: return self.store_dtype @@ -227,6 +257,9 @@ class MLASubPoolSpec(SubPoolSpec): assert self.qk_rope_head_dim > 0, ( f"qk_rope_head_dim must be positive; got {self.qk_rope_head_dim}" ) + assert self.draft_region is None, ( + "MLA sub-pools do not carry a fused draft region yet" + ) @property def kv_cache_dim(self) -> int: @@ -271,6 +304,9 @@ class MambaSubPoolSpec(SubPoolSpec): def __post_init__(self): super().__post_init__() assert len(self.conv_state_shapes) > 0, "conv_state_shapes must be non-empty" + assert self.draft_region is None, ( + "mamba state pages carry no fused draft region yet" + ) def conv_row_bytes(self, idx: int) -> int: return _prod(self.conv_state_shapes[idx]) * self.conv_dtype.itemsize @@ -350,6 +386,7 @@ class UnifiedKVPool: page_size: int = 1, post_capture_active: bool = False, bs1_floor_terms: Optional[List[Tuple[str, int]]] = None, + fused_draft: Optional[FusedDraftPlacement] = None, ): assert page_size >= 1, f"page_size must be >= 1; got {page_size}" assert len(sub_pool_specs) >= 2, ( @@ -386,6 +423,17 @@ class UnifiedKVPool: self.post_capture_active = post_capture_active self._post_capture_owner: Optional[KvVmmBufferOwner] = None self._bs1_floor_terms = bs1_floor_terms or [] + self.fused_draft = fused_draft + for spec in sub_pool_specs: + expected = ( + fused_draft.region + if fused_draft is not None and spec.name == "full" + else None + ) + assert spec.draft_region is expected, ( + f"sub-pool {spec.name!r}: draft_region {spec.draft_region} does " + f"not match the fused draft placement's {expected}" + ) self.memory_saver_adapter = TorchMemorySaverAdapter.create( enable=enable_memory_saver @@ -588,6 +636,17 @@ class UnifiedKVPool: ) return s + def require_draft_host_spec(self, name: str) -> SubPoolSpec: + """The sub-pool spec whose entries carry a fused draft region. + + Kind-agnostic: any spec that resolves a `draft_region` also lays the + draft parts out in its `layout()`.""" + s = self._specs_by_name[name] + assert s.draft_region is not None, ( + f"sub-pool {name!r} carries no fused draft region" + ) + return s + def max_slots(self, name: str) -> int: return self._max_slots[name] @@ -630,6 +689,27 @@ class UnifiedKVPool: ) return k_views, v_views + def build_dense_draft_views( + self, sub_pool_name: str + ) -> Tuple[List[torch.Tensor], List[torch.Tensor]]: + """Per-layer K/V views of the DRAFT parts fused into ``sub_pool_name``'s + entries: same pages, same slot ids, same v2p table as the host.""" + spec = self.require_draft_host_spec(sub_pool_name) + layout = spec.layout() + page_size = self._page_size + num_slots = self.max_slots(sub_pool_name) // page_size * page_size + k_views, v_views = ( + build_dense_views( + self._raw, + layout=layout, + part_name=name, + num_slots=num_slots, + anchor_bytes=self._anchor_bytes[sub_pool_name], + ) + for name in ("draft_k", "draft_v") + ) + return k_views, v_views + def _build_mla_views( self, spec: MLASubPoolSpec, @@ -1899,8 +1979,15 @@ def init_unified_swa_pools( lazy_compaction: bool = False, model_context_len: Optional[int] = None, sliding_window_size: Optional[int] = None, + fused_draft: Optional[FusedDraftPlacement] = None, ) -> UnifiedSWAPoolBundle: - """Build the SWA-hybrid unified-memory-pool stack.""" + """Build the SWA-hybrid unified-memory-pool stack. + + With ``fused_draft``, every entry of the "full" sub-pool carries the + draft model's K/V parts after the host parts, and each draft runner + binds a `UnifiedDraftKVPool` over its own lanes instead of allocating a + pool of its own. + """ from sglang.srt.mem_cache.allocator.unified_hybrid_swa import ( UnifiedSWATokenToKVPoolAllocator, ) @@ -1926,6 +2013,7 @@ def init_unified_swa_pools( store_dtype=store_dtype, kv_cache_dtype=kv_cache_dtype, grow_direction="down", + draft_region=None if fused_draft is None else fused_draft.region, ) swa_spec = MHASubPoolSpec( name="swa", @@ -1981,6 +2069,7 @@ def init_unified_swa_pools( page_size=page_size, post_capture_active=post_capture_active, bs1_floor_terms=bs1_floor_terms, + fused_draft=fused_draft, ) token_to_kv_pool = UnifiedSWAKVPool( unified_buffer=shared_pool, diff --git a/python/sglang/srt/model_executor/model_runner_components/spec_aux_hidden_state.py b/python/sglang/srt/model_executor/model_runner_components/spec_aux_hidden_state.py index 1690eb4c3d..1ad8fda9df 100644 --- a/python/sglang/srt/model_executor/model_runner_components/spec_aux_hidden_state.py +++ b/python/sglang/srt/model_executor/model_runner_components/spec_aux_hidden_state.py @@ -44,6 +44,11 @@ class SpecAuxHiddenStateConfig(msgspec.Struct, kw_only=True): dflash_target_layer_ids: Any = None # DFLASH draft KV bytes/token; None when unresolved. dflash_draft_cell_size_per_token: int | None = None + # The draft checkpoint read in draft mode, and its KV layer count: the + # fused-draft placement's input. `eagle_draft_num_layers` above keeps + # sizing a private draft pool exactly as before. + draft_model_config: Optional[ModelConfig] = None + draft_kv_num_layers: Optional[int] = None def resolve_spec_aux_hidden_state_config( @@ -85,24 +90,38 @@ def _resolve_eagle_aux_hidden_state( ): return - draft_model_config = model_config - if get_spec().speculative_draft_model_path: - draft_model_config = ModelConfig.from_server_args( - server_args, - model_path=get_spec().speculative_draft_model_path, - model_revision=get_spec().speculative_draft_model_revision, - is_draft_model=True, - ) - num_nextn_predict_layers = draft_model_config.num_nextn_predict_layers - if num_nextn_predict_layers is not None: - config.eagle_draft_num_layers = int(num_nextn_predict_layers) - elif get_spec().speculative_draft_model_path: - config.eagle_draft_num_layers = int( + draft_path = get_spec().speculative_draft_model_path + # The draft read in draft mode. A path-less NEXTN/MTP head is the TARGET + # checkpoint rewritten to its MTP form, which is where its + # `num_nextn_predict_layers` gets filled in. + draft_model_config = ModelConfig.from_server_args( + server_args, + model_path=draft_path, + model_revision=get_spec().speculative_draft_model_revision, + is_draft_model=True, + ) + if draft_model_config.num_nextn_predict_layers is not None: + config.draft_kv_num_layers = int(draft_model_config.num_nextn_predict_layers) + elif draft_path is not None: + config.draft_kv_num_layers = int( max( draft_model_config.num_hidden_layers, draft_model_config.num_attention_layers, ) ) + if config.draft_kv_num_layers is not None: + config.draft_model_config = draft_model_config + + # The private draft pool's budget keeps its reading: a path-less run sizes + # the draft off the TARGET config. Reading the draft-mode config here would + # change the token budget of every path-less MTP deployment. + if not draft_path: + draft_model_config = model_config + num_nextn_predict_layers = draft_model_config.num_nextn_predict_layers + if num_nextn_predict_layers is not None: + config.eagle_draft_num_layers = int(num_nextn_predict_layers) + elif draft_path: + config.eagle_draft_num_layers = config.draft_kv_num_layers else: return diff --git a/python/sglang/srt/model_executor/pool_configurator.py b/python/sglang/srt/model_executor/pool_configurator.py index 3986089f98..fba401b218 100644 --- a/python/sglang/srt/model_executor/pool_configurator.py +++ b/python/sglang/srt/model_executor/pool_configurator.py @@ -714,9 +714,27 @@ class HybridSWAPoolConfigurator(MemoryPoolConfigurator): ) self._draft_cell_size = _dflash_draft_cell_size(kvc) + self._fused_full_entry = kvc.fused_entry_bytes("full") + if self._fused_full_entry is not None: + self._draft_full_layers_num = 0 + self._draft_swa_layers_num = 0 + self._draft_swa_full_layers_num = 0 + self._draft_cell_size = 0 self._recompute_cell_size() + def _full_cell_bytes(self) -> int: + """Bytes per full-side token: the fused entry when the draft is placed + there, else the target's rows plus the private draft's full rows.""" + if self._fused_full_entry is not None: + return self._fused_full_entry + return self._full_per_token * ( + self._full_layers_num + self._draft_full_layers_num + ) + + def _swa_cell_bytes(self) -> int: + return self._swa_per_token * (self._swa_layers_num + self._draft_swa_layers_num) + def _recompute_cell_size(self) -> None: # Bytes per token of max_total_num_tokens: full_tokens when hybrid, else # swa_tokens with no ratio, since all-SWA has no full pool to relate to. @@ -730,12 +748,9 @@ class HybridSWAPoolConfigurator(MemoryPoolConfigurator): ) else: self._cell_size = ( - self._full_per_token - * (self._full_layers_num + self._draft_full_layers_num) + self._full_cell_bytes() + self._swa_per_token * self._draft_swa_full_layers_num - + self._swa_full_tokens_ratio - * self._swa_per_token - * (self._swa_layers_num + self._draft_swa_layers_num) + + self._swa_full_tokens_ratio * self._swa_cell_bytes() + self._draft_cell_size ) @@ -747,9 +762,17 @@ class HybridSWAPoolConfigurator(MemoryPoolConfigurator): + self._draft_cell_size ) + def _unified_full_bytes_per_token(self) -> int: + """Bytes one full-side token takes in the unified pool: the fused + entry (host + draft + pad) when the draft lives in it, else the + target's own rows. A private draft pool is priced separately.""" + if self._fused_full_entry is not None: + return self._fused_full_entry + return self._full_per_token * self._full_layers_num + def _unified_pool_bytes(self, full_tokens: int, swa_tokens: int) -> int: return ( - full_tokens * self._full_per_token * self._full_layers_num + full_tokens * self._unified_full_bytes_per_token() + swa_tokens * self._swa_per_token * self._swa_layers_num ) @@ -761,7 +784,7 @@ class HybridSWAPoolConfigurator(MemoryPoolConfigurator): ) -> int: """Find the largest page-aligned full capacity whose allocations fit.""" draft_bytes_per_token = self._draft_pool_bytes_per_token() - target_full_bytes_per_token = self._full_per_token * self._full_layers_num + target_full_bytes_per_token = self._unified_full_bytes_per_token() assert target_full_bytes_per_token > 0 def allocation_bytes(full_pages: int) -> int: @@ -930,19 +953,14 @@ class SWAChunkCapPoolConfigurator(HybridSWAPoolConfigurator): ) -> MemoryPoolConfig: # SWA pool sized tightly from the cap; the rest of the budget goes to full. swa_tokens = ceil_align(self._swa_cap, page_size) - fixed_swa_bytes = ( - swa_tokens - * self._swa_per_token - * (self._swa_layers_num + self._draft_swa_layers_num) - ) - if self._enable_unified_memory: + fixed_swa_bytes = swa_tokens * self._swa_cell_bytes() + if self._enable_unified_memory and self._draft_pool_bytes_per_token() > 0: full_tokens = self._max_unified_full_tokens( available_bytes, page_size, fixed_swa_tokens=swa_tokens ) else: full_cell_size = ( - self._full_per_token - * (self._full_layers_num + self._draft_full_layers_num) + self._full_cell_bytes() + self._swa_per_token * self._draft_swa_full_layers_num ) full_tokens = ( diff --git a/python/sglang/srt/speculative/base_spec_worker.py b/python/sglang/srt/speculative/base_spec_worker.py index d8bab5c428..ae7910ea67 100644 --- a/python/sglang/srt/speculative/base_spec_worker.py +++ b/python/sglang/srt/speculative/base_spec_worker.py @@ -180,8 +180,15 @@ class BaseSpecWorker(ABC): @property def primary_draft_kv_pool(self) -> Optional[object]: + from sglang.srt.mem_cache.unified_draft_pool import fused_draft_host_allocator + draft_runners = self._draft_model_runners() - return draft_runners[0].token_to_kv_pool if draft_runners else None + if not draft_runners: + return None + pool = draft_runners[0].token_to_kv_pool + if fused_draft_host_allocator(pool) is not None: + return None + return pool @property def target_worker(self) -> TpModelWorker: diff --git a/python/sglang/srt/speculative/eagle_worker_v2.py b/python/sglang/srt/speculative/eagle_worker_v2.py index 7afc056dd0..b05bd5872d 100644 --- a/python/sglang/srt/speculative/eagle_worker_v2.py +++ b/python/sglang/srt/speculative/eagle_worker_v2.py @@ -389,6 +389,15 @@ class EagleDraftWorker(EagleDraftWorkerBase): self._configure_qsa_mtp_index_share() self.tree_mask_mode = default_tree_mask_mode() + # Boot guard: every backend a draft forward reaches must carry its + # runner's translator or the index builders emit virtual ids. + translator = self.draft_runner.kv_index_translator + if translator.is_translating: + backends = [self.draft_attn_backend, self.draft_extend_attn_backend] + if self.draft_attn_backend is not None: + backends += self.draft_attn_backend.attn_backends + translator.bind_and_verify_backends(backends) + def _configure_qsa_mtp_index_share(self) -> None: """Reuse the draft-extend QSA selection across the MTP decode steps; chain speculation only: with topk > 1 decode rows are not request-major.""" diff --git a/python/sglang/srt/speculative/multi_layer_eagle_worker_v2.py b/python/sglang/srt/speculative/multi_layer_eagle_worker_v2.py index cd23867273..e5cdfaa07d 100644 --- a/python/sglang/srt/speculative/multi_layer_eagle_worker_v2.py +++ b/python/sglang/srt/speculative/multi_layer_eagle_worker_v2.py @@ -398,6 +398,13 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase): self.draft_runner_list[ step ].attn_backend = self.draft_extend_attn_backend_list[-1] + # Boot guard: every backend a draft forward reaches must carry + # its runner's translator or the index builders emit virtual ids. + translator = self.draft_runner_list[step].kv_index_translator + if translator.is_translating: + translator.bind_and_verify_backends( + [self.draft_extend_attn_backend_list[-1]] + ) def _capture_cuda_graphs(self): self.cuda_graph_runner = None diff --git a/test/registered/kernels/ops/speculative/test_draft_decode_window.py b/test/registered/kernels/ops/speculative/test_draft_decode_window.py index 0617d295f7..f214911d71 100644 --- a/test/registered/kernels/ops/speculative/test_draft_decode_window.py +++ b/test/registered/kernels/ops/speculative/test_draft_decode_window.py @@ -117,6 +117,7 @@ def _run(io, window_size=0, sink_size=0): kv_indices, kv_indptr, io["positions"], + None, # v2p: no unified-pool translation io["pool_len"], kv_indices.shape[1], kv_indptr.shape[1], diff --git a/test/registered/kernels/ops/speculative/test_generate_draft_decode_kv_indices_translate.py b/test/registered/kernels/ops/speculative/test_generate_draft_decode_kv_indices_translate.py new file mode 100644 index 0000000000..ad66497b3f --- /dev/null +++ b/test/registered/kernels/ops/speculative/test_generate_draft_decode_kv_indices_translate.py @@ -0,0 +1,172 @@ +# Copyright 2023-2026 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""`generate_draft_decode_kv_indices` under the unified pool's translation. + +BUG REGRESSION. The multi-step draft-decode kernel copied req_to_token values +verbatim into the per-step kv_indices. Under `--enable-unified-memory` those +values are VIRTUAL ids while the fused draft pool is indexed by physical +ids, so with `speculative_num_steps > 1` every draft decode step read the +wrong KV rows -- silently: bad drafts only depress acceptance, verify stays +correct. Pinned here: TRANSLATE=True must emit exactly `translate_kv_loc`'s +physical ids, and TRANSLATE=False must stay byte-identical to the historical +raw copy. + + python -m pytest test/registered/kernels/test_generate_draft_decode_kv_indices_translate.py -v +""" + +import unittest + +import torch + +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=30, stage="base-b-kernel-unit", runner_config="1-gpu-large") + +_SENTINEL = -7 + + +def _run_kernel( + *, + req_to_token, + seq_lens, + positions, + num_steps, + topk, + page_size, + v2p, + translate, +): + from sglang.kernels.ops.speculative.cache_locs import ( + generate_draft_decode_kv_indices, + ) + from sglang.srt.utils import next_power_of_2 + + num_seqs = seq_lens.numel() + bs = num_seqs * topk + width = int((seq_lens.sum().item() + num_steps * bs) * 2 + 64) + kv_indices = torch.full( + (num_steps, width), _SENTINEL, dtype=torch.int64, device="cuda" + ) + kv_indptr = torch.zeros((num_steps, bs + 1), dtype=torch.int32, device="cuda") + generate_draft_decode_kv_indices[(num_steps, num_seqs, topk)]( + torch.arange(num_seqs, dtype=torch.int64, device="cuda"), + req_to_token, + seq_lens, + kv_indices, + kv_indptr, + positions, + v2p if translate else None, + req_to_token.shape[1], + width, + kv_indptr.shape[1], + next_power_of_2(num_seqs), + next_power_of_2(num_steps), + next_power_of_2(bs), + page_size, + TRANSLATE=translate, + ) + return kv_indices, kv_indptr + + +class TestGenerateDraftDecodeKVIndicesTranslate(CustomTestCase): + def _check(self, *, page_size, num_steps=3, topk=1, unused_slots=False): + torch.manual_seed(7) + num_seqs, max_context = 3, 64 + num_pages = max_context // page_size + 1 + # Virtual ids: a shuffled page layout so translation is not identity. + perm = torch.randperm(num_pages, device="cuda") + v2p = torch.empty(num_pages, dtype=torch.int64, device="cuda") + v2p[perm] = torch.arange(num_pages, device="cuda") + req_to_token = ( + torch.arange(max_context, dtype=torch.int64, device="cuda") + .flip(0) + .repeat(num_seqs, 1) + ) + if unused_slots: + # Negative entries inside the read window must land on the sink. + req_to_token[:, ::3] = -1 + seq_lens = torch.tensor([5, 1, 9], dtype=torch.int64, device="cuda")[:num_seqs] + positions = seq_lens.repeat_interleave(topk) + + common = dict( + req_to_token=req_to_token, + seq_lens=seq_lens, + positions=positions, + num_steps=num_steps, + topk=topk, + page_size=page_size, + v2p=v2p, + ) + raw, raw_indptr = _run_kernel(translate=False, **common) + out, out_indptr = _run_kernel(translate=True, **common) + + torch.testing.assert_close(raw_indptr, out_indptr, rtol=0, atol=0) + written = raw != _SENTINEL + # Unwritten lanes stay untouched in both compilations. + self.assertTrue(bool((out[~written] == _SENTINEL).all())) + # Written lanes: translate_kv_loc's formula over the raw ids. + virt = raw[written] + page = torch.where(virt < 0, 0, virt // page_size) + expected = torch.where( + virt < 0, + 0, + torch.clamp_min(v2p[page] * page_size + virt % page_size, 0), + ) + torch.testing.assert_close(out[written], expected, rtol=0, atol=0) + + def test_page_size_one(self): + if not torch.cuda.is_available(): + self.skipTest("CUDA required") + self._check(page_size=1) + + def test_page_size_two(self): + if not torch.cuda.is_available(): + self.skipTest("CUDA required") + self._check(page_size=2) + + def test_negative_ids_map_to_the_sink(self): + if not torch.cuda.is_available(): + self.skipTest("CUDA required") + for page_size in (1, 2): + self._check(page_size=page_size, unused_slots=True) + + def test_translate_false_is_a_raw_copy(self): + """The static-pool compilation must stay byte-identical to the + historical kernel: the emitted indices ARE the req_to_token values.""" + if not torch.cuda.is_available(): + self.skipTest("CUDA required") + torch.manual_seed(11) + num_seqs, max_context, page_size = 2, 32, 1 + req_to_token = torch.randint( + 0, 1 << 20, (num_seqs, max_context), dtype=torch.int64, device="cuda" + ) + seq_lens = torch.tensor([4, 7], dtype=torch.int64, device="cuda") + raw, _ = _run_kernel( + req_to_token=req_to_token, + seq_lens=seq_lens, + positions=seq_lens, + num_steps=2, + topk=1, + page_size=page_size, + v2p=None, + translate=False, + ) + written = raw != _SENTINEL + emitted = raw[written] + self.assertTrue(bool(torch.isin(emitted, req_to_token.flatten()).all())) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/kernels/ops/speculative/test_spec_kv_indices_grid.py b/test/registered/kernels/ops/speculative/test_spec_kv_indices_grid.py index d93ad83dee..dece431904 100644 --- a/test/registered/kernels/ops/speculative/test_spec_kv_indices_grid.py +++ b/test/registered/kernels/ops/speculative/test_spec_kv_indices_grid.py @@ -60,6 +60,7 @@ def _run_draft(kern_inputs, topk, steps, page_size, nb, kw): kv_i, kv_p, positions, + None, # v2p: no unified-pool translation POOL_LEN, kv_i.shape[1], kv_p.shape[1], diff --git a/test/registered/unit/mem_cache/test_dense_fused_draft_geometry.py b/test/registered/unit/mem_cache/test_dense_fused_draft_geometry.py new file mode 100644 index 0000000000..4b493a5703 --- /dev/null +++ b/test/registered/unit/mem_cache/test_dense_fused_draft_geometry.py @@ -0,0 +1,308 @@ +"""Dense fused-draft geometry: the draft model's KV as two more parts of every +host slot's entry. + +Derived-property pins for the fused entry layout + + [ K_0 | V_0 | ... | K_{Lh-1} | V_{Lh-1} | dK_0 | dV_0 | ... | pad ] + +the draft parts start at the 16-B-aligned end of the host parts and the entry +rounds up to the 32-B entry alignment (never to an lcm of row widths), so the +draft's row width is free to differ from the host's; host and draft views +share the slot stride and are indexed by the same physical token id; and +writes through either family's views land inside their own part of their own +slot (compaction moves whole page envelopes, so confinement IS the correctness +of the fused move). + + python -m pytest test/registered/unit/mem_cache/test_dense_fused_draft_geometry.py -v +""" + +import unittest + +import torch + +from sglang.srt.mem_cache.layout.fused_draft import ( + DenseDraftRegion, + FusedDraftPlacement, +) +from sglang.srt.mem_cache.layout.token_major import ( + ENTRY_ALIGN_BYTES, + ROW_ALIGN_BYTES, + align_entry_bytes, + align_part_offset, +) +from sglang.srt.mem_cache.unified_draft_pool import UnifiedDraftKVPool +from sglang.srt.mem_cache.unified_memory_pool import ( + MambaSubPoolSpec, + MHASubPoolSpec, + MLASubPoolSpec, + UnifiedKVPool, + UnifiedMHATokenToKVPool, +) +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=10, suite="base-a-test-cpu") + +_DEV = "cpu" +_DTYPE = torch.bfloat16 +_ITEM = _DTYPE.itemsize + + +def _host_spec(draft_region=None, layer_num=2, head_num=2, head_dim=4): + return MHASubPoolSpec( + name="full", + layer_num=layer_num, + head_num=head_num, + head_dim=head_dim, + store_dtype=_DTYPE, + grow_direction="down", + draft_region=draft_region, + ) + + +def _draft_region(): + # Deliberately a different row width from the host's (16 B): K rows of + # 48 B, V rows of 16 B -- asymmetric draft rows are ordinary parts. + return DenseDraftRegion( + lane_num=1, head_num=1, head_dim=24, v_head_dim=8, store_dtype=_DTYPE + ) + + +def _placement(region): + return FusedDraftPlacement(region=region, runner_lane_counts=(region.lane_num,)) + + +class TestFusedSpecMath(unittest.TestCase): + def test_unfused_spec_is_byte_identical_to_before(self): + s = _host_spec() + self.assertEqual(s.entry_bytes(), 2 * (16 + 16)) + self.assertEqual(s.host_entry_bytes(), s.entry_bytes()) + self.assertEqual([p.name for p in s.layout().parts], ["k", "v"]) + + def test_fused_entry_appends_the_draft_parts(self): + f = _host_spec(_draft_region()) + r = f.draft_region + self.assertEqual(r.k_row_bytes(), 48) + self.assertEqual(r.v_row_bytes(), 16) + self.assertEqual(r.entry_bytes(), 64) + self.assertEqual( + f.draft_offset_in_entry(), align_part_offset(f.host_entry_bytes()) + ) + self.assertEqual(f.draft_offset_in_entry(), 64) + self.assertEqual(f.entry_bytes(), align_entry_bytes(64 + 64)) + self.assertEqual(f.entry_bytes() % ENTRY_ALIGN_BYTES, 0) + layout = f.layout() + self.assertEqual( + [p.name for p in layout.parts], ["k", "v", "draft_k", "draft_v"] + ) + self.assertEqual(layout.part("draft_k").offset_bytes, 64) + self.assertEqual(layout.part("draft_v").offset_bytes, 64 + 48) + self.assertEqual(layout.part("draft_k").layer_stride_bytes, 64) + + def test_entry_alignment_pads_only_to_the_entry_quantum(self): + # host 32 B + draft 48 B = 80 B of rows -> one 96 B entry (16 B pad), + # not an lcm of the two row widths. + f = _host_spec( + DenseDraftRegion( + lane_num=1, head_num=1, head_dim=8, v_head_dim=16, store_dtype=_DTYPE + ), + layer_num=1, + head_num=1, + head_dim=8, + ) + self.assertEqual(f.host_entry_bytes(), 32) + self.assertEqual(f.draft_region.entry_bytes(), 48) + self.assertEqual(f.entry_bytes(), 96) + self.assertEqual(f.draft_offset_in_entry() % ROW_ALIGN_BYTES, 0) + f.layout() # parts fit and do not overlap + + def test_draft_rows_must_be_row_aligned(self): + with self.assertRaises(AssertionError): + _host_spec( + DenseDraftRegion(lane_num=1, head_num=1, head_dim=3, store_dtype=_DTYPE) + ).layout() + + def test_only_page_envelope_kinds_accept_a_draft_region(self): + """`draft_region` is a universal spec field (None = unfused), but a + kind without the fused entry layout must refuse one at construction; + a silently-carried region would never reach the layout.""" + with self.assertRaises(AssertionError): + MLASubPoolSpec( + name="full", + layer_num=2, + kv_lora_rank=8, + qk_rope_head_dim=4, + store_dtype=torch.bfloat16, + grow_direction="down", + draft_region=_draft_region(), + ) + with self.assertRaises(AssertionError): + MambaSubPoolSpec( + name="mamba", + layer_num=1, + conv_state_shapes=((2, 2),), + conv_dtype=torch.float32, + temporal_state_shape=(2,), + temporal_dtype=torch.float32, + grow_direction="up", + draft_region=_draft_region(), + ) + + +class TestFusedRegionConfinement(unittest.TestCase): + """Writes through host/draft views must land in their own part of their + own slot.""" + + PS = 2 + PAGES = 3 + + def _build(self): + spec = _host_spec(_draft_region()) + swa = MHASubPoolSpec( + name="swa", + layer_num=1, + head_num=2, + head_dim=4, + store_dtype=_DTYPE, + grow_direction="up", + ) + total = 8 * self.PS * spec.entry_bytes() + 8 * self.PS * swa.entry_bytes() + pool = UnifiedKVPool( + total_bytes=total, + sub_pool_specs=[spec, swa], + device=_DEV, + enable_memory_saver=False, + page_size=self.PS, + fused_draft=_placement(spec.draft_region), + ) + hk, hv = pool.mha_views_for("full") + dk, dv = pool.build_dense_draft_views("full") + return spec, pool, (hk, hv), (dk, dv) + + def test_host_and_draft_writes_confine_to_their_parts(self): + spec, pool, (hk, hv), (dk, dv) = self._build() + layout = spec.layout() + raw = pool._raw + entry = spec.entry_bytes() + families = ( + ("k", hk), + ("v", hv), + ("draft_k", dk), + ("draft_v", dv), + ) + for t in range(self.PAGES * self.PS): + for name, views in families: + part = layout.part(name) + for layer, view in enumerate(views): + raw.zero_() + view[t] = 1.0 + nz = raw.nonzero().flatten() + lo = t * entry + part.layer_offset_bytes(layer) + hi = lo + part.row_bytes() + self.assertEqual(int(nz.min()), lo, f"{name}[{layer}] t={t}") + self.assertEqual(int(nz.max()), hi - 1, f"{name}[{layer}] t={t}") + # ... and inside the slot's page envelope. + page = t // self.PS + self.assertGreaterEqual(lo, page * self.PS * entry) + self.assertLessEqual(hi, (page + 1) * self.PS * entry) + + def test_host_and_draft_views_share_the_slot_stride(self): + spec, _, (hk, hv), (dk, dv) = self._build() + stride = spec.entry_bytes() + for view in (*hk, *hv, *dk, *dv): + self.assertEqual(view.stride(0) * view.element_size(), stride) + self.assertEqual(view.shape[0], hk[0].shape[0]) + self.assertEqual(tuple(dk[0].shape[1:]), (1, 24)) + self.assertEqual(tuple(dv[0].shape[1:]), (1, 8)) + + def test_pool_helper_refuses_an_unfused_sub_pool(self): + _, pool, _, _ = self._build() + with self.assertRaises(AssertionError): + pool.build_dense_draft_views("swa") # no fused region there + + +class TestUnifiedDraftKVPool(unittest.TestCase): + PS = 2 + PAGES = 8 + + def _pool(self): + spec = _host_spec(_draft_region()) + swa = MHASubPoolSpec( + name="swa", + layer_num=1, + head_num=2, + head_dim=4, + store_dtype=torch.bfloat16, + grow_direction="up", + ) + total = self.PAGES * self.PS * (spec.entry_bytes() + swa.entry_bytes()) + return UnifiedKVPool( + total_bytes=total, + sub_pool_specs=[spec, swa], + device=_DEV, + enable_memory_saver=False, + page_size=self.PS, + fused_draft=_placement(spec.draft_region), + ) + + def _draft_pool(self, pool): + sentinel_allocator = object() + dp = UnifiedDraftKVPool( + unified_buffer=pool, + host_sub_pool_name="full", + host_allocator=sentinel_allocator, + layer_lanes={0: 0}, + page_size=self.PS, + ) + return dp, sentinel_allocator + + def test_probe_surface_and_view_binding(self): + pool = self._pool() + dp, alloc = self._draft_pool(pool) + spec = pool.mha_spec("full") + self.assertIs(dp.host_allocator, alloc) + self.assertTrue(dp.requires_physical_write_loc) + self.assertEqual(len(dp.k_buffer), 1) + self.assertEqual(dp.k_buffer[0].shape[1:], (1, 24)) + self.assertEqual(dp.v_buffer[0].shape[1:], (1, 8)) + # Same slot space as the host: one row per physical token, the slot + # stride being the whole fused entry. + n_rows = pool.max_slots("full") // self.PS * self.PS + self.assertEqual(dp.k_buffer[0].shape[0], n_rows) + self.assertEqual(dp.size, n_rows - self.PS) + self.assertEqual( + dp.k_buffer[0].stride(0) * dp.k_buffer[0].element_size(), spec.entry_bytes() + ) + + def test_host_page_move_carries_the_draft_bytes(self): + # THE fused-layout property: compaction relocates whole page envelopes + # on the HOST pool; a draft marker written in page A must arrive at + # page B after host.move_kv_cache(B, A), with zero draft-side moves. + pool = self._pool() + dp, _ = self._draft_pool(pool) + host = UnifiedMHATokenToKVPool( + unified_buffer=pool, sub_pool_name="full", page_size=self.PS + ) + src_page, dst_page = 3, 5 + src_t, dst_t = src_page * self.PS, dst_page * self.PS + dp.k_buffer[0][src_t] = 7.0 + self.assertEqual(float(dp.k_buffer[0][dst_t].sum()), 0.0) + + ps = self.PS + to_tokens = lambda p: torch.arange(p * ps, (p + 1) * ps, dtype=torch.int64) + host.move_kv_cache(to_tokens(dst_page), to_tokens(src_page)) + self.assertEqual(float(dp.k_buffer[0][dst_t].sum()), 7.0 * 24) + + def test_draft_side_moves_and_transfers_fail_loudly(self): + pool = self._pool() + dp, _ = self._draft_pool(pool) + one = torch.zeros(self.PS, dtype=torch.int64) + with self.assertRaises(NotImplementedError): + dp.move_kv_cache(one, one) + with self.assertRaises(NotImplementedError): + dp.get_contiguous_buf_infos() + with self.assertRaises(NotImplementedError): + dp.get_cpu_copy(one) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/mem_cache/test_fused_draft_placement.py b/test/registered/unit/mem_cache/test_fused_draft_placement.py new file mode 100644 index 0000000000..98db659458 --- /dev/null +++ b/test/registered/unit/mem_cache/test_fused_draft_placement.py @@ -0,0 +1,241 @@ +# Copyright 2023-2026 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""`place_fused_draft`: which host/draft pairs fuse, and where each runner's +layers land. + +Every decline is the private-pool fallback, so a wrong answer never fails +the boot -- it silently changes what the draft binds and what the boot solve +prices. Pinned: + - a replicated head under multi-layer EAGLE gets one lane RANGE per runner + (one shared region would let the runners clobber each other's KV); + - a per-depth head serves one depth per runner and needs one runner per + depth; + - a draft with SWA layers of its own, or asymmetric K/V rows, declines: + the fused arm binds one dense pool over the host's full slots; + - the profile divides the draft's heads by attn_tp, as the target does; + - a placement whose runner lane counts do not fill its region is refused; + - the priced entry counts the layers THIS runner owns, not the whole model's; + - only a two-pool hybrid-SWA target with a stateless draft is placed. + + python -m pytest test/registered/unit/mem_cache/test_fused_draft_placement.py -v +""" + +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +import torch + +from sglang.srt.mem_cache.layout.fused_draft import ( + DenseDraftRegion, + DraftKVGeometry, + DraftKVProfile, + FusedDraftPlacement, + draft_kv_profile, + place_fused_draft, +) +from sglang.srt.runtime_context import get_parallel +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + +_DTYPE = torch.bfloat16 + + +def _profile( + *, + num_layers=1, + swa_layer_ids=(), + num_depths=1, + head_dim=64, + v_head_dim=64, +): + return DraftKVProfile( + num_layers=num_layers, + full=DraftKVGeometry(head_num=4, head_dim=head_dim, v_head_dim=v_head_dim), + swa_layer_ids=swa_layer_ids, + num_depths=num_depths, + ) + + +def _place(profile, num_runners=1): + return place_fused_draft( + profile=profile, + num_runners=num_runners, + store_dtype=_DTYPE, + ) + + +class TestPlaceFusedDraft(CustomTestCase): + def test_replicated_head_gets_one_slot_range_per_runner(self): + decision = _place(_profile(num_layers=2), num_runners=3) + placement = decision.placement + self.assertIsNotNone(placement, decision.declined) + self.assertEqual(placement.region.lane_num, 6) + for r in range(3): + self.assertEqual(placement.lanes_for(r), range(2 * r, 2 * r + 2)) + + def test_per_depth_head_serves_one_depth_per_runner(self): + decision = _place(_profile(num_layers=8, num_depths=8), num_runners=2) + placement = decision.placement + self.assertIsNotNone(placement, decision.declined) + self.assertEqual(placement.region.lane_num, 2) + self.assertEqual(placement.lanes_for(1), range(1, 2)) + + def test_per_depth_head_needs_one_runner_per_depth(self): + self.assertIsNone(_place(_profile(num_layers=8, num_depths=8), 1).placement) + self.assertIsNone(_place(_profile(num_layers=8, num_depths=8), 9).placement) + + def test_swa_asymmetric_and_misaligned_drafts_decline(self): + for profile in ( + _profile(swa_layer_ids=(0,)), + _profile(head_dim=64, v_head_dim=32), + # 4 heads x 5 dims x 2 B = 40 B: not a 16-B-aligned entry part. + _profile(head_dim=5, v_head_dim=5), + ): + decision = _place(profile) + self.assertIsNone(decision.placement) + self.assertIsNotNone(decision.declined) + + def test_region_carries_the_profile_geometry(self): + placement = _place(_profile()).placement + self.assertEqual( + placement.region, + DenseDraftRegion( + lane_num=1, head_num=4, head_dim=64, v_head_dim=64, store_dtype=_DTYPE + ), + ) + + +class TestDraftKVProfile(CustomTestCase): + def test_heads_are_divided_by_attn_tp(self): + mc = SimpleNamespace( + is_hybrid_swa=True, + is_deepseek_v4_arch=False, + swa_attention_layer_ids=[0], + num_nextn_predict_layers=None, + get_num_kv_heads=lambda tp: max(1, 8 // tp), + head_dim=64, + v_head_dim=32, + ) + profile = draft_kv_profile(mc, num_layers=1, attn_tp_size=2) + self.assertEqual( + profile.full, DraftKVGeometry(head_num=4, head_dim=64, v_head_dim=32) + ) + self.assertEqual(profile.swa_layer_ids, (0,)) + self.assertEqual(profile.num_depths, 1) + + +class TestFusedDraftPlacement(CustomTestCase): + def test_runner_lane_counts_must_fill_the_region(self): + region = DenseDraftRegion( + lane_num=2, head_num=1, head_dim=8, store_dtype=_DTYPE + ) + with self.assertRaises(AssertionError): + FusedDraftPlacement(region=region, runner_lane_counts=(1,)) + with self.assertRaises(AssertionError): + FusedDraftPlacement(region=region, runner_lane_counts=()) + + +class TestFusedEntryPricing(CustomTestCase): + """The boot solve prices a fused entry through the same spec the pool + factory builds, so the two cannot drift. The factory builds the full + sub-pool from this runner's OWN layer slice; pricing from the whole-model + split over-counts every layer another pipeline rank holds, and the solve + then hands out fewer tokens than fit.""" + + def _configurator(self, *, owned, whole): + from sglang.srt.mem_cache.kv_cache_configurator import KVCacheConfigurator + + cfg = KVCacheConfigurator.__new__(KVCacheConfigurator) + cfg.layer_info = SimpleNamespace(full_attention_layer_ids=owned) + cfg.model_config = SimpleNamespace( + full_attention_layer_ids=whole, + head_dim=8, + get_num_kv_heads=lambda *_: 1, + ) + cfg.kv_cache_dtype = _DTYPE + return cfg + + def test_the_priced_entry_counts_only_this_runners_layers(self): + region = DenseDraftRegion( + lane_num=1, head_num=1, head_dim=8, store_dtype=_DTYPE + ) + cfg = self._configurator(owned=[0, 1], whole=[0, 1, 2, 3]) + with get_parallel().override(attn_tp_size=1, attn_dcp_size=1): + spec = cfg._full_host_spec(region) + self.assertEqual(spec.layer_num, 2) + + +class TestFusedDraftGate(CustomTestCase): + """Only the two-pool hybrid-SWA factory takes a placement here. A target + that also carries recurrent state builds its pools unfused, so answering + a placement for it would price a fused entry the factory never allocates; + a draft with recurrent state needs a state pool the fused arm lacks.""" + + def _decide(self, *, target_state=None, draft_state=None, draft_kv_dtype=None): + from sglang.srt.mem_cache import kv_cache_configurator as kvc + + cfg = kvc.KVCacheConfigurator.__new__(kvc.KVCacheConfigurator) + cfg.is_hybrid_swa = True + cfg.mambaish_config = target_state + cfg.is_draft_worker = False + cfg.spec_algorithm = SimpleNamespace(is_eagle=lambda: True) + cfg.model_config = SimpleNamespace(is_multi_layer_eagle=False) + cfg.kv_cache_dtype = _DTYPE + cfg.spec_aux_config = SimpleNamespace( + eagle_draft_num_layers=1, + draft_kv_num_layers=1, + draft_model_config=SimpleNamespace( + is_hybrid_swa=False, + is_deepseek_v4_arch=False, + num_nextn_predict_layers=None, + get_num_kv_heads=lambda tp: 4, + head_dim=64, + v_head_dim=64, + dtype=torch.bfloat16, + ), + ) + memory = SimpleNamespace(enable_unified_memory=True) + spec = SimpleNamespace( + speculative_num_steps=1, speculative_draft_kv_cache_dtype=draft_kv_dtype + ) + with ( + patch.object(kvc, "get_memory", return_value=memory), + patch.object(kvc, "get_spec", return_value=spec), + patch.object(kvc, "mambaish_config", return_value=draft_state), + get_parallel().override(attn_tp_size=1), + ): + return cfg._fused_draft_decision() + + def test_a_two_pool_target_places_a_stateless_draft(self): + self.assertIsNotNone(self._decide().placement) + + def test_a_draft_kv_dtype_unlike_the_host_keeps_the_private_pool(self): + """A fused draft stores its rows in the host's KV dtype; an explicit + draft dtype that differs declines instead of mis-typing those rows.""" + self.assertIsNotNone(self._decide(draft_kv_dtype="auto").placement) + declined = self._decide(draft_kv_dtype="fp8_e4m3") + self.assertIsNone(declined.placement) + self.assertIn("KV cache dtype", declined.declined) + + def test_recurrent_state_on_either_side_keeps_the_private_pool(self): + state = SimpleNamespace() + for kw in ({"target_state": state}, {"draft_state": state}): + self.assertIsNone(self._decide(**kw).placement, kw) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/mem_cache/test_kv_cache_configurator_draft_gate.py b/test/registered/unit/mem_cache/test_kv_cache_configurator_draft_gate.py new file mode 100644 index 0000000000..07673bfdee --- /dev/null +++ b/test/registered/unit/mem_cache/test_kv_cache_configurator_draft_gate.py @@ -0,0 +1,249 @@ +# Copyright 2023-2026 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""How a draft-shaped KVCacheConfigurator binds its KV under the unified pool. + +DERIVED PROPERTY (binding dispatch). The fused-vs-private draft binding +dispatches on the spec algorithm, not the allocator kind: a non-EAGLE draft on +a unified SWA target takes the private arm, sized by the full sub-allocator's +VIRTUAL id space (`max_slots - 1`). The SWA allocator's `size_full` reports +the static token budget — smaller than the id space — so sizing by it would +put verify-window writes at high virtual ids out of bounds. + + python -m pytest test/registered/unit/mem_cache/test_kv_cache_configurator_draft_gate.py -v +""" + +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +import torch + +from sglang.srt.mem_cache import kv_cache_configurator as kcc +from sglang.srt.mem_cache import unified_draft_pool +from sglang.srt.mem_cache.allocator.unified_hybrid_swa import ( + UnifiedSWATokenToKVPoolAllocator, +) +from sglang.srt.mem_cache.layout.fused_draft import ( + DenseDraftRegion, + FusedDraftPlacement, +) +from sglang.srt.mem_cache.unified_draft_pool import UnifiedDraftKVPool +from sglang.srt.mem_cache.unified_memory_pool import MHASubPoolSpec, UnifiedKVPool +from sglang.srt.runtime_context import publish, reset_context +from sglang.srt.server_args import ServerArgs +from sglang.srt.speculative.spec_info import SpeculativeAlgorithm +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + + +class _FakeKVCache: + def __init__(self, max_slots): + self.buf = torch.full((max_slots,), -1, dtype=torch.int64) + self.allocator = None + + def attach_allocator(self, allocator): + self.allocator = allocator + + +class _FakeUnifiedSWAKVPool: + def __init__(self, shared_pool): + self.full_kv_pool = _FakeKVCache(shared_pool.max_slots("full")) + self.swa_kv_pool = _FakeKVCache(shared_pool.max_slots("swa")) + self.full_to_swa_index_mapping = None + + def attach_allocators(self, *, full_allocator, swa_allocator): + self._full_allocator = full_allocator + self._swa_allocator = swa_allocator + + +class _CapturedSizes(Exception): + """Sentinel carrying the sizes handed to the token-pool build.""" + + def __init__(self, sizes): + self.sizes = sizes + + +_PS = 2 + + +class TestDraftBindingDispatch(CustomTestCase): + def setUp(self): + # `KVIndexTranslator.__init__` reads `attn_dcp_size`, a derived + # parallel width that only exists once a config is published. + reset_context() + self.addCleanup(reset_context) + publish(ServerArgs(model_path="dummy"), role="tokenizer") + + def _swa_allocator(self, *, with_draft_region: bool, n_full=32, n_swa=16): + region = ( + DenseDraftRegion( + lane_num=1, head_num=1, head_dim=8, store_dtype=torch.bfloat16 + ) + if with_draft_region + else None + ) + full_spec = MHASubPoolSpec( + name="full", + layer_num=2, + head_num=2, + head_dim=4, + store_dtype=torch.bfloat16, + grow_direction="down", + draft_region=region, + ) + swa_spec = MHASubPoolSpec( + name="swa", + layer_num=1, + head_num=2, + head_dim=4, + store_dtype=torch.bfloat16, + grow_direction="up", + ) + total = n_full * full_spec.entry_bytes() + n_swa * swa_spec.entry_bytes() + pool = UnifiedKVPool( + total_bytes=total, + sub_pool_specs=[full_spec, swa_spec], + device="cpu", + enable_memory_saver=False, + page_size=_PS, + fused_draft=( + FusedDraftPlacement(region=region, runner_lane_counts=(1,)) + if region is not None + else None + ), + ) + return UnifiedSWATokenToKVPoolAllocator( + unified_buffer=pool, + kvcache=_FakeUnifiedSWAKVPool(pool), + device="cpu", + full_max_total_num_tokens=n_full, + swa_max_total_num_tokens=n_swa, + page_size=_PS, + need_sort=False, + forward_stream=None, + ) + + def _run(self, *, algorithm, alloc, max_total_num_tokens, kv_dtype=torch.bfloat16): + cfg = kcc.KVCacheConfigurator.__new__(kcc.KVCacheConfigurator) + cfg.kv_cache_dtype = kv_dtype + cfg.is_draft_worker = True + cfg.draft_model_idx = None + cfg.spec_algorithm = algorithm + cfg.page_size = _PS + cfg.is_hybrid_swa = False + cfg.is_hybrid_swa_mtp_draft = False + cfg.model = object() + cfg.model_config = SimpleNamespace(hf_config=None, is_hybrid_swa=False) + sizes = kcc._PoolSizes( + max_total_num_tokens=max_total_num_tokens, + max_running_requests=8, + full_max_total_num_tokens=max_total_num_tokens, + swa_max_total_num_tokens=16, + c4_max_total_num_tokens=0, + c128_max_total_num_tokens=0, + c4_state_pool_size=0, + c128_state_pool_size=0, + c4_state_dtype=None, + c128_state_dtype=None, + ) + + def _capture(self, *, sizes, **kw): + raise _CapturedSizes(sizes) + + with ( + patch.object( + kcc, + "get_memory", + return_value=SimpleNamespace(enable_unified_memory=True), + ), + patch.object( + kcc, + "get_schedule", + return_value=SimpleNamespace(page_size=_PS), + ), + patch.object(kcc, "is_deepseek_dsa", return_value=False), + patch.object(kcc, "is_deepseek_v4", return_value=False), + patch.object( + kcc.KVCacheConfigurator, + "_validate_prefill_only_disable_kv_cache_pool_family", + lambda self, *a, **kw: None, + ), + patch.object(kcc.KVCacheConfigurator, "_build_token_to_kv_pool", _capture), + # The real scan walks the draft nn.Module for RadixAttention layers. + patch.object(unified_draft_pool, "draft_kv_layer_ids", return_value=[0]), + ): + return cfg._init_pools( + sizes=sizes, + req_to_token_pool=object(), + token_to_kv_pool_allocator=alloc, + ) + + def test_non_eagle_draft_takes_the_private_arm_sized_by_the_id_space(self): + alloc = self._swa_allocator(with_draft_region=False) + id_space = alloc.full_attn_allocator.max_slots - 1 + # The distinction under test only exists while budget < id space. + self.assertLess(alloc.size_full, id_space) + with self.assertRaises(_CapturedSizes) as caught: + self._run( + algorithm=SpeculativeAlgorithm.DSPARK, + alloc=alloc, + max_total_num_tokens=alloc.size_full, + ) + sized = caught.exception.sizes.max_total_num_tokens + self.assertEqual(sized, (id_space + _PS - 1) // _PS * _PS) + + def test_eagle_draft_still_binds_the_fused_pool(self): + alloc = self._swa_allocator(with_draft_region=True) + pools = self._run( + algorithm=SpeculativeAlgorithm.EAGLE3, + alloc=alloc, + max_total_num_tokens=alloc.size_full, + ) + self.assertIsInstance(pools.token_to_kv_pool, UnifiedDraftKVPool) + self.assertIs(pools.token_to_kv_pool_allocator, alloc) + + def test_a_draft_kv_dtype_unlike_the_region_refuses_to_bind(self): + """The region stores the target's KV dtype. A draft that resolved its + own -- `auto` picking up an fp8 quant config the target lacks, say -- + must fail loudly rather than read those rows as another dtype.""" + alloc = self._swa_allocator(with_draft_region=True) + with self.assertRaisesRegex(ValueError, "speculative-draft-kv-cache-dtype"): + self._run( + algorithm=SpeculativeAlgorithm.EAGLE3, + alloc=alloc, + max_total_num_tokens=alloc.size_full, + kv_dtype=torch.float8_e4m3fn, + ) + + def test_eagle_draft_without_a_placement_falls_back_to_the_private_arm(self): + """Target boot declines a placement for legitimate configurations (a + draft with SWA layers of its own, asymmetric rows), so a + placement-less EAGLE draft binds the private pool -- sized by the id + space like any other private draft -- instead of failing the boot.""" + alloc = self._swa_allocator(with_draft_region=False) + id_space = alloc.full_attn_allocator.max_slots - 1 + with self.assertRaises(_CapturedSizes) as caught: + self._run( + algorithm=SpeculativeAlgorithm.EAGLE3, + alloc=alloc, + max_total_num_tokens=alloc.size_full, + ) + sized = caught.exception.sizes.max_total_num_tokens + self.assertEqual(sized, (id_space + _PS - 1) // _PS * _PS) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/mem_cache/test_kv_index_translator_draft_disposition.py b/test/registered/unit/mem_cache/test_kv_index_translator_draft_disposition.py new file mode 100644 index 0000000000..c634e9a5a0 --- /dev/null +++ b/test/registered/unit/mem_cache/test_kv_index_translator_draft_disposition.py @@ -0,0 +1,369 @@ +# Copyright 2023-2026 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""KVIndexTranslator's three id-space dispositions with a fused draft region. + +1. TARGET runner (allocator's own kvcache): translates to physical ids. +2. FUSED-DRAFT runner (`UnifiedDraftKVPool` bound to this allocator): the + SAME translate -- same pages, same v2p table, same physical ids (the draft + parts sit inside the host entry). A dense draft pool routes no window + layers, so it has no swa id space: the window index table falls back to + the one dense table and `sliding_window_write_loc_for()` answers None (a + separate-swa assumption here crashed the read side and left the write side + with nothing to derive). +3. PRIVATE-POOL draft (DSPARK/DFLASH shape: target's allocator, own + virtual-indexed buffer): strict passthrough — translating such a runner + would address a slot-count buffer with dense ids (OOB both directions), + so the no-op is load-bearing, not a default. + + python -m pytest test/registered/unit/mem_cache/test_kv_index_translator_draft_disposition.py -v +""" + +import ast +import pathlib +import unittest +from types import SimpleNamespace + +import torch + +from sglang.srt.mem_cache import kv_index_translator +from sglang.srt.mem_cache.allocator.unified_hybrid_swa import ( + UnifiedSWATokenToKVPoolAllocator, +) +from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator +from sglang.srt.mem_cache.layout.fused_draft import ( + DenseDraftRegion, + FusedDraftPlacement, +) +from sglang.srt.mem_cache.unified_draft_pool import UnifiedDraftKVPool +from sglang.srt.mem_cache.unified_memory_pool import MHASubPoolSpec, UnifiedKVPool +from sglang.srt.runtime_context import publish, reset_context +from sglang.srt.server_args import ServerArgs +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=10, suite="base-a-test-cpu") + +_DEV = "cpu" +_PS = 2 + + +class _FakeKVCache: + def __init__(self, max_slots): + self.buf = torch.full((max_slots,), -1, dtype=torch.int64) + self.allocator = None + + def attach_allocator(self, allocator): + self.allocator = allocator + + def move_kv_cache(self, dst_loc, src_loc): + self.buf[dst_loc] = self.buf[src_loc].clone() + + +class _FakeUnifiedSWAKVPool: + def __init__(self, shared_pool): + self.full_kv_pool = _FakeKVCache(shared_pool.max_slots("full")) + self.swa_kv_pool = _FakeKVCache(shared_pool.max_slots("swa")) + self.full_to_swa_index_mapping = None + + def attach_allocators(self, *, full_allocator, swa_allocator): + self._full_allocator = full_allocator + self._swa_allocator = swa_allocator + + +def _build(n_full=32, n_swa=16): + full_spec = MHASubPoolSpec( + name="full", + layer_num=2, + head_num=2, + head_dim=4, + store_dtype=torch.bfloat16, + grow_direction="down", + draft_region=DenseDraftRegion( + lane_num=1, head_num=1, head_dim=8, store_dtype=torch.bfloat16 + ), + ) + swa_spec = MHASubPoolSpec( + name="swa", + layer_num=1, + head_num=2, + head_dim=4, + store_dtype=torch.bfloat16, + grow_direction="up", + ) + total = n_full * full_spec.entry_bytes() + n_swa * swa_spec.entry_bytes() + pool = UnifiedKVPool( + total_bytes=total, + sub_pool_specs=[full_spec, swa_spec], + device=_DEV, + enable_memory_saver=False, + page_size=_PS, + fused_draft=FusedDraftPlacement( + region=full_spec.draft_region, runner_lane_counts=(1,) + ), + ) + kvcache = _FakeUnifiedSWAKVPool(pool) + allocator = UnifiedSWATokenToKVPoolAllocator( + unified_buffer=pool, + kvcache=kvcache, + device=_DEV, + full_max_total_num_tokens=n_full, + swa_max_total_num_tokens=n_swa, + page_size=_PS, + need_sort=False, + forward_stream=None, + ) + draft_pool = UnifiedDraftKVPool( + unified_buffer=pool, + host_sub_pool_name="full", + host_allocator=allocator, + layer_lanes={0: 0}, + page_size=_PS, + ) + return pool, allocator, kvcache, draft_pool + + +def _source(allocator, pool_obj): + return KVIndexTranslator( + req_to_token=torch.zeros((4, 16), dtype=torch.int32), + token_to_kv_pool_allocator=allocator, + token_to_kv_pool=pool_obj, + page_size=_PS, + device=_DEV, + ) + + +class TestKVIndexSourceDraftDisposition(unittest.TestCase): + def setUp(self): + # `KVIndexTranslator.__init__` reads `attn_dcp_size`, a derived + # parallel width that only exists once a config is published. + reset_context() + self.addCleanup(reset_context) + publish(ServerArgs(model_path="dummy"), role="tokenizer") + + def test_target_runner_translates_to_physical_ids(self): + _, allocator, kvcache, _ = _build() + src = _source(allocator, kvcache) + self.assertTrue(src.is_translating) + v = allocator.alloc(2 * _PS) + self.assertIsNotNone(v) + fb = SimpleNamespace(out_cache_loc=v) + src.rebind_write_loc(fb) + self.assertTrue(torch.equal(fb.out_cache_loc, allocator.translate_kv_loc(v))) + + def test_fused_draft_runner_translates_to_the_same_physical_ids(self): + _, allocator, _, draft_pool = _build() + src = _source(allocator, draft_pool) + self.assertTrue(src.is_translating) + self.assertIsNone(src._swa_v2p_table) # single space: no swa id space + + # rebind_write_loc translates to the HOST's physical ids: the draft + # parts live inside the same slots. + v = allocator.alloc(2 * _PS) + self.assertIsNotNone(v) + fb = SimpleNamespace(out_cache_loc=v) + src.rebind_write_loc(fb) + self.assertIsNot(fb.out_cache_loc, v) + fa = allocator.full_attn_allocator + expected = torch.clamp_min(fa.virtual_to_physical[v // _PS] * _PS + v % _PS, 0) + torch.testing.assert_close(fb.out_cache_loc, expected, rtol=0, atol=0) + self.assertTrue(torch.equal(fb.out_cache_loc, allocator.translate_kv_loc(v))) + # A dense draft pool routes no window layers: no swa write loc. + self.assertIsNone(src.sliding_window_write_loc_for(fb.out_cache_loc)) + + def test_dense_fused_draft_has_no_window_write_loc(self): + """`sliding_window_write_loc_for()` on a dense fused-draft batch answers + None: the pool routes no window layers, so there is nothing to + derive. The hybrid-SWA TARGET on the same allocator keeps a genuinely + derived swa loc, so the None can never paper over a real swa id + space.""" + _, allocator, kvcache, draft_pool = _build() + src = _source(allocator, draft_pool) + v = allocator.alloc(_PS) + self.assertIsNotNone(v) + fb = SimpleNamespace(out_cache_loc=v) + src.rebind_write_loc(fb) + self.assertIsNone(src.sliding_window_write_loc_for(fb.out_cache_loc)) + # Target contrast: a real swa side derives a DIFFERENT loc. + tgt = _source(allocator, kvcache) + tv = allocator.alloc(_PS) + self.assertIsNotNone(tv) + tfb = SimpleNamespace(out_cache_loc=tv) + tgt.rebind_write_loc(tfb) + tswa = tgt.sliding_window_write_loc_for(tfb.out_cache_loc) + self.assertIsNotNone(tswa) + self.assertIsNot(tswa, tfb.out_cache_loc) + self.assertTrue(torch.equal(tswa, allocator.translate_loc_from_full_to_swa(tv))) + + def test_private_pool_draft_stays_a_strict_passthrough(self): + _, allocator, _, _ = _build() + private_draft_pool = _FakeKVCache(64) # own buffer, not the allocator's + src = _source(allocator, private_draft_pool) + self.assertFalse(src.is_translating) + v = torch.arange(2 * _PS, dtype=torch.int64) + fb = SimpleNamespace(out_cache_loc=v) + src.rebind_write_loc(fb) + self.assertIs(fb.out_cache_loc, v) # untouched, not even a copy + + def test_foreign_allocator_draft_pool_is_not_enabled(self): + # A UnifiedDraftKVPool bound to a DIFFERENT allocator must not enable + # against this one (identity, not type, decides). + pool, allocator, kvcache, _ = _build() + _, other_alloc, _, other_draft = _build() + src = _source(allocator, other_draft) + self.assertFalse(src.is_translating) + + def test_full_flat_v2p_per_disposition(self): + """The flat-translate accessor must hand a kernel exactly what + `translate_kv_loc` would use: the shared v2p table on the target and + on the fused draft, and None on a pass-through runner (a translated + pass-through would address a private buffer with the target's ids).""" + _, allocator, kvcache, draft_pool = _build() + + target = _source(allocator, kvcache) + self.assertIs(target.full_flat_v2p(), allocator.full_v2p_page_table) + + draft = _source(allocator, draft_pool) + self.assertIs(draft.full_flat_v2p(), allocator.full_v2p_page_table) + + passthrough = _source(allocator, _FakeKVCache(64)) + self.assertIsNone(passthrough.full_flat_v2p()) + + def test_multi_step_containers_pass_the_v2p_table_to_the_kernel(self): + """Every `generate_draft_decode_kv_indices` launch must thread the + runner's v2p table and a matching TRANSLATE flag. A launch + without them emits raw req_to_token values, which under the unified + pool are VIRTUAL ids the fused draft pool cannot address — the exact + silent-garbage bug this series fixed.""" + import pathlib + import re + + import sglang.srt.mem_cache.kv_index_translator as _kit + + root = pathlib.Path(_kit.__file__).parent.parent / "layers" / "attention" + launching = {} + for path in sorted(root.glob("*.py")): + text = path.read_text() + launches = len(re.findall(r"generate_draft_decode_kv_indices\[", text)) + if launches: + launching[path.name] = ( + launches, + len(re.findall(r"TRANSLATE=v2p is not None", text)), + "full_flat_v2p" in text, + ) + self.assertGreaterEqual( + len(launching), 4, f"launch sites disappeared: {sorted(launching)}" + ) + for name, (launches, translated, has_accessor) in launching.items(): + self.assertEqual( + launches, + translated, + f"{name}: {launches} kernel launch(es) but only {translated} " + "carry TRANSLATE=v2p is not None", + ) + self.assertTrue( + has_accessor, + f"{name} launches the kernel without full_flat_v2p", + ) + + +class TestDispositionBranchesAgree(unittest.TestCase): + """Every `__init__` disposition must assign the SAME attribute set. + + BUG REGRESSION. `KVIndexTranslator.__init__` picks a disposition -- + translating (unified target or fused draft) or passthrough -- and later + methods read attributes off `self` unconditionally. When upstream adds an + attribute it naturally adds it to the branches IT knows about; a branch + added here is silently left short, and nothing fails until the missing + attribute is read at RUNTIME. That is exactly how the fused-draft branch + lost `_translate_write_full` and `defer_read_translate` across a rebase: + py_compile passes (the attribute is only ever read, never declared), the + undefined-NAME check passes (it is an attribute, not a bare name), and the + target and passthrough paths both work -- only a fused-draft forward raises + `AttributeError: 'KVIndexTranslator' object has no attribute + '_translate_write_full'`. + + Comparing the branches against EACH OTHER needs no list to maintain: a new + attribute is covered the moment any one branch sets it. + + python -m pytest test/registered/unit/mem_cache/test_kv_index_translator_draft_disposition.py -v + """ + + def setUp(self): + # `KVIndexTranslator.__init__` reads `attn_dcp_size`, a derived + # parallel width that only exists once a config is published. + reset_context() + self.addCleanup(reset_context) + publish(ServerArgs(model_path="dummy"), role="tokenizer") + + def _init_branch_assignments(self): + """{branch index: {attr names it assigns}} for __init__'s if/elif/else.""" + src = pathlib.Path(kv_index_translator.__file__).read_text() + tree = ast.parse(src) + cls = next( + n + for n in ast.walk(tree) + if isinstance(n, ast.ClassDef) and n.name == "KVIndexTranslator" + ) + init = next( + n + for n in cls.body + if isinstance(n, ast.FunctionDef) and n.name == "__init__" + ) + + # The disposition chain is the `if` whose body assigns the id-space + # fields; find it by the attribute every disposition must set. + def assigns(stmts): + out = set() + for st in stmts: + for node in ast.walk(st): + if isinstance(node, ast.Assign): + for t in node.targets: + if ( + isinstance(t, ast.Attribute) + and isinstance(t.value, ast.Name) + and t.value.id == "self" + ): + out.add(t.attr) + return out + + for node in init.body: + if not isinstance(node, ast.If): + continue + branches, cur = [], node + while True: + branches.append(assigns(cur.body)) + if len(cur.orelse) == 1 and isinstance(cur.orelse[0], ast.If): + cur = cur.orelse[0] + continue + if cur.orelse: + branches.append(assigns(cur.orelse)) + break + if len(branches) >= 2 and all("_full_v2p_table" in b for b in branches): + return branches + self.fail("could not locate the disposition if/elif/else in __init__") + + def test_every_disposition_assigns_the_same_attributes(self): + branches = self._init_branch_assignments() + union = set().union(*branches) + missing = {i: sorted(union - b) for i, b in enumerate(branches) if union - b} + self.assertEqual( + missing, + {}, + "a KVIndexTranslator.__init__ disposition does not assign every " + "attribute its siblings do; the branch raises AttributeError only " + "when that path is taken at runtime: " + repr(missing), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/model_executor/test_pool_configurator.py b/test/registered/unit/model_executor/test_pool_configurator.py index 3fa29e5ce0..0846bd8f54 100644 --- a/test/registered/unit/model_executor/test_pool_configurator.py +++ b/test/registered/unit/model_executor/test_pool_configurator.py @@ -179,6 +179,9 @@ def _make_model_runner( eagle_draft_swa_num_layers=None, dflash_draft_num_layers=None, ) + # Fused draft KV is off unless a test opts in (a bare MagicMock return + # would read as a truthy entry size and take the fused pricing branch). + mr.fused_entry_bytes.return_value = None return mr @@ -1430,5 +1433,106 @@ class TestSWAPoolFloor(CustomTestCase): ) +class TestFusedDraftPricing(unittest.TestCase): + """Fused draft KV: the boot solve must price the EXACT fused entry. + + The fused full-side entry (host + draft + alignment pad) replaces BOTH the + per-token full term and the per-draft-layer approximation; charging both + would over-reserve, charging only the approximation would under-reserve + and OOM at pool construction (the factory allocates tokens x fused entry). + """ + + def _make(self, fused_entry, draft_layers, **runner_kwargs): + mr = _make_model_runner( + self, + is_hybrid_swa=True, + full_attention_layer_ids=list(range(4)), + swa_attention_layer_ids=list(range(4, 12)), + swa_num_kv_heads=4, + page_size=1, + swa_full_tokens_ratio=0.5, + **runner_kwargs, + ) + mr.spec_algorithm.is_eagle.return_value = True + mr.spec_algorithm.is_standalone.return_value = False + mr.spec_algorithm.is_none.return_value = False + mr.spec_aux_config.eagle_draft_num_layers = draft_layers + mr.fused_entry_bytes.side_effect = lambda name: ( + fused_entry if name == "full" else None + ) + with mock_cpu_env(): + from sglang.srt.model_executor.pool_configurator import ( + create_memory_pool_configurator, + ) + + return mr, create_memory_pool_configurator(mr) + + def _expected_cell(self, cfg, full_term): + return ( + full_term + + cfg._swa_per_token * cfg._draft_swa_full_layers_num + + cfg._swa_full_tokens_ratio * cfg._swa_per_token * cfg._swa_layers_num + + cfg._draft_cell_size + ) + + def test_unfused_keeps_the_draft_layer_approximation(self): + _, cfg = self._make(fused_entry=None, draft_layers=2) + self.assertEqual(cfg._draft_full_layers_num, 2) + self.assertEqual( + cfg._cell_size, + self._expected_cell(cfg, cfg._full_per_token * (4 + 2)), + ) + + def test_fused_entry_replaces_full_and_draft_terms(self): + fused_entry = 54_321 + _, cfg = self._make(fused_entry=fused_entry, draft_layers=2) + self.assertEqual(cfg._draft_full_layers_num, 0) + self.assertEqual(cfg._cell_size, self._expected_cell(cfg, fused_entry)) + + def _assert_unified_bytes_price_the_fused_entry(self, cfg, config, available): + """The bytes handed to the unified pool must buy every full token at + the fused entry -- the factory divides them by it -- and must not leave + a full token's worth of the budget unspent.""" + fused_entry = cfg._fused_full_entry + swa_bytes = ( + config.swa_max_total_num_tokens * cfg._swa_per_token * cfg._swa_layers_num + ) + self.assertEqual( + config.unified_memory_pool_bytes, + config.full_max_total_num_tokens * fused_entry + swa_bytes, + ) + self.assertLessEqual(config.unified_memory_pool_bytes, available) + self.assertGreater( + config.unified_memory_pool_bytes + 2 * fused_entry, available + ) + + def test_unified_solve_prices_the_fused_entry(self): + available = 50_000_000 + _, cfg = self._make( + fused_entry=54_321, draft_layers=2, enable_unified_memory=True + ) + config = cfg.calculate_pool_sizes(available, 1) + self._assert_unified_bytes_price_the_fused_entry(cfg, config, available) + + def test_chunk_cap_solve_prices_the_fused_entry(self): + from sglang.srt.model_executor.pool_configurator import ( + SWAChunkCapPoolConfigurator, + ) + + available = 50_000_000 + _, cfg = self._make( + fused_entry=54_321, + draft_layers=2, + enable_unified_memory=True, + disable_radix_cache=True, + chunked_prefill_size=4, + sliding_window_size=8, + max_running_requests=2, + ) + self.assertIsInstance(cfg, SWAChunkCapPoolConfigurator) + config = cfg.calculate_pool_sizes(available, 1) + self._assert_unified_bytes_price_the_fused_entry(cfg, config, available) + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/model_executor/test_spec_aux_hidden_state_boot_order.py b/test/registered/unit/model_executor/test_spec_aux_hidden_state_boot_order.py new file mode 100644 index 0000000000..566cf048cc --- /dev/null +++ b/test/registered/unit/model_executor/test_spec_aux_hidden_state_boot_order.py @@ -0,0 +1,191 @@ +# Copyright 2023-2026 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""`resolve_spec_aux_hidden_state_config` runs before the TP group exists. + +BUG REGRESSION. `ModelRunner.init_spec_aux_hidden_state()` runs BEFORE +`init_torch_distributed()`, so the attention-TP group does not exist while +this resolver runs. A `get_parallel().attn_tp_size` read inside it therefore +raises `AssertionError: attention tensor model parallel group is not +initialized` and kills the server at startup -- for EVERY EAGLE-family run +with a draft path, unified pool or not. It is invisible to CPU unit gates +(they never boot a runner) and was caught only by a GPU boot guard. + +The resolver therefore records the draft's `ModelConfig` whole, and the +per-GPU geometry is derived at pool-build time (`draft_kv_profile`), where +the group exists -- the same point the target divides its own heads. + + python -m pytest test/registered/unit/model_executor/test_spec_aux_hidden_state_boot_order.py -v +""" + +import ast +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +import sglang.srt.model_executor.model_runner_components.spec_aux_hidden_state as _mod +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + +_SRC = Path(_mod.__file__) + + +def _function_named(tree, name): + for node in ast.walk(tree): + if isinstance(node, ast.FunctionDef) and node.name == name: + return node + raise AssertionError(f"{name} not found in {_SRC}") + + +def _config(*, num_nextn_predict_layers, num_hidden_layers=1): + return SimpleNamespace( + num_nextn_predict_layers=num_nextn_predict_layers, + num_hidden_layers=num_hidden_layers, + num_attention_layers=num_hidden_layers, + is_hybrid_swa=False, + is_deepseek_v4_arch=False, + ) + + +def _spec_algorithm(*, eagle: bool, eagle3: bool = False): + return SimpleNamespace( + is_eagle=lambda: eagle, + is_standalone=lambda: False, + is_eagle3=lambda: eagle3, + ) + + +class TestResolverTouchesNoParallelState(unittest.TestCase): + def test_no_parallel_state_in_the_eagle_resolver(self): + """AST pin: the EAGLE resolver runs pre-distributed-init, so any + parallel-state read in it -- `get_parallel()` directly, or a per-GPU + head count that divides by it -- is a boot crash, not a wrong number.""" + tree = ast.parse(_SRC.read_text()) + fn = _function_named(tree, "_resolve_eagle_aux_hidden_state") + called = { + n.func.id + for n in ast.walk(fn) + if isinstance(n, ast.Call) and isinstance(n.func, ast.Name) + } + self.assertNotIn( + "get_parallel", + called, + "_resolve_eagle_aux_hidden_state reads parallel state, but it runs " + "before init_torch_distributed() -- the attention-TP group does not " + "exist yet and the server dies at startup.", + ) + methods = { + n.func.attr + for n in ast.walk(fn) + if isinstance(n, ast.Call) and isinstance(n.func, ast.Attribute) + } + self.assertNotIn("get_num_kv_heads", methods) + imported = { + alias.name + for n in ast.walk(fn) + if isinstance(n, ast.ImportFrom) + for alias in n.names + } + self.assertNotIn("get_parallel", imported) + + def _resolve_with(self, *, draft_path, target, draft): + """Run the EAGLE resolver with the config factory stubbed: `draft` + answers the draft-mode read, `target` is the runner's own config.""" + config = _mod.SpecAuxHiddenStateConfig() + spec = SimpleNamespace( + speculative_draft_model_path=draft_path, + speculative_draft_model_revision=None, + ) + with ( + patch.object(_mod, "get_spec", return_value=spec), + patch.object( + _mod.ModelConfig, "from_server_args", return_value=draft + ) as factory, + ): + _mod._resolve_eagle_aux_hidden_state( + config=config, + server_args=None, + model_config=target, + spec_algorithm=_spec_algorithm(eagle=True), + is_draft_worker=False, + ) + self.assertTrue(factory.call_args.kwargs["is_draft_model"]) + return config + + def test_path_less_nextn_fuses_without_resizing_the_private_pool(self): + """Every `num_nextn_predict_layers = 1` assignment in model_config.py + is guarded by `if is_draft_model`, so a path-less MTP target answers + None for it. The fused placement must read the draft-mode config, or a + path-less NEXTN head never fuses; the private-pool budget must keep + reading the target, or every path-less MTP deployment's token budget + changes.""" + target = _config(num_nextn_predict_layers=None) + draft = _config(num_nextn_predict_layers=1) + config = self._resolve_with(draft_path=None, target=target, draft=draft) + self.assertIs(config.draft_model_config, draft) + self.assertEqual(config.draft_kv_num_layers, 1) + self.assertIsNone(config.eagle_draft_num_layers) + + def test_draft_path_sizes_both_from_the_draft_checkpoint(self): + target = _config(num_nextn_predict_layers=None) + draft = _config(num_nextn_predict_layers=None, num_hidden_layers=2) + config = self._resolve_with(draft_path="/draft", target=target, draft=draft) + self.assertIs(config.draft_model_config, draft) + self.assertEqual(config.draft_kv_num_layers, 2) + self.assertEqual(config.eagle_draft_num_layers, 2) + + def test_the_draft_config_is_recorded_whole(self): + """The aux config carries the draft `ModelConfig`, never a pre-divided + geometry: a per-GPU count recorded here would be computed without the + TP group (every unit fixture runs at tp=1, where that is invisible).""" + fields = _mod.SpecAuxHiddenStateConfig.__struct_fields__ + self.assertIn("draft_model_config", fields) + self.assertNotIn("eagle_draft_total_kv_heads", fields) + self.assertNotIn("eagle_draft_kv_head_num", fields) + + +class TestResolverIsInertOffTheTargetEagleArm(unittest.TestCase): + """BUG REGRESSION. The draft-config block is guarded by "EAGLE-family + target"; a rebase left the hybrid-SWA read outside that guard, so every + draft worker and every spec-off runner raised `UnboundLocalError` at + boot, and the EAGLE3 arm's try/except masked the same read into a silent + `None`. Off the target EAGLE arm the resolver must return untouched.""" + + def _resolve(self, *, spec_algorithm, is_draft_worker): + config = _mod.SpecAuxHiddenStateConfig() + _mod._resolve_eagle_aux_hidden_state( + config=config, + server_args=None, + model_config=None, + spec_algorithm=spec_algorithm, + is_draft_worker=is_draft_worker, + ) + return config + + def test_draft_workers_and_spec_off_runners_resolve_nothing(self): + untouched = _mod.SpecAuxHiddenStateConfig() + for spec_algorithm, is_draft_worker in ( + (_spec_algorithm(eagle=True), True), + (_spec_algorithm(eagle=True, eagle3=True), True), + (_spec_algorithm(eagle=False), False), + ): + config = self._resolve( + spec_algorithm=spec_algorithm, is_draft_worker=is_draft_worker + ) + self.assertEqual(config, untouched) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/spec/test_eagle_worker_v2_topk1_fastpath.py b/test/registered/unit/spec/test_eagle_worker_v2_topk1_fastpath.py index 72f02dfc08..ba35f2612f 100644 --- a/test/registered/unit/spec/test_eagle_worker_v2_topk1_fastpath.py +++ b/test/registered/unit/spec/test_eagle_worker_v2_topk1_fastpath.py @@ -269,6 +269,7 @@ class TestEagleWorkerV2BackendFallback(CustomTestCase): worker.draft_runner = SimpleNamespace( attn_backend=existing_backend, model_config=SimpleNamespace(hf_config=SimpleNamespace()), + kv_index_translator=SimpleNamespace(is_translating=False), ) worker.topk = 1 worker.speculative_num_steps = 2 @@ -294,6 +295,7 @@ class TestEagleWorkerV2BackendFallback(CustomTestCase): worker.draft_runner = SimpleNamespace( attn_backend=existing_backend, model_config=SimpleNamespace(hf_config=SimpleNamespace()), + kv_index_translator=SimpleNamespace(is_translating=False), ) worker.topk = 1 worker.speculative_num_steps = 2