[DeepSeek-V4] Enable non-paged indexer by default for large prefill chunks (#30140)
This commit is contained in:
@@ -842,7 +842,10 @@ class Envs:
|
||||
SGLANG_OPT_FUSE_MHC_POST_PRE = EnvBool(False)
|
||||
SGLANG_OPT_USE_TILELANG_INDEXER = EnvBool(False)
|
||||
SGLANG_OPT_USE_AITER_INDEXER = EnvBool(False)
|
||||
SGLANG_OPT_DSV4_NONPAGED_INDEXER = EnvBool(False)
|
||||
SGLANG_OPT_DSV4_NONPAGED_INDEXER = EnvBool(True)
|
||||
# Per-rank local query rows (after DP-attention sharding when enabled),
|
||||
# not request ISL.
|
||||
SGLANG_OPT_DSV4_NONPAGED_INDEXER_MIN_QUERY_TOKENS = EnvInt(8192)
|
||||
SGLANG_OPT_USE_JIT_INDEXER_METADATA = EnvBool(True)
|
||||
SGLANG_OPT_USE_ONLINE_COMPRESS = EnvBool(False)
|
||||
SGLANG_EXPERIMENTAL_ONLINE_C128_MTP = EnvBool(False)
|
||||
|
||||
@@ -482,6 +482,8 @@ class C4IndexerBackendMixin:
|
||||
c4_seq_lens: torch.Tensor,
|
||||
query_rows: int,
|
||||
) -> Optional[NonPagedIndexerPlan]:
|
||||
if query_rows < envs.SGLANG_OPT_DSV4_NONPAGED_INDEXER_MIN_QUERY_TOKENS.get():
|
||||
return None
|
||||
if not self._can_use_nonpaged_indexer(
|
||||
c4_indexer=c4_indexer,
|
||||
forward_batch=forward_batch,
|
||||
|
||||
Reference in New Issue
Block a user