diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index cdeff9b23..150687781 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -246,16 +246,39 @@ class DsparkFoldedSampling(IntEnum): class Envs: + # Organization principles for this registry: + # - Put every field in exactly one topical section. Prefer an existing + # section; add a new one only when no current section is a clear fit. + # - Group by the behavior and owning call sites, not by name similarity + # alone. Keep closely related lifecycle or feature knobs adjacent. + # - Keep each section focused and below 30 fields. Split growing sections + # by subsystem or lifecycle instead of creating catch-all groups. + # - Order broad runtime subsystems before shared storage and backends; keep + # platform- and model-specific integrations in dedicated later sections. + # - Use the same three-line section header everywhere; do not add ad hoc + # one-line headings or append unrelated fields at the end of a section. + # - Keep vendor-specific aliases with their owning integration, and keep + # test/debug knobs with the feature or test workflow they exercise. + # - Keep explanatory comments attached to their field when moving it. + # - For organization-only changes, AST-check that field names, descriptor + # types, and defaults are unchanged and that only field order moved. + # =================================================================== + # Runtime configuration and process identity + # =================================================================== # Per-role config-namespace bookkeeping: off / record / enforce (value is # validated fail-loud in runtime_context, which resolves it once at import # so the read stays dynamo-prunable). SGLANG_ROLE_NAMESPACES = EnvStr("off") - # record mode: append each newly observed (role, namespace) pair to this + # Record mode: append each newly observed (role, namespace) pair to this # file so the audit survives signal-killed workers. SGLANG_ROLE_NAMESPACES_OUT = EnvStr(None) + IS_H200 = EnvBool(False) + SGLANG_ENABLE_TORCH_INFERENCE_MODE = EnvBool(False) - # Model & File Download + # =================================================================== + # Model configuration, discovery, and weight loading + # =================================================================== SGLANG_USE_MODELSCOPE = EnvBool(False) # Controls weight-file ordering for load-time I/O optimization. # -1 : no sorting, no staggering; preserves original file order. @@ -269,14 +292,36 @@ class Envs: SGLANG_PREFETCH_BLOCK_SIZE_MB = EnvInt(16) SGLANG_GEMMA_OUT_OF_PLACE_POSITION_MUTATION = EnvBool(False) SGLANG_ENABLE_WEIGHT_LOADER_V2 = EnvBool(False) + # Copy rank-local MoE slices into independent CPU storage before H2D when + # they reference a larger mmap-backed checkpoint storage. + SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D = EnvBool(False) + SGLANG_LOAD_SNAPSHOT_USE_ZMQ = EnvBool(False) + SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN = EnvBool(False) + HF_HUB_DISABLE_XET = EnvBool(False) + # In seconds. If a warmup forward batch takes longer than this, the server will crash to prevent hanging. + # Recommend to increase warmup timeout to 1800 to accommodate some kernel JIT precache e.g. deep gemm + SGLANG_WARMUP_TIMEOUT = EnvFloat(-1) + SGLANG_EXTERNAL_MODEL_PACKAGE = EnvStr("") + SGLANG_EXTERNAL_MM_MODEL_ARCH = EnvStr("") + SGLANG_EXTERNAL_MM_PROCESSOR_PACKAGE = EnvStr("") - # HTTP server + # =================================================================== + # HTTP server and health + # =================================================================== # Decompress request bodies tagged with `x-body-compressed`. SGLANG_ENABLE_REQUEST_DECOMPRESSION = EnvBool(False) # Override parsed request fields from headers. SGLANG_ENABLE_REQUEST_HEADER_OVERRIDES = EnvBool(False) + DISABLE_OPENAPI_DOC = EnvBool(False) + SGLANG_TIMEOUT_KEEP_ALIVE = EnvInt(5) + # Uvicorn multiprocess supervisor pings each worker on this interval; default 5s is + # too short when many workers cold-start and load tokenizers in parallel. + SGLANG_UVICORN_WORKER_HEALTHCHECK_TIMEOUT = EnvInt(10) + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION = EnvBool(True) - # Logging Options + # =================================================================== + # Logging + # =================================================================== SGLANG_LOG_GC = EnvBool(False) SGLANG_LOG_FORWARD_ITERS = EnvBool(False) SGLANG_LOG_DECODE_GRAPH_KEY = EnvBool(False) @@ -286,37 +331,82 @@ class Envs: SGLANG_LOG_SCHEDULER_STATUS_TARGET = EnvStr("") SGLANG_LOG_SCHEDULER_STATUS_INTERVAL = EnvFloat(60.0) - # IPC + # =================================================================== + # IPC, broadcasters, and ports + # =================================================================== SGLANG_USE_PICKLE_IPC = EnvBool(True) # Log top-level PickleWrapper frames unwrapped on msgpack IPC decode. SGLANG_LOG_PICKLE_IPC_OBJECTS = EnvBool(False) + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER = EnvBool(True) + SGLANG_TCP_STORE_PORT = EnvInt(29600) + # Base port hint for ephemeral sockets (ZMQ, SHM broadcaster, etc.). + # When set, get_open_port() and shm_broadcast search upwards from this + # value instead of asking the OS for a random port. Useful to keep all + # SGLang ports in a predictable range behind a firewall. + SGLANG_PORT = EnvInt(None) + SGLANG_BACKUP_PORT_BASE = EnvInt(10000) - # SGLang CI + # =================================================================== + # CI and test execution + # =================================================================== SGLANG_IS_IN_CI = EnvBool(False) SGLANG_IS_IN_CI_AMD = EnvBool(False) + SGLANG_TEST_MAX_RETRY = EnvInt(None) + # Expand jit_kernel test grids to their full parameter ranges (nightly). + SGLANG_JIT_KERNEL_RUN_FULL_TESTS = EnvBool(False) + SGLANG_SKIP_SGL_KERNEL_VERSION_CHECK = EnvBool(False) + + # =================================================================== + # Crash diagnostics and shutdown + # =================================================================== SGLANG_CUDA_COREDUMP = EnvBool(False) # None = unset, letting get_dump_dir() resolve the base (RUNNER_TEMP in CI, # else /tmp); see debug_utils/cuda_coredump.py. SGLANG_CUDA_COREDUMP_DIR = EnvStr(None) - SGLANG_TEST_MAX_RETRY = EnvInt(None) - # Expand jit_kernel test grids to their full parameter ranges (nightly). - SGLANG_JIT_KERNEL_RUN_FULL_TESTS = EnvBool(False) + SGLANG_FORCE_SHUTDOWN = EnvBool(False) + SGLANG_PYSPY_DUMP_BEFORE_CRASH = EnvBool(True) + SGLANG_CUDA_COREDUMP_BEFORE_CRASH = EnvBool(True) + SGLANG_CUDA_COREDUMP_BEFORE_CRASH_WAIT_SECS = EnvFloat(60.0) - # Constrained Decoding (Grammar) + # =================================================================== + # Constrained decoding and grammar + # =================================================================== SGLANG_GRAMMAR_POLL_INTERVAL = EnvFloat(0.005) SGLANG_GRAMMAR_MAX_POLL_ITERATIONS = EnvInt(10000) SGLANG_DISABLE_OUTLINES_DISK_CACHE = EnvBool(False) - # Test & Debug - SGLANG_DETECT_SLOW_RANK = EnvBool(False) + # =================================================================== + # Fault injection and regression tests + # =================================================================== SGLANG_TEST_STUCK_DETOKENIZER = EnvFloat(0) SGLANG_TEST_STUCK_DP_CONTROLLER = EnvFloat(0) SGLANG_TEST_STUCK_SCHEDULER_INIT = EnvFloat(0) SGLANG_TEST_STUCK_TOKENIZER = EnvFloat(0) SGLANG_TEST_CRASH_AFTER_STREAM_OUTPUTS = EnvInt(0) - IS_H200 = EnvBool(False) - SGLANG_SET_CPU_AFFINITY = EnvBool(False) - SGLANG_ENABLE_CP_V2 = EnvBool(False) + SGLANG_TEST_REQUEST_TIME_STATS = EnvBool(False) + SGLANG_TEST_DISAGG_FAILURE_PROB = EnvFloat(0.0) + SGLANG_TEST_RETRACT = EnvBool(False) + SGLANG_TEST_RETRACT_INTERVAL = EnvInt(3) + SGLANG_TEST_RETRACT_NO_PREFILL_BS = EnvInt(2**31) + # Scheduler: force lazy extra_buffer prealloc to fail at decode boundaries + SGLANG_TEST_MAMBA_LAZY_ALLOC_FAIL = EnvBool(False) + # KL tests: skip the cache-hit count assertion (e.g. when alloc failure reduces hits) + SGLANG_TEST_SKIP_CACHE_HIT_ASSERT = EnvBool(False) + + # =================================================================== + # PD and scripted-runtime tests + # =================================================================== + SGLANG_TEST_PD_DISAGG_BACKEND = EnvStr("mooncake") + SGLANG_TEST_PD_DISAGG_DEVICES = EnvStr(None) + SGLANG_TEST_FORCE_OPTIMISTIC_PREFILL_RETRY_PROB = EnvFloat(0.0) + SGLANG_TEST_SCRIPTED_RUNTIME = EnvBool(False) + SGLANG_TEST_SCRIPTED_RUNTIME_IPC_ADDR = EnvStr(None) + SGLANG_TEST_SCRIPTED_RUNTIME_OUT_OF_BAND_ERROR_PATH = EnvStr(None) + SGLANG_TEST_SCRIPTED_RUNTIME_SYS_PATH_ENTRY = EnvStr(None) + + # =================================================================== + # Profiling, tracing, and metrics + # =================================================================== SGLANG_PROFILE_WITH_STACK = EnvBool(True) SGLANG_PROFILE_RECORD_SHAPES = EnvBool(True) SGLANG_PROFILE_V2 = EnvBool(False) @@ -332,10 +422,59 @@ class Envs: # SGLANG_ENABLE_CUDA_GRAPH_CAPTURE_TRACE (single combined trace) takes # precedence when both are set. SGLANG_GRAPH_BATCH_CAPTURE = EnvBool(False) - SGLANG_FORCE_SHUTDOWN = EnvBool(False) + SGLANG_TORCH_PROFILER_DIR = EnvStr("/tmp") + # Allocator-history buffer for /start_profile activities=["MEM"]; the + # default truncates long windows (each entry is one alloc/free event). + SGLANG_MEM_PROFILE_MAX_ENTRIES = EnvInt(100000) + SGLANG_OTLP_EXPORTER_SCHEDULE_DELAY_MILLIS = EnvInt(500) + SGLANG_OTLP_EXPORTER_MAX_EXPORT_BATCH_SIZE = EnvInt(64) + SGLANG_TRACE_ASYNC = EnvBool(False) + SGLANG_TRACE_ASYNC_FLUSH_THRESHOLD = EnvInt(100) + SGLANG_ENABLE_METRICS_DEVICE_TIMER = EnvBool(False) + SGLANG_ENABLE_METRICS_DP_ATTENTION = EnvBool(False) + + # =================================================================== + # Debugging and invariant checks + # =================================================================== + SGLANG_DETECT_SLOW_RANK = EnvBool(False) SGLANG_DEBUG_MEMORY_POOL = EnvBool(False) # NaN-fill the unified memory pool at boot (debug repro switch). SGLANG_DEBUG_POISON_POOL = EnvBool(False) + SGLANG_DEBUG_REVERT_PR = EnvInt(0) + SGLANG_PHASE_CHECKER_DEBUG = EnvBool(False) + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK = EnvBool(False) + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK = EnvBool(True) + SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY = EnvInt(0) + SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_IDLE = EnvBool(True) + # Physical KV-page checks: committed<=allocated + no page alias. + SGLANG_CHECK_KV_PAGE_INVARIANTS = EnvBool(False) + SGLANG_TBO_DEBUG = EnvBool(False) + # Timing probe: run the swap-in fully but skip the host->device KV bytes, + # measuring the "IO is free" floor. GARBAGE OUTPUT -- benchmarking only. + SGLANG_DEBUG_HISPARSE_SKIP_IO = EnvBool(False) + # Master switch for all async-asserted invariant probes (NaN, Inf, OOB, + # page alignment). Off in prod; tests turn it on to fail-fast on + # numerical / index violations instead of getting silent NaN cascades. + SGLANG_ENABLE_ASYNC_ASSERT = EnvBool(False) + # Signal level for value/index validity checks (nan/inf/oob/...); see + # invariants.py. OFF (prod default) runs only the free data layer, WARN + # adds throttled logging, STRICT (CI default) crashes on violations. + # Supersedes SGLANG_ENABLE_ASYNC_ASSERT, which is bridged as STRICT until + # every callsite migrates. + SGLANG_INVARIANT_CHECK = EnvInt(InvariantCheckLevel.OFF) + + # =================================================================== + # Runtime simulations + # =================================================================== + SGLANG_SIMULATE_ACC_LEN = EnvFloat(-1) + SGLANG_SIMULATE_ACC_METHOD = EnvStr("match-expected") + SGLANG_SIMULATE_ACC_TOKEN_MODE = EnvStr("fixed") + SGLANG_SIMULATE_UNIFORM_EXPERTS = EnvBool(False) + SGLANG_SIMULATE_ROUND_ROBIN_EXPERTS = EnvBool(False) + + # =================================================================== + # DSpark speculative decoding + # =================================================================== SGLANG_DSPARK_DEBUG_CONFIDENCE_PREFIX_SCHEDULER = EnvBool(False) SGLANG_DSPARK_DEBUG_CONFIDENCE_METRICS = EnvBool(False) SGLANG_DSPARK_DEBUG_DUMP = EnvTuple(tuple()) @@ -354,23 +493,11 @@ class Envs: SGLANG_DSPARK_OPT_MARKOV_W2_BF16 = EnvBool(True) SGLANG_DSPARK_OPT_MARKOV_W2_TP_SHARD = EnvBool(True) SGLANG_DSPARK_ENABLE_MULTI_STREAM = EnvBool(True) - SGLANG_DEBUG_REVERT_PR = EnvInt(0) - SGLANG_PHASE_CHECKER_DEBUG = EnvBool(False) - SGLANG_TEST_REQUEST_TIME_STATS = EnvBool(False) - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK = EnvBool(False) - SGLANG_SIMULATE_ACC_LEN = EnvFloat(-1) - SGLANG_SIMULATE_ACC_METHOD = EnvStr("match-expected") - SGLANG_SIMULATE_ACC_TOKEN_MODE = EnvStr("fixed") - SGLANG_SIMULATE_UNIFORM_EXPERTS = EnvBool(False) - SGLANG_SIMULATE_ROUND_ROBIN_EXPERTS = EnvBool(False) - SGLANG_TORCH_PROFILER_DIR = EnvStr("/tmp") - # Allocator-history buffer for /start_profile activities=["MEM"]; the - # default truncates long windows (each entry is one alloc/free event). - SGLANG_MEM_PROFILE_MAX_ENTRIES = EnvInt(100000) - SGLANG_OTLP_EXPORTER_SCHEDULE_DELAY_MILLIS = EnvInt(500) - SGLANG_OTLP_EXPORTER_MAX_EXPORT_BATCH_SIZE = EnvInt(64) - SGLANG_TRACE_ASYNC = EnvBool(False) - SGLANG_TRACE_ASYNC_FLUSH_THRESHOLD = EnvInt(100) + SGLANG_DSPARK_CONFIDENCE_RELAY_LAG_STEPS = EnvInt(2) + + # =================================================================== + # Memory pools and KV-cache sizing + # =================================================================== SGLANG_NATIVE_MOVE_KV_CACHE = EnvBool(False) # Disable lazy compaction in the unified memory pool allocator and # fall back to the per-free eager compaction. Used for production @@ -381,47 +508,97 @@ class Envs: # Periodically log lazy-compaction stats per sub-pool (observability only). SGLANG_LOG_LAZY_COMPACTION_STATS = EnvBool(False) SGLANG_LOG_LAZY_COMPACTION_STATS_INTERVAL_SEC = EnvInt(30) - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK = EnvBool(True) - SGLANG_TEST_DISAGG_FAILURE_PROB = EnvFloat(0.0) - # HND KV layout folds (page, head) into one paged index for per-kv-head sparse # page tables (DP attn); paged backends like trtllm_mha consume it directly. SGLANG_USE_HND_KVCACHE = EnvBool(False) - - # size the KV pool after CUDA-graph capture + # Size the KV pool after CUDA-graph capture. SGLANG_ENABLE_POST_CAPTURE_KV_SIZING = EnvBool(False) - # Scheduler: memory leak test - SGLANG_TEST_RETRACT = EnvBool(False) - SGLANG_TEST_RETRACT_INTERVAL = EnvInt(3) - SGLANG_TEST_RETRACT_NO_PREFILL_BS = EnvInt(2**31) - # Scheduler: force lazy extra_buffer prealloc to fail at decode boundaries - SGLANG_TEST_MAMBA_LAZY_ALLOC_FAIL = EnvBool(False) - # KL tests: skip the cache-hit count assertion (e.g. when alloc failure reduces hits) - SGLANG_TEST_SKIP_CACHE_HIT_ASSERT = EnvBool(False) - SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY = EnvInt(0) - SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_IDLE = EnvBool(True) - # Physical KV-page checks: committed<=allocated + no page alias. - SGLANG_CHECK_KV_PAGE_INVARIANTS = EnvBool(False) - - # Load snapshot backend - SGLANG_LOAD_SNAPSHOT_USE_ZMQ = EnvBool(False) - - # Scheduler: new token ratio hyperparameters + # =================================================================== + # Scheduler token budgeting and admission + # =================================================================== SGLANG_INIT_NEW_TOKEN_RATIO = EnvFloat(0.7) SGLANG_MIN_NEW_TOKEN_RATIO_FACTOR = EnvFloat(0.14) SGLANG_NEW_TOKEN_RATIO_DECAY_STEPS = EnvInt(600) SGLANG_RETRACT_DECODE_STEPS = EnvInt(20) SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION = EnvInt(4096) SGLANG_MAX_NEW_TOKENS_LIMIT = EnvInt(None) + SGLANG_DYNAMIC_CHUNKING_SMOOTH_FACTOR = EnvFloat(0.75) + SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES = EnvInt(None) + SGLANG_PREFILL_DELAYER_TOKEN_USAGE_LOW_WATERMARK = EnvFloat(None) + SGLANG_DATA_PARALLEL_BUDGET_INTERVAL = EnvInt(1) + # Compact extend-attention scheduler tile-budget admission (AMD/HIP-only). + # Budget <= 0 disables; >0 sets the max prefix-extend tiles per batch. + SGLANG_PREFILL_TILE_BUDGET = EnvInt(0) + # Tile-budget mode: "compact" (default, counts actual per-request tiles) or + # "legacy" (rectangular grid, max_extend_len-shaped). + # Internal/testing only - users should not need to change this. + SGLANG_PREFILL_TILE_BUDGET_MODE = EnvStr("compact") - # Scheduler: recv interval + # =================================================================== + # Scheduler polling, timeouts, and output + # =================================================================== SGLANG_SCHEDULER_RECV_SKIPPER_WEIGHT_DEFAULT = EnvInt(1000) SGLANG_SCHEDULER_RECV_SKIPPER_WEIGHT_DECODE = EnvInt(1) SGLANG_SCHEDULER_RECV_SKIPPER_WEIGHT_TARGET_VERIFY = EnvInt(1) SGLANG_SCHEDULER_RECV_SKIPPER_WEIGHT_NONE = EnvInt(1) + # in seconds. Set if you observe high memory accumulation over a long serving period. + SGLANG_EMPTY_CACHE_INTERVAL = EnvFloat(-1) + SGLANG_SCHEDULER_MAX_RECV_PER_POLL = EnvInt(-1) + SGLANG_SCHEDULER_SKIP_ALL_GATHER = EnvBool(False) + SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE = EnvBool(False) + SGLANG_KILLPG_ON_SCHEDULER_EXCEPTION = EnvBool(False) + SGLANG_REQ_WAITING_TIMEOUT = EnvFloat(-1) # in seconds + SGLANG_REQ_RUNNING_TIMEOUT = EnvFloat(-1) # in seconds + # For non-streaming requests, the scheduler still flushes intermediate + # output batches to the tokenizer manager every N decoded tokens so that + # `first_token_time`/TTFT can be recorded. Lower this (e.g. to 1) to get + # an accurate TTFT for benchmarking; the upstream default of 50 trades + # off some TTFT-metric accuracy for less IPC overhead. + SGLANG_FORCE_STREAM_INTERVAL = EnvInt(50) - # PD Disaggregation (runtime) + # =================================================================== + # Overlap scheduler and pipeline parallelism + # =================================================================== + SGLANG_DISABLE_CONSECUTIVE_PREFILL_OVERLAP = EnvBool(False) + # Force delay_sample_func for all overlap decode (not just grammar mode), + # allowing CPU result processing to overlap with subsequent forward computation + # and reducing the impact of sampling overhead on the critical path. + SGLANG_ENABLE_DELAY_SAMPLE = EnvBool(False) + # Force-enable the WAR (write-after-read) barrier for the overlap scheduler + # even when is_cuda() is False (e.g. AMD/ROCm). On CUDA the barrier is + # already enabled regardless of this flag (see start_event_loop). + SGLANG_ENABLE_WAR_BARRIER = EnvBool(False) + # Force the WAR barrier to wait for the whole forward instead of the + # read-done fastpath event. + SGLANG_FORCE_COARSE_WAR_BARRIER = EnvBool(False) + # Enable prefill read-done publication after compliant metadata initialization. + SGLANG_ENABLE_PREFILL_WAR_READ_DONE = EnvBool(False) + # PP: skip output send/recv when the entire batch consists of non-final chunked prefill requests, + # since process_batch_result_prefill discards next_token_ids for those anyway. + SGLANG_PP_SKIP_PURE_CHUNKED_OUTPUT_COMM = EnvBool(False) + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH = EnvBool(False) + + # =================================================================== + # Radix and sparse KV caches + # =================================================================== + SGLANG_EXPERIMENTAL_CPP_RADIX_TREE = EnvBool(False) + SGLANG_RADIX_FORCE_MISS = EnvBool(False) + SGLANG_CHUNKED_PREFIX_CACHE_THRESHOLD = EnvInt(8192) + SGLANG_MAX_KV_CHUNK_CAPACITY = EnvInt(128 * 1024) + # Kill-switch for the shared-index (IndexShare) swap-in prefetch + # (auto-enabled for GLM-5.2-style DSA); set True to A/B synchronous swap-in. + SGLANG_DISABLE_HISPARSE_PREFETCH = EnvBool(False) + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS = EnvBool(True) + # Decode batches between SWA out-of-window evictions. + SGLANG_SWA_EVICTION_INTERVAL = EnvInt(128) + SGLANG_ENABLE_UNIFIED_RADIX_TREE = EnvBool(False) + # Registered TreeCore backend serving the unified radix cache. + SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND = EnvStr("python") + + # =================================================================== + # PD disaggregation runtime + # =================================================================== # NOTE: For SGLANG_DISAGGREGATION_THREAD_POOL_SIZE, the effective default is # computed dynamically at runtime based on cpu_count; see disaggregation backends. SGLANG_DISAGGREGATION_THREAD_POOL_SIZE = EnvInt(None) @@ -438,82 +615,34 @@ class Envs: SGLANG_DISAGGREGATION_ALL_CP_RANKS_TRANSFER = EnvBool(False) SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK = EnvBool(False) SGLANG_DISAGGREGATION_SAMPLING_MASK_MAX_TOKENS = EnvInt(0) - - # Scheduler: others: - # in seconds. Set if you observe high memory accumulation over a long serving period. - SGLANG_EMPTY_CACHE_INTERVAL = EnvFloat(-1) - SGLANG_DISABLE_CONSECUTIVE_PREFILL_OVERLAP = EnvBool(False) - # Force-enable the WAR (write-after-read) barrier for the overlap scheduler - # even when is_cuda() is False (e.g. AMD/ROCm). On CUDA the barrier is - # already enabled regardless of this flag (see start_event_loop). - SGLANG_ENABLE_WAR_BARRIER = EnvBool(False) - # Force the WAR barrier to wait for the whole forward instead of the - # read-done fastpath event. - SGLANG_FORCE_COARSE_WAR_BARRIER = EnvBool(False) - # Enable prefill read-done publication after compliant metadata initialization. - SGLANG_ENABLE_PREFILL_WAR_READ_DONE = EnvBool(False) - # PP: skip output send/recv when the entire batch consists of non-final chunked prefill requests, - # since process_batch_result_prefill discards next_token_ids for those anyway. - SGLANG_PP_SKIP_PURE_CHUNKED_OUTPUT_COMM = EnvBool(False) - SGLANG_SCHEDULER_MAX_RECV_PER_POLL = EnvInt(-1) - SGLANG_EXPERIMENTAL_CPP_RADIX_TREE = EnvBool(False) - SGLANG_RADIX_FORCE_MISS = EnvBool(False) - SGLANG_DYNAMIC_CHUNKING_SMOOTH_FACTOR = EnvFloat(0.75) - SGLANG_SCHEDULER_SKIP_ALL_GATHER = EnvBool(False) - SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE = EnvBool(False) - SGLANG_KILLPG_ON_SCHEDULER_EXCEPTION = EnvBool(False) - SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES = EnvInt(None) - SGLANG_PREFILL_DELAYER_TOKEN_USAGE_LOW_WATERMARK = EnvFloat(None) - SGLANG_DATA_PARALLEL_BUDGET_INTERVAL = EnvInt(1) - SGLANG_REQ_WAITING_TIMEOUT = EnvFloat(-1) # in seconds - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH = EnvBool(False) - SGLANG_REQ_RUNNING_TIMEOUT = EnvFloat(-1) # in seconds SGLANG_DISAGGREGATION_BOOTSTRAP_ENTRY_CLEANUP_INTERVAL = EnvInt(120) - # Decode batches between SWA out-of-window evictions. - SGLANG_SWA_EVICTION_INTERVAL = EnvInt(128) - # For non-streaming requests, the scheduler still flushes intermediate - # output batches to the tokenizer manager every N decoded tokens so that - # `first_token_time`/TTFT can be recorded. Lower this (e.g. to 1) to get - # an accurate TTFT for benchmarking; the upstream default of 50 trades - # off some TTFT-metric accuracy for less IPC overhead. - SGLANG_FORCE_STREAM_INTERVAL = EnvInt(50) - # Compact extend-attention scheduler tile-budget admission (AMD/HIP-only). - # Budget <= 0 disables; >0 sets the max prefix-extend tiles per batch. - SGLANG_PREFILL_TILE_BUDGET = EnvInt(0) - # Tile-budget mode: "compact" (default, counts actual per-request tiles) or - # "legacy" (rectangular grid, max_extend_len-shaped). - # Internal/testing only - users should not need to change this. - SGLANG_PREFILL_TILE_BUDGET_MODE = EnvStr("compact") - # Test: pd-disaggregation - SGLANG_TEST_PD_DISAGG_BACKEND = EnvStr("mooncake") - SGLANG_TEST_PD_DISAGG_DEVICES = EnvStr(None) - SGLANG_TEST_FORCE_OPTIMISTIC_PREFILL_RETRY_PROB = EnvFloat(0.0) - - SGLANG_TEST_SCRIPTED_RUNTIME = EnvBool(False) - SGLANG_TEST_SCRIPTED_RUNTIME_IPC_ADDR = EnvStr(None) - SGLANG_TEST_SCRIPTED_RUNTIME_OUT_OF_BAND_ERROR_PATH = EnvStr(None) - SGLANG_TEST_SCRIPTED_RUNTIME_SYS_PATH_ENTRY = EnvStr(None) - - # Model Parallel - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER = EnvBool(True) + # =================================================================== + # Distributed and model-parallel runtime + # =================================================================== + SGLANG_ENABLE_CP_V2 = EnvBool(False) SGLANG_ONE_VISIBLE_DEVICE_PER_PROCESS = EnvBool(False) # Comma-separated bundle indices for Ray Custom PG mode (e.g., "0,1,2,7"). SGLANG_RAY_BUNDLE_INDICES = EnvStr("") # Override the distributed init method used by torch.distributed.init_process_group. # Set to "env://" to use an externally-created TCPStore via MASTER_ADDR/MASTER_PORT. SGLANG_DISTRIBUTED_INIT_METHOD_OVERRIDE = EnvStr(None) - SGLANG_TCP_STORE_PORT = EnvInt(29600) + SGLANG_IS_FIRST_RANK_ON_NODE = EnvBool(True) + SGLANG_SYNC_TOKEN_IDS_ACROSS_TP = EnvBool(False) + SGLANG_ENABLE_COLOCATED_BATCH_GEN = EnvBool(False) + SGLANG_SHARED_EXPERT_TP1 = EnvBool(False) + # Replicate the input embedding across TP ranks instead of sharding it + # along the vocab dimension (saves an all-reduce/all-gather in the embed + # lookup at the cost of replicated embedding weights). Drives both the + # target and every draft that shares its embedding (see + # get_embedding_tp_kwargs); they must stay in lock-step. Currently only + # applies to the Deepseek-V2 family (Deepseek V3.1, Kimi K2.5) + drafts. + SGLANG_ENABLE_EMBED_REPLICATION = EnvBool(False) - # Base port hint for ephemeral sockets (ZMQ, SHM broadcaster, etc.). - # When set, get_open_port() and shm_broadcast search upwards from this - # value instead of asking the OS for a random port. Useful to keep all - # SGLang ports in a predictable range behind a firewall. - SGLANG_PORT = EnvInt(None) - - # Tool Calling + # =================================================================== + # Tool calling and native web search + # =================================================================== SGLANG_FORWARD_UNKNOWN_TOOLS = EnvBool(False) - # Native web search (Exa). EXA_API_KEY is the vendor BYOK credential # (kept as-is, not renamed to SGLANG_*); the SGLANG_EXA_* knobs tune the # request defaults for the built-in GPT-OSS web_search tool. @@ -521,8 +650,11 @@ class Envs: SGLANG_EXA_NUM_RESULTS = EnvInt(10) SGLANG_EXA_SEARCH_TYPE = EnvStr("auto") SGLANG_EXA_INCLUDE_HIGHLIGHTS = EnvBool(True) + SGLANG_TOOL_STRICT_LEVEL = EnvInt(ToolStrictLevel.OFF) - # Hi-Cache + # =================================================================== + # HiCache storage backends and mmap allocation + # =================================================================== SGLANG_HICACHE_HF3FS_CONFIG_PATH = EnvStr(None) SGLANG_HICACHE_DECODE_OFFLOAD_STRIDE = EnvInt(None) SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR = EnvStr(None) @@ -540,13 +672,16 @@ class Envs: # "use_direct_io": false key in --hicache-storage-backend-extra-config. SGLANG_HICACHE_NIXL_USE_DIRECT_IO = EnvBool(True) SGLANG_HUGEPAGE_SIZE = EnvStr("") + + # =================================================================== + # KV-transfer staging and Mooncake transport + # =================================================================== # Staging buffer for heterogeneous TP KV transfer SGLANG_DISAGG_STAGING_BUFFER = EnvBool(False) SGLANG_DISAGG_STAGING_POOL_SIZE_MB = EnvInt(4096) # TODO(yangminl): remove SGLANG_STAGING_USE_TORCH and the torch fallback in # staging_buffer.py once Triton kernels are fully validated in production. SGLANG_STAGING_USE_TORCH = EnvBool(False) - # Mooncake KV Transfer SGLANG_MOONCAKE_CUSTOM_MEM_POOL = EnvStr(None) ENABLE_ASCEND_TRANSFER_WITH_MOONCAKE = EnvBool(False) ASCEND_NPU_PHY_ID = EnvInt(-1) @@ -554,7 +689,9 @@ class Envs: SGLANG_ENABLE_FAILED_SESSION_PROBE = EnvBool(False) SGLANG_FAILED_SESSION_PROBE_INTERVAL_S = EnvFloat(30.0) - # Mooncake Store + # =================================================================== + # Mooncake store + # =================================================================== SGLANG_HICACHE_MOONCAKE_CONFIG_PATH = EnvStr(None) SGLANG_HICACHE_MOONCAKE_REUSE_TE = EnvBool(True) MOONCAKE_MASTER = EnvStr(None) @@ -571,7 +708,9 @@ class Envs: MOONCAKE_OFFLOAD_FILE_STORAGE_PATH = EnvStr(None) MOONCAKE_TENANT_ID = EnvStr("default") - # MoRI KV Transfer + # =================================================================== + # MoRI transport and expert dispatch + # =================================================================== # Send CPU-resident AUX data via RDMA instead of ZMQ TCP (default: TCP). SGLANG_MORI_SEND_AUX_RDMA = EnvBool(False) # Number of RDMA Queue Pairs (QPs) used per transfer operation. Higher @@ -596,8 +735,11 @@ class Envs: # Per-transfer SLA (ms) before a KV transfer is failed; 0 disables the SLA # and relies on the RDMA retry-exceeded timeout only. SGLANG_MORI_TRANSFER_TIMEOUT_MS = EnvInt(0) + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK = EnvInt(4096) - # AMD & ROCm + # =================================================================== + # AMD, ROCm, and AITER + # =================================================================== SGLANG_USE_AITER = EnvBool(False) SGLANG_USE_AITER_AG = EnvBool(True) # Use reduce_scatter (instead of all_reduce + dp_scatter) for the equal-chunk @@ -642,13 +784,15 @@ class Envs: SGLANG_ROCM_FUSED_DECODE_MLA = EnvBool(False) SGLANG_ROCM_DISABLE_LINEARQUANT = EnvBool(False) USE_ROCM_AITER_ROPE_BACKEND = EnvStr("0") - SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK = EnvInt(4096) # Enable dual-stream MoE (shared experts vs routed experts) on the # ROCm/AITER path. Requires GPU_MAX_HW_QUEUES>=5 to avoid HW-queue serialization. SGLANG_ROCM_USE_MULTI_STREAM = EnvBool(False) SGLANG_HACK_FLASHMLA_BACKEND = EnvStr("tilelang") + SGLANG_USE_AITER_FP8_PER_TOKEN = EnvBool(False) - # MPS (Apple Silicon) + # =================================================================== + # Apple Silicon and MLX + # =================================================================== SGLANG_USE_MLX = EnvBool(False) SGLANG_MLX_USE_CUSTOM_ROPE = EnvBool(False) SGLANG_MLX_FUSE_SWIGLU = EnvBool(False) @@ -658,7 +802,9 @@ class Envs: # MLX buffer-cache cap in GB. SGLANG_MLX_CACHE_LIMIT_GB = EnvFloat(None) - # NPU + # =================================================================== + # Ascend NPU + # =================================================================== SGLANG_NPU_DISABLE_ACL_FORMAT_WEIGHT = EnvBool(False) SGLANG_NPU_USE_MULTI_STREAM = EnvBool(False) SGLANG_NPU_USE_MLAPO = EnvBool(False) @@ -668,9 +814,6 @@ class Envs: SGLANG_NPU_FORWARD_NATIVE_GEMMA_RMS_NORM = EnvBool(False) # Delay all-gather after qlora for better performance for Deepseek v3.2 SGLANG_USE_AG_AFTER_QLORA = EnvBool(False) - # Master switch for the experimental TRT-LLM LoRA fast path; when OFF (default) every - # fine-grained opt switch reads False, keeping non-experimental paths byte-identical. - SGLANG_EXPERIMENTAL_LORA_OPTI = EnvBool(False) # Enable int4x2 weights loading SGLANG_NPU_W4A4_NEW_PACKING = EnvBool(False) # Keep K3 shared experts and dense MLPs sharded over attention TP. @@ -683,11 +826,17 @@ class Envs: # Quantize x to int8 in the dispatch operator DEEP_NORMAL_MODE_USE_INT8_QUANT = EnvBool(False) # This argument is deprecated SGLANG_NPU_FUSED_MOE_MODE = EnvInt(1) + SGLANG_ZBAL_LOCAL_MEM_SIZE = EnvInt(0) + SGLANG_ZBAL_BOOTSTRAP_URL = EnvStr("") - # MTHREADS & MUSA + # =================================================================== + # MUSA + # =================================================================== SGLANG_MUSA_FA3_FORCE_UPDATE_METADATA = EnvBool(False) + # =================================================================== # Quantization + # =================================================================== SGLANG_INT4_WEIGHT = EnvBool(False) SGLANG_CPU_QUANTIZATION = EnvBool(False) SGLANG_USE_DYNAMIC_MXFP4_LINEAR = EnvBool(False) @@ -699,13 +848,17 @@ class Envs: SGLANG_FP8_IGNORED_LAYERS = EnvStr("") SGLANG_FP4_IGNORED_LAYERS = EnvStr("") - # Quantization (Humming) + # =================================================================== + # Humming quantization + # =================================================================== SGLANG_HUMMING_ONLINE_QUANT_CONFIG = EnvJSON(None) SGLANG_HUMMING_INPUT_QUANT_CONFIG = EnvJSON(None) SGLANG_HUMMING_USE_F16_ACCUM = EnvBool(False) SGLANG_HUMMING_MOE_GEMM_TYPE = EnvStr("") - # Flashinfer + # =================================================================== + # FlashInfer, FlashMLA, and TRT-LLM + # =================================================================== SGLANG_IS_FLASHINFER_AVAILABLE = EnvBool(True) SGLANG_FLASHINFER_USE_PAGED = EnvBool(False) # Default to the pick from flashinfer @@ -718,6 +871,9 @@ class Envs: # Launch the TRT-LLM MoE grouped GEMMs with PDL only at or below this # token count. SGLANG_TRTLLM_MOE_PDL_MAX_TOKENS = EnvInt(8192) + # Master switch for the experimental TRT-LLM LoRA fast path; when OFF (default) every + # fine-grained opt switch reads False, keeping non-experimental paths byte-identical. + SGLANG_EXPERIMENTAL_LORA_OPTI = EnvBool(False) # SGLang needs to know FlashInfer NVFP4 4over6 config to compute the global scale factor. FLASHINFER_NVFP4_4OVER6 = EnvBool(False) FLASHINFER_NVFP4_4OVER6_E4M3_USE_256 = EnvBool(False) @@ -727,8 +883,17 @@ class Envs: SGLANG_SKIP_SOFTMAX_DECODE_THRESHOLD_SCALE_FACTOR = EnvFloat(None) # SM120 FlashMLA decode backend: "flashinfer" (default), "triton", or "torch". SGLANG_SM120_FLASHMLA_BACKEND = EnvStr("flashinfer") + SGLANG_FLASHINFER_PREFILL_SPLIT_TILE_SIZE = EnvInt(4096) + SGLANG_FLASHINFER_DECODE_SPLIT_TILE_SIZE = EnvInt(2048) + SGLANG_FLASHINFER_AUTOTUNE_CACHE = EnvBool(True) + # Also autotune one EXTEND-shaped dummy at max_prefill_tokens during + # warmup. Opt-in: the extra forward needs transient activation headroom + # that small-VRAM or tightly-packed configs may not have. + SGLANG_FLASHINFER_AUTOTUNE_EXTEND = EnvBool(False) - # Triton + # =================================================================== + # Triton and Torch compilation + # =================================================================== SGLANG_TRITON_DECODE_ATTN_STATIC_KV_SPLITS = EnvBool(False) SGLANG_USE_CUSTOM_TRITON_KERNEL_CACHE = EnvBool(False) # Compact extend-attention query-tile grid: AMD/HIP-only optimization @@ -741,11 +906,13 @@ class Envs: SGLANG_CRASH_ON_TRITON_LOAD_AFTER_READY = EnvBool(False) SGLANG_TRITON_SLOW_COMPILE_THRESHOLD_SECS = EnvFloat(1.0) SGLANG_TRITON_LOAD_WARNING_THRESHOLD_GB = EnvFloat(1.0) - - # Torch Compile SGLANG_ENABLE_TORCH_COMPILE = EnvBool(False) + SGLANG_TRITON_PREFILL_TRUNCATION_ALIGN_SIZE = EnvInt(4096) + SGLANG_TRITON_DECODE_SPLIT_TILE_SIZE = EnvInt(256) - # EPLB + # =================================================================== + # Expert parallel load balancing + # =================================================================== SGLANG_EXPERT_LOCATION_UPDATER_LOG_INPUT = EnvBool(False) SGLANG_EXPERT_LOCATION_UPDATER_CANARY = EnvBool(False) SGLANG_EXPERT_LOCATION_UPDATER_LOG_METRICS = EnvBool(False) @@ -759,10 +926,9 @@ class Envs: 32, deprecated_name="SGLANG_EPLB_ROCM_P2P_BATCH_CHUNK_SIZE" ) - # TBO - SGLANG_TBO_DEBUG = EnvBool(False) - - # DeepGemm + # =================================================================== + # DeepGEMM + # =================================================================== SGLANG_ENABLE_JIT_DEEPGEMM = EnvBool(True) SGLANG_DEEPGEMM_STANDARD_LAYOUT = EnvStr("auto") SGLANG_DEEPGEMM_MASKED_MEMORY_BUDGET_FRACTION = EnvFloat(0.25) @@ -790,11 +956,14 @@ class Envs: SGLANG_DEEPGEMM_PDL = EnvBool(True) SGLANG_PP_PARALLEL_DEEPGEMM_WARMUP = EnvBool(False) - # DeepSeek MHA Optimization - SGLANG_CHUNKED_PREFIX_CACHE_THRESHOLD = EnvInt(8192) - SGLANG_MAX_KV_CHUNK_CAPACITY = EnvInt(128 * 1024) + # =================================================================== + # Cache directories + # =================================================================== + SGLANG_CACHE_DIR = EnvStr(os.path.expanduser("~/.cache/sglang")) - # DeepEP + # =================================================================== + # Expert-parallel dispatch and MoE execution + # =================================================================== SGLANG_DEEPEP_BF16_DISPATCH = EnvBool(False) # This argument is deprecated SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK = EnvInt(128) SGLANG_DEEPEP_LL_COMBINE_SEND_NUM_SMS = EnvInt(32) @@ -802,26 +971,22 @@ class Envs: # Force dynamic Waterfill with runtime EP all-reduce instead of the default # static local-batch path. SGLANG_DISABLE_STATIC_WATERFILL = EnvBool(False) - - # NIXL-EP SGLANG_NIXL_EP_BF16_DISPATCH = EnvBool(False) SGLANG_NIXL_EP_NUM_MAX_DISPATCH_TOKENS_PER_RANK = EnvInt(128) - - # PPLX-EP (Perplexity pplx-kernels NVSHMEM all-to-all) SGLANG_PPLX_NUM_MAX_DISPATCH_TOKENS_PER_RANK = EnvInt(128) + SGLANG_ENABLE_MOE_DEFERRED_FINALIZE = EnvBool(True) + # DeepSeek/GLM MoE (deepseek_v2.py): quantize the (dp-gathered) MoE input + # to per-token-group-128 fp8 ONCE and feed both the fused shared-expert + # GEMM (cutlass w8a8 linear) and the routed experts' triton fused runner, + # instead of quantizing the same [T, hidden] tensor twice with different + # scale layouts. Only engages on CUDA with fp8 block-128 weights, the + # standard dispatcher, and the triton MoE runner; falls back silently + # otherwise. + SGLANG_OPT_MOE_QUANT_ONCE = EnvBool(False) - # HiSparse - # Kill-switch for the shared-index (IndexShare) swap-in prefetch - # (auto-enabled for GLM-5.2-style DSA); set True to A/B synchronous swap-in. - SGLANG_DISABLE_HISPARSE_PREFETCH = EnvBool(False) - # Timing probe: run the swap-in fully but skip the host->device KV bytes, - # measuring the "IO is free" floor. GARBAGE OUTPUT -- benchmarking only. - SGLANG_DEBUG_HISPARSE_SKIP_IO = EnvBool(False) - - # Unified radix cache - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS = EnvBool(True) - - # DeepGemm Mega MoE + # =================================================================== + # DeepGEMM Mega MoE + # =================================================================== SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE = EnvBool(False) SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK = EnvInt(8192) # When set, the mega-MoE x slot is packed E2M1 (FP4) instead of FP8 E4M3. @@ -836,7 +1001,9 @@ class Envs: SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND = EnvBool(False) SGLANG_OPT_FIX_MEGA_MOE_MEMORY = EnvBool(False) - # TopK + # =================================================================== + # Top-k kernels + # =================================================================== SGLANG_OPT_USE_FUSED_HASH_TOPK = EnvBool(True) SGLANG_OPT_USE_JIT_KERNEL_FUSED_TOPK = EnvBool(True) # Opt-in: route DeepSeek-V3 grouped topk through the unified Triton router @@ -846,31 +1013,40 @@ class Envs: SGLANG_OPT_USE_JIT_KERNEL_GROUPED_TOPK = EnvBool(False) SGLANG_OPT_USE_TOPK_V2 = EnvBool(True) - # sgl-kernel - SGLANG_SKIP_SGL_KERNEL_VERSION_CHECK = EnvBool(False) - - # Flash Attention + # =================================================================== + # Kernel selection and fused backends + # =================================================================== SGLANG_USE_SGL_FA3_KERNEL = EnvBool(True) - - # Kernels # Force every sglang.kernels BaseFusedOp onto one backend (a KernelBackend # value, e.g. "torch" / "torch_compile" / "triton" / "aot"); unset = # auto-select by priority. "torch" flips all fused ops to their pure-torch # reference implementations for numerical-bug bisection. SGLANG_FORCE_FUSED_OP_BACKEND = EnvStr(None) USE_TRITON_W8A8_FP8_KERNEL = EnvBool(False) - SGLANG_RETURN_ORIGINAL_LOGPROB = EnvBool(False) - SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN = EnvBool(False) SGLANG_MOE_PADDING = EnvBool(False) SGLANG_CUTLASS_MOE = EnvBool(False) - HF_HUB_DISABLE_XET = EnvBool(False) - DISABLE_OPENAPI_DOC = EnvBool(False) - SGLANG_ENABLE_TORCH_INFERENCE_MODE = EnvBool(False) - SGLANG_IS_FIRST_RANK_ON_NODE = EnvBool(True) - SGLANG_SYNC_TOKEN_IDS_ACROSS_TP = EnvBool(False) - SGLANG_ENABLE_COLOCATED_BATCH_GEN = EnvBool(False) - # Deterministic inference + # =================================================================== + # Logits and log-probability processing + # =================================================================== + SGLANG_RETURN_ORIGINAL_LOGPROB = EnvBool(False) + # Sanitize NaN logits before sampling kernels and log a throttled warning + # (see sanitize_nan_logits). + SGLANG_SANITIZE_NAN_LOGITS = EnvBool(False) + SGLANG_ENABLE_LOGPROB_CHUNK = EnvBoolWithAlias( + True, deprecated_name="SGLANG_ENABLE_LOGITS_PROCESSER_CHUNK" + ) + SGLANG_LOGPROB_CHUNK_SIZE = EnvIntWithAlias( + 2048, deprecated_name="SGLANG_LOGITS_PROCESSER_CHUNK_SIZE" + ) + # Compute input logprobs from logits via per-row logsumexp instead of + # materializing the full-vocab log-softmax. Escape hatch only; the two + # paths are mathematically identical. + SGLANG_ENABLE_FAST_INPUT_LOGPROBS = EnvBool(True) + + # =================================================================== + # Deterministic inference and all-reduce + # =================================================================== SGLANG_ENABLE_DETERMINISTIC_INFERENCE = EnvBool(False) # Use 1-stage all-reduce kernel on AMD (deterministic, fixed accumulation order) # If not set: auto (enabled when --enable-deterministic-inference is on) @@ -895,25 +1071,22 @@ class Envs: # MNNVL-fabric devices (GB200/GB300) when nnodes > 1; set 0/1 to # override in either direction. SGLANG_ENABLE_CUSTOM_ALL_REDUCE_V2_MULTINODE = EnvBool(False) - SGLANG_FLASHINFER_PREFILL_SPLIT_TILE_SIZE = EnvInt(4096) - SGLANG_FLASHINFER_DECODE_SPLIT_TILE_SIZE = EnvInt(2048) - SGLANG_TRITON_PREFILL_TRUNCATION_ALIGN_SIZE = EnvInt(4096) - SGLANG_TRITON_DECODE_SPLIT_TILE_SIZE = EnvInt(256) - # RoPE cache configuration + # =================================================================== + # RoPE cache + # =================================================================== SGLANG_SPEC_EXPANSION_SAFETY_FACTOR = EnvInt(2) SGLANG_ROPE_CACHE_FP32 = EnvBool(False) SGLANG_ROPE_CACHE_SAFETY_MARGIN = EnvInt(256) SGLANG_ROPE_CACHE_ALIGN = EnvInt(128) - # Overlap Spec V2 + # =================================================================== + # Speculative decoding + # =================================================================== SGLANG_ENABLE_OVERLAP_PLAN_STREAM = EnvBool(False) - - # Spec Config # A/B: keep the DFLASH draft greedy head eager (not folded in-graph). SGLANG_DFLASH_EAGER_DRAFT_SAMPLER = EnvBool(False) SGLANG_RAGGED_VERIFY_MODE = EnvStr("static") - SGLANG_DSPARK_CONFIDENCE_RELAY_LAG_STEPS = EnvInt(2) SGLANG_TEST_RAGGED_VERIFY_FORCE_UNIFORM_CAPTURE = EnvBool(False) # Skip draft_extend while adaptive spec is at steps=0 (drafting disabled). # Saves the per-step draft forward, but the draft KV goes stale: an upshift @@ -929,21 +1102,11 @@ class Envs: # extend_attention_fwd for unsupported cases or when set false (e.g. for # debugging). Correctness is unaffected; this only changes performance. SGLANG_ENABLE_SPLITKV_VERIFY = EnvBool(True) - # Master switch for all async-asserted invariant probes (NaN, Inf, OOB, - # page alignment). Off in prod; tests turn it on to fail-fast on - # numerical / index violations instead of getting silent NaN cascades. - SGLANG_ENABLE_ASYNC_ASSERT = EnvBool(False) - # Signal level for value/index validity checks (nan/inf/oob/...); see - # invariants.py. OFF (prod default) runs only the free data layer, WARN - # adds throttled logging, STRICT (CI default) crashes on violations. - # Supersedes SGLANG_ENABLE_ASYNC_ASSERT, which is bridged as STRICT until - # every callsite migrates. - SGLANG_INVARIANT_CHECK = EnvInt(InvariantCheckLevel.OFF) - # Sanitize NaN logits before sampling kernels and log a throttled warning - # (see sanitize_nan_logits). - SGLANG_SANITIZE_NAN_LOGITS = EnvBool(False) + SGLANG_NGRAM_FORCE_GREEDY_VERIFY = EnvBool(False) - # VLM + # =================================================================== + # Multimodal processing + # =================================================================== SGLANG_VLM_CACHE_SIZE_MB = EnvInt(100) SGLANG_IMAGE_MAX_PIXELS = EnvInt(16384 * 28 * 28) SGLANG_RESIZE_RESAMPLE = EnvStr("") @@ -959,7 +1122,9 @@ class Envs: # preserve the user's original tokens to avoid retokenization drift. SGLANG_MM_AVOID_RETOKENIZE = EnvBool(True) - # VLM Item CUDA IPC Transport + # =================================================================== + # Multimodal CUDA IPC transport + # =================================================================== SGLANG_USE_CUDA_IPC_TRANSPORT = EnvBool(False) # Reuse the mapping for the already-allocated bounded CUDA IPC pool. This # has no effect unless CUDA IPC feature transport is explicitly selected. @@ -967,7 +1132,9 @@ class Envs: SGLANG_MM_FEATURE_CACHE_MB = EnvInt(1 * 1024) SGLANG_MM_ITEM_MEM_POOL_RECYCLE_INTERVAL_SEC = EnvFloat(0.05) - # Mamba + # =================================================================== + # Mamba state and cache + # =================================================================== SGLANG_MAMBA_CONV_DTYPE = EnvStr("bfloat16") SGLANG_MAMBA_SSM_DTYPE = EnvStr(None) # Kill-switch for the fused per-slot conv clear/copy kernel (MambaPool); @@ -978,67 +1145,55 @@ class Envs: # mamba pool ratio accordingly. Frees one resident slot per running request, # raising max_running_requests. Off = original locking + ratio (escape hatch). SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK = EnvBool(False) - # Unified Radix Tree - SGLANG_ENABLE_UNIFIED_RADIX_TREE = EnvBool(False) - # Registered TreeCore backend serving the unified radix cache. - SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND = EnvStr("python") - # CUDA Graph + # =================================================================== + # CUDA graphs and execution buffers + # =================================================================== SGLANG_USE_BREAKABLE_CUDA_GRAPH = EnvBool(False) # Guards CUDA graph executable dedup via cudaGraphExecUpdate. SGLANG_ENABLE_CUDA_GRAPH_DEDUP = EnvBool(False) - - # Release & Resume Memory SGLANG_MEMORY_SAVER_CUDA_GRAPH = EnvBool(False) + # Eager forward wraps the ForwardBatch's own tensors instead of copying them + # into the CUDA graph buffer registry (no per-iter device-to-device copy). + SGLANG_EAGER_INPUT_NO_COPY = EnvBool(False) - # Sparse Embeddings + # =================================================================== + # Tokenizer, request state, embeddings, and reasoning controls + # =================================================================== SGLANG_EMBEDDINGS_SPARSE_HEAD = EnvStr(None) - - # Logprob processor - SGLANG_ENABLE_LOGPROB_CHUNK = EnvBoolWithAlias( - True, deprecated_name="SGLANG_ENABLE_LOGITS_PROCESSER_CHUNK" - ) - SGLANG_LOGPROB_CHUNK_SIZE = EnvIntWithAlias( - 2048, deprecated_name="SGLANG_LOGITS_PROCESSER_CHUNK_SIZE" - ) - # Compute input logprobs from logits via per-row logsumexp instead of - # materializing the full-vocab log-softmax. Escape hatch only; the two - # paths are mathematically identical. - SGLANG_ENABLE_FAST_INPUT_LOGPROBS = EnvBool(True) - - # Tool-Call behavior - SGLANG_TOOL_STRICT_LEVEL = EnvInt(ToolStrictLevel.OFF) - # Think tokens budget: negative means unlimited, >= 0 caps thinking tokens SGLANG_MAX_THINK_TOKENS = EnvInt(-1) + SGLANG_PATCH_TOKENIZER = EnvBool(True) + SGLANG_REQUEST_STATE_WAIT_TIMEOUT = EnvInt(4) + SGLANG_DEFAULT_THINKING = EnvBool(False) - # Ngram - SGLANG_NGRAM_FORCE_GREEDY_VERIFY = EnvBool(False) - - # Warmup - # in seconds. If a warmup forward batch takes longer than this, the server will crash to prevent hanging. - # Recommend to increase warmup timeout to 1800 to accommodate some kernel JIT precache e.g. deep gemm - SGLANG_WARMUP_TIMEOUT = EnvFloat(-1) - - # HTTP Server - SGLANG_TIMEOUT_KEEP_ALIVE = EnvInt(5) - # Uvicorn multiprocess supervisor pings each worker on this interval; default 5s is - # too short when many workers cold-start and load tokenizers in parallel. - SGLANG_UVICORN_WORKER_HEALTHCHECK_TIMEOUT = EnvInt(10) - - # Health Check - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION = EnvBool(True) - - # Crash diagnostics - SGLANG_PYSPY_DUMP_BEFORE_CRASH = EnvBool(True) - SGLANG_CUDA_COREDUMP_BEFORE_CRASH = EnvBool(True) - SGLANG_CUDA_COREDUMP_BEFORE_CRASH_WAIT_SECS = EnvFloat(60.0) - - # Encoder gRPC + # =================================================================== + # Encoder pipeline and disaggregation + # =================================================================== SGLANG_ENCODER_GRPC_TIMEOUT_SECS = EnvInt(60) # Encoder receiver selection: http|grpc (used by EPD paths). SGLANG_ENCODER_MM_RECEIVER_MODE = EnvStr("http") + SGLANG_ENCODER_RECV_TIMEOUT = EnvFloat(180.0) + SGLANG_ENCODER_SEND_TIMEOUT = EnvFloat(180.0) + SGLANG_ENCODER_HTTP_TIMEOUT = EnvFloat(1800.0) + SGLANG_ENCODER_REQ_TIMEOUT = EnvFloat(180.0) + SGLANG_ENCODER_DISPATCH_MIN_ITEMS = EnvInt(2) + SGLANG_ENCODER_IMAGE_PROCESSOR_USE_GPU = EnvBool(False) + SGLANG_ENCODER_MAX_BATCH_SIZE = EnvInt(8) + SGLANG_ENCODER_PREPROC_WORKERS = EnvInt(8) + # EncoderBootstrapServer health-check tuning. Interval == 0 disables it. + SGLANG_ENCODER_BOOTSTRAP_HEALTH_CHECK_INTERVAL = EnvFloat(10.0) + SGLANG_ENCODER_BOOTSTRAP_HEALTH_CHECK_TIMEOUT = EnvFloat(2.0) + # Seconds before permanently dropping an unhealthy encoder (0 = keep probing). + SGLANG_ENCODER_BOOTSTRAP_EVICTED_TTL = EnvFloat(600.0) + # Persistent receiver-side GPU embedding pool size for mooncake EPD transport. + # 0 disables (per-request register/deregister). 4096 = 4GB default per TP + SGLANG_EMBEDDING_POOL_SIZE_MB = EnvInt(4096) + SGLANG_ENCODER_DP_WORKER_MAX_INFLIGHT = EnvInt(64) + # =================================================================== + # Native gRPC server + # =================================================================== # Native gRPC server. SGLANG_GRPC_PORT is the env fallback for the # --grpc-port CLI flag; setting either enables the native server alongside # HTTP. The worker-threads knob stays env-only (internal tuning, no CLI @@ -1046,34 +1201,17 @@ class Envs: SGLANG_GRPC_PORT = EnvInt(None) SGLANG_GRPC_WORKER_THREADS = EnvInt(4) - # External models - SGLANG_EXTERNAL_MODEL_PACKAGE = EnvStr("") - SGLANG_EXTERNAL_MM_MODEL_ARCH = EnvStr("") - SGLANG_EXTERNAL_MM_PROCESSOR_PACKAGE = EnvStr("") - - # Numa + # =================================================================== + # NUMA and CPU affinity + # =================================================================== + SGLANG_SET_CPU_AFFINITY = EnvBool(False) SGLANG_NUMA_BIND_V2 = EnvBool(True) SGLANG_AUTO_NUMA_BIND = EnvBool(True) SGLANG_CRASH_ON_NUMA_BIND_FAILURE = EnvBool(False) - # Metrics - SGLANG_ENABLE_METRICS_DEVICE_TIMER = EnvBool(False) - SGLANG_ENABLE_METRICS_DP_ATTENTION = EnvBool(False) - - # Tokenizer (Kimi tiktoken: cache all_special_tokens / all_special_ids; the ITL can differ by +10x under high batch size). - SGLANG_PATCH_TOKENIZER = EnvBool(True) - - # TokenizerManager - SGLANG_REQUEST_STATE_WAIT_TIMEOUT = EnvInt(4) - - # ZBAL, zero buffer accelerate library, currently worked only in npu - SGLANG_ZBAL_LOCAL_MEM_SIZE = EnvInt(0) - SGLANG_ZBAL_BOOTSTRAP_URL = EnvStr("") - - SGLANG_DEFAULT_THINKING = EnvBool(False) - - # ==================================================================== - # DeepSeek V4 + # =================================================================== + # DeepSeek V4 - model and quantization + # =================================================================== SGLANG_OPT_DPSK_V4_RADIX = EnvBool(True) SGLANG_OPT_USE_OLD_COMPRESSOR = EnvBool(False) SGLANG_OPT_USE_TRITON_SWA_PREPARE = EnvBool(True) @@ -1086,20 +1224,18 @@ class Envs: SGLANG_OPT_USE_FUSED_CLAMP_ACT_MUL = EnvBool(True) SGLANG_ENABLE_NVFP4_GEMM_SWIGLU_FUSION = EnvBool(True) SGLANG_FIX_MTP_HC_HIDDEN = EnvBool(False) - # Set False when using FP4-to-FP8 converted DeepSeek V4 checkpoint. SGLANG_DSV4_FP4_EXPERTS = EnvBool(True) SGLANG_DSV4_FP4_DEQUANT = EnvBool(False) - # Copy rank-local MoE slices into independent CPU storage before H2D when - # they reference a larger mmap-backed checkpoint storage. - SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D = EnvBool(False) # Flash-0731 also accepts "low"; the active profile is checkpoint-resolved. SGLANG_DSV4_REASONING_EFFORT = EnvStr("") # Quantize the SWA fp8 KV cache from bf16-rounded values (matches # trainer-side QAT and the DSA-CP path) instead of fp32 registers. SGLANG_DSV4_USE_BF16_KV_QUANT_SOURCE = EnvBool(False) - # CUDA kernels + # =================================================================== + # DeepSeek V4 - kernels and indexer + # =================================================================== SGLANG_OPT_DEEPGEMM_HC_PRENORM = EnvBool(True) SGLANG_OPT_USE_TILELANG_MHC_PRE = EnvBool(True) SGLANG_OPT_USE_TILELANG_MHC_POST = EnvBool(True) @@ -1123,49 +1259,27 @@ class Envs: SGLANG_TOPK_TRANSFORM_512_TORCH = EnvBool(False) SGLANG_OPT_FLASHMLA_SPARSE_PREFILL = EnvBool(True) - # SWA radix cache + # =================================================================== + # DeepSeek V4 - cache, GEMM, and distributed + # =================================================================== # TODO(DSV4): @ispobock this has bug on main branch when retract SGLANG_OPT_SWA_RADIX_CACHE_COMPACT = EnvBool(False) SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT = EnvBool(False) SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW = EnvBool(False) - - # GEMM / kernel fusion SGLANG_OPT_FP8_WO_A_GEMM = EnvBool(True) SGLANG_OPT_BF16_FP32_GEMM_ALGO = EnvStr("cublas") SGLANG_OPT_USE_JIT_EP_ACTIVATION = EnvBool(True) SGLANG_OPT_FUSE_WQA_WKV = EnvBool(True) SGLANG_OPT_SWIGLU_CLAMP_FUSION = EnvBool(True) - # DeepSeek/GLM MoE (deepseek_v2.py): quantize the (dp-gathered) MoE input - # to per-token-group-128 fp8 ONCE and feed both the fused shared-expert - # GEMM (cutlass w8a8 linear) and the routed experts' triton fused runner, - # instead of quantizing the same [T, hidden] tensor twice with different - # scale layouts. Only engages on CUDA with fp8 block-128 weights, the - # standard dispatcher, and the triton MoE runner; falls back silently - # otherwise. - SGLANG_OPT_MOE_QUANT_ONCE = EnvBool(False) - - # Cache / overlap SGLANG_OPT_USE_FUSED_STORE_CACHE = EnvBool(True) SGLANG_OPT_USE_JIT_NORM = EnvBool(True) SGLANG_OPT_USE_MULTI_STREAM_OVERLAP = EnvBool(True) - # Force delay_sample_func for all overlap decode (not just grammar mode), - # allowing CPU result processing to overlap with subsequent forward computation - # and reducing the impact of sampling overhead on the critical path. - SGLANG_ENABLE_DELAY_SAMPLE = EnvBool(False) - - # CUDA graph SGLANG_PREP_IN_CUDA_GRAPH = EnvBool(True) - - # Eager forward wraps the ForwardBatch's own tensors instead of copying them - # into the CUDA graph buffer registry (no per-iter device-to-device copy). - SGLANG_EAGER_INPUT_NO_COPY = EnvBool(False) - - # Distributed SGLANG_DSV4_FIX_TP_ATTN_A2A_SCATTER = EnvBool(True) - # ==================================================================== - # ==================================================================== + # =================================================================== # Inkling + # =================================================================== SGLANG_OPT_USE_FUSED_GATE_TOPK = EnvBool(True) # Inside the fused gate: use the CUDA JIT top-k+renorm kernel (v2) instead # of the triton kernel when the production Inkling shape applies. @@ -1221,7 +1335,6 @@ class Envs: # above the band, single-launch tau-folded kernel in the small-t tau # band. Bit-identical to the plain einsum; flag-off restores it. SGLANG_OPT_USE_INKLING_REL_PROJ_DISPATCH = EnvBool(True) - # Quantize and store MXFP8 K/V data and scales in one fused kernel. SGLANG_OPT_INKLING_MXFP8_FUSED_QUANT_STORE = EnvBool(True) # Default reasoning effort in [0.0, 0.99] when omitted by a request. @@ -1229,10 +1342,10 @@ class Envs: # directive is always emitted. SGLANG_INKLING_DEFAULT_REASONING_EFFORT = EnvStr("0.9") SGLANG_INKLING_RS_MM_PREPROCESS = EnvBool(True) - # ==================================================================== - # ==================================================================== - # DSA Backend (GLM 5 series/DeepSeek v3.2) + # =================================================================== + # DSA backend (GLM 5 and DeepSeek V3.2) + # =================================================================== SGLANG_DSA_FUSE_TOPK = EnvBoolWithAlias( True, deprecated_name="SGLANG_NSA_FUSE_TOPK" ) @@ -1277,10 +1390,10 @@ class Envs: # "cuda" = the hand-written SM90 WGMMA kernel (bitwise identical to the # Triton two_dot variant, 1.16-1.38x faster across GLM/DS shapes). SGLANG_OPT_Q8KV8_QPREP_VARIANT = EnvStr("auto") - # ==================================================================== - # ==================================================================== - # Minimax M3 + # =================================================================== + # MiniMax M3 + # =================================================================== SGLANG_OPT_USE_BF16_ROUTER_GEMM = EnvBool(True) SGLANG_OPT_USE_MINIMAX_DENSE_SPARSE_DECODE = EnvBool(False) SGLANG_DISABLE_MSA = EnvBool(False) @@ -1289,42 +1402,34 @@ class Envs: # forces the pre-fp8 behavior (bf16 indexer + widening sparse path, bf16 q) # even when kv_cache_dtype fp8_e4m3 + trtllm_mha + SM100 would activate it. SGLANG_DISABLE_M3_FP8_ATTN_GEMM = EnvBool(False) - # MiniMax-M3 sparse decode indexer: single JIT radix-select kernel replaces the 2-stage split-K Triton topk. SGLANG_OPT_USE_MINIMAX_DECODE_TOPK_RADIX = EnvBool(True) - # Fused JIT store (minimax_store_kv_index) of main+index K/V instead of separate # set_*_buffer copies; falls back when main/index dtypes differ or non-CUDA. SGLANG_OPT_USE_MINIMAX_FUSED_KV_INDEX_STORE = EnvBool(True) - # MiniMax-M3 MXFP8 MoE experimental fusion toggles (default off; A/B only). SGLANG_MINIMAX_M3_FUSED_SWIGLU_MXFP8 = EnvBool(False) SGLANG_MINIMAX_M3_FUSED_MOE_COMBINE = EnvBool(False) - # MiniMax M3 NPU prefill MAIN-attention: route the sparse main attention through # the native Ascend FA op `torch.ops.npu.npu_fused_infer_attention_score` (FIA) # with a per-query CUSTOM block_table SGLANG_MINIMAX_NPU_PREFILL_FIA = EnvBool(True) - # MiniMax-M3 NPU sparse INDEXER (decode + verify topk block selection): route # through the native AscendC packed indexer op instead of the Triton indexer. SGLANG_MINIMAX_NPU_NATIVE_INDEXER = EnvBool(False) - # MiniMax-M3 NPU sparse MAIN-attention (decode-main + verify-main): route the # sparse main attention through the native AscendC sparse-attention op with the # cached block_table override. SGLANG_MINIMAX_NPU_NATIVE_ATTN = EnvBool(False) - # MiniMax-M3 on ROCm force-disables custom all-reduce in its model override # (arg_groups/overrides.py) when aiter all-reduce fusion is off. Set this to # opt back in and keep custom/quick all-reduce enabled -- e.g. to run the # INT4 quick-reduce path via ROCM_QUICK_REDUCE_QUANTIZATION={INT4,INT6,INT8}. SGLANG_M3_ALLOW_CUSTOM_AR = EnvBool(False) - # ==================================================================== - - # ==================================================================== - # Kimi-K3 + # =================================================================== + # Kimi K3 + # =================================================================== # MNNVL fused all-reduce (bf16, TP8): zero-copy 1shot multicast-push for # small messages and in-place NVLS 2shot on symmetric-memory tensors for # large ones, with an optional fused residual add. Covers the KDA o_proj @@ -1351,65 +1456,24 @@ class Envs: # front reads hidden_states once, and run the top-k plus the bf16 cast in one # epilogue kernel. See kernels/ops/moe/moe_front.py. Default on. SGLANG_K3_FUSED_FRONT = EnvBool(True) - - # VLM SGLANG_KIMI_K3_VIT_CUDA_GRAPH_CACHE_CAPACITY = EnvInt(2) SGLANG_KIMI_K3_VIT_CUDA_GRAPH_MIN_HITS = EnvInt(2) SGLANG_KIMI_K3_VIT_CUDA_GRAPH_MAX_SEQLEN = EnvInt(6144) - # ==================================================================== - SGLANG_SHARED_EXPERT_TP1 = EnvBool(False) - # Replicate the input embedding across TP ranks instead of sharding it - # along the vocab dimension (saves an all-reduce/all-gather in the embed - # lookup at the cost of replicated embedding weights). Drives both the - # target and every draft that shares its embedding (see - # get_embedding_tp_kwargs); they must stay in lock-step. Currently only - # applies to the Deepseek-V2 family (Deepseek V3.1, Kimi K2.5) + drafts. - SGLANG_ENABLE_EMBED_REPLICATION = EnvBool(False) - # Symmetric Memory + # =================================================================== + # Symmetric memory + # =================================================================== SGLANG_SYMM_MEM_PREALLOC_GB_SIZE = EnvInt(-1) SGLANG_DEBUG_SYMM_MEM = EnvBool(False) - # Aiter - SGLANG_USE_AITER_FP8_PER_TOKEN = EnvBool(False) - - # EPD - SGLANG_ENCODER_RECV_TIMEOUT = EnvFloat(180.0) - SGLANG_ENCODER_SEND_TIMEOUT = EnvFloat(180.0) - SGLANG_ENCODER_HTTP_TIMEOUT = EnvFloat(1800.0) - SGLANG_ENCODER_REQ_TIMEOUT = EnvFloat(180.0) - SGLANG_ENCODER_DISPATCH_MIN_ITEMS = EnvInt(2) - SGLANG_ENCODER_IMAGE_PROCESSOR_USE_GPU = EnvBool(False) - SGLANG_ENCODER_MAX_BATCH_SIZE = EnvInt(8) - SGLANG_ENCODER_PREPROC_WORKERS = EnvInt(8) - # EncoderBootstrapServer health-check tuning. Interval == 0 disables it. - SGLANG_ENCODER_BOOTSTRAP_HEALTH_CHECK_INTERVAL = EnvFloat(10.0) - SGLANG_ENCODER_BOOTSTRAP_HEALTH_CHECK_TIMEOUT = EnvFloat(2.0) - # Seconds before permanently dropping an unhealthy encoder (0 = keep probing). - SGLANG_ENCODER_BOOTSTRAP_EVICTED_TTL = EnvFloat(600.0) - # Persistent receiver-side GPU embedding pool size for mooncake EPD transport. - # 0 disables (per-request register/deregister). 4096 = 4GB default per TP - SGLANG_EMBEDDING_POOL_SIZE_MB = EnvInt(4096) - SGLANG_ENCODER_DP_WORKER_MAX_INFLIGHT = EnvInt(64) - - # Elastic EP Backup Port - SGLANG_BACKUP_PORT_BASE = EnvInt(10000) - - # Sglang Cache Dir - SGLANG_CACHE_DIR = EnvStr(os.path.expanduser("~/.cache/sglang")) - SGLANG_FLASHINFER_AUTOTUNE_CACHE = EnvBool(True) - # Also autotune one EXTEND-shaped dummy at max_prefill_tokens during - # warmup. Opt-in: the extra forward needs transient activation headroom - # that small-VRAM or tightly-packed configs may not have. - SGLANG_FLASHINFER_AUTOTUNE_EXTEND = EnvBool(False) - SGLANG_ENABLE_MOE_DEFERRED_FINALIZE = EnvBool(True) - + # =================================================================== # Plugin system + # =================================================================== SGLANG_PLATFORM = EnvStr("") SGLANG_PLUGINS = EnvStr("") # =================================================================== - # KV-Canary / Token-Oracle (testing-only) + # KV-Canary and Token-Oracle (testing only) # =================================================================== SGLANG_KV_CANARY_RING_CAPACITY = EnvInt(1024) SGLANG_KV_CANARY_STATS_PRINT_EVERY_N_STEPS = EnvInt(100) @@ -1427,7 +1491,7 @@ class Envs: SGLANG_KV_CANARY_ENABLE_MHA_V = EnvBool(False) # =================================================================== - # Rust Server specific envs. + # Rust server # =================================================================== SGLANG_RUST_SERVER = EnvBool(False) # Most batched requests one /generate HTTP call may expand into.