diff --git a/docs_new/docs/references/environment_variables.mdx b/docs_new/docs/references/environment_variables.mdx index 3efc356a1..4ae94069c 100644 --- a/docs_new/docs/references/environment_variables.mdx +++ b/docs_new/docs/references/environment_variables.mdx @@ -127,6 +127,11 @@ SGLang supports various environment variables that can be used to configure its Control FlashInfer availability check `true` + + `SGLANG_FLASHINFER_AUTOTUNE_CACHE` + Reuse persisted FlashInfer autotune results from `SGLANG_CACHE_DIR` across runs. Set to `0` to force re-autotuning on every startup; the fresh result is written to a `runs/..json` sibling file (the canonical cache is left untouched). + `true` + `SGLANG_SKIP_P2P_CHECK` Skip P2P (peer-to-peer) access check diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index ef1c3869a..9821b2b73 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -727,6 +727,7 @@ class Envs: # Sglang Cache Dir SGLANG_CACHE_DIR = EnvStr(os.path.expanduser("~/.cache/sglang")) + SGLANG_FLASHINFER_AUTOTUNE_CACHE = EnvBool(True) # Plugin system SGLANG_PLATFORM = EnvStr("") diff --git a/python/sglang/srt/model_executor/model_runner.py b/python/sglang/srt/model_executor/model_runner.py index afe4be2cd..3e124ce8f 100644 --- a/python/sglang/srt/model_executor/model_runner.py +++ b/python/sglang/srt/model_executor/model_runner.py @@ -2379,13 +2379,27 @@ class ModelRunner(ModelRunnerKVCacheMixin): from flashinfer.autotuner import autotune cache_path = self._flashinfer_autotune_cache_path() - logger.info("Running FlashInfer autotune with cache: %s", cache_path) + if envs.SGLANG_FLASHINFER_AUTOTUNE_CACHE.get(): + autotune_cache = cache_path + logger.info("Running FlashInfer autotune with cache: %s", autotune_cache) + else: + timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + runs_dir = cache_path.parent / "runs" + runs_dir.mkdir(parents=True, exist_ok=True) + autotune_cache = ( + runs_dir / f"{cache_path.stem}.{timestamp}{cache_path.suffix}" + ) + logger.info( + "Running FlashInfer autotune (cache reuse DISABLED via " + "SGLANG_FLASHINFER_AUTOTUNE_CACHE=0); writing fresh result to: %s", + autotune_cache, + ) # Run warmup on the non-default stream to avoid NCCL 2.29+ cudaMemcpyBatchAsync # calls on default stream (unsupported by CUDA) when --enable-symm-mem is used. self.forward_stream.wait_stream(torch.cuda.current_stream()) with torch.get_device_module(self.device).stream(self.forward_stream): - with torch.inference_mode(), autotune(True, cache=str(cache_path)): + with torch.inference_mode(), autotune(True, cache=str(autotune_cache)): self._dummy_run(batch_size=self.req_to_token_pool.size) torch.cuda.current_stream().wait_stream(self.forward_stream) logger.info("FlashInfer autotune completed.")