From 93fa577bb95a37699e7f1f56a486e436d5792b71 Mon Sep 17 00:00:00 2001 From: Lianmin Zheng Date: Sun, 24 May 2026 14:35:15 -0700 Subject: [PATCH] Clean up server startup log noise (#26205) --- .claude/skills/clean-startup-log/SKILL.md | 173 ++++++++++++++---- python/sglang/srt/configs/model_config.py | 6 +- .../layers/attention/flashinfer_backend.py | 6 +- .../sglang/srt/managers/template_detection.py | 11 -- .../sglang/srt/managers/template_manager.py | 13 +- python/sglang/srt/mem_cache/memory_pool.py | 4 +- .../sglang/srt/model_executor/model_runner.py | 2 - python/sglang/srt/models/gpt_oss.py | 4 +- .../srt/utils/hf_transformers/tokenizer.py | 6 +- 9 files changed, 157 insertions(+), 68 deletions(-) diff --git a/.claude/skills/clean-startup-log/SKILL.md b/.claude/skills/clean-startup-log/SKILL.md index 8f7c25411..c1b9e886f 100644 --- a/.claude/skills/clean-startup-log/SKILL.md +++ b/.claude/skills/clean-startup-log/SKILL.md @@ -23,6 +23,11 @@ For TP>1 testing: uv run sglang serve --model-path Qwen/Qwen3-8B --tp 2 2>&1 | tee /tmp/startup_log.txt ``` +For MoE / hybrid-SWA models (e.g. gpt-oss), test separately — they exercise different code paths: +```bash +uv run sglang serve --model-path openai/gpt-oss-20b 2>&1 | tee /tmp/startup_log.txt +``` + ### 2. Compare against the clean reference log Read `/tmp/startup_log.txt` and compare it against the reference log at the bottom of this file. Identify lines that: @@ -31,6 +36,7 @@ Read `/tmp/startup_log.txt` and compare it against the reference log at the bott - Contain `WARNING`, `deprecated`, `is deprecated`, or similar noise - Are printed by third-party libraries (transformers, torchao, NCCL, Gloo, tqdm, etc.) - Are duplicate/redundant with information already logged by SGLang +- Appear multiple times due to `ModelConfig` being constructed in multiple processes ### 3. Classify each noisy line @@ -40,6 +46,7 @@ For each noisy line, determine: |----------|--------| | **SGLang code using wrong API** | Fix the SGLang code (e.g., replace deprecated API with new one) | | **SGLang code logging at wrong level** | Change log level (e.g., warning -> debug for non-actionable messages) | +| **Duplicated across processes** | Downgrade to debug — info logged in one process becomes noise in 3-4 | | **Third-party lib prints at import time** | Suppress the logger or redirect stdout during that import | | **C-level print from .so library** | Redirect fd 1 during the specific C call, or accept it if too invasive | | **Real warning the user should see** | Keep it | @@ -52,6 +59,23 @@ List all noisy lines with their source and proposed fix. Ask the user to review After approval, apply fixes one at a time, re-launch the server, and verify each fix works. +## Key Architecture: Why Logs Repeat + +`ModelConfig` is constructed **3-4 times** during startup across different processes: +1. Main process: `ServerArgs.__post_init__()` → `get_model_config()` → `ModelConfig()` +2. Scheduler subprocess: `Scheduler.init_model_config()` → `ModelConfig.from_server_args()` +3. Scheduler subprocess: `TpModelWorker._init_model_config()` → `ModelConfig.from_server_args()` +4. Main process: `TokenizerManager.init_model_config()` → `ModelConfig.from_server_args()` + +Similarly, `get_tokenizer()` is called **5 times** across processes: +1. `resolve_auto_parsers` (main) — `template_detection.py` +2. `Scheduler.init_tokenizer()` (scheduler subprocess) — `scheduler.py` +3. `DetokenizerManager` (detokenizer subprocess) — `detokenizer_manager.py` +4. `TpModelWorker.__init__()` (scheduler subprocess) — `tp_worker.py` +5. `TokenizerManager` (main) — `tokenizer_manager.py` + +Any `logger.info()` or `logger.warning()` in `ModelConfig.__init__()` or `get_tokenizer()` will appear 3-5 times. **Keep these at `logger.debug()`.** + ## Known Noise Sources and Fixes (from past sessions) ### 1. torchao "Skipping import of cpp extensions due to incompatible torch version" @@ -69,11 +93,21 @@ After approval, apply fixes one at a time, re-launch the server, and verify each _torchao_logger.setLevel(_prev_level) ``` -### 2. "`torch_dtype` is deprecated! Use `dtype` instead!" +### 2. "`torch_dtype` is deprecated! Use `dtype` instead!" (PARTIALLY FIXED) - **Source:** `transformers/configuration_utils.py` — the `torch_dtype` property warns via `logger.warning_once()` -- **Trigger:** `get_hf_text_config()` in `sglang/srt/utils/hf_transformers/common.py` accesses `config.torch_dtype` -- **Fix:** Replace all `getattr(config, "torch_dtype", ...)` with `getattr(config, "dtype", ...)` and `config.torch_dtype = X` with `config.dtype = X` in `common.py` +- **Trigger:** Model files accessing `config.torch_dtype` instead of `config.dtype` +- **Fix applied so far:** Only `models/gpt_oss.py` (lines 222, 471) — tested with `openai/gpt-oss-20b`. +- **Remaining files that still use `config.torch_dtype`** (fix each only after testing with the corresponding model): + - `models/bailing_moe.py` (line 302) + - `models/llada2.py` (line 313) + - `models/qwen3_next.py` (lines 192, 209) + - `models/qwen3_5.py` (line 245) + - `models/nano_nemotron_vl.py` (lines 79, 102, 284) + - `models/llava.py` (lines 732, 734-737) + - `model_loader/loader.py` (line 649) +- **Note:** `common.py` was already fixed in a prior session. If new model files are added with `config.torch_dtype`, the warning will reappear — grep for `\.torch_dtype` to find them. +- **Important:** Only change `config.torch_dtype` → `config.dtype` for models you have actually tested. The `dtype` property should return the same value, but verify per-model to avoid regressions. ### 3. "`BaseImageProcessorFast` is deprecated" @@ -105,6 +139,67 @@ After approval, apply fixes one at a time, re-launch the server, and verify each - **Status:** These are expected and useful. They show progress during weight loading and CUDA graph capture. Keep them. +### 9. CUTE_DSL "Unexpected error during package walk" — double-logged (FIXED) + +- **Source:** `nvidia-cutlass-dsl` package at `.venv/.../cutlass/cutlass_dsl/cutlass.py`, line 391. Logger named `CUTE_DSL` with its own `StreamHandler`. +- **Trigger:** During CUDA graph capture, cutlass DSL walks packages and hits an unexpected error for `cutlass.cute.experimental`. +- **Root cause of double-logging:** The CUTE_DSL logger has `propagate=True` (default), so the warning is emitted by both the CUTE_DSL handler (with its format) and the root logger (SGLang's format). +- **Fix applied:** In `entrypoints/engine.py`, changed `CUTE_DSL_LOG_LEVEL` from `"30"` (WARNING) to `"40"` (ERROR). This suppresses the WARNING at both the CUTE_DSL logger and root propagation levels. The env var controls both `logger.setLevel()` and `console_handler.setLevel()` in cutlass's `setup_log()`. + +### 10. ModelConfig init logs repeated 3x (FIXED) + +- **Lines:** `"Downcasting torch.float32 to ..."`, `"Hybrid swa model: ..."`, `"DeepGemm is enabled but ..."` +- **Source:** `configs/model_config.py` — `_get_and_verify_dtype()` (line 1457), `_derive_hybrid_model()` (line 497), `_verify_quantization()` (line 1236) +- **Root cause:** `ModelConfig.__init__()` is called 3-4 times in different processes (see "Key Architecture" above). Each construction fires the same log lines. +- **Fix applied:** Downgraded all three from `logger.info()`/`logger.warning()` to `logger.debug()`. The dtype is already visible in `server_args` and `Load weight end`. Hybrid SWA info appears in `Tree cache initialized`. DeepGemm is not actionable. + +### 11. Tokenizer retry/fallback messages repeated 3-4x (FIXED) + +- **Lines:** `"Tokenizer loaded as generic TokenizersBackend ... retrying"`, `"Loading tokenizer ... directly as PreTrainedTokenizerFast"`, `"Tokenizer for ... loaded as generic TokenizersBackend. Set --trust-remote-code"` +- **Source:** `utils/hf_transformers/tokenizer.py` — `_resolve_tokenizers_backend()` (line 215), `_load_tokenizer_by_declared_class()` (line 110), final warning (line 244) +- **Root cause:** 5 separate `get_tokenizer()` calls across processes (see "Key Architecture" above). Each produces 3 log lines. Concurrent subprocess launches cause interleaved/doubled output. +- **Fix applied:** Downgraded all three from `logger.warning()`/`logger.info()` to `logger.debug()`. + +### 12. Template detection logs — 5 lines consolidated to 1 (FIXED) + +- **Lines:** `"Detected reasoning config '...' from template rule '...'"`, `"Detected reasoning parser '...' from template rule '...'"`, `"Detected tool-call parser '...' from template rule '...'"`, `"Auto-detected reasoning parser: ..."`, `"Auto-detected tool-call parser: ..."` +- **Source:** `managers/template_detection.py` (lines 337, 370) logged each detection rule match. `managers/template_manager.py` (lines 177-182) logged summary lines that duplicated the detection logs. +- **Fix applied:** Removed per-rule logs from `template_detection.py`. Consolidated the 5 lines in `template_manager.py` into a single summary: `"Auto-detected template features: reasoning_config=..., reasoning_parser=..., tool_call_parser=..."` + +### 13. KV cache dtype logged separately from allocation (FIXED) + +- **Lines:** `"Using KV cache dtype: torch.bfloat16"` then `"KV Cache is allocated. #tokens: ..., K size: ..., V size: ..."` +- **Source:** `model_executor/model_runner.py` (line 2217) and `mem_cache/memory_pool.py` (line 740) +- **Fix applied:** Removed the standalone dtype log from `model_runner.py`. Added `dtype` field to the allocation log in `memory_pool.py`: `"KV Cache is allocated. dtype: torch.bfloat16, #tokens: ..., K size: ..., V size: ..."` + +### 14. CUTLASS backend warning — B200 → SM100, warning → info (FIXED) + +- **Line:** `"CUTLASS backend is disabled when piecewise cuda graph is enabled due to TMA descriptor initialization issues on B200."` +- **Source:** `layers/attention/flashinfer_backend.py` (line 249) +- **Fix applied:** Changed "B200" to "SM100 GPUs" (the condition checks `is_sm100_supported()` which matches SM10x, not just B200). Downgraded from `logger.warning()` to `logger.info()` since it's an expected automatic fallback. + +### 15. `max_total_num_tokens` and `Tree cache initialized` log ordering + +- **Issue:** `max_total_num_tokens=...` appears before `Tree cache initialized:...` even though tree cache is conceptually part of memory setup. +- **Root cause:** `max_total_num_tokens` is logged inside `init_model_worker()` (scheduler.py:972), which runs before `build_kv_cache()` (scheduler.py:425) where tree cache is created. +- **Status:** Not fixed — reordering was reverted. Acceptable as-is. + +### 16. `Ignore import error when loading sglang.srt.models.midashenglm` + +- **Source:** `models/registry.py` (line 109) — `logger.warning()` during `import_model_classes()` which iterates all model modules via `pkgutil.iter_modules` +- **Trigger:** The `midashenglm` model depends on `torchaudio`, which fails to load +- **Status:** Should be downgraded to `logger.debug()` — not actionable when loading an unrelated model. Same pattern exists in `managers/multimodal_processor.py`, `dllm/algorithm/__init__.py`, `multimodal_gen/runtime/models/registry.py`. + +### 17. `Multiple NUMA nodes found for GPU X` + +- **Source:** `utils/numa_utils.py` (line 112) — `logger.warning()` +- **Status:** Could be downgraded to `logger.info()`. The situation is handled gracefully ("Using the first one") and not actionable. + +### 18. Warmup `/model_info` access log + +- **Source:** Uvicorn access log, triggered by SGLang's own warmup at `entrypoints/http_server.py` (line 1877) +- **Status:** SGLang talking to itself. Could suppress uvicorn access logger during warmup, or exclude `/model_info` from warmup access logging. + ## Investigation Techniques ### Trace what triggers an import @@ -137,43 +232,51 @@ logging.getLogger('TARGET_LOGGER_NAME').addHandler(h) strings /path/to/library.so | grep "SEARCH_STRING" ``` +### Find all config.torch_dtype accesses (for deprecation warning) +```bash +grep -rn '\.torch_dtype' python/sglang/srt/models/ python/sglang/srt/model_loader/ python/sglang/srt/utils/hf_transformers/ +``` + ## Reference: Clean Startup Log (TP=1, Qwen3-8B) ``` -[2026-04-27 02:35:53] Attention backend not specified. Use trtllm_mha backend by default. -[2026-04-27 02:35:53] TensorRT-LLM MHA only supports page_size of 16, 32 or 64, changing page_size from None to 64. -[2026-04-27 02:35:54] server_args=ServerArgs(model_path='Qwen/Qwen3-8B', ...) -[2026-04-27 02:35:56] Using default HuggingFace chat template with detected content format: string -[2026-04-27 02:36:03] Init torch distributed begin. +[2026-05-24 00:52:39] Attention backend not specified. Use trtllm_mha backend by default. +[2026-05-24 00:52:39] TensorRT-LLM MHA only supports page_size of 16, 32 or 64, changing page_size from None to 64. +[2026-05-24 00:52:40] server_args=ServerArgs(model_path='Qwen/Qwen3-8B', ...) +[2026-05-24 00:52:40] Multiple NUMA nodes found for GPU 0: [...]. Using the first one. +[2026-05-24 00:52:42] Using default HuggingFace chat template with detected content format: string +[2026-05-24 00:52:42] Auto-detected template features: reasoning_config=..., reasoning_parser=qwen3, tool_call_parser=qwen +[2026-05-24 00:52:50] Init torch distributed begin. [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[2026-04-27 02:36:03] Init torch distributed ends. elapsed=0.27 s, mem usage=0.09 GB -[2026-04-27 02:36:04] Load weight begin. avail mem=177.57 GB -[2026-04-27 02:36:04] Found local HF snapshot for Qwen/Qwen3-8B at ...; skipping download. -Multi-thread loading shards: 100% Completed | 5/5 [00:01<00:00, 3.08it/s] -[2026-04-27 02:36:06] Load weight end. elapsed=1.97 s, type=Qwen3ForCausalLM, avail mem=162.30 GB, mem usage=15.28 GB. -[2026-04-27 02:36:06] Using KV cache dtype: torch.bfloat16 -[2026-04-27 02:36:06] KV Cache is allocated. #tokens: 992896, K size: 68.18 GB, V size: 68.18 GB -[2026-04-27 02:36:06] Memory pool end. avail mem=25.26 GB -[2026-04-27 02:36:06] Capture cuda graph begin. This can take up to several minutes. avail mem=24.14 GB -[2026-04-27 02:36:06] Capture cuda graph bs [1, 2, 4, ...] -Capturing batches (bs=1 avail_mem=23.54 GB): 100% | 52/52 [00:03<00:00, 16.76it/s] -[2026-04-27 02:36:09] Capture cuda graph end. Time elapsed: 3.74 s. mem usage=0.60 GB. avail mem=23.54 GB. -[2026-04-27 02:36:09] Capture piecewise CUDA graph begin. avail mem=23.54 GB -[2026-04-27 02:36:09] Capture cuda graph num tokens [4, 8, 12, ...] -Compiling num tokens (num_tokens=4): 100% | 74/74 [00:09<00:00, 8.16it/s] -Capturing num tokens (num_tokens=4 avail_mem=21.23 GB): 100% | 74/74 [00:08<00:00, 9.11it/s] -[2026-04-27 02:36:27] Capture piecewise CUDA graph end. Time elapsed: 17.62 s. mem usage=2.32 GB. avail mem=21.22 GB. -[2026-04-27 02:36:28] max_total_num_tokens=992896, chunked_prefill_size=16384, ... -[2026-04-27 02:36:29] INFO: Started server process [399368] -[2026-04-27 02:36:29] INFO: Waiting for application startup. -[2026-04-27 02:36:29] Using default chat sampling params from model generation config: ... -[2026-04-27 02:36:29] INFO: Application startup complete. -[2026-04-27 02:36:29] INFO: Uvicorn running on http://127.0.0.1:30000 (Press CTRL+C to quit) -[2026-04-27 02:36:30] Prefill batch, #new-seq: 1, #new-token: 64, ... -[2026-04-27 02:36:30] INFO: 127.0.0.1:34916 - "POST /generate HTTP/1.1" 200 OK -[2026-04-27 02:36:30] The server is fired up and ready to roll! +[2026-05-24 00:52:50] Init torch distributed ends. elapsed=0.21 s, mem usage=0.10 GB +[2026-05-24 00:52:51] Load weight begin. avail mem=275.75 GB +[2026-05-24 00:52:51] Found local HF snapshot for Qwen/Qwen3-8B at ...; skipping download. +Multi-thread loading shards: 100% Completed | 5/5 [00:01<00:00, 2.62it/s] +[2026-05-24 00:52:54] Load weight end. elapsed=2.62 s, type=Qwen3ForCausalLM, avail mem=260.48 GB, mem usage=15.28 GB. +[2026-05-24 00:52:54] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1707904, K size: 117.28 GB, V size: 117.28 GB +[2026-05-24 00:52:54] Memory pool end. avail mem=25.28 GB +[2026-05-24 00:52:54] CUTLASS backend is disabled when piecewise cuda graph is enabled due to TMA descriptor initialization issues on SM100 GPUs. Using auto backend instead for stability. +[2026-05-24 00:52:54] Capture cuda graph begin. This can take up to several minutes. avail mem=24.16 GB +[2026-05-24 00:52:54] Capture cuda graph bs [1, 2, 4, ...] +Capturing batches (bs=1 avail_mem=23.56 GB): 100% | 52/52 [00:05<00:00, 10.36it/s] +[2026-05-24 00:53:00] Capture cuda graph end. Time elapsed: 5.38 s. mem usage=0.60 GB. avail mem=23.56 GB. +[2026-05-24 00:53:00] Capture piecewise CUDA graph begin. avail mem=23.56 GB +[2026-05-24 00:53:00] Capture cuda graph num tokens [4, 8, 12, ...] +Compiling num tokens (num_tokens=4): 100% | 74/74 [00:09<00:00, 7.44it/s] +Capturing num tokens (num_tokens=4 avail_mem=21.24 GB): 100% | 74/74 [00:07<00:00, 10.44it/s] +[2026-05-24 00:53:18] Capture piecewise CUDA graph end. Time elapsed: 18.18 s. mem usage=2.32 GB. avail mem=21.24 GB. +[2026-05-24 00:53:20] Tree cache initialized: source=default impl=RadixCache hybrid_swa=False hybrid_ssm=False hierarchical=False streaming_wrapped=False +[2026-05-24 00:53:20] max_total_num_tokens=1707904, chunked_prefill_size=16384, max_prefill_tokens=16384, max_running_requests=4096, context_len=40960, available_gpu_mem=21.24 GB +[2026-05-24 00:53:20] INFO: Started server process [1964249] +[2026-05-24 00:53:20] INFO: Waiting for application startup. +[2026-05-24 00:53:20] Using default chat sampling params from model generation config: {'temperature': 0.6, 'top_k': 20, 'top_p': 0.95} +[2026-05-24 00:53:20] INFO: Application startup complete. +[2026-05-24 00:53:20] INFO: Uvicorn running on http://127.0.0.1:30000 (Press CTRL+C to quit) +[2026-05-24 00:53:21] Prefill batch, #new-seq: 1, #new-token: 64, ... +[2026-05-24 00:53:21] INFO: 127.0.0.1:... - "POST /generate HTTP/1.1" 200 OK +[2026-05-24 00:53:21] The server is fired up and ready to roll! ``` -Note: `[Gloo]` messages and tqdm progress bars are acceptable. The key is no warnings or deprecation messages from transformers, torchao, or other third-party libraries. +Note: `[Gloo]` messages and tqdm progress bars are acceptable. The key is no warnings or deprecation messages from transformers, torchao, or other third-party libraries. The `CUTLASS backend is disabled` message is now `info` level, not a warning. diff --git a/python/sglang/srt/configs/model_config.py b/python/sglang/srt/configs/model_config.py index e3551392a..4511fb298 100644 --- a/python/sglang/srt/configs/model_config.py +++ b/python/sglang/srt/configs/model_config.py @@ -1450,15 +1450,15 @@ def _get_and_verify_dtype( if torch_dtype != config_dtype: if torch_dtype == torch.float32: # Upcasting to float32 is allowed. - logger.info("Upcasting %s to %s.", config_dtype, torch_dtype) + logger.debug("Upcasting %s to %s.", config_dtype, torch_dtype) pass elif config_dtype == torch.float32: # Downcasting from float32 to float16 or bfloat16 is allowed. - logger.info("Downcasting %s to %s.", config_dtype, torch_dtype) + logger.debug("Downcasting %s to %s.", config_dtype, torch_dtype) pass else: # Casting between float16 and bfloat16 is allowed with a warning. - logger.warning("Casting %s to %s.", config_dtype, torch_dtype) + logger.debug("Casting %s to %s.", config_dtype, torch_dtype) return torch_dtype diff --git a/python/sglang/srt/layers/attention/flashinfer_backend.py b/python/sglang/srt/layers/attention/flashinfer_backend.py index 13930a752..cd8cc2dcd 100644 --- a/python/sglang/srt/layers/attention/flashinfer_backend.py +++ b/python/sglang/srt/layers/attention/flashinfer_backend.py @@ -243,12 +243,10 @@ class FlashInferAttnBackend(AttentionBackend): fmha_backend = "auto" if is_sm100_supported(): - # Disable CUTLASS backend when piecewise cuda graph is enabled - # due to TMA descriptor initialization issues on B200 if not model_runner.server_args.disable_piecewise_cuda_graph: - logger.warning( + logger.info( "CUTLASS backend is disabled when piecewise cuda graph is enabled " - "due to TMA descriptor initialization issues on B200. " + "due to TMA descriptor initialization issues on SM100 GPUs. " "Using auto backend instead for stability." ) else: diff --git a/python/sglang/srt/managers/template_detection.py b/python/sglang/srt/managers/template_detection.py index ac3710b15..190cec67d 100644 --- a/python/sglang/srt/managers/template_detection.py +++ b/python/sglang/srt/managers/template_detection.py @@ -334,12 +334,6 @@ def match_rules( for rule in rules: try: if rule.predicate(ctx): - logger.info( - "Detected %s '%s' from template rule '%s'.", - label, - rule.value, - rule.name, - ) return rule.value except Exception as e: logger.warning( @@ -367,11 +361,6 @@ def detect_reasoning_pattern( ) for rule in REASONING_MODE_RULES: if rule.predicate(ctx): - logger.info( - "Detected reasoning config '%s' from template rule '%s'.", - rule.value, - rule.name, - ) return rule.value.always_on, rule.value return False, None diff --git a/python/sglang/srt/managers/template_manager.py b/python/sglang/srt/managers/template_manager.py index 4328120a2..95337ad2a 100644 --- a/python/sglang/srt/managers/template_manager.py +++ b/python/sglang/srt/managers/template_manager.py @@ -173,14 +173,15 @@ class TemplateManager: if tokenizer_manager.tokenizer: template = tokenizer_manager.tokenizer.chat_template self._run_template_detection(template, tokenizer_manager.tokenizer) + parts = [] + if self._reasoning_config: + parts.append(f"reasoning_config={self._reasoning_config}") if self._suggested_reasoning_parser: - logger.info( - f"Auto-detected reasoning parser: {self._suggested_reasoning_parser}" - ) + parts.append(f"reasoning_parser={self._suggested_reasoning_parser}") if self._suggested_tool_call_parser: - logger.info( - f"Auto-detected tool-call parser: {self._suggested_tool_call_parser}" - ) + parts.append(f"tool_call_parser={self._suggested_tool_call_parser}") + if parts: + logger.info(f"Auto-detected template features: {', '.join(parts)}") def _load_explicit_chat_template( self, tokenizer_manager: TokenizerManager, chat_template_arg: str diff --git a/python/sglang/srt/mem_cache/memory_pool.py b/python/sglang/srt/mem_cache/memory_pool.py index b1353902b..78127c05f 100644 --- a/python/sglang/srt/mem_cache/memory_pool.py +++ b/python/sglang/srt/mem_cache/memory_pool.py @@ -738,13 +738,13 @@ class KVCache(abc.ABC): k_size_GB = k_size / GB v_size_GB = v_size / GB logger.info( - f"KV Cache is allocated. #tokens: {num_tokens}, K size: {k_size_GB:.2f} GB, V size: {v_size_GB:.2f} GB" + f"KV Cache is allocated. dtype: {self.dtype}, #tokens: {num_tokens}, K size: {k_size_GB:.2f} GB, V size: {v_size_GB:.2f} GB" ) self.mem_usage = k_size_GB + v_size_GB else: kv_size_GB = kv_size_bytes / GB logger.info( - f"KV Cache is allocated. #tokens: {num_tokens}, KV size: {kv_size_GB:.2f} GB" + f"KV Cache is allocated. dtype: {self.dtype}, #tokens: {num_tokens}, KV size: {kv_size_GB:.2f} GB" ) self.mem_usage = kv_size_GB diff --git a/python/sglang/srt/model_executor/model_runner.py b/python/sglang/srt/model_executor/model_runner.py index eb0a633a3..71b86ed5e 100644 --- a/python/sglang/srt/model_executor/model_runner.py +++ b/python/sglang/srt/model_executor/model_runner.py @@ -2214,8 +2214,6 @@ class ModelRunner(ModelRunnerKVCacheMixin): f"Unsupported kv_cache_dtype: {self.server_args.kv_cache_dtype}." ) - log_info_on_rank0(logger, f"Using KV cache dtype: {self.kv_cache_dtype}") - def init_cublas(self): """We need to run a small matmul to init cublas. Otherwise, it will raise some errors later.""" dtype = torch.float16 diff --git a/python/sglang/srt/models/gpt_oss.py b/python/sglang/srt/models/gpt_oss.py index f6f2e72df..84f8890ee 100644 --- a/python/sglang/srt/models/gpt_oss.py +++ b/python/sglang/srt/models/gpt_oss.py @@ -219,7 +219,7 @@ class GptOssSparseMoeBlock(nn.Module): bias=True, quant_config=None, prefix=add_prefix("gate", prefix), - params_dtype=config.torch_dtype, + params_dtype=config.dtype, ) def forward( @@ -468,7 +468,7 @@ class GptOssDecoderLayer(nn.Module): prefix=add_prefix("self_attn", prefix), sliding_window_size=self.sliding_window_size, layer_type=config.layer_types[layer_id], - params_dtype=config.torch_dtype, + params_dtype=config.dtype, ) self.layer_id = layer_id diff --git a/python/sglang/srt/utils/hf_transformers/tokenizer.py b/python/sglang/srt/utils/hf_transformers/tokenizer.py index 9a0fafb0f..40b1693a3 100644 --- a/python/sglang/srt/utils/hf_transformers/tokenizer.py +++ b/python/sglang/srt/utils/hf_transformers/tokenizer.py @@ -105,7 +105,7 @@ def _load_tokenizer_by_declared_class(tokenizer_name, *args, **kwargs): if tok_cls is None: return None - logger.info( + logger.debug( "Loading tokenizer for %s directly as %s (bypassing AutoTokenizer)", tokenizer_name, tok_class_name, @@ -208,7 +208,7 @@ def _resolve_tokenizers_backend(tokenizer_name, *args, **common_kwargs): ``tokenizer_config.json``. May still return a ``TokenizersBackend`` if all retries fail (with a warning). """ - logger.warning( + logger.debug( "Tokenizer loaded as generic TokenizersBackend for %s, " "retrying with use_fast=False", tokenizer_name, @@ -239,7 +239,7 @@ def _resolve_tokenizers_backend(tokenizer_name, *args, **common_kwargs): tokenizer_name, ) else: - logger.warning( + logger.debug( "Tokenizer for %s loaded as generic TokenizersBackend. " "Set --trust-remote-code to load the model-specific tokenizer.", tokenizer_name,