[Auto Sync] Update detokenizer_manager.py, io_struct.py, mu... (20260120) (#17442)

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: Wangfan Fu <wangfan@x.ai>

docs/get_started/install.md

@@ -15,25 +15,20 @@ pip install uv
 uv pip install "sglang"
 ```
 
-For different CUDA versions, you can try to add `--extra-index-url` to find the correct [PyTorch](https://pytorch.org/get-started/locally/) wheel.
-For example, on GB200, you will need to do the following. Otherwise, it will install the CPU version of PyTorch.
-```
-uv pip install "sglang" --extra-index-url https://download.pytorch.org/whl/cu129
-```
-
-For CUDA 13, Docker is recommended (see Method 3 note on B300/GB300/CUDA 13). If you do not have Docker access, installing the matching `sgl_kernel` wheel from [the sgl-project whl releases](https://github.com/sgl-project/whl/releases) after installing SGLang also works. Replace `X.Y.Z` with the `sgl_kernel` version required by your SGLang install (you can find this by running `uv pip show sgl_kernel`). Examples:
-
-```bash
-# x86_64
-uv pip install "https://github.com/sgl-project/whl/releases/download/vX.Y.Z/sgl_kernel-X.Y.Z+cu130-cp310-abi3-manylinux2014_x86_64.whl"
-
-# aarch64
-uv pip install "https://github.com/sgl-project/whl/releases/download/vX.Y.Z/sgl_kernel-X.Y.Z+cu130-cp310-abi3-manylinux2014_aarch64.whl"
-```
-
 **Quick fixes to common problems**
+- In some cases (e.g., GB200), the above command might install a wrong torch version (e.g., the CPU version) due to dependency resolution. To fix this, you can first run the above command and then force-reinstall the correct [PyTorch](https://pytorch.org/get-started/locally/) with the following:
+  ```
+  uv pip install "torch" --extra-index-url https://download.pytorch.org/whl/cu129 --force-reinstall
+  ```
+- For CUDA 13, Docker is recommended (see the Method 3 note on B300/GB300/CUDA 13). If you do not have Docker access, installing the matching `sgl_kernel` wheel from [the sgl-project whl releases](https://github.com/sgl-project/whl/releases) after installing SGLang also works. Replace `X.Y.Z` with the `sgl_kernel` version required by your SGLang (you can find this by running `uv pip show sgl_kernel`). Examples:
+  ```bash
+  # x86_64
+  uv pip install "https://github.com/sgl-project/whl/releases/download/vX.Y.Z/sgl_kernel-X.Y.Z+cu130-cp310-abi3-manylinux2014_x86_64.whl"
 
-- If you encounter `OSError: CUDA_HOME environment variable is not set`. Please set it to your CUDA install root with either of the following solutions:
+  # aarch64
+  uv pip install "https://github.com/sgl-project/whl/releases/download/vX.Y.Z/sgl_kernel-X.Y.Z+cu130-cp310-abi3-manylinux2014_aarch64.whl"
+  ```
+- If you encounter `OSError: CUDA_HOME environment variable is not set`, set it to your CUDA install root with either of the following solutions:
   1. Use `export CUDA_HOME=/usr/local/cuda-<your-cuda-version>` to set the `CUDA_HOME` environment variable.
   2. Install FlashInfer first following [FlashInfer installation doc](https://docs.flashinfer.ai/installation.html), then install SGLang as described above.
 
@@ -84,9 +79,8 @@ docker run --gpus all \
 
 You can also find the nightly docker images [here](https://hub.docker.com/r/lmsysorg/sglang/tags?name=nightly).
 
-On B300/GB300 (SM103) or CUDA 13 environment, we recommend using the nightly image at `lmsysorg/sglang:dev-cu13` or stable image at `lmsysorg/sglang:latest-cu130-runtime`.
+Notes:
-Please, do not re-install the project as editable inside the docker image, since it will override the version of
+- On B300/GB300 (SM103) or CUDA 13 environment, we recommend using the nightly image at `lmsysorg/sglang:dev-cu13` or stable image at `lmsysorg/sglang:latest-cu130-runtime`. Please, do not re-install the project as editable inside the docker image, since it will override the version of libraries specified by the cu13 docker image.
-libraries specified by the cu13 docker image.
 
 ## Method 4: Using Kubernetes
 

python/pyproject.toml

@@ -69,7 +69,6 @@ dependencies = [
   "torchaudio==2.9.1",
   "torchcodec==0.8.0 ; sys_platform != 'linux' or (sys_platform == 'linux' and platform_machine != 'aarch64' and platform_machine != 'arm64' and platform_machine != 'armv7l')", # torchcodec does not exist in those systems. If not provided, transformer will use torchvision instead by default.
   "torchvision",
-  "torchao==0.9.0",
   "tqdm",
   "transformers==4.57.1",
   "uvicorn",

python/pyproject_cpu.toml

@@ -57,7 +57,6 @@ dependencies = [
   "soundfile==0.13.1",
   "tiktoken",
   "timm==1.0.16",
-  "torchao==0.9.0",
   "tqdm",
   "transformers==4.57.1",
   "uvicorn",

python/pyproject_other.toml

@@ -57,7 +57,6 @@ runtime_common = [
   "soundfile==0.13.1",
   "tiktoken",
   "timm==1.0.16",
-  "torchao==0.9.0",
   "tqdm",
   "transformers==4.57.1",
   "uvicorn",

python/pyproject_xpu.toml

@@ -61,7 +61,6 @@ dependencies = [
   "soundfile==0.13.1",
   "tiktoken",
   "timm==1.0.16",
-  "torchao==0.9.0",
   "tqdm",
   "transformers==4.57.1",
   "uvicorn",

python/sglang/srt/entrypoints/http_server.py

@@ -46,7 +46,7 @@ import orjson
 import requests
 import uvicorn
 import uvloop
-from fastapi import Depends, FastAPI, HTTPException, Request, UploadFile
+from fastapi import Depends, FastAPI, HTTPException, Request
 from fastapi.exceptions import RequestValidationError
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import ORJSONResponse, Response, StreamingResponse
@@ -552,26 +552,20 @@ async def model_info():
         "has_audio_understanding": model_config.is_audio_understandable_model,
         "model_type": getattr(model_config.hf_config, "model_type", None),
         "architectures": getattr(model_config.hf_config, "architectures", None),
+        "weight_version": _global_state.tokenizer_manager.server_args.weight_version,
+        # "hf_config": model_config.hf_config.to_dict(),
     }
     return result
 
 
 @app.get("/get_weight_version")
-async def get_weight_version():
-    """Get the current weight version (deprecated - use /weight_version instead)."""
-    logger.warning(
-        "Endpoint '/get_weight_version' is deprecated and will be removed in a future version. "
-        "Please use '/weight_version' instead."
-    )
-    return await weight_version()
-
-
 @app.get("/weight_version")
 async def weight_version():
     """Get the current weight version."""
-    return {
+    raise HTTPException(
-        "weight_version": _global_state.tokenizer_manager.server_args.weight_version
+        status_code=404,
-    }
+        detail="Endpoint '/get_weight_version' or '/weight_version' is deprecated. Please use '/model_info' instead.",
+    )
 
 
 @app.get("/get_server_info")
@@ -661,30 +655,6 @@ async def generate_request(obj: GenerateReqInput, request: Request):
             return _create_error_response(e)
 
 
-@app.api_route("/generate_from_file", methods=["POST"])
-async def generate_from_file_request(file: UploadFile, request: Request):
-    """Handle a generate request, this is purely to work with input_embeds."""
-    content = await file.read()
-    input_embeds = orjson.loads(content.decode("utf-8"))
-
-    obj = GenerateReqInput(
-        input_embeds=input_embeds,
-        sampling_params={
-            "temperature": 0.0,
-            "max_new_tokens": 512,
-        },
-    )
-
-    try:
-        ret = await _global_state.tokenizer_manager.generate_request(
-            obj, request
-        ).__anext__()
-        return ret
-    except ValueError as e:
-        logger.error(f"Error: {e}")
-        return _create_error_response(e)
-
-
 @app.api_route("/encode", methods=["POST", "PUT"])
 async def encode_request(obj: EmbeddingReqInput, request: Request):
     """Handle an embedding request."""

python/sglang/srt/managers/detokenizer_manager.py

@@ -326,20 +326,20 @@ class DetokenizerManager(MultiHttpWorkerDetokenizerMixin):
 
         return output_strs
 
-    def _extract_routed_experts(self, recv_obj: BatchTokenIDOutput) -> List[List[int]]:
+    def _extract_routed_experts(
-        output_routed_experts = None
+        self, recv_obj: BatchTokenIDOutput
-        if recv_obj.output_routed_experts is not None:
+    ) -> List[List[int]] | None:
-            output_routed_experts = [
+        routed_experts = None
+        if recv_obj.routed_experts is not None:
+            routed_experts = [
                 (
-                    pybase64.b64encode(output_routed_experts.numpy().tobytes()).decode(
+                    pybase64.b64encode(routed_experts.numpy().tobytes()).decode("utf-8")
-                        "utf-8"
+                    if routed_experts is not None
-                    )
-                    if output_routed_experts is not None
                     else []
                 )
-                for output_routed_experts in recv_obj.output_routed_experts
+                for routed_experts in recv_obj.routed_experts
             ]
-        return output_routed_experts
+        return routed_experts
 
     def handle_batch_token_id_out(self, recv_obj: BatchTokenIDOutput):
         # If handling idle batch, set output_strs to [].
@@ -348,7 +348,7 @@ class DetokenizerManager(MultiHttpWorkerDetokenizerMixin):
             if len(recv_obj.rids) > 0
             else []
         )
-        output_routed_experts = self._extract_routed_experts(recv_obj)
+        routed_experts = self._extract_routed_experts(recv_obj)
 
         return BatchStrOutput(
             rids=recv_obj.rids,
@@ -375,7 +375,7 @@ class DetokenizerManager(MultiHttpWorkerDetokenizerMixin):
             output_token_ids_logprobs_idx=recv_obj.output_token_ids_logprobs_idx,
             output_token_entropy_val=recv_obj.output_token_entropy_val,
             output_hidden_states=recv_obj.output_hidden_states,
-            output_routed_experts=output_routed_experts,
+            routed_experts=routed_experts,
             customized_info=recv_obj.customized_info,
             placeholder_tokens_idx=None,
             placeholder_tokens_val=None,

python/sglang/srt/managers/io_struct.py

@@ -202,6 +202,8 @@ class GenerateReqInput(BaseReq, APIServingTimingMixin):
     return_hidden_states: Union[List[bool], bool] = False
     # Whether to return captured routed experts
     return_routed_experts: bool = False
+    # The start location in the prompt for returning routed experts.
+    routed_experts_start_len: int = 0
 
     # The modalities of the image data [image, multi-images, video]
     modalities: Optional[List[str]] = None
@@ -709,6 +711,8 @@ class TokenizedGenerateReqInput(BaseReq):
 
     # Whether to return captured routed experts
     return_routed_experts: bool = False
+    # The start location in the prompt for returning routed experts.
+    routed_experts_start_len: int = 0
 
     # The input embeds
     input_embeds: Optional[Union[List[List[List[float]]], List[List[float]]]] = None
@@ -981,8 +985,9 @@ class BatchTokenIDOutput(
     # Hidden states
     output_hidden_states: List[List[float]]
 
-    # The routed experts for each output token
+    # The routed experts for each token, including both input and output tokens
-    output_routed_experts: List[torch.Tensor]
+    # routed_experts[i] is a tensor of shape (token, layer, top_k) for request i
+    routed_experts: List[Optional[torch.Tensor]]
 
     # The information of placeholder tokens (e.g., image token)
     # idx is the index of the token in the prompt after expansion.
@@ -1068,8 +1073,9 @@ class BatchStrOutput(
     # Hidden states
     output_hidden_states: List[List[float]]
 
-    # The routed experts for each output token
+    # The routed experts for each token, including both input and output tokens
-    output_routed_experts: List[List[int]]
+    # routed_experts[i] is a tensor of shape (token, layer, top_k) for request i
+    routed_experts: List[Optional[torch.Tensor]]
 
     # The information of placeholder tokens (e.g., image token)
     # idx is the index of the token in the prompt after expansion.

python/sglang/srt/managers/multi_tokenizer_mixin.py

@@ -278,8 +278,8 @@ def _handle_output_by_index(output, i):
             output_hidden_states=_extract_field_by_index(
                 output, "output_hidden_states", i, check_length=False
             ),
-            output_routed_experts=_extract_field_by_index(
+            routed_experts=_extract_field_by_index(
-                output, "output_routed_experts", i, check_length=False
+                output, "routed_experts", i, check_length=False
             ),
             customized_info=_extract_field_by_index(
                 output, "customized_info", i, check_length=False

python/sglang/srt/managers/scheduler_output_processor_mixin.py

@@ -854,7 +854,7 @@ class SchedulerOutputProcessorMixin:
         retraction_counts = []
         output_hidden_states = None
         load = self.get_load()
-        output_routed_experts = None
+        routed_experts = None
         customized_info = {}
 
         queue_times = []
@@ -1051,9 +1051,9 @@ class SchedulerOutputProcessorMixin:
                         output_hidden_states = []
                     output_hidden_states.append(req.hidden_states)
                 if req.return_routed_experts:
-                    if output_routed_experts is None:
+                    if routed_experts is None:
-                        output_routed_experts = []
+                        routed_experts = []
-                    output_routed_experts.append(req.routed_experts)
+                    routed_experts.append(req.routed_experts)
 
                 if req.customized_info is not None:
                     for k, v in req.customized_info.items():
@@ -1108,7 +1108,7 @@ class SchedulerOutputProcessorMixin:
                     output_token_ids_logprobs_idx=output_token_ids_logprobs_idx,
                     output_token_entropy_val=None,
                     output_hidden_states=output_hidden_states,
-                    output_routed_experts=output_routed_experts,
+                    routed_experts=routed_experts,
                     customized_info=customized_info,
                     placeholder_tokens_idx=None,
                     placeholder_tokens_val=None,

python/sglang/srt/managers/tokenizer_manager.py

@@ -1534,8 +1534,8 @@ class TokenizerManager(TokenizerCommunicatorMixin, TokenizerManagerMultiItemMixi
 
             if getattr(recv_obj, "output_hidden_states", None):
                 meta_info["hidden_states"] = recv_obj.output_hidden_states[i]
-            if getattr(recv_obj, "output_routed_experts", None):
+            if getattr(recv_obj, "routed_experts", None):
-                meta_info["routed_experts"] = recv_obj.output_routed_experts[i]
+                meta_info["routed_experts"] = recv_obj.routed_experts[i]
             if getattr(recv_obj, "customized_info", None):
                 for k, v in recv_obj.customized_info.items():
                     meta_info[k] = v[i]

test/registered/openai_server/basic/test_openai_server.py

@@ -89,7 +89,9 @@ class TestOpenAIServer(CustomTestCase):
 
         if logprobs:
             assert response.choices[0].logprobs
-            assert isinstance(response.choices[0].logprobs.tokens[0], str)
+            assert isinstance(
+                response.choices[0].logprobs.tokens[0], str
+            ), f"{response=}"
             assert isinstance(response.choices[0].logprobs.top_logprobs[1], dict)
             ret_num_top_logprobs = len(response.choices[0].logprobs.top_logprobs[1])
 
@@ -155,6 +157,9 @@ class TestOpenAIServer(CustomTestCase):
 
             if logprobs:
                 assert response.choices[0].logprobs, f"no logprobs in response"
+                print(
+                    f"{response=}, {response.choices[0]=}, {response.choices[0].logprobs=}"
+                )
                 assert isinstance(
                     response.choices[0].logprobs.tokens[0], str
                 ), f"{response.choices[0].logprobs.tokens[0]} is not a string"

test/registered/quant/test_gptqmodel_dynamic.py

@@ -54,11 +54,11 @@ def check_quant_method(model_path: str, use_marlin_kernel: bool):
         model_config=model_config, load_config=load_config, device_config=device_config
     )
 
-    from sglang.srt.layers.linear import UnquantizedLinearMethod
     from sglang.srt.layers.quantization.gptq import (
         GPTQLinearMethod,
         GPTQMarlinLinearMethod,
     )
+    from sglang.srt.layers.quantization.unquant import UnquantizedLinearMethod
 
     linear_method_cls = (
         GPTQMarlinLinearMethod if use_marlin_kernel else (GPTQLinearMethod)