minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[GLM4.6v] Required changes for bumping up to transformer 5.x (#13229)

Browse Source
This commit is contained in:
Binyao Jiang
2025-11-18 10:58:00 +08:00
committed by GitHub
parent d7984f3125
commit 90c18a16cb
11 changed files with 29 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/advanced_features/separate_reasoning.ipynb
+2 -6
View File
@@ -256,9 +256,7 @@
"\n", "\n",
"tokenizer = AutoTokenizer.from_pretrained(\"deepseek-ai/DeepSeek-R1-Distill-Qwen-7B\")\n", "tokenizer = AutoTokenizer.from_pretrained(\"deepseek-ai/DeepSeek-R1-Distill-Qwen-7B\")\n",
"input = tokenizer.apply_chat_template(\n", "input = tokenizer.apply_chat_template(\n",
" messages,\n", " messages, tokenize=False, add_generation_prompt=True, return_dict=False\n",
" tokenize=False,\n",
" add_generation_prompt=True,\n",
")\n", ")\n",
"\n", "\n",
"gen_url = f\"http://localhost:{port}/generate\"\n", "gen_url = f\"http://localhost:{port}/generate\"\n",
@@ -319,9 +317,7 @@
"llm = sgl.Engine(model_path=\"deepseek-ai/DeepSeek-R1-Distill-Qwen-7B\")\n", "llm = sgl.Engine(model_path=\"deepseek-ai/DeepSeek-R1-Distill-Qwen-7B\")\n",
"tokenizer = AutoTokenizer.from_pretrained(\"deepseek-ai/DeepSeek-R1-Distill-Qwen-7B\")\n", "tokenizer = AutoTokenizer.from_pretrained(\"deepseek-ai/DeepSeek-R1-Distill-Qwen-7B\")\n",
"input = tokenizer.apply_chat_template(\n", "input = tokenizer.apply_chat_template(\n",
" messages,\n", " messages, tokenize=False, add_generation_prompt=True, return_dict=False\n",
" tokenize=False,\n",
" add_generation_prompt=True,\n",
")\n", ")\n",
"sampling_params = {\n", "sampling_params = {\n",
" \"max_new_tokens\": 1024,\n", " \"max_new_tokens\": 1024,\n",
docs/advanced_features/structured_outputs.ipynb
+5 -5
View File
@@ -443,7 +443,7 @@
" }\n", " }\n",
"]\n", "]\n",
"text = tokenizer.apply_chat_template(\n", "text = tokenizer.apply_chat_template(\n",
" messages, tokenize=False, add_generation_prompt=True\n", " messages, tokenize=False, add_generation_prompt=True, return_dict=False\n",
")\n", ")\n",
"response = requests.post(\n", "response = requests.post(\n",
" f\"http://localhost:{port}/generate\",\n", " f\"http://localhost:{port}/generate\",\n",
@@ -525,7 +525,7 @@
" }\n", " }\n",
"]\n", "]\n",
"text = tokenizer.apply_chat_template(\n", "text = tokenizer.apply_chat_template(\n",
" messages, tokenize=False, add_generation_prompt=True\n", " messages, tokenize=False, add_generation_prompt=True, return_dict=False\n",
")\n", ")\n",
"response = requests.post(\n", "response = requests.post(\n",
" f\"http://localhost:{port}/generate\",\n", " f\"http://localhost:{port}/generate\",\n",
@@ -571,7 +571,7 @@
" }\n", " }\n",
"]\n", "]\n",
"text = tokenizer.apply_chat_template(\n", "text = tokenizer.apply_chat_template(\n",
" messages, tokenize=False, add_generation_prompt=True\n", " messages, tokenize=False, add_generation_prompt=True, return_dict=False\n",
")\n", ")\n",
"response = requests.post(\n", "response = requests.post(\n",
" f\"http://localhost:{port}/generate\",\n", " f\"http://localhost:{port}/generate\",\n",
@@ -606,7 +606,7 @@
"tokenizer = AutoTokenizer.from_pretrained(\"meta-llama/Meta-Llama-3.1-8B-Instruct\")\n", "tokenizer = AutoTokenizer.from_pretrained(\"meta-llama/Meta-Llama-3.1-8B-Instruct\")\n",
"\n", "\n",
"text = tokenizer.apply_chat_template(\n", "text = tokenizer.apply_chat_template(\n",
" messages, tokenize=False, add_generation_prompt=True\n", " messages, tokenize=False, add_generation_prompt=True, return_dict=False\n",
")\n", ")\n",
"payload = {\n", "payload = {\n",
" \"text\": text,\n", " \"text\": text,\n",
@@ -883,7 +883,7 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"text = tokenizer.apply_chat_template(\n", "text = tokenizer.apply_chat_template(\n",
" messages, tokenize=False, add_generation_prompt=True\n", " messages, tokenize=False, add_generation_prompt=True, return_dict=False\n",
")\n", ")\n",
"prompts = [text]\n", "prompts = [text]\n",
"\n", "\n",
docs/advanced_features/structured_outputs_for_reasoning_models.ipynb
+4 -4
View File
@@ -400,7 +400,7 @@
" },\n", " },\n",
"]\n", "]\n",
"text = tokenizer.apply_chat_template(\n", "text = tokenizer.apply_chat_template(\n",
" messages, tokenize=False, add_generation_prompt=True\n", " messages, tokenize=False, add_generation_prompt=True, return_dict=False\n",
")\n", ")\n",
"# Make API request\n", "# Make API request\n",
"response = requests.post(\n", "response = requests.post(\n",
@@ -448,7 +448,7 @@
"\n", "\n",
"# JSON\n", "# JSON\n",
"text = tokenizer.apply_chat_template(\n", "text = tokenizer.apply_chat_template(\n",
" messages, tokenize=False, add_generation_prompt=True\n", " messages, tokenize=False, add_generation_prompt=True, return_dict=False\n",
")\n", ")\n",
"response = requests.post(\n", "response = requests.post(\n",
" f\"http://localhost:{port}/generate\",\n", " f\"http://localhost:{port}/generate\",\n",
@@ -543,7 +543,7 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"text = tokenizer.apply_chat_template(\n", "text = tokenizer.apply_chat_template(\n",
" messages, tokenize=False, add_generation_prompt=True\n", " messages, tokenize=False, add_generation_prompt=True, return_dict=False\n",
")\n", ")\n",
"payload = {\n", "payload = {\n",
" \"text\": text,\n", " \"text\": text,\n",
@@ -765,7 +765,7 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"text = tokenizer.apply_chat_template(\n", "text = tokenizer.apply_chat_template(\n",
" messages, tokenize=False, add_generation_prompt=True\n", " messages, tokenize=False, add_generation_prompt=True, return_dict=False\n",
")\n", ")\n",
"prompts = [text]\n", "prompts = [text]\n",
"\n", "\n",
docs/advanced_features/tool_parser.ipynb
+2 -5
View File
@@ -391,10 +391,7 @@
"messages = get_messages()\n", "messages = get_messages()\n",
"\n", "\n",
"input = tokenizer.apply_chat_template(\n", "input = tokenizer.apply_chat_template(\n",
" messages,\n", " messages, tokenize=False, add_generation_prompt=True, tools=tools, return_dict=False\n",
" tokenize=False,\n",
" add_generation_prompt=True,\n",
" tools=tools,\n",
")\n", ")\n",
"\n", "\n",
"gen_url = f\"http://localhost:{port}/generate\"\n", "gen_url = f\"http://localhost:{port}/generate\"\n",
@@ -459,7 +456,7 @@
"llm = sgl.Engine(model_path=\"Qwen/Qwen2.5-7B-Instruct\")\n", "llm = sgl.Engine(model_path=\"Qwen/Qwen2.5-7B-Instruct\")\n",
"tokenizer = llm.tokenizer_manager.tokenizer\n", "tokenizer = llm.tokenizer_manager.tokenizer\n",
"input_ids = tokenizer.apply_chat_template(\n", "input_ids = tokenizer.apply_chat_template(\n",
" messages, tokenize=True, add_generation_prompt=True, tools=tools\n", " messages, tokenize=True, add_generation_prompt=True, tools=tools, return_dict=False\n",
")\n", ")\n",
"\n", "\n",
"# Note that for gpt-oss tool parser, adding \"no_stop_trim\": True\n", "# Note that for gpt-oss tool parser, adding \"no_stop_trim\": True\n",
docs/basic_usage/native_api.ipynb
+1 -1
View File
@@ -410,7 +410,7 @@
"]\n", "]\n",
"\n", "\n",
"tokenizer = AutoTokenizer.from_pretrained(\"Skywork/Skywork-Reward-Llama-3.1-8B-v0.2\")\n", "tokenizer = AutoTokenizer.from_pretrained(\"Skywork/Skywork-Reward-Llama-3.1-8B-v0.2\")\n",
"prompts = tokenizer.apply_chat_template(CONVS, tokenize=False)\n", "prompts = tokenizer.apply_chat_template(CONVS, tokenize=False, return_dict=False)\n",
"\n", "\n",
"url = f\"http://localhost:{port}/classify\"\n", "url = f\"http://localhost:{port}/classify\"\n",
"data = {\"model\": \"Skywork/Skywork-Reward-Llama-3.1-8B-v0.2\", \"text\": prompts}\n", "data = {\"model\": \"Skywork/Skywork-Reward-Llama-3.1-8B-v0.2\", \"text\": prompts}\n",
python/sglang/bench_serving.py
+5 -1
View File
@@ -995,7 +995,10 @@ async def get_mooncake_request_over_time(
# Form the full prompt from history # Form the full prompt from history
try: try:
full_prompt_text = tokenizer.apply_chat_template( full_prompt_text = tokenizer.apply_chat_template(
chat_history, tokenize=False, add_generation_prompt=True chat_history,
tokenize=False,
add_generation_prompt=True,
return_dict=False,
) )
except Exception: except Exception:
full_prompt_text = "\n".join( full_prompt_text = "\n".join(
@@ -1161,6 +1164,7 @@ def sample_sharegpt_requests(
[{"role": "user", "content": prompt}], [{"role": "user", "content": prompt}],
add_generation_prompt=True, add_generation_prompt=True,
tokenize=False, tokenize=False,
return_dict=False,
) )
if tokenizer.bos_token: if tokenizer.bos_token:
prompt = prompt.replace(tokenizer.bos_token, "") prompt = prompt.replace(tokenizer.bos_token, "")
python/sglang/multimodal_gen/runtime/loader/weight_utils.py
+2 -1
View File
@@ -46,7 +46,8 @@ enable_hf_transfer()
class DisabledTqdm(tqdm): class DisabledTqdm(tqdm):
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs, disable=True) kwargs["disable"] = True
super().__init__(*args, **kwargs)
def get_lock(model_name_or_path: str | Path, cache_dir: str | None = None): def get_lock(model_name_or_path: str | Path, cache_dir: str | None = None):
python/sglang/srt/entrypoints/openai/serving_chat.py
+2
View File
@@ -324,6 +324,7 @@ class OpenAIServingChat(OpenAIServingBase):
**( **(
request.chat_template_kwargs if request.chat_template_kwargs else {} request.chat_template_kwargs if request.chat_template_kwargs else {}
), ),
return_dict=False,
) )
except Exception: except Exception:
# This except branch will be triggered when the chosen model # This except branch will be triggered when the chosen model
@@ -343,6 +344,7 @@ class OpenAIServingChat(OpenAIServingBase):
**( **(
request.chat_template_kwargs if request.chat_template_kwargs else {} request.chat_template_kwargs if request.chat_template_kwargs else {}
), ),
return_dict=False,
) )
if assistant_prefix: if assistant_prefix:
python/sglang/srt/model_loader/weight_utils.py
+2 -1
View File
@@ -70,7 +70,8 @@ enable_hf_transfer()
class DisabledTqdm(tqdm): class DisabledTqdm(tqdm):
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs, disable=True) kwargs["disable"] = True
super().__init__(*args, **kwargs)
def get_lock(model_name_or_path: str, cache_dir: Optional[str] = None): def get_lock(model_name_or_path: str, cache_dir: Optional[str] = None):
python/sglang/test/runners.py
+1 -1
View File
@@ -353,7 +353,7 @@ class HFRunner:
scores = [] scores = []
for conv in prompts: for conv in prompts:
conv_formatted = self.tokenizer.apply_chat_template( conv_formatted = self.tokenizer.apply_chat_template(
conv, tokenize=False conv, tokenize=False, return_dict=False
) )
conv_tokenized = self.tokenizer( conv_tokenized = self.tokenizer(
conv_formatted, return_tensors="pt" conv_formatted, return_tensors="pt"
test/srt/models/test_reward_models.py
+3 -1
View File
@@ -68,7 +68,9 @@ class TestRewardModels(CustomTestCase):
torch_dtype=torch_dtype, torch_dtype=torch_dtype,
model_type="reward", model_type="reward",
) as srt_runner: ) as srt_runner:
prompts = srt_runner.tokenizer.apply_chat_template(convs, tokenize=False) prompts = srt_runner.tokenizer.apply_chat_template(
convs, tokenize=False, return_dict=False
)
srt_outputs = srt_runner.forward(prompts) srt_outputs = srt_runner.forward(prompts)
hf_scores = torch.tensor(hf_outputs.scores) hf_scores = torch.tensor(hf_outputs.scores)