Fix dataclasses.asdict on the msgspec ServerArgs (#38958)
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
52c191da52
commit
00143e9c23
@@ -4,7 +4,8 @@ python3 offline_batch_inference.py --model meta-llama/Llama-3.1-8B-Instruct
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import dataclasses
|
||||
|
||||
import msgspec
|
||||
|
||||
import sglang as sgl
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
@@ -24,7 +25,7 @@ def main(
|
||||
sampling_params = {"temperature": 0.8, "top_p": 0.95}
|
||||
|
||||
# Create an LLM.
|
||||
llm = sgl.Engine(**dataclasses.asdict(server_args))
|
||||
llm = sgl.Engine(**msgspec.structs.asdict(server_args))
|
||||
|
||||
outputs = llm.generate(prompts, sampling_params)
|
||||
# Print the outputs.
|
||||
|
||||
@@ -9,9 +9,10 @@ which is useful to implement an online-like generation with batched inference.
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import dataclasses
|
||||
import time
|
||||
|
||||
import msgspec
|
||||
|
||||
import sglang as sgl
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
|
||||
@@ -26,7 +27,7 @@ class InferenceEngine:
|
||||
|
||||
|
||||
async def run_server(server_args):
|
||||
inference = InferenceEngine(**dataclasses.asdict(server_args))
|
||||
inference = InferenceEngine(**msgspec.structs.asdict(server_args))
|
||||
|
||||
# Sample prompts.
|
||||
prompts = [
|
||||
|
||||
@@ -4,7 +4,8 @@ python offline_batch_inference_vlm.py --model-path Qwen/Qwen2-VL-7B-Instruct
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import dataclasses
|
||||
|
||||
import msgspec
|
||||
|
||||
import sglang as sgl
|
||||
from sglang.srt.parser.conversation import chat_templates
|
||||
@@ -14,7 +15,7 @@ from sglang.srt.server_args import ServerArgs
|
||||
def main(
|
||||
server_args: ServerArgs,
|
||||
):
|
||||
vlm = sgl.Engine(**dataclasses.asdict(server_args))
|
||||
vlm = sgl.Engine(**msgspec.structs.asdict(server_args))
|
||||
|
||||
conv = chat_templates[server_args.chat_template].copy()
|
||||
image_token = conv.image_token
|
||||
|
||||
Reference in New Issue
Block a user