Fix dataclasses.asdict on the msgspec ServerArgs (#38958)
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
52c191da52
commit
00143e9c23
@@ -4,10 +4,10 @@ import asyncio
|
|||||||
import atexit
|
import atexit
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
from dataclasses import asdict
|
|
||||||
from typing import Iterator
|
from typing import Iterator
|
||||||
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
|
from msgspec.structs import asdict
|
||||||
from sglang_simulator.compat import apply_simulator_server_args
|
from sglang_simulator.compat import apply_simulator_server_args
|
||||||
from sglang_simulator.dataset import BaseDataset, GenericRequest
|
from sglang_simulator.dataset import BaseDataset, GenericRequest
|
||||||
from sglang_simulator.simulation.benchmark import BaseBenchmarkRunner, BenchmarkConfig
|
from sglang_simulator.simulation.benchmark import BaseBenchmarkRunner, BenchmarkConfig
|
||||||
|
|||||||
@@ -4,7 +4,8 @@ python3 offline_batch_inference.py --model meta-llama/Llama-3.1-8B-Instruct
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
import dataclasses
|
|
||||||
|
import msgspec
|
||||||
|
|
||||||
import sglang as sgl
|
import sglang as sgl
|
||||||
from sglang.srt.server_args import ServerArgs
|
from sglang.srt.server_args import ServerArgs
|
||||||
@@ -24,7 +25,7 @@ def main(
|
|||||||
sampling_params = {"temperature": 0.8, "top_p": 0.95}
|
sampling_params = {"temperature": 0.8, "top_p": 0.95}
|
||||||
|
|
||||||
# Create an LLM.
|
# Create an LLM.
|
||||||
llm = sgl.Engine(**dataclasses.asdict(server_args))
|
llm = sgl.Engine(**msgspec.structs.asdict(server_args))
|
||||||
|
|
||||||
outputs = llm.generate(prompts, sampling_params)
|
outputs = llm.generate(prompts, sampling_params)
|
||||||
# Print the outputs.
|
# Print the outputs.
|
||||||
|
|||||||
@@ -9,9 +9,10 @@ which is useful to implement an online-like generation with batched inference.
|
|||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
import asyncio
|
import asyncio
|
||||||
import dataclasses
|
|
||||||
import time
|
import time
|
||||||
|
|
||||||
|
import msgspec
|
||||||
|
|
||||||
import sglang as sgl
|
import sglang as sgl
|
||||||
from sglang.srt.server_args import ServerArgs
|
from sglang.srt.server_args import ServerArgs
|
||||||
|
|
||||||
@@ -26,7 +27,7 @@ class InferenceEngine:
|
|||||||
|
|
||||||
|
|
||||||
async def run_server(server_args):
|
async def run_server(server_args):
|
||||||
inference = InferenceEngine(**dataclasses.asdict(server_args))
|
inference = InferenceEngine(**msgspec.structs.asdict(server_args))
|
||||||
|
|
||||||
# Sample prompts.
|
# Sample prompts.
|
||||||
prompts = [
|
prompts = [
|
||||||
|
|||||||
@@ -4,7 +4,8 @@ python offline_batch_inference_vlm.py --model-path Qwen/Qwen2-VL-7B-Instruct
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
import dataclasses
|
|
||||||
|
import msgspec
|
||||||
|
|
||||||
import sglang as sgl
|
import sglang as sgl
|
||||||
from sglang.srt.parser.conversation import chat_templates
|
from sglang.srt.parser.conversation import chat_templates
|
||||||
@@ -14,7 +15,7 @@ from sglang.srt.server_args import ServerArgs
|
|||||||
def main(
|
def main(
|
||||||
server_args: ServerArgs,
|
server_args: ServerArgs,
|
||||||
):
|
):
|
||||||
vlm = sgl.Engine(**dataclasses.asdict(server_args))
|
vlm = sgl.Engine(**msgspec.structs.asdict(server_args))
|
||||||
|
|
||||||
conv = chat_templates[server_args.chat_template].copy()
|
conv = chat_templates[server_args.chat_template].copy()
|
||||||
image_token = conv.image_token
|
image_token = conv.image_token
|
||||||
|
|||||||
Reference in New Issue
Block a user