Files
sglang/test/registered/model_loading/test_instanttensor_loader.py
T

64 lines
1.9 KiB
Python

import multiprocessing as mp
import os
import unittest
import torch
from sglang.srt.utils.common import temp_set_env
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.runners import SRTRunner, check_close_model_outputs
from sglang.test.test_utils import CustomTestCase
register_cuda_ci(est_time=120, stage="weekly", runner_config="2-gpu-large")
MODEL = "Qwen/Qwen2-0.5B"
class TestInstantTensorLoader(CustomTestCase):
@classmethod
def setUpClass(cls):
mp.set_start_method("spawn", force=True)
def test_generation_matches_safetensors(self):
prompts = ["The capital of France is"]
runner_args = dict(
torch_dtype=torch.float16,
model_type="generation",
tp_size=2,
disable_cuda_graph=True,
disable_radix_cache=True,
max_total_tokens=256,
)
with temp_set_env(
**{key: None for key in os.environ if key.startswith("INSTANTTENSOR_")}
):
with SRTRunner(MODEL, load_format="safetensors", **runner_args) as runner:
expected = runner.forward(prompts, max_new_tokens=16)
with SRTRunner(
MODEL,
load_format="instanttensor",
model_loader_extra_config={
"backend": "MMAP",
"chunk_size": 8 * 1024 * 1024,
"io_depth": 2,
},
**runner_args,
) as runner:
actual = runner.forward(prompts, max_new_tokens=16)
self.assertEqual(expected.output_strs, actual.output_strs)
check_close_model_outputs(
hf_outputs=expected,
srt_outputs=actual,
prefill_tolerance=1e-6,
decode_tolerance=1e-6,
rouge_l_tolerance=1.0,
debug_text="safetensors vs InstantTensor weight loading",
)
if __name__ == "__main__":
unittest.main()