Remove the torchao integration (--torchao-config) (#34304)
Co-authored-by: Brayden Zhong <brayden@radixark.ai>
This commit is contained in:
co-authored by
Brayden Zhong
parent
fe0c18effd
commit
5e65dd01a7
@@ -32,29 +32,6 @@ class TestSRTEngineWithQuantArgs(CustomTestCase):
|
||||
engine.generate(prompt, sampling_params)
|
||||
engine.shutdown()
|
||||
|
||||
def test_2_torchao_args(self):
|
||||
|
||||
# we don't test int8dq because currently there is conflict between int8dq and capture cuda graph
|
||||
torchao_args_list = [
|
||||
# "int8dq",
|
||||
"int8wo",
|
||||
"fp8wo",
|
||||
"fp8dq-per_tensor",
|
||||
"fp8dq-per_row",
|
||||
] + [f"int4wo-{group_size}" for group_size in [32, 64, 128, 256]]
|
||||
|
||||
prompt = "Today is a sunny day and I like"
|
||||
model_path = DEFAULT_SMALL_MODEL_NAME_FOR_TEST
|
||||
|
||||
sampling_params = {"temperature": 0, "max_new_tokens": 8}
|
||||
|
||||
for torchao_config in torchao_args_list:
|
||||
engine = sgl.Engine(
|
||||
model_path=model_path, random_seed=42, torchao_config=torchao_config
|
||||
)
|
||||
engine.generate(prompt, sampling_params)
|
||||
engine.shutdown()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user