minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[TestFix] change LoRA tests to use NVIDIA adapter instead of Nutanix (#19642)

Browse Source 
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
This commit is contained in:
Glen Liu
2026-03-02 12:55:41 -08:00
committed by GitHub
co-authored by gemini-code-assist[bot]
parent 51ee17ce44
commit cc860a2198
3 changed files with 60 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/test/lora_utils.py
+2 -2
View File
@@ -68,7 +68,7 @@ ALL_OTHER_LORA_MODELS = [
base="meta-llama/Llama-3.1-8B-Instruct", base="meta-llama/Llama-3.1-8B-Instruct",
adaptors=[ adaptors=[
LoRAAdaptor( LoRAAdaptor(
name="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", name="nvidia/llama-3.1-nemoguard-8b-topic-control",
prefill_tolerance=1e-1, prefill_tolerance=1e-1,
), ),
], ],
@@ -109,7 +109,7 @@ ALL_OTHER_MULTI_LORA_MODELS = [
prefill_tolerance=1e-1, prefill_tolerance=1e-1,
), ),
LoRAAdaptor( LoRAAdaptor(
name="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", name="nvidia/llama-3.1-nemoguard-8b-topic-control",
prefill_tolerance=1e-1, prefill_tolerance=1e-1,
), ),
], ],
test/registered/lora/test_lora_update.py
+56 -71
View File
@@ -37,7 +37,6 @@ from sglang.test.test_utils import (
register_cuda_ci( register_cuda_ci(
est_time=487, est_time=487,
suite="stage-b-test-large-1-gpu", suite="stage-b-test-large-1-gpu",
disabled="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16 removed from HuggingFace",
) )
PROMPTS = [ PROMPTS = [
@@ -98,13 +97,13 @@ BASIC_TESTS = [
max_loras_per_batch=3, max_loras_per_batch=3,
all_adapters=[ all_adapters=[
"philschmid/code-llama-3-1-8b-text-to-sql-lora", "philschmid/code-llama-3-1-8b-text-to-sql-lora",
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", "nvidia/llama-3.1-nemoguard-8b-topic-control",
"pbevan11/llama-3.1-8b-ocr-correction", "pbevan11/llama-3.1-8b-ocr-correction",
], ],
initial_adapters=[ initial_adapters=[
# Testing 3 supported lora-path formats. # Testing 3 supported lora-path formats.
"philschmid/code-llama-3-1-8b-text-to-sql-lora", "philschmid/code-llama-3-1-8b-text-to-sql-lora",
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16=Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", "nvidia/llama-3.1-nemoguard-8b-topic-control=nvidia/llama-3.1-nemoguard-8b-topic-control",
{ {
"lora_name": "pbevan11/llama-3.1-8b-ocr-correction", "lora_name": "pbevan11/llama-3.1-8b-ocr-correction",
"lora_path": "pbevan11/llama-3.1-8b-ocr-correction", "lora_path": "pbevan11/llama-3.1-8b-ocr-correction",
@@ -130,14 +129,14 @@ BASIC_TESTS = [
data=create_batch_data( data=create_batch_data(
[ [
"philschmid/code-llama-3-1-8b-text-to-sql-lora", "philschmid/code-llama-3-1-8b-text-to-sql-lora",
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", "nvidia/llama-3.1-nemoguard-8b-topic-control",
"pbevan11/llama-3.1-8b-ocr-correction", "pbevan11/llama-3.1-8b-ocr-correction",
] ]
), ),
), ),
Operation( Operation(
type=OperationType.UNLOAD, type=OperationType.UNLOAD,
data="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", data="nvidia/llama-3.1-nemoguard-8b-topic-control",
), ),
Operation( Operation(
type=OperationType.UNLOAD, type=OperationType.UNLOAD,
@@ -149,9 +148,7 @@ BASIC_TESTS = [
), ),
Operation( Operation(
type=OperationType.FORWARD, type=OperationType.FORWARD,
data=create_batch_data( data=create_batch_data("nvidia/llama-3.1-nemoguard-8b-topic-control"),
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16"
),
), ),
Operation( Operation(
type=OperationType.FORWARD, type=OperationType.FORWARD,
@@ -159,7 +156,7 @@ BASIC_TESTS = [
), ),
Operation( Operation(
type=OperationType.LOAD, type=OperationType.LOAD,
data="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", data="nvidia/llama-3.1-nemoguard-8b-topic-control",
expected_error="already loaded", expected_error="already loaded",
), ),
Operation( Operation(
@@ -172,7 +169,7 @@ BASIC_TESTS = [
data=create_batch_data( data=create_batch_data(
[ [
"philschmid/code-llama-3-1-8b-text-to-sql-lora", "philschmid/code-llama-3-1-8b-text-to-sql-lora",
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", "nvidia/llama-3.1-nemoguard-8b-topic-control",
"pbevan11/llama-3.1-8b-ocr-correction", "pbevan11/llama-3.1-8b-ocr-correction",
] ]
), ),
@@ -189,14 +186,14 @@ BASIC_TESTS = [
type=OperationType.FORWARD, type=OperationType.FORWARD,
data=create_batch_data( data=create_batch_data(
[ [
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", "nvidia/llama-3.1-nemoguard-8b-topic-control",
"pbevan11/llama-3.1-8b-ocr-correction", "pbevan11/llama-3.1-8b-ocr-correction",
] ]
), ),
), ),
Operation( Operation(
type=OperationType.UNLOAD, type=OperationType.UNLOAD,
data="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", data="nvidia/llama-3.1-nemoguard-8b-topic-control",
), ),
Operation( Operation(
type=OperationType.UNLOAD, type=OperationType.UNLOAD,
@@ -204,9 +201,7 @@ BASIC_TESTS = [
), ),
Operation( Operation(
type=OperationType.FORWARD, type=OperationType.FORWARD,
data=create_batch_data( data=create_batch_data("nvidia/llama-3.1-nemoguard-8b-topic-control"),
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16"
),
), ),
Operation( Operation(
type=OperationType.FORWARD, type=OperationType.FORWARD,
@@ -229,7 +224,7 @@ BASIC_TESTS = [
max_loras_per_batch=4, max_loras_per_batch=4,
all_adapters=[ all_adapters=[
"philschmid/code-llama-3-1-8b-text-to-sql-lora", "philschmid/code-llama-3-1-8b-text-to-sql-lora",
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", "nvidia/llama-3.1-nemoguard-8b-topic-control",
"pbevan11/llama-3.1-8b-ocr-correction", "pbevan11/llama-3.1-8b-ocr-correction",
], ],
op_sequence=[ op_sequence=[
@@ -239,7 +234,7 @@ BASIC_TESTS = [
), ),
Operation( Operation(
type=OperationType.LOAD, type=OperationType.LOAD,
data="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", data="nvidia/llama-3.1-nemoguard-8b-topic-control",
), ),
Operation( Operation(
type=OperationType.LOAD, type=OperationType.LOAD,
@@ -263,7 +258,7 @@ BASIC_TESTS = [
data=create_batch_data( data=create_batch_data(
[ [
"philschmid/code-llama-3-1-8b-text-to-sql-lora", "philschmid/code-llama-3-1-8b-text-to-sql-lora",
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", "nvidia/llama-3.1-nemoguard-8b-topic-control",
"pbevan11/llama-3.1-8b-ocr-correction", "pbevan11/llama-3.1-8b-ocr-correction",
None, None,
] ]
@@ -282,7 +277,7 @@ BASIC_TESTS = [
data=create_batch_data( data=create_batch_data(
[ [
None, None,
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", "nvidia/llama-3.1-nemoguard-8b-topic-control",
"pbevan11/llama-3.1-8b-ocr-correction", "pbevan11/llama-3.1-8b-ocr-correction",
None, None,
] ]
@@ -290,7 +285,7 @@ BASIC_TESTS = [
), ),
Operation( Operation(
type=OperationType.UNLOAD, type=OperationType.UNLOAD,
data="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", data="nvidia/llama-3.1-nemoguard-8b-topic-control",
), ),
Operation( Operation(
type=OperationType.UNLOAD, type=OperationType.UNLOAD,
@@ -298,9 +293,7 @@ BASIC_TESTS = [
), ),
Operation( Operation(
type=OperationType.FORWARD, type=OperationType.FORWARD,
data=create_batch_data( data=create_batch_data("nvidia/llama-3.1-nemoguard-8b-topic-control"),
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16"
),
), ),
Operation( Operation(
type=OperationType.FORWARD, type=OperationType.FORWARD,
@@ -317,7 +310,7 @@ BASIC_TESTS = [
), ),
Operation( Operation(
type=OperationType.LOAD, type=OperationType.LOAD,
data="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", data="nvidia/llama-3.1-nemoguard-8b-topic-control",
expected_error="already loaded", expected_error="already loaded",
), ),
Operation( Operation(
@@ -330,7 +323,7 @@ BASIC_TESTS = [
data=create_batch_data( data=create_batch_data(
[ [
"philschmid/code-llama-3-1-8b-text-to-sql-lora", "philschmid/code-llama-3-1-8b-text-to-sql-lora",
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", "nvidia/llama-3.1-nemoguard-8b-topic-control",
"pbevan11/llama-3.1-8b-ocr-correction", "pbevan11/llama-3.1-8b-ocr-correction",
None, None,
] ]
@@ -347,7 +340,7 @@ TARGET_MODULE_TESTS = [
lora_target_modules=["all"], lora_target_modules=["all"],
max_lora_rank=64, max_lora_rank=64,
all_adapters=[ all_adapters=[
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", # target_modules = q, k, v, o, gate, up, down "nvidia/llama-3.1-nemoguard-8b-topic-control", # target_modules = q, k, v, o, gate, up, down
"algoprog/fact-generation-llama-3.1-8b-instruct-lora", # target_modules = q, k, v, o, gate "algoprog/fact-generation-llama-3.1-8b-instruct-lora", # target_modules = q, k, v, o, gate
], ],
initial_adapters=["algoprog/fact-generation-llama-3.1-8b-instruct-lora"], initial_adapters=["algoprog/fact-generation-llama-3.1-8b-instruct-lora"],
@@ -360,21 +353,19 @@ TARGET_MODULE_TESTS = [
), ),
Operation( Operation(
type=OperationType.FORWARD, type=OperationType.FORWARD,
data=create_batch_data( data=create_batch_data("nvidia/llama-3.1-nemoguard-8b-topic-control"),
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16"
),
expected_error="never been loaded", expected_error="never been loaded",
), ),
Operation( Operation(
type=OperationType.LOAD, type=OperationType.LOAD,
data="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", data="nvidia/llama-3.1-nemoguard-8b-topic-control",
), ),
Operation( Operation(
type=OperationType.FORWARD, type=OperationType.FORWARD,
data=create_batch_data( data=create_batch_data(
[ [
"algoprog/fact-generation-llama-3.1-8b-instruct-lora", "algoprog/fact-generation-llama-3.1-8b-instruct-lora",
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", "nvidia/llama-3.1-nemoguard-8b-topic-control",
None, None,
] ]
), ),
@@ -387,16 +378,14 @@ TARGET_MODULE_TESTS = [
max_loras_per_batch=3, max_loras_per_batch=3,
max_lora_rank=64, max_lora_rank=64,
all_adapters=[ all_adapters=[
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", # target_modules = q, k, v, o, gate, up, down "nvidia/llama-3.1-nemoguard-8b-topic-control", # target_modules = q, k, v, o, gate, up, down
"algoprog/fact-generation-llama-3.1-8b-instruct-lora", # target_modules = q, k, v, o, gate "algoprog/fact-generation-llama-3.1-8b-instruct-lora", # target_modules = q, k, v, o, gate
], ],
initial_adapters=["Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16"], initial_adapters=["nvidia/llama-3.1-nemoguard-8b-topic-control"],
op_sequence=[ op_sequence=[
Operation( Operation(
type=OperationType.FORWARD, type=OperationType.FORWARD,
data=create_batch_data( data=create_batch_data("nvidia/llama-3.1-nemoguard-8b-topic-control"),
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16"
),
), ),
Operation( Operation(
type=OperationType.FORWARD, type=OperationType.FORWARD,
@@ -414,7 +403,7 @@ TARGET_MODULE_TESTS = [
data=create_batch_data( data=create_batch_data(
[ [
"algoprog/fact-generation-llama-3.1-8b-instruct-lora", "algoprog/fact-generation-llama-3.1-8b-instruct-lora",
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", "nvidia/llama-3.1-nemoguard-8b-topic-control",
None, None,
] ]
), ),
@@ -427,7 +416,7 @@ TARGET_MODULE_TESTS = [
max_loras_per_batch=3, max_loras_per_batch=3,
max_lora_rank=64, max_lora_rank=64,
all_adapters=[ all_adapters=[
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", # target_modules = q, k, v, o, gate, up, down "nvidia/llama-3.1-nemoguard-8b-topic-control", # target_modules = q, k, v, o, gate, up, down
"algoprog/fact-generation-llama-3.1-8b-instruct-lora", # target_modules = q, k, v, o, gate "algoprog/fact-generation-llama-3.1-8b-instruct-lora", # target_modules = q, k, v, o, gate
], ],
initial_adapters=["algoprog/fact-generation-llama-3.1-8b-instruct-lora"], initial_adapters=["algoprog/fact-generation-llama-3.1-8b-instruct-lora"],
@@ -440,14 +429,12 @@ TARGET_MODULE_TESTS = [
), ),
Operation( Operation(
type=OperationType.FORWARD, type=OperationType.FORWARD,
data=create_batch_data( data=create_batch_data("nvidia/llama-3.1-nemoguard-8b-topic-control"),
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16"
),
expected_error="never been loaded", expected_error="never been loaded",
), ),
Operation( Operation(
type=OperationType.LOAD, type=OperationType.LOAD,
data="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", data="nvidia/llama-3.1-nemoguard-8b-topic-control",
expected_error="incompatible", expected_error="incompatible",
), ),
Operation( Operation(
@@ -469,17 +456,15 @@ MAX_LORA_RANK_TESTS = [
max_loras_per_batch=3, max_loras_per_batch=3,
max_lora_rank=32, max_lora_rank=32,
all_adapters=[ all_adapters=[
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", # r = 4 "nvidia/llama-3.1-nemoguard-8b-topic-control", # r = 4
"pbevan11/llama-3.1-8b-ocr-correction", # r = 32 "pbevan11/llama-3.1-8b-ocr-correction", # r = 32
"philschmid/code-llama-3-1-8b-text-to-sql-lora", # r = 256 "philschmid/code-llama-3-1-8b-text-to-sql-lora", # r = 256
], ],
initial_adapters=["Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16"], initial_adapters=["nvidia/llama-3.1-nemoguard-8b-topic-control"],
op_sequence=[ op_sequence=[
Operation( Operation(
type=OperationType.FORWARD, type=OperationType.FORWARD,
data=create_batch_data( data=create_batch_data("nvidia/llama-3.1-nemoguard-8b-topic-control"),
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16"
),
), ),
Operation( Operation(
type=OperationType.FORWARD, type=OperationType.FORWARD,
@@ -500,7 +485,7 @@ MAX_LORA_RANK_TESTS = [
data=create_batch_data( data=create_batch_data(
[ [
"pbevan11/llama-3.1-8b-ocr-correction", "pbevan11/llama-3.1-8b-ocr-correction",
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", "nvidia/llama-3.1-nemoguard-8b-topic-control",
None, None,
] ]
), ),
@@ -522,7 +507,7 @@ MAX_LORA_RANK_TESTS = [
data=create_batch_data( data=create_batch_data(
[ [
"pbevan11/llama-3.1-8b-ocr-correction", "pbevan11/llama-3.1-8b-ocr-correction",
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", "nvidia/llama-3.1-nemoguard-8b-topic-control",
None, None,
] ]
), ),
@@ -534,7 +519,7 @@ MAX_LORA_RANK_TESTS = [
base="meta-llama/Llama-3.1-8B-Instruct", base="meta-llama/Llama-3.1-8B-Instruct",
max_loras_per_batch=3, max_loras_per_batch=3,
all_adapters=[ all_adapters=[
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", # r = 4 "nvidia/llama-3.1-nemoguard-8b-topic-control", # r = 4
"pbevan11/llama-3.1-8b-ocr-correction", # r = 32 "pbevan11/llama-3.1-8b-ocr-correction", # r = 32
"philschmid/code-llama-3-1-8b-text-to-sql-lora", # r = 256 "philschmid/code-llama-3-1-8b-text-to-sql-lora", # r = 256
], ],
@@ -556,19 +541,19 @@ MAX_LORA_RANK_TESTS = [
), ),
Operation( Operation(
type=OperationType.LOAD, type=OperationType.LOAD,
data="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", data="nvidia/llama-3.1-nemoguard-8b-topic-control",
), ),
Operation( Operation(
type=OperationType.FORWARD, type=OperationType.FORWARD,
data=create_batch_data( data=create_batch_data(
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", "nvidia/llama-3.1-nemoguard-8b-topic-control",
), ),
), ),
Operation( Operation(
type=OperationType.FORWARD, type=OperationType.FORWARD,
data=create_batch_data( data=create_batch_data(
[ [
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", "nvidia/llama-3.1-nemoguard-8b-topic-control",
"pbevan11/llama-3.1-8b-ocr-correction", "pbevan11/llama-3.1-8b-ocr-correction",
None, None,
] ]
@@ -585,14 +570,14 @@ MAX_LOADED_LORAS_TESTS = [
max_loaded_loras=2, max_loaded_loras=2,
all_adapters=[ all_adapters=[
"philschmid/code-llama-3-1-8b-text-to-sql-lora", "philschmid/code-llama-3-1-8b-text-to-sql-lora",
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", "nvidia/llama-3.1-nemoguard-8b-topic-control",
"pbevan11/llama-3.1-8b-ocr-correction", "pbevan11/llama-3.1-8b-ocr-correction",
], ],
initial_adapters=["philschmid/code-llama-3-1-8b-text-to-sql-lora"], initial_adapters=["philschmid/code-llama-3-1-8b-text-to-sql-lora"],
op_sequence=[ op_sequence=[
Operation( Operation(
type=OperationType.LOAD, type=OperationType.LOAD,
data="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", data="nvidia/llama-3.1-nemoguard-8b-topic-control",
), ),
Operation( Operation(
type=OperationType.LOAD, type=OperationType.LOAD,
@@ -606,7 +591,7 @@ MAX_LOADED_LORAS_TESTS = [
type=OperationType.FORWARD, type=OperationType.FORWARD,
data=create_batch_data( data=create_batch_data(
[ [
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", "nvidia/llama-3.1-nemoguard-8b-topic-control",
"philschmid/code-llama-3-1-8b-text-to-sql-lora", "philschmid/code-llama-3-1-8b-text-to-sql-lora",
] ]
), ),
@@ -620,7 +605,7 @@ MAX_LOADED_LORAS_TESTS = [
type=OperationType.FORWARD, type=OperationType.FORWARD,
data=create_batch_data( data=create_batch_data(
[ [
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", "nvidia/llama-3.1-nemoguard-8b-topic-control",
] ]
), ),
), ),
@@ -628,13 +613,13 @@ MAX_LOADED_LORAS_TESTS = [
type=OperationType.LOAD, type=OperationType.LOAD,
data="philschmid/code-llama-3-1-8b-text-to-sql-lora", data="philschmid/code-llama-3-1-8b-text-to-sql-lora",
), ),
# Implicitly load "pbevan11/llama-3.1-8b-ocr-correction" and make sure that "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16" # Implicitly load "pbevan11/llama-3.1-8b-ocr-correction" and make sure that "nvidia/llama-3.1-nemoguard-8b-topic-control"
# isn't implicitly unloaded even though it is LRU because it is needed for this forward pass # isn't implicitly unloaded even though it is LRU because it is needed for this forward pass
Operation( Operation(
type=OperationType.FORWARD, type=OperationType.FORWARD,
data=create_batch_data( data=create_batch_data(
[ [
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", "nvidia/llama-3.1-nemoguard-8b-topic-control",
"pbevan11/llama-3.1-8b-ocr-correction", "pbevan11/llama-3.1-8b-ocr-correction",
] ]
), ),
@@ -644,7 +629,7 @@ MAX_LOADED_LORAS_TESTS = [
), ),
Operation( Operation(
type=OperationType.UNLOAD, type=OperationType.UNLOAD,
data="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", data="nvidia/llama-3.1-nemoguard-8b-topic-control",
), ),
Operation( Operation(
type=OperationType.LOAD, type=OperationType.LOAD,
@@ -654,7 +639,7 @@ MAX_LOADED_LORAS_TESTS = [
type=OperationType.FORWARD, type=OperationType.FORWARD,
data=create_batch_data( data=create_batch_data(
[ [
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", "nvidia/llama-3.1-nemoguard-8b-topic-control",
"philschmid/code-llama-3-1-8b-text-to-sql-lora", "philschmid/code-llama-3-1-8b-text-to-sql-lora",
] ]
), ),
@@ -672,7 +657,7 @@ MAX_LOADED_LORAS_TESTS = [
max_loaded_loras=2, max_loaded_loras=2,
all_adapters=[ all_adapters=[
"philschmid/code-llama-3-1-8b-text-to-sql-lora", "philschmid/code-llama-3-1-8b-text-to-sql-lora",
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", "nvidia/llama-3.1-nemoguard-8b-topic-control",
"pbevan11/llama-3.1-8b-ocr-correction", "pbevan11/llama-3.1-8b-ocr-correction",
], ],
initial_adapters=[ initial_adapters=[
@@ -685,22 +670,22 @@ MAX_LOADED_LORAS_TESTS = [
op_sequence=[ op_sequence=[
Operation( Operation(
type=OperationType.LOAD, type=OperationType.LOAD,
data="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", data="nvidia/llama-3.1-nemoguard-8b-topic-control",
), ),
Operation( Operation(
type=OperationType.LOAD, type=OperationType.LOAD,
data="pbevan11/llama-3.1-8b-ocr-correction", data="pbevan11/llama-3.1-8b-ocr-correction",
expected_implicit_evictions={ expected_implicit_evictions={
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16" "nvidia/llama-3.1-nemoguard-8b-topic-control"
}, },
), ),
# Implicitly load "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16" # Implicitly load "nvidia/llama-3.1-nemoguard-8b-topic-control"
Operation( Operation(
type=OperationType.FORWARD, type=OperationType.FORWARD,
data=create_batch_data( data=create_batch_data(
[ [
"philschmid/code-llama-3-1-8b-text-to-sql-lora", "philschmid/code-llama-3-1-8b-text-to-sql-lora",
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", "nvidia/llama-3.1-nemoguard-8b-topic-control",
] ]
), ),
expected_implicit_evictions={"pbevan11/llama-3.1-8b-ocr-correction"}, expected_implicit_evictions={"pbevan11/llama-3.1-8b-ocr-correction"},
@@ -730,7 +715,7 @@ MAX_LOADED_LORAS_TESTS = [
type=OperationType.FORWARD, type=OperationType.FORWARD,
data=create_batch_data( data=create_batch_data(
[ [
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", "nvidia/llama-3.1-nemoguard-8b-topic-control",
"pbevan11/llama-3.1-8b-ocr-correction", "pbevan11/llama-3.1-8b-ocr-correction",
] ]
), ),
@@ -745,7 +730,7 @@ EVICTION_TESTS = [
max_loras_per_batch=2, max_loras_per_batch=2,
all_adapters=[ all_adapters=[
"lora1=philschmid/code-llama-3-1-8b-text-to-sql-lora", "lora1=philschmid/code-llama-3-1-8b-text-to-sql-lora",
"lora2=Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", "lora2=nvidia/llama-3.1-nemoguard-8b-topic-control",
"lora3=pbevan11/llama-3.1-8b-ocr-correction", "lora3=pbevan11/llama-3.1-8b-ocr-correction",
], ],
enable_lora=True, enable_lora=True,
@@ -764,7 +749,7 @@ EVICTION_TESTS = [
type=OperationType.LOAD, type=OperationType.LOAD,
data={ data={
"lora_name": "lora2", "lora_name": "lora2",
"lora_path": "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", "lora_path": "nvidia/llama-3.1-nemoguard-8b-topic-control",
"pinned": True, "pinned": True,
}, },
expected_error="starvation", expected_error="starvation",
@@ -773,7 +758,7 @@ EVICTION_TESTS = [
type=OperationType.LOAD, type=OperationType.LOAD,
data={ data={
"lora_name": "lora2", "lora_name": "lora2",
"lora_path": "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", "lora_path": "nvidia/llama-3.1-nemoguard-8b-topic-control",
"pinned": False, "pinned": False,
}, },
), ),
@@ -1492,7 +1477,7 @@ class TestLoRADynamicUpdate(CustomTestCase):
""" """
adapters = [ adapters = [
"philschmid/code-llama-3-1-8b-text-to-sql-lora", "philschmid/code-llama-3-1-8b-text-to-sql-lora",
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", "nvidia/llama-3.1-nemoguard-8b-topic-control",
] ]
with LoRAUpdateTestSession( with LoRAUpdateTestSession(
test/registered/perf/test_bench_serving_1gpu_part1.py
+2 -8
View File
@@ -141,9 +141,6 @@ class TestBenchServing1GPUPart1(CustomTestCase):
self.assertLess(res["median_ttft_ms"], 86) self.assertLess(res["median_ttft_ms"], 86)
self.assertLess(res["median_itl_ms"], 10) self.assertLess(res["median_itl_ms"], 10)
@unittest.skip(
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16 removed from HuggingFace"
)
def test_lora_online_latency(self): def test_lora_online_latency(self):
if is_in_amd_ci(): if is_in_amd_ci():
pass pass
@@ -159,9 +156,6 @@ class TestBenchServing1GPUPart1(CustomTestCase):
self.assertLess(res["median_e2e_latency_ms"], 2400) self.assertLess(res["median_e2e_latency_ms"], 2400)
self.assertLess(res["median_ttft_ms"], 58) self.assertLess(res["median_ttft_ms"], 58)
@unittest.skip(
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16 removed from HuggingFace"
)
def test_lora_online_latency_with_concurrent_adapter_updates(self): def test_lora_online_latency_with_concurrent_adapter_updates(self):
if is_in_amd_ci(): if is_in_amd_ci():
pass pass
@@ -247,14 +241,14 @@ class TestBenchServing1GPUPart1(CustomTestCase):
"--mem-fraction-static", "--mem-fraction-static",
"0.8", "0.8",
"--lora-paths", "--lora-paths",
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", "nvidia/llama-3.1-nemoguard-8b-topic-control",
"--max-lora-rank", "--max-lora-rank",
"256", "256",
], ],
dataset_name="random", dataset_name="random",
random_input_len=256, random_input_len=256,
random_output_len=256, random_output_len=256,
lora_name=["Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16"], lora_name=["nvidia/llama-3.1-nemoguard-8b-topic-control"],
background_task=background_task, background_task=background_task,
) )