minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 13 Packages Projects Releases Wiki Activity

[Config] Retire get_global_server_args, and clear the deprecated flags that have a replacement (#38375)

Browse Source
This commit is contained in:
Cheng Wan
2026-09-08 16:42:12 -07:00
committed by GitHub
parent 5177a3ec08
commit db272201a2
213 changed files with 625 additions and 892 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/manual/4-gpu-models/test_qwen35_fp4_triton.py
+1 -1
View File
@@ -21,7 +21,7 @@ class TestQwen35FP4(CustomTestCase):
"4",
"--chunked-prefill-size",
"2048",
"--mamba-scheduler-strategy",
"--mamba-radix-cache-strategy",
"extra_buffer",
"--mamba-track-interval",
"128",
test/manual/4-gpu-models/test_qwen35_models_archived.py
+2 -2
View File
@@ -37,7 +37,7 @@ class TestQwen35FP4(CustomTestCase):
"4",
"--chunked-prefill-size",
"2048",
"--mamba-scheduler-strategy",
"--mamba-radix-cache-strategy",
"extra_buffer",
"--mamba-track-interval",
"128",
@@ -103,7 +103,7 @@ class TestQwen35FP4MTP(ReasoningTokenUsageMixin, CustomTestCase):
"4",
"--chunked-prefill-size",
"2048",
"--mamba-scheduler-strategy",
"--mamba-radix-cache-strategy",
"extra_buffer",
"--mamba-track-interval",
"128",
test/manual/4-gpu-models/test_qwen3_next_models.py
+2 -2
View File
@@ -20,7 +20,7 @@ class TestQwen3Next(
"4",
"--chunked-prefill-size",
"1024",
"--mamba-scheduler-strategy",
"--mamba-radix-cache-strategy",
"extra_buffer",
"--mamba-track-interval",
"2",
@@ -45,7 +45,7 @@ class TestQwen3NextLazyExtraBuffer(
"4",
"--chunked-prefill-size",
"1024",
"--mamba-scheduler-strategy",
"--mamba-radix-cache-strategy",
"extra_buffer_lazy",
"--mamba-track-interval",
"2",
test/manual/4-gpu-models/test_qwen3_next_models_mtp_archived.py
+1 -1
View File
@@ -34,7 +34,7 @@ class TestQwen3NextMTP(GSM8KMixin, KLDivergenceMixin, DefaultServerBase):
"4",
"--chunked-prefill-size",
"2048",
"--mamba-scheduler-strategy",
"--mamba-radix-cache-strategy",
"no_buffer",
"--disable-radix-cache",
]
test/manual/chunked_prefill/test_e2e_hybrid_swa.py
+1 -1
View File
@@ -14,7 +14,7 @@ class TestChunkedFeatureHybridSWA(ChunkedTestBase):
feature_args = [
"--mem-fraction-static",
"0.70",
"--disable-piecewise-cuda-graph",
"--cuda-graph-backend-prefill=disabled",
]
test/manual/ep/test_eplb.py
-1
View File
@@ -73,7 +73,6 @@ class _BaseTestDynamicEPLB(CustomTestCase):
"--expert-distribution-recorder-buffer-size",
"50",
# TODO pr-chain: enable later
# "--enable-expert-distribution-metrics",
# TODO auto determine these flags
"--expert-distribution-recorder-mode",
"stat",
test/manual/lora/run_spec_lora_matrix.py
+3 -3
View File
@@ -14,7 +14,7 @@ without spec, then with spec) and runs four checks:
2. distinct — each adapter's output differs from base (LoRA really applied)
3. mixed — a batch interleaving every adapter matches the solo outputs
(crossed verify segments serve a request the wrong adapter)
4. eager — a batch wider than --cuda-graph-max-bs still matches
4. eager — a batch wider than --cuda-graph-max-bs-decode still matches
(exercises the non-cuda-graph target-verify path)
It also reports per-adapter accept length from each response's
@@ -99,7 +99,7 @@ CONFIGS = {
common_args=[
"--mem-fraction-static=0.7",
"--max-lora-rank=128",
"--cuda-graph-max-bs=2",
"--cuda-graph-max-bs-decode=2",
],
tp=1,
),
@@ -813,7 +813,7 @@ def main() -> int:
"--wide-batch",
type=int,
default=16,
help="batch size for the eager-path check; keep it above --cuda-graph-max-bs",
help="batch size for the eager-path check; keep it above --cuda-graph-max-bs-decode",
)
parser.add_argument(
"--lora-backend", help="override the config's LoRA kernel backend"
test/manual/models/test_granite_moe_hybrid.py
+1 -1
View File
@@ -24,7 +24,7 @@ class TestGraniteMoeHybridExtraBuffer(
other_args = [
"--mem-fraction-static",
"0.8",
"--mamba-scheduler-strategy",
"--mamba-radix-cache-strategy",
"extra_buffer",
]
test/manual/models/test_nvidia_nemotron_nano_v2.py
+2 -2
View File
@@ -34,7 +34,7 @@ class TestNvidiaNemotronNanoV2BF16ExtraBuffer(
"256",
"--mem-fraction-static",
"0.8",
"--mamba-scheduler-strategy",
"--mamba-radix-cache-strategy",
"extra_buffer",
]
@@ -88,7 +88,7 @@ class TestNvidiaNemotronNanoV2SpeculativeDecodingExtraBuffer(
model = NVIDIA_NEMOTRON_NANO_V2_MODEL
gsm8k_accuracy_thres = 0.87
other_args = SPECULATIVE_DECODING_OTHER_ARGS + [
"--mamba-scheduler-strategy",
"--mamba-radix-cache-strategy",
"extra_buffer",
]
test/manual/nightly/test_vlms_piecewise_cuda_graph.py
+3 -3
View File
@@ -135,11 +135,11 @@ class TestVLMPiecewiseCudaGraph(CustomTestCase):
api_key=self.api_key,
other_args=[
"--trust-remote-code",
"--piecewise-cuda-graph-max-tokens",
"--cuda-graph-max-bs-prefill",
"8192",
"--enforce-piecewise-cuda-graph",
"--cuda-graph-backend-prefill=tc_piecewise",
"--tp=8",
"--piecewise-cuda-graph-compiler=eager",
"--cuda-graph-tc-compiler=eager",
"--disable-radix-cache",
"--log-level",
log_level,
test/manual/nightly/test_vlms_vit_cuda_graph.py
+3 -3
View File
@@ -140,14 +140,14 @@ class TestVLMViTCudaGraph(CustomTestCase):
other_args=[
"--mm-attention-backend",
"fa3",
"--enforce-piecewise-cuda-graph",
"--piecewise-cuda-graph-max-tokens",
"--cuda-graph-backend-prefill=tc_piecewise",
"--cuda-graph-max-bs-prefill",
"8192",
"--chunked-prefill-size",
"8192",
"--disable-radix-cache",
"--disable-overlap-schedule",
"--piecewise-cuda-graph-compiler",
"--cuda-graph-tc-compiler",
"eager",
],
env=process_env,
test/manual/piecewise_cuda_graph/test_piecewise_cuda_graph_support_1_gpu_archived.py
+1 -1
View File
@@ -31,7 +31,7 @@ class TestPiecewiseCudaGraphInternVL25(CustomTestCase):
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--enforce-piecewise-cuda-graph",
"--cuda-graph-backend-prefill=tc_piecewise",
"--disable-radix-cache",
],
)
test/manual/piecewise_cudagraph/test_disaggregation_piecewise_cuda_graph.py
+1 -1
View File
@@ -38,7 +38,7 @@ class TestDisaggregationPiecewiseCudaGraph(PDDisaggregationServerBase):
"prefill",
"--tp",
"1",
"--enforce-piecewise-cuda-graph",
"--cuda-graph-backend-prefill=tc_piecewise",
]
prefill_args += cls.transfer_backend + cls.rdma_devices
cls.process_prefill = popen_launch_pd_server(
test/manual/test_dsa_alias_cli_registry_env.py
-63
View File
@@ -41,12 +41,6 @@ class TestDSAChoicesAndFields(unittest.TestCase):
self.assertIn("tilelang", choices)
self.assertIn("flashinfer_sparse_mla", choices)
def test_nsa_choices_match_dsa_choices(self):
self.assertEqual(
self.actions["--nsa-prefill-backend"].choices,
self.actions["--dsa-prefill-backend"].choices,
)
def test_serverargs_has_dsa_fields(self):
sa = self.ServerArgs
self.assertTrue(hasattr(sa, "dsa_prefill_backend"))
@@ -94,63 +88,6 @@ class TestCLICanonicalFlags(unittest.TestCase):
args = self._parse(["--attention-backend", "dsa"])
self.assertEqual(args.attention_backend, "dsa")
class TestCLIDeprecatedFlags(unittest.TestCase):
"""Deprecated flags write to canonical attributes and emit logger warning."""
def setUp(self):
import logging
from sglang.srt.server_args import ServerArgs
self.parser = argparse.ArgumentParser()
ServerArgs.add_cli_args(self.parser)
# Capture log output to detect deprecation warnings
self.log_records = []
handler = (
logging.handlers_collector(self.log_records)
if hasattr(logging, "handlers_collector")
else None
)
def _parse(self, extra_args):
return self.parser.parse_args(["--model", "dummy"] + extra_args)
def _parse_capture_warnings(self, extra_args):
"""Parse and capture both warnings.warn and logger output."""
import io
import logging
log_stream = io.StringIO()
handler = logging.StreamHandler(log_stream)
handler.setLevel(logging.WARNING)
root = logging.getLogger()
root.addHandler(handler)
try:
args = self._parse(extra_args)
finally:
root.removeHandler(handler)
return args, log_stream.getvalue()
def test_nsa_prefill_backend_deprecated_writes_to_dsa(self):
args, log_output = self._parse_capture_warnings(
["--nsa-prefill-backend", "fa3"]
)
self.assertEqual(args.dsa_prefill_backend, "fa3")
self.assertIn(
"deprecated",
log_output.lower(),
f"Expected deprecation warning in log; got: {log_output!r}",
)
def test_nsa_decode_backend_deprecated_writes_to_dsa(self):
args, log_output = self._parse_capture_warnings(
["--nsa-decode-backend", "tilelang"]
)
self.assertEqual(args.dsa_decode_backend, "tilelang")
self.assertIn("deprecated", log_output.lower())
def test_attention_backend_nsa_still_accepted(self):
"""attention_backend='nsa' still parses without error (registry handles the deprecation)."""
args = self._parse(["--attention-backend", "nsa"])
test/manual/test_w4a8_deepseek_v3.py
+3 -3
View File
@@ -63,7 +63,7 @@ class TestDeepseekV3W4Afp8Mtp(CustomTestCase):
"--trust-remote-code",
"--ep-size",
"8",
"--cuda-graph-bs",
"--cuda-graph-bs-decode",
"256",
"--disable-radix-cache",
"--speculative-algorithm",
@@ -130,7 +130,7 @@ class TestDeepseekV3W4Afp8DeepepNormal(CustomTestCase):
"--trust-remote-code",
"--ep-size",
"8",
"--cuda-graph-bs",
"--cuda-graph-bs-decode",
"256",
"--disable-radix-cache",
"--moe-a2a-backend",
@@ -185,7 +185,7 @@ class TestDeepseekV3W4Afp8DeepepAutoMtp(CustomTestCase):
"--trust-remote-code",
"--ep-size",
"8",
"--cuda-graph-bs",
"--cuda-graph-bs-decode",
"256",
"--disable-radix-cache",
"--moe-a2a-backend",
test/registered/amd/accuracy/mi45x/test_deepseek_v4_flash_eval_mi45x.py
+1 -1
View File
@@ -77,7 +77,7 @@ MI45X_DEEPSEEK_V4_FLASH_MODELS = [
"deepseek-v4",
"--chunked-prefill-size",
"8192",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"256",
"--max-running-requests",
"256",
test/registered/amd/test_deepseek_r1_mxfp4_8gpu.py
+3 -3
View File
@@ -40,10 +40,10 @@ class TestDeepseekR1MXFP4(CustomTestCase):
"131072",
"--model-loader-extra-config",
'{"enable_multithread_load": true}',
"--enforce-piecewise-cuda-graph",
"--piecewise-cuda-graph-compiler",
"--cuda-graph-backend-prefill=tc_piecewise",
"--cuda-graph-tc-compiler",
"eager",
"--piecewise-cuda-graph-max-tokens",
"--cuda-graph-max-bs-prefill",
"8192",
]
cls.process = popen_launch_server(
test/registered/amd/test_deepseek_v4_flash_fp8_tbo.py
+1 -1
View File
@@ -105,7 +105,7 @@ class TestDeepseekV4FlashFp8Tbo(CustomTestCase):
"fp8_e4m3",
"--max-running-requests",
"512",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"512",
"--page-size",
"256",
test/registered/amd/test_deepseek_v4_pro_fp4_dspark.py
+1 -1
View File
@@ -145,7 +145,7 @@ class TestDeepseekV4DSparkUnifiedKVGSM8K(CustomTestCase):
"fp8_e4m3",
"--chunked-prefill-size",
"65536",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"512",
"--max-running-requests",
"512",
test/registered/amd/test_deepseek_v4_pro_fp4_tbo.py
+1 -1
View File
@@ -95,7 +95,7 @@ class TestDeepseekV4ProFp4Tbo(CustomTestCase):
"fp8_e4m3",
"--max-running-requests",
"512",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"512",
"--page-size",
"256",
test/registered/amd/test_deepseek_v4_pro_fp4_tbo_mtp.py
+1 -1
View File
@@ -93,7 +93,7 @@ class TestDeepseekV4ProFp4TboMTP(CustomTestCase):
"4",
"--max-running-requests",
"512",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"512",
"--page-size",
"256",
test/registered/attention/test_qwen35_deterministic.py
+1 -1
View File
@@ -30,7 +30,7 @@ class TestQwen35Fa3Deterministic(TestDeterministicBase):
"--attention-backend",
"fa3",
"--skip-server-warmup",
"--mamba-scheduler-strategy",
"--mamba-radix-cache-strategy",
"extra_buffer",
"--enable-flashinfer-allreduce-fusion",
"--tokenizer-worker-num",
test/registered/core/test_basic_sanity_dflash.py
+2 -2
View File
@@ -63,12 +63,12 @@ class TestBasicSanityDFlash(
"DFLASH",
"--speculative-draft-model-path",
DEFAULT_DRAFT_MODEL_DFLASH,
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"4",
"--mem-fraction-static",
"0.7",
"--enable-metrics",
"--disable-piecewise-cuda-graph",
"--cuda-graph-backend-prefill=disabled",
],
env={"SGLANG_ENABLE_METRICS_DEVICE_TIMER": "1"},
)
test/registered/core/test_basic_sanity_dspark.py
+1 -1
View File
@@ -81,7 +81,7 @@ class TestBasicSanityDSpark(
"--page-size",
"1",
"--enable-metrics",
"--disable-piecewise-cuda-graph",
"--cuda-graph-backend-prefill=disabled",
],
env={
"SGLANG_ENABLE_METRICS_DEVICE_TIMER": "1",
test/registered/core/test_basic_sanity_eagle3.py
+1 -1
View File
@@ -75,7 +75,7 @@ class TestBasicSanityEagle3(
"--mem-fraction-static",
"0.7",
"--enable-metrics",
"--disable-piecewise-cuda-graph",
"--cuda-graph-backend-prefill=disabled",
],
env={"SGLANG_ENABLE_METRICS_DEVICE_TIMER": "1"},
)
test/registered/cp/test_dsa_prefill_cp.py
+1 -1
View File
@@ -42,7 +42,7 @@ class TestDSACPInterleave(CustomTestCase):
"4",
"--mem-frac",
"0.85",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"32",
"--max-running-requests",
"32",
test/registered/cp/test_gqa_prefill_cp.py
+3 -3
View File
@@ -44,7 +44,7 @@ class TestGQACP2TP2EP2(CustomTestCase):
"--max-running-requests",
"32",
"--trust-remote-code",
"--disable-piecewise-cuda-graph",
"--cuda-graph-backend-prefill=disabled",
"--model-loader-extra-config",
'{"enable_multithread_load": true, "num_threads": 64}',
],
@@ -102,7 +102,7 @@ class TestGQACPTP2CP2EP4(CustomTestCase):
"--max-running-requests",
"32",
"--trust-remote-code",
"--disable-piecewise-cuda-graph",
"--cuda-graph-backend-prefill=disabled",
"--model-loader-extra-config",
'{"enable_multithread_load": true, "num_threads": 64}',
],
@@ -162,7 +162,7 @@ class TestGQACPCP4EP4(CustomTestCase):
"--max-running-requests",
"32",
"--trust-remote-code",
"--disable-piecewise-cuda-graph",
"--cuda-graph-backend-prefill=disabled",
"--model-loader-extra-config",
'{"enable_multithread_load": true, "num_threads": 64}',
],
test/registered/cpu/test_cpu_graph.py
+2 -2
View File
@@ -34,7 +34,7 @@ class TestCPUGraph(CustomTestCase):
"--enable-torch-compile",
"--torch-compile-max-bs",
"2",
"--cuda-graph-bs",
"--cuda-graph-bs-decode",
"2",
],
min_throughput=7,
@@ -62,7 +62,7 @@ class TestCPUGraph(CustomTestCase):
"--trust-remote-code",
"--disable-overlap-schedule",
"--enable-torch-compile",
"--cuda-graph-bs",
"--cuda-graph-bs-decode",
"2",
"--tp",
f"{n_numa_node}",
test/registered/cuda_graph/breakable/test_breakable_cuda_graph.py
+2 -2
View File
@@ -4,7 +4,7 @@ Two test classes:
- TestBreakableCUDAGraphBasic / TestCopyOutput / TestBreakGraphHelper:
unit tests for the core capture / replay mechanism (simple tensor ops).
- TestBreakableCudaGraph: integration test — spin up Qwen3-8B with
--enable-breakable-cuda-graph and check mgsm_en accuracy.
--cuda-graph-backend-prefill=breakable and check mgsm_en accuracy.
"""
import unittest
@@ -364,7 +364,7 @@ class TestBreakGraphHelper(CustomTestCase):
class TestBreakableCudaGraph(CustomTestCase):
"""Integration: Qwen3-8B with --enable-breakable-cuda-graph on mgsm_en."""
"""Integration: Qwen3-8B with --cuda-graph-backend-prefill=breakable on mgsm_en."""
@classmethod
def setUpClass(cls):
test/registered/cuda_graph/piecewise/test_pcg_with_speculative_decoding_extra.py
+1 -1
View File
@@ -21,7 +21,7 @@ class TestPCGWithMTP(PCGSpecBase, unittest.TestCase):
"--trust-remote-code",
"--quantization",
"fp8",
"--mamba-scheduler-strategy",
"--mamba-radix-cache-strategy",
"extra_buffer",
"--speculative-algorithm",
"NEXTN",
test/registered/dcp/test_dsv31_dcp8_gsm8k.py
+1 -1
View File
@@ -77,7 +77,7 @@ _COMMON_SERVER_ARGS = [
"256",
"--attention-backend",
"flashinfer",
"--disable-piecewise-cuda-graph",
"--cuda-graph-backend-prefill=disabled",
"--log-level",
"info",
"--log-requests",
test/registered/debug_utils/test_nightly_precision_regression.py
+1 -1
View File
@@ -608,7 +608,7 @@ def _run_server_and_dump(
"--mem-fraction-static",
"0.9",
"--disable-cuda-graph",
"--disable-piecewise-cuda-graph",
"--cuda-graph-backend-prefill=disabled",
"--disable-radix-cache",
"--flashinfer-allreduce-fusion-backend",
PRECISION_FUSION_BACKEND,
test/registered/disaggregation/test_disaggregation_hybrid_attention.py
+4 -4
View File
@@ -114,7 +114,7 @@ class TestDisaggregationHybridAttentionGDNExtraBuffer(PDDisaggregationServerBase
cls.bootstrap_port,
"--tp",
"4",
"--mamba-scheduler-strategy",
"--mamba-radix-cache-strategy",
"extra_buffer",
]
prefill_args += cls.transfer_backend + cls.rdma_devices
@@ -137,7 +137,7 @@ class TestDisaggregationHybridAttentionGDNExtraBuffer(PDDisaggregationServerBase
"4",
"--base-gpu-id",
"4",
"--mamba-scheduler-strategy",
"--mamba-radix-cache-strategy",
"extra_buffer",
]
decode_args += cls.transfer_backend + cls.rdma_devices
@@ -342,7 +342,7 @@ class TestDisaggregationHybridAttentionMambaExtraBuffer(PDDisaggregationServerBa
cls.bootstrap_port,
"--tp",
"4",
"--mamba-scheduler-strategy",
"--mamba-radix-cache-strategy",
"extra_buffer",
]
prefill_args += cls.transfer_backend + cls.rdma_devices
@@ -365,7 +365,7 @@ class TestDisaggregationHybridAttentionMambaExtraBuffer(PDDisaggregationServerBa
"4",
"--base-gpu-id",
"4",
"--mamba-scheduler-strategy",
"--mamba-radix-cache-strategy",
"extra_buffer",
]
decode_args += cls.transfer_backend + cls.rdma_devices
test/registered/dllm/test_dllm_batching_fdfo.py
+1 -1
View File
@@ -41,7 +41,7 @@ class TestBatchingFDFO(CustomTestCase):
"--dllm-algorithm",
"LowConfidence",
"--dllm-fdfo",
"--cuda-graph-bs",
"--cuda-graph-bs-decode",
"1",
"2",
"3",
test/registered/dllm/test_dllm_fdfo_joint_threshold.py
+1 -1
View File
@@ -44,7 +44,7 @@ class TestBatchingFDFOJointThreshold(CustomTestCase):
"flashinfer",
"--dllm-algorithm",
"JointThreshold",
"--cuda-graph-bs",
"--cuda-graph-bs-decode",
"1",
]
# FDFO is the default; the sync arm must opt out explicitly.
test/registered/e2e/models/test_deepseek_v4_flash_fp4_b200.py
+1 -1
View File
@@ -192,7 +192,7 @@ class TestDSV4FlashFP4BreakableCudaGraphB200(
DEEPEP_CONFIG,
"--chunked-prefill-size",
"4096",
"--piecewise-cuda-graph-max-tokens",
"--cuda-graph-max-bs-prefill",
"1024",
"--mem-fraction-static",
"0.80",
test/registered/e2e/models/test_qwen35_fp4_mtp.py
+1 -1
View File
@@ -24,7 +24,7 @@ MTP_BASE_ARGS = [
"4",
"--chunked-prefill-size",
"2048",
"--mamba-scheduler-strategy",
"--mamba-radix-cache-strategy",
"extra_buffer",
"--mamba-track-interval",
"128",
test/registered/e2e/models/test_qwen3_next_models.py
+1 -1
View File
@@ -16,7 +16,7 @@ _COMMON_ARGS = [
"4",
"--chunked-prefill-size",
"2048",
"--mamba-scheduler-strategy",
"--mamba-radix-cache-strategy",
"extra_buffer_lazy",
"--attention-backend",
"triton",
test/registered/e2e/models/test_qwen3_next_models_extra.py
+1 -1
View File
@@ -15,7 +15,7 @@ _COMMON_ARGS = [
"4",
"--chunked-prefill-size",
"2048",
"--mamba-scheduler-strategy",
"--mamba-radix-cache-strategy",
"extra_buffer_lazy",
"--attention-backend",
"triton",
test/registered/e2e/models/test_qwen3_next_models_mtp.py
+3 -3
View File
@@ -29,7 +29,7 @@ def _mtp_args(*, strategy, steps, topk, draft_tokens, track_interval):
"4",
"--chunked-prefill-size",
"2048",
"--mamba-scheduler-strategy",
"--mamba-radix-cache-strategy",
strategy,
"--mamba-track-interval",
str(track_interval),
@@ -62,7 +62,7 @@ class TestQwen3NextMTPTopk(
"4",
"--chunked-prefill-size",
"2048",
"--mamba-scheduler-strategy",
"--mamba-radix-cache-strategy",
"extra_buffer",
"--mamba-track-interval",
"128",
@@ -94,7 +94,7 @@ class TestQwen3NextMTPV2(GSM8KMixin, KLDivergenceMixin, DefaultServerBase):
"4",
"--chunked-prefill-size",
"2048",
"--mamba-scheduler-strategy",
"--mamba-radix-cache-strategy",
"extra_buffer",
"--mamba-track-interval",
"128",
test/registered/e2e/models_large/test_deepseek_v3_cutedsl_4gpu.py
+1 -1
View File
@@ -109,7 +109,7 @@ class TestDummyWithSBO(CustomTestCase):
"256",
"--chunked-prefill-size",
"1024",
"--cuda-graph-bs",
"--cuda-graph-bs-decode",
"64",
"--tp",
"4",
test/registered/e2e/models_large/test_ling_2_6_flash.py
+1 -1
View File
@@ -26,7 +26,7 @@ class TestLing26Flash(GSM8KMixin, DefaultServerBase):
"--tp-size",
"4",
"--trust-remote-code",
"--mamba-scheduler-strategy",
"--mamba-radix-cache-strategy",
"extra_buffer",
"--mem-fraction-static",
"0.75",
test/registered/e2e/models_large/test_qwen35.py
+1 -1
View File
@@ -37,7 +37,7 @@ class TestQwen35(unittest.TestCase):
"--speculative-num-steps=3",
"--speculative-eagle-topk=1",
"--speculative-num-draft-tokens=4",
"--mamba-scheduler-strategy=extra_buffer",
"--mamba-radix-cache-strategy=extra_buffer",
]
variants = [
test/registered/ep/test_deepep_large.py
+2 -2
View File
@@ -51,7 +51,7 @@ class TestDeepseek(CustomTestCase):
"dynamic",
"--eplb-algorithm",
"deepseek",
"--cuda-graph-bs",
"--cuda-graph-bs-decode",
"256",
"--max-running-requests",
"2048",
@@ -112,7 +112,7 @@ class TestDeepseekMTP(CustomTestCase):
"dynamic",
"--eplb-algorithm",
"deepseek",
"--cuda-graph-bs",
"--cuda-graph-bs-decode",
"64", # TODO: increase it to 128 when TBO is supported in draft_extend
"--max-running-requests",
"512",
test/registered/gb300/test_qwen35_fp8_dp.py
+1 -1
View File
@@ -18,7 +18,7 @@ COMMON_ARGS = [
"--enable-flashinfer-allreduce-fusion",
"--attention-backend=trtllm_mha",
"--mem-fraction-static=0.8",
"--mamba-scheduler-strategy=extra_buffer",
"--mamba-radix-cache-strategy=extra_buffer",
"--enable-multimodal",
"--enable-metrics",
"--nccl-port",
test/registered/gb300/test_qwen35_fp8_tp.py
+1 -1
View File
@@ -18,7 +18,7 @@ COMMON_ARGS = [
"--enable-flashinfer-allreduce-fusion",
"--attention-backend=trtllm_mha",
"--mem-fraction-static=0.8",
"--mamba-scheduler-strategy=extra_buffer",
"--mamba-radix-cache-strategy=extra_buffer",
"--enable-multimodal",
"--enable-metrics",
"--nccl-port",
test/registered/hicache/test_qwen35_hicache.py
+1 -1
View File
@@ -51,7 +51,7 @@ class TestQwen35WithHiCache(CustomTestCase):
"120000",
"--chunked-prefill-size",
"2048",
"--mamba-scheduler-strategy",
"--mamba-radix-cache-strategy",
"extra_buffer",
"--mamba-track-interval",
"128",
test/registered/kv_canary/test_self_e2e_bench_speed.py
+2 -2
View File
@@ -36,11 +36,11 @@ _PROFILE_NO_GRAPH_STEPS = 3
def _make_server_args(
*, canary_on: bool, disable_cuda_graph: bool = False
) -> ServerArgs:
# install_canary asserts --disable-piecewise-cuda-graph; pass on both sides for apples-to-apples.
# install_canary asserts --cuda-graph-backend-prefill=disabled; pass on both sides for apples-to-apples.
extra = [
"--model-path",
_QWEN3_MODEL,
"--disable-piecewise-cuda-graph",
"--cuda-graph-backend-prefill=disabled",
]
if disable_cuda_graph:
extra.append("--disable-cuda-graph")
test/registered/mock_model/test_e2e_pd.py
+1 -1
View File
@@ -27,7 +27,7 @@ register_amd_ci(est_time=165, stage="extra-a", runner_config="2-gpu-large-amd")
# full graph silently bypasses the only path that exercises that invariant
# end-to-end.
#
# --disable-piecewise-cuda-graph is REQUIRED by canary: install_canary
# --cuda-graph-backend-prefill=disabled is REQUIRED by canary: install_canary
# (api.py) asserts it, and the SingleForwardManager design depends on it.
# mock_model_server_args() already passes it; do not remove it.
_NUM_PROMPTS = 32
test/registered/musa/test_llm_server_smoke_musa.py
+1 -1
View File
@@ -34,7 +34,7 @@ class TestMusaDeepSeekV2LiteChatServerSmoke(DefaultServerBase):
"1",
"--chunked-prefill-size",
"-1",
"--disable-piecewise-cuda-graph",
"--cuda-graph-backend-prefill=disabled",
"--context-length",
"4096",
"--max-total-tokens",
test/registered/npu/accuracy/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_gpqa.py
+1 -1
View File
@@ -93,7 +93,7 @@ DEEPSEEK_V4_FLASH_W8A8_DSPARK_8P_OTHER_ARGS = [
"--speculative-dspark-block-size",
5,
"--skip-server-warmup",
"--cuda-graph-bs",
"--cuda-graph-bs-decode",
1,
2,
4,
test/registered/npu/accuracy/glm4_6v_flash/test_npu_glm4_6v_flash_1p_mmmu.py
+1 -1
View File
@@ -41,7 +41,7 @@ OTHER_ARGS = [
"--trust-remote-code",
"--mem-fraction-static",
0.5,
"--cuda-graph-bs",
"--cuda-graph-bs-decode",
1,
2,
4,
test/registered/npu/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_aime25.py
+1 -1
View File
@@ -40,7 +40,7 @@ OTHER_ARGS = [
"--trust-remote-code",
"--mem-fraction-static",
0.75,
"--cuda-graph-bs",
"--cuda-graph-bs-decode",
1,
2,
4,
test/registered/npu/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_gsm8k.py
+1 -1
View File
@@ -36,7 +36,7 @@ OTHER_ARGS = [
"--trust-remote-code",
"--mem-fraction-static",
0.75,
"--cuda-graph-bs",
"--cuda-graph-bs-decode",
1,
2,
4,
test/registered/npu/accuracy/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26.py
+1 -2
View File
@@ -127,7 +127,7 @@ GLM_5_1_PD_SEP_DECODE_ARGS = [
"deepep",
"--deepep-mode",
"low_latency",
"--cuda-graph-bs",
"--cuda-graph-bs-decode",
1,
2,
3,
@@ -139,7 +139,6 @@ GLM_5_1_PD_SEP_DECODE_ARGS = [
180000,
"--tokenizer-worker-num",
16,
"--prefill-round-robin-balance",
"--disable-shared-experts-fusion",
"--dtype",
"bfloat16",
test/registered/npu/accuracy/glm5_2/test_npu_glm_5_2_w4a8_16p_gpqa.py
+1 -1
View File
@@ -56,7 +56,7 @@ GLM_5_2_W4A8_16P_TWO_NODE_OTHER_ARGS = [
135000,
"--served-model-name",
"glm-5.2-w4a8",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
4,
"--max-running-requests",
32,
test/registered/npu/accuracy/glm5_top64_pruned/test_npu_glm5_top64_pruned_bf16_8p_gsm8k.py
+1 -1
View File
@@ -40,7 +40,7 @@ OTHER_ARGS = [
"deepep",
"--deepep-mode",
"auto",
"--cuda-graph-bs",
"--cuda-graph-bs-decode",
16,
]
test/registered/npu/accuracy/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25.py
+1 -1
View File
@@ -63,7 +63,7 @@ OTHER_ARGS = [
"deepep",
"--deepep-mode",
"auto",
"--cuda-graph-bs",
"--cuda-graph-bs-decode",
1,
"--disable-radix-cache",
"--speculative-algorithm",
test/registered/npu/accuracy/kimi_k3/test_npu_kimi_k3_w4a8_32p_gpqa.py
+1 -1
View File
@@ -70,7 +70,7 @@ KIMI_K3_W4A8_32P_OTHER_ARGS = [
0.72,
"--chunked-prefill-size",
8192,
"--cuda-graph-bs",
"--cuda-graph-bs-decode",
1,
4,
16,
test/registered/npu/accuracy/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms_gpqa.py
+1 -1
View File
@@ -50,7 +50,7 @@ MINIMAX_M2_5_W8A8_4P_IN64K_OUT1K_PREFIX90_OTHER_ARGS = [
67072,
"--max-prefill-token",
67000,
"--cuda-graph-bs",
"--cuda-graph-bs-decode",
2,
4,
8,
test/registered/npu/accuracy/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa.py
+1 -1
View File
@@ -53,7 +53,7 @@ MINIMAX_M2_5_HIGH_THROUGHPUT_OTHER_ARGS = [
196608,
"--max-prefill-token",
8192,
"--cuda-graph-bs",
"--cuda-graph-bs-decode",
1,
2,
4,
test/registered/npu/accuracy/moonshotai_moonlight_16b_a3b/test_npu_moonlight_16b_a3b_bf16_1p_gsm8k.py
+1 -1
View File
@@ -38,7 +38,7 @@ MODEL_OTHER_ARGS = [
4096,
"--max-prefill-tokens",
16384,
"--cuda-graph-bs",
"--cuda-graph-bs-decode",
1,
2,
4,
test/registered/npu/accuracy/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py
+1 -1
View File
@@ -49,7 +49,7 @@ QWEN3_8B_OTHER_ARGS = [
1,
"--mem-fraction-static",
0.85,
"--cuda-graph-bs",
"--cuda-graph-bs-decode",
8,
12,
24,
test/registered/npu/accuracy/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in6k_out1k5_bs16_gpqa.py
+1 -1
View File
@@ -47,7 +47,7 @@ QWEN3_8B_OTHER_ARGS = [
2,
"--mem-fraction-static",
0.894,
"--cuda-graph-bs",
"--cuda-graph-bs-decode",
1,
5,
15,
test/registered/npu/accuracy/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms_aime25.py
+1 -1
View File
@@ -63,7 +63,7 @@ QWEN3_30B_A3B_OTHER_ARGS = [
2,
"--mem-fraction-static",
0.87,
"--cuda-graph-bs",
"--cuda-graph-bs-decode",
1,
5,
15,
test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_bf16_8p_gpqa.py
+1 -1
View File
@@ -59,7 +59,7 @@ QWEN3_32B_OTHER_ARGS = [
16,
"--mem-fraction-static",
0.72,
"--cuda-graph-bs",
"--cuda-graph-bs-decode",
64,
"--dtype",
"bfloat16",
test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa.py
+1 -1
View File
@@ -62,7 +62,7 @@ QWEN3_32B_OTHER_ARGS = [
4,
"--mem-fraction-static",
0.845,
"--cuda-graph-bs",
"--cuda-graph-bs-decode",
16,
32,
64,
test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa_a2.py
+1 -1
View File
@@ -62,7 +62,7 @@ QWEN3_32B_OTHER_ARGS = [
4,
"--mem-fraction-static",
0.845,
"--cuda-graph-bs",
"--cuda-graph-bs-decode",
16,
32,
64,
test/registered/npu/accuracy/qwen3_5_9b/test_npu_qwen3_5_9b_bf16_1p_gsm8k.py
+1 -1
View File
@@ -39,7 +39,7 @@ QWEN3_5_9B_OTHER_ARGS = [
"--trust-remote-code",
"--mem-fraction-static",
0.7,
"--cuda-graph-bs",
"--cuda-graph-bs-decode",
16,
"--enable-multimodal",
"--mm-attention-backend",
test/registered/npu/accuracy/qwen3_6_27b/test_npu_qwen3_6_27b_1p_gpqa.py
+2 -2
View File
@@ -43,7 +43,7 @@ QWEN3_6_27B_64K_PREFIX_OTHER_ARGS = [
32768,
"--max-prefill-tokens",
32768,
"--mamba-scheduler-strategy",
"--mamba-radix-cache-strategy",
"extra_buffer",
"--trust-remote-code",
"--max-running-requests",
@@ -52,7 +52,7 @@ QWEN3_6_27B_64K_PREFIX_OTHER_ARGS = [
120,
"--mem-fraction-static",
0.8,
"--cuda-graph-bs",
"--cuda-graph-bs-decode",
1,
2,
4,
test/registered/npu/accuracy/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py
+1 -1
View File
@@ -48,7 +48,7 @@ QWEN3_6_27B_3K5_1K5_OTHER_ARGS = [
74,
"--mem-fraction-static",
0.7,
"--cuda-graph-bs",
"--cuda-graph-bs-decode",
2,
8,
16,
test/registered/npu/accuracy/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_aime26.py
+1 -1
View File
@@ -45,7 +45,7 @@ QWEN3_6_35B_A3B_3K5_1K5_OTHER_ARGS = [
4,
"--mem-fraction-static",
0.7,
"--cuda-graph-bs",
"--cuda-graph-bs-decode",
1,
2,
3,
test/registered/npu/accuracy/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms_aime26.py
+2 -2
View File
@@ -43,7 +43,7 @@ QWEN3_6_35B_A3B_64K_PREFIX_OTHER_ARGS = [
"--max-prefill-tokens",
65536,
"--trust-remote-code",
"--mamba-scheduler-strategy",
"--mamba-radix-cache-strategy",
"extra_buffer",
"--max-running-requests",
40,
@@ -51,7 +51,7 @@ QWEN3_6_35B_A3B_64K_PREFIX_OTHER_ARGS = [
200,
"--mem-fraction-static",
0.9,
"--cuda-graph-bs",
"--cuda-graph-bs-decode",
2,
8,
16,
test/registered/npu/accuracy/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25.py
+1 -1
View File
@@ -80,7 +80,7 @@ QWEN3_NEXT_80B_A3B_OTHER_ARGS = [
-1,
"--max-running-requests",
16,
"--cuda-graph-bs",
"--cuda-graph-bs-decode",
2,
4,
8,
test/registered/npu/accuracy/qwen3_omni_30b_a3b_thinking/test_npu_qwen3_omni_30b_a3b_thinking_1p_mmmu.py
+1 -1
View File
@@ -63,7 +63,7 @@ OTHER_ARGS = [
2,
"--mem-fraction-static",
0.87,
"--cuda-graph-bs",
"--cuda-graph-bs-decode",
1,
5,
15,
test/registered/npu/accuracy/qwen3_vl_8b/test_npu_qwen3_vl_8b_bf16_2p_gsm8k.py
+1 -1
View File
@@ -40,7 +40,7 @@ QWEN3_VL_8B_OTHER_ARGS = [
"qwen",
"--reasoning-parser",
"qwen3",
"--cuda-graph-bs",
"--cuda-graph-bs-decode",
8,
16,
32,
test/registered/npu/basic_function/optimization_debug/test_npu_piecewise_graph_prefill.py
+1 -1
View File
@@ -27,7 +27,7 @@ class TestPiecewiseGraphPrefillCorrectness(GSM8KAscendMixin, CustomTestCase):
0.8,
"--attention-backend",
"ascend",
"--cuda-graph-bs",
"--cuda-graph-bs-decode",
128,
"--cuda-graph-backend-prefill=tc_piecewise",
"--cuda-graph-bs-prefill",
test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_qwen3_480b.py
+1 -1
View File
@@ -57,7 +57,7 @@ class TestDeepEpQwen(GSM8KAscendMixin, TestMMLU, CustomTestCase):
"--enable-dp-lm-head",
"--mem-fraction-static",
0.7,
"--cuda-graph-bs",
"--cuda-graph-bs-decode",
16,
20,
24,
test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_qwen3_next.py
+1 -1
View File
@@ -41,7 +41,7 @@ class TestQwen3Next(GSM8KAscendMixin, TestMMLU, CustomTestCase):
"--watchdog-timeout",
9000,
"--disable-radix-cache",
"--cuda-graph-bs",
"--cuda-graph-bs-decode",
2,
4,
6,
test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_qwen3_480b.py
+1 -1
View File
@@ -58,7 +58,7 @@ class TestDeepEpQwen(GSM8KAscendMixin, TestMMLU, CustomTestCase):
"--enable-dp-lm-head",
"--mem-fraction-static",
0.7,
"--cuda-graph-bs",
"--cuda-graph-bs-decode",
16,
20,
24,
test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_qwen3_next.py
+1 -1
View File
@@ -41,7 +41,7 @@ class TestQwen3Next(GSM8KAscendMixin, TestMMLU, CustomTestCase):
"--watchdog-timeout",
9000,
"--disable-radix-cache",
"--cuda-graph-bs",
"--cuda-graph-bs-decode",
2,
4,
6,
test/registered/npu/basic_function/quant/test_npu_w4a4_quantization.py
+1 -1
View File
@@ -29,7 +29,7 @@ class TestAscendW4A4(GSM8KAscendMixin, CustomTestCase):
"4",
"--mem-fraction-static",
"0.8",
"--cuda-graph-bs",
"--cuda-graph-bs-decode",
"64",
"--disable-radix-cache",
]
test/registered/npu/basic_function/speculative_inference/test_npu_basic_sanity_eagle3.py
+2 -2
View File
@@ -56,11 +56,11 @@ class TestBasicSanityEagle3(
"1",
"--speculative-num-draft-tokens",
"2",
"--cuda-graph-max-bs",
"--cuda-graph-max-bs-decode",
"4",
"--mem-fraction-static",
"0.7",
"--disable-piecewise-cuda-graph",
"--cuda-graph-backend-prefill=disabled",
],
)
test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_1p1d_16p_in8k_out1k_50ms.py
+1 -1
View File
@@ -171,7 +171,7 @@ DEEPSEEK_V4_FLASH_W8A8_1P1D_DECODE_ARGS = [
"--enable-dp-lm-head",
"--kv-cache-dtype",
"bfloat16",
"--cuda-graph-bs",
"--cuda-graph-bs-decode",
1,
2,
4,
test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in32k_out1k_50ms.py
+1 -1
View File
@@ -80,7 +80,7 @@ DEEPSEEK_V4_FLASH_W8A8_8P_OTHER_ARGS = [
"--kv-cache-dtype",
"bfloat16",
"--skip-server-warmup",
"--cuda-graph-bs",
"--cuda-graph-bs-decode",
1,
2,
4,
test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py
+1 -1
View File
@@ -79,7 +79,7 @@ DEEPSEEK_V4_FLASH_W8A8_8P_OTHER_ARGS = [
"--kv-cache-dtype",
"bfloat16",
"--skip-server-warmup",
"--cuda-graph-bs",
"--cuda-graph-bs-decode",
1,
2,
4,
test/registered/npu/performance/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms.py
+1 -2
View File
@@ -127,7 +127,7 @@ GLM_5_1_PD_SEP_DECODE_ARGS = [
"deepep",
"--deepep-mode",
"low_latency",
"--cuda-graph-bs",
"--cuda-graph-bs-decode",
1,
2,
3,
@@ -139,7 +139,6 @@ GLM_5_1_PD_SEP_DECODE_ARGS = [
180000,
"--tokenizer-worker-num",
16,
"--prefill-round-robin-balance",
"--disable-shared-experts-fusion",
"--dtype",
"bfloat16",
test/registered/npu/performance/qwen3_5_397b/test_npu_qwen3_5_397b_w4a8_8p_in3k5_out1k5_50ms.py
+1 -1
View File
@@ -53,7 +53,7 @@ QWEN3_5_397B_A17B_3K5_1K5_OTHER_ARGS = [
432,
"--mem-fraction-static",
0.8,
"--cuda-graph-bs",
"--cuda-graph-bs-decode",
2,
4,
6,
test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1024x1024_30_out1024_50ms.py
+1 -1
View File
@@ -49,7 +49,7 @@ QWEN3_6_27B_1024_OTHER_ARGS = [
60,
"--mem-fraction-static",
0.74,
"--cuda-graph-bs",
"--cuda-graph-bs-decode",
2,
4,
8,
test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1080p_30_out256_50ms.py
+1 -1
View File
@@ -49,7 +49,7 @@ QWEN3_6_27B_1080P_OTHER_ARGS = [
38,
"--mem-fraction-static",
0.70,
"--cuda-graph-bs",
"--cuda-graph-bs-decode",
1,
2,
4,
test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_2p_in64k_out1k_prefix90_50ms.py
+2 -2
View File
@@ -38,7 +38,7 @@ QWEN3_6_27B_64K_PREFIX_OTHER_ARGS = [
32768,
"--max-prefill-tokens",
32768,
"--mamba-scheduler-strategy",
"--mamba-radix-cache-strategy",
"extra_buffer",
"--trust-remote-code",
"--max-running-requests",
@@ -47,7 +47,7 @@ QWEN3_6_27B_64K_PREFIX_OTHER_ARGS = [
160,
"--mem-fraction-static",
0.82,
"--cuda-graph-bs",
"--cuda-graph-bs-decode",
1,
2,
5,
test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms.py
+1 -1
View File
@@ -48,7 +48,7 @@ QWEN3_6_27B_3K5_1K5_OTHER_ARGS = [
74,
"--mem-fraction-static",
0.7,
"--cuda-graph-bs",
"--cuda-graph-bs-decode",
2,
8,
16,
test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in64k_out1k_50ms.py
+1 -1
View File
@@ -41,7 +41,7 @@ QWEN3_6_27B_64K_1K_OTHER_ARGS = [
16,
"--mem-fraction-static",
0.6,
"--cuda-graph-bs",
"--cuda-graph-bs-decode",
1,
2,
4,
test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in128k_out1k_50ms.py
+1 -1
View File
@@ -48,7 +48,7 @@ QWEN3_6_27B_128K_OTHER_ARGS = [
7,
"--mem-fraction-static",
0.63,
"--cuda-graph-bs",
"--cuda-graph-bs-decode",
1,
2,
4,
test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in16k_out1k_50ms.py
+1 -1
View File
@@ -49,7 +49,7 @@ QWEN3_6_27B_16K_1k_OTHER_ARGS = [
74,
"--mem-fraction-static",
0.70,
"--cuda-graph-bs",
"--cuda-graph-bs-decode",
1,
2,
3,
test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_50ms.py
+1 -1
View File
@@ -51,7 +51,7 @@ QWEN3_6_35B_A3B_128K_1K_OTHER_ARGS = [
3,
"--mem-fraction-static",
0.9,
"--cuda-graph-bs",
"--cuda-graph-bs-decode",
1,
2,
3,
test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_prefix90_50ms.py
+2 -2
View File
@@ -40,7 +40,7 @@ QWEN3_6_35B_A3B_128K_PREFIX_OTHER_ARGS = [
65536,
"--trust-remote-code",
"--enable-prefill-delayer",
"--mamba-scheduler-strategy",
"--mamba-radix-cache-strategy",
"extra_buffer",
"--max-running-requests",
103,
@@ -48,7 +48,7 @@ QWEN3_6_35B_A3B_128K_PREFIX_OTHER_ARGS = [
85,
"--mem-fraction-static",
0.85,
"--cuda-graph-bs",
"--cuda-graph-bs-decode",
2,
4,
8,
test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in3k5_out1k5_50ms.py
+1 -1
View File
@@ -47,7 +47,7 @@ QWEN3_6_35B_A3B_3K5_1K5_OTHER_ARGS = [
122,
"--mem-fraction-static",
0.9,
"--cuda-graph-bs",
"--cuda-graph-bs-decode",
4,
16,
32,
test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_50ms.py
+1 -1
View File
@@ -46,7 +46,7 @@ QWEN3_6_35B_A3B_64K_1K_OTHER_ARGS = [
20,
"--mem-fraction-static",
0.65,
"--cuda-graph-bs",
"--cuda-graph-bs-decode",
2,
4,
8,
test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms.py
+2 -2
View File
@@ -40,7 +40,7 @@ QWEN3_6_35B_A3B_64K_PREFIX_OTHER_ARGS = [
"--max-prefill-tokens",
65536,
"--trust-remote-code",
"--mamba-scheduler-strategy",
"--mamba-radix-cache-strategy",
"extra_buffer",
"--max-running-requests",
40,
@@ -48,7 +48,7 @@ QWEN3_6_35B_A3B_64K_PREFIX_OTHER_ARGS = [
200,
"--mem-fraction-static",
0.9,
"--cuda-graph-bs",
"--cuda-graph-bs-decode",
2,
8,
16,

Some files were not shown because too many files have changed in this diff Show More