[Config] Round 5.1: the published-side readers ask the bags, and a platform fact gets one address (#37086)
This commit is contained in:
@@ -25,6 +25,7 @@ from sglang.srt.model_executor.forward_batch_info import (
|
||||
from sglang.srt.model_executor.runner.prefill_cuda_graph_runner import (
|
||||
PrefillCudaGraphRunner,
|
||||
)
|
||||
from sglang.srt.runtime_context import override_platform
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
@@ -84,16 +85,12 @@ class TestMultimodalPiecewiseCudaGraph(CustomTestCase):
|
||||
)
|
||||
args._cuda_graph_config_locked = set()
|
||||
|
||||
with (
|
||||
patch(
|
||||
"sglang.srt.arg_groups.cuda_graph_hook"
|
||||
".disable_tc_piecewise_cudagraph_if_incompatible"
|
||||
) as disable_if_incompatible,
|
||||
patch(
|
||||
"sglang.srt.arg_groups.overrides.attention_backends_of",
|
||||
return_value=("fa3", "fa3"),
|
||||
),
|
||||
):
|
||||
args.attention_backend = "fa3"
|
||||
|
||||
with patch(
|
||||
"sglang.srt.arg_groups.cuda_graph_hook"
|
||||
".disable_tc_piecewise_cudagraph_if_incompatible"
|
||||
) as disable_if_incompatible:
|
||||
apply_cuda_graph_compatibility(args)
|
||||
|
||||
self.assertEqual(
|
||||
@@ -118,11 +115,9 @@ class TestMultimodalPiecewiseCudaGraph(CustomTestCase):
|
||||
)
|
||||
args._cuda_graph_config_locked = set()
|
||||
|
||||
with patch(
|
||||
"sglang.srt.arg_groups.overrides.attention_backends_of",
|
||||
return_value=("trtllm_mla", "trtllm_mla"),
|
||||
):
|
||||
apply_cuda_graph_compatibility(args)
|
||||
args.attention_backend = "trtllm_mla"
|
||||
|
||||
apply_cuda_graph_compatibility(args)
|
||||
|
||||
self.assertEqual(
|
||||
resolution_result(args, "cuda_graph_config").prefill.backend,
|
||||
@@ -136,11 +131,9 @@ class TestMultimodalPiecewiseCudaGraph(CustomTestCase):
|
||||
)
|
||||
args._cuda_graph_config_locked = {(Phase.PREFILL, "backend")}
|
||||
|
||||
with patch(
|
||||
"sglang.srt.arg_groups.overrides.attention_backends_of",
|
||||
return_value=("trtllm_mla", "trtllm_mla"),
|
||||
):
|
||||
apply_cuda_graph_compatibility(args)
|
||||
args.attention_backend = "trtllm_mla"
|
||||
|
||||
apply_cuda_graph_compatibility(args)
|
||||
|
||||
self.assertEqual(
|
||||
resolution_result(args, "cuda_graph_config").prefill.backend,
|
||||
@@ -183,7 +176,7 @@ class TestMultimodalPiecewiseCudaGraph(CustomTestCase):
|
||||
args.disable_radix_cache = False
|
||||
args.chunked_prefill_size = 2048
|
||||
|
||||
with (patch("sglang.srt.arg_groups.model_hook.is_cuda", return_value=True),):
|
||||
with (override_platform(is_cuda=True),):
|
||||
handle_model_capability_adjustments(args)
|
||||
|
||||
self.assertTrue(resolution_result(args, "disable_radix_cache"))
|
||||
|
||||
@@ -20,6 +20,7 @@ import unittest
|
||||
from concurrent.futures import Future
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from sglang.srt.arg_groups.overrides import resolution_result
|
||||
from sglang.srt.constrained.base_grammar_backend import (
|
||||
GRAMMAR_BACKEND_REGISTRY,
|
||||
BaseGrammarBackend,
|
||||
@@ -355,7 +356,10 @@ class TestCreateGrammarBackend(unittest.TestCase):
|
||||
self.assertEqual(
|
||||
get_context().resolved_server_args_dict()["grammar_backend"], "none"
|
||||
)
|
||||
self.assertEqual(server_args.grammar_backend, "xgrammar")
|
||||
# The record is not written any more: what the caller asked for is a
|
||||
# declaration on it, and the runtime fallback to "none" lives in the bag
|
||||
# (asserted above). The two are meant to differ here.
|
||||
self.assertEqual(resolution_result(server_args, "grammar_backend"), "xgrammar")
|
||||
|
||||
@patch("sglang.srt.constrained.llguidance_backend.GuidanceBackend")
|
||||
def test_llguidance_backend(self, mock_guidance_cls):
|
||||
|
||||
@@ -8,6 +8,7 @@ from sglang.srt.layers.attention.linear.kda_backend import KDAKernelDispatcher
|
||||
from sglang.srt.layers.attention.linear.kernels.kda_helion import HelionKDAKernel
|
||||
from sglang.srt.layers.attention.linear.kernels.kda_triton import TritonKDAKernel
|
||||
from sglang.srt.layers.attention.linear.utils import LinearAttnKernelBackend
|
||||
from sglang.srt.runtime_context import override_platform
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
|
||||
@@ -160,11 +161,8 @@ class TestHelionKDADispatcher(unittest.TestCase):
|
||||
|
||||
def test_replayssm_accepts_helion_and_rejects_other_backends(self):
|
||||
with (
|
||||
patch(
|
||||
"sglang.srt.arg_groups.attention_hook.is_sm100_supported",
|
||||
return_value=False,
|
||||
),
|
||||
patch("sglang.srt.arg_groups.attention_hook.is_cuda", return_value=False),
|
||||
override_platform(is_sm100=False),
|
||||
override_platform(is_cuda=False),
|
||||
):
|
||||
helion_args = ServerArgs(
|
||||
model_path="dummy",
|
||||
@@ -188,11 +186,8 @@ class TestHelionKDADispatcher(unittest.TestCase):
|
||||
mamba_ssm_dtype="bfloat16",
|
||||
)
|
||||
with (
|
||||
patch(
|
||||
"sglang.srt.arg_groups.attention_hook.is_sm100_supported",
|
||||
return_value=True,
|
||||
),
|
||||
patch("sglang.srt.arg_groups.attention_hook.is_cuda", return_value=False),
|
||||
override_platform(is_sm100=True),
|
||||
override_platform(is_cuda=False),
|
||||
):
|
||||
handle_linear_attn_backend(args)
|
||||
|
||||
|
||||
+5
-12
@@ -12,6 +12,7 @@ from sglang.srt.layers.quantization.compressed_tensors.schemes import (
|
||||
CompressedTensorsWNA16MoE,
|
||||
CompressedTensorsWNA16TritonMoE,
|
||||
)
|
||||
from sglang.srt.runtime_context import override_platform
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
@@ -90,9 +91,7 @@ class TestWNA16MoENoLinearGroup(CustomTestCase):
|
||||
"get_moe_runner_backend",
|
||||
return_value=MoeRunnerBackend.AUTO,
|
||||
),
|
||||
mock.patch.object(
|
||||
compressed_tensors, "is_sm100_supported", return_value=True
|
||||
),
|
||||
override_platform(is_sm100=True),
|
||||
):
|
||||
scheme = quant_config.get_moe_scheme(
|
||||
torch.nn.Module(), layer_name=EXPERTS_LAYER
|
||||
@@ -120,9 +119,7 @@ class TestWNA16MoENoLinearGroup(CustomTestCase):
|
||||
"get_moe_runner_backend",
|
||||
return_value=MoeRunnerBackend.AUTO,
|
||||
),
|
||||
mock.patch.object(
|
||||
compressed_tensors, "is_sm100_supported", return_value=True
|
||||
),
|
||||
override_platform(is_sm100=True),
|
||||
):
|
||||
scheme = quant_config.get_moe_scheme(
|
||||
torch.nn.Module(), layer_name=EXPERTS_LAYER
|
||||
@@ -157,9 +154,7 @@ class TestWNA16MoENoLinearGroup(CustomTestCase):
|
||||
"get_moe_runner_backend",
|
||||
return_value=MoeRunnerBackend.MARLIN,
|
||||
),
|
||||
mock.patch.object(
|
||||
compressed_tensors, "is_sm100_supported", return_value=True
|
||||
),
|
||||
override_platform(is_sm100=True),
|
||||
):
|
||||
scheme = quant_config.get_moe_scheme(
|
||||
torch.nn.Module(), layer_name=EXPERTS_LAYER
|
||||
@@ -178,9 +173,7 @@ class TestWNA16MoENoLinearGroup(CustomTestCase):
|
||||
"get_moe_runner_backend",
|
||||
return_value=MoeRunnerBackend.AUTO,
|
||||
),
|
||||
mock.patch.object(
|
||||
compressed_tensors, "is_sm100_supported", return_value=True
|
||||
),
|
||||
override_platform(is_sm100=True),
|
||||
):
|
||||
scheme = quant_config.get_moe_scheme(
|
||||
torch.nn.Module(), layer_name=EXPERTS_LAYER
|
||||
|
||||
@@ -11,6 +11,7 @@ from types import SimpleNamespace
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from sglang.srt.runtime_context import override_platform
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=120, stage="base-b", runner_config="1-gpu-small")
|
||||
@@ -81,10 +82,12 @@ def test_cutlass_adapter_import_does_not_require_flashinfer(monkeypatch):
|
||||
sys.modules[module_name] = cached_module
|
||||
|
||||
|
||||
def test_dsv4_sm120_load_contract(monkeypatch):
|
||||
def test_dsv4_sm120_load_contract(monkeypatch, request):
|
||||
import sglang.srt.layers.quantization.mxfp4_flashinfer_cutlass_moe as adapter_module
|
||||
|
||||
monkeypatch.setattr(adapter_module, "is_sm120_supported", lambda: True)
|
||||
platform = override_platform(is_sm120=True)
|
||||
platform.install()
|
||||
request.addfinalizer(platform.restore)
|
||||
|
||||
captured = {}
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@ from unittest.mock import MagicMock, patch
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers import flashinfer_comm_fusion as fusion
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.runtime_context import get_parallel, override_platform
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
@@ -101,7 +101,7 @@ class TestFlashInferCommFusion(CustomTestCase):
|
||||
multi_node = ("auto", True)
|
||||
|
||||
# Blackwell: mnnvl on both single-node and multi-node.
|
||||
with patch.object(fusion, "is_sm100_supported", return_value=True):
|
||||
with override_platform(is_sm100=True):
|
||||
self.assertEqual(
|
||||
fusion._resolve_backend(*single_node),
|
||||
"mnnvl",
|
||||
@@ -110,8 +110,8 @@ class TestFlashInferCommFusion(CustomTestCase):
|
||||
|
||||
# SM90: auto uses trtllm on single-node, multi-node is unsupported.
|
||||
with (
|
||||
patch.object(fusion, "is_sm100_supported", return_value=False),
|
||||
patch.object(fusion, "is_sm90_supported", return_value=True),
|
||||
override_platform(is_sm100=False),
|
||||
override_platform(is_sm90=True),
|
||||
):
|
||||
self.assertEqual(
|
||||
fusion._resolve_backend(*single_node),
|
||||
@@ -125,8 +125,8 @@ class TestFlashInferCommFusion(CustomTestCase):
|
||||
for arch in ("pre_sm90", "post_sm10x"):
|
||||
with (
|
||||
self.subTest(arch=arch),
|
||||
patch.object(fusion, "is_sm100_supported", return_value=False),
|
||||
patch.object(fusion, "is_sm90_supported", return_value=False),
|
||||
override_platform(is_sm100=False),
|
||||
override_platform(is_sm90=False),
|
||||
):
|
||||
with self.assertRaises(ValueError):
|
||||
fusion._resolve_backend(*single_node)
|
||||
@@ -140,8 +140,8 @@ class TestFlashInferCommFusion(CustomTestCase):
|
||||
multi_node_trtllm = ("trtllm", True)
|
||||
|
||||
with (
|
||||
patch.object(fusion, "is_sm100_supported", return_value=False),
|
||||
patch.object(fusion, "is_sm90_supported", return_value=True),
|
||||
override_platform(is_sm100=False),
|
||||
override_platform(is_sm90=True),
|
||||
):
|
||||
self.assertEqual(
|
||||
fusion._resolve_backend(*single_node_mnnvl),
|
||||
@@ -156,7 +156,7 @@ class TestFlashInferCommFusion(CustomTestCase):
|
||||
with self.assertRaises(ValueError):
|
||||
fusion._resolve_backend(*multi_node_trtllm)
|
||||
|
||||
with patch.object(fusion, "is_sm100_supported", return_value=True):
|
||||
with override_platform(is_sm100=True):
|
||||
self.assertEqual(
|
||||
fusion._resolve_backend(*multi_node_mnnvl),
|
||||
"mnnvl",
|
||||
@@ -167,8 +167,8 @@ class TestFlashInferCommFusion(CustomTestCase):
|
||||
for arch in ("pre_sm90", "post_sm10x"):
|
||||
with (
|
||||
self.subTest(arch=arch),
|
||||
patch.object(fusion, "is_sm100_supported", return_value=False),
|
||||
patch.object(fusion, "is_sm90_supported", return_value=False),
|
||||
override_platform(is_sm100=False),
|
||||
override_platform(is_sm90=False),
|
||||
):
|
||||
for args in (
|
||||
single_node_mnnvl,
|
||||
|
||||
@@ -6,6 +6,7 @@ from unittest.mock import MagicMock, Mock, patch
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.runtime_context import override_platform
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
@@ -96,7 +97,7 @@ def _construct_sparse_backend(
|
||||
with (
|
||||
get_schedule().override(chunked_prefill_size=chunked_prefill_size),
|
||||
patch.object(backend_module, "MiniCPMHybridConfig", SimpleNamespace),
|
||||
patch.object(backend_module, "is_blackwell_supported", return_value=blackwell),
|
||||
override_platform(is_blackwell=blackwell),
|
||||
patch.object(
|
||||
backend_module,
|
||||
"FlashAttentionBackend",
|
||||
@@ -292,7 +293,7 @@ class TestMiniCPMSparseMetadata(CustomTestCase):
|
||||
{"sglang.srt.layers.attention.minicpm.fuse_kernel": fake_fuse_kernel},
|
||||
),
|
||||
patch.object(backend_module, "MiniCPMHybridConfig", SimpleNamespace),
|
||||
patch.object(backend_module, "is_blackwell_supported", return_value=True),
|
||||
override_platform(is_blackwell=True),
|
||||
patch.object(
|
||||
backend_module,
|
||||
"FlashAttentionBackend",
|
||||
@@ -318,7 +319,7 @@ class TestMiniCPMSparseMetadata(CustomTestCase):
|
||||
model_config.num_attention_heads = 8
|
||||
with (
|
||||
patch.object(backend_module, "MiniCPMHybridConfig", SimpleNamespace),
|
||||
patch.object(backend_module, "is_blackwell_supported", return_value=True),
|
||||
override_platform(is_blackwell=True),
|
||||
patch.object(
|
||||
backend_module,
|
||||
"FlashAttentionBackend",
|
||||
@@ -337,7 +338,7 @@ class TestMiniCPMSparseMetadata(CustomTestCase):
|
||||
model_runner.server_args.attention_backend = "minicpm_flashattn"
|
||||
with (
|
||||
patch.object(backend_module, "MiniCPMHybridConfig", SimpleNamespace),
|
||||
patch.object(backend_module, "is_blackwell_supported", return_value=False),
|
||||
override_platform(is_blackwell=False),
|
||||
patch.object(
|
||||
backend_module,
|
||||
"FlashAttentionBackend",
|
||||
@@ -397,7 +398,7 @@ class TestMiniCPMSparseMetadata(CustomTestCase):
|
||||
with (
|
||||
backend_module.envs.SGLANG_MINICPM_DENSE_AS_SPARSE.override(True),
|
||||
patch.object(backend_module, "MiniCPMHybridConfig", SimpleNamespace),
|
||||
patch.object(backend_module, "is_blackwell_supported", return_value=False),
|
||||
override_platform(is_blackwell=False),
|
||||
patch.object(
|
||||
backend_module,
|
||||
"FlashAttentionBackend",
|
||||
|
||||
@@ -403,8 +403,12 @@ class TestStreamOrderedMmFeaturePool(CustomTestCase):
|
||||
def test_consumer_slot_uses_global_tp_rank(self):
|
||||
from sglang.srt.multimodal.transport.memory_pool import resolve_consumer_rank
|
||||
|
||||
parallel = SimpleNamespace(tp_rank=6, attn_tp_rank=2)
|
||||
with patch("sglang.srt.runtime_context.get_parallel", return_value=parallel):
|
||||
# State the topology on the context, not by stubbing the accessor:
|
||||
# `memory_pool` imports `get_parallel` at module scope, so a patch on
|
||||
# the defining module never reaches the copy doing the reading.
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
with get_parallel().override(tp_rank=6, attn_tp_rank=2):
|
||||
self.assertEqual(resolve_consumer_rank(8), 6)
|
||||
|
||||
def test_complete_group_acknowledges_each_consumer_slot(self):
|
||||
|
||||
@@ -13,6 +13,7 @@ from sglang.srt.mem_cache.base_prefix_cache import (
|
||||
DecLockRefResult,
|
||||
IncLockRefResult,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_context
|
||||
from sglang.srt.server_args import ServerArgs, set_global_server_args_for_scheduler
|
||||
from sglang.srt.utils.common import Range
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
@@ -79,14 +80,14 @@ class TestPrefillAdder(CustomTestCase):
|
||||
batch.filter_batch.return_value = None
|
||||
return batch
|
||||
|
||||
def create_server_args(
|
||||
self, *, schedule_low_priority_values_first: bool
|
||||
) -> MagicMock:
|
||||
server_args = MagicMock()
|
||||
server_args.schedule_low_priority_values_first = (
|
||||
schedule_low_priority_values_first
|
||||
def scheduling_order(self, *, schedule_low_priority_values_first: bool):
|
||||
"""State the policy on the context, which is where the scheduler reads
|
||||
it: `preempt_to_schedule` takes no record to state it on."""
|
||||
override = get_context().override_server_args(
|
||||
schedule_low_priority_values_first=schedule_low_priority_values_first
|
||||
)
|
||||
return server_args
|
||||
override.install()
|
||||
self.addCleanup(override.restore)
|
||||
|
||||
def create_mock_req(self, rid, priority, max_new_tokens, output_len=0, wait_time=0):
|
||||
req = MagicMock(spec=Req)
|
||||
@@ -129,9 +130,7 @@ class TestPrefillAdder(CustomTestCase):
|
||||
self.create_mock_req(rid, priority, max_new_tokens)
|
||||
for rid, priority, max_new_tokens in params
|
||||
]
|
||||
mock_server_args = self.create_server_args(
|
||||
schedule_low_priority_values_first=False
|
||||
)
|
||||
self.scheduling_order(schedule_low_priority_values_first=False)
|
||||
running_batch = self.create_running_batch(running_reqs)
|
||||
adder = self.create_adder(running_batch)
|
||||
|
||||
@@ -144,7 +143,7 @@ class TestPrefillAdder(CustomTestCase):
|
||||
|
||||
new_req = self.create_mock_req("new1", priority=1, max_new_tokens=49)
|
||||
|
||||
success = adder.preempt_to_schedule(new_req, mock_server_args)
|
||||
success = adder.preempt_to_schedule(new_req)
|
||||
|
||||
self.assertTrue(success)
|
||||
self.assertIn(running_reqs[0], adder.preempt_list)
|
||||
@@ -161,9 +160,7 @@ class TestPrefillAdder(CustomTestCase):
|
||||
self.create_mock_req(rid, priority, max_new_tokens)
|
||||
for rid, priority, max_new_tokens in params
|
||||
]
|
||||
mock_server_args = self.create_server_args(
|
||||
schedule_low_priority_values_first=True
|
||||
)
|
||||
self.scheduling_order(schedule_low_priority_values_first=True)
|
||||
running_batch = self.create_running_batch(running_reqs)
|
||||
adder = self.create_adder(running_batch)
|
||||
|
||||
@@ -176,7 +173,7 @@ class TestPrefillAdder(CustomTestCase):
|
||||
|
||||
new_req = self.create_mock_req("new1", priority=1, max_new_tokens=49)
|
||||
|
||||
success = adder.preempt_to_schedule(new_req, mock_server_args)
|
||||
success = adder.preempt_to_schedule(new_req)
|
||||
|
||||
self.assertTrue(success)
|
||||
self.assertIn(running_reqs[2], adder.preempt_list)
|
||||
@@ -193,9 +190,7 @@ class TestPrefillAdder(CustomTestCase):
|
||||
self.create_mock_req(rid, priority, max_new_tokens)
|
||||
for rid, priority, max_new_tokens in params
|
||||
]
|
||||
mock_server_args = self.create_server_args(
|
||||
schedule_low_priority_values_first=True
|
||||
)
|
||||
self.scheduling_order(schedule_low_priority_values_first=True)
|
||||
running_batch = self.create_running_batch(running_reqs)
|
||||
adder = self.create_adder(running_batch)
|
||||
|
||||
@@ -211,7 +206,7 @@ class TestPrefillAdder(CustomTestCase):
|
||||
)
|
||||
|
||||
success_by_priority_check = adder.preempt_to_schedule(
|
||||
new_req_fail_by_priority_check, mock_server_args
|
||||
new_req_fail_by_priority_check
|
||||
)
|
||||
self.assertFalse(success_by_priority_check)
|
||||
|
||||
@@ -219,7 +214,7 @@ class TestPrefillAdder(CustomTestCase):
|
||||
"new2", priority=1, max_new_tokens=110
|
||||
)
|
||||
success_by_capacity_check = adder.preempt_to_schedule(
|
||||
new_req_fail_by_priority_check, mock_server_args
|
||||
new_req_fail_by_priority_check
|
||||
)
|
||||
self.assertFalse(success_by_capacity_check)
|
||||
|
||||
@@ -233,9 +228,7 @@ class TestPrefillAdder(CustomTestCase):
|
||||
self.create_mock_req(rid, priority, max_new_tokens)
|
||||
for rid, priority, max_new_tokens in params
|
||||
]
|
||||
mock_server_args = self.create_server_args(
|
||||
schedule_low_priority_values_first=False
|
||||
)
|
||||
self.scheduling_order(schedule_low_priority_values_first=False)
|
||||
running_batch = self.create_running_batch(running_reqs)
|
||||
adder = self.create_adder(running_batch)
|
||||
|
||||
@@ -251,7 +244,7 @@ class TestPrefillAdder(CustomTestCase):
|
||||
)
|
||||
|
||||
success_by_priority_check = adder.preempt_to_schedule(
|
||||
new_req_fail_by_priority_check, mock_server_args
|
||||
new_req_fail_by_priority_check
|
||||
)
|
||||
self.assertFalse(success_by_priority_check)
|
||||
|
||||
@@ -259,7 +252,7 @@ class TestPrefillAdder(CustomTestCase):
|
||||
"new2", priority=-1, max_new_tokens=110
|
||||
)
|
||||
success_by_capacity_check = adder.preempt_to_schedule(
|
||||
new_req_fail_by_priority_check, mock_server_args
|
||||
new_req_fail_by_priority_check
|
||||
)
|
||||
self.assertFalse(success_by_capacity_check)
|
||||
|
||||
@@ -273,9 +266,7 @@ class TestPrefillAdder(CustomTestCase):
|
||||
self.create_mock_req(rid, priority, max_new_tokens)
|
||||
for rid, priority, max_new_tokens in params
|
||||
]
|
||||
mock_server_args = self.create_server_args(
|
||||
schedule_low_priority_values_first=False
|
||||
)
|
||||
self.scheduling_order(schedule_low_priority_values_first=False)
|
||||
running_batch = self.create_running_batch(running_reqs)
|
||||
adder = self.create_adder(running_batch)
|
||||
|
||||
@@ -288,7 +279,7 @@ class TestPrefillAdder(CustomTestCase):
|
||||
first_req = self.create_mock_req(
|
||||
"new_req_prio_1", priority=1, max_new_tokens=49
|
||||
)
|
||||
first_success = adder.preempt_to_schedule(first_req, mock_server_args)
|
||||
first_success = adder.preempt_to_schedule(first_req)
|
||||
self.assertTrue(first_success)
|
||||
self.assertIn(running_reqs[0], adder.preempt_list)
|
||||
self.assertEqual(adder.rem_total_token_offset, 175)
|
||||
@@ -299,7 +290,7 @@ class TestPrefillAdder(CustomTestCase):
|
||||
second_req = self.create_mock_req(
|
||||
"second_new_req_prio_1", priority=1, max_new_tokens=76
|
||||
)
|
||||
second_success = adder.preempt_to_schedule(second_req, mock_server_args)
|
||||
second_success = adder.preempt_to_schedule(second_req)
|
||||
|
||||
self.assertFalse(second_success)
|
||||
self.assertEqual(adder.rem_total_token_offset, 175)
|
||||
@@ -318,9 +309,7 @@ class TestPrefillAdder(CustomTestCase):
|
||||
self.create_mock_req(rid, priority, max_new_tokens)
|
||||
for rid, priority, max_new_tokens in params
|
||||
]
|
||||
mock_server_args = self.create_server_args(
|
||||
schedule_low_priority_values_first=True
|
||||
)
|
||||
self.scheduling_order(schedule_low_priority_values_first=True)
|
||||
running_batch = self.create_running_batch(running_reqs)
|
||||
adder = self.create_adder(running_batch)
|
||||
|
||||
@@ -333,7 +322,7 @@ class TestPrefillAdder(CustomTestCase):
|
||||
|
||||
new_req = self.create_mock_req("new1", priority=1, max_new_tokens=75)
|
||||
|
||||
success = adder.preempt_to_schedule(new_req, mock_server_args)
|
||||
success = adder.preempt_to_schedule(new_req)
|
||||
self.assertTrue(success)
|
||||
self.assertIn(running_reqs[2], adder.preempt_list)
|
||||
self.assertEqual(
|
||||
@@ -353,9 +342,7 @@ class TestPrefillAdder(CustomTestCase):
|
||||
self.create_mock_req(rid, priority, max_new_tokens)
|
||||
for rid, priority, max_new_tokens in params
|
||||
]
|
||||
mock_server_args = self.create_server_args(
|
||||
schedule_low_priority_values_first=True
|
||||
)
|
||||
self.scheduling_order(schedule_low_priority_values_first=True)
|
||||
running_batch = self.create_running_batch(running_reqs)
|
||||
adder = self.create_adder(running_batch)
|
||||
|
||||
@@ -368,7 +355,7 @@ class TestPrefillAdder(CustomTestCase):
|
||||
|
||||
new_req = self.create_mock_req("new1", priority=1, max_new_tokens=200)
|
||||
|
||||
success = adder.preempt_to_schedule(new_req, mock_server_args)
|
||||
success = adder.preempt_to_schedule(new_req)
|
||||
self.assertTrue(success)
|
||||
self.assertIn(running_reqs[2], adder.preempt_list)
|
||||
self.assertIn(running_reqs[3], adder.preempt_list)
|
||||
|
||||
@@ -9,6 +9,7 @@ and the published instance stays as the launcher left it.
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
from sglang.srt.arg_groups.overrides import resolution_result
|
||||
from sglang.srt.managers.io_struct import (
|
||||
AttachHiCacheStorageReqInput,
|
||||
DetachHiCacheStorageReqInput,
|
||||
@@ -69,7 +70,12 @@ class TestSchedulerHiCacheAttach(CustomTestCase):
|
||||
self.assertIsNone(
|
||||
get_context().resolved_server_args_dict()["hicache_storage_backend"]
|
||||
)
|
||||
self.assertEqual(self.server_args.hicache_storage_backend, "file")
|
||||
# The record is not written any more: the attach is a declaration on
|
||||
# it and the detach is a bag override (asserted above), so the two are
|
||||
# meant to differ here.
|
||||
self.assertEqual(
|
||||
resolution_result(self.server_args, "hicache_storage_backend"), "file"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -3,6 +3,7 @@ from unittest import mock
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.models.deepseek_common.amd import deepseek_v4_fused_mhc
|
||||
from sglang.srt.runtime_context import override_platform
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
|
||||
register_cpu_ci(est_time=4, suite="base-a-test-cpu")
|
||||
@@ -19,8 +20,8 @@ class TestAmdFusedMhcCrossLayerGating(unittest.TestCase):
|
||||
):
|
||||
self.assertTrue(deepseek_v4_fused_mhc.is_cross_layer_mhc_fusion_enabled())
|
||||
|
||||
@mock.patch.object(deepseek_v4_fused_mhc, "is_sm120_supported", return_value=True)
|
||||
def test_sm120_enables_fusion_with_tilelang_pre_disabled(self, _mock_sm120):
|
||||
@override_platform(is_sm120=True)
|
||||
def test_sm120_enables_fusion_with_tilelang_pre_disabled(self):
|
||||
# Regression (PR review): consolidating _is_fused_mhc_post_pre_enabled into
|
||||
# this module must preserve the SM120 special case. SM120 disables the
|
||||
# standalone TileLang pre path, but mhc_fused_post_pre dispatches
|
||||
@@ -35,8 +36,8 @@ class TestAmdFusedMhcCrossLayerGating(unittest.TestCase):
|
||||
):
|
||||
self.assertTrue(deepseek_v4_fused_mhc._is_fused_mhc_post_pre_enabled())
|
||||
|
||||
@mock.patch.object(deepseek_v4_fused_mhc, "is_sm120_supported", return_value=False)
|
||||
def test_no_sm120_still_requires_tilelang_pre(self, _mock_sm120):
|
||||
@override_platform(is_sm120=False)
|
||||
def test_no_sm120_still_requires_tilelang_pre(self):
|
||||
# Negative branch: the (pre OR sm120) clause must not degrade to
|
||||
# always-true. With SM120 unsupported and the pre flag off, fuse+post
|
||||
# alone must not enable the standalone TileLang fused path.
|
||||
@@ -68,11 +69,7 @@ class TestAmdFusedMhcCrossLayerGating(unittest.TestCase):
|
||||
envs.SGLANG_OPT_FUSE_MHC_POST_PRE.override(fuse),
|
||||
envs.SGLANG_OPT_USE_TILELANG_MHC_PRE.override(pre),
|
||||
envs.SGLANG_OPT_USE_TILELANG_MHC_POST.override(post),
|
||||
mock.patch.object(
|
||||
deepseek_v4_fused_mhc,
|
||||
"is_sm120_supported",
|
||||
return_value=sm120,
|
||||
),
|
||||
override_platform(is_sm120=sm120),
|
||||
):
|
||||
self.assertEqual(
|
||||
deepseek_v4_fused_mhc._is_fused_mhc_post_pre_enabled(),
|
||||
@@ -83,7 +80,6 @@ class TestAmdFusedMhcCrossLayerGating(unittest.TestCase):
|
||||
@mock.patch.object(deepseek_v4_fused_mhc, "get_bool_env_var", return_value=True)
|
||||
@mock.patch.object(deepseek_v4_fused_mhc, "_is_hip", True)
|
||||
def test_aiter_gfx95_enables_cross_layer_fusion(self, _mock_aiter, _mock_gfx95):
|
||||
# TileLang flags off: fusion must still enable via the aiter gfx95 path.
|
||||
with (
|
||||
envs.SGLANG_OPT_FUSE_MHC_POST_PRE.override(False),
|
||||
envs.SGLANG_OPT_USE_TILELANG_MHC_PRE.override(False),
|
||||
|
||||
@@ -55,10 +55,9 @@ def _accessor_names():
|
||||
for node in tree.body
|
||||
if isinstance(node, ast.FunctionDef) and node.name.startswith("get_")
|
||||
}
|
||||
# The context object itself is not a bag: it exists before anything is
|
||||
# published, and `declare_late_resolution` calls it deliberately to find
|
||||
# out whether the record it was handed has been published yet.
|
||||
return frozenset(names - {"get_context"})
|
||||
# Two that are not bags: the context object itself, and the platform facts.
|
||||
# Both answer before anything is published.
|
||||
return frozenset(names - {"get_context", "get_platform"})
|
||||
|
||||
|
||||
_BAG_ACCESSORS = _accessor_names()
|
||||
|
||||
@@ -471,7 +471,7 @@ class TestResolutionReadsTheDeclarations(CustomTestCase):
|
||||
members = _record_members()
|
||||
# The floor is here to catch the scan collapsing, not to pin the
|
||||
# class's size.
|
||||
self.assertGreater(len(members), 25, f"only {len(members)} members were found")
|
||||
self.assertGreater(len(members), 15, f"only {len(members)} members were found")
|
||||
offenders = []
|
||||
for name, fn in sorted(members.items()):
|
||||
holders = _holders(fn) | {"self"}
|
||||
|
||||
@@ -73,7 +73,12 @@ from sglang.srt.model_executor.cuda_graph_config import (
|
||||
Phase,
|
||||
PhaseConfig,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_context, get_serving
|
||||
from sglang.srt.runtime_context import (
|
||||
describe_kv_events_publisher,
|
||||
get_context,
|
||||
get_serving,
|
||||
override_platform,
|
||||
)
|
||||
from sglang.srt.server_args import PortArgs, ServerArgs, prepare_server_args
|
||||
from sglang.srt.utils.server_args_config_parser import ConfigArgumentMerger
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
@@ -333,8 +338,8 @@ class TestMultimodalFeatureTransport(CustomTestCase):
|
||||
def _set_model_type(server_args, *, is_multimodal):
|
||||
server_args._model_config = SimpleNamespace(is_multimodal=is_multimodal)
|
||||
|
||||
@patch("sglang.srt.arg_groups.serving_hook.is_cuda", return_value=True)
|
||||
def test_cuda_ipc_is_explicit_and_bounded(self, _mock_is_cuda):
|
||||
@override_platform(is_cuda=True)
|
||||
def test_cuda_ipc_is_explicit_and_bounded(self):
|
||||
server_args = ServerArgs(
|
||||
model_path="dummy",
|
||||
mm_feature_transport="cuda_ipc",
|
||||
@@ -355,8 +360,8 @@ class TestMultimodalFeatureTransport(CustomTestCase):
|
||||
self.assertIn("base GPU 2", output)
|
||||
self.assertIn("4 tokenizer worker", output)
|
||||
|
||||
@patch("sglang.srt.arg_groups.serving_hook.is_cuda", return_value=True)
|
||||
def test_legacy_keep_flag_maps_to_cuda_ipc(self, _mock_is_cuda):
|
||||
@override_platform(is_cuda=True)
|
||||
def test_legacy_keep_flag_maps_to_cuda_ipc(self):
|
||||
server_args = ServerArgs(model_path="dummy", keep_mm_feature_on_device=True)
|
||||
|
||||
with patch.dict(os.environ, {"SGLANG_USE_CUDA_IPC_TRANSPORT": "0"}):
|
||||
@@ -383,8 +388,8 @@ class TestMultimodalFeatureTransport(CustomTestCase):
|
||||
with self.assertRaisesRegex(ValueError, "conflicts.*cuda_vmm"):
|
||||
handle_multimodal_feature_transport(server_args)
|
||||
|
||||
@patch("sglang.srt.arg_groups.serving_hook.is_cuda", return_value=True)
|
||||
def test_explicit_cpu_overrides_legacy_environment(self, _mock_is_cuda):
|
||||
@override_platform(is_cuda=True)
|
||||
def test_explicit_cpu_overrides_legacy_environment(self):
|
||||
server_args = ServerArgs(model_path="dummy", mm_feature_transport="cpu")
|
||||
|
||||
with patch.dict(os.environ, {"SGLANG_USE_CUDA_IPC_TRANSPORT": "1"}):
|
||||
@@ -409,8 +414,8 @@ class TestMultimodalFeatureTransport(CustomTestCase):
|
||||
)
|
||||
self.assertFalse(envs.SGLANG_USE_CUDA_IPC_TRANSPORT.get())
|
||||
|
||||
@patch("sglang.srt.arg_groups.serving_hook.is_cuda", return_value=True)
|
||||
def test_default_transport_is_cpu_for_text_only_model(self, _mock_is_cuda):
|
||||
@override_platform(is_cuda=True)
|
||||
def test_default_transport_is_cpu_for_text_only_model(self):
|
||||
server_args = ServerArgs(model_path="dummy")
|
||||
self._set_model_type(server_args, is_multimodal=False)
|
||||
|
||||
@@ -424,8 +429,8 @@ class TestMultimodalFeatureTransport(CustomTestCase):
|
||||
)
|
||||
self.assertFalse(envs.SGLANG_USE_CUDA_IPC_TRANSPORT.get())
|
||||
|
||||
@patch("sglang.srt.arg_groups.serving_hook.is_cuda", return_value=True)
|
||||
def test_default_transport_is_cpu_for_multimodal_model(self, _mock_is_cuda):
|
||||
@override_platform(is_cuda=True)
|
||||
def test_default_transport_is_cpu_for_multimodal_model(self):
|
||||
server_args = ServerArgs(model_path="dummy")
|
||||
self._set_model_type(server_args, is_multimodal=True)
|
||||
|
||||
@@ -443,17 +448,13 @@ class TestMultimodalFeatureTransport(CustomTestCase):
|
||||
@patch(
|
||||
"sglang.srt.arg_groups.serving_hook.is_mnnvl_fabric_device", return_value=True
|
||||
)
|
||||
@patch("sglang.srt.arg_groups.serving_hook.is_cuda", return_value=True)
|
||||
@override_platform(is_cuda=True)
|
||||
@patch(
|
||||
"sglang.srt.model_loader.utils.supports_cuda_vmm_feature_transport",
|
||||
return_value=True,
|
||||
)
|
||||
def test_default_transport_is_cuda_vmm_for_supported_multinode_mnnvl(
|
||||
self,
|
||||
_mock_supports_cuda_vmm,
|
||||
_mock_is_cuda,
|
||||
_mock_is_mnnvl,
|
||||
_mock_path_exists,
|
||||
self, _mock_supports_cuda_vmm, _mock_is_cuda, _mock_is_mnnvl
|
||||
):
|
||||
server_args = ServerArgs(model_path="dummy", nnodes=2)
|
||||
self._set_model_type(server_args, is_multimodal=True)
|
||||
@@ -476,17 +477,13 @@ class TestMultimodalFeatureTransport(CustomTestCase):
|
||||
@patch(
|
||||
"sglang.srt.arg_groups.serving_hook.is_mnnvl_fabric_device", return_value=True
|
||||
)
|
||||
@patch("sglang.srt.arg_groups.serving_hook.is_cuda", return_value=True)
|
||||
@override_platform(is_cuda=True)
|
||||
@patch(
|
||||
"sglang.srt.model_loader.utils.supports_cuda_vmm_feature_transport",
|
||||
return_value=False,
|
||||
)
|
||||
def test_default_transport_is_cpu_for_unsupported_multinode_model(
|
||||
self,
|
||||
_mock_supports_cuda_vmm,
|
||||
_mock_is_cuda,
|
||||
_mock_is_mnnvl,
|
||||
_mock_path_exists,
|
||||
self, _mock_supports_cuda_vmm, _mock_is_cuda, _mock_is_mnnvl
|
||||
):
|
||||
server_args = ServerArgs(model_path="dummy", nnodes=2)
|
||||
self._set_model_type(server_args, is_multimodal=True)
|
||||
@@ -501,9 +498,9 @@ class TestMultimodalFeatureTransport(CustomTestCase):
|
||||
@patch(
|
||||
"sglang.srt.arg_groups.serving_hook.is_mnnvl_fabric_device", return_value=True
|
||||
)
|
||||
@patch("sglang.srt.arg_groups.serving_hook.is_cuda", return_value=True)
|
||||
@override_platform(is_cuda=True)
|
||||
def test_default_transport_is_cpu_without_imex_channel(
|
||||
self, _mock_is_cuda, _mock_is_mnnvl, _mock_path_exists
|
||||
self, _mock_is_cuda, _mock_is_mnnvl
|
||||
):
|
||||
server_args = ServerArgs(model_path="dummy", nnodes=2)
|
||||
self._set_model_type(server_args, is_multimodal=True)
|
||||
@@ -522,10 +519,8 @@ class TestMultimodalFeatureTransport(CustomTestCase):
|
||||
@patch(
|
||||
"sglang.srt.arg_groups.serving_hook.is_mnnvl_fabric_device", return_value=False
|
||||
)
|
||||
@patch("sglang.srt.arg_groups.serving_hook.is_cuda", return_value=True)
|
||||
def test_default_transport_is_cpu_for_multinode_non_mnnvl(
|
||||
self, _mock_is_cuda, _mock_is_mnnvl
|
||||
):
|
||||
@override_platform(is_cuda=True)
|
||||
def test_default_transport_is_cpu_for_multinode_non_mnnvl(self, _mock_is_cuda):
|
||||
server_args = ServerArgs(model_path="dummy", nnodes=2)
|
||||
self._set_model_type(server_args, is_multimodal=True)
|
||||
|
||||
@@ -538,8 +533,8 @@ class TestMultimodalFeatureTransport(CustomTestCase):
|
||||
)
|
||||
self.assertFalse(envs.SGLANG_USE_CUDA_IPC_TRANSPORT.get())
|
||||
|
||||
@patch("sglang.srt.arg_groups.serving_hook.is_cuda", return_value=True)
|
||||
def test_default_transport_is_cpu_for_language_only_model(self, _mock_is_cuda):
|
||||
@override_platform(is_cuda=True)
|
||||
def test_default_transport_is_cpu_for_language_only_model(self):
|
||||
server_args = ServerArgs(model_path="dummy", language_only=True)
|
||||
self._set_model_type(server_args, is_multimodal=True)
|
||||
|
||||
@@ -552,15 +547,15 @@ class TestMultimodalFeatureTransport(CustomTestCase):
|
||||
)
|
||||
self.assertFalse(envs.SGLANG_USE_CUDA_IPC_TRANSPORT.get())
|
||||
|
||||
@patch("sglang.srt.arg_groups.serving_hook.is_cuda", return_value=False)
|
||||
def test_cuda_ipc_rejects_non_nvidia_platforms(self, _mock_is_cuda):
|
||||
@override_platform(is_cuda=False)
|
||||
def test_cuda_ipc_rejects_non_nvidia_platforms(self):
|
||||
server_args = ServerArgs(model_path="dummy", mm_feature_transport="cuda_ipc")
|
||||
|
||||
with self.assertRaisesRegex(ValueError, "requires NVIDIA CUDA"):
|
||||
handle_multimodal_feature_transport(server_args)
|
||||
|
||||
@patch("sglang.srt.arg_groups.serving_hook.is_cuda", return_value=True)
|
||||
def test_cuda_ipc_rejects_multi_node(self, _mock_is_cuda):
|
||||
@override_platform(is_cuda=True)
|
||||
def test_cuda_ipc_rejects_multi_node(self):
|
||||
server_args = ServerArgs(
|
||||
model_path="dummy", mm_feature_transport="cuda_ipc", nnodes=2
|
||||
)
|
||||
@@ -568,8 +563,8 @@ class TestMultimodalFeatureTransport(CustomTestCase):
|
||||
with self.assertRaisesRegex(ValueError, "single node"):
|
||||
handle_multimodal_feature_transport(server_args)
|
||||
|
||||
@patch("sglang.srt.arg_groups.serving_hook.is_cuda", return_value=True)
|
||||
def test_cuda_vmm_is_explicit_and_uses_shared_budget(self, _mock_is_cuda):
|
||||
@override_platform(is_cuda=True)
|
||||
def test_cuda_vmm_is_explicit_and_uses_shared_budget(self):
|
||||
server_args = ServerArgs(
|
||||
model_path="dummy",
|
||||
mm_feature_transport="cuda_vmm",
|
||||
@@ -595,15 +590,15 @@ class TestMultimodalFeatureTransport(CustomTestCase):
|
||||
self.assertIn("2 tokenizer worker", output)
|
||||
self.assertIn("falls back to inline CPU", output)
|
||||
|
||||
@patch("sglang.srt.arg_groups.serving_hook.is_cuda", return_value=False)
|
||||
def test_cuda_vmm_rejects_non_nvidia_platforms(self, _mock_is_cuda):
|
||||
@override_platform(is_cuda=False)
|
||||
def test_cuda_vmm_rejects_non_nvidia_platforms(self):
|
||||
server_args = ServerArgs(model_path="dummy", mm_feature_transport="cuda_vmm")
|
||||
|
||||
with self.assertRaisesRegex(ValueError, "requires NVIDIA CUDA"):
|
||||
handle_multimodal_feature_transport(server_args)
|
||||
|
||||
@patch("sglang.srt.arg_groups.serving_hook.is_cuda", return_value=True)
|
||||
def test_cuda_vmm_rejects_rust_server(self, _mock_is_cuda):
|
||||
@override_platform(is_cuda=True)
|
||||
def test_cuda_vmm_rejects_rust_server(self):
|
||||
server_args = ServerArgs(model_path="dummy", mm_feature_transport="cuda_vmm")
|
||||
|
||||
with (
|
||||
@@ -612,8 +607,8 @@ class TestMultimodalFeatureTransport(CustomTestCase):
|
||||
):
|
||||
handle_multimodal_feature_transport(server_args)
|
||||
|
||||
@patch("sglang.srt.arg_groups.serving_hook.is_cuda", return_value=True)
|
||||
def test_cuda_vmm_rejects_pipeline_parallelism(self, _mock_is_cuda):
|
||||
@override_platform(is_cuda=True)
|
||||
def test_cuda_vmm_rejects_pipeline_parallelism(self):
|
||||
server_args = ServerArgs(
|
||||
model_path="dummy", mm_feature_transport="cuda_vmm", pp_size=2
|
||||
)
|
||||
@@ -633,8 +628,8 @@ class TestMambaCacheStochasticRounding(unittest.TestCase):
|
||||
with self.assertRaisesRegex(ValueError, "--mamba-ssm-dtype float16"):
|
||||
handle_mamba_backend(server_args)
|
||||
|
||||
@patch("sglang.srt.arg_groups.mamba_hook.is_cuda", return_value=False)
|
||||
def test_rejects_non_cuda(self, _mock_is_cuda):
|
||||
@override_platform(is_cuda=False)
|
||||
def test_rejects_non_cuda(self):
|
||||
server_args = ServerArgs(
|
||||
model_path="dummy",
|
||||
mamba_ssm_dtype="float16",
|
||||
@@ -644,9 +639,9 @@ class TestMambaCacheStochasticRounding(unittest.TestCase):
|
||||
with self.assertRaisesRegex(ValueError, "NVIDIA CUDA"):
|
||||
handle_mamba_backend(server_args)
|
||||
|
||||
@patch("sglang.srt.arg_groups.mamba_hook.is_cuda", return_value=True)
|
||||
@patch("sglang.srt.arg_groups.mamba_hook.is_sm100_supported", return_value=False)
|
||||
def test_rejects_triton_without_sm100(self, _mock_sm100, _mock_is_cuda):
|
||||
@override_platform(is_cuda=True)
|
||||
@override_platform(is_sm100=False)
|
||||
def test_rejects_triton_without_sm100(self):
|
||||
server_args = ServerArgs(
|
||||
model_path="dummy",
|
||||
mamba_ssm_dtype="float16",
|
||||
@@ -832,8 +827,8 @@ class TestHiSparseDsaBackendPolicy(unittest.TestCase):
|
||||
)
|
||||
with (
|
||||
patch("sglang.srt.configs.model_config.is_deepseek_dsa", return_value=True),
|
||||
patch("sglang.srt.arg_groups.overrides.is_npu", return_value=False),
|
||||
patch("sglang.srt.arg_groups.overrides.is_xpu", return_value=False),
|
||||
override_platform(is_npu=False),
|
||||
override_platform(is_xpu=False),
|
||||
patch("torch.cuda.get_device_capability", return_value=(9, 0)),
|
||||
):
|
||||
declared = _dsa_split_backend_resolution(view)
|
||||
@@ -846,22 +841,22 @@ class TestHiSparseDsaBackendPolicy(unittest.TestCase):
|
||||
),
|
||||
}
|
||||
|
||||
@patch("sglang.srt.arg_groups.hisparse_hook._is_hip", return_value=False)
|
||||
def test_hisparse_defaults_to_flashmla_sparse_on_cuda_bfloat16(self, _mock_is_hip):
|
||||
@override_platform(is_hip=False)
|
||||
def test_hisparse_defaults_to_flashmla_sparse_on_cuda_bfloat16(self):
|
||||
resolved = self._resolve("bfloat16")
|
||||
|
||||
self.assertEqual(resolved["dsa_prefill_backend"], "flashmla_sparse")
|
||||
self.assertEqual(resolved["dsa_decode_backend"], "flashmla_sparse")
|
||||
|
||||
@patch("sglang.srt.arg_groups.hisparse_hook._is_hip", return_value=False)
|
||||
def test_hisparse_defaults_to_flashmla_kv_on_cuda_fp8(self, _mock_is_hip):
|
||||
@override_platform(is_hip=False)
|
||||
def test_hisparse_defaults_to_flashmla_kv_on_cuda_fp8(self):
|
||||
resolved = self._resolve("fp8_e4m3")
|
||||
|
||||
self.assertEqual(resolved["dsa_prefill_backend"], "flashmla_kv")
|
||||
self.assertEqual(resolved["dsa_decode_backend"], "flashmla_kv")
|
||||
|
||||
@patch("sglang.srt.arg_groups.hisparse_hook._is_hip", return_value=False)
|
||||
def test_hisparse_accepts_flashinfer_sparse_mla_on_cuda_fp8(self, _mock_is_hip):
|
||||
@override_platform(is_hip=False)
|
||||
def test_hisparse_accepts_flashinfer_sparse_mla_on_cuda_fp8(self):
|
||||
"""SM120 GLM DSA resolves both DSA backends to flashinfer_sparse_mla, so
|
||||
the fp8 hisparse allow-set must admit it or --enable-hisparse cannot
|
||||
start there at all. The device/arch narrowing happens later, in
|
||||
@@ -877,24 +872,22 @@ class TestHiSparseDsaBackendPolicy(unittest.TestCase):
|
||||
validate_hisparse_dsa_backend(server_args, "dsa_prefill_backend", "prefill")
|
||||
validate_hisparse_dsa_backend(server_args, "dsa_decode_backend", "decode")
|
||||
|
||||
@patch("sglang.srt.arg_groups.hisparse_hook._is_hip", return_value=True)
|
||||
def test_hisparse_defaults_to_tilelang_on_rocm(self, _mock_is_hip):
|
||||
@override_platform(is_hip=True)
|
||||
def test_hisparse_defaults_to_tilelang_on_rocm(self):
|
||||
resolved = self._resolve("bfloat16")
|
||||
|
||||
self.assertEqual(resolved["dsa_prefill_backend"], "tilelang")
|
||||
self.assertEqual(resolved["dsa_decode_backend"], "tilelang")
|
||||
|
||||
@patch("sglang.srt.arg_groups.hisparse_hook._is_hip", return_value=True)
|
||||
def test_hisparse_preserves_rocm_user_backend_and_defaults_missing_side(
|
||||
self, _mock_is_hip
|
||||
):
|
||||
@override_platform(is_hip=True)
|
||||
def test_hisparse_preserves_rocm_user_backend_and_defaults_missing_side(self):
|
||||
resolved = self._resolve("bfloat16", dsa_prefill_backend="tilelang")
|
||||
|
||||
self.assertEqual(resolved["dsa_prefill_backend"], "tilelang")
|
||||
self.assertEqual(resolved["dsa_decode_backend"], "tilelang")
|
||||
|
||||
@patch("sglang.srt.arg_groups.hisparse_hook._is_hip", return_value=True)
|
||||
def test_hisparse_accepts_aiter_backend_on_rocm(self, _mock_is_hip):
|
||||
@override_platform(is_hip=True)
|
||||
def test_hisparse_accepts_aiter_backend_on_rocm(self):
|
||||
server_args = ServerArgs(
|
||||
model_path="dummy",
|
||||
enable_hisparse=True,
|
||||
@@ -906,8 +899,8 @@ class TestHiSparseDsaBackendPolicy(unittest.TestCase):
|
||||
validate_hisparse_dsa_backend(server_args, "dsa_prefill_backend", "prefill")
|
||||
validate_hisparse_dsa_backend(server_args, "dsa_decode_backend", "decode")
|
||||
|
||||
@patch("sglang.srt.arg_groups.hisparse_hook._is_hip", return_value=True)
|
||||
def test_hisparse_rejects_cuda_backend_on_rocm(self, _mock_is_hip):
|
||||
@override_platform(is_hip=True)
|
||||
def test_hisparse_rejects_cuda_backend_on_rocm(self):
|
||||
server_args = ServerArgs(
|
||||
model_path="dummy",
|
||||
enable_hisparse=True,
|
||||
@@ -918,8 +911,8 @@ class TestHiSparseDsaBackendPolicy(unittest.TestCase):
|
||||
with self.assertRaisesRegex(ValueError, "tilelang"):
|
||||
validate_hisparse_dsa_backend(server_args, "dsa_prefill_backend", "prefill")
|
||||
|
||||
@patch("sglang.srt.arg_groups.hisparse_hook._is_hip", return_value=False)
|
||||
def test_hisparse_rejects_rocm_backend_on_cuda(self, _mock_is_hip):
|
||||
@override_platform(is_hip=False)
|
||||
def test_hisparse_rejects_rocm_backend_on_cuda(self):
|
||||
server_args = ServerArgs(
|
||||
model_path="dummy",
|
||||
enable_hisparse=True,
|
||||
@@ -971,15 +964,15 @@ class TestFa4PageSizeAutoForce(CustomTestCase):
|
||||
args.decode_attention_backend = decode
|
||||
args.page_size = page_size
|
||||
# Short-circuit model_config_of(): the fa4 page_size branch only needs
|
||||
# use_mla_backend() (mocked) and is_sm100_supported() (mocked), not a
|
||||
# use_mla_backend() (mocked) and override_platform(is_sm100=...), not a
|
||||
# real model_config. Pre-set the attribute so get_model_config returns
|
||||
# early without touching ModelConfig.from_server_args.
|
||||
args._model_config = MagicMock()
|
||||
args._model_config.hf_config.dual_chunk_attention_config = None
|
||||
return args
|
||||
|
||||
@patch("sglang.srt.arg_groups.overrides.is_sm100_supported", return_value=True)
|
||||
def test_combined_attention_backend_fa4_forces_page_size_128(self, _mock_sm100):
|
||||
@override_platform(is_sm100=True)
|
||||
def test_combined_attention_backend_fa4_forces_page_size_128(self):
|
||||
# `--attention-backend fa4` (combined): prefill/decode fields stay None.
|
||||
args = self._make_args(attention_backend="fa4")
|
||||
|
||||
@@ -990,8 +983,8 @@ class TestFa4PageSizeAutoForce(CustomTestCase):
|
||||
self.assertEqual(args.page_size, 1) # the field stays pristine
|
||||
self.assertEqual(resolved_view(args).page_size, 128)
|
||||
|
||||
@patch("sglang.srt.arg_groups.overrides.is_sm100_supported", return_value=True)
|
||||
def test_explicit_prefill_fa4_forces_page_size_128(self, _mock_sm100):
|
||||
@override_platform(is_sm100=True)
|
||||
def test_explicit_prefill_fa4_forces_page_size_128(self):
|
||||
# `--prefill-attention-backend fa4`: the previously-covered path.
|
||||
args = self._make_args(attention_backend=None, prefill="fa4", page_size=1)
|
||||
|
||||
@@ -1969,11 +1962,11 @@ class TestPrefillCudaGraphLoRACompatibility(CustomTestCase):
|
||||
prefill=PhaseConfig(backend=Backend.TC_PIECEWISE)
|
||||
)
|
||||
with (
|
||||
patch("sglang.srt.arg_groups.cuda_graph_hook.is_hip", return_value=False),
|
||||
patch("sglang.srt.arg_groups.cuda_graph_hook.is_npu", return_value=False),
|
||||
override_platform(is_hip=False),
|
||||
override_platform(is_npu=False),
|
||||
patch("sglang.srt.arg_groups.cuda_graph_hook.is_cpu", return_value=False),
|
||||
patch("sglang.srt.arg_groups.cuda_graph_hook.is_mps", return_value=False),
|
||||
patch("sglang.srt.arg_groups.cuda_graph_hook.is_xpu", return_value=False),
|
||||
override_platform(is_xpu=False),
|
||||
):
|
||||
disable_tc_piecewise_cudagraph_if_incompatible(args)
|
||||
|
||||
@@ -2602,15 +2595,27 @@ class TestGrpcServerArgs(CustomTestCase):
|
||||
],
|
||||
host="127.0.0.1",
|
||||
)
|
||||
# The port the sidecar dials is the resolved one, off the bag.
|
||||
override = get_context_for_config().override_server_args(grpc_port=50051)
|
||||
# Every value the sidecar reads is resolved config, so the case states
|
||||
# them all through the context rather than half here and half in a
|
||||
# stand-in the readers no longer consult.
|
||||
override = get_context_for_config().override_server_args(
|
||||
grpc_port=50051,
|
||||
sidecar="example.sidecar",
|
||||
sidecar_args=[
|
||||
"--sidecar-shutdown-timeout",
|
||||
"42",
|
||||
"--grpc-connections",
|
||||
"2",
|
||||
],
|
||||
host="127.0.0.1",
|
||||
)
|
||||
override.install()
|
||||
self.addCleanup(override.restore)
|
||||
with (
|
||||
patch("sglang.srt.entrypoints.sidecar.mp.get_context") as get_context,
|
||||
patch("sglang.srt.entrypoints.sidecar.Sidecar") as sidecar_class,
|
||||
):
|
||||
start_sidecar(server_args)
|
||||
start_sidecar()
|
||||
|
||||
process_kwargs = get_context.return_value.Process.call_args.kwargs
|
||||
self.assertEqual(process_kwargs["name"], "sglang_sidecar_example.sidecar")
|
||||
@@ -2826,11 +2831,11 @@ class TestDcpKvEventContract(CustomTestCase):
|
||||
page_size=64,
|
||||
kv_events_config=self.KV_EVENTS,
|
||||
)
|
||||
self.assertEqual(args.describe_kv_events_publisher()["block_size"], 256)
|
||||
self.assertEqual(describe_kv_events_publisher(args)["block_size"], 256)
|
||||
args = ServerArgs(
|
||||
model_path="dummy", page_size=64, kv_events_config=self.KV_EVENTS
|
||||
)
|
||||
self.assertEqual(args.describe_kv_events_publisher()["block_size"], 64)
|
||||
self.assertEqual(describe_kv_events_publisher(args)["block_size"], 64)
|
||||
|
||||
def test_kv_event_block_size_widens_a_single_token_page(self):
|
||||
# page_size=1 + DCP is a real deployment shape: the allocator is still
|
||||
|
||||
@@ -16,6 +16,7 @@ from sglang.test.test_utils import CustomTestCase
|
||||
register_cuda_ci(est_time=10, stage="base-b", runner_config="1-gpu-small")
|
||||
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardMode
|
||||
from sglang.srt.runtime_context import override_platform
|
||||
from sglang.srt.speculative import dflash_info
|
||||
from sglang.srt.speculative.dflash_info import DFlashVerifyInput
|
||||
|
||||
@@ -29,6 +30,9 @@ def _lazy_view(**overrides):
|
||||
mamba_track_interval=256,
|
||||
page_size=64,
|
||||
chunked_prefill_size=None,
|
||||
# The cases name a supported arch, so the view states the backend
|
||||
# rather than the answer.
|
||||
linear_attn_backend="triton",
|
||||
)
|
||||
for key, value in overrides.items():
|
||||
setattr(view, key, value)
|
||||
@@ -39,15 +43,9 @@ class TestValidateMambaExtraBufferLazyDflash(CustomTestCase):
|
||||
"""The DFLASH rejection is gone; the neighboring invariants still hold."""
|
||||
|
||||
def _validate(self, view):
|
||||
with mock.patch(
|
||||
"sglang.srt.arg_groups.overrides.supports_mamba_cache_extra_buffer",
|
||||
return_value=True,
|
||||
), mock.patch(
|
||||
# Keep the test runnable on CPU-only hosts: the platform assert is
|
||||
# not what is under test here.
|
||||
"sglang.srt.arg_groups.mamba_hook.is_cuda",
|
||||
return_value=True,
|
||||
):
|
||||
# Keep the test runnable on CPU-only hosts: the platform assert is
|
||||
# not what is under test here.
|
||||
with override_platform(is_cuda=True):
|
||||
validate_mamba_extra_buffer(
|
||||
view,
|
||||
"Qwen3NextForCausalLM",
|
||||
@@ -81,10 +79,7 @@ class TestValidateMambaExtraBufferLazyDflash(CustomTestCase):
|
||||
def _must_not_be_read():
|
||||
raise AssertionError("the chunk size was read before page_size resolved")
|
||||
|
||||
with mock.patch(
|
||||
"sglang.srt.arg_groups.overrides.supports_mamba_cache_extra_buffer",
|
||||
return_value=True,
|
||||
), mock.patch("sglang.srt.arg_groups.mamba_hook.is_cuda", return_value=True):
|
||||
with override_platform(is_cuda=True):
|
||||
validate_mamba_extra_buffer(
|
||||
_lazy_view(page_size=None),
|
||||
"Qwen3NextForCausalLM",
|
||||
|
||||
@@ -15,7 +15,6 @@ from types import SimpleNamespace
|
||||
from typing import Optional
|
||||
from unittest.mock import patch
|
||||
|
||||
from sglang.srt.arg_groups import attention_hook
|
||||
from sglang.srt.arg_groups import overrides as overrides_module
|
||||
from sglang.srt.arg_groups.arg_utils import A, Arg, resolvable_fields
|
||||
from sglang.srt.arg_groups.overrides import (
|
||||
@@ -31,6 +30,7 @@ from sglang.srt.runtime_context import (
|
||||
get_context,
|
||||
get_exec,
|
||||
get_server_args,
|
||||
override_platform,
|
||||
reset_context,
|
||||
)
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
@@ -427,11 +427,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
)
|
||||
|
||||
def test_sparse_minicpm_defaults_to_sparse_attention_backend(self):
|
||||
with patch.object(
|
||||
overrides_module,
|
||||
"is_blackwell_supported",
|
||||
return_value=False,
|
||||
):
|
||||
with override_platform(is_blackwell=False):
|
||||
for architecture in ("MiniCPMForCausalLM", "MiniCPMSALAForCausalLM"):
|
||||
with self.subTest(architecture=architecture):
|
||||
self.assertEqual(
|
||||
@@ -456,20 +452,14 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
has_lightning_layers=False,
|
||||
)
|
||||
|
||||
with patch.object(
|
||||
overrides_module, "is_blackwell_supported", return_value=False
|
||||
):
|
||||
with override_platform(is_blackwell=False):
|
||||
overrides = overrides_module._minicpm_sala_overrides(args, config)
|
||||
|
||||
self.assertTrue(overrides["disable_radix_cache"])
|
||||
self.assertEqual(overrides["attention_backend"], "minicpm_flashattn")
|
||||
|
||||
def test_sparse_minicpm_defaults_to_flashinfer_on_blackwell(self):
|
||||
with patch.object(
|
||||
overrides_module,
|
||||
"is_blackwell_supported",
|
||||
return_value=True,
|
||||
):
|
||||
with override_platform(is_blackwell=True):
|
||||
self.assertEqual(
|
||||
self._minicpm_overrides(
|
||||
"MiniCPMSALAForCausalLM",
|
||||
@@ -531,11 +521,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
)["attention_backend"],
|
||||
"flashinfer",
|
||||
)
|
||||
with patch.object(
|
||||
overrides_module,
|
||||
"is_blackwell_supported",
|
||||
return_value=True,
|
||||
):
|
||||
with override_platform(is_blackwell=True):
|
||||
self.assertEqual(
|
||||
self._minicpm_overrides(
|
||||
"MiniCPMSALAForCausalLM",
|
||||
@@ -544,11 +530,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
)["attention_backend"],
|
||||
"fa4",
|
||||
)
|
||||
with patch.object(
|
||||
overrides_module,
|
||||
"is_blackwell_supported",
|
||||
return_value=False,
|
||||
):
|
||||
with override_platform(is_blackwell=False):
|
||||
split_overrides = self._minicpm_overrides(
|
||||
"MiniCPMSALAForCausalLM",
|
||||
sparse_attention=True,
|
||||
@@ -625,9 +607,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
"""MiniMax-M2 NVFP4 auto must avoid the unsupported plain TRT-LLM path."""
|
||||
# Every module that asks: the attention handler validates what the
|
||||
# override family picks, and each holds its own import.
|
||||
with patch.object(
|
||||
overrides_module, "is_sm100_supported", return_value=True
|
||||
), patch.object(attention_hook, "is_sm100_supported", return_value=True):
|
||||
with override_platform(is_sm100=True), override_platform(is_sm100=True):
|
||||
explicit = self._construct(
|
||||
"MiniMaxM2ForCausalLM",
|
||||
"llama",
|
||||
@@ -666,8 +646,8 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
|
||||
# Thor (SM110) and other architectures keep the existing auto behavior.
|
||||
with (
|
||||
patch.object(overrides_module, "is_sm100_supported", return_value=False),
|
||||
patch.object(overrides_module, "is_sm120_supported", return_value=False),
|
||||
override_platform(is_sm100=False),
|
||||
override_platform(is_sm120=False),
|
||||
):
|
||||
non_sm10x = self._construct(
|
||||
"MiniMaxM2ForCausalLM", "llama", quantization="modelopt_fp4"
|
||||
@@ -688,7 +668,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
return SimpleNamespace(**defaults)
|
||||
|
||||
# Non-SM100: the MoE pin must not fire, so hf_config is never inspected.
|
||||
with patch.object(overrides_module, "is_sm100_supported", return_value=False):
|
||||
with override_platform(is_sm100=False):
|
||||
self.assertEqual(
|
||||
_mimo_v2_overrides(_args(speculative_algorithm="EAGLE"), None),
|
||||
{"enable_multi_layer_eagle": True},
|
||||
@@ -704,7 +684,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
defaults.update(kw)
|
||||
return SimpleNamespace(**defaults)
|
||||
|
||||
with patch.object(overrides_module, "is_sm100_supported", return_value=True):
|
||||
with override_platform(is_sm100=True):
|
||||
with patch.object(
|
||||
overrides_module, "get_quantization_config", return_value="fp8"
|
||||
):
|
||||
@@ -724,7 +704,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
self.assertEqual(_mimo_v2_overrides(_args(), None), {})
|
||||
|
||||
def test_mimo_v2_family_is_registered(self):
|
||||
with patch.object(overrides_module, "is_sm100_supported", return_value=False):
|
||||
with override_platform(is_sm100=False):
|
||||
self.assertEqual(
|
||||
collect_model_override_declarations(
|
||||
"MiMoV2FlashForCausalLM",
|
||||
@@ -785,8 +765,8 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
)
|
||||
|
||||
with (
|
||||
patch.object(overrides_module, "is_sm100_supported", return_value=True),
|
||||
patch.object(overrides_module, "is_blackwell_supported", return_value=True),
|
||||
override_platform(is_sm100=True),
|
||||
override_platform(is_blackwell=True),
|
||||
):
|
||||
self.assertEqual(
|
||||
_nemotron_h_overrides(server_args, hf_config),
|
||||
@@ -815,8 +795,8 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
)
|
||||
|
||||
with (
|
||||
patch.object(overrides_module, "is_sm100_supported", return_value=True),
|
||||
patch.object(overrides_module, "is_blackwell_supported", return_value=True),
|
||||
override_platform(is_sm100=True),
|
||||
override_platform(is_blackwell=True),
|
||||
):
|
||||
self.assertEqual(
|
||||
_nemotron_h_overrides(server_args, hf_config),
|
||||
@@ -848,16 +828,8 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
server_args.speculative_algorithm = "EAGLE"
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
overrides_module,
|
||||
"is_blackwell_supported",
|
||||
return_value=True,
|
||||
),
|
||||
patch.object(
|
||||
overrides_module,
|
||||
"is_sm100_supported",
|
||||
return_value=is_sm100,
|
||||
),
|
||||
override_platform(is_blackwell=True),
|
||||
override_platform(is_sm100=is_sm100),
|
||||
):
|
||||
overrides = _nemotron_h_overrides(server_args, hf_config)
|
||||
for key, value in expected.items():
|
||||
@@ -871,12 +843,8 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
server_args, hf_config = self._nemotron_h_args(quantized_layers={})
|
||||
server_args.speculative_algorithm = algorithm
|
||||
with (
|
||||
patch.object(
|
||||
overrides_module, "is_blackwell_supported", return_value=True
|
||||
),
|
||||
patch.object(
|
||||
overrides_module, "is_sm100_supported", return_value=True
|
||||
),
|
||||
override_platform(is_blackwell=True),
|
||||
override_platform(is_sm100=True),
|
||||
):
|
||||
overrides = _nemotron_h_overrides(server_args, hf_config)
|
||||
self.assertEqual(overrides["attention_backend"], "trtllm_mha")
|
||||
@@ -888,8 +856,8 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
server_args, hf_config = self._nemotron_h_args(quantized_layers={})
|
||||
server_args.speculative_algorithm = "DFLASH"
|
||||
with (
|
||||
patch.object(overrides_module, "is_blackwell_supported", return_value=True),
|
||||
patch.object(overrides_module, "is_sm100_supported", return_value=True),
|
||||
override_platform(is_blackwell=True),
|
||||
override_platform(is_sm100=True),
|
||||
):
|
||||
overrides = _nemotron_h_overrides(server_args, hf_config)
|
||||
self.assertEqual(overrides["attention_backend"], "trtllm_mha")
|
||||
@@ -905,8 +873,8 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
server_args.speculative_draft_attention_backend = "flashinfer"
|
||||
|
||||
with (
|
||||
patch.object(overrides_module, "is_blackwell_supported", return_value=True),
|
||||
patch.object(overrides_module, "is_sm100_supported", return_value=True),
|
||||
override_platform(is_blackwell=True),
|
||||
override_platform(is_sm100=True),
|
||||
):
|
||||
overrides = _nemotron_h_overrides(server_args, hf_config)
|
||||
|
||||
@@ -923,8 +891,8 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
server_args.speculative_eagle_topk = 4
|
||||
|
||||
with (
|
||||
patch.object(overrides_module, "is_blackwell_supported", return_value=True),
|
||||
patch.object(overrides_module, "is_sm100_supported", return_value=True),
|
||||
override_platform(is_blackwell=True),
|
||||
override_platform(is_sm100=True),
|
||||
):
|
||||
overrides = _nemotron_h_overrides(server_args, hf_config)
|
||||
|
||||
@@ -939,8 +907,8 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
server_args, hf_config = self._nemotron_h_args(quantized_layers={})
|
||||
|
||||
with (
|
||||
patch.object(overrides_module, "is_blackwell_supported", return_value=True),
|
||||
patch.object(overrides_module, "is_sm100_supported", return_value=False),
|
||||
override_platform(is_blackwell=True),
|
||||
override_platform(is_sm100=False),
|
||||
):
|
||||
self.assertNotIn(
|
||||
"attention_backend", _nemotron_h_overrides(server_args, hf_config)
|
||||
@@ -952,8 +920,8 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
server_args, hf_config = self._nemotron_h_args(quantized_layers={})
|
||||
|
||||
with (
|
||||
patch.object(overrides_module, "is_blackwell_supported", return_value=True),
|
||||
patch.object(overrides_module, "is_sm100_supported", return_value=True),
|
||||
override_platform(is_blackwell=True),
|
||||
override_platform(is_sm100=True),
|
||||
):
|
||||
self.assertEqual(
|
||||
_nemotron_h_overrides(server_args, hf_config)["attention_backend"],
|
||||
@@ -969,8 +937,8 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
server_args.speculative_draft_attention_backend = "fa3"
|
||||
|
||||
with (
|
||||
patch.object(overrides_module, "is_blackwell_supported", return_value=True),
|
||||
patch.object(overrides_module, "is_sm100_supported", return_value=True),
|
||||
override_platform(is_blackwell=True),
|
||||
override_platform(is_sm100=True),
|
||||
):
|
||||
overrides = _nemotron_h_overrides(server_args, hf_config)
|
||||
self.assertNotIn("attention_backend", overrides)
|
||||
@@ -1083,7 +1051,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
def test_gpt_oss_xpu_dtype_validation_reads_pristine(self):
|
||||
from sglang.srt.arg_groups.overrides import _gpt_oss_overrides
|
||||
|
||||
with patch.object(overrides_module, "is_xpu", return_value=True):
|
||||
with override_platform(is_xpu=True):
|
||||
with self.assertRaises(NotImplementedError):
|
||||
_gpt_oss_overrides(
|
||||
SimpleNamespace(
|
||||
@@ -1299,12 +1267,12 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
view = ResolvedView(
|
||||
SimpleNamespace(attention_backend="intel_amx", device="cpu")
|
||||
)
|
||||
with patch.object(overrides_module, "cpu_has_amx_support", return_value=False):
|
||||
with override_platform(has_amx=False):
|
||||
self.assertEqual(
|
||||
_attention_backend_platform_fallbacks(view),
|
||||
{"attention_backend": "torch_native"},
|
||||
)
|
||||
with patch.object(overrides_module, "cpu_has_amx_support", return_value=True):
|
||||
with override_platform(has_amx=True):
|
||||
self.assertEqual(_attention_backend_platform_fallbacks(view), {})
|
||||
|
||||
# dual-chunk config: mismatched explicit backend raises verbatim
|
||||
@@ -1338,20 +1306,20 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
defaults.update(kw)
|
||||
return ResolvedView(SimpleNamespace(**defaults))
|
||||
|
||||
with patch.object(overrides_module, "is_hip", return_value=True):
|
||||
with override_platform(is_hip=True):
|
||||
self.assertEqual(
|
||||
_dllm_attention_backend(_view()), {"attention_backend": "triton"}
|
||||
)
|
||||
self.assertEqual(
|
||||
_dllm_attention_backend(_view(attention_backend="aiter")), {}
|
||||
)
|
||||
with patch.object(overrides_module, "is_hip", return_value=False):
|
||||
with patch.object(overrides_module, "is_npu", return_value=True):
|
||||
with override_platform(is_hip=False):
|
||||
with override_platform(is_npu=True):
|
||||
self.assertEqual(
|
||||
_dllm_attention_backend(_view()),
|
||||
{"attention_backend": "ascend"},
|
||||
)
|
||||
with patch.object(overrides_module, "is_npu", return_value=False):
|
||||
with override_platform(is_npu=False):
|
||||
# cuda graph disabled -> nothing to force
|
||||
self.assertEqual(_dllm_attention_backend(_view()), {})
|
||||
self.assertEqual(
|
||||
@@ -1366,13 +1334,13 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
_page_size_default(ResolvedView(SimpleNamespace(page_size=64))), {}
|
||||
)
|
||||
# default fill on non-HIP/non-MUSA platforms is 1
|
||||
with patch.object(overrides_module, "is_hip", return_value=False):
|
||||
with patch.object(overrides_module, "is_musa", return_value=False):
|
||||
with override_platform(is_hip=False):
|
||||
with override_platform(is_musa=False):
|
||||
self.assertEqual(
|
||||
_page_size_default(ResolvedView(SimpleNamespace(page_size=None))),
|
||||
{"page_size": 1},
|
||||
)
|
||||
with patch.object(overrides_module, "is_musa", return_value=True):
|
||||
with override_platform(is_musa=True):
|
||||
self.assertEqual(
|
||||
_page_size_default(ResolvedView(SimpleNamespace(page_size=None))),
|
||||
{"page_size": 64},
|
||||
@@ -1477,7 +1445,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
|
||||
with (
|
||||
envs.SGLANG_DSV4_FP4_DEQUANT.override(False),
|
||||
patch.object(overrides_module, "is_sm100_supported", return_value=True),
|
||||
override_platform(is_sm100=True),
|
||||
):
|
||||
self.assertEqual(
|
||||
_deepseek_v4_overrides(_args(), hf),
|
||||
@@ -1505,7 +1473,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
# FlashInfer MXFP4 only supports the standard (non-A2A) dispatcher.
|
||||
with (
|
||||
envs.SGLANG_DSV4_FP4_DEQUANT.override(False),
|
||||
patch.object(overrides_module, "is_sm100_supported", return_value=True),
|
||||
override_platform(is_sm100=True),
|
||||
):
|
||||
self.assertNotIn(
|
||||
"moe_runner_backend",
|
||||
@@ -1514,7 +1482,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
# Runtime FP4-to-FP8 dequantization must retain the generic FP8 runner.
|
||||
with (
|
||||
envs.SGLANG_DSV4_FP4_DEQUANT.override(True),
|
||||
patch.object(overrides_module, "is_sm100_supported", return_value=True),
|
||||
override_platform(is_sm100=True),
|
||||
):
|
||||
self.assertNotIn(
|
||||
"moe_runner_backend",
|
||||
@@ -1531,7 +1499,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
"moe_runner_backend",
|
||||
_deepseek_v4_overrides(_args(device="npu"), hf),
|
||||
)
|
||||
with patch.object(overrides_module, "is_hip", return_value=True):
|
||||
with override_platform(is_hip=True):
|
||||
self.assertNotIn(
|
||||
"moe_runner_backend",
|
||||
_deepseek_v4_overrides(_args(), hf),
|
||||
@@ -1539,9 +1507,9 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
# Unsupported NVIDIA architectures keep the generic auto-resolution
|
||||
# path instead of selecting a FlashInfer kernel that cannot launch.
|
||||
with (
|
||||
patch.object(overrides_module, "is_sm90_supported", return_value=False),
|
||||
patch.object(overrides_module, "is_sm100_supported", return_value=False),
|
||||
patch.object(overrides_module, "is_sm120_supported", return_value=False),
|
||||
override_platform(is_sm90=False),
|
||||
override_platform(is_sm100=False),
|
||||
override_platform(is_sm120=False),
|
||||
):
|
||||
self.assertNotIn(
|
||||
"moe_runner_backend",
|
||||
@@ -1550,9 +1518,9 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
# SM120 uses the same model hook; no later pass is needed.
|
||||
with (
|
||||
envs.SGLANG_DSV4_FP4_DEQUANT.override(False),
|
||||
patch.object(overrides_module, "is_sm90_supported", return_value=False),
|
||||
patch.object(overrides_module, "is_sm100_supported", return_value=False),
|
||||
patch.object(overrides_module, "is_sm120_supported", return_value=True),
|
||||
override_platform(is_sm90=False),
|
||||
override_platform(is_sm100=False),
|
||||
override_platform(is_sm120=True),
|
||||
):
|
||||
self.assertEqual(
|
||||
_deepseek_v4_overrides(_args(), hf)["moe_runner_backend"],
|
||||
@@ -1605,8 +1573,8 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
|
||||
hf = _hf()
|
||||
with (
|
||||
patch.object(overrides_module, "is_sm100_supported", return_value=True),
|
||||
patch.object(overrides_module, "is_blackwell_supported", return_value=True),
|
||||
override_platform(is_sm100=True),
|
||||
override_platform(is_blackwell=True),
|
||||
):
|
||||
# modelopt checkpoint: quant algo resolution + sm100 defaults
|
||||
self.assertEqual(
|
||||
@@ -1625,11 +1593,9 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
"modelopt_mixed",
|
||||
)
|
||||
with (
|
||||
patch.object(overrides_module, "is_sm100_supported", return_value=False),
|
||||
patch.object(overrides_module, "is_cuda", return_value=True),
|
||||
patch.object(
|
||||
overrides_module, "get_device_capability", return_value=(9, 0)
|
||||
),
|
||||
override_platform(is_sm100=False),
|
||||
override_platform(is_cuda=True),
|
||||
override_platform(device_capability=(9, 0)),
|
||||
):
|
||||
# SM80-SM90 fp4: marlin
|
||||
self.assertEqual(
|
||||
@@ -1651,8 +1617,8 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
|
||||
hf_without_quant_cfg = _hf(include_quantization_config=False)
|
||||
with (
|
||||
patch.object(overrides_module, "is_sm100_supported", return_value=True),
|
||||
patch.object(overrides_module, "is_blackwell_supported", return_value=True),
|
||||
override_platform(is_sm100=True),
|
||||
override_platform(is_blackwell=True),
|
||||
):
|
||||
for modelopt_quantization in ("modelopt_fp8", "modelopt_fp4"):
|
||||
with self.subTest(modelopt_quantization=modelopt_quantization):
|
||||
@@ -1718,9 +1684,9 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
|
||||
with (
|
||||
patch("sglang.srt.configs.model_config.is_deepseek_dsa", return_value=True),
|
||||
patch.object(overrides_module, "is_npu", return_value=False),
|
||||
patch.object(overrides_module, "is_xpu", return_value=False),
|
||||
patch.object(overrides_module, "is_hip", return_value=False),
|
||||
override_platform(is_npu=False),
|
||||
override_platform(is_xpu=False),
|
||||
override_platform(is_hip=False),
|
||||
patch("torch.cuda.get_device_capability", return_value=(9, 0)),
|
||||
):
|
||||
# Hopper FP8 -> flashmla_kv both
|
||||
@@ -1758,9 +1724,9 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
)
|
||||
with (
|
||||
patch("sglang.srt.configs.model_config.is_deepseek_dsa", return_value=True),
|
||||
patch.object(overrides_module, "is_npu", return_value=False),
|
||||
patch.object(overrides_module, "is_xpu", return_value=False),
|
||||
patch.object(overrides_module, "is_hip", return_value=False),
|
||||
override_platform(is_npu=False),
|
||||
override_platform(is_xpu=False),
|
||||
override_platform(is_hip=False),
|
||||
patch("torch.cuda.get_device_capability", return_value=(12, 0)),
|
||||
):
|
||||
self.assertEqual(
|
||||
@@ -1772,9 +1738,9 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
)
|
||||
with (
|
||||
patch("sglang.srt.configs.model_config.is_deepseek_dsa", return_value=True),
|
||||
patch.object(overrides_module, "is_npu", return_value=False),
|
||||
patch.object(overrides_module, "is_xpu", return_value=False),
|
||||
patch.object(overrides_module, "is_hip", return_value=True),
|
||||
override_platform(is_npu=False),
|
||||
override_platform(is_xpu=False),
|
||||
override_platform(is_hip=True),
|
||||
patch("torch.cuda.get_device_capability", return_value=(9, 4)),
|
||||
):
|
||||
# ROCm with both unset -> tilelang
|
||||
@@ -1811,8 +1777,8 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
)
|
||||
|
||||
with (
|
||||
patch.object(overrides_module, "is_sm90_supported", return_value=True),
|
||||
patch.object(overrides_module, "is_sm100_supported", return_value=False),
|
||||
override_platform(is_sm90=True),
|
||||
override_platform(is_sm100=False),
|
||||
):
|
||||
self.assertEqual(
|
||||
_flashinfer_allreduce_fusion_auto_enable(_view()),
|
||||
@@ -1897,7 +1863,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
defaults.update(kw)
|
||||
return ResolvedView(SimpleNamespace(**defaults))
|
||||
|
||||
with patch.object(overrides_module, "is_sm100_supported", return_value=True):
|
||||
with override_platform(is_sm100=True):
|
||||
# decode-only cutedsl: prefill defaults to trtllm_mla
|
||||
self.assertEqual(
|
||||
_cutedsl_prefill_backend_fill(_view()),
|
||||
@@ -1921,7 +1887,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
_cutedsl_prefill_backend_fill(_view(decode_attention_backend=None)),
|
||||
{},
|
||||
)
|
||||
with patch.object(overrides_module, "is_sm100_supported", return_value=False):
|
||||
with override_platform(is_sm100=False):
|
||||
with self.assertRaises(ValueError):
|
||||
_cutedsl_prefill_backend_fill(_view())
|
||||
|
||||
@@ -1971,8 +1937,8 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
|
||||
with (
|
||||
patch("sglang.srt.configs.model_config.is_deepseek_dsa", return_value=True),
|
||||
patch.object(overrides_module, "is_npu", return_value=False),
|
||||
patch.object(overrides_module, "is_xpu", return_value=False),
|
||||
override_platform(is_npu=False),
|
||||
override_platform(is_xpu=False),
|
||||
):
|
||||
with patch("torch.cuda.get_device_capability", return_value=(9, 0)):
|
||||
# Hopper: auto -> bfloat16
|
||||
@@ -2052,7 +2018,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
SimpleNamespace(_model_config=SimpleNamespace(hf_config=hf), **defaults)
|
||||
)
|
||||
|
||||
with patch.object(overrides_module, "is_hip", return_value=True):
|
||||
with override_platform(is_hip=True):
|
||||
with patch.object(
|
||||
envs.SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE, "get", return_value=False
|
||||
):
|
||||
@@ -2093,7 +2059,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
with self.assertRaises(ValueError):
|
||||
_deepseek_spec_moe_resolution(_view(ep_size=1))
|
||||
# the arm is HIP-only
|
||||
with patch.object(overrides_module, "is_hip", return_value=False):
|
||||
with override_platform(is_hip=False):
|
||||
self.assertEqual(_deepseek_spec_moe_resolution(_view()), {})
|
||||
|
||||
def test_mamba_radix_cache_resolution_pass(self):
|
||||
@@ -2234,9 +2200,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
args._model_config = SimpleNamespace(attention_arch=AttentionArch.MHA)
|
||||
return args
|
||||
|
||||
with patch.object(
|
||||
overrides_module, "is_sm100_supported", return_value=True
|
||||
), patch.object(
|
||||
with override_platform(is_sm100=True), patch.object(
|
||||
overrides_module,
|
||||
"get_default_attn_backend",
|
||||
lambda server_args, **_: server_args.default_backend_for_test,
|
||||
@@ -2278,13 +2242,13 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
),
|
||||
{"attention_backend": "trtllm_mha", "page_size": 64},
|
||||
)
|
||||
with patch.object(overrides_module, "is_sm100_supported", return_value=False):
|
||||
with override_platform(is_sm100=False):
|
||||
self.assertEqual(_qwen3_5_hybrid_overrides(_args("fa3"), None), {})
|
||||
|
||||
def test_qwen3vl_page_size(self):
|
||||
from sglang.srt.arg_groups.overrides import _qwen3vl_overrides
|
||||
|
||||
with patch.object(overrides_module, "is_hip", return_value=True):
|
||||
with override_platform(is_hip=True):
|
||||
with patch("sglang.srt.environ.envs.SGLANG_USE_AITER_UNIFIED_ATTN") as e:
|
||||
e.get.return_value = True
|
||||
self.assertEqual(
|
||||
@@ -2306,7 +2270,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
defaults.update(kw)
|
||||
return ResolvedView(SimpleNamespace(**defaults))
|
||||
|
||||
with patch.object(overrides_module, "is_sm100_supported", return_value=True):
|
||||
with override_platform(is_sm100=True):
|
||||
self.assertEqual(
|
||||
_moe_runner_backend_quant_constraints(
|
||||
_view(quantization="nvfp4_online")
|
||||
@@ -2321,7 +2285,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
_moe_runner_backend_quant_constraints(_view(quantization="mxfp8")),
|
||||
{"moe_runner_backend": "flashinfer_trtllm"},
|
||||
)
|
||||
with patch.object(overrides_module, "is_sm120_supported", return_value=True):
|
||||
with override_platform(is_sm120=True):
|
||||
self.assertEqual(
|
||||
_moe_runner_backend_quant_constraints(
|
||||
_view(quantization="modelopt_fp4")
|
||||
@@ -2366,7 +2330,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
defaults.update(kw)
|
||||
return SimpleNamespace(**defaults)
|
||||
|
||||
with patch("sglang.srt.utils.common.is_sm100_supported", return_value=True):
|
||||
with override_platform(is_sm100=True):
|
||||
# e4m3 + trtllm_mha + SM100: mode active
|
||||
self.assertTrue(m3_fp8_attn_gemm_enabled(_args()))
|
||||
# fa4 dense backend: mode inactive (no fp8-q GEMM path)
|
||||
@@ -2380,7 +2344,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
# otherwise-active config
|
||||
with envs.SGLANG_DISABLE_M3_FP8_ATTN_GEMM.override(True):
|
||||
self.assertFalse(m3_fp8_attn_gemm_enabled(_args()))
|
||||
with patch("sglang.srt.utils.common.is_sm100_supported", return_value=False):
|
||||
with override_platform(is_sm100=False):
|
||||
# non-SM100: mode inactive
|
||||
self.assertFalse(m3_fp8_attn_gemm_enabled(_args()))
|
||||
|
||||
@@ -2400,8 +2364,8 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
return ns
|
||||
|
||||
hf = SimpleNamespace()
|
||||
with patch.object(overrides_module, "is_hip", return_value=False), patch.object(
|
||||
overrides_module, "is_sm100_supported", return_value=True
|
||||
with override_platform(is_hip=False), override_platform(
|
||||
is_sm100=True
|
||||
), patch.object(overrides_module, "get_quantization_config", return_value=None):
|
||||
# fp8_e4m3 KV: SM100 backend default flips to trtllm_mha (the only
|
||||
# dense backend with the fp8-q GEMM path); page snaps to 128
|
||||
@@ -2501,7 +2465,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
# no matching backend: nothing declared
|
||||
self.assertEqual(_mla_backend_page_constraints(_view()), {})
|
||||
|
||||
with patch.object(overrides_module, "is_sm100_supported", return_value=True):
|
||||
with override_platform(is_sm100=True):
|
||||
self.assertEqual(
|
||||
_fa4_page_constraint(
|
||||
_view(
|
||||
@@ -2565,7 +2529,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
defaults.update(kw)
|
||||
return SimpleNamespace(**defaults)
|
||||
|
||||
with patch.object(overrides_module, "is_sm100_supported", return_value=True):
|
||||
with override_platform(is_sm100=True):
|
||||
self.assertEqual(
|
||||
_llama4_overrides(_args(), None), {"attention_backend": "trtllm_mha"}
|
||||
)
|
||||
@@ -2599,9 +2563,9 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
self.assertEqual(
|
||||
_lfm2_overrides(_args(), None), {"attention_backend": "flashinfer"}
|
||||
)
|
||||
with patch.object(overrides_module, "is_sm100_supported", return_value=False):
|
||||
with override_platform(is_sm100=False):
|
||||
self.assertEqual(_minicpm_v4_6_overrides(_args(), None), {})
|
||||
with patch.object(overrides_module, "is_sm90_supported", return_value=True):
|
||||
with override_platform(is_sm90=True):
|
||||
self.assertEqual(
|
||||
_llama4_overrides(_args(), None), {"attention_backend": "fa3"}
|
||||
)
|
||||
@@ -2609,11 +2573,11 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
_gemma4_overrides(_args(), None), {"attention_backend": "triton"}
|
||||
)
|
||||
# Glm4Moe: unconditional tf32 declaration + (sm100) quant/moe absorption
|
||||
with patch.object(overrides_module, "is_sm100_supported", return_value=False):
|
||||
with override_platform(is_sm100=False):
|
||||
self.assertEqual(
|
||||
_glm4_moe_overrides(None, None), {"enable_tf32_matmul": True}
|
||||
)
|
||||
with patch.object(overrides_module, "is_sm100_supported", return_value=True):
|
||||
with override_platform(is_sm100=True):
|
||||
self.assertEqual(
|
||||
_glm4_moe_overrides(
|
||||
SimpleNamespace(
|
||||
@@ -2654,7 +2618,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
defaults.update(kw)
|
||||
return ResolvedView(SimpleNamespace(**defaults))
|
||||
|
||||
with patch.object(overrides_module, "is_sm100_supported", return_value=True):
|
||||
with override_platform(is_sm100=True):
|
||||
with patch.object(
|
||||
overrides_module, "get_quantization_config", return_value="fp8"
|
||||
):
|
||||
@@ -2670,7 +2634,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
self.assertEqual(
|
||||
_deepseek_moe_quant_resolution(_view(arch="LlamaForCausalLM")), {}
|
||||
)
|
||||
with patch.object(overrides_module, "is_sm100_supported", return_value=False):
|
||||
with override_platform(is_sm100=False):
|
||||
self.assertEqual(_deepseek_moe_quant_resolution(_view()), {})
|
||||
|
||||
def test_data_parallelism_and_a2a_passes(self):
|
||||
@@ -2725,15 +2689,15 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
with patch(
|
||||
"sglang.srt.configs.model_config.is_deepseek_dsa", return_value=True
|
||||
):
|
||||
with patch.object(overrides_module, "is_npu", return_value=False):
|
||||
with patch.object(overrides_module, "is_xpu", return_value=False):
|
||||
with patch.object(overrides_module, "is_hip", return_value=False):
|
||||
with override_platform(is_npu=False):
|
||||
with override_platform(is_xpu=False):
|
||||
with override_platform(is_hip=False):
|
||||
self.assertEqual(
|
||||
_deepseek_family_overrides(_args(), None),
|
||||
{"attention_backend": "dsa", "page_size": 64},
|
||||
)
|
||||
# HIP without the preshuffle path: page 1
|
||||
with patch.object(overrides_module, "is_hip", return_value=True):
|
||||
with override_platform(is_hip=True):
|
||||
with patch(
|
||||
"sglang.srt.layers.attention.dsa.utils.aiter_can_use_preshuffle_paged_mqa",
|
||||
return_value=False,
|
||||
@@ -2746,9 +2710,9 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
with patch(
|
||||
"sglang.srt.configs.model_config.is_deepseek_dsa", return_value=True
|
||||
):
|
||||
with patch.object(overrides_module, "is_npu", return_value=False):
|
||||
with patch.object(overrides_module, "is_xpu", return_value=False):
|
||||
with patch.object(overrides_module, "is_hip", return_value=False):
|
||||
with override_platform(is_npu=False):
|
||||
with override_platform(is_xpu=False):
|
||||
with override_platform(is_hip=False):
|
||||
result = _deepseek_family_overrides(
|
||||
_args(
|
||||
enable_prefill_cp=True,
|
||||
@@ -2789,9 +2753,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
with patch(
|
||||
"sglang.srt.configs.model_config.is_deepseek_dsa", return_value=False
|
||||
):
|
||||
with patch.object(
|
||||
overrides_module, "is_sm100_supported", return_value=True
|
||||
):
|
||||
with override_platform(is_sm100=True):
|
||||
self.assertEqual(
|
||||
_deepseek_family_overrides(_args(), None),
|
||||
{"attention_backend": "trtllm_mla"},
|
||||
@@ -2802,15 +2764,13 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
),
|
||||
{},
|
||||
)
|
||||
with patch.object(
|
||||
overrides_module, "is_sm100_supported", return_value=False
|
||||
):
|
||||
with override_platform(is_sm100=False):
|
||||
self.assertEqual(_deepseek_family_overrides(_args(), None), {})
|
||||
|
||||
def test_qwen3_moe_family_quant_absorption(self):
|
||||
from sglang.srt.arg_groups.overrides import _qwen3_moe_family_overrides
|
||||
|
||||
with patch.object(overrides_module, "is_sm100_supported", return_value=True):
|
||||
with override_platform(is_sm100=True):
|
||||
with patch.object(
|
||||
overrides_module, "get_quantization_config", return_value="fp8"
|
||||
):
|
||||
@@ -2829,7 +2789,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
"moe_runner_backend": "flashinfer_trtllm",
|
||||
},
|
||||
)
|
||||
with patch.object(overrides_module, "is_sm100_supported", return_value=False):
|
||||
with override_platform(is_sm100=False):
|
||||
self.assertEqual(_qwen3_moe_family_overrides(None, None), {})
|
||||
|
||||
def test_step3p_declarations_at_callable_level(self):
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
"""No module-scope name may freeze a platform fact.
|
||||
|
||||
The address exists so `override_platform(...)` reaches every reader at once.
|
||||
A module-level `_is_sm120 = get_platform().is_sm120` defeats that completely:
|
||||
the value is read when the module is first imported and never again, so whether
|
||||
an override is visible depends on import order -- and the line *looks* like it
|
||||
went through the address, which is worse than the bare probe it replaced.
|
||||
|
||||
Four of these were written during this refactor's own conversion (three in
|
||||
`fp8_utils`, one in `deepseek_v4_backend`), by substituting the accessor into
|
||||
lines that were already frozen. Substituting the call is not the conversion; the
|
||||
conversion is the reader asking at the point of decision.
|
||||
"""
|
||||
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
|
||||
register_cpu_ci(est_time=6, suite="base-a-test-cpu")
|
||||
|
||||
import ast
|
||||
import pathlib
|
||||
import unittest
|
||||
|
||||
import sglang
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
_ROOT = pathlib.Path(next(iter(sglang.__path__))) / "srt"
|
||||
|
||||
|
||||
def _frozen_platform_reads():
|
||||
"""(file, line, name) for each module-scope `x = get_platform().y`."""
|
||||
found = []
|
||||
for path in sorted(_ROOT.rglob("*.py")):
|
||||
source = path.read_text(encoding="utf-8-sig")
|
||||
if "get_platform" not in source:
|
||||
continue
|
||||
try:
|
||||
tree = ast.parse(source)
|
||||
except SyntaxError:
|
||||
continue
|
||||
# Module scope only: inside a function the call runs per invocation,
|
||||
# which is the shape the address is for.
|
||||
for node in tree.body:
|
||||
if not isinstance(node, ast.Assign):
|
||||
continue
|
||||
value = node.value
|
||||
if not (
|
||||
isinstance(value, ast.Attribute)
|
||||
and isinstance(value.value, ast.Call)
|
||||
and getattr(value.value.func, "id", None) == "get_platform"
|
||||
):
|
||||
continue
|
||||
for target in node.targets:
|
||||
if isinstance(target, ast.Name):
|
||||
rel = path.relative_to(_ROOT).as_posix()
|
||||
found.append(f"{rel}:{node.lineno} {target.id}")
|
||||
return found
|
||||
|
||||
|
||||
class TestPlatformAddressNotFrozen(CustomTestCase):
|
||||
def test_the_scan_reaches_the_address(self):
|
||||
"""The premise: `get_platform()` is used somewhere under srt/."""
|
||||
users = [
|
||||
path
|
||||
for path in _ROOT.rglob("*.py")
|
||||
if "get_platform()" in path.read_text(encoding="utf-8-sig")
|
||||
]
|
||||
self.assertGreater(len(users), 20, "the scan found almost no readers")
|
||||
|
||||
def test_no_module_scope_name_freezes_a_platform_fact(self):
|
||||
frozen = _frozen_platform_reads()
|
||||
self.assertEqual(
|
||||
[],
|
||||
frozen,
|
||||
"these read a platform fact once at import and keep the answer, so "
|
||||
"`override_platform(...)` cannot reach them and the result depends "
|
||||
f"on import order. Ask at the point of decision instead: {frozen}",
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,90 @@
|
||||
"""A platform fact has one address, and one place to change it.
|
||||
|
||||
The value lives in one object, so `override_platform(...)` reaches every reader
|
||||
at once, and the facts answer before anything is published.
|
||||
"""
|
||||
|
||||
import unittest
|
||||
|
||||
from sglang.srt.runtime_context import (
|
||||
_PLATFORM_PROBES,
|
||||
_PLATFORM_VALUES,
|
||||
get_platform,
|
||||
override_platform,
|
||||
)
|
||||
from sglang.srt.utils import common as _common
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
|
||||
|
||||
|
||||
class TestPlatformContext(CustomTestCase):
|
||||
def test_every_name_maps_to_a_real_probe(self):
|
||||
"""A typo in either table would answer with an AttributeError at runtime."""
|
||||
for name, probe in {**_PLATFORM_PROBES, **_PLATFORM_VALUES}.items():
|
||||
self.assertTrue(
|
||||
callable(getattr(_common, probe, None)),
|
||||
f"{name} maps to {probe!r}, which is not callable in utils.common",
|
||||
)
|
||||
|
||||
def test_it_answers_before_anything_is_published(self):
|
||||
"""Unlike a config bag: a launcher asks these before it publishes."""
|
||||
platform = get_platform()
|
||||
for name in _PLATFORM_PROBES:
|
||||
self.assertIsInstance(getattr(platform, name), bool)
|
||||
|
||||
def test_the_probe_is_what_it_answers_with(self):
|
||||
platform = get_platform()
|
||||
for name, probe in {**_PLATFORM_PROBES, **_PLATFORM_VALUES}.items():
|
||||
self.assertEqual(getattr(platform, name), getattr(_common, probe)())
|
||||
|
||||
def test_an_override_is_scoped_and_restores(self):
|
||||
platform = get_platform()
|
||||
before = platform.is_sm100
|
||||
with override_platform(is_sm100=not before):
|
||||
self.assertEqual(platform.is_sm100, not before)
|
||||
self.assertEqual(platform.is_sm100, before)
|
||||
|
||||
def test_overrides_nest(self):
|
||||
platform = get_platform()
|
||||
with override_platform(is_sm100=True):
|
||||
self.assertTrue(platform.is_sm100)
|
||||
with override_platform(is_sm100=False):
|
||||
self.assertFalse(platform.is_sm100)
|
||||
self.assertTrue(platform.is_sm100)
|
||||
|
||||
def test_an_unknown_fact_is_refused_on_both_paths(self):
|
||||
with self.assertRaises(AttributeError):
|
||||
get_platform().is_sm999
|
||||
with self.assertRaises(ValueError):
|
||||
override_platform(is_sm999=True).install()
|
||||
|
||||
def test_a_fact_is_not_assigned(self):
|
||||
"""Assignment would move it for one reader, which is the defect."""
|
||||
with self.assertRaises(AttributeError):
|
||||
get_platform().is_sm100 = True
|
||||
|
||||
def test_one_statement_reaches_every_reader(self):
|
||||
"""The point of the address.
|
||||
|
||||
Two modules that both ask the context see the same answer from one
|
||||
override -- which is what the round-4 failure needed and could not say.
|
||||
"""
|
||||
|
||||
def module_a():
|
||||
return get_platform().is_sm100
|
||||
|
||||
def module_b():
|
||||
return get_platform().is_sm100
|
||||
|
||||
with override_platform(is_sm100=True):
|
||||
self.assertTrue(module_a())
|
||||
self.assertTrue(module_b())
|
||||
with override_platform(is_sm100=False):
|
||||
self.assertFalse(module_a())
|
||||
self.assertFalse(module_b())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,134 @@
|
||||
"""The launch path runs before `publish`, so nothing on it may ask a config bag.
|
||||
|
||||
`publish` is what projects the bags; every accessor fails closed until then
|
||||
(`config namespace 'observability' not published`). Most readers live deep in a
|
||||
runtime path and are safely downstream of it, so converting one to a bag is
|
||||
normally free. The launcher's own reads are not: everything
|
||||
`_launch_subprocesses` calls before its `publish` runs with no bags at all, and
|
||||
`multimodal_gen` calls into the same code with a `ServerArgs` of its own that
|
||||
never publishes them.
|
||||
|
||||
This is a class no other test in the tree catches: a converted reader is
|
||||
exercised everywhere by tests that publish first, so it passes unit CI and then
|
||||
takes the server down on the first real launch -- which is how `configure_logger`
|
||||
shipped. So the protected set is *derived from the launch path* rather than
|
||||
listed here: the callees named in `_launch_subprocesses` above its `publish` are
|
||||
read out of the source, and each is called against a context where nothing has
|
||||
been published. A conversion of any of them turns this red without a boot.
|
||||
"""
|
||||
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
|
||||
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
|
||||
|
||||
import ast
|
||||
import logging
|
||||
import pathlib
|
||||
import unittest
|
||||
|
||||
import sglang
|
||||
from sglang.srt.entrypoints.engine import _set_envs_and_config
|
||||
from sglang.srt.runtime_context import get_observability, reset_context
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
from sglang.srt.utils.common import configure_logger
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
# Callees this guard can drive directly. `_set_envs_and_config` is here too: it
|
||||
# is the other one that runs on every launch before anything is published.
|
||||
_EXERCISED = {
|
||||
"configure_logger": configure_logger,
|
||||
"_set_envs_and_config": _set_envs_and_config,
|
||||
}
|
||||
|
||||
# Named by the launcher before `publish`, but none of them reads config out of a
|
||||
# record. Listed so the set above stays a statement about all of the callees.
|
||||
_NOT_EXERCISED = {
|
||||
"load_plugins",
|
||||
"resolve_auto_parsers",
|
||||
"snapshot_context",
|
||||
"resolving_view",
|
||||
}
|
||||
|
||||
|
||||
def _pre_publish_callees():
|
||||
"""Functions `_launch_subprocesses` calls before it publishes.
|
||||
|
||||
Read from the source so the set cannot go stale: a call added above the
|
||||
`publish(...)` line joins the protected set on its own.
|
||||
"""
|
||||
source = (
|
||||
pathlib.Path(next(iter(sglang.__path__))) / "srt" / "entrypoints" / "engine.py"
|
||||
).read_text(encoding="utf-8-sig")
|
||||
tree = ast.parse(source)
|
||||
launcher = next(
|
||||
node
|
||||
for node in ast.walk(tree)
|
||||
if isinstance(node, ast.FunctionDef) and node.name == "_launch_subprocesses"
|
||||
)
|
||||
publish_line = min(
|
||||
node.lineno
|
||||
for node in ast.walk(launcher)
|
||||
if isinstance(node, ast.Call) and getattr(node.func, "id", None) == "publish"
|
||||
)
|
||||
return {
|
||||
node.func.id
|
||||
for node in ast.walk(launcher)
|
||||
if isinstance(node, ast.Call)
|
||||
and isinstance(node.func, ast.Name)
|
||||
and node.lineno < publish_line
|
||||
}
|
||||
|
||||
|
||||
class TestPrePublishReaders(CustomTestCase):
|
||||
def setUp(self):
|
||||
self._levels = {
|
||||
name: logging.getLogger(name).level
|
||||
for name in (None, "sglang", "httpx", "httpcore")
|
||||
}
|
||||
reset_context()
|
||||
self.addCleanup(self._restore_levels)
|
||||
self.addCleanup(reset_context)
|
||||
|
||||
def _restore_levels(self):
|
||||
for name, level in self._levels.items():
|
||||
logging.getLogger(name).setLevel(level)
|
||||
|
||||
def test_nothing_is_published_here(self):
|
||||
"""The premise: this fixture really is a pre-publish context."""
|
||||
with self.assertRaises(ValueError) as caught:
|
||||
get_observability()
|
||||
self.assertIn("not published", str(caught.exception))
|
||||
|
||||
def test_the_protected_set_is_what_the_launcher_calls(self):
|
||||
"""If the launcher stops calling one of these, or starts calling
|
||||
something new before publishing, this file has to be looked at."""
|
||||
self.assertEqual(set(_EXERCISED) | _NOT_EXERCISED, _pre_publish_callees())
|
||||
|
||||
def test_none_of_them_asks_a_bag(self):
|
||||
"""Each is called with nothing published. A bag read raises
|
||||
`config namespace ... not published` -- any other failure is the
|
||||
callee's own business and does not belong to this guard."""
|
||||
server_args = ServerArgs(model_path="dummy", log_level="warning")
|
||||
for name, call in sorted(_EXERCISED.items()):
|
||||
with self.subTest(callee=name):
|
||||
reset_context()
|
||||
try:
|
||||
call(server_args)
|
||||
except Exception as exc: # noqa: BLE001 -- see the docstring
|
||||
self.assertNotIn(
|
||||
"not published",
|
||||
str(exc),
|
||||
f"{name} runs before publish and asked a config bag",
|
||||
)
|
||||
|
||||
def test_configure_logger_reads_the_record_it_was_handed(self):
|
||||
"""The one that shipped broken, pinned by value rather than by not
|
||||
raising: `multimodal_gen` hands it a ServerArgs that never publishes."""
|
||||
configure_logger(ServerArgs(model_path="dummy", log_level="warning"))
|
||||
self.assertEqual(logging.getLogger().level, logging.WARNING)
|
||||
configure_logger(ServerArgs(model_path="dummy", log_level="error"))
|
||||
self.assertEqual(logging.getLogger().level, logging.ERROR)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -26,6 +26,7 @@ from sglang.srt.arg_groups.overrides import (
|
||||
max_prefill_buffer_tokens as max_prefill_buffer_tokens_of,
|
||||
)
|
||||
from sglang.srt.arg_groups.overrides import (
|
||||
resolution_result,
|
||||
resolved_view,
|
||||
)
|
||||
from sglang.srt.runtime_context import (
|
||||
@@ -377,10 +378,12 @@ class TestServerArgsScopedOverride(_IsolatedServerArgs):
|
||||
)
|
||||
published = override.install()
|
||||
self.assertIs(get_server_args(), published)
|
||||
self.assertEqual(published.attention_backend, "triton")
|
||||
self.assertEqual(published.chunked_prefill_size, -1)
|
||||
# The hook declares; the record keeps the operator's input, so the
|
||||
# values are read where resolution puts them.
|
||||
self.assertEqual(resolution_result(published, "attention_backend"), "triton")
|
||||
self.assertEqual(resolution_result(published, "chunked_prefill_size"), -1)
|
||||
# unnamed fields keep their dataclass defaults
|
||||
self.assertEqual(published.tp_size, 1)
|
||||
self.assertEqual(resolution_result(published, "tp_size"), 1)
|
||||
|
||||
def test_unknown_fields_are_rejected(self):
|
||||
with self.assertRaises(ValueError):
|
||||
@@ -391,7 +394,7 @@ class TestServerArgsScopedOverride(_IsolatedServerArgs):
|
||||
get_context().set_server_args(previous)
|
||||
override = get_context().override_server_args(tp_size=8)
|
||||
override.install()
|
||||
self.assertEqual(get_server_args().tp_size, 8)
|
||||
self.assertEqual(get_parallel().tp_size, 8)
|
||||
override.restore()
|
||||
self.assertIs(get_server_args(), previous)
|
||||
|
||||
@@ -406,9 +409,9 @@ class TestServerArgsScopedOverride(_IsolatedServerArgs):
|
||||
reset_context()
|
||||
with get_context().override_server_args(tp_size=2) as outer:
|
||||
with get_context().override_server_args(tp_size=4):
|
||||
self.assertEqual(get_server_args().tp_size, 4)
|
||||
self.assertEqual(get_parallel().tp_size, 4)
|
||||
self.assertIs(get_server_args(), outer)
|
||||
self.assertEqual(get_server_args().tp_size, 2)
|
||||
self.assertEqual(get_parallel().tp_size, 2)
|
||||
|
||||
def test_private_attribute_seeding(self):
|
||||
# Property caches (e.g. _mamba_cache_chunk_size) are seeded through
|
||||
@@ -418,13 +421,35 @@ class TestServerArgsScopedOverride(_IsolatedServerArgs):
|
||||
)
|
||||
self.assertEqual(mamba_cache_chunk_size_of(published), 64)
|
||||
|
||||
def test_an_underscore_field_is_declared_like_any_other(self):
|
||||
"""The split is fields vs not-fields, not the leading underscore.
|
||||
|
||||
`_speculative_draft_quantization_explicitly_set` is a real field
|
||||
published under `spec`. Seeding it as a raw attribute instead of
|
||||
declaring it would leave the earlier resolution authoritative, so both
|
||||
the resolution and the bag would keep answering the pre-override value
|
||||
while the record said otherwise.
|
||||
"""
|
||||
from sglang.srt.arg_groups.overrides import resolution_result
|
||||
from sglang.srt.runtime_context import get_spec
|
||||
|
||||
name = "_speculative_draft_quantization_explicitly_set"
|
||||
self.assertIn(name, ServerArgs.__dataclass_fields__)
|
||||
|
||||
published = get_context().override_server_args(**{name: True}).install()
|
||||
# The record keeps the operator's input, as it does for every other
|
||||
# field; the override travels as a declaration.
|
||||
self.assertIsNone(getattr(published, name))
|
||||
self.assertIs(resolution_result(published, name), True)
|
||||
self.assertIs(getattr(get_spec(), name), True)
|
||||
|
||||
def test_installed_config_arms_the_strict_guard(self):
|
||||
# The published dummy must behave like a resolved config: bare writes
|
||||
# raise.
|
||||
published = get_context().override_server_args(tp_size=2).install()
|
||||
with self.assertRaises(AttributeError):
|
||||
published.tp_size = 4
|
||||
self.assertEqual(published.tp_size, 2)
|
||||
self.assertEqual(resolution_result(published, "tp_size"), 2)
|
||||
|
||||
def test_restore_resets_the_capture_seed(self):
|
||||
# install() seeds flags.capture from the published dummy; restore()
|
||||
|
||||
@@ -98,7 +98,10 @@ class TestNoServerArgsMutationEntry(CustomTestCase):
|
||||
)
|
||||
|
||||
def test_late_resolution_refuses_the_published_config(self):
|
||||
from sglang.srt.arg_groups.overrides import declare_late_resolution
|
||||
from sglang.srt.arg_groups.overrides import (
|
||||
declare_late_resolution,
|
||||
resolution_result,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_context
|
||||
|
||||
override = get_context().override_server_args(tp_size=2)
|
||||
@@ -107,7 +110,9 @@ class TestNoServerArgsMutationEntry(CustomTestCase):
|
||||
|
||||
with self.assertRaises(ValueError):
|
||||
declare_late_resolution(published, "test", tp_size=4)
|
||||
self.assertEqual(published.tp_size, 2)
|
||||
# The refusal left the resolution alone: the hook's declaration stands,
|
||||
# and the record still carries the operator's input.
|
||||
self.assertEqual(resolution_result(published, "tp_size"), 2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user