[CI] Derive registered-test kind from the registry call instead of the path (#40294)

This commit is contained in:
Liangsheng Yin
2026-09-20 02:01:22 -07:00
committed by GitHub
parent 671630abf1
commit 0024efa0de
24 changed files with 41 additions and 744 deletions
+7 -11
View File
@@ -72,18 +72,14 @@ Parameters: `est_time` (seconds), `stage` + `runner_config` (target stage and ru
Keep `est_time`, `stage`, `runner_config` as **literal values** — `run_suite.py` collects them by AST parsing.
New and renamed non-kernel tests use this layout:
```text
test/registered/<kind>/<subsystem>/test_*.py
```
`<kind>` is one of `unit`, `e2e`, `accuracy`, `perf`, or `stress`. Kernel tests
use `test/registered/kernels/{ops,benchmark}/<group>/`, retaining the established
Directories under `test/registered/` group tests by topic and are free-form
(`lora/`, `hicache/`, `disaggregation/`, `perf/`, ...); unit tests cover one srt
module, so they mirror the source tree under `unit/`. What a test costs, which
stage gates it and which runner it needs are declared by its `register_*_ci`
call -- including hardware, which is expressed by one or more `register_*_ci`
calls and never by a new top-level directory. Kernel tests use
`test/registered/kernels/{ops,benchmark}/<group>/`, retaining the established
plural `kernels` root.
Hardware is expressed by one or more `register_*_ci` calls, never by creating a
new top-level hardware directory. The admission checker applies the layout and
kind/suite contract incrementally while legacy paths are migrated.
Diffusion workflows also enter through `test/run_suite.py`; registered bridge
files preserve their case-level pytest partitioning until the remaining
@@ -3,7 +3,7 @@
Covers:
1. Triton LSE combine kernel correctness vs CPU reference (base-e and base-2)
2. Various DCP world sizes (N=1,2,4,8)
3. Edge cases: single shard, dominant LSE, equal LSE, NaN/inf
3. Edge cases: single shard, dominant LSE, equal LSE
4. return_lse mode
5. dcp_a2a_lse_reduce with pre-allocated CUDA graph buffers
"""
@@ -239,31 +239,8 @@ class TestLSECombineEdgeCases(CustomTestCase):
)
class TestCPUReference(CustomTestCase):
"""Test the CPU reference implementation independently."""
def test_basic_combine(self):
from sglang.kernels.ops.attention.dcp_kernels import _lse_weighted_combine_cpu
N, B, H, D = 2, 2, 4, 8
outputs = torch.randn(N, B, H, D)
lses = torch.randn(N, B, H)
result = _lse_weighted_combine_cpu(outputs, lses, is_lse_base_on_e=True)
self.assertEqual(result.shape, (B, H, D))
self.assertFalse(torch.isnan(result).any())
def test_base2_vs_base_e(self):
from sglang.kernels.ops.attention.dcp_kernels import _lse_weighted_combine_cpu
N, B, H, D = 2, 2, 4, 8
outputs = torch.randn(N, B, H, D)
lses = torch.randn(N, B, H) * 3.0
result_e = _lse_weighted_combine_cpu(outputs, lses, is_lse_base_on_e=True)
result_2 = _lse_weighted_combine_cpu(outputs, lses, is_lse_base_on_e=False)
self.assertFalse(torch.allclose(result_e, result_2, atol=1e-3))
class TestLSEBaseByBackend(CustomTestCase):
"""Which attention backends report LSE in natural log."""
def test_natural_log_lse_backends(self):
from sglang.srt.models.deepseek_common.attention_forward_methods.forward_mla import (
@@ -277,26 +254,6 @@ class TestCPUReference(CustomTestCase):
self.assertFalse(is_mla_dcp_lse_base_on_e("trtllm_mla"))
self.assertFalse(is_mla_dcp_lse_base_on_e(None))
def test_nan_lse_handled(self):
from sglang.kernels.ops.attention.dcp_kernels import _lse_weighted_combine_cpu
N, B, H, D = 2, 1, 1, 8
outputs = torch.randn(N, B, H, D)
lses = torch.tensor([[[5.0]], [[float("nan")]]])
result = _lse_weighted_combine_cpu(outputs, lses, is_lse_base_on_e=True)
self.assertFalse(torch.isnan(result).any())
def test_inf_lse_handled(self):
from sglang.kernels.ops.attention.dcp_kernels import _lse_weighted_combine_cpu
N, B, H, D = 2, 1, 1, 8
outputs = torch.randn(N, B, H, D)
lses = torch.tensor([[[5.0]], [[float("inf")]]])
result = _lse_weighted_combine_cpu(outputs, lses, is_lse_base_on_e=True)
self.assertFalse(torch.isnan(result).any())
class TestDCPA2AReduceWithCUDAGraphBuffers(CustomTestCase):
"""Test dcp_a2a_lse_reduce with pre-allocated CUDA graph buffers."""
@@ -368,40 +325,6 @@ class TestDCPA2AReduceWithCUDAGraphBuffers(CustomTestCase):
rtol=1e-5,
)
def test_cuda_graph_buffers_n4(self):
from sglang.srt.layers.dcp import dcp_a2a_lse_reduce
torch.manual_seed(456)
N, B, H_per_rank, D = 4, 2, 4, 64
H = H_per_rank * N
max_bs = 8
group = self._make_mock_group(N)
attn_out = torch.randn(B, H, D, device=self.device, dtype=torch.bfloat16)
attn_lse = torch.randn(B, H, device=self.device, dtype=torch.float32)
result_dynamic = dcp_a2a_lse_reduce(
attn_out.clone(), attn_lse.clone(), group, is_lse_base_on_e=True
)
cuda_graph_buffers = self._make_cuda_graph_buffers(N, max_bs, H_per_rank, D)
result_graph = dcp_a2a_lse_reduce(
attn_out.clone(),
attn_lse.clone(),
group,
is_lse_base_on_e=True,
cuda_graph_buffers=cuda_graph_buffers,
)
torch.testing.assert_close(
result_graph.float().cpu(),
result_dynamic.float().cpu(),
atol=1e-5,
rtol=1e-5,
)
def test_cuda_graph_buffers_partial_batch(self):
"""Buffer max_bs > actual B -- should correctly slice."""
from sglang.srt.layers.dcp import dcp_a2a_lse_reduce
@@ -429,29 +352,6 @@ class TestDCPA2AReduceWithCUDAGraphBuffers(CustomTestCase):
self.assertEqual(result.shape, (B, H_per_rank, D))
self.assertFalse(torch.isnan(result).any())
def test_a2a_reduce_allocates_when_no_buffers(self):
"""Without cuda_graph_buffers, dcp_a2a_lse_reduce still works (eager mode)."""
from sglang.srt.layers.dcp import dcp_a2a_lse_reduce
N, B, H_per_rank, D = 2, 4, 8, 64
H = H_per_rank * N
group = self._make_mock_group(N)
attn_out = torch.randn(B, H, D, device=self.device, dtype=torch.bfloat16)
attn_lse = torch.randn(B, H, device=self.device, dtype=torch.float32)
result = dcp_a2a_lse_reduce(
attn_out,
attn_lse,
group,
is_lse_base_on_e=True,
cuda_graph_buffers=None,
)
self.assertEqual(result.shape, (B, H_per_rank, D))
self.assertFalse(torch.isnan(result).any())
def test_pack_matches_the_copy_formulation_it_replaces(self):
from sglang.kernels.ops.attention.dcp_kernels import (
_lse_pack_dim,
@@ -1,315 +0,0 @@
"""
End-to-end tests for strict reasoning + constrained decoding.
Tests that the full pipeline works:
- AC-5.1: Strict reasoning + JSON schema constrained generation
- AC-5.2: Strict reasoning + tool call parsing (basic validation only)
These tests launch a real server with a small model and verify
the constrained decoding pipeline produces valid output.
"""
import json
import unittest
import requests
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
CustomTestCase,
popen_launch_server,
)
register_cuda_ci(est_time=96, stage="base-b", runner_config="1-gpu-small")
register_amd_ci(est_time=120, suite="stage-b-test-1-gpu-small-amd")
MODEL = "Qwen/Qwen3-0.6B"
BASE_URL = "http://127.0.0.1:39877"
API_KEY = "sk-test-1234"
class TestConstrainedReasoningE2E(CustomTestCase):
@classmethod
def setUpClass(cls):
cls.model = MODEL
cls.base_url = BASE_URL
cls.api_key = API_KEY
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
api_key=cls.api_key,
other_args=[
"--reasoning-parser",
"qwen3",
],
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def _chat(self, **kwargs):
default = {
"model": self.model,
"messages": [
{
"role": "user",
"content": "What is 2+2? Answer with just the number.",
}
],
"temperature": 0,
"max_tokens": 256,
}
default.update(kwargs)
resp = requests.post(
f"{self.base_url}/v1/chat/completions",
headers={"Authorization": f"Bearer {self.api_key}"},
json=default,
timeout=60,
)
self.assertEqual(resp.status_code, 200, f"Request failed: {resp.text}")
return resp.json()
def test_reasoning_with_json_schema(self):
"""AC-5.1: Reasoning + JSON schema produces valid JSON output."""
schema = {
"type": "object",
"properties": {
"answer": {"type": "integer"},
},
"required": ["answer"],
}
data = self._chat(
response_format={
"type": "json_schema",
"json_schema": {
"name": "answer_schema",
"schema": schema,
},
},
chat_template_kwargs={"enable_thinking": True},
separate_reasoning=True,
)
choice = data["choices"][0]
content = choice["message"]["content"] or ""
# Content should be valid JSON conforming to schema when non-empty.
# With small models + separate_reasoning, content may be empty if the
# model puts everything in reasoning_content. That's acceptable.
if content.strip():
try:
parsed = json.loads(content)
self.assertIn("answer", parsed)
self.assertIsInstance(parsed["answer"], int)
except (json.JSONDecodeError, TypeError):
# Small models may produce imperfect JSON
self.assertTrue(
content.strip().startswith("{"),
f"Expected JSON-like output, got: {content!r}",
)
# Content should NOT contain <think> tags (those go to reasoning_content)
self.assertNotIn("<think>", content)
def test_reasoning_disabled_with_json_schema(self):
"""JSON schema still works when reasoning is explicitly disabled."""
schema = {
"type": "object",
"properties": {
"answer": {"type": "integer"},
},
"required": ["answer"],
}
data = self._chat(
response_format={
"type": "json_schema",
"json_schema": {
"name": "answer_schema",
"schema": schema,
},
},
chat_template_kwargs={"enable_thinking": False},
)
choice = data["choices"][0]
content = choice["message"]["content"]
# Should still produce valid JSON
parsed = json.loads(content)
self.assertIn("answer", parsed)
def test_reasoning_with_separate_output(self):
"""Reasoning content is correctly separated from normal content."""
data = self._chat(
chat_template_kwargs={"enable_thinking": True},
separate_reasoning=True,
)
choice = data["choices"][0]
content = choice["message"]["content"]
reasoning = choice["message"].get("reasoning_content")
# Content should not contain think tags
self.assertNotIn("<think>", content)
self.assertNotIn("</think>", content)
def test_tool_call_after_reasoning(self):
"""AC-5.2: Tool call parsing works with reasoning enabled."""
tools = [
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get the current weather",
"parameters": {
"type": "object",
"properties": {
"location": {"type": "string"},
},
"required": ["location"],
},
},
}
]
data = self._chat(
messages=[
{
"role": "user",
"content": "What's the weather in Paris?",
}
],
tools=tools,
chat_template_kwargs={"enable_thinking": True},
separate_reasoning=True,
)
choice = data["choices"][0]
# The model may or may not produce tool calls (depends on model capability)
# but the response should be well-formed (no crashes)
self.assertIn("message", choice)
self.assertIn("finish_reason", choice)
# finish_reason should be either "stop" or "tool_calls"
self.assertIn(choice["finish_reason"], ["stop", "tool_calls", "length"])
class TestStrictThinkingE2E(CustomTestCase):
"""E2E tests with --enable-strict-thinking flag.
Validates that the strict thinking flag is correctly propagated through
the full pipeline: server_args -> grammar_backend -> ReasonerGrammarBackend
-> token filtering during thinking phase.
"""
@classmethod
def setUpClass(cls):
cls.model = MODEL
cls.base_url = "http://127.0.0.1:39878"
cls.api_key = API_KEY
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
api_key=cls.api_key,
other_args=[
"--reasoning-parser",
"qwen3",
"--enable-strict-thinking",
],
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def _chat(self, **kwargs):
default = {
"model": self.model,
"messages": [
{
"role": "user",
"content": "What is 2+2? Answer with just the number.",
}
],
"temperature": 0,
"max_tokens": 256,
}
default.update(kwargs)
resp = requests.post(
f"{self.base_url}/v1/chat/completions",
headers={"Authorization": f"Bearer {self.api_key}"},
json=default,
timeout=60,
)
self.assertEqual(resp.status_code, 200, f"Request failed: {resp.text}")
return resp.json()
def test_strict_thinking_with_json_schema(self):
"""Strict thinking + JSON schema: server starts and produces valid output."""
schema = {
"type": "object",
"properties": {
"answer": {"type": "integer"},
},
"required": ["answer"],
}
data = self._chat(
response_format={
"type": "json_schema",
"json_schema": {
"name": "answer_schema",
"schema": schema,
},
},
chat_template_kwargs={"enable_thinking": True},
separate_reasoning=True,
)
choice = data["choices"][0]
content = choice["message"]["content"] or ""
if content.strip():
try:
parsed = json.loads(content)
self.assertIn("answer", parsed)
except (json.JSONDecodeError, TypeError):
self.assertTrue(
content.strip().startswith("{"),
f"Expected JSON-like output, got: {content!r}",
)
# Think tags must not leak into content
self.assertNotIn("<think>", content)
def test_strict_thinking_disabled_per_request(self):
"""When thinking is disabled per-request, strict server still works."""
data = self._chat(
chat_template_kwargs={"enable_thinking": False},
)
choice = data["choices"][0]
self.assertIn("message", choice)
self.assertIn("finish_reason", choice)
# Should complete normally without errors
self.assertIn(choice["finish_reason"], ["stop", "length"])
def test_strict_thinking_separate_reasoning(self):
"""Strict thinking with separate_reasoning produces well-formed output."""
data = self._chat(
chat_template_kwargs={"enable_thinking": True},
separate_reasoning=True,
)
choice = data["choices"][0]
content = choice["message"]["content"] or ""
# Think tags must not leak into content
self.assertNotIn("<think>", content)
self.assertNotIn("</think>", content)
if __name__ == "__main__":
unittest.main()
@@ -445,16 +445,13 @@ def test_fused_moe_compile_hook_is_bs1_only():
# --- tracing --------------------------------------------------------------------
def test_trace_labels_platform_and_backend(monkeypatch):
def test_trace_labels_explicit_backend(monkeypatch):
_mock_platform(monkeypatch, key="cuda", info=_CUDA)
op = _CudaOnlyPlatformOp()
fo.enable_fused_op_trace()
op(torch.zeros(2, 3))
op(torch.zeros(2, 3), backend=KernelBackend.TORCH)
auto_rec, explicit_rec = fo.get_fused_op_trace()
assert auto_rec.op == "test.cuda_only_platform"
assert auto_rec.backend == "cuda"
assert auto_rec.tensor_args == ("torch.float32[2, 3]",)
_, explicit_rec = fo.get_fused_op_trace()
assert explicit_rec.backend == "torch"
@@ -512,46 +509,6 @@ def test_deprecated_alias_keeps_legacy_platform_defaults(monkeypatch):
_NativeOnlyLegacy()(torch.zeros(1)) # old CUDA behavior preserved
# --- migration completeness -------------------------------------------------------
_MIGRATED_OPS = [
("sglang.srt.layers.activation", "SiluAndMul"),
("sglang.srt.layers.activation", "GeluAndMul"),
("sglang.srt.layers.activation", "NewGELU"),
("sglang.srt.layers.activation", "ReLU2"),
("sglang.srt.layers.activation", "QuickGELU"),
("sglang.srt.layers.activation", "XIELU"),
("sglang.srt.layers.layernorm", "RMSNorm"),
("sglang.srt.layers.layernorm", "LayerNorm"),
("sglang.srt.layers.layernorm", "GemmaRMSNorm"),
("sglang.srt.layers.layernorm", "Gemma3RMSNorm"),
("sglang.srt.layers.layernorm", "Gemma4RMSNorm"),
("sglang.srt.layers.layernorm", "RMSNormWithoutScale"),
("sglang.srt.layers.conv", "Conv2dLayer"),
("sglang.srt.layers.conv", "Conv3dLayer"),
("sglang.srt.layers.moe.topk", "TopK"),
("sglang.srt.layers.rotary_embedding.base", "RotaryEmbedding"),
("sglang.srt.layers.rotary_embedding.rope_variant", "DualChunkRotaryEmbedding"),
("sglang.srt.layers.attention.dsa.dsa_indexer", "Indexer"),
("sglang.srt.layers.attention.dsv4.compressor", "Compressor"),
("sglang.srt.layers.attention.mamba.mixer2_rms_norm_gated", "Mixer2RMSNormGated"),
("sglang.srt.layers.quantization.unquant", "UnquantizedFusedMoEMethod"),
]
@pytest.mark.parametrize("module_name, cls_name", _MIGRATED_OPS)
def test_migrated_ops_subclass_base_fused_op(module_name, cls_name):
"""Production ops must extend BaseFusedOp directly, never the deprecated
MultiPlatformOp alias (which exists only for out-of-tree users)."""
import importlib
from sglang.srt.layers.utils.multi_platform import MultiPlatformOp
cls = getattr(importlib.import_module(module_name), cls_name)
assert issubclass(cls, BaseFusedOp)
assert MultiPlatformOp not in cls.__mro__
if __name__ == "__main__":
import sys