chore: cleanup garbage code (#29770)
This commit is contained in:
@@ -131,7 +131,7 @@ def run_baseline(inp):
|
||||
|
||||
def run_packed(inp):
|
||||
"""Packed path: single fused kernel directly on mixed_qkv."""
|
||||
B, HV, K, V = inp["B"], inp["HV"], inp["K"], inp["V"]
|
||||
B, HV, V = inp["B"], inp["HV"], inp["V"]
|
||||
ssm_states = inp["ssm_states"].clone()
|
||||
out = inp["mixed_qkv"].new_empty(B, 1, HV, V)
|
||||
|
||||
@@ -173,7 +173,7 @@ def check_correctness(B, H, HV, K, V, pool_size, device, dtype, seed=42):
|
||||
try:
|
||||
torch.testing.assert_close(o_packed, o_baseline, atol=atol, rtol=rtol)
|
||||
output_ok = True
|
||||
except AssertionError as e:
|
||||
except AssertionError:
|
||||
output_ok = False
|
||||
out_diff = (o_packed - o_baseline).abs().max().item()
|
||||
|
||||
|
||||
@@ -171,7 +171,7 @@ if __name__ == "__main__":
|
||||
"--limit",
|
||||
type=int,
|
||||
default=None,
|
||||
help="If set, only evaluate this many samples (debug / smoke runs).",
|
||||
help="If set, only evaluate this many samples (debug / quick runs).",
|
||||
)
|
||||
EvalArgs.add_cli_args(parser)
|
||||
args = parser.parse_args()
|
||||
|
||||
@@ -32,12 +32,12 @@ cargo bench --bench radix_tree_benchmark -- --sample-size 30 --measurement-time
|
||||
cargo bench --bench manual_policy_benchmark
|
||||
```
|
||||
|
||||
For the quick smoke runs whose numbers are reproduced below: drop
|
||||
For the quick runs whose numbers are reproduced below: drop
|
||||
`--sample-size` to 10 and `--measurement-time` to 2 (Criterion will
|
||||
warn about reduced statistical confidence but the order-of-magnitude
|
||||
comparison stands).
|
||||
|
||||
## Smoke-run data points (M1 MacBook, release profile)
|
||||
## Quick-run Data Points (M1 MacBook, release profile)
|
||||
|
||||
These are NOT the real acceptance numbers — they're a sanity check
|
||||
that the sgl-router routing primitives are in the same ballpark as the
|
||||
|
||||
@@ -680,7 +680,7 @@ mod tests {
|
||||
let _ = manager_handle.await;
|
||||
}
|
||||
|
||||
/// End-to-end wiring smoke test: spin up a fake worker, run the
|
||||
/// End-to-end wiring check: spin up a fake worker, run the
|
||||
/// manager with a real `KvEventIndex` against that worker URL, and
|
||||
/// verify both `Added` and `Removed` propagate through to the
|
||||
/// index's internal worker map.
|
||||
|
||||
@@ -7,7 +7,7 @@ The shape:
|
||||
- non-streaming + streaming chat completion
|
||||
- assistant message non-empty, role correct, finish_reason set
|
||||
|
||||
These are the smoke tests that run first; if they pass, the heavier
|
||||
These quick checks run first; if they pass, the heavier
|
||||
multi-worker acceptance tests are worth running.
|
||||
"""
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
Two flavors of fixtures coexist here:
|
||||
|
||||
1. **Session-scoped smoke fixtures** (``sglang_server`` + ``router``) —
|
||||
1. **Session-scoped sanity fixtures** (``sglang_server`` + ``router``) —
|
||||
launch ONE SGLang worker + ONE router on fixed ports for the whole
|
||||
test session. Used by the lightweight ``test_chat_smoke.py`` /
|
||||
``test_tokenize_smoke.py`` files. These are the cheap "did the
|
||||
@@ -174,7 +174,7 @@ def build_smoke_router_args(
|
||||
tokenizer_path: str,
|
||||
sglang_url: str,
|
||||
) -> list[str]:
|
||||
"""Build the sgl-router CLI flags the smoke ``router`` fixture launches.
|
||||
"""Build the sgl-router CLI flags the single-worker ``router`` fixture launches.
|
||||
|
||||
Static single-worker discovery (``--worker-urls``) pointed at the one
|
||||
SGLang worker, serving exactly one model.
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
minversion = "8.0"
|
||||
# Default discovery: smoke tests (top-level test_*.py) and the
|
||||
# Default discovery: top-level test_*.py files and the
|
||||
# multi-worker chat_completions suite. k8s_integration is intentionally
|
||||
# not in the default set — it requires a kind/k8s cluster and is
|
||||
# invoked explicitly.
|
||||
|
||||
@@ -9,7 +9,7 @@ pytest-asyncio==0.24.0
|
||||
# pin of `huggingface_hub==0.26.2` here got installed AFTER the SGLang
|
||||
# deps and downgraded huggingface_hub past `is_offline_mode`'s top-
|
||||
# level export, which broke `from sglang.srt.server_args import …`
|
||||
# at module import time and turned every smoke test into a 5-minute
|
||||
# at module import time and turned every quick check into a 5-minute
|
||||
# `/health` timeout with no actionable signal.
|
||||
# The e2e suite only uses huggingface_hub's `try_to_load_from_cache`,
|
||||
# which is available in every release SGLang would install.
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
//! (existing code path; pinned here so a future PD wiring change
|
||||
//! doesn't silently swap codes).
|
||||
//! * A PD-disagg model with both pools healthy → request flows to the
|
||||
//! prefill worker (smoke; the decode worker MUST NOT be selected for
|
||||
//! prefill worker (sanity check; the decode worker MUST NOT be selected for
|
||||
//! the chat route).
|
||||
|
||||
use axum::body::Body;
|
||||
|
||||
@@ -52,7 +52,6 @@ class CustomOp(nn.Module):
|
||||
def forward_tpu(self, *args, **kwargs) -> Any:
|
||||
# By default, we assume that TPU ops are compatible with the
|
||||
# PyTorch-native implementation.
|
||||
# NOTE(woosuk): This is a placeholder for future extensions.
|
||||
return self.forward_native(*args, **kwargs)
|
||||
|
||||
def forward_musa(self, *args, **kwargs) -> Any:
|
||||
|
||||
@@ -854,7 +854,7 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
prompt_ids = self.tokenizer_manager.tokenizer.encode(
|
||||
rendered_prompt, **encode_kwargs
|
||||
)
|
||||
except Exception as e:
|
||||
except Exception:
|
||||
# If the first attempt fails, try with flat function-only format.
|
||||
# Some templates (e.g. Mistral) expect tools without the OpenAI wrapper.
|
||||
tools = (
|
||||
|
||||
@@ -1689,7 +1689,7 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
)
|
||||
|
||||
async def empty_async_generator():
|
||||
if False:
|
||||
for _ in ():
|
||||
yield
|
||||
|
||||
final_response = await self.responses_full_generator(
|
||||
|
||||
@@ -801,13 +801,9 @@ class _DetailAccumulator(_UtilizationRateAccumulatorMixin):
|
||||
self._records = []
|
||||
|
||||
def get_single_pass_gatherer_keys(self):
|
||||
if False: # TODO `server_args.enable_two_batch_overlap`
|
||||
return [_SINGLE_PASS_GATHERER_KEY_PRIMARY, "child_a", "child_b"]
|
||||
return super().get_single_pass_gatherer_keys()
|
||||
|
||||
def get_single_pass_gatherer_key(self, debug_name: Optional[str]):
|
||||
if False: # TODO `server_args.enable_two_batch_overlap`
|
||||
return debug_name or _SINGLE_PASS_GATHERER_KEY_PRIMARY
|
||||
return super().get_single_pass_gatherer_key(debug_name)
|
||||
|
||||
def append(
|
||||
|
||||
@@ -280,14 +280,8 @@ class XIELU(MultiPlatformOp):
|
||||
)
|
||||
self._xielu_cuda_fn = self._xielu_cuda
|
||||
logger.warning_once(msg)
|
||||
except Exception as err:
|
||||
except Exception:
|
||||
pass
|
||||
# logger.warning_once(
|
||||
# "CUDA-fused xIELU not available (%s) –"
|
||||
# " falling back to a Python version.\n"
|
||||
# "For CUDA xIELU (experimental), `pip install git+https://github.com/nickjbrowning/XIELU`",
|
||||
# str(err),
|
||||
# )
|
||||
|
||||
def _xielu_python(self, x: torch.Tensor) -> torch.Tensor:
|
||||
alpha_p = nn.functional.softplus(self.alpha_p)
|
||||
|
||||
@@ -253,135 +253,11 @@ class GetKAndS:
|
||||
)
|
||||
|
||||
|
||||
class SetK:
|
||||
@classmethod
|
||||
def execute(cls, *args, buf, **kwargs):
|
||||
return cls.torch_fast(*args, **kwargs, buf=buf)
|
||||
|
||||
@classmethod
|
||||
def slow(
|
||||
cls,
|
||||
pool: "DSATokenToKVPool",
|
||||
buf: torch.Tensor,
|
||||
loc: torch.Tensor,
|
||||
index_k: torch.Tensor,
|
||||
):
|
||||
for i in range(len(loc)):
|
||||
page_index = loc[i] // pool.page_size
|
||||
offset = loc[i] % pool.page_size
|
||||
buf[
|
||||
page_index,
|
||||
offset * pool.index_head_dim : (offset + 1) * pool.index_head_dim,
|
||||
] = index_k[i].view(torch.uint8)
|
||||
|
||||
@classmethod
|
||||
def torch_fast(
|
||||
cls,
|
||||
pool: "DSATokenToKVPool",
|
||||
buf: torch.Tensor,
|
||||
loc: torch.Tensor,
|
||||
index_k: torch.Tensor,
|
||||
):
|
||||
(num_tokens_to_write,) = loc.shape
|
||||
buf_numel_per_page = buf.shape[1]
|
||||
num_k_bytes_per_token = pool.index_head_dim
|
||||
|
||||
# loc: (num_tokens_to_write,), int32, element := the token index to write to
|
||||
loc_page_index = loc // pool.page_size
|
||||
loc_token_offset_in_page = loc % pool.page_size
|
||||
|
||||
flat_buf = buf.flatten()
|
||||
flat_indices = (
|
||||
(loc_page_index * buf_numel_per_page)[:, None]
|
||||
+ (loc_token_offset_in_page * num_k_bytes_per_token)[:, None]
|
||||
+ torch.arange(num_k_bytes_per_token, dtype=torch.int32, device="cuda")[
|
||||
None, :
|
||||
]
|
||||
)
|
||||
num_k_bytes_total = num_tokens_to_write * num_k_bytes_per_token
|
||||
flat_indices = flat_indices.flatten()[:num_k_bytes_total]
|
||||
flat_buf[flat_indices] = index_k.view(torch.uint8).flatten()
|
||||
|
||||
|
||||
class SetS:
|
||||
@classmethod
|
||||
def execute(cls, *args, buf, **kwargs):
|
||||
return cls.torch_fast(*args, **kwargs, buf=buf)
|
||||
|
||||
@classmethod
|
||||
def slow(
|
||||
cls,
|
||||
pool: "DSATokenToKVPool",
|
||||
buf: torch.Tensor,
|
||||
loc: torch.Tensor,
|
||||
index_k_scale: torch.Tensor,
|
||||
):
|
||||
for i in range(len(loc)):
|
||||
page_index = loc[i] // pool.page_size
|
||||
offset = loc[i] % pool.page_size
|
||||
start = pool.page_size * pool.index_head_dim
|
||||
buf[page_index, start + offset * 4 : start + (offset + 1) * 4] = (
|
||||
index_k_scale[i].view(torch.uint8)
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def torch_fast(
|
||||
cls,
|
||||
pool: "DSATokenToKVPool",
|
||||
buf: torch.Tensor,
|
||||
loc: torch.Tensor,
|
||||
index_k_scale: torch.Tensor,
|
||||
):
|
||||
(num_tokens_to_write,) = loc.shape
|
||||
buf_numel_per_page = buf.shape[1]
|
||||
num_s_bytes_per_token = 4
|
||||
s_offset_in_page = pool.page_size * pool.index_head_dim
|
||||
|
||||
# loc: (num_tokens_to_write,), int32, element := the token index to write to
|
||||
loc_page_index = loc // pool.page_size
|
||||
loc_token_offset_in_page = loc % pool.page_size
|
||||
|
||||
flat_buf = buf.flatten()
|
||||
flat_indices = (
|
||||
(loc_page_index * buf_numel_per_page)[:, None]
|
||||
+ s_offset_in_page
|
||||
+ (loc_token_offset_in_page * num_s_bytes_per_token)[:, None]
|
||||
+ torch.arange(num_s_bytes_per_token, dtype=torch.int32, device="cuda")[
|
||||
None, :
|
||||
]
|
||||
)
|
||||
number_s_bytes_total = num_tokens_to_write * num_s_bytes_per_token
|
||||
flat_indices = flat_indices.flatten()[:number_s_bytes_total]
|
||||
flat_buf[flat_indices] = index_k_scale.view(torch.uint8).flatten()
|
||||
|
||||
|
||||
class SetKAndS:
|
||||
@classmethod
|
||||
def execute(cls, *args, buf, **kwargs):
|
||||
if 0:
|
||||
# print("SetK, SetS comparison test")
|
||||
buf_cloned = buf.clone()
|
||||
cls.vanilla(*args, **kwargs, buf=buf)
|
||||
cls.triton(*args, **kwargs, buf=buf_cloned)
|
||||
|
||||
def _clear_token_0(target):
|
||||
target[0, :128] = target[0, 64 * 128 : 64 * 128 + 4] = 0
|
||||
|
||||
_clear_token_0(buf)
|
||||
_clear_token_0(buf_cloned)
|
||||
|
||||
assert torch.all(
|
||||
buf == buf_cloned
|
||||
), f"{buf=} {buf_cloned=} {kwargs['loc'].to_list()=}"
|
||||
return
|
||||
|
||||
cls.triton(*args, **kwargs, buf=buf)
|
||||
|
||||
@classmethod
|
||||
def vanilla(cls, pool, buf, loc, index_k, index_k_scale):
|
||||
SetK.execute(pool=pool, buf=buf, loc=loc, index_k=index_k)
|
||||
SetS.execute(pool=pool, buf=buf, loc=loc, index_k_scale=index_k_scale)
|
||||
|
||||
@classmethod
|
||||
def triton(cls, pool, buf, loc, index_k, index_k_scale):
|
||||
loc = loc.to(torch.int64)
|
||||
|
||||
@@ -252,8 +252,7 @@ def pre_permute_standard_to_triton(
|
||||
running_state: dict,
|
||||
) -> TritonRunnerInput:
|
||||
|
||||
# NOTE: this is dead code as a fused func for standard format is registered.
|
||||
# This is left here for testing and examples.
|
||||
# Registered fallback for format-conversion tests and examples.
|
||||
|
||||
from sglang.srt.layers.moe.moe_runner.triton_utils.fused_moe import (
|
||||
_prepare_fused_moe_run,
|
||||
@@ -309,8 +308,7 @@ def post_permute_triton_to_standard(
|
||||
running_state: dict,
|
||||
) -> StandardCombineInput:
|
||||
|
||||
# NOTE: this is dead code as a fused func for standard format is registered.
|
||||
# This is left here for testing and examples.
|
||||
# Registered fallback for format-conversion tests and examples.
|
||||
|
||||
from sglang.srt.layers.moe.token_dispatcher.standard import StandardCombineInput
|
||||
|
||||
|
||||
@@ -195,7 +195,7 @@ if _use_aiter:
|
||||
if _is_musa:
|
||||
try:
|
||||
from mate import moe_fused_gate
|
||||
except ImportError as e:
|
||||
except ImportError:
|
||||
raise ImportError("mate is required for the biased grouped topk.")
|
||||
|
||||
from sglang.srt.hardware_backend.musa.kernels.topk import topk_sigmoid, topk_softmax
|
||||
|
||||
@@ -94,11 +94,7 @@ def hash_tiles32_kernel_blocked(
|
||||
h1 ^= nbytes
|
||||
h2 ^= nbytes
|
||||
h1 = _fmix32(h1, C1=FM_C1, C2=FM_C2)
|
||||
h2 = (
|
||||
_fmix32(h2, C1=FMIX32_C1, C2=FMIX32_C2)
|
||||
if False
|
||||
else _fmix32(h2, C1=FM_C1, C2=FM_C2)
|
||||
)
|
||||
h2 = _fmix32(h2, C1=FM_C1, C2=FM_C2)
|
||||
|
||||
out = (h1.to(tl.uint64) << 32) | h2.to(tl.uint64)
|
||||
tl.store(out_ptr + pid, out)
|
||||
|
||||
@@ -9,7 +9,7 @@ import torch
|
||||
|
||||
try:
|
||||
from aiter.ops.triton.quant import dynamic_mxfp4_quant
|
||||
except ImportError as err:
|
||||
except ImportError:
|
||||
|
||||
def raise_aiter_import_error(*args, **kwargs):
|
||||
raise ImportError(
|
||||
|
||||
@@ -68,7 +68,7 @@ def init_feature_buffer(device):
|
||||
num_elements, dtype=torch.float32, device=device
|
||||
)
|
||||
logger.info(f"Preallocated {size_mb}MB GPU buffer")
|
||||
except RuntimeError as e:
|
||||
except RuntimeError:
|
||||
_GPU_FEATURE_BUFFER = None
|
||||
|
||||
|
||||
@@ -160,7 +160,7 @@ class TransportProxyTensor(torch.Tensor):
|
||||
"storage_offset": self.storage_offset(),
|
||||
}
|
||||
state["tensor_data"] = None
|
||||
except Exception as e:
|
||||
except Exception:
|
||||
# Failed to get CUDA IPC handle (possibly tp). Falling back to default transport.
|
||||
state["metadata"]["transport_mode"] = "default"
|
||||
state["tensor_data"] = self.as_subclass(torch.Tensor)
|
||||
|
||||
@@ -1064,7 +1064,7 @@ class ModelRunner(ModelRunnerKVCacheMixin):
|
||||
def remote_instance_init_transfer_engine(self):
|
||||
try:
|
||||
from mooncake.engine import TransferEngine
|
||||
except ImportError as e:
|
||||
except ImportError:
|
||||
logger.warning(
|
||||
"Please install mooncake for using remote instance transfer engine: pip install mooncake-transfer-engine"
|
||||
)
|
||||
|
||||
@@ -531,22 +531,15 @@ class HunYuanModel(nn.Module):
|
||||
hidden_states = self.get_input_embeddings(input_ids)
|
||||
residual = None
|
||||
|
||||
prev_kv_states = None
|
||||
for i in range(len(self.layers)):
|
||||
layer = self.layers[i]
|
||||
hidden_states, residual, kv_states = layer(
|
||||
for layer in self.layers:
|
||||
hidden_states, residual, _ = layer(
|
||||
positions,
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
residual,
|
||||
prev_kv_states,
|
||||
None,
|
||||
)
|
||||
|
||||
if False: # (i - self.start_layer) % cla_factor == 0:
|
||||
prev_kv_states = kv_states
|
||||
else:
|
||||
prev_kv_states = None
|
||||
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
return hidden_states
|
||||
|
||||
|
||||
@@ -213,7 +213,7 @@ def process_anyres_image(image, processor, grid_pinpoints):
|
||||
if isinstance(grid_pinpoints, str) and "x" in grid_pinpoints:
|
||||
try:
|
||||
patch_size = processor.size[0]
|
||||
except Exception as e:
|
||||
except Exception:
|
||||
patch_size = processor.size["shortest_edge"]
|
||||
assert patch_size in [
|
||||
224,
|
||||
|
||||
@@ -1857,7 +1857,7 @@ def get_npu_memory_capacity():
|
||||
return envs.SGLANG_ZBAL_LOCAL_MEM_SIZE.get() # unit: MB
|
||||
else:
|
||||
return torch.npu.mem_get_info()[1] // 1024 // 1024 # unit: MB
|
||||
except ImportError as e:
|
||||
except ImportError:
|
||||
raise ImportError("torch_npu is required when run on npu device.")
|
||||
|
||||
|
||||
@@ -2210,7 +2210,7 @@ def get_compiler_backend(mode=None) -> str:
|
||||
import torchair
|
||||
import torchair.ge_concrete_graph.ge_converter.experimental.patch_for_hcom_allreduce
|
||||
from torchair.configs.compiler_config import CompilerConfig
|
||||
except ImportError as e:
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"NPU detected, but torchair package is not installed. "
|
||||
"Please install torchair for torch.compile support on NPU."
|
||||
|
||||
@@ -364,7 +364,7 @@ class CudaIpcTensorTransportProxy:
|
||||
"recons_dtype": info_data.dtype,
|
||||
}
|
||||
state["tensor_data"] = None
|
||||
except Exception as e:
|
||||
except Exception:
|
||||
# Failed to get CUDA IPC handle (possibly tp). Falling back to default transport.
|
||||
state["ipc_extra"] = None
|
||||
state["tensor_data"] = data
|
||||
|
||||
@@ -368,11 +368,11 @@ def rpd_to_chrome_trace(
|
||||
if row[0] == "0": # Frame start
|
||||
if key not in stacks:
|
||||
stacks[key] = []
|
||||
stack = stacks[key].append((row[1], row[4]))
|
||||
stacks[key].append((row[1], row[4]))
|
||||
# print(f"0: new api frame: pid_tid={key} -> stack={stacks}")
|
||||
|
||||
elif row[0] == "1": # Frame end
|
||||
completed = stacks[key].pop()
|
||||
stacks[key].pop()
|
||||
# print(f"1: end api frame: pid_tid={key} -> stack={stacks}")
|
||||
|
||||
elif row[0] == "2": # API + Op
|
||||
@@ -400,8 +400,6 @@ def rpd_to_chrome_trace(
|
||||
or abs(gpuFrame.start - row[8]) < 200
|
||||
)
|
||||
):
|
||||
# if gpuFrame.id == frame[0] and gpuFrame.name == frame[1]: # Another op under the same frame -> union them
|
||||
# if False: # Turn off frame joining
|
||||
if row[7] < gpuFrame.start:
|
||||
gpuFrame.start = row[7]
|
||||
if row[8] > gpuFrame.end:
|
||||
|
||||
@@ -779,7 +779,7 @@ def wait_server_ready(url, timeout=LOCAL_TIMEOUT):
|
||||
logger.info(
|
||||
f"Server {url} returned status code: {response.status_code}"
|
||||
)
|
||||
except Exception as e:
|
||||
except Exception:
|
||||
# logger.error(f"Server {url} request error: {e}, retrying...")
|
||||
pass
|
||||
|
||||
|
||||
@@ -1104,7 +1104,7 @@ def _seed_c4_if_needed(
|
||||
fixture: DSV4AttentionFixture, *, num_entries: int = _DSV4_EXTRA_ENTRIES
|
||||
) -> None:
|
||||
"""For compress_ratio=4, seed the C4 metadata the exercised path consumes
|
||||
(the C4Indexer would normally populate it; the smoke fixture skips the
|
||||
(the C4Indexer would normally populate it; the compact fixture skips the
|
||||
indexer): `c4_sparse_page_indices` for the dense extend path,
|
||||
`c4_sparse_raw_indices` for sparse prefill. No-op for other compress_ratios.
|
||||
"""
|
||||
@@ -1383,7 +1383,7 @@ def _seed_c4_sparse_indices(
|
||||
) -> None:
|
||||
"""For compress_ratio=4 the production `init_flashmla_related` initializes
|
||||
`c4_sparse_page_indices` to all `-1` (the C4Indexer fills it in later).
|
||||
Since the smoke fixture does not run the indexer, the C4 path attends to
|
||||
Since the compact fixture does not run the indexer, the C4 path attends to
|
||||
zero extra entries unless we seed the indices ourselves. Seed each query
|
||||
row to point to `[0, 1, ..., num_entries - 1]` so the backend reads the
|
||||
same `num_entries` C4 K's that the reference also reads, exercising the
|
||||
@@ -1614,7 +1614,7 @@ def run_dsv4_compress_attention_case(
|
||||
assert case.compress_ratio in (
|
||||
4,
|
||||
128,
|
||||
), f"smoke runner requires compress_ratio in (4, 128); got {case.compress_ratio}"
|
||||
), f"DSV4 compact runner requires compress_ratio in (4, 128); got {case.compress_ratio}"
|
||||
if sparse_prefill:
|
||||
assert (
|
||||
case.forward_mode.is_extend_without_speculative()
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
Probes that catch the model producing wrong output: weight load
|
||||
failure, sampling path bugs, KV / attention corruption, and cuda graph
|
||||
edge cases. Single-prompt smoke only -- dataset-driven accuracy gates
|
||||
edge cases. Single-prompt coverage only -- dataset-driven accuracy gates
|
||||
belong to the consuming test class, not this kit.
|
||||
|
||||
Mix into any ``CustomTestCase`` subclass that exposes ``self.base_url``
|
||||
|
||||
@@ -175,7 +175,7 @@ class MGSMEval(Eval):
|
||||
]
|
||||
try:
|
||||
response_text = sampler(prompt_messages)
|
||||
except Exception as e:
|
||||
except Exception:
|
||||
response_text = ""
|
||||
|
||||
answer_prefix = LANG_TO_ANSWER_PREFIX[language]
|
||||
|
||||
@@ -150,8 +150,6 @@ else
|
||||
docker cp human-eval ci_sglang:/
|
||||
install_with_retry docker exec -w /human-eval ci_sglang pip install --cache-dir=/sgl-data/pip-cache -e .
|
||||
|
||||
docker exec -w / ci_sglang mkdir -p /dummy-grok
|
||||
# Create dummy grok config inline (bypasses Azure blob storage which may have auth issues)
|
||||
mkdir -p dummy-grok
|
||||
cat > dummy-grok/config.json << 'EOF'
|
||||
{
|
||||
@@ -176,9 +174,8 @@ else
|
||||
"torch_dtype": "bfloat16"
|
||||
}
|
||||
EOF
|
||||
# docker exec -w / ci_sglang mkdir -p /dummy-grok
|
||||
# mkdir -p dummy-grok && wget https://sharkpublic.blob.core.windows.net/sharkpublic/sglang/dummy_grok.json -O dummy-grok/config.json
|
||||
# docker cp ./dummy-grok ci_sglang:/
|
||||
docker exec -w / ci_sglang mkdir -p /dummy-grok
|
||||
docker cp ./dummy-grok/config.json ci_sglang:/dummy-grok/config.json
|
||||
|
||||
docker exec ci_sglang pip install --cache-dir=/sgl-data/pip-cache huggingface_hub[hf_xet]
|
||||
docker exec ci_sglang pip install --cache-dir=/sgl-data/pip-cache pytest
|
||||
|
||||
@@ -143,7 +143,7 @@ def parse_args() -> argparse.Namespace:
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def build_sandbox_image() -> "modal.Image":
|
||||
def build_sandbox_image() -> modal.Image:
|
||||
if modal is None:
|
||||
raise RuntimeError(
|
||||
"The 'modal' package is required. Run this script with "
|
||||
@@ -213,7 +213,7 @@ Do not skip issue filing. The whole point of this system is automated triage.
|
||||
"""
|
||||
|
||||
|
||||
def upload_tree(sandbox: "modal.Sandbox", log_dir: Path) -> None:
|
||||
def upload_tree(sandbox: modal.Sandbox, log_dir: Path) -> None:
|
||||
log_files = [path for path in log_dir.rglob("*") if path.is_file()]
|
||||
logger.info("Uploading %d log files into the sandbox", len(log_files))
|
||||
for index, log_file in enumerate(log_files, start=1):
|
||||
@@ -225,7 +225,7 @@ def upload_tree(sandbox: "modal.Sandbox", log_dir: Path) -> None:
|
||||
logger.info("Uploaded %d/%d files", index, len(log_files))
|
||||
|
||||
|
||||
def clone_context_repos(sandbox: "modal.Sandbox", repo_urls: list[str]) -> None:
|
||||
def clone_context_repos(sandbox: modal.Sandbox, repo_urls: list[str]) -> None:
|
||||
if not repo_urls:
|
||||
return
|
||||
|
||||
@@ -244,7 +244,7 @@ def clone_context_repos(sandbox: "modal.Sandbox", repo_urls: list[str]) -> None:
|
||||
).wait()
|
||||
|
||||
|
||||
def read_optional_file(sandbox: "modal.Sandbox", path: str) -> str | None:
|
||||
def read_optional_file(sandbox: modal.Sandbox, path: str) -> str | None:
|
||||
try:
|
||||
return sandbox.filesystem.read_text(path)
|
||||
except Exception:
|
||||
|
||||
@@ -231,10 +231,10 @@ docker run $DOCKER_COMMON --name mi355x_decode \
|
||||
$COMMON_FLAGS --disaggregation-mode decode --disaggregation-bootstrap-port $DBOOT
|
||||
EOF
|
||||
|
||||
# Smoke-test payload + validator (separate files to avoid quoting inside the
|
||||
# Probe payload + validator (separate files to avoid quoting inside the
|
||||
# bench.sh `bash -lc '...'` block). One real request exercises the full
|
||||
# prefill->decode KV handoff before we commit to the whole sweep.
|
||||
cat > "$WORKDIR/smoke.json" <<'JSON'
|
||||
cat > "$WORKDIR/probe.json" <<'JSON'
|
||||
{"text": "The capital of France is", "sampling_params": {"max_new_tokens": 16, "temperature": 0.0}}
|
||||
JSON
|
||||
cat > "$WORKDIR/assert_nonempty.py" <<'PY'
|
||||
@@ -242,9 +242,9 @@ import sys, json
|
||||
d = json.load(sys.stdin)
|
||||
t = d.get("text", "") if isinstance(d, dict) else ""
|
||||
if not (t and t.strip()):
|
||||
print("[smoke] empty/invalid output:", str(d)[:200])
|
||||
print("[probe] empty/invalid output:", str(d)[:200])
|
||||
sys.exit(1)
|
||||
print("[smoke] ok:", t[:80].replace("\n", " "))
|
||||
print("[probe] ok:", t[:80].replace("\n", " "))
|
||||
PY
|
||||
|
||||
# Bench script runs on the prefill node; \$PIP/\$DIP injected at srun time.
|
||||
@@ -267,12 +267,12 @@ docker run $DOCKER_COMMON --name mi355x_bench \
|
||||
--disable-circuit-breaker &
|
||||
for i in \$(seq 1 30); do curl -sf http://127.0.0.1:$LBPORT/health >/dev/null && break; sleep 2; done
|
||||
CIDIR=/host_home/.mi355x_ci/${MATRIX_CONFIG_NAME}
|
||||
echo "[smoke] PD end-to-end check via LB"
|
||||
echo "[probe] PD end-to-end check via LB"
|
||||
curl -sf -X POST http://127.0.0.1:$LBPORT/generate \
|
||||
-H "content-type: application/json" -d @\$CIDIR/smoke.json > \$CIDIR/smoke_out.json \
|
||||
|| { echo "[smoke] request failed -- PD path not serving; aborting before sweep"; exit 1; }
|
||||
python3 \$CIDIR/assert_nonempty.py < \$CIDIR/smoke_out.json \
|
||||
|| { echo "[smoke] empty/invalid generation; aborting before sweep"; exit 1; }
|
||||
-H "content-type: application/json" -d @\$CIDIR/probe.json > \$CIDIR/probe_out.json \
|
||||
|| { echo "[probe] request failed -- PD path not serving; aborting before sweep"; exit 1; }
|
||||
python3 \$CIDIR/assert_nonempty.py < \$CIDIR/probe_out.json \
|
||||
|| { echo "[probe] empty/invalid generation; aborting before sweep"; exit 1; }
|
||||
# Correctness gate runs BEFORE the perf sweep: if the model is wrong there
|
||||
# is no point spending ~15min measuring how fast it is wrong, so a failure
|
||||
# here exits immediately and the sweep never runs.
|
||||
|
||||
@@ -35,7 +35,7 @@ HWBackend = _ci_register.HWBackend
|
||||
# pr-test-amd.yml / pr-test-npu.yml have their own dispatch.
|
||||
_TARGET_BACKENDS = {HWBackend.CUDA, HWBackend.CPU}
|
||||
|
||||
# base-a is the critical-path entry gate; pin its fanout to smoke-coverage
|
||||
# base-a is the critical-path entry gate; pin its fanout to sanity-coverage
|
||||
# defaults instead of est_time. max_parallel = size (no throttle).
|
||||
_BASE_A_OVERRIDES = {
|
||||
"base-a-test-cpu": 8,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import time
|
||||
from collections import defaultdict
|
||||
from typing import Dict, List
|
||||
from typing import Dict
|
||||
|
||||
|
||||
class Node:
|
||||
@@ -111,9 +111,6 @@ class MultiTenantRadixTree:
|
||||
curr = self.root
|
||||
curr_idx = 0
|
||||
|
||||
ret_text = ""
|
||||
ret_tenant = None
|
||||
|
||||
while curr_idx < len(s):
|
||||
matched_node = None
|
||||
if s[curr_idx] in curr.children:
|
||||
@@ -228,18 +225,6 @@ class MultiTenantRadixTree:
|
||||
|
||||
return used_size_per_tenant
|
||||
|
||||
def remove_tenant(self, tenant_id: str) -> None:
|
||||
"""
|
||||
Remove all data associated with a specific tenant from the tree.
|
||||
This operation maintains the integrity of the shared tree structure while
|
||||
removing only the specified tenant's access information.
|
||||
|
||||
Args:
|
||||
tenant_id: The identifier of the tenant whose data should be removed
|
||||
"""
|
||||
# TODO: Implementation needed
|
||||
pass
|
||||
|
||||
def pretty_print(self) -> str:
|
||||
"""
|
||||
Returns a string representation of the tree showing the structure, tenant ownership,
|
||||
|
||||
@@ -79,9 +79,6 @@ def bench_es(
|
||||
b_tensors = []
|
||||
a_scales_tensors = []
|
||||
b_scales_tensors = []
|
||||
if False:
|
||||
print("Token Distributtion: ", group_ms[0:num_groups])
|
||||
print("Token Count: ", sum(group_ms[0:num_groups]))
|
||||
for g in range(num_groups):
|
||||
m_g = group_ms[g]
|
||||
expert_offsets[g + 1] = expert_offsets[g] + m_g
|
||||
|
||||
@@ -32,19 +32,19 @@ else:
|
||||
|
||||
|
||||
def rope_pool_fused(
|
||||
q: "mx.array",
|
||||
k: "mx.array",
|
||||
v: "mx.array",
|
||||
positions: "mx.array",
|
||||
slots: "mx.array",
|
||||
k_pool: "mx.array",
|
||||
v_pool: "mx.array",
|
||||
q: mx.array,
|
||||
k: mx.array,
|
||||
v: mx.array,
|
||||
positions: mx.array,
|
||||
slots: mx.array,
|
||||
k_pool: mx.array,
|
||||
v_pool: mx.array,
|
||||
*,
|
||||
head_dim: int,
|
||||
num_qo_heads: int,
|
||||
num_kv_heads: int,
|
||||
rope_base: float,
|
||||
) -> tuple["mx.array", "mx.array", "mx.array", "mx.array"]:
|
||||
) -> tuple[mx.array, mx.array, mx.array, mx.array]:
|
||||
"""Apply NeoX RoPE to Q/K and scatter K/V into the MLX KV pool.
|
||||
|
||||
Args:
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
"""Tests for DeepSeek-V4 fused norm + RoPE kernels."""
|
||||
|
||||
import math
|
||||
|
||||
import pytest
|
||||
import sgl_kernel
|
||||
import torch
|
||||
@@ -93,7 +91,7 @@ def test_fused_q_norm_rope_preallocated_output():
|
||||
|
||||
@pytest.mark.parametrize("batch_size", [1, 8])
|
||||
def test_fused_q_indexer_rope_hadamard_quant_runs(batch_size):
|
||||
"""Smoke test: kernel runs without errors and produces finite results."""
|
||||
"""Basic launch coverage with finite output checks."""
|
||||
torch.manual_seed(42)
|
||||
num_heads = 4
|
||||
head_dim = 128
|
||||
|
||||
@@ -18,7 +18,7 @@ pub fn generate_tool_call_id(
|
||||
}
|
||||
}
|
||||
|
||||
/// Generate tool constraints (placeholder implementation)
|
||||
/// Return an explicit error for tool-constraint generation through this FFI.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `tools_json` - JSON array of tools
|
||||
@@ -37,8 +37,6 @@ pub unsafe extern "C" fn sgl_generate_tool_constraints(
|
||||
_constraint_schema_out: *mut *mut std::os::raw::c_char,
|
||||
error_out: *mut *mut std::os::raw::c_char,
|
||||
) -> super::error::SglErrorCode {
|
||||
// Implementation would parse JSON and call generate_tool_constraints
|
||||
// This is a placeholder
|
||||
super::error::set_error_message(error_out, "Tool constraint generation not yet implemented in FFI");
|
||||
super::error::SglErrorCode::UnknownError
|
||||
}
|
||||
|
||||
@@ -336,7 +336,7 @@ async def _get_model_info_impl():
|
||||
model_info_json = await response.json()
|
||||
return ORJSONResponse(content=model_info_json)
|
||||
|
||||
except aiohttp.ClientError as e:
|
||||
except aiohttp.ClientError:
|
||||
raise HTTPException(
|
||||
status_code=HTTPStatus.SERVICE_UNAVAILABLE,
|
||||
detail=f"Failed to get model info from backend",
|
||||
|
||||
@@ -112,7 +112,7 @@ class TestRegressionBasic(ScriptedTestCase):
|
||||
"dbdcdde245 mamba_pool_idx cleanup-skip is mamba-architecture-specific. "
|
||||
"The shared test fixture does not configure a mamba model; running this "
|
||||
"regression against the default transformer model would not exercise the "
|
||||
"mamba NO_TOKEN cleanup path, so the body would be a pure smoke test "
|
||||
"mamba NO_TOKEN cleanup path, so the body would be a check "
|
||||
"with no real protection. Re-enable when a mamba fixture is wired in."
|
||||
)
|
||||
def test_mamba_chunked_resume_no_token(self):
|
||||
|
||||
@@ -8,7 +8,7 @@ and contains one ``CustomTestCase`` subclass per recipe
|
||||
Each subclass launches the server with the cookbook's exact flags and
|
||||
runs two sgl-eval evaluations (https://github.com/sgl-project/sgl-eval):
|
||||
- ``test_smoke_gsm8k`` — short, cheap GSM8K pass to verify the server
|
||||
can produce coherent math answers at all (smoke gate).
|
||||
can produce coherent math answers at all (sanity gate).
|
||||
- ``test_aime25`` — full AIME25 accuracy run (heavy; 16 repeats default).
|
||||
|
||||
Cookbook reference:
|
||||
@@ -19,7 +19,7 @@ These are MANUAL tests (not CI). ``sgl-eval`` must be on PATH.
|
||||
Per-variant defaults (set on the Flash/Pro intermediate base classes):
|
||||
Flash recipes -> AIME25 score threshold 0.93
|
||||
Pro recipes -> AIME25 score threshold 0.95
|
||||
GSM8K smoke threshold (0.93) is shared across Flash and Pro.
|
||||
GSM8K sanity threshold (0.93) is shared across Flash and Pro.
|
||||
|
||||
AIME25 knobs (env vars):
|
||||
DSV4_AIME25_NUM_REPEATS (default 16 -> --n-repeats)
|
||||
@@ -30,7 +30,7 @@ AIME25 knobs (env vars):
|
||||
DSV4_AIME25_SCORE_METRIC (default "score"; sgl-eval JSON key under "aggregate")
|
||||
DSV4_AIME25_SCORE_THRESHOLD (default 0; >0 overrides per-variant default)
|
||||
|
||||
GSM8K smoke knobs (env vars):
|
||||
GSM8K sanity knobs (env vars):
|
||||
DSV4_GSM8K_NUM_EXAMPLES (default 50 -> --num-examples)
|
||||
DSV4_GSM8K_N_REPEATS (default 1 -> --n-repeats)
|
||||
DSV4_GSM8K_TEMPERATURE (default 0.6 -> --temperature)
|
||||
|
||||
@@ -20,7 +20,7 @@ class TestLing26Flash(GSM8KMixin, DefaultServerBase):
|
||||
|
||||
# Native 128K context (no YaRN) — avoids the
|
||||
# SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN env-var dance and keeps the
|
||||
# smoke test focused on the dispatcher / hybrid-attention path.
|
||||
# coverage focused on the dispatcher / hybrid-attention path.
|
||||
other_args = [
|
||||
"--tp-size",
|
||||
"4",
|
||||
|
||||
@@ -39,7 +39,7 @@ class MoriTransferEngineBase(PDDisaggregationServerBase):
|
||||
raise unittest.SkipTest("torch.cuda is not available.")
|
||||
if torch.cuda.device_count() < cls.required_gpus:
|
||||
raise unittest.SkipTest(
|
||||
f"MORI PD smoke test requires >= {cls.required_gpus} visible GPUs."
|
||||
f"MORI PD check requires >= {cls.required_gpus} visible GPUs."
|
||||
)
|
||||
except Exception as e:
|
||||
raise unittest.SkipTest(f"torch is not available/usable: {e}")
|
||||
|
||||
@@ -46,7 +46,7 @@ class NixlTransferEngineBase(PDDisaggregationServerBase):
|
||||
raise unittest.SkipTest("torch.cuda is not available.")
|
||||
if torch.cuda.device_count() < cls.required_gpus:
|
||||
raise unittest.SkipTest(
|
||||
f"NIXL PD smoke test requires >= {cls.required_gpus} visible GPUs."
|
||||
f"NIXL PD check requires >= {cls.required_gpus} visible GPUs."
|
||||
)
|
||||
except unittest.SkipTest:
|
||||
raise
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Stage-a basic sanity: small-but-broad server smoke that downstream
|
||||
"""Stage-a basic sanity: small-but-broad server coverage that downstream
|
||||
stages depend on. Multiple sanity-kit mixins driving one shared server,
|
||||
covering protocol, decode correctness, scheduler stress, occupancy, and
|
||||
hellaswag accuracy."""
|
||||
|
||||
@@ -302,7 +302,7 @@ def test_load_weights_from_remote_instance(
|
||||
try:
|
||||
key, value = param_queue.get(timeout=5)
|
||||
results[key] = value
|
||||
except Exception as e:
|
||||
except Exception:
|
||||
if all(not p.is_alive() for p in context.processes):
|
||||
break
|
||||
|
||||
|
||||
@@ -310,7 +310,7 @@ def test_load_weights_from_remote_instance(
|
||||
try:
|
||||
key, value = param_queue.get(timeout=5)
|
||||
results[key] = value
|
||||
except Exception as e:
|
||||
except Exception:
|
||||
if all(not p.is_alive() for p in context.processes):
|
||||
break
|
||||
|
||||
|
||||
@@ -18,7 +18,7 @@ _REQUEST_TIMEOUT = 60
|
||||
"MUSA device not available",
|
||||
)
|
||||
class TestMusaDeepSeekV2LiteChatServerSmoke(DefaultServerBase):
|
||||
"""MUSA LLM server smoke test: launch, health check, and non-empty generation."""
|
||||
"""MUSA LLM server sanity check: launch, health check, and non-empty generation."""
|
||||
|
||||
model = os.getenv("SGLANG_MUSA_LLM_MODEL", "deepseek-ai/DeepSeek-V2-Lite-Chat")
|
||||
served_model_name = "deepseek-v2-lite-chat"
|
||||
|
||||
@@ -112,7 +112,7 @@ class TestUnifiedDeepSeekV4FlashHiCache(UnifiedRadixTreeTestMixin, CustomTestCas
|
||||
class TestUnifiedDeepSeekV4FlashHiCachePageFirstDirect(
|
||||
TestUnifiedDeepSeekV4FlashHiCache
|
||||
):
|
||||
"""DeepSeek V4 Flash HiCache layout smoke: page_first_direct + direct."""
|
||||
"""DeepSeek V4 Flash HiCache layout check: page_first_direct + direct."""
|
||||
|
||||
hicache_io_backend = "kernel"
|
||||
hicache_mem_layout = "layer_first"
|
||||
|
||||
@@ -396,7 +396,8 @@ def init_process_sgl(
|
||||
return response.json()
|
||||
|
||||
with ThreadPoolExecutor(32) as executor:
|
||||
futures = [executor.submit(run_decode, 1000) for _ in range(32)]
|
||||
for _ in range(32):
|
||||
executor.submit(run_decode, 1000)
|
||||
time.sleep(2)
|
||||
|
||||
# The last parameter is lm_head.weight, which is tied
|
||||
@@ -573,7 +574,7 @@ def test_update_weights_from_distributed(
|
||||
try:
|
||||
key, value = param_queue.get(timeout=5)
|
||||
results[key] = value
|
||||
except Exception as e:
|
||||
except Exception:
|
||||
if all(not p.is_alive() for p in context.processes):
|
||||
break
|
||||
|
||||
|
||||
@@ -736,9 +736,7 @@ class TestScriptedRuntimeCore(ScriptedTestCase):
|
||||
|
||||
@staticmethod
|
||||
def _script_empty_return(t: ScriptedContext):
|
||||
if False:
|
||||
yield
|
||||
return
|
||||
yield from ()
|
||||
|
||||
def test_failing_script_surfaces_and_session_survives(self):
|
||||
with self.assertRaises(AssertionError) as ctx:
|
||||
@@ -753,7 +751,7 @@ class TestScriptedRuntimeCore(ScriptedTestCase):
|
||||
|
||||
@staticmethod
|
||||
def _script_minimal_ok(t: ScriptedContext):
|
||||
r = t.start_req(prompt_len=_SHORT_PROMPT_LEN, max_new_tokens=2)
|
||||
t.start_req(prompt_len=_SHORT_PROMPT_LEN, max_new_tokens=2)
|
||||
yield
|
||||
yield
|
||||
|
||||
|
||||
@@ -280,7 +280,7 @@ class FullResponseUsageTestCase(unittest.TestCase):
|
||||
metadata = RequestResponseMetadata(request_id=request.request_id)
|
||||
|
||||
async def empty_generator():
|
||||
if False:
|
||||
for _ in ():
|
||||
yield None
|
||||
|
||||
response = asyncio.run(
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
Covers:
|
||||
- mlx_q4 / mlx_q8 quantize fp16 weights to QuantizedLinear in-place
|
||||
- active-memory drops after quantization
|
||||
- smoke /generate still works post-quantize
|
||||
- /generate still works post-quantize
|
||||
- pre-quantized HF repos still load (regression guard for mlx_lm passthrough)
|
||||
- mlx_q4 flag on an already-quantized model is a no-op (skip + log)
|
||||
|
||||
|
||||
@@ -265,7 +265,7 @@ def _image_item(feature, grid_hws):
|
||||
|
||||
|
||||
class TestGetImageFeatureWiring(CustomTestCase):
|
||||
"""Forward-shape smoke test for get_image_feature.
|
||||
"""Forward-shape coverage for get_image_feature.
|
||||
|
||||
Guards the production path (pixel concat -> vision tower -> projector) and
|
||||
the precomputed-embedding passthrough so a future change to the wiring or
|
||||
|
||||
Reference in New Issue
Block a user