chore: cleanup garbage code (#29770)
This commit is contained in:
@@ -131,7 +131,7 @@ def run_baseline(inp):
|
|||||||
|
|
||||||
def run_packed(inp):
|
def run_packed(inp):
|
||||||
"""Packed path: single fused kernel directly on mixed_qkv."""
|
"""Packed path: single fused kernel directly on mixed_qkv."""
|
||||||
B, HV, K, V = inp["B"], inp["HV"], inp["K"], inp["V"]
|
B, HV, V = inp["B"], inp["HV"], inp["V"]
|
||||||
ssm_states = inp["ssm_states"].clone()
|
ssm_states = inp["ssm_states"].clone()
|
||||||
out = inp["mixed_qkv"].new_empty(B, 1, HV, V)
|
out = inp["mixed_qkv"].new_empty(B, 1, HV, V)
|
||||||
|
|
||||||
@@ -173,7 +173,7 @@ def check_correctness(B, H, HV, K, V, pool_size, device, dtype, seed=42):
|
|||||||
try:
|
try:
|
||||||
torch.testing.assert_close(o_packed, o_baseline, atol=atol, rtol=rtol)
|
torch.testing.assert_close(o_packed, o_baseline, atol=atol, rtol=rtol)
|
||||||
output_ok = True
|
output_ok = True
|
||||||
except AssertionError as e:
|
except AssertionError:
|
||||||
output_ok = False
|
output_ok = False
|
||||||
out_diff = (o_packed - o_baseline).abs().max().item()
|
out_diff = (o_packed - o_baseline).abs().max().item()
|
||||||
|
|
||||||
|
|||||||
@@ -171,7 +171,7 @@ if __name__ == "__main__":
|
|||||||
"--limit",
|
"--limit",
|
||||||
type=int,
|
type=int,
|
||||||
default=None,
|
default=None,
|
||||||
help="If set, only evaluate this many samples (debug / smoke runs).",
|
help="If set, only evaluate this many samples (debug / quick runs).",
|
||||||
)
|
)
|
||||||
EvalArgs.add_cli_args(parser)
|
EvalArgs.add_cli_args(parser)
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|||||||
@@ -32,12 +32,12 @@ cargo bench --bench radix_tree_benchmark -- --sample-size 30 --measurement-time
|
|||||||
cargo bench --bench manual_policy_benchmark
|
cargo bench --bench manual_policy_benchmark
|
||||||
```
|
```
|
||||||
|
|
||||||
For the quick smoke runs whose numbers are reproduced below: drop
|
For the quick runs whose numbers are reproduced below: drop
|
||||||
`--sample-size` to 10 and `--measurement-time` to 2 (Criterion will
|
`--sample-size` to 10 and `--measurement-time` to 2 (Criterion will
|
||||||
warn about reduced statistical confidence but the order-of-magnitude
|
warn about reduced statistical confidence but the order-of-magnitude
|
||||||
comparison stands).
|
comparison stands).
|
||||||
|
|
||||||
## Smoke-run data points (M1 MacBook, release profile)
|
## Quick-run Data Points (M1 MacBook, release profile)
|
||||||
|
|
||||||
These are NOT the real acceptance numbers — they're a sanity check
|
These are NOT the real acceptance numbers — they're a sanity check
|
||||||
that the sgl-router routing primitives are in the same ballpark as the
|
that the sgl-router routing primitives are in the same ballpark as the
|
||||||
|
|||||||
@@ -680,7 +680,7 @@ mod tests {
|
|||||||
let _ = manager_handle.await;
|
let _ = manager_handle.await;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// End-to-end wiring smoke test: spin up a fake worker, run the
|
/// End-to-end wiring check: spin up a fake worker, run the
|
||||||
/// manager with a real `KvEventIndex` against that worker URL, and
|
/// manager with a real `KvEventIndex` against that worker URL, and
|
||||||
/// verify both `Added` and `Removed` propagate through to the
|
/// verify both `Added` and `Removed` propagate through to the
|
||||||
/// index's internal worker map.
|
/// index's internal worker map.
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ The shape:
|
|||||||
- non-streaming + streaming chat completion
|
- non-streaming + streaming chat completion
|
||||||
- assistant message non-empty, role correct, finish_reason set
|
- assistant message non-empty, role correct, finish_reason set
|
||||||
|
|
||||||
These are the smoke tests that run first; if they pass, the heavier
|
These quick checks run first; if they pass, the heavier
|
||||||
multi-worker acceptance tests are worth running.
|
multi-worker acceptance tests are worth running.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
Two flavors of fixtures coexist here:
|
Two flavors of fixtures coexist here:
|
||||||
|
|
||||||
1. **Session-scoped smoke fixtures** (``sglang_server`` + ``router``) —
|
1. **Session-scoped sanity fixtures** (``sglang_server`` + ``router``) —
|
||||||
launch ONE SGLang worker + ONE router on fixed ports for the whole
|
launch ONE SGLang worker + ONE router on fixed ports for the whole
|
||||||
test session. Used by the lightweight ``test_chat_smoke.py`` /
|
test session. Used by the lightweight ``test_chat_smoke.py`` /
|
||||||
``test_tokenize_smoke.py`` files. These are the cheap "did the
|
``test_tokenize_smoke.py`` files. These are the cheap "did the
|
||||||
@@ -174,7 +174,7 @@ def build_smoke_router_args(
|
|||||||
tokenizer_path: str,
|
tokenizer_path: str,
|
||||||
sglang_url: str,
|
sglang_url: str,
|
||||||
) -> list[str]:
|
) -> list[str]:
|
||||||
"""Build the sgl-router CLI flags the smoke ``router`` fixture launches.
|
"""Build the sgl-router CLI flags the single-worker ``router`` fixture launches.
|
||||||
|
|
||||||
Static single-worker discovery (``--worker-urls``) pointed at the one
|
Static single-worker discovery (``--worker-urls``) pointed at the one
|
||||||
SGLang worker, serving exactly one model.
|
SGLang worker, serving exactly one model.
|
||||||
|
|||||||
@@ -4,7 +4,7 @@
|
|||||||
|
|
||||||
[tool.pytest.ini_options]
|
[tool.pytest.ini_options]
|
||||||
minversion = "8.0"
|
minversion = "8.0"
|
||||||
# Default discovery: smoke tests (top-level test_*.py) and the
|
# Default discovery: top-level test_*.py files and the
|
||||||
# multi-worker chat_completions suite. k8s_integration is intentionally
|
# multi-worker chat_completions suite. k8s_integration is intentionally
|
||||||
# not in the default set — it requires a kind/k8s cluster and is
|
# not in the default set — it requires a kind/k8s cluster and is
|
||||||
# invoked explicitly.
|
# invoked explicitly.
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ pytest-asyncio==0.24.0
|
|||||||
# pin of `huggingface_hub==0.26.2` here got installed AFTER the SGLang
|
# pin of `huggingface_hub==0.26.2` here got installed AFTER the SGLang
|
||||||
# deps and downgraded huggingface_hub past `is_offline_mode`'s top-
|
# deps and downgraded huggingface_hub past `is_offline_mode`'s top-
|
||||||
# level export, which broke `from sglang.srt.server_args import …`
|
# level export, which broke `from sglang.srt.server_args import …`
|
||||||
# at module import time and turned every smoke test into a 5-minute
|
# at module import time and turned every quick check into a 5-minute
|
||||||
# `/health` timeout with no actionable signal.
|
# `/health` timeout with no actionable signal.
|
||||||
# The e2e suite only uses huggingface_hub's `try_to_load_from_cache`,
|
# The e2e suite only uses huggingface_hub's `try_to_load_from_cache`,
|
||||||
# which is available in every release SGLang would install.
|
# which is available in every release SGLang would install.
|
||||||
|
|||||||
@@ -13,7 +13,7 @@
|
|||||||
//! (existing code path; pinned here so a future PD wiring change
|
//! (existing code path; pinned here so a future PD wiring change
|
||||||
//! doesn't silently swap codes).
|
//! doesn't silently swap codes).
|
||||||
//! * A PD-disagg model with both pools healthy → request flows to the
|
//! * A PD-disagg model with both pools healthy → request flows to the
|
||||||
//! prefill worker (smoke; the decode worker MUST NOT be selected for
|
//! prefill worker (sanity check; the decode worker MUST NOT be selected for
|
||||||
//! the chat route).
|
//! the chat route).
|
||||||
|
|
||||||
use axum::body::Body;
|
use axum::body::Body;
|
||||||
|
|||||||
@@ -52,7 +52,6 @@ class CustomOp(nn.Module):
|
|||||||
def forward_tpu(self, *args, **kwargs) -> Any:
|
def forward_tpu(self, *args, **kwargs) -> Any:
|
||||||
# By default, we assume that TPU ops are compatible with the
|
# By default, we assume that TPU ops are compatible with the
|
||||||
# PyTorch-native implementation.
|
# PyTorch-native implementation.
|
||||||
# NOTE(woosuk): This is a placeholder for future extensions.
|
|
||||||
return self.forward_native(*args, **kwargs)
|
return self.forward_native(*args, **kwargs)
|
||||||
|
|
||||||
def forward_musa(self, *args, **kwargs) -> Any:
|
def forward_musa(self, *args, **kwargs) -> Any:
|
||||||
|
|||||||
@@ -854,7 +854,7 @@ class OpenAIServingChat(OpenAIServingBase):
|
|||||||
prompt_ids = self.tokenizer_manager.tokenizer.encode(
|
prompt_ids = self.tokenizer_manager.tokenizer.encode(
|
||||||
rendered_prompt, **encode_kwargs
|
rendered_prompt, **encode_kwargs
|
||||||
)
|
)
|
||||||
except Exception as e:
|
except Exception:
|
||||||
# If the first attempt fails, try with flat function-only format.
|
# If the first attempt fails, try with flat function-only format.
|
||||||
# Some templates (e.g. Mistral) expect tools without the OpenAI wrapper.
|
# Some templates (e.g. Mistral) expect tools without the OpenAI wrapper.
|
||||||
tools = (
|
tools = (
|
||||||
|
|||||||
@@ -1689,7 +1689,7 @@ class OpenAIServingResponses(OpenAIServingChat):
|
|||||||
)
|
)
|
||||||
|
|
||||||
async def empty_async_generator():
|
async def empty_async_generator():
|
||||||
if False:
|
for _ in ():
|
||||||
yield
|
yield
|
||||||
|
|
||||||
final_response = await self.responses_full_generator(
|
final_response = await self.responses_full_generator(
|
||||||
|
|||||||
@@ -801,13 +801,9 @@ class _DetailAccumulator(_UtilizationRateAccumulatorMixin):
|
|||||||
self._records = []
|
self._records = []
|
||||||
|
|
||||||
def get_single_pass_gatherer_keys(self):
|
def get_single_pass_gatherer_keys(self):
|
||||||
if False: # TODO `server_args.enable_two_batch_overlap`
|
|
||||||
return [_SINGLE_PASS_GATHERER_KEY_PRIMARY, "child_a", "child_b"]
|
|
||||||
return super().get_single_pass_gatherer_keys()
|
return super().get_single_pass_gatherer_keys()
|
||||||
|
|
||||||
def get_single_pass_gatherer_key(self, debug_name: Optional[str]):
|
def get_single_pass_gatherer_key(self, debug_name: Optional[str]):
|
||||||
if False: # TODO `server_args.enable_two_batch_overlap`
|
|
||||||
return debug_name or _SINGLE_PASS_GATHERER_KEY_PRIMARY
|
|
||||||
return super().get_single_pass_gatherer_key(debug_name)
|
return super().get_single_pass_gatherer_key(debug_name)
|
||||||
|
|
||||||
def append(
|
def append(
|
||||||
|
|||||||
@@ -280,14 +280,8 @@ class XIELU(MultiPlatformOp):
|
|||||||
)
|
)
|
||||||
self._xielu_cuda_fn = self._xielu_cuda
|
self._xielu_cuda_fn = self._xielu_cuda
|
||||||
logger.warning_once(msg)
|
logger.warning_once(msg)
|
||||||
except Exception as err:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
# logger.warning_once(
|
|
||||||
# "CUDA-fused xIELU not available (%s) –"
|
|
||||||
# " falling back to a Python version.\n"
|
|
||||||
# "For CUDA xIELU (experimental), `pip install git+https://github.com/nickjbrowning/XIELU`",
|
|
||||||
# str(err),
|
|
||||||
# )
|
|
||||||
|
|
||||||
def _xielu_python(self, x: torch.Tensor) -> torch.Tensor:
|
def _xielu_python(self, x: torch.Tensor) -> torch.Tensor:
|
||||||
alpha_p = nn.functional.softplus(self.alpha_p)
|
alpha_p = nn.functional.softplus(self.alpha_p)
|
||||||
|
|||||||
@@ -253,135 +253,11 @@ class GetKAndS:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class SetK:
|
|
||||||
@classmethod
|
|
||||||
def execute(cls, *args, buf, **kwargs):
|
|
||||||
return cls.torch_fast(*args, **kwargs, buf=buf)
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def slow(
|
|
||||||
cls,
|
|
||||||
pool: "DSATokenToKVPool",
|
|
||||||
buf: torch.Tensor,
|
|
||||||
loc: torch.Tensor,
|
|
||||||
index_k: torch.Tensor,
|
|
||||||
):
|
|
||||||
for i in range(len(loc)):
|
|
||||||
page_index = loc[i] // pool.page_size
|
|
||||||
offset = loc[i] % pool.page_size
|
|
||||||
buf[
|
|
||||||
page_index,
|
|
||||||
offset * pool.index_head_dim : (offset + 1) * pool.index_head_dim,
|
|
||||||
] = index_k[i].view(torch.uint8)
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def torch_fast(
|
|
||||||
cls,
|
|
||||||
pool: "DSATokenToKVPool",
|
|
||||||
buf: torch.Tensor,
|
|
||||||
loc: torch.Tensor,
|
|
||||||
index_k: torch.Tensor,
|
|
||||||
):
|
|
||||||
(num_tokens_to_write,) = loc.shape
|
|
||||||
buf_numel_per_page = buf.shape[1]
|
|
||||||
num_k_bytes_per_token = pool.index_head_dim
|
|
||||||
|
|
||||||
# loc: (num_tokens_to_write,), int32, element := the token index to write to
|
|
||||||
loc_page_index = loc // pool.page_size
|
|
||||||
loc_token_offset_in_page = loc % pool.page_size
|
|
||||||
|
|
||||||
flat_buf = buf.flatten()
|
|
||||||
flat_indices = (
|
|
||||||
(loc_page_index * buf_numel_per_page)[:, None]
|
|
||||||
+ (loc_token_offset_in_page * num_k_bytes_per_token)[:, None]
|
|
||||||
+ torch.arange(num_k_bytes_per_token, dtype=torch.int32, device="cuda")[
|
|
||||||
None, :
|
|
||||||
]
|
|
||||||
)
|
|
||||||
num_k_bytes_total = num_tokens_to_write * num_k_bytes_per_token
|
|
||||||
flat_indices = flat_indices.flatten()[:num_k_bytes_total]
|
|
||||||
flat_buf[flat_indices] = index_k.view(torch.uint8).flatten()
|
|
||||||
|
|
||||||
|
|
||||||
class SetS:
|
|
||||||
@classmethod
|
|
||||||
def execute(cls, *args, buf, **kwargs):
|
|
||||||
return cls.torch_fast(*args, **kwargs, buf=buf)
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def slow(
|
|
||||||
cls,
|
|
||||||
pool: "DSATokenToKVPool",
|
|
||||||
buf: torch.Tensor,
|
|
||||||
loc: torch.Tensor,
|
|
||||||
index_k_scale: torch.Tensor,
|
|
||||||
):
|
|
||||||
for i in range(len(loc)):
|
|
||||||
page_index = loc[i] // pool.page_size
|
|
||||||
offset = loc[i] % pool.page_size
|
|
||||||
start = pool.page_size * pool.index_head_dim
|
|
||||||
buf[page_index, start + offset * 4 : start + (offset + 1) * 4] = (
|
|
||||||
index_k_scale[i].view(torch.uint8)
|
|
||||||
)
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def torch_fast(
|
|
||||||
cls,
|
|
||||||
pool: "DSATokenToKVPool",
|
|
||||||
buf: torch.Tensor,
|
|
||||||
loc: torch.Tensor,
|
|
||||||
index_k_scale: torch.Tensor,
|
|
||||||
):
|
|
||||||
(num_tokens_to_write,) = loc.shape
|
|
||||||
buf_numel_per_page = buf.shape[1]
|
|
||||||
num_s_bytes_per_token = 4
|
|
||||||
s_offset_in_page = pool.page_size * pool.index_head_dim
|
|
||||||
|
|
||||||
# loc: (num_tokens_to_write,), int32, element := the token index to write to
|
|
||||||
loc_page_index = loc // pool.page_size
|
|
||||||
loc_token_offset_in_page = loc % pool.page_size
|
|
||||||
|
|
||||||
flat_buf = buf.flatten()
|
|
||||||
flat_indices = (
|
|
||||||
(loc_page_index * buf_numel_per_page)[:, None]
|
|
||||||
+ s_offset_in_page
|
|
||||||
+ (loc_token_offset_in_page * num_s_bytes_per_token)[:, None]
|
|
||||||
+ torch.arange(num_s_bytes_per_token, dtype=torch.int32, device="cuda")[
|
|
||||||
None, :
|
|
||||||
]
|
|
||||||
)
|
|
||||||
number_s_bytes_total = num_tokens_to_write * num_s_bytes_per_token
|
|
||||||
flat_indices = flat_indices.flatten()[:number_s_bytes_total]
|
|
||||||
flat_buf[flat_indices] = index_k_scale.view(torch.uint8).flatten()
|
|
||||||
|
|
||||||
|
|
||||||
class SetKAndS:
|
class SetKAndS:
|
||||||
@classmethod
|
@classmethod
|
||||||
def execute(cls, *args, buf, **kwargs):
|
def execute(cls, *args, buf, **kwargs):
|
||||||
if 0:
|
|
||||||
# print("SetK, SetS comparison test")
|
|
||||||
buf_cloned = buf.clone()
|
|
||||||
cls.vanilla(*args, **kwargs, buf=buf)
|
|
||||||
cls.triton(*args, **kwargs, buf=buf_cloned)
|
|
||||||
|
|
||||||
def _clear_token_0(target):
|
|
||||||
target[0, :128] = target[0, 64 * 128 : 64 * 128 + 4] = 0
|
|
||||||
|
|
||||||
_clear_token_0(buf)
|
|
||||||
_clear_token_0(buf_cloned)
|
|
||||||
|
|
||||||
assert torch.all(
|
|
||||||
buf == buf_cloned
|
|
||||||
), f"{buf=} {buf_cloned=} {kwargs['loc'].to_list()=}"
|
|
||||||
return
|
|
||||||
|
|
||||||
cls.triton(*args, **kwargs, buf=buf)
|
cls.triton(*args, **kwargs, buf=buf)
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def vanilla(cls, pool, buf, loc, index_k, index_k_scale):
|
|
||||||
SetK.execute(pool=pool, buf=buf, loc=loc, index_k=index_k)
|
|
||||||
SetS.execute(pool=pool, buf=buf, loc=loc, index_k_scale=index_k_scale)
|
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def triton(cls, pool, buf, loc, index_k, index_k_scale):
|
def triton(cls, pool, buf, loc, index_k, index_k_scale):
|
||||||
loc = loc.to(torch.int64)
|
loc = loc.to(torch.int64)
|
||||||
|
|||||||
@@ -252,8 +252,7 @@ def pre_permute_standard_to_triton(
|
|||||||
running_state: dict,
|
running_state: dict,
|
||||||
) -> TritonRunnerInput:
|
) -> TritonRunnerInput:
|
||||||
|
|
||||||
# NOTE: this is dead code as a fused func for standard format is registered.
|
# Registered fallback for format-conversion tests and examples.
|
||||||
# This is left here for testing and examples.
|
|
||||||
|
|
||||||
from sglang.srt.layers.moe.moe_runner.triton_utils.fused_moe import (
|
from sglang.srt.layers.moe.moe_runner.triton_utils.fused_moe import (
|
||||||
_prepare_fused_moe_run,
|
_prepare_fused_moe_run,
|
||||||
@@ -309,8 +308,7 @@ def post_permute_triton_to_standard(
|
|||||||
running_state: dict,
|
running_state: dict,
|
||||||
) -> StandardCombineInput:
|
) -> StandardCombineInput:
|
||||||
|
|
||||||
# NOTE: this is dead code as a fused func for standard format is registered.
|
# Registered fallback for format-conversion tests and examples.
|
||||||
# This is left here for testing and examples.
|
|
||||||
|
|
||||||
from sglang.srt.layers.moe.token_dispatcher.standard import StandardCombineInput
|
from sglang.srt.layers.moe.token_dispatcher.standard import StandardCombineInput
|
||||||
|
|
||||||
|
|||||||
@@ -195,7 +195,7 @@ if _use_aiter:
|
|||||||
if _is_musa:
|
if _is_musa:
|
||||||
try:
|
try:
|
||||||
from mate import moe_fused_gate
|
from mate import moe_fused_gate
|
||||||
except ImportError as e:
|
except ImportError:
|
||||||
raise ImportError("mate is required for the biased grouped topk.")
|
raise ImportError("mate is required for the biased grouped topk.")
|
||||||
|
|
||||||
from sglang.srt.hardware_backend.musa.kernels.topk import topk_sigmoid, topk_softmax
|
from sglang.srt.hardware_backend.musa.kernels.topk import topk_sigmoid, topk_softmax
|
||||||
|
|||||||
@@ -94,11 +94,7 @@ def hash_tiles32_kernel_blocked(
|
|||||||
h1 ^= nbytes
|
h1 ^= nbytes
|
||||||
h2 ^= nbytes
|
h2 ^= nbytes
|
||||||
h1 = _fmix32(h1, C1=FM_C1, C2=FM_C2)
|
h1 = _fmix32(h1, C1=FM_C1, C2=FM_C2)
|
||||||
h2 = (
|
h2 = _fmix32(h2, C1=FM_C1, C2=FM_C2)
|
||||||
_fmix32(h2, C1=FMIX32_C1, C2=FMIX32_C2)
|
|
||||||
if False
|
|
||||||
else _fmix32(h2, C1=FM_C1, C2=FM_C2)
|
|
||||||
)
|
|
||||||
|
|
||||||
out = (h1.to(tl.uint64) << 32) | h2.to(tl.uint64)
|
out = (h1.to(tl.uint64) << 32) | h2.to(tl.uint64)
|
||||||
tl.store(out_ptr + pid, out)
|
tl.store(out_ptr + pid, out)
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ import torch
|
|||||||
|
|
||||||
try:
|
try:
|
||||||
from aiter.ops.triton.quant import dynamic_mxfp4_quant
|
from aiter.ops.triton.quant import dynamic_mxfp4_quant
|
||||||
except ImportError as err:
|
except ImportError:
|
||||||
|
|
||||||
def raise_aiter_import_error(*args, **kwargs):
|
def raise_aiter_import_error(*args, **kwargs):
|
||||||
raise ImportError(
|
raise ImportError(
|
||||||
|
|||||||
@@ -68,7 +68,7 @@ def init_feature_buffer(device):
|
|||||||
num_elements, dtype=torch.float32, device=device
|
num_elements, dtype=torch.float32, device=device
|
||||||
)
|
)
|
||||||
logger.info(f"Preallocated {size_mb}MB GPU buffer")
|
logger.info(f"Preallocated {size_mb}MB GPU buffer")
|
||||||
except RuntimeError as e:
|
except RuntimeError:
|
||||||
_GPU_FEATURE_BUFFER = None
|
_GPU_FEATURE_BUFFER = None
|
||||||
|
|
||||||
|
|
||||||
@@ -160,7 +160,7 @@ class TransportProxyTensor(torch.Tensor):
|
|||||||
"storage_offset": self.storage_offset(),
|
"storage_offset": self.storage_offset(),
|
||||||
}
|
}
|
||||||
state["tensor_data"] = None
|
state["tensor_data"] = None
|
||||||
except Exception as e:
|
except Exception:
|
||||||
# Failed to get CUDA IPC handle (possibly tp). Falling back to default transport.
|
# Failed to get CUDA IPC handle (possibly tp). Falling back to default transport.
|
||||||
state["metadata"]["transport_mode"] = "default"
|
state["metadata"]["transport_mode"] = "default"
|
||||||
state["tensor_data"] = self.as_subclass(torch.Tensor)
|
state["tensor_data"] = self.as_subclass(torch.Tensor)
|
||||||
|
|||||||
@@ -1064,7 +1064,7 @@ class ModelRunner(ModelRunnerKVCacheMixin):
|
|||||||
def remote_instance_init_transfer_engine(self):
|
def remote_instance_init_transfer_engine(self):
|
||||||
try:
|
try:
|
||||||
from mooncake.engine import TransferEngine
|
from mooncake.engine import TransferEngine
|
||||||
except ImportError as e:
|
except ImportError:
|
||||||
logger.warning(
|
logger.warning(
|
||||||
"Please install mooncake for using remote instance transfer engine: pip install mooncake-transfer-engine"
|
"Please install mooncake for using remote instance transfer engine: pip install mooncake-transfer-engine"
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -531,22 +531,15 @@ class HunYuanModel(nn.Module):
|
|||||||
hidden_states = self.get_input_embeddings(input_ids)
|
hidden_states = self.get_input_embeddings(input_ids)
|
||||||
residual = None
|
residual = None
|
||||||
|
|
||||||
prev_kv_states = None
|
for layer in self.layers:
|
||||||
for i in range(len(self.layers)):
|
hidden_states, residual, _ = layer(
|
||||||
layer = self.layers[i]
|
|
||||||
hidden_states, residual, kv_states = layer(
|
|
||||||
positions,
|
positions,
|
||||||
hidden_states,
|
hidden_states,
|
||||||
forward_batch,
|
forward_batch,
|
||||||
residual,
|
residual,
|
||||||
prev_kv_states,
|
None,
|
||||||
)
|
)
|
||||||
|
|
||||||
if False: # (i - self.start_layer) % cla_factor == 0:
|
|
||||||
prev_kv_states = kv_states
|
|
||||||
else:
|
|
||||||
prev_kv_states = None
|
|
||||||
|
|
||||||
hidden_states, _ = self.norm(hidden_states, residual)
|
hidden_states, _ = self.norm(hidden_states, residual)
|
||||||
return hidden_states
|
return hidden_states
|
||||||
|
|
||||||
|
|||||||
@@ -213,7 +213,7 @@ def process_anyres_image(image, processor, grid_pinpoints):
|
|||||||
if isinstance(grid_pinpoints, str) and "x" in grid_pinpoints:
|
if isinstance(grid_pinpoints, str) and "x" in grid_pinpoints:
|
||||||
try:
|
try:
|
||||||
patch_size = processor.size[0]
|
patch_size = processor.size[0]
|
||||||
except Exception as e:
|
except Exception:
|
||||||
patch_size = processor.size["shortest_edge"]
|
patch_size = processor.size["shortest_edge"]
|
||||||
assert patch_size in [
|
assert patch_size in [
|
||||||
224,
|
224,
|
||||||
|
|||||||
@@ -1857,7 +1857,7 @@ def get_npu_memory_capacity():
|
|||||||
return envs.SGLANG_ZBAL_LOCAL_MEM_SIZE.get() # unit: MB
|
return envs.SGLANG_ZBAL_LOCAL_MEM_SIZE.get() # unit: MB
|
||||||
else:
|
else:
|
||||||
return torch.npu.mem_get_info()[1] // 1024 // 1024 # unit: MB
|
return torch.npu.mem_get_info()[1] // 1024 // 1024 # unit: MB
|
||||||
except ImportError as e:
|
except ImportError:
|
||||||
raise ImportError("torch_npu is required when run on npu device.")
|
raise ImportError("torch_npu is required when run on npu device.")
|
||||||
|
|
||||||
|
|
||||||
@@ -2210,7 +2210,7 @@ def get_compiler_backend(mode=None) -> str:
|
|||||||
import torchair
|
import torchair
|
||||||
import torchair.ge_concrete_graph.ge_converter.experimental.patch_for_hcom_allreduce
|
import torchair.ge_concrete_graph.ge_converter.experimental.patch_for_hcom_allreduce
|
||||||
from torchair.configs.compiler_config import CompilerConfig
|
from torchair.configs.compiler_config import CompilerConfig
|
||||||
except ImportError as e:
|
except ImportError:
|
||||||
raise ImportError(
|
raise ImportError(
|
||||||
"NPU detected, but torchair package is not installed. "
|
"NPU detected, but torchair package is not installed. "
|
||||||
"Please install torchair for torch.compile support on NPU."
|
"Please install torchair for torch.compile support on NPU."
|
||||||
|
|||||||
@@ -364,7 +364,7 @@ class CudaIpcTensorTransportProxy:
|
|||||||
"recons_dtype": info_data.dtype,
|
"recons_dtype": info_data.dtype,
|
||||||
}
|
}
|
||||||
state["tensor_data"] = None
|
state["tensor_data"] = None
|
||||||
except Exception as e:
|
except Exception:
|
||||||
# Failed to get CUDA IPC handle (possibly tp). Falling back to default transport.
|
# Failed to get CUDA IPC handle (possibly tp). Falling back to default transport.
|
||||||
state["ipc_extra"] = None
|
state["ipc_extra"] = None
|
||||||
state["tensor_data"] = data
|
state["tensor_data"] = data
|
||||||
|
|||||||
@@ -368,11 +368,11 @@ def rpd_to_chrome_trace(
|
|||||||
if row[0] == "0": # Frame start
|
if row[0] == "0": # Frame start
|
||||||
if key not in stacks:
|
if key not in stacks:
|
||||||
stacks[key] = []
|
stacks[key] = []
|
||||||
stack = stacks[key].append((row[1], row[4]))
|
stacks[key].append((row[1], row[4]))
|
||||||
# print(f"0: new api frame: pid_tid={key} -> stack={stacks}")
|
# print(f"0: new api frame: pid_tid={key} -> stack={stacks}")
|
||||||
|
|
||||||
elif row[0] == "1": # Frame end
|
elif row[0] == "1": # Frame end
|
||||||
completed = stacks[key].pop()
|
stacks[key].pop()
|
||||||
# print(f"1: end api frame: pid_tid={key} -> stack={stacks}")
|
# print(f"1: end api frame: pid_tid={key} -> stack={stacks}")
|
||||||
|
|
||||||
elif row[0] == "2": # API + Op
|
elif row[0] == "2": # API + Op
|
||||||
@@ -400,8 +400,6 @@ def rpd_to_chrome_trace(
|
|||||||
or abs(gpuFrame.start - row[8]) < 200
|
or abs(gpuFrame.start - row[8]) < 200
|
||||||
)
|
)
|
||||||
):
|
):
|
||||||
# if gpuFrame.id == frame[0] and gpuFrame.name == frame[1]: # Another op under the same frame -> union them
|
|
||||||
# if False: # Turn off frame joining
|
|
||||||
if row[7] < gpuFrame.start:
|
if row[7] < gpuFrame.start:
|
||||||
gpuFrame.start = row[7]
|
gpuFrame.start = row[7]
|
||||||
if row[8] > gpuFrame.end:
|
if row[8] > gpuFrame.end:
|
||||||
|
|||||||
@@ -779,7 +779,7 @@ def wait_server_ready(url, timeout=LOCAL_TIMEOUT):
|
|||||||
logger.info(
|
logger.info(
|
||||||
f"Server {url} returned status code: {response.status_code}"
|
f"Server {url} returned status code: {response.status_code}"
|
||||||
)
|
)
|
||||||
except Exception as e:
|
except Exception:
|
||||||
# logger.error(f"Server {url} request error: {e}, retrying...")
|
# logger.error(f"Server {url} request error: {e}, retrying...")
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|||||||
@@ -1104,7 +1104,7 @@ def _seed_c4_if_needed(
|
|||||||
fixture: DSV4AttentionFixture, *, num_entries: int = _DSV4_EXTRA_ENTRIES
|
fixture: DSV4AttentionFixture, *, num_entries: int = _DSV4_EXTRA_ENTRIES
|
||||||
) -> None:
|
) -> None:
|
||||||
"""For compress_ratio=4, seed the C4 metadata the exercised path consumes
|
"""For compress_ratio=4, seed the C4 metadata the exercised path consumes
|
||||||
(the C4Indexer would normally populate it; the smoke fixture skips the
|
(the C4Indexer would normally populate it; the compact fixture skips the
|
||||||
indexer): `c4_sparse_page_indices` for the dense extend path,
|
indexer): `c4_sparse_page_indices` for the dense extend path,
|
||||||
`c4_sparse_raw_indices` for sparse prefill. No-op for other compress_ratios.
|
`c4_sparse_raw_indices` for sparse prefill. No-op for other compress_ratios.
|
||||||
"""
|
"""
|
||||||
@@ -1383,7 +1383,7 @@ def _seed_c4_sparse_indices(
|
|||||||
) -> None:
|
) -> None:
|
||||||
"""For compress_ratio=4 the production `init_flashmla_related` initializes
|
"""For compress_ratio=4 the production `init_flashmla_related` initializes
|
||||||
`c4_sparse_page_indices` to all `-1` (the C4Indexer fills it in later).
|
`c4_sparse_page_indices` to all `-1` (the C4Indexer fills it in later).
|
||||||
Since the smoke fixture does not run the indexer, the C4 path attends to
|
Since the compact fixture does not run the indexer, the C4 path attends to
|
||||||
zero extra entries unless we seed the indices ourselves. Seed each query
|
zero extra entries unless we seed the indices ourselves. Seed each query
|
||||||
row to point to `[0, 1, ..., num_entries - 1]` so the backend reads the
|
row to point to `[0, 1, ..., num_entries - 1]` so the backend reads the
|
||||||
same `num_entries` C4 K's that the reference also reads, exercising the
|
same `num_entries` C4 K's that the reference also reads, exercising the
|
||||||
@@ -1614,7 +1614,7 @@ def run_dsv4_compress_attention_case(
|
|||||||
assert case.compress_ratio in (
|
assert case.compress_ratio in (
|
||||||
4,
|
4,
|
||||||
128,
|
128,
|
||||||
), f"smoke runner requires compress_ratio in (4, 128); got {case.compress_ratio}"
|
), f"DSV4 compact runner requires compress_ratio in (4, 128); got {case.compress_ratio}"
|
||||||
if sparse_prefill:
|
if sparse_prefill:
|
||||||
assert (
|
assert (
|
||||||
case.forward_mode.is_extend_without_speculative()
|
case.forward_mode.is_extend_without_speculative()
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
Probes that catch the model producing wrong output: weight load
|
Probes that catch the model producing wrong output: weight load
|
||||||
failure, sampling path bugs, KV / attention corruption, and cuda graph
|
failure, sampling path bugs, KV / attention corruption, and cuda graph
|
||||||
edge cases. Single-prompt smoke only -- dataset-driven accuracy gates
|
edge cases. Single-prompt coverage only -- dataset-driven accuracy gates
|
||||||
belong to the consuming test class, not this kit.
|
belong to the consuming test class, not this kit.
|
||||||
|
|
||||||
Mix into any ``CustomTestCase`` subclass that exposes ``self.base_url``
|
Mix into any ``CustomTestCase`` subclass that exposes ``self.base_url``
|
||||||
|
|||||||
@@ -175,7 +175,7 @@ class MGSMEval(Eval):
|
|||||||
]
|
]
|
||||||
try:
|
try:
|
||||||
response_text = sampler(prompt_messages)
|
response_text = sampler(prompt_messages)
|
||||||
except Exception as e:
|
except Exception:
|
||||||
response_text = ""
|
response_text = ""
|
||||||
|
|
||||||
answer_prefix = LANG_TO_ANSWER_PREFIX[language]
|
answer_prefix = LANG_TO_ANSWER_PREFIX[language]
|
||||||
|
|||||||
@@ -150,8 +150,6 @@ else
|
|||||||
docker cp human-eval ci_sglang:/
|
docker cp human-eval ci_sglang:/
|
||||||
install_with_retry docker exec -w /human-eval ci_sglang pip install --cache-dir=/sgl-data/pip-cache -e .
|
install_with_retry docker exec -w /human-eval ci_sglang pip install --cache-dir=/sgl-data/pip-cache -e .
|
||||||
|
|
||||||
docker exec -w / ci_sglang mkdir -p /dummy-grok
|
|
||||||
# Create dummy grok config inline (bypasses Azure blob storage which may have auth issues)
|
|
||||||
mkdir -p dummy-grok
|
mkdir -p dummy-grok
|
||||||
cat > dummy-grok/config.json << 'EOF'
|
cat > dummy-grok/config.json << 'EOF'
|
||||||
{
|
{
|
||||||
@@ -176,9 +174,8 @@ else
|
|||||||
"torch_dtype": "bfloat16"
|
"torch_dtype": "bfloat16"
|
||||||
}
|
}
|
||||||
EOF
|
EOF
|
||||||
# docker exec -w / ci_sglang mkdir -p /dummy-grok
|
docker exec -w / ci_sglang mkdir -p /dummy-grok
|
||||||
# mkdir -p dummy-grok && wget https://sharkpublic.blob.core.windows.net/sharkpublic/sglang/dummy_grok.json -O dummy-grok/config.json
|
docker cp ./dummy-grok/config.json ci_sglang:/dummy-grok/config.json
|
||||||
# docker cp ./dummy-grok ci_sglang:/
|
|
||||||
|
|
||||||
docker exec ci_sglang pip install --cache-dir=/sgl-data/pip-cache huggingface_hub[hf_xet]
|
docker exec ci_sglang pip install --cache-dir=/sgl-data/pip-cache huggingface_hub[hf_xet]
|
||||||
docker exec ci_sglang pip install --cache-dir=/sgl-data/pip-cache pytest
|
docker exec ci_sglang pip install --cache-dir=/sgl-data/pip-cache pytest
|
||||||
|
|||||||
@@ -143,7 +143,7 @@ def parse_args() -> argparse.Namespace:
|
|||||||
return parser.parse_args()
|
return parser.parse_args()
|
||||||
|
|
||||||
|
|
||||||
def build_sandbox_image() -> "modal.Image":
|
def build_sandbox_image() -> modal.Image:
|
||||||
if modal is None:
|
if modal is None:
|
||||||
raise RuntimeError(
|
raise RuntimeError(
|
||||||
"The 'modal' package is required. Run this script with "
|
"The 'modal' package is required. Run this script with "
|
||||||
@@ -213,7 +213,7 @@ Do not skip issue filing. The whole point of this system is automated triage.
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
|
|
||||||
def upload_tree(sandbox: "modal.Sandbox", log_dir: Path) -> None:
|
def upload_tree(sandbox: modal.Sandbox, log_dir: Path) -> None:
|
||||||
log_files = [path for path in log_dir.rglob("*") if path.is_file()]
|
log_files = [path for path in log_dir.rglob("*") if path.is_file()]
|
||||||
logger.info("Uploading %d log files into the sandbox", len(log_files))
|
logger.info("Uploading %d log files into the sandbox", len(log_files))
|
||||||
for index, log_file in enumerate(log_files, start=1):
|
for index, log_file in enumerate(log_files, start=1):
|
||||||
@@ -225,7 +225,7 @@ def upload_tree(sandbox: "modal.Sandbox", log_dir: Path) -> None:
|
|||||||
logger.info("Uploaded %d/%d files", index, len(log_files))
|
logger.info("Uploaded %d/%d files", index, len(log_files))
|
||||||
|
|
||||||
|
|
||||||
def clone_context_repos(sandbox: "modal.Sandbox", repo_urls: list[str]) -> None:
|
def clone_context_repos(sandbox: modal.Sandbox, repo_urls: list[str]) -> None:
|
||||||
if not repo_urls:
|
if not repo_urls:
|
||||||
return
|
return
|
||||||
|
|
||||||
@@ -244,7 +244,7 @@ def clone_context_repos(sandbox: "modal.Sandbox", repo_urls: list[str]) -> None:
|
|||||||
).wait()
|
).wait()
|
||||||
|
|
||||||
|
|
||||||
def read_optional_file(sandbox: "modal.Sandbox", path: str) -> str | None:
|
def read_optional_file(sandbox: modal.Sandbox, path: str) -> str | None:
|
||||||
try:
|
try:
|
||||||
return sandbox.filesystem.read_text(path)
|
return sandbox.filesystem.read_text(path)
|
||||||
except Exception:
|
except Exception:
|
||||||
|
|||||||
@@ -231,10 +231,10 @@ docker run $DOCKER_COMMON --name mi355x_decode \
|
|||||||
$COMMON_FLAGS --disaggregation-mode decode --disaggregation-bootstrap-port $DBOOT
|
$COMMON_FLAGS --disaggregation-mode decode --disaggregation-bootstrap-port $DBOOT
|
||||||
EOF
|
EOF
|
||||||
|
|
||||||
# Smoke-test payload + validator (separate files to avoid quoting inside the
|
# Probe payload + validator (separate files to avoid quoting inside the
|
||||||
# bench.sh `bash -lc '...'` block). One real request exercises the full
|
# bench.sh `bash -lc '...'` block). One real request exercises the full
|
||||||
# prefill->decode KV handoff before we commit to the whole sweep.
|
# prefill->decode KV handoff before we commit to the whole sweep.
|
||||||
cat > "$WORKDIR/smoke.json" <<'JSON'
|
cat > "$WORKDIR/probe.json" <<'JSON'
|
||||||
{"text": "The capital of France is", "sampling_params": {"max_new_tokens": 16, "temperature": 0.0}}
|
{"text": "The capital of France is", "sampling_params": {"max_new_tokens": 16, "temperature": 0.0}}
|
||||||
JSON
|
JSON
|
||||||
cat > "$WORKDIR/assert_nonempty.py" <<'PY'
|
cat > "$WORKDIR/assert_nonempty.py" <<'PY'
|
||||||
@@ -242,9 +242,9 @@ import sys, json
|
|||||||
d = json.load(sys.stdin)
|
d = json.load(sys.stdin)
|
||||||
t = d.get("text", "") if isinstance(d, dict) else ""
|
t = d.get("text", "") if isinstance(d, dict) else ""
|
||||||
if not (t and t.strip()):
|
if not (t and t.strip()):
|
||||||
print("[smoke] empty/invalid output:", str(d)[:200])
|
print("[probe] empty/invalid output:", str(d)[:200])
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
print("[smoke] ok:", t[:80].replace("\n", " "))
|
print("[probe] ok:", t[:80].replace("\n", " "))
|
||||||
PY
|
PY
|
||||||
|
|
||||||
# Bench script runs on the prefill node; \$PIP/\$DIP injected at srun time.
|
# Bench script runs on the prefill node; \$PIP/\$DIP injected at srun time.
|
||||||
@@ -267,12 +267,12 @@ docker run $DOCKER_COMMON --name mi355x_bench \
|
|||||||
--disable-circuit-breaker &
|
--disable-circuit-breaker &
|
||||||
for i in \$(seq 1 30); do curl -sf http://127.0.0.1:$LBPORT/health >/dev/null && break; sleep 2; done
|
for i in \$(seq 1 30); do curl -sf http://127.0.0.1:$LBPORT/health >/dev/null && break; sleep 2; done
|
||||||
CIDIR=/host_home/.mi355x_ci/${MATRIX_CONFIG_NAME}
|
CIDIR=/host_home/.mi355x_ci/${MATRIX_CONFIG_NAME}
|
||||||
echo "[smoke] PD end-to-end check via LB"
|
echo "[probe] PD end-to-end check via LB"
|
||||||
curl -sf -X POST http://127.0.0.1:$LBPORT/generate \
|
curl -sf -X POST http://127.0.0.1:$LBPORT/generate \
|
||||||
-H "content-type: application/json" -d @\$CIDIR/smoke.json > \$CIDIR/smoke_out.json \
|
-H "content-type: application/json" -d @\$CIDIR/probe.json > \$CIDIR/probe_out.json \
|
||||||
|| { echo "[smoke] request failed -- PD path not serving; aborting before sweep"; exit 1; }
|
|| { echo "[probe] request failed -- PD path not serving; aborting before sweep"; exit 1; }
|
||||||
python3 \$CIDIR/assert_nonempty.py < \$CIDIR/smoke_out.json \
|
python3 \$CIDIR/assert_nonempty.py < \$CIDIR/probe_out.json \
|
||||||
|| { echo "[smoke] empty/invalid generation; aborting before sweep"; exit 1; }
|
|| { echo "[probe] empty/invalid generation; aborting before sweep"; exit 1; }
|
||||||
# Correctness gate runs BEFORE the perf sweep: if the model is wrong there
|
# Correctness gate runs BEFORE the perf sweep: if the model is wrong there
|
||||||
# is no point spending ~15min measuring how fast it is wrong, so a failure
|
# is no point spending ~15min measuring how fast it is wrong, so a failure
|
||||||
# here exits immediately and the sweep never runs.
|
# here exits immediately and the sweep never runs.
|
||||||
|
|||||||
@@ -35,7 +35,7 @@ HWBackend = _ci_register.HWBackend
|
|||||||
# pr-test-amd.yml / pr-test-npu.yml have their own dispatch.
|
# pr-test-amd.yml / pr-test-npu.yml have their own dispatch.
|
||||||
_TARGET_BACKENDS = {HWBackend.CUDA, HWBackend.CPU}
|
_TARGET_BACKENDS = {HWBackend.CUDA, HWBackend.CPU}
|
||||||
|
|
||||||
# base-a is the critical-path entry gate; pin its fanout to smoke-coverage
|
# base-a is the critical-path entry gate; pin its fanout to sanity-coverage
|
||||||
# defaults instead of est_time. max_parallel = size (no throttle).
|
# defaults instead of est_time. max_parallel = size (no throttle).
|
||||||
_BASE_A_OVERRIDES = {
|
_BASE_A_OVERRIDES = {
|
||||||
"base-a-test-cpu": 8,
|
"base-a-test-cpu": 8,
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import time
|
import time
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
from typing import Dict, List
|
from typing import Dict
|
||||||
|
|
||||||
|
|
||||||
class Node:
|
class Node:
|
||||||
@@ -111,9 +111,6 @@ class MultiTenantRadixTree:
|
|||||||
curr = self.root
|
curr = self.root
|
||||||
curr_idx = 0
|
curr_idx = 0
|
||||||
|
|
||||||
ret_text = ""
|
|
||||||
ret_tenant = None
|
|
||||||
|
|
||||||
while curr_idx < len(s):
|
while curr_idx < len(s):
|
||||||
matched_node = None
|
matched_node = None
|
||||||
if s[curr_idx] in curr.children:
|
if s[curr_idx] in curr.children:
|
||||||
@@ -228,18 +225,6 @@ class MultiTenantRadixTree:
|
|||||||
|
|
||||||
return used_size_per_tenant
|
return used_size_per_tenant
|
||||||
|
|
||||||
def remove_tenant(self, tenant_id: str) -> None:
|
|
||||||
"""
|
|
||||||
Remove all data associated with a specific tenant from the tree.
|
|
||||||
This operation maintains the integrity of the shared tree structure while
|
|
||||||
removing only the specified tenant's access information.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
tenant_id: The identifier of the tenant whose data should be removed
|
|
||||||
"""
|
|
||||||
# TODO: Implementation needed
|
|
||||||
pass
|
|
||||||
|
|
||||||
def pretty_print(self) -> str:
|
def pretty_print(self) -> str:
|
||||||
"""
|
"""
|
||||||
Returns a string representation of the tree showing the structure, tenant ownership,
|
Returns a string representation of the tree showing the structure, tenant ownership,
|
||||||
|
|||||||
@@ -79,9 +79,6 @@ def bench_es(
|
|||||||
b_tensors = []
|
b_tensors = []
|
||||||
a_scales_tensors = []
|
a_scales_tensors = []
|
||||||
b_scales_tensors = []
|
b_scales_tensors = []
|
||||||
if False:
|
|
||||||
print("Token Distributtion: ", group_ms[0:num_groups])
|
|
||||||
print("Token Count: ", sum(group_ms[0:num_groups]))
|
|
||||||
for g in range(num_groups):
|
for g in range(num_groups):
|
||||||
m_g = group_ms[g]
|
m_g = group_ms[g]
|
||||||
expert_offsets[g + 1] = expert_offsets[g] + m_g
|
expert_offsets[g + 1] = expert_offsets[g] + m_g
|
||||||
|
|||||||
@@ -32,19 +32,19 @@ else:
|
|||||||
|
|
||||||
|
|
||||||
def rope_pool_fused(
|
def rope_pool_fused(
|
||||||
q: "mx.array",
|
q: mx.array,
|
||||||
k: "mx.array",
|
k: mx.array,
|
||||||
v: "mx.array",
|
v: mx.array,
|
||||||
positions: "mx.array",
|
positions: mx.array,
|
||||||
slots: "mx.array",
|
slots: mx.array,
|
||||||
k_pool: "mx.array",
|
k_pool: mx.array,
|
||||||
v_pool: "mx.array",
|
v_pool: mx.array,
|
||||||
*,
|
*,
|
||||||
head_dim: int,
|
head_dim: int,
|
||||||
num_qo_heads: int,
|
num_qo_heads: int,
|
||||||
num_kv_heads: int,
|
num_kv_heads: int,
|
||||||
rope_base: float,
|
rope_base: float,
|
||||||
) -> tuple["mx.array", "mx.array", "mx.array", "mx.array"]:
|
) -> tuple[mx.array, mx.array, mx.array, mx.array]:
|
||||||
"""Apply NeoX RoPE to Q/K and scatter K/V into the MLX KV pool.
|
"""Apply NeoX RoPE to Q/K and scatter K/V into the MLX KV pool.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
|
|||||||
@@ -1,7 +1,5 @@
|
|||||||
"""Tests for DeepSeek-V4 fused norm + RoPE kernels."""
|
"""Tests for DeepSeek-V4 fused norm + RoPE kernels."""
|
||||||
|
|
||||||
import math
|
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
import sgl_kernel
|
import sgl_kernel
|
||||||
import torch
|
import torch
|
||||||
@@ -93,7 +91,7 @@ def test_fused_q_norm_rope_preallocated_output():
|
|||||||
|
|
||||||
@pytest.mark.parametrize("batch_size", [1, 8])
|
@pytest.mark.parametrize("batch_size", [1, 8])
|
||||||
def test_fused_q_indexer_rope_hadamard_quant_runs(batch_size):
|
def test_fused_q_indexer_rope_hadamard_quant_runs(batch_size):
|
||||||
"""Smoke test: kernel runs without errors and produces finite results."""
|
"""Basic launch coverage with finite output checks."""
|
||||||
torch.manual_seed(42)
|
torch.manual_seed(42)
|
||||||
num_heads = 4
|
num_heads = 4
|
||||||
head_dim = 128
|
head_dim = 128
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ pub fn generate_tool_call_id(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Generate tool constraints (placeholder implementation)
|
/// Return an explicit error for tool-constraint generation through this FFI.
|
||||||
///
|
///
|
||||||
/// # Arguments
|
/// # Arguments
|
||||||
/// * `tools_json` - JSON array of tools
|
/// * `tools_json` - JSON array of tools
|
||||||
@@ -37,8 +37,6 @@ pub unsafe extern "C" fn sgl_generate_tool_constraints(
|
|||||||
_constraint_schema_out: *mut *mut std::os::raw::c_char,
|
_constraint_schema_out: *mut *mut std::os::raw::c_char,
|
||||||
error_out: *mut *mut std::os::raw::c_char,
|
error_out: *mut *mut std::os::raw::c_char,
|
||||||
) -> super::error::SglErrorCode {
|
) -> super::error::SglErrorCode {
|
||||||
// Implementation would parse JSON and call generate_tool_constraints
|
|
||||||
// This is a placeholder
|
|
||||||
super::error::set_error_message(error_out, "Tool constraint generation not yet implemented in FFI");
|
super::error::set_error_message(error_out, "Tool constraint generation not yet implemented in FFI");
|
||||||
super::error::SglErrorCode::UnknownError
|
super::error::SglErrorCode::UnknownError
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -336,7 +336,7 @@ async def _get_model_info_impl():
|
|||||||
model_info_json = await response.json()
|
model_info_json = await response.json()
|
||||||
return ORJSONResponse(content=model_info_json)
|
return ORJSONResponse(content=model_info_json)
|
||||||
|
|
||||||
except aiohttp.ClientError as e:
|
except aiohttp.ClientError:
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
status_code=HTTPStatus.SERVICE_UNAVAILABLE,
|
status_code=HTTPStatus.SERVICE_UNAVAILABLE,
|
||||||
detail=f"Failed to get model info from backend",
|
detail=f"Failed to get model info from backend",
|
||||||
|
|||||||
@@ -112,7 +112,7 @@ class TestRegressionBasic(ScriptedTestCase):
|
|||||||
"dbdcdde245 mamba_pool_idx cleanup-skip is mamba-architecture-specific. "
|
"dbdcdde245 mamba_pool_idx cleanup-skip is mamba-architecture-specific. "
|
||||||
"The shared test fixture does not configure a mamba model; running this "
|
"The shared test fixture does not configure a mamba model; running this "
|
||||||
"regression against the default transformer model would not exercise the "
|
"regression against the default transformer model would not exercise the "
|
||||||
"mamba NO_TOKEN cleanup path, so the body would be a pure smoke test "
|
"mamba NO_TOKEN cleanup path, so the body would be a check "
|
||||||
"with no real protection. Re-enable when a mamba fixture is wired in."
|
"with no real protection. Re-enable when a mamba fixture is wired in."
|
||||||
)
|
)
|
||||||
def test_mamba_chunked_resume_no_token(self):
|
def test_mamba_chunked_resume_no_token(self):
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ and contains one ``CustomTestCase`` subclass per recipe
|
|||||||
Each subclass launches the server with the cookbook's exact flags and
|
Each subclass launches the server with the cookbook's exact flags and
|
||||||
runs two sgl-eval evaluations (https://github.com/sgl-project/sgl-eval):
|
runs two sgl-eval evaluations (https://github.com/sgl-project/sgl-eval):
|
||||||
- ``test_smoke_gsm8k`` — short, cheap GSM8K pass to verify the server
|
- ``test_smoke_gsm8k`` — short, cheap GSM8K pass to verify the server
|
||||||
can produce coherent math answers at all (smoke gate).
|
can produce coherent math answers at all (sanity gate).
|
||||||
- ``test_aime25`` — full AIME25 accuracy run (heavy; 16 repeats default).
|
- ``test_aime25`` — full AIME25 accuracy run (heavy; 16 repeats default).
|
||||||
|
|
||||||
Cookbook reference:
|
Cookbook reference:
|
||||||
@@ -19,7 +19,7 @@ These are MANUAL tests (not CI). ``sgl-eval`` must be on PATH.
|
|||||||
Per-variant defaults (set on the Flash/Pro intermediate base classes):
|
Per-variant defaults (set on the Flash/Pro intermediate base classes):
|
||||||
Flash recipes -> AIME25 score threshold 0.93
|
Flash recipes -> AIME25 score threshold 0.93
|
||||||
Pro recipes -> AIME25 score threshold 0.95
|
Pro recipes -> AIME25 score threshold 0.95
|
||||||
GSM8K smoke threshold (0.93) is shared across Flash and Pro.
|
GSM8K sanity threshold (0.93) is shared across Flash and Pro.
|
||||||
|
|
||||||
AIME25 knobs (env vars):
|
AIME25 knobs (env vars):
|
||||||
DSV4_AIME25_NUM_REPEATS (default 16 -> --n-repeats)
|
DSV4_AIME25_NUM_REPEATS (default 16 -> --n-repeats)
|
||||||
@@ -30,7 +30,7 @@ AIME25 knobs (env vars):
|
|||||||
DSV4_AIME25_SCORE_METRIC (default "score"; sgl-eval JSON key under "aggregate")
|
DSV4_AIME25_SCORE_METRIC (default "score"; sgl-eval JSON key under "aggregate")
|
||||||
DSV4_AIME25_SCORE_THRESHOLD (default 0; >0 overrides per-variant default)
|
DSV4_AIME25_SCORE_THRESHOLD (default 0; >0 overrides per-variant default)
|
||||||
|
|
||||||
GSM8K smoke knobs (env vars):
|
GSM8K sanity knobs (env vars):
|
||||||
DSV4_GSM8K_NUM_EXAMPLES (default 50 -> --num-examples)
|
DSV4_GSM8K_NUM_EXAMPLES (default 50 -> --num-examples)
|
||||||
DSV4_GSM8K_N_REPEATS (default 1 -> --n-repeats)
|
DSV4_GSM8K_N_REPEATS (default 1 -> --n-repeats)
|
||||||
DSV4_GSM8K_TEMPERATURE (default 0.6 -> --temperature)
|
DSV4_GSM8K_TEMPERATURE (default 0.6 -> --temperature)
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ class TestLing26Flash(GSM8KMixin, DefaultServerBase):
|
|||||||
|
|
||||||
# Native 128K context (no YaRN) — avoids the
|
# Native 128K context (no YaRN) — avoids the
|
||||||
# SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN env-var dance and keeps the
|
# SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN env-var dance and keeps the
|
||||||
# smoke test focused on the dispatcher / hybrid-attention path.
|
# coverage focused on the dispatcher / hybrid-attention path.
|
||||||
other_args = [
|
other_args = [
|
||||||
"--tp-size",
|
"--tp-size",
|
||||||
"4",
|
"4",
|
||||||
|
|||||||
@@ -39,7 +39,7 @@ class MoriTransferEngineBase(PDDisaggregationServerBase):
|
|||||||
raise unittest.SkipTest("torch.cuda is not available.")
|
raise unittest.SkipTest("torch.cuda is not available.")
|
||||||
if torch.cuda.device_count() < cls.required_gpus:
|
if torch.cuda.device_count() < cls.required_gpus:
|
||||||
raise unittest.SkipTest(
|
raise unittest.SkipTest(
|
||||||
f"MORI PD smoke test requires >= {cls.required_gpus} visible GPUs."
|
f"MORI PD check requires >= {cls.required_gpus} visible GPUs."
|
||||||
)
|
)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
raise unittest.SkipTest(f"torch is not available/usable: {e}")
|
raise unittest.SkipTest(f"torch is not available/usable: {e}")
|
||||||
|
|||||||
@@ -46,7 +46,7 @@ class NixlTransferEngineBase(PDDisaggregationServerBase):
|
|||||||
raise unittest.SkipTest("torch.cuda is not available.")
|
raise unittest.SkipTest("torch.cuda is not available.")
|
||||||
if torch.cuda.device_count() < cls.required_gpus:
|
if torch.cuda.device_count() < cls.required_gpus:
|
||||||
raise unittest.SkipTest(
|
raise unittest.SkipTest(
|
||||||
f"NIXL PD smoke test requires >= {cls.required_gpus} visible GPUs."
|
f"NIXL PD check requires >= {cls.required_gpus} visible GPUs."
|
||||||
)
|
)
|
||||||
except unittest.SkipTest:
|
except unittest.SkipTest:
|
||||||
raise
|
raise
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
"""Stage-a basic sanity: small-but-broad server smoke that downstream
|
"""Stage-a basic sanity: small-but-broad server coverage that downstream
|
||||||
stages depend on. Multiple sanity-kit mixins driving one shared server,
|
stages depend on. Multiple sanity-kit mixins driving one shared server,
|
||||||
covering protocol, decode correctness, scheduler stress, occupancy, and
|
covering protocol, decode correctness, scheduler stress, occupancy, and
|
||||||
hellaswag accuracy."""
|
hellaswag accuracy."""
|
||||||
|
|||||||
@@ -302,7 +302,7 @@ def test_load_weights_from_remote_instance(
|
|||||||
try:
|
try:
|
||||||
key, value = param_queue.get(timeout=5)
|
key, value = param_queue.get(timeout=5)
|
||||||
results[key] = value
|
results[key] = value
|
||||||
except Exception as e:
|
except Exception:
|
||||||
if all(not p.is_alive() for p in context.processes):
|
if all(not p.is_alive() for p in context.processes):
|
||||||
break
|
break
|
||||||
|
|
||||||
|
|||||||
@@ -310,7 +310,7 @@ def test_load_weights_from_remote_instance(
|
|||||||
try:
|
try:
|
||||||
key, value = param_queue.get(timeout=5)
|
key, value = param_queue.get(timeout=5)
|
||||||
results[key] = value
|
results[key] = value
|
||||||
except Exception as e:
|
except Exception:
|
||||||
if all(not p.is_alive() for p in context.processes):
|
if all(not p.is_alive() for p in context.processes):
|
||||||
break
|
break
|
||||||
|
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ _REQUEST_TIMEOUT = 60
|
|||||||
"MUSA device not available",
|
"MUSA device not available",
|
||||||
)
|
)
|
||||||
class TestMusaDeepSeekV2LiteChatServerSmoke(DefaultServerBase):
|
class TestMusaDeepSeekV2LiteChatServerSmoke(DefaultServerBase):
|
||||||
"""MUSA LLM server smoke test: launch, health check, and non-empty generation."""
|
"""MUSA LLM server sanity check: launch, health check, and non-empty generation."""
|
||||||
|
|
||||||
model = os.getenv("SGLANG_MUSA_LLM_MODEL", "deepseek-ai/DeepSeek-V2-Lite-Chat")
|
model = os.getenv("SGLANG_MUSA_LLM_MODEL", "deepseek-ai/DeepSeek-V2-Lite-Chat")
|
||||||
served_model_name = "deepseek-v2-lite-chat"
|
served_model_name = "deepseek-v2-lite-chat"
|
||||||
|
|||||||
@@ -112,7 +112,7 @@ class TestUnifiedDeepSeekV4FlashHiCache(UnifiedRadixTreeTestMixin, CustomTestCas
|
|||||||
class TestUnifiedDeepSeekV4FlashHiCachePageFirstDirect(
|
class TestUnifiedDeepSeekV4FlashHiCachePageFirstDirect(
|
||||||
TestUnifiedDeepSeekV4FlashHiCache
|
TestUnifiedDeepSeekV4FlashHiCache
|
||||||
):
|
):
|
||||||
"""DeepSeek V4 Flash HiCache layout smoke: page_first_direct + direct."""
|
"""DeepSeek V4 Flash HiCache layout check: page_first_direct + direct."""
|
||||||
|
|
||||||
hicache_io_backend = "kernel"
|
hicache_io_backend = "kernel"
|
||||||
hicache_mem_layout = "layer_first"
|
hicache_mem_layout = "layer_first"
|
||||||
|
|||||||
@@ -396,7 +396,8 @@ def init_process_sgl(
|
|||||||
return response.json()
|
return response.json()
|
||||||
|
|
||||||
with ThreadPoolExecutor(32) as executor:
|
with ThreadPoolExecutor(32) as executor:
|
||||||
futures = [executor.submit(run_decode, 1000) for _ in range(32)]
|
for _ in range(32):
|
||||||
|
executor.submit(run_decode, 1000)
|
||||||
time.sleep(2)
|
time.sleep(2)
|
||||||
|
|
||||||
# The last parameter is lm_head.weight, which is tied
|
# The last parameter is lm_head.weight, which is tied
|
||||||
@@ -573,7 +574,7 @@ def test_update_weights_from_distributed(
|
|||||||
try:
|
try:
|
||||||
key, value = param_queue.get(timeout=5)
|
key, value = param_queue.get(timeout=5)
|
||||||
results[key] = value
|
results[key] = value
|
||||||
except Exception as e:
|
except Exception:
|
||||||
if all(not p.is_alive() for p in context.processes):
|
if all(not p.is_alive() for p in context.processes):
|
||||||
break
|
break
|
||||||
|
|
||||||
|
|||||||
@@ -736,9 +736,7 @@ class TestScriptedRuntimeCore(ScriptedTestCase):
|
|||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _script_empty_return(t: ScriptedContext):
|
def _script_empty_return(t: ScriptedContext):
|
||||||
if False:
|
yield from ()
|
||||||
yield
|
|
||||||
return
|
|
||||||
|
|
||||||
def test_failing_script_surfaces_and_session_survives(self):
|
def test_failing_script_surfaces_and_session_survives(self):
|
||||||
with self.assertRaises(AssertionError) as ctx:
|
with self.assertRaises(AssertionError) as ctx:
|
||||||
@@ -753,7 +751,7 @@ class TestScriptedRuntimeCore(ScriptedTestCase):
|
|||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _script_minimal_ok(t: ScriptedContext):
|
def _script_minimal_ok(t: ScriptedContext):
|
||||||
r = t.start_req(prompt_len=_SHORT_PROMPT_LEN, max_new_tokens=2)
|
t.start_req(prompt_len=_SHORT_PROMPT_LEN, max_new_tokens=2)
|
||||||
yield
|
yield
|
||||||
yield
|
yield
|
||||||
|
|
||||||
|
|||||||
@@ -280,7 +280,7 @@ class FullResponseUsageTestCase(unittest.TestCase):
|
|||||||
metadata = RequestResponseMetadata(request_id=request.request_id)
|
metadata = RequestResponseMetadata(request_id=request.request_id)
|
||||||
|
|
||||||
async def empty_generator():
|
async def empty_generator():
|
||||||
if False:
|
for _ in ():
|
||||||
yield None
|
yield None
|
||||||
|
|
||||||
response = asyncio.run(
|
response = asyncio.run(
|
||||||
|
|||||||
@@ -3,7 +3,7 @@
|
|||||||
Covers:
|
Covers:
|
||||||
- mlx_q4 / mlx_q8 quantize fp16 weights to QuantizedLinear in-place
|
- mlx_q4 / mlx_q8 quantize fp16 weights to QuantizedLinear in-place
|
||||||
- active-memory drops after quantization
|
- active-memory drops after quantization
|
||||||
- smoke /generate still works post-quantize
|
- /generate still works post-quantize
|
||||||
- pre-quantized HF repos still load (regression guard for mlx_lm passthrough)
|
- pre-quantized HF repos still load (regression guard for mlx_lm passthrough)
|
||||||
- mlx_q4 flag on an already-quantized model is a no-op (skip + log)
|
- mlx_q4 flag on an already-quantized model is a no-op (skip + log)
|
||||||
|
|
||||||
|
|||||||
@@ -265,7 +265,7 @@ def _image_item(feature, grid_hws):
|
|||||||
|
|
||||||
|
|
||||||
class TestGetImageFeatureWiring(CustomTestCase):
|
class TestGetImageFeatureWiring(CustomTestCase):
|
||||||
"""Forward-shape smoke test for get_image_feature.
|
"""Forward-shape coverage for get_image_feature.
|
||||||
|
|
||||||
Guards the production path (pixel concat -> vision tower -> projector) and
|
Guards the production path (pixel concat -> vision tower -> projector) and
|
||||||
the precomputed-embedding passthrough so a future change to the wiring or
|
the precomputed-embedding passthrough so a future change to the wiring or
|
||||||
|
|||||||
Reference in New Issue
Block a user