[mem_cache] Clean up SWA/Mamba radix cache leftovers and drop SGLANG_ENABLE_UNIFIED_RADIX_TREE (#40780)
This commit is contained in:
@@ -739,7 +739,6 @@ Pending update...
|
||||
|
||||
**TP8, concurrency 1–8:**
|
||||
```bash Command
|
||||
SGLANG_ENABLE_UNIFIED_RADIX_TREE=1 \
|
||||
python3 -m sglang.launch_server \
|
||||
--model-path deepseek-ai/DeepSeek-V4-Pro-0813 \
|
||||
--trust-remote-code \
|
||||
@@ -765,7 +764,6 @@ DSv4 HiCache sizes the host tier with `--hicache-ratio` (host/device token ratio
|
||||
|
||||
**DEP8 (DP Attention), concurrency 64–160:**
|
||||
```bash Command
|
||||
SGLANG_ENABLE_UNIFIED_RADIX_TREE=1 \
|
||||
SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=8320 \
|
||||
python3 -m sglang.launch_server \
|
||||
--model-path deepseek-ai/DeepSeek-V4-Pro-0813 \
|
||||
@@ -883,7 +881,6 @@ DeepSeek-V4-Pro-0813 bundles the DSpark head, so `--speculative-draft-model-path
|
||||
|
||||
**TP8, concurrency 1–16:**
|
||||
```bash Command
|
||||
SGLANG_ENABLE_UNIFIED_RADIX_TREE=1 \
|
||||
SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS=1 \
|
||||
SGLANG_USE_ROCM700A=0 \
|
||||
TORCH_BLAS_PREFER_HIPBLASLT=1 \
|
||||
@@ -922,7 +919,6 @@ Concurrency 32–48 keeps the same TP-only command and adds HiCache DRAM offload
|
||||
|
||||
**DP8 (DP Attention), concurrency 128–256:**
|
||||
```bash Command
|
||||
SGLANG_ENABLE_UNIFIED_RADIX_TREE=1 \
|
||||
SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS=1 \
|
||||
SGLANG_USE_ROCM700A=0 \
|
||||
TORCH_BLAS_PREFER_HIPBLASLT=1 \
|
||||
|
||||
@@ -13,7 +13,7 @@ Session references are soft protection, not memory pins. Referenced KV can still
|
||||
This feature is implemented only by `UnifiedRadixCache`.
|
||||
|
||||
```bash Command
|
||||
SGLANG_ENABLE_UNIFIED_RADIX_TREE=1 python3 -m sglang.launch_server \
|
||||
python3 -m sglang.launch_server \
|
||||
--model-path MODEL_PATH \
|
||||
--enable-session-radix-cache
|
||||
```
|
||||
|
||||
@@ -1414,11 +1414,6 @@ SGLang supports various environment variables that can be used to configure its
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Multiplier applied to the sliding-window-attention eviction interval.</td>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}><code>1.0</code></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}><code>SGLANG_ENABLE_UNIFIED_RADIX_TREE</code></td>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Use the unified radix-tree cache implementation.</td>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}><code>false</code></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}><code>SGLANG_RADIX_FORCE_MISS</code></td>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Force radix-cache misses (debugging/benchmarking).</td>
|
||||
|
||||
@@ -209,9 +209,7 @@ export const config = {
|
||||
{
|
||||
match: { hw: "b200", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" },
|
||||
verified: true,
|
||||
env: [
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
],
|
||||
env: [],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
@@ -236,9 +234,7 @@ export const config = {
|
||||
{
|
||||
match: { hw: "b300", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" },
|
||||
verified: true,
|
||||
env: [
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
],
|
||||
env: [],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
@@ -262,9 +258,7 @@ export const config = {
|
||||
},
|
||||
{
|
||||
match: { hw: "gb200", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" },
|
||||
env: [
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
],
|
||||
env: [],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
@@ -289,9 +283,7 @@ export const config = {
|
||||
{
|
||||
match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" },
|
||||
verified: true,
|
||||
env: [
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
],
|
||||
env: [],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
@@ -320,9 +312,7 @@ export const config = {
|
||||
{
|
||||
match: { hw: "h200", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" },
|
||||
verified: true,
|
||||
env: [
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
],
|
||||
env: [],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
@@ -352,9 +342,7 @@ export const config = {
|
||||
{
|
||||
match: { hw: "dgx-spark", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "multi-2" },
|
||||
verified: true,
|
||||
env: [
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
],
|
||||
env: [],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
@@ -384,7 +372,6 @@ export const config = {
|
||||
verified: true,
|
||||
env: [
|
||||
"SGLANG_USE_AITER=1",
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
@@ -410,7 +397,6 @@ export const config = {
|
||||
verified: true,
|
||||
env: [
|
||||
"SGLANG_USE_AITER=1",
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
@@ -442,9 +428,7 @@ export const config = {
|
||||
{
|
||||
match: { hw: "b200", variant: "default", quant: "nvfp4", strategy: "long_context", nodes: "single" },
|
||||
verified: true,
|
||||
env: [
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
],
|
||||
env: [],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
@@ -470,9 +454,7 @@ export const config = {
|
||||
{
|
||||
match: { hw: "b300", variant: "default", quant: "nvfp4", strategy: "long_context", nodes: "single" },
|
||||
verified: true,
|
||||
env: [
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
],
|
||||
env: [],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
@@ -497,9 +479,7 @@ export const config = {
|
||||
},
|
||||
{
|
||||
match: { hw: "gb200", variant: "default", quant: "nvfp4", strategy: "long_context", nodes: "single" },
|
||||
env: [
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
],
|
||||
env: [],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
@@ -525,9 +505,7 @@ export const config = {
|
||||
{
|
||||
match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "long_context", nodes: "single" },
|
||||
verified: true,
|
||||
env: [
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
],
|
||||
env: [],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
@@ -560,9 +538,7 @@ export const config = {
|
||||
{
|
||||
match: { hw: "b200", variant: "default", quant: "nvfp4", strategy: "mtp", nodes: "single" },
|
||||
verified: true,
|
||||
env: [
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
],
|
||||
env: [],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
@@ -593,9 +569,7 @@ export const config = {
|
||||
{
|
||||
match: { hw: "b300", variant: "default", quant: "nvfp4", strategy: "mtp", nodes: "single" },
|
||||
verified: true,
|
||||
env: [
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
],
|
||||
env: [],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
@@ -625,9 +599,7 @@ export const config = {
|
||||
},
|
||||
{
|
||||
match: { hw: "gb200", variant: "default", quant: "nvfp4", strategy: "mtp", nodes: "single" },
|
||||
env: [
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
],
|
||||
env: [],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
@@ -658,9 +630,7 @@ export const config = {
|
||||
{
|
||||
match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "mtp", nodes: "single" },
|
||||
verified: true,
|
||||
env: [
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
],
|
||||
env: [],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
@@ -691,9 +661,7 @@ export const config = {
|
||||
{
|
||||
match: { hw: "h200", variant: "default", quant: "nvfp4", strategy: "mtp", nodes: "single" },
|
||||
verified: true,
|
||||
env: [
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
],
|
||||
env: [],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
@@ -729,9 +697,7 @@ export const config = {
|
||||
{
|
||||
match: { hw: "b200", variant: "default", quant: "nvfp4", strategy: "dspark", nodes: "single" },
|
||||
verified: true,
|
||||
env: [
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
],
|
||||
env: [],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
@@ -764,9 +730,7 @@ export const config = {
|
||||
{
|
||||
match: { hw: "b300", variant: "default", quant: "nvfp4", strategy: "dspark", nodes: "single" },
|
||||
verified: true,
|
||||
env: [
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
],
|
||||
env: [],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
@@ -799,9 +763,7 @@ export const config = {
|
||||
{
|
||||
match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "dspark", nodes: "single" },
|
||||
verified: true,
|
||||
env: [
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
],
|
||||
env: [],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
@@ -834,9 +796,7 @@ export const config = {
|
||||
{
|
||||
match: { hw: "h200", variant: "default", quant: "nvfp4", strategy: "dspark", nodes: "single" },
|
||||
verified: true,
|
||||
env: [
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
],
|
||||
env: [],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
@@ -879,7 +839,6 @@ export const config = {
|
||||
"NCCL_MNNVL_ENABLE=1",
|
||||
"NCCL_NVLS_ENABLE=1",
|
||||
"NCCL_CUMEM_ENABLE=1",
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
@@ -908,7 +867,6 @@ export const config = {
|
||||
"NCCL_MNNVL_ENABLE=1",
|
||||
"NCCL_NVLS_ENABLE=1",
|
||||
"NCCL_CUMEM_ENABLE=1",
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
@@ -940,9 +898,7 @@ export const config = {
|
||||
{
|
||||
match: { hw: "b300", variant: "default", quant: "bf16", strategy: "balanced", nodes: "single" },
|
||||
verified: true,
|
||||
env: [
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
],
|
||||
env: [],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
@@ -967,9 +923,7 @@ export const config = {
|
||||
{
|
||||
match: { hw: "b300", variant: "default", quant: "bf16", strategy: "mtp", nodes: "single" },
|
||||
verified: true,
|
||||
env: [
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
],
|
||||
env: [],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
@@ -997,9 +951,7 @@ export const config = {
|
||||
},
|
||||
{
|
||||
match: { hw: "b200", variant: "default", quant: "bf16", strategy: "balanced", nodes: "multi-2" },
|
||||
env: [
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
],
|
||||
env: [],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
@@ -1021,9 +973,7 @@ export const config = {
|
||||
},
|
||||
{
|
||||
match: { hw: "b200", variant: "default", quant: "bf16", strategy: "mtp", nodes: "multi-2" },
|
||||
env: [
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
],
|
||||
env: [],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
@@ -1058,7 +1008,6 @@ export const config = {
|
||||
{
|
||||
match: { hw: "b200", variant: "lora", quant: "nvfp4", strategy: "balanced", nodes: "single" },
|
||||
env: [
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
"SGLANG_EXPERIMENTAL_LORA_OPTI=1",
|
||||
"SGLANG_OPT_LORA_OVERLAP_MAIN_ALLOC=1",
|
||||
],
|
||||
@@ -1092,7 +1041,6 @@ export const config = {
|
||||
{
|
||||
match: { hw: "b300", variant: "lora", quant: "nvfp4", strategy: "balanced", nodes: "single" },
|
||||
env: [
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
"SGLANG_EXPERIMENTAL_LORA_OPTI=1",
|
||||
"SGLANG_OPT_LORA_OVERLAP_MAIN_ALLOC=1",
|
||||
],
|
||||
@@ -1126,7 +1074,6 @@ export const config = {
|
||||
{
|
||||
match: { hw: "gb200", variant: "lora", quant: "nvfp4", strategy: "balanced", nodes: "single" },
|
||||
env: [
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
"SGLANG_EXPERIMENTAL_LORA_OPTI=1",
|
||||
"SGLANG_OPT_LORA_OVERLAP_MAIN_ALLOC=1",
|
||||
],
|
||||
@@ -1161,7 +1108,6 @@ export const config = {
|
||||
match: { hw: "gb300", variant: "lora", quant: "nvfp4", strategy: "balanced", nodes: "single" },
|
||||
verified: true,
|
||||
env: [
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
"SGLANG_EXPERIMENTAL_LORA_OPTI=1",
|
||||
"SGLANG_OPT_LORA_OVERLAP_MAIN_ALLOC=1",
|
||||
],
|
||||
@@ -1195,7 +1141,6 @@ export const config = {
|
||||
match: { hw: "h200", variant: "lora", quant: "nvfp4", strategy: "balanced", nodes: "single" },
|
||||
verified: true,
|
||||
env: [
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
"SGLANG_EXPERIMENTAL_LORA_OPTI=1",
|
||||
"SGLANG_OPT_LORA_OVERLAP_MAIN_ALLOC=1",
|
||||
"SGLANG_OPT_USE_INKLING_SHEARED_BIAS=0",
|
||||
@@ -1231,7 +1176,6 @@ export const config = {
|
||||
match: { hw: "gb300", variant: "lora", quant: "bf16", strategy: "balanced", nodes: "single" },
|
||||
verified: true,
|
||||
env: [
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
"SGLANG_EXPERIMENTAL_LORA_OPTI=1",
|
||||
"SGLANG_OPT_LORA_OVERLAP_MAIN_ALLOC=1",
|
||||
"SGLANG_OPT_USE_JIT_KERNEL_MOE_ALIGN=1",
|
||||
@@ -1264,7 +1208,6 @@ export const config = {
|
||||
match: { hw: "h200", variant: "lora", quant: "bf16", strategy: "balanced", nodes: "single" },
|
||||
verified: true,
|
||||
env: [
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
"SGLANG_EXPERIMENTAL_LORA_OPTI=1",
|
||||
"SGLANG_OPT_LORA_OVERLAP_MAIN_ALLOC=1",
|
||||
"SGLANG_OPT_USE_JIT_KERNEL_MOE_ALIGN=1",
|
||||
|
||||
@@ -206,9 +206,7 @@ export const config = {
|
||||
{
|
||||
match: { hw: "b200", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" },
|
||||
verified: true,
|
||||
env: [
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
],
|
||||
env: [],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
@@ -233,9 +231,7 @@ export const config = {
|
||||
{
|
||||
match: { hw: "b300", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" },
|
||||
verified: true,
|
||||
env: [
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
],
|
||||
env: [],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
@@ -259,9 +255,7 @@ export const config = {
|
||||
},
|
||||
{
|
||||
match: { hw: "gb200", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" },
|
||||
env: [
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
],
|
||||
env: [],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
@@ -286,9 +280,7 @@ export const config = {
|
||||
{
|
||||
match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" },
|
||||
verified: true,
|
||||
env: [
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
],
|
||||
env: [],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
@@ -317,9 +309,7 @@ export const config = {
|
||||
{
|
||||
match: { hw: "h200", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" },
|
||||
verified: true,
|
||||
env: [
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
],
|
||||
env: [],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
@@ -349,7 +339,6 @@ export const config = {
|
||||
verified: true,
|
||||
env: [
|
||||
"SGLANG_USE_AITER=1",
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
@@ -375,7 +364,6 @@ export const config = {
|
||||
verified: true,
|
||||
env: [
|
||||
"SGLANG_USE_AITER=1",
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
@@ -407,9 +395,7 @@ export const config = {
|
||||
{
|
||||
match: { hw: "b200", variant: "default", quant: "nvfp4", strategy: "long_context", nodes: "single" },
|
||||
verified: true,
|
||||
env: [
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
],
|
||||
env: [],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
@@ -435,9 +421,7 @@ export const config = {
|
||||
{
|
||||
match: { hw: "b300", variant: "default", quant: "nvfp4", strategy: "long_context", nodes: "single" },
|
||||
verified: true,
|
||||
env: [
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
],
|
||||
env: [],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
@@ -462,9 +446,7 @@ export const config = {
|
||||
},
|
||||
{
|
||||
match: { hw: "gb200", variant: "default", quant: "nvfp4", strategy: "long_context", nodes: "single" },
|
||||
env: [
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
],
|
||||
env: [],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
@@ -490,9 +472,7 @@ export const config = {
|
||||
{
|
||||
match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "long_context", nodes: "single" },
|
||||
verified: true,
|
||||
env: [
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
],
|
||||
env: [],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
@@ -525,9 +505,7 @@ export const config = {
|
||||
{
|
||||
match: { hw: "b200", variant: "default", quant: "nvfp4", strategy: "mtp", nodes: "single" },
|
||||
verified: true,
|
||||
env: [
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
],
|
||||
env: [],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
@@ -558,9 +536,7 @@ export const config = {
|
||||
{
|
||||
match: { hw: "b300", variant: "default", quant: "nvfp4", strategy: "mtp", nodes: "single" },
|
||||
verified: true,
|
||||
env: [
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
],
|
||||
env: [],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
@@ -590,9 +566,7 @@ export const config = {
|
||||
},
|
||||
{
|
||||
match: { hw: "gb200", variant: "default", quant: "nvfp4", strategy: "mtp", nodes: "single" },
|
||||
env: [
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
],
|
||||
env: [],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
@@ -623,9 +597,7 @@ export const config = {
|
||||
{
|
||||
match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "mtp", nodes: "single" },
|
||||
verified: true,
|
||||
env: [
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
],
|
||||
env: [],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
@@ -656,9 +628,7 @@ export const config = {
|
||||
{
|
||||
match: { hw: "h200", variant: "default", quant: "nvfp4", strategy: "mtp", nodes: "single" },
|
||||
verified: true,
|
||||
env: [
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
],
|
||||
env: [],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
@@ -696,9 +666,7 @@ export const config = {
|
||||
{
|
||||
match: { hw: "b200", variant: "default", quant: "nvfp4", strategy: "dspark", nodes: "single" },
|
||||
verified: true,
|
||||
env: [
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
],
|
||||
env: [],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
@@ -742,7 +710,6 @@ export const config = {
|
||||
"NCCL_MNNVL_ENABLE=1",
|
||||
"NCCL_NVLS_ENABLE=1",
|
||||
"NCCL_CUMEM_ENABLE=1",
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
@@ -772,7 +739,6 @@ export const config = {
|
||||
"NCCL_MNNVL_ENABLE=1",
|
||||
"NCCL_NVLS_ENABLE=1",
|
||||
"NCCL_CUMEM_ENABLE=1",
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
@@ -804,9 +770,7 @@ export const config = {
|
||||
{
|
||||
match: { hw: "b300", variant: "default", quant: "bf16", strategy: "balanced", nodes: "single" },
|
||||
verified: true,
|
||||
env: [
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
],
|
||||
env: [],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
@@ -831,9 +795,7 @@ export const config = {
|
||||
{
|
||||
match: { hw: "b300", variant: "default", quant: "bf16", strategy: "mtp", nodes: "single" },
|
||||
verified: true,
|
||||
env: [
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
],
|
||||
env: [],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
@@ -863,9 +825,7 @@ export const config = {
|
||||
},
|
||||
{
|
||||
match: { hw: "b200", variant: "default", quant: "bf16", strategy: "balanced", nodes: "multi-2" },
|
||||
env: [
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
],
|
||||
env: [],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
@@ -887,9 +847,7 @@ export const config = {
|
||||
},
|
||||
{
|
||||
match: { hw: "b200", variant: "default", quant: "bf16", strategy: "mtp", nodes: "multi-2" },
|
||||
env: [
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
],
|
||||
env: [],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
@@ -925,7 +883,6 @@ export const config = {
|
||||
match: { hw: "b200", variant: "lora", quant: "nvfp4", strategy: "balanced", nodes: "single" },
|
||||
verified: true,
|
||||
env: [
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
"SGLANG_EXPERIMENTAL_LORA_OPTI=1",
|
||||
"SGLANG_OPT_LORA_OVERLAP_MAIN_ALLOC=1",
|
||||
],
|
||||
@@ -960,7 +917,6 @@ export const config = {
|
||||
match: { hw: "b300", variant: "lora", quant: "nvfp4", strategy: "balanced", nodes: "single" },
|
||||
verified: true,
|
||||
env: [
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
"SGLANG_EXPERIMENTAL_LORA_OPTI=1",
|
||||
"SGLANG_OPT_LORA_OVERLAP_MAIN_ALLOC=1",
|
||||
],
|
||||
@@ -994,7 +950,6 @@ export const config = {
|
||||
{
|
||||
match: { hw: "gb200", variant: "lora", quant: "nvfp4", strategy: "balanced", nodes: "single" },
|
||||
env: [
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
"SGLANG_EXPERIMENTAL_LORA_OPTI=1",
|
||||
"SGLANG_OPT_LORA_OVERLAP_MAIN_ALLOC=1",
|
||||
],
|
||||
@@ -1029,7 +984,6 @@ export const config = {
|
||||
match: { hw: "gb300", variant: "lora", quant: "nvfp4", strategy: "balanced", nodes: "single" },
|
||||
verified: true,
|
||||
env: [
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
"SGLANG_EXPERIMENTAL_LORA_OPTI=1",
|
||||
"SGLANG_OPT_LORA_OVERLAP_MAIN_ALLOC=1",
|
||||
],
|
||||
@@ -1064,7 +1018,6 @@ export const config = {
|
||||
match: { hw: "h200", variant: "lora", quant: "nvfp4", strategy: "balanced", nodes: "single" },
|
||||
verified: true,
|
||||
env: [
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
"SGLANG_EXPERIMENTAL_LORA_OPTI=1",
|
||||
"SGLANG_OPT_LORA_OVERLAP_MAIN_ALLOC=1",
|
||||
],
|
||||
@@ -1102,7 +1055,6 @@ export const config = {
|
||||
"NCCL_MNNVL_ENABLE=1",
|
||||
"NCCL_NVLS_ENABLE=1",
|
||||
"NCCL_CUMEM_ENABLE=1",
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
"SGLANG_EXPERIMENTAL_LORA_OPTI=1",
|
||||
"SGLANG_OPT_LORA_OVERLAP_MAIN_ALLOC=1",
|
||||
"SGLANG_OPT_USE_JIT_KERNEL_MOE_ALIGN=1",
|
||||
|
||||
@@ -11,7 +11,6 @@ from sglang.srt.arg_groups.model_override_base import (
|
||||
is_attention_backend_not_set,
|
||||
resolving_view,
|
||||
)
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.runtime_context import get_platform
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -76,5 +75,4 @@ def _inkling_overrides(server_args: Any, hf_config: Any) -> dict:
|
||||
f"Use {inkling_attn_backend} as the attention backend for Inkling "
|
||||
"(requires fa4 or triton)."
|
||||
)
|
||||
envs.SGLANG_ENABLE_UNIFIED_RADIX_TREE.set(True)
|
||||
return overrides
|
||||
|
||||
@@ -671,10 +671,6 @@ class Envs:
|
||||
SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS = EnvBool(True)
|
||||
# Decode batches between SWA out-of-window evictions.
|
||||
SGLANG_SWA_EVICTION_INTERVAL = EnvInt(128)
|
||||
# Deprecated: the unified radix tree is the default tree cache now, so the
|
||||
# registry no longer reads this. Kept because a few model/arch call sites
|
||||
# still assert on it; do not use in new code.
|
||||
SGLANG_ENABLE_UNIFIED_RADIX_TREE = EnvBool(False)
|
||||
# Registered TreeCore backend serving the unified radix cache.
|
||||
SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND = EnvStr("python")
|
||||
SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW = EnvBool(False)
|
||||
@@ -1975,8 +1971,7 @@ _DEPRECATED_ENVS: Dict[str, _DeprecatedEnv] = {
|
||||
"To override the threshold, use '--min-free-slots-delay'."
|
||||
),
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE": _DeprecatedEnv(
|
||||
note="The unified radix tree is the default tree cache now; unset this "
|
||||
"env. The field is still defined for legacy call sites."
|
||||
note="The unified radix tree is the default tree cache now; unset this env."
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
@@ -38,13 +38,14 @@ def walk_radix_cache_for_canary(
|
||||
f"walk_radix_cache_for_canary does not support {cache_type.__name__}"
|
||||
)
|
||||
|
||||
# RadixCache has no SWA tier, so every node it holds is SWA-resident and
|
||||
# swa_resident_only is a no-op here.
|
||||
slot_buf: list[int] = []
|
||||
position_buf: list[int] = []
|
||||
prev_slot_buf: list[int] = []
|
||||
|
||||
_walk_radix_subtree(
|
||||
node=radix_cache.root_node,
|
||||
radix_cache=radix_cache,
|
||||
depth=0,
|
||||
parent_last_slot=-1,
|
||||
slot_buf=slot_buf,
|
||||
@@ -52,7 +53,6 @@ def walk_radix_cache_for_canary(
|
||||
prev_slot_buf=prev_slot_buf,
|
||||
is_root=True,
|
||||
unlocked_only=unlocked_only,
|
||||
swa_resident_only=swa_resident_only,
|
||||
)
|
||||
|
||||
slot_tensor = torch.tensor(slot_buf, dtype=torch.int64)
|
||||
@@ -68,7 +68,6 @@ def walk_radix_cache_for_canary(
|
||||
def _walk_radix_subtree(
|
||||
*,
|
||||
node: TreeNode,
|
||||
radix_cache: BasePrefixCache,
|
||||
depth: int,
|
||||
parent_last_slot: int,
|
||||
slot_buf: list[int],
|
||||
@@ -76,21 +75,10 @@ def _walk_radix_subtree(
|
||||
prev_slot_buf: list[int],
|
||||
is_root: bool,
|
||||
unlocked_only: bool,
|
||||
swa_resident_only: bool,
|
||||
) -> None:
|
||||
node_slots = _node_slots_for_canary(node=node)
|
||||
|
||||
if unlocked_only:
|
||||
emit_slots = not is_root and _node_is_unlocked_for_canary(
|
||||
node=node, radix_cache=radix_cache
|
||||
)
|
||||
else:
|
||||
emit_slots = not is_root
|
||||
if swa_resident_only:
|
||||
emit_slots = emit_slots and _node_is_swa_resident_for_canary(
|
||||
node=node,
|
||||
radix_cache=radix_cache,
|
||||
)
|
||||
emit_slots = not is_root and (not unlocked_only or node.lock_ref == 0)
|
||||
|
||||
chain_last_slot = parent_last_slot
|
||||
for j, slot in enumerate(node_slots):
|
||||
@@ -105,7 +93,6 @@ def _walk_radix_subtree(
|
||||
for child in node.children.values():
|
||||
_walk_radix_subtree(
|
||||
node=child,
|
||||
radix_cache=radix_cache,
|
||||
depth=child_depth,
|
||||
parent_last_slot=chain_last_slot,
|
||||
slot_buf=slot_buf,
|
||||
@@ -113,7 +100,6 @@ def _walk_radix_subtree(
|
||||
prev_slot_buf=prev_slot_buf,
|
||||
is_root=False,
|
||||
unlocked_only=unlocked_only,
|
||||
swa_resident_only=swa_resident_only,
|
||||
)
|
||||
|
||||
|
||||
@@ -122,25 +108,3 @@ def _node_slots_for_canary(*, node: TreeNode) -> list[int]:
|
||||
if isinstance(value, torch.Tensor):
|
||||
return [int(s) for s in value.tolist()]
|
||||
return []
|
||||
|
||||
|
||||
def _node_is_unlocked_for_canary(
|
||||
*,
|
||||
node: TreeNode,
|
||||
radix_cache: BasePrefixCache,
|
||||
) -> bool:
|
||||
if type(radix_cache) is RadixCache:
|
||||
return node.lock_ref == 0
|
||||
|
||||
raise NotImplementedError(
|
||||
f"walk_radix_cache_for_canary does not support {type(radix_cache).__name__}"
|
||||
)
|
||||
|
||||
|
||||
def _node_is_swa_resident_for_canary(
|
||||
*,
|
||||
node: TreeNode,
|
||||
radix_cache: BasePrefixCache,
|
||||
) -> bool:
|
||||
# RadixCache has no SWA tier, so every node it holds is resident.
|
||||
return True
|
||||
|
||||
@@ -65,7 +65,7 @@ def free_swa_out_of_window_slots(
|
||||
if not req.kv.holds_kv:
|
||||
return
|
||||
|
||||
# For swa radix cache, we need to evict the tokens that are not in the tree cache and also not in the sliding window
|
||||
# For SWA-capable tree caches, we need to evict the tokens that are not in the tree cache and also not in the sliding window
|
||||
assert req.kv.cache_protected_len % page_size == 0, (
|
||||
"cache_protected_len must be page aligned"
|
||||
)
|
||||
|
||||
@@ -36,7 +36,6 @@ from sglang.srt.configs.hybrid_arch import (
|
||||
)
|
||||
from sglang.srt.configs.model_config import ModelImpl, is_deepseek_dsa
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.hardware_backend.mlx.runtime import use_mlx
|
||||
from sglang.srt.managers.mm_schedule import init_mm_embedding_cache
|
||||
from sglang.srt.mem_cache.base_swa_memory_pool import BaseSWAKVPool
|
||||
from sglang.srt.mem_cache.cache_init_params import CacheInitParams
|
||||
@@ -314,12 +313,6 @@ def build_kv_cache(
|
||||
and get_disagg().disaggregation_mode == "decode"
|
||||
):
|
||||
if is_hybrid_swa:
|
||||
if not (envs.SGLANG_ENABLE_UNIFIED_RADIX_TREE.get() or use_mlx()):
|
||||
raise ValueError(
|
||||
"--disaggregation-decode-enable-radix-cache with sliding "
|
||||
"window attention (SWA) models requires the unified radix "
|
||||
"tree (set SGLANG_ENABLE_UNIFIED_RADIX_TREE=1)."
|
||||
)
|
||||
if enable_hierarchical_cache:
|
||||
raise ValueError(
|
||||
"--disaggregation-decode-enable-radix-cache with sliding "
|
||||
|
||||
@@ -367,18 +367,6 @@ class UnifiedLRUList:
|
||||
return None
|
||||
return x
|
||||
|
||||
def get_prev_leaf_no_lock(self, node: UnifiedTreeNode, check_id: bool = True):
|
||||
if check_id:
|
||||
assert node.id in self.cache
|
||||
pt = self._pt
|
||||
ct = self.component_type
|
||||
x = node.lru_prev[pt]
|
||||
while x.component_data[ct].lock_ref > 0 or len(x.children) > 0:
|
||||
x = x.lru_prev[pt]
|
||||
if x == self.head:
|
||||
return None
|
||||
return x
|
||||
|
||||
def get_prev_no_host_lock(self, node: UnifiedTreeNode, check_id: bool = True):
|
||||
"""Host-LRU walker: skip nodes whose component host_lock_ref > 0."""
|
||||
if check_id:
|
||||
@@ -395,9 +383,6 @@ class UnifiedLRUList:
|
||||
def get_lru_no_lock(self):
|
||||
return self.get_prev_no_lock(self.tail, check_id=False)
|
||||
|
||||
def get_leaf_lru_no_lock(self):
|
||||
return self.get_prev_leaf_no_lock(self.tail, check_id=False)
|
||||
|
||||
def get_lru_no_host_lock(self):
|
||||
return self.get_prev_no_host_lock(self.tail, check_id=False)
|
||||
|
||||
|
||||
@@ -1014,7 +1014,6 @@ class InklingForConditionalGeneration(nn.Module):
|
||||
self.config = config
|
||||
self.text_config = config.text_config
|
||||
|
||||
assert envs.SGLANG_ENABLE_UNIFIED_RADIX_TREE.get()
|
||||
if get_disagg().disaggregation_mode != "decode":
|
||||
assert not get_memory().disable_radix_cache
|
||||
assert not get_schedule().disable_hybrid_swa_memory
|
||||
|
||||
@@ -668,7 +668,6 @@ def unified_radix_tree_server_env(
|
||||
**os.environ,
|
||||
**extra_env,
|
||||
"SGLANG_ENABLE_RANK_CONSENSUS_CHECKER": "1",
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE": "1",
|
||||
"SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND": tree_core_backend,
|
||||
}
|
||||
|
||||
|
||||
@@ -44,9 +44,6 @@ class TestDisaggregationDecodeRadixCacheSWANixl(
|
||||
# so keep the original 0.45 absolute floor and rely on the two-pass
|
||||
# non-regression check below to catch decode-cache corruption.
|
||||
gsm8k_min_score = 0.45
|
||||
# SWA + decode-side radix cache is gated to the unified radix tree.
|
||||
extra_prefill_env = {"SGLANG_ENABLE_UNIFIED_RADIX_TREE": "1"}
|
||||
extra_decode_env = {"SGLANG_ENABLE_UNIFIED_RADIX_TREE": "1"}
|
||||
extra_prefill_args = SWA_SERVER_ARGS
|
||||
extra_decode_args = [
|
||||
"--disaggregation-decode-enable-radix-cache",
|
||||
|
||||
@@ -28,10 +28,6 @@ MODEL = os.environ.get(
|
||||
"INKLING_SMALL_TEST_MODEL_PATH", "thinkingmachines/Inkling-Small-NVFP4"
|
||||
)
|
||||
|
||||
# The unified radix tree is what merges the three components into one tree, so
|
||||
# it is a precondition rather than a tuning knob here.
|
||||
ENV = {"SGLANG_ENABLE_UNIFIED_RADIX_TREE": "1"}
|
||||
|
||||
# Shared with the single-server Inkling recipe; TP is per role.
|
||||
COMMON_ARGS = [
|
||||
"--tp",
|
||||
@@ -104,7 +100,6 @@ class TestDisaggregationInklingMXFP8(PDDisaggregationServerBase, GSM8KMixin):
|
||||
cls.prefill_url,
|
||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
other_args=prefill_args,
|
||||
env={**os.environ, **ENV},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
@@ -125,7 +120,6 @@ class TestDisaggregationInklingMXFP8(PDDisaggregationServerBase, GSM8KMixin):
|
||||
cls.decode_url,
|
||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
other_args=decode_args,
|
||||
env={**os.environ, **ENV},
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -37,9 +37,6 @@ register_cuda_ci(est_time=900, stage="extra-a", runner_config="2-gpu-large")
|
||||
_MODEL_PATH = os.environ.get("INKLING_TEST_MODEL_PATH", "thinkingmachines/Inkling")
|
||||
_MODEL_REVISION = os.environ.get("INKLING_TEST_MODEL_REVISION", "test")
|
||||
|
||||
# The unified radix tree is what merges the three components into one tree.
|
||||
SERVER_ENV = {"SGLANG_ENABLE_UNIFIED_RADIX_TREE": "1"}
|
||||
|
||||
UNIFIED_TRI_ARGS = [
|
||||
"--skip-tokenizer-init",
|
||||
"--random-seed",
|
||||
@@ -72,8 +69,6 @@ class TestUnifiedMemoryDisaggregationTriPool(
|
||||
"""1 prefill + 1 decode, both unified, vs a non-PD unified reference."""
|
||||
|
||||
model = _MODEL_PATH
|
||||
extra_prefill_env = SERVER_ENV
|
||||
extra_decode_env = SERVER_ENV
|
||||
prefill_tp_size = 1
|
||||
decode_tp_size = 1
|
||||
decode_base_gpu_id = 1
|
||||
|
||||
@@ -6,7 +6,6 @@ same unified-memory configuration to keep attention reduction order comparable.
|
||||
Covers GDN, SWA, tri-pool, and MLA layouts.
|
||||
"""
|
||||
|
||||
import os
|
||||
import time
|
||||
import unittest
|
||||
|
||||
@@ -67,7 +66,6 @@ class UnifiedMemoryHiCacheBase(CustomTestCase):
|
||||
|
||||
model: str = ""
|
||||
extra_args: list = []
|
||||
server_env: dict = {}
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
@@ -76,7 +74,6 @@ class UnifiedMemoryHiCacheBase(CustomTestCase):
|
||||
base_args = _COMMON_ARGS + cls.extra_args
|
||||
cls.hicache_url = "http://127.0.0.1:8157"
|
||||
cls.reference_url = "http://127.0.0.1:8158"
|
||||
env = {**os.environ, **cls.server_env} if cls.server_env else None
|
||||
hicache_args = ["--enable-hierarchical-cache"]
|
||||
if "--hicache-size" not in base_args:
|
||||
hicache_args += ["--hicache-ratio", "4"]
|
||||
@@ -85,7 +82,6 @@ class UnifiedMemoryHiCacheBase(CustomTestCase):
|
||||
cls.hicache_url,
|
||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
other_args=base_args + hicache_args,
|
||||
env=env,
|
||||
)
|
||||
cls.addClassCleanup(kill_process_tree, cls.process_hicache.pid)
|
||||
cls.process_reference = popen_launch_server(
|
||||
@@ -93,7 +89,6 @@ class UnifiedMemoryHiCacheBase(CustomTestCase):
|
||||
cls.reference_url,
|
||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
other_args=base_args + ["--base-gpu-id", "1"],
|
||||
env=env,
|
||||
)
|
||||
cls.addClassCleanup(kill_process_tree, cls.process_reference.pid)
|
||||
|
||||
@@ -188,7 +183,6 @@ class TestUnifiedMemoryHiCacheTriPool(UnifiedMemoryHiCacheBase):
|
||||
|
||||
# The test revision is the reduced checkpoint used by Inkling CI.
|
||||
model = "thinkingmachines/Inkling"
|
||||
server_env = {"SGLANG_ENABLE_UNIFIED_RADIX_TREE": "1"}
|
||||
extra_args = _SMALL_POOL + [
|
||||
"--revision",
|
||||
"test",
|
||||
|
||||
@@ -86,7 +86,6 @@ class TestInklingServer(CustomTestCase):
|
||||
cls.base_url,
|
||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
other_args=other_args,
|
||||
env={**os.environ, "SGLANG_ENABLE_UNIFIED_RADIX_TREE": "1"},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
|
||||
@@ -112,7 +112,6 @@ class TestInklingSmallNvfp4(CustomTestCase):
|
||||
"--mem-fraction-static",
|
||||
"0.85",
|
||||
],
|
||||
env={**os.environ, "SGLANG_ENABLE_UNIFIED_RADIX_TREE": "1"},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
@@ -202,7 +201,6 @@ class TestInklingSmallNvfp4DsparkDeterministic(CustomTestCase):
|
||||
"--speculative-draft-attention-backend",
|
||||
"fa4",
|
||||
],
|
||||
env={**os.environ, "SGLANG_ENABLE_UNIFIED_RADIX_TREE": "1"},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
@@ -316,7 +314,6 @@ class TestInklingSmallNvfp4HiCacheDeterministic(CustomTestCase):
|
||||
"4",
|
||||
"--enable-deterministic-inference",
|
||||
],
|
||||
env={**os.environ, "SGLANG_ENABLE_UNIFIED_RADIX_TREE": "1"},
|
||||
)
|
||||
cls.input_ids = get_input_ids(cls.model, num_samples=18, trust_remote_code=True)
|
||||
|
||||
|
||||
@@ -141,7 +141,6 @@ class TestInklingUnifiedTriPool(CustomTestCase):
|
||||
cls.base_url,
|
||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
other_args=cls.server_args(),
|
||||
env={**os.environ, "SGLANG_ENABLE_UNIFIED_RADIX_TREE": "1"},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
@@ -235,7 +234,6 @@ class TestInklingUnifiedVsStaticParity(CustomTestCase):
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
other_args=other_args,
|
||||
env={**os.environ, "SGLANG_ENABLE_UNIFIED_RADIX_TREE": "1"},
|
||||
)
|
||||
try:
|
||||
out = []
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import unittest
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
|
||||
from sglang.test.server_fixtures.mmmu_fixture import MMMUServerBase
|
||||
@@ -50,11 +49,6 @@ class TestMiMoV2(GSM8KMixin, MMMUServerBase):
|
||||
server_api_key = None
|
||||
other_args = MIMO_V2_MTP_OTHER_ARGS
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
with envs.SGLANG_ENABLE_UNIFIED_RADIX_TREE.override(True):
|
||||
super().setUpClass()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import unittest
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.utils import is_blackwell
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
|
||||
@@ -54,11 +53,6 @@ class TestMiMoV2Flash(GSM8KMixin, SpecDecodingMixin, DefaultServerBase):
|
||||
bs_1_speed_thres = 170
|
||||
accept_length_thres = 3.2
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
with envs.SGLANG_ENABLE_UNIFIED_RADIX_TREE.override(True):
|
||||
super().setUpClass()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import unittest
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.test.accuracy_test_runner import AccuracyTestParams
|
||||
from sglang.test.cache_consistency_jitter import get_jitter_engine, run_jitter_test
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
@@ -58,24 +57,23 @@ class TestInklingNVFP4Nightly(unittest.TestCase):
|
||||
),
|
||||
]
|
||||
|
||||
with envs.SGLANG_ENABLE_UNIFIED_RADIX_TREE.override(1):
|
||||
run_combined_tests(
|
||||
models=variants,
|
||||
test_name="Inkling-NVFP4",
|
||||
accuracy_params=AccuracyTestParams(
|
||||
dataset="gsm8k",
|
||||
baseline_accuracy=GSM8K_BASELINE,
|
||||
num_examples=1314,
|
||||
num_threads=512,
|
||||
max_tokens=16000,
|
||||
temperature=1.0,
|
||||
top_p=0.95,
|
||||
repeat=1,
|
||||
),
|
||||
performance_params=PerformanceTestParams(
|
||||
result_dir="performance_results_inkling_nvfp4",
|
||||
),
|
||||
)
|
||||
run_combined_tests(
|
||||
models=variants,
|
||||
test_name="Inkling-NVFP4",
|
||||
accuracy_params=AccuracyTestParams(
|
||||
dataset="gsm8k",
|
||||
baseline_accuracy=GSM8K_BASELINE,
|
||||
num_examples=1314,
|
||||
num_threads=512,
|
||||
max_tokens=16000,
|
||||
temperature=1.0,
|
||||
top_p=0.95,
|
||||
repeat=1,
|
||||
),
|
||||
performance_params=PerformanceTestParams(
|
||||
result_dir="performance_results_inkling_nvfp4",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class TestInklingSmallCacheConsistencyNightly(unittest.TestCase):
|
||||
@@ -85,27 +83,26 @@ class TestInklingSmallCacheConsistencyNightly(unittest.TestCase):
|
||||
|
||||
@unittest.skipIf(not is_blackwell_system(), "NVFP4 requires Blackwell")
|
||||
def test_scored_contexts_are_bitwise_identical(self):
|
||||
with envs.SGLANG_ENABLE_UNIFIED_RADIX_TREE.override(1):
|
||||
with get_jitter_engine(
|
||||
model_path=INKLING_SMALL_NVFP4_MODEL,
|
||||
tp_size=8,
|
||||
trust_remote_code=True,
|
||||
quantization="modelopt_fp4",
|
||||
attention_backend="fa4",
|
||||
page_size=128,
|
||||
fp4_gemm_runner_backend="flashinfer_trtllm",
|
||||
moe_runner_backend="flashinfer_trtllm_routed",
|
||||
enable_torch_symm_mem=True,
|
||||
mamba_radix_cache_strategy="extra_buffer",
|
||||
swa_full_tokens_ratio=0.1,
|
||||
mamba_full_memory_ratio=0.1,
|
||||
mem_fraction_static=0.6,
|
||||
enable_deterministic_inference=True,
|
||||
# The harness pins a tight pool; this checkpoint needs room for
|
||||
# the 2048-4096 token prefixes the default workload draws.
|
||||
max_total_tokens=131_072,
|
||||
) as engine:
|
||||
run_jitter_test(engine)
|
||||
with get_jitter_engine(
|
||||
model_path=INKLING_SMALL_NVFP4_MODEL,
|
||||
tp_size=8,
|
||||
trust_remote_code=True,
|
||||
quantization="modelopt_fp4",
|
||||
attention_backend="fa4",
|
||||
page_size=128,
|
||||
fp4_gemm_runner_backend="flashinfer_trtllm",
|
||||
moe_runner_backend="flashinfer_trtllm_routed",
|
||||
enable_torch_symm_mem=True,
|
||||
mamba_radix_cache_strategy="extra_buffer",
|
||||
swa_full_tokens_ratio=0.1,
|
||||
mamba_full_memory_ratio=0.1,
|
||||
mem_fraction_static=0.6,
|
||||
enable_deterministic_inference=True,
|
||||
# The harness pins a tight pool; this checkpoint needs room for
|
||||
# the 2048-4096 token prefixes the default workload draws.
|
||||
max_total_tokens=131_072,
|
||||
) as engine:
|
||||
run_jitter_test(engine)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -32,9 +32,6 @@ register_cuda_ci(est_time=308, stage="base-b", runner_config="2-gpu-large")
|
||||
|
||||
|
||||
class TestHiCacheStorageRuntimeAttachDetach(CustomTestCase):
|
||||
# Extra server env; subclasses use it to select a tree_cache implementation.
|
||||
extra_env: dict = {}
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.temp_dir = tempfile.mkdtemp()
|
||||
@@ -64,7 +61,6 @@ class TestHiCacheStorageRuntimeAttachDetach(CustomTestCase):
|
||||
# Make runs less flaky for CI/dev.
|
||||
"SGLANG_ENABLE_DETERMINISTIC_INFERENCE": "1",
|
||||
"SGLANG_ENABLE_RANK_CONSENSUS_CHECKER": "1",
|
||||
**cls.extra_env,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
@@ -372,18 +368,5 @@ class TestHiCacheStorageRuntimeAttachDetach(CustomTestCase):
|
||||
time.sleep(2)
|
||||
|
||||
|
||||
class TestUnifiedRadixCacheStorageRuntimeAttachDetach(
|
||||
TestHiCacheStorageRuntimeAttachDetach
|
||||
):
|
||||
"""Same runtime attach/detach lifecycle, backed by UnifiedRadixCache."""
|
||||
|
||||
extra_env = {"SGLANG_ENABLE_UNIFIED_RADIX_TREE": "1"}
|
||||
|
||||
def test_runtime_attach_detach(self):
|
||||
# Admin-key gating (phase A of the base test) lives in the HTTP layer and is
|
||||
# independent of the tree cache implementation, so only the lifecycle is run.
|
||||
self._check_attach_detach_lifecycle()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(verbosity=2)
|
||||
|
||||
@@ -126,7 +126,6 @@ class TestHiCacheStorageUMBPBackend(CustomTestCase):
|
||||
{
|
||||
"SGLANG_ENABLE_DETERMINISTIC_INFERENCE": "1",
|
||||
"SGLANG_ENABLE_RANK_CONSENSUS_CHECKER": "1",
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE": "1",
|
||||
"SGLANG_DSV4_FP4_EXPERTS": "0",
|
||||
"SGLANG_HACK_FLASHMLA_BACKEND": "unified_kv_triton",
|
||||
"SGLANG_USE_ROCM700A": "0",
|
||||
|
||||
@@ -21,7 +21,6 @@ Usage:
|
||||
python3 -m unittest test_int8_mamba_checkpoint_e2e
|
||||
"""
|
||||
|
||||
import time
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from urllib.parse import urlparse
|
||||
@@ -30,21 +29,19 @@ from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.kits.kl_divergence_kit import KLDivergenceMixin
|
||||
from sglang.test.server_fixtures.default_fixture import (
|
||||
DefaultServerBase,
|
||||
openai_api_env,
|
||||
)
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_HYBRID_MAMBA_MODEL_NAME_FOR_TEST,
|
||||
popen_launch_server,
|
||||
terminate_and_kill_process_tree,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=344, stage="extra-b", runner_config="4-gpu-h100")
|
||||
register_cuda_ci(est_time=172, stage="extra-b", runner_config="4-gpu-h100")
|
||||
|
||||
|
||||
class TestInt8MambaCheckpointE2E(KLDivergenceMixin, DefaultServerBase):
|
||||
"""int8 mamba checkpoint pool on Qwen3-Next-80B-A3B (GDN-hybrid)."""
|
||||
|
||||
model = DEFAULT_HYBRID_MAMBA_MODEL_NAME_FOR_TEST
|
||||
server_env = {"SGLANG_ENABLE_RANK_CONSENSUS_CHECKER": "1"}
|
||||
|
||||
# Cache-hit KL: int8 is a lossy codec, so its cache-hit divergence is
|
||||
# inherently larger than the bf16/fp8 reuse the other KL tests bound (~0.005),
|
||||
@@ -97,30 +94,5 @@ class TestInt8MambaCheckpointE2E(KLDivergenceMixin, DefaultServerBase):
|
||||
self.assertGreaterEqual(metrics["accuracy"], self.gsm8k_threshold)
|
||||
|
||||
|
||||
class TestUnifiedRadixTreeInt8MambaCheckpointE2E(TestInt8MambaCheckpointE2E):
|
||||
"""Run the same int8 mamba checkpoint checks with UnifiedRadixTree forced on."""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
assert cls.model is not None, "Please set cls.model in subclass"
|
||||
|
||||
with openai_api_env(cls.api_key):
|
||||
cls.process = popen_launch_server(
|
||||
cls.model,
|
||||
cls.base_url,
|
||||
timeout=cls.timeout,
|
||||
other_args=cls.other_args,
|
||||
env={
|
||||
"SGLANG_ENABLE_RANK_CONSENSUS_CHECKER": "1",
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE": "1",
|
||||
},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
terminate_and_kill_process_tree(cls.process, wait_timeout=60)
|
||||
time.sleep(2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
-1
@@ -73,7 +73,6 @@ class TestDeepSeekV4FlashUnifiedCacheLinkerKL(
|
||||
env={
|
||||
**cls.mooncake.server_env(),
|
||||
"SGLANG_DSV4_FP4_EXPERTS": "0",
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE": "1",
|
||||
},
|
||||
)
|
||||
cls.input_ids = get_input_ids(cls.model, num_samples=18)
|
||||
|
||||
-2
@@ -101,7 +101,6 @@ class TestUnifiedQwen3HiCachePP(UnifiedRadixTreeTestMixin, CustomTestCase):
|
||||
],
|
||||
env={
|
||||
"SGLANG_ENABLE_RANK_CONSENSUS_CHECKER": "1",
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE": "1",
|
||||
},
|
||||
)
|
||||
cls.input_ids = get_input_ids(cls.model, num_samples=18)
|
||||
@@ -154,7 +153,6 @@ class TestUnifiedQwen3HiCachePPL3(AccuracyTwoPassMixin, CustomTestCase):
|
||||
],
|
||||
env={
|
||||
"SGLANG_ENABLE_RANK_CONSENSUS_CHECKER": "1",
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE": "1",
|
||||
"SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR": cls.hicache_dir,
|
||||
},
|
||||
)
|
||||
|
||||
@@ -65,7 +65,6 @@ class TestUnifiedQwen3HiCacheCP(UnifiedRadixTreeTestMixin, CustomTestCase):
|
||||
],
|
||||
env={
|
||||
"SGLANG_ENABLE_RANK_CONSENSUS_CHECKER": "1",
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE": "1",
|
||||
},
|
||||
)
|
||||
cls.input_ids = get_input_ids(cls.model, num_samples=18)
|
||||
|
||||
@@ -92,7 +92,6 @@ class TestUnifiedKimiLinearDcpHiCache(UnifiedRadixTreeTestMixin, CustomTestCase)
|
||||
],
|
||||
env={
|
||||
"SGLANG_ENABLE_RANK_CONSENSUS_CHECKER": "1",
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE": "1",
|
||||
},
|
||||
)
|
||||
cls.input_ids = get_input_ids(cls.model, num_samples=18, trust_remote_code=True)
|
||||
|
||||
@@ -71,7 +71,6 @@ class TestGLM5UnifiedRadixCacheL3Accuracy(AccuracyTwoPassMixin, CustomTestCase):
|
||||
env={
|
||||
"SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR": cls.hicache_dir,
|
||||
"SGLANG_ENABLE_RANK_CONSENSUS_CHECKER": "1",
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE": "1",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@@ -65,7 +65,6 @@ class TestUnifiedMambaRadixCache(UnifiedRadixTreeTestMixin, CustomTestCase):
|
||||
],
|
||||
env={
|
||||
"SGLANG_ENABLE_RANK_CONSENSUS_CHECKER": "1",
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE": "1",
|
||||
},
|
||||
)
|
||||
cls.input_ids = get_input_ids(cls.model, num_samples=18)
|
||||
@@ -127,7 +126,6 @@ class TestUnifiedMambaHiCache(UnifiedRadixTreeTestMixin, CustomTestCase):
|
||||
],
|
||||
env={
|
||||
"SGLANG_ENABLE_RANK_CONSENSUS_CHECKER": "1",
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE": "1",
|
||||
},
|
||||
)
|
||||
cls.input_ids = get_input_ids(cls.model, num_samples=18)
|
||||
@@ -197,7 +195,6 @@ class TestUnifiedMambaHiCacheL3(AccuracyTwoPassMixin, CustomTestCase):
|
||||
],
|
||||
env={
|
||||
"SGLANG_ENABLE_RANK_CONSENSUS_CHECKER": "1",
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE": "1",
|
||||
"SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR": cls.hicache_dir,
|
||||
},
|
||||
)
|
||||
|
||||
@@ -91,7 +91,6 @@ class TestUnifiedMiMoHiCacheLoadBackKL(CustomTestCase):
|
||||
],
|
||||
env={
|
||||
"SGLANG_ENABLE_RANK_CONSENSUS_CHECKER": "1",
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE": "1",
|
||||
"SGLANG_USE_CUDA_IPC_TRANSPORT": "1",
|
||||
},
|
||||
)
|
||||
|
||||
@@ -39,7 +39,6 @@ class TestUnifiedSWARadixCache(UnifiedRadixTreeTestMixin, CustomTestCase):
|
||||
],
|
||||
env={
|
||||
"SGLANG_ENABLE_RANK_CONSENSUS_CHECKER": "1",
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE": "1",
|
||||
},
|
||||
)
|
||||
cls.input_ids = get_input_ids(cls.model, num_samples=18)
|
||||
|
||||
@@ -29,10 +29,6 @@ register_amd_ci(est_time=9, suite="stage-b-test-1-gpu-small-amd")
|
||||
register_xpu_ci(est_time=20, suite="stage-b-test-1-gpu-xpu")
|
||||
|
||||
|
||||
def _event_hashes(events):
|
||||
return [block_hash for event in events for block_hash in event.block_hashes]
|
||||
|
||||
|
||||
class TestMamba(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
|
||||
@@ -364,7 +364,7 @@ class TestDefaultRadixCacheFactory(CustomTestCase):
|
||||
)
|
||||
self.assertIs(result, cache)
|
||||
|
||||
def test_swa_radix_cache_when_hybrid_swa(self):
|
||||
def test_unified_radix_cache_when_hybrid_swa(self):
|
||||
ctx = _make_ctx(self, is_hybrid_swa=True)
|
||||
# SWA hybrid models now default to the unified radix tree.
|
||||
fake_components = MagicMock()
|
||||
@@ -390,7 +390,7 @@ class TestDefaultRadixCacheFactory(CustomTestCase):
|
||||
PureSWA.assert_called_once_with(params=ctx.params)
|
||||
self.assertIs(result, PureSWA.return_value)
|
||||
|
||||
def test_mamba_radix_cache_when_hybrid_ssm(self):
|
||||
def test_unified_radix_cache_when_hybrid_ssm(self):
|
||||
ctx = _make_ctx(self, is_hybrid_ssm=True)
|
||||
# Mamba hybrid models now default to the unified radix tree.
|
||||
fake_components = MagicMock()
|
||||
|
||||
@@ -27,22 +27,12 @@ register_cuda_ci(est_time=9, stage="base-b", runner_config="1-gpu-large")
|
||||
register_amd_ci(est_time=10, suite="stage-b-test-1-gpu-small-amd")
|
||||
|
||||
|
||||
class _DummyReq:
|
||||
def __init__(self):
|
||||
self._kv_committed_len = 0
|
||||
self.swa_prefix_lock_released = False
|
||||
self.kv = SimpleNamespace(swa_evicted_seqlen=0, cache_protected_len=0)
|
||||
|
||||
|
||||
def _build_swa_tree(
|
||||
is_eagle: bool,
|
||||
def _build_swa_allocator(
|
||||
page_size: int = 1,
|
||||
req_size: int = 8,
|
||||
max_context_len: int = 64,
|
||||
kv_size: int = 64,
|
||||
kv_size_swa: int = 32,
|
||||
sliding_window_size: int = 4,
|
||||
enable_kv_cache_events: bool = False,
|
||||
swa_req_ring_size: int | None = None,
|
||||
):
|
||||
head_num = 8
|
||||
@@ -149,12 +139,10 @@ class TestSWA(unittest.TestCase):
|
||||
|
||||
def test_swa_memory_pool_paged_free_clears_full_page_mapping(self):
|
||||
page_size = 4
|
||||
allocator, _ = _build_swa_tree(
|
||||
is_eagle=False,
|
||||
allocator, _ = _build_swa_allocator(
|
||||
page_size=page_size,
|
||||
kv_size=16,
|
||||
kv_size_swa=16,
|
||||
sliding_window_size=page_size,
|
||||
)
|
||||
|
||||
full_indices = _swa_alloc(allocator, page_size)
|
||||
@@ -176,7 +164,7 @@ class TestSWA(unittest.TestCase):
|
||||
"""Clearing the full-to-SWA mapping must not block the stream; writing a
|
||||
host-resident scalar into it does.
|
||||
"""
|
||||
allocator, _ = _build_swa_tree(is_eagle=False)
|
||||
allocator, _ = _build_swa_allocator()
|
||||
full_indices = _swa_alloc(allocator, 4)
|
||||
mapping = allocator.full_to_swa_index_mapping
|
||||
|
||||
@@ -202,8 +190,7 @@ class TestSWA(unittest.TestCase):
|
||||
self._free_swa_group_owns_deferred_indices(page_size)
|
||||
|
||||
def _free_swa_group_owns_deferred_indices(self, page_size):
|
||||
allocator, _ = _build_swa_tree(
|
||||
is_eagle=False,
|
||||
allocator, _ = _build_swa_allocator(
|
||||
page_size=page_size,
|
||||
kv_size=32 * page_size,
|
||||
kv_size_swa=32 * page_size,
|
||||
@@ -239,8 +226,7 @@ class TestSWA(unittest.TestCase):
|
||||
)
|
||||
|
||||
def test_free_swa_group_owns_mapping_at_enqueue_time(self):
|
||||
allocator, _ = _build_swa_tree(
|
||||
is_eagle=False,
|
||||
allocator, _ = _build_swa_allocator(
|
||||
kv_size=8,
|
||||
kv_size_swa=8,
|
||||
)
|
||||
@@ -271,8 +257,7 @@ class TestSWA(unittest.TestCase):
|
||||
)
|
||||
|
||||
def _build_two_mapped_slots(self, page_size=1):
|
||||
allocator, _ = _build_swa_tree(
|
||||
is_eagle=False,
|
||||
allocator, _ = _build_swa_allocator(
|
||||
page_size=page_size,
|
||||
kv_size=8 * page_size,
|
||||
kv_size_swa=8 * page_size,
|
||||
@@ -372,7 +357,7 @@ class TestFreeFullPartition(CustomTestCase):
|
||||
"""`free_full` releases only the full side of a hybrid SWA allocator."""
|
||||
|
||||
def setUp(self):
|
||||
self.allocator, _ = _build_swa_tree(is_eagle=False)
|
||||
self.allocator, _ = _build_swa_allocator()
|
||||
self.full_baseline = self.allocator.full_available_size()
|
||||
self.swa_baseline = self.allocator.swa_available_size()
|
||||
|
||||
@@ -420,7 +405,7 @@ class TestFreeKvRow(CustomTestCase):
|
||||
whole, the SWA side only from the floor up."""
|
||||
|
||||
def setUp(self):
|
||||
self.allocator, _ = _build_swa_tree(is_eagle=False)
|
||||
self.allocator, _ = _build_swa_allocator()
|
||||
self.full_baseline = self.allocator.full_available_size()
|
||||
self.swa_baseline = self.allocator.swa_available_size()
|
||||
|
||||
@@ -457,7 +442,7 @@ class TestFreeKvRow(CustomTestCase):
|
||||
self.assertEqual(self._sizes(), (self.full_baseline, self.swa_baseline))
|
||||
|
||||
def test_below_floor_pieces_go_back_through_the_full_side(self):
|
||||
allocator, _ = _build_swa_tree(is_eagle=False, page_size=4)
|
||||
allocator, _ = _build_swa_allocator(page_size=4)
|
||||
indices = _swa_alloc(allocator, 8)
|
||||
allocator.free_swa(indices)
|
||||
after_alloc = allocator.full_available_size()
|
||||
@@ -477,7 +462,7 @@ class TestFreeKvRow(CustomTestCase):
|
||||
self.assertEqual(allocator.full_available_size(), after_alloc + 8)
|
||||
|
||||
def test_grouped_full_side_frees_defer_and_skip_the_unique_path(self):
|
||||
allocator, _ = _build_swa_tree(is_eagle=False, page_size=4)
|
||||
allocator, _ = _build_swa_allocator(page_size=4)
|
||||
indices = _swa_alloc(allocator, 12)
|
||||
allocator.free_swa(indices[:8])
|
||||
after_alloc = allocator.full_available_size()
|
||||
@@ -540,7 +525,7 @@ class TestSWAPeerMappedContract(CustomTestCase):
|
||||
return bool(assert_async.call_args.args[0])
|
||||
|
||||
def test_segment_free_flags_a_page_whose_peer_is_already_gone(self):
|
||||
allocator, _ = _build_swa_tree(is_eagle=False, page_size=4)
|
||||
allocator, _ = _build_swa_allocator(page_size=4)
|
||||
live = _swa_alloc(allocator, 8)
|
||||
stale = _swa_alloc(allocator, 8)
|
||||
allocator.clear_full_to_swa_mapping(stale)
|
||||
@@ -553,7 +538,7 @@ class TestSWAPeerMappedContract(CustomTestCase):
|
||||
"""page_size > 1: page reps by stride replace the page expansion's
|
||||
filter and the inner allocator's torch.unique, in and out of a group."""
|
||||
ps = 4
|
||||
allocator, _ = _build_swa_tree(is_eagle=False, page_size=ps)
|
||||
allocator, _ = _build_swa_allocator(page_size=ps)
|
||||
|
||||
def grouped(indices):
|
||||
allocator.free_group_begin()
|
||||
@@ -581,7 +566,7 @@ class TestSWAPeerMappedContract(CustomTestCase):
|
||||
self.assertIsNone(_sync_error(lambda: grouped(second[: 2 * ps - 1])))
|
||||
|
||||
def test_free_swa_flags_a_slot_whose_peer_is_already_gone(self):
|
||||
allocator, _ = _build_swa_tree(is_eagle=False)
|
||||
allocator, _ = _build_swa_allocator()
|
||||
live = _swa_alloc(allocator, 4)
|
||||
stale = _swa_alloc(allocator, 4)
|
||||
# Whoever released the peer left the mapping reading as the padding slot.
|
||||
@@ -594,7 +579,7 @@ class TestSWAPeerMappedContract(CustomTestCase):
|
||||
def test_free_swa_does_not_synchronize(self):
|
||||
"""The filter's output shape was data-dependent, so it read a count back
|
||||
to the host; the gather that replaced it has a fixed shape."""
|
||||
allocator, _ = _build_swa_tree(is_eagle=False)
|
||||
allocator, _ = _build_swa_allocator()
|
||||
mapping = allocator.full_to_swa_index_mapping
|
||||
|
||||
# Warm up outside the window: a first-time cudaMalloc can synchronize on
|
||||
@@ -618,8 +603,7 @@ class TestSWAReqRingFree(CustomTestCase):
|
||||
|
||||
def _allocated_ring(self):
|
||||
ps = self.PS
|
||||
allocator, req_pool = _build_swa_tree(
|
||||
is_eagle=False,
|
||||
allocator, req_pool = _build_swa_allocator(
|
||||
page_size=ps,
|
||||
req_size=2,
|
||||
max_context_len=4 * ps,
|
||||
@@ -738,7 +722,7 @@ class TestSWAPageRepsFree(CustomTestCase):
|
||||
PS = 4
|
||||
|
||||
def _allocator(self):
|
||||
allocator, _ = _build_swa_tree(is_eagle=False, page_size=self.PS)
|
||||
allocator, _ = _build_swa_allocator(page_size=self.PS)
|
||||
return allocator
|
||||
|
||||
def _sizes(self, allocator):
|
||||
@@ -748,8 +732,7 @@ class TestSWAPageRepsFree(CustomTestCase):
|
||||
def test_free_swa_segment_npu_uses_reference_path(self):
|
||||
for page_size in (1, 4):
|
||||
with self.subTest(page_size=page_size):
|
||||
allocator, _ = _build_swa_tree(
|
||||
is_eagle=False,
|
||||
allocator, _ = _build_swa_allocator(
|
||||
page_size=page_size,
|
||||
kv_size=8 * page_size,
|
||||
kv_size_swa=8 * page_size,
|
||||
@@ -800,8 +783,7 @@ class TestSWAPageRepsFree(CustomTestCase):
|
||||
with self.subTest(name=name):
|
||||
num_allocated_tokens = max(2 * page_size, num_tokens)
|
||||
kv_size = max(8 * page_size, num_allocated_tokens)
|
||||
allocator, _ = _build_swa_tree(
|
||||
is_eagle=False,
|
||||
allocator, _ = _build_swa_allocator(
|
||||
page_size=page_size,
|
||||
kv_size=kv_size,
|
||||
kv_size_swa=kv_size,
|
||||
|
||||
Reference in New Issue
Block a user