Support deepseek v4 and kimi k3 on ssd (#35314)
Co-authored-by: 1BIN4 <1741738350@qq.com> Co-authored-by: L-Ark <fliangae@connect.ust.hk> Co-authored-by: Chikati <jxudn@connect.ust.hk> Co-authored-by: mengzili <zilim@ust.hk>
This commit is contained in:
hujianmin
co-authored by
1BIN4
L-Ark
Chikati
mengzili
1BIN4
L-Ark
Chikati
mengzili
parent
bec6248272
commit
2d8484740d
@@ -0,0 +1,360 @@
|
||||
"""CUDA unit tests for the MXFP4 expert-pack kernels."""
|
||||
|
||||
import unittest
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.moe.expert_pack_mxfp4 import (
|
||||
mxfp4_marlin_repack,
|
||||
mxfp4_matvec,
|
||||
mxfp4_matvec_dual,
|
||||
)
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=90, stage="base-b", runner_config="1-gpu-small")
|
||||
|
||||
|
||||
_FP4_VALUES = (
|
||||
0.0,
|
||||
0.5,
|
||||
1.0,
|
||||
1.5,
|
||||
2.0,
|
||||
3.0,
|
||||
4.0,
|
||||
6.0,
|
||||
0.0,
|
||||
-0.5,
|
||||
-1.0,
|
||||
-1.5,
|
||||
-2.0,
|
||||
-3.0,
|
||||
-4.0,
|
||||
-6.0,
|
||||
)
|
||||
_PACK_INDEX = (0, 2, 4, 6, 1, 3, 5, 7)
|
||||
|
||||
|
||||
def _matvec_reference(
|
||||
input_cpu,
|
||||
cache_cpu,
|
||||
slots_cpu,
|
||||
role_offset,
|
||||
input_size,
|
||||
output_size,
|
||||
records_per_input,
|
||||
):
|
||||
blocks = input_size // 32
|
||||
row_bytes = blocks * 17
|
||||
result = torch.zeros((slots_cpu.numel(), output_size), dtype=torch.float32)
|
||||
for record, slot in enumerate(slots_cpu.tolist()):
|
||||
input_row = record // records_per_input
|
||||
for output_row in range(output_size):
|
||||
total = 0.0
|
||||
row_offset = (
|
||||
int(slot) * cache_cpu.stride(0) + role_offset + output_row * row_bytes
|
||||
)
|
||||
for block in range(blocks):
|
||||
quant_offset = row_offset + block * 17
|
||||
scale = 2.0 ** (int(cache_cpu.view(-1)[quant_offset]) - 127)
|
||||
block_sum = 0.0
|
||||
for index in range(16):
|
||||
packed = int(cache_cpu.view(-1)[quant_offset + index + 1])
|
||||
block_sum += (
|
||||
float(input_cpu[input_row, block * 32 + index])
|
||||
* _FP4_VALUES[packed & 0xF]
|
||||
)
|
||||
block_sum += (
|
||||
float(input_cpu[input_row, block * 32 + index + 16])
|
||||
* _FP4_VALUES[packed >> 4]
|
||||
)
|
||||
total += block_sum * scale
|
||||
result[record, output_row] = total
|
||||
return result
|
||||
|
||||
|
||||
def _fill_mxfp4_cache(slots, role_bytes, input_size, device):
|
||||
blocks = input_size // 32
|
||||
cache_cpu = torch.zeros((slots, 3 * role_bytes), dtype=torch.uint8)
|
||||
for slot in range(slots):
|
||||
for role in range(3):
|
||||
role_rows = role_bytes // (blocks * 17)
|
||||
for row in range(role_rows):
|
||||
for block in range(blocks):
|
||||
offset = (
|
||||
slot * cache_cpu.stride(0)
|
||||
+ role * role_bytes
|
||||
+ row * blocks * 17
|
||||
+ block * 17
|
||||
)
|
||||
cache_cpu.view(-1)[offset] = 126 + ((slot + role + row + block) % 3)
|
||||
for byte in range(16):
|
||||
low = (slot + role * 3 + row + block + byte) % 16
|
||||
high = (15 + slot + role + row + block - byte) % 16
|
||||
cache_cpu.view(-1)[offset + byte + 1] = low | (high << 4)
|
||||
return cache_cpu.to(device=device)
|
||||
|
||||
|
||||
def _load_raw_word(raw_cpu, slot, role_offset, row, blocks_per_row, packed_word):
|
||||
block = packed_word // 4
|
||||
word_in_block = packed_word & 3
|
||||
offset = (
|
||||
slot * raw_cpu.stride(0)
|
||||
+ role_offset
|
||||
+ row * blocks_per_row * 17
|
||||
+ block * 17
|
||||
+ 1
|
||||
+ word_in_block * 4
|
||||
)
|
||||
word = 0
|
||||
for byte in range(4):
|
||||
word |= int(raw_cpu.view(-1)[offset + byte]) << (8 * byte)
|
||||
return word
|
||||
|
||||
|
||||
def _marlin_nibble(word, value_index):
|
||||
return (word >> ((value_index & 7) * 4)) & 0xF
|
||||
|
||||
|
||||
def _marlin_scale_perm(index):
|
||||
local_perm = (0, 2, 1, 3)
|
||||
interleaved = (index // 4) * 4 + local_perm[index & 3]
|
||||
return ((interleaved & 7) * 8) + (interleaved >> 3)
|
||||
|
||||
|
||||
def _repack_reference(
|
||||
raw_cpu, source_slot, role_bytes, input_size, output_size, gate_up
|
||||
):
|
||||
blocks = input_size // 32
|
||||
total_words = (input_size // 16) * output_size * 2
|
||||
output = torch.empty(total_words, dtype=torch.int32)
|
||||
tile_span = (output_size // 64) * 128
|
||||
rows_per_role = output_size // 2 if gate_up else output_size
|
||||
for index in range(total_words):
|
||||
tile_k, tile_rem = divmod(index, tile_span)
|
||||
tile_n, local = divmod(tile_rem, 128)
|
||||
warp, thread = local & 3, local >> 2
|
||||
cur_n = warp * 16 + thread // 4
|
||||
tc_row = (thread & 3) * 2
|
||||
values = []
|
||||
for high in (False, True):
|
||||
source_row = tile_n * 64 + cur_n + (8 if high else 0)
|
||||
role = (
|
||||
1 if gate_up and source_row >= rows_per_role else (0 if gate_up else 2)
|
||||
)
|
||||
row = source_row % rows_per_role if gate_up else source_row
|
||||
for offset in (0, 1, 8, 9):
|
||||
value_index = tc_row + offset
|
||||
word = _load_raw_word(
|
||||
raw_cpu,
|
||||
source_slot,
|
||||
role * role_bytes,
|
||||
row,
|
||||
blocks,
|
||||
tile_k * 2 + value_index // 8,
|
||||
)
|
||||
values.append(_marlin_nibble(word, value_index))
|
||||
packed = 0
|
||||
for output_index, value_index in enumerate(_PACK_INDEX):
|
||||
packed |= values[value_index] << (output_index * 4)
|
||||
# The CUDA kernel stores the packed uint32 bit pattern in int32.
|
||||
output[index] = packed if packed < (1 << 31) else packed - (1 << 32)
|
||||
return output
|
||||
|
||||
|
||||
def _scale_reference(
|
||||
raw_cpu, source_slot, role_bytes, input_size, output_size, gate_up
|
||||
):
|
||||
blocks = input_size // 32
|
||||
output = torch.empty(blocks * output_size, dtype=torch.uint8)
|
||||
rows_per_role = output_size // 2 if gate_up else output_size
|
||||
for index in range(output.numel()):
|
||||
group, column = divmod(index, output_size)
|
||||
source_column = (column // 64) * 64 + _marlin_scale_perm(column & 63)
|
||||
role = (
|
||||
1 if gate_up and source_column >= rows_per_role else (0 if gate_up else 2)
|
||||
)
|
||||
row = source_column % rows_per_role if gate_up else source_column
|
||||
offset = (
|
||||
source_slot * raw_cpu.stride(0)
|
||||
+ role * role_bytes
|
||||
+ row * blocks * 17
|
||||
+ group * 17
|
||||
)
|
||||
output[index] = raw_cpu.view(-1)[offset]
|
||||
return output
|
||||
|
||||
|
||||
@unittest.skipUnless(torch.cuda.is_available(), "MXFP4 kernel tests require CUDA")
|
||||
class TestExpertPackMxfp4(unittest.TestCase):
|
||||
def test_matvec_fp16_and_bf16(self):
|
||||
input_size, output_size, records_per_input = 64, 5, 2
|
||||
blocks = input_size // 32
|
||||
role_bytes = output_size * blocks * 17
|
||||
cache = _fill_mxfp4_cache(2, role_bytes, input_size, "cuda")
|
||||
slots = torch.tensor([1, 0, 1, 0], dtype=torch.int32, device="cuda")
|
||||
for dtype in (torch.float16, torch.bfloat16):
|
||||
if dtype is torch.bfloat16 and not torch.cuda.is_bf16_supported():
|
||||
continue
|
||||
input_tensor = torch.arange(
|
||||
2 * input_size, dtype=torch.float32, device="cuda"
|
||||
).reshape(2, input_size)
|
||||
input_tensor = ((input_tensor % 19) - 9).to(dtype)
|
||||
output = mxfp4_matvec(
|
||||
input_tensor,
|
||||
cache,
|
||||
slots,
|
||||
role_offset=role_bytes,
|
||||
role_bytes=role_bytes,
|
||||
input_size=input_size,
|
||||
output_size=output_size,
|
||||
records_per_input=records_per_input,
|
||||
)
|
||||
reference = _matvec_reference(
|
||||
input_tensor.cpu(),
|
||||
cache.cpu(),
|
||||
slots.cpu(),
|
||||
role_bytes,
|
||||
input_size,
|
||||
output_size,
|
||||
records_per_input,
|
||||
).to(dtype)
|
||||
torch.testing.assert_close(output.cpu(), reference, rtol=0.03, atol=0.25)
|
||||
|
||||
def test_matvec_dual_matches_two_roles(self):
|
||||
input_size, output_size, records_per_input = 64, 17, 2
|
||||
blocks = input_size // 32
|
||||
role_bytes = output_size * blocks * 17
|
||||
cache = _fill_mxfp4_cache(2, role_bytes, input_size, "cuda")
|
||||
slots = torch.tensor([0, 1, 1, 0], dtype=torch.int32, device="cuda")
|
||||
input_tensor = (torch.randn(2, input_size, device="cuda") * 0.5).to(
|
||||
torch.float16
|
||||
)
|
||||
output_a, output_b = mxfp4_matvec_dual(
|
||||
input_tensor,
|
||||
cache,
|
||||
slots,
|
||||
gate_role_offset=0,
|
||||
up_role_offset=role_bytes,
|
||||
role_bytes=role_bytes,
|
||||
input_size=input_size,
|
||||
output_size=output_size,
|
||||
records_per_input=records_per_input,
|
||||
)
|
||||
expected_a = mxfp4_matvec(
|
||||
input_tensor,
|
||||
cache,
|
||||
slots,
|
||||
role_offset=0,
|
||||
role_bytes=role_bytes,
|
||||
input_size=input_size,
|
||||
output_size=output_size,
|
||||
records_per_input=records_per_input,
|
||||
)
|
||||
expected_b = mxfp4_matvec(
|
||||
input_tensor,
|
||||
cache,
|
||||
slots,
|
||||
role_offset=role_bytes,
|
||||
role_bytes=role_bytes,
|
||||
input_size=input_size,
|
||||
output_size=output_size,
|
||||
records_per_input=records_per_input,
|
||||
)
|
||||
torch.testing.assert_close(output_a, expected_a)
|
||||
torch.testing.assert_close(output_b, expected_b)
|
||||
|
||||
def test_marlin_repack_weights_and_scales(self):
|
||||
hidden_size, intermediate_size = 64, 32
|
||||
w13_n, w2_n = 2 * intermediate_size, hidden_size
|
||||
role_bytes = intermediate_size * (hidden_size // 32) * 17
|
||||
raw_cpu = torch.zeros((4, 3 * role_bytes), dtype=torch.uint8)
|
||||
for slot in range(4):
|
||||
for role, rows, size in (
|
||||
(0, intermediate_size, hidden_size),
|
||||
(1, intermediate_size, hidden_size),
|
||||
(2, hidden_size, intermediate_size),
|
||||
):
|
||||
groups = size // 32
|
||||
for row in range(rows):
|
||||
for group in range(groups):
|
||||
offset = (
|
||||
slot * raw_cpu.stride(0)
|
||||
+ role * role_bytes
|
||||
+ row * groups * 17
|
||||
+ group * 17
|
||||
)
|
||||
raw_cpu.view(-1)[offset] = (
|
||||
100 + slot * 7 + role * 11 + row + group
|
||||
) % 256
|
||||
for byte in range(16):
|
||||
raw_cpu.view(-1)[offset + byte + 1] = (
|
||||
(byte + row + role) % 16
|
||||
) | (((15 - byte + slot + group) % 16) << 4)
|
||||
raw = raw_cpu.cuda()
|
||||
source_slots = torch.tensor([1, 0], dtype=torch.int32, device="cuda")
|
||||
target_slots = torch.tensor([2, 3], dtype=torch.int32, device="cuda")
|
||||
w13_words = (hidden_size // 16) * w13_n * 2
|
||||
w2_words = (intermediate_size // 16) * w2_n * 2
|
||||
w13_scales = (hidden_size // 32) * w13_n
|
||||
w2_scales = (intermediate_size // 32) * w2_n
|
||||
w13 = torch.full((4, w13_words), -1, dtype=torch.int32, device="cuda")
|
||||
w2 = torch.full((4, w2_words), -1, dtype=torch.int32, device="cuda")
|
||||
w13_scale = torch.full((4, w13_scales), 255, dtype=torch.uint8, device="cuda")
|
||||
w2_scale = torch.full((4, w2_scales), 255, dtype=torch.uint8, device="cuda")
|
||||
mxfp4_marlin_repack(
|
||||
raw,
|
||||
source_slots,
|
||||
target_slots,
|
||||
role_bytes=role_bytes,
|
||||
hidden_size=hidden_size,
|
||||
intermediate_size=intermediate_size,
|
||||
w13=w13,
|
||||
w2=w2,
|
||||
w13_scale=w13_scale,
|
||||
w2_scale=w2_scale,
|
||||
)
|
||||
torch.cuda.synchronize()
|
||||
for source_slot, target_slot in zip(
|
||||
source_slots.cpu().tolist(), target_slots.cpu().tolist()
|
||||
):
|
||||
expected_w13 = _repack_reference(
|
||||
raw_cpu, source_slot, role_bytes, hidden_size, w13_n, True
|
||||
)
|
||||
expected_w2 = _repack_reference(
|
||||
raw_cpu, source_slot, role_bytes, intermediate_size, w2_n, False
|
||||
)
|
||||
expected_w13_scale = _scale_reference(
|
||||
raw_cpu, source_slot, role_bytes, hidden_size, w13_n, True
|
||||
)
|
||||
expected_w2_scale = _scale_reference(
|
||||
raw_cpu, source_slot, role_bytes, intermediate_size, w2_n, False
|
||||
)
|
||||
torch.testing.assert_close(w13[target_slot].cpu(), expected_w13)
|
||||
torch.testing.assert_close(w2[target_slot].cpu(), expected_w2)
|
||||
torch.testing.assert_close(w13_scale[target_slot].cpu(), expected_w13_scale)
|
||||
torch.testing.assert_close(w2_scale[target_slot].cpu(), expected_w2_scale)
|
||||
self.assertTrue(torch.all(w13[0] == -1))
|
||||
self.assertTrue(torch.all(w2[0] == -1))
|
||||
self.assertTrue(torch.all(w13_scale[0] == 255))
|
||||
self.assertTrue(torch.all(w2_scale[0] == 255))
|
||||
|
||||
def test_invalid_dimensions_are_rejected(self):
|
||||
input_tensor = torch.zeros((1, 31), dtype=torch.float16, device="cuda")
|
||||
cache = torch.zeros((1, 17), dtype=torch.uint8, device="cuda")
|
||||
slots = torch.zeros((1,), dtype=torch.int32, device="cuda")
|
||||
with self.assertRaisesRegex(RuntimeError, "divisible by 32"):
|
||||
mxfp4_matvec(
|
||||
input_tensor,
|
||||
cache,
|
||||
slots,
|
||||
role_offset=0,
|
||||
role_bytes=17,
|
||||
input_size=31,
|
||||
output_size=1,
|
||||
records_per_input=1,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,444 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
|
||||
register_cpu_ci(est_time=15, suite="base-a-test-cpu")
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
TOOLS = ROOT / "tools" / "expert_pack"
|
||||
sys.path.insert(0, str(TOOLS))
|
||||
|
||||
from format import ( # noqa: E402
|
||||
ENTRY_STRUCT,
|
||||
FLAG_IDENTITY_PAYLOAD,
|
||||
FLAG_TRIPLET_OBJECTS,
|
||||
HEADER_STRUCT,
|
||||
IndexEntry,
|
||||
PackHeader,
|
||||
align_up,
|
||||
read_header,
|
||||
read_index,
|
||||
)
|
||||
|
||||
from sglang.srt.layers.moe.expert_pack import ( # noqa: E402
|
||||
ExpertPackStore,
|
||||
_CacheSlot,
|
||||
)
|
||||
|
||||
|
||||
def _sha256(path: Path) -> str:
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def _make_pack(directory: Path) -> tuple[Path, Path, dict[str, str]]:
|
||||
layers, experts, top_k = 1, 2, 1
|
||||
role_bytes = 17
|
||||
object_stride = 4096
|
||||
index_count = layers * experts * 3
|
||||
data_start = align_up(HEADER_STRUCT.size + index_count * ENTRY_STRUCT.size, 4096)
|
||||
digests = {
|
||||
"model_identity": hashlib.sha256(b"model-identity").hexdigest(),
|
||||
"source": hashlib.sha256(b"source").hexdigest(),
|
||||
"config": hashlib.sha256(b"config").hexdigest(),
|
||||
}
|
||||
header = PackHeader(
|
||||
flags=FLAG_IDENTITY_PAYLOAD | FLAG_TRIPLET_OBJECTS,
|
||||
index_count=index_count,
|
||||
data_start=data_start,
|
||||
alignment=4096,
|
||||
num_layers=layers,
|
||||
num_experts=experts,
|
||||
top_k=top_k,
|
||||
role_count=3,
|
||||
model_identity_sha256=digests["model_identity"],
|
||||
source_blob_sha256=digests["source"],
|
||||
config_sha256=digests["config"],
|
||||
)
|
||||
entries = []
|
||||
payloads = []
|
||||
for expert in range(experts):
|
||||
object_offset = data_start + expert * object_stride
|
||||
generation = expert + 100
|
||||
for role_id, role in enumerate(("gate", "up", "down")):
|
||||
payload = bytes([expert * 3 + role_id]) * role_bytes
|
||||
payload_hash = hashlib.sha256(payload).hexdigest()
|
||||
entries.append(
|
||||
IndexEntry(
|
||||
layer=0,
|
||||
expert=expert,
|
||||
role=role,
|
||||
dtype_id=39,
|
||||
dtype="MXFP4",
|
||||
tensor_name=f"blk.0.ffn_{role}_exps.weight",
|
||||
source_tensor_offset=0,
|
||||
source_tensor_nbytes=role_bytes * experts,
|
||||
source_slice_offset=expert * role_bytes,
|
||||
source_slice_nbytes=role_bytes,
|
||||
pack_offset=object_offset + role_id * role_bytes,
|
||||
pack_nbytes=role_bytes,
|
||||
source_tensor_sha256=payload_hash,
|
||||
source_slice_sha256=payload_hash,
|
||||
checksum=payload_hash,
|
||||
shape=(32, 1),
|
||||
quant_scheme="MXFP4",
|
||||
transform_id="identity-v1",
|
||||
block_size=32,
|
||||
generation=generation,
|
||||
)
|
||||
)
|
||||
payloads.append((object_offset + role_id * role_bytes, payload))
|
||||
|
||||
pack = directory / "runtime.expert-pack"
|
||||
with pack.open("w+b") as stream:
|
||||
stream.write(header.pack())
|
||||
for entry in entries:
|
||||
stream.write(entry.pack())
|
||||
stream.truncate(data_start + experts * object_stride)
|
||||
for offset, payload in payloads:
|
||||
stream.seek(offset)
|
||||
stream.write(payload)
|
||||
manifest = directory / "runtime.expert-pack.manifest.json"
|
||||
manifest.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"complete": True,
|
||||
"object_stride": object_stride,
|
||||
"pack_sha256": _sha256(pack),
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
return pack, manifest, digests
|
||||
|
||||
|
||||
def _refresh_manifest_hash(pack: Path, manifest: Path) -> None:
|
||||
value = json.loads(manifest.read_text(encoding="utf-8"))
|
||||
value["pack_sha256"] = _sha256(pack)
|
||||
manifest.write_text(json.dumps(value), encoding="utf-8")
|
||||
|
||||
|
||||
class TestExpertPackRuntime(unittest.TestCase):
|
||||
def test_victim_prefers_low_frequency_then_lru(self):
|
||||
store = object.__new__(ExpertPackStore)
|
||||
keys = [(0, 0), (0, 1), (0, 2)]
|
||||
store._cache_slots = [
|
||||
_CacheSlot(key=keys[0], frequency=5),
|
||||
_CacheSlot(key=keys[1], frequency=1),
|
||||
_CacheSlot(key=keys[2], frequency=1),
|
||||
]
|
||||
store._key_to_slot = {key: index for index, key in enumerate(keys)}
|
||||
store._lru = dict.fromkeys(keys)
|
||||
|
||||
self.assertEqual(store._victim_slot(set()), 1)
|
||||
self.assertEqual(store._victim_slot({keys[1]}), 2)
|
||||
self.assertEqual(store._victim_slot(set(), preserve_oldest=True), 2)
|
||||
with self.assertRaisesRegex(RuntimeError, "active top-k"):
|
||||
store._victim_slot(set(keys))
|
||||
|
||||
def test_zero_staging_slots_are_rejected(self):
|
||||
with tempfile.TemporaryDirectory() as value:
|
||||
root = Path(value)
|
||||
pack, manifest, digests = _make_pack(root)
|
||||
with self.assertRaisesRegex(ValueError, "staging budgets"):
|
||||
ExpertPackStore(
|
||||
pack,
|
||||
manifest_path=manifest,
|
||||
expected_layers=1,
|
||||
expected_experts=2,
|
||||
expected_top_k=1,
|
||||
expected_source_sha256=digests["source"],
|
||||
expected_model_identity_sha256=digests["model_identity"],
|
||||
expected_config_sha256=digests["config"],
|
||||
cache_vram_mib=1,
|
||||
stage_slots=0,
|
||||
)
|
||||
|
||||
def test_full_pack_verification_is_opt_in(self):
|
||||
with tempfile.TemporaryDirectory() as value:
|
||||
root = Path(value)
|
||||
pack, manifest, digests = _make_pack(root)
|
||||
store = ExpertPackStore(
|
||||
pack,
|
||||
manifest_path=manifest,
|
||||
expected_layers=1,
|
||||
expected_experts=2,
|
||||
expected_top_k=1,
|
||||
expected_source_sha256=digests["source"],
|
||||
expected_model_identity_sha256=digests["model_identity"],
|
||||
expected_config_sha256=digests["config"],
|
||||
cache_vram_mib=1,
|
||||
stage_slots=1,
|
||||
)
|
||||
self.assertEqual(len(store.entries), 6)
|
||||
self.assertEqual(store.object_payload_bytes, 51)
|
||||
staging = torch.empty(51, dtype=torch.uint8)
|
||||
read_bytes, elapsed_ns = store._read_object(0, 1, staging)
|
||||
self.assertEqual(read_bytes, 51)
|
||||
self.assertGreaterEqual(elapsed_ns, 0)
|
||||
self.assertEqual(staging.tolist(), [3] * 17 + [4] * 17 + [5] * 17)
|
||||
store.close()
|
||||
|
||||
with pack.open("r+b") as stream:
|
||||
stream.seek(-1, 2)
|
||||
stream.write(b"\x01")
|
||||
|
||||
store = ExpertPackStore(
|
||||
pack,
|
||||
manifest_path=manifest,
|
||||
expected_layers=1,
|
||||
expected_experts=2,
|
||||
expected_top_k=1,
|
||||
expected_source_sha256=digests["source"],
|
||||
expected_model_identity_sha256=digests["model_identity"],
|
||||
expected_config_sha256=digests["config"],
|
||||
cache_vram_mib=1,
|
||||
stage_slots=1,
|
||||
)
|
||||
store.close()
|
||||
|
||||
with self.assertRaisesRegex(ValueError, "SHA-256"):
|
||||
ExpertPackStore(
|
||||
pack,
|
||||
manifest_path=manifest,
|
||||
expected_layers=1,
|
||||
expected_experts=2,
|
||||
expected_top_k=1,
|
||||
expected_source_sha256=digests["source"],
|
||||
expected_model_identity_sha256=digests["model_identity"],
|
||||
expected_config_sha256=digests["config"],
|
||||
cache_vram_mib=1,
|
||||
stage_slots=1,
|
||||
verify_pack_sha256=True,
|
||||
)
|
||||
|
||||
def test_split_read_ranges_reconstruct_exact_object(self):
|
||||
with tempfile.TemporaryDirectory() as value:
|
||||
root = Path(value)
|
||||
pack, manifest, digests = _make_pack(root)
|
||||
store = ExpertPackStore(
|
||||
pack,
|
||||
manifest_path=manifest,
|
||||
expected_layers=1,
|
||||
expected_experts=2,
|
||||
expected_top_k=1,
|
||||
expected_source_sha256=digests["source"],
|
||||
expected_model_identity_sha256=digests["model_identity"],
|
||||
expected_config_sha256=digests["config"],
|
||||
cache_vram_mib=1,
|
||||
stage_slots=1,
|
||||
)
|
||||
ranges = store._object_read_ranges()
|
||||
self.assertEqual(len(ranges), 4)
|
||||
self.assertEqual(ranges[0][0], 0)
|
||||
self.assertEqual(sum(length for _, length in ranges), 51)
|
||||
self.assertTrue(all(length > 0 for _, length in ranges))
|
||||
self.assertTrue(
|
||||
all(
|
||||
ranges[index][0] + ranges[index][1] == ranges[index + 1][0]
|
||||
for index in range(len(ranges) - 1)
|
||||
)
|
||||
)
|
||||
|
||||
staging = torch.empty(51, dtype=torch.uint8)
|
||||
for start, length in ranges:
|
||||
read_bytes, elapsed_ns = store._read_object_range(
|
||||
0, 1, staging, start=start, length=length
|
||||
)
|
||||
self.assertEqual(read_bytes, length)
|
||||
self.assertGreaterEqual(elapsed_ns, 0)
|
||||
self.assertEqual(staging.tolist(), [3] * 17 + [4] * 17 + [5] * 17)
|
||||
store.close()
|
||||
|
||||
def test_read_splits_follow_runtime_configuration(self):
|
||||
with tempfile.TemporaryDirectory() as value:
|
||||
root = Path(value)
|
||||
pack, manifest, digests = _make_pack(root)
|
||||
store = ExpertPackStore(
|
||||
pack,
|
||||
manifest_path=manifest,
|
||||
expected_layers=1,
|
||||
expected_experts=2,
|
||||
expected_top_k=1,
|
||||
expected_source_sha256=digests["source"],
|
||||
expected_model_identity_sha256=digests["model_identity"],
|
||||
expected_config_sha256=digests["config"],
|
||||
cache_vram_mib=1,
|
||||
stage_slots=1,
|
||||
read_splits=2,
|
||||
stats_flush_interval=7,
|
||||
)
|
||||
ranges = store._object_read_ranges()
|
||||
self.assertEqual(store.stats["read_splits"], 2)
|
||||
self.assertEqual(store.stats_flush_interval, 7)
|
||||
self.assertEqual(len(ranges), 2)
|
||||
self.assertEqual(ranges[0][0], 0)
|
||||
self.assertEqual(sum(length for _, length in ranges), 51)
|
||||
store.close()
|
||||
|
||||
def test_short_object_read_raises(self):
|
||||
with tempfile.TemporaryDirectory() as value:
|
||||
root = Path(value)
|
||||
pack, manifest, digests = _make_pack(root)
|
||||
store = ExpertPackStore(
|
||||
pack,
|
||||
manifest_path=manifest,
|
||||
expected_layers=1,
|
||||
expected_experts=2,
|
||||
expected_top_k=1,
|
||||
expected_source_sha256=digests["source"],
|
||||
expected_model_identity_sha256=digests["model_identity"],
|
||||
expected_config_sha256=digests["config"],
|
||||
cache_vram_mib=1,
|
||||
stage_slots=1,
|
||||
)
|
||||
second_object = store.object_offsets[(0, 1)]
|
||||
with pack.open("r+b", buffering=0) as stream:
|
||||
stream.truncate(second_object + 10)
|
||||
staging = torch.empty(store.object_payload_bytes, dtype=torch.uint8)
|
||||
with self.assertRaisesRegex(OSError, "short expert-pack read"):
|
||||
store._read_object(0, 1, staging)
|
||||
store.close()
|
||||
|
||||
def test_duplicate_entry_is_rejected_even_with_valid_pack_hash(self):
|
||||
with tempfile.TemporaryDirectory() as value:
|
||||
root = Path(value)
|
||||
pack, manifest, digests = _make_pack(root)
|
||||
with pack.open("r+b", buffering=0) as stream:
|
||||
header = read_header(stream)
|
||||
entries = read_index(stream, header)
|
||||
duplicate = replace(
|
||||
entries[1],
|
||||
layer=entries[0].layer,
|
||||
expert=entries[0].expert,
|
||||
role=entries[0].role,
|
||||
)
|
||||
stream.seek(HEADER_STRUCT.size + ENTRY_STRUCT.size)
|
||||
stream.write(duplicate.pack())
|
||||
_refresh_manifest_hash(pack, manifest)
|
||||
with self.assertRaisesRegex(ValueError, "duplicate expert-pack entry"):
|
||||
ExpertPackStore(
|
||||
pack,
|
||||
manifest_path=manifest,
|
||||
expected_layers=1,
|
||||
expected_experts=2,
|
||||
expected_top_k=1,
|
||||
expected_source_sha256=digests["source"],
|
||||
expected_model_identity_sha256=digests["model_identity"],
|
||||
expected_config_sha256=digests["config"],
|
||||
cache_vram_mib=1,
|
||||
stage_slots=1,
|
||||
)
|
||||
|
||||
def test_out_of_range_entry_is_rejected_even_with_valid_pack_hash(self):
|
||||
with tempfile.TemporaryDirectory() as value:
|
||||
root = Path(value)
|
||||
pack, manifest, digests = _make_pack(root)
|
||||
with pack.open("r+b", buffering=0) as stream:
|
||||
header = read_header(stream)
|
||||
entries = read_index(stream, header)
|
||||
invalid = replace(
|
||||
entries[0], pack_offset=pack.stat().st_size + header.alignment
|
||||
)
|
||||
stream.seek(HEADER_STRUCT.size)
|
||||
stream.write(invalid.pack())
|
||||
_refresh_manifest_hash(pack, manifest)
|
||||
with self.assertRaisesRegex(ValueError, "object layout mismatch"):
|
||||
ExpertPackStore(
|
||||
pack,
|
||||
manifest_path=manifest,
|
||||
expected_layers=1,
|
||||
expected_experts=2,
|
||||
expected_top_k=1,
|
||||
expected_source_sha256=digests["source"],
|
||||
expected_model_identity_sha256=digests["model_identity"],
|
||||
expected_config_sha256=digests["config"],
|
||||
cache_vram_mib=1,
|
||||
stage_slots=1,
|
||||
)
|
||||
|
||||
def test_source_identity_mismatch_is_rejected(self):
|
||||
with tempfile.TemporaryDirectory() as value:
|
||||
root = Path(value)
|
||||
pack, manifest, digests = _make_pack(root)
|
||||
with self.assertRaisesRegex(ValueError, "source_blob_sha256"):
|
||||
ExpertPackStore(
|
||||
pack,
|
||||
manifest_path=manifest,
|
||||
expected_layers=1,
|
||||
expected_experts=2,
|
||||
expected_top_k=1,
|
||||
expected_source_sha256="0" * 64,
|
||||
expected_model_identity_sha256=digests["model_identity"],
|
||||
expected_config_sha256=digests["config"],
|
||||
cache_vram_mib=1,
|
||||
stage_slots=1,
|
||||
)
|
||||
|
||||
@unittest.skipUnless(torch.cuda.is_available(), "requires CUDA pinned memory")
|
||||
def test_direct_io_rejects_unaligned_object_ranges(self):
|
||||
with tempfile.TemporaryDirectory() as value:
|
||||
root = Path(value)
|
||||
pack, manifest, digests = _make_pack(root)
|
||||
store = ExpertPackStore(
|
||||
pack,
|
||||
manifest_path=manifest,
|
||||
expected_layers=1,
|
||||
expected_experts=2,
|
||||
expected_top_k=1,
|
||||
expected_source_sha256=digests["source"],
|
||||
expected_model_identity_sha256=digests["model_identity"],
|
||||
expected_config_sha256=digests["config"],
|
||||
cache_vram_mib=1,
|
||||
cache_vram_reserve_mib=1,
|
||||
stage_slots=1,
|
||||
direct_io=True,
|
||||
)
|
||||
with self.assertRaisesRegex(ValueError, "aligned read ranges"):
|
||||
store.initialize_device_cache("cuda")
|
||||
store.close()
|
||||
|
||||
def test_close_flushes_stats_atomically(self):
|
||||
with tempfile.TemporaryDirectory() as value:
|
||||
root = Path(value)
|
||||
pack, manifest, digests = _make_pack(root)
|
||||
stats_path = root / "stats.json"
|
||||
store = ExpertPackStore(
|
||||
pack,
|
||||
manifest_path=manifest,
|
||||
expected_layers=1,
|
||||
expected_experts=2,
|
||||
expected_top_k=1,
|
||||
expected_source_sha256=digests["source"],
|
||||
expected_model_identity_sha256=digests["model_identity"],
|
||||
expected_config_sha256=digests["config"],
|
||||
cache_vram_mib=1,
|
||||
stage_slots=1,
|
||||
stats_path=stats_path,
|
||||
)
|
||||
store._route_calls_by_layer[0] = 2
|
||||
store._route_tokens_by_layer[0] = 3
|
||||
store.stats["pack_reads"] = 4
|
||||
store.stats["pack_read_bytes"] = 204
|
||||
store.close()
|
||||
stats = json.loads(stats_path.read_text(encoding="utf-8"))
|
||||
self.assertEqual(stats["pack_reads"], 4)
|
||||
self.assertEqual(stats["pack_read_bytes"], 204)
|
||||
self.assertEqual(stats["route_calls_by_layer"], [2])
|
||||
self.assertEqual(stats["route_tokens_by_layer"], [3])
|
||||
self.assertFalse(list(root.glob("stats.json.*.tmp")))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,115 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers.quantization.gguf import (
|
||||
GGUFLinearMethod,
|
||||
_ordered_gguf_shard_ids,
|
||||
)
|
||||
from sglang.srt.model_loader.kimi_k3_gguf import (
|
||||
_kda_a_log_target_value,
|
||||
_residual_target_value,
|
||||
kimi_k3_checkpoint_targets,
|
||||
routed_expert_tensor,
|
||||
)
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
|
||||
register_cpu_ci(est_time=10, suite="base-a-test-cpu")
|
||||
|
||||
|
||||
class TestKimiK3GGUFMapping(unittest.TestCase):
|
||||
def test_maps_dense_kda_mla_moe_and_residual_tensors(self) -> None:
|
||||
cases = {
|
||||
"token_embd.weight": ("model.embed_tokens.weight",),
|
||||
"blk.0.ffn_gate.weight": ("model.layers.0.mlp.gate_proj.weight",),
|
||||
"blk.1.ssm_g.weight": ("model.layers.1.self_attn.g_proj.weight",),
|
||||
"blk.3.attn_q_b.weight": ("model.layers.3.self_attn.q_b_proj.weight",),
|
||||
"blk.3.attn_k_b.weight": ("model.layers.3.self_attn.k_b_qweight",),
|
||||
"blk.3.attn_v_b.weight": ("model.layers.3.self_attn.v_b_qweight",),
|
||||
"blk.1.ffn_routed_down.weight": (
|
||||
"model.layers.1.mlp.routed_expert_down_proj.weight",
|
||||
),
|
||||
"blk.1.ffn_gate_shexp.weight": (
|
||||
"model.layers.1.mlp.shared_experts.gate_proj.weight",
|
||||
),
|
||||
"blk.2.attn_res_score.weight": (
|
||||
"model.layers.2.self_attention_res_proj.weight",
|
||||
"model.layers.2.self_attention_res_norm.weight",
|
||||
),
|
||||
"output_res_score.weight": (
|
||||
"model.output_attn_res_proj.weight",
|
||||
"model.output_attn_res_norm.weight",
|
||||
),
|
||||
}
|
||||
for source, expected in cases.items():
|
||||
with self.subTest(source=source):
|
||||
self.assertEqual(kimi_k3_checkpoint_targets(source), expected)
|
||||
|
||||
def test_only_routed_aggregate_tensors_are_skipped(self) -> None:
|
||||
self.assertTrue(routed_expert_tensor("blk.92.ffn_up_exps.weight"))
|
||||
self.assertTrue(routed_expert_tensor("blk.1.ffn_down_exps.weight"))
|
||||
self.assertFalse(routed_expert_tensor("blk.1.ffn_up_shexp.weight"))
|
||||
self.assertFalse(routed_expert_tensor("blk.1.ffn_routed_up.weight"))
|
||||
|
||||
def test_unknown_tensor_fails_closed(self) -> None:
|
||||
with self.assertRaisesRegex(KeyError, "unsupported Kimi-K3"):
|
||||
kimi_k3_checkpoint_targets("blk.7.unexpected.weight")
|
||||
|
||||
def test_residual_score_preserves_exact_combined_weight(self) -> None:
|
||||
source = torch.tensor([0.5, -1.25, 3.0], dtype=torch.float32)
|
||||
projection = _residual_target_value(source, 0)
|
||||
norm = _residual_target_value(source, 1)
|
||||
self.assertEqual(tuple(projection.shape), (1, 3))
|
||||
self.assertEqual(tuple(norm.shape), (3,))
|
||||
torch.testing.assert_close(norm * projection.squeeze(0), source)
|
||||
|
||||
def test_residual_score_rejects_non_vector_source(self) -> None:
|
||||
with self.assertRaisesRegex(ValueError, "must be a vector"):
|
||||
_residual_target_value(torch.ones(1, 3), 0)
|
||||
|
||||
def test_restores_kda_a_log_from_gguf_transform(self) -> None:
|
||||
original = torch.tensor([-0.75, 0.0, 1.5], dtype=torch.float32)
|
||||
stored = -torch.exp(original)
|
||||
torch.testing.assert_close(_kda_a_log_target_value(stored), original)
|
||||
with self.assertRaisesRegex(ValueError, "only -exp"):
|
||||
_kda_a_log_target_value(torch.tensor([-1.0, 0.0]))
|
||||
with self.assertRaisesRegex(ValueError, "finite"):
|
||||
_kda_a_log_target_value(torch.tensor([-1.0, float("nan")]))
|
||||
|
||||
def test_merged_gguf_output_uses_logical_shard_order(self) -> None:
|
||||
qweight = torch.tensor([[30], [0], [20], [10]], dtype=torch.uint8)
|
||||
qweight.shard_id = [3, 0, 2, 1]
|
||||
qweight.shard_offset_map = {
|
||||
3: (0, 1, 1),
|
||||
0: (1, 2, 1),
|
||||
2: (2, 3, 1),
|
||||
1: (3, 4, 1),
|
||||
}
|
||||
qweight.gguf_prefix = ""
|
||||
layer = SimpleNamespace(
|
||||
qweight=qweight,
|
||||
qweight_type=SimpleNamespace(shard_weight_type={0: 0, 1: 0, 2: 0, 3: 0}),
|
||||
)
|
||||
method = object.__new__(GGUFLinearMethod)
|
||||
|
||||
def fake_matmul(_x, weight, _weight_type):
|
||||
return weight[:, 0].float().unsqueeze(0)
|
||||
|
||||
with patch(
|
||||
"sglang.srt.layers.quantization.gguf.fused_mul_mat_gguf",
|
||||
side_effect=fake_matmul,
|
||||
):
|
||||
output = method.apply(layer, torch.zeros(1, 1))
|
||||
torch.testing.assert_close(output, torch.tensor([[0.0, 10.0, 20.0, 30.0]]))
|
||||
|
||||
def test_unknown_gguf_shard_layouts_preserve_checkpoint_order(self) -> None:
|
||||
self.assertEqual(_ordered_gguf_shard_ids(["q", "k"]), ["q", "k"])
|
||||
self.assertEqual(_ordered_gguf_shard_ids([4, 2]), [4, 2])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,7 +1,7 @@
|
||||
import subprocess
|
||||
import sys
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from types import ModuleType, SimpleNamespace
|
||||
from unittest.mock import MagicMock, Mock, patch
|
||||
|
||||
import torch
|
||||
@@ -279,7 +279,18 @@ class TestMiniCPMSparseMetadata(CustomTestCase):
|
||||
|
||||
model_runner.server_args.attention_backend = "minicpm_flashinfer"
|
||||
flashinfer_adapter = object()
|
||||
fake_fuse_kernel = ModuleType("sglang.srt.layers.attention.minicpm.fuse_kernel")
|
||||
fake_fuse_kernel.fused_attn_pooling_online_topk_prefill = Mock(
|
||||
return_value="prefill"
|
||||
)
|
||||
fake_fuse_kernel.fused_attn_pooling_online_topk_decode = Mock(
|
||||
return_value="decode"
|
||||
)
|
||||
with (
|
||||
patch.dict(
|
||||
sys.modules,
|
||||
{"sglang.srt.layers.attention.minicpm.fuse_kernel": fake_fuse_kernel},
|
||||
),
|
||||
patch.object(backend_module, "MiniCPMHybridConfig", SimpleNamespace),
|
||||
patch.object(backend_module, "is_blackwell_supported", return_value=True),
|
||||
patch.object(
|
||||
@@ -297,11 +308,6 @@ class TestMiniCPMSparseMetadata(CustomTestCase):
|
||||
"get_parallel",
|
||||
return_value=SimpleNamespace(attn_tp_size=1),
|
||||
),
|
||||
patch(
|
||||
"sglang.srt.layers.attention.minicpm.fuse_kernel."
|
||||
"fused_attn_pooling_online_topk_prefill",
|
||||
return_value="prefill",
|
||||
),
|
||||
patch.object(backend_module, "attach_compressed_cache"),
|
||||
):
|
||||
backend = MiniCPMSparseBackend(model_runner, use_flashinfer=True)
|
||||
@@ -1079,10 +1085,16 @@ class TestMiniCPMSparseMetadata(CustomTestCase):
|
||||
backend._get_fused_topk_kernel.assert_called_once_with(1, is_prefill=False)
|
||||
|
||||
def test_fused_topk_prefill_kernels_compile_for_all_batches_at_startup(self):
|
||||
with patch(
|
||||
"sglang.srt.layers.attention.minicpm.fuse_kernel."
|
||||
"fused_attn_pooling_online_topk_prefill",
|
||||
side_effect=lambda **kwargs: f"prefill-{kwargs['batch_size']}",
|
||||
fake_fuse_kernel = ModuleType("sglang.srt.layers.attention.minicpm.fuse_kernel")
|
||||
fake_fuse_kernel.fused_attn_pooling_online_topk_prefill = Mock(
|
||||
side_effect=lambda **kwargs: f"prefill-{kwargs['batch_size']}"
|
||||
)
|
||||
fake_fuse_kernel.fused_attn_pooling_online_topk_decode = Mock(
|
||||
side_effect=lambda **kwargs: f"decode-{kwargs['batch_size']}"
|
||||
)
|
||||
with patch.dict(
|
||||
sys.modules,
|
||||
{"sglang.srt.layers.attention.minicpm.fuse_kernel": fake_fuse_kernel},
|
||||
):
|
||||
backend, *_ = _construct_sparse_backend(
|
||||
max_running_requests=3,
|
||||
@@ -1104,17 +1116,14 @@ class TestMiniCPMSparseMetadata(CustomTestCase):
|
||||
backend.fused_kernel_kwargs = {"topk": 8}
|
||||
backend.prefill_kernel_max_seqlen_q_grid = 64
|
||||
|
||||
with (
|
||||
patch(
|
||||
"sglang.srt.layers.attention.minicpm.fuse_kernel."
|
||||
"fused_attn_pooling_online_topk_prefill",
|
||||
return_value="prefill",
|
||||
) as prefill,
|
||||
patch(
|
||||
"sglang.srt.layers.attention.minicpm.fuse_kernel."
|
||||
"fused_attn_pooling_online_topk_decode",
|
||||
return_value="decode",
|
||||
) as decode,
|
||||
fake_fuse_kernel = ModuleType("sglang.srt.layers.attention.minicpm.fuse_kernel")
|
||||
prefill = Mock(return_value="prefill")
|
||||
decode = Mock(return_value="decode")
|
||||
fake_fuse_kernel.fused_attn_pooling_online_topk_prefill = prefill
|
||||
fake_fuse_kernel.fused_attn_pooling_online_topk_decode = decode
|
||||
with patch.dict(
|
||||
sys.modules,
|
||||
{"sglang.srt.layers.attention.minicpm.fuse_kernel": fake_fuse_kernel},
|
||||
):
|
||||
self.assertEqual(
|
||||
backend._get_fused_topk_kernel(3, is_prefill=True), "prefill"
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
from sglang.srt.lora.deepseek_mla_correction import is_kv_b_lora_active
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
|
||||
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
|
||||
|
||||
|
||||
class TestDeepseekMLACorrection(unittest.TestCase):
|
||||
def test_kv_b_lora_probe(self):
|
||||
self.assertFalse(is_kv_b_lora_active(SimpleNamespace()))
|
||||
self.assertFalse(
|
||||
is_kv_b_lora_active(SimpleNamespace(kv_b_proj=SimpleNamespace()))
|
||||
)
|
||||
self.assertTrue(
|
||||
is_kv_b_lora_active(
|
||||
SimpleNamespace(kv_b_proj=SimpleNamespace(set_lora=True))
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -636,7 +636,10 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
)
|
||||
|
||||
# Thor (SM110) and other architectures keep the existing auto behavior.
|
||||
with patch.object(overrides_module, "is_sm100_supported", return_value=False):
|
||||
with (
|
||||
patch.object(overrides_module, "is_sm100_supported", return_value=False),
|
||||
patch.object(overrides_module, "is_sm120_supported", return_value=False),
|
||||
):
|
||||
non_sm10x = self._construct(
|
||||
"MiniMaxM2ForCausalLM", "llama", quantization="modelopt_fp4"
|
||||
)
|
||||
|
||||
@@ -139,7 +139,9 @@ _EXPOSED = {
|
||||
("dllm/config.py", "model_path"),
|
||||
("multimodal/processors/base_processor.py", "image_processor_backend"),
|
||||
("speculative/spec_registry.py", "disable_overlap_schedule"),
|
||||
("disaggregation/encoder/server.py", "model_loader_extra_config"),
|
||||
("layers/moe/utils.py", "deepep_mode"),
|
||||
("layers/moe/utils.py", "disable_shared_experts_fusion"),
|
||||
("layers/moe/utils.py", "moe_a2a_backend"),
|
||||
("layers/moe/utils.py", "moe_runner_backend"),
|
||||
("layers/moe/utils.py", "quantization"),
|
||||
@@ -171,6 +173,8 @@ _EXPOSED = {
|
||||
("layers/flashinfer_comm_fusion.py", "flashinfer_allreduce_fusion_backend"),
|
||||
("lora/lora_manager.py", "enable_lora_overlap_loading"),
|
||||
("lora/marlin_lora_temp/policy.py", "lora_paths"),
|
||||
("model_loader/expert_pack_runtime.py", "model_path"),
|
||||
("model_loader/expert_pack_runtime.py", "tokenizer_path"),
|
||||
("parser/template_detection.py", "model_path"),
|
||||
("speculative/adaptive_spec_params.py", "speculative_algorithm"),
|
||||
("speculative/adaptive_spec_params.py", "speculative_eagle_topk"),
|
||||
@@ -189,6 +193,7 @@ _EXPOSED = {
|
||||
("weight_cache/daemon.py", "enable_dp_lm_head"),
|
||||
("weight_cache/daemon.py", "ep_size"),
|
||||
("weight_cache/daemon.py", "load_format"),
|
||||
("weight_cache/daemon.py", "model_loader_extra_config"),
|
||||
("weight_cache/daemon.py", "model_path"),
|
||||
("weight_cache/daemon.py", "moe_a2a_backend"),
|
||||
("weight_cache/daemon.py", "moe_dense_tp_size"),
|
||||
@@ -212,6 +217,7 @@ _OVERRIDDEN_AND_READ = {
|
||||
("dllm/config.py", "model_path"),
|
||||
("entrypoints/engine.py", "reasoning_parser"),
|
||||
("entrypoints/engine.py", "tool_call_parser"),
|
||||
("model_loader/expert_pack_runtime.py", "model_path"),
|
||||
("weight_cache/daemon.py", "dp_size"),
|
||||
("weight_cache/daemon.py", "dtype"),
|
||||
("weight_cache/daemon.py", "ep_size"),
|
||||
|
||||
Reference in New Issue
Block a user