Compare commits
44
Commits
g9i-est-td
...
h20-est
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4e60127fb4 | ||
|
|
e9fdf3ad0a | ||
|
|
2087760ea8 | ||
|
|
ef1a5955f3 | ||
|
|
7e17dee8b7 | ||
|
|
0090df5d64 | ||
|
|
4938eea2dc | ||
|
|
8c6cbd8c40 | ||
|
|
c82aed9fef | ||
|
|
e6f6420275 | ||
|
|
566176d0cf | ||
|
|
4f52678224 | ||
|
|
068acff09d | ||
|
|
f9056f0fcd | ||
|
|
2aae0cb4ee | ||
|
|
ea5fc832c1 | ||
|
|
9894dc789c | ||
|
|
77e7555ae3 | ||
|
|
319ee3c056 | ||
|
|
e59014be87 | ||
|
|
fd86688428 | ||
|
|
da343ba5d1 | ||
|
|
9ea02a7201 | ||
|
|
6c1c54b514 | ||
|
|
51624199b4 | ||
|
|
31b3758c99 | ||
|
|
740372bef0 | ||
|
|
19fe20baec | ||
|
|
383df91fcc | ||
|
|
534842ac08 | ||
|
|
4dd95a8817 | ||
|
|
656e00dcd3 | ||
|
|
a4b1363bf4 | ||
|
|
ad2acaa2df | ||
|
|
ecaaa37cb0 | ||
|
|
1b70fc192c | ||
|
|
3ebbb2d536 | ||
|
|
5d52cccbee | ||
|
|
74d03143e2 | ||
|
|
2b192aade6 | ||
|
|
9247c97919 | ||
|
|
5e7190b87a | ||
|
|
e00930c548 | ||
|
|
d7ff74b984 |
@@ -0,0 +1,146 @@
|
||||
"""Compare default, H200, and H20-3e block-FP8 configs on an H20-3e.
|
||||
|
||||
Example:
|
||||
python benchmark/kernels/deepseek/benchmark_h20_block_fp8.py \
|
||||
--output h20_block_fp8.jsonl
|
||||
|
||||
This measures individual GEMMs, not end-to-end model throughput. Run on an
|
||||
otherwise idle GPU. Inputs are synthetic; no model weights are needed.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import pathlib
|
||||
import statistics
|
||||
from unittest.mock import patch
|
||||
|
||||
import torch
|
||||
import triton
|
||||
|
||||
from sglang.kernels.ops.gemm import fp8_kernel
|
||||
|
||||
SHAPES = [
|
||||
(1792, 5120),
|
||||
(25600, 6144),
|
||||
(4096, 1280),
|
||||
(5120, 1024),
|
||||
(5120, 288),
|
||||
(576, 5120),
|
||||
]
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--output", type=pathlib.Path, required=True)
|
||||
parser.add_argument(
|
||||
"--batch-sizes",
|
||||
type=int,
|
||||
nargs="+",
|
||||
default=[1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64],
|
||||
)
|
||||
parser.add_argument("--trials", type=int, default=5)
|
||||
parser.add_argument("--rep-ms", type=int, default=50)
|
||||
parser.add_argument("--seed", type=int, default=98765)
|
||||
args = parser.parse_args()
|
||||
assert torch.cuda.get_device_name() == "NVIDIA H20-3e"
|
||||
assert args.trials > 0 and args.rep_ms > 0 and all(m > 0 for m in args.batch_sizes)
|
||||
torch.backends.cuda.matmul.allow_tf32 = False
|
||||
torch.manual_seed(args.seed)
|
||||
config_dir = pathlib.Path(fp8_kernel.__file__).parent / "configs"
|
||||
with args.output.open("x") as output:
|
||||
|
||||
def emit(row):
|
||||
text = json.dumps(row)
|
||||
output.write(text + "\n")
|
||||
output.flush()
|
||||
print(text, flush=True)
|
||||
|
||||
emit(
|
||||
{
|
||||
"kind": "environment",
|
||||
"device": torch.cuda.get_device_name(),
|
||||
"torch": torch.__version__,
|
||||
"triton": triton.__version__,
|
||||
"seed": args.seed,
|
||||
"batch_sizes": args.batch_sizes,
|
||||
}
|
||||
)
|
||||
for n, k in SHAPES:
|
||||
b = (torch.randn(n, k, device="cuda") * 0.2).to(torch.float8_e4m3fn)
|
||||
bs = torch.rand(n // 32, k // 32, device="cuda") + 0.1
|
||||
b_dequant = b.float() * bs.repeat_interleave(32, 0).repeat_interleave(32, 1)
|
||||
tables = {"default": None}
|
||||
for device in ("NVIDIA_H200", "NVIDIA_H20-3e"):
|
||||
path = (
|
||||
config_dir
|
||||
/ f"N={n},K={k},device_name={device},dtype=fp8_w8a8,block_shape=[32, 32].json"
|
||||
)
|
||||
if path.exists():
|
||||
tables[device] = {
|
||||
int(m): cfg for m, cfg in json.loads(path.read_text()).items()
|
||||
}
|
||||
assert "NVIDIA_H20-3e" in tables, (
|
||||
"Install the H20-3e configuration files before benchmarking"
|
||||
)
|
||||
for m in args.batch_sizes:
|
||||
a = (torch.randn(m, k, device="cuda") * 0.2).to(torch.float8_e4m3fn)
|
||||
scales = (torch.rand(k // 32, m, device="cuda") + 0.1).T
|
||||
reference = (
|
||||
(a.float() * scales.repeat_interleave(32, 1)) @ b_dequant.T
|
||||
).to(torch.bfloat16)
|
||||
results = {}
|
||||
# Compile and validate every variant before timing any variant.
|
||||
for label, table in tables.items():
|
||||
with patch.object(
|
||||
fp8_kernel, "get_w8a8_block_fp8_configs", return_value=table
|
||||
):
|
||||
actual = fp8_kernel.w8a8_block_fp8_matmul_triton(
|
||||
a, b, scales, bs, [32, 32], torch.bfloat16
|
||||
)
|
||||
relative_l2 = (
|
||||
torch.linalg.vector_norm(actual.float() - reference.float())
|
||||
/ torch.linalg.vector_norm(reference.float())
|
||||
).item()
|
||||
assert torch.isfinite(actual).all() and relative_l2 < 0.01
|
||||
results[label] = {
|
||||
"config": None
|
||||
if table is None
|
||||
else table[min(table, key=lambda key: abs(key - m))],
|
||||
"relative_l2": relative_l2,
|
||||
"trials_us": [],
|
||||
}
|
||||
for trial in range(args.trials):
|
||||
labels = list(tables)
|
||||
# Rotate variant order to reduce systematic timing-order bias.
|
||||
labels = (
|
||||
labels[trial % len(labels) :] + labels[: trial % len(labels)]
|
||||
)
|
||||
for label in labels:
|
||||
with patch.object(
|
||||
fp8_kernel,
|
||||
"get_w8a8_block_fp8_configs",
|
||||
return_value=tables[label],
|
||||
):
|
||||
|
||||
def invoke():
|
||||
return fp8_kernel.w8a8_block_fp8_matmul_triton(
|
||||
a, b, scales, bs, [32, 32], torch.bfloat16
|
||||
)
|
||||
|
||||
elapsed = (
|
||||
triton.testing.do_bench_cudagraph(
|
||||
invoke, rep=args.rep_ms
|
||||
)
|
||||
* 1000
|
||||
)
|
||||
results[label]["trials_us"].append(elapsed)
|
||||
for result in results.values():
|
||||
result["median_us"] = statistics.median(result["trials_us"])
|
||||
emit(
|
||||
{"kind": "measurement", "m": m, "n": n, "k": k, "results": results}
|
||||
)
|
||||
emit({"kind": "complete"})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,181 @@
|
||||
"""Finite cold-cache decode benchmark using synthetic token IDs.
|
||||
|
||||
Run against an otherwise idle SGLang server. This clears its prefix cache.
|
||||
Example:
|
||||
python benchmark/serving/benchmark_h20_decode.py \
|
||||
--model-path /path/to/DeepSeek-V4.1-Flash --output before.jsonl
|
||||
|
||||
Repeat after installing the H20-3e configurations with identical server flags.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import concurrent.futures
|
||||
import datetime
|
||||
import hashlib
|
||||
import json
|
||||
import pathlib
|
||||
import statistics
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
from tokenizers import Tokenizer
|
||||
|
||||
p = argparse.ArgumentParser(description=__doc__)
|
||||
p.add_argument("--label", default="measurement")
|
||||
p.add_argument("--url", default="http://127.0.0.1:8000")
|
||||
p.add_argument("--model-path", type=pathlib.Path, required=True)
|
||||
p.add_argument("--output", type=pathlib.Path, required=True)
|
||||
args = p.parse_args()
|
||||
out = args.output
|
||||
assert not out.exists()
|
||||
tokenizer = Tokenizer.from_file(str(args.model_path / "tokenizer.json"))
|
||||
filler = tokenizer.encode(
|
||||
(
|
||||
"The research server processes independent records. Each record contains an identifier, a timestamp, and a measured value. We compare latency using the same data and the same output budget.\n"
|
||||
)
|
||||
* 16000,
|
||||
add_special_tokens=False,
|
||||
).ids
|
||||
|
||||
|
||||
def emit(row):
|
||||
row = {
|
||||
"time": datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
||||
"label": args.label,
|
||||
**row,
|
||||
}
|
||||
with out.open("a") as f:
|
||||
f.write(json.dumps(row) + "\n")
|
||||
print(json.dumps(row), flush=True)
|
||||
|
||||
|
||||
def request(path, body=None):
|
||||
return urllib.request.urlopen(
|
||||
urllib.request.Request(
|
||||
args.url.rstrip("/") + path,
|
||||
data=None if body is None else json.dumps(body).encode(),
|
||||
headers={"Content-Type": "application/json"},
|
||||
),
|
||||
timeout=None,
|
||||
)
|
||||
|
||||
|
||||
def flush():
|
||||
with request("/flush_cache?timeout=30", {}) as r:
|
||||
r.read()
|
||||
|
||||
|
||||
def native(case, isl, osl, index):
|
||||
ids = (
|
||||
tokenizer.encode(
|
||||
f"Independent record {index}: {case}.\n", add_special_tokens=False
|
||||
).ids
|
||||
+ filler
|
||||
)[:isl]
|
||||
start = time.perf_counter()
|
||||
first = last = None
|
||||
final = None
|
||||
events = 0
|
||||
body = {
|
||||
"input_ids": ids,
|
||||
"sampling_params": {
|
||||
"temperature": 0.0,
|
||||
"max_new_tokens": osl,
|
||||
"ignore_eos": True,
|
||||
},
|
||||
"stream": True,
|
||||
}
|
||||
with request("/generate", body) as response:
|
||||
for line in response:
|
||||
if not line.startswith(b"data:"):
|
||||
continue
|
||||
value = line[5:].strip()
|
||||
if value == b"[DONE]":
|
||||
break
|
||||
obj = json.loads(value)
|
||||
now = time.perf_counter()
|
||||
if obj.get("error"):
|
||||
raise RuntimeError(str(obj["error"]))
|
||||
if obj.get("meta_info", {}).get("completion_tokens", 0) > 0:
|
||||
first = first or now
|
||||
last = now
|
||||
final = obj
|
||||
events += 1
|
||||
elapsed = time.perf_counter() - start
|
||||
assert final is not None, (case, index)
|
||||
meta = final["meta_info"]
|
||||
assert meta["completion_tokens"] == osl and meta["prompt_tokens"] == isl, meta
|
||||
assert meta.get("cached_tokens") == 0, meta
|
||||
return {
|
||||
"kind": "request",
|
||||
"case": case,
|
||||
"index": index,
|
||||
"input_hash": hashlib.sha256(json.dumps(ids).encode()).hexdigest(),
|
||||
"prompt_tokens": isl,
|
||||
"completion_tokens": osl,
|
||||
"cached_tokens": meta["cached_tokens"],
|
||||
"ttft_s": first - start,
|
||||
"elapsed_s": elapsed,
|
||||
"decode_tokens_per_s": (osl - 1) / (last - first) if osl > 1 else None,
|
||||
"tpot_ms": 1000 * (last - first) / (osl - 1) if osl > 1 else None,
|
||||
"events": events,
|
||||
}
|
||||
|
||||
|
||||
cases = [
|
||||
("8k_1000_c1", 8192, 1000, 1),
|
||||
("1k_128_c1", 1024, 128, 1),
|
||||
("1k_128_c8", 1024, 128, 8),
|
||||
("1k_128_c32", 1024, 128, 32),
|
||||
("8k_128_c8", 8192, 128, 8),
|
||||
("32k_128_c1", 32768, 128, 1),
|
||||
]
|
||||
emit(
|
||||
{
|
||||
"kind": "start",
|
||||
"cases": cases,
|
||||
"repetitions": 3,
|
||||
}
|
||||
)
|
||||
try:
|
||||
for case, isl, osl, concurrency in cases:
|
||||
flush()
|
||||
emit({"kind": "warmup_start", "case": case})
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as pool:
|
||||
list(
|
||||
pool.map(
|
||||
lambda i: native("warm_" + case, isl, 16, i), range(concurrency)
|
||||
)
|
||||
)
|
||||
for repeat in range(3):
|
||||
flush()
|
||||
start = time.perf_counter()
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as pool:
|
||||
rows = list(
|
||||
pool.map(
|
||||
lambda i: native(case, isl, osl, i + repeat * 100),
|
||||
range(concurrency),
|
||||
)
|
||||
)
|
||||
wall = time.perf_counter() - start
|
||||
for row in rows:
|
||||
emit({**row, "repeat": repeat})
|
||||
emit(
|
||||
{
|
||||
"kind": "batch",
|
||||
"case": case,
|
||||
"repeat": repeat,
|
||||
"concurrency": concurrency,
|
||||
"wall_s": wall,
|
||||
"output_tokens_per_s": concurrency * osl / wall,
|
||||
"median_ttft_s": statistics.median(x["ttft_s"] for x in rows),
|
||||
"median_decode_tokens_per_s": statistics.median(
|
||||
x["decode_tokens_per_s"] for x in rows
|
||||
),
|
||||
"median_tpot_ms": statistics.median(x["tpot_ms"] for x in rows),
|
||||
}
|
||||
)
|
||||
emit({"kind": "complete"})
|
||||
except Exception as exc:
|
||||
emit({"kind": "failed", "error": repr(exc)})
|
||||
raise
|
||||
@@ -74,10 +74,12 @@ dependencies = [
|
||||
"regex",
|
||||
"requests",
|
||||
"scipy",
|
||||
"sentencepiece",
|
||||
"sentencepiece==0.2.1", # 0.2.2 rejects null pieces in InternVL tokenizers.
|
||||
"setproctitle",
|
||||
"sgl-deep-ep==0.1.2",
|
||||
"sgl-deep-gemm==0.2.0",
|
||||
"sgl-deep-ep==0.1.2.post1",
|
||||
"sgl-deep-gemm==0.2.2",
|
||||
# 0.4.9 wheels are built against torch 2.14 (CUDAErrorLogCapture); this
|
||||
# branch pins torch 2.13, so stay on the v0.5.21-paired 0.4.7.
|
||||
"sglang-kernel==0.4.7",
|
||||
"smg-grpc-servicer>=0.9.0",
|
||||
"soundfile==0.13.1",
|
||||
|
||||
@@ -54,7 +54,7 @@ dependencies = [
|
||||
"regex",
|
||||
"requests",
|
||||
"scipy",
|
||||
"sentencepiece",
|
||||
"sentencepiece==0.2.1", # 0.2.2 rejects null pieces in InternVL tokenizers.
|
||||
"setproctitle",
|
||||
"smg-grpc-servicer>=0.9.0",
|
||||
"soundfile==0.13.1",
|
||||
|
||||
@@ -55,7 +55,7 @@ dependencies = [
|
||||
"regex",
|
||||
"requests",
|
||||
"scipy",
|
||||
"sentencepiece",
|
||||
"sentencepiece==0.2.1", # 0.2.2 rejects null pieces in InternVL tokenizers.
|
||||
"setproctitle",
|
||||
"smg-grpc-servicer>=0.9.0",
|
||||
"soundfile==0.13.1",
|
||||
|
||||
@@ -56,7 +56,7 @@ runtime_base = [
|
||||
"regex",
|
||||
"requests",
|
||||
"scipy",
|
||||
"sentencepiece",
|
||||
"sentencepiece==0.2.1", # 0.2.2 rejects null pieces in InternVL tokenizers.
|
||||
"setproctitle",
|
||||
"smg-grpc-servicer>=0.9.0",
|
||||
"soundfile==0.13.1",
|
||||
|
||||
@@ -54,7 +54,7 @@ dependencies = [
|
||||
"regex",
|
||||
"requests",
|
||||
"scipy",
|
||||
"sentencepiece",
|
||||
"sentencepiece==0.2.1", # 0.2.2 rejects null pieces in InternVL tokenizers.
|
||||
"setproctitle",
|
||||
"sglang-kernel-xpu @ https://github.com/sgl-project/whl/releases/download/v0.3.0+xpu/sglang_kernel_xpu-0.3.0-cp310-abi3-manylinux_2_38_x86_64.whl",
|
||||
"smg-grpc-servicer>=0.9.0",
|
||||
|
||||
@@ -18,6 +18,8 @@
|
||||
|
||||
namespace sglang {
|
||||
|
||||
using device::math::fast_mod_div_u32_t;
|
||||
|
||||
/// \brief Ratio-2 decode compressor: pair-pool, RMSNorm and the main-KV write.
|
||||
///
|
||||
/// `kv_input` and `kv_state` rows are `2 * kHeadDim` floats, kv then score.
|
||||
@@ -34,8 +36,7 @@ struct Compress2DecodeParams {
|
||||
const int64_t* __restrict__ req; // [num_tokens], req_pool_idx per token
|
||||
const void* __restrict__ raw_out_loc; // [num_tokens] LocT, the FULL slot; 0 marks a padded row
|
||||
uint8_t* __restrict__ kvcache; // [npages, kPageBytes] uint8
|
||||
/// Positions per request slot in the pair-state ring.
|
||||
uint32_t ring_size;
|
||||
fast_mod_div_u32_t ring_size; // Positions per request slot in the pair-state ring.
|
||||
float eps;
|
||||
};
|
||||
|
||||
@@ -43,21 +44,32 @@ struct Compress2DecodeParams {
|
||||
/// At (512, 64) it also keeps the nope/rope split warp-aligned, as the fp8 amax reduction requires.
|
||||
constexpr uint32_t kC2VecSize = 2;
|
||||
|
||||
/// \brief grid = num_tokens, block = kHeadDim / kC2VecSize.
|
||||
/// \brief Where a completing row finds its partner, and who parks a pending row.
|
||||
enum class C2Mode {
|
||||
/// One row per request: the partner comes from the ring, a pending row parks there.
|
||||
Decode,
|
||||
/// `draft_len` consecutive positions per request: `blockIdx.x` is the position inside the
|
||||
/// block, `blockIdx.y` the request, and every row but the first takes its partner from the
|
||||
/// previous `kv_input` row instead of the ring.
|
||||
Verify,
|
||||
/// Arbitrary rows, paired as `_low_ratio_pair_partners` pairs them: the partner is the previous
|
||||
/// row when it is the same request's previous position, else the ring. A pending row does
|
||||
/// nothing here; `c2_prefill_store_kernel` parks the ring rows after this kernel read them.
|
||||
Prefill,
|
||||
};
|
||||
|
||||
/// \brief grid = num_tokens (Decode, Prefill) or `draft_len x batch` (Verify),
|
||||
/// block = kHeadDim / kC2VecSize.
|
||||
///
|
||||
/// An odd position completes a group with its even predecessor; an even one parks in the state.
|
||||
///
|
||||
/// Under target-verify, `draft_len` consecutive positions per request: `blockIdx.x` is the
|
||||
/// position inside the block, `blockIdx.y` the request, and every row but the first takes its
|
||||
/// partner from the previous `kv_input` row instead of the ring.
|
||||
///
|
||||
/// The three reductions have different widths and are not interchangeable: the RMSNorm
|
||||
/// statistic spans the row, an fp8 store scale 64 elements, an fp4 block 16.
|
||||
///
|
||||
/// kLayout is the cache's page format: V4 and V41 store the fake-quantized value; V41_FP4
|
||||
/// stores the e2m1 codes and their e4m3 scales directly, so the fp4 rounding happens once.
|
||||
template <
|
||||
bool kVerify,
|
||||
C2Mode kMode,
|
||||
int64_t kHeadDim,
|
||||
int64_t kRopeDim,
|
||||
int32_t kPageBits,
|
||||
@@ -91,16 +103,21 @@ __global__ __launch_bounds__(kHeadDim / kC2VecSize) void flash_c2_decode_kernel(
|
||||
|
||||
const auto tx = threadIdx.x;
|
||||
// Verify gives each request a CTA column; decode a flat grid of one row each.
|
||||
const auto row = kVerify ? blockIdx.y * gridDim.x + blockIdx.x : blockIdx.x;
|
||||
const auto row = kMode == C2Mode::Verify ? blockIdx.y * gridDim.x + blockIdx.x : blockIdx.x;
|
||||
PDLWaitPrimary<kUsePDL>();
|
||||
// Slots fit in int32 whatever width the scheduler hands them in.
|
||||
const auto raw_out_loc = static_cast<int32_t>(static_cast<const LocT*>(params.raw_out_loc)[row]);
|
||||
// CUDA graph padding is a completely inert row: do not read schedule, input,
|
||||
// state, or RoPE data, and do not publish output or update the cache.
|
||||
if (raw_out_loc == 0) return PDLTriggerSecondary<kUsePDL>();
|
||||
const auto pos = static_cast<const PosT*>(params.positions)[row];
|
||||
// A completing row reads the slot left by `pos - 1`;
|
||||
// a pending row writes its own slot, so reads and writes stay disjoint.
|
||||
const auto* loc_ptr = static_cast<const LocT*>(params.raw_out_loc);
|
||||
const auto* pos_ptr = static_cast<const PosT*>(params.positions);
|
||||
const auto raw_out_loc = loc_ptr[row];
|
||||
// Prefill rows are many, so a CUDA graph padding row exits before any load.
|
||||
// Decode and verify rows are few: every row issues its loads to keep more in flight.
|
||||
if constexpr (kMode == C2Mode::Prefill) {
|
||||
if (raw_out_loc == 0) return PDLTriggerSecondary<kUsePDL>();
|
||||
}
|
||||
const auto pos = pos_ptr[row];
|
||||
// Prefill leaves a pending row's latent unwritten; its index-K slot is -1, so nothing reads it.
|
||||
if constexpr (kMode == C2Mode::Prefill) {
|
||||
if ((pos & 1) == 0) return PDLTriggerSecondary<kUsePDL>();
|
||||
}
|
||||
const auto rid = params.req[row];
|
||||
|
||||
fp32_vec_t kv_new, score_new;
|
||||
@@ -108,21 +125,32 @@ __global__ __launch_bounds__(kHeadDim / kC2VecSize) void flash_c2_decode_kernel(
|
||||
score_new.load(params.kv_input + row * kStride, tx + kCTASize);
|
||||
|
||||
const auto ring = static_cast<int64_t>(rid) * params.ring_size;
|
||||
const auto read_row = ring + (pos - 1 + params.ring_size) % params.ring_size;
|
||||
const auto write_row = ring + pos % params.ring_size;
|
||||
const auto write_offset = static_cast<uint32_t>(pos) % params.ring_size;
|
||||
const auto read_offset = write_offset == 0 ? params.ring_size - 1 : write_offset - 1;
|
||||
const auto write_row = ring + write_offset;
|
||||
const auto read_row = ring + read_offset;
|
||||
|
||||
fp32_vec_t kv_old, score_old;
|
||||
const float* partner = params.kv_state + read_row * kStride;
|
||||
if constexpr (kVerify) {
|
||||
if constexpr (kMode == C2Mode::Verify) {
|
||||
if (blockIdx.x != 0) partner = params.kv_input + static_cast<int64_t>(row - 1) * kStride;
|
||||
}
|
||||
if constexpr (kMode == C2Mode::Prefill) {
|
||||
if (row > 0 && params.req[row - 1] == rid && pos_ptr[row - 1] == pos - 1 && loc_ptr[row - 1] != 0) {
|
||||
partner = params.kv_input + static_cast<int64_t>(row - 1) * kStride;
|
||||
}
|
||||
}
|
||||
kv_old.load(partner, tx);
|
||||
score_old.load(partner, tx + kCTASize);
|
||||
|
||||
if ((pos & 1) == 0) {
|
||||
kv_new.store(params.kv_state + write_row * kStride, tx);
|
||||
score_new.store(params.kv_state + write_row * kStride, tx + kCTASize);
|
||||
return PDLTriggerSecondary<kUsePDL>();
|
||||
if constexpr (kMode != C2Mode::Prefill) {
|
||||
// CUDA graph padding case
|
||||
if (raw_out_loc == 0) return PDLTriggerSecondary<kUsePDL>();
|
||||
if ((pos & 1) == 0) {
|
||||
kv_new.store(params.kv_state + write_row * kStride, tx);
|
||||
score_new.store(params.kv_state + write_row * kStride, tx + kCTASize);
|
||||
return PDLTriggerSecondary<kUsePDL>();
|
||||
}
|
||||
}
|
||||
|
||||
constexpr uint32_t kNumWarps = kCTASize / kWarpThreads;
|
||||
@@ -203,9 +231,9 @@ __global__ __launch_bounds__(kHeadDim / kC2VecSize) void flash_c2_decode_kernel(
|
||||
}
|
||||
}
|
||||
|
||||
const int32_t out_loc = static_cast<uint32_t>(raw_out_loc) >> 1;
|
||||
if constexpr (kLayout == KVLayout::V41_FP4) {
|
||||
// The fp4 cache takes the rotated bf16 value as is: its row quantizer is the fake quant, minus the dequant.
|
||||
const int32_t out_loc = raw_out_loc >> 1;
|
||||
const auto kv_row = Paged::row(params.kvcache, out_loc);
|
||||
return deepseek_v4::v41::store_row<kLayout>(kv_row.data, kv_row.scale, tx, staged);
|
||||
}
|
||||
@@ -228,7 +256,6 @@ __global__ __launch_bounds__(kHeadDim / kC2VecSize) void flash_c2_decode_kernel(
|
||||
}
|
||||
|
||||
// `raw_out_loc / ratio`; ratio 2 makes it a shift.
|
||||
const int32_t out_loc = raw_out_loc >> 1;
|
||||
const auto kv_row = Paged::row(params.kvcache, out_loc);
|
||||
|
||||
if constexpr (kLayout == KVLayout::V41) {
|
||||
@@ -264,21 +291,63 @@ __global__ __launch_bounds__(kHeadDim / kC2VecSize) void flash_c2_decode_kernel(
|
||||
}
|
||||
}
|
||||
|
||||
/// \brief Prefill ring write-back, launched after `flash_c2_decode_kernel<C2Mode::Prefill>`.
|
||||
///
|
||||
/// Each request's last `ring_size` rows park their `| kv | score |` at `req * ring_size +
|
||||
/// pos % ring_size` for a later step's first partner. It cannot run inside the pooling kernel:
|
||||
/// a chunk whose first row is odd and whose last row is even reads and writes the same ring
|
||||
/// slot. grid = `ring_size x num_reqs`, CTA (k, i) parks the k-th row from the end of request i,
|
||||
/// one float4 per thread; those rows are consecutive positions, so their slots are distinct.
|
||||
template <int64_t kHeadDim, typename PosT, typename LocT, bool kUsePDL>
|
||||
__global__ __launch_bounds__(kHeadDim / 2) void c2_prefill_store_kernel(
|
||||
const Compress2DecodeParams params,
|
||||
const int32_t* __restrict__ extend_start_loc,
|
||||
const int32_t* __restrict__ extend_seq_lens) {
|
||||
using namespace device;
|
||||
constexpr int64_t kStride = kHeadDim * 2;
|
||||
using vec_t = AlignedVector<float, 4>;
|
||||
static_assert(kStride == 4 * (kHeadDim / 2));
|
||||
|
||||
// The pooling kernel reads the ring rows this one overwrites.
|
||||
PDLWaitPrimary<kUsePDL>();
|
||||
const auto bx = static_cast<int32_t>(blockIdx.x);
|
||||
const auto len = extend_seq_lens[blockIdx.y];
|
||||
const auto start_loc = extend_start_loc[blockIdx.y];
|
||||
if (bx >= len) return;
|
||||
const auto row = start_loc + (len - 1 - bx);
|
||||
const auto rid = params.req[row];
|
||||
const auto out = static_cast<const LocT*>(params.raw_out_loc)[row];
|
||||
const auto pos = static_cast<uint32_t>(static_cast<const PosT*>(params.positions)[row]);
|
||||
const auto dst_row = rid * params.ring_size + pos % params.ring_size;
|
||||
if (out == 0) return;
|
||||
vec_t buf;
|
||||
buf.load(params.kv_input + row * kStride, threadIdx.x);
|
||||
PDLTriggerSecondary<kUsePDL>();
|
||||
buf.store(params.kv_state + dst_row * kStride, threadIdx.x);
|
||||
}
|
||||
|
||||
template <int64_t kHeadDim, int64_t kRopeDim, uint32_t kPageSize, deepseek_v4::KVLayout kLayout, bool kUsePDL>
|
||||
struct FlashCompress2Kernel {
|
||||
static constexpr uint32_t kBlockSize = kHeadDim / kC2VecSize;
|
||||
static constexpr int32_t kPageBits = std::bit_width(kPageSize) - 1;
|
||||
static constexpr int64_t kPageBytes = deepseek_v4::kv_page_bytes<kLayout>(kPageSize);
|
||||
static_assert(kLayout != deepseek_v4::KVLayout::V4 || kPageBytes == host::div_ceil(584ll * kPageSize, 576) * 576);
|
||||
template <bool kVerify, typename PosT, typename LocT>
|
||||
template <C2Mode kMode, typename PosT, typename LocT>
|
||||
static constexpr auto kernel =
|
||||
flash_c2_decode_kernel<kVerify, kHeadDim, kRopeDim, kPageBits, PosT, LocT, kLayout, kUsePDL>;
|
||||
flash_c2_decode_kernel<kMode, kHeadDim, kRopeDim, kPageBits, PosT, LocT, kLayout, kUsePDL>;
|
||||
template <typename PosT, typename LocT>
|
||||
static constexpr auto store_kernel = c2_prefill_store_kernel<kHeadDim, PosT, LocT, kUsePDL>;
|
||||
|
||||
/// \brief The (`positions`, `raw_out_loc`) dtype pair, resolved at run time.
|
||||
template <bool kVerify>
|
||||
template <C2Mode kMode>
|
||||
static auto select(const bool pos_i32, const bool loc_i32) {
|
||||
if (pos_i32) return loc_i32 ? kernel<kVerify, int32_t, int32_t> : kernel<kVerify, int32_t, int64_t>;
|
||||
return loc_i32 ? kernel<kVerify, int64_t, int32_t> : kernel<kVerify, int64_t, int64_t>;
|
||||
if (pos_i32) return loc_i32 ? kernel<kMode, int32_t, int32_t> : kernel<kMode, int32_t, int64_t>;
|
||||
return loc_i32 ? kernel<kMode, int64_t, int32_t> : kernel<kMode, int64_t, int64_t>;
|
||||
}
|
||||
|
||||
static auto select_store(const bool pos_i32, const bool loc_i32) {
|
||||
if (pos_i32) return loc_i32 ? store_kernel<int32_t, int32_t> : store_kernel<int32_t, int64_t>;
|
||||
return loc_i32 ? store_kernel<int64_t, int32_t> : store_kernel<int64_t, int64_t>;
|
||||
}
|
||||
|
||||
// The sum of squares reduces through a fixed-size shared array, so the CTA must be whole warps.
|
||||
@@ -302,7 +371,7 @@ struct FlashCompress2Kernel {
|
||||
const tvm::ffi::TensorView kvcache,
|
||||
const int64_t ring_size,
|
||||
const int64_t draft_len) {
|
||||
launch(
|
||||
launch<false>(
|
||||
kv_input,
|
||||
kv_state,
|
||||
kv_output,
|
||||
@@ -317,7 +386,62 @@ struct FlashCompress2Kernel {
|
||||
draft_len);
|
||||
}
|
||||
|
||||
/// \brief The extend-batch pooling kernel, then the ring write-back.
|
||||
///
|
||||
/// Rows may mix requests, prefix offsets and padding; a pending (even) row's `kv_output`
|
||||
/// row is left unwritten. `extend_start_loc` / `extend_seq_lens` are int32 `[num_reqs]`:
|
||||
/// request i owns rows `[start[i], start[i] + len[i])`; zero-length requests are skipped.
|
||||
static void run_prefill_fusion(
|
||||
const tvm::ffi::TensorView kv_input,
|
||||
const tvm::ffi::TensorView kv_state,
|
||||
const tvm::ffi::TensorView kv_output,
|
||||
const tvm::ffi::TensorView norm_weight,
|
||||
const tvm::ffi::TensorView positions,
|
||||
const tvm::ffi::TensorView req,
|
||||
const tvm::ffi::TensorView raw_out_loc,
|
||||
const float eps,
|
||||
const tvm::ffi::TensorView freqs_cis,
|
||||
const tvm::ffi::TensorView kvcache,
|
||||
const int64_t ring_size,
|
||||
const tvm::ffi::TensorView extend_start_loc,
|
||||
const tvm::ffi::TensorView extend_seq_lens) {
|
||||
using namespace host;
|
||||
auto B = SymbolicSize{"num_reqs"};
|
||||
auto device_ = SymbolicDevice{};
|
||||
device_.set_options<kDLGPU>();
|
||||
TensorMatcher({B}).with_dtype<int32_t>().with_device(device_).verify(extend_start_loc);
|
||||
TensorMatcher({B}).with_dtype<int32_t>().with_device(device_).verify(extend_seq_lens);
|
||||
const auto requests = StoreRequests{
|
||||
.extend_start_loc = static_cast<const int32_t*>(extend_start_loc.data_ptr()),
|
||||
.extend_seq_lens = static_cast<const int32_t*>(extend_seq_lens.data_ptr()),
|
||||
.num_reqs = static_cast<uint32_t>(B.unwrap()),
|
||||
};
|
||||
launch<true>(
|
||||
kv_input,
|
||||
kv_state,
|
||||
kv_output,
|
||||
norm_weight,
|
||||
positions,
|
||||
req,
|
||||
raw_out_loc,
|
||||
eps,
|
||||
ring_size,
|
||||
freqs_cis,
|
||||
kvcache,
|
||||
1,
|
||||
&requests);
|
||||
}
|
||||
|
||||
private:
|
||||
struct StoreRequests {
|
||||
const int32_t* extend_start_loc;
|
||||
const int32_t* extend_seq_lens;
|
||||
uint32_t num_reqs;
|
||||
};
|
||||
|
||||
/// \tparam kPrefill `C2Mode::Prefill` plus the ring write-back of `requests`; otherwise
|
||||
/// Decode, or Verify when `draft_len > 1`.
|
||||
template <bool kPrefill>
|
||||
static void launch(
|
||||
const tvm::ffi::TensorView kv_input,
|
||||
const tvm::ffi::TensorView kv_state,
|
||||
@@ -330,7 +454,8 @@ struct FlashCompress2Kernel {
|
||||
const int64_t ring_size,
|
||||
const tvm::ffi::TensorView freqs_cis,
|
||||
const tvm::ffi::TensorView kvcache,
|
||||
const int64_t draft_len) {
|
||||
const int64_t draft_len,
|
||||
const StoreRequests* requests = nullptr) {
|
||||
using namespace host;
|
||||
|
||||
auto N = SymbolicSize{"num_tokens"};
|
||||
@@ -358,7 +483,7 @@ struct FlashCompress2Kernel {
|
||||
|
||||
const auto num_tokens = static_cast<uint32_t>(N.unwrap());
|
||||
if (num_tokens == 0) return;
|
||||
const auto is_verify = draft_len > 1;
|
||||
const auto is_verify = !kPrefill && draft_len > 1;
|
||||
CHECK_HOST(ring_size > 0 && draft_len >= 1);
|
||||
CHECK_HOST(!is_verify || num_tokens % draft_len == 0);
|
||||
CHECK_HOST(!is_verify || ring_size > draft_len)
|
||||
@@ -373,19 +498,28 @@ struct FlashCompress2Kernel {
|
||||
.req = static_cast<const int64_t*>(req.data_ptr()),
|
||||
.raw_out_loc = raw_out_loc.data_ptr(),
|
||||
.kvcache = static_cast<uint8_t*>(kvcache.data_ptr()),
|
||||
.ring_size = static_cast<uint32_t>(ring_size),
|
||||
.ring_size = static_cast<fast_mod_div_u32_t>(ring_size),
|
||||
.eps = eps,
|
||||
};
|
||||
// `LaunchKernel` is move-only, so each arm builds its own.
|
||||
const auto pos_i32 = pos_dtype.is_type<int32_t>();
|
||||
const auto loc_i32 = loc_dtype.is_type<int32_t>();
|
||||
if (is_verify) {
|
||||
if constexpr (kPrefill) {
|
||||
const auto k = select<C2Mode::Prefill>(pos_i32, loc_i32);
|
||||
LaunchKernel(num_tokens, kBlockSize, device_.unwrap()) //
|
||||
.enable_pdl(kUsePDL)(k, params);
|
||||
if (requests->num_reqs == 0) return;
|
||||
const auto store = select_store(pos_i32, loc_i32);
|
||||
const auto grid = dim3{static_cast<uint32_t>(ring_size), requests->num_reqs};
|
||||
LaunchKernel(grid, kHeadDim / 2, device_.unwrap()) //
|
||||
.enable_pdl(kUsePDL)(store, params, requests->extend_start_loc, requests->extend_seq_lens);
|
||||
} else if (is_verify) {
|
||||
const auto block = static_cast<uint32_t>(draft_len);
|
||||
const auto k = select<true>(pos_i32, loc_i32);
|
||||
const auto k = select<C2Mode::Verify>(pos_i32, loc_i32);
|
||||
LaunchKernel(dim3{block, num_tokens / block}, kBlockSize, device_.unwrap()) //
|
||||
.enable_pdl(kUsePDL)(k, params);
|
||||
} else {
|
||||
const auto k = select<false>(pos_i32, loc_i32);
|
||||
const auto k = select<C2Mode::Decode>(pos_i32, loc_i32);
|
||||
LaunchKernel(num_tokens, kBlockSize, device_.unwrap()) //
|
||||
.enable_pdl(kUsePDL)(k, params);
|
||||
}
|
||||
|
||||
@@ -80,7 +80,7 @@ struct FusedQNormRopeParams {
|
||||
float eps;
|
||||
};
|
||||
|
||||
template <typename DType, int64_t kHeadDim, int64_t kRopeDim, typename PosT, bool kUsePDL>
|
||||
template <typename DType, int64_t kHeadDim, int64_t kRopeDim, typename PosT, bool kUsePDL, bool kApplyNorm>
|
||||
Q_KERNEL void fused_q_norm_rope(const __grid_constant__ FusedQNormRopeParams params) {
|
||||
using namespace device;
|
||||
|
||||
@@ -129,24 +129,26 @@ Q_KERNEL void fused_q_norm_rope(const __grid_constant__ FusedQNormRopeParams par
|
||||
|
||||
const auto freq = mem_freq.load(params.freqs_cis + position * kRopeDim);
|
||||
|
||||
float sum_of_squares = 0.0f;
|
||||
if constexpr (kApplyNorm) {
|
||||
float sum_of_squares = 0.0f;
|
||||
#pragma unroll
|
||||
for (int i = 0; i < kLocalSize; ++i) {
|
||||
for (int i = 0; i < kLocalSize; ++i) {
|
||||
#pragma unroll
|
||||
for (int j = 0; j < kVecSize; ++j) {
|
||||
const auto x = cast<float>(input_vec[i][j]);
|
||||
sum_of_squares += x * x;
|
||||
for (int j = 0; j < kVecSize; ++j) {
|
||||
const auto x = cast<float>(input_vec[i][j]);
|
||||
sum_of_squares += x * x;
|
||||
}
|
||||
}
|
||||
}
|
||||
sum_of_squares = warp::reduce_sum(sum_of_squares);
|
||||
const auto norm_factor = math::rsqrt(sum_of_squares / kHeadDim + params.eps);
|
||||
sum_of_squares = warp::reduce_sum(sum_of_squares);
|
||||
const auto norm_factor = math::rsqrt(sum_of_squares / kHeadDim + params.eps);
|
||||
|
||||
#pragma unroll
|
||||
for (int i = 0; i < kLocalSize; ++i) {
|
||||
for (int i = 0; i < kLocalSize; ++i) {
|
||||
#pragma unroll
|
||||
for (int j = 0; j < kVecSize; ++j) {
|
||||
const auto x = cast<float>(input_vec[i][j]);
|
||||
input_vec[i][j] = cast<DType>(x * norm_factor);
|
||||
for (int j = 0; j < kVecSize; ++j) {
|
||||
const auto x = cast<float>(input_vec[i][j]);
|
||||
input_vec[i][j] = cast<DType>(x * norm_factor);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -172,17 +174,34 @@ Q_KERNEL void fused_q_norm_rope(const __grid_constant__ FusedQNormRopeParams par
|
||||
const auto elem = mem_elem.load(s_rope[warp_id]);
|
||||
const auto [x_real, x_imag] = cast<fp32x2_t>(elem);
|
||||
const auto [freq_real, freq_imag] = freq;
|
||||
const fp32x2_t rotated = {
|
||||
x_real * freq_real - x_imag * freq_imag,
|
||||
x_real * freq_imag + x_imag * freq_real,
|
||||
};
|
||||
#ifdef USE_ROCM
|
||||
// gfx950 batched RoPE rounds the sine product, then one fma with the cosine. Only the
|
||||
// kApplyNorm = false specialization replaces a kernel that rounded that way, so only it
|
||||
// matches it; the norm path keeps the order it has always had.
|
||||
constexpr bool kBatchedRopeOrder = !kApplyNorm;
|
||||
#else
|
||||
constexpr bool kBatchedRopeOrder = false;
|
||||
#endif
|
||||
fp32x2_t rotated;
|
||||
if constexpr (kBatchedRopeOrder) {
|
||||
rotated = {
|
||||
fmaf(x_real, freq_real, -__fmul_rn(x_imag, freq_imag)),
|
||||
fmaf(x_imag, freq_real, __fmul_rn(x_real, freq_imag)),
|
||||
};
|
||||
} else {
|
||||
rotated = {
|
||||
x_real * freq_real - x_imag * freq_imag,
|
||||
x_real * freq_imag + x_imag * freq_real,
|
||||
};
|
||||
}
|
||||
mem_elem.store(output_ptr + (kHeadDim - kRopeDim), cast<DType2>(rotated));
|
||||
}
|
||||
|
||||
template <typename DType, int64_t kHeadDim, int64_t kRopeDim, bool kUsePDL>
|
||||
// kApplyNorm = false only rotates the rope tail while copying Q into a (padded) output.
|
||||
template <typename DType, int64_t kHeadDim, int64_t kRopeDim, bool kApplyNorm, bool kUsePDL>
|
||||
struct FusedQNormRopeKernel {
|
||||
template <typename PosT>
|
||||
static constexpr auto kernel = fused_q_norm_rope<DType, kHeadDim, kRopeDim, PosT, kUsePDL>;
|
||||
static constexpr auto kernel = fused_q_norm_rope<DType, kHeadDim, kRopeDim, PosT, kUsePDL, kApplyNorm>;
|
||||
|
||||
static void forward(
|
||||
const tvm::ffi::TensorView q_input,
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
#include <sgl_kernel/tensor.h>
|
||||
#include <sgl_kernel/utils.h>
|
||||
|
||||
#include <sgl_kernel/math.cuh>
|
||||
#include <sgl_kernel/runtime.cuh>
|
||||
#include <sgl_kernel/type.cuh>
|
||||
#include <sgl_kernel/utils.cuh>
|
||||
@@ -21,31 +22,7 @@ namespace sglang {
|
||||
|
||||
using device::distributed::PushWorkSpace;
|
||||
using device::distributed::Semaphore;
|
||||
|
||||
// Runtime uint32 division as a multiply-high and a shift (round-up magic,
|
||||
// exact below 2^31); cuda::fast_mod_div needs a newer CCCL than CUDA 13 bundles.
|
||||
struct fast_mod_div_u32_t {
|
||||
uint32_t divisor;
|
||||
uint32_t magic;
|
||||
uint32_t shift;
|
||||
|
||||
__host__ explicit fast_mod_div_u32_t(uint32_t d) : divisor(d), magic(0), shift(0) {
|
||||
if (d > 1) {
|
||||
const uint32_t log2_ceil = 32 - std::countl_zero(d - 1);
|
||||
const uint32_t p = 31 + log2_ceil;
|
||||
magic = static_cast<uint32_t>(((uint64_t{1} << p) + d - 1) / d);
|
||||
shift = p - 32;
|
||||
}
|
||||
}
|
||||
|
||||
__device__ friend uint32_t operator/(uint32_t n, const fast_mod_div_u32_t& fd) {
|
||||
return fd.divisor == 1 ? n : __umulhi(n, fd.magic) >> fd.shift;
|
||||
}
|
||||
|
||||
__device__ friend uint32_t operator%(uint32_t n, const fast_mod_div_u32_t& fd) {
|
||||
return n - (n / fd) * fd.divisor;
|
||||
}
|
||||
};
|
||||
using device::math::fast_mod_div_u32_t;
|
||||
|
||||
template <uint32_t kWorldSize>
|
||||
struct NVLinkCommPushParams {
|
||||
@@ -168,8 +145,7 @@ PUSH_KERNEL void nvlink_push_kernel(const __grid_constant__ NVLinkCommPushParams
|
||||
ptx::st_multimem_16B(vec, dst_ptr_mc, vid);
|
||||
}
|
||||
} else /* reduce-scatter only */ {
|
||||
const auto token_id = vid / params.vecs_per_token_div;
|
||||
const auto offset = vid % params.vecs_per_token_div;
|
||||
const auto [token_id, offset] = math::div(vid, params.vecs_per_token_div);
|
||||
// Both by a compile-time constant, so this is a mask and a shift.
|
||||
const auto dst_rank = token_id % kWorldSize;
|
||||
const auto dst_token_id = token_id / kWorldSize;
|
||||
|
||||
@@ -53,6 +53,7 @@ struct ActivationParams {
|
||||
// for per-token routing and BLOCK_SIZE_M for sorted/TMA routing.
|
||||
const int32_t* __restrict__ expert_ids;
|
||||
uint32_t expert_step;
|
||||
float clamp_limit;
|
||||
};
|
||||
|
||||
template <
|
||||
@@ -61,7 +62,8 @@ template <
|
||||
bool kUsePDL,
|
||||
bool kFilterExpert,
|
||||
bool kRoundActivation = false,
|
||||
bool kReuseInput = false>
|
||||
bool kReuseInput = false,
|
||||
bool kClamp = false>
|
||||
__global__ void act_and_mul_kernel(const __grid_constant__ ActivationParams params) {
|
||||
using namespace device;
|
||||
constexpr auto kVecSize = kMaxVecBytes / sizeof(T);
|
||||
@@ -80,11 +82,17 @@ __global__ void act_and_mul_kernel(const __grid_constant__ ActivationParams para
|
||||
PDLWaitPrimary<kUsePDL>();
|
||||
const auto gate = device::load_as<vec_t>(params.input, input_offset);
|
||||
const auto up = device::load_as<vec_t>(params.input, input_offset + num_vecs);
|
||||
const float limit = device::cast<fp32_t>(device::cast<T>(params.clamp_limit));
|
||||
vec_t out;
|
||||
#pragma unroll
|
||||
for (int i = 0; i < kVecSize; ++i) {
|
||||
const float gate_f32 = device::cast<fp32_t>(gate[i]);
|
||||
const float up_f32 = device::cast<fp32_t>(up[i]);
|
||||
float gate_f32 = device::cast<fp32_t>(gate[i]);
|
||||
float up_f32 = device::cast<fp32_t>(up[i]);
|
||||
if constexpr (kClamp) {
|
||||
static_assert(kAct == ActivationKind::kSiLU);
|
||||
gate_f32 = gate_f32 > limit ? limit : gate_f32;
|
||||
up_f32 = up_f32 > limit ? limit : (up_f32 < -limit ? -limit : up_f32);
|
||||
}
|
||||
if constexpr (kRoundActivation) {
|
||||
const T activated = device::cast<T>(apply_activation_f32<kAct>(gate_f32));
|
||||
out[i] = device::cast<T>(device::cast<fp32_t>(activated) * up_f32);
|
||||
@@ -153,13 +161,14 @@ struct ActivationKernel {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
template <bool kRoundActivation = false, bool kReuseInput = false>
|
||||
template <bool kRoundActivation = false, bool kReuseInput = false, bool kClamp = false>
|
||||
static void launch(
|
||||
const tvm::ffi::TensorView& input,
|
||||
const tvm::ffi::TensorView& out,
|
||||
const std::string& type,
|
||||
const int32_t* expert_ids,
|
||||
uint32_t expert_step) {
|
||||
uint32_t expert_step,
|
||||
float clamp_limit = 0.0f) {
|
||||
using namespace host;
|
||||
|
||||
auto N = SymbolicSize{"num_tokens"};
|
||||
@@ -198,8 +207,14 @@ struct ActivationKernel {
|
||||
.num_tokens = num_tokens,
|
||||
.expert_ids = expert_ids,
|
||||
.expert_step = expert_step,
|
||||
.clamp_limit = clamp_limit,
|
||||
};
|
||||
if (expert_ids != nullptr) {
|
||||
if constexpr (kClamp) {
|
||||
RuntimeCheck(type == "silu" && expert_ids == nullptr, "clamping requires unfiltered SiLU");
|
||||
const auto kernel =
|
||||
act_and_mul_kernel<T, ActivationKind::kSiLU, kUsePDL, false, kRoundActivation, kReuseInput, true>;
|
||||
LaunchKernel(num_blocks, kBlockSize, device).enable_pdl(kUsePDL)(kernel, params);
|
||||
} else if (expert_ids != nullptr) {
|
||||
RuntimeCheck(expert_step > 0, "expert_step must be positive");
|
||||
const auto kernel = select_kernel<true, kRoundActivation, kReuseInput>(type);
|
||||
LaunchKernel(num_blocks, kBlockSize, device).enable_pdl(kUsePDL)(kernel, params);
|
||||
@@ -213,9 +228,13 @@ struct ActivationKernel {
|
||||
launch(input, out, type, /*expert_ids=*/nullptr, /*expert_step=*/1);
|
||||
}
|
||||
|
||||
static void
|
||||
run_activation_with_rounding(const tvm::ffi::TensorView input, const tvm::ffi::TensorView out, std::string type) {
|
||||
launch<true>(input, out, type, /*expert_ids=*/nullptr, /*expert_step=*/1);
|
||||
static void run_activation_with_rounding(
|
||||
const tvm::ffi::TensorView input, const tvm::ffi::TensorView out, std::string type, double clamp_limit) {
|
||||
if (clamp_limit > 0) {
|
||||
launch<true, false, true>(input, out, type, /*expert_ids=*/nullptr, /*expert_step=*/1, clamp_limit);
|
||||
} else {
|
||||
launch<true>(input, out, type, /*expert_ids=*/nullptr, /*expert_step=*/1);
|
||||
}
|
||||
}
|
||||
|
||||
static void run_activation_with_rounding_input_inplace(
|
||||
|
||||
@@ -404,6 +404,7 @@ bool is_valid_config(
|
||||
#define MXFP4_GET_IF(W_TYPE) \
|
||||
MXFP4_GET_IF_M1(W_TYPE, 8, 8, 256) \
|
||||
MXFP4_GET_IF_M1(W_TYPE, 8, 4, 128) \
|
||||
MXFP4_GET_IF_M1(W_TYPE, 4, 8, 128) \
|
||||
MXFP4_GET_IF_M234(W_TYPE, 16, 4, 256) \
|
||||
MXFP4_GET_IF_M234(W_TYPE, 8, 4, 128)
|
||||
|
||||
@@ -492,6 +493,16 @@ exec_config_t determine_exec_config(
|
||||
bool is_zp_float,
|
||||
int max_shared_mem,
|
||||
int sms) {
|
||||
#if SGL_CUDA_ARCH == 900
|
||||
if constexpr (std::is_same_v<scalar_t, __nv_bfloat16> && !kIsEP && !kHasBias) {
|
||||
// H200 TP8 DeepSeek-V4.1 gate/up: six sparse expert rows need a narrower
|
||||
// N tile and fewer persistent blocks than the occupancy-only heuristic.
|
||||
if (q_type == host::kFE2M1f && group_size == 32 && prob_m == 1 && prob_n == 640 && prob_k == 5120 && top_k == 6 &&
|
||||
thread_m_blocks == 1 && m_block_size_8 && !has_act_order && !has_zp) {
|
||||
return exec_config_t{2, thread_config_t{128, 64, 128}};
|
||||
}
|
||||
}
|
||||
#endif
|
||||
exec_config_t exec_cfg = exec_config_t{1, thread_config_t{-1, -1, -1}};
|
||||
thread_config_t* thread_configs = thread_m_blocks > 1 ? large_batch_thread_configs : small_batch_thread_configs;
|
||||
int thread_configs_size = thread_m_blocks > 1 ? sizeof(large_batch_thread_configs) / sizeof(thread_config_t)
|
||||
|
||||
@@ -112,6 +112,53 @@ SGL_DEVICE float fma_f32_bf16(bf16_t a, float b, float acc) {
|
||||
return fmaf(cast<fp32_t>(a), b, acc);
|
||||
}
|
||||
|
||||
/**
|
||||
* \brief Fast uint32 division and modulo by a compile-time constant divisor.
|
||||
*
|
||||
* This struct is only exact when the divisor is less than `2^31`. Larger dividends
|
||||
* will lead to incorrect results. Be careful when using this struct.
|
||||
*/
|
||||
struct fast_mod_div_u32_t {
|
||||
public:
|
||||
explicit constexpr fast_mod_div_u32_t(uint32_t d) : m_divisor(d) {
|
||||
if (d > 1) {
|
||||
const uint32_t log2_ceil = 32 - std::countl_zero(d - 1);
|
||||
const uint32_t p = 31 + log2_ceil;
|
||||
m_magic = static_cast<uint32_t>(((uint64_t{1} << p) + d - 1) / d);
|
||||
m_shift = p - 32;
|
||||
} else {
|
||||
m_magic = 0;
|
||||
m_shift = 0;
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
SGL_DEVICE friend uint32_t operator/(T n, const fast_mod_div_u32_t& d) {
|
||||
static_assert(std::is_unsigned_v<T> && sizeof(T) <= sizeof(uint32_t));
|
||||
return d.m_divisor == 1 ? n : __umulhi(n, d.m_magic) >> d.m_shift;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
SGL_DEVICE friend uint32_t operator%(T n, const fast_mod_div_u32_t& d) {
|
||||
static_assert(std::is_unsigned_v<T> && sizeof(T) <= sizeof(uint32_t));
|
||||
return n - (n / d) * d.m_divisor;
|
||||
}
|
||||
|
||||
constexpr operator uint32_t() const {
|
||||
return m_divisor;
|
||||
}
|
||||
|
||||
private:
|
||||
uint32_t m_divisor;
|
||||
uint32_t m_magic;
|
||||
uint32_t m_shift;
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
SGL_DEVICE uint2 div(T n, const fast_mod_div_u32_t& d) {
|
||||
return {n / d, n % d};
|
||||
}
|
||||
|
||||
} // namespace device::math
|
||||
|
||||
} // namespace sglang
|
||||
|
||||
@@ -67,14 +67,14 @@ def _run_activation_inplace(
|
||||
|
||||
@register_custom_op(mutates_args=["out"])
|
||||
def _run_activation_with_rounding_inplace(
|
||||
op_name: str, input: torch.Tensor, out: torch.Tensor
|
||||
op_name: str, input: torch.Tensor, out: torch.Tensor, clamp_limit: float = 0.0
|
||||
) -> None:
|
||||
hidden_size = input.shape[-1] // 2
|
||||
# Fast-math changes FP16 SiLU at eager rounding boundaries on SM90.
|
||||
module = activation_module(input.dtype, fast_math=False)
|
||||
input_2d = input.view(-1, hidden_size * 2)
|
||||
out_2d = out.view(-1, hidden_size)
|
||||
module.run_activation_with_rounding(input_2d, out_2d, op_name)
|
||||
module.run_activation_with_rounding(input_2d, out_2d, op_name, clamp_limit)
|
||||
|
||||
|
||||
@register_custom_op(mutates_args=["input"])
|
||||
@@ -184,11 +184,13 @@ def silu_and_mul(
|
||||
def silu_and_mul_with_activation_rounding(
|
||||
input: torch.Tensor,
|
||||
out: Optional[torch.Tensor] = None,
|
||||
*,
|
||||
clamp_limit: float = 0.0,
|
||||
) -> torch.Tensor:
|
||||
hidden_size = input.shape[-1] // 2
|
||||
if out is None:
|
||||
out = input.new_empty(*input.shape[:-1], hidden_size)
|
||||
_run_activation_with_rounding_inplace("silu", input, out)
|
||||
_run_activation_with_rounding_inplace("silu", input, out, clamp_limit)
|
||||
return out
|
||||
|
||||
|
||||
|
||||
@@ -184,6 +184,7 @@ for _mod, _fn in [
|
||||
("dsa.cp_split", "dsa_cp_interleave_q_seqs_kernel"),
|
||||
("dsv4.fp4_indexer", "quantize_fp4_indexer_tensor"),
|
||||
("dsv4.fp4_indexer", "store_fp4_index_k_cache"),
|
||||
("dsv4.fp4_indexer_prefill", "fused_index_scores"),
|
||||
]:
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
@@ -359,3 +360,14 @@ register_kernel(
|
||||
capabilities=frozenset({CapabilityRequirement.CUDA}),
|
||||
)
|
||||
)
|
||||
|
||||
for _fn in ("fp4_index_logits_paged", "finish_paged_indexer_topk"):
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op=f"attention.{_fn}",
|
||||
backend=KernelBackend.TRITON,
|
||||
target=f"sglang.kernels.ops.attention.dsv4.fp4_indexer:{_fn}",
|
||||
capabilities=frozenset({CapabilityRequirement.CUDA}),
|
||||
)
|
||||
)
|
||||
del _fn
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
"""Direct-candidate prefill scorer; Torch remains the independent reference.
|
||||
|
||||
Read candidates from shared, already dequantized K. Neither gathered K nor
|
||||
per-head scores are materialized in global memory. Top-K stays with the caller.
|
||||
"""
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _candidate_bf16_mqa(
|
||||
Q,
|
||||
K,
|
||||
W,
|
||||
BLOCKS,
|
||||
LENS,
|
||||
OUT,
|
||||
LENGTH: tl.constexpr,
|
||||
WIDTH: tl.constexpr,
|
||||
QR: tl.constexpr,
|
||||
QH: tl.constexpr,
|
||||
QD: tl.constexpr,
|
||||
KR: tl.constexpr,
|
||||
KD: tl.constexpr,
|
||||
WR: tl.constexpr,
|
||||
WH: tl.constexpr,
|
||||
BR: tl.constexpr,
|
||||
BC: tl.constexpr,
|
||||
LS: tl.constexpr,
|
||||
TILE: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
col = tl.program_id(1) * TILE + tl.arange(0, TILE)
|
||||
head = tl.arange(0, 32)
|
||||
dim = tl.arange(0, 128)
|
||||
block = tl.load(BLOCKS + row * BR + (col // 8) * BC, mask=col < WIDTH, other=-1).to(
|
||||
tl.int64
|
||||
)
|
||||
pos = block * 8 + col % 8
|
||||
visible = tl.minimum(tl.load(LENS + row * LS), LENGTH)
|
||||
valid = (col < WIDTH) & (pos >= 0) & (pos < visible)
|
||||
# Masked K loads also protect padded/invalid candidate addresses.
|
||||
keys = tl.load(
|
||||
K + dim[:, None] * KD + pos[None, :] * KR, mask=valid[None, :], other=0
|
||||
)
|
||||
query = tl.load(Q + row * QR + head[:, None] * QH + dim[None, :] * QD)
|
||||
dot = tl.dot(query, keys).to(tl.bfloat16).to(tl.float32)
|
||||
weight = tl.load(W + row * WR + head * WH).to(tl.float32)
|
||||
weighted = (tl.maximum(dot, 0.0) * weight[:, None]).to(tl.bfloat16)
|
||||
# Preserve Torch's BF16 output rounding after multiplication and head sum.
|
||||
score = tl.sum(weighted.to(tl.float32), axis=0).to(tl.bfloat16).to(tl.float32)
|
||||
tl.store(
|
||||
OUT + row * WIDTH + col, tl.where(valid, score, -float("inf")), mask=col < WIDTH
|
||||
)
|
||||
|
||||
|
||||
def can_use_candidate_bf16_mqa(q, keys, weights, block_size):
|
||||
"""Whether the operands fit the compact prefill scorer; otherwise use Torch."""
|
||||
return (
|
||||
q.is_cuda
|
||||
and torch.version.cuda is not None
|
||||
and q.ndim == 3
|
||||
and q.shape[1:] == (32, 128)
|
||||
and keys.ndim == 2
|
||||
and keys.shape[1] == 128
|
||||
and weights.shape == q.shape[:2]
|
||||
and q.dtype == keys.dtype == weights.dtype == torch.bfloat16
|
||||
and q.device == keys.device == weights.device
|
||||
and block_size == 8
|
||||
)
|
||||
|
||||
|
||||
def candidate_bf16_mqa_logits(q, keys, weights, blocks, visible_lens):
|
||||
"""Score candidate blocks without a per-query K copy.
|
||||
|
||||
q [R,32,128], keys [L,128], weights [R,32] are BF16. blocks [R,C]
|
||||
contains logical block IDs (8 positions/block), with -1 padding.
|
||||
visible_lens [R] is exclusive causal visibility in the same K coordinates.
|
||||
Return FP32 [R,C*8], with -inf for invalid positions. Candidate order is
|
||||
preserved, so the caller can reuse existing Top-K and index restoration.
|
||||
Only finite model activations are supported; this is not a quantizer.
|
||||
"""
|
||||
if q.ndim != 3 or q.shape[1:] != (32, 128):
|
||||
raise ValueError("q must have shape [rows, 32, 128]")
|
||||
rows = q.shape[0]
|
||||
if keys.ndim != 2 or keys.shape[1] != 128:
|
||||
raise ValueError("keys must have shape [length, 128]")
|
||||
if weights.shape != (rows, 32):
|
||||
raise ValueError("weights must have shape [rows, 32]")
|
||||
if blocks.ndim != 2 or blocks.shape[0] != rows:
|
||||
raise ValueError("blocks must have shape [rows, candidate_blocks]")
|
||||
if visible_lens.shape != (rows,):
|
||||
raise ValueError("visible_lens must have shape [rows]")
|
||||
if any(t.dtype != torch.bfloat16 for t in (q, keys, weights)):
|
||||
raise ValueError("q, keys and weights must be BF16")
|
||||
if any(t.dtype not in (torch.int32, torch.int64) for t in (blocks, visible_lens)):
|
||||
raise ValueError("blocks and visible_lens must be int32 or int64")
|
||||
if not q.is_cuda or any(
|
||||
t.device != q.device for t in (keys, weights, blocks, visible_lens)
|
||||
):
|
||||
raise ValueError("all inputs must be on the same CUDA device")
|
||||
width = blocks.shape[1] * 8
|
||||
out = torch.empty((rows, width), dtype=torch.float32, device=q.device)
|
||||
if not rows or not width:
|
||||
return out
|
||||
if not keys.shape[0]:
|
||||
return out.fill_(-torch.inf)
|
||||
_candidate_bf16_mqa[(rows, triton.cdiv(width, 64))](
|
||||
q,
|
||||
keys,
|
||||
weights,
|
||||
blocks,
|
||||
visible_lens,
|
||||
out,
|
||||
keys.shape[0],
|
||||
width,
|
||||
*q.stride(),
|
||||
*keys.stride(),
|
||||
*weights.stride(),
|
||||
*blocks.stride(),
|
||||
visible_lens.stride(0),
|
||||
TILE=64,
|
||||
num_warps=4,
|
||||
enable_fp_fusion=False,
|
||||
)
|
||||
return out
|
||||
@@ -143,6 +143,18 @@ def amax_topk_blocks(
|
||||
return blocks
|
||||
|
||||
|
||||
def candidate_block_mask(
|
||||
blocks: torch.Tensor, width: int, block_size: int
|
||||
) -> torch.Tensor:
|
||||
"""Materialize a source's block IDs once for Hopper paged-score consumers."""
|
||||
num_blocks = (width + block_size - 1) // block_size
|
||||
keep = torch.zeros(
|
||||
(*blocks.shape[:-1], num_blocks + 1), dtype=torch.bool, device=blocks.device
|
||||
)
|
||||
keep.scatter_(-1, blocks.to(torch.int64).masked_fill(blocks < 0, num_blocks), True)
|
||||
return keep[..., :num_blocks].repeat_interleave(block_size, dim=-1)[..., :width]
|
||||
|
||||
|
||||
def select_candidate_block_ids(
|
||||
logits: torch.Tensor,
|
||||
compress_lens: Union[torch.Tensor, int],
|
||||
|
||||
@@ -37,9 +37,10 @@ def _jit_main_q_norm_rope_module(
|
||||
dtype: torch.dtype,
|
||||
head_dim: int,
|
||||
rope_dim: int,
|
||||
apply_norm: bool,
|
||||
):
|
||||
"""Main MLA path Q kernel: rmsnorm-self + RoPE, warp per (token, head)."""
|
||||
args = make_cpp_args(dtype, head_dim, rope_dim, is_arch_support_pdl())
|
||||
args = make_cpp_args(dtype, head_dim, rope_dim, apply_norm, is_arch_support_pdl())
|
||||
return load_jit(
|
||||
make_name("main_q_norm_rope"),
|
||||
*args,
|
||||
@@ -170,7 +171,7 @@ def fused_rope_inplace(
|
||||
def fused_q_norm_rope(
|
||||
q_input: torch.Tensor,
|
||||
q_output: torch.Tensor,
|
||||
eps: float,
|
||||
eps: Optional[float],
|
||||
freqs_cis: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
) -> None:
|
||||
@@ -178,10 +179,13 @@ def fused_q_norm_rope(
|
||||
head_dim = q_input.shape[-1]
|
||||
rope_dim = freqs_real.shape[-1]
|
||||
if _is_xpu:
|
||||
assert eps is not None
|
||||
fused_q_norm_rope_xpu(q_input, q_output, freqs_real, positions, eps)
|
||||
else:
|
||||
module = _jit_main_q_norm_rope_module(q_input.dtype, head_dim, rope_dim)
|
||||
module.forward(q_input, q_output, freqs_real, positions, eps)
|
||||
module = _jit_main_q_norm_rope_module(
|
||||
q_input.dtype, head_dim, rope_dim, eps is not None
|
||||
)
|
||||
module.forward(q_input, q_output, freqs_real, positions, eps or 0.0)
|
||||
|
||||
|
||||
def fused_q_indexer_rope_hadamard_quant(
|
||||
|
||||
@@ -15,6 +15,208 @@ INDEX_K_SCALE_BYTES = tl.constexpr(4)
|
||||
INDEX_K_SLOT_BYTES = INDEX_K_PAYLOAD_BYTES.value + INDEX_K_SCALE_BYTES.value
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _finish_paged_indexer_topk_kernel(
|
||||
Indices,
|
||||
Scores,
|
||||
Lengths,
|
||||
Requests,
|
||||
ReqTable,
|
||||
Pages,
|
||||
Raw,
|
||||
Blocks,
|
||||
OutputRows,
|
||||
HAS_OUTPUT_ROWS: tl.constexpr,
|
||||
WRITE_PAGES: tl.constexpr,
|
||||
BLOCK_STRIDE: tl.constexpr,
|
||||
HAS_CANDIDATES: tl.constexpr,
|
||||
BATCH: tl.constexpr,
|
||||
K: tl.constexpr,
|
||||
WIDTH: tl.constexpr,
|
||||
OUT_WIDTH: tl.constexpr,
|
||||
IDX_STRIDE: tl.constexpr,
|
||||
SCORE_STRIDE: tl.constexpr,
|
||||
TABLE_STRIDE: tl.constexpr,
|
||||
PAGE_STRIDE: tl.constexpr,
|
||||
RAW_STRIDE: tl.constexpr,
|
||||
RATIO: tl.constexpr,
|
||||
MASK_SCORES: tl.constexpr,
|
||||
WRITE_RAW: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
col = tl.arange(0, BLOCK)
|
||||
idx = tl.load(Indices + row * IDX_STRIDE + col, (row < BATCH) & (col < K), WIDTH)
|
||||
if MASK_SCORES:
|
||||
valid = (row < BATCH) & (col < K) & (idx >= 0) & (idx < WIDTH)
|
||||
score = tl.load(Scores + row * SCORE_STRIDE + idx, valid, -float("inf"))
|
||||
idx = tl.where(valid & (score > -float("inf")), idx, WIDTH)
|
||||
if HAS_CANDIDATES:
|
||||
valid = (row < BATCH) & (col < K) & (idx >= 0) & (idx < WIDTH)
|
||||
block = tl.load(Blocks + row * BLOCK_STRIDE + idx // 8, valid, -1)
|
||||
# Sort logical positions, not the unordered candidate-table columns.
|
||||
idx = tl.where(valid & (block >= 0), block * 8 + idx % 8, 0x7FFFFFFF)
|
||||
idx = tl.sort(tl.where(idx >= 0, idx, WIDTH), descending=False)
|
||||
length = tl.load(Lengths + row, row < BATCH, 0)
|
||||
output_row = tl.load(OutputRows + row) if HAS_OUTPUT_ROWS else row
|
||||
valid = (row < BATCH) & (col < K) & (idx < length)
|
||||
if WRITE_PAGES:
|
||||
req = tl.load(Requests + row, row < BATCH, 0)
|
||||
slot = tl.load(
|
||||
ReqTable + req.to(tl.int64) * TABLE_STRIDE + idx.to(tl.int64) * RATIO,
|
||||
valid,
|
||||
0,
|
||||
)
|
||||
tl.store(
|
||||
Pages + output_row * PAGE_STRIDE + col,
|
||||
tl.where(valid, slot.to(tl.int64) // RATIO, -1),
|
||||
col < OUT_WIDTH,
|
||||
)
|
||||
if WRITE_RAW:
|
||||
tl.store(
|
||||
Raw + output_row * RAW_STRIDE + col,
|
||||
tl.where(valid, idx, -1),
|
||||
col < OUT_WIDTH,
|
||||
)
|
||||
|
||||
|
||||
def finish_paged_indexer_topk(
|
||||
indices: torch.Tensor,
|
||||
scores: torch.Tensor,
|
||||
lengths: torch.Tensor,
|
||||
req: torch.Tensor | None,
|
||||
req_table: torch.Tensor | None,
|
||||
page_indices: torch.Tensor | None,
|
||||
raw_indices: torch.Tensor | None,
|
||||
ratio: int,
|
||||
mask_scores: bool,
|
||||
*,
|
||||
candidate_blocks: torch.Tensor | None = None,
|
||||
output_rows: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
"""Sort selected positions and map them to compressed KV slots.
|
||||
|
||||
Candidate consumers discard selected masked scores, including top-k underfill.
|
||||
With output_rows, write only those rows (unique, in-range destination indices).
|
||||
Otherwise write every output row, including padded decode rows. Unselected
|
||||
columns are -1. Raw-only prefill does not read the request table.
|
||||
"""
|
||||
if candidate_blocks is not None:
|
||||
assert candidate_blocks.shape == (lengths.numel(), scores.shape[1] // 8)
|
||||
assert candidate_blocks.stride(1) == 1
|
||||
out = page_indices if page_indices is not None else raw_indices
|
||||
assert out is not None
|
||||
if page_indices is not None:
|
||||
assert req is not None and req_table is not None
|
||||
if raw_indices is not None:
|
||||
assert raw_indices.shape == page_indices.shape
|
||||
if output_rows is not None:
|
||||
assert output_rows.shape == (lengths.numel(),) and output_rows.is_contiguous()
|
||||
rows = out.shape[0] if output_rows is None else output_rows.numel()
|
||||
if not rows or not out.shape[1]:
|
||||
return
|
||||
_finish_paged_indexer_topk_kernel[(rows,)](
|
||||
indices,
|
||||
scores,
|
||||
lengths,
|
||||
req,
|
||||
req_table,
|
||||
page_indices,
|
||||
raw_indices,
|
||||
candidate_blocks,
|
||||
output_rows,
|
||||
output_rows is not None,
|
||||
page_indices is not None,
|
||||
candidate_blocks.stride(0) if candidate_blocks is not None else 0,
|
||||
candidate_blocks is not None,
|
||||
lengths.numel(),
|
||||
indices.shape[1],
|
||||
scores.shape[1],
|
||||
out.shape[1],
|
||||
indices.stride(0),
|
||||
scores.stride(0),
|
||||
req_table.stride(0) if req_table is not None else 0,
|
||||
page_indices.stride(0) if page_indices is not None else 0,
|
||||
raw_indices.stride(0) if raw_indices is not None else 0,
|
||||
ratio,
|
||||
mask_scores or candidate_blocks is not None,
|
||||
raw_indices is not None,
|
||||
triton.next_power_of_2(max(out.shape[1], indices.shape[1])),
|
||||
num_warps=4,
|
||||
)
|
||||
|
||||
|
||||
# Visible key counts change across batches; keep them out of the JIT key.
|
||||
@triton.jit(do_not_specialize=["NUM_SLOTS"])
|
||||
def _finish_flat_indexer_topk_kernel(
|
||||
Indices,
|
||||
Slots,
|
||||
Starts,
|
||||
Pages,
|
||||
Raw,
|
||||
K: tl.constexpr,
|
||||
NUM_SLOTS,
|
||||
IDX_STRIDE: tl.constexpr,
|
||||
PAGE_STRIDE: tl.constexpr,
|
||||
RAW_STRIDE: tl.constexpr,
|
||||
WRITE_RAW: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
col = tl.arange(0, BLOCK)
|
||||
unselected = 2147483647
|
||||
idx = tl.load(Indices + row * IDX_STRIDE + col, col < K, unselected)
|
||||
idx = tl.sort(tl.where(idx < 0, unselected, idx), descending=False)
|
||||
chosen = (col < K) & (idx != unselected)
|
||||
slot = tl.load(Slots + tl.minimum(idx, NUM_SLOTS - 1), chosen, -1)
|
||||
tl.store(Pages + row * PAGE_STRIDE + col, slot, col < K)
|
||||
if WRITE_RAW:
|
||||
start = tl.load(Starts + row)
|
||||
tl.store(
|
||||
Raw + row * RAW_STRIDE + col, tl.where(chosen, idx - start, -1), col < K
|
||||
)
|
||||
|
||||
|
||||
def finish_flat_indexer_topk(
|
||||
indices: torch.Tensor,
|
||||
slots: torch.Tensor,
|
||||
request_starts: torch.Tensor,
|
||||
page_indices: torch.Tensor,
|
||||
raw_indices: torch.Tensor | None,
|
||||
) -> None:
|
||||
"""Sort flattened prefill positions and map them to KV slots in one kernel.
|
||||
|
||||
Negative positions and INT32_MAX are unselected. Write only the selected
|
||||
rows and columns; callers retain responsibility for resetting output padding.
|
||||
"""
|
||||
rows, k = indices.shape
|
||||
if not rows or not k:
|
||||
return
|
||||
assert indices.dtype == torch.int32 and indices.stride(1) == 1
|
||||
assert slots.ndim == 1 and slots.is_contiguous() and slots.numel() > 0
|
||||
assert request_starts.shape == (rows,) and request_starts.is_contiguous()
|
||||
assert page_indices.shape[0] >= rows and page_indices.shape[1] >= k
|
||||
assert page_indices.dtype == torch.int32 and page_indices.stride(1) == 1
|
||||
if raw_indices is not None:
|
||||
assert raw_indices.shape[0] >= rows and raw_indices.shape[1] >= k
|
||||
assert raw_indices.dtype == torch.int32 and raw_indices.stride(1) == 1
|
||||
_finish_flat_indexer_topk_kernel[(rows,)](
|
||||
indices,
|
||||
slots,
|
||||
request_starts,
|
||||
page_indices,
|
||||
raw_indices,
|
||||
k,
|
||||
slots.numel(),
|
||||
indices.stride(0),
|
||||
page_indices.stride(0),
|
||||
raw_indices.stride(0) if raw_indices is not None else 0,
|
||||
raw_indices is not None,
|
||||
triton.next_power_of_2(k),
|
||||
num_warps=4,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _select_group_value(group, v0, v1, v2, v3):
|
||||
return tl.where(
|
||||
@@ -297,6 +499,67 @@ def store_fp4_index_k_cache(
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _gather_fp4_index_k_kernel(
|
||||
cache,
|
||||
cache_stride,
|
||||
slots,
|
||||
payload,
|
||||
scales,
|
||||
n,
|
||||
page_size: tl.constexpr,
|
||||
BLOCK_ROWS: tl.constexpr,
|
||||
):
|
||||
# int32 view of a page: page_size * 16 payload words, then page_size scale words.
|
||||
rows = tl.program_id(0) * BLOCK_ROWS + tl.arange(0, BLOCK_ROWS)
|
||||
valid = rows < n
|
||||
slot = tl.load(slots + rows, valid, 0).to(tl.int64)
|
||||
page = slot // page_size
|
||||
offset = slot - page * page_size
|
||||
page_base = cache + page * cache_stride
|
||||
words = tl.arange(0, 16)
|
||||
payload_words = tl.load(
|
||||
page_base[:, None] + offset[:, None] * 16 + words[None, :], valid[:, None]
|
||||
)
|
||||
tl.store(
|
||||
payload + rows[:, None].to(tl.int64) * 16 + words[None, :],
|
||||
payload_words,
|
||||
valid[:, None],
|
||||
)
|
||||
tl.store(scales + rows, tl.load(page_base + page_size * 16 + offset, valid), valid)
|
||||
|
||||
|
||||
def gather_fp4_index_k(
|
||||
cache: torch.Tensor, slots: torch.Tensor, *, page_size: int
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Packed fp4 rows at ``slots`` of the index-K ``cache`` (``[pages, page_size *
|
||||
68]`` uint8, ``store_fp4_index_k_cache``'s layout): (payload int8 ``[n, 64]``,
|
||||
ue8m0 scales packed int32 ``[n]``), the layout ``quantize_fp4_indexer_tensor``
|
||||
returns."""
|
||||
assert cache.shape[1] == page_size * INDEX_K_SLOT_BYTES and cache.stride(1) == 1
|
||||
# The kernel addresses slots as `slots + rows`, so a strided view would be read
|
||||
# as if its elements were packed.
|
||||
assert slots.is_contiguous()
|
||||
words = cache.view(torch.int32)
|
||||
n = slots.shape[0]
|
||||
payload = torch.empty((n, 16), device=cache.device, dtype=torch.int32)
|
||||
scales = torch.empty((n,), device=cache.device, dtype=torch.int32)
|
||||
if n > 0:
|
||||
block_rows = 64
|
||||
_gather_fp4_index_k_kernel[(triton.cdiv(n, block_rows),)](
|
||||
words,
|
||||
words.stride(0),
|
||||
slots,
|
||||
payload,
|
||||
scales,
|
||||
n,
|
||||
page_size,
|
||||
BLOCK_ROWS=block_rows,
|
||||
num_warps=4,
|
||||
)
|
||||
return payload.view(torch.int8), scales
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _index_k_rope_pack_kernel(
|
||||
X,
|
||||
@@ -430,6 +693,73 @@ def _e2m1_decode(code):
|
||||
return tl.where((code >> 3) == 1, -v, v)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _fp4_index_logits_tile(
|
||||
q_ptr,
|
||||
w_ptr,
|
||||
table_ptr,
|
||||
b,
|
||||
slot,
|
||||
valid,
|
||||
page_size,
|
||||
row_stride,
|
||||
stride_qb,
|
||||
stride_qh,
|
||||
stride_wb,
|
||||
H: tl.constexpr,
|
||||
HALF_D: tl.constexpr,
|
||||
):
|
||||
offs_h = tl.arange(0, H)
|
||||
offs_i = tl.arange(0, HALF_D)
|
||||
page = slot // page_size
|
||||
off = slot % page_size
|
||||
row_base = page * row_stride
|
||||
# K payload: [BLOCK_L, HALF_D] uint8
|
||||
pay = tl.load(
|
||||
table_ptr
|
||||
+ row_base[:, None]
|
||||
+ off[:, None] * INDEX_K_PAYLOAD_BYTES
|
||||
+ offs_i[None, :],
|
||||
mask=valid[:, None],
|
||||
other=0,
|
||||
)
|
||||
low = _e2m1_decode(pay & 0x0F)
|
||||
high = _e2m1_decode((pay >> 4) & 0x0F)
|
||||
# e8m0 block scales: element j uses block j // 32 -> byte i uses block i // 16.
|
||||
sc_idx = offs_i // 16
|
||||
exps = tl.load(
|
||||
table_ptr
|
||||
+ row_base[:, None]
|
||||
+ page_size * INDEX_K_PAYLOAD_BYTES
|
||||
+ off[:, None] * INDEX_K_SCALE_BYTES
|
||||
+ sc_idx[None, :],
|
||||
mask=valid[:, None],
|
||||
other=127,
|
||||
)
|
||||
scale = tl.exp2(exps.to(tl.float32) - 127.0)
|
||||
k_low = (low * scale).to(tl.bfloat16) # [BLOCK_L, HALF_D] elements 2i
|
||||
k_high = (high * scale).to(tl.bfloat16) # elements 2i+1
|
||||
|
||||
# queries: even / odd elements, [H, HALF_D] bf16
|
||||
q_even = tl.load(
|
||||
q_ptr + b * stride_qb + offs_h[:, None] * stride_qh + 2 * offs_i[None, :]
|
||||
)
|
||||
q_odd = tl.load(
|
||||
q_ptr + b * stride_qb + offs_h[:, None] * stride_qh + 2 * offs_i[None, :] + 1
|
||||
)
|
||||
|
||||
acc = tl.dot(q_even, tl.trans(k_low)) # [H, BLOCK_L] fp32
|
||||
acc += tl.dot(q_odd, tl.trans(k_high))
|
||||
# reference rounding points: bf16 dot -> relu -> * bf16 weight -> bf16 -> sum -> bf16
|
||||
s = acc.to(tl.bfloat16).to(tl.float32)
|
||||
s = tl.maximum(s, 0.0)
|
||||
w = tl.load(w_ptr + b * stride_wb + offs_h).to(tl.float32)
|
||||
s = (s * w[:, None]).to(tl.bfloat16).to(tl.float32)
|
||||
logit = tl.sum(s, axis=0).to(tl.bfloat16).to(tl.float32)
|
||||
logit = tl.where(valid, logit, float("-inf"))
|
||||
return logit
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _fp4_index_logits_kernel(
|
||||
q_ptr, # [B, H, D] bf16, fq4 queries (already rope'd)
|
||||
@@ -456,11 +786,6 @@ def _fp4_index_logits_kernel(
|
||||
b = tl.program_id(0)
|
||||
lb = tl.program_id(1)
|
||||
offs_l = lb * BLOCK_L + tl.arange(0, BLOCK_L)
|
||||
offs_h = tl.arange(0, H)
|
||||
offs_i = tl.arange(
|
||||
0, HALF_D
|
||||
) # byte index i holds elements 2i (low nibble), 2i+1 (high nibble)
|
||||
|
||||
n_vis = tl.load(lens_ptr + b)
|
||||
# Graph replay keeps the capacity-sized grid even for short live contexts.
|
||||
# Skip whole invisible tiles using the current device-side length; masking
|
||||
@@ -472,57 +797,21 @@ def _fp4_index_logits_kernel(
|
||||
slot = tl.load(slots_ptr + b * L + offs_l, mask=offs_l < L, other=0).to(
|
||||
tl.int64
|
||||
)
|
||||
page = slot // page_size
|
||||
off = slot % page_size
|
||||
row_base = page * row_stride
|
||||
|
||||
# K payload: [BLOCK_L, HALF_D] uint8
|
||||
pay = tl.load(
|
||||
table_ptr
|
||||
+ row_base[:, None]
|
||||
+ off[:, None] * INDEX_K_PAYLOAD_BYTES
|
||||
+ offs_i[None, :],
|
||||
mask=valid[:, None],
|
||||
other=0,
|
||||
logit = _fp4_index_logits_tile(
|
||||
q_ptr,
|
||||
w_ptr,
|
||||
table_ptr,
|
||||
b,
|
||||
slot,
|
||||
valid,
|
||||
page_size,
|
||||
row_stride,
|
||||
stride_qb,
|
||||
stride_qh,
|
||||
stride_wb,
|
||||
H,
|
||||
HALF_D,
|
||||
)
|
||||
low = _e2m1_decode(pay & 0x0F)
|
||||
high = _e2m1_decode((pay >> 4) & 0x0F)
|
||||
# e8m0 block scales: element j uses block j // 32 -> byte i uses block i // 16.
|
||||
sc_idx = offs_i // 16
|
||||
exps = tl.load(
|
||||
table_ptr
|
||||
+ row_base[:, None]
|
||||
+ page_size * INDEX_K_PAYLOAD_BYTES
|
||||
+ off[:, None] * INDEX_K_SCALE_BYTES
|
||||
+ sc_idx[None, :],
|
||||
mask=valid[:, None],
|
||||
other=127,
|
||||
)
|
||||
scale = tl.exp2(exps.to(tl.float32) - 127.0)
|
||||
k_low = (low * scale).to(tl.bfloat16) # [BLOCK_L, HALF_D] elements 2i
|
||||
k_high = (high * scale).to(tl.bfloat16) # elements 2i+1
|
||||
|
||||
# queries: even / odd elements, [H, HALF_D] bf16
|
||||
q_even = tl.load(
|
||||
q_ptr + b * stride_qb + offs_h[:, None] * stride_qh + 2 * offs_i[None, :]
|
||||
)
|
||||
q_odd = tl.load(
|
||||
q_ptr
|
||||
+ b * stride_qb
|
||||
+ offs_h[:, None] * stride_qh
|
||||
+ 2 * offs_i[None, :]
|
||||
+ 1
|
||||
)
|
||||
|
||||
acc = tl.dot(q_even, tl.trans(k_low)) # [H, BLOCK_L] fp32
|
||||
acc += tl.dot(q_odd, tl.trans(k_high))
|
||||
# reference rounding points: bf16 dot -> relu -> * bf16 weight -> bf16 -> sum -> bf16
|
||||
s = acc.to(tl.bfloat16).to(tl.float32)
|
||||
s = tl.maximum(s, 0.0)
|
||||
w = tl.load(w_ptr + b * stride_wb + offs_h).to(tl.float32)
|
||||
s = (s * w[:, None]).to(tl.bfloat16).to(tl.float32)
|
||||
logit = tl.sum(s, axis=0).to(tl.bfloat16).to(tl.float32)
|
||||
logit = tl.where(valid, logit, float("-inf"))
|
||||
tl.store(out_ptr + b * L + offs_l, logit, mask=offs_l < L)
|
||||
|
||||
|
||||
@@ -569,3 +858,252 @@ def fp4_index_logits_decode(
|
||||
num_warps=4,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _fp4_index_logits_paged_kernel(
|
||||
q_ptr,
|
||||
w_ptr,
|
||||
req_ptr,
|
||||
req_table_ptr,
|
||||
lens_ptr,
|
||||
table_ptr,
|
||||
mask_ptr,
|
||||
out_ptr,
|
||||
L,
|
||||
page_size,
|
||||
row_stride,
|
||||
req_stride,
|
||||
mask_stride,
|
||||
out_stride,
|
||||
stride_qb,
|
||||
stride_qh,
|
||||
stride_wb,
|
||||
H: tl.constexpr,
|
||||
HALF_D: tl.constexpr,
|
||||
RATIO: tl.constexpr,
|
||||
HAS_MASK: tl.constexpr,
|
||||
BLOCK_L: tl.constexpr,
|
||||
):
|
||||
b = tl.program_id(0)
|
||||
n_vis = tl.minimum(tl.load(lens_ptr + b), L)
|
||||
req = tl.load(req_ptr + b).to(tl.int64)
|
||||
for tile in range(tl.program_id(1), tl.cdiv(n_vis, BLOCK_L), tl.num_programs(1)):
|
||||
offs_l = tile * BLOCK_L + tl.arange(0, BLOCK_L)
|
||||
valid = offs_l < n_vis
|
||||
if HAS_MASK:
|
||||
valid = valid & tl.load(
|
||||
mask_ptr + b * mask_stride + offs_l, mask=offs_l < n_vis, other=False
|
||||
)
|
||||
slot = (
|
||||
tl.load(
|
||||
req_table_ptr + req * req_stride + offs_l * RATIO, mask=valid, other=0
|
||||
).to(tl.int64)
|
||||
// RATIO
|
||||
)
|
||||
logit = _fp4_index_logits_tile(
|
||||
q_ptr,
|
||||
w_ptr,
|
||||
table_ptr,
|
||||
b,
|
||||
slot,
|
||||
valid,
|
||||
page_size,
|
||||
row_stride,
|
||||
stride_qb,
|
||||
stride_qh,
|
||||
stride_wb,
|
||||
H,
|
||||
HALF_D,
|
||||
)
|
||||
tl.store(out_ptr + b * out_stride + offs_l, logit, mask=offs_l < L)
|
||||
|
||||
|
||||
def fp4_index_logits_paged(
|
||||
q: torch.Tensor,
|
||||
weights: torch.Tensor,
|
||||
req: torch.Tensor,
|
||||
req_table: torch.Tensor,
|
||||
lens: torch.Tensor,
|
||||
table: torch.Tensor,
|
||||
page_size: int,
|
||||
max_len: int,
|
||||
ratio: int,
|
||||
candidate_mask: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Score visible compressed positions directly from the request table.
|
||||
|
||||
The output has graph-stable capacity, but only the first ``lens[b]`` values
|
||||
are initialized. Consumers must use the current device-side lengths.
|
||||
The FP4 decoding and BF16 rounding are shared with the dense-slot path.
|
||||
"""
|
||||
assert q.dtype == torch.bfloat16 and q.shape[-1] == INDEX_HEAD_DIM
|
||||
assert table.dtype == torch.uint8 and table.ndim == 2
|
||||
assert ratio in (1, 2) and max_len <= req_table.shape[1] // ratio
|
||||
assert req_table.stride(1) == 1
|
||||
if candidate_mask is not None:
|
||||
assert candidate_mask.shape[1] >= max_len and candidate_mask.stride(1) == 1
|
||||
batch, heads, _ = q.shape
|
||||
q = q.contiguous()
|
||||
weights = weights.to(torch.bfloat16).contiguous()
|
||||
req = req.contiguous()
|
||||
lens = lens.contiguous()
|
||||
# Rows are padded to 32 floats (128 bytes): the deep_select top-k kernel
|
||||
# requires 128-byte-aligned input row strides (kInputAlignmentBytes in
|
||||
# entry.cuh); the old eight-float alignment (candidate-amax) is subsumed.
|
||||
out = torch.empty(
|
||||
(batch, triton.cdiv(max_len, 32) * 32), device=q.device, dtype=torch.float32
|
||||
)
|
||||
if not batch or not max_len:
|
||||
return out[:, :max_len]
|
||||
num_sms = torch.cuda.get_device_properties(q.device).multi_processor_count
|
||||
workers = min(triton.cdiv(max_len, 64), triton.cdiv(num_sms * 4, batch))
|
||||
_fp4_index_logits_paged_kernel[(batch, workers)](
|
||||
q,
|
||||
weights,
|
||||
req,
|
||||
req_table,
|
||||
lens,
|
||||
table,
|
||||
candidate_mask,
|
||||
out,
|
||||
max_len,
|
||||
page_size,
|
||||
table.stride(0),
|
||||
req_table.stride(0),
|
||||
candidate_mask.stride(0) if candidate_mask is not None else 0,
|
||||
out.stride(0),
|
||||
q.stride(0),
|
||||
q.stride(1),
|
||||
weights.stride(0),
|
||||
H=heads,
|
||||
HALF_D=INDEX_HEAD_DIM // 2,
|
||||
RATIO=ratio,
|
||||
HAS_MASK=candidate_mask is not None,
|
||||
BLOCK_L=64,
|
||||
num_warps=4,
|
||||
)
|
||||
return out[:, :max_len]
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _fp4_index_logits_candidates_kernel(
|
||||
Q,
|
||||
W,
|
||||
Requests,
|
||||
ReqTable,
|
||||
Lengths,
|
||||
Cache,
|
||||
Blocks,
|
||||
Out,
|
||||
WIDTH: tl.constexpr,
|
||||
OUT_STRIDE: tl.constexpr,
|
||||
MAX_LEN: tl.constexpr,
|
||||
PAGE_SIZE: tl.constexpr,
|
||||
CACHE_STRIDE: tl.constexpr,
|
||||
TABLE_STRIDE: tl.constexpr,
|
||||
BLOCK_STRIDE: tl.constexpr,
|
||||
QB: tl.constexpr,
|
||||
QH: tl.constexpr,
|
||||
WB: tl.constexpr,
|
||||
H: tl.constexpr,
|
||||
RATIO: tl.constexpr,
|
||||
TILE: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
req = tl.load(Requests + row).to(tl.int64)
|
||||
visible = tl.minimum(tl.load(Lengths + row), MAX_LEN)
|
||||
for tile in range(tl.program_id(1), tl.cdiv(WIDTH, TILE), tl.num_programs(1)):
|
||||
compact = tile * TILE + tl.arange(0, TILE)
|
||||
block = tl.load(
|
||||
Blocks + row * BLOCK_STRIDE + compact // 8, mask=compact < WIDTH, other=-1
|
||||
).to(tl.int64)
|
||||
logical = block * 8 + compact % 8
|
||||
valid = (compact < WIDTH) & (block >= 0) & (logical < visible)
|
||||
slot = (
|
||||
tl.load(
|
||||
ReqTable + req * TABLE_STRIDE + logical * RATIO, mask=valid, other=0
|
||||
).to(tl.int64)
|
||||
// RATIO
|
||||
)
|
||||
score = _fp4_index_logits_tile(
|
||||
Q,
|
||||
W,
|
||||
Cache,
|
||||
row,
|
||||
slot,
|
||||
valid,
|
||||
PAGE_SIZE,
|
||||
CACHE_STRIDE,
|
||||
QB,
|
||||
QH,
|
||||
WB,
|
||||
H,
|
||||
64,
|
||||
)
|
||||
tl.store(Out + row * OUT_STRIDE + compact, score, mask=compact < WIDTH)
|
||||
|
||||
|
||||
def fp4_index_logits_candidates(
|
||||
q: torch.Tensor,
|
||||
weights: torch.Tensor,
|
||||
req: torch.Tensor,
|
||||
req_table: torch.Tensor,
|
||||
lens: torch.Tensor,
|
||||
table: torch.Tensor,
|
||||
page_size: int,
|
||||
max_len: int,
|
||||
ratio: int,
|
||||
blocks: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Paged FP4 scores in candidate order, [rows, blocks_per_row * 8].
|
||||
|
||||
Read the request table directly, as in the dense paged scorer. Padding and
|
||||
causal-invisible candidates produce -inf; no full-width mask or slot map.
|
||||
"""
|
||||
assert q.dtype == torch.bfloat16 and q.shape[-1] == INDEX_HEAD_DIM
|
||||
assert table.dtype == torch.uint8 and table.ndim == 2
|
||||
assert ratio in (1, 2) and max_len <= req_table.shape[1] // ratio
|
||||
assert req_table.stride(1) == 1
|
||||
batch, heads, _ = q.shape
|
||||
assert blocks.ndim == 2 and blocks.shape[0] == batch
|
||||
assert blocks.dtype in (torch.int32, torch.int64)
|
||||
assert req.shape == lens.shape == (batch,)
|
||||
assert weights.shape == (batch, heads)
|
||||
q = q.contiguous()
|
||||
weights = weights.to(torch.bfloat16).contiguous()
|
||||
req, lens, blocks = req.contiguous(), lens.contiguous(), blocks.contiguous()
|
||||
width = blocks.shape[1] * 8
|
||||
# Pad rows to 32 floats (128 bytes): the deep_select top-k kernel requires
|
||||
# 128-byte-aligned input row strides (kInputAlignmentBytes in entry.cuh).
|
||||
out_stride = triton.cdiv(width, 32) * 32
|
||||
out = torch.empty((batch, out_stride), device=q.device, dtype=torch.float32)
|
||||
if not batch or not width:
|
||||
return out[:, :width]
|
||||
num_sms = torch.cuda.get_device_properties(q.device).multi_processor_count
|
||||
workers = min(triton.cdiv(width, 64), triton.cdiv(num_sms * 4, batch))
|
||||
_fp4_index_logits_candidates_kernel[(batch, workers)](
|
||||
q,
|
||||
weights,
|
||||
req,
|
||||
req_table,
|
||||
lens,
|
||||
table,
|
||||
blocks,
|
||||
out,
|
||||
width,
|
||||
out_stride,
|
||||
max_len,
|
||||
page_size,
|
||||
table.stride(0),
|
||||
req_table.stride(0),
|
||||
blocks.stride(0),
|
||||
q.stride(0),
|
||||
q.stride(1),
|
||||
weights.stride(0),
|
||||
heads,
|
||||
ratio,
|
||||
64,
|
||||
num_warps=4,
|
||||
)
|
||||
return out[:, :width]
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
"""Prefill-time low-ratio indexer scores for Hopper.
|
||||
|
||||
The torch prefill path scores a chunk of query rows against the request's
|
||||
dequantized index K as `einsum -> relu -> * weights -> sum over heads`, which
|
||||
materializes a bf16 [rows, heads, lc] tensor and streams it through HBM several
|
||||
times. This Triton kernel keeps the head reduction in a [BLOCK_M, BLOCK_N]
|
||||
register tile and writes only the fp32 [rows, lc] result, with the length mask
|
||||
already applied.
|
||||
|
||||
Numerics follow the torch path and the decode kernel in fp4_indexer.py
|
||||
(bf16 dot, bf16 relu/weight product, bf16 head reduction), so prefill and decode
|
||||
select the same positions; only the head summation order differs.
|
||||
"""
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.kernels.ops.attention.dsv4.fp4_indexer import INDEX_HEAD_DIM
|
||||
|
||||
# Measured best on H200 among 7 swept tiles. The grid walks query-row blocks
|
||||
# outermost so a row block's queries (BLOCK_M * heads * d bf16) stay in L2
|
||||
# while the index K is streamed past them.
|
||||
_CONFIG = dict(BLOCK_M=64, BLOCK_N=128, num_warps=4, num_stages=2)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _index_scores_kernel(
|
||||
q_ptr, # [M, H, D] bf16
|
||||
k_ptr, # [N, D] bf16
|
||||
w_ptr, # [M, H] bf16
|
||||
lens_ptr, # [M] int, visible positions per row
|
||||
out_ptr, # [M, N] fp32, -inf at positions >= lens
|
||||
M,
|
||||
N,
|
||||
stride_qm,
|
||||
stride_qh,
|
||||
stride_kn,
|
||||
stride_wm,
|
||||
stride_om,
|
||||
H: tl.constexpr,
|
||||
D: tl.constexpr,
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
):
|
||||
pid = tl.program_id(0)
|
||||
n_blocks = tl.cdiv(N, BLOCK_N)
|
||||
pid_m = pid // n_blocks
|
||||
pid_n = pid % n_blocks
|
||||
offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)
|
||||
offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
|
||||
offs_d = tl.arange(0, D)
|
||||
mask_m = offs_m < M
|
||||
mask_n = offs_n < N
|
||||
|
||||
k = tl.load(
|
||||
k_ptr + offs_n[:, None].to(tl.int64) * stride_kn + offs_d[None, :],
|
||||
mask=mask_n[:, None],
|
||||
other=0.0,
|
||||
)
|
||||
k_t = tl.trans(k)
|
||||
q_rows = q_ptr + offs_m[:, None].to(tl.int64) * stride_qm + offs_d[None, :]
|
||||
w_rows = w_ptr + offs_m.to(tl.int64) * stride_wm
|
||||
acc = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
|
||||
for h in range(H):
|
||||
q = tl.load(q_rows + h * stride_qh, mask=mask_m[:, None], other=0.0)
|
||||
w = tl.load(w_rows + h, mask=mask_m, other=0.0).to(tl.float32)
|
||||
# reference rounding points, as in the decode kernel:
|
||||
# bf16 dot -> relu -> * bf16 weight -> bf16 -> sum -> bf16
|
||||
s = tl.dot(q, k_t).to(tl.bfloat16).to(tl.float32)
|
||||
acc += (tl.maximum(s, 0.0) * w[:, None]).to(tl.bfloat16).to(tl.float32)
|
||||
acc = acc.to(tl.bfloat16).to(tl.float32)
|
||||
|
||||
lens = tl.load(lens_ptr + offs_m, mask=mask_m, other=0)
|
||||
acc = tl.where(offs_n[None, :] < lens[:, None], acc, float("-inf"))
|
||||
tl.store(
|
||||
out_ptr + offs_m[:, None].to(tl.int64) * stride_om + offs_n[None, :],
|
||||
acc,
|
||||
mask=mask_m[:, None] & mask_n[None, :],
|
||||
)
|
||||
|
||||
|
||||
def fused_index_scores(
|
||||
q: torch.Tensor, k: torch.Tensor, weights: torch.Tensor, lens: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
"""q [t, H, d] bf16, k [n, d] bf16, weights [t, H] bf16, lens [t] int ->
|
||||
[t, n] fp32: sum over heads of relu(q_h k^T) * w_h, -inf at positions >= lens."""
|
||||
t, heads, head_dim = q.shape
|
||||
n = k.shape[0]
|
||||
assert q.dtype == k.dtype == torch.bfloat16 and head_dim == INDEX_HEAD_DIM
|
||||
assert q.stride(2) == 1 and k.stride(1) == 1 and weights.stride(1) == 1
|
||||
assert lens.stride(0) == 1
|
||||
out = torch.empty(t, n, dtype=torch.float32, device=q.device)
|
||||
grid = (triton.cdiv(t, _CONFIG["BLOCK_M"]) * triton.cdiv(n, _CONFIG["BLOCK_N"]),)
|
||||
_index_scores_kernel[grid](
|
||||
q,
|
||||
k,
|
||||
weights,
|
||||
lens,
|
||||
out,
|
||||
t,
|
||||
n,
|
||||
q.stride(0),
|
||||
q.stride(1),
|
||||
k.stride(0),
|
||||
weights.stride(0),
|
||||
out.stride(0),
|
||||
H=heads,
|
||||
D=head_dim,
|
||||
**_CONFIG,
|
||||
)
|
||||
return out
|
||||
@@ -1,5 +1,6 @@
|
||||
"""Fused ratio-1 and ratio-2 decode compressors: RMSNorm, RoPE and the FlashMLA
|
||||
cache write in one launch.
|
||||
"""Fused ratio-1 and ratio-2 compressors: RMSNorm, RoPE and the FlashMLA cache
|
||||
write in one launch. The ratio-1 kernel is per token, so decode and extend share
|
||||
it; ratio 2 has a decode/verify entry point and an extend one.
|
||||
|
||||
Ratio 1 takes the bf16 ``wkv`` projection as is: RoPE uses the token's own
|
||||
position and the compressed slot equals the FULL slot. Ratio 2 pair-pools the
|
||||
@@ -121,11 +122,12 @@ def _jit_c2_module(
|
||||
is_arch_support_pdl(),
|
||||
)
|
||||
return load_jit(
|
||||
make_name("c2_decode"),
|
||||
make_name("c2"),
|
||||
*args,
|
||||
cuda_files=["deepseek_v4/c2.cuh"],
|
||||
cuda_wrappers=[
|
||||
("decode_fusion", f"FlashCompress2Kernel<{args}>::run_decode_fusion"),
|
||||
("prefill_fusion", f"FlashCompress2Kernel<{args}>::run_prefill_fusion"),
|
||||
],
|
||||
)
|
||||
|
||||
@@ -151,6 +153,12 @@ def c2_decode_norm_rope_store(
|
||||
|
||||
The cache store uses ``raw_out_loc // 2`` as its slot.
|
||||
|
||||
A padded graph row is marked by ``raw_out_loc == 0`` and writes nothing, but
|
||||
decode and verify rows issue their ring load before that check, so a padded
|
||||
row's ``req`` and ``positions`` must still address an in-bounds ring slot.
|
||||
The decode graph buffers zero both tails (``PaddingPolicy.ZERO``); a runner
|
||||
that left stale request indices there would read out of the ring pool.
|
||||
|
||||
:param freqs_cis: ``[max_pos, rope_dim]`` fp32, real/imag interleaved --
|
||||
``torch.view_as_real(freqs).flatten(-2)``. Indexed
|
||||
in-kernel at ``positions - 1``, the position the latent
|
||||
@@ -184,3 +192,60 @@ def c2_decode_norm_rope_store(
|
||||
draft_len,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def c2_prefill_norm_rope_store(
|
||||
kv_input: torch.Tensor,
|
||||
kv_state: torch.Tensor,
|
||||
norm_weight: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
req: torch.Tensor,
|
||||
raw_out_loc: torch.Tensor,
|
||||
eps: float,
|
||||
freqs_cis: torch.Tensor,
|
||||
k_cache: torch.Tensor,
|
||||
extend_start_loc: torch.Tensor,
|
||||
extend_seq_lens: torch.Tensor,
|
||||
*,
|
||||
page_size: int,
|
||||
ring_size: int,
|
||||
layout: Union[KVLayout, str] = KVLayout.V4,
|
||||
out: Optional[torch.Tensor] = None,
|
||||
) -> torch.Tensor:
|
||||
""":func:`c2_decode_norm_rope_store` for an extend batch, then the ring write-back.
|
||||
|
||||
Rows may mix requests, prefix offsets and CUDA graph padding (``raw_out_loc
|
||||
== 0``). An odd position pairs with the previous row when that row is the
|
||||
same request's previous position, else with its ring slot. Afterwards each
|
||||
request's last ``ring_size`` rows are saved to the ring, which a later step
|
||||
reads for its first partner.
|
||||
|
||||
:param extend_start_loc: int32 ``[num_reqs]``, each request's first row.
|
||||
:param extend_seq_lens: int32 ``[num_reqs]``, each request's row count.
|
||||
:return: ``out``, ``[num_tokens, head_dim]`` bf16. Only the rows of odd
|
||||
positions are written; the rest are left as they were.
|
||||
"""
|
||||
num_tokens, fused_dim = kv_input.shape
|
||||
head_dim = fused_dim // 2
|
||||
if out is None:
|
||||
out = kv_input.new_empty((num_tokens, head_dim), dtype=torch.bfloat16)
|
||||
|
||||
layout = KVLayout.parse(layout)
|
||||
module = _jit_c2_module(head_dim, freqs_cis.shape[-1], page_size, layout)
|
||||
module.prefill_fusion(
|
||||
kv_input,
|
||||
kv_state,
|
||||
out,
|
||||
norm_weight,
|
||||
positions,
|
||||
req,
|
||||
raw_out_loc,
|
||||
eps,
|
||||
freqs_cis,
|
||||
# HIP's fp8_e4m3_t is uint8_t, so the kernel matches the fp8 pool as bytes
|
||||
k_cache.view(torch.uint8) if is_hip_runtime() else k_cache,
|
||||
ring_size,
|
||||
extend_start_loc,
|
||||
extend_seq_lens,
|
||||
)
|
||||
return out
|
||||
|
||||
@@ -1,126 +0,0 @@
|
||||
"""Rotate Q while writing the padded buffer consumed by sparse attention."""
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.kernels.jit.utils import is_arch_support_pdl
|
||||
from sglang.srt.utils import is_hip
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _q_rope_store(
|
||||
X,
|
||||
Y,
|
||||
F,
|
||||
POS,
|
||||
SX: tl.constexpr,
|
||||
SY: tl.constexpr,
|
||||
IS_HIP: tl.constexpr,
|
||||
USE_GDC: tl.constexpr = False,
|
||||
):
|
||||
row, head = tl.program_id(0), tl.program_id(1)
|
||||
r = tl.arange(0, 512)
|
||||
if USE_GDC:
|
||||
tl.extra.cuda.gdc_wait() # X is the wq_b GEMM output; POS may be in-graph
|
||||
value = tl.load(X + row * SX + head * 512 + r).to(tl.float32)
|
||||
partner = tl.gather(value, r ^ 1, 0)
|
||||
position = tl.load(POS + row)
|
||||
cos = tl.load(F + position * 64 + (r - 448) // 2 * 2, r >= 448, 0)
|
||||
sin = tl.load(F + position * 64 + (r - 448) // 2 * 2 + 1, r >= 448, 0)
|
||||
# Match the operation order of deepseek_rope_kernel before BF16 rounding.
|
||||
even = tl.fma(value, cos, -partner * sin)
|
||||
if IS_HIP:
|
||||
# Match AMD flat RoPE: round the sine product before the cosine FMA.
|
||||
odd = tl.fma(value, cos, partner * sin)
|
||||
else:
|
||||
odd = tl.fma(partner, sin, value * cos)
|
||||
rotated = tl.where((r & 1) == 0, even, odd)
|
||||
tl.store(Y + row * SY + head * 512 + r, tl.where(r >= 448, rotated, value))
|
||||
if USE_GDC:
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _q_rope_store_prefill(
|
||||
X,
|
||||
Y,
|
||||
F,
|
||||
POS,
|
||||
M,
|
||||
H: tl.constexpr,
|
||||
SX: tl.constexpr,
|
||||
SY: tl.constexpr,
|
||||
BLOCK_HEADS: tl.constexpr,
|
||||
IS_HIP: tl.constexpr,
|
||||
):
|
||||
# Keep token count dynamic to avoid compiling every prefill batch length.
|
||||
heads = tl.program_id(0) * BLOCK_HEADS + tl.arange(0, BLOCK_HEADS)
|
||||
row, head = heads // H, heads % H
|
||||
r = tl.arange(0, 512)
|
||||
# The padded output can exceed 2 GiB at the 64K prefill ceiling.
|
||||
x_offset = row[:, None].to(tl.int64) * SX + head[:, None] * 512 + r[None, :]
|
||||
value = tl.load(X + x_offset, row[:, None] < M, 0).to(tl.float32)
|
||||
partner = tl.gather(value, tl.broadcast_to((r ^ 1)[None, :], (BLOCK_HEADS, 512)), 1)
|
||||
position = tl.load(POS + row, row < M, 0)
|
||||
freq_offset = position[:, None].to(tl.int64) * 64 + (r[None, :] - 448) // 2 * 2
|
||||
rope_mask = (row[:, None] < M) & (r[None, :] >= 448)
|
||||
cos = tl.load(F + freq_offset, rope_mask, 0)
|
||||
sin = tl.load(F + freq_offset + 1, rope_mask, 0)
|
||||
# Keep the same arithmetic and BF16 rounding as the decode kernel.
|
||||
even = tl.fma(value, cos, -partner * sin)
|
||||
if IS_HIP:
|
||||
# Match AMD flat RoPE: round the sine product before the cosine FMA.
|
||||
odd = tl.fma(value, cos, partner * sin)
|
||||
else:
|
||||
odd = tl.fma(partner, sin, value * cos)
|
||||
rotated = tl.where((r[None, :] & 1) == 0, even, odd)
|
||||
y_offset = row[:, None].to(tl.int64) * SY + head[:, None] * 512 + r[None, :]
|
||||
tl.store(
|
||||
Y + y_offset,
|
||||
tl.where(r[None, :] >= 448, rotated, value),
|
||||
row[:, None] < M,
|
||||
)
|
||||
|
||||
|
||||
def q_rope_store(
|
||||
q: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
freqs_cis: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
) -> None:
|
||||
"""Apply 64-wide forward RoPE to 512-wide heads without changing Q padding."""
|
||||
assert q.shape == output.shape and q.ndim == 3 and q.shape[2] == 512
|
||||
assert q.dtype == output.dtype == torch.bfloat16
|
||||
assert q.stride(2) == output.stride(2) == 1
|
||||
assert q.stride(1) == output.stride(1) == 512
|
||||
assert freqs_cis.dtype == torch.complex64 and freqs_cis.is_contiguous()
|
||||
assert freqs_cis.shape[1] == 32 and positions.shape == (q.shape[0],)
|
||||
assert positions.dtype in (torch.int32, torch.int64) and positions.is_contiguous()
|
||||
if q.shape[0] >= 4096 and q.shape[1] == 16:
|
||||
_q_rope_store_prefill[(triton.cdiv(q.shape[0] * q.shape[1], 4),)](
|
||||
q,
|
||||
output,
|
||||
torch.view_as_real(freqs_cis),
|
||||
positions,
|
||||
q.shape[0],
|
||||
q.shape[1],
|
||||
q.stride(0),
|
||||
output.stride(0),
|
||||
BLOCK_HEADS=4,
|
||||
IS_HIP=is_hip(),
|
||||
num_warps=4,
|
||||
)
|
||||
return
|
||||
pdl_kwargs = {"USE_GDC": True, "launch_pdl": True} if is_arch_support_pdl() else {}
|
||||
_q_rope_store[(q.shape[0], q.shape[1])](
|
||||
q,
|
||||
output,
|
||||
torch.view_as_real(freqs_cis),
|
||||
positions,
|
||||
q.stride(0),
|
||||
output.stride(0),
|
||||
IS_HIP=is_hip(),
|
||||
num_warps=4,
|
||||
**pdl_kwargs,
|
||||
)
|
||||
@@ -0,0 +1,170 @@
|
||||
"""Fused request-to-KV mapping for opt-in SM90 32-head static verification."""
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
INDEX_HEAD_DIM = 128
|
||||
PAYLOAD_BYTES = tl.constexpr(64)
|
||||
SCALE_BYTES = tl.constexpr(4)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _e2m1_decode(code):
|
||||
# code: uint 0..15 -> e2m1 value. exp = bits 2..1, mantissa = bit 0, sign = bit 3.
|
||||
e = (code >> 1) & 3
|
||||
m = (code & 1).to(tl.float32)
|
||||
sub = m * 0.5
|
||||
exp = tl.where(e == 1, 1.0, tl.where(e == 2, 2.0, 4.0))
|
||||
nor = (1.0 + m * 0.5) * exp
|
||||
v = tl.where(e == 0, sub, nor)
|
||||
return tl.where((code >> 3) == 1, -v, v)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _fp4_index_logits_kernel(
|
||||
q_ptr, # [B, H, D] bf16, fq4 queries (already rope'd)
|
||||
w_ptr, # [B, H] bf16 head weights (softmax scale folded in)
|
||||
slots_ptr, # [requests, capacity * ratio] int32 request-to-token mapping
|
||||
lens_ptr, # [B] int64 visible compressed positions per request
|
||||
table_ptr, # [num_pages, page_size * 64 + page_size * 4] uint8
|
||||
out_ptr, # [B, L] fp32 logits, -inf beyond lens
|
||||
L,
|
||||
page_size,
|
||||
row_stride,
|
||||
stride_qb,
|
||||
stride_qh,
|
||||
stride_wb,
|
||||
H: tl.constexpr,
|
||||
HALF_D: tl.constexpr, # D // 2 == 64 nibble-pairs per row
|
||||
BLOCK_L: tl.constexpr,
|
||||
req_ptr,
|
||||
req_stride: tl.constexpr,
|
||||
ratio: tl.constexpr,
|
||||
):
|
||||
b = tl.program_id(0)
|
||||
lb = tl.program_id(1)
|
||||
offs_l = lb * BLOCK_L + tl.arange(0, BLOCK_L)
|
||||
offs_h = tl.arange(0, H)
|
||||
offs_i = tl.arange(
|
||||
0, HALF_D
|
||||
) # byte index i holds elements 2i (low nibble), 2i+1 (high nibble)
|
||||
|
||||
n_vis = tl.load(lens_ptr + b)
|
||||
# Keep capacity-sized CUDA Graph buffers while skipping invisible tiles,
|
||||
# including mapping loads, dequantization, dot products and head reduction.
|
||||
if lb * BLOCK_L >= n_vis:
|
||||
tl.store(out_ptr + b * L + offs_l, float("-inf"), mask=offs_l < L)
|
||||
else:
|
||||
valid = offs_l < tl.minimum(n_vis, L)
|
||||
request = tl.load(req_ptr + b).to(tl.int64)
|
||||
slot = (
|
||||
tl.load(
|
||||
slots_ptr + request * req_stride + offs_l * ratio,
|
||||
mask=valid,
|
||||
other=0,
|
||||
).to(tl.int64)
|
||||
// ratio
|
||||
)
|
||||
page = slot // page_size
|
||||
off = slot % page_size
|
||||
row_base = page * row_stride
|
||||
|
||||
# K payload: [BLOCK_L, HALF_D] uint8
|
||||
pay = tl.load(
|
||||
table_ptr
|
||||
+ row_base[:, None]
|
||||
+ off[:, None] * PAYLOAD_BYTES
|
||||
+ offs_i[None, :],
|
||||
mask=valid[:, None],
|
||||
other=0,
|
||||
)
|
||||
low = _e2m1_decode(pay & 0x0F)
|
||||
high = _e2m1_decode((pay >> 4) & 0x0F)
|
||||
# e8m0 block scales: each scale covers 16 payload bytes (32 elements).
|
||||
offs_s = tl.arange(0, SCALE_BYTES)
|
||||
exps = tl.load(
|
||||
table_ptr
|
||||
+ row_base[:, None]
|
||||
+ page_size * PAYLOAD_BYTES
|
||||
+ off[:, None] * SCALE_BYTES
|
||||
+ offs_s[None, :],
|
||||
mask=valid[:, None],
|
||||
other=127,
|
||||
)
|
||||
scale = tl.exp2(exps.to(tl.float32) - 127.0)
|
||||
scale = scale[:, :, None]
|
||||
k_shape: tl.constexpr = (BLOCK_L, SCALE_BYTES, HALF_D // SCALE_BYTES)
|
||||
k_low = tl.reshape(tl.reshape(low, k_shape) * scale, (BLOCK_L, HALF_D)).to(
|
||||
tl.bfloat16
|
||||
)
|
||||
k_high = tl.reshape(tl.reshape(high, k_shape) * scale, (BLOCK_L, HALF_D)).to(
|
||||
tl.bfloat16
|
||||
)
|
||||
|
||||
# queries: even / odd elements, [H, HALF_D] bf16
|
||||
q_even = tl.load(
|
||||
q_ptr + b * stride_qb + offs_h[:, None] * stride_qh + 2 * offs_i[None, :]
|
||||
)
|
||||
q_odd = tl.load(
|
||||
q_ptr
|
||||
+ b * stride_qb
|
||||
+ offs_h[:, None] * stride_qh
|
||||
+ 2 * offs_i[None, :]
|
||||
+ 1
|
||||
)
|
||||
|
||||
acc = tl.dot(q_even, tl.trans(k_low)) # [H, BLOCK_L] fp32
|
||||
acc += tl.dot(q_odd, tl.trans(k_high))
|
||||
# reference rounding points: bf16 dot -> relu -> * bf16 weight -> bf16 -> sum -> bf16
|
||||
s = acc.to(tl.bfloat16).to(tl.float32)
|
||||
s = tl.maximum(s, 0.0)
|
||||
w = tl.load(w_ptr + b * stride_wb + offs_h).to(tl.float32)
|
||||
s = (s * w[:, None]).to(tl.bfloat16).to(tl.float32)
|
||||
logit = tl.sum(s, axis=0).to(tl.bfloat16).to(tl.float32)
|
||||
logit = tl.where(valid, logit, float("-inf"))
|
||||
tl.store(out_ptr + b * L + offs_l, logit, mask=offs_l < L)
|
||||
|
||||
|
||||
def fp4_index_logits_mapped_sm90(
|
||||
q,
|
||||
weights,
|
||||
req_to_token,
|
||||
req,
|
||||
lens,
|
||||
table,
|
||||
page_size,
|
||||
ratio,
|
||||
width,
|
||||
):
|
||||
"""Score with request mapping fused into the Triton kernel.
|
||||
|
||||
The caller validates the common tensor/device contract. No dense slots
|
||||
tensor or host read of dynamic lengths is needed during graph replay.
|
||||
"""
|
||||
rows, heads, _ = q.shape
|
||||
out = torch.empty((rows, width), dtype=torch.float32, device=q.device)
|
||||
if rows == 0 or width == 0:
|
||||
return out
|
||||
_fp4_index_logits_kernel[(rows, triton.cdiv(width, 64))](
|
||||
q,
|
||||
weights,
|
||||
req_to_token,
|
||||
lens,
|
||||
table,
|
||||
out,
|
||||
width,
|
||||
page_size,
|
||||
table.stride(0),
|
||||
q.stride(0),
|
||||
q.stride(1),
|
||||
weights.stride(0),
|
||||
H=heads,
|
||||
HALF_D=64,
|
||||
BLOCK_L=64,
|
||||
req_ptr=req,
|
||||
req_stride=req_to_token.stride(0),
|
||||
ratio=ratio,
|
||||
num_warps=4,
|
||||
)
|
||||
return out
|
||||
@@ -0,0 +1,86 @@
|
||||
"""Sort selected logical positions and map slots without dense intermediates."""
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _sort_map_topk(
|
||||
selected,
|
||||
lens,
|
||||
req,
|
||||
mapping,
|
||||
pages,
|
||||
raw,
|
||||
selected_row_stride: tl.constexpr,
|
||||
selected_col_stride: tl.constexpr,
|
||||
lens_stride: tl.constexpr,
|
||||
req_stride: tl.constexpr,
|
||||
mapping_row_stride: tl.constexpr,
|
||||
pages_row_stride: tl.constexpr,
|
||||
raw_row_stride: tl.constexpr,
|
||||
K: tl.constexpr,
|
||||
RATIO: tl.constexpr,
|
||||
HAS_RAW: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
col = tl.arange(0, BLOCK)
|
||||
logical = tl.load(
|
||||
selected + row * selected_row_stride + col * selected_col_stride,
|
||||
col < K,
|
||||
other=0x7FFFFFFFFFFFFFFF,
|
||||
)
|
||||
logical = tl.sort(logical, descending=False)
|
||||
visible = tl.load(lens + row * lens_stride)
|
||||
request = tl.load(req + row * req_stride).to(tl.int64)
|
||||
valid = (col < K) & (logical < visible)
|
||||
# Invalid candidate positions can equal the scan bound; never read their
|
||||
# mappings. The backend has already filled the unused output tail with -1.
|
||||
position = tl.where(valid, logical, 0)
|
||||
slot = tl.load(
|
||||
mapping + request * mapping_row_stride + position * RATIO,
|
||||
valid,
|
||||
other=0,
|
||||
)
|
||||
tl.store(
|
||||
pages + row * pages_row_stride + col,
|
||||
tl.where(valid, slot // RATIO, -1),
|
||||
col < K,
|
||||
)
|
||||
if HAS_RAW:
|
||||
tl.store(
|
||||
raw + row * raw_row_stride + col, tl.where(valid, logical, -1), col < K
|
||||
)
|
||||
|
||||
|
||||
def sort_map_topk(selected, lens, req, mapping, pages, raw, ratio):
|
||||
"""Publish sorted TopK values; selection and tie-breaking stay in PyTorch.
|
||||
|
||||
The caller supplies int64 selected positions, with invalid candidates
|
||||
replaced by the scan bound, and pre-fills both output tensors with -1.
|
||||
Only the opt-in backend uses this small-K epilogue.
|
||||
"""
|
||||
rows, k = selected.shape
|
||||
assert selected.dtype == torch.int64 and 0 < k <= 1024
|
||||
_sort_map_topk[(rows,)](
|
||||
selected,
|
||||
lens,
|
||||
req,
|
||||
mapping,
|
||||
pages,
|
||||
raw if raw is not None else pages,
|
||||
selected.stride(0),
|
||||
selected.stride(1),
|
||||
lens.stride(0),
|
||||
req.stride(0),
|
||||
mapping.stride(0),
|
||||
pages.stride(0),
|
||||
raw.stride(0) if raw is not None else 0,
|
||||
K=k,
|
||||
RATIO=ratio,
|
||||
HAS_RAW=raw is not None,
|
||||
BLOCK=triton.next_power_of_2(k),
|
||||
num_warps=4,
|
||||
)
|
||||
@@ -0,0 +1,524 @@
|
||||
"""Opt-in 32-head Hopper indexer with device-side prefix lengths.
|
||||
|
||||
Score buffers keep their graph-stable capacity, but only the visible prefix
|
||||
(and its final 64-element tile) is initialized. Every consumer MUST use the
|
||||
device lengths. TopK v2 is exact by score, with arbitrary cutoff-tie selection;
|
||||
it does not promise PyTorch's choice of indices with equal scores.
|
||||
"""
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from .sm90_fp4_indexer import _e2m1_decode
|
||||
from .topk import topk_transform_ragged_v2
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _prepare_lengths(
|
||||
POS,
|
||||
CAPACITY,
|
||||
VISIBLE,
|
||||
LENGTHS,
|
||||
ROWS: tl.constexpr,
|
||||
POS_STRIDE: tl.constexpr,
|
||||
RATIO: tl.constexpr,
|
||||
WIDTH: tl.constexpr,
|
||||
HAS_CANDIDATES: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0) * 256 + tl.arange(0, 256)
|
||||
pos = tl.load(POS + row * POS_STRIDE, row < ROWS, other=-1)
|
||||
n = tl.minimum(tl.maximum((pos + 1) // RATIO, 0), WIDTH).to(tl.int32)
|
||||
tl.store(VISIBLE + row, n, row < ROWS)
|
||||
if HAS_CANDIDATES:
|
||||
capacity = tl.load(CAPACITY + row, row < ROWS, other=0)
|
||||
tl.store(LENGTHS + row, tl.minimum(n, capacity), row < ROWS)
|
||||
|
||||
|
||||
def prepare_candidate_lengths(pos, ratio, width, candidates=None):
|
||||
"""Fuse position conversion, clamping, casting and compact score bounds."""
|
||||
visible = torch.empty(pos.shape[0], dtype=torch.int32, device=pos.device)
|
||||
lengths = torch.empty_like(visible) if candidates is not None else visible
|
||||
_prepare_lengths[(triton.cdiv(pos.shape[0], 256),)](
|
||||
pos,
|
||||
candidates.lengths if candidates is not None else visible,
|
||||
visible,
|
||||
lengths,
|
||||
pos.shape[0],
|
||||
pos.stride(0),
|
||||
ratio,
|
||||
width,
|
||||
candidates is not None,
|
||||
num_warps=4,
|
||||
)
|
||||
return visible, lengths
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _prefix_logits(
|
||||
Q,
|
||||
W,
|
||||
MAPPING,
|
||||
REQ,
|
||||
LENS,
|
||||
TABLE,
|
||||
MASK,
|
||||
OUT,
|
||||
BLOCKS,
|
||||
IS_PREFIX,
|
||||
VISIBLE,
|
||||
Q_STRIDE: tl.constexpr,
|
||||
W_STRIDE: tl.constexpr,
|
||||
MAP_STRIDE: tl.constexpr,
|
||||
TABLE_STRIDE: tl.constexpr,
|
||||
MASK_STRIDE: tl.constexpr,
|
||||
OUT_STRIDE: tl.constexpr,
|
||||
BLOCK_STRIDE: tl.constexpr,
|
||||
PAGE: tl.constexpr,
|
||||
RATIO: tl.constexpr,
|
||||
HAS_MASK: tl.constexpr,
|
||||
HAS_BLOCKS: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
n = tl.load(LENS + row)
|
||||
request = tl.load(REQ + row).to(tl.int64)
|
||||
h = tl.arange(0, 32)
|
||||
d = tl.arange(0, 64)
|
||||
# Reuse query/weights across the tiles owned by this CTA.
|
||||
qe = tl.load(Q + row * Q_STRIDE + h[:, None] * 128 + 2 * d[None, :])
|
||||
qo = tl.load(Q + row * Q_STRIDE + h[:, None] * 128 + 2 * d[None, :] + 1)
|
||||
weight = tl.load(W + row * W_STRIDE + h).to(tl.float32)
|
||||
if HAS_BLOCKS:
|
||||
visible_n = tl.load(VISIBLE + row)
|
||||
is_prefix = tl.load(IS_PREFIX + row)
|
||||
for tile in range(tl.program_id(1), tl.cdiv(n, 64), tl.num_programs(1)):
|
||||
col = tile * 64 + tl.arange(0, 64)
|
||||
visible = col < n
|
||||
valid = visible
|
||||
position = col
|
||||
if HAS_BLOCKS:
|
||||
# If all selected blocks form a prefix, keep the short-context
|
||||
# scoring path's contiguous accesses and avoid the block gather.
|
||||
if is_prefix == 0:
|
||||
block = tl.load(
|
||||
BLOCKS + row * BLOCK_STRIDE + col // BLOCK_SIZE,
|
||||
visible,
|
||||
other=-1,
|
||||
)
|
||||
position = block * BLOCK_SIZE + col % BLOCK_SIZE
|
||||
valid = valid & (position >= 0) & (position < visible_n)
|
||||
if HAS_MASK:
|
||||
valid = valid & tl.load(
|
||||
MASK + row * MASK_STRIDE + col, visible, other=False
|
||||
)
|
||||
if tl.sum(valid.to(tl.int32), 0) > 0:
|
||||
slot = (
|
||||
tl.load(
|
||||
MAPPING + request * MAP_STRIDE + position * RATIO, valid, other=0
|
||||
).to(tl.int64)
|
||||
// RATIO
|
||||
)
|
||||
base = (slot // PAGE) * TABLE_STRIDE
|
||||
off = slot % PAGE
|
||||
payload = tl.load(
|
||||
TABLE + base[:, None] + off[:, None] * 64 + d[None, :],
|
||||
valid[:, None],
|
||||
other=0,
|
||||
)
|
||||
scales = tl.load(
|
||||
TABLE
|
||||
+ base[:, None]
|
||||
+ PAGE * 64
|
||||
+ off[:, None] * 4
|
||||
+ tl.arange(0, 4)[None, :],
|
||||
valid[:, None],
|
||||
other=127,
|
||||
)
|
||||
scale = tl.exp2(scales.to(tl.float32) - 127.0)[:, :, None]
|
||||
lo = _e2m1_decode(payload & 15)
|
||||
hi = _e2m1_decode(payload >> 4)
|
||||
kl = tl.reshape(tl.reshape(lo, (64, 4, 16)) * scale, (64, 64)).to(
|
||||
tl.bfloat16
|
||||
)
|
||||
kh = tl.reshape(tl.reshape(hi, (64, 4, 16)) * scale, (64, 64)).to(
|
||||
tl.bfloat16
|
||||
)
|
||||
acc = tl.dot(qe, tl.trans(kl))
|
||||
acc += tl.dot(qo, tl.trans(kh))
|
||||
score = tl.maximum(acc.to(tl.bfloat16).to(tl.float32), 0.0)
|
||||
score = (score * weight[:, None]).to(tl.bfloat16).to(tl.float32)
|
||||
score = tl.sum(score, 0).to(tl.bfloat16).to(tl.float32)
|
||||
score = tl.where(valid, score, -float("inf"))
|
||||
else:
|
||||
score = tl.full((64,), -float("inf"), tl.float32)
|
||||
# TopK uses 16-byte loads at the prefix boundary. Initialize the whole
|
||||
# last tile, including the <= 3 padding floats beyond the logical width.
|
||||
tl.store(OUT + row * OUT_STRIDE + col, score, col < OUT_STRIDE)
|
||||
|
||||
|
||||
def prefix_logits(
|
||||
q,
|
||||
weights,
|
||||
mapping,
|
||||
req,
|
||||
lens,
|
||||
table,
|
||||
page,
|
||||
ratio,
|
||||
width,
|
||||
mask=None,
|
||||
*,
|
||||
candidates=None,
|
||||
visible=None,
|
||||
):
|
||||
"""Return partially initialized scores, bounded by int32 device lengths.
|
||||
|
||||
lens bounds the score buffer in [0, width]. With candidates, visible is the
|
||||
original prefix length and lens bounds the compact candidate positions.
|
||||
"""
|
||||
assert candidates is None or (visible is not None and mask is None)
|
||||
rows = q.shape[0]
|
||||
stride = triton.cdiv(width, 4) * 4
|
||||
out = torch.empty((rows, stride), dtype=torch.float32, device=q.device)
|
||||
sm_count = torch.cuda.get_device_properties(q.device).multi_processor_count
|
||||
tiles = min(
|
||||
triton.cdiv(width, 64),
|
||||
max(8, triton.next_power_of_2(triton.cdiv(8 * sm_count, rows))),
|
||||
)
|
||||
_prefix_logits[(rows, tiles)](
|
||||
q,
|
||||
weights,
|
||||
mapping,
|
||||
req,
|
||||
lens,
|
||||
table,
|
||||
mask if mask is not None else out,
|
||||
out,
|
||||
candidates.blocks if candidates is not None else out,
|
||||
candidates.is_prefix if candidates is not None else lens,
|
||||
visible if visible is not None else lens,
|
||||
q.stride(0),
|
||||
weights.stride(0),
|
||||
mapping.stride(0),
|
||||
table.stride(0),
|
||||
mask.stride(0) if mask is not None else 0,
|
||||
stride,
|
||||
candidates.blocks.stride(0) if candidates is not None else 0,
|
||||
page,
|
||||
ratio,
|
||||
mask is not None,
|
||||
candidates is not None,
|
||||
candidates.block_size if candidates is not None else 1,
|
||||
num_warps=4,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def select_prefix_topk(scores, lens, k):
|
||||
"""Exact TopK by score over each visible prefix; -1 pads short rows."""
|
||||
selected = torch.empty(
|
||||
(scores.shape[0], k), dtype=torch.int32, device=scores.device
|
||||
)
|
||||
# The paged small-batch dispatch can request a 16-CTA cluster, while H20
|
||||
# supports at most 8. Reuse v2's length-aware non-cluster entry point.
|
||||
offsets = torch.zeros_like(lens)
|
||||
topk_transform_ragged_v2(scores, lens, out_offsets=offsets, out_indices=selected)
|
||||
return selected
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _block_scores(
|
||||
SCORES,
|
||||
LENS,
|
||||
OUT,
|
||||
BLOCK_LENS,
|
||||
SCORE_STRIDE: tl.constexpr,
|
||||
OUT_STRIDE: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
TILE: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
n = tl.load(LENS + row)
|
||||
count = tl.cdiv(n, BLOCK_SIZE)
|
||||
if tl.program_id(1) == 0:
|
||||
tl.store(BLOCK_LENS + row, count)
|
||||
for tile in range(tl.program_id(1), tl.cdiv(count, TILE), tl.num_programs(1)):
|
||||
block = tile * TILE + tl.arange(0, TILE)
|
||||
pos = block[:, None] * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)[None, :]
|
||||
v = tl.load(SCORES + row * SCORE_STRIDE + pos, pos < n, other=-float("inf"))
|
||||
score = tl.max(v, 1)
|
||||
score = tl.where(block == count - 1, float("inf"), score)
|
||||
tl.store(OUT + row * OUT_STRIDE + block, score, block < OUT_STRIDE)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _publish_blocks(
|
||||
SELECTED,
|
||||
SCORES,
|
||||
MASK,
|
||||
K: tl.constexpr,
|
||||
SCORE_STRIDE: tl.constexpr,
|
||||
WIDTH: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
i = tl.program_id(1) * 256 + tl.arange(0, 256)
|
||||
block = tl.load(SELECTED + row * K + i // BLOCK_SIZE, i < K * BLOCK_SIZE, other=-1)
|
||||
valid = (i < K * BLOCK_SIZE) & (block >= 0)
|
||||
score = tl.load(
|
||||
SCORES + row * SCORE_STRIDE + tl.maximum(block, 0), valid, other=-float("inf")
|
||||
)
|
||||
pos = block * BLOCK_SIZE + i % BLOCK_SIZE
|
||||
# Keep the whole last block, exactly as the existing candidate mask does;
|
||||
# visibility is enforced independently by every score consumer.
|
||||
tl.store(
|
||||
MASK + row * WIDTH + pos, True, valid & (score > -float("inf")) & (pos < WIDTH)
|
||||
)
|
||||
|
||||
|
||||
def _candidate_block_topk(scores, lens, width, topk_blocks, block_size):
|
||||
rows = scores.shape[0]
|
||||
blocks = triton.cdiv(width, block_size)
|
||||
stride = triton.cdiv(blocks, 4) * 4
|
||||
block_scores = torch.empty(
|
||||
(rows, stride), dtype=torch.float32, device=scores.device
|
||||
)
|
||||
block_lens = torch.empty_like(lens)
|
||||
_block_scores[(rows, min(16, triton.cdiv(blocks, 128)))](
|
||||
scores,
|
||||
lens,
|
||||
block_scores,
|
||||
block_lens,
|
||||
scores.stride(0),
|
||||
stride,
|
||||
block_size,
|
||||
128,
|
||||
num_warps=4,
|
||||
)
|
||||
k = min(topk_blocks, blocks)
|
||||
selected = select_prefix_topk(block_scores, block_lens, k)
|
||||
return selected, block_scores
|
||||
|
||||
|
||||
def candidate_mask(scores, lens, width, topk_blocks, block_size):
|
||||
selected, block_scores = _candidate_block_topk(
|
||||
scores, lens, width, topk_blocks, block_size
|
||||
)
|
||||
rows, k = selected.shape
|
||||
# Preserve CandidateMasks' full-width contract for fallback consumers. Only
|
||||
# the source layer initializes this bool buffer; consumers read valid tiles.
|
||||
mask = torch.zeros((rows, width), dtype=torch.bool, device=scores.device)
|
||||
_publish_blocks[(rows, triton.cdiv(k * block_size, 256))](
|
||||
selected,
|
||||
block_scores,
|
||||
mask,
|
||||
k,
|
||||
block_scores.stride(0),
|
||||
width,
|
||||
block_size,
|
||||
num_warps=4,
|
||||
)
|
||||
return mask
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _sort_blocks(
|
||||
SELECTED,
|
||||
SCORES,
|
||||
LENS,
|
||||
BLOCKS,
|
||||
LENGTHS,
|
||||
IS_PREFIX,
|
||||
K: tl.constexpr,
|
||||
SCORE_STRIDE: tl.constexpr,
|
||||
WIDTH: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
i = tl.arange(0, BLOCK)
|
||||
count = tl.cdiv(tl.load(LENS + row), BLOCK_SIZE)
|
||||
if count <= K:
|
||||
# Short contexts keep every finite block. Avoid a 2048-element bitonic
|
||||
# sort when the candidate list is already the contiguous prefix.
|
||||
block = i
|
||||
valid = (i < K) & (i < count)
|
||||
else:
|
||||
block = tl.load(SELECTED + row * K + i, i < K, other=-1)
|
||||
valid = (i < K) & (block >= 0)
|
||||
score = tl.load(
|
||||
SCORES + row * SCORE_STRIDE + tl.maximum(block, 0),
|
||||
valid,
|
||||
other=-float("inf"),
|
||||
)
|
||||
valid = valid & (score > -float("inf"))
|
||||
block = tl.where(valid, block, 0x7FFFFFFF)
|
||||
# All -inf blocks are excluded even when K covers the prefix. Such holes
|
||||
# need compaction just like the selected subset of a long context does.
|
||||
if count > K or tl.sum(valid.to(tl.int32), 0) != count:
|
||||
block = tl.sort(block, descending=False)
|
||||
valid = block != 0x7FFFFFFF
|
||||
count = tl.sum(valid.to(tl.int32), 0)
|
||||
tl.store(BLOCKS + row * K + i, tl.where(valid, block, -1), i < K)
|
||||
# Keep entire selected blocks, matching CandidateMasks. Consumer visibility
|
||||
# independently masks the final partial block and any differing ratio.
|
||||
tl.store(LENGTHS + row, tl.minimum(count * BLOCK_SIZE, WIDTH))
|
||||
tl.store(IS_PREFIX + row, tl.sum((valid & (block != i)).to(tl.int32), 0) == 0)
|
||||
|
||||
|
||||
def candidate_blocks(scores, lens, width, topk_blocks, block_size):
|
||||
"""Publish a compact list using prefix scores and non-cluster TopK."""
|
||||
from sglang.srt.layers.attention.dsv4.v41_indexer.sm90_decode import CandidateBlocks
|
||||
|
||||
selected, block_scores = _candidate_block_topk(
|
||||
scores, lens, width, topk_blocks, block_size
|
||||
)
|
||||
rows, k = selected.shape
|
||||
blocks = torch.empty_like(selected)
|
||||
lengths = torch.empty_like(lens)
|
||||
is_prefix = torch.empty_like(lens)
|
||||
_sort_blocks[(rows,)](
|
||||
selected,
|
||||
block_scores,
|
||||
lens,
|
||||
blocks,
|
||||
lengths,
|
||||
is_prefix,
|
||||
k,
|
||||
block_scores.stride(0),
|
||||
width,
|
||||
block_size,
|
||||
triton.next_power_of_2(k),
|
||||
num_warps=4,
|
||||
)
|
||||
return CandidateBlocks(blocks, lengths, is_prefix, width, block_size)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _blocks_to_mask(
|
||||
BLOCKS,
|
||||
MASK,
|
||||
K: tl.constexpr,
|
||||
WIDTH: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
i = tl.program_id(1) * 256 + tl.arange(0, 256)
|
||||
block = tl.load(BLOCKS + row * K + i // BLOCK_SIZE, i < K * BLOCK_SIZE, other=-1)
|
||||
pos = block * BLOCK_SIZE + i % BLOCK_SIZE
|
||||
tl.store(
|
||||
MASK + row * WIDTH + pos,
|
||||
True,
|
||||
(i < K * BLOCK_SIZE) & (block >= 0) & (pos < WIDTH),
|
||||
)
|
||||
|
||||
|
||||
def materialize_candidate_mask(candidates):
|
||||
"""Build the legacy mask only when a later layer takes a fallback path."""
|
||||
rows, k = candidates.blocks.shape
|
||||
mask = torch.zeros(
|
||||
(rows, candidates.width), dtype=torch.bool, device=candidates.blocks.device
|
||||
)
|
||||
_blocks_to_mask[(rows, triton.cdiv(k * candidates.block_size, 256))](
|
||||
candidates.blocks,
|
||||
mask,
|
||||
k,
|
||||
candidates.width,
|
||||
candidates.block_size,
|
||||
num_warps=4,
|
||||
)
|
||||
return mask
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _publish_topk(
|
||||
SELECTED,
|
||||
SCORES,
|
||||
LENS,
|
||||
REQ,
|
||||
MAPPING,
|
||||
PAGES,
|
||||
RAW,
|
||||
BLOCKS,
|
||||
K: tl.constexpr,
|
||||
SCORE_STRIDE: tl.constexpr,
|
||||
MAP_STRIDE: tl.constexpr,
|
||||
PAGE_STRIDE: tl.constexpr,
|
||||
RAW_STRIDE: tl.constexpr,
|
||||
BLOCK_STRIDE: tl.constexpr,
|
||||
RATIO: tl.constexpr,
|
||||
HAS_RAW: tl.constexpr,
|
||||
FILTER_MASKED: tl.constexpr,
|
||||
HAS_BLOCKS: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
col = tl.arange(0, BLOCK)
|
||||
logical = tl.load(SELECTED + row * K + col, col < K, other=-1)
|
||||
n = tl.load(LENS + row)
|
||||
valid = (logical >= 0) & (logical < n) & (col < K)
|
||||
if FILTER_MASKED:
|
||||
score = tl.load(
|
||||
SCORES + row * SCORE_STRIDE + tl.maximum(logical, 0),
|
||||
valid,
|
||||
other=-float("inf"),
|
||||
)
|
||||
valid = valid & (score > -float("inf"))
|
||||
if HAS_BLOCKS:
|
||||
block = tl.load(
|
||||
BLOCKS + row * BLOCK_STRIDE + tl.maximum(logical, 0) // BLOCK_SIZE,
|
||||
valid,
|
||||
other=-1,
|
||||
)
|
||||
logical = block * BLOCK_SIZE + logical % BLOCK_SIZE
|
||||
logical = tl.sort(tl.where(valid, logical, 0x7FFFFFFF), descending=False)
|
||||
valid = (logical != 0x7FFFFFFF) & (col < K)
|
||||
request = tl.load(REQ + row).to(tl.int64)
|
||||
position = tl.where(valid, logical, 0).to(tl.int64)
|
||||
slot = tl.load(MAPPING + request * MAP_STRIDE + position * RATIO, valid, other=0)
|
||||
tl.store(
|
||||
PAGES + row * PAGE_STRIDE + col, tl.where(valid, slot // RATIO, -1), col < K
|
||||
)
|
||||
if HAS_RAW:
|
||||
tl.store(RAW + row * RAW_STRIDE + col, tl.where(valid, logical, -1), col < K)
|
||||
|
||||
|
||||
def publish_topk(
|
||||
selected,
|
||||
scores,
|
||||
lens,
|
||||
req,
|
||||
mapping,
|
||||
pages,
|
||||
raw,
|
||||
ratio,
|
||||
filter_masked,
|
||||
*,
|
||||
candidates=None,
|
||||
):
|
||||
rows, k = selected.shape
|
||||
_publish_topk[(rows,)](
|
||||
selected,
|
||||
scores,
|
||||
lens,
|
||||
req,
|
||||
mapping,
|
||||
pages,
|
||||
raw if raw is not None else pages,
|
||||
candidates.blocks if candidates is not None else selected,
|
||||
k,
|
||||
scores.stride(0),
|
||||
mapping.stride(0),
|
||||
pages.stride(0),
|
||||
raw.stride(0) if raw is not None else 0,
|
||||
candidates.blocks.stride(0) if candidates is not None else 0,
|
||||
ratio,
|
||||
raw is not None,
|
||||
filter_masked,
|
||||
candidates is not None,
|
||||
candidates.block_size if candidates is not None else 1,
|
||||
triton.next_power_of_2(k),
|
||||
num_warps=4,
|
||||
)
|
||||
@@ -36,7 +36,7 @@ def _build_swa_token_ids_kernel(
|
||||
tl.store(out_ptr + out_off + i, swa_id)
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["top_k"])
|
||||
@triton.jit(do_not_specialize=["num_tokens", "num_reqs", "top_k"])
|
||||
def _combine_topk_swa_indices_kernel(
|
||||
combined_indices_ptr,
|
||||
combined_indices_stride,
|
||||
@@ -49,59 +49,76 @@ def _combine_topk_swa_indices_kernel(
|
||||
gather_lens_ptr,
|
||||
compressed_base_ptr,
|
||||
swa_base_ptr,
|
||||
num_tokens,
|
||||
num_reqs,
|
||||
top_k,
|
||||
COMPRESS_RATIO: tl.constexpr,
|
||||
WINDOW_SIZE: tl.constexpr,
|
||||
PADDED_TOP_K: tl.constexpr,
|
||||
WIDTH: tl.constexpr,
|
||||
PADDED_WIDTH: tl.constexpr,
|
||||
BLOCK_T: tl.constexpr,
|
||||
SEARCH_STEPS: tl.constexpr,
|
||||
):
|
||||
batch_idx = tl.program_id(0)
|
||||
worker_id = tl.program_id(1)
|
||||
num_workers = tl.num_programs(1)
|
||||
# BLOCK_T consecutive tokens per program, so the grid scales with tokens, not requests.
|
||||
token = tl.program_id(0) * BLOCK_T + tl.arange(0, BLOCK_T)
|
||||
valid = token < num_tokens
|
||||
|
||||
# query_start_loc may be a global tensor; rebase to chunk-local offsets
|
||||
# by subtracting the chunk's starting value.
|
||||
base = tl.load(query_start_loc_ptr)
|
||||
query_start = tl.load(query_start_loc_ptr + batch_idx) - base
|
||||
query_end = tl.load(query_start_loc_ptr + batch_idx + 1) - base
|
||||
seq_len = tl.load(seq_lens_ptr + batch_idx)
|
||||
gather_len = tl.load(gather_lens_ptr + batch_idx)
|
||||
compressed_base = tl.load(compressed_base_ptr + batch_idx)
|
||||
swa_base = tl.load(swa_base_ptr + batch_idx)
|
||||
# Each token's request: the last r with query_start_loc[r] - base <= token.
|
||||
lo = tl.zeros([BLOCK_T], dtype=tl.int32)
|
||||
hi = tl.full([BLOCK_T], 0, dtype=tl.int32) + num_reqs
|
||||
for _ in tl.static_range(SEARCH_STEPS):
|
||||
mid = (lo + hi + 1) // 2
|
||||
start = tl.load(query_start_loc_ptr + mid, mask=lo < hi, other=0) - base
|
||||
go_right = (lo < hi) & (start <= token)
|
||||
lo = tl.where(go_right, mid, lo)
|
||||
hi = tl.where((lo < hi) & ~go_right, mid - 1, hi)
|
||||
batch_idx = tl.minimum(lo, num_reqs - 1)
|
||||
# Rows past the last request's queries belong to no request: -1 rows, length 0.
|
||||
owned = valid & (token < tl.load(query_start_loc_ptr + num_reqs) - base)
|
||||
|
||||
pos = tl.load(query_pos_ptr + token, mask=owned, other=0)
|
||||
seq_len = tl.load(seq_lens_ptr + batch_idx, mask=owned, other=0)
|
||||
gather_len = tl.load(gather_lens_ptr + batch_idx, mask=owned, other=0)
|
||||
compressed_base = tl.load(compressed_base_ptr + batch_idx, mask=owned, other=0)
|
||||
swa_base = tl.load(swa_base_ptr + batch_idx, mask=owned, other=0)
|
||||
# SWA portion of the gathered buffer starts from position
|
||||
# (seq_len - gather_len), not 0. The +pos-gather_start formula maps a
|
||||
# query's window back into the workspace's SWA region.
|
||||
gather_start = seq_len - gather_len
|
||||
|
||||
for token_idx in range(query_start + worker_id, query_end, num_workers):
|
||||
pos = tl.load(query_pos_ptr + token_idx)
|
||||
# -1 entries inside the top-k span stay -1 (attention skips them).
|
||||
# top_k=0 disables the compressed portion for SWA-only layers.
|
||||
topk_len = tl.minimum((pos + 1) // COMPRESS_RATIO, top_k)
|
||||
swa_len = tl.minimum(pos + 1, WINDOW_SIZE)
|
||||
# -1 entries inside the top-k span stay -1 (attention skips them).
|
||||
# top_k=0 disables the compressed portion for SWA-only layers.
|
||||
topk_len = tl.where(owned, tl.minimum((pos + 1) // COMPRESS_RATIO, top_k), 0)
|
||||
swa_len = tl.where(owned, tl.minimum(pos + 1, WINDOW_SIZE), 0)
|
||||
|
||||
combined_row = token_idx.to(tl.int64) * combined_indices_stride
|
||||
topk_row = token_idx.to(tl.int64) * topk_indices_stride
|
||||
combined_row = token.to(tl.int64)[:, None] * combined_indices_stride
|
||||
topk_row = token.to(tl.int64)[:, None] * topk_indices_stride
|
||||
|
||||
offset = tl.arange(0, PADDED_TOP_K)
|
||||
mask = offset < topk_len
|
||||
topk_vals = tl.load(
|
||||
topk_indices_ptr + topk_row + offset,
|
||||
mask=mask,
|
||||
)
|
||||
tl.store(
|
||||
combined_indices_ptr + combined_row + offset,
|
||||
tl.where(topk_vals >= 0, topk_vals + compressed_base, -1),
|
||||
mask=mask,
|
||||
)
|
||||
# Whole rows, -1 padded, so the output needs no fill pass:
|
||||
# [top-k (-1 holes kept) | window | -1 up to WIDTH].
|
||||
offset = tl.arange(0, PADDED_WIDTH)[None, :]
|
||||
in_topk = offset < topk_len[:, None]
|
||||
topk_vals = tl.load(
|
||||
topk_indices_ptr + topk_row + offset, mask=valid[:, None] & in_topk, other=-1
|
||||
)
|
||||
# Workspace SWA index: swa_base[r] + (gather_offset_in_buffer).
|
||||
# For positions [pos - swa_len + 1, pos], the buffer offsets are
|
||||
# [pos - swa_len + 1 - gather_start, pos - gather_start].
|
||||
swa_start = swa_base + pos - swa_len + 1 - gather_start
|
||||
swa_vals = swa_start[:, None] + (offset - topk_len[:, None])
|
||||
in_swa = offset < (topk_len + swa_len)[:, None]
|
||||
vals = tl.where(
|
||||
in_topk,
|
||||
tl.where(topk_vals >= 0, topk_vals + compressed_base[:, None], -1),
|
||||
tl.where(in_swa, swa_vals, -1),
|
||||
)
|
||||
tl.store(
|
||||
combined_indices_ptr + combined_row + offset,
|
||||
vals,
|
||||
mask=valid[:, None] & (offset < WIDTH),
|
||||
)
|
||||
|
||||
offset = tl.arange(0, WINDOW_SIZE)
|
||||
# Workspace SWA index: swa_base[r] + (gather_offset_in_buffer).
|
||||
# For positions [pos - swa_len + 1, pos], the buffer offsets are
|
||||
# [pos - swa_len + 1 - gather_start, pos - gather_start].
|
||||
tl.store(
|
||||
combined_indices_ptr + combined_row + topk_len + offset,
|
||||
swa_base + offset + pos - swa_len + 1 - gather_start,
|
||||
mask=offset < swa_len,
|
||||
)
|
||||
|
||||
tl.store(combined_lens_ptr + token_idx, topk_len + swa_len)
|
||||
tl.store(combined_lens_ptr + token, topk_len + swa_len, mask=valid)
|
||||
|
||||
@@ -98,6 +98,8 @@ def _engram_hash_kernel(
|
||||
out_ptr,
|
||||
num_tokens,
|
||||
num_real,
|
||||
extend_bs,
|
||||
search_steps,
|
||||
pad_id,
|
||||
image_token_id,
|
||||
mm_pad_shift,
|
||||
@@ -111,6 +113,7 @@ def _engram_hash_kernel(
|
||||
L: tl.constexpr,
|
||||
H: tl.constexpr,
|
||||
BLOCK_T: tl.constexpr,
|
||||
ROWS_FROM_STARTS: tl.constexpr = False,
|
||||
):
|
||||
COLS: tl.constexpr = (N - 1) * H
|
||||
t = tl.program_id(0) * BLOCK_T + tl.arange(0, BLOCK_T)
|
||||
@@ -124,7 +127,25 @@ def _engram_hash_kernel(
|
||||
r = (t // BLOCK).to(tl.int64)
|
||||
off = t - (t // BLOCK) * BLOCK
|
||||
else:
|
||||
r = tl.load(row_ptr + t, mask=real, other=0).to(tl.int64)
|
||||
if ROWS_FROM_STARTS:
|
||||
# Upper bound handles adjacent starts from zero-length requests.
|
||||
# Resolve rows in registers instead of materializing repeat_interleave.
|
||||
# The batch size and step count are runtime values: as constexprs every
|
||||
# new batch size would compile and device-load a fresh specialization
|
||||
# mid-serving.
|
||||
lo = tl.zeros((BLOCK_T,), tl.int32)
|
||||
hi = tl.zeros((BLOCK_T,), tl.int32) + extend_bs
|
||||
for _ in range(search_steps):
|
||||
mid = (lo + hi) // 2
|
||||
start = tl.load(
|
||||
starts_ptr + mid, mask=real & (mid < extend_bs), other=0
|
||||
)
|
||||
right = (mid < extend_bs) & (start <= t)
|
||||
lo = tl.where(right, mid + 1, lo)
|
||||
hi = tl.where(right, hi, mid)
|
||||
r = tl.maximum(lo - 1, 0).to(tl.int64)
|
||||
else:
|
||||
r = tl.load(row_ptr + t, mask=real, other=0).to(tl.int64)
|
||||
off = t - tl.load(starts_ptr + r, mask=real, other=0).to(tl.int32)
|
||||
if HIST_VIA_SLOTS:
|
||||
hrow = tl.load(slots_ptr + r, mask=real, other=0).to(tl.int64)
|
||||
@@ -207,6 +228,7 @@ def _launch_hash_kernel(
|
||||
out_cache_loc: Optional[torch.Tensor],
|
||||
write_tokens: bool,
|
||||
block_t: int,
|
||||
extend_bs: int = 0,
|
||||
) -> tuple[torch.Tensor, Optional[torch.Tensor]]:
|
||||
num_tokens = input_ids.shape[0]
|
||||
L, N = multipliers.shape
|
||||
@@ -215,7 +237,7 @@ def _launch_hash_kernel(
|
||||
assert primes.shape == (L, N - 1, H) and offsets.shape == (L, (N - 1) * H)
|
||||
assert history.dim() == 2 and history.shape[1] == N - 1, history.shape
|
||||
if mode == MODE_EXTEND:
|
||||
assert row is not None and starts is not None
|
||||
assert (row is not None or extend_bs > 0) and starts is not None
|
||||
if out_cache_loc is not None:
|
||||
assert mode == MODE_DECODE and req_slots is not None, "commit is decode-only"
|
||||
assert out_cache_loc.shape[0] == num_tokens, out_cache_loc.shape
|
||||
@@ -247,6 +269,8 @@ def _launch_hash_kernel(
|
||||
out,
|
||||
num_tokens,
|
||||
num_real,
|
||||
extend_bs,
|
||||
extend_bs.bit_length(),
|
||||
pad_id,
|
||||
image_token_id if image_token_id is not None else -1,
|
||||
mm_pad_shift,
|
||||
@@ -260,11 +284,111 @@ def _launch_hash_kernel(
|
||||
L=L,
|
||||
H=H,
|
||||
BLOCK_T=block_t,
|
||||
ROWS_FROM_STARTS=extend_bs > 0,
|
||||
num_warps=4,
|
||||
)
|
||||
return out, tokens
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _engram_commit_extend_history_kernel(
|
||||
history_ptr,
|
||||
tokens_ptr,
|
||||
slots_ptr,
|
||||
starts_ptr,
|
||||
lens_ptr,
|
||||
out_loc_ptr,
|
||||
num_tokens,
|
||||
N: tl.constexpr,
|
||||
HAS_OUT_LOC: tl.constexpr,
|
||||
):
|
||||
r = tl.program_id(0)
|
||||
length = tl.load(lens_ptr + r)
|
||||
start = tl.load(starts_ptr + r)
|
||||
last = tl.minimum(tl.maximum(start + length - 1, 0), num_tokens - 1)
|
||||
live = length > 0
|
||||
if HAS_OUT_LOC:
|
||||
live = live & (tl.load(out_loc_ptr + last) != 0)
|
||||
slot = tl.load(slots_ptr + r).to(tl.int64)
|
||||
col = tl.arange(0, N)
|
||||
value = tl.load(
|
||||
tokens_ptr + last * N + (N - 2 - col),
|
||||
mask=live & (col < N - 1),
|
||||
other=0,
|
||||
)
|
||||
tl.store(history_ptr + slot * (N - 1) + col, value, mask=live & (col < N - 1))
|
||||
|
||||
|
||||
def engram_hash_extend_and_commit(
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
*,
|
||||
history: torch.Tensor,
|
||||
commit_history: torch.Tensor,
|
||||
req_slots: torch.Tensor,
|
||||
starts: torch.Tensor,
|
||||
lengths: torch.Tensor,
|
||||
num_real: int,
|
||||
history_via_slots: bool,
|
||||
out_cache_loc: Optional[torch.Tensor],
|
||||
token_map: torch.Tensor,
|
||||
multipliers: torch.Tensor,
|
||||
primes: torch.Tensor,
|
||||
offsets: torch.Tensor,
|
||||
pad_id: int,
|
||||
image_token_id: Optional[int] = None,
|
||||
mm_pad_shift: int = 0,
|
||||
) -> torch.Tensor:
|
||||
"""Eager extend: hash without a row-map tensor, then commit in one launch.
|
||||
|
||||
Hashing must finish before history is updated: different token programs may
|
||||
still read a request's old history. Keep these as two ordered kernels.
|
||||
Empty/padded requests leave live history untouched; the spare padding row is
|
||||
intentionally not updated. Request slots must be distinct for live requests.
|
||||
"""
|
||||
bs = req_slots.numel()
|
||||
assert 0 <= num_real <= input_ids.numel()
|
||||
assert starts.numel() == lengths.numel() == bs
|
||||
assert history.is_contiguous() and commit_history.is_contiguous()
|
||||
assert all(t.ndim == 1 and t.stride(0) == 1 for t in (req_slots, starts, lengths))
|
||||
out, tokens = _launch_hash_kernel(
|
||||
input_ids,
|
||||
positions,
|
||||
mode=MODE_EXTEND,
|
||||
history=history,
|
||||
token_map=token_map,
|
||||
multipliers=multipliers,
|
||||
primes=primes,
|
||||
offsets=offsets,
|
||||
pad_id=pad_id,
|
||||
num_real=num_real,
|
||||
req_slots=req_slots if history_via_slots else None,
|
||||
block=1,
|
||||
row=None,
|
||||
starts=starts,
|
||||
image_token_id=image_token_id,
|
||||
mm_pad_shift=mm_pad_shift,
|
||||
out_cache_loc=None,
|
||||
write_tokens=True,
|
||||
block_t=32,
|
||||
extend_bs=bs,
|
||||
)
|
||||
if input_ids.numel() and bs:
|
||||
_engram_commit_extend_history_kernel[(bs,)](
|
||||
commit_history,
|
||||
tokens,
|
||||
req_slots,
|
||||
starts,
|
||||
lengths,
|
||||
out_cache_loc if out_cache_loc is not None else req_slots,
|
||||
input_ids.numel(),
|
||||
N=multipliers.shape[1],
|
||||
HAS_OUT_LOC=out_cache_loc is not None,
|
||||
num_warps=4,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def engram_hash_ids(
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
|
||||
+138
@@ -0,0 +1,138 @@
|
||||
{
|
||||
"0": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 16
|
||||
},
|
||||
"2": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 16
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 16
|
||||
},
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 16
|
||||
},
|
||||
"12": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 16
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 16
|
||||
},
|
||||
"24": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"40": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"48": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"56": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"65": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
}
|
||||
}
|
||||
+138
@@ -0,0 +1,138 @@
|
||||
{
|
||||
"0": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 16
|
||||
},
|
||||
"2": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 16
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 16
|
||||
},
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 16
|
||||
},
|
||||
"12": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 16
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 16
|
||||
},
|
||||
"24": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"40": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"48": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"56": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"65": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
}
|
||||
}
|
||||
+62
-2
@@ -5,11 +5,71 @@
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 8
|
||||
"SPLIT_K": 16
|
||||
},
|
||||
"2": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 16
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 16
|
||||
},
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 16
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 16
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 16
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 16
|
||||
},
|
||||
"65": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
|
||||
+126
@@ -0,0 +1,126 @@
|
||||
{
|
||||
"0": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"2": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 2
|
||||
},
|
||||
"12": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 2
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 2
|
||||
},
|
||||
"24": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"40": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"48": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"56": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"65": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
}
|
||||
}
|
||||
+126
@@ -0,0 +1,126 @@
|
||||
{
|
||||
"0": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"2": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 2
|
||||
},
|
||||
"12": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 2
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 2
|
||||
},
|
||||
"24": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"40": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"48": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"56": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"65": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
}
|
||||
}
|
||||
+63
-2
@@ -5,10 +5,71 @@
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"2": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 2
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 1
|
||||
},
|
||||
"65": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
|
||||
+138
@@ -0,0 +1,138 @@
|
||||
{
|
||||
"0": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"2": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"12": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"24": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"40": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"48": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"56": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"65": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
}
|
||||
}
|
||||
+138
@@ -0,0 +1,138 @@
|
||||
{
|
||||
"0": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"2": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"12": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"24": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"40": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"48": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"56": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"65": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
}
|
||||
}
|
||||
+62
-2
@@ -1,15 +1,75 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"2": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"2": {
|
||||
"65": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
|
||||
+132
@@ -0,0 +1,132 @@
|
||||
{
|
||||
"0": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 2,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"2": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"12": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"24": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"40": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"48": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"56": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"65": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
}
|
||||
}
|
||||
+132
@@ -0,0 +1,132 @@
|
||||
{
|
||||
"0": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 2,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"2": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"12": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"24": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"40": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"48": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"56": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"65": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
}
|
||||
}
|
||||
+80
@@ -0,0 +1,80 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"2": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"65": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
}
|
||||
}
|
||||
+126
@@ -0,0 +1,126 @@
|
||||
{
|
||||
"0": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 1
|
||||
},
|
||||
"2": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 1
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 1
|
||||
},
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 1
|
||||
},
|
||||
"12": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 1
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 1
|
||||
},
|
||||
"24": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"40": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"48": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"56": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"65": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
}
|
||||
}
|
||||
+126
@@ -0,0 +1,126 @@
|
||||
{
|
||||
"0": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 1
|
||||
},
|
||||
"2": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 1
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 1
|
||||
},
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 1
|
||||
},
|
||||
"12": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 1
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 1
|
||||
},
|
||||
"24": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"40": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"48": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"56": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"65": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
}
|
||||
}
|
||||
+86
@@ -0,0 +1,86 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"2": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"33": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"65": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
}
|
||||
}
|
||||
+138
@@ -0,0 +1,138 @@
|
||||
{
|
||||
"0": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 16
|
||||
},
|
||||
"2": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 16
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 16
|
||||
},
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 16
|
||||
},
|
||||
"12": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 16
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 16
|
||||
},
|
||||
"24": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"40": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"48": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"56": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"65": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
}
|
||||
}
|
||||
+138
@@ -0,0 +1,138 @@
|
||||
{
|
||||
"0": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 16
|
||||
},
|
||||
"2": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 16
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 16
|
||||
},
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 16
|
||||
},
|
||||
"12": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 16
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 16
|
||||
},
|
||||
"24": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"40": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"48": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"56": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"65": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
}
|
||||
}
|
||||
+80
@@ -0,0 +1,80 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 16
|
||||
},
|
||||
"2": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 16
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 16
|
||||
},
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 16
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 16
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 16
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 16
|
||||
},
|
||||
"65": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
}
|
||||
}
|
||||
@@ -33,8 +33,11 @@ def deep_gemm_fp8_fp8_bf16_nt(
|
||||
B: torch.Tensor,
|
||||
Bs: torch.Tensor,
|
||||
C: torch.Tensor,
|
||||
block_n: int = 128,
|
||||
block_k: int = 128,
|
||||
) -> None:
|
||||
deep_gemm_wrapper.gemm_nt_f8f8bf16((A, As), (B, Bs), C)
|
||||
recipe = (1, block_n, block_k) if (block_n, block_k) != (128, 128) else None
|
||||
deep_gemm_wrapper.gemm_nt_f8f8bf16((A, As), (B, Bs), C, recipe=recipe)
|
||||
|
||||
|
||||
@register_custom_op(mutates_args=["C"])
|
||||
@@ -746,7 +749,7 @@ def w8a8_block_fp8_matmul_deepgemm(
|
||||
# Deepgemm only supports output tensor type as bfloat16
|
||||
assert C.dtype == torch.bfloat16 and deep_gemm_wrapper.ENABLE_JIT_DEEPGEMM
|
||||
|
||||
deep_gemm_fp8_fp8_bf16_nt(A, As, B, Bs, C)
|
||||
deep_gemm_fp8_fp8_bf16_nt(A, As, B, Bs, C, block_size[0], block_size[1])
|
||||
|
||||
return C
|
||||
|
||||
|
||||
@@ -1888,6 +1888,34 @@ def hc_expand(x: torch.Tensor, n: int) -> torch.Tensor:
|
||||
return x.repeat(1, n)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _hc_broadcast_kernel(
|
||||
x_ptr, y_ptr, H, x_stride_m, HC: tl.constexpr, BLOCK_H: tl.constexpr
|
||||
):
|
||||
row = tl.program_id(0).to(tl.int64)
|
||||
offs = tl.program_id(1) * BLOCK_H + tl.arange(0, BLOCK_H)
|
||||
mask = offs < H
|
||||
x = tl.load(x_ptr + row * x_stride_m + offs, mask=mask)
|
||||
y_row = y_ptr + row * (HC * H)
|
||||
for k in tl.static_range(HC):
|
||||
tl.store(y_row + k * H + offs, x, mask=mask)
|
||||
|
||||
|
||||
def hc_broadcast(x: torch.Tensor, hc_mult: int) -> torch.Tensor:
|
||||
"""[T, H] -> [T, hc_mult, H], every copy equal to x: the initial mHC residual."""
|
||||
if not x.is_cuda or x.stride(-1) != 1:
|
||||
return x.unsqueeze(1).repeat(1, hc_mult, 1)
|
||||
num_tokens, hidden = x.shape
|
||||
y = x.new_empty(num_tokens, hc_mult, hidden)
|
||||
if num_tokens == 0:
|
||||
return y
|
||||
block_h = 1024
|
||||
_hc_broadcast_kernel[(num_tokens, triton.cdiv(hidden, block_h))](
|
||||
x, y, hidden, x.stride(0), HC=hc_mult, BLOCK_H=block_h, num_warps=4
|
||||
)
|
||||
return y
|
||||
|
||||
|
||||
def hc_contract(x: torch.Tensor, n: int) -> torch.Tensor:
|
||||
return x.unflatten(-1, (n, -1)).mean(dim=-2)
|
||||
|
||||
@@ -2611,7 +2639,10 @@ def hc_mix_stats_sinkhorn_bf16x3(
|
||||
m, k = x.shape
|
||||
mix = (2 + hc_mult) * hc_mult
|
||||
slices = _HC_MIX_COMPENSATED_SLICES
|
||||
assert x.is_contiguous() and x.dtype == torch.bfloat16 and 4096 <= m <= 65536
|
||||
hopper_medium = get_platform().is_sm90 and 32 <= m < 4096
|
||||
block_m = 64 if hopper_medium else _HC_MIX_BF16X3_BLOCK_M
|
||||
assert x.is_contiguous() and x.dtype == torch.bfloat16
|
||||
assert hopper_medium or 4096 <= m <= 65536
|
||||
assert k % (slices * _HC_MIX_BLOCK_K) == 0
|
||||
assert len(weight_parts) == 3
|
||||
assert all(
|
||||
@@ -2623,7 +2654,7 @@ def hc_mix_stats_sinkhorn_bf16x3(
|
||||
pre = torch.empty((m, hc_mult), device=x.device, dtype=torch.float32)
|
||||
post = torch.empty_like(pre)
|
||||
comb = torch.empty((m, hc_mult, hc_mult), device=x.device, dtype=torch.float32)
|
||||
_hc_mix_stats_bf16x3_kernel[(triton.cdiv(m, _HC_MIX_BF16X3_BLOCK_M), slices)](
|
||||
_hc_mix_stats_bf16x3_kernel[(triton.cdiv(m, block_m), slices)](
|
||||
x,
|
||||
*weight_parts,
|
||||
part_mix,
|
||||
@@ -2634,7 +2665,7 @@ def hc_mix_stats_sinkhorn_bf16x3(
|
||||
MIX_COLS=mix,
|
||||
MIX_PAD=triton.next_power_of_2(mix),
|
||||
BLOCK_K=_HC_MIX_BLOCK_K,
|
||||
BLOCK_M=_HC_MIX_BF16X3_BLOCK_M,
|
||||
BLOCK_M=block_m,
|
||||
num_warps=4,
|
||||
num_stages=3,
|
||||
)
|
||||
|
||||
@@ -172,6 +172,7 @@ _TRITON_KERNELS = [
|
||||
("fp8_kernel", "sglang_per_token_group_quant_8bit"),
|
||||
("fp8_kernel", "sglang_per_token_quant_fp8"),
|
||||
("fp8_kernel", "static_quant_fp8"),
|
||||
("fp8_kernel", "dequant_group_fp8_to_bf16"),
|
||||
("fp8_kernel", "per_tensor_quant_mla_fp8"),
|
||||
("fp8_kernel", "per_token_group_quant_mla_deep_gemm_masked_fp8"),
|
||||
("fp8_kernel", "per_token_group_quant_fp8_hopper_moe_mn_major"),
|
||||
|
||||
+134
@@ -0,0 +1,134 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"2": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"256": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 2
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"8192": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
}
|
||||
}
|
||||
+134
@@ -0,0 +1,134 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"2": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"256": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 2
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"8192": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
}
|
||||
}
|
||||
+128
@@ -0,0 +1,128 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"2": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 2
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"256": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"8192": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
}
|
||||
}
|
||||
+128
@@ -0,0 +1,128 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"2": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 2
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"256": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"8192": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
}
|
||||
}
|
||||
+128
@@ -0,0 +1,128 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"2": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 2
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 1
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 2
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"256": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"8192": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
}
|
||||
}
|
||||
+128
@@ -0,0 +1,128 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"2": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 2
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 1
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 2
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"256": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"8192": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
}
|
||||
}
|
||||
+126
@@ -0,0 +1,126 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"2": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 1
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 1
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"256": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"8192": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
}
|
||||
}
|
||||
+126
@@ -0,0 +1,126 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"2": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 1
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 1
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"256": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"8192": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
}
|
||||
}
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 1
|
||||
},
|
||||
"2": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 1
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 1
|
||||
}
|
||||
}
|
||||
+122
@@ -0,0 +1,122 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 1
|
||||
},
|
||||
"2": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 1
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 1
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 1
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"256": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"8192": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
}
|
||||
}
|
||||
+122
@@ -0,0 +1,122 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 1
|
||||
},
|
||||
"2": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 1
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 1
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 1
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 32,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"256": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"8192": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
}
|
||||
}
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 1
|
||||
},
|
||||
"2": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 1
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 1
|
||||
}
|
||||
}
|
||||
+138
@@ -0,0 +1,138 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 16
|
||||
},
|
||||
"2": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 16
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 16
|
||||
},
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 16
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 16
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 16
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"256": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 2
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 2
|
||||
},
|
||||
"8192": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
}
|
||||
}
|
||||
+138
@@ -0,0 +1,138 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 16
|
||||
},
|
||||
"2": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 16
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 16
|
||||
},
|
||||
"8": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 16
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 16
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 8
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 16
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"256": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 4
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 2
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": false,
|
||||
"SPLIT_K": 2
|
||||
},
|
||||
"8192": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3
|
||||
}
|
||||
}
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 16
|
||||
},
|
||||
"2": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 64,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 16
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 32,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 4,
|
||||
"SWAP_AB": true,
|
||||
"SPLIT_K": 8
|
||||
}
|
||||
}
|
||||
@@ -1389,6 +1389,25 @@ def per_group_transpose(
|
||||
return trans_a
|
||||
|
||||
|
||||
# input - [M, K]
|
||||
# weight - [K, N]
|
||||
# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/layers/quantization/compressed_tensors/triton_scaled_mm.py
|
||||
@triton.jit
|
||||
def _dequant_group_fp8_kernel(
|
||||
X, S, Y, M, K: tl.constexpr, SS: tl.constexpr, BLOCK: tl.constexpr
|
||||
):
|
||||
offsets = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
|
||||
rows, cols = offsets // K, offsets % K
|
||||
value = tl.load(X + offsets, offsets < M * K, 0.0).to(tl.float32)
|
||||
scale = tl.load(S + rows * SS + cols // 32, rows < M, 0.0)
|
||||
tl.store(Y + offsets, value * scale, offsets < M * K)
|
||||
|
||||
|
||||
def dequant_group_fp8_to_bf16(x: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:
|
||||
"""Expand row-wise group32 FP8 values; UE8M0 scaling is exact in BF16."""
|
||||
m, k = x.shape
|
||||
assert x.is_contiguous() and k % 32 == 0
|
||||
assert scale.shape == (m, k // 32) and scale.stride(1) == 1
|
||||
out = torch.empty_like(x, dtype=torch.bfloat16)
|
||||
if m:
|
||||
_dequant_group_fp8_kernel[(triton.cdiv(m * k, 1024),)](
|
||||
x, scale, out, m, k, scale.stride(0), 1024
|
||||
)
|
||||
return out
|
||||
|
||||
@@ -172,15 +172,25 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
|
||||
)
|
||||
|
||||
cfg = resolving_view(server_args)
|
||||
if model_config_of(server_args).hf_config.model_type != "deepseek_v41":
|
||||
hf_config = model_config_of(server_args).hf_config
|
||||
if hf_config.model_type != "deepseek_v41":
|
||||
if cfg.enable_encoder_swa_bounded_replay:
|
||||
raise ValueError(
|
||||
"--enable-encoder-swa-bounded-replay requires DeepSeek-V4.1"
|
||||
)
|
||||
return
|
||||
if cfg.enable_encoder_swa_bounded_replay:
|
||||
from sglang.srt.model_executor.cuda_graph_config import Backend
|
||||
from sglang.srt.model_executor.cuda_graph_config import Backend
|
||||
|
||||
if hf_config.vision_n_layers > 0 and cfg.enable_prefill_cp:
|
||||
_validate_deepseek_v41_vision_prefill_cp(server_args)
|
||||
if (
|
||||
cfg.dsv4_attn_backend == "trtllm"
|
||||
and cfg.cuda_graph_config.prefill.backend != Backend.DISABLED
|
||||
):
|
||||
raise ValueError(
|
||||
"DeepSeek-V4.1 TRT-LLM requires --cuda-graph-backend-prefill disabled"
|
||||
)
|
||||
if cfg.enable_encoder_swa_bounded_replay:
|
||||
incompatible = (
|
||||
("non-CUDA hardware", not get_platform().is_cuda),
|
||||
(
|
||||
@@ -188,7 +198,8 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
|
||||
cfg.cuda_graph_config.prefill.backend != Backend.DISABLED,
|
||||
),
|
||||
("DP attention", cfg.enable_dp_attention),
|
||||
("context parallelism", cfg.attn_cp_size > 1),
|
||||
# Prefill CP declares attn_cp_size and DP attention only later.
|
||||
("context parallelism", cfg.attn_cp_size > 1 or cfg.enable_prefill_cp),
|
||||
("external cache linker", cfg.enable_unified_cache_external_linker),
|
||||
("unified memory", cfg.enable_unified_memory),
|
||||
("PD disaggregation", cfg.disaggregation_mode != "null"),
|
||||
@@ -237,26 +248,43 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
|
||||
read_ragged_verify_mode,
|
||||
)
|
||||
|
||||
# CP size may still be implicit here; validate_deepseek_v4_cp resolves it.
|
||||
supported_prefill_cp = (
|
||||
cfg.disaggregation_mode == "prefill"
|
||||
and cfg.enable_prefill_cp
|
||||
and cfg.cp_strategy == "interleave"
|
||||
)
|
||||
if (
|
||||
read_ragged_verify_mode() is not RaggedVerifyMode.STATIC
|
||||
or cfg.disaggregation_transfer_backend != "mooncake"
|
||||
or cfg.dp_size != 1
|
||||
or cfg.enable_dp_attention
|
||||
or cfg.attn_cp_size != 1
|
||||
or (
|
||||
(cfg.enable_prefill_cp or cfg.attn_cp_size != 1)
|
||||
and not supported_prefill_cp
|
||||
)
|
||||
or cfg.dcp_size != 1
|
||||
):
|
||||
raise ValueError(
|
||||
"DeepSeek-V4.1 DSpark PD requires static verify, Mooncake, "
|
||||
"DP=1 and CP=1. Both servers must enable DSpark with the same "
|
||||
"block size and TP size."
|
||||
"DCP=1, and CP=1 on decode. Prefill CP requires "
|
||||
"--enable-prefill-cp with the interleave strategy. "
|
||||
"DP attention is supported when "
|
||||
"both servers use the same block size and target/draft KV layout."
|
||||
)
|
||||
|
||||
from sglang.srt.model_executor.cuda_graph_config import Backend, Phase, with_phase
|
||||
|
||||
prefill_graph = cfg.cuda_graph_config.prefill
|
||||
if prefill_graph.backend != Backend.DISABLED and prefill_graph.max_seq_len is None:
|
||||
# The captured low-ratio indexer scores a static context width; 16k
|
||||
# keeps it inside the candidate window at under 1 ms per layer.
|
||||
cp_breakable_prefill = (
|
||||
cfg.enable_prefill_cp
|
||||
and cfg.cp_strategy == "interleave"
|
||||
and cfg.tp_size > 1
|
||||
and prefill_graph.backend == Backend.BREAKABLE
|
||||
)
|
||||
if (
|
||||
prefill_graph.backend != Backend.DISABLED
|
||||
and prefill_graph.max_seq_len is None
|
||||
and not cp_breakable_prefill
|
||||
):
|
||||
declare_resolution(
|
||||
server_args,
|
||||
"validate_deepseek_v41_features",
|
||||
@@ -287,3 +315,34 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
|
||||
"--enable-decoder-swa-bounded-replay cannot be combined with "
|
||||
f"{feature} yet; disable one of them."
|
||||
)
|
||||
|
||||
|
||||
def _validate_deepseek_v41_vision_prefill_cp(server_args: ServerArgs) -> None:
|
||||
cfg = resolving_view(server_args)
|
||||
from sglang.srt.model_executor.cuda_graph_config import Backend, Phase, with_phase
|
||||
|
||||
if cfg.cp_strategy != "interleave":
|
||||
raise ValueError(
|
||||
"DeepSeek-V4.1 vision with prefill CP requires --cp-strategy "
|
||||
f"interleave; got {cfg.cp_strategy!r}."
|
||||
)
|
||||
if cfg.cuda_graph_config.prefill.backend != Backend.DISABLED:
|
||||
locked = getattr(server_args, "_cuda_graph_config_locked", set())
|
||||
if (Phase.PREFILL, "backend") in locked:
|
||||
raise ValueError(
|
||||
"DeepSeek-V4.1 vision with prefill CP runs eager prefill; remove "
|
||||
"the explicit prefill CUDA graph backend."
|
||||
)
|
||||
declare_resolution(
|
||||
server_args,
|
||||
"validate_deepseek_v41_features",
|
||||
cuda_graph_config=with_phase(
|
||||
cfg.cuda_graph_config, Phase.PREFILL, backend=Backend.DISABLED
|
||||
),
|
||||
)
|
||||
logger.warning(
|
||||
"Disabling the prefill CUDA graph for DeepSeek-V4.1 vision with prefill CP."
|
||||
)
|
||||
# Eager CP gathers decoder-replay outputs with the tail layout and forwards
|
||||
# their global token indices to DSpark. The ordinary graph/DP replay guards
|
||||
# in validate_deepseek_v41_features still apply.
|
||||
|
||||
@@ -980,12 +980,14 @@ def handle_language_model_only(server_args: Any):
|
||||
):
|
||||
if flag:
|
||||
raise ValueError(f"--language-model-only cannot be combined with {name}")
|
||||
if cfg.disaggregation_mode != "null":
|
||||
hf_config = model_config_of(server_args).hf_config
|
||||
# V4.1 text-only workers use the standard PD KV transfer path.
|
||||
if cfg.disaggregation_mode != "null" and hf_config.model_type != "deepseek_v41":
|
||||
raise ValueError(
|
||||
"--language-model-only is incompatible with --disaggregation-mode "
|
||||
"prefill/decode"
|
||||
)
|
||||
architectures = model_config_of(server_args).hf_config.architectures
|
||||
architectures = hf_config.architectures
|
||||
if not any(
|
||||
a in server_args.LANGUAGE_MODEL_ONLY_ARCHITECTURES for a in architectures
|
||||
):
|
||||
|
||||
@@ -1780,6 +1780,18 @@ class _SGLangPlugin(_FrameworkPlugin):
|
||||
if isinstance(value, self.PPProxyTensors):
|
||||
return {k: v for k, v in value.tensors.items()}
|
||||
|
||||
from sglang.srt.layers.layer_boundary.output import UnreducedOutput
|
||||
from sglang.srt.layers.layer_boundary.residual.stream import OwedOutput
|
||||
|
||||
# Observe rank-local storage without completing the deferred reduction or
|
||||
# consuming the residual stream. Opaque handoffs and consumed handles have
|
||||
# no tensor to dump.
|
||||
if isinstance(value, UnreducedOutput):
|
||||
return {"": value.partial}
|
||||
if isinstance(value, OwedOutput):
|
||||
tensor = value.contribution.value
|
||||
return {"": tensor} if tensor is not None else {}
|
||||
|
||||
return None
|
||||
|
||||
def detect_layer_id(self, module: "torch.nn.Module") -> Optional[int]:
|
||||
|
||||
@@ -986,9 +986,20 @@ class CommonKVManager(BaseKVManager):
|
||||
"enable DSpark with the same block size and target/draft KV "
|
||||
"layout. Upgrade both servers together."
|
||||
)
|
||||
if info.attn_tp_size != self.attn_tp_size:
|
||||
|
||||
decode_tp_size_aligns_prefill_world_size = (
|
||||
info.attn_cp_size > 1
|
||||
and (self.is_mla_backend or self.is_hybrid_mla_backend)
|
||||
and self.attn_cp_size == 1
|
||||
and info.attn_tp_size * info.attn_cp_size == self.attn_tp_size
|
||||
)
|
||||
if (
|
||||
info.attn_tp_size != self.attn_tp_size
|
||||
and not decode_tp_size_aligns_prefill_world_size
|
||||
):
|
||||
raise RuntimeError(
|
||||
"DeepSeek-V4.1 DSpark PD requires the same TP size on both servers"
|
||||
"DeepSeek-V4.1 DSpark PD requires the same TP size on both "
|
||||
"servers (including prefill CP ranks for MLA)"
|
||||
)
|
||||
|
||||
if self.dcp_size > 1:
|
||||
|
||||
@@ -890,6 +890,10 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
continue
|
||||
|
||||
self.cache_unfinished_disagg_prefill(req)
|
||||
# G9: the disagg-prefill path bypasses batch_result_processor's
|
||||
# prefill loop, so record the drain event here — final chunk
|
||||
# done, request heading to the inflight (KV transfer) queue.
|
||||
self.record_prefill_drain(req.seqlen)
|
||||
self.disagg_prefill_inflight_queue.append(req)
|
||||
if self.spec_algorithm.is_eagle() and draft_input is not None:
|
||||
req.output_topk_p = draft_input.topk_p[i]
|
||||
|
||||
@@ -1784,6 +1784,18 @@ async def openai_v1_chat_completions(
|
||||
)
|
||||
|
||||
|
||||
@app.post("/query_cache_hit", dependencies=[Depends(validate_json_request)])
|
||||
async def query_cache_hit(request: ChatCompletionRequest, raw_request: Request):
|
||||
"""G9: probe prefix-cache hit (L1/L2 + L3 storage) for a chat request.
|
||||
|
||||
Deployed on decode engines; the PD router calls it before dispatch to
|
||||
decide conditional bypass. Accepts a normal chat body or `input_ids`.
|
||||
"""
|
||||
return await raw_request.app.state.openai_serving_chat.query_cache_hit(
|
||||
request, raw_request
|
||||
)
|
||||
|
||||
|
||||
@app.post(
|
||||
"/v1/embeddings",
|
||||
response_class=ORJSONResponse,
|
||||
|
||||
@@ -25,6 +25,12 @@ LATEST_REMINDER_SP_TOKEN = "<|latest_reminder|>"
|
||||
|
||||
IMAGE_PLACEHOLDER = "<|deepseek_image|>"
|
||||
|
||||
# ASCII-pipe spelling of IMAGE_PLACEHOLDER. User-supplied text that happens to
|
||||
# contain the literal fullwidth-bar token is rewritten to this spelling
|
||||
# instead of being rejected, so it can never be confused with a genuine
|
||||
# image placeholder inserted by _process_image_blocks for real image content.
|
||||
IMAGE_PLACEHOLDER_ESCAPED = "<|deepseek_image|>"
|
||||
|
||||
# Task special tokens for internal classification tasks
|
||||
DS_TASK_SP_TOKENS = {
|
||||
"action": "<|action|>",
|
||||
@@ -528,9 +534,26 @@ def _extract_image(block: Dict[str, Any]) -> Dict[str, Any]:
|
||||
return {"url": url}
|
||||
|
||||
|
||||
def _escape_image_placeholder(text: str) -> str:
|
||||
"""Rewrite a literal image-placeholder token to its ASCII-pipe spelling.
|
||||
|
||||
Applied to user-supplied text (never to a placeholder this module inserts
|
||||
itself for a genuine image block), so the escaped spelling can never be
|
||||
mistaken for a real image placeholder downstream.
|
||||
"""
|
||||
return text.replace(IMAGE_PLACEHOLDER, IMAGE_PLACEHOLDER_ESCAPED)
|
||||
|
||||
|
||||
def _process_image_blocks(
|
||||
blocks: List[Any], image_placeholder: str = IMAGE_PLACEHOLDER
|
||||
) -> Tuple[List[Any], List[Dict[str, Any]]]:
|
||||
"""Replace image blocks with placeholders and collect their records in order.
|
||||
|
||||
Any literal image-placeholder token found in user-supplied text (a text
|
||||
block, or a tool-result block's text content) is escaped to its
|
||||
ASCII-pipe spelling rather than rejected; only a genuine image/image_url
|
||||
block inserts the real placeholder.
|
||||
"""
|
||||
new_blocks: List[Any] = []
|
||||
images: List[Dict[str, Any]] = []
|
||||
for block in blocks:
|
||||
@@ -549,31 +572,41 @@ def _process_image_blocks(
|
||||
)
|
||||
new_blocks.append(block)
|
||||
images.extend(nested_images)
|
||||
elif block.get("type") == "tool_result" and isinstance(
|
||||
block.get("content"), str
|
||||
):
|
||||
content = block.get("content", "")
|
||||
if IMAGE_PLACEHOLDER in content:
|
||||
block = copy.copy(block)
|
||||
block["content"] = _escape_image_placeholder(content)
|
||||
new_blocks.append(block)
|
||||
elif block.get("type") == "text":
|
||||
text = block.get("text") or ""
|
||||
if IMAGE_PLACEHOLDER in text:
|
||||
raise ValueError(
|
||||
f"Text block contains image placeholder '{IMAGE_PLACEHOLDER}': "
|
||||
f"'{text[:100]}'. Images should be separate content blocks."
|
||||
)
|
||||
block = copy.copy(block)
|
||||
block["text"] = _escape_image_placeholder(text)
|
||||
new_blocks.append(block)
|
||||
else:
|
||||
new_blocks.append(block)
|
||||
return new_blocks, images
|
||||
|
||||
|
||||
def _validate_no_image_sp_tokens(msg: Dict[str, Any]) -> None:
|
||||
def _escape_image_sp_tokens(msg: Dict[str, Any]) -> None:
|
||||
"""Escape user-supplied image placeholder tokens in textual fields.
|
||||
|
||||
A literal image-placeholder token typed or echoed back by the user (in
|
||||
``content``, ``reasoning_content``, or a tool message's ``content``) is
|
||||
rewritten in place to its ASCII-pipe spelling instead of being rejected,
|
||||
so it can never be mistaken for a genuine image placeholder this module
|
||||
inserts for real image content.
|
||||
"""
|
||||
content = msg.get("content")
|
||||
if isinstance(content, str) and IMAGE_PLACEHOLDER in content:
|
||||
raise ValueError(
|
||||
f"Message content contains image special token '{IMAGE_PLACEHOLDER}'. "
|
||||
"Images should be provided as image content blocks."
|
||||
)
|
||||
msg["content"] = _escape_image_placeholder(content)
|
||||
|
||||
reasoning_content = msg.get("reasoning_content")
|
||||
if isinstance(reasoning_content, str) and IMAGE_PLACEHOLDER in reasoning_content:
|
||||
raise ValueError(
|
||||
f"reasoning_content contains image special token '{IMAGE_PLACEHOLDER}'"
|
||||
)
|
||||
msg["reasoning_content"] = _escape_image_placeholder(reasoning_content)
|
||||
|
||||
|
||||
def process_image_messages(
|
||||
@@ -583,7 +616,7 @@ def process_image_messages(
|
||||
images: List[Dict[str, Any]] = []
|
||||
for msg in messages:
|
||||
msg = copy.deepcopy(msg)
|
||||
_validate_no_image_sp_tokens(msg)
|
||||
_escape_image_sp_tokens(msg)
|
||||
|
||||
if isinstance(msg.get("content"), list) and "content_blocks" not in msg:
|
||||
msg["content_blocks"] = msg.pop("content")
|
||||
|
||||
@@ -1154,6 +1154,73 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
return "text", processed_messages.prompt_ids
|
||||
return "input_ids", processed_messages.prompt_ids
|
||||
|
||||
async def query_cache_hit(
|
||||
self, request: ChatCompletionRequest, raw_request: Request = None
|
||||
) -> ORJSONResponse:
|
||||
"""G9 probe: report how much of this chat request's prompt is already
|
||||
cached (per-rank-best L1/L2 radix + shared L3 storage) without running
|
||||
any prefill. The router uses it for conditional PD-bypass decisions.
|
||||
|
||||
Multimodal requests are rejected (v1): their sentinel-token expansion
|
||||
costs ~10-100ms per image for only ~2% of traffic; the router skips
|
||||
them for bypass anyway.
|
||||
"""
|
||||
try:
|
||||
internal_req, _ = self._convert_to_internal_request(request, raw_request)
|
||||
except Exception as e:
|
||||
return ORJSONResponse(
|
||||
status_code=400,
|
||||
content={"error": f"failed to convert chat request: {e}"},
|
||||
)
|
||||
if internal_req.contains_mm_input():
|
||||
return ORJSONResponse(
|
||||
status_code=400,
|
||||
content={"error": "multimodal requests are not supported by /query_cache_hit"},
|
||||
)
|
||||
try:
|
||||
# 只取 input_ids,不走 _tokenize_one_request——它要求 rid_to_state
|
||||
# 里已有 ReqState(正常生成路径在 tokenize 前注册),纯查询没有。
|
||||
# dsv41 系 chat encoding 直接渲染 prompt_ids;否则对渲染后的
|
||||
# text 走 tokenizer(_tokenize_texts 与真实路径同一入口)。
|
||||
if internal_req.input_ids is not None:
|
||||
input_ids = list(internal_req.input_ids)
|
||||
elif internal_req.text:
|
||||
input_ids, _ = await self.tokenizer_manager._tokenize_texts(
|
||||
internal_req.text, False
|
||||
)
|
||||
else:
|
||||
return ORJSONResponse(
|
||||
status_code=400,
|
||||
content={"error": "request produced neither input_ids nor text"},
|
||||
)
|
||||
from sglang.srt.managers.io_struct import QueryCacheHitReqInput
|
||||
|
||||
result = await self.tokenizer_manager.query_cache_hit(
|
||||
QueryCacheHitReqInput(
|
||||
input_ids=list(input_ids),
|
||||
extra_key=getattr(internal_req, "extra_key", None),
|
||||
cache_salt=getattr(internal_req, "cache_salt", None),
|
||||
)
|
||||
)
|
||||
except Exception as e:
|
||||
return ORJSONResponse(
|
||||
status_code=500, content={"error": f"cache-hit query failed: {e}"}
|
||||
)
|
||||
if result.error is not None:
|
||||
return ORJSONResponse(status_code=500, content={"error": result.error})
|
||||
hit = result.l1_hit_tokens + result.l2_hit_tokens + result.l3_hit_tokens
|
||||
return ORJSONResponse(
|
||||
content={
|
||||
"total_tokens": result.total_tokens,
|
||||
"l1_hit_tokens": result.l1_hit_tokens,
|
||||
"l2_hit_tokens": result.l2_hit_tokens,
|
||||
"l3_hit_tokens": result.l3_hit_tokens,
|
||||
"hit_tokens": hit,
|
||||
"new_tokens": max(0, result.total_tokens - hit),
|
||||
"dp_rank": result.dp_rank,
|
||||
}
|
||||
)
|
||||
|
||||
def _convert_to_internal_request(
|
||||
self,
|
||||
request: ChatCompletionRequest,
|
||||
|
||||
@@ -1015,6 +1015,9 @@ class Envs:
|
||||
SGLANG_CPU_QUANTIZATION = EnvBool(False)
|
||||
SGLANG_USE_DYNAMIC_MXFP4_LINEAR = EnvBool(False)
|
||||
SGLANG_FORCE_FP8_MARLIN = EnvBool(False)
|
||||
# Cache BF16 expansions of Hopper group32 FP8 weights for larger GEMMs.
|
||||
# Disable to save the additional weight memory or allow online updates.
|
||||
SGLANG_OPT_HOPPER_BLOCK_FP8_BF16 = EnvBool(True)
|
||||
SGLANG_MOE_NVFP4_DISPATCH = EnvBool(False)
|
||||
SGLANG_NVFP4_CKPT_FP8_GEMM_IN_ATTN = EnvBool(False)
|
||||
SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE = EnvBool(False)
|
||||
@@ -1550,6 +1553,8 @@ class Envs:
|
||||
# DeepSeek-V4.1 engram host table: keep the tables in host memory (layout
|
||||
# below) and gather rows from the GPU instead of sharding them over HBM.
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE = EnvBool(False)
|
||||
SGLANG_OPT_ENGRAM_EXTEND_HASH = EnvBool(False)
|
||||
SGLANG_OPT_DSV41_MEGA_MHC_PREFILL = EnvBool(False)
|
||||
# "shared" is one buffer for the whole TP group, mapped by every rank, with no
|
||||
# lookup all-reduce (the ranks must share a PID namespace); "per_rank" is one
|
||||
# anonymous mapping per rank holding only its rows, gathered with the
|
||||
@@ -1574,6 +1579,17 @@ class Envs:
|
||||
# Run the DeepSeek-V4.1 ratio-1/2 prefill indexer on the torch path instead
|
||||
# of the DeepGEMM dense fp4 logits kernel (test oracle / fallback).
|
||||
SGLANG_DSV41_TORCH_PREFILL_INDEXER = EnvBool(False)
|
||||
# Fused mapping for SM90 32-head, ratio-1/2 static target verification.
|
||||
SGLANG_OPT_DSV41_SM90_GROUPED_INDEXER = EnvBool(False)
|
||||
# Persistent prefix scoring and length-aware TopK; requires GROUPED_INDEXER.
|
||||
# Equal-score cutoff ties can select different positions than PyTorch.
|
||||
SGLANG_OPT_DSV41_SM90_LENGTH_AWARE_INDEXER = EnvBool(False)
|
||||
# Compact candidate lists; requires the length-aware static-verify path.
|
||||
SGLANG_OPT_DSV41_SM90_COMPACT_CANDIDATES = EnvBool(False)
|
||||
# Score the torch prefill indexer with one Triton kernel instead of the
|
||||
# einsum/relu/weight/sum chain; False, or SGLANG_DSV41_TORCH_PREFILL_INDEXER,
|
||||
# restores the torch scoring.
|
||||
SGLANG_OPT_USE_FUSED_DSV41_INDEXER_SCORES = EnvBool(True)
|
||||
SGLANG_FP8_PAGED_MQA_LOGITS_TORCH = EnvBool(False)
|
||||
SGLANG_OPT_FLASHMLA_SPARSE_PREFILL = EnvBool(True)
|
||||
|
||||
|
||||
@@ -82,7 +82,6 @@ from sglang.srt.layers.attention.dsv4.v41_indexer import (
|
||||
CapturedPrefillInputs,
|
||||
DecodeInputs,
|
||||
PrefillInputs,
|
||||
Selection,
|
||||
has_dense_fp4_indexer,
|
||||
is_sm100_or_newer,
|
||||
make_candidate_indexer,
|
||||
@@ -122,6 +121,7 @@ from sglang.srt.speculative.ragged_verify import (
|
||||
resolve_ragged_verify_layout,
|
||||
)
|
||||
from sglang.srt.utils import ceil_align, is_cuda, is_xpu
|
||||
from sglang.srt.utils.common import async_h2d
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sgl_kernel.flash_mla import FlashMLASchedMeta
|
||||
@@ -888,6 +888,16 @@ class LateLayerTail(msgspec.Struct, frozen=True):
|
||||
local_lens_cpu: Optional[List[int]] = None
|
||||
req_global: Optional[torch.Tensor] = None
|
||||
pos_global: Optional[torch.Tensor] = None
|
||||
# DSpark injection consumes global extend positions after the CP gather.
|
||||
global_token_indices: Optional[torch.Tensor] = None
|
||||
|
||||
@property
|
||||
def output_token_indices(self) -> torch.Tensor:
|
||||
return (
|
||||
self.global_token_indices
|
||||
if self.global_token_indices is not None
|
||||
else self.token_indices
|
||||
)
|
||||
|
||||
def rows(self, t: torch.Tensor) -> torch.Tensor:
|
||||
rows = self.real_rows(t)
|
||||
@@ -1189,6 +1199,11 @@ class DeepseekV4AttnBackend(
|
||||
] = None
|
||||
self.online_c128_mtp = OnlineC128MTPController(self)
|
||||
self.sparse_prefill_workspace = SparsePrefillWorkspace(self.device)
|
||||
# CP V4.1 consumers share compressed KV across layers. Separate ratio
|
||||
# workspaces keep those prefixes intact while each layer refreshes SWA.
|
||||
self.shared_compressed_prefill_workspaces = {
|
||||
ratio: SparsePrefillWorkspace(self.device) for ratio in (1, 2)
|
||||
}
|
||||
spec_alg = model_runner.spec_algorithm
|
||||
self.needs_cpu_seq_lens = not spec_alg.is_dspark() and (
|
||||
not _is_cuda or self.online_c128_mtp.enabled()
|
||||
@@ -1499,7 +1514,13 @@ class DeepseekV4AttnBackend(
|
||||
|
||||
@property
|
||||
def low_ratio_prefill_graph(self) -> bool:
|
||||
return bool(self.low_ratios) and has_dense_fp4_indexer() and is_sm100_or_newer()
|
||||
"""Whether ratio-1/2 sources use captured projections and indexer metadata."""
|
||||
return (
|
||||
bool(self.low_ratios)
|
||||
and has_dense_fp4_indexer()
|
||||
and is_sm100_or_newer()
|
||||
and get_parallel().attn_cp_size == 1
|
||||
)
|
||||
|
||||
def can_run_prefill_cuda_graph(self, forward_batch: ForwardBatch) -> bool:
|
||||
max_seq_len = _prefill_graph_max_seq_len()
|
||||
@@ -1592,6 +1613,7 @@ class DeepseekV4AttnBackend(
|
||||
local_lens_cpu=cp_tail["local_lens_cpu"],
|
||||
req_global=metadata.low_ratio_req_indices,
|
||||
pos_global=metadata.low_ratio_pos_i64,
|
||||
global_token_indices=token_indices,
|
||||
)
|
||||
return metadata
|
||||
|
||||
@@ -2420,7 +2442,7 @@ class DeepseekV4AttnBackend(
|
||||
req_to_token=self.req_to_token,
|
||||
full_to_swa=self.token_to_kv_pool.full_to_swa_index_mapping,
|
||||
swa_window_size=SWA_WINDOW,
|
||||
swa_page_size=self.token_to_kv_pool.swa_kv_pool.page_size,
|
||||
swa_page_size=self.token_to_kv_pool.get_swa_key_page_size(),
|
||||
num_qo_tokens=num_qo_tokens,
|
||||
max_seq_len=max(seq_lens_cpu_list),
|
||||
total_swa=total_swa,
|
||||
@@ -2803,6 +2825,22 @@ class DeepseekV4AttnBackend(
|
||||
self._low_ratio_compress_fused(
|
||||
layer, x, req, pos, draft_len=self.speculative_num_draft_tokens
|
||||
)
|
||||
elif (
|
||||
layer.compressor.use_fused_compress
|
||||
and forward_batch.forward_mode.is_extend_without_speculative()
|
||||
and forward_batch.extend_start_loc is not None
|
||||
):
|
||||
# DP padding extends extend_seq_lens but not extend_start_loc, whose
|
||||
# length is the request count; padded rows belong to no request.
|
||||
start_loc = forward_batch.extend_start_loc
|
||||
seq_lens = forward_batch.extend_seq_lens[: start_loc.shape[0]]
|
||||
self._low_ratio_compress_fused(
|
||||
layer,
|
||||
x,
|
||||
req,
|
||||
pos,
|
||||
extend_offsets=(start_loc.to(torch.int32), seq_lens.to(torch.int32)),
|
||||
)
|
||||
else:
|
||||
self._low_ratio_compress_torch(
|
||||
layer,
|
||||
@@ -2872,13 +2910,18 @@ class DeepseekV4AttnBackend(
|
||||
fuse_index_store=is_sm100_or_newer(),
|
||||
)
|
||||
|
||||
def _low_ratio_compress_fused(self, layer, x, req, pos, *, draft_len=1) -> None:
|
||||
def _low_ratio_compress_fused(
|
||||
self, layer, x, req, pos, *, draft_len=1, extend_offsets=None
|
||||
) -> None:
|
||||
"""Decode or target-verify; an extend batch passes ``extend_offsets``, the int32
|
||||
(first row, row count) of each request."""
|
||||
from sglang.kernels.ops.attention.dsv4.fp4_indexer_rope import (
|
||||
index_k_norm_rope_pack_store,
|
||||
)
|
||||
from sglang.kernels.ops.attention.dsv4.low_ratio_compress import (
|
||||
c1_decode_norm_rope_store,
|
||||
c2_decode_norm_rope_store,
|
||||
c2_prefill_norm_rope_store,
|
||||
)
|
||||
|
||||
pool = self.token_to_kv_pool
|
||||
@@ -2911,7 +2954,7 @@ class DeepseekV4AttnBackend(
|
||||
# CompressStatePool stores each request's pending pairs in a position ring.
|
||||
# KVAndScore rows use | kv | score |, addressed as req * ring_size + pos % ring_size.
|
||||
state = pool.get_attention_compress_states(layer_id)
|
||||
latent = c2_decode_norm_rope_store(
|
||||
args = (
|
||||
compressor.project_fused(x),
|
||||
state.kv_score_buffer.kv_score,
|
||||
compressor.norm.weight.data,
|
||||
@@ -2921,11 +2964,23 @@ class DeepseekV4AttnBackend(
|
||||
compressor.norm.eps,
|
||||
freqs_cis,
|
||||
kv_cache,
|
||||
page_size=page_size,
|
||||
ring_size=state.ring_size,
|
||||
draft_len=draft_len,
|
||||
layout=kv_layout,
|
||||
)
|
||||
if extend_offsets is None:
|
||||
latent = c2_decode_norm_rope_store(
|
||||
*args,
|
||||
page_size=page_size,
|
||||
ring_size=state.ring_size,
|
||||
draft_len=draft_len,
|
||||
layout=kv_layout,
|
||||
)
|
||||
else:
|
||||
latent = c2_prefill_norm_rope_store(
|
||||
*args,
|
||||
*extend_offsets,
|
||||
page_size=page_size,
|
||||
ring_size=state.ring_size,
|
||||
layout=kv_layout,
|
||||
)
|
||||
out_loc = core.c2_out_loc
|
||||
|
||||
indexer = layer.indexer
|
||||
@@ -3084,7 +3139,6 @@ class DeepseekV4AttnBackend(
|
||||
is_source = layer.indexer.is_candidate_source
|
||||
is_consumer = layer.indexer.uses_candidates
|
||||
ratio = layer.compress_ratio
|
||||
out = self._get_low_ratio_selection(ratio)
|
||||
published = self.forward_metadata.candidate_metadata
|
||||
mode = forward_batch.forward_mode
|
||||
if mode.is_decode() or mode.is_target_verify():
|
||||
@@ -3099,30 +3153,31 @@ class DeepseekV4AttnBackend(
|
||||
is_source = is_consumer = False
|
||||
|
||||
inputs = self._make_low_ratio_decode_indexer_inputs(
|
||||
layer, x, q_lora, req, pos, mode
|
||||
layer, x, q_lora, req, pos, forward_batch
|
||||
)
|
||||
if is_source:
|
||||
published = self.decode_candidates.publish_decode(inputs, out)
|
||||
published = self.decode_candidates.publish_decode(inputs)
|
||||
self._publish_candidate_metadata(published)
|
||||
elif is_consumer:
|
||||
self.decode_candidates.consume_decode(inputs, published, out)
|
||||
self.decode_candidates.consume_decode(inputs, published)
|
||||
else:
|
||||
self.full_topk_indexer.topk_decode(inputs, out)
|
||||
self.full_topk_indexer.topk_decode(inputs)
|
||||
else:
|
||||
inputs = self._make_low_ratio_prefill_indexer_inputs(
|
||||
layer, x, q_lora, req, pos, forward_batch, rows_per_request
|
||||
)
|
||||
if is_source:
|
||||
published = self.prefill_candidates.publish_prefill(inputs, out)
|
||||
published = self.prefill_candidates.publish_prefill(inputs)
|
||||
self._publish_candidate_metadata(published)
|
||||
elif is_consumer:
|
||||
self.prefill_candidates.consume_prefill(inputs, published, out)
|
||||
self.prefill_candidates.consume_prefill(inputs, published)
|
||||
else:
|
||||
self.full_topk_indexer.topk_prefill(inputs, out)
|
||||
self.full_topk_indexer.topk_prefill(inputs)
|
||||
|
||||
def _low_ratio_index_topk_captured(self, layer, projected_q, projected_w) -> None:
|
||||
ratio = layer.compress_ratio
|
||||
indexer = layer.indexer
|
||||
core = self.forward_metadata.core_metadata
|
||||
metadata = (
|
||||
self.forward_metadata.c1_indexer_metadata
|
||||
if ratio == 1
|
||||
@@ -3141,18 +3196,14 @@ class DeepseekV4AttnBackend(
|
||||
q=projected_q,
|
||||
weights=projected_w,
|
||||
paged_metadata=metadata,
|
||||
out_page_indices=core.sparse_page_indices(ratio),
|
||||
out_raw_indices=core.sparse_raw_indices(ratio),
|
||||
)
|
||||
out = self._get_low_ratio_selection(ratio)
|
||||
self.full_topk_indexer.topk_prefill_captured(inputs, out)
|
||||
self.full_topk_indexer.topk_prefill_captured(inputs)
|
||||
|
||||
def _get_low_ratio_selection(self, compress_ratio: int) -> Selection:
|
||||
core = self.forward_metadata.core_metadata
|
||||
return Selection(
|
||||
page_indices=core.sparse_page_indices(compress_ratio),
|
||||
raw_indices=core.sparse_raw_indices(compress_ratio),
|
||||
)
|
||||
|
||||
def _make_low_ratio_decode_indexer_inputs(self, layer, x, q_lora, req, pos, mode):
|
||||
def _make_low_ratio_decode_indexer_inputs(
|
||||
self, layer, x, q_lora, req, pos, forward_batch
|
||||
):
|
||||
ratio = layer.compress_ratio
|
||||
metadata = (
|
||||
self.forward_metadata.c1_indexer_metadata
|
||||
@@ -3160,6 +3211,24 @@ class DeepseekV4AttnBackend(
|
||||
else self.forward_metadata.c2_indexer_metadata
|
||||
)
|
||||
assert metadata is not None, f"no decode indexer metadata for {ratio = }"
|
||||
group_size = 1
|
||||
if (
|
||||
envs.SGLANG_OPT_DSV41_SM90_GROUPED_INDEXER.get()
|
||||
and forward_batch.forward_mode.is_target_verify()
|
||||
and not self.is_dspark_draft
|
||||
and read_ragged_verify_mode() is RaggedVerifyMode.STATIC
|
||||
and self.forward_metadata.late_layer_tail is None
|
||||
and self.speculative_num_draft_tokens is not None
|
||||
and self.speculative_num_draft_tokens > 1
|
||||
and forward_batch.spec_info.draft_token_num
|
||||
== self.speculative_num_draft_tokens
|
||||
and req.shape[0]
|
||||
== forward_batch.batch_size * self.speculative_num_draft_tokens
|
||||
and x.shape[0] == q_lora.shape[0] == req.shape[0]
|
||||
):
|
||||
# Only static verify guarantees fixed request-major groups.
|
||||
# Do not read GPU request ids or lengths during graph capture.
|
||||
group_size = self.speculative_num_draft_tokens
|
||||
return DecodeInputs(
|
||||
indexer=layer.indexer,
|
||||
layer_id=layer.layer_id,
|
||||
@@ -3170,7 +3239,11 @@ class DeepseekV4AttnBackend(
|
||||
positions=pos,
|
||||
req_rows=req,
|
||||
paged_metadata=metadata,
|
||||
is_verify=mode.is_target_verify(),
|
||||
is_verify=forward_batch.forward_mode.is_target_verify(),
|
||||
group_size=group_size,
|
||||
out_page_indices=self.forward_metadata.core_metadata.sparse_page_indices(
|
||||
ratio
|
||||
),
|
||||
)
|
||||
|
||||
def _make_low_ratio_prefill_indexer_inputs(
|
||||
@@ -3183,9 +3256,10 @@ class DeepseekV4AttnBackend(
|
||||
forward_batch,
|
||||
rows_per_request,
|
||||
) -> PrefillInputs:
|
||||
core = self.forward_metadata.core_metadata
|
||||
tail = self.forward_metadata.late_layer_tail
|
||||
if rows_per_request is not None:
|
||||
rows_per_request_device = torch.tensor(
|
||||
rows_per_request_device = async_h2d(
|
||||
rows_per_request, dtype=torch.int32, device=x.device
|
||||
)
|
||||
elif tail is not None:
|
||||
@@ -3204,10 +3278,29 @@ class DeepseekV4AttnBackend(
|
||||
positions=pos,
|
||||
req_rows=req,
|
||||
req_pool_indices=forward_batch.req_pool_indices,
|
||||
kv_page_table=self.forward_metadata.core_metadata.page_table,
|
||||
kv_page_table=core.page_table,
|
||||
seq_lens_cpu=_as_int_list(forward_batch.seq_lens_cpu),
|
||||
rows_per_request=rows_per_request,
|
||||
rows_per_request_device=rows_per_request_device,
|
||||
out_raw_indices=core.sparse_raw_indices(layer.compress_ratio),
|
||||
out_page_indices=(
|
||||
core.sparse_page_indices(layer.compress_ratio)
|
||||
if self._low_ratio_prefill_reads_page_indices(forward_batch)
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
def _low_ratio_prefill_reads_page_indices(self, forward_batch: ForwardBatch):
|
||||
# The negation of the sparse-prefill gate in forward(), which reads only
|
||||
# raw_indices; the query-count clause is dropped, so this errs to True.
|
||||
return (
|
||||
self.trtllm_attn
|
||||
or not forward_batch.forward_mode.is_extend_without_speculative()
|
||||
or get_platform().is_sm120
|
||||
or self.tail_forward_metadata is not None
|
||||
or self.forward_metadata.late_layer_tail is not None
|
||||
or self.token_to_kv_pool.request_window is not None
|
||||
or not envs.SGLANG_OPT_FLASHMLA_SPARSE_PREFILL.get()
|
||||
)
|
||||
|
||||
def _publish_candidate_metadata(self, published: Optional[CandidateMetadata]):
|
||||
@@ -3307,7 +3400,7 @@ class DeepseekV4AttnBackend(
|
||||
compress_ratio
|
||||
)
|
||||
|
||||
swa_kv_page_size = token_to_kv_pool.swa_kv_pool.page_size
|
||||
swa_kv_page_size = token_to_kv_pool.get_swa_key_page_size()
|
||||
assert swa_k_cache.ndim == 2
|
||||
# The kernel detects each cache's format from the last dim of this
|
||||
# view: 584 (V4), 528 (V4.1 fp8) or 288 (V4.1 fp4, extra cache only).
|
||||
@@ -3550,20 +3643,38 @@ class DeepseekV4AttnBackend(
|
||||
compress_ratio, core_attn_metadata, extra_page_size
|
||||
)
|
||||
n_compressed = flat_token_ids.shape[0]
|
||||
workspace = self.sparse_prefill_workspace.get(
|
||||
n_compressed + cache.swa_token_ids.shape[0]
|
||||
reuse_compressed = compress_ratio in (1, 2) and is_cp_active(forward_batch)
|
||||
workspace_pool = (
|
||||
self.shared_compressed_prefill_workspaces[compress_ratio]
|
||||
if reuse_compressed
|
||||
else self.sparse_prefill_workspace
|
||||
)
|
||||
workspace = workspace_pool.get(n_compressed + cache.swa_token_ids.shape[0])
|
||||
compressed_slice = workspace[:n_compressed]
|
||||
swa_slice = workspace[n_compressed:]
|
||||
|
||||
if compressed_slice is not None:
|
||||
dequantize_k_cache_paged(
|
||||
extra_k_cache,
|
||||
flat_token_ids,
|
||||
page_size=extra_page_size,
|
||||
out=compressed_slice,
|
||||
layout=token_to_kv_pool.get_extra_key_layout(layer_id),
|
||||
)
|
||||
source_key = None
|
||||
if reuse_compressed:
|
||||
source_layer = token_to_kv_pool.source_layer_of(layer_id)
|
||||
source_key = (source_layer, workspace.data_ptr())
|
||||
gather = cache.compressed[compress_ratio]
|
||||
# A source layer may have just updated its cache in place. Consumer
|
||||
# layers only reuse the compressed prefix; their top-k and SWA stay live.
|
||||
if (
|
||||
source_key is None
|
||||
or layer_id == source_key[0]
|
||||
or gather.dequantized_source != source_key
|
||||
):
|
||||
dequantize_k_cache_paged(
|
||||
extra_k_cache,
|
||||
flat_token_ids,
|
||||
page_size=extra_page_size,
|
||||
out=compressed_slice,
|
||||
layout=token_to_kv_pool.get_extra_key_layout(layer_id),
|
||||
)
|
||||
if source_key is not None:
|
||||
gather.dequantized_source = source_key
|
||||
dequantize_k_cache_paged(
|
||||
token_to_kv_pool.get_swa_key_buffer_radix(layer_id),
|
||||
cache.swa_token_ids,
|
||||
|
||||
@@ -19,7 +19,7 @@ from sglang.kernels.ops.gemm.bf16_fp32 import linear_bf16_fp32
|
||||
from sglang.kernels.ops.layernorm.rmsnorm_fp32 import rmsnorm_fp32
|
||||
from sglang.srt.layers.linear import ReplicatedLinear
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.utils import add_prefix
|
||||
from sglang.srt.utils import add_prefix, is_gfx95_supported
|
||||
|
||||
|
||||
def _rope_fq4(x, freqs, rope_dim, *, compressed_kv=False):
|
||||
@@ -87,12 +87,14 @@ def rope_tail(
|
||||
|
||||
|
||||
def fused_low_ratio_compress_supported() -> bool:
|
||||
"""The fused c1 / c2 / index-K decode kernels pack fp4 with
|
||||
`cvt.rn.satfinite.e2m1x2`, an sm100+ instruction; the answer also fixes the
|
||||
ratio-2 weight layout (`wkv_gate`, or `wkv` plus `wgate`)."""
|
||||
if not torch.cuda.is_available() or torch.version.hip is not None:
|
||||
"""The fused c1 / c2 / index-K decode kernels need sm90+ on CUDA or gfx950 on
|
||||
ROCm; the answer also fixes the ratio-2 weight layout (`wkv_gate`, or `wkv`
|
||||
plus `wgate`)."""
|
||||
if not torch.cuda.is_available():
|
||||
return False
|
||||
return torch.cuda.get_device_capability()[0] >= 10
|
||||
if torch.version.hip is not None:
|
||||
return is_gfx95_supported()
|
||||
return torch.cuda.get_device_capability()[0] >= 9
|
||||
|
||||
|
||||
class DeepseekV41Compressor(nn.Module):
|
||||
|
||||
@@ -78,10 +78,11 @@ def use_dsv4_q8kv8_sparse_prefill(dsv4_prefill_backend: str = "auto") -> bool:
|
||||
class SparsePrefillWorkspace:
|
||||
"""Backend-owned scratch storage for sparse prefill KV dequantization.
|
||||
|
||||
The workspace contents are fully overwritten before every attention call,
|
||||
so token buckets and compression ratios can safely share one buffer. Sparse
|
||||
prefill executes eagerly and serially on the supported paths, which makes it
|
||||
safe to replace the scratch allocation when a larger extent is needed.
|
||||
Callers normally overwrite the entire workspace. Shared compressed-KV
|
||||
callers keep separate workspaces per ratio and track prefix validity in the
|
||||
per-forward gather cache, including the allocation address. Sparse prefill
|
||||
executes eagerly and serially on the supported paths, so the allocation can
|
||||
be replaced when a larger extent is needed.
|
||||
"""
|
||||
|
||||
def __init__(self, device: torch.device):
|
||||
@@ -135,8 +136,8 @@ def combine_topk_swa_indices(
|
||||
|
||||
Returns int32 indices [num_tokens, padded_topk_swa] and per-token scanned-prefix
|
||||
lengths, including -1 entries skipped by attention. Width is padded to 128.
|
||||
Preallocated out_indices must contain -1 outside the written prefix;
|
||||
reuse requires chunk-invariant scanned-prefix lengths. out_lens is overwritten.
|
||||
Rows are written whole and -1 padded, so preallocated out_indices / out_lens
|
||||
are overwritten entirely; rows past the last request are -1 with length 0.
|
||||
"""
|
||||
assert topk_indices.dtype == torch.int32
|
||||
assert query_start_loc.dtype == torch.int32
|
||||
@@ -154,9 +155,9 @@ def combine_topk_swa_indices(
|
||||
num_reqs = seq_lens.shape[0]
|
||||
combined_topk = combined_topk_width(topk, window_size)
|
||||
if out_indices is None:
|
||||
combined_indices = torch.full(
|
||||
# The kernel writes whole -1 padded rows.
|
||||
combined_indices = torch.empty(
|
||||
(num_tokens, combined_topk),
|
||||
-1,
|
||||
dtype=torch.int32,
|
||||
device=topk_indices.device,
|
||||
)
|
||||
@@ -165,7 +166,7 @@ def combine_topk_swa_indices(
|
||||
assert out_indices.dtype == torch.int32
|
||||
combined_indices = out_indices
|
||||
if out_lens is None:
|
||||
combined_lens = torch.zeros(
|
||||
combined_lens = torch.empty(
|
||||
num_tokens, dtype=torch.int32, device=topk_indices.device
|
||||
)
|
||||
else:
|
||||
@@ -173,8 +174,10 @@ def combine_topk_swa_indices(
|
||||
assert out_lens.dtype == torch.int32
|
||||
combined_lens = out_lens
|
||||
|
||||
NUM_WORKERS = 128
|
||||
_combine_topk_swa_indices_kernel[(num_reqs, NUM_WORKERS)](
|
||||
if num_tokens == 0:
|
||||
return combined_indices, combined_lens
|
||||
block_t = 2 # Measured on B200 with num_warps=2, 16384 tokens, bs 1 to 64.
|
||||
_combine_topk_swa_indices_kernel[(triton.cdiv(num_tokens, block_t),)](
|
||||
combined_indices,
|
||||
combined_indices.stride(0),
|
||||
combined_lens,
|
||||
@@ -186,10 +189,16 @@ def combine_topk_swa_indices(
|
||||
gather_lens,
|
||||
compressed_base,
|
||||
swa_base,
|
||||
num_tokens,
|
||||
num_reqs,
|
||||
top_k=topk,
|
||||
COMPRESS_RATIO=compress_ratio,
|
||||
WINDOW_SIZE=window_size,
|
||||
PADDED_TOP_K=triton.next_power_of_2(topk_indices.shape[-1]),
|
||||
WIDTH=combined_topk,
|
||||
PADDED_WIDTH=triton.next_power_of_2(combined_topk),
|
||||
BLOCK_T=block_t,
|
||||
SEARCH_STEPS=max(num_reqs, 1).bit_length(),
|
||||
num_warps=2,
|
||||
)
|
||||
return combined_indices, combined_lens
|
||||
|
||||
@@ -275,14 +284,18 @@ class CompressedGather:
|
||||
# chunk-invariant per request; subsequent layers only overwrite that prefix.
|
||||
combined_indices: Optional[torch.Tensor] = None
|
||||
combined_lens: Optional[torch.Tensor] = None
|
||||
# Valid only for this forward's gather layout. Each ratio has its own
|
||||
# workspace; its compressed prefix survives consumer layers' SWA writes.
|
||||
dequantized_source: Optional[tuple[int, int]] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class SparsePrefillChunkCache:
|
||||
"""Cache prefill-chunk metadata shared across layers.
|
||||
|
||||
Fields depend on request/token mappings and compressed page tables, not
|
||||
per-layer k_cache; per-layer top-k combinations are recomputed into reused
|
||||
Gather layouts depend on request/token mappings and compressed page tables.
|
||||
Shared-source dequantization keys live only for this forward; per-layer
|
||||
top-k combinations are recomputed into reused
|
||||
buffers.
|
||||
"""
|
||||
|
||||
|
||||
@@ -17,7 +17,6 @@ from .types import (
|
||||
DecodeInputs,
|
||||
PrefillCandidates,
|
||||
PrefillInputs,
|
||||
Selection,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -31,7 +30,6 @@ __all__ = [
|
||||
"FullTopKIndexer",
|
||||
"PrefillCandidates",
|
||||
"PrefillInputs",
|
||||
"Selection",
|
||||
"has_dense_fp4_indexer",
|
||||
"is_sm100_or_newer",
|
||||
"make_candidate_indexer",
|
||||
@@ -104,6 +102,10 @@ def make_candidate_indexer(
|
||||
candidate_topk_blocks=candidate_topk_blocks,
|
||||
candidate_block_size=candidate_block_size,
|
||||
use_deep_gemm_prefill=use_deep_gemm_prefill,
|
||||
use_compact_candidates=(
|
||||
torch.version.cuda is not None
|
||||
and torch.cuda.get_device_capability()[0] == 9
|
||||
),
|
||||
)
|
||||
# Without candidate blocks no layer is a candidate source or consumer.
|
||||
if not is_sm100_or_newer() or candidate_topk_blocks <= 0:
|
||||
|
||||
@@ -3,30 +3,35 @@ its best blocks, a consumer takes its top-k among them."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, List, Optional, Tuple
|
||||
from typing import TYPE_CHECKING, List, Optional, Tuple, Union
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.attention.dsv4.candidate_blocks import (
|
||||
amax_topk_blocks,
|
||||
candidate_block_mask,
|
||||
select_candidate_block_ids,
|
||||
topk_among_blocks,
|
||||
)
|
||||
|
||||
from .scoring import (
|
||||
DeepGEMMPrefillData,
|
||||
PagedDecodeScores,
|
||||
decode_scores,
|
||||
get_deep_gemm_prefill_data,
|
||||
prefill_requests,
|
||||
score_tiles,
|
||||
select_compact_prefill,
|
||||
select_decode,
|
||||
write_decode,
|
||||
write_prefill,
|
||||
)
|
||||
from .sm90_decode import CandidateBlocks, try_sm90_decode
|
||||
from .types import (
|
||||
CandidateMetadata,
|
||||
DecodeInputs,
|
||||
PrefillInputs,
|
||||
Selection,
|
||||
get_tail_row_indices,
|
||||
)
|
||||
|
||||
@@ -40,6 +45,8 @@ class BlockIds(CandidateMetadata, msgspec.Struct):
|
||||
blocks: torch.Tensor
|
||||
# prefill: query rows of each request in row order, for the late-layer tail
|
||||
rows_per_request: Optional[List[int]] = None
|
||||
# Dense Hopper decode reuses this mask; compact consumers only need blocks.
|
||||
decode_mask: Optional[torch.Tensor] = None
|
||||
|
||||
def tail(self, rows_per_request: List[int]) -> BlockIds:
|
||||
assert self.rows_per_request is not None, "prefill block ids missing"
|
||||
@@ -60,76 +67,119 @@ class DenseBlocksBackend:
|
||||
candidate_topk_blocks: int,
|
||||
candidate_block_size: int,
|
||||
use_deep_gemm_prefill: bool,
|
||||
use_compact_candidates: bool = False,
|
||||
) -> None:
|
||||
self.token_to_kv_pool = token_to_kv_pool
|
||||
self.req_to_token = req_to_token
|
||||
self.topk_blocks = candidate_topk_blocks
|
||||
self.block_size = candidate_block_size
|
||||
self.use_deep_gemm_prefill = use_deep_gemm_prefill
|
||||
self.use_compact_candidates = (
|
||||
use_compact_candidates and candidate_block_size == 8
|
||||
)
|
||||
|
||||
def publish_prefill(self, inputs: PrefillInputs, out: Selection):
|
||||
def publish_prefill(self, inputs: PrefillInputs):
|
||||
if self.use_deep_gemm_prefill:
|
||||
return self._deep_gemm_publish_prefill(inputs, out)
|
||||
return self._torch_publish_prefill(inputs, out)
|
||||
return self._deep_gemm_publish_prefill(inputs)
|
||||
return self._torch_publish_prefill(inputs)
|
||||
|
||||
def consume_prefill(
|
||||
self,
|
||||
inputs: PrefillInputs,
|
||||
published: Optional[BlockIds],
|
||||
out: Selection,
|
||||
) -> None:
|
||||
if self.use_deep_gemm_prefill:
|
||||
self._deep_gemm_consume_prefill(inputs, published, out)
|
||||
self._deep_gemm_consume_prefill(inputs, published)
|
||||
else:
|
||||
self._torch_consume_prefill(inputs, published, out)
|
||||
self._torch_consume_prefill(inputs, published)
|
||||
|
||||
def publish_decode(self, inputs: DecodeInputs, out: Selection):
|
||||
def publish_decode(self, inputs: DecodeInputs):
|
||||
handled, published = try_sm90_decode(
|
||||
inputs=inputs,
|
||||
token_to_kv_pool=self.token_to_kv_pool,
|
||||
req_to_token=self.req_to_token,
|
||||
is_source=True,
|
||||
)
|
||||
if handled:
|
||||
return published
|
||||
d = decode_scores(
|
||||
inputs=inputs,
|
||||
out=out,
|
||||
token_to_kv_pool=self.token_to_kv_pool,
|
||||
req_to_token=self.req_to_token,
|
||||
)
|
||||
if d is None:
|
||||
return None
|
||||
published = BlockIds(
|
||||
blocks=select_candidate_block_ids(
|
||||
d.scores,
|
||||
d.lens[:, None],
|
||||
topk_blocks=self.topk_blocks,
|
||||
block_size=self.block_size,
|
||||
if isinstance(d, PagedDecodeScores):
|
||||
assert self.block_size == 8
|
||||
blocks = amax_topk_blocks(
|
||||
d.scores, d.lens, (d.lens + 7) // 8, self.topk_blocks
|
||||
)
|
||||
)
|
||||
k = min(inputs.indexer.index_topk, d.lmax)
|
||||
idx = d.scores.topk(k, dim=-1, sorted=False).indices
|
||||
write_decode(out, d, idx)
|
||||
published = BlockIds(
|
||||
blocks=blocks,
|
||||
decode_mask=(
|
||||
None
|
||||
if self.use_compact_candidates
|
||||
else candidate_block_mask(blocks, d.lmax, self.block_size)
|
||||
),
|
||||
)
|
||||
else:
|
||||
published = BlockIds(
|
||||
blocks=select_candidate_block_ids(
|
||||
d.scores,
|
||||
d.lens[:, None],
|
||||
topk_blocks=self.topk_blocks,
|
||||
block_size=self.block_size,
|
||||
)
|
||||
)
|
||||
select_decode(inputs, d, inputs.indexer.index_topk)
|
||||
return published
|
||||
|
||||
def consume_decode(
|
||||
self,
|
||||
inputs: DecodeInputs,
|
||||
published: Optional[BlockIds],
|
||||
out: Selection,
|
||||
published: Optional[Union[BlockIds, CandidateBlocks]],
|
||||
) -> None:
|
||||
d = decode_scores(
|
||||
handled, _ = try_sm90_decode(
|
||||
inputs=inputs,
|
||||
out=out,
|
||||
token_to_kv_pool=self.token_to_kv_pool,
|
||||
req_to_token=self.req_to_token,
|
||||
is_consumer=True,
|
||||
published=published,
|
||||
)
|
||||
if handled:
|
||||
return
|
||||
d = decode_scores(
|
||||
inputs=inputs,
|
||||
token_to_kv_pool=self.token_to_kv_pool,
|
||||
req_to_token=self.req_to_token,
|
||||
candidate_mask=(
|
||||
published.decode_mask
|
||||
if published is not None and not self.use_compact_candidates
|
||||
else None
|
||||
),
|
||||
candidate_blocks=(
|
||||
published.blocks
|
||||
if published is not None and self.use_compact_candidates
|
||||
else None
|
||||
),
|
||||
)
|
||||
if d is None:
|
||||
return
|
||||
assert published is not None and published.blocks.shape[0] == d.bs
|
||||
if isinstance(d, PagedDecodeScores):
|
||||
assert d.candidate_blocks is not None or published.decode_mask is not None
|
||||
select_decode(inputs, d, inputs.indexer.index_topk)
|
||||
return
|
||||
k = min(inputs.indexer.index_topk, d.lmax)
|
||||
idx = topk_among_blocks(
|
||||
d.scores, d.lens, published.blocks, k, block_size=self.block_size
|
||||
)
|
||||
write_decode(out, d, idx.masked_fill(idx < 0, d.lmax))
|
||||
write_decode(inputs, d, idx.masked_fill(idx < 0, d.lmax))
|
||||
|
||||
# ---------- DeepGEMM: flattened-K scores ----------
|
||||
|
||||
def _deep_gemm_publish_prefill(self, inputs: PrefillInputs, out: Selection):
|
||||
out.reset()
|
||||
def _deep_gemm_publish_prefill(self, inputs: PrefillInputs):
|
||||
inputs.reset_outputs()
|
||||
data = get_deep_gemm_prefill_data(inputs, self.req_to_token)
|
||||
if data is None:
|
||||
return None
|
||||
@@ -142,16 +192,15 @@ class DenseBlocksBackend:
|
||||
topk_blocks=self.topk_blocks,
|
||||
block_size=self.block_size,
|
||||
)
|
||||
data.write_selection(selected=selected, out=out)
|
||||
data.write_selection(selected, inputs)
|
||||
return BlockIds(blocks=blocks, rows_per_request=data.rows_per_request)
|
||||
|
||||
def _deep_gemm_consume_prefill(
|
||||
self,
|
||||
inputs: PrefillInputs,
|
||||
published: Optional[BlockIds],
|
||||
out: Selection,
|
||||
) -> None:
|
||||
out.reset()
|
||||
inputs.reset_outputs()
|
||||
data = get_deep_gemm_prefill_data(inputs, self.req_to_token)
|
||||
if data is None:
|
||||
return
|
||||
@@ -165,15 +214,14 @@ class DenseBlocksBackend:
|
||||
blocks=published.blocks,
|
||||
block_size=self.block_size,
|
||||
)
|
||||
data.write_selection(selected=selected, out=out)
|
||||
data.write_selection(selected, inputs)
|
||||
|
||||
# ---------- torch: per-request scores ----------
|
||||
|
||||
def _torch_publish_prefill(self, inputs: PrefillInputs, out: Selection):
|
||||
def _torch_publish_prefill(self, inputs: PrefillInputs):
|
||||
picked = [] # (query rows, their block ids) per scored chunk
|
||||
for request, chunks in prefill_requests(
|
||||
inputs=inputs,
|
||||
out=out,
|
||||
token_to_kv_pool=self.token_to_kv_pool,
|
||||
req_to_token=self.req_to_token,
|
||||
):
|
||||
@@ -190,7 +238,7 @@ class DenseBlocksBackend:
|
||||
)
|
||||
)
|
||||
idx = chunk.scores.topk(request.k, dim=-1, sorted=False).indices
|
||||
write_prefill(out, request, chunk, idx)
|
||||
write_prefill(inputs, request, chunk, idx)
|
||||
width = max((ids.shape[1] for _, ids in picked), default=0)
|
||||
blocks = torch.full(
|
||||
(inputs.positions.shape[0], width),
|
||||
@@ -206,16 +254,19 @@ class DenseBlocksBackend:
|
||||
self,
|
||||
inputs: PrefillInputs,
|
||||
published: Optional[BlockIds],
|
||||
out: Selection,
|
||||
) -> None:
|
||||
assert published is not None
|
||||
for request, chunks in prefill_requests(
|
||||
inputs=inputs,
|
||||
out=out,
|
||||
token_to_kv_pool=self.token_to_kv_pool,
|
||||
req_to_token=self.req_to_token,
|
||||
candidate_blocks=published.blocks if self.use_compact_candidates else None,
|
||||
block_size=self.block_size,
|
||||
):
|
||||
for chunk in chunks:
|
||||
if chunk.candidate_blocks is not None:
|
||||
select_compact_prefill(inputs, request, chunk, self.req_to_token)
|
||||
continue
|
||||
idx = topk_among_blocks(
|
||||
chunk.scores,
|
||||
chunk.lens,
|
||||
@@ -223,7 +274,9 @@ class DenseBlocksBackend:
|
||||
request.k,
|
||||
block_size=self.block_size,
|
||||
)
|
||||
write_prefill(out, request, chunk, idx.masked_fill(idx < 0, request.lc))
|
||||
write_prefill(
|
||||
inputs, request, chunk, idx.masked_fill(idx < 0, request.lc)
|
||||
)
|
||||
|
||||
|
||||
def _publish_prefill_blocks(
|
||||
|
||||
@@ -7,6 +7,10 @@ from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.attention.deep_select import (
|
||||
is_deep_select_supported,
|
||||
topk_page_transform,
|
||||
)
|
||||
from sglang.kernels.ops.attention.dsv4.index_logits import (
|
||||
deep_gemm_fp4_paged_mqa_logits,
|
||||
)
|
||||
@@ -21,14 +25,14 @@ from .scoring import (
|
||||
get_index_k_cache,
|
||||
prefill_requests,
|
||||
quantize_index_q,
|
||||
write_decode,
|
||||
select_decode,
|
||||
write_prefill,
|
||||
)
|
||||
from .sm90_decode import try_sm90_decode
|
||||
from .types import (
|
||||
CapturedPrefillInputs,
|
||||
DecodeInputs,
|
||||
PrefillInputs,
|
||||
Selection,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -48,46 +52,48 @@ class FullTopKIndexer:
|
||||
self.req_to_token = req_to_token
|
||||
self.use_deep_gemm_prefill = use_deep_gemm_prefill
|
||||
self.use_deep_gemm_decode = use_deep_gemm_decode
|
||||
self.use_deep_select_decode = (
|
||||
not use_deep_gemm_decode and is_deep_select_supported()
|
||||
)
|
||||
|
||||
def topk_prefill(self, inputs: PrefillInputs, out: Selection) -> None:
|
||||
def topk_prefill(self, inputs: PrefillInputs) -> None:
|
||||
if self.use_deep_gemm_prefill:
|
||||
self._deep_gemm_prefill(inputs, out)
|
||||
self._deep_gemm_prefill(inputs)
|
||||
else:
|
||||
self._torch_prefill(inputs, out)
|
||||
self._torch_prefill(inputs)
|
||||
|
||||
def topk_prefill_captured(self, inputs: CapturedPrefillInputs, out: Selection):
|
||||
self._deep_gemm_prefill_captured(inputs, out)
|
||||
def topk_prefill_captured(self, inputs: CapturedPrefillInputs) -> None:
|
||||
self._deep_gemm_prefill_captured(inputs)
|
||||
|
||||
def topk_decode(self, inputs: DecodeInputs, out: Selection) -> None:
|
||||
def topk_decode(self, inputs: DecodeInputs) -> None:
|
||||
if self.use_deep_gemm_decode:
|
||||
self._deep_gemm_decode(inputs, out)
|
||||
self._deep_gemm_decode(inputs)
|
||||
else:
|
||||
self._torch_decode(inputs, out)
|
||||
self._torch_decode(inputs)
|
||||
|
||||
def _deep_gemm_prefill(self, inputs: PrefillInputs, out: Selection) -> None:
|
||||
out.reset()
|
||||
def _deep_gemm_prefill(self, inputs: PrefillInputs) -> None:
|
||||
inputs.reset_outputs()
|
||||
data = get_deep_gemm_prefill_data(inputs, self.req_to_token)
|
||||
if data is None:
|
||||
return
|
||||
kv = self.token_to_kv_pool.get_low_ratio_index_k_fp4(
|
||||
inputs.layer_id, data.k_slots
|
||||
)
|
||||
selected = dense_prefill_topk(data, kv, topk=inputs.indexer.index_topk)
|
||||
data.write_selection(selected=selected, out=out)
|
||||
dense_prefill_topk(data, kv, out=inputs.out_raw_indices[: data.num_rows])
|
||||
data.write_page_indices(inputs)
|
||||
|
||||
def _torch_prefill(self, inputs: PrefillInputs, out: Selection) -> None:
|
||||
def _torch_prefill(self, inputs: PrefillInputs) -> None:
|
||||
requests = prefill_requests(
|
||||
inputs=inputs,
|
||||
out=out,
|
||||
token_to_kv_pool=self.token_to_kv_pool,
|
||||
req_to_token=self.req_to_token,
|
||||
)
|
||||
for request, chunks in requests:
|
||||
for chunk in chunks:
|
||||
idx = chunk.scores.topk(request.k, dim=-1, sorted=False).indices
|
||||
write_prefill(out, request, chunk, idx)
|
||||
write_prefill(inputs, request, chunk, idx)
|
||||
|
||||
def _deep_gemm_decode(self, inputs: DecodeInputs, out: Selection) -> None:
|
||||
def _deep_gemm_decode(self, inputs: DecodeInputs) -> None:
|
||||
data = get_deep_gemm_decode_data(inputs, self.token_to_kv_pool)
|
||||
metadata = inputs.paged_metadata
|
||||
if isinstance(metadata.deep_gemm_metadata, list):
|
||||
@@ -107,8 +113,7 @@ class FullTopKIndexer:
|
||||
topk_transform_paged_from_metadata(
|
||||
logits,
|
||||
metadata,
|
||||
out.page_indices,
|
||||
out.raw_indices,
|
||||
inputs.out_page_indices,
|
||||
rows=rows,
|
||||
topk_metadata=(
|
||||
topk_plans[chunk_idx] if topk_plans is not None else None
|
||||
@@ -126,25 +131,39 @@ class FullTopKIndexer:
|
||||
)
|
||||
# TODO(dark): add bf16 topk
|
||||
topk_transform_paged_from_metadata(
|
||||
logits, metadata, out.page_indices, out.raw_indices
|
||||
logits, metadata, inputs.out_page_indices, None
|
||||
)
|
||||
|
||||
def _torch_decode(self, inputs: DecodeInputs, out: Selection) -> None:
|
||||
def _torch_decode(self, inputs: DecodeInputs) -> None:
|
||||
handled, _ = try_sm90_decode(
|
||||
inputs=inputs,
|
||||
token_to_kv_pool=self.token_to_kv_pool,
|
||||
req_to_token=self.req_to_token,
|
||||
)
|
||||
if handled:
|
||||
return
|
||||
d = decode_scores(
|
||||
inputs=inputs,
|
||||
out=out,
|
||||
token_to_kv_pool=self.token_to_kv_pool,
|
||||
req_to_token=self.req_to_token,
|
||||
)
|
||||
if d is None:
|
||||
return
|
||||
k = min(inputs.indexer.index_topk, d.lmax)
|
||||
idx = d.scores.topk(k, dim=-1, sorted=False).indices
|
||||
write_decode(out, d, idx)
|
||||
if self.use_deep_select_decode:
|
||||
metadata = inputs.paged_metadata
|
||||
topk_page_transform(
|
||||
d.scores,
|
||||
inputs.indexer.index_topk,
|
||||
page_table=metadata.page_table[: d.bs],
|
||||
page_size=metadata.compressed_page_size,
|
||||
end=d.lens.to(torch.int32),
|
||||
sorted_index=False,
|
||||
output_idx=inputs.out_page_indices[: d.bs],
|
||||
)
|
||||
return
|
||||
select_decode(inputs, d, inputs.indexer.index_topk)
|
||||
|
||||
def _deep_gemm_prefill_captured(
|
||||
self, inputs: CapturedPrefillInputs, out: Selection
|
||||
) -> None:
|
||||
def _deep_gemm_prefill_captured(self, inputs: CapturedPrefillInputs) -> None:
|
||||
indexer = inputs.indexer
|
||||
metadata = inputs.paged_metadata
|
||||
assert indexer.n_local_heads == indexer.n_heads
|
||||
@@ -178,7 +197,7 @@ class FullTopKIndexer:
|
||||
logits,
|
||||
lens[rows],
|
||||
page_table[rows],
|
||||
out.page_indices[rows, :topk],
|
||||
inputs.out_page_indices[rows, :topk],
|
||||
page_size,
|
||||
out.raw_indices[rows, :topk] if out.raw_indices is not None else None,
|
||||
inputs.out_raw_indices[rows, :topk],
|
||||
)
|
||||
|
||||
@@ -4,19 +4,31 @@ a selection back into pool slots."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import itertools
|
||||
from typing import TYPE_CHECKING, Generator, Iterator, List, Optional, Tuple
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.attention.dsv4.fp4_indexer import fp4_index_logits_decode
|
||||
from sglang.kernels.ops.attention.dsv4.index_logits import flat_index_logits_tiles
|
||||
|
||||
from .types import (
|
||||
DecodeInputs,
|
||||
PrefillInputs,
|
||||
Selection,
|
||||
from sglang.kernels.ops.attention.dsv4.fp4_indexer import (
|
||||
finish_paged_indexer_topk,
|
||||
fp4_index_logits_candidates,
|
||||
fp4_index_logits_decode,
|
||||
fp4_index_logits_paged,
|
||||
)
|
||||
from sglang.kernels.ops.attention.dsv4.fp4_indexer_prefill import fused_index_scores
|
||||
from sglang.kernels.ops.attention.dsv4.fp4_indexer_rope import index_q_rope_pack_weights
|
||||
from sglang.kernels.ops.attention.dsv4.index_logits import flat_index_logits_tiles
|
||||
from sglang.kernels.ops.attention.dsv4.topk import (
|
||||
plan_topk_v2,
|
||||
topk_transform_paged_v2,
|
||||
topk_transform_ragged_v2,
|
||||
)
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.runtime_context import get_platform
|
||||
from sglang.srt.utils.common import async_h2d
|
||||
|
||||
from .types import DecodeInputs, PrefillInputs, materialize_prefill_request_rows
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.layers.attention.dsv4.dsv41_sparse import DeepseekV41Indexer
|
||||
@@ -56,18 +68,27 @@ class DeepGEMMPrefillData(msgspec.Struct, frozen=True):
|
||||
(self.num_rows, topk), -1, dtype=torch.int32
|
||||
)
|
||||
|
||||
def write_selection(self, selected: torch.Tensor, out: Selection) -> None:
|
||||
def write_page_indices(self, inputs: PrefillInputs) -> None:
|
||||
"""The pool slots of the request-local ``inputs.out_raw_indices`` rows."""
|
||||
if inputs.out_page_indices is None:
|
||||
return
|
||||
raw = inputs.out_raw_indices[: self.num_rows]
|
||||
columns = (raw + self.request_starts[:, None]).clamp_min(0)
|
||||
inputs.out_page_indices[: self.num_rows, : raw.shape[1]] = torch.where(
|
||||
raw >= 0, self.k_slots[columns], -1
|
||||
)
|
||||
|
||||
def write_selection(self, selected: torch.Tensor, inputs: PrefillInputs) -> None:
|
||||
"""Write flattened-K columns ``selected`` into the selection buffers."""
|
||||
num_tokens, topk = selected.shape
|
||||
unselected = torch.iinfo(torch.int32).max
|
||||
selected = selected.masked_fill(selected < 0, unselected).sort(dim=-1).values
|
||||
chosen = selected != unselected
|
||||
out.page_indices[:num_tokens, :topk] = torch.where(
|
||||
chosen, self.k_slots[selected.clamp_max(self.k_slots.shape[0] - 1)], -1
|
||||
).to(torch.int32)
|
||||
if out.raw_indices is not None:
|
||||
out.raw_indices[:num_tokens, :topk] = torch.where(
|
||||
chosen, selected - self.request_starts[:, None], -1
|
||||
)
|
||||
chosen = selected >= 0
|
||||
if inputs.out_page_indices is not None:
|
||||
inputs.out_page_indices[:num_tokens, :topk] = torch.where(
|
||||
chosen, self.k_slots[selected.clamp_min(0)], -1
|
||||
).to(torch.int32)
|
||||
inputs.out_raw_indices[:num_tokens, :topk] = torch.where(
|
||||
chosen, selected - self.request_starts[:, None], -1
|
||||
)
|
||||
|
||||
|
||||
def quantize_index_q(q: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
@@ -93,38 +114,42 @@ def get_index_k_cache(
|
||||
return k_cache.view(k_cache.shape[0], page_size, 1, 68)
|
||||
|
||||
|
||||
def _index_q_and_weights(
|
||||
*,
|
||||
indexer: DeepseekV41Indexer,
|
||||
x: torch.Tensor,
|
||||
q_lora: torch.Tensor,
|
||||
freqs_cis: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""fp4 index queries ``[rows, heads, 64]`` int8 with ``[rows, heads]`` packed
|
||||
ue8m0 scales, and fp32 head weights ``[rows, heads]``."""
|
||||
rows, heads = x.shape[0], indexer.n_local_heads
|
||||
q, _ = indexer.wq_b(q_lora)
|
||||
q_fp4, q_sf, weights = index_q_rope_pack_weights(
|
||||
q.view(rows, heads, indexer.index_head_dim),
|
||||
torch.view_as_real(freqs_cis).flatten(-2),
|
||||
positions,
|
||||
indexer.head_weights_raw(x),
|
||||
indexer.head_weight_scale,
|
||||
)
|
||||
return q_fp4.view(rows, heads, 64), q_sf.view(rows, heads), weights
|
||||
|
||||
|
||||
def get_deep_gemm_decode_data(
|
||||
inputs: DecodeInputs, token_to_kv_pool: DeepSeekV4TokenToKVPool
|
||||
) -> DeepGEMMDecodeData:
|
||||
indexer = inputs.indexer
|
||||
x, q_lora, pos = inputs.x, inputs.q_lora, inputs.positions
|
||||
# The kernel sums head scores locally, so the indexer heads must be replicated.
|
||||
assert indexer.n_local_heads == indexer.n_heads
|
||||
if (
|
||||
x.is_cuda
|
||||
and torch.version.cuda is not None
|
||||
and x.dtype == torch.bfloat16
|
||||
and indexer.index_head_dim == 128
|
||||
):
|
||||
from sglang.kernels.ops.attention.dsv4.fp4_indexer_rope import (
|
||||
index_q_rope_pack_weights,
|
||||
)
|
||||
|
||||
q, _ = indexer.wq_b(q_lora)
|
||||
q = q.view(q.shape[0], indexer.n_local_heads, indexer.index_head_dim)
|
||||
# The fused pack also computes head_weights(x).float(), same rounding.
|
||||
q_fp4, q_sf, weights = index_q_rope_pack_weights(
|
||||
q,
|
||||
torch.view_as_real(inputs.freqs_cis).flatten(-2),
|
||||
pos,
|
||||
indexer.head_weights_raw(x), # [bs, n_local] bf16, n32k5120
|
||||
indexer.head_weight_scale,
|
||||
)
|
||||
else:
|
||||
q = indexer.queries(q_lora, inputs.freqs_cis[pos])
|
||||
q_fp4, q_sf = quantize_index_q(q)
|
||||
weights = indexer.head_weights(x).float() # [bs, n_local]
|
||||
bs = q.shape[0]
|
||||
q_fp4, q_sf, weights = _index_q_and_weights(
|
||||
indexer=indexer,
|
||||
x=inputs.x,
|
||||
q_lora=inputs.q_lora,
|
||||
freqs_cis=inputs.freqs_cis,
|
||||
positions=inputs.positions,
|
||||
)
|
||||
bs = q_fp4.shape[0]
|
||||
q_fp4 = q_fp4.view(bs, 1, indexer.n_local_heads, 64)
|
||||
q_sf = q_sf.view(bs, 1, indexer.n_local_heads)
|
||||
|
||||
@@ -142,47 +167,44 @@ def get_deep_gemm_prefill_data(
|
||||
) -> Optional[DeepGEMMPrefillData]:
|
||||
"""None when the chunk has no row or no visible compressed position."""
|
||||
ratio = inputs.compress_ratio
|
||||
indexer = inputs.indexer
|
||||
pos = inputs.positions
|
||||
seq_lens_cpu = inputs.seq_lens_cpu
|
||||
assert (
|
||||
seq_lens_cpu is not None
|
||||
inputs.seq_lens_cpu is not None
|
||||
and inputs.rows_per_request is not None
|
||||
and inputs.rows_per_request_device is not None
|
||||
), "the DeepGEMM prefill indexer needs the CPU length vectors"
|
||||
device = pos.device
|
||||
# Visible compressed positions per request; (pos + 1) // ratio bounds each row.
|
||||
lc_per_req = [s // ratio for s in seq_lens_cpu]
|
||||
req_pool_indices = inputs.req_pool_indices.to(torch.int64)
|
||||
slot_chunks, starts, start = [], [], 0
|
||||
for r, lc in enumerate(lc_per_req):
|
||||
starts.append(start)
|
||||
if lc == 0:
|
||||
continue
|
||||
j = torch.arange(lc, device=device)
|
||||
slot_chunks.append(
|
||||
req_to_token[req_pool_indices[r], j * ratio].to(torch.int64) // ratio
|
||||
)
|
||||
start += lc
|
||||
lens_per_request = [s // ratio for s in inputs.seq_lens_cpu]
|
||||
columns = sum(lens_per_request)
|
||||
num_tokens = pos.shape[0]
|
||||
if not slot_chunks or num_tokens == 0:
|
||||
if columns == 0 or num_tokens == 0:
|
||||
return None
|
||||
k_slots = torch.cat(slot_chunks)
|
||||
q = indexer.queries(inputs.q_lora, inputs.freqs_cis[pos])
|
||||
q_fp4, q_sf = quantize_index_q(q)
|
||||
weights = indexer.head_weights(inputs.x).float()
|
||||
compress_lens = ((pos + 1) // ratio).to(torch.int32)
|
||||
request_starts = torch.repeat_interleave(
|
||||
torch.tensor(starts, dtype=torch.int32, device=device),
|
||||
inputs.rows_per_request_device.to(torch.int64),
|
||||
output_size=num_tokens,
|
||||
starts = list(itertools.accumulate(lens_per_request, initial=0))[:-1]
|
||||
lens = async_h2d(lens_per_request, dtype=torch.int64, device=device)
|
||||
starts_dev = async_h2d(starts, dtype=torch.int64, device=device)
|
||||
request = torch.repeat_interleave(
|
||||
torch.arange(len(lens_per_request), device=device), lens, output_size=columns
|
||||
)
|
||||
position = torch.arange(columns, device=device) - starts_dev[request]
|
||||
pool_rows = inputs.req_pool_indices.to(torch.int64)[request]
|
||||
q_fp4, q_sf, weights = _index_q_and_weights(
|
||||
indexer=inputs.indexer,
|
||||
x=inputs.x,
|
||||
q_lora=inputs.q_lora,
|
||||
freqs_cis=inputs.freqs_cis,
|
||||
positions=pos,
|
||||
)
|
||||
return DeepGEMMPrefillData(
|
||||
k_slots=k_slots,
|
||||
request_starts=request_starts,
|
||||
lens_per_request=lc_per_req,
|
||||
k_slots=req_to_token[pool_rows, position * ratio].to(torch.int64) // ratio,
|
||||
request_starts=torch.repeat_interleave(
|
||||
starts_dev.to(torch.int32),
|
||||
inputs.rows_per_request_device.to(torch.int64),
|
||||
output_size=num_tokens,
|
||||
),
|
||||
lens_per_request=lens_per_request,
|
||||
rows_per_request=inputs.rows_per_request,
|
||||
compress_lens=compress_lens,
|
||||
compress_lens=((pos + 1) // ratio).to(torch.int32),
|
||||
q_fp4=q_fp4,
|
||||
q_sf=q_sf,
|
||||
weights=weights,
|
||||
@@ -211,22 +233,22 @@ def dense_prefill_topk(
|
||||
data: DeepGEMMPrefillData,
|
||||
kv: Tuple[torch.Tensor, torch.Tensor],
|
||||
*,
|
||||
topk: int,
|
||||
) -> torch.Tensor:
|
||||
"""Flattened-K columns, ``-1`` padded, unordered."""
|
||||
out: torch.Tensor,
|
||||
) -> None:
|
||||
"""Request-local compressed positions into ``out``, ``-1`` padded, unordered."""
|
||||
from sglang.kernels.ops.attention.dsv4 import topk_transform_ragged_v2
|
||||
|
||||
selected = data.empty_selection(topk)
|
||||
# Score columns are request-relative, so a zero offset emits request-local positions.
|
||||
zero_offsets = torch.zeros_like(data.request_starts)
|
||||
for tile, logits in score_tiles(data, kv, width_align=4):
|
||||
topk_transform_ragged_v2(
|
||||
logits,
|
||||
data.compress_lens[tile],
|
||||
out_offsets=data.request_starts[tile],
|
||||
out_indices=selected[tile],
|
||||
out_offsets=zero_offsets[tile],
|
||||
out_indices=out[tile],
|
||||
)
|
||||
# Free this tile's logits before the generator scores the next one.
|
||||
del logits
|
||||
return selected
|
||||
|
||||
|
||||
# ---------- torch ----------
|
||||
@@ -249,7 +271,8 @@ class ChunkScores(msgspec.Struct, frozen=True):
|
||||
rows: slice # into the request's rows
|
||||
tok: torch.Tensor
|
||||
lens: torch.Tensor
|
||||
scores: torch.Tensor # [rows, lc] bf16, -inf past each row's length
|
||||
scores: torch.Tensor # [rows, lc], or compact candidate scores
|
||||
candidate_blocks: Optional[torch.Tensor] = None
|
||||
|
||||
|
||||
class DecodeScores(msgspec.Struct, frozen=True):
|
||||
@@ -263,17 +286,17 @@ class DecodeScores(msgspec.Struct, frozen=True):
|
||||
def prefill_requests(
|
||||
*,
|
||||
inputs: PrefillInputs,
|
||||
out: Selection,
|
||||
token_to_kv_pool: DeepSeekV4TokenToKVPool,
|
||||
req_to_token: torch.Tensor,
|
||||
candidate_blocks: Optional[torch.Tensor] = None,
|
||||
block_size: int = 8,
|
||||
) -> Iterator[tuple[RequestScores, Iterator[ChunkScores]]]:
|
||||
"""Requests with nothing visible yet are skipped."""
|
||||
pool = token_to_kv_pool
|
||||
ratio = inputs.compress_ratio
|
||||
indexer = inputs.indexer
|
||||
req, pos = inputs.req_rows, inputs.positions
|
||||
assert req is not None, "the torch indexer needs the batch's rows"
|
||||
out.reset()
|
||||
req, pos = materialize_prefill_request_rows(inputs), inputs.positions
|
||||
inputs.reset_outputs()
|
||||
q = indexer.queries(inputs.q_lora, inputs.freqs_cis[pos])
|
||||
weights = indexer.head_weights(inputs.x)
|
||||
# A compressed position is visible once the query has passed its last token.
|
||||
@@ -295,7 +318,13 @@ def prefill_requests(
|
||||
yield (
|
||||
request,
|
||||
_score_chunks(
|
||||
indexer=indexer, q=q, weights=weights, index_k=index_k, request=request
|
||||
indexer=indexer,
|
||||
q=q,
|
||||
weights=weights,
|
||||
index_k=index_k,
|
||||
request=request,
|
||||
candidate_blocks=candidate_blocks,
|
||||
block_size=block_size,
|
||||
),
|
||||
)
|
||||
|
||||
@@ -307,49 +336,113 @@ def _score_chunks(
|
||||
weights: torch.Tensor,
|
||||
index_k: torch.Tensor,
|
||||
request: RequestScores,
|
||||
candidate_blocks: Optional[torch.Tensor] = None,
|
||||
block_size: int = 8,
|
||||
) -> Iterator[ChunkScores]:
|
||||
lc, j = request.lc, request.columns
|
||||
# The torch prefill indexer flag asks for the torch oracle, not this kernel.
|
||||
fused = (
|
||||
q.is_cuda
|
||||
and envs.SGLANG_OPT_USE_FUSED_DSV41_INDEXER_SCORES.get()
|
||||
and not envs.SGLANG_DSV41_TORCH_PREFILL_INDEXER.get()
|
||||
)
|
||||
# Chunk rows so the [rows, heads, lc] bf16 scores stay under the budget.
|
||||
rows_per_chunk = max(1, _TORCH_SCORE_BUDGET_BYTES // (q.shape[1] * lc * 2))
|
||||
# The fused kernel holds only the fp32 [rows, lc] scores; at a quarter of
|
||||
# the budget its [rows, lc] copies downstream stay under the torch peak.
|
||||
row_bytes = lc * 16 if fused else q.shape[1] * lc * 2
|
||||
rows_per_chunk = max(1, _TORCH_SCORE_BUDGET_BYTES // row_bytes)
|
||||
if candidate_blocks is not None:
|
||||
from sglang.kernels.ops.attention.dsv4.candidate_bf16_mqa import (
|
||||
can_use_candidate_bf16_mqa,
|
||||
candidate_bf16_mqa_logits,
|
||||
)
|
||||
|
||||
if not can_use_candidate_bf16_mqa(q, index_k, weights, block_size):
|
||||
candidate_blocks = None
|
||||
if candidate_blocks is not None:
|
||||
# No [rows, heads, lc] or per-query K tensor on this path. Bound the
|
||||
# FP32 output and the logical-index/top-k temporaries instead.
|
||||
width = candidate_blocks.shape[1] * 8
|
||||
rows_per_chunk = max(1, _TORCH_SCORE_BUDGET_BYTES // max(1, width * 32))
|
||||
for start in range(0, request.tok.numel(), rows_per_chunk):
|
||||
rows = slice(start, start + rows_per_chunk)
|
||||
tok_c, lens_c = request.tok[rows], request.lens[rows]
|
||||
s = indexer.scores(q[tok_c], index_k, weights[tok_c])
|
||||
s = s.masked_fill(j[None, :] >= lens_c[:, None], -torch.inf)
|
||||
yield ChunkScores(rows=rows, tok=tok_c, lens=lens_c, scores=s)
|
||||
blocks = None if candidate_blocks is None else candidate_blocks[tok_c]
|
||||
if blocks is not None:
|
||||
s = candidate_bf16_mqa_logits(
|
||||
q[tok_c], index_k, weights[tok_c], blocks, lens_c
|
||||
)
|
||||
elif fused:
|
||||
s = fused_index_scores(q[tok_c], index_k, weights[tok_c], lens_c)
|
||||
else:
|
||||
s = indexer.scores(q[tok_c], index_k, weights[tok_c])
|
||||
s = s.masked_fill(j[None, :] >= lens_c[:, None], -torch.inf)
|
||||
yield ChunkScores(
|
||||
rows=rows, tok=tok_c, lens=lens_c, scores=s, candidate_blocks=blocks
|
||||
)
|
||||
|
||||
|
||||
def write_prefill(
|
||||
out: Selection, request: RequestScores, chunk: ChunkScores, idx: torch.Tensor
|
||||
inputs: PrefillInputs,
|
||||
request: RequestScores,
|
||||
chunk: ChunkScores,
|
||||
idx: torch.Tensor,
|
||||
) -> None:
|
||||
k, lc = request.k, request.lc
|
||||
idx = idx.sort(dim=-1).values
|
||||
reach = idx < chunk.lens[:, None]
|
||||
out.page_indices[chunk.tok, :k] = torch.where(
|
||||
reach, request.slots[idx.clamp_max(lc - 1)], -1
|
||||
).to(torch.int32)
|
||||
if out.raw_indices is not None:
|
||||
out.raw_indices[chunk.tok, :k] = torch.where(reach, idx, -1).to(torch.int32)
|
||||
if inputs.out_page_indices is not None:
|
||||
inputs.out_page_indices[chunk.tok, :k] = torch.where(
|
||||
reach, request.slots[idx.clamp_max(lc - 1)], -1
|
||||
).to(torch.int32)
|
||||
inputs.out_raw_indices[chunk.tok, :k] = torch.where(reach, idx, -1).to(torch.int32)
|
||||
|
||||
|
||||
class PagedDecodeScores(msgspec.Struct, frozen=True):
|
||||
"""Paged decode scores with graph-stable logical capacity ``lmax``.
|
||||
|
||||
Only positions below each device-side ``lens`` are initialized. Top-k
|
||||
consumers must respect those lengths. With ``has_candidate_mask``, masked
|
||||
positions have -inf scores and must be discarded during selection writeback.
|
||||
Compact candidates instead initialize every score column; score_lens is
|
||||
their scan width, while lens remains the causal bound in logical K space.
|
||||
"""
|
||||
|
||||
bs: int
|
||||
lmax: int
|
||||
lens: torch.Tensor
|
||||
scores: torch.Tensor
|
||||
req: torch.Tensor
|
||||
req_to_token: torch.Tensor
|
||||
ratio: int
|
||||
plan: torch.Tensor
|
||||
has_candidate_mask: bool
|
||||
# Compact columns need their own scan lengths; lens stays in logical K space.
|
||||
candidate_blocks: Optional[torch.Tensor] = None
|
||||
score_lens: Optional[torch.Tensor] = None
|
||||
|
||||
|
||||
def decode_scores(
|
||||
*,
|
||||
inputs: DecodeInputs,
|
||||
out: Selection,
|
||||
token_to_kv_pool: DeepSeekV4TokenToKVPool,
|
||||
req_to_token: torch.Tensor,
|
||||
) -> Optional[DecodeScores]:
|
||||
candidate_mask: Optional[torch.Tensor] = None,
|
||||
compute_topk_plan: bool = True,
|
||||
candidate_blocks: Optional[torch.Tensor] = None,
|
||||
) -> Optional[DecodeScores | PagedDecodeScores]:
|
||||
"""None when there is no row, or nothing visible yet."""
|
||||
pool = token_to_kv_pool
|
||||
ratio = inputs.compress_ratio
|
||||
indexer = inputs.indexer
|
||||
req, pos = inputs.req_rows, inputs.positions
|
||||
out.reset()
|
||||
paged = get_platform().is_sm90
|
||||
bs = req.shape[0]
|
||||
assert pos.shape[0] == bs, (
|
||||
f"decode expects one token per request, {pos.shape=} {bs=}"
|
||||
)
|
||||
if bs == 0:
|
||||
inputs.reset_outputs()
|
||||
return None
|
||||
lens = (pos + 1) // ratio
|
||||
metadata = inputs.paged_metadata
|
||||
@@ -357,26 +450,142 @@ def decode_scores(
|
||||
# A capture-time length read would both synchronize and truncate replay.
|
||||
lmax = min(metadata.max_compressed_seq_len, req_to_token.shape[1] // ratio)
|
||||
if lmax == 0:
|
||||
inputs.reset_outputs()
|
||||
return None
|
||||
q = indexer.queries(inputs.q_lora, inputs.freqs_cis[pos])
|
||||
weights = indexer.head_weights(inputs.x)
|
||||
table = pool.get_index_k_with_scale_buffer(inputs.layer_id)
|
||||
if paged:
|
||||
lens = lens.to(torch.int32)
|
||||
if candidate_blocks is not None:
|
||||
assert candidate_mask is None
|
||||
# Valid candidates may be interspersed with padding/causal holes.
|
||||
score_lens = torch.full_like(lens, candidate_blocks.shape[1] * 8)
|
||||
plan = plan_topk_v2(score_lens) if compute_topk_plan else None
|
||||
scores = fp4_index_logits_candidates(
|
||||
q,
|
||||
weights,
|
||||
req,
|
||||
req_to_token,
|
||||
lens,
|
||||
table,
|
||||
table.shape[1] // 68,
|
||||
lmax,
|
||||
ratio,
|
||||
candidate_blocks,
|
||||
)
|
||||
else:
|
||||
score_lens = None
|
||||
# Prepare the plan before scoring, away from its top-k consumer.
|
||||
plan = plan_topk_v2(lens) if compute_topk_plan else None
|
||||
scores = fp4_index_logits_paged(
|
||||
q,
|
||||
weights,
|
||||
req,
|
||||
req_to_token,
|
||||
lens,
|
||||
table,
|
||||
table.shape[1] // 68,
|
||||
lmax,
|
||||
ratio,
|
||||
candidate_mask,
|
||||
)
|
||||
return PagedDecodeScores(
|
||||
bs=bs,
|
||||
lmax=lmax,
|
||||
lens=lens,
|
||||
scores=scores,
|
||||
req=req,
|
||||
req_to_token=req_to_token,
|
||||
ratio=ratio,
|
||||
plan=plan,
|
||||
has_candidate_mask=candidate_mask is not None,
|
||||
candidate_blocks=candidate_blocks,
|
||||
score_lens=score_lens,
|
||||
)
|
||||
assert candidate_blocks is None, "compact paged scoring requires SM90"
|
||||
inputs.reset_outputs()
|
||||
j = torch.arange(lmax, device=pos.device)
|
||||
valid = j[None, :] < lens[:, None]
|
||||
slots = req_to_token[req[:, None], (j * ratio)[None, :]].to(torch.int64) // ratio
|
||||
slots = slots.masked_fill(~valid, 0)
|
||||
table = pool.get_index_k_with_scale_buffer(inputs.layer_id)
|
||||
scores = fp4_index_logits_decode(
|
||||
q, weights, slots, lens, table, table.shape[1] // 68
|
||||
)
|
||||
return DecodeScores(bs=bs, lmax=lmax, lens=lens, slots=slots, scores=scores)
|
||||
|
||||
|
||||
def write_decode(out: Selection, d: DecodeScores, idx: torch.Tensor) -> None:
|
||||
def select_decode(
|
||||
inputs: DecodeInputs, d: DecodeScores | PagedDecodeScores, topk: int
|
||||
) -> None:
|
||||
k = min(topk, d.lmax)
|
||||
if isinstance(d, PagedDecodeScores):
|
||||
assert d.plan is not None, "paged select_decode needs the top-k v2 plan"
|
||||
k = min(k, d.scores.shape[1])
|
||||
if not k:
|
||||
inputs.reset_outputs()
|
||||
return
|
||||
idx = torch.empty((d.bs, k), dtype=torch.int32, device=d.scores.device)
|
||||
scan_lens = d.lens if d.score_lens is None else d.score_lens
|
||||
topk_transform_paged_v2(d.scores, scan_lens, None, idx, 1, d.plan)
|
||||
finish_paged_indexer_topk(
|
||||
idx,
|
||||
d.scores,
|
||||
d.lens,
|
||||
d.req,
|
||||
d.req_to_token,
|
||||
inputs.out_page_indices,
|
||||
None,
|
||||
d.ratio,
|
||||
d.has_candidate_mask,
|
||||
candidate_blocks=d.candidate_blocks,
|
||||
)
|
||||
else:
|
||||
write_decode(inputs, d, d.scores.topk(k, dim=-1, sorted=False).indices)
|
||||
|
||||
|
||||
def write_decode(inputs: DecodeInputs, d: DecodeScores, idx: torch.Tensor) -> None:
|
||||
k = idx.shape[1]
|
||||
idx = idx.sort(dim=-1).values
|
||||
reach = idx < d.lens[:, None]
|
||||
out.page_indices[: d.bs, :k] = torch.where(
|
||||
inputs.out_page_indices[: d.bs, :k] = torch.where(
|
||||
reach, d.slots.gather(1, idx.clamp_max(d.lmax - 1)), -1
|
||||
).to(torch.int32)
|
||||
if out.raw_indices is not None:
|
||||
out.raw_indices[: d.bs, :k] = torch.where(reach, idx, -1).to(torch.int32)
|
||||
|
||||
|
||||
def select_compact_prefill(
|
||||
inputs: PrefillInputs,
|
||||
request: RequestScores,
|
||||
chunk: ChunkScores,
|
||||
req_to_token: torch.Tensor,
|
||||
) -> None:
|
||||
rows, width = chunk.scores.shape
|
||||
k = min(request.k, width)
|
||||
if not rows or not k:
|
||||
return # prefill_requests has reset the output to -1.
|
||||
indices = torch.empty((rows, k), dtype=torch.int32, device=chunk.scores.device)
|
||||
# Scan the entire compact row: padding/causal holes can precede valid blocks.
|
||||
scan_lens = torch.full(
|
||||
(rows,), width, dtype=torch.int32, device=chunk.scores.device
|
||||
)
|
||||
topk_transform_ragged_v2(
|
||||
chunk.scores,
|
||||
scan_lens,
|
||||
out_offsets=torch.zeros_like(scan_lens),
|
||||
out_indices=indices,
|
||||
)
|
||||
finish_paged_indexer_topk(
|
||||
indices,
|
||||
chunk.scores,
|
||||
chunk.lens,
|
||||
materialize_prefill_request_rows(inputs)[chunk.tok]
|
||||
if inputs.out_page_indices is not None
|
||||
else None,
|
||||
req_to_token,
|
||||
inputs.out_page_indices,
|
||||
inputs.out_raw_indices,
|
||||
inputs.compress_ratio,
|
||||
True,
|
||||
candidate_blocks=chunk.candidate_blocks,
|
||||
output_rows=chunk.tok,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,256 @@
|
||||
"""Opt-in SM90 static-verification selection for the V4.1 decode backends."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Optional, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
|
||||
from .types import CandidateMetadata, DecodeInputs
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool
|
||||
|
||||
|
||||
@dataclass
|
||||
class CandidateBlocks(CandidateMetadata):
|
||||
"""Sorted block IDs with device lengths refreshed on each graph replay.
|
||||
|
||||
Ordinary decode consumers can use ``blocks`` directly. Length-aware
|
||||
consumers also use the device lengths; never cache a derived full mask.
|
||||
"""
|
||||
|
||||
blocks: torch.Tensor # [rows, topk_blocks] int32, -1 padding
|
||||
lengths: torch.Tensor # [rows] int32, selected capacity in positions
|
||||
is_prefix: torch.Tensor # [rows] int32, blocks are consecutive from zero
|
||||
width: int # source capacity in compressed positions
|
||||
block_size: int
|
||||
|
||||
|
||||
def _candidate_mask(published, width: int, block_size: int) -> torch.Tensor:
|
||||
"""Adapt a standard block-ID publish to prefix scoring, without caching."""
|
||||
if isinstance(published, CandidateBlocks):
|
||||
from sglang.kernels.ops.attention.dsv4.sm90_length_aware_indexer import (
|
||||
materialize_candidate_mask,
|
||||
)
|
||||
|
||||
assert published.width >= width and published.block_size == block_size
|
||||
return materialize_candidate_mask(published)
|
||||
blocks = published.blocks
|
||||
num_blocks = (width + block_size - 1) // block_size
|
||||
keep = torch.zeros(
|
||||
(*blocks.shape[:-1], num_blocks + 1), dtype=torch.bool, device=blocks.device
|
||||
)
|
||||
ids = blocks.to(torch.int64)
|
||||
ids = ids.masked_fill((ids < 0) | (ids >= num_blocks), num_blocks)
|
||||
keep.scatter_(-1, ids, True)
|
||||
return keep[..., :num_blocks].repeat_interleave(block_size, dim=-1)[..., :width]
|
||||
|
||||
|
||||
def try_sm90_decode(
|
||||
*,
|
||||
inputs: DecodeInputs,
|
||||
token_to_kv_pool: DeepSeekV4TokenToKVPool,
|
||||
req_to_token: torch.Tensor,
|
||||
is_source: bool = False,
|
||||
is_consumer: bool = False,
|
||||
published: Optional[CandidateMetadata] = None,
|
||||
) -> Tuple[bool, Optional[CandidateMetadata]]:
|
||||
"""Return whether selection was handled, plus a source's published blocks.
|
||||
|
||||
Candidate roles come from the caller: graph variants may bypass filtering
|
||||
even for a layer whose indexer normally publishes or consumes candidates.
|
||||
"""
|
||||
ratio = inputs.compress_ratio
|
||||
if not (
|
||||
envs.SGLANG_OPT_DSV41_SM90_GROUPED_INDEXER.get()
|
||||
and inputs.is_verify
|
||||
and inputs.group_size > 1
|
||||
and ratio in (1, 2)
|
||||
):
|
||||
return False, None
|
||||
indexer = inputs.indexer
|
||||
req, pos = inputs.req_rows, inputs.positions
|
||||
bs = req.shape[0]
|
||||
assert pos.shape[0] == bs, (
|
||||
f"decode expects one token per request, {pos.shape=} {bs=}"
|
||||
)
|
||||
# Metadata reserves graph replay capacity; visibility remains on device.
|
||||
lmax = min(
|
||||
inputs.paged_metadata.max_compressed_seq_len, req_to_token.shape[1] // ratio
|
||||
)
|
||||
if bs == 0 or lmax == 0:
|
||||
inputs.reset_outputs()
|
||||
return True, None
|
||||
q = indexer.queries(inputs.q_lora, inputs.freqs_cis[pos])
|
||||
weights = indexer.head_weights(inputs.x)
|
||||
table = token_to_kv_pool.get_index_k_with_scale_buffer(inputs.layer_id)
|
||||
if not (
|
||||
q.is_cuda
|
||||
and torch.version.cuda is not None
|
||||
and torch.cuda.get_device_capability(q.device)[0] == 9
|
||||
and q.dtype == weights.dtype == torch.bfloat16
|
||||
and q.shape[1:] == (32, 128)
|
||||
and q.is_contiguous()
|
||||
and weights.is_contiguous()
|
||||
and req_to_token.dtype == torch.int32
|
||||
and req_to_token.stride(1) == 1
|
||||
and table.dtype == torch.uint8
|
||||
and table.dim() == 2
|
||||
and table.stride(1) == 1
|
||||
):
|
||||
return False, None
|
||||
|
||||
from sglang.kernels.ops.attention.dsv4.candidate_blocks import (
|
||||
select_candidate_block_ids,
|
||||
topk_among_blocks,
|
||||
)
|
||||
|
||||
# Imported only after dispatch to avoid a module-initialization cycle.
|
||||
from .dense_blocks import BlockIds
|
||||
|
||||
inputs.reset_outputs()
|
||||
if is_consumer:
|
||||
assert published is not None and published.blocks.shape[0] == bs
|
||||
use_length_aware = (
|
||||
envs.SGLANG_OPT_DSV41_SM90_LENGTH_AWARE_INDEXER.get()
|
||||
and 0 < indexer.index_topk <= 2048
|
||||
and (
|
||||
not is_source
|
||||
or (
|
||||
0 < indexer.candidate_topk_blocks <= 2048
|
||||
and indexer.candidate_block_size in (1, 2, 4, 8, 16, 32, 64, 128)
|
||||
)
|
||||
)
|
||||
)
|
||||
if use_length_aware:
|
||||
from sglang.kernels.ops.attention.dsv4.sm90_length_aware_indexer import (
|
||||
candidate_blocks,
|
||||
prefix_logits,
|
||||
prepare_candidate_lengths,
|
||||
publish_topk,
|
||||
select_prefix_topk,
|
||||
)
|
||||
|
||||
request = req.to(torch.int64).contiguous()
|
||||
compact = envs.SGLANG_OPT_DSV41_SM90_COMPACT_CANDIDATES.get() and (
|
||||
is_source or is_consumer
|
||||
)
|
||||
candidates = (
|
||||
published
|
||||
if compact and is_consumer and isinstance(published, CandidateBlocks)
|
||||
else None
|
||||
)
|
||||
if compact:
|
||||
visible, score_lens = prepare_candidate_lengths(
|
||||
pos, ratio, lmax, candidates
|
||||
)
|
||||
else:
|
||||
visible = ((pos + 1) // ratio).clamp(0, lmax).to(torch.int32).contiguous()
|
||||
score_lens = visible
|
||||
consume = None
|
||||
if is_consumer and candidates is None:
|
||||
consume = _candidate_mask(published, lmax, indexer.candidate_block_size)
|
||||
assert consume.shape[0] == bs and consume.shape[1] >= lmax
|
||||
assert consume.stride(1) == 1
|
||||
score_width = lmax
|
||||
if candidates is not None:
|
||||
assert candidates.width >= lmax
|
||||
assert candidates.block_size == indexer.candidate_block_size
|
||||
score_width = min(lmax, candidates.blocks.shape[1] * candidates.block_size)
|
||||
scores = prefix_logits(
|
||||
q,
|
||||
weights,
|
||||
req_to_token,
|
||||
request,
|
||||
score_lens,
|
||||
table,
|
||||
table.shape[1] // 68,
|
||||
ratio,
|
||||
score_width,
|
||||
consume,
|
||||
candidates=candidates,
|
||||
visible=visible,
|
||||
)
|
||||
result = None
|
||||
if is_source:
|
||||
result = candidate_blocks(
|
||||
scores,
|
||||
visible,
|
||||
lmax,
|
||||
indexer.candidate_topk_blocks,
|
||||
indexer.candidate_block_size,
|
||||
)
|
||||
if not compact:
|
||||
# Main's decode protocol carries block IDs even without the
|
||||
# compact scoring flag. Drop only the optional device lengths.
|
||||
result = BlockIds(blocks=result.blocks)
|
||||
idx = select_prefix_topk(
|
||||
scores, score_lens, min(indexer.index_topk, score_width)
|
||||
)
|
||||
publish_topk(
|
||||
idx,
|
||||
scores,
|
||||
score_lens,
|
||||
request,
|
||||
req_to_token,
|
||||
inputs.out_page_indices,
|
||||
None,
|
||||
ratio,
|
||||
is_consumer,
|
||||
candidates=candidates,
|
||||
)
|
||||
return True, result
|
||||
|
||||
from sglang.kernels.ops.attention.dsv4.sm90_fp4_indexer import (
|
||||
fp4_index_logits_mapped_sm90,
|
||||
)
|
||||
|
||||
lens = (pos + 1) // ratio
|
||||
scores = fp4_index_logits_mapped_sm90(
|
||||
q,
|
||||
weights,
|
||||
req_to_token,
|
||||
req.to(torch.int64).contiguous(),
|
||||
lens.to(torch.int64).contiguous(),
|
||||
table,
|
||||
table.shape[1] // 68,
|
||||
ratio,
|
||||
lmax,
|
||||
)
|
||||
result = None
|
||||
if is_source:
|
||||
result = BlockIds(
|
||||
blocks=select_candidate_block_ids(
|
||||
scores,
|
||||
lens[:, None],
|
||||
indexer.candidate_topk_blocks,
|
||||
indexer.candidate_block_size,
|
||||
)
|
||||
)
|
||||
k = min(indexer.index_topk, lmax)
|
||||
if is_consumer:
|
||||
idx = topk_among_blocks(
|
||||
scores, lens, published.blocks, k, block_size=indexer.candidate_block_size
|
||||
)
|
||||
idx = idx.masked_fill(idx < 0, lmax)
|
||||
else:
|
||||
idx = scores.topk(k, dim=-1, sorted=False).indices
|
||||
if 0 < k <= 1024:
|
||||
from sglang.kernels.ops.attention.dsv4.sm90_fp4_topk import sort_map_topk
|
||||
|
||||
sort_map_topk(
|
||||
idx, lens, req, req_to_token, inputs.out_page_indices, None, ratio
|
||||
)
|
||||
else:
|
||||
idx = idx.sort(dim=-1).values
|
||||
reach = idx < lens[:, None]
|
||||
slots = (
|
||||
req_to_token[req[:, None], idx.clamp_max(lmax - 1) * ratio].to(torch.int64)
|
||||
// ratio
|
||||
)
|
||||
inputs.out_page_indices[:bs, :k] = torch.where(reach, slots, -1).to(torch.int32)
|
||||
return True, result
|
||||
@@ -30,6 +30,7 @@ from sglang.kernels.ops.attention.dsv4.topk import (
|
||||
)
|
||||
from sglang.srt.layers.attention.dsv4.indexer import topk_transform_paged_from_metadata
|
||||
from sglang.srt.layers.attention.dsv4.metadata import expand_index_page_table
|
||||
from sglang.srt.utils.common import async_h2d
|
||||
|
||||
from .scoring import (
|
||||
DeepGEMMDecodeData,
|
||||
@@ -43,7 +44,6 @@ from .types import (
|
||||
CandidateMetadata,
|
||||
DecodeInputs,
|
||||
PrefillInputs,
|
||||
Selection,
|
||||
get_tail_row_indices,
|
||||
)
|
||||
|
||||
@@ -73,7 +73,7 @@ class _SparsePrefillTable(_SparseTable):
|
||||
|
||||
compress_lens: torch.Tensor # [rows] int32
|
||||
page_table: torch.Tensor # [rows, index pages] int32
|
||||
request_ids: torch.Tensor # [rows] int32
|
||||
request_ids: torch.Tensor # [rows] int32 row-pair ids, see _row_pair_ids
|
||||
q_dtype: torch.dtype
|
||||
page_size: int # index-K pool page size (slots)
|
||||
rows_per_request: List[int] # rows of each request, in row order
|
||||
@@ -118,12 +118,11 @@ class SparseTableBackend:
|
||||
# captured graphs keep reading the buffers they saw
|
||||
self._retired_cached_row_ids: list = []
|
||||
|
||||
def publish_prefill(self, inputs: PrefillInputs, out: Selection):
|
||||
out.reset()
|
||||
def publish_prefill(self, inputs: PrefillInputs):
|
||||
inputs.reset_outputs()
|
||||
data = get_deep_gemm_prefill_data(inputs, self.req_to_token)
|
||||
if data is None:
|
||||
return None
|
||||
selected = data.empty_selection(inputs.indexer.index_topk)
|
||||
index_page_size = self.token_to_kv_pool.get_index_k_page_size(
|
||||
inputs.compress_ratio
|
||||
)
|
||||
@@ -140,23 +139,21 @@ class SparseTableBackend:
|
||||
).contiguous(),
|
||||
index_page_size=index_page_size,
|
||||
topk_blocks=self.topk_blocks,
|
||||
out_positions=selected,
|
||||
out_positions=inputs.out_raw_indices[: data.num_rows],
|
||||
)
|
||||
data.write_selection(selected=selected, out=out)
|
||||
data.write_page_indices(inputs)
|
||||
return table
|
||||
|
||||
def consume_prefill(
|
||||
self,
|
||||
inputs: PrefillInputs,
|
||||
published: Optional[_SparsePrefillTable],
|
||||
out: Selection,
|
||||
) -> None:
|
||||
out.reset()
|
||||
inputs.reset_outputs()
|
||||
data = get_deep_gemm_prefill_data(inputs, self.req_to_token)
|
||||
if data is None:
|
||||
return
|
||||
assert published is not None
|
||||
selected = data.empty_selection(inputs.indexer.index_topk)
|
||||
select_prefill_table(
|
||||
table=published,
|
||||
data=data,
|
||||
@@ -167,16 +164,16 @@ class SparseTableBackend:
|
||||
inputs.compress_ratio
|
||||
),
|
||||
),
|
||||
out_positions=selected,
|
||||
out_positions=inputs.out_raw_indices[: data.num_rows],
|
||||
)
|
||||
data.write_selection(selected=selected, out=out)
|
||||
data.write_page_indices(inputs)
|
||||
|
||||
def publish_decode(self, inputs: DecodeInputs, out: Selection):
|
||||
def publish_decode(self, inputs: DecodeInputs):
|
||||
data = get_deep_gemm_decode_data(inputs, self.token_to_kv_pool)
|
||||
metadata = inputs.paged_metadata
|
||||
seq_lens = metadata.compressed_seq_lens.reshape(-1)
|
||||
if isinstance(metadata.deep_gemm_metadata, list):
|
||||
return self._publish_decode_chunked(inputs, data, seq_lens, out)
|
||||
return self._publish_decode_chunked(inputs, data, seq_lens)
|
||||
logits = deep_gemm_fp4_paged_mqa_logits(
|
||||
(data.q_fp4, data.q_sf),
|
||||
data.k_cache,
|
||||
@@ -195,10 +192,9 @@ class SparseTableBackend:
|
||||
logits,
|
||||
seq_lens,
|
||||
metadata.page_table,
|
||||
out.page_indices,
|
||||
inputs.out_page_indices,
|
||||
metadata.compressed_page_size,
|
||||
metadata.topk_metadata,
|
||||
out_raw_indices=out.raw_indices,
|
||||
)
|
||||
# per row: block count for the block top-k, sparse-row length for the consumers
|
||||
with torch.cuda.stream(self.alt_stream):
|
||||
@@ -237,7 +233,6 @@ class SparseTableBackend:
|
||||
inputs: DecodeInputs,
|
||||
data: DeepGEMMDecodeData,
|
||||
seq_lens: torch.Tensor,
|
||||
out: Selection,
|
||||
) -> _SparseTable:
|
||||
"""On the current stream, so each chunk's logits are released before the next."""
|
||||
metadata = inputs.paged_metadata
|
||||
@@ -259,8 +254,8 @@ class SparseTableBackend:
|
||||
topk_transform_paged_from_metadata(
|
||||
logits,
|
||||
metadata,
|
||||
out.page_indices,
|
||||
out.raw_indices,
|
||||
inputs.out_page_indices,
|
||||
None,
|
||||
rows=rows,
|
||||
topk_metadata=(
|
||||
topk_plans[chunk_idx] if topk_plans is not None else None
|
||||
@@ -304,7 +299,6 @@ class SparseTableBackend:
|
||||
self,
|
||||
inputs: DecodeInputs,
|
||||
published: Optional[_SparseTable],
|
||||
out: Selection,
|
||||
) -> None:
|
||||
assert published is not None
|
||||
data = get_deep_gemm_decode_data(inputs, self.token_to_kv_pool)
|
||||
@@ -317,9 +311,8 @@ class SparseTableBackend:
|
||||
published.schedule,
|
||||
published.blocks.shape[1],
|
||||
)
|
||||
assert out.raw_indices is None
|
||||
topk_transform_sparse(
|
||||
logits, published.valid_lens, published.phys_blocks, out.page_indices
|
||||
logits, published.valid_lens, published.phys_blocks, inputs.out_page_indices
|
||||
)
|
||||
|
||||
def _get_request_ids(self, inputs: DecodeInputs, rows: int, device: torch.device):
|
||||
@@ -372,6 +365,25 @@ def _build_prefill_table(
|
||||
)
|
||||
|
||||
|
||||
def _row_pair_ids(rows_per_request: List[int], *, device: torch.device) -> torch.Tensor:
|
||||
"""[rows] int32: one id per consecutive row pair of a request, never across two.
|
||||
DeepGEMM's schedule walks back over equal ids to a row's request start once per
|
||||
row, quadratic in a request's rows; per-pair ids bound the walk to one step.
|
||||
|
||||
Pairs, not larger groups, because DeepGEMM's paged scheduler has BLOCK_Q == 2:
|
||||
the ids only have to delimit what it already groups, and its page table is
|
||||
indexed by Q row rather than by the id value, so the ids never address anything.
|
||||
A wider BLOCK_Q would need the pair width here to follow it."""
|
||||
counts = async_h2d(rows_per_request, dtype=torch.int64, device=device)
|
||||
rows = sum(rows_per_request)
|
||||
row = torch.arange(rows, device=device)
|
||||
request = torch.repeat_interleave(
|
||||
torch.arange(counts.numel(), device=device), counts, output_size=rows
|
||||
)
|
||||
first_row = torch.cumsum(counts, 0) - counts
|
||||
return (row - ((row - first_row[request]) & 1)).to(torch.int32)
|
||||
|
||||
|
||||
# TODO(dark): support fusion of publish + topk of publish layer
|
||||
def publish_prefill_table(
|
||||
*,
|
||||
@@ -382,8 +394,9 @@ def publish_prefill_table(
|
||||
topk_blocks: int,
|
||||
out_positions: torch.Tensor,
|
||||
) -> _SparsePrefillTable:
|
||||
"""The source's own top-k into ``out_positions`` and the chunk's table, from one
|
||||
pass over its dense scores; ``index_page_table`` is at ``index_page_size`` slots."""
|
||||
"""The source's own top-k into ``out_positions`` (request-local compressed
|
||||
positions) and the chunk's table, from one pass over its dense scores;
|
||||
``index_page_table`` is at ``index_page_size`` slots."""
|
||||
rows = data.num_rows
|
||||
device = data.q_fp4.device
|
||||
nblocks, valid_lens = candidate_row_lens(data.compress_lens, topk_blocks)
|
||||
@@ -395,7 +408,7 @@ def publish_prefill_table(
|
||||
topk_transform_ragged_v2(
|
||||
logits,
|
||||
lens,
|
||||
out_offsets=data.request_starts[tile],
|
||||
out_offsets=zero_offsets[tile],
|
||||
out_indices=out_positions[tile],
|
||||
)
|
||||
# keys past a row's block count stay unset; the top-k reads a row
|
||||
@@ -411,17 +424,12 @@ def publish_prefill_table(
|
||||
out_offsets=zero_offsets[: logits.shape[0]],
|
||||
out_indices=blocks[tile],
|
||||
)
|
||||
request_ids = torch.repeat_interleave(
|
||||
torch.arange(len(data.rows_per_request), dtype=torch.int32, device=device),
|
||||
torch.tensor(data.rows_per_request, dtype=torch.int64, device=device),
|
||||
output_size=rows,
|
||||
)
|
||||
return _build_prefill_table(
|
||||
blocks=blocks,
|
||||
compress_lens=data.compress_lens,
|
||||
page_table=index_page_table,
|
||||
page_size=index_page_size,
|
||||
request_ids=request_ids,
|
||||
request_ids=_row_pair_ids(data.rows_per_request, device=device),
|
||||
rows_per_request=data.rows_per_request,
|
||||
q_dtype=data.q_fp4.dtype,
|
||||
valid_lens=valid_lens,
|
||||
@@ -446,4 +454,3 @@ def select_prefill_table(
|
||||
)
|
||||
# request-relative compressed positions, -1 padded
|
||||
topk_transform_sparse(logits, table.valid_lens, table.blocks, out_positions)
|
||||
out_positions.add_(torch.where(out_positions >= 0, data.request_starts[:, None], 0))
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""The inputs and outputs of the V4.1 indexer backends."""
|
||||
"""The inputs of the V4.1 indexer backends, including the selection buffers they fill."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -7,6 +7,8 @@ from typing import TYPE_CHECKING, List, Optional, Protocol, TypeVar
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
from sglang.srt.utils.common import async_h2d
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.layers.attention.dsv4.dsv41_sparse import DeepseekV41Indexer
|
||||
from sglang.srt.layers.attention.dsv4.metadata import PagedIndexerMetadata
|
||||
@@ -22,11 +24,7 @@ def get_tail_row_indices(
|
||||
for n, t in zip(full_rows_per_request, tail_rows_per_request):
|
||||
rows.extend(range(start + n - t, start + n))
|
||||
start += n
|
||||
device = torch.device(device)
|
||||
if device.type != "cuda":
|
||||
return torch.tensor(rows, dtype=torch.int64, device=device)
|
||||
staged = torch.tensor(rows, dtype=torch.int64, pin_memory=True)
|
||||
return staged.to(device, non_blocking=True)
|
||||
return async_h2d(rows, dtype=torch.int64, device=device)
|
||||
|
||||
|
||||
class CandidateMetadata:
|
||||
@@ -34,16 +32,6 @@ class CandidateMetadata:
|
||||
raise NotImplementedError(f"{type(self).__name__} is not a prefill publish")
|
||||
|
||||
|
||||
class Selection(msgspec.Struct, frozen=True):
|
||||
page_indices: torch.Tensor # index-K pool slots, ascending
|
||||
raw_indices: Optional[torch.Tensor] # the same picks as compressed positions
|
||||
|
||||
def reset(self) -> None:
|
||||
self.page_indices.fill_(-1)
|
||||
if self.raw_indices is not None:
|
||||
self.raw_indices.fill_(-1)
|
||||
|
||||
|
||||
class PrefillInputs(msgspec.Struct, frozen=True, kw_only=True):
|
||||
"""An eager extend: the backend projects the queries and gathers the index K."""
|
||||
|
||||
@@ -68,6 +56,37 @@ class PrefillInputs(msgspec.Struct, frozen=True, kw_only=True):
|
||||
rows_per_request: Optional[List[int]]
|
||||
rows_per_request_device: Optional[torch.Tensor]
|
||||
|
||||
# The selection, unordered, -1 padded: [rows, topk] int32 request-local
|
||||
# compressed positions, and the same picks as index-K pool slots, which is
|
||||
# None when this forward's attention reads only the positions (sparse prefill).
|
||||
out_raw_indices: torch.Tensor
|
||||
out_page_indices: Optional[torch.Tensor]
|
||||
|
||||
def reset_outputs(self) -> None:
|
||||
self.out_raw_indices.fill_(-1)
|
||||
if self.out_page_indices is not None:
|
||||
self.out_page_indices.fill_(-1)
|
||||
|
||||
|
||||
def materialize_prefill_request_rows(inputs: PrefillInputs) -> torch.Tensor:
|
||||
"""Map real local query rows to request-pool IDs, including interleaved CP.
|
||||
|
||||
CP keeps request order but changes each request's local row count. Pool IDs
|
||||
are not batch ordinals, and zero-row requests must remain in the count map.
|
||||
The CPU counts also exclude padding and account for a bounded-replay tail.
|
||||
"""
|
||||
if inputs.req_rows is not None:
|
||||
return inputs.req_rows
|
||||
counts = inputs.rows_per_request
|
||||
assert counts is not None and len(counts) == inputs.req_pool_indices.numel()
|
||||
assert sum(counts) == inputs.positions.shape[0]
|
||||
repeats = inputs.rows_per_request_device
|
||||
if repeats is None:
|
||||
repeats = async_h2d(counts, dtype=torch.int32, device=inputs.positions.device)
|
||||
return torch.repeat_interleave(
|
||||
inputs.req_pool_indices, repeats, output_size=inputs.positions.shape[0]
|
||||
)
|
||||
|
||||
|
||||
class DecodeInputs(msgspec.Struct, frozen=True, kw_only=True):
|
||||
"""Decode (one row per request) or verify (one row per draft token)."""
|
||||
@@ -82,6 +101,14 @@ class DecodeInputs(msgspec.Struct, frozen=True, kw_only=True):
|
||||
req_rows: torch.Tensor # [rows] req_to_token row of each query row
|
||||
paged_metadata: PagedIndexerMetadata
|
||||
is_verify: bool
|
||||
# Fixed request-major groups, validated by the attention backend for static verify.
|
||||
group_size: int = 1
|
||||
|
||||
# [rows, topk] int32 index-K pool slots of the selection, unordered, -1 padded.
|
||||
out_page_indices: torch.Tensor
|
||||
|
||||
def reset_outputs(self) -> None:
|
||||
self.out_page_indices.fill_(-1)
|
||||
|
||||
|
||||
class CapturedPrefillInputs(msgspec.Struct, frozen=True, kw_only=True):
|
||||
@@ -92,6 +119,9 @@ class CapturedPrefillInputs(msgspec.Struct, frozen=True, kw_only=True):
|
||||
q: torch.Tensor # [rows, heads, 128] bf16, roped
|
||||
weights: torch.Tensor # [rows, heads] bf16 head weights
|
||||
paged_metadata: PagedIndexerMetadata
|
||||
# The selection as pool slots and as request-local compressed positions.
|
||||
out_page_indices: torch.Tensor
|
||||
out_raw_indices: torch.Tensor
|
||||
|
||||
|
||||
Metadata = TypeVar("Metadata", bound=CandidateMetadata)
|
||||
@@ -100,30 +130,16 @@ Metadata = TypeVar("Metadata", bound=CandidateMetadata)
|
||||
class PrefillCandidates(Protocol[Metadata]):
|
||||
"""The attention backend keeps a publish alive and hands it back unread."""
|
||||
|
||||
def publish_prefill(
|
||||
self,
|
||||
inputs: PrefillInputs,
|
||||
out: Selection,
|
||||
) -> Optional[Metadata]: ...
|
||||
def publish_prefill(self, inputs: PrefillInputs) -> Optional[Metadata]: ...
|
||||
|
||||
def consume_prefill(
|
||||
self,
|
||||
inputs: PrefillInputs,
|
||||
published: Optional[Metadata],
|
||||
out: Selection,
|
||||
self, inputs: PrefillInputs, published: Optional[Metadata]
|
||||
) -> None: ...
|
||||
|
||||
|
||||
class DecodeCandidates(Protocol[Metadata]):
|
||||
def publish_decode(
|
||||
self,
|
||||
inputs: DecodeInputs,
|
||||
out: Selection,
|
||||
) -> Optional[Metadata]: ...
|
||||
def publish_decode(self, inputs: DecodeInputs) -> Optional[Metadata]: ...
|
||||
|
||||
def consume_decode(
|
||||
self,
|
||||
inputs: DecodeInputs,
|
||||
published: Optional[Metadata],
|
||||
out: Selection,
|
||||
self, inputs: DecodeInputs, published: Optional[Metadata]
|
||||
) -> None: ...
|
||||
|
||||
@@ -110,8 +110,11 @@ def create_dsv41_candidate_graph_variants(
|
||||
if not (
|
||||
(capture_forward_mode == ForwardMode.DECODE or dspark_target_verify)
|
||||
and model_runner.device == "cuda"
|
||||
and not is_hip()
|
||||
and torch.cuda.get_device_capability(model_runner.gpu_id)[0] >= 10
|
||||
and (
|
||||
is_gfx95_supported()
|
||||
if is_hip()
|
||||
else torch.cuda.get_device_capability(model_runner.gpu_id)[0] >= 9
|
||||
)
|
||||
and getattr(text_config, "model_type", None) == "deepseek_v41"
|
||||
and getattr(text_config, "candidate_source_layer_id", -1) >= 0
|
||||
):
|
||||
@@ -122,7 +125,7 @@ def create_dsv41_candidate_graph_variants(
|
||||
ratios = set(text_config.compress_ratios) & {1, 2}
|
||||
topk = text_config.index_topk
|
||||
variants = []
|
||||
# Verify needs per-query causal top-k, so it always keeps candidate filtering.
|
||||
# Verify needs per-query causal top-k, so it cannot skip low-ratio scoring.
|
||||
if topk > 0 and ratios and not dspark_target_verify:
|
||||
variants.append(("candidate_all", topk * min(ratios)))
|
||||
if ratios == {1, 2}:
|
||||
|
||||
@@ -231,6 +231,24 @@ def cp_interleave_input_ids(input_ids: Any, forward_batch):
|
||||
return padded_input_ids.view(-1, get_parallel().attn_cp_size).T.flatten()
|
||||
|
||||
|
||||
def cp_relayout_tail_input_ids(input_ids, full_metadata, tail_metadata, token_indices):
|
||||
"""Select canonical routing IDs without changing their original CP owner.
|
||||
|
||||
A request tail need not start at a multiple of CP size. Re-interleaving the
|
||||
compact tails would move IDs to different ranks than their hidden states.
|
||||
Both input and output use the rank-major layout consumed by the MoE gather.
|
||||
"""
|
||||
cp_size = len(full_metadata.per_rank_actual_token)
|
||||
full_rank_len = max(full_metadata.per_rank_actual_token)
|
||||
full_slots = (
|
||||
token_indices.remainder(cp_size) * full_rank_len + token_indices // cp_size
|
||||
)
|
||||
tail_ids = input_ids.index_select(0, full_slots)
|
||||
result = input_ids.new_zeros(sum(tail_metadata.per_rank_actual_token))
|
||||
result.index_copy_(0, tail_metadata.gather_index, tail_ids)
|
||||
return result
|
||||
|
||||
|
||||
def cp_gather_after_forward(x: Any, forward_batch, stream: Optional[Any] = None):
|
||||
"""Gather CP hidden states at the model boundary when this batch is active."""
|
||||
assert is_cp_active(forward_batch)
|
||||
|
||||
@@ -5,6 +5,7 @@ import os
|
||||
import time
|
||||
from contextlib import contextmanager, nullcontext
|
||||
from enum import IntEnum, auto
|
||||
from functools import partial
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Tuple
|
||||
|
||||
@@ -16,7 +17,11 @@ from sglang.srt.distributed.device_communicators.pynccl_allocator import (
|
||||
restore_symmetric_memory_context,
|
||||
)
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.deep_gemm_wrapper.configurer import ENABLE_JIT_DEEPGEMM
|
||||
from sglang.srt.layers.deep_gemm_wrapper.configurer import (
|
||||
DEEPGEMM_NEED_TMA_ALIGNED_SCALES,
|
||||
DEEPGEMM_SCALE_UE8M0,
|
||||
ENABLE_JIT_DEEPGEMM,
|
||||
)
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardMode
|
||||
from sglang.srt.runtime_context import (
|
||||
get_device,
|
||||
@@ -111,6 +116,7 @@ class DeepGemmKernelType(IntEnum):
|
||||
GROUPED_GEMM_NT_BF16_MASKED = auto()
|
||||
GROUPED_GEMM_NT_BF16_CONTIG = auto()
|
||||
GEMM_NT_F8F8BF16 = auto()
|
||||
GEMM_NT_F8F8BF16_BLOCK32 = auto()
|
||||
GEMM_NT_BF16BF16F32 = auto()
|
||||
|
||||
|
||||
@@ -265,6 +271,9 @@ class _BaseWarmupExecutor:
|
||||
def create(kernel_type: DeepGemmKernelType, **kwargs):
|
||||
return {
|
||||
DeepGemmKernelType.GEMM_NT_F8F8BF16: _NormalWarmupExecutor,
|
||||
DeepGemmKernelType.GEMM_NT_F8F8BF16_BLOCK32: partial(
|
||||
_NormalWarmupExecutor, block_size=32
|
||||
),
|
||||
DeepGemmKernelType.GROUPED_GEMM_NT_F8F8BF16_CONTIG: _GroupedContWarmupExecutor,
|
||||
DeepGemmKernelType.GROUPED_GEMM_NT_F8F8BF16_MASKED: _GroupedMaskedWarmupExecutor,
|
||||
DeepGemmKernelType.GEMM_NT_BF16BF16F32: _BF16F32WarmupExecutor,
|
||||
@@ -278,8 +287,38 @@ class _BaseWarmupExecutor:
|
||||
) -> int:
|
||||
# Return the required memory space in GB for warmup executor
|
||||
_GB = 1 << 30
|
||||
if kernel_type == DeepGemmKernelType.GEMM_NT_F8F8BF16:
|
||||
return (max_m * k + n * k + max_m * n * 2) / _GB
|
||||
is_dense_fp8 = kernel_type in (
|
||||
DeepGemmKernelType.GEMM_NT_F8F8BF16,
|
||||
DeepGemmKernelType.GEMM_NT_F8F8BF16_BLOCK32,
|
||||
)
|
||||
block_size = (
|
||||
32
|
||||
if kernel_type == DeepGemmKernelType.GEMM_NT_F8F8BF16_BLOCK32
|
||||
else _BLOCK_SIZE
|
||||
)
|
||||
lhs_scales = max_m * ceil_div(k, block_size) * 4
|
||||
rhs_scales = ceil_div(n, block_size) * ceil_div(k, block_size) * 4
|
||||
scale_workspace = _get_fp8_scale_workspace(
|
||||
max_m,
|
||||
n,
|
||||
k,
|
||||
lhs_groups=(
|
||||
num_groups
|
||||
if kernel_type == DeepGemmKernelType.GROUPED_GEMM_NT_F8F8BF16_MASKED
|
||||
else 1
|
||||
),
|
||||
rhs_groups=1 if is_dense_fp8 else num_groups,
|
||||
block_size=block_size,
|
||||
)
|
||||
if is_dense_fp8:
|
||||
return (
|
||||
max_m * k
|
||||
+ n * k
|
||||
+ max_m * n * 2
|
||||
+ lhs_scales
|
||||
+ rhs_scales
|
||||
+ scale_workspace
|
||||
) / _GB
|
||||
elif kernel_type == DeepGemmKernelType.GROUPED_GEMM_NT_F8F8BF16_CONTIG:
|
||||
return (max_m * k + num_groups * n * k + max_m * 4 + max_m * n * 2) / _GB
|
||||
elif kernel_type == DeepGemmKernelType.GROUPED_GEMM_NT_BF16_CONTIG:
|
||||
@@ -310,22 +349,49 @@ class _BaseWarmupExecutor:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
def _empty_token_fp8(size):
|
||||
def _packed_scale_workspace(rows: int, cols: int, groups: int) -> tuple[int, int]:
|
||||
packed_bytes = groups * ceil_align(rows, 4) * ceil_div(cols, 4) * 4
|
||||
peak_bytes = packed_bytes
|
||||
if groups > 1 and rows * cols % 4:
|
||||
# DeepGEMM's PyTorch fallback keeps the initial packed allocation while
|
||||
# shifting FP32 bits to uint8, padding, and transposing a second copy.
|
||||
token_scales = groups * rows * cols
|
||||
peak_bytes += max(5 * token_scales, token_scales + 2 * packed_bytes)
|
||||
return packed_bytes, peak_bytes
|
||||
|
||||
|
||||
def _get_fp8_scale_workspace(
|
||||
max_m: int, n: int, k: int, lhs_groups: int, rhs_groups: int, block_size: int = 128
|
||||
) -> int:
|
||||
scale_k = ceil_div(k, block_size)
|
||||
if DEEPGEMM_NEED_TMA_ALIGNED_SCALES:
|
||||
return lhs_groups * ceil_align(max_m, 4) * scale_k * 4
|
||||
if DEEPGEMM_SCALE_UE8M0:
|
||||
lhs_packed, lhs_peak = _packed_scale_workspace(max_m, scale_k, lhs_groups)
|
||||
_, rhs_peak = _packed_scale_workspace(n, scale_k, rhs_groups)
|
||||
# RHS block scales are broadcast to token scales before packing. The
|
||||
# int64 index tensor dies before packing; the packed LHS stays live.
|
||||
rhs_broadcast = rhs_groups * n * scale_k * 4
|
||||
return max(lhs_peak, lhs_packed + rhs_broadcast + max(n * 8, rhs_peak))
|
||||
return 0
|
||||
|
||||
|
||||
def _empty_token_fp8(size, block_size=128):
|
||||
*dims, k = size
|
||||
return (
|
||||
torch.empty(size, device="cuda", dtype=torch.float8_e4m3fn),
|
||||
torch.ones(
|
||||
(*dims, ceil_div(k, _BLOCK_SIZE)), device="cuda", dtype=torch.float32
|
||||
(*dims, ceil_div(k, block_size)), device="cuda", dtype=torch.float32
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _empty_block_fp8(size):
|
||||
def _empty_block_fp8(size, block_size=128):
|
||||
*dims, n, k = size
|
||||
return (
|
||||
torch.empty(size, device="cuda", dtype=torch.float8_e4m3fn),
|
||||
torch.ones(
|
||||
(*dims, ceil_div(n, _BLOCK_SIZE), ceil_div(k, _BLOCK_SIZE)),
|
||||
(*dims, ceil_div(n, block_size), ceil_div(k, block_size)),
|
||||
device="cuda",
|
||||
dtype=torch.float32,
|
||||
),
|
||||
@@ -336,9 +402,12 @@ _BLOCK_SIZE = 128
|
||||
|
||||
|
||||
class _NormalWarmupExecutor(_BaseWarmupExecutor):
|
||||
def __init__(self, max_m: int, n: int, k: int, num_groups: int):
|
||||
self.lhs_q, self.lhs_s = _empty_token_fp8((max_m, k))
|
||||
self.rhs_q, self.rhs_s = _empty_block_fp8((n, k))
|
||||
def __init__(
|
||||
self, max_m: int, n: int, k: int, num_groups: int, block_size: int = 128
|
||||
):
|
||||
self.recipe = (1, block_size, block_size)
|
||||
self.lhs_q, self.lhs_s = _empty_token_fp8((max_m, k), block_size)
|
||||
self.rhs_q, self.rhs_s = _empty_block_fp8((n, k), block_size)
|
||||
self.out = torch.empty((max_m, n), device="cuda", dtype=torch.bfloat16)
|
||||
|
||||
def execute(self, m):
|
||||
@@ -346,6 +415,7 @@ class _NormalWarmupExecutor(_BaseWarmupExecutor):
|
||||
(self.lhs_q[:m], self.lhs_s[:m]),
|
||||
(self.rhs_q, self.rhs_s),
|
||||
self.out[:m],
|
||||
recipe=self.recipe,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -220,11 +220,16 @@ def gemm_nt_f8f8bf16(
|
||||
lhs: Tuple[torch.Tensor, torch.Tensor],
|
||||
rhs: Tuple[torch.Tensor, torch.Tensor],
|
||||
out: torch.Tensor,
|
||||
recipe: Optional[Tuple[int, int, int]] = None,
|
||||
):
|
||||
m, k = lhs[0].shape
|
||||
n, _ = rhs[0].shape
|
||||
num_groups = 1
|
||||
kernel_type = compile_utils.DeepGemmKernelType.GEMM_NT_F8F8BF16
|
||||
kernel_type = (
|
||||
compile_utils.DeepGemmKernelType.GEMM_NT_F8F8BF16_BLOCK32
|
||||
if recipe == (1, 32, 32)
|
||||
else compile_utils.DeepGemmKernelType.GEMM_NT_F8F8BF16
|
||||
)
|
||||
|
||||
_sanity_check_input(lhs)
|
||||
_sanity_check_input(rhs)
|
||||
@@ -234,6 +239,7 @@ def gemm_nt_f8f8bf16(
|
||||
lhs,
|
||||
rhs,
|
||||
out,
|
||||
**({"recipe": recipe} if recipe is not None else {}),
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -29,6 +29,7 @@ from sglang.kernels.ops.embeddings.engram_hash import (
|
||||
MODE_EXTEND,
|
||||
MODE_VERIFY,
|
||||
engram_commit_history,
|
||||
engram_hash_extend_and_commit,
|
||||
engram_hash_ids,
|
||||
engram_hash_ids_and_commit,
|
||||
)
|
||||
@@ -239,6 +240,11 @@ class EngramHasher(nn.Module):
|
||||
self.image_token_id: Optional[int] = None
|
||||
self.history: Optional[torch.Tensor] = None
|
||||
self.pad_row = 0
|
||||
# typing only
|
||||
self.token_map: torch.Tensor
|
||||
self.multipliers: torch.Tensor
|
||||
self.primes: torch.Tensor
|
||||
self.offsets: torch.Tensor
|
||||
|
||||
def init_history(self, num_req_slots: int, device) -> None:
|
||||
"""Allocate oldest-first history with a spare row for graph padding."""
|
||||
@@ -285,6 +291,33 @@ class EngramHasher(nn.Module):
|
||||
device=input_ids.device,
|
||||
)
|
||||
mode = forward_batch.forward_mode
|
||||
if (
|
||||
mode.is_extend()
|
||||
and not mode.is_target_verify()
|
||||
and envs.SGLANG_OPT_ENGRAM_EXTEND_HASH.get()
|
||||
and _cuda_kernels(input_ids)
|
||||
and forward_batch.extend_seq_lens_cpu is not None
|
||||
):
|
||||
history = forward_batch.engram_history
|
||||
return engram_hash_extend_and_commit(
|
||||
input_ids,
|
||||
forward_batch.positions,
|
||||
history=self.history if history is None else history,
|
||||
commit_history=self.history,
|
||||
req_slots=forward_batch.req_pool_indices,
|
||||
starts=forward_batch.extend_start_loc,
|
||||
lengths=forward_batch.extend_seq_lens,
|
||||
num_real=sum(forward_batch.extend_seq_lens_cpu),
|
||||
history_via_slots=history is None,
|
||||
out_cache_loc=forward_batch.out_cache_loc,
|
||||
token_map=self.token_map,
|
||||
multipliers=self.multipliers,
|
||||
primes=self.primes,
|
||||
offsets=self.offsets,
|
||||
pad_id=self.pad_id,
|
||||
image_token_id=self.image_token_id,
|
||||
mm_pad_shift=MM_PAD_SHIFT_VALUE,
|
||||
)
|
||||
req_slots = forward_batch.req_pool_indices
|
||||
bs = req_slots.shape[0]
|
||||
device = input_ids.device
|
||||
@@ -309,7 +342,13 @@ class EngramHasher(nn.Module):
|
||||
)
|
||||
lens = forward_batch.extend_seq_lens.to(torch.int64)
|
||||
starts = forward_batch.extend_start_loc.to(torch.int64)
|
||||
row = torch.repeat_interleave(torch.arange(bs, device=device), lens)
|
||||
lens_cpu = forward_batch.extend_seq_lens_cpu
|
||||
row = torch.repeat_interleave(
|
||||
torch.arange(bs, device=device),
|
||||
lens,
|
||||
# The host total skips the device sum's sync.
|
||||
output_size=sum(lens_cpu) if lens_cpu is not None else None,
|
||||
)
|
||||
num_real = row.shape[0]
|
||||
kmode = MODE_EXTEND
|
||||
if forward_batch.engram_history is not None:
|
||||
|
||||
@@ -6,6 +6,7 @@ import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.srt.layers import zero_copy_context
|
||||
from sglang.srt.runtime_context import get_platform
|
||||
from sglang.srt.utils import is_cuda
|
||||
from sglang.srt.utils.custom_op import register_custom_op
|
||||
|
||||
@@ -14,7 +15,10 @@ _is_cuda = is_cuda()
|
||||
if _is_cuda:
|
||||
from sgl_kernel import moe_sum_reduce
|
||||
|
||||
from sglang.kernels.ops.activation.activation import silu_and_mul
|
||||
from sglang.kernels.ops.activation.activation import (
|
||||
silu_and_mul,
|
||||
silu_and_mul_with_activation_rounding,
|
||||
)
|
||||
from sglang.kernels.ops.moe.moe_wna16_marlin import moe_wna16_marlin_gemm
|
||||
|
||||
|
||||
@@ -109,6 +113,17 @@ def swiglu_limit_func(
|
||||
swiglu_limit: float = 0.0,
|
||||
) -> None:
|
||||
d = input.shape[1] // 2
|
||||
if (
|
||||
_is_cuda
|
||||
and get_platform().is_sm90
|
||||
and input.is_cuda
|
||||
and input.dtype in (torch.bfloat16, torch.float16)
|
||||
and d % 16 == 0
|
||||
and input.is_contiguous()
|
||||
and output.is_contiguous()
|
||||
):
|
||||
silu_and_mul_with_activation_rounding(input, output, clamp_limit=swiglu_limit)
|
||||
return
|
||||
gate = input[:, :d]
|
||||
up = input[:, d:]
|
||||
|
||||
|
||||
@@ -53,6 +53,7 @@ from sglang.srt.layers.quantization.base_config import (
|
||||
QuantizeMethodBase,
|
||||
)
|
||||
from sglang.srt.layers.quantization.fp8_utils import (
|
||||
_deepgemm_block_fp8_supported,
|
||||
_use_aiter_bpreshuffle_gfx95,
|
||||
apply_fp8_linear,
|
||||
block_fp8_scale_to_mxfp8_e8m0,
|
||||
@@ -722,6 +723,10 @@ class Fp8LinearMethod(LinearMethodBase):
|
||||
)
|
||||
|
||||
def process_weights_after_loading_block_quant(self, layer: Module) -> None:
|
||||
use_deepgemm_runner = (
|
||||
getattr(self.w8a8_block_fp8_linear, "func", self.w8a8_block_fp8_linear)
|
||||
is deepgemm_w8a8_block_fp8_linear_with_fallback
|
||||
)
|
||||
if self.convert_mxfp8_to_block:
|
||||
from sglang.srt.layers.quantization.mxfp8_block_convert import (
|
||||
convert_mxfp8_weight_to_block_fp8,
|
||||
@@ -803,10 +808,6 @@ class Fp8LinearMethod(LinearMethodBase):
|
||||
return
|
||||
else:
|
||||
# Requantize block scales to UE8M0 when DeepGEMM is the active runner.
|
||||
use_deepgemm_runner = (
|
||||
self.w8a8_block_fp8_linear
|
||||
is deepgemm_w8a8_block_fp8_linear_with_fallback
|
||||
)
|
||||
requant_block_scale_ue8m0_for_deepgemm(
|
||||
layer.weight,
|
||||
layer.weight_scale_inv,
|
||||
@@ -819,6 +820,54 @@ class Fp8LinearMethod(LinearMethodBase):
|
||||
|
||||
layer.weight.data = weight.data
|
||||
layer.weight_scale_inv.data = weight_scale.data
|
||||
if hasattr(layer, "_block_fp8_bf16_weight"):
|
||||
layer._block_fp8_bf16_weight = None
|
||||
layer._derived_weight_cache_error = None
|
||||
if (
|
||||
_is_cuda
|
||||
and get_platform().is_sm90
|
||||
and envs.SGLANG_OPT_HOPPER_BLOCK_FP8_BF16.get()
|
||||
and not (
|
||||
use_deepgemm_runner
|
||||
and _deepgemm_block_fp8_supported(
|
||||
weight.shape,
|
||||
self.weight_block_size,
|
||||
getattr(layer, "orig_dtype", None),
|
||||
)
|
||||
)
|
||||
and weight.is_cuda
|
||||
and weight.dtype == torch.float8_e4m3fn
|
||||
and self.weight_block_size == [32, 32]
|
||||
and getattr(self.quant_config, "scale_fmt", None) == "ue8m0"
|
||||
and getattr(layer, "orig_dtype", None) == torch.bfloat16
|
||||
and not self.use_marlin
|
||||
and not getattr(layer, "keep_plain_weight_layout", False)
|
||||
):
|
||||
from sglang.srt.batch_invariant_ops import is_batch_invariant_mode_enabled
|
||||
from sglang.srt.layers.quantization.fp8_utils import block_quant_dequant
|
||||
|
||||
# Keep even the smallest/largest finite E4M3 value representable
|
||||
# as normal BF16, and reject mislabeled non-power-of-two scales.
|
||||
exact_bf16_scales = (
|
||||
(weight_scale >= 2.0**-117)
|
||||
& (weight_scale <= 2.0**119)
|
||||
& (torch.frexp(weight_scale)[0] == 0.5)
|
||||
).all()
|
||||
if not is_batch_invariant_mode_enabled() and exact_bf16_scales.item():
|
||||
# Hopper group32 FP8 accumulates each 32-wide block separately.
|
||||
# Reuse a BF16 weight expansion for larger GEMMs, keeping the
|
||||
# original FP8 weights for decode and weight reloads.
|
||||
layer.register_buffer(
|
||||
"_block_fp8_bf16_weight",
|
||||
block_quant_dequant(weight, weight_scale, [32, 32], torch.bfloat16),
|
||||
persistent=False,
|
||||
)
|
||||
layer._derived_weight_cache_error = (
|
||||
"Online weight updates are not supported while Hopper FP8 "
|
||||
"BF16 weight caches are active: captured CUDA graphs retain "
|
||||
"these derived weights. Restart with "
|
||||
"SGLANG_OPT_HOPPER_BLOCK_FP8_BF16=0 to allow online updates."
|
||||
)
|
||||
if self.block_fp8_as_mxfp8:
|
||||
self._prepare_block_fp8_as_mxfp8(layer)
|
||||
|
||||
@@ -1233,6 +1282,10 @@ class Fp8LinearMethod(LinearMethodBase):
|
||||
True, # is_vnni
|
||||
)
|
||||
|
||||
cached_weight = getattr(layer, "_block_fp8_bf16_weight", None)
|
||||
extra_kwargs = (
|
||||
{"weight_bf16": cached_weight} if cached_weight is not None else {}
|
||||
)
|
||||
if isinstance(x, tuple):
|
||||
return self.w8a8_block_fp8_linear(
|
||||
input=x[0],
|
||||
@@ -1241,6 +1294,7 @@ class Fp8LinearMethod(LinearMethodBase):
|
||||
weight_scale=layer.weight_scale_inv,
|
||||
input_scale=x[1],
|
||||
bias=bias,
|
||||
**extra_kwargs,
|
||||
)
|
||||
|
||||
return self.w8a8_block_fp8_linear(
|
||||
@@ -1250,6 +1304,7 @@ class Fp8LinearMethod(LinearMethodBase):
|
||||
weight_scale=layer.weight_scale_inv,
|
||||
input_scale=None,
|
||||
bias=bias,
|
||||
**extra_kwargs,
|
||||
)
|
||||
|
||||
if use_intel_amx_backend(layer):
|
||||
|
||||
@@ -14,6 +14,7 @@ from sglang.kernels.ops.gemm.fp8_kernel import (
|
||||
w8a8_block_fp8_matmul_triton,
|
||||
)
|
||||
from sglang.kernels.ops.quantization.fp8_kernel import (
|
||||
dequant_group_fp8_to_bf16,
|
||||
fp8_dtype,
|
||||
fp8_max,
|
||||
fp8_min,
|
||||
@@ -469,6 +470,12 @@ if get_platform().is_blackwell and is_flashinfer_available():
|
||||
from flashinfer import mxfp8_quantize as _raw_flashinfer_mxfp8_quantize
|
||||
from flashinfer.gemm import gemm_fp8_nt_groupwise as _raw_gemm_fp8_nt_groupwise
|
||||
|
||||
from sglang.srt.layers.quantization.mxfp8_dispatch_cache import (
|
||||
maybe_cache_mxfp8_dispatch,
|
||||
)
|
||||
|
||||
_flashinfer_mm_mxfp8_impl = maybe_cache_mxfp8_dispatch(_raw_flashinfer_mm_mxfp8)
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _get_flashinfer_groupwise_backend() -> str:
|
||||
if get_fp8_gemm_runner_backend().is_flashinfer_cutlass():
|
||||
@@ -563,7 +570,7 @@ if get_platform().is_blackwell and is_flashinfer_available():
|
||||
use_8x4_sf_layout: bool = False,
|
||||
backend: str = "auto",
|
||||
) -> torch.Tensor:
|
||||
return _raw_flashinfer_mm_mxfp8(
|
||||
return _flashinfer_mm_mxfp8_impl(
|
||||
q_input,
|
||||
weight_t,
|
||||
x_scale_u8,
|
||||
@@ -590,19 +597,32 @@ def dispatch_w8a8_block_fp8_linear(
|
||||
1. The --fp8-gemm-backend server argument (preferred)
|
||||
2. Auto-detection based on hardware capabilities
|
||||
"""
|
||||
# Only Triton reads the block size at launch; DeepGEMM, the FlashInfer
|
||||
# groupwise kernels and CUTLASS take 128-wide K blocks only.
|
||||
backend = get_fp8_gemm_runner_backend()
|
||||
if (
|
||||
weight_block_size == [32, 32]
|
||||
and get_platform().is_sm90
|
||||
and deep_gemm_wrapper.ENABLE_JIT_DEEPGEMM
|
||||
and (backend.is_auto() or backend.is_deep_gemm())
|
||||
):
|
||||
return partial(
|
||||
deepgemm_w8a8_block_fp8_linear_with_fallback,
|
||||
act_scale_ue8m0=act_scale_ue8m0,
|
||||
)
|
||||
|
||||
# Other non-128 layouts still require Triton or the separate MXFP8 route.
|
||||
if weight_block_size is not None and weight_block_size[1] != 128:
|
||||
return partial(triton_w8a8_block_fp8_linear, act_scale_ue8m0=act_scale_ue8m0)
|
||||
|
||||
backend = get_fp8_gemm_runner_backend()
|
||||
|
||||
# Handle explicit backend selection via --fp8-gemm-backend
|
||||
if not backend.is_auto():
|
||||
return _dispatch_explicit_backend(backend)
|
||||
|
||||
# Auto mode: Select based purely on hardware/backend availability
|
||||
return _dispatch_auto_backend()
|
||||
linear = _dispatch_explicit_backend(backend)
|
||||
else:
|
||||
# Auto mode: Select based purely on hardware/backend availability
|
||||
linear = _dispatch_auto_backend()
|
||||
# Preserve the unwrapped callable when no activation policy needs binding.
|
||||
if act_scale_ue8m0 and linear is deepgemm_w8a8_block_fp8_linear_with_fallback:
|
||||
return partial(linear, act_scale_ue8m0=True)
|
||||
return linear
|
||||
|
||||
|
||||
def torch_w8a8_block_fp8_linear(
|
||||
@@ -1185,6 +1205,17 @@ def cutlass_w8a8_block_fp8_linear_with_fallback(
|
||||
return output.to(dtype=input_2d.dtype).view(*output_shape)
|
||||
|
||||
|
||||
def _deepgemm_block_fp8_supported(
|
||||
weight_shape: Tuple[int, int], block_size: List[int], output_dtype: torch.dtype
|
||||
) -> bool:
|
||||
if output_dtype != torch.bfloat16:
|
||||
return False
|
||||
n, k = weight_shape
|
||||
if list(block_size) == [32, 32] and get_platform().is_sm90:
|
||||
return n % 8 == 0 and k % 32 == 0
|
||||
return list(block_size) == [128, 128] and n % 64 == 0 and k % 128 == 0
|
||||
|
||||
|
||||
def deepgemm_w8a8_block_fp8_linear_with_fallback(
|
||||
input: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
@@ -1192,43 +1223,16 @@ def deepgemm_w8a8_block_fp8_linear_with_fallback(
|
||||
weight_scale: torch.Tensor,
|
||||
input_scale: Optional[torch.Tensor] = None,
|
||||
bias: Optional[torch.Tensor] = None,
|
||||
act_scale_ue8m0: bool = False,
|
||||
weight_bf16: Optional[torch.Tensor] = None,
|
||||
) -> torch.Tensor:
|
||||
if input_scale is not None:
|
||||
# Pre-quantized activation (SGLANG_OPT_MOE_QUANT_ONCE): ``input`` is
|
||||
# the fp8 per-token-group-128 q with rows padded to a multiple of 4
|
||||
# and ``input_scale`` the matching column-major fp32 scales
|
||||
# (stride == (1, padded_rows)) -- identical to the MN-major
|
||||
# TMA-aligned layout this path's own quant would produce below.
|
||||
# Output keeps the padded row count; the caller slices back.
|
||||
# UE8M0 packed scales (Blackwell DeepGEMM) use a different layout;
|
||||
# the caller gates on it.
|
||||
assert not deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0
|
||||
assert input.dtype == torch.float8_e4m3fn
|
||||
assert weight.shape[0] % 64 == 0 and weight.shape[1] % 128 == 0, (
|
||||
"pre-quantized fp8 input requires DeepGEMM-supported weight shapes "
|
||||
f"(got {tuple(weight.shape)})"
|
||||
)
|
||||
input_2d = input.view(-1, input.shape[-1])
|
||||
output = w8a8_block_fp8_matmul_deepgemm(
|
||||
input_2d,
|
||||
weight,
|
||||
input_scale,
|
||||
weight_scale,
|
||||
block_size,
|
||||
output_dtype=torch.bfloat16,
|
||||
)
|
||||
if bias is not None:
|
||||
output += bias
|
||||
return output.view(*input.shape[:-1], weight.shape[0])
|
||||
|
||||
output_dtype = input.dtype
|
||||
dtype_supported = output_dtype == torch.bfloat16
|
||||
output_dtype = torch.bfloat16 if input_scale is not None else input.dtype
|
||||
|
||||
# TODO: https://github.com/sgl-project/sglang/pull/6890#issuecomment-2943395737
|
||||
shape_supported = weight.shape[0] % 64 == 0 and weight.shape[1] % 128 == 0
|
||||
block_supported = list(block_size) == [128, 128]
|
||||
|
||||
if not (shape_supported and dtype_supported and block_supported):
|
||||
if not _deepgemm_block_fp8_supported(weight.shape, block_size, output_dtype):
|
||||
# fall back to triton
|
||||
# If weight_scale is in UE8M0 packed format (int32), convert back to float32
|
||||
# UE8M0 format has shape (N, K//block_k//4) with dtype int32
|
||||
@@ -1238,13 +1242,42 @@ def deepgemm_w8a8_block_fp8_linear_with_fallback(
|
||||
weight_scale, weight.shape, block_size
|
||||
)
|
||||
return triton_w8a8_block_fp8_linear(
|
||||
input, weight, block_size, weight_scale, input_scale, bias
|
||||
input,
|
||||
weight,
|
||||
block_size,
|
||||
weight_scale,
|
||||
input_scale,
|
||||
bias,
|
||||
act_scale_ue8m0=act_scale_ue8m0,
|
||||
weight_bf16=weight_bf16,
|
||||
)
|
||||
|
||||
if input_scale is not None:
|
||||
# Pre-quantized activations retain their per-token group size and
|
||||
# fp32 scales. DeepGEMM aligns scale storage for TMA when needed.
|
||||
input_2d = input.view(-1, input.shape[-1])
|
||||
output = w8a8_block_fp8_matmul_deepgemm(
|
||||
input_2d,
|
||||
weight,
|
||||
input_scale,
|
||||
weight_scale,
|
||||
block_size,
|
||||
output_dtype=output_dtype,
|
||||
)
|
||||
if bias is not None:
|
||||
output += bias
|
||||
return output.view(*input.shape[:-1], weight.shape[0])
|
||||
|
||||
input_2d = input.view(-1, input.shape[-1])
|
||||
output_shape = [*input.shape[:-1], weight.shape[0]]
|
||||
|
||||
if not _is_musa:
|
||||
if get_platform().is_sm90 and act_scale_ue8m0:
|
||||
# Keep the checkpoint's power-of-two activation quantization. SM90
|
||||
# consumes fp32 scales; DeepGEMM aligns their storage for TMA.
|
||||
q_input, x_scale = sglang_per_token_group_quant_fp8(
|
||||
input_2d, block_size[1], scale_ue8m0=True
|
||||
)
|
||||
elif not _is_musa:
|
||||
q_input, x_scale = sglang_per_token_group_quant_fp8(
|
||||
input_2d,
|
||||
block_size[1],
|
||||
@@ -1430,6 +1463,7 @@ def triton_w8a8_block_fp8_linear(
|
||||
input_scale: Optional[torch.Tensor] = None,
|
||||
bias: Optional[torch.Tensor] = None,
|
||||
act_scale_ue8m0: bool = False,
|
||||
weight_bf16: Optional[torch.Tensor] = None,
|
||||
) -> torch.Tensor:
|
||||
if input_scale is not None:
|
||||
# Pre-quantized input: ``input`` is already fp8 and ``input_scale`` is
|
||||
@@ -1454,9 +1488,30 @@ def triton_w8a8_block_fp8_linear(
|
||||
input_2d, block_size[1], column_major_scales=False
|
||||
)
|
||||
|
||||
output = w8a8_block_fp8_matmul_triton(
|
||||
q_input, weight, x_scale, weight_scale, block_size, output_dtype=output_dtype
|
||||
)
|
||||
from sglang.srt.batch_invariant_ops import is_batch_invariant_mode_enabled
|
||||
|
||||
if (
|
||||
weight_bf16 is not None
|
||||
and input_scale is None
|
||||
and act_scale_ue8m0
|
||||
and block_size == [32, 32]
|
||||
and q_input.shape[0] >= 64
|
||||
and output_dtype == torch.bfloat16
|
||||
and not is_batch_invariant_mode_enabled()
|
||||
):
|
||||
# Keep activation quantization: only the GEMM implementation changes.
|
||||
output = torch.nn.functional.linear(
|
||||
dequant_group_fp8_to_bf16(q_input, x_scale), weight_bf16
|
||||
)
|
||||
else:
|
||||
output = w8a8_block_fp8_matmul_triton(
|
||||
q_input,
|
||||
weight,
|
||||
x_scale,
|
||||
weight_scale,
|
||||
block_size,
|
||||
output_dtype=output_dtype,
|
||||
)
|
||||
if bias is not None:
|
||||
output += bias
|
||||
return output.to(dtype=output_dtype).view(*output_shape)
|
||||
|
||||
@@ -71,7 +71,13 @@ class Mxfp4MarlinMoEMethod:
|
||||
|
||||
layer._dsv4_mxfp4_backend = None # set in process_weights_after_loading
|
||||
fp4_block_k = 32
|
||||
intermediate_size_per_partition = round_up(intermediate_size_per_partition, 128)
|
||||
# Hopper repacking pads the loaded gate/up halves and down columns to
|
||||
# the required tile sizes. Keep their logical widths until then: early
|
||||
# padding to 128 turns a TP8 width of 288 into 384 instead of 320.
|
||||
if not get_platform().is_sm90:
|
||||
intermediate_size_per_partition = round_up(
|
||||
intermediate_size_per_partition, 128
|
||||
)
|
||||
hidden_size = round_up(hidden_size, 256)
|
||||
self.hidden_pad = hidden_size - layer.hidden_size
|
||||
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
"""Cache compatible FlashInfer 0.7.0+ MXFP8 host dispatch.
|
||||
|
||||
Keep FlashInfer's output allocation, autotuner and kernel invocation intact.
|
||||
Only reuse the stateless CuTe runner and validation of identical metadata.
|
||||
"""
|
||||
|
||||
from functools import lru_cache
|
||||
from inspect import Parameter, signature
|
||||
|
||||
from packaging.version import InvalidVersion, Version
|
||||
|
||||
try:
|
||||
from flashinfer import __version__ as flashinfer_version
|
||||
from flashinfer.gemm import gemm_base
|
||||
except ImportError:
|
||||
flashinfer_version = ""
|
||||
gemm_base = None
|
||||
|
||||
_cached_runner_factories = set()
|
||||
|
||||
|
||||
def maybe_cache_mxfp8_dispatch(raw_mm):
|
||||
"""Keep unsupported FlashInfer versions and APIs on the original path."""
|
||||
if gemm_base is None:
|
||||
return raw_mm
|
||||
try:
|
||||
if Version(flashinfer_version) < Version("0.7.0"):
|
||||
return raw_mm
|
||||
except InvalidVersion:
|
||||
return raw_mm
|
||||
try:
|
||||
if not callable(gemm_base._cute_dsl_gemm_mxfp8_runner):
|
||||
return raw_mm
|
||||
parameters = signature(raw_mm, follow_wrapped=False).parameters
|
||||
except (AttributeError, TypeError, ValueError):
|
||||
return raw_mm
|
||||
skip_check = parameters.get("skip_check")
|
||||
accepts_keyword = skip_check is not None and skip_check.kind in (
|
||||
Parameter.POSITIONAL_OR_KEYWORD,
|
||||
Parameter.KEYWORD_ONLY,
|
||||
)
|
||||
if not accepts_keyword and not any(
|
||||
p.kind == Parameter.VAR_KEYWORD for p in parameters.values()
|
||||
):
|
||||
return raw_mm
|
||||
return Mxfp8DispatchCache(raw_mm, gemm_base)
|
||||
|
||||
|
||||
class Mxfp8DispatchCache:
|
||||
def __init__(self, raw_mm, gemm_module):
|
||||
self.raw_mm = raw_mm
|
||||
self.validated_set = set()
|
||||
factory = gemm_module._cute_dsl_gemm_mxfp8_runner
|
||||
# This process-wide patch only memoizes the stateless
|
||||
# runner factory. The factory arguments include SM, PDL and output dtype.
|
||||
if factory not in _cached_runner_factories:
|
||||
factory = lru_cache(maxsize=16)(factory)
|
||||
gemm_module._cute_dsl_gemm_mxfp8_runner = factory
|
||||
_cached_runner_factories.add(factory)
|
||||
self.runner_factory = factory
|
||||
|
||||
def __call__(self, a, b, a_scale, b_scale, out_dtype, use_8x4_sf_layout, backend):
|
||||
kwargs = dict(
|
||||
out_dtype=out_dtype,
|
||||
use_8x4_sf_layout=use_8x4_sf_layout,
|
||||
backend=backend,
|
||||
)
|
||||
if backend != "cute-dsl":
|
||||
return self.raw_mm(a, b, a_scale, b_scale, **kwargs)
|
||||
|
||||
# Validation depends on metadata, never tensor contents. A changed
|
||||
# shape, stride, dtype or device must pass the original checks again.
|
||||
key = (
|
||||
out_dtype,
|
||||
use_8x4_sf_layout,
|
||||
tuple(
|
||||
(t.shape, t.stride(), t.dtype, t.device, t.layout)
|
||||
for t in (a, b, a_scale, b_scale)
|
||||
),
|
||||
)
|
||||
is_validated = key in self.validated_set
|
||||
result = self.raw_mm(a, b, a_scale, b_scale, skip_check=is_validated, **kwargs)
|
||||
if not is_validated:
|
||||
# Bound metadata memory for varying workloads. Only successful
|
||||
# executions qualify; a failed validation is never cached.
|
||||
if len(self.validated_set) >= 1024:
|
||||
self.validated_set.clear()
|
||||
self.validated_set.add(key)
|
||||
return result
|
||||
@@ -2249,6 +2249,27 @@ class SetInternalStateReqOutput(BaseReq, kw_only=True):
|
||||
updated: bool
|
||||
|
||||
|
||||
class QueryCacheHitReqInput(BaseReq, kw_only=True):
|
||||
"""G9: read-only probe of prefix-cache hit for a token sequence.
|
||||
|
||||
Sent to every DP rank of a (decode) engine; each rank answers with its own
|
||||
L1/L2 radix match plus the shared L3 storage hit continuing from it.
|
||||
"""
|
||||
|
||||
input_ids: List[int]
|
||||
extra_key: Optional[str] = None
|
||||
cache_salt: Optional[str] = None
|
||||
|
||||
|
||||
class QueryCacheHitReqOutput(BaseReq, kw_only=True):
|
||||
total_tokens: int = 0
|
||||
l1_hit_tokens: int = 0
|
||||
l2_hit_tokens: int = 0
|
||||
l3_hit_tokens: int = 0
|
||||
dp_rank: int = 0
|
||||
error: Optional[str] = None
|
||||
|
||||
|
||||
class ProfileReqType(Enum):
|
||||
START_PROFILE = 1
|
||||
STOP_PROFILE = 2
|
||||
|
||||
@@ -197,6 +197,12 @@ class LoadSnapshot(msgspec.Struct, omit_defaults=True):
|
||||
num_running_reqs: int = 0
|
||||
num_waiting_reqs: int = 0
|
||||
num_waiting_uncached_tokens: int = 0
|
||||
# G9 queue-time estimation (总输入长度口径, self-consistent numerator/denominator):
|
||||
# sum of total input lengths (req.seqlen) over all pending queues, and the
|
||||
# self-learned drain rate in the same unit — completed prefills' total
|
||||
# input tokens / wall second over a sliding window.
|
||||
queued_total_input_tokens: int = 0
|
||||
drain_rate_tokens_per_s: float = 0.0
|
||||
num_used_tokens: int = 0
|
||||
num_total_tokens: int = 0
|
||||
# num_total_tokens minus tokens still awaiting a KV transfer (equal to it
|
||||
|
||||
@@ -0,0 +1,464 @@
|
||||
"""One owner rank encodes each image span and broadcasts it to the ranks that
|
||||
run the same prefill chunk; every agreement precedes the payload it guards."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from contextlib import contextmanager
|
||||
from typing import Any, Callable, Dict, Iterator, List, Optional, Sequence, Tuple
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
from sglang.srt.distributed.device_communicators.pynccl_allocator import (
|
||||
disable_symmetric_memory_context,
|
||||
restore_symmetric_memory_context,
|
||||
)
|
||||
from sglang.srt.mem_cache.multimodal_cache import EmbeddingResult, MultiModalStaticCache
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SpanKey = Tuple[Optional[int], int]
|
||||
SpanEncoder = Callable[[List[Any]], torch.Tensor | List[torch.Tensor]]
|
||||
|
||||
LOCAL_HIT = 0
|
||||
OWNER_CACHE_BROADCAST = 1
|
||||
OWNER_ENCODE_BROADCAST = 2
|
||||
|
||||
PHASE_PREPARE = "prepare"
|
||||
PHASE_FEATURES = "features"
|
||||
PHASE_FINALIZE = "finalize"
|
||||
|
||||
|
||||
class MmOwnerProtocolError(RuntimeError):
|
||||
"""Raised with identical text on every group member after a group-agreed failure."""
|
||||
|
||||
|
||||
class ImageSpanRequest(msgspec.Struct, frozen=True):
|
||||
hash: Optional[int]
|
||||
span_len: int
|
||||
item: Any
|
||||
duplicates: List[Any] = []
|
||||
|
||||
|
||||
class ImageSpanKey(msgspec.Struct, frozen=True):
|
||||
hash: Optional[int]
|
||||
span_len: int
|
||||
geometry: Optional[Tuple[Any, ...]]
|
||||
|
||||
|
||||
class RankManifest(msgspec.Struct, frozen=True):
|
||||
rank: int
|
||||
keys: List[ImageSpanKey]
|
||||
cached: List[bool]
|
||||
dtype: str
|
||||
width: int
|
||||
rids: List[str]
|
||||
error: Optional[str] = None
|
||||
|
||||
|
||||
class OwnerPlan(msgspec.Struct, frozen=True):
|
||||
actions: List[int] = []
|
||||
owners: List[int] = []
|
||||
error: Optional[str] = None
|
||||
|
||||
|
||||
class RankStatus(msgspec.Struct, frozen=True):
|
||||
error: Optional[str] = None
|
||||
|
||||
|
||||
def select_owner_group(parallel) -> Optional[Any]:
|
||||
"""The group whose members all execute the same requests; None keeps each
|
||||
rank encoding its own images."""
|
||||
replication = parallel.tp_size // parallel.attn_dp_size
|
||||
if replication <= 1:
|
||||
return None
|
||||
if parallel.attn_cp_size == 1:
|
||||
group = parallel.attn_tp_group
|
||||
elif parallel.attn_dp_size == 1 and parallel.attn_cp_size == parallel.tp_size:
|
||||
group = parallel.attn_cp_group
|
||||
else:
|
||||
return None
|
||||
return group if group.world_size == replication else None
|
||||
|
||||
|
||||
def has_owner_span_work(
|
||||
mm_inputs: Sequence[Any],
|
||||
extend_prefix_lens: Sequence[int],
|
||||
extend_seq_lens: Sequence[int],
|
||||
) -> bool:
|
||||
"""Host-side mirror of the per-image scheduling path: does any raw
|
||||
single-span image overlap the chunk on every rank of the group."""
|
||||
for mm_input, prefix_len, extend_len in zip(
|
||||
mm_inputs, extend_prefix_lens, extend_seq_lens
|
||||
):
|
||||
if mm_input is None or extend_len <= 0:
|
||||
continue
|
||||
items = [item for item in mm_input.mm_items if item is not None]
|
||||
if any(
|
||||
item.precomputed_embeddings is not None or len(item.offsets) != 1
|
||||
for item in items
|
||||
):
|
||||
continue
|
||||
for item in items:
|
||||
start, end = item.offsets[0]
|
||||
if end >= prefix_len and start < prefix_len + extend_len:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
class MmOwnerSession(msgspec.Struct):
|
||||
group: Any
|
||||
device: Any
|
||||
dtype: Any
|
||||
width: int
|
||||
rids: List[str]
|
||||
signature: Any
|
||||
phase: str = PHASE_PREPARE
|
||||
in_collective: bool = False
|
||||
|
||||
def resolve(
|
||||
self,
|
||||
requests: Sequence[ImageSpanRequest],
|
||||
cache: MultiModalStaticCache,
|
||||
encode: SpanEncoder,
|
||||
) -> Dict[SpanKey, torch.Tensor]:
|
||||
# Owners allocate different amounts than receivers, so none of these
|
||||
# buffers may come out of a symmetric pool.
|
||||
saved_context = disable_symmetric_memory_context()
|
||||
try:
|
||||
return _resolve_owner_features(self, requests, cache, encode)
|
||||
finally:
|
||||
restore_symmetric_memory_context(saved_context)
|
||||
|
||||
def features_ready(self) -> None:
|
||||
self._complete()
|
||||
self.phase = PHASE_FINALIZE
|
||||
|
||||
@contextmanager
|
||||
def uncaptured(self) -> Iterator[None]:
|
||||
# A failure inside a collective leaves the group in an unknown state;
|
||||
# no later exchange may try to agree on it.
|
||||
self.in_collective = True
|
||||
yield
|
||||
self.in_collective = False
|
||||
|
||||
@contextmanager
|
||||
def fence(self) -> Iterator[None]:
|
||||
try:
|
||||
yield
|
||||
except Exception as exc:
|
||||
self._fail(exc)
|
||||
raise
|
||||
self._complete()
|
||||
|
||||
def _fail(self, exc: BaseException) -> None:
|
||||
if self.in_collective or isinstance(exc, MmOwnerProtocolError):
|
||||
raise exc
|
||||
text = _describe(self, self.phase, exc)
|
||||
if self.phase == PHASE_PREPARE:
|
||||
try:
|
||||
_exchange_manifest(self, _manifest(self, [], [], error=text))
|
||||
except MmOwnerProtocolError as agreed:
|
||||
raise agreed from exc
|
||||
_exchange_status(self, text, exc)
|
||||
|
||||
def _complete(self) -> None:
|
||||
if self.phase == PHASE_PREPARE:
|
||||
raise RuntimeError(
|
||||
f"owner protocol {self.phase} completed without a manifest exchange"
|
||||
)
|
||||
_drain_and_agree(self, self.phase)
|
||||
|
||||
|
||||
def _manifest(
|
||||
session: MmOwnerSession,
|
||||
keys: List[ImageSpanKey],
|
||||
cached: List[bool],
|
||||
error: Optional[str] = None,
|
||||
) -> RankManifest:
|
||||
return RankManifest(
|
||||
rank=session.group.rank_in_group,
|
||||
keys=keys,
|
||||
cached=cached,
|
||||
dtype=str(session.dtype),
|
||||
width=session.width,
|
||||
rids=list(session.rids),
|
||||
error=error,
|
||||
)
|
||||
|
||||
|
||||
def _exchange_manifest(session: MmOwnerSession, manifest: RankManifest) -> OwnerPlan:
|
||||
group = session.group
|
||||
with session.uncaptured():
|
||||
manifests = group.all_gather_object(manifest)
|
||||
plan = _plan_or_error(session, manifests) if group.rank_in_group == 0 else None
|
||||
plan = group.broadcast_object(plan, src=0)
|
||||
session.phase = PHASE_FEATURES
|
||||
if plan.error is not None:
|
||||
raise MmOwnerProtocolError(plan.error)
|
||||
return plan
|
||||
|
||||
|
||||
def _plan_or_error(session: MmOwnerSession, manifests: List[RankManifest]) -> OwnerPlan:
|
||||
try:
|
||||
return _make_plan(manifests)
|
||||
except Exception as exc:
|
||||
return OwnerPlan(error=_describe(session, "plan", exc))
|
||||
|
||||
|
||||
def _exchange_status(
|
||||
session: MmOwnerSession, error: Optional[str], cause: Optional[BaseException]
|
||||
) -> None:
|
||||
with session.uncaptured():
|
||||
statuses = session.group.all_gather_object(RankStatus(error=error))
|
||||
for status in statuses:
|
||||
if status.error is not None:
|
||||
raise MmOwnerProtocolError(status.error) from cause
|
||||
|
||||
|
||||
def _drain_and_agree(
|
||||
session: MmOwnerSession, stage: str, step: Optional[Callable[[], Any]] = None
|
||||
) -> Any:
|
||||
result, error, cause = None, None, None
|
||||
try:
|
||||
result = step() if step is not None else None
|
||||
_synchronize(session.device)
|
||||
except Exception as exc:
|
||||
error, cause = _describe(session, stage, exc), exc
|
||||
_exchange_status(session, error, cause)
|
||||
return result
|
||||
|
||||
|
||||
def _resolve_owner_features(
|
||||
session: MmOwnerSession,
|
||||
requests: Sequence[ImageSpanRequest],
|
||||
cache: MultiModalStaticCache,
|
||||
encode: SpanEncoder,
|
||||
) -> Dict[SpanKey, torch.Tensor]:
|
||||
group = session.group
|
||||
features: Dict[SpanKey, torch.Tensor] = {}
|
||||
keys: List[ImageSpanKey] = []
|
||||
cached: List[bool] = []
|
||||
error = None
|
||||
try:
|
||||
keys, cached = _pin_local_cache(session, requests, cache, features)
|
||||
except Exception as exc:
|
||||
error = _describe(session, "manifest", exc)
|
||||
plan = _exchange_manifest(session, _manifest(session, keys, cached, error))
|
||||
|
||||
if all(action == LOCAL_HIT for action in plan.actions):
|
||||
return features
|
||||
|
||||
buffers = _drain_and_agree(
|
||||
session,
|
||||
"encode",
|
||||
lambda: _prepare_transfers(session, requests, keys, plan, features, encode),
|
||||
)
|
||||
|
||||
with session.uncaptured():
|
||||
for index, (action, owner) in enumerate(zip(plan.actions, plan.owners)):
|
||||
if action != LOCAL_HIT:
|
||||
group.broadcast(buffers[index], src=owner)
|
||||
|
||||
for index, key in enumerate(keys):
|
||||
if plan.actions[index] == LOCAL_HIT:
|
||||
continue
|
||||
span = buffers[index]
|
||||
features[(key.hash, key.span_len)] = span
|
||||
cache.set(key.hash, EmbeddingResult(embedding=span))
|
||||
return features
|
||||
|
||||
|
||||
def _pin_local_cache(
|
||||
session: MmOwnerSession,
|
||||
requests: Sequence[ImageSpanRequest],
|
||||
cache: MultiModalStaticCache,
|
||||
features: Dict[SpanKey, torch.Tensor],
|
||||
) -> Tuple[List[ImageSpanKey], List[bool]]:
|
||||
keys: List[ImageSpanKey] = []
|
||||
cached: List[bool] = []
|
||||
for request in requests:
|
||||
if request.hash is None:
|
||||
raise ValueError(
|
||||
f"image span of {request.span_len} tokens has no content hash"
|
||||
)
|
||||
geometry = session.signature(request.item, request.span_len)
|
||||
for duplicate in request.duplicates:
|
||||
other = session.signature(duplicate, request.span_len)
|
||||
if other != geometry:
|
||||
raise ValueError(
|
||||
f"image hash {request.hash} ({request.span_len} tokens) occurs "
|
||||
f"with different geometry: {geometry} vs {other}"
|
||||
)
|
||||
keys.append(
|
||||
ImageSpanKey(
|
||||
hash=request.hash, span_len=request.span_len, geometry=geometry
|
||||
)
|
||||
)
|
||||
span = _valid_cached_span(session, cache, request)
|
||||
if span is not None:
|
||||
features[(request.hash, request.span_len)] = span
|
||||
cached.append(span is not None)
|
||||
return keys, cached
|
||||
|
||||
|
||||
def _valid_cached_span(
|
||||
session: MmOwnerSession,
|
||||
cache: MultiModalStaticCache,
|
||||
request: ImageSpanRequest,
|
||||
) -> Optional[torch.Tensor]:
|
||||
entry = cache.get_single(request.hash)
|
||||
if entry is None:
|
||||
return None
|
||||
span = entry.embedding
|
||||
if (
|
||||
span.dim() == 2
|
||||
and span.shape[0] == request.span_len
|
||||
and span.shape[1] == session.width
|
||||
and span.dtype == session.dtype
|
||||
and span.device == session.device
|
||||
):
|
||||
return span
|
||||
logger.warning(
|
||||
"Discarding cached multimodal embedding that cannot serve the current "
|
||||
"image span: cache_key=%s expected=(%d, %d, %s) cached=(%s, %s).",
|
||||
request.hash,
|
||||
request.span_len,
|
||||
session.width,
|
||||
session.dtype,
|
||||
tuple(span.shape),
|
||||
span.dtype,
|
||||
)
|
||||
cache.free(request.hash, None)
|
||||
return None
|
||||
|
||||
|
||||
def _make_plan(manifests: List[RankManifest]) -> OwnerPlan:
|
||||
for manifest in manifests:
|
||||
if manifest.error is not None:
|
||||
return OwnerPlan(error=manifest.error)
|
||||
lead = manifests[0]
|
||||
for manifest in manifests[1:]:
|
||||
if (manifest.keys, manifest.dtype, manifest.width, manifest.rids) != (
|
||||
lead.keys,
|
||||
lead.dtype,
|
||||
lead.width,
|
||||
lead.rids,
|
||||
):
|
||||
return OwnerPlan(
|
||||
error=(
|
||||
"image manifest mismatch between group ranks 0 and "
|
||||
f"{manifest.rank}: rids={lead.rids} vs {manifest.rids}, "
|
||||
f"keys={lead.keys} vs {manifest.keys}, "
|
||||
f"dtype={lead.dtype} vs {manifest.dtype}, "
|
||||
f"width={lead.width} vs {manifest.width}"
|
||||
),
|
||||
)
|
||||
replication = len(manifests)
|
||||
actions: List[int] = []
|
||||
owners: List[int] = []
|
||||
for index, key in enumerate(lead.keys):
|
||||
owner = key.hash % replication
|
||||
if all(manifest.cached[index] for manifest in manifests):
|
||||
action = LOCAL_HIT
|
||||
elif manifests[owner].cached[index]:
|
||||
action = OWNER_CACHE_BROADCAST
|
||||
else:
|
||||
action = OWNER_ENCODE_BROADCAST
|
||||
actions.append(action)
|
||||
owners.append(owner)
|
||||
return OwnerPlan(actions=actions, owners=owners)
|
||||
|
||||
|
||||
def _prepare_transfers(
|
||||
session: MmOwnerSession,
|
||||
requests: Sequence[ImageSpanRequest],
|
||||
keys: List[ImageSpanKey],
|
||||
plan: OwnerPlan,
|
||||
features: Dict[SpanKey, torch.Tensor],
|
||||
encode: SpanEncoder,
|
||||
) -> Dict[int, torch.Tensor]:
|
||||
rank = session.group.rank_in_group
|
||||
buffers: Dict[int, torch.Tensor] = {}
|
||||
owned: List[int] = []
|
||||
for index, (action, owner) in enumerate(zip(plan.actions, plan.owners)):
|
||||
if action == LOCAL_HIT:
|
||||
continue
|
||||
if owner != rank:
|
||||
try:
|
||||
buffers[index] = _new_span_buffer(session, keys[index])
|
||||
except Exception as exc:
|
||||
raise RuntimeError(
|
||||
f"receive buffer for image hash {keys[index].hash} shape "
|
||||
f"{(keys[index].span_len, session.width)} {session.dtype} "
|
||||
f"failed: {type(exc).__name__}: {exc}"
|
||||
) from exc
|
||||
elif action == OWNER_CACHE_BROADCAST:
|
||||
key = (keys[index].hash, keys[index].span_len)
|
||||
buffers[index] = features[key].contiguous()
|
||||
else:
|
||||
owned.append(index)
|
||||
if owned:
|
||||
try:
|
||||
encoded = encode([requests[index].item for index in owned])
|
||||
except Exception as exc:
|
||||
raise RuntimeError(
|
||||
f"owner encode of image hashes {[keys[i].hash for i in owned]} failed: "
|
||||
f"{type(exc).__name__}: {exc}"
|
||||
) from exc
|
||||
spans = _split_spans(encoded, [keys[index].span_len for index in owned])
|
||||
for index, span in zip(owned, spans):
|
||||
buffers[index] = _validated_span(session, keys[index], span)
|
||||
return buffers
|
||||
|
||||
|
||||
def _new_span_buffer(session: MmOwnerSession, key: ImageSpanKey) -> torch.Tensor:
|
||||
return torch.empty(
|
||||
(key.span_len, session.width), device=session.device, dtype=session.dtype
|
||||
)
|
||||
|
||||
|
||||
def _split_spans(
|
||||
encoded: torch.Tensor | List[torch.Tensor], span_lens: List[int]
|
||||
) -> List[torch.Tensor]:
|
||||
if isinstance(encoded, list):
|
||||
if len(encoded) != len(span_lens):
|
||||
raise ValueError(
|
||||
f"encoder returned {len(encoded)} spans for {len(span_lens)} images"
|
||||
)
|
||||
return [span.reshape(-1, span.shape[-1]) for span in encoded]
|
||||
encoded = encoded.reshape(-1, encoded.shape[-1])
|
||||
if encoded.shape[0] != sum(span_lens):
|
||||
raise ValueError(
|
||||
f"encoder returned {encoded.shape[0]} rows for spans of {span_lens}"
|
||||
)
|
||||
return list(torch.split(encoded, span_lens, dim=0))
|
||||
|
||||
|
||||
def _validated_span(
|
||||
session: MmOwnerSession, key: ImageSpanKey, span: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
expected = (key.span_len, session.width)
|
||||
if tuple(span.shape) != expected or span.dtype != session.dtype:
|
||||
raise ValueError(
|
||||
f"encoded span for hash={key.hash} has shape {tuple(span.shape)} "
|
||||
f"dtype {span.dtype}; expected {expected} {session.dtype}"
|
||||
)
|
||||
if span.device != session.device:
|
||||
span = span.to(session.device)
|
||||
return span.contiguous()
|
||||
|
||||
|
||||
def _synchronize(device) -> None:
|
||||
if device.type == "cuda":
|
||||
torch.cuda.current_stream(device).synchronize()
|
||||
|
||||
|
||||
def _describe(session: MmOwnerSession, stage: str, exc: BaseException) -> str:
|
||||
return (
|
||||
f"multimodal owner protocol failed during {stage} on group rank "
|
||||
f"{session.group.rank_in_group} (global rank "
|
||||
f"{session.group.ranks[session.group.rank_in_group]}, rids={list(session.rids)}): "
|
||||
f"{type(exc).__name__}: {exc}"
|
||||
)
|
||||
@@ -5,6 +5,7 @@ from typing import Callable, Dict, List, Optional, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.managers.mm_owner_embedding import ImageSpanRequest, MmOwnerSession
|
||||
from sglang.srt.managers.schedule_batch import MultimodalDataItem
|
||||
from sglang.srt.mem_cache.multimodal_cache import EmbeddingResult, MultiModalStaticCache
|
||||
from sglang.srt.multimodal.evs import EVSEmbeddingResult
|
||||
@@ -430,6 +431,56 @@ def _batch_encode_per_image_misses(
|
||||
return hash_to_embedding
|
||||
|
||||
|
||||
def _item_overlap(
|
||||
item: MultimodalDataItem, chunk_start: int, chunk_end: int
|
||||
) -> Optional[int]:
|
||||
"""Token count of an item with several runs, or None if none intersects the chunk."""
|
||||
if len(item.offsets) == 1:
|
||||
start, end = item.offsets[0]
|
||||
return end - start + 1 if end >= chunk_start and start < chunk_end else None
|
||||
if any(end >= chunk_start and start < chunk_end for start, end in item.offsets):
|
||||
return sum(end - start + 1 for start, end in item.offsets)
|
||||
return None
|
||||
|
||||
|
||||
def _collect_image_span_requests(
|
||||
per_image_requests: List[PerImageRequestInfo],
|
||||
) -> List[ImageSpanRequest]:
|
||||
spans: Dict[Tuple[Optional[int], int], ImageSpanRequest] = {}
|
||||
for req_info in per_image_requests:
|
||||
chunk_start = req_info.extend_prefix_len
|
||||
chunk_end = chunk_start + req_info.extend_seq_len # exclusive
|
||||
# PerImageRequestInfo.overlapping is the 4-tuple (idx, item, start, end)
|
||||
# convention shared with _batch_encode_per_image_misses /
|
||||
# _assemble_per_image_chunk; the span cache key is the item's full
|
||||
# token count, matching cache_key = (item.hash, end - start + 1).
|
||||
req_info.overlapping = [
|
||||
(idx, item, start, end)
|
||||
for idx, (item, (start, end)) in enumerate(
|
||||
zip(req_info.items, req_info.items_offset)
|
||||
)
|
||||
if _item_overlap(item, chunk_start, chunk_end) is not None
|
||||
]
|
||||
for _idx, item, start, end in req_info.overlapping:
|
||||
token_count = end - start + 1
|
||||
span = spans.get((item.hash, token_count))
|
||||
if span is None:
|
||||
spans[(item.hash, token_count)] = ImageSpanRequest(
|
||||
hash=item.hash, span_len=token_count, item=item
|
||||
)
|
||||
elif span.item is not item:
|
||||
span.duplicates.append(item)
|
||||
return list(spans.values())
|
||||
|
||||
|
||||
def _owner_span_encoder(data_embedding_func: DataEmbeddingFunc, device: torch.device):
|
||||
def encode(items: List[MultimodalDataItem]):
|
||||
_move_items_to_device(items, device, data_embedding_func)
|
||||
return data_embedding_func(items)
|
||||
|
||||
return encode
|
||||
|
||||
|
||||
def _get_chunked_embedding_by_item(
|
||||
data_embedding_func: DataEmbeddingFunc,
|
||||
embedding_items_per_req: List[MultimodalDataItem],
|
||||
@@ -554,6 +605,7 @@ def _get_chunked_prefill_embedding(
|
||||
extend_length: List[int],
|
||||
items_offset_list: List[List[Tuple[int, int]]],
|
||||
input_ids: torch.Tensor,
|
||||
mm_owner: Optional[MmOwnerSession] = None,
|
||||
) -> tuple[torch.Tensor | None, torch.Tensor]:
|
||||
"""
|
||||
Chunked prefill embedding: encode items across all requests and extract
|
||||
@@ -615,7 +667,15 @@ def _get_chunked_prefill_embedding(
|
||||
|
||||
# Phase 1: batch encode all per-image cache misses in ONE ViT call
|
||||
hash_to_embedding: Dict[Tuple[Optional[int], int], torch.Tensor] = {}
|
||||
if per_image_requests:
|
||||
if per_image_requests and mm_owner is not None:
|
||||
# The owner protocol must see every overlapping span before any local
|
||||
# cache filtering: a rank-local hit can never skip a group collective.
|
||||
hash_to_embedding = mm_owner.resolve(
|
||||
_collect_image_span_requests(per_image_requests),
|
||||
cache=embedding_cache,
|
||||
encode=_owner_span_encoder(data_embedding_func, device),
|
||||
)
|
||||
elif per_image_requests:
|
||||
hash_to_embedding = _batch_encode_per_image_misses(
|
||||
data_embedding_func, per_image_requests, device
|
||||
)
|
||||
@@ -718,6 +778,7 @@ def get_embedding_and_mask(
|
||||
prefix_length: List[int],
|
||||
extend_length: List[int],
|
||||
items_offset_list: List[List[Tuple[int, int]]],
|
||||
mm_owner: Optional[MmOwnerSession] = None,
|
||||
) -> Tuple[torch.Tensor | None, torch.Tensor | None, torch.Tensor]:
|
||||
"""
|
||||
Generate multimodal embeddings and create a mask for identifying their positions in the input sequence.
|
||||
@@ -758,6 +819,7 @@ def get_embedding_and_mask(
|
||||
extend_length,
|
||||
items_offset_list,
|
||||
input_ids,
|
||||
mm_owner=mm_owner,
|
||||
)
|
||||
if embedding is None:
|
||||
return None, None, input_ids
|
||||
|
||||
@@ -11,6 +11,7 @@ import sys
|
||||
from abc import abstractmethod
|
||||
from array import array
|
||||
from collections import defaultdict
|
||||
from contextlib import nullcontext
|
||||
from multiprocessing import shared_memory
|
||||
from typing import Any, Callable, Dict, List, Optional, Tuple
|
||||
|
||||
@@ -26,6 +27,7 @@ from sglang.srt.managers.io_struct import (
|
||||
TokenizedEmbeddingReqInput,
|
||||
TokenizedGenerateReqInput,
|
||||
)
|
||||
from sglang.srt.managers.mm_owner_embedding import MmOwnerSession
|
||||
|
||||
# Preserve the existing initialization import for downstream callers.
|
||||
from sglang.srt.managers.mm_schedule import (
|
||||
@@ -416,6 +418,7 @@ def embed_mm_inputs(
|
||||
data_embedding_func_mapping: Dict[Modality, DataEmbeddingFunc] = None,
|
||||
placeholder_tokens: dict[Modality, List[int]] = None,
|
||||
use_deepstack: Dict[Modality, bool] = {},
|
||||
mm_owner: Optional[MmOwnerSession] = None,
|
||||
) -> Optional[torch.Tensor]:
|
||||
"""
|
||||
Embed multimodal inputs and integrate them with text token embeddings.
|
||||
@@ -497,6 +500,7 @@ def embed_mm_inputs(
|
||||
prefix_length=extend_prefix_lens,
|
||||
extend_length=extend_seq_lens,
|
||||
items_offset_list=items_offsets,
|
||||
mm_owner=mm_owner,
|
||||
)
|
||||
|
||||
if use_deepstack.get(modality, None) and embedding is not None:
|
||||
@@ -517,7 +521,12 @@ def embed_mm_inputs(
|
||||
# filled with the hash values of the multimodal for the prefix matching in the radix attention.
|
||||
# There values are useless because their embeddings will be replaced by vision embeddings anyway.
|
||||
input_ids.clamp_(min=0, max=vocab_size - 1)
|
||||
input_embeds = input_embedding(input_ids)
|
||||
if mm_owner is not None:
|
||||
# The text embedding may all-reduce across TP; a rank-local failure in
|
||||
# feature preparation has to be agreed on before any rank enters it.
|
||||
mm_owner.features_ready()
|
||||
with mm_owner.uncaptured() if mm_owner is not None else nullcontext():
|
||||
input_embeds = input_embedding(input_ids)
|
||||
|
||||
# deepstack embedding
|
||||
if use_deepstack:
|
||||
|
||||
@@ -1111,6 +1111,15 @@ class PrefillAdder:
|
||||
else AddReqResult.CONTINUE
|
||||
)
|
||||
|
||||
def can_share_extend_batch(self, req: Req) -> bool:
|
||||
# Token embedding overrides embed the batch's raw input_ids before the
|
||||
# model runs, and that lookup cannot index multimodal placeholder hash IDs.
|
||||
if req.positional_embed_overrides is not None:
|
||||
return all(r.multimodal_inputs is None for r in self.can_run_list)
|
||||
if req.multimodal_inputs is not None:
|
||||
return all(r.positional_embed_overrides is None for r in self.can_run_list)
|
||||
return True
|
||||
|
||||
def add_chunked_req(self, req: Req):
|
||||
if self.dllm_config is not None:
|
||||
_rem_tokens = self._get_dllm_remain_tokens(req)
|
||||
|
||||
@@ -163,6 +163,8 @@ from sglang.srt.managers.io_struct import (
|
||||
PauseGenerationReqInput,
|
||||
PdRoleSwitchReqInput,
|
||||
ProfileReq,
|
||||
QueryCacheHitReqInput,
|
||||
QueryCacheHitReqOutput,
|
||||
ReleaseMemoryOccupationReqInput,
|
||||
RemoveExternalCorpusReqInput,
|
||||
RemoveExternalCorpusReqOutput,
|
||||
@@ -1799,6 +1801,7 @@ class Scheduler(
|
||||
(ShutdownReq, self.handle_shutdown),
|
||||
(GetInternalStateReq, self.get_internal_state),
|
||||
(SetInternalStateReq, self.set_internal_state),
|
||||
(QueryCacheHitReqInput, self.handle_query_cache_hit),
|
||||
(RpcReqInput, self.handle_rpc_request),
|
||||
(ExpertDistributionReq, self.expert_distribution_handle),
|
||||
(LoadLoRAAdapterReqInput, self.load_lora_adapter),
|
||||
@@ -2090,9 +2093,13 @@ class Scheduler(
|
||||
vmm_errors = self._materialize_cuda_vmm_inputs(recv_req)
|
||||
|
||||
# Skip health check when server is busy — ongoing requests already carry health info.
|
||||
if is_health_check_generate_req(recv_req) and not self.is_fully_idle(
|
||||
for_health_check=True
|
||||
):
|
||||
# NOTE: the admit/skip decision must be identical on every CP/TP rank.
|
||||
# is_fully_idle() includes rank-local hicache drain queues, which diverge
|
||||
# across ranks right after activity; a divergent decision lets one rank
|
||||
# dispatch the health-check generate while others piggyback-skip, breaking
|
||||
# collective ordering (deadlock: one rank blocks in the hicache drain
|
||||
# all_reduce while another waits in the CP request broadcast).
|
||||
if is_health_check_generate_req(recv_req) and not self.is_sched_idle_cp_symmetric():
|
||||
self.return_health_check_ipcs.append(
|
||||
getattr(recv_req, "http_worker_ipc", None)
|
||||
)
|
||||
@@ -2417,6 +2424,11 @@ class Scheduler(
|
||||
self._prev_step: Optional[Tuple[int, float, bool]] = None
|
||||
self._prev_prefill_end_ts: Optional[float] = None
|
||||
self._sched_idled = False
|
||||
# G9: sliding-window drain events (monotonic_ts, total_input_tokens) for
|
||||
# the queue-time estimator. Recorded once per request when its final
|
||||
# prefill chunk completes (batch_result_processor), in the same
|
||||
# total-input-length unit as LoadSnapshot.queued_total_input_tokens.
|
||||
self.prefill_drain_window: deque = deque()
|
||||
self.load_inquirer = SchedulerLoadInquirer(
|
||||
disaggregation_mode=self.disaggregation_mode,
|
||||
server_args=self.server_args,
|
||||
@@ -2456,8 +2468,35 @@ class Scheduler(
|
||||
),
|
||||
get_total_prefill_busy_us=lambda: self.total_prefill_busy_us,
|
||||
get_decode_moment_totals=lambda: self.decode_moment_totals,
|
||||
get_prefill_drain_rate=self.get_prefill_drain_rate,
|
||||
)
|
||||
|
||||
PREFILL_DRAIN_WINDOW_S = 60.0
|
||||
|
||||
def record_prefill_drain(self, total_input_tokens: int) -> None:
|
||||
"""G9: one drain event per request whose final prefill chunk completed.
|
||||
|
||||
Tokens are the request's total input length (seqlen), matching the
|
||||
queued_total_input_tokens numerator so the router's
|
||||
wait = queued_tokens / drain_rate is unit-consistent (决策②).
|
||||
"""
|
||||
now = time.monotonic()
|
||||
self.prefill_drain_window.append((now, total_input_tokens))
|
||||
cutoff = now - self.PREFILL_DRAIN_WINDOW_S
|
||||
while self.prefill_drain_window and self.prefill_drain_window[0][0] < cutoff:
|
||||
self.prefill_drain_window.popleft()
|
||||
|
||||
def get_prefill_drain_rate(self) -> float:
|
||||
"""Completed prefill input tokens per wall second over the window."""
|
||||
now = time.monotonic()
|
||||
cutoff = now - self.PREFILL_DRAIN_WINDOW_S
|
||||
while self.prefill_drain_window and self.prefill_drain_window[0][0] < cutoff:
|
||||
self.prefill_drain_window.popleft()
|
||||
if not self.prefill_drain_window:
|
||||
return 0.0
|
||||
span = max(now - self.prefill_drain_window[0][0], 1.0)
|
||||
return sum(tokens for _, tokens in self.prefill_drain_window) / span
|
||||
|
||||
def init_output_streamer(self) -> None:
|
||||
self.output_streamer = self.get_output_streamer_class()(
|
||||
send_to_detokenizer=self.ipc_channels.send_to_detokenizer,
|
||||
@@ -2510,6 +2549,7 @@ class Scheduler(
|
||||
output_streamer=self.output_streamer,
|
||||
beam_coordinator=self.beam_coordinator,
|
||||
abort_request=self.abort_request,
|
||||
record_prefill_drain=self.record_prefill_drain,
|
||||
)
|
||||
|
||||
def init_req_max_new_tokens(self, req):
|
||||
@@ -3954,6 +3994,8 @@ class Scheduler(
|
||||
for req in self.waiting_queue:
|
||||
if self.enable_lora and not self.can_schedule_lora_req(req, running_loras):
|
||||
continue
|
||||
if not adder.can_share_extend_batch(req):
|
||||
break
|
||||
|
||||
# A forward batch runs one pooling mode, so setwise readout requests
|
||||
# (token_indices_to_pool) cannot share a batch with last-token ones.
|
||||
@@ -5008,6 +5050,31 @@ class Scheduler(
|
||||
else:
|
||||
self.metrics_reporter.record_scheduler_active()
|
||||
|
||||
def is_sched_idle_cp_symmetric(self) -> bool:
|
||||
"""Idle check using only state that is identical across CP/TP ranks.
|
||||
|
||||
Request/batch/queue state is collectively maintained (requests arrive
|
||||
via broadcast, batches are collectively scheduled), so every rank
|
||||
computes the same result. Rank-local hicache drain and disagg transfer
|
||||
queues are deliberately excluded: those are exactly the terms that
|
||||
diverge across ranks and caused the CP health-check deadlock
|
||||
(hicache drain all_reduce vs CP request broadcast cross-collective
|
||||
wait, seen on cp2/cp4 + hicache L3 right after router health checks).
|
||||
|
||||
Used only for health-check admission; all other idle logic keeps using
|
||||
is_fully_idle().
|
||||
"""
|
||||
return (
|
||||
self.running_batch.is_empty()
|
||||
and self.chunked_req is None
|
||||
and not self.dllm_manager.any_staging_reqs()
|
||||
and (self.last_batch is None or self.last_batch.is_empty())
|
||||
and (not self.enable_overlap or len(self.result_queue) == 0)
|
||||
and self._pp_microbatches_drained()
|
||||
and len(self.waiting_queue) == 0
|
||||
and len(self.grammar_manager.grammar_queue) == 0
|
||||
)
|
||||
|
||||
def is_fully_idle(self, for_health_check=False, ignore_waiting=False) -> bool:
|
||||
# Health check piggybacks on running requests in process_output.
|
||||
# Only running_batch + waiting_queue guarantee active GPU processing;
|
||||
@@ -5215,6 +5282,86 @@ class Scheduler(
|
||||
success = False
|
||||
return success
|
||||
|
||||
def handle_query_cache_hit(self, recv_req: QueryCacheHitReqInput):
|
||||
"""G9: read-only prefix-hit probe (L1/L2 radix + L3 storage).
|
||||
|
||||
The router calls this (via /query_cache_hit on the decode engine) to
|
||||
decide conditional PD-bypass. Runs inside the scheduler event loop so
|
||||
radix/storage state is consistent, and reuses the exact admission-time
|
||||
match + L3 query logic (mirrors DecodeHiCachePreallocMixin
|
||||
._build_decode_prefix_match) without creating a Req or mutating
|
||||
request state. Node access metadata (LRU) may be touched; a hit found
|
||||
here can still be evicted before the real request arrives (inherent
|
||||
TOCTOU, acceptable for an estimate).
|
||||
"""
|
||||
from sglang.srt.mem_cache.base_prefix_cache import MatchPrefixParams
|
||||
from sglang.srt.mem_cache.radix_cache import RadixKey
|
||||
|
||||
total = len(recv_req.input_ids)
|
||||
# 树节点键是 array("q"),RadixKey.match_at 断言两侧类型一致——
|
||||
# 查询侧必须同样用 array,否则树非空时匹配直接炸(2026-10-11 实测)。
|
||||
from array import array
|
||||
|
||||
query_ids = array("q", recv_req.input_ids)
|
||||
try:
|
||||
match_result = self.tree_cache.match_prefix(
|
||||
MatchPrefixParams(
|
||||
key=RadixKey(
|
||||
token_ids=query_ids,
|
||||
extra_key=recv_req.extra_key,
|
||||
cache_salt=recv_req.cache_salt,
|
||||
)
|
||||
)
|
||||
)
|
||||
l1_hit = len(match_result.device_indices)
|
||||
l2_hit = getattr(match_result, "host_hit_length", 0) or 0
|
||||
l3_hit = 0
|
||||
last_host_node = getattr(match_result, "last_host_node", None)
|
||||
# L3 查询条件:本调度器挂了 hicache 存储后端即可(P 节点 HiCache
|
||||
# write_through/write_back 场景),不必是 decode-hicache 模式。
|
||||
# SWA 模型的 decode radix 与 hicache 互斥(kv_cache_builder 硬限制),
|
||||
# 若只认 enable_decode_hicache,P 节点永远查不到 L3。
|
||||
can_query_l3 = getattr(self, "enable_decode_hicache", False) or getattr(
|
||||
self, "enable_hicache_storage", False
|
||||
)
|
||||
if (
|
||||
can_query_l3
|
||||
and last_host_node is not None
|
||||
and hasattr(self.tree_cache, "query_storage_hit_length")
|
||||
and (
|
||||
self.tree_cache.is_backuped(last_host_node)
|
||||
or self.tree_cache.is_root(last_host_node)
|
||||
)
|
||||
):
|
||||
matched_len = l1_hit + l2_hit
|
||||
suffix_tokens = query_ids[matched_len:]
|
||||
last_hash = self.tree_cache.get_last_hash_value(last_host_node)
|
||||
prefix_keys = (
|
||||
self.tree_cache.get_prefix_hash_values(last_host_node)
|
||||
if getattr(
|
||||
self.tree_cache, "hicache_storage_pass_prefix_keys", False
|
||||
)
|
||||
else None
|
||||
)
|
||||
l3_hit = self.tree_cache.query_storage_hit_length(
|
||||
last_host_node,
|
||||
suffix_tokens,
|
||||
last_hash,
|
||||
prefix_keys,
|
||||
extra_key=recv_req.extra_key,
|
||||
cache_salt=recv_req.cache_salt,
|
||||
)
|
||||
return QueryCacheHitReqOutput(
|
||||
total_tokens=total,
|
||||
l1_hit_tokens=l1_hit,
|
||||
l2_hit_tokens=l2_hit,
|
||||
l3_hit_tokens=l3_hit,
|
||||
dp_rank=int(get_parallel().dp_rank or 0),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"handle_query_cache_hit failed: {e}")
|
||||
return QueryCacheHitReqOutput(total_tokens=total, error=str(e))
|
||||
|
||||
def get_internal_state(self, recv_req: GetInternalStateReq):
|
||||
# Resolved config (pristine server_args + post-publish overrides) so a
|
||||
# readback reflects values changed via /set_internal_state, not startup.
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user