diff --git a/benchmark/kernels/deepseek/benchmark_h20_block_fp8.py b/benchmark/kernels/deepseek/benchmark_h20_block_fp8.py new file mode 100644 index 0000000000..bd50fbe354 --- /dev/null +++ b/benchmark/kernels/deepseek/benchmark_h20_block_fp8.py @@ -0,0 +1,146 @@ +"""Compare default, H200, and H20-3e block-FP8 configs on an H20-3e. + +Example: + python benchmark/kernels/deepseek/benchmark_h20_block_fp8.py \ + --output h20_block_fp8.jsonl + +This measures individual GEMMs, not end-to-end model throughput. Run on an +otherwise idle GPU. Inputs are synthetic; no model weights are needed. +""" + +import argparse +import json +import pathlib +import statistics +from unittest.mock import patch + +import torch +import triton + +from sglang.kernels.ops.gemm import fp8_kernel + +SHAPES = [ + (1792, 5120), + (25600, 6144), + (4096, 1280), + (5120, 1024), + (5120, 288), + (576, 5120), +] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=pathlib.Path, required=True) + parser.add_argument( + "--batch-sizes", + type=int, + nargs="+", + default=[1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64], + ) + parser.add_argument("--trials", type=int, default=5) + parser.add_argument("--rep-ms", type=int, default=50) + parser.add_argument("--seed", type=int, default=98765) + args = parser.parse_args() + assert torch.cuda.get_device_name() == "NVIDIA H20-3e" + assert args.trials > 0 and args.rep_ms > 0 and all(m > 0 for m in args.batch_sizes) + torch.backends.cuda.matmul.allow_tf32 = False + torch.manual_seed(args.seed) + config_dir = pathlib.Path(fp8_kernel.__file__).parent / "configs" + with args.output.open("x") as output: + + def emit(row): + text = json.dumps(row) + output.write(text + "\n") + output.flush() + print(text, flush=True) + + emit( + { + "kind": "environment", + "device": torch.cuda.get_device_name(), + "torch": torch.__version__, + "triton": triton.__version__, + "seed": args.seed, + "batch_sizes": args.batch_sizes, + } + ) + for n, k in SHAPES: + b = (torch.randn(n, k, device="cuda") * 0.2).to(torch.float8_e4m3fn) + bs = torch.rand(n // 32, k // 32, device="cuda") + 0.1 + b_dequant = b.float() * bs.repeat_interleave(32, 0).repeat_interleave(32, 1) + tables = {"default": None} + for device in ("NVIDIA_H200", "NVIDIA_H20-3e"): + path = ( + config_dir + / f"N={n},K={k},device_name={device},dtype=fp8_w8a8,block_shape=[32, 32].json" + ) + if path.exists(): + tables[device] = { + int(m): cfg for m, cfg in json.loads(path.read_text()).items() + } + assert "NVIDIA_H20-3e" in tables, ( + "Install the H20-3e configuration files before benchmarking" + ) + for m in args.batch_sizes: + a = (torch.randn(m, k, device="cuda") * 0.2).to(torch.float8_e4m3fn) + scales = (torch.rand(k // 32, m, device="cuda") + 0.1).T + reference = ( + (a.float() * scales.repeat_interleave(32, 1)) @ b_dequant.T + ).to(torch.bfloat16) + results = {} + # Compile and validate every variant before timing any variant. + for label, table in tables.items(): + with patch.object( + fp8_kernel, "get_w8a8_block_fp8_configs", return_value=table + ): + actual = fp8_kernel.w8a8_block_fp8_matmul_triton( + a, b, scales, bs, [32, 32], torch.bfloat16 + ) + relative_l2 = ( + torch.linalg.vector_norm(actual.float() - reference.float()) + / torch.linalg.vector_norm(reference.float()) + ).item() + assert torch.isfinite(actual).all() and relative_l2 < 0.01 + results[label] = { + "config": None + if table is None + else table[min(table, key=lambda key: abs(key - m))], + "relative_l2": relative_l2, + "trials_us": [], + } + for trial in range(args.trials): + labels = list(tables) + # Rotate variant order to reduce systematic timing-order bias. + labels = ( + labels[trial % len(labels) :] + labels[: trial % len(labels)] + ) + for label in labels: + with patch.object( + fp8_kernel, + "get_w8a8_block_fp8_configs", + return_value=tables[label], + ): + + def invoke(): + return fp8_kernel.w8a8_block_fp8_matmul_triton( + a, b, scales, bs, [32, 32], torch.bfloat16 + ) + + elapsed = ( + triton.testing.do_bench_cudagraph( + invoke, rep=args.rep_ms + ) + * 1000 + ) + results[label]["trials_us"].append(elapsed) + for result in results.values(): + result["median_us"] = statistics.median(result["trials_us"]) + emit( + {"kind": "measurement", "m": m, "n": n, "k": k, "results": results} + ) + emit({"kind": "complete"}) + + +if __name__ == "__main__": + main() diff --git a/benchmark/serving/benchmark_h20_decode.py b/benchmark/serving/benchmark_h20_decode.py new file mode 100644 index 0000000000..d03b72ad44 --- /dev/null +++ b/benchmark/serving/benchmark_h20_decode.py @@ -0,0 +1,181 @@ +"""Finite cold-cache decode benchmark using synthetic token IDs. + +Run against an otherwise idle SGLang server. This clears its prefix cache. +Example: + python benchmark/serving/benchmark_h20_decode.py \ + --model-path /path/to/DeepSeek-V4.1-Flash --output before.jsonl + +Repeat after installing the H20-3e configurations with identical server flags. +""" + +import argparse +import concurrent.futures +import datetime +import hashlib +import json +import pathlib +import statistics +import time +import urllib.request + +from tokenizers import Tokenizer + +p = argparse.ArgumentParser(description=__doc__) +p.add_argument("--label", default="measurement") +p.add_argument("--url", default="http://127.0.0.1:8000") +p.add_argument("--model-path", type=pathlib.Path, required=True) +p.add_argument("--output", type=pathlib.Path, required=True) +args = p.parse_args() +out = args.output +assert not out.exists() +tokenizer = Tokenizer.from_file(str(args.model_path / "tokenizer.json")) +filler = tokenizer.encode( + ( + "The research server processes independent records. Each record contains an identifier, a timestamp, and a measured value. We compare latency using the same data and the same output budget.\n" + ) + * 16000, + add_special_tokens=False, +).ids + + +def emit(row): + row = { + "time": datetime.datetime.now(datetime.timezone.utc).isoformat(), + "label": args.label, + **row, + } + with out.open("a") as f: + f.write(json.dumps(row) + "\n") + print(json.dumps(row), flush=True) + + +def request(path, body=None): + return urllib.request.urlopen( + urllib.request.Request( + args.url.rstrip("/") + path, + data=None if body is None else json.dumps(body).encode(), + headers={"Content-Type": "application/json"}, + ), + timeout=None, + ) + + +def flush(): + with request("/flush_cache?timeout=30", {}) as r: + r.read() + + +def native(case, isl, osl, index): + ids = ( + tokenizer.encode( + f"Independent record {index}: {case}.\n", add_special_tokens=False + ).ids + + filler + )[:isl] + start = time.perf_counter() + first = last = None + final = None + events = 0 + body = { + "input_ids": ids, + "sampling_params": { + "temperature": 0.0, + "max_new_tokens": osl, + "ignore_eos": True, + }, + "stream": True, + } + with request("/generate", body) as response: + for line in response: + if not line.startswith(b"data:"): + continue + value = line[5:].strip() + if value == b"[DONE]": + break + obj = json.loads(value) + now = time.perf_counter() + if obj.get("error"): + raise RuntimeError(str(obj["error"])) + if obj.get("meta_info", {}).get("completion_tokens", 0) > 0: + first = first or now + last = now + final = obj + events += 1 + elapsed = time.perf_counter() - start + assert final is not None, (case, index) + meta = final["meta_info"] + assert meta["completion_tokens"] == osl and meta["prompt_tokens"] == isl, meta + assert meta.get("cached_tokens") == 0, meta + return { + "kind": "request", + "case": case, + "index": index, + "input_hash": hashlib.sha256(json.dumps(ids).encode()).hexdigest(), + "prompt_tokens": isl, + "completion_tokens": osl, + "cached_tokens": meta["cached_tokens"], + "ttft_s": first - start, + "elapsed_s": elapsed, + "decode_tokens_per_s": (osl - 1) / (last - first) if osl > 1 else None, + "tpot_ms": 1000 * (last - first) / (osl - 1) if osl > 1 else None, + "events": events, + } + + +cases = [ + ("8k_1000_c1", 8192, 1000, 1), + ("1k_128_c1", 1024, 128, 1), + ("1k_128_c8", 1024, 128, 8), + ("1k_128_c32", 1024, 128, 32), + ("8k_128_c8", 8192, 128, 8), + ("32k_128_c1", 32768, 128, 1), +] +emit( + { + "kind": "start", + "cases": cases, + "repetitions": 3, + } +) +try: + for case, isl, osl, concurrency in cases: + flush() + emit({"kind": "warmup_start", "case": case}) + with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as pool: + list( + pool.map( + lambda i: native("warm_" + case, isl, 16, i), range(concurrency) + ) + ) + for repeat in range(3): + flush() + start = time.perf_counter() + with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as pool: + rows = list( + pool.map( + lambda i: native(case, isl, osl, i + repeat * 100), + range(concurrency), + ) + ) + wall = time.perf_counter() - start + for row in rows: + emit({**row, "repeat": repeat}) + emit( + { + "kind": "batch", + "case": case, + "repeat": repeat, + "concurrency": concurrency, + "wall_s": wall, + "output_tokens_per_s": concurrency * osl / wall, + "median_ttft_s": statistics.median(x["ttft_s"] for x in rows), + "median_decode_tokens_per_s": statistics.median( + x["decode_tokens_per_s"] for x in rows + ), + "median_tpot_ms": statistics.median(x["tpot_ms"] for x in rows), + } + ) + emit({"kind": "complete"}) +except Exception as exc: + emit({"kind": "failed", "error": repr(exc)}) + raise diff --git a/python/sglang/kernels/ops/gemm/configs/N=1792,K=5120,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[32, 32].json b/python/sglang/kernels/ops/gemm/configs/N=1792,K=5120,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[32, 32].json new file mode 100644 index 0000000000..5401a493f4 --- /dev/null +++ b/python/sglang/kernels/ops/gemm/configs/N=1792,K=5120,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[32, 32].json @@ -0,0 +1,138 @@ +{ + "0": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": true, + "SPLIT_K": 16 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": true, + "SPLIT_K": 16 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": true, + "SPLIT_K": 16 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": true, + "SPLIT_K": 16 + }, + "12": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": true, + "SPLIT_K": 16 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 16 + }, + "24": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": false, + "SPLIT_K": 8 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": false, + "SPLIT_K": 8 + }, + "40": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": false, + "SPLIT_K": 8 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": false, + "SPLIT_K": 8 + }, + "56": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": false, + "SPLIT_K": 8 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": false, + "SPLIT_K": 8 + }, + "65": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/kernels/ops/gemm/configs/N=1792,K=5120,device_name=NVIDIA_H20-3e,dtype=fp8_w8a8,block_shape=[32, 32].json b/python/sglang/kernels/ops/gemm/configs/N=1792,K=5120,device_name=NVIDIA_H20-3e,dtype=fp8_w8a8,block_shape=[32, 32].json new file mode 100644 index 0000000000..5401a493f4 --- /dev/null +++ b/python/sglang/kernels/ops/gemm/configs/N=1792,K=5120,device_name=NVIDIA_H20-3e,dtype=fp8_w8a8,block_shape=[32, 32].json @@ -0,0 +1,138 @@ +{ + "0": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": true, + "SPLIT_K": 16 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": true, + "SPLIT_K": 16 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": true, + "SPLIT_K": 16 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": true, + "SPLIT_K": 16 + }, + "12": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": true, + "SPLIT_K": 16 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 16 + }, + "24": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": false, + "SPLIT_K": 8 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": false, + "SPLIT_K": 8 + }, + "40": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": false, + "SPLIT_K": 8 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": false, + "SPLIT_K": 8 + }, + "56": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": false, + "SPLIT_K": 8 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": false, + "SPLIT_K": 8 + }, + "65": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/kernels/ops/gemm/configs/N=25600,K=6144,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[32, 32].json b/python/sglang/kernels/ops/gemm/configs/N=25600,K=6144,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[32, 32].json new file mode 100644 index 0000000000..4207135048 --- /dev/null +++ b/python/sglang/kernels/ops/gemm/configs/N=25600,K=6144,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[32, 32].json @@ -0,0 +1,126 @@ +{ + "0": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 8 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 8 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 8 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 2 + }, + "12": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 2 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 2 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "40": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "56": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "65": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/kernels/ops/gemm/configs/N=25600,K=6144,device_name=NVIDIA_H20-3e,dtype=fp8_w8a8,block_shape=[32, 32].json b/python/sglang/kernels/ops/gemm/configs/N=25600,K=6144,device_name=NVIDIA_H20-3e,dtype=fp8_w8a8,block_shape=[32, 32].json new file mode 100644 index 0000000000..4207135048 --- /dev/null +++ b/python/sglang/kernels/ops/gemm/configs/N=25600,K=6144,device_name=NVIDIA_H20-3e,dtype=fp8_w8a8,block_shape=[32, 32].json @@ -0,0 +1,126 @@ +{ + "0": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 8 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 8 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 8 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 2 + }, + "12": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 2 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 2 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "40": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "56": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "65": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/kernels/ops/gemm/configs/N=4096,K=1280,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[32, 32].json b/python/sglang/kernels/ops/gemm/configs/N=4096,K=1280,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[32, 32].json new file mode 100644 index 0000000000..4f779051ff --- /dev/null +++ b/python/sglang/kernels/ops/gemm/configs/N=4096,K=1280,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[32, 32].json @@ -0,0 +1,138 @@ +{ + "0": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": true, + "SPLIT_K": 8 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": true, + "SPLIT_K": 8 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": true, + "SPLIT_K": 8 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": true, + "SPLIT_K": 8 + }, + "12": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": true, + "SPLIT_K": 8 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": true, + "SPLIT_K": 8 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": false, + "SPLIT_K": 4 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": false, + "SPLIT_K": 4 + }, + "40": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": false, + "SPLIT_K": 4 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": false, + "SPLIT_K": 4 + }, + "56": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": false, + "SPLIT_K": 4 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": false, + "SPLIT_K": 4 + }, + "65": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/kernels/ops/gemm/configs/N=4096,K=1280,device_name=NVIDIA_H20-3e,dtype=fp8_w8a8,block_shape=[32, 32].json b/python/sglang/kernels/ops/gemm/configs/N=4096,K=1280,device_name=NVIDIA_H20-3e,dtype=fp8_w8a8,block_shape=[32, 32].json new file mode 100644 index 0000000000..4f779051ff --- /dev/null +++ b/python/sglang/kernels/ops/gemm/configs/N=4096,K=1280,device_name=NVIDIA_H20-3e,dtype=fp8_w8a8,block_shape=[32, 32].json @@ -0,0 +1,138 @@ +{ + "0": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": true, + "SPLIT_K": 8 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": true, + "SPLIT_K": 8 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": true, + "SPLIT_K": 8 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": true, + "SPLIT_K": 8 + }, + "12": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": true, + "SPLIT_K": 8 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": true, + "SPLIT_K": 8 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": false, + "SPLIT_K": 4 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": false, + "SPLIT_K": 4 + }, + "40": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": false, + "SPLIT_K": 4 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": false, + "SPLIT_K": 4 + }, + "56": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": false, + "SPLIT_K": 4 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": false, + "SPLIT_K": 4 + }, + "65": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/kernels/ops/gemm/configs/N=5120,K=1024,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[32, 32].json b/python/sglang/kernels/ops/gemm/configs/N=5120,K=1024,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[32, 32].json new file mode 100644 index 0000000000..0b1cb1b98e --- /dev/null +++ b/python/sglang/kernels/ops/gemm/configs/N=5120,K=1024,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[32, 32].json @@ -0,0 +1,132 @@ +{ + "0": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "SWAP_AB": true, + "SPLIT_K": 8 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": true, + "SPLIT_K": 8 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": true, + "SPLIT_K": 8 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": true, + "SPLIT_K": 8 + }, + "12": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": true, + "SPLIT_K": 8 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": true, + "SPLIT_K": 8 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": false, + "SPLIT_K": 4 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": false, + "SPLIT_K": 4 + }, + "40": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": false, + "SPLIT_K": 4 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "56": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "65": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/kernels/ops/gemm/configs/N=5120,K=1024,device_name=NVIDIA_H20-3e,dtype=fp8_w8a8,block_shape=[32, 32].json b/python/sglang/kernels/ops/gemm/configs/N=5120,K=1024,device_name=NVIDIA_H20-3e,dtype=fp8_w8a8,block_shape=[32, 32].json new file mode 100644 index 0000000000..0b1cb1b98e --- /dev/null +++ b/python/sglang/kernels/ops/gemm/configs/N=5120,K=1024,device_name=NVIDIA_H20-3e,dtype=fp8_w8a8,block_shape=[32, 32].json @@ -0,0 +1,132 @@ +{ + "0": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "SWAP_AB": true, + "SPLIT_K": 8 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": true, + "SPLIT_K": 8 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": true, + "SPLIT_K": 8 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": true, + "SPLIT_K": 8 + }, + "12": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": true, + "SPLIT_K": 8 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": true, + "SPLIT_K": 8 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": false, + "SPLIT_K": 4 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": false, + "SPLIT_K": 4 + }, + "40": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": false, + "SPLIT_K": 4 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "56": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "65": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/kernels/ops/gemm/configs/N=5120,K=288,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[32, 32].json b/python/sglang/kernels/ops/gemm/configs/N=5120,K=288,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[32, 32].json new file mode 100644 index 0000000000..84633f2187 --- /dev/null +++ b/python/sglang/kernels/ops/gemm/configs/N=5120,K=288,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[32, 32].json @@ -0,0 +1,126 @@ +{ + "0": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 1 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 1 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": true, + "SPLIT_K": 1 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": true, + "SPLIT_K": 1 + }, + "12": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 1 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 1 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "40": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "56": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "65": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/kernels/ops/gemm/configs/N=5120,K=288,device_name=NVIDIA_H20-3e,dtype=fp8_w8a8,block_shape=[32, 32].json b/python/sglang/kernels/ops/gemm/configs/N=5120,K=288,device_name=NVIDIA_H20-3e,dtype=fp8_w8a8,block_shape=[32, 32].json new file mode 100644 index 0000000000..84633f2187 --- /dev/null +++ b/python/sglang/kernels/ops/gemm/configs/N=5120,K=288,device_name=NVIDIA_H20-3e,dtype=fp8_w8a8,block_shape=[32, 32].json @@ -0,0 +1,126 @@ +{ + "0": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 1 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 1 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": true, + "SPLIT_K": 1 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": true, + "SPLIT_K": 1 + }, + "12": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 1 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 1 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "40": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "56": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "65": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/kernels/ops/gemm/configs/N=576,K=5120,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[32, 32].json b/python/sglang/kernels/ops/gemm/configs/N=576,K=5120,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[32, 32].json new file mode 100644 index 0000000000..0f54ee0450 --- /dev/null +++ b/python/sglang/kernels/ops/gemm/configs/N=576,K=5120,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[32, 32].json @@ -0,0 +1,138 @@ +{ + "0": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": true, + "SPLIT_K": 16 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": true, + "SPLIT_K": 16 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": true, + "SPLIT_K": 16 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": true, + "SPLIT_K": 16 + }, + "12": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": true, + "SPLIT_K": 16 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": true, + "SPLIT_K": 16 + }, + "24": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": false, + "SPLIT_K": 8 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": false, + "SPLIT_K": 8 + }, + "40": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": false, + "SPLIT_K": 8 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": false, + "SPLIT_K": 8 + }, + "56": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": false, + "SPLIT_K": 8 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": false, + "SPLIT_K": 8 + }, + "65": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/kernels/ops/gemm/configs/N=576,K=5120,device_name=NVIDIA_H20-3e,dtype=fp8_w8a8,block_shape=[32, 32].json b/python/sglang/kernels/ops/gemm/configs/N=576,K=5120,device_name=NVIDIA_H20-3e,dtype=fp8_w8a8,block_shape=[32, 32].json new file mode 100644 index 0000000000..0f54ee0450 --- /dev/null +++ b/python/sglang/kernels/ops/gemm/configs/N=576,K=5120,device_name=NVIDIA_H20-3e,dtype=fp8_w8a8,block_shape=[32, 32].json @@ -0,0 +1,138 @@ +{ + "0": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": true, + "SPLIT_K": 16 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": true, + "SPLIT_K": 16 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": true, + "SPLIT_K": 16 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": true, + "SPLIT_K": 16 + }, + "12": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": true, + "SPLIT_K": 16 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": true, + "SPLIT_K": 16 + }, + "24": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": false, + "SPLIT_K": 8 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": false, + "SPLIT_K": 8 + }, + "40": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": false, + "SPLIT_K": 8 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": false, + "SPLIT_K": 8 + }, + "56": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": false, + "SPLIT_K": 8 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": false, + "SPLIT_K": 8 + }, + "65": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/kernels/ops/quantization/configs/N=1792,K=5120,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[32, 32].json b/python/sglang/kernels/ops/quantization/configs/N=1792,K=5120,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[32, 32].json new file mode 100644 index 0000000000..ef49380a6e --- /dev/null +++ b/python/sglang/kernels/ops/quantization/configs/N=1792,K=5120,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[32, 32].json @@ -0,0 +1,134 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 8 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 8 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 8 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 8 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 8 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": false, + "SPLIT_K": 8 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": false, + "SPLIT_K": 8 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": false, + "SPLIT_K": 4 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": false, + "SPLIT_K": 4 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": false, + "SPLIT_K": 2 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "8192": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + } +} diff --git a/python/sglang/kernels/ops/quantization/configs/N=25600,K=6144,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[32, 32].json b/python/sglang/kernels/ops/quantization/configs/N=25600,K=6144,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[32, 32].json new file mode 100644 index 0000000000..5db0cd1bc4 --- /dev/null +++ b/python/sglang/kernels/ops/quantization/configs/N=25600,K=6144,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[32, 32].json @@ -0,0 +1,128 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 4 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": false, + "SPLIT_K": 4 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": false, + "SPLIT_K": 2 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "8192": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/kernels/ops/quantization/configs/N=4096,K=1280,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[32, 32].json b/python/sglang/kernels/ops/quantization/configs/N=4096,K=1280,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[32, 32].json new file mode 100644 index 0000000000..dddbb88774 --- /dev/null +++ b/python/sglang/kernels/ops/quantization/configs/N=4096,K=1280,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[32, 32].json @@ -0,0 +1,128 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 2 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 1 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": false, + "SPLIT_K": 2 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "8192": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/kernels/ops/quantization/configs/N=5120,K=1024,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[32, 32].json b/python/sglang/kernels/ops/quantization/configs/N=5120,K=1024,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[32, 32].json new file mode 100644 index 0000000000..e05c17ccc2 --- /dev/null +++ b/python/sglang/kernels/ops/quantization/configs/N=5120,K=1024,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[32, 32].json @@ -0,0 +1,126 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 1 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 1 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "8192": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/kernels/ops/quantization/configs/N=5120,K=288,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[32, 32].json b/python/sglang/kernels/ops/quantization/configs/N=5120,K=288,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[32, 32].json new file mode 100644 index 0000000000..484bd0de8c --- /dev/null +++ b/python/sglang/kernels/ops/quantization/configs/N=5120,K=288,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[32, 32].json @@ -0,0 +1,122 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 1 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 1 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 1 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 1 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "8192": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/kernels/ops/quantization/configs/N=576,K=5120,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[32, 32].json b/python/sglang/kernels/ops/quantization/configs/N=576,K=5120,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[32, 32].json new file mode 100644 index 0000000000..583e274c21 --- /dev/null +++ b/python/sglang/kernels/ops/quantization/configs/N=576,K=5120,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[32, 32].json @@ -0,0 +1,138 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 16 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 16 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 16 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 16 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 16 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4, + "SWAP_AB": true, + "SPLIT_K": 8 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": false, + "SPLIT_K": 16 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": false, + "SPLIT_K": 4 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": false, + "SPLIT_K": 4 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": false, + "SPLIT_K": 4 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": false, + "SPLIT_K": 2 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + "SWAP_AB": false, + "SPLIT_K": 2 + }, + "8192": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/test/manual/quant/test_h20_block_fp8_configs.py b/test/manual/quant/test_h20_block_fp8_configs.py new file mode 100644 index 0000000000..e210bec1b9 --- /dev/null +++ b/test/manual/quant/test_h20_block_fp8_configs.py @@ -0,0 +1,94 @@ +"""Validate H20-3e decode tables, including nearest-M interpolation. + +Run on an H20-3e with the proposed configuration files installed: + python test/manual/quant/test_h20_block_fp8_configs.py -v +""" + +import unittest + +import torch + +from sglang.kernels.ops.gemm.fp8_kernel import ( + get_w8a8_block_fp8_configs, + w8a8_block_fp8_matmul_triton, +) + +SHAPES = [ + (1792, 5120), + (25600, 6144), + (4096, 1280), + (5120, 1024), + (5120, 288), + (576, 5120), +] +DEFAULT_CONFIG = { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, +} + + +class TestH20BlockFP8Configs(unittest.TestCase): + @classmethod + def setUpClass(cls): + if ( + not torch.cuda.is_available() + or torch.cuda.get_device_name() != "NVIDIA H20-3e" + ): + raise unittest.SkipTest( + "These configurations are specific to NVIDIA H20-3e" + ) + torch.backends.cuda.matmul.allow_tf32 = False + + def test_untuned_batch_sizes_keep_default(self): + for n, k in SHAPES: + configs = get_w8a8_block_fp8_configs(n, k, 32, 32) + self.assertIsNotNone(configs, (n, k)) + for m in [0, *range(65, 8193), 32768, 131072, 1048576]: + selected = configs[min(configs, key=lambda key: abs(key - m))] + self.assertEqual(selected, DEFAULT_CONFIG, (m, n, k)) + + def test_all_decode_batches_against_fp32(self): + # Independent seeds and both scale strides exercise masked M tails, + # K=288's partial Split-K partition, and both supported output dtypes. + for seed in (1234, 5678): + torch.manual_seed(seed) + for n, k in SHAPES: + b = (torch.randn(n, k, device="cuda") * 0.2).to(torch.float8_e4m3fn) + bs = torch.rand(n // 32, k // 32, device="cuda") + 0.1 + b_dequant = b.float() * bs.repeat_interleave(32, 0).repeat_interleave( + 32, 1 + ) + for m in range(1, 66): + a = (torch.randn(m, k, device="cuda") * 0.2).to(torch.float8_e4m3fn) + scales = torch.rand(m, k // 32, device="cuda") + 0.1 + reference = ( + a.float() * scales.repeat_interleave(32, 1) + ) @ b_dequant.T + for column_major in (False, True): + a_scales = scales.T.contiguous().T if column_major else scales + for dtype in (torch.bfloat16, torch.float16): + with self.subTest( + seed=seed, + m=m, + n=n, + k=k, + column_major=column_major, + dtype=dtype, + ): + actual = w8a8_block_fp8_matmul_triton( + a, b, a_scales, bs, [32, 32], dtype + ) + expected = reference.to(dtype).float() + self.assertTrue(torch.isfinite(actual).all().item()) + relative_l2 = torch.linalg.vector_norm( + actual.float() - expected + ) / torch.linalg.vector_norm(expected) + self.assertLess(relative_l2.item(), 0.01) + + +if __name__ == "__main__": + unittest.main()