diff --git a/.claude/rules/kernel-benchmark.md b/.claude/rules/kernel-benchmark.md new file mode 100644 index 0000000000..9b2c8d434a --- /dev/null +++ b/.claude/rules/kernel-benchmark.md @@ -0,0 +1,20 @@ +--- +paths: + - "**/bench_*.py" + - "**/benchmark/**/*.py" + - "benchmark/kernels/**" +--- + +# Kernel benchmarks + +How a benchmark is written is up to you. The requirement is only that the numbers are valid. + +## Tools + +In SGLang we recommend `marker.do_bench` (`python/sglang/kernels/jit/benchmark/marker.py`). It works for any callable, not only JIT kernels, and it measures with a cold L2 by default. It is not mandatory. CUPTI (e.g. CUDA 13 `cupti-python`), the torch profiler / `bench_kineto`, Triton's `do_bench` / proton, `flashinfer.testing`, Nsight Systems / Nsight Compute, or a hand-written loop are all fine. Each pitfall below must be handled, whichever tool you use. + +## L2 cache reuse + +Timing the same kernel on the same inputs in a loop measures an L2-hot kernel, not the one serving runs. `triton.testing.do_bench_cudagraph` with fixed arguments and bare `for` loops between two events both do this. Measure cold L2 by flushing between calls or by rotating enough input copies to exceed L2 several times over. Hot L2 is acceptable only when the working set far exceeds L2 or when hot L2 is the real serving pattern, and the reason must be stated. + +Host-memory (zero-copy) loads do not go through device L2. Peer-GPU (NVLink) reads still see reuse (on remote GPU). diff --git a/.claude/skills/add-jit-kernel/SKILL.md b/.claude/skills/add-jit-kernel/SKILL.md index b5289b80d0..f68b43dd58 100644 --- a/.claude/skills/add-jit-kernel/SKILL.md +++ b/.claude/skills/add-jit-kernel/SKILL.md @@ -641,6 +641,8 @@ if __name__ == "__main__": Benchmarks are `bench_*.py` files under `test/registered/kernels/benchmark//`. They are picked up by the same `run_suite.py` machinery as unit tests. Register them for **`base-b-kernel-benchmark-test-1-gpu-large`** (PR JIT benchmark job: `python3 run_suite.py --hw cuda --suite base-b-kernel-benchmark-test-1-gpu-large`). +Every benchmark must account for L2 cache reuse — see [`rules/kernel-benchmark.md`](../../rules/kernel-benchmark.md). + Benchmarks use the project's own `marker` framework (in `python/sglang/kernels/jit/benchmark/marker.py`) — **do not** use `triton.testing.perf_report` / `triton.testing.do_bench` directly. The marker framework provides (public names: `benchmark`, `parametrize`, `do_bench`, `skip`, `BenchResult`, `BenchSkip`): - **`@marker.benchmark(line_arg, line_vals, *, unit="us")`** — the **innermost** decorator (bottom of the stack, directly above `def benchmark`). Declares the column axis: each value in `line_vals` becomes a result column, and `line_arg` is the parameter name passed into the benchmark function. `unit` is one of `"us" | "ms" | "s"`. @@ -649,7 +651,7 @@ Benchmarks use the project's own `marker` framework (in `python/sglang/kernels/j - `memory_args`: defaults to `"all"` (footprint derived from all input args/kwargs). Pass an explicit tuple of tensors (e.g. `(k, v, indices)`) to count only the inputs the kernel actually touches. - `memory_output`: defaults to `"out"` — re-runs `fn` once to capture its **returned** tensor and counts it. For in-place kernels (which return `None`), pass the written tensors explicitly (e.g. `memory_output=(k, v)`); the re-run is then skipped. Set to `None` to count no output. - Together `memory_args` + `memory_output` give the GB/s column; with both defaults a function `out = f(src)` already reports `bytes(src) + bytes(out)`. - - `graph_clone_args` / `graph_clone_kwargs`: which inputs to clone per CUDA-graph iteration to defeat L2 cache reuse. Defaults to `"all"` — pass an iterable of indices/keys to limit to the *read* args (writes don't need cloning). + - `graph_clone_args` / `graph_clone_kwargs`: which inputs to clone per CUDA-graph iteration to defeat L2 cache reuse. Defaults to `"all"`; if you narrow it, keep every read, in-place, and output tensor. - `use_cuda_graph=False` for kernels that can't be captured. - `metrics=(0.5, "avg")` controls reported quantiles (the first metric becomes the table latency column). - `disable_log_bandwidth` (defaults from `SGLANG_JIT_BENCHMARK_DISABLE_LOG_BANDWIDTH=1`) skips the bandwidth column entirely. diff --git a/.claude/skills/add-sgl-kernel/SKILL.md b/.claude/skills/add-sgl-kernel/SKILL.md index 6aaad777d5..6895b180de 100644 --- a/.claude/skills/add-sgl-kernel/SKILL.md +++ b/.claude/skills/add-sgl-kernel/SKILL.md @@ -29,7 +29,7 @@ Add a new operation that scales each element of a tensor by a scalar factor: In addition, every new kernel must ship with: - **Tests** (pytest) -- **A benchmark script** (triton.testing) +- **A benchmark script** (`marker.do_bench` from `sglang.kernels.jit.benchmark` — it works for any callable, not only JIT kernels) --- @@ -260,62 +260,46 @@ if __name__ == "__main__": ## Step 7: Add a benchmark (required) +Every benchmark must account for L2 cache reuse — see [`rules/kernel-benchmark.md`](../../rules/kernel-benchmark.md). + Create `python/sglang/kernels/aot/benchmark/bench_scale.py`: ```python -import itertools - import torch -import triton -import triton.testing import sgl_kernel -from sglang.utils import is_in_ci - -IS_CI = is_in_ci() - -dtypes = [torch.float16] if IS_CI else [torch.float16, torch.bfloat16, torch.float32] -sizes = [4096] if IS_CI else [2**n for n in range(10, 20)] # 1K … 512K -factors = [2.0] - -configs = list(itertools.product(dtypes, sizes)) +from sglang.kernels.jit.benchmark import marker -def torch_scale(input: torch.Tensor, factor: float) -> torch.Tensor: - return input * factor +def sglang_scale(input: torch.Tensor, factor: float, out: torch.Tensor) -> None: + sgl_kernel.scale(input, factor, out=out) -@triton.testing.perf_report( - triton.testing.Benchmark( - x_names=["dtype", "size"], - x_vals=configs, - line_arg="provider", - line_vals=["sglang", "torch"], - line_names=["SGL Kernel", "PyTorch"], - styles=[("green", "-"), ("red", "--")], - ylabel="µs (median)", - plot_name="scale-performance", - args={}, +def torch_scale(input: torch.Tensor, factor: float, out: torch.Tensor) -> None: + torch.mul(input, factor, out=out) + + +FN_MAP = {"sglang": sglang_scale, "torch": torch_scale} + + +@marker.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32], [torch.float16]) +@marker.parametrize("size", [2**n for n in range(10, 20)], [4096]) # 1K .. 512K +@marker.benchmark("provider", ["sglang", "torch"]) +def benchmark(dtype: torch.dtype, size: int, provider: str): + input = torch.randn(size, dtype=dtype, device="cuda") + out = torch.empty_like(input) + return marker.do_bench( + FN_MAP[provider], + # Pass every tensor through input_args (not a closure) so marker rotates + # them across CUDA-graph calls to defeat L2 reuse. + input_args=(input, 2.0, out), + # Bandwidth = bytes(input) + bytes(out), both already in input_args. + memory_output=None, ) -) -def benchmark(dtype, size, provider): - input = torch.randn(size, dtype=dtype, device="cuda") - out = torch.empty_like(input) - factor = 2.0 - - if provider == "sglang": - fn = lambda: sgl_kernel.scale(input, factor, out=out) - else: - fn = lambda: torch_scale(input, factor) - - ms, min_ms, max_ms = triton.testing.do_bench_cudagraph( - fn, quantiles=[0.5, 0.2, 0.8] - ) - return 1000 * ms, 1000 * max_ms, 1000 * min_ms if __name__ == "__main__": - benchmark.run(print_data=True) + benchmark.run() ``` ---