From e667ab10d5f76f34e3a949ec0e82f5568f7a0afc Mon Sep 17 00:00:00 2001 From: "Alex O. P." Date: Fri, 2 Oct 2026 17:26:28 +0200 Subject: [PATCH] [diffusion] quantization: ConvRot INT8 online W8A8 for DiTs (sgl-kernel + `convrot_int8_customkernel`) (#38040) Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com> --- .../cookbook/diffusion/MiniMax/MiniMax-H3.mdx | 42 +- .../environment_variables.mdx | 9 +- docs/docs/sglang-diffusion/quantization.mdx | 119 +- .../gemm/convrot_int8/convrot_int8_entry.cuh | 1021 +++++++++++++++++ .../epilogue/epilogue_per_row_per_col_scale.h | 341 ++++++ .../gemm/gemm_universal_base_compat.h | 388 +++++++ .../gemm/gemm_with_epilogue_visitor.h | 524 +++++++++ python/sglang/kernels/ops/gemm/__init__.py | 44 + .../sglang/kernels/ops/gemm/convrot_int8.py | 231 ++++ .../sglang-diffusion-performance/SKILL.md | 2 +- python/sglang/multimodal_gen/envs.py | 8 + .../runtime/layers/quantization/__init__.py | 12 +- .../configs/convrot_int8_config.py | 377 ++++++ .../configs/kitchen_int8_config.py | 189 --- .../configs/kitchen_w4a4_config.py | 6 +- ..._int8.py => convrot_int8_comfy_kitchen.py} | 37 +- .../layers/quantization/convrot_int8_jit.py | 384 +++++++ .../layers/quantization/method_names.py | 22 + .../layers/quantization/modelopt_quant.py | 8 +- .../component_loaders/text_encoder_loader.py | 13 +- .../component_loaders/transformer_loader.py | 14 +- .../runtime/loader/transformer_load_utils.py | 24 +- .../runtime/models/dits/common.py | 27 +- .../runtime/models/dits/qwen_image.py | 259 ++++- .../runtime/server_args/server_args.py | 18 +- .../runtime/utils/quantization_utils.py | 6 +- .../test/unit/test_convrot_int8_backends.py | 672 +++++++++++ .../test/unit/test_qwen_image_joint_qkv.py | 480 ++++++++ .../test/unit/test_server_args.py | 2 +- .../test/unit/test_text_encoder_loader.py | 21 +- .../test/unit/test_transformer_quant.py | 66 +- .../benchmark/gemm/bench_convrot_int8.py | 72 ++ .../kernels/ops/gemm/test_convrot_int8.py | 405 +++++++ .../unit/kernels/test_kernels_namespace.py | 24 + 34 files changed, 5545 insertions(+), 322 deletions(-) create mode 100644 python/sglang/kernels/jit/csrc/gemm/convrot_int8/convrot_int8_entry.cuh create mode 100644 python/sglang/kernels/jit/csrc/gemm/convrot_int8/cutlass_extensions/epilogue/epilogue_per_row_per_col_scale.h create mode 100644 python/sglang/kernels/jit/csrc/gemm/convrot_int8/cutlass_extensions/gemm/gemm_universal_base_compat.h create mode 100644 python/sglang/kernels/jit/csrc/gemm/convrot_int8/cutlass_extensions/gemm/gemm_with_epilogue_visitor.h create mode 100644 python/sglang/kernels/ops/gemm/convrot_int8.py create mode 100644 python/sglang/multimodal_gen/runtime/layers/quantization/configs/convrot_int8_config.py delete mode 100644 python/sglang/multimodal_gen/runtime/layers/quantization/configs/kitchen_int8_config.py rename python/sglang/multimodal_gen/runtime/layers/quantization/{kitchen_int8.py => convrot_int8_comfy_kitchen.py} (87%) create mode 100644 python/sglang/multimodal_gen/runtime/layers/quantization/convrot_int8_jit.py create mode 100644 python/sglang/multimodal_gen/runtime/layers/quantization/method_names.py create mode 100644 python/sglang/multimodal_gen/test/unit/test_convrot_int8_backends.py create mode 100644 python/sglang/multimodal_gen/test/unit/test_qwen_image_joint_qkv.py create mode 100644 test/registered/kernels/benchmark/gemm/bench_convrot_int8.py create mode 100644 test/registered/kernels/ops/gemm/test_convrot_int8.py diff --git a/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx b/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx index 8988c9c165..ff72d9d4d0 100644 --- a/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx +++ b/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx @@ -115,13 +115,13 @@ may be either a normal style adapter or a timestep-distilled Turbo adapter. | DiT | [Full or AdaLN-pruned](https://huggingface.co/Comfy-Org/MiniMax-H3/tree/main/diffusion_models), or [LoRA-merged/remixed](https://huggingface.co/FX-FeiHou/MiniMax-H3-Remix) BF16 safetensors | `--component-weights-paths.transformer OWNER/REPO/path/FILE.safetensors` | Weight-only override for a native full/pruned H3 layout. Match the FL2VA/Ref2VA partition; pruned, merged, or dtype-converted exports are approximate, and any author-specific sampler remains a separate requirement. | | DiT | FL/Ref hybrid fine-tunes, including [Singularity v1.3](https://huggingface.co/WarmBloodAban/Minimax-h3_Singularity) (full or pruned INT8) | Replace `--model-variant` with `hybrid`; add `--component-weights-paths.transformer OWNER/REPO/FILE.safetensors` | Explicit merged-weight deployment serving `t2va`, `fl2va`, and `ref2va` on one native pipeline. Quantization and pruning are detected from the file; INT8 requires `comfy-kitchen`. See [Singularity](#singularity-hybrid-weights). | | DiT | Comfy FP8 or self-describing MXFP8 safetensors | `--component-weights-paths.transformer OWNER/REPO/path/FILE.safetensors` | Per-layer metadata selects static/dynamic FP8 or MXFP8 automatically. | -| DiT | ConvRot INT8, W4A8, W4A4, or mixed W4A4+INT8 safetensors ([INT8](https://huggingface.co/Comfy-Org/MiniMax-H3/tree/main/diffusion_models), [W4A8](https://huggingface.co/Winnougan/MiniMax-H3-INT4_Convrot_ComfyUI), [W4A4](https://huggingface.co/Merserk/MiniMax-H3-INT4-ConvRot)) | `--component-weights-paths.transformer OWNER/REPO/path/FILE.safetensors` | Auto-detected; requires `comfy-kitchen`. TP must preserve each file's ConvRot group boundaries. | +| DiT | ConvRot INT8, W4A8, W4A4, or mixed W4A4+INT8 safetensors ([INT8](https://huggingface.co/Comfy-Org/MiniMax-H3/tree/main/diffusion_models), [W4A8](https://huggingface.co/Winnougan/MiniMax-H3-INT4_Convrot_ComfyUI), [W4A4](https://huggingface.co/Merserk/MiniMax-H3-INT4-ConvRot)) | `--component-weights-paths.transformer OWNER/REPO/path/FILE.safetensors` | Auto-detected. INT8 layers run on SGLang's JIT-compiled ConvRot ops on CC 9.0, 10.0, 12.0 and 12.1 and on `comfy-kitchen` elsewhere; W4A8 and W4A4 layers require `comfy-kitchen`. TP must preserve each file's ConvRot group boundaries. | | DiT | [NVFP4, optionally mixed with INT8 or FP8](https://huggingface.co/Abiray/Minimax-H3-nvfp4-INT4-INT8-Convrot) | `--component-weights-paths.transformer OWNER/REPO/path/FILE.safetensors` | Auto-detected; NVFP4 execution requires NVIDIA compute capability 10.0+. | | DiT | [AutoRound W4A16 component](https://huggingface.co/Ar4ikov/MiniMax-H3-transformer-W4A16-RTN) | `--component-paths.transformer Ar4ikov/MiniMax-H3-transformer-W4A16-RTN` | Self-describing Diffusers component; SGLang reuses the SRT GPTQ/Marlin backend. The linked export is FL2VA. | | DiT | GGUF, full or AdaLN-pruned ([full](https://huggingface.co/leejet/MiniMax-H3-GGUF), [pruned](https://huggingface.co/unsloth/MiniMax-H3-GGUF)) | `--component-weights-paths.transformer OWNER/REPO/FILE.gguf` | CUDA capacity path; aligned TP and layerwise offload are supported, FSDP and LoRA are not. | | Text encoder | Architecture-compatible Qwen3-VL BF16 finetune ([Heretic example](https://huggingface.co/llmfan46/Qwen3-VL-32B-Instruct-ultra-uncensored-heretic)) | `--component-paths.text_encoder OWNER/REPO` | Reuses the native H3 extractor, including its vision tower and layer-50 hidden-state selection. Finetuning changes conditioning, not the sampling schedule. | | Text encoder | [Serialized FP8 component](https://huggingface.co/Qwen/Qwen3-VL-32B-Instruct-FP8) | `--component-paths.text_encoder Qwen/Qwen3-VL-32B-Instruct-FP8` | Only eligible language-model linears use FP8; embeddings, norms, and the vision tower keep their declared precision. | -| Text encoder | ConvRot INT8, W4A8, or W4A4 safetensors ([INT8](https://huggingface.co/Comfy-Org/MiniMax-H3/tree/main/text_encoders), [Heretic INT8](https://huggingface.co/ethanfel/Qwen3-VL-32B-Ultra-Heretic-H3-ComfyUI-INT8-ConvRot), [W4A8](https://huggingface.co/Winnougan/MiniMax-H3-INT4_Convrot_ComfyUI), [W4A4](https://huggingface.co/Merserk/MiniMax-H3-INT4-ConvRot)) | `--component-weights-paths.text_encoder OWNER/REPO/path/FILE.safetensors` | Auto-detected; requires `comfy-kitchen`. INT8/W4A8 files may include a scalar-scale INT8 embedding. Unmarked tensors retain their declared precision. | +| Text encoder | ConvRot INT8, W4A8, or W4A4 safetensors ([INT8](https://huggingface.co/Comfy-Org/MiniMax-H3/tree/main/text_encoders), [Heretic INT8](https://huggingface.co/ethanfel/Qwen3-VL-32B-Ultra-Heretic-H3-ComfyUI-INT8-ConvRot), [W4A8](https://huggingface.co/Winnougan/MiniMax-H3-INT4_Convrot_ComfyUI), [W4A4](https://huggingface.co/Merserk/MiniMax-H3-INT4-ConvRot)) | `--component-weights-paths.text_encoder OWNER/REPO/path/FILE.safetensors` | Auto-detected. INT8 layers run on SGLang's JIT-compiled ConvRot ops on CC 9.0, 10.0, 12.0 and 12.1 and on `comfy-kitchen` elsewhere; W4A8 and W4A4 layers require `comfy-kitchen`. INT8/W4A8 files may include a scalar-scale INT8 embedding, which requires `comfy-kitchen`. Unmarked tensors retain their declared precision. | | Text encoder | Comfy NVFP4 with dynamic activation quantization ([Heretic example](https://huggingface.co/sakamakismile/Qwen3-VL-32B-Heretic-MiniMax-H3-NVFP4)) | `--component-weights-paths.text_encoder OWNER/REPO/path/FILE.safetensors` | Requires `comfy-kitchen` and NVIDIA compute capability 10.0+. Per-layer metadata selects native NVFP4 matmul; scalar/row-wise INT8 embeddings remain packed. | | Text encoder | [NVFP4-AWQ](https://huggingface.co/Comfy-Org/MiniMax-H3/tree/main/text_encoders) or [Quanto qint8](https://huggingface.co/DeepBeepMeep/MiniMax-H3/tree/main/Qwen3-VL-32B-Instruct) safetensors | `--component-weights-paths.text_encoder OWNER/REPO/path/FILE.safetensors` | Memory-oriented formats: compressed storage is restored, then each active matrix uses BF16/FP16 compute. | | Text encoder | [GGUF Qwen3-VL](https://huggingface.co/DeepBeepMeep/MiniMax-H3/tree/main/Qwen3-VL-32B-Instruct) | `--component-weights-paths.text_encoder OWNER/REPO/FILE.gguf` | CUDA capacity path with encoder TP/layerwise support; encoder FSDP is not supported. | @@ -249,18 +249,20 @@ transfer overhead. This exact recipe was validated on latency and memory comparison is collected in the benchmark section below. For a single 24 GB consumer card (RTX 4090), stream the DiT and text encoder -and quantize DiT linear layers online with `kitchen_int8`. Keep `vae` out of +and quantize DiT linear layers online with `convrot_int8`. Keep `vae` out of `--layerwise-offload-components`: putting the VAE decoder in layerwise offload re-streams about 9 GiB on each of 167 decode tiles. Default attention stays `fa` (exact). Approximate backends are opt-in; see [Attention Backends](/docs/sglang-diffusion/attention_backends#sage-then-sol-hybrid). -Install `comfy-kitchen` first (`pip install comfy-kitchen`). +On the RTX 4090 (CC 8.9) `convrot_int8` runs on `comfy-kitchen`, so install it +first (`pip install comfy-kitchen`); on CC 9.0, 10.0, 12.0 and 12.1 it uses +SGLang's JIT-compiled ConvRot ops instead (built once per machine on first use). ```bash 1×RTX 4090 24GB sglang generate \ --model-path MiniMaxAI/MiniMax-H3 \ --model-variant fl2va \ - --quantization kitchen_int8 \ + --quantization convrot_int8 \ --attention-backend fa \ --performance-mode memory \ --layerwise-offload-components dit,text_encoder \ @@ -1259,9 +1261,11 @@ This recipe is limited to the resident B200 and B300 topologies used for real H3 validation runs. -On a single 24 GB card, use `kitchen_int8` instead of FP8. It quantizes the +On a single 24 GB card, use `convrot_int8` instead of FP8. It quantizes the four GEMMs per DiT block online from the Hub BF16 weights (data-free, no -calibration) and dispatches them through `comfy_kitchen.int8_linear`. +calibration) and runs them on `comfy_kitchen.int8_linear` on the 4090 (CC 8.9) +or on SGLang's JIT-compiled ConvRot ops on CC 9.0, 10.0, 12.0 and 12.1 +(see [Quantization](/docs/sglang-diffusion/quantization#convrot-int8)). Quantization happens after H3's grouped `qkv` reorder, so do not load an externally pre-quantized INT8 checkpoint here. @@ -1269,7 +1273,7 @@ externally pre-quantized INT8 checkpoint here. sglang generate \ --model-path MiniMaxAI/MiniMax-H3 \ --model-variant fl2va \ - --quantization kitchen_int8 \ + --quantization convrot_int8 \ --attention-backend fa \ --performance-mode memory \ --layerwise-offload-components dit,text_encoder \ @@ -1284,13 +1288,15 @@ sglang generate \ `--attention-backend sol_attn` with `--attention-backend-config dense_backend=sage_attn,dense_steps=10` and `--component-attention-backends text_encoder=torch_sdpa,transformer=sol_attn`. -See [Quantization](/docs/sglang-diffusion/quantization#kitchen-int8) +See [Quantization](/docs/sglang-diffusion/quantization#convrot-int8) and [Attention Backends](/docs/sglang-diffusion/attention_backends#sage-then-sol-hybrid). -`kitchen_int8` changes Linear numerics. `sol_attn` / `sage_attn` also change +`convrot_int8` changes Linear numerics. `sol_attn` / `sage_attn` also change the attention algorithm. Neither is a consistency ground-truth mode. The -BF16 path is unchanged when `comfy-kitchen` is not installed. +BF16 path never depends on `comfy-kitchen`; `convrot_int8` needs it only on +GPUs the JIT ops do not cover (any CC other than 9.0, 10.0, 12.0 and 12.1) or +when no CUDA toolchain is available to build them. Pre-quantized and compact H3 text encoders are listed once in @@ -1474,7 +1480,7 @@ the configurations with collected measurements: | Ascend NPU | 8 NPUs, TP2 + SP4, Laser Attention | 4 NPUs, TP2 + SP2, Laser Attention | | MI300X / MI355X | 8× Ulysses8 resident | 1×, 2×, and 4× scaling runs | | RTX 5090 | 1× layerwise offload, measured on a physical desktop | 2× TP2 + layerwise offload | -| RTX 4090 24 GB | 1× layerwise offload + `kitchen_int8` | Spectrum and approximate attention backends are opt-in | +| RTX 4090 24 GB | 1× layerwise offload + `convrot_int8` | Spectrum and approximate attention backends are opt-in | ### GB300 single-host @@ -2113,15 +2119,15 @@ timed (seed 42); only the timed pass is reported. GPU peak stayed about | Config | Timed e2e | Denoise | vs BF16 | PSNR vs BF16 | | --- | ---: | ---: | ---: | ---: | | BF16 + FlashAttention | 405.6 s | 370.2 s | 1.00× | — | -| `kitchen_int8` + FA | 303.3 s | 273.7 s | 1.34× | 24.81 dB | -| `kitchen_int8` + `sol_attn` | 223.9 s | 203.9 s | 1.81× | 24.44 dB | -| `kitchen_int8` + `sage_attn` | 174.9 s | 154.2 s | 2.32× | 23.51 dB | -| `kitchen_int8` + Sage→Sol hybrid | 163.8 s | 143.1 s | 2.48× | 23.04 dB | +| `convrot_int8` + FA | 303.3 s | 273.7 s | 1.34× | 24.81 dB | +| `convrot_int8` + `sol_attn` | 223.9 s | 203.9 s | 1.81× | 24.44 dB | +| `convrot_int8` + `sage_attn` | 174.9 s | 154.2 s | 2.32× | 23.51 dB | +| `convrot_int8` + Sage→Sol hybrid | 163.8 s | 143.1 s | 2.48× | 23.04 dB | -`kitchen_int8` + FA changes Linear numerics only. The `sol_attn` / +`convrot_int8` + FA changes Linear numerics only. The `sol_attn` / `sage_attn` / hybrid rows also change the attention algorithm, so speed and pixel fidelity rank in opposite orders there. Default remains -`kitchen_int8` + `fa`. Cache-DiT can share this layerwise recipe; omit +`convrot_int8` + `fa`. Cache-DiT can share this layerwise recipe; omit `quality` and see the quality-level section. Same card, 50-step run. Speedup is vs BF16+FA. PSNR is vs same-tree INT8+FA (Spectrum off). diff --git a/docs/docs/sglang-diffusion/environment_variables.mdx b/docs/docs/sglang-diffusion/environment_variables.mdx index 163dede820..67699723de 100644 --- a/docs/docs/sglang-diffusion/environment_variables.mdx +++ b/docs/docs/sglang-diffusion/environment_variables.mdx @@ -98,15 +98,20 @@ description: "Configure SGLang diffusion behavior with environment variables." true Use Run:AI model streamer for model loading + + SGLANG_DIFFUSION_CONVROT_INT8_BACKEND + auto + Kernel backend for convrot_int8: auto takes SGLang's JIT-compiled ConvRot ops where they build and run (CC 9.0, 10.0, 12.0, 12.1 with nvcc) and comfy_kitchen otherwise; jit or comfy_kitchen forces one. + SGLANG_KITCHEN_INT8_MAX_ROWS 8192 - Max activation rows per kitchen_int8 fused GEMM. Set 0 to disable row splitting. + Max activation rows per convrot_int8 fused GEMM on the comfy_kitchen backend. Set 0 to disable row splitting. SGLANG_KITCHEN_INT8_MIN_SPLIT_N 8192 - Minimum output features before kitchen_int8 splits large-M GEMMs. Narrow outputs stay as a single call. + Minimum output features before convrot_int8 (comfy_kitchen backend) splits large-M GEMMs. Narrow outputs stay as a single call. diff --git a/docs/docs/sglang-diffusion/quantization.mdx b/docs/docs/sglang-diffusion/quantization.mdx index 90873d9c8a..227b6d0797 100644 --- a/docs/docs/sglang-diffusion/quantization.mdx +++ b/docs/docs/sglang-diffusion/quantization.mdx @@ -122,12 +122,12 @@ backend. MXFP4 requires ROCm and MI350+ (gfx95x). Weights quantized at load time, activations quantized to fp8 / mxfp4 dynamically. - kitchen_int8 (online quantization) + convrot_int8 (online quantization; kitchen_int8 is a deprecated alias) Unquantized BF16/FP16 checkpoint - --component-quantizations.<component> kitchen_int8 - MiniMax-H3 (validated on 1× RTX 4090 24 GB) - comfy-kitchen - Data-free INT8 ConvRot at load time via comfy_kitchen.int8_linear. Layers whose input dim is not divisible by the group size stay in BF16. Requires Turing+ (SM75). + --component-quantizations.<component> convrot_int8 + MiniMax-H3, Qwen-Image, Qwen-Image-Edit (quality-validated); other BF16 DiTs load but are not quality-validated + SGLang JIT ops (bundled, need nvcc) or comfy-kitchen + Data-free INT8 ConvRot W8A8 at load time, per layer on the fused JIT-compiled ops (CC 9.0, 10.0, 12.0, 12.1) or on comfy_kitchen.int8_linear (Turing+); SGLANG_DIFFUSION_CONVROT_INT8_BACKEND forces one. Layers whose input dim is not divisible by the group size stay in BF16. TP, LoRA, FSDP and memory notes under ConvRot INT8 below. fp8 (offline quantization) @@ -182,8 +182,8 @@ backend. One selected safetensors file with per-layer int8_tensorwise and ConvRot metadata --component-weights-paths.<component> Native DiTs and encoders whose parameter mappings preserve each marked linear; MiniMax-H3 DiT and Qwen3-VL encoder checkpoints have validated tensor contracts - comfy-kitchen - CUDA; auto-detected; uses the fused Kitchen INT8 kernel and validates weight/scale layout before model construction. TP requires every row-parallel input shard to preserve the checkpoint's ConvRot group boundary. The H3 256-group DiT supports TP1/2/4, not TP8; its Qwen3-VL encoder keeps TP8 by replicating only incompatible row projections. Offload is supported; FSDP is not. + SGLang JIT ops (bundled, need nvcc) or comfy-kitchen + CUDA; auto-detected; each layer runs on the JIT-compiled ConvRot ops where they apply (CC 9.0/10.0/12.0/12.1, group size 64 or 256) and on comfy_kitchen.int8_linear otherwise; weight/scale layout is validated before model construction. TP requires every row-parallel input shard to preserve the checkpoint's ConvRot group boundary. The H3 256-group DiT supports TP1/2/4, not TP8; its Qwen3-VL encoder keeps TP8 by replicating only incompatible row projections. Offload is supported; FSDP is not. mxfp8 @@ -389,31 +389,61 @@ sglang generate \ ``` **Note:** Requires `aiter` package with MXFP4 kernel support -### Kitchen INT8 +### ConvRot INT8 -Serialized Comfy ConvRot INT8 DiTs and compatible native encoders use -`--component-weights-paths.`. Both are auto-detected from per-layer -markers and load INT8 weights and row scales directly; omit online -quantization. `--transformer-weights-path` remains the primary-DiT convenience -spelling. +`convrot_int8` (formerly `kitchen_int8`, still accepted as a deprecated alias) +is online W8A8 INT8 for DiTs and compatible native encoders: +every linear whose input dim is divisible by the Hadamard group size (256) is +rotated and quantized per output channel after loading a stock BF16 checkpoint +(ConvRot, arXiv:2512.03673; no calibration data). Each forward call runs one +fused op per linear: group-wise Hadamard rotation, dynamic per-row INT8 +activation quantization, INT8 GEMM, dequantization and bias add. -For a BF16 checkpoint, `--component-quantizations.transformer kitchen_int8` -instead performs online quantization after loading: +Two kernel backends implement the same quantization and consume the same INT8 +weights and per-row scales, so the choice is made per layer at load time: -`kitchen_int8` quantizes DiT linear weights online from the stock BF16 -checkpoint. Forward uses the fused `comfy_kitchen.int8_linear` op (rotation, -dynamic per-row activation quantization, INT8 GEMM, dequant, and bias). -Install the optional dependency first: +- `jit`: the `convrot_int8` ops in `sglang.kernels.ops.gemm.convrot_int8`, + compiled by SGLang's JIT for the local GPU on first use (about a minute with + `nvcc` on the machine; cached under `SGLANG_JIT_CACHE_DIR`, default + `~/.cache/sglang/jit`, for later runs). CC 9.0 (Hopper), 10.0 (B200, GB200) + and 12.0 / 12.1 (RTX PRO 6000 Blackwell, RTX 50 series, DGX Spark); group + size 64, 128, 256 or 512; output width a multiple of 8. CC 10.3 (B300, + GB300) is refused on purpose: Blackwell Ultra cuts INT8 tensor-core + throughput to a fraction of its BF16 rate, so use FP8 or NVFP4 there. Linears sharing one input (a block's Q/K/V projections) + rotate and quantize it once, and the Qwen-Image FFN down-projection folds its + GELU(tanh) into the rotate kernel; both are bitwise identical to the + per-linear ops. +- `comfy_kitchen`: `comfy_kitchen.int8_linear` from the optional + `comfy-kitchen` package (`pip install comfy-kitchen`). Turing and later + (SM75); group size 16, 64 or 256. + +The default `auto` takes `jit` wherever its ops build and run (no CUDA +toolchain, or a failed build, counts as unavailable and is logged) and +`comfy_kitchen` otherwise; loading stops with an install hint if +`comfy-kitchen` is then missing. A layer whose output width is not a multiple +of 8 goes to `comfy_kitchen` when it is installed and stays BF16 otherwise. +`SGLANG_DIFFUSION_CONVROT_INT8_BACKEND=jit` or `=comfy_kitchen` forces one +backend. LoRA adapters run in dynamic mode (they cannot be merged into +rotated INT8 weights); under FSDP `auto` resolves to `comfy_kitchen` and an +explicit `jit` is refused. Serialized Comfy ConvRot INT8 DiTs and native +encoders use `--component-weights-paths.`; they are auto-detected +from per-layer markers, load INT8 weights and row scales directly (omit online +quantization), and follow the same per-layer backend choice. +`--transformer-weights-path` remains the primary-DiT convenience spelling. ```bash -pip install comfy-kitchen +sglang generate \ + --model-path Qwen/Qwen-Image \ + --quantization convrot_int8 \ + --prompt "a beautiful sunset" \ + --save-output ``` ```bash sglang generate \ --model-path MiniMaxAI/MiniMax-H3 \ --model-variant fl2va \ - --component-quantizations.transformer kitchen_int8 \ + --component-quantizations.transformer convrot_int8 \ --attention-backend fa \ --performance-mode memory \ --layerwise-offload-components dit,text_encoder \ @@ -426,21 +456,52 @@ sglang generate \ Quantization runs after the model's weight loaders, so MiniMax-H3's grouped `qkv` reorder is already applied. Layers whose input dim is not divisible by -the group size (256) stay in BF16 instead of failing the load; H3's AdaLN -projections take that path. +the group size stay in BF16 instead of failing the load (H3's AdaLN +projections take that path); under TP every row-parallel input shard must also +divide it, or the load stops naming the layer, which +`--quantization-ignored-layers` keeps in BF16. Without offload, loading stages +the BF16 weights on the GPU before quantizing them, so it needs the BF16 +checkpoint's worth of free VRAM (38 GiB for Qwen-Image); under layerwise +offload the weights stay on the host and are quantized one layer at a time. +`--quantization-ignored-layers img_mod txt_mod txt_mlp.net.2` keeps the +few-row Qwen-Image GEMMs (AdaLN modulation, +text-stream FFN down-projection) in BF16 at the cost of memory (45.5 GB peak) +for no consistent speed gain, so the default quantizes everything. + +Measured against BF16 on GB200 in eager execution with every eligible layer +quantized: + +| Model | Backend | Speedup vs BF16 | Peak GPU memory | +| --- | --- | ---: | --- | +| Qwen-Image 1024x1024, 50 steps, CFG 4, 1 GPU | `jit` | 1.19x (1.13x on 2 GPUs, CFG parallel) | 56 -> 37 GB | +| Qwen-Image-Edit 1024x1024, 1 GPU | `jit` | 1.32x | | +| MiniMax-H3 1344x768, 5 s, 50 steps, 1 GPU | `jit` | 1.07x (1.15x on 4 GPUs, Ulysses 4) | 125 -> 107 GiB | +| MiniMax-H3, same | `comfy_kitchen` | 0.86x | 125 -> 107 GiB | + +On MiniMax-H3 both backends quantize the same 209 linears and sit in the same +quality class against BF16 over six prompt/seed pairs (video PSNR 23.4 vs 23.8 +dB, SSIM 0.80 vs 0.81, audio spectral similarity 0.96 vs 0.97; the same-seed +distance between the two backends equals each one's distance to BF16). Loading +an online-quantized H3 without offload peaks at 115 GiB resident; the +serialized Comfy INT8 checkpoint loads at 96 GiB resident and runs on `jit` at +the same speed. -`kitchen_int8` is approximate and is not a consistency ground-truth mode. -The BF16 path is unchanged when `comfy-kitchen` is not installed. See the +`convrot_int8` is approximate: same-seed outputs are not bit-exact with BF16 or +between the two backends (both stay within INT8 quantization noise of BF16), so +it is not a consistency ground-truth mode. Quality was validated against BF16 on +MiniMax-H3, Qwen-Image and Qwen-Image-Edit; other DiTs load but are not +quality-validated. Without `convrot_int8` neither backend is loaded and the BF16 +path is untouched. See the [MiniMax-H3 cookbook](/cookbook/diffusion/MiniMax/MiniMax-H3#9-feature-contracts-and-advanced-recipes) for the 24 GB offload recipe, including why `vae` must stay out of `--layerwise-offload-components`. -Large-M GEMMs (`rows > 8192` and `out_features >= 8192`) are split by rows so -the fused kernel stays on the data-parallel CUTLASS config. Override the -thresholds with `SGLANG_KITCHEN_INT8_MAX_ROWS` and -`SGLANG_KITCHEN_INT8_MIN_SPLIT_N`. +With the `comfy_kitchen` backend, large-M GEMMs (`rows > 8192` and +`out_features >= 8192`) are split by rows so the fused kernel stays on the +data-parallel CUTLASS config. Override the thresholds with +`SGLANG_KITCHEN_INT8_MAX_ROWS` and `SGLANG_KITCHEN_INT8_MIN_SPLIT_N`. ### Skipping Layers diff --git a/python/sglang/kernels/jit/csrc/gemm/convrot_int8/convrot_int8_entry.cuh b/python/sglang/kernels/jit/csrc/gemm/convrot_int8/convrot_int8_entry.cuh new file mode 100644 index 0000000000..3c11ba2d7b --- /dev/null +++ b/python/sglang/kernels/jit/csrc/gemm/convrot_int8/convrot_int8_entry.cuh @@ -0,0 +1,1021 @@ +/* Copyright 2026 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// ConvRot (arXiv:2512.03673) INT8 W8A8 linear. Activations pass through a +// group-wise Hadamard rotation (Kronecker power of the regular 4x4 matrix, see +// the rotate kernel) and per-row dynamic INT8 quantization; +// weights are rotated and quantized offline with the same row-wise transform +// (convrot_rotate_quantize_activation on the [N, K] weight). The GEMM is a +// CUTLASS dense INT8 kernel whose epilogue applies the per-row (activation) x +// per-column (weight) dequant into BF16: CUTLASS 3.x WGMMA on Sm90, tcgen05 on +// Sm100, and the CUTLASS 2.x mma.sync path on the CC 12.x parts that have neither. +// The JIT build targets the local GPU, so SGL_CUDA_ARCH picks the GEMM path at +// compile time; the Python wrapper refuses every other capability before +// compiling. + +#pragma once + +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cutlass_extensions/epilogue/epilogue_per_row_per_col_scale.h" +#include "cutlass_extensions/gemm/gemm_universal_base_compat.h" +#include "cutlass_extensions/gemm/gemm_with_epilogue_visitor.h" +#include +#include +#include +#include + +#if SGL_CUDA_ARCH != 900 && SGL_CUDA_ARCH != 1000 && SGL_CUDA_ARCH != 1200 && SGL_CUDA_ARCH != 1210 +#error "convrot_int8: no GEMM path for this SGL_CUDA_ARCH (supported: 900, 1000, 1200, 1210)" +#endif + +namespace sglang { + +namespace convrot_int8 { + +using namespace cute; +using namespace host; +using tvm::ffi::TensorView; + +// ---------------------------- Rotate + quantize ---------------------------- +// Register-resident: a warp holds a 1024-element tile of one row as 32 fp32 per +// lane, so there is no K-proportional shared memory and no smem bound on K. Lanes +// are split into GroupSize/32-lane clusters, one group per cluster (q = lane within +// the cluster, r = register 0..31). In-group index bits held in r are {0,1,2,4,6} +// (G >= 128) or {0..4} (G = 64), the rest come from q: bits 0-2 in r make each +// lane's data four runs of 8 contiguous elements (16-B loads, 8-B int8 stores), and +// every radix-4 digit above the first has one bit in r and one in q, so a stage +// costs one shuffle per element and the transform needs no block barrier. A row is +// ceil(K / 1024) warps; up to 32 of them (K <= 32768) keep the rotated row in +// registers across the single abs-max barrier. Wider rows take the two-pass variant +// (rotate for the abs-max, re-read the row from L2 and rotate again to quantize). +constexpr int kRotTileElems = 1024; +constexpr int kRotElemsPerLane = 32; +constexpr int kRotMaxWarps = 32; +constexpr int kRotSmallTiles = 16; // n_tiles <= 16 (K <= 16384) runs under a 512-thread register cap +constexpr int kRotTargetWarpsPerBlock = 8; +constexpr unsigned kRotFullMask = 0xffffffffu; + +// GELU (tanh approximation) matching ATen's CUDA kernel bit-for-bit: same fp32 +// constants and association order, libdevice tanhf. A --use_fast_math build +// rewrites ::tanhf into the MUFU approximation (measured: 22% of BF16 outputs +// then shift one INT8 step); the libdevice symbol is exempt from that +// substitution, so the result does not depend on the build flags. +extern "C" __device__ float __nv_tanhf(float); + +struct ATenGeluTanh { + SGL_DEVICE float operator()(float x) const { + const float x_cube = x * x * x; + const float inner = 0.7978845608028654f * (x + 0.044715f * x_cube); + return 0.5f * x * (1.0f + __nv_tanhf(inner)); + } +}; + +constexpr bool is_supported_group_size(int64_t group_size) { + return group_size == 64 || group_size == 128 || group_size == 256 || group_size == 512; +} + +template +struct RotLayout { + static_assert(is_supported_group_size(GroupSize), ""); + static constexpr int kLanesPerGroup = GroupSize / 32; + static constexpr int kGroupsPerTile = 32 / kLanesPerGroup; + // In-group offset of register run j (registers 8j .. 8j+7) for lane-in-cluster q. + SGL_DEVICE static int run_offset(int q, int j) { + if constexpr (GroupSize == 64) { + return (q << 5) | (j << 3); + } else { + return ((q & 1) << 3) | ((j & 1) << 4) | (((q >> 1) & 1) << 5) | ((j >> 1) << 6) | (((q >> 2) & 1) << 7) | + (((q >> 3) & 1) << 8); + } + } +}; + +// Regular (non-Sylvester) Hadamard, the Kronecker power of +// H4 = [[1,1,1,-1], [1,1,-1,1], [1,-1,1,1], [-1,1,1,1]]: +// one 4-point stage per base-4 digit of the in-group index, y_d = S - 2 x_{d^3} +// with S the sum of the four elements that differ only in that digit. Every row +// of the product sums to +1, so a group's mean stays spread over the +// coefficients. The Sylvester transform (row 0 all ones) concentrates it into +// one coefficient at sqrt(GroupSize) x; that coefficient becomes the row absmax +// and coarsens the INT8 step for the whole row, measured 1.3-1.9x more error on +// GELU outputs and other non-zero-mean inputs. Same matrix as comfy-kitchen's +// ConvRot path. Group sizes with an odd bit count finish with one 2-point stage +// on the top bit. +// S is always formed as (own + low-bit partner) + (high-bit partner + both), which +// is the same fp32 expression for the four members of a quartet (a + b == b + a +// bitwise), so the result does not depend on which lane or register holds an +// element. All v[] indices are compile-time constants after unrolling; a +// lane-dependent index would put the array in local memory. + +// Both digit bits in the register index (B0 = low bit, B1 = high bit). +template +SGL_DEVICE void rot_stage_regreg(float (&v)[kRotElemsPerLane]) { + CUTLASS_PRAGMA_UNROLL + for (int r = 0; r < kRotElemsPerLane; r++) { + if ((r & (B0 | B1)) == 0) { + const float a0 = v[r], a1 = v[r | B0], a2 = v[r | B1], a3 = v[r | B0 | B1]; + const float S = (a0 + a1) + (a2 + a3); + v[r] = S - 2.f * a3; + v[r | B0] = S - 2.f * a2; + v[r | B1] = S - 2.f * a1; + v[r | B0 | B1] = S - 2.f * a0; + } + } +} + +// Low digit bit in the register index (RB), high digit bit in the lane (xor X). +template +SGL_DEVICE void rot_stage_split(float (&v)[kRotElemsPerLane]) { + CUTLASS_PRAGMA_UNROLL + for (int r = 0; r < kRotElemsPerLane; r++) { + if ((r & RB) == 0) { + const float a = v[r], b = v[r | RB]; + const float ap = __shfl_xor_sync(kRotFullMask, a, X); + const float bp = __shfl_xor_sync(kRotFullMask, b, X); + const float S = (a + b) + (ap + bp); + v[r] = S - 2.f * bp; + v[r | RB] = S - 2.f * ap; + } + } +} + +// 2-point stage on a register bit (RB) ... +template +SGL_DEVICE void rot_stage_2pt_reg(float (&v)[kRotElemsPerLane]) { + CUTLASS_PRAGMA_UNROLL + for (int r = 0; r < kRotElemsPerLane; r++) { + if ((r & RB) == 0) { + const float a = v[r], b = v[r | RB]; + v[r] = a + b; + v[r | RB] = a - b; + } + } +} + +// ... or on a lane bit (xor X); `hi` = this lane holds the bit-set element. +template +SGL_DEVICE void rot_stage_2pt_lane(float (&v)[kRotElemsPerLane], bool hi) { + CUTLASS_PRAGMA_UNROLL + for (int r = 0; r < kRotElemsPerLane; r++) { + const float mine = v[r]; + const float p = __shfl_xor_sync(kRotFullMask, mine, X); + v[r] = hi ? (p - mine) : (mine + p); + } +} + +template +SGL_DEVICE void rot_rotate_tile(float (&v)[kRotElemsPerLane], int q) { + if constexpr (GroupSize == 64) { + rot_stage_regreg<1, 2>(v); // digit 0: in-group bits 0,1 + rot_stage_regreg<4, 8>(v); // digit 1: bits 2,3 + rot_stage_split<16, 1>(v); // digit 2: bit 4 (reg) / bit 5 (lane) + } else { + rot_stage_regreg<1, 2>(v); // digit 0: bits 0,1 + rot_stage_split<4, 1>(v); // digit 1: bit 2 (reg) / bit 3 (lane) + rot_stage_split<8, 2>(v); // digit 2: bit 4 (reg) / bit 5 (lane) + if constexpr (GroupSize == 128) { + rot_stage_2pt_reg<16>(v); // bit 6 (reg) + } else { + rot_stage_split<16, 4>(v); // digit 3: bit 6 (reg) / bit 7 (lane) + if constexpr (GroupSize == 512) { + rot_stage_2pt_lane<8>(v, (q & 8) != 0); // bit 8 (lane) + } + } + } +} + +// GeluInput fuses the F.gelu(approximate="tanh") that otherwise precedes an +// FFN down-projection. The GELU result is rounded to BF16 and back before the +// butterfly, so the rotated values are bitwise those of the eager +// store-then-reload path. +template +SGL_DEVICE void rot_load_tile(const __nv_bfloat16* __restrict__ grp, int q, bool active, float (&v)[kRotElemsPerLane]) { + CUTLASS_PRAGMA_UNROLL + for (int j = 0; j < 4; j++) { + uint4 pk = make_uint4(0u, 0u, 0u, 0u); + if (active) pk = *reinterpret_cast(grp + RotLayout::run_offset(q, j)); + const uint32_t w[4] = {pk.x, pk.y, pk.z, pk.w}; + CUTLASS_PRAGMA_UNROLL + for (int e = 0; e < 4; e++) { + v[8 * j + 2 * e] = __uint_as_float(w[e] << 16); + v[8 * j + 2 * e + 1] = __uint_as_float(w[e] & 0xffff0000u); + } + } + if constexpr (GeluInput) { + CUTLASS_PRAGMA_UNROLL + for (int r = 0; r < kRotElemsPerLane; r++) { + v[r] = __bfloat162float(__float2bfloat16(ATenGeluTanh{}(v[r]))); + } + } +} + +template +SGL_DEVICE void rot_quant_store_tile( + int8_t* __restrict__ grp_out, int q, bool active, const float (&v)[kRotElemsPerLane], float inv_scale) { + CUTLASS_PRAGMA_UNROLL + for (int j = 0; j < 4; j++) { + uint32_t w[2]; + CUTLASS_PRAGMA_UNROLL + for (int h = 0; h < 2; h++) { + int32_t qv[4]; + CUTLASS_PRAGMA_UNROLL + for (int e = 0; e < 4; e++) { + float t = v[8 * j + 4 * h + e] * inv_scale; + t = fmaxf(-127.f, fminf(127.f, t)); + qv[e] = __float2int_rn(t); // == lrintf on the clamped range + } + const uint32_t lo = __byte_perm((uint32_t)qv[0], (uint32_t)qv[1], 0x0040); + const uint32_t hi = __byte_perm((uint32_t)qv[2], (uint32_t)qv[3], 0x0040); + w[h] = __byte_perm(lo, hi, 0x5410); + } + if (active) *reinterpret_cast(grp_out + RotLayout::run_offset(q, j)) = make_uint2(w[0], w[1]); + } +} + +SGL_DEVICE float rot_warp_max(float a) { + CUTLASS_PRAGMA_UNROLL + for (int off = 16; off > 0; off >>= 1) { + a = fmaxf(a, __shfl_xor_sync(kRotFullMask, a, off)); + } + return a; +} + +// Block = dim3(32 * warps_per_row, rows_per_block): threadIdx.y is the row within +// the block, threadIdx.x >> 5 the tile within the row. Resident: warps_per_row == +// n_tiles and the rotated row stays in registers across the barrier. Two-pass +// (!Resident, K > 32768): 32 warps stride over the tiles. +template +__global__ void __launch_bounds__(MaxThreads) convrot_rotate_quantize_activation_kernel( + const __nv_bfloat16* __restrict__ x, int8_t* __restrict__ x_q, float* __restrict__ row_scale, int M, int K) { + using Layout = RotLayout; + __shared__ float warp_amax[kRotMaxWarps]; + + const int lane = threadIdx.x & 31; + const int tile0 = threadIdx.x >> 5; + const int warps_per_row = blockDim.x >> 5; + const int row_in_block = threadIdx.y; + const int block_warp = row_in_block * warps_per_row + tile0; + const int row = blockIdx.x * blockDim.y + row_in_block; + const bool row_valid = row < M; // padding rows compute on zeros and still reach the barrier + const int c = lane / Layout::kLanesPerGroup; // group within the tile + const int q = lane % Layout::kLanesPerGroup; // lane within the group + const int num_groups = K / GroupSize; + const int n_tiles = (K + kRotTileElems - 1) / kRotTileElems; + const __nv_bfloat16* row_in = x + (int64_t)row * K; + int8_t* row_out = x_q + (int64_t)row * K; + // H / sqrt(GroupSize) on both operands keeps (Ux) . (Uw) == x . w exactly. + const float inv_sqrt_group = rsqrtf((float)GroupSize); + + float v[kRotElemsPerLane]; + float local_amax = 0.f; + const int g0 = tile0 * Layout::kGroupsPerTile + c; + const bool active0 = row_valid && (g0 < num_groups); + if constexpr (Resident) { + rot_load_tile(row_in + (int64_t)g0 * GroupSize, q, active0, v); + rot_rotate_tile(v, q); + CUTLASS_PRAGMA_UNROLL + for (int r = 0; r < kRotElemsPerLane; r++) { + v[r] *= inv_sqrt_group; + local_amax = fmaxf(local_amax, fabsf(v[r])); + } + } else { + CUTLASS_PRAGMA_NO_UNROLL + for (int t = tile0; t < n_tiles; t += warps_per_row) { + const int g = t * Layout::kGroupsPerTile + c; + const bool active = row_valid && (g < num_groups); + rot_load_tile(row_in + (int64_t)g * GroupSize, q, active, v); + rot_rotate_tile(v, q); + CUTLASS_PRAGMA_UNROLL + for (int r = 0; r < kRotElemsPerLane; r++) { + local_amax = fmaxf(local_amax, fabsf(v[r] * inv_sqrt_group)); + } + } + } + + local_amax = rot_warp_max(local_amax); + if (lane == 0) warp_amax[block_warp] = local_amax; + __syncthreads(); + + // Every thread folds its row's warp partials (max is order-independent), so no + // second barrier is needed; the scale expressions are the same as before. + const float amax = rot_warp_max(lane < warps_per_row ? warp_amax[row_in_block * warps_per_row + lane] : 0.f); + const float scale = (amax > 0.f) ? (amax / 127.f) : 1.f; + const float inv_scale = 1.f / scale; + if (row_valid && tile0 == 0 && lane == 0) row_scale[row] = scale; + + if constexpr (Resident) { + rot_quant_store_tile(row_out + (int64_t)g0 * GroupSize, q, active0, v, inv_scale); + } else { + CUTLASS_PRAGMA_NO_UNROLL + for (int t = tile0; t < n_tiles; t += warps_per_row) { + const int g = t * Layout::kGroupsPerTile + c; + const bool active = row_valid && (g < num_groups); + rot_load_tile(row_in + (int64_t)g * GroupSize, q, active, v); + rot_rotate_tile(v, q); + CUTLASS_PRAGMA_UNROLL + for (int r = 0; r < kRotElemsPerLane; r++) + v[r] *= inv_sqrt_group; + rot_quant_store_tile(row_out + (int64_t)g * GroupSize, q, active, v, inv_scale); + } + } +} + +struct RotateOperands { + const bf16_t* x; // [M, K] + int8_t* x_q; // [M, K] + float* x_scale; // [M] + int32_t M; + int32_t K; + cudaStream_t stream; +}; + +template +void launch_rotate_quantize_kernel(const RotateOperands& op) { + const int n_tiles = (op.K + kRotTileElems - 1) / kRotTileElems; + if (n_tiles <= kRotMaxWarps) { + const int rows_per_block = std::max(1, std::min(op.M, kRotTargetWarpsPerBlock / n_tiles)); + const dim3 block(32 * n_tiles, rows_per_block); + const dim3 grid((op.M + rows_per_block - 1) / rows_per_block); + if (n_tiles <= kRotSmallTiles) { + LaunchKernel(grid, block, op.stream)( + convrot_rotate_quantize_activation_kernel, + op.x, + op.x_q, + op.x_scale, + op.M, + op.K); + } else { + LaunchKernel(grid, block, op.stream)( + convrot_rotate_quantize_activation_kernel, + op.x, + op.x_q, + op.x_scale, + op.M, + op.K); + } + } else { + LaunchKernel(dim3(op.M), dim3(32 * kRotMaxWarps, 1), op.stream)( + convrot_rotate_quantize_activation_kernel, + op.x, + op.x_q, + op.x_scale, + op.M, + op.K); + } +} + +template +void launch_rotate_quantize(const RotateOperands& op, bool gelu_input) { + if (gelu_input) { + launch_rotate_quantize_kernel(op); + } else { + launch_rotate_quantize_kernel(op); + } +} + +void check_group_size(int64_t K, int64_t group_size) { + CHECK_HOST(is_supported_group_size(group_size)) + << "convrot_int8: unsupported group_size " << group_size << " (supported: 64, 128, 256, 512)"; + CHECK_HOST(K % group_size == 0) << "convrot_int8: K (" << K << ") must be a multiple of group_size (" << group_size + << ")"; +} + +void rotate_quantize_rowwise(const RotateOperands& op, int64_t group_size, bool gelu_input) { + switch (group_size) { + case 64: + launch_rotate_quantize<64>(op, gelu_input); + break; + case 128: + launch_rotate_quantize<128>(op, gelu_input); + break; + case 256: + launch_rotate_quantize<256>(op, gelu_input); + break; + case 512: + launch_rotate_quantize<512>(op, gelu_input); + break; + default: + Panic("convrot_int8: unreachable group_size ", group_size); + } +} + +// ----------------------------------- GEMM ----------------------------------- +using ElementA = int8_t; +using ElementB = int8_t; +using ElementAccumulator = int32_t; +using ElementCompute = float; +using ElementOutput = cutlass::bfloat16_t; + +// x_q is [M, K] row-major; weight_q is [N, K] row-major, i.e. B as K x N column-major. +using LayoutA = cutlass::layout::RowMajor; +using LayoutB = cutlass::layout::ColumnMajor; +using LayoutC = cutlass::layout::RowMajor; +using LayoutD = cutlass::layout::RowMajor; + +constexpr int AlignmentA = 128 / cutlass::sizeof_bits::value; +constexpr int AlignmentB = 128 / cutlass::sizeof_bits::value; +constexpr int AlignmentC = 128 / cutlass::sizeof_bits::value; +constexpr int AlignmentD = 128 / cutlass::sizeof_bits::value; + +struct GemmOperands { + ElementOutput* out; // [M, N] + ElementA* x_q; // [M, K] + ElementB* w_q; // [N, K] + ElementCompute* x_scale; // [M] + ElementCompute* w_scale; // [N] + ElementOutput* bias; // [N], nullptr without bias + int32_t M; + int32_t N; + int32_t K; + DLDevice device; + cudaStream_t stream; +}; + +void check_cutlass(cutlass::Status status, const char* what) { + CHECK_HOST(status == cutlass::Status::kSuccess) + << "convrot_int8: " << what << " failed: " << cutlassGetStatusString(status); +} + +#if SGL_CUDA_ARCH >= 1200 + +// CC 12.0 / 12.1 (RTX PRO 6000 Blackwell, RTX 50 series, DGX Spark) have neither +// WGMMA nor tcgen05, so the dequant GEMM runs on the CUTLASS 2.x mma.sync INT8 +// path sgl-kernel already uses for its sm89 int8_scaled_mm (int8_gemm_kernel.cu), +// whose per-row x per-column epilogue visitor computes the same +// bf16(x_scale[m] * w_scale[n] * acc + bias[n]). +template +void run_convrot_int8_gemm_mma_sync(const GemmOperands& op) { + using ArchTag = cutlass::arch::Sm80; + using InstructionShape = cutlass::gemm::GemmShape<16, 8, 32>; + using OperatorClass = cutlass::arch::OpClassTensorOp; + using ThreadblockSwizzle = cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<8>; + using DefaultGemmConf = cutlass::gemm::device:: + DefaultGemmConfiguration; + using EpilogueOutputOp = typename DefaultGemmConf::EpilogueOutputOp; + + using GemmKernel_ = typename cutlass::gemm::kernel::DefaultGemm< + ElementA, + LayoutA, + DefaultGemmConf::kAlignmentA, + ElementB, + LayoutB, + DefaultGemmConf::kAlignmentB, + ElementOutput, + LayoutC, + ElementAccumulator, + OperatorClass, + ArchTag, + ThreadblockShape, + WarpShape, + InstructionShape, + EpilogueOutputOp, + ThreadblockSwizzle, + NumStages, + true, + typename DefaultGemmConf::Operator>::GemmKernel; + + using AlphaColTileIterator = cutlass::epilogue::threadblock::PredicatedTileIterator< + cutlass::epilogue::threadblock::OutputTileOptimalThreadMap< + typename GemmKernel_::Epilogue::OutputTileIterator::ThreadMap::Shape, + typename GemmKernel_::Epilogue::OutputTileIterator::ThreadMap::Count, + GemmKernel_::Epilogue::OutputTileIterator::ThreadMap::kThreads, + GemmKernel_::Epilogue::OutputTileIterator::kElementsPerAccess, + cutlass::sizeof_bits::value>, + ElementCompute>; + using EpilogueVisitor = typename cutlass::epilogue::threadblock::EpilogueVisitorPerRowPerCol< + ThreadblockShape, + GemmKernel_::kThreadCount, + AlphaColTileIterator, + typename GemmKernel_::Epilogue::OutputTileIterator, + ElementAccumulator, + ElementCompute, + EpilogueOutputOp>; + using Epilogue = typename cutlass::epilogue::threadblock:: + EpilogueWithVisitorFromExistingEpilogue::Epilogue; + using GemmKernel = + cutlass::gemm::kernel::GemmWithEpilogueVisitor; + using Gemm = cutlass::gemm::device::GemmUniversalBaseCompat; + + const int M = op.M, N = op.N, K = op.K; + // w_q is [N, K] row-major, i.e. B as K x N column-major with ldb = K; the bias + // TensorRef with stride 0 broadcasts one row. + typename EpilogueOutputOp::Params linear_scaling_params; + typename EpilogueVisitor::Arguments visitor_args{linear_scaling_params}; + typename Gemm::Arguments args{ + {M, N, K}, + {op.x_q, static_cast(K)}, + {op.w_q, static_cast(K)}, + {op.w_scale, 0}, + {op.x_scale, 0}, + {op.bias, 0}, + {op.out, static_cast(N)}, + visitor_args}; + + Gemm gemm_op; + check_cutlass(gemm_op.can_implement(args), "can_implement"); + const size_t workspace_size = gemm_op.get_workspace_size(args); + auto workspace = host::ffi::alloc_workspace_tensor(workspace_size, op.device); + void* workspace_ptr = workspace_size == 0 ? nullptr : workspace.data_ptr(); + check_cutlass(gemm_op(args, workspace_ptr, op.stream), "run"); +} + +// Tile table of sm89_dispatch_shape (int8_gemm_kernel.cu): the 100 KB shared-memory +// class the CC 12.x parts share with Ada. +void dispatch_convrot_int8_gemm(const GemmOperands& op) { + using cutlass::gemm::GemmShape; + const int M = op.M, N = op.N; + if (M <= 16) { + if (N <= 8192) { + run_convrot_int8_gemm_mma_sync, GemmShape<16, 64, 64>, 5>(op); + } else { + run_convrot_int8_gemm_mma_sync, GemmShape<16, 64, 64>, 4>(op); + } + } else if (M <= 32) { + if (N <= 8192) { + run_convrot_int8_gemm_mma_sync, GemmShape<16, 64, 64>, 5>(op); + } else { + run_convrot_int8_gemm_mma_sync, GemmShape<32, 64, 64>, 4>(op); + } + } else if (M <= 64) { + if (N <= 8192) { + run_convrot_int8_gemm_mma_sync, GemmShape<32, 64, 64>, 5>(op); + } else { + run_convrot_int8_gemm_mma_sync, GemmShape<64, 64, 64>, 3>(op); + } + } else if (M <= 128) { + if (N <= 8192) { + run_convrot_int8_gemm_mma_sync, GemmShape<32, 64, 64>, 3>(op); + } else if (N <= 16384) { + run_convrot_int8_gemm_mma_sync, GemmShape<64, 64, 64>, 5>(op); + } else { + run_convrot_int8_gemm_mma_sync, GemmShape<32, 64, 64>, 5>(op); + } + } else if (M <= 256) { + if (N <= 4096) { + run_convrot_int8_gemm_mma_sync, GemmShape<64, 64, 64>, 3>(op); + } else if (N <= 8192) { + run_convrot_int8_gemm_mma_sync, GemmShape<64, 64, 64>, 5>(op); + } else if (N <= 16384) { + run_convrot_int8_gemm_mma_sync, GemmShape<64, 64, 64>, 3>(op); + } else { + run_convrot_int8_gemm_mma_sync, GemmShape<64, 64, 64>, 5>(op); + } + } else { + run_convrot_int8_gemm_mma_sync, GemmShape<64, 64, 64>, 5>(op); + } +} + +#else // SGL_CUDA_ARCH == 900 || SGL_CUDA_ARCH == 1000 + +// out[m, n] = bf16(x_scale[m] * (w_scale[n] * acc[m, n]) [+ bias[n]]). The Sm90 +// EVT nodes alias to their Sm100 counterparts, so one tree serves both archs. +template +struct ConvRotDequantEpilogue { + using XScale = cutlass::epilogue::fusion:: + Sm90ColBroadcast<0, MmaTileShape, ElementCompute, ElementCompute, Stride, Int<0>, Int<0> > >; + using WScale = cutlass::epilogue::fusion:: + Sm90RowBroadcast<0, MmaTileShape, ElementCompute, ElementCompute, Stride, Int<1>, Int<0> > >; + using Bias = cutlass::epilogue::fusion:: + Sm90RowBroadcast<0, MmaTileShape, ElementOutput, ElementOutput, Stride, Int<1>, Int<0> > >; + using Accum = cutlass::epilogue::fusion::Sm90AccFetch; + + using Compute0 = cutlass::epilogue::fusion:: + Sm90Compute; + using EVTCompute0 = cutlass::epilogue::fusion::Sm90EVT; + + using Compute1 = cutlass::epilogue::fusion:: + Sm90Compute; + using WithoutBias = cutlass::epilogue::fusion::Sm90EVT; + + using ComputeWithBias = cutlass::epilogue::fusion:: + Sm90Compute; + using WithBias = cutlass::epilogue::fusion::Sm90EVT; + + template + using EVT = std::conditional_t; +}; + +// Builder arguments follow CUTLASS's sm100 s8_s8_void_s32 unit-test config +// (KernelScheduleAuto / EpilogueScheduleAuto). +template +struct ConvRotGemmSm100 { + template + struct Select { + using EpilogueEVT = typename ConvRotDequantEpilogue::template EVT; + + using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder< + cutlass::arch::Sm100, + cutlass::arch::OpClassTensorOp, + TileShape_, + ClusterShape_, + cutlass::epilogue::collective::EpilogueTileAuto, + ElementAccumulator, + ElementCompute, + ElementOutput, + LayoutC, + AlignmentC, + ElementOutput, + LayoutD, + AlignmentD, + cutlass::epilogue::collective::EpilogueScheduleAuto, + EpilogueEVT>::CollectiveOp; + + using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder< + cutlass::arch::Sm100, + cutlass::arch::OpClassTensorOp, + ElementA, + LayoutA, + AlignmentA, + ElementB, + LayoutB, + AlignmentB, + ElementAccumulator, + TileShape_, + ClusterShape_, + cutlass::gemm::collective::StageCountAutoCarveout( + sizeof(typename CollectiveEpilogue::SharedStorage))>, + cutlass::gemm::collective::KernelScheduleAuto>::CollectiveOp; + + using GemmKernel = + cutlass::gemm::kernel::GemmUniversal, CollectiveMainloop, CollectiveEpilogue>; + using Gemm = cutlass::gemm::device::GemmUniversalAdapter; + }; +}; + +// Mirrors cutlass_int8_scaled_mm_sm90 (int8_gemm_kernel.cu): TMA warp-specialized +// epilogue, caller-chosen mainloop schedule, persistent tile scheduler. +template +struct ConvRotGemmSm90 { + template + struct Select { + using EpilogueEVT = typename ConvRotDequantEpilogue::template EVT; + + using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder< + cutlass::arch::Sm90, + cutlass::arch::OpClassTensorOp, + TileShape_, + ClusterShape_, + cutlass::epilogue::collective::EpilogueTileAuto, + ElementAccumulator, + ElementCompute, + ElementOutput, + LayoutC, + AlignmentC, + ElementOutput, + LayoutD, + AlignmentD, + cutlass::epilogue::TmaWarpSpecialized, + EpilogueEVT>::CollectiveOp; + + using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder< + cutlass::arch::Sm90, + cutlass::arch::OpClassTensorOp, + ElementA, + LayoutA, + AlignmentA, + ElementB, + LayoutB, + AlignmentB, + ElementAccumulator, + TileShape_, + ClusterShape_, + cutlass::gemm::collective::StageCountAutoCarveout( + sizeof(typename CollectiveEpilogue::SharedStorage))>, + MainloopSchedule_>::CollectiveOp; + + using GemmKernel = cutlass::gemm::kernel::GemmUniversal< + Shape, + CollectiveMainloop, + CollectiveEpilogue, + cutlass::gemm::PersistentScheduler>; + using Gemm = cutlass::gemm::device::GemmUniversalAdapter; + }; +}; + +using ConvRotSm100Default = ConvRotGemmSm100, Shape<_2, _1, _1> >; +using ConvRotSm100WideN = ConvRotGemmSm100, Shape<_2, _1, _1> >; + +// Tile choices follow sm90_dispatch_shape (int8_gemm_kernel.cu). +using ConvRotSm90LargeM = + ConvRotGemmSm90, Shape<_2, _1, _1>, cutlass::gemm::KernelTmaWarpSpecializedPingpong>; +using ConvRotSm90SmallMNarrowN = + ConvRotGemmSm90, Shape<_2, _1, _1>, cutlass::gemm::KernelTmaWarpSpecialized>; +using ConvRotSm90SmallMWideN = + ConvRotGemmSm90, Shape<_2, _1, _1>, cutlass::gemm::KernelTmaWarpSpecialized>; + +template +void run_convrot_int8_gemm(const GemmOperands& op) { + using Gemm = typename Types::template Select::Gemm; + using StrideA = typename Gemm::GemmKernel::StrideA; + using StrideB = typename Gemm::GemmKernel::StrideB; + using StrideC = typename Gemm::GemmKernel::StrideC; + using StrideD = typename Gemm::GemmKernel::StrideD; + + const int M = op.M, N = op.N, K = op.K; + + StrideA stride_A = cutlass::make_cute_packed_stride(StrideA{}, make_shape(M, K, 1)); + StrideB stride_B = cutlass::make_cute_packed_stride(StrideB{}, make_shape(N, K, 1)); + StrideC stride_C; + StrideD stride_D = cutlass::make_cute_packed_stride(StrideD{}, make_shape(M, N, 1)); + + typename Gemm::Arguments args{ + cutlass::gemm::GemmUniversalMode::kGemm, + {M, N, K, 1}, + {op.x_q, stride_A, op.w_q, stride_B}, + {{}, nullptr, stride_C, op.out, stride_D}}; + + const ElementCompute* x_s = op.x_scale; + const ElementCompute* w_s = op.w_scale; + if constexpr (WithBias) { + const ElementOutput* bias_ptr = op.bias; + args.epilogue.thread = {{x_s}, {{w_s}, {}, {}}, {bias_ptr}, {}}; + } else { + args.epilogue.thread = {{x_s}, {{w_s}, {}, {}}, {}}; + } + + Gemm gemm_op; + check_cutlass(gemm_op.can_implement(args), "can_implement"); + const size_t workspace_size = Gemm::get_workspace_size(args); + auto workspace = host::ffi::alloc_workspace_tensor(workspace_size, op.device); + void* workspace_ptr = workspace_size == 0 ? nullptr : workspace.data_ptr(); + check_cutlass(gemm_op.initialize(args, workspace_ptr, op.stream), "initialize"); + check_cutlass(gemm_op.run(op.stream), "run"); +} + +template +void dispatch_convrot_int8_gemm_typed(const GemmOperands& op) { +#if SGL_CUDA_ARCH == 900 + const int M = op.M, N = op.N; + if (M > 128) { + run_convrot_int8_gemm(op); + } else if (N <= 4096) { + run_convrot_int8_gemm(op); + } else { + run_convrot_int8_gemm(op); + } +#else + const int M = op.M, K = op.K; + // Measured on B200-class parts with bias: 128x256 wins at large M except the M >= 4096, + // K = 12288 down-projection. The no-bias path is unmeasured; constexpr keeps its wide tile uninstantiated. + if constexpr (WithBias) { + if (M >= 256 && (K <= 8192 || M <= 2048)) { + run_convrot_int8_gemm(op); + return; + } + } + run_convrot_int8_gemm(op); +#endif +} + +void dispatch_convrot_int8_gemm(const GemmOperands& op) { + if (op.bias != nullptr) { + dispatch_convrot_int8_gemm_typed(op); + } else { + dispatch_convrot_int8_gemm_typed(op); + } +} + +#endif // SGL_CUDA_ARCH + +// weight_q [N, K] int8; weight_scale holds N floats, either [N] or the [N, 1] a +// serialized Comfy ConvRot checkpoint stores; bias [N] BF16 or absent. +void verify_weight( + TensorView weight_q, + TensorView weight_scale, + const tvm::ffi::Optional& bias, + SymbolicSize& N, + SymbolicSize& K, + SymbolicDevice& device) { + TensorMatcher({N, K}).with_dtype().with_device(device).ensure_alignment(16).verify(weight_q); + // Every GEMM path stores the BF16 output 8 elements at a time. + CHECK_HOST(N.unwrap() % 8 == 0) << "convrot_int8: N (weight rows) must be a multiple of 8, got " << N.unwrap(); + if (weight_scale.dim() == 2) { + TensorMatcher({N, 1}).with_dtype().with_device(device).verify(weight_scale); + } else { + TensorMatcher({N}).with_dtype().with_device(device).verify(weight_scale); + } + if (bias.has_value()) { + TensorMatcher({N}).with_dtype().with_device(device).verify(bias.value()); + } +} + +// A contiguous [M, K] activation with K a multiple of the group size has 16-byte +// aligned rows on its own, so the vectorized loads and the GEMM only need the +// base pointer checked; the shape and group checks come first so a bad K is +// reported as such. +template +void verify_activation( + TensorView view, const char* name, SymbolicSize& M, SymbolicSize& K, SymbolicDevice& device, int64_t group_size) { + TensorMatcher({M, K}).template with_dtype().template with_device(device).verify(view); + check_group_size(K.unwrap(), group_size); + CHECK_HOST((reinterpret_cast(view.data_ptr()) & 15) == 0) + << "convrot_int8: " << name << " must be 16-byte aligned"; +} + +// CUTLASS problem shapes and the rotate kernel index with int32. +int32_t checked_int32(int64_t value, const char* name) { + CHECK_HOST(value <= INT32_MAX) << "convrot_int8: " << name << " = " << value << " exceeds int32"; + return static_cast(value); +} + +ElementOutput* bias_ptr_or_null(const tvm::ffi::Optional& bias) { + return bias.has_value() ? static_cast(bias.value().data_ptr()) : nullptr; +} + +} // namespace convrot_int8 + +/** + * \brief Group-wise Hadamard rotation followed by per-row dynamic INT8 quantization. + * + * Applies H / sqrt(group_size) to each group_size-wide slice of every row, then + * quantizes the row symmetrically with scale = absmax / 127 (scale 1 for an + * all-zero row). This is the activation half of the fused linear below, run by + * the same kernel, and also the offline transform for the [N, K] weight that + * yields weight_q / weight_scale. + * + * \param x_q int8 [M, K] output, contiguous. + * \param x_scale float32 [M] output. + * \param x BF16 [M, K], contiguous; K must be a multiple of group_size. + * \param group_size Hadamard group width: 64, 128, 256 or 512. + */ +void convrot_rotate_quantize_activation( + tvm::ffi::TensorView x_q, tvm::ffi::TensorView x_scale, tvm::ffi::TensorView x, int64_t group_size) { + using namespace convrot_int8; + SymbolicSize M{"M"}, K{"K"}; + SymbolicDevice device; + device.set_options(); + verify_activation(x, "x", M, K, device, group_size); + verify_activation(x_q, "x_q", M, K, device, group_size); + TensorMatcher({M}).with_dtype().with_device(device).verify(x_scale); + if (M.unwrap() == 0) return; + const DLDevice dev = device.unwrap(); + // Launches and CUTLASS setup run on the tensors' device, not on whatever + // device the caller left current. + const tvm::ffi::CUDADeviceGuard guard(dev.device_id); + rotate_quantize_rowwise( + RotateOperands{ + .x = static_cast(x.data_ptr()), + .x_q = static_cast(x_q.data_ptr()), + .x_scale = static_cast(x_scale.data_ptr()), + .M = checked_int32(M.unwrap(), "M"), + .K = checked_int32(K.unwrap(), "K"), + .stream = LaunchKernel::resolve_device(dev), + }, + group_size, + /*gelu_input=*/false); +} + +/** + * \brief ConvRot INT8 linear: rotate and quantize x, then INT8 GEMM with fused dequant. + * + * out[m, n] = bf16(x_scale[m] * (weight_scale[n] * sum_k x_q[m, k] * weight_q[n, k]) + bias[n]). + * Bitwise equal to convrot_int8_linear_prequant applied to the output of + * convrot_rotate_quantize_activation(x, group_size). With gelu_input, x is + * passed through GELU(tanh) inside the rotate kernel first, using ATen's fp32 + * formula rounded to BF16, so the result is bitwise that of the eager + * F.gelu(x, approximate="tanh") followed by this op. + * + * \param out BF16 [M, N] output, contiguous. + * \param x BF16 [M, K], contiguous; K must be a multiple of group_size. + * \param weight_q int8 [N, K] from convrot_rotate_quantize_activation(weight, group_size). + * \param weight_scale float32 with N elements, [N] or [N, 1]. + * \param bias Optional BF16 [N]. + * \param group_size Hadamard group width shared by both operands. + * \param gelu_input Apply GELU(tanh) to x before the rotation. + */ +void convrot_int8_fused_linear( + tvm::ffi::TensorView out, + tvm::ffi::TensorView x, + tvm::ffi::TensorView weight_q, + tvm::ffi::TensorView weight_scale, + tvm::ffi::Optional bias, + int64_t group_size, + bool gelu_input) { + using namespace convrot_int8; + SymbolicSize M{"M"}, K{"K"}, N{"N"}; + SymbolicDevice device; + device.set_options(); + verify_activation(x, "x", M, K, device, group_size); + verify_weight(weight_q, weight_scale, bias, N, K, device); + TensorMatcher({M, N}).with_dtype().with_device(device).ensure_alignment(16).verify(out); + if (M.unwrap() == 0) return; + + const DLDevice dev = device.unwrap(); + const tvm::ffi::CUDADeviceGuard guard(dev.device_id); + const cudaStream_t stream = LaunchKernel::resolve_device(dev); + const int64_t m = M.unwrap(), n = N.unwrap(), k = K.unwrap(); + auto x_q = host::ffi::empty({m, k}, DLDataType{kDLInt, 8, 1}, dev); + auto x_scale = host::ffi::empty({m}, DLDataType{kDLFloat, 32, 1}, dev); + rotate_quantize_rowwise( + RotateOperands{ + .x = static_cast(x.data_ptr()), + .x_q = static_cast(x_q.data_ptr()), + .x_scale = static_cast(x_scale.data_ptr()), + .M = checked_int32(m, "M"), + .K = checked_int32(k, "K"), + .stream = stream, + }, + group_size, + gelu_input); + dispatch_convrot_int8_gemm( + GemmOperands{ + .out = static_cast(out.data_ptr()), + .x_q = static_cast(x_q.data_ptr()), + .w_q = static_cast(weight_q.data_ptr()), + .x_scale = static_cast(x_scale.data_ptr()), + .w_scale = static_cast(weight_scale.data_ptr()), + .bias = bias_ptr_or_null(bias), + .M = checked_int32(m, "M"), + .N = checked_int32(n, "N"), + .K = checked_int32(k, "K"), + .device = dev, + .stream = stream, + }); +} + +/** + * \brief INT8 GEMM with fused dequant on an already rotated and quantized activation. + * + * Bitwise equal to convrot_int8_fused_linear on the x that produced (xq, xs): + * the tile configuration is a function of (M, K, N, bias) alone and the epilogue + * is evaluated per element. Lets several linears sharing one input (e.g. a + * q/k/v projection trio) quantize it once. + * + * \param out BF16 [M, N] output, contiguous. + * \param xq int8 [M, K] from convrot_rotate_quantize_activation. + * \param xs float32 [M] from convrot_rotate_quantize_activation. + * \param weight_q int8 [N, K]. + * \param weight_scale float32 with N elements, [N] or [N, 1]. + * \param bias Optional BF16 [N]. + * \param group_size Hadamard group width xq was produced with; K must be a multiple of it. + */ +void convrot_int8_linear_prequant( + tvm::ffi::TensorView out, + tvm::ffi::TensorView xq, + tvm::ffi::TensorView xs, + tvm::ffi::TensorView weight_q, + tvm::ffi::TensorView weight_scale, + tvm::ffi::Optional bias, + int64_t group_size) { + using namespace convrot_int8; + SymbolicSize M{"M"}, K{"K"}, N{"N"}; + SymbolicDevice device; + device.set_options(); + verify_activation(xq, "xq", M, K, device, group_size); + TensorMatcher({M}).with_dtype().with_device(device).verify(xs); + verify_weight(weight_q, weight_scale, bias, N, K, device); + TensorMatcher({M, N}).with_dtype().with_device(device).ensure_alignment(16).verify(out); + if (M.unwrap() == 0) return; + + const DLDevice dev = device.unwrap(); + const tvm::ffi::CUDADeviceGuard guard(dev.device_id); + dispatch_convrot_int8_gemm( + GemmOperands{ + .out = static_cast(out.data_ptr()), + .x_q = static_cast(xq.data_ptr()), + .w_q = static_cast(weight_q.data_ptr()), + .x_scale = static_cast(xs.data_ptr()), + .w_scale = static_cast(weight_scale.data_ptr()), + .bias = bias_ptr_or_null(bias), + .M = checked_int32(M.unwrap(), "M"), + .N = checked_int32(N.unwrap(), "N"), + .K = checked_int32(K.unwrap(), "K"), + .device = dev, + .stream = LaunchKernel::resolve_device(dev), + }); +} + +} // namespace sglang diff --git a/python/sglang/kernels/jit/csrc/gemm/convrot_int8/cutlass_extensions/epilogue/epilogue_per_row_per_col_scale.h b/python/sglang/kernels/jit/csrc/gemm/convrot_int8/cutlass_extensions/epilogue/epilogue_per_row_per_col_scale.h new file mode 100644 index 0000000000..b949ba791a --- /dev/null +++ b/python/sglang/kernels/jit/csrc/gemm/convrot_int8/cutlass_extensions/epilogue/epilogue_per_row_per_col_scale.h @@ -0,0 +1,341 @@ +/* Copyright 2026 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// Adapted from +// https://github.com/NVIDIA/TensorRT-LLM/blob/be1788106245496872d18e702978e59b6bfd50e0/cpp/tensorrt_llm/cutlass_extensions/include/cutlass_extensions/epilogue/threadblock/epilogue_per_row_per_col_scale.h +// Modifications: clang-format reflow, includes trimmed to CUTLASS, no tensorrt_llm dependency. + +/*************************************************************************************************** + * Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ + +#pragma once + +#include +#include + +namespace cutlass { +namespace epilogue { +namespace threadblock { + +template < + typename ThreadblockShape_, + int ThreadCount, + typename ScaleTileIterator_, + typename OutputTileIterator_, + typename ElementAccumulator_, + typename ElementCompute_, + typename ElementwiseFunctor_, + bool UseMasking_ = false> +class EpilogueVisitorPerRowPerCol { + public: + using ThreadblockShape = ThreadblockShape_; + static int const kThreadCount = ThreadCount; + + using ScaleTileIterator = ScaleTileIterator_; + using OutputTileIterator = OutputTileIterator_; + using ElementwiseFunctor = ElementwiseFunctor_; + + static int const kIterations = OutputTileIterator::kIterations; + static int const kElementsPerAccess = OutputTileIterator::kElementsPerAccess; + + using ElementOutput = typename OutputTileIterator::Element; + using LayoutOutput = cutlass::layout::RowMajor; + using ElementAccumulator = ElementAccumulator_; + + using AlphaScaleElementType = typename ScaleTileIterator::Element; + + using ElementCompute = ElementCompute_; + using AccumulatorFragment = Array; + using ComputeFragment = Array; + using OutputVector = Array; + + static int const kThreadsPerRow = OutputTileIterator::ThreadMap::Detail::kAccessWidth; + static bool const kHasMultiStepsInRow = (OutputTileIterator::ThreadMap::Iterations::kColumn > 1); + + /// Argument structure + struct Arguments { + typename ElementwiseFunctor::Params elementwise; + int64_t batch_stride_alpha; + int64_t batch_stride_C; + int64_t batch_stride_D; + + // + // Methods + // + Arguments() : batch_stride_alpha(0), batch_stride_C(0), batch_stride_D(0) {} + + Arguments(typename ElementwiseFunctor::Params elementwise_) + : elementwise(elementwise_), batch_stride_alpha(0), batch_stride_C(0), batch_stride_D(0) {} + + Arguments( + typename ElementwiseFunctor::Params elementwise_, + int64_t batch_stride_alpha_, + int64_t batch_stride_C_, + int64_t batch_stride_D_) + : elementwise(elementwise_), + batch_stride_alpha(batch_stride_alpha_), + batch_stride_C(batch_stride_C_), + batch_stride_D(batch_stride_D_) {} + }; + + struct Params { + typename ElementwiseFunctor::Params elementwise; + int64_t batch_stride_alpha; + int64_t batch_stride_C; + int64_t batch_stride_D; + + // + // Methods + // + CUTLASS_HOST_DEVICE + Params() {} + + CUTLASS_HOST_DEVICE + Params(Arguments const& args) + : elementwise(args.elementwise), + batch_stride_alpha(args.batch_stride_alpha), + batch_stride_C(args.batch_stride_C), + batch_stride_D(args.batch_stride_D) {} + }; + + /// Shared storage + struct SharedStorage {}; + + private: + Params const& params_; + SharedStorage& shared_storage_; + MatrixCoord extent_; + MatrixCoord extent_real_; + ElementwiseFunctor elementwise_; + + bool const with_bias_; + bool const per_token_quant_; + bool const per_channel_quant_; + + AlphaScaleElementType* ptr_alpha_row_; + AlphaScaleElementType* ptr_alpha_col_; + ScaleTileIterator iterator_alpha_col_; + OutputTileIterator iterator_C_; + OutputTileIterator iterator_D_; + + AlphaScaleElementType element_alpha_row_ = 1.0f; + AlphaScaleElementType element_alpha_col_ = 1.0f; + typename ScaleTileIterator::Fragment fragment_alpha_col_; + typename OutputTileIterator::Fragment fragment_C_; + typename OutputTileIterator::Fragment fragment_D_; + + ElementAccumulator beta_; + + int column_offset_; + + MatrixCoord thread_offset_; + + public: + CUTLASS_DEVICE + EpilogueVisitorPerRowPerCol( + Params const& params, + SharedStorage& shared_storage, + cutlass::MatrixCoord const& problem_size, + int thread_idx, + int warp_idx, + int lane_idx, + typename ScaleTileIterator::Params params_alpha_col, + typename OutputTileIterator::Params params_C, + typename OutputTileIterator::Params params_D, + bool with_bias, + bool per_token_quant, + bool per_channel_quant, + AlphaScaleElementType* ptr_alpha_row, + AlphaScaleElementType* ptr_alpha_col, + typename OutputTileIterator::Element* ptr_C, + typename OutputTileIterator::Element* ptr_D, + cutlass::MatrixCoord const& threadblock_offset = cutlass::MatrixCoord(0, 0), + int column_offset = 0, + cutlass::MatrixCoord const& problem_size_real = cutlass::MatrixCoord(0, 0)) + : params_(params), + shared_storage_(shared_storage), + extent_(problem_size), + elementwise_(params.elementwise), + with_bias_(with_bias), + per_token_quant_(per_token_quant), + per_channel_quant_(per_channel_quant), + ptr_alpha_row_(ptr_alpha_row), + ptr_alpha_col_(ptr_alpha_col), + iterator_alpha_col_(params_alpha_col, ptr_alpha_col, problem_size, thread_idx, threadblock_offset), + iterator_C_(params_C, ptr_C, problem_size, thread_idx, threadblock_offset), + iterator_D_(params_D, ptr_D, problem_size, thread_idx, threadblock_offset), + extent_real_(problem_size_real) { + if (!per_channel_quant_ && (ptr_alpha_col_ != nullptr)) { + element_alpha_col_ = *ptr_alpha_col_; + } + + if (!per_token_quant_ && (ptr_alpha_row_ != nullptr)) { + element_alpha_row_ = *ptr_alpha_row_; + } + } + + /// Helper to indicate split-K behavior + CUTLASS_DEVICE + void set_k_partition( + int split_k_index, ///< Index of this threadblock within split-K partitioned scheme + int split_k_slices) { ///< Total number of split-K slices + } + + /// Called to set the batch index + CUTLASS_DEVICE + void set_batch_index(int batch_idx) { + iterator_alpha_col_.add_pointer_offset(batch_idx * params_.batch_stride_alpha); + iterator_C_.add_pointer_offset(batch_idx * params_.batch_stride_C); + iterator_D_.add_pointer_offset(batch_idx * params_.batch_stride_D); + } + + /// Called at the start of the epilogue just before iterating over accumulator slices + CUTLASS_DEVICE + void begin_epilogue() { + if (per_channel_quant_) { + iterator_alpha_col_.load(fragment_alpha_col_); + } + + if (with_bias_) { + iterator_C_.load(fragment_C_); + } + } + + /// Called at the start of one step before starting accumulator exchange + CUTLASS_DEVICE + void begin_step(int step_idx) { + fragment_D_.clear(); + } + + /// Called at the start of a row + CUTLASS_DEVICE + void begin_row(int row_idx) { + // load alpha_row in begin_step only when per token(row) scaling is used + if (per_token_quant_) { + int thread_offset_row = + iterator_D_.thread_start_row() + OutputTileIterator::ThreadMap::iteration_offset(row_idx).row(); + + arch::global_load( + element_alpha_row_, ptr_alpha_row_ + thread_offset_row, thread_offset_row < extent_.row()); + } + } + + /// Called after accumulators have been exchanged for each accumulator vector + CUTLASS_DEVICE + void visit(int iter_idx, int row_idx, int column_idx, int frag_idx, AccumulatorFragment const& accum) { + NumericArrayConverter source_converter; + + ComputeFragment result = source_converter(accum); + if (per_channel_quant_) { + ComputeFragment alpha_col = reinterpret_cast(&fragment_alpha_col_)[column_idx]; + result = per_token_channel_scale_accumulator_(result, alpha_col, element_alpha_row_); + } else { + result = per_token_scale_accumulator_(result, element_alpha_col_, element_alpha_row_); + } + + if (with_bias_) { + NumericArrayConverter bias_converter; + OutputVector bias = reinterpret_cast(&fragment_C_)[column_idx]; + result = bias_accumulator_(result, bias_converter(bias)); + } + + // Convert to the output + NumericArrayConverter output_converter; + OutputVector& output = reinterpret_cast(&fragment_D_)[frag_idx]; + output = output_converter(result); + } + + /// Called at the end of a row + CUTLASS_DEVICE + void end_row(int row_idx) {} + + /// Called after all accumulator elements have been visited + CUTLASS_DEVICE + void end_step(int step_idx) { + iterator_D_.store(fragment_D_); + ++iterator_D_; + } + + /// Called after all steps have been completed + CUTLASS_DEVICE + void end_epilogue() {} + + private: + CUTLASS_DEVICE + ComputeFragment per_token_channel_scale_accumulator_( + ComputeFragment const& accum, ComputeFragment const& scale_col, AlphaScaleElementType const& scale_row) { + ComputeFragment result; + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < ComputeFragment::kElements; ++i) { + result[i] = accum[i] * (scale_col[i] * scale_row); + } + + return result; + } + + CUTLASS_DEVICE + ComputeFragment per_token_scale_accumulator_( + ComputeFragment const& accum, AlphaScaleElementType const& scale_col, AlphaScaleElementType const& scale_row) { + ComputeFragment result; + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < ComputeFragment::kElements; ++i) { + result[i] = accum[i] * (scale_col * scale_row); + } + + return result; + } + + CUTLASS_DEVICE + ComputeFragment bias_accumulator_(ComputeFragment const& accum, ComputeFragment const& bias) { + ComputeFragment result; + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < OutputVector::kElements; ++i) { + result[i] = accum[i] + bias[i]; + } + return result; + } +}; + +} // namespace threadblock +} // namespace epilogue +} // namespace cutlass diff --git a/python/sglang/kernels/jit/csrc/gemm/convrot_int8/cutlass_extensions/gemm/gemm_universal_base_compat.h b/python/sglang/kernels/jit/csrc/gemm/convrot_int8/cutlass_extensions/gemm/gemm_universal_base_compat.h new file mode 100644 index 0000000000..bc807921fd --- /dev/null +++ b/python/sglang/kernels/jit/csrc/gemm/convrot_int8/cutlass_extensions/gemm/gemm_universal_base_compat.h @@ -0,0 +1,388 @@ +/* Copyright 2026 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// Adapted from +// https://github.com/NVIDIA/TensorRT-LLM/blob/be1788106245496872d18e702978e59b6bfd50e0/cpp/tensorrt_llm/cutlass_extensions/include/cutlass_extensions/gemm/device/gemm_universal_base_compat.h +// Modifications: clang-format reflow, includes trimmed to CUTLASS, no tensorrt_llm dependency. + +/*************************************************************************************************** + * Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +#pragma once + +#include +#include +#include + +//////////////////////////////////////////////////////////////////////////////// + +namespace cutlass { +namespace gemm { +namespace device { + +///////////////////////////////////////////////////////////////////////////////////////////////// + +/* + This is the device layer from CUTLASS 2.10 (SHA - cc85b64cf676c45f98a17e3a47c0aafcf817f088) + It is replicated here since we needed to duplicate kernel level APIs for mixed dtype GEMMs + and SmoothQuant. The newer device layer is not compatible with these older kernel level APIs. + + Note: While CUTLASS 3.x supports stream-k, none of the kernels in the extensions folder support + that feature at the moment. + */ + +template +class GemmUniversalBaseCompat { + public: + using GemmKernel = GemmKernel_; + using ThreadblockShape = typename GemmKernel::Mma::Shape; + + using ElementA = typename GemmKernel::ElementA; + using LayoutA = typename GemmKernel::LayoutA; + using TensorRefA = TensorRef; + static ComplexTransform const kTransformA = GemmKernel::kTransformA; + + using ElementB = typename GemmKernel::ElementB; + using LayoutB = typename GemmKernel::LayoutB; + using TensorRefB = TensorRef; + static ComplexTransform const kTransformB = GemmKernel::kTransformB; + + using ElementC = typename GemmKernel::ElementC; + using LayoutC = typename GemmKernel::LayoutC; + using TensorRefC = TensorRef; + using TensorRefD = TensorRef; + + using ElementAccumulator = typename GemmKernel::Mma::Policy::Operator::ElementC; + + using EpilogueOutputOp = typename GemmKernel::EpilogueOutputOp; + using ThreadblockSwizzle = typename GemmKernel::ThreadblockSwizzle; + using Operator = typename GemmKernel::Operator; + + /// Argument structure + using Arguments = typename GemmKernel::Arguments; + + protected: + /// Kernel parameters object + typename GemmKernel::Params params_; + + protected: + /// Private helper to obtain the grid dimensions with fix-up for split-K + static void get_grid_shape_(gemm::GemmCoord& grid_tiled_shape, int& gemm_k_size, Arguments const& args) { + // Determine grid shape + ThreadblockSwizzle threadblock_swizzle; + + grid_tiled_shape = threadblock_swizzle.get_tiled_shape( + args.problem_size, {ThreadblockShape::kM, ThreadblockShape::kN, ThreadblockShape::kK}, args.batch_count); + + gemm_k_size = args.problem_size.k(); + + if (args.mode == GemmUniversalMode::kGemm || args.mode == GemmUniversalMode::kGemmSplitKParallel) { + int const kAlignK = + const_max(const_max(128 / sizeof_bits::value, 128 / sizeof_bits::value), 1); + + gemm_k_size = round_up(ceil_div(args.problem_size.k(), args.batch_count), kAlignK); + + if (gemm_k_size) { + grid_tiled_shape.k() = ceil_div(args.problem_size.k(), gemm_k_size); + } + } + } + + public: + /// Constructs the GEMM. + GemmUniversalBaseCompat() {} + + /// Determines whether the GEMM can execute the given problem. + static Status can_implement(Arguments const& args) { + // Determine grid shape + cutlass::gemm::GemmCoord grid_tiled_shape; + int gemm_k_size = 0; + + get_grid_shape_(grid_tiled_shape, gemm_k_size, args); + + ThreadblockSwizzle threadblock_swizzle; + dim3 grid = threadblock_swizzle.get_grid_shape(grid_tiled_shape); + + uint32_t const kGridYZMax = ((1 << (sizeof(uint16_t) * 8)) - 1); + + if (!(grid.y <= kGridYZMax && grid.z <= kGridYZMax)) { + return Status::kErrorInvalidProblem; + } + + return GemmKernel::can_implement(args); + } + + /// Gets the workspace size + static size_t get_workspace_size(Arguments const& args) { + CUTLASS_TRACE_HOST("GemmUniversalBaseCompat::get_workspace_size()"); + + size_t workspace_bytes = 0; + + // Determine grid shape + cutlass::gemm::GemmCoord grid_tiled_shape; + int gemm_k_size = 0; + + get_grid_shape_(grid_tiled_shape, gemm_k_size, args); + + if (args.mode == GemmUniversalMode::kGemmSplitKParallel) { + // Split-K parallel always requires a temporary workspace + workspace_bytes = sizeof(ElementC) * size_t(args.batch_stride_D) * size_t(grid_tiled_shape.k()); + } else if (args.mode == GemmUniversalMode::kGemm && grid_tiled_shape.k() > 1) { + // Serial split-K only requires a temporary workspace if the number of partitions along the + // GEMM K dimension is greater than one. + workspace_bytes = sizeof(int) * size_t(grid_tiled_shape.m()) * size_t(grid_tiled_shape.n()); + } + + CUTLASS_TRACE_HOST(" workspace_bytes: " << workspace_bytes); + + workspace_bytes += GemmKernel::get_extra_workspace_size(args, grid_tiled_shape); + + return workspace_bytes; + } + + /// Computes the grid shape + static dim3 get_grid_shape(Arguments const& args) { + CUTLASS_TRACE_HOST("GemmUniversalBaseCompat::get_grid_shape()"); + + ThreadblockSwizzle threadblock_swizzle; + + cutlass::gemm::GemmCoord grid_tiled_shape; + int gemm_k_size = 0; + + get_grid_shape_(grid_tiled_shape, gemm_k_size, args); + dim3 result = threadblock_swizzle.get_grid_shape(grid_tiled_shape); + + CUTLASS_TRACE_HOST( + " grid_tiled_shape: " << grid_tiled_shape << "\n" + << " result = {" << result << "}"); + + return result; + } + + /// Computes the maximum number of active blocks per multiprocessor + static int maximum_active_blocks(int smem_capacity = -1) { + CUTLASS_TRACE_HOST("GemmUniversalBaseCompat::maximum_active_blocks()"); + + int max_active_blocks = -1; + int smem_size = int(sizeof(typename GemmKernel::SharedStorage)); + + CUTLASS_TRACE_HOST(" smem_size: " << smem_size << " bytes"); + + if (smem_size <= (48 << 10)) { + cudaError_t result = cudaOccupancyMaxActiveBlocksPerMultiprocessor( + &max_active_blocks, Kernel, GemmKernel::kThreadCount, smem_size); + + if (result == cudaSuccess) { + CUTLASS_TRACE_HOST(" max_active_blocks: " << max_active_blocks); + return max_active_blocks; + } + } else { + // Query assuming zero shared memory then compute occupancy limit based on SMEM + cudaError_t result = cudaOccupancyMaxActiveBlocksPerMultiprocessor( + &max_active_blocks, Kernel, GemmKernel::kThreadCount, 0); + + if (result != cudaSuccess) { + CUTLASS_TRACE_HOST( + " cudaOccupancyMaxActiveBlocksPerMultiprocessor() returned error " << cudaGetErrorString(result)); + + return -1; + } + + if (smem_capacity < 0) { + int device_idx = 0; + result = cudaGetDevice(&device_idx); + + if (result != cudaSuccess) { + return -1; + } + + cudaDeviceProp properties; + result = cudaGetDeviceProperties(&properties, device_idx); + + if (result != cudaSuccess) { + return -1; + } + + smem_capacity = static_cast(properties.sharedMemPerMultiprocessor); + } + + int occupancy = std::min(max_active_blocks, smem_capacity / smem_size); + + CUTLASS_TRACE_HOST(" occupancy: " << occupancy); + + return occupancy; + } + + CUTLASS_TRACE_HOST(" returning internal error"); + + return -1; + } + + /// Initializes GEMM state from arguments. + Status initialize(Arguments const& args, void* workspace = nullptr, cudaStream_t stream = nullptr) { + CUTLASS_TRACE_HOST( + "GemmUniversalBaseCompat::initialize() - workspace " << workspace + << ", stream: " << (stream ? "non-null" : "null")); + + size_t workspace_bytes = get_workspace_size(args); + + CUTLASS_TRACE_HOST(" workspace_bytes: " << workspace_bytes); + + if (workspace_bytes) { + if (!workspace) { + CUTLASS_TRACE_HOST(" error: device workspace must not be null"); + + return Status::kErrorWorkspaceNull; + } + + if (args.mode == GemmUniversalMode::kGemm) { + CUTLASS_TRACE_HOST(" clearing device workspace"); + cudaError_t result = cudaMemsetAsync(workspace, 0, workspace_bytes, stream); + + if (result != cudaSuccess) { + CUTLASS_TRACE_HOST(" cudaMemsetAsync() returned error " << cudaGetErrorString(result)); + + return Status::kErrorInternal; + } + } + } + + // Get CUDA grid shape + cutlass::gemm::GemmCoord grid_tiled_shape; + int gemm_k_size = 0; + + get_grid_shape_(grid_tiled_shape, gemm_k_size, args); + + // Initialize the Params structure + params_ = typename GemmKernel::Params(args, grid_tiled_shape, gemm_k_size, static_cast(workspace)); + + // Specify shared memory capacity for kernel. + int smem_size = int(sizeof(typename GemmKernel::SharedStorage)); + + if (smem_size >= (48 << 10)) { + cudaError_t result = + cudaFuncSetAttribute(Kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size); + + if (result != cudaSuccess) { + return Status::kErrorInternal; + } + } + + return Status::kSuccess; + } + + /// Lightweight update given a subset of arguments + Status update(Arguments const& args, void* workspace = nullptr) { + CUTLASS_TRACE_HOST("GemmUniversalBaseCompat()::update() - workspace: " << workspace); + + size_t workspace_bytes = get_workspace_size(args); + + if (workspace_bytes && !workspace) { + return Status::kErrorWorkspaceNull; + } + + params_.update(args, workspace); + + return Status::kSuccess; + } + + /// Runs the kernel using initialized state. + Status run(cudaStream_t stream = nullptr) { + CUTLASS_TRACE_HOST("GemmUniversalBaseCompat::run()"); + + // + // Configure grid and block dimensions + // + + ThreadblockSwizzle threadblock_swizzle; + + dim3 grid = threadblock_swizzle.get_grid_shape(params_.grid_tiled_shape); + dim3 block(GemmKernel::kThreadCount, 1, 1); + + int smem_size = int(sizeof(typename GemmKernel::SharedStorage)); + + // + // Launch kernel + // + + CUTLASS_TRACE_HOST(" grid: (" << grid << "), block: (" << block << "), SMEM: " << smem_size << " bytes"); + + // Launch + cutlass::Kernel<<>>(params_); + + // + // Query for errors + // + cudaError_t result = cudaGetLastError(); + + if (result != cudaSuccess) { + CUTLASS_TRACE_HOST(" grid launch failed with error " << cudaGetErrorString(result)); + return Status::kErrorInternal; + } + + return Status::kSuccess; + } + + /// Runs the kernel using initialized state. + Status operator()(cudaStream_t stream = nullptr) { + return run(stream); + } + + /// Runs the kernel using initialized state. + Status operator()(Arguments const& args, void* workspace = nullptr, cudaStream_t stream = nullptr) { + Status status = initialize(args, workspace, stream); + + if (status == Status::kSuccess) { + status = run(stream); + } + + return status; + } +}; + +///////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace device +} // namespace gemm +} // namespace cutlass + +///////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/python/sglang/kernels/jit/csrc/gemm/convrot_int8/cutlass_extensions/gemm/gemm_with_epilogue_visitor.h b/python/sglang/kernels/jit/csrc/gemm/convrot_int8/cutlass_extensions/gemm/gemm_with_epilogue_visitor.h new file mode 100644 index 0000000000..d16de87abf --- /dev/null +++ b/python/sglang/kernels/jit/csrc/gemm/convrot_int8/cutlass_extensions/gemm/gemm_with_epilogue_visitor.h @@ -0,0 +1,524 @@ +/* Copyright 2026 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// Adapted from +// https://github.com/NVIDIA/TensorRT-LLM/blob/be1788106245496872d18e702978e59b6bfd50e0/cpp/tensorrt_llm/cutlass_extensions/include/cutlass_extensions/gemm/kernel/gemm_with_epilogue_visitor.h +// Modifications: clang-format reflow, includes trimmed to CUTLASS, no tensorrt_llm dependency. + +/*************************************************************************************************** + * Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ + +#pragma once + +#include +#include +#include +#include +#include + +///////////////////////////////////////////////////////////////////////////////////////////////// + +namespace cutlass { +namespace gemm { +namespace kernel { + +///////////////////////////////////////////////////////////////////////////////////////////////// + +template < + typename Mma_, ///! Threadblock-scoped matrix multiply-accumulate + typename Epilogue_, ///! Epilogue + typename ThreadblockSwizzle_ ///! Threadblock swizzling function + > +struct GemmWithEpilogueVisitor { + public: + using Mma = Mma_; + using Epilogue = Epilogue_; + using EpilogueVisitor = typename Epilogue::Visitor; + using ThreadblockSwizzle = ThreadblockSwizzle_; + + using ElementA = typename Mma::IteratorA::Element; + using LayoutA = typename Mma::IteratorA::Layout; + using TensorRefA = TensorRef; + + using ElementB = typename Mma::IteratorB::Element; + using LayoutB = typename Mma::IteratorB::Layout; + using TensorRefB = TensorRef; + + using ElementCompute = typename EpilogueVisitor::ElementCompute; + using LayoutAlphaCol = cutlass::layout::RowMajor; + using LayoutAlphaRow = cutlass::layout::ColumnMajor; + using TensorRefAlphaCol = TensorRef; + using TensorRefAlphaRow = TensorRef; + + using ElementC = typename EpilogueVisitor::ElementOutput; + using LayoutC = typename Epilogue::Layout; + using TensorRefC = TensorRef; + + static ComplexTransform const kTransformA = Mma::kTransformA; + static ComplexTransform const kTransformB = Mma::kTransformB; + using Operator = typename Mma::Operator; + + using OperatorClass = typename Mma::Operator::OperatorClass; + using ThreadblockShape = typename Mma::Shape; + using WarpShape = typename Mma::Operator::Shape; + using InstructionShape = typename Mma::Policy::Operator::InstructionShape; + using ArchTag = typename Mma::ArchTag; + using EpilogueOutputOp = + typename Epilogue::Visitor::ElementwiseFunctor; // Define type so GemmUniversalBase doesn't complain + + static int const kStages = Mma::kStages; + static int const kAlignmentA = Mma::IteratorA::AccessType::kElements; + static int const kAlignmentB = Mma::IteratorB::AccessType::kElements; + static int const kAlignmentC = EpilogueVisitor::kElementsPerAccess; + + /// Warp count (concept: GemmShape) + using WarpCount = typename Mma::WarpCount; + static int const kThreadCount = 32 * WarpCount::kCount; + + /// Split-K preserves splits that are 128b aligned + static int const kSplitKAlignment = const_max(128 / sizeof_bits::value, 128 / sizeof_bits::value); + + // + // Structures + // + + /// Argument structure + struct Arguments { + // + // Data members + // + + GemmUniversalMode mode; + GemmCoord problem_size; + int batch_count; + + TensorRefA ref_A; + TensorRefB ref_B; + TensorRefAlphaCol ref_alpha_col; + TensorRefAlphaRow ref_alpha_row; + TensorRefC ref_C; + TensorRefC ref_D; + + int64_t batch_stride_A; + int64_t batch_stride_B; + int64_t batch_stride_D; + + typename EpilogueVisitor::Arguments epilogue_visitor; + + // + // Methods + // + + Arguments() : mode(GemmUniversalMode::kGemm), batch_count(1) {} + + /// constructs an arguments structure + Arguments( + GemmCoord problem_size_, + TensorRefA ref_A_, + TensorRefB ref_B_, + TensorRefAlphaCol ref_alpha_col_, + TensorRefAlphaRow ref_alpha_row_, + TensorRefC ref_C_, + TensorRefC ref_D_, + typename EpilogueVisitor::Arguments epilogue_visitor_) + : mode(GemmUniversalMode::kGemm), + problem_size(problem_size_), + batch_count(1), + ref_A(ref_A_), + ref_B(ref_B_), + ref_alpha_col(ref_alpha_col_), + ref_alpha_row(ref_alpha_row_), + ref_C(ref_C_), + ref_D(ref_D_), + batch_stride_A(0), + batch_stride_B(0), + batch_stride_D(0), + epilogue_visitor(epilogue_visitor_) {} + }; + + // + // Structure for precomputing values in host memory and passing to kernels + // + + /// Parameters structure + struct Params { + cutlass::gemm::GemmCoord problem_size; + cutlass::gemm::GemmCoord grid_tiled_shape; + int swizzle_log_tile; + + typename Mma::IteratorA::Params params_A; + typename Mma::IteratorB::Params params_B; + typename EpilogueVisitor::ScaleTileIterator::Params params_alpha_col; + typename EpilogueVisitor::ScaleTileIterator::Params params_alpha_row; + typename EpilogueVisitor::OutputTileIterator::Params params_C; + typename EpilogueVisitor::OutputTileIterator::Params params_D; + + GemmUniversalMode mode; + int batch_count; + int gemm_k_size; + + void* ptr_A; + void* ptr_B; + typename EpilogueVisitor::ScaleTileIterator::Element* ptr_alpha_col; + typename EpilogueVisitor::ScaleTileIterator::Element* ptr_alpha_row; + ElementC* ptr_C; + ElementC* ptr_D; + + int64_t batch_stride_A; + int64_t batch_stride_B; + + typename EpilogueVisitor::Params epilogue_visitor; + + // + // Methods + // + + CUTLASS_HOST_DEVICE + Params() + : swizzle_log_tile(0), + params_A(0), + params_B(0), + params_alpha_col(0), + params_C(0), + params_D(0), + batch_count(0), + gemm_k_size(0), + mode(cutlass::gemm::GemmUniversalMode::kGemm), + ptr_A(nullptr), + ptr_B(nullptr), + ptr_alpha_col(nullptr), + ptr_alpha_row(nullptr), + ptr_C(nullptr), + ptr_D(nullptr), + batch_stride_A(0), + batch_stride_B(0) {} + + Params(Arguments const& args, cutlass::gemm::GemmCoord const& grid_tiled_shape_, int gemm_k_size_, int* workspace_) + : problem_size(args.problem_size), + swizzle_log_tile(0), + params_A(args.ref_A.layout()), + params_B(args.ref_B.layout()), + params_alpha_col(args.ref_alpha_col.layout()), + params_alpha_row(args.ref_alpha_col.layout()), + params_C(args.ref_C.layout()), + params_D(args.ref_D.layout()), + mode(args.mode), + batch_count(args.batch_count), + gemm_k_size(args.problem_size.k()), + ptr_A(args.ref_A.data()), + ptr_B(args.ref_B.data()), + ptr_alpha_col(args.ref_alpha_col.data()), + ptr_alpha_row(args.ref_alpha_row.data()), + ptr_C(args.ref_C.data()), + ptr_D(args.ref_D.data()), + batch_stride_A(args.batch_stride_A), + batch_stride_B(args.batch_stride_B), + epilogue_visitor(args.epilogue_visitor) { + ThreadblockSwizzle threadblock_swizzle; + + grid_tiled_shape = threadblock_swizzle.get_tiled_shape( + args.problem_size, {ThreadblockShape::kM, ThreadblockShape::kN, ThreadblockShape::kK}, args.batch_count); + + if (args.mode == GemmUniversalMode::kGemm || args.mode == GemmUniversalMode::kGemmSplitKParallel) { + int const kAlignK = + const_max(const_max(128 / sizeof_bits::value, 128 / sizeof_bits::value), 1); + + gemm_k_size = round_up(ceil_div(args.problem_size.k(), args.batch_count), kAlignK); + + if (gemm_k_size) { + grid_tiled_shape.k() = ceil_div(args.problem_size.k(), gemm_k_size); + } + } + + swizzle_log_tile = threadblock_swizzle.get_log_tile(grid_tiled_shape); + } + }; + + /// Shared memory storage structure + union SharedStorage { + typename Mma::SharedStorage main_loop; + + struct { + typename Epilogue::SharedStorage epilogue; + typename EpilogueVisitor::SharedStorage visitor; + } epilogue; + }; + + public: + // + // Methods + // + + CUTLASS_DEVICE + GemmWithEpilogueVisitor() {} + + /// Determines whether kernel satisfies alignment + static Status can_implement(cutlass::gemm::GemmCoord const& problem_size) { + CUTLASS_TRACE_HOST("GemmWithEpilogueVisitor::can_implement()"); + + static int const kAlignmentA = Mma::IteratorA::AccessType::kElements; + static int const kAlignmentB = Mma::IteratorB::AccessType::kElements; + static int const kAlignmentC = EpilogueVisitor::OutputTileIterator::kElementsPerAccess; + + bool isAMisaligned = false; + bool isBMisaligned = false; + bool isCMisaligned = false; + + if (platform::is_same::value) { + isAMisaligned = problem_size.k() % kAlignmentA; + } else if (platform::is_same::value) { + isAMisaligned = problem_size.m() % kAlignmentA; + } else if ( + platform::is_same>::value || + platform::is_same>::value) { + isAMisaligned = problem_size.k() % kAlignmentA; + } + + if (platform::is_same::value) { + isBMisaligned = problem_size.n() % kAlignmentB; + } else if (platform::is_same::value) { + isBMisaligned = problem_size.k() % kAlignmentB; + } else if ( + platform::is_same>::value || + platform::is_same>::value) { + isBMisaligned = problem_size.k() % kAlignmentB; + } + + if (platform::is_same::value) { + isCMisaligned = problem_size.n() % kAlignmentC; + } else if (platform::is_same::value) { + isCMisaligned = problem_size.m() % kAlignmentC; + } else if ( + platform::is_same>::value || + platform::is_same>::value) { + isCMisaligned = problem_size.n() % kAlignmentC; + } + + if (isAMisaligned) { + CUTLASS_TRACE_HOST(" returning kErrorMisalignedOperand for A operand"); + return Status::kErrorMisalignedOperand; + } + + if (isBMisaligned) { + CUTLASS_TRACE_HOST(" returning kErrorMisalignedOperand for B operand"); + return Status::kErrorMisalignedOperand; + } + + if (isCMisaligned) { + CUTLASS_TRACE_HOST(" returning kErrorMisalignedOperand for C operand"); + return Status::kErrorMisalignedOperand; + } + + CUTLASS_TRACE_HOST(" returning kSuccess"); + + return Status::kSuccess; + } + + static Status can_implement(Arguments const& args) { + return can_implement(args.problem_size); + } + + static size_t get_extra_workspace_size(Arguments const& args, cutlass::gemm::GemmCoord const& grid_tiled_shape) { + return 0; + } + +#define SPLIT_K_ENABLED 1 + + /// Executes one GEMM + CUTLASS_DEVICE + void run_kernel_(Params const& params, SharedStorage& shared_storage) { + // Compute threadblock location + ThreadblockSwizzle threadblock_swizzle; + + cutlass::gemm::GemmCoord threadblock_tile_offset = threadblock_swizzle.get_tile_offset(params.swizzle_log_tile); + + // Early exit if CTA is out of range + if (params.grid_tiled_shape.m() <= threadblock_tile_offset.m() || + params.grid_tiled_shape.n() <= threadblock_tile_offset.n()) { + return; + } + + int offset_k = 0; + int problem_size_k = params.problem_size.k(); + + ElementA* ptr_A = static_cast(params.ptr_A); + ElementB* ptr_B = static_cast(params.ptr_B); + +#if SPLIT_K_ENABLED + // + // Fetch pointers based on mode. + // + if (params.mode == GemmUniversalMode::kGemm || params.mode == GemmUniversalMode::kGemmSplitKParallel) { + if (threadblock_tile_offset.k() + 1 < params.grid_tiled_shape.k()) { + problem_size_k = (threadblock_tile_offset.k() + 1) * params.gemm_k_size; + } + + offset_k = threadblock_tile_offset.k() * params.gemm_k_size; + } else if (params.mode == GemmUniversalMode::kBatched) { + ptr_A += threadblock_tile_offset.k() * params.batch_stride_A; + ptr_B += threadblock_tile_offset.k() * params.batch_stride_B; + } else if (params.mode == GemmUniversalMode::kArray) { + ptr_A = static_cast(params.ptr_A)[threadblock_tile_offset.k()]; + ptr_B = static_cast(params.ptr_B)[threadblock_tile_offset.k()]; + } +#endif + + // Compute initial location in logical coordinates + cutlass::MatrixCoord tb_offset_A{ + threadblock_tile_offset.m() * Mma::Shape::kM, + offset_k, + }; + + cutlass::MatrixCoord tb_offset_B{offset_k, threadblock_tile_offset.n() * Mma::Shape::kN}; + + // Compute position within threadblock + int thread_idx = threadIdx.x; + + // Construct iterators to A and B operands + typename Mma::IteratorA iterator_A( + params.params_A, ptr_A, {params.problem_size.m(), problem_size_k}, thread_idx, tb_offset_A); + + typename Mma::IteratorB iterator_B( + params.params_B, ptr_B, {problem_size_k, params.problem_size.n()}, thread_idx, tb_offset_B); + + // Broadcast the warp_id computed by lane 0 to ensure dependent code + // is compiled as warp-uniform. + int warp_idx = __shfl_sync(0xffffffff, threadIdx.x / 32, 0); + + int lane_idx = threadIdx.x % 32; + + // + // Main loop + // + + // Construct thread-scoped matrix multiply + Mma mma(shared_storage.main_loop, thread_idx, warp_idx, lane_idx); + + typename Mma::FragmentC accumulators; + + accumulators.clear(); + + // Compute threadblock-scoped matrix multiply-add + int gemm_k_iterations = (problem_size_k - offset_k + Mma::Shape::kK - 1) / Mma::Shape::kK; + + // Compute threadblock-scoped matrix multiply-add + mma(gemm_k_iterations, accumulators, iterator_A, iterator_B, accumulators); + + // + // Masked tile iterators constructed from members + // + + threadblock_tile_offset = threadblock_swizzle.get_tile_offset(params.swizzle_log_tile); + + // assume identity swizzle + MatrixCoord threadblock_offset( + threadblock_tile_offset.m() * Mma::Shape::kM, threadblock_tile_offset.n() * Mma::Shape::kN); + + int block_idx = threadblock_tile_offset.m() + threadblock_tile_offset.n() * params.grid_tiled_shape.m(); + + // + // Construct the epilogue visitor + // + + bool with_bias = true; + if (params.ptr_C == nullptr) { + with_bias = false; + } + + EpilogueVisitor epilogue_visitor( + params.epilogue_visitor, + shared_storage.epilogue.visitor, + params.problem_size.mn(), + thread_idx, + warp_idx, + lane_idx, + params.params_alpha_col, + params.params_C, + params.params_D, + with_bias, + true, + true, + params.ptr_alpha_row, + params.ptr_alpha_col, + params.ptr_C, + params.ptr_D, + threadblock_offset, + blockIdx.y * params.problem_size.m()); + + if (params.mode == GemmUniversalMode::kGemm) { + // Indicate which position in a serial reduction the output operator is currently updating + epilogue_visitor.set_k_partition(threadblock_tile_offset.k(), params.grid_tiled_shape.k()); + } else if (params.mode == GemmUniversalMode::kBatched || params.mode == GemmUniversalMode::kArray) { + epilogue_visitor.set_batch_index(threadblock_tile_offset.k()); + } + + // Construct the epilogue + Epilogue epilogue(shared_storage.epilogue.epilogue, thread_idx, warp_idx, lane_idx); + + // Execute the epilogue operator to update the destination tensor. + epilogue(epilogue_visitor, accumulators); + } + + template + CUTLASS_DEVICE void run_kernel(Params const& params, SharedStorage& shared_storage) { + if constexpr (platform::is_same::value) { + run_kernel_(params, shared_storage); + } else { + CUTLASS_NOT_IMPLEMENTED(); + } + } + + /// Executes one GEMM + CUTLASS_DEVICE + void operator()(Params const& params, SharedStorage& shared_storage) { + run_kernel(params, shared_storage); + } +}; + +///////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace kernel +} // namespace gemm +} // namespace cutlass + +///////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/python/sglang/kernels/ops/gemm/__init__.py b/python/sglang/kernels/ops/gemm/__init__.py index 49a962d3bb..688873c55e 100644 --- a/python/sglang/kernels/ops/gemm/__init__.py +++ b/python/sglang/kernels/ops/gemm/__init__.py @@ -480,6 +480,50 @@ register_kernel( ), ) ) +# Mirrors SUPPORTED_CAPABILITIES in ops.gemm.convrot_int8 (pinned by a test). +_CONVROT_INT8 = frozenset( + CapabilityRequirement.cuda(min_sm=sm, max_sm=sm) + for sm in ((9, 0), (10, 0), (12, 0), (12, 1)) +) +register_kernel( + KernelSpec( + op="gemm.convrot_rotate_quantize_activation", + backend=KernelBackend.JIT, + target="sglang.kernels.ops.gemm.convrot_int8:convrot_rotate_quantize_activation", + capabilities=_CONVROT_INT8, + format_signature=FormatSignature( + supported_dtypes=("bfloat16",), + description="group-wise regular Hadamard rotation + per-row INT8 quant of BF16 [M, K] -> (int8 [M, K], fp32 [M])", + ), + description="ConvRot INT8 activation / weight quantizer (sglang.kernels.jit).", + ) +) +register_kernel( + KernelSpec( + op="gemm.convrot_int8_fused_linear", + backend=KernelBackend.JIT, + target="sglang.kernels.ops.gemm.convrot_int8:convrot_int8_fused_linear", + capabilities=_CONVROT_INT8, + format_signature=FormatSignature( + supported_dtypes=("bfloat16", "int8"), + description="BF16 [M, K] x int8 [N, K] -> BF16 [M, N] with in-kernel rotation, INT8 quant and fused per-row x per-column dequant (+ bias)", + ), + description="ConvRot INT8 W8A8 linear (sglang.kernels.jit).", + ) +) +register_kernel( + KernelSpec( + op="gemm.convrot_int8_linear_prequant", + backend=KernelBackend.JIT, + target="sglang.kernels.ops.gemm.convrot_int8:convrot_int8_linear_prequant", + capabilities=_CONVROT_INT8, + format_signature=FormatSignature( + supported_dtypes=("int8", "float32", "bfloat16"), + description="int8 [M, K] x int8 [N, K] with per-row / per-column fp32 scales -> BF16 [M, N] (+ bias); for linears sharing one rotated input", + ), + description="ConvRot INT8 GEMM on a pre-quantized activation (sglang.kernels.jit).", + ) +) register_kernel( KernelSpec( op="gemm.sm120_fp8_gemv", diff --git a/python/sglang/kernels/ops/gemm/convrot_int8.py b/python/sglang/kernels/ops/gemm/convrot_int8.py new file mode 100644 index 0000000000..0d1d2b151e --- /dev/null +++ b/python/sglang/kernels/ops/gemm/convrot_int8.py @@ -0,0 +1,231 @@ +"""ConvRot INT8 W8A8 linear (arXiv:2512.03673), JIT-compiled for the local GPU. + +One entry rotates a BF16 activation group-wise with the regular Hadamard, +quantizes it per row to INT8 and runs the dense INT8 GEMM with the per-row x +per-column dequant (and bias) fused into the epilogue; the rotate step alone +is the offline transform for the [N, K] weight. CUTLASS 3.x WGMMA on CC 9.0, +tcgen05 on CC 10.0 and the CUTLASS 2.x mma.sync path on CC 12.0 / 12.1; the +module compiles once per machine on first use and is cached by the JIT. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import torch + +from sglang.kernels.jit.utils import cache_once, get_jit_cuda_arch, load_jit +from sglang.kernels.jit.utils.common import is_hip_runtime, is_musa_runtime +from sglang.kernels.kernel_api_logging import debug_kernel_api +from sglang.srt.utils.custom_op import register_custom_op + +if TYPE_CHECKING: + from tvm_ffi.module import Module + +# Exact compute capabilities the entry header carries a GEMM path for: sm_90a +# and sm_100a hold the WGMMA / tcgen05 kernels, CC 12.0 / 12.1 the mma.sync +# path. CC 10.3 is left out on purpose (INT8 tensor-core rate); the quant +# method reports the reason. +SUPPORTED_CAPABILITIES: frozenset[tuple[int, int]] = frozenset( + {(9, 0), (10, 0), (12, 0), (12, 1)} +) +# Hadamard group widths the rotate kernel is instantiated for. +GROUP_SIZES: tuple[int, ...] = (64, 128, 256, 512) + + +def convrot_int8_supported_capabilities() -> frozenset[tuple[int, int]]: + """(major, minor) pairs the convrot_int8 ops compile for.""" + return SUPPORTED_CAPABILITIES + + +def _convrot_int8_cuda_flags() -> list[str]: + return [ + "-DNDEBUG", + "-DCUTE_USE_PACKED_TUPLE=1", + "-DCUTLASS_ENABLE_TENSOR_CORE_MMA=1", + "-DCUTLASS_VERSIONS_GENERATED", + "-DCUTLASS_TEST_LEVEL=0", + "-DCUTLASS_TEST_ENABLE_CACHED_RESULTS=1", + "-DCUTLASS_DEBUG_TRACE_LEVEL=0", + "--expt-relaxed-constexpr", + "--expt-extended-lambda", + # Only the row-scale division and denormal handling see this flag; a + # precise build rounds a few INT8 codes one step away from the + # validated numerics. The GELU tanh is pinned to libdevice __nv_tanhf. + "--use_fast_math", + ] + + +def convrot_int8_local_capability() -> tuple[int, int] | None: + """(major, minor) the JIT build would target, or None off CUDA (ROCm and + MUSA capability numbers are not SM versions).""" + if is_hip_runtime() or is_musa_runtime(): + return None + arch = get_jit_cuda_arch() + return (arch.major, arch.minor) + + +@cache_once +def _jit_convrot_int8_module() -> Module: + """Compile and cache the convrot_int8 module for the local GPU.""" + capability = convrot_int8_local_capability() + if capability not in SUPPORTED_CAPABILITIES: + supported = ", ".join(f"{a}.{b}" for a, b in sorted(SUPPORTED_CAPABILITIES)) + where = ( + "a non-CUDA device" + if capability is None + else f"CC {capability[0]}.{capability[1]}" + ) + raise RuntimeError( + f"convrot_int8: no kernel for {where} (supported: {supported})" + ) + return load_jit( + "convrot_int8", + cuda_files=["gemm/convrot_int8/convrot_int8_entry.cuh"], + cuda_wrappers=[ + ( + "convrot_rotate_quantize_activation", + "convrot_rotate_quantize_activation", + ), + ("convrot_int8_fused_linear", "convrot_int8_fused_linear"), + ("convrot_int8_linear_prequant", "convrot_int8_linear_prequant"), + ], + extra_dependencies=["cutlass"], + extra_cuda_cflags=_convrot_int8_cuda_flags(), + ) + + +def load_convrot_int8_module() -> Module: + """Build (first use) or load the cached module; raises when the GPU is + unsupported, the CUDA toolchain is missing or the build fails.""" + return _jit_convrot_int8_module() + + +# The custom ops below exist for torch.compile: dynamo needs an opaque op. Eager +# calls go straight to the module instead, because the dispatcher round trip of +# a Python-implemented op costs ~9 us on top of the ~10 us tvm-ffi call +# (measured on GB200) and a Qwen-Image DiT forward issues ~850 of these calls. +@register_custom_op( + op_name="convrot_rotate_quantize_activation", + mutates_args=["x_q", "x_scale"], +) +def _convrot_rotate_quantize_activation_op( + x_q: torch.Tensor, + x_scale: torch.Tensor, + x: torch.Tensor, + group_size: int, +) -> None: + module = _jit_convrot_int8_module() + module.convrot_rotate_quantize_activation(x_q, x_scale, x, group_size) + + +@register_custom_op(op_name="convrot_int8_fused_linear", mutates_args=["out"]) +def _convrot_int8_fused_linear_op( + out: torch.Tensor, + x: torch.Tensor, + weight_q: torch.Tensor, + weight_scale: torch.Tensor, + bias: torch.Tensor | None, + group_size: int, + gelu_input: bool, +) -> None: + module = _jit_convrot_int8_module() + module.convrot_int8_fused_linear( + out, x, weight_q, weight_scale, bias, group_size, gelu_input + ) + + +@register_custom_op(op_name="convrot_int8_linear_prequant", mutates_args=["out"]) +def _convrot_int8_linear_prequant_op( + out: torch.Tensor, + xq: torch.Tensor, + xs: torch.Tensor, + weight_q: torch.Tensor, + weight_scale: torch.Tensor, + bias: torch.Tensor | None, + group_size: int, +) -> None: + module = _jit_convrot_int8_module() + module.convrot_int8_linear_prequant( + out, xq, xs, weight_q, weight_scale, bias, group_size + ) + + +@debug_kernel_api +def convrot_rotate_quantize_activation( + x: torch.Tensor, group_size: int +) -> tuple[torch.Tensor, torch.Tensor]: + """Group Hadamard rotation + per-row INT8 quant of a contiguous BF16 [M, K]. + + Returns (x_q int8 [M, K], x_scale float32 [M]). Also the offline transform + for a [N, K] weight, yielding (weight_q, weight_scale). + """ + x_q = torch.empty(x.shape, dtype=torch.int8, device=x.device) + x_scale = torch.empty(x.shape[0], dtype=torch.float32, device=x.device) + if torch.compiler.is_compiling(): + _convrot_rotate_quantize_activation_op(x_q, x_scale, x, group_size) + else: + _jit_convrot_int8_module().convrot_rotate_quantize_activation( + x_q, x_scale, x, group_size + ) + return x_q, x_scale + + +@debug_kernel_api +def convrot_int8_fused_linear( + x: torch.Tensor, + weight_q: torch.Tensor, + weight_scale: torch.Tensor, + bias: torch.Tensor | None, + group_size: int, + *, + out: torch.Tensor | None = None, + gelu_input: bool = False, +) -> torch.Tensor: + """BF16 [M, K] x int8 [N, K] -> BF16 [M, N]; x is rotated and quantized in-kernel. + + ``out`` must be a contiguous BF16 [M, N] when given (a row slice of a larger + buffer qualifies). ``gelu_input`` applies GELU(tanh) to x inside the rotate + kernel, bitwise equal to the eager ``F.gelu(x, approximate="tanh")`` first. + """ + if out is None: + out = torch.empty( + (x.shape[0], weight_q.shape[0]), dtype=torch.bfloat16, device=x.device + ) + if torch.compiler.is_compiling(): + _convrot_int8_fused_linear_op( + out, x, weight_q, weight_scale, bias, group_size, gelu_input + ) + else: + _jit_convrot_int8_module().convrot_int8_fused_linear( + out, x, weight_q, weight_scale, bias, group_size, gelu_input + ) + return out + + +@debug_kernel_api +def convrot_int8_linear_prequant( + xq: torch.Tensor, + xs: torch.Tensor, + weight_q: torch.Tensor, + weight_scale: torch.Tensor, + bias: torch.Tensor | None, + group_size: int, + *, + out: torch.Tensor | None = None, +) -> torch.Tensor: + """GEMM + dequant on (xq int8 [M, K], xs float32 [M]) from + convrot_rotate_quantize_activation; bitwise equal to the fused op.""" + if out is None: + out = torch.empty( + (xq.shape[0], weight_q.shape[0]), dtype=torch.bfloat16, device=xq.device + ) + if torch.compiler.is_compiling(): + _convrot_int8_linear_prequant_op( + out, xq, xs, weight_q, weight_scale, bias, group_size + ) + else: + _jit_convrot_int8_module().convrot_int8_linear_prequant( + out, xq, xs, weight_q, weight_scale, bias, group_size + ) + return out diff --git a/python/sglang/multimodal_gen/.agents/skills/sglang-diffusion-performance/SKILL.md b/python/sglang/multimodal_gen/.agents/skills/sglang-diffusion-performance/SKILL.md index 7abdf1ee94..c248c241a9 100644 --- a/python/sglang/multimodal_gen/.agents/skills/sglang-diffusion-performance/SKILL.md +++ b/python/sglang/multimodal_gen/.agents/skills/sglang-diffusion-performance/SKILL.md @@ -172,7 +172,7 @@ These options **trade output quality** for speed or VRAM savings. Results will d | **Progressive Resolution** | `--progressive-mode dct_rewind --progressive-levels N --progressive-delta D` | Runs early denoising at lower latent resolution, then spectrally upsamples and switches to the target resolution. | Model- and schedule-dependent | Approximate and pipeline-specific. Keep the switch schedule fixed and compare detail, composition, and temporal stability. | | **Causal KV-Cache Quantization** | `--kv-cache-quant int4\|int2` plus optional `--kv-cache-quant-*` controls | Compresses completed causal KV-cache chunks with Quant-VideoGen PRQ while keeping the mutable/current chunk and recent chunks in BF16. | Primarily a long-session memory saving | Currently limited to LingBot World realtime causal serving; requires `quant-videogen`. INT4 is the starting point; INT2 saves more memory with more error. It quantizes cache state, not checkpoint weights. | | **Quantized Models (Nunchaku / SVDQuant)** | `--enable-svdquant --transformer-weights-path ` + optional `--quantization-precision int4\|nvfp4`, `--quantization-rank 32` | W4A4-style quantization via [Nunchaku](https://nunchaku.tech). Reduces DiT weight memory by ~4x. Precision/rank can be auto-inferred from weight filename or set explicitly. | ~1.5–2x compute speedup | Lossy quantization; quality depends on rank and precision. Requires pre-quantized weights. Ampere (SM8x) or SM12x only (no Hopper SM90). Higher rank = better quality but more memory. | -| **GGUF Transformer** | `--transformer-weights-path ` | Loads a community-quantized DiT from one `.gguf`; other components stay on the base model. **Shrinks the checkpoint, not the peak VRAM** — offload already bounds peak, so reach for this when the *download* or the host RAM offload pins is the problem (MiniMax-H3 17.5 vs 61.7 GiB), not when VRAM is. For a 24 GB card `kitchen_int8` is the faster option if you can afford the full BF16 checkpoint on disk. | None; expect a small slowdown from per-step dequantization | Lossy (4-bit families ~0.997 cosine vs BF16). CUDA only, `--tp-size 1`, no FSDP, no LoRA, no `--quantization`, no `--enable-svdquant`, and mutually exclusive with the H3 AdaLN cache/online flags — each rejected at startup. Validated on MiniMax-H3 `fl2va` Q4_K_M, 1 GPU. | +| **GGUF Transformer** | `--transformer-weights-path ` | Loads a community-quantized DiT from one `.gguf`; other components stay on the base model. **Shrinks the checkpoint, not the peak VRAM** — offload already bounds peak, so reach for this when the *download* or the host RAM offload pins is the problem (MiniMax-H3 17.5 vs 61.7 GiB), not when VRAM is. For a 24 GB card `convrot_int8` is the faster option if you can afford the full BF16 checkpoint on disk. | None; expect a small slowdown from per-step dequantization | Lossy (4-bit families ~0.997 cosine vs BF16). CUDA only, `--tp-size 1`, no FSDP, no LoRA, no `--quantization`, no `--enable-svdquant`, and mutually exclusive with the H3 AdaLN cache/online flags — each rejected at startup. Validated on MiniMax-H3 `fl2va` Q4_K_M, 1 GPU. | | **Pre-quantized Transformer Override** | `--transformer-path ` / `--transformer-weights-path ` | Load a quantized transformer component or raw transformer weights. For converted ModelOpt FP8/NVFP4 directories, prefer `--transformer-path`; use `--transformer-weights-path` for weight-only artifacts the model loader expects. | ~1.3–1.5x compute (dtype dependent) | Requires a validated quantized transformer override, such as one produced by the ModelOpt helper tools. Quality is usually slightly worse than BF16 and depends on the format, fallback layers, and calibration scope. | | **Component Precision Override** | `--dit-precision fp16`, `--vae-precision fp16\|bf16` | On-the-fly dtype conversion for individual components. E.g. convert a BF16 model to FP16 at load time, or run VAE in BF16 instead of FP32. | Reduces memory; FP16 can be faster on some GPUs | May affect numerical stability. VAE is FP32 by default for accuracy; lowering it is lossy. DiT defaults to BF16. | | **Fewer Inference Steps** | `--num-inference-steps N` (sampling param) | Reduces the number of denoising steps. Fewer steps = faster. | Linear speedup | Quality degrades with too few steps. Model-dependent optimal range. | diff --git a/python/sglang/multimodal_gen/envs.py b/python/sglang/multimodal_gen/envs.py index 0a0ee5b0e9..af3fe5413e 100644 --- a/python/sglang/multimodal_gen/envs.py +++ b/python/sglang/multimodal_gen/envs.py @@ -47,6 +47,7 @@ if TYPE_CHECKING: SGLANG_DIFFUSION_DISABLE_AUTO_RESIDENCY: bool = False SGLANG_DIFFUSION_MINIMAX_H3_ADALN_GPU_PLANS: int = 64 SGLANG_DIFFUSION_MINIMAX_H3_ADALN_FP32: bool = False + SGLANG_DIFFUSION_CONVROT_INT8_BACKEND: str = "auto" SGLANG_DIFFUSION_MINIMAX_H3_PDD_HEADS: str | None = None SGLANG_DIFFUSION_FLUX3_NATTEN_BACKEND: str | None = None SGLANG_DIFFUSION_CFG_GATE_STEP: float = 1.0 @@ -329,6 +330,13 @@ environment_variables: dict[str, Callable[[], Any]] = { "SGLANG_DIFFUSION_CFG_GATE_STEP": _lazy_float( "SGLANG_DIFFUSION_CFG_GATE_STEP", 1.0 ), + # Kernel backend for convrot_int8 (online or serialized ConvRot INT8): + # "auto" prefers SGLang's JIT-compiled convrot_int8 ops where they build + # and run (CC 9.0, 10.0, 12.0, 12.1 with nvcc) and falls back to + # comfy_kitchen; "jit" or "comfy_kitchen" forces one backend. + "SGLANG_DIFFUSION_CONVROT_INT8_BACKEND": _lazy_str( + "SGLANG_DIFFUSION_CONVROT_INT8_BACKEND", "auto" + ), # Path to a Parallel Decoding Distillation head stack (one fused output head # per denoise step, produced by fuse_minimax_h3_pdd_heads.py). Set only when serving a # PDD-distilled checkpoint; an ordinary run leaves the projection alone. diff --git a/python/sglang/multimodal_gen/runtime/layers/quantization/__init__.py b/python/sglang/multimodal_gen/runtime/layers/quantization/__init__.py index ef6f0621d8..d35c7d9b2e 100644 --- a/python/sglang/multimodal_gen/runtime/layers/quantization/__init__.py +++ b/python/sglang/multimodal_gen/runtime/layers/quantization/__init__.py @@ -11,10 +11,13 @@ from sglang.multimodal_gen.runtime.layers.quantization.bitsandbytes import ( from sglang.multimodal_gen.runtime.layers.quantization.configs.base_config import ( QuantizationConfig, ) -from sglang.multimodal_gen.runtime.layers.quantization.configs.kitchen_int8_config import ( - KitchenInt8Config, +from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config import ( + ConvRotInt8Config, ) from sglang.multimodal_gen.runtime.layers.quantization.fp8 import Fp8Config +from sglang.multimodal_gen.runtime.layers.quantization.method_names import ( + canonical_quantization_method, +) from sglang.multimodal_gen.runtime.layers.quantization.modelopt_fp8 import ( ModelOptFp8Config as ModelOptFp8DiffusionConfig, ) @@ -40,6 +43,8 @@ QuantizationMethods = Literal[ "mxfp8", "mxfp4", "mxfp4_npu", + "convrot_int8", + # deprecated alias of convrot_int8 "kitchen_int8", ] @@ -57,7 +62,7 @@ _CUSTOMIZED_METHOD_TO_QUANT_CONFIG = { "mxfp4": Mxfp4Config, "mxfp8": MXFP8Config, "mxfp4_npu": NPUMXFP4Config, - "kitchen_int8": KitchenInt8Config, + "convrot_int8": ConvRotInt8Config, } @@ -90,6 +95,7 @@ def register_quantization_config(quantization: str): def get_quantization_config(quantization: str) -> type[QuantizationConfig]: + quantization = canonical_quantization_method(quantization) if quantization not in QUANTIZATION_METHODS: raise ValueError(f"Invalid quantization method: {quantization}") diff --git a/python/sglang/multimodal_gen/runtime/layers/quantization/configs/convrot_int8_config.py b/python/sglang/multimodal_gen/runtime/layers/quantization/configs/convrot_int8_config.py new file mode 100644 index 0000000000..b4aa4ea69d --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/layers/quantization/configs/convrot_int8_config.py @@ -0,0 +1,377 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Config for convrot_int8: online or serialized ConvRot INT8 on the JIT ops or comfy_kitchen. + +Both backends implement the same quantization (arXiv:2512.03673): group-wise +regular Hadamard rotation, per-row dynamic INT8 activations and per-output- +channel INT8 weights, with interchangeable weight and scale tensors. ``auto`` +picks SGLang's JIT-compiled fused ``convrot_int8`` ops where they build and run +(CC 9.0, 10.0, 12.0 and 12.1 with a CUDA toolchain; group size 64, 128, 256 or +512; output width a multiple of 8) and ``comfy_kitchen.int8_linear`` otherwise +(Turing and later; group size 16, 64 or 256; the load fails with an install hint +when the package is missing). Only a layer whose output width the JIT epilogue +cannot store falls to comfy_kitchen or, without it, stays BF16. +""" + +from __future__ import annotations + +import functools +from typing import Any + +import torch + +from sglang.kernels.ops.gemm.convrot_int8 import GROUP_SIZES as JIT_GROUP_SIZES +from sglang.multimodal_gen import envs +from sglang.multimodal_gen.runtime.layers.linear import ( + ColumnParallelLinear, + LinearBase, + UnquantizedLinearMethod, +) +from sglang.multimodal_gen.runtime.layers.quantization.comfy_int8 import ( + ComfyInt8EmbeddingMethod, + is_comfy_int8_embedding, +) +from sglang.multimodal_gen.runtime.layers.quantization.configs.base_config import ( + QuantizationConfig, + QuantizeMethodBase, +) +from sglang.multimodal_gen.runtime.layers.vocab_parallel_embedding import ( + VocabParallelEmbedding, +) +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger +from sglang.srt.layers.quantization.utils import is_layer_skipped + +logger = init_logger(__name__) + +COMFY_KITCHEN = "comfy_kitchen" +JIT = "jit" +BACKENDS = ("auto", COMFY_KITCHEN, JIT) +# Group widths each backend's kernels are instantiated for. +BACKEND_GROUP_SIZES = {COMFY_KITCHEN: (16, 64, 256), JIT: JIT_GROUP_SIZES} +_SUPPORTED_GROUP_SIZES = tuple( + sorted({*BACKEND_GROUP_SIZES[COMFY_KITCHEN], *JIT_GROUP_SIZES}) +) + + +def _jit_available() -> bool: + from sglang.multimodal_gen.runtime.layers.quantization.convrot_int8_jit import ( + jit_convrot_unavailable_reason, + ) + + reason = jit_convrot_unavailable_reason() + if reason is not None: + _log_jit_fallback(reason) + return reason is None + + +@functools.cache +def _log_jit_fallback(reason: str) -> None: + # Once per process: the refused-GPU reason (for example CC 10.3) or a failed + # build would otherwise only surface with an explicit jit backend. + logger.info( + "convrot_int8: JIT backend unavailable (%s); auto uses comfy_kitchen", + reason, + ) + + +def _comfy_kitchen_available() -> bool: + from sglang.multimodal_gen.runtime.layers.quantization.convrot_int8_comfy_kitchen import ( + comfy_kitchen_available, + ) + + return comfy_kitchen_available() + + +class ConvRotInt8Config(QuantizationConfig): + """Dispatch online or serialized Comfy ConvRot layers to a kernel backend.""" + + def __init__( + self, + group_size: int = 256, + ignored_layers: list[str] | None = None, + packed_modules_mapping: dict[str, list[str]] | None = None, + layer_markers: dict[str, dict[str, Any]] | None = None, + backend: str | None = None, + ) -> None: + super().__init__() + if group_size not in _SUPPORTED_GROUP_SIZES: + raise ValueError( + f"convrot_int8 group_size must be one of {_SUPPORTED_GROUP_SIZES}, " + f"got {group_size}" + ) + if backend is None: + backend = envs.SGLANG_DIFFUSION_CONVROT_INT8_BACKEND + if backend not in BACKENDS: + raise ValueError( + f"convrot_int8 backend must be one of {BACKENDS}, got {backend!r}" + ) + if backend != "auto" and group_size not in BACKEND_GROUP_SIZES[backend]: + raise ValueError( + f"convrot_int8 backend {backend} has kernels for group sizes " + f"{BACKEND_GROUP_SIZES[backend]}, got {group_size}" + ) + self.group_size = group_size + self.backend = backend + self.ignored_layers = ignored_layers or [] + self.packed_modules_mapping = packed_modules_mapping or {} + self.layer_markers = layer_markers + self.is_checkpoint_int8_serialized = layer_markers is not None + self.checkpoint_uses_native_qkv_layout = self.is_checkpoint_int8_serialized + self._serialized_group_sizes: dict[str, int] = {} + if layer_markers is not None: + for prefix, marker in layer_markers.items(): + if is_comfy_int8_embedding(marker): + continue + if marker.get("format") != "int8_tensorwise": + raise ValueError( + f"Unsupported Comfy INT8 format for {prefix!r}: " + f"{marker.get('format')!r}" + ) + if marker.get("convrot") is not True: + raise ValueError( + f"Serialized convrot_int8 layer {prefix!r} must set " + "convrot=true" + ) + marker_group_size = marker.get("convrot_groupsize") + if marker_group_size not in BACKEND_GROUP_SIZES[COMFY_KITCHEN]: + raise ValueError( + f"Serialized convrot_int8 layer {prefix!r} must declare " + f"convrot_groupsize in {BACKEND_GROUP_SIZES[COMFY_KITCHEN]}, " + f"got {marker_group_size!r}" + ) + self._serialized_group_sizes[prefix] = marker_group_size + elif backend == "auto" and self.resolve_backend() is None: + raise ValueError( + f"convrot_int8: no backend on this machine serves group size " + f"{group_size} (the JIT backend needs CC 9.0/10.0/12.0/12.1, a CUDA " + f"toolchain and one of {BACKEND_GROUP_SIZES[JIT]}; comfy_kitchen serves " + f"{BACKEND_GROUP_SIZES[COMFY_KITCHEN]})" + ) + # Logged per backend at the end of loading: a silent fallback to BF16 + # or to the slower backend looks exactly like a slow kernel. + self.selected: list[str] = [] + # Embeddings count as selected for the loader's marker check but never + # report back through note_quantized / note_loaded. + self.selected_embeddings: list[str] = [] + self.selected_by_backend: dict[str, list[str]] = { + COMFY_KITCHEN: [], + JIT: [], + } + self.skipped: list[str] = [] + self._processed = 0 + self._quantized_bytes = 0 + + @classmethod + def get_name(cls) -> str: + return "convrot_int8" + + @classmethod + def get_supported_act_dtypes(cls) -> list[torch.dtype]: + return [torch.bfloat16, torch.float16] + + @classmethod + def get_min_capability(cls) -> int: + # INT8 tensor cores land on Turing (comfy_kitchen); the JIT backend + # checks its own table when it is selected. + return 75 + + @classmethod + def get_config_filenames(cls) -> list[str]: + return [] + + @classmethod + def from_config(cls, config: dict[str, Any]) -> ConvRotInt8Config: + return cls( + group_size=cls.get_from_keys_or(config, ["group_size"], 256), + ignored_layers=cls.get_from_keys_or(config, ["ignored_layers"], None), + backend=cls.get_from_keys_or(config, ["backend"], None), + ) + + def resolve_backend(self, group_size: int | None = None) -> str | None: + """Backend serving ``group_size`` (default: the online group size), or + None when the requested backend has no kernel for it. ``auto`` prefers + the JIT ops where they build and run on this GPU; an explicit backend + is never substituted.""" + group_size = self.group_size if group_size is None else group_size + if self.backend == "auto": + if group_size in BACKEND_GROUP_SIZES[JIT] and _jit_available(): + return JIT + if group_size in BACKEND_GROUP_SIZES[COMFY_KITCHEN]: + return COMFY_KITCHEN + return None + return self.backend if group_size in BACKEND_GROUP_SIZES[self.backend] else None + + def require_comfy_kitchen(self, reason: str) -> None: + """Serve every layer with comfy_kitchen; an explicit jit request is refused.""" + if self.backend == JIT: + raise ValueError( + f"convrot_int8 backend jit {reason}; use backend " + "comfy_kitchen or TP and/or sequence parallelism instead" + ) + if self.group_size not in BACKEND_GROUP_SIZES[COMFY_KITCHEN]: + raise ValueError( + f"convrot_int8 backend comfy_kitchen has no kernel for group size " + f"{self.group_size}, and the jit backend {reason}" + ) + if self.backend == "auto": + logger.info( + "convrot_int8: using the comfy_kitchen backend because %s", reason + ) + self.backend = COMFY_KITCHEN + + def _backend_for(self, *, group_size: int, out_size: int) -> str | None: + backend = self.resolve_backend(group_size) + if backend == JIT and out_size % 8: + # The JIT GEMM epilogue stores 8 BF16 outputs at a time. + if ( + self.backend == "auto" + and group_size in BACKEND_GROUP_SIZES[COMFY_KITCHEN] + and _comfy_kitchen_available() + ): + return COMFY_KITCHEN + return None + return backend + + def _no_kernel_reason(self, *, group_size: int, out_size: int) -> str: + comfy = ( + "installed" + if _comfy_kitchen_available() + else "not installed (pip install comfy-kitchen)" + ) + return ( + f"ConvRot group {group_size}, {out_size} outputs; backend {self.backend!r}; " + f"jit serves group sizes {BACKEND_GROUP_SIZES[JIT]} with an " + f"output width that is a multiple of 8, comfy_kitchen serves group sizes " + f"{BACKEND_GROUP_SIZES[COMFY_KITCHEN]} and is {comfy}" + ) + + def _select( + self, *, prefix: str, backend: str, group_size: int, serialized: bool + ) -> QuantizeMethodBase: + self.selected.append(prefix) + self.selected_by_backend[backend].append(prefix) + if backend == JIT: + from sglang.multimodal_gen.runtime.layers.quantization.convrot_int8_jit import ( + ConvRotInt8JitLinearMethod, + ) + + return ConvRotInt8JitLinearMethod( + self, group_size=group_size, is_checkpoint_serialized=serialized + ) + from sglang.multimodal_gen.runtime.layers.quantization.convrot_int8_comfy_kitchen import ( + ConvRotInt8ComfyKitchenLinearMethod, + ) + + return ConvRotInt8ComfyKitchenLinearMethod( + self, group_size=group_size, is_checkpoint_serialized=serialized + ) + + def get_quant_method( + self, layer: torch.nn.Module, prefix: str + ) -> QuantizeMethodBase | None: + if isinstance(layer, VocabParallelEmbedding) and self.quantizes_embedding( + prefix + ): + self.selected.append(prefix) + self.selected_embeddings.append(prefix) + return ComfyInt8EmbeddingMethod( + tensorwise=bool(self.layer_markers[prefix].get("_is_tensorwise_scalar")) + ) + if not isinstance(layer, LinearBase): + return None + # A column-parallel layer shards its output; every rank sees the same + # per-shard width, so the decision below is rank-consistent. + out_size = ( + layer.output_size_per_partition + if isinstance(layer, ColumnParallelLinear) + else layer.output_size + ) + if self.layer_markers is not None: + marker_group_size = self._serialized_group_sizes.get(prefix) + if marker_group_size is None: + return UnquantizedLinearMethod() + if layer.input_size % marker_group_size: + raise ValueError( + f"Serialized convrot_int8 layer {prefix!r} has input size " + f"{layer.input_size}, which is not divisible by its " + f"ConvRot group size {marker_group_size}" + ) + backend = self._backend_for(group_size=marker_group_size, out_size=out_size) + if backend is None: + raise ValueError( + f"Serialized convrot_int8 layer {prefix!r} has no kernel: " + + self._no_kernel_reason( + group_size=marker_group_size, out_size=out_size + ) + ) + return self._select( + prefix=prefix, + backend=backend, + group_size=marker_group_size, + serialized=True, + ) + if is_layer_skipped( + prefix, self.ignored_layers, fused_mapping=self.packed_modules_mapping + ): + self.skipped.append(prefix) + return UnquantizedLinearMethod() + # The rotation partitions the input dim into fixed-size groups, so a + # layer whose input does not divide evenly simply stays in BF16 rather + # than failing the whole model. H3's adaln projections (in=2688) are + # the case this exists for, and they cost 0.2% of a step anyway. + if layer.input_size % self.group_size: + self.skipped.append(f"{prefix}(in={layer.input_size})") + return UnquantizedLinearMethod() + backend = self._backend_for(group_size=self.group_size, out_size=out_size) + if backend is None: + self.skipped.append(f"{prefix}(out={out_size})") + return UnquantizedLinearMethod() + return self._select( + prefix=prefix, backend=backend, group_size=self.group_size, serialized=False + ) + + def note_quantized(self, saved_bytes: int) -> None: + self._processed += 1 + self._quantized_bytes += saved_bytes + if self._processed == len(self.selected) - len(self.selected_embeddings): + logger.info( + "convrot_int8: quantized %d linear layers (%.2f GiB of BF16 weights " + "-> %.2f GiB INT8; jit %d, comfy_kitchen %d), left %d in BF16", + self._processed, + self._quantized_bytes / 1024**3, + self._quantized_bytes / 2 / 1024**3, + len(self.selected_by_backend[JIT]), + len(self.selected_by_backend[COMFY_KITCHEN]), + len(self.skipped), + ) + logger.debug("convrot_int8: layers left in BF16: %s", self.skipped) + + def note_loaded(self) -> None: + """A serialized layer's INT8 weights are in place; logs the backend split once.""" + self._processed += 1 + if self._processed == len(self.selected) - len(self.selected_embeddings): + logger.info( + "convrot_int8: loaded %d serialized INT8 linear layers " + "(jit %d, comfy_kitchen %d)", + self._processed, + len(self.selected_by_backend[JIT]), + len(self.selected_by_backend[COMFY_KITCHEN]), + ) + + def get_scaled_act_names(self) -> list[str]: + return [] + + def quantizes_embedding(self, prefix: str) -> bool: + return self.layer_markers is not None and is_comfy_int8_embedding( + self.layer_markers.get(prefix) + ) + + def supports_input_partition( + self, prefix: str, input_size_per_partition: int + ) -> bool: + group_size = self.group_size + if self.layer_markers is not None: + marker_group_size = self._serialized_group_sizes.get(prefix) + if marker_group_size is None: + return True + group_size = marker_group_size + return input_size_per_partition % group_size == 0 diff --git a/python/sglang/multimodal_gen/runtime/layers/quantization/configs/kitchen_int8_config.py b/python/sglang/multimodal_gen/runtime/layers/quantization/configs/kitchen_int8_config.py deleted file mode 100644 index 8f86297eb0..0000000000 --- a/python/sglang/multimodal_gen/runtime/layers/quantization/configs/kitchen_int8_config.py +++ /dev/null @@ -1,189 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -"""Config for online or serialized INT8 ConvRot via comfy_kitchen.""" - -from __future__ import annotations - -from typing import Any - -import torch - -from sglang.multimodal_gen.runtime.layers.linear import UnquantizedLinearMethod -from sglang.multimodal_gen.runtime.layers.quantization.comfy_int8 import ( - ComfyInt8EmbeddingMethod, - is_comfy_int8_embedding, -) -from sglang.multimodal_gen.runtime.layers.quantization.configs.base_config import ( - QuantizationConfig, - QuantizeMethodBase, -) -from sglang.multimodal_gen.runtime.layers.vocab_parallel_embedding import ( - VocabParallelEmbedding, -) -from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger -from sglang.srt.layers.quantization.utils import is_layer_skipped - -logger = init_logger(__name__) - -_SUPPORTED_GROUP_SIZES = (16, 64, 256) - - -class KitchenInt8Config(QuantizationConfig): - """Dispatch online quantization or serialized Comfy ConvRot layers.""" - - def __init__( - self, - group_size: int = 256, - ignored_layers: list[str] | None = None, - packed_modules_mapping: dict[str, list[str]] | None = None, - layer_markers: dict[str, dict[str, Any]] | None = None, - ) -> None: - super().__init__() - if group_size not in _SUPPORTED_GROUP_SIZES: - raise ValueError( - f"kitchen_int8 group_size must be one of {_SUPPORTED_GROUP_SIZES}, " - f"got {group_size}" - ) - self.group_size = group_size - self.ignored_layers = ignored_layers or [] - self.packed_modules_mapping = packed_modules_mapping or {} - self.layer_markers = layer_markers - self.is_checkpoint_int8_serialized = layer_markers is not None - self.checkpoint_uses_native_qkv_layout = self.is_checkpoint_int8_serialized - self._serialized_group_sizes: dict[str, int] = {} - if layer_markers is not None: - for prefix, marker in layer_markers.items(): - if is_comfy_int8_embedding(marker): - continue - if marker.get("format") != "int8_tensorwise": - raise ValueError( - f"Unsupported Comfy INT8 format for {prefix!r}: " - f"{marker.get('format')!r}" - ) - if marker.get("convrot") is not True: - raise ValueError( - f"Serialized kitchen_int8 layer {prefix!r} must set " - "convrot=true" - ) - marker_group_size = marker.get("convrot_groupsize") - if marker_group_size not in _SUPPORTED_GROUP_SIZES: - raise ValueError( - f"Serialized kitchen_int8 layer {prefix!r} must declare " - f"convrot_groupsize in {_SUPPORTED_GROUP_SIZES}, got " - f"{marker_group_size!r}" - ) - self._serialized_group_sizes[prefix] = marker_group_size - # Which layers actually got quantized is worth stating plainly in the - # log: a silent fallback to BF16 looks exactly like a slow kernel. - self.selected: list[str] = [] - self.skipped: list[str] = [] - self._processed = 0 - self._quantized_bytes = 0 - - @classmethod - def get_name(cls) -> str: - return "kitchen_int8" - - @classmethod - def get_supported_act_dtypes(cls) -> list[torch.dtype]: - return [torch.bfloat16, torch.float16] - - @classmethod - def get_min_capability(cls) -> int: - # INT8 tensor cores land on Turing. - return 75 - - @classmethod - def get_config_filenames(cls) -> list[str]: - return [] - - @classmethod - def from_config(cls, config: dict[str, Any]) -> KitchenInt8Config: - return cls( - group_size=cls.get_from_keys_or(config, ["group_size"], 256), - ignored_layers=cls.get_from_keys_or(config, ["ignored_layers"], None), - ) - - def get_quant_method( - self, layer: torch.nn.Module, prefix: str - ) -> QuantizeMethodBase | None: - from sglang.multimodal_gen.runtime.layers.linear import LinearBase - from sglang.multimodal_gen.runtime.layers.quantization.kitchen_int8 import ( - KitchenInt8LinearMethod, - ) - - if isinstance(layer, VocabParallelEmbedding) and self.quantizes_embedding( - prefix - ): - self.selected.append(prefix) - return ComfyInt8EmbeddingMethod( - tensorwise=bool(self.layer_markers[prefix].get("_is_tensorwise_scalar")) - ) - if not isinstance(layer, LinearBase): - return None - if self.layer_markers is not None: - marker_group_size = self._serialized_group_sizes.get(prefix) - if marker_group_size is None: - return UnquantizedLinearMethod() - if layer.input_size % marker_group_size: - raise ValueError( - f"Serialized kitchen_int8 layer {prefix!r} has input size " - f"{layer.input_size}, which is not divisible by its " - f"ConvRot group size {marker_group_size}" - ) - self.selected.append(prefix) - return KitchenInt8LinearMethod( - self, - group_size=marker_group_size, - is_checkpoint_serialized=True, - ) - if is_layer_skipped( - prefix, self.ignored_layers, fused_mapping=self.packed_modules_mapping - ): - self.skipped.append(prefix) - return UnquantizedLinearMethod() - # The rotation partitions the input dim into fixed-size groups, so a - # layer whose input does not divide evenly simply stays in BF16 rather - # than failing the whole model. H3's adaln projections (in=2688) are - # the case this exists for, and they cost 0.2% of a step anyway. - if layer.input_size % self.group_size: - self.skipped.append(f"{prefix}(in={layer.input_size})") - return UnquantizedLinearMethod() - self.selected.append(prefix) - return KitchenInt8LinearMethod( - self, - group_size=self.group_size, - is_checkpoint_serialized=False, - ) - - def note_quantized(self, saved_bytes: int) -> None: - self._processed += 1 - self._quantized_bytes += saved_bytes - if self._processed == len(self.selected): - logger.info( - "kitchen_int8: quantized %d linear layers (%.2f GiB of BF16 weights " - "-> %.2f GiB INT8), left %d in BF16", - self._processed, - self._quantized_bytes / 1024**3, - self._quantized_bytes / 2 / 1024**3, - len(self.skipped), - ) - logger.debug("kitchen_int8: layers left in BF16: %s", self.skipped) - - def get_scaled_act_names(self) -> list[str]: - return [] - - def supports_input_partition( - self, prefix: str, input_size_per_partition: int - ) -> bool: - group_size = self.group_size - if self.layer_markers is not None: - marker_group_size = self._serialized_group_sizes.get(prefix) - if marker_group_size is None: - return True - group_size = marker_group_size - return input_size_per_partition % group_size == 0 - - def quantizes_embedding(self, prefix: str) -> bool: - return self.layer_markers is not None and is_comfy_int8_embedding( - self.layer_markers.get(prefix) - ) diff --git a/python/sglang/multimodal_gen/runtime/layers/quantization/configs/kitchen_w4a4_config.py b/python/sglang/multimodal_gen/runtime/layers/quantization/configs/kitchen_w4a4_config.py index 357a35e4ab..07ea4849c7 100644 --- a/python/sglang/multimodal_gen/runtime/layers/quantization/configs/kitchen_w4a4_config.py +++ b/python/sglang/multimodal_gen/runtime/layers/quantization/configs/kitchen_w4a4_config.py @@ -15,8 +15,8 @@ from sglang.multimodal_gen.runtime.layers.quantization.configs.base_config impor QuantizationConfig, QuantizeMethodBase, ) -from sglang.multimodal_gen.runtime.layers.quantization.configs.kitchen_int8_config import ( - KitchenInt8Config, +from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config import ( + ConvRotInt8Config, ) from sglang.multimodal_gen.runtime.layers.quantization.kitchen_w4a4 import ( KitchenW4A4LinearMethod, @@ -55,7 +55,7 @@ class KitchenW4A4Config(QuantizationConfig): if marker.get("format") == "int8_tensorwise" } self._int8_config = ( - KitchenInt8Config(layer_markers=int8_markers) if int8_markers else None + ConvRotInt8Config(layer_markers=int8_markers) if int8_markers else None ) for prefix, marker in layer_markers.items(): diff --git a/python/sglang/multimodal_gen/runtime/layers/quantization/kitchen_int8.py b/python/sglang/multimodal_gen/runtime/layers/quantization/convrot_int8_comfy_kitchen.py similarity index 87% rename from python/sglang/multimodal_gen/runtime/layers/quantization/kitchen_int8.py rename to python/sglang/multimodal_gen/runtime/layers/quantization/convrot_int8_comfy_kitchen.py index ae443bfc96..b33a9c1eb4 100644 --- a/python/sglang/multimodal_gen/runtime/layers/quantization/kitchen_int8.py +++ b/python/sglang/multimodal_gen/runtime/layers/quantization/convrot_int8_comfy_kitchen.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -"""INT8 weight-only-storage linear backed by comfy_kitchen's fused ConvRot kernel. +"""convrot_int8's comfy_kitchen backend: INT8 linear on comfy_kitchen's fused ConvRot kernel. On Ada (RTX 4090) INT8 is only worth doing with the right kernel: on MiniMax H3 shapes `torch._int_mm` measures 0.46-0.90x of BF16 (i.e. slower) and a Triton @@ -15,18 +15,19 @@ Comfy checkpoints instead load their INT8 weights and row scales directly. from __future__ import annotations +import functools import os import torch from torch.nn.parameter import Parameter from sglang.multimodal_gen.runtime.layers.linear import LinearMethodBase -from sglang.multimodal_gen.runtime.layers.quantization.configs.kitchen_int8_config import ( - KitchenInt8Config, +from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config import ( + ConvRotInt8Config, ) from sglang.multimodal_gen.runtime.utils.weight_attrs import set_weight_attrs -__all__ = ["KitchenInt8Config", "KitchenInt8LinearMethod"] +__all__ = ["ConvRotInt8ComfyKitchenLinearMethod"] # comfy_kitchen's dtype codes for the fused op's output. _OUT_DTYPE_CODE = {torch.float32: 0, torch.float16: 1, torch.bfloat16: 2} @@ -55,12 +56,22 @@ def _row_split(rows: int, out_features: int) -> int | None: return _MAX_ROWS_PER_CALL +@functools.cache +def comfy_kitchen_available() -> bool: + """Whether comfy_kitchen is importable and registers int8_linear.""" + try: + import comfy_kitchen # noqa: F401 + except ImportError: + return False + return hasattr(torch.ops.comfy_kitchen, "int8_linear") + + def _load_comfy_kitchen(): try: import comfy_kitchen # noqa: F401 except ImportError as exc: # pragma: no cover - depends on optional dep raise ImportError( - "kitchen_int8 quantization requires the `comfy-kitchen` package " + "convrot_int8 quantization requires the `comfy-kitchen` package " "(pip install comfy-kitchen). It is a self-contained abi3 extension " "and does not link against libtorch, so any torch version works." ) from exc @@ -71,12 +82,12 @@ def _load_comfy_kitchen(): ) -class KitchenInt8LinearMethod(LinearMethodBase): +class ConvRotInt8ComfyKitchenLinearMethod(LinearMethodBase): """Loads or creates ConvRot INT8 weights and runs the fused kernel.""" def __init__( self, - quant_config: KitchenInt8Config, + quant_config: ConvRotInt8Config, *, group_size: int, is_checkpoint_serialized: bool, @@ -101,9 +112,10 @@ class KitchenInt8LinearMethod(LinearMethodBase): # dimension the rotation groups over. if input_size_per_partition % self.group_size: raise ValueError( - f"kitchen_int8 needs input_size_per_partition " + f"convrot_int8 needs input_size_per_partition " f"({input_size_per_partition}) divisible by group_size " - f"{self.group_size}" + f"{self.group_size}; leave the layer in BF16 with " + "--quantization-ignored-layers" ) # The online path initially matches UnquantizedLinearMethod so the @@ -135,7 +147,10 @@ class KitchenInt8LinearMethod(LinearMethodBase): def process_weights_after_loading(self, layer: torch.nn.Module) -> None: weight = layer.weight.data - if self.is_checkpoint_serialized or weight.dtype == torch.int8: + if self.is_checkpoint_serialized: + self.quant_config.note_loaded() + return + if weight.dtype == torch.int8: return from comfy_kitchen.tensor.int8 import TensorWiseINT8Layout @@ -172,7 +187,7 @@ class KitchenInt8LinearMethod(LinearMethodBase): out_code = _OUT_DTYPE_CODE.get(x.dtype) if out_code is None: raise ValueError( - f"kitchen_int8 does not support activation dtype {x.dtype}" + f"convrot_int8 does not support activation dtype {x.dtype}" ) # The kernel takes 2D activations; callers may pass [..., K]. diff --git a/python/sglang/multimodal_gen/runtime/layers/quantization/convrot_int8_jit.py b/python/sglang/multimodal_gen/runtime/layers/quantization/convrot_int8_jit.py new file mode 100644 index 0000000000..5e4469246f --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/layers/quantization/convrot_int8_jit.py @@ -0,0 +1,384 @@ +# SPDX-License-Identifier: Apache-2.0 +"""convrot_int8's JIT backend: INT8 W8A8 linear on the fused ConvRot ops. + +One op takes a BF16 activation and performs the group-wise Hadamard rotation, +dynamic per-row INT8 quantization, INT8 GEMM, dequantization and bias add +without materializing the intermediates. Weights receive the same rotation and +per-output-channel quantization once, after loading a stock BF16 checkpoint; a +serialized Comfy ConvRot INT8 checkpoint loads its INT8 weights and row scales +directly, the same tensors comfy_kitchen consumes. + +The ops live in ``sglang.kernels.ops.gemm.convrot_int8`` and are compiled for +the local GPU on first use (nvcc, about a minute once per machine; cached +afterwards). A missing toolchain or a failed build makes this backend +unavailable, and ``auto`` then takes comfy_kitchen. + +Two variants of that op are exposed to the model code as helpers below: a +shared-input form that rotates and quantizes an activation once for several +linears consuming it (a q/k/v trio), and a gelu-input form that applies +GELU(tanh) inside the rotate kernel for an FFN down-projection. Both are +bitwise identical to the plain op on the equivalent eager input. +""" + +from __future__ import annotations + +import functools +import time +from collections.abc import Sequence + +import torch +from torch.nn.parameter import Parameter + +from sglang.kernels.ops.gemm.convrot_int8 import ( + convrot_int8_fused_linear, + convrot_int8_linear_prequant, + convrot_int8_local_capability, + convrot_int8_supported_capabilities, + convrot_rotate_quantize_activation, + load_convrot_int8_module, +) +from sglang.multimodal_gen.runtime.layers.linear import ( + ColumnParallelLinear, + LinearBase, + LinearMethodBase, + ReplicatedLinear, + RowParallelLinear, +) +from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config import ( + ConvRotInt8Config, +) +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger +from sglang.multimodal_gen.runtime.utils.weight_attrs import set_weight_attrs +from sglang.srt.environ import envs + +logger = init_logger(__name__) + +__all__ = [ + "ConvRotInt8JitLinearMethod", + "REFUSED_CAPABILITY_REASONS", + "apply_convrot_int8_gelu_input", + "apply_convrot_int8_shared_input", + "apply_convrot_int8_shared_input_out", + "convrot_int8_fuses_gelu_input", + "convrot_int8_shares_input", + "jit_convrot_available", + "jit_convrot_unavailable_reason", +] + +# Parts the ops deliberately leave out, with the reason shown at load time. The +# supported table itself lives with the ops (convrot_int8_supported_capabilities); +# nothing here duplicates it. +REFUSED_CAPABILITY_REASONS: dict[tuple[int, int], str] = { + (10, 3): ( + "Blackwell Ultra cuts INT8 tensor-core throughput to a fraction of its " + "BF16 rate, so W8A8 INT8 would be a slowdown there; use FP8 or NVFP4 " + "quantization on this GPU" + ), +} + + +def _unsupported_capability_reason(capability: tuple[int, int]) -> str: + major, minor = capability + reason = REFUSED_CAPABILITY_REASONS.get( + capability, "the convrot_int8 ops carry no code for it" + ) + supported = ", ".join( + f"{a}.{b}" for a, b in sorted(convrot_int8_supported_capabilities()) + ) + return f"CC {major}.{minor} is not supported: {reason} (supported: {supported})" + + +@functools.cache +def jit_convrot_unavailable_reason() -> str | None: + """Why the JIT backend cannot run here, or None when it can. + + Builds the ops on the way (once per machine; cached afterwards), so a + missing CUDA toolchain or a failed build shows up here as a reason instead + of failing the first quantized layer. + """ + if not torch.cuda.is_available(): + return "no CUDA device" + capability = convrot_int8_local_capability() + if capability is None: + return "not a CUDA device" + if capability not in convrot_int8_supported_capabilities(): + return _unsupported_capability_reason(capability) + logger.info( + "convrot_int8: loading the JIT ops for CC %d.%d (the first use on a " + "machine compiles them, about a minute)", + *capability, + ) + started = time.monotonic() + try: + load_convrot_int8_module() + except Exception as exc: # nvcc / ninja missing, or the build itself failed + if envs.SGLANG_CRASH_ON_JIT_COMPILE.get(): + raise + logger.warning("convrot_int8: the JIT build failed: %s", exc) + detail = str(exc).strip().splitlines() + return ( + f"the JIT build of the convrot_int8 ops failed " + f"({type(exc).__name__}: {detail[0] if detail else 'no message'}; " + "the compiler output is in the log above)" + ) + elapsed = time.monotonic() - started + # A cache hit loads in well under a second; anything longer was a build. + if elapsed > 5: + logger.info( + "convrot_int8: compiled the JIT ops for CC %d.%d in %.0f s " + "(cached for later runs)", + *capability, + elapsed, + ) + return None + + +def jit_convrot_available() -> bool: + """Whether the JIT convrot ops run (and built) on the current GPU.""" + return jit_convrot_unavailable_reason() is None + + +def _load_jit_kernel() -> None: + reason = jit_convrot_unavailable_reason() + if reason is not None: + raise RuntimeError(f"convrot_int8 backend jit is unavailable: {reason}") + + +def _as_rows(x: torch.Tensor) -> torch.Tensor: + # The ops take BF16 only. FP16 activations (--dit-precision fp16) are + # cast on the way in and back on the way out; the BF16 rounding is an + # order of magnitude below the INT8 quantization error. + if x.dtype == torch.float16: + x = x.to(torch.bfloat16) + elif x.dtype != torch.bfloat16: + raise ValueError( + f"convrot_int8 backend jit does not support activation dtype {x.dtype}" + ) + return x.reshape(-1, x.shape[-1]).contiguous() + + +def _like_input(out: torch.Tensor, x: torch.Tensor) -> torch.Tensor: + out = out.reshape(*x.shape[:-1], out.shape[-1]) + return out if out.dtype == x.dtype else out.to(x.dtype) + + +class ConvRotInt8JitLinearMethod(LinearMethodBase): + """Loads or creates ConvRot INT8 weights and runs the JIT fused ops.""" + + def __init__( + self, + quant_config: ConvRotInt8Config, + *, + group_size: int, + is_checkpoint_serialized: bool, + ) -> None: + self.quant_config = quant_config + self.group_size = group_size + self.is_checkpoint_serialized = is_checkpoint_serialized + _load_jit_kernel() + + def create_weights( + self, + layer: torch.nn.Module, + input_size_per_partition: int, + output_partition_sizes: list[int], + input_size: int, + output_size: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ) -> None: + # get_quant_method screened the unsharded input size and the per-shard + # output width, so these fire only under TP > 1, where a row-parallel + # layer splits the very dimension the rotation groups over. + if input_size_per_partition % self.group_size: + raise ValueError( + f"convrot_int8 needs input_size_per_partition " + f"({input_size_per_partition}) divisible by group_size " + f"{self.group_size}; leave the layer in BF16 with " + "--quantization-ignored-layers" + ) + if sum(output_partition_sizes) % 8: + raise ValueError( + f"convrot_int8 backend jit needs the output size per " + f"partition ({sum(output_partition_sizes)}) to be a multiple of 8" + ) + # The online path initially matches UnquantizedLinearMethod so the + # source weights load in BF16 before quantization. Serialized weights + # allocate their final INT8 storage immediately. + weight = Parameter( + torch.empty( + sum(output_partition_sizes), + input_size_per_partition, + dtype=(torch.int8 if self.is_checkpoint_serialized else params_dtype), + ), + requires_grad=False, + ) + set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0}) + layer.register_parameter("weight", weight) + set_weight_attrs(weight, extra_weight_attrs) + if self.is_checkpoint_serialized: + # Comfy stores the per-row scale as [N, 1]; the ops read N contiguous + # floats and accept that shape unchanged. + weight_scale = Parameter( + torch.empty(sum(output_partition_sizes), 1, dtype=torch.float32), + requires_grad=False, + ) + set_weight_attrs(weight_scale, {"output_dim": 0}) + set_weight_attrs(weight_scale, extra_weight_attrs) + layer.register_parameter("weight_scale", weight_scale) + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + weight = layer.weight.data + if self.is_checkpoint_serialized: + self.quant_config.note_loaded() + elif weight.dtype != torch.int8: + # Quantization runs on CUDA; a layer staged on CPU is round-tripped + # on its own, so an offloaded model never holds more than one BF16 + # layer on the GPU during load. + home = weight.device + weight_q, weight_scale = convrot_rotate_quantize_activation( + weight.to("cuda", non_blocking=True).to(torch.bfloat16), + self.group_size, + ) + layer.weight = Parameter(weight_q.to(home), requires_grad=False) + layer.register_parameter( + "weight_scale", + Parameter(weight_scale.to(home), requires_grad=False), + ) + self.quant_config.note_quantized(weight.numel() * weight.element_size()) + del weight_q, weight_scale + torch.cuda.empty_cache() + # The ops add a BF16 bias; an FP16 pipeline loads FP16 parameters. A + # deferred bias (skip_bias_add) never reaches the ops and keeps its dtype. + if ( + layer.bias is not None + and layer.bias.dtype != torch.bfloat16 + and not layer.skip_bias_add + ): + layer.bias = Parameter( + layer.bias.data.to(torch.bfloat16), requires_grad=False + ) + + def apply( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: torch.Tensor | None = None, + ) -> torch.Tensor: + out = convrot_int8_fused_linear( + _as_rows(x), + layer.weight, + layer.weight_scale, + bias, + self.group_size, + ) + return _like_input(out=out, x=x) + + +def _forward_is_plain_apply(layer: LinearBase) -> bool: + # True only when layer(x) is exactly quant_method.apply(layer, x, layer.bias): + # no collective, no deferred bias, no rank-dependent bias. + if layer.skip_bias_add: + return False + if isinstance(layer, ReplicatedLinear): + return True + if isinstance(layer, ColumnParallelLinear): + return not layer.gather_output + if isinstance(layer, RowParallelLinear): + return layer.tp_size == 1 + return False + + +def _is_jit_int8(layer: torch.nn.Module) -> bool: + # LoRA wrappers and other non-LinearBase modules carry no quant_method. + return isinstance(layer, LinearBase) and isinstance( + layer.quant_method, ConvRotInt8JitLinearMethod + ) + + +def convrot_int8_shares_input(layers: Sequence[torch.nn.Module]) -> bool: + """Whether ``apply_convrot_int8_shared_input`` reproduces ``layer(x)`` for + every layer in ``layers``.""" + if not layers or not all( + _is_jit_int8(layer) and _forward_is_plain_apply(layer) for layer in layers + ): + return False + # One rotated input serves every layer only at one group width. + return len({layer.quant_method.group_size for layer in layers}) == 1 + + +def apply_convrot_int8_shared_input( + x: torch.Tensor, layers: Sequence[LinearBase] +) -> list[torch.Tensor]: + """``[layer(x)[0] for layer in layers]`` with ``x`` rotated and quantized once. + + Bitwise identical to applying each layer on its own; see + ``convrot_int8_linear_prequant`` in ``sglang.kernels.ops.gemm.convrot_int8``. + """ + group_size = layers[0].quant_method.group_size + x_q, x_scale = convrot_rotate_quantize_activation(_as_rows(x), group_size) + outs = [] + for layer in layers: + out = convrot_int8_linear_prequant( + x_q, + x_scale, + layer.weight, + layer.weight_scale, + layer.bias, + group_size, + ) + outs.append(_like_input(out=out, x=x)) + return outs + + +def apply_convrot_int8_shared_input_out( + x: torch.Tensor, layers: Sequence[LinearBase], outs: Sequence[torch.Tensor] +) -> None: + """Writes ``layer(x)[0]`` into ``out`` for each (layer, out) pair with ``x`` + rotated and quantized once; ``out`` must be a contiguous BF16 slice shaped + like ``layer(x)[0]``, so ``x`` must be BF16 as well (an FP16 ``x`` would + have ``layer(x)`` return FP16; use ``apply_convrot_int8_shared_input`` for + it). Bitwise identical to ``apply_convrot_int8_shared_input`` and to + ``layer(x)``; see ``convrot_int8_linear_prequant`` in + ``sglang.kernels.ops.gemm.convrot_int8``. + """ + if x.dtype != torch.bfloat16: + raise ValueError( + "apply_convrot_int8_shared_input_out writes BF16 outputs, so its " + f"input must be BF16, got {x.dtype}" + ) + group_size = layers[0].quant_method.group_size + x_q, x_scale = convrot_rotate_quantize_activation(_as_rows(x), group_size) + for layer, out in zip(layers, outs, strict=True): + # view() rather than reshape(): a copy here would silently drop the write. + convrot_int8_linear_prequant( + x_q, + x_scale, + layer.weight, + layer.weight_scale, + layer.bias, + group_size, + out=out.view(-1, out.shape[-1]), + ) + + +def convrot_int8_fuses_gelu_input(layer: torch.nn.Module) -> bool: + """Whether ``apply_convrot_int8_gelu_input`` reproduces + ``layer(F.gelu(x, approximate="tanh"))[0]``.""" + return _is_jit_int8(layer) and _forward_is_plain_apply(layer) + + +def apply_convrot_int8_gelu_input(layer: LinearBase, x: torch.Tensor) -> torch.Tensor: + """``layer(F.gelu(x, approximate="tanh"))[0]`` as one op, bitwise identical + to the eager GELU followed by the layer for a BF16 ``x`` (an FP16 ``x`` is + cast to BF16 before the GELU); see ``convrot_int8_fused_linear`` in + ``sglang.kernels.ops.gemm.convrot_int8``.""" + out = convrot_int8_fused_linear( + _as_rows(x), + layer.weight, + layer.weight_scale, + layer.bias, + layer.quant_method.group_size, + gelu_input=True, + ) + return _like_input(out=out, x=x) diff --git a/python/sglang/multimodal_gen/runtime/layers/quantization/method_names.py b/python/sglang/multimodal_gen/runtime/layers/quantization/method_names.py new file mode 100644 index 0000000000..5f28fa31da --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/layers/quantization/method_names.py @@ -0,0 +1,22 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Quantization method names and their deprecated aliases.""" + +from __future__ import annotations + +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) + +# Deprecated public name -> current name. +QUANTIZATION_METHOD_ALIASES: dict[str, str] = {"kitchen_int8": "convrot_int8"} + + +def canonical_quantization_method(name: str) -> str: + """Map a deprecated method name to its current name, warning once per alias.""" + canonical = QUANTIZATION_METHOD_ALIASES.get(name) + if canonical is None: + return name + logger.warning_once( + f"quantization method {name!r} is a deprecated alias of {canonical!r}" + ) + return canonical diff --git a/python/sglang/multimodal_gen/runtime/layers/quantization/modelopt_quant.py b/python/sglang/multimodal_gen/runtime/layers/quantization/modelopt_quant.py index 6d6c7b48fe..6ed273bf8c 100755 --- a/python/sglang/multimodal_gen/runtime/layers/quantization/modelopt_quant.py +++ b/python/sglang/multimodal_gen/runtime/layers/quantization/modelopt_quant.py @@ -18,8 +18,8 @@ from sglang.multimodal_gen.runtime.layers.quantization.configs.base_config impor QuantizationConfig, QuantizeMethodBase, ) -from sglang.multimodal_gen.runtime.layers.quantization.configs.kitchen_int8_config import ( - KitchenInt8Config, +from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config import ( + ConvRotInt8Config, ) from sglang.multimodal_gen.runtime.models.parameter import ( ModelWeightParameter, @@ -328,7 +328,7 @@ class ModelOptFp4Config(ModelOptQuantConfig): self.swap_weight_nibbles = swap_weight_nibbles self.checkpoint_weight_scale_layout = checkpoint_weight_scale_layout self.checkpoint_uses_comfy_quantization = checkpoint_uses_comfy_quantization - self._comfy_int8_config: KitchenInt8Config | None = None + self._comfy_int8_config: ConvRotInt8Config | None = None self._comfy_fp8_config: ComfyFp8Config | None = None def set_comfy_layer_markers(self, layer_markers: dict[str, dict[str, Any]]) -> None: @@ -346,7 +346,7 @@ class ModelOptFp4Config(ModelOptQuantConfig): if marker.get("format") == "int8_tensorwise" } self._comfy_int8_config = ( - KitchenInt8Config(layer_markers=int8_markers) if int8_markers else None + ConvRotInt8Config(layer_markers=int8_markers) if int8_markers else None ) fp8_markers = { prefix: marker diff --git a/python/sglang/multimodal_gen/runtime/loader/component_loaders/text_encoder_loader.py b/python/sglang/multimodal_gen/runtime/loader/component_loaders/text_encoder_loader.py index cfa7555ecc..0ef27fb542 100644 --- a/python/sglang/multimodal_gen/runtime/loader/component_loaders/text_encoder_loader.py +++ b/python/sglang/multimodal_gen/runtime/loader/component_loaders/text_encoder_loader.py @@ -34,8 +34,8 @@ from sglang.multimodal_gen.runtime.layers.quantization.comfy_nvfp4 import ( from sglang.multimodal_gen.runtime.layers.quantization.configs.base_config import ( QuantizationConfig, ) -from sglang.multimodal_gen.runtime.layers.quantization.configs.kitchen_int8_config import ( - KitchenInt8Config, +from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config import ( + ConvRotInt8Config, ) from sglang.multimodal_gen.runtime.layers.quantization.configs.kitchen_w4a4_config import ( KitchenW4A4Config, @@ -107,7 +107,7 @@ from sglang.srt.model_loader.checkpoint_quantization import ( logger = init_logger(__name__) -_ONLINE_ENCODER_QUANTIZATIONS = frozenset({"fp8", "kitchen_int8", "mxfp4"}) +_ONLINE_ENCODER_QUANTIZATIONS = frozenset({"fp8", "convrot_int8", "mxfp4"}) _TRANSFORMERS_ENCODER_ONLY_CLASSES = { "T5EncoderModel": transformers.T5EncoderModel, @@ -398,12 +398,13 @@ def _require_quantized_encoder_layers( ( ComfyFp8Config, ComfyNvfp4Config, - KitchenInt8Config, + ConvRotInt8Config, KitchenW4A4Config, KitchenW4A8Config, ), ): - expected = set(quant_config.layer_markers) + # Online convrot_int8 carries no markers; there is nothing to consume. + expected = set(quant_config.layer_markers or ()) selected = set(quant_config.selected) elif isinstance(quant_config, QuantoInt8Config): expected = quant_config.layer_prefixes @@ -744,7 +745,7 @@ class TextEncoderLoader(OnlineQuantizationComponentLoader): if quant_config is not None and not isinstance(quant_config, GGUFConfig): postprocess_device: torch.device | None = local_torch_device if isinstance(quant_config, (ComfyNvfp4Config, QuantoInt8Config)) or ( - isinstance(quant_config, KitchenInt8Config) + isinstance(quant_config, ConvRotInt8Config) and quant_config.is_checkpoint_int8_serialized ): postprocess_device = None diff --git a/python/sglang/multimodal_gen/runtime/loader/component_loaders/transformer_loader.py b/python/sglang/multimodal_gen/runtime/loader/component_loaders/transformer_loader.py index a9fa3993bc..d8bcf5468e 100644 --- a/python/sglang/multimodal_gen/runtime/loader/component_loaders/transformer_loader.py +++ b/python/sglang/multimodal_gen/runtime/loader/component_loaders/transformer_loader.py @@ -13,6 +13,9 @@ from sglang.multimodal_gen.runtime.layers.attention.selector import ( get_component_forced_attn_backend, get_global_forced_attn_backend, ) +from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config import ( + ConvRotInt8Config, +) from sglang.multimodal_gen.runtime.loader.component_loaders.component_loader import ( OnlineQuantizationComponentLoader, ) @@ -379,6 +382,15 @@ class TransformerLoader(OnlineQuantizationComponentLoader): "AutoRound checkpoints do not support diffusion FSDP inference; " "use TP and/or sequence parallelism instead" ) + if ( + use_fsdp + and isinstance(quant_spec.quant_config, ConvRotInt8Config) + and not quant_spec.quant_config.is_checkpoint_int8_serialized + ): + # Only the comfy_kitchen backend has run under FSDP. + quant_spec.quant_config.require_comfy_kitchen( + "has not been validated with diffusion FSDP inference" + ) if quant_spec.gguf_file is not None: logger.info( @@ -476,7 +488,7 @@ class TransformerLoader(OnlineQuantizationComponentLoader): runtime_quant_config=quant_spec.runtime_quant_config, quantized_cpu_load_supported=( quant_spec.gguf_file is not None - or quant_spec.is_serialized_kitchen_int8 + or quant_spec.is_convrot_int8 or quant_spec.is_serialized_kitchen_w4a8 ), ) diff --git a/python/sglang/multimodal_gen/runtime/loader/transformer_load_utils.py b/python/sglang/multimodal_gen/runtime/loader/transformer_load_utils.py index d6f13135e1..5a650e45ce 100644 --- a/python/sglang/multimodal_gen/runtime/loader/transformer_load_utils.py +++ b/python/sglang/multimodal_gen/runtime/loader/transformer_load_utils.py @@ -19,8 +19,8 @@ from torch import nn from sglang.multimodal_gen.configs.models.dits.base import DiTArchConfig from sglang.multimodal_gen.runtime.layers.quantization import QuantizationConfig -from sglang.multimodal_gen.runtime.layers.quantization.configs.kitchen_int8_config import ( - KitchenInt8Config, +from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config import ( + ConvRotInt8Config, ) from sglang.multimodal_gen.runtime.layers.quantization.configs.kitchen_w4a4_config import ( KitchenW4A4Config, @@ -175,9 +175,13 @@ class TransformerQuantLoadSpec: return _get_quant_config_name(self.quant_config) == "comfy_fp8" @property - def is_serialized_kitchen_int8(self) -> bool: + def is_convrot_int8(self) -> bool: + return isinstance(self.quant_config, ConvRotInt8Config) + + @property + def is_serialized_convrot_int8(self) -> bool: return ( - isinstance(self.quant_config, KitchenInt8Config) + isinstance(self.quant_config, ConvRotInt8Config) and self.quant_config.is_checkpoint_int8_serialized ) @@ -193,7 +197,7 @@ class TransformerQuantLoadSpec: def uses_comfy_layer_markers(self) -> bool: return ( self.is_comfy_fp8 - or self.is_serialized_kitchen_int8 + or self.is_serialized_convrot_int8 or self.is_serialized_kitchen_w4a4 or self.is_serialized_kitchen_w4a8 or ( @@ -816,10 +820,8 @@ def _needs_device_weight_postprocess( quant_name = _get_quant_config_name(quant_config) if quant_name in ("modelopt_fp8", "comfy_fp8", "auto-round", "mxfp8"): return True - if quant_name == "kitchen_int8": - assert isinstance(quant_config, KitchenInt8Config) - return not quant_config.is_checkpoint_int8_serialized - + # convrot_int8 is deliberately absent: both backends quantize each layer + # through their own CUDA round trip, so an offloaded DiT stays on the host. serialized_flag_by_quant_name = { "fp8": "is_checkpoint_fp8_serialized", "mxfp4": "is_checkpoint_mxfp4_serialized", @@ -1047,13 +1049,13 @@ def _resolve_quant_config( "such as owner/repo:Q4_K_M)." ) - # Online-quant convention: for `fp8`, `mxfp4` and `kitchen_int8`, a + # Online-quant convention: for `fp8`, `mxfp4` and `convrot_int8`, a # no-arg QuantizationConfig() selects the post-load path -- weights # load in source dtype and are quantized in # process_weights_after_loading. quant_cls = get_quantization_config(server_args.quantization) quant_kwargs = {} - if server_args.quantization in {"fp8", "mxfp4", "kitchen_int8"}: + if server_args.quantization in {"fp8", "mxfp4", "convrot_int8"}: quant_kwargs["ignored_layers"] = getattr( server_args, "quantization_ignored_layers", None ) diff --git a/python/sglang/multimodal_gen/runtime/models/dits/common.py b/python/sglang/multimodal_gen/runtime/models/dits/common.py index 70cba347a2..7629acad0b 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/common.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/common.py @@ -2,10 +2,16 @@ from __future__ import annotations +from collections.abc import Sequence from typing import Any import torch +from sglang.multimodal_gen.runtime.layers.quantization.convrot_int8_jit import ( + apply_convrot_int8_shared_input, + convrot_int8_shares_input, +) + def modulate( x: torch.Tensor, @@ -22,6 +28,14 @@ def modulate( return x * (1 + scale.unsqueeze(1)) + shift.unsqueeze(1) +def _project_shared_input( + x: torch.Tensor, layers: Sequence[torch.nn.Module] +) -> list[torch.Tensor]: + if convrot_int8_shares_input(layers): + return apply_convrot_int8_shared_input(x=x, layers=layers) + return [layer(x)[0] for layer in layers] + + def get_qkv_projections( attn: Any, hidden_states: torch.Tensor, @@ -53,9 +67,9 @@ def get_qkv_projections( if make_contiguous: query, key, value = [t.contiguous() for t in (query, key, value)] else: - query, _ = attn.to_q(hidden_states) - key, _ = attn.to_k(hidden_states) - value, _ = attn.to_v(hidden_states) + query, key, value = _project_shared_input( + x=hidden_states, layers=(attn.to_q, attn.to_k, attn.to_v) + ) encoder_query = encoder_key = encoder_value = None if encoder_hidden_states is not None and attn.added_kv_proj_dim is not None: @@ -67,8 +81,9 @@ def get_qkv_projections( t.contiguous() for t in (encoder_query, encoder_key, encoder_value) ] else: - encoder_query, _ = attn.add_q_proj(encoder_hidden_states) - encoder_key, _ = attn.add_k_proj(encoder_hidden_states) - encoder_value, _ = attn.add_v_proj(encoder_hidden_states) + encoder_query, encoder_key, encoder_value = _project_shared_input( + x=encoder_hidden_states, + layers=(attn.add_q_proj, attn.add_k_proj, attn.add_v_proj), + ) return query, key, value, encoder_query, encoder_key, encoder_value diff --git a/python/sglang/multimodal_gen/runtime/models/dits/qwen_image.py b/python/sglang/multimodal_gen/runtime/models/dits/qwen_image.py index 9d7357dd01..d1bf04f3dd 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/qwen_image.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/qwen_image.py @@ -73,6 +73,7 @@ from sglang.multimodal_gen.runtime.layers.linear import ( MergedColumnParallelLinear, ReplicatedLinear, RowParallelLinear, + UnquantizedLinearMethod, apply_unquantized_linear, ) from sglang.multimodal_gen.runtime.layers.quantization.configs.base_config import ( @@ -82,6 +83,13 @@ from sglang.multimodal_gen.runtime.layers.quantization.configs.nunchaku_config i NunchakuConfig, is_nunchaku_available, ) +from sglang.multimodal_gen.runtime.layers.quantization.convrot_int8_jit import ( + apply_convrot_int8_gelu_input, + apply_convrot_int8_shared_input, + apply_convrot_int8_shared_input_out, + convrot_int8_fuses_gelu_input, + convrot_int8_shares_input, +) from sglang.multimodal_gen.runtime.layers.quantization.modelopt_quant import ( ModelOptFp4LinearMethod, ModelOptFp8LinearMethod, @@ -720,7 +728,161 @@ class QwenEmbedLayer3DRope(nn.Module): return freqs.clone().contiguous() +def _joint_qkv_layers( + attn: "QwenImageCrossAttention", +) -> tuple[nn.Module, nn.Module, nn.Module, nn.Module, nn.Module, nn.Module]: + return ( + attn.to_q, + attn.to_k, + attn.to_v, + attn.add_q_proj, + attn.add_k_proj, + attn.add_v_proj, + ) + + +def _joint_qkv_operands_match( + *, + attn: "QwenImageCrossAttention", + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor, +) -> bool: + dtype = hidden_states.dtype + if encoder_hidden_states.dtype != dtype: + return False + layers = _joint_qkv_layers(attn) + # A LoRA wrapper forwards .weight to its base layer and adds its delta in + # forward, so the bare addmm below would silently drop it. + if not all(isinstance(layer, ColumnParallelLinear) for layer in layers): + return False + if attn.separate_convrot_qkv_proj: + # apply_convrot_int8_shared_input_out stores BF16 only; FP16 streams + # take apply_convrot_int8_shared_input, which casts at the op boundary. + return dtype == torch.bfloat16 + # The out= GEMMs below take no part in autocast and do no promotion, so + # every operand must already be in the dtype F.linear would compute in. + device_type = hidden_states.device.type + if ( + torch.is_autocast_enabled(device_type) + and torch.get_autocast_dtype(device_type) != dtype + ): + return False + return all( + layer.weight.dtype == dtype + and (layer.bias is None or layer.bias.dtype == dtype) + for layer in layers + ) + + +def _use_joint_qkv_buffers( + *, + attn: "QwenImageCrossAttention", + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor, + masked: bool, +) -> bool: + # Batch 1 keeps each slice contiguous for the out= writes and the in-place + # QK-norm; SP>1 and masked sequences never take join_seqs. + return ( + (attn.separate_unquantized_qkv_proj or attn.separate_convrot_qkv_proj) + and not masked + # The out= GEMMs below reject grad-tracking operands. + and not torch.is_grad_enabled() + and hidden_states.shape[0] == 1 + and hidden_states.is_contiguous() + and encoder_hidden_states.is_contiguous() + and get_sp_world_size() <= 1 + and _joint_qkv_operands_match( + attn=attn, + hidden_states=hidden_states, + encoder_hidden_states=encoder_hidden_states, + ) + ) + + +def _project_into( + *, x: torch.Tensor, layer: ColumnParallelLinear, out: torch.Tensor +) -> None: + # The addmm F.linear dispatches for a contiguous x, so the out= write is + # bitwise identical to layer(x). + x_2d = x.reshape(-1, x.shape[-1]) + # view() rather than reshape(): a copy here would silently drop the write. + out_2d = out.view(-1, out.shape[-1]) + if layer.bias is None: + torch.mm(x_2d, layer.weight.t(), out=out_2d) + else: + torch.addmm(layer.bias, x_2d, layer.weight.t(), out=out_2d) + + +def _project_qkv_into_joint_buffers( + *, + attn: "QwenImageCrossAttention", + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor, + seq_len_txt: int, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + # Three [B, txt + img, local_inner_dim] buffers, text rows first. + batch_size, seq_len_img, _ = hidden_states.shape + inner_dim_local = attn.local_num_heads * attn.head_dim + shape = (batch_size, seq_len_txt + seq_len_img, inner_dim_local) + bufs = tuple( + torch.empty(shape, dtype=hidden_states.dtype, device=hidden_states.device) + for _ in range(3) + ) + to_q, to_k, to_v, add_q_proj, add_k_proj, add_v_proj = _joint_qkv_layers(attn) + img_layers = (to_q, to_k, to_v) + txt_layers = (add_q_proj, add_k_proj, add_v_proj) + if attn.separate_convrot_qkv_proj: + apply_convrot_int8_shared_input_out( + x=hidden_states, + layers=img_layers, + outs=[buf[:, seq_len_txt:] for buf in bufs], + ) + apply_convrot_int8_shared_input_out( + x=encoder_hidden_states, + layers=txt_layers, + outs=[buf[:, :seq_len_txt] for buf in bufs], + ) + return bufs + for buf, img_layer, txt_layer in zip(bufs, img_layers, txt_layers): + _project_into(x=hidden_states, layer=img_layer, out=buf[:, seq_len_txt:]) + _project_into( + x=encoder_hidden_states, layer=txt_layer, out=buf[:, :seq_len_txt] + ) + return bufs + + +def _joint_qkv_head_views( + bufs: tuple[torch.Tensor, torch.Tensor, torch.Tensor], + *, + seq_len_txt: int, + num_heads: int, + head_dim: int, +) -> tuple[torch.Tensor, ...]: + # (img_q, img_k, img_v, txt_q, txt_k, txt_v), each a [B, S, H, D] view. + img_views = tuple( + buf[:, seq_len_txt:].unflatten(-1, (num_heads, head_dim)) for buf in bufs + ) + txt_views = tuple( + buf[:, :seq_len_txt].unflatten(-1, (num_heads, head_dim)) for buf in bufs + ) + return (*img_views, *txt_views) + + +def _sync_into_buffer(*, value: torch.Tensor, buf_view: torch.Tensor) -> None: + # In-place QK-norm/RoPE aliases value to buf_view; only the non-fused + # fallback returns a fresh tensor that must be copied back. + # Dynamo cannot trace data_ptr(); a self-aliased copy_ is a no-op anyway. + if torch.compiler.is_compiling() or value.data_ptr() != buf_view.data_ptr(): + buf_view.copy_(value) + + class QwenImageCrossAttention(nn.Module): + # Per-instance values are set in __init__; the class defaults keep forward() + # valid on modules built around it (object.__new__ in the epilogue tests). + separate_unquantized_qkv_proj = False + separate_convrot_qkv_proj = False + def __init__( self, dim: int, # query_dim @@ -776,6 +938,8 @@ class QwenImageCrossAttention(nn.Module): ) self.local_num_heads = self.num_heads // tp_size self._unquantized_added_qkv_is_packed = False + self.separate_unquantized_qkv_proj = False + self.separate_convrot_qkv_proj = False if self.use_fused_qkv: # Use fused QKV projection for nunchaku quantization @@ -860,6 +1024,24 @@ class QwenImageCrossAttention(nn.Module): quant_config=quant_config, prefix=f"{prefix}.add_v_proj", ) + # Six plain linears can write straight into joint text-image + # buffers (see _use_joint_qkv_buffers); the fused-epilogue + # path consumes packed projection strides instead. + self.separate_unquantized_qkv_proj = ( + not self.use_fused_qkv + and not self.use_fused_qkv_epilogue + and all( + isinstance(layer.quant_method, UnquantizedLinearMethod) + for layer in _joint_qkv_layers(self) + ) + ) + # Six ConvRot INT8 linears share one rotated+quantized input per + # stream and also write straight into the joint buffers. + self.separate_convrot_qkv_proj = ( + not self.use_fused_qkv + and not self.use_fused_qkv_epilogue + and convrot_int8_shares_input(_joint_qkv_layers(self)) + ) if context_pre_only is not None and not context_pre_only: self.to_add_out = RowParallelLinear( @@ -961,7 +1143,35 @@ class QwenImageCrossAttention(nn.Module): # Rows of tail padding inside THIS rank's text chunk (sp_shard meta). sp_txt_pad = _attn_mask_meta_local_pad(attn_mask_meta) - if self._unquantized_added_qkv_is_packed and not qwen_image_added_qkv_active( + # Joint [text, image] destination buffers take precedence over the + # packed added-QKV GEMM, which in turn precedes per-layer projections. + joint_qkv_bufs = None + if _use_joint_qkv_buffers( + attn=self, + hidden_states=hidden_states, + encoder_hidden_states=encoder_hidden_states, + masked=attn_mask is not None or encoder_hidden_states_mask is not None, + ): + joint_qkv_bufs = _project_qkv_into_joint_buffers( + attn=self, + hidden_states=hidden_states, + encoder_hidden_states=encoder_hidden_states, + seq_len_txt=seq_len_txt, + ) + ( + img_query, + img_key, + img_value, + txt_query, + txt_key, + txt_value, + ) = _joint_qkv_head_views( + joint_qkv_bufs, + seq_len_txt=seq_len_txt, + num_heads=self.local_num_heads, + head_dim=self.head_dim, + ) + elif self._unquantized_added_qkv_is_packed and not qwen_image_added_qkv_active( self ): img_query, img_key, img_value, _, _, _ = _get_qkv_projections( @@ -972,6 +1182,20 @@ class QwenImageCrossAttention(nn.Module): txt_query, txt_key, txt_value = self._get_added_qkv_projections( encoder_hidden_states ) + elif self.separate_convrot_qkv_proj and convrot_int8_shares_input( + _joint_qkv_layers(self) + ): + # Same rotate-once projections as the joint-buffer path, into fresh + # tensors when the joint layout is not applicable (SP, masks, B>1). + # Re-checked per call: LoRA mounting swaps the projections for + # wrappers that add their delta in forward. + img_query, img_key, img_value = apply_convrot_int8_shared_input( + x=hidden_states, layers=(self.to_q, self.to_k, self.to_v) + ) + txt_query, txt_key, txt_value = apply_convrot_int8_shared_input( + x=encoder_hidden_states, + layers=(self.add_q_proj, self.add_k_proj, self.add_v_proj), + ) else: ( img_query, @@ -993,14 +1217,15 @@ class QwenImageCrossAttention(nn.Module): else: img_complex = txt_complex = None - # Reshape for multi-head attention - img_query = img_query.unflatten(-1, (self.local_num_heads, self.head_dim)) - img_key = img_key.unflatten(-1, (self.local_num_heads, self.head_dim)) - img_value = img_value.unflatten(-1, (self.local_num_heads, self.head_dim)) + if joint_qkv_bufs is None: + # Reshape for multi-head attention + img_query = img_query.unflatten(-1, (self.local_num_heads, self.head_dim)) + img_key = img_key.unflatten(-1, (self.local_num_heads, self.head_dim)) + img_value = img_value.unflatten(-1, (self.local_num_heads, self.head_dim)) - txt_query = txt_query.unflatten(-1, (self.local_num_heads, self.head_dim)) - txt_key = txt_key.unflatten(-1, (self.local_num_heads, self.head_dim)) - txt_value = txt_value.unflatten(-1, (self.local_num_heads, self.head_dim)) + txt_query = txt_query.unflatten(-1, (self.local_num_heads, self.head_dim)) + txt_key = txt_key.unflatten(-1, (self.local_num_heads, self.head_dim)) + txt_value = txt_value.unflatten(-1, (self.local_num_heads, self.head_dim)) img_cache = txt_cache = None if image_rotary_emb is not None: @@ -1117,6 +1342,17 @@ class QwenImageCrossAttention(nn.Module): joint_query, joint_key, joint_value = joint_qkv elif seg_qkv is not None: joint_query, joint_key, joint_value = seg_qkv + elif joint_qkv_bufs is not None: + # The buffers already hold the [text, image] layout: Q/K/V were + # projected into them and QK-norm/RoPE ran in place on views. + joint_query, joint_key, joint_value = ( + buf.unflatten(-1, (self.local_num_heads, self.head_dim)) + for buf in joint_qkv_bufs + ) + _sync_into_buffer(value=img_query, buf_view=joint_query[:, seq_len_txt:]) + _sync_into_buffer(value=img_key, buf_view=joint_key[:, seq_len_txt:]) + _sync_into_buffer(value=txt_query, buf_view=joint_query[:, :seq_len_txt]) + _sync_into_buffer(value=txt_key, buf_view=joint_key[:, :seq_len_txt]) elif attn_mask is not None and not sp_text_sharded: # Let the eager attention break point pack directly from the text # and image segments. Materializing three dense joint tensors here @@ -1269,6 +1505,13 @@ class QwenImageFeedForward(nn.Module): def forward_with_bias( self, hidden_states: torch.Tensor | tuple[torch.Tensor, torch.Tensor] ) -> tuple[torch.Tensor, Optional[torch.Tensor]]: + # Not cached at construction: LoRA mounting swaps net.2 for a wrapper. + if convrot_int8_fuses_gelu_input(self.net[2]): + # The down-projection's rotate kernel applies GELU(tanh) to its + # input bit-exactly, so the up-projection runs bare and the + # standalone activation kernel is skipped. + up, _ = self.net[0].proj(hidden_states) + return apply_convrot_int8_gelu_input(layer=self.net[2], x=up), None hidden_states = self.net[0](hidden_states) hidden_states = self.net[1](hidden_states) return self.net[2](hidden_states) diff --git a/python/sglang/multimodal_gen/runtime/server_args/server_args.py b/python/sglang/multimodal_gen/runtime/server_args/server_args.py index 75b683f989..c0cdcf0534 100644 --- a/python/sglang/multimodal_gen/runtime/server_args/server_args.py +++ b/python/sglang/multimodal_gen/runtime/server_args/server_args.py @@ -32,6 +32,9 @@ from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType from sglang.multimodal_gen.runtime.layers.quantization.configs.nunchaku_config import ( NunchakuConfig, ) +from sglang.multimodal_gen.runtime.layers.quantization.method_names import ( + canonical_quantization_method, +) from sglang.multimodal_gen.runtime.loader.utils import BYTES_PER_GB from sglang.multimodal_gen.runtime.managers.memory_managers.component_residency import ( COMPONENT_OFFLOAD, @@ -1981,10 +1984,14 @@ class ServerArgs(DisaggServerArgsMixin): ) normalized_direct_gpu_loading[component_name] = enabled self.component_direct_gpu_weight_loading = normalized_direct_gpu_loading + if self.quantization is not None: + self.quantization = canonical_quantization_method(self.quantization) normalized_quantizations: dict[str, str] = {} for component, quantization in self.component_quantizations.items(): component = str(component).strip().replace("-", "_") - quantization = str(quantization).strip().lower() + quantization = canonical_quantization_method( + str(quantization).strip().lower() + ) if not component or not quantization: raise ValueError( "Component quantization entries require a component and method" @@ -2789,7 +2796,11 @@ class ServerArgs(DisaggServerArgsMixin): "auto-detected from the checkpoint config or safetensors metadata when " "possible. Use this flag to override auto-detection. " "Online (post-load) quantization from a BF16/FP16 checkpoint " - "is supported for 'fp8' and 'mxfp4'. Other methods " + "is supported for 'fp8', 'mxfp4' and 'convrot_int8' (ConvRot INT8 " + "W8A8; runs on SGLang's JIT-compiled fused ops on CC 9.0, 10.0, " + "12.0 and 12.1, else on comfy_kitchen; see " + "SGLANG_DIFFUSION_CONVROT_INT8_BACKEND; 'kitchen_int8' is a " + "deprecated alias). Other methods " "('modelopt', 'modelopt_fp8', 'modelopt_fp4', 'mxfp8', " "'mxfp4_npu', 'modelslim') require a pre-quantized checkpoint. " "Note: 'mxfp4' targets ROCm + MI350+ (gfx95x); " @@ -2803,7 +2814,8 @@ class ServerArgs(DisaggServerArgsMixin): default=ServerArgs.quantization_ignored_layers, help=( "Layer name patterns to keep unquantized during online quantization " - "(fp8/mxfp4). Each pattern is matched against the layer prefix. " + "(fp8/mxfp4/convrot_int8). Each pattern is matched against the " + "layer prefix. " "Example: --quantization-ignored-layers img_mod txt_mod to_out" ), ) diff --git a/python/sglang/multimodal_gen/runtime/utils/quantization_utils.py b/python/sglang/multimodal_gen/runtime/utils/quantization_utils.py index 29d79d49b3..cd48f0666c 100644 --- a/python/sglang/multimodal_gen/runtime/utils/quantization_utils.py +++ b/python/sglang/multimodal_gen/runtime/utils/quantization_utils.py @@ -22,8 +22,8 @@ from sglang.multimodal_gen.runtime.layers.quantization.comfy_fp8 import ComfyFp8 from sglang.multimodal_gen.runtime.layers.quantization.comfy_nvfp4 import ( ComfyNvfp4Config, ) -from sglang.multimodal_gen.runtime.layers.quantization.configs.kitchen_int8_config import ( - KitchenInt8Config, +from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config import ( + ConvRotInt8Config, ) from sglang.multimodal_gen.runtime.layers.quantization.configs.kitchen_w4a4_config import ( KitchenW4A4Config, @@ -395,7 +395,7 @@ def resolve_comfy_checkpoint_quantization( return None formats = sorted({str(marker.get("format")) for marker in layer_markers.values()}) if formats == ["int8_tensorwise"]: - return KitchenInt8Config(layer_markers=layer_markers) + return ConvRotInt8Config(layer_markers=layer_markers) if formats == ["asym_w4a8_int8"]: return KitchenW4A8Config(layer_markers) if formats == ["asym_w4a8_int8", "int8_tensorwise"]: diff --git a/python/sglang/multimodal_gen/test/unit/test_convrot_int8_backends.py b/python/sglang/multimodal_gen/test/unit/test_convrot_int8_backends.py new file mode 100644 index 0000000000..9c52ee18ba --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_convrot_int8_backends.py @@ -0,0 +1,672 @@ +"""Unit tests for convrot_int8's backend selection and its JIT backend.""" + +from types import SimpleNamespace +from unittest.mock import patch + +import pytest +import torch +import torch.nn.functional as F + +from sglang.kernels.ops.gemm.convrot_int8 import convrot_int8_supported_capabilities +from sglang.multimodal_gen.runtime.layers.linear import ( + LinearBase, + ReplicatedLinear, + UnquantizedLinearMethod, +) +from sglang.multimodal_gen.runtime.layers.lora.linear import wrap_with_lora_layer +from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config import ( + ConvRotInt8Config, +) +from sglang.multimodal_gen.runtime.layers.quantization.convrot_int8_comfy_kitchen import ( + ConvRotInt8ComfyKitchenLinearMethod, +) +from sglang.multimodal_gen.runtime.layers.quantization.convrot_int8_jit import ( + ConvRotInt8JitLinearMethod, + apply_convrot_int8_gelu_input, + apply_convrot_int8_shared_input, + apply_convrot_int8_shared_input_out, + convrot_int8_fuses_gelu_input, + convrot_int8_shares_input, +) + +_CONFIG_MODULE = ( + "sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config" +) +_LOAD_JIT_KERNEL = ( + "sglang.multimodal_gen.runtime.layers.quantization.convrot_int8_jit." + "_load_jit_kernel" +) +_LOAD_COMFY_KITCHEN = "sglang.multimodal_gen.runtime.layers.quantization.convrot_int8_comfy_kitchen._load_comfy_kitchen" + +QWEN_IMAGE_IGNORED_LAYERS = ["img_mod", "txt_mod", "txt_mlp.net.2"] + + +# Capability only: on a supported GPU a missing toolchain or a failed build must +# fail these tests, not skip them. +requires_kernel = pytest.mark.skipif( + not ( + torch.cuda.is_available() + and torch.cuda.get_device_capability() in convrot_int8_supported_capabilities() + ), + reason="needs a GPU in the convrot_int8 capability table", +) + + +def _jit_config(**kwargs) -> ConvRotInt8Config: + return ConvRotInt8Config(backend="jit", **kwargs) + + +def _method_for(config, prefix, input_size=3072): + layer = LinearBase(input_size=input_size, output_size=32) + return config.get_quant_method(layer, prefix) + + +@patch(_LOAD_JIT_KERNEL) +def test_ignored_layer_patterns_select_bf16_on_module_path_boundaries(_load): + """A pattern must match its own module path only: `txt_mlp.net.2` keeps the + text FFN down-projection in BF16 without also catching `img_mlp.net.2` or + the text FFN up-projection, and `img_mod` must not catch `img_mlp`.""" + config = _jit_config(ignored_layers=QWEN_IMAGE_IGNORED_LAYERS) + + kept_bf16 = [ + "transformer_blocks.0.img_mod.1", + "transformer_blocks.3.txt_mod.1", + "transformer_blocks.7.txt_mlp.net.2", + ] + quantized = [ + "transformer_blocks.0.img_mlp.net.2", + "transformer_blocks.7.txt_mlp.net.0.proj", + "transformer_blocks.0.attn.to_q", + "transformer_blocks.0.attn.add_k_proj", + ] + for prefix in kept_bf16: + assert isinstance(_method_for(config, prefix), UnquantizedLinearMethod) + for prefix in quantized: + assert isinstance(_method_for(config, prefix), ConvRotInt8JitLinearMethod) + assert config.skipped == kept_bf16 + assert config.selected == quantized + + +@patch(_LOAD_JIT_KERNEL) +def test_input_dim_not_divisible_by_group_stays_bf16(_load): + config = _jit_config() + method = _method_for(config, "blocks.0.mod", input_size=2688) + assert isinstance(method, UnquantizedLinearMethod) + assert config.skipped == ["blocks.0.mod(in=2688)"] + assert config.selected == [] + assert not config.supports_input_partition("blocks.0.attn.to_out", 1536 + 128) + assert config.supports_input_partition("blocks.0.attn.to_out", 1536) + + +@patch(_LOAD_JIT_KERNEL) +def test_shared_input_and_gelu_helpers_refuse_deferred_bias_and_lora(_load): + """The helpers stand in for `layer(x)`; a layer that returns its bias + separately would silently lose it, and a LoRA-wrapped projection is not a + `LinearBase` at all, so eligibility must say no to both instead of + crashing on the wrapper.""" + config = _jit_config() + plain = ReplicatedLinear(256, 8, quant_config=config, prefix="q") + deferred = ReplicatedLinear( + 256, 8, skip_bias_add=True, quant_config=config, prefix="k" + ) + bf16 = ReplicatedLinear(256, 8, quant_config=None, prefix="v") + wrapped = wrap_with_lora_layer(plain) + + assert convrot_int8_shares_input([plain, plain]) + assert not convrot_int8_shares_input([plain, deferred]) + assert not convrot_int8_shares_input([plain, bf16]) + assert not convrot_int8_shares_input([wrapped, plain]) + assert not convrot_int8_shares_input([]) + assert convrot_int8_fuses_gelu_input(plain) + assert not convrot_int8_fuses_gelu_input(deferred) + assert not convrot_int8_fuses_gelu_input(bf16) + assert not convrot_int8_fuses_gelu_input(wrapped) + + +def _quantized_layer( + config, in_features, out_features, prefix, seed, dtype=torch.bfloat16 +): + gen = torch.Generator(device="cuda").manual_seed(seed) + layer = ReplicatedLinear( + in_features, + out_features, + params_dtype=dtype, + quant_config=config, + prefix=prefix, + ).cuda() + with torch.no_grad(): + layer.weight.copy_( + torch.randn(layer.weight.shape, device="cuda", generator=gen) * 0.02 + ) + layer.bias.copy_(torch.randn(layer.bias.shape, device="cuda", generator=gen)) + layer.bias.requires_grad_(False) + reference_weight = layer.weight.detach().clone() + layer.quant_method.process_weights_after_loading(layer) + return layer, reference_weight + + +@requires_kernel +def test_quantized_layer_tracks_bf16_reference(): + config = _jit_config() + layer, weight = _quantized_layer(config, 3072, 3072, "attn.to_q", seed=0) + assert layer.weight.dtype == torch.int8 + assert layer.weight_scale.shape == (3072,) + + x = torch.randn(2, 1024, 3072, device="cuda", dtype=torch.bfloat16) + out, out_bias = layer(x) + assert out_bias is None + ref = F.linear(x, weight, layer.bias) + err = torch.linalg.vector_norm(out.float() - ref.float()) + rel_l2 = (err / torch.linalg.vector_norm(ref.float())).item() + # Arbitrary loose bound: a wrong scale or rotation lands far above it, + # W8A8 group-256 quantization noise on Gaussian data far below. + assert rel_l2 < 2e-2, rel_l2 + + +@requires_kernel +def test_shared_input_helper_is_bitwise_three_layer_calls(): + config = _jit_config() + layers = [ + _quantized_layer(config, 3072, 3072, f"attn.to_{n}", seed=i)[0] + for i, n in enumerate("qkv") + ] + x = torch.randn(1, 4096, 3072, device="cuda", dtype=torch.bfloat16) + + assert convrot_int8_shares_input(layers) + shared = apply_convrot_int8_shared_input(x=x, layers=layers) + for out, layer in zip(shared, layers, strict=True): + assert torch.equal(out, layer(x)[0]) + + +@requires_kernel +def test_shared_input_out_helper_is_bitwise_into_joint_buffer_slices(): + """The joint text-image Q/K/V path writes each projection into a row slice + of one larger buffer; the slice write must equal the allocating helper.""" + config = _jit_config() + layers = [ + _quantized_layer(config, 3072, 3072, f"attn.to_{n}", seed=10 + i)[0] + for i, n in enumerate("qkv") + ] + seq_len_txt, seq_len_img = 20, 4096 + x = torch.randn(1, seq_len_img, 3072, device="cuda", dtype=torch.bfloat16) + bufs = [ + torch.zeros( + 1, seq_len_txt + seq_len_img, 3072, device="cuda", dtype=torch.bfloat16 + ) + for _ in layers + ] + + apply_convrot_int8_shared_input_out( + x=x, layers=layers, outs=[buf[:, seq_len_txt:] for buf in bufs] + ) + for out, buf, layer in zip( + apply_convrot_int8_shared_input(x=x, layers=layers), bufs, layers, strict=True + ): + assert torch.equal(buf[:, seq_len_txt:], out) + assert torch.equal(buf[:, seq_len_txt:], layer(x)[0]) + assert not buf[:, :seq_len_txt].any() + + +@requires_kernel +def test_gelu_input_helper_is_bitwise_eager_gelu_then_layer(): + config = _jit_config() + down, _ = _quantized_layer(config, 12288, 3072, "img_mlp.net.2", seed=3) + up = torch.randn(1, 2048, 12288, device="cuda", dtype=torch.bfloat16) + + assert convrot_int8_fuses_gelu_input(down) + fused = apply_convrot_int8_gelu_input(layer=down, x=up) + eager, _ = down(F.gelu(up, approximate="tanh")) + assert torch.equal(fused, eager) + + +class _FakeTpGroup: + pass + + +def _with_tp(monkeypatch, tp_size): + import sglang.multimodal_gen.runtime.layers.linear as linear + + monkeypatch.setattr(linear, "get_tp_group", lambda: _FakeTpGroup()) + monkeypatch.setattr(linear, "get_group_rank", lambda group: 0) + monkeypatch.setattr(linear, "get_group_size", lambda group: tp_size) + + +@patch(_LOAD_JIT_KERNEL) +@pytest.mark.parametrize("tp_size,input_size", [(4, 13824), (8, 3072)]) +def test_row_parallel_shard_not_divisible_by_group_is_refused( + _load, monkeypatch, tp_size, input_size +): + from sglang.multimodal_gen.runtime.layers.linear import RowParallelLinear + + _with_tp(monkeypatch, tp_size) + config = _jit_config() + # convrot_int8 semantics: the unsharded input divides, so the layer is + # selected, and a shard that does not divide is refused at construction. + with pytest.raises(ValueError, match="divisible by group_size"): + RowParallelLinear( + input_size, + 5120, + bias=True, + quant_config=config, + prefix="blocks.0.ffn.fc_out", + ) + + +@patch(_LOAD_JIT_KERNEL) +def test_row_parallel_shard_divisible_by_group_is_quantized(_load, monkeypatch): + from sglang.multimodal_gen.runtime.layers.linear import RowParallelLinear + + _with_tp(monkeypatch, 4) + config = _jit_config() + layer = RowParallelLinear( + 12288, 3072, bias=True, quant_config=config, prefix="blocks.0.ffn.net.2" + ) + assert isinstance(layer.quant_method, ConvRotInt8JitLinearMethod) + assert config.selected == ["blocks.0.ffn.net.2"] + + +@patch(_LOAD_JIT_KERNEL) +def test_output_width_not_multiple_of_8_stays_bf16(_load): + config = _jit_config() + layer = LinearBase(input_size=4096, output_size=4) + method = config.get_quant_method(layer, "blocks.0.attn.to_gate_logits") + assert isinstance(method, UnquantizedLinearMethod) + assert config.skipped == ["blocks.0.attn.to_gate_logits(out=4)"] + + +@patch(_LOAD_JIT_KERNEL) +def test_column_parallel_shard_output_not_multiple_of_8_stays_bf16(_load, monkeypatch): + from sglang.multimodal_gen.runtime.layers.linear import ColumnParallelLinear + + _with_tp(monkeypatch, 8) + config = _jit_config() + # 32 output rows split eight ways leave 4 per rank; the epilogue stores 8 at + # a time, so the layer stays in BF16 instead of failing in the first forward. + layer = ColumnParallelLinear( + 4096, 32, bias=True, quant_config=config, prefix="blocks.0.attn.to_gate_logits" + ) + assert isinstance(layer.quant_method, UnquantizedLinearMethod) + assert config.skipped == ["blocks.0.attn.to_gate_logits(out=4)"] + + +@requires_kernel +def test_fp16_activations_are_cast_and_returned_in_fp16(): + config = _jit_config() + layer, weight = _quantized_layer(config, 3072, 3072, "attn.to_q", seed=3) + x = torch.randn(2, 64, 3072, device="cuda", dtype=torch.float16) + out, _ = layer(x) + assert out.dtype == torch.float16 + ref = F.linear(x.to(torch.bfloat16), weight, layer.bias).to(torch.float16) + err = torch.linalg.vector_norm(out.float() - ref.float()) + rel_l2 = (err / torch.linalg.vector_norm(ref.float())).item() + assert rel_l2 < 2e-2, rel_l2 + + +@requires_kernel +def test_fp16_parameters_are_cast_for_the_kernel_in_every_helper(): + """FP16 parameters (--dit-precision fp16) must run through the plain apply + and every fused helper; the out= helper must refuse an FP16 input.""" + config = _jit_config() + layers = [] + weights = [] + for i, n in enumerate("qkv"): + layer, weight = _quantized_layer( + config, 3072, 3072, f"attn.to_{n}", seed=20 + i, dtype=torch.float16 + ) + layers.append(layer) + weights.append(weight) + assert layers[0].weight.dtype == torch.int8 + assert layers[0].bias.dtype == torch.bfloat16 + x = torch.randn(1, 128, 3072, device="cuda", dtype=torch.float16) + + for layer, weight in zip(layers, weights, strict=True): + out, _ = layer(x) + assert out.dtype == torch.float16 + ref = F.linear(x.float(), weight.float(), layer.bias.float()) + err = torch.linalg.vector_norm(out.float() - ref) + rel_l2 = (err / torch.linalg.vector_norm(ref)).item() + assert rel_l2 < 2e-2, rel_l2 + + shared = apply_convrot_int8_shared_input(x=x, layers=layers) + for out, layer in zip(shared, layers, strict=True): + assert out.dtype == torch.float16 + assert torch.equal(out, layer(x)[0]) + + down, _ = _quantized_layer( + config, 12288, 3072, "img_mlp.net.2", seed=23, dtype=torch.float16 + ) + up = torch.randn(1, 128, 12288, device="cuda", dtype=torch.float16) + fused = apply_convrot_int8_gelu_input(layer=down, x=up) + assert fused.dtype == torch.float16 + # The GELU runs on the BF16-cast input inside the kernel, so the eager + # reference is the layer on GELU of that cast, not GELU in FP16. + eager, _ = down(F.gelu(up.to(torch.bfloat16), approximate="tanh")) + assert torch.equal(fused, eager.to(torch.float16)) + + outs = [ + torch.empty(1, 128, 3072, device="cuda", dtype=torch.bfloat16) for _ in layers + ] + with pytest.raises(ValueError, match="must be BF16"): + apply_convrot_int8_shared_input_out(x=x, layers=layers, outs=outs) + x_bf16 = x.to(torch.bfloat16) + apply_convrot_int8_shared_input_out(x=x_bf16, layers=layers, outs=outs) + for out, layer in zip(outs, layers, strict=True): + assert torch.equal(out, layer(x_bf16)[0]) + + +@requires_kernel +def test_lora_merge_mode_is_redirected_to_dynamic_on_int8_base(): + from sglang.multimodal_gen.runtime.pipelines_core.lora.pipeline import LoRAPipeline + + config = _jit_config() + layer, weight = _quantized_layer( + config, 3072, 3072, "transformer_blocks.0.attn.to_q", seed=4 + ) + lora = wrap_with_lora_layer(layer, lora_rank=16, lora_alpha=16, snapshot_base=True) + layers = {"transformer_blocks.0.attn.to_q": lora} + # The rotated, per-channel scaled INT8 matrix cannot take a merged delta. + assert lora.can_merge_base_weight is False + # LoRAPipeline is abstract; the decision only needs its static helper. + stand_in = SimpleNamespace(_uses_dtensor_weights=LoRAPipeline._uses_dtensor_weights) + + def decide(module_name, layers, mode): + return LoRAPipeline._should_merge_lora_for_layers( + stand_in, module_name, layers, mode + ) + + assert decide("transformer_blocks.0.attn.to_q", layers, "auto") is False + with pytest.raises(ValueError, match="use merge mode 'dynamic'"): + decide("transformer_blocks.0.attn.to_q", layers, "merge") + + gen = torch.Generator(device="cuda").manual_seed(4) + lora_a = (torch.randn(16, 3072, device="cuda", generator=gen) * 0.05).to( + torch.bfloat16 + ) + lora_b = (torch.randn(3072, 16, device="cuda", generator=gen) * 0.05).to( + torch.bfloat16 + ) + lora.set_lora_weights( + lora_a, lora_b, lora_path="test", strength=1.0, merge_weights=False + ) + x = torch.randn(8, 3072, device="cuda", dtype=torch.bfloat16, generator=gen) + out, _ = lora(x) + delta = (lora_b.float() @ lora_a.float()).to(torch.bfloat16) + ref = F.linear(x, weight + delta, layer.bias) + err = torch.linalg.vector_norm(out.float() - ref.float()) + rel_l2 = (err / torch.linalg.vector_norm(ref.float())).item() + assert rel_l2 < 2e-2, rel_l2 + + +_JIT_AVAILABLE = f"{_CONFIG_MODULE}._jit_available" +_COMFY_AVAILABLE = f"{_CONFIG_MODULE}._comfy_kitchen_available" + + +@patch(_LOAD_COMFY_KITCHEN) +@patch(_LOAD_JIT_KERNEL) +def test_auto_backend_prefers_jit_and_falls_back_to_comfy(_jit, _comfy): + """``auto`` must pick the JIT ops where they run and comfy_kitchen + otherwise; the choice is per config, not per import.""" + with ( + patch(_JIT_AVAILABLE, return_value=True), + patch(_COMFY_AVAILABLE, return_value=True), + ): + config = ConvRotInt8Config() + assert config.resolve_backend() == "jit" + method = config.get_quant_method(LinearBase(3072, 3072), "blocks.0.attn.to_q") + assert isinstance(method, ConvRotInt8JitLinearMethod) + assert config.selected_by_backend["jit"] == ["blocks.0.attn.to_q"] + with ( + patch(_JIT_AVAILABLE, return_value=False), + patch(_COMFY_AVAILABLE, return_value=True), + ): + config = ConvRotInt8Config() + assert config.resolve_backend() == "comfy_kitchen" + method = config.get_quant_method(LinearBase(3072, 3072), "blocks.0.attn.to_q") + assert isinstance(method, ConvRotInt8ComfyKitchenLinearMethod) + assert config.selected_by_backend["comfy_kitchen"] == ["blocks.0.attn.to_q"] + + +@patch(_LOAD_COMFY_KITCHEN) +@patch(_LOAD_JIT_KERNEL) +def test_auto_backend_serves_narrow_outputs_with_comfy_or_leaves_bf16(_jit, _comfy): + """The JIT epilogue stores 8 outputs at a time; a narrower layer goes + to comfy_kitchen when it is installed and stays BF16 otherwise.""" + with patch(_JIT_AVAILABLE, return_value=True): + with patch(_COMFY_AVAILABLE, return_value=True): + config = ConvRotInt8Config() + method = config.get_quant_method(LinearBase(4096, 4), "blocks.0.gate") + assert isinstance(method, ConvRotInt8ComfyKitchenLinearMethod) + assert config.selected_by_backend["comfy_kitchen"] == ["blocks.0.gate"] + with patch(_COMFY_AVAILABLE, return_value=False): + config = ConvRotInt8Config() + method = config.get_quant_method(LinearBase(4096, 4), "blocks.0.gate") + assert isinstance(method, UnquantizedLinearMethod) + assert config.skipped == ["blocks.0.gate(out=4)"] + + +@patch(_LOAD_COMFY_KITCHEN) +@patch(_LOAD_JIT_KERNEL) +def test_serialized_layers_pick_the_backend_per_marker_group_size(_jit, _comfy): + """A Comfy checkpoint may mix group sizes; 16 exists only in comfy_kitchen + while 256 runs on the JIT ops with the stored [N, 1] scale.""" + markers = { + "visual.proj": { + "format": "int8_tensorwise", + "convrot": True, + "convrot_groupsize": 16, + }, + "blocks.0.fc1": { + "format": "int8_tensorwise", + "convrot": True, + "convrot_groupsize": 256, + }, + } + with ( + patch(_JIT_AVAILABLE, return_value=True), + patch(_COMFY_AVAILABLE, return_value=True), + ): + config = ConvRotInt8Config(layer_markers=markers) + small = ReplicatedLinear( + 64, 16, bias=False, quant_config=config, prefix="visual.proj" + ) + fc1 = ReplicatedLinear( + 256, 16, bias=False, quant_config=config, prefix="blocks.0.fc1" + ) + assert isinstance(small.quant_method, ConvRotInt8ComfyKitchenLinearMethod) + assert isinstance(fc1.quant_method, ConvRotInt8JitLinearMethod) + assert fc1.quant_method.is_checkpoint_serialized + assert fc1.weight.dtype == torch.int8 and fc1.weight.shape == (16, 256) + assert fc1.weight_scale.dtype == torch.float32 + assert fc1.weight_scale.shape == (16, 1) + assert config.selected == ["visual.proj", "blocks.0.fc1"] + + +def test_explicit_backend_rejects_group_sizes_it_has_no_kernel_for(): + with pytest.raises(ValueError, match="group sizes"): + ConvRotInt8Config(group_size=16, backend="jit") + with pytest.raises(ValueError, match="group sizes"): + ConvRotInt8Config(group_size=128, backend="comfy_kitchen") + with pytest.raises(ValueError, match="backend must be one of"): + ConvRotInt8Config(backend="triton") + + +def test_require_comfy_kitchen_pins_auto_and_refuses_explicit_jit(): + with ( + patch(_JIT_AVAILABLE, return_value=True), + patch(_COMFY_AVAILABLE, return_value=True), + ): + config = ConvRotInt8Config() + config.require_comfy_kitchen("is not validated here") + assert config.resolve_backend() == "comfy_kitchen" + with pytest.raises(ValueError, match="jit is not validated here"): + ConvRotInt8Config(backend="jit").require_comfy_kitchen( + "is not validated here" + ) + + +@requires_kernel +def test_serialized_layer_on_jit_matches_online_quantization_bitwise(): + """Loading the online path's INT8 weight and fp32 [N, 1] row scale through + the serialized method must reproduce the online layer bit for bit.""" + online, _ = _quantized_layer(_jit_config(), 3072, 3072, "attn.to_q", seed=30) + markers = { + "attn.to_q": { + "format": "int8_tensorwise", + "convrot": True, + "convrot_groupsize": 256, + } + } + config = ConvRotInt8Config(layer_markers=markers, backend="jit") + serialized = ReplicatedLinear( + 3072, 3072, params_dtype=torch.bfloat16, quant_config=config, prefix="attn.to_q" + ).cuda() + assert isinstance(serialized.quant_method, ConvRotInt8JitLinearMethod) + with torch.no_grad(): + serialized.weight.copy_(online.weight) + serialized.weight_scale.copy_(online.weight_scale.view(-1, 1)) + serialized.bias.copy_(online.bias) + serialized.quant_method.process_weights_after_loading(serialized) + + x = torch.randn(2, 512, 3072, device="cuda", dtype=torch.bfloat16) + assert torch.equal(serialized(x)[0], online(x)[0]) + assert convrot_int8_shares_input([serialized, online]) + + +def test_auto_logs_why_the_jit_backend_is_unavailable(caplog): + """A refused GPU (CC 10.3) must still say why once the method is convrot_int8 + with auto backend, not only when jit is requested explicitly.""" + import logging + + from sglang.multimodal_gen.runtime.layers.quantization.configs import ( + convrot_int8_config, + ) + + convrot_int8_config._log_jit_fallback.cache_clear() + reason = ( + "CC 10.3 is not supported: Blackwell Ultra cuts INT8 tensor-core throughput" + ) + with ( + patch( + "sglang.multimodal_gen.runtime.layers.quantization.convrot_int8_jit." + "jit_convrot_unavailable_reason", + return_value=reason, + ), + patch(_COMFY_AVAILABLE, return_value=True), + caplog.at_level(logging.INFO), + ): + config = ConvRotInt8Config() + assert config.resolve_backend() == "comfy_kitchen" + assert config.resolve_backend() == "comfy_kitchen" + messages = [ + r.getMessage() + for r in caplog.records + if "JIT backend unavailable" in r.getMessage() + ] + assert len(messages) == 1 and reason in messages[0] + + +@patch(_LOAD_JIT_KERNEL) +def test_shared_input_requires_one_group_size(_jit): + """One rotated activation serves several layers only at one group width; a + serialized checkpoint may mix 64- and 256-group layers.""" + markers = { + "a": {"format": "int8_tensorwise", "convrot": True, "convrot_groupsize": 256}, + "b": {"format": "int8_tensorwise", "convrot": True, "convrot_groupsize": 64}, + "c": {"format": "int8_tensorwise", "convrot": True, "convrot_groupsize": 256}, + } + config = ConvRotInt8Config(layer_markers=markers, backend="jit") + a, b, c = ( + ReplicatedLinear(256, 16, bias=False, quant_config=config, prefix=name) + for name in "abc" + ) + assert convrot_int8_shares_input([a, c]) + assert not convrot_int8_shares_input([a, b]) + + +def test_every_no_kernel_branch_raises_instead_of_falling_back(): + """A serialized INT8 layer that no backend serves must not land in a BF16 + parameter, and an impossible auto config must fail at construction.""" + with ( + patch(_JIT_AVAILABLE, return_value=False), + patch(_COMFY_AVAILABLE, return_value=True), + ): + with pytest.raises(ValueError, match="no backend on this machine"): + ConvRotInt8Config(group_size=128) + with ( + patch(_JIT_AVAILABLE, return_value=True), + patch(_COMFY_AVAILABLE, return_value=False), + ): + with pytest.raises(ValueError, match="no kernel for group size 128"): + ConvRotInt8Config(group_size=128).require_comfy_kitchen("is pinned") + markers = { + "gate": { + "format": "int8_tensorwise", + "convrot": True, + "convrot_groupsize": 256, + } + } + config = ConvRotInt8Config(layer_markers=markers) + with pytest.raises(ValueError, match="pip install comfy-kitchen"): + config.get_quant_method(LinearBase(256, 4), "gate") + + +@patch(_LOAD_COMFY_KITCHEN) +@patch(_LOAD_JIT_KERNEL) +def test_serialized_layers_log_the_backend_split_once_loaded(_jit, _comfy, caplog): + """Serialized layers skip online quantization, so the backend split must be + reported when their weights are in place instead.""" + import logging + + markers = { + "fc1": {"format": "int8_tensorwise", "convrot": True, "convrot_groupsize": 256}, + "small": { + "format": "int8_tensorwise", + "convrot": True, + "convrot_groupsize": 16, + }, + } + with ( + patch(_JIT_AVAILABLE, return_value=True), + patch(_COMFY_AVAILABLE, return_value=True), + caplog.at_level(logging.INFO), + ): + config = ConvRotInt8Config(layer_markers=markers) + fc1 = ReplicatedLinear(256, 16, bias=False, quant_config=config, prefix="fc1") + small = ReplicatedLinear( + 64, 16, bias=False, quant_config=config, prefix="small" + ) + fc1.quant_method.process_weights_after_loading(fc1) + assert not [r for r in caplog.records if "serialized INT8" in r.getMessage()] + small.quant_method.process_weights_after_loading(small) + lines = [ + r.getMessage() for r in caplog.records if "serialized INT8" in r.getMessage() + ] + assert lines == [ + "convrot_int8: loaded 2 serialized INT8 linear layers (jit 1, comfy_kitchen 1)" + ] + + +def test_kitchen_int8_is_a_deprecated_alias_of_convrot_int8(caplog): + """Existing `--quantization kitchen_int8` commands must keep working under the + new name, with one deprecation warning per process.""" + import logging + + from sglang.multimodal_gen.runtime.layers.quantization import ( + get_quantization_config, + method_names, + ) + from sglang.multimodal_gen.runtime.utils.logging_utils import _print_warning_once + + _print_warning_once.cache_clear() + with caplog.at_level(logging.WARNING): + assert ( + method_names.canonical_quantization_method("kitchen_int8") == "convrot_int8" + ) + assert ( + method_names.canonical_quantization_method("kitchen_int8") == "convrot_int8" + ) + assert method_names.canonical_quantization_method("fp8") == "fp8" + warnings = [r for r in caplog.records if "deprecated alias" in r.getMessage()] + assert len(warnings) == 1 + assert get_quantization_config("kitchen_int8") is ConvRotInt8Config + assert ConvRotInt8Config.get_name() == "convrot_int8" diff --git a/python/sglang/multimodal_gen/test/unit/test_qwen_image_joint_qkv.py b/python/sglang/multimodal_gen/test/unit/test_qwen_image_joint_qkv.py new file mode 100644 index 0000000000..7f889ea2f7 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_qwen_image_joint_qkv.py @@ -0,0 +1,480 @@ +"""Unit tests for Qwen-Image joint text-image Q/K/V destination buffers.""" + +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +import torch +import torch.nn as nn + +from sglang.kernels.ops.gemm.convrot_int8 import convrot_int8_supported_capabilities +from sglang.multimodal_gen.runtime.distributed.sp_shard_utils import join_seqs +from sglang.multimodal_gen.runtime.layers.layernorm import RMSNorm +from sglang.multimodal_gen.runtime.layers.linear import ( + ColumnParallelLinear, + apply_unquantized_linear, +) +from sglang.multimodal_gen.runtime.layers.lora.linear import BaseLayerWithLoRA +from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config import ( + ConvRotInt8Config, +) +from sglang.multimodal_gen.runtime.models.dits.qwen_image import ( + QwenImageCrossAttention, + _joint_qkv_head_views, + _project_qkv_into_joint_buffers, + _sync_into_buffer, + _use_joint_qkv_buffers, +) +from sglang.test.test_utils import CustomTestCase + +DIM = 32 +# ConvRot rotates 256-wide input groups. +CONVROT_DIM = 256 +NUM_HEADS = 2 +HEAD_DIM = 8 +INNER_DIM = NUM_HEADS * HEAD_DIM +SEQ_TXT = 5 +SEQ_IMG = 13 +QWEN_IMAGE = "sglang.multimodal_gen.runtime.models.dits.qwen_image" +SP_WORLD_SIZE = f"{QWEN_IMAGE}.get_sp_world_size" +PROJECT_INTO_JOINT_BUFFERS = f"{QWEN_IMAGE}._project_qkv_into_joint_buffers" +FUSED_INPLACE_QKNORM = ( + "sglang.multimodal_gen.runtime.layers.layernorm.can_use_fused_inplace_qknorm" +) +# ColumnParallelLinear reads only the group's size and rank. +_TP_GROUP = SimpleNamespace(world_size=1, rank_in_group=0) + + +# Capability only: on a supported GPU a missing toolchain or a failed build must +# fail these tests, not skip them. +requires_convrot_kernel = unittest.skipUnless( + torch.cuda.is_available() + and torch.cuda.get_device_capability() in convrot_int8_supported_capabilities(), + "needs a GPU in the convrot_int8 capability table", +) + + +def _linear( + *, + bias: bool = True, + dim: int = DIM, + dtype: torch.dtype = torch.bfloat16, + quant_config: ConvRotInt8Config | None = None, + prefix: str = "", +) -> ColumnParallelLinear: + linear = ColumnParallelLinear( + dim, + INNER_DIM, + bias=bias, + gather_output=False, + params_dtype=dtype, + quant_config=quant_config, + prefix=prefix, + tp_group=_TP_GROUP, + ) + with torch.no_grad(): + linear.weight.normal_() + if bias: + linear.bias.normal_() + linear = linear.cuda() + if quant_config is not None: + linear.quant_method.process_weights_after_loading(linear) + return linear + + +def _passthrough(x: torch.Tensor) -> tuple[torch.Tensor, None]: + return x, None + + +class _RecordingAttention: + # Stands in for USPAttention: keeps the Q/K/V it was handed and returns a + # deterministic function of them in the [B, S, H, D] layout. + def __init__(self) -> None: + self.calls: list[tuple[torch.Tensor, torch.Tensor, torch.Tensor, dict]] = [] + + def __call__( + self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, **kwargs + ) -> torch.Tensor: + self.calls.append((q.clone(), k.clone(), v.clone(), kwargs)) + if kwargs["q_prefix"] is not None: + q = torch.cat([kwargs["q_prefix"], q], dim=1) + k = torch.cat([kwargs["k_prefix"], k], dim=1) + v = torch.cat([kwargs["v_prefix"], v], dim=1) + return q + k + v + + +def _attention( + *, + bias: bool = True, + dtype: torch.dtype = torch.bfloat16, + quant_config: ConvRotInt8Config | None = None, +) -> QwenImageCrossAttention: + dim = DIM if quant_config is None else CONVROT_DIM + attn = object.__new__(QwenImageCrossAttention) + nn.Module.__init__(attn) + attn.head_dim = HEAD_DIM + attn.local_num_heads = NUM_HEADS + attn.added_kv_proj_dim = dim + attn.qk_norm = True + attn.use_fused_qkv = False + attn.use_fused_qkv_epilogue = False + attn.use_fused_added_qkv = False + attn._unquantized_added_qkv_is_packed = False + attn.separate_unquantized_qkv_proj = quant_config is None + attn.separate_convrot_qkv_proj = quant_config is not None + for name in ("to_q", "to_k", "to_v", "add_q_proj", "add_k_proj", "add_v_proj"): + setattr( + attn, + name, + _linear( + bias=bias, + dim=dim, + dtype=dtype, + quant_config=quant_config, + prefix=f"transformer_blocks.0.attn.{name}", + ), + ) + for name in ("norm_q", "norm_k", "norm_added_q", "norm_added_k"): + norm = RMSNorm(HEAD_DIM, eps=1e-6).to(device="cuda", dtype=dtype) + with torch.no_grad(): + norm.weight.normal_() + setattr(attn, name, norm) + attn.to_out = [_passthrough] + attn.to_add_out = _passthrough + attn.attn = _RecordingAttention() + return attn + + +def _streams( + *, dim: int = DIM, dtype: torch.dtype = torch.bfloat16 +) -> tuple[torch.Tensor, torch.Tensor]: + hidden = torch.randn(1, SEQ_IMG, dim, device="cuda", dtype=dtype) + encoder = torch.randn(1, SEQ_TXT, dim, device="cuda", dtype=dtype) + return hidden, encoder + + +def _joint_flag(attn: QwenImageCrossAttention) -> str: + return ( + "separate_convrot_qkv_proj" + if attn.separate_convrot_qkv_proj + else "separate_unquantized_qkv_proj" + ) + + +class _JointQkvCase(CustomTestCase): + # Forward helpers shared by the unquantized and the ConvRot test classes. + + def setUp(self) -> None: + torch.manual_seed(20260902) + # The runtime forwards under no_grad; the out= GEMMs of the joint path + # reject grad-tracking operands. + grad_was_enabled = torch.is_grad_enabled() + torch.set_grad_enabled(False) + self.addCleanup(torch.set_grad_enabled, grad_was_enabled) + + def _forward( + self, + attn: QwenImageCrossAttention, + *, + dtype: torch.dtype = torch.bfloat16, + **kwargs, + ): + hidden, encoder = _streams(dim=attn.added_kv_proj_dim, dtype=dtype) + with patch(SP_WORLD_SIZE, return_value=1): + return attn.forward( + hidden_states=hidden, + encoder_hidden_states=encoder, + image_rotary_emb=None, + **kwargs, + ) + + def _assert_joint_path_matches_join_seqs_path( + self, attn: QwenImageCrossAttention, *, dtype: torch.dtype = torch.bfloat16 + ) -> None: + flag = _joint_flag(attn) + outputs = [] + for joint in (True, False): + torch.manual_seed(20260902) + setattr(attn, flag, joint) + attn.attn.calls.clear() + with patch( + PROJECT_INTO_JOINT_BUFFERS, wraps=_project_qkv_into_joint_buffers + ) as project: + out = self._forward(attn, dtype=dtype) + self.assertEqual(project.call_count, 1 if joint else 0) + outputs.append((out, attn.attn.calls[0])) + (joint_out, joint_call), (ref_out, ref_call) = outputs + + self.assertIsNone(joint_call[3]["q_prefix"]) + for joint_qkv, ref_qkv in zip(joint_call[:3], ref_call[:3]): + self.assertEqual( + joint_qkv.shape, (1, SEQ_TXT + SEQ_IMG, NUM_HEADS, HEAD_DIM) + ) + self.assertTrue(torch.equal(joint_qkv, ref_qkv)) + self.assertTrue(torch.equal(joint_out[0], ref_out[0])) + self.assertTrue(torch.equal(joint_out[1], ref_out[1])) + + def _assert_compiled_forward_matches_eager(self, attn) -> None: + with patch(FUSED_INPLACE_QKNORM, return_value=False): + torch.manual_seed(20260902) + eager_out = self._forward(attn) + torch.manual_seed(20260902) + compiled = torch.compile(attn, dynamic=False) + hidden, encoder = _streams(dim=attn.added_kv_proj_dim, dtype=torch.bfloat16) + with patch(SP_WORLD_SIZE, return_value=1): + compiled_out = compiled( + hidden_states=hidden, + encoder_hidden_states=encoder, + image_rotary_emb=None, + ) + torch.testing.assert_close(compiled_out[0], eager_out[0], rtol=2e-2, atol=2e-2) + torch.testing.assert_close(compiled_out[1], eager_out[1], rtol=2e-2, atol=2e-2) + + def _assert_forward_bypasses_joint_buffers( + self, attn: QwenImageCrossAttention, *, dtype: torch.dtype + ) -> tuple[torch.Tensor, torch.Tensor]: + # The out= kernels reject these operand dtypes, so the joint path must + # not be entered; the flag-off forward is the same code the fallback runs. + with patch( + PROJECT_INTO_JOINT_BUFFERS, wraps=_project_qkv_into_joint_buffers + ) as project: + out = self._forward(attn, dtype=dtype) + project.assert_not_called() + return out + + +@unittest.skipUnless(torch.cuda.is_available(), "CUDA required") +class TestQwenImageJointQkvBuffers(_JointQkvCase): + def test_forward_is_bitwise_identical_to_join_seqs_path(self): + self._assert_joint_path_matches_join_seqs_path(_attention()) + + def test_forward_copies_back_when_qk_norm_returns_fresh_tensors(self): + """The non-fused QK-norm returns new tensors instead of writing the + buffer views in place; their values must still reach the joint buffers.""" + with patch(FUSED_INPLACE_QKNORM, return_value=False): + self._assert_joint_path_matches_join_seqs_path(_attention()) + + def test_compiled_forward_matches_eager_when_qk_norm_returns_fresh_tensors(self): + """With the in-place QK-norm off, the joint buffers are filled by the + copy-back; that path must compile and agree with eager.""" + self._assert_compiled_forward_matches_eager(_attention()) + + def test_fp16_parameters_and_streams_take_the_joint_path(self): + """--dit-precision fp16 keeps every operand FP16, which the out= GEMMs + accept like BF16.""" + self._assert_joint_path_matches_join_seqs_path( + _attention(dtype=torch.float16), dtype=torch.float16 + ) + + def test_fp32_streams_under_bf16_autocast_bypass_joint_buffers(self): + """FP32 streams under BF16 autocast must take the join_seqs path; the + out= GEMMs of the joint path do not autocast.""" + attn = _attention() + with torch.autocast("cuda", dtype=torch.bfloat16): + self._assert_forward_bypasses_joint_buffers(attn, dtype=torch.float32) + + def test_masked_forward_bypasses_joint_buffers(self): + """A masked sequence hands text Q/K/V to attention as a prefix, so the + joint buffers must not be built for it.""" + attn = _attention() + mask = torch.ones(1, SEQ_TXT, dtype=torch.bool, device="cuda") + with patch(PROJECT_INTO_JOINT_BUFFERS) as project: + self._forward(attn, encoder_hidden_states_mask=mask) + project.assert_not_called() + self.assertIsNotNone(attn.attn.calls[0][3]["q_prefix"]) + + def test_stale_convrot_flag_falls_through_to_per_layer_projections(self): + """LoRA mounting swaps the projections after construction, so the + ConvRot flag alone must not select the shared-input helper.""" + attn = _attention() + attn.separate_unquantized_qkv_proj = False + mask = torch.ones(1, SEQ_TXT, dtype=torch.bool, device="cuda") + outputs = [] + for convrot in (False, True): + torch.manual_seed(20260902) + attn.separate_convrot_qkv_proj = convrot + outputs.append(self._forward(attn, encoder_hidden_states_mask=mask)) + self.assertTrue(torch.equal(outputs[0][0], outputs[1][0])) + self.assertTrue(torch.equal(outputs[0][1], outputs[1][1])) + + def test_joint_projection_without_bias_matches_reference(self): + attn = _attention(bias=False) + hidden, encoder = _streams() + bufs = _project_qkv_into_joint_buffers( + attn=attn, + hidden_states=hidden, + encoder_hidden_states=encoder, + seq_len_txt=SEQ_TXT, + ) + img_layers = (attn.to_q, attn.to_k, attn.to_v) + txt_layers = (attn.add_q_proj, attn.add_k_proj, attn.add_v_proj) + for buf, img_layer, txt_layer in zip(bufs, img_layers, txt_layers): + expected = join_seqs( + apply_unquantized_linear(encoder, txt_layer.weight, None), + apply_unquantized_linear(hidden, img_layer.weight, None), + 0, + ) + self.assertTrue(torch.equal(buf, expected)) + + def test_head_views_alias_the_joint_buffers(self): + hidden, encoder = _streams() + bufs = _project_qkv_into_joint_buffers( + attn=_attention(), + hidden_states=hidden, + encoder_hidden_states=encoder, + seq_len_txt=SEQ_TXT, + ) + views = _joint_qkv_head_views( + bufs, seq_len_txt=SEQ_TXT, num_heads=NUM_HEADS, head_dim=HEAD_DIM + ) + img_query, txt_query = views[0], views[3] + + self.assertEqual(img_query.shape, (1, SEQ_IMG, NUM_HEADS, HEAD_DIM)) + self.assertEqual(txt_query.shape, (1, SEQ_TXT, NUM_HEADS, HEAD_DIM)) + self.assertTrue(img_query.is_contiguous()) + self.assertTrue(txt_query.is_contiguous()) + # In-place QK-norm/RoPE on the views must land in the joint buffer. + img_query.zero_() + txt_query.fill_(1.0) + self.assertTrue(torch.all(bufs[0][:, SEQ_TXT:] == 0)) + self.assertTrue(torch.all(bufs[0][:, :SEQ_TXT] == 1)) + + def test_eligibility_rejects_every_unsupported_layout(self): + attn = _attention() + hidden, encoder = _streams() + + def eligible(h=hidden, e=encoder, *, masked=False, sp=1, attention=attn): + with patch(SP_WORLD_SIZE, return_value=sp): + return _use_joint_qkv_buffers( + attn=attention, + hidden_states=h, + encoder_hidden_states=e, + masked=masked, + ) + + self.assertTrue(eligible()) + self.assertFalse(eligible(masked=True)) + self.assertFalse(eligible(sp=2)) + self.assertFalse(eligible(h=hidden.expand(2, -1, -1).contiguous())) + strided_hidden = hidden.transpose(1, 2).contiguous().transpose(1, 2) + strided_encoder = encoder.transpose(1, 2).contiguous().transpose(1, 2) + self.assertFalse(eligible(h=strided_hidden)) + self.assertFalse(eligible(e=strided_encoder)) + packed = _attention() + packed.separate_unquantized_qkv_proj = False + self.assertFalse(eligible(attention=packed)) + # An unmerged LoRA adds its delta in forward; the bare addmm would drop it. + lora_wrapped = _attention() + lora_wrapped.to_q = BaseLayerWithLoRA(lora_wrapped.to_q) + self.assertFalse(eligible(attention=lora_wrapped)) + + def test_eligibility_requires_one_dtype_across_streams_weights_and_autocast( + self, + ): + """The out= GEMMs neither promote nor autocast, so they stand in for + F.linear only when every operand already has the dtype it would use.""" + attn = _attention() + hidden, encoder = _streams() + + def eligible(h=hidden, e=encoder, *, attention=attn): + with patch(SP_WORLD_SIZE, return_value=1): + return _use_joint_qkv_buffers( + attn=attention, + hidden_states=h, + encoder_hidden_states=e, + masked=False, + ) + + self.assertFalse(eligible(h=hidden.float(), e=encoder.float())) + self.assertFalse(eligible(e=encoder.float())) + fp32_bias = _attention() + fp32_bias.to_k.bias.data = fp32_bias.to_k.bias.data.float() + self.assertFalse(eligible(attention=fp32_bias)) + with torch.autocast("cuda", dtype=torch.float16): + self.assertFalse(eligible()) + with torch.autocast("cuda", dtype=torch.bfloat16): + self.assertTrue(eligible()) + self.assertFalse(eligible(h=hidden.float(), e=encoder.float())) + self.assertTrue(eligible()) + fp16 = _attention(dtype=torch.float16) + h16, e16 = _streams(dtype=torch.float16) + self.assertTrue(eligible(h=h16, e=e16, attention=fp16)) + + +class TestSyncIntoBufferUnderCompile(CustomTestCase): + def test_out_of_place_sync_compiles_without_graph_breaks(self): + """Tracing the copy-back must produce one graph: no storage-pointer + comparison while Dynamo is compiling.""" + + @torch.no_grad() + def probe(x): + out = torch.empty_like(x) + _sync_into_buffer(value=x + 1, buf_view=out) + return out * 2 + + x = torch.arange(8, dtype=torch.float32) + report = torch._dynamo.explain(probe)(x) + self.assertEqual(report.graph_break_count, 0, report.break_reasons) + self.assertEqual(report.graph_count, 1) + torch._dynamo.reset() + self.assertTrue(torch.equal(torch.compile(probe)(x), (x + 1) * 2)) + + def test_eager_sync_copies_fresh_values_and_keeps_aliases(self): + """Eager keeps the storage check: a fresh value is copied back, an + aliased one must not pay a redundant copy_. The version counter is + shared by a tensor and its views, so it counts in-place writes.""" + buf = torch.zeros(8) + _sync_into_buffer(value=torch.ones(8), buf_view=buf) + self.assertTrue(torch.equal(buf, torch.ones(8))) + version = buf._version + _sync_into_buffer(value=buf[2:6], buf_view=buf[2:6]) + self.assertEqual(buf._version, version) + + +@requires_convrot_kernel +class TestQwenImageJointQkvBuffersConvRot(_JointQkvCase): + """The same forward with the six projections on convrot_int8's JIT + backend, which writes the joint buffers through its out= op.""" + + def setUp(self) -> None: + super().setUp() + self.config = ConvRotInt8Config(backend="jit") + + def test_forward_is_bitwise_identical_to_join_seqs_path(self): + attn = _attention(quant_config=self.config) + self.assertEqual(attn.to_q.weight.dtype, torch.int8) + self._assert_joint_path_matches_join_seqs_path(attn) + + def test_compiled_forward_matches_eager(self): + """The out= writes into the joint buffers go through the registered + custom ops under torch.compile; they must trace and agree with eager.""" + self._assert_compiled_forward_matches_eager( + _attention(quant_config=self.config) + ) + + def test_fp16_streams_bypass_joint_buffers_and_return_fp16(self): + """FP16 streams must skip the joint buffers (the out= kernel stores BF16 + only) and the forward must still return FP16.""" + attn = _attention(dtype=torch.float16, quant_config=self.config) + self.assertEqual(attn.to_q.bias.dtype, torch.bfloat16) + out = self._assert_forward_bypasses_joint_buffers(attn, dtype=torch.float16) + self.assertEqual(out[0].dtype, torch.float16) + self.assertEqual(out[1].dtype, torch.float16) + + def test_eligibility_requires_bf16_streams(self): + attn = _attention(quant_config=self.config) + hidden, encoder = _streams(dim=CONVROT_DIM) + + def eligible(h, e): + with patch(SP_WORLD_SIZE, return_value=1): + return _use_joint_qkv_buffers( + attn=attn, hidden_states=h, encoder_hidden_states=e, masked=False + ) + + self.assertTrue(eligible(hidden, encoder)) + self.assertFalse(eligible(hidden.half(), encoder.half())) + self.assertFalse(eligible(hidden, encoder.half())) + + +if __name__ == "__main__": + unittest.main() diff --git a/python/sglang/multimodal_gen/test/unit/test_server_args.py b/python/sglang/multimodal_gen/test/unit/test_server_args.py index cb4bdf69e9..afbca6522e 100644 --- a/python/sglang/multimodal_gen/test/unit/test_server_args.py +++ b/python/sglang/multimodal_gen/test/unit/test_server_args.py @@ -623,7 +623,7 @@ class TestServerArgsPathExpansion(unittest.TestCase): server_args.component_attention_backends, ) self.assertEqual( - {"text_encoder": "kitchen_int8", "transformer": "fp8"}, + {"text_encoder": "convrot_int8", "transformer": "fp8"}, server_args.component_quantizations, ) self.assertEqual( diff --git a/python/sglang/multimodal_gen/test/unit/test_text_encoder_loader.py b/python/sglang/multimodal_gen/test/unit/test_text_encoder_loader.py index b4d784f6a9..ca39ab0583 100644 --- a/python/sglang/multimodal_gen/test/unit/test_text_encoder_loader.py +++ b/python/sglang/multimodal_gen/test/unit/test_text_encoder_loader.py @@ -21,8 +21,8 @@ from sglang.multimodal_gen.runtime.layers.quantization.comfy_nvfp4 import ( ComfyNvfp4Config, ComfyNvfp4LinearMethod, ) -from sglang.multimodal_gen.runtime.layers.quantization.configs.kitchen_int8_config import ( - KitchenInt8Config, +from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config import ( + ConvRotInt8Config, ) from sglang.multimodal_gen.runtime.layers.quantization.configs.kitchen_w4a4_config import ( KitchenW4A4Config, @@ -623,11 +623,11 @@ class TestTextEncoderQuantization(unittest.TestCase): "/model/text_encoder", "/model/text_encoder", "text_encoder", - explicit_quantization="kitchen_int8", + explicit_quantization="convrot_int8", ignored_layers=["lm_head"], ) - self.assertIsInstance(model_config.quant_config, KitchenInt8Config) + self.assertIsInstance(model_config.quant_config, ConvRotInt8Config) self.assertFalse(model_config.quant_config.is_checkpoint_int8_serialized) self.assertEqual(model_config.quant_config.ignored_layers, ["lm_head"]) @@ -688,7 +688,7 @@ class TestTextEncoderQuantization(unittest.TestCase): "text_encoder", ) - self.assertIsInstance(model_config.quant_config, KitchenInt8Config) + self.assertIsInstance(model_config.quant_config, ConvRotInt8Config) self.assertEqual( set(model_config.quant_config.layer_markers), {"model.visual.blocks.0.attn.qkv_proj"}, @@ -1129,8 +1129,17 @@ class TestQuantizedTextEncoderPostprocess(unittest.TestCase): ): _require_quantized_encoder_layers(nn.Linear(2, 2), "text_encoder") + def test_online_convrot_int8_has_no_markers_to_consume(self): + """An online convrot_int8 encoder carries no serialized markers, so the + consumption check must pass instead of failing on the absent map.""" + _require_quantized_encoder_layers( + _QuantizedEncoder(_RecordingQuantMethod()), + "text_encoder", + quant_config=ConvRotInt8Config(ignored_layers=["lm_head"]), + ) + def test_rejects_unconsumed_comfy_marker(self): - config = KitchenInt8Config( + config = ConvRotInt8Config( layer_markers={ "visual.proj": { "format": "int8_tensorwise", diff --git a/python/sglang/multimodal_gen/test/unit/test_transformer_quant.py b/python/sglang/multimodal_gen/test/unit/test_transformer_quant.py index 810b7b1585..2c03f34035 100644 --- a/python/sglang/multimodal_gen/test/unit/test_transformer_quant.py +++ b/python/sglang/multimodal_gen/test/unit/test_transformer_quant.py @@ -58,8 +58,8 @@ from sglang.multimodal_gen.runtime.layers.quantization.comfy_fp8 import ( ComfyFp8Config, ComfyFullPrecisionFp8LinearMethod, ) -from sglang.multimodal_gen.runtime.layers.quantization.configs.kitchen_int8_config import ( - KitchenInt8Config, +from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config import ( + ConvRotInt8Config, ) from sglang.multimodal_gen.runtime.layers.quantization.configs.kitchen_w4a4_config import ( KitchenW4A4Config, @@ -70,13 +70,13 @@ from sglang.multimodal_gen.runtime.layers.quantization.configs.kitchen_w4a8_conf from sglang.multimodal_gen.runtime.layers.quantization.configs.nunchaku_config import ( NunchakuConfig, ) +from sglang.multimodal_gen.runtime.layers.quantization.convrot_int8_comfy_kitchen import ( + ConvRotInt8ComfyKitchenLinearMethod, +) from sglang.multimodal_gen.runtime.layers.quantization.fp8 import ( Fp8Config, Fp8LinearMethod, ) -from sglang.multimodal_gen.runtime.layers.quantization.kitchen_int8 import ( - KitchenInt8LinearMethod, -) from sglang.multimodal_gen.runtime.layers.quantization.modelopt_quant import ( ModelOptFp4Config, ModelOptFp4LinearMethod, @@ -141,6 +141,13 @@ from sglang.srt.layers.quantization.bitsandbytes import ( from sglang.srt.layers.quantization.fp8 import Fp8Config as SRTFp8Config from sglang.srt.layers.quantization.fp8 import Fp8LinearMethod as SRTFp8LinearMethod +# convrot_int8 picks its kernel backend from the GPU; pin comfy_kitchen so these +# cases do not depend on the runner. +_CONVROT_JIT_AVAILABLE = ( + "sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config." + "_jit_available" +) + class _FakeFluxTransformer: pass @@ -707,10 +714,17 @@ class TestTransformerQuantHelpers(unittest.TestCase): ) @patch( - "sglang.multimodal_gen.runtime.layers.quantization.kitchen_int8." + "sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config." + "_comfy_kitchen_available", + return_value=True, + ) + @patch( + "sglang.multimodal_gen.runtime.layers.quantization.convrot_int8_comfy_kitchen." "_load_comfy_kitchen" ) - def test_inspect_minimax_h3_safetensors_detects_curve_and_comfy_format(self, _load): + def test_inspect_minimax_h3_safetensors_detects_curve_and_comfy_format( + self, _load, _available + ): marker = json.dumps( { "format": "int8_tensorwise", @@ -847,14 +861,27 @@ class TestTransformerQuantHelpers(unittest.TestCase): } ) - self.assertIsInstance(config, KitchenInt8Config) + self.assertIsInstance(config, ConvRotInt8Config) self.assertTrue(config.is_checkpoint_int8_serialized) self.assertTrue(config.checkpoint_uses_native_qkv_layout) - self.assertFalse(KitchenInt8Config().checkpoint_uses_native_qkv_layout) + self.assertFalse(ConvRotInt8Config().checkpoint_uses_native_qkv_layout) self.assertFalse(_needs_device_weight_postprocess(config)) self.assertTrue(config.supports_input_partition("blocks.0.mlp.fc1", 6400)) self.assertFalse(config.supports_input_partition("blocks.0.mlp.fc1", 3200)) + @patch(_CONVROT_JIT_AVAILABLE, return_value=False) + def test_online_convrot_int8_loads_on_the_host_under_offload(self, _jit): + """Online convrot_int8 quantizes each layer through its own CUDA round + trip, so the loader must not require the whole BF16 DiT on the GPU + (an offloaded 62 GB H3 DiT has to stay on the host during load).""" + config = ConvRotInt8Config() + + self.assertFalse(config.is_checkpoint_int8_serialized) + self.assertFalse(_needs_device_weight_postprocess(config)) + spec = TransformerQuantLoadSpec([], config, None, None) + self.assertTrue(spec.is_convrot_int8) + self.assertFalse(spec.is_serialized_convrot_int8) + def test_minimax_h3_w4a8_metadata_resolves_serialized_kitchen(self): metadata = { "_quantization_metadata": json.dumps( @@ -1003,9 +1030,10 @@ class TestTransformerQuantHelpers(unittest.TestCase): new=object(), ), patch( - "sglang.multimodal_gen.runtime.layers.quantization.kitchen_int8." + "sglang.multimodal_gen.runtime.layers.quantization.convrot_int8_comfy_kitchen." "_load_comfy_kitchen" ), + patch(_CONVROT_JIT_AVAILABLE, return_value=False), ): config = resolve_minimax_h3_checkpoint_quantization(markers) w4a4 = ReplicatedLinear( @@ -1030,12 +1058,13 @@ class TestTransformerQuantHelpers(unittest.TestCase): self.assertEqual(int8.weight.shape, (3, 256)) self.assertEqual(set(config.selected), {"w4a4", "int8"}) + @patch(_CONVROT_JIT_AVAILABLE, return_value=False) @patch( - "sglang.multimodal_gen.runtime.layers.quantization.kitchen_int8." + "sglang.multimodal_gen.runtime.layers.quantization.convrot_int8_comfy_kitchen." "_load_comfy_kitchen" ) - def test_serialized_kitchen_constructs_int8_weight_and_row_scale(self, _load): - config = KitchenInt8Config( + def test_serialized_kitchen_constructs_int8_weight_and_row_scale(self, _load, _jit): + config = ConvRotInt8Config( layer_markers={ "proj": { "format": "int8_tensorwise", @@ -1731,16 +1760,19 @@ class TestTransformerQuantHelpers(unittest.TestCase): ), ModelOptFp4LinearMethod, ) - with patch( - "sglang.multimodal_gen.runtime.layers.quantization." - "kitchen_int8._load_comfy_kitchen" + with ( + patch( + "sglang.multimodal_gen.runtime.layers.quantization." + "convrot_int8_comfy_kitchen._load_comfy_kitchen" + ), + patch(_CONVROT_JIT_AVAILABLE, return_value=False), ): self.assertIsInstance( config.get_quant_method( LinearBase(input_size=256, output_size=32), "blocks.0.attn.out_proj", ), - KitchenInt8LinearMethod, + ConvRotInt8ComfyKitchenLinearMethod, ) self.assertIsInstance( config.get_quant_method( diff --git a/test/registered/kernels/benchmark/gemm/bench_convrot_int8.py b/test/registered/kernels/benchmark/gemm/bench_convrot_int8.py new file mode 100644 index 0000000000..b4afa1e603 --- /dev/null +++ b/test/registered/kernels/benchmark/gemm/bench_convrot_int8.py @@ -0,0 +1,72 @@ +"""ConvRot INT8 linear (rotate + quant + GEMM, and GEMM alone) vs torch BF16 addmm.""" + +import itertools + +import torch + +from sglang.kernels.jit.benchmark import marker +from sglang.kernels.jit.benchmark.utils import create_random +from sglang.kernels.ops.gemm.convrot_int8 import ( + SUPPORTED_CAPABILITIES, + convrot_int8_fused_linear, + convrot_int8_linear_prequant, + convrot_rotate_quantize_activation, +) +from sglang.test.ci.ci_register import register_cuda_ci + +register_cuda_ci( + est_time=120, stage="base-b-kernel-benchmark", runner_config="1-gpu-large" +) + +GROUP_SIZE = 256 + +# Qwen-Image DiT linears at 1024x1024; same shapes as ops/gemm/test_convrot_int8.py. +M_VALUES = [3, 20, 2048, 4096] +KN_VALUES = [(3072, 3072), (3072, 12288), (12288, 3072)] +SHAPES = [(M, K, N) for M, (K, N) in itertools.product(M_VALUES, KN_VALUES)] + + +def _fused(x, weight_q, weight_scale, bias): + return convrot_int8_fused_linear( + x, weight_q, weight_scale, bias=bias, group_size=GROUP_SIZE + ) + + +def _prequant(x_q, x_scale, weight_q, weight_scale, bias): + return convrot_int8_linear_prequant( + x_q, x_scale, weight_q, weight_scale, bias=bias, group_size=GROUP_SIZE + ) + + +def _torch_bf16(x, weight, bias): + return torch.addmm(bias, x, weight.t()) + + +@marker.parametrize("shape", SHAPES, [(2048, 3072, 3072), (2048, 12288, 3072)]) +@marker.benchmark("provider", ["convrot_int8", "convrot_int8_prequant", "torch_bf16"]) +def benchmark(shape, provider): + if torch.cuda.get_device_capability() not in SUPPORTED_CAPABILITIES: + marker.skip("convrot_int8 kernels carry no code for this GPU") + M, K, N = shape + x = create_random(M, K) + weight = create_random(N, K) * 0.02 + bias = create_random(N) + weight_q, weight_scale = convrot_rotate_quantize_activation( + weight, group_size=GROUP_SIZE + ) + x_q, x_scale = convrot_rotate_quantize_activation(x, group_size=GROUP_SIZE) + # Each provider gets only the tensors it reads, so the bandwidth column and + # the per-iteration L2 rotation count what that provider actually touches. + fn, args = { + "convrot_int8": (_fused, (x, weight_q, weight_scale, bias)), + "convrot_int8_prequant": ( + _prequant, + (x_q, x_scale, weight_q, weight_scale, bias), + ), + "torch_bf16": (_torch_bf16, (x, weight, bias)), + }[provider] + return marker.do_bench(fn, input_args=args, flops=2.0 * M * N * K) + + +if __name__ == "__main__": + benchmark.run() diff --git a/test/registered/kernels/ops/gemm/test_convrot_int8.py b/test/registered/kernels/ops/gemm/test_convrot_int8.py new file mode 100644 index 0000000000..6959cc59f2 --- /dev/null +++ b/test/registered/kernels/ops/gemm/test_convrot_int8.py @@ -0,0 +1,405 @@ +import math +import sys + +import pytest +import torch +import torch.nn.functional as F + +from sglang.kernels.ops.gemm.convrot_int8 import ( + GROUP_SIZES, + SUPPORTED_CAPABILITIES, + convrot_int8_fused_linear, + convrot_int8_linear_prequant, + convrot_int8_supported_capabilities, + convrot_rotate_quantize_activation, +) +from sglang.test.ci.ci_register import register_cuda_ci + +# One registration per GEMM path: H100 (sm_90a), B200 (sm_100a), RTX 5090 (CC 12.0). +register_cuda_ci(est_time=120, stage="base-b-kernel-unit", runner_config="1-gpu-large") +register_cuda_ci(est_time=120, stage="base-b-kernel-unit", runner_config="4-gpu-b200") +register_cuda_ci(est_time=120, stage="base-b-kernel-unit", runner_config="1-gpu-small") + + +def _device_supported() -> bool: + return ( + torch.cuda.is_available() + and torch.cuda.get_device_capability() in SUPPORTED_CAPABILITIES + ) + + +pytestmark = pytest.mark.skipif( + not _device_supported(), + reason="convrot_int8 kernels carry no code for this GPU (see SUPPORTED_CAPABILITIES)", +) + +GROUP_SIZE = 256 + +# Qwen-Image DiT linears at 1024x1024: text-stream rows (3, 20), image-stream +# rows (2048, 4096); (K, N) = attention projections, FFN up, FFN down. Together +# they hit every tile branch: SM90 small-M narrow/wide-N and large-M, SM100 +# default and wide-N, and the small/large-M rows of the CC 12.x mma.sync table. +PRODUCTION_M = [3, 20, 2048, 4096] +PRODUCTION_KN = [(3072, 3072), (3072, 12288), (12288, 3072)] + + +def _channel_shift(z, gen): + K = z.shape[-1] + scale = (1 + 0.3 * torch.randn(K, device="cuda", generator=gen)).abs() + shift = 0.5 + 0.3 * torch.randn(K, device="cuda", generator=gen) + return (z * scale + shift).to(z.dtype) + + +# Activation statistics a data-free rotation must be indifferent to. GELU +# outputs and shifted rows are what the DiT feeds the down-projections; the +# Sylvester transform concentrated a group's mean into one coefficient and +# ran 1.8-2.8x the Gaussian error on these, the regular Hadamard stays within +# 1.15x. +INPUT_FAMILIES = { + "gauss": lambda z, gen: z, + "gelu": lambda z, gen: torch.nn.functional.gelu(2 * z, approximate="tanh"), + "dc": lambda z, gen: z + 1.0, + "chan_shift": _channel_shift, +} + + +def _make_inputs(M, K, N, with_bias, seed=0, family="gauss"): + gen = torch.Generator(device="cuda").manual_seed(seed) + x = torch.randn(M, K, device="cuda", dtype=torch.bfloat16, generator=gen) + x = INPUT_FAMILIES[family](x, gen) + weight = ( + torch.randn(N, K, device="cuda", dtype=torch.bfloat16, generator=gen) * 0.02 + ) + bias = ( + torch.randn(N, device="cuda", dtype=torch.bfloat16, generator=gen) + if with_bias + else None + ) + weight_q, weight_scale = convrot_rotate_quantize_activation( + weight, group_size=GROUP_SIZE + ) + return x, weight, weight_q, weight_scale, bias + + +def _regular_hadamard(n, device): + """kron([H2,] H4, H4, ...) / sqrt(n): 4-point stages from the lowest index digit + up, one 2-point stage on the top bit when log2(n) is odd. Every row sums to +1 (up to + the 2-point factor), unlike the Sylvester transform whose first row is all ones.""" + h4 = torch.tensor( + [[1, 1, 1, -1], [1, 1, -1, 1], [1, -1, 1, 1], [-1, 1, 1, 1]], + device=device, + dtype=torch.float64, + ) + h2 = torch.tensor([[1, 1], [1, -1]], device=device, dtype=torch.float64) + bits = int(math.log2(n)) + h = torch.ones(1, 1, device=device, dtype=torch.float64) + for _ in range(bits // 2): + h = torch.kron(h4, h) # the lowest digit is the rightmost Kronecker factor + if bits % 2: + h = torch.kron(h2, h) + return (h / math.sqrt(n)).float() + + +def _reference_rotate_quantize(x, group_size): + M, K = x.shape + h = _regular_hadamard(group_size, x.device) + rotated = (x.float().view(M, K // group_size, group_size) @ h).view(M, K) + amax = rotated.abs().amax(dim=1) + scale = torch.where(amax > 0, amax / 127.0, torch.ones_like(amax)) + x_q = torch.round(rotated / scale[:, None]).clamp(-127, 127).to(torch.int8) + return x_q, scale + + +@pytest.mark.parametrize("group_size", [64, 128, 256, 512]) +@pytest.mark.parametrize("M", [3, 2048]) +def test_rotate_quantize_matches_hadamard_reference(M, group_size): + """The in-kernel stages must be the orthonormal regular Hadamard (Kronecker + power of the 4x4) the weight side was quantized with, not merely some + consistent orthogonal map.""" + x, *_ = _make_inputs(M, 3072, 8, with_bias=False) + x_q, x_scale = convrot_rotate_quantize_activation(x, group_size=group_size) + ref_q, ref_scale = _reference_rotate_quantize(x, group_size=group_size) + # The butterfly sums the same terms in a different fp32 order than the + # dense reference matmul; measured drift on the row absmax is ~1e-4. + torch.testing.assert_close(x_scale, ref_scale, rtol=1e-3, atol=0) + assert (x_q.int() - ref_q.int()).abs().max().item() <= 1 + + +def _fused_rel_l2(M, K, N, family): + x, weight, weight_q, weight_scale, bias = _make_inputs( + M, K, N, with_bias=True, family=family + ) + out = convrot_int8_fused_linear( + x, weight_q, weight_scale, bias=bias, group_size=GROUP_SIZE + ) + ref = torch.nn.functional.linear(x, weight, bias) + err = torch.linalg.vector_norm(out.float() - ref.float()) + return (err / torch.linalg.vector_norm(ref.float())).item() + + +@pytest.mark.parametrize("family", [f for f in INPUT_FAMILIES if f != "gauss"]) +@pytest.mark.parametrize("K,N", PRODUCTION_KN) +def test_error_is_independent_of_input_statistics(family, K, N): + M = 2048 + gauss = _fused_rel_l2(M, K, N, "gauss") + other = _fused_rel_l2(M, K, N, family) + assert other < 2e-2, other + assert other / gauss < 1.25, (family, other, gauss) + + +@pytest.mark.parametrize("group_size", [64, 128, 256, 512]) +@pytest.mark.parametrize("K", [1024, 3072, 12288, 16384, 33792, 57856]) +def test_rotate_quantize_matches_hadamard_reference_wide_rows(K, group_size): + # K + group_size leaves a partial last tile; 16384 < K selects the + # 1024-thread launch and K > 32768 the two-pass one. + gen = torch.Generator(device="cuda").manual_seed(0) + for k in (K, K + group_size): + x = torch.randn(3, k, device="cuda", dtype=torch.bfloat16, generator=gen) + x_q, x_scale = convrot_rotate_quantize_activation(x, group_size=group_size) + ref_q, ref_scale = _reference_rotate_quantize(x, group_size=group_size) + torch.testing.assert_close(x_scale, ref_scale, rtol=1e-3, atol=0) + assert (x_q.int() - ref_q.int()).abs().max().item() <= 1 + + +def test_rejects_misaligned_view(): + base = torch.randn(20 * 3072 + 8, device="cuda", dtype=torch.bfloat16) + # Contiguous, but the storage offset is one element: the vectorized loads + # need 16-byte alignment and the op says so instead of faulting. + x = base[1 : 1 + 20 * 3072].view(20, 3072) + with pytest.raises(RuntimeError, match="aligned"): + convrot_rotate_quantize_activation(x, group_size=GROUP_SIZE) + + +@pytest.mark.parametrize("M", PRODUCTION_M) +@pytest.mark.parametrize("K,N", PRODUCTION_KN) +@pytest.mark.parametrize("with_bias", [True, False]) +def test_matches_bf16_reference(M, K, N, with_bias): + x, weight, weight_q, weight_scale, bias = _make_inputs(M, K, N, with_bias=with_bias) + out = convrot_int8_fused_linear( + x, weight_q, weight_scale, bias=bias, group_size=GROUP_SIZE + ) + ref = torch.addmm(bias, x, weight.t()) if with_bias else x @ weight.t() + err = torch.linalg.vector_norm(out.float() - ref.float()) + rel_l2 = (err / torch.linalg.vector_norm(ref.float())).item() + assert rel_l2 < 2e-2, rel_l2 + + +@pytest.mark.parametrize("M", PRODUCTION_M) +@pytest.mark.parametrize("K,N", PRODUCTION_KN) +@pytest.mark.parametrize("with_bias", [True, False]) +def test_prequant_bitwise_equals_fused(M, K, N, with_bias): + x, _, weight_q, weight_scale, bias = _make_inputs(M, K, N, with_bias=with_bias) + x_q, x_scale = convrot_rotate_quantize_activation(x, group_size=GROUP_SIZE) + fused = convrot_int8_fused_linear( + x, weight_q, weight_scale, bias=bias, group_size=GROUP_SIZE + ) + prequant = convrot_int8_linear_prequant( + x_q, x_scale, weight_q, weight_scale, bias=bias, group_size=GROUP_SIZE + ) + assert torch.equal(fused, prequant) + + +@pytest.mark.parametrize("M", [20, 2048]) +@pytest.mark.parametrize("K,N", [(3072, 3072), (3072, 12288)]) +def test_out_variants_bitwise_equal_plain(M, K, N): + x, _, weight_q, weight_scale, bias = _make_inputs(M, K, N, with_bias=True) + x_q, x_scale = convrot_rotate_quantize_activation(x, group_size=GROUP_SIZE) + plain = convrot_int8_fused_linear( + x, weight_q, weight_scale, bias=bias, group_size=GROUP_SIZE + ) + + # A row slice of a larger joint buffer is the production target of `out`. + joint = torch.empty(M + 7, N, device="cuda", dtype=torch.bfloat16) + out = joint[7:] + ret = convrot_int8_fused_linear( + x, weight_q, weight_scale, bias=bias, group_size=GROUP_SIZE, out=out + ) + assert ret.data_ptr() == out.data_ptr() + assert torch.equal(out, plain) + + out2 = torch.empty_like(plain) + ret2 = convrot_int8_linear_prequant( + x_q, x_scale, weight_q, weight_scale, bias=bias, group_size=GROUP_SIZE, out=out2 + ) + assert ret2.data_ptr() == out2.data_ptr() + assert torch.equal(out2, plain) + + +def test_serialized_scale_layout_bitwise_equal_flat(): + """A serialized Comfy ConvRot checkpoint stores weight_scale as [N, 1]; the + ops must accept it unchanged and compute exactly what the flat [N] gives.""" + M, K, N = 20, 3072, 3072 + x, _, weight_q, weight_scale, bias = _make_inputs(M, K, N, with_bias=True) + flat = convrot_int8_fused_linear( + x, weight_q, weight_scale, bias=bias, group_size=GROUP_SIZE + ) + column = convrot_int8_fused_linear( + x, weight_q, weight_scale.view(N, 1), bias=bias, group_size=GROUP_SIZE + ) + assert torch.equal(flat, column) + + +@pytest.mark.parametrize("M", [3, 20, 2048, 4096]) +def test_gelu_input_bitwise_equals_eager_gelu(M): + K, N = 12288, 3072 + x, _, weight_q, weight_scale, bias = _make_inputs(M, K, N, with_bias=True) + fused = convrot_int8_fused_linear( + x, weight_q, weight_scale, bias=bias, group_size=GROUP_SIZE, gelu_input=True + ) + x_gelu = F.gelu(x, approximate="tanh") + ref = convrot_int8_fused_linear( + x_gelu, weight_q, weight_scale, bias=bias, group_size=GROUP_SIZE + ) + assert torch.equal(fused, ref) + + +def test_empty_batch(): + x, _, weight_q, weight_scale, bias = _make_inputs(0, 3072, 3072, with_bias=True) + out = convrot_int8_fused_linear( + x, weight_q, weight_scale, bias=bias, group_size=GROUP_SIZE + ) + assert out.shape == (0, 3072) and out.dtype == torch.bfloat16 + x_q, x_scale = convrot_rotate_quantize_activation(x, group_size=GROUP_SIZE) + assert x_q.shape == (0, 3072) and x_scale.shape == (0,) + + +def test_rejects_invalid_arguments(): + M, K, N = 20, 3072, 3072 + x, _, weight_q, weight_scale, bias = _make_inputs(M, K, N, with_bias=True) + x_q, x_scale = convrot_rotate_quantize_activation(x, group_size=GROUP_SIZE) + + with pytest.raises(RuntimeError, match="unsupported group_size"): + convrot_int8_fused_linear(x, weight_q, weight_scale, bias=bias, group_size=96) + with pytest.raises(RuntimeError, match="multiple of group_size"): + convrot_rotate_quantize_activation( + x[:, :3000].contiguous(), group_size=GROUP_SIZE + ) + with pytest.raises( + RuntimeError, match=r"\[float32\] not in the allowed options: \[bfloat16\]" + ): + convrot_int8_fused_linear( + x.float(), weight_q, weight_scale, bias=bias, group_size=GROUP_SIZE + ) + x_strided = x.t().contiguous().t() + with pytest.raises(RuntimeError, match="Tensor is not contiguous"): + convrot_int8_fused_linear( + x_strided, weight_q, weight_scale, bias=bias, group_size=GROUP_SIZE + ) + with pytest.raises( + RuntimeError, match=r"\[int32\] not in the allowed options: \[int8\]" + ): + convrot_int8_fused_linear( + x, weight_q.to(torch.int32), weight_scale, bias=bias, group_size=GROUP_SIZE + ) + with pytest.raises(RuntimeError, match="multiple of 8"): + convrot_int8_fused_linear( + x, + weight_q[: N - 4].contiguous(), + weight_scale[: N - 4], + bias=bias[: N - 4], + group_size=GROUP_SIZE, + ) + narrow_weight_q = weight_q[:, :2048].contiguous() + with pytest.raises( + RuntimeError, match=r"shape#1\('K'\): expected 3072 but got 2048" + ): + convrot_int8_fused_linear( + x, narrow_weight_q, weight_scale, bias=bias, group_size=GROUP_SIZE + ) + with pytest.raises( + RuntimeError, match=r"shape#0\('N'\): expected 3072 but got 3071" + ): + convrot_int8_fused_linear( + x, weight_q, weight_scale[:-1], bias=bias, group_size=GROUP_SIZE + ) + with pytest.raises( + RuntimeError, match=r"\[float32\] not in the allowed options: \[bfloat16\]" + ): + convrot_int8_fused_linear( + x, weight_q, weight_scale, bias=bias.float(), group_size=GROUP_SIZE + ) + wrong_out = torch.empty(M, N + 1, device="cuda", dtype=torch.bfloat16) + with pytest.raises( + RuntimeError, match=r"shape#1\('N'\): expected 3072 but got 3073" + ): + convrot_int8_fused_linear( + x, weight_q, weight_scale, bias=bias, group_size=GROUP_SIZE, out=wrong_out + ) + fp16_out = torch.empty(M, N, device="cuda", dtype=torch.float16) + with pytest.raises( + RuntimeError, match=r"\[float16\] not in the allowed options: \[bfloat16\]" + ): + convrot_int8_fused_linear( + x, weight_q, weight_scale, bias=bias, group_size=GROUP_SIZE, out=fp16_out + ) + strided_out = torch.empty(M, 2 * N, device="cuda", dtype=torch.bfloat16)[:, :N] + with pytest.raises(RuntimeError, match="Tensor is not contiguous"): + convrot_int8_fused_linear( + x, weight_q, weight_scale, bias=bias, group_size=GROUP_SIZE, out=strided_out + ) + x_q_i32 = x_q.to(torch.int32) + with pytest.raises( + RuntimeError, match=r"\[int32\] not in the allowed options: \[int8\]" + ): + convrot_int8_linear_prequant( + x_q_i32, x_scale, weight_q, weight_scale, bias=bias, group_size=GROUP_SIZE + ) + with pytest.raises(RuntimeError, match=r"shape#0\('M'\): expected 20 but got 19"): + convrot_int8_linear_prequant( + x_q, x_scale[:-1], weight_q, weight_scale, bias=bias, group_size=GROUP_SIZE + ) + with pytest.raises( + RuntimeError, match=r"\[cpu\] not in the allowed options: \[cuda\]" + ): + convrot_int8_fused_linear( + x.cpu(), weight_q, weight_scale, bias=bias, group_size=GROUP_SIZE + ) + + +def test_supported_capabilities_is_the_published_table(): + """The op module is the single source of truth for the supported parts and + group widths; the quantization method and the harness read these tables. + Extending them is a deliberate change, so pin them here.""" + assert convrot_int8_supported_capabilities() == frozenset( + {(9, 0), (10, 0), (12, 0), (12, 1)} + ) + assert GROUP_SIZES == (64, 128, 256, 512) + assert torch.cuda.get_device_capability() in convrot_int8_supported_capabilities() + + +@pytest.mark.parametrize("with_bias", [True, False]) +def test_ops_compile_fullgraph_and_match_eager(with_bias): + """torch.compile routes through the registered custom ops instead of the + eager module call; the compiled graph must not break and must reproduce + eager bit for bit, including the out= write into a row slice.""" + x, _, weight_q, weight_scale, bias = _make_inputs( + 64, 1024, 512, with_bias=with_bias + ) + # Zeros, not empty: the seven rows above the slice are compared too. + joint = torch.zeros(64 + 7, 512, device="cuda", dtype=torch.bfloat16) + + def fn(x, weight_q, weight_scale, bias, out): + x_q, x_scale = convrot_rotate_quantize_activation(x, group_size=GROUP_SIZE) + gelu = convrot_int8_fused_linear( + x, weight_q, weight_scale, bias=bias, group_size=GROUP_SIZE, gelu_input=True + ) + convrot_int8_linear_prequant( + x_q, + x_scale, + weight_q, + weight_scale, + bias=bias, + group_size=GROUP_SIZE, + out=out[7:], + ) + return x_q, x_scale, gelu, out + + args = (x, weight_q, weight_scale, bias) + eager = fn(*args, joint.clone()) + compiled = torch.compile(fn, fullgraph=True)(*args, joint.clone()) + for got, want in zip(compiled, eager, strict=True): + assert torch.equal(got, want) + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__])) diff --git a/test/registered/unit/kernels/test_kernels_namespace.py b/test/registered/unit/kernels/test_kernels_namespace.py index 43092fbba7..d001be9a81 100644 --- a/test/registered/unit/kernels/test_kernels_namespace.py +++ b/test/registered/unit/kernels/test_kernels_namespace.py @@ -296,6 +296,13 @@ def test_reclassified_public_entry_points_are_inventoried(): ("attention.deep_select_topk", (10, 1), False), ("attention.deep_select_topk", (10, 3), True), ("attention.deep_select_topk", (12, 0), False), + ("gemm.convrot_int8_fused_linear", (8, 9), False), + ("gemm.convrot_int8_fused_linear", (9, 0), True), + ("gemm.convrot_int8_fused_linear", (10, 0), True), + ("gemm.convrot_int8_fused_linear", (10, 3), False), + ("gemm.convrot_int8_fused_linear", (12, 0), True), + ("gemm.convrot_int8_fused_linear", (12, 1), True), + ("gemm.convrot_int8_fused_linear", (12, 2), False), ], ) def test_registered_architecture_boundaries(op, sm, expected): @@ -307,6 +314,23 @@ def test_registered_architecture_boundaries(op, sm, expected): assert not K.capabilities_satisfied(spec.capabilities, _CPU) +def test_convrot_int8_specs_match_wrapper_capabilities(): + """The gemm group cannot import the wrapper, so this pins its three registry + entries to the wrapper's SUPPORTED_CAPABILITIES table.""" + from sglang.kernels.ops.gemm import convrot_int8 + + expected = frozenset( + Cap.cuda(min_sm=sm, max_sm=sm) for sm in convrot_int8.SUPPORTED_CAPABILITIES + ) + for op in ( + "gemm.convrot_rotate_quantize_activation", + "gemm.convrot_int8_fused_linear", + "gemm.convrot_int8_linear_prequant", + ): + spec = K.registry.get_backend(op, KernelBackend.JIT) + assert spec.capabilities == expected, op + + def test_deep_select_spec_matches_wrapper_architectures(): """The group cannot import the wrapper, so this is the only link between the two SM lists.""" from sglang.kernels.ops.attention import deep_select