From 1962cbeebe3b1e177ea2f2f4844f7aabb92701bf Mon Sep 17 00:00:00 2001
From: WenhaoZhang <42087078+niehen6174@users.noreply.github.com>
Date: Thu, 22 Oct 2026 11:30:32 +0800
Subject: [PATCH] [Perf] diffusion: convrot_int8 comfy_kitchen issues one GEMM
per layer by default (#44179)
---
.../sglang-diffusion/environment_variables.mdx | 6 +++---
docs/docs/sglang-diffusion/quantization.mdx | 10 ++++++----
.../quantization/convrot_int8_comfy_kitchen.py | 15 +++++----------
3 files changed, 14 insertions(+), 17 deletions(-)
diff --git a/docs/docs/sglang-diffusion/environment_variables.mdx b/docs/docs/sglang-diffusion/environment_variables.mdx
index 80b8ce404c..350a2adb86 100644
--- a/docs/docs/sglang-diffusion/environment_variables.mdx
+++ b/docs/docs/sglang-diffusion/environment_variables.mdx
@@ -85,13 +85,13 @@ description: "Configure SGLang diffusion behavior with environment variables."
SGLANG_KITCHEN_INT8_MAX_ROWS |
- 8192 |
- Max activation rows per convrot_int8 fused GEMM on the comfy_kitchen backend. Set 0 to disable row splitting. |
+ 0 |
+ Max activation rows per convrot_int8 fused GEMM on the comfy_kitchen backend. 0 (default) issues one call; a positive value splits large-M GEMMs by rows. |
SGLANG_KITCHEN_INT8_MIN_SPLIT_N |
8192 |
- Minimum output features before convrot_int8 (comfy_kitchen backend) splits large-M GEMMs. Narrow outputs stay as a single call. |
+ Minimum output features before convrot_int8 (comfy_kitchen backend) splits large-M GEMMs when SGLANG_KITCHEN_INT8_MAX_ROWS is positive. Narrow outputs stay as a single call. |
diff --git a/docs/docs/sglang-diffusion/quantization.mdx b/docs/docs/sglang-diffusion/quantization.mdx
index bfc60e1749..e2de3bcb35 100644
--- a/docs/docs/sglang-diffusion/quantization.mdx
+++ b/docs/docs/sglang-diffusion/quantization.mdx
@@ -498,10 +498,12 @@ for the 24 GB offload recipe, including why `vae` must stay out of
`--layerwise-offload-components`.
-With the `comfy_kitchen` backend, large-M GEMMs (`rows > 8192` and
-`out_features >= 8192`) are split by rows so the fused kernel stays on the
-data-parallel CUTLASS config. Override the thresholds with
-`SGLANG_KITCHEN_INT8_MAX_ROWS` and `SGLANG_KITCHEN_INT8_MIN_SPLIT_N`.
+With the `comfy_kitchen` backend, each GEMM is one fused call by default.
+`SGLANG_KITCHEN_INT8_MAX_ROWS=N` splits large-M GEMMs (`rows > N` and
+`out_features >= SGLANG_KITCHEN_INT8_MIN_SPLIT_N`, default 8192) into N-row
+calls, which keeps them on comfy_kitchen's data-parallel CUTLASS config; on
+MiniMax-H3 with comfy-kitchen 0.2.37 the extra calls and output copies cost more
+than they save.
### Skipping Layers
diff --git a/python/sglang/multimodal_gen/runtime/layers/quantization/convrot_int8_comfy_kitchen.py b/python/sglang/multimodal_gen/runtime/layers/quantization/convrot_int8_comfy_kitchen.py
index 98c82544ae..6819d2535b 100644
--- a/python/sglang/multimodal_gen/runtime/layers/quantization/convrot_int8_comfy_kitchen.py
+++ b/python/sglang/multimodal_gen/runtime/layers/quantization/convrot_int8_comfy_kitchen.py
@@ -35,16 +35,11 @@ __all__ = ["ConvRotInt8ComfyKitchenLinearMethod", "comfy_kitchen_swiglu_mlp"]
# comfy_kitchen's dtype codes for the fused op's output.
_OUT_DTYPE_CODE = {torch.float32: 0, torch.float16: 1, torch.bfloat16: 2}
-# comfy_kitchen picks its CUTLASS tile configuration from a threshold tree
-# (select_fused_int8_config in cutlass_gemm_int8.cu). Shapes whose N falls under
-# its 24832 cutoff but whose M is large get a Stream-K schedule, which exists to
-# balance load when there are too few tiles to fill the GPU. At H3's 32700 tokens
-# qkv_proj already launches ~21k CTAs over the 4090's 128 SMs, so Stream-K's
-# workspace and fixup reduction are pure overhead: 26.5 ms against 17.9 ms for
-# the identical tile without it. Capping rows per call keeps the plain
-# data-parallel config, and is bit-exact because splitting rows does not change
-# any single row's arithmetic.
-_MAX_ROWS_PER_CALL = int(os.environ.get("SGLANG_KITCHEN_INT8_MAX_ROWS", "8192"))
+# Rows per comfy_kitchen int8_linear call; 0 issues one call. Splitting keeps tall,
+# narrow-N GEMMs off comfy_kitchen's Stream-K schedule, but every extra call also
+# copies its slice of the output, and whole-model MiniMax-H3 runs (comfy-kitchen
+# 0.2.37, pruned INT8) are 1.3-3.9% faster unsplit on RTX 4090, RTX 5090 and H200.
+_MAX_ROWS_PER_CALL = int(os.environ.get("SGLANG_KITCHEN_INT8_MAX_ROWS", "0"))
# Narrow outputs do not recover the cost of writing results back through a
# preallocated buffer; H3's out_proj and fc2 (N=5376) both measure slower split.
_MIN_SPLIT_OUTPUT = int(os.environ.get("SGLANG_KITCHEN_INT8_MIN_SPLIT_N", "8192"))