[diffusion] quantization: ConvRot INT8 online W8A8 for DiTs (sgl-kernel + convrot_int8_customkernel) (#38040)

Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com>
This commit is contained in:
Alex O. P.
2026-10-02 23:26:28 +08:00
committed by GitHub
co-authored by Po-Han Huang
parent 1a2eb6c426
commit e667ab10d5
34 changed files with 5545 additions and 322 deletions
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,341 @@
/* Copyright 2026 SGLang Team. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// Adapted from
// https://github.com/NVIDIA/TensorRT-LLM/blob/be1788106245496872d18e702978e59b6bfd50e0/cpp/tensorrt_llm/cutlass_extensions/include/cutlass_extensions/epilogue/threadblock/epilogue_per_row_per_col_scale.h
// Modifications: clang-format reflow, includes trimmed to CUTLASS, no tensorrt_llm dependency.
/***************************************************************************************************
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
#pragma once
#include <cutlass/arch/memory.h>
#include <cutlass/numeric_conversion.h>
namespace cutlass {
namespace epilogue {
namespace threadblock {
template <
typename ThreadblockShape_,
int ThreadCount,
typename ScaleTileIterator_,
typename OutputTileIterator_,
typename ElementAccumulator_,
typename ElementCompute_,
typename ElementwiseFunctor_,
bool UseMasking_ = false>
class EpilogueVisitorPerRowPerCol {
public:
using ThreadblockShape = ThreadblockShape_;
static int const kThreadCount = ThreadCount;
using ScaleTileIterator = ScaleTileIterator_;
using OutputTileIterator = OutputTileIterator_;
using ElementwiseFunctor = ElementwiseFunctor_;
static int const kIterations = OutputTileIterator::kIterations;
static int const kElementsPerAccess = OutputTileIterator::kElementsPerAccess;
using ElementOutput = typename OutputTileIterator::Element;
using LayoutOutput = cutlass::layout::RowMajor;
using ElementAccumulator = ElementAccumulator_;
using AlphaScaleElementType = typename ScaleTileIterator::Element;
using ElementCompute = ElementCompute_;
using AccumulatorFragment = Array<ElementAccumulator, kElementsPerAccess>;
using ComputeFragment = Array<ElementCompute_, kElementsPerAccess>;
using OutputVector = Array<ElementOutput, kElementsPerAccess>;
static int const kThreadsPerRow = OutputTileIterator::ThreadMap::Detail::kAccessWidth;
static bool const kHasMultiStepsInRow = (OutputTileIterator::ThreadMap::Iterations::kColumn > 1);
/// Argument structure
struct Arguments {
typename ElementwiseFunctor::Params elementwise;
int64_t batch_stride_alpha;
int64_t batch_stride_C;
int64_t batch_stride_D;
//
// Methods
//
Arguments() : batch_stride_alpha(0), batch_stride_C(0), batch_stride_D(0) {}
Arguments(typename ElementwiseFunctor::Params elementwise_)
: elementwise(elementwise_), batch_stride_alpha(0), batch_stride_C(0), batch_stride_D(0) {}
Arguments(
typename ElementwiseFunctor::Params elementwise_,
int64_t batch_stride_alpha_,
int64_t batch_stride_C_,
int64_t batch_stride_D_)
: elementwise(elementwise_),
batch_stride_alpha(batch_stride_alpha_),
batch_stride_C(batch_stride_C_),
batch_stride_D(batch_stride_D_) {}
};
struct Params {
typename ElementwiseFunctor::Params elementwise;
int64_t batch_stride_alpha;
int64_t batch_stride_C;
int64_t batch_stride_D;
//
// Methods
//
CUTLASS_HOST_DEVICE
Params() {}
CUTLASS_HOST_DEVICE
Params(Arguments const& args)
: elementwise(args.elementwise),
batch_stride_alpha(args.batch_stride_alpha),
batch_stride_C(args.batch_stride_C),
batch_stride_D(args.batch_stride_D) {}
};
/// Shared storage
struct SharedStorage {};
private:
Params const& params_;
SharedStorage& shared_storage_;
MatrixCoord extent_;
MatrixCoord extent_real_;
ElementwiseFunctor elementwise_;
bool const with_bias_;
bool const per_token_quant_;
bool const per_channel_quant_;
AlphaScaleElementType* ptr_alpha_row_;
AlphaScaleElementType* ptr_alpha_col_;
ScaleTileIterator iterator_alpha_col_;
OutputTileIterator iterator_C_;
OutputTileIterator iterator_D_;
AlphaScaleElementType element_alpha_row_ = 1.0f;
AlphaScaleElementType element_alpha_col_ = 1.0f;
typename ScaleTileIterator::Fragment fragment_alpha_col_;
typename OutputTileIterator::Fragment fragment_C_;
typename OutputTileIterator::Fragment fragment_D_;
ElementAccumulator beta_;
int column_offset_;
MatrixCoord thread_offset_;
public:
CUTLASS_DEVICE
EpilogueVisitorPerRowPerCol(
Params const& params,
SharedStorage& shared_storage,
cutlass::MatrixCoord const& problem_size,
int thread_idx,
int warp_idx,
int lane_idx,
typename ScaleTileIterator::Params params_alpha_col,
typename OutputTileIterator::Params params_C,
typename OutputTileIterator::Params params_D,
bool with_bias,
bool per_token_quant,
bool per_channel_quant,
AlphaScaleElementType* ptr_alpha_row,
AlphaScaleElementType* ptr_alpha_col,
typename OutputTileIterator::Element* ptr_C,
typename OutputTileIterator::Element* ptr_D,
cutlass::MatrixCoord const& threadblock_offset = cutlass::MatrixCoord(0, 0),
int column_offset = 0,
cutlass::MatrixCoord const& problem_size_real = cutlass::MatrixCoord(0, 0))
: params_(params),
shared_storage_(shared_storage),
extent_(problem_size),
elementwise_(params.elementwise),
with_bias_(with_bias),
per_token_quant_(per_token_quant),
per_channel_quant_(per_channel_quant),
ptr_alpha_row_(ptr_alpha_row),
ptr_alpha_col_(ptr_alpha_col),
iterator_alpha_col_(params_alpha_col, ptr_alpha_col, problem_size, thread_idx, threadblock_offset),
iterator_C_(params_C, ptr_C, problem_size, thread_idx, threadblock_offset),
iterator_D_(params_D, ptr_D, problem_size, thread_idx, threadblock_offset),
extent_real_(problem_size_real) {
if (!per_channel_quant_ && (ptr_alpha_col_ != nullptr)) {
element_alpha_col_ = *ptr_alpha_col_;
}
if (!per_token_quant_ && (ptr_alpha_row_ != nullptr)) {
element_alpha_row_ = *ptr_alpha_row_;
}
}
/// Helper to indicate split-K behavior
CUTLASS_DEVICE
void set_k_partition(
int split_k_index, ///< Index of this threadblock within split-K partitioned scheme
int split_k_slices) { ///< Total number of split-K slices
}
/// Called to set the batch index
CUTLASS_DEVICE
void set_batch_index(int batch_idx) {
iterator_alpha_col_.add_pointer_offset(batch_idx * params_.batch_stride_alpha);
iterator_C_.add_pointer_offset(batch_idx * params_.batch_stride_C);
iterator_D_.add_pointer_offset(batch_idx * params_.batch_stride_D);
}
/// Called at the start of the epilogue just before iterating over accumulator slices
CUTLASS_DEVICE
void begin_epilogue() {
if (per_channel_quant_) {
iterator_alpha_col_.load(fragment_alpha_col_);
}
if (with_bias_) {
iterator_C_.load(fragment_C_);
}
}
/// Called at the start of one step before starting accumulator exchange
CUTLASS_DEVICE
void begin_step(int step_idx) {
fragment_D_.clear();
}
/// Called at the start of a row
CUTLASS_DEVICE
void begin_row(int row_idx) {
// load alpha_row in begin_step only when per token(row) scaling is used
if (per_token_quant_) {
int thread_offset_row =
iterator_D_.thread_start_row() + OutputTileIterator::ThreadMap::iteration_offset(row_idx).row();
arch::global_load<AlphaScaleElementType, sizeof(AlphaScaleElementType)>(
element_alpha_row_, ptr_alpha_row_ + thread_offset_row, thread_offset_row < extent_.row());
}
}
/// Called after accumulators have been exchanged for each accumulator vector
CUTLASS_DEVICE
void visit(int iter_idx, int row_idx, int column_idx, int frag_idx, AccumulatorFragment const& accum) {
NumericArrayConverter<ElementCompute, ElementAccumulator, kElementsPerAccess> source_converter;
ComputeFragment result = source_converter(accum);
if (per_channel_quant_) {
ComputeFragment alpha_col = reinterpret_cast<ComputeFragment*>(&fragment_alpha_col_)[column_idx];
result = per_token_channel_scale_accumulator_(result, alpha_col, element_alpha_row_);
} else {
result = per_token_scale_accumulator_(result, element_alpha_col_, element_alpha_row_);
}
if (with_bias_) {
NumericArrayConverter<ElementCompute, ElementOutput, kElementsPerAccess> bias_converter;
OutputVector bias = reinterpret_cast<OutputVector*>(&fragment_C_)[column_idx];
result = bias_accumulator_(result, bias_converter(bias));
}
// Convert to the output
NumericArrayConverter<ElementOutput, ElementCompute, kElementsPerAccess> output_converter;
OutputVector& output = reinterpret_cast<OutputVector*>(&fragment_D_)[frag_idx];
output = output_converter(result);
}
/// Called at the end of a row
CUTLASS_DEVICE
void end_row(int row_idx) {}
/// Called after all accumulator elements have been visited
CUTLASS_DEVICE
void end_step(int step_idx) {
iterator_D_.store(fragment_D_);
++iterator_D_;
}
/// Called after all steps have been completed
CUTLASS_DEVICE
void end_epilogue() {}
private:
CUTLASS_DEVICE
ComputeFragment per_token_channel_scale_accumulator_(
ComputeFragment const& accum, ComputeFragment const& scale_col, AlphaScaleElementType const& scale_row) {
ComputeFragment result;
CUTLASS_PRAGMA_UNROLL
for (int i = 0; i < ComputeFragment::kElements; ++i) {
result[i] = accum[i] * (scale_col[i] * scale_row);
}
return result;
}
CUTLASS_DEVICE
ComputeFragment per_token_scale_accumulator_(
ComputeFragment const& accum, AlphaScaleElementType const& scale_col, AlphaScaleElementType const& scale_row) {
ComputeFragment result;
CUTLASS_PRAGMA_UNROLL
for (int i = 0; i < ComputeFragment::kElements; ++i) {
result[i] = accum[i] * (scale_col * scale_row);
}
return result;
}
CUTLASS_DEVICE
ComputeFragment bias_accumulator_(ComputeFragment const& accum, ComputeFragment const& bias) {
ComputeFragment result;
CUTLASS_PRAGMA_UNROLL
for (int i = 0; i < OutputVector::kElements; ++i) {
result[i] = accum[i] + bias[i];
}
return result;
}
};
} // namespace threadblock
} // namespace epilogue
} // namespace cutlass
@@ -0,0 +1,388 @@
/* Copyright 2026 SGLang Team. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// Adapted from
// https://github.com/NVIDIA/TensorRT-LLM/blob/be1788106245496872d18e702978e59b6bfd50e0/cpp/tensorrt_llm/cutlass_extensions/include/cutlass_extensions/gemm/device/gemm_universal_base_compat.h
// Modifications: clang-format reflow, includes trimmed to CUTLASS, no tensorrt_llm dependency.
/***************************************************************************************************
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
#pragma once
#include <cutlass/cutlass.h>
#include <cutlass/device_kernel.h>
#include <cutlass/trace.h>
////////////////////////////////////////////////////////////////////////////////
namespace cutlass {
namespace gemm {
namespace device {
/////////////////////////////////////////////////////////////////////////////////////////////////
/*
This is the device layer from CUTLASS 2.10 (SHA - cc85b64cf676c45f98a17e3a47c0aafcf817f088)
It is replicated here since we needed to duplicate kernel level APIs for mixed dtype GEMMs
and SmoothQuant. The newer device layer is not compatible with these older kernel level APIs.
Note: While CUTLASS 3.x supports stream-k, none of the kernels in the extensions folder support
that feature at the moment.
*/
template <typename GemmKernel_>
class GemmUniversalBaseCompat {
public:
using GemmKernel = GemmKernel_;
using ThreadblockShape = typename GemmKernel::Mma::Shape;
using ElementA = typename GemmKernel::ElementA;
using LayoutA = typename GemmKernel::LayoutA;
using TensorRefA = TensorRef<ElementA const, LayoutA>;
static ComplexTransform const kTransformA = GemmKernel::kTransformA;
using ElementB = typename GemmKernel::ElementB;
using LayoutB = typename GemmKernel::LayoutB;
using TensorRefB = TensorRef<ElementB const, LayoutB>;
static ComplexTransform const kTransformB = GemmKernel::kTransformB;
using ElementC = typename GemmKernel::ElementC;
using LayoutC = typename GemmKernel::LayoutC;
using TensorRefC = TensorRef<ElementC const, LayoutC>;
using TensorRefD = TensorRef<ElementC, LayoutC>;
using ElementAccumulator = typename GemmKernel::Mma::Policy::Operator::ElementC;
using EpilogueOutputOp = typename GemmKernel::EpilogueOutputOp;
using ThreadblockSwizzle = typename GemmKernel::ThreadblockSwizzle;
using Operator = typename GemmKernel::Operator;
/// Argument structure
using Arguments = typename GemmKernel::Arguments;
protected:
/// Kernel parameters object
typename GemmKernel::Params params_;
protected:
/// Private helper to obtain the grid dimensions with fix-up for split-K
static void get_grid_shape_(gemm::GemmCoord& grid_tiled_shape, int& gemm_k_size, Arguments const& args) {
// Determine grid shape
ThreadblockSwizzle threadblock_swizzle;
grid_tiled_shape = threadblock_swizzle.get_tiled_shape(
args.problem_size, {ThreadblockShape::kM, ThreadblockShape::kN, ThreadblockShape::kK}, args.batch_count);
gemm_k_size = args.problem_size.k();
if (args.mode == GemmUniversalMode::kGemm || args.mode == GemmUniversalMode::kGemmSplitKParallel) {
int const kAlignK =
const_max(const_max(128 / sizeof_bits<ElementA>::value, 128 / sizeof_bits<ElementB>::value), 1);
gemm_k_size = round_up(ceil_div(args.problem_size.k(), args.batch_count), kAlignK);
if (gemm_k_size) {
grid_tiled_shape.k() = ceil_div(args.problem_size.k(), gemm_k_size);
}
}
}
public:
/// Constructs the GEMM.
GemmUniversalBaseCompat() {}
/// Determines whether the GEMM can execute the given problem.
static Status can_implement(Arguments const& args) {
// Determine grid shape
cutlass::gemm::GemmCoord grid_tiled_shape;
int gemm_k_size = 0;
get_grid_shape_(grid_tiled_shape, gemm_k_size, args);
ThreadblockSwizzle threadblock_swizzle;
dim3 grid = threadblock_swizzle.get_grid_shape(grid_tiled_shape);
uint32_t const kGridYZMax = ((1 << (sizeof(uint16_t) * 8)) - 1);
if (!(grid.y <= kGridYZMax && grid.z <= kGridYZMax)) {
return Status::kErrorInvalidProblem;
}
return GemmKernel::can_implement(args);
}
/// Gets the workspace size
static size_t get_workspace_size(Arguments const& args) {
CUTLASS_TRACE_HOST("GemmUniversalBaseCompat::get_workspace_size()");
size_t workspace_bytes = 0;
// Determine grid shape
cutlass::gemm::GemmCoord grid_tiled_shape;
int gemm_k_size = 0;
get_grid_shape_(grid_tiled_shape, gemm_k_size, args);
if (args.mode == GemmUniversalMode::kGemmSplitKParallel) {
// Split-K parallel always requires a temporary workspace
workspace_bytes = sizeof(ElementC) * size_t(args.batch_stride_D) * size_t(grid_tiled_shape.k());
} else if (args.mode == GemmUniversalMode::kGemm && grid_tiled_shape.k() > 1) {
// Serial split-K only requires a temporary workspace if the number of partitions along the
// GEMM K dimension is greater than one.
workspace_bytes = sizeof(int) * size_t(grid_tiled_shape.m()) * size_t(grid_tiled_shape.n());
}
CUTLASS_TRACE_HOST(" workspace_bytes: " << workspace_bytes);
workspace_bytes += GemmKernel::get_extra_workspace_size(args, grid_tiled_shape);
return workspace_bytes;
}
/// Computes the grid shape
static dim3 get_grid_shape(Arguments const& args) {
CUTLASS_TRACE_HOST("GemmUniversalBaseCompat::get_grid_shape()");
ThreadblockSwizzle threadblock_swizzle;
cutlass::gemm::GemmCoord grid_tiled_shape;
int gemm_k_size = 0;
get_grid_shape_(grid_tiled_shape, gemm_k_size, args);
dim3 result = threadblock_swizzle.get_grid_shape(grid_tiled_shape);
CUTLASS_TRACE_HOST(
" grid_tiled_shape: " << grid_tiled_shape << "\n"
<< " result = {" << result << "}");
return result;
}
/// Computes the maximum number of active blocks per multiprocessor
static int maximum_active_blocks(int smem_capacity = -1) {
CUTLASS_TRACE_HOST("GemmUniversalBaseCompat::maximum_active_blocks()");
int max_active_blocks = -1;
int smem_size = int(sizeof(typename GemmKernel::SharedStorage));
CUTLASS_TRACE_HOST(" smem_size: " << smem_size << " bytes");
if (smem_size <= (48 << 10)) {
cudaError_t result = cudaOccupancyMaxActiveBlocksPerMultiprocessor(
&max_active_blocks, Kernel<GemmKernel>, GemmKernel::kThreadCount, smem_size);
if (result == cudaSuccess) {
CUTLASS_TRACE_HOST(" max_active_blocks: " << max_active_blocks);
return max_active_blocks;
}
} else {
// Query assuming zero shared memory then compute occupancy limit based on SMEM
cudaError_t result = cudaOccupancyMaxActiveBlocksPerMultiprocessor(
&max_active_blocks, Kernel<GemmKernel>, GemmKernel::kThreadCount, 0);
if (result != cudaSuccess) {
CUTLASS_TRACE_HOST(
" cudaOccupancyMaxActiveBlocksPerMultiprocessor() returned error " << cudaGetErrorString(result));
return -1;
}
if (smem_capacity < 0) {
int device_idx = 0;
result = cudaGetDevice(&device_idx);
if (result != cudaSuccess) {
return -1;
}
cudaDeviceProp properties;
result = cudaGetDeviceProperties(&properties, device_idx);
if (result != cudaSuccess) {
return -1;
}
smem_capacity = static_cast<int>(properties.sharedMemPerMultiprocessor);
}
int occupancy = std::min(max_active_blocks, smem_capacity / smem_size);
CUTLASS_TRACE_HOST(" occupancy: " << occupancy);
return occupancy;
}
CUTLASS_TRACE_HOST(" returning internal error");
return -1;
}
/// Initializes GEMM state from arguments.
Status initialize(Arguments const& args, void* workspace = nullptr, cudaStream_t stream = nullptr) {
CUTLASS_TRACE_HOST(
"GemmUniversalBaseCompat::initialize() - workspace " << workspace
<< ", stream: " << (stream ? "non-null" : "null"));
size_t workspace_bytes = get_workspace_size(args);
CUTLASS_TRACE_HOST(" workspace_bytes: " << workspace_bytes);
if (workspace_bytes) {
if (!workspace) {
CUTLASS_TRACE_HOST(" error: device workspace must not be null");
return Status::kErrorWorkspaceNull;
}
if (args.mode == GemmUniversalMode::kGemm) {
CUTLASS_TRACE_HOST(" clearing device workspace");
cudaError_t result = cudaMemsetAsync(workspace, 0, workspace_bytes, stream);
if (result != cudaSuccess) {
CUTLASS_TRACE_HOST(" cudaMemsetAsync() returned error " << cudaGetErrorString(result));
return Status::kErrorInternal;
}
}
}
// Get CUDA grid shape
cutlass::gemm::GemmCoord grid_tiled_shape;
int gemm_k_size = 0;
get_grid_shape_(grid_tiled_shape, gemm_k_size, args);
// Initialize the Params structure
params_ = typename GemmKernel::Params(args, grid_tiled_shape, gemm_k_size, static_cast<int*>(workspace));
// Specify shared memory capacity for kernel.
int smem_size = int(sizeof(typename GemmKernel::SharedStorage));
if (smem_size >= (48 << 10)) {
cudaError_t result =
cudaFuncSetAttribute(Kernel<GemmKernel>, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size);
if (result != cudaSuccess) {
return Status::kErrorInternal;
}
}
return Status::kSuccess;
}
/// Lightweight update given a subset of arguments
Status update(Arguments const& args, void* workspace = nullptr) {
CUTLASS_TRACE_HOST("GemmUniversalBaseCompat()::update() - workspace: " << workspace);
size_t workspace_bytes = get_workspace_size(args);
if (workspace_bytes && !workspace) {
return Status::kErrorWorkspaceNull;
}
params_.update(args, workspace);
return Status::kSuccess;
}
/// Runs the kernel using initialized state.
Status run(cudaStream_t stream = nullptr) {
CUTLASS_TRACE_HOST("GemmUniversalBaseCompat::run()");
//
// Configure grid and block dimensions
//
ThreadblockSwizzle threadblock_swizzle;
dim3 grid = threadblock_swizzle.get_grid_shape(params_.grid_tiled_shape);
dim3 block(GemmKernel::kThreadCount, 1, 1);
int smem_size = int(sizeof(typename GemmKernel::SharedStorage));
//
// Launch kernel
//
CUTLASS_TRACE_HOST(" grid: (" << grid << "), block: (" << block << "), SMEM: " << smem_size << " bytes");
// Launch
cutlass::Kernel<GemmKernel><<<grid, block, smem_size, stream>>>(params_);
//
// Query for errors
//
cudaError_t result = cudaGetLastError();
if (result != cudaSuccess) {
CUTLASS_TRACE_HOST(" grid launch failed with error " << cudaGetErrorString(result));
return Status::kErrorInternal;
}
return Status::kSuccess;
}
/// Runs the kernel using initialized state.
Status operator()(cudaStream_t stream = nullptr) {
return run(stream);
}
/// Runs the kernel using initialized state.
Status operator()(Arguments const& args, void* workspace = nullptr, cudaStream_t stream = nullptr) {
Status status = initialize(args, workspace, stream);
if (status == Status::kSuccess) {
status = run(stream);
}
return status;
}
};
/////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace device
} // namespace gemm
} // namespace cutlass
/////////////////////////////////////////////////////////////////////////////////////////////////
@@ -0,0 +1,524 @@
/* Copyright 2026 SGLang Team. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// Adapted from
// https://github.com/NVIDIA/TensorRT-LLM/blob/be1788106245496872d18e702978e59b6bfd50e0/cpp/tensorrt_llm/cutlass_extensions/include/cutlass_extensions/gemm/kernel/gemm_with_epilogue_visitor.h
// Modifications: clang-format reflow, includes trimmed to CUTLASS, no tensorrt_llm dependency.
/***************************************************************************************************
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
#pragma once
#include <cutlass/complex.h>
#include <cutlass/cutlass.h>
#include <cutlass/fast_math.h>
#include <cutlass/matrix_coord.h>
#include <cutlass/trace.h>
/////////////////////////////////////////////////////////////////////////////////////////////////
namespace cutlass {
namespace gemm {
namespace kernel {
/////////////////////////////////////////////////////////////////////////////////////////////////
template <
typename Mma_, ///! Threadblock-scoped matrix multiply-accumulate
typename Epilogue_, ///! Epilogue
typename ThreadblockSwizzle_ ///! Threadblock swizzling function
>
struct GemmWithEpilogueVisitor {
public:
using Mma = Mma_;
using Epilogue = Epilogue_;
using EpilogueVisitor = typename Epilogue::Visitor;
using ThreadblockSwizzle = ThreadblockSwizzle_;
using ElementA = typename Mma::IteratorA::Element;
using LayoutA = typename Mma::IteratorA::Layout;
using TensorRefA = TensorRef<ElementA, LayoutA>;
using ElementB = typename Mma::IteratorB::Element;
using LayoutB = typename Mma::IteratorB::Layout;
using TensorRefB = TensorRef<ElementB, LayoutB>;
using ElementCompute = typename EpilogueVisitor::ElementCompute;
using LayoutAlphaCol = cutlass::layout::RowMajor;
using LayoutAlphaRow = cutlass::layout::ColumnMajor;
using TensorRefAlphaCol = TensorRef<ElementCompute, LayoutAlphaCol>;
using TensorRefAlphaRow = TensorRef<ElementCompute, LayoutAlphaRow>;
using ElementC = typename EpilogueVisitor::ElementOutput;
using LayoutC = typename Epilogue::Layout;
using TensorRefC = TensorRef<ElementC, LayoutC>;
static ComplexTransform const kTransformA = Mma::kTransformA;
static ComplexTransform const kTransformB = Mma::kTransformB;
using Operator = typename Mma::Operator;
using OperatorClass = typename Mma::Operator::OperatorClass;
using ThreadblockShape = typename Mma::Shape;
using WarpShape = typename Mma::Operator::Shape;
using InstructionShape = typename Mma::Policy::Operator::InstructionShape;
using ArchTag = typename Mma::ArchTag;
using EpilogueOutputOp =
typename Epilogue::Visitor::ElementwiseFunctor; // Define type so GemmUniversalBase doesn't complain
static int const kStages = Mma::kStages;
static int const kAlignmentA = Mma::IteratorA::AccessType::kElements;
static int const kAlignmentB = Mma::IteratorB::AccessType::kElements;
static int const kAlignmentC = EpilogueVisitor::kElementsPerAccess;
/// Warp count (concept: GemmShape)
using WarpCount = typename Mma::WarpCount;
static int const kThreadCount = 32 * WarpCount::kCount;
/// Split-K preserves splits that are 128b aligned
static int const kSplitKAlignment = const_max(128 / sizeof_bits<ElementA>::value, 128 / sizeof_bits<ElementB>::value);
//
// Structures
//
/// Argument structure
struct Arguments {
//
// Data members
//
GemmUniversalMode mode;
GemmCoord problem_size;
int batch_count;
TensorRefA ref_A;
TensorRefB ref_B;
TensorRefAlphaCol ref_alpha_col;
TensorRefAlphaRow ref_alpha_row;
TensorRefC ref_C;
TensorRefC ref_D;
int64_t batch_stride_A;
int64_t batch_stride_B;
int64_t batch_stride_D;
typename EpilogueVisitor::Arguments epilogue_visitor;
//
// Methods
//
Arguments() : mode(GemmUniversalMode::kGemm), batch_count(1) {}
/// constructs an arguments structure
Arguments(
GemmCoord problem_size_,
TensorRefA ref_A_,
TensorRefB ref_B_,
TensorRefAlphaCol ref_alpha_col_,
TensorRefAlphaRow ref_alpha_row_,
TensorRefC ref_C_,
TensorRefC ref_D_,
typename EpilogueVisitor::Arguments epilogue_visitor_)
: mode(GemmUniversalMode::kGemm),
problem_size(problem_size_),
batch_count(1),
ref_A(ref_A_),
ref_B(ref_B_),
ref_alpha_col(ref_alpha_col_),
ref_alpha_row(ref_alpha_row_),
ref_C(ref_C_),
ref_D(ref_D_),
batch_stride_A(0),
batch_stride_B(0),
batch_stride_D(0),
epilogue_visitor(epilogue_visitor_) {}
};
//
// Structure for precomputing values in host memory and passing to kernels
//
/// Parameters structure
struct Params {
cutlass::gemm::GemmCoord problem_size;
cutlass::gemm::GemmCoord grid_tiled_shape;
int swizzle_log_tile;
typename Mma::IteratorA::Params params_A;
typename Mma::IteratorB::Params params_B;
typename EpilogueVisitor::ScaleTileIterator::Params params_alpha_col;
typename EpilogueVisitor::ScaleTileIterator::Params params_alpha_row;
typename EpilogueVisitor::OutputTileIterator::Params params_C;
typename EpilogueVisitor::OutputTileIterator::Params params_D;
GemmUniversalMode mode;
int batch_count;
int gemm_k_size;
void* ptr_A;
void* ptr_B;
typename EpilogueVisitor::ScaleTileIterator::Element* ptr_alpha_col;
typename EpilogueVisitor::ScaleTileIterator::Element* ptr_alpha_row;
ElementC* ptr_C;
ElementC* ptr_D;
int64_t batch_stride_A;
int64_t batch_stride_B;
typename EpilogueVisitor::Params epilogue_visitor;
//
// Methods
//
CUTLASS_HOST_DEVICE
Params()
: swizzle_log_tile(0),
params_A(0),
params_B(0),
params_alpha_col(0),
params_C(0),
params_D(0),
batch_count(0),
gemm_k_size(0),
mode(cutlass::gemm::GemmUniversalMode::kGemm),
ptr_A(nullptr),
ptr_B(nullptr),
ptr_alpha_col(nullptr),
ptr_alpha_row(nullptr),
ptr_C(nullptr),
ptr_D(nullptr),
batch_stride_A(0),
batch_stride_B(0) {}
Params(Arguments const& args, cutlass::gemm::GemmCoord const& grid_tiled_shape_, int gemm_k_size_, int* workspace_)
: problem_size(args.problem_size),
swizzle_log_tile(0),
params_A(args.ref_A.layout()),
params_B(args.ref_B.layout()),
params_alpha_col(args.ref_alpha_col.layout()),
params_alpha_row(args.ref_alpha_col.layout()),
params_C(args.ref_C.layout()),
params_D(args.ref_D.layout()),
mode(args.mode),
batch_count(args.batch_count),
gemm_k_size(args.problem_size.k()),
ptr_A(args.ref_A.data()),
ptr_B(args.ref_B.data()),
ptr_alpha_col(args.ref_alpha_col.data()),
ptr_alpha_row(args.ref_alpha_row.data()),
ptr_C(args.ref_C.data()),
ptr_D(args.ref_D.data()),
batch_stride_A(args.batch_stride_A),
batch_stride_B(args.batch_stride_B),
epilogue_visitor(args.epilogue_visitor) {
ThreadblockSwizzle threadblock_swizzle;
grid_tiled_shape = threadblock_swizzle.get_tiled_shape(
args.problem_size, {ThreadblockShape::kM, ThreadblockShape::kN, ThreadblockShape::kK}, args.batch_count);
if (args.mode == GemmUniversalMode::kGemm || args.mode == GemmUniversalMode::kGemmSplitKParallel) {
int const kAlignK =
const_max(const_max(128 / sizeof_bits<ElementA>::value, 128 / sizeof_bits<ElementB>::value), 1);
gemm_k_size = round_up(ceil_div(args.problem_size.k(), args.batch_count), kAlignK);
if (gemm_k_size) {
grid_tiled_shape.k() = ceil_div(args.problem_size.k(), gemm_k_size);
}
}
swizzle_log_tile = threadblock_swizzle.get_log_tile(grid_tiled_shape);
}
};
/// Shared memory storage structure
union SharedStorage {
typename Mma::SharedStorage main_loop;
struct {
typename Epilogue::SharedStorage epilogue;
typename EpilogueVisitor::SharedStorage visitor;
} epilogue;
};
public:
//
// Methods
//
CUTLASS_DEVICE
GemmWithEpilogueVisitor() {}
/// Determines whether kernel satisfies alignment
static Status can_implement(cutlass::gemm::GemmCoord const& problem_size) {
CUTLASS_TRACE_HOST("GemmWithEpilogueVisitor::can_implement()");
static int const kAlignmentA = Mma::IteratorA::AccessType::kElements;
static int const kAlignmentB = Mma::IteratorB::AccessType::kElements;
static int const kAlignmentC = EpilogueVisitor::OutputTileIterator::kElementsPerAccess;
bool isAMisaligned = false;
bool isBMisaligned = false;
bool isCMisaligned = false;
if (platform::is_same<LayoutA, layout::RowMajor>::value) {
isAMisaligned = problem_size.k() % kAlignmentA;
} else if (platform::is_same<LayoutA, layout::ColumnMajor>::value) {
isAMisaligned = problem_size.m() % kAlignmentA;
} else if (
platform::is_same<LayoutA, layout::ColumnMajorInterleaved<32>>::value ||
platform::is_same<LayoutA, layout::ColumnMajorInterleaved<64>>::value) {
isAMisaligned = problem_size.k() % kAlignmentA;
}
if (platform::is_same<LayoutB, layout::RowMajor>::value) {
isBMisaligned = problem_size.n() % kAlignmentB;
} else if (platform::is_same<LayoutB, layout::ColumnMajor>::value) {
isBMisaligned = problem_size.k() % kAlignmentB;
} else if (
platform::is_same<LayoutB, layout::RowMajorInterleaved<32>>::value ||
platform::is_same<LayoutB, layout::RowMajorInterleaved<64>>::value) {
isBMisaligned = problem_size.k() % kAlignmentB;
}
if (platform::is_same<LayoutC, layout::RowMajor>::value) {
isCMisaligned = problem_size.n() % kAlignmentC;
} else if (platform::is_same<LayoutC, layout::ColumnMajor>::value) {
isCMisaligned = problem_size.m() % kAlignmentC;
} else if (
platform::is_same<LayoutC, layout::ColumnMajorInterleaved<32>>::value ||
platform::is_same<LayoutC, layout::ColumnMajorInterleaved<64>>::value) {
isCMisaligned = problem_size.n() % kAlignmentC;
}
if (isAMisaligned) {
CUTLASS_TRACE_HOST(" returning kErrorMisalignedOperand for A operand");
return Status::kErrorMisalignedOperand;
}
if (isBMisaligned) {
CUTLASS_TRACE_HOST(" returning kErrorMisalignedOperand for B operand");
return Status::kErrorMisalignedOperand;
}
if (isCMisaligned) {
CUTLASS_TRACE_HOST(" returning kErrorMisalignedOperand for C operand");
return Status::kErrorMisalignedOperand;
}
CUTLASS_TRACE_HOST(" returning kSuccess");
return Status::kSuccess;
}
static Status can_implement(Arguments const& args) {
return can_implement(args.problem_size);
}
static size_t get_extra_workspace_size(Arguments const& args, cutlass::gemm::GemmCoord const& grid_tiled_shape) {
return 0;
}
#define SPLIT_K_ENABLED 1
/// Executes one GEMM
CUTLASS_DEVICE
void run_kernel_(Params const& params, SharedStorage& shared_storage) {
// Compute threadblock location
ThreadblockSwizzle threadblock_swizzle;
cutlass::gemm::GemmCoord threadblock_tile_offset = threadblock_swizzle.get_tile_offset(params.swizzle_log_tile);
// Early exit if CTA is out of range
if (params.grid_tiled_shape.m() <= threadblock_tile_offset.m() ||
params.grid_tiled_shape.n() <= threadblock_tile_offset.n()) {
return;
}
int offset_k = 0;
int problem_size_k = params.problem_size.k();
ElementA* ptr_A = static_cast<ElementA*>(params.ptr_A);
ElementB* ptr_B = static_cast<ElementB*>(params.ptr_B);
#if SPLIT_K_ENABLED
//
// Fetch pointers based on mode.
//
if (params.mode == GemmUniversalMode::kGemm || params.mode == GemmUniversalMode::kGemmSplitKParallel) {
if (threadblock_tile_offset.k() + 1 < params.grid_tiled_shape.k()) {
problem_size_k = (threadblock_tile_offset.k() + 1) * params.gemm_k_size;
}
offset_k = threadblock_tile_offset.k() * params.gemm_k_size;
} else if (params.mode == GemmUniversalMode::kBatched) {
ptr_A += threadblock_tile_offset.k() * params.batch_stride_A;
ptr_B += threadblock_tile_offset.k() * params.batch_stride_B;
} else if (params.mode == GemmUniversalMode::kArray) {
ptr_A = static_cast<ElementA* const*>(params.ptr_A)[threadblock_tile_offset.k()];
ptr_B = static_cast<ElementB* const*>(params.ptr_B)[threadblock_tile_offset.k()];
}
#endif
// Compute initial location in logical coordinates
cutlass::MatrixCoord tb_offset_A{
threadblock_tile_offset.m() * Mma::Shape::kM,
offset_k,
};
cutlass::MatrixCoord tb_offset_B{offset_k, threadblock_tile_offset.n() * Mma::Shape::kN};
// Compute position within threadblock
int thread_idx = threadIdx.x;
// Construct iterators to A and B operands
typename Mma::IteratorA iterator_A(
params.params_A, ptr_A, {params.problem_size.m(), problem_size_k}, thread_idx, tb_offset_A);
typename Mma::IteratorB iterator_B(
params.params_B, ptr_B, {problem_size_k, params.problem_size.n()}, thread_idx, tb_offset_B);
// Broadcast the warp_id computed by lane 0 to ensure dependent code
// is compiled as warp-uniform.
int warp_idx = __shfl_sync(0xffffffff, threadIdx.x / 32, 0);
int lane_idx = threadIdx.x % 32;
//
// Main loop
//
// Construct thread-scoped matrix multiply
Mma mma(shared_storage.main_loop, thread_idx, warp_idx, lane_idx);
typename Mma::FragmentC accumulators;
accumulators.clear();
// Compute threadblock-scoped matrix multiply-add
int gemm_k_iterations = (problem_size_k - offset_k + Mma::Shape::kK - 1) / Mma::Shape::kK;
// Compute threadblock-scoped matrix multiply-add
mma(gemm_k_iterations, accumulators, iterator_A, iterator_B, accumulators);
//
// Masked tile iterators constructed from members
//
threadblock_tile_offset = threadblock_swizzle.get_tile_offset(params.swizzle_log_tile);
// assume identity swizzle
MatrixCoord threadblock_offset(
threadblock_tile_offset.m() * Mma::Shape::kM, threadblock_tile_offset.n() * Mma::Shape::kN);
int block_idx = threadblock_tile_offset.m() + threadblock_tile_offset.n() * params.grid_tiled_shape.m();
//
// Construct the epilogue visitor
//
bool with_bias = true;
if (params.ptr_C == nullptr) {
with_bias = false;
}
EpilogueVisitor epilogue_visitor(
params.epilogue_visitor,
shared_storage.epilogue.visitor,
params.problem_size.mn(),
thread_idx,
warp_idx,
lane_idx,
params.params_alpha_col,
params.params_C,
params.params_D,
with_bias,
true,
true,
params.ptr_alpha_row,
params.ptr_alpha_col,
params.ptr_C,
params.ptr_D,
threadblock_offset,
blockIdx.y * params.problem_size.m());
if (params.mode == GemmUniversalMode::kGemm) {
// Indicate which position in a serial reduction the output operator is currently updating
epilogue_visitor.set_k_partition(threadblock_tile_offset.k(), params.grid_tiled_shape.k());
} else if (params.mode == GemmUniversalMode::kBatched || params.mode == GemmUniversalMode::kArray) {
epilogue_visitor.set_batch_index(threadblock_tile_offset.k());
}
// Construct the epilogue
Epilogue epilogue(shared_storage.epilogue.epilogue, thread_idx, warp_idx, lane_idx);
// Execute the epilogue operator to update the destination tensor.
epilogue(epilogue_visitor, accumulators);
}
template <typename CompilationArch>
CUTLASS_DEVICE void run_kernel(Params const& params, SharedStorage& shared_storage) {
if constexpr (platform::is_same<ArchTag, CompilationArch>::value) {
run_kernel_(params, shared_storage);
} else {
CUTLASS_NOT_IMPLEMENTED();
}
}
/// Executes one GEMM
CUTLASS_DEVICE
void operator()(Params const& params, SharedStorage& shared_storage) {
run_kernel<ArchTag>(params, shared_storage);
}
};
/////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace kernel
} // namespace gemm
} // namespace cutlass
/////////////////////////////////////////////////////////////////////////////////////////////////
@@ -480,6 +480,50 @@ register_kernel(
),
)
)
# Mirrors SUPPORTED_CAPABILITIES in ops.gemm.convrot_int8 (pinned by a test).
_CONVROT_INT8 = frozenset(
CapabilityRequirement.cuda(min_sm=sm, max_sm=sm)
for sm in ((9, 0), (10, 0), (12, 0), (12, 1))
)
register_kernel(
KernelSpec(
op="gemm.convrot_rotate_quantize_activation",
backend=KernelBackend.JIT,
target="sglang.kernels.ops.gemm.convrot_int8:convrot_rotate_quantize_activation",
capabilities=_CONVROT_INT8,
format_signature=FormatSignature(
supported_dtypes=("bfloat16",),
description="group-wise regular Hadamard rotation + per-row INT8 quant of BF16 [M, K] -> (int8 [M, K], fp32 [M])",
),
description="ConvRot INT8 activation / weight quantizer (sglang.kernels.jit).",
)
)
register_kernel(
KernelSpec(
op="gemm.convrot_int8_fused_linear",
backend=KernelBackend.JIT,
target="sglang.kernels.ops.gemm.convrot_int8:convrot_int8_fused_linear",
capabilities=_CONVROT_INT8,
format_signature=FormatSignature(
supported_dtypes=("bfloat16", "int8"),
description="BF16 [M, K] x int8 [N, K] -> BF16 [M, N] with in-kernel rotation, INT8 quant and fused per-row x per-column dequant (+ bias)",
),
description="ConvRot INT8 W8A8 linear (sglang.kernels.jit).",
)
)
register_kernel(
KernelSpec(
op="gemm.convrot_int8_linear_prequant",
backend=KernelBackend.JIT,
target="sglang.kernels.ops.gemm.convrot_int8:convrot_int8_linear_prequant",
capabilities=_CONVROT_INT8,
format_signature=FormatSignature(
supported_dtypes=("int8", "float32", "bfloat16"),
description="int8 [M, K] x int8 [N, K] with per-row / per-column fp32 scales -> BF16 [M, N] (+ bias); for linears sharing one rotated input",
),
description="ConvRot INT8 GEMM on a pre-quantized activation (sglang.kernels.jit).",
)
)
register_kernel(
KernelSpec(
op="gemm.sm120_fp8_gemv",
@@ -0,0 +1,231 @@
"""ConvRot INT8 W8A8 linear (arXiv:2512.03673), JIT-compiled for the local GPU.
One entry rotates a BF16 activation group-wise with the regular Hadamard,
quantizes it per row to INT8 and runs the dense INT8 GEMM with the per-row x
per-column dequant (and bias) fused into the epilogue; the rotate step alone
is the offline transform for the [N, K] weight. CUTLASS 3.x WGMMA on CC 9.0,
tcgen05 on CC 10.0 and the CUTLASS 2.x mma.sync path on CC 12.0 / 12.1; the
module compiles once per machine on first use and is cached by the JIT.
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import torch
from sglang.kernels.jit.utils import cache_once, get_jit_cuda_arch, load_jit
from sglang.kernels.jit.utils.common import is_hip_runtime, is_musa_runtime
from sglang.kernels.kernel_api_logging import debug_kernel_api
from sglang.srt.utils.custom_op import register_custom_op
if TYPE_CHECKING:
from tvm_ffi.module import Module
# Exact compute capabilities the entry header carries a GEMM path for: sm_90a
# and sm_100a hold the WGMMA / tcgen05 kernels, CC 12.0 / 12.1 the mma.sync
# path. CC 10.3 is left out on purpose (INT8 tensor-core rate); the quant
# method reports the reason.
SUPPORTED_CAPABILITIES: frozenset[tuple[int, int]] = frozenset(
{(9, 0), (10, 0), (12, 0), (12, 1)}
)
# Hadamard group widths the rotate kernel is instantiated for.
GROUP_SIZES: tuple[int, ...] = (64, 128, 256, 512)
def convrot_int8_supported_capabilities() -> frozenset[tuple[int, int]]:
"""(major, minor) pairs the convrot_int8 ops compile for."""
return SUPPORTED_CAPABILITIES
def _convrot_int8_cuda_flags() -> list[str]:
return [
"-DNDEBUG",
"-DCUTE_USE_PACKED_TUPLE=1",
"-DCUTLASS_ENABLE_TENSOR_CORE_MMA=1",
"-DCUTLASS_VERSIONS_GENERATED",
"-DCUTLASS_TEST_LEVEL=0",
"-DCUTLASS_TEST_ENABLE_CACHED_RESULTS=1",
"-DCUTLASS_DEBUG_TRACE_LEVEL=0",
"--expt-relaxed-constexpr",
"--expt-extended-lambda",
# Only the row-scale division and denormal handling see this flag; a
# precise build rounds a few INT8 codes one step away from the
# validated numerics. The GELU tanh is pinned to libdevice __nv_tanhf.
"--use_fast_math",
]
def convrot_int8_local_capability() -> tuple[int, int] | None:
"""(major, minor) the JIT build would target, or None off CUDA (ROCm and
MUSA capability numbers are not SM versions)."""
if is_hip_runtime() or is_musa_runtime():
return None
arch = get_jit_cuda_arch()
return (arch.major, arch.minor)
@cache_once
def _jit_convrot_int8_module() -> Module:
"""Compile and cache the convrot_int8 module for the local GPU."""
capability = convrot_int8_local_capability()
if capability not in SUPPORTED_CAPABILITIES:
supported = ", ".join(f"{a}.{b}" for a, b in sorted(SUPPORTED_CAPABILITIES))
where = (
"a non-CUDA device"
if capability is None
else f"CC {capability[0]}.{capability[1]}"
)
raise RuntimeError(
f"convrot_int8: no kernel for {where} (supported: {supported})"
)
return load_jit(
"convrot_int8",
cuda_files=["gemm/convrot_int8/convrot_int8_entry.cuh"],
cuda_wrappers=[
(
"convrot_rotate_quantize_activation",
"convrot_rotate_quantize_activation",
),
("convrot_int8_fused_linear", "convrot_int8_fused_linear"),
("convrot_int8_linear_prequant", "convrot_int8_linear_prequant"),
],
extra_dependencies=["cutlass"],
extra_cuda_cflags=_convrot_int8_cuda_flags(),
)
def load_convrot_int8_module() -> Module:
"""Build (first use) or load the cached module; raises when the GPU is
unsupported, the CUDA toolchain is missing or the build fails."""
return _jit_convrot_int8_module()
# The custom ops below exist for torch.compile: dynamo needs an opaque op. Eager
# calls go straight to the module instead, because the dispatcher round trip of
# a Python-implemented op costs ~9 us on top of the ~10 us tvm-ffi call
# (measured on GB200) and a Qwen-Image DiT forward issues ~850 of these calls.
@register_custom_op(
op_name="convrot_rotate_quantize_activation",
mutates_args=["x_q", "x_scale"],
)
def _convrot_rotate_quantize_activation_op(
x_q: torch.Tensor,
x_scale: torch.Tensor,
x: torch.Tensor,
group_size: int,
) -> None:
module = _jit_convrot_int8_module()
module.convrot_rotate_quantize_activation(x_q, x_scale, x, group_size)
@register_custom_op(op_name="convrot_int8_fused_linear", mutates_args=["out"])
def _convrot_int8_fused_linear_op(
out: torch.Tensor,
x: torch.Tensor,
weight_q: torch.Tensor,
weight_scale: torch.Tensor,
bias: torch.Tensor | None,
group_size: int,
gelu_input: bool,
) -> None:
module = _jit_convrot_int8_module()
module.convrot_int8_fused_linear(
out, x, weight_q, weight_scale, bias, group_size, gelu_input
)
@register_custom_op(op_name="convrot_int8_linear_prequant", mutates_args=["out"])
def _convrot_int8_linear_prequant_op(
out: torch.Tensor,
xq: torch.Tensor,
xs: torch.Tensor,
weight_q: torch.Tensor,
weight_scale: torch.Tensor,
bias: torch.Tensor | None,
group_size: int,
) -> None:
module = _jit_convrot_int8_module()
module.convrot_int8_linear_prequant(
out, xq, xs, weight_q, weight_scale, bias, group_size
)
@debug_kernel_api
def convrot_rotate_quantize_activation(
x: torch.Tensor, group_size: int
) -> tuple[torch.Tensor, torch.Tensor]:
"""Group Hadamard rotation + per-row INT8 quant of a contiguous BF16 [M, K].
Returns (x_q int8 [M, K], x_scale float32 [M]). Also the offline transform
for a [N, K] weight, yielding (weight_q, weight_scale).
"""
x_q = torch.empty(x.shape, dtype=torch.int8, device=x.device)
x_scale = torch.empty(x.shape[0], dtype=torch.float32, device=x.device)
if torch.compiler.is_compiling():
_convrot_rotate_quantize_activation_op(x_q, x_scale, x, group_size)
else:
_jit_convrot_int8_module().convrot_rotate_quantize_activation(
x_q, x_scale, x, group_size
)
return x_q, x_scale
@debug_kernel_api
def convrot_int8_fused_linear(
x: torch.Tensor,
weight_q: torch.Tensor,
weight_scale: torch.Tensor,
bias: torch.Tensor | None,
group_size: int,
*,
out: torch.Tensor | None = None,
gelu_input: bool = False,
) -> torch.Tensor:
"""BF16 [M, K] x int8 [N, K] -> BF16 [M, N]; x is rotated and quantized in-kernel.
``out`` must be a contiguous BF16 [M, N] when given (a row slice of a larger
buffer qualifies). ``gelu_input`` applies GELU(tanh) to x inside the rotate
kernel, bitwise equal to the eager ``F.gelu(x, approximate="tanh")`` first.
"""
if out is None:
out = torch.empty(
(x.shape[0], weight_q.shape[0]), dtype=torch.bfloat16, device=x.device
)
if torch.compiler.is_compiling():
_convrot_int8_fused_linear_op(
out, x, weight_q, weight_scale, bias, group_size, gelu_input
)
else:
_jit_convrot_int8_module().convrot_int8_fused_linear(
out, x, weight_q, weight_scale, bias, group_size, gelu_input
)
return out
@debug_kernel_api
def convrot_int8_linear_prequant(
xq: torch.Tensor,
xs: torch.Tensor,
weight_q: torch.Tensor,
weight_scale: torch.Tensor,
bias: torch.Tensor | None,
group_size: int,
*,
out: torch.Tensor | None = None,
) -> torch.Tensor:
"""GEMM + dequant on (xq int8 [M, K], xs float32 [M]) from
convrot_rotate_quantize_activation; bitwise equal to the fused op."""
if out is None:
out = torch.empty(
(xq.shape[0], weight_q.shape[0]), dtype=torch.bfloat16, device=xq.device
)
if torch.compiler.is_compiling():
_convrot_int8_linear_prequant_op(
out, xq, xs, weight_q, weight_scale, bias, group_size
)
else:
_jit_convrot_int8_module().convrot_int8_linear_prequant(
out, xq, xs, weight_q, weight_scale, bias, group_size
)
return out
@@ -172,7 +172,7 @@ These options **trade output quality** for speed or VRAM savings. Results will d
| **Progressive Resolution** | `--progressive-mode dct_rewind --progressive-levels N --progressive-delta D` | Runs early denoising at lower latent resolution, then spectrally upsamples and switches to the target resolution. | Model- and schedule-dependent | Approximate and pipeline-specific. Keep the switch schedule fixed and compare detail, composition, and temporal stability. |
| **Causal KV-Cache Quantization** | `--kv-cache-quant int4\|int2` plus optional `--kv-cache-quant-*` controls | Compresses completed causal KV-cache chunks with Quant-VideoGen PRQ while keeping the mutable/current chunk and recent chunks in BF16. | Primarily a long-session memory saving | Currently limited to LingBot World realtime causal serving; requires `quant-videogen`. INT4 is the starting point; INT2 saves more memory with more error. It quantizes cache state, not checkpoint weights. |
| **Quantized Models (Nunchaku / SVDQuant)** | `--enable-svdquant --transformer-weights-path <path>` + optional `--quantization-precision int4\|nvfp4`, `--quantization-rank 32` | W4A4-style quantization via [Nunchaku](https://nunchaku.tech). Reduces DiT weight memory by ~4x. Precision/rank can be auto-inferred from weight filename or set explicitly. | ~1.5–2x compute speedup | Lossy quantization; quality depends on rank and precision. Requires pre-quantized weights. Ampere (SM8x) or SM12x only (no Hopper SM90). Higher rank = better quality but more memory. |
| **GGUF Transformer** | `--transformer-weights-path <file.gguf\|owner/repo:QUANT>` | Loads a community-quantized DiT from one `.gguf`; other components stay on the base model. **Shrinks the checkpoint, not the peak VRAM** — offload already bounds peak, so reach for this when the *download* or the host RAM offload pins is the problem (MiniMax-H3 17.5 vs 61.7 GiB), not when VRAM is. For a 24 GB card `kitchen_int8` is the faster option if you can afford the full BF16 checkpoint on disk. | None; expect a small slowdown from per-step dequantization | Lossy (4-bit families ~0.997 cosine vs BF16). CUDA only, `--tp-size 1`, no FSDP, no LoRA, no `--quantization`, no `--enable-svdquant`, and mutually exclusive with the H3 AdaLN cache/online flags — each rejected at startup. Validated on MiniMax-H3 `fl2va` Q4_K_M, 1 GPU. |
| **GGUF Transformer** | `--transformer-weights-path <file.gguf\|owner/repo:QUANT>` | Loads a community-quantized DiT from one `.gguf`; other components stay on the base model. **Shrinks the checkpoint, not the peak VRAM** — offload already bounds peak, so reach for this when the *download* or the host RAM offload pins is the problem (MiniMax-H3 17.5 vs 61.7 GiB), not when VRAM is. For a 24 GB card `convrot_int8` is the faster option if you can afford the full BF16 checkpoint on disk. | None; expect a small slowdown from per-step dequantization | Lossy (4-bit families ~0.997 cosine vs BF16). CUDA only, `--tp-size 1`, no FSDP, no LoRA, no `--quantization`, no `--enable-svdquant`, and mutually exclusive with the H3 AdaLN cache/online flags — each rejected at startup. Validated on MiniMax-H3 `fl2va` Q4_K_M, 1 GPU. |
| **Pre-quantized Transformer Override** | `--transformer-path <dir-or-repo>` / `--transformer-weights-path <path>` | Load a quantized transformer component or raw transformer weights. For converted ModelOpt FP8/NVFP4 directories, prefer `--transformer-path`; use `--transformer-weights-path` for weight-only artifacts the model loader expects. | ~1.3–1.5x compute (dtype dependent) | Requires a validated quantized transformer override, such as one produced by the ModelOpt helper tools. Quality is usually slightly worse than BF16 and depends on the format, fallback layers, and calibration scope. |
| **Component Precision Override** | `--dit-precision fp16`, `--vae-precision fp16\|bf16` | On-the-fly dtype conversion for individual components. E.g. convert a BF16 model to FP16 at load time, or run VAE in BF16 instead of FP32. | Reduces memory; FP16 can be faster on some GPUs | May affect numerical stability. VAE is FP32 by default for accuracy; lowering it is lossy. DiT defaults to BF16. |
| **Fewer Inference Steps** | `--num-inference-steps N` (sampling param) | Reduces the number of denoising steps. Fewer steps = faster. | Linear speedup | Quality degrades with too few steps. Model-dependent optimal range. |
+8
View File
@@ -47,6 +47,7 @@ if TYPE_CHECKING:
SGLANG_DIFFUSION_DISABLE_AUTO_RESIDENCY: bool = False
SGLANG_DIFFUSION_MINIMAX_H3_ADALN_GPU_PLANS: int = 64
SGLANG_DIFFUSION_MINIMAX_H3_ADALN_FP32: bool = False
SGLANG_DIFFUSION_CONVROT_INT8_BACKEND: str = "auto"
SGLANG_DIFFUSION_MINIMAX_H3_PDD_HEADS: str | None = None
SGLANG_DIFFUSION_FLUX3_NATTEN_BACKEND: str | None = None
SGLANG_DIFFUSION_CFG_GATE_STEP: float = 1.0
@@ -329,6 +330,13 @@ environment_variables: dict[str, Callable[[], Any]] = {
"SGLANG_DIFFUSION_CFG_GATE_STEP": _lazy_float(
"SGLANG_DIFFUSION_CFG_GATE_STEP", 1.0
),
# Kernel backend for convrot_int8 (online or serialized ConvRot INT8):
# "auto" prefers SGLang's JIT-compiled convrot_int8 ops where they build
# and run (CC 9.0, 10.0, 12.0, 12.1 with nvcc) and falls back to
# comfy_kitchen; "jit" or "comfy_kitchen" forces one backend.
"SGLANG_DIFFUSION_CONVROT_INT8_BACKEND": _lazy_str(
"SGLANG_DIFFUSION_CONVROT_INT8_BACKEND", "auto"
),
# Path to a Parallel Decoding Distillation head stack (one fused output head
# per denoise step, produced by fuse_minimax_h3_pdd_heads.py). Set only when serving a
# PDD-distilled checkpoint; an ordinary run leaves the projection alone.
@@ -11,10 +11,13 @@ from sglang.multimodal_gen.runtime.layers.quantization.bitsandbytes import (
from sglang.multimodal_gen.runtime.layers.quantization.configs.base_config import (
QuantizationConfig,
)
from sglang.multimodal_gen.runtime.layers.quantization.configs.kitchen_int8_config import (
KitchenInt8Config,
from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config import (
ConvRotInt8Config,
)
from sglang.multimodal_gen.runtime.layers.quantization.fp8 import Fp8Config
from sglang.multimodal_gen.runtime.layers.quantization.method_names import (
canonical_quantization_method,
)
from sglang.multimodal_gen.runtime.layers.quantization.modelopt_fp8 import (
ModelOptFp8Config as ModelOptFp8DiffusionConfig,
)
@@ -40,6 +43,8 @@ QuantizationMethods = Literal[
"mxfp8",
"mxfp4",
"mxfp4_npu",
"convrot_int8",
# deprecated alias of convrot_int8
"kitchen_int8",
]
@@ -57,7 +62,7 @@ _CUSTOMIZED_METHOD_TO_QUANT_CONFIG = {
"mxfp4": Mxfp4Config,
"mxfp8": MXFP8Config,
"mxfp4_npu": NPUMXFP4Config,
"kitchen_int8": KitchenInt8Config,
"convrot_int8": ConvRotInt8Config,
}
@@ -90,6 +95,7 @@ def register_quantization_config(quantization: str):
def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
quantization = canonical_quantization_method(quantization)
if quantization not in QUANTIZATION_METHODS:
raise ValueError(f"Invalid quantization method: {quantization}")
@@ -0,0 +1,377 @@
# SPDX-License-Identifier: Apache-2.0
"""Config for convrot_int8: online or serialized ConvRot INT8 on the JIT ops or comfy_kitchen.
Both backends implement the same quantization (arXiv:2512.03673): group-wise
regular Hadamard rotation, per-row dynamic INT8 activations and per-output-
channel INT8 weights, with interchangeable weight and scale tensors. ``auto``
picks SGLang's JIT-compiled fused ``convrot_int8`` ops where they build and run
(CC 9.0, 10.0, 12.0 and 12.1 with a CUDA toolchain; group size 64, 128, 256 or
512; output width a multiple of 8) and ``comfy_kitchen.int8_linear`` otherwise
(Turing and later; group size 16, 64 or 256; the load fails with an install hint
when the package is missing). Only a layer whose output width the JIT epilogue
cannot store falls to comfy_kitchen or, without it, stays BF16.
"""
from __future__ import annotations
import functools
from typing import Any
import torch
from sglang.kernels.ops.gemm.convrot_int8 import GROUP_SIZES as JIT_GROUP_SIZES
from sglang.multimodal_gen import envs
from sglang.multimodal_gen.runtime.layers.linear import (
ColumnParallelLinear,
LinearBase,
UnquantizedLinearMethod,
)
from sglang.multimodal_gen.runtime.layers.quantization.comfy_int8 import (
ComfyInt8EmbeddingMethod,
is_comfy_int8_embedding,
)
from sglang.multimodal_gen.runtime.layers.quantization.configs.base_config import (
QuantizationConfig,
QuantizeMethodBase,
)
from sglang.multimodal_gen.runtime.layers.vocab_parallel_embedding import (
VocabParallelEmbedding,
)
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
from sglang.srt.layers.quantization.utils import is_layer_skipped
logger = init_logger(__name__)
COMFY_KITCHEN = "comfy_kitchen"
JIT = "jit"
BACKENDS = ("auto", COMFY_KITCHEN, JIT)
# Group widths each backend's kernels are instantiated for.
BACKEND_GROUP_SIZES = {COMFY_KITCHEN: (16, 64, 256), JIT: JIT_GROUP_SIZES}
_SUPPORTED_GROUP_SIZES = tuple(
sorted({*BACKEND_GROUP_SIZES[COMFY_KITCHEN], *JIT_GROUP_SIZES})
)
def _jit_available() -> bool:
from sglang.multimodal_gen.runtime.layers.quantization.convrot_int8_jit import (
jit_convrot_unavailable_reason,
)
reason = jit_convrot_unavailable_reason()
if reason is not None:
_log_jit_fallback(reason)
return reason is None
@functools.cache
def _log_jit_fallback(reason: str) -> None:
# Once per process: the refused-GPU reason (for example CC 10.3) or a failed
# build would otherwise only surface with an explicit jit backend.
logger.info(
"convrot_int8: JIT backend unavailable (%s); auto uses comfy_kitchen",
reason,
)
def _comfy_kitchen_available() -> bool:
from sglang.multimodal_gen.runtime.layers.quantization.convrot_int8_comfy_kitchen import (
comfy_kitchen_available,
)
return comfy_kitchen_available()
class ConvRotInt8Config(QuantizationConfig):
"""Dispatch online or serialized Comfy ConvRot layers to a kernel backend."""
def __init__(
self,
group_size: int = 256,
ignored_layers: list[str] | None = None,
packed_modules_mapping: dict[str, list[str]] | None = None,
layer_markers: dict[str, dict[str, Any]] | None = None,
backend: str | None = None,
) -> None:
super().__init__()
if group_size not in _SUPPORTED_GROUP_SIZES:
raise ValueError(
f"convrot_int8 group_size must be one of {_SUPPORTED_GROUP_SIZES}, "
f"got {group_size}"
)
if backend is None:
backend = envs.SGLANG_DIFFUSION_CONVROT_INT8_BACKEND
if backend not in BACKENDS:
raise ValueError(
f"convrot_int8 backend must be one of {BACKENDS}, got {backend!r}"
)
if backend != "auto" and group_size not in BACKEND_GROUP_SIZES[backend]:
raise ValueError(
f"convrot_int8 backend {backend} has kernels for group sizes "
f"{BACKEND_GROUP_SIZES[backend]}, got {group_size}"
)
self.group_size = group_size
self.backend = backend
self.ignored_layers = ignored_layers or []
self.packed_modules_mapping = packed_modules_mapping or {}
self.layer_markers = layer_markers
self.is_checkpoint_int8_serialized = layer_markers is not None
self.checkpoint_uses_native_qkv_layout = self.is_checkpoint_int8_serialized
self._serialized_group_sizes: dict[str, int] = {}
if layer_markers is not None:
for prefix, marker in layer_markers.items():
if is_comfy_int8_embedding(marker):
continue
if marker.get("format") != "int8_tensorwise":
raise ValueError(
f"Unsupported Comfy INT8 format for {prefix!r}: "
f"{marker.get('format')!r}"
)
if marker.get("convrot") is not True:
raise ValueError(
f"Serialized convrot_int8 layer {prefix!r} must set "
"convrot=true"
)
marker_group_size = marker.get("convrot_groupsize")
if marker_group_size not in BACKEND_GROUP_SIZES[COMFY_KITCHEN]:
raise ValueError(
f"Serialized convrot_int8 layer {prefix!r} must declare "
f"convrot_groupsize in {BACKEND_GROUP_SIZES[COMFY_KITCHEN]}, "
f"got {marker_group_size!r}"
)
self._serialized_group_sizes[prefix] = marker_group_size
elif backend == "auto" and self.resolve_backend() is None:
raise ValueError(
f"convrot_int8: no backend on this machine serves group size "
f"{group_size} (the JIT backend needs CC 9.0/10.0/12.0/12.1, a CUDA "
f"toolchain and one of {BACKEND_GROUP_SIZES[JIT]}; comfy_kitchen serves "
f"{BACKEND_GROUP_SIZES[COMFY_KITCHEN]})"
)
# Logged per backend at the end of loading: a silent fallback to BF16
# or to the slower backend looks exactly like a slow kernel.
self.selected: list[str] = []
# Embeddings count as selected for the loader's marker check but never
# report back through note_quantized / note_loaded.
self.selected_embeddings: list[str] = []
self.selected_by_backend: dict[str, list[str]] = {
COMFY_KITCHEN: [],
JIT: [],
}
self.skipped: list[str] = []
self._processed = 0
self._quantized_bytes = 0
@classmethod
def get_name(cls) -> str:
return "convrot_int8"
@classmethod
def get_supported_act_dtypes(cls) -> list[torch.dtype]:
return [torch.bfloat16, torch.float16]
@classmethod
def get_min_capability(cls) -> int:
# INT8 tensor cores land on Turing (comfy_kitchen); the JIT backend
# checks its own table when it is selected.
return 75
@classmethod
def get_config_filenames(cls) -> list[str]:
return []
@classmethod
def from_config(cls, config: dict[str, Any]) -> ConvRotInt8Config:
return cls(
group_size=cls.get_from_keys_or(config, ["group_size"], 256),
ignored_layers=cls.get_from_keys_or(config, ["ignored_layers"], None),
backend=cls.get_from_keys_or(config, ["backend"], None),
)
def resolve_backend(self, group_size: int | None = None) -> str | None:
"""Backend serving ``group_size`` (default: the online group size), or
None when the requested backend has no kernel for it. ``auto`` prefers
the JIT ops where they build and run on this GPU; an explicit backend
is never substituted."""
group_size = self.group_size if group_size is None else group_size
if self.backend == "auto":
if group_size in BACKEND_GROUP_SIZES[JIT] and _jit_available():
return JIT
if group_size in BACKEND_GROUP_SIZES[COMFY_KITCHEN]:
return COMFY_KITCHEN
return None
return self.backend if group_size in BACKEND_GROUP_SIZES[self.backend] else None
def require_comfy_kitchen(self, reason: str) -> None:
"""Serve every layer with comfy_kitchen; an explicit jit request is refused."""
if self.backend == JIT:
raise ValueError(
f"convrot_int8 backend jit {reason}; use backend "
"comfy_kitchen or TP and/or sequence parallelism instead"
)
if self.group_size not in BACKEND_GROUP_SIZES[COMFY_KITCHEN]:
raise ValueError(
f"convrot_int8 backend comfy_kitchen has no kernel for group size "
f"{self.group_size}, and the jit backend {reason}"
)
if self.backend == "auto":
logger.info(
"convrot_int8: using the comfy_kitchen backend because %s", reason
)
self.backend = COMFY_KITCHEN
def _backend_for(self, *, group_size: int, out_size: int) -> str | None:
backend = self.resolve_backend(group_size)
if backend == JIT and out_size % 8:
# The JIT GEMM epilogue stores 8 BF16 outputs at a time.
if (
self.backend == "auto"
and group_size in BACKEND_GROUP_SIZES[COMFY_KITCHEN]
and _comfy_kitchen_available()
):
return COMFY_KITCHEN
return None
return backend
def _no_kernel_reason(self, *, group_size: int, out_size: int) -> str:
comfy = (
"installed"
if _comfy_kitchen_available()
else "not installed (pip install comfy-kitchen)"
)
return (
f"ConvRot group {group_size}, {out_size} outputs; backend {self.backend!r}; "
f"jit serves group sizes {BACKEND_GROUP_SIZES[JIT]} with an "
f"output width that is a multiple of 8, comfy_kitchen serves group sizes "
f"{BACKEND_GROUP_SIZES[COMFY_KITCHEN]} and is {comfy}"
)
def _select(
self, *, prefix: str, backend: str, group_size: int, serialized: bool
) -> QuantizeMethodBase:
self.selected.append(prefix)
self.selected_by_backend[backend].append(prefix)
if backend == JIT:
from sglang.multimodal_gen.runtime.layers.quantization.convrot_int8_jit import (
ConvRotInt8JitLinearMethod,
)
return ConvRotInt8JitLinearMethod(
self, group_size=group_size, is_checkpoint_serialized=serialized
)
from sglang.multimodal_gen.runtime.layers.quantization.convrot_int8_comfy_kitchen import (
ConvRotInt8ComfyKitchenLinearMethod,
)
return ConvRotInt8ComfyKitchenLinearMethod(
self, group_size=group_size, is_checkpoint_serialized=serialized
)
def get_quant_method(
self, layer: torch.nn.Module, prefix: str
) -> QuantizeMethodBase | None:
if isinstance(layer, VocabParallelEmbedding) and self.quantizes_embedding(
prefix
):
self.selected.append(prefix)
self.selected_embeddings.append(prefix)
return ComfyInt8EmbeddingMethod(
tensorwise=bool(self.layer_markers[prefix].get("_is_tensorwise_scalar"))
)
if not isinstance(layer, LinearBase):
return None
# A column-parallel layer shards its output; every rank sees the same
# per-shard width, so the decision below is rank-consistent.
out_size = (
layer.output_size_per_partition
if isinstance(layer, ColumnParallelLinear)
else layer.output_size
)
if self.layer_markers is not None:
marker_group_size = self._serialized_group_sizes.get(prefix)
if marker_group_size is None:
return UnquantizedLinearMethod()
if layer.input_size % marker_group_size:
raise ValueError(
f"Serialized convrot_int8 layer {prefix!r} has input size "
f"{layer.input_size}, which is not divisible by its "
f"ConvRot group size {marker_group_size}"
)
backend = self._backend_for(group_size=marker_group_size, out_size=out_size)
if backend is None:
raise ValueError(
f"Serialized convrot_int8 layer {prefix!r} has no kernel: "
+ self._no_kernel_reason(
group_size=marker_group_size, out_size=out_size
)
)
return self._select(
prefix=prefix,
backend=backend,
group_size=marker_group_size,
serialized=True,
)
if is_layer_skipped(
prefix, self.ignored_layers, fused_mapping=self.packed_modules_mapping
):
self.skipped.append(prefix)
return UnquantizedLinearMethod()
# The rotation partitions the input dim into fixed-size groups, so a
# layer whose input does not divide evenly simply stays in BF16 rather
# than failing the whole model. H3's adaln projections (in=2688) are
# the case this exists for, and they cost 0.2% of a step anyway.
if layer.input_size % self.group_size:
self.skipped.append(f"{prefix}(in={layer.input_size})")
return UnquantizedLinearMethod()
backend = self._backend_for(group_size=self.group_size, out_size=out_size)
if backend is None:
self.skipped.append(f"{prefix}(out={out_size})")
return UnquantizedLinearMethod()
return self._select(
prefix=prefix, backend=backend, group_size=self.group_size, serialized=False
)
def note_quantized(self, saved_bytes: int) -> None:
self._processed += 1
self._quantized_bytes += saved_bytes
if self._processed == len(self.selected) - len(self.selected_embeddings):
logger.info(
"convrot_int8: quantized %d linear layers (%.2f GiB of BF16 weights "
"-> %.2f GiB INT8; jit %d, comfy_kitchen %d), left %d in BF16",
self._processed,
self._quantized_bytes / 1024**3,
self._quantized_bytes / 2 / 1024**3,
len(self.selected_by_backend[JIT]),
len(self.selected_by_backend[COMFY_KITCHEN]),
len(self.skipped),
)
logger.debug("convrot_int8: layers left in BF16: %s", self.skipped)
def note_loaded(self) -> None:
"""A serialized layer's INT8 weights are in place; logs the backend split once."""
self._processed += 1
if self._processed == len(self.selected) - len(self.selected_embeddings):
logger.info(
"convrot_int8: loaded %d serialized INT8 linear layers "
"(jit %d, comfy_kitchen %d)",
self._processed,
len(self.selected_by_backend[JIT]),
len(self.selected_by_backend[COMFY_KITCHEN]),
)
def get_scaled_act_names(self) -> list[str]:
return []
def quantizes_embedding(self, prefix: str) -> bool:
return self.layer_markers is not None and is_comfy_int8_embedding(
self.layer_markers.get(prefix)
)
def supports_input_partition(
self, prefix: str, input_size_per_partition: int
) -> bool:
group_size = self.group_size
if self.layer_markers is not None:
marker_group_size = self._serialized_group_sizes.get(prefix)
if marker_group_size is None:
return True
group_size = marker_group_size
return input_size_per_partition % group_size == 0
@@ -1,189 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
"""Config for online or serialized INT8 ConvRot via comfy_kitchen."""
from __future__ import annotations
from typing import Any
import torch
from sglang.multimodal_gen.runtime.layers.linear import UnquantizedLinearMethod
from sglang.multimodal_gen.runtime.layers.quantization.comfy_int8 import (
ComfyInt8EmbeddingMethod,
is_comfy_int8_embedding,
)
from sglang.multimodal_gen.runtime.layers.quantization.configs.base_config import (
QuantizationConfig,
QuantizeMethodBase,
)
from sglang.multimodal_gen.runtime.layers.vocab_parallel_embedding import (
VocabParallelEmbedding,
)
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
from sglang.srt.layers.quantization.utils import is_layer_skipped
logger = init_logger(__name__)
_SUPPORTED_GROUP_SIZES = (16, 64, 256)
class KitchenInt8Config(QuantizationConfig):
"""Dispatch online quantization or serialized Comfy ConvRot layers."""
def __init__(
self,
group_size: int = 256,
ignored_layers: list[str] | None = None,
packed_modules_mapping: dict[str, list[str]] | None = None,
layer_markers: dict[str, dict[str, Any]] | None = None,
) -> None:
super().__init__()
if group_size not in _SUPPORTED_GROUP_SIZES:
raise ValueError(
f"kitchen_int8 group_size must be one of {_SUPPORTED_GROUP_SIZES}, "
f"got {group_size}"
)
self.group_size = group_size
self.ignored_layers = ignored_layers or []
self.packed_modules_mapping = packed_modules_mapping or {}
self.layer_markers = layer_markers
self.is_checkpoint_int8_serialized = layer_markers is not None
self.checkpoint_uses_native_qkv_layout = self.is_checkpoint_int8_serialized
self._serialized_group_sizes: dict[str, int] = {}
if layer_markers is not None:
for prefix, marker in layer_markers.items():
if is_comfy_int8_embedding(marker):
continue
if marker.get("format") != "int8_tensorwise":
raise ValueError(
f"Unsupported Comfy INT8 format for {prefix!r}: "
f"{marker.get('format')!r}"
)
if marker.get("convrot") is not True:
raise ValueError(
f"Serialized kitchen_int8 layer {prefix!r} must set "
"convrot=true"
)
marker_group_size = marker.get("convrot_groupsize")
if marker_group_size not in _SUPPORTED_GROUP_SIZES:
raise ValueError(
f"Serialized kitchen_int8 layer {prefix!r} must declare "
f"convrot_groupsize in {_SUPPORTED_GROUP_SIZES}, got "
f"{marker_group_size!r}"
)
self._serialized_group_sizes[prefix] = marker_group_size
# Which layers actually got quantized is worth stating plainly in the
# log: a silent fallback to BF16 looks exactly like a slow kernel.
self.selected: list[str] = []
self.skipped: list[str] = []
self._processed = 0
self._quantized_bytes = 0
@classmethod
def get_name(cls) -> str:
return "kitchen_int8"
@classmethod
def get_supported_act_dtypes(cls) -> list[torch.dtype]:
return [torch.bfloat16, torch.float16]
@classmethod
def get_min_capability(cls) -> int:
# INT8 tensor cores land on Turing.
return 75
@classmethod
def get_config_filenames(cls) -> list[str]:
return []
@classmethod
def from_config(cls, config: dict[str, Any]) -> KitchenInt8Config:
return cls(
group_size=cls.get_from_keys_or(config, ["group_size"], 256),
ignored_layers=cls.get_from_keys_or(config, ["ignored_layers"], None),
)
def get_quant_method(
self, layer: torch.nn.Module, prefix: str
) -> QuantizeMethodBase | None:
from sglang.multimodal_gen.runtime.layers.linear import LinearBase
from sglang.multimodal_gen.runtime.layers.quantization.kitchen_int8 import (
KitchenInt8LinearMethod,
)
if isinstance(layer, VocabParallelEmbedding) and self.quantizes_embedding(
prefix
):
self.selected.append(prefix)
return ComfyInt8EmbeddingMethod(
tensorwise=bool(self.layer_markers[prefix].get("_is_tensorwise_scalar"))
)
if not isinstance(layer, LinearBase):
return None
if self.layer_markers is not None:
marker_group_size = self._serialized_group_sizes.get(prefix)
if marker_group_size is None:
return UnquantizedLinearMethod()
if layer.input_size % marker_group_size:
raise ValueError(
f"Serialized kitchen_int8 layer {prefix!r} has input size "
f"{layer.input_size}, which is not divisible by its "
f"ConvRot group size {marker_group_size}"
)
self.selected.append(prefix)
return KitchenInt8LinearMethod(
self,
group_size=marker_group_size,
is_checkpoint_serialized=True,
)
if is_layer_skipped(
prefix, self.ignored_layers, fused_mapping=self.packed_modules_mapping
):
self.skipped.append(prefix)
return UnquantizedLinearMethod()
# The rotation partitions the input dim into fixed-size groups, so a
# layer whose input does not divide evenly simply stays in BF16 rather
# than failing the whole model. H3's adaln projections (in=2688) are
# the case this exists for, and they cost 0.2% of a step anyway.
if layer.input_size % self.group_size:
self.skipped.append(f"{prefix}(in={layer.input_size})")
return UnquantizedLinearMethod()
self.selected.append(prefix)
return KitchenInt8LinearMethod(
self,
group_size=self.group_size,
is_checkpoint_serialized=False,
)
def note_quantized(self, saved_bytes: int) -> None:
self._processed += 1
self._quantized_bytes += saved_bytes
if self._processed == len(self.selected):
logger.info(
"kitchen_int8: quantized %d linear layers (%.2f GiB of BF16 weights "
"-> %.2f GiB INT8), left %d in BF16",
self._processed,
self._quantized_bytes / 1024**3,
self._quantized_bytes / 2 / 1024**3,
len(self.skipped),
)
logger.debug("kitchen_int8: layers left in BF16: %s", self.skipped)
def get_scaled_act_names(self) -> list[str]:
return []
def supports_input_partition(
self, prefix: str, input_size_per_partition: int
) -> bool:
group_size = self.group_size
if self.layer_markers is not None:
marker_group_size = self._serialized_group_sizes.get(prefix)
if marker_group_size is None:
return True
group_size = marker_group_size
return input_size_per_partition % group_size == 0
def quantizes_embedding(self, prefix: str) -> bool:
return self.layer_markers is not None and is_comfy_int8_embedding(
self.layer_markers.get(prefix)
)
@@ -15,8 +15,8 @@ from sglang.multimodal_gen.runtime.layers.quantization.configs.base_config impor
QuantizationConfig,
QuantizeMethodBase,
)
from sglang.multimodal_gen.runtime.layers.quantization.configs.kitchen_int8_config import (
KitchenInt8Config,
from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config import (
ConvRotInt8Config,
)
from sglang.multimodal_gen.runtime.layers.quantization.kitchen_w4a4 import (
KitchenW4A4LinearMethod,
@@ -55,7 +55,7 @@ class KitchenW4A4Config(QuantizationConfig):
if marker.get("format") == "int8_tensorwise"
}
self._int8_config = (
KitchenInt8Config(layer_markers=int8_markers) if int8_markers else None
ConvRotInt8Config(layer_markers=int8_markers) if int8_markers else None
)
for prefix, marker in layer_markers.items():
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
"""INT8 weight-only-storage linear backed by comfy_kitchen's fused ConvRot kernel.
"""convrot_int8's comfy_kitchen backend: INT8 linear on comfy_kitchen's fused ConvRot kernel.
On Ada (RTX 4090) INT8 is only worth doing with the right kernel: on MiniMax H3
shapes `torch._int_mm` measures 0.46-0.90x of BF16 (i.e. slower) and a Triton
@@ -15,18 +15,19 @@ Comfy checkpoints instead load their INT8 weights and row scales directly.
from __future__ import annotations
import functools
import os
import torch
from torch.nn.parameter import Parameter
from sglang.multimodal_gen.runtime.layers.linear import LinearMethodBase
from sglang.multimodal_gen.runtime.layers.quantization.configs.kitchen_int8_config import (
KitchenInt8Config,
from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config import (
ConvRotInt8Config,
)
from sglang.multimodal_gen.runtime.utils.weight_attrs import set_weight_attrs
__all__ = ["KitchenInt8Config", "KitchenInt8LinearMethod"]
__all__ = ["ConvRotInt8ComfyKitchenLinearMethod"]
# comfy_kitchen's dtype codes for the fused op's output.
_OUT_DTYPE_CODE = {torch.float32: 0, torch.float16: 1, torch.bfloat16: 2}
@@ -55,12 +56,22 @@ def _row_split(rows: int, out_features: int) -> int | None:
return _MAX_ROWS_PER_CALL
@functools.cache
def comfy_kitchen_available() -> bool:
"""Whether comfy_kitchen is importable and registers int8_linear."""
try:
import comfy_kitchen # noqa: F401
except ImportError:
return False
return hasattr(torch.ops.comfy_kitchen, "int8_linear")
def _load_comfy_kitchen():
try:
import comfy_kitchen # noqa: F401
except ImportError as exc: # pragma: no cover - depends on optional dep
raise ImportError(
"kitchen_int8 quantization requires the `comfy-kitchen` package "
"convrot_int8 quantization requires the `comfy-kitchen` package "
"(pip install comfy-kitchen). It is a self-contained abi3 extension "
"and does not link against libtorch, so any torch version works."
) from exc
@@ -71,12 +82,12 @@ def _load_comfy_kitchen():
)
class KitchenInt8LinearMethod(LinearMethodBase):
class ConvRotInt8ComfyKitchenLinearMethod(LinearMethodBase):
"""Loads or creates ConvRot INT8 weights and runs the fused kernel."""
def __init__(
self,
quant_config: KitchenInt8Config,
quant_config: ConvRotInt8Config,
*,
group_size: int,
is_checkpoint_serialized: bool,
@@ -101,9 +112,10 @@ class KitchenInt8LinearMethod(LinearMethodBase):
# dimension the rotation groups over.
if input_size_per_partition % self.group_size:
raise ValueError(
f"kitchen_int8 needs input_size_per_partition "
f"convrot_int8 needs input_size_per_partition "
f"({input_size_per_partition}) divisible by group_size "
f"{self.group_size}"
f"{self.group_size}; leave the layer in BF16 with "
"--quantization-ignored-layers"
)
# The online path initially matches UnquantizedLinearMethod so the
@@ -135,7 +147,10 @@ class KitchenInt8LinearMethod(LinearMethodBase):
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
weight = layer.weight.data
if self.is_checkpoint_serialized or weight.dtype == torch.int8:
if self.is_checkpoint_serialized:
self.quant_config.note_loaded()
return
if weight.dtype == torch.int8:
return
from comfy_kitchen.tensor.int8 import TensorWiseINT8Layout
@@ -172,7 +187,7 @@ class KitchenInt8LinearMethod(LinearMethodBase):
out_code = _OUT_DTYPE_CODE.get(x.dtype)
if out_code is None:
raise ValueError(
f"kitchen_int8 does not support activation dtype {x.dtype}"
f"convrot_int8 does not support activation dtype {x.dtype}"
)
# The kernel takes 2D activations; callers may pass [..., K].
@@ -0,0 +1,384 @@
# SPDX-License-Identifier: Apache-2.0
"""convrot_int8's JIT backend: INT8 W8A8 linear on the fused ConvRot ops.
One op takes a BF16 activation and performs the group-wise Hadamard rotation,
dynamic per-row INT8 quantization, INT8 GEMM, dequantization and bias add
without materializing the intermediates. Weights receive the same rotation and
per-output-channel quantization once, after loading a stock BF16 checkpoint; a
serialized Comfy ConvRot INT8 checkpoint loads its INT8 weights and row scales
directly, the same tensors comfy_kitchen consumes.
The ops live in ``sglang.kernels.ops.gemm.convrot_int8`` and are compiled for
the local GPU on first use (nvcc, about a minute once per machine; cached
afterwards). A missing toolchain or a failed build makes this backend
unavailable, and ``auto`` then takes comfy_kitchen.
Two variants of that op are exposed to the model code as helpers below: a
shared-input form that rotates and quantizes an activation once for several
linears consuming it (a q/k/v trio), and a gelu-input form that applies
GELU(tanh) inside the rotate kernel for an FFN down-projection. Both are
bitwise identical to the plain op on the equivalent eager input.
"""
from __future__ import annotations
import functools
import time
from collections.abc import Sequence
import torch
from torch.nn.parameter import Parameter
from sglang.kernels.ops.gemm.convrot_int8 import (
convrot_int8_fused_linear,
convrot_int8_linear_prequant,
convrot_int8_local_capability,
convrot_int8_supported_capabilities,
convrot_rotate_quantize_activation,
load_convrot_int8_module,
)
from sglang.multimodal_gen.runtime.layers.linear import (
ColumnParallelLinear,
LinearBase,
LinearMethodBase,
ReplicatedLinear,
RowParallelLinear,
)
from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config import (
ConvRotInt8Config,
)
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
from sglang.multimodal_gen.runtime.utils.weight_attrs import set_weight_attrs
from sglang.srt.environ import envs
logger = init_logger(__name__)
__all__ = [
"ConvRotInt8JitLinearMethod",
"REFUSED_CAPABILITY_REASONS",
"apply_convrot_int8_gelu_input",
"apply_convrot_int8_shared_input",
"apply_convrot_int8_shared_input_out",
"convrot_int8_fuses_gelu_input",
"convrot_int8_shares_input",
"jit_convrot_available",
"jit_convrot_unavailable_reason",
]
# Parts the ops deliberately leave out, with the reason shown at load time. The
# supported table itself lives with the ops (convrot_int8_supported_capabilities);
# nothing here duplicates it.
REFUSED_CAPABILITY_REASONS: dict[tuple[int, int], str] = {
(10, 3): (
"Blackwell Ultra cuts INT8 tensor-core throughput to a fraction of its "
"BF16 rate, so W8A8 INT8 would be a slowdown there; use FP8 or NVFP4 "
"quantization on this GPU"
),
}
def _unsupported_capability_reason(capability: tuple[int, int]) -> str:
major, minor = capability
reason = REFUSED_CAPABILITY_REASONS.get(
capability, "the convrot_int8 ops carry no code for it"
)
supported = ", ".join(
f"{a}.{b}" for a, b in sorted(convrot_int8_supported_capabilities())
)
return f"CC {major}.{minor} is not supported: {reason} (supported: {supported})"
@functools.cache
def jit_convrot_unavailable_reason() -> str | None:
"""Why the JIT backend cannot run here, or None when it can.
Builds the ops on the way (once per machine; cached afterwards), so a
missing CUDA toolchain or a failed build shows up here as a reason instead
of failing the first quantized layer.
"""
if not torch.cuda.is_available():
return "no CUDA device"
capability = convrot_int8_local_capability()
if capability is None:
return "not a CUDA device"
if capability not in convrot_int8_supported_capabilities():
return _unsupported_capability_reason(capability)
logger.info(
"convrot_int8: loading the JIT ops for CC %d.%d (the first use on a "
"machine compiles them, about a minute)",
*capability,
)
started = time.monotonic()
try:
load_convrot_int8_module()
except Exception as exc: # nvcc / ninja missing, or the build itself failed
if envs.SGLANG_CRASH_ON_JIT_COMPILE.get():
raise
logger.warning("convrot_int8: the JIT build failed: %s", exc)
detail = str(exc).strip().splitlines()
return (
f"the JIT build of the convrot_int8 ops failed "
f"({type(exc).__name__}: {detail[0] if detail else 'no message'}; "
"the compiler output is in the log above)"
)
elapsed = time.monotonic() - started
# A cache hit loads in well under a second; anything longer was a build.
if elapsed > 5:
logger.info(
"convrot_int8: compiled the JIT ops for CC %d.%d in %.0f s "
"(cached for later runs)",
*capability,
elapsed,
)
return None
def jit_convrot_available() -> bool:
"""Whether the JIT convrot ops run (and built) on the current GPU."""
return jit_convrot_unavailable_reason() is None
def _load_jit_kernel() -> None:
reason = jit_convrot_unavailable_reason()
if reason is not None:
raise RuntimeError(f"convrot_int8 backend jit is unavailable: {reason}")
def _as_rows(x: torch.Tensor) -> torch.Tensor:
# The ops take BF16 only. FP16 activations (--dit-precision fp16) are
# cast on the way in and back on the way out; the BF16 rounding is an
# order of magnitude below the INT8 quantization error.
if x.dtype == torch.float16:
x = x.to(torch.bfloat16)
elif x.dtype != torch.bfloat16:
raise ValueError(
f"convrot_int8 backend jit does not support activation dtype {x.dtype}"
)
return x.reshape(-1, x.shape[-1]).contiguous()
def _like_input(out: torch.Tensor, x: torch.Tensor) -> torch.Tensor:
out = out.reshape(*x.shape[:-1], out.shape[-1])
return out if out.dtype == x.dtype else out.to(x.dtype)
class ConvRotInt8JitLinearMethod(LinearMethodBase):
"""Loads or creates ConvRot INT8 weights and runs the JIT fused ops."""
def __init__(
self,
quant_config: ConvRotInt8Config,
*,
group_size: int,
is_checkpoint_serialized: bool,
) -> None:
self.quant_config = quant_config
self.group_size = group_size
self.is_checkpoint_serialized = is_checkpoint_serialized
_load_jit_kernel()
def create_weights(
self,
layer: torch.nn.Module,
input_size_per_partition: int,
output_partition_sizes: list[int],
input_size: int,
output_size: int,
params_dtype: torch.dtype,
**extra_weight_attrs,
) -> None:
# get_quant_method screened the unsharded input size and the per-shard
# output width, so these fire only under TP > 1, where a row-parallel
# layer splits the very dimension the rotation groups over.
if input_size_per_partition % self.group_size:
raise ValueError(
f"convrot_int8 needs input_size_per_partition "
f"({input_size_per_partition}) divisible by group_size "
f"{self.group_size}; leave the layer in BF16 with "
"--quantization-ignored-layers"
)
if sum(output_partition_sizes) % 8:
raise ValueError(
f"convrot_int8 backend jit needs the output size per "
f"partition ({sum(output_partition_sizes)}) to be a multiple of 8"
)
# The online path initially matches UnquantizedLinearMethod so the
# source weights load in BF16 before quantization. Serialized weights
# allocate their final INT8 storage immediately.
weight = Parameter(
torch.empty(
sum(output_partition_sizes),
input_size_per_partition,
dtype=(torch.int8 if self.is_checkpoint_serialized else params_dtype),
),
requires_grad=False,
)
set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0})
layer.register_parameter("weight", weight)
set_weight_attrs(weight, extra_weight_attrs)
if self.is_checkpoint_serialized:
# Comfy stores the per-row scale as [N, 1]; the ops read N contiguous
# floats and accept that shape unchanged.
weight_scale = Parameter(
torch.empty(sum(output_partition_sizes), 1, dtype=torch.float32),
requires_grad=False,
)
set_weight_attrs(weight_scale, {"output_dim": 0})
set_weight_attrs(weight_scale, extra_weight_attrs)
layer.register_parameter("weight_scale", weight_scale)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
weight = layer.weight.data
if self.is_checkpoint_serialized:
self.quant_config.note_loaded()
elif weight.dtype != torch.int8:
# Quantization runs on CUDA; a layer staged on CPU is round-tripped
# on its own, so an offloaded model never holds more than one BF16
# layer on the GPU during load.
home = weight.device
weight_q, weight_scale = convrot_rotate_quantize_activation(
weight.to("cuda", non_blocking=True).to(torch.bfloat16),
self.group_size,
)
layer.weight = Parameter(weight_q.to(home), requires_grad=False)
layer.register_parameter(
"weight_scale",
Parameter(weight_scale.to(home), requires_grad=False),
)
self.quant_config.note_quantized(weight.numel() * weight.element_size())
del weight_q, weight_scale
torch.cuda.empty_cache()
# The ops add a BF16 bias; an FP16 pipeline loads FP16 parameters. A
# deferred bias (skip_bias_add) never reaches the ops and keeps its dtype.
if (
layer.bias is not None
and layer.bias.dtype != torch.bfloat16
and not layer.skip_bias_add
):
layer.bias = Parameter(
layer.bias.data.to(torch.bfloat16), requires_grad=False
)
def apply(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
out = convrot_int8_fused_linear(
_as_rows(x),
layer.weight,
layer.weight_scale,
bias,
self.group_size,
)
return _like_input(out=out, x=x)
def _forward_is_plain_apply(layer: LinearBase) -> bool:
# True only when layer(x) is exactly quant_method.apply(layer, x, layer.bias):
# no collective, no deferred bias, no rank-dependent bias.
if layer.skip_bias_add:
return False
if isinstance(layer, ReplicatedLinear):
return True
if isinstance(layer, ColumnParallelLinear):
return not layer.gather_output
if isinstance(layer, RowParallelLinear):
return layer.tp_size == 1
return False
def _is_jit_int8(layer: torch.nn.Module) -> bool:
# LoRA wrappers and other non-LinearBase modules carry no quant_method.
return isinstance(layer, LinearBase) and isinstance(
layer.quant_method, ConvRotInt8JitLinearMethod
)
def convrot_int8_shares_input(layers: Sequence[torch.nn.Module]) -> bool:
"""Whether ``apply_convrot_int8_shared_input`` reproduces ``layer(x)`` for
every layer in ``layers``."""
if not layers or not all(
_is_jit_int8(layer) and _forward_is_plain_apply(layer) for layer in layers
):
return False
# One rotated input serves every layer only at one group width.
return len({layer.quant_method.group_size for layer in layers}) == 1
def apply_convrot_int8_shared_input(
x: torch.Tensor, layers: Sequence[LinearBase]
) -> list[torch.Tensor]:
"""``[layer(x)[0] for layer in layers]`` with ``x`` rotated and quantized once.
Bitwise identical to applying each layer on its own; see
``convrot_int8_linear_prequant`` in ``sglang.kernels.ops.gemm.convrot_int8``.
"""
group_size = layers[0].quant_method.group_size
x_q, x_scale = convrot_rotate_quantize_activation(_as_rows(x), group_size)
outs = []
for layer in layers:
out = convrot_int8_linear_prequant(
x_q,
x_scale,
layer.weight,
layer.weight_scale,
layer.bias,
group_size,
)
outs.append(_like_input(out=out, x=x))
return outs
def apply_convrot_int8_shared_input_out(
x: torch.Tensor, layers: Sequence[LinearBase], outs: Sequence[torch.Tensor]
) -> None:
"""Writes ``layer(x)[0]`` into ``out`` for each (layer, out) pair with ``x``
rotated and quantized once; ``out`` must be a contiguous BF16 slice shaped
like ``layer(x)[0]``, so ``x`` must be BF16 as well (an FP16 ``x`` would
have ``layer(x)`` return FP16; use ``apply_convrot_int8_shared_input`` for
it). Bitwise identical to ``apply_convrot_int8_shared_input`` and to
``layer(x)``; see ``convrot_int8_linear_prequant`` in
``sglang.kernels.ops.gemm.convrot_int8``.
"""
if x.dtype != torch.bfloat16:
raise ValueError(
"apply_convrot_int8_shared_input_out writes BF16 outputs, so its "
f"input must be BF16, got {x.dtype}"
)
group_size = layers[0].quant_method.group_size
x_q, x_scale = convrot_rotate_quantize_activation(_as_rows(x), group_size)
for layer, out in zip(layers, outs, strict=True):
# view() rather than reshape(): a copy here would silently drop the write.
convrot_int8_linear_prequant(
x_q,
x_scale,
layer.weight,
layer.weight_scale,
layer.bias,
group_size,
out=out.view(-1, out.shape[-1]),
)
def convrot_int8_fuses_gelu_input(layer: torch.nn.Module) -> bool:
"""Whether ``apply_convrot_int8_gelu_input`` reproduces
``layer(F.gelu(x, approximate="tanh"))[0]``."""
return _is_jit_int8(layer) and _forward_is_plain_apply(layer)
def apply_convrot_int8_gelu_input(layer: LinearBase, x: torch.Tensor) -> torch.Tensor:
"""``layer(F.gelu(x, approximate="tanh"))[0]`` as one op, bitwise identical
to the eager GELU followed by the layer for a BF16 ``x`` (an FP16 ``x`` is
cast to BF16 before the GELU); see ``convrot_int8_fused_linear`` in
``sglang.kernels.ops.gemm.convrot_int8``."""
out = convrot_int8_fused_linear(
_as_rows(x),
layer.weight,
layer.weight_scale,
layer.bias,
layer.quant_method.group_size,
gelu_input=True,
)
return _like_input(out=out, x=x)
@@ -0,0 +1,22 @@
# SPDX-License-Identifier: Apache-2.0
"""Quantization method names and their deprecated aliases."""
from __future__ import annotations
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
logger = init_logger(__name__)
# Deprecated public name -> current name.
QUANTIZATION_METHOD_ALIASES: dict[str, str] = {"kitchen_int8": "convrot_int8"}
def canonical_quantization_method(name: str) -> str:
"""Map a deprecated method name to its current name, warning once per alias."""
canonical = QUANTIZATION_METHOD_ALIASES.get(name)
if canonical is None:
return name
logger.warning_once(
f"quantization method {name!r} is a deprecated alias of {canonical!r}"
)
return canonical
@@ -18,8 +18,8 @@ from sglang.multimodal_gen.runtime.layers.quantization.configs.base_config impor
QuantizationConfig,
QuantizeMethodBase,
)
from sglang.multimodal_gen.runtime.layers.quantization.configs.kitchen_int8_config import (
KitchenInt8Config,
from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config import (
ConvRotInt8Config,
)
from sglang.multimodal_gen.runtime.models.parameter import (
ModelWeightParameter,
@@ -328,7 +328,7 @@ class ModelOptFp4Config(ModelOptQuantConfig):
self.swap_weight_nibbles = swap_weight_nibbles
self.checkpoint_weight_scale_layout = checkpoint_weight_scale_layout
self.checkpoint_uses_comfy_quantization = checkpoint_uses_comfy_quantization
self._comfy_int8_config: KitchenInt8Config | None = None
self._comfy_int8_config: ConvRotInt8Config | None = None
self._comfy_fp8_config: ComfyFp8Config | None = None
def set_comfy_layer_markers(self, layer_markers: dict[str, dict[str, Any]]) -> None:
@@ -346,7 +346,7 @@ class ModelOptFp4Config(ModelOptQuantConfig):
if marker.get("format") == "int8_tensorwise"
}
self._comfy_int8_config = (
KitchenInt8Config(layer_markers=int8_markers) if int8_markers else None
ConvRotInt8Config(layer_markers=int8_markers) if int8_markers else None
)
fp8_markers = {
prefix: marker
@@ -34,8 +34,8 @@ from sglang.multimodal_gen.runtime.layers.quantization.comfy_nvfp4 import (
from sglang.multimodal_gen.runtime.layers.quantization.configs.base_config import (
QuantizationConfig,
)
from sglang.multimodal_gen.runtime.layers.quantization.configs.kitchen_int8_config import (
KitchenInt8Config,
from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config import (
ConvRotInt8Config,
)
from sglang.multimodal_gen.runtime.layers.quantization.configs.kitchen_w4a4_config import (
KitchenW4A4Config,
@@ -107,7 +107,7 @@ from sglang.srt.model_loader.checkpoint_quantization import (
logger = init_logger(__name__)
_ONLINE_ENCODER_QUANTIZATIONS = frozenset({"fp8", "kitchen_int8", "mxfp4"})
_ONLINE_ENCODER_QUANTIZATIONS = frozenset({"fp8", "convrot_int8", "mxfp4"})
_TRANSFORMERS_ENCODER_ONLY_CLASSES = {
"T5EncoderModel": transformers.T5EncoderModel,
@@ -398,12 +398,13 @@ def _require_quantized_encoder_layers(
(
ComfyFp8Config,
ComfyNvfp4Config,
KitchenInt8Config,
ConvRotInt8Config,
KitchenW4A4Config,
KitchenW4A8Config,
),
):
expected = set(quant_config.layer_markers)
# Online convrot_int8 carries no markers; there is nothing to consume.
expected = set(quant_config.layer_markers or ())
selected = set(quant_config.selected)
elif isinstance(quant_config, QuantoInt8Config):
expected = quant_config.layer_prefixes
@@ -744,7 +745,7 @@ class TextEncoderLoader(OnlineQuantizationComponentLoader):
if quant_config is not None and not isinstance(quant_config, GGUFConfig):
postprocess_device: torch.device | None = local_torch_device
if isinstance(quant_config, (ComfyNvfp4Config, QuantoInt8Config)) or (
isinstance(quant_config, KitchenInt8Config)
isinstance(quant_config, ConvRotInt8Config)
and quant_config.is_checkpoint_int8_serialized
):
postprocess_device = None
@@ -13,6 +13,9 @@ from sglang.multimodal_gen.runtime.layers.attention.selector import (
get_component_forced_attn_backend,
get_global_forced_attn_backend,
)
from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config import (
ConvRotInt8Config,
)
from sglang.multimodal_gen.runtime.loader.component_loaders.component_loader import (
OnlineQuantizationComponentLoader,
)
@@ -379,6 +382,15 @@ class TransformerLoader(OnlineQuantizationComponentLoader):
"AutoRound checkpoints do not support diffusion FSDP inference; "
"use TP and/or sequence parallelism instead"
)
if (
use_fsdp
and isinstance(quant_spec.quant_config, ConvRotInt8Config)
and not quant_spec.quant_config.is_checkpoint_int8_serialized
):
# Only the comfy_kitchen backend has run under FSDP.
quant_spec.quant_config.require_comfy_kitchen(
"has not been validated with diffusion FSDP inference"
)
if quant_spec.gguf_file is not None:
logger.info(
@@ -476,7 +488,7 @@ class TransformerLoader(OnlineQuantizationComponentLoader):
runtime_quant_config=quant_spec.runtime_quant_config,
quantized_cpu_load_supported=(
quant_spec.gguf_file is not None
or quant_spec.is_serialized_kitchen_int8
or quant_spec.is_convrot_int8
or quant_spec.is_serialized_kitchen_w4a8
),
)
@@ -19,8 +19,8 @@ from torch import nn
from sglang.multimodal_gen.configs.models.dits.base import DiTArchConfig
from sglang.multimodal_gen.runtime.layers.quantization import QuantizationConfig
from sglang.multimodal_gen.runtime.layers.quantization.configs.kitchen_int8_config import (
KitchenInt8Config,
from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config import (
ConvRotInt8Config,
)
from sglang.multimodal_gen.runtime.layers.quantization.configs.kitchen_w4a4_config import (
KitchenW4A4Config,
@@ -175,9 +175,13 @@ class TransformerQuantLoadSpec:
return _get_quant_config_name(self.quant_config) == "comfy_fp8"
@property
def is_serialized_kitchen_int8(self) -> bool:
def is_convrot_int8(self) -> bool:
return isinstance(self.quant_config, ConvRotInt8Config)
@property
def is_serialized_convrot_int8(self) -> bool:
return (
isinstance(self.quant_config, KitchenInt8Config)
isinstance(self.quant_config, ConvRotInt8Config)
and self.quant_config.is_checkpoint_int8_serialized
)
@@ -193,7 +197,7 @@ class TransformerQuantLoadSpec:
def uses_comfy_layer_markers(self) -> bool:
return (
self.is_comfy_fp8
or self.is_serialized_kitchen_int8
or self.is_serialized_convrot_int8
or self.is_serialized_kitchen_w4a4
or self.is_serialized_kitchen_w4a8
or (
@@ -816,10 +820,8 @@ def _needs_device_weight_postprocess(
quant_name = _get_quant_config_name(quant_config)
if quant_name in ("modelopt_fp8", "comfy_fp8", "auto-round", "mxfp8"):
return True
if quant_name == "kitchen_int8":
assert isinstance(quant_config, KitchenInt8Config)
return not quant_config.is_checkpoint_int8_serialized
# convrot_int8 is deliberately absent: both backends quantize each layer
# through their own CUDA round trip, so an offloaded DiT stays on the host.
serialized_flag_by_quant_name = {
"fp8": "is_checkpoint_fp8_serialized",
"mxfp4": "is_checkpoint_mxfp4_serialized",
@@ -1047,13 +1049,13 @@ def _resolve_quant_config(
"such as owner/repo:Q4_K_M)."
)
# Online-quant convention: for `fp8`, `mxfp4` and `kitchen_int8`, a
# Online-quant convention: for `fp8`, `mxfp4` and `convrot_int8`, a
# no-arg QuantizationConfig() selects the post-load path -- weights
# load in source dtype and are quantized in
# process_weights_after_loading.
quant_cls = get_quantization_config(server_args.quantization)
quant_kwargs = {}
if server_args.quantization in {"fp8", "mxfp4", "kitchen_int8"}:
if server_args.quantization in {"fp8", "mxfp4", "convrot_int8"}:
quant_kwargs["ignored_layers"] = getattr(
server_args, "quantization_ignored_layers", None
)
@@ -2,10 +2,16 @@
from __future__ import annotations
from collections.abc import Sequence
from typing import Any
import torch
from sglang.multimodal_gen.runtime.layers.quantization.convrot_int8_jit import (
apply_convrot_int8_shared_input,
convrot_int8_shares_input,
)
def modulate(
x: torch.Tensor,
@@ -22,6 +28,14 @@ def modulate(
return x * (1 + scale.unsqueeze(1)) + shift.unsqueeze(1)
def _project_shared_input(
x: torch.Tensor, layers: Sequence[torch.nn.Module]
) -> list[torch.Tensor]:
if convrot_int8_shares_input(layers):
return apply_convrot_int8_shared_input(x=x, layers=layers)
return [layer(x)[0] for layer in layers]
def get_qkv_projections(
attn: Any,
hidden_states: torch.Tensor,
@@ -53,9 +67,9 @@ def get_qkv_projections(
if make_contiguous:
query, key, value = [t.contiguous() for t in (query, key, value)]
else:
query, _ = attn.to_q(hidden_states)
key, _ = attn.to_k(hidden_states)
value, _ = attn.to_v(hidden_states)
query, key, value = _project_shared_input(
x=hidden_states, layers=(attn.to_q, attn.to_k, attn.to_v)
)
encoder_query = encoder_key = encoder_value = None
if encoder_hidden_states is not None and attn.added_kv_proj_dim is not None:
@@ -67,8 +81,9 @@ def get_qkv_projections(
t.contiguous() for t in (encoder_query, encoder_key, encoder_value)
]
else:
encoder_query, _ = attn.add_q_proj(encoder_hidden_states)
encoder_key, _ = attn.add_k_proj(encoder_hidden_states)
encoder_value, _ = attn.add_v_proj(encoder_hidden_states)
encoder_query, encoder_key, encoder_value = _project_shared_input(
x=encoder_hidden_states,
layers=(attn.add_q_proj, attn.add_k_proj, attn.add_v_proj),
)
return query, key, value, encoder_query, encoder_key, encoder_value
@@ -73,6 +73,7 @@ from sglang.multimodal_gen.runtime.layers.linear import (
MergedColumnParallelLinear,
ReplicatedLinear,
RowParallelLinear,
UnquantizedLinearMethod,
apply_unquantized_linear,
)
from sglang.multimodal_gen.runtime.layers.quantization.configs.base_config import (
@@ -82,6 +83,13 @@ from sglang.multimodal_gen.runtime.layers.quantization.configs.nunchaku_config i
NunchakuConfig,
is_nunchaku_available,
)
from sglang.multimodal_gen.runtime.layers.quantization.convrot_int8_jit import (
apply_convrot_int8_gelu_input,
apply_convrot_int8_shared_input,
apply_convrot_int8_shared_input_out,
convrot_int8_fuses_gelu_input,
convrot_int8_shares_input,
)
from sglang.multimodal_gen.runtime.layers.quantization.modelopt_quant import (
ModelOptFp4LinearMethod,
ModelOptFp8LinearMethod,
@@ -720,7 +728,161 @@ class QwenEmbedLayer3DRope(nn.Module):
return freqs.clone().contiguous()
def _joint_qkv_layers(
attn: "QwenImageCrossAttention",
) -> tuple[nn.Module, nn.Module, nn.Module, nn.Module, nn.Module, nn.Module]:
return (
attn.to_q,
attn.to_k,
attn.to_v,
attn.add_q_proj,
attn.add_k_proj,
attn.add_v_proj,
)
def _joint_qkv_operands_match(
*,
attn: "QwenImageCrossAttention",
hidden_states: torch.Tensor,
encoder_hidden_states: torch.Tensor,
) -> bool:
dtype = hidden_states.dtype
if encoder_hidden_states.dtype != dtype:
return False
layers = _joint_qkv_layers(attn)
# A LoRA wrapper forwards .weight to its base layer and adds its delta in
# forward, so the bare addmm below would silently drop it.
if not all(isinstance(layer, ColumnParallelLinear) for layer in layers):
return False
if attn.separate_convrot_qkv_proj:
# apply_convrot_int8_shared_input_out stores BF16 only; FP16 streams
# take apply_convrot_int8_shared_input, which casts at the op boundary.
return dtype == torch.bfloat16
# The out= GEMMs below take no part in autocast and do no promotion, so
# every operand must already be in the dtype F.linear would compute in.
device_type = hidden_states.device.type
if (
torch.is_autocast_enabled(device_type)
and torch.get_autocast_dtype(device_type) != dtype
):
return False
return all(
layer.weight.dtype == dtype
and (layer.bias is None or layer.bias.dtype == dtype)
for layer in layers
)
def _use_joint_qkv_buffers(
*,
attn: "QwenImageCrossAttention",
hidden_states: torch.Tensor,
encoder_hidden_states: torch.Tensor,
masked: bool,
) -> bool:
# Batch 1 keeps each slice contiguous for the out= writes and the in-place
# QK-norm; SP>1 and masked sequences never take join_seqs.
return (
(attn.separate_unquantized_qkv_proj or attn.separate_convrot_qkv_proj)
and not masked
# The out= GEMMs below reject grad-tracking operands.
and not torch.is_grad_enabled()
and hidden_states.shape[0] == 1
and hidden_states.is_contiguous()
and encoder_hidden_states.is_contiguous()
and get_sp_world_size() <= 1
and _joint_qkv_operands_match(
attn=attn,
hidden_states=hidden_states,
encoder_hidden_states=encoder_hidden_states,
)
)
def _project_into(
*, x: torch.Tensor, layer: ColumnParallelLinear, out: torch.Tensor
) -> None:
# The addmm F.linear dispatches for a contiguous x, so the out= write is
# bitwise identical to layer(x).
x_2d = x.reshape(-1, x.shape[-1])
# view() rather than reshape(): a copy here would silently drop the write.
out_2d = out.view(-1, out.shape[-1])
if layer.bias is None:
torch.mm(x_2d, layer.weight.t(), out=out_2d)
else:
torch.addmm(layer.bias, x_2d, layer.weight.t(), out=out_2d)
def _project_qkv_into_joint_buffers(
*,
attn: "QwenImageCrossAttention",
hidden_states: torch.Tensor,
encoder_hidden_states: torch.Tensor,
seq_len_txt: int,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
# Three [B, txt + img, local_inner_dim] buffers, text rows first.
batch_size, seq_len_img, _ = hidden_states.shape
inner_dim_local = attn.local_num_heads * attn.head_dim
shape = (batch_size, seq_len_txt + seq_len_img, inner_dim_local)
bufs = tuple(
torch.empty(shape, dtype=hidden_states.dtype, device=hidden_states.device)
for _ in range(3)
)
to_q, to_k, to_v, add_q_proj, add_k_proj, add_v_proj = _joint_qkv_layers(attn)
img_layers = (to_q, to_k, to_v)
txt_layers = (add_q_proj, add_k_proj, add_v_proj)
if attn.separate_convrot_qkv_proj:
apply_convrot_int8_shared_input_out(
x=hidden_states,
layers=img_layers,
outs=[buf[:, seq_len_txt:] for buf in bufs],
)
apply_convrot_int8_shared_input_out(
x=encoder_hidden_states,
layers=txt_layers,
outs=[buf[:, :seq_len_txt] for buf in bufs],
)
return bufs
for buf, img_layer, txt_layer in zip(bufs, img_layers, txt_layers):
_project_into(x=hidden_states, layer=img_layer, out=buf[:, seq_len_txt:])
_project_into(
x=encoder_hidden_states, layer=txt_layer, out=buf[:, :seq_len_txt]
)
return bufs
def _joint_qkv_head_views(
bufs: tuple[torch.Tensor, torch.Tensor, torch.Tensor],
*,
seq_len_txt: int,
num_heads: int,
head_dim: int,
) -> tuple[torch.Tensor, ...]:
# (img_q, img_k, img_v, txt_q, txt_k, txt_v), each a [B, S, H, D] view.
img_views = tuple(
buf[:, seq_len_txt:].unflatten(-1, (num_heads, head_dim)) for buf in bufs
)
txt_views = tuple(
buf[:, :seq_len_txt].unflatten(-1, (num_heads, head_dim)) for buf in bufs
)
return (*img_views, *txt_views)
def _sync_into_buffer(*, value: torch.Tensor, buf_view: torch.Tensor) -> None:
# In-place QK-norm/RoPE aliases value to buf_view; only the non-fused
# fallback returns a fresh tensor that must be copied back.
# Dynamo cannot trace data_ptr(); a self-aliased copy_ is a no-op anyway.
if torch.compiler.is_compiling() or value.data_ptr() != buf_view.data_ptr():
buf_view.copy_(value)
class QwenImageCrossAttention(nn.Module):
# Per-instance values are set in __init__; the class defaults keep forward()
# valid on modules built around it (object.__new__ in the epilogue tests).
separate_unquantized_qkv_proj = False
separate_convrot_qkv_proj = False
def __init__(
self,
dim: int, # query_dim
@@ -776,6 +938,8 @@ class QwenImageCrossAttention(nn.Module):
)
self.local_num_heads = self.num_heads // tp_size
self._unquantized_added_qkv_is_packed = False
self.separate_unquantized_qkv_proj = False
self.separate_convrot_qkv_proj = False
if self.use_fused_qkv:
# Use fused QKV projection for nunchaku quantization
@@ -860,6 +1024,24 @@ class QwenImageCrossAttention(nn.Module):
quant_config=quant_config,
prefix=f"{prefix}.add_v_proj",
)
# Six plain linears can write straight into joint text-image
# buffers (see _use_joint_qkv_buffers); the fused-epilogue
# path consumes packed projection strides instead.
self.separate_unquantized_qkv_proj = (
not self.use_fused_qkv
and not self.use_fused_qkv_epilogue
and all(
isinstance(layer.quant_method, UnquantizedLinearMethod)
for layer in _joint_qkv_layers(self)
)
)
# Six ConvRot INT8 linears share one rotated+quantized input per
# stream and also write straight into the joint buffers.
self.separate_convrot_qkv_proj = (
not self.use_fused_qkv
and not self.use_fused_qkv_epilogue
and convrot_int8_shares_input(_joint_qkv_layers(self))
)
if context_pre_only is not None and not context_pre_only:
self.to_add_out = RowParallelLinear(
@@ -961,7 +1143,35 @@ class QwenImageCrossAttention(nn.Module):
# Rows of tail padding inside THIS rank's text chunk (sp_shard meta).
sp_txt_pad = _attn_mask_meta_local_pad(attn_mask_meta)
if self._unquantized_added_qkv_is_packed and not qwen_image_added_qkv_active(
# Joint [text, image] destination buffers take precedence over the
# packed added-QKV GEMM, which in turn precedes per-layer projections.
joint_qkv_bufs = None
if _use_joint_qkv_buffers(
attn=self,
hidden_states=hidden_states,
encoder_hidden_states=encoder_hidden_states,
masked=attn_mask is not None or encoder_hidden_states_mask is not None,
):
joint_qkv_bufs = _project_qkv_into_joint_buffers(
attn=self,
hidden_states=hidden_states,
encoder_hidden_states=encoder_hidden_states,
seq_len_txt=seq_len_txt,
)
(
img_query,
img_key,
img_value,
txt_query,
txt_key,
txt_value,
) = _joint_qkv_head_views(
joint_qkv_bufs,
seq_len_txt=seq_len_txt,
num_heads=self.local_num_heads,
head_dim=self.head_dim,
)
elif self._unquantized_added_qkv_is_packed and not qwen_image_added_qkv_active(
self
):
img_query, img_key, img_value, _, _, _ = _get_qkv_projections(
@@ -972,6 +1182,20 @@ class QwenImageCrossAttention(nn.Module):
txt_query, txt_key, txt_value = self._get_added_qkv_projections(
encoder_hidden_states
)
elif self.separate_convrot_qkv_proj and convrot_int8_shares_input(
_joint_qkv_layers(self)
):
# Same rotate-once projections as the joint-buffer path, into fresh
# tensors when the joint layout is not applicable (SP, masks, B>1).
# Re-checked per call: LoRA mounting swaps the projections for
# wrappers that add their delta in forward.
img_query, img_key, img_value = apply_convrot_int8_shared_input(
x=hidden_states, layers=(self.to_q, self.to_k, self.to_v)
)
txt_query, txt_key, txt_value = apply_convrot_int8_shared_input(
x=encoder_hidden_states,
layers=(self.add_q_proj, self.add_k_proj, self.add_v_proj),
)
else:
(
img_query,
@@ -993,14 +1217,15 @@ class QwenImageCrossAttention(nn.Module):
else:
img_complex = txt_complex = None
# Reshape for multi-head attention
img_query = img_query.unflatten(-1, (self.local_num_heads, self.head_dim))
img_key = img_key.unflatten(-1, (self.local_num_heads, self.head_dim))
img_value = img_value.unflatten(-1, (self.local_num_heads, self.head_dim))
if joint_qkv_bufs is None:
# Reshape for multi-head attention
img_query = img_query.unflatten(-1, (self.local_num_heads, self.head_dim))
img_key = img_key.unflatten(-1, (self.local_num_heads, self.head_dim))
img_value = img_value.unflatten(-1, (self.local_num_heads, self.head_dim))
txt_query = txt_query.unflatten(-1, (self.local_num_heads, self.head_dim))
txt_key = txt_key.unflatten(-1, (self.local_num_heads, self.head_dim))
txt_value = txt_value.unflatten(-1, (self.local_num_heads, self.head_dim))
txt_query = txt_query.unflatten(-1, (self.local_num_heads, self.head_dim))
txt_key = txt_key.unflatten(-1, (self.local_num_heads, self.head_dim))
txt_value = txt_value.unflatten(-1, (self.local_num_heads, self.head_dim))
img_cache = txt_cache = None
if image_rotary_emb is not None:
@@ -1117,6 +1342,17 @@ class QwenImageCrossAttention(nn.Module):
joint_query, joint_key, joint_value = joint_qkv
elif seg_qkv is not None:
joint_query, joint_key, joint_value = seg_qkv
elif joint_qkv_bufs is not None:
# The buffers already hold the [text, image] layout: Q/K/V were
# projected into them and QK-norm/RoPE ran in place on views.
joint_query, joint_key, joint_value = (
buf.unflatten(-1, (self.local_num_heads, self.head_dim))
for buf in joint_qkv_bufs
)
_sync_into_buffer(value=img_query, buf_view=joint_query[:, seq_len_txt:])
_sync_into_buffer(value=img_key, buf_view=joint_key[:, seq_len_txt:])
_sync_into_buffer(value=txt_query, buf_view=joint_query[:, :seq_len_txt])
_sync_into_buffer(value=txt_key, buf_view=joint_key[:, :seq_len_txt])
elif attn_mask is not None and not sp_text_sharded:
# Let the eager attention break point pack directly from the text
# and image segments. Materializing three dense joint tensors here
@@ -1269,6 +1505,13 @@ class QwenImageFeedForward(nn.Module):
def forward_with_bias(
self, hidden_states: torch.Tensor | tuple[torch.Tensor, torch.Tensor]
) -> tuple[torch.Tensor, Optional[torch.Tensor]]:
# Not cached at construction: LoRA mounting swaps net.2 for a wrapper.
if convrot_int8_fuses_gelu_input(self.net[2]):
# The down-projection's rotate kernel applies GELU(tanh) to its
# input bit-exactly, so the up-projection runs bare and the
# standalone activation kernel is skipped.
up, _ = self.net[0].proj(hidden_states)
return apply_convrot_int8_gelu_input(layer=self.net[2], x=up), None
hidden_states = self.net[0](hidden_states)
hidden_states = self.net[1](hidden_states)
return self.net[2](hidden_states)
@@ -32,6 +32,9 @@ from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType
from sglang.multimodal_gen.runtime.layers.quantization.configs.nunchaku_config import (
NunchakuConfig,
)
from sglang.multimodal_gen.runtime.layers.quantization.method_names import (
canonical_quantization_method,
)
from sglang.multimodal_gen.runtime.loader.utils import BYTES_PER_GB
from sglang.multimodal_gen.runtime.managers.memory_managers.component_residency import (
COMPONENT_OFFLOAD,
@@ -1981,10 +1984,14 @@ class ServerArgs(DisaggServerArgsMixin):
)
normalized_direct_gpu_loading[component_name] = enabled
self.component_direct_gpu_weight_loading = normalized_direct_gpu_loading
if self.quantization is not None:
self.quantization = canonical_quantization_method(self.quantization)
normalized_quantizations: dict[str, str] = {}
for component, quantization in self.component_quantizations.items():
component = str(component).strip().replace("-", "_")
quantization = str(quantization).strip().lower()
quantization = canonical_quantization_method(
str(quantization).strip().lower()
)
if not component or not quantization:
raise ValueError(
"Component quantization entries require a component and method"
@@ -2789,7 +2796,11 @@ class ServerArgs(DisaggServerArgsMixin):
"auto-detected from the checkpoint config or safetensors metadata when "
"possible. Use this flag to override auto-detection. "
"Online (post-load) quantization from a BF16/FP16 checkpoint "
"is supported for 'fp8' and 'mxfp4'. Other methods "
"is supported for 'fp8', 'mxfp4' and 'convrot_int8' (ConvRot INT8 "
"W8A8; runs on SGLang's JIT-compiled fused ops on CC 9.0, 10.0, "
"12.0 and 12.1, else on comfy_kitchen; see "
"SGLANG_DIFFUSION_CONVROT_INT8_BACKEND; 'kitchen_int8' is a "
"deprecated alias). Other methods "
"('modelopt', 'modelopt_fp8', 'modelopt_fp4', 'mxfp8', "
"'mxfp4_npu', 'modelslim') require a pre-quantized checkpoint. "
"Note: 'mxfp4' targets ROCm + MI350+ (gfx95x); "
@@ -2803,7 +2814,8 @@ class ServerArgs(DisaggServerArgsMixin):
default=ServerArgs.quantization_ignored_layers,
help=(
"Layer name patterns to keep unquantized during online quantization "
"(fp8/mxfp4). Each pattern is matched against the layer prefix. "
"(fp8/mxfp4/convrot_int8). Each pattern is matched against the "
"layer prefix. "
"Example: --quantization-ignored-layers img_mod txt_mod to_out"
),
)
@@ -22,8 +22,8 @@ from sglang.multimodal_gen.runtime.layers.quantization.comfy_fp8 import ComfyFp8
from sglang.multimodal_gen.runtime.layers.quantization.comfy_nvfp4 import (
ComfyNvfp4Config,
)
from sglang.multimodal_gen.runtime.layers.quantization.configs.kitchen_int8_config import (
KitchenInt8Config,
from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config import (
ConvRotInt8Config,
)
from sglang.multimodal_gen.runtime.layers.quantization.configs.kitchen_w4a4_config import (
KitchenW4A4Config,
@@ -395,7 +395,7 @@ def resolve_comfy_checkpoint_quantization(
return None
formats = sorted({str(marker.get("format")) for marker in layer_markers.values()})
if formats == ["int8_tensorwise"]:
return KitchenInt8Config(layer_markers=layer_markers)
return ConvRotInt8Config(layer_markers=layer_markers)
if formats == ["asym_w4a8_int8"]:
return KitchenW4A8Config(layer_markers)
if formats == ["asym_w4a8_int8", "int8_tensorwise"]:
@@ -0,0 +1,672 @@
"""Unit tests for convrot_int8's backend selection and its JIT backend."""
from types import SimpleNamespace
from unittest.mock import patch
import pytest
import torch
import torch.nn.functional as F
from sglang.kernels.ops.gemm.convrot_int8 import convrot_int8_supported_capabilities
from sglang.multimodal_gen.runtime.layers.linear import (
LinearBase,
ReplicatedLinear,
UnquantizedLinearMethod,
)
from sglang.multimodal_gen.runtime.layers.lora.linear import wrap_with_lora_layer
from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config import (
ConvRotInt8Config,
)
from sglang.multimodal_gen.runtime.layers.quantization.convrot_int8_comfy_kitchen import (
ConvRotInt8ComfyKitchenLinearMethod,
)
from sglang.multimodal_gen.runtime.layers.quantization.convrot_int8_jit import (
ConvRotInt8JitLinearMethod,
apply_convrot_int8_gelu_input,
apply_convrot_int8_shared_input,
apply_convrot_int8_shared_input_out,
convrot_int8_fuses_gelu_input,
convrot_int8_shares_input,
)
_CONFIG_MODULE = (
"sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config"
)
_LOAD_JIT_KERNEL = (
"sglang.multimodal_gen.runtime.layers.quantization.convrot_int8_jit."
"_load_jit_kernel"
)
_LOAD_COMFY_KITCHEN = "sglang.multimodal_gen.runtime.layers.quantization.convrot_int8_comfy_kitchen._load_comfy_kitchen"
QWEN_IMAGE_IGNORED_LAYERS = ["img_mod", "txt_mod", "txt_mlp.net.2"]
# Capability only: on a supported GPU a missing toolchain or a failed build must
# fail these tests, not skip them.
requires_kernel = pytest.mark.skipif(
not (
torch.cuda.is_available()
and torch.cuda.get_device_capability() in convrot_int8_supported_capabilities()
),
reason="needs a GPU in the convrot_int8 capability table",
)
def _jit_config(**kwargs) -> ConvRotInt8Config:
return ConvRotInt8Config(backend="jit", **kwargs)
def _method_for(config, prefix, input_size=3072):
layer = LinearBase(input_size=input_size, output_size=32)
return config.get_quant_method(layer, prefix)
@patch(_LOAD_JIT_KERNEL)
def test_ignored_layer_patterns_select_bf16_on_module_path_boundaries(_load):
"""A pattern must match its own module path only: `txt_mlp.net.2` keeps the
text FFN down-projection in BF16 without also catching `img_mlp.net.2` or
the text FFN up-projection, and `img_mod` must not catch `img_mlp`."""
config = _jit_config(ignored_layers=QWEN_IMAGE_IGNORED_LAYERS)
kept_bf16 = [
"transformer_blocks.0.img_mod.1",
"transformer_blocks.3.txt_mod.1",
"transformer_blocks.7.txt_mlp.net.2",
]
quantized = [
"transformer_blocks.0.img_mlp.net.2",
"transformer_blocks.7.txt_mlp.net.0.proj",
"transformer_blocks.0.attn.to_q",
"transformer_blocks.0.attn.add_k_proj",
]
for prefix in kept_bf16:
assert isinstance(_method_for(config, prefix), UnquantizedLinearMethod)
for prefix in quantized:
assert isinstance(_method_for(config, prefix), ConvRotInt8JitLinearMethod)
assert config.skipped == kept_bf16
assert config.selected == quantized
@patch(_LOAD_JIT_KERNEL)
def test_input_dim_not_divisible_by_group_stays_bf16(_load):
config = _jit_config()
method = _method_for(config, "blocks.0.mod", input_size=2688)
assert isinstance(method, UnquantizedLinearMethod)
assert config.skipped == ["blocks.0.mod(in=2688)"]
assert config.selected == []
assert not config.supports_input_partition("blocks.0.attn.to_out", 1536 + 128)
assert config.supports_input_partition("blocks.0.attn.to_out", 1536)
@patch(_LOAD_JIT_KERNEL)
def test_shared_input_and_gelu_helpers_refuse_deferred_bias_and_lora(_load):
"""The helpers stand in for `layer(x)`; a layer that returns its bias
separately would silently lose it, and a LoRA-wrapped projection is not a
`LinearBase` at all, so eligibility must say no to both instead of
crashing on the wrapper."""
config = _jit_config()
plain = ReplicatedLinear(256, 8, quant_config=config, prefix="q")
deferred = ReplicatedLinear(
256, 8, skip_bias_add=True, quant_config=config, prefix="k"
)
bf16 = ReplicatedLinear(256, 8, quant_config=None, prefix="v")
wrapped = wrap_with_lora_layer(plain)
assert convrot_int8_shares_input([plain, plain])
assert not convrot_int8_shares_input([plain, deferred])
assert not convrot_int8_shares_input([plain, bf16])
assert not convrot_int8_shares_input([wrapped, plain])
assert not convrot_int8_shares_input([])
assert convrot_int8_fuses_gelu_input(plain)
assert not convrot_int8_fuses_gelu_input(deferred)
assert not convrot_int8_fuses_gelu_input(bf16)
assert not convrot_int8_fuses_gelu_input(wrapped)
def _quantized_layer(
config, in_features, out_features, prefix, seed, dtype=torch.bfloat16
):
gen = torch.Generator(device="cuda").manual_seed(seed)
layer = ReplicatedLinear(
in_features,
out_features,
params_dtype=dtype,
quant_config=config,
prefix=prefix,
).cuda()
with torch.no_grad():
layer.weight.copy_(
torch.randn(layer.weight.shape, device="cuda", generator=gen) * 0.02
)
layer.bias.copy_(torch.randn(layer.bias.shape, device="cuda", generator=gen))
layer.bias.requires_grad_(False)
reference_weight = layer.weight.detach().clone()
layer.quant_method.process_weights_after_loading(layer)
return layer, reference_weight
@requires_kernel
def test_quantized_layer_tracks_bf16_reference():
config = _jit_config()
layer, weight = _quantized_layer(config, 3072, 3072, "attn.to_q", seed=0)
assert layer.weight.dtype == torch.int8
assert layer.weight_scale.shape == (3072,)
x = torch.randn(2, 1024, 3072, device="cuda", dtype=torch.bfloat16)
out, out_bias = layer(x)
assert out_bias is None
ref = F.linear(x, weight, layer.bias)
err = torch.linalg.vector_norm(out.float() - ref.float())
rel_l2 = (err / torch.linalg.vector_norm(ref.float())).item()
# Arbitrary loose bound: a wrong scale or rotation lands far above it,
# W8A8 group-256 quantization noise on Gaussian data far below.
assert rel_l2 < 2e-2, rel_l2
@requires_kernel
def test_shared_input_helper_is_bitwise_three_layer_calls():
config = _jit_config()
layers = [
_quantized_layer(config, 3072, 3072, f"attn.to_{n}", seed=i)[0]
for i, n in enumerate("qkv")
]
x = torch.randn(1, 4096, 3072, device="cuda", dtype=torch.bfloat16)
assert convrot_int8_shares_input(layers)
shared = apply_convrot_int8_shared_input(x=x, layers=layers)
for out, layer in zip(shared, layers, strict=True):
assert torch.equal(out, layer(x)[0])
@requires_kernel
def test_shared_input_out_helper_is_bitwise_into_joint_buffer_slices():
"""The joint text-image Q/K/V path writes each projection into a row slice
of one larger buffer; the slice write must equal the allocating helper."""
config = _jit_config()
layers = [
_quantized_layer(config, 3072, 3072, f"attn.to_{n}", seed=10 + i)[0]
for i, n in enumerate("qkv")
]
seq_len_txt, seq_len_img = 20, 4096
x = torch.randn(1, seq_len_img, 3072, device="cuda", dtype=torch.bfloat16)
bufs = [
torch.zeros(
1, seq_len_txt + seq_len_img, 3072, device="cuda", dtype=torch.bfloat16
)
for _ in layers
]
apply_convrot_int8_shared_input_out(
x=x, layers=layers, outs=[buf[:, seq_len_txt:] for buf in bufs]
)
for out, buf, layer in zip(
apply_convrot_int8_shared_input(x=x, layers=layers), bufs, layers, strict=True
):
assert torch.equal(buf[:, seq_len_txt:], out)
assert torch.equal(buf[:, seq_len_txt:], layer(x)[0])
assert not buf[:, :seq_len_txt].any()
@requires_kernel
def test_gelu_input_helper_is_bitwise_eager_gelu_then_layer():
config = _jit_config()
down, _ = _quantized_layer(config, 12288, 3072, "img_mlp.net.2", seed=3)
up = torch.randn(1, 2048, 12288, device="cuda", dtype=torch.bfloat16)
assert convrot_int8_fuses_gelu_input(down)
fused = apply_convrot_int8_gelu_input(layer=down, x=up)
eager, _ = down(F.gelu(up, approximate="tanh"))
assert torch.equal(fused, eager)
class _FakeTpGroup:
pass
def _with_tp(monkeypatch, tp_size):
import sglang.multimodal_gen.runtime.layers.linear as linear
monkeypatch.setattr(linear, "get_tp_group", lambda: _FakeTpGroup())
monkeypatch.setattr(linear, "get_group_rank", lambda group: 0)
monkeypatch.setattr(linear, "get_group_size", lambda group: tp_size)
@patch(_LOAD_JIT_KERNEL)
@pytest.mark.parametrize("tp_size,input_size", [(4, 13824), (8, 3072)])
def test_row_parallel_shard_not_divisible_by_group_is_refused(
_load, monkeypatch, tp_size, input_size
):
from sglang.multimodal_gen.runtime.layers.linear import RowParallelLinear
_with_tp(monkeypatch, tp_size)
config = _jit_config()
# convrot_int8 semantics: the unsharded input divides, so the layer is
# selected, and a shard that does not divide is refused at construction.
with pytest.raises(ValueError, match="divisible by group_size"):
RowParallelLinear(
input_size,
5120,
bias=True,
quant_config=config,
prefix="blocks.0.ffn.fc_out",
)
@patch(_LOAD_JIT_KERNEL)
def test_row_parallel_shard_divisible_by_group_is_quantized(_load, monkeypatch):
from sglang.multimodal_gen.runtime.layers.linear import RowParallelLinear
_with_tp(monkeypatch, 4)
config = _jit_config()
layer = RowParallelLinear(
12288, 3072, bias=True, quant_config=config, prefix="blocks.0.ffn.net.2"
)
assert isinstance(layer.quant_method, ConvRotInt8JitLinearMethod)
assert config.selected == ["blocks.0.ffn.net.2"]
@patch(_LOAD_JIT_KERNEL)
def test_output_width_not_multiple_of_8_stays_bf16(_load):
config = _jit_config()
layer = LinearBase(input_size=4096, output_size=4)
method = config.get_quant_method(layer, "blocks.0.attn.to_gate_logits")
assert isinstance(method, UnquantizedLinearMethod)
assert config.skipped == ["blocks.0.attn.to_gate_logits(out=4)"]
@patch(_LOAD_JIT_KERNEL)
def test_column_parallel_shard_output_not_multiple_of_8_stays_bf16(_load, monkeypatch):
from sglang.multimodal_gen.runtime.layers.linear import ColumnParallelLinear
_with_tp(monkeypatch, 8)
config = _jit_config()
# 32 output rows split eight ways leave 4 per rank; the epilogue stores 8 at
# a time, so the layer stays in BF16 instead of failing in the first forward.
layer = ColumnParallelLinear(
4096, 32, bias=True, quant_config=config, prefix="blocks.0.attn.to_gate_logits"
)
assert isinstance(layer.quant_method, UnquantizedLinearMethod)
assert config.skipped == ["blocks.0.attn.to_gate_logits(out=4)"]
@requires_kernel
def test_fp16_activations_are_cast_and_returned_in_fp16():
config = _jit_config()
layer, weight = _quantized_layer(config, 3072, 3072, "attn.to_q", seed=3)
x = torch.randn(2, 64, 3072, device="cuda", dtype=torch.float16)
out, _ = layer(x)
assert out.dtype == torch.float16
ref = F.linear(x.to(torch.bfloat16), weight, layer.bias).to(torch.float16)
err = torch.linalg.vector_norm(out.float() - ref.float())
rel_l2 = (err / torch.linalg.vector_norm(ref.float())).item()
assert rel_l2 < 2e-2, rel_l2
@requires_kernel
def test_fp16_parameters_are_cast_for_the_kernel_in_every_helper():
"""FP16 parameters (--dit-precision fp16) must run through the plain apply
and every fused helper; the out= helper must refuse an FP16 input."""
config = _jit_config()
layers = []
weights = []
for i, n in enumerate("qkv"):
layer, weight = _quantized_layer(
config, 3072, 3072, f"attn.to_{n}", seed=20 + i, dtype=torch.float16
)
layers.append(layer)
weights.append(weight)
assert layers[0].weight.dtype == torch.int8
assert layers[0].bias.dtype == torch.bfloat16
x = torch.randn(1, 128, 3072, device="cuda", dtype=torch.float16)
for layer, weight in zip(layers, weights, strict=True):
out, _ = layer(x)
assert out.dtype == torch.float16
ref = F.linear(x.float(), weight.float(), layer.bias.float())
err = torch.linalg.vector_norm(out.float() - ref)
rel_l2 = (err / torch.linalg.vector_norm(ref)).item()
assert rel_l2 < 2e-2, rel_l2
shared = apply_convrot_int8_shared_input(x=x, layers=layers)
for out, layer in zip(shared, layers, strict=True):
assert out.dtype == torch.float16
assert torch.equal(out, layer(x)[0])
down, _ = _quantized_layer(
config, 12288, 3072, "img_mlp.net.2", seed=23, dtype=torch.float16
)
up = torch.randn(1, 128, 12288, device="cuda", dtype=torch.float16)
fused = apply_convrot_int8_gelu_input(layer=down, x=up)
assert fused.dtype == torch.float16
# The GELU runs on the BF16-cast input inside the kernel, so the eager
# reference is the layer on GELU of that cast, not GELU in FP16.
eager, _ = down(F.gelu(up.to(torch.bfloat16), approximate="tanh"))
assert torch.equal(fused, eager.to(torch.float16))
outs = [
torch.empty(1, 128, 3072, device="cuda", dtype=torch.bfloat16) for _ in layers
]
with pytest.raises(ValueError, match="must be BF16"):
apply_convrot_int8_shared_input_out(x=x, layers=layers, outs=outs)
x_bf16 = x.to(torch.bfloat16)
apply_convrot_int8_shared_input_out(x=x_bf16, layers=layers, outs=outs)
for out, layer in zip(outs, layers, strict=True):
assert torch.equal(out, layer(x_bf16)[0])
@requires_kernel
def test_lora_merge_mode_is_redirected_to_dynamic_on_int8_base():
from sglang.multimodal_gen.runtime.pipelines_core.lora.pipeline import LoRAPipeline
config = _jit_config()
layer, weight = _quantized_layer(
config, 3072, 3072, "transformer_blocks.0.attn.to_q", seed=4
)
lora = wrap_with_lora_layer(layer, lora_rank=16, lora_alpha=16, snapshot_base=True)
layers = {"transformer_blocks.0.attn.to_q": lora}
# The rotated, per-channel scaled INT8 matrix cannot take a merged delta.
assert lora.can_merge_base_weight is False
# LoRAPipeline is abstract; the decision only needs its static helper.
stand_in = SimpleNamespace(_uses_dtensor_weights=LoRAPipeline._uses_dtensor_weights)
def decide(module_name, layers, mode):
return LoRAPipeline._should_merge_lora_for_layers(
stand_in, module_name, layers, mode
)
assert decide("transformer_blocks.0.attn.to_q", layers, "auto") is False
with pytest.raises(ValueError, match="use merge mode 'dynamic'"):
decide("transformer_blocks.0.attn.to_q", layers, "merge")
gen = torch.Generator(device="cuda").manual_seed(4)
lora_a = (torch.randn(16, 3072, device="cuda", generator=gen) * 0.05).to(
torch.bfloat16
)
lora_b = (torch.randn(3072, 16, device="cuda", generator=gen) * 0.05).to(
torch.bfloat16
)
lora.set_lora_weights(
lora_a, lora_b, lora_path="test", strength=1.0, merge_weights=False
)
x = torch.randn(8, 3072, device="cuda", dtype=torch.bfloat16, generator=gen)
out, _ = lora(x)
delta = (lora_b.float() @ lora_a.float()).to(torch.bfloat16)
ref = F.linear(x, weight + delta, layer.bias)
err = torch.linalg.vector_norm(out.float() - ref.float())
rel_l2 = (err / torch.linalg.vector_norm(ref.float())).item()
assert rel_l2 < 2e-2, rel_l2
_JIT_AVAILABLE = f"{_CONFIG_MODULE}._jit_available"
_COMFY_AVAILABLE = f"{_CONFIG_MODULE}._comfy_kitchen_available"
@patch(_LOAD_COMFY_KITCHEN)
@patch(_LOAD_JIT_KERNEL)
def test_auto_backend_prefers_jit_and_falls_back_to_comfy(_jit, _comfy):
"""``auto`` must pick the JIT ops where they run and comfy_kitchen
otherwise; the choice is per config, not per import."""
with (
patch(_JIT_AVAILABLE, return_value=True),
patch(_COMFY_AVAILABLE, return_value=True),
):
config = ConvRotInt8Config()
assert config.resolve_backend() == "jit"
method = config.get_quant_method(LinearBase(3072, 3072), "blocks.0.attn.to_q")
assert isinstance(method, ConvRotInt8JitLinearMethod)
assert config.selected_by_backend["jit"] == ["blocks.0.attn.to_q"]
with (
patch(_JIT_AVAILABLE, return_value=False),
patch(_COMFY_AVAILABLE, return_value=True),
):
config = ConvRotInt8Config()
assert config.resolve_backend() == "comfy_kitchen"
method = config.get_quant_method(LinearBase(3072, 3072), "blocks.0.attn.to_q")
assert isinstance(method, ConvRotInt8ComfyKitchenLinearMethod)
assert config.selected_by_backend["comfy_kitchen"] == ["blocks.0.attn.to_q"]
@patch(_LOAD_COMFY_KITCHEN)
@patch(_LOAD_JIT_KERNEL)
def test_auto_backend_serves_narrow_outputs_with_comfy_or_leaves_bf16(_jit, _comfy):
"""The JIT epilogue stores 8 outputs at a time; a narrower layer goes
to comfy_kitchen when it is installed and stays BF16 otherwise."""
with patch(_JIT_AVAILABLE, return_value=True):
with patch(_COMFY_AVAILABLE, return_value=True):
config = ConvRotInt8Config()
method = config.get_quant_method(LinearBase(4096, 4), "blocks.0.gate")
assert isinstance(method, ConvRotInt8ComfyKitchenLinearMethod)
assert config.selected_by_backend["comfy_kitchen"] == ["blocks.0.gate"]
with patch(_COMFY_AVAILABLE, return_value=False):
config = ConvRotInt8Config()
method = config.get_quant_method(LinearBase(4096, 4), "blocks.0.gate")
assert isinstance(method, UnquantizedLinearMethod)
assert config.skipped == ["blocks.0.gate(out=4)"]
@patch(_LOAD_COMFY_KITCHEN)
@patch(_LOAD_JIT_KERNEL)
def test_serialized_layers_pick_the_backend_per_marker_group_size(_jit, _comfy):
"""A Comfy checkpoint may mix group sizes; 16 exists only in comfy_kitchen
while 256 runs on the JIT ops with the stored [N, 1] scale."""
markers = {
"visual.proj": {
"format": "int8_tensorwise",
"convrot": True,
"convrot_groupsize": 16,
},
"blocks.0.fc1": {
"format": "int8_tensorwise",
"convrot": True,
"convrot_groupsize": 256,
},
}
with (
patch(_JIT_AVAILABLE, return_value=True),
patch(_COMFY_AVAILABLE, return_value=True),
):
config = ConvRotInt8Config(layer_markers=markers)
small = ReplicatedLinear(
64, 16, bias=False, quant_config=config, prefix="visual.proj"
)
fc1 = ReplicatedLinear(
256, 16, bias=False, quant_config=config, prefix="blocks.0.fc1"
)
assert isinstance(small.quant_method, ConvRotInt8ComfyKitchenLinearMethod)
assert isinstance(fc1.quant_method, ConvRotInt8JitLinearMethod)
assert fc1.quant_method.is_checkpoint_serialized
assert fc1.weight.dtype == torch.int8 and fc1.weight.shape == (16, 256)
assert fc1.weight_scale.dtype == torch.float32
assert fc1.weight_scale.shape == (16, 1)
assert config.selected == ["visual.proj", "blocks.0.fc1"]
def test_explicit_backend_rejects_group_sizes_it_has_no_kernel_for():
with pytest.raises(ValueError, match="group sizes"):
ConvRotInt8Config(group_size=16, backend="jit")
with pytest.raises(ValueError, match="group sizes"):
ConvRotInt8Config(group_size=128, backend="comfy_kitchen")
with pytest.raises(ValueError, match="backend must be one of"):
ConvRotInt8Config(backend="triton")
def test_require_comfy_kitchen_pins_auto_and_refuses_explicit_jit():
with (
patch(_JIT_AVAILABLE, return_value=True),
patch(_COMFY_AVAILABLE, return_value=True),
):
config = ConvRotInt8Config()
config.require_comfy_kitchen("is not validated here")
assert config.resolve_backend() == "comfy_kitchen"
with pytest.raises(ValueError, match="jit is not validated here"):
ConvRotInt8Config(backend="jit").require_comfy_kitchen(
"is not validated here"
)
@requires_kernel
def test_serialized_layer_on_jit_matches_online_quantization_bitwise():
"""Loading the online path's INT8 weight and fp32 [N, 1] row scale through
the serialized method must reproduce the online layer bit for bit."""
online, _ = _quantized_layer(_jit_config(), 3072, 3072, "attn.to_q", seed=30)
markers = {
"attn.to_q": {
"format": "int8_tensorwise",
"convrot": True,
"convrot_groupsize": 256,
}
}
config = ConvRotInt8Config(layer_markers=markers, backend="jit")
serialized = ReplicatedLinear(
3072, 3072, params_dtype=torch.bfloat16, quant_config=config, prefix="attn.to_q"
).cuda()
assert isinstance(serialized.quant_method, ConvRotInt8JitLinearMethod)
with torch.no_grad():
serialized.weight.copy_(online.weight)
serialized.weight_scale.copy_(online.weight_scale.view(-1, 1))
serialized.bias.copy_(online.bias)
serialized.quant_method.process_weights_after_loading(serialized)
x = torch.randn(2, 512, 3072, device="cuda", dtype=torch.bfloat16)
assert torch.equal(serialized(x)[0], online(x)[0])
assert convrot_int8_shares_input([serialized, online])
def test_auto_logs_why_the_jit_backend_is_unavailable(caplog):
"""A refused GPU (CC 10.3) must still say why once the method is convrot_int8
with auto backend, not only when jit is requested explicitly."""
import logging
from sglang.multimodal_gen.runtime.layers.quantization.configs import (
convrot_int8_config,
)
convrot_int8_config._log_jit_fallback.cache_clear()
reason = (
"CC 10.3 is not supported: Blackwell Ultra cuts INT8 tensor-core throughput"
)
with (
patch(
"sglang.multimodal_gen.runtime.layers.quantization.convrot_int8_jit."
"jit_convrot_unavailable_reason",
return_value=reason,
),
patch(_COMFY_AVAILABLE, return_value=True),
caplog.at_level(logging.INFO),
):
config = ConvRotInt8Config()
assert config.resolve_backend() == "comfy_kitchen"
assert config.resolve_backend() == "comfy_kitchen"
messages = [
r.getMessage()
for r in caplog.records
if "JIT backend unavailable" in r.getMessage()
]
assert len(messages) == 1 and reason in messages[0]
@patch(_LOAD_JIT_KERNEL)
def test_shared_input_requires_one_group_size(_jit):
"""One rotated activation serves several layers only at one group width; a
serialized checkpoint may mix 64- and 256-group layers."""
markers = {
"a": {"format": "int8_tensorwise", "convrot": True, "convrot_groupsize": 256},
"b": {"format": "int8_tensorwise", "convrot": True, "convrot_groupsize": 64},
"c": {"format": "int8_tensorwise", "convrot": True, "convrot_groupsize": 256},
}
config = ConvRotInt8Config(layer_markers=markers, backend="jit")
a, b, c = (
ReplicatedLinear(256, 16, bias=False, quant_config=config, prefix=name)
for name in "abc"
)
assert convrot_int8_shares_input([a, c])
assert not convrot_int8_shares_input([a, b])
def test_every_no_kernel_branch_raises_instead_of_falling_back():
"""A serialized INT8 layer that no backend serves must not land in a BF16
parameter, and an impossible auto config must fail at construction."""
with (
patch(_JIT_AVAILABLE, return_value=False),
patch(_COMFY_AVAILABLE, return_value=True),
):
with pytest.raises(ValueError, match="no backend on this machine"):
ConvRotInt8Config(group_size=128)
with (
patch(_JIT_AVAILABLE, return_value=True),
patch(_COMFY_AVAILABLE, return_value=False),
):
with pytest.raises(ValueError, match="no kernel for group size 128"):
ConvRotInt8Config(group_size=128).require_comfy_kitchen("is pinned")
markers = {
"gate": {
"format": "int8_tensorwise",
"convrot": True,
"convrot_groupsize": 256,
}
}
config = ConvRotInt8Config(layer_markers=markers)
with pytest.raises(ValueError, match="pip install comfy-kitchen"):
config.get_quant_method(LinearBase(256, 4), "gate")
@patch(_LOAD_COMFY_KITCHEN)
@patch(_LOAD_JIT_KERNEL)
def test_serialized_layers_log_the_backend_split_once_loaded(_jit, _comfy, caplog):
"""Serialized layers skip online quantization, so the backend split must be
reported when their weights are in place instead."""
import logging
markers = {
"fc1": {"format": "int8_tensorwise", "convrot": True, "convrot_groupsize": 256},
"small": {
"format": "int8_tensorwise",
"convrot": True,
"convrot_groupsize": 16,
},
}
with (
patch(_JIT_AVAILABLE, return_value=True),
patch(_COMFY_AVAILABLE, return_value=True),
caplog.at_level(logging.INFO),
):
config = ConvRotInt8Config(layer_markers=markers)
fc1 = ReplicatedLinear(256, 16, bias=False, quant_config=config, prefix="fc1")
small = ReplicatedLinear(
64, 16, bias=False, quant_config=config, prefix="small"
)
fc1.quant_method.process_weights_after_loading(fc1)
assert not [r for r in caplog.records if "serialized INT8" in r.getMessage()]
small.quant_method.process_weights_after_loading(small)
lines = [
r.getMessage() for r in caplog.records if "serialized INT8" in r.getMessage()
]
assert lines == [
"convrot_int8: loaded 2 serialized INT8 linear layers (jit 1, comfy_kitchen 1)"
]
def test_kitchen_int8_is_a_deprecated_alias_of_convrot_int8(caplog):
"""Existing `--quantization kitchen_int8` commands must keep working under the
new name, with one deprecation warning per process."""
import logging
from sglang.multimodal_gen.runtime.layers.quantization import (
get_quantization_config,
method_names,
)
from sglang.multimodal_gen.runtime.utils.logging_utils import _print_warning_once
_print_warning_once.cache_clear()
with caplog.at_level(logging.WARNING):
assert (
method_names.canonical_quantization_method("kitchen_int8") == "convrot_int8"
)
assert (
method_names.canonical_quantization_method("kitchen_int8") == "convrot_int8"
)
assert method_names.canonical_quantization_method("fp8") == "fp8"
warnings = [r for r in caplog.records if "deprecated alias" in r.getMessage()]
assert len(warnings) == 1
assert get_quantization_config("kitchen_int8") is ConvRotInt8Config
assert ConvRotInt8Config.get_name() == "convrot_int8"
@@ -0,0 +1,480 @@
"""Unit tests for Qwen-Image joint text-image Q/K/V destination buffers."""
import unittest
from types import SimpleNamespace
from unittest.mock import patch
import torch
import torch.nn as nn
from sglang.kernels.ops.gemm.convrot_int8 import convrot_int8_supported_capabilities
from sglang.multimodal_gen.runtime.distributed.sp_shard_utils import join_seqs
from sglang.multimodal_gen.runtime.layers.layernorm import RMSNorm
from sglang.multimodal_gen.runtime.layers.linear import (
ColumnParallelLinear,
apply_unquantized_linear,
)
from sglang.multimodal_gen.runtime.layers.lora.linear import BaseLayerWithLoRA
from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config import (
ConvRotInt8Config,
)
from sglang.multimodal_gen.runtime.models.dits.qwen_image import (
QwenImageCrossAttention,
_joint_qkv_head_views,
_project_qkv_into_joint_buffers,
_sync_into_buffer,
_use_joint_qkv_buffers,
)
from sglang.test.test_utils import CustomTestCase
DIM = 32
# ConvRot rotates 256-wide input groups.
CONVROT_DIM = 256
NUM_HEADS = 2
HEAD_DIM = 8
INNER_DIM = NUM_HEADS * HEAD_DIM
SEQ_TXT = 5
SEQ_IMG = 13
QWEN_IMAGE = "sglang.multimodal_gen.runtime.models.dits.qwen_image"
SP_WORLD_SIZE = f"{QWEN_IMAGE}.get_sp_world_size"
PROJECT_INTO_JOINT_BUFFERS = f"{QWEN_IMAGE}._project_qkv_into_joint_buffers"
FUSED_INPLACE_QKNORM = (
"sglang.multimodal_gen.runtime.layers.layernorm.can_use_fused_inplace_qknorm"
)
# ColumnParallelLinear reads only the group's size and rank.
_TP_GROUP = SimpleNamespace(world_size=1, rank_in_group=0)
# Capability only: on a supported GPU a missing toolchain or a failed build must
# fail these tests, not skip them.
requires_convrot_kernel = unittest.skipUnless(
torch.cuda.is_available()
and torch.cuda.get_device_capability() in convrot_int8_supported_capabilities(),
"needs a GPU in the convrot_int8 capability table",
)
def _linear(
*,
bias: bool = True,
dim: int = DIM,
dtype: torch.dtype = torch.bfloat16,
quant_config: ConvRotInt8Config | None = None,
prefix: str = "",
) -> ColumnParallelLinear:
linear = ColumnParallelLinear(
dim,
INNER_DIM,
bias=bias,
gather_output=False,
params_dtype=dtype,
quant_config=quant_config,
prefix=prefix,
tp_group=_TP_GROUP,
)
with torch.no_grad():
linear.weight.normal_()
if bias:
linear.bias.normal_()
linear = linear.cuda()
if quant_config is not None:
linear.quant_method.process_weights_after_loading(linear)
return linear
def _passthrough(x: torch.Tensor) -> tuple[torch.Tensor, None]:
return x, None
class _RecordingAttention:
# Stands in for USPAttention: keeps the Q/K/V it was handed and returns a
# deterministic function of them in the [B, S, H, D] layout.
def __init__(self) -> None:
self.calls: list[tuple[torch.Tensor, torch.Tensor, torch.Tensor, dict]] = []
def __call__(
self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, **kwargs
) -> torch.Tensor:
self.calls.append((q.clone(), k.clone(), v.clone(), kwargs))
if kwargs["q_prefix"] is not None:
q = torch.cat([kwargs["q_prefix"], q], dim=1)
k = torch.cat([kwargs["k_prefix"], k], dim=1)
v = torch.cat([kwargs["v_prefix"], v], dim=1)
return q + k + v
def _attention(
*,
bias: bool = True,
dtype: torch.dtype = torch.bfloat16,
quant_config: ConvRotInt8Config | None = None,
) -> QwenImageCrossAttention:
dim = DIM if quant_config is None else CONVROT_DIM
attn = object.__new__(QwenImageCrossAttention)
nn.Module.__init__(attn)
attn.head_dim = HEAD_DIM
attn.local_num_heads = NUM_HEADS
attn.added_kv_proj_dim = dim
attn.qk_norm = True
attn.use_fused_qkv = False
attn.use_fused_qkv_epilogue = False
attn.use_fused_added_qkv = False
attn._unquantized_added_qkv_is_packed = False
attn.separate_unquantized_qkv_proj = quant_config is None
attn.separate_convrot_qkv_proj = quant_config is not None
for name in ("to_q", "to_k", "to_v", "add_q_proj", "add_k_proj", "add_v_proj"):
setattr(
attn,
name,
_linear(
bias=bias,
dim=dim,
dtype=dtype,
quant_config=quant_config,
prefix=f"transformer_blocks.0.attn.{name}",
),
)
for name in ("norm_q", "norm_k", "norm_added_q", "norm_added_k"):
norm = RMSNorm(HEAD_DIM, eps=1e-6).to(device="cuda", dtype=dtype)
with torch.no_grad():
norm.weight.normal_()
setattr(attn, name, norm)
attn.to_out = [_passthrough]
attn.to_add_out = _passthrough
attn.attn = _RecordingAttention()
return attn
def _streams(
*, dim: int = DIM, dtype: torch.dtype = torch.bfloat16
) -> tuple[torch.Tensor, torch.Tensor]:
hidden = torch.randn(1, SEQ_IMG, dim, device="cuda", dtype=dtype)
encoder = torch.randn(1, SEQ_TXT, dim, device="cuda", dtype=dtype)
return hidden, encoder
def _joint_flag(attn: QwenImageCrossAttention) -> str:
return (
"separate_convrot_qkv_proj"
if attn.separate_convrot_qkv_proj
else "separate_unquantized_qkv_proj"
)
class _JointQkvCase(CustomTestCase):
# Forward helpers shared by the unquantized and the ConvRot test classes.
def setUp(self) -> None:
torch.manual_seed(20260902)
# The runtime forwards under no_grad; the out= GEMMs of the joint path
# reject grad-tracking operands.
grad_was_enabled = torch.is_grad_enabled()
torch.set_grad_enabled(False)
self.addCleanup(torch.set_grad_enabled, grad_was_enabled)
def _forward(
self,
attn: QwenImageCrossAttention,
*,
dtype: torch.dtype = torch.bfloat16,
**kwargs,
):
hidden, encoder = _streams(dim=attn.added_kv_proj_dim, dtype=dtype)
with patch(SP_WORLD_SIZE, return_value=1):
return attn.forward(
hidden_states=hidden,
encoder_hidden_states=encoder,
image_rotary_emb=None,
**kwargs,
)
def _assert_joint_path_matches_join_seqs_path(
self, attn: QwenImageCrossAttention, *, dtype: torch.dtype = torch.bfloat16
) -> None:
flag = _joint_flag(attn)
outputs = []
for joint in (True, False):
torch.manual_seed(20260902)
setattr(attn, flag, joint)
attn.attn.calls.clear()
with patch(
PROJECT_INTO_JOINT_BUFFERS, wraps=_project_qkv_into_joint_buffers
) as project:
out = self._forward(attn, dtype=dtype)
self.assertEqual(project.call_count, 1 if joint else 0)
outputs.append((out, attn.attn.calls[0]))
(joint_out, joint_call), (ref_out, ref_call) = outputs
self.assertIsNone(joint_call[3]["q_prefix"])
for joint_qkv, ref_qkv in zip(joint_call[:3], ref_call[:3]):
self.assertEqual(
joint_qkv.shape, (1, SEQ_TXT + SEQ_IMG, NUM_HEADS, HEAD_DIM)
)
self.assertTrue(torch.equal(joint_qkv, ref_qkv))
self.assertTrue(torch.equal(joint_out[0], ref_out[0]))
self.assertTrue(torch.equal(joint_out[1], ref_out[1]))
def _assert_compiled_forward_matches_eager(self, attn) -> None:
with patch(FUSED_INPLACE_QKNORM, return_value=False):
torch.manual_seed(20260902)
eager_out = self._forward(attn)
torch.manual_seed(20260902)
compiled = torch.compile(attn, dynamic=False)
hidden, encoder = _streams(dim=attn.added_kv_proj_dim, dtype=torch.bfloat16)
with patch(SP_WORLD_SIZE, return_value=1):
compiled_out = compiled(
hidden_states=hidden,
encoder_hidden_states=encoder,
image_rotary_emb=None,
)
torch.testing.assert_close(compiled_out[0], eager_out[0], rtol=2e-2, atol=2e-2)
torch.testing.assert_close(compiled_out[1], eager_out[1], rtol=2e-2, atol=2e-2)
def _assert_forward_bypasses_joint_buffers(
self, attn: QwenImageCrossAttention, *, dtype: torch.dtype
) -> tuple[torch.Tensor, torch.Tensor]:
# The out= kernels reject these operand dtypes, so the joint path must
# not be entered; the flag-off forward is the same code the fallback runs.
with patch(
PROJECT_INTO_JOINT_BUFFERS, wraps=_project_qkv_into_joint_buffers
) as project:
out = self._forward(attn, dtype=dtype)
project.assert_not_called()
return out
@unittest.skipUnless(torch.cuda.is_available(), "CUDA required")
class TestQwenImageJointQkvBuffers(_JointQkvCase):
def test_forward_is_bitwise_identical_to_join_seqs_path(self):
self._assert_joint_path_matches_join_seqs_path(_attention())
def test_forward_copies_back_when_qk_norm_returns_fresh_tensors(self):
"""The non-fused QK-norm returns new tensors instead of writing the
buffer views in place; their values must still reach the joint buffers."""
with patch(FUSED_INPLACE_QKNORM, return_value=False):
self._assert_joint_path_matches_join_seqs_path(_attention())
def test_compiled_forward_matches_eager_when_qk_norm_returns_fresh_tensors(self):
"""With the in-place QK-norm off, the joint buffers are filled by the
copy-back; that path must compile and agree with eager."""
self._assert_compiled_forward_matches_eager(_attention())
def test_fp16_parameters_and_streams_take_the_joint_path(self):
"""--dit-precision fp16 keeps every operand FP16, which the out= GEMMs
accept like BF16."""
self._assert_joint_path_matches_join_seqs_path(
_attention(dtype=torch.float16), dtype=torch.float16
)
def test_fp32_streams_under_bf16_autocast_bypass_joint_buffers(self):
"""FP32 streams under BF16 autocast must take the join_seqs path; the
out= GEMMs of the joint path do not autocast."""
attn = _attention()
with torch.autocast("cuda", dtype=torch.bfloat16):
self._assert_forward_bypasses_joint_buffers(attn, dtype=torch.float32)
def test_masked_forward_bypasses_joint_buffers(self):
"""A masked sequence hands text Q/K/V to attention as a prefix, so the
joint buffers must not be built for it."""
attn = _attention()
mask = torch.ones(1, SEQ_TXT, dtype=torch.bool, device="cuda")
with patch(PROJECT_INTO_JOINT_BUFFERS) as project:
self._forward(attn, encoder_hidden_states_mask=mask)
project.assert_not_called()
self.assertIsNotNone(attn.attn.calls[0][3]["q_prefix"])
def test_stale_convrot_flag_falls_through_to_per_layer_projections(self):
"""LoRA mounting swaps the projections after construction, so the
ConvRot flag alone must not select the shared-input helper."""
attn = _attention()
attn.separate_unquantized_qkv_proj = False
mask = torch.ones(1, SEQ_TXT, dtype=torch.bool, device="cuda")
outputs = []
for convrot in (False, True):
torch.manual_seed(20260902)
attn.separate_convrot_qkv_proj = convrot
outputs.append(self._forward(attn, encoder_hidden_states_mask=mask))
self.assertTrue(torch.equal(outputs[0][0], outputs[1][0]))
self.assertTrue(torch.equal(outputs[0][1], outputs[1][1]))
def test_joint_projection_without_bias_matches_reference(self):
attn = _attention(bias=False)
hidden, encoder = _streams()
bufs = _project_qkv_into_joint_buffers(
attn=attn,
hidden_states=hidden,
encoder_hidden_states=encoder,
seq_len_txt=SEQ_TXT,
)
img_layers = (attn.to_q, attn.to_k, attn.to_v)
txt_layers = (attn.add_q_proj, attn.add_k_proj, attn.add_v_proj)
for buf, img_layer, txt_layer in zip(bufs, img_layers, txt_layers):
expected = join_seqs(
apply_unquantized_linear(encoder, txt_layer.weight, None),
apply_unquantized_linear(hidden, img_layer.weight, None),
0,
)
self.assertTrue(torch.equal(buf, expected))
def test_head_views_alias_the_joint_buffers(self):
hidden, encoder = _streams()
bufs = _project_qkv_into_joint_buffers(
attn=_attention(),
hidden_states=hidden,
encoder_hidden_states=encoder,
seq_len_txt=SEQ_TXT,
)
views = _joint_qkv_head_views(
bufs, seq_len_txt=SEQ_TXT, num_heads=NUM_HEADS, head_dim=HEAD_DIM
)
img_query, txt_query = views[0], views[3]
self.assertEqual(img_query.shape, (1, SEQ_IMG, NUM_HEADS, HEAD_DIM))
self.assertEqual(txt_query.shape, (1, SEQ_TXT, NUM_HEADS, HEAD_DIM))
self.assertTrue(img_query.is_contiguous())
self.assertTrue(txt_query.is_contiguous())
# In-place QK-norm/RoPE on the views must land in the joint buffer.
img_query.zero_()
txt_query.fill_(1.0)
self.assertTrue(torch.all(bufs[0][:, SEQ_TXT:] == 0))
self.assertTrue(torch.all(bufs[0][:, :SEQ_TXT] == 1))
def test_eligibility_rejects_every_unsupported_layout(self):
attn = _attention()
hidden, encoder = _streams()
def eligible(h=hidden, e=encoder, *, masked=False, sp=1, attention=attn):
with patch(SP_WORLD_SIZE, return_value=sp):
return _use_joint_qkv_buffers(
attn=attention,
hidden_states=h,
encoder_hidden_states=e,
masked=masked,
)
self.assertTrue(eligible())
self.assertFalse(eligible(masked=True))
self.assertFalse(eligible(sp=2))
self.assertFalse(eligible(h=hidden.expand(2, -1, -1).contiguous()))
strided_hidden = hidden.transpose(1, 2).contiguous().transpose(1, 2)
strided_encoder = encoder.transpose(1, 2).contiguous().transpose(1, 2)
self.assertFalse(eligible(h=strided_hidden))
self.assertFalse(eligible(e=strided_encoder))
packed = _attention()
packed.separate_unquantized_qkv_proj = False
self.assertFalse(eligible(attention=packed))
# An unmerged LoRA adds its delta in forward; the bare addmm would drop it.
lora_wrapped = _attention()
lora_wrapped.to_q = BaseLayerWithLoRA(lora_wrapped.to_q)
self.assertFalse(eligible(attention=lora_wrapped))
def test_eligibility_requires_one_dtype_across_streams_weights_and_autocast(
self,
):
"""The out= GEMMs neither promote nor autocast, so they stand in for
F.linear only when every operand already has the dtype it would use."""
attn = _attention()
hidden, encoder = _streams()
def eligible(h=hidden, e=encoder, *, attention=attn):
with patch(SP_WORLD_SIZE, return_value=1):
return _use_joint_qkv_buffers(
attn=attention,
hidden_states=h,
encoder_hidden_states=e,
masked=False,
)
self.assertFalse(eligible(h=hidden.float(), e=encoder.float()))
self.assertFalse(eligible(e=encoder.float()))
fp32_bias = _attention()
fp32_bias.to_k.bias.data = fp32_bias.to_k.bias.data.float()
self.assertFalse(eligible(attention=fp32_bias))
with torch.autocast("cuda", dtype=torch.float16):
self.assertFalse(eligible())
with torch.autocast("cuda", dtype=torch.bfloat16):
self.assertTrue(eligible())
self.assertFalse(eligible(h=hidden.float(), e=encoder.float()))
self.assertTrue(eligible())
fp16 = _attention(dtype=torch.float16)
h16, e16 = _streams(dtype=torch.float16)
self.assertTrue(eligible(h=h16, e=e16, attention=fp16))
class TestSyncIntoBufferUnderCompile(CustomTestCase):
def test_out_of_place_sync_compiles_without_graph_breaks(self):
"""Tracing the copy-back must produce one graph: no storage-pointer
comparison while Dynamo is compiling."""
@torch.no_grad()
def probe(x):
out = torch.empty_like(x)
_sync_into_buffer(value=x + 1, buf_view=out)
return out * 2
x = torch.arange(8, dtype=torch.float32)
report = torch._dynamo.explain(probe)(x)
self.assertEqual(report.graph_break_count, 0, report.break_reasons)
self.assertEqual(report.graph_count, 1)
torch._dynamo.reset()
self.assertTrue(torch.equal(torch.compile(probe)(x), (x + 1) * 2))
def test_eager_sync_copies_fresh_values_and_keeps_aliases(self):
"""Eager keeps the storage check: a fresh value is copied back, an
aliased one must not pay a redundant copy_. The version counter is
shared by a tensor and its views, so it counts in-place writes."""
buf = torch.zeros(8)
_sync_into_buffer(value=torch.ones(8), buf_view=buf)
self.assertTrue(torch.equal(buf, torch.ones(8)))
version = buf._version
_sync_into_buffer(value=buf[2:6], buf_view=buf[2:6])
self.assertEqual(buf._version, version)
@requires_convrot_kernel
class TestQwenImageJointQkvBuffersConvRot(_JointQkvCase):
"""The same forward with the six projections on convrot_int8's JIT
backend, which writes the joint buffers through its out= op."""
def setUp(self) -> None:
super().setUp()
self.config = ConvRotInt8Config(backend="jit")
def test_forward_is_bitwise_identical_to_join_seqs_path(self):
attn = _attention(quant_config=self.config)
self.assertEqual(attn.to_q.weight.dtype, torch.int8)
self._assert_joint_path_matches_join_seqs_path(attn)
def test_compiled_forward_matches_eager(self):
"""The out= writes into the joint buffers go through the registered
custom ops under torch.compile; they must trace and agree with eager."""
self._assert_compiled_forward_matches_eager(
_attention(quant_config=self.config)
)
def test_fp16_streams_bypass_joint_buffers_and_return_fp16(self):
"""FP16 streams must skip the joint buffers (the out= kernel stores BF16
only) and the forward must still return FP16."""
attn = _attention(dtype=torch.float16, quant_config=self.config)
self.assertEqual(attn.to_q.bias.dtype, torch.bfloat16)
out = self._assert_forward_bypasses_joint_buffers(attn, dtype=torch.float16)
self.assertEqual(out[0].dtype, torch.float16)
self.assertEqual(out[1].dtype, torch.float16)
def test_eligibility_requires_bf16_streams(self):
attn = _attention(quant_config=self.config)
hidden, encoder = _streams(dim=CONVROT_DIM)
def eligible(h, e):
with patch(SP_WORLD_SIZE, return_value=1):
return _use_joint_qkv_buffers(
attn=attn, hidden_states=h, encoder_hidden_states=e, masked=False
)
self.assertTrue(eligible(hidden, encoder))
self.assertFalse(eligible(hidden.half(), encoder.half()))
self.assertFalse(eligible(hidden, encoder.half()))
if __name__ == "__main__":
unittest.main()
@@ -623,7 +623,7 @@ class TestServerArgsPathExpansion(unittest.TestCase):
server_args.component_attention_backends,
)
self.assertEqual(
{"text_encoder": "kitchen_int8", "transformer": "fp8"},
{"text_encoder": "convrot_int8", "transformer": "fp8"},
server_args.component_quantizations,
)
self.assertEqual(
@@ -21,8 +21,8 @@ from sglang.multimodal_gen.runtime.layers.quantization.comfy_nvfp4 import (
ComfyNvfp4Config,
ComfyNvfp4LinearMethod,
)
from sglang.multimodal_gen.runtime.layers.quantization.configs.kitchen_int8_config import (
KitchenInt8Config,
from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config import (
ConvRotInt8Config,
)
from sglang.multimodal_gen.runtime.layers.quantization.configs.kitchen_w4a4_config import (
KitchenW4A4Config,
@@ -623,11 +623,11 @@ class TestTextEncoderQuantization(unittest.TestCase):
"/model/text_encoder",
"/model/text_encoder",
"text_encoder",
explicit_quantization="kitchen_int8",
explicit_quantization="convrot_int8",
ignored_layers=["lm_head"],
)
self.assertIsInstance(model_config.quant_config, KitchenInt8Config)
self.assertIsInstance(model_config.quant_config, ConvRotInt8Config)
self.assertFalse(model_config.quant_config.is_checkpoint_int8_serialized)
self.assertEqual(model_config.quant_config.ignored_layers, ["lm_head"])
@@ -688,7 +688,7 @@ class TestTextEncoderQuantization(unittest.TestCase):
"text_encoder",
)
self.assertIsInstance(model_config.quant_config, KitchenInt8Config)
self.assertIsInstance(model_config.quant_config, ConvRotInt8Config)
self.assertEqual(
set(model_config.quant_config.layer_markers),
{"model.visual.blocks.0.attn.qkv_proj"},
@@ -1129,8 +1129,17 @@ class TestQuantizedTextEncoderPostprocess(unittest.TestCase):
):
_require_quantized_encoder_layers(nn.Linear(2, 2), "text_encoder")
def test_online_convrot_int8_has_no_markers_to_consume(self):
"""An online convrot_int8 encoder carries no serialized markers, so the
consumption check must pass instead of failing on the absent map."""
_require_quantized_encoder_layers(
_QuantizedEncoder(_RecordingQuantMethod()),
"text_encoder",
quant_config=ConvRotInt8Config(ignored_layers=["lm_head"]),
)
def test_rejects_unconsumed_comfy_marker(self):
config = KitchenInt8Config(
config = ConvRotInt8Config(
layer_markers={
"visual.proj": {
"format": "int8_tensorwise",
@@ -58,8 +58,8 @@ from sglang.multimodal_gen.runtime.layers.quantization.comfy_fp8 import (
ComfyFp8Config,
ComfyFullPrecisionFp8LinearMethod,
)
from sglang.multimodal_gen.runtime.layers.quantization.configs.kitchen_int8_config import (
KitchenInt8Config,
from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config import (
ConvRotInt8Config,
)
from sglang.multimodal_gen.runtime.layers.quantization.configs.kitchen_w4a4_config import (
KitchenW4A4Config,
@@ -70,13 +70,13 @@ from sglang.multimodal_gen.runtime.layers.quantization.configs.kitchen_w4a8_conf
from sglang.multimodal_gen.runtime.layers.quantization.configs.nunchaku_config import (
NunchakuConfig,
)
from sglang.multimodal_gen.runtime.layers.quantization.convrot_int8_comfy_kitchen import (
ConvRotInt8ComfyKitchenLinearMethod,
)
from sglang.multimodal_gen.runtime.layers.quantization.fp8 import (
Fp8Config,
Fp8LinearMethod,
)
from sglang.multimodal_gen.runtime.layers.quantization.kitchen_int8 import (
KitchenInt8LinearMethod,
)
from sglang.multimodal_gen.runtime.layers.quantization.modelopt_quant import (
ModelOptFp4Config,
ModelOptFp4LinearMethod,
@@ -141,6 +141,13 @@ from sglang.srt.layers.quantization.bitsandbytes import (
from sglang.srt.layers.quantization.fp8 import Fp8Config as SRTFp8Config
from sglang.srt.layers.quantization.fp8 import Fp8LinearMethod as SRTFp8LinearMethod
# convrot_int8 picks its kernel backend from the GPU; pin comfy_kitchen so these
# cases do not depend on the runner.
_CONVROT_JIT_AVAILABLE = (
"sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config."
"_jit_available"
)
class _FakeFluxTransformer:
pass
@@ -707,10 +714,17 @@ class TestTransformerQuantHelpers(unittest.TestCase):
)
@patch(
"sglang.multimodal_gen.runtime.layers.quantization.kitchen_int8."
"sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config."
"_comfy_kitchen_available",
return_value=True,
)
@patch(
"sglang.multimodal_gen.runtime.layers.quantization.convrot_int8_comfy_kitchen."
"_load_comfy_kitchen"
)
def test_inspect_minimax_h3_safetensors_detects_curve_and_comfy_format(self, _load):
def test_inspect_minimax_h3_safetensors_detects_curve_and_comfy_format(
self, _load, _available
):
marker = json.dumps(
{
"format": "int8_tensorwise",
@@ -847,14 +861,27 @@ class TestTransformerQuantHelpers(unittest.TestCase):
}
)
self.assertIsInstance(config, KitchenInt8Config)
self.assertIsInstance(config, ConvRotInt8Config)
self.assertTrue(config.is_checkpoint_int8_serialized)
self.assertTrue(config.checkpoint_uses_native_qkv_layout)
self.assertFalse(KitchenInt8Config().checkpoint_uses_native_qkv_layout)
self.assertFalse(ConvRotInt8Config().checkpoint_uses_native_qkv_layout)
self.assertFalse(_needs_device_weight_postprocess(config))
self.assertTrue(config.supports_input_partition("blocks.0.mlp.fc1", 6400))
self.assertFalse(config.supports_input_partition("blocks.0.mlp.fc1", 3200))
@patch(_CONVROT_JIT_AVAILABLE, return_value=False)
def test_online_convrot_int8_loads_on_the_host_under_offload(self, _jit):
"""Online convrot_int8 quantizes each layer through its own CUDA round
trip, so the loader must not require the whole BF16 DiT on the GPU
(an offloaded 62 GB H3 DiT has to stay on the host during load)."""
config = ConvRotInt8Config()
self.assertFalse(config.is_checkpoint_int8_serialized)
self.assertFalse(_needs_device_weight_postprocess(config))
spec = TransformerQuantLoadSpec([], config, None, None)
self.assertTrue(spec.is_convrot_int8)
self.assertFalse(spec.is_serialized_convrot_int8)
def test_minimax_h3_w4a8_metadata_resolves_serialized_kitchen(self):
metadata = {
"_quantization_metadata": json.dumps(
@@ -1003,9 +1030,10 @@ class TestTransformerQuantHelpers(unittest.TestCase):
new=object(),
),
patch(
"sglang.multimodal_gen.runtime.layers.quantization.kitchen_int8."
"sglang.multimodal_gen.runtime.layers.quantization.convrot_int8_comfy_kitchen."
"_load_comfy_kitchen"
),
patch(_CONVROT_JIT_AVAILABLE, return_value=False),
):
config = resolve_minimax_h3_checkpoint_quantization(markers)
w4a4 = ReplicatedLinear(
@@ -1030,12 +1058,13 @@ class TestTransformerQuantHelpers(unittest.TestCase):
self.assertEqual(int8.weight.shape, (3, 256))
self.assertEqual(set(config.selected), {"w4a4", "int8"})
@patch(_CONVROT_JIT_AVAILABLE, return_value=False)
@patch(
"sglang.multimodal_gen.runtime.layers.quantization.kitchen_int8."
"sglang.multimodal_gen.runtime.layers.quantization.convrot_int8_comfy_kitchen."
"_load_comfy_kitchen"
)
def test_serialized_kitchen_constructs_int8_weight_and_row_scale(self, _load):
config = KitchenInt8Config(
def test_serialized_kitchen_constructs_int8_weight_and_row_scale(self, _load, _jit):
config = ConvRotInt8Config(
layer_markers={
"proj": {
"format": "int8_tensorwise",
@@ -1731,16 +1760,19 @@ class TestTransformerQuantHelpers(unittest.TestCase):
),
ModelOptFp4LinearMethod,
)
with patch(
"sglang.multimodal_gen.runtime.layers.quantization."
"kitchen_int8._load_comfy_kitchen"
with (
patch(
"sglang.multimodal_gen.runtime.layers.quantization."
"convrot_int8_comfy_kitchen._load_comfy_kitchen"
),
patch(_CONVROT_JIT_AVAILABLE, return_value=False),
):
self.assertIsInstance(
config.get_quant_method(
LinearBase(input_size=256, output_size=32),
"blocks.0.attn.out_proj",
),
KitchenInt8LinearMethod,
ConvRotInt8ComfyKitchenLinearMethod,
)
self.assertIsInstance(
config.get_quant_method(