[diffusion] quantization: ConvRot INT8 online W8A8 for DiTs (sgl-kernel + convrot_int8_customkernel) (#38040)
Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com>
This commit is contained in:
co-authored by
Po-Han Huang
parent
1a2eb6c426
commit
e667ab10d5
File diff suppressed because it is too large
Load Diff
+341
@@ -0,0 +1,341 @@
|
||||
/* Copyright 2026 SGLang Team. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
// Adapted from
|
||||
// https://github.com/NVIDIA/TensorRT-LLM/blob/be1788106245496872d18e702978e59b6bfd50e0/cpp/tensorrt_llm/cutlass_extensions/include/cutlass_extensions/epilogue/threadblock/epilogue_per_row_per_col_scale.h
|
||||
// Modifications: clang-format reflow, includes trimmed to CUTLASS, no tensorrt_llm dependency.
|
||||
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cutlass/arch/memory.h>
|
||||
#include <cutlass/numeric_conversion.h>
|
||||
|
||||
namespace cutlass {
|
||||
namespace epilogue {
|
||||
namespace threadblock {
|
||||
|
||||
template <
|
||||
typename ThreadblockShape_,
|
||||
int ThreadCount,
|
||||
typename ScaleTileIterator_,
|
||||
typename OutputTileIterator_,
|
||||
typename ElementAccumulator_,
|
||||
typename ElementCompute_,
|
||||
typename ElementwiseFunctor_,
|
||||
bool UseMasking_ = false>
|
||||
class EpilogueVisitorPerRowPerCol {
|
||||
public:
|
||||
using ThreadblockShape = ThreadblockShape_;
|
||||
static int const kThreadCount = ThreadCount;
|
||||
|
||||
using ScaleTileIterator = ScaleTileIterator_;
|
||||
using OutputTileIterator = OutputTileIterator_;
|
||||
using ElementwiseFunctor = ElementwiseFunctor_;
|
||||
|
||||
static int const kIterations = OutputTileIterator::kIterations;
|
||||
static int const kElementsPerAccess = OutputTileIterator::kElementsPerAccess;
|
||||
|
||||
using ElementOutput = typename OutputTileIterator::Element;
|
||||
using LayoutOutput = cutlass::layout::RowMajor;
|
||||
using ElementAccumulator = ElementAccumulator_;
|
||||
|
||||
using AlphaScaleElementType = typename ScaleTileIterator::Element;
|
||||
|
||||
using ElementCompute = ElementCompute_;
|
||||
using AccumulatorFragment = Array<ElementAccumulator, kElementsPerAccess>;
|
||||
using ComputeFragment = Array<ElementCompute_, kElementsPerAccess>;
|
||||
using OutputVector = Array<ElementOutput, kElementsPerAccess>;
|
||||
|
||||
static int const kThreadsPerRow = OutputTileIterator::ThreadMap::Detail::kAccessWidth;
|
||||
static bool const kHasMultiStepsInRow = (OutputTileIterator::ThreadMap::Iterations::kColumn > 1);
|
||||
|
||||
/// Argument structure
|
||||
struct Arguments {
|
||||
typename ElementwiseFunctor::Params elementwise;
|
||||
int64_t batch_stride_alpha;
|
||||
int64_t batch_stride_C;
|
||||
int64_t batch_stride_D;
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
Arguments() : batch_stride_alpha(0), batch_stride_C(0), batch_stride_D(0) {}
|
||||
|
||||
Arguments(typename ElementwiseFunctor::Params elementwise_)
|
||||
: elementwise(elementwise_), batch_stride_alpha(0), batch_stride_C(0), batch_stride_D(0) {}
|
||||
|
||||
Arguments(
|
||||
typename ElementwiseFunctor::Params elementwise_,
|
||||
int64_t batch_stride_alpha_,
|
||||
int64_t batch_stride_C_,
|
||||
int64_t batch_stride_D_)
|
||||
: elementwise(elementwise_),
|
||||
batch_stride_alpha(batch_stride_alpha_),
|
||||
batch_stride_C(batch_stride_C_),
|
||||
batch_stride_D(batch_stride_D_) {}
|
||||
};
|
||||
|
||||
struct Params {
|
||||
typename ElementwiseFunctor::Params elementwise;
|
||||
int64_t batch_stride_alpha;
|
||||
int64_t batch_stride_C;
|
||||
int64_t batch_stride_D;
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
CUTLASS_HOST_DEVICE
|
||||
Params() {}
|
||||
|
||||
CUTLASS_HOST_DEVICE
|
||||
Params(Arguments const& args)
|
||||
: elementwise(args.elementwise),
|
||||
batch_stride_alpha(args.batch_stride_alpha),
|
||||
batch_stride_C(args.batch_stride_C),
|
||||
batch_stride_D(args.batch_stride_D) {}
|
||||
};
|
||||
|
||||
/// Shared storage
|
||||
struct SharedStorage {};
|
||||
|
||||
private:
|
||||
Params const& params_;
|
||||
SharedStorage& shared_storage_;
|
||||
MatrixCoord extent_;
|
||||
MatrixCoord extent_real_;
|
||||
ElementwiseFunctor elementwise_;
|
||||
|
||||
bool const with_bias_;
|
||||
bool const per_token_quant_;
|
||||
bool const per_channel_quant_;
|
||||
|
||||
AlphaScaleElementType* ptr_alpha_row_;
|
||||
AlphaScaleElementType* ptr_alpha_col_;
|
||||
ScaleTileIterator iterator_alpha_col_;
|
||||
OutputTileIterator iterator_C_;
|
||||
OutputTileIterator iterator_D_;
|
||||
|
||||
AlphaScaleElementType element_alpha_row_ = 1.0f;
|
||||
AlphaScaleElementType element_alpha_col_ = 1.0f;
|
||||
typename ScaleTileIterator::Fragment fragment_alpha_col_;
|
||||
typename OutputTileIterator::Fragment fragment_C_;
|
||||
typename OutputTileIterator::Fragment fragment_D_;
|
||||
|
||||
ElementAccumulator beta_;
|
||||
|
||||
int column_offset_;
|
||||
|
||||
MatrixCoord thread_offset_;
|
||||
|
||||
public:
|
||||
CUTLASS_DEVICE
|
||||
EpilogueVisitorPerRowPerCol(
|
||||
Params const& params,
|
||||
SharedStorage& shared_storage,
|
||||
cutlass::MatrixCoord const& problem_size,
|
||||
int thread_idx,
|
||||
int warp_idx,
|
||||
int lane_idx,
|
||||
typename ScaleTileIterator::Params params_alpha_col,
|
||||
typename OutputTileIterator::Params params_C,
|
||||
typename OutputTileIterator::Params params_D,
|
||||
bool with_bias,
|
||||
bool per_token_quant,
|
||||
bool per_channel_quant,
|
||||
AlphaScaleElementType* ptr_alpha_row,
|
||||
AlphaScaleElementType* ptr_alpha_col,
|
||||
typename OutputTileIterator::Element* ptr_C,
|
||||
typename OutputTileIterator::Element* ptr_D,
|
||||
cutlass::MatrixCoord const& threadblock_offset = cutlass::MatrixCoord(0, 0),
|
||||
int column_offset = 0,
|
||||
cutlass::MatrixCoord const& problem_size_real = cutlass::MatrixCoord(0, 0))
|
||||
: params_(params),
|
||||
shared_storage_(shared_storage),
|
||||
extent_(problem_size),
|
||||
elementwise_(params.elementwise),
|
||||
with_bias_(with_bias),
|
||||
per_token_quant_(per_token_quant),
|
||||
per_channel_quant_(per_channel_quant),
|
||||
ptr_alpha_row_(ptr_alpha_row),
|
||||
ptr_alpha_col_(ptr_alpha_col),
|
||||
iterator_alpha_col_(params_alpha_col, ptr_alpha_col, problem_size, thread_idx, threadblock_offset),
|
||||
iterator_C_(params_C, ptr_C, problem_size, thread_idx, threadblock_offset),
|
||||
iterator_D_(params_D, ptr_D, problem_size, thread_idx, threadblock_offset),
|
||||
extent_real_(problem_size_real) {
|
||||
if (!per_channel_quant_ && (ptr_alpha_col_ != nullptr)) {
|
||||
element_alpha_col_ = *ptr_alpha_col_;
|
||||
}
|
||||
|
||||
if (!per_token_quant_ && (ptr_alpha_row_ != nullptr)) {
|
||||
element_alpha_row_ = *ptr_alpha_row_;
|
||||
}
|
||||
}
|
||||
|
||||
/// Helper to indicate split-K behavior
|
||||
CUTLASS_DEVICE
|
||||
void set_k_partition(
|
||||
int split_k_index, ///< Index of this threadblock within split-K partitioned scheme
|
||||
int split_k_slices) { ///< Total number of split-K slices
|
||||
}
|
||||
|
||||
/// Called to set the batch index
|
||||
CUTLASS_DEVICE
|
||||
void set_batch_index(int batch_idx) {
|
||||
iterator_alpha_col_.add_pointer_offset(batch_idx * params_.batch_stride_alpha);
|
||||
iterator_C_.add_pointer_offset(batch_idx * params_.batch_stride_C);
|
||||
iterator_D_.add_pointer_offset(batch_idx * params_.batch_stride_D);
|
||||
}
|
||||
|
||||
/// Called at the start of the epilogue just before iterating over accumulator slices
|
||||
CUTLASS_DEVICE
|
||||
void begin_epilogue() {
|
||||
if (per_channel_quant_) {
|
||||
iterator_alpha_col_.load(fragment_alpha_col_);
|
||||
}
|
||||
|
||||
if (with_bias_) {
|
||||
iterator_C_.load(fragment_C_);
|
||||
}
|
||||
}
|
||||
|
||||
/// Called at the start of one step before starting accumulator exchange
|
||||
CUTLASS_DEVICE
|
||||
void begin_step(int step_idx) {
|
||||
fragment_D_.clear();
|
||||
}
|
||||
|
||||
/// Called at the start of a row
|
||||
CUTLASS_DEVICE
|
||||
void begin_row(int row_idx) {
|
||||
// load alpha_row in begin_step only when per token(row) scaling is used
|
||||
if (per_token_quant_) {
|
||||
int thread_offset_row =
|
||||
iterator_D_.thread_start_row() + OutputTileIterator::ThreadMap::iteration_offset(row_idx).row();
|
||||
|
||||
arch::global_load<AlphaScaleElementType, sizeof(AlphaScaleElementType)>(
|
||||
element_alpha_row_, ptr_alpha_row_ + thread_offset_row, thread_offset_row < extent_.row());
|
||||
}
|
||||
}
|
||||
|
||||
/// Called after accumulators have been exchanged for each accumulator vector
|
||||
CUTLASS_DEVICE
|
||||
void visit(int iter_idx, int row_idx, int column_idx, int frag_idx, AccumulatorFragment const& accum) {
|
||||
NumericArrayConverter<ElementCompute, ElementAccumulator, kElementsPerAccess> source_converter;
|
||||
|
||||
ComputeFragment result = source_converter(accum);
|
||||
if (per_channel_quant_) {
|
||||
ComputeFragment alpha_col = reinterpret_cast<ComputeFragment*>(&fragment_alpha_col_)[column_idx];
|
||||
result = per_token_channel_scale_accumulator_(result, alpha_col, element_alpha_row_);
|
||||
} else {
|
||||
result = per_token_scale_accumulator_(result, element_alpha_col_, element_alpha_row_);
|
||||
}
|
||||
|
||||
if (with_bias_) {
|
||||
NumericArrayConverter<ElementCompute, ElementOutput, kElementsPerAccess> bias_converter;
|
||||
OutputVector bias = reinterpret_cast<OutputVector*>(&fragment_C_)[column_idx];
|
||||
result = bias_accumulator_(result, bias_converter(bias));
|
||||
}
|
||||
|
||||
// Convert to the output
|
||||
NumericArrayConverter<ElementOutput, ElementCompute, kElementsPerAccess> output_converter;
|
||||
OutputVector& output = reinterpret_cast<OutputVector*>(&fragment_D_)[frag_idx];
|
||||
output = output_converter(result);
|
||||
}
|
||||
|
||||
/// Called at the end of a row
|
||||
CUTLASS_DEVICE
|
||||
void end_row(int row_idx) {}
|
||||
|
||||
/// Called after all accumulator elements have been visited
|
||||
CUTLASS_DEVICE
|
||||
void end_step(int step_idx) {
|
||||
iterator_D_.store(fragment_D_);
|
||||
++iterator_D_;
|
||||
}
|
||||
|
||||
/// Called after all steps have been completed
|
||||
CUTLASS_DEVICE
|
||||
void end_epilogue() {}
|
||||
|
||||
private:
|
||||
CUTLASS_DEVICE
|
||||
ComputeFragment per_token_channel_scale_accumulator_(
|
||||
ComputeFragment const& accum, ComputeFragment const& scale_col, AlphaScaleElementType const& scale_row) {
|
||||
ComputeFragment result;
|
||||
CUTLASS_PRAGMA_UNROLL
|
||||
for (int i = 0; i < ComputeFragment::kElements; ++i) {
|
||||
result[i] = accum[i] * (scale_col[i] * scale_row);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
CUTLASS_DEVICE
|
||||
ComputeFragment per_token_scale_accumulator_(
|
||||
ComputeFragment const& accum, AlphaScaleElementType const& scale_col, AlphaScaleElementType const& scale_row) {
|
||||
ComputeFragment result;
|
||||
CUTLASS_PRAGMA_UNROLL
|
||||
for (int i = 0; i < ComputeFragment::kElements; ++i) {
|
||||
result[i] = accum[i] * (scale_col * scale_row);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
CUTLASS_DEVICE
|
||||
ComputeFragment bias_accumulator_(ComputeFragment const& accum, ComputeFragment const& bias) {
|
||||
ComputeFragment result;
|
||||
CUTLASS_PRAGMA_UNROLL
|
||||
for (int i = 0; i < OutputVector::kElements; ++i) {
|
||||
result[i] = accum[i] + bias[i];
|
||||
}
|
||||
return result;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace threadblock
|
||||
} // namespace epilogue
|
||||
} // namespace cutlass
|
||||
+388
@@ -0,0 +1,388 @@
|
||||
/* Copyright 2026 SGLang Team. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
// Adapted from
|
||||
// https://github.com/NVIDIA/TensorRT-LLM/blob/be1788106245496872d18e702978e59b6bfd50e0/cpp/tensorrt_llm/cutlass_extensions/include/cutlass_extensions/gemm/device/gemm_universal_base_compat.h
|
||||
// Modifications: clang-format reflow, includes trimmed to CUTLASS, no tensorrt_llm dependency.
|
||||
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
#pragma once
|
||||
|
||||
#include <cutlass/cutlass.h>
|
||||
#include <cutlass/device_kernel.h>
|
||||
#include <cutlass/trace.h>
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace cutlass {
|
||||
namespace gemm {
|
||||
namespace device {
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/*
|
||||
This is the device layer from CUTLASS 2.10 (SHA - cc85b64cf676c45f98a17e3a47c0aafcf817f088)
|
||||
It is replicated here since we needed to duplicate kernel level APIs for mixed dtype GEMMs
|
||||
and SmoothQuant. The newer device layer is not compatible with these older kernel level APIs.
|
||||
|
||||
Note: While CUTLASS 3.x supports stream-k, none of the kernels in the extensions folder support
|
||||
that feature at the moment.
|
||||
*/
|
||||
|
||||
template <typename GemmKernel_>
|
||||
class GemmUniversalBaseCompat {
|
||||
public:
|
||||
using GemmKernel = GemmKernel_;
|
||||
using ThreadblockShape = typename GemmKernel::Mma::Shape;
|
||||
|
||||
using ElementA = typename GemmKernel::ElementA;
|
||||
using LayoutA = typename GemmKernel::LayoutA;
|
||||
using TensorRefA = TensorRef<ElementA const, LayoutA>;
|
||||
static ComplexTransform const kTransformA = GemmKernel::kTransformA;
|
||||
|
||||
using ElementB = typename GemmKernel::ElementB;
|
||||
using LayoutB = typename GemmKernel::LayoutB;
|
||||
using TensorRefB = TensorRef<ElementB const, LayoutB>;
|
||||
static ComplexTransform const kTransformB = GemmKernel::kTransformB;
|
||||
|
||||
using ElementC = typename GemmKernel::ElementC;
|
||||
using LayoutC = typename GemmKernel::LayoutC;
|
||||
using TensorRefC = TensorRef<ElementC const, LayoutC>;
|
||||
using TensorRefD = TensorRef<ElementC, LayoutC>;
|
||||
|
||||
using ElementAccumulator = typename GemmKernel::Mma::Policy::Operator::ElementC;
|
||||
|
||||
using EpilogueOutputOp = typename GemmKernel::EpilogueOutputOp;
|
||||
using ThreadblockSwizzle = typename GemmKernel::ThreadblockSwizzle;
|
||||
using Operator = typename GemmKernel::Operator;
|
||||
|
||||
/// Argument structure
|
||||
using Arguments = typename GemmKernel::Arguments;
|
||||
|
||||
protected:
|
||||
/// Kernel parameters object
|
||||
typename GemmKernel::Params params_;
|
||||
|
||||
protected:
|
||||
/// Private helper to obtain the grid dimensions with fix-up for split-K
|
||||
static void get_grid_shape_(gemm::GemmCoord& grid_tiled_shape, int& gemm_k_size, Arguments const& args) {
|
||||
// Determine grid shape
|
||||
ThreadblockSwizzle threadblock_swizzle;
|
||||
|
||||
grid_tiled_shape = threadblock_swizzle.get_tiled_shape(
|
||||
args.problem_size, {ThreadblockShape::kM, ThreadblockShape::kN, ThreadblockShape::kK}, args.batch_count);
|
||||
|
||||
gemm_k_size = args.problem_size.k();
|
||||
|
||||
if (args.mode == GemmUniversalMode::kGemm || args.mode == GemmUniversalMode::kGemmSplitKParallel) {
|
||||
int const kAlignK =
|
||||
const_max(const_max(128 / sizeof_bits<ElementA>::value, 128 / sizeof_bits<ElementB>::value), 1);
|
||||
|
||||
gemm_k_size = round_up(ceil_div(args.problem_size.k(), args.batch_count), kAlignK);
|
||||
|
||||
if (gemm_k_size) {
|
||||
grid_tiled_shape.k() = ceil_div(args.problem_size.k(), gemm_k_size);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public:
|
||||
/// Constructs the GEMM.
|
||||
GemmUniversalBaseCompat() {}
|
||||
|
||||
/// Determines whether the GEMM can execute the given problem.
|
||||
static Status can_implement(Arguments const& args) {
|
||||
// Determine grid shape
|
||||
cutlass::gemm::GemmCoord grid_tiled_shape;
|
||||
int gemm_k_size = 0;
|
||||
|
||||
get_grid_shape_(grid_tiled_shape, gemm_k_size, args);
|
||||
|
||||
ThreadblockSwizzle threadblock_swizzle;
|
||||
dim3 grid = threadblock_swizzle.get_grid_shape(grid_tiled_shape);
|
||||
|
||||
uint32_t const kGridYZMax = ((1 << (sizeof(uint16_t) * 8)) - 1);
|
||||
|
||||
if (!(grid.y <= kGridYZMax && grid.z <= kGridYZMax)) {
|
||||
return Status::kErrorInvalidProblem;
|
||||
}
|
||||
|
||||
return GemmKernel::can_implement(args);
|
||||
}
|
||||
|
||||
/// Gets the workspace size
|
||||
static size_t get_workspace_size(Arguments const& args) {
|
||||
CUTLASS_TRACE_HOST("GemmUniversalBaseCompat::get_workspace_size()");
|
||||
|
||||
size_t workspace_bytes = 0;
|
||||
|
||||
// Determine grid shape
|
||||
cutlass::gemm::GemmCoord grid_tiled_shape;
|
||||
int gemm_k_size = 0;
|
||||
|
||||
get_grid_shape_(grid_tiled_shape, gemm_k_size, args);
|
||||
|
||||
if (args.mode == GemmUniversalMode::kGemmSplitKParallel) {
|
||||
// Split-K parallel always requires a temporary workspace
|
||||
workspace_bytes = sizeof(ElementC) * size_t(args.batch_stride_D) * size_t(grid_tiled_shape.k());
|
||||
} else if (args.mode == GemmUniversalMode::kGemm && grid_tiled_shape.k() > 1) {
|
||||
// Serial split-K only requires a temporary workspace if the number of partitions along the
|
||||
// GEMM K dimension is greater than one.
|
||||
workspace_bytes = sizeof(int) * size_t(grid_tiled_shape.m()) * size_t(grid_tiled_shape.n());
|
||||
}
|
||||
|
||||
CUTLASS_TRACE_HOST(" workspace_bytes: " << workspace_bytes);
|
||||
|
||||
workspace_bytes += GemmKernel::get_extra_workspace_size(args, grid_tiled_shape);
|
||||
|
||||
return workspace_bytes;
|
||||
}
|
||||
|
||||
/// Computes the grid shape
|
||||
static dim3 get_grid_shape(Arguments const& args) {
|
||||
CUTLASS_TRACE_HOST("GemmUniversalBaseCompat::get_grid_shape()");
|
||||
|
||||
ThreadblockSwizzle threadblock_swizzle;
|
||||
|
||||
cutlass::gemm::GemmCoord grid_tiled_shape;
|
||||
int gemm_k_size = 0;
|
||||
|
||||
get_grid_shape_(grid_tiled_shape, gemm_k_size, args);
|
||||
dim3 result = threadblock_swizzle.get_grid_shape(grid_tiled_shape);
|
||||
|
||||
CUTLASS_TRACE_HOST(
|
||||
" grid_tiled_shape: " << grid_tiled_shape << "\n"
|
||||
<< " result = {" << result << "}");
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/// Computes the maximum number of active blocks per multiprocessor
|
||||
static int maximum_active_blocks(int smem_capacity = -1) {
|
||||
CUTLASS_TRACE_HOST("GemmUniversalBaseCompat::maximum_active_blocks()");
|
||||
|
||||
int max_active_blocks = -1;
|
||||
int smem_size = int(sizeof(typename GemmKernel::SharedStorage));
|
||||
|
||||
CUTLASS_TRACE_HOST(" smem_size: " << smem_size << " bytes");
|
||||
|
||||
if (smem_size <= (48 << 10)) {
|
||||
cudaError_t result = cudaOccupancyMaxActiveBlocksPerMultiprocessor(
|
||||
&max_active_blocks, Kernel<GemmKernel>, GemmKernel::kThreadCount, smem_size);
|
||||
|
||||
if (result == cudaSuccess) {
|
||||
CUTLASS_TRACE_HOST(" max_active_blocks: " << max_active_blocks);
|
||||
return max_active_blocks;
|
||||
}
|
||||
} else {
|
||||
// Query assuming zero shared memory then compute occupancy limit based on SMEM
|
||||
cudaError_t result = cudaOccupancyMaxActiveBlocksPerMultiprocessor(
|
||||
&max_active_blocks, Kernel<GemmKernel>, GemmKernel::kThreadCount, 0);
|
||||
|
||||
if (result != cudaSuccess) {
|
||||
CUTLASS_TRACE_HOST(
|
||||
" cudaOccupancyMaxActiveBlocksPerMultiprocessor() returned error " << cudaGetErrorString(result));
|
||||
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (smem_capacity < 0) {
|
||||
int device_idx = 0;
|
||||
result = cudaGetDevice(&device_idx);
|
||||
|
||||
if (result != cudaSuccess) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
cudaDeviceProp properties;
|
||||
result = cudaGetDeviceProperties(&properties, device_idx);
|
||||
|
||||
if (result != cudaSuccess) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
smem_capacity = static_cast<int>(properties.sharedMemPerMultiprocessor);
|
||||
}
|
||||
|
||||
int occupancy = std::min(max_active_blocks, smem_capacity / smem_size);
|
||||
|
||||
CUTLASS_TRACE_HOST(" occupancy: " << occupancy);
|
||||
|
||||
return occupancy;
|
||||
}
|
||||
|
||||
CUTLASS_TRACE_HOST(" returning internal error");
|
||||
|
||||
return -1;
|
||||
}
|
||||
|
||||
/// Initializes GEMM state from arguments.
|
||||
Status initialize(Arguments const& args, void* workspace = nullptr, cudaStream_t stream = nullptr) {
|
||||
CUTLASS_TRACE_HOST(
|
||||
"GemmUniversalBaseCompat::initialize() - workspace " << workspace
|
||||
<< ", stream: " << (stream ? "non-null" : "null"));
|
||||
|
||||
size_t workspace_bytes = get_workspace_size(args);
|
||||
|
||||
CUTLASS_TRACE_HOST(" workspace_bytes: " << workspace_bytes);
|
||||
|
||||
if (workspace_bytes) {
|
||||
if (!workspace) {
|
||||
CUTLASS_TRACE_HOST(" error: device workspace must not be null");
|
||||
|
||||
return Status::kErrorWorkspaceNull;
|
||||
}
|
||||
|
||||
if (args.mode == GemmUniversalMode::kGemm) {
|
||||
CUTLASS_TRACE_HOST(" clearing device workspace");
|
||||
cudaError_t result = cudaMemsetAsync(workspace, 0, workspace_bytes, stream);
|
||||
|
||||
if (result != cudaSuccess) {
|
||||
CUTLASS_TRACE_HOST(" cudaMemsetAsync() returned error " << cudaGetErrorString(result));
|
||||
|
||||
return Status::kErrorInternal;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Get CUDA grid shape
|
||||
cutlass::gemm::GemmCoord grid_tiled_shape;
|
||||
int gemm_k_size = 0;
|
||||
|
||||
get_grid_shape_(grid_tiled_shape, gemm_k_size, args);
|
||||
|
||||
// Initialize the Params structure
|
||||
params_ = typename GemmKernel::Params(args, grid_tiled_shape, gemm_k_size, static_cast<int*>(workspace));
|
||||
|
||||
// Specify shared memory capacity for kernel.
|
||||
int smem_size = int(sizeof(typename GemmKernel::SharedStorage));
|
||||
|
||||
if (smem_size >= (48 << 10)) {
|
||||
cudaError_t result =
|
||||
cudaFuncSetAttribute(Kernel<GemmKernel>, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size);
|
||||
|
||||
if (result != cudaSuccess) {
|
||||
return Status::kErrorInternal;
|
||||
}
|
||||
}
|
||||
|
||||
return Status::kSuccess;
|
||||
}
|
||||
|
||||
/// Lightweight update given a subset of arguments
|
||||
Status update(Arguments const& args, void* workspace = nullptr) {
|
||||
CUTLASS_TRACE_HOST("GemmUniversalBaseCompat()::update() - workspace: " << workspace);
|
||||
|
||||
size_t workspace_bytes = get_workspace_size(args);
|
||||
|
||||
if (workspace_bytes && !workspace) {
|
||||
return Status::kErrorWorkspaceNull;
|
||||
}
|
||||
|
||||
params_.update(args, workspace);
|
||||
|
||||
return Status::kSuccess;
|
||||
}
|
||||
|
||||
/// Runs the kernel using initialized state.
|
||||
Status run(cudaStream_t stream = nullptr) {
|
||||
CUTLASS_TRACE_HOST("GemmUniversalBaseCompat::run()");
|
||||
|
||||
//
|
||||
// Configure grid and block dimensions
|
||||
//
|
||||
|
||||
ThreadblockSwizzle threadblock_swizzle;
|
||||
|
||||
dim3 grid = threadblock_swizzle.get_grid_shape(params_.grid_tiled_shape);
|
||||
dim3 block(GemmKernel::kThreadCount, 1, 1);
|
||||
|
||||
int smem_size = int(sizeof(typename GemmKernel::SharedStorage));
|
||||
|
||||
//
|
||||
// Launch kernel
|
||||
//
|
||||
|
||||
CUTLASS_TRACE_HOST(" grid: (" << grid << "), block: (" << block << "), SMEM: " << smem_size << " bytes");
|
||||
|
||||
// Launch
|
||||
cutlass::Kernel<GemmKernel><<<grid, block, smem_size, stream>>>(params_);
|
||||
|
||||
//
|
||||
// Query for errors
|
||||
//
|
||||
cudaError_t result = cudaGetLastError();
|
||||
|
||||
if (result != cudaSuccess) {
|
||||
CUTLASS_TRACE_HOST(" grid launch failed with error " << cudaGetErrorString(result));
|
||||
return Status::kErrorInternal;
|
||||
}
|
||||
|
||||
return Status::kSuccess;
|
||||
}
|
||||
|
||||
/// Runs the kernel using initialized state.
|
||||
Status operator()(cudaStream_t stream = nullptr) {
|
||||
return run(stream);
|
||||
}
|
||||
|
||||
/// Runs the kernel using initialized state.
|
||||
Status operator()(Arguments const& args, void* workspace = nullptr, cudaStream_t stream = nullptr) {
|
||||
Status status = initialize(args, workspace, stream);
|
||||
|
||||
if (status == Status::kSuccess) {
|
||||
status = run(stream);
|
||||
}
|
||||
|
||||
return status;
|
||||
}
|
||||
};
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace device
|
||||
} // namespace gemm
|
||||
} // namespace cutlass
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
+524
@@ -0,0 +1,524 @@
|
||||
/* Copyright 2026 SGLang Team. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
// Adapted from
|
||||
// https://github.com/NVIDIA/TensorRT-LLM/blob/be1788106245496872d18e702978e59b6bfd50e0/cpp/tensorrt_llm/cutlass_extensions/include/cutlass_extensions/gemm/kernel/gemm_with_epilogue_visitor.h
|
||||
// Modifications: clang-format reflow, includes trimmed to CUTLASS, no tensorrt_llm dependency.
|
||||
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017 - 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cutlass/complex.h>
|
||||
#include <cutlass/cutlass.h>
|
||||
#include <cutlass/fast_math.h>
|
||||
#include <cutlass/matrix_coord.h>
|
||||
#include <cutlass/trace.h>
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace cutlass {
|
||||
namespace gemm {
|
||||
namespace kernel {
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <
|
||||
typename Mma_, ///! Threadblock-scoped matrix multiply-accumulate
|
||||
typename Epilogue_, ///! Epilogue
|
||||
typename ThreadblockSwizzle_ ///! Threadblock swizzling function
|
||||
>
|
||||
struct GemmWithEpilogueVisitor {
|
||||
public:
|
||||
using Mma = Mma_;
|
||||
using Epilogue = Epilogue_;
|
||||
using EpilogueVisitor = typename Epilogue::Visitor;
|
||||
using ThreadblockSwizzle = ThreadblockSwizzle_;
|
||||
|
||||
using ElementA = typename Mma::IteratorA::Element;
|
||||
using LayoutA = typename Mma::IteratorA::Layout;
|
||||
using TensorRefA = TensorRef<ElementA, LayoutA>;
|
||||
|
||||
using ElementB = typename Mma::IteratorB::Element;
|
||||
using LayoutB = typename Mma::IteratorB::Layout;
|
||||
using TensorRefB = TensorRef<ElementB, LayoutB>;
|
||||
|
||||
using ElementCompute = typename EpilogueVisitor::ElementCompute;
|
||||
using LayoutAlphaCol = cutlass::layout::RowMajor;
|
||||
using LayoutAlphaRow = cutlass::layout::ColumnMajor;
|
||||
using TensorRefAlphaCol = TensorRef<ElementCompute, LayoutAlphaCol>;
|
||||
using TensorRefAlphaRow = TensorRef<ElementCompute, LayoutAlphaRow>;
|
||||
|
||||
using ElementC = typename EpilogueVisitor::ElementOutput;
|
||||
using LayoutC = typename Epilogue::Layout;
|
||||
using TensorRefC = TensorRef<ElementC, LayoutC>;
|
||||
|
||||
static ComplexTransform const kTransformA = Mma::kTransformA;
|
||||
static ComplexTransform const kTransformB = Mma::kTransformB;
|
||||
using Operator = typename Mma::Operator;
|
||||
|
||||
using OperatorClass = typename Mma::Operator::OperatorClass;
|
||||
using ThreadblockShape = typename Mma::Shape;
|
||||
using WarpShape = typename Mma::Operator::Shape;
|
||||
using InstructionShape = typename Mma::Policy::Operator::InstructionShape;
|
||||
using ArchTag = typename Mma::ArchTag;
|
||||
using EpilogueOutputOp =
|
||||
typename Epilogue::Visitor::ElementwiseFunctor; // Define type so GemmUniversalBase doesn't complain
|
||||
|
||||
static int const kStages = Mma::kStages;
|
||||
static int const kAlignmentA = Mma::IteratorA::AccessType::kElements;
|
||||
static int const kAlignmentB = Mma::IteratorB::AccessType::kElements;
|
||||
static int const kAlignmentC = EpilogueVisitor::kElementsPerAccess;
|
||||
|
||||
/// Warp count (concept: GemmShape)
|
||||
using WarpCount = typename Mma::WarpCount;
|
||||
static int const kThreadCount = 32 * WarpCount::kCount;
|
||||
|
||||
/// Split-K preserves splits that are 128b aligned
|
||||
static int const kSplitKAlignment = const_max(128 / sizeof_bits<ElementA>::value, 128 / sizeof_bits<ElementB>::value);
|
||||
|
||||
//
|
||||
// Structures
|
||||
//
|
||||
|
||||
/// Argument structure
|
||||
struct Arguments {
|
||||
//
|
||||
// Data members
|
||||
//
|
||||
|
||||
GemmUniversalMode mode;
|
||||
GemmCoord problem_size;
|
||||
int batch_count;
|
||||
|
||||
TensorRefA ref_A;
|
||||
TensorRefB ref_B;
|
||||
TensorRefAlphaCol ref_alpha_col;
|
||||
TensorRefAlphaRow ref_alpha_row;
|
||||
TensorRefC ref_C;
|
||||
TensorRefC ref_D;
|
||||
|
||||
int64_t batch_stride_A;
|
||||
int64_t batch_stride_B;
|
||||
int64_t batch_stride_D;
|
||||
|
||||
typename EpilogueVisitor::Arguments epilogue_visitor;
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
Arguments() : mode(GemmUniversalMode::kGemm), batch_count(1) {}
|
||||
|
||||
/// constructs an arguments structure
|
||||
Arguments(
|
||||
GemmCoord problem_size_,
|
||||
TensorRefA ref_A_,
|
||||
TensorRefB ref_B_,
|
||||
TensorRefAlphaCol ref_alpha_col_,
|
||||
TensorRefAlphaRow ref_alpha_row_,
|
||||
TensorRefC ref_C_,
|
||||
TensorRefC ref_D_,
|
||||
typename EpilogueVisitor::Arguments epilogue_visitor_)
|
||||
: mode(GemmUniversalMode::kGemm),
|
||||
problem_size(problem_size_),
|
||||
batch_count(1),
|
||||
ref_A(ref_A_),
|
||||
ref_B(ref_B_),
|
||||
ref_alpha_col(ref_alpha_col_),
|
||||
ref_alpha_row(ref_alpha_row_),
|
||||
ref_C(ref_C_),
|
||||
ref_D(ref_D_),
|
||||
batch_stride_A(0),
|
||||
batch_stride_B(0),
|
||||
batch_stride_D(0),
|
||||
epilogue_visitor(epilogue_visitor_) {}
|
||||
};
|
||||
|
||||
//
|
||||
// Structure for precomputing values in host memory and passing to kernels
|
||||
//
|
||||
|
||||
/// Parameters structure
|
||||
struct Params {
|
||||
cutlass::gemm::GemmCoord problem_size;
|
||||
cutlass::gemm::GemmCoord grid_tiled_shape;
|
||||
int swizzle_log_tile;
|
||||
|
||||
typename Mma::IteratorA::Params params_A;
|
||||
typename Mma::IteratorB::Params params_B;
|
||||
typename EpilogueVisitor::ScaleTileIterator::Params params_alpha_col;
|
||||
typename EpilogueVisitor::ScaleTileIterator::Params params_alpha_row;
|
||||
typename EpilogueVisitor::OutputTileIterator::Params params_C;
|
||||
typename EpilogueVisitor::OutputTileIterator::Params params_D;
|
||||
|
||||
GemmUniversalMode mode;
|
||||
int batch_count;
|
||||
int gemm_k_size;
|
||||
|
||||
void* ptr_A;
|
||||
void* ptr_B;
|
||||
typename EpilogueVisitor::ScaleTileIterator::Element* ptr_alpha_col;
|
||||
typename EpilogueVisitor::ScaleTileIterator::Element* ptr_alpha_row;
|
||||
ElementC* ptr_C;
|
||||
ElementC* ptr_D;
|
||||
|
||||
int64_t batch_stride_A;
|
||||
int64_t batch_stride_B;
|
||||
|
||||
typename EpilogueVisitor::Params epilogue_visitor;
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
CUTLASS_HOST_DEVICE
|
||||
Params()
|
||||
: swizzle_log_tile(0),
|
||||
params_A(0),
|
||||
params_B(0),
|
||||
params_alpha_col(0),
|
||||
params_C(0),
|
||||
params_D(0),
|
||||
batch_count(0),
|
||||
gemm_k_size(0),
|
||||
mode(cutlass::gemm::GemmUniversalMode::kGemm),
|
||||
ptr_A(nullptr),
|
||||
ptr_B(nullptr),
|
||||
ptr_alpha_col(nullptr),
|
||||
ptr_alpha_row(nullptr),
|
||||
ptr_C(nullptr),
|
||||
ptr_D(nullptr),
|
||||
batch_stride_A(0),
|
||||
batch_stride_B(0) {}
|
||||
|
||||
Params(Arguments const& args, cutlass::gemm::GemmCoord const& grid_tiled_shape_, int gemm_k_size_, int* workspace_)
|
||||
: problem_size(args.problem_size),
|
||||
swizzle_log_tile(0),
|
||||
params_A(args.ref_A.layout()),
|
||||
params_B(args.ref_B.layout()),
|
||||
params_alpha_col(args.ref_alpha_col.layout()),
|
||||
params_alpha_row(args.ref_alpha_col.layout()),
|
||||
params_C(args.ref_C.layout()),
|
||||
params_D(args.ref_D.layout()),
|
||||
mode(args.mode),
|
||||
batch_count(args.batch_count),
|
||||
gemm_k_size(args.problem_size.k()),
|
||||
ptr_A(args.ref_A.data()),
|
||||
ptr_B(args.ref_B.data()),
|
||||
ptr_alpha_col(args.ref_alpha_col.data()),
|
||||
ptr_alpha_row(args.ref_alpha_row.data()),
|
||||
ptr_C(args.ref_C.data()),
|
||||
ptr_D(args.ref_D.data()),
|
||||
batch_stride_A(args.batch_stride_A),
|
||||
batch_stride_B(args.batch_stride_B),
|
||||
epilogue_visitor(args.epilogue_visitor) {
|
||||
ThreadblockSwizzle threadblock_swizzle;
|
||||
|
||||
grid_tiled_shape = threadblock_swizzle.get_tiled_shape(
|
||||
args.problem_size, {ThreadblockShape::kM, ThreadblockShape::kN, ThreadblockShape::kK}, args.batch_count);
|
||||
|
||||
if (args.mode == GemmUniversalMode::kGemm || args.mode == GemmUniversalMode::kGemmSplitKParallel) {
|
||||
int const kAlignK =
|
||||
const_max(const_max(128 / sizeof_bits<ElementA>::value, 128 / sizeof_bits<ElementB>::value), 1);
|
||||
|
||||
gemm_k_size = round_up(ceil_div(args.problem_size.k(), args.batch_count), kAlignK);
|
||||
|
||||
if (gemm_k_size) {
|
||||
grid_tiled_shape.k() = ceil_div(args.problem_size.k(), gemm_k_size);
|
||||
}
|
||||
}
|
||||
|
||||
swizzle_log_tile = threadblock_swizzle.get_log_tile(grid_tiled_shape);
|
||||
}
|
||||
};
|
||||
|
||||
/// Shared memory storage structure
|
||||
union SharedStorage {
|
||||
typename Mma::SharedStorage main_loop;
|
||||
|
||||
struct {
|
||||
typename Epilogue::SharedStorage epilogue;
|
||||
typename EpilogueVisitor::SharedStorage visitor;
|
||||
} epilogue;
|
||||
};
|
||||
|
||||
public:
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
CUTLASS_DEVICE
|
||||
GemmWithEpilogueVisitor() {}
|
||||
|
||||
/// Determines whether kernel satisfies alignment
|
||||
static Status can_implement(cutlass::gemm::GemmCoord const& problem_size) {
|
||||
CUTLASS_TRACE_HOST("GemmWithEpilogueVisitor::can_implement()");
|
||||
|
||||
static int const kAlignmentA = Mma::IteratorA::AccessType::kElements;
|
||||
static int const kAlignmentB = Mma::IteratorB::AccessType::kElements;
|
||||
static int const kAlignmentC = EpilogueVisitor::OutputTileIterator::kElementsPerAccess;
|
||||
|
||||
bool isAMisaligned = false;
|
||||
bool isBMisaligned = false;
|
||||
bool isCMisaligned = false;
|
||||
|
||||
if (platform::is_same<LayoutA, layout::RowMajor>::value) {
|
||||
isAMisaligned = problem_size.k() % kAlignmentA;
|
||||
} else if (platform::is_same<LayoutA, layout::ColumnMajor>::value) {
|
||||
isAMisaligned = problem_size.m() % kAlignmentA;
|
||||
} else if (
|
||||
platform::is_same<LayoutA, layout::ColumnMajorInterleaved<32>>::value ||
|
||||
platform::is_same<LayoutA, layout::ColumnMajorInterleaved<64>>::value) {
|
||||
isAMisaligned = problem_size.k() % kAlignmentA;
|
||||
}
|
||||
|
||||
if (platform::is_same<LayoutB, layout::RowMajor>::value) {
|
||||
isBMisaligned = problem_size.n() % kAlignmentB;
|
||||
} else if (platform::is_same<LayoutB, layout::ColumnMajor>::value) {
|
||||
isBMisaligned = problem_size.k() % kAlignmentB;
|
||||
} else if (
|
||||
platform::is_same<LayoutB, layout::RowMajorInterleaved<32>>::value ||
|
||||
platform::is_same<LayoutB, layout::RowMajorInterleaved<64>>::value) {
|
||||
isBMisaligned = problem_size.k() % kAlignmentB;
|
||||
}
|
||||
|
||||
if (platform::is_same<LayoutC, layout::RowMajor>::value) {
|
||||
isCMisaligned = problem_size.n() % kAlignmentC;
|
||||
} else if (platform::is_same<LayoutC, layout::ColumnMajor>::value) {
|
||||
isCMisaligned = problem_size.m() % kAlignmentC;
|
||||
} else if (
|
||||
platform::is_same<LayoutC, layout::ColumnMajorInterleaved<32>>::value ||
|
||||
platform::is_same<LayoutC, layout::ColumnMajorInterleaved<64>>::value) {
|
||||
isCMisaligned = problem_size.n() % kAlignmentC;
|
||||
}
|
||||
|
||||
if (isAMisaligned) {
|
||||
CUTLASS_TRACE_HOST(" returning kErrorMisalignedOperand for A operand");
|
||||
return Status::kErrorMisalignedOperand;
|
||||
}
|
||||
|
||||
if (isBMisaligned) {
|
||||
CUTLASS_TRACE_HOST(" returning kErrorMisalignedOperand for B operand");
|
||||
return Status::kErrorMisalignedOperand;
|
||||
}
|
||||
|
||||
if (isCMisaligned) {
|
||||
CUTLASS_TRACE_HOST(" returning kErrorMisalignedOperand for C operand");
|
||||
return Status::kErrorMisalignedOperand;
|
||||
}
|
||||
|
||||
CUTLASS_TRACE_HOST(" returning kSuccess");
|
||||
|
||||
return Status::kSuccess;
|
||||
}
|
||||
|
||||
static Status can_implement(Arguments const& args) {
|
||||
return can_implement(args.problem_size);
|
||||
}
|
||||
|
||||
static size_t get_extra_workspace_size(Arguments const& args, cutlass::gemm::GemmCoord const& grid_tiled_shape) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
#define SPLIT_K_ENABLED 1
|
||||
|
||||
/// Executes one GEMM
|
||||
CUTLASS_DEVICE
|
||||
void run_kernel_(Params const& params, SharedStorage& shared_storage) {
|
||||
// Compute threadblock location
|
||||
ThreadblockSwizzle threadblock_swizzle;
|
||||
|
||||
cutlass::gemm::GemmCoord threadblock_tile_offset = threadblock_swizzle.get_tile_offset(params.swizzle_log_tile);
|
||||
|
||||
// Early exit if CTA is out of range
|
||||
if (params.grid_tiled_shape.m() <= threadblock_tile_offset.m() ||
|
||||
params.grid_tiled_shape.n() <= threadblock_tile_offset.n()) {
|
||||
return;
|
||||
}
|
||||
|
||||
int offset_k = 0;
|
||||
int problem_size_k = params.problem_size.k();
|
||||
|
||||
ElementA* ptr_A = static_cast<ElementA*>(params.ptr_A);
|
||||
ElementB* ptr_B = static_cast<ElementB*>(params.ptr_B);
|
||||
|
||||
#if SPLIT_K_ENABLED
|
||||
//
|
||||
// Fetch pointers based on mode.
|
||||
//
|
||||
if (params.mode == GemmUniversalMode::kGemm || params.mode == GemmUniversalMode::kGemmSplitKParallel) {
|
||||
if (threadblock_tile_offset.k() + 1 < params.grid_tiled_shape.k()) {
|
||||
problem_size_k = (threadblock_tile_offset.k() + 1) * params.gemm_k_size;
|
||||
}
|
||||
|
||||
offset_k = threadblock_tile_offset.k() * params.gemm_k_size;
|
||||
} else if (params.mode == GemmUniversalMode::kBatched) {
|
||||
ptr_A += threadblock_tile_offset.k() * params.batch_stride_A;
|
||||
ptr_B += threadblock_tile_offset.k() * params.batch_stride_B;
|
||||
} else if (params.mode == GemmUniversalMode::kArray) {
|
||||
ptr_A = static_cast<ElementA* const*>(params.ptr_A)[threadblock_tile_offset.k()];
|
||||
ptr_B = static_cast<ElementB* const*>(params.ptr_B)[threadblock_tile_offset.k()];
|
||||
}
|
||||
#endif
|
||||
|
||||
// Compute initial location in logical coordinates
|
||||
cutlass::MatrixCoord tb_offset_A{
|
||||
threadblock_tile_offset.m() * Mma::Shape::kM,
|
||||
offset_k,
|
||||
};
|
||||
|
||||
cutlass::MatrixCoord tb_offset_B{offset_k, threadblock_tile_offset.n() * Mma::Shape::kN};
|
||||
|
||||
// Compute position within threadblock
|
||||
int thread_idx = threadIdx.x;
|
||||
|
||||
// Construct iterators to A and B operands
|
||||
typename Mma::IteratorA iterator_A(
|
||||
params.params_A, ptr_A, {params.problem_size.m(), problem_size_k}, thread_idx, tb_offset_A);
|
||||
|
||||
typename Mma::IteratorB iterator_B(
|
||||
params.params_B, ptr_B, {problem_size_k, params.problem_size.n()}, thread_idx, tb_offset_B);
|
||||
|
||||
// Broadcast the warp_id computed by lane 0 to ensure dependent code
|
||||
// is compiled as warp-uniform.
|
||||
int warp_idx = __shfl_sync(0xffffffff, threadIdx.x / 32, 0);
|
||||
|
||||
int lane_idx = threadIdx.x % 32;
|
||||
|
||||
//
|
||||
// Main loop
|
||||
//
|
||||
|
||||
// Construct thread-scoped matrix multiply
|
||||
Mma mma(shared_storage.main_loop, thread_idx, warp_idx, lane_idx);
|
||||
|
||||
typename Mma::FragmentC accumulators;
|
||||
|
||||
accumulators.clear();
|
||||
|
||||
// Compute threadblock-scoped matrix multiply-add
|
||||
int gemm_k_iterations = (problem_size_k - offset_k + Mma::Shape::kK - 1) / Mma::Shape::kK;
|
||||
|
||||
// Compute threadblock-scoped matrix multiply-add
|
||||
mma(gemm_k_iterations, accumulators, iterator_A, iterator_B, accumulators);
|
||||
|
||||
//
|
||||
// Masked tile iterators constructed from members
|
||||
//
|
||||
|
||||
threadblock_tile_offset = threadblock_swizzle.get_tile_offset(params.swizzle_log_tile);
|
||||
|
||||
// assume identity swizzle
|
||||
MatrixCoord threadblock_offset(
|
||||
threadblock_tile_offset.m() * Mma::Shape::kM, threadblock_tile_offset.n() * Mma::Shape::kN);
|
||||
|
||||
int block_idx = threadblock_tile_offset.m() + threadblock_tile_offset.n() * params.grid_tiled_shape.m();
|
||||
|
||||
//
|
||||
// Construct the epilogue visitor
|
||||
//
|
||||
|
||||
bool with_bias = true;
|
||||
if (params.ptr_C == nullptr) {
|
||||
with_bias = false;
|
||||
}
|
||||
|
||||
EpilogueVisitor epilogue_visitor(
|
||||
params.epilogue_visitor,
|
||||
shared_storage.epilogue.visitor,
|
||||
params.problem_size.mn(),
|
||||
thread_idx,
|
||||
warp_idx,
|
||||
lane_idx,
|
||||
params.params_alpha_col,
|
||||
params.params_C,
|
||||
params.params_D,
|
||||
with_bias,
|
||||
true,
|
||||
true,
|
||||
params.ptr_alpha_row,
|
||||
params.ptr_alpha_col,
|
||||
params.ptr_C,
|
||||
params.ptr_D,
|
||||
threadblock_offset,
|
||||
blockIdx.y * params.problem_size.m());
|
||||
|
||||
if (params.mode == GemmUniversalMode::kGemm) {
|
||||
// Indicate which position in a serial reduction the output operator is currently updating
|
||||
epilogue_visitor.set_k_partition(threadblock_tile_offset.k(), params.grid_tiled_shape.k());
|
||||
} else if (params.mode == GemmUniversalMode::kBatched || params.mode == GemmUniversalMode::kArray) {
|
||||
epilogue_visitor.set_batch_index(threadblock_tile_offset.k());
|
||||
}
|
||||
|
||||
// Construct the epilogue
|
||||
Epilogue epilogue(shared_storage.epilogue.epilogue, thread_idx, warp_idx, lane_idx);
|
||||
|
||||
// Execute the epilogue operator to update the destination tensor.
|
||||
epilogue(epilogue_visitor, accumulators);
|
||||
}
|
||||
|
||||
template <typename CompilationArch>
|
||||
CUTLASS_DEVICE void run_kernel(Params const& params, SharedStorage& shared_storage) {
|
||||
if constexpr (platform::is_same<ArchTag, CompilationArch>::value) {
|
||||
run_kernel_(params, shared_storage);
|
||||
} else {
|
||||
CUTLASS_NOT_IMPLEMENTED();
|
||||
}
|
||||
}
|
||||
|
||||
/// Executes one GEMM
|
||||
CUTLASS_DEVICE
|
||||
void operator()(Params const& params, SharedStorage& shared_storage) {
|
||||
run_kernel<ArchTag>(params, shared_storage);
|
||||
}
|
||||
};
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace kernel
|
||||
} // namespace gemm
|
||||
} // namespace cutlass
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
@@ -480,6 +480,50 @@ register_kernel(
|
||||
),
|
||||
)
|
||||
)
|
||||
# Mirrors SUPPORTED_CAPABILITIES in ops.gemm.convrot_int8 (pinned by a test).
|
||||
_CONVROT_INT8 = frozenset(
|
||||
CapabilityRequirement.cuda(min_sm=sm, max_sm=sm)
|
||||
for sm in ((9, 0), (10, 0), (12, 0), (12, 1))
|
||||
)
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="gemm.convrot_rotate_quantize_activation",
|
||||
backend=KernelBackend.JIT,
|
||||
target="sglang.kernels.ops.gemm.convrot_int8:convrot_rotate_quantize_activation",
|
||||
capabilities=_CONVROT_INT8,
|
||||
format_signature=FormatSignature(
|
||||
supported_dtypes=("bfloat16",),
|
||||
description="group-wise regular Hadamard rotation + per-row INT8 quant of BF16 [M, K] -> (int8 [M, K], fp32 [M])",
|
||||
),
|
||||
description="ConvRot INT8 activation / weight quantizer (sglang.kernels.jit).",
|
||||
)
|
||||
)
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="gemm.convrot_int8_fused_linear",
|
||||
backend=KernelBackend.JIT,
|
||||
target="sglang.kernels.ops.gemm.convrot_int8:convrot_int8_fused_linear",
|
||||
capabilities=_CONVROT_INT8,
|
||||
format_signature=FormatSignature(
|
||||
supported_dtypes=("bfloat16", "int8"),
|
||||
description="BF16 [M, K] x int8 [N, K] -> BF16 [M, N] with in-kernel rotation, INT8 quant and fused per-row x per-column dequant (+ bias)",
|
||||
),
|
||||
description="ConvRot INT8 W8A8 linear (sglang.kernels.jit).",
|
||||
)
|
||||
)
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="gemm.convrot_int8_linear_prequant",
|
||||
backend=KernelBackend.JIT,
|
||||
target="sglang.kernels.ops.gemm.convrot_int8:convrot_int8_linear_prequant",
|
||||
capabilities=_CONVROT_INT8,
|
||||
format_signature=FormatSignature(
|
||||
supported_dtypes=("int8", "float32", "bfloat16"),
|
||||
description="int8 [M, K] x int8 [N, K] with per-row / per-column fp32 scales -> BF16 [M, N] (+ bias); for linears sharing one rotated input",
|
||||
),
|
||||
description="ConvRot INT8 GEMM on a pre-quantized activation (sglang.kernels.jit).",
|
||||
)
|
||||
)
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="gemm.sm120_fp8_gemv",
|
||||
|
||||
@@ -0,0 +1,231 @@
|
||||
"""ConvRot INT8 W8A8 linear (arXiv:2512.03673), JIT-compiled for the local GPU.
|
||||
|
||||
One entry rotates a BF16 activation group-wise with the regular Hadamard,
|
||||
quantizes it per row to INT8 and runs the dense INT8 GEMM with the per-row x
|
||||
per-column dequant (and bias) fused into the epilogue; the rotate step alone
|
||||
is the offline transform for the [N, K] weight. CUTLASS 3.x WGMMA on CC 9.0,
|
||||
tcgen05 on CC 10.0 and the CUTLASS 2.x mma.sync path on CC 12.0 / 12.1; the
|
||||
module compiles once per machine on first use and is cached by the JIT.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.jit.utils import cache_once, get_jit_cuda_arch, load_jit
|
||||
from sglang.kernels.jit.utils.common import is_hip_runtime, is_musa_runtime
|
||||
from sglang.kernels.kernel_api_logging import debug_kernel_api
|
||||
from sglang.srt.utils.custom_op import register_custom_op
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from tvm_ffi.module import Module
|
||||
|
||||
# Exact compute capabilities the entry header carries a GEMM path for: sm_90a
|
||||
# and sm_100a hold the WGMMA / tcgen05 kernels, CC 12.0 / 12.1 the mma.sync
|
||||
# path. CC 10.3 is left out on purpose (INT8 tensor-core rate); the quant
|
||||
# method reports the reason.
|
||||
SUPPORTED_CAPABILITIES: frozenset[tuple[int, int]] = frozenset(
|
||||
{(9, 0), (10, 0), (12, 0), (12, 1)}
|
||||
)
|
||||
# Hadamard group widths the rotate kernel is instantiated for.
|
||||
GROUP_SIZES: tuple[int, ...] = (64, 128, 256, 512)
|
||||
|
||||
|
||||
def convrot_int8_supported_capabilities() -> frozenset[tuple[int, int]]:
|
||||
"""(major, minor) pairs the convrot_int8 ops compile for."""
|
||||
return SUPPORTED_CAPABILITIES
|
||||
|
||||
|
||||
def _convrot_int8_cuda_flags() -> list[str]:
|
||||
return [
|
||||
"-DNDEBUG",
|
||||
"-DCUTE_USE_PACKED_TUPLE=1",
|
||||
"-DCUTLASS_ENABLE_TENSOR_CORE_MMA=1",
|
||||
"-DCUTLASS_VERSIONS_GENERATED",
|
||||
"-DCUTLASS_TEST_LEVEL=0",
|
||||
"-DCUTLASS_TEST_ENABLE_CACHED_RESULTS=1",
|
||||
"-DCUTLASS_DEBUG_TRACE_LEVEL=0",
|
||||
"--expt-relaxed-constexpr",
|
||||
"--expt-extended-lambda",
|
||||
# Only the row-scale division and denormal handling see this flag; a
|
||||
# precise build rounds a few INT8 codes one step away from the
|
||||
# validated numerics. The GELU tanh is pinned to libdevice __nv_tanhf.
|
||||
"--use_fast_math",
|
||||
]
|
||||
|
||||
|
||||
def convrot_int8_local_capability() -> tuple[int, int] | None:
|
||||
"""(major, minor) the JIT build would target, or None off CUDA (ROCm and
|
||||
MUSA capability numbers are not SM versions)."""
|
||||
if is_hip_runtime() or is_musa_runtime():
|
||||
return None
|
||||
arch = get_jit_cuda_arch()
|
||||
return (arch.major, arch.minor)
|
||||
|
||||
|
||||
@cache_once
|
||||
def _jit_convrot_int8_module() -> Module:
|
||||
"""Compile and cache the convrot_int8 module for the local GPU."""
|
||||
capability = convrot_int8_local_capability()
|
||||
if capability not in SUPPORTED_CAPABILITIES:
|
||||
supported = ", ".join(f"{a}.{b}" for a, b in sorted(SUPPORTED_CAPABILITIES))
|
||||
where = (
|
||||
"a non-CUDA device"
|
||||
if capability is None
|
||||
else f"CC {capability[0]}.{capability[1]}"
|
||||
)
|
||||
raise RuntimeError(
|
||||
f"convrot_int8: no kernel for {where} (supported: {supported})"
|
||||
)
|
||||
return load_jit(
|
||||
"convrot_int8",
|
||||
cuda_files=["gemm/convrot_int8/convrot_int8_entry.cuh"],
|
||||
cuda_wrappers=[
|
||||
(
|
||||
"convrot_rotate_quantize_activation",
|
||||
"convrot_rotate_quantize_activation",
|
||||
),
|
||||
("convrot_int8_fused_linear", "convrot_int8_fused_linear"),
|
||||
("convrot_int8_linear_prequant", "convrot_int8_linear_prequant"),
|
||||
],
|
||||
extra_dependencies=["cutlass"],
|
||||
extra_cuda_cflags=_convrot_int8_cuda_flags(),
|
||||
)
|
||||
|
||||
|
||||
def load_convrot_int8_module() -> Module:
|
||||
"""Build (first use) or load the cached module; raises when the GPU is
|
||||
unsupported, the CUDA toolchain is missing or the build fails."""
|
||||
return _jit_convrot_int8_module()
|
||||
|
||||
|
||||
# The custom ops below exist for torch.compile: dynamo needs an opaque op. Eager
|
||||
# calls go straight to the module instead, because the dispatcher round trip of
|
||||
# a Python-implemented op costs ~9 us on top of the ~10 us tvm-ffi call
|
||||
# (measured on GB200) and a Qwen-Image DiT forward issues ~850 of these calls.
|
||||
@register_custom_op(
|
||||
op_name="convrot_rotate_quantize_activation",
|
||||
mutates_args=["x_q", "x_scale"],
|
||||
)
|
||||
def _convrot_rotate_quantize_activation_op(
|
||||
x_q: torch.Tensor,
|
||||
x_scale: torch.Tensor,
|
||||
x: torch.Tensor,
|
||||
group_size: int,
|
||||
) -> None:
|
||||
module = _jit_convrot_int8_module()
|
||||
module.convrot_rotate_quantize_activation(x_q, x_scale, x, group_size)
|
||||
|
||||
|
||||
@register_custom_op(op_name="convrot_int8_fused_linear", mutates_args=["out"])
|
||||
def _convrot_int8_fused_linear_op(
|
||||
out: torch.Tensor,
|
||||
x: torch.Tensor,
|
||||
weight_q: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
bias: torch.Tensor | None,
|
||||
group_size: int,
|
||||
gelu_input: bool,
|
||||
) -> None:
|
||||
module = _jit_convrot_int8_module()
|
||||
module.convrot_int8_fused_linear(
|
||||
out, x, weight_q, weight_scale, bias, group_size, gelu_input
|
||||
)
|
||||
|
||||
|
||||
@register_custom_op(op_name="convrot_int8_linear_prequant", mutates_args=["out"])
|
||||
def _convrot_int8_linear_prequant_op(
|
||||
out: torch.Tensor,
|
||||
xq: torch.Tensor,
|
||||
xs: torch.Tensor,
|
||||
weight_q: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
bias: torch.Tensor | None,
|
||||
group_size: int,
|
||||
) -> None:
|
||||
module = _jit_convrot_int8_module()
|
||||
module.convrot_int8_linear_prequant(
|
||||
out, xq, xs, weight_q, weight_scale, bias, group_size
|
||||
)
|
||||
|
||||
|
||||
@debug_kernel_api
|
||||
def convrot_rotate_quantize_activation(
|
||||
x: torch.Tensor, group_size: int
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Group Hadamard rotation + per-row INT8 quant of a contiguous BF16 [M, K].
|
||||
|
||||
Returns (x_q int8 [M, K], x_scale float32 [M]). Also the offline transform
|
||||
for a [N, K] weight, yielding (weight_q, weight_scale).
|
||||
"""
|
||||
x_q = torch.empty(x.shape, dtype=torch.int8, device=x.device)
|
||||
x_scale = torch.empty(x.shape[0], dtype=torch.float32, device=x.device)
|
||||
if torch.compiler.is_compiling():
|
||||
_convrot_rotate_quantize_activation_op(x_q, x_scale, x, group_size)
|
||||
else:
|
||||
_jit_convrot_int8_module().convrot_rotate_quantize_activation(
|
||||
x_q, x_scale, x, group_size
|
||||
)
|
||||
return x_q, x_scale
|
||||
|
||||
|
||||
@debug_kernel_api
|
||||
def convrot_int8_fused_linear(
|
||||
x: torch.Tensor,
|
||||
weight_q: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
bias: torch.Tensor | None,
|
||||
group_size: int,
|
||||
*,
|
||||
out: torch.Tensor | None = None,
|
||||
gelu_input: bool = False,
|
||||
) -> torch.Tensor:
|
||||
"""BF16 [M, K] x int8 [N, K] -> BF16 [M, N]; x is rotated and quantized in-kernel.
|
||||
|
||||
``out`` must be a contiguous BF16 [M, N] when given (a row slice of a larger
|
||||
buffer qualifies). ``gelu_input`` applies GELU(tanh) to x inside the rotate
|
||||
kernel, bitwise equal to the eager ``F.gelu(x, approximate="tanh")`` first.
|
||||
"""
|
||||
if out is None:
|
||||
out = torch.empty(
|
||||
(x.shape[0], weight_q.shape[0]), dtype=torch.bfloat16, device=x.device
|
||||
)
|
||||
if torch.compiler.is_compiling():
|
||||
_convrot_int8_fused_linear_op(
|
||||
out, x, weight_q, weight_scale, bias, group_size, gelu_input
|
||||
)
|
||||
else:
|
||||
_jit_convrot_int8_module().convrot_int8_fused_linear(
|
||||
out, x, weight_q, weight_scale, bias, group_size, gelu_input
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
@debug_kernel_api
|
||||
def convrot_int8_linear_prequant(
|
||||
xq: torch.Tensor,
|
||||
xs: torch.Tensor,
|
||||
weight_q: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
bias: torch.Tensor | None,
|
||||
group_size: int,
|
||||
*,
|
||||
out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""GEMM + dequant on (xq int8 [M, K], xs float32 [M]) from
|
||||
convrot_rotate_quantize_activation; bitwise equal to the fused op."""
|
||||
if out is None:
|
||||
out = torch.empty(
|
||||
(xq.shape[0], weight_q.shape[0]), dtype=torch.bfloat16, device=xq.device
|
||||
)
|
||||
if torch.compiler.is_compiling():
|
||||
_convrot_int8_linear_prequant_op(
|
||||
out, xq, xs, weight_q, weight_scale, bias, group_size
|
||||
)
|
||||
else:
|
||||
_jit_convrot_int8_module().convrot_int8_linear_prequant(
|
||||
out, xq, xs, weight_q, weight_scale, bias, group_size
|
||||
)
|
||||
return out
|
||||
@@ -172,7 +172,7 @@ These options **trade output quality** for speed or VRAM savings. Results will d
|
||||
| **Progressive Resolution** | `--progressive-mode dct_rewind --progressive-levels N --progressive-delta D` | Runs early denoising at lower latent resolution, then spectrally upsamples and switches to the target resolution. | Model- and schedule-dependent | Approximate and pipeline-specific. Keep the switch schedule fixed and compare detail, composition, and temporal stability. |
|
||||
| **Causal KV-Cache Quantization** | `--kv-cache-quant int4\|int2` plus optional `--kv-cache-quant-*` controls | Compresses completed causal KV-cache chunks with Quant-VideoGen PRQ while keeping the mutable/current chunk and recent chunks in BF16. | Primarily a long-session memory saving | Currently limited to LingBot World realtime causal serving; requires `quant-videogen`. INT4 is the starting point; INT2 saves more memory with more error. It quantizes cache state, not checkpoint weights. |
|
||||
| **Quantized Models (Nunchaku / SVDQuant)** | `--enable-svdquant --transformer-weights-path <path>` + optional `--quantization-precision int4\|nvfp4`, `--quantization-rank 32` | W4A4-style quantization via [Nunchaku](https://nunchaku.tech). Reduces DiT weight memory by ~4x. Precision/rank can be auto-inferred from weight filename or set explicitly. | ~1.5–2x compute speedup | Lossy quantization; quality depends on rank and precision. Requires pre-quantized weights. Ampere (SM8x) or SM12x only (no Hopper SM90). Higher rank = better quality but more memory. |
|
||||
| **GGUF Transformer** | `--transformer-weights-path <file.gguf\|owner/repo:QUANT>` | Loads a community-quantized DiT from one `.gguf`; other components stay on the base model. **Shrinks the checkpoint, not the peak VRAM** — offload already bounds peak, so reach for this when the *download* or the host RAM offload pins is the problem (MiniMax-H3 17.5 vs 61.7 GiB), not when VRAM is. For a 24 GB card `kitchen_int8` is the faster option if you can afford the full BF16 checkpoint on disk. | None; expect a small slowdown from per-step dequantization | Lossy (4-bit families ~0.997 cosine vs BF16). CUDA only, `--tp-size 1`, no FSDP, no LoRA, no `--quantization`, no `--enable-svdquant`, and mutually exclusive with the H3 AdaLN cache/online flags — each rejected at startup. Validated on MiniMax-H3 `fl2va` Q4_K_M, 1 GPU. |
|
||||
| **GGUF Transformer** | `--transformer-weights-path <file.gguf\|owner/repo:QUANT>` | Loads a community-quantized DiT from one `.gguf`; other components stay on the base model. **Shrinks the checkpoint, not the peak VRAM** — offload already bounds peak, so reach for this when the *download* or the host RAM offload pins is the problem (MiniMax-H3 17.5 vs 61.7 GiB), not when VRAM is. For a 24 GB card `convrot_int8` is the faster option if you can afford the full BF16 checkpoint on disk. | None; expect a small slowdown from per-step dequantization | Lossy (4-bit families ~0.997 cosine vs BF16). CUDA only, `--tp-size 1`, no FSDP, no LoRA, no `--quantization`, no `--enable-svdquant`, and mutually exclusive with the H3 AdaLN cache/online flags — each rejected at startup. Validated on MiniMax-H3 `fl2va` Q4_K_M, 1 GPU. |
|
||||
| **Pre-quantized Transformer Override** | `--transformer-path <dir-or-repo>` / `--transformer-weights-path <path>` | Load a quantized transformer component or raw transformer weights. For converted ModelOpt FP8/NVFP4 directories, prefer `--transformer-path`; use `--transformer-weights-path` for weight-only artifacts the model loader expects. | ~1.3–1.5x compute (dtype dependent) | Requires a validated quantized transformer override, such as one produced by the ModelOpt helper tools. Quality is usually slightly worse than BF16 and depends on the format, fallback layers, and calibration scope. |
|
||||
| **Component Precision Override** | `--dit-precision fp16`, `--vae-precision fp16\|bf16` | On-the-fly dtype conversion for individual components. E.g. convert a BF16 model to FP16 at load time, or run VAE in BF16 instead of FP32. | Reduces memory; FP16 can be faster on some GPUs | May affect numerical stability. VAE is FP32 by default for accuracy; lowering it is lossy. DiT defaults to BF16. |
|
||||
| **Fewer Inference Steps** | `--num-inference-steps N` (sampling param) | Reduces the number of denoising steps. Fewer steps = faster. | Linear speedup | Quality degrades with too few steps. Model-dependent optimal range. |
|
||||
|
||||
@@ -47,6 +47,7 @@ if TYPE_CHECKING:
|
||||
SGLANG_DIFFUSION_DISABLE_AUTO_RESIDENCY: bool = False
|
||||
SGLANG_DIFFUSION_MINIMAX_H3_ADALN_GPU_PLANS: int = 64
|
||||
SGLANG_DIFFUSION_MINIMAX_H3_ADALN_FP32: bool = False
|
||||
SGLANG_DIFFUSION_CONVROT_INT8_BACKEND: str = "auto"
|
||||
SGLANG_DIFFUSION_MINIMAX_H3_PDD_HEADS: str | None = None
|
||||
SGLANG_DIFFUSION_FLUX3_NATTEN_BACKEND: str | None = None
|
||||
SGLANG_DIFFUSION_CFG_GATE_STEP: float = 1.0
|
||||
@@ -329,6 +330,13 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"SGLANG_DIFFUSION_CFG_GATE_STEP": _lazy_float(
|
||||
"SGLANG_DIFFUSION_CFG_GATE_STEP", 1.0
|
||||
),
|
||||
# Kernel backend for convrot_int8 (online or serialized ConvRot INT8):
|
||||
# "auto" prefers SGLang's JIT-compiled convrot_int8 ops where they build
|
||||
# and run (CC 9.0, 10.0, 12.0, 12.1 with nvcc) and falls back to
|
||||
# comfy_kitchen; "jit" or "comfy_kitchen" forces one backend.
|
||||
"SGLANG_DIFFUSION_CONVROT_INT8_BACKEND": _lazy_str(
|
||||
"SGLANG_DIFFUSION_CONVROT_INT8_BACKEND", "auto"
|
||||
),
|
||||
# Path to a Parallel Decoding Distillation head stack (one fused output head
|
||||
# per denoise step, produced by fuse_minimax_h3_pdd_heads.py). Set only when serving a
|
||||
# PDD-distilled checkpoint; an ordinary run leaves the projection alone.
|
||||
|
||||
@@ -11,10 +11,13 @@ from sglang.multimodal_gen.runtime.layers.quantization.bitsandbytes import (
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.configs.base_config import (
|
||||
QuantizationConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.configs.kitchen_int8_config import (
|
||||
KitchenInt8Config,
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config import (
|
||||
ConvRotInt8Config,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.fp8 import Fp8Config
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.method_names import (
|
||||
canonical_quantization_method,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.modelopt_fp8 import (
|
||||
ModelOptFp8Config as ModelOptFp8DiffusionConfig,
|
||||
)
|
||||
@@ -40,6 +43,8 @@ QuantizationMethods = Literal[
|
||||
"mxfp8",
|
||||
"mxfp4",
|
||||
"mxfp4_npu",
|
||||
"convrot_int8",
|
||||
# deprecated alias of convrot_int8
|
||||
"kitchen_int8",
|
||||
]
|
||||
|
||||
@@ -57,7 +62,7 @@ _CUSTOMIZED_METHOD_TO_QUANT_CONFIG = {
|
||||
"mxfp4": Mxfp4Config,
|
||||
"mxfp8": MXFP8Config,
|
||||
"mxfp4_npu": NPUMXFP4Config,
|
||||
"kitchen_int8": KitchenInt8Config,
|
||||
"convrot_int8": ConvRotInt8Config,
|
||||
}
|
||||
|
||||
|
||||
@@ -90,6 +95,7 @@ def register_quantization_config(quantization: str):
|
||||
|
||||
|
||||
def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
|
||||
quantization = canonical_quantization_method(quantization)
|
||||
if quantization not in QUANTIZATION_METHODS:
|
||||
raise ValueError(f"Invalid quantization method: {quantization}")
|
||||
|
||||
|
||||
+377
@@ -0,0 +1,377 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Config for convrot_int8: online or serialized ConvRot INT8 on the JIT ops or comfy_kitchen.
|
||||
|
||||
Both backends implement the same quantization (arXiv:2512.03673): group-wise
|
||||
regular Hadamard rotation, per-row dynamic INT8 activations and per-output-
|
||||
channel INT8 weights, with interchangeable weight and scale tensors. ``auto``
|
||||
picks SGLang's JIT-compiled fused ``convrot_int8`` ops where they build and run
|
||||
(CC 9.0, 10.0, 12.0 and 12.1 with a CUDA toolchain; group size 64, 128, 256 or
|
||||
512; output width a multiple of 8) and ``comfy_kitchen.int8_linear`` otherwise
|
||||
(Turing and later; group size 16, 64 or 256; the load fails with an install hint
|
||||
when the package is missing). Only a layer whose output width the JIT epilogue
|
||||
cannot store falls to comfy_kitchen or, without it, stays BF16.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.gemm.convrot_int8 import GROUP_SIZES as JIT_GROUP_SIZES
|
||||
from sglang.multimodal_gen import envs
|
||||
from sglang.multimodal_gen.runtime.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
LinearBase,
|
||||
UnquantizedLinearMethod,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.comfy_int8 import (
|
||||
ComfyInt8EmbeddingMethod,
|
||||
is_comfy_int8_embedding,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.configs.base_config import (
|
||||
QuantizationConfig,
|
||||
QuantizeMethodBase,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
|
||||
from sglang.srt.layers.quantization.utils import is_layer_skipped
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
COMFY_KITCHEN = "comfy_kitchen"
|
||||
JIT = "jit"
|
||||
BACKENDS = ("auto", COMFY_KITCHEN, JIT)
|
||||
# Group widths each backend's kernels are instantiated for.
|
||||
BACKEND_GROUP_SIZES = {COMFY_KITCHEN: (16, 64, 256), JIT: JIT_GROUP_SIZES}
|
||||
_SUPPORTED_GROUP_SIZES = tuple(
|
||||
sorted({*BACKEND_GROUP_SIZES[COMFY_KITCHEN], *JIT_GROUP_SIZES})
|
||||
)
|
||||
|
||||
|
||||
def _jit_available() -> bool:
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.convrot_int8_jit import (
|
||||
jit_convrot_unavailable_reason,
|
||||
)
|
||||
|
||||
reason = jit_convrot_unavailable_reason()
|
||||
if reason is not None:
|
||||
_log_jit_fallback(reason)
|
||||
return reason is None
|
||||
|
||||
|
||||
@functools.cache
|
||||
def _log_jit_fallback(reason: str) -> None:
|
||||
# Once per process: the refused-GPU reason (for example CC 10.3) or a failed
|
||||
# build would otherwise only surface with an explicit jit backend.
|
||||
logger.info(
|
||||
"convrot_int8: JIT backend unavailable (%s); auto uses comfy_kitchen",
|
||||
reason,
|
||||
)
|
||||
|
||||
|
||||
def _comfy_kitchen_available() -> bool:
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.convrot_int8_comfy_kitchen import (
|
||||
comfy_kitchen_available,
|
||||
)
|
||||
|
||||
return comfy_kitchen_available()
|
||||
|
||||
|
||||
class ConvRotInt8Config(QuantizationConfig):
|
||||
"""Dispatch online or serialized Comfy ConvRot layers to a kernel backend."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
group_size: int = 256,
|
||||
ignored_layers: list[str] | None = None,
|
||||
packed_modules_mapping: dict[str, list[str]] | None = None,
|
||||
layer_markers: dict[str, dict[str, Any]] | None = None,
|
||||
backend: str | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
if group_size not in _SUPPORTED_GROUP_SIZES:
|
||||
raise ValueError(
|
||||
f"convrot_int8 group_size must be one of {_SUPPORTED_GROUP_SIZES}, "
|
||||
f"got {group_size}"
|
||||
)
|
||||
if backend is None:
|
||||
backend = envs.SGLANG_DIFFUSION_CONVROT_INT8_BACKEND
|
||||
if backend not in BACKENDS:
|
||||
raise ValueError(
|
||||
f"convrot_int8 backend must be one of {BACKENDS}, got {backend!r}"
|
||||
)
|
||||
if backend != "auto" and group_size not in BACKEND_GROUP_SIZES[backend]:
|
||||
raise ValueError(
|
||||
f"convrot_int8 backend {backend} has kernels for group sizes "
|
||||
f"{BACKEND_GROUP_SIZES[backend]}, got {group_size}"
|
||||
)
|
||||
self.group_size = group_size
|
||||
self.backend = backend
|
||||
self.ignored_layers = ignored_layers or []
|
||||
self.packed_modules_mapping = packed_modules_mapping or {}
|
||||
self.layer_markers = layer_markers
|
||||
self.is_checkpoint_int8_serialized = layer_markers is not None
|
||||
self.checkpoint_uses_native_qkv_layout = self.is_checkpoint_int8_serialized
|
||||
self._serialized_group_sizes: dict[str, int] = {}
|
||||
if layer_markers is not None:
|
||||
for prefix, marker in layer_markers.items():
|
||||
if is_comfy_int8_embedding(marker):
|
||||
continue
|
||||
if marker.get("format") != "int8_tensorwise":
|
||||
raise ValueError(
|
||||
f"Unsupported Comfy INT8 format for {prefix!r}: "
|
||||
f"{marker.get('format')!r}"
|
||||
)
|
||||
if marker.get("convrot") is not True:
|
||||
raise ValueError(
|
||||
f"Serialized convrot_int8 layer {prefix!r} must set "
|
||||
"convrot=true"
|
||||
)
|
||||
marker_group_size = marker.get("convrot_groupsize")
|
||||
if marker_group_size not in BACKEND_GROUP_SIZES[COMFY_KITCHEN]:
|
||||
raise ValueError(
|
||||
f"Serialized convrot_int8 layer {prefix!r} must declare "
|
||||
f"convrot_groupsize in {BACKEND_GROUP_SIZES[COMFY_KITCHEN]}, "
|
||||
f"got {marker_group_size!r}"
|
||||
)
|
||||
self._serialized_group_sizes[prefix] = marker_group_size
|
||||
elif backend == "auto" and self.resolve_backend() is None:
|
||||
raise ValueError(
|
||||
f"convrot_int8: no backend on this machine serves group size "
|
||||
f"{group_size} (the JIT backend needs CC 9.0/10.0/12.0/12.1, a CUDA "
|
||||
f"toolchain and one of {BACKEND_GROUP_SIZES[JIT]}; comfy_kitchen serves "
|
||||
f"{BACKEND_GROUP_SIZES[COMFY_KITCHEN]})"
|
||||
)
|
||||
# Logged per backend at the end of loading: a silent fallback to BF16
|
||||
# or to the slower backend looks exactly like a slow kernel.
|
||||
self.selected: list[str] = []
|
||||
# Embeddings count as selected for the loader's marker check but never
|
||||
# report back through note_quantized / note_loaded.
|
||||
self.selected_embeddings: list[str] = []
|
||||
self.selected_by_backend: dict[str, list[str]] = {
|
||||
COMFY_KITCHEN: [],
|
||||
JIT: [],
|
||||
}
|
||||
self.skipped: list[str] = []
|
||||
self._processed = 0
|
||||
self._quantized_bytes = 0
|
||||
|
||||
@classmethod
|
||||
def get_name(cls) -> str:
|
||||
return "convrot_int8"
|
||||
|
||||
@classmethod
|
||||
def get_supported_act_dtypes(cls) -> list[torch.dtype]:
|
||||
return [torch.bfloat16, torch.float16]
|
||||
|
||||
@classmethod
|
||||
def get_min_capability(cls) -> int:
|
||||
# INT8 tensor cores land on Turing (comfy_kitchen); the JIT backend
|
||||
# checks its own table when it is selected.
|
||||
return 75
|
||||
|
||||
@classmethod
|
||||
def get_config_filenames(cls) -> list[str]:
|
||||
return []
|
||||
|
||||
@classmethod
|
||||
def from_config(cls, config: dict[str, Any]) -> ConvRotInt8Config:
|
||||
return cls(
|
||||
group_size=cls.get_from_keys_or(config, ["group_size"], 256),
|
||||
ignored_layers=cls.get_from_keys_or(config, ["ignored_layers"], None),
|
||||
backend=cls.get_from_keys_or(config, ["backend"], None),
|
||||
)
|
||||
|
||||
def resolve_backend(self, group_size: int | None = None) -> str | None:
|
||||
"""Backend serving ``group_size`` (default: the online group size), or
|
||||
None when the requested backend has no kernel for it. ``auto`` prefers
|
||||
the JIT ops where they build and run on this GPU; an explicit backend
|
||||
is never substituted."""
|
||||
group_size = self.group_size if group_size is None else group_size
|
||||
if self.backend == "auto":
|
||||
if group_size in BACKEND_GROUP_SIZES[JIT] and _jit_available():
|
||||
return JIT
|
||||
if group_size in BACKEND_GROUP_SIZES[COMFY_KITCHEN]:
|
||||
return COMFY_KITCHEN
|
||||
return None
|
||||
return self.backend if group_size in BACKEND_GROUP_SIZES[self.backend] else None
|
||||
|
||||
def require_comfy_kitchen(self, reason: str) -> None:
|
||||
"""Serve every layer with comfy_kitchen; an explicit jit request is refused."""
|
||||
if self.backend == JIT:
|
||||
raise ValueError(
|
||||
f"convrot_int8 backend jit {reason}; use backend "
|
||||
"comfy_kitchen or TP and/or sequence parallelism instead"
|
||||
)
|
||||
if self.group_size not in BACKEND_GROUP_SIZES[COMFY_KITCHEN]:
|
||||
raise ValueError(
|
||||
f"convrot_int8 backend comfy_kitchen has no kernel for group size "
|
||||
f"{self.group_size}, and the jit backend {reason}"
|
||||
)
|
||||
if self.backend == "auto":
|
||||
logger.info(
|
||||
"convrot_int8: using the comfy_kitchen backend because %s", reason
|
||||
)
|
||||
self.backend = COMFY_KITCHEN
|
||||
|
||||
def _backend_for(self, *, group_size: int, out_size: int) -> str | None:
|
||||
backend = self.resolve_backend(group_size)
|
||||
if backend == JIT and out_size % 8:
|
||||
# The JIT GEMM epilogue stores 8 BF16 outputs at a time.
|
||||
if (
|
||||
self.backend == "auto"
|
||||
and group_size in BACKEND_GROUP_SIZES[COMFY_KITCHEN]
|
||||
and _comfy_kitchen_available()
|
||||
):
|
||||
return COMFY_KITCHEN
|
||||
return None
|
||||
return backend
|
||||
|
||||
def _no_kernel_reason(self, *, group_size: int, out_size: int) -> str:
|
||||
comfy = (
|
||||
"installed"
|
||||
if _comfy_kitchen_available()
|
||||
else "not installed (pip install comfy-kitchen)"
|
||||
)
|
||||
return (
|
||||
f"ConvRot group {group_size}, {out_size} outputs; backend {self.backend!r}; "
|
||||
f"jit serves group sizes {BACKEND_GROUP_SIZES[JIT]} with an "
|
||||
f"output width that is a multiple of 8, comfy_kitchen serves group sizes "
|
||||
f"{BACKEND_GROUP_SIZES[COMFY_KITCHEN]} and is {comfy}"
|
||||
)
|
||||
|
||||
def _select(
|
||||
self, *, prefix: str, backend: str, group_size: int, serialized: bool
|
||||
) -> QuantizeMethodBase:
|
||||
self.selected.append(prefix)
|
||||
self.selected_by_backend[backend].append(prefix)
|
||||
if backend == JIT:
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.convrot_int8_jit import (
|
||||
ConvRotInt8JitLinearMethod,
|
||||
)
|
||||
|
||||
return ConvRotInt8JitLinearMethod(
|
||||
self, group_size=group_size, is_checkpoint_serialized=serialized
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.convrot_int8_comfy_kitchen import (
|
||||
ConvRotInt8ComfyKitchenLinearMethod,
|
||||
)
|
||||
|
||||
return ConvRotInt8ComfyKitchenLinearMethod(
|
||||
self, group_size=group_size, is_checkpoint_serialized=serialized
|
||||
)
|
||||
|
||||
def get_quant_method(
|
||||
self, layer: torch.nn.Module, prefix: str
|
||||
) -> QuantizeMethodBase | None:
|
||||
if isinstance(layer, VocabParallelEmbedding) and self.quantizes_embedding(
|
||||
prefix
|
||||
):
|
||||
self.selected.append(prefix)
|
||||
self.selected_embeddings.append(prefix)
|
||||
return ComfyInt8EmbeddingMethod(
|
||||
tensorwise=bool(self.layer_markers[prefix].get("_is_tensorwise_scalar"))
|
||||
)
|
||||
if not isinstance(layer, LinearBase):
|
||||
return None
|
||||
# A column-parallel layer shards its output; every rank sees the same
|
||||
# per-shard width, so the decision below is rank-consistent.
|
||||
out_size = (
|
||||
layer.output_size_per_partition
|
||||
if isinstance(layer, ColumnParallelLinear)
|
||||
else layer.output_size
|
||||
)
|
||||
if self.layer_markers is not None:
|
||||
marker_group_size = self._serialized_group_sizes.get(prefix)
|
||||
if marker_group_size is None:
|
||||
return UnquantizedLinearMethod()
|
||||
if layer.input_size % marker_group_size:
|
||||
raise ValueError(
|
||||
f"Serialized convrot_int8 layer {prefix!r} has input size "
|
||||
f"{layer.input_size}, which is not divisible by its "
|
||||
f"ConvRot group size {marker_group_size}"
|
||||
)
|
||||
backend = self._backend_for(group_size=marker_group_size, out_size=out_size)
|
||||
if backend is None:
|
||||
raise ValueError(
|
||||
f"Serialized convrot_int8 layer {prefix!r} has no kernel: "
|
||||
+ self._no_kernel_reason(
|
||||
group_size=marker_group_size, out_size=out_size
|
||||
)
|
||||
)
|
||||
return self._select(
|
||||
prefix=prefix,
|
||||
backend=backend,
|
||||
group_size=marker_group_size,
|
||||
serialized=True,
|
||||
)
|
||||
if is_layer_skipped(
|
||||
prefix, self.ignored_layers, fused_mapping=self.packed_modules_mapping
|
||||
):
|
||||
self.skipped.append(prefix)
|
||||
return UnquantizedLinearMethod()
|
||||
# The rotation partitions the input dim into fixed-size groups, so a
|
||||
# layer whose input does not divide evenly simply stays in BF16 rather
|
||||
# than failing the whole model. H3's adaln projections (in=2688) are
|
||||
# the case this exists for, and they cost 0.2% of a step anyway.
|
||||
if layer.input_size % self.group_size:
|
||||
self.skipped.append(f"{prefix}(in={layer.input_size})")
|
||||
return UnquantizedLinearMethod()
|
||||
backend = self._backend_for(group_size=self.group_size, out_size=out_size)
|
||||
if backend is None:
|
||||
self.skipped.append(f"{prefix}(out={out_size})")
|
||||
return UnquantizedLinearMethod()
|
||||
return self._select(
|
||||
prefix=prefix, backend=backend, group_size=self.group_size, serialized=False
|
||||
)
|
||||
|
||||
def note_quantized(self, saved_bytes: int) -> None:
|
||||
self._processed += 1
|
||||
self._quantized_bytes += saved_bytes
|
||||
if self._processed == len(self.selected) - len(self.selected_embeddings):
|
||||
logger.info(
|
||||
"convrot_int8: quantized %d linear layers (%.2f GiB of BF16 weights "
|
||||
"-> %.2f GiB INT8; jit %d, comfy_kitchen %d), left %d in BF16",
|
||||
self._processed,
|
||||
self._quantized_bytes / 1024**3,
|
||||
self._quantized_bytes / 2 / 1024**3,
|
||||
len(self.selected_by_backend[JIT]),
|
||||
len(self.selected_by_backend[COMFY_KITCHEN]),
|
||||
len(self.skipped),
|
||||
)
|
||||
logger.debug("convrot_int8: layers left in BF16: %s", self.skipped)
|
||||
|
||||
def note_loaded(self) -> None:
|
||||
"""A serialized layer's INT8 weights are in place; logs the backend split once."""
|
||||
self._processed += 1
|
||||
if self._processed == len(self.selected) - len(self.selected_embeddings):
|
||||
logger.info(
|
||||
"convrot_int8: loaded %d serialized INT8 linear layers "
|
||||
"(jit %d, comfy_kitchen %d)",
|
||||
self._processed,
|
||||
len(self.selected_by_backend[JIT]),
|
||||
len(self.selected_by_backend[COMFY_KITCHEN]),
|
||||
)
|
||||
|
||||
def get_scaled_act_names(self) -> list[str]:
|
||||
return []
|
||||
|
||||
def quantizes_embedding(self, prefix: str) -> bool:
|
||||
return self.layer_markers is not None and is_comfy_int8_embedding(
|
||||
self.layer_markers.get(prefix)
|
||||
)
|
||||
|
||||
def supports_input_partition(
|
||||
self, prefix: str, input_size_per_partition: int
|
||||
) -> bool:
|
||||
group_size = self.group_size
|
||||
if self.layer_markers is not None:
|
||||
marker_group_size = self._serialized_group_sizes.get(prefix)
|
||||
if marker_group_size is None:
|
||||
return True
|
||||
group_size = marker_group_size
|
||||
return input_size_per_partition % group_size == 0
|
||||
-189
@@ -1,189 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Config for online or serialized INT8 ConvRot via comfy_kitchen."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.runtime.layers.linear import UnquantizedLinearMethod
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.comfy_int8 import (
|
||||
ComfyInt8EmbeddingMethod,
|
||||
is_comfy_int8_embedding,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.configs.base_config import (
|
||||
QuantizationConfig,
|
||||
QuantizeMethodBase,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
|
||||
from sglang.srt.layers.quantization.utils import is_layer_skipped
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_SUPPORTED_GROUP_SIZES = (16, 64, 256)
|
||||
|
||||
|
||||
class KitchenInt8Config(QuantizationConfig):
|
||||
"""Dispatch online quantization or serialized Comfy ConvRot layers."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
group_size: int = 256,
|
||||
ignored_layers: list[str] | None = None,
|
||||
packed_modules_mapping: dict[str, list[str]] | None = None,
|
||||
layer_markers: dict[str, dict[str, Any]] | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
if group_size not in _SUPPORTED_GROUP_SIZES:
|
||||
raise ValueError(
|
||||
f"kitchen_int8 group_size must be one of {_SUPPORTED_GROUP_SIZES}, "
|
||||
f"got {group_size}"
|
||||
)
|
||||
self.group_size = group_size
|
||||
self.ignored_layers = ignored_layers or []
|
||||
self.packed_modules_mapping = packed_modules_mapping or {}
|
||||
self.layer_markers = layer_markers
|
||||
self.is_checkpoint_int8_serialized = layer_markers is not None
|
||||
self.checkpoint_uses_native_qkv_layout = self.is_checkpoint_int8_serialized
|
||||
self._serialized_group_sizes: dict[str, int] = {}
|
||||
if layer_markers is not None:
|
||||
for prefix, marker in layer_markers.items():
|
||||
if is_comfy_int8_embedding(marker):
|
||||
continue
|
||||
if marker.get("format") != "int8_tensorwise":
|
||||
raise ValueError(
|
||||
f"Unsupported Comfy INT8 format for {prefix!r}: "
|
||||
f"{marker.get('format')!r}"
|
||||
)
|
||||
if marker.get("convrot") is not True:
|
||||
raise ValueError(
|
||||
f"Serialized kitchen_int8 layer {prefix!r} must set "
|
||||
"convrot=true"
|
||||
)
|
||||
marker_group_size = marker.get("convrot_groupsize")
|
||||
if marker_group_size not in _SUPPORTED_GROUP_SIZES:
|
||||
raise ValueError(
|
||||
f"Serialized kitchen_int8 layer {prefix!r} must declare "
|
||||
f"convrot_groupsize in {_SUPPORTED_GROUP_SIZES}, got "
|
||||
f"{marker_group_size!r}"
|
||||
)
|
||||
self._serialized_group_sizes[prefix] = marker_group_size
|
||||
# Which layers actually got quantized is worth stating plainly in the
|
||||
# log: a silent fallback to BF16 looks exactly like a slow kernel.
|
||||
self.selected: list[str] = []
|
||||
self.skipped: list[str] = []
|
||||
self._processed = 0
|
||||
self._quantized_bytes = 0
|
||||
|
||||
@classmethod
|
||||
def get_name(cls) -> str:
|
||||
return "kitchen_int8"
|
||||
|
||||
@classmethod
|
||||
def get_supported_act_dtypes(cls) -> list[torch.dtype]:
|
||||
return [torch.bfloat16, torch.float16]
|
||||
|
||||
@classmethod
|
||||
def get_min_capability(cls) -> int:
|
||||
# INT8 tensor cores land on Turing.
|
||||
return 75
|
||||
|
||||
@classmethod
|
||||
def get_config_filenames(cls) -> list[str]:
|
||||
return []
|
||||
|
||||
@classmethod
|
||||
def from_config(cls, config: dict[str, Any]) -> KitchenInt8Config:
|
||||
return cls(
|
||||
group_size=cls.get_from_keys_or(config, ["group_size"], 256),
|
||||
ignored_layers=cls.get_from_keys_or(config, ["ignored_layers"], None),
|
||||
)
|
||||
|
||||
def get_quant_method(
|
||||
self, layer: torch.nn.Module, prefix: str
|
||||
) -> QuantizeMethodBase | None:
|
||||
from sglang.multimodal_gen.runtime.layers.linear import LinearBase
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.kitchen_int8 import (
|
||||
KitchenInt8LinearMethod,
|
||||
)
|
||||
|
||||
if isinstance(layer, VocabParallelEmbedding) and self.quantizes_embedding(
|
||||
prefix
|
||||
):
|
||||
self.selected.append(prefix)
|
||||
return ComfyInt8EmbeddingMethod(
|
||||
tensorwise=bool(self.layer_markers[prefix].get("_is_tensorwise_scalar"))
|
||||
)
|
||||
if not isinstance(layer, LinearBase):
|
||||
return None
|
||||
if self.layer_markers is not None:
|
||||
marker_group_size = self._serialized_group_sizes.get(prefix)
|
||||
if marker_group_size is None:
|
||||
return UnquantizedLinearMethod()
|
||||
if layer.input_size % marker_group_size:
|
||||
raise ValueError(
|
||||
f"Serialized kitchen_int8 layer {prefix!r} has input size "
|
||||
f"{layer.input_size}, which is not divisible by its "
|
||||
f"ConvRot group size {marker_group_size}"
|
||||
)
|
||||
self.selected.append(prefix)
|
||||
return KitchenInt8LinearMethod(
|
||||
self,
|
||||
group_size=marker_group_size,
|
||||
is_checkpoint_serialized=True,
|
||||
)
|
||||
if is_layer_skipped(
|
||||
prefix, self.ignored_layers, fused_mapping=self.packed_modules_mapping
|
||||
):
|
||||
self.skipped.append(prefix)
|
||||
return UnquantizedLinearMethod()
|
||||
# The rotation partitions the input dim into fixed-size groups, so a
|
||||
# layer whose input does not divide evenly simply stays in BF16 rather
|
||||
# than failing the whole model. H3's adaln projections (in=2688) are
|
||||
# the case this exists for, and they cost 0.2% of a step anyway.
|
||||
if layer.input_size % self.group_size:
|
||||
self.skipped.append(f"{prefix}(in={layer.input_size})")
|
||||
return UnquantizedLinearMethod()
|
||||
self.selected.append(prefix)
|
||||
return KitchenInt8LinearMethod(
|
||||
self,
|
||||
group_size=self.group_size,
|
||||
is_checkpoint_serialized=False,
|
||||
)
|
||||
|
||||
def note_quantized(self, saved_bytes: int) -> None:
|
||||
self._processed += 1
|
||||
self._quantized_bytes += saved_bytes
|
||||
if self._processed == len(self.selected):
|
||||
logger.info(
|
||||
"kitchen_int8: quantized %d linear layers (%.2f GiB of BF16 weights "
|
||||
"-> %.2f GiB INT8), left %d in BF16",
|
||||
self._processed,
|
||||
self._quantized_bytes / 1024**3,
|
||||
self._quantized_bytes / 2 / 1024**3,
|
||||
len(self.skipped),
|
||||
)
|
||||
logger.debug("kitchen_int8: layers left in BF16: %s", self.skipped)
|
||||
|
||||
def get_scaled_act_names(self) -> list[str]:
|
||||
return []
|
||||
|
||||
def supports_input_partition(
|
||||
self, prefix: str, input_size_per_partition: int
|
||||
) -> bool:
|
||||
group_size = self.group_size
|
||||
if self.layer_markers is not None:
|
||||
marker_group_size = self._serialized_group_sizes.get(prefix)
|
||||
if marker_group_size is None:
|
||||
return True
|
||||
group_size = marker_group_size
|
||||
return input_size_per_partition % group_size == 0
|
||||
|
||||
def quantizes_embedding(self, prefix: str) -> bool:
|
||||
return self.layer_markers is not None and is_comfy_int8_embedding(
|
||||
self.layer_markers.get(prefix)
|
||||
)
|
||||
+3
-3
@@ -15,8 +15,8 @@ from sglang.multimodal_gen.runtime.layers.quantization.configs.base_config impor
|
||||
QuantizationConfig,
|
||||
QuantizeMethodBase,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.configs.kitchen_int8_config import (
|
||||
KitchenInt8Config,
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config import (
|
||||
ConvRotInt8Config,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.kitchen_w4a4 import (
|
||||
KitchenW4A4LinearMethod,
|
||||
@@ -55,7 +55,7 @@ class KitchenW4A4Config(QuantizationConfig):
|
||||
if marker.get("format") == "int8_tensorwise"
|
||||
}
|
||||
self._int8_config = (
|
||||
KitchenInt8Config(layer_markers=int8_markers) if int8_markers else None
|
||||
ConvRotInt8Config(layer_markers=int8_markers) if int8_markers else None
|
||||
)
|
||||
|
||||
for prefix, marker in layer_markers.items():
|
||||
|
||||
+26
-11
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""INT8 weight-only-storage linear backed by comfy_kitchen's fused ConvRot kernel.
|
||||
"""convrot_int8's comfy_kitchen backend: INT8 linear on comfy_kitchen's fused ConvRot kernel.
|
||||
|
||||
On Ada (RTX 4090) INT8 is only worth doing with the right kernel: on MiniMax H3
|
||||
shapes `torch._int_mm` measures 0.46-0.90x of BF16 (i.e. slower) and a Triton
|
||||
@@ -15,18 +15,19 @@ Comfy checkpoints instead load their INT8 weights and row scales directly.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
import os
|
||||
|
||||
import torch
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
from sglang.multimodal_gen.runtime.layers.linear import LinearMethodBase
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.configs.kitchen_int8_config import (
|
||||
KitchenInt8Config,
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config import (
|
||||
ConvRotInt8Config,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.utils.weight_attrs import set_weight_attrs
|
||||
|
||||
__all__ = ["KitchenInt8Config", "KitchenInt8LinearMethod"]
|
||||
__all__ = ["ConvRotInt8ComfyKitchenLinearMethod"]
|
||||
|
||||
# comfy_kitchen's dtype codes for the fused op's output.
|
||||
_OUT_DTYPE_CODE = {torch.float32: 0, torch.float16: 1, torch.bfloat16: 2}
|
||||
@@ -55,12 +56,22 @@ def _row_split(rows: int, out_features: int) -> int | None:
|
||||
return _MAX_ROWS_PER_CALL
|
||||
|
||||
|
||||
@functools.cache
|
||||
def comfy_kitchen_available() -> bool:
|
||||
"""Whether comfy_kitchen is importable and registers int8_linear."""
|
||||
try:
|
||||
import comfy_kitchen # noqa: F401
|
||||
except ImportError:
|
||||
return False
|
||||
return hasattr(torch.ops.comfy_kitchen, "int8_linear")
|
||||
|
||||
|
||||
def _load_comfy_kitchen():
|
||||
try:
|
||||
import comfy_kitchen # noqa: F401
|
||||
except ImportError as exc: # pragma: no cover - depends on optional dep
|
||||
raise ImportError(
|
||||
"kitchen_int8 quantization requires the `comfy-kitchen` package "
|
||||
"convrot_int8 quantization requires the `comfy-kitchen` package "
|
||||
"(pip install comfy-kitchen). It is a self-contained abi3 extension "
|
||||
"and does not link against libtorch, so any torch version works."
|
||||
) from exc
|
||||
@@ -71,12 +82,12 @@ def _load_comfy_kitchen():
|
||||
)
|
||||
|
||||
|
||||
class KitchenInt8LinearMethod(LinearMethodBase):
|
||||
class ConvRotInt8ComfyKitchenLinearMethod(LinearMethodBase):
|
||||
"""Loads or creates ConvRot INT8 weights and runs the fused kernel."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
quant_config: KitchenInt8Config,
|
||||
quant_config: ConvRotInt8Config,
|
||||
*,
|
||||
group_size: int,
|
||||
is_checkpoint_serialized: bool,
|
||||
@@ -101,9 +112,10 @@ class KitchenInt8LinearMethod(LinearMethodBase):
|
||||
# dimension the rotation groups over.
|
||||
if input_size_per_partition % self.group_size:
|
||||
raise ValueError(
|
||||
f"kitchen_int8 needs input_size_per_partition "
|
||||
f"convrot_int8 needs input_size_per_partition "
|
||||
f"({input_size_per_partition}) divisible by group_size "
|
||||
f"{self.group_size}"
|
||||
f"{self.group_size}; leave the layer in BF16 with "
|
||||
"--quantization-ignored-layers"
|
||||
)
|
||||
|
||||
# The online path initially matches UnquantizedLinearMethod so the
|
||||
@@ -135,7 +147,10 @@ class KitchenInt8LinearMethod(LinearMethodBase):
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
weight = layer.weight.data
|
||||
if self.is_checkpoint_serialized or weight.dtype == torch.int8:
|
||||
if self.is_checkpoint_serialized:
|
||||
self.quant_config.note_loaded()
|
||||
return
|
||||
if weight.dtype == torch.int8:
|
||||
return
|
||||
|
||||
from comfy_kitchen.tensor.int8 import TensorWiseINT8Layout
|
||||
@@ -172,7 +187,7 @@ class KitchenInt8LinearMethod(LinearMethodBase):
|
||||
out_code = _OUT_DTYPE_CODE.get(x.dtype)
|
||||
if out_code is None:
|
||||
raise ValueError(
|
||||
f"kitchen_int8 does not support activation dtype {x.dtype}"
|
||||
f"convrot_int8 does not support activation dtype {x.dtype}"
|
||||
)
|
||||
|
||||
# The kernel takes 2D activations; callers may pass [..., K].
|
||||
@@ -0,0 +1,384 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""convrot_int8's JIT backend: INT8 W8A8 linear on the fused ConvRot ops.
|
||||
|
||||
One op takes a BF16 activation and performs the group-wise Hadamard rotation,
|
||||
dynamic per-row INT8 quantization, INT8 GEMM, dequantization and bias add
|
||||
without materializing the intermediates. Weights receive the same rotation and
|
||||
per-output-channel quantization once, after loading a stock BF16 checkpoint; a
|
||||
serialized Comfy ConvRot INT8 checkpoint loads its INT8 weights and row scales
|
||||
directly, the same tensors comfy_kitchen consumes.
|
||||
|
||||
The ops live in ``sglang.kernels.ops.gemm.convrot_int8`` and are compiled for
|
||||
the local GPU on first use (nvcc, about a minute once per machine; cached
|
||||
afterwards). A missing toolchain or a failed build makes this backend
|
||||
unavailable, and ``auto`` then takes comfy_kitchen.
|
||||
|
||||
Two variants of that op are exposed to the model code as helpers below: a
|
||||
shared-input form that rotates and quantizes an activation once for several
|
||||
linears consuming it (a q/k/v trio), and a gelu-input form that applies
|
||||
GELU(tanh) inside the rotate kernel for an FFN down-projection. Both are
|
||||
bitwise identical to the plain op on the equivalent eager input.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
import time
|
||||
from collections.abc import Sequence
|
||||
|
||||
import torch
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
from sglang.kernels.ops.gemm.convrot_int8 import (
|
||||
convrot_int8_fused_linear,
|
||||
convrot_int8_linear_prequant,
|
||||
convrot_int8_local_capability,
|
||||
convrot_int8_supported_capabilities,
|
||||
convrot_rotate_quantize_activation,
|
||||
load_convrot_int8_module,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
LinearBase,
|
||||
LinearMethodBase,
|
||||
ReplicatedLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config import (
|
||||
ConvRotInt8Config,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
|
||||
from sglang.multimodal_gen.runtime.utils.weight_attrs import set_weight_attrs
|
||||
from sglang.srt.environ import envs
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
__all__ = [
|
||||
"ConvRotInt8JitLinearMethod",
|
||||
"REFUSED_CAPABILITY_REASONS",
|
||||
"apply_convrot_int8_gelu_input",
|
||||
"apply_convrot_int8_shared_input",
|
||||
"apply_convrot_int8_shared_input_out",
|
||||
"convrot_int8_fuses_gelu_input",
|
||||
"convrot_int8_shares_input",
|
||||
"jit_convrot_available",
|
||||
"jit_convrot_unavailable_reason",
|
||||
]
|
||||
|
||||
# Parts the ops deliberately leave out, with the reason shown at load time. The
|
||||
# supported table itself lives with the ops (convrot_int8_supported_capabilities);
|
||||
# nothing here duplicates it.
|
||||
REFUSED_CAPABILITY_REASONS: dict[tuple[int, int], str] = {
|
||||
(10, 3): (
|
||||
"Blackwell Ultra cuts INT8 tensor-core throughput to a fraction of its "
|
||||
"BF16 rate, so W8A8 INT8 would be a slowdown there; use FP8 or NVFP4 "
|
||||
"quantization on this GPU"
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _unsupported_capability_reason(capability: tuple[int, int]) -> str:
|
||||
major, minor = capability
|
||||
reason = REFUSED_CAPABILITY_REASONS.get(
|
||||
capability, "the convrot_int8 ops carry no code for it"
|
||||
)
|
||||
supported = ", ".join(
|
||||
f"{a}.{b}" for a, b in sorted(convrot_int8_supported_capabilities())
|
||||
)
|
||||
return f"CC {major}.{minor} is not supported: {reason} (supported: {supported})"
|
||||
|
||||
|
||||
@functools.cache
|
||||
def jit_convrot_unavailable_reason() -> str | None:
|
||||
"""Why the JIT backend cannot run here, or None when it can.
|
||||
|
||||
Builds the ops on the way (once per machine; cached afterwards), so a
|
||||
missing CUDA toolchain or a failed build shows up here as a reason instead
|
||||
of failing the first quantized layer.
|
||||
"""
|
||||
if not torch.cuda.is_available():
|
||||
return "no CUDA device"
|
||||
capability = convrot_int8_local_capability()
|
||||
if capability is None:
|
||||
return "not a CUDA device"
|
||||
if capability not in convrot_int8_supported_capabilities():
|
||||
return _unsupported_capability_reason(capability)
|
||||
logger.info(
|
||||
"convrot_int8: loading the JIT ops for CC %d.%d (the first use on a "
|
||||
"machine compiles them, about a minute)",
|
||||
*capability,
|
||||
)
|
||||
started = time.monotonic()
|
||||
try:
|
||||
load_convrot_int8_module()
|
||||
except Exception as exc: # nvcc / ninja missing, or the build itself failed
|
||||
if envs.SGLANG_CRASH_ON_JIT_COMPILE.get():
|
||||
raise
|
||||
logger.warning("convrot_int8: the JIT build failed: %s", exc)
|
||||
detail = str(exc).strip().splitlines()
|
||||
return (
|
||||
f"the JIT build of the convrot_int8 ops failed "
|
||||
f"({type(exc).__name__}: {detail[0] if detail else 'no message'}; "
|
||||
"the compiler output is in the log above)"
|
||||
)
|
||||
elapsed = time.monotonic() - started
|
||||
# A cache hit loads in well under a second; anything longer was a build.
|
||||
if elapsed > 5:
|
||||
logger.info(
|
||||
"convrot_int8: compiled the JIT ops for CC %d.%d in %.0f s "
|
||||
"(cached for later runs)",
|
||||
*capability,
|
||||
elapsed,
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def jit_convrot_available() -> bool:
|
||||
"""Whether the JIT convrot ops run (and built) on the current GPU."""
|
||||
return jit_convrot_unavailable_reason() is None
|
||||
|
||||
|
||||
def _load_jit_kernel() -> None:
|
||||
reason = jit_convrot_unavailable_reason()
|
||||
if reason is not None:
|
||||
raise RuntimeError(f"convrot_int8 backend jit is unavailable: {reason}")
|
||||
|
||||
|
||||
def _as_rows(x: torch.Tensor) -> torch.Tensor:
|
||||
# The ops take BF16 only. FP16 activations (--dit-precision fp16) are
|
||||
# cast on the way in and back on the way out; the BF16 rounding is an
|
||||
# order of magnitude below the INT8 quantization error.
|
||||
if x.dtype == torch.float16:
|
||||
x = x.to(torch.bfloat16)
|
||||
elif x.dtype != torch.bfloat16:
|
||||
raise ValueError(
|
||||
f"convrot_int8 backend jit does not support activation dtype {x.dtype}"
|
||||
)
|
||||
return x.reshape(-1, x.shape[-1]).contiguous()
|
||||
|
||||
|
||||
def _like_input(out: torch.Tensor, x: torch.Tensor) -> torch.Tensor:
|
||||
out = out.reshape(*x.shape[:-1], out.shape[-1])
|
||||
return out if out.dtype == x.dtype else out.to(x.dtype)
|
||||
|
||||
|
||||
class ConvRotInt8JitLinearMethod(LinearMethodBase):
|
||||
"""Loads or creates ConvRot INT8 weights and runs the JIT fused ops."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
quant_config: ConvRotInt8Config,
|
||||
*,
|
||||
group_size: int,
|
||||
is_checkpoint_serialized: bool,
|
||||
) -> None:
|
||||
self.quant_config = quant_config
|
||||
self.group_size = group_size
|
||||
self.is_checkpoint_serialized = is_checkpoint_serialized
|
||||
_load_jit_kernel()
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
input_size_per_partition: int,
|
||||
output_partition_sizes: list[int],
|
||||
input_size: int,
|
||||
output_size: int,
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
) -> None:
|
||||
# get_quant_method screened the unsharded input size and the per-shard
|
||||
# output width, so these fire only under TP > 1, where a row-parallel
|
||||
# layer splits the very dimension the rotation groups over.
|
||||
if input_size_per_partition % self.group_size:
|
||||
raise ValueError(
|
||||
f"convrot_int8 needs input_size_per_partition "
|
||||
f"({input_size_per_partition}) divisible by group_size "
|
||||
f"{self.group_size}; leave the layer in BF16 with "
|
||||
"--quantization-ignored-layers"
|
||||
)
|
||||
if sum(output_partition_sizes) % 8:
|
||||
raise ValueError(
|
||||
f"convrot_int8 backend jit needs the output size per "
|
||||
f"partition ({sum(output_partition_sizes)}) to be a multiple of 8"
|
||||
)
|
||||
# The online path initially matches UnquantizedLinearMethod so the
|
||||
# source weights load in BF16 before quantization. Serialized weights
|
||||
# allocate their final INT8 storage immediately.
|
||||
weight = Parameter(
|
||||
torch.empty(
|
||||
sum(output_partition_sizes),
|
||||
input_size_per_partition,
|
||||
dtype=(torch.int8 if self.is_checkpoint_serialized else params_dtype),
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0})
|
||||
layer.register_parameter("weight", weight)
|
||||
set_weight_attrs(weight, extra_weight_attrs)
|
||||
if self.is_checkpoint_serialized:
|
||||
# Comfy stores the per-row scale as [N, 1]; the ops read N contiguous
|
||||
# floats and accept that shape unchanged.
|
||||
weight_scale = Parameter(
|
||||
torch.empty(sum(output_partition_sizes), 1, dtype=torch.float32),
|
||||
requires_grad=False,
|
||||
)
|
||||
set_weight_attrs(weight_scale, {"output_dim": 0})
|
||||
set_weight_attrs(weight_scale, extra_weight_attrs)
|
||||
layer.register_parameter("weight_scale", weight_scale)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
weight = layer.weight.data
|
||||
if self.is_checkpoint_serialized:
|
||||
self.quant_config.note_loaded()
|
||||
elif weight.dtype != torch.int8:
|
||||
# Quantization runs on CUDA; a layer staged on CPU is round-tripped
|
||||
# on its own, so an offloaded model never holds more than one BF16
|
||||
# layer on the GPU during load.
|
||||
home = weight.device
|
||||
weight_q, weight_scale = convrot_rotate_quantize_activation(
|
||||
weight.to("cuda", non_blocking=True).to(torch.bfloat16),
|
||||
self.group_size,
|
||||
)
|
||||
layer.weight = Parameter(weight_q.to(home), requires_grad=False)
|
||||
layer.register_parameter(
|
||||
"weight_scale",
|
||||
Parameter(weight_scale.to(home), requires_grad=False),
|
||||
)
|
||||
self.quant_config.note_quantized(weight.numel() * weight.element_size())
|
||||
del weight_q, weight_scale
|
||||
torch.cuda.empty_cache()
|
||||
# The ops add a BF16 bias; an FP16 pipeline loads FP16 parameters. A
|
||||
# deferred bias (skip_bias_add) never reaches the ops and keeps its dtype.
|
||||
if (
|
||||
layer.bias is not None
|
||||
and layer.bias.dtype != torch.bfloat16
|
||||
and not layer.skip_bias_add
|
||||
):
|
||||
layer.bias = Parameter(
|
||||
layer.bias.data.to(torch.bfloat16), requires_grad=False
|
||||
)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
out = convrot_int8_fused_linear(
|
||||
_as_rows(x),
|
||||
layer.weight,
|
||||
layer.weight_scale,
|
||||
bias,
|
||||
self.group_size,
|
||||
)
|
||||
return _like_input(out=out, x=x)
|
||||
|
||||
|
||||
def _forward_is_plain_apply(layer: LinearBase) -> bool:
|
||||
# True only when layer(x) is exactly quant_method.apply(layer, x, layer.bias):
|
||||
# no collective, no deferred bias, no rank-dependent bias.
|
||||
if layer.skip_bias_add:
|
||||
return False
|
||||
if isinstance(layer, ReplicatedLinear):
|
||||
return True
|
||||
if isinstance(layer, ColumnParallelLinear):
|
||||
return not layer.gather_output
|
||||
if isinstance(layer, RowParallelLinear):
|
||||
return layer.tp_size == 1
|
||||
return False
|
||||
|
||||
|
||||
def _is_jit_int8(layer: torch.nn.Module) -> bool:
|
||||
# LoRA wrappers and other non-LinearBase modules carry no quant_method.
|
||||
return isinstance(layer, LinearBase) and isinstance(
|
||||
layer.quant_method, ConvRotInt8JitLinearMethod
|
||||
)
|
||||
|
||||
|
||||
def convrot_int8_shares_input(layers: Sequence[torch.nn.Module]) -> bool:
|
||||
"""Whether ``apply_convrot_int8_shared_input`` reproduces ``layer(x)`` for
|
||||
every layer in ``layers``."""
|
||||
if not layers or not all(
|
||||
_is_jit_int8(layer) and _forward_is_plain_apply(layer) for layer in layers
|
||||
):
|
||||
return False
|
||||
# One rotated input serves every layer only at one group width.
|
||||
return len({layer.quant_method.group_size for layer in layers}) == 1
|
||||
|
||||
|
||||
def apply_convrot_int8_shared_input(
|
||||
x: torch.Tensor, layers: Sequence[LinearBase]
|
||||
) -> list[torch.Tensor]:
|
||||
"""``[layer(x)[0] for layer in layers]`` with ``x`` rotated and quantized once.
|
||||
|
||||
Bitwise identical to applying each layer on its own; see
|
||||
``convrot_int8_linear_prequant`` in ``sglang.kernels.ops.gemm.convrot_int8``.
|
||||
"""
|
||||
group_size = layers[0].quant_method.group_size
|
||||
x_q, x_scale = convrot_rotate_quantize_activation(_as_rows(x), group_size)
|
||||
outs = []
|
||||
for layer in layers:
|
||||
out = convrot_int8_linear_prequant(
|
||||
x_q,
|
||||
x_scale,
|
||||
layer.weight,
|
||||
layer.weight_scale,
|
||||
layer.bias,
|
||||
group_size,
|
||||
)
|
||||
outs.append(_like_input(out=out, x=x))
|
||||
return outs
|
||||
|
||||
|
||||
def apply_convrot_int8_shared_input_out(
|
||||
x: torch.Tensor, layers: Sequence[LinearBase], outs: Sequence[torch.Tensor]
|
||||
) -> None:
|
||||
"""Writes ``layer(x)[0]`` into ``out`` for each (layer, out) pair with ``x``
|
||||
rotated and quantized once; ``out`` must be a contiguous BF16 slice shaped
|
||||
like ``layer(x)[0]``, so ``x`` must be BF16 as well (an FP16 ``x`` would
|
||||
have ``layer(x)`` return FP16; use ``apply_convrot_int8_shared_input`` for
|
||||
it). Bitwise identical to ``apply_convrot_int8_shared_input`` and to
|
||||
``layer(x)``; see ``convrot_int8_linear_prequant`` in
|
||||
``sglang.kernels.ops.gemm.convrot_int8``.
|
||||
"""
|
||||
if x.dtype != torch.bfloat16:
|
||||
raise ValueError(
|
||||
"apply_convrot_int8_shared_input_out writes BF16 outputs, so its "
|
||||
f"input must be BF16, got {x.dtype}"
|
||||
)
|
||||
group_size = layers[0].quant_method.group_size
|
||||
x_q, x_scale = convrot_rotate_quantize_activation(_as_rows(x), group_size)
|
||||
for layer, out in zip(layers, outs, strict=True):
|
||||
# view() rather than reshape(): a copy here would silently drop the write.
|
||||
convrot_int8_linear_prequant(
|
||||
x_q,
|
||||
x_scale,
|
||||
layer.weight,
|
||||
layer.weight_scale,
|
||||
layer.bias,
|
||||
group_size,
|
||||
out=out.view(-1, out.shape[-1]),
|
||||
)
|
||||
|
||||
|
||||
def convrot_int8_fuses_gelu_input(layer: torch.nn.Module) -> bool:
|
||||
"""Whether ``apply_convrot_int8_gelu_input`` reproduces
|
||||
``layer(F.gelu(x, approximate="tanh"))[0]``."""
|
||||
return _is_jit_int8(layer) and _forward_is_plain_apply(layer)
|
||||
|
||||
|
||||
def apply_convrot_int8_gelu_input(layer: LinearBase, x: torch.Tensor) -> torch.Tensor:
|
||||
"""``layer(F.gelu(x, approximate="tanh"))[0]`` as one op, bitwise identical
|
||||
to the eager GELU followed by the layer for a BF16 ``x`` (an FP16 ``x`` is
|
||||
cast to BF16 before the GELU); see ``convrot_int8_fused_linear`` in
|
||||
``sglang.kernels.ops.gemm.convrot_int8``."""
|
||||
out = convrot_int8_fused_linear(
|
||||
_as_rows(x),
|
||||
layer.weight,
|
||||
layer.weight_scale,
|
||||
layer.bias,
|
||||
layer.quant_method.group_size,
|
||||
gelu_input=True,
|
||||
)
|
||||
return _like_input(out=out, x=x)
|
||||
@@ -0,0 +1,22 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Quantization method names and their deprecated aliases."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Deprecated public name -> current name.
|
||||
QUANTIZATION_METHOD_ALIASES: dict[str, str] = {"kitchen_int8": "convrot_int8"}
|
||||
|
||||
|
||||
def canonical_quantization_method(name: str) -> str:
|
||||
"""Map a deprecated method name to its current name, warning once per alias."""
|
||||
canonical = QUANTIZATION_METHOD_ALIASES.get(name)
|
||||
if canonical is None:
|
||||
return name
|
||||
logger.warning_once(
|
||||
f"quantization method {name!r} is a deprecated alias of {canonical!r}"
|
||||
)
|
||||
return canonical
|
||||
@@ -18,8 +18,8 @@ from sglang.multimodal_gen.runtime.layers.quantization.configs.base_config impor
|
||||
QuantizationConfig,
|
||||
QuantizeMethodBase,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.configs.kitchen_int8_config import (
|
||||
KitchenInt8Config,
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config import (
|
||||
ConvRotInt8Config,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.parameter import (
|
||||
ModelWeightParameter,
|
||||
@@ -328,7 +328,7 @@ class ModelOptFp4Config(ModelOptQuantConfig):
|
||||
self.swap_weight_nibbles = swap_weight_nibbles
|
||||
self.checkpoint_weight_scale_layout = checkpoint_weight_scale_layout
|
||||
self.checkpoint_uses_comfy_quantization = checkpoint_uses_comfy_quantization
|
||||
self._comfy_int8_config: KitchenInt8Config | None = None
|
||||
self._comfy_int8_config: ConvRotInt8Config | None = None
|
||||
self._comfy_fp8_config: ComfyFp8Config | None = None
|
||||
|
||||
def set_comfy_layer_markers(self, layer_markers: dict[str, dict[str, Any]]) -> None:
|
||||
@@ -346,7 +346,7 @@ class ModelOptFp4Config(ModelOptQuantConfig):
|
||||
if marker.get("format") == "int8_tensorwise"
|
||||
}
|
||||
self._comfy_int8_config = (
|
||||
KitchenInt8Config(layer_markers=int8_markers) if int8_markers else None
|
||||
ConvRotInt8Config(layer_markers=int8_markers) if int8_markers else None
|
||||
)
|
||||
fp8_markers = {
|
||||
prefix: marker
|
||||
|
||||
+7
-6
@@ -34,8 +34,8 @@ from sglang.multimodal_gen.runtime.layers.quantization.comfy_nvfp4 import (
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.configs.base_config import (
|
||||
QuantizationConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.configs.kitchen_int8_config import (
|
||||
KitchenInt8Config,
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config import (
|
||||
ConvRotInt8Config,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.configs.kitchen_w4a4_config import (
|
||||
KitchenW4A4Config,
|
||||
@@ -107,7 +107,7 @@ from sglang.srt.model_loader.checkpoint_quantization import (
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_ONLINE_ENCODER_QUANTIZATIONS = frozenset({"fp8", "kitchen_int8", "mxfp4"})
|
||||
_ONLINE_ENCODER_QUANTIZATIONS = frozenset({"fp8", "convrot_int8", "mxfp4"})
|
||||
|
||||
_TRANSFORMERS_ENCODER_ONLY_CLASSES = {
|
||||
"T5EncoderModel": transformers.T5EncoderModel,
|
||||
@@ -398,12 +398,13 @@ def _require_quantized_encoder_layers(
|
||||
(
|
||||
ComfyFp8Config,
|
||||
ComfyNvfp4Config,
|
||||
KitchenInt8Config,
|
||||
ConvRotInt8Config,
|
||||
KitchenW4A4Config,
|
||||
KitchenW4A8Config,
|
||||
),
|
||||
):
|
||||
expected = set(quant_config.layer_markers)
|
||||
# Online convrot_int8 carries no markers; there is nothing to consume.
|
||||
expected = set(quant_config.layer_markers or ())
|
||||
selected = set(quant_config.selected)
|
||||
elif isinstance(quant_config, QuantoInt8Config):
|
||||
expected = quant_config.layer_prefixes
|
||||
@@ -744,7 +745,7 @@ class TextEncoderLoader(OnlineQuantizationComponentLoader):
|
||||
if quant_config is not None and not isinstance(quant_config, GGUFConfig):
|
||||
postprocess_device: torch.device | None = local_torch_device
|
||||
if isinstance(quant_config, (ComfyNvfp4Config, QuantoInt8Config)) or (
|
||||
isinstance(quant_config, KitchenInt8Config)
|
||||
isinstance(quant_config, ConvRotInt8Config)
|
||||
and quant_config.is_checkpoint_int8_serialized
|
||||
):
|
||||
postprocess_device = None
|
||||
|
||||
+13
-1
@@ -13,6 +13,9 @@ from sglang.multimodal_gen.runtime.layers.attention.selector import (
|
||||
get_component_forced_attn_backend,
|
||||
get_global_forced_attn_backend,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config import (
|
||||
ConvRotInt8Config,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.loader.component_loaders.component_loader import (
|
||||
OnlineQuantizationComponentLoader,
|
||||
)
|
||||
@@ -379,6 +382,15 @@ class TransformerLoader(OnlineQuantizationComponentLoader):
|
||||
"AutoRound checkpoints do not support diffusion FSDP inference; "
|
||||
"use TP and/or sequence parallelism instead"
|
||||
)
|
||||
if (
|
||||
use_fsdp
|
||||
and isinstance(quant_spec.quant_config, ConvRotInt8Config)
|
||||
and not quant_spec.quant_config.is_checkpoint_int8_serialized
|
||||
):
|
||||
# Only the comfy_kitchen backend has run under FSDP.
|
||||
quant_spec.quant_config.require_comfy_kitchen(
|
||||
"has not been validated with diffusion FSDP inference"
|
||||
)
|
||||
|
||||
if quant_spec.gguf_file is not None:
|
||||
logger.info(
|
||||
@@ -476,7 +488,7 @@ class TransformerLoader(OnlineQuantizationComponentLoader):
|
||||
runtime_quant_config=quant_spec.runtime_quant_config,
|
||||
quantized_cpu_load_supported=(
|
||||
quant_spec.gguf_file is not None
|
||||
or quant_spec.is_serialized_kitchen_int8
|
||||
or quant_spec.is_convrot_int8
|
||||
or quant_spec.is_serialized_kitchen_w4a8
|
||||
),
|
||||
)
|
||||
|
||||
@@ -19,8 +19,8 @@ from torch import nn
|
||||
|
||||
from sglang.multimodal_gen.configs.models.dits.base import DiTArchConfig
|
||||
from sglang.multimodal_gen.runtime.layers.quantization import QuantizationConfig
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.configs.kitchen_int8_config import (
|
||||
KitchenInt8Config,
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config import (
|
||||
ConvRotInt8Config,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.configs.kitchen_w4a4_config import (
|
||||
KitchenW4A4Config,
|
||||
@@ -175,9 +175,13 @@ class TransformerQuantLoadSpec:
|
||||
return _get_quant_config_name(self.quant_config) == "comfy_fp8"
|
||||
|
||||
@property
|
||||
def is_serialized_kitchen_int8(self) -> bool:
|
||||
def is_convrot_int8(self) -> bool:
|
||||
return isinstance(self.quant_config, ConvRotInt8Config)
|
||||
|
||||
@property
|
||||
def is_serialized_convrot_int8(self) -> bool:
|
||||
return (
|
||||
isinstance(self.quant_config, KitchenInt8Config)
|
||||
isinstance(self.quant_config, ConvRotInt8Config)
|
||||
and self.quant_config.is_checkpoint_int8_serialized
|
||||
)
|
||||
|
||||
@@ -193,7 +197,7 @@ class TransformerQuantLoadSpec:
|
||||
def uses_comfy_layer_markers(self) -> bool:
|
||||
return (
|
||||
self.is_comfy_fp8
|
||||
or self.is_serialized_kitchen_int8
|
||||
or self.is_serialized_convrot_int8
|
||||
or self.is_serialized_kitchen_w4a4
|
||||
or self.is_serialized_kitchen_w4a8
|
||||
or (
|
||||
@@ -816,10 +820,8 @@ def _needs_device_weight_postprocess(
|
||||
quant_name = _get_quant_config_name(quant_config)
|
||||
if quant_name in ("modelopt_fp8", "comfy_fp8", "auto-round", "mxfp8"):
|
||||
return True
|
||||
if quant_name == "kitchen_int8":
|
||||
assert isinstance(quant_config, KitchenInt8Config)
|
||||
return not quant_config.is_checkpoint_int8_serialized
|
||||
|
||||
# convrot_int8 is deliberately absent: both backends quantize each layer
|
||||
# through their own CUDA round trip, so an offloaded DiT stays on the host.
|
||||
serialized_flag_by_quant_name = {
|
||||
"fp8": "is_checkpoint_fp8_serialized",
|
||||
"mxfp4": "is_checkpoint_mxfp4_serialized",
|
||||
@@ -1047,13 +1049,13 @@ def _resolve_quant_config(
|
||||
"such as owner/repo:Q4_K_M)."
|
||||
)
|
||||
|
||||
# Online-quant convention: for `fp8`, `mxfp4` and `kitchen_int8`, a
|
||||
# Online-quant convention: for `fp8`, `mxfp4` and `convrot_int8`, a
|
||||
# no-arg QuantizationConfig() selects the post-load path -- weights
|
||||
# load in source dtype and are quantized in
|
||||
# process_weights_after_loading.
|
||||
quant_cls = get_quantization_config(server_args.quantization)
|
||||
quant_kwargs = {}
|
||||
if server_args.quantization in {"fp8", "mxfp4", "kitchen_int8"}:
|
||||
if server_args.quantization in {"fp8", "mxfp4", "convrot_int8"}:
|
||||
quant_kwargs["ignored_layers"] = getattr(
|
||||
server_args, "quantization_ignored_layers", None
|
||||
)
|
||||
|
||||
@@ -2,10 +2,16 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Sequence
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.convrot_int8_jit import (
|
||||
apply_convrot_int8_shared_input,
|
||||
convrot_int8_shares_input,
|
||||
)
|
||||
|
||||
|
||||
def modulate(
|
||||
x: torch.Tensor,
|
||||
@@ -22,6 +28,14 @@ def modulate(
|
||||
return x * (1 + scale.unsqueeze(1)) + shift.unsqueeze(1)
|
||||
|
||||
|
||||
def _project_shared_input(
|
||||
x: torch.Tensor, layers: Sequence[torch.nn.Module]
|
||||
) -> list[torch.Tensor]:
|
||||
if convrot_int8_shares_input(layers):
|
||||
return apply_convrot_int8_shared_input(x=x, layers=layers)
|
||||
return [layer(x)[0] for layer in layers]
|
||||
|
||||
|
||||
def get_qkv_projections(
|
||||
attn: Any,
|
||||
hidden_states: torch.Tensor,
|
||||
@@ -53,9 +67,9 @@ def get_qkv_projections(
|
||||
if make_contiguous:
|
||||
query, key, value = [t.contiguous() for t in (query, key, value)]
|
||||
else:
|
||||
query, _ = attn.to_q(hidden_states)
|
||||
key, _ = attn.to_k(hidden_states)
|
||||
value, _ = attn.to_v(hidden_states)
|
||||
query, key, value = _project_shared_input(
|
||||
x=hidden_states, layers=(attn.to_q, attn.to_k, attn.to_v)
|
||||
)
|
||||
|
||||
encoder_query = encoder_key = encoder_value = None
|
||||
if encoder_hidden_states is not None and attn.added_kv_proj_dim is not None:
|
||||
@@ -67,8 +81,9 @@ def get_qkv_projections(
|
||||
t.contiguous() for t in (encoder_query, encoder_key, encoder_value)
|
||||
]
|
||||
else:
|
||||
encoder_query, _ = attn.add_q_proj(encoder_hidden_states)
|
||||
encoder_key, _ = attn.add_k_proj(encoder_hidden_states)
|
||||
encoder_value, _ = attn.add_v_proj(encoder_hidden_states)
|
||||
encoder_query, encoder_key, encoder_value = _project_shared_input(
|
||||
x=encoder_hidden_states,
|
||||
layers=(attn.add_q_proj, attn.add_k_proj, attn.add_v_proj),
|
||||
)
|
||||
|
||||
return query, key, value, encoder_query, encoder_key, encoder_value
|
||||
|
||||
@@ -73,6 +73,7 @@ from sglang.multimodal_gen.runtime.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
ReplicatedLinear,
|
||||
RowParallelLinear,
|
||||
UnquantizedLinearMethod,
|
||||
apply_unquantized_linear,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.configs.base_config import (
|
||||
@@ -82,6 +83,13 @@ from sglang.multimodal_gen.runtime.layers.quantization.configs.nunchaku_config i
|
||||
NunchakuConfig,
|
||||
is_nunchaku_available,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.convrot_int8_jit import (
|
||||
apply_convrot_int8_gelu_input,
|
||||
apply_convrot_int8_shared_input,
|
||||
apply_convrot_int8_shared_input_out,
|
||||
convrot_int8_fuses_gelu_input,
|
||||
convrot_int8_shares_input,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.modelopt_quant import (
|
||||
ModelOptFp4LinearMethod,
|
||||
ModelOptFp8LinearMethod,
|
||||
@@ -720,7 +728,161 @@ class QwenEmbedLayer3DRope(nn.Module):
|
||||
return freqs.clone().contiguous()
|
||||
|
||||
|
||||
def _joint_qkv_layers(
|
||||
attn: "QwenImageCrossAttention",
|
||||
) -> tuple[nn.Module, nn.Module, nn.Module, nn.Module, nn.Module, nn.Module]:
|
||||
return (
|
||||
attn.to_q,
|
||||
attn.to_k,
|
||||
attn.to_v,
|
||||
attn.add_q_proj,
|
||||
attn.add_k_proj,
|
||||
attn.add_v_proj,
|
||||
)
|
||||
|
||||
|
||||
def _joint_qkv_operands_match(
|
||||
*,
|
||||
attn: "QwenImageCrossAttention",
|
||||
hidden_states: torch.Tensor,
|
||||
encoder_hidden_states: torch.Tensor,
|
||||
) -> bool:
|
||||
dtype = hidden_states.dtype
|
||||
if encoder_hidden_states.dtype != dtype:
|
||||
return False
|
||||
layers = _joint_qkv_layers(attn)
|
||||
# A LoRA wrapper forwards .weight to its base layer and adds its delta in
|
||||
# forward, so the bare addmm below would silently drop it.
|
||||
if not all(isinstance(layer, ColumnParallelLinear) for layer in layers):
|
||||
return False
|
||||
if attn.separate_convrot_qkv_proj:
|
||||
# apply_convrot_int8_shared_input_out stores BF16 only; FP16 streams
|
||||
# take apply_convrot_int8_shared_input, which casts at the op boundary.
|
||||
return dtype == torch.bfloat16
|
||||
# The out= GEMMs below take no part in autocast and do no promotion, so
|
||||
# every operand must already be in the dtype F.linear would compute in.
|
||||
device_type = hidden_states.device.type
|
||||
if (
|
||||
torch.is_autocast_enabled(device_type)
|
||||
and torch.get_autocast_dtype(device_type) != dtype
|
||||
):
|
||||
return False
|
||||
return all(
|
||||
layer.weight.dtype == dtype
|
||||
and (layer.bias is None or layer.bias.dtype == dtype)
|
||||
for layer in layers
|
||||
)
|
||||
|
||||
|
||||
def _use_joint_qkv_buffers(
|
||||
*,
|
||||
attn: "QwenImageCrossAttention",
|
||||
hidden_states: torch.Tensor,
|
||||
encoder_hidden_states: torch.Tensor,
|
||||
masked: bool,
|
||||
) -> bool:
|
||||
# Batch 1 keeps each slice contiguous for the out= writes and the in-place
|
||||
# QK-norm; SP>1 and masked sequences never take join_seqs.
|
||||
return (
|
||||
(attn.separate_unquantized_qkv_proj or attn.separate_convrot_qkv_proj)
|
||||
and not masked
|
||||
# The out= GEMMs below reject grad-tracking operands.
|
||||
and not torch.is_grad_enabled()
|
||||
and hidden_states.shape[0] == 1
|
||||
and hidden_states.is_contiguous()
|
||||
and encoder_hidden_states.is_contiguous()
|
||||
and get_sp_world_size() <= 1
|
||||
and _joint_qkv_operands_match(
|
||||
attn=attn,
|
||||
hidden_states=hidden_states,
|
||||
encoder_hidden_states=encoder_hidden_states,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _project_into(
|
||||
*, x: torch.Tensor, layer: ColumnParallelLinear, out: torch.Tensor
|
||||
) -> None:
|
||||
# The addmm F.linear dispatches for a contiguous x, so the out= write is
|
||||
# bitwise identical to layer(x).
|
||||
x_2d = x.reshape(-1, x.shape[-1])
|
||||
# view() rather than reshape(): a copy here would silently drop the write.
|
||||
out_2d = out.view(-1, out.shape[-1])
|
||||
if layer.bias is None:
|
||||
torch.mm(x_2d, layer.weight.t(), out=out_2d)
|
||||
else:
|
||||
torch.addmm(layer.bias, x_2d, layer.weight.t(), out=out_2d)
|
||||
|
||||
|
||||
def _project_qkv_into_joint_buffers(
|
||||
*,
|
||||
attn: "QwenImageCrossAttention",
|
||||
hidden_states: torch.Tensor,
|
||||
encoder_hidden_states: torch.Tensor,
|
||||
seq_len_txt: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
# Three [B, txt + img, local_inner_dim] buffers, text rows first.
|
||||
batch_size, seq_len_img, _ = hidden_states.shape
|
||||
inner_dim_local = attn.local_num_heads * attn.head_dim
|
||||
shape = (batch_size, seq_len_txt + seq_len_img, inner_dim_local)
|
||||
bufs = tuple(
|
||||
torch.empty(shape, dtype=hidden_states.dtype, device=hidden_states.device)
|
||||
for _ in range(3)
|
||||
)
|
||||
to_q, to_k, to_v, add_q_proj, add_k_proj, add_v_proj = _joint_qkv_layers(attn)
|
||||
img_layers = (to_q, to_k, to_v)
|
||||
txt_layers = (add_q_proj, add_k_proj, add_v_proj)
|
||||
if attn.separate_convrot_qkv_proj:
|
||||
apply_convrot_int8_shared_input_out(
|
||||
x=hidden_states,
|
||||
layers=img_layers,
|
||||
outs=[buf[:, seq_len_txt:] for buf in bufs],
|
||||
)
|
||||
apply_convrot_int8_shared_input_out(
|
||||
x=encoder_hidden_states,
|
||||
layers=txt_layers,
|
||||
outs=[buf[:, :seq_len_txt] for buf in bufs],
|
||||
)
|
||||
return bufs
|
||||
for buf, img_layer, txt_layer in zip(bufs, img_layers, txt_layers):
|
||||
_project_into(x=hidden_states, layer=img_layer, out=buf[:, seq_len_txt:])
|
||||
_project_into(
|
||||
x=encoder_hidden_states, layer=txt_layer, out=buf[:, :seq_len_txt]
|
||||
)
|
||||
return bufs
|
||||
|
||||
|
||||
def _joint_qkv_head_views(
|
||||
bufs: tuple[torch.Tensor, torch.Tensor, torch.Tensor],
|
||||
*,
|
||||
seq_len_txt: int,
|
||||
num_heads: int,
|
||||
head_dim: int,
|
||||
) -> tuple[torch.Tensor, ...]:
|
||||
# (img_q, img_k, img_v, txt_q, txt_k, txt_v), each a [B, S, H, D] view.
|
||||
img_views = tuple(
|
||||
buf[:, seq_len_txt:].unflatten(-1, (num_heads, head_dim)) for buf in bufs
|
||||
)
|
||||
txt_views = tuple(
|
||||
buf[:, :seq_len_txt].unflatten(-1, (num_heads, head_dim)) for buf in bufs
|
||||
)
|
||||
return (*img_views, *txt_views)
|
||||
|
||||
|
||||
def _sync_into_buffer(*, value: torch.Tensor, buf_view: torch.Tensor) -> None:
|
||||
# In-place QK-norm/RoPE aliases value to buf_view; only the non-fused
|
||||
# fallback returns a fresh tensor that must be copied back.
|
||||
# Dynamo cannot trace data_ptr(); a self-aliased copy_ is a no-op anyway.
|
||||
if torch.compiler.is_compiling() or value.data_ptr() != buf_view.data_ptr():
|
||||
buf_view.copy_(value)
|
||||
|
||||
|
||||
class QwenImageCrossAttention(nn.Module):
|
||||
# Per-instance values are set in __init__; the class defaults keep forward()
|
||||
# valid on modules built around it (object.__new__ in the epilogue tests).
|
||||
separate_unquantized_qkv_proj = False
|
||||
separate_convrot_qkv_proj = False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
dim: int, # query_dim
|
||||
@@ -776,6 +938,8 @@ class QwenImageCrossAttention(nn.Module):
|
||||
)
|
||||
self.local_num_heads = self.num_heads // tp_size
|
||||
self._unquantized_added_qkv_is_packed = False
|
||||
self.separate_unquantized_qkv_proj = False
|
||||
self.separate_convrot_qkv_proj = False
|
||||
|
||||
if self.use_fused_qkv:
|
||||
# Use fused QKV projection for nunchaku quantization
|
||||
@@ -860,6 +1024,24 @@ class QwenImageCrossAttention(nn.Module):
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.add_v_proj",
|
||||
)
|
||||
# Six plain linears can write straight into joint text-image
|
||||
# buffers (see _use_joint_qkv_buffers); the fused-epilogue
|
||||
# path consumes packed projection strides instead.
|
||||
self.separate_unquantized_qkv_proj = (
|
||||
not self.use_fused_qkv
|
||||
and not self.use_fused_qkv_epilogue
|
||||
and all(
|
||||
isinstance(layer.quant_method, UnquantizedLinearMethod)
|
||||
for layer in _joint_qkv_layers(self)
|
||||
)
|
||||
)
|
||||
# Six ConvRot INT8 linears share one rotated+quantized input per
|
||||
# stream and also write straight into the joint buffers.
|
||||
self.separate_convrot_qkv_proj = (
|
||||
not self.use_fused_qkv
|
||||
and not self.use_fused_qkv_epilogue
|
||||
and convrot_int8_shares_input(_joint_qkv_layers(self))
|
||||
)
|
||||
|
||||
if context_pre_only is not None and not context_pre_only:
|
||||
self.to_add_out = RowParallelLinear(
|
||||
@@ -961,7 +1143,35 @@ class QwenImageCrossAttention(nn.Module):
|
||||
# Rows of tail padding inside THIS rank's text chunk (sp_shard meta).
|
||||
sp_txt_pad = _attn_mask_meta_local_pad(attn_mask_meta)
|
||||
|
||||
if self._unquantized_added_qkv_is_packed and not qwen_image_added_qkv_active(
|
||||
# Joint [text, image] destination buffers take precedence over the
|
||||
# packed added-QKV GEMM, which in turn precedes per-layer projections.
|
||||
joint_qkv_bufs = None
|
||||
if _use_joint_qkv_buffers(
|
||||
attn=self,
|
||||
hidden_states=hidden_states,
|
||||
encoder_hidden_states=encoder_hidden_states,
|
||||
masked=attn_mask is not None or encoder_hidden_states_mask is not None,
|
||||
):
|
||||
joint_qkv_bufs = _project_qkv_into_joint_buffers(
|
||||
attn=self,
|
||||
hidden_states=hidden_states,
|
||||
encoder_hidden_states=encoder_hidden_states,
|
||||
seq_len_txt=seq_len_txt,
|
||||
)
|
||||
(
|
||||
img_query,
|
||||
img_key,
|
||||
img_value,
|
||||
txt_query,
|
||||
txt_key,
|
||||
txt_value,
|
||||
) = _joint_qkv_head_views(
|
||||
joint_qkv_bufs,
|
||||
seq_len_txt=seq_len_txt,
|
||||
num_heads=self.local_num_heads,
|
||||
head_dim=self.head_dim,
|
||||
)
|
||||
elif self._unquantized_added_qkv_is_packed and not qwen_image_added_qkv_active(
|
||||
self
|
||||
):
|
||||
img_query, img_key, img_value, _, _, _ = _get_qkv_projections(
|
||||
@@ -972,6 +1182,20 @@ class QwenImageCrossAttention(nn.Module):
|
||||
txt_query, txt_key, txt_value = self._get_added_qkv_projections(
|
||||
encoder_hidden_states
|
||||
)
|
||||
elif self.separate_convrot_qkv_proj and convrot_int8_shares_input(
|
||||
_joint_qkv_layers(self)
|
||||
):
|
||||
# Same rotate-once projections as the joint-buffer path, into fresh
|
||||
# tensors when the joint layout is not applicable (SP, masks, B>1).
|
||||
# Re-checked per call: LoRA mounting swaps the projections for
|
||||
# wrappers that add their delta in forward.
|
||||
img_query, img_key, img_value = apply_convrot_int8_shared_input(
|
||||
x=hidden_states, layers=(self.to_q, self.to_k, self.to_v)
|
||||
)
|
||||
txt_query, txt_key, txt_value = apply_convrot_int8_shared_input(
|
||||
x=encoder_hidden_states,
|
||||
layers=(self.add_q_proj, self.add_k_proj, self.add_v_proj),
|
||||
)
|
||||
else:
|
||||
(
|
||||
img_query,
|
||||
@@ -993,14 +1217,15 @@ class QwenImageCrossAttention(nn.Module):
|
||||
else:
|
||||
img_complex = txt_complex = None
|
||||
|
||||
# Reshape for multi-head attention
|
||||
img_query = img_query.unflatten(-1, (self.local_num_heads, self.head_dim))
|
||||
img_key = img_key.unflatten(-1, (self.local_num_heads, self.head_dim))
|
||||
img_value = img_value.unflatten(-1, (self.local_num_heads, self.head_dim))
|
||||
if joint_qkv_bufs is None:
|
||||
# Reshape for multi-head attention
|
||||
img_query = img_query.unflatten(-1, (self.local_num_heads, self.head_dim))
|
||||
img_key = img_key.unflatten(-1, (self.local_num_heads, self.head_dim))
|
||||
img_value = img_value.unflatten(-1, (self.local_num_heads, self.head_dim))
|
||||
|
||||
txt_query = txt_query.unflatten(-1, (self.local_num_heads, self.head_dim))
|
||||
txt_key = txt_key.unflatten(-1, (self.local_num_heads, self.head_dim))
|
||||
txt_value = txt_value.unflatten(-1, (self.local_num_heads, self.head_dim))
|
||||
txt_query = txt_query.unflatten(-1, (self.local_num_heads, self.head_dim))
|
||||
txt_key = txt_key.unflatten(-1, (self.local_num_heads, self.head_dim))
|
||||
txt_value = txt_value.unflatten(-1, (self.local_num_heads, self.head_dim))
|
||||
|
||||
img_cache = txt_cache = None
|
||||
if image_rotary_emb is not None:
|
||||
@@ -1117,6 +1342,17 @@ class QwenImageCrossAttention(nn.Module):
|
||||
joint_query, joint_key, joint_value = joint_qkv
|
||||
elif seg_qkv is not None:
|
||||
joint_query, joint_key, joint_value = seg_qkv
|
||||
elif joint_qkv_bufs is not None:
|
||||
# The buffers already hold the [text, image] layout: Q/K/V were
|
||||
# projected into them and QK-norm/RoPE ran in place on views.
|
||||
joint_query, joint_key, joint_value = (
|
||||
buf.unflatten(-1, (self.local_num_heads, self.head_dim))
|
||||
for buf in joint_qkv_bufs
|
||||
)
|
||||
_sync_into_buffer(value=img_query, buf_view=joint_query[:, seq_len_txt:])
|
||||
_sync_into_buffer(value=img_key, buf_view=joint_key[:, seq_len_txt:])
|
||||
_sync_into_buffer(value=txt_query, buf_view=joint_query[:, :seq_len_txt])
|
||||
_sync_into_buffer(value=txt_key, buf_view=joint_key[:, :seq_len_txt])
|
||||
elif attn_mask is not None and not sp_text_sharded:
|
||||
# Let the eager attention break point pack directly from the text
|
||||
# and image segments. Materializing three dense joint tensors here
|
||||
@@ -1269,6 +1505,13 @@ class QwenImageFeedForward(nn.Module):
|
||||
def forward_with_bias(
|
||||
self, hidden_states: torch.Tensor | tuple[torch.Tensor, torch.Tensor]
|
||||
) -> tuple[torch.Tensor, Optional[torch.Tensor]]:
|
||||
# Not cached at construction: LoRA mounting swaps net.2 for a wrapper.
|
||||
if convrot_int8_fuses_gelu_input(self.net[2]):
|
||||
# The down-projection's rotate kernel applies GELU(tanh) to its
|
||||
# input bit-exactly, so the up-projection runs bare and the
|
||||
# standalone activation kernel is skipped.
|
||||
up, _ = self.net[0].proj(hidden_states)
|
||||
return apply_convrot_int8_gelu_input(layer=self.net[2], x=up), None
|
||||
hidden_states = self.net[0](hidden_states)
|
||||
hidden_states = self.net[1](hidden_states)
|
||||
return self.net[2](hidden_states)
|
||||
|
||||
@@ -32,6 +32,9 @@ from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.configs.nunchaku_config import (
|
||||
NunchakuConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.method_names import (
|
||||
canonical_quantization_method,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.loader.utils import BYTES_PER_GB
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.component_residency import (
|
||||
COMPONENT_OFFLOAD,
|
||||
@@ -1981,10 +1984,14 @@ class ServerArgs(DisaggServerArgsMixin):
|
||||
)
|
||||
normalized_direct_gpu_loading[component_name] = enabled
|
||||
self.component_direct_gpu_weight_loading = normalized_direct_gpu_loading
|
||||
if self.quantization is not None:
|
||||
self.quantization = canonical_quantization_method(self.quantization)
|
||||
normalized_quantizations: dict[str, str] = {}
|
||||
for component, quantization in self.component_quantizations.items():
|
||||
component = str(component).strip().replace("-", "_")
|
||||
quantization = str(quantization).strip().lower()
|
||||
quantization = canonical_quantization_method(
|
||||
str(quantization).strip().lower()
|
||||
)
|
||||
if not component or not quantization:
|
||||
raise ValueError(
|
||||
"Component quantization entries require a component and method"
|
||||
@@ -2789,7 +2796,11 @@ class ServerArgs(DisaggServerArgsMixin):
|
||||
"auto-detected from the checkpoint config or safetensors metadata when "
|
||||
"possible. Use this flag to override auto-detection. "
|
||||
"Online (post-load) quantization from a BF16/FP16 checkpoint "
|
||||
"is supported for 'fp8' and 'mxfp4'. Other methods "
|
||||
"is supported for 'fp8', 'mxfp4' and 'convrot_int8' (ConvRot INT8 "
|
||||
"W8A8; runs on SGLang's JIT-compiled fused ops on CC 9.0, 10.0, "
|
||||
"12.0 and 12.1, else on comfy_kitchen; see "
|
||||
"SGLANG_DIFFUSION_CONVROT_INT8_BACKEND; 'kitchen_int8' is a "
|
||||
"deprecated alias). Other methods "
|
||||
"('modelopt', 'modelopt_fp8', 'modelopt_fp4', 'mxfp8', "
|
||||
"'mxfp4_npu', 'modelslim') require a pre-quantized checkpoint. "
|
||||
"Note: 'mxfp4' targets ROCm + MI350+ (gfx95x); "
|
||||
@@ -2803,7 +2814,8 @@ class ServerArgs(DisaggServerArgsMixin):
|
||||
default=ServerArgs.quantization_ignored_layers,
|
||||
help=(
|
||||
"Layer name patterns to keep unquantized during online quantization "
|
||||
"(fp8/mxfp4). Each pattern is matched against the layer prefix. "
|
||||
"(fp8/mxfp4/convrot_int8). Each pattern is matched against the "
|
||||
"layer prefix. "
|
||||
"Example: --quantization-ignored-layers img_mod txt_mod to_out"
|
||||
),
|
||||
)
|
||||
|
||||
@@ -22,8 +22,8 @@ from sglang.multimodal_gen.runtime.layers.quantization.comfy_fp8 import ComfyFp8
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.comfy_nvfp4 import (
|
||||
ComfyNvfp4Config,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.configs.kitchen_int8_config import (
|
||||
KitchenInt8Config,
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config import (
|
||||
ConvRotInt8Config,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.configs.kitchen_w4a4_config import (
|
||||
KitchenW4A4Config,
|
||||
@@ -395,7 +395,7 @@ def resolve_comfy_checkpoint_quantization(
|
||||
return None
|
||||
formats = sorted({str(marker.get("format")) for marker in layer_markers.values()})
|
||||
if formats == ["int8_tensorwise"]:
|
||||
return KitchenInt8Config(layer_markers=layer_markers)
|
||||
return ConvRotInt8Config(layer_markers=layer_markers)
|
||||
if formats == ["asym_w4a8_int8"]:
|
||||
return KitchenW4A8Config(layer_markers)
|
||||
if formats == ["asym_w4a8_int8", "int8_tensorwise"]:
|
||||
|
||||
@@ -0,0 +1,672 @@
|
||||
"""Unit tests for convrot_int8's backend selection and its JIT backend."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from sglang.kernels.ops.gemm.convrot_int8 import convrot_int8_supported_capabilities
|
||||
from sglang.multimodal_gen.runtime.layers.linear import (
|
||||
LinearBase,
|
||||
ReplicatedLinear,
|
||||
UnquantizedLinearMethod,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.lora.linear import wrap_with_lora_layer
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config import (
|
||||
ConvRotInt8Config,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.convrot_int8_comfy_kitchen import (
|
||||
ConvRotInt8ComfyKitchenLinearMethod,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.convrot_int8_jit import (
|
||||
ConvRotInt8JitLinearMethod,
|
||||
apply_convrot_int8_gelu_input,
|
||||
apply_convrot_int8_shared_input,
|
||||
apply_convrot_int8_shared_input_out,
|
||||
convrot_int8_fuses_gelu_input,
|
||||
convrot_int8_shares_input,
|
||||
)
|
||||
|
||||
_CONFIG_MODULE = (
|
||||
"sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config"
|
||||
)
|
||||
_LOAD_JIT_KERNEL = (
|
||||
"sglang.multimodal_gen.runtime.layers.quantization.convrot_int8_jit."
|
||||
"_load_jit_kernel"
|
||||
)
|
||||
_LOAD_COMFY_KITCHEN = "sglang.multimodal_gen.runtime.layers.quantization.convrot_int8_comfy_kitchen._load_comfy_kitchen"
|
||||
|
||||
QWEN_IMAGE_IGNORED_LAYERS = ["img_mod", "txt_mod", "txt_mlp.net.2"]
|
||||
|
||||
|
||||
# Capability only: on a supported GPU a missing toolchain or a failed build must
|
||||
# fail these tests, not skip them.
|
||||
requires_kernel = pytest.mark.skipif(
|
||||
not (
|
||||
torch.cuda.is_available()
|
||||
and torch.cuda.get_device_capability() in convrot_int8_supported_capabilities()
|
||||
),
|
||||
reason="needs a GPU in the convrot_int8 capability table",
|
||||
)
|
||||
|
||||
|
||||
def _jit_config(**kwargs) -> ConvRotInt8Config:
|
||||
return ConvRotInt8Config(backend="jit", **kwargs)
|
||||
|
||||
|
||||
def _method_for(config, prefix, input_size=3072):
|
||||
layer = LinearBase(input_size=input_size, output_size=32)
|
||||
return config.get_quant_method(layer, prefix)
|
||||
|
||||
|
||||
@patch(_LOAD_JIT_KERNEL)
|
||||
def test_ignored_layer_patterns_select_bf16_on_module_path_boundaries(_load):
|
||||
"""A pattern must match its own module path only: `txt_mlp.net.2` keeps the
|
||||
text FFN down-projection in BF16 without also catching `img_mlp.net.2` or
|
||||
the text FFN up-projection, and `img_mod` must not catch `img_mlp`."""
|
||||
config = _jit_config(ignored_layers=QWEN_IMAGE_IGNORED_LAYERS)
|
||||
|
||||
kept_bf16 = [
|
||||
"transformer_blocks.0.img_mod.1",
|
||||
"transformer_blocks.3.txt_mod.1",
|
||||
"transformer_blocks.7.txt_mlp.net.2",
|
||||
]
|
||||
quantized = [
|
||||
"transformer_blocks.0.img_mlp.net.2",
|
||||
"transformer_blocks.7.txt_mlp.net.0.proj",
|
||||
"transformer_blocks.0.attn.to_q",
|
||||
"transformer_blocks.0.attn.add_k_proj",
|
||||
]
|
||||
for prefix in kept_bf16:
|
||||
assert isinstance(_method_for(config, prefix), UnquantizedLinearMethod)
|
||||
for prefix in quantized:
|
||||
assert isinstance(_method_for(config, prefix), ConvRotInt8JitLinearMethod)
|
||||
assert config.skipped == kept_bf16
|
||||
assert config.selected == quantized
|
||||
|
||||
|
||||
@patch(_LOAD_JIT_KERNEL)
|
||||
def test_input_dim_not_divisible_by_group_stays_bf16(_load):
|
||||
config = _jit_config()
|
||||
method = _method_for(config, "blocks.0.mod", input_size=2688)
|
||||
assert isinstance(method, UnquantizedLinearMethod)
|
||||
assert config.skipped == ["blocks.0.mod(in=2688)"]
|
||||
assert config.selected == []
|
||||
assert not config.supports_input_partition("blocks.0.attn.to_out", 1536 + 128)
|
||||
assert config.supports_input_partition("blocks.0.attn.to_out", 1536)
|
||||
|
||||
|
||||
@patch(_LOAD_JIT_KERNEL)
|
||||
def test_shared_input_and_gelu_helpers_refuse_deferred_bias_and_lora(_load):
|
||||
"""The helpers stand in for `layer(x)`; a layer that returns its bias
|
||||
separately would silently lose it, and a LoRA-wrapped projection is not a
|
||||
`LinearBase` at all, so eligibility must say no to both instead of
|
||||
crashing on the wrapper."""
|
||||
config = _jit_config()
|
||||
plain = ReplicatedLinear(256, 8, quant_config=config, prefix="q")
|
||||
deferred = ReplicatedLinear(
|
||||
256, 8, skip_bias_add=True, quant_config=config, prefix="k"
|
||||
)
|
||||
bf16 = ReplicatedLinear(256, 8, quant_config=None, prefix="v")
|
||||
wrapped = wrap_with_lora_layer(plain)
|
||||
|
||||
assert convrot_int8_shares_input([plain, plain])
|
||||
assert not convrot_int8_shares_input([plain, deferred])
|
||||
assert not convrot_int8_shares_input([plain, bf16])
|
||||
assert not convrot_int8_shares_input([wrapped, plain])
|
||||
assert not convrot_int8_shares_input([])
|
||||
assert convrot_int8_fuses_gelu_input(plain)
|
||||
assert not convrot_int8_fuses_gelu_input(deferred)
|
||||
assert not convrot_int8_fuses_gelu_input(bf16)
|
||||
assert not convrot_int8_fuses_gelu_input(wrapped)
|
||||
|
||||
|
||||
def _quantized_layer(
|
||||
config, in_features, out_features, prefix, seed, dtype=torch.bfloat16
|
||||
):
|
||||
gen = torch.Generator(device="cuda").manual_seed(seed)
|
||||
layer = ReplicatedLinear(
|
||||
in_features,
|
||||
out_features,
|
||||
params_dtype=dtype,
|
||||
quant_config=config,
|
||||
prefix=prefix,
|
||||
).cuda()
|
||||
with torch.no_grad():
|
||||
layer.weight.copy_(
|
||||
torch.randn(layer.weight.shape, device="cuda", generator=gen) * 0.02
|
||||
)
|
||||
layer.bias.copy_(torch.randn(layer.bias.shape, device="cuda", generator=gen))
|
||||
layer.bias.requires_grad_(False)
|
||||
reference_weight = layer.weight.detach().clone()
|
||||
layer.quant_method.process_weights_after_loading(layer)
|
||||
return layer, reference_weight
|
||||
|
||||
|
||||
@requires_kernel
|
||||
def test_quantized_layer_tracks_bf16_reference():
|
||||
config = _jit_config()
|
||||
layer, weight = _quantized_layer(config, 3072, 3072, "attn.to_q", seed=0)
|
||||
assert layer.weight.dtype == torch.int8
|
||||
assert layer.weight_scale.shape == (3072,)
|
||||
|
||||
x = torch.randn(2, 1024, 3072, device="cuda", dtype=torch.bfloat16)
|
||||
out, out_bias = layer(x)
|
||||
assert out_bias is None
|
||||
ref = F.linear(x, weight, layer.bias)
|
||||
err = torch.linalg.vector_norm(out.float() - ref.float())
|
||||
rel_l2 = (err / torch.linalg.vector_norm(ref.float())).item()
|
||||
# Arbitrary loose bound: a wrong scale or rotation lands far above it,
|
||||
# W8A8 group-256 quantization noise on Gaussian data far below.
|
||||
assert rel_l2 < 2e-2, rel_l2
|
||||
|
||||
|
||||
@requires_kernel
|
||||
def test_shared_input_helper_is_bitwise_three_layer_calls():
|
||||
config = _jit_config()
|
||||
layers = [
|
||||
_quantized_layer(config, 3072, 3072, f"attn.to_{n}", seed=i)[0]
|
||||
for i, n in enumerate("qkv")
|
||||
]
|
||||
x = torch.randn(1, 4096, 3072, device="cuda", dtype=torch.bfloat16)
|
||||
|
||||
assert convrot_int8_shares_input(layers)
|
||||
shared = apply_convrot_int8_shared_input(x=x, layers=layers)
|
||||
for out, layer in zip(shared, layers, strict=True):
|
||||
assert torch.equal(out, layer(x)[0])
|
||||
|
||||
|
||||
@requires_kernel
|
||||
def test_shared_input_out_helper_is_bitwise_into_joint_buffer_slices():
|
||||
"""The joint text-image Q/K/V path writes each projection into a row slice
|
||||
of one larger buffer; the slice write must equal the allocating helper."""
|
||||
config = _jit_config()
|
||||
layers = [
|
||||
_quantized_layer(config, 3072, 3072, f"attn.to_{n}", seed=10 + i)[0]
|
||||
for i, n in enumerate("qkv")
|
||||
]
|
||||
seq_len_txt, seq_len_img = 20, 4096
|
||||
x = torch.randn(1, seq_len_img, 3072, device="cuda", dtype=torch.bfloat16)
|
||||
bufs = [
|
||||
torch.zeros(
|
||||
1, seq_len_txt + seq_len_img, 3072, device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
for _ in layers
|
||||
]
|
||||
|
||||
apply_convrot_int8_shared_input_out(
|
||||
x=x, layers=layers, outs=[buf[:, seq_len_txt:] for buf in bufs]
|
||||
)
|
||||
for out, buf, layer in zip(
|
||||
apply_convrot_int8_shared_input(x=x, layers=layers), bufs, layers, strict=True
|
||||
):
|
||||
assert torch.equal(buf[:, seq_len_txt:], out)
|
||||
assert torch.equal(buf[:, seq_len_txt:], layer(x)[0])
|
||||
assert not buf[:, :seq_len_txt].any()
|
||||
|
||||
|
||||
@requires_kernel
|
||||
def test_gelu_input_helper_is_bitwise_eager_gelu_then_layer():
|
||||
config = _jit_config()
|
||||
down, _ = _quantized_layer(config, 12288, 3072, "img_mlp.net.2", seed=3)
|
||||
up = torch.randn(1, 2048, 12288, device="cuda", dtype=torch.bfloat16)
|
||||
|
||||
assert convrot_int8_fuses_gelu_input(down)
|
||||
fused = apply_convrot_int8_gelu_input(layer=down, x=up)
|
||||
eager, _ = down(F.gelu(up, approximate="tanh"))
|
||||
assert torch.equal(fused, eager)
|
||||
|
||||
|
||||
class _FakeTpGroup:
|
||||
pass
|
||||
|
||||
|
||||
def _with_tp(monkeypatch, tp_size):
|
||||
import sglang.multimodal_gen.runtime.layers.linear as linear
|
||||
|
||||
monkeypatch.setattr(linear, "get_tp_group", lambda: _FakeTpGroup())
|
||||
monkeypatch.setattr(linear, "get_group_rank", lambda group: 0)
|
||||
monkeypatch.setattr(linear, "get_group_size", lambda group: tp_size)
|
||||
|
||||
|
||||
@patch(_LOAD_JIT_KERNEL)
|
||||
@pytest.mark.parametrize("tp_size,input_size", [(4, 13824), (8, 3072)])
|
||||
def test_row_parallel_shard_not_divisible_by_group_is_refused(
|
||||
_load, monkeypatch, tp_size, input_size
|
||||
):
|
||||
from sglang.multimodal_gen.runtime.layers.linear import RowParallelLinear
|
||||
|
||||
_with_tp(monkeypatch, tp_size)
|
||||
config = _jit_config()
|
||||
# convrot_int8 semantics: the unsharded input divides, so the layer is
|
||||
# selected, and a shard that does not divide is refused at construction.
|
||||
with pytest.raises(ValueError, match="divisible by group_size"):
|
||||
RowParallelLinear(
|
||||
input_size,
|
||||
5120,
|
||||
bias=True,
|
||||
quant_config=config,
|
||||
prefix="blocks.0.ffn.fc_out",
|
||||
)
|
||||
|
||||
|
||||
@patch(_LOAD_JIT_KERNEL)
|
||||
def test_row_parallel_shard_divisible_by_group_is_quantized(_load, monkeypatch):
|
||||
from sglang.multimodal_gen.runtime.layers.linear import RowParallelLinear
|
||||
|
||||
_with_tp(monkeypatch, 4)
|
||||
config = _jit_config()
|
||||
layer = RowParallelLinear(
|
||||
12288, 3072, bias=True, quant_config=config, prefix="blocks.0.ffn.net.2"
|
||||
)
|
||||
assert isinstance(layer.quant_method, ConvRotInt8JitLinearMethod)
|
||||
assert config.selected == ["blocks.0.ffn.net.2"]
|
||||
|
||||
|
||||
@patch(_LOAD_JIT_KERNEL)
|
||||
def test_output_width_not_multiple_of_8_stays_bf16(_load):
|
||||
config = _jit_config()
|
||||
layer = LinearBase(input_size=4096, output_size=4)
|
||||
method = config.get_quant_method(layer, "blocks.0.attn.to_gate_logits")
|
||||
assert isinstance(method, UnquantizedLinearMethod)
|
||||
assert config.skipped == ["blocks.0.attn.to_gate_logits(out=4)"]
|
||||
|
||||
|
||||
@patch(_LOAD_JIT_KERNEL)
|
||||
def test_column_parallel_shard_output_not_multiple_of_8_stays_bf16(_load, monkeypatch):
|
||||
from sglang.multimodal_gen.runtime.layers.linear import ColumnParallelLinear
|
||||
|
||||
_with_tp(monkeypatch, 8)
|
||||
config = _jit_config()
|
||||
# 32 output rows split eight ways leave 4 per rank; the epilogue stores 8 at
|
||||
# a time, so the layer stays in BF16 instead of failing in the first forward.
|
||||
layer = ColumnParallelLinear(
|
||||
4096, 32, bias=True, quant_config=config, prefix="blocks.0.attn.to_gate_logits"
|
||||
)
|
||||
assert isinstance(layer.quant_method, UnquantizedLinearMethod)
|
||||
assert config.skipped == ["blocks.0.attn.to_gate_logits(out=4)"]
|
||||
|
||||
|
||||
@requires_kernel
|
||||
def test_fp16_activations_are_cast_and_returned_in_fp16():
|
||||
config = _jit_config()
|
||||
layer, weight = _quantized_layer(config, 3072, 3072, "attn.to_q", seed=3)
|
||||
x = torch.randn(2, 64, 3072, device="cuda", dtype=torch.float16)
|
||||
out, _ = layer(x)
|
||||
assert out.dtype == torch.float16
|
||||
ref = F.linear(x.to(torch.bfloat16), weight, layer.bias).to(torch.float16)
|
||||
err = torch.linalg.vector_norm(out.float() - ref.float())
|
||||
rel_l2 = (err / torch.linalg.vector_norm(ref.float())).item()
|
||||
assert rel_l2 < 2e-2, rel_l2
|
||||
|
||||
|
||||
@requires_kernel
|
||||
def test_fp16_parameters_are_cast_for_the_kernel_in_every_helper():
|
||||
"""FP16 parameters (--dit-precision fp16) must run through the plain apply
|
||||
and every fused helper; the out= helper must refuse an FP16 input."""
|
||||
config = _jit_config()
|
||||
layers = []
|
||||
weights = []
|
||||
for i, n in enumerate("qkv"):
|
||||
layer, weight = _quantized_layer(
|
||||
config, 3072, 3072, f"attn.to_{n}", seed=20 + i, dtype=torch.float16
|
||||
)
|
||||
layers.append(layer)
|
||||
weights.append(weight)
|
||||
assert layers[0].weight.dtype == torch.int8
|
||||
assert layers[0].bias.dtype == torch.bfloat16
|
||||
x = torch.randn(1, 128, 3072, device="cuda", dtype=torch.float16)
|
||||
|
||||
for layer, weight in zip(layers, weights, strict=True):
|
||||
out, _ = layer(x)
|
||||
assert out.dtype == torch.float16
|
||||
ref = F.linear(x.float(), weight.float(), layer.bias.float())
|
||||
err = torch.linalg.vector_norm(out.float() - ref)
|
||||
rel_l2 = (err / torch.linalg.vector_norm(ref)).item()
|
||||
assert rel_l2 < 2e-2, rel_l2
|
||||
|
||||
shared = apply_convrot_int8_shared_input(x=x, layers=layers)
|
||||
for out, layer in zip(shared, layers, strict=True):
|
||||
assert out.dtype == torch.float16
|
||||
assert torch.equal(out, layer(x)[0])
|
||||
|
||||
down, _ = _quantized_layer(
|
||||
config, 12288, 3072, "img_mlp.net.2", seed=23, dtype=torch.float16
|
||||
)
|
||||
up = torch.randn(1, 128, 12288, device="cuda", dtype=torch.float16)
|
||||
fused = apply_convrot_int8_gelu_input(layer=down, x=up)
|
||||
assert fused.dtype == torch.float16
|
||||
# The GELU runs on the BF16-cast input inside the kernel, so the eager
|
||||
# reference is the layer on GELU of that cast, not GELU in FP16.
|
||||
eager, _ = down(F.gelu(up.to(torch.bfloat16), approximate="tanh"))
|
||||
assert torch.equal(fused, eager.to(torch.float16))
|
||||
|
||||
outs = [
|
||||
torch.empty(1, 128, 3072, device="cuda", dtype=torch.bfloat16) for _ in layers
|
||||
]
|
||||
with pytest.raises(ValueError, match="must be BF16"):
|
||||
apply_convrot_int8_shared_input_out(x=x, layers=layers, outs=outs)
|
||||
x_bf16 = x.to(torch.bfloat16)
|
||||
apply_convrot_int8_shared_input_out(x=x_bf16, layers=layers, outs=outs)
|
||||
for out, layer in zip(outs, layers, strict=True):
|
||||
assert torch.equal(out, layer(x_bf16)[0])
|
||||
|
||||
|
||||
@requires_kernel
|
||||
def test_lora_merge_mode_is_redirected_to_dynamic_on_int8_base():
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.lora.pipeline import LoRAPipeline
|
||||
|
||||
config = _jit_config()
|
||||
layer, weight = _quantized_layer(
|
||||
config, 3072, 3072, "transformer_blocks.0.attn.to_q", seed=4
|
||||
)
|
||||
lora = wrap_with_lora_layer(layer, lora_rank=16, lora_alpha=16, snapshot_base=True)
|
||||
layers = {"transformer_blocks.0.attn.to_q": lora}
|
||||
# The rotated, per-channel scaled INT8 matrix cannot take a merged delta.
|
||||
assert lora.can_merge_base_weight is False
|
||||
# LoRAPipeline is abstract; the decision only needs its static helper.
|
||||
stand_in = SimpleNamespace(_uses_dtensor_weights=LoRAPipeline._uses_dtensor_weights)
|
||||
|
||||
def decide(module_name, layers, mode):
|
||||
return LoRAPipeline._should_merge_lora_for_layers(
|
||||
stand_in, module_name, layers, mode
|
||||
)
|
||||
|
||||
assert decide("transformer_blocks.0.attn.to_q", layers, "auto") is False
|
||||
with pytest.raises(ValueError, match="use merge mode 'dynamic'"):
|
||||
decide("transformer_blocks.0.attn.to_q", layers, "merge")
|
||||
|
||||
gen = torch.Generator(device="cuda").manual_seed(4)
|
||||
lora_a = (torch.randn(16, 3072, device="cuda", generator=gen) * 0.05).to(
|
||||
torch.bfloat16
|
||||
)
|
||||
lora_b = (torch.randn(3072, 16, device="cuda", generator=gen) * 0.05).to(
|
||||
torch.bfloat16
|
||||
)
|
||||
lora.set_lora_weights(
|
||||
lora_a, lora_b, lora_path="test", strength=1.0, merge_weights=False
|
||||
)
|
||||
x = torch.randn(8, 3072, device="cuda", dtype=torch.bfloat16, generator=gen)
|
||||
out, _ = lora(x)
|
||||
delta = (lora_b.float() @ lora_a.float()).to(torch.bfloat16)
|
||||
ref = F.linear(x, weight + delta, layer.bias)
|
||||
err = torch.linalg.vector_norm(out.float() - ref.float())
|
||||
rel_l2 = (err / torch.linalg.vector_norm(ref.float())).item()
|
||||
assert rel_l2 < 2e-2, rel_l2
|
||||
|
||||
|
||||
_JIT_AVAILABLE = f"{_CONFIG_MODULE}._jit_available"
|
||||
_COMFY_AVAILABLE = f"{_CONFIG_MODULE}._comfy_kitchen_available"
|
||||
|
||||
|
||||
@patch(_LOAD_COMFY_KITCHEN)
|
||||
@patch(_LOAD_JIT_KERNEL)
|
||||
def test_auto_backend_prefers_jit_and_falls_back_to_comfy(_jit, _comfy):
|
||||
"""``auto`` must pick the JIT ops where they run and comfy_kitchen
|
||||
otherwise; the choice is per config, not per import."""
|
||||
with (
|
||||
patch(_JIT_AVAILABLE, return_value=True),
|
||||
patch(_COMFY_AVAILABLE, return_value=True),
|
||||
):
|
||||
config = ConvRotInt8Config()
|
||||
assert config.resolve_backend() == "jit"
|
||||
method = config.get_quant_method(LinearBase(3072, 3072), "blocks.0.attn.to_q")
|
||||
assert isinstance(method, ConvRotInt8JitLinearMethod)
|
||||
assert config.selected_by_backend["jit"] == ["blocks.0.attn.to_q"]
|
||||
with (
|
||||
patch(_JIT_AVAILABLE, return_value=False),
|
||||
patch(_COMFY_AVAILABLE, return_value=True),
|
||||
):
|
||||
config = ConvRotInt8Config()
|
||||
assert config.resolve_backend() == "comfy_kitchen"
|
||||
method = config.get_quant_method(LinearBase(3072, 3072), "blocks.0.attn.to_q")
|
||||
assert isinstance(method, ConvRotInt8ComfyKitchenLinearMethod)
|
||||
assert config.selected_by_backend["comfy_kitchen"] == ["blocks.0.attn.to_q"]
|
||||
|
||||
|
||||
@patch(_LOAD_COMFY_KITCHEN)
|
||||
@patch(_LOAD_JIT_KERNEL)
|
||||
def test_auto_backend_serves_narrow_outputs_with_comfy_or_leaves_bf16(_jit, _comfy):
|
||||
"""The JIT epilogue stores 8 outputs at a time; a narrower layer goes
|
||||
to comfy_kitchen when it is installed and stays BF16 otherwise."""
|
||||
with patch(_JIT_AVAILABLE, return_value=True):
|
||||
with patch(_COMFY_AVAILABLE, return_value=True):
|
||||
config = ConvRotInt8Config()
|
||||
method = config.get_quant_method(LinearBase(4096, 4), "blocks.0.gate")
|
||||
assert isinstance(method, ConvRotInt8ComfyKitchenLinearMethod)
|
||||
assert config.selected_by_backend["comfy_kitchen"] == ["blocks.0.gate"]
|
||||
with patch(_COMFY_AVAILABLE, return_value=False):
|
||||
config = ConvRotInt8Config()
|
||||
method = config.get_quant_method(LinearBase(4096, 4), "blocks.0.gate")
|
||||
assert isinstance(method, UnquantizedLinearMethod)
|
||||
assert config.skipped == ["blocks.0.gate(out=4)"]
|
||||
|
||||
|
||||
@patch(_LOAD_COMFY_KITCHEN)
|
||||
@patch(_LOAD_JIT_KERNEL)
|
||||
def test_serialized_layers_pick_the_backend_per_marker_group_size(_jit, _comfy):
|
||||
"""A Comfy checkpoint may mix group sizes; 16 exists only in comfy_kitchen
|
||||
while 256 runs on the JIT ops with the stored [N, 1] scale."""
|
||||
markers = {
|
||||
"visual.proj": {
|
||||
"format": "int8_tensorwise",
|
||||
"convrot": True,
|
||||
"convrot_groupsize": 16,
|
||||
},
|
||||
"blocks.0.fc1": {
|
||||
"format": "int8_tensorwise",
|
||||
"convrot": True,
|
||||
"convrot_groupsize": 256,
|
||||
},
|
||||
}
|
||||
with (
|
||||
patch(_JIT_AVAILABLE, return_value=True),
|
||||
patch(_COMFY_AVAILABLE, return_value=True),
|
||||
):
|
||||
config = ConvRotInt8Config(layer_markers=markers)
|
||||
small = ReplicatedLinear(
|
||||
64, 16, bias=False, quant_config=config, prefix="visual.proj"
|
||||
)
|
||||
fc1 = ReplicatedLinear(
|
||||
256, 16, bias=False, quant_config=config, prefix="blocks.0.fc1"
|
||||
)
|
||||
assert isinstance(small.quant_method, ConvRotInt8ComfyKitchenLinearMethod)
|
||||
assert isinstance(fc1.quant_method, ConvRotInt8JitLinearMethod)
|
||||
assert fc1.quant_method.is_checkpoint_serialized
|
||||
assert fc1.weight.dtype == torch.int8 and fc1.weight.shape == (16, 256)
|
||||
assert fc1.weight_scale.dtype == torch.float32
|
||||
assert fc1.weight_scale.shape == (16, 1)
|
||||
assert config.selected == ["visual.proj", "blocks.0.fc1"]
|
||||
|
||||
|
||||
def test_explicit_backend_rejects_group_sizes_it_has_no_kernel_for():
|
||||
with pytest.raises(ValueError, match="group sizes"):
|
||||
ConvRotInt8Config(group_size=16, backend="jit")
|
||||
with pytest.raises(ValueError, match="group sizes"):
|
||||
ConvRotInt8Config(group_size=128, backend="comfy_kitchen")
|
||||
with pytest.raises(ValueError, match="backend must be one of"):
|
||||
ConvRotInt8Config(backend="triton")
|
||||
|
||||
|
||||
def test_require_comfy_kitchen_pins_auto_and_refuses_explicit_jit():
|
||||
with (
|
||||
patch(_JIT_AVAILABLE, return_value=True),
|
||||
patch(_COMFY_AVAILABLE, return_value=True),
|
||||
):
|
||||
config = ConvRotInt8Config()
|
||||
config.require_comfy_kitchen("is not validated here")
|
||||
assert config.resolve_backend() == "comfy_kitchen"
|
||||
with pytest.raises(ValueError, match="jit is not validated here"):
|
||||
ConvRotInt8Config(backend="jit").require_comfy_kitchen(
|
||||
"is not validated here"
|
||||
)
|
||||
|
||||
|
||||
@requires_kernel
|
||||
def test_serialized_layer_on_jit_matches_online_quantization_bitwise():
|
||||
"""Loading the online path's INT8 weight and fp32 [N, 1] row scale through
|
||||
the serialized method must reproduce the online layer bit for bit."""
|
||||
online, _ = _quantized_layer(_jit_config(), 3072, 3072, "attn.to_q", seed=30)
|
||||
markers = {
|
||||
"attn.to_q": {
|
||||
"format": "int8_tensorwise",
|
||||
"convrot": True,
|
||||
"convrot_groupsize": 256,
|
||||
}
|
||||
}
|
||||
config = ConvRotInt8Config(layer_markers=markers, backend="jit")
|
||||
serialized = ReplicatedLinear(
|
||||
3072, 3072, params_dtype=torch.bfloat16, quant_config=config, prefix="attn.to_q"
|
||||
).cuda()
|
||||
assert isinstance(serialized.quant_method, ConvRotInt8JitLinearMethod)
|
||||
with torch.no_grad():
|
||||
serialized.weight.copy_(online.weight)
|
||||
serialized.weight_scale.copy_(online.weight_scale.view(-1, 1))
|
||||
serialized.bias.copy_(online.bias)
|
||||
serialized.quant_method.process_weights_after_loading(serialized)
|
||||
|
||||
x = torch.randn(2, 512, 3072, device="cuda", dtype=torch.bfloat16)
|
||||
assert torch.equal(serialized(x)[0], online(x)[0])
|
||||
assert convrot_int8_shares_input([serialized, online])
|
||||
|
||||
|
||||
def test_auto_logs_why_the_jit_backend_is_unavailable(caplog):
|
||||
"""A refused GPU (CC 10.3) must still say why once the method is convrot_int8
|
||||
with auto backend, not only when jit is requested explicitly."""
|
||||
import logging
|
||||
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.configs import (
|
||||
convrot_int8_config,
|
||||
)
|
||||
|
||||
convrot_int8_config._log_jit_fallback.cache_clear()
|
||||
reason = (
|
||||
"CC 10.3 is not supported: Blackwell Ultra cuts INT8 tensor-core throughput"
|
||||
)
|
||||
with (
|
||||
patch(
|
||||
"sglang.multimodal_gen.runtime.layers.quantization.convrot_int8_jit."
|
||||
"jit_convrot_unavailable_reason",
|
||||
return_value=reason,
|
||||
),
|
||||
patch(_COMFY_AVAILABLE, return_value=True),
|
||||
caplog.at_level(logging.INFO),
|
||||
):
|
||||
config = ConvRotInt8Config()
|
||||
assert config.resolve_backend() == "comfy_kitchen"
|
||||
assert config.resolve_backend() == "comfy_kitchen"
|
||||
messages = [
|
||||
r.getMessage()
|
||||
for r in caplog.records
|
||||
if "JIT backend unavailable" in r.getMessage()
|
||||
]
|
||||
assert len(messages) == 1 and reason in messages[0]
|
||||
|
||||
|
||||
@patch(_LOAD_JIT_KERNEL)
|
||||
def test_shared_input_requires_one_group_size(_jit):
|
||||
"""One rotated activation serves several layers only at one group width; a
|
||||
serialized checkpoint may mix 64- and 256-group layers."""
|
||||
markers = {
|
||||
"a": {"format": "int8_tensorwise", "convrot": True, "convrot_groupsize": 256},
|
||||
"b": {"format": "int8_tensorwise", "convrot": True, "convrot_groupsize": 64},
|
||||
"c": {"format": "int8_tensorwise", "convrot": True, "convrot_groupsize": 256},
|
||||
}
|
||||
config = ConvRotInt8Config(layer_markers=markers, backend="jit")
|
||||
a, b, c = (
|
||||
ReplicatedLinear(256, 16, bias=False, quant_config=config, prefix=name)
|
||||
for name in "abc"
|
||||
)
|
||||
assert convrot_int8_shares_input([a, c])
|
||||
assert not convrot_int8_shares_input([a, b])
|
||||
|
||||
|
||||
def test_every_no_kernel_branch_raises_instead_of_falling_back():
|
||||
"""A serialized INT8 layer that no backend serves must not land in a BF16
|
||||
parameter, and an impossible auto config must fail at construction."""
|
||||
with (
|
||||
patch(_JIT_AVAILABLE, return_value=False),
|
||||
patch(_COMFY_AVAILABLE, return_value=True),
|
||||
):
|
||||
with pytest.raises(ValueError, match="no backend on this machine"):
|
||||
ConvRotInt8Config(group_size=128)
|
||||
with (
|
||||
patch(_JIT_AVAILABLE, return_value=True),
|
||||
patch(_COMFY_AVAILABLE, return_value=False),
|
||||
):
|
||||
with pytest.raises(ValueError, match="no kernel for group size 128"):
|
||||
ConvRotInt8Config(group_size=128).require_comfy_kitchen("is pinned")
|
||||
markers = {
|
||||
"gate": {
|
||||
"format": "int8_tensorwise",
|
||||
"convrot": True,
|
||||
"convrot_groupsize": 256,
|
||||
}
|
||||
}
|
||||
config = ConvRotInt8Config(layer_markers=markers)
|
||||
with pytest.raises(ValueError, match="pip install comfy-kitchen"):
|
||||
config.get_quant_method(LinearBase(256, 4), "gate")
|
||||
|
||||
|
||||
@patch(_LOAD_COMFY_KITCHEN)
|
||||
@patch(_LOAD_JIT_KERNEL)
|
||||
def test_serialized_layers_log_the_backend_split_once_loaded(_jit, _comfy, caplog):
|
||||
"""Serialized layers skip online quantization, so the backend split must be
|
||||
reported when their weights are in place instead."""
|
||||
import logging
|
||||
|
||||
markers = {
|
||||
"fc1": {"format": "int8_tensorwise", "convrot": True, "convrot_groupsize": 256},
|
||||
"small": {
|
||||
"format": "int8_tensorwise",
|
||||
"convrot": True,
|
||||
"convrot_groupsize": 16,
|
||||
},
|
||||
}
|
||||
with (
|
||||
patch(_JIT_AVAILABLE, return_value=True),
|
||||
patch(_COMFY_AVAILABLE, return_value=True),
|
||||
caplog.at_level(logging.INFO),
|
||||
):
|
||||
config = ConvRotInt8Config(layer_markers=markers)
|
||||
fc1 = ReplicatedLinear(256, 16, bias=False, quant_config=config, prefix="fc1")
|
||||
small = ReplicatedLinear(
|
||||
64, 16, bias=False, quant_config=config, prefix="small"
|
||||
)
|
||||
fc1.quant_method.process_weights_after_loading(fc1)
|
||||
assert not [r for r in caplog.records if "serialized INT8" in r.getMessage()]
|
||||
small.quant_method.process_weights_after_loading(small)
|
||||
lines = [
|
||||
r.getMessage() for r in caplog.records if "serialized INT8" in r.getMessage()
|
||||
]
|
||||
assert lines == [
|
||||
"convrot_int8: loaded 2 serialized INT8 linear layers (jit 1, comfy_kitchen 1)"
|
||||
]
|
||||
|
||||
|
||||
def test_kitchen_int8_is_a_deprecated_alias_of_convrot_int8(caplog):
|
||||
"""Existing `--quantization kitchen_int8` commands must keep working under the
|
||||
new name, with one deprecation warning per process."""
|
||||
import logging
|
||||
|
||||
from sglang.multimodal_gen.runtime.layers.quantization import (
|
||||
get_quantization_config,
|
||||
method_names,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.utils.logging_utils import _print_warning_once
|
||||
|
||||
_print_warning_once.cache_clear()
|
||||
with caplog.at_level(logging.WARNING):
|
||||
assert (
|
||||
method_names.canonical_quantization_method("kitchen_int8") == "convrot_int8"
|
||||
)
|
||||
assert (
|
||||
method_names.canonical_quantization_method("kitchen_int8") == "convrot_int8"
|
||||
)
|
||||
assert method_names.canonical_quantization_method("fp8") == "fp8"
|
||||
warnings = [r for r in caplog.records if "deprecated alias" in r.getMessage()]
|
||||
assert len(warnings) == 1
|
||||
assert get_quantization_config("kitchen_int8") is ConvRotInt8Config
|
||||
assert ConvRotInt8Config.get_name() == "convrot_int8"
|
||||
@@ -0,0 +1,480 @@
|
||||
"""Unit tests for Qwen-Image joint text-image Q/K/V destination buffers."""
|
||||
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from sglang.kernels.ops.gemm.convrot_int8 import convrot_int8_supported_capabilities
|
||||
from sglang.multimodal_gen.runtime.distributed.sp_shard_utils import join_seqs
|
||||
from sglang.multimodal_gen.runtime.layers.layernorm import RMSNorm
|
||||
from sglang.multimodal_gen.runtime.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
apply_unquantized_linear,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.lora.linear import BaseLayerWithLoRA
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config import (
|
||||
ConvRotInt8Config,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.dits.qwen_image import (
|
||||
QwenImageCrossAttention,
|
||||
_joint_qkv_head_views,
|
||||
_project_qkv_into_joint_buffers,
|
||||
_sync_into_buffer,
|
||||
_use_joint_qkv_buffers,
|
||||
)
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
DIM = 32
|
||||
# ConvRot rotates 256-wide input groups.
|
||||
CONVROT_DIM = 256
|
||||
NUM_HEADS = 2
|
||||
HEAD_DIM = 8
|
||||
INNER_DIM = NUM_HEADS * HEAD_DIM
|
||||
SEQ_TXT = 5
|
||||
SEQ_IMG = 13
|
||||
QWEN_IMAGE = "sglang.multimodal_gen.runtime.models.dits.qwen_image"
|
||||
SP_WORLD_SIZE = f"{QWEN_IMAGE}.get_sp_world_size"
|
||||
PROJECT_INTO_JOINT_BUFFERS = f"{QWEN_IMAGE}._project_qkv_into_joint_buffers"
|
||||
FUSED_INPLACE_QKNORM = (
|
||||
"sglang.multimodal_gen.runtime.layers.layernorm.can_use_fused_inplace_qknorm"
|
||||
)
|
||||
# ColumnParallelLinear reads only the group's size and rank.
|
||||
_TP_GROUP = SimpleNamespace(world_size=1, rank_in_group=0)
|
||||
|
||||
|
||||
# Capability only: on a supported GPU a missing toolchain or a failed build must
|
||||
# fail these tests, not skip them.
|
||||
requires_convrot_kernel = unittest.skipUnless(
|
||||
torch.cuda.is_available()
|
||||
and torch.cuda.get_device_capability() in convrot_int8_supported_capabilities(),
|
||||
"needs a GPU in the convrot_int8 capability table",
|
||||
)
|
||||
|
||||
|
||||
def _linear(
|
||||
*,
|
||||
bias: bool = True,
|
||||
dim: int = DIM,
|
||||
dtype: torch.dtype = torch.bfloat16,
|
||||
quant_config: ConvRotInt8Config | None = None,
|
||||
prefix: str = "",
|
||||
) -> ColumnParallelLinear:
|
||||
linear = ColumnParallelLinear(
|
||||
dim,
|
||||
INNER_DIM,
|
||||
bias=bias,
|
||||
gather_output=False,
|
||||
params_dtype=dtype,
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
tp_group=_TP_GROUP,
|
||||
)
|
||||
with torch.no_grad():
|
||||
linear.weight.normal_()
|
||||
if bias:
|
||||
linear.bias.normal_()
|
||||
linear = linear.cuda()
|
||||
if quant_config is not None:
|
||||
linear.quant_method.process_weights_after_loading(linear)
|
||||
return linear
|
||||
|
||||
|
||||
def _passthrough(x: torch.Tensor) -> tuple[torch.Tensor, None]:
|
||||
return x, None
|
||||
|
||||
|
||||
class _RecordingAttention:
|
||||
# Stands in for USPAttention: keeps the Q/K/V it was handed and returns a
|
||||
# deterministic function of them in the [B, S, H, D] layout.
|
||||
def __init__(self) -> None:
|
||||
self.calls: list[tuple[torch.Tensor, torch.Tensor, torch.Tensor, dict]] = []
|
||||
|
||||
def __call__(
|
||||
self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, **kwargs
|
||||
) -> torch.Tensor:
|
||||
self.calls.append((q.clone(), k.clone(), v.clone(), kwargs))
|
||||
if kwargs["q_prefix"] is not None:
|
||||
q = torch.cat([kwargs["q_prefix"], q], dim=1)
|
||||
k = torch.cat([kwargs["k_prefix"], k], dim=1)
|
||||
v = torch.cat([kwargs["v_prefix"], v], dim=1)
|
||||
return q + k + v
|
||||
|
||||
|
||||
def _attention(
|
||||
*,
|
||||
bias: bool = True,
|
||||
dtype: torch.dtype = torch.bfloat16,
|
||||
quant_config: ConvRotInt8Config | None = None,
|
||||
) -> QwenImageCrossAttention:
|
||||
dim = DIM if quant_config is None else CONVROT_DIM
|
||||
attn = object.__new__(QwenImageCrossAttention)
|
||||
nn.Module.__init__(attn)
|
||||
attn.head_dim = HEAD_DIM
|
||||
attn.local_num_heads = NUM_HEADS
|
||||
attn.added_kv_proj_dim = dim
|
||||
attn.qk_norm = True
|
||||
attn.use_fused_qkv = False
|
||||
attn.use_fused_qkv_epilogue = False
|
||||
attn.use_fused_added_qkv = False
|
||||
attn._unquantized_added_qkv_is_packed = False
|
||||
attn.separate_unquantized_qkv_proj = quant_config is None
|
||||
attn.separate_convrot_qkv_proj = quant_config is not None
|
||||
for name in ("to_q", "to_k", "to_v", "add_q_proj", "add_k_proj", "add_v_proj"):
|
||||
setattr(
|
||||
attn,
|
||||
name,
|
||||
_linear(
|
||||
bias=bias,
|
||||
dim=dim,
|
||||
dtype=dtype,
|
||||
quant_config=quant_config,
|
||||
prefix=f"transformer_blocks.0.attn.{name}",
|
||||
),
|
||||
)
|
||||
for name in ("norm_q", "norm_k", "norm_added_q", "norm_added_k"):
|
||||
norm = RMSNorm(HEAD_DIM, eps=1e-6).to(device="cuda", dtype=dtype)
|
||||
with torch.no_grad():
|
||||
norm.weight.normal_()
|
||||
setattr(attn, name, norm)
|
||||
attn.to_out = [_passthrough]
|
||||
attn.to_add_out = _passthrough
|
||||
attn.attn = _RecordingAttention()
|
||||
return attn
|
||||
|
||||
|
||||
def _streams(
|
||||
*, dim: int = DIM, dtype: torch.dtype = torch.bfloat16
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
hidden = torch.randn(1, SEQ_IMG, dim, device="cuda", dtype=dtype)
|
||||
encoder = torch.randn(1, SEQ_TXT, dim, device="cuda", dtype=dtype)
|
||||
return hidden, encoder
|
||||
|
||||
|
||||
def _joint_flag(attn: QwenImageCrossAttention) -> str:
|
||||
return (
|
||||
"separate_convrot_qkv_proj"
|
||||
if attn.separate_convrot_qkv_proj
|
||||
else "separate_unquantized_qkv_proj"
|
||||
)
|
||||
|
||||
|
||||
class _JointQkvCase(CustomTestCase):
|
||||
# Forward helpers shared by the unquantized and the ConvRot test classes.
|
||||
|
||||
def setUp(self) -> None:
|
||||
torch.manual_seed(20260902)
|
||||
# The runtime forwards under no_grad; the out= GEMMs of the joint path
|
||||
# reject grad-tracking operands.
|
||||
grad_was_enabled = torch.is_grad_enabled()
|
||||
torch.set_grad_enabled(False)
|
||||
self.addCleanup(torch.set_grad_enabled, grad_was_enabled)
|
||||
|
||||
def _forward(
|
||||
self,
|
||||
attn: QwenImageCrossAttention,
|
||||
*,
|
||||
dtype: torch.dtype = torch.bfloat16,
|
||||
**kwargs,
|
||||
):
|
||||
hidden, encoder = _streams(dim=attn.added_kv_proj_dim, dtype=dtype)
|
||||
with patch(SP_WORLD_SIZE, return_value=1):
|
||||
return attn.forward(
|
||||
hidden_states=hidden,
|
||||
encoder_hidden_states=encoder,
|
||||
image_rotary_emb=None,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
def _assert_joint_path_matches_join_seqs_path(
|
||||
self, attn: QwenImageCrossAttention, *, dtype: torch.dtype = torch.bfloat16
|
||||
) -> None:
|
||||
flag = _joint_flag(attn)
|
||||
outputs = []
|
||||
for joint in (True, False):
|
||||
torch.manual_seed(20260902)
|
||||
setattr(attn, flag, joint)
|
||||
attn.attn.calls.clear()
|
||||
with patch(
|
||||
PROJECT_INTO_JOINT_BUFFERS, wraps=_project_qkv_into_joint_buffers
|
||||
) as project:
|
||||
out = self._forward(attn, dtype=dtype)
|
||||
self.assertEqual(project.call_count, 1 if joint else 0)
|
||||
outputs.append((out, attn.attn.calls[0]))
|
||||
(joint_out, joint_call), (ref_out, ref_call) = outputs
|
||||
|
||||
self.assertIsNone(joint_call[3]["q_prefix"])
|
||||
for joint_qkv, ref_qkv in zip(joint_call[:3], ref_call[:3]):
|
||||
self.assertEqual(
|
||||
joint_qkv.shape, (1, SEQ_TXT + SEQ_IMG, NUM_HEADS, HEAD_DIM)
|
||||
)
|
||||
self.assertTrue(torch.equal(joint_qkv, ref_qkv))
|
||||
self.assertTrue(torch.equal(joint_out[0], ref_out[0]))
|
||||
self.assertTrue(torch.equal(joint_out[1], ref_out[1]))
|
||||
|
||||
def _assert_compiled_forward_matches_eager(self, attn) -> None:
|
||||
with patch(FUSED_INPLACE_QKNORM, return_value=False):
|
||||
torch.manual_seed(20260902)
|
||||
eager_out = self._forward(attn)
|
||||
torch.manual_seed(20260902)
|
||||
compiled = torch.compile(attn, dynamic=False)
|
||||
hidden, encoder = _streams(dim=attn.added_kv_proj_dim, dtype=torch.bfloat16)
|
||||
with patch(SP_WORLD_SIZE, return_value=1):
|
||||
compiled_out = compiled(
|
||||
hidden_states=hidden,
|
||||
encoder_hidden_states=encoder,
|
||||
image_rotary_emb=None,
|
||||
)
|
||||
torch.testing.assert_close(compiled_out[0], eager_out[0], rtol=2e-2, atol=2e-2)
|
||||
torch.testing.assert_close(compiled_out[1], eager_out[1], rtol=2e-2, atol=2e-2)
|
||||
|
||||
def _assert_forward_bypasses_joint_buffers(
|
||||
self, attn: QwenImageCrossAttention, *, dtype: torch.dtype
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
# The out= kernels reject these operand dtypes, so the joint path must
|
||||
# not be entered; the flag-off forward is the same code the fallback runs.
|
||||
with patch(
|
||||
PROJECT_INTO_JOINT_BUFFERS, wraps=_project_qkv_into_joint_buffers
|
||||
) as project:
|
||||
out = self._forward(attn, dtype=dtype)
|
||||
project.assert_not_called()
|
||||
return out
|
||||
|
||||
|
||||
@unittest.skipUnless(torch.cuda.is_available(), "CUDA required")
|
||||
class TestQwenImageJointQkvBuffers(_JointQkvCase):
|
||||
def test_forward_is_bitwise_identical_to_join_seqs_path(self):
|
||||
self._assert_joint_path_matches_join_seqs_path(_attention())
|
||||
|
||||
def test_forward_copies_back_when_qk_norm_returns_fresh_tensors(self):
|
||||
"""The non-fused QK-norm returns new tensors instead of writing the
|
||||
buffer views in place; their values must still reach the joint buffers."""
|
||||
with patch(FUSED_INPLACE_QKNORM, return_value=False):
|
||||
self._assert_joint_path_matches_join_seqs_path(_attention())
|
||||
|
||||
def test_compiled_forward_matches_eager_when_qk_norm_returns_fresh_tensors(self):
|
||||
"""With the in-place QK-norm off, the joint buffers are filled by the
|
||||
copy-back; that path must compile and agree with eager."""
|
||||
self._assert_compiled_forward_matches_eager(_attention())
|
||||
|
||||
def test_fp16_parameters_and_streams_take_the_joint_path(self):
|
||||
"""--dit-precision fp16 keeps every operand FP16, which the out= GEMMs
|
||||
accept like BF16."""
|
||||
self._assert_joint_path_matches_join_seqs_path(
|
||||
_attention(dtype=torch.float16), dtype=torch.float16
|
||||
)
|
||||
|
||||
def test_fp32_streams_under_bf16_autocast_bypass_joint_buffers(self):
|
||||
"""FP32 streams under BF16 autocast must take the join_seqs path; the
|
||||
out= GEMMs of the joint path do not autocast."""
|
||||
attn = _attention()
|
||||
with torch.autocast("cuda", dtype=torch.bfloat16):
|
||||
self._assert_forward_bypasses_joint_buffers(attn, dtype=torch.float32)
|
||||
|
||||
def test_masked_forward_bypasses_joint_buffers(self):
|
||||
"""A masked sequence hands text Q/K/V to attention as a prefix, so the
|
||||
joint buffers must not be built for it."""
|
||||
attn = _attention()
|
||||
mask = torch.ones(1, SEQ_TXT, dtype=torch.bool, device="cuda")
|
||||
with patch(PROJECT_INTO_JOINT_BUFFERS) as project:
|
||||
self._forward(attn, encoder_hidden_states_mask=mask)
|
||||
project.assert_not_called()
|
||||
self.assertIsNotNone(attn.attn.calls[0][3]["q_prefix"])
|
||||
|
||||
def test_stale_convrot_flag_falls_through_to_per_layer_projections(self):
|
||||
"""LoRA mounting swaps the projections after construction, so the
|
||||
ConvRot flag alone must not select the shared-input helper."""
|
||||
attn = _attention()
|
||||
attn.separate_unquantized_qkv_proj = False
|
||||
mask = torch.ones(1, SEQ_TXT, dtype=torch.bool, device="cuda")
|
||||
outputs = []
|
||||
for convrot in (False, True):
|
||||
torch.manual_seed(20260902)
|
||||
attn.separate_convrot_qkv_proj = convrot
|
||||
outputs.append(self._forward(attn, encoder_hidden_states_mask=mask))
|
||||
self.assertTrue(torch.equal(outputs[0][0], outputs[1][0]))
|
||||
self.assertTrue(torch.equal(outputs[0][1], outputs[1][1]))
|
||||
|
||||
def test_joint_projection_without_bias_matches_reference(self):
|
||||
attn = _attention(bias=False)
|
||||
hidden, encoder = _streams()
|
||||
bufs = _project_qkv_into_joint_buffers(
|
||||
attn=attn,
|
||||
hidden_states=hidden,
|
||||
encoder_hidden_states=encoder,
|
||||
seq_len_txt=SEQ_TXT,
|
||||
)
|
||||
img_layers = (attn.to_q, attn.to_k, attn.to_v)
|
||||
txt_layers = (attn.add_q_proj, attn.add_k_proj, attn.add_v_proj)
|
||||
for buf, img_layer, txt_layer in zip(bufs, img_layers, txt_layers):
|
||||
expected = join_seqs(
|
||||
apply_unquantized_linear(encoder, txt_layer.weight, None),
|
||||
apply_unquantized_linear(hidden, img_layer.weight, None),
|
||||
0,
|
||||
)
|
||||
self.assertTrue(torch.equal(buf, expected))
|
||||
|
||||
def test_head_views_alias_the_joint_buffers(self):
|
||||
hidden, encoder = _streams()
|
||||
bufs = _project_qkv_into_joint_buffers(
|
||||
attn=_attention(),
|
||||
hidden_states=hidden,
|
||||
encoder_hidden_states=encoder,
|
||||
seq_len_txt=SEQ_TXT,
|
||||
)
|
||||
views = _joint_qkv_head_views(
|
||||
bufs, seq_len_txt=SEQ_TXT, num_heads=NUM_HEADS, head_dim=HEAD_DIM
|
||||
)
|
||||
img_query, txt_query = views[0], views[3]
|
||||
|
||||
self.assertEqual(img_query.shape, (1, SEQ_IMG, NUM_HEADS, HEAD_DIM))
|
||||
self.assertEqual(txt_query.shape, (1, SEQ_TXT, NUM_HEADS, HEAD_DIM))
|
||||
self.assertTrue(img_query.is_contiguous())
|
||||
self.assertTrue(txt_query.is_contiguous())
|
||||
# In-place QK-norm/RoPE on the views must land in the joint buffer.
|
||||
img_query.zero_()
|
||||
txt_query.fill_(1.0)
|
||||
self.assertTrue(torch.all(bufs[0][:, SEQ_TXT:] == 0))
|
||||
self.assertTrue(torch.all(bufs[0][:, :SEQ_TXT] == 1))
|
||||
|
||||
def test_eligibility_rejects_every_unsupported_layout(self):
|
||||
attn = _attention()
|
||||
hidden, encoder = _streams()
|
||||
|
||||
def eligible(h=hidden, e=encoder, *, masked=False, sp=1, attention=attn):
|
||||
with patch(SP_WORLD_SIZE, return_value=sp):
|
||||
return _use_joint_qkv_buffers(
|
||||
attn=attention,
|
||||
hidden_states=h,
|
||||
encoder_hidden_states=e,
|
||||
masked=masked,
|
||||
)
|
||||
|
||||
self.assertTrue(eligible())
|
||||
self.assertFalse(eligible(masked=True))
|
||||
self.assertFalse(eligible(sp=2))
|
||||
self.assertFalse(eligible(h=hidden.expand(2, -1, -1).contiguous()))
|
||||
strided_hidden = hidden.transpose(1, 2).contiguous().transpose(1, 2)
|
||||
strided_encoder = encoder.transpose(1, 2).contiguous().transpose(1, 2)
|
||||
self.assertFalse(eligible(h=strided_hidden))
|
||||
self.assertFalse(eligible(e=strided_encoder))
|
||||
packed = _attention()
|
||||
packed.separate_unquantized_qkv_proj = False
|
||||
self.assertFalse(eligible(attention=packed))
|
||||
# An unmerged LoRA adds its delta in forward; the bare addmm would drop it.
|
||||
lora_wrapped = _attention()
|
||||
lora_wrapped.to_q = BaseLayerWithLoRA(lora_wrapped.to_q)
|
||||
self.assertFalse(eligible(attention=lora_wrapped))
|
||||
|
||||
def test_eligibility_requires_one_dtype_across_streams_weights_and_autocast(
|
||||
self,
|
||||
):
|
||||
"""The out= GEMMs neither promote nor autocast, so they stand in for
|
||||
F.linear only when every operand already has the dtype it would use."""
|
||||
attn = _attention()
|
||||
hidden, encoder = _streams()
|
||||
|
||||
def eligible(h=hidden, e=encoder, *, attention=attn):
|
||||
with patch(SP_WORLD_SIZE, return_value=1):
|
||||
return _use_joint_qkv_buffers(
|
||||
attn=attention,
|
||||
hidden_states=h,
|
||||
encoder_hidden_states=e,
|
||||
masked=False,
|
||||
)
|
||||
|
||||
self.assertFalse(eligible(h=hidden.float(), e=encoder.float()))
|
||||
self.assertFalse(eligible(e=encoder.float()))
|
||||
fp32_bias = _attention()
|
||||
fp32_bias.to_k.bias.data = fp32_bias.to_k.bias.data.float()
|
||||
self.assertFalse(eligible(attention=fp32_bias))
|
||||
with torch.autocast("cuda", dtype=torch.float16):
|
||||
self.assertFalse(eligible())
|
||||
with torch.autocast("cuda", dtype=torch.bfloat16):
|
||||
self.assertTrue(eligible())
|
||||
self.assertFalse(eligible(h=hidden.float(), e=encoder.float()))
|
||||
self.assertTrue(eligible())
|
||||
fp16 = _attention(dtype=torch.float16)
|
||||
h16, e16 = _streams(dtype=torch.float16)
|
||||
self.assertTrue(eligible(h=h16, e=e16, attention=fp16))
|
||||
|
||||
|
||||
class TestSyncIntoBufferUnderCompile(CustomTestCase):
|
||||
def test_out_of_place_sync_compiles_without_graph_breaks(self):
|
||||
"""Tracing the copy-back must produce one graph: no storage-pointer
|
||||
comparison while Dynamo is compiling."""
|
||||
|
||||
@torch.no_grad()
|
||||
def probe(x):
|
||||
out = torch.empty_like(x)
|
||||
_sync_into_buffer(value=x + 1, buf_view=out)
|
||||
return out * 2
|
||||
|
||||
x = torch.arange(8, dtype=torch.float32)
|
||||
report = torch._dynamo.explain(probe)(x)
|
||||
self.assertEqual(report.graph_break_count, 0, report.break_reasons)
|
||||
self.assertEqual(report.graph_count, 1)
|
||||
torch._dynamo.reset()
|
||||
self.assertTrue(torch.equal(torch.compile(probe)(x), (x + 1) * 2))
|
||||
|
||||
def test_eager_sync_copies_fresh_values_and_keeps_aliases(self):
|
||||
"""Eager keeps the storage check: a fresh value is copied back, an
|
||||
aliased one must not pay a redundant copy_. The version counter is
|
||||
shared by a tensor and its views, so it counts in-place writes."""
|
||||
buf = torch.zeros(8)
|
||||
_sync_into_buffer(value=torch.ones(8), buf_view=buf)
|
||||
self.assertTrue(torch.equal(buf, torch.ones(8)))
|
||||
version = buf._version
|
||||
_sync_into_buffer(value=buf[2:6], buf_view=buf[2:6])
|
||||
self.assertEqual(buf._version, version)
|
||||
|
||||
|
||||
@requires_convrot_kernel
|
||||
class TestQwenImageJointQkvBuffersConvRot(_JointQkvCase):
|
||||
"""The same forward with the six projections on convrot_int8's JIT
|
||||
backend, which writes the joint buffers through its out= op."""
|
||||
|
||||
def setUp(self) -> None:
|
||||
super().setUp()
|
||||
self.config = ConvRotInt8Config(backend="jit")
|
||||
|
||||
def test_forward_is_bitwise_identical_to_join_seqs_path(self):
|
||||
attn = _attention(quant_config=self.config)
|
||||
self.assertEqual(attn.to_q.weight.dtype, torch.int8)
|
||||
self._assert_joint_path_matches_join_seqs_path(attn)
|
||||
|
||||
def test_compiled_forward_matches_eager(self):
|
||||
"""The out= writes into the joint buffers go through the registered
|
||||
custom ops under torch.compile; they must trace and agree with eager."""
|
||||
self._assert_compiled_forward_matches_eager(
|
||||
_attention(quant_config=self.config)
|
||||
)
|
||||
|
||||
def test_fp16_streams_bypass_joint_buffers_and_return_fp16(self):
|
||||
"""FP16 streams must skip the joint buffers (the out= kernel stores BF16
|
||||
only) and the forward must still return FP16."""
|
||||
attn = _attention(dtype=torch.float16, quant_config=self.config)
|
||||
self.assertEqual(attn.to_q.bias.dtype, torch.bfloat16)
|
||||
out = self._assert_forward_bypasses_joint_buffers(attn, dtype=torch.float16)
|
||||
self.assertEqual(out[0].dtype, torch.float16)
|
||||
self.assertEqual(out[1].dtype, torch.float16)
|
||||
|
||||
def test_eligibility_requires_bf16_streams(self):
|
||||
attn = _attention(quant_config=self.config)
|
||||
hidden, encoder = _streams(dim=CONVROT_DIM)
|
||||
|
||||
def eligible(h, e):
|
||||
with patch(SP_WORLD_SIZE, return_value=1):
|
||||
return _use_joint_qkv_buffers(
|
||||
attn=attn, hidden_states=h, encoder_hidden_states=e, masked=False
|
||||
)
|
||||
|
||||
self.assertTrue(eligible(hidden, encoder))
|
||||
self.assertFalse(eligible(hidden.half(), encoder.half()))
|
||||
self.assertFalse(eligible(hidden, encoder.half()))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -623,7 +623,7 @@ class TestServerArgsPathExpansion(unittest.TestCase):
|
||||
server_args.component_attention_backends,
|
||||
)
|
||||
self.assertEqual(
|
||||
{"text_encoder": "kitchen_int8", "transformer": "fp8"},
|
||||
{"text_encoder": "convrot_int8", "transformer": "fp8"},
|
||||
server_args.component_quantizations,
|
||||
)
|
||||
self.assertEqual(
|
||||
|
||||
@@ -21,8 +21,8 @@ from sglang.multimodal_gen.runtime.layers.quantization.comfy_nvfp4 import (
|
||||
ComfyNvfp4Config,
|
||||
ComfyNvfp4LinearMethod,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.configs.kitchen_int8_config import (
|
||||
KitchenInt8Config,
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config import (
|
||||
ConvRotInt8Config,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.configs.kitchen_w4a4_config import (
|
||||
KitchenW4A4Config,
|
||||
@@ -623,11 +623,11 @@ class TestTextEncoderQuantization(unittest.TestCase):
|
||||
"/model/text_encoder",
|
||||
"/model/text_encoder",
|
||||
"text_encoder",
|
||||
explicit_quantization="kitchen_int8",
|
||||
explicit_quantization="convrot_int8",
|
||||
ignored_layers=["lm_head"],
|
||||
)
|
||||
|
||||
self.assertIsInstance(model_config.quant_config, KitchenInt8Config)
|
||||
self.assertIsInstance(model_config.quant_config, ConvRotInt8Config)
|
||||
self.assertFalse(model_config.quant_config.is_checkpoint_int8_serialized)
|
||||
self.assertEqual(model_config.quant_config.ignored_layers, ["lm_head"])
|
||||
|
||||
@@ -688,7 +688,7 @@ class TestTextEncoderQuantization(unittest.TestCase):
|
||||
"text_encoder",
|
||||
)
|
||||
|
||||
self.assertIsInstance(model_config.quant_config, KitchenInt8Config)
|
||||
self.assertIsInstance(model_config.quant_config, ConvRotInt8Config)
|
||||
self.assertEqual(
|
||||
set(model_config.quant_config.layer_markers),
|
||||
{"model.visual.blocks.0.attn.qkv_proj"},
|
||||
@@ -1129,8 +1129,17 @@ class TestQuantizedTextEncoderPostprocess(unittest.TestCase):
|
||||
):
|
||||
_require_quantized_encoder_layers(nn.Linear(2, 2), "text_encoder")
|
||||
|
||||
def test_online_convrot_int8_has_no_markers_to_consume(self):
|
||||
"""An online convrot_int8 encoder carries no serialized markers, so the
|
||||
consumption check must pass instead of failing on the absent map."""
|
||||
_require_quantized_encoder_layers(
|
||||
_QuantizedEncoder(_RecordingQuantMethod()),
|
||||
"text_encoder",
|
||||
quant_config=ConvRotInt8Config(ignored_layers=["lm_head"]),
|
||||
)
|
||||
|
||||
def test_rejects_unconsumed_comfy_marker(self):
|
||||
config = KitchenInt8Config(
|
||||
config = ConvRotInt8Config(
|
||||
layer_markers={
|
||||
"visual.proj": {
|
||||
"format": "int8_tensorwise",
|
||||
|
||||
@@ -58,8 +58,8 @@ from sglang.multimodal_gen.runtime.layers.quantization.comfy_fp8 import (
|
||||
ComfyFp8Config,
|
||||
ComfyFullPrecisionFp8LinearMethod,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.configs.kitchen_int8_config import (
|
||||
KitchenInt8Config,
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config import (
|
||||
ConvRotInt8Config,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.configs.kitchen_w4a4_config import (
|
||||
KitchenW4A4Config,
|
||||
@@ -70,13 +70,13 @@ from sglang.multimodal_gen.runtime.layers.quantization.configs.kitchen_w4a8_conf
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.configs.nunchaku_config import (
|
||||
NunchakuConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.convrot_int8_comfy_kitchen import (
|
||||
ConvRotInt8ComfyKitchenLinearMethod,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.fp8 import (
|
||||
Fp8Config,
|
||||
Fp8LinearMethod,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.kitchen_int8 import (
|
||||
KitchenInt8LinearMethod,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.modelopt_quant import (
|
||||
ModelOptFp4Config,
|
||||
ModelOptFp4LinearMethod,
|
||||
@@ -141,6 +141,13 @@ from sglang.srt.layers.quantization.bitsandbytes import (
|
||||
from sglang.srt.layers.quantization.fp8 import Fp8Config as SRTFp8Config
|
||||
from sglang.srt.layers.quantization.fp8 import Fp8LinearMethod as SRTFp8LinearMethod
|
||||
|
||||
# convrot_int8 picks its kernel backend from the GPU; pin comfy_kitchen so these
|
||||
# cases do not depend on the runner.
|
||||
_CONVROT_JIT_AVAILABLE = (
|
||||
"sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config."
|
||||
"_jit_available"
|
||||
)
|
||||
|
||||
|
||||
class _FakeFluxTransformer:
|
||||
pass
|
||||
@@ -707,10 +714,17 @@ class TestTransformerQuantHelpers(unittest.TestCase):
|
||||
)
|
||||
|
||||
@patch(
|
||||
"sglang.multimodal_gen.runtime.layers.quantization.kitchen_int8."
|
||||
"sglang.multimodal_gen.runtime.layers.quantization.configs.convrot_int8_config."
|
||||
"_comfy_kitchen_available",
|
||||
return_value=True,
|
||||
)
|
||||
@patch(
|
||||
"sglang.multimodal_gen.runtime.layers.quantization.convrot_int8_comfy_kitchen."
|
||||
"_load_comfy_kitchen"
|
||||
)
|
||||
def test_inspect_minimax_h3_safetensors_detects_curve_and_comfy_format(self, _load):
|
||||
def test_inspect_minimax_h3_safetensors_detects_curve_and_comfy_format(
|
||||
self, _load, _available
|
||||
):
|
||||
marker = json.dumps(
|
||||
{
|
||||
"format": "int8_tensorwise",
|
||||
@@ -847,14 +861,27 @@ class TestTransformerQuantHelpers(unittest.TestCase):
|
||||
}
|
||||
)
|
||||
|
||||
self.assertIsInstance(config, KitchenInt8Config)
|
||||
self.assertIsInstance(config, ConvRotInt8Config)
|
||||
self.assertTrue(config.is_checkpoint_int8_serialized)
|
||||
self.assertTrue(config.checkpoint_uses_native_qkv_layout)
|
||||
self.assertFalse(KitchenInt8Config().checkpoint_uses_native_qkv_layout)
|
||||
self.assertFalse(ConvRotInt8Config().checkpoint_uses_native_qkv_layout)
|
||||
self.assertFalse(_needs_device_weight_postprocess(config))
|
||||
self.assertTrue(config.supports_input_partition("blocks.0.mlp.fc1", 6400))
|
||||
self.assertFalse(config.supports_input_partition("blocks.0.mlp.fc1", 3200))
|
||||
|
||||
@patch(_CONVROT_JIT_AVAILABLE, return_value=False)
|
||||
def test_online_convrot_int8_loads_on_the_host_under_offload(self, _jit):
|
||||
"""Online convrot_int8 quantizes each layer through its own CUDA round
|
||||
trip, so the loader must not require the whole BF16 DiT on the GPU
|
||||
(an offloaded 62 GB H3 DiT has to stay on the host during load)."""
|
||||
config = ConvRotInt8Config()
|
||||
|
||||
self.assertFalse(config.is_checkpoint_int8_serialized)
|
||||
self.assertFalse(_needs_device_weight_postprocess(config))
|
||||
spec = TransformerQuantLoadSpec([], config, None, None)
|
||||
self.assertTrue(spec.is_convrot_int8)
|
||||
self.assertFalse(spec.is_serialized_convrot_int8)
|
||||
|
||||
def test_minimax_h3_w4a8_metadata_resolves_serialized_kitchen(self):
|
||||
metadata = {
|
||||
"_quantization_metadata": json.dumps(
|
||||
@@ -1003,9 +1030,10 @@ class TestTransformerQuantHelpers(unittest.TestCase):
|
||||
new=object(),
|
||||
),
|
||||
patch(
|
||||
"sglang.multimodal_gen.runtime.layers.quantization.kitchen_int8."
|
||||
"sglang.multimodal_gen.runtime.layers.quantization.convrot_int8_comfy_kitchen."
|
||||
"_load_comfy_kitchen"
|
||||
),
|
||||
patch(_CONVROT_JIT_AVAILABLE, return_value=False),
|
||||
):
|
||||
config = resolve_minimax_h3_checkpoint_quantization(markers)
|
||||
w4a4 = ReplicatedLinear(
|
||||
@@ -1030,12 +1058,13 @@ class TestTransformerQuantHelpers(unittest.TestCase):
|
||||
self.assertEqual(int8.weight.shape, (3, 256))
|
||||
self.assertEqual(set(config.selected), {"w4a4", "int8"})
|
||||
|
||||
@patch(_CONVROT_JIT_AVAILABLE, return_value=False)
|
||||
@patch(
|
||||
"sglang.multimodal_gen.runtime.layers.quantization.kitchen_int8."
|
||||
"sglang.multimodal_gen.runtime.layers.quantization.convrot_int8_comfy_kitchen."
|
||||
"_load_comfy_kitchen"
|
||||
)
|
||||
def test_serialized_kitchen_constructs_int8_weight_and_row_scale(self, _load):
|
||||
config = KitchenInt8Config(
|
||||
def test_serialized_kitchen_constructs_int8_weight_and_row_scale(self, _load, _jit):
|
||||
config = ConvRotInt8Config(
|
||||
layer_markers={
|
||||
"proj": {
|
||||
"format": "int8_tensorwise",
|
||||
@@ -1731,16 +1760,19 @@ class TestTransformerQuantHelpers(unittest.TestCase):
|
||||
),
|
||||
ModelOptFp4LinearMethod,
|
||||
)
|
||||
with patch(
|
||||
"sglang.multimodal_gen.runtime.layers.quantization."
|
||||
"kitchen_int8._load_comfy_kitchen"
|
||||
with (
|
||||
patch(
|
||||
"sglang.multimodal_gen.runtime.layers.quantization."
|
||||
"convrot_int8_comfy_kitchen._load_comfy_kitchen"
|
||||
),
|
||||
patch(_CONVROT_JIT_AVAILABLE, return_value=False),
|
||||
):
|
||||
self.assertIsInstance(
|
||||
config.get_quant_method(
|
||||
LinearBase(input_size=256, output_size=32),
|
||||
"blocks.0.attn.out_proj",
|
||||
),
|
||||
KitchenInt8LinearMethod,
|
||||
ConvRotInt8ComfyKitchenLinearMethod,
|
||||
)
|
||||
self.assertIsInstance(
|
||||
config.get_quant_method(
|
||||
|
||||
Reference in New Issue
Block a user