[diffusion] perf: FastH3 8-Step V2 native support, native Blackwell VSA and batched H3 VAE decode (#37662)
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com> Co-authored-by: mickqian <mickqian@users.noreply.github.com> Co-authored-by: Mick <mickjagger19@icloud.com>
This commit is contained in:
co-authored by
Claude Fable 5.1
mickqian
Mick
parent
57e9ff5403
commit
4f320fb85a
@@ -0,0 +1,85 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
#include <sgl_kernel/tensor.h>
|
||||
#include <sgl_kernel/utils.h>
|
||||
|
||||
#include <sgl_kernel/utils.cuh>
|
||||
|
||||
#include <dlpack/dlpack.h>
|
||||
#include <tvm/ffi/container/tensor.h>
|
||||
|
||||
#define VSA_BHSD true
|
||||
#include "vsa_block_sparse_sm100/block_sparse_launch_sm100a.cuh"
|
||||
|
||||
namespace sglang {
|
||||
|
||||
/**
|
||||
* \brief FastVideo's warp-specialized tcgen05 block-sparse attention forward for 64-token
|
||||
* tiles (sm_100a / sm_103a).
|
||||
*
|
||||
* \param q, k, v, out [B, H, S, 128] bf16, contiguous; S == num_blocks * 64, num_blocks even.
|
||||
* \param q2k_idx [B * H * num_blocks, max_kv] int32 key-tile lists (valid below q2k_num).
|
||||
* \param q2k_num [B * H * num_blocks] int32 per-query-tile list lengths (0 allowed).
|
||||
* \param block_sizes [num_blocks] int32 valid tokens per key tile.
|
||||
*/
|
||||
struct VsaBlockSparseSm100Kernel {
|
||||
static void
|
||||
run(const tvm::ffi::TensorView q,
|
||||
const tvm::ffi::TensorView k,
|
||||
const tvm::ffi::TensorView v,
|
||||
const tvm::ffi::TensorView q2k_idx,
|
||||
const tvm::ffi::TensorView q2k_num,
|
||||
const tvm::ffi::TensorView block_sizes,
|
||||
tvm::ffi::TensorView out,
|
||||
double sm_scale) {
|
||||
using namespace host;
|
||||
|
||||
auto B = SymbolicSize{"batch"};
|
||||
auto H = SymbolicSize{"heads"};
|
||||
auto S = SymbolicSize{"seq_pad"};
|
||||
auto R = SymbolicSize{"rows"};
|
||||
auto N = SymbolicSize{"num_blocks"};
|
||||
auto MAXKV = SymbolicSize{"max_kv"};
|
||||
auto device = SymbolicDevice{};
|
||||
device.set_options<kDLCUDA>();
|
||||
|
||||
TensorMatcher({B, H, S, HEAD_DIM})
|
||||
.with_dtype<bf16_t>()
|
||||
.with_device(device)
|
||||
.verify(q)
|
||||
.verify(k)
|
||||
.verify(v)
|
||||
.verify(out);
|
||||
TensorMatcher({R, MAXKV}).with_dtype<int32_t>().with_device(device).verify(q2k_idx);
|
||||
TensorMatcher({R}).with_dtype<int32_t>().with_device(device).verify(q2k_num);
|
||||
TensorMatcher({N}).with_dtype<int32_t>().with_device(device).verify(block_sizes);
|
||||
|
||||
const int64_t num_blocks = N.unwrap();
|
||||
CHECK_HOST(num_blocks % 2 == 0) << "num_blocks must be even, got " << num_blocks;
|
||||
CHECK_HOST(S.unwrap() == num_blocks * BLOCK) << "seq_pad " << S.unwrap() << " != num_blocks * " << BLOCK;
|
||||
CHECK_HOST(R.unwrap() == B.unwrap() * H.unwrap() * num_blocks) << "q2k rows must be batch * heads * num_blocks";
|
||||
|
||||
BlockSparseVsaArgs a{};
|
||||
a.q = static_cast<const __nv_bfloat16*>(q.data_ptr());
|
||||
a.k = static_cast<const __nv_bfloat16*>(k.data_ptr());
|
||||
a.v = static_cast<const __nv_bfloat16*>(v.data_ptr());
|
||||
a.v_t = nullptr;
|
||||
a.o = static_cast<__nv_bfloat16*>(out.data_ptr());
|
||||
a.lse = nullptr;
|
||||
a.q2k_idx = static_cast<const int*>(q2k_idx.data_ptr());
|
||||
a.q2k_num = static_cast<const int*>(q2k_num.data_ptr());
|
||||
a.variable_block_sizes = static_cast<const int*>(block_sizes.data_ptr());
|
||||
a.batch = static_cast<int>(B.unwrap());
|
||||
a.num_heads = static_cast<int>(H.unwrap());
|
||||
a.seqlen = static_cast<int>(S.unwrap());
|
||||
a.head_dim = HEAD_DIM;
|
||||
a.num_blocks = static_cast<int>(num_blocks);
|
||||
a.max_kv = static_cast<int>(MAXKV.unwrap());
|
||||
a.sm_scale = static_cast<float>(sm_scale);
|
||||
|
||||
const DLDevice dev = device.unwrap();
|
||||
auto stream = static_cast<cudaStream_t>(::TVMFFIEnvGetStream(dev.device_type, dev.device_id));
|
||||
CHECK_CUDA(launch_block_sparse_sm100a(a, stream)) << "vsa_block_sparse_sm100 launch";
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace sglang
|
||||
+1083
File diff suppressed because it is too large
Load Diff
+205
@@ -0,0 +1,205 @@
|
||||
// Copied and adapted from: https://github.com/hao-ai-lab/FastVideo
|
||||
// (fastvideo-kernel/csrc/attention/block_sparse_launch_sm100a.cuh, Apache-2.0). Inference-only
|
||||
// forward; the sm_103a device pass is admitted alongside sm_100a.
|
||||
|
||||
#ifndef BLOCK_SPARSE_VSA_LAUNCH_SM100A_CUH
|
||||
#define BLOCK_SPARSE_VSA_LAUNCH_SM100A_CUH
|
||||
|
||||
// Launch surface for the sm_100a VSA block-sparse FMHA forward.
|
||||
//
|
||||
// Everything a caller needs: a POD argument struct, a predicate saying whether this build can
|
||||
// run those arguments, and one launch entry point. The benchmark in
|
||||
// block_sparse_bench_sm100a.cu and the torch binding both go through here, so there is
|
||||
// one tensormap construction and one launch configuration rather than two that can drift.
|
||||
//
|
||||
// Two compile-time knobs select the four builds:
|
||||
// VSA_BLK128 false -> 64-token sparse blocks, true -> 128-token
|
||||
// VSA_BHSD false -> [token][head][dim] (BSHD), true -> [batch][head][token][dim] (BHSD)
|
||||
|
||||
#include "block_sparse_kernel_sm100a.cuh"
|
||||
|
||||
namespace VSA_NAMESPACE {
|
||||
|
||||
struct BlockSparseVsaArgs {
|
||||
const __nv_bfloat16* q;
|
||||
const __nv_bfloat16* k;
|
||||
const __nv_bfloat16* v; // natural layout; only blk128 reads it (blk64 still needs v_t)
|
||||
const __nv_bfloat16* v_t; // unused: kept so the bench's V_T buffer still binds
|
||||
__nv_bfloat16* o;
|
||||
float* lse; // [batch, num_heads, seqlen] fp32, or nullptr
|
||||
|
||||
const int* q2k_idx; // [batch*num_heads*num_blocks, max_kv] int32
|
||||
const int* q2k_num; // [batch*num_heads*num_blocks] int32
|
||||
const int* variable_block_sizes; // [num_blocks] int32, valid tokens per block
|
||||
|
||||
int batch;
|
||||
int num_heads;
|
||||
int seqlen;
|
||||
int head_dim;
|
||||
int num_blocks;
|
||||
int max_kv;
|
||||
float sm_scale;
|
||||
};
|
||||
|
||||
// cudaSuccess iff this build can run `a`. Deliberately conservative: the caller is expected
|
||||
// to fall back to its own implementation rather than get a wrong answer.
|
||||
__host__ inline cudaError_t block_sparse_supported(const BlockSparseVsaArgs& a) {
|
||||
if (a.head_dim != HEAD_DIM) return cudaErrorInvalidValue; // compile-time in the kernel
|
||||
if (a.num_blocks % 2 != 0) return cudaErrorInvalidValue; // a CTA owns an adjacent pair
|
||||
if (a.seqlen != a.num_blocks * BLOCK) return cudaErrorInvalidValue;
|
||||
if (a.max_kv < 1 || a.num_blocks < 1) return cudaErrorInvalidValue;
|
||||
if (a.q == nullptr || a.k == nullptr || a.o == nullptr) return cudaErrorInvalidValue;
|
||||
if (a.q2k_idx == nullptr || a.q2k_num == nullptr) return cudaErrorInvalidValue;
|
||||
// FastVideo always supplies this; without it padded keys would be attended as real zeros.
|
||||
if (a.variable_block_sizes == nullptr) return cudaErrorInvalidValue;
|
||||
// V is read MN-major at BOTH block sizes now, so no pre-transposed V_T is ever needed.
|
||||
if (a.v == nullptr) return cudaErrorInvalidValue;
|
||||
return cudaSuccess;
|
||||
}
|
||||
|
||||
__host__ inline cudaError_t launch_block_sparse_sm100a(const BlockSparseVsaArgs& a,
|
||||
cudaStream_t stream) {
|
||||
const cudaError_t sup = block_sparse_supported(a);
|
||||
if (sup != cudaSuccess) return sup;
|
||||
|
||||
const int B = a.batch, H = a.num_heads, S = a.seqlen, hd = a.head_dim;
|
||||
const int num_blocks = a.num_blocks, max_kv = a.max_kv;
|
||||
const long tq = (long)B * S;
|
||||
const int packed_mtiles_per_seq = num_blocks / 2;
|
||||
const int total_work = B * H * packed_mtiles_per_seq;
|
||||
constexpr bool BHSD = VSA_BHSD;
|
||||
|
||||
CUtensorMap tq_, tk_, tvt_, tv_, to_;
|
||||
{
|
||||
uint64_t gd[4] = { (uint64_t)SUB_COLS_BF16, BHSD ? (uint64_t)((long)B * H) : (uint64_t)H,
|
||||
BHSD ? (uint64_t)S : (uint64_t)tq, (uint64_t)Q_SUBTILES };
|
||||
uint64_t gs[3] = { BHSD ? (uint64_t)((long)S * hd) * 2u : (uint64_t)hd * 2u,
|
||||
BHSD ? (uint64_t)hd * 2u : (uint64_t)((long)H * hd) * 2u,
|
||||
(uint64_t)SUB_COLS_BF16 * 2u };
|
||||
uint32_t bd[4] = { (uint32_t)SUB_COLS_BF16, 1u, (uint32_t)M_TILE, (uint32_t)Q_SUBTILES };
|
||||
uint32_t es[4] = { 1u, 1u, 1u, 1u };
|
||||
if (cuTensorMapEncodeTiled(&tq_, CU_TENSOR_MAP_DATA_TYPE_BFLOAT16, 4,
|
||||
const_cast<__nv_bfloat16*>(a.q), gd, gs, bd, es,
|
||||
CU_TENSOR_MAP_INTERLEAVE_NONE, CU_TENSOR_MAP_SWIZZLE_128B,
|
||||
CU_TENSOR_MAP_L2_PROMOTION_L2_128B,
|
||||
CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE) != CUDA_SUCCESS)
|
||||
return cudaErrorInvalidValue;
|
||||
if (cuTensorMapEncodeTiled(&to_, CU_TENSOR_MAP_DATA_TYPE_BFLOAT16, 4, a.o, gd, gs, bd, es,
|
||||
CU_TENSOR_MAP_INTERLEAVE_NONE, CU_TENSOR_MAP_SWIZZLE_128B,
|
||||
CU_TENSOR_MAP_L2_PROMOTION_L2_128B,
|
||||
CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE) != CUDA_SUCCESS)
|
||||
return cudaErrorInvalidValue;
|
||||
}
|
||||
{
|
||||
uint64_t gd[4] = { (uint64_t)SUB_COLS_BF16,
|
||||
BHSD ? (uint64_t)S : (uint64_t)tq,
|
||||
BHSD ? (uint64_t)(hd / SUB_COLS_BF16)
|
||||
: (uint64_t)((long)H * hd / SUB_COLS_BF16),
|
||||
(uint64_t)((long)B * H) };
|
||||
uint64_t gs[3] = { BHSD ? (uint64_t)hd * 2u : (uint64_t)((long)H * hd) * 2u,
|
||||
(uint64_t)SUB_COLS_BF16 * 2u,
|
||||
(uint64_t)((long)S * hd) * 2u };
|
||||
uint32_t bd[4] = { (uint32_t)SUB_COLS_BF16, (uint32_t)BLOCK,
|
||||
BLK128 ? (uint32_t)K_SUBTILES : 1u, 1u };
|
||||
uint32_t es[4] = { 1u, 1u, 1u, 1u };
|
||||
if (cuTensorMapEncodeTiled(&tk_, CU_TENSOR_MAP_DATA_TYPE_BFLOAT16, BHSD ? 4 : 3,
|
||||
const_cast<__nv_bfloat16*>(a.k), gd, gs, bd, es,
|
||||
CU_TENSOR_MAP_INTERLEAVE_NONE, CU_TENSOR_MAP_SWIZZLE_128B,
|
||||
CU_TENSOR_MAP_L2_PROMOTION_L2_128B,
|
||||
CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE) != CUDA_SUCCESS)
|
||||
return cudaErrorInvalidValue;
|
||||
// V map is byte-for-byte the K map over a.v: MN-major V needs no transpose (blk128).
|
||||
const __nv_bfloat16* vbase = a.v ? a.v : a.k;
|
||||
if (cuTensorMapEncodeTiled(&tv_, CU_TENSOR_MAP_DATA_TYPE_BFLOAT16, BHSD ? 4 : 3,
|
||||
const_cast<__nv_bfloat16*>(vbase), gd, gs, bd, es,
|
||||
CU_TENSOR_MAP_INTERLEAVE_NONE, CU_TENSOR_MAP_SWIZZLE_128B,
|
||||
CU_TENSOR_MAP_L2_PROMOTION_L2_128B,
|
||||
CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE) != CUDA_SUCCESS)
|
||||
return cudaErrorInvalidValue;
|
||||
}
|
||||
// V_T map: blk64 only. Unused at blk128 but must still be a valid tensormap to pass by value.
|
||||
{
|
||||
const __nv_bfloat16* vt = a.v_t ? a.v_t : a.k;
|
||||
if constexpr (BLK128) {
|
||||
uint64_t gd[3] = { (uint64_t)SUB_COLS_BF16, (uint64_t)((long)H * hd),
|
||||
(uint64_t)((long)tq / SUB_COLS_BF16) };
|
||||
uint64_t gs[2] = { (uint64_t)tq * 2u, (uint64_t)SUB_COLS_BF16 * 2u };
|
||||
uint32_t bd[3] = { (uint32_t)SUB_COLS_BF16, (uint32_t)hd, (uint32_t)V_SUBTILES };
|
||||
uint32_t es[3] = { 1u, 1u, 1u };
|
||||
if (cuTensorMapEncodeTiled(&tvt_, CU_TENSOR_MAP_DATA_TYPE_BFLOAT16, 3,
|
||||
const_cast<__nv_bfloat16*>(vt), gd, gs, bd, es,
|
||||
CU_TENSOR_MAP_INTERLEAVE_NONE, CU_TENSOR_MAP_SWIZZLE_128B,
|
||||
CU_TENSOR_MAP_L2_PROMOTION_L2_128B,
|
||||
CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE) != CUDA_SUCCESS)
|
||||
return cudaErrorInvalidValue;
|
||||
} else {
|
||||
if (make_tma_2d_tiled(&tvt_, const_cast<__nv_bfloat16*>(vt), (long)H * hd, (int)tq, hd,
|
||||
SUB_COLS_BF16, 2, CU_TENSOR_MAP_DATA_TYPE_BFLOAT16,
|
||||
CU_TENSOR_MAP_SWIZZLE_128B) != cudaSuccess)
|
||||
return cudaErrorInvalidValue;
|
||||
}
|
||||
}
|
||||
|
||||
const size_t smem =
|
||||
(size_t)2 * Q_TILE_BYTES + NUM_KV_STAGES * KV_RING_SLOT_BYTES
|
||||
+ (size_t)2 * M_TILE * HEAD_DIM * sizeof(__nv_bfloat16)
|
||||
+ (2 * NUM_KV_STAGES + 22) * 8
|
||||
+ (size_t)CLC_STAGES * (2 * 8 + 16) + 16
|
||||
+ 8
|
||||
+ (size_t)2 * STAT_REGIONS * STATS * sizeof(float)
|
||||
+ 256;
|
||||
|
||||
#ifndef VSA_NAMED_BAR
|
||||
#define VSA_NAMED_BAR false
|
||||
#endif
|
||||
#ifndef VSA_THROTTLE
|
||||
#define VSA_THROTTLE false
|
||||
#endif
|
||||
#ifndef VSA_USE_CLC
|
||||
#define VSA_USE_CLC true
|
||||
#endif
|
||||
constexpr bool FULL_NAMED_BAR = VSA_NAMED_BAR, EX2_EMU = true, SPLIT_P = true,
|
||||
SOFTMAX_THROTTLE = VSA_THROTTLE, USE_CLC = VSA_USE_CLC,
|
||||
Q_RASTER = true, MHA = true;
|
||||
auto kfn = &fmha_context_bf16_gen_kernel<32, FULL_NAMED_BAR, EX2_EMU, SPLIT_P,
|
||||
SOFTMAX_THROTTLE, USE_CLC, Q_RASTER, MHA,
|
||||
/*RESCALE_THRESHOLD=*/8, /*BHSD=*/VSA_BHSD>;
|
||||
cudaError_t e = cudaFuncSetAttribute(kfn, cudaFuncAttributeMaxDynamicSharedMemorySize, (int)smem);
|
||||
if (e != cudaSuccess) return e;
|
||||
|
||||
const unsigned long long magic0 = make_magic((unsigned)(H * packed_mtiles_per_seq));
|
||||
const unsigned long long magic1 = make_magic((unsigned)H);
|
||||
const unsigned long long magic2 = make_magic((unsigned)packed_mtiles_per_seq);
|
||||
const float scale_log2 = a.sm_scale * (float)M_LOG2E;
|
||||
|
||||
int numSM = 0;
|
||||
e = cudaDeviceGetAttribute(&numSM, cudaDevAttrMultiProcessorCount, 0);
|
||||
if (e != cudaSuccess) return e;
|
||||
const int num_ctas = USE_CLC ? total_work : (total_work < numSM ? total_work : numSM);
|
||||
dim3 grid(num_ctas, 1, 1), block(N_WARPS * 32, 1, 1);
|
||||
|
||||
if (USE_CLC) {
|
||||
cudaLaunchConfig_t cfg = {};
|
||||
cfg.gridDim = grid; cfg.blockDim = block; cfg.dynamicSmemBytes = smem; cfg.stream = stream;
|
||||
cudaLaunchAttribute cfgAttr[1];
|
||||
cfgAttr[0].id = cudaLaunchAttributeClusterDimension;
|
||||
cfgAttr[0].val.clusterDim.x = 1; cfgAttr[0].val.clusterDim.y = 1;
|
||||
cfgAttr[0].val.clusterDim.z = 1;
|
||||
cfg.attrs = cfgAttr; cfg.numAttrs = 1;
|
||||
return cudaLaunchKernelEx(&cfg, kfn, tq_, tk_, tvt_, tv_, to_, S, H, scale_log2, B,
|
||||
num_blocks, packed_mtiles_per_seq, max_kv, magic0, magic1, magic2,
|
||||
a.q2k_idx, a.q2k_num, a.variable_block_sizes, a.lse);
|
||||
}
|
||||
kfn<<<grid, block, smem, stream>>>(tq_, tk_, tvt_, tv_, to_, S, H, scale_log2, B, num_blocks,
|
||||
packed_mtiles_per_seq, max_kv, magic0, magic1, magic2,
|
||||
a.q2k_idx, a.q2k_num, a.variable_block_sizes, a.lse);
|
||||
return cudaGetLastError();
|
||||
}
|
||||
|
||||
} // namespace VSA_NAMESPACE
|
||||
|
||||
// Callers (the bench, the torch binding) keep using unqualified names; each translation unit
|
||||
// only ever sees the one configuration its VSA_BLK128 selected.
|
||||
using namespace VSA_NAMESPACE;
|
||||
|
||||
#endif // BLOCK_SPARSE_VSA_LAUNCH_SM100A_CUH
|
||||
@@ -0,0 +1,880 @@
|
||||
// Copied and adapted from: https://github.com/hao-ai-lab/FastVideo
|
||||
// (fastvideo-kernel/csrc/attention/primitives.cuh, Apache-2.0). Inference-only
|
||||
// forward; the sm_103a device pass is admitted alongside sm_100a.
|
||||
|
||||
// primitives.cuh -- device primitives for the sm_100a VSA block-sparse attention
|
||||
// forward: tcgen05 (alloc / mma / ld / st / commit / wait / fence), TMA load / store /
|
||||
// tensormap, mbarrier, cluster launch control, setmaxnreg, fast math, and the FMHA helpers.
|
||||
//
|
||||
// Generated and pruned to what the kernel reaches -- do not edit by hand.
|
||||
#pragma once
|
||||
#include <cstdint>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <cuda.h>
|
||||
#include <cuda_bf16.h>
|
||||
#include <cuda_fp16.h>
|
||||
#include <cuda_runtime.h>
|
||||
#include <cmath>
|
||||
#include <cassert>
|
||||
#include <cstring>
|
||||
#include <vector_types.h>
|
||||
|
||||
#ifndef CUDA_CHECK
|
||||
#define CUDA_CHECK(stmt) do { \
|
||||
cudaError_t _e = (stmt); \
|
||||
if (_e != cudaSuccess) { \
|
||||
fprintf(stderr, "CUDA error %s:%d: %s -> %s\n", \
|
||||
__FILE__, __LINE__, #stmt, cudaGetErrorString(_e)); \
|
||||
std::exit(1); \
|
||||
} \
|
||||
} while (0)
|
||||
#endif
|
||||
|
||||
__device__ __forceinline__
|
||||
uint64_t mbarrier_arrive(uint32_t mbar_smem) {
|
||||
uint64_t state;
|
||||
asm volatile("mbarrier.arrive.shared::cta.b64 %0, [%1];\n"
|
||||
: "=l"(state) : "r"(mbar_smem) : "memory");
|
||||
return state;
|
||||
}
|
||||
|
||||
__device__ __forceinline__
|
||||
void mbarrier_arrive_nostate(uint32_t mbar_smem) {
|
||||
asm volatile("mbarrier.arrive.shared::cta.b64 _, [%0];\n"
|
||||
:: "r"(mbar_smem) : "memory");
|
||||
}
|
||||
|
||||
__device__ __forceinline__
|
||||
void mbarrier_arrive_cluster_default(uint32_t cluster_smem_addr) {
|
||||
asm volatile("mbarrier.arrive.shared::cluster.b64 _, [%0];\n"
|
||||
:: "r"(cluster_smem_addr) : "memory");
|
||||
}
|
||||
|
||||
__device__ __forceinline__
|
||||
void mbarrier_arrive_expect_tx(uint32_t mbar_smem, uint32_t expected_bytes) {
|
||||
asm volatile("mbarrier.arrive.expect_tx.release.cta.shared::cta.b64 _, [%0], %1;\n"
|
||||
:: "r"(mbar_smem), "r"(expected_bytes) : "memory");
|
||||
}
|
||||
|
||||
__device__ __forceinline__
|
||||
void mbarrier_wait_parity_suspend(uint32_t mbar_smem, uint32_t phase_parity) {
|
||||
asm volatile(
|
||||
"{\n"
|
||||
".reg .pred P1;\n"
|
||||
"LAB_WAIT:\n"
|
||||
"mbarrier.try_wait.parity.shared::cta.b64 P1, [%0], %1, 10000000;\n"
|
||||
"@!P1 bra.uni LAB_WAIT;\n"
|
||||
"}\n"
|
||||
:: "r"(mbar_smem), "r"(phase_parity) : "memory");
|
||||
}
|
||||
|
||||
__device__ __forceinline__
|
||||
void mbarrier_wait_parity(uint32_t mbar_smem, uint32_t phase_parity) {
|
||||
asm volatile(
|
||||
"{\n"
|
||||
".reg .pred P1;\n"
|
||||
"LAB_WAIT_HOT:\n"
|
||||
"mbarrier.try_wait.parity.shared::cta.b64 P1, [%0], %1;\n"
|
||||
"@!P1 bra.uni LAB_WAIT_HOT;\n"
|
||||
"}\n"
|
||||
:: "r"(mbar_smem), "r"(phase_parity) : "memory");
|
||||
}
|
||||
|
||||
__device__ __forceinline__
|
||||
void fence_proxy_async_shared_cta() {
|
||||
asm volatile("fence.proxy.async.shared::cta;\n" ::: "memory");
|
||||
}
|
||||
|
||||
__device__ __forceinline__
|
||||
void fence_proxy_async_shared() {
|
||||
asm volatile("fence.proxy.async.shared::cta;\n" ::: "memory");
|
||||
}
|
||||
|
||||
__device__ __forceinline__ void clc_try_cancel_async(
|
||||
uint32_t smem_dst, uint32_t mbar_smem) {
|
||||
asm volatile(
|
||||
"clusterlaunchcontrol.try_cancel.async.shared::cta.mbarrier::complete_tx::bytes.b128"
|
||||
" [%0], [%1];\n"
|
||||
:: "r"(smem_dst), "r"(mbar_smem) : "memory");
|
||||
}
|
||||
|
||||
__device__ __forceinline__ void clc_load_response(
|
||||
uint32_t smem_slot, uint32_t& r0, uint32_t& r1,
|
||||
uint32_t& r2, uint32_t& r3) {
|
||||
asm volatile("ld.shared::cta.v4.b32 {%0, %1, %2, %3}, [%4];\n"
|
||||
: "=r"(r0), "=r"(r1), "=r"(r2), "=r"(r3)
|
||||
: "r"(smem_slot));
|
||||
}
|
||||
|
||||
template <int NUM_STAGES>
|
||||
struct MbarrierPhaseTracker {
|
||||
|
||||
uint32_t phase[NUM_STAGES];
|
||||
int idx;
|
||||
|
||||
__device__ __forceinline__
|
||||
void init() {
|
||||
for (int i = 0; i < NUM_STAGES; ++i) phase[i] = 0;
|
||||
idx = 0;
|
||||
}
|
||||
|
||||
__device__ __forceinline__
|
||||
uint32_t current_phase() const { return phase[idx]; }
|
||||
|
||||
__device__ __forceinline__
|
||||
void advance() {
|
||||
phase[idx] ^= 1u;
|
||||
idx = (idx + 1) % NUM_STAGES;
|
||||
}
|
||||
|
||||
__device__ __forceinline__
|
||||
int stage() const { return idx; }
|
||||
};
|
||||
|
||||
template <int NUM_STAGES>
|
||||
struct PhaseTracker {
|
||||
int stage;
|
||||
uint32_t phase;
|
||||
|
||||
__device__ __forceinline__
|
||||
PhaseTracker() : stage(0), phase(0) {}
|
||||
|
||||
__device__ __forceinline__
|
||||
void advance() {
|
||||
stage++;
|
||||
if (stage == NUM_STAGES) {
|
||||
stage = 0;
|
||||
phase ^= 1;
|
||||
}
|
||||
}
|
||||
|
||||
__device__ __forceinline__
|
||||
int get_stage() const { return stage; }
|
||||
|
||||
__device__ __forceinline__
|
||||
uint32_t get_phase() const { return phase; }
|
||||
};
|
||||
|
||||
template <int NUM_STAGES>
|
||||
struct EmptyPhaseTracker {
|
||||
int stage;
|
||||
uint32_t phase;
|
||||
|
||||
__device__ __forceinline__
|
||||
EmptyPhaseTracker() : stage(0), phase(1) {}
|
||||
|
||||
__device__ __forceinline__
|
||||
void advance() {
|
||||
stage++;
|
||||
if (stage == NUM_STAGES) {
|
||||
stage = 0;
|
||||
phase ^= 1;
|
||||
}
|
||||
}
|
||||
|
||||
__device__ __forceinline__
|
||||
int get_stage() const { return stage; }
|
||||
|
||||
__device__ __forceinline__
|
||||
uint32_t get_phase() const { return phase; }
|
||||
};
|
||||
|
||||
template <int STAGES>
|
||||
__device__ __forceinline__
|
||||
void advance_stage_phase(int& stage, uint32_t& phase) {
|
||||
++stage;
|
||||
if (stage == STAGES) {
|
||||
stage = 0;
|
||||
phase ^= 1u;
|
||||
}
|
||||
}
|
||||
|
||||
static constexpr uint32_t SM100_CLC_PEER_MASK = 0xFEFFFFFF;
|
||||
|
||||
struct ClcTileInfo {
|
||||
int m_tile;
|
||||
int n_tile;
|
||||
bool valid;
|
||||
};
|
||||
|
||||
enum class ClcRasterOrder { AlongN, AlongM };
|
||||
|
||||
__device__ __forceinline__
|
||||
void clc_arrive_expect_tx_cta(uint32_t clc_full_local_addr, uint32_t tx_bytes) {
|
||||
if ((threadIdx.x & 31) == 0) {
|
||||
mbarrier_arrive_expect_tx(clc_full_local_addr, tx_bytes);
|
||||
}
|
||||
}
|
||||
|
||||
__device__ __forceinline__
|
||||
void clc_consumer_release(uint32_t clc_empty_local_addr) {
|
||||
uint32_t peer0_addr = clc_empty_local_addr & SM100_CLC_PEER_MASK;
|
||||
mbarrier_arrive_cluster_default(peer0_addr);
|
||||
}
|
||||
|
||||
__device__ __forceinline__
|
||||
void clc_consumer_release_cta(uint32_t clc_empty_local_addr) {
|
||||
mbarrier_arrive_nostate(clc_empty_local_addr);
|
||||
}
|
||||
|
||||
template <int CLUSTER_SHAPE_M, int CLUSTER_SHAPE_N, ClcRasterOrder ORDER>
|
||||
__device__ __forceinline__
|
||||
ClcTileInfo clc_parse_response(uint32_t resp_smem_addr) {
|
||||
uint32_t d0, d1, d2, d3;
|
||||
fence_proxy_async_shared_cta();
|
||||
clc_load_response(resp_smem_addr, d0, d1, d2, d3);
|
||||
const int ctaid_x = static_cast<int>(d0);
|
||||
const int ctaid_y = static_cast<int>(d1 & 0xFFFFu);
|
||||
const bool valid = (d2 & 1u) != 0u;
|
||||
(void)d3;
|
||||
|
||||
ClcTileInfo info;
|
||||
info.valid = valid;
|
||||
if constexpr (ORDER == ClcRasterOrder::AlongN) {
|
||||
info.m_tile = ctaid_y / CLUSTER_SHAPE_M;
|
||||
info.n_tile = ctaid_x / CLUSTER_SHAPE_N;
|
||||
} else {
|
||||
info.m_tile = ctaid_x / CLUSTER_SHAPE_M;
|
||||
info.n_tile = ctaid_y / CLUSTER_SHAPE_N;
|
||||
}
|
||||
return info;
|
||||
}
|
||||
|
||||
template <int CLUSTER_SHAPE_M, int CLUSTER_SHAPE_N, ClcRasterOrder ORDER,
|
||||
int CTA_GROUP = 2, bool SUSPEND = false>
|
||||
__device__ __forceinline__
|
||||
ClcTileInfo clc_fetch_next_tile(
|
||||
uint64_t* clc_full_bar, uint64_t* clc_empty_bar, uint32_t* clc_response,
|
||||
int clc_cons_stage, uint32_t clc_cons_phase, bool do_release) {
|
||||
uint32_t full_addr = static_cast<uint32_t>(
|
||||
__cvta_generic_to_shared(&clc_full_bar[clc_cons_stage]));
|
||||
if constexpr (SUSPEND) mbarrier_wait_parity_suspend(full_addr, clc_cons_phase);
|
||||
else mbarrier_wait_parity(full_addr, clc_cons_phase);
|
||||
uint32_t resp_addr = static_cast<uint32_t>(
|
||||
__cvta_generic_to_shared(&clc_response[clc_cons_stage * 4]));
|
||||
ClcTileInfo t = clc_parse_response<
|
||||
CLUSTER_SHAPE_M, CLUSTER_SHAPE_N, ORDER>(resp_addr);
|
||||
if (do_release) {
|
||||
uint32_t empty_local = static_cast<uint32_t>(
|
||||
__cvta_generic_to_shared(&clc_empty_bar[clc_cons_stage]));
|
||||
if constexpr (CTA_GROUP == 1) {
|
||||
clc_consumer_release_cta(empty_local);
|
||||
} else {
|
||||
clc_consumer_release(empty_local);
|
||||
}
|
||||
}
|
||||
return t;
|
||||
}
|
||||
|
||||
template <int STAGES = 2>
|
||||
__device__ __forceinline__
|
||||
void clc_fetch_next_tile_advance(int& clc_cons_stage,
|
||||
uint32_t& clc_cons_phase) {
|
||||
advance_stage_phase<STAGES>(clc_cons_stage, clc_cons_phase);
|
||||
}
|
||||
|
||||
__device__ __forceinline__ unsigned fdiv(unsigned n, unsigned long long pk) {
|
||||
unsigned M = (unsigned)pk;
|
||||
if (M == 0u) return n;
|
||||
return __umulhi(n, M) >> (unsigned)(pk >> 32);
|
||||
}
|
||||
__host__ inline unsigned long long make_magic(unsigned d) {
|
||||
if (d <= 1u) return 0ULL;
|
||||
unsigned l = 0; while ((1u << (l + 1)) <= d) ++l;
|
||||
unsigned p = 31u + l;
|
||||
unsigned long long m = ((1ull << p) + (unsigned long long)d - 1ull) / d;
|
||||
return (m & 0xffffffffULL) | ((unsigned long long)(p - 32u) << 32);
|
||||
}
|
||||
|
||||
template <int BARRIER_ID>
|
||||
__device__ __forceinline__
|
||||
void bar_sync(uint32_t thread_count) {
|
||||
static_assert(BARRIER_ID >= 0 && BARRIER_ID <= 15,
|
||||
"bar.sync: BARRIER_ID must be in [0, 15]");
|
||||
asm volatile("bar.sync %0, %1;\n"
|
||||
:: "n"(BARRIER_ID), "r"(thread_count) : "memory");
|
||||
}
|
||||
|
||||
template <int BARRIER_ID>
|
||||
__device__ __forceinline__
|
||||
void bar_arrive(uint32_t thread_count) {
|
||||
static_assert(BARRIER_ID >= 0 && BARRIER_ID <= 15,
|
||||
"bar.arrive: BARRIER_ID must be in [0, 15]");
|
||||
asm volatile("bar.arrive %0, %1;\n"
|
||||
:: "n"(BARRIER_ID), "r"(thread_count) : "memory");
|
||||
}
|
||||
|
||||
__device__ __forceinline__ void full_bar_arrive(int m_tile, int band) {
|
||||
switch (1 + m_tile * 4 + band) {
|
||||
case 1: bar_arrive<1>(64); break;
|
||||
case 2: bar_arrive<2>(64); break;
|
||||
case 3: bar_arrive<3>(64); break;
|
||||
case 4: bar_arrive<4>(64); break;
|
||||
case 5: bar_arrive<5>(64); break;
|
||||
case 6: bar_arrive<6>(64); break;
|
||||
case 7: bar_arrive<7>(64); break;
|
||||
case 8: bar_arrive<8>(64); break;
|
||||
}
|
||||
}
|
||||
__device__ __forceinline__ void full_bar_wait(int m_tile, int band) {
|
||||
switch (1 + m_tile * 4 + band) {
|
||||
case 1: bar_sync<1>(64); break;
|
||||
case 2: bar_sync<2>(64); break;
|
||||
case 3: bar_sync<3>(64); break;
|
||||
case 4: bar_sync<4>(64); break;
|
||||
case 5: bar_sync<5>(64); break;
|
||||
case 6: bar_sync<6>(64); break;
|
||||
case 7: bar_sync<7>(64); break;
|
||||
case 8: bar_sync<8>(64); break;
|
||||
}
|
||||
}
|
||||
|
||||
template <bool IS_CAUSAL, int K_TILE>
|
||||
__device__ __forceinline__ void mask_s_row_r2p(float* scores, int k_offset, int q_pos, int seqlen_k) {
|
||||
int n_keep = seqlen_k - k_offset;
|
||||
if constexpr (IS_CAUSAL) {
|
||||
const int causal = q_pos - k_offset + 1;
|
||||
n_keep = n_keep < causal ? n_keep : causal;
|
||||
}
|
||||
#pragma unroll
|
||||
for (int s = 0; s < K_TILE / 32; ++s) {
|
||||
int m = (s + 1) * 32 - n_keep;
|
||||
m = m < 0 ? 0 : (m > 32 ? 32 : m);
|
||||
const uint32_t keep = (m >= 32) ? 0u : (0xFFFFFFFFu >> m);
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 32; ++i)
|
||||
if (!(keep & (1u << i))) scores[s * 32 + i] = -INFINITY;
|
||||
}
|
||||
}
|
||||
|
||||
template <int CTA_GROUP>
|
||||
__device__ __forceinline__ void tcgen05_alloc(uint32_t smem_dst_ptr,
|
||||
uint32_t n_cols) {
|
||||
static_assert(CTA_GROUP == 1 || CTA_GROUP == 2,
|
||||
"tcgen05_alloc: CTA_GROUP must be 1 or 2");
|
||||
if constexpr (CTA_GROUP == 1) {
|
||||
asm volatile(
|
||||
"tcgen05.alloc.cta_group::1.sync.aligned.shared::cta.b32 [%0], %1;\n"
|
||||
:: "r"(smem_dst_ptr), "r"(n_cols));
|
||||
} else {
|
||||
asm volatile(
|
||||
"tcgen05.alloc.cta_group::2.sync.aligned.shared::cta.b32 [%0], %1;\n"
|
||||
:: "r"(smem_dst_ptr), "r"(n_cols));
|
||||
}
|
||||
}
|
||||
|
||||
__device__ __forceinline__ void tcgen05_st_32x32b_x16(
|
||||
uint32_t tmem_addr, const uint32_t (&r)[16]) {
|
||||
asm volatile("tcgen05.st.sync.aligned.32x32b.x16.b32 "
|
||||
"[%0], {%1,%2,%3,%4,%5,%6,%7,%8,%9,%10,"
|
||||
"%11,%12,%13,%14,%15,%16};\n"
|
||||
:: "r"(tmem_addr),
|
||||
"r"(r[0]),"r"(r[1]),"r"(r[2]),"r"(r[3]),
|
||||
"r"(r[4]),"r"(r[5]),"r"(r[6]),"r"(r[7]),
|
||||
"r"(r[8]),"r"(r[9]),"r"(r[10]),"r"(r[11]),
|
||||
"r"(r[12]),"r"(r[13]),"r"(r[14]),"r"(r[15]));
|
||||
}
|
||||
|
||||
__device__ __forceinline__ void tcgen05_st_32x32b_x32(
|
||||
uint32_t tmem_addr, const uint32_t (&r)[32]) {
|
||||
asm volatile("tcgen05.st.sync.aligned.32x32b.x32.b32 "
|
||||
"[%0], {%1,%2,%3,%4,%5,%6,%7,%8,%9,%10,"
|
||||
"%11,%12,%13,%14,%15,%16,%17,%18,%19,%20,"
|
||||
"%21,%22,%23,%24,%25,%26,%27,%28,%29,%30,"
|
||||
"%31,%32};\n"
|
||||
:: "r"(tmem_addr),
|
||||
"r"(r[0]),"r"(r[1]),"r"(r[2]),"r"(r[3]),
|
||||
"r"(r[4]),"r"(r[5]),"r"(r[6]),"r"(r[7]),
|
||||
"r"(r[8]),"r"(r[9]),"r"(r[10]),"r"(r[11]),
|
||||
"r"(r[12]),"r"(r[13]),"r"(r[14]),"r"(r[15]),
|
||||
"r"(r[16]),"r"(r[17]),"r"(r[18]),"r"(r[19]),
|
||||
"r"(r[20]),"r"(r[21]),"r"(r[22]),"r"(r[23]),
|
||||
"r"(r[24]),"r"(r[25]),"r"(r[26]),"r"(r[27]),
|
||||
"r"(r[28]),"r"(r[29]),"r"(r[30]),"r"(r[31]));
|
||||
}
|
||||
|
||||
__device__ __forceinline__ void tcgen05_commit1_lead(uint32_t lead, uint32_t mbar_smem_addr) {
|
||||
asm volatile(
|
||||
"{\n\t"
|
||||
".reg .pred q;\n\t"
|
||||
"setp.ne.b32 q, %0, 0;\n\t"
|
||||
"@q tcgen05.commit.cta_group::1.mbarrier::arrive::one.b64 [%1];\n\t"
|
||||
"}\n"
|
||||
:: "r"(lead), "r"(mbar_smem_addr));
|
||||
}
|
||||
|
||||
__device__ __forceinline__ void tcgen05_wait_st() {
|
||||
asm volatile("tcgen05.wait::st.sync.aligned;\n" ::: "memory");
|
||||
}
|
||||
|
||||
__device__ __forceinline__ void tcgen05_fence_before_thread_sync() {
|
||||
asm volatile("tcgen05.fence::before_thread_sync;\n" ::: "memory");
|
||||
}
|
||||
|
||||
__device__ __forceinline__
|
||||
void tma_load_2d(uint32_t smem_dst, const void* tensormap_ptr,
|
||||
uint32_t mbar_smem, int coord_x, int coord_y) {
|
||||
asm volatile(
|
||||
"cp.async.bulk.tensor.2d.shared::cluster.global.tile"
|
||||
".mbarrier::complete_tx::bytes"
|
||||
" [%0], [%1, {%3, %4}], [%2];\n"
|
||||
:: "r"(smem_dst), "l"(tensormap_ptr),
|
||||
"r"(mbar_smem), "r"(coord_x), "r"(coord_y)
|
||||
: "memory");
|
||||
}
|
||||
|
||||
__device__ __forceinline__
|
||||
void tma_load_3d(uint32_t smem_dst, const void* tensormap_ptr,
|
||||
uint32_t mbar_smem, int c0, int c1, int c2) {
|
||||
asm volatile(
|
||||
"cp.async.bulk.tensor.3d.shared::cluster.global.tile"
|
||||
".mbarrier::complete_tx::bytes"
|
||||
" [%0], [%1, {%3, %4, %5}], [%2];\n"
|
||||
:: "r"(smem_dst), "l"(tensormap_ptr),
|
||||
"r"(mbar_smem), "r"(c0), "r"(c1), "r"(c2)
|
||||
: "memory");
|
||||
}
|
||||
|
||||
__device__ __forceinline__
|
||||
void tma_load_4d(uint32_t smem_dst, const void* tensormap_ptr,
|
||||
uint32_t mbar_smem, int c0, int c1, int c2, int c3) {
|
||||
asm volatile(
|
||||
"cp.async.bulk.tensor.4d.shared::cluster.global.tile"
|
||||
".mbarrier::complete_tx::bytes"
|
||||
" [%0], [%1, {%3, %4, %5, %6}], [%2];\n"
|
||||
:: "r"(smem_dst), "l"(tensormap_ptr),
|
||||
"r"(mbar_smem), "r"(c0), "r"(c1), "r"(c2), "r"(c3)
|
||||
: "memory");
|
||||
}
|
||||
|
||||
template <int CTA_GROUP>
|
||||
__device__ __forceinline__ void tcgen05_dealloc(uint32_t tmem_addr,
|
||||
uint32_t n_cols) {
|
||||
static_assert(CTA_GROUP == 1 || CTA_GROUP == 2,
|
||||
"tcgen05_dealloc: CTA_GROUP must be 1 or 2");
|
||||
if constexpr (CTA_GROUP == 1) {
|
||||
asm volatile(
|
||||
"tcgen05.dealloc.cta_group::1.sync.aligned.b32 %0, %1;\n"
|
||||
:: "r"(tmem_addr), "r"(n_cols));
|
||||
} else {
|
||||
asm volatile(
|
||||
"tcgen05.dealloc.cta_group::2.sync.aligned.b32 %0, %1;\n"
|
||||
:: "r"(tmem_addr), "r"(n_cols));
|
||||
}
|
||||
}
|
||||
|
||||
__device__ __forceinline__
|
||||
void tma_store_2d(const void* tensormap_ptr, int coord_x, int coord_y,
|
||||
uint32_t smem_src) {
|
||||
asm volatile(
|
||||
"cp.async.bulk.tensor.2d.global.shared::cta.tile.bulk_group"
|
||||
" [%0, {%1, %2}], [%3];\n"
|
||||
:: "l"(tensormap_ptr), "r"(coord_x), "r"(coord_y),
|
||||
"r"(smem_src)
|
||||
: "memory");
|
||||
}
|
||||
|
||||
__device__ __forceinline__
|
||||
void tma_store_3d(const void* tensormap_ptr, int c0, int c1, int c2,
|
||||
uint32_t smem_src) {
|
||||
asm volatile(
|
||||
"cp.async.bulk.tensor.3d.global.shared::cta.tile.bulk_group"
|
||||
" [%0, {%1, %2, %3}], [%4];\n"
|
||||
:: "l"(tensormap_ptr), "r"(c0), "r"(c1), "r"(c2), "r"(smem_src)
|
||||
: "memory");
|
||||
}
|
||||
|
||||
__device__ __forceinline__
|
||||
void tma_store_4d(const void* tensormap_ptr, int c0, int c1, int c2, int c3,
|
||||
uint32_t smem_src) {
|
||||
asm volatile(
|
||||
"cp.async.bulk.tensor.4d.global.shared::cta.tile.bulk_group"
|
||||
" [%0, {%1, %2, %3, %4}], [%5];\n"
|
||||
:: "l"(tensormap_ptr), "r"(c0), "r"(c1), "r"(c2), "r"(c3), "r"(smem_src)
|
||||
: "memory");
|
||||
}
|
||||
|
||||
inline cudaError_t make_tma_2d_tiled(
|
||||
CUtensorMap* out,
|
||||
const void* ptr, int rows, int cols, int box_rows, int box_cols,
|
||||
int elem_bytes, CUtensorMapDataType dtype,
|
||||
CUtensorMapSwizzle swizzle = CU_TENSOR_MAP_SWIZZLE_128B,
|
||||
CUtensorMapL2promotion l2 = CU_TENSOR_MAP_L2_PROMOTION_L2_128B,
|
||||
CUtensorMapFloatOOBfill oob = CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE) {
|
||||
uint64_t globalDim[2] = { (uint64_t)cols, (uint64_t)rows };
|
||||
uint64_t globalStrides[1] = { (uint64_t)cols * (uint64_t)elem_bytes };
|
||||
uint32_t boxDim[2] = { (uint32_t)box_cols, (uint32_t)box_rows };
|
||||
uint32_t elemStrides[2] = { 1u, 1u };
|
||||
|
||||
CUresult r = cuTensorMapEncodeTiled(
|
||||
out, dtype, 2,
|
||||
const_cast<void*>(ptr), globalDim, globalStrides,
|
||||
boxDim, elemStrides,
|
||||
CU_TENSOR_MAP_INTERLEAVE_NONE,
|
||||
swizzle, l2, oob);
|
||||
return (r == CUDA_SUCCESS) ? cudaSuccess : cudaErrorInvalidValue;
|
||||
}
|
||||
|
||||
__device__ __forceinline__
|
||||
void cp_async_bulk_commit_group() {
|
||||
asm volatile("cp.async.bulk.commit_group;\n" ::: "memory");
|
||||
}
|
||||
|
||||
template <int N>
|
||||
__device__ __forceinline__
|
||||
void cp_async_bulk_wait_group_read() {
|
||||
asm volatile("cp.async.bulk.wait_group.read %0;\n" :: "n"(N) : "memory");
|
||||
}
|
||||
|
||||
__device__ __forceinline__
|
||||
void mbarrier_init(uint32_t mbar_smem, uint32_t arrive_count) {
|
||||
asm volatile("mbarrier.init.shared::cta.b64 [%0], %1;\n"
|
||||
:: "r"(mbar_smem), "r"(arrive_count) : "memory");
|
||||
}
|
||||
|
||||
template <int CTA_GROUP>
|
||||
__device__ __forceinline__ void tcgen05_relinquish_alloc_permit() {
|
||||
static_assert(CTA_GROUP == 1 || CTA_GROUP == 2,
|
||||
"tcgen05_relinquish_alloc_permit: CTA_GROUP must be 1 or 2");
|
||||
if constexpr (CTA_GROUP == 1) {
|
||||
asm volatile("tcgen05.relinquish_alloc_permit.cta_group::1.sync.aligned;\n" ::);
|
||||
} else {
|
||||
asm volatile("tcgen05.relinquish_alloc_permit.cta_group::2.sync.aligned;\n" ::);
|
||||
}
|
||||
}
|
||||
|
||||
__device__ __forceinline__
|
||||
void fence_mbarrier_init_release_cluster() {
|
||||
asm volatile("fence.mbarrier_init.release.cluster;\n" ::: "memory");
|
||||
}
|
||||
|
||||
__device__ __forceinline__ void tcgen05_mma_f16_ss_lead(uint32_t lead,
|
||||
uint32_t tmem_c, uint64_t desc_a, uint64_t desc_b, uint32_t idesc,
|
||||
bool enable_input_d) {
|
||||
asm volatile(
|
||||
"{\n\t"
|
||||
".reg .pred p, q;\n\t"
|
||||
"setp.ne.b32 q, %0, 0;\n\t"
|
||||
"setp.ne.b32 p, %5, 0;\n\t"
|
||||
"@q tcgen05.mma.cta_group::1.kind::f16 [%1], %2, %3, %4, {%6, %7, %8, %9}, p;\n\t"
|
||||
"}\n"
|
||||
:: "r"(lead), "r"(tmem_c), "l"(desc_a), "l"(desc_b), "r"(idesc),
|
||||
"r"(enable_input_d ? 1u : 0u), "r"(0u), "r"(0u), "r"(0u), "r"(0u));
|
||||
}
|
||||
|
||||
__device__ __forceinline__ void tcgen05_mma_f16_ts_1sm_lead(uint32_t lead,
|
||||
uint32_t tmem_c, uint32_t tmem_a, uint64_t desc_b, uint32_t idesc,
|
||||
bool enable_input_d) {
|
||||
asm volatile(
|
||||
"{\n\t"
|
||||
".reg .pred p, q;\n\t"
|
||||
"setp.ne.b32 q, %0, 0;\n\t"
|
||||
"setp.ne.b32 p, %5, 0;\n\t"
|
||||
"@q tcgen05.mma.cta_group::1.kind::f16 [%1], [%2], %3, %4, {%6, %7, %8, %9}, p;\n\t"
|
||||
"}\n"
|
||||
:: "r"(lead), "r"(tmem_c), "r"(tmem_a), "l"(desc_b), "r"(idesc),
|
||||
"r"(enable_input_d ? 1u : 0u), "r"(0u), "r"(0u), "r"(0u), "r"(0u));
|
||||
}
|
||||
|
||||
enum class SmemSwizzleBlackwell : uint32_t {
|
||||
None = 0,
|
||||
B128_32atom = 1,
|
||||
B128 = 2,
|
||||
B64 = 4,
|
||||
B32 = 6,
|
||||
};
|
||||
|
||||
__device__ __host__ __forceinline__ uint64_t build_smem_desc_blackwell(
|
||||
uint32_t smem_addr,
|
||||
uint32_t stride_byte_offset,
|
||||
uint32_t leading_byte_offset,
|
||||
SmemSwizzleBlackwell swizzle = SmemSwizzleBlackwell::B128,
|
||||
uint32_t base_offset = 0) {
|
||||
uint64_t d = 0;
|
||||
d |= static_cast<uint64_t>((smem_addr >> 4) & 0x3FFF);
|
||||
d |= static_cast<uint64_t>((leading_byte_offset >> 4) & 0x3FFF) << 16;
|
||||
d |= static_cast<uint64_t>((stride_byte_offset >> 4) & 0x3FFF) << 32;
|
||||
d |= static_cast<uint64_t>(1) << 46;
|
||||
d |= (static_cast<uint64_t>(base_offset) & 0x7) << 49;
|
||||
d |= static_cast<uint64_t>(static_cast<uint32_t>(swizzle) & 0x7) << 61;
|
||||
return d;
|
||||
}
|
||||
|
||||
__device__ __forceinline__
|
||||
uint32_t elect_one_sync() {
|
||||
uint32_t elected;
|
||||
asm volatile(
|
||||
"{\n\t"
|
||||
".reg .pred p;\n\t"
|
||||
"elect.sync %0|p, 0xffffffff;\n\t"
|
||||
"selp.b32 %0, 1, 0, p;\n\t"
|
||||
"}\n"
|
||||
: "=r"(elected));
|
||||
return elected;
|
||||
}
|
||||
|
||||
__device__ __forceinline__
|
||||
uint32_t elect_one_sync(uint32_t membermask) {
|
||||
uint32_t elected;
|
||||
asm volatile(
|
||||
"{\n\t"
|
||||
".reg .pred p;\n\t"
|
||||
"elect.sync %0|p, %1;\n\t"
|
||||
"selp.b32 %0, 1, 0, p;\n\t"
|
||||
"}\n"
|
||||
: "=r"(elected) : "r"(membermask));
|
||||
return elected;
|
||||
}
|
||||
|
||||
template <int N>
|
||||
__device__ __forceinline__
|
||||
void setmaxnreg_dec() {
|
||||
static_assert(N >= 24 && N <= 256 && N % 8 == 0,
|
||||
"setmaxnreg_dec: N must be in [24, 256] and a multiple of 8");
|
||||
asm volatile("setmaxnreg.dec.sync.aligned.u32 %0;\n"
|
||||
:: "n"(N) : "memory");
|
||||
}
|
||||
|
||||
template <int N>
|
||||
__device__ __forceinline__
|
||||
void setmaxnreg_inc() {
|
||||
static_assert(N >= 24 && N <= 256 && N % 8 == 0,
|
||||
"setmaxnreg_inc: N must be in [24, 256] and a multiple of 8");
|
||||
asm volatile("setmaxnreg.inc.sync.aligned.u32 %0;\n"
|
||||
:: "n"(N) : "memory");
|
||||
}
|
||||
|
||||
__device__ __forceinline__
|
||||
uint32_t cvt_f32x2_to_bf16x2(float a, float b) {
|
||||
uint32_t r;
|
||||
asm volatile("cvt.rn.bf16x2.f32 %0, %2, %1;\n"
|
||||
: "=r"(r) : "f"(a), "f"(b));
|
||||
return r;
|
||||
}
|
||||
|
||||
namespace {
|
||||
__device__ __forceinline__ uint64_t f32x2_bits(float2 v) {
|
||||
uint64_t b; __builtin_memcpy(&b, &v, 8); return b;
|
||||
}
|
||||
__device__ __forceinline__ float2 f32x2_make(uint64_t b) {
|
||||
float2 v; __builtin_memcpy(&v, &b, 8); return v;
|
||||
}
|
||||
}
|
||||
|
||||
__device__ __forceinline__ float2 fmul2(float2 a, float2 b) {
|
||||
uint64_t d;
|
||||
asm volatile("mul.f32x2 %0, %1, %2;\n" : "=l"(d) : "l"(f32x2_bits(a)), "l"(f32x2_bits(b)));
|
||||
return f32x2_make(d);
|
||||
}
|
||||
|
||||
__device__ __forceinline__ float2 fadd2(float2 a, float2 b) {
|
||||
uint64_t d;
|
||||
asm volatile("add.f32x2 %0, %1, %2;\n" : "=l"(d) : "l"(f32x2_bits(a)), "l"(f32x2_bits(b)));
|
||||
return f32x2_make(d);
|
||||
}
|
||||
|
||||
__device__ __forceinline__ float2 ffma2(float2 a, float2 b, float2 c) {
|
||||
uint64_t d;
|
||||
asm volatile("fma.rn.f32x2 %0, %1, %2, %3;\n"
|
||||
: "=l"(d) : "l"(f32x2_bits(a)), "l"(f32x2_bits(b)), "l"(f32x2_bits(c)));
|
||||
return f32x2_make(d);
|
||||
}
|
||||
|
||||
__device__ __forceinline__ float2 f32x2_splat(float s) { return make_float2(s, s); }
|
||||
|
||||
__device__ __forceinline__ float ex2_approx_f32(float z) {
|
||||
float d;
|
||||
asm volatile("ex2.approx.ftz.f32 %0, %1;\n" : "=f"(d) : "f"(z));
|
||||
return d;
|
||||
}
|
||||
|
||||
__device__ __forceinline__ float2 ex2_emu_f32x2(float x, float y) {
|
||||
uint32_t ox, oy;
|
||||
asm volatile(
|
||||
"{\n\t"
|
||||
".reg .f32 f1,f2,f3,f4,f5,f6,f7;\n\t"
|
||||
".reg .b64 l1,l2,l3,l4,l5,l6,l7,l8,l9,l10;\n\t"
|
||||
".reg .s32 r1,r2,r3,r4,r5,r6,r7,r8;\n\t"
|
||||
"max.f32 f1, %2, 0fC2FE0000;\n\t"
|
||||
"max.f32 f2, %3, 0fC2FE0000;\n\t"
|
||||
"mov.b64 l1, {f1, f2};\n\t"
|
||||
"mov.f32 f3, 0f4B400000;\n\t"
|
||||
"mov.b64 l2, {f3, f3};\n\t"
|
||||
"add.rm.f32x2 l7, l1, l2;\n\t"
|
||||
"sub.rn.f32x2 l8, l7, l2;\n\t"
|
||||
"sub.rn.f32x2 l9, l1, l8;\n\t"
|
||||
"mov.f32 f7, 0f3D9DF09D;\n\t"
|
||||
"mov.b64 l6, {f7, f7};\n\t"
|
||||
"mov.f32 f6, 0f3E6906A4;\n\t"
|
||||
"mov.b64 l5, {f6, f6};\n\t"
|
||||
"mov.f32 f5, 0f3F31F519;\n\t"
|
||||
"mov.b64 l4, {f5, f5};\n\t"
|
||||
"mov.f32 f4, 0f3F800000;\n\t"
|
||||
"mov.b64 l3, {f4, f4};\n\t"
|
||||
"fma.rn.f32x2 l10, l9, l6, l5;\n\t"
|
||||
"fma.rn.f32x2 l10, l10, l9, l4;\n\t"
|
||||
"fma.rn.f32x2 l10, l10, l9, l3;\n\t"
|
||||
"mov.b64 {r1, r2}, l7;\n\t"
|
||||
"mov.b64 {r3, r4}, l10;\n\t"
|
||||
"shl.b32 r5, r1, 23;\n\t"
|
||||
"add.s32 r7, r5, r3;\n\t"
|
||||
"shl.b32 r6, r2, 23;\n\t"
|
||||
"add.s32 r8, r6, r4;\n\t"
|
||||
"mov.b32 %0, r7;\n\t"
|
||||
"mov.b32 %1, r8;\n\t"
|
||||
"}\n"
|
||||
: "=r"(ox), "=r"(oy) : "f"(x), "f"(y));
|
||||
float2 r; __builtin_memcpy(&r.x, &ox, 4); __builtin_memcpy(&r.y, &oy, 4);
|
||||
return r;
|
||||
}
|
||||
|
||||
__device__ __forceinline__ float rcp_approx_ftz_f32(float x) {
|
||||
float d;
|
||||
asm("rcp.approx.ftz.f32 %0, %1;\n" : "=f"(d) : "f"(x));
|
||||
return d;
|
||||
}
|
||||
|
||||
__device__ __forceinline__ uint32_t make_idesc_table44(
|
||||
int M, int N,
|
||||
uint32_t dtype, uint32_t atype, uint32_t btype,
|
||||
bool transpose_a = false, bool transpose_b = false,
|
||||
bool negate_a = false, bool negate_b = false) {
|
||||
uint32_t idesc = 0;
|
||||
idesc |= (dtype & 0x3) << 4;
|
||||
idesc |= (atype & 0x7) << 7;
|
||||
idesc |= (btype & 0x7) << 10;
|
||||
idesc |= (negate_a ? 1u : 0u) << 13;
|
||||
idesc |= (negate_b ? 1u : 0u) << 14;
|
||||
idesc |= (transpose_a ? 1u : 0u) << 15;
|
||||
idesc |= (transpose_b ? 1u : 0u) << 16;
|
||||
idesc |= ((static_cast<uint32_t>(N) >> 3) & 0x3F) << 17;
|
||||
idesc |= ((static_cast<uint32_t>(M) >> 4) & 0x1F) << 24;
|
||||
return idesc;
|
||||
}
|
||||
|
||||
__device__ __forceinline__ uint32_t make_idesc_bf16_f32(
|
||||
int M, int N, bool ta = false, bool tb = false) {
|
||||
return make_idesc_table44(M, N, 1,
|
||||
1, 1, ta, tb);
|
||||
}
|
||||
|
||||
__device__ __forceinline__ void tcgen05_ld_32x32b_x16(
|
||||
uint32_t tmem_addr, uint32_t (&r)[16]) {
|
||||
asm volatile("tcgen05.ld.sync.aligned.32x32b.x16.b32 "
|
||||
"{%0,%1,%2,%3,%4,%5,%6,%7,%8,%9,"
|
||||
"%10,%11,%12,%13,%14,%15}, [%16];\n"
|
||||
: "=r"(r[0]),"=r"(r[1]),"=r"(r[2]),"=r"(r[3]),
|
||||
"=r"(r[4]),"=r"(r[5]),"=r"(r[6]),"=r"(r[7]),
|
||||
"=r"(r[8]),"=r"(r[9]),"=r"(r[10]),"=r"(r[11]),
|
||||
"=r"(r[12]),"=r"(r[13]),"=r"(r[14]),"=r"(r[15])
|
||||
: "r"(tmem_addr));
|
||||
}
|
||||
|
||||
__device__ __forceinline__ void tcgen05_ld_32x32b_x32(
|
||||
uint32_t tmem_addr, uint32_t (&r)[32]) {
|
||||
asm volatile("tcgen05.ld.sync.aligned.32x32b.x32.b32 "
|
||||
"{%0,%1,%2,%3,%4,%5,%6,%7,%8,%9,"
|
||||
"%10,%11,%12,%13,%14,%15,%16,%17,%18,%19,"
|
||||
"%20,%21,%22,%23,%24,%25,%26,%27,%28,%29,"
|
||||
"%30,%31}, [%32];\n"
|
||||
: "=r"(r[0]),"=r"(r[1]),"=r"(r[2]),"=r"(r[3]),
|
||||
"=r"(r[4]),"=r"(r[5]),"=r"(r[6]),"=r"(r[7]),
|
||||
"=r"(r[8]),"=r"(r[9]),"=r"(r[10]),"=r"(r[11]),
|
||||
"=r"(r[12]),"=r"(r[13]),"=r"(r[14]),"=r"(r[15]),
|
||||
"=r"(r[16]),"=r"(r[17]),"=r"(r[18]),"=r"(r[19]),
|
||||
"=r"(r[20]),"=r"(r[21]),"=r"(r[22]),"=r"(r[23]),
|
||||
"=r"(r[24]),"=r"(r[25]),"=r"(r[26]),"=r"(r[27]),
|
||||
"=r"(r[28]),"=r"(r[29]),"=r"(r[30]),"=r"(r[31])
|
||||
: "r"(tmem_addr));
|
||||
}
|
||||
|
||||
__device__ __forceinline__ void tcgen05_ld_32x32b_x64(
|
||||
uint32_t tmem_addr, uint32_t (&r)[64]) {
|
||||
asm volatile("tcgen05.ld.sync.aligned.32x32b.x64.b32 "
|
||||
"{%0,%1,%2,%3,%4,%5,%6,%7,%8,%9,"
|
||||
"%10,%11,%12,%13,%14,%15,%16,%17,%18,%19,"
|
||||
"%20,%21,%22,%23,%24,%25,%26,%27,%28,%29,"
|
||||
"%30,%31,%32,%33,%34,%35,%36,%37,%38,%39,"
|
||||
"%40,%41,%42,%43,%44,%45,%46,%47,%48,%49,"
|
||||
"%50,%51,%52,%53,%54,%55,%56,%57,%58,%59,"
|
||||
"%60,%61,%62,%63}, [%64];\n"
|
||||
: "=r"(r[0]),"=r"(r[1]),"=r"(r[2]),"=r"(r[3]),
|
||||
"=r"(r[4]),"=r"(r[5]),"=r"(r[6]),"=r"(r[7]),
|
||||
"=r"(r[8]),"=r"(r[9]),"=r"(r[10]),"=r"(r[11]),
|
||||
"=r"(r[12]),"=r"(r[13]),"=r"(r[14]),"=r"(r[15]),
|
||||
"=r"(r[16]),"=r"(r[17]),"=r"(r[18]),"=r"(r[19]),
|
||||
"=r"(r[20]),"=r"(r[21]),"=r"(r[22]),"=r"(r[23]),
|
||||
"=r"(r[24]),"=r"(r[25]),"=r"(r[26]),"=r"(r[27]),
|
||||
"=r"(r[28]),"=r"(r[29]),"=r"(r[30]),"=r"(r[31]),
|
||||
"=r"(r[32]),"=r"(r[33]),"=r"(r[34]),"=r"(r[35]),
|
||||
"=r"(r[36]),"=r"(r[37]),"=r"(r[38]),"=r"(r[39]),
|
||||
"=r"(r[40]),"=r"(r[41]),"=r"(r[42]),"=r"(r[43]),
|
||||
"=r"(r[44]),"=r"(r[45]),"=r"(r[46]),"=r"(r[47]),
|
||||
"=r"(r[48]),"=r"(r[49]),"=r"(r[50]),"=r"(r[51]),
|
||||
"=r"(r[52]),"=r"(r[53]),"=r"(r[54]),"=r"(r[55]),
|
||||
"=r"(r[56]),"=r"(r[57]),"=r"(r[58]),"=r"(r[59]),
|
||||
"=r"(r[60]),"=r"(r[61]),"=r"(r[62]),"=r"(r[63])
|
||||
: "r"(tmem_addr));
|
||||
}
|
||||
|
||||
__device__ __forceinline__ void tcgen05_ld_32x32b_x128(
|
||||
uint32_t tmem_addr, uint32_t (&r)[128]) {
|
||||
asm volatile("tcgen05.ld.sync.aligned.32x32b.x128.b32 "
|
||||
"{%0,%1,%2,%3,%4,%5,%6,%7,%8,%9,"
|
||||
"%10,%11,%12,%13,%14,%15,%16,%17,%18,%19,"
|
||||
"%20,%21,%22,%23,%24,%25,%26,%27,%28,%29,"
|
||||
"%30,%31,%32,%33,%34,%35,%36,%37,%38,%39,"
|
||||
"%40,%41,%42,%43,%44,%45,%46,%47,%48,%49,"
|
||||
"%50,%51,%52,%53,%54,%55,%56,%57,%58,%59,"
|
||||
"%60,%61,%62,%63,%64,%65,%66,%67,%68,%69,"
|
||||
"%70,%71,%72,%73,%74,%75,%76,%77,%78,%79,"
|
||||
"%80,%81,%82,%83,%84,%85,%86,%87,%88,%89,"
|
||||
"%90,%91,%92,%93,%94,%95,%96,%97,%98,%99,"
|
||||
"%100,%101,%102,%103,%104,%105,%106,%107,%108,%109,"
|
||||
"%110,%111,%112,%113,%114,%115,%116,%117,%118,%119,"
|
||||
"%120,%121,%122,%123,%124,%125,%126,%127}, [%128];\n"
|
||||
: "=r"(r[ 0]),"=r"(r[ 1]),"=r"(r[ 2]),"=r"(r[ 3]),
|
||||
"=r"(r[ 4]),"=r"(r[ 5]),"=r"(r[ 6]),"=r"(r[ 7]),
|
||||
"=r"(r[ 8]),"=r"(r[ 9]),"=r"(r[ 10]),"=r"(r[ 11]),
|
||||
"=r"(r[ 12]),"=r"(r[ 13]),"=r"(r[ 14]),"=r"(r[ 15]),
|
||||
"=r"(r[ 16]),"=r"(r[ 17]),"=r"(r[ 18]),"=r"(r[ 19]),
|
||||
"=r"(r[ 20]),"=r"(r[ 21]),"=r"(r[ 22]),"=r"(r[ 23]),
|
||||
"=r"(r[ 24]),"=r"(r[ 25]),"=r"(r[ 26]),"=r"(r[ 27]),
|
||||
"=r"(r[ 28]),"=r"(r[ 29]),"=r"(r[ 30]),"=r"(r[ 31]),
|
||||
"=r"(r[ 32]),"=r"(r[ 33]),"=r"(r[ 34]),"=r"(r[ 35]),
|
||||
"=r"(r[ 36]),"=r"(r[ 37]),"=r"(r[ 38]),"=r"(r[ 39]),
|
||||
"=r"(r[ 40]),"=r"(r[ 41]),"=r"(r[ 42]),"=r"(r[ 43]),
|
||||
"=r"(r[ 44]),"=r"(r[ 45]),"=r"(r[ 46]),"=r"(r[ 47]),
|
||||
"=r"(r[ 48]),"=r"(r[ 49]),"=r"(r[ 50]),"=r"(r[ 51]),
|
||||
"=r"(r[ 52]),"=r"(r[ 53]),"=r"(r[ 54]),"=r"(r[ 55]),
|
||||
"=r"(r[ 56]),"=r"(r[ 57]),"=r"(r[ 58]),"=r"(r[ 59]),
|
||||
"=r"(r[ 60]),"=r"(r[ 61]),"=r"(r[ 62]),"=r"(r[ 63]),
|
||||
"=r"(r[ 64]),"=r"(r[ 65]),"=r"(r[ 66]),"=r"(r[ 67]),
|
||||
"=r"(r[ 68]),"=r"(r[ 69]),"=r"(r[ 70]),"=r"(r[ 71]),
|
||||
"=r"(r[ 72]),"=r"(r[ 73]),"=r"(r[ 74]),"=r"(r[ 75]),
|
||||
"=r"(r[ 76]),"=r"(r[ 77]),"=r"(r[ 78]),"=r"(r[ 79]),
|
||||
"=r"(r[ 80]),"=r"(r[ 81]),"=r"(r[ 82]),"=r"(r[ 83]),
|
||||
"=r"(r[ 84]),"=r"(r[ 85]),"=r"(r[ 86]),"=r"(r[ 87]),
|
||||
"=r"(r[ 88]),"=r"(r[ 89]),"=r"(r[ 90]),"=r"(r[ 91]),
|
||||
"=r"(r[ 92]),"=r"(r[ 93]),"=r"(r[ 94]),"=r"(r[ 95]),
|
||||
"=r"(r[ 96]),"=r"(r[ 97]),"=r"(r[ 98]),"=r"(r[ 99]),
|
||||
"=r"(r[100]),"=r"(r[101]),"=r"(r[102]),"=r"(r[103]),
|
||||
"=r"(r[104]),"=r"(r[105]),"=r"(r[106]),"=r"(r[107]),
|
||||
"=r"(r[108]),"=r"(r[109]),"=r"(r[110]),"=r"(r[111]),
|
||||
"=r"(r[112]),"=r"(r[113]),"=r"(r[114]),"=r"(r[115]),
|
||||
"=r"(r[116]),"=r"(r[117]),"=r"(r[118]),"=r"(r[119]),
|
||||
"=r"(r[120]),"=r"(r[121]),"=r"(r[122]),"=r"(r[123]),
|
||||
"=r"(r[124]),"=r"(r[125]),"=r"(r[126]),"=r"(r[127])
|
||||
: "r"(tmem_addr));
|
||||
}
|
||||
|
||||
__device__ __forceinline__
|
||||
uint32_t smem_ptr_u32(const void* ptr) {
|
||||
return static_cast<uint32_t>(__cvta_generic_to_shared(ptr));
|
||||
}
|
||||
|
||||
__device__ __forceinline__
|
||||
void sts_f32(uint32_t smem_addr, float val) {
|
||||
asm volatile("st.shared.f32 [%0], %1;" :: "r"(smem_addr), "f"(val) : "memory");
|
||||
}
|
||||
@@ -252,6 +252,13 @@ _SPECS: tuple[tuple[str, KernelBackend, str, frozenset, str], ...] = (
|
||||
_CUDA,
|
||||
"Fused in-place QK RMS-norm + RoPE.",
|
||||
),
|
||||
(
|
||||
"diffusion.vsa_block_sparse_sm100",
|
||||
KernelBackend.JIT,
|
||||
"attention.vsa_block_sparse_sm100_jit:vsa_block_sparse_sm100",
|
||||
frozenset({CapabilityRequirement.cuda(min_sm=(10, 0), max_sm=(10, 3))}),
|
||||
"FastVideo's warp-specialized tcgen05 block-sparse VSA forward (64-token tiles).",
|
||||
),
|
||||
(
|
||||
"diffusion.h3_vae_rmsnorm",
|
||||
KernelBackend.TRITON,
|
||||
@@ -715,6 +722,8 @@ _EXPORTS: dict[str, str] = {
|
||||
"prepare_rope_tables": "attention.sana_wm_gdn_triton",
|
||||
"_attn_fwd": "attention.sparse_linear_attn_triton",
|
||||
"get_block_map": "attention.sparse_linear_attn_triton",
|
||||
"can_use_vsa_block_sparse_sm100": "attention.vsa_block_sparse_sm100_jit",
|
||||
"vsa_block_sparse_sm100": "attention.vsa_block_sparse_sm100_jit",
|
||||
# MoE routing
|
||||
"can_use_group_limited_topk": "routing.group_limited_topk_triton",
|
||||
"group_limited_topk": "routing.group_limited_topk_triton",
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
"""FastVideo's native Blackwell block-sparse VSA forward (64-token tiles).
|
||||
|
||||
A CTA owns two adjacent query tiles, so the tile count must be even. Requires
|
||||
sm_100a or sm_103a, bf16, head_dim 128 and contiguous [B, H, S, D].
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.jit.utils import cache_once, load_jit
|
||||
from sglang.srt.utils.custom_op import register_custom_op
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from tvm_ffi.module import Module
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_SUPPORTED_CAPABILITIES = ((10, 0), (10, 3))
|
||||
_HEAD_DIM = 128
|
||||
|
||||
|
||||
@cache_once
|
||||
def _module() -> Module:
|
||||
return load_jit(
|
||||
"vsa_block_sparse_sm100",
|
||||
cuda_files=["diffusion/vsa_block_sparse_sm100.cuh"],
|
||||
cuda_wrappers=[("vsa_block_sparse_sm100", "VsaBlockSparseSm100Kernel::run")],
|
||||
extra_ldflags=["-lcuda"],
|
||||
)
|
||||
|
||||
|
||||
@cache_once
|
||||
def can_use_vsa_block_sparse_sm100(
|
||||
device_index: int, dtype: torch.dtype, head_dim: int
|
||||
) -> bool:
|
||||
if dtype != torch.bfloat16 or head_dim != _HEAD_DIM:
|
||||
return False
|
||||
if torch.cuda.get_device_capability(device_index) not in _SUPPORTED_CAPABILITIES:
|
||||
return False
|
||||
try:
|
||||
_module()
|
||||
except Exception as e:
|
||||
logger.warning("Failed to load the JIT VSA block-sparse sm100 kernel: %s", e)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
@register_custom_op(mutates_args=["out"])
|
||||
def vsa_block_sparse_sm100(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
q2k_idx: torch.Tensor,
|
||||
q2k_num: torch.Tensor,
|
||||
block_sizes: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
sm_scale: float,
|
||||
) -> None:
|
||||
"""``out[b, h, i*64:(i+1)*64]`` = attention of query tile ``i`` over its
|
||||
``q2k_num`` listed key tiles, keys past each tile's ``block_sizes`` masked."""
|
||||
_module().vsa_block_sparse_sm100(
|
||||
q, k, v, q2k_idx, q2k_num, block_sizes, out, sm_scale
|
||||
)
|
||||
+1
-1
@@ -314,7 +314,7 @@ Use the preset categories this way:
|
||||
| `cosmos3-super-t2v-cfg2tp2` | `nvidia/Cosmos3-Super` | No | Explicit four-GPU TP2 x CFG2 throughput comparator. On H200 it was 48.00% faster end to end than TP2, but the topology changed the deterministic output (SSIM 0.914244, PSNR 29.469771 dB), so do not treat it as lossless-equivalent or select it automatically. |
|
||||
| `wan-i2v` | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | Yes: `wan22_i2v_a14b_720p` | Nightly cat image and motion prompt, 1280x720, 81 frames, 4 GPUs, CFG parallel, Ulysses degree 2, text encoder CPU offload and pinned CPU memory |
|
||||
| `minimax-h3-t2va` | `MiniMaxAI/MiniMax-H3` | Yes: `minimax_h3_t2va_5s` | H3 FL2VA-partition T2VA baseline: 1344x768 resolved canvas, 5 seconds / 124 frames at 24 fps, 50 joint video-audio steps, 4 GPUs, TP2 + Ulysses2, eager BF16/FP32. The helper writes H3's request contract to a generated config. |
|
||||
| `fasth3-t2va-vsa` | `FastVideo/FastVideo-FastH3-4-step-Preview-v1-VSA-DataFree` | No | FastH3 4-step distilled T2VA on the trained VSA-H3 backend: 1344x768, 10 seconds / 243 frames, five sigma points = four DiT forwards, 4 GPUs, Ulysses 4, eager, 2-step warmup request. Compare against `--attention-backend fa` on the same weights for the dense-fallback gap. |
|
||||
| `fasth3-t2va-vsa` | `FastVideo/FastVideo-FastH3-8-Step-V2` | No | FastH3 8-step distilled T2VA on the trained VSA-H3 backend (sparsity 0.8): 1344x768, 10 seconds / 243 frames, nine sigma points = eight DiT forwards, 4 GPUs, Ulysses 4, eager, 2-step warmup request. Compare against `--attention-backend fa` on the same weights for the dense-fallback gap. |
|
||||
| `longcat-image` | `meituan-longcat/LongCat-Image` | No | Eager DiT baseline at 1024x1024, 50 steps, guidance 4.5; prompt rewrite is disabled so Qwen2.5-VL does not contaminate the DiT A/B. |
|
||||
| `longcat-image-edit` | `meituan-longcat/LongCat-Image-Edit` | No | Native edit baseline using the public SGLang edit fixture. Its 1536x1024 source resolves to 1264x848 under the checkpoint's roughly-one-megapixel aspect-ratio rule, and the BCG comparator captures that exact serving canvas; prompt rewrite is disabled to isolate the DiT. |
|
||||
| `longcat-image-edit-turbo` | `meituan-longcat/LongCat-Image-Edit-Turbo` | No | Matching distilled edit baseline using the same public fixture, prompt, and 1264x848 BCG canvas. Its registered sampling class owns the eight-step, guidance-1 schedule. |
|
||||
|
||||
+3
-4
@@ -427,7 +427,7 @@ MODELS = {
|
||||
},
|
||||
# H3 rejects a 1-step warmup request, hence --warmup-steps=2.
|
||||
"fasth3-t2va-vsa": {
|
||||
"path": "FastVideo/FastVideo-FastH3-4-step-Preview-v1-VSA-DataFree",
|
||||
"path": "FastVideo/FastVideo-FastH3-8-Step-V2",
|
||||
"prompt": (
|
||||
"A curious raccoon peers through a vibrant field of yellow "
|
||||
"sunflowers, its eyes wide with interest."
|
||||
@@ -441,12 +441,11 @@ MODELS = {
|
||||
"aspect_ratio": "16:9",
|
||||
"duration_seconds": 10.0,
|
||||
},
|
||||
"num_inference_steps": 5,
|
||||
"num_inference_steps": 9,
|
||||
},
|
||||
"extra_args": [
|
||||
"--num-gpus=4",
|
||||
"--attention-backend=video_sparse_attn_h3",
|
||||
'--attention-backend-config={"VSA_sparsity": 0.9}',
|
||||
"--component-attention-backends=transformer=video_sparse_attn_h3",
|
||||
"--enable-torch-compile=false",
|
||||
"--warmup-steps=2",
|
||||
],
|
||||
|
||||
@@ -37,6 +37,9 @@ SGLang Diffusion supports Apple Silicon (M-series) via the MPS backend. Since Tr
|
||||
|
||||
Docker is recommended for Linux GPU deployments. Follow the [installation guide](https://docs.sglang.io/docs/sglang-diffusion/installation) to start a container with diffusion dependencies included, or choose pip/uv, source installation, or another platform's setup. Run the examples below inside that environment.
|
||||
|
||||
Video output requires a system `ffmpeg` with the `libx264` encoder on `PATH`. For Ubuntu source or pip installations, install it with `sudo apt-get install ffmpeg`; on macOS, use `brew install ffmpeg`.
|
||||
|
||||
|
||||
## Inference
|
||||
|
||||
Here's a minimal example to generate a video using the default settings:
|
||||
|
||||
@@ -33,6 +33,8 @@ class MiniMaxH3VideoVAEConfig(VAEConfig):
|
||||
load_decoder: bool = True
|
||||
use_tiling: bool = True
|
||||
use_parallel_tiling: bool = True
|
||||
# Decode each rank's spatial tiles in one batched call instead of one by one.
|
||||
stack_tiling: bool = False
|
||||
# The released checkpoint's quality contract uses overlapping latent
|
||||
# tiles. Parallel tiling distributes whole tiles without changing that
|
||||
# recipe. Spatial-shard decode is rejected because validation found output
|
||||
|
||||
@@ -57,6 +57,8 @@ class MiniMaxH3PipelineConfig(PipelineConfig):
|
||||
vae_precision: str = "fp32"
|
||||
vae_decode_precision: str = "fp16"
|
||||
audio_vae_precision: str = "fp32"
|
||||
# Default VSA-H3 sparsity; --attention-backend-config VSA_sparsity overrides.
|
||||
vsa_sparsity: float = 0.9
|
||||
text_encoder_configs: tuple[MiniMaxH3Qwen3VLConfig, ...] = field(
|
||||
default_factory=lambda: (MiniMaxH3Qwen3VLConfig(),)
|
||||
)
|
||||
@@ -333,16 +335,20 @@ class MiniMaxH3PipelineConfig(PipelineConfig):
|
||||
|
||||
@dataclass
|
||||
class FastH3PipelineConfig(MiniMaxH3PipelineConfig):
|
||||
"""FastH3: 4-step VSA-distilled MiniMax-H3, t2va only."""
|
||||
"""FastH3 8-Step V2: VSA-distilled MiniMax-H3, t2va only."""
|
||||
|
||||
# The checkpoint's trained sparsity (fastvideo_inference.json).
|
||||
vsa_sparsity: float = 0.8
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
self.dit_config.arch_config.has_gate_compress = True
|
||||
self.vae_config.stack_tiling = True
|
||||
|
||||
def validate_quality_deployment(self, server_args) -> None:
|
||||
raise ValueError(
|
||||
'quality="high" is audited only for the base MiniMax-H3 50-step '
|
||||
"4xH200 deployment; the FastH3 4-step distilled checkpoint has no "
|
||||
'audited high-quality deployment. Use quality="exact".'
|
||||
"4xH200 deployment; FastH3 has no audited high-quality deployment. "
|
||||
'Use quality="exact", or the default "lossless".'
|
||||
)
|
||||
|
||||
def validate_server_args(self, server_args) -> None:
|
||||
@@ -383,7 +389,7 @@ def register():
|
||||
sampling_param_cls=FastH3SamplingParams,
|
||||
pipeline_config_cls=FastH3PipelineConfig,
|
||||
hf_model_paths=[
|
||||
"FastVideo/FastVideo-FastH3-4-step-Preview-v1-VSA-DataFree",
|
||||
"FastVideo/FastVideo-FastH3-8-Step-V2",
|
||||
],
|
||||
model_detectors=[
|
||||
lambda model_id: (
|
||||
|
||||
@@ -357,15 +357,16 @@ class MiniMaxH3SamplingParams(SamplingParams):
|
||||
|
||||
@dataclass
|
||||
class FastH3SamplingParams(MiniMaxH3SamplingParams):
|
||||
"""FastH3: five sigma grid points, i.e. the four distilled DiT forwards."""
|
||||
"""FastH3 8-Step V2: nine sigma points, i.e. eight DiT forwards on the trained rungs."""
|
||||
|
||||
num_inference_steps: int = 5
|
||||
num_inference_steps: int = 9
|
||||
quality: str = "lossless"
|
||||
|
||||
def _validate(self) -> None:
|
||||
super()._validate()
|
||||
if self.num_inference_steps != 5:
|
||||
if self.num_inference_steps != 9:
|
||||
raise ValueError(
|
||||
"FastH3 is distilled for exactly five sigma grid points (four DiT "
|
||||
"FastH3 is distilled for exactly nine sigma grid points (eight DiT "
|
||||
f"forwards); got num_inference_steps={self.num_inference_steps}. "
|
||||
"Use MiniMaxAI/MiniMax-H3 for other schedules."
|
||||
)
|
||||
|
||||
@@ -176,7 +176,7 @@ KNOWN_NON_DIFFUSERS_DIFFUSION_MODEL_PATTERNS: Dict[str, str] = {
|
||||
"ming-image-0.1-design": "MingImagePipeline",
|
||||
"minimaxai/minimax-h3": "MiniMaxH3Pipeline",
|
||||
"minimax/minimax-h3": "MiniMaxH3Pipeline",
|
||||
"fastvideo/fastvideo-fasth3-4-step-preview-v1-vsa-datafree": "FastH3Pipeline",
|
||||
"fastvideo/fastvideo-fasth3-8-step-v2": "FastH3Pipeline",
|
||||
"openvdn/vdn-minimax-h3": "VDNH3Pipeline",
|
||||
"lerobot/pi05": "Pi05Pipeline",
|
||||
"pi05": "Pi05Pipeline",
|
||||
|
||||
@@ -13,7 +13,6 @@ import json
|
||||
import mmap
|
||||
import os
|
||||
import queue
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
import threading
|
||||
@@ -23,6 +22,7 @@ from dataclasses import dataclass, field
|
||||
from typing import Any, Callable, List, Optional, Sequence, Union
|
||||
|
||||
import imageio
|
||||
import imageio_ffmpeg
|
||||
import numpy as np
|
||||
import torch
|
||||
from PIL import Image
|
||||
@@ -32,11 +32,6 @@ try:
|
||||
except ImportError: # pragma: no cover
|
||||
scipy_wavfile = None
|
||||
|
||||
try:
|
||||
import imageio_ffmpeg as _imageio_ffmpeg
|
||||
except ImportError: # pragma: no cover
|
||||
_imageio_ffmpeg = None
|
||||
|
||||
from sglang.multimodal_gen.configs.sample.sampling_params import (
|
||||
DataType,
|
||||
SamplingParams,
|
||||
@@ -300,25 +295,8 @@ def _pick_audio_sample_rate(
|
||||
|
||||
|
||||
def _resolve_ffmpeg_exe() -> str:
|
||||
ffmpeg_exe = "ffmpeg"
|
||||
ffmpeg_on_path = shutil.which("ffmpeg")
|
||||
if ffmpeg_on_path:
|
||||
ffmpeg_exe = ffmpeg_on_path
|
||||
try:
|
||||
if _imageio_ffmpeg is not None:
|
||||
ffmpeg_exe = _imageio_ffmpeg.get_ffmpeg_exe()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
ffmpeg_ok = False
|
||||
if ffmpeg_exe:
|
||||
if os.path.isabs(ffmpeg_exe):
|
||||
ffmpeg_ok = os.path.exists(ffmpeg_exe)
|
||||
else:
|
||||
ffmpeg_ok = shutil.which(ffmpeg_exe) is not None
|
||||
if not ffmpeg_ok:
|
||||
raise RuntimeError("ffmpeg not found")
|
||||
return ffmpeg_exe
|
||||
# keep the pinned encoder binary: x264 upgrades change lossy video pixels
|
||||
return imageio_ffmpeg.get_ffmpeg_exe()
|
||||
|
||||
|
||||
# ffmpeg's implicit libx264 default is `medium`. On diffusion output `fast` is
|
||||
@@ -716,7 +694,7 @@ def _try_save_cuda_video_direct(
|
||||
if encoder is not None:
|
||||
encoder.abort()
|
||||
logger.warning_once(
|
||||
"Direct CUDA video save failed; falling back to imageio. "
|
||||
"Direct CUDA video save failed; falling back to CPU ffmpeg encoding. "
|
||||
"Enable debug logging for exception details."
|
||||
)
|
||||
logger.debug("Direct CUDA video save failure", exc_info=True)
|
||||
@@ -894,6 +872,83 @@ def _maybe_mux_audio_into_mp4(
|
||||
)
|
||||
|
||||
|
||||
def _save_video_ffmpeg(
|
||||
save_file_path: str,
|
||||
frames: list,
|
||||
*,
|
||||
fps: int,
|
||||
quality: float,
|
||||
x264_preset: Optional[str] = None,
|
||||
audio_path: Optional[str] = None,
|
||||
) -> None:
|
||||
if not frames:
|
||||
raise ValueError("video output requires at least one frame")
|
||||
if not 1 <= quality <= 10:
|
||||
raise ValueError("video quality must be between 1 and 10")
|
||||
first_frame = np.asarray(frames[0])
|
||||
height, width = first_frame.shape[:2]
|
||||
channels = first_frame.shape[2] if first_frame.ndim == 3 else 1
|
||||
pixel_format = {1: "gray", 3: "rgb24", 4: "rgba"}[channels]
|
||||
command = [
|
||||
_resolve_ffmpeg_exe(),
|
||||
"-y",
|
||||
"-f",
|
||||
"rawvideo",
|
||||
"-vcodec",
|
||||
"rawvideo",
|
||||
"-s",
|
||||
f"{width}x{height}",
|
||||
"-pix_fmt",
|
||||
pixel_format,
|
||||
"-r",
|
||||
f"{fps:.02f}",
|
||||
"-i",
|
||||
"pipe:0",
|
||||
]
|
||||
if audio_path is None:
|
||||
command += ["-an"]
|
||||
else:
|
||||
command += ["-i", audio_path, "-acodec", "aac"]
|
||||
command += [
|
||||
"-vcodec",
|
||||
"libx264",
|
||||
"-preset",
|
||||
x264_preset or X264_PRESET,
|
||||
"-pix_fmt",
|
||||
"yuv420p",
|
||||
"-crf",
|
||||
str(int((1 - quality / 10.0) * 51)),
|
||||
]
|
||||
if width % 16 or height % 16:
|
||||
command += [
|
||||
"-vf",
|
||||
f"scale={(width + 15) // 16 * 16}:{(height + 15) // 16 * 16}",
|
||||
]
|
||||
command += [save_file_path]
|
||||
with tempfile.TemporaryFile() as stderr:
|
||||
with subprocess.Popen(
|
||||
command,
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=stderr,
|
||||
) as process:
|
||||
try:
|
||||
for frame in frames:
|
||||
frame = np.asarray(frame)
|
||||
if frame.shape != first_frame.shape or frame.dtype != np.uint8:
|
||||
raise ValueError(
|
||||
"video frames must have matching shapes and uint8 dtype"
|
||||
)
|
||||
process.stdin.write(memoryview(np.ascontiguousarray(frame)))
|
||||
finally:
|
||||
process.stdin.close()
|
||||
if process.wait() != 0:
|
||||
stderr.seek(0)
|
||||
raise RuntimeError(
|
||||
f"ffmpeg video encoding failed: {stderr.read().decode(errors='replace')}"
|
||||
)
|
||||
|
||||
|
||||
def _try_save_video_with_audio(
|
||||
*,
|
||||
save_file_path: str,
|
||||
@@ -901,7 +956,6 @@ def _try_save_video_with_audio(
|
||||
fps: int,
|
||||
audio: Any,
|
||||
audio_sample_rate: Optional[int],
|
||||
output_format: str,
|
||||
quality: float,
|
||||
x264_preset: Optional[str] = None,
|
||||
) -> bool:
|
||||
@@ -925,16 +979,13 @@ def _try_save_video_with_audio(
|
||||
with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as f:
|
||||
tmp_wav_path = f.name
|
||||
scipy_wavfile.write(tmp_wav_path, selected_sr, audio_np)
|
||||
imageio.mimsave(
|
||||
_save_video_ffmpeg(
|
||||
save_file_path,
|
||||
frames,
|
||||
fps=fps,
|
||||
format=output_format,
|
||||
codec="libx264",
|
||||
quality=quality,
|
||||
audio_path=tmp_wav_path,
|
||||
audio_codec="aac",
|
||||
output_params=["-preset", x264_preset or X264_PRESET],
|
||||
x264_preset=x264_preset,
|
||||
)
|
||||
return True
|
||||
except Exception as e:
|
||||
@@ -1151,26 +1202,22 @@ def save_materialized_output(
|
||||
os.makedirs(os.path.dirname(save_file_path), exist_ok=True)
|
||||
if data_type == DataType.VIDEO:
|
||||
quality = output_compression / 10 if output_compression is not None else 5
|
||||
output_format = data_type.get_default_extension()
|
||||
saved_with_audio = _try_save_video_with_audio(
|
||||
save_file_path=save_file_path,
|
||||
frames=materialized.frames,
|
||||
fps=materialized.fps,
|
||||
audio=materialized.audio,
|
||||
audio_sample_rate=audio_sample_rate,
|
||||
output_format=output_format,
|
||||
quality=quality,
|
||||
x264_preset=x264_preset,
|
||||
)
|
||||
if not saved_with_audio:
|
||||
imageio.mimsave(
|
||||
_save_video_ffmpeg(
|
||||
save_file_path,
|
||||
materialized.frames,
|
||||
fps=materialized.fps,
|
||||
format=output_format,
|
||||
codec="libx264",
|
||||
quality=quality,
|
||||
output_params=["-preset", x264_preset or X264_PRESET],
|
||||
x264_preset=x264_preset,
|
||||
)
|
||||
|
||||
_maybe_mux_audio_into_mp4(
|
||||
|
||||
+91
-14
@@ -29,6 +29,7 @@ from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.diffusion import can_use_vsa_block_sparse_sm100
|
||||
from sglang.multimodal_gen.runtime.layers.attention.backends.attention_backend import (
|
||||
AttentionBackend,
|
||||
AttentionImpl,
|
||||
@@ -42,7 +43,9 @@ from sglang.multimodal_gen.runtime.layers.attention.backends.video_sparse_attn i
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.attention.backends.vsa_h3_kernels import (
|
||||
vsa_h3_block_sparse_attn_forward,
|
||||
vsa_h3_gate_add,
|
||||
vsa_h3_pack_tiles,
|
||||
vsa_h3_topk_lists,
|
||||
vsa_h3_untile,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum
|
||||
@@ -238,6 +241,7 @@ def _topk_tile_lists(
|
||||
prefix = num_prefix_tiles
|
||||
keep = _compute_topk(sparsity, num_video_tiles)
|
||||
video_rows = scores[:, prefix:, :]
|
||||
# serving takes vsa_h3_topk_lists for this branch; it stays as its reference
|
||||
if exempt or prefix == 0:
|
||||
picked = video_rows[:, :, prefix:].topk(keep, dim=-1).indices + prefix
|
||||
picked = picked.sort(dim=-1).values
|
||||
@@ -277,6 +281,8 @@ class _Workspace:
|
||||
"""Per-geometry scratch: tiled q/k/v(/gate) [3|4, H, S_pad, D], pooled fp32
|
||||
tile means [3, H, n_tiles, D], and the kernel index lists (prefix rows and
|
||||
prefix columns are static; only the top-k video columns change per layer).
|
||||
The native kernel pairs query tiles, so the tile axis is padded to an even
|
||||
count with one empty tile.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -289,9 +295,14 @@ class _Workspace:
|
||||
device: torch.device,
|
||||
) -> None:
|
||||
n_tiles = meta.num_tiles
|
||||
seq_pad = n_tiles * VSA_H3_TILE_ELEMS
|
||||
self.n_tiles = n_tiles
|
||||
self.use_sm100_kernel = can_use_vsa_block_sparse_sm100(
|
||||
device.index, dtype, head_dim
|
||||
)
|
||||
n_tiles_padded = n_tiles + (n_tiles % 2 if self.use_sm100_kernel else 0)
|
||||
seq_pad = n_tiles_padded * VSA_H3_TILE_ELEMS
|
||||
self.key = _workspace_key(meta, heads, head_dim, has_gate, dtype, device)
|
||||
self.tiled = torch.empty(
|
||||
self.tiled = torch.zeros(
|
||||
(3 + int(has_gate), heads, seq_pad, head_dim), dtype=dtype, device=device
|
||||
)
|
||||
self.pooled = torch.empty(
|
||||
@@ -300,23 +311,67 @@ class _Workspace:
|
||||
self.out_tiled = torch.empty(
|
||||
(heads, seq_pad, head_dim), dtype=dtype, device=device
|
||||
)
|
||||
self.block_sizes = torch.zeros(n_tiles_padded, dtype=torch.int32, device=device)
|
||||
self.block_sizes[:n_tiles] = meta.variable_block_sizes
|
||||
all_tiles = torch.arange(n_tiles, dtype=torch.int32, device=device)
|
||||
self.dense_index = all_tiles.repeat(heads, n_tiles, 1)
|
||||
self.dense_num = torch.full(
|
||||
(heads, n_tiles), n_tiles, dtype=torch.int32, device=device
|
||||
self.dense_index = torch.zeros(
|
||||
(heads, n_tiles_padded, n_tiles), dtype=torch.int32, device=device
|
||||
)
|
||||
self.dense_index[:, :n_tiles] = all_tiles
|
||||
self.dense_num = torch.zeros(
|
||||
(heads, n_tiles_padded), dtype=torch.int32, device=device
|
||||
)
|
||||
self.dense_num[:, :n_tiles] = n_tiles
|
||||
self.q2k_index = self.dense_index.clone()
|
||||
self.q2k_num = self.dense_num.clone()
|
||||
num_video_tiles = meta.num_video_tiles
|
||||
# Zeroed scratch for the exempt top-k compaction; kept zero between uses.
|
||||
self.topk_mask = torch.zeros(
|
||||
(heads, num_video_tiles, num_video_tiles), dtype=torch.int8, device=device
|
||||
)
|
||||
|
||||
def sparse_lists(
|
||||
self, video_lists: torch.Tensor, num_prefix_tiles: int
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
width = video_lists.shape[-1]
|
||||
self.q2k_index[:, num_prefix_tiles:, :width] = video_lists
|
||||
self.q2k_num[:, num_prefix_tiles:] = width
|
||||
self.q2k_index[:, num_prefix_tiles : self.n_tiles, :width] = video_lists
|
||||
self.q2k_num[:, num_prefix_tiles : self.n_tiles] = width
|
||||
return self.q2k_index, self.q2k_num
|
||||
|
||||
|
||||
def _gate_tile_index(
|
||||
meta: VideoSparseAttentionH3Metadata, row_start: int, num_rows: int
|
||||
) -> torch.Tensor:
|
||||
"""Tile of each packed row in the shard, -1 for pad rows."""
|
||||
key = ("gate_tile_index", row_start, num_rows)
|
||||
tile_index = meta.workspace_cache.get(key)
|
||||
if tile_index is None:
|
||||
unpack_index = meta.unpack_index
|
||||
rows = torch.arange(row_start, row_start + num_rows, device=unpack_index.device)
|
||||
tile_index = torch.full_like(rows, -1, dtype=torch.int32)
|
||||
in_used = rows < meta.total_seq_length
|
||||
tile_index[in_used] = unpack_index[rows[in_used]] // VSA_H3_TILE_ELEMS
|
||||
meta.workspace_cache[key] = tile_index
|
||||
return tile_index
|
||||
|
||||
|
||||
def vsa_h3_fold_gate(
|
||||
out: torch.Tensor,
|
||||
gate_compress: torch.Tensor,
|
||||
out_compress: torch.Tensor,
|
||||
meta: VideoSparseAttentionH3Metadata,
|
||||
row_start: int,
|
||||
) -> None:
|
||||
"""Fold the compression branch into ``out`` [rows, H, D], the row shard at
|
||||
``row_start``."""
|
||||
vsa_h3_gate_add(
|
||||
out,
|
||||
gate_compress,
|
||||
out_compress,
|
||||
_gate_tile_index(meta, row_start, out.shape[0]),
|
||||
)
|
||||
|
||||
|
||||
def _get_workspace(
|
||||
meta: VideoSparseAttentionH3Metadata, query: torch.Tensor, has_gate: bool
|
||||
) -> _Workspace:
|
||||
@@ -341,6 +396,15 @@ def _select_kv_lists(
|
||||
keep = _compute_topk(sparsity, meta.num_video_tiles)
|
||||
if sparsity <= 0.0 or keep >= meta.num_video_tiles:
|
||||
return ws.dense_index, ws.dense_num
|
||||
prefix = meta.num_prefix_tiles
|
||||
if meta.exempt or prefix == 0:
|
||||
# Same lists as _topk_tile_lists without sorting values or indices.
|
||||
picked = scores[:, prefix:, prefix:].topk(keep, dim=-1, sorted=False).indices
|
||||
vsa_h3_topk_lists(
|
||||
picked, ws.topk_mask, ws.q2k_index[:, prefix : ws.n_tiles], prefix
|
||||
)
|
||||
ws.q2k_num[:, prefix : ws.n_tiles] = prefix + keep
|
||||
return ws.q2k_index, ws.q2k_num
|
||||
return ws.sparse_lists(
|
||||
_topk_tile_lists(
|
||||
scores,
|
||||
@@ -393,7 +457,9 @@ class VideoSparseAttentionH3Impl(AttentionImpl):
|
||||
attn_metadata: AttentionMetadata,
|
||||
) -> torch.Tensor:
|
||||
raise NotImplementedError(
|
||||
"VSA-H3 serves MiniMax-H3's packed varlen attention; use forward_varlen."
|
||||
"VSA-H3 serves MiniMax-H3's packed varlen attention only; select it "
|
||||
"for the transformer with --component-attention-backends "
|
||||
"transformer=video_sparse_attn_h3 so dense layers keep their backends."
|
||||
)
|
||||
|
||||
def forward_varlen(
|
||||
@@ -407,8 +473,15 @@ class VideoSparseAttentionH3Impl(AttentionImpl):
|
||||
cu_seqlens_host: tuple[int, ...] | None = None,
|
||||
attn_metadata: VideoSparseAttentionH3Metadata | None = None,
|
||||
gate_compress: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""query/key/value: [T, H, D] packed rows (post-norm, post-RoPE)."""
|
||||
return_compress: bool = False,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
"""query/key/value: [T, H, D] packed rows (post-norm, post-RoPE).
|
||||
``return_compress`` returns the compression branch's ``[H, n_tiles, D]``
|
||||
output instead of folding it, for ``vsa_h3_fold_gate`` after the
|
||||
caller's collectives."""
|
||||
assert not return_compress or (
|
||||
gate_compress is None and attn_metadata is not None
|
||||
), "return_compress replaces gate_compress and needs VSA-H3 metadata"
|
||||
if self.layer_idx is None or attn_metadata is None:
|
||||
if attn_metadata is None and self.layer_idx is not None:
|
||||
raise RuntimeError(
|
||||
@@ -441,6 +514,7 @@ class VideoSparseAttentionH3Impl(AttentionImpl):
|
||||
|
||||
sparsity = 0.0 if self.layer_idx in meta.dense_layers else meta.VSA_sparsity
|
||||
has_gate = gate_compress is not None
|
||||
needs_compress = has_gate or return_compress
|
||||
ws = _get_workspace(meta, query, has_gate)
|
||||
|
||||
vsa_h3_pack_tiles(
|
||||
@@ -456,7 +530,7 @@ class VideoSparseAttentionH3Impl(AttentionImpl):
|
||||
q_pooled, k_pooled, v_pooled = ws.pooled
|
||||
|
||||
scores = None
|
||||
if sparsity > 0.0 or has_gate:
|
||||
if sparsity > 0.0 or needs_compress:
|
||||
scores = torch.matmul(q_pooled, k_pooled.transpose(-2, -1)) * (
|
||||
self.head_size**-0.5
|
||||
)
|
||||
@@ -468,21 +542,24 @@ class VideoSparseAttentionH3Impl(AttentionImpl):
|
||||
ws.tiled[2:3],
|
||||
q2k_index[None],
|
||||
q2k_num[None],
|
||||
meta.variable_block_sizes,
|
||||
ws.block_sizes,
|
||||
out=ws.out_tiled[None],
|
||||
use_sm100_kernel=ws.use_sm100_kernel,
|
||||
)
|
||||
|
||||
out_compress = None
|
||||
if has_gate:
|
||||
if needs_compress:
|
||||
out_compress = torch.matmul(torch.softmax(scores, dim=-1), v_pooled)
|
||||
|
||||
result = torch.empty(query.shape, dtype=query.dtype, device=query.device)
|
||||
vsa_h3_untile(
|
||||
ws.out_tiled,
|
||||
ws.tiled[3] if has_gate else None,
|
||||
out_compress,
|
||||
out_compress if has_gate else None,
|
||||
meta.unpack_index,
|
||||
used,
|
||||
result,
|
||||
)
|
||||
if return_compress:
|
||||
return result, out_compress
|
||||
return result
|
||||
|
||||
@@ -17,6 +17,7 @@ padded tile layout the attention kernel reads and pools each tile in the same
|
||||
pass; ``vsa_h3_untile`` scatters the attention output back to packed rows and
|
||||
folds in the gated compression branch. Both replace chains of index copies
|
||||
and transposes that otherwise cost more than the attention kernel itself.
|
||||
``vsa_h3_gate_add`` folds the same branch after the caller's collectives.
|
||||
"""
|
||||
|
||||
import math
|
||||
@@ -26,6 +27,8 @@ import triton
|
||||
import triton.language as tl
|
||||
from triton.tools.tensor_descriptor import TensorDescriptor
|
||||
|
||||
from sglang.kernels.ops.diffusion import vsa_block_sparse_sm100
|
||||
|
||||
# BLOCK_M / BLOCK_N are structural, not tunable: the kernel indexes the top-k
|
||||
# list per BLOCK_M q-tile and addresses keys as kv_idx * BLOCK_N, so both must
|
||||
# match the granularity q2k_index and variable_block_sizes were built at.
|
||||
@@ -106,10 +109,12 @@ def vsa_h3_block_sparse_attn_forward(
|
||||
q2k_num: torch.Tensor,
|
||||
variable_block_sizes: torch.Tensor,
|
||||
out: torch.Tensor | None = None,
|
||||
use_sm100_kernel: bool = False,
|
||||
) -> torch.Tensor:
|
||||
"""q/k/v: contiguous [B, H, S_pad, D] bf16 with S_pad = n_tiles * 64; pad
|
||||
rows zero. q2k_index/q2k_num: contiguous [B, H, n_tiles, max_kv] /
|
||||
[B, H, n_tiles] int32."""
|
||||
[B, H, n_tiles] int32. ``use_sm100_kernel`` runs FastVideo's tcgen05 kernel (sm_100a /
|
||||
sm_103a, even n_tiles) instead of the Triton kernel."""
|
||||
batch, heads, seq_q, head_dim = q.shape
|
||||
seq_kv = k.shape[2]
|
||||
if seq_q % VSA_H3_KERNEL_BLOCK or seq_kv % VSA_H3_KERNEL_BLOCK:
|
||||
@@ -124,6 +129,18 @@ def vsa_h3_block_sparse_attn_forward(
|
||||
)
|
||||
if out is None:
|
||||
out = torch.empty_like(q)
|
||||
if use_sm100_kernel:
|
||||
vsa_block_sparse_sm100(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
q2k_index.view(-1, q2k_index.shape[-1]),
|
||||
q2k_num.view(-1),
|
||||
variable_block_sizes,
|
||||
out,
|
||||
1.0 / math.sqrt(head_dim),
|
||||
)
|
||||
return out
|
||||
block = [1, 1, VSA_H3_KERNEL_BLOCK, head_dim]
|
||||
desc_q, desc_k, desc_v, desc_o = (
|
||||
TensorDescriptor.from_tensor(t, block_shape=block) for t in (q, k, v, out)
|
||||
@@ -243,9 +260,10 @@ def vsa_h3_pack_tiles(
|
||||
"""Gather packed [T, H, D] rows into ``tiled`` [3|4, H, S_pad, D] and write
|
||||
fp32 per-tile means of q/k/v into ``pooled`` [3, H, n_tiles, D].
|
||||
``src_index`` maps each padded position to its packed row, or -1 (pad -> 0).
|
||||
S_pad may hold one extra empty tile past ``n_tiles``; it is left untouched.
|
||||
"""
|
||||
_, heads, seq_pad, head_dim = tiled.shape
|
||||
n_tiles = seq_pad // VSA_H3_KERNEL_BLOCK
|
||||
n_tiles = pooled.shape[2]
|
||||
has_gate = gate is not None
|
||||
g = gate if has_gate else q
|
||||
assert all(t.stride(-1) == 1 for t in (q, k, v, g))
|
||||
@@ -327,6 +345,8 @@ def vsa_h3_untile(
|
||||
heads, seq_pad, head_dim = out_tiled.shape
|
||||
total = result.shape[0]
|
||||
has_gate = gate_tiled is not None
|
||||
# S_pad may carry one padded tile; the compression output does not
|
||||
n_tiles = out_compress.shape[1] if has_gate else seq_pad // VSA_H3_KERNEL_BLOCK
|
||||
_untile_kernel[(triton.cdiv(total, VSA_H3_KERNEL_BLOCK), heads)](
|
||||
out_tiled,
|
||||
gate_tiled if has_gate else out_tiled,
|
||||
@@ -336,8 +356,126 @@ def vsa_h3_untile(
|
||||
used,
|
||||
total,
|
||||
seq_pad,
|
||||
seq_pad // VSA_H3_KERNEL_BLOCK,
|
||||
n_tiles,
|
||||
HAS_GATE=has_gate,
|
||||
HEAD_DIM=head_dim,
|
||||
BLOCK=VSA_H3_KERNEL_BLOCK,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _gate_add_kernel(
|
||||
Out,
|
||||
Gate,
|
||||
Compress,
|
||||
tile_index,
|
||||
num_rows,
|
||||
stride_o_row,
|
||||
stride_o_head,
|
||||
stride_g_row,
|
||||
stride_g_head,
|
||||
N_TILES,
|
||||
HEAD_DIM: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
row_block = tl.program_id(0)
|
||||
h = tl.program_id(1)
|
||||
rows = row_block * BLOCK + tl.arange(0, BLOCK)
|
||||
cols = tl.arange(0, HEAD_DIM)
|
||||
in_rows = rows < num_rows
|
||||
tile = tl.load(tile_index + rows, mask=in_rows, other=-1)
|
||||
valid = (tile >= 0)[:, None]
|
||||
o_off = (
|
||||
rows.to(tl.int64)[:, None] * stride_o_row + h * stride_o_head + cols[None, :]
|
||||
)
|
||||
g_off = (
|
||||
rows.to(tl.int64)[:, None] * stride_g_row + h * stride_g_head + cols[None, :]
|
||||
)
|
||||
o = tl.load(Out + o_off, mask=valid, other=0.0).to(tl.float32)
|
||||
g = tl.load(Gate + g_off, mask=valid, other=0.0).to(tl.float32)
|
||||
c = tl.load(
|
||||
Compress + (h * N_TILES + tile)[:, None] * HEAD_DIM + cols[None, :],
|
||||
mask=valid,
|
||||
other=0.0,
|
||||
)
|
||||
o = o + c * g
|
||||
tl.store(Out + o_off, o.to(Out.type.element_ty), mask=valid)
|
||||
|
||||
|
||||
def vsa_h3_gate_add(
|
||||
out: torch.Tensor,
|
||||
gate: torch.Tensor,
|
||||
out_compress: torch.Tensor,
|
||||
tile_index: torch.Tensor,
|
||||
) -> None:
|
||||
"""In place: ``out[t, h] += out_compress[h, tile_index[t]] * gate[t, h]``
|
||||
where ``tile_index[t] >= 0``; same fp32 arithmetic as ``vsa_h3_untile``."""
|
||||
num_rows, heads, head_dim = out.shape
|
||||
assert out.stride(-1) == 1 and gate.stride(-1) == 1
|
||||
assert out_compress.is_contiguous() and out_compress.shape[0] == heads
|
||||
_gate_add_kernel[(triton.cdiv(num_rows, VSA_H3_KERNEL_BLOCK), heads)](
|
||||
out,
|
||||
gate,
|
||||
out_compress,
|
||||
tile_index,
|
||||
num_rows,
|
||||
out.stride(0),
|
||||
out.stride(1),
|
||||
gate.stride(0),
|
||||
gate.stride(1),
|
||||
out_compress.shape[1],
|
||||
HEAD_DIM=head_dim,
|
||||
BLOCK=VSA_H3_KERNEL_BLOCK,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _topk_lists_kernel(
|
||||
mask_ptr,
|
||||
out_ptr,
|
||||
num_video_tiles,
|
||||
prefix,
|
||||
stride_mh,
|
||||
stride_mr,
|
||||
stride_oh,
|
||||
stride_or,
|
||||
VIDEO_POW2: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
head = tl.program_id(1)
|
||||
mask_row = mask_ptr + head * stride_mh + row * stride_mr
|
||||
out_row = out_ptr + head * stride_oh + row * stride_or
|
||||
cols = tl.arange(0, VIDEO_POW2)
|
||||
in_range = cols < num_video_tiles
|
||||
picked = tl.load(mask_row + cols, mask=in_range, other=0).to(tl.int32)
|
||||
slot = tl.cumsum(picked, axis=0) - 1
|
||||
tl.store(out_row + prefix + slot, cols + prefix, mask=in_range & (picked != 0))
|
||||
# Leave the mask zeroed for the next layer's scatter.
|
||||
tl.store(mask_row + cols, tl.zeros_like(picked).to(tl.int8), mask=in_range)
|
||||
|
||||
|
||||
def vsa_h3_topk_lists(
|
||||
picked: torch.Tensor, mask: torch.Tensor, out: torch.Tensor, num_prefix_tiles: int
|
||||
) -> None:
|
||||
"""Write ``sort(picked + prefix)`` into ``out[h, r, prefix:]``.
|
||||
|
||||
``picked`` [H, rows, keep] holds distinct video-tile indices in any order;
|
||||
``mask`` [H, rows, num_video_tiles] is an all-zero int8 scratch, left
|
||||
zeroed; ``out`` [H, rows, >= prefix + keep] is int32 whose first ``prefix``
|
||||
columns already hold ``[0, prefix)``."""
|
||||
heads, rows, _ = picked.shape
|
||||
num_video_tiles = mask.shape[-1]
|
||||
assert mask.dtype == torch.int8 and mask.is_contiguous()
|
||||
assert out.dtype == torch.int32 and out.stride(-1) == 1
|
||||
mask.scatter_(-1, picked, 1)
|
||||
_topk_lists_kernel[(rows, heads)](
|
||||
mask,
|
||||
out,
|
||||
num_video_tiles,
|
||||
num_prefix_tiles,
|
||||
mask.stride(0),
|
||||
mask.stride(1),
|
||||
out.stride(0),
|
||||
out.stride(1),
|
||||
VIDEO_POW2=triton.next_power_of_2(num_video_tiles),
|
||||
)
|
||||
|
||||
@@ -109,6 +109,20 @@ def _usp_all_to_all_single(x: torch.Tensor, role: str | None = None) -> torch.Te
|
||||
return output.reshape(x_shape)
|
||||
|
||||
|
||||
def _usp_all_gather(x: torch.Tensor) -> torch.Tensor:
|
||||
"""Concatenate ``x`` from every Ulysses rank along dim 0, rank order."""
|
||||
ulysses_pg = get_sp_group().ulysses_group
|
||||
assert ulysses_pg is not None, "Ulysses process group is not initialized."
|
||||
x = x.contiguous()
|
||||
output = torch.empty(
|
||||
(get_ulysses_parallel_world_size() * x.shape[0], *x.shape[1:]),
|
||||
dtype=x.dtype,
|
||||
device=x.device,
|
||||
)
|
||||
dist.all_gather_into_tensor(output, x, group=ulysses_pg)
|
||||
return output
|
||||
|
||||
|
||||
def _usp_all_to_all_single_varlen(
|
||||
x: torch.Tensor,
|
||||
output_split_sizes: list[int],
|
||||
|
||||
@@ -59,6 +59,9 @@ from sglang.multimodal_gen.runtime.distributed.parallel_state import (
|
||||
from sglang.multimodal_gen.runtime.layers.attention.backends.attention_backend import (
|
||||
AttentionRequirements,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.attention.backends.video_sparse_attn_h3 import (
|
||||
vsa_h3_fold_gate,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.attention.selector import (
|
||||
claim_deferred_component_attn_backend,
|
||||
get_attn_backend,
|
||||
@@ -673,14 +676,12 @@ def _minimax_h3_attention_core_impl(
|
||||
|
||||
if ulysses_active:
|
||||
from sglang.multimodal_gen.runtime.layers.usp import (
|
||||
_usp_input_all_to_all,
|
||||
_usp_all_gather,
|
||||
_usp_input_all_to_all_packed_qkv,
|
||||
_usp_output_all_to_all,
|
||||
)
|
||||
|
||||
q, k, v = _usp_input_all_to_all_packed_qkv(q, k, v)
|
||||
if gate_compress is not None:
|
||||
gate_compress = _usp_input_all_to_all(gate_compress[None], head_dim=2)[0]
|
||||
|
||||
if attention._attention_impl is None:
|
||||
attention._set_attention_backend(
|
||||
@@ -698,7 +699,9 @@ def _minimax_h3_attention_core_impl(
|
||||
if attention.prefix.startswith("blocks.")
|
||||
else None
|
||||
)
|
||||
out = attention._attention_impl.forward_varlen(
|
||||
# the gate stays on this rank's row shard; fold it after the output all-to-all
|
||||
fold_after_exchange = ulysses_active and gate_compress is not None
|
||||
attn_result = attention._attention_impl.forward_varlen(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
@@ -706,10 +709,24 @@ def _minimax_h3_attention_core_impl(
|
||||
max_seqlen=max_seqlen,
|
||||
cu_seqlens_host=cu_seqlens_host,
|
||||
attn_metadata=attn_metadata,
|
||||
gate_compress=gate_compress,
|
||||
gate_compress=None if fold_after_exchange else gate_compress,
|
||||
return_compress=fold_after_exchange,
|
||||
)
|
||||
if fold_after_exchange:
|
||||
out, out_compress = attn_result
|
||||
else:
|
||||
out = attn_result
|
||||
if ulysses_active:
|
||||
out = _usp_output_all_to_all(out[None], head_dim=2)[0]
|
||||
if fold_after_exchange:
|
||||
_, ulysses_rank = get_ulysses_ctx()
|
||||
vsa_h3_fold_gate(
|
||||
out,
|
||||
gate_compress,
|
||||
_usp_all_gather(out_compress),
|
||||
attn_metadata,
|
||||
row_start=ulysses_rank * out.shape[0],
|
||||
)
|
||||
return out
|
||||
|
||||
if ring_active:
|
||||
|
||||
@@ -73,6 +73,7 @@ class MiniMaxH3VideoVAE(AutoencoderKLLegacy, LayerwiseOffloadableModuleMixin):
|
||||
and config.use_parallel_decode
|
||||
and config.use_parallel_tiling
|
||||
and bool(arch.vae_parallel_tiling),
|
||||
stack_tiling=config.stack_tiling,
|
||||
tile_size=int(arch.vae_tile_size),
|
||||
tile_overlap_min=int(arch.vae_tile_overlap_min),
|
||||
encoder_parallel=False,
|
||||
|
||||
@@ -0,0 +1,181 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""CUDA fast path for the MiniMax-H3 video VAE decoder (ViT3DDecoder).
|
||||
|
||||
Fuses each decoder attention block's per-head QK RMSNorm + NeoX RoPE into one
|
||||
in-place launch over the strided Q/K views of the QKV buffer and runs the
|
||||
block's attention on cuDNN SDPA. Forwards are bound once at VAE load and
|
||||
dispatch on a decode-scoped :class:`VaeFastPathGate`: ``quality="lossless"``
|
||||
and ``"high"`` take the fast path (rounding-level differences), the
|
||||
``"exact"`` runs the original module path bit-for-bit. Install is
|
||||
all-or-nothing and fail-closed.
|
||||
"""
|
||||
|
||||
from types import MethodType
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from torch.nn.attention import SDPBackend, sdpa_kernel
|
||||
|
||||
from sglang.kernels.ops.diffusion import (
|
||||
can_use_fused_inplace_qknorm_rope,
|
||||
fused_inplace_qknorm_rope,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.vaes.fast_path_gate import (
|
||||
VaeFastPathGate,
|
||||
register_vae_fast_path_gate,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.vaes.minimax_h3_video_vae.attention import (
|
||||
Attention,
|
||||
_apply_qk_norm,
|
||||
_sdpa_attention,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.vaes.minimax_h3_video_vae.vit_utils import (
|
||||
_env_flag,
|
||||
apply_rotary_pos_emb_qk,
|
||||
native_rope_cache,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _fused_qknorm_rope(self, query, key, rotary_pos_emb) -> bool:
|
||||
native = native_rope_cache(query, key, rotary_pos_emb)
|
||||
if native is None:
|
||||
return False
|
||||
cache, positions = native
|
||||
if not can_use_fused_inplace_qknorm_rope(
|
||||
head_dim=self.dim_head,
|
||||
rope_dim=cache.shape[1],
|
||||
is_neox=True,
|
||||
dtype=query.dtype,
|
||||
cache_dtype=cache.dtype,
|
||||
round_norm_before_rope=True,
|
||||
):
|
||||
return False
|
||||
weight = self._sgl_unit_weight
|
||||
if weight is None or weight.dtype != query.dtype or weight.device != query.device:
|
||||
weight = torch.ones(self.dim_head, dtype=query.dtype, device=query.device)
|
||||
self._sgl_unit_weight = weight
|
||||
# Batched decoder tiles flatten into rows; a view, so the update is in place.
|
||||
fused_inplace_qknorm_rope(
|
||||
query.flatten(0, 1),
|
||||
key.flatten(0, 1),
|
||||
weight,
|
||||
weight,
|
||||
cache,
|
||||
positions,
|
||||
is_neox=True,
|
||||
eps=self.norm_q.eps,
|
||||
head_dim=self.dim_head,
|
||||
rope_dim=cache.shape[1],
|
||||
round_norm_before_rope=True,
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def _cudnn_attention(self, query, key, value):
|
||||
if not self._sgl_cudnn_failed:
|
||||
try:
|
||||
with sdpa_kernel(SDPBackend.CUDNN_ATTENTION):
|
||||
return F.scaled_dot_product_attention(
|
||||
query.transpose(1, 2), key.transpose(1, 2), value.transpose(1, 2)
|
||||
).transpose(1, 2)
|
||||
except RuntimeError as e:
|
||||
logger.warning(
|
||||
"MiniMax-H3 VAE: cuDNN SDPA failed (%s); using the layer's "
|
||||
"attention backend.",
|
||||
e,
|
||||
)
|
||||
self._sgl_cudnn_failed = True
|
||||
return self.attn(query, key, value)
|
||||
|
||||
|
||||
def _attn_fast_forward(self, hidden_states, rotary_pos_emb=None):
|
||||
if (
|
||||
not self._sgl_gate.enabled
|
||||
or rotary_pos_emb is None
|
||||
or torch.is_grad_enabled()
|
||||
or torch.compiler.is_compiling()
|
||||
):
|
||||
return type(self).forward(self, hidden_states, rotary_pos_emb)
|
||||
|
||||
batch_size, seq_len, _ = hidden_states.shape
|
||||
qkv = self.to_qkv(hidden_states)
|
||||
qkv = qkv.view(batch_size, seq_len, -1, 3 * self.dim_head)
|
||||
query, key, value = torch.chunk(qkv, 3, dim=-1)
|
||||
|
||||
if not _fused_qknorm_rope(self, query, key, rotary_pos_emb):
|
||||
query = _apply_qk_norm(self.norm_q, query)
|
||||
key = _apply_qk_norm(self.norm_k, key)
|
||||
query, key = apply_rotary_pos_emb_qk(query, key, rotary_pos_emb)
|
||||
|
||||
if self.attn is not None and query.dtype in (torch.float16, torch.bfloat16):
|
||||
hidden_states = _cudnn_attention(self, query, key, value)
|
||||
else:
|
||||
hidden_states = _sdpa_attention(query, key, value)
|
||||
|
||||
hidden_states = hidden_states.reshape(batch_size, seq_len, -1)
|
||||
return self.to_out(hidden_states)
|
||||
|
||||
|
||||
def _plain_rms_norm(module) -> bool:
|
||||
return isinstance(module, nn.RMSNorm) and module.weight is None
|
||||
|
||||
|
||||
def _attn_fast_compatible(attn: nn.Module) -> bool:
|
||||
return (
|
||||
type(attn) is Attention
|
||||
and _plain_rms_norm(attn.norm_q)
|
||||
and _plain_rms_norm(attn.norm_k)
|
||||
and attn.norm_q.eps == attn.norm_k.eps
|
||||
)
|
||||
|
||||
|
||||
def _install_fast_attention(
|
||||
attn_modules: list[nn.Module], gate: VaeFastPathGate
|
||||
) -> None:
|
||||
for attn in attn_modules:
|
||||
attn._sgl_gate = gate
|
||||
attn._sgl_unit_weight = None
|
||||
attn._sgl_cudnn_failed = False
|
||||
attn.forward = MethodType(_attn_fast_forward, attn)
|
||||
|
||||
|
||||
def maybe_optimize_minimax_h3_vae(vae: nn.Module) -> nn.Module:
|
||||
"""Install the quality-gated CUDA MiniMax-H3 VAE decoder fast path."""
|
||||
from sglang.multimodal_gen.runtime.models.vaes.minimax_h3 import MiniMaxH3VideoVAE
|
||||
from sglang.multimodal_gen.runtime.models.vaes.minimax_h3_video_vae.vae_vit import (
|
||||
ViT3DDecoder,
|
||||
)
|
||||
|
||||
if not isinstance(vae, MiniMaxH3VideoVAE):
|
||||
return vae
|
||||
decoder = vae.decoder
|
||||
if type(decoder) is not ViT3DDecoder:
|
||||
return vae
|
||||
if not _env_flag("MINIMAX_H3_VAE_DECODER_VIT_FP32_NORM", "1"):
|
||||
logger.info("MiniMax-H3 VAE: fp32 QK norm disabled; skipping fast path.")
|
||||
return vae
|
||||
|
||||
attn_modules = [block.attn for block in decoder.transformer_blocks]
|
||||
eligible = [attn for attn in attn_modules if _attn_fast_compatible(attn)]
|
||||
if len(eligible) != len(attn_modules):
|
||||
logger.warning(
|
||||
"MiniMax-H3 VAE: %d/%d decoder attention blocks non-standard; "
|
||||
"skipping fast path.",
|
||||
len(attn_modules) - len(eligible),
|
||||
len(attn_modules),
|
||||
)
|
||||
return vae
|
||||
|
||||
gate = VaeFastPathGate()
|
||||
_install_fast_attention(eligible, gate)
|
||||
register_vae_fast_path_gate(vae, gate)
|
||||
logger.info(
|
||||
"MiniMax-H3 VAE: installed quality-gated fast path (%d QK RMSNorm+RoPE "
|
||||
"fusions, cuDNN SDPA).",
|
||||
len(eligible),
|
||||
)
|
||||
return vae
|
||||
@@ -426,6 +426,7 @@ class ViT3DDecoder(ViTBase):
|
||||
rotary_pos_emb = prepare_rotary_pos_emb(
|
||||
self.pos_embed(img_ids[:1]),
|
||||
dtype=rotary_dtype,
|
||||
batch=B,
|
||||
)
|
||||
if B > 1:
|
||||
rotary_pos_emb = _expand_rotary_batch(rotary_pos_emb, B)
|
||||
|
||||
+50
-32
@@ -115,8 +115,10 @@ def prepare_rotary_pos_emb(
|
||||
rotary_pos_emb: Tuple[torch.Tensor, torch.Tensor],
|
||||
*,
|
||||
dtype: torch.dtype,
|
||||
batch: int = 1,
|
||||
) -> tuple[torch.Tensor, ...]:
|
||||
"""Prebuild the native Q/K rotary cache once per ViT decoder forward."""
|
||||
"""Prebuild the native Q/K rotary cache once per ViT decoder forward; its
|
||||
positions cover ``batch`` samples sharing one token grid."""
|
||||
cos, sin = rotary_pos_emb
|
||||
if (
|
||||
not cos.is_cuda
|
||||
@@ -143,7 +145,7 @@ def prepare_rotary_pos_emb(
|
||||
cos.shape[1],
|
||||
dtype=torch.long,
|
||||
device=cos.device,
|
||||
)
|
||||
).repeat(batch)
|
||||
return cos, sin, cache, positions
|
||||
|
||||
|
||||
@@ -209,13 +211,15 @@ def apply_rotary_pos_emb(
|
||||
raise
|
||||
|
||||
|
||||
def apply_rotary_pos_emb_qk(
|
||||
def native_rope_cache(
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
rotary_pos_emb: Sequence[torch.Tensor],
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Apply the exact native NeoX rotary kernel to Q/K together when possible."""
|
||||
if (
|
||||
) -> tuple[torch.Tensor, torch.Tensor] | None:
|
||||
"""``(cache, positions)`` from ``prepare_rotary_pos_emb`` when the native
|
||||
kernels can consume it for this Q/K, else None. ``positions`` covers the
|
||||
batch-major rows of ``query.flatten(0, 1)``."""
|
||||
if not (
|
||||
len(rotary_pos_emb) == 4
|
||||
and query.is_cuda
|
||||
and query.shape == key.shape
|
||||
@@ -224,33 +228,47 @@ def apply_rotary_pos_emb_qk(
|
||||
and query.dim() == 4
|
||||
and not torch.compiler.is_compiling()
|
||||
):
|
||||
_, _, cache, positions = rotary_pos_emb
|
||||
batch, seq_len = query.shape[:2]
|
||||
if (
|
||||
cache.is_cuda
|
||||
and cache.dtype == query.dtype
|
||||
and cache.dim() == 2
|
||||
and cache.shape[0] == seq_len
|
||||
and cache.shape[1] <= query.shape[-1]
|
||||
and positions.is_cuda
|
||||
and positions.shape == (seq_len,)
|
||||
):
|
||||
from sgl_kernel import rotary_embedding
|
||||
return None
|
||||
_, _, cache, positions = rotary_pos_emb
|
||||
if not (
|
||||
cache.is_cuda
|
||||
and cache.dtype == query.dtype
|
||||
and cache.dim() == 2
|
||||
and cache.shape[0] == query.shape[1]
|
||||
and cache.shape[1] <= query.shape[-1]
|
||||
and positions.is_cuda
|
||||
and positions.shape in ((query.shape[1],), (query.shape[0] * query.shape[1],))
|
||||
):
|
||||
return None
|
||||
# Accept both main's shared sequence positions and the pre-expanded
|
||||
# batch-major positions used by the decoder cache and fused fast path.
|
||||
if positions.shape == (query.shape[1],) and query.shape[0] > 1:
|
||||
positions = positions.repeat(query.shape[0])
|
||||
return cache, positions
|
||||
|
||||
# Stacked spatial tiles share one rotary table. Flatten the
|
||||
# batch and replay that table once per tile.
|
||||
query = query.contiguous()
|
||||
key = key.contiguous()
|
||||
token_positions = positions if batch == 1 else positions.repeat(batch)
|
||||
rotary_embedding(
|
||||
token_positions,
|
||||
query.view(batch * seq_len, -1),
|
||||
key.view(batch * seq_len, -1),
|
||||
query.shape[-1],
|
||||
cache,
|
||||
True,
|
||||
)
|
||||
return query, key
|
||||
|
||||
def apply_rotary_pos_emb_qk(
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
rotary_pos_emb: Sequence[torch.Tensor],
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Apply the exact native NeoX rotary kernel to Q/K together when possible."""
|
||||
native = native_rope_cache(query, key, rotary_pos_emb)
|
||||
if native is not None:
|
||||
from sgl_kernel import rotary_embedding
|
||||
|
||||
cache, positions = native
|
||||
query = query.contiguous()
|
||||
key = key.contiguous()
|
||||
rotary_embedding(
|
||||
positions,
|
||||
query.view(positions.shape[0], -1),
|
||||
key.view(positions.shape[0], -1),
|
||||
query.shape[-1],
|
||||
cache,
|
||||
True,
|
||||
)
|
||||
return query, key
|
||||
|
||||
return (
|
||||
apply_rotary_pos_emb(query, rotary_pos_emb),
|
||||
|
||||
@@ -175,6 +175,11 @@ class MiniMaxH3Pipeline(LoRAPipeline, ComposedPipelineBase):
|
||||
self.add_stage(
|
||||
MiniMaxH3TimestepPreparationStage(
|
||||
sigma_shift_scales=sigma_shift_scales,
|
||||
dmd_denoising_steps=(
|
||||
release_metadata.dmd_denoising_steps
|
||||
if release_metadata is not None
|
||||
else None
|
||||
),
|
||||
)
|
||||
)
|
||||
self.add_stage(
|
||||
@@ -201,7 +206,7 @@ class MiniMaxH3Pipeline(LoRAPipeline, ComposedPipelineBase):
|
||||
|
||||
|
||||
class FastH3Pipeline(MiniMaxH3Pipeline):
|
||||
"""FastH3: 4-step DMD2-distilled MiniMax-H3 (t2va only).
|
||||
"""FastH3: 8-step DMD2-distilled MiniMax-H3 (V2, t2va only).
|
||||
|
||||
The flat single-partition repo is materialized into the base-H3 layout by
|
||||
the bundled model overlay (see model_overlays/), so every stage, loader,
|
||||
|
||||
+18
@@ -52,6 +52,8 @@ class MiniMaxH3ReleaseMetadata:
|
||||
task_aliases: Mapping[str, str]
|
||||
video_sigma_shift: float
|
||||
audio_sigma_shift: float
|
||||
# trained DMD rungs of a distilled release, unshifted; None keeps the uniform grid
|
||||
dmd_denoising_steps: tuple[int, ...] | None = None
|
||||
|
||||
@classmethod
|
||||
def from_model_index(
|
||||
@@ -92,6 +94,21 @@ class MiniMaxH3ReleaseMetadata:
|
||||
"model_index.json._minimax_h3.sigma_shift_scales requires numeric "
|
||||
"video and audio values"
|
||||
) from exc
|
||||
dmd_steps = raw.get("dmd_denoising_steps")
|
||||
if dmd_steps is not None:
|
||||
if (
|
||||
not isinstance(dmd_steps, list)
|
||||
or not dmd_steps
|
||||
or any(
|
||||
type(step) is not int or not 0 < step <= 1000 for step in dmd_steps
|
||||
)
|
||||
or any(left <= right for left, right in zip(dmd_steps, dmd_steps[1:]))
|
||||
):
|
||||
raise ValueError(
|
||||
"model_index.json._minimax_h3.dmd_denoising_steps must be "
|
||||
"strictly decreasing integers in (0, 1000]"
|
||||
)
|
||||
dmd_steps = tuple(dmd_steps)
|
||||
metadata = cls(
|
||||
schema_version=1,
|
||||
partition=partition,
|
||||
@@ -99,6 +116,7 @@ class MiniMaxH3ReleaseMetadata:
|
||||
task_aliases=dict(aliases),
|
||||
video_sigma_shift=video_sigma,
|
||||
audio_sigma_shift=audio_sigma,
|
||||
dmd_denoising_steps=dmd_steps,
|
||||
)
|
||||
for task in metadata.tasks:
|
||||
if canonical_minimax_h3_task(task) != task:
|
||||
|
||||
+11
-1
@@ -7,6 +7,9 @@ from contextlib import contextmanager, nullcontext
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.configs.sample.sampling_params import (
|
||||
quality_allows,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType
|
||||
from sglang.multimodal_gen.runtime.distributed import (
|
||||
get_replica_group,
|
||||
@@ -16,6 +19,7 @@ from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_c
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
|
||||
ComponentUse,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.vaes.fast_path_gate import use_vae_fast_path
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch, Req
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.base import (
|
||||
StageParallelismType,
|
||||
@@ -494,7 +498,13 @@ class MiniMaxH3DecodingStage(DecodingStage):
|
||||
)
|
||||
decode_kwargs["on_frames"] = stream
|
||||
try:
|
||||
with set_forward_context(current_timestep=0, attn_metadata=None):
|
||||
with (
|
||||
use_vae_fast_path(
|
||||
selected_video_vae,
|
||||
quality_allows(batch.sampling_params.quality, "lossless"),
|
||||
),
|
||||
set_forward_context(current_timestep=0, attn_metadata=None),
|
||||
):
|
||||
decoded = video_decode(visual_decode_latent, **decode_kwargs)
|
||||
if stream is not None:
|
||||
output_file_paths = stream.finish()
|
||||
|
||||
+4
-1
@@ -1077,7 +1077,10 @@ def _maybe_prepare_vsa_h3_step_metadata(
|
||||
"VSA-H3 in SGLang serves the trained 64-token (4, 4, 4) tile "
|
||||
f"geometry; got vsa_tile_size={tile_size}."
|
||||
)
|
||||
sparsity = float(config.get("VSA_sparsity", config.get("sparsity", 0.9)))
|
||||
default_sparsity = server_args.pipeline_config.vsa_sparsity
|
||||
sparsity = float(
|
||||
config.get("VSA_sparsity", config.get("sparsity", default_sparsity))
|
||||
)
|
||||
if not 0.0 <= sparsity < 1.0:
|
||||
raise ValueError(f"VSA sparsity must be in [0, 1), got {sparsity}")
|
||||
mode = str(config.get("vsa_mode", "exempt"))
|
||||
|
||||
+15
-2
@@ -26,12 +26,17 @@ class MiniMaxH3TimestepPreparationStage(PipelineStage):
|
||||
deduplicated_tensor_tree_output_fields = ("timesteps", "sigmas")
|
||||
deduplicated_extra_tensor_tree_output_keys = (MINIMAX_H3_SIGMAS_EXTRA_KEY,)
|
||||
|
||||
def __init__(self, sigma_shift_scales=None) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
sigma_shift_scales=None,
|
||||
dmd_denoising_steps: tuple[int, ...] | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
# Per-model sigma shift override (model_index.json "_minimax_h3" release
|
||||
# block, sigma_shift_scales): the schedule constants are a MODEL
|
||||
# serving contract — fl2va and ref2va use video 12 / audio 3 by default.
|
||||
self.sigma_shift_scales = sigma_shift_scales
|
||||
self.dmd_denoising_steps = dmd_denoising_steps
|
||||
self._pdd_config = None
|
||||
pdd_heads = envs.SGLANG_DIFFUSION_MINIMAX_H3_PDD_HEADS
|
||||
if pdd_heads:
|
||||
@@ -197,12 +202,20 @@ class MiniMaxH3TimestepPreparationStage(PipelineStage):
|
||||
task_default=plan.default_audio_flow_shift,
|
||||
),
|
||||
}
|
||||
dmd_steps = self.dmd_denoising_steps
|
||||
num_steps = requested_num_steps
|
||||
if dmd_steps is not None and batch.is_warmup:
|
||||
# warmup truncates the trained rungs instead of rescaling a shorter grid
|
||||
num_steps = len(dmd_steps) + 1
|
||||
sigmas: dict[str, list[float]] = {}
|
||||
for modality in ("video", "audio"):
|
||||
sigmas[modality] = minimax_h3_time_shift_sigmas(
|
||||
num_steps=requested_num_steps,
|
||||
num_steps=num_steps,
|
||||
shift_scale=scales[modality],
|
||||
dmd_steps=dmd_steps,
|
||||
)
|
||||
if num_steps != requested_num_steps:
|
||||
sigmas[modality] = sigmas[modality][: max(2, requested_num_steps)]
|
||||
batch.extra[MINIMAX_H3_SIGMAS_EXTRA_KEY] = sigmas
|
||||
|
||||
def verify_input(self, batch: Req, server_args: ServerArgs) -> VerificationResult:
|
||||
|
||||
+21
-8
@@ -33,6 +33,7 @@ def minimax_h3_time_shift_sigmas(
|
||||
*,
|
||||
num_steps: int = 50,
|
||||
shift_scale: float = 6.0,
|
||||
dmd_steps: tuple[int, ...] | None = None,
|
||||
) -> list[float]:
|
||||
if shift_scale <= 0:
|
||||
raise ValueError("MiniMax H3 shift_scale must be > 0")
|
||||
@@ -41,14 +42,26 @@ def minimax_h3_time_shift_sigmas(
|
||||
|
||||
import torch
|
||||
|
||||
# The rectified-flow sigma range is fixed at [1.0, 0.0].
|
||||
base = torch.linspace(
|
||||
1.0,
|
||||
0.0,
|
||||
int(num_steps),
|
||||
device="cpu",
|
||||
dtype=torch.float32,
|
||||
)
|
||||
if dmd_steps is not None:
|
||||
if int(num_steps) != len(dmd_steps) + 1:
|
||||
raise ValueError(
|
||||
f"this checkpoint is distilled for {len(dmd_steps)} DiT forwards "
|
||||
f"({len(dmd_steps) + 1} sigma grid points); got "
|
||||
f"num_inference_steps={num_steps}"
|
||||
)
|
||||
# trained rungs are unshifted timesteps over 1000; append the final step to sigma 0
|
||||
base = torch.tensor(
|
||||
[step / 1000.0 for step in dmd_steps] + [0.0], dtype=torch.float32
|
||||
)
|
||||
else:
|
||||
# The rectified-flow sigma range is fixed at [1.0, 0.0].
|
||||
base = torch.linspace(
|
||||
1.0,
|
||||
0.0,
|
||||
int(num_steps),
|
||||
device="cpu",
|
||||
dtype=torch.float32,
|
||||
)
|
||||
shifted = float(shift_scale) * base / (1 + (float(shift_scale) - 1) * base)
|
||||
shifted, _ = torch.unique_consecutive(shifted, return_counts=True)
|
||||
# A one-point request is still exactly one point. Normal serving uses
|
||||
|
||||
@@ -847,8 +847,8 @@ class CudaPlatformBase(Platform):
|
||||
|
||||
@classmethod
|
||||
def optimize_vae(cls, vae: torch.nn.Module) -> torch.nn.Module:
|
||||
"""Install the quality-gated FLUX.2 / AutoencoderKL / Wan / Qwen-Image
|
||||
VAE decoder fast paths.
|
||||
"""Install the quality-gated FLUX.2 / AutoencoderKL / Wan / Qwen-Image /
|
||||
MiniMax-H3 VAE decoder fast paths.
|
||||
|
||||
Requests with quality="lossless" or "high" run the fast paths; the
|
||||
"exact" default runs the original module path bit-for-bit. See
|
||||
@@ -859,6 +859,9 @@ class CudaPlatformBase(Platform):
|
||||
maybe_optimize_autoencoder_kl,
|
||||
maybe_optimize_flux2_vae,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.vaes.minimax_h3_vae_cuda_opt import (
|
||||
maybe_optimize_minimax_h3_vae,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.vaes.wan_vae_cuda_opt import (
|
||||
maybe_optimize_qwen_image_vae,
|
||||
maybe_optimize_wan_vae,
|
||||
@@ -868,6 +871,7 @@ class CudaPlatformBase(Platform):
|
||||
vae = maybe_optimize_autoencoder_kl(vae)
|
||||
vae = maybe_optimize_wan_vae(vae)
|
||||
vae = maybe_optimize_qwen_image_vae(vae)
|
||||
vae = maybe_optimize_minimax_h3_vae(vae)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Failed to apply CUDA VAE optimizations; using the unmodified VAE.",
|
||||
|
||||
@@ -45,9 +45,9 @@ BUILTIN_MODEL_OVERLAY_REGISTRY: dict[str, dict[str, Any]] = {
|
||||
"overlay_repo_id": "AgainstEntropy/SANA-WM_streaming-overlay",
|
||||
"overlay_revision": "62c6840871ecc3559189047513ba0670e1bf62e7",
|
||||
},
|
||||
"FastVideo/FastVideo-FastH3-4-step-Preview-v1-VSA-DataFree": {
|
||||
"overlay_repo_id": "kevin-mi/FastH3-4step-Preview-overlay",
|
||||
"overlay_revision": "f769cb8001dae335089de7b250364335bc7cb183",
|
||||
"FastVideo/FastVideo-FastH3-8-Step-V2": {
|
||||
"overlay_repo_id": "kevin-mi/FastH3-8step-V2-overlay",
|
||||
"overlay_revision": "308e2dd627218c40ae759a93745429800771a625",
|
||||
},
|
||||
"OpenVDN/vdn-minimax-h3": {
|
||||
"overlay_repo_id": "kevin-mi/VDN-H3-overlay",
|
||||
|
||||
@@ -775,14 +775,12 @@ MINIMAX_H3_FOUR_GPU_H100_CASES = [
|
||||
DiffusionTestCase(
|
||||
"fasth3_t2va_vsa_4gpu_h100",
|
||||
DiffusionServerArgs(
|
||||
model_path="FastVideo/FastVideo-FastH3-4-step-Preview-v1-VSA-DataFree",
|
||||
model_path="FastVideo/FastVideo-FastH3-8-Step-V2",
|
||||
modality="video",
|
||||
num_gpus=4,
|
||||
extras=[
|
||||
"--attention-backend",
|
||||
"video_sparse_attn_h3",
|
||||
"--attention-backend-config",
|
||||
'{"VSA_sparsity": 0.9}',
|
||||
"--component-attention-backends",
|
||||
"transformer=video_sparse_attn_h3",
|
||||
"--enable-torch-compile",
|
||||
"false",
|
||||
],
|
||||
@@ -805,7 +803,7 @@ MINIMAX_H3_FOUR_GPU_H100_CASES = [
|
||||
"aspect_ratio": "16:9",
|
||||
"duration_seconds": 5.0,
|
||||
},
|
||||
"num_inference_steps": 5,
|
||||
"num_inference_steps": 9,
|
||||
"seed": 42,
|
||||
},
|
||||
),
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""FastH3 (4-step VSA-distilled MiniMax-H3) registration and admission contracts."""
|
||||
"""FastH3 8-Step V2 (VSA-distilled MiniMax-H3) registration and admission contracts."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -25,11 +25,18 @@ from sglang.multimodal_gen.runtime.distributed.parallel_state import (
|
||||
from sglang.multimodal_gen.runtime.layers.linear import UnquantizedLinearMethod
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.fp8 import Fp8Config
|
||||
from sglang.multimodal_gen.runtime.models.dits.minimax_h3 import MiniMaxH3DiTModel
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.release_metadata import (
|
||||
MiniMaxH3ReleaseMetadata,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.time_request import (
|
||||
minimax_h3_time_shift_sigmas,
|
||||
)
|
||||
from sglang.multimodal_gen.test.single_test_file.component_accuracy.utils import (
|
||||
ensure_distributed_env_defaults,
|
||||
)
|
||||
|
||||
FASTH3_MODEL_ID = "FastVideo/FastVideo-FastH3-4-step-Preview-v1-VSA-DataFree"
|
||||
FASTH3_MODEL_ID = "FastVideo/FastVideo-FastH3-8-Step-V2"
|
||||
DMD_STEPS = (999, 874, 749, 624, 500, 375, 250, 125)
|
||||
|
||||
|
||||
def _ensure_single_process_parallel_runtime() -> None:
|
||||
@@ -45,18 +52,20 @@ def test_registry_resolves_fasth3_configs() -> None:
|
||||
assert info.pipeline_config_cls is FastH3PipelineConfig
|
||||
assert get_non_diffusers_pipeline_name(FASTH3_MODEL_ID) == "FastH3Pipeline"
|
||||
materialized = (
|
||||
"/cache/materialized_models/"
|
||||
"FastVideo__FastVideo-FastH3-4-step-Preview-v1-VSA-DataFree-0123abcd"
|
||||
"/cache/materialized_models/FastVideo__FastVideo-FastH3-8-Step-V2-0123abcd"
|
||||
)
|
||||
assert get_model_info(materialized).sampling_param_cls is FastH3SamplingParams
|
||||
|
||||
|
||||
def test_fasth3_sampling_defaults_and_task_rejection() -> None:
|
||||
params = FastH3SamplingParams(prompt="p")
|
||||
assert params.num_inference_steps == 5
|
||||
assert params.num_inference_steps == 9
|
||||
assert params.guidance_scale == 1.0
|
||||
assert params.quality == "lossless"
|
||||
assert FastH3SamplingParams(prompt="p", quality="extra-high").quality == "lossless"
|
||||
assert FastH3SamplingParams(prompt="p", quality="exact").quality == "exact"
|
||||
|
||||
with pytest.raises(ValueError, match="exactly five sigma grid points"):
|
||||
with pytest.raises(ValueError, match="exactly nine sigma grid points"):
|
||||
FastH3SamplingParams(prompt="p", num_inference_steps=50)
|
||||
|
||||
with pytest.raises(ValueError, match="distilled for t2va only"):
|
||||
@@ -72,8 +81,56 @@ def test_fasth3_sampling_defaults_and_task_rejection() -> None:
|
||||
)
|
||||
|
||||
|
||||
def test_fasth3_dmd_schedule_matches_trained_rungs() -> None:
|
||||
"""The distilled rungs are shifted exactly once, as FastVideo serves them, and
|
||||
a wrong step count or non-decreasing rungs are rejected."""
|
||||
metadata = MiniMaxH3ReleaseMetadata.from_model_index(
|
||||
{
|
||||
"_minimax_h3": {
|
||||
"schema_version": 1,
|
||||
"partition": "fl2va",
|
||||
"tasks": ["t2va"],
|
||||
"sigma_shift_scales": {"video": 10.0, "audio": 3.0},
|
||||
"dmd_denoising_steps": list(DMD_STEPS),
|
||||
}
|
||||
}
|
||||
)
|
||||
assert metadata.dmd_denoising_steps == DMD_STEPS
|
||||
# FastVideo's _set_dmd_schedule output for the video / audio shifts 10 / 3
|
||||
served = {
|
||||
10.0: [
|
||||
*(0.9999, 0.985788, 0.967575, 0.943168, 0.909091),
|
||||
*(0.857143, 0.769231, 0.588235, 0.0),
|
||||
],
|
||||
3.0: [
|
||||
*(0.999666, 0.954148, 0.89952, 0.83274, 0.75),
|
||||
*(0.642857, 0.5, 0.3, 0.0),
|
||||
],
|
||||
}
|
||||
for shift, expected in served.items():
|
||||
sigmas = minimax_h3_time_shift_sigmas(
|
||||
num_steps=9, shift_scale=shift, dmd_steps=DMD_STEPS
|
||||
)
|
||||
assert sigmas == pytest.approx(expected, abs=1e-6)
|
||||
with pytest.raises(ValueError, match="8 DiT forwards"):
|
||||
minimax_h3_time_shift_sigmas(num_steps=5, shift_scale=10.0, dmd_steps=DMD_STEPS)
|
||||
with pytest.raises(ValueError, match="strictly decreasing"):
|
||||
MiniMaxH3ReleaseMetadata.from_model_index(
|
||||
{
|
||||
"_minimax_h3": {
|
||||
"schema_version": 1,
|
||||
"partition": "fl2va",
|
||||
"tasks": ["t2va"],
|
||||
"sigma_shift_scales": {"video": 10.0, "audio": 3.0},
|
||||
"dmd_denoising_steps": [999, 999],
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_fasth3_pipeline_config_gates_and_rejections() -> None:
|
||||
config = FastH3PipelineConfig()
|
||||
assert config.vsa_sparsity == 0.8
|
||||
assert config.dit_config.arch_config.has_gate_compress
|
||||
assert not MiniMaxH3PipelineConfig().dit_config.arch_config.has_gate_compress
|
||||
mapping = config.dit_config.arch_config.param_names_mapping
|
||||
|
||||
@@ -5,6 +5,7 @@ import os
|
||||
import subprocess
|
||||
import sys
|
||||
import textwrap
|
||||
from types import SimpleNamespace
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
@@ -14,10 +15,17 @@ import torch.nn as nn
|
||||
from sglang.multimodal_gen.configs.models.vaes.minimax_h3_video import (
|
||||
MiniMaxH3VideoVAEConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context
|
||||
from sglang.multimodal_gen.runtime.models.vaes.fast_path_gate import VaeFastPathGate
|
||||
from sglang.multimodal_gen.runtime.models.vaes.minimax_h3 import MiniMaxH3VideoVAE
|
||||
from sglang.multimodal_gen.runtime.models.vaes.minimax_h3_audio_vae.audio_vae import (
|
||||
CausalAttention,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.vaes.minimax_h3_vae_cuda_opt import (
|
||||
_attn_fast_compatible,
|
||||
_fused_qknorm_rope,
|
||||
_install_fast_attention,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.vaes.minimax_h3_video_vae import (
|
||||
AutoencoderKLLegacy,
|
||||
)
|
||||
@@ -25,14 +33,23 @@ from sglang.multimodal_gen.runtime.models.vaes.minimax_h3_video_vae.attention im
|
||||
Attention,
|
||||
_apply_qk_norm,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.vaes.minimax_h3_video_vae.base_module import (
|
||||
RotaryEmbeddingND,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.vaes.minimax_h3_video_vae.vae_vit import (
|
||||
ViT3DDecoder,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.vaes.minimax_h3_video_vae.vit_utils import (
|
||||
apply_rotary_pos_emb_qk,
|
||||
create_token_ids,
|
||||
prepare_rotary_pos_emb,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum
|
||||
|
||||
requires_cuda = pytest.mark.skipif(
|
||||
not torch.cuda.is_available(), reason="fused qk-norm+RoPE kernel needs CUDA"
|
||||
)
|
||||
|
||||
|
||||
def _init_kwargs(config: MiniMaxH3VideoVAEConfig):
|
||||
with mock.patch.object(
|
||||
@@ -97,6 +114,77 @@ def test_vit_qk_norm_supports_affine_free_rmsnorm():
|
||||
assert output.shape == hidden_states.shape
|
||||
|
||||
|
||||
@requires_cuda
|
||||
def test_vit_fast_path_is_gated_and_matches_reference():
|
||||
"""Gate closed: bit-identical to the original forward. Gate open: the fused
|
||||
qk-norm+RoPE kernel and cuDNN SDPA run and match to rounding level."""
|
||||
device = torch.device("cuda")
|
||||
dtype = torch.float16
|
||||
heads, dim_head, rope_dim = 4, 64, 48
|
||||
attention = Attention(
|
||||
heads=heads, dim_head=dim_head, qk_norm_type="rms_norm", eps=1e-5
|
||||
).to(device=device, dtype=dtype)
|
||||
assert _attn_fast_compatible(attention)
|
||||
|
||||
pos_embed = RotaryEmbeddingND(rope_dim, 100.0, n_dim=3, use_angle=True).to(device)
|
||||
ids = create_token_ids((2, 4, 4), device, dtype)
|
||||
ids = torch.cat([ids, torch.zeros((1, 5, 3), device=device, dtype=dtype)], 1)
|
||||
rotary = prepare_rotary_pos_emb(pos_embed(ids), dtype=dtype)
|
||||
generator = torch.Generator(device="cpu").manual_seed(0)
|
||||
hidden = torch.randn((1, ids.shape[1], heads * dim_head), generator=generator)
|
||||
hidden = hidden.to(device=device, dtype=dtype)
|
||||
|
||||
with (
|
||||
torch.no_grad(),
|
||||
torch.autocast("cuda", dtype=dtype),
|
||||
set_forward_context(current_timestep=0, attn_metadata=None),
|
||||
):
|
||||
reference = attention(hidden, rotary)
|
||||
gate = VaeFastPathGate()
|
||||
_install_fast_attention([attention], gate)
|
||||
assert torch.equal(attention(hidden, rotary), reference)
|
||||
gate.enabled = True
|
||||
fused = attention(hidden, rotary)
|
||||
assert attention._sgl_unit_weight is not None, "fused qk-norm+RoPE did not run"
|
||||
assert attention._sgl_cudnn_failed is False, "cuDNN SDPA fell back"
|
||||
torch.testing.assert_close(fused, reference, atol=2e-2, rtol=1e-2)
|
||||
|
||||
|
||||
@requires_cuda
|
||||
def test_vit_rope_batched_tiles_match_per_tile():
|
||||
"""Stacked decoder tiles share one RoPE row; the fused (quality) and native
|
||||
(exact) RoPE kernels over the flattened batch are bit-identical to one
|
||||
tile at a time."""
|
||||
device, dtype = torch.device("cuda"), torch.float16
|
||||
tiles, heads, dim_head, rope_dim = 3, 4, 64, 48
|
||||
pos_embed = RotaryEmbeddingND(rope_dim, 100.0, n_dim=3, use_angle=True).to(device)
|
||||
ids = create_token_ids((2, 4, 4), device, dtype)
|
||||
rotary = prepare_rotary_pos_emb(pos_embed(ids), dtype=dtype)
|
||||
batched_rotary = prepare_rotary_pos_emb(pos_embed(ids), dtype=dtype, batch=tiles)
|
||||
generator = torch.Generator(device="cpu").manual_seed(0)
|
||||
qkv = torch.randn(
|
||||
(tiles, ids.shape[1], heads, 3 * dim_head), generator=generator
|
||||
).to(device=device, dtype=dtype)
|
||||
attn = SimpleNamespace(
|
||||
dim_head=dim_head, _sgl_unit_weight=None, norm_q=SimpleNamespace(eps=1e-5)
|
||||
)
|
||||
|
||||
per_tile = qkv.clone()
|
||||
for tile in range(tiles):
|
||||
query, key, _ = per_tile[tile : tile + 1].chunk(3, dim=-1)
|
||||
assert _fused_qknorm_rope(attn, query, key, rotary)
|
||||
query, key, _ = qkv.chunk(3, dim=-1)
|
||||
assert _fused_qknorm_rope(attn, query, key, batched_rotary)
|
||||
assert torch.equal(qkv, per_tile)
|
||||
|
||||
query, key, _ = qkv.chunk(3, dim=-1)
|
||||
batched = apply_rotary_pos_emb_qk(query, key, batched_rotary)
|
||||
for tile in range(tiles):
|
||||
query, key, _ = qkv[tile : tile + 1].chunk(3, dim=-1)
|
||||
for got, want in zip(batched, apply_rotary_pos_emb_qk(query, key, rotary)):
|
||||
assert torch.equal(got[tile : tile + 1], want)
|
||||
|
||||
|
||||
def test_audio_vae_attention_defaults_to_local_sdpa_and_allows_fa():
|
||||
class RecordingFA(nn.Module):
|
||||
backend = AttentionBackendEnum.FA
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import shutil
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
@@ -100,15 +102,15 @@ def test_video_with_audio_uses_single_pass_encoder(tmp_path, monkeypatch):
|
||||
def write(*_args, **_kwargs):
|
||||
pass
|
||||
|
||||
def mimsave_spy(path, frames, **kwargs):
|
||||
def encode_spy(path, frames, **kwargs):
|
||||
calls.append((path, frames, kwargs))
|
||||
assert kwargs["audio_path"].endswith(".wav")
|
||||
assert kwargs["audio_codec"] == "aac"
|
||||
assert kwargs["quality"] == 5
|
||||
|
||||
def fail_legacy_mux(**_kwargs):
|
||||
raise AssertionError("the two-pass mux path should not run")
|
||||
|
||||
monkeypatch.setattr(output_utils.imageio, "mimsave", mimsave_spy)
|
||||
monkeypatch.setattr(output_utils, "_save_video_ffmpeg", encode_spy)
|
||||
monkeypatch.setattr(output_utils, "scipy_wavfile", FakeWavFile)
|
||||
monkeypatch.setattr(output_utils, "_maybe_mux_audio_into_mp4", fail_legacy_mux)
|
||||
|
||||
@@ -144,13 +146,13 @@ def test_x264_preset_reaches_every_video_save_path(tmp_path, monkeypatch):
|
||||
presets.append(("direct", kwargs["x264_preset"]))
|
||||
return False
|
||||
|
||||
def mimsave_spy(_path, _frames, **kwargs):
|
||||
encoder = "single-pass" if "audio_path" in kwargs else "imageio"
|
||||
presets.append((encoder, kwargs["output_params"][-1]))
|
||||
def encode_spy(_path, _frames, **kwargs):
|
||||
encoder = "single-pass" if "audio_path" in kwargs else "ffmpeg"
|
||||
presets.append((encoder, kwargs["x264_preset"]))
|
||||
|
||||
monkeypatch.setattr(output_utils, "_try_save_cuda_videos_direct", parallel_save)
|
||||
monkeypatch.setattr(output_utils, "_try_save_cuda_video_direct", direct_save)
|
||||
monkeypatch.setattr(output_utils.imageio, "mimsave", mimsave_spy)
|
||||
monkeypatch.setattr(output_utils, "_save_video_ffmpeg", encode_spy)
|
||||
monkeypatch.setattr(output_utils, "scipy_wavfile", FakeWavFile)
|
||||
|
||||
output_utils.save_outputs(
|
||||
@@ -177,7 +179,7 @@ def test_x264_preset_reaches_every_video_save_path(tmp_path, monkeypatch):
|
||||
assert {encoder for encoder, _ in presets} == {
|
||||
"parallel",
|
||||
"direct",
|
||||
"imageio",
|
||||
"ffmpeg",
|
||||
"single-pass",
|
||||
}
|
||||
assert {preset for _, preset in presets} == {"ultrafast"}
|
||||
@@ -193,12 +195,12 @@ def test_video_audio_single_pass_failure_falls_back(tmp_path, monkeypatch):
|
||||
def write(*_args, **_kwargs):
|
||||
pass
|
||||
|
||||
def mimsave_spy(path, frames, **kwargs):
|
||||
def encode_spy(path, frames, **kwargs):
|
||||
calls.append((path, frames, kwargs))
|
||||
if "audio_path" in kwargs:
|
||||
raise RuntimeError("unsupported audio input")
|
||||
|
||||
monkeypatch.setattr(output_utils.imageio, "mimsave", mimsave_spy)
|
||||
monkeypatch.setattr(output_utils, "_save_video_ffmpeg", encode_spy)
|
||||
monkeypatch.setattr(output_utils, "scipy_wavfile", FakeWavFile)
|
||||
monkeypatch.setattr(
|
||||
output_utils,
|
||||
@@ -316,3 +318,11 @@ def test_multiple_videos_use_parallel_direct_save_with_serial_fallback(
|
||||
assert kwargs["fps"] == 24
|
||||
assert len(serial_calls) == 1
|
||||
assert serial_calls[0]["save_file_path"] == paths[1]
|
||||
|
||||
|
||||
def test_video_output_preserves_pinned_encoder(monkeypatch):
|
||||
monkeypatch.setattr(shutil, "which", lambda _: "/system/ffmpeg")
|
||||
monkeypatch.setattr(
|
||||
output_utils.imageio_ffmpeg, "get_ffmpeg_exe", lambda: "/pinned/ffmpeg"
|
||||
)
|
||||
assert output_utils._resolve_ffmpeg_exe() == "/pinned/ffmpeg"
|
||||
|
||||
@@ -7,6 +7,7 @@ catches the failure this guards against -- the preset never being passed, and
|
||||
ffmpeg silently applying its own default.
|
||||
"""
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import subprocess
|
||||
|
||||
@@ -14,11 +15,17 @@ import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import sglang.multimodal_gen.runtime.entrypoints.utils as output_utils
|
||||
from sglang.multimodal_gen.configs.sample.sampling_params import (
|
||||
DataType,
|
||||
SamplingParams,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.entrypoints.utils import X264_PRESET, save_outputs
|
||||
from sglang.multimodal_gen.runtime.entrypoints.utils import (
|
||||
X264_PRESET,
|
||||
MaterializedOutput,
|
||||
save_materialized_output,
|
||||
save_outputs,
|
||||
)
|
||||
|
||||
FPS = 8
|
||||
FRAMES = 8
|
||||
@@ -97,7 +104,7 @@ def test_saved_video_carries_the_configured_preset(tmp_path):
|
||||
@pytest.mark.parametrize(
|
||||
"device",
|
||||
[
|
||||
# a host tensor saves through imageio, a CUDA one through the ffmpeg pipe
|
||||
# host and CUDA tensors use the same system ffmpeg encoder
|
||||
"cpu",
|
||||
pytest.param(
|
||||
"cuda",
|
||||
@@ -166,3 +173,75 @@ def test_saved_video_decodes_to_every_frame(tmp_path):
|
||||
codec, count = probe.split(",")
|
||||
assert codec == "h264"
|
||||
assert int(count) == FRAMES
|
||||
|
||||
|
||||
@pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="needs ffmpeg")
|
||||
@pytest.mark.parametrize("with_audio", [False, True])
|
||||
@pytest.mark.parametrize("compression", [10, 100])
|
||||
def test_materialized_video_uses_ffmpeg(tmp_path, monkeypatch, with_audio, compression):
|
||||
def reject_imageio(*args, **kwargs):
|
||||
raise AssertionError("video output must use ffmpeg directly")
|
||||
|
||||
monkeypatch.setattr(output_utils.imageio, "mimsave", reject_imageio)
|
||||
rng = np.random.default_rng(3)
|
||||
# non-contiguous RGB and non-macroblock dimensions exercise the raw pipe and scaling
|
||||
frames = rng.integers(0, 256, (FRAMES, 30, 34, 3), dtype=np.uint8)[:, :, ::-1]
|
||||
saved = tmp_path / "materialized.mp4"
|
||||
save_materialized_output(
|
||||
MaterializedOutput(
|
||||
sample=None,
|
||||
frames=list(frames),
|
||||
audio=np.zeros((32000, 2), dtype=np.float32) if with_audio else None,
|
||||
fps=FPS,
|
||||
),
|
||||
DataType.VIDEO,
|
||||
str(saved),
|
||||
audio_sample_rate=32000,
|
||||
output_compression=compression,
|
||||
x264_preset="ultrafast",
|
||||
)
|
||||
streams = json.loads(
|
||||
subprocess.run(
|
||||
[
|
||||
"ffprobe",
|
||||
"-v",
|
||||
"error",
|
||||
"-count_frames",
|
||||
"-show_streams",
|
||||
"-of",
|
||||
"json",
|
||||
str(saved),
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
).stdout
|
||||
)["streams"]
|
||||
video = next(stream for stream in streams if stream["codec_type"] == "video")
|
||||
assert (video["width"], video["height"]) == (48, 32)
|
||||
assert video["codec_name"] == "h264"
|
||||
assert int(video["nb_read_frames"]) == FRAMES
|
||||
assert video["r_frame_rate"] == f"{FPS}/1"
|
||||
options = _x264_options(saved)
|
||||
if compression == 100:
|
||||
assert options["qp"] == "0"
|
||||
else:
|
||||
assert options["crf"] == f"{int((1 - compression / 100) * 51):.1f}"
|
||||
audio = [stream for stream in streams if stream["codec_type"] == "audio"]
|
||||
assert bool(audio) == with_audio
|
||||
if with_audio:
|
||||
assert audio[0]["codec_name"] == "aac"
|
||||
assert audio[0]["sample_rate"] == "32000"
|
||||
assert audio[0]["channels"] == 2
|
||||
|
||||
|
||||
@pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="needs ffmpeg")
|
||||
def test_ffmpeg_failure_is_reported(tmp_path):
|
||||
with pytest.raises(RuntimeError, match="ffmpeg video encoding failed"):
|
||||
output_utils._save_video_ffmpeg(
|
||||
str(tmp_path / "invalid.mp4"),
|
||||
[np.zeros((16, 16, 3), dtype=np.uint8)],
|
||||
fps=FPS,
|
||||
quality=5,
|
||||
x264_preset="invalid",
|
||||
)
|
||||
|
||||
@@ -19,6 +19,10 @@ from sglang.multimodal_gen.runtime.layers.attention.backends.video_sparse_attn_h
|
||||
VideoSparseAttentionH3Impl,
|
||||
VideoSparseAttentionH3MetadataBuilder,
|
||||
_topk_tile_lists,
|
||||
vsa_h3_fold_gate,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.attention.backends.vsa_h3_kernels import (
|
||||
vsa_h3_topk_lists,
|
||||
)
|
||||
|
||||
requires_cuda = pytest.mark.skipif(
|
||||
@@ -163,6 +167,29 @@ def test_topk_tile_list_semantics() -> None:
|
||||
assert (_lists_to_mask(compete, n_tiles).sum(dim=-1) == keep + num_prefix).all()
|
||||
|
||||
|
||||
@requires_cuda
|
||||
@pytest.mark.parametrize(
|
||||
"num_prefix, num_video, sparsity",
|
||||
[(29, 528, 0.8), (0, 64, 0.9), (30, 1541, 0.05)],
|
||||
)
|
||||
def test_topk_list_kernel_matches_sorted_topk(num_prefix, num_video, sparsity) -> None:
|
||||
"""The unsorted-top-k list kernel must emit the sorted lists and leave its
|
||||
scratch mask zeroed for the next layer."""
|
||||
n_tiles = num_prefix + num_video
|
||||
scores = torch.randn(HEADS, n_tiles, n_tiles, device="cuda")
|
||||
reference = _topk_tile_lists(scores, num_prefix, num_video, sparsity, True)
|
||||
keep = reference.shape[-1] - num_prefix
|
||||
out = torch.full((HEADS, num_video, n_tiles), -1, dtype=torch.int32, device="cuda")
|
||||
out[..., :num_prefix] = torch.arange(num_prefix, dtype=torch.int32, device="cuda")
|
||||
mask = torch.zeros(HEADS, num_video, num_video, dtype=torch.int8, device="cuda")
|
||||
for _ in range(2): # the scratch mask must come back zeroed
|
||||
picked = scores[:, num_prefix:, num_prefix:].topk(keep, sorted=False).indices
|
||||
vsa_h3_topk_lists(picked, mask, out, num_prefix)
|
||||
assert torch.equal(out[..., : num_prefix + keep], reference)
|
||||
assert not mask.any()
|
||||
assert (out[..., num_prefix + keep :] == -1).all()
|
||||
|
||||
|
||||
def _masked_dense_reference(meta, used, q, k, v, gate, sparsity):
|
||||
"""fp32 reference over the padded tile layout with the top-k tile mask."""
|
||||
n_tiles = meta.num_tiles
|
||||
@@ -215,6 +242,68 @@ def test_sparse_gated_matches_masked_dense_reference() -> None:
|
||||
assert torch.all(out[used:] == 0)
|
||||
|
||||
|
||||
@requires_cuda
|
||||
def test_return_compress_fold_on_row_shards_is_bit_identical() -> None:
|
||||
"""The deferred row-shard fold (Ulysses) must reproduce the in-kernel fold
|
||||
bit for bit: same tile index derivation, same fp32 FMA order."""
|
||||
device = torch.device("cuda")
|
||||
meta = _build_metadata(0.5, device)
|
||||
used, total, (q, k, v) = _packed_qkv(device)
|
||||
gate = (torch.randn_like(q) * 0.1).to(torch.bfloat16)
|
||||
impl = _impl()
|
||||
|
||||
folded = _run(impl, meta, used, total, q, k, v, gate=gate)
|
||||
cu = torch.tensor([0, used, total], dtype=torch.int32, device=device)
|
||||
out, out_compress = impl.forward_varlen(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
cu_seqlens=cu,
|
||||
max_seqlen=used,
|
||||
cu_seqlens_host=(0, used, total),
|
||||
attn_metadata=meta,
|
||||
return_compress=True,
|
||||
)
|
||||
shard = total // 2
|
||||
for row_start in (0, shard):
|
||||
rows = slice(row_start, row_start + shard)
|
||||
vsa_h3_fold_gate(out[rows], gate[rows], out_compress, meta, row_start)
|
||||
assert torch.equal(out, folded)
|
||||
|
||||
|
||||
@requires_cuda
|
||||
@pytest.mark.parametrize("sparsity", [0.0, 0.8])
|
||||
def test_odd_tile_count_matches_reference(sparsity: float) -> None:
|
||||
"""An odd tile count (padded to even for the SM100 kernel) must keep the
|
||||
pooled means, the top-k lists and the gate fold on the real tiles."""
|
||||
device = torch.device("cuda")
|
||||
prefix = (70, 0, 40)
|
||||
meta = VideoSparseAttentionH3MetadataBuilder().build(
|
||||
current_timestep=0,
|
||||
raw_latent_shape=VIDEO_SHAPE,
|
||||
patch_size=(1, 1, 1),
|
||||
VSA_sparsity=sparsity,
|
||||
prefix_segments=prefix,
|
||||
device=device,
|
||||
)
|
||||
assert meta.num_tiles % 2 == 1
|
||||
used = sum(prefix) + math.prod(VIDEO_SHAPE)
|
||||
total = (used + 63) // 64 * 64
|
||||
generator = torch.Generator(device="cpu").manual_seed(11)
|
||||
q, k, v, gate = (
|
||||
torch.randn((total, HEADS, HEAD_DIM), generator=generator).to(
|
||||
device=device, dtype=torch.bfloat16
|
||||
)
|
||||
for _ in range(4)
|
||||
)
|
||||
gate = gate * 0.1
|
||||
out = _run(_impl(), meta, used, total, q, k, v, gate=gate)
|
||||
reference = _masked_dense_reference(meta, used, q, k, v, gate, sparsity)
|
||||
diff = (out[:used].float() - reference).abs().max().item()
|
||||
assert diff < 2e-2, f"odd tile count, sparsity {sparsity}: max diff {diff}"
|
||||
assert torch.all(out[used:] == 0)
|
||||
|
||||
|
||||
def test_metadata_tile_geometry_accounts_every_row() -> None:
|
||||
device = torch.device("cpu")
|
||||
meta = _build_metadata(0.9, device)
|
||||
|
||||
Reference in New Issue
Block a user