[diffusion] perf: FastH3 8-Step V2 native support, native Blackwell VSA and batched H3 VAE decode (#37662)

Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
Co-authored-by: mickqian <mickqian@users.noreply.github.com>
Co-authored-by: Mick <mickjagger19@icloud.com>
This commit is contained in:
Kevin Mi
2026-10-08 14:35:27 +08:00
committed by GitHub
co-authored by Claude Fable 5.1 mickqian Mick
parent 57e9ff5403
commit 4f320fb85a
43 changed files with 3567 additions and 229 deletions
@@ -0,0 +1,85 @@
// SPDX-License-Identifier: Apache-2.0
#include <sgl_kernel/tensor.h>
#include <sgl_kernel/utils.h>
#include <sgl_kernel/utils.cuh>
#include <dlpack/dlpack.h>
#include <tvm/ffi/container/tensor.h>
#define VSA_BHSD true
#include "vsa_block_sparse_sm100/block_sparse_launch_sm100a.cuh"
namespace sglang {
/**
* \brief FastVideo's warp-specialized tcgen05 block-sparse attention forward for 64-token
* tiles (sm_100a / sm_103a).
*
* \param q, k, v, out [B, H, S, 128] bf16, contiguous; S == num_blocks * 64, num_blocks even.
* \param q2k_idx [B * H * num_blocks, max_kv] int32 key-tile lists (valid below q2k_num).
* \param q2k_num [B * H * num_blocks] int32 per-query-tile list lengths (0 allowed).
* \param block_sizes [num_blocks] int32 valid tokens per key tile.
*/
struct VsaBlockSparseSm100Kernel {
static void
run(const tvm::ffi::TensorView q,
const tvm::ffi::TensorView k,
const tvm::ffi::TensorView v,
const tvm::ffi::TensorView q2k_idx,
const tvm::ffi::TensorView q2k_num,
const tvm::ffi::TensorView block_sizes,
tvm::ffi::TensorView out,
double sm_scale) {
using namespace host;
auto B = SymbolicSize{"batch"};
auto H = SymbolicSize{"heads"};
auto S = SymbolicSize{"seq_pad"};
auto R = SymbolicSize{"rows"};
auto N = SymbolicSize{"num_blocks"};
auto MAXKV = SymbolicSize{"max_kv"};
auto device = SymbolicDevice{};
device.set_options<kDLCUDA>();
TensorMatcher({B, H, S, HEAD_DIM})
.with_dtype<bf16_t>()
.with_device(device)
.verify(q)
.verify(k)
.verify(v)
.verify(out);
TensorMatcher({R, MAXKV}).with_dtype<int32_t>().with_device(device).verify(q2k_idx);
TensorMatcher({R}).with_dtype<int32_t>().with_device(device).verify(q2k_num);
TensorMatcher({N}).with_dtype<int32_t>().with_device(device).verify(block_sizes);
const int64_t num_blocks = N.unwrap();
CHECK_HOST(num_blocks % 2 == 0) << "num_blocks must be even, got " << num_blocks;
CHECK_HOST(S.unwrap() == num_blocks * BLOCK) << "seq_pad " << S.unwrap() << " != num_blocks * " << BLOCK;
CHECK_HOST(R.unwrap() == B.unwrap() * H.unwrap() * num_blocks) << "q2k rows must be batch * heads * num_blocks";
BlockSparseVsaArgs a{};
a.q = static_cast<const __nv_bfloat16*>(q.data_ptr());
a.k = static_cast<const __nv_bfloat16*>(k.data_ptr());
a.v = static_cast<const __nv_bfloat16*>(v.data_ptr());
a.v_t = nullptr;
a.o = static_cast<__nv_bfloat16*>(out.data_ptr());
a.lse = nullptr;
a.q2k_idx = static_cast<const int*>(q2k_idx.data_ptr());
a.q2k_num = static_cast<const int*>(q2k_num.data_ptr());
a.variable_block_sizes = static_cast<const int*>(block_sizes.data_ptr());
a.batch = static_cast<int>(B.unwrap());
a.num_heads = static_cast<int>(H.unwrap());
a.seqlen = static_cast<int>(S.unwrap());
a.head_dim = HEAD_DIM;
a.num_blocks = static_cast<int>(num_blocks);
a.max_kv = static_cast<int>(MAXKV.unwrap());
a.sm_scale = static_cast<float>(sm_scale);
const DLDevice dev = device.unwrap();
auto stream = static_cast<cudaStream_t>(::TVMFFIEnvGetStream(dev.device_type, dev.device_id));
CHECK_CUDA(launch_block_sparse_sm100a(a, stream)) << "vsa_block_sparse_sm100 launch";
}
};
} // namespace sglang
@@ -0,0 +1,205 @@
// Copied and adapted from: https://github.com/hao-ai-lab/FastVideo
// (fastvideo-kernel/csrc/attention/block_sparse_launch_sm100a.cuh, Apache-2.0). Inference-only
// forward; the sm_103a device pass is admitted alongside sm_100a.
#ifndef BLOCK_SPARSE_VSA_LAUNCH_SM100A_CUH
#define BLOCK_SPARSE_VSA_LAUNCH_SM100A_CUH
// Launch surface for the sm_100a VSA block-sparse FMHA forward.
//
// Everything a caller needs: a POD argument struct, a predicate saying whether this build can
// run those arguments, and one launch entry point. The benchmark in
// block_sparse_bench_sm100a.cu and the torch binding both go through here, so there is
// one tensormap construction and one launch configuration rather than two that can drift.
//
// Two compile-time knobs select the four builds:
// VSA_BLK128 false -> 64-token sparse blocks, true -> 128-token
// VSA_BHSD false -> [token][head][dim] (BSHD), true -> [batch][head][token][dim] (BHSD)
#include "block_sparse_kernel_sm100a.cuh"
namespace VSA_NAMESPACE {
struct BlockSparseVsaArgs {
const __nv_bfloat16* q;
const __nv_bfloat16* k;
const __nv_bfloat16* v; // natural layout; only blk128 reads it (blk64 still needs v_t)
const __nv_bfloat16* v_t; // unused: kept so the bench's V_T buffer still binds
__nv_bfloat16* o;
float* lse; // [batch, num_heads, seqlen] fp32, or nullptr
const int* q2k_idx; // [batch*num_heads*num_blocks, max_kv] int32
const int* q2k_num; // [batch*num_heads*num_blocks] int32
const int* variable_block_sizes; // [num_blocks] int32, valid tokens per block
int batch;
int num_heads;
int seqlen;
int head_dim;
int num_blocks;
int max_kv;
float sm_scale;
};
// cudaSuccess iff this build can run `a`. Deliberately conservative: the caller is expected
// to fall back to its own implementation rather than get a wrong answer.
__host__ inline cudaError_t block_sparse_supported(const BlockSparseVsaArgs& a) {
if (a.head_dim != HEAD_DIM) return cudaErrorInvalidValue; // compile-time in the kernel
if (a.num_blocks % 2 != 0) return cudaErrorInvalidValue; // a CTA owns an adjacent pair
if (a.seqlen != a.num_blocks * BLOCK) return cudaErrorInvalidValue;
if (a.max_kv < 1 || a.num_blocks < 1) return cudaErrorInvalidValue;
if (a.q == nullptr || a.k == nullptr || a.o == nullptr) return cudaErrorInvalidValue;
if (a.q2k_idx == nullptr || a.q2k_num == nullptr) return cudaErrorInvalidValue;
// FastVideo always supplies this; without it padded keys would be attended as real zeros.
if (a.variable_block_sizes == nullptr) return cudaErrorInvalidValue;
// V is read MN-major at BOTH block sizes now, so no pre-transposed V_T is ever needed.
if (a.v == nullptr) return cudaErrorInvalidValue;
return cudaSuccess;
}
__host__ inline cudaError_t launch_block_sparse_sm100a(const BlockSparseVsaArgs& a,
cudaStream_t stream) {
const cudaError_t sup = block_sparse_supported(a);
if (sup != cudaSuccess) return sup;
const int B = a.batch, H = a.num_heads, S = a.seqlen, hd = a.head_dim;
const int num_blocks = a.num_blocks, max_kv = a.max_kv;
const long tq = (long)B * S;
const int packed_mtiles_per_seq = num_blocks / 2;
const int total_work = B * H * packed_mtiles_per_seq;
constexpr bool BHSD = VSA_BHSD;
CUtensorMap tq_, tk_, tvt_, tv_, to_;
{
uint64_t gd[4] = { (uint64_t)SUB_COLS_BF16, BHSD ? (uint64_t)((long)B * H) : (uint64_t)H,
BHSD ? (uint64_t)S : (uint64_t)tq, (uint64_t)Q_SUBTILES };
uint64_t gs[3] = { BHSD ? (uint64_t)((long)S * hd) * 2u : (uint64_t)hd * 2u,
BHSD ? (uint64_t)hd * 2u : (uint64_t)((long)H * hd) * 2u,
(uint64_t)SUB_COLS_BF16 * 2u };
uint32_t bd[4] = { (uint32_t)SUB_COLS_BF16, 1u, (uint32_t)M_TILE, (uint32_t)Q_SUBTILES };
uint32_t es[4] = { 1u, 1u, 1u, 1u };
if (cuTensorMapEncodeTiled(&tq_, CU_TENSOR_MAP_DATA_TYPE_BFLOAT16, 4,
const_cast<__nv_bfloat16*>(a.q), gd, gs, bd, es,
CU_TENSOR_MAP_INTERLEAVE_NONE, CU_TENSOR_MAP_SWIZZLE_128B,
CU_TENSOR_MAP_L2_PROMOTION_L2_128B,
CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE) != CUDA_SUCCESS)
return cudaErrorInvalidValue;
if (cuTensorMapEncodeTiled(&to_, CU_TENSOR_MAP_DATA_TYPE_BFLOAT16, 4, a.o, gd, gs, bd, es,
CU_TENSOR_MAP_INTERLEAVE_NONE, CU_TENSOR_MAP_SWIZZLE_128B,
CU_TENSOR_MAP_L2_PROMOTION_L2_128B,
CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE) != CUDA_SUCCESS)
return cudaErrorInvalidValue;
}
{
uint64_t gd[4] = { (uint64_t)SUB_COLS_BF16,
BHSD ? (uint64_t)S : (uint64_t)tq,
BHSD ? (uint64_t)(hd / SUB_COLS_BF16)
: (uint64_t)((long)H * hd / SUB_COLS_BF16),
(uint64_t)((long)B * H) };
uint64_t gs[3] = { BHSD ? (uint64_t)hd * 2u : (uint64_t)((long)H * hd) * 2u,
(uint64_t)SUB_COLS_BF16 * 2u,
(uint64_t)((long)S * hd) * 2u };
uint32_t bd[4] = { (uint32_t)SUB_COLS_BF16, (uint32_t)BLOCK,
BLK128 ? (uint32_t)K_SUBTILES : 1u, 1u };
uint32_t es[4] = { 1u, 1u, 1u, 1u };
if (cuTensorMapEncodeTiled(&tk_, CU_TENSOR_MAP_DATA_TYPE_BFLOAT16, BHSD ? 4 : 3,
const_cast<__nv_bfloat16*>(a.k), gd, gs, bd, es,
CU_TENSOR_MAP_INTERLEAVE_NONE, CU_TENSOR_MAP_SWIZZLE_128B,
CU_TENSOR_MAP_L2_PROMOTION_L2_128B,
CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE) != CUDA_SUCCESS)
return cudaErrorInvalidValue;
// V map is byte-for-byte the K map over a.v: MN-major V needs no transpose (blk128).
const __nv_bfloat16* vbase = a.v ? a.v : a.k;
if (cuTensorMapEncodeTiled(&tv_, CU_TENSOR_MAP_DATA_TYPE_BFLOAT16, BHSD ? 4 : 3,
const_cast<__nv_bfloat16*>(vbase), gd, gs, bd, es,
CU_TENSOR_MAP_INTERLEAVE_NONE, CU_TENSOR_MAP_SWIZZLE_128B,
CU_TENSOR_MAP_L2_PROMOTION_L2_128B,
CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE) != CUDA_SUCCESS)
return cudaErrorInvalidValue;
}
// V_T map: blk64 only. Unused at blk128 but must still be a valid tensormap to pass by value.
{
const __nv_bfloat16* vt = a.v_t ? a.v_t : a.k;
if constexpr (BLK128) {
uint64_t gd[3] = { (uint64_t)SUB_COLS_BF16, (uint64_t)((long)H * hd),
(uint64_t)((long)tq / SUB_COLS_BF16) };
uint64_t gs[2] = { (uint64_t)tq * 2u, (uint64_t)SUB_COLS_BF16 * 2u };
uint32_t bd[3] = { (uint32_t)SUB_COLS_BF16, (uint32_t)hd, (uint32_t)V_SUBTILES };
uint32_t es[3] = { 1u, 1u, 1u };
if (cuTensorMapEncodeTiled(&tvt_, CU_TENSOR_MAP_DATA_TYPE_BFLOAT16, 3,
const_cast<__nv_bfloat16*>(vt), gd, gs, bd, es,
CU_TENSOR_MAP_INTERLEAVE_NONE, CU_TENSOR_MAP_SWIZZLE_128B,
CU_TENSOR_MAP_L2_PROMOTION_L2_128B,
CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE) != CUDA_SUCCESS)
return cudaErrorInvalidValue;
} else {
if (make_tma_2d_tiled(&tvt_, const_cast<__nv_bfloat16*>(vt), (long)H * hd, (int)tq, hd,
SUB_COLS_BF16, 2, CU_TENSOR_MAP_DATA_TYPE_BFLOAT16,
CU_TENSOR_MAP_SWIZZLE_128B) != cudaSuccess)
return cudaErrorInvalidValue;
}
}
const size_t smem =
(size_t)2 * Q_TILE_BYTES + NUM_KV_STAGES * KV_RING_SLOT_BYTES
+ (size_t)2 * M_TILE * HEAD_DIM * sizeof(__nv_bfloat16)
+ (2 * NUM_KV_STAGES + 22) * 8
+ (size_t)CLC_STAGES * (2 * 8 + 16) + 16
+ 8
+ (size_t)2 * STAT_REGIONS * STATS * sizeof(float)
+ 256;
#ifndef VSA_NAMED_BAR
#define VSA_NAMED_BAR false
#endif
#ifndef VSA_THROTTLE
#define VSA_THROTTLE false
#endif
#ifndef VSA_USE_CLC
#define VSA_USE_CLC true
#endif
constexpr bool FULL_NAMED_BAR = VSA_NAMED_BAR, EX2_EMU = true, SPLIT_P = true,
SOFTMAX_THROTTLE = VSA_THROTTLE, USE_CLC = VSA_USE_CLC,
Q_RASTER = true, MHA = true;
auto kfn = &fmha_context_bf16_gen_kernel<32, FULL_NAMED_BAR, EX2_EMU, SPLIT_P,
SOFTMAX_THROTTLE, USE_CLC, Q_RASTER, MHA,
/*RESCALE_THRESHOLD=*/8, /*BHSD=*/VSA_BHSD>;
cudaError_t e = cudaFuncSetAttribute(kfn, cudaFuncAttributeMaxDynamicSharedMemorySize, (int)smem);
if (e != cudaSuccess) return e;
const unsigned long long magic0 = make_magic((unsigned)(H * packed_mtiles_per_seq));
const unsigned long long magic1 = make_magic((unsigned)H);
const unsigned long long magic2 = make_magic((unsigned)packed_mtiles_per_seq);
const float scale_log2 = a.sm_scale * (float)M_LOG2E;
int numSM = 0;
e = cudaDeviceGetAttribute(&numSM, cudaDevAttrMultiProcessorCount, 0);
if (e != cudaSuccess) return e;
const int num_ctas = USE_CLC ? total_work : (total_work < numSM ? total_work : numSM);
dim3 grid(num_ctas, 1, 1), block(N_WARPS * 32, 1, 1);
if (USE_CLC) {
cudaLaunchConfig_t cfg = {};
cfg.gridDim = grid; cfg.blockDim = block; cfg.dynamicSmemBytes = smem; cfg.stream = stream;
cudaLaunchAttribute cfgAttr[1];
cfgAttr[0].id = cudaLaunchAttributeClusterDimension;
cfgAttr[0].val.clusterDim.x = 1; cfgAttr[0].val.clusterDim.y = 1;
cfgAttr[0].val.clusterDim.z = 1;
cfg.attrs = cfgAttr; cfg.numAttrs = 1;
return cudaLaunchKernelEx(&cfg, kfn, tq_, tk_, tvt_, tv_, to_, S, H, scale_log2, B,
num_blocks, packed_mtiles_per_seq, max_kv, magic0, magic1, magic2,
a.q2k_idx, a.q2k_num, a.variable_block_sizes, a.lse);
}
kfn<<<grid, block, smem, stream>>>(tq_, tk_, tvt_, tv_, to_, S, H, scale_log2, B, num_blocks,
packed_mtiles_per_seq, max_kv, magic0, magic1, magic2,
a.q2k_idx, a.q2k_num, a.variable_block_sizes, a.lse);
return cudaGetLastError();
}
} // namespace VSA_NAMESPACE
// Callers (the bench, the torch binding) keep using unqualified names; each translation unit
// only ever sees the one configuration its VSA_BLK128 selected.
using namespace VSA_NAMESPACE;
#endif // BLOCK_SPARSE_VSA_LAUNCH_SM100A_CUH
@@ -0,0 +1,880 @@
// Copied and adapted from: https://github.com/hao-ai-lab/FastVideo
// (fastvideo-kernel/csrc/attention/primitives.cuh, Apache-2.0). Inference-only
// forward; the sm_103a device pass is admitted alongside sm_100a.
// primitives.cuh -- device primitives for the sm_100a VSA block-sparse attention
// forward: tcgen05 (alloc / mma / ld / st / commit / wait / fence), TMA load / store /
// tensormap, mbarrier, cluster launch control, setmaxnreg, fast math, and the FMHA helpers.
//
// Generated and pruned to what the kernel reaches -- do not edit by hand.
#pragma once
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <cuda.h>
#include <cuda_bf16.h>
#include <cuda_fp16.h>
#include <cuda_runtime.h>
#include <cmath>
#include <cassert>
#include <cstring>
#include <vector_types.h>
#ifndef CUDA_CHECK
#define CUDA_CHECK(stmt) do { \
cudaError_t _e = (stmt); \
if (_e != cudaSuccess) { \
fprintf(stderr, "CUDA error %s:%d: %s -> %s\n", \
__FILE__, __LINE__, #stmt, cudaGetErrorString(_e)); \
std::exit(1); \
} \
} while (0)
#endif
__device__ __forceinline__
uint64_t mbarrier_arrive(uint32_t mbar_smem) {
uint64_t state;
asm volatile("mbarrier.arrive.shared::cta.b64 %0, [%1];\n"
: "=l"(state) : "r"(mbar_smem) : "memory");
return state;
}
__device__ __forceinline__
void mbarrier_arrive_nostate(uint32_t mbar_smem) {
asm volatile("mbarrier.arrive.shared::cta.b64 _, [%0];\n"
:: "r"(mbar_smem) : "memory");
}
__device__ __forceinline__
void mbarrier_arrive_cluster_default(uint32_t cluster_smem_addr) {
asm volatile("mbarrier.arrive.shared::cluster.b64 _, [%0];\n"
:: "r"(cluster_smem_addr) : "memory");
}
__device__ __forceinline__
void mbarrier_arrive_expect_tx(uint32_t mbar_smem, uint32_t expected_bytes) {
asm volatile("mbarrier.arrive.expect_tx.release.cta.shared::cta.b64 _, [%0], %1;\n"
:: "r"(mbar_smem), "r"(expected_bytes) : "memory");
}
__device__ __forceinline__
void mbarrier_wait_parity_suspend(uint32_t mbar_smem, uint32_t phase_parity) {
asm volatile(
"{\n"
".reg .pred P1;\n"
"LAB_WAIT:\n"
"mbarrier.try_wait.parity.shared::cta.b64 P1, [%0], %1, 10000000;\n"
"@!P1 bra.uni LAB_WAIT;\n"
"}\n"
:: "r"(mbar_smem), "r"(phase_parity) : "memory");
}
__device__ __forceinline__
void mbarrier_wait_parity(uint32_t mbar_smem, uint32_t phase_parity) {
asm volatile(
"{\n"
".reg .pred P1;\n"
"LAB_WAIT_HOT:\n"
"mbarrier.try_wait.parity.shared::cta.b64 P1, [%0], %1;\n"
"@!P1 bra.uni LAB_WAIT_HOT;\n"
"}\n"
:: "r"(mbar_smem), "r"(phase_parity) : "memory");
}
__device__ __forceinline__
void fence_proxy_async_shared_cta() {
asm volatile("fence.proxy.async.shared::cta;\n" ::: "memory");
}
__device__ __forceinline__
void fence_proxy_async_shared() {
asm volatile("fence.proxy.async.shared::cta;\n" ::: "memory");
}
__device__ __forceinline__ void clc_try_cancel_async(
uint32_t smem_dst, uint32_t mbar_smem) {
asm volatile(
"clusterlaunchcontrol.try_cancel.async.shared::cta.mbarrier::complete_tx::bytes.b128"
" [%0], [%1];\n"
:: "r"(smem_dst), "r"(mbar_smem) : "memory");
}
__device__ __forceinline__ void clc_load_response(
uint32_t smem_slot, uint32_t& r0, uint32_t& r1,
uint32_t& r2, uint32_t& r3) {
asm volatile("ld.shared::cta.v4.b32 {%0, %1, %2, %3}, [%4];\n"
: "=r"(r0), "=r"(r1), "=r"(r2), "=r"(r3)
: "r"(smem_slot));
}
template <int NUM_STAGES>
struct MbarrierPhaseTracker {
uint32_t phase[NUM_STAGES];
int idx;
__device__ __forceinline__
void init() {
for (int i = 0; i < NUM_STAGES; ++i) phase[i] = 0;
idx = 0;
}
__device__ __forceinline__
uint32_t current_phase() const { return phase[idx]; }
__device__ __forceinline__
void advance() {
phase[idx] ^= 1u;
idx = (idx + 1) % NUM_STAGES;
}
__device__ __forceinline__
int stage() const { return idx; }
};
template <int NUM_STAGES>
struct PhaseTracker {
int stage;
uint32_t phase;
__device__ __forceinline__
PhaseTracker() : stage(0), phase(0) {}
__device__ __forceinline__
void advance() {
stage++;
if (stage == NUM_STAGES) {
stage = 0;
phase ^= 1;
}
}
__device__ __forceinline__
int get_stage() const { return stage; }
__device__ __forceinline__
uint32_t get_phase() const { return phase; }
};
template <int NUM_STAGES>
struct EmptyPhaseTracker {
int stage;
uint32_t phase;
__device__ __forceinline__
EmptyPhaseTracker() : stage(0), phase(1) {}
__device__ __forceinline__
void advance() {
stage++;
if (stage == NUM_STAGES) {
stage = 0;
phase ^= 1;
}
}
__device__ __forceinline__
int get_stage() const { return stage; }
__device__ __forceinline__
uint32_t get_phase() const { return phase; }
};
template <int STAGES>
__device__ __forceinline__
void advance_stage_phase(int& stage, uint32_t& phase) {
++stage;
if (stage == STAGES) {
stage = 0;
phase ^= 1u;
}
}
static constexpr uint32_t SM100_CLC_PEER_MASK = 0xFEFFFFFF;
struct ClcTileInfo {
int m_tile;
int n_tile;
bool valid;
};
enum class ClcRasterOrder { AlongN, AlongM };
__device__ __forceinline__
void clc_arrive_expect_tx_cta(uint32_t clc_full_local_addr, uint32_t tx_bytes) {
if ((threadIdx.x & 31) == 0) {
mbarrier_arrive_expect_tx(clc_full_local_addr, tx_bytes);
}
}
__device__ __forceinline__
void clc_consumer_release(uint32_t clc_empty_local_addr) {
uint32_t peer0_addr = clc_empty_local_addr & SM100_CLC_PEER_MASK;
mbarrier_arrive_cluster_default(peer0_addr);
}
__device__ __forceinline__
void clc_consumer_release_cta(uint32_t clc_empty_local_addr) {
mbarrier_arrive_nostate(clc_empty_local_addr);
}
template <int CLUSTER_SHAPE_M, int CLUSTER_SHAPE_N, ClcRasterOrder ORDER>
__device__ __forceinline__
ClcTileInfo clc_parse_response(uint32_t resp_smem_addr) {
uint32_t d0, d1, d2, d3;
fence_proxy_async_shared_cta();
clc_load_response(resp_smem_addr, d0, d1, d2, d3);
const int ctaid_x = static_cast<int>(d0);
const int ctaid_y = static_cast<int>(d1 & 0xFFFFu);
const bool valid = (d2 & 1u) != 0u;
(void)d3;
ClcTileInfo info;
info.valid = valid;
if constexpr (ORDER == ClcRasterOrder::AlongN) {
info.m_tile = ctaid_y / CLUSTER_SHAPE_M;
info.n_tile = ctaid_x / CLUSTER_SHAPE_N;
} else {
info.m_tile = ctaid_x / CLUSTER_SHAPE_M;
info.n_tile = ctaid_y / CLUSTER_SHAPE_N;
}
return info;
}
template <int CLUSTER_SHAPE_M, int CLUSTER_SHAPE_N, ClcRasterOrder ORDER,
int CTA_GROUP = 2, bool SUSPEND = false>
__device__ __forceinline__
ClcTileInfo clc_fetch_next_tile(
uint64_t* clc_full_bar, uint64_t* clc_empty_bar, uint32_t* clc_response,
int clc_cons_stage, uint32_t clc_cons_phase, bool do_release) {
uint32_t full_addr = static_cast<uint32_t>(
__cvta_generic_to_shared(&clc_full_bar[clc_cons_stage]));
if constexpr (SUSPEND) mbarrier_wait_parity_suspend(full_addr, clc_cons_phase);
else mbarrier_wait_parity(full_addr, clc_cons_phase);
uint32_t resp_addr = static_cast<uint32_t>(
__cvta_generic_to_shared(&clc_response[clc_cons_stage * 4]));
ClcTileInfo t = clc_parse_response<
CLUSTER_SHAPE_M, CLUSTER_SHAPE_N, ORDER>(resp_addr);
if (do_release) {
uint32_t empty_local = static_cast<uint32_t>(
__cvta_generic_to_shared(&clc_empty_bar[clc_cons_stage]));
if constexpr (CTA_GROUP == 1) {
clc_consumer_release_cta(empty_local);
} else {
clc_consumer_release(empty_local);
}
}
return t;
}
template <int STAGES = 2>
__device__ __forceinline__
void clc_fetch_next_tile_advance(int& clc_cons_stage,
uint32_t& clc_cons_phase) {
advance_stage_phase<STAGES>(clc_cons_stage, clc_cons_phase);
}
__device__ __forceinline__ unsigned fdiv(unsigned n, unsigned long long pk) {
unsigned M = (unsigned)pk;
if (M == 0u) return n;
return __umulhi(n, M) >> (unsigned)(pk >> 32);
}
__host__ inline unsigned long long make_magic(unsigned d) {
if (d <= 1u) return 0ULL;
unsigned l = 0; while ((1u << (l + 1)) <= d) ++l;
unsigned p = 31u + l;
unsigned long long m = ((1ull << p) + (unsigned long long)d - 1ull) / d;
return (m & 0xffffffffULL) | ((unsigned long long)(p - 32u) << 32);
}
template <int BARRIER_ID>
__device__ __forceinline__
void bar_sync(uint32_t thread_count) {
static_assert(BARRIER_ID >= 0 && BARRIER_ID <= 15,
"bar.sync: BARRIER_ID must be in [0, 15]");
asm volatile("bar.sync %0, %1;\n"
:: "n"(BARRIER_ID), "r"(thread_count) : "memory");
}
template <int BARRIER_ID>
__device__ __forceinline__
void bar_arrive(uint32_t thread_count) {
static_assert(BARRIER_ID >= 0 && BARRIER_ID <= 15,
"bar.arrive: BARRIER_ID must be in [0, 15]");
asm volatile("bar.arrive %0, %1;\n"
:: "n"(BARRIER_ID), "r"(thread_count) : "memory");
}
__device__ __forceinline__ void full_bar_arrive(int m_tile, int band) {
switch (1 + m_tile * 4 + band) {
case 1: bar_arrive<1>(64); break;
case 2: bar_arrive<2>(64); break;
case 3: bar_arrive<3>(64); break;
case 4: bar_arrive<4>(64); break;
case 5: bar_arrive<5>(64); break;
case 6: bar_arrive<6>(64); break;
case 7: bar_arrive<7>(64); break;
case 8: bar_arrive<8>(64); break;
}
}
__device__ __forceinline__ void full_bar_wait(int m_tile, int band) {
switch (1 + m_tile * 4 + band) {
case 1: bar_sync<1>(64); break;
case 2: bar_sync<2>(64); break;
case 3: bar_sync<3>(64); break;
case 4: bar_sync<4>(64); break;
case 5: bar_sync<5>(64); break;
case 6: bar_sync<6>(64); break;
case 7: bar_sync<7>(64); break;
case 8: bar_sync<8>(64); break;
}
}
template <bool IS_CAUSAL, int K_TILE>
__device__ __forceinline__ void mask_s_row_r2p(float* scores, int k_offset, int q_pos, int seqlen_k) {
int n_keep = seqlen_k - k_offset;
if constexpr (IS_CAUSAL) {
const int causal = q_pos - k_offset + 1;
n_keep = n_keep < causal ? n_keep : causal;
}
#pragma unroll
for (int s = 0; s < K_TILE / 32; ++s) {
int m = (s + 1) * 32 - n_keep;
m = m < 0 ? 0 : (m > 32 ? 32 : m);
const uint32_t keep = (m >= 32) ? 0u : (0xFFFFFFFFu >> m);
#pragma unroll
for (int i = 0; i < 32; ++i)
if (!(keep & (1u << i))) scores[s * 32 + i] = -INFINITY;
}
}
template <int CTA_GROUP>
__device__ __forceinline__ void tcgen05_alloc(uint32_t smem_dst_ptr,
uint32_t n_cols) {
static_assert(CTA_GROUP == 1 || CTA_GROUP == 2,
"tcgen05_alloc: CTA_GROUP must be 1 or 2");
if constexpr (CTA_GROUP == 1) {
asm volatile(
"tcgen05.alloc.cta_group::1.sync.aligned.shared::cta.b32 [%0], %1;\n"
:: "r"(smem_dst_ptr), "r"(n_cols));
} else {
asm volatile(
"tcgen05.alloc.cta_group::2.sync.aligned.shared::cta.b32 [%0], %1;\n"
:: "r"(smem_dst_ptr), "r"(n_cols));
}
}
__device__ __forceinline__ void tcgen05_st_32x32b_x16(
uint32_t tmem_addr, const uint32_t (&r)[16]) {
asm volatile("tcgen05.st.sync.aligned.32x32b.x16.b32 "
"[%0], {%1,%2,%3,%4,%5,%6,%7,%8,%9,%10,"
"%11,%12,%13,%14,%15,%16};\n"
:: "r"(tmem_addr),
"r"(r[0]),"r"(r[1]),"r"(r[2]),"r"(r[3]),
"r"(r[4]),"r"(r[5]),"r"(r[6]),"r"(r[7]),
"r"(r[8]),"r"(r[9]),"r"(r[10]),"r"(r[11]),
"r"(r[12]),"r"(r[13]),"r"(r[14]),"r"(r[15]));
}
__device__ __forceinline__ void tcgen05_st_32x32b_x32(
uint32_t tmem_addr, const uint32_t (&r)[32]) {
asm volatile("tcgen05.st.sync.aligned.32x32b.x32.b32 "
"[%0], {%1,%2,%3,%4,%5,%6,%7,%8,%9,%10,"
"%11,%12,%13,%14,%15,%16,%17,%18,%19,%20,"
"%21,%22,%23,%24,%25,%26,%27,%28,%29,%30,"
"%31,%32};\n"
:: "r"(tmem_addr),
"r"(r[0]),"r"(r[1]),"r"(r[2]),"r"(r[3]),
"r"(r[4]),"r"(r[5]),"r"(r[6]),"r"(r[7]),
"r"(r[8]),"r"(r[9]),"r"(r[10]),"r"(r[11]),
"r"(r[12]),"r"(r[13]),"r"(r[14]),"r"(r[15]),
"r"(r[16]),"r"(r[17]),"r"(r[18]),"r"(r[19]),
"r"(r[20]),"r"(r[21]),"r"(r[22]),"r"(r[23]),
"r"(r[24]),"r"(r[25]),"r"(r[26]),"r"(r[27]),
"r"(r[28]),"r"(r[29]),"r"(r[30]),"r"(r[31]));
}
__device__ __forceinline__ void tcgen05_commit1_lead(uint32_t lead, uint32_t mbar_smem_addr) {
asm volatile(
"{\n\t"
".reg .pred q;\n\t"
"setp.ne.b32 q, %0, 0;\n\t"
"@q tcgen05.commit.cta_group::1.mbarrier::arrive::one.b64 [%1];\n\t"
"}\n"
:: "r"(lead), "r"(mbar_smem_addr));
}
__device__ __forceinline__ void tcgen05_wait_st() {
asm volatile("tcgen05.wait::st.sync.aligned;\n" ::: "memory");
}
__device__ __forceinline__ void tcgen05_fence_before_thread_sync() {
asm volatile("tcgen05.fence::before_thread_sync;\n" ::: "memory");
}
__device__ __forceinline__
void tma_load_2d(uint32_t smem_dst, const void* tensormap_ptr,
uint32_t mbar_smem, int coord_x, int coord_y) {
asm volatile(
"cp.async.bulk.tensor.2d.shared::cluster.global.tile"
".mbarrier::complete_tx::bytes"
" [%0], [%1, {%3, %4}], [%2];\n"
:: "r"(smem_dst), "l"(tensormap_ptr),
"r"(mbar_smem), "r"(coord_x), "r"(coord_y)
: "memory");
}
__device__ __forceinline__
void tma_load_3d(uint32_t smem_dst, const void* tensormap_ptr,
uint32_t mbar_smem, int c0, int c1, int c2) {
asm volatile(
"cp.async.bulk.tensor.3d.shared::cluster.global.tile"
".mbarrier::complete_tx::bytes"
" [%0], [%1, {%3, %4, %5}], [%2];\n"
:: "r"(smem_dst), "l"(tensormap_ptr),
"r"(mbar_smem), "r"(c0), "r"(c1), "r"(c2)
: "memory");
}
__device__ __forceinline__
void tma_load_4d(uint32_t smem_dst, const void* tensormap_ptr,
uint32_t mbar_smem, int c0, int c1, int c2, int c3) {
asm volatile(
"cp.async.bulk.tensor.4d.shared::cluster.global.tile"
".mbarrier::complete_tx::bytes"
" [%0], [%1, {%3, %4, %5, %6}], [%2];\n"
:: "r"(smem_dst), "l"(tensormap_ptr),
"r"(mbar_smem), "r"(c0), "r"(c1), "r"(c2), "r"(c3)
: "memory");
}
template <int CTA_GROUP>
__device__ __forceinline__ void tcgen05_dealloc(uint32_t tmem_addr,
uint32_t n_cols) {
static_assert(CTA_GROUP == 1 || CTA_GROUP == 2,
"tcgen05_dealloc: CTA_GROUP must be 1 or 2");
if constexpr (CTA_GROUP == 1) {
asm volatile(
"tcgen05.dealloc.cta_group::1.sync.aligned.b32 %0, %1;\n"
:: "r"(tmem_addr), "r"(n_cols));
} else {
asm volatile(
"tcgen05.dealloc.cta_group::2.sync.aligned.b32 %0, %1;\n"
:: "r"(tmem_addr), "r"(n_cols));
}
}
__device__ __forceinline__
void tma_store_2d(const void* tensormap_ptr, int coord_x, int coord_y,
uint32_t smem_src) {
asm volatile(
"cp.async.bulk.tensor.2d.global.shared::cta.tile.bulk_group"
" [%0, {%1, %2}], [%3];\n"
:: "l"(tensormap_ptr), "r"(coord_x), "r"(coord_y),
"r"(smem_src)
: "memory");
}
__device__ __forceinline__
void tma_store_3d(const void* tensormap_ptr, int c0, int c1, int c2,
uint32_t smem_src) {
asm volatile(
"cp.async.bulk.tensor.3d.global.shared::cta.tile.bulk_group"
" [%0, {%1, %2, %3}], [%4];\n"
:: "l"(tensormap_ptr), "r"(c0), "r"(c1), "r"(c2), "r"(smem_src)
: "memory");
}
__device__ __forceinline__
void tma_store_4d(const void* tensormap_ptr, int c0, int c1, int c2, int c3,
uint32_t smem_src) {
asm volatile(
"cp.async.bulk.tensor.4d.global.shared::cta.tile.bulk_group"
" [%0, {%1, %2, %3, %4}], [%5];\n"
:: "l"(tensormap_ptr), "r"(c0), "r"(c1), "r"(c2), "r"(c3), "r"(smem_src)
: "memory");
}
inline cudaError_t make_tma_2d_tiled(
CUtensorMap* out,
const void* ptr, int rows, int cols, int box_rows, int box_cols,
int elem_bytes, CUtensorMapDataType dtype,
CUtensorMapSwizzle swizzle = CU_TENSOR_MAP_SWIZZLE_128B,
CUtensorMapL2promotion l2 = CU_TENSOR_MAP_L2_PROMOTION_L2_128B,
CUtensorMapFloatOOBfill oob = CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE) {
uint64_t globalDim[2] = { (uint64_t)cols, (uint64_t)rows };
uint64_t globalStrides[1] = { (uint64_t)cols * (uint64_t)elem_bytes };
uint32_t boxDim[2] = { (uint32_t)box_cols, (uint32_t)box_rows };
uint32_t elemStrides[2] = { 1u, 1u };
CUresult r = cuTensorMapEncodeTiled(
out, dtype, 2,
const_cast<void*>(ptr), globalDim, globalStrides,
boxDim, elemStrides,
CU_TENSOR_MAP_INTERLEAVE_NONE,
swizzle, l2, oob);
return (r == CUDA_SUCCESS) ? cudaSuccess : cudaErrorInvalidValue;
}
__device__ __forceinline__
void cp_async_bulk_commit_group() {
asm volatile("cp.async.bulk.commit_group;\n" ::: "memory");
}
template <int N>
__device__ __forceinline__
void cp_async_bulk_wait_group_read() {
asm volatile("cp.async.bulk.wait_group.read %0;\n" :: "n"(N) : "memory");
}
__device__ __forceinline__
void mbarrier_init(uint32_t mbar_smem, uint32_t arrive_count) {
asm volatile("mbarrier.init.shared::cta.b64 [%0], %1;\n"
:: "r"(mbar_smem), "r"(arrive_count) : "memory");
}
template <int CTA_GROUP>
__device__ __forceinline__ void tcgen05_relinquish_alloc_permit() {
static_assert(CTA_GROUP == 1 || CTA_GROUP == 2,
"tcgen05_relinquish_alloc_permit: CTA_GROUP must be 1 or 2");
if constexpr (CTA_GROUP == 1) {
asm volatile("tcgen05.relinquish_alloc_permit.cta_group::1.sync.aligned;\n" ::);
} else {
asm volatile("tcgen05.relinquish_alloc_permit.cta_group::2.sync.aligned;\n" ::);
}
}
__device__ __forceinline__
void fence_mbarrier_init_release_cluster() {
asm volatile("fence.mbarrier_init.release.cluster;\n" ::: "memory");
}
__device__ __forceinline__ void tcgen05_mma_f16_ss_lead(uint32_t lead,
uint32_t tmem_c, uint64_t desc_a, uint64_t desc_b, uint32_t idesc,
bool enable_input_d) {
asm volatile(
"{\n\t"
".reg .pred p, q;\n\t"
"setp.ne.b32 q, %0, 0;\n\t"
"setp.ne.b32 p, %5, 0;\n\t"
"@q tcgen05.mma.cta_group::1.kind::f16 [%1], %2, %3, %4, {%6, %7, %8, %9}, p;\n\t"
"}\n"
:: "r"(lead), "r"(tmem_c), "l"(desc_a), "l"(desc_b), "r"(idesc),
"r"(enable_input_d ? 1u : 0u), "r"(0u), "r"(0u), "r"(0u), "r"(0u));
}
__device__ __forceinline__ void tcgen05_mma_f16_ts_1sm_lead(uint32_t lead,
uint32_t tmem_c, uint32_t tmem_a, uint64_t desc_b, uint32_t idesc,
bool enable_input_d) {
asm volatile(
"{\n\t"
".reg .pred p, q;\n\t"
"setp.ne.b32 q, %0, 0;\n\t"
"setp.ne.b32 p, %5, 0;\n\t"
"@q tcgen05.mma.cta_group::1.kind::f16 [%1], [%2], %3, %4, {%6, %7, %8, %9}, p;\n\t"
"}\n"
:: "r"(lead), "r"(tmem_c), "r"(tmem_a), "l"(desc_b), "r"(idesc),
"r"(enable_input_d ? 1u : 0u), "r"(0u), "r"(0u), "r"(0u), "r"(0u));
}
enum class SmemSwizzleBlackwell : uint32_t {
None = 0,
B128_32atom = 1,
B128 = 2,
B64 = 4,
B32 = 6,
};
__device__ __host__ __forceinline__ uint64_t build_smem_desc_blackwell(
uint32_t smem_addr,
uint32_t stride_byte_offset,
uint32_t leading_byte_offset,
SmemSwizzleBlackwell swizzle = SmemSwizzleBlackwell::B128,
uint32_t base_offset = 0) {
uint64_t d = 0;
d |= static_cast<uint64_t>((smem_addr >> 4) & 0x3FFF);
d |= static_cast<uint64_t>((leading_byte_offset >> 4) & 0x3FFF) << 16;
d |= static_cast<uint64_t>((stride_byte_offset >> 4) & 0x3FFF) << 32;
d |= static_cast<uint64_t>(1) << 46;
d |= (static_cast<uint64_t>(base_offset) & 0x7) << 49;
d |= static_cast<uint64_t>(static_cast<uint32_t>(swizzle) & 0x7) << 61;
return d;
}
__device__ __forceinline__
uint32_t elect_one_sync() {
uint32_t elected;
asm volatile(
"{\n\t"
".reg .pred p;\n\t"
"elect.sync %0|p, 0xffffffff;\n\t"
"selp.b32 %0, 1, 0, p;\n\t"
"}\n"
: "=r"(elected));
return elected;
}
__device__ __forceinline__
uint32_t elect_one_sync(uint32_t membermask) {
uint32_t elected;
asm volatile(
"{\n\t"
".reg .pred p;\n\t"
"elect.sync %0|p, %1;\n\t"
"selp.b32 %0, 1, 0, p;\n\t"
"}\n"
: "=r"(elected) : "r"(membermask));
return elected;
}
template <int N>
__device__ __forceinline__
void setmaxnreg_dec() {
static_assert(N >= 24 && N <= 256 && N % 8 == 0,
"setmaxnreg_dec: N must be in [24, 256] and a multiple of 8");
asm volatile("setmaxnreg.dec.sync.aligned.u32 %0;\n"
:: "n"(N) : "memory");
}
template <int N>
__device__ __forceinline__
void setmaxnreg_inc() {
static_assert(N >= 24 && N <= 256 && N % 8 == 0,
"setmaxnreg_inc: N must be in [24, 256] and a multiple of 8");
asm volatile("setmaxnreg.inc.sync.aligned.u32 %0;\n"
:: "n"(N) : "memory");
}
__device__ __forceinline__
uint32_t cvt_f32x2_to_bf16x2(float a, float b) {
uint32_t r;
asm volatile("cvt.rn.bf16x2.f32 %0, %2, %1;\n"
: "=r"(r) : "f"(a), "f"(b));
return r;
}
namespace {
__device__ __forceinline__ uint64_t f32x2_bits(float2 v) {
uint64_t b; __builtin_memcpy(&b, &v, 8); return b;
}
__device__ __forceinline__ float2 f32x2_make(uint64_t b) {
float2 v; __builtin_memcpy(&v, &b, 8); return v;
}
}
__device__ __forceinline__ float2 fmul2(float2 a, float2 b) {
uint64_t d;
asm volatile("mul.f32x2 %0, %1, %2;\n" : "=l"(d) : "l"(f32x2_bits(a)), "l"(f32x2_bits(b)));
return f32x2_make(d);
}
__device__ __forceinline__ float2 fadd2(float2 a, float2 b) {
uint64_t d;
asm volatile("add.f32x2 %0, %1, %2;\n" : "=l"(d) : "l"(f32x2_bits(a)), "l"(f32x2_bits(b)));
return f32x2_make(d);
}
__device__ __forceinline__ float2 ffma2(float2 a, float2 b, float2 c) {
uint64_t d;
asm volatile("fma.rn.f32x2 %0, %1, %2, %3;\n"
: "=l"(d) : "l"(f32x2_bits(a)), "l"(f32x2_bits(b)), "l"(f32x2_bits(c)));
return f32x2_make(d);
}
__device__ __forceinline__ float2 f32x2_splat(float s) { return make_float2(s, s); }
__device__ __forceinline__ float ex2_approx_f32(float z) {
float d;
asm volatile("ex2.approx.ftz.f32 %0, %1;\n" : "=f"(d) : "f"(z));
return d;
}
__device__ __forceinline__ float2 ex2_emu_f32x2(float x, float y) {
uint32_t ox, oy;
asm volatile(
"{\n\t"
".reg .f32 f1,f2,f3,f4,f5,f6,f7;\n\t"
".reg .b64 l1,l2,l3,l4,l5,l6,l7,l8,l9,l10;\n\t"
".reg .s32 r1,r2,r3,r4,r5,r6,r7,r8;\n\t"
"max.f32 f1, %2, 0fC2FE0000;\n\t"
"max.f32 f2, %3, 0fC2FE0000;\n\t"
"mov.b64 l1, {f1, f2};\n\t"
"mov.f32 f3, 0f4B400000;\n\t"
"mov.b64 l2, {f3, f3};\n\t"
"add.rm.f32x2 l7, l1, l2;\n\t"
"sub.rn.f32x2 l8, l7, l2;\n\t"
"sub.rn.f32x2 l9, l1, l8;\n\t"
"mov.f32 f7, 0f3D9DF09D;\n\t"
"mov.b64 l6, {f7, f7};\n\t"
"mov.f32 f6, 0f3E6906A4;\n\t"
"mov.b64 l5, {f6, f6};\n\t"
"mov.f32 f5, 0f3F31F519;\n\t"
"mov.b64 l4, {f5, f5};\n\t"
"mov.f32 f4, 0f3F800000;\n\t"
"mov.b64 l3, {f4, f4};\n\t"
"fma.rn.f32x2 l10, l9, l6, l5;\n\t"
"fma.rn.f32x2 l10, l10, l9, l4;\n\t"
"fma.rn.f32x2 l10, l10, l9, l3;\n\t"
"mov.b64 {r1, r2}, l7;\n\t"
"mov.b64 {r3, r4}, l10;\n\t"
"shl.b32 r5, r1, 23;\n\t"
"add.s32 r7, r5, r3;\n\t"
"shl.b32 r6, r2, 23;\n\t"
"add.s32 r8, r6, r4;\n\t"
"mov.b32 %0, r7;\n\t"
"mov.b32 %1, r8;\n\t"
"}\n"
: "=r"(ox), "=r"(oy) : "f"(x), "f"(y));
float2 r; __builtin_memcpy(&r.x, &ox, 4); __builtin_memcpy(&r.y, &oy, 4);
return r;
}
__device__ __forceinline__ float rcp_approx_ftz_f32(float x) {
float d;
asm("rcp.approx.ftz.f32 %0, %1;\n" : "=f"(d) : "f"(x));
return d;
}
__device__ __forceinline__ uint32_t make_idesc_table44(
int M, int N,
uint32_t dtype, uint32_t atype, uint32_t btype,
bool transpose_a = false, bool transpose_b = false,
bool negate_a = false, bool negate_b = false) {
uint32_t idesc = 0;
idesc |= (dtype & 0x3) << 4;
idesc |= (atype & 0x7) << 7;
idesc |= (btype & 0x7) << 10;
idesc |= (negate_a ? 1u : 0u) << 13;
idesc |= (negate_b ? 1u : 0u) << 14;
idesc |= (transpose_a ? 1u : 0u) << 15;
idesc |= (transpose_b ? 1u : 0u) << 16;
idesc |= ((static_cast<uint32_t>(N) >> 3) & 0x3F) << 17;
idesc |= ((static_cast<uint32_t>(M) >> 4) & 0x1F) << 24;
return idesc;
}
__device__ __forceinline__ uint32_t make_idesc_bf16_f32(
int M, int N, bool ta = false, bool tb = false) {
return make_idesc_table44(M, N, 1,
1, 1, ta, tb);
}
__device__ __forceinline__ void tcgen05_ld_32x32b_x16(
uint32_t tmem_addr, uint32_t (&r)[16]) {
asm volatile("tcgen05.ld.sync.aligned.32x32b.x16.b32 "
"{%0,%1,%2,%3,%4,%5,%6,%7,%8,%9,"
"%10,%11,%12,%13,%14,%15}, [%16];\n"
: "=r"(r[0]),"=r"(r[1]),"=r"(r[2]),"=r"(r[3]),
"=r"(r[4]),"=r"(r[5]),"=r"(r[6]),"=r"(r[7]),
"=r"(r[8]),"=r"(r[9]),"=r"(r[10]),"=r"(r[11]),
"=r"(r[12]),"=r"(r[13]),"=r"(r[14]),"=r"(r[15])
: "r"(tmem_addr));
}
__device__ __forceinline__ void tcgen05_ld_32x32b_x32(
uint32_t tmem_addr, uint32_t (&r)[32]) {
asm volatile("tcgen05.ld.sync.aligned.32x32b.x32.b32 "
"{%0,%1,%2,%3,%4,%5,%6,%7,%8,%9,"
"%10,%11,%12,%13,%14,%15,%16,%17,%18,%19,"
"%20,%21,%22,%23,%24,%25,%26,%27,%28,%29,"
"%30,%31}, [%32];\n"
: "=r"(r[0]),"=r"(r[1]),"=r"(r[2]),"=r"(r[3]),
"=r"(r[4]),"=r"(r[5]),"=r"(r[6]),"=r"(r[7]),
"=r"(r[8]),"=r"(r[9]),"=r"(r[10]),"=r"(r[11]),
"=r"(r[12]),"=r"(r[13]),"=r"(r[14]),"=r"(r[15]),
"=r"(r[16]),"=r"(r[17]),"=r"(r[18]),"=r"(r[19]),
"=r"(r[20]),"=r"(r[21]),"=r"(r[22]),"=r"(r[23]),
"=r"(r[24]),"=r"(r[25]),"=r"(r[26]),"=r"(r[27]),
"=r"(r[28]),"=r"(r[29]),"=r"(r[30]),"=r"(r[31])
: "r"(tmem_addr));
}
__device__ __forceinline__ void tcgen05_ld_32x32b_x64(
uint32_t tmem_addr, uint32_t (&r)[64]) {
asm volatile("tcgen05.ld.sync.aligned.32x32b.x64.b32 "
"{%0,%1,%2,%3,%4,%5,%6,%7,%8,%9,"
"%10,%11,%12,%13,%14,%15,%16,%17,%18,%19,"
"%20,%21,%22,%23,%24,%25,%26,%27,%28,%29,"
"%30,%31,%32,%33,%34,%35,%36,%37,%38,%39,"
"%40,%41,%42,%43,%44,%45,%46,%47,%48,%49,"
"%50,%51,%52,%53,%54,%55,%56,%57,%58,%59,"
"%60,%61,%62,%63}, [%64];\n"
: "=r"(r[0]),"=r"(r[1]),"=r"(r[2]),"=r"(r[3]),
"=r"(r[4]),"=r"(r[5]),"=r"(r[6]),"=r"(r[7]),
"=r"(r[8]),"=r"(r[9]),"=r"(r[10]),"=r"(r[11]),
"=r"(r[12]),"=r"(r[13]),"=r"(r[14]),"=r"(r[15]),
"=r"(r[16]),"=r"(r[17]),"=r"(r[18]),"=r"(r[19]),
"=r"(r[20]),"=r"(r[21]),"=r"(r[22]),"=r"(r[23]),
"=r"(r[24]),"=r"(r[25]),"=r"(r[26]),"=r"(r[27]),
"=r"(r[28]),"=r"(r[29]),"=r"(r[30]),"=r"(r[31]),
"=r"(r[32]),"=r"(r[33]),"=r"(r[34]),"=r"(r[35]),
"=r"(r[36]),"=r"(r[37]),"=r"(r[38]),"=r"(r[39]),
"=r"(r[40]),"=r"(r[41]),"=r"(r[42]),"=r"(r[43]),
"=r"(r[44]),"=r"(r[45]),"=r"(r[46]),"=r"(r[47]),
"=r"(r[48]),"=r"(r[49]),"=r"(r[50]),"=r"(r[51]),
"=r"(r[52]),"=r"(r[53]),"=r"(r[54]),"=r"(r[55]),
"=r"(r[56]),"=r"(r[57]),"=r"(r[58]),"=r"(r[59]),
"=r"(r[60]),"=r"(r[61]),"=r"(r[62]),"=r"(r[63])
: "r"(tmem_addr));
}
__device__ __forceinline__ void tcgen05_ld_32x32b_x128(
uint32_t tmem_addr, uint32_t (&r)[128]) {
asm volatile("tcgen05.ld.sync.aligned.32x32b.x128.b32 "
"{%0,%1,%2,%3,%4,%5,%6,%7,%8,%9,"
"%10,%11,%12,%13,%14,%15,%16,%17,%18,%19,"
"%20,%21,%22,%23,%24,%25,%26,%27,%28,%29,"
"%30,%31,%32,%33,%34,%35,%36,%37,%38,%39,"
"%40,%41,%42,%43,%44,%45,%46,%47,%48,%49,"
"%50,%51,%52,%53,%54,%55,%56,%57,%58,%59,"
"%60,%61,%62,%63,%64,%65,%66,%67,%68,%69,"
"%70,%71,%72,%73,%74,%75,%76,%77,%78,%79,"
"%80,%81,%82,%83,%84,%85,%86,%87,%88,%89,"
"%90,%91,%92,%93,%94,%95,%96,%97,%98,%99,"
"%100,%101,%102,%103,%104,%105,%106,%107,%108,%109,"
"%110,%111,%112,%113,%114,%115,%116,%117,%118,%119,"
"%120,%121,%122,%123,%124,%125,%126,%127}, [%128];\n"
: "=r"(r[ 0]),"=r"(r[ 1]),"=r"(r[ 2]),"=r"(r[ 3]),
"=r"(r[ 4]),"=r"(r[ 5]),"=r"(r[ 6]),"=r"(r[ 7]),
"=r"(r[ 8]),"=r"(r[ 9]),"=r"(r[ 10]),"=r"(r[ 11]),
"=r"(r[ 12]),"=r"(r[ 13]),"=r"(r[ 14]),"=r"(r[ 15]),
"=r"(r[ 16]),"=r"(r[ 17]),"=r"(r[ 18]),"=r"(r[ 19]),
"=r"(r[ 20]),"=r"(r[ 21]),"=r"(r[ 22]),"=r"(r[ 23]),
"=r"(r[ 24]),"=r"(r[ 25]),"=r"(r[ 26]),"=r"(r[ 27]),
"=r"(r[ 28]),"=r"(r[ 29]),"=r"(r[ 30]),"=r"(r[ 31]),
"=r"(r[ 32]),"=r"(r[ 33]),"=r"(r[ 34]),"=r"(r[ 35]),
"=r"(r[ 36]),"=r"(r[ 37]),"=r"(r[ 38]),"=r"(r[ 39]),
"=r"(r[ 40]),"=r"(r[ 41]),"=r"(r[ 42]),"=r"(r[ 43]),
"=r"(r[ 44]),"=r"(r[ 45]),"=r"(r[ 46]),"=r"(r[ 47]),
"=r"(r[ 48]),"=r"(r[ 49]),"=r"(r[ 50]),"=r"(r[ 51]),
"=r"(r[ 52]),"=r"(r[ 53]),"=r"(r[ 54]),"=r"(r[ 55]),
"=r"(r[ 56]),"=r"(r[ 57]),"=r"(r[ 58]),"=r"(r[ 59]),
"=r"(r[ 60]),"=r"(r[ 61]),"=r"(r[ 62]),"=r"(r[ 63]),
"=r"(r[ 64]),"=r"(r[ 65]),"=r"(r[ 66]),"=r"(r[ 67]),
"=r"(r[ 68]),"=r"(r[ 69]),"=r"(r[ 70]),"=r"(r[ 71]),
"=r"(r[ 72]),"=r"(r[ 73]),"=r"(r[ 74]),"=r"(r[ 75]),
"=r"(r[ 76]),"=r"(r[ 77]),"=r"(r[ 78]),"=r"(r[ 79]),
"=r"(r[ 80]),"=r"(r[ 81]),"=r"(r[ 82]),"=r"(r[ 83]),
"=r"(r[ 84]),"=r"(r[ 85]),"=r"(r[ 86]),"=r"(r[ 87]),
"=r"(r[ 88]),"=r"(r[ 89]),"=r"(r[ 90]),"=r"(r[ 91]),
"=r"(r[ 92]),"=r"(r[ 93]),"=r"(r[ 94]),"=r"(r[ 95]),
"=r"(r[ 96]),"=r"(r[ 97]),"=r"(r[ 98]),"=r"(r[ 99]),
"=r"(r[100]),"=r"(r[101]),"=r"(r[102]),"=r"(r[103]),
"=r"(r[104]),"=r"(r[105]),"=r"(r[106]),"=r"(r[107]),
"=r"(r[108]),"=r"(r[109]),"=r"(r[110]),"=r"(r[111]),
"=r"(r[112]),"=r"(r[113]),"=r"(r[114]),"=r"(r[115]),
"=r"(r[116]),"=r"(r[117]),"=r"(r[118]),"=r"(r[119]),
"=r"(r[120]),"=r"(r[121]),"=r"(r[122]),"=r"(r[123]),
"=r"(r[124]),"=r"(r[125]),"=r"(r[126]),"=r"(r[127])
: "r"(tmem_addr));
}
__device__ __forceinline__
uint32_t smem_ptr_u32(const void* ptr) {
return static_cast<uint32_t>(__cvta_generic_to_shared(ptr));
}
__device__ __forceinline__
void sts_f32(uint32_t smem_addr, float val) {
asm volatile("st.shared.f32 [%0], %1;" :: "r"(smem_addr), "f"(val) : "memory");
}
@@ -252,6 +252,13 @@ _SPECS: tuple[tuple[str, KernelBackend, str, frozenset, str], ...] = (
_CUDA,
"Fused in-place QK RMS-norm + RoPE.",
),
(
"diffusion.vsa_block_sparse_sm100",
KernelBackend.JIT,
"attention.vsa_block_sparse_sm100_jit:vsa_block_sparse_sm100",
frozenset({CapabilityRequirement.cuda(min_sm=(10, 0), max_sm=(10, 3))}),
"FastVideo's warp-specialized tcgen05 block-sparse VSA forward (64-token tiles).",
),
(
"diffusion.h3_vae_rmsnorm",
KernelBackend.TRITON,
@@ -715,6 +722,8 @@ _EXPORTS: dict[str, str] = {
"prepare_rope_tables": "attention.sana_wm_gdn_triton",
"_attn_fwd": "attention.sparse_linear_attn_triton",
"get_block_map": "attention.sparse_linear_attn_triton",
"can_use_vsa_block_sparse_sm100": "attention.vsa_block_sparse_sm100_jit",
"vsa_block_sparse_sm100": "attention.vsa_block_sparse_sm100_jit",
# MoE routing
"can_use_group_limited_topk": "routing.group_limited_topk_triton",
"group_limited_topk": "routing.group_limited_topk_triton",
@@ -0,0 +1,67 @@
"""FastVideo's native Blackwell block-sparse VSA forward (64-token tiles).
A CTA owns two adjacent query tiles, so the tile count must be even. Requires
sm_100a or sm_103a, bf16, head_dim 128 and contiguous [B, H, S, D].
"""
from __future__ import annotations
import logging
from typing import TYPE_CHECKING
import torch
from sglang.kernels.jit.utils import cache_once, load_jit
from sglang.srt.utils.custom_op import register_custom_op
if TYPE_CHECKING:
from tvm_ffi.module import Module
logger = logging.getLogger(__name__)
_SUPPORTED_CAPABILITIES = ((10, 0), (10, 3))
_HEAD_DIM = 128
@cache_once
def _module() -> Module:
return load_jit(
"vsa_block_sparse_sm100",
cuda_files=["diffusion/vsa_block_sparse_sm100.cuh"],
cuda_wrappers=[("vsa_block_sparse_sm100", "VsaBlockSparseSm100Kernel::run")],
extra_ldflags=["-lcuda"],
)
@cache_once
def can_use_vsa_block_sparse_sm100(
device_index: int, dtype: torch.dtype, head_dim: int
) -> bool:
if dtype != torch.bfloat16 or head_dim != _HEAD_DIM:
return False
if torch.cuda.get_device_capability(device_index) not in _SUPPORTED_CAPABILITIES:
return False
try:
_module()
except Exception as e:
logger.warning("Failed to load the JIT VSA block-sparse sm100 kernel: %s", e)
return False
return True
@register_custom_op(mutates_args=["out"])
def vsa_block_sparse_sm100(
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
q2k_idx: torch.Tensor,
q2k_num: torch.Tensor,
block_sizes: torch.Tensor,
out: torch.Tensor,
sm_scale: float,
) -> None:
"""``out[b, h, i*64:(i+1)*64]`` = attention of query tile ``i`` over its
``q2k_num`` listed key tiles, keys past each tile's ``block_sizes`` masked."""
_module().vsa_block_sparse_sm100(
q, k, v, q2k_idx, q2k_num, block_sizes, out, sm_scale
)
@@ -314,7 +314,7 @@ Use the preset categories this way:
| `cosmos3-super-t2v-cfg2tp2` | `nvidia/Cosmos3-Super` | No | Explicit four-GPU TP2 x CFG2 throughput comparator. On H200 it was 48.00% faster end to end than TP2, but the topology changed the deterministic output (SSIM 0.914244, PSNR 29.469771 dB), so do not treat it as lossless-equivalent or select it automatically. |
| `wan-i2v` | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | Yes: `wan22_i2v_a14b_720p` | Nightly cat image and motion prompt, 1280x720, 81 frames, 4 GPUs, CFG parallel, Ulysses degree 2, text encoder CPU offload and pinned CPU memory |
| `minimax-h3-t2va` | `MiniMaxAI/MiniMax-H3` | Yes: `minimax_h3_t2va_5s` | H3 FL2VA-partition T2VA baseline: 1344x768 resolved canvas, 5 seconds / 124 frames at 24 fps, 50 joint video-audio steps, 4 GPUs, TP2 + Ulysses2, eager BF16/FP32. The helper writes H3's request contract to a generated config. |
| `fasth3-t2va-vsa` | `FastVideo/FastVideo-FastH3-4-step-Preview-v1-VSA-DataFree` | No | FastH3 4-step distilled T2VA on the trained VSA-H3 backend: 1344x768, 10 seconds / 243 frames, five sigma points = four DiT forwards, 4 GPUs, Ulysses 4, eager, 2-step warmup request. Compare against `--attention-backend fa` on the same weights for the dense-fallback gap. |
| `fasth3-t2va-vsa` | `FastVideo/FastVideo-FastH3-8-Step-V2` | No | FastH3 8-step distilled T2VA on the trained VSA-H3 backend (sparsity 0.8): 1344x768, 10 seconds / 243 frames, nine sigma points = eight DiT forwards, 4 GPUs, Ulysses 4, eager, 2-step warmup request. Compare against `--attention-backend fa` on the same weights for the dense-fallback gap. |
| `longcat-image` | `meituan-longcat/LongCat-Image` | No | Eager DiT baseline at 1024x1024, 50 steps, guidance 4.5; prompt rewrite is disabled so Qwen2.5-VL does not contaminate the DiT A/B. |
| `longcat-image-edit` | `meituan-longcat/LongCat-Image-Edit` | No | Native edit baseline using the public SGLang edit fixture. Its 1536x1024 source resolves to 1264x848 under the checkpoint's roughly-one-megapixel aspect-ratio rule, and the BCG comparator captures that exact serving canvas; prompt rewrite is disabled to isolate the DiT. |
| `longcat-image-edit-turbo` | `meituan-longcat/LongCat-Image-Edit-Turbo` | No | Matching distilled edit baseline using the same public fixture, prompt, and 1264x848 BCG canvas. Its registered sampling class owns the eight-step, guidance-1 schedule. |
@@ -427,7 +427,7 @@ MODELS = {
},
# H3 rejects a 1-step warmup request, hence --warmup-steps=2.
"fasth3-t2va-vsa": {
"path": "FastVideo/FastVideo-FastH3-4-step-Preview-v1-VSA-DataFree",
"path": "FastVideo/FastVideo-FastH3-8-Step-V2",
"prompt": (
"A curious raccoon peers through a vibrant field of yellow "
"sunflowers, its eyes wide with interest."
@@ -441,12 +441,11 @@ MODELS = {
"aspect_ratio": "16:9",
"duration_seconds": 10.0,
},
"num_inference_steps": 5,
"num_inference_steps": 9,
},
"extra_args": [
"--num-gpus=4",
"--attention-backend=video_sparse_attn_h3",
'--attention-backend-config={"VSA_sparsity": 0.9}',
"--component-attention-backends=transformer=video_sparse_attn_h3",
"--enable-torch-compile=false",
"--warmup-steps=2",
],
+3
View File
@@ -37,6 +37,9 @@ SGLang Diffusion supports Apple Silicon (M-series) via the MPS backend. Since Tr
Docker is recommended for Linux GPU deployments. Follow the [installation guide](https://docs.sglang.io/docs/sglang-diffusion/installation) to start a container with diffusion dependencies included, or choose pip/uv, source installation, or another platform's setup. Run the examples below inside that environment.
Video output requires a system `ffmpeg` with the `libx264` encoder on `PATH`. For Ubuntu source or pip installations, install it with `sudo apt-get install ffmpeg`; on macOS, use `brew install ffmpeg`.
## Inference
Here's a minimal example to generate a video using the default settings:
@@ -33,6 +33,8 @@ class MiniMaxH3VideoVAEConfig(VAEConfig):
load_decoder: bool = True
use_tiling: bool = True
use_parallel_tiling: bool = True
# Decode each rank's spatial tiles in one batched call instead of one by one.
stack_tiling: bool = False
# The released checkpoint's quality contract uses overlapping latent
# tiles. Parallel tiling distributes whole tiles without changing that
# recipe. Spatial-shard decode is rejected because validation found output
@@ -57,6 +57,8 @@ class MiniMaxH3PipelineConfig(PipelineConfig):
vae_precision: str = "fp32"
vae_decode_precision: str = "fp16"
audio_vae_precision: str = "fp32"
# Default VSA-H3 sparsity; --attention-backend-config VSA_sparsity overrides.
vsa_sparsity: float = 0.9
text_encoder_configs: tuple[MiniMaxH3Qwen3VLConfig, ...] = field(
default_factory=lambda: (MiniMaxH3Qwen3VLConfig(),)
)
@@ -333,16 +335,20 @@ class MiniMaxH3PipelineConfig(PipelineConfig):
@dataclass
class FastH3PipelineConfig(MiniMaxH3PipelineConfig):
"""FastH3: 4-step VSA-distilled MiniMax-H3, t2va only."""
"""FastH3 8-Step V2: VSA-distilled MiniMax-H3, t2va only."""
# The checkpoint's trained sparsity (fastvideo_inference.json).
vsa_sparsity: float = 0.8
def __post_init__(self) -> None:
self.dit_config.arch_config.has_gate_compress = True
self.vae_config.stack_tiling = True
def validate_quality_deployment(self, server_args) -> None:
raise ValueError(
'quality="high" is audited only for the base MiniMax-H3 50-step '
"4xH200 deployment; the FastH3 4-step distilled checkpoint has no "
'audited high-quality deployment. Use quality="exact".'
"4xH200 deployment; FastH3 has no audited high-quality deployment. "
'Use quality="exact", or the default "lossless".'
)
def validate_server_args(self, server_args) -> None:
@@ -383,7 +389,7 @@ def register():
sampling_param_cls=FastH3SamplingParams,
pipeline_config_cls=FastH3PipelineConfig,
hf_model_paths=[
"FastVideo/FastVideo-FastH3-4-step-Preview-v1-VSA-DataFree",
"FastVideo/FastVideo-FastH3-8-Step-V2",
],
model_detectors=[
lambda model_id: (
@@ -357,15 +357,16 @@ class MiniMaxH3SamplingParams(SamplingParams):
@dataclass
class FastH3SamplingParams(MiniMaxH3SamplingParams):
"""FastH3: five sigma grid points, i.e. the four distilled DiT forwards."""
"""FastH3 8-Step V2: nine sigma points, i.e. eight DiT forwards on the trained rungs."""
num_inference_steps: int = 5
num_inference_steps: int = 9
quality: str = "lossless"
def _validate(self) -> None:
super()._validate()
if self.num_inference_steps != 5:
if self.num_inference_steps != 9:
raise ValueError(
"FastH3 is distilled for exactly five sigma grid points (four DiT "
"FastH3 is distilled for exactly nine sigma grid points (eight DiT "
f"forwards); got num_inference_steps={self.num_inference_steps}. "
"Use MiniMaxAI/MiniMax-H3 for other schedules."
)
+1 -1
View File
@@ -176,7 +176,7 @@ KNOWN_NON_DIFFUSERS_DIFFUSION_MODEL_PATTERNS: Dict[str, str] = {
"ming-image-0.1-design": "MingImagePipeline",
"minimaxai/minimax-h3": "MiniMaxH3Pipeline",
"minimax/minimax-h3": "MiniMaxH3Pipeline",
"fastvideo/fastvideo-fasth3-4-step-preview-v1-vsa-datafree": "FastH3Pipeline",
"fastvideo/fastvideo-fasth3-8-step-v2": "FastH3Pipeline",
"openvdn/vdn-minimax-h3": "VDNH3Pipeline",
"lerobot/pi05": "Pi05Pipeline",
"pi05": "Pi05Pipeline",
@@ -13,7 +13,6 @@ import json
import mmap
import os
import queue
import shutil
import subprocess
import tempfile
import threading
@@ -23,6 +22,7 @@ from dataclasses import dataclass, field
from typing import Any, Callable, List, Optional, Sequence, Union
import imageio
import imageio_ffmpeg
import numpy as np
import torch
from PIL import Image
@@ -32,11 +32,6 @@ try:
except ImportError: # pragma: no cover
scipy_wavfile = None
try:
import imageio_ffmpeg as _imageio_ffmpeg
except ImportError: # pragma: no cover
_imageio_ffmpeg = None
from sglang.multimodal_gen.configs.sample.sampling_params import (
DataType,
SamplingParams,
@@ -300,25 +295,8 @@ def _pick_audio_sample_rate(
def _resolve_ffmpeg_exe() -> str:
ffmpeg_exe = "ffmpeg"
ffmpeg_on_path = shutil.which("ffmpeg")
if ffmpeg_on_path:
ffmpeg_exe = ffmpeg_on_path
try:
if _imageio_ffmpeg is not None:
ffmpeg_exe = _imageio_ffmpeg.get_ffmpeg_exe()
except Exception:
pass
ffmpeg_ok = False
if ffmpeg_exe:
if os.path.isabs(ffmpeg_exe):
ffmpeg_ok = os.path.exists(ffmpeg_exe)
else:
ffmpeg_ok = shutil.which(ffmpeg_exe) is not None
if not ffmpeg_ok:
raise RuntimeError("ffmpeg not found")
return ffmpeg_exe
# keep the pinned encoder binary: x264 upgrades change lossy video pixels
return imageio_ffmpeg.get_ffmpeg_exe()
# ffmpeg's implicit libx264 default is `medium`. On diffusion output `fast` is
@@ -716,7 +694,7 @@ def _try_save_cuda_video_direct(
if encoder is not None:
encoder.abort()
logger.warning_once(
"Direct CUDA video save failed; falling back to imageio. "
"Direct CUDA video save failed; falling back to CPU ffmpeg encoding. "
"Enable debug logging for exception details."
)
logger.debug("Direct CUDA video save failure", exc_info=True)
@@ -894,6 +872,83 @@ def _maybe_mux_audio_into_mp4(
)
def _save_video_ffmpeg(
save_file_path: str,
frames: list,
*,
fps: int,
quality: float,
x264_preset: Optional[str] = None,
audio_path: Optional[str] = None,
) -> None:
if not frames:
raise ValueError("video output requires at least one frame")
if not 1 <= quality <= 10:
raise ValueError("video quality must be between 1 and 10")
first_frame = np.asarray(frames[0])
height, width = first_frame.shape[:2]
channels = first_frame.shape[2] if first_frame.ndim == 3 else 1
pixel_format = {1: "gray", 3: "rgb24", 4: "rgba"}[channels]
command = [
_resolve_ffmpeg_exe(),
"-y",
"-f",
"rawvideo",
"-vcodec",
"rawvideo",
"-s",
f"{width}x{height}",
"-pix_fmt",
pixel_format,
"-r",
f"{fps:.02f}",
"-i",
"pipe:0",
]
if audio_path is None:
command += ["-an"]
else:
command += ["-i", audio_path, "-acodec", "aac"]
command += [
"-vcodec",
"libx264",
"-preset",
x264_preset or X264_PRESET,
"-pix_fmt",
"yuv420p",
"-crf",
str(int((1 - quality / 10.0) * 51)),
]
if width % 16 or height % 16:
command += [
"-vf",
f"scale={(width + 15) // 16 * 16}:{(height + 15) // 16 * 16}",
]
command += [save_file_path]
with tempfile.TemporaryFile() as stderr:
with subprocess.Popen(
command,
stdin=subprocess.PIPE,
stdout=subprocess.DEVNULL,
stderr=stderr,
) as process:
try:
for frame in frames:
frame = np.asarray(frame)
if frame.shape != first_frame.shape or frame.dtype != np.uint8:
raise ValueError(
"video frames must have matching shapes and uint8 dtype"
)
process.stdin.write(memoryview(np.ascontiguousarray(frame)))
finally:
process.stdin.close()
if process.wait() != 0:
stderr.seek(0)
raise RuntimeError(
f"ffmpeg video encoding failed: {stderr.read().decode(errors='replace')}"
)
def _try_save_video_with_audio(
*,
save_file_path: str,
@@ -901,7 +956,6 @@ def _try_save_video_with_audio(
fps: int,
audio: Any,
audio_sample_rate: Optional[int],
output_format: str,
quality: float,
x264_preset: Optional[str] = None,
) -> bool:
@@ -925,16 +979,13 @@ def _try_save_video_with_audio(
with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as f:
tmp_wav_path = f.name
scipy_wavfile.write(tmp_wav_path, selected_sr, audio_np)
imageio.mimsave(
_save_video_ffmpeg(
save_file_path,
frames,
fps=fps,
format=output_format,
codec="libx264",
quality=quality,
audio_path=tmp_wav_path,
audio_codec="aac",
output_params=["-preset", x264_preset or X264_PRESET],
x264_preset=x264_preset,
)
return True
except Exception as e:
@@ -1151,26 +1202,22 @@ def save_materialized_output(
os.makedirs(os.path.dirname(save_file_path), exist_ok=True)
if data_type == DataType.VIDEO:
quality = output_compression / 10 if output_compression is not None else 5
output_format = data_type.get_default_extension()
saved_with_audio = _try_save_video_with_audio(
save_file_path=save_file_path,
frames=materialized.frames,
fps=materialized.fps,
audio=materialized.audio,
audio_sample_rate=audio_sample_rate,
output_format=output_format,
quality=quality,
x264_preset=x264_preset,
)
if not saved_with_audio:
imageio.mimsave(
_save_video_ffmpeg(
save_file_path,
materialized.frames,
fps=materialized.fps,
format=output_format,
codec="libx264",
quality=quality,
output_params=["-preset", x264_preset or X264_PRESET],
x264_preset=x264_preset,
)
_maybe_mux_audio_into_mp4(
@@ -29,6 +29,7 @@ from typing import Any
import torch
from sglang.kernels.ops.diffusion import can_use_vsa_block_sparse_sm100
from sglang.multimodal_gen.runtime.layers.attention.backends.attention_backend import (
AttentionBackend,
AttentionImpl,
@@ -42,7 +43,9 @@ from sglang.multimodal_gen.runtime.layers.attention.backends.video_sparse_attn i
)
from sglang.multimodal_gen.runtime.layers.attention.backends.vsa_h3_kernels import (
vsa_h3_block_sparse_attn_forward,
vsa_h3_gate_add,
vsa_h3_pack_tiles,
vsa_h3_topk_lists,
vsa_h3_untile,
)
from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum
@@ -238,6 +241,7 @@ def _topk_tile_lists(
prefix = num_prefix_tiles
keep = _compute_topk(sparsity, num_video_tiles)
video_rows = scores[:, prefix:, :]
# serving takes vsa_h3_topk_lists for this branch; it stays as its reference
if exempt or prefix == 0:
picked = video_rows[:, :, prefix:].topk(keep, dim=-1).indices + prefix
picked = picked.sort(dim=-1).values
@@ -277,6 +281,8 @@ class _Workspace:
"""Per-geometry scratch: tiled q/k/v(/gate) [3|4, H, S_pad, D], pooled fp32
tile means [3, H, n_tiles, D], and the kernel index lists (prefix rows and
prefix columns are static; only the top-k video columns change per layer).
The native kernel pairs query tiles, so the tile axis is padded to an even
count with one empty tile.
"""
def __init__(
@@ -289,9 +295,14 @@ class _Workspace:
device: torch.device,
) -> None:
n_tiles = meta.num_tiles
seq_pad = n_tiles * VSA_H3_TILE_ELEMS
self.n_tiles = n_tiles
self.use_sm100_kernel = can_use_vsa_block_sparse_sm100(
device.index, dtype, head_dim
)
n_tiles_padded = n_tiles + (n_tiles % 2 if self.use_sm100_kernel else 0)
seq_pad = n_tiles_padded * VSA_H3_TILE_ELEMS
self.key = _workspace_key(meta, heads, head_dim, has_gate, dtype, device)
self.tiled = torch.empty(
self.tiled = torch.zeros(
(3 + int(has_gate), heads, seq_pad, head_dim), dtype=dtype, device=device
)
self.pooled = torch.empty(
@@ -300,23 +311,67 @@ class _Workspace:
self.out_tiled = torch.empty(
(heads, seq_pad, head_dim), dtype=dtype, device=device
)
self.block_sizes = torch.zeros(n_tiles_padded, dtype=torch.int32, device=device)
self.block_sizes[:n_tiles] = meta.variable_block_sizes
all_tiles = torch.arange(n_tiles, dtype=torch.int32, device=device)
self.dense_index = all_tiles.repeat(heads, n_tiles, 1)
self.dense_num = torch.full(
(heads, n_tiles), n_tiles, dtype=torch.int32, device=device
self.dense_index = torch.zeros(
(heads, n_tiles_padded, n_tiles), dtype=torch.int32, device=device
)
self.dense_index[:, :n_tiles] = all_tiles
self.dense_num = torch.zeros(
(heads, n_tiles_padded), dtype=torch.int32, device=device
)
self.dense_num[:, :n_tiles] = n_tiles
self.q2k_index = self.dense_index.clone()
self.q2k_num = self.dense_num.clone()
num_video_tiles = meta.num_video_tiles
# Zeroed scratch for the exempt top-k compaction; kept zero between uses.
self.topk_mask = torch.zeros(
(heads, num_video_tiles, num_video_tiles), dtype=torch.int8, device=device
)
def sparse_lists(
self, video_lists: torch.Tensor, num_prefix_tiles: int
) -> tuple[torch.Tensor, torch.Tensor]:
width = video_lists.shape[-1]
self.q2k_index[:, num_prefix_tiles:, :width] = video_lists
self.q2k_num[:, num_prefix_tiles:] = width
self.q2k_index[:, num_prefix_tiles : self.n_tiles, :width] = video_lists
self.q2k_num[:, num_prefix_tiles : self.n_tiles] = width
return self.q2k_index, self.q2k_num
def _gate_tile_index(
meta: VideoSparseAttentionH3Metadata, row_start: int, num_rows: int
) -> torch.Tensor:
"""Tile of each packed row in the shard, -1 for pad rows."""
key = ("gate_tile_index", row_start, num_rows)
tile_index = meta.workspace_cache.get(key)
if tile_index is None:
unpack_index = meta.unpack_index
rows = torch.arange(row_start, row_start + num_rows, device=unpack_index.device)
tile_index = torch.full_like(rows, -1, dtype=torch.int32)
in_used = rows < meta.total_seq_length
tile_index[in_used] = unpack_index[rows[in_used]] // VSA_H3_TILE_ELEMS
meta.workspace_cache[key] = tile_index
return tile_index
def vsa_h3_fold_gate(
out: torch.Tensor,
gate_compress: torch.Tensor,
out_compress: torch.Tensor,
meta: VideoSparseAttentionH3Metadata,
row_start: int,
) -> None:
"""Fold the compression branch into ``out`` [rows, H, D], the row shard at
``row_start``."""
vsa_h3_gate_add(
out,
gate_compress,
out_compress,
_gate_tile_index(meta, row_start, out.shape[0]),
)
def _get_workspace(
meta: VideoSparseAttentionH3Metadata, query: torch.Tensor, has_gate: bool
) -> _Workspace:
@@ -341,6 +396,15 @@ def _select_kv_lists(
keep = _compute_topk(sparsity, meta.num_video_tiles)
if sparsity <= 0.0 or keep >= meta.num_video_tiles:
return ws.dense_index, ws.dense_num
prefix = meta.num_prefix_tiles
if meta.exempt or prefix == 0:
# Same lists as _topk_tile_lists without sorting values or indices.
picked = scores[:, prefix:, prefix:].topk(keep, dim=-1, sorted=False).indices
vsa_h3_topk_lists(
picked, ws.topk_mask, ws.q2k_index[:, prefix : ws.n_tiles], prefix
)
ws.q2k_num[:, prefix : ws.n_tiles] = prefix + keep
return ws.q2k_index, ws.q2k_num
return ws.sparse_lists(
_topk_tile_lists(
scores,
@@ -393,7 +457,9 @@ class VideoSparseAttentionH3Impl(AttentionImpl):
attn_metadata: AttentionMetadata,
) -> torch.Tensor:
raise NotImplementedError(
"VSA-H3 serves MiniMax-H3's packed varlen attention; use forward_varlen."
"VSA-H3 serves MiniMax-H3's packed varlen attention only; select it "
"for the transformer with --component-attention-backends "
"transformer=video_sparse_attn_h3 so dense layers keep their backends."
)
def forward_varlen(
@@ -407,8 +473,15 @@ class VideoSparseAttentionH3Impl(AttentionImpl):
cu_seqlens_host: tuple[int, ...] | None = None,
attn_metadata: VideoSparseAttentionH3Metadata | None = None,
gate_compress: torch.Tensor | None = None,
) -> torch.Tensor:
"""query/key/value: [T, H, D] packed rows (post-norm, post-RoPE)."""
return_compress: bool = False,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
"""query/key/value: [T, H, D] packed rows (post-norm, post-RoPE).
``return_compress`` returns the compression branch's ``[H, n_tiles, D]``
output instead of folding it, for ``vsa_h3_fold_gate`` after the
caller's collectives."""
assert not return_compress or (
gate_compress is None and attn_metadata is not None
), "return_compress replaces gate_compress and needs VSA-H3 metadata"
if self.layer_idx is None or attn_metadata is None:
if attn_metadata is None and self.layer_idx is not None:
raise RuntimeError(
@@ -441,6 +514,7 @@ class VideoSparseAttentionH3Impl(AttentionImpl):
sparsity = 0.0 if self.layer_idx in meta.dense_layers else meta.VSA_sparsity
has_gate = gate_compress is not None
needs_compress = has_gate or return_compress
ws = _get_workspace(meta, query, has_gate)
vsa_h3_pack_tiles(
@@ -456,7 +530,7 @@ class VideoSparseAttentionH3Impl(AttentionImpl):
q_pooled, k_pooled, v_pooled = ws.pooled
scores = None
if sparsity > 0.0 or has_gate:
if sparsity > 0.0 or needs_compress:
scores = torch.matmul(q_pooled, k_pooled.transpose(-2, -1)) * (
self.head_size**-0.5
)
@@ -468,21 +542,24 @@ class VideoSparseAttentionH3Impl(AttentionImpl):
ws.tiled[2:3],
q2k_index[None],
q2k_num[None],
meta.variable_block_sizes,
ws.block_sizes,
out=ws.out_tiled[None],
use_sm100_kernel=ws.use_sm100_kernel,
)
out_compress = None
if has_gate:
if needs_compress:
out_compress = torch.matmul(torch.softmax(scores, dim=-1), v_pooled)
result = torch.empty(query.shape, dtype=query.dtype, device=query.device)
vsa_h3_untile(
ws.out_tiled,
ws.tiled[3] if has_gate else None,
out_compress,
out_compress if has_gate else None,
meta.unpack_index,
used,
result,
)
if return_compress:
return result, out_compress
return result
@@ -17,6 +17,7 @@ padded tile layout the attention kernel reads and pools each tile in the same
pass; ``vsa_h3_untile`` scatters the attention output back to packed rows and
folds in the gated compression branch. Both replace chains of index copies
and transposes that otherwise cost more than the attention kernel itself.
``vsa_h3_gate_add`` folds the same branch after the caller's collectives.
"""
import math
@@ -26,6 +27,8 @@ import triton
import triton.language as tl
from triton.tools.tensor_descriptor import TensorDescriptor
from sglang.kernels.ops.diffusion import vsa_block_sparse_sm100
# BLOCK_M / BLOCK_N are structural, not tunable: the kernel indexes the top-k
# list per BLOCK_M q-tile and addresses keys as kv_idx * BLOCK_N, so both must
# match the granularity q2k_index and variable_block_sizes were built at.
@@ -106,10 +109,12 @@ def vsa_h3_block_sparse_attn_forward(
q2k_num: torch.Tensor,
variable_block_sizes: torch.Tensor,
out: torch.Tensor | None = None,
use_sm100_kernel: bool = False,
) -> torch.Tensor:
"""q/k/v: contiguous [B, H, S_pad, D] bf16 with S_pad = n_tiles * 64; pad
rows zero. q2k_index/q2k_num: contiguous [B, H, n_tiles, max_kv] /
[B, H, n_tiles] int32."""
[B, H, n_tiles] int32. ``use_sm100_kernel`` runs FastVideo's tcgen05 kernel (sm_100a /
sm_103a, even n_tiles) instead of the Triton kernel."""
batch, heads, seq_q, head_dim = q.shape
seq_kv = k.shape[2]
if seq_q % VSA_H3_KERNEL_BLOCK or seq_kv % VSA_H3_KERNEL_BLOCK:
@@ -124,6 +129,18 @@ def vsa_h3_block_sparse_attn_forward(
)
if out is None:
out = torch.empty_like(q)
if use_sm100_kernel:
vsa_block_sparse_sm100(
q,
k,
v,
q2k_index.view(-1, q2k_index.shape[-1]),
q2k_num.view(-1),
variable_block_sizes,
out,
1.0 / math.sqrt(head_dim),
)
return out
block = [1, 1, VSA_H3_KERNEL_BLOCK, head_dim]
desc_q, desc_k, desc_v, desc_o = (
TensorDescriptor.from_tensor(t, block_shape=block) for t in (q, k, v, out)
@@ -243,9 +260,10 @@ def vsa_h3_pack_tiles(
"""Gather packed [T, H, D] rows into ``tiled`` [3|4, H, S_pad, D] and write
fp32 per-tile means of q/k/v into ``pooled`` [3, H, n_tiles, D].
``src_index`` maps each padded position to its packed row, or -1 (pad -> 0).
S_pad may hold one extra empty tile past ``n_tiles``; it is left untouched.
"""
_, heads, seq_pad, head_dim = tiled.shape
n_tiles = seq_pad // VSA_H3_KERNEL_BLOCK
n_tiles = pooled.shape[2]
has_gate = gate is not None
g = gate if has_gate else q
assert all(t.stride(-1) == 1 for t in (q, k, v, g))
@@ -327,6 +345,8 @@ def vsa_h3_untile(
heads, seq_pad, head_dim = out_tiled.shape
total = result.shape[0]
has_gate = gate_tiled is not None
# S_pad may carry one padded tile; the compression output does not
n_tiles = out_compress.shape[1] if has_gate else seq_pad // VSA_H3_KERNEL_BLOCK
_untile_kernel[(triton.cdiv(total, VSA_H3_KERNEL_BLOCK), heads)](
out_tiled,
gate_tiled if has_gate else out_tiled,
@@ -336,8 +356,126 @@ def vsa_h3_untile(
used,
total,
seq_pad,
seq_pad // VSA_H3_KERNEL_BLOCK,
n_tiles,
HAS_GATE=has_gate,
HEAD_DIM=head_dim,
BLOCK=VSA_H3_KERNEL_BLOCK,
)
@triton.jit
def _gate_add_kernel(
Out,
Gate,
Compress,
tile_index,
num_rows,
stride_o_row,
stride_o_head,
stride_g_row,
stride_g_head,
N_TILES,
HEAD_DIM: tl.constexpr,
BLOCK: tl.constexpr,
):
row_block = tl.program_id(0)
h = tl.program_id(1)
rows = row_block * BLOCK + tl.arange(0, BLOCK)
cols = tl.arange(0, HEAD_DIM)
in_rows = rows < num_rows
tile = tl.load(tile_index + rows, mask=in_rows, other=-1)
valid = (tile >= 0)[:, None]
o_off = (
rows.to(tl.int64)[:, None] * stride_o_row + h * stride_o_head + cols[None, :]
)
g_off = (
rows.to(tl.int64)[:, None] * stride_g_row + h * stride_g_head + cols[None, :]
)
o = tl.load(Out + o_off, mask=valid, other=0.0).to(tl.float32)
g = tl.load(Gate + g_off, mask=valid, other=0.0).to(tl.float32)
c = tl.load(
Compress + (h * N_TILES + tile)[:, None] * HEAD_DIM + cols[None, :],
mask=valid,
other=0.0,
)
o = o + c * g
tl.store(Out + o_off, o.to(Out.type.element_ty), mask=valid)
def vsa_h3_gate_add(
out: torch.Tensor,
gate: torch.Tensor,
out_compress: torch.Tensor,
tile_index: torch.Tensor,
) -> None:
"""In place: ``out[t, h] += out_compress[h, tile_index[t]] * gate[t, h]``
where ``tile_index[t] >= 0``; same fp32 arithmetic as ``vsa_h3_untile``."""
num_rows, heads, head_dim = out.shape
assert out.stride(-1) == 1 and gate.stride(-1) == 1
assert out_compress.is_contiguous() and out_compress.shape[0] == heads
_gate_add_kernel[(triton.cdiv(num_rows, VSA_H3_KERNEL_BLOCK), heads)](
out,
gate,
out_compress,
tile_index,
num_rows,
out.stride(0),
out.stride(1),
gate.stride(0),
gate.stride(1),
out_compress.shape[1],
HEAD_DIM=head_dim,
BLOCK=VSA_H3_KERNEL_BLOCK,
)
@triton.jit
def _topk_lists_kernel(
mask_ptr,
out_ptr,
num_video_tiles,
prefix,
stride_mh,
stride_mr,
stride_oh,
stride_or,
VIDEO_POW2: tl.constexpr,
):
row = tl.program_id(0)
head = tl.program_id(1)
mask_row = mask_ptr + head * stride_mh + row * stride_mr
out_row = out_ptr + head * stride_oh + row * stride_or
cols = tl.arange(0, VIDEO_POW2)
in_range = cols < num_video_tiles
picked = tl.load(mask_row + cols, mask=in_range, other=0).to(tl.int32)
slot = tl.cumsum(picked, axis=0) - 1
tl.store(out_row + prefix + slot, cols + prefix, mask=in_range & (picked != 0))
# Leave the mask zeroed for the next layer's scatter.
tl.store(mask_row + cols, tl.zeros_like(picked).to(tl.int8), mask=in_range)
def vsa_h3_topk_lists(
picked: torch.Tensor, mask: torch.Tensor, out: torch.Tensor, num_prefix_tiles: int
) -> None:
"""Write ``sort(picked + prefix)`` into ``out[h, r, prefix:]``.
``picked`` [H, rows, keep] holds distinct video-tile indices in any order;
``mask`` [H, rows, num_video_tiles] is an all-zero int8 scratch, left
zeroed; ``out`` [H, rows, >= prefix + keep] is int32 whose first ``prefix``
columns already hold ``[0, prefix)``."""
heads, rows, _ = picked.shape
num_video_tiles = mask.shape[-1]
assert mask.dtype == torch.int8 and mask.is_contiguous()
assert out.dtype == torch.int32 and out.stride(-1) == 1
mask.scatter_(-1, picked, 1)
_topk_lists_kernel[(rows, heads)](
mask,
out,
num_video_tiles,
num_prefix_tiles,
mask.stride(0),
mask.stride(1),
out.stride(0),
out.stride(1),
VIDEO_POW2=triton.next_power_of_2(num_video_tiles),
)
@@ -109,6 +109,20 @@ def _usp_all_to_all_single(x: torch.Tensor, role: str | None = None) -> torch.Te
return output.reshape(x_shape)
def _usp_all_gather(x: torch.Tensor) -> torch.Tensor:
"""Concatenate ``x`` from every Ulysses rank along dim 0, rank order."""
ulysses_pg = get_sp_group().ulysses_group
assert ulysses_pg is not None, "Ulysses process group is not initialized."
x = x.contiguous()
output = torch.empty(
(get_ulysses_parallel_world_size() * x.shape[0], *x.shape[1:]),
dtype=x.dtype,
device=x.device,
)
dist.all_gather_into_tensor(output, x, group=ulysses_pg)
return output
def _usp_all_to_all_single_varlen(
x: torch.Tensor,
output_split_sizes: list[int],
@@ -59,6 +59,9 @@ from sglang.multimodal_gen.runtime.distributed.parallel_state import (
from sglang.multimodal_gen.runtime.layers.attention.backends.attention_backend import (
AttentionRequirements,
)
from sglang.multimodal_gen.runtime.layers.attention.backends.video_sparse_attn_h3 import (
vsa_h3_fold_gate,
)
from sglang.multimodal_gen.runtime.layers.attention.selector import (
claim_deferred_component_attn_backend,
get_attn_backend,
@@ -673,14 +676,12 @@ def _minimax_h3_attention_core_impl(
if ulysses_active:
from sglang.multimodal_gen.runtime.layers.usp import (
_usp_input_all_to_all,
_usp_all_gather,
_usp_input_all_to_all_packed_qkv,
_usp_output_all_to_all,
)
q, k, v = _usp_input_all_to_all_packed_qkv(q, k, v)
if gate_compress is not None:
gate_compress = _usp_input_all_to_all(gate_compress[None], head_dim=2)[0]
if attention._attention_impl is None:
attention._set_attention_backend(
@@ -698,7 +699,9 @@ def _minimax_h3_attention_core_impl(
if attention.prefix.startswith("blocks.")
else None
)
out = attention._attention_impl.forward_varlen(
# the gate stays on this rank's row shard; fold it after the output all-to-all
fold_after_exchange = ulysses_active and gate_compress is not None
attn_result = attention._attention_impl.forward_varlen(
q,
k,
v,
@@ -706,10 +709,24 @@ def _minimax_h3_attention_core_impl(
max_seqlen=max_seqlen,
cu_seqlens_host=cu_seqlens_host,
attn_metadata=attn_metadata,
gate_compress=gate_compress,
gate_compress=None if fold_after_exchange else gate_compress,
return_compress=fold_after_exchange,
)
if fold_after_exchange:
out, out_compress = attn_result
else:
out = attn_result
if ulysses_active:
out = _usp_output_all_to_all(out[None], head_dim=2)[0]
if fold_after_exchange:
_, ulysses_rank = get_ulysses_ctx()
vsa_h3_fold_gate(
out,
gate_compress,
_usp_all_gather(out_compress),
attn_metadata,
row_start=ulysses_rank * out.shape[0],
)
return out
if ring_active:
@@ -73,6 +73,7 @@ class MiniMaxH3VideoVAE(AutoencoderKLLegacy, LayerwiseOffloadableModuleMixin):
and config.use_parallel_decode
and config.use_parallel_tiling
and bool(arch.vae_parallel_tiling),
stack_tiling=config.stack_tiling,
tile_size=int(arch.vae_tile_size),
tile_overlap_min=int(arch.vae_tile_overlap_min),
encoder_parallel=False,
@@ -0,0 +1,181 @@
# SPDX-License-Identifier: Apache-2.0
"""CUDA fast path for the MiniMax-H3 video VAE decoder (ViT3DDecoder).
Fuses each decoder attention block's per-head QK RMSNorm + NeoX RoPE into one
in-place launch over the strided Q/K views of the QKV buffer and runs the
block's attention on cuDNN SDPA. Forwards are bound once at VAE load and
dispatch on a decode-scoped :class:`VaeFastPathGate`: ``quality="lossless"``
and ``"high"`` take the fast path (rounding-level differences), the
``"exact"`` runs the original module path bit-for-bit. Install is
all-or-nothing and fail-closed.
"""
from types import MethodType
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.nn.attention import SDPBackend, sdpa_kernel
from sglang.kernels.ops.diffusion import (
can_use_fused_inplace_qknorm_rope,
fused_inplace_qknorm_rope,
)
from sglang.multimodal_gen.runtime.models.vaes.fast_path_gate import (
VaeFastPathGate,
register_vae_fast_path_gate,
)
from sglang.multimodal_gen.runtime.models.vaes.minimax_h3_video_vae.attention import (
Attention,
_apply_qk_norm,
_sdpa_attention,
)
from sglang.multimodal_gen.runtime.models.vaes.minimax_h3_video_vae.vit_utils import (
_env_flag,
apply_rotary_pos_emb_qk,
native_rope_cache,
)
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
logger = init_logger(__name__)
def _fused_qknorm_rope(self, query, key, rotary_pos_emb) -> bool:
native = native_rope_cache(query, key, rotary_pos_emb)
if native is None:
return False
cache, positions = native
if not can_use_fused_inplace_qknorm_rope(
head_dim=self.dim_head,
rope_dim=cache.shape[1],
is_neox=True,
dtype=query.dtype,
cache_dtype=cache.dtype,
round_norm_before_rope=True,
):
return False
weight = self._sgl_unit_weight
if weight is None or weight.dtype != query.dtype or weight.device != query.device:
weight = torch.ones(self.dim_head, dtype=query.dtype, device=query.device)
self._sgl_unit_weight = weight
# Batched decoder tiles flatten into rows; a view, so the update is in place.
fused_inplace_qknorm_rope(
query.flatten(0, 1),
key.flatten(0, 1),
weight,
weight,
cache,
positions,
is_neox=True,
eps=self.norm_q.eps,
head_dim=self.dim_head,
rope_dim=cache.shape[1],
round_norm_before_rope=True,
)
return True
def _cudnn_attention(self, query, key, value):
if not self._sgl_cudnn_failed:
try:
with sdpa_kernel(SDPBackend.CUDNN_ATTENTION):
return F.scaled_dot_product_attention(
query.transpose(1, 2), key.transpose(1, 2), value.transpose(1, 2)
).transpose(1, 2)
except RuntimeError as e:
logger.warning(
"MiniMax-H3 VAE: cuDNN SDPA failed (%s); using the layer's "
"attention backend.",
e,
)
self._sgl_cudnn_failed = True
return self.attn(query, key, value)
def _attn_fast_forward(self, hidden_states, rotary_pos_emb=None):
if (
not self._sgl_gate.enabled
or rotary_pos_emb is None
or torch.is_grad_enabled()
or torch.compiler.is_compiling()
):
return type(self).forward(self, hidden_states, rotary_pos_emb)
batch_size, seq_len, _ = hidden_states.shape
qkv = self.to_qkv(hidden_states)
qkv = qkv.view(batch_size, seq_len, -1, 3 * self.dim_head)
query, key, value = torch.chunk(qkv, 3, dim=-1)
if not _fused_qknorm_rope(self, query, key, rotary_pos_emb):
query = _apply_qk_norm(self.norm_q, query)
key = _apply_qk_norm(self.norm_k, key)
query, key = apply_rotary_pos_emb_qk(query, key, rotary_pos_emb)
if self.attn is not None and query.dtype in (torch.float16, torch.bfloat16):
hidden_states = _cudnn_attention(self, query, key, value)
else:
hidden_states = _sdpa_attention(query, key, value)
hidden_states = hidden_states.reshape(batch_size, seq_len, -1)
return self.to_out(hidden_states)
def _plain_rms_norm(module) -> bool:
return isinstance(module, nn.RMSNorm) and module.weight is None
def _attn_fast_compatible(attn: nn.Module) -> bool:
return (
type(attn) is Attention
and _plain_rms_norm(attn.norm_q)
and _plain_rms_norm(attn.norm_k)
and attn.norm_q.eps == attn.norm_k.eps
)
def _install_fast_attention(
attn_modules: list[nn.Module], gate: VaeFastPathGate
) -> None:
for attn in attn_modules:
attn._sgl_gate = gate
attn._sgl_unit_weight = None
attn._sgl_cudnn_failed = False
attn.forward = MethodType(_attn_fast_forward, attn)
def maybe_optimize_minimax_h3_vae(vae: nn.Module) -> nn.Module:
"""Install the quality-gated CUDA MiniMax-H3 VAE decoder fast path."""
from sglang.multimodal_gen.runtime.models.vaes.minimax_h3 import MiniMaxH3VideoVAE
from sglang.multimodal_gen.runtime.models.vaes.minimax_h3_video_vae.vae_vit import (
ViT3DDecoder,
)
if not isinstance(vae, MiniMaxH3VideoVAE):
return vae
decoder = vae.decoder
if type(decoder) is not ViT3DDecoder:
return vae
if not _env_flag("MINIMAX_H3_VAE_DECODER_VIT_FP32_NORM", "1"):
logger.info("MiniMax-H3 VAE: fp32 QK norm disabled; skipping fast path.")
return vae
attn_modules = [block.attn for block in decoder.transformer_blocks]
eligible = [attn for attn in attn_modules if _attn_fast_compatible(attn)]
if len(eligible) != len(attn_modules):
logger.warning(
"MiniMax-H3 VAE: %d/%d decoder attention blocks non-standard; "
"skipping fast path.",
len(attn_modules) - len(eligible),
len(attn_modules),
)
return vae
gate = VaeFastPathGate()
_install_fast_attention(eligible, gate)
register_vae_fast_path_gate(vae, gate)
logger.info(
"MiniMax-H3 VAE: installed quality-gated fast path (%d QK RMSNorm+RoPE "
"fusions, cuDNN SDPA).",
len(eligible),
)
return vae
@@ -426,6 +426,7 @@ class ViT3DDecoder(ViTBase):
rotary_pos_emb = prepare_rotary_pos_emb(
self.pos_embed(img_ids[:1]),
dtype=rotary_dtype,
batch=B,
)
if B > 1:
rotary_pos_emb = _expand_rotary_batch(rotary_pos_emb, B)
@@ -115,8 +115,10 @@ def prepare_rotary_pos_emb(
rotary_pos_emb: Tuple[torch.Tensor, torch.Tensor],
*,
dtype: torch.dtype,
batch: int = 1,
) -> tuple[torch.Tensor, ...]:
"""Prebuild the native Q/K rotary cache once per ViT decoder forward."""
"""Prebuild the native Q/K rotary cache once per ViT decoder forward; its
positions cover ``batch`` samples sharing one token grid."""
cos, sin = rotary_pos_emb
if (
not cos.is_cuda
@@ -143,7 +145,7 @@ def prepare_rotary_pos_emb(
cos.shape[1],
dtype=torch.long,
device=cos.device,
)
).repeat(batch)
return cos, sin, cache, positions
@@ -209,13 +211,15 @@ def apply_rotary_pos_emb(
raise
def apply_rotary_pos_emb_qk(
def native_rope_cache(
query: torch.Tensor,
key: torch.Tensor,
rotary_pos_emb: Sequence[torch.Tensor],
) -> tuple[torch.Tensor, torch.Tensor]:
"""Apply the exact native NeoX rotary kernel to Q/K together when possible."""
if (
) -> tuple[torch.Tensor, torch.Tensor] | None:
"""``(cache, positions)`` from ``prepare_rotary_pos_emb`` when the native
kernels can consume it for this Q/K, else None. ``positions`` covers the
batch-major rows of ``query.flatten(0, 1)``."""
if not (
len(rotary_pos_emb) == 4
and query.is_cuda
and query.shape == key.shape
@@ -224,33 +228,47 @@ def apply_rotary_pos_emb_qk(
and query.dim() == 4
and not torch.compiler.is_compiling()
):
_, _, cache, positions = rotary_pos_emb
batch, seq_len = query.shape[:2]
if (
cache.is_cuda
and cache.dtype == query.dtype
and cache.dim() == 2
and cache.shape[0] == seq_len
and cache.shape[1] <= query.shape[-1]
and positions.is_cuda
and positions.shape == (seq_len,)
):
from sgl_kernel import rotary_embedding
return None
_, _, cache, positions = rotary_pos_emb
if not (
cache.is_cuda
and cache.dtype == query.dtype
and cache.dim() == 2
and cache.shape[0] == query.shape[1]
and cache.shape[1] <= query.shape[-1]
and positions.is_cuda
and positions.shape in ((query.shape[1],), (query.shape[0] * query.shape[1],))
):
return None
# Accept both main's shared sequence positions and the pre-expanded
# batch-major positions used by the decoder cache and fused fast path.
if positions.shape == (query.shape[1],) and query.shape[0] > 1:
positions = positions.repeat(query.shape[0])
return cache, positions
# Stacked spatial tiles share one rotary table. Flatten the
# batch and replay that table once per tile.
query = query.contiguous()
key = key.contiguous()
token_positions = positions if batch == 1 else positions.repeat(batch)
rotary_embedding(
token_positions,
query.view(batch * seq_len, -1),
key.view(batch * seq_len, -1),
query.shape[-1],
cache,
True,
)
return query, key
def apply_rotary_pos_emb_qk(
query: torch.Tensor,
key: torch.Tensor,
rotary_pos_emb: Sequence[torch.Tensor],
) -> tuple[torch.Tensor, torch.Tensor]:
"""Apply the exact native NeoX rotary kernel to Q/K together when possible."""
native = native_rope_cache(query, key, rotary_pos_emb)
if native is not None:
from sgl_kernel import rotary_embedding
cache, positions = native
query = query.contiguous()
key = key.contiguous()
rotary_embedding(
positions,
query.view(positions.shape[0], -1),
key.view(positions.shape[0], -1),
query.shape[-1],
cache,
True,
)
return query, key
return (
apply_rotary_pos_emb(query, rotary_pos_emb),
@@ -175,6 +175,11 @@ class MiniMaxH3Pipeline(LoRAPipeline, ComposedPipelineBase):
self.add_stage(
MiniMaxH3TimestepPreparationStage(
sigma_shift_scales=sigma_shift_scales,
dmd_denoising_steps=(
release_metadata.dmd_denoising_steps
if release_metadata is not None
else None
),
)
)
self.add_stage(
@@ -201,7 +206,7 @@ class MiniMaxH3Pipeline(LoRAPipeline, ComposedPipelineBase):
class FastH3Pipeline(MiniMaxH3Pipeline):
"""FastH3: 4-step DMD2-distilled MiniMax-H3 (t2va only).
"""FastH3: 8-step DMD2-distilled MiniMax-H3 (V2, t2va only).
The flat single-partition repo is materialized into the base-H3 layout by
the bundled model overlay (see model_overlays/), so every stage, loader,
@@ -52,6 +52,8 @@ class MiniMaxH3ReleaseMetadata:
task_aliases: Mapping[str, str]
video_sigma_shift: float
audio_sigma_shift: float
# trained DMD rungs of a distilled release, unshifted; None keeps the uniform grid
dmd_denoising_steps: tuple[int, ...] | None = None
@classmethod
def from_model_index(
@@ -92,6 +94,21 @@ class MiniMaxH3ReleaseMetadata:
"model_index.json._minimax_h3.sigma_shift_scales requires numeric "
"video and audio values"
) from exc
dmd_steps = raw.get("dmd_denoising_steps")
if dmd_steps is not None:
if (
not isinstance(dmd_steps, list)
or not dmd_steps
or any(
type(step) is not int or not 0 < step <= 1000 for step in dmd_steps
)
or any(left <= right for left, right in zip(dmd_steps, dmd_steps[1:]))
):
raise ValueError(
"model_index.json._minimax_h3.dmd_denoising_steps must be "
"strictly decreasing integers in (0, 1000]"
)
dmd_steps = tuple(dmd_steps)
metadata = cls(
schema_version=1,
partition=partition,
@@ -99,6 +116,7 @@ class MiniMaxH3ReleaseMetadata:
task_aliases=dict(aliases),
video_sigma_shift=video_sigma,
audio_sigma_shift=audio_sigma,
dmd_denoising_steps=dmd_steps,
)
for task in metadata.tasks:
if canonical_minimax_h3_task(task) != task:
@@ -7,6 +7,9 @@ from contextlib import contextmanager, nullcontext
import torch
from sglang.multimodal_gen.configs.sample.sampling_params import (
quality_allows,
)
from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType
from sglang.multimodal_gen.runtime.distributed import (
get_replica_group,
@@ -16,6 +19,7 @@ from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_c
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
ComponentUse,
)
from sglang.multimodal_gen.runtime.models.vaes.fast_path_gate import use_vae_fast_path
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch, Req
from sglang.multimodal_gen.runtime.pipelines_core.stages.base import (
StageParallelismType,
@@ -494,7 +498,13 @@ class MiniMaxH3DecodingStage(DecodingStage):
)
decode_kwargs["on_frames"] = stream
try:
with set_forward_context(current_timestep=0, attn_metadata=None):
with (
use_vae_fast_path(
selected_video_vae,
quality_allows(batch.sampling_params.quality, "lossless"),
),
set_forward_context(current_timestep=0, attn_metadata=None),
):
decoded = video_decode(visual_decode_latent, **decode_kwargs)
if stream is not None:
output_file_paths = stream.finish()
@@ -1077,7 +1077,10 @@ def _maybe_prepare_vsa_h3_step_metadata(
"VSA-H3 in SGLang serves the trained 64-token (4, 4, 4) tile "
f"geometry; got vsa_tile_size={tile_size}."
)
sparsity = float(config.get("VSA_sparsity", config.get("sparsity", 0.9)))
default_sparsity = server_args.pipeline_config.vsa_sparsity
sparsity = float(
config.get("VSA_sparsity", config.get("sparsity", default_sparsity))
)
if not 0.0 <= sparsity < 1.0:
raise ValueError(f"VSA sparsity must be in [0, 1), got {sparsity}")
mode = str(config.get("vsa_mode", "exempt"))
@@ -26,12 +26,17 @@ class MiniMaxH3TimestepPreparationStage(PipelineStage):
deduplicated_tensor_tree_output_fields = ("timesteps", "sigmas")
deduplicated_extra_tensor_tree_output_keys = (MINIMAX_H3_SIGMAS_EXTRA_KEY,)
def __init__(self, sigma_shift_scales=None) -> None:
def __init__(
self,
sigma_shift_scales=None,
dmd_denoising_steps: tuple[int, ...] | None = None,
) -> None:
super().__init__()
# Per-model sigma shift override (model_index.json "_minimax_h3" release
# block, sigma_shift_scales): the schedule constants are a MODEL
# serving contract — fl2va and ref2va use video 12 / audio 3 by default.
self.sigma_shift_scales = sigma_shift_scales
self.dmd_denoising_steps = dmd_denoising_steps
self._pdd_config = None
pdd_heads = envs.SGLANG_DIFFUSION_MINIMAX_H3_PDD_HEADS
if pdd_heads:
@@ -197,12 +202,20 @@ class MiniMaxH3TimestepPreparationStage(PipelineStage):
task_default=plan.default_audio_flow_shift,
),
}
dmd_steps = self.dmd_denoising_steps
num_steps = requested_num_steps
if dmd_steps is not None and batch.is_warmup:
# warmup truncates the trained rungs instead of rescaling a shorter grid
num_steps = len(dmd_steps) + 1
sigmas: dict[str, list[float]] = {}
for modality in ("video", "audio"):
sigmas[modality] = minimax_h3_time_shift_sigmas(
num_steps=requested_num_steps,
num_steps=num_steps,
shift_scale=scales[modality],
dmd_steps=dmd_steps,
)
if num_steps != requested_num_steps:
sigmas[modality] = sigmas[modality][: max(2, requested_num_steps)]
batch.extra[MINIMAX_H3_SIGMAS_EXTRA_KEY] = sigmas
def verify_input(self, batch: Req, server_args: ServerArgs) -> VerificationResult:
@@ -33,6 +33,7 @@ def minimax_h3_time_shift_sigmas(
*,
num_steps: int = 50,
shift_scale: float = 6.0,
dmd_steps: tuple[int, ...] | None = None,
) -> list[float]:
if shift_scale <= 0:
raise ValueError("MiniMax H3 shift_scale must be > 0")
@@ -41,14 +42,26 @@ def minimax_h3_time_shift_sigmas(
import torch
# The rectified-flow sigma range is fixed at [1.0, 0.0].
base = torch.linspace(
1.0,
0.0,
int(num_steps),
device="cpu",
dtype=torch.float32,
)
if dmd_steps is not None:
if int(num_steps) != len(dmd_steps) + 1:
raise ValueError(
f"this checkpoint is distilled for {len(dmd_steps)} DiT forwards "
f"({len(dmd_steps) + 1} sigma grid points); got "
f"num_inference_steps={num_steps}"
)
# trained rungs are unshifted timesteps over 1000; append the final step to sigma 0
base = torch.tensor(
[step / 1000.0 for step in dmd_steps] + [0.0], dtype=torch.float32
)
else:
# The rectified-flow sigma range is fixed at [1.0, 0.0].
base = torch.linspace(
1.0,
0.0,
int(num_steps),
device="cpu",
dtype=torch.float32,
)
shifted = float(shift_scale) * base / (1 + (float(shift_scale) - 1) * base)
shifted, _ = torch.unique_consecutive(shifted, return_counts=True)
# A one-point request is still exactly one point. Normal serving uses
@@ -847,8 +847,8 @@ class CudaPlatformBase(Platform):
@classmethod
def optimize_vae(cls, vae: torch.nn.Module) -> torch.nn.Module:
"""Install the quality-gated FLUX.2 / AutoencoderKL / Wan / Qwen-Image
VAE decoder fast paths.
"""Install the quality-gated FLUX.2 / AutoencoderKL / Wan / Qwen-Image /
MiniMax-H3 VAE decoder fast paths.
Requests with quality="lossless" or "high" run the fast paths; the
"exact" default runs the original module path bit-for-bit. See
@@ -859,6 +859,9 @@ class CudaPlatformBase(Platform):
maybe_optimize_autoencoder_kl,
maybe_optimize_flux2_vae,
)
from sglang.multimodal_gen.runtime.models.vaes.minimax_h3_vae_cuda_opt import (
maybe_optimize_minimax_h3_vae,
)
from sglang.multimodal_gen.runtime.models.vaes.wan_vae_cuda_opt import (
maybe_optimize_qwen_image_vae,
maybe_optimize_wan_vae,
@@ -868,6 +871,7 @@ class CudaPlatformBase(Platform):
vae = maybe_optimize_autoencoder_kl(vae)
vae = maybe_optimize_wan_vae(vae)
vae = maybe_optimize_qwen_image_vae(vae)
vae = maybe_optimize_minimax_h3_vae(vae)
except Exception:
logger.warning(
"Failed to apply CUDA VAE optimizations; using the unmodified VAE.",
@@ -45,9 +45,9 @@ BUILTIN_MODEL_OVERLAY_REGISTRY: dict[str, dict[str, Any]] = {
"overlay_repo_id": "AgainstEntropy/SANA-WM_streaming-overlay",
"overlay_revision": "62c6840871ecc3559189047513ba0670e1bf62e7",
},
"FastVideo/FastVideo-FastH3-4-step-Preview-v1-VSA-DataFree": {
"overlay_repo_id": "kevin-mi/FastH3-4step-Preview-overlay",
"overlay_revision": "f769cb8001dae335089de7b250364335bc7cb183",
"FastVideo/FastVideo-FastH3-8-Step-V2": {
"overlay_repo_id": "kevin-mi/FastH3-8step-V2-overlay",
"overlay_revision": "308e2dd627218c40ae759a93745429800771a625",
},
"OpenVDN/vdn-minimax-h3": {
"overlay_repo_id": "kevin-mi/VDN-H3-overlay",
@@ -775,14 +775,12 @@ MINIMAX_H3_FOUR_GPU_H100_CASES = [
DiffusionTestCase(
"fasth3_t2va_vsa_4gpu_h100",
DiffusionServerArgs(
model_path="FastVideo/FastVideo-FastH3-4-step-Preview-v1-VSA-DataFree",
model_path="FastVideo/FastVideo-FastH3-8-Step-V2",
modality="video",
num_gpus=4,
extras=[
"--attention-backend",
"video_sparse_attn_h3",
"--attention-backend-config",
'{"VSA_sparsity": 0.9}',
"--component-attention-backends",
"transformer=video_sparse_attn_h3",
"--enable-torch-compile",
"false",
],
@@ -805,7 +803,7 @@ MINIMAX_H3_FOUR_GPU_H100_CASES = [
"aspect_ratio": "16:9",
"duration_seconds": 5.0,
},
"num_inference_steps": 5,
"num_inference_steps": 9,
"seed": 42,
},
),
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
"""FastH3 (4-step VSA-distilled MiniMax-H3) registration and admission contracts."""
"""FastH3 8-Step V2 (VSA-distilled MiniMax-H3) registration and admission contracts."""
from __future__ import annotations
@@ -25,11 +25,18 @@ from sglang.multimodal_gen.runtime.distributed.parallel_state import (
from sglang.multimodal_gen.runtime.layers.linear import UnquantizedLinearMethod
from sglang.multimodal_gen.runtime.layers.quantization.fp8 import Fp8Config
from sglang.multimodal_gen.runtime.models.dits.minimax_h3 import MiniMaxH3DiTModel
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.release_metadata import (
MiniMaxH3ReleaseMetadata,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.time_request import (
minimax_h3_time_shift_sigmas,
)
from sglang.multimodal_gen.test.single_test_file.component_accuracy.utils import (
ensure_distributed_env_defaults,
)
FASTH3_MODEL_ID = "FastVideo/FastVideo-FastH3-4-step-Preview-v1-VSA-DataFree"
FASTH3_MODEL_ID = "FastVideo/FastVideo-FastH3-8-Step-V2"
DMD_STEPS = (999, 874, 749, 624, 500, 375, 250, 125)
def _ensure_single_process_parallel_runtime() -> None:
@@ -45,18 +52,20 @@ def test_registry_resolves_fasth3_configs() -> None:
assert info.pipeline_config_cls is FastH3PipelineConfig
assert get_non_diffusers_pipeline_name(FASTH3_MODEL_ID) == "FastH3Pipeline"
materialized = (
"/cache/materialized_models/"
"FastVideo__FastVideo-FastH3-4-step-Preview-v1-VSA-DataFree-0123abcd"
"/cache/materialized_models/FastVideo__FastVideo-FastH3-8-Step-V2-0123abcd"
)
assert get_model_info(materialized).sampling_param_cls is FastH3SamplingParams
def test_fasth3_sampling_defaults_and_task_rejection() -> None:
params = FastH3SamplingParams(prompt="p")
assert params.num_inference_steps == 5
assert params.num_inference_steps == 9
assert params.guidance_scale == 1.0
assert params.quality == "lossless"
assert FastH3SamplingParams(prompt="p", quality="extra-high").quality == "lossless"
assert FastH3SamplingParams(prompt="p", quality="exact").quality == "exact"
with pytest.raises(ValueError, match="exactly five sigma grid points"):
with pytest.raises(ValueError, match="exactly nine sigma grid points"):
FastH3SamplingParams(prompt="p", num_inference_steps=50)
with pytest.raises(ValueError, match="distilled for t2va only"):
@@ -72,8 +81,56 @@ def test_fasth3_sampling_defaults_and_task_rejection() -> None:
)
def test_fasth3_dmd_schedule_matches_trained_rungs() -> None:
"""The distilled rungs are shifted exactly once, as FastVideo serves them, and
a wrong step count or non-decreasing rungs are rejected."""
metadata = MiniMaxH3ReleaseMetadata.from_model_index(
{
"_minimax_h3": {
"schema_version": 1,
"partition": "fl2va",
"tasks": ["t2va"],
"sigma_shift_scales": {"video": 10.0, "audio": 3.0},
"dmd_denoising_steps": list(DMD_STEPS),
}
}
)
assert metadata.dmd_denoising_steps == DMD_STEPS
# FastVideo's _set_dmd_schedule output for the video / audio shifts 10 / 3
served = {
10.0: [
*(0.9999, 0.985788, 0.967575, 0.943168, 0.909091),
*(0.857143, 0.769231, 0.588235, 0.0),
],
3.0: [
*(0.999666, 0.954148, 0.89952, 0.83274, 0.75),
*(0.642857, 0.5, 0.3, 0.0),
],
}
for shift, expected in served.items():
sigmas = minimax_h3_time_shift_sigmas(
num_steps=9, shift_scale=shift, dmd_steps=DMD_STEPS
)
assert sigmas == pytest.approx(expected, abs=1e-6)
with pytest.raises(ValueError, match="8 DiT forwards"):
minimax_h3_time_shift_sigmas(num_steps=5, shift_scale=10.0, dmd_steps=DMD_STEPS)
with pytest.raises(ValueError, match="strictly decreasing"):
MiniMaxH3ReleaseMetadata.from_model_index(
{
"_minimax_h3": {
"schema_version": 1,
"partition": "fl2va",
"tasks": ["t2va"],
"sigma_shift_scales": {"video": 10.0, "audio": 3.0},
"dmd_denoising_steps": [999, 999],
}
}
)
def test_fasth3_pipeline_config_gates_and_rejections() -> None:
config = FastH3PipelineConfig()
assert config.vsa_sparsity == 0.8
assert config.dit_config.arch_config.has_gate_compress
assert not MiniMaxH3PipelineConfig().dit_config.arch_config.has_gate_compress
mapping = config.dit_config.arch_config.param_names_mapping
@@ -5,6 +5,7 @@ import os
import subprocess
import sys
import textwrap
from types import SimpleNamespace
from unittest import mock
import pytest
@@ -14,10 +15,17 @@ import torch.nn as nn
from sglang.multimodal_gen.configs.models.vaes.minimax_h3_video import (
MiniMaxH3VideoVAEConfig,
)
from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context
from sglang.multimodal_gen.runtime.models.vaes.fast_path_gate import VaeFastPathGate
from sglang.multimodal_gen.runtime.models.vaes.minimax_h3 import MiniMaxH3VideoVAE
from sglang.multimodal_gen.runtime.models.vaes.minimax_h3_audio_vae.audio_vae import (
CausalAttention,
)
from sglang.multimodal_gen.runtime.models.vaes.minimax_h3_vae_cuda_opt import (
_attn_fast_compatible,
_fused_qknorm_rope,
_install_fast_attention,
)
from sglang.multimodal_gen.runtime.models.vaes.minimax_h3_video_vae import (
AutoencoderKLLegacy,
)
@@ -25,14 +33,23 @@ from sglang.multimodal_gen.runtime.models.vaes.minimax_h3_video_vae.attention im
Attention,
_apply_qk_norm,
)
from sglang.multimodal_gen.runtime.models.vaes.minimax_h3_video_vae.base_module import (
RotaryEmbeddingND,
)
from sglang.multimodal_gen.runtime.models.vaes.minimax_h3_video_vae.vae_vit import (
ViT3DDecoder,
)
from sglang.multimodal_gen.runtime.models.vaes.minimax_h3_video_vae.vit_utils import (
apply_rotary_pos_emb_qk,
create_token_ids,
prepare_rotary_pos_emb,
)
from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum
requires_cuda = pytest.mark.skipif(
not torch.cuda.is_available(), reason="fused qk-norm+RoPE kernel needs CUDA"
)
def _init_kwargs(config: MiniMaxH3VideoVAEConfig):
with mock.patch.object(
@@ -97,6 +114,77 @@ def test_vit_qk_norm_supports_affine_free_rmsnorm():
assert output.shape == hidden_states.shape
@requires_cuda
def test_vit_fast_path_is_gated_and_matches_reference():
"""Gate closed: bit-identical to the original forward. Gate open: the fused
qk-norm+RoPE kernel and cuDNN SDPA run and match to rounding level."""
device = torch.device("cuda")
dtype = torch.float16
heads, dim_head, rope_dim = 4, 64, 48
attention = Attention(
heads=heads, dim_head=dim_head, qk_norm_type="rms_norm", eps=1e-5
).to(device=device, dtype=dtype)
assert _attn_fast_compatible(attention)
pos_embed = RotaryEmbeddingND(rope_dim, 100.0, n_dim=3, use_angle=True).to(device)
ids = create_token_ids((2, 4, 4), device, dtype)
ids = torch.cat([ids, torch.zeros((1, 5, 3), device=device, dtype=dtype)], 1)
rotary = prepare_rotary_pos_emb(pos_embed(ids), dtype=dtype)
generator = torch.Generator(device="cpu").manual_seed(0)
hidden = torch.randn((1, ids.shape[1], heads * dim_head), generator=generator)
hidden = hidden.to(device=device, dtype=dtype)
with (
torch.no_grad(),
torch.autocast("cuda", dtype=dtype),
set_forward_context(current_timestep=0, attn_metadata=None),
):
reference = attention(hidden, rotary)
gate = VaeFastPathGate()
_install_fast_attention([attention], gate)
assert torch.equal(attention(hidden, rotary), reference)
gate.enabled = True
fused = attention(hidden, rotary)
assert attention._sgl_unit_weight is not None, "fused qk-norm+RoPE did not run"
assert attention._sgl_cudnn_failed is False, "cuDNN SDPA fell back"
torch.testing.assert_close(fused, reference, atol=2e-2, rtol=1e-2)
@requires_cuda
def test_vit_rope_batched_tiles_match_per_tile():
"""Stacked decoder tiles share one RoPE row; the fused (quality) and native
(exact) RoPE kernels over the flattened batch are bit-identical to one
tile at a time."""
device, dtype = torch.device("cuda"), torch.float16
tiles, heads, dim_head, rope_dim = 3, 4, 64, 48
pos_embed = RotaryEmbeddingND(rope_dim, 100.0, n_dim=3, use_angle=True).to(device)
ids = create_token_ids((2, 4, 4), device, dtype)
rotary = prepare_rotary_pos_emb(pos_embed(ids), dtype=dtype)
batched_rotary = prepare_rotary_pos_emb(pos_embed(ids), dtype=dtype, batch=tiles)
generator = torch.Generator(device="cpu").manual_seed(0)
qkv = torch.randn(
(tiles, ids.shape[1], heads, 3 * dim_head), generator=generator
).to(device=device, dtype=dtype)
attn = SimpleNamespace(
dim_head=dim_head, _sgl_unit_weight=None, norm_q=SimpleNamespace(eps=1e-5)
)
per_tile = qkv.clone()
for tile in range(tiles):
query, key, _ = per_tile[tile : tile + 1].chunk(3, dim=-1)
assert _fused_qknorm_rope(attn, query, key, rotary)
query, key, _ = qkv.chunk(3, dim=-1)
assert _fused_qknorm_rope(attn, query, key, batched_rotary)
assert torch.equal(qkv, per_tile)
query, key, _ = qkv.chunk(3, dim=-1)
batched = apply_rotary_pos_emb_qk(query, key, batched_rotary)
for tile in range(tiles):
query, key, _ = qkv[tile : tile + 1].chunk(3, dim=-1)
for got, want in zip(batched, apply_rotary_pos_emb_qk(query, key, rotary)):
assert torch.equal(got[tile : tile + 1], want)
def test_audio_vae_attention_defaults_to_local_sdpa_and_allows_fa():
class RecordingFA(nn.Module):
backend = AttentionBackendEnum.FA
@@ -1,3 +1,5 @@
import shutil
import numpy as np
import pytest
import torch
@@ -100,15 +102,15 @@ def test_video_with_audio_uses_single_pass_encoder(tmp_path, monkeypatch):
def write(*_args, **_kwargs):
pass
def mimsave_spy(path, frames, **kwargs):
def encode_spy(path, frames, **kwargs):
calls.append((path, frames, kwargs))
assert kwargs["audio_path"].endswith(".wav")
assert kwargs["audio_codec"] == "aac"
assert kwargs["quality"] == 5
def fail_legacy_mux(**_kwargs):
raise AssertionError("the two-pass mux path should not run")
monkeypatch.setattr(output_utils.imageio, "mimsave", mimsave_spy)
monkeypatch.setattr(output_utils, "_save_video_ffmpeg", encode_spy)
monkeypatch.setattr(output_utils, "scipy_wavfile", FakeWavFile)
monkeypatch.setattr(output_utils, "_maybe_mux_audio_into_mp4", fail_legacy_mux)
@@ -144,13 +146,13 @@ def test_x264_preset_reaches_every_video_save_path(tmp_path, monkeypatch):
presets.append(("direct", kwargs["x264_preset"]))
return False
def mimsave_spy(_path, _frames, **kwargs):
encoder = "single-pass" if "audio_path" in kwargs else "imageio"
presets.append((encoder, kwargs["output_params"][-1]))
def encode_spy(_path, _frames, **kwargs):
encoder = "single-pass" if "audio_path" in kwargs else "ffmpeg"
presets.append((encoder, kwargs["x264_preset"]))
monkeypatch.setattr(output_utils, "_try_save_cuda_videos_direct", parallel_save)
monkeypatch.setattr(output_utils, "_try_save_cuda_video_direct", direct_save)
monkeypatch.setattr(output_utils.imageio, "mimsave", mimsave_spy)
monkeypatch.setattr(output_utils, "_save_video_ffmpeg", encode_spy)
monkeypatch.setattr(output_utils, "scipy_wavfile", FakeWavFile)
output_utils.save_outputs(
@@ -177,7 +179,7 @@ def test_x264_preset_reaches_every_video_save_path(tmp_path, monkeypatch):
assert {encoder for encoder, _ in presets} == {
"parallel",
"direct",
"imageio",
"ffmpeg",
"single-pass",
}
assert {preset for _, preset in presets} == {"ultrafast"}
@@ -193,12 +195,12 @@ def test_video_audio_single_pass_failure_falls_back(tmp_path, monkeypatch):
def write(*_args, **_kwargs):
pass
def mimsave_spy(path, frames, **kwargs):
def encode_spy(path, frames, **kwargs):
calls.append((path, frames, kwargs))
if "audio_path" in kwargs:
raise RuntimeError("unsupported audio input")
monkeypatch.setattr(output_utils.imageio, "mimsave", mimsave_spy)
monkeypatch.setattr(output_utils, "_save_video_ffmpeg", encode_spy)
monkeypatch.setattr(output_utils, "scipy_wavfile", FakeWavFile)
monkeypatch.setattr(
output_utils,
@@ -316,3 +318,11 @@ def test_multiple_videos_use_parallel_direct_save_with_serial_fallback(
assert kwargs["fps"] == 24
assert len(serial_calls) == 1
assert serial_calls[0]["save_file_path"] == paths[1]
def test_video_output_preserves_pinned_encoder(monkeypatch):
monkeypatch.setattr(shutil, "which", lambda _: "/system/ffmpeg")
monkeypatch.setattr(
output_utils.imageio_ffmpeg, "get_ffmpeg_exe", lambda: "/pinned/ffmpeg"
)
assert output_utils._resolve_ffmpeg_exe() == "/pinned/ffmpeg"
@@ -7,6 +7,7 @@ catches the failure this guards against -- the preset never being passed, and
ffmpeg silently applying its own default.
"""
import json
import shutil
import subprocess
@@ -14,11 +15,17 @@ import numpy as np
import pytest
import torch
import sglang.multimodal_gen.runtime.entrypoints.utils as output_utils
from sglang.multimodal_gen.configs.sample.sampling_params import (
DataType,
SamplingParams,
)
from sglang.multimodal_gen.runtime.entrypoints.utils import X264_PRESET, save_outputs
from sglang.multimodal_gen.runtime.entrypoints.utils import (
X264_PRESET,
MaterializedOutput,
save_materialized_output,
save_outputs,
)
FPS = 8
FRAMES = 8
@@ -97,7 +104,7 @@ def test_saved_video_carries_the_configured_preset(tmp_path):
@pytest.mark.parametrize(
"device",
[
# a host tensor saves through imageio, a CUDA one through the ffmpeg pipe
# host and CUDA tensors use the same system ffmpeg encoder
"cpu",
pytest.param(
"cuda",
@@ -166,3 +173,75 @@ def test_saved_video_decodes_to_every_frame(tmp_path):
codec, count = probe.split(",")
assert codec == "h264"
assert int(count) == FRAMES
@pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="needs ffmpeg")
@pytest.mark.parametrize("with_audio", [False, True])
@pytest.mark.parametrize("compression", [10, 100])
def test_materialized_video_uses_ffmpeg(tmp_path, monkeypatch, with_audio, compression):
def reject_imageio(*args, **kwargs):
raise AssertionError("video output must use ffmpeg directly")
monkeypatch.setattr(output_utils.imageio, "mimsave", reject_imageio)
rng = np.random.default_rng(3)
# non-contiguous RGB and non-macroblock dimensions exercise the raw pipe and scaling
frames = rng.integers(0, 256, (FRAMES, 30, 34, 3), dtype=np.uint8)[:, :, ::-1]
saved = tmp_path / "materialized.mp4"
save_materialized_output(
MaterializedOutput(
sample=None,
frames=list(frames),
audio=np.zeros((32000, 2), dtype=np.float32) if with_audio else None,
fps=FPS,
),
DataType.VIDEO,
str(saved),
audio_sample_rate=32000,
output_compression=compression,
x264_preset="ultrafast",
)
streams = json.loads(
subprocess.run(
[
"ffprobe",
"-v",
"error",
"-count_frames",
"-show_streams",
"-of",
"json",
str(saved),
],
capture_output=True,
text=True,
check=True,
).stdout
)["streams"]
video = next(stream for stream in streams if stream["codec_type"] == "video")
assert (video["width"], video["height"]) == (48, 32)
assert video["codec_name"] == "h264"
assert int(video["nb_read_frames"]) == FRAMES
assert video["r_frame_rate"] == f"{FPS}/1"
options = _x264_options(saved)
if compression == 100:
assert options["qp"] == "0"
else:
assert options["crf"] == f"{int((1 - compression / 100) * 51):.1f}"
audio = [stream for stream in streams if stream["codec_type"] == "audio"]
assert bool(audio) == with_audio
if with_audio:
assert audio[0]["codec_name"] == "aac"
assert audio[0]["sample_rate"] == "32000"
assert audio[0]["channels"] == 2
@pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="needs ffmpeg")
def test_ffmpeg_failure_is_reported(tmp_path):
with pytest.raises(RuntimeError, match="ffmpeg video encoding failed"):
output_utils._save_video_ffmpeg(
str(tmp_path / "invalid.mp4"),
[np.zeros((16, 16, 3), dtype=np.uint8)],
fps=FPS,
quality=5,
x264_preset="invalid",
)
@@ -19,6 +19,10 @@ from sglang.multimodal_gen.runtime.layers.attention.backends.video_sparse_attn_h
VideoSparseAttentionH3Impl,
VideoSparseAttentionH3MetadataBuilder,
_topk_tile_lists,
vsa_h3_fold_gate,
)
from sglang.multimodal_gen.runtime.layers.attention.backends.vsa_h3_kernels import (
vsa_h3_topk_lists,
)
requires_cuda = pytest.mark.skipif(
@@ -163,6 +167,29 @@ def test_topk_tile_list_semantics() -> None:
assert (_lists_to_mask(compete, n_tiles).sum(dim=-1) == keep + num_prefix).all()
@requires_cuda
@pytest.mark.parametrize(
"num_prefix, num_video, sparsity",
[(29, 528, 0.8), (0, 64, 0.9), (30, 1541, 0.05)],
)
def test_topk_list_kernel_matches_sorted_topk(num_prefix, num_video, sparsity) -> None:
"""The unsorted-top-k list kernel must emit the sorted lists and leave its
scratch mask zeroed for the next layer."""
n_tiles = num_prefix + num_video
scores = torch.randn(HEADS, n_tiles, n_tiles, device="cuda")
reference = _topk_tile_lists(scores, num_prefix, num_video, sparsity, True)
keep = reference.shape[-1] - num_prefix
out = torch.full((HEADS, num_video, n_tiles), -1, dtype=torch.int32, device="cuda")
out[..., :num_prefix] = torch.arange(num_prefix, dtype=torch.int32, device="cuda")
mask = torch.zeros(HEADS, num_video, num_video, dtype=torch.int8, device="cuda")
for _ in range(2): # the scratch mask must come back zeroed
picked = scores[:, num_prefix:, num_prefix:].topk(keep, sorted=False).indices
vsa_h3_topk_lists(picked, mask, out, num_prefix)
assert torch.equal(out[..., : num_prefix + keep], reference)
assert not mask.any()
assert (out[..., num_prefix + keep :] == -1).all()
def _masked_dense_reference(meta, used, q, k, v, gate, sparsity):
"""fp32 reference over the padded tile layout with the top-k tile mask."""
n_tiles = meta.num_tiles
@@ -215,6 +242,68 @@ def test_sparse_gated_matches_masked_dense_reference() -> None:
assert torch.all(out[used:] == 0)
@requires_cuda
def test_return_compress_fold_on_row_shards_is_bit_identical() -> None:
"""The deferred row-shard fold (Ulysses) must reproduce the in-kernel fold
bit for bit: same tile index derivation, same fp32 FMA order."""
device = torch.device("cuda")
meta = _build_metadata(0.5, device)
used, total, (q, k, v) = _packed_qkv(device)
gate = (torch.randn_like(q) * 0.1).to(torch.bfloat16)
impl = _impl()
folded = _run(impl, meta, used, total, q, k, v, gate=gate)
cu = torch.tensor([0, used, total], dtype=torch.int32, device=device)
out, out_compress = impl.forward_varlen(
q,
k,
v,
cu_seqlens=cu,
max_seqlen=used,
cu_seqlens_host=(0, used, total),
attn_metadata=meta,
return_compress=True,
)
shard = total // 2
for row_start in (0, shard):
rows = slice(row_start, row_start + shard)
vsa_h3_fold_gate(out[rows], gate[rows], out_compress, meta, row_start)
assert torch.equal(out, folded)
@requires_cuda
@pytest.mark.parametrize("sparsity", [0.0, 0.8])
def test_odd_tile_count_matches_reference(sparsity: float) -> None:
"""An odd tile count (padded to even for the SM100 kernel) must keep the
pooled means, the top-k lists and the gate fold on the real tiles."""
device = torch.device("cuda")
prefix = (70, 0, 40)
meta = VideoSparseAttentionH3MetadataBuilder().build(
current_timestep=0,
raw_latent_shape=VIDEO_SHAPE,
patch_size=(1, 1, 1),
VSA_sparsity=sparsity,
prefix_segments=prefix,
device=device,
)
assert meta.num_tiles % 2 == 1
used = sum(prefix) + math.prod(VIDEO_SHAPE)
total = (used + 63) // 64 * 64
generator = torch.Generator(device="cpu").manual_seed(11)
q, k, v, gate = (
torch.randn((total, HEADS, HEAD_DIM), generator=generator).to(
device=device, dtype=torch.bfloat16
)
for _ in range(4)
)
gate = gate * 0.1
out = _run(_impl(), meta, used, total, q, k, v, gate=gate)
reference = _masked_dense_reference(meta, used, q, k, v, gate, sparsity)
diff = (out[:used].float() - reference).abs().max().item()
assert diff < 2e-2, f"odd tile count, sparsity {sparsity}: max diff {diff}"
assert torch.all(out[used:] == 0)
def test_metadata_tile_geometry_accounts_every_row() -> None:
device = torch.device("cpu")
meta = _build_metadata(0.9, device)