[LoRA] Reorganize kernels and add CODEOWNERS (#42299)

This commit is contained in:
Yanbin Jiang
2026-10-03 13:43:40 -07:00
committed by GitHub
parent fd5e68f99e
commit 749a89abd6
83 changed files with 179 additions and 157 deletions
+9 -2
View File
@@ -26,13 +26,20 @@ sglang/kernels/
Operator groups (all populated): `activation`, `attention`, `communication`,
`diffusion`, `elementwise`, `embeddings`, `gemm`, `grammar`, `kv_canary`,
`kvcache`, `layernorm`, `lplb`, `mamba`, `memory`, `mm`, `moe`, `quantization`,
`sampling`, `speculative`.
`kvcache`, `layernorm`, `lora`, `lplb`, `mamba`, `memory`, `mm`, `moe`,
`quantization`, `sampling`, `speculative`.
Place model-specific implementations and tuning data inside the corresponding
logical operator group, rather than creating a model-specific top-level group
under `ops/`.
`lora` is its own group because it has a distinct responsibility: the adapter
kernels that only the LoRA runtime in `srt/lora` calls (`dense/` SGMV GEMMs and
their tuning data, `moe/` fused MoE-LoRA). It also keeps the experimental
TRT-LLM LoRA path whole as `dense/trtllm_lora_temp/` and
`moe/trtllm_lora_temp/`; the latter includes the FlashInfer overlay and the
flag-gated routing kernels that `srt/layers/moe/topk.py` reaches.
As of the RFC #29630 finale (#32072) the legacy `sglang.jit_kernel` package has
been **removed**: its shared build/runtime infra moved to `sglang.kernels.jit`
and each JIT-backed operator into its group as
@@ -58,7 +58,7 @@ __device__ float convert_to_float(T x) {
}
}
// Reference pack (bit-identical to kernels/ops/moe/trtllm_lora_temp/topk_pack.py):
// Reference pack (bit-identical to kernels/ops/lora/moe/trtllm_lora_temp/topk_pack.py):
// low 16 bits = bf16(weight) bits (round-to-nearest-even, same as torch/triton
// `.to(bfloat16)`), high 16 bits = int16 expert id.
__device__ __forceinline__ int32_t pack_routed(int32_t id, float w) {
+1
View File
@@ -25,6 +25,7 @@ _GROUPS = (
"grammar",
"kvcache",
"layernorm",
"lora",
"mamba",
"memory",
"mm",
@@ -397,33 +397,6 @@ __all__ = [
]
# LoRA SGMV Triton kernels migrated into this group (from lora/triton_ops);
# registered for inventory. Import them from their modules.
_TRITON_KERNELS = [
("chunked_embedding_lora_a", "chunked_embedding_lora_a_forward"),
("chunked_sgmv_expand", "chunked_sgmv_lora_expand_forward"),
("chunked_sgmv_shrink", "chunked_sgmv_lora_shrink_forward"),
("embedding_lora_a", "embedding_lora_a_fwd"),
("gate_up_lora_b", "gate_up_lora_b_fwd"),
("qkv_lora_b", "qkv_lora_b_fwd"),
("sgemm_lora_a", "sgemm_lora_a_fwd"),
("sgemm_lora_b", "sgemm_lora_b_fwd"),
("kv_b_lora_absorbed", "step_a_q_fwd"),
("kv_b_lora_absorbed", "step_b_q_fwd"),
("kv_b_lora_absorbed", "step_a_v_fwd"),
("kv_b_lora_absorbed", "step_b_v_fwd"),
]
for _mod, _fn in _TRITON_KERNELS:
register_kernel(
KernelSpec(
op=f"gemm.{_fn}",
backend=KernelBackend.TRITON,
target=f"sglang.kernels.ops.gemm.{_mod}:{_fn}",
)
)
del _mod, _fn
# Public entry points inventoried by logical operator group (RFC #29630).
register_kernel(
KernelSpec(
@@ -0,0 +1,34 @@
"""LoRA adapter kernels that only the LoRA runtime calls: ``dense/`` SGMV GEMMs,
``moe/`` fused MoE-LoRA, and the experimental TRT-LLM LoRA path."""
from sglang.kernels.registry import register_kernel
from sglang.kernels.spec import KernelBackend, KernelSpec
# Triton kernels registered for inventory. Import them from their modules.
_TRITON_KERNELS = [
("dense.chunked_embedding_lora_a", "chunked_embedding_lora_a_forward"),
("dense.chunked_sgmv_expand", "chunked_sgmv_lora_expand_forward"),
("dense.chunked_sgmv_shrink", "chunked_sgmv_lora_shrink_forward"),
("dense.embedding_lora_a", "embedding_lora_a_fwd"),
("dense.gate_up_lora_b", "gate_up_lora_b_fwd"),
("dense.qkv_lora_b", "qkv_lora_b_fwd"),
("dense.sgemm_lora_a", "sgemm_lora_a_fwd"),
("dense.sgemm_lora_b", "sgemm_lora_b_fwd"),
("dense.kv_b_lora_absorbed", "step_a_q_fwd"),
("dense.kv_b_lora_absorbed", "step_b_q_fwd"),
("dense.kv_b_lora_absorbed", "step_a_v_fwd"),
("dense.kv_b_lora_absorbed", "step_b_v_fwd"),
("moe.fused_moe_lora_kernel", "fused_moe_lora"),
("moe.virtual_experts", "merged_experts_fused_moe_lora_add"),
]
for _mod, _fn in _TRITON_KERNELS:
register_kernel(
KernelSpec(
op=f"lora.{_fn}",
backend=KernelBackend.TRITON,
target=f"sglang.kernels.ops.lora.{_mod}:{_fn}",
)
)
del _mod, _fn
__all__: list[str] = []
@@ -0,0 +1,2 @@
"""Triton LoRA GEMM kernels (SGMV, chunked SGMV, absorbed-MLA correction) and the
CSGMV tuning-config loader with its ``csgmv_configs/`` data."""
@@ -4,7 +4,7 @@ import torch
import triton
import triton.language as tl
from sglang.kernels.ops.gemm.lora_tuning_config import get_lora_expand_config
from sglang.kernels.ops.lora.dense.lora_tuning_config import get_lora_expand_config
from sglang.srt.lora.utils import LoRABatchInfo
from sglang.srt.utils import cached_triton_kernel
@@ -2,7 +2,7 @@ import torch
import triton
import triton.language as tl
from sglang.kernels.ops.gemm.lora_tuning_config import get_lora_shrink_config
from sglang.kernels.ops.lora.dense.lora_tuning_config import get_lora_shrink_config
from sglang.srt.lora.utils import LoRABatchInfo
from sglang.srt.utils import cached_triton_kernel
@@ -2,7 +2,7 @@ import torch
import triton
import triton.language as tl
from sglang.kernels.ops.gemm.kernel_utils import _resolve_token_positions
from sglang.kernels.ops.lora.dense.kernel_utils import _resolve_token_positions
from sglang.srt.lora.utils import LoRABatchInfo
@@ -48,7 +48,7 @@ import torch
import triton
import triton.language as tl
from sglang.kernels.ops.gemm.kernel_utils import _resolve_token_positions
from sglang.kernels.ops.lora.dense.kernel_utils import _resolve_token_positions
from sglang.srt.lora.utils import LoRABatchInfo
# ---------------------------------------------------------------------------
@@ -20,7 +20,7 @@ Usage:
python3 benchmark/kernels/lora_csgmv/tune_lora_csgmv.py \
--model Qwen/Qwen3-Embedding-0.6B --max-lora-rank 64
# Configs saved to python/sglang/kernels/ops/gemm/configs/
# Configs saved to python/sglang/kernels/ops/lora/dense/csgmv_configs/
# Server automatically picks them up:
python3 -m sglang.launch_server --model ... --enable-lora --lora-backend csgmv
@@ -2,7 +2,7 @@ import torch
import triton
import triton.language as tl
from sglang.kernels.ops.gemm.kernel_utils import _resolve_token_positions
from sglang.kernels.ops.lora.dense.kernel_utils import _resolve_token_positions
from sglang.srt.lora.utils import LoRABatchInfo
@@ -2,7 +2,7 @@ import torch
import triton
import triton.language as tl
from sglang.kernels.ops.gemm.kernel_utils import _resolve_token_positions
from sglang.kernels.ops.lora.dense.kernel_utils import _resolve_token_positions
from sglang.srt.lora.utils import LoRABatchInfo
@@ -2,7 +2,7 @@ import torch
import triton
import triton.language as tl
from sglang.kernels.ops.gemm.kernel_utils import _resolve_token_positions
from sglang.kernels.ops.lora.dense.kernel_utils import _resolve_token_positions
from sglang.srt.lora.utils import LoRABatchInfo
@@ -2,7 +2,7 @@ import torch
import triton
import triton.language as tl
from sglang.kernels.ops.gemm.trtllm_lora_temp.kernel_utils import (
from sglang.kernels.ops.lora.dense.trtllm_lora_temp.kernel_utils import (
_resolve_token_positions,
get_pdl_launch_metadata,
)
@@ -1,5 +1,5 @@
from sglang.kernels.jit.utils import is_arch_support_pdl
from sglang.kernels.ops.gemm.kernel_utils import (
from sglang.kernels.ops.lora.dense.kernel_utils import (
_resolve_token_positions as _resolve_token_positions,
)
@@ -48,7 +48,7 @@ import torch
import triton
import triton.language as tl
from sglang.kernels.ops.gemm.trtllm_lora_temp.kernel_utils import (
from sglang.kernels.ops.lora.dense.trtllm_lora_temp.kernel_utils import (
_resolve_token_positions,
get_pdl_launch_metadata,
)
@@ -4,7 +4,7 @@ import torch
import triton
import triton.language as tl
from sglang.kernels.ops.gemm.trtllm_lora_temp.kernel_utils import (
from sglang.kernels.ops.lora.dense.trtllm_lora_temp.kernel_utils import (
_resolve_token_positions,
get_pdl_launch_metadata,
)
@@ -4,7 +4,7 @@ import torch
import triton
import triton.language as tl
from sglang.kernels.ops.gemm.trtllm_lora_temp.kernel_utils import (
from sglang.kernels.ops.lora.dense.trtllm_lora_temp.kernel_utils import (
_resolve_token_positions,
get_pdl_launch_metadata,
)
@@ -2,10 +2,10 @@ import torch
import triton
import triton.language as tl
from sglang.kernels.ops.gemm.trtllm_lora_temp.gate_up_lora_b import (
from sglang.kernels.ops.lora.dense.trtllm_lora_temp.gate_up_lora_b import (
_CUBLAS_MIN_S_RANK,
)
from sglang.kernels.ops.gemm.trtllm_lora_temp.kernel_utils import (
from sglang.kernels.ops.lora.dense.trtllm_lora_temp.kernel_utils import (
_resolve_token_positions,
get_pdl_launch_metadata,
)
@@ -0,0 +1,2 @@
"""Fused MoE-LoRA kernels: the fused shrink/expand Triton kernel, LoRA-aware
block alignment (JIT), and the merged virtual-expert path."""
@@ -2,8 +2,7 @@
Migrated from ``sglang.srt.lora.trtllm_lora_temp.triton_ops`` (RFC #29630)."""
# --- merged from sglang.kernels.ops.moe.trtllm_lora_temp (RFC #29630 Phase 4) ---
from sglang.kernels.ops.moe.trtllm_lora_temp.core import (
from sglang.kernels.ops.lora.moe.trtllm_lora_temp.core import (
trtllm_bf16_routed_moe_lora,
trtllm_fp4_block_scale_moe_lora_finalize,
trtllm_fp4_block_scale_routed_moe_lora,
@@ -10,7 +10,7 @@ from sglang.srt.lora.trtllm_lora_temp.environ import lora_envs
def get_sgl_trtllm_moe_sm100_module():
import flashinfer.fused_moe.core as fi_core
from sglang.kernels.ops.moe.trtllm_lora_temp.jit import (
from sglang.kernels.ops.lora.moe.trtllm_lora_temp.jit import (
gen_sgl_trtllm_gen_fused_moe_sm100_module,
)
@@ -30,7 +30,7 @@ def get_sgl_trtllm_moe_sm100_module():
def get_sgl_trtllm_moe_sm100_raw_module():
from flashinfer.fused_moe.core import setup_cubin_loader
from sglang.kernels.ops.moe.trtllm_lora_temp.jit import (
from sglang.kernels.ops.lora.moe.trtllm_lora_temp.jit import (
gen_sgl_trtllm_gen_fused_moe_sm100_module,
)
@@ -14,7 +14,7 @@ if TYPE_CHECKING:
def _jit_kimi_k2_moe_fused_gate_module() -> Module:
return load_jit(
"kimi_k2_moe_fused_gate",
cuda_files=["trtllm_lora_temp/kimi_k2_moe_fused_gate.cuh"],
cuda_files=["lora/trtllm_lora_temp/kimi_k2_moe_fused_gate.cuh"],
cuda_wrappers=[
("kimi_k2_moe_fused_gate", "KimiK2MoEFusedGateKernel::run"),
],
@@ -16,7 +16,7 @@ def _jit_module(dtype: torch.dtype) -> Module:
return load_jit(
"moe_lora_merged_align",
*args,
cuda_files=["trtllm_lora_temp/moe_lora_merged_align_kernel.cu"],
cuda_files=["lora/trtllm_lora_temp/moe_lora_merged_align_kernel.cu"],
cuda_wrappers=[
("moe_lora_merged_align", f"MoeLoraMergedAlignKernel<{args}>::run"),
],
@@ -31,7 +31,7 @@ if TYPE_CHECKING:
def _jit_topk_softmax_pack_module() -> Module:
return load_jit(
"topk_softmax_pack",
cuda_files=["trtllm_lora_temp/topk_softmax_pack.cuh"],
cuda_files=["lora/trtllm_lora_temp/topk_softmax_pack.cuh"],
cuda_wrappers=[("topk_softmax_pack", "topk_softmax_pack")],
)
@@ -9,10 +9,10 @@ import torch
import triton
import triton.language as tl
from sglang.kernels.ops.gemm.trtllm_lora_temp.kernel_utils import (
from sglang.kernels.ops.lora.dense.trtllm_lora_temp.kernel_utils import (
get_pdl_launch_metadata,
)
from sglang.kernels.ops.moe.virtual_experts import _align_block_size_large
from sglang.kernels.ops.lora.moe.virtual_experts import _align_block_size_large
from sglang.srt.lora.trtllm_lora_temp.environ import lora_envs
@@ -394,7 +394,7 @@ def _get_moe_lora_shrink_split_k(
return max(1, min(triton.cdiv(target, base_grid), max_split_k, 8))
# Rank-specialized LoRA-B expand kernel lives in lora/trtllm_lora_temp/.
# Rank-specialized LoRA-B expand kernel lives in srt/lora/trtllm_lora_temp/.
# Re-export so existing call sites (and any external imports) keep working.
from sglang.srt.lora.trtllm_lora_temp.specialized_expand import ( # noqa: E402,F401
_invoke_moe_lora_expand_add,
@@ -574,7 +574,7 @@ def _merged_experts_fused_moe_lora_add_impl(
and bucket_experts + 1 <= 1024
and topk_ids.shape[0] < 512
):
from sglang.kernels.ops.moe.trtllm_lora_temp.moe_lora_merged_align import (
from sglang.kernels.ops.lora.moe.trtllm_lora_temp.moe_lora_merged_align import (
moe_lora_merged_align,
)
-17
View File
@@ -161,23 +161,6 @@ def situ_and_mul_masked_post_quant(
__all__ = ["situ_and_mul_masked_post_quant", "moe_align_block_size", "topk_softmax"]
# Fused MoE-LoRA Triton kernels migrated into this group (from lora/triton_ops);
# registered for inventory. Import them from their modules.
_TRITON_KERNELS = [
("fused_moe_lora_kernel", "fused_moe_lora"),
("virtual_experts", "merged_experts_fused_moe_lora_add"),
]
for _mod, _fn in _TRITON_KERNELS:
register_kernel(
KernelSpec(
op=f"moe.{_fn}",
backend=KernelBackend.TRITON,
target=f"sglang.kernels.ops.moe.{_mod}:{_fn}",
)
)
del _mod, _fn
# Triton kernels migrated from srt/layers/moe (RFC #29630, Phase 2.5);
# registered for inventory. Import them from their modules.
_PHASE25_TRITON_KERNELS = [
@@ -43,7 +43,7 @@ def _fused_unpack_packed_topk(packed: torch.Tensor):
Returns (topk_ids int32, topk_weights float32). Collapses the ~5 elementwise
ops (shift / mask / int16 / bitcast / cast) the torch reference emits per call
into one Triton launch. Mirrors _pack_topk_kernel (trtllm_lora_temp/topk_pack).
into one Triton launch. Mirrors _pack_topk_kernel (ops/lora/moe/trtllm_lora_temp/topk_pack).
"""
packed = packed.contiguous()
ids = torch.empty_like(packed, dtype=torch.int32)
@@ -77,7 +77,7 @@ def moe_align_block_size(
from sglang.srt.lora.trtllm_lora_temp.environ import lora_envs
if lora_envs.SGLANG_OPT_USE_JIT_KERNEL_MOE_ALIGN.get() and num_experts <= 8191:
from sglang.kernels.ops.moe.trtllm_lora_temp.virtual_experts import (
from sglang.kernels.ops.lora.moe.virtual_experts import (
_align_block_size_jit,
)
+2 -2
View File
@@ -1061,7 +1061,7 @@ def fused_topk(
elif packed_out is not None:
# Fused gating + routed pack (SGLANG_OPT_LORA_FUSED_TOPK_PACK): one JIT kernel
# writes topk_weights/topk_ids AND the FlashInfer packed topk in one launch.
from sglang.kernels.ops.moe.trtllm_lora_temp.topk_softmax_pack import (
from sglang.kernels.ops.lora.moe.trtllm_lora_temp.topk_softmax_pack import (
topk_softmax_pack,
)
@@ -1966,7 +1966,7 @@ def biased_grouped_topk_gpu(
and lora_envs.SGLANG_OPT_KIMI_GATE_BF16_INPUT.get()
)
if _use_jit_bf16_gate:
from sglang.kernels.ops.moe.trtllm_lora_temp.kimi_k2_moe_fused_gate import (
from sglang.kernels.ops.lora.moe.trtllm_lora_temp.kimi_k2_moe_fused_gate import (
kimi_k2_moe_fused_gate as _kimi_k2_moe_fused_gate,
)
@@ -3,11 +3,15 @@ from typing import List, Optional, Tuple
import torch
from sglang.kernels.ops.gemm.chunked_embedding_lora_a import (
from sglang.kernels.ops.lora.dense.chunked_embedding_lora_a import (
chunked_embedding_lora_a_forward,
)
from sglang.kernels.ops.gemm.chunked_sgmv_expand import chunked_sgmv_lora_expand_forward
from sglang.kernels.ops.gemm.chunked_sgmv_shrink import chunked_sgmv_lora_shrink_forward
from sglang.kernels.ops.lora.dense.chunked_sgmv_expand import (
chunked_sgmv_lora_expand_forward,
)
from sglang.kernels.ops.lora.dense.chunked_sgmv_shrink import (
chunked_sgmv_lora_shrink_forward,
)
from sglang.srt.lora.backend.base_backend import BaseLoRABackend
from sglang.srt.lora.utils import (
LoRABatchInfo,
@@ -3,11 +3,11 @@ from typing import List, Optional, Tuple
import torch
from sglang.kernels.ops.gemm.embedding_lora_a import embedding_lora_a_fwd
from sglang.kernels.ops.gemm.gate_up_lora_b import gate_up_lora_b_fwd
from sglang.kernels.ops.gemm.qkv_lora_b import qkv_lora_b_fwd
from sglang.kernels.ops.gemm.sgemm_lora_a import sgemm_lora_a_fwd
from sglang.kernels.ops.gemm.sgemm_lora_b import sgemm_lora_b_fwd
from sglang.kernels.ops.lora.dense.embedding_lora_a import embedding_lora_a_fwd
from sglang.kernels.ops.lora.dense.gate_up_lora_b import gate_up_lora_b_fwd
from sglang.kernels.ops.lora.dense.qkv_lora_b import qkv_lora_b_fwd
from sglang.kernels.ops.lora.dense.sgemm_lora_a import sgemm_lora_a_fwd
from sglang.kernels.ops.lora.dense.sgemm_lora_b import sgemm_lora_b_fwd
from sglang.srt.environ import envs
from sglang.srt.layers.dp_attention import (
DpPaddingMode,
@@ -19,7 +19,7 @@ from typing import TYPE_CHECKING, Optional, Tuple
import torch
from sglang.kernels.ops.gemm.kv_b_lora_absorbed import (
from sglang.kernels.ops.lora.dense.kv_b_lora_absorbed import (
step_a_q_fwd,
step_a_v_fwd,
step_b_q_fwd,
+9 -5
View File
@@ -37,7 +37,7 @@ _is_hip = is_hip()
_is_xpu = is_xpu()
if _is_cuda or _is_hip or _is_xpu:
from sglang.kernels.ops.moe.moe_lora_align import moe_lora_align_block_size
from sglang.kernels.ops.lora.moe.moe_lora_align import moe_lora_align_block_size
def _get_moe_lora_block_config(max_lora_rank: int) -> dict:
@@ -316,8 +316,10 @@ def _add_lora_gate_up_delta(
routing_cache: dict | None = None,
) -> None:
"""Add LoRA gate_up delta to intermediate_cache in-place."""
from sglang.kernels.ops.moe.fused_moe_lora_kernel import fused_moe_lora
from sglang.kernels.ops.moe.virtual_experts import merged_experts_fused_moe_lora_add
from sglang.kernels.ops.lora.moe.fused_moe_lora_kernel import fused_moe_lora
from sglang.kernels.ops.lora.moe.virtual_experts import (
merged_experts_fused_moe_lora_add,
)
if lora_info is None or lora_info.max_lora_rank == 0:
return
@@ -407,8 +409,10 @@ def _add_lora_down_delta(
routing_cache: dict | None = None,
) -> None:
"""Add LoRA down delta to intermediate_cache in-place."""
from sglang.kernels.ops.moe.fused_moe_lora_kernel import fused_moe_lora
from sglang.kernels.ops.moe.virtual_experts import merged_experts_fused_moe_lora_add
from sglang.kernels.ops.lora.moe.fused_moe_lora_kernel import fused_moe_lora
from sglang.kernels.ops.lora.moe.virtual_experts import (
merged_experts_fused_moe_lora_add,
)
if lora_info.max_lora_rank == 0:
return
@@ -24,13 +24,13 @@ _is_cuda = is_cuda()
if _is_cuda:
from sgl_kernel import silu_and_mul
from sglang.kernels.ops.lora.moe.virtual_experts import (
_align_block_size_jit as moe_align_block_size,
)
from sglang.kernels.ops.moe.fused_moe_triton_kernels import (
moe_sum_reduce_triton,
)
from sglang.kernels.ops.moe.moe_wna16_marlin import moe_wna16_marlin_gemm
from sglang.kernels.ops.moe.virtual_experts import (
_align_block_size_jit as moe_align_block_size,
)
from sglang.srt.layers.moe.fused_moe_triton.fused_marlin_moe import (
get_scalar_type,
)
@@ -208,7 +208,7 @@ def fused_experts_experimental_sgl_marlin_lora(
runner_config: MoeRunnerConfig,
lora_info,
) -> StandardCombineInput:
from sglang.kernels.ops.moe.trtllm_lora_temp.virtual_experts import (
from sglang.kernels.ops.lora.moe.trtllm_lora_temp.virtual_experts import (
merged_experts_fused_moe_lora_add,
)
from sglang.srt.layers.moe.token_dispatcher.standard import (
@@ -42,8 +42,10 @@ def qkv_proj_lora_forward(self, input_: torch.Tensor):
):
return get_original_qkv_forward()(self, input_)
from sglang.kernels.ops.gemm.trtllm_lora_temp.qkv_lora_b import qkv_lora_b_fwd
from sglang.kernels.ops.gemm.trtllm_lora_temp.sgemm_lora_a import sgemm_lora_a_fwd
from sglang.kernels.ops.lora.dense.trtllm_lora_temp.qkv_lora_b import qkv_lora_b_fwd
from sglang.kernels.ops.lora.dense.trtllm_lora_temp.sgemm_lora_a import (
sgemm_lora_a_fwd,
)
bias = self.base_layer.bias if not self.base_layer.skip_bias_add else None
side_stream = get_lora_side_stream()
@@ -121,10 +123,10 @@ def row_parallel_lora_forward(
_alloc = lora_overlap_alloc_stream() # capture MAIN stream here (before the fork)
side_stream.wait_stream(torch.cuda.current_stream())
with torch.cuda.stream(side_stream):
from sglang.kernels.ops.gemm.trtllm_lora_temp.sgemm_lora_a import (
from sglang.kernels.ops.lora.dense.trtllm_lora_temp.sgemm_lora_a import (
sgemm_lora_a_fwd,
)
from sglang.kernels.ops.gemm.trtllm_lora_temp.sgemm_lora_b import (
from sglang.kernels.ops.lora.dense.trtllm_lora_temp.sgemm_lora_b import (
sgemm_lora_b_fwd,
)
@@ -189,10 +191,10 @@ def column_parallel_lora_forward(self, input_: torch.Tensor):
_alloc = lora_overlap_alloc_stream() # capture MAIN stream here (before the fork)
side_stream.wait_stream(torch.cuda.current_stream())
with torch.cuda.stream(side_stream):
from sglang.kernels.ops.gemm.trtllm_lora_temp.sgemm_lora_a import (
from sglang.kernels.ops.lora.dense.trtllm_lora_temp.sgemm_lora_a import (
sgemm_lora_a_fwd,
)
from sglang.kernels.ops.gemm.trtllm_lora_temp.sgemm_lora_b import (
from sglang.kernels.ops.lora.dense.trtllm_lora_temp.sgemm_lora_b import (
sgemm_lora_b_fwd,
)
@@ -245,10 +247,10 @@ def replicated_lora_forward(self, x: torch.Tensor):
_alloc = lora_overlap_alloc_stream() # capture MAIN stream here (before the fork)
side_stream.wait_stream(torch.cuda.current_stream())
with torch.cuda.stream(side_stream):
from sglang.kernels.ops.gemm.trtllm_lora_temp.sgemm_lora_a import (
from sglang.kernels.ops.lora.dense.trtllm_lora_temp.sgemm_lora_a import (
sgemm_lora_a_fwd,
)
from sglang.kernels.ops.gemm.trtllm_lora_temp.sgemm_lora_b import (
from sglang.kernels.ops.lora.dense.trtllm_lora_temp.sgemm_lora_b import (
sgemm_lora_b_fwd,
)
@@ -267,7 +269,9 @@ def replicated_lora_forward(self, x: torch.Tensor):
if first_dim == 0:
output = sgemm_lora_b_fwd(lora_a_output, self.B_buffer, sgemm_info, output)
else:
from sglang.kernels.ops.gemm.trtllm_lora_temp.qkv_lora_b import qkv_lora_b_fwd
from sglang.kernels.ops.lora.dense.trtllm_lora_temp.qkv_lora_b import (
qkv_lora_b_fwd,
)
output = qkv_lora_b_fwd(
lora_a_output,
@@ -29,16 +29,16 @@ step_a_q_fwd = step_a_v_fwd = step_b_q_fwd = step_b_v_fwd = None
def _ensure_step_kernels() -> None:
global step_a_q_fwd, step_a_v_fwd, step_b_q_fwd, step_b_v_fwd
if step_a_q_fwd is None:
from sglang.kernels.ops.gemm.trtllm_lora_temp.kv_b_lora_absorbed import (
from sglang.kernels.ops.lora.dense.trtllm_lora_temp.kv_b_lora_absorbed import (
step_a_q_fwd as _aq,
)
from sglang.kernels.ops.gemm.trtllm_lora_temp.kv_b_lora_absorbed import (
from sglang.kernels.ops.lora.dense.trtllm_lora_temp.kv_b_lora_absorbed import (
step_a_v_fwd as _av,
)
from sglang.kernels.ops.gemm.trtllm_lora_temp.kv_b_lora_absorbed import (
from sglang.kernels.ops.lora.dense.trtllm_lora_temp.kv_b_lora_absorbed import (
step_b_q_fwd as _bq,
)
from sglang.kernels.ops.gemm.trtllm_lora_temp.kv_b_lora_absorbed import (
from sglang.kernels.ops.lora.dense.trtllm_lora_temp.kv_b_lora_absorbed import (
step_b_v_fwd as _bv,
)
@@ -108,7 +108,7 @@ class _LoraEnvs:
# diagnostics / tuning:
SGLANG_OPT_LORA_SHRINK_TUNE = _GatedBool("SGLANG_OPT_LORA_SHRINK_TUNE", False)
# kimi NVFP4 permute+quant fuse — read in kernels/ops/moe/trtllm_lora_temp/core.py (Python) to pass
# kimi NVFP4 permute+quant fuse — read in kernels/ops/lora/moe/trtllm_lora_temp/core.py (Python) to pass
# a bool to the kernel, AND C++-side via getenv in the launcher. Default off (kimi-only).
SGLANG_OPT_FUSED_PERMUTE_QUANT = _GatedBool("SGLANG_OPT_FUSED_PERMUTE_QUANT", False)
@@ -70,7 +70,7 @@ def sgl_trtllm_fp8_block_scale_moe_wrapper(
"TRTLLM enums and cubin-loader utilities."
) from e
from sglang.kernels.ops.moe.trtllm_lora_temp import trtllm_fp8_block_scale_moe
from sglang.kernels.ops.lora.moe.trtllm_lora_temp import trtllm_fp8_block_scale_moe
kwargs = {
"routing_logits": routing_logits,
@@ -168,7 +168,7 @@ def sgl_trtllm_fp8_block_scale_routed_moe_wrapper(
"TRTLLM enums and cubin-loader utilities."
) from e
from sglang.kernels.ops.moe.trtllm_lora_temp import (
from sglang.kernels.ops.lora.moe.trtllm_lora_temp import (
trtllm_fp8_block_scale_routed_moe,
)
@@ -41,10 +41,10 @@ def apply_multi_lora(
) -> torch.Tensor:
if getattr(layer.lora_backend, "name", None) != "triton" or not inputs.is_cuda:
raise RuntimeError("Multi-slot dense LoRA requires Triton on CUDA")
from sglang.kernels.ops.gemm.trtllm_lora_temp.sgemm_lora_a import (
from sglang.kernels.ops.lora.dense.trtllm_lora_temp.sgemm_lora_a import (
shared_sink_sgemm_lora_a_fwd,
)
from sglang.kernels.ops.gemm.trtllm_lora_temp.sgemm_lora_b import (
from sglang.kernels.ops.lora.dense.trtllm_lora_temp.sgemm_lora_b import (
shared_sink_sgemm_lora_b_fwd,
)
@@ -86,7 +86,7 @@ def _apply_per_expert_lora(
lora_a: torch.Tensor,
lora_b: torch.Tensor,
) -> None:
from sglang.kernels.ops.moe.virtual_experts import (
from sglang.kernels.ops.lora.moe.virtual_experts import (
merged_experts_fused_moe_lora_add,
)
@@ -156,7 +156,7 @@ def forward_with_lora(
if single_gate_up:
torch.mm(x_td, a_gate_up.T, out=gate_up_shrink)
else:
from sglang.kernels.ops.gemm.trtllm_lora_temp.sgemm_lora_a import (
from sglang.kernels.ops.lora.dense.trtllm_lora_temp.sgemm_lora_a import (
shared_sink_sgemm_lora_a_fwd,
)
@@ -190,7 +190,7 @@ def forward_with_lora(
y_flat.addmm_(gate_up_shrink, layer._w1_delta[0].T)
y = y_flat.view(t, n, -1)
else:
from sglang.kernels.ops.gemm.trtllm_lora_temp.sgemm_lora_b import (
from sglang.kernels.ops.lora.dense.trtllm_lora_temp.sgemm_lora_b import (
shared_sink_sgemm_lora_b_fwd,
)
@@ -47,12 +47,12 @@ def fused_experts_none_to_experimental_sgl_trtllm_fp8_lora(
) -> StandardCombineInput:
from flashinfer.fused_moe import Fp8QuantizationType
from sglang.kernels.ops.moe.trtllm_lora_temp import (
from sglang.kernels.ops.lora.moe.trtllm_lora_temp import (
trtllm_fp8_block_scale_moe_lora_finalize,
trtllm_fp8_block_scale_routed_moe_lora,
)
from sglang.kernels.ops.moe.trtllm_lora_temp.topk_pack import fused_pack_topk
from sglang.kernels.ops.moe.trtllm_lora_temp.virtual_experts import (
from sglang.kernels.ops.lora.moe.trtllm_lora_temp.topk_pack import fused_pack_topk
from sglang.kernels.ops.lora.moe.trtllm_lora_temp.virtual_experts import (
merged_experts_fused_moe_lora_add,
)
from sglang.srt.layers.moe.token_dispatcher.standard import StandardCombineInput
@@ -316,9 +316,9 @@ def fused_experts_none_to_experimental_sgl_trtllm_bf16_lora(
down grouped GEMM -> finalize, then the virtual-experts down-LoRA is merged into
the output. Single-stream version (no two-stream overlap yet — phase 2).
"""
from sglang.kernels.ops.moe.trtllm_lora_temp import trtllm_bf16_routed_moe_lora
from sglang.kernels.ops.moe.trtllm_lora_temp.topk_pack import fused_pack_topk
from sglang.kernels.ops.moe.trtllm_lora_temp.virtual_experts import (
from sglang.kernels.ops.lora.moe.trtllm_lora_temp import trtllm_bf16_routed_moe_lora
from sglang.kernels.ops.lora.moe.trtllm_lora_temp.topk_pack import fused_pack_topk
from sglang.kernels.ops.lora.moe.trtllm_lora_temp.virtual_experts import (
merged_experts_fused_moe_lora_add,
)
from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import (
@@ -474,11 +474,11 @@ def fused_experts_none_to_experimental_sgl_trtllm_fp4_lora(
then the virtual-experts down-LoRA is merged into the output. Single-stream
version; ``moe_overlap.py`` provides the two-stream variant.
"""
from sglang.kernels.ops.moe.trtllm_lora_temp import (
from sglang.kernels.ops.lora.moe.trtllm_lora_temp import (
trtllm_fp4_block_scale_routed_moe_lora,
)
from sglang.kernels.ops.moe.trtllm_lora_temp.topk_pack import fused_pack_topk
from sglang.kernels.ops.moe.trtllm_lora_temp.virtual_experts import (
from sglang.kernels.ops.lora.moe.trtllm_lora_temp.topk_pack import fused_pack_topk
from sglang.kernels.ops.lora.moe.trtllm_lora_temp.virtual_experts import (
merged_experts_fused_moe_lora_add,
)
from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import (
@@ -39,7 +39,7 @@ def _warm_sgl_trtllm_moe_module() -> None:
global _SGL_TRTLLM_MODULE_WARMED
if _SGL_TRTLLM_MODULE_WARMED:
return
from sglang.kernels.ops.moe.trtllm_lora_temp.core import (
from sglang.kernels.ops.lora.moe.trtllm_lora_temp.core import (
get_sgl_trtllm_moe_sm100_raw_module,
)
@@ -37,11 +37,13 @@ def merged_column_lora_forward(self, input_: torch.Tensor):
):
return get_original_merged_column_forward()(self, input_)
from sglang.kernels.ops.gemm.trtllm_lora_temp.gate_up_lora_b import (
from sglang.kernels.ops.lora.dense.trtllm_lora_temp.gate_up_lora_b import (
gate_up_lora_b_fwd,
)
from sglang.kernels.ops.gemm.trtllm_lora_temp.qkv_lora_b import qkv_lora_b_fwd
from sglang.kernels.ops.gemm.trtllm_lora_temp.sgemm_lora_a import sgemm_lora_a_fwd
from sglang.kernels.ops.lora.dense.trtllm_lora_temp.qkv_lora_b import qkv_lora_b_fwd
from sglang.kernels.ops.lora.dense.trtllm_lora_temp.sgemm_lora_a import (
sgemm_lora_a_fwd,
)
bias = self.base_layer.bias if not self.base_layer.skip_bias_add else None
side_stream = get_lora_side_stream()
@@ -54,11 +54,11 @@ def fused_experts_none_to_experimental_sgl_trtllm_fp8_lora_two_stream(
# ---- two-stream fast path ----
from flashinfer.fused_moe import Fp8QuantizationType
from sglang.kernels.ops.moe.trtllm_lora_temp import (
from sglang.kernels.ops.lora.moe.trtllm_lora_temp import (
trtllm_fp8_block_scale_routed_moe_lora,
)
from sglang.kernels.ops.moe.trtllm_lora_temp.topk_pack import fused_pack_topk
from sglang.kernels.ops.moe.trtllm_lora_temp.virtual_experts import (
from sglang.kernels.ops.lora.moe.trtllm_lora_temp.topk_pack import fused_pack_topk
from sglang.kernels.ops.lora.moe.trtllm_lora_temp.virtual_experts import (
merged_experts_fused_moe_lora_add,
)
from sglang.kernels.ops.quantization.fp8_kernel import per_token_group_quant_fp8
@@ -360,11 +360,11 @@ def fused_experts_none_to_experimental_sgl_trtllm_fp4_lora_two_stream(
)
# ---- two-stream fast path ----
from sglang.kernels.ops.moe.trtllm_lora_temp import (
from sglang.kernels.ops.lora.moe.trtllm_lora_temp import (
trtllm_fp4_block_scale_routed_moe_lora,
)
from sglang.kernels.ops.moe.trtllm_lora_temp.topk_pack import fused_pack_topk
from sglang.kernels.ops.moe.trtllm_lora_temp.virtual_experts import (
from sglang.kernels.ops.lora.moe.trtllm_lora_temp.topk_pack import fused_pack_topk
from sglang.kernels.ops.lora.moe.trtllm_lora_temp.virtual_experts import (
merged_experts_fused_moe_lora_add,
)
from sglang.srt.distributed.device_communicators.pynccl_allocator import (
@@ -610,9 +610,9 @@ def fused_experts_none_to_experimental_sgl_trtllm_bf16_lora_two_stream(
)
# ---- two-stream fast path ----
from sglang.kernels.ops.moe.trtllm_lora_temp import trtllm_bf16_routed_moe_lora
from sglang.kernels.ops.moe.trtllm_lora_temp.topk_pack import fused_pack_topk
from sglang.kernels.ops.moe.trtllm_lora_temp.virtual_experts import (
from sglang.kernels.ops.lora.moe.trtllm_lora_temp import trtllm_bf16_routed_moe_lora
from sglang.kernels.ops.lora.moe.trtllm_lora_temp.topk_pack import fused_pack_topk
from sglang.kernels.ops.lora.moe.trtllm_lora_temp.virtual_experts import (
merged_experts_fused_moe_lora_add,
)
from sglang.srt.distributed.device_communicators.pynccl_allocator import (
@@ -27,7 +27,7 @@ def fused_experts_fp8_sgl(
# <-> quantization import cycle at load time.
from flashinfer.fused_moe import Fp8QuantizationType
from sglang.kernels.ops.moe.trtllm_lora_temp.topk_pack import fused_pack_topk
from sglang.kernels.ops.lora.moe.trtllm_lora_temp.topk_pack import fused_pack_topk
from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import (
is_allocation_symmetric,
next_power_of_2,
@@ -5,7 +5,7 @@ It is rank-specialized: the ``R`` dimension (LoRA rank) is a Triton
``constexpr``, so each rank value used at runtime gets its own JIT-compiled
specialization.
Called from :mod:`sglang.kernels.ops.moe.virtual_experts` when
Called from :mod:`sglang.kernels.ops.lora.moe.trtllm_lora_temp.virtual_experts` when
``use_direct_expand_add=True``. Ranks above 64 are accumulated in multiple
rank tiles.
"""