diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index c3f3f8ccdf..1d98d046df 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -7,6 +7,8 @@ /python/sglang/kernels @DarkSharpness @HaiShaw @BBuf @celve @HydraQYH @yuan-luo /python/sglang/kernels/ops/diffusion @yingluosanqian @BBuf @mickqian @triple-mu @niehen6174 /python/sglang/kernels/ops/attention/fla @yizhang2077 @hebiao064 @yuan-luo +/python/sglang/kernels/ops/lora @Ying1123 @Fridge003 @lifuhuang @yushengsu-thu @jybsuper +/python/sglang/kernels/jit/csrc/lora @Ying1123 @Fridge003 @lifuhuang @yushengsu-thu @jybsuper /python/sglang/multimodal_gen @mickqian @ping1jing2 @HaiShaw @yichiche @AgainstEntropy @BBuf @triple-mu @kevin-mii @niehen6174 /python/sglang/multimodal_gen/runtime/cache @DefTruth /python/sglang/multimodal_gen/runtime/layers/quantization/ @HaiShaw @BowenBao @OrangeRedeng @TallMessiWu diff --git a/benchmark/kernels/lora_csgmv/tune_lora_csgmv.py b/benchmark/kernels/lora_csgmv/tune_lora_csgmv.py index 84a2d8f5a5..1bd4f08e61 100755 --- a/benchmark/kernels/lora_csgmv/tune_lora_csgmv.py +++ b/benchmark/kernels/lora_csgmv/tune_lora_csgmv.py @@ -58,9 +58,13 @@ from typing import Any, Dict, List, Optional import torch import triton -from sglang.kernels.ops.gemm.chunked_sgmv_expand import _chunked_lora_expand_kernel -from sglang.kernels.ops.gemm.chunked_sgmv_shrink import _chunked_lora_shrink_kernel -from sglang.kernels.ops.gemm.lora_tuning_config import ( +from sglang.kernels.ops.lora.dense.chunked_sgmv_expand import ( + _chunked_lora_expand_kernel, +) +from sglang.kernels.ops.lora.dense.chunked_sgmv_shrink import ( + _chunked_lora_shrink_kernel, +) +from sglang.kernels.ops.lora.dense.lora_tuning_config import ( DEFAULT_EXPAND_CONFIG, DEFAULT_SHRINK_CONFIG, get_lora_config_file_name, @@ -364,7 +368,8 @@ def save_config( "sglang", "kernels", "ops", - "gemm", + "lora", + "dense", "csgmv_configs", version_dir, ) diff --git a/python/sglang/kernels/README.md b/python/sglang/kernels/README.md index d4c538390c..8856617eaf 100644 --- a/python/sglang/kernels/README.md +++ b/python/sglang/kernels/README.md @@ -26,13 +26,20 @@ sglang/kernels/ Operator groups (all populated): `activation`, `attention`, `communication`, `diffusion`, `elementwise`, `embeddings`, `gemm`, `grammar`, `kv_canary`, -`kvcache`, `layernorm`, `lplb`, `mamba`, `memory`, `mm`, `moe`, `quantization`, -`sampling`, `speculative`. +`kvcache`, `layernorm`, `lora`, `lplb`, `mamba`, `memory`, `mm`, `moe`, +`quantization`, `sampling`, `speculative`. Place model-specific implementations and tuning data inside the corresponding logical operator group, rather than creating a model-specific top-level group under `ops/`. +`lora` is its own group because it has a distinct responsibility: the adapter +kernels that only the LoRA runtime in `srt/lora` calls (`dense/` SGMV GEMMs and +their tuning data, `moe/` fused MoE-LoRA). It also keeps the experimental +TRT-LLM LoRA path whole as `dense/trtllm_lora_temp/` and +`moe/trtllm_lora_temp/`; the latter includes the FlashInfer overlay and the +flag-gated routing kernels that `srt/layers/moe/topk.py` reaches. + As of the RFC #29630 finale (#32072) the legacy `sglang.jit_kernel` package has been **removed**: its shared build/runtime infra moved to `sglang.kernels.jit` and each JIT-backed operator into its group as diff --git a/python/sglang/kernels/jit/csrc/trtllm_lora_temp/kimi_k2_moe_fused_gate.cuh b/python/sglang/kernels/jit/csrc/lora/trtllm_lora_temp/kimi_k2_moe_fused_gate.cuh similarity index 100% rename from python/sglang/kernels/jit/csrc/trtllm_lora_temp/kimi_k2_moe_fused_gate.cuh rename to python/sglang/kernels/jit/csrc/lora/trtllm_lora_temp/kimi_k2_moe_fused_gate.cuh diff --git a/python/sglang/kernels/jit/csrc/trtllm_lora_temp/moe_lora_merged_align_kernel.cu b/python/sglang/kernels/jit/csrc/lora/trtllm_lora_temp/moe_lora_merged_align_kernel.cu similarity index 100% rename from python/sglang/kernels/jit/csrc/trtllm_lora_temp/moe_lora_merged_align_kernel.cu rename to python/sglang/kernels/jit/csrc/lora/trtllm_lora_temp/moe_lora_merged_align_kernel.cu diff --git a/python/sglang/kernels/jit/csrc/trtllm_lora_temp/topk_softmax_pack.cuh b/python/sglang/kernels/jit/csrc/lora/trtllm_lora_temp/topk_softmax_pack.cuh similarity index 99% rename from python/sglang/kernels/jit/csrc/trtllm_lora_temp/topk_softmax_pack.cuh rename to python/sglang/kernels/jit/csrc/lora/trtllm_lora_temp/topk_softmax_pack.cuh index 87d4956b6c..f44d0d0c8b 100644 --- a/python/sglang/kernels/jit/csrc/trtllm_lora_temp/topk_softmax_pack.cuh +++ b/python/sglang/kernels/jit/csrc/lora/trtllm_lora_temp/topk_softmax_pack.cuh @@ -58,7 +58,7 @@ __device__ float convert_to_float(T x) { } } -// Reference pack (bit-identical to kernels/ops/moe/trtllm_lora_temp/topk_pack.py): +// Reference pack (bit-identical to kernels/ops/lora/moe/trtllm_lora_temp/topk_pack.py): // low 16 bits = bf16(weight) bits (round-to-nearest-even, same as torch/triton // `.to(bfloat16)`), high 16 bits = int16 expert id. __device__ __forceinline__ int32_t pack_routed(int32_t id, float w) { diff --git a/python/sglang/kernels/ops/__init__.py b/python/sglang/kernels/ops/__init__.py index ee1c1b2a11..b27d804ab6 100644 --- a/python/sglang/kernels/ops/__init__.py +++ b/python/sglang/kernels/ops/__init__.py @@ -25,6 +25,7 @@ _GROUPS = ( "grammar", "kvcache", "layernorm", + "lora", "mamba", "memory", "mm", diff --git a/python/sglang/kernels/ops/gemm/__init__.py b/python/sglang/kernels/ops/gemm/__init__.py index 688873c55e..b7c826d8aa 100644 --- a/python/sglang/kernels/ops/gemm/__init__.py +++ b/python/sglang/kernels/ops/gemm/__init__.py @@ -397,33 +397,6 @@ __all__ = [ ] -# LoRA SGMV Triton kernels migrated into this group (from lora/triton_ops); -# registered for inventory. Import them from their modules. -_TRITON_KERNELS = [ - ("chunked_embedding_lora_a", "chunked_embedding_lora_a_forward"), - ("chunked_sgmv_expand", "chunked_sgmv_lora_expand_forward"), - ("chunked_sgmv_shrink", "chunked_sgmv_lora_shrink_forward"), - ("embedding_lora_a", "embedding_lora_a_fwd"), - ("gate_up_lora_b", "gate_up_lora_b_fwd"), - ("qkv_lora_b", "qkv_lora_b_fwd"), - ("sgemm_lora_a", "sgemm_lora_a_fwd"), - ("sgemm_lora_b", "sgemm_lora_b_fwd"), - ("kv_b_lora_absorbed", "step_a_q_fwd"), - ("kv_b_lora_absorbed", "step_b_q_fwd"), - ("kv_b_lora_absorbed", "step_a_v_fwd"), - ("kv_b_lora_absorbed", "step_b_v_fwd"), -] -for _mod, _fn in _TRITON_KERNELS: - register_kernel( - KernelSpec( - op=f"gemm.{_fn}", - backend=KernelBackend.TRITON, - target=f"sglang.kernels.ops.gemm.{_mod}:{_fn}", - ) - ) -del _mod, _fn - - # Public entry points inventoried by logical operator group (RFC #29630). register_kernel( KernelSpec( diff --git a/python/sglang/kernels/ops/lora/__init__.py b/python/sglang/kernels/ops/lora/__init__.py new file mode 100644 index 0000000000..a6b5a074bb --- /dev/null +++ b/python/sglang/kernels/ops/lora/__init__.py @@ -0,0 +1,34 @@ +"""LoRA adapter kernels that only the LoRA runtime calls: ``dense/`` SGMV GEMMs, +``moe/`` fused MoE-LoRA, and the experimental TRT-LLM LoRA path.""" + +from sglang.kernels.registry import register_kernel +from sglang.kernels.spec import KernelBackend, KernelSpec + +# Triton kernels registered for inventory. Import them from their modules. +_TRITON_KERNELS = [ + ("dense.chunked_embedding_lora_a", "chunked_embedding_lora_a_forward"), + ("dense.chunked_sgmv_expand", "chunked_sgmv_lora_expand_forward"), + ("dense.chunked_sgmv_shrink", "chunked_sgmv_lora_shrink_forward"), + ("dense.embedding_lora_a", "embedding_lora_a_fwd"), + ("dense.gate_up_lora_b", "gate_up_lora_b_fwd"), + ("dense.qkv_lora_b", "qkv_lora_b_fwd"), + ("dense.sgemm_lora_a", "sgemm_lora_a_fwd"), + ("dense.sgemm_lora_b", "sgemm_lora_b_fwd"), + ("dense.kv_b_lora_absorbed", "step_a_q_fwd"), + ("dense.kv_b_lora_absorbed", "step_b_q_fwd"), + ("dense.kv_b_lora_absorbed", "step_a_v_fwd"), + ("dense.kv_b_lora_absorbed", "step_b_v_fwd"), + ("moe.fused_moe_lora_kernel", "fused_moe_lora"), + ("moe.virtual_experts", "merged_experts_fused_moe_lora_add"), +] +for _mod, _fn in _TRITON_KERNELS: + register_kernel( + KernelSpec( + op=f"lora.{_fn}", + backend=KernelBackend.TRITON, + target=f"sglang.kernels.ops.lora.{_mod}:{_fn}", + ) + ) +del _mod, _fn + +__all__: list[str] = [] diff --git a/python/sglang/kernels/ops/lora/dense/__init__.py b/python/sglang/kernels/ops/lora/dense/__init__.py new file mode 100644 index 0000000000..c55c3ca9ce --- /dev/null +++ b/python/sglang/kernels/ops/lora/dense/__init__.py @@ -0,0 +1,2 @@ +"""Triton LoRA GEMM kernels (SGMV, chunked SGMV, absorbed-MLA correction) and the +CSGMV tuning-config loader with its ``csgmv_configs/`` data.""" diff --git a/python/sglang/kernels/ops/gemm/chunked_embedding_lora_a.py b/python/sglang/kernels/ops/lora/dense/chunked_embedding_lora_a.py similarity index 100% rename from python/sglang/kernels/ops/gemm/chunked_embedding_lora_a.py rename to python/sglang/kernels/ops/lora/dense/chunked_embedding_lora_a.py diff --git a/python/sglang/kernels/ops/gemm/chunked_sgmv_expand.py b/python/sglang/kernels/ops/lora/dense/chunked_sgmv_expand.py similarity index 98% rename from python/sglang/kernels/ops/gemm/chunked_sgmv_expand.py rename to python/sglang/kernels/ops/lora/dense/chunked_sgmv_expand.py index 673d340c67..8dd7430ec5 100644 --- a/python/sglang/kernels/ops/gemm/chunked_sgmv_expand.py +++ b/python/sglang/kernels/ops/lora/dense/chunked_sgmv_expand.py @@ -4,7 +4,7 @@ import torch import triton import triton.language as tl -from sglang.kernels.ops.gemm.lora_tuning_config import get_lora_expand_config +from sglang.kernels.ops.lora.dense.lora_tuning_config import get_lora_expand_config from sglang.srt.lora.utils import LoRABatchInfo from sglang.srt.utils import cached_triton_kernel diff --git a/python/sglang/kernels/ops/gemm/chunked_sgmv_shrink.py b/python/sglang/kernels/ops/lora/dense/chunked_sgmv_shrink.py similarity index 98% rename from python/sglang/kernels/ops/gemm/chunked_sgmv_shrink.py rename to python/sglang/kernels/ops/lora/dense/chunked_sgmv_shrink.py index 9d7a822bff..8502cfc8b0 100644 --- a/python/sglang/kernels/ops/gemm/chunked_sgmv_shrink.py +++ b/python/sglang/kernels/ops/lora/dense/chunked_sgmv_shrink.py @@ -2,7 +2,7 @@ import torch import triton import triton.language as tl -from sglang.kernels.ops.gemm.lora_tuning_config import get_lora_shrink_config +from sglang.kernels.ops.lora.dense.lora_tuning_config import get_lora_shrink_config from sglang.srt.lora.utils import LoRABatchInfo from sglang.srt.utils import cached_triton_kernel diff --git a/python/sglang/kernels/ops/gemm/csgmv_configs/triton_3_5_1/lora_expand,K=1024,R=64,S=1,device=NVIDIA_H200.json b/python/sglang/kernels/ops/lora/dense/csgmv_configs/triton_3_5_1/lora_expand,K=1024,R=64,S=1,device=NVIDIA_H200.json similarity index 100% rename from python/sglang/kernels/ops/gemm/csgmv_configs/triton_3_5_1/lora_expand,K=1024,R=64,S=1,device=NVIDIA_H200.json rename to python/sglang/kernels/ops/lora/dense/csgmv_configs/triton_3_5_1/lora_expand,K=1024,R=64,S=1,device=NVIDIA_H200.json diff --git a/python/sglang/kernels/ops/gemm/csgmv_configs/triton_3_5_1/lora_expand,K=4096,R=64,S=3,device=NVIDIA_H200.json b/python/sglang/kernels/ops/lora/dense/csgmv_configs/triton_3_5_1/lora_expand,K=4096,R=64,S=3,device=NVIDIA_H200.json similarity index 100% rename from python/sglang/kernels/ops/gemm/csgmv_configs/triton_3_5_1/lora_expand,K=4096,R=64,S=3,device=NVIDIA_H200.json rename to python/sglang/kernels/ops/lora/dense/csgmv_configs/triton_3_5_1/lora_expand,K=4096,R=64,S=3,device=NVIDIA_H200.json diff --git a/python/sglang/kernels/ops/gemm/csgmv_configs/triton_3_5_1/lora_expand,K=6144,R=64,S=2,device=NVIDIA_H200.json b/python/sglang/kernels/ops/lora/dense/csgmv_configs/triton_3_5_1/lora_expand,K=6144,R=64,S=2,device=NVIDIA_H200.json similarity index 100% rename from python/sglang/kernels/ops/gemm/csgmv_configs/triton_3_5_1/lora_expand,K=6144,R=64,S=2,device=NVIDIA_H200.json rename to python/sglang/kernels/ops/lora/dense/csgmv_configs/triton_3_5_1/lora_expand,K=6144,R=64,S=2,device=NVIDIA_H200.json diff --git a/python/sglang/kernels/ops/gemm/csgmv_configs/triton_3_5_1/lora_shrink,K=1024,R=64,S=2,device=NVIDIA_H200.json b/python/sglang/kernels/ops/lora/dense/csgmv_configs/triton_3_5_1/lora_shrink,K=1024,R=64,S=2,device=NVIDIA_H200.json similarity index 100% rename from python/sglang/kernels/ops/gemm/csgmv_configs/triton_3_5_1/lora_shrink,K=1024,R=64,S=2,device=NVIDIA_H200.json rename to python/sglang/kernels/ops/lora/dense/csgmv_configs/triton_3_5_1/lora_shrink,K=1024,R=64,S=2,device=NVIDIA_H200.json diff --git a/python/sglang/kernels/ops/gemm/csgmv_configs/triton_3_5_1/lora_shrink,K=1024,R=64,S=3,device=NVIDIA_H200.json b/python/sglang/kernels/ops/lora/dense/csgmv_configs/triton_3_5_1/lora_shrink,K=1024,R=64,S=3,device=NVIDIA_H200.json similarity index 100% rename from python/sglang/kernels/ops/gemm/csgmv_configs/triton_3_5_1/lora_shrink,K=1024,R=64,S=3,device=NVIDIA_H200.json rename to python/sglang/kernels/ops/lora/dense/csgmv_configs/triton_3_5_1/lora_shrink,K=1024,R=64,S=3,device=NVIDIA_H200.json diff --git a/python/sglang/kernels/ops/gemm/csgmv_configs/triton_3_5_1/lora_shrink,K=2048,R=64,S=1,device=NVIDIA_H200.json b/python/sglang/kernels/ops/lora/dense/csgmv_configs/triton_3_5_1/lora_shrink,K=2048,R=64,S=1,device=NVIDIA_H200.json similarity index 100% rename from python/sglang/kernels/ops/gemm/csgmv_configs/triton_3_5_1/lora_shrink,K=2048,R=64,S=1,device=NVIDIA_H200.json rename to python/sglang/kernels/ops/lora/dense/csgmv_configs/triton_3_5_1/lora_shrink,K=2048,R=64,S=1,device=NVIDIA_H200.json diff --git a/python/sglang/kernels/ops/gemm/csgmv_configs/triton_3_5_1/lora_shrink,K=3072,R=64,S=1,device=NVIDIA_H200.json b/python/sglang/kernels/ops/lora/dense/csgmv_configs/triton_3_5_1/lora_shrink,K=3072,R=64,S=1,device=NVIDIA_H200.json similarity index 100% rename from python/sglang/kernels/ops/gemm/csgmv_configs/triton_3_5_1/lora_shrink,K=3072,R=64,S=1,device=NVIDIA_H200.json rename to python/sglang/kernels/ops/lora/dense/csgmv_configs/triton_3_5_1/lora_shrink,K=3072,R=64,S=1,device=NVIDIA_H200.json diff --git a/python/sglang/kernels/ops/gemm/embedding_lora_a.py b/python/sglang/kernels/ops/lora/dense/embedding_lora_a.py similarity index 100% rename from python/sglang/kernels/ops/gemm/embedding_lora_a.py rename to python/sglang/kernels/ops/lora/dense/embedding_lora_a.py diff --git a/python/sglang/kernels/ops/gemm/gate_up_lora_b.py b/python/sglang/kernels/ops/lora/dense/gate_up_lora_b.py similarity index 98% rename from python/sglang/kernels/ops/gemm/gate_up_lora_b.py rename to python/sglang/kernels/ops/lora/dense/gate_up_lora_b.py index bc89fac403..4368656a1d 100644 --- a/python/sglang/kernels/ops/gemm/gate_up_lora_b.py +++ b/python/sglang/kernels/ops/lora/dense/gate_up_lora_b.py @@ -2,7 +2,7 @@ import torch import triton import triton.language as tl -from sglang.kernels.ops.gemm.kernel_utils import _resolve_token_positions +from sglang.kernels.ops.lora.dense.kernel_utils import _resolve_token_positions from sglang.srt.lora.utils import LoRABatchInfo diff --git a/python/sglang/kernels/ops/gemm/kernel_utils.py b/python/sglang/kernels/ops/lora/dense/kernel_utils.py similarity index 100% rename from python/sglang/kernels/ops/gemm/kernel_utils.py rename to python/sglang/kernels/ops/lora/dense/kernel_utils.py diff --git a/python/sglang/kernels/ops/gemm/kv_b_lora_absorbed.py b/python/sglang/kernels/ops/lora/dense/kv_b_lora_absorbed.py similarity index 99% rename from python/sglang/kernels/ops/gemm/kv_b_lora_absorbed.py rename to python/sglang/kernels/ops/lora/dense/kv_b_lora_absorbed.py index 99a7710813..fa63689157 100644 --- a/python/sglang/kernels/ops/gemm/kv_b_lora_absorbed.py +++ b/python/sglang/kernels/ops/lora/dense/kv_b_lora_absorbed.py @@ -48,7 +48,7 @@ import torch import triton import triton.language as tl -from sglang.kernels.ops.gemm.kernel_utils import _resolve_token_positions +from sglang.kernels.ops.lora.dense.kernel_utils import _resolve_token_positions from sglang.srt.lora.utils import LoRABatchInfo # --------------------------------------------------------------------------- diff --git a/python/sglang/kernels/ops/gemm/lora_tuning_config.py b/python/sglang/kernels/ops/lora/dense/lora_tuning_config.py similarity index 98% rename from python/sglang/kernels/ops/gemm/lora_tuning_config.py rename to python/sglang/kernels/ops/lora/dense/lora_tuning_config.py index d7335694f1..a8be93b718 100644 --- a/python/sglang/kernels/ops/gemm/lora_tuning_config.py +++ b/python/sglang/kernels/ops/lora/dense/lora_tuning_config.py @@ -20,7 +20,7 @@ Usage: python3 benchmark/kernels/lora_csgmv/tune_lora_csgmv.py \ --model Qwen/Qwen3-Embedding-0.6B --max-lora-rank 64 - # Configs saved to python/sglang/kernels/ops/gemm/configs/ + # Configs saved to python/sglang/kernels/ops/lora/dense/csgmv_configs/ # Server automatically picks them up: python3 -m sglang.launch_server --model ... --enable-lora --lora-backend csgmv diff --git a/python/sglang/kernels/ops/gemm/qkv_lora_b.py b/python/sglang/kernels/ops/lora/dense/qkv_lora_b.py similarity index 98% rename from python/sglang/kernels/ops/gemm/qkv_lora_b.py rename to python/sglang/kernels/ops/lora/dense/qkv_lora_b.py index 1b1cff2b96..10c473b052 100644 --- a/python/sglang/kernels/ops/gemm/qkv_lora_b.py +++ b/python/sglang/kernels/ops/lora/dense/qkv_lora_b.py @@ -2,7 +2,7 @@ import torch import triton import triton.language as tl -from sglang.kernels.ops.gemm.kernel_utils import _resolve_token_positions +from sglang.kernels.ops.lora.dense.kernel_utils import _resolve_token_positions from sglang.srt.lora.utils import LoRABatchInfo diff --git a/python/sglang/kernels/ops/gemm/sgemm_lora_a.py b/python/sglang/kernels/ops/lora/dense/sgemm_lora_a.py similarity index 98% rename from python/sglang/kernels/ops/gemm/sgemm_lora_a.py rename to python/sglang/kernels/ops/lora/dense/sgemm_lora_a.py index 549620e420..d8b24f27a8 100644 --- a/python/sglang/kernels/ops/gemm/sgemm_lora_a.py +++ b/python/sglang/kernels/ops/lora/dense/sgemm_lora_a.py @@ -2,7 +2,7 @@ import torch import triton import triton.language as tl -from sglang.kernels.ops.gemm.kernel_utils import _resolve_token_positions +from sglang.kernels.ops.lora.dense.kernel_utils import _resolve_token_positions from sglang.srt.lora.utils import LoRABatchInfo diff --git a/python/sglang/kernels/ops/gemm/sgemm_lora_b.py b/python/sglang/kernels/ops/lora/dense/sgemm_lora_b.py similarity index 98% rename from python/sglang/kernels/ops/gemm/sgemm_lora_b.py rename to python/sglang/kernels/ops/lora/dense/sgemm_lora_b.py index c1cfb2ea4b..b973306984 100644 --- a/python/sglang/kernels/ops/gemm/sgemm_lora_b.py +++ b/python/sglang/kernels/ops/lora/dense/sgemm_lora_b.py @@ -2,7 +2,7 @@ import torch import triton import triton.language as tl -from sglang.kernels.ops.gemm.kernel_utils import _resolve_token_positions +from sglang.kernels.ops.lora.dense.kernel_utils import _resolve_token_positions from sglang.srt.lora.utils import LoRABatchInfo diff --git a/python/sglang/kernels/ops/gemm/trtllm_lora_temp/__init__.py b/python/sglang/kernels/ops/lora/dense/trtllm_lora_temp/__init__.py similarity index 100% rename from python/sglang/kernels/ops/gemm/trtllm_lora_temp/__init__.py rename to python/sglang/kernels/ops/lora/dense/trtllm_lora_temp/__init__.py diff --git a/python/sglang/kernels/ops/gemm/trtllm_lora_temp/gate_up_lora_b.py b/python/sglang/kernels/ops/lora/dense/trtllm_lora_temp/gate_up_lora_b.py similarity index 99% rename from python/sglang/kernels/ops/gemm/trtllm_lora_temp/gate_up_lora_b.py rename to python/sglang/kernels/ops/lora/dense/trtllm_lora_temp/gate_up_lora_b.py index f9da4af0aa..9f4e0f88fa 100644 --- a/python/sglang/kernels/ops/gemm/trtllm_lora_temp/gate_up_lora_b.py +++ b/python/sglang/kernels/ops/lora/dense/trtllm_lora_temp/gate_up_lora_b.py @@ -2,7 +2,7 @@ import torch import triton import triton.language as tl -from sglang.kernels.ops.gemm.trtllm_lora_temp.kernel_utils import ( +from sglang.kernels.ops.lora.dense.trtllm_lora_temp.kernel_utils import ( _resolve_token_positions, get_pdl_launch_metadata, ) diff --git a/python/sglang/kernels/ops/gemm/trtllm_lora_temp/kernel_utils.py b/python/sglang/kernels/ops/lora/dense/trtllm_lora_temp/kernel_utils.py similarity index 90% rename from python/sglang/kernels/ops/gemm/trtllm_lora_temp/kernel_utils.py rename to python/sglang/kernels/ops/lora/dense/trtllm_lora_temp/kernel_utils.py index 2ef9513386..b0f0bef3af 100644 --- a/python/sglang/kernels/ops/gemm/trtllm_lora_temp/kernel_utils.py +++ b/python/sglang/kernels/ops/lora/dense/trtllm_lora_temp/kernel_utils.py @@ -1,5 +1,5 @@ from sglang.kernels.jit.utils import is_arch_support_pdl -from sglang.kernels.ops.gemm.kernel_utils import ( +from sglang.kernels.ops.lora.dense.kernel_utils import ( _resolve_token_positions as _resolve_token_positions, ) diff --git a/python/sglang/kernels/ops/gemm/trtllm_lora_temp/kv_b_lora_absorbed.py b/python/sglang/kernels/ops/lora/dense/trtllm_lora_temp/kv_b_lora_absorbed.py similarity index 99% rename from python/sglang/kernels/ops/gemm/trtllm_lora_temp/kv_b_lora_absorbed.py rename to python/sglang/kernels/ops/lora/dense/trtllm_lora_temp/kv_b_lora_absorbed.py index 4ee5ce7f48..923abc3d96 100644 --- a/python/sglang/kernels/ops/gemm/trtllm_lora_temp/kv_b_lora_absorbed.py +++ b/python/sglang/kernels/ops/lora/dense/trtllm_lora_temp/kv_b_lora_absorbed.py @@ -48,7 +48,7 @@ import torch import triton import triton.language as tl -from sglang.kernels.ops.gemm.trtllm_lora_temp.kernel_utils import ( +from sglang.kernels.ops.lora.dense.trtllm_lora_temp.kernel_utils import ( _resolve_token_positions, get_pdl_launch_metadata, ) diff --git a/python/sglang/kernels/ops/gemm/trtllm_lora_temp/qkv_lora_b.py b/python/sglang/kernels/ops/lora/dense/trtllm_lora_temp/qkv_lora_b.py similarity index 99% rename from python/sglang/kernels/ops/gemm/trtllm_lora_temp/qkv_lora_b.py rename to python/sglang/kernels/ops/lora/dense/trtllm_lora_temp/qkv_lora_b.py index 2667e03e82..57df5ef684 100644 --- a/python/sglang/kernels/ops/gemm/trtllm_lora_temp/qkv_lora_b.py +++ b/python/sglang/kernels/ops/lora/dense/trtllm_lora_temp/qkv_lora_b.py @@ -4,7 +4,7 @@ import torch import triton import triton.language as tl -from sglang.kernels.ops.gemm.trtllm_lora_temp.kernel_utils import ( +from sglang.kernels.ops.lora.dense.trtllm_lora_temp.kernel_utils import ( _resolve_token_positions, get_pdl_launch_metadata, ) diff --git a/python/sglang/kernels/ops/gemm/trtllm_lora_temp/sgemm_lora_a.py b/python/sglang/kernels/ops/lora/dense/trtllm_lora_temp/sgemm_lora_a.py similarity index 99% rename from python/sglang/kernels/ops/gemm/trtllm_lora_temp/sgemm_lora_a.py rename to python/sglang/kernels/ops/lora/dense/trtllm_lora_temp/sgemm_lora_a.py index 6e528bf886..330370097b 100644 --- a/python/sglang/kernels/ops/gemm/trtllm_lora_temp/sgemm_lora_a.py +++ b/python/sglang/kernels/ops/lora/dense/trtllm_lora_temp/sgemm_lora_a.py @@ -4,7 +4,7 @@ import torch import triton import triton.language as tl -from sglang.kernels.ops.gemm.trtllm_lora_temp.kernel_utils import ( +from sglang.kernels.ops.lora.dense.trtllm_lora_temp.kernel_utils import ( _resolve_token_positions, get_pdl_launch_metadata, ) diff --git a/python/sglang/kernels/ops/gemm/trtllm_lora_temp/sgemm_lora_b.py b/python/sglang/kernels/ops/lora/dense/trtllm_lora_temp/sgemm_lora_b.py similarity index 98% rename from python/sglang/kernels/ops/gemm/trtllm_lora_temp/sgemm_lora_b.py rename to python/sglang/kernels/ops/lora/dense/trtllm_lora_temp/sgemm_lora_b.py index 7d0f79dce3..c904212cef 100644 --- a/python/sglang/kernels/ops/gemm/trtllm_lora_temp/sgemm_lora_b.py +++ b/python/sglang/kernels/ops/lora/dense/trtllm_lora_temp/sgemm_lora_b.py @@ -2,10 +2,10 @@ import torch import triton import triton.language as tl -from sglang.kernels.ops.gemm.trtllm_lora_temp.gate_up_lora_b import ( +from sglang.kernels.ops.lora.dense.trtllm_lora_temp.gate_up_lora_b import ( _CUBLAS_MIN_S_RANK, ) -from sglang.kernels.ops.gemm.trtllm_lora_temp.kernel_utils import ( +from sglang.kernels.ops.lora.dense.trtllm_lora_temp.kernel_utils import ( _resolve_token_positions, get_pdl_launch_metadata, ) diff --git a/python/sglang/kernels/ops/lora/moe/__init__.py b/python/sglang/kernels/ops/lora/moe/__init__.py new file mode 100644 index 0000000000..ec88850bfb --- /dev/null +++ b/python/sglang/kernels/ops/lora/moe/__init__.py @@ -0,0 +1,2 @@ +"""Fused MoE-LoRA kernels: the fused shrink/expand Triton kernel, LoRA-aware +block alignment (JIT), and the merged virtual-expert path.""" diff --git a/python/sglang/kernels/ops/moe/fused_moe_lora_kernel.py b/python/sglang/kernels/ops/lora/moe/fused_moe_lora_kernel.py similarity index 100% rename from python/sglang/kernels/ops/moe/fused_moe_lora_kernel.py rename to python/sglang/kernels/ops/lora/moe/fused_moe_lora_kernel.py diff --git a/python/sglang/kernels/ops/moe/moe_lora_align.py b/python/sglang/kernels/ops/lora/moe/moe_lora_align.py similarity index 100% rename from python/sglang/kernels/ops/moe/moe_lora_align.py rename to python/sglang/kernels/ops/lora/moe/moe_lora_align.py diff --git a/python/sglang/kernels/ops/moe/trtllm_lora_temp/SOURCE.md b/python/sglang/kernels/ops/lora/moe/trtllm_lora_temp/SOURCE.md similarity index 100% rename from python/sglang/kernels/ops/moe/trtllm_lora_temp/SOURCE.md rename to python/sglang/kernels/ops/lora/moe/trtllm_lora_temp/SOURCE.md diff --git a/python/sglang/kernels/ops/moe/trtllm_lora_temp/__init__.py b/python/sglang/kernels/ops/lora/moe/trtllm_lora_temp/__init__.py similarity index 84% rename from python/sglang/kernels/ops/moe/trtllm_lora_temp/__init__.py rename to python/sglang/kernels/ops/lora/moe/trtllm_lora_temp/__init__.py index c4b98801c7..fff8a16858 100644 --- a/python/sglang/kernels/ops/moe/trtllm_lora_temp/__init__.py +++ b/python/sglang/kernels/ops/lora/moe/trtllm_lora_temp/__init__.py @@ -2,8 +2,7 @@ Migrated from ``sglang.srt.lora.trtllm_lora_temp.triton_ops`` (RFC #29630).""" -# --- merged from sglang.kernels.ops.moe.trtllm_lora_temp (RFC #29630 Phase 4) --- -from sglang.kernels.ops.moe.trtllm_lora_temp.core import ( +from sglang.kernels.ops.lora.moe.trtllm_lora_temp.core import ( trtllm_bf16_routed_moe_lora, trtllm_fp4_block_scale_moe_lora_finalize, trtllm_fp4_block_scale_routed_moe_lora, diff --git a/python/sglang/kernels/ops/moe/trtllm_lora_temp/core.py b/python/sglang/kernels/ops/lora/moe/trtllm_lora_temp/core.py similarity index 99% rename from python/sglang/kernels/ops/moe/trtllm_lora_temp/core.py rename to python/sglang/kernels/ops/lora/moe/trtllm_lora_temp/core.py index 50d151abc5..1b2b7b18a4 100644 --- a/python/sglang/kernels/ops/moe/trtllm_lora_temp/core.py +++ b/python/sglang/kernels/ops/lora/moe/trtllm_lora_temp/core.py @@ -10,7 +10,7 @@ from sglang.srt.lora.trtllm_lora_temp.environ import lora_envs def get_sgl_trtllm_moe_sm100_module(): import flashinfer.fused_moe.core as fi_core - from sglang.kernels.ops.moe.trtllm_lora_temp.jit import ( + from sglang.kernels.ops.lora.moe.trtllm_lora_temp.jit import ( gen_sgl_trtllm_gen_fused_moe_sm100_module, ) @@ -30,7 +30,7 @@ def get_sgl_trtllm_moe_sm100_module(): def get_sgl_trtllm_moe_sm100_raw_module(): from flashinfer.fused_moe.core import setup_cubin_loader - from sglang.kernels.ops.moe.trtllm_lora_temp.jit import ( + from sglang.kernels.ops.lora.moe.trtllm_lora_temp.jit import ( gen_sgl_trtllm_gen_fused_moe_sm100_module, ) diff --git a/python/sglang/kernels/ops/moe/trtllm_lora_temp/data/csrc/fused_activation_quant.cuh b/python/sglang/kernels/ops/lora/moe/trtllm_lora_temp/data/csrc/fused_activation_quant.cuh similarity index 100% rename from python/sglang/kernels/ops/moe/trtllm_lora_temp/data/csrc/fused_activation_quant.cuh rename to python/sglang/kernels/ops/lora/moe/trtllm_lora_temp/data/csrc/fused_activation_quant.cuh diff --git a/python/sglang/kernels/ops/moe/trtllm_lora_temp/data/csrc/fused_moe/trtllm_backend/trtllm_fused_moe_dev_kernel.cu b/python/sglang/kernels/ops/lora/moe/trtllm_lora_temp/data/csrc/fused_moe/trtllm_backend/trtllm_fused_moe_dev_kernel.cu similarity index 100% rename from python/sglang/kernels/ops/moe/trtllm_lora_temp/data/csrc/fused_moe/trtllm_backend/trtllm_fused_moe_dev_kernel.cu rename to python/sglang/kernels/ops/lora/moe/trtllm_lora_temp/data/csrc/fused_moe/trtllm_backend/trtllm_fused_moe_dev_kernel.cu diff --git a/python/sglang/kernels/ops/moe/trtllm_lora_temp/data/csrc/fused_permute_quant.cuh b/python/sglang/kernels/ops/lora/moe/trtllm_lora_temp/data/csrc/fused_permute_quant.cuh similarity index 100% rename from python/sglang/kernels/ops/moe/trtllm_lora_temp/data/csrc/fused_permute_quant.cuh rename to python/sglang/kernels/ops/lora/moe/trtllm_lora_temp/data/csrc/fused_permute_quant.cuh diff --git a/python/sglang/kernels/ops/moe/trtllm_lora_temp/data/csrc/trtllm_fused_moe_kernel_launcher.cu b/python/sglang/kernels/ops/lora/moe/trtllm_lora_temp/data/csrc/trtllm_fused_moe_kernel_launcher.cu similarity index 100% rename from python/sglang/kernels/ops/moe/trtllm_lora_temp/data/csrc/trtllm_fused_moe_kernel_launcher.cu rename to python/sglang/kernels/ops/lora/moe/trtllm_lora_temp/data/csrc/trtllm_fused_moe_kernel_launcher.cu diff --git a/python/sglang/kernels/ops/moe/trtllm_lora_temp/data/csrc/trtllm_fused_moe_runner.cu b/python/sglang/kernels/ops/lora/moe/trtllm_lora_temp/data/csrc/trtllm_fused_moe_runner.cu similarity index 100% rename from python/sglang/kernels/ops/moe/trtllm_lora_temp/data/csrc/trtllm_fused_moe_runner.cu rename to python/sglang/kernels/ops/lora/moe/trtllm_lora_temp/data/csrc/trtllm_fused_moe_runner.cu diff --git a/python/sglang/kernels/ops/moe/trtllm_lora_temp/data/include/flashinfer/trtllm/fused_moe/DevKernel.h b/python/sglang/kernels/ops/lora/moe/trtllm_lora_temp/data/include/flashinfer/trtllm/fused_moe/DevKernel.h similarity index 100% rename from python/sglang/kernels/ops/moe/trtllm_lora_temp/data/include/flashinfer/trtllm/fused_moe/DevKernel.h rename to python/sglang/kernels/ops/lora/moe/trtllm_lora_temp/data/include/flashinfer/trtllm/fused_moe/DevKernel.h diff --git a/python/sglang/kernels/ops/moe/trtllm_lora_temp/data/include/flashinfer/trtllm/fused_moe/runner.h b/python/sglang/kernels/ops/lora/moe/trtllm_lora_temp/data/include/flashinfer/trtllm/fused_moe/runner.h similarity index 100% rename from python/sglang/kernels/ops/moe/trtllm_lora_temp/data/include/flashinfer/trtllm/fused_moe/runner.h rename to python/sglang/kernels/ops/lora/moe/trtllm_lora_temp/data/include/flashinfer/trtllm/fused_moe/runner.h diff --git a/python/sglang/kernels/ops/moe/trtllm_lora_temp/jit.py b/python/sglang/kernels/ops/lora/moe/trtllm_lora_temp/jit.py similarity index 100% rename from python/sglang/kernels/ops/moe/trtllm_lora_temp/jit.py rename to python/sglang/kernels/ops/lora/moe/trtllm_lora_temp/jit.py diff --git a/python/sglang/kernels/ops/moe/trtllm_lora_temp/kimi_k2_moe_fused_gate.py b/python/sglang/kernels/ops/lora/moe/trtllm_lora_temp/kimi_k2_moe_fused_gate.py similarity index 96% rename from python/sglang/kernels/ops/moe/trtllm_lora_temp/kimi_k2_moe_fused_gate.py rename to python/sglang/kernels/ops/lora/moe/trtllm_lora_temp/kimi_k2_moe_fused_gate.py index 7ad2476f49..17a79b0dc3 100644 --- a/python/sglang/kernels/ops/moe/trtllm_lora_temp/kimi_k2_moe_fused_gate.py +++ b/python/sglang/kernels/ops/lora/moe/trtllm_lora_temp/kimi_k2_moe_fused_gate.py @@ -14,7 +14,7 @@ if TYPE_CHECKING: def _jit_kimi_k2_moe_fused_gate_module() -> Module: return load_jit( "kimi_k2_moe_fused_gate", - cuda_files=["trtllm_lora_temp/kimi_k2_moe_fused_gate.cuh"], + cuda_files=["lora/trtllm_lora_temp/kimi_k2_moe_fused_gate.cuh"], cuda_wrappers=[ ("kimi_k2_moe_fused_gate", "KimiK2MoEFusedGateKernel::run"), ], diff --git a/python/sglang/kernels/ops/moe/trtllm_lora_temp/moe_lora_merged_align.py b/python/sglang/kernels/ops/lora/moe/trtllm_lora_temp/moe_lora_merged_align.py similarity index 98% rename from python/sglang/kernels/ops/moe/trtllm_lora_temp/moe_lora_merged_align.py rename to python/sglang/kernels/ops/lora/moe/trtllm_lora_temp/moe_lora_merged_align.py index c55e702ee5..183e77b5f1 100644 --- a/python/sglang/kernels/ops/moe/trtllm_lora_temp/moe_lora_merged_align.py +++ b/python/sglang/kernels/ops/lora/moe/trtllm_lora_temp/moe_lora_merged_align.py @@ -16,7 +16,7 @@ def _jit_module(dtype: torch.dtype) -> Module: return load_jit( "moe_lora_merged_align", *args, - cuda_files=["trtllm_lora_temp/moe_lora_merged_align_kernel.cu"], + cuda_files=["lora/trtllm_lora_temp/moe_lora_merged_align_kernel.cu"], cuda_wrappers=[ ("moe_lora_merged_align", f"MoeLoraMergedAlignKernel<{args}>::run"), ], diff --git a/python/sglang/kernels/ops/moe/trtllm_lora_temp/topk_pack.py b/python/sglang/kernels/ops/lora/moe/trtllm_lora_temp/topk_pack.py similarity index 100% rename from python/sglang/kernels/ops/moe/trtllm_lora_temp/topk_pack.py rename to python/sglang/kernels/ops/lora/moe/trtllm_lora_temp/topk_pack.py diff --git a/python/sglang/kernels/ops/moe/trtllm_lora_temp/topk_softmax_pack.py b/python/sglang/kernels/ops/lora/moe/trtllm_lora_temp/topk_softmax_pack.py similarity index 97% rename from python/sglang/kernels/ops/moe/trtllm_lora_temp/topk_softmax_pack.py rename to python/sglang/kernels/ops/lora/moe/trtllm_lora_temp/topk_softmax_pack.py index fb3455727c..550f6bcb92 100644 --- a/python/sglang/kernels/ops/moe/trtllm_lora_temp/topk_softmax_pack.py +++ b/python/sglang/kernels/ops/lora/moe/trtllm_lora_temp/topk_softmax_pack.py @@ -31,7 +31,7 @@ if TYPE_CHECKING: def _jit_topk_softmax_pack_module() -> Module: return load_jit( "topk_softmax_pack", - cuda_files=["trtllm_lora_temp/topk_softmax_pack.cuh"], + cuda_files=["lora/trtllm_lora_temp/topk_softmax_pack.cuh"], cuda_wrappers=[("topk_softmax_pack", "topk_softmax_pack")], ) diff --git a/python/sglang/kernels/ops/moe/trtllm_lora_temp/virtual_experts.py b/python/sglang/kernels/ops/lora/moe/trtllm_lora_temp/virtual_experts.py similarity index 99% rename from python/sglang/kernels/ops/moe/trtllm_lora_temp/virtual_experts.py rename to python/sglang/kernels/ops/lora/moe/trtllm_lora_temp/virtual_experts.py index e3a7332065..0836bccb3b 100644 --- a/python/sglang/kernels/ops/moe/trtllm_lora_temp/virtual_experts.py +++ b/python/sglang/kernels/ops/lora/moe/trtllm_lora_temp/virtual_experts.py @@ -9,10 +9,10 @@ import torch import triton import triton.language as tl -from sglang.kernels.ops.gemm.trtllm_lora_temp.kernel_utils import ( +from sglang.kernels.ops.lora.dense.trtllm_lora_temp.kernel_utils import ( get_pdl_launch_metadata, ) -from sglang.kernels.ops.moe.virtual_experts import _align_block_size_large +from sglang.kernels.ops.lora.moe.virtual_experts import _align_block_size_large from sglang.srt.lora.trtllm_lora_temp.environ import lora_envs @@ -394,7 +394,7 @@ def _get_moe_lora_shrink_split_k( return max(1, min(triton.cdiv(target, base_grid), max_split_k, 8)) -# Rank-specialized LoRA-B expand kernel lives in lora/trtllm_lora_temp/. +# Rank-specialized LoRA-B expand kernel lives in srt/lora/trtllm_lora_temp/. # Re-export so existing call sites (and any external imports) keep working. from sglang.srt.lora.trtllm_lora_temp.specialized_expand import ( # noqa: E402,F401 _invoke_moe_lora_expand_add, @@ -574,7 +574,7 @@ def _merged_experts_fused_moe_lora_add_impl( and bucket_experts + 1 <= 1024 and topk_ids.shape[0] < 512 ): - from sglang.kernels.ops.moe.trtllm_lora_temp.moe_lora_merged_align import ( + from sglang.kernels.ops.lora.moe.trtllm_lora_temp.moe_lora_merged_align import ( moe_lora_merged_align, ) diff --git a/python/sglang/kernels/ops/moe/virtual_experts.py b/python/sglang/kernels/ops/lora/moe/virtual_experts.py similarity index 100% rename from python/sglang/kernels/ops/moe/virtual_experts.py rename to python/sglang/kernels/ops/lora/moe/virtual_experts.py diff --git a/python/sglang/kernels/ops/moe/__init__.py b/python/sglang/kernels/ops/moe/__init__.py index 02657fded5..9ae48045b0 100644 --- a/python/sglang/kernels/ops/moe/__init__.py +++ b/python/sglang/kernels/ops/moe/__init__.py @@ -161,23 +161,6 @@ def situ_and_mul_masked_post_quant( __all__ = ["situ_and_mul_masked_post_quant", "moe_align_block_size", "topk_softmax"] -# Fused MoE-LoRA Triton kernels migrated into this group (from lora/triton_ops); -# registered for inventory. Import them from their modules. -_TRITON_KERNELS = [ - ("fused_moe_lora_kernel", "fused_moe_lora"), - ("virtual_experts", "merged_experts_fused_moe_lora_add"), -] -for _mod, _fn in _TRITON_KERNELS: - register_kernel( - KernelSpec( - op=f"moe.{_fn}", - backend=KernelBackend.TRITON, - target=f"sglang.kernels.ops.moe.{_mod}:{_fn}", - ) - ) -del _mod, _fn - - # Triton kernels migrated from srt/layers/moe (RFC #29630, Phase 2.5); # registered for inventory. Import them from their modules. _PHASE25_TRITON_KERNELS = [ diff --git a/python/sglang/srt/layers/moe/moe_runner/marlin.py b/python/sglang/srt/layers/moe/moe_runner/marlin.py index 0c5ea7fae8..df62ab6b78 100644 --- a/python/sglang/srt/layers/moe/moe_runner/marlin.py +++ b/python/sglang/srt/layers/moe/moe_runner/marlin.py @@ -43,7 +43,7 @@ def _fused_unpack_packed_topk(packed: torch.Tensor): Returns (topk_ids int32, topk_weights float32). Collapses the ~5 elementwise ops (shift / mask / int16 / bitcast / cast) the torch reference emits per call - into one Triton launch. Mirrors _pack_topk_kernel (trtllm_lora_temp/topk_pack). + into one Triton launch. Mirrors _pack_topk_kernel (ops/lora/moe/trtllm_lora_temp/topk_pack). """ packed = packed.contiguous() ids = torch.empty_like(packed, dtype=torch.int32) diff --git a/python/sglang/srt/layers/moe/moe_runner/triton_utils/moe_align_block_size.py b/python/sglang/srt/layers/moe/moe_runner/triton_utils/moe_align_block_size.py index b36b37ea5e..6ad8973393 100644 --- a/python/sglang/srt/layers/moe/moe_runner/triton_utils/moe_align_block_size.py +++ b/python/sglang/srt/layers/moe/moe_runner/triton_utils/moe_align_block_size.py @@ -77,7 +77,7 @@ def moe_align_block_size( from sglang.srt.lora.trtllm_lora_temp.environ import lora_envs if lora_envs.SGLANG_OPT_USE_JIT_KERNEL_MOE_ALIGN.get() and num_experts <= 8191: - from sglang.kernels.ops.moe.trtllm_lora_temp.virtual_experts import ( + from sglang.kernels.ops.lora.moe.virtual_experts import ( _align_block_size_jit, ) diff --git a/python/sglang/srt/layers/moe/topk.py b/python/sglang/srt/layers/moe/topk.py index c8a1eceb4d..801bbb6b8b 100644 --- a/python/sglang/srt/layers/moe/topk.py +++ b/python/sglang/srt/layers/moe/topk.py @@ -1061,7 +1061,7 @@ def fused_topk( elif packed_out is not None: # Fused gating + routed pack (SGLANG_OPT_LORA_FUSED_TOPK_PACK): one JIT kernel # writes topk_weights/topk_ids AND the FlashInfer packed topk in one launch. - from sglang.kernels.ops.moe.trtllm_lora_temp.topk_softmax_pack import ( + from sglang.kernels.ops.lora.moe.trtllm_lora_temp.topk_softmax_pack import ( topk_softmax_pack, ) @@ -1966,7 +1966,7 @@ def biased_grouped_topk_gpu( and lora_envs.SGLANG_OPT_KIMI_GATE_BF16_INPUT.get() ) if _use_jit_bf16_gate: - from sglang.kernels.ops.moe.trtllm_lora_temp.kimi_k2_moe_fused_gate import ( + from sglang.kernels.ops.lora.moe.trtllm_lora_temp.kimi_k2_moe_fused_gate import ( kimi_k2_moe_fused_gate as _kimi_k2_moe_fused_gate, ) diff --git a/python/sglang/srt/lora/backend/chunked_backend.py b/python/sglang/srt/lora/backend/chunked_backend.py index c22549897a..64533908ef 100644 --- a/python/sglang/srt/lora/backend/chunked_backend.py +++ b/python/sglang/srt/lora/backend/chunked_backend.py @@ -3,11 +3,15 @@ from typing import List, Optional, Tuple import torch -from sglang.kernels.ops.gemm.chunked_embedding_lora_a import ( +from sglang.kernels.ops.lora.dense.chunked_embedding_lora_a import ( chunked_embedding_lora_a_forward, ) -from sglang.kernels.ops.gemm.chunked_sgmv_expand import chunked_sgmv_lora_expand_forward -from sglang.kernels.ops.gemm.chunked_sgmv_shrink import chunked_sgmv_lora_shrink_forward +from sglang.kernels.ops.lora.dense.chunked_sgmv_expand import ( + chunked_sgmv_lora_expand_forward, +) +from sglang.kernels.ops.lora.dense.chunked_sgmv_shrink import ( + chunked_sgmv_lora_shrink_forward, +) from sglang.srt.lora.backend.base_backend import BaseLoRABackend from sglang.srt.lora.utils import ( LoRABatchInfo, diff --git a/python/sglang/srt/lora/backend/triton_backend.py b/python/sglang/srt/lora/backend/triton_backend.py index 30560a0b33..ddd291d2c2 100644 --- a/python/sglang/srt/lora/backend/triton_backend.py +++ b/python/sglang/srt/lora/backend/triton_backend.py @@ -3,11 +3,11 @@ from typing import List, Optional, Tuple import torch -from sglang.kernels.ops.gemm.embedding_lora_a import embedding_lora_a_fwd -from sglang.kernels.ops.gemm.gate_up_lora_b import gate_up_lora_b_fwd -from sglang.kernels.ops.gemm.qkv_lora_b import qkv_lora_b_fwd -from sglang.kernels.ops.gemm.sgemm_lora_a import sgemm_lora_a_fwd -from sglang.kernels.ops.gemm.sgemm_lora_b import sgemm_lora_b_fwd +from sglang.kernels.ops.lora.dense.embedding_lora_a import embedding_lora_a_fwd +from sglang.kernels.ops.lora.dense.gate_up_lora_b import gate_up_lora_b_fwd +from sglang.kernels.ops.lora.dense.qkv_lora_b import qkv_lora_b_fwd +from sglang.kernels.ops.lora.dense.sgemm_lora_a import sgemm_lora_a_fwd +from sglang.kernels.ops.lora.dense.sgemm_lora_b import sgemm_lora_b_fwd from sglang.srt.environ import envs from sglang.srt.layers.dp_attention import ( DpPaddingMode, diff --git a/python/sglang/srt/lora/deepseek_mla_correction.py b/python/sglang/srt/lora/deepseek_mla_correction.py index ddc1a378a3..9cddf4daf9 100644 --- a/python/sglang/srt/lora/deepseek_mla_correction.py +++ b/python/sglang/srt/lora/deepseek_mla_correction.py @@ -19,7 +19,7 @@ from typing import TYPE_CHECKING, Optional, Tuple import torch -from sglang.kernels.ops.gemm.kv_b_lora_absorbed import ( +from sglang.kernels.ops.lora.dense.kv_b_lora_absorbed import ( step_a_q_fwd, step_a_v_fwd, step_b_q_fwd, diff --git a/python/sglang/srt/lora/lora_moe_runners.py b/python/sglang/srt/lora/lora_moe_runners.py index 8b1d3e5e45..b630b2db79 100644 --- a/python/sglang/srt/lora/lora_moe_runners.py +++ b/python/sglang/srt/lora/lora_moe_runners.py @@ -37,7 +37,7 @@ _is_hip = is_hip() _is_xpu = is_xpu() if _is_cuda or _is_hip or _is_xpu: - from sglang.kernels.ops.moe.moe_lora_align import moe_lora_align_block_size + from sglang.kernels.ops.lora.moe.moe_lora_align import moe_lora_align_block_size def _get_moe_lora_block_config(max_lora_rank: int) -> dict: @@ -316,8 +316,10 @@ def _add_lora_gate_up_delta( routing_cache: dict | None = None, ) -> None: """Add LoRA gate_up delta to intermediate_cache in-place.""" - from sglang.kernels.ops.moe.fused_moe_lora_kernel import fused_moe_lora - from sglang.kernels.ops.moe.virtual_experts import merged_experts_fused_moe_lora_add + from sglang.kernels.ops.lora.moe.fused_moe_lora_kernel import fused_moe_lora + from sglang.kernels.ops.lora.moe.virtual_experts import ( + merged_experts_fused_moe_lora_add, + ) if lora_info is None or lora_info.max_lora_rank == 0: return @@ -407,8 +409,10 @@ def _add_lora_down_delta( routing_cache: dict | None = None, ) -> None: """Add LoRA down delta to intermediate_cache in-place.""" - from sglang.kernels.ops.moe.fused_moe_lora_kernel import fused_moe_lora - from sglang.kernels.ops.moe.virtual_experts import merged_experts_fused_moe_lora_add + from sglang.kernels.ops.lora.moe.fused_moe_lora_kernel import fused_moe_lora + from sglang.kernels.ops.lora.moe.virtual_experts import ( + merged_experts_fused_moe_lora_add, + ) if lora_info.max_lora_rank == 0: return diff --git a/python/sglang/srt/lora/marlin_lora_temp/moe_runner.py b/python/sglang/srt/lora/marlin_lora_temp/moe_runner.py index 8b00244d71..16201396ab 100644 --- a/python/sglang/srt/lora/marlin_lora_temp/moe_runner.py +++ b/python/sglang/srt/lora/marlin_lora_temp/moe_runner.py @@ -24,13 +24,13 @@ _is_cuda = is_cuda() if _is_cuda: from sgl_kernel import silu_and_mul + from sglang.kernels.ops.lora.moe.virtual_experts import ( + _align_block_size_jit as moe_align_block_size, + ) from sglang.kernels.ops.moe.fused_moe_triton_kernels import ( moe_sum_reduce_triton, ) from sglang.kernels.ops.moe.moe_wna16_marlin import moe_wna16_marlin_gemm - from sglang.kernels.ops.moe.virtual_experts import ( - _align_block_size_jit as moe_align_block_size, - ) from sglang.srt.layers.moe.fused_moe_triton.fused_marlin_moe import ( get_scalar_type, ) @@ -208,7 +208,7 @@ def fused_experts_experimental_sgl_marlin_lora( runner_config: MoeRunnerConfig, lora_info, ) -> StandardCombineInput: - from sglang.kernels.ops.moe.trtllm_lora_temp.virtual_experts import ( + from sglang.kernels.ops.lora.moe.trtllm_lora_temp.virtual_experts import ( merged_experts_fused_moe_lora_add, ) from sglang.srt.layers.moe.token_dispatcher.standard import ( diff --git a/python/sglang/srt/lora/trtllm_lora_temp/attention.py b/python/sglang/srt/lora/trtllm_lora_temp/attention.py index 4dceab6486..332325a4af 100644 --- a/python/sglang/srt/lora/trtllm_lora_temp/attention.py +++ b/python/sglang/srt/lora/trtllm_lora_temp/attention.py @@ -42,8 +42,10 @@ def qkv_proj_lora_forward(self, input_: torch.Tensor): ): return get_original_qkv_forward()(self, input_) - from sglang.kernels.ops.gemm.trtllm_lora_temp.qkv_lora_b import qkv_lora_b_fwd - from sglang.kernels.ops.gemm.trtllm_lora_temp.sgemm_lora_a import sgemm_lora_a_fwd + from sglang.kernels.ops.lora.dense.trtllm_lora_temp.qkv_lora_b import qkv_lora_b_fwd + from sglang.kernels.ops.lora.dense.trtllm_lora_temp.sgemm_lora_a import ( + sgemm_lora_a_fwd, + ) bias = self.base_layer.bias if not self.base_layer.skip_bias_add else None side_stream = get_lora_side_stream() @@ -121,10 +123,10 @@ def row_parallel_lora_forward( _alloc = lora_overlap_alloc_stream() # capture MAIN stream here (before the fork) side_stream.wait_stream(torch.cuda.current_stream()) with torch.cuda.stream(side_stream): - from sglang.kernels.ops.gemm.trtllm_lora_temp.sgemm_lora_a import ( + from sglang.kernels.ops.lora.dense.trtllm_lora_temp.sgemm_lora_a import ( sgemm_lora_a_fwd, ) - from sglang.kernels.ops.gemm.trtllm_lora_temp.sgemm_lora_b import ( + from sglang.kernels.ops.lora.dense.trtllm_lora_temp.sgemm_lora_b import ( sgemm_lora_b_fwd, ) @@ -189,10 +191,10 @@ def column_parallel_lora_forward(self, input_: torch.Tensor): _alloc = lora_overlap_alloc_stream() # capture MAIN stream here (before the fork) side_stream.wait_stream(torch.cuda.current_stream()) with torch.cuda.stream(side_stream): - from sglang.kernels.ops.gemm.trtllm_lora_temp.sgemm_lora_a import ( + from sglang.kernels.ops.lora.dense.trtllm_lora_temp.sgemm_lora_a import ( sgemm_lora_a_fwd, ) - from sglang.kernels.ops.gemm.trtllm_lora_temp.sgemm_lora_b import ( + from sglang.kernels.ops.lora.dense.trtllm_lora_temp.sgemm_lora_b import ( sgemm_lora_b_fwd, ) @@ -245,10 +247,10 @@ def replicated_lora_forward(self, x: torch.Tensor): _alloc = lora_overlap_alloc_stream() # capture MAIN stream here (before the fork) side_stream.wait_stream(torch.cuda.current_stream()) with torch.cuda.stream(side_stream): - from sglang.kernels.ops.gemm.trtllm_lora_temp.sgemm_lora_a import ( + from sglang.kernels.ops.lora.dense.trtllm_lora_temp.sgemm_lora_a import ( sgemm_lora_a_fwd, ) - from sglang.kernels.ops.gemm.trtllm_lora_temp.sgemm_lora_b import ( + from sglang.kernels.ops.lora.dense.trtllm_lora_temp.sgemm_lora_b import ( sgemm_lora_b_fwd, ) @@ -267,7 +269,9 @@ def replicated_lora_forward(self, x: torch.Tensor): if first_dim == 0: output = sgemm_lora_b_fwd(lora_a_output, self.B_buffer, sgemm_info, output) else: - from sglang.kernels.ops.gemm.trtllm_lora_temp.qkv_lora_b import qkv_lora_b_fwd + from sglang.kernels.ops.lora.dense.trtllm_lora_temp.qkv_lora_b import ( + qkv_lora_b_fwd, + ) output = qkv_lora_b_fwd( lora_a_output, diff --git a/python/sglang/srt/lora/trtllm_lora_temp/deepseek_mla_correction.py b/python/sglang/srt/lora/trtllm_lora_temp/deepseek_mla_correction.py index dc4d5a46cd..8ae3b7e2c3 100644 --- a/python/sglang/srt/lora/trtllm_lora_temp/deepseek_mla_correction.py +++ b/python/sglang/srt/lora/trtllm_lora_temp/deepseek_mla_correction.py @@ -29,16 +29,16 @@ step_a_q_fwd = step_a_v_fwd = step_b_q_fwd = step_b_v_fwd = None def _ensure_step_kernels() -> None: global step_a_q_fwd, step_a_v_fwd, step_b_q_fwd, step_b_v_fwd if step_a_q_fwd is None: - from sglang.kernels.ops.gemm.trtllm_lora_temp.kv_b_lora_absorbed import ( + from sglang.kernels.ops.lora.dense.trtllm_lora_temp.kv_b_lora_absorbed import ( step_a_q_fwd as _aq, ) - from sglang.kernels.ops.gemm.trtllm_lora_temp.kv_b_lora_absorbed import ( + from sglang.kernels.ops.lora.dense.trtllm_lora_temp.kv_b_lora_absorbed import ( step_a_v_fwd as _av, ) - from sglang.kernels.ops.gemm.trtllm_lora_temp.kv_b_lora_absorbed import ( + from sglang.kernels.ops.lora.dense.trtllm_lora_temp.kv_b_lora_absorbed import ( step_b_q_fwd as _bq, ) - from sglang.kernels.ops.gemm.trtllm_lora_temp.kv_b_lora_absorbed import ( + from sglang.kernels.ops.lora.dense.trtllm_lora_temp.kv_b_lora_absorbed import ( step_b_v_fwd as _bv, ) diff --git a/python/sglang/srt/lora/trtllm_lora_temp/environ.py b/python/sglang/srt/lora/trtllm_lora_temp/environ.py index 1ab0611dc7..779ad40c7c 100644 --- a/python/sglang/srt/lora/trtllm_lora_temp/environ.py +++ b/python/sglang/srt/lora/trtllm_lora_temp/environ.py @@ -108,7 +108,7 @@ class _LoraEnvs: # diagnostics / tuning: SGLANG_OPT_LORA_SHRINK_TUNE = _GatedBool("SGLANG_OPT_LORA_SHRINK_TUNE", False) - # kimi NVFP4 permute+quant fuse — read in kernels/ops/moe/trtllm_lora_temp/core.py (Python) to pass + # kimi NVFP4 permute+quant fuse — read in kernels/ops/lora/moe/trtllm_lora_temp/core.py (Python) to pass # a bool to the kernel, AND C++-side via getenv in the launcher. Default off (kimi-only). SGLANG_OPT_FUSED_PERMUTE_QUANT = _GatedBool("SGLANG_OPT_FUSED_PERMUTE_QUANT", False) diff --git a/python/sglang/srt/lora/trtllm_lora_temp/experimental_sgl_trtllm_moe.py b/python/sglang/srt/lora/trtllm_lora_temp/experimental_sgl_trtllm_moe.py index 9d1b2e9a8a..f4e0e0ae3e 100644 --- a/python/sglang/srt/lora/trtllm_lora_temp/experimental_sgl_trtllm_moe.py +++ b/python/sglang/srt/lora/trtllm_lora_temp/experimental_sgl_trtllm_moe.py @@ -70,7 +70,7 @@ def sgl_trtllm_fp8_block_scale_moe_wrapper( "TRTLLM enums and cubin-loader utilities." ) from e - from sglang.kernels.ops.moe.trtllm_lora_temp import trtllm_fp8_block_scale_moe + from sglang.kernels.ops.lora.moe.trtllm_lora_temp import trtllm_fp8_block_scale_moe kwargs = { "routing_logits": routing_logits, @@ -168,7 +168,7 @@ def sgl_trtllm_fp8_block_scale_routed_moe_wrapper( "TRTLLM enums and cubin-loader utilities." ) from e - from sglang.kernels.ops.moe.trtllm_lora_temp import ( + from sglang.kernels.ops.lora.moe.trtllm_lora_temp import ( trtllm_fp8_block_scale_routed_moe, ) diff --git a/python/sglang/srt/lora/trtllm_lora_temp/inkling_dense.py b/python/sglang/srt/lora/trtllm_lora_temp/inkling_dense.py index 25d5184a3a..7a2557c692 100644 --- a/python/sglang/srt/lora/trtllm_lora_temp/inkling_dense.py +++ b/python/sglang/srt/lora/trtllm_lora_temp/inkling_dense.py @@ -41,10 +41,10 @@ def apply_multi_lora( ) -> torch.Tensor: if getattr(layer.lora_backend, "name", None) != "triton" or not inputs.is_cuda: raise RuntimeError("Multi-slot dense LoRA requires Triton on CUDA") - from sglang.kernels.ops.gemm.trtllm_lora_temp.sgemm_lora_a import ( + from sglang.kernels.ops.lora.dense.trtllm_lora_temp.sgemm_lora_a import ( shared_sink_sgemm_lora_a_fwd, ) - from sglang.kernels.ops.gemm.trtllm_lora_temp.sgemm_lora_b import ( + from sglang.kernels.ops.lora.dense.trtllm_lora_temp.sgemm_lora_b import ( shared_sink_sgemm_lora_b_fwd, ) @@ -86,7 +86,7 @@ def _apply_per_expert_lora( lora_a: torch.Tensor, lora_b: torch.Tensor, ) -> None: - from sglang.kernels.ops.moe.virtual_experts import ( + from sglang.kernels.ops.lora.moe.virtual_experts import ( merged_experts_fused_moe_lora_add, ) @@ -156,7 +156,7 @@ def forward_with_lora( if single_gate_up: torch.mm(x_td, a_gate_up.T, out=gate_up_shrink) else: - from sglang.kernels.ops.gemm.trtllm_lora_temp.sgemm_lora_a import ( + from sglang.kernels.ops.lora.dense.trtllm_lora_temp.sgemm_lora_a import ( shared_sink_sgemm_lora_a_fwd, ) @@ -190,7 +190,7 @@ def forward_with_lora( y_flat.addmm_(gate_up_shrink, layer._w1_delta[0].T) y = y_flat.view(t, n, -1) else: - from sglang.kernels.ops.gemm.trtllm_lora_temp.sgemm_lora_b import ( + from sglang.kernels.ops.lora.dense.trtllm_lora_temp.sgemm_lora_b import ( shared_sink_sgemm_lora_b_fwd, ) diff --git a/python/sglang/srt/lora/trtllm_lora_temp/lora_dispatch.py b/python/sglang/srt/lora/trtllm_lora_temp/lora_dispatch.py index 4b062858c7..d5aecb570d 100644 --- a/python/sglang/srt/lora/trtllm_lora_temp/lora_dispatch.py +++ b/python/sglang/srt/lora/trtllm_lora_temp/lora_dispatch.py @@ -47,12 +47,12 @@ def fused_experts_none_to_experimental_sgl_trtllm_fp8_lora( ) -> StandardCombineInput: from flashinfer.fused_moe import Fp8QuantizationType - from sglang.kernels.ops.moe.trtllm_lora_temp import ( + from sglang.kernels.ops.lora.moe.trtllm_lora_temp import ( trtllm_fp8_block_scale_moe_lora_finalize, trtllm_fp8_block_scale_routed_moe_lora, ) - from sglang.kernels.ops.moe.trtllm_lora_temp.topk_pack import fused_pack_topk - from sglang.kernels.ops.moe.trtllm_lora_temp.virtual_experts import ( + from sglang.kernels.ops.lora.moe.trtllm_lora_temp.topk_pack import fused_pack_topk + from sglang.kernels.ops.lora.moe.trtllm_lora_temp.virtual_experts import ( merged_experts_fused_moe_lora_add, ) from sglang.srt.layers.moe.token_dispatcher.standard import StandardCombineInput @@ -316,9 +316,9 @@ def fused_experts_none_to_experimental_sgl_trtllm_bf16_lora( down grouped GEMM -> finalize, then the virtual-experts down-LoRA is merged into the output. Single-stream version (no two-stream overlap yet — phase 2). """ - from sglang.kernels.ops.moe.trtllm_lora_temp import trtllm_bf16_routed_moe_lora - from sglang.kernels.ops.moe.trtllm_lora_temp.topk_pack import fused_pack_topk - from sglang.kernels.ops.moe.trtllm_lora_temp.virtual_experts import ( + from sglang.kernels.ops.lora.moe.trtllm_lora_temp import trtllm_bf16_routed_moe_lora + from sglang.kernels.ops.lora.moe.trtllm_lora_temp.topk_pack import fused_pack_topk + from sglang.kernels.ops.lora.moe.trtllm_lora_temp.virtual_experts import ( merged_experts_fused_moe_lora_add, ) from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import ( @@ -474,11 +474,11 @@ def fused_experts_none_to_experimental_sgl_trtllm_fp4_lora( then the virtual-experts down-LoRA is merged into the output. Single-stream version; ``moe_overlap.py`` provides the two-stream variant. """ - from sglang.kernels.ops.moe.trtllm_lora_temp import ( + from sglang.kernels.ops.lora.moe.trtllm_lora_temp import ( trtllm_fp4_block_scale_routed_moe_lora, ) - from sglang.kernels.ops.moe.trtllm_lora_temp.topk_pack import fused_pack_topk - from sglang.kernels.ops.moe.trtllm_lora_temp.virtual_experts import ( + from sglang.kernels.ops.lora.moe.trtllm_lora_temp.topk_pack import fused_pack_topk + from sglang.kernels.ops.lora.moe.trtllm_lora_temp.virtual_experts import ( merged_experts_fused_moe_lora_add, ) from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import ( diff --git a/python/sglang/srt/lora/trtllm_lora_temp/lora_layer.py b/python/sglang/srt/lora/trtllm_lora_temp/lora_layer.py index 40ec490cac..045be3a47c 100644 --- a/python/sglang/srt/lora/trtllm_lora_temp/lora_layer.py +++ b/python/sglang/srt/lora/trtllm_lora_temp/lora_layer.py @@ -39,7 +39,7 @@ def _warm_sgl_trtllm_moe_module() -> None: global _SGL_TRTLLM_MODULE_WARMED if _SGL_TRTLLM_MODULE_WARMED: return - from sglang.kernels.ops.moe.trtllm_lora_temp.core import ( + from sglang.kernels.ops.lora.moe.trtllm_lora_temp.core import ( get_sgl_trtllm_moe_sm100_raw_module, ) diff --git a/python/sglang/srt/lora/trtllm_lora_temp/merged_column.py b/python/sglang/srt/lora/trtllm_lora_temp/merged_column.py index f3cbf3abbf..bde335c5d2 100644 --- a/python/sglang/srt/lora/trtllm_lora_temp/merged_column.py +++ b/python/sglang/srt/lora/trtllm_lora_temp/merged_column.py @@ -37,11 +37,13 @@ def merged_column_lora_forward(self, input_: torch.Tensor): ): return get_original_merged_column_forward()(self, input_) - from sglang.kernels.ops.gemm.trtllm_lora_temp.gate_up_lora_b import ( + from sglang.kernels.ops.lora.dense.trtllm_lora_temp.gate_up_lora_b import ( gate_up_lora_b_fwd, ) - from sglang.kernels.ops.gemm.trtllm_lora_temp.qkv_lora_b import qkv_lora_b_fwd - from sglang.kernels.ops.gemm.trtllm_lora_temp.sgemm_lora_a import sgemm_lora_a_fwd + from sglang.kernels.ops.lora.dense.trtllm_lora_temp.qkv_lora_b import qkv_lora_b_fwd + from sglang.kernels.ops.lora.dense.trtllm_lora_temp.sgemm_lora_a import ( + sgemm_lora_a_fwd, + ) bias = self.base_layer.bias if not self.base_layer.skip_bias_add else None side_stream = get_lora_side_stream() diff --git a/python/sglang/srt/lora/trtllm_lora_temp/moe_overlap.py b/python/sglang/srt/lora/trtllm_lora_temp/moe_overlap.py index 9d2ba8244d..b14cd1478c 100644 --- a/python/sglang/srt/lora/trtllm_lora_temp/moe_overlap.py +++ b/python/sglang/srt/lora/trtllm_lora_temp/moe_overlap.py @@ -54,11 +54,11 @@ def fused_experts_none_to_experimental_sgl_trtllm_fp8_lora_two_stream( # ---- two-stream fast path ---- from flashinfer.fused_moe import Fp8QuantizationType - from sglang.kernels.ops.moe.trtllm_lora_temp import ( + from sglang.kernels.ops.lora.moe.trtllm_lora_temp import ( trtllm_fp8_block_scale_routed_moe_lora, ) - from sglang.kernels.ops.moe.trtllm_lora_temp.topk_pack import fused_pack_topk - from sglang.kernels.ops.moe.trtllm_lora_temp.virtual_experts import ( + from sglang.kernels.ops.lora.moe.trtllm_lora_temp.topk_pack import fused_pack_topk + from sglang.kernels.ops.lora.moe.trtllm_lora_temp.virtual_experts import ( merged_experts_fused_moe_lora_add, ) from sglang.kernels.ops.quantization.fp8_kernel import per_token_group_quant_fp8 @@ -360,11 +360,11 @@ def fused_experts_none_to_experimental_sgl_trtllm_fp4_lora_two_stream( ) # ---- two-stream fast path ---- - from sglang.kernels.ops.moe.trtllm_lora_temp import ( + from sglang.kernels.ops.lora.moe.trtllm_lora_temp import ( trtllm_fp4_block_scale_routed_moe_lora, ) - from sglang.kernels.ops.moe.trtllm_lora_temp.topk_pack import fused_pack_topk - from sglang.kernels.ops.moe.trtllm_lora_temp.virtual_experts import ( + from sglang.kernels.ops.lora.moe.trtllm_lora_temp.topk_pack import fused_pack_topk + from sglang.kernels.ops.lora.moe.trtllm_lora_temp.virtual_experts import ( merged_experts_fused_moe_lora_add, ) from sglang.srt.distributed.device_communicators.pynccl_allocator import ( @@ -610,9 +610,9 @@ def fused_experts_none_to_experimental_sgl_trtllm_bf16_lora_two_stream( ) # ---- two-stream fast path ---- - from sglang.kernels.ops.moe.trtllm_lora_temp import trtllm_bf16_routed_moe_lora - from sglang.kernels.ops.moe.trtllm_lora_temp.topk_pack import fused_pack_topk - from sglang.kernels.ops.moe.trtllm_lora_temp.virtual_experts import ( + from sglang.kernels.ops.lora.moe.trtllm_lora_temp import trtllm_bf16_routed_moe_lora + from sglang.kernels.ops.lora.moe.trtllm_lora_temp.topk_pack import fused_pack_topk + from sglang.kernels.ops.lora.moe.trtllm_lora_temp.virtual_experts import ( merged_experts_fused_moe_lora_add, ) from sglang.srt.distributed.device_communicators.pynccl_allocator import ( diff --git a/python/sglang/srt/lora/trtllm_lora_temp/sgl_fp8_moe.py b/python/sglang/srt/lora/trtllm_lora_temp/sgl_fp8_moe.py index 732f5a2624..c21288e0d0 100644 --- a/python/sglang/srt/lora/trtllm_lora_temp/sgl_fp8_moe.py +++ b/python/sglang/srt/lora/trtllm_lora_temp/sgl_fp8_moe.py @@ -27,7 +27,7 @@ def fused_experts_fp8_sgl( # <-> quantization import cycle at load time. from flashinfer.fused_moe import Fp8QuantizationType - from sglang.kernels.ops.moe.trtllm_lora_temp.topk_pack import fused_pack_topk + from sglang.kernels.ops.lora.moe.trtllm_lora_temp.topk_pack import fused_pack_topk from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import ( is_allocation_symmetric, next_power_of_2, diff --git a/python/sglang/srt/lora/trtllm_lora_temp/specialized_expand.py b/python/sglang/srt/lora/trtllm_lora_temp/specialized_expand.py index dd4bf04b53..0875364d30 100644 --- a/python/sglang/srt/lora/trtllm_lora_temp/specialized_expand.py +++ b/python/sglang/srt/lora/trtllm_lora_temp/specialized_expand.py @@ -5,7 +5,7 @@ It is rank-specialized: the ``R`` dimension (LoRA rank) is a Triton ``constexpr``, so each rank value used at runtime gets its own JIT-compiled specialization. -Called from :mod:`sglang.kernels.ops.moe.virtual_experts` when +Called from :mod:`sglang.kernels.ops.lora.moe.trtllm_lora_temp.virtual_experts` when ``use_direct_expand_add=True``. Ranks above 64 are accumulated in multiple rank tiles. """ diff --git a/test/manual/lora/test_lora_tuning_config.py b/test/manual/lora/test_lora_tuning_config.py index 002c97cecd..e7057f49ca 100644 --- a/test/manual/lora/test_lora_tuning_config.py +++ b/test/manual/lora/test_lora_tuning_config.py @@ -6,7 +6,7 @@ import tempfile import unittest from unittest.mock import patch -from sglang.kernels.ops.gemm.lora_tuning_config import ( +from sglang.kernels.ops.lora.dense.lora_tuning_config import ( DEFAULT_EXPAND_CONFIG, DEFAULT_SHRINK_CONFIG, get_lora_config_file_name, @@ -15,7 +15,7 @@ from sglang.kernels.ops.gemm.lora_tuning_config import ( get_lora_shrink_config, ) -_MODULE = "sglang.kernels.ops.gemm.lora_tuning_config" +_MODULE = "sglang.kernels.ops.lora.dense.lora_tuning_config" # Shared fixture _TUNED_CONFIGS = { @@ -88,7 +88,7 @@ class TestConfigSelection(unittest.TestCase): def setUp(self): get_lora_configs.cache_clear() - from sglang.kernels.ops.gemm import lora_tuning_config + from sglang.kernels.ops.lora.dense import lora_tuning_config lora_tuning_config._logged_configs.clear() diff --git a/test/registered/kernels/ops/gemm/test_chunked_sgmv_cuda_graph.py b/test/registered/kernels/ops/lora/test_chunked_sgmv_cuda_graph.py similarity index 98% rename from test/registered/kernels/ops/gemm/test_chunked_sgmv_cuda_graph.py rename to test/registered/kernels/ops/lora/test_chunked_sgmv_cuda_graph.py index b47f44481c..7c069fa679 100644 --- a/test/registered/kernels/ops/gemm/test_chunked_sgmv_cuda_graph.py +++ b/test/registered/kernels/ops/lora/test_chunked_sgmv_cuda_graph.py @@ -17,15 +17,15 @@ import pytest import torch -from sglang.kernels.ops.gemm.chunked_sgmv_expand import ( +from sglang.kernels.ops.lora.dense.chunked_sgmv_expand import ( _chunked_lora_expand_kernel, chunked_sgmv_lora_expand_forward, ) -from sglang.kernels.ops.gemm.chunked_sgmv_shrink import ( +from sglang.kernels.ops.lora.dense.chunked_sgmv_shrink import ( _chunked_lora_shrink_kernel, chunked_sgmv_lora_shrink_forward, ) -from sglang.kernels.ops.gemm.kv_b_lora_absorbed import ( +from sglang.kernels.ops.lora.dense.kv_b_lora_absorbed import ( step_a_q_fwd, step_a_v_fwd, step_b_q_fwd, diff --git a/test/registered/kernels/ops/moe/test_moe_lora_align_block_size.py b/test/registered/kernels/ops/lora/test_moe_lora_align_block_size.py similarity index 98% rename from test/registered/kernels/ops/moe/test_moe_lora_align_block_size.py rename to test/registered/kernels/ops/lora/test_moe_lora_align_block_size.py index 35cc314270..6884fcc1be 100644 --- a/test/registered/kernels/ops/moe/test_moe_lora_align_block_size.py +++ b/test/registered/kernels/ops/lora/test_moe_lora_align_block_size.py @@ -8,7 +8,7 @@ import torch # --------------------------------------------------------- # IMPORT PREBUILT KERNEL # --------------------------------------------------------- -from sglang.kernels.ops.moe.moe_lora_align import moe_lora_align_block_size +from sglang.kernels.ops.lora.moe.moe_lora_align import moe_lora_align_block_size from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci register_cuda_ci(est_time=28, stage="base-b-kernel-unit", runner_config="1-gpu-large") diff --git a/test/registered/kernels/ops/moe/test_marlin_packed_topk_unpack.py b/test/registered/kernels/ops/moe/test_marlin_packed_topk_unpack.py index 09062c8969..e0e541eee3 100644 --- a/test/registered/kernels/ops/moe/test_marlin_packed_topk_unpack.py +++ b/test/registered/kernels/ops/moe/test_marlin_packed_topk_unpack.py @@ -27,7 +27,7 @@ def _torch_unpack(packed: torch.Tensor): def _torch_pack(ids: torch.Tensor, weights: torch.Tensor) -> torch.Tensor: - # inverse of the unpack, matching trtllm_lora_temp/topk_pack._pack_topk_kernel + # inverse of the unpack, matching ops/lora/moe/trtllm_lora_temp/topk_pack._pack_topk_kernel wbits = weights.to(torch.bfloat16).view(torch.int16).to(torch.int32) & 0xFFFF return (ids.to(torch.int32) << 16) | wbits diff --git a/test/registered/lora/test_fused_moe_lora_kernel.py b/test/registered/lora/test_fused_moe_lora_kernel.py index 8b63796ce7..89dd760e29 100644 --- a/test/registered/lora/test_fused_moe_lora_kernel.py +++ b/test/registered/lora/test_fused_moe_lora_kernel.py @@ -5,12 +5,12 @@ import sys import pytest import torch -from sglang.kernels.ops.moe.fused_moe_lora_kernel import fused_moe_lora +from sglang.kernels.ops.lora.moe.fused_moe_lora_kernel import fused_moe_lora # ============================================================================== # IMPORT PREBUILT KERNEL # ============================================================================== -from sglang.kernels.ops.moe.moe_lora_align import moe_lora_align_block_size +from sglang.kernels.ops.lora.moe.moe_lora_align import moe_lora_align_block_size from sglang.srt.lora.lora_moe_runners import _naive_moe_lora_align_block_size from sglang.srt.utils import get_device, is_xpu, set_random_seed from sglang.test.ci.ci_register import register_cuda_ci diff --git a/test/registered/lora/test_virtual_experts_kernels.py b/test/registered/lora/test_virtual_experts_kernels.py index e6c8d65968..8b420eaceb 100644 --- a/test/registered/lora/test_virtual_experts_kernels.py +++ b/test/registered/lora/test_virtual_experts_kernels.py @@ -42,7 +42,7 @@ def _require_accelerator_device(): return get_device(0) -from sglang.kernels.ops.moe.virtual_experts import ( +from sglang.kernels.ops.lora.moe.virtual_experts import ( _align_block_size_jit, _align_block_size_torch, _fused_virtual_topk_ids, diff --git a/test/registered/unit/lora/test_experimental_sgl_marlin_multi_prefill.py b/test/registered/unit/lora/test_experimental_sgl_marlin_multi_prefill.py index 25bb30675c..2dc7c90316 100644 --- a/test/registered/unit/lora/test_experimental_sgl_marlin_multi_prefill.py +++ b/test/registered/unit/lora/test_experimental_sgl_marlin_multi_prefill.py @@ -121,7 +121,7 @@ def _run_factored_pipeline( collapsed_routing_cache: dict, routed_scaling_factor: float, ) -> None: - from sglang.kernels.ops.moe.trtllm_lora_temp.virtual_experts import ( + from sglang.kernels.ops.lora.moe.trtllm_lora_temp.virtual_experts import ( merged_experts_fused_moe_lora_add, ) from sglang.srt.lora.marlin_lora_temp.shared_outer import weighted_topk_rank_sum diff --git a/test/registered/unit/lora/test_experimental_sgl_marlin_runtime_unit.py b/test/registered/unit/lora/test_experimental_sgl_marlin_runtime_unit.py index 5cd7a3bd0d..d92c7df229 100644 --- a/test/registered/unit/lora/test_experimental_sgl_marlin_runtime_unit.py +++ b/test/registered/unit/lora/test_experimental_sgl_marlin_runtime_unit.py @@ -273,7 +273,7 @@ def _run_marlin_flow( ) _stub_module( monkeypatch, - "sglang.kernels.ops.moe.trtllm_lora_temp.virtual_experts", + "sglang.kernels.ops.lora.moe.trtllm_lora_temp.virtual_experts", merged_experts_fused_moe_lora_add=merged_experts_fused_moe_lora_add, ) _stub_module(