[LoRA] Reorganize kernels and add CODEOWNERS (#42299)
This commit is contained in:
@@ -26,13 +26,20 @@ sglang/kernels/
|
||||
|
||||
Operator groups (all populated): `activation`, `attention`, `communication`,
|
||||
`diffusion`, `elementwise`, `embeddings`, `gemm`, `grammar`, `kv_canary`,
|
||||
`kvcache`, `layernorm`, `lplb`, `mamba`, `memory`, `mm`, `moe`, `quantization`,
|
||||
`sampling`, `speculative`.
|
||||
`kvcache`, `layernorm`, `lora`, `lplb`, `mamba`, `memory`, `mm`, `moe`,
|
||||
`quantization`, `sampling`, `speculative`.
|
||||
|
||||
Place model-specific implementations and tuning data inside the corresponding
|
||||
logical operator group, rather than creating a model-specific top-level group
|
||||
under `ops/`.
|
||||
|
||||
`lora` is its own group because it has a distinct responsibility: the adapter
|
||||
kernels that only the LoRA runtime in `srt/lora` calls (`dense/` SGMV GEMMs and
|
||||
their tuning data, `moe/` fused MoE-LoRA). It also keeps the experimental
|
||||
TRT-LLM LoRA path whole as `dense/trtllm_lora_temp/` and
|
||||
`moe/trtllm_lora_temp/`; the latter includes the FlashInfer overlay and the
|
||||
flag-gated routing kernels that `srt/layers/moe/topk.py` reaches.
|
||||
|
||||
As of the RFC #29630 finale (#32072) the legacy `sglang.jit_kernel` package has
|
||||
been **removed**: its shared build/runtime infra moved to `sglang.kernels.jit`
|
||||
and each JIT-backed operator into its group as
|
||||
|
||||
+1
-1
@@ -58,7 +58,7 @@ __device__ float convert_to_float(T x) {
|
||||
}
|
||||
}
|
||||
|
||||
// Reference pack (bit-identical to kernels/ops/moe/trtllm_lora_temp/topk_pack.py):
|
||||
// Reference pack (bit-identical to kernels/ops/lora/moe/trtllm_lora_temp/topk_pack.py):
|
||||
// low 16 bits = bf16(weight) bits (round-to-nearest-even, same as torch/triton
|
||||
// `.to(bfloat16)`), high 16 bits = int16 expert id.
|
||||
__device__ __forceinline__ int32_t pack_routed(int32_t id, float w) {
|
||||
@@ -25,6 +25,7 @@ _GROUPS = (
|
||||
"grammar",
|
||||
"kvcache",
|
||||
"layernorm",
|
||||
"lora",
|
||||
"mamba",
|
||||
"memory",
|
||||
"mm",
|
||||
|
||||
@@ -397,33 +397,6 @@ __all__ = [
|
||||
]
|
||||
|
||||
|
||||
# LoRA SGMV Triton kernels migrated into this group (from lora/triton_ops);
|
||||
# registered for inventory. Import them from their modules.
|
||||
_TRITON_KERNELS = [
|
||||
("chunked_embedding_lora_a", "chunked_embedding_lora_a_forward"),
|
||||
("chunked_sgmv_expand", "chunked_sgmv_lora_expand_forward"),
|
||||
("chunked_sgmv_shrink", "chunked_sgmv_lora_shrink_forward"),
|
||||
("embedding_lora_a", "embedding_lora_a_fwd"),
|
||||
("gate_up_lora_b", "gate_up_lora_b_fwd"),
|
||||
("qkv_lora_b", "qkv_lora_b_fwd"),
|
||||
("sgemm_lora_a", "sgemm_lora_a_fwd"),
|
||||
("sgemm_lora_b", "sgemm_lora_b_fwd"),
|
||||
("kv_b_lora_absorbed", "step_a_q_fwd"),
|
||||
("kv_b_lora_absorbed", "step_b_q_fwd"),
|
||||
("kv_b_lora_absorbed", "step_a_v_fwd"),
|
||||
("kv_b_lora_absorbed", "step_b_v_fwd"),
|
||||
]
|
||||
for _mod, _fn in _TRITON_KERNELS:
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op=f"gemm.{_fn}",
|
||||
backend=KernelBackend.TRITON,
|
||||
target=f"sglang.kernels.ops.gemm.{_mod}:{_fn}",
|
||||
)
|
||||
)
|
||||
del _mod, _fn
|
||||
|
||||
|
||||
# Public entry points inventoried by logical operator group (RFC #29630).
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
"""LoRA adapter kernels that only the LoRA runtime calls: ``dense/`` SGMV GEMMs,
|
||||
``moe/`` fused MoE-LoRA, and the experimental TRT-LLM LoRA path."""
|
||||
|
||||
from sglang.kernels.registry import register_kernel
|
||||
from sglang.kernels.spec import KernelBackend, KernelSpec
|
||||
|
||||
# Triton kernels registered for inventory. Import them from their modules.
|
||||
_TRITON_KERNELS = [
|
||||
("dense.chunked_embedding_lora_a", "chunked_embedding_lora_a_forward"),
|
||||
("dense.chunked_sgmv_expand", "chunked_sgmv_lora_expand_forward"),
|
||||
("dense.chunked_sgmv_shrink", "chunked_sgmv_lora_shrink_forward"),
|
||||
("dense.embedding_lora_a", "embedding_lora_a_fwd"),
|
||||
("dense.gate_up_lora_b", "gate_up_lora_b_fwd"),
|
||||
("dense.qkv_lora_b", "qkv_lora_b_fwd"),
|
||||
("dense.sgemm_lora_a", "sgemm_lora_a_fwd"),
|
||||
("dense.sgemm_lora_b", "sgemm_lora_b_fwd"),
|
||||
("dense.kv_b_lora_absorbed", "step_a_q_fwd"),
|
||||
("dense.kv_b_lora_absorbed", "step_b_q_fwd"),
|
||||
("dense.kv_b_lora_absorbed", "step_a_v_fwd"),
|
||||
("dense.kv_b_lora_absorbed", "step_b_v_fwd"),
|
||||
("moe.fused_moe_lora_kernel", "fused_moe_lora"),
|
||||
("moe.virtual_experts", "merged_experts_fused_moe_lora_add"),
|
||||
]
|
||||
for _mod, _fn in _TRITON_KERNELS:
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op=f"lora.{_fn}",
|
||||
backend=KernelBackend.TRITON,
|
||||
target=f"sglang.kernels.ops.lora.{_mod}:{_fn}",
|
||||
)
|
||||
)
|
||||
del _mod, _fn
|
||||
|
||||
__all__: list[str] = []
|
||||
@@ -0,0 +1,2 @@
|
||||
"""Triton LoRA GEMM kernels (SGMV, chunked SGMV, absorbed-MLA correction) and the
|
||||
CSGMV tuning-config loader with its ``csgmv_configs/`` data."""
|
||||
+1
-1
@@ -4,7 +4,7 @@ import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.kernels.ops.gemm.lora_tuning_config import get_lora_expand_config
|
||||
from sglang.kernels.ops.lora.dense.lora_tuning_config import get_lora_expand_config
|
||||
from sglang.srt.lora.utils import LoRABatchInfo
|
||||
from sglang.srt.utils import cached_triton_kernel
|
||||
|
||||
+1
-1
@@ -2,7 +2,7 @@ import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.kernels.ops.gemm.lora_tuning_config import get_lora_shrink_config
|
||||
from sglang.kernels.ops.lora.dense.lora_tuning_config import get_lora_shrink_config
|
||||
from sglang.srt.lora.utils import LoRABatchInfo
|
||||
from sglang.srt.utils import cached_triton_kernel
|
||||
|
||||
+1
-1
@@ -2,7 +2,7 @@ import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.kernels.ops.gemm.kernel_utils import _resolve_token_positions
|
||||
from sglang.kernels.ops.lora.dense.kernel_utils import _resolve_token_positions
|
||||
from sglang.srt.lora.utils import LoRABatchInfo
|
||||
|
||||
|
||||
+1
-1
@@ -48,7 +48,7 @@ import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.kernels.ops.gemm.kernel_utils import _resolve_token_positions
|
||||
from sglang.kernels.ops.lora.dense.kernel_utils import _resolve_token_positions
|
||||
from sglang.srt.lora.utils import LoRABatchInfo
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
+1
-1
@@ -20,7 +20,7 @@ Usage:
|
||||
python3 benchmark/kernels/lora_csgmv/tune_lora_csgmv.py \
|
||||
--model Qwen/Qwen3-Embedding-0.6B --max-lora-rank 64
|
||||
|
||||
# Configs saved to python/sglang/kernels/ops/gemm/configs/
|
||||
# Configs saved to python/sglang/kernels/ops/lora/dense/csgmv_configs/
|
||||
|
||||
# Server automatically picks them up:
|
||||
python3 -m sglang.launch_server --model ... --enable-lora --lora-backend csgmv
|
||||
+1
-1
@@ -2,7 +2,7 @@ import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.kernels.ops.gemm.kernel_utils import _resolve_token_positions
|
||||
from sglang.kernels.ops.lora.dense.kernel_utils import _resolve_token_positions
|
||||
from sglang.srt.lora.utils import LoRABatchInfo
|
||||
|
||||
|
||||
+1
-1
@@ -2,7 +2,7 @@ import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.kernels.ops.gemm.kernel_utils import _resolve_token_positions
|
||||
from sglang.kernels.ops.lora.dense.kernel_utils import _resolve_token_positions
|
||||
from sglang.srt.lora.utils import LoRABatchInfo
|
||||
|
||||
|
||||
+1
-1
@@ -2,7 +2,7 @@ import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.kernels.ops.gemm.kernel_utils import _resolve_token_positions
|
||||
from sglang.kernels.ops.lora.dense.kernel_utils import _resolve_token_positions
|
||||
from sglang.srt.lora.utils import LoRABatchInfo
|
||||
|
||||
|
||||
+1
-1
@@ -2,7 +2,7 @@ import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.kernels.ops.gemm.trtllm_lora_temp.kernel_utils import (
|
||||
from sglang.kernels.ops.lora.dense.trtllm_lora_temp.kernel_utils import (
|
||||
_resolve_token_positions,
|
||||
get_pdl_launch_metadata,
|
||||
)
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
from sglang.kernels.jit.utils import is_arch_support_pdl
|
||||
from sglang.kernels.ops.gemm.kernel_utils import (
|
||||
from sglang.kernels.ops.lora.dense.kernel_utils import (
|
||||
_resolve_token_positions as _resolve_token_positions,
|
||||
)
|
||||
|
||||
+1
-1
@@ -48,7 +48,7 @@ import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.kernels.ops.gemm.trtllm_lora_temp.kernel_utils import (
|
||||
from sglang.kernels.ops.lora.dense.trtllm_lora_temp.kernel_utils import (
|
||||
_resolve_token_positions,
|
||||
get_pdl_launch_metadata,
|
||||
)
|
||||
+1
-1
@@ -4,7 +4,7 @@ import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.kernels.ops.gemm.trtllm_lora_temp.kernel_utils import (
|
||||
from sglang.kernels.ops.lora.dense.trtllm_lora_temp.kernel_utils import (
|
||||
_resolve_token_positions,
|
||||
get_pdl_launch_metadata,
|
||||
)
|
||||
+1
-1
@@ -4,7 +4,7 @@ import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.kernels.ops.gemm.trtllm_lora_temp.kernel_utils import (
|
||||
from sglang.kernels.ops.lora.dense.trtllm_lora_temp.kernel_utils import (
|
||||
_resolve_token_positions,
|
||||
get_pdl_launch_metadata,
|
||||
)
|
||||
+2
-2
@@ -2,10 +2,10 @@ import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.kernels.ops.gemm.trtllm_lora_temp.gate_up_lora_b import (
|
||||
from sglang.kernels.ops.lora.dense.trtllm_lora_temp.gate_up_lora_b import (
|
||||
_CUBLAS_MIN_S_RANK,
|
||||
)
|
||||
from sglang.kernels.ops.gemm.trtllm_lora_temp.kernel_utils import (
|
||||
from sglang.kernels.ops.lora.dense.trtllm_lora_temp.kernel_utils import (
|
||||
_resolve_token_positions,
|
||||
get_pdl_launch_metadata,
|
||||
)
|
||||
@@ -0,0 +1,2 @@
|
||||
"""Fused MoE-LoRA kernels: the fused shrink/expand Triton kernel, LoRA-aware
|
||||
block alignment (JIT), and the merged virtual-expert path."""
|
||||
+1
-2
@@ -2,8 +2,7 @@
|
||||
|
||||
Migrated from ``sglang.srt.lora.trtllm_lora_temp.triton_ops`` (RFC #29630)."""
|
||||
|
||||
# --- merged from sglang.kernels.ops.moe.trtllm_lora_temp (RFC #29630 Phase 4) ---
|
||||
from sglang.kernels.ops.moe.trtllm_lora_temp.core import (
|
||||
from sglang.kernels.ops.lora.moe.trtllm_lora_temp.core import (
|
||||
trtllm_bf16_routed_moe_lora,
|
||||
trtllm_fp4_block_scale_moe_lora_finalize,
|
||||
trtllm_fp4_block_scale_routed_moe_lora,
|
||||
+2
-2
@@ -10,7 +10,7 @@ from sglang.srt.lora.trtllm_lora_temp.environ import lora_envs
|
||||
def get_sgl_trtllm_moe_sm100_module():
|
||||
import flashinfer.fused_moe.core as fi_core
|
||||
|
||||
from sglang.kernels.ops.moe.trtllm_lora_temp.jit import (
|
||||
from sglang.kernels.ops.lora.moe.trtllm_lora_temp.jit import (
|
||||
gen_sgl_trtllm_gen_fused_moe_sm100_module,
|
||||
)
|
||||
|
||||
@@ -30,7 +30,7 @@ def get_sgl_trtllm_moe_sm100_module():
|
||||
def get_sgl_trtllm_moe_sm100_raw_module():
|
||||
from flashinfer.fused_moe.core import setup_cubin_loader
|
||||
|
||||
from sglang.kernels.ops.moe.trtllm_lora_temp.jit import (
|
||||
from sglang.kernels.ops.lora.moe.trtllm_lora_temp.jit import (
|
||||
gen_sgl_trtllm_gen_fused_moe_sm100_module,
|
||||
)
|
||||
|
||||
+1
-1
@@ -14,7 +14,7 @@ if TYPE_CHECKING:
|
||||
def _jit_kimi_k2_moe_fused_gate_module() -> Module:
|
||||
return load_jit(
|
||||
"kimi_k2_moe_fused_gate",
|
||||
cuda_files=["trtllm_lora_temp/kimi_k2_moe_fused_gate.cuh"],
|
||||
cuda_files=["lora/trtllm_lora_temp/kimi_k2_moe_fused_gate.cuh"],
|
||||
cuda_wrappers=[
|
||||
("kimi_k2_moe_fused_gate", "KimiK2MoEFusedGateKernel::run"),
|
||||
],
|
||||
+1
-1
@@ -16,7 +16,7 @@ def _jit_module(dtype: torch.dtype) -> Module:
|
||||
return load_jit(
|
||||
"moe_lora_merged_align",
|
||||
*args,
|
||||
cuda_files=["trtllm_lora_temp/moe_lora_merged_align_kernel.cu"],
|
||||
cuda_files=["lora/trtllm_lora_temp/moe_lora_merged_align_kernel.cu"],
|
||||
cuda_wrappers=[
|
||||
("moe_lora_merged_align", f"MoeLoraMergedAlignKernel<{args}>::run"),
|
||||
],
|
||||
+1
-1
@@ -31,7 +31,7 @@ if TYPE_CHECKING:
|
||||
def _jit_topk_softmax_pack_module() -> Module:
|
||||
return load_jit(
|
||||
"topk_softmax_pack",
|
||||
cuda_files=["trtllm_lora_temp/topk_softmax_pack.cuh"],
|
||||
cuda_files=["lora/trtllm_lora_temp/topk_softmax_pack.cuh"],
|
||||
cuda_wrappers=[("topk_softmax_pack", "topk_softmax_pack")],
|
||||
)
|
||||
|
||||
+4
-4
@@ -9,10 +9,10 @@ import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.kernels.ops.gemm.trtllm_lora_temp.kernel_utils import (
|
||||
from sglang.kernels.ops.lora.dense.trtllm_lora_temp.kernel_utils import (
|
||||
get_pdl_launch_metadata,
|
||||
)
|
||||
from sglang.kernels.ops.moe.virtual_experts import _align_block_size_large
|
||||
from sglang.kernels.ops.lora.moe.virtual_experts import _align_block_size_large
|
||||
from sglang.srt.lora.trtllm_lora_temp.environ import lora_envs
|
||||
|
||||
|
||||
@@ -394,7 +394,7 @@ def _get_moe_lora_shrink_split_k(
|
||||
return max(1, min(triton.cdiv(target, base_grid), max_split_k, 8))
|
||||
|
||||
|
||||
# Rank-specialized LoRA-B expand kernel lives in lora/trtllm_lora_temp/.
|
||||
# Rank-specialized LoRA-B expand kernel lives in srt/lora/trtllm_lora_temp/.
|
||||
# Re-export so existing call sites (and any external imports) keep working.
|
||||
from sglang.srt.lora.trtllm_lora_temp.specialized_expand import ( # noqa: E402,F401
|
||||
_invoke_moe_lora_expand_add,
|
||||
@@ -574,7 +574,7 @@ def _merged_experts_fused_moe_lora_add_impl(
|
||||
and bucket_experts + 1 <= 1024
|
||||
and topk_ids.shape[0] < 512
|
||||
):
|
||||
from sglang.kernels.ops.moe.trtllm_lora_temp.moe_lora_merged_align import (
|
||||
from sglang.kernels.ops.lora.moe.trtllm_lora_temp.moe_lora_merged_align import (
|
||||
moe_lora_merged_align,
|
||||
)
|
||||
|
||||
@@ -161,23 +161,6 @@ def situ_and_mul_masked_post_quant(
|
||||
__all__ = ["situ_and_mul_masked_post_quant", "moe_align_block_size", "topk_softmax"]
|
||||
|
||||
|
||||
# Fused MoE-LoRA Triton kernels migrated into this group (from lora/triton_ops);
|
||||
# registered for inventory. Import them from their modules.
|
||||
_TRITON_KERNELS = [
|
||||
("fused_moe_lora_kernel", "fused_moe_lora"),
|
||||
("virtual_experts", "merged_experts_fused_moe_lora_add"),
|
||||
]
|
||||
for _mod, _fn in _TRITON_KERNELS:
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op=f"moe.{_fn}",
|
||||
backend=KernelBackend.TRITON,
|
||||
target=f"sglang.kernels.ops.moe.{_mod}:{_fn}",
|
||||
)
|
||||
)
|
||||
del _mod, _fn
|
||||
|
||||
|
||||
# Triton kernels migrated from srt/layers/moe (RFC #29630, Phase 2.5);
|
||||
# registered for inventory. Import them from their modules.
|
||||
_PHASE25_TRITON_KERNELS = [
|
||||
|
||||
@@ -43,7 +43,7 @@ def _fused_unpack_packed_topk(packed: torch.Tensor):
|
||||
|
||||
Returns (topk_ids int32, topk_weights float32). Collapses the ~5 elementwise
|
||||
ops (shift / mask / int16 / bitcast / cast) the torch reference emits per call
|
||||
into one Triton launch. Mirrors _pack_topk_kernel (trtllm_lora_temp/topk_pack).
|
||||
into one Triton launch. Mirrors _pack_topk_kernel (ops/lora/moe/trtllm_lora_temp/topk_pack).
|
||||
"""
|
||||
packed = packed.contiguous()
|
||||
ids = torch.empty_like(packed, dtype=torch.int32)
|
||||
|
||||
@@ -77,7 +77,7 @@ def moe_align_block_size(
|
||||
from sglang.srt.lora.trtllm_lora_temp.environ import lora_envs
|
||||
|
||||
if lora_envs.SGLANG_OPT_USE_JIT_KERNEL_MOE_ALIGN.get() and num_experts <= 8191:
|
||||
from sglang.kernels.ops.moe.trtllm_lora_temp.virtual_experts import (
|
||||
from sglang.kernels.ops.lora.moe.virtual_experts import (
|
||||
_align_block_size_jit,
|
||||
)
|
||||
|
||||
|
||||
@@ -1061,7 +1061,7 @@ def fused_topk(
|
||||
elif packed_out is not None:
|
||||
# Fused gating + routed pack (SGLANG_OPT_LORA_FUSED_TOPK_PACK): one JIT kernel
|
||||
# writes topk_weights/topk_ids AND the FlashInfer packed topk in one launch.
|
||||
from sglang.kernels.ops.moe.trtllm_lora_temp.topk_softmax_pack import (
|
||||
from sglang.kernels.ops.lora.moe.trtllm_lora_temp.topk_softmax_pack import (
|
||||
topk_softmax_pack,
|
||||
)
|
||||
|
||||
@@ -1966,7 +1966,7 @@ def biased_grouped_topk_gpu(
|
||||
and lora_envs.SGLANG_OPT_KIMI_GATE_BF16_INPUT.get()
|
||||
)
|
||||
if _use_jit_bf16_gate:
|
||||
from sglang.kernels.ops.moe.trtllm_lora_temp.kimi_k2_moe_fused_gate import (
|
||||
from sglang.kernels.ops.lora.moe.trtllm_lora_temp.kimi_k2_moe_fused_gate import (
|
||||
kimi_k2_moe_fused_gate as _kimi_k2_moe_fused_gate,
|
||||
)
|
||||
|
||||
|
||||
@@ -3,11 +3,15 @@ from typing import List, Optional, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.gemm.chunked_embedding_lora_a import (
|
||||
from sglang.kernels.ops.lora.dense.chunked_embedding_lora_a import (
|
||||
chunked_embedding_lora_a_forward,
|
||||
)
|
||||
from sglang.kernels.ops.gemm.chunked_sgmv_expand import chunked_sgmv_lora_expand_forward
|
||||
from sglang.kernels.ops.gemm.chunked_sgmv_shrink import chunked_sgmv_lora_shrink_forward
|
||||
from sglang.kernels.ops.lora.dense.chunked_sgmv_expand import (
|
||||
chunked_sgmv_lora_expand_forward,
|
||||
)
|
||||
from sglang.kernels.ops.lora.dense.chunked_sgmv_shrink import (
|
||||
chunked_sgmv_lora_shrink_forward,
|
||||
)
|
||||
from sglang.srt.lora.backend.base_backend import BaseLoRABackend
|
||||
from sglang.srt.lora.utils import (
|
||||
LoRABatchInfo,
|
||||
|
||||
@@ -3,11 +3,11 @@ from typing import List, Optional, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.gemm.embedding_lora_a import embedding_lora_a_fwd
|
||||
from sglang.kernels.ops.gemm.gate_up_lora_b import gate_up_lora_b_fwd
|
||||
from sglang.kernels.ops.gemm.qkv_lora_b import qkv_lora_b_fwd
|
||||
from sglang.kernels.ops.gemm.sgemm_lora_a import sgemm_lora_a_fwd
|
||||
from sglang.kernels.ops.gemm.sgemm_lora_b import sgemm_lora_b_fwd
|
||||
from sglang.kernels.ops.lora.dense.embedding_lora_a import embedding_lora_a_fwd
|
||||
from sglang.kernels.ops.lora.dense.gate_up_lora_b import gate_up_lora_b_fwd
|
||||
from sglang.kernels.ops.lora.dense.qkv_lora_b import qkv_lora_b_fwd
|
||||
from sglang.kernels.ops.lora.dense.sgemm_lora_a import sgemm_lora_a_fwd
|
||||
from sglang.kernels.ops.lora.dense.sgemm_lora_b import sgemm_lora_b_fwd
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
DpPaddingMode,
|
||||
|
||||
@@ -19,7 +19,7 @@ from typing import TYPE_CHECKING, Optional, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.gemm.kv_b_lora_absorbed import (
|
||||
from sglang.kernels.ops.lora.dense.kv_b_lora_absorbed import (
|
||||
step_a_q_fwd,
|
||||
step_a_v_fwd,
|
||||
step_b_q_fwd,
|
||||
|
||||
@@ -37,7 +37,7 @@ _is_hip = is_hip()
|
||||
_is_xpu = is_xpu()
|
||||
|
||||
if _is_cuda or _is_hip or _is_xpu:
|
||||
from sglang.kernels.ops.moe.moe_lora_align import moe_lora_align_block_size
|
||||
from sglang.kernels.ops.lora.moe.moe_lora_align import moe_lora_align_block_size
|
||||
|
||||
|
||||
def _get_moe_lora_block_config(max_lora_rank: int) -> dict:
|
||||
@@ -316,8 +316,10 @@ def _add_lora_gate_up_delta(
|
||||
routing_cache: dict | None = None,
|
||||
) -> None:
|
||||
"""Add LoRA gate_up delta to intermediate_cache in-place."""
|
||||
from sglang.kernels.ops.moe.fused_moe_lora_kernel import fused_moe_lora
|
||||
from sglang.kernels.ops.moe.virtual_experts import merged_experts_fused_moe_lora_add
|
||||
from sglang.kernels.ops.lora.moe.fused_moe_lora_kernel import fused_moe_lora
|
||||
from sglang.kernels.ops.lora.moe.virtual_experts import (
|
||||
merged_experts_fused_moe_lora_add,
|
||||
)
|
||||
|
||||
if lora_info is None or lora_info.max_lora_rank == 0:
|
||||
return
|
||||
@@ -407,8 +409,10 @@ def _add_lora_down_delta(
|
||||
routing_cache: dict | None = None,
|
||||
) -> None:
|
||||
"""Add LoRA down delta to intermediate_cache in-place."""
|
||||
from sglang.kernels.ops.moe.fused_moe_lora_kernel import fused_moe_lora
|
||||
from sglang.kernels.ops.moe.virtual_experts import merged_experts_fused_moe_lora_add
|
||||
from sglang.kernels.ops.lora.moe.fused_moe_lora_kernel import fused_moe_lora
|
||||
from sglang.kernels.ops.lora.moe.virtual_experts import (
|
||||
merged_experts_fused_moe_lora_add,
|
||||
)
|
||||
|
||||
if lora_info.max_lora_rank == 0:
|
||||
return
|
||||
|
||||
@@ -24,13 +24,13 @@ _is_cuda = is_cuda()
|
||||
if _is_cuda:
|
||||
from sgl_kernel import silu_and_mul
|
||||
|
||||
from sglang.kernels.ops.lora.moe.virtual_experts import (
|
||||
_align_block_size_jit as moe_align_block_size,
|
||||
)
|
||||
from sglang.kernels.ops.moe.fused_moe_triton_kernels import (
|
||||
moe_sum_reduce_triton,
|
||||
)
|
||||
from sglang.kernels.ops.moe.moe_wna16_marlin import moe_wna16_marlin_gemm
|
||||
from sglang.kernels.ops.moe.virtual_experts import (
|
||||
_align_block_size_jit as moe_align_block_size,
|
||||
)
|
||||
from sglang.srt.layers.moe.fused_moe_triton.fused_marlin_moe import (
|
||||
get_scalar_type,
|
||||
)
|
||||
@@ -208,7 +208,7 @@ def fused_experts_experimental_sgl_marlin_lora(
|
||||
runner_config: MoeRunnerConfig,
|
||||
lora_info,
|
||||
) -> StandardCombineInput:
|
||||
from sglang.kernels.ops.moe.trtllm_lora_temp.virtual_experts import (
|
||||
from sglang.kernels.ops.lora.moe.trtllm_lora_temp.virtual_experts import (
|
||||
merged_experts_fused_moe_lora_add,
|
||||
)
|
||||
from sglang.srt.layers.moe.token_dispatcher.standard import (
|
||||
|
||||
@@ -42,8 +42,10 @@ def qkv_proj_lora_forward(self, input_: torch.Tensor):
|
||||
):
|
||||
return get_original_qkv_forward()(self, input_)
|
||||
|
||||
from sglang.kernels.ops.gemm.trtllm_lora_temp.qkv_lora_b import qkv_lora_b_fwd
|
||||
from sglang.kernels.ops.gemm.trtllm_lora_temp.sgemm_lora_a import sgemm_lora_a_fwd
|
||||
from sglang.kernels.ops.lora.dense.trtllm_lora_temp.qkv_lora_b import qkv_lora_b_fwd
|
||||
from sglang.kernels.ops.lora.dense.trtllm_lora_temp.sgemm_lora_a import (
|
||||
sgemm_lora_a_fwd,
|
||||
)
|
||||
|
||||
bias = self.base_layer.bias if not self.base_layer.skip_bias_add else None
|
||||
side_stream = get_lora_side_stream()
|
||||
@@ -121,10 +123,10 @@ def row_parallel_lora_forward(
|
||||
_alloc = lora_overlap_alloc_stream() # capture MAIN stream here (before the fork)
|
||||
side_stream.wait_stream(torch.cuda.current_stream())
|
||||
with torch.cuda.stream(side_stream):
|
||||
from sglang.kernels.ops.gemm.trtllm_lora_temp.sgemm_lora_a import (
|
||||
from sglang.kernels.ops.lora.dense.trtllm_lora_temp.sgemm_lora_a import (
|
||||
sgemm_lora_a_fwd,
|
||||
)
|
||||
from sglang.kernels.ops.gemm.trtllm_lora_temp.sgemm_lora_b import (
|
||||
from sglang.kernels.ops.lora.dense.trtllm_lora_temp.sgemm_lora_b import (
|
||||
sgemm_lora_b_fwd,
|
||||
)
|
||||
|
||||
@@ -189,10 +191,10 @@ def column_parallel_lora_forward(self, input_: torch.Tensor):
|
||||
_alloc = lora_overlap_alloc_stream() # capture MAIN stream here (before the fork)
|
||||
side_stream.wait_stream(torch.cuda.current_stream())
|
||||
with torch.cuda.stream(side_stream):
|
||||
from sglang.kernels.ops.gemm.trtllm_lora_temp.sgemm_lora_a import (
|
||||
from sglang.kernels.ops.lora.dense.trtllm_lora_temp.sgemm_lora_a import (
|
||||
sgemm_lora_a_fwd,
|
||||
)
|
||||
from sglang.kernels.ops.gemm.trtllm_lora_temp.sgemm_lora_b import (
|
||||
from sglang.kernels.ops.lora.dense.trtllm_lora_temp.sgemm_lora_b import (
|
||||
sgemm_lora_b_fwd,
|
||||
)
|
||||
|
||||
@@ -245,10 +247,10 @@ def replicated_lora_forward(self, x: torch.Tensor):
|
||||
_alloc = lora_overlap_alloc_stream() # capture MAIN stream here (before the fork)
|
||||
side_stream.wait_stream(torch.cuda.current_stream())
|
||||
with torch.cuda.stream(side_stream):
|
||||
from sglang.kernels.ops.gemm.trtllm_lora_temp.sgemm_lora_a import (
|
||||
from sglang.kernels.ops.lora.dense.trtllm_lora_temp.sgemm_lora_a import (
|
||||
sgemm_lora_a_fwd,
|
||||
)
|
||||
from sglang.kernels.ops.gemm.trtllm_lora_temp.sgemm_lora_b import (
|
||||
from sglang.kernels.ops.lora.dense.trtllm_lora_temp.sgemm_lora_b import (
|
||||
sgemm_lora_b_fwd,
|
||||
)
|
||||
|
||||
@@ -267,7 +269,9 @@ def replicated_lora_forward(self, x: torch.Tensor):
|
||||
if first_dim == 0:
|
||||
output = sgemm_lora_b_fwd(lora_a_output, self.B_buffer, sgemm_info, output)
|
||||
else:
|
||||
from sglang.kernels.ops.gemm.trtllm_lora_temp.qkv_lora_b import qkv_lora_b_fwd
|
||||
from sglang.kernels.ops.lora.dense.trtllm_lora_temp.qkv_lora_b import (
|
||||
qkv_lora_b_fwd,
|
||||
)
|
||||
|
||||
output = qkv_lora_b_fwd(
|
||||
lora_a_output,
|
||||
|
||||
@@ -29,16 +29,16 @@ step_a_q_fwd = step_a_v_fwd = step_b_q_fwd = step_b_v_fwd = None
|
||||
def _ensure_step_kernels() -> None:
|
||||
global step_a_q_fwd, step_a_v_fwd, step_b_q_fwd, step_b_v_fwd
|
||||
if step_a_q_fwd is None:
|
||||
from sglang.kernels.ops.gemm.trtllm_lora_temp.kv_b_lora_absorbed import (
|
||||
from sglang.kernels.ops.lora.dense.trtllm_lora_temp.kv_b_lora_absorbed import (
|
||||
step_a_q_fwd as _aq,
|
||||
)
|
||||
from sglang.kernels.ops.gemm.trtllm_lora_temp.kv_b_lora_absorbed import (
|
||||
from sglang.kernels.ops.lora.dense.trtllm_lora_temp.kv_b_lora_absorbed import (
|
||||
step_a_v_fwd as _av,
|
||||
)
|
||||
from sglang.kernels.ops.gemm.trtllm_lora_temp.kv_b_lora_absorbed import (
|
||||
from sglang.kernels.ops.lora.dense.trtllm_lora_temp.kv_b_lora_absorbed import (
|
||||
step_b_q_fwd as _bq,
|
||||
)
|
||||
from sglang.kernels.ops.gemm.trtllm_lora_temp.kv_b_lora_absorbed import (
|
||||
from sglang.kernels.ops.lora.dense.trtllm_lora_temp.kv_b_lora_absorbed import (
|
||||
step_b_v_fwd as _bv,
|
||||
)
|
||||
|
||||
|
||||
@@ -108,7 +108,7 @@ class _LoraEnvs:
|
||||
# diagnostics / tuning:
|
||||
SGLANG_OPT_LORA_SHRINK_TUNE = _GatedBool("SGLANG_OPT_LORA_SHRINK_TUNE", False)
|
||||
|
||||
# kimi NVFP4 permute+quant fuse — read in kernels/ops/moe/trtllm_lora_temp/core.py (Python) to pass
|
||||
# kimi NVFP4 permute+quant fuse — read in kernels/ops/lora/moe/trtllm_lora_temp/core.py (Python) to pass
|
||||
# a bool to the kernel, AND C++-side via getenv in the launcher. Default off (kimi-only).
|
||||
SGLANG_OPT_FUSED_PERMUTE_QUANT = _GatedBool("SGLANG_OPT_FUSED_PERMUTE_QUANT", False)
|
||||
|
||||
|
||||
@@ -70,7 +70,7 @@ def sgl_trtllm_fp8_block_scale_moe_wrapper(
|
||||
"TRTLLM enums and cubin-loader utilities."
|
||||
) from e
|
||||
|
||||
from sglang.kernels.ops.moe.trtllm_lora_temp import trtllm_fp8_block_scale_moe
|
||||
from sglang.kernels.ops.lora.moe.trtllm_lora_temp import trtllm_fp8_block_scale_moe
|
||||
|
||||
kwargs = {
|
||||
"routing_logits": routing_logits,
|
||||
@@ -168,7 +168,7 @@ def sgl_trtllm_fp8_block_scale_routed_moe_wrapper(
|
||||
"TRTLLM enums and cubin-loader utilities."
|
||||
) from e
|
||||
|
||||
from sglang.kernels.ops.moe.trtllm_lora_temp import (
|
||||
from sglang.kernels.ops.lora.moe.trtllm_lora_temp import (
|
||||
trtllm_fp8_block_scale_routed_moe,
|
||||
)
|
||||
|
||||
|
||||
@@ -41,10 +41,10 @@ def apply_multi_lora(
|
||||
) -> torch.Tensor:
|
||||
if getattr(layer.lora_backend, "name", None) != "triton" or not inputs.is_cuda:
|
||||
raise RuntimeError("Multi-slot dense LoRA requires Triton on CUDA")
|
||||
from sglang.kernels.ops.gemm.trtllm_lora_temp.sgemm_lora_a import (
|
||||
from sglang.kernels.ops.lora.dense.trtllm_lora_temp.sgemm_lora_a import (
|
||||
shared_sink_sgemm_lora_a_fwd,
|
||||
)
|
||||
from sglang.kernels.ops.gemm.trtllm_lora_temp.sgemm_lora_b import (
|
||||
from sglang.kernels.ops.lora.dense.trtllm_lora_temp.sgemm_lora_b import (
|
||||
shared_sink_sgemm_lora_b_fwd,
|
||||
)
|
||||
|
||||
@@ -86,7 +86,7 @@ def _apply_per_expert_lora(
|
||||
lora_a: torch.Tensor,
|
||||
lora_b: torch.Tensor,
|
||||
) -> None:
|
||||
from sglang.kernels.ops.moe.virtual_experts import (
|
||||
from sglang.kernels.ops.lora.moe.virtual_experts import (
|
||||
merged_experts_fused_moe_lora_add,
|
||||
)
|
||||
|
||||
@@ -156,7 +156,7 @@ def forward_with_lora(
|
||||
if single_gate_up:
|
||||
torch.mm(x_td, a_gate_up.T, out=gate_up_shrink)
|
||||
else:
|
||||
from sglang.kernels.ops.gemm.trtllm_lora_temp.sgemm_lora_a import (
|
||||
from sglang.kernels.ops.lora.dense.trtllm_lora_temp.sgemm_lora_a import (
|
||||
shared_sink_sgemm_lora_a_fwd,
|
||||
)
|
||||
|
||||
@@ -190,7 +190,7 @@ def forward_with_lora(
|
||||
y_flat.addmm_(gate_up_shrink, layer._w1_delta[0].T)
|
||||
y = y_flat.view(t, n, -1)
|
||||
else:
|
||||
from sglang.kernels.ops.gemm.trtllm_lora_temp.sgemm_lora_b import (
|
||||
from sglang.kernels.ops.lora.dense.trtllm_lora_temp.sgemm_lora_b import (
|
||||
shared_sink_sgemm_lora_b_fwd,
|
||||
)
|
||||
|
||||
|
||||
@@ -47,12 +47,12 @@ def fused_experts_none_to_experimental_sgl_trtllm_fp8_lora(
|
||||
) -> StandardCombineInput:
|
||||
from flashinfer.fused_moe import Fp8QuantizationType
|
||||
|
||||
from sglang.kernels.ops.moe.trtllm_lora_temp import (
|
||||
from sglang.kernels.ops.lora.moe.trtllm_lora_temp import (
|
||||
trtllm_fp8_block_scale_moe_lora_finalize,
|
||||
trtllm_fp8_block_scale_routed_moe_lora,
|
||||
)
|
||||
from sglang.kernels.ops.moe.trtllm_lora_temp.topk_pack import fused_pack_topk
|
||||
from sglang.kernels.ops.moe.trtllm_lora_temp.virtual_experts import (
|
||||
from sglang.kernels.ops.lora.moe.trtllm_lora_temp.topk_pack import fused_pack_topk
|
||||
from sglang.kernels.ops.lora.moe.trtllm_lora_temp.virtual_experts import (
|
||||
merged_experts_fused_moe_lora_add,
|
||||
)
|
||||
from sglang.srt.layers.moe.token_dispatcher.standard import StandardCombineInput
|
||||
@@ -316,9 +316,9 @@ def fused_experts_none_to_experimental_sgl_trtllm_bf16_lora(
|
||||
down grouped GEMM -> finalize, then the virtual-experts down-LoRA is merged into
|
||||
the output. Single-stream version (no two-stream overlap yet — phase 2).
|
||||
"""
|
||||
from sglang.kernels.ops.moe.trtllm_lora_temp import trtllm_bf16_routed_moe_lora
|
||||
from sglang.kernels.ops.moe.trtllm_lora_temp.topk_pack import fused_pack_topk
|
||||
from sglang.kernels.ops.moe.trtllm_lora_temp.virtual_experts import (
|
||||
from sglang.kernels.ops.lora.moe.trtllm_lora_temp import trtllm_bf16_routed_moe_lora
|
||||
from sglang.kernels.ops.lora.moe.trtllm_lora_temp.topk_pack import fused_pack_topk
|
||||
from sglang.kernels.ops.lora.moe.trtllm_lora_temp.virtual_experts import (
|
||||
merged_experts_fused_moe_lora_add,
|
||||
)
|
||||
from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import (
|
||||
@@ -474,11 +474,11 @@ def fused_experts_none_to_experimental_sgl_trtllm_fp4_lora(
|
||||
then the virtual-experts down-LoRA is merged into the output. Single-stream
|
||||
version; ``moe_overlap.py`` provides the two-stream variant.
|
||||
"""
|
||||
from sglang.kernels.ops.moe.trtllm_lora_temp import (
|
||||
from sglang.kernels.ops.lora.moe.trtllm_lora_temp import (
|
||||
trtllm_fp4_block_scale_routed_moe_lora,
|
||||
)
|
||||
from sglang.kernels.ops.moe.trtllm_lora_temp.topk_pack import fused_pack_topk
|
||||
from sglang.kernels.ops.moe.trtllm_lora_temp.virtual_experts import (
|
||||
from sglang.kernels.ops.lora.moe.trtllm_lora_temp.topk_pack import fused_pack_topk
|
||||
from sglang.kernels.ops.lora.moe.trtllm_lora_temp.virtual_experts import (
|
||||
merged_experts_fused_moe_lora_add,
|
||||
)
|
||||
from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import (
|
||||
|
||||
@@ -39,7 +39,7 @@ def _warm_sgl_trtllm_moe_module() -> None:
|
||||
global _SGL_TRTLLM_MODULE_WARMED
|
||||
if _SGL_TRTLLM_MODULE_WARMED:
|
||||
return
|
||||
from sglang.kernels.ops.moe.trtllm_lora_temp.core import (
|
||||
from sglang.kernels.ops.lora.moe.trtllm_lora_temp.core import (
|
||||
get_sgl_trtllm_moe_sm100_raw_module,
|
||||
)
|
||||
|
||||
|
||||
@@ -37,11 +37,13 @@ def merged_column_lora_forward(self, input_: torch.Tensor):
|
||||
):
|
||||
return get_original_merged_column_forward()(self, input_)
|
||||
|
||||
from sglang.kernels.ops.gemm.trtllm_lora_temp.gate_up_lora_b import (
|
||||
from sglang.kernels.ops.lora.dense.trtllm_lora_temp.gate_up_lora_b import (
|
||||
gate_up_lora_b_fwd,
|
||||
)
|
||||
from sglang.kernels.ops.gemm.trtllm_lora_temp.qkv_lora_b import qkv_lora_b_fwd
|
||||
from sglang.kernels.ops.gemm.trtllm_lora_temp.sgemm_lora_a import sgemm_lora_a_fwd
|
||||
from sglang.kernels.ops.lora.dense.trtllm_lora_temp.qkv_lora_b import qkv_lora_b_fwd
|
||||
from sglang.kernels.ops.lora.dense.trtllm_lora_temp.sgemm_lora_a import (
|
||||
sgemm_lora_a_fwd,
|
||||
)
|
||||
|
||||
bias = self.base_layer.bias if not self.base_layer.skip_bias_add else None
|
||||
side_stream = get_lora_side_stream()
|
||||
|
||||
@@ -54,11 +54,11 @@ def fused_experts_none_to_experimental_sgl_trtllm_fp8_lora_two_stream(
|
||||
# ---- two-stream fast path ----
|
||||
from flashinfer.fused_moe import Fp8QuantizationType
|
||||
|
||||
from sglang.kernels.ops.moe.trtllm_lora_temp import (
|
||||
from sglang.kernels.ops.lora.moe.trtllm_lora_temp import (
|
||||
trtllm_fp8_block_scale_routed_moe_lora,
|
||||
)
|
||||
from sglang.kernels.ops.moe.trtllm_lora_temp.topk_pack import fused_pack_topk
|
||||
from sglang.kernels.ops.moe.trtllm_lora_temp.virtual_experts import (
|
||||
from sglang.kernels.ops.lora.moe.trtllm_lora_temp.topk_pack import fused_pack_topk
|
||||
from sglang.kernels.ops.lora.moe.trtllm_lora_temp.virtual_experts import (
|
||||
merged_experts_fused_moe_lora_add,
|
||||
)
|
||||
from sglang.kernels.ops.quantization.fp8_kernel import per_token_group_quant_fp8
|
||||
@@ -360,11 +360,11 @@ def fused_experts_none_to_experimental_sgl_trtllm_fp4_lora_two_stream(
|
||||
)
|
||||
|
||||
# ---- two-stream fast path ----
|
||||
from sglang.kernels.ops.moe.trtllm_lora_temp import (
|
||||
from sglang.kernels.ops.lora.moe.trtllm_lora_temp import (
|
||||
trtllm_fp4_block_scale_routed_moe_lora,
|
||||
)
|
||||
from sglang.kernels.ops.moe.trtllm_lora_temp.topk_pack import fused_pack_topk
|
||||
from sglang.kernels.ops.moe.trtllm_lora_temp.virtual_experts import (
|
||||
from sglang.kernels.ops.lora.moe.trtllm_lora_temp.topk_pack import fused_pack_topk
|
||||
from sglang.kernels.ops.lora.moe.trtllm_lora_temp.virtual_experts import (
|
||||
merged_experts_fused_moe_lora_add,
|
||||
)
|
||||
from sglang.srt.distributed.device_communicators.pynccl_allocator import (
|
||||
@@ -610,9 +610,9 @@ def fused_experts_none_to_experimental_sgl_trtllm_bf16_lora_two_stream(
|
||||
)
|
||||
|
||||
# ---- two-stream fast path ----
|
||||
from sglang.kernels.ops.moe.trtllm_lora_temp import trtllm_bf16_routed_moe_lora
|
||||
from sglang.kernels.ops.moe.trtllm_lora_temp.topk_pack import fused_pack_topk
|
||||
from sglang.kernels.ops.moe.trtllm_lora_temp.virtual_experts import (
|
||||
from sglang.kernels.ops.lora.moe.trtllm_lora_temp import trtllm_bf16_routed_moe_lora
|
||||
from sglang.kernels.ops.lora.moe.trtllm_lora_temp.topk_pack import fused_pack_topk
|
||||
from sglang.kernels.ops.lora.moe.trtllm_lora_temp.virtual_experts import (
|
||||
merged_experts_fused_moe_lora_add,
|
||||
)
|
||||
from sglang.srt.distributed.device_communicators.pynccl_allocator import (
|
||||
|
||||
@@ -27,7 +27,7 @@ def fused_experts_fp8_sgl(
|
||||
# <-> quantization import cycle at load time.
|
||||
from flashinfer.fused_moe import Fp8QuantizationType
|
||||
|
||||
from sglang.kernels.ops.moe.trtllm_lora_temp.topk_pack import fused_pack_topk
|
||||
from sglang.kernels.ops.lora.moe.trtllm_lora_temp.topk_pack import fused_pack_topk
|
||||
from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import (
|
||||
is_allocation_symmetric,
|
||||
next_power_of_2,
|
||||
|
||||
@@ -5,7 +5,7 @@ It is rank-specialized: the ``R`` dimension (LoRA rank) is a Triton
|
||||
``constexpr``, so each rank value used at runtime gets its own JIT-compiled
|
||||
specialization.
|
||||
|
||||
Called from :mod:`sglang.kernels.ops.moe.virtual_experts` when
|
||||
Called from :mod:`sglang.kernels.ops.lora.moe.trtllm_lora_temp.virtual_experts` when
|
||||
``use_direct_expand_add=True``. Ranks above 64 are accumulated in multiple
|
||||
rank tiles.
|
||||
"""
|
||||
|
||||
Reference in New Issue
Block a user