[Refactor] Build the Llama and Nemotron-NAS decoders from stage boundaries (#42308)
This commit is contained in:
@@ -85,10 +85,13 @@ def _reduce_update_read(
|
||||
group: SumGroup = SumGroup.ATTN_TP,
|
||||
read: ResidualReadout = NORM_READOUT,
|
||||
update: ResidualUpdate = PLAIN_ADD,
|
||||
quant_format: str = "",
|
||||
post_residual_addition: Optional[torch.Tensor] = None,
|
||||
):
|
||||
"""Complete the sum the input owes over ``group`` on the rows it is on,
|
||||
unless one of ``fusions`` does it with the residual add and the norm, then
|
||||
write it into the residual and read the input."""
|
||||
write it into the residual and read the input (an attention's read takes
|
||||
``quant_format`` and ``post_residual_addition``)."""
|
||||
if gathers_residual:
|
||||
residual = update.gather_residual_attn_tp(residual)
|
||||
for fused in fusions:
|
||||
@@ -104,7 +107,14 @@ def _reduce_update_read(
|
||||
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
|
||||
if _is_npu and cache is not None:
|
||||
_ = prepare_weight_cache(hidden_states, cache)
|
||||
return read.update_and_read(update, hidden_states, residual, norm)
|
||||
return read.update_and_read(
|
||||
update,
|
||||
hidden_states,
|
||||
residual,
|
||||
norm,
|
||||
quant_format=quant_format,
|
||||
post_residual_addition=post_residual_addition,
|
||||
)
|
||||
|
||||
|
||||
def _reduce_update_read_dp_gather(
|
||||
|
||||
@@ -29,6 +29,14 @@ from sglang.srt.distributed import (
|
||||
get_pp_indices,
|
||||
)
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layer_boundary.residual.add_norm import NORM_QUANT_READOUT
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
@@ -155,6 +163,7 @@ class LlamaAttention(nn.Module):
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
prefix: str = "",
|
||||
bias: bool = False,
|
||||
reduce_results: bool = True,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
@@ -199,6 +208,7 @@ class LlamaAttention(nn.Module):
|
||||
hidden_size,
|
||||
bias=bias,
|
||||
quant_config=quant_config,
|
||||
reduce_results=reduce_results,
|
||||
prefix=add_prefix("o_proj", prefix),
|
||||
)
|
||||
|
||||
@@ -292,7 +302,11 @@ class LlamaDecoderLayer(nn.Module):
|
||||
start_layer: int = 0,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
prefix: str = "",
|
||||
stage_boundaries: bool = True,
|
||||
) -> None:
|
||||
"""``stage_boundaries=False`` is for a subclass that drives its own
|
||||
residual in its own forward(): the attention and the MLP then complete
|
||||
their own sums and no boundaries are built."""
|
||||
super().__init__()
|
||||
self.hidden_size = config.hidden_size
|
||||
rope_parameters = getattr(config, "rope_parameters", None)
|
||||
@@ -329,6 +343,7 @@ class LlamaDecoderLayer(nn.Module):
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("self_attn", prefix),
|
||||
bias=attention_bias,
|
||||
reduce_results=not stage_boundaries,
|
||||
)
|
||||
self.mlp = LlamaMLP(
|
||||
hidden_size=self.hidden_size,
|
||||
@@ -336,29 +351,40 @@ class LlamaDecoderLayer(nn.Module):
|
||||
hidden_act=config.hidden_act,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
reduce_results=not stage_boundaries,
|
||||
)
|
||||
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.post_attention_layernorm = RMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
if not stage_boundaries:
|
||||
return
|
||||
self.input_layernorm.fuse_input_quant(self.self_attn.qkv_proj)
|
||||
self.post_attention_layernorm.fuse_input_quant(self.mlp.gate_up_proj)
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
(declare_attn(read=self._attn_readout(layer_id)), self.input_layernorm),
|
||||
(
|
||||
declare_ffn(sparse=False, next_layer_sparse=False),
|
||||
self.post_attention_layernorm,
|
||||
),
|
||||
previous=declare_ffn() if layer_id != 0 else None,
|
||||
terminal=layer_id == config.num_hidden_layers - 1,
|
||||
)
|
||||
|
||||
def _attn_readout(self, layer_id: int):
|
||||
return NORM_QUANT_READOUT
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor],
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
capture_output=None,
|
||||
) -> torch.Tensor:
|
||||
# Self Attention
|
||||
if residual is None:
|
||||
residual = hidden_states
|
||||
hidden_states = self.input_layernorm(
|
||||
hidden_states, quant_linear=self.self_attn.qkv_proj
|
||||
)
|
||||
else:
|
||||
hidden_states, residual = self.input_layernorm(
|
||||
hidden_states, residual, quant_linear=self.self_attn.qkv_proj
|
||||
)
|
||||
hidden_states = self.attn_boundary.prepare(
|
||||
hidden_states, forward_batch, capture=capture_output
|
||||
)
|
||||
hidden_states = self.self_attn(
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
@@ -366,11 +392,11 @@ class LlamaDecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
# Fully Connected
|
||||
hidden_states, residual = self.post_attention_layernorm(
|
||||
hidden_states, residual, quant_linear=self.mlp.gate_up_proj
|
||||
)
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
return hidden_states, residual
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
return ffn_exit.finish(hidden_states)
|
||||
|
||||
|
||||
class LlamaModel(nn.Module):
|
||||
@@ -431,35 +457,29 @@ class LlamaModel(nn.Module):
|
||||
hidden_states = self.embed_tokens(input_ids)
|
||||
else:
|
||||
hidden_states = input_embeds
|
||||
residual = None
|
||||
residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
# FIXME(@ying): reduce the number of proxy tensors by not fusing layer norms
|
||||
hidden_states = pp_proxy_tensors["hidden_states"]
|
||||
residual = pp_proxy_tensors["residual"]
|
||||
deferred_norm = None
|
||||
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
|
||||
pp_proxy_tensors, forward_batch
|
||||
)
|
||||
|
||||
aux_hidden_states = []
|
||||
aux_hidden_states = AuxHiddenStateList()
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
if i in self.layers_to_capture:
|
||||
aux_hidden_states.append(hidden_states + residual)
|
||||
layer = self.layers[i]
|
||||
hidden_states, residual = layer(
|
||||
hidden_states = self.layers[i](
|
||||
positions,
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
residual,
|
||||
capture_output=aux_hidden_states.capture
|
||||
if i in self.layers_to_capture
|
||||
else None,
|
||||
)
|
||||
|
||||
if not self.pp_group.is_last_rank:
|
||||
return PPProxyTensors(
|
||||
{
|
||||
"hidden_states": hidden_states,
|
||||
"residual": residual,
|
||||
}
|
||||
)
|
||||
else:
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
return residual_batch.to_pp(hidden_states, forward_batch)
|
||||
hidden_states = residual_batch.final_norm(
|
||||
hidden_states, forward_batch, self.norm
|
||||
)
|
||||
|
||||
if len(aux_hidden_states) == 0:
|
||||
return hidden_states
|
||||
@@ -609,6 +629,7 @@ class LlamaForCausalLM(nn.Module):
|
||||
start, end = split_interval
|
||||
# embed
|
||||
if start == 0:
|
||||
residual_batch.start(forward_batch)
|
||||
if input_embeds is None:
|
||||
forward_batch.hidden_states = self.model.embed_tokens(input_ids)
|
||||
else:
|
||||
@@ -616,19 +637,17 @@ class LlamaForCausalLM(nn.Module):
|
||||
# decoder layer
|
||||
for i in range(start, end):
|
||||
layer = self.model.layers[i]
|
||||
forward_batch.hidden_states, forward_batch.residual = layer(
|
||||
forward_batch.hidden_states = layer(
|
||||
positions,
|
||||
forward_batch.hidden_states,
|
||||
forward_batch,
|
||||
forward_batch.residual,
|
||||
)
|
||||
|
||||
if end == self.model.config.num_hidden_layers:
|
||||
# norm
|
||||
hidden_states, _ = self.model.norm(
|
||||
forward_batch.hidden_states, forward_batch.residual
|
||||
forward_batch.hidden_states = residual_batch.final_norm(
|
||||
forward_batch.hidden_states, forward_batch, self.model.norm
|
||||
)
|
||||
forward_batch.hidden_states = hidden_states
|
||||
# logits process
|
||||
result = self.logits_processor(
|
||||
input_ids, forward_batch.hidden_states, self.lm_head, forward_batch
|
||||
|
||||
@@ -25,6 +25,8 @@ import torch
|
||||
from torch import nn
|
||||
from transformers import LlamaConfig
|
||||
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layer_boundary.residual.post_norm import PLAIN_READOUT
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.vocab_parallel_embedding import (
|
||||
@@ -50,7 +52,11 @@ class LlamaDecoderLayer(LlamaDecoderLayer):
|
||||
# https://github.com/SafeAILab/EAGLE/blob/35c78f6cdc19a73e05cf5c330b4c358dad970c6a/eagle/model/cnets.py#L427
|
||||
if layer_id == 0:
|
||||
del self.input_layernorm
|
||||
setattr(self, "input_layernorm", lambda x, quant_linear=None: x)
|
||||
|
||||
def _attn_readout(self, layer_id: int):
|
||||
if layer_id == 0:
|
||||
return PLAIN_READOUT
|
||||
return super()._attn_readout(layer_id)
|
||||
|
||||
|
||||
class LlamaModel(nn.Module):
|
||||
@@ -98,16 +104,11 @@ class LlamaModel(nn.Module):
|
||||
torch.cat((hidden_states, forward_batch.spec_info.hidden_states), dim=-1)
|
||||
)
|
||||
|
||||
residual = None
|
||||
for i in range(len(self.layers)):
|
||||
layer = self.layers[i]
|
||||
hidden_states, residual = layer(
|
||||
positions,
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
residual,
|
||||
)
|
||||
return hidden_states + residual
|
||||
residual_batch.start(forward_batch)
|
||||
for layer in self.layers:
|
||||
hidden_states = layer(positions, hidden_states, forward_batch)
|
||||
hidden_states = residual_batch.fold(hidden_states, forward_batch)
|
||||
return residual_batch.take_output(hidden_states, forward_batch)
|
||||
|
||||
|
||||
class LlamaForCausalLMEagle(LlamaForCausalLM):
|
||||
|
||||
@@ -49,7 +49,13 @@ class LlamaDecoderLayer(LlamaDecoderLayer):
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__(config, layer_id, quant_config=quant_config, prefix=prefix)
|
||||
super().__init__(
|
||||
config,
|
||||
layer_id,
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
stage_boundaries=False,
|
||||
)
|
||||
|
||||
# Input layer concats embeds + target_hidden before qkv (input dim 2x).
|
||||
self.is_input_layer = layer_id == 0
|
||||
|
||||
@@ -39,6 +39,7 @@ class Ministral3Attention(LlamaAttention):
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
prefix: str = "",
|
||||
bias: bool = False,
|
||||
reduce_results: bool = True,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
config=config,
|
||||
@@ -54,6 +55,7 @@ class Ministral3Attention(LlamaAttention):
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
bias=bias,
|
||||
reduce_results=reduce_results,
|
||||
)
|
||||
# Ministral3 specific: llama 4 style scaling beta
|
||||
self.llama_4_scaling_beta = config.rope_parameters.get("llama_4_scaling_beta")
|
||||
@@ -128,7 +130,9 @@ class Ministral3DecoderLayer(LlamaDecoderLayer):
|
||||
prefix=add_prefix("self_attn", prefix),
|
||||
bias=getattr(config, "attention_bias", False)
|
||||
or getattr(config, "bias", False),
|
||||
reduce_results=False,
|
||||
)
|
||||
self.input_layernorm.fuse_input_quant(self.self_attn.qkv_proj)
|
||||
|
||||
|
||||
class Ministral3Model(LlamaModel):
|
||||
|
||||
@@ -40,6 +40,7 @@ import torch
|
||||
from torch import nn
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import RowParallelLinear
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
@@ -120,12 +121,11 @@ class MistralEagleModel(nn.Module):
|
||||
)
|
||||
)
|
||||
|
||||
residual = None
|
||||
residual_batch.start(forward_batch)
|
||||
for layer in self.layers:
|
||||
hidden_states, residual = layer(
|
||||
positions, hidden_states, forward_batch, residual
|
||||
)
|
||||
return hidden_states + residual
|
||||
hidden_states = layer(positions, hidden_states, forward_batch)
|
||||
hidden_states = residual_batch.fold(hidden_states, forward_batch)
|
||||
return residual_batch.take_output(hidden_states, forward_batch)
|
||||
|
||||
|
||||
class MistralForCausalLMEagle(LlamaForCausalLMEagle):
|
||||
|
||||
@@ -22,6 +22,7 @@ from transformers.models.mllama.modeling_mllama import (
|
||||
|
||||
from sglang.srt.layers.activation import get_act_fn
|
||||
from sglang.srt.layers.attention.vision import VisionAttention
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
@@ -725,13 +726,14 @@ class MllamaTextModel(nn.Module):
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
elif isinstance(decoder_layer, LlamaDecoderLayer):
|
||||
hidden_states, residual = decoder_layer(
|
||||
residual_batch.start(forward_batch)
|
||||
hidden_states = decoder_layer(
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
residual=None,
|
||||
)
|
||||
hidden_states = hidden_states + residual
|
||||
hidden_states = residual_batch.fold(hidden_states, forward_batch)
|
||||
hidden_states = residual_batch.take_output(hidden_states, forward_batch)
|
||||
else:
|
||||
raise ValueError(f"Unknown decoder layer type {type(decoder_layer)}")
|
||||
hidden_states = self.norm(hidden_states)
|
||||
|
||||
@@ -23,6 +23,12 @@ import torch
|
||||
from torch import nn
|
||||
from transformers import LlamaConfig
|
||||
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
make_stages,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor, LogitsProcessorOutput
|
||||
from sglang.srt.layers.pooler import Pooler, PoolingType
|
||||
@@ -57,6 +63,21 @@ def _find_multiple(n: int, k: int) -> int:
|
||||
return n + k - (n % k)
|
||||
|
||||
|
||||
def _previous_stage(block_configs, layer_idx: int):
|
||||
"""The declaration of the last attention or FFN before this layer, or None
|
||||
when this layer's first stage starts the layer stack."""
|
||||
for block in reversed(block_configs[:layer_idx]):
|
||||
if not block.ffn.no_op:
|
||||
return declare_ffn()
|
||||
if not block.attention.no_op:
|
||||
return declare_attn()
|
||||
return None
|
||||
|
||||
|
||||
def _has_stage(block) -> bool:
|
||||
return not (block.attention.no_op and block.ffn.no_op)
|
||||
|
||||
|
||||
class DeciLMDecoderLayer(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
@@ -107,6 +128,7 @@ class DeciLMDecoderLayer(nn.Module):
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("self_attn", prefix),
|
||||
bias=attention_bias,
|
||||
reduce_results=False,
|
||||
)
|
||||
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
@@ -121,41 +143,61 @@ class DeciLMDecoderLayer(nn.Module):
|
||||
hidden_act=config.hidden_act,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
reduce_results=False,
|
||||
)
|
||||
self.post_attention_layernorm = RMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
|
||||
# A layer whose attention and FFN are both no-ops has no stages.
|
||||
stages = []
|
||||
if not self._is_no_op_attention:
|
||||
stages.append((declare_attn(), self.input_layernorm))
|
||||
if not self._is_no_op_ffn:
|
||||
stages.append(
|
||||
(
|
||||
declare_ffn(sparse=False, next_layer_sparse=False),
|
||||
self.post_attention_layernorm,
|
||||
)
|
||||
)
|
||||
self.entry_boundary = None
|
||||
if stages:
|
||||
boundaries = make_stages(
|
||||
*stages,
|
||||
previous=_previous_stage(config.block_configs, layer_idx),
|
||||
terminal=not any(
|
||||
_has_stage(block) for block in config.block_configs[layer_idx + 1 :]
|
||||
),
|
||||
)
|
||||
self.entry_boundary = boundaries[0]
|
||||
if not self._is_no_op_attention:
|
||||
self.attn_boundary = boundaries[0]
|
||||
if not self._is_no_op_ffn:
|
||||
self.ffn_boundary = boundaries[-1]
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
residual: Optional[torch.Tensor],
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
) -> torch.Tensor:
|
||||
# Self Attention
|
||||
|
||||
if self._is_no_op_attention:
|
||||
pass
|
||||
else:
|
||||
if residual is None:
|
||||
residual = hidden_states
|
||||
hidden_states = self.input_layernorm(hidden_states)
|
||||
else:
|
||||
hidden_states, residual = self.input_layernorm(hidden_states, residual)
|
||||
if not self._is_no_op_attention:
|
||||
hidden_states = self.attn_boundary.prepare(hidden_states, forward_batch)
|
||||
hidden_states = self.self_attn(
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
|
||||
# Fully Connected
|
||||
if not self._is_no_op_ffn:
|
||||
hidden_states, residual = self.post_attention_layernorm(
|
||||
hidden_states, residual
|
||||
)
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
return hidden_states, residual
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
return hidden_states
|
||||
|
||||
|
||||
class DeciModel(nn.Module):
|
||||
@@ -223,32 +265,23 @@ class DeciModel(nn.Module):
|
||||
hidden_states = inputs_embeds
|
||||
else:
|
||||
hidden_states = self.get_input_embeddings(input_ids)
|
||||
residual = None
|
||||
residual_batch.start(forward_batch)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states = pp_proxy_tensors["hidden_states"]
|
||||
residual = pp_proxy_tensors["residual"]
|
||||
entry = next(
|
||||
self.layers[i].entry_boundary
|
||||
for i in range(self.start_layer, self.end_layer)
|
||||
if self.layers[i].entry_boundary is not None
|
||||
)
|
||||
hidden_states = entry.from_pp(pp_proxy_tensors, forward_batch)
|
||||
|
||||
kv_cache_index = 0
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
layer = self.layers[i]
|
||||
if not layer._is_no_op_attention:
|
||||
hidden_states, residual = layer(
|
||||
positions, hidden_states, forward_batch, residual
|
||||
)
|
||||
kv_cache_index += 1
|
||||
else:
|
||||
hidden_states, residual = layer(
|
||||
positions, hidden_states, forward_batch, residual
|
||||
)
|
||||
hidden_states = self.layers[i](positions, hidden_states, forward_batch)
|
||||
|
||||
if not get_parallel().pp_group.is_last_rank:
|
||||
return PPProxyTensors(
|
||||
{"hidden_states": hidden_states, "residual": residual}
|
||||
)
|
||||
return residual_batch.to_pp(hidden_states, forward_batch)
|
||||
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
return hidden_states
|
||||
return residual_batch.final_norm(hidden_states, forward_batch, self.norm)
|
||||
|
||||
|
||||
class DeciLMForCausalLM(nn.Module):
|
||||
|
||||
Reference in New Issue
Block a user