MOSS模型结构

""" PyTorch Moss model."""

from typing import Optional, Tuple, Union

import torch
import torch.utils.checkpoint
from torch import nn
from torch.nn import CrossEntropyLoss
import transformers
from transformers.activations import ACT2FN
from transformers.modeling_utils import PreTrainedModel
from transformers.modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast
from transformers.utils import (
    add_code_sample_docstrings, 
    add_start_docstrings, 
    add_start_docstrings_to_model_forward, 
    logging
)

from .configuration_moss import MossConfig

logger = logging.get_logger(__name__)

_CHECKPOINT_FOR_DOC = "fnlp/moss-moon-003-base"
_CONFIG_FOR_DOC = "MossConfig"


MOSS_PRETRAINED_MODEL_ARCHIVE_LIST = [
    "fnlp/moss-moon-003-base",
    "fnlp/moss-moon-003-sft",
    "fnlp/moss-moon-003-sft-plugin",
    "fnlp/moss-moon-003-sft-