pegasus_fineturn_and_evaluation_script.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FTPcjFWFllMP"
      },
      "source": [
        "# Import Data and Split"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Run all the commented out code the first time you run the script."
      ],
      "metadata": {
        "id": "MnhYF2Anionv"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "QDqI_igTkTLf",
        "tags": []
      },
      "outputs": [],
      "source": [
        "# !ls results\n",
        "# !pip install scikit-learn\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "AjJEV0oyEFpf"
      },
      "outputs": [],
      "source": [
        "# !rm -rf ./Dataset\n",
        "# !rm -rf ./mimicDataset/*\n",
        "# !nproc\n",
        "# !rm -rf ./results"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "bdYtptRHgPKz"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "\n",
        "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "jupyter": {
          "outputs_hidden": true
        },
        "tags": [],
        "id": "DR-GD9j-gPKz"
      },
      "outputs": [],
      "source": [
        "# zip file\n",
        "\n",
        "# import zipfile\n",
        "# import os\n",
        "\n",
        "# # Specify the path to the ZIP file\n",
        "# zip_path = 'mimicDataset/mimicDataset.zip'\n",
        "\n",
        "# # Open the ZIP file\n",
        "# with zipfile.ZipFile(zip_path, 'r') as zip_ref:\n",
        "#     # Extract all the files in the same folder\n",
        "#     zip_ref.extractall()\n",
        "\n",
        "# # Get the directory containing the ZIP file\n",
        "# zip_dir = os.path.dirname(zip_path)\n",
        "\n",
        "# # Print the list of extracted files\n",
        "# extracted_files = zip_ref.namelist()\n",
        "# for file in extracted_files:\n",
        "#     print(os.path.join(zip_dir, file))\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8Gb1cRCHJvQJ",
        "tags": []
      },
      "outputs": [],
      "source": [
        "import json\n",
        "import os\n",
        "import shutil\n",
        " \n",
        "# set env and make folder\n",
        "dataset_folder_path = 'Dataset'\n",
        "mimic_dataset_path = 'mimicDataset'\n",
        "\n",
        "\n",
        "if not os.path.exists(dataset_folder_path):\n",
        "    os.mkdir(dataset_folder_path)\n",
        "if not os.path.exists(mimic_dataset_path):\n",
        "    os.mkdir(mimic_dataset_path)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Qh_hcyUvmM7n",
        "tags": []
      },
      "outputs": [],
      "source": [
        "def save_to_json(data, file_path):\n",
        "    with open(file_path, 'w') as json_file:\n",
        "        json.dump(data, json_file)\n",
        "\n",
        "def load_from_json(file_path):\n",
        "    with open(file_path, 'r') as json_file:\n",
        "        data = json.load(json_file)\n",
        "    return data\n",
        "\n",
        "def zip_directory(source_dir, output_zip):\n",
        "    shutil.make_archive(output_zip, 'zip', source_dir)\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "BtWZyeIbRMp-",
        "tags": []
      },
      "outputs": [],
      "source": [
        "# load data from dataset file\n",
        "\n",
        "# with open(\"mimicbertSum.json\", \"r\") as f:\n",
        "#     data = json.load(f)\n",
        "\n",
        "# data = data[:2464]\n",
        "# print(len(data))\n",
        "# extractives = [item[\"topK_importance\"] for item in data]\n",
        "# summaries = [item[\"summary\"] for item in data]\n",
        "# # print(\" \".join(extractives[0]))\n",
        "\n",
        "# extractives = [\" \".join(extract) for extract in extractives]\n",
        "# summaries = [\" \".join(summary) for summary in summaries]\n",
        "\n",
        "# print('finished')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wQLfW6jVRNV8"
      },
      "source": [
        "### Split data and save to seperate file"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ywgCHarmmVut",
        "tags": []
      },
      "outputs": [],
      "source": [
        "# from sklearn.model_selection import train_test_split\n",
        "\n",
        "# train_extractives, test_extractives, train_summaries, test_summaries = train_test_split(extractives, summaries, test_size=0.1, random_state=42)\n",
        "# train_extractives, val_extractives, train_summaries, val_summaries = train_test_split(train_extractives, train_summaries, test_size=0.1, random_state=42)\n",
        "\n",
        "\n",
        "# dataset_dir = mimic_dataset_path\n",
        "\n",
        "# train_extractives = save_to_json(train_extractives, os.path.join(dataset_dir,'train_extractives.json'))\n",
        "# # test_extractives = save_to_json(test_extractives, os.path.join(dataset_dir, 'test_extractives.json'))\n",
        "# train_summaries = save_to_json(train_summaries, os.path.join(dataset_dir, 'train_summaries.json'))\n",
        "# # test_summaries = save_to_json(test_summaries, os.path.join(dataset_dir, 'test_summaries.json'))\n",
        "# val_extractives = save_to_json(val_extractives, os.path.join(dataset_dir, 'val_extractives.json'))\n",
        "# val_summaries = save_to_json(val_summaries, os.path.join(dataset_dir, 'val_summaries.json'))\n",
        "\n",
        "# print('finished')\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "NHolwmKWLwqu",
        "tags": []
      },
      "outputs": [],
      "source": [
        "# zip_directory(dataset_folder_path, dataset_folder_path+'-700-end')\n",
        "# # files.download(dataset_folder_path+'.zip')\n",
        "\n",
        "# zip_directory(mimic_dataset_path, mimic_dataset_path)\n",
        "# files.download(mimic_dataset_path+'.zip')"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### read data from seperated file"
      ],
      "metadata": {
        "id": "pVbkGXKFgT9O"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "iX-o3cy-tJXH",
        "tags": [],
        "outputId": "c52a17df-169f-4668-b275-161cd1ccb777"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "test_extractives.json  train_extractives.json  val_extractives.json\n",
            "test_summaries.json    train_summaries.json    val_summaries.json\n"
          ]
        }
      ],
      "source": [
        "# read from json\n",
        "dataset_dir = mimic_dataset_path\n",
        "!ls mimicDataset\n",
        "train_extractives = load_from_json(os.path.join(dataset_dir,'train_extractives.json'))\n",
        "test_extractives = load_from_json(os.path.join(dataset_dir, 'test_extractives.json'))\n",
        "train_summaries = load_from_json(os.path.join(dataset_dir, 'train_summaries.json'))\n",
        "test_summaries = load_from_json(os.path.join(dataset_dir, 'test_summaries.json'))\n",
        "val_extractives = load_from_json(os.path.join(dataset_dir, 'val_extractives.json'))\n",
        "val_summaries = load_from_json(os.path.join(dataset_dir, 'val_summaries.json'))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-lT5twE5M5k9",
        "outputId": "998e6af8-12ca-4819-8496-101411e69a4d",
        "tags": []
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2000\n",
            "223\n",
            "241\n"
          ]
        }
      ],
      "source": [
        "print(len(train_extractives))\n",
        "print(len(val_extractives))\n",
        "print(len(test_extractives))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "t3D7-hJVl9nB"
      },
      "source": [
        "# Train and Validate Different HPs"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "82msUL9LsovS",
        "tags": []
      },
      "outputs": [],
      "source": [
        "# !pip install transformers datasets sentencepiece\n",
        "# !pip install --upgrade accelerate"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "i07kdk7VoES5",
        "tags": [],
        "outputId": "4147474e-d2a2-4909-be91-617cc5e080e1"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "prepare_data finished runing!\n"
          ]
        }
      ],
      "source": [
        "from transformers import PegasusForConditionalGeneration, PegasusTokenizer, Trainer, TrainingArguments\n",
        "from transformers import BigBirdPegasusForConditionalGeneration, AutoTokenizer\n",
        "from transformers import PegasusXForConditionalGeneration\n",
        "import torch\n",
        "\n",
        "\n",
        "\"\"\"\n",
        "some part of this block modified from https://gist.github.com/jiahao87/50cec29725824da7ff6dd9314b53c4b3\n",
        "\"\"\"\n",
        "\n",
        "# class PegasusDataset(torch.utils.data.Dataset):\n",
        "#     def __init__(self, encodings, labels):\n",
        "#         self.encodings = encodings\n",
        "#         self.labels = labels\n",
        "\n",
        "#     def __getitem__(self, idx):\n",
        "#         item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
        "#         item[\"labels\"] = torch.tensor(self.labels[\"input_ids\"][idx])\n",
        "#         return item\n",
        "\n",
        "#     def __len__(self):\n",
        "#         return len(self.encodings[\"input_ids\"])\n",
        "\n",
        "# class PegasusDataset(torch.utils.data.Dataset):\n",
        "#     def __init__(self, encodings, labels, input_ids=None):\n",
        "#         self.encodings = encodings\n",
        "#         self.labels = labels\n",
        "#         self.input_ids = input_ids\n",
        "\n",
        "#     def __getitem__(self, idx):\n",
        "#         item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
        "#         item[\"labels\"] = torch.tensor(self.labels[\"input_ids\"][idx])\n",
        "#         if self.input_ids is not None:\n",
        "#             item[\"input_ids\"] = torch.tensor(self.input_ids[idx])\n",
        "#         return item\n",
        "\n",
        "#     def __len__(self):\n",
        "#         return len(self.encodings[\"input_ids\"])\n",
        "class PegasusDataset(torch.utils.data.Dataset):\n",
        "    def __init__(self, encodings, labels, input_ids=None):\n",
        "        self.encodings = encodings\n",
        "        self.labels = labels\n",
        "        self.input_ids = input_ids\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        item = {key: val[idx] for key, val in self.encodings.items()}\n",
        "        item[\"labels\"] = self.labels[\"input_ids\"][idx]\n",
        "        if self.input_ids is not None:\n",
        "            item[\"input_ids\"] = self.input_ids[idx]\n",
        "        return item\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.encodings[\"input_ids\"])\n",
        "\n",
        "\n",
        "'''\n",
        "make ready-to-use dataset\n",
        "'''      \n",
        "def prepare_data(model_name, \n",
        "                 train_texts, train_labels, \n",
        "                 val_texts=None, val_labels=None, \n",
        "                 test_texts=None, test_labels=None):\n",
        "\n",
        "  # tokenizer = PegasusTokenizer.from_pretrained(model_name)\n",
        "  # tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
        "  tokenizer = AutoTokenizer.from_pretrained(\"google/pegasus-x-large\")\n",
        "  print(tokenizer)\n",
        "  prepare_val = False if val_texts is None or val_labels is None else True\n",
        "  prepare_test = False if test_texts is None or test_labels is None else True\n",
        "\n",
        "  # def tokenize_data(texts, labels):\n",
        "  #   encodings = tokenizer(texts, truncation=True, padding=True, max_length=512)\n",
        "  #   decodings = tokenizer(labels, truncation=True, padding=True, max_length=512)\n",
        "  #   tokenized_dataset = PegasusDataset(encodings, decodings)\n",
        "  #   return tokenized_dataset\n",
        "  def tokenize_data(texts, labels):\n",
        "    encodings = tokenizer(texts, truncation=True, padding='longest', max_length=1024, return_tensors=\"pt\")\n",
        "    decodings = tokenizer(labels, truncation=True, padding='longest', max_length=1024, return_tensors=\"pt\")\n",
        "    tokenized_dataset = PegasusDataset(encodings, decodings, input_ids=encodings[\"input_ids\"])\n",
        "    return tokenized_dataset\n",
        "\n",
        "\n",
        "  train_dataset = tokenize_data(train_texts, train_labels)\n",
        "  val_dataset = tokenize_data(val_texts, val_labels) if prepare_val else None\n",
        "  test_dataset = tokenize_data(test_texts, test_labels) if prepare_test else None\n",
        "\n",
        "  return train_dataset, val_dataset, test_dataset, tokenizer\n",
        "\n",
        "\n",
        "print('prepare_data finished runing!')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "gdr4CCUVmfw6",
        "tags": [],
        "outputId": "46691fc2-cc70-470e-b480-dd86dc95d70d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "prepare_fine_tuning finished runing!\n"
          ]
        }
      ],
      "source": [
        "\"\"\"\n",
        "some part of this block modified from https://gist.github.com/jiahao87/50cec29725824da7ff6dd9314b53c4b3\n",
        "\"\"\"\n",
        "\n",
        "\n",
        "def prepare_fine_tuning(model_name, tokenizer, train_dataset, val_dataset=None, freeze_encoder=False, output_dir='./results'):\n",
        "  \"\"\"\n",
        "  Prepare configurations and base model for fine-tuning\n",
        "  \"\"\"\n",
        "  torch_device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
        "  # model = PegasusForConditionalGeneration.from_pretrained(model_name).to(torch_device)\n",
        "  # model = BigBirdPegasusForConditionalGeneration.from_pretrained(model_name).to(torch_device)\n",
        "  model = PegasusXForConditionalGeneration.from_pretrained(model_name).to(torch_device) # for Pegasus-x\n",
        "  print(model)\n",
        "    \n",
        "  if freeze_encoder:\n",
        "    for param in model.model.encoder.parameters():\n",
        "      param.requires_grad = False\n",
        "\n",
        "  if val_dataset is not None:\n",
        "    training_args = TrainingArguments(\n",
        "      output_dir=output_dir,           # output directory\n",
        "      num_train_epochs=30,           # total number of training epochs\n",
        "      per_device_train_batch_size=7,   # batch size per device during training, can increase if memory allows\n",
        "      per_device_eval_batch_size=7,    # batch size for evaluation, can increase if memory allows\n",
        "      save_steps=500,                  # number of updates steps before checkpoint saves\n",
        "      save_total_limit=4,              # limit the total amount of checkpoints and deletes the older checkpoints\n",
        "      evaluation_strategy='steps',     # evaluation strategy to adopt during training\n",
        "      eval_steps=100,                  # number of update steps before evaluation\n",
        "      learning_rate=3e-5,              # set beginning lr \n",
        "      # lr_scheduler_type='cosine_with_restarts', # set scheduler type\n",
        "      warmup_steps=500,                # number of warmup steps for learning rate scheduler\n",
        "      weight_decay=0.08,               # strength of weight decay\n",
        "      logging_dir='./logs',            # directory for storing logs\n",
        "      logging_steps=100,\n",
        "      log_level='info',\n",
        "    )\n",
        "\n",
        "    trainer = Trainer(\n",
        "      model=model,                         # the instantiated 🤗 Transformers model to be trained\n",
        "      args=training_args,                  # training arguments, defined above\n",
        "      train_dataset=train_dataset,         # training dataset\n",
        "      eval_dataset=val_dataset,            # evaluation dataset\n",
        "      tokenizer=tokenizer\n",
        "    )\n",
        "\n",
        "  else:\n",
        "    training_args = TrainingArguments(\n",
        "      output_dir=output_dir,           # output directory\n",
        "      num_train_epochs=100,           # total number of training epochs\n",
        "      per_device_train_batch_size=8,   # batch size per device during training, can increase if memory allows\n",
        "      save_steps=500,                  # number of updates steps before checkpoint saves\n",
        "      save_total_limit=5,              # limit the total amount of checkpoints and deletes the older checkpoints\n",
        "      warmup_steps=500,                # number of warmup steps for learning rate scheduler\n",
        "      weight_decay=0.005,               # strength of weight decay\n",
        "      logging_dir='./logs',            # directory for storing logs\n",
        "      logging_steps=10,\n",
        "    )\n",
        "\n",
        "    trainer = Trainer(\n",
        "      model=model,                         # the instantiated 🤗 Transformers model to be trained\n",
        "      args=training_args,                  # training arguments, defined above\n",
        "      train_dataset=train_dataset,         # training dataset\n",
        "      tokenizer=tokenizer\n",
        "    )\n",
        "\n",
        "  return trainer\n",
        "\n",
        "\n",
        "print('prepare_fine_tuning finished runing!')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 565
        },
        "id": "q8y8EPy2oIwp",
        "outputId": "ec51d8c5-53bf-46e9-88ed-86fa31642dd3",
        "tags": []
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "start train!\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loading file spiece.model from cache at /root/.cache/huggingface/hub/models--google--pegasus-x-large/snapshots/ff38c5db1f5b97b923dbecdaf58b15c8e213a27a/spiece.model\n",
            "loading file tokenizer.json from cache at /root/.cache/huggingface/hub/models--google--pegasus-x-large/snapshots/ff38c5db1f5b97b923dbecdaf58b15c8e213a27a/tokenizer.json\n",
            "loading file added_tokens.json from cache at None\n",
            "loading file special_tokens_map.json from cache at /root/.cache/huggingface/hub/models--google--pegasus-x-large/snapshots/ff38c5db1f5b97b923dbecdaf58b15c8e213a27a/special_tokens_map.json\n",
            "loading file tokenizer_config.json from cache at /root/.cache/huggingface/hub/models--google--pegasus-x-large/snapshots/ff38c5db1f5b97b923dbecdaf58b15c8e213a27a/tokenizer_config.json\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "PegasusTokenizerFast(name_or_path='google/pegasus-x-large', vocab_size=96103, model_max_length=1024, is_fast=True, padding_side='right', truncation_side='right', special_tokens={'eos_token': '</s>', 'unk_token': '<unk>', 'pad_token': '<pad>', 'mask_token': '<mask_2>', 'additional_special_tokens': ['<mask_1>', '<unk_2>', '<unk_3>', '<unk_4>', '<unk_5>', '<unk_6>', '<unk_7>', '<unk_8>', '<unk_9>', '<unk_10>', '<unk_11>', '<unk_12>', '<unk_13>', '<unk_14>', '<unk_15>', '<unk_16>', '<unk_17>', '<unk_18>', '<unk_19>', '<unk_20>', '<unk_21>', '<unk_22>', '<unk_23>', '<unk_24>', '<unk_25>', '<unk_26>', '<unk_27>', '<unk_28>', '<unk_29>', '<unk_30>', '<unk_31>', '<unk_32>', '<unk_33>', '<unk_34>', '<unk_35>', '<unk_36>', '<unk_37>', '<unk_38>', '<unk_39>', '<unk_40>', '<unk_41>', '<unk_42>', '<unk_43>', '<unk_44>', '<unk_45>', '<unk_46>', '<unk_47>', '<unk_48>', '<unk_49>', '<unk_50>', '<unk_51>', '<unk_52>', '<unk_53>', '<unk_54>', '<unk_55>', '<unk_56>', '<unk_57>', '<unk_58>', '<unk_59>', '<unk_60>', '<unk_61>', '<unk_62>', '<unk_63>', '<unk_64>', '<unk_65>', '<unk_66>', '<unk_67>', '<unk_68>', '<unk_69>', '<unk_70>', '<unk_71>', '<unk_72>', '<unk_73>', '<unk_74>', '<unk_75>', '<unk_76>', '<unk_77>', '<unk_78>', '<unk_79>', '<unk_80>', '<unk_81>', '<unk_82>', '<unk_83>', '<unk_84>', '<unk_85>', '<unk_86>', '<unk_87>', '<unk_88>', '<unk_89>', '<unk_90>', '<unk_91>', '<unk_92>', '<unk_93>', '<unk_94>', '<unk_95>', '<unk_96>', '<unk_97>', '<unk_98>', '<unk_99>', '<unk_100>', '<unk_101>', '<unk_102>']})\n"
          ]
        }
      ],
      "source": [
        "if __name__=='__main__':\n",
        "  # use XSum dataset as example, with first 1000 docs as training data\n",
        "  # from datasets import load_dataset\n",
        "  # dataset = load_dataset(\"xsum\")\n",
        "  # train_texts, train_labels = dataset['train']['document'][:1000], dataset['train']['summary'][:1000]\n",
        "  \n",
        "  # use Pegasus Large model as base for fine-tuning\n",
        "  print('start train!')\n",
        "  model_name = 'google/pegasus-x-large'\n",
        "  # _, _, test_dataset, _ = prepare_data(model_name, train_extractives, train_summaries, val_extractives, val_summaries, test_extractives, test_summaries)\n",
        "  train_dataset, val_dataset, test_dataset, tokenizer = prepare_data(model_name, train_extractives, train_summaries, val_extractives, val_summaries)\n",
        "  trainer = prepare_fine_tuning(model_name, tokenizer, train_dataset, val_dataset, freeze_encoder=False)\n",
        "  trainer.train()\n",
        "\n",
        "  # write training log to checkpoint\n",
        "  log_file_path = './logs/training_logs.txt'\n",
        "  with open(log_file_path, 'w') as log_file:\n",
        "    log_file.write(f\"Training Arguments:\\n{json.dumps(trainer.args.to_dict(), indent=2)}\\n\\n\")\n",
        "    log_file.write(\"Training and Validation Losses:\\n\")\n",
        "    with open(trainer.args.logging_dir + '/info.log', 'r') as log_info:\n",
        "      log_file.writelines(log_info.readlines())"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_ZNQ_4aylwib",
        "tags": []
      },
      "source": [
        "## TEST USE"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ILBzMwY2jgYw"
      },
      "outputs": [],
      "source": [
        "# # 1. Load the checkpoint model\n",
        "# checkpoint_path = \"./results/checkpoint-500\"\n",
        "# eval_model = PegasusXForConditionalGeneration.from_pretrained(checkpoint_path)\n",
        "\n",
        "# # 2. Load the tokenizer\n",
        "# eval_tokenizer = AutoTokenizer.from_pretrained(\"google/pegasus-x-large\")\n",
        "    \n",
        "# # Check if GPU is available and set the device accordingly\n",
        "# device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "# # Move the model to the device\n",
        "# model.to(device)\n",
        "\n",
        "# # Modify the generate_summary function to use GPU\n",
        "# def generate_summary(model, tokenizer, text, max_length=4096):\n",
        "#     inputs = tokenizer(text, return_tensors=\"pt\", max_length=max_length, truncation=True)\n",
        "\n",
        "#     # Move input tensors to the device\n",
        "#     inputs = {key: tensor.to(device) for key, tensor in inputs.items()}\n",
        "\n",
        "#     summary_ids = model.generate(**inputs, no_repeat_ngram_size=3)\n",
        "#     summary = tokenizer.decode(summary_ids[0], skip_special_tokens=True)\n",
        "#     return summary\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "K4_6nt85pbRj"
      },
      "outputs": [],
      "source": [
        "# # Test the model on your input text\n",
        "# input_text = \"\"\"\n",
        "# Most movements are comprised of sequences. From the complex routines that gymnasts perform to intricate dance numbers to simply reaching for an object, our movements are comprised of sequences of movements that are learned through practice. The cerebellum has been long implicated in learning and execution of accurate movements (Doyon et al., 1997; Shin and Ivry, 2003; Lehéricy et al., 2005; Krupa et al., 1993; Lisberger, 1994; Diener and Dichgans, 1992). Movement sequences as well as multi-joint movements are particularly sensitive to cerebellar dysfunction (Shin and Ivry, 2003; Diener and Dichgans, 1992; Doyon et al., 2002). For example, one of the hallmark deficits of cerebellar pathology is dysdiadochokinesia (Diener and Dichgans, 1992) – an inability to perform a rapid alternating sequence of movements. Patients with cerebellar lesions display severe deficits (Doyon et al., 1997) in sequence learning or are unable to learn sequences at all (Shin and Ivry, 2003), even with modest impairments in learning of single-component, directly cued movements (Spencer and Ivry, 2009). Studies of sequence learning in nonhuman primates (Desmurget and Turner, 2010; Rünger et al., 2013) demonstrated that with repeated training components of movement in the sequence start to be initiated predictively, before the arrival of sensory cue. Since one of the fundamental properties of cerebellar learning is the ability to learn a predictive response (Marr, 1969; Bastian, 2006; Shadmehr et al., 2010; Therrien and Bastian, 2015), these results also indirectly imply a strong cerebellar contribution to sequence learning. Although results of these studies suggest cerebellar involvement in the learning and execution of movement sequences, most of what we know about cerebellar mechanisms of learning comes from studies utilizing single-component movements. These include adaptation of the vestibule-ocular reflex, adaptation of saccadic and smooth pursuit eye movements and conditioning of eyelid responses. In principle, the cerebellar mechanisms underlying movement sequences could be different from, or at least somewhat different from those mediating single-component movements. If so, these mechanisms are largely unknown. In contrast, we tested the hypothesis that a simple elaboration of cerebellar mechanisms that mediate learning single-component movements is sufficient to explain cerebellar learning and implementation of movement sequences. Three possible (but not mutually exclusive) ways of implementing cerebellar learning of movement sequences are illustrated in Figure 1. With the first possibility a specific external cue is associated with a specific movement component in the sequence (Figure 1A). The second option is a variant of the first one. Here only single external cue is present, but it persists in time through the whole sequence, so that different movement components are elicited by signals associated with different times during the cue (Figure 1B). Finally, Figure 1C illustrates a possibility where feedback signals from one movement component are used to learn the next component. The design of most experiments does not permit distinguishing between these possibilities. For example, a number of studies (Choi and Moore, 2003; Moore and Choi, 1997; Freeman et al., 2003; Halverson et al., 2015; Jirenhed et al., 2017) used eyelid conditioning to train subjects to respond with a sequence of two eyelid responses. In these experiments, however, the external cue either explicitly extended through the whole sequence\n",
        "# \"\"\"\n",
        "# print(input_text)\n",
        "# summary = generate_summary(eval_model, eval_tokenizer, input_text)\n",
        "# print(summary)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rvgLI5Z9-ppX"
      },
      "outputs": [],
      "source": [
        "# !nproc\n",
        "# !tar cf - results | pigz -0 -p 12 > results.tar.gz\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uBcg9ZBwlcLL"
      },
      "source": [
        "# Evaluation on Held-Out Test Set"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": true,
        "id": "PpPjNzlPm5-p",
        "jupyter": {
          "outputs_hidden": true
        },
        "tags": [],
        "outputId": "3293f425-3c91-4f69-891b-ba919e825b10"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Looking in indexes: https://repo.huaweicloud.com/repository/pypi/simple\n",
            "Requirement already satisfied: datasets in ./miniconda3/lib/python3.8/site-packages (2.12.0)\n",
            "Requirement already satisfied: tqdm>=4.62.1 in ./miniconda3/lib/python3.8/site-packages (from datasets) (4.63.1)\n",
            "Requirement already satisfied: multiprocess in ./miniconda3/lib/python3.8/site-packages (from datasets) (0.70.12.2)\n",
            "Requirement already satisfied: aiohttp in ./miniconda3/lib/python3.8/site-packages (from datasets) (3.8.4)\n",
            "Requirement already satisfied: xxhash in ./miniconda3/lib/python3.8/site-packages (from datasets) (3.0.0)\n",
            "Requirement already satisfied: pandas in ./miniconda3/lib/python3.8/site-packages (from datasets) (2.0.0)\n",
            "Requirement already satisfied: huggingface-hub<1.0.0,>=0.11.0 in ./miniconda3/lib/python3.8/site-packages (from datasets) (0.13.4)\n",
            "Requirement already satisfied: dill<0.3.7,>=0.3.0 in ./miniconda3/lib/python3.8/site-packages (from datasets) (0.3.5.1)\n",
            "Requirement already satisfied: numpy>=1.17 in ./miniconda3/lib/python3.8/site-packages (from datasets) (1.24.2)\n",
            "Requirement already satisfied: pyarrow>=8.0.0 in ./miniconda3/lib/python3.8/site-packages (from datasets) (10.0.1)\n",
            "Requirement already satisfied: packaging in ./miniconda3/lib/python3.8/site-packages (from datasets) (23.0)\n",
            "Requirement already satisfied: responses<0.19 in ./miniconda3/lib/python3.8/site-packages (from datasets) (0.18.0)\n",
            "Requirement already satisfied: pyyaml>=5.1 in ./miniconda3/lib/python3.8/site-packages (from datasets) (6.0)\n",
            "Requirement already satisfied: requests>=2.19.0 in ./miniconda3/lib/python3.8/site-packages (from datasets) (2.28.2)\n",
            "Requirement already satisfied: fsspec[http]>=2021.11.1 in ./miniconda3/lib/python3.8/site-packages (from datasets) (2022.11.0)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in ./miniconda3/lib/python3.8/site-packages (from aiohttp->datasets) (1.8.1)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in ./miniconda3/lib/python3.8/site-packages (from aiohttp->datasets) (6.0.2)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in ./miniconda3/lib/python3.8/site-packages (from aiohttp->datasets) (4.0.2)\n",
            "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in ./miniconda3/lib/python3.8/site-packages (from aiohttp->datasets) (3.1.0)\n",
            "Requirement already satisfied: attrs>=17.3.0 in ./miniconda3/lib/python3.8/site-packages (from aiohttp->datasets) (22.2.0)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in ./miniconda3/lib/python3.8/site-packages (from aiohttp->datasets) (1.3.3)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in ./miniconda3/lib/python3.8/site-packages (from aiohttp->datasets) (1.3.1)\n",
            "Requirement already satisfied: filelock in ./miniconda3/lib/python3.8/site-packages (from huggingface-hub<1.0.0,>=0.11.0->datasets) (3.10.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in ./miniconda3/lib/python3.8/site-packages (from huggingface-hub<1.0.0,>=0.11.0->datasets) (4.5.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in ./miniconda3/lib/python3.8/site-packages (from requests>=2.19.0->datasets) (2021.5.30)\n",
            "Requirement already satisfied: idna<4,>=2.5 in ./miniconda3/lib/python3.8/site-packages (from requests>=2.19.0->datasets) (2.10)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in ./miniconda3/lib/python3.8/site-packages (from requests>=2.19.0->datasets) (1.26.6)\n",
            "Requirement already satisfied: python-dateutil>=2.8.2 in ./miniconda3/lib/python3.8/site-packages (from pandas->datasets) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in ./miniconda3/lib/python3.8/site-packages (from pandas->datasets) (2022.7.1)\n",
            "Requirement already satisfied: tzdata>=2022.1 in ./miniconda3/lib/python3.8/site-packages (from pandas->datasets) (2022.7)\n",
            "Requirement already satisfied: six>=1.5 in ./miniconda3/lib/python3.8/site-packages (from python-dateutil>=2.8.2->pandas->datasets) (1.16.0)\n",
            "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\n",
            "Looking in indexes: https://repo.huaweicloud.com/repository/pypi/simple\n",
            "Collecting rouge_score\n",
            "  Downloading https://repo.huaweicloud.com/repository/pypi/packages/e2/c5/9136736c37022a6ad27fea38f3111eb8f02fe75d067f9a985cc358653102/rouge_score-0.1.2.tar.gz (17 kB)\n",
            "Requirement already satisfied: absl-py in ./miniconda3/lib/python3.8/site-packages (from rouge_score) (1.4.0)\n",
            "Collecting nltk\n",
            "  Downloading https://repo.huaweicloud.com/repository/pypi/packages/43/0b/8298798bc5a9a007b7cae3f846a3d9a325953e0f9c238affa478b4d59324/nltk-3.7-py3-none-any.whl (1.5 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.5 MB 1.7 MB/s eta 0:00:01\n",
            "\u001b[?25hRequirement already satisfied: numpy in ./miniconda3/lib/python3.8/site-packages (from rouge_score) (1.24.2)\n",
            "Requirement already satisfied: six>=1.14.0 in ./miniconda3/lib/python3.8/site-packages (from rouge_score) (1.16.0)\n",
            "Collecting click\n",
            "  Downloading https://repo.huaweicloud.com/repository/pypi/packages/c2/f1/df59e28c642d583f7dacffb1e0965d0e00b218e0186d7858ac5233dce840/click-8.1.3-py3-none-any.whl (96 kB)\n",
            "\u001b[K     |████████████████████████████████| 96 kB 22.7 MB/s eta 0:00:01\n",
            "\u001b[?25hRequirement already satisfied: tqdm in ./miniconda3/lib/python3.8/site-packages (from nltk->rouge_score) (4.63.1)\n",
            "Requirement already satisfied: regex>=2021.8.3 in ./miniconda3/lib/python3.8/site-packages (from nltk->rouge_score) (2022.10.31)\n",
            "Requirement already satisfied: joblib in ./miniconda3/lib/python3.8/site-packages (from nltk->rouge_score) (1.1.0)\n",
            "Building wheels for collected packages: rouge-score\n",
            "  Building wheel for rouge-score (setup.py) ... \u001b[?25ldone\n",
            "\u001b[?25h  Created wheel for rouge-score: filename=rouge_score-0.1.2-py3-none-any.whl size=24956 sha256=e6d5db37fc3c1c78863f964ea36a17591550c08d1c98f32d337da81b8db71f9a\n",
            "  Stored in directory: /root/.cache/pip/wheels/fa/9b/18/7518ccfe3107aca76ec573a2fa1264e00efb0f3168a66f09c2\n",
            "Successfully built rouge-score\n",
            "Installing collected packages: click, nltk, rouge-score\n",
            "Successfully installed click-8.1.3 nltk-3.7 rouge-score-0.1.2\n",
            "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\n"
          ]
        }
      ],
      "source": [
        "# %%capture\n",
        "# !pip3 install datasets\n",
        "# !pip3 install rouge_score"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "SDWwfl8FldB9",
        "tags": [],
        "outputId": "5e1dd845-ef8a-4d36-f185-4d0bfdd46b9f"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/tmp/ipykernel_11011/1100646279.py:2: FutureWarning: list_metrics is deprecated and will be removed in the next major version of datasets. Use 'evaluate.list_evaluation_modules' instead, from the new library 🤗 Evaluate: https://huggingface.co/docs/evaluate\n",
            "  print(list_metrics())\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "['accuracy', 'bertscore', 'bleu', 'bleurt', 'brier_score', 'cer', 'character', 'charcut_mt', 'chrf', 'code_eval', 'comet', 'competition_math', 'coval', 'cuad', 'exact_match', 'f1', 'frugalscore', 'glue', 'google_bleu', 'indic_glue', 'mae', 'mahalanobis', 'mape', 'mase', 'matthews_correlation', 'mauve', 'mean_iou', 'meteor', 'mse', 'nist_mt', 'pearsonr', 'perplexity', 'poseval', 'precision', 'r_squared', 'recall', 'rl_reliability', 'roc_auc', 'rouge', 'sacrebleu', 'sari', 'seqeval', 'smape', 'spearmanr', 'squad', 'squad_v2', 'super_glue', 'ter', 'trec_eval', 'wer', 'wiki_split', 'xnli', 'xtreme_s', 'AlhitawiMohammed22/CER_Hu-Evaluation-Metrics', 'Asmedeus/perplexity', 'BucketHeadP65/confusion_matrix', 'BucketHeadP65/roc_curve', 'Drunper/metrica_tesi', 'Felipehonorato/eer', 'GMFTBY/dailydialog_evaluate', 'GMFTBY/dailydialogevaluate', 'JP-SystemsX/nDCG', 'Josh98/nl2bash_m', 'KevinSpaghetti/accuracyk', 'Muennighoff/code_eval', 'NCSOFT/harim_plus', 'NikitaMartynov/spell-check-metric', 'NimaBoscarino/weat', 'Ochiroo/rouge_mn', 'Splend1dchan/cosine_similarity', 'Vertaix/vendiscore', 'Viona/fuzzy_reordering', 'Viona/infolm', 'Viona/kendall_tau', 'Vlasta/pr_auc', 'Yeshwant123/mcc', 'abdusah/aradiawer', 'abidlabs/mean_iou', 'abidlabs/mean_iou2', 'andstor/code_perplexity', 'angelina-wang/directional_bias_amplification', 'anz2/iliauniiccocrevaluation', 'bstrai/classification_report', 'cakiki/ndcg', 'chanelcolgate/average_precision', 'ckb/unigram', 'codeparrot/apps_metric', 'cpllab/syntaxgym', 'daiyizheng/valid', 'dvitel/codebleu', 'ecody726/bertscore', 'erntkn/dice_coefficient', 'giulio98/code_eval_outputs', 'giulio98/codebleu', 'gnail/cosine_similarity', 'gorkaartola/metric_for_tp_fp_samples', 'hack/test_metric', 'harshhpareek/bertscore', 'hpi-dhc/FairEval', 'hxw15/sari_metric', 'hynky/sklearn_proxy', 'idsedykh/codebleu', 'idsedykh/codebleu2', 'idsedykh/megaglue', 'idsedykh/metric', 'jordyvl/ece', 'jpxkqx/peak_signal_to_noise_ratio', 'jpxkqx/signal_to_reconstrution_error', 'jzm-mailchimp/joshs_second_test_metric', 'kaggle/ai4code', 'kaggle/amex', 'kashif/mape', 'kyokote/my_metric2', 'leslyarun/fbeta_score', 'lhy/hamming_loss', 'lhy/ranking_loss', 'loubnabnl/apps_metric2', 'lvwerra/accuracy_score', 'lvwerra/bary_score', 'lvwerra/test', 'manueldeprada/beer', 'mfumanelli/geometric_mean', 'mgfrantz/roc_auc_macro', 'ola13/precision_at_k', 'omidf/squad_precision_recall', 'posicube/mean_reciprocal_rank', 'ronaldahmed/nwentfaithfulness', 'shunzh/apps_metric', 'sma2023/wil', 'sportlosos/sescore', 'tialaeMceryu/unigram', 'transZ/sbert_cosine', 'transZ/test_parascore', 'transformersegmentation/segmentation_scores', 'unnati/kendall_tau_distance', 'weiqis/pajm', 'xu1998hz/sescore', 'xu1998hz/sescore_english_coco', 'xu1998hz/sescore_english_mt', 'xu1998hz/sescore_english_webnlg', 'xu1998hz/sescore_german_mt', 'ybelkada/cocoevaluate', 'yonting/average_precision_score', 'yulong-me/yl_metric', 'yuyijiong/quad_match_score', 'yzha/ctc_eval', 'zbeloki/m2']\n"
          ]
        }
      ],
      "source": [
        "from datasets import load_metric, list_metrics\n",
        "print(list_metrics())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": true,
        "id": "2g-lMsTbrAd2",
        "jupyter": {
          "outputs_hidden": true
        },
        "tags": [],
        "outputId": "23db795f-e14f-4cb9-e49c-c780f6709f3b"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loading configuration file ./results/checkpoint-2500/config.json\n",
            "Model config PegasusXConfig {\n",
            "  \"_name_or_path\": \"google/pegasus-x-large\",\n",
            "  \"activation_dropout\": 0.1,\n",
            "  \"activation_function\": \"relu\",\n",
            "  \"add_bias_logits\": false,\n",
            "  \"add_final_layer_norm\": true,\n",
            "  \"architectures\": [\n",
            "    \"PegasusXForConditionalGeneration\"\n",
            "  ],\n",
            "  \"attention_dropout\": 0.1,\n",
            "  \"block_size\": 512,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classif_dropout\": 0.0,\n",
            "  \"classifier_dropout\": 0.0,\n",
            "  \"d_model\": 1024,\n",
            "  \"decoder_attention_heads\": 16,\n",
            "  \"decoder_ffn_dim\": 4096,\n",
            "  \"decoder_layerdrop\": 0.0,\n",
            "  \"decoder_layers\": 16,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"dropout\": 0.1,\n",
            "  \"encoder_attention_heads\": 16,\n",
            "  \"encoder_ffn_dim\": 4096,\n",
            "  \"encoder_layerdrop\": 0.0,\n",
            "  \"encoder_layers\": 16,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"extra_pos_embeddings\": 1,\n",
            "  \"force_bos_token_to_be_generated\": false,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"gradient_checkpointing\": false,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\"\n",
            "  },\n",
            "  \"init_std\": 0.02,\n",
            "  \"is_encoder_decoder\": true,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2\n",
            "  },\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"max_position_embeddings\": 16384,\n",
            "  \"model_type\": \"pegasus_x\",\n",
            "  \"normalize_before\": true,\n",
            "  \"normalize_embedding\": false,\n",
            "  \"num_beams\": 8,\n",
            "  \"num_global_tokens\": 128,\n",
            "  \"num_hidden_layers\": 16,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"scale_embedding\": true,\n",
            "  \"stagger_local_blocks\": true,\n",
            "  \"static_position_embeddings\": true,\n",
            "  \"torch_dtype\": \"float32\",\n",
            "  \"transformers_version\": \"4.27.4\",\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 96103\n",
            "}\n",
            "\n",
            "loading weights file ./results/checkpoint-2500/pytorch_model.bin\n",
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n",
            "All model checkpoint weights were used when initializing PegasusXForConditionalGeneration.\n",
            "\n",
            "All the weights of PegasusXForConditionalGeneration were initialized from the model checkpoint at ./results/checkpoint-2500.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use PegasusXForConditionalGeneration for predictions without further training.\n",
            "loading configuration file ./results/checkpoint-2500/generation_config.json\n",
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "finished this!\n"
          ]
        }
      ],
      "source": [
        "# model_name = 'google/bigbird-pegasus-large-pubmed'\n",
        "# model_path = \"checkpoint-3500\"\n",
        "# model_path = \"checkpoint-4000\"\n",
        "checkpoint_path = \"./results/checkpoint-2500\"\n",
        "\n",
        "\n",
        "torch_device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
        "eval_model = PegasusXForConditionalGeneration.from_pretrained(checkpoint_path).to(torch_device)\n",
        "\n",
        "# Put the model in evaluation mode\n",
        "eval_model.eval()\n",
        "\n",
        "print(\"finished this!\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "c_s74ClXrI8d"
      },
      "source": [
        "Let's take 2 samples and verify the predictions to be sure everything works as expected"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "RsuhzbMNrJVA",
        "outputId": "1c55b950-fc75-457c-c82b-3d2bc51e7b8b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "241\n"
          ]
        }
      ],
      "source": [
        "print(len(test_dataset))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "W9cZ2UX9stMI"
      },
      "source": [
        "Finally, we can evaluate the predictions using the *rouge* metric."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "tags": [],
        "id": "6SE3numjgPK7",
        "outputId": "24531e95-8ce3-4208-cd3b-8087944f6352"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "241\n",
            "Average number of test extractives tokens: 820.5311203319502\n",
            "Average number of test summary tokens: 383.09128630705396\n"
          ]
        }
      ],
      "source": [
        "# Read and parse the JSON file into a Python list\n",
        "with open(\"mimicDataset/test_extractives.json\", \"r\") as file:\n",
        "    data = json.load(file)\n",
        "print(len(data))\n",
        "# Calculate the total number of tokens (words) in the list\n",
        "total_tokens = 0\n",
        "for string in data:\n",
        "    tokens = string.split()  # Split the string into tokens by whitespace\n",
        "    total_tokens += len(tokens)\n",
        "\n",
        "# Calculate the average number of tokens per string\n",
        "average_tokens_ext = total_tokens / len(data)\n",
        "print(f\"Average number of test extractives tokens: {average_tokens_ext}\")\n",
        "\n",
        "with open(\"mimicDataset/test_summaries.json\", \"r\") as file:\n",
        "    data = json.load(file)\n",
        "\n",
        "# Calculate the total number of tokens (words) in the list\n",
        "total_tokens = 0\n",
        "for string in data:\n",
        "    tokens = string.split()  # Split the string into tokens by whitespace\n",
        "    total_tokens += len(tokens)\n",
        "\n",
        "# Calculate the average number of tokens per string\n",
        "average_tokens_sums = total_tokens / len(data)\n",
        "print(f\"Average number of test summary tokens: {average_tokens_sums}\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "tags": [],
        "id": "CRuEPGywgPK7",
        "outputId": "f21f06f8-c04e-494a-e555-5004529b1f2f"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "0\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "3\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "4\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "5\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "6\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "7\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "8\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "9\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "10\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "11\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "12\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "13\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "14\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "15\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "16\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "17\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "18\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "19\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "20\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "21\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "22\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "23\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "24\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "25\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "26\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "27\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "28\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "29\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "30\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "31\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "32\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "33\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "34\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "35\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "36\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "37\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "38\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "39\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "40\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "41\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "42\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "43\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "44\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "45\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "46\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "47\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "48\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "49\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "50\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "51\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "52\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "53\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "54\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "55\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "56\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "57\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "58\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "59\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "60\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "61\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "62\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "63\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "64\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "65\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "66\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "67\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "68\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "69\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "70\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "71\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "72\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "73\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "74\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "75\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "76\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "77\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "78\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "79\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "80\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "81\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "82\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "83\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "84\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "85\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "86\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "87\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "88\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "89\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "90\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "91\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "92\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "93\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "94\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "95\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "96\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "97\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "98\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "99\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "100\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "101\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "102\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "103\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "104\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "105\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "106\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "107\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "108\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "109\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "110\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "111\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "112\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "113\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "114\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "115\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "116\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "117\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "118\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "119\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "120\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "121\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "122\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "123\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "124\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "125\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "126\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "127\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "128\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "129\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "130\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "131\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "132\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "133\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "134\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "135\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "136\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "137\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "138\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "139\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "140\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "141\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "142\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "143\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "144\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "145\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "146\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "147\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "148\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "149\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "150\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "151\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "152\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "153\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "154\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "155\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "156\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "157\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "158\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "159\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "160\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "161\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "162\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "163\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "164\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "165\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "166\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "167\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "168\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "169\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "170\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "171\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "172\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "173\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "174\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "175\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "176\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "177\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "178\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "179\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "180\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "181\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "182\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "183\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "184\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "185\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "186\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "187\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "188\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "189\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "190\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "191\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "192\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "193\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "194\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "195\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "196\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "197\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "198\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "199\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "200\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "201\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "202\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "203\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "204\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "205\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "206\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "207\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "208\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "209\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "210\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "211\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "212\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "213\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "214\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "215\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "216\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "217\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "218\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "219\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "220\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "221\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "222\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "223\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "224\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "225\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "226\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "227\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "228\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "229\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "230\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "231\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "232\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "233\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "234\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "235\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "236\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "237\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "238\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "239\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generate config GenerationConfig {\n",
            "  \"_from_model_config\": true,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"forced_eos_token_id\": 1,\n",
            "  \"length_penalty\": 0.8,\n",
            "  \"max_length\": 16384,\n",
            "  \"num_beams\": 8,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"transformers_version\": \"4.27.4\"\n",
            "}\n",
            "\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "240\n",
            "finished generate!\n"
          ]
        }
      ],
      "source": [
        "import numpy as np\n",
        "# Generate summaries for the test dataset\n",
        "generated_summaries = []\n",
        "for num, example in enumerate(test_dataset):\n",
        "    print(num)\n",
        "    input_ids = example['input_ids'].unsqueeze(0)\n",
        "    input_ids = input_ids.to(eval_model.device)\n",
        "    attention_mask = example['attention_mask'].unsqueeze(0)\n",
        "    attention_mask = attention_mask.to(eval_model.device)\n",
        "    summary_ids = eval_model.generate(input_ids=input_ids, attention_mask=attention_mask, max_length=round(average_tokens_sums), num_beams=3, length_penalty=0.8)\n",
        "    summary = tokenizer.decode(summary_ids[0], skip_special_tokens=True)\n",
        "    generated_summaries.append(summary)\n",
        "    # print(summary)\n",
        "np.save('gen_sums_finetuned_bertsum.npy', np.array(generated_summaries))\n",
        "\n",
        "print(\"finished generate!\")"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Rouge"
      ],
      "metadata": {
        "id": "nluwh7C3gzKE"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "TjvDOFM7gPK7",
        "outputId": "08ac54c8-836a-4041-8766-3ee13702636f"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "load Rouge matric\n",
            "241\n"
          ]
        }
      ],
      "source": [
        "print(\"load Rouge matric\")\n",
        "rouge = load_metric(\"rouge\")\n",
        "print(len(test_summaries))\n",
        "# Create a list to store the reference summaries\n",
        "gen_sums = np.load('gen_sums_finetuned_bertsum.npy')\n",
        "reference_summaries = test_summaries\n",
        "\n",
        "# Compute ROUGE-L scores\n",
        "rouge_results = rouge.compute(predictions=gen_sums, references=reference_summaries, rouge_types=[\"rougeL\", \"rouge1\", \"rouge2\"], use_aggregator=True)\n",
        "\n",
        "# Access the ROUGE-L score\n",
        "rouge_l_score = rouge_results[\"rougeL\"].mid.fmeasure\n",
        "rouge_1_score = rouge_results[\"rouge1\"].mid.fmeasure\n",
        "rouge_2_score = rouge_results[\"rouge2\"].mid.fmeasure\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "tags": [],
        "id": "dpXCYL17gPK7",
        "outputId": "db664f1a-b2ea-4d1b-b5f6-060b7a9f54a0"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Rouge scores for finetuned finetuned Pegasus-x-large on 2k andy data\n",
            "ROUGE-L Aggregated F1 Score: 0.18643294804538302\n",
            "ROUGE-1 Aggregated F1 Score: 0.33473625347582453\n",
            "ROUGE-2 Aggregated F1 Score: 0.0684724793511705\n"
          ]
        }
      ],
      "source": [
        "# Print the ROUGE-L score\n",
        "print(\"Rouge scores for finetuned finetuned Pegasus-x-large on 2k andy data\")\n",
        "print(\"ROUGE-L Aggregated F1 Score:\", rouge_l_score)\n",
        "print(\"ROUGE-1 Aggregated F1 Score:\", rouge_1_score)\n",
        "print(\"ROUGE-2 Aggregated F1 Score:\", rouge_2_score)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### BLEU(NOT USE)"
      ],
      "metadata": {
        "id": "6nBpCtRqg1tF"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "S7GYrurNgPK7"
      },
      "outputs": [],
      "source": [
        "# # BLEU\n",
        "# import nltk\n",
        "# nltk.download('punkt')\n",
        "# bleu = load_metric(\"bleu\")\n",
        "\n",
        "# gen_sums = np.load('gen_sums_bertsum.npy').reshape(-1,1).tolist()\n",
        "# ref_sums = np.array(test_summaries)\n",
        "# ref_sums = np.expand_dims(np.expand_dims(ref_sums, axis=1), axis=1).tolist()\n",
        "# # print(len(gen_sums[0][0]))\n",
        "# # print(len(ref_sums[0][0][0]))\n",
        "# # print(gen_sums[0][0])\n",
        "# # print(ref_sums[0][0][0])\n",
        "\n",
        "# def custom_tokenizer(text):\n",
        "#     return nltk.word_tokenize(text)\n",
        "\n",
        "# # Tokenize the predictions and references\n",
        "# # tokenized_preds = [[custom_tokenizer(pred) for pred in pred] for pred in gen_sums]\n",
        "# # tokenized_references = [[[custom_tokenizer(ref) for ref in refs] for refs in group] for group in ref_sums]\n",
        "\n",
        "# # print(tokenized_preds)\n",
        "\n",
        "# bleu_results = bleu.compute(predictions=gen_sums, references=ref_sums, max_order=1, tokenizer=word_tokenize)\n",
        "\n",
        "# # Access the BLEU score\n",
        "# # bleu_score = bleu_results[\"bleu\"]\n",
        "\n",
        "# # print(\"BLEU Score n=1:\", bleu_score)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": true,
        "jupyter": {
          "outputs_hidden": true
        },
        "tags": [],
        "id": "2KP2W8wIgPK8",
        "outputId": "b608bcef-5c5c-4830-f438-05c149588dd8"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Looking in indexes: https://repo.huaweicloud.com/repository/pypi/simple\n",
            "Collecting bert_score\n",
            "  Downloading https://repo.huaweicloud.com/repository/pypi/packages/c6/8c/bc5457de4c004b1a623b31f7bc8d0375fb699b7d67df11879098b4b7b7c8/bert_score-0.3.13-py3-none-any.whl (61 kB)\n",
            "\u001b[K     |████████████████████████████████| 61 kB 1.5 MB/s eta 0:00:01\n",
            "\u001b[?25hRequirement already satisfied: packaging>=20.9 in ./miniconda3/lib/python3.8/site-packages (from bert_score) (23.0)\n",
            "Requirement already satisfied: matplotlib in ./miniconda3/lib/python3.8/site-packages (from bert_score) (3.7.1)\n",
            "Requirement already satisfied: tqdm>=4.31.1 in ./miniconda3/lib/python3.8/site-packages (from bert_score) (4.63.1)\n",
            "Requirement already satisfied: torch>=1.0.0 in ./miniconda3/lib/python3.8/site-packages (from bert_score) (2.0.0+cu118)\n",
            "Requirement already satisfied: requests in ./miniconda3/lib/python3.8/site-packages (from bert_score) (2.28.2)\n",
            "Requirement already satisfied: pandas>=1.0.1 in ./miniconda3/lib/python3.8/site-packages (from bert_score) (2.0.0)\n",
            "Requirement already satisfied: numpy in ./miniconda3/lib/python3.8/site-packages (from bert_score) (1.24.2)\n",
            "Requirement already satisfied: transformers>=3.0.0 in ./miniconda3/lib/python3.8/site-packages (from bert_score) (4.27.4)\n",
            "Requirement already satisfied: pytz>=2020.1 in ./miniconda3/lib/python3.8/site-packages (from pandas>=1.0.1->bert_score) (2022.7.1)\n",
            "Requirement already satisfied: python-dateutil>=2.8.2 in ./miniconda3/lib/python3.8/site-packages (from pandas>=1.0.1->bert_score) (2.8.2)\n",
            "Requirement already satisfied: tzdata>=2022.1 in ./miniconda3/lib/python3.8/site-packages (from pandas>=1.0.1->bert_score) (2022.7)\n",
            "Requirement already satisfied: six>=1.5 in ./miniconda3/lib/python3.8/site-packages (from python-dateutil>=2.8.2->pandas>=1.0.1->bert_score) (1.16.0)\n",
            "Requirement already satisfied: networkx in ./miniconda3/lib/python3.8/site-packages (from torch>=1.0.0->bert_score) (3.0)\n",
            "Requirement already satisfied: typing-extensions in ./miniconda3/lib/python3.8/site-packages (from torch>=1.0.0->bert_score) (4.5.0)\n",
            "Requirement already satisfied: filelock in ./miniconda3/lib/python3.8/site-packages (from torch>=1.0.0->bert_score) (3.10.0)\n",
            "Requirement already satisfied: sympy in ./miniconda3/lib/python3.8/site-packages (from torch>=1.0.0->bert_score) (1.11.1)\n",
            "Requirement already satisfied: jinja2 in ./miniconda3/lib/python3.8/site-packages (from torch>=1.0.0->bert_score) (3.1.2)\n",
            "Requirement already satisfied: triton==2.0.0 in ./miniconda3/lib/python3.8/site-packages (from torch>=1.0.0->bert_score) (2.0.0)\n",
            "Requirement already satisfied: lit in ./miniconda3/lib/python3.8/site-packages (from triton==2.0.0->torch>=1.0.0->bert_score) (15.0.7)\n",
            "Requirement already satisfied: cmake in ./miniconda3/lib/python3.8/site-packages (from triton==2.0.0->torch>=1.0.0->bert_score) (3.26.0)\n",
            "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in ./miniconda3/lib/python3.8/site-packages (from transformers>=3.0.0->bert_score) (0.13.2)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.11.0 in ./miniconda3/lib/python3.8/site-packages (from transformers>=3.0.0->bert_score) (0.13.4)\n",
            "Requirement already satisfied: regex!=2019.12.17 in ./miniconda3/lib/python3.8/site-packages (from transformers>=3.0.0->bert_score) (2022.10.31)\n",
            "Requirement already satisfied: pyyaml>=5.1 in ./miniconda3/lib/python3.8/site-packages (from transformers>=3.0.0->bert_score) (6.0)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in ./miniconda3/lib/python3.8/site-packages (from jinja2->torch>=1.0.0->bert_score) (2.1.2)\n",
            "Requirement already satisfied: cycler>=0.10 in ./miniconda3/lib/python3.8/site-packages (from matplotlib->bert_score) (0.11.0)\n",
            "Requirement already satisfied: contourpy>=1.0.1 in ./miniconda3/lib/python3.8/site-packages (from matplotlib->bert_score) (1.0.7)\n",
            "Requirement already satisfied: pyparsing>=2.3.1 in ./miniconda3/lib/python3.8/site-packages (from matplotlib->bert_score) (3.0.9)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in ./miniconda3/lib/python3.8/site-packages (from matplotlib->bert_score) (1.4.4)\n",
            "Requirement already satisfied: importlib-resources>=3.2.0 in ./miniconda3/lib/python3.8/site-packages (from matplotlib->bert_score) (5.12.0)\n",
            "Requirement already satisfied: fonttools>=4.22.0 in ./miniconda3/lib/python3.8/site-packages (from matplotlib->bert_score) (4.39.0)\n",
            "Requirement already satisfied: pillow>=6.2.0 in ./miniconda3/lib/python3.8/site-packages (from matplotlib->bert_score) (9.4.0)\n",
            "Requirement already satisfied: zipp>=3.1.0 in ./miniconda3/lib/python3.8/site-packages (from importlib-resources>=3.2.0->matplotlib->bert_score) (3.15.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in ./miniconda3/lib/python3.8/site-packages (from requests->bert_score) (3.1.0)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in ./miniconda3/lib/python3.8/site-packages (from requests->bert_score) (1.26.6)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in ./miniconda3/lib/python3.8/site-packages (from requests->bert_score) (2021.5.30)\n",
            "Requirement already satisfied: idna<4,>=2.5 in ./miniconda3/lib/python3.8/site-packages (from requests->bert_score) (2.10)\n",
            "Requirement already satisfied: mpmath>=0.19 in ./miniconda3/lib/python3.8/site-packages (from sympy->torch>=1.0.0->bert_score) (1.3.0)\n",
            "Installing collected packages: bert-score\n",
            "Successfully installed bert-score-0.3.13\n",
            "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\n"
          ]
        }
      ],
      "source": [
        "# !pip install bert_score"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### BERTScore"
      ],
      "metadata": {
        "id": "TQKPYVE3g5hW"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "tags": [],
        "id": "wicRNYUGgPK8",
        "outputId": "dd98c72d-9c27-43be-81c5-f18d2a2b271f"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Load the BERTScore metric\n",
            "Compute BERTScore using the loaded metric\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Could not locate the tokenizer configuration file, will try to use the model config instead.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-large/snapshots/716877d372b884cad6d419d828bac6c85b3b18d9/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"_name_or_path\": \"roberta-large\",\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 1024,\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 4096,\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 16,\n",
            "  \"num_hidden_layers\": 24,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.27.4\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading file vocab.json from cache at /root/.cache/huggingface/hub/models--roberta-large/snapshots/716877d372b884cad6d419d828bac6c85b3b18d9/vocab.json\n",
            "loading file merges.txt from cache at /root/.cache/huggingface/hub/models--roberta-large/snapshots/716877d372b884cad6d419d828bac6c85b3b18d9/merges.txt\n",
            "loading file added_tokens.json from cache at None\n",
            "loading file special_tokens_map.json from cache at None\n",
            "loading file tokenizer_config.json from cache at None\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-large/snapshots/716877d372b884cad6d419d828bac6c85b3b18d9/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"_name_or_path\": \"roberta-large\",\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 1024,\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 4096,\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 16,\n",
            "  \"num_hidden_layers\": 24,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.27.4\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-large/snapshots/716877d372b884cad6d419d828bac6c85b3b18d9/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"_name_or_path\": \"roberta-large\",\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 1024,\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 4096,\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 16,\n",
            "  \"num_hidden_layers\": 24,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.27.4\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-large/snapshots/716877d372b884cad6d419d828bac6c85b3b18d9/pytorch_model.bin\n",
            "All the weights of RobertaModel were initialized from the model checkpoint at roberta-large.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use RobertaModel for predictions without further training.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "compute BERTScore finished!\n"
          ]
        }
      ],
      "source": [
        "# Load the BERTScore metric\n",
        "print(\"Load the BERTScore metric\")\n",
        "bert_score_metric = load_metric(\"bertscore\")\n",
        "\n",
        "gen_sums = np.load('gen_sums_finetuned_bertsum-xLarge1.npy')\n",
        "ref_sums = test_summaries\n",
        "\n",
        "# Compute BERTScore using the loaded metric\n",
        "print(\"Compute BERTScore using the loaded metric\")\n",
        "bertscore_results = bert_score_metric.compute(predictions=gen_sums, references=ref_sums, lang=\"en\")\n",
        "\n",
        "print(\"compute BERTScore finished!\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "gSbGOkr3gPK8",
        "outputId": "5a03fdf8-ace6-4192-ab1a-ac44be2f26de"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Avg BertScore-F1 of finetuned Pegasus-x-large on andy data 2k:  0.8180586609108319\n"
          ]
        }
      ],
      "source": [
        "avg_bert_score = np.array(bertscore_results['f1']).mean()\n",
        "print(\"Avg BertScore-F1 of finetuned Pegasus-x-large on andy data 2k: \", avg_bert_score)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Meteor"
      ],
      "metadata": {
        "id": "JKHrhkOlg8xK"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "tags": [],
        "id": "d-kHSVsngPK8",
        "outputId": "3a20d459-4eb7-4ffb-bc49-221b7e7ef868"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "load meteor!\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]   Package wordnet is already up-to-date!\n",
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n",
            "[nltk_data] Downloading package omw-1.4 to /root/nltk_data...\n"
          ]
        },
        {
          "ename": "KeyboardInterrupt",
          "evalue": "",
          "output_type": "error",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "Cell \u001b[0;32mIn[23], line 2\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mload meteor!\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m----> 2\u001b[0m meteor \u001b[38;5;241m=\u001b[39m \u001b[43mload_metric\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mmeteor\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m      4\u001b[0m gen_sums \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39mload(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mgen_sums_finetuned_bertsum.npy\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m      5\u001b[0m ref_sums \u001b[38;5;241m=\u001b[39m test_summaries\n",
            "File \u001b[0;32m~/miniconda3/lib/python3.8/site-packages/datasets/utils/deprecation_utils.py:46\u001b[0m, in \u001b[0;36mdeprecated.<locals>.decorator.<locals>.wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     44\u001b[0m     warnings\u001b[38;5;241m.\u001b[39mwarn(warning_msg, category\u001b[38;5;241m=\u001b[39m\u001b[38;5;167;01mFutureWarning\u001b[39;00m, stacklevel\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m2\u001b[39m)\n\u001b[1;32m     45\u001b[0m     _emitted_deprecation_warnings\u001b[38;5;241m.\u001b[39madd(func_hash)\n\u001b[0;32m---> 46\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mdeprecated_function\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
            "File \u001b[0;32m~/miniconda3/lib/python3.8/site-packages/datasets/load.py:1400\u001b[0m, in \u001b[0;36mload_metric\u001b[0;34m(path, config_name, process_id, num_process, cache_dir, experiment_id, keep_in_memory, download_config, download_mode, revision, **metric_init_kwargs)\u001b[0m\n\u001b[1;32m   1389\u001b[0m metric \u001b[38;5;241m=\u001b[39m metric_cls(\n\u001b[1;32m   1390\u001b[0m     config_name\u001b[38;5;241m=\u001b[39mconfig_name,\n\u001b[1;32m   1391\u001b[0m     process_id\u001b[38;5;241m=\u001b[39mprocess_id,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   1396\u001b[0m     \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mmetric_init_kwargs,\n\u001b[1;32m   1397\u001b[0m )\n\u001b[1;32m   1399\u001b[0m \u001b[38;5;66;03m# Download and prepare resources for the metric\u001b[39;00m\n\u001b[0;32m-> 1400\u001b[0m \u001b[43mmetric\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdownload_and_prepare\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdownload_config\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdownload_config\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1402\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m metric\n",
            "File \u001b[0;32m~/miniconda3/lib/python3.8/site-packages/datasets/metric.py:625\u001b[0m, in \u001b[0;36mMetric.download_and_prepare\u001b[0;34m(self, download_config, dl_manager)\u001b[0m\n\u001b[1;32m    619\u001b[0m         download_config\u001b[38;5;241m.\u001b[39mforce_download \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[1;32m    621\u001b[0m     dl_manager \u001b[38;5;241m=\u001b[39m DownloadManager(\n\u001b[1;32m    622\u001b[0m         dataset_name\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mname, download_config\u001b[38;5;241m=\u001b[39mdownload_config, data_dir\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdata_dir\n\u001b[1;32m    623\u001b[0m     )\n\u001b[0;32m--> 625\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_download_and_prepare\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdl_manager\u001b[49m\u001b[43m)\u001b[49m\n",
            "File \u001b[0;32m~/.cache/huggingface/modules/datasets_modules/metrics/meteor/87ffc79bd41e3f7bc54f1d2829208e20ce309558c8340daad0d369e0802455d6/meteor.py:111\u001b[0m, in \u001b[0;36mMeteor._download_and_prepare\u001b[0;34m(self, dl_manager)\u001b[0m\n\u001b[1;32m    109\u001b[0m     nltk\u001b[38;5;241m.\u001b[39mdownload(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpunkt\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m    110\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m NLTK_VERSION \u001b[38;5;241m>\u001b[39m\u001b[38;5;241m=\u001b[39m version\u001b[38;5;241m.\u001b[39mVersion(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m3.6.6\u001b[39m\u001b[38;5;124m\"\u001b[39m):\n\u001b[0;32m--> 111\u001b[0m     \u001b[43mnltk\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdownload\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43momw-1.4\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n",
            "File \u001b[0;32m~/miniconda3/lib/python3.8/site-packages/nltk/downloader.py:777\u001b[0m, in \u001b[0;36mDownloader.download\u001b[0;34m(self, info_or_id, download_dir, quiet, force, prefix, halt_on_error, raise_on_error, print_error_to)\u001b[0m\n\u001b[1;32m    768\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mshow\u001b[39m(s, prefix2\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m\"\u001b[39m):\n\u001b[1;32m    769\u001b[0m     print_to(\n\u001b[1;32m    770\u001b[0m         textwrap\u001b[38;5;241m.\u001b[39mfill(\n\u001b[1;32m    771\u001b[0m             s,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    774\u001b[0m         )\n\u001b[1;32m    775\u001b[0m     )\n\u001b[0;32m--> 777\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m msg \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mincr_download(info_or_id, download_dir, force):\n\u001b[1;32m    778\u001b[0m     \u001b[38;5;66;03m# Error messages\u001b[39;00m\n\u001b[1;32m    779\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(msg, ErrorMessage):\n\u001b[1;32m    780\u001b[0m         show(msg\u001b[38;5;241m.\u001b[39mmessage)\n",
            "File \u001b[0;32m~/miniconda3/lib/python3.8/site-packages/nltk/downloader.py:642\u001b[0m, in \u001b[0;36mDownloader.incr_download\u001b[0;34m(self, info_or_id, download_dir, force)\u001b[0m\n\u001b[1;32m    638\u001b[0m     \u001b[38;5;28;01myield\u001b[39;00m FinishCollectionMessage(info)\n\u001b[1;32m    640\u001b[0m \u001b[38;5;66;03m# Handle Packages (delegate to a helper function).\u001b[39;00m\n\u001b[1;32m    641\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 642\u001b[0m     \u001b[38;5;28;01myield from\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_download_package(info, download_dir, force)\n",
            "File \u001b[0;32m~/miniconda3/lib/python3.8/site-packages/nltk/downloader.py:712\u001b[0m, in \u001b[0;36mDownloader._download_package\u001b[0;34m(self, info, download_dir, force)\u001b[0m\n\u001b[1;32m    710\u001b[0m num_blocks \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mmax\u001b[39m(\u001b[38;5;241m1\u001b[39m, info\u001b[38;5;241m.\u001b[39msize \u001b[38;5;241m/\u001b[39m (\u001b[38;5;241m1024\u001b[39m \u001b[38;5;241m*\u001b[39m \u001b[38;5;241m16\u001b[39m))\n\u001b[1;32m    711\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m block \u001b[38;5;129;01min\u001b[39;00m itertools\u001b[38;5;241m.\u001b[39mcount():\n\u001b[0;32m--> 712\u001b[0m     s \u001b[38;5;241m=\u001b[39m \u001b[43minfile\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m1024\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;241;43m16\u001b[39;49m\u001b[43m)\u001b[49m  \u001b[38;5;66;03m# 16k blocks.\u001b[39;00m\n\u001b[1;32m    713\u001b[0m     outfile\u001b[38;5;241m.\u001b[39mwrite(s)\n\u001b[1;32m    714\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m s:\n",
            "File \u001b[0;32m~/miniconda3/lib/python3.8/http/client.py:455\u001b[0m, in \u001b[0;36mHTTPResponse.read\u001b[0;34m(self, amt)\u001b[0m\n\u001b[1;32m    452\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m amt \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    453\u001b[0m     \u001b[38;5;66;03m# Amount is given, implement using readinto\u001b[39;00m\n\u001b[1;32m    454\u001b[0m     b \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mbytearray\u001b[39m(amt)\n\u001b[0;32m--> 455\u001b[0m     n \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mreadinto\u001b[49m\u001b[43m(\u001b[49m\u001b[43mb\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    456\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mmemoryview\u001b[39m(b)[:n]\u001b[38;5;241m.\u001b[39mtobytes()\n\u001b[1;32m    457\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    458\u001b[0m     \u001b[38;5;66;03m# Amount is not given (unbounded read) so we must check self.length\u001b[39;00m\n\u001b[1;32m    459\u001b[0m     \u001b[38;5;66;03m# and self.chunked\u001b[39;00m\n",
            "File \u001b[0;32m~/miniconda3/lib/python3.8/http/client.py:499\u001b[0m, in \u001b[0;36mHTTPResponse.readinto\u001b[0;34m(self, b)\u001b[0m\n\u001b[1;32m    494\u001b[0m         b \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mmemoryview\u001b[39m(b)[\u001b[38;5;241m0\u001b[39m:\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlength]\n\u001b[1;32m    496\u001b[0m \u001b[38;5;66;03m# we do not use _safe_read() here because this may be a .will_close\u001b[39;00m\n\u001b[1;32m    497\u001b[0m \u001b[38;5;66;03m# connection, and the user is reading more bytes than will be provided\u001b[39;00m\n\u001b[1;32m    498\u001b[0m \u001b[38;5;66;03m# (for example, reading in 1k chunks)\u001b[39;00m\n\u001b[0;32m--> 499\u001b[0m n \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mreadinto\u001b[49m\u001b[43m(\u001b[49m\u001b[43mb\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    500\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m n \u001b[38;5;129;01mand\u001b[39;00m b:\n\u001b[1;32m    501\u001b[0m     \u001b[38;5;66;03m# Ideally, we would raise IncompleteRead if the content-length\u001b[39;00m\n\u001b[1;32m    502\u001b[0m     \u001b[38;5;66;03m# wasn't satisfied, but it might break compatibility.\u001b[39;00m\n\u001b[1;32m    503\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_close_conn()\n",
            "File \u001b[0;32m~/miniconda3/lib/python3.8/socket.py:669\u001b[0m, in \u001b[0;36mSocketIO.readinto\u001b[0;34m(self, b)\u001b[0m\n\u001b[1;32m    667\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[1;32m    668\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 669\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_sock\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrecv_into\u001b[49m\u001b[43m(\u001b[49m\u001b[43mb\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    670\u001b[0m     \u001b[38;5;28;01mexcept\u001b[39;00m timeout:\n\u001b[1;32m    671\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_timeout_occurred \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n",
            "File \u001b[0;32m~/miniconda3/lib/python3.8/ssl.py:1241\u001b[0m, in \u001b[0;36mSSLSocket.recv_into\u001b[0;34m(self, buffer, nbytes, flags)\u001b[0m\n\u001b[1;32m   1237\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m flags \u001b[38;5;241m!=\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[1;32m   1238\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m   1239\u001b[0m           \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnon-zero flags not allowed in calls to recv_into() on \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m%\u001b[39m\n\u001b[1;32m   1240\u001b[0m           \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__class__\u001b[39m)\n\u001b[0;32m-> 1241\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnbytes\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbuffer\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1242\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m   1243\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39mrecv_into(buffer, nbytes, flags)\n",
            "File \u001b[0;32m~/miniconda3/lib/python3.8/ssl.py:1099\u001b[0m, in \u001b[0;36mSSLSocket.read\u001b[0;34m(self, len, buffer)\u001b[0m\n\u001b[1;32m   1097\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1098\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m buffer \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m-> 1099\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_sslobj\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mlen\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbuffer\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1100\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m   1101\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_sslobj\u001b[38;5;241m.\u001b[39mread(\u001b[38;5;28mlen\u001b[39m)\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ],
      "source": [
        "print(\"load meteor!\")\n",
        "meteor = load_metric(\"meteor\")\n",
        "\n",
        "gen_sums = np.load('gen_sums_finetuned_bertsum.npy')\n",
        "ref_sums = test_summaries\n",
        "\n",
        "print(\"Compute METEOR scores\")\n",
        "# Compute METEOR scores\n",
        "meteor_scores = meteor.compute(predictions=gen_sums, references=ref_sums, alpha=0.001, beta=30, gamma=0.001)\n",
        "\n",
        "# Access the METEOR score\n",
        "meteor_score = meteor_scores[\"meteor\"]\n",
        "\n",
        "# Print the METEOR score\n",
        "print(\"METEOR of finetuned Pegasus-x-large on andy data 2k: \", meteor_score)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "dnwUBGdGgPK8"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "gpuType": "T4",
      "machine_shape": "hm",
      "provenance": []
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3 (ipykernel)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.10"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}