Test_Alpaca_Lora_with_Colab.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "machine_shape": "hm",
      "gpuClass": "premium",
      "gpuType": "A100"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "gpuClass": "premium",
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "Mount Google Drive"
      ],
      "metadata": {
        "id": "ozmGRpVglvQv"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "62dxWS420AoV",
        "outputId": "a7ca9b8c-8408-486b-9114-cc0bfcf85550"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/gdrive\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Check the GPU device you are using"
      ],
      "metadata": {
        "id": "kQR57sBKl2Q-"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!nvidia-smi"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gxy0cfEioRdg",
        "outputId": "dfc2842c-4da2-460d-d399-e09aaaac4a42"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Sat Jun 24 05:42:49 2023       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 525.85.12    Driver Version: 525.85.12    CUDA Version: 12.0     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  NVIDIA A100-SXM...  Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   31C    P0    42W / 400W |      0MiB / 40960MiB |      0%      Default |\n",
            "|                               |                      |             Disabled |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%cd /content/gdrive/MyDrive"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2Xc-OQk50h9g",
        "outputId": "b6a2fec0-af99-43e7-afa6-92362dddbc79"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/gdrive/MyDrive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Git Clone the Alpaca-Lora Project"
      ],
      "metadata": {
        "id": "BiFAsLrama15"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!git clone https://github.com/xuwenhao/alpaca-lora.git"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YVR7flzH0nT3",
        "outputId": "909320c5-c480-433f-a478-90d6bc022ed6"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "fatal: destination path 'alpaca-lora' already exists and is not an empty directory.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%cd alpaca-lora"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "f0kYjrcs1CoL",
        "outputId": "748e8194-c70d-47ff-a539-74ff8c28681c"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/gdrive/MyDrive/alpaca-lora\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!git pull"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EIg83p7sKFXL",
        "outputId": "282ecc16-e775-4ac1-c3bb-1eaf88dd852a"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Already up to date.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Install the package"
      ],
      "metadata": {
        "id": "H1vhW0nWmf68"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -r requirements.txt"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "v-zUvUth1FsN",
        "outputId": "518a948b-47a9-4b6a-bb4d-06299760cb26"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting accelerate (from -r requirements.txt (line 1))\n",
            "  Downloading accelerate-0.20.3-py3-none-any.whl (227 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m227.6/227.6 kB\u001b[0m \u001b[31m16.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: appdirs in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 2)) (1.4.4)\n",
            "Collecting loralib (from -r requirements.txt (line 3))\n",
            "  Downloading loralib-0.1.1-py3-none-any.whl (8.8 kB)\n",
            "Collecting bitsandbytes==0.37.2 (from -r requirements.txt (line 4))\n",
            "  Downloading bitsandbytes-0.37.2-py3-none-any.whl (84.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m84.2/84.2 MB\u001b[0m \u001b[31m19.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting black (from -r requirements.txt (line 5))\n",
            "  Downloading black-23.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.7 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.7/1.7 MB\u001b[0m \u001b[31m99.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting datasets (from -r requirements.txt (line 7))\n",
            "  Downloading datasets-2.13.1-py3-none-any.whl (486 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m486.2/486.2 kB\u001b[0m \u001b[31m58.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting fire (from -r requirements.txt (line 8))\n",
            "  Downloading fire-0.5.0.tar.gz (88 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m88.3/88.3 kB\u001b[0m \u001b[31m16.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting peft==0.2.0 (from -r requirements.txt (line 9))\n",
            "  Downloading peft-0.2.0-py3-none-any.whl (40 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m40.3/40.3 kB\u001b[0m \u001b[31m6.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting transformers==4.29.2 (from -r requirements.txt (line 10))\n",
            "  Downloading transformers-4.29.2-py3-none-any.whl (7.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.1/7.1 MB\u001b[0m \u001b[31m92.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting sentencepiece (from -r requirements.txt (line 11))\n",
            "  Downloading sentencepiece-0.1.99-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m92.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting gradio (from -r requirements.txt (line 12))\n",
            "  Downloading gradio-3.35.2-py3-none-any.whl (19.7 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m19.7/19.7 MB\u001b[0m \u001b[31m61.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from peft==0.2.0->-r requirements.txt (line 9)) (1.22.4)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from peft==0.2.0->-r requirements.txt (line 9)) (23.1)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from peft==0.2.0->-r requirements.txt (line 9)) (5.9.5)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.10/dist-packages (from peft==0.2.0->-r requirements.txt (line 9)) (6.0)\n",
            "Requirement already satisfied: torch>=1.13.0 in /usr/local/lib/python3.10/dist-packages (from peft==0.2.0->-r requirements.txt (line 9)) (2.0.1+cu118)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers==4.29.2->-r requirements.txt (line 10)) (3.12.2)\n",
            "Collecting huggingface-hub<1.0,>=0.14.1 (from transformers==4.29.2->-r requirements.txt (line 10))\n",
            "  Downloading huggingface_hub-0.15.1-py3-none-any.whl (236 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m236.8/236.8 kB\u001b[0m \u001b[31m35.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers==4.29.2->-r requirements.txt (line 10)) (2022.10.31)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers==4.29.2->-r requirements.txt (line 10)) (2.27.1)\n",
            "Collecting tokenizers!=0.11.3,<0.14,>=0.11.1 (from transformers==4.29.2->-r requirements.txt (line 10))\n",
            "  Downloading tokenizers-0.13.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.8/7.8 MB\u001b[0m \u001b[31m126.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers==4.29.2->-r requirements.txt (line 10)) (4.65.0)\n",
            "Requirement already satisfied: click>=8.0.0 in /usr/local/lib/python3.10/dist-packages (from black->-r requirements.txt (line 5)) (8.1.3)\n",
            "Collecting mypy-extensions>=0.4.3 (from black->-r requirements.txt (line 5))\n",
            "  Downloading mypy_extensions-1.0.0-py3-none-any.whl (4.7 kB)\n",
            "Collecting pathspec>=0.9.0 (from black->-r requirements.txt (line 5))\n",
            "  Downloading pathspec-0.11.1-py3-none-any.whl (29 kB)\n",
            "Requirement already satisfied: platformdirs>=2 in /usr/local/lib/python3.10/dist-packages (from black->-r requirements.txt (line 5)) (3.7.0)\n",
            "Requirement already satisfied: tomli>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from black->-r requirements.txt (line 5)) (2.0.1)\n",
            "Requirement already satisfied: ipython>=7.8.0 in /usr/local/lib/python3.10/dist-packages (from black->-r requirements.txt (line 5)) (7.34.0)\n",
            "Collecting tokenize-rt>=3.2.0 (from black->-r requirements.txt (line 5))\n",
            "  Downloading tokenize_rt-5.1.0-py2.py3-none-any.whl (5.8 kB)\n",
            "Requirement already satisfied: pyarrow>=8.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets->-r requirements.txt (line 7)) (9.0.0)\n",
            "Collecting dill<0.3.7,>=0.3.0 (from datasets->-r requirements.txt (line 7))\n",
            "  Downloading dill-0.3.6-py3-none-any.whl (110 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m110.5/110.5 kB\u001b[0m \u001b[31m20.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets->-r requirements.txt (line 7)) (1.5.3)\n",
            "Collecting xxhash (from datasets->-r requirements.txt (line 7))\n",
            "  Downloading xxhash-3.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (212 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m212.5/212.5 kB\u001b[0m \u001b[31m33.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting multiprocess (from datasets->-r requirements.txt (line 7))\n",
            "  Downloading multiprocess-0.70.14-py310-none-any.whl (134 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.3/134.3 kB\u001b[0m \u001b[31m24.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: fsspec[http]>=2021.11.1 in /usr/local/lib/python3.10/dist-packages (from datasets->-r requirements.txt (line 7)) (2023.6.0)\n",
            "Collecting aiohttp (from datasets->-r requirements.txt (line 7))\n",
            "  Downloading aiohttp-3.8.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.0/1.0 MB\u001b[0m \u001b[31m83.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: six in /usr/local/lib/python3.10/dist-packages (from fire->-r requirements.txt (line 8)) (1.16.0)\n",
            "Requirement already satisfied: termcolor in /usr/local/lib/python3.10/dist-packages (from fire->-r requirements.txt (line 8)) (2.3.0)\n",
            "Collecting aiofiles (from gradio->-r requirements.txt (line 12))\n",
            "  Downloading aiofiles-23.1.0-py3-none-any.whl (14 kB)\n",
            "Requirement already satisfied: altair>=4.2.0 in /usr/local/lib/python3.10/dist-packages (from gradio->-r requirements.txt (line 12)) (4.2.2)\n",
            "Collecting fastapi (from gradio->-r requirements.txt (line 12))\n",
            "  Downloading fastapi-0.98.0-py3-none-any.whl (56 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m57.0/57.0 kB\u001b[0m \u001b[31m10.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting ffmpy (from gradio->-r requirements.txt (line 12))\n",
            "  Downloading ffmpy-0.3.0.tar.gz (4.8 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting gradio-client>=0.2.7 (from gradio->-r requirements.txt (line 12))\n",
            "  Downloading gradio_client-0.2.7-py3-none-any.whl (288 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m288.4/288.4 kB\u001b[0m \u001b[31m43.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting httpx (from gradio->-r requirements.txt (line 12))\n",
            "  Downloading httpx-0.24.1-py3-none-any.whl (75 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.4/75.4 kB\u001b[0m \u001b[31m13.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from gradio->-r requirements.txt (line 12)) (3.1.2)\n",
            "Requirement already satisfied: markdown-it-py[linkify]>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from gradio->-r requirements.txt (line 12)) (3.0.0)\n",
            "Requirement already satisfied: markupsafe in /usr/local/lib/python3.10/dist-packages (from gradio->-r requirements.txt (line 12)) (2.1.3)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.10/dist-packages (from gradio->-r requirements.txt (line 12)) (3.7.1)\n",
            "Collecting mdit-py-plugins<=0.3.3 (from gradio->-r requirements.txt (line 12))\n",
            "  Downloading mdit_py_plugins-0.3.3-py3-none-any.whl (50 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m50.5/50.5 kB\u001b[0m \u001b[31m9.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting orjson (from gradio->-r requirements.txt (line 12))\n",
            "  Downloading orjson-3.9.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (136 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m137.0/137.0 kB\u001b[0m \u001b[31m24.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pillow in /usr/local/lib/python3.10/dist-packages (from gradio->-r requirements.txt (line 12)) (8.4.0)\n",
            "Requirement already satisfied: pydantic in /usr/local/lib/python3.10/dist-packages (from gradio->-r requirements.txt (line 12)) (1.10.9)\n",
            "Collecting pydub (from gradio->-r requirements.txt (line 12))\n",
            "  Downloading pydub-0.25.1-py2.py3-none-any.whl (32 kB)\n",
            "Requirement already satisfied: pygments>=2.12.0 in /usr/local/lib/python3.10/dist-packages (from gradio->-r requirements.txt (line 12)) (2.14.0)\n",
            "Collecting python-multipart (from gradio->-r requirements.txt (line 12))\n",
            "  Downloading python_multipart-0.0.6-py3-none-any.whl (45 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m45.7/45.7 kB\u001b[0m \u001b[31m7.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting semantic-version (from gradio->-r requirements.txt (line 12))\n",
            "  Downloading semantic_version-2.10.0-py2.py3-none-any.whl (15 kB)\n",
            "Collecting uvicorn>=0.14.0 (from gradio->-r requirements.txt (line 12))\n",
            "  Downloading uvicorn-0.22.0-py3-none-any.whl (58 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m11.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting websockets>=10.0 (from gradio->-r requirements.txt (line 12))\n",
            "  Downloading websockets-11.0.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (129 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m129.9/129.9 kB\u001b[0m \u001b[31m23.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: entrypoints in /usr/local/lib/python3.10/dist-packages (from altair>=4.2.0->gradio->-r requirements.txt (line 12)) (0.4)\n",
            "Requirement already satisfied: jsonschema>=3.0 in /usr/local/lib/python3.10/dist-packages (from altair>=4.2.0->gradio->-r requirements.txt (line 12)) (4.3.3)\n",
            "Requirement already satisfied: toolz in /usr/local/lib/python3.10/dist-packages (from altair>=4.2.0->gradio->-r requirements.txt (line 12)) (0.12.0)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->-r requirements.txt (line 7)) (23.1.0)\n",
            "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->-r requirements.txt (line 7)) (2.0.12)\n",
            "Collecting multidict<7.0,>=4.5 (from aiohttp->datasets->-r requirements.txt (line 7))\n",
            "  Downloading multidict-6.0.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (114 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m114.5/114.5 kB\u001b[0m \u001b[31m21.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting async-timeout<5.0,>=4.0.0a3 (from aiohttp->datasets->-r requirements.txt (line 7))\n",
            "  Downloading async_timeout-4.0.2-py3-none-any.whl (5.8 kB)\n",
            "Collecting yarl<2.0,>=1.0 (from aiohttp->datasets->-r requirements.txt (line 7))\n",
            "  Downloading yarl-1.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (268 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m268.8/268.8 kB\u001b[0m \u001b[31m39.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting frozenlist>=1.1.1 (from aiohttp->datasets->-r requirements.txt (line 7))\n",
            "  Downloading frozenlist-1.3.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (149 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m149.6/149.6 kB\u001b[0m \u001b[31m27.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting aiosignal>=1.1.2 (from aiohttp->datasets->-r requirements.txt (line 7))\n",
            "  Downloading aiosignal-1.3.1-py3-none-any.whl (7.6 kB)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from gradio-client>=0.2.7->gradio->-r requirements.txt (line 12)) (4.6.3)\n",
            "Requirement already satisfied: setuptools>=18.5 in /usr/local/lib/python3.10/dist-packages (from ipython>=7.8.0->black->-r requirements.txt (line 5)) (67.7.2)\n",
            "Collecting jedi>=0.16 (from ipython>=7.8.0->black->-r requirements.txt (line 5))\n",
            "  Downloading jedi-0.18.2-py2.py3-none-any.whl (1.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m93.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: decorator in /usr/local/lib/python3.10/dist-packages (from ipython>=7.8.0->black->-r requirements.txt (line 5)) (4.4.2)\n",
            "Requirement already satisfied: pickleshare in /usr/local/lib/python3.10/dist-packages (from ipython>=7.8.0->black->-r requirements.txt (line 5)) (0.7.5)\n",
            "Requirement already satisfied: traitlets>=4.2 in /usr/local/lib/python3.10/dist-packages (from ipython>=7.8.0->black->-r requirements.txt (line 5)) (5.7.1)\n",
            "Requirement already satisfied: prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from ipython>=7.8.0->black->-r requirements.txt (line 5)) (3.0.38)\n",
            "Requirement already satisfied: backcall in /usr/local/lib/python3.10/dist-packages (from ipython>=7.8.0->black->-r requirements.txt (line 5)) (0.2.0)\n",
            "Requirement already satisfied: matplotlib-inline in /usr/local/lib/python3.10/dist-packages (from ipython>=7.8.0->black->-r requirements.txt (line 5)) (0.1.6)\n",
            "Requirement already satisfied: pexpect>4.3 in /usr/local/lib/python3.10/dist-packages (from ipython>=7.8.0->black->-r requirements.txt (line 5)) (4.8.0)\n",
            "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.10/dist-packages (from markdown-it-py[linkify]>=2.0.0->gradio->-r requirements.txt (line 12)) (0.1.2)\n",
            "Collecting linkify-it-py<3,>=1 (from markdown-it-py[linkify]>=2.0.0->gradio->-r requirements.txt (line 12))\n",
            "  Downloading linkify_it_py-2.0.2-py3-none-any.whl (19 kB)\n",
            "INFO: pip is looking at multiple versions of mdit-py-plugins to determine which version is compatible with other requirements. This could take a while.\n",
            "Collecting mdit-py-plugins<=0.3.3 (from gradio->-r requirements.txt (line 12))\n",
            "  Downloading mdit_py_plugins-0.3.2-py3-none-any.whl (50 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m50.4/50.4 kB\u001b[0m \u001b[31m9.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Downloading mdit_py_plugins-0.3.1-py3-none-any.whl (46 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m46.5/46.5 kB\u001b[0m \u001b[31m8.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Downloading mdit_py_plugins-0.3.0-py3-none-any.whl (43 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m43.7/43.7 kB\u001b[0m \u001b[31m7.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Downloading mdit_py_plugins-0.2.8-py3-none-any.whl (41 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m41.0/41.0 kB\u001b[0m \u001b[31m7.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Downloading mdit_py_plugins-0.2.7-py3-none-any.whl (41 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m41.0/41.0 kB\u001b[0m \u001b[31m7.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Downloading mdit_py_plugins-0.2.6-py3-none-any.whl (39 kB)\n",
            "  Downloading mdit_py_plugins-0.2.5-py3-none-any.whl (39 kB)\n",
            "INFO: pip is looking at multiple versions of mdit-py-plugins to determine which version is compatible with other requirements. This could take a while.\n",
            "  Downloading mdit_py_plugins-0.2.4-py3-none-any.whl (39 kB)\n",
            "  Downloading mdit_py_plugins-0.2.3-py3-none-any.whl (39 kB)\n",
            "  Downloading mdit_py_plugins-0.2.2-py3-none-any.whl (39 kB)\n",
            "  Downloading mdit_py_plugins-0.2.1-py3-none-any.whl (38 kB)\n",
            "  Downloading mdit_py_plugins-0.2.0-py3-none-any.whl (38 kB)\n",
            "INFO: This is taking longer than usual. You might need to provide the dependency resolver with stricter constraints to reduce runtime. See https://pip.pypa.io/warnings/backtracking for guidance. If you want to abort this run, press Ctrl + C.\n",
            "  Downloading mdit_py_plugins-0.1.0-py3-none-any.whl (37 kB)\n",
            "Collecting markdown-it-py[linkify]>=2.0.0 (from gradio->-r requirements.txt (line 12))\n",
            "  Downloading markdown_it_py-3.0.0-py3-none-any.whl (87 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m87.5/87.5 kB\u001b[0m \u001b[31m16.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Downloading markdown_it_py-2.2.0-py3-none-any.whl (84 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m84.5/84.5 kB\u001b[0m \u001b[31m15.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets->-r requirements.txt (line 7)) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets->-r requirements.txt (line 7)) (2022.7.1)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers==4.29.2->-r requirements.txt (line 10)) (1.26.16)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers==4.29.2->-r requirements.txt (line 10)) (2023.5.7)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers==4.29.2->-r requirements.txt (line 10)) (3.4)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.0->peft==0.2.0->-r requirements.txt (line 9)) (1.11.1)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.0->peft==0.2.0->-r requirements.txt (line 9)) (3.1)\n",
            "Requirement already satisfied: triton==2.0.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.0->peft==0.2.0->-r requirements.txt (line 9)) (2.0.0)\n",
            "Requirement already satisfied: cmake in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.13.0->peft==0.2.0->-r requirements.txt (line 9)) (3.25.2)\n",
            "Requirement already satisfied: lit in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.13.0->peft==0.2.0->-r requirements.txt (line 9)) (16.0.6)\n",
            "Collecting h11>=0.8 (from uvicorn>=0.14.0->gradio->-r requirements.txt (line 12))\n",
            "  Downloading h11-0.14.0-py3-none-any.whl (58 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m10.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting starlette<0.28.0,>=0.27.0 (from fastapi->gradio->-r requirements.txt (line 12))\n",
            "  Downloading starlette-0.27.0-py3-none-any.whl (66 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m67.0/67.0 kB\u001b[0m \u001b[31m12.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting httpcore<0.18.0,>=0.15.0 (from httpx->gradio->-r requirements.txt (line 12))\n",
            "  Downloading httpcore-0.17.2-py3-none-any.whl (72 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m72.5/72.5 kB\u001b[0m \u001b[31m1.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from httpx->gradio->-r requirements.txt (line 12)) (1.3.0)\n",
            "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->gradio->-r requirements.txt (line 12)) (1.1.0)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib->gradio->-r requirements.txt (line 12)) (0.11.0)\n",
            "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib->gradio->-r requirements.txt (line 12)) (4.40.0)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->gradio->-r requirements.txt (line 12)) (1.4.4)\n",
            "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->gradio->-r requirements.txt (line 12)) (3.1.0)\n",
            "Requirement already satisfied: anyio<5.0,>=3.0 in /usr/local/lib/python3.10/dist-packages (from httpcore<0.18.0,>=0.15.0->httpx->gradio->-r requirements.txt (line 12)) (3.7.0)\n",
            "Requirement already satisfied: parso<0.9.0,>=0.8.0 in /usr/local/lib/python3.10/dist-packages (from jedi>=0.16->ipython>=7.8.0->black->-r requirements.txt (line 5)) (0.8.3)\n",
            "Requirement already satisfied: pyrsistent!=0.17.0,!=0.17.1,!=0.17.2,>=0.14.0 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=3.0->altair>=4.2.0->gradio->-r requirements.txt (line 12)) (0.19.3)\n",
            "Collecting uc-micro-py (from linkify-it-py<3,>=1->markdown-it-py[linkify]>=2.0.0->gradio->-r requirements.txt (line 12))\n",
            "  Downloading uc_micro_py-1.0.2-py3-none-any.whl (6.2 kB)\n",
            "Requirement already satisfied: ptyprocess>=0.5 in /usr/local/lib/python3.10/dist-packages (from pexpect>4.3->ipython>=7.8.0->black->-r requirements.txt (line 5)) (0.7.0)\n",
            "Requirement already satisfied: wcwidth in /usr/local/lib/python3.10/dist-packages (from prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0->ipython>=7.8.0->black->-r requirements.txt (line 5)) (0.2.6)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.13.0->peft==0.2.0->-r requirements.txt (line 9)) (1.3.0)\n",
            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<5.0,>=3.0->httpcore<0.18.0,>=0.15.0->httpx->gradio->-r requirements.txt (line 12)) (1.1.1)\n",
            "Building wheels for collected packages: fire, ffmpy\n",
            "  Building wheel for fire (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for fire: filename=fire-0.5.0-py2.py3-none-any.whl size=116932 sha256=cc1830713e07c2f3790b69c4b819768fa013b617fce4b53fe91568422accda22\n",
            "  Stored in directory: /root/.cache/pip/wheels/90/d4/f7/9404e5db0116bd4d43e5666eaa3e70ab53723e1e3ea40c9a95\n",
            "  Building wheel for ffmpy (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for ffmpy: filename=ffmpy-0.3.0-py3-none-any.whl size=4694 sha256=c3d7e4a351a4e053d8bd6358a47d82186bc916cc048a5e22c62b4b6d8a8825f1\n",
            "  Stored in directory: /root/.cache/pip/wheels/0c/c2/0e/3b9c6845c6a4e35beb90910cc70d9ac9ab5d47402bd62af0df\n",
            "Successfully built fire ffmpy\n",
            "Installing collected packages: tokenizers, sentencepiece, pydub, ffmpy, bitsandbytes, xxhash, websockets, uc-micro-py, tokenize-rt, semantic-version, python-multipart, pathspec, orjson, mypy-extensions, multidict, markdown-it-py, loralib, jedi, h11, frozenlist, fire, dill, async-timeout, aiofiles, yarl, uvicorn, starlette, multiprocess, mdit-py-plugins, linkify-it-py, huggingface-hub, httpcore, black, aiosignal, transformers, httpx, fastapi, aiohttp, gradio-client, gradio, datasets, accelerate, peft\n",
            "  Attempting uninstall: markdown-it-py\n",
            "    Found existing installation: markdown-it-py 3.0.0\n",
            "    Uninstalling markdown-it-py-3.0.0:\n",
            "      Successfully uninstalled markdown-it-py-3.0.0\n",
            "Successfully installed accelerate-0.20.3 aiofiles-23.1.0 aiohttp-3.8.4 aiosignal-1.3.1 async-timeout-4.0.2 bitsandbytes-0.37.2 black-23.3.0 datasets-2.13.1 dill-0.3.6 fastapi-0.98.0 ffmpy-0.3.0 fire-0.5.0 frozenlist-1.3.3 gradio-3.35.2 gradio-client-0.2.7 h11-0.14.0 httpcore-0.17.2 httpx-0.24.1 huggingface-hub-0.15.1 jedi-0.18.2 linkify-it-py-2.0.2 loralib-0.1.1 markdown-it-py-2.2.0 mdit-py-plugins-0.3.3 multidict-6.0.4 multiprocess-0.70.14 mypy-extensions-1.0.0 orjson-3.9.1 pathspec-0.11.1 peft-0.2.0 pydub-0.25.1 python-multipart-0.0.6 semantic-version-2.10.0 sentencepiece-0.1.99 starlette-0.27.0 tokenize-rt-5.1.0 tokenizers-0.13.3 transformers-4.29.2 uc-micro-py-1.0.2 uvicorn-0.22.0 websockets-11.0.3 xxhash-3.2.0 yarl-1.9.2\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install wandb\n"
      ],
      "metadata": {
        "id": "WCBOnCmGJYPP",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "650e72b0-ee57-4eee-993e-4fa0e3df31a8"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting wandb\n",
            "  Downloading wandb-0.15.4-py3-none-any.whl (2.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.1/2.1 MB\u001b[0m \u001b[31m63.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: Click!=8.0.0,>=7.0 in /usr/local/lib/python3.10/dist-packages (from wandb) (8.1.3)\n",
            "Collecting GitPython!=3.1.29,>=1.0.0 (from wandb)\n",
            "  Downloading GitPython-3.1.31-py3-none-any.whl (184 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m184.3/184.3 kB\u001b[0m \u001b[31m29.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: requests<3,>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from wandb) (2.27.1)\n",
            "Requirement already satisfied: psutil>=5.0.0 in /usr/local/lib/python3.10/dist-packages (from wandb) (5.9.5)\n",
            "Collecting sentry-sdk>=1.0.0 (from wandb)\n",
            "  Downloading sentry_sdk-1.26.0-py2.py3-none-any.whl (209 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m209.4/209.4 kB\u001b[0m \u001b[31m29.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting docker-pycreds>=0.4.0 (from wandb)\n",
            "  Downloading docker_pycreds-0.4.0-py2.py3-none-any.whl (9.0 kB)\n",
            "Requirement already satisfied: PyYAML in /usr/local/lib/python3.10/dist-packages (from wandb) (6.0)\n",
            "Collecting pathtools (from wandb)\n",
            "  Downloading pathtools-0.1.2.tar.gz (11 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting setproctitle (from wandb)\n",
            "  Downloading setproctitle-1.3.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (30 kB)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from wandb) (67.7.2)\n",
            "Requirement already satisfied: appdirs>=1.4.3 in /usr/local/lib/python3.10/dist-packages (from wandb) (1.4.4)\n",
            "Requirement already satisfied: protobuf!=4.21.0,<5,>=3.19.0 in /usr/local/lib/python3.10/dist-packages (from wandb) (3.20.3)\n",
            "Requirement already satisfied: six>=1.4.0 in /usr/local/lib/python3.10/dist-packages (from docker-pycreds>=0.4.0->wandb) (1.16.0)\n",
            "Collecting gitdb<5,>=4.0.1 (from GitPython!=3.1.29,>=1.0.0->wandb)\n",
            "  Downloading gitdb-4.0.10-py3-none-any.whl (62 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.7/62.7 kB\u001b[0m \u001b[31m10.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.0.0->wandb) (1.26.16)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.0.0->wandb) (2023.5.7)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.0.0->wandb) (2.0.12)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.0.0->wandb) (3.4)\n",
            "Collecting smmap<6,>=3.0.1 (from gitdb<5,>=4.0.1->GitPython!=3.1.29,>=1.0.0->wandb)\n",
            "  Downloading smmap-5.0.0-py3-none-any.whl (24 kB)\n",
            "Building wheels for collected packages: pathtools\n",
            "  Building wheel for pathtools (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pathtools: filename=pathtools-0.1.2-py3-none-any.whl size=8791 sha256=2fa902831893e247ae941e26a4874921ff64115c4686f9db36bc01ef2e8cf5f7\n",
            "  Stored in directory: /root/.cache/pip/wheels/e7/f3/22/152153d6eb222ee7a56ff8617d80ee5207207a8c00a7aab794\n",
            "Successfully built pathtools\n",
            "Installing collected packages: pathtools, smmap, setproctitle, sentry-sdk, docker-pycreds, gitdb, GitPython, wandb\n",
            "Successfully installed GitPython-3.1.31 docker-pycreds-0.4.0 gitdb-4.0.10 pathtools-0.1.2 sentry-sdk-1.26.0 setproctitle-1.3.2 smmap-5.0.0 wandb-0.15.4\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import wandb\n",
        "wandb.login()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 158
        },
        "id": "eqd4WbH0JmTI",
        "outputId": "a74a3bbe-1027-4518-b579-8a6510ad815a"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "\n",
              "        window._wandbApiKey = new Promise((resolve, reject) => {\n",
              "            function loadScript(url) {\n",
              "            return new Promise(function(resolve, reject) {\n",
              "                let newScript = document.createElement(\"script\");\n",
              "                newScript.onerror = reject;\n",
              "                newScript.onload = resolve;\n",
              "                document.body.appendChild(newScript);\n",
              "                newScript.src = url;\n",
              "            });\n",
              "            }\n",
              "            loadScript(\"https://cdn.jsdelivr.net/npm/postmate/build/postmate.min.js\").then(() => {\n",
              "            const iframe = document.createElement('iframe')\n",
              "            iframe.style.cssText = \"width:0;height:0;border:none\"\n",
              "            document.body.appendChild(iframe)\n",
              "            const handshake = new Postmate({\n",
              "                container: iframe,\n",
              "                url: 'https://wandb.ai/authorize'\n",
              "            });\n",
              "            const timeout = setTimeout(() => reject(\"Couldn't auto authenticate\"), 5000)\n",
              "            handshake.then(function(child) {\n",
              "                child.on('authorize', data => {\n",
              "                    clearTimeout(timeout)\n",
              "                    resolve(data)\n",
              "                });\n",
              "            });\n",
              "            })\n",
              "        });\n",
              "    "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Logging into wandb.ai. (Learn how to deploy a W&B server locally: https://wandb.me/wandb-server)\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: You can find your API key in your browser here: https://wandb.ai/authorize\n",
            "wandb: Paste an API key from your profile and hit enter, or press ctrl+c to quit:"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " ··········\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: /root/.netrc\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Fine-tunning on the Llama-7b.\n",
        "One tricky part is to tune the micro_batch_size such that your memory can be fully utilized. Here I choose it to be 50 for 40GB-A100. You may want to optimize it by yourself."
      ],
      "metadata": {
        "id": "HqC2UgQCmry1"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!python finetune.py \\\n",
        "    --base_model 'huggyllama/llama-7b' \\\n",
        "    --data_path './alpaca_data.json' \\\n",
        "    --output_dir './lora-alpaca-20230624-01' \\\n",
        "    --batch_size 128 \\\n",
        "    --micro_batch_size 50 \\\n",
        "    --num_epochs 3 \\\n",
        "    --learning_rate 1e-4 \\\n",
        "    --cutoff_len 512 \\\n",
        "    --val_set_size 2000 \\\n",
        "    --lora_r 8 \\\n",
        "    --lora_alpha 16 \\\n",
        "    --lora_dropout 0.05 \\\n",
        "    --lora_target_modules '[q_proj,v_proj]' \\\n",
        "    --train_on_inputs \\\n",
        "    --group_by_length \\\n",
        "    --wandb_project 'mactalk-demo' \\\n",
        "    --wandb_run_name 'alpaca-20230624' \\\n",
        "    --wandb_log_model 'true' \\\n",
        "    --wandb_watch 'false'"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "J8bDF1kkFOfF",
        "outputId": "d84e853d-296d-4383-f15f-0650575b2b47"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2023-06-24 05:46:42.049610: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
            "\n",
            "===================================BUG REPORT===================================\n",
            "Welcome to bitsandbytes. For bug reports, please submit your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\n",
            "================================================================================\n",
            "/usr/local/lib/python3.10/dist-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: /usr/lib64-nvidia did not contain libcudart.so as expected! Searching further paths...\n",
            "  warn(msg)\n",
            "/usr/local/lib/python3.10/dist-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/sys/fs/cgroup/memory.events /var/colab/cgroup/jupyter-children/memory.events')}\n",
            "  warn(msg)\n",
            "/usr/local/lib/python3.10/dist-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('http'), PosixPath('//172.28.0.1'), PosixPath('8013')}\n",
            "  warn(msg)\n",
            "/usr/local/lib/python3.10/dist-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('--logtostderr --listen_host=172.28.0.12 --target_host=172.28.0.12 --tunnel_background_save_url=https'), PosixPath('//colab.research.google.com/tun/m/cc48301118ce562b961b3c22d803539adc1e0c19/gpu-a100-s-3qgjg4qb81vsg --tunnel_background_save_delay=10s --tunnel_periodic_background_save_frequency=30m0s --enable_output_coalescing=true --output_coalescing_required=true')}\n",
            "  warn(msg)\n",
            "/usr/local/lib/python3.10/dist-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/env/python')}\n",
            "  warn(msg)\n",
            "/usr/local/lib/python3.10/dist-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('//ipykernel.pylab.backend_inline'), PosixPath('module')}\n",
            "  warn(msg)\n",
            "CUDA_SETUP: WARNING! libcudart.so not found in any environmental path. Searching /usr/local/cuda/lib64...\n",
            "CUDA SETUP: CUDA runtime path found: /usr/local/cuda/lib64/libcudart.so\n",
            "CUDA SETUP: Highest compute capability among GPUs detected: 8.0\n",
            "CUDA SETUP: Detected CUDA version 118\n",
            "CUDA SETUP: Loading binary /usr/local/lib/python3.10/dist-packages/bitsandbytes/libbitsandbytes_cuda118.so...\n",
            "Training Alpaca-LoRA model with params:\n",
            "base_model: huggyllama/llama-7b\n",
            "data_path: ./alpaca_data.json\n",
            "output_dir: ./lora-alpaca-20230624-01\n",
            "batch_size: 128\n",
            "micro_batch_size: 50\n",
            "num_epochs: 3\n",
            "learning_rate: 0.0001\n",
            "cutoff_len: 512\n",
            "val_set_size: 2000\n",
            "lora_r: 8\n",
            "lora_alpha: 16\n",
            "lora_dropout: 0.05\n",
            "lora_target_modules: ['q_proj', 'v_proj']\n",
            "train_on_inputs: True\n",
            "add_eos_token: False\n",
            "group_by_length: True\n",
            "wandb_project: mactalk-demo\n",
            "wandb_run_name: alpaca-20230624\n",
            "wandb_watch: false\n",
            "wandb_log_model: true\n",
            "resume_from_checkpoint: False\n",
            "prompt template: alpaca\n",
            "\n",
            "Downloading (…)lve/main/config.json: 100% 594/594 [00:00<00:00, 3.97MB/s]\n",
            "Downloading (…)model.bin.index.json: 26.8kB [00:00, 70.6MB/s]\n",
            "Downloading shards:   0% 0/2 [00:00<?, ?it/s]\n",
            "Downloading (…)l-00001-of-00002.bin:   0% 0.00/9.98G [00:00<?, ?B/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:   1% 52.4M/9.98G [00:00<00:23, 429MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:   1% 105M/9.98G [00:00<00:21, 450MB/s] \u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:   2% 157M/9.98G [00:00<00:22, 445MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:   2% 210M/9.98G [00:00<00:22, 431MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:   3% 262M/9.98G [00:00<00:21, 449MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:   3% 315M/9.98G [00:00<00:21, 460MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:   4% 367M/9.98G [00:00<00:22, 437MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:   4% 419M/9.98G [00:00<00:21, 442MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:   5% 472M/9.98G [00:01<00:21, 443MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:   5% 524M/9.98G [00:01<00:21, 446MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:   6% 577M/9.98G [00:01<00:20, 462MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:   6% 629M/9.98G [00:01<00:20, 448MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:   7% 682M/9.98G [00:01<00:21, 441MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:   7% 734M/9.98G [00:01<00:22, 417MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:   8% 786M/9.98G [00:01<00:22, 407MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:   8% 839M/9.98G [00:01<00:22, 412MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:   9% 881M/9.98G [00:02<00:23, 389MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:   9% 923M/9.98G [00:02<00:24, 365MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  10% 975M/9.98G [00:02<00:22, 395MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  10% 1.02G/9.98G [00:02<00:22, 396MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  11% 1.08G/9.98G [00:02<00:20, 427MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  11% 1.13G/9.98G [00:02<00:20, 434MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  12% 1.18G/9.98G [00:02<00:20, 440MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  12% 1.24G/9.98G [00:02<00:21, 410MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  13% 1.28G/9.98G [00:03<00:21, 405MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  13% 1.33G/9.98G [00:03<00:20, 421MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  14% 1.38G/9.98G [00:03<00:19, 448MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  14% 1.44G/9.98G [00:03<00:19, 445MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  15% 1.49G/9.98G [00:03<00:19, 443MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  15% 1.54G/9.98G [00:03<00:18, 447MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  16% 1.59G/9.98G [00:03<00:18, 446MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  17% 1.65G/9.98G [00:03<00:17, 464MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  17% 1.71G/9.98G [00:03<00:16, 488MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  18% 1.76G/9.98G [00:04<00:16, 492MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  18% 1.81G/9.98G [00:04<00:16, 500MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  19% 1.88G/9.98G [00:04<00:15, 516MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  19% 1.94G/9.98G [00:04<00:15, 524MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  20% 1.99G/9.98G [00:04<00:16, 486MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  20% 2.04G/9.98G [00:04<00:16, 477MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  21% 2.10G/9.98G [00:04<00:16, 473MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  22% 2.15G/9.98G [00:04<00:16, 463MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  22% 2.20G/9.98G [00:04<00:18, 427MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  23% 2.25G/9.98G [00:05<00:17, 439MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  23% 2.31G/9.98G [00:05<00:18, 416MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  24% 2.36G/9.98G [00:05<00:22, 346MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  24% 2.40G/9.98G [00:05<00:25, 293MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  24% 2.44G/9.98G [00:05<00:28, 262MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  25% 2.47G/9.98G [00:06<00:30, 247MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  25% 2.51G/9.98G [00:06<00:32, 231MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  25% 2.54G/9.98G [00:06<00:33, 220MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  26% 2.57G/9.98G [00:06<00:34, 216MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  26% 2.60G/9.98G [00:06<00:34, 214MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  26% 2.63G/9.98G [00:06<00:34, 213MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  27% 2.66G/9.98G [00:06<00:33, 220MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  27% 2.69G/9.98G [00:07<00:32, 224MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  27% 2.73G/9.98G [00:07<00:31, 233MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  28% 2.76G/9.98G [00:07<00:30, 236MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  28% 2.80G/9.98G [00:07<00:27, 261MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  28% 2.84G/9.98G [00:07<00:25, 284MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  29% 2.88G/9.98G [00:07<00:24, 293MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  29% 2.92G/9.98G [00:07<00:23, 298MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  30% 2.96G/9.98G [00:07<00:22, 318MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  30% 3.00G/9.98G [00:08<00:21, 321MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  30% 3.04G/9.98G [00:08<00:20, 339MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  31% 3.08G/9.98G [00:08<00:22, 308MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  31% 3.12G/9.98G [00:08<00:21, 315MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  32% 3.17G/9.98G [00:08<00:20, 339MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  32% 3.22G/9.98G [00:08<00:18, 363MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  33% 3.27G/9.98G [00:08<00:17, 378MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  33% 3.32G/9.98G [00:08<00:16, 403MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  34% 3.38G/9.98G [00:09<00:15, 419MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  34% 3.43G/9.98G [00:09<00:15, 411MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  35% 3.48G/9.98G [00:09<00:15, 419MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  35% 3.53G/9.98G [00:09<00:16, 382MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  36% 3.60G/9.98G [00:09<00:15, 424MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  37% 3.65G/9.98G [00:09<00:14, 434MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  37% 3.70G/9.98G [00:09<00:14, 440MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  38% 3.75G/9.98G [00:09<00:13, 457MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  38% 3.81G/9.98G [00:10<00:13, 446MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  39% 3.86G/9.98G [00:10<00:13, 453MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  39% 3.91G/9.98G [00:10<00:14, 409MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  40% 3.96G/9.98G [00:10<00:18, 320MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  40% 4.01G/9.98G [00:10<00:20, 290MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  41% 4.05G/9.98G [00:10<00:21, 274MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  41% 4.08G/9.98G [00:11<00:22, 265MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  41% 4.11G/9.98G [00:11<00:22, 263MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  42% 4.14G/9.98G [00:11<00:22, 263MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  42% 4.17G/9.98G [00:11<00:22, 263MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  42% 4.20G/9.98G [00:11<00:22, 258MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  42% 4.24G/9.98G [00:11<00:22, 260MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  43% 4.27G/9.98G [00:11<00:21, 267MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  43% 4.30G/9.98G [00:11<00:21, 269MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  44% 4.34G/9.98G [00:11<00:18, 306MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  44% 4.38G/9.98G [00:12<00:18, 299MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  44% 4.42G/9.98G [00:12<00:17, 320MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  45% 4.47G/9.98G [00:12<00:16, 338MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  45% 4.51G/9.98G [00:12<00:16, 334MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  46% 4.55G/9.98G [00:12<00:16, 327MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  46% 4.59G/9.98G [00:12<00:16, 335MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  46% 4.63G/9.98G [00:12<00:16, 316MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  47% 4.68G/9.98G [00:12<00:16, 326MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  47% 4.72G/9.98G [00:13<00:15, 332MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  48% 4.76G/9.98G [00:13<00:15, 329MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  48% 4.80G/9.98G [00:13<00:15, 335MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  49% 4.84G/9.98G [00:13<00:15, 332MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  49% 4.89G/9.98G [00:13<00:16, 305MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  49% 4.93G/9.98G [00:13<00:15, 325MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  50% 4.97G/9.98G [00:13<00:19, 260MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  50% 5.01G/9.98G [00:14<00:17, 287MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  51% 5.05G/9.98G [00:14<00:15, 316MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  51% 5.11G/9.98G [00:14<00:13, 352MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  52% 5.16G/9.98G [00:14<00:12, 383MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  52% 5.21G/9.98G [00:14<00:11, 412MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  53% 5.26G/9.98G [00:14<00:10, 428MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  53% 5.33G/9.98G [00:14<00:09, 474MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  54% 5.38G/9.98G [00:14<00:10, 436MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  54% 5.43G/9.98G [00:15<00:11, 391MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  55% 5.47G/9.98G [00:15<00:12, 370MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  55% 5.52G/9.98G [00:15<00:12, 351MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  56% 5.56G/9.98G [00:15<00:12, 342MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  56% 5.60G/9.98G [00:15<00:13, 328MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  57% 5.64G/9.98G [00:15<00:13, 319MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  57% 5.68G/9.98G [00:15<00:13, 317MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  57% 5.73G/9.98G [00:16<00:13, 304MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  58% 5.77G/9.98G [00:16<00:13, 308MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  58% 5.80G/9.98G [00:16<00:13, 299MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  58% 5.83G/9.98G [00:16<00:13, 299MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  59% 5.87G/9.98G [00:16<00:13, 306MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  59% 5.90G/9.98G [00:16<00:13, 305MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  60% 5.95G/9.98G [00:16<00:13, 308MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  60% 5.98G/9.98G [00:16<00:14, 282MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  60% 6.03G/9.98G [00:17<00:12, 318MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  61% 6.08G/9.98G [00:17<00:11, 351MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  61% 6.12G/9.98G [00:17<00:10, 365MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  62% 6.18G/9.98G [00:17<00:09, 389MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  62% 6.23G/9.98G [00:17<00:08, 417MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  63% 6.28G/9.98G [00:17<00:08, 424MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  64% 6.34G/9.98G [00:17<00:07, 459MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  64% 6.40G/9.98G [00:17<00:07, 454MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  65% 6.45G/9.98G [00:17<00:07, 445MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  65% 6.50G/9.98G [00:18<00:07, 444MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  66% 6.55G/9.98G [00:18<00:07, 465MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  66% 6.61G/9.98G [00:18<00:07, 457MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  67% 6.66G/9.98G [00:18<00:07, 460MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  67% 6.72G/9.98G [00:18<00:06, 483MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  68% 6.77G/9.98G [00:18<00:07, 455MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  68% 6.83G/9.98G [00:18<00:07, 439MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  69% 6.88G/9.98G [00:19<00:10, 291MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  69% 6.93G/9.98G [00:19<00:09, 327MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  70% 6.98G/9.98G [00:19<00:08, 356MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  70% 7.03G/9.98G [00:19<00:08, 363MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  71% 7.08G/9.98G [00:19<00:07, 393MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  71% 7.13G/9.98G [00:19<00:06, 411MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  72% 7.18G/9.98G [00:19<00:06, 410MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  73% 7.24G/9.98G [00:19<00:06, 400MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  73% 7.29G/9.98G [00:20<00:06, 420MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  74% 7.34G/9.98G [00:20<00:06, 431MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  74% 7.39G/9.98G [00:20<00:05, 450MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  75% 7.46G/9.98G [00:20<00:05, 480MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  75% 7.52G/9.98G [00:20<00:05, 490MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  76% 7.57G/9.98G [00:20<00:05, 463MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  76% 7.62G/9.98G [00:20<00:05, 455MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  77% 7.68G/9.98G [00:20<00:05, 433MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  77% 7.73G/9.98G [00:21<00:05, 423MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  78% 7.78G/9.98G [00:21<00:05, 436MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  79% 7.83G/9.98G [00:21<00:05, 359MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  79% 7.87G/9.98G [00:21<00:06, 315MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  79% 7.92G/9.98G [00:21<00:07, 285MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  80% 7.95G/9.98G [00:21<00:07, 271MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  80% 7.98G/9.98G [00:21<00:07, 257MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  80% 8.01G/9.98G [00:22<00:08, 242MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  81% 8.04G/9.98G [00:22<00:08, 227MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  81% 8.07G/9.98G [00:22<00:09, 211MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  81% 8.11G/9.98G [00:22<00:09, 195MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  81% 8.13G/9.98G [00:22<00:09, 188MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  82% 8.15G/9.98G [00:22<00:10, 174MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  82% 8.17G/9.98G [00:23<00:10, 169MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  82% 8.19G/9.98G [00:23<00:10, 164MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  82% 8.22G/9.98G [00:23<00:09, 186MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  83% 8.26G/9.98G [00:23<00:07, 230MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  83% 8.32G/9.98G [00:23<00:05, 282MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  84% 8.37G/9.98G [00:23<00:04, 327MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  84% 8.42G/9.98G [00:23<00:04, 366MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  85% 8.48G/9.98G [00:23<00:03, 420MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  86% 8.54G/9.98G [00:24<00:03, 435MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  86% 8.59G/9.98G [00:24<00:03, 453MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  87% 8.64G/9.98G [00:24<00:02, 454MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  87% 8.69G/9.98G [00:24<00:02, 462MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  88% 8.75G/9.98G [00:24<00:02, 457MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  88% 8.80G/9.98G [00:24<00:02, 454MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  89% 8.85G/9.98G [00:24<00:02, 466MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  89% 8.90G/9.98G [00:24<00:02, 449MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  90% 8.95G/9.98G [00:24<00:02, 446MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  90% 9.01G/9.98G [00:25<00:02, 432MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  91% 9.06G/9.98G [00:25<00:02, 442MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  91% 9.11G/9.98G [00:25<00:01, 447MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  92% 9.16G/9.98G [00:25<00:01, 438MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  92% 9.22G/9.98G [00:25<00:01, 446MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  93% 9.27G/9.98G [00:25<00:01, 413MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  93% 9.32G/9.98G [00:25<00:01, 418MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  94% 9.37G/9.98G [00:25<00:01, 423MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  94% 9.43G/9.98G [00:26<00:01, 393MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  95% 9.47G/9.98G [00:26<00:01, 334MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  95% 9.51G/9.98G [00:26<00:01, 299MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  96% 9.54G/9.98G [00:26<00:01, 280MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  96% 9.57G/9.98G [00:26<00:01, 262MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  96% 9.60G/9.98G [00:26<00:01, 247MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  97% 9.64G/9.98G [00:27<00:01, 227MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  97% 9.67G/9.98G [00:27<00:01, 214MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  97% 9.70G/9.98G [00:27<00:01, 203MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  97% 9.72G/9.98G [00:27<00:01, 190MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  98% 9.74G/9.98G [00:27<00:01, 186MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  98% 9.76G/9.98G [00:27<00:01, 179MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  98% 9.78G/9.98G [00:27<00:01, 180MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  98% 9.80G/9.98G [00:28<00:00, 176MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  99% 9.84G/9.98G [00:28<00:00, 189MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  99% 9.87G/9.98G [00:28<00:00, 216MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin:  99% 9.91G/9.98G [00:28<00:00, 257MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00002.bin: 100% 9.98G/9.98G [00:28<00:00, 349MB/s]\n",
            "Downloading shards:  50% 1/2 [00:31<00:31, 31.74s/it]\n",
            "Downloading (…)l-00002-of-00002.bin:   0% 0.00/3.50G [00:00<?, ?B/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:   1% 21.0M/3.50G [00:00<00:25, 136MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:   1% 52.4M/3.50G [00:00<00:16, 203MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:   3% 115M/3.50G [00:00<00:09, 350MB/s] \u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:   5% 168M/3.50G [00:00<00:08, 389MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:   6% 220M/3.50G [00:00<00:07, 429MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:   8% 273M/3.50G [00:00<00:07, 446MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:   9% 325M/3.50G [00:00<00:06, 456MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  11% 377M/3.50G [00:00<00:06, 450MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  12% 430M/3.50G [00:01<00:07, 436MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  14% 482M/3.50G [00:01<00:06, 440MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  15% 535M/3.50G [00:01<00:06, 441MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  17% 587M/3.50G [00:01<00:06, 445MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  18% 640M/3.50G [00:01<00:06, 460MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  20% 692M/3.50G [00:01<00:05, 472MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  21% 744M/3.50G [00:01<00:05, 469MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  23% 797M/3.50G [00:01<00:05, 479MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  24% 849M/3.50G [00:01<00:05, 481MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  26% 902M/3.50G [00:02<00:05, 492MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  27% 954M/3.50G [00:02<00:05, 461MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  29% 1.01G/3.50G [00:02<00:06, 381MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  30% 1.05G/3.50G [00:02<00:07, 338MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  31% 1.09G/3.50G [00:02<00:07, 305MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  32% 1.13G/3.50G [00:03<00:11, 211MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  34% 1.18G/3.50G [00:03<00:09, 257MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  35% 1.23G/3.50G [00:03<00:08, 281MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  36% 1.27G/3.50G [00:03<00:08, 275MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  37% 1.30G/3.50G [00:03<00:08, 254MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  38% 1.33G/3.50G [00:03<00:08, 246MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  39% 1.36G/3.50G [00:03<00:08, 248MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  40% 1.41G/3.50G [00:04<00:07, 270MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  41% 1.45G/3.50G [00:04<00:06, 300MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  43% 1.50G/3.50G [00:04<00:05, 343MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  44% 1.55G/3.50G [00:04<00:05, 372MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  46% 1.60G/3.50G [00:04<00:04, 405MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  47% 1.66G/3.50G [00:04<00:04, 425MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  49% 1.72G/3.50G [00:04<00:03, 458MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  51% 1.77G/3.50G [00:04<00:03, 450MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  52% 1.82G/3.50G [00:04<00:03, 450MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  54% 1.89G/3.50G [00:05<00:03, 476MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  55% 1.94G/3.50G [00:05<00:03, 476MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  57% 2.00G/3.50G [00:05<00:03, 499MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  59% 2.06G/3.50G [00:05<00:02, 496MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  60% 2.11G/3.50G [00:05<00:02, 483MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  62% 2.16G/3.50G [00:05<00:03, 431MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  63% 2.21G/3.50G [00:05<00:02, 439MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  65% 2.26G/3.50G [00:05<00:02, 455MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  66% 2.32G/3.50G [00:05<00:02, 447MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  68% 2.37G/3.50G [00:06<00:02, 434MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  69% 2.43G/3.50G [00:06<00:02, 483MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  71% 2.49G/3.50G [00:06<00:02, 443MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  72% 2.54G/3.50G [00:06<00:02, 389MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  74% 2.58G/3.50G [00:06<00:02, 363MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  75% 2.62G/3.50G [00:06<00:02, 343MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  76% 2.66G/3.50G [00:06<00:02, 327MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  77% 2.71G/3.50G [00:07<00:02, 313MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  78% 2.75G/3.50G [00:07<00:02, 303MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  79% 2.78G/3.50G [00:07<00:02, 298MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  80% 2.81G/3.50G [00:07<00:02, 295MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  81% 2.84G/3.50G [00:07<00:02, 286MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  82% 2.87G/3.50G [00:07<00:02, 279MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  83% 2.90G/3.50G [00:07<00:02, 275MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  84% 2.94G/3.50G [00:07<00:02, 268MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  85% 2.97G/3.50G [00:08<00:02, 253MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  86% 3.00G/3.50G [00:08<00:02, 248MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  87% 3.03G/3.50G [00:08<00:01, 241MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  87% 3.06G/3.50G [00:08<00:01, 225MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  88% 3.09G/3.50G [00:08<00:01, 219MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  89% 3.12G/3.50G [00:08<00:01, 210MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  90% 3.16G/3.50G [00:09<00:01, 203MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  91% 3.18G/3.50G [00:09<00:01, 196MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  91% 3.20G/3.50G [00:09<00:01, 196MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  92% 3.22G/3.50G [00:09<00:01, 191MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  93% 3.24G/3.50G [00:09<00:01, 186MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  93% 3.27G/3.50G [00:09<00:01, 200MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  95% 3.31G/3.50G [00:09<00:00, 236MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  96% 3.36G/3.50G [00:09<00:00, 281MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin:  97% 3.41G/3.50G [00:09<00:00, 330MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00002.bin: 100% 3.50G/3.50G [00:10<00:00, 344MB/s]\n",
            "Downloading shards: 100% 2/2 [00:42<00:00, 21.10s/it]\n",
            "The model weights are not tied. Please use the `tie_weights` method before using the `infer_auto_device` function.\n",
            "Loading checkpoint shards: 100% 2/2 [00:10<00:00,  5.48s/it]\n",
            "Downloading (…)neration_config.json: 100% 137/137 [00:00<00:00, 927kB/s]\n",
            "Downloading tokenizer.model: 100% 500k/500k [00:01<00:00, 411kB/s]\n",
            "Downloading (…)cial_tokens_map.json: 100% 411/411 [00:00<00:00, 3.45MB/s]\n",
            "Downloading (…)okenizer_config.json: 100% 700/700 [00:00<00:00, 5.54MB/s]\n",
            "Downloading and preparing dataset json/default to /root/.cache/huggingface/datasets/json/default-cfa1d5e7c46929e4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96...\n",
            "Downloading data files: 100% 1/1 [00:00<00:00, 3872.86it/s]\n",
            "Extracting data files: 100% 1/1 [00:02<00:00,  2.03s/it]\n",
            "Dataset json downloaded and prepared to /root/.cache/huggingface/datasets/json/default-cfa1d5e7c46929e4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96. Subsequent calls will reuse this data.\n",
            "100% 1/1 [00:00<00:00, 229.61it/s]\n",
            "trainable params: 4194304 || all params: 6742609920 || trainable%: 0.06220594176090199\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mxuwenhao\u001b[0m. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.15.4\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run data is saved locally in \u001b[35m\u001b[1m/content/gdrive/MyDrive/alpaca-lora/wandb/run-20230624_054902-h5xtoucw\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb offline`\u001b[0m to turn off syncing.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Syncing run \u001b[33malpaca-20230624\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at \u001b[34m\u001b[4mhttps://wandb.ai/xuwenhao/mactalk-demo\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run at \u001b[34m\u001b[4mhttps://wandb.ai/xuwenhao/mactalk-demo/runs/h5xtoucw\u001b[0m\n",
            "{'loss': 1.8828, 'learning_rate': 1e-05, 'epoch': 0.02}\n",
            "{'loss': 2.1034, 'learning_rate': 2e-05, 'epoch': 0.04}\n",
            "{'loss': 2.2946, 'learning_rate': 3e-05, 'epoch': 0.06}\n",
            "{'loss': 2.3642, 'learning_rate': 4e-05, 'epoch': 0.08}\n",
            "{'loss': 2.1826, 'learning_rate': 5e-05, 'epoch': 0.1}\n",
            "{'loss': 1.4636, 'learning_rate': 6e-05, 'epoch': 0.12}\n",
            "{'loss': 1.3705, 'learning_rate': 7e-05, 'epoch': 0.14}\n",
            "{'loss': 1.0137, 'learning_rate': 8e-05, 'epoch': 0.16}\n",
            "{'loss': 0.7729, 'learning_rate': 9e-05, 'epoch': 0.18}\n",
            "{'loss': 0.7018, 'learning_rate': 0.0001, 'epoch': 0.2}\n",
            "{'loss': 1.0843, 'learning_rate': 9.92857142857143e-05, 'epoch': 0.22}\n",
            "{'loss': 0.9953, 'learning_rate': 9.857142857142858e-05, 'epoch': 0.24}\n",
            "{'loss': 0.8688, 'learning_rate': 9.785714285714286e-05, 'epoch': 0.26}\n",
            "{'loss': 0.7079, 'learning_rate': 9.714285714285715e-05, 'epoch': 0.28}\n",
            "{'loss': 0.6484, 'learning_rate': 9.642857142857143e-05, 'epoch': 0.3}\n",
            "{'loss': 1.0624, 'learning_rate': 9.571428571428573e-05, 'epoch': 0.32}\n",
            "{'loss': 0.9657, 'learning_rate': 9.5e-05, 'epoch': 0.34}\n",
            "{'loss': 0.824, 'learning_rate': 9.428571428571429e-05, 'epoch': 0.36}\n",
            "{'loss': 0.7031, 'learning_rate': 9.357142857142858e-05, 'epoch': 0.38}\n",
            "{'loss': 0.6489, 'learning_rate': 9.285714285714286e-05, 'epoch': 0.4}\n",
            " 13% 200/1500 [17:29<1:03:44,  2.94s/it]\n",
            "  0% 0/250 [00:00<?, ?it/s]\u001b[A\n",
            "  1% 2/250 [00:00<00:39,  6.21it/s]\u001b[A\n",
            "  1% 3/250 [00:00<00:55,  4.49it/s]\u001b[A\n",
            "  2% 4/250 [00:01<01:09,  3.55it/s]\u001b[A\n",
            "  2% 5/250 [00:01<01:14,  3.29it/s]\u001b[A\n",
            "  2% 6/250 [00:01<01:14,  3.29it/s]\u001b[A\n",
            "  3% 7/250 [00:01<01:12,  3.33it/s]\u001b[A\n",
            "  3% 8/250 [00:02<01:17,  3.13it/s]\u001b[A\n",
            "  4% 9/250 [00:02<01:20,  2.98it/s]\u001b[A\n",
            "  4% 10/250 [00:03<01:27,  2.75it/s]\u001b[A\n",
            "  4% 11/250 [00:03<01:23,  2.86it/s]\u001b[A\n",
            "  5% 12/250 [00:03<01:23,  2.86it/s]\u001b[A\n",
            "  5% 13/250 [00:04<01:19,  2.99it/s]\u001b[A\n",
            "  6% 14/250 [00:04<01:16,  3.08it/s]\u001b[A\n",
            "  6% 15/250 [00:04<01:17,  3.03it/s]\u001b[A\n",
            "  6% 16/250 [00:05<01:15,  3.09it/s]\u001b[A\n",
            "  7% 17/250 [00:05<01:16,  3.05it/s]\u001b[A\n",
            "  7% 18/250 [00:05<01:17,  3.01it/s]\u001b[A\n",
            "  8% 19/250 [00:06<01:16,  3.03it/s]\u001b[A\n",
            "  8% 20/250 [00:06<01:17,  2.98it/s]\u001b[A\n",
            "  8% 21/250 [00:06<01:15,  3.03it/s]\u001b[A\n",
            "  9% 22/250 [00:07<01:16,  3.00it/s]\u001b[A\n",
            "  9% 23/250 [00:07<01:12,  3.12it/s]\u001b[A\n",
            " 10% 24/250 [00:07<01:14,  3.03it/s]\u001b[A\n",
            " 10% 25/250 [00:08<01:22,  2.74it/s]\u001b[A\n",
            " 10% 26/250 [00:08<01:23,  2.68it/s]\u001b[A\n",
            " 11% 27/250 [00:08<01:19,  2.82it/s]\u001b[A\n",
            " 11% 28/250 [00:09<01:14,  2.97it/s]\u001b[A\n",
            " 12% 29/250 [00:09<01:13,  3.02it/s]\u001b[A\n",
            " 12% 30/250 [00:09<01:12,  3.03it/s]\u001b[A\n",
            " 12% 31/250 [00:10<01:10,  3.11it/s]\u001b[A\n",
            " 13% 32/250 [00:10<01:09,  3.13it/s]\u001b[A\n",
            " 13% 33/250 [00:10<01:09,  3.10it/s]\u001b[A\n",
            " 14% 34/250 [00:11<01:08,  3.17it/s]\u001b[A\n",
            " 14% 35/250 [00:11<01:07,  3.16it/s]\u001b[A\n",
            " 14% 36/250 [00:11<01:13,  2.92it/s]\u001b[A\n",
            " 15% 37/250 [00:12<01:12,  2.95it/s]\u001b[A\n",
            " 15% 38/250 [00:12<01:10,  3.00it/s]\u001b[A\n",
            " 16% 39/250 [00:12<01:11,  2.95it/s]\u001b[A\n",
            " 16% 40/250 [00:13<01:15,  2.79it/s]\u001b[A\n",
            " 16% 41/250 [00:13<01:13,  2.83it/s]\u001b[A\n",
            " 17% 42/250 [00:13<01:13,  2.83it/s]\u001b[A\n",
            " 17% 43/250 [00:14<01:14,  2.78it/s]\u001b[A\n",
            " 18% 44/250 [00:14<01:11,  2.89it/s]\u001b[A\n",
            " 18% 45/250 [00:14<01:11,  2.85it/s]\u001b[A\n",
            " 18% 46/250 [00:15<01:09,  2.93it/s]\u001b[A\n",
            " 19% 47/250 [00:15<01:11,  2.82it/s]\u001b[A\n",
            " 19% 48/250 [00:15<01:10,  2.88it/s]\u001b[A\n",
            " 20% 49/250 [00:16<01:08,  2.96it/s]\u001b[A\n",
            " 20% 50/250 [00:16<01:08,  2.93it/s]\u001b[A\n",
            " 20% 51/250 [00:16<01:06,  2.99it/s]\u001b[A\n",
            " 21% 52/250 [00:17<01:05,  3.01it/s]\u001b[A\n",
            " 21% 53/250 [00:17<01:08,  2.87it/s]\u001b[A\n",
            " 22% 54/250 [00:17<01:08,  2.85it/s]\u001b[A\n",
            " 22% 55/250 [00:18<01:06,  2.93it/s]\u001b[A\n",
            " 22% 56/250 [00:18<01:06,  2.93it/s]\u001b[A\n",
            " 23% 57/250 [00:19<01:06,  2.89it/s]\u001b[A\n",
            " 23% 58/250 [00:19<01:04,  2.98it/s]\u001b[A\n",
            " 24% 59/250 [00:19<01:05,  2.91it/s]\u001b[A\n",
            " 24% 60/250 [00:20<01:15,  2.51it/s]\u001b[A\n",
            " 24% 61/250 [00:20<01:11,  2.66it/s]\u001b[A\n",
            " 25% 62/250 [00:20<01:13,  2.55it/s]\u001b[A\n",
            " 25% 63/250 [00:21<01:08,  2.73it/s]\u001b[A\n",
            " 26% 64/250 [00:21<01:07,  2.77it/s]\u001b[A\n",
            " 26% 65/250 [00:22<01:08,  2.70it/s]\u001b[A\n",
            " 26% 66/250 [00:22<01:07,  2.74it/s]\u001b[A\n",
            " 27% 67/250 [00:22<01:05,  2.79it/s]\u001b[A\n",
            " 27% 68/250 [00:23<01:02,  2.91it/s]\u001b[A\n",
            " 28% 69/250 [00:23<01:11,  2.54it/s]\u001b[A\n",
            " 28% 70/250 [00:23<01:10,  2.55it/s]\u001b[A\n",
            " 28% 71/250 [00:24<01:06,  2.71it/s]\u001b[A\n",
            " 29% 72/250 [00:24<01:03,  2.82it/s]\u001b[A\n",
            " 29% 73/250 [00:24<01:01,  2.87it/s]\u001b[A\n",
            " 30% 74/250 [00:25<01:08,  2.56it/s]\u001b[A\n",
            " 30% 75/250 [00:25<01:05,  2.67it/s]\u001b[A\n",
            " 30% 76/250 [00:26<01:04,  2.72it/s]\u001b[A\n",
            " 31% 77/250 [00:26<01:00,  2.86it/s]\u001b[A\n",
            " 31% 78/250 [00:26<00:58,  2.96it/s]\u001b[A\n",
            " 32% 79/250 [00:27<01:03,  2.70it/s]\u001b[A\n",
            " 32% 80/250 [00:27<01:05,  2.59it/s]\u001b[A\n",
            " 32% 81/250 [00:27<01:01,  2.75it/s]\u001b[A\n",
            " 33% 82/250 [00:28<00:58,  2.88it/s]\u001b[A\n",
            " 33% 83/250 [00:28<01:00,  2.74it/s]\u001b[A\n",
            " 34% 84/250 [00:28<00:59,  2.78it/s]\u001b[A\n",
            " 34% 85/250 [00:29<00:57,  2.85it/s]\u001b[A\n",
            " 34% 86/250 [00:29<01:03,  2.59it/s]\u001b[A\n",
            " 35% 87/250 [00:30<01:00,  2.67it/s]\u001b[A\n",
            " 35% 88/250 [00:30<01:02,  2.58it/s]\u001b[A\n",
            " 36% 89/250 [00:30<00:59,  2.72it/s]\u001b[A\n",
            " 36% 90/250 [00:31<01:08,  2.35it/s]\u001b[A\n",
            " 36% 91/250 [00:31<01:03,  2.51it/s]\u001b[A\n",
            " 37% 92/250 [00:32<00:59,  2.64it/s]\u001b[A\n",
            " 37% 93/250 [00:32<01:00,  2.58it/s]\u001b[A\n",
            " 38% 94/250 [00:32<00:59,  2.62it/s]\u001b[A\n",
            " 38% 95/250 [00:33<00:59,  2.61it/s]\u001b[A\n",
            " 38% 96/250 [00:33<01:07,  2.29it/s]\u001b[A\n",
            " 39% 97/250 [00:34<01:01,  2.51it/s]\u001b[A\n",
            " 39% 98/250 [00:34<00:56,  2.67it/s]\u001b[A\n",
            " 40% 99/250 [00:34<00:53,  2.83it/s]\u001b[A\n",
            " 40% 100/250 [00:35<00:51,  2.88it/s]\u001b[A\n",
            " 40% 101/250 [00:35<00:50,  2.94it/s]\u001b[A\n",
            " 41% 102/250 [00:35<00:50,  2.92it/s]\u001b[A\n",
            " 41% 103/250 [00:36<00:51,  2.86it/s]\u001b[A\n",
            " 42% 104/250 [00:36<00:50,  2.88it/s]\u001b[A\n",
            " 42% 105/250 [00:36<00:48,  2.99it/s]\u001b[A\n",
            " 42% 106/250 [00:37<00:55,  2.60it/s]\u001b[A\n",
            " 43% 107/250 [00:37<00:58,  2.45it/s]\u001b[A\n",
            " 43% 108/250 [00:37<00:53,  2.64it/s]\u001b[A\n",
            " 44% 109/250 [00:38<00:50,  2.78it/s]\u001b[A\n",
            " 44% 110/250 [00:38<00:50,  2.78it/s]\u001b[A\n",
            " 44% 111/250 [00:38<00:48,  2.86it/s]\u001b[A\n",
            " 45% 112/250 [00:39<00:50,  2.74it/s]\u001b[A\n",
            " 45% 113/250 [00:39<00:48,  2.84it/s]\u001b[A\n",
            " 46% 114/250 [00:40<00:46,  2.93it/s]\u001b[A\n",
            " 46% 115/250 [00:40<00:47,  2.83it/s]\u001b[A\n",
            " 46% 116/250 [00:40<00:50,  2.65it/s]\u001b[A\n",
            " 47% 117/250 [00:41<00:48,  2.73it/s]\u001b[A\n",
            " 47% 118/250 [00:41<00:49,  2.66it/s]\u001b[A\n",
            " 48% 119/250 [00:41<00:47,  2.74it/s]\u001b[A\n",
            " 48% 120/250 [00:42<00:46,  2.78it/s]\u001b[A\n",
            " 48% 121/250 [00:42<00:45,  2.82it/s]\u001b[A\n",
            " 49% 122/250 [00:42<00:44,  2.88it/s]\u001b[A\n",
            " 49% 123/250 [00:43<00:42,  2.98it/s]\u001b[A\n",
            " 50% 124/250 [00:43<00:41,  3.05it/s]\u001b[A\n",
            " 50% 125/250 [00:43<00:40,  3.08it/s]\u001b[A\n",
            " 50% 126/250 [00:44<00:40,  3.09it/s]\u001b[A\n",
            " 51% 127/250 [00:44<00:41,  2.99it/s]\u001b[A\n",
            " 51% 128/250 [00:45<00:49,  2.48it/s]\u001b[A\n",
            " 52% 129/250 [00:45<00:45,  2.64it/s]\u001b[A\n",
            " 52% 130/250 [00:45<00:46,  2.57it/s]\u001b[A\n",
            " 52% 131/250 [00:46<00:46,  2.57it/s]\u001b[A\n",
            " 53% 132/250 [00:46<00:44,  2.65it/s]\u001b[A\n",
            " 53% 133/250 [00:46<00:43,  2.67it/s]\u001b[A\n",
            " 54% 134/250 [00:47<00:43,  2.64it/s]\u001b[A\n",
            " 54% 135/250 [00:47<00:42,  2.73it/s]\u001b[A\n",
            " 54% 136/250 [00:47<00:39,  2.87it/s]\u001b[A\n",
            " 55% 137/250 [00:48<00:40,  2.79it/s]\u001b[A\n",
            " 55% 138/250 [00:48<00:38,  2.91it/s]\u001b[A\n",
            " 56% 139/250 [00:49<00:40,  2.74it/s]\u001b[A\n",
            " 56% 140/250 [00:49<00:38,  2.82it/s]\u001b[A\n",
            " 56% 141/250 [00:49<00:37,  2.93it/s]\u001b[A\n",
            " 57% 142/250 [00:50<00:43,  2.50it/s]\u001b[A\n",
            " 57% 143/250 [00:50<00:42,  2.54it/s]\u001b[A\n",
            " 58% 144/250 [00:50<00:39,  2.68it/s]\u001b[A\n",
            " 58% 145/250 [00:51<00:38,  2.74it/s]\u001b[A\n",
            " 58% 146/250 [00:51<00:36,  2.87it/s]\u001b[A\n",
            " 59% 147/250 [00:51<00:35,  2.93it/s]\u001b[A\n",
            " 59% 148/250 [00:52<00:33,  3.03it/s]\u001b[A\n",
            " 60% 149/250 [00:52<00:34,  2.95it/s]\u001b[A\n",
            " 60% 150/250 [00:52<00:33,  3.01it/s]\u001b[A\n",
            " 60% 151/250 [00:53<00:33,  2.97it/s]\u001b[A\n",
            " 61% 152/250 [00:53<00:34,  2.87it/s]\u001b[A\n",
            " 61% 153/250 [00:53<00:33,  2.93it/s]\u001b[A\n",
            " 62% 154/250 [00:54<00:33,  2.89it/s]\u001b[A\n",
            " 62% 155/250 [00:54<00:31,  3.04it/s]\u001b[A\n",
            " 62% 156/250 [00:55<00:33,  2.77it/s]\u001b[A\n",
            " 63% 157/250 [00:55<00:33,  2.80it/s]\u001b[A\n",
            " 63% 158/250 [00:55<00:33,  2.74it/s]\u001b[A\n",
            " 64% 159/250 [00:56<00:31,  2.86it/s]\u001b[A\n",
            " 64% 160/250 [00:56<00:31,  2.89it/s]\u001b[A\n",
            " 64% 161/250 [00:56<00:30,  2.89it/s]\u001b[A\n",
            " 65% 162/250 [00:57<00:30,  2.93it/s]\u001b[A\n",
            " 65% 163/250 [00:57<00:29,  2.95it/s]\u001b[A\n",
            " 66% 164/250 [00:57<00:29,  2.92it/s]\u001b[A\n",
            " 66% 165/250 [00:58<00:29,  2.90it/s]\u001b[A\n",
            " 66% 166/250 [00:58<00:29,  2.81it/s]\u001b[A\n",
            " 67% 167/250 [00:58<00:28,  2.92it/s]\u001b[A\n",
            " 67% 168/250 [00:59<00:28,  2.91it/s]\u001b[A\n",
            " 68% 169/250 [00:59<00:29,  2.70it/s]\u001b[A\n",
            " 68% 170/250 [00:59<00:29,  2.72it/s]\u001b[A\n",
            " 68% 171/250 [01:00<00:27,  2.88it/s]\u001b[A\n",
            " 69% 172/250 [01:00<00:26,  2.96it/s]\u001b[A\n",
            " 69% 173/250 [01:00<00:26,  2.96it/s]\u001b[A\n",
            " 70% 174/250 [01:01<00:25,  3.00it/s]\u001b[A\n",
            " 70% 175/250 [01:01<00:24,  3.03it/s]\u001b[A\n",
            " 70% 176/250 [01:01<00:24,  3.02it/s]\u001b[A\n",
            " 71% 177/250 [01:02<00:24,  3.02it/s]\u001b[A\n",
            " 71% 178/250 [01:02<00:24,  2.99it/s]\u001b[A\n",
            " 72% 179/250 [01:02<00:23,  3.03it/s]\u001b[A\n",
            " 72% 180/250 [01:03<00:23,  2.95it/s]\u001b[A\n",
            " 72% 181/250 [01:03<00:22,  3.04it/s]\u001b[A\n",
            " 73% 182/250 [01:03<00:22,  3.05it/s]\u001b[A\n",
            " 73% 183/250 [01:04<00:22,  3.00it/s]\u001b[A\n",
            " 74% 184/250 [01:04<00:22,  2.95it/s]\u001b[A\n",
            " 74% 185/250 [01:04<00:22,  2.94it/s]\u001b[A\n",
            " 74% 186/250 [01:05<00:21,  3.00it/s]\u001b[A\n",
            " 75% 187/250 [01:05<00:21,  2.98it/s]\u001b[A\n",
            " 75% 188/250 [01:05<00:20,  2.98it/s]\u001b[A\n",
            " 76% 189/250 [01:06<00:20,  2.97it/s]\u001b[A\n",
            " 76% 190/250 [01:06<00:19,  3.03it/s]\u001b[A\n",
            " 76% 191/250 [01:06<00:19,  3.05it/s]\u001b[A\n",
            " 77% 192/250 [01:07<00:19,  2.93it/s]\u001b[A\n",
            " 77% 193/250 [01:07<00:19,  2.96it/s]\u001b[A\n",
            " 78% 194/250 [01:07<00:18,  3.05it/s]\u001b[A\n",
            " 78% 195/250 [01:08<00:17,  3.06it/s]\u001b[A\n",
            " 78% 196/250 [01:08<00:17,  3.10it/s]\u001b[A\n",
            " 79% 197/250 [01:08<00:17,  3.02it/s]\u001b[A\n",
            " 79% 198/250 [01:09<00:17,  2.90it/s]\u001b[A\n",
            " 80% 199/250 [01:09<00:17,  2.97it/s]\u001b[A\n",
            " 80% 200/250 [01:09<00:17,  2.94it/s]\u001b[A\n",
            " 80% 201/250 [01:10<00:16,  2.93it/s]\u001b[A\n",
            " 81% 202/250 [01:10<00:16,  2.97it/s]\u001b[A\n",
            " 81% 203/250 [01:10<00:15,  2.96it/s]\u001b[A\n",
            " 82% 204/250 [01:11<00:16,  2.81it/s]\u001b[A\n",
            " 82% 205/250 [01:11<00:15,  2.85it/s]\u001b[A\n",
            " 82% 206/250 [01:12<00:15,  2.88it/s]\u001b[A\n",
            " 83% 207/250 [01:12<00:15,  2.85it/s]\u001b[A\n",
            " 83% 208/250 [01:12<00:14,  2.86it/s]\u001b[A\n",
            " 84% 209/250 [01:13<00:14,  2.84it/s]\u001b[A\n",
            " 84% 210/250 [01:13<00:13,  2.95it/s]\u001b[A\n",
            " 84% 211/250 [01:13<00:12,  3.01it/s]\u001b[A\n",
            " 85% 212/250 [01:14<00:12,  3.10it/s]\u001b[A\n",
            " 85% 213/250 [01:14<00:12,  3.01it/s]\u001b[A\n",
            " 86% 214/250 [01:14<00:11,  3.05it/s]\u001b[A\n",
            " 86% 215/250 [01:15<00:11,  3.05it/s]\u001b[A\n",
            " 86% 216/250 [01:15<00:11,  3.05it/s]\u001b[A\n",
            " 87% 217/250 [01:15<00:11,  2.98it/s]\u001b[A\n",
            " 87% 218/250 [01:16<00:10,  2.94it/s]\u001b[A\n",
            " 88% 219/250 [01:16<00:10,  3.03it/s]\u001b[A\n",
            " 88% 220/250 [01:16<00:09,  3.12it/s]\u001b[A\n",
            " 88% 221/250 [01:16<00:09,  3.13it/s]\u001b[A\n",
            " 89% 222/250 [01:17<00:09,  2.98it/s]\u001b[A\n",
            " 89% 223/250 [01:17<00:09,  2.93it/s]\u001b[A\n",
            " 90% 224/250 [01:18<00:08,  3.02it/s]\u001b[A\n",
            " 90% 225/250 [01:18<00:08,  2.93it/s]\u001b[A\n",
            " 90% 226/250 [01:18<00:08,  2.92it/s]\u001b[A\n",
            " 91% 227/250 [01:19<00:07,  2.99it/s]\u001b[A\n",
            " 91% 228/250 [01:19<00:07,  2.97it/s]\u001b[A\n",
            " 92% 229/250 [01:19<00:07,  2.75it/s]\u001b[A\n",
            " 92% 230/250 [01:20<00:07,  2.80it/s]\u001b[A\n",
            " 92% 231/250 [01:20<00:07,  2.67it/s]\u001b[A\n",
            " 93% 232/250 [01:21<00:07,  2.33it/s]\u001b[A\n",
            " 93% 233/250 [01:21<00:06,  2.48it/s]\u001b[A\n",
            " 94% 234/250 [01:21<00:06,  2.66it/s]\u001b[A\n",
            " 94% 235/250 [01:22<00:05,  2.80it/s]\u001b[A\n",
            " 94% 236/250 [01:22<00:05,  2.40it/s]\u001b[A\n",
            " 95% 237/250 [01:23<00:05,  2.44it/s]\u001b[A\n",
            " 95% 238/250 [01:23<00:04,  2.42it/s]\u001b[A\n",
            " 96% 239/250 [01:23<00:04,  2.49it/s]\u001b[A\n",
            " 96% 240/250 [01:24<00:03,  2.59it/s]\u001b[A\n",
            " 96% 241/250 [01:24<00:03,  2.64it/s]\u001b[A\n",
            " 97% 242/250 [01:24<00:03,  2.63it/s]\u001b[A\n",
            " 97% 243/250 [01:25<00:02,  2.83it/s]\u001b[A\n",
            " 98% 244/250 [01:25<00:02,  2.91it/s]\u001b[A\n",
            " 98% 245/250 [01:25<00:01,  3.01it/s]\u001b[A\n",
            " 98% 246/250 [01:26<00:01,  3.03it/s]\u001b[A\n",
            " 99% 247/250 [01:26<00:00,  3.08it/s]\u001b[A\n",
            " 99% 248/250 [01:26<00:00,  2.97it/s]\u001b[A\n",
            "100% 249/250 [01:27<00:00,  2.86it/s]\u001b[A\n",
            "100% 250/250 [01:27<00:00,  2.87it/s]\u001b[A\n",
            "{'eval_loss': 0.9394135475158691, 'eval_runtime': 88.0179, 'eval_samples_per_second': 22.723, 'eval_steps_per_second': 2.84, 'epoch': 0.4}\n",
            "\n",
            " 13% 200/1500 [18:57<1:03:44,  2.94s/it]\n",
            "{'loss': 1.0634, 'learning_rate': 9.214285714285714e-05, 'epoch': 0.42}\n",
            "{'loss': 0.9565, 'learning_rate': 9.142857142857143e-05, 'epoch': 0.44}\n",
            "{'loss': 0.8381, 'learning_rate': 9.071428571428571e-05, 'epoch': 0.46}\n",
            "{'loss': 0.6824, 'learning_rate': 9e-05, 'epoch': 0.48}\n",
            "{'loss': 0.6406, 'learning_rate': 8.92857142857143e-05, 'epoch': 0.5}\n",
            "{'loss': 1.0623, 'learning_rate': 8.857142857142857e-05, 'epoch': 0.52}\n",
            "{'loss': 0.9539, 'learning_rate': 8.785714285714286e-05, 'epoch': 0.54}\n",
            "{'loss': 0.8286, 'learning_rate': 8.714285714285715e-05, 'epoch': 0.56}\n",
            "{'loss': 0.682, 'learning_rate': 8.642857142857143e-05, 'epoch': 0.58}\n",
            "{'loss': 0.6302, 'learning_rate': 8.571428571428571e-05, 'epoch': 0.6}\n",
            "{'loss': 1.0559, 'learning_rate': 8.5e-05, 'epoch': 0.62}\n",
            "{'loss': 0.9502, 'learning_rate': 8.428571428571429e-05, 'epoch': 0.64}\n",
            "{'loss': 0.8221, 'learning_rate': 8.357142857142858e-05, 'epoch': 0.66}\n",
            "{'loss': 0.6731, 'learning_rate': 8.285714285714287e-05, 'epoch': 0.68}\n",
            "{'loss': 0.627, 'learning_rate': 8.214285714285714e-05, 'epoch': 0.7}\n",
            "{'loss': 1.0687, 'learning_rate': 8.142857142857143e-05, 'epoch': 0.72}\n",
            "{'loss': 0.9492, 'learning_rate': 8.071428571428573e-05, 'epoch': 0.74}\n",
            "{'loss': 0.8198, 'learning_rate': 8e-05, 'epoch': 0.76}\n",
            "{'loss': 0.6647, 'learning_rate': 7.928571428571429e-05, 'epoch': 0.78}\n",
            "{'loss': 0.6199, 'learning_rate': 7.857142857142858e-05, 'epoch': 0.8}\n",
            " 27% 400/1500 [36:31<53:56,  2.94s/it]\n",
            "  0% 0/250 [00:00<?, ?it/s]\u001b[A\n",
            "  1% 2/250 [00:00<00:40,  6.15it/s]\u001b[A\n",
            "  1% 3/250 [00:00<00:55,  4.44it/s]\u001b[A\n",
            "  2% 4/250 [00:01<01:09,  3.53it/s]\u001b[A\n",
            "  2% 5/250 [00:01<01:15,  3.26it/s]\u001b[A\n",
            "  2% 6/250 [00:01<01:15,  3.24it/s]\u001b[A\n",
            "  3% 7/250 [00:01<01:14,  3.27it/s]\u001b[A\n",
            "  3% 8/250 [00:02<01:18,  3.09it/s]\u001b[A\n",
            "  4% 9/250 [00:02<01:21,  2.94it/s]\u001b[A\n",
            "  4% 10/250 [00:03<01:27,  2.74it/s]\u001b[A\n",
            "  4% 11/250 [00:03<01:24,  2.84it/s]\u001b[A\n",
            "  5% 12/250 [00:03<01:23,  2.86it/s]\u001b[A\n",
            "  5% 13/250 [00:04<01:19,  2.98it/s]\u001b[A\n",
            "  6% 14/250 [00:04<01:17,  3.03it/s]\u001b[A\n",
            "  6% 15/250 [00:04<01:18,  2.98it/s]\u001b[A\n",
            "  6% 16/250 [00:05<01:16,  3.05it/s]\u001b[A\n",
            "  7% 17/250 [00:05<01:17,  3.00it/s]\u001b[A\n",
            "  7% 18/250 [00:05<01:18,  2.97it/s]\u001b[A\n",
            "  8% 19/250 [00:06<01:17,  2.98it/s]\u001b[A\n",
            "  8% 20/250 [00:06<01:18,  2.94it/s]\u001b[A\n",
            "  8% 21/250 [00:06<01:16,  2.98it/s]\u001b[A\n",
            "  9% 22/250 [00:07<01:17,  2.95it/s]\u001b[A\n",
            "  9% 23/250 [00:07<01:14,  3.05it/s]\u001b[A\n",
            " 10% 24/250 [00:07<01:16,  2.97it/s]\u001b[A\n",
            " 10% 25/250 [00:08<01:23,  2.70it/s]\u001b[A\n",
            " 10% 26/250 [00:08<01:24,  2.65it/s]\u001b[A\n",
            " 11% 27/250 [00:08<01:20,  2.78it/s]\u001b[A\n",
            " 11% 28/250 [00:09<01:16,  2.91it/s]\u001b[A\n",
            " 12% 29/250 [00:09<01:14,  2.96it/s]\u001b[A\n",
            " 12% 30/250 [00:09<01:13,  2.99it/s]\u001b[A\n",
            " 12% 31/250 [00:10<01:11,  3.07it/s]\u001b[A\n",
            " 13% 32/250 [00:10<01:10,  3.08it/s]\u001b[A\n",
            " 13% 33/250 [00:10<01:11,  3.03it/s]\u001b[A\n",
            " 14% 34/250 [00:11<01:09,  3.09it/s]\u001b[A\n",
            " 14% 35/250 [00:11<01:09,  3.09it/s]\u001b[A\n",
            " 14% 36/250 [00:11<01:14,  2.88it/s]\u001b[A\n",
            " 15% 37/250 [00:12<01:12,  2.93it/s]\u001b[A\n",
            " 15% 38/250 [00:12<01:10,  2.99it/s]\u001b[A\n",
            " 16% 39/250 [00:12<01:11,  2.94it/s]\u001b[A\n",
            " 16% 40/250 [00:13<01:16,  2.76it/s]\u001b[A\n",
            " 16% 41/250 [00:13<01:14,  2.79it/s]\u001b[A\n",
            " 17% 42/250 [00:14<01:14,  2.80it/s]\u001b[A\n",
            " 17% 43/250 [00:14<01:15,  2.74it/s]\u001b[A\n",
            " 18% 44/250 [00:14<01:11,  2.87it/s]\u001b[A\n",
            " 18% 45/250 [00:15<01:12,  2.83it/s]\u001b[A\n",
            " 18% 46/250 [00:15<01:09,  2.92it/s]\u001b[A\n",
            " 19% 47/250 [00:15<01:11,  2.84it/s]\u001b[A\n",
            " 19% 48/250 [00:16<01:09,  2.91it/s]\u001b[A\n",
            " 20% 49/250 [00:16<01:07,  2.99it/s]\u001b[A\n",
            " 20% 50/250 [00:16<01:07,  2.96it/s]\u001b[A\n",
            " 20% 51/250 [00:17<01:05,  3.02it/s]\u001b[A\n",
            " 21% 52/250 [00:17<01:05,  3.04it/s]\u001b[A\n",
            " 21% 53/250 [00:17<01:07,  2.91it/s]\u001b[A\n",
            " 22% 54/250 [00:18<01:07,  2.89it/s]\u001b[A\n",
            " 22% 55/250 [00:18<01:05,  2.97it/s]\u001b[A\n",
            " 22% 56/250 [00:18<01:05,  2.96it/s]\u001b[A\n",
            " 23% 57/250 [00:19<01:06,  2.92it/s]\u001b[A\n",
            " 23% 58/250 [00:19<01:03,  3.01it/s]\u001b[A\n",
            " 24% 59/250 [00:19<01:04,  2.94it/s]\u001b[A\n",
            " 24% 60/250 [00:20<01:15,  2.53it/s]\u001b[A\n",
            " 24% 61/250 [00:20<01:10,  2.68it/s]\u001b[A\n",
            " 25% 62/250 [00:21<01:13,  2.56it/s]\u001b[A\n",
            " 25% 63/250 [00:21<01:08,  2.74it/s]\u001b[A\n",
            " 26% 64/250 [00:21<01:07,  2.78it/s]\u001b[A\n",
            " 26% 65/250 [00:22<01:08,  2.71it/s]\u001b[A\n",
            " 26% 66/250 [00:22<01:07,  2.73it/s]\u001b[A\n",
            " 27% 67/250 [00:22<01:05,  2.79it/s]\u001b[A\n",
            " 27% 68/250 [00:23<01:02,  2.91it/s]\u001b[A\n",
            " 28% 69/250 [00:23<01:11,  2.53it/s]\u001b[A\n",
            " 28% 70/250 [00:24<01:10,  2.55it/s]\u001b[A\n",
            " 28% 71/250 [00:24<01:06,  2.71it/s]\u001b[A\n",
            " 29% 72/250 [00:24<01:03,  2.82it/s]\u001b[A\n",
            " 29% 73/250 [00:25<01:01,  2.86it/s]\u001b[A\n",
            " 30% 74/250 [00:25<01:08,  2.55it/s]\u001b[A\n",
            " 30% 75/250 [00:25<01:05,  2.68it/s]\u001b[A\n",
            " 30% 76/250 [00:26<01:04,  2.71it/s]\u001b[A\n",
            " 31% 77/250 [00:26<01:00,  2.88it/s]\u001b[A\n",
            " 31% 78/250 [00:26<00:57,  2.98it/s]\u001b[A\n",
            " 32% 79/250 [00:27<01:02,  2.72it/s]\u001b[A\n",
            " 32% 80/250 [00:27<01:05,  2.60it/s]\u001b[A\n",
            " 32% 81/250 [00:27<01:01,  2.75it/s]\u001b[A\n",
            " 33% 82/250 [00:28<00:58,  2.89it/s]\u001b[A\n",
            " 33% 83/250 [00:28<01:00,  2.75it/s]\u001b[A\n",
            " 34% 84/250 [00:29<00:59,  2.78it/s]\u001b[A\n",
            " 34% 85/250 [00:29<00:57,  2.85it/s]\u001b[A\n",
            " 34% 86/250 [00:29<01:03,  2.59it/s]\u001b[A\n",
            " 35% 87/250 [00:30<01:00,  2.68it/s]\u001b[A\n",
            " 35% 88/250 [00:30<01:02,  2.59it/s]\u001b[A\n",
            " 36% 89/250 [00:30<00:58,  2.75it/s]\u001b[A\n",
            " 36% 90/250 [00:31<01:07,  2.36it/s]\u001b[A\n",
            " 36% 91/250 [00:31<01:02,  2.53it/s]\u001b[A\n",
            " 37% 92/250 [00:32<00:59,  2.66it/s]\u001b[A\n",
            " 37% 93/250 [00:32<01:00,  2.58it/s]\u001b[A\n",
            " 38% 94/250 [00:32<00:59,  2.63it/s]\u001b[A\n",
            " 38% 95/250 [00:33<00:59,  2.61it/s]\u001b[A\n",
            " 38% 96/250 [00:33<01:07,  2.29it/s]\u001b[A\n",
            " 39% 97/250 [00:34<01:01,  2.50it/s]\u001b[A\n",
            " 39% 98/250 [00:34<00:56,  2.68it/s]\u001b[A\n",
            " 40% 99/250 [00:34<00:53,  2.84it/s]\u001b[A\n",
            " 40% 100/250 [00:35<00:52,  2.88it/s]\u001b[A\n",
            " 40% 101/250 [00:35<00:50,  2.94it/s]\u001b[A\n",
            " 41% 102/250 [00:35<00:50,  2.90it/s]\u001b[A\n",
            " 41% 103/250 [00:36<00:51,  2.84it/s]\u001b[A\n",
            " 42% 104/250 [00:36<00:51,  2.86it/s]\u001b[A\n",
            " 42% 105/250 [00:36<00:48,  2.97it/s]\u001b[A\n",
            " 42% 106/250 [00:37<00:55,  2.58it/s]\u001b[A\n",
            " 43% 107/250 [00:37<00:58,  2.45it/s]\u001b[A\n",
            " 43% 108/250 [00:38<00:54,  2.63it/s]\u001b[A\n",
            " 44% 109/250 [00:38<00:50,  2.78it/s]\u001b[A\n",
            " 44% 110/250 [00:38<00:50,  2.78it/s]\u001b[A\n",
            " 44% 111/250 [00:39<00:48,  2.85it/s]\u001b[A\n",
            " 45% 112/250 [00:39<00:50,  2.73it/s]\u001b[A\n",
            " 45% 113/250 [00:39<00:48,  2.83it/s]\u001b[A\n",
            " 46% 114/250 [00:40<00:46,  2.91it/s]\u001b[A\n",
            " 46% 115/250 [00:40<00:48,  2.81it/s]\u001b[A\n",
            " 46% 116/250 [00:40<00:50,  2.63it/s]\u001b[A\n",
            " 47% 117/250 [00:41<00:49,  2.70it/s]\u001b[A\n",
            " 47% 118/250 [00:41<00:49,  2.64it/s]\u001b[A\n",
            " 48% 119/250 [00:42<00:48,  2.73it/s]\u001b[A\n",
            " 48% 120/250 [00:42<00:47,  2.76it/s]\u001b[A\n",
            " 48% 121/250 [00:42<00:46,  2.79it/s]\u001b[A\n",
            " 49% 122/250 [00:43<00:44,  2.86it/s]\u001b[A\n",
            " 49% 123/250 [00:43<00:42,  2.96it/s]\u001b[A\n",
            " 50% 124/250 [00:43<00:41,  3.03it/s]\u001b[A\n",
            " 50% 125/250 [00:44<00:40,  3.08it/s]\u001b[A\n",
            " 50% 126/250 [00:44<00:40,  3.09it/s]\u001b[A\n",
            " 51% 127/250 [00:44<00:40,  3.00it/s]\u001b[A\n",
            " 51% 128/250 [00:45<00:49,  2.49it/s]\u001b[A\n",
            " 52% 129/250 [00:45<00:45,  2.63it/s]\u001b[A\n",
            " 52% 130/250 [00:46<00:46,  2.55it/s]\u001b[A\n",
            " 52% 131/250 [00:46<00:46,  2.56it/s]\u001b[A\n",
            " 53% 132/250 [00:46<00:44,  2.63it/s]\u001b[A\n",
            " 53% 133/250 [00:47<00:44,  2.65it/s]\u001b[A\n",
            " 54% 134/250 [00:47<00:43,  2.64it/s]\u001b[A\n",
            " 54% 135/250 [00:47<00:42,  2.73it/s]\u001b[A\n",
            " 54% 136/250 [00:48<00:39,  2.87it/s]\u001b[A\n",
            " 55% 137/250 [00:48<00:40,  2.78it/s]\u001b[A\n",
            " 55% 138/250 [00:48<00:38,  2.90it/s]\u001b[A\n",
            " 56% 139/250 [00:49<00:40,  2.72it/s]\u001b[A\n",
            " 56% 140/250 [00:49<00:39,  2.82it/s]\u001b[A\n",
            " 56% 141/250 [00:49<00:37,  2.93it/s]\u001b[A\n",
            " 57% 142/250 [00:50<00:43,  2.51it/s]\u001b[A\n",
            " 57% 143/250 [00:50<00:42,  2.54it/s]\u001b[A\n",
            " 58% 144/250 [00:51<00:39,  2.67it/s]\u001b[A\n",
            " 58% 145/250 [00:51<00:38,  2.73it/s]\u001b[A\n",
            " 58% 146/250 [00:51<00:36,  2.87it/s]\u001b[A\n",
            " 59% 147/250 [00:52<00:35,  2.93it/s]\u001b[A\n",
            " 59% 148/250 [00:52<00:33,  3.03it/s]\u001b[A\n",
            " 60% 149/250 [00:52<00:34,  2.97it/s]\u001b[A\n",
            " 60% 150/250 [00:53<00:32,  3.03it/s]\u001b[A\n",
            " 60% 151/250 [00:53<00:33,  3.00it/s]\u001b[A\n",
            " 61% 152/250 [00:53<00:33,  2.91it/s]\u001b[A\n",
            " 61% 153/250 [00:54<00:32,  2.96it/s]\u001b[A\n",
            " 62% 154/250 [00:54<00:32,  2.91it/s]\u001b[A\n",
            " 62% 155/250 [00:54<00:31,  3.05it/s]\u001b[A\n",
            " 62% 156/250 [00:55<00:33,  2.77it/s]\u001b[A\n",
            " 63% 157/250 [00:55<00:33,  2.80it/s]\u001b[A\n",
            " 63% 158/250 [00:55<00:33,  2.74it/s]\u001b[A\n",
            " 64% 159/250 [00:56<00:31,  2.87it/s]\u001b[A\n",
            " 64% 160/250 [00:56<00:31,  2.89it/s]\u001b[A\n",
            " 64% 161/250 [00:56<00:30,  2.88it/s]\u001b[A\n",
            " 65% 162/250 [00:57<00:30,  2.93it/s]\u001b[A\n",
            " 65% 163/250 [00:57<00:29,  2.95it/s]\u001b[A\n",
            " 66% 164/250 [00:57<00:29,  2.92it/s]\u001b[A\n",
            " 66% 165/250 [00:58<00:29,  2.91it/s]\u001b[A\n",
            " 66% 166/250 [00:58<00:29,  2.81it/s]\u001b[A\n",
            " 67% 167/250 [00:59<00:28,  2.93it/s]\u001b[A\n",
            " 67% 168/250 [00:59<00:28,  2.90it/s]\u001b[A\n",
            " 68% 169/250 [00:59<00:30,  2.70it/s]\u001b[A\n",
            " 68% 170/250 [01:00<00:29,  2.71it/s]\u001b[A\n",
            " 68% 171/250 [01:00<00:27,  2.86it/s]\u001b[A\n",
            " 69% 172/250 [01:00<00:26,  2.95it/s]\u001b[A\n",
            " 69% 173/250 [01:01<00:26,  2.94it/s]\u001b[A\n",
            " 70% 174/250 [01:01<00:25,  2.96it/s]\u001b[A\n",
            " 70% 175/250 [01:01<00:25,  3.00it/s]\u001b[A\n",
            " 70% 176/250 [01:02<00:24,  2.99it/s]\u001b[A\n",
            " 71% 177/250 [01:02<00:24,  2.99it/s]\u001b[A\n",
            " 71% 178/250 [01:02<00:24,  2.98it/s]\u001b[A\n",
            " 72% 179/250 [01:03<00:23,  3.03it/s]\u001b[A\n",
            " 72% 180/250 [01:03<00:23,  2.95it/s]\u001b[A\n",
            " 72% 181/250 [01:03<00:22,  3.04it/s]\u001b[A\n",
            " 73% 182/250 [01:04<00:22,  3.06it/s]\u001b[A\n",
            " 73% 183/250 [01:04<00:22,  3.00it/s]\u001b[A\n",
            " 74% 184/250 [01:04<00:22,  2.95it/s]\u001b[A\n",
            " 74% 185/250 [01:05<00:21,  2.96it/s]\u001b[A\n",
            " 74% 186/250 [01:05<00:21,  3.03it/s]\u001b[A\n",
            " 75% 187/250 [01:05<00:20,  3.03it/s]\u001b[A\n",
            " 75% 188/250 [01:06<00:20,  3.03it/s]\u001b[A\n",
            " 76% 189/250 [01:06<00:20,  3.02it/s]\u001b[A\n",
            " 76% 190/250 [01:06<00:19,  3.06it/s]\u001b[A\n",
            " 76% 191/250 [01:07<00:19,  3.08it/s]\u001b[A\n",
            " 77% 192/250 [01:07<00:19,  2.95it/s]\u001b[A\n",
            " 77% 193/250 [01:07<00:19,  2.96it/s]\u001b[A\n",
            " 78% 194/250 [01:08<00:18,  3.05it/s]\u001b[A\n",
            " 78% 195/250 [01:08<00:18,  3.05it/s]\u001b[A\n",
            " 78% 196/250 [01:08<00:17,  3.09it/s]\u001b[A\n",
            " 79% 197/250 [01:09<00:17,  3.00it/s]\u001b[A\n",
            " 79% 198/250 [01:09<00:18,  2.88it/s]\u001b[A\n",
            " 80% 199/250 [01:09<00:17,  2.95it/s]\u001b[A\n",
            " 80% 200/250 [01:10<00:17,  2.91it/s]\u001b[A\n",
            " 80% 201/250 [01:10<00:16,  2.89it/s]\u001b[A\n",
            " 81% 202/250 [01:10<00:16,  2.91it/s]\u001b[A\n",
            " 81% 203/250 [01:11<00:16,  2.92it/s]\u001b[A\n",
            " 82% 204/250 [01:11<00:16,  2.77it/s]\u001b[A\n",
            " 82% 205/250 [01:11<00:15,  2.81it/s]\u001b[A\n",
            " 82% 206/250 [01:12<00:15,  2.85it/s]\u001b[A\n",
            " 83% 207/250 [01:12<00:15,  2.84it/s]\u001b[A\n",
            " 83% 208/250 [01:12<00:14,  2.85it/s]\u001b[A\n",
            " 84% 209/250 [01:13<00:14,  2.82it/s]\u001b[A\n",
            " 84% 210/250 [01:13<00:13,  2.91it/s]\u001b[A\n",
            " 84% 211/250 [01:13<00:13,  2.96it/s]\u001b[A\n",
            " 85% 212/250 [01:14<00:12,  3.03it/s]\u001b[A\n",
            " 85% 213/250 [01:14<00:12,  2.95it/s]\u001b[A\n",
            " 86% 214/250 [01:14<00:12,  2.99it/s]\u001b[A\n",
            " 86% 215/250 [01:15<00:11,  2.99it/s]\u001b[A\n",
            " 86% 216/250 [01:15<00:11,  3.00it/s]\u001b[A\n",
            " 87% 217/250 [01:15<00:11,  2.95it/s]\u001b[A\n",
            " 87% 218/250 [01:16<00:10,  2.92it/s]\u001b[A\n",
            " 88% 219/250 [01:16<00:10,  3.01it/s]\u001b[A\n",
            " 88% 220/250 [01:16<00:09,  3.09it/s]\u001b[A\n",
            " 88% 221/250 [01:17<00:09,  3.11it/s]\u001b[A\n",
            " 89% 222/250 [01:17<00:09,  2.95it/s]\u001b[A\n",
            " 89% 223/250 [01:17<00:09,  2.91it/s]\u001b[A\n",
            " 90% 224/250 [01:18<00:08,  3.01it/s]\u001b[A\n",
            " 90% 225/250 [01:18<00:08,  2.93it/s]\u001b[A\n",
            " 90% 226/250 [01:18<00:08,  2.93it/s]\u001b[A\n",
            " 91% 227/250 [01:19<00:07,  2.99it/s]\u001b[A\n",
            " 91% 228/250 [01:19<00:07,  2.97it/s]\u001b[A\n",
            " 92% 229/250 [01:20<00:07,  2.75it/s]\u001b[A\n",
            " 92% 230/250 [01:20<00:07,  2.81it/s]\u001b[A\n",
            " 92% 231/250 [01:20<00:07,  2.68it/s]\u001b[A\n",
            " 93% 232/250 [01:21<00:07,  2.33it/s]\u001b[A\n",
            " 93% 233/250 [01:21<00:06,  2.47it/s]\u001b[A\n",
            " 94% 234/250 [01:22<00:06,  2.64it/s]\u001b[A\n",
            " 94% 235/250 [01:22<00:05,  2.78it/s]\u001b[A\n",
            " 94% 236/250 [01:22<00:05,  2.38it/s]\u001b[A\n",
            " 95% 237/250 [01:23<00:05,  2.43it/s]\u001b[A\n",
            " 95% 238/250 [01:23<00:04,  2.42it/s]\u001b[A\n",
            " 96% 239/250 [01:24<00:04,  2.48it/s]\u001b[A\n",
            " 96% 240/250 [01:24<00:03,  2.58it/s]\u001b[A\n",
            " 96% 241/250 [01:24<00:03,  2.63it/s]\u001b[A\n",
            " 97% 242/250 [01:25<00:03,  2.63it/s]\u001b[A\n",
            " 97% 243/250 [01:25<00:02,  2.81it/s]\u001b[A\n",
            " 98% 244/250 [01:25<00:02,  2.88it/s]\u001b[A\n",
            " 98% 245/250 [01:26<00:01,  3.01it/s]\u001b[A\n",
            " 98% 246/250 [01:26<00:01,  3.04it/s]\u001b[A\n",
            " 99% 247/250 [01:26<00:00,  3.10it/s]\u001b[A\n",
            " 99% 248/250 [01:27<00:00,  2.99it/s]\u001b[A\n",
            "100% 249/250 [01:27<00:00,  2.87it/s]\u001b[A\n",
            "100% 250/250 [01:27<00:00,  2.89it/s]\u001b[A\n",
            "{'eval_loss': 0.906471848487854, 'eval_runtime': 88.2821, 'eval_samples_per_second': 22.655, 'eval_steps_per_second': 2.832, 'epoch': 0.8}\n",
            "\n",
            " 27% 400/1500 [37:59<53:56,  2.94s/it]\n",
            "{'loss': 1.0414, 'learning_rate': 7.785714285714286e-05, 'epoch': 0.82}\n",
            "{'loss': 0.9337, 'learning_rate': 7.714285714285715e-05, 'epoch': 0.84}\n",
            "{'loss': 0.8203, 'learning_rate': 7.642857142857143e-05, 'epoch': 0.86}\n",
            "{'loss': 0.6737, 'learning_rate': 7.571428571428571e-05, 'epoch': 0.88}\n",
            "{'loss': 0.6239, 'learning_rate': 7.500000000000001e-05, 'epoch': 0.9}\n",
            "{'loss': 1.0446, 'learning_rate': 7.428571428571429e-05, 'epoch': 0.92}\n",
            "{'loss': 0.9406, 'learning_rate': 7.357142857142858e-05, 'epoch': 0.94}\n",
            "{'loss': 0.81, 'learning_rate': 7.285714285714286e-05, 'epoch': 0.96}\n",
            "{'loss': 0.6719, 'learning_rate': 7.214285714285714e-05, 'epoch': 0.98}\n",
            "{'loss': 0.6235, 'learning_rate': 7.142857142857143e-05, 'epoch': 1.0}\n",
            "{'loss': 1.0333, 'learning_rate': 7.071428571428573e-05, 'epoch': 1.02}\n",
            "{'loss': 0.9397, 'learning_rate': 7e-05, 'epoch': 1.04}\n",
            "{'loss': 0.8164, 'learning_rate': 6.928571428571429e-05, 'epoch': 1.06}\n",
            "{'loss': 0.671, 'learning_rate': 6.857142857142858e-05, 'epoch': 1.08}\n",
            "{'loss': 0.6283, 'learning_rate': 6.785714285714286e-05, 'epoch': 1.1}\n",
            "{'loss': 1.0111, 'learning_rate': 6.714285714285714e-05, 'epoch': 1.12}\n",
            "{'loss': 0.9314, 'learning_rate': 6.642857142857143e-05, 'epoch': 1.14}\n",
            "{'loss': 0.8177, 'learning_rate': 6.571428571428571e-05, 'epoch': 1.16}\n",
            "{'loss': 0.6798, 'learning_rate': 6.500000000000001e-05, 'epoch': 1.18}\n",
            "{'loss': 0.6207, 'learning_rate': 6.428571428571429e-05, 'epoch': 1.2}\n",
            " 40% 600/1500 [55:32<45:17,  3.02s/it]\n",
            "  0% 0/250 [00:00<?, ?it/s]\u001b[A\n",
            "  1% 2/250 [00:00<00:39,  6.24it/s]\u001b[A\n",
            "  1% 3/250 [00:00<00:55,  4.49it/s]\u001b[A\n",
            "  2% 4/250 [00:01<01:09,  3.56it/s]\u001b[A\n",
            "  2% 5/250 [00:01<01:14,  3.30it/s]\u001b[A\n",
            "  2% 6/250 [00:01<01:14,  3.27it/s]\u001b[A\n",
            "  3% 7/250 [00:01<01:13,  3.30it/s]\u001b[A\n",
            "  3% 8/250 [00:02<01:17,  3.10it/s]\u001b[A\n",
            "  4% 9/250 [00:02<01:21,  2.95it/s]\u001b[A\n",
            "  4% 10/250 [00:03<01:27,  2.74it/s]\u001b[A\n",
            "  4% 11/250 [00:03<01:24,  2.83it/s]\u001b[A\n",
            "  5% 12/250 [00:03<01:23,  2.84it/s]\u001b[A\n",
            "  5% 13/250 [00:04<01:20,  2.94it/s]\u001b[A\n",
            "  6% 14/250 [00:04<01:18,  3.01it/s]\u001b[A\n",
            "  6% 15/250 [00:04<01:19,  2.96it/s]\u001b[A\n",
            "  6% 16/250 [00:05<01:17,  3.02it/s]\u001b[A\n",
            "  7% 17/250 [00:05<01:17,  2.99it/s]\u001b[A\n",
            "  7% 18/250 [00:05<01:18,  2.95it/s]\u001b[A\n",
            "  8% 19/250 [00:06<01:18,  2.95it/s]\u001b[A\n",
            "  8% 20/250 [00:06<01:19,  2.91it/s]\u001b[A\n",
            "  8% 21/250 [00:06<01:17,  2.95it/s]\u001b[A\n",
            "  9% 22/250 [00:07<01:17,  2.92it/s]\u001b[A\n",
            "  9% 23/250 [00:07<01:14,  3.03it/s]\u001b[A\n",
            " 10% 24/250 [00:07<01:16,  2.95it/s]\u001b[A\n",
            " 10% 25/250 [00:08<01:24,  2.68it/s]\u001b[A\n",
            " 10% 26/250 [00:08<01:24,  2.64it/s]\u001b[A\n",
            " 11% 27/250 [00:08<01:20,  2.76it/s]\u001b[A\n",
            " 11% 28/250 [00:09<01:17,  2.88it/s]\u001b[A\n",
            " 12% 29/250 [00:09<01:15,  2.93it/s]\u001b[A\n",
            " 12% 30/250 [00:09<01:14,  2.95it/s]\u001b[A\n",
            " 12% 31/250 [00:10<01:12,  3.03it/s]\u001b[A\n",
            " 13% 32/250 [00:10<01:11,  3.05it/s]\u001b[A\n",
            " 13% 33/250 [00:10<01:12,  3.00it/s]\u001b[A\n",
            " 14% 34/250 [00:11<01:10,  3.06it/s]\u001b[A\n",
            " 14% 35/250 [00:11<01:09,  3.09it/s]\u001b[A\n",
            " 14% 36/250 [00:11<01:14,  2.88it/s]\u001b[A\n",
            " 15% 37/250 [00:12<01:11,  2.96it/s]\u001b[A\n",
            " 15% 38/250 [00:12<01:10,  3.01it/s]\u001b[A\n",
            " 16% 39/250 [00:12<01:11,  2.96it/s]\u001b[A\n",
            " 16% 40/250 [00:13<01:15,  2.79it/s]\u001b[A\n",
            " 16% 41/250 [00:13<01:14,  2.82it/s]\u001b[A\n",
            " 17% 42/250 [00:14<01:13,  2.82it/s]\u001b[A\n",
            " 17% 43/250 [00:14<01:15,  2.75it/s]\u001b[A\n",
            " 18% 44/250 [00:14<01:11,  2.87it/s]\u001b[A\n",
            " 18% 45/250 [00:15<01:12,  2.83it/s]\u001b[A\n",
            " 18% 46/250 [00:15<01:10,  2.91it/s]\u001b[A\n",
            " 19% 47/250 [00:15<01:12,  2.82it/s]\u001b[A\n",
            " 19% 48/250 [00:16<01:09,  2.89it/s]\u001b[A\n",
            " 20% 49/250 [00:16<01:08,  2.95it/s]\u001b[A\n",
            " 20% 50/250 [00:16<01:08,  2.91it/s]\u001b[A\n",
            " 20% 51/250 [00:17<01:06,  2.98it/s]\u001b[A\n",
            " 21% 52/250 [00:17<01:05,  3.00it/s]\u001b[A\n",
            " 21% 53/250 [00:17<01:08,  2.87it/s]\u001b[A\n",
            " 22% 54/250 [00:18<01:08,  2.86it/s]\u001b[A\n",
            " 22% 55/250 [00:18<01:06,  2.94it/s]\u001b[A\n",
            " 22% 56/250 [00:18<01:06,  2.93it/s]\u001b[A\n",
            " 23% 57/250 [00:19<01:06,  2.90it/s]\u001b[A\n",
            " 23% 58/250 [00:19<01:04,  2.99it/s]\u001b[A\n",
            " 24% 59/250 [00:19<01:05,  2.91it/s]\u001b[A\n",
            " 24% 60/250 [00:20<01:15,  2.51it/s]\u001b[A\n",
            " 24% 61/250 [00:20<01:11,  2.65it/s]\u001b[A\n",
            " 25% 62/250 [00:21<01:13,  2.54it/s]\u001b[A\n",
            " 25% 63/250 [00:21<01:08,  2.71it/s]\u001b[A\n",
            " 26% 64/250 [00:21<01:07,  2.76it/s]\u001b[A\n",
            " 26% 65/250 [00:22<01:08,  2.70it/s]\u001b[A\n",
            " 26% 66/250 [00:22<01:07,  2.73it/s]\u001b[A\n",
            " 27% 67/250 [00:22<01:05,  2.78it/s]\u001b[A\n",
            " 27% 68/250 [00:23<01:02,  2.90it/s]\u001b[A\n",
            " 28% 69/250 [00:23<01:11,  2.53it/s]\u001b[A\n",
            " 28% 70/250 [00:24<01:10,  2.54it/s]\u001b[A\n",
            " 28% 71/250 [00:24<01:06,  2.69it/s]\u001b[A\n",
            " 29% 72/250 [00:24<01:03,  2.80it/s]\u001b[A\n",
            " 29% 73/250 [00:25<01:02,  2.85it/s]\u001b[A\n",
            " 30% 74/250 [00:25<01:09,  2.55it/s]\u001b[A\n",
            " 30% 75/250 [00:25<01:05,  2.66it/s]\u001b[A\n",
            " 30% 76/250 [00:26<01:04,  2.72it/s]\u001b[A\n",
            " 31% 77/250 [00:26<01:00,  2.86it/s]\u001b[A\n",
            " 31% 78/250 [00:26<00:58,  2.95it/s]\u001b[A\n",
            " 32% 79/250 [00:27<01:03,  2.71it/s]\u001b[A\n",
            " 32% 80/250 [00:27<01:05,  2.59it/s]\u001b[A\n",
            " 32% 81/250 [00:28<01:01,  2.76it/s]\u001b[A\n",
            " 33% 82/250 [00:28<00:58,  2.89it/s]\u001b[A\n",
            " 33% 83/250 [00:28<01:00,  2.75it/s]\u001b[A\n",
            " 34% 84/250 [00:29<00:59,  2.77it/s]\u001b[A\n",
            " 34% 85/250 [00:29<00:58,  2.84it/s]\u001b[A\n",
            " 34% 86/250 [00:29<01:03,  2.59it/s]\u001b[A\n",
            " 35% 87/250 [00:30<01:00,  2.68it/s]\u001b[A\n",
            " 35% 88/250 [00:30<01:02,  2.59it/s]\u001b[A\n",
            " 36% 89/250 [00:31<00:58,  2.73it/s]\u001b[A\n",
            " 36% 90/250 [00:31<01:08,  2.35it/s]\u001b[A\n",
            " 36% 91/250 [00:31<01:03,  2.51it/s]\u001b[A\n",
            " 37% 92/250 [00:32<00:59,  2.64it/s]\u001b[A\n",
            " 37% 93/250 [00:32<01:01,  2.57it/s]\u001b[A\n",
            " 38% 94/250 [00:33<00:59,  2.62it/s]\u001b[A\n",
            " 38% 95/250 [00:33<00:59,  2.59it/s]\u001b[A\n",
            " 38% 96/250 [00:34<01:07,  2.28it/s]\u001b[A\n",
            " 39% 97/250 [00:34<01:01,  2.48it/s]\u001b[A\n",
            " 39% 98/250 [00:34<00:57,  2.65it/s]\u001b[A\n",
            " 40% 99/250 [00:34<00:53,  2.80it/s]\u001b[A\n",
            " 40% 100/250 [00:35<00:52,  2.86it/s]\u001b[A\n",
            " 40% 101/250 [00:35<00:51,  2.91it/s]\u001b[A\n",
            " 41% 102/250 [00:35<00:51,  2.89it/s]\u001b[A\n",
            " 41% 103/250 [00:36<00:51,  2.85it/s]\u001b[A\n",
            " 42% 104/250 [00:36<00:50,  2.87it/s]\u001b[A\n",
            " 42% 105/250 [00:36<00:48,  2.97it/s]\u001b[A\n",
            " 42% 106/250 [00:37<00:55,  2.58it/s]\u001b[A\n",
            " 43% 107/250 [00:37<00:58,  2.44it/s]\u001b[A\n",
            " 43% 108/250 [00:38<00:54,  2.61it/s]\u001b[A\n",
            " 44% 109/250 [00:38<00:51,  2.75it/s]\u001b[A\n",
            " 44% 110/250 [00:38<00:50,  2.75it/s]\u001b[A\n",
            " 44% 111/250 [00:39<00:49,  2.83it/s]\u001b[A\n",
            " 45% 112/250 [00:39<00:50,  2.72it/s]\u001b[A\n",
            " 45% 113/250 [00:40<00:48,  2.82it/s]\u001b[A\n",
            " 46% 114/250 [00:40<00:46,  2.92it/s]\u001b[A\n",
            " 46% 115/250 [00:40<00:47,  2.82it/s]\u001b[A\n",
            " 46% 116/250 [00:41<00:50,  2.66it/s]\u001b[A\n",
            " 47% 117/250 [00:41<00:48,  2.74it/s]\u001b[A\n",
            " 47% 118/250 [00:41<00:49,  2.67it/s]\u001b[A\n",
            " 48% 119/250 [00:42<00:47,  2.75it/s]\u001b[A\n",
            " 48% 120/250 [00:42<00:46,  2.78it/s]\u001b[A\n",
            " 48% 121/250 [00:42<00:45,  2.81it/s]\u001b[A\n",
            " 49% 122/250 [00:43<00:44,  2.89it/s]\u001b[A\n",
            " 49% 123/250 [00:43<00:42,  2.99it/s]\u001b[A\n",
            " 50% 124/250 [00:43<00:41,  3.06it/s]\u001b[A\n",
            " 50% 125/250 [00:44<00:40,  3.10it/s]\u001b[A\n",
            " 50% 126/250 [00:44<00:39,  3.11it/s]\u001b[A\n",
            " 51% 127/250 [00:44<00:40,  3.01it/s]\u001b[A\n",
            " 51% 128/250 [00:45<00:49,  2.49it/s]\u001b[A\n",
            " 52% 129/250 [00:45<00:45,  2.64it/s]\u001b[A\n",
            " 52% 130/250 [00:46<00:46,  2.56it/s]\u001b[A\n",
            " 52% 131/250 [00:46<00:46,  2.55it/s]\u001b[A\n",
            " 53% 132/250 [00:46<00:45,  2.62it/s]\u001b[A\n",
            " 53% 133/250 [00:47<00:44,  2.64it/s]\u001b[A\n",
            " 54% 134/250 [00:47<00:44,  2.63it/s]\u001b[A\n",
            " 54% 135/250 [00:47<00:42,  2.71it/s]\u001b[A\n",
            " 54% 136/250 [00:48<00:40,  2.84it/s]\u001b[A\n",
            " 55% 137/250 [00:48<00:41,  2.75it/s]\u001b[A\n",
            " 55% 138/250 [00:49<00:38,  2.88it/s]\u001b[A\n",
            " 56% 139/250 [00:49<00:40,  2.73it/s]\u001b[A\n",
            " 56% 140/250 [00:49<00:39,  2.81it/s]\u001b[A\n",
            " 56% 141/250 [00:50<00:37,  2.90it/s]\u001b[A\n",
            " 57% 142/250 [00:50<00:43,  2.49it/s]\u001b[A\n",
            " 57% 143/250 [00:50<00:42,  2.52it/s]\u001b[A\n",
            " 58% 144/250 [00:51<00:40,  2.65it/s]\u001b[A\n",
            " 58% 145/250 [00:51<00:38,  2.71it/s]\u001b[A\n",
            " 58% 146/250 [00:51<00:36,  2.84it/s]\u001b[A\n",
            " 59% 147/250 [00:52<00:35,  2.89it/s]\u001b[A\n",
            " 59% 148/250 [00:52<00:34,  2.98it/s]\u001b[A\n",
            " 60% 149/250 [00:52<00:34,  2.90it/s]\u001b[A\n",
            " 60% 150/250 [00:53<00:33,  2.96it/s]\u001b[A\n",
            " 60% 151/250 [00:53<00:33,  2.93it/s]\u001b[A\n",
            " 61% 152/250 [00:54<00:34,  2.84it/s]\u001b[A\n",
            " 61% 153/250 [00:54<00:33,  2.90it/s]\u001b[A\n",
            " 62% 154/250 [00:54<00:33,  2.86it/s]\u001b[A\n",
            " 62% 155/250 [00:55<00:31,  2.99it/s]\u001b[A\n",
            " 62% 156/250 [00:55<00:34,  2.74it/s]\u001b[A\n",
            " 63% 157/250 [00:55<00:33,  2.77it/s]\u001b[A\n",
            " 63% 158/250 [00:56<00:33,  2.71it/s]\u001b[A\n",
            " 64% 159/250 [00:56<00:31,  2.86it/s]\u001b[A\n",
            " 64% 160/250 [00:56<00:31,  2.88it/s]\u001b[A\n",
            " 64% 161/250 [00:57<00:30,  2.88it/s]\u001b[A\n",
            " 65% 162/250 [00:57<00:30,  2.93it/s]\u001b[A\n",
            " 65% 163/250 [00:57<00:29,  2.94it/s]\u001b[A\n",
            " 66% 164/250 [00:58<00:29,  2.91it/s]\u001b[A\n",
            " 66% 165/250 [00:58<00:29,  2.90it/s]\u001b[A\n",
            " 66% 166/250 [00:58<00:30,  2.80it/s]\u001b[A\n",
            " 67% 167/250 [00:59<00:28,  2.92it/s]\u001b[A\n",
            " 67% 168/250 [00:59<00:28,  2.90it/s]\u001b[A\n",
            " 68% 169/250 [01:00<00:29,  2.70it/s]\u001b[A\n",
            " 68% 170/250 [01:00<00:29,  2.73it/s]\u001b[A\n",
            " 68% 171/250 [01:00<00:27,  2.87it/s]\u001b[A\n",
            " 69% 172/250 [01:01<00:26,  2.94it/s]\u001b[A\n",
            " 69% 173/250 [01:01<00:26,  2.93it/s]\u001b[A\n",
            " 70% 174/250 [01:01<00:25,  2.98it/s]\u001b[A\n",
            " 70% 175/250 [01:01<00:24,  3.03it/s]\u001b[A\n",
            " 70% 176/250 [01:02<00:24,  3.01it/s]\u001b[A\n",
            " 71% 177/250 [01:02<00:24,  3.02it/s]\u001b[A\n",
            " 71% 178/250 [01:03<00:24,  2.98it/s]\u001b[A\n",
            " 72% 179/250 [01:03<00:23,  3.03it/s]\u001b[A\n",
            " 72% 180/250 [01:03<00:23,  2.94it/s]\u001b[A\n",
            " 72% 181/250 [01:04<00:22,  3.02it/s]\u001b[A\n",
            " 73% 182/250 [01:04<00:22,  3.03it/s]\u001b[A\n",
            " 73% 183/250 [01:04<00:22,  2.97it/s]\u001b[A\n",
            " 74% 184/250 [01:05<00:22,  2.92it/s]\u001b[A\n",
            " 74% 185/250 [01:05<00:22,  2.91it/s]\u001b[A\n",
            " 74% 186/250 [01:05<00:21,  2.99it/s]\u001b[A\n",
            " 75% 187/250 [01:06<00:21,  2.98it/s]\u001b[A\n",
            " 75% 188/250 [01:06<00:20,  2.98it/s]\u001b[A\n",
            " 76% 189/250 [01:06<00:20,  2.97it/s]\u001b[A\n",
            " 76% 190/250 [01:07<00:19,  3.01it/s]\u001b[A\n",
            " 76% 191/250 [01:07<00:19,  3.04it/s]\u001b[A\n",
            " 77% 192/250 [01:07<00:19,  2.91it/s]\u001b[A\n",
            " 77% 193/250 [01:08<00:19,  2.92it/s]\u001b[A\n",
            " 78% 194/250 [01:08<00:18,  2.98it/s]\u001b[A\n",
            " 78% 195/250 [01:08<00:18,  3.01it/s]\u001b[A\n",
            " 78% 196/250 [01:09<00:17,  3.06it/s]\u001b[A\n",
            " 79% 197/250 [01:09<00:17,  3.00it/s]\u001b[A\n",
            " 79% 198/250 [01:09<00:18,  2.89it/s]\u001b[A\n",
            " 80% 199/250 [01:10<00:17,  2.96it/s]\u001b[A\n",
            " 80% 200/250 [01:10<00:17,  2.92it/s]\u001b[A\n",
            " 80% 201/250 [01:10<00:16,  2.91it/s]\u001b[A\n",
            " 81% 202/250 [01:11<00:16,  2.93it/s]\u001b[A\n",
            " 81% 203/250 [01:11<00:16,  2.93it/s]\u001b[A\n",
            " 82% 204/250 [01:11<00:16,  2.78it/s]\u001b[A\n",
            " 82% 205/250 [01:12<00:15,  2.82it/s]\u001b[A\n",
            " 82% 206/250 [01:12<00:15,  2.87it/s]\u001b[A\n",
            " 83% 207/250 [01:12<00:15,  2.84it/s]\u001b[A\n",
            " 83% 208/250 [01:13<00:14,  2.87it/s]\u001b[A\n",
            " 84% 209/250 [01:13<00:14,  2.85it/s]\u001b[A\n",
            " 84% 210/250 [01:13<00:13,  2.95it/s]\u001b[A\n",
            " 84% 211/250 [01:14<00:13,  2.99it/s]\u001b[A\n",
            " 85% 212/250 [01:14<00:12,  3.07it/s]\u001b[A\n",
            " 85% 213/250 [01:14<00:12,  3.00it/s]\u001b[A\n",
            " 86% 214/250 [01:15<00:11,  3.04it/s]\u001b[A\n",
            " 86% 215/250 [01:15<00:11,  3.05it/s]\u001b[A\n",
            " 86% 216/250 [01:15<00:11,  3.04it/s]\u001b[A\n",
            " 87% 217/250 [01:16<00:11,  2.98it/s]\u001b[A\n",
            " 87% 218/250 [01:16<00:10,  2.93it/s]\u001b[A\n",
            " 88% 219/250 [01:16<00:10,  3.02it/s]\u001b[A\n",
            " 88% 220/250 [01:17<00:09,  3.09it/s]\u001b[A\n",
            " 88% 221/250 [01:17<00:09,  3.11it/s]\u001b[A\n",
            " 89% 222/250 [01:17<00:09,  2.95it/s]\u001b[A\n",
            " 89% 223/250 [01:18<00:09,  2.91it/s]\u001b[A\n",
            " 90% 224/250 [01:18<00:08,  3.01it/s]\u001b[A\n",
            " 90% 225/250 [01:18<00:08,  2.92it/s]\u001b[A\n",
            " 90% 226/250 [01:19<00:08,  2.91it/s]\u001b[A\n",
            " 91% 227/250 [01:19<00:07,  2.98it/s]\u001b[A\n",
            " 91% 228/250 [01:19<00:07,  2.95it/s]\u001b[A\n",
            " 92% 229/250 [01:20<00:07,  2.74it/s]\u001b[A\n",
            " 92% 230/250 [01:20<00:07,  2.79it/s]\u001b[A\n",
            " 92% 231/250 [01:21<00:07,  2.66it/s]\u001b[A\n",
            " 93% 232/250 [01:21<00:07,  2.32it/s]\u001b[A\n",
            " 93% 233/250 [01:22<00:06,  2.47it/s]\u001b[A\n",
            " 94% 234/250 [01:22<00:06,  2.64it/s]\u001b[A\n",
            " 94% 235/250 [01:22<00:05,  2.77it/s]\u001b[A\n",
            " 94% 236/250 [01:23<00:05,  2.37it/s]\u001b[A\n",
            " 95% 237/250 [01:23<00:05,  2.42it/s]\u001b[A\n",
            " 95% 238/250 [01:24<00:04,  2.41it/s]\u001b[A\n",
            " 96% 239/250 [01:24<00:04,  2.48it/s]\u001b[A\n",
            " 96% 240/250 [01:24<00:03,  2.58it/s]\u001b[A\n",
            " 96% 241/250 [01:25<00:03,  2.63it/s]\u001b[A\n",
            " 97% 242/250 [01:25<00:03,  2.63it/s]\u001b[A\n",
            " 97% 243/250 [01:25<00:02,  2.82it/s]\u001b[A\n",
            " 98% 244/250 [01:26<00:02,  2.89it/s]\u001b[A\n",
            " 98% 245/250 [01:26<00:01,  3.01it/s]\u001b[A\n",
            " 98% 246/250 [01:26<00:01,  3.03it/s]\u001b[A\n",
            " 99% 247/250 [01:27<00:00,  3.08it/s]\u001b[A\n",
            " 99% 248/250 [01:27<00:00,  2.97it/s]\u001b[A\n",
            "100% 249/250 [01:27<00:00,  2.85it/s]\u001b[A\n",
            "100% 250/250 [01:28<00:00,  2.87it/s]\u001b[A\n",
            "{'eval_loss': 0.8862542510032654, 'eval_runtime': 88.5659, 'eval_samples_per_second': 22.582, 'eval_steps_per_second': 2.823, 'epoch': 1.2}\n",
            "\n",
            " 40% 600/1500 [57:01<45:17,  3.02s/it]\n",
            "{'loss': 1.001, 'learning_rate': 6.357142857142857e-05, 'epoch': 1.22}\n",
            "{'loss': 0.9437, 'learning_rate': 6.285714285714286e-05, 'epoch': 1.24}\n",
            "{'loss': 0.8046, 'learning_rate': 6.214285714285714e-05, 'epoch': 1.26}\n",
            "{'loss': 0.6589, 'learning_rate': 6.142857142857143e-05, 'epoch': 1.28}\n",
            "{'loss': 0.6158, 'learning_rate': 6.0714285714285715e-05, 'epoch': 1.3}\n",
            "{'loss': 0.9993, 'learning_rate': 6e-05, 'epoch': 1.32}\n",
            "{'loss': 0.9486, 'learning_rate': 5.928571428571429e-05, 'epoch': 1.34}\n",
            "{'loss': 0.8142, 'learning_rate': 5.8571428571428575e-05, 'epoch': 1.36}\n",
            "{'loss': 0.6897, 'learning_rate': 5.785714285714287e-05, 'epoch': 1.38}\n",
            "{'loss': 0.6204, 'learning_rate': 5.714285714285714e-05, 'epoch': 1.4}\n",
            "{'loss': 1.0103, 'learning_rate': 5.642857142857143e-05, 'epoch': 1.42}\n",
            "{'loss': 0.9259, 'learning_rate': 5.571428571428572e-05, 'epoch': 1.44}\n",
            "{'loss': 0.806, 'learning_rate': 5.500000000000001e-05, 'epoch': 1.46}\n",
            "{'loss': 0.6663, 'learning_rate': 5.428571428571428e-05, 'epoch': 1.48}\n",
            "{'loss': 0.6164, 'learning_rate': 5.3571428571428575e-05, 'epoch': 1.5}\n",
            "{'loss': 1.024, 'learning_rate': 5.285714285714286e-05, 'epoch': 1.52}\n",
            "{'loss': 0.9313, 'learning_rate': 5.214285714285715e-05, 'epoch': 1.54}\n",
            "{'loss': 0.8058, 'learning_rate': 5.142857142857143e-05, 'epoch': 1.56}\n",
            "{'loss': 0.6617, 'learning_rate': 5.0714285714285716e-05, 'epoch': 1.58}\n",
            "{'loss': 0.6182, 'learning_rate': 5e-05, 'epoch': 1.6}\n",
            " 53% 800/1500 [1:14:29<34:52,  2.99s/it]\n",
            "  0% 0/250 [00:00<?, ?it/s]\u001b[A\n",
            "  1% 2/250 [00:00<00:39,  6.24it/s]\u001b[A\n",
            "  1% 3/250 [00:00<00:54,  4.49it/s]\u001b[A\n",
            "  2% 4/250 [00:01<01:09,  3.56it/s]\u001b[A\n",
            "  2% 5/250 [00:01<01:14,  3.29it/s]\u001b[A\n",
            "  2% 6/250 [00:01<01:14,  3.26it/s]\u001b[A\n",
            "  3% 7/250 [00:01<01:14,  3.27it/s]\u001b[A\n",
            "  3% 8/250 [00:02<01:18,  3.08it/s]\u001b[A\n",
            "  4% 9/250 [00:02<01:22,  2.94it/s]\u001b[A\n",
            "  4% 10/250 [00:03<01:28,  2.72it/s]\u001b[A\n",
            "  4% 11/250 [00:03<01:24,  2.82it/s]\u001b[A\n",
            "  5% 12/250 [00:03<01:24,  2.83it/s]\u001b[A\n",
            "  5% 13/250 [00:04<01:20,  2.96it/s]\u001b[A\n",
            "  6% 14/250 [00:04<01:17,  3.03it/s]\u001b[A\n",
            "  6% 15/250 [00:04<01:18,  2.98it/s]\u001b[A\n",
            "  6% 16/250 [00:05<01:17,  3.03it/s]\u001b[A\n",
            "  7% 17/250 [00:05<01:17,  3.00it/s]\u001b[A\n",
            "  7% 18/250 [00:05<01:17,  2.98it/s]\u001b[A\n",
            "  8% 19/250 [00:06<01:17,  2.97it/s]\u001b[A\n",
            "  8% 20/250 [00:06<01:18,  2.94it/s]\u001b[A\n",
            "  8% 21/250 [00:06<01:16,  2.98it/s]\u001b[A\n",
            "  9% 22/250 [00:07<01:16,  2.96it/s]\u001b[A\n",
            "  9% 23/250 [00:07<01:14,  3.05it/s]\u001b[A\n",
            " 10% 24/250 [00:07<01:16,  2.97it/s]\u001b[A\n",
            " 10% 25/250 [00:08<01:23,  2.70it/s]\u001b[A\n",
            " 10% 26/250 [00:08<01:24,  2.64it/s]\u001b[A\n",
            " 11% 27/250 [00:08<01:20,  2.78it/s]\u001b[A\n",
            " 11% 28/250 [00:09<01:16,  2.91it/s]\u001b[A\n",
            " 12% 29/250 [00:09<01:14,  2.98it/s]\u001b[A\n",
            " 12% 30/250 [00:09<01:13,  3.00it/s]\u001b[A\n",
            " 12% 31/250 [00:10<01:10,  3.09it/s]\u001b[A\n",
            " 13% 32/250 [00:10<01:09,  3.12it/s]\u001b[A\n",
            " 13% 33/250 [00:10<01:10,  3.07it/s]\u001b[A\n",
            " 14% 34/250 [00:11<01:08,  3.15it/s]\u001b[A\n",
            " 14% 35/250 [00:11<01:08,  3.16it/s]\u001b[A\n",
            " 14% 36/250 [00:11<01:13,  2.92it/s]\u001b[A\n",
            " 15% 37/250 [00:12<01:11,  2.97it/s]\u001b[A\n",
            " 15% 38/250 [00:12<01:10,  3.03it/s]\u001b[A\n",
            " 16% 39/250 [00:12<01:10,  2.97it/s]\u001b[A\n",
            " 16% 40/250 [00:13<01:14,  2.80it/s]\u001b[A\n",
            " 16% 41/250 [00:13<01:13,  2.83it/s]\u001b[A\n",
            " 17% 42/250 [00:13<01:13,  2.83it/s]\u001b[A\n",
            " 17% 43/250 [00:14<01:14,  2.79it/s]\u001b[A\n",
            " 18% 44/250 [00:14<01:10,  2.90it/s]\u001b[A\n",
            " 18% 45/250 [00:15<01:11,  2.85it/s]\u001b[A\n",
            " 18% 46/250 [00:15<01:09,  2.93it/s]\u001b[A\n",
            " 19% 47/250 [00:15<01:11,  2.83it/s]\u001b[A\n",
            " 19% 48/250 [00:16<01:09,  2.91it/s]\u001b[A\n",
            " 20% 49/250 [00:16<01:07,  3.00it/s]\u001b[A\n",
            " 20% 50/250 [00:16<01:07,  2.95it/s]\u001b[A\n",
            " 20% 51/250 [00:17<01:06,  3.01it/s]\u001b[A\n",
            " 21% 52/250 [00:17<01:05,  3.02it/s]\u001b[A\n",
            " 21% 53/250 [00:17<01:08,  2.88it/s]\u001b[A\n",
            " 22% 54/250 [00:18<01:08,  2.87it/s]\u001b[A\n",
            " 22% 55/250 [00:18<01:06,  2.95it/s]\u001b[A\n",
            " 22% 56/250 [00:18<01:06,  2.94it/s]\u001b[A\n",
            " 23% 57/250 [00:19<01:06,  2.90it/s]\u001b[A\n",
            " 23% 58/250 [00:19<01:03,  3.00it/s]\u001b[A\n",
            " 24% 59/250 [00:19<01:05,  2.93it/s]\u001b[A\n",
            " 24% 60/250 [00:20<01:15,  2.53it/s]\u001b[A\n",
            " 24% 61/250 [00:20<01:10,  2.68it/s]\u001b[A\n",
            " 25% 62/250 [00:21<01:13,  2.57it/s]\u001b[A\n",
            " 25% 63/250 [00:21<01:07,  2.76it/s]\u001b[A\n",
            " 26% 64/250 [00:21<01:05,  2.82it/s]\u001b[A\n",
            " 26% 65/250 [00:22<01:07,  2.75it/s]\u001b[A\n",
            " 26% 66/250 [00:22<01:06,  2.79it/s]\u001b[A\n",
            " 27% 67/250 [00:22<01:04,  2.84it/s]\u001b[A\n",
            " 27% 68/250 [00:23<01:01,  2.96it/s]\u001b[A\n",
            " 28% 69/250 [00:23<01:10,  2.56it/s]\u001b[A\n",
            " 28% 70/250 [00:23<01:09,  2.58it/s]\u001b[A\n",
            " 28% 71/250 [00:24<01:05,  2.75it/s]\u001b[A\n",
            " 29% 72/250 [00:24<01:02,  2.86it/s]\u001b[A\n",
            " 29% 73/250 [00:24<01:00,  2.91it/s]\u001b[A\n",
            " 30% 74/250 [00:25<01:08,  2.59it/s]\u001b[A\n",
            " 30% 75/250 [00:25<01:04,  2.70it/s]\u001b[A\n",
            " 30% 76/250 [00:26<01:03,  2.75it/s]\u001b[A\n",
            " 31% 77/250 [00:26<00:59,  2.89it/s]\u001b[A\n",
            " 31% 78/250 [00:26<00:57,  3.00it/s]\u001b[A\n",
            " 32% 79/250 [00:27<01:02,  2.72it/s]\u001b[A\n",
            " 32% 80/250 [00:27<01:05,  2.60it/s]\u001b[A\n",
            " 32% 81/250 [00:27<01:01,  2.77it/s]\u001b[A\n",
            " 33% 82/250 [00:28<00:58,  2.90it/s]\u001b[A\n",
            " 33% 83/250 [00:28<01:00,  2.76it/s]\u001b[A\n",
            " 34% 84/250 [00:28<00:59,  2.80it/s]\u001b[A\n",
            " 34% 85/250 [00:29<00:57,  2.86it/s]\u001b[A\n",
            " 34% 86/250 [00:29<01:02,  2.61it/s]\u001b[A\n",
            " 35% 87/250 [00:30<01:00,  2.69it/s]\u001b[A\n",
            " 35% 88/250 [00:30<01:02,  2.60it/s]\u001b[A\n",
            " 36% 89/250 [00:30<00:58,  2.76it/s]\u001b[A\n",
            " 36% 90/250 [00:31<01:07,  2.37it/s]\u001b[A\n",
            " 36% 91/250 [00:31<01:02,  2.53it/s]\u001b[A\n",
            " 37% 92/250 [00:31<00:59,  2.66it/s]\u001b[A\n",
            " 37% 93/250 [00:32<01:00,  2.58it/s]\u001b[A\n",
            " 38% 94/250 [00:32<00:59,  2.63it/s]\u001b[A\n",
            " 38% 95/250 [00:33<00:59,  2.61it/s]\u001b[A\n",
            " 38% 96/250 [00:33<01:07,  2.29it/s]\u001b[A\n",
            " 39% 97/250 [00:34<01:01,  2.50it/s]\u001b[A\n",
            " 39% 98/250 [00:34<00:57,  2.66it/s]\u001b[A\n",
            " 40% 99/250 [00:34<00:53,  2.81it/s]\u001b[A\n",
            " 40% 100/250 [00:34<00:52,  2.86it/s]\u001b[A\n",
            " 40% 101/250 [00:35<00:50,  2.93it/s]\u001b[A\n",
            " 41% 102/250 [00:35<00:50,  2.92it/s]\u001b[A\n",
            " 41% 103/250 [00:36<00:51,  2.88it/s]\u001b[A\n",
            " 42% 104/250 [00:36<00:50,  2.90it/s]\u001b[A\n",
            " 42% 105/250 [00:36<00:48,  3.02it/s]\u001b[A\n",
            " 42% 106/250 [00:37<00:55,  2.61it/s]\u001b[A\n",
            " 43% 107/250 [00:37<00:58,  2.46it/s]\u001b[A\n",
            " 43% 108/250 [00:37<00:53,  2.64it/s]\u001b[A\n",
            " 44% 109/250 [00:38<00:50,  2.80it/s]\u001b[A\n",
            " 44% 110/250 [00:38<00:49,  2.80it/s]\u001b[A\n",
            " 44% 111/250 [00:38<00:48,  2.87it/s]\u001b[A\n",
            " 45% 112/250 [00:39<00:49,  2.76it/s]\u001b[A\n",
            " 45% 113/250 [00:39<00:47,  2.86it/s]\u001b[A\n",
            " 46% 114/250 [00:39<00:46,  2.95it/s]\u001b[A\n",
            " 46% 115/250 [00:40<00:47,  2.86it/s]\u001b[A\n",
            " 46% 116/250 [00:40<00:49,  2.68it/s]\u001b[A\n",
            " 47% 117/250 [00:41<00:48,  2.76it/s]\u001b[A\n",
            " 47% 118/250 [00:41<00:49,  2.68it/s]\u001b[A\n",
            " 48% 119/250 [00:41<00:47,  2.75it/s]\u001b[A\n",
            " 48% 120/250 [00:42<00:46,  2.78it/s]\u001b[A\n",
            " 48% 121/250 [00:42<00:45,  2.82it/s]\u001b[A\n",
            " 49% 122/250 [00:42<00:44,  2.89it/s]\u001b[A\n",
            " 49% 123/250 [00:43<00:42,  2.98it/s]\u001b[A\n",
            " 50% 124/250 [00:43<00:41,  3.05it/s]\u001b[A\n",
            " 50% 125/250 [00:43<00:40,  3.08it/s]\u001b[A\n",
            " 50% 126/250 [00:44<00:40,  3.09it/s]\u001b[A\n",
            " 51% 127/250 [00:44<00:40,  3.01it/s]\u001b[A\n",
            " 51% 128/250 [00:45<00:49,  2.49it/s]\u001b[A\n",
            " 52% 129/250 [00:45<00:45,  2.64it/s]\u001b[A\n",
            " 52% 130/250 [00:45<00:46,  2.58it/s]\u001b[A\n",
            " 52% 131/250 [00:46<00:46,  2.56it/s]\u001b[A\n",
            " 53% 132/250 [00:46<00:44,  2.63it/s]\u001b[A\n",
            " 53% 133/250 [00:46<00:44,  2.64it/s]\u001b[A\n",
            " 54% 134/250 [00:47<00:44,  2.62it/s]\u001b[A\n",
            " 54% 135/250 [00:47<00:42,  2.71it/s]\u001b[A\n",
            " 54% 136/250 [00:47<00:40,  2.84it/s]\u001b[A\n",
            " 55% 137/250 [00:48<00:41,  2.76it/s]\u001b[A\n",
            " 55% 138/250 [00:48<00:38,  2.89it/s]\u001b[A\n",
            " 56% 139/250 [00:49<00:40,  2.73it/s]\u001b[A\n",
            " 56% 140/250 [00:49<00:39,  2.82it/s]\u001b[A\n",
            " 56% 141/250 [00:49<00:37,  2.92it/s]\u001b[A\n",
            " 57% 142/250 [00:50<00:43,  2.51it/s]\u001b[A\n",
            " 57% 143/250 [00:50<00:42,  2.54it/s]\u001b[A\n",
            " 58% 144/250 [00:50<00:39,  2.67it/s]\u001b[A\n",
            " 58% 145/250 [00:51<00:38,  2.73it/s]\u001b[A\n",
            " 58% 146/250 [00:51<00:36,  2.87it/s]\u001b[A\n",
            " 59% 147/250 [00:51<00:35,  2.93it/s]\u001b[A\n",
            " 59% 148/250 [00:52<00:33,  3.03it/s]\u001b[A\n",
            " 60% 149/250 [00:52<00:34,  2.96it/s]\u001b[A\n",
            " 60% 150/250 [00:52<00:33,  3.03it/s]\u001b[A\n",
            " 60% 151/250 [00:53<00:32,  3.01it/s]\u001b[A\n",
            " 61% 152/250 [00:53<00:33,  2.91it/s]\u001b[A\n",
            " 61% 153/250 [00:53<00:32,  2.96it/s]\u001b[A\n",
            " 62% 154/250 [00:54<00:33,  2.91it/s]\u001b[A\n",
            " 62% 155/250 [00:54<00:31,  3.05it/s]\u001b[A\n",
            " 62% 156/250 [00:54<00:33,  2.78it/s]\u001b[A\n",
            " 63% 157/250 [00:55<00:33,  2.80it/s]\u001b[A\n",
            " 63% 158/250 [00:55<00:33,  2.75it/s]\u001b[A\n",
            " 64% 159/250 [00:56<00:31,  2.88it/s]\u001b[A\n",
            " 64% 160/250 [00:56<00:30,  2.90it/s]\u001b[A\n",
            " 64% 161/250 [00:56<00:30,  2.90it/s]\u001b[A\n",
            " 65% 162/250 [00:57<00:29,  2.94it/s]\u001b[A\n",
            " 65% 163/250 [00:57<00:29,  2.95it/s]\u001b[A\n",
            " 66% 164/250 [00:57<00:29,  2.93it/s]\u001b[A\n",
            " 66% 165/250 [00:58<00:29,  2.91it/s]\u001b[A\n",
            " 66% 166/250 [00:58<00:29,  2.81it/s]\u001b[A\n",
            " 67% 167/250 [00:58<00:28,  2.94it/s]\u001b[A\n",
            " 67% 168/250 [00:59<00:28,  2.92it/s]\u001b[A\n",
            " 68% 169/250 [00:59<00:29,  2.71it/s]\u001b[A\n",
            " 68% 170/250 [00:59<00:29,  2.73it/s]\u001b[A\n",
            " 68% 171/250 [01:00<00:27,  2.86it/s]\u001b[A\n",
            " 69% 172/250 [01:00<00:26,  2.96it/s]\u001b[A\n",
            " 69% 173/250 [01:00<00:26,  2.95it/s]\u001b[A\n",
            " 70% 174/250 [01:01<00:25,  2.99it/s]\u001b[A\n",
            " 70% 175/250 [01:01<00:24,  3.01it/s]\u001b[A\n",
            " 70% 176/250 [01:01<00:24,  3.00it/s]\u001b[A\n",
            " 71% 177/250 [01:02<00:24,  3.00it/s]\u001b[A\n",
            " 71% 178/250 [01:02<00:24,  2.96it/s]\u001b[A\n",
            " 72% 179/250 [01:02<00:23,  3.00it/s]\u001b[A\n",
            " 72% 180/250 [01:03<00:23,  2.94it/s]\u001b[A\n",
            " 72% 181/250 [01:03<00:22,  3.03it/s]\u001b[A\n",
            " 73% 182/250 [01:03<00:22,  3.06it/s]\u001b[A\n",
            " 73% 183/250 [01:04<00:22,  3.02it/s]\u001b[A\n",
            " 74% 184/250 [01:04<00:22,  2.97it/s]\u001b[A\n",
            " 74% 185/250 [01:04<00:21,  2.97it/s]\u001b[A\n",
            " 74% 186/250 [01:05<00:21,  3.03it/s]\u001b[A\n",
            " 75% 187/250 [01:05<00:20,  3.01it/s]\u001b[A\n",
            " 75% 188/250 [01:05<00:20,  3.01it/s]\u001b[A\n",
            " 76% 189/250 [01:06<00:20,  2.99it/s]\u001b[A\n",
            " 76% 190/250 [01:06<00:19,  3.03it/s]\u001b[A\n",
            " 76% 191/250 [01:06<00:19,  3.06it/s]\u001b[A\n",
            " 77% 192/250 [01:07<00:19,  2.93it/s]\u001b[A\n",
            " 77% 193/250 [01:07<00:19,  2.96it/s]\u001b[A\n",
            " 78% 194/250 [01:07<00:18,  3.05it/s]\u001b[A\n",
            " 78% 195/250 [01:08<00:17,  3.06it/s]\u001b[A\n",
            " 78% 196/250 [01:08<00:17,  3.10it/s]\u001b[A\n",
            " 79% 197/250 [01:08<00:17,  3.01it/s]\u001b[A\n",
            " 79% 198/250 [01:09<00:17,  2.90it/s]\u001b[A\n",
            " 80% 199/250 [01:09<00:17,  2.99it/s]\u001b[A\n",
            " 80% 200/250 [01:09<00:16,  2.95it/s]\u001b[A\n",
            " 80% 201/250 [01:10<00:16,  2.93it/s]\u001b[A\n",
            " 81% 202/250 [01:10<00:16,  2.96it/s]\u001b[A\n",
            " 81% 203/250 [01:10<00:15,  2.97it/s]\u001b[A\n",
            " 82% 204/250 [01:11<00:16,  2.82it/s]\u001b[A\n",
            " 82% 205/250 [01:11<00:15,  2.87it/s]\u001b[A\n",
            " 82% 206/250 [01:11<00:15,  2.92it/s]\u001b[A\n",
            " 83% 207/250 [01:12<00:14,  2.89it/s]\u001b[A\n",
            " 83% 208/250 [01:12<00:14,  2.92it/s]\u001b[A\n",
            " 84% 209/250 [01:12<00:14,  2.89it/s]\u001b[A\n",
            " 84% 210/250 [01:13<00:13,  2.99it/s]\u001b[A\n",
            " 84% 211/250 [01:13<00:12,  3.04it/s]\u001b[A\n",
            " 85% 212/250 [01:13<00:12,  3.12it/s]\u001b[A\n",
            " 85% 213/250 [01:14<00:12,  3.04it/s]\u001b[A\n",
            " 86% 214/250 [01:14<00:11,  3.09it/s]\u001b[A\n",
            " 86% 215/250 [01:14<00:11,  3.08it/s]\u001b[A\n",
            " 86% 216/250 [01:15<00:11,  3.07it/s]\u001b[A\n",
            " 87% 217/250 [01:15<00:10,  3.01it/s]\u001b[A\n",
            " 87% 218/250 [01:15<00:10,  2.96it/s]\u001b[A\n",
            " 88% 219/250 [01:16<00:10,  3.05it/s]\u001b[A\n",
            " 88% 220/250 [01:16<00:09,  3.13it/s]\u001b[A\n",
            " 88% 221/250 [01:16<00:09,  3.13it/s]\u001b[A\n",
            " 89% 222/250 [01:17<00:09,  2.98it/s]\u001b[A\n",
            " 89% 223/250 [01:17<00:09,  2.94it/s]\u001b[A\n",
            " 90% 224/250 [01:17<00:08,  3.03it/s]\u001b[A\n",
            " 90% 225/250 [01:18<00:08,  2.95it/s]\u001b[A\n",
            " 90% 226/250 [01:18<00:08,  2.95it/s]\u001b[A\n",
            " 91% 227/250 [01:18<00:07,  3.03it/s]\u001b[A\n",
            " 91% 228/250 [01:19<00:07,  2.99it/s]\u001b[A\n",
            " 92% 229/250 [01:19<00:07,  2.76it/s]\u001b[A\n",
            " 92% 230/250 [01:19<00:07,  2.81it/s]\u001b[A\n",
            " 92% 231/250 [01:20<00:07,  2.68it/s]\u001b[A\n",
            " 93% 232/250 [01:20<00:07,  2.34it/s]\u001b[A\n",
            " 93% 233/250 [01:21<00:06,  2.49it/s]\u001b[A\n",
            " 94% 234/250 [01:21<00:06,  2.66it/s]\u001b[A\n",
            " 94% 235/250 [01:21<00:05,  2.80it/s]\u001b[A\n",
            " 94% 236/250 [01:22<00:05,  2.39it/s]\u001b[A\n",
            " 95% 237/250 [01:22<00:05,  2.44it/s]\u001b[A\n",
            " 95% 238/250 [01:23<00:04,  2.43it/s]\u001b[A\n",
            " 96% 239/250 [01:23<00:04,  2.51it/s]\u001b[A\n",
            " 96% 240/250 [01:24<00:03,  2.62it/s]\u001b[A\n",
            " 96% 241/250 [01:24<00:03,  2.67it/s]\u001b[A\n",
            " 97% 242/250 [01:24<00:02,  2.67it/s]\u001b[A\n",
            " 97% 243/250 [01:25<00:02,  2.85it/s]\u001b[A\n",
            " 98% 244/250 [01:25<00:02,  2.94it/s]\u001b[A\n",
            " 98% 245/250 [01:25<00:01,  3.07it/s]\u001b[A\n",
            " 98% 246/250 [01:25<00:01,  3.09it/s]\u001b[A\n",
            " 99% 247/250 [01:26<00:00,  3.13it/s]\u001b[A\n",
            " 99% 248/250 [01:26<00:00,  3.01it/s]\u001b[A\n",
            "100% 249/250 [01:27<00:00,  2.89it/s]\u001b[A\n",
            "100% 250/250 [01:27<00:00,  2.91it/s]\u001b[A\n",
            "{'eval_loss': 0.8749068975448608, 'eval_runtime': 87.78, 'eval_samples_per_second': 22.784, 'eval_steps_per_second': 2.848, 'epoch': 1.6}\n",
            "\n",
            " 53% 800/1500 [1:15:57<34:52,  2.99s/it]\n",
            "{'loss': 1.0019, 'learning_rate': 4.928571428571429e-05, 'epoch': 1.62}\n",
            "{'loss': 0.9479, 'learning_rate': 4.8571428571428576e-05, 'epoch': 1.64}\n",
            "{'loss': 0.8141, 'learning_rate': 4.785714285714286e-05, 'epoch': 1.66}\n",
            "{'loss': 0.6676, 'learning_rate': 4.714285714285714e-05, 'epoch': 1.68}\n",
            "{'loss': 0.6251, 'learning_rate': 4.642857142857143e-05, 'epoch': 1.7}\n",
            "{'loss': 1.0051, 'learning_rate': 4.5714285714285716e-05, 'epoch': 1.72}\n",
            "{'loss': 0.9395, 'learning_rate': 4.5e-05, 'epoch': 1.74}\n",
            "{'loss': 0.8161, 'learning_rate': 4.428571428571428e-05, 'epoch': 1.76}\n",
            "{'loss': 0.6718, 'learning_rate': 4.3571428571428576e-05, 'epoch': 1.78}\n",
            "{'loss': 0.6145, 'learning_rate': 4.2857142857142856e-05, 'epoch': 1.8}\n",
            "{'loss': 1.0038, 'learning_rate': 4.214285714285714e-05, 'epoch': 1.82}\n",
            "{'loss': 0.9371, 'learning_rate': 4.1428571428571437e-05, 'epoch': 1.84}\n",
            "{'loss': 0.8074, 'learning_rate': 4.0714285714285717e-05, 'epoch': 1.86}\n",
            "{'loss': 0.6716, 'learning_rate': 4e-05, 'epoch': 1.88}\n",
            "{'loss': 0.623, 'learning_rate': 3.928571428571429e-05, 'epoch': 1.9}\n",
            "{'loss': 0.9893, 'learning_rate': 3.857142857142858e-05, 'epoch': 1.92}\n",
            "{'loss': 0.9173, 'learning_rate': 3.785714285714286e-05, 'epoch': 1.94}\n",
            "{'loss': 0.7971, 'learning_rate': 3.7142857142857143e-05, 'epoch': 1.96}\n",
            "{'loss': 0.6536, 'learning_rate': 3.642857142857143e-05, 'epoch': 1.98}\n",
            "{'loss': 0.6142, 'learning_rate': 3.571428571428572e-05, 'epoch': 2.0}\n",
            " 67% 1000/1500 [1:33:29<25:02,  3.01s/it]\n",
            "  0% 0/250 [00:00<?, ?it/s]\u001b[A\n",
            "  1% 2/250 [00:00<00:40,  6.16it/s]\u001b[A\n",
            "  1% 3/250 [00:00<00:55,  4.42it/s]\u001b[A\n",
            "  2% 4/250 [00:01<01:09,  3.53it/s]\u001b[A\n",
            "  2% 5/250 [00:01<01:15,  3.26it/s]\u001b[A\n",
            "  2% 6/250 [00:01<01:15,  3.23it/s]\u001b[A\n",
            "  3% 7/250 [00:01<01:15,  3.23it/s]\u001b[A\n",
            "  3% 8/250 [00:02<01:19,  3.05it/s]\u001b[A\n",
            "  4% 9/250 [00:02<01:22,  2.90it/s]\u001b[A\n",
            "  4% 10/250 [00:03<01:28,  2.71it/s]\u001b[A\n",
            "  4% 11/250 [00:03<01:25,  2.79it/s]\u001b[A\n",
            "  5% 12/250 [00:03<01:24,  2.81it/s]\u001b[A\n",
            "  5% 13/250 [00:04<01:20,  2.93it/s]\u001b[A\n",
            "  6% 14/250 [00:04<01:18,  3.01it/s]\u001b[A\n",
            "  6% 15/250 [00:04<01:18,  2.98it/s]\u001b[A\n",
            "  6% 16/250 [00:05<01:16,  3.05it/s]\u001b[A\n",
            "  7% 17/250 [00:05<01:17,  3.01it/s]\u001b[A\n",
            "  7% 18/250 [00:05<01:17,  2.98it/s]\u001b[A\n",
            "  8% 19/250 [00:06<01:17,  2.97it/s]\u001b[A\n",
            "  8% 20/250 [00:06<01:18,  2.93it/s]\u001b[A\n",
            "  8% 21/250 [00:06<01:17,  2.97it/s]\u001b[A\n",
            "  9% 22/250 [00:07<01:17,  2.95it/s]\u001b[A\n",
            "  9% 23/250 [00:07<01:14,  3.06it/s]\u001b[A\n",
            " 10% 24/250 [00:07<01:16,  2.97it/s]\u001b[A\n",
            " 10% 25/250 [00:08<01:23,  2.70it/s]\u001b[A\n",
            " 10% 26/250 [00:08<01:24,  2.64it/s]\u001b[A\n",
            " 11% 27/250 [00:08<01:20,  2.77it/s]\u001b[A\n",
            " 11% 28/250 [00:09<01:16,  2.88it/s]\u001b[A\n",
            " 12% 29/250 [00:09<01:15,  2.93it/s]\u001b[A\n",
            " 12% 30/250 [00:09<01:14,  2.95it/s]\u001b[A\n",
            " 12% 31/250 [00:10<01:12,  3.02it/s]\u001b[A\n",
            " 13% 32/250 [00:10<01:11,  3.03it/s]\u001b[A\n",
            " 13% 33/250 [00:10<01:12,  2.99it/s]\u001b[A\n",
            " 14% 34/250 [00:11<01:10,  3.05it/s]\u001b[A\n",
            " 14% 35/250 [00:11<01:10,  3.06it/s]\u001b[A\n",
            " 14% 36/250 [00:11<01:14,  2.86it/s]\u001b[A\n",
            " 15% 37/250 [00:12<01:12,  2.94it/s]\u001b[A\n",
            " 15% 38/250 [00:12<01:10,  2.99it/s]\u001b[A\n",
            " 16% 39/250 [00:12<01:11,  2.95it/s]\u001b[A\n",
            " 16% 40/250 [00:13<01:15,  2.78it/s]\u001b[A\n",
            " 16% 41/250 [00:13<01:14,  2.82it/s]\u001b[A\n",
            " 17% 42/250 [00:14<01:13,  2.84it/s]\u001b[A\n",
            " 17% 43/250 [00:14<01:13,  2.80it/s]\u001b[A\n",
            " 18% 44/250 [00:14<01:10,  2.91it/s]\u001b[A\n",
            " 18% 45/250 [00:15<01:11,  2.85it/s]\u001b[A\n",
            " 18% 46/250 [00:15<01:09,  2.94it/s]\u001b[A\n",
            " 19% 47/250 [00:15<01:11,  2.84it/s]\u001b[A\n",
            " 19% 48/250 [00:16<01:09,  2.92it/s]\u001b[A\n",
            " 20% 49/250 [00:16<01:07,  3.00it/s]\u001b[A\n",
            " 20% 50/250 [00:16<01:07,  2.97it/s]\u001b[A\n",
            " 20% 51/250 [00:17<01:05,  3.05it/s]\u001b[A\n",
            " 21% 52/250 [00:17<01:04,  3.06it/s]\u001b[A\n",
            " 21% 53/250 [00:17<01:07,  2.91it/s]\u001b[A\n",
            " 22% 54/250 [00:18<01:07,  2.89it/s]\u001b[A\n",
            " 22% 55/250 [00:18<01:05,  2.97it/s]\u001b[A\n",
            " 22% 56/250 [00:18<01:05,  2.97it/s]\u001b[A\n",
            " 23% 57/250 [00:19<01:05,  2.93it/s]\u001b[A\n",
            " 23% 58/250 [00:19<01:03,  3.02it/s]\u001b[A\n",
            " 24% 59/250 [00:19<01:04,  2.94it/s]\u001b[A\n",
            " 24% 60/250 [00:20<01:15,  2.52it/s]\u001b[A\n",
            " 24% 61/250 [00:20<01:10,  2.67it/s]\u001b[A\n",
            " 25% 62/250 [00:21<01:13,  2.56it/s]\u001b[A\n",
            " 25% 63/250 [00:21<01:08,  2.74it/s]\u001b[A\n",
            " 26% 64/250 [00:21<01:06,  2.78it/s]\u001b[A\n",
            " 26% 65/250 [00:22<01:08,  2.72it/s]\u001b[A\n",
            " 26% 66/250 [00:22<01:07,  2.73it/s]\u001b[A\n",
            " 27% 67/250 [00:22<01:06,  2.77it/s]\u001b[A\n",
            " 27% 68/250 [00:23<01:03,  2.88it/s]\u001b[A\n",
            " 28% 69/250 [00:23<01:12,  2.51it/s]\u001b[A\n",
            " 28% 70/250 [00:24<01:11,  2.52it/s]\u001b[A\n",
            " 28% 71/250 [00:24<01:06,  2.68it/s]\u001b[A\n",
            " 29% 72/250 [00:24<01:03,  2.79it/s]\u001b[A\n",
            " 29% 73/250 [00:25<01:02,  2.83it/s]\u001b[A\n",
            " 30% 74/250 [00:25<01:09,  2.53it/s]\u001b[A\n",
            " 30% 75/250 [00:25<01:06,  2.63it/s]\u001b[A\n",
            " 30% 76/250 [00:26<01:04,  2.69it/s]\u001b[A\n",
            " 31% 77/250 [00:26<01:01,  2.83it/s]\u001b[A\n",
            " 31% 78/250 [00:26<00:58,  2.94it/s]\u001b[A\n",
            " 32% 79/250 [00:27<01:03,  2.69it/s]\u001b[A\n",
            " 32% 80/250 [00:27<01:05,  2.58it/s]\u001b[A\n",
            " 32% 81/250 [00:28<01:01,  2.75it/s]\u001b[A\n",
            " 33% 82/250 [00:28<00:58,  2.88it/s]\u001b[A\n",
            " 33% 83/250 [00:28<01:00,  2.74it/s]\u001b[A\n",
            " 34% 84/250 [00:29<00:59,  2.78it/s]\u001b[A\n",
            " 34% 85/250 [00:29<00:57,  2.85it/s]\u001b[A\n",
            " 34% 86/250 [00:29<01:03,  2.59it/s]\u001b[A\n",
            " 35% 87/250 [00:30<01:01,  2.67it/s]\u001b[A\n",
            " 35% 88/250 [00:30<01:03,  2.57it/s]\u001b[A\n",
            " 36% 89/250 [00:31<00:59,  2.73it/s]\u001b[A\n",
            " 36% 90/250 [00:31<01:08,  2.35it/s]\u001b[A\n",
            " 36% 91/250 [00:31<01:03,  2.51it/s]\u001b[A\n",
            " 37% 92/250 [00:32<00:59,  2.65it/s]\u001b[A\n",
            " 37% 93/250 [00:32<01:00,  2.58it/s]\u001b[A\n",
            " 38% 94/250 [00:33<00:59,  2.62it/s]\u001b[A\n",
            " 38% 95/250 [00:33<00:59,  2.60it/s]\u001b[A\n",
            " 38% 96/250 [00:33<01:07,  2.28it/s]\u001b[A\n",
            " 39% 97/250 [00:34<01:01,  2.50it/s]\u001b[A\n",
            " 39% 98/250 [00:34<00:57,  2.66it/s]\u001b[A\n",
            " 40% 99/250 [00:34<00:53,  2.81it/s]\u001b[A\n",
            " 40% 100/250 [00:35<00:52,  2.87it/s]\u001b[A\n",
            " 40% 101/250 [00:35<00:51,  2.92it/s]\u001b[A\n",
            " 41% 102/250 [00:35<00:51,  2.90it/s]\u001b[A\n",
            " 41% 103/250 [00:36<00:51,  2.85it/s]\u001b[A\n",
            " 42% 104/250 [00:36<00:50,  2.87it/s]\u001b[A\n",
            " 42% 105/250 [00:36<00:49,  2.96it/s]\u001b[A\n",
            " 42% 106/250 [00:37<00:56,  2.57it/s]\u001b[A\n",
            " 43% 107/250 [00:37<00:58,  2.43it/s]\u001b[A\n",
            " 43% 108/250 [00:38<00:54,  2.58it/s]\u001b[A\n",
            " 44% 109/250 [00:38<00:51,  2.72it/s]\u001b[A\n",
            " 44% 110/250 [00:38<00:51,  2.71it/s]\u001b[A\n",
            " 44% 111/250 [00:39<00:49,  2.78it/s]\u001b[A\n",
            " 45% 112/250 [00:39<00:51,  2.67it/s]\u001b[A\n",
            " 45% 113/250 [00:40<00:49,  2.78it/s]\u001b[A\n",
            " 46% 114/250 [00:40<00:47,  2.86it/s]\u001b[A\n",
            " 46% 115/250 [00:40<00:48,  2.77it/s]\u001b[A\n",
            " 46% 116/250 [00:41<00:51,  2.61it/s]\u001b[A\n",
            " 47% 117/250 [00:41<00:49,  2.69it/s]\u001b[A\n",
            " 47% 118/250 [00:41<00:50,  2.63it/s]\u001b[A\n",
            " 48% 119/250 [00:42<00:48,  2.72it/s]\u001b[A\n",
            " 48% 120/250 [00:42<00:47,  2.75it/s]\u001b[A\n",
            " 48% 121/250 [00:42<00:46,  2.78it/s]\u001b[A\n",
            " 49% 122/250 [00:43<00:44,  2.86it/s]\u001b[A\n",
            " 49% 123/250 [00:43<00:42,  2.96it/s]\u001b[A\n",
            " 50% 124/250 [00:43<00:41,  3.03it/s]\u001b[A\n",
            " 50% 125/250 [00:44<00:40,  3.07it/s]\u001b[A\n",
            " 50% 126/250 [00:44<00:40,  3.09it/s]\u001b[A\n",
            " 51% 127/250 [00:44<00:40,  3.01it/s]\u001b[A\n",
            " 51% 128/250 [00:45<00:48,  2.50it/s]\u001b[A\n",
            " 52% 129/250 [00:45<00:45,  2.65it/s]\u001b[A\n",
            " 52% 130/250 [00:46<00:46,  2.57it/s]\u001b[A\n",
            " 52% 131/250 [00:46<00:46,  2.57it/s]\u001b[A\n",
            " 53% 132/250 [00:46<00:44,  2.64it/s]\u001b[A\n",
            " 53% 133/250 [00:47<00:44,  2.65it/s]\u001b[A\n",
            " 54% 134/250 [00:47<00:44,  2.63it/s]\u001b[A\n",
            " 54% 135/250 [00:48<00:42,  2.72it/s]\u001b[A\n",
            " 54% 136/250 [00:48<00:39,  2.86it/s]\u001b[A\n",
            " 55% 137/250 [00:48<00:40,  2.77it/s]\u001b[A\n",
            " 55% 138/250 [00:49<00:38,  2.89it/s]\u001b[A\n",
            " 56% 139/250 [00:49<00:40,  2.72it/s]\u001b[A\n",
            " 56% 140/250 [00:49<00:39,  2.81it/s]\u001b[A\n",
            " 56% 141/250 [00:50<00:37,  2.92it/s]\u001b[A\n",
            " 57% 142/250 [00:50<00:43,  2.49it/s]\u001b[A\n",
            " 57% 143/250 [00:51<00:42,  2.53it/s]\u001b[A\n",
            " 58% 144/250 [00:51<00:40,  2.65it/s]\u001b[A\n",
            " 58% 145/250 [00:51<00:39,  2.69it/s]\u001b[A\n",
            " 58% 146/250 [00:52<00:37,  2.81it/s]\u001b[A\n",
            " 59% 147/250 [00:52<00:36,  2.85it/s]\u001b[A\n",
            " 59% 148/250 [00:52<00:34,  2.94it/s]\u001b[A\n",
            " 60% 149/250 [00:53<00:35,  2.87it/s]\u001b[A\n",
            " 60% 150/250 [00:53<00:34,  2.93it/s]\u001b[A\n",
            " 60% 151/250 [00:53<00:34,  2.90it/s]\u001b[A\n",
            " 61% 152/250 [00:54<00:34,  2.81it/s]\u001b[A\n",
            " 61% 153/250 [00:54<00:33,  2.86it/s]\u001b[A\n",
            " 62% 154/250 [00:54<00:34,  2.82it/s]\u001b[A\n",
            " 62% 155/250 [00:55<00:32,  2.94it/s]\u001b[A\n",
            " 62% 156/250 [00:55<00:34,  2.70it/s]\u001b[A\n",
            " 63% 157/250 [00:55<00:33,  2.74it/s]\u001b[A\n",
            " 63% 158/250 [00:56<00:34,  2.69it/s]\u001b[A\n",
            " 64% 159/250 [00:56<00:32,  2.83it/s]\u001b[A\n",
            " 64% 160/250 [00:56<00:31,  2.84it/s]\u001b[A\n",
            " 64% 161/250 [00:57<00:31,  2.85it/s]\u001b[A\n",
            " 65% 162/250 [00:57<00:30,  2.89it/s]\u001b[A\n",
            " 65% 163/250 [00:57<00:29,  2.90it/s]\u001b[A\n",
            " 66% 164/250 [00:58<00:29,  2.88it/s]\u001b[A\n",
            " 66% 165/250 [00:58<00:29,  2.86it/s]\u001b[A\n",
            " 66% 166/250 [00:59<00:30,  2.76it/s]\u001b[A\n",
            " 67% 167/250 [00:59<00:28,  2.88it/s]\u001b[A\n",
            " 67% 168/250 [00:59<00:28,  2.87it/s]\u001b[A\n",
            " 68% 169/250 [01:00<00:30,  2.68it/s]\u001b[A\n",
            " 68% 170/250 [01:00<00:29,  2.70it/s]\u001b[A\n",
            " 68% 171/250 [01:00<00:27,  2.83it/s]\u001b[A\n",
            " 69% 172/250 [01:01<00:26,  2.92it/s]\u001b[A\n",
            " 69% 173/250 [01:01<00:26,  2.90it/s]\u001b[A\n",
            " 70% 174/250 [01:01<00:25,  2.96it/s]\u001b[A\n",
            " 70% 175/250 [01:02<00:24,  3.01it/s]\u001b[A\n",
            " 70% 176/250 [01:02<00:24,  3.00it/s]\u001b[A\n",
            " 71% 177/250 [01:02<00:24,  2.99it/s]\u001b[A\n",
            " 71% 178/250 [01:03<00:24,  2.97it/s]\u001b[A\n",
            " 72% 179/250 [01:03<00:23,  3.01it/s]\u001b[A\n",
            " 72% 180/250 [01:03<00:23,  2.93it/s]\u001b[A\n",
            " 72% 181/250 [01:04<00:22,  3.01it/s]\u001b[A\n",
            " 73% 182/250 [01:04<00:22,  3.03it/s]\u001b[A\n",
            " 73% 183/250 [01:04<00:22,  2.97it/s]\u001b[A\n",
            " 74% 184/250 [01:05<00:22,  2.92it/s]\u001b[A\n",
            " 74% 185/250 [01:05<00:22,  2.90it/s]\u001b[A\n",
            " 74% 186/250 [01:05<00:21,  2.96it/s]\u001b[A\n",
            " 75% 187/250 [01:06<00:21,  2.93it/s]\u001b[A\n",
            " 75% 188/250 [01:06<00:21,  2.93it/s]\u001b[A\n",
            " 76% 189/250 [01:06<00:20,  2.91it/s]\u001b[A\n",
            " 76% 190/250 [01:07<00:20,  2.96it/s]\u001b[A\n",
            " 76% 191/250 [01:07<00:19,  2.97it/s]\u001b[A\n",
            " 77% 192/250 [01:07<00:20,  2.83it/s]\u001b[A\n",
            " 77% 193/250 [01:08<00:19,  2.85it/s]\u001b[A\n",
            " 78% 194/250 [01:08<00:19,  2.95it/s]\u001b[A\n",
            " 78% 195/250 [01:08<00:18,  2.91it/s]\u001b[A\n",
            " 78% 196/250 [01:09<00:18,  2.91it/s]\u001b[A\n",
            " 79% 197/250 [01:09<00:18,  2.82it/s]\u001b[A\n",
            " 79% 198/250 [01:10<00:18,  2.74it/s]\u001b[A\n",
            " 80% 199/250 [01:10<00:17,  2.84it/s]\u001b[A\n",
            " 80% 200/250 [01:10<00:17,  2.84it/s]\u001b[A\n",
            " 80% 201/250 [01:11<00:17,  2.85it/s]\u001b[A\n",
            " 81% 202/250 [01:11<00:16,  2.88it/s]\u001b[A\n",
            " 81% 203/250 [01:11<00:16,  2.89it/s]\u001b[A\n",
            " 82% 204/250 [01:12<00:16,  2.75it/s]\u001b[A\n",
            " 82% 205/250 [01:12<00:16,  2.79it/s]\u001b[A\n",
            " 82% 206/250 [01:12<00:15,  2.86it/s]\u001b[A\n",
            " 83% 207/250 [01:13<00:15,  2.84it/s]\u001b[A\n",
            " 83% 208/250 [01:13<00:14,  2.85it/s]\u001b[A\n",
            " 84% 209/250 [01:13<00:14,  2.83it/s]\u001b[A\n",
            " 84% 210/250 [01:14<00:13,  2.94it/s]\u001b[A\n",
            " 84% 211/250 [01:14<00:13,  2.99it/s]\u001b[A\n",
            " 85% 212/250 [01:14<00:12,  3.07it/s]\u001b[A\n",
            " 85% 213/250 [01:15<00:12,  3.00it/s]\u001b[A\n",
            " 86% 214/250 [01:15<00:11,  3.02it/s]\u001b[A\n",
            " 86% 215/250 [01:15<00:11,  3.03it/s]\u001b[A\n",
            " 86% 216/250 [01:16<00:11,  3.02it/s]\u001b[A\n",
            " 87% 217/250 [01:16<00:11,  2.96it/s]\u001b[A\n",
            " 87% 218/250 [01:16<00:11,  2.91it/s]\u001b[A\n",
            " 88% 219/250 [01:17<00:10,  3.00it/s]\u001b[A\n",
            " 88% 220/250 [01:17<00:09,  3.07it/s]\u001b[A\n",
            " 88% 221/250 [01:17<00:09,  3.09it/s]\u001b[A\n",
            " 89% 222/250 [01:18<00:09,  2.93it/s]\u001b[A\n",
            " 89% 223/250 [01:18<00:09,  2.90it/s]\u001b[A\n",
            " 90% 224/250 [01:18<00:08,  2.99it/s]\u001b[A\n",
            " 90% 225/250 [01:19<00:08,  2.91it/s]\u001b[A\n",
            " 90% 226/250 [01:19<00:08,  2.91it/s]\u001b[A\n",
            " 91% 227/250 [01:19<00:07,  2.97it/s]\u001b[A\n",
            " 91% 228/250 [01:20<00:07,  2.94it/s]\u001b[A\n",
            " 92% 229/250 [01:20<00:07,  2.72it/s]\u001b[A\n",
            " 92% 230/250 [01:21<00:07,  2.76it/s]\u001b[A\n",
            " 92% 231/250 [01:21<00:07,  2.63it/s]\u001b[A\n",
            " 93% 232/250 [01:22<00:07,  2.30it/s]\u001b[A\n",
            " 93% 233/250 [01:22<00:06,  2.44it/s]\u001b[A\n",
            " 94% 234/250 [01:22<00:06,  2.62it/s]\u001b[A\n",
            " 94% 235/250 [01:23<00:05,  2.75it/s]\u001b[A\n",
            " 94% 236/250 [01:23<00:05,  2.36it/s]\u001b[A\n",
            " 95% 237/250 [01:24<00:05,  2.40it/s]\u001b[A\n",
            " 95% 238/250 [01:24<00:05,  2.39it/s]\u001b[A\n",
            " 96% 239/250 [01:24<00:04,  2.47it/s]\u001b[A\n",
            " 96% 240/250 [01:25<00:03,  2.57it/s]\u001b[A\n",
            " 96% 241/250 [01:25<00:03,  2.62it/s]\u001b[A\n",
            " 97% 242/250 [01:25<00:03,  2.63it/s]\u001b[A\n",
            " 97% 243/250 [01:26<00:02,  2.81it/s]\u001b[A\n",
            " 98% 244/250 [01:26<00:02,  2.90it/s]\u001b[A\n",
            " 98% 245/250 [01:26<00:01,  3.01it/s]\u001b[A\n",
            " 98% 246/250 [01:27<00:01,  3.04it/s]\u001b[A\n",
            " 99% 247/250 [01:27<00:00,  3.08it/s]\u001b[A\n",
            " 99% 248/250 [01:27<00:00,  2.96it/s]\u001b[A\n",
            "100% 249/250 [01:28<00:00,  2.85it/s]\u001b[A\n",
            "100% 250/250 [01:28<00:00,  2.86it/s]\u001b[A\n",
            "{'eval_loss': 0.8692930936813354, 'eval_runtime': 88.9932, 'eval_samples_per_second': 22.474, 'eval_steps_per_second': 2.809, 'epoch': 2.0}\n",
            "\n",
            " 67% 1000/1500 [1:34:58<25:02,  3.01s/it]\n",
            "{'loss': 1.0045, 'learning_rate': 3.5e-05, 'epoch': 2.02}\n",
            "{'loss': 0.9361, 'learning_rate': 3.428571428571429e-05, 'epoch': 2.04}\n",
            "{'loss': 0.8205, 'learning_rate': 3.357142857142857e-05, 'epoch': 2.06}\n",
            "{'loss': 0.6618, 'learning_rate': 3.285714285714286e-05, 'epoch': 2.08}\n",
            "{'loss': 0.6217, 'learning_rate': 3.2142857142857144e-05, 'epoch': 2.1}\n",
            "{'loss': 0.9787, 'learning_rate': 3.142857142857143e-05, 'epoch': 2.12}\n",
            "{'loss': 0.9354, 'learning_rate': 3.071428571428572e-05, 'epoch': 2.14}\n",
            "{'loss': 0.8114, 'learning_rate': 3e-05, 'epoch': 2.16}\n",
            "{'loss': 0.6695, 'learning_rate': 2.9285714285714288e-05, 'epoch': 2.18}\n",
            "{'loss': 0.6115, 'learning_rate': 2.857142857142857e-05, 'epoch': 2.2}\n",
            "{'loss': 0.9789, 'learning_rate': 2.785714285714286e-05, 'epoch': 2.22}\n",
            "{'loss': 0.9233, 'learning_rate': 2.714285714285714e-05, 'epoch': 2.24}\n",
            "{'loss': 0.8174, 'learning_rate': 2.642857142857143e-05, 'epoch': 2.26}\n",
            "{'loss': 0.6633, 'learning_rate': 2.5714285714285714e-05, 'epoch': 2.28}\n",
            "{'loss': 0.6211, 'learning_rate': 2.5e-05, 'epoch': 2.3}\n",
            "{'loss': 0.9716, 'learning_rate': 2.4285714285714288e-05, 'epoch': 2.32}\n",
            "{'loss': 0.9343, 'learning_rate': 2.357142857142857e-05, 'epoch': 2.34}\n",
            "{'loss': 0.8229, 'learning_rate': 2.2857142857142858e-05, 'epoch': 2.36}\n",
            "{'loss': 0.6706, 'learning_rate': 2.214285714285714e-05, 'epoch': 2.38}\n",
            "{'loss': 0.6203, 'learning_rate': 2.1428571428571428e-05, 'epoch': 2.4}\n",
            " 80% 1200/1500 [1:52:32<15:24,  3.08s/it]\n",
            "  0% 0/250 [00:00<?, ?it/s]\u001b[A\n",
            "  1% 2/250 [00:00<00:40,  6.15it/s]\u001b[A\n",
            "  1% 3/250 [00:00<00:56,  4.41it/s]\u001b[A\n",
            "  2% 4/250 [00:01<01:09,  3.54it/s]\u001b[A\n",
            "  2% 5/250 [00:01<01:14,  3.28it/s]\u001b[A\n",
            "  2% 6/250 [00:01<01:15,  3.25it/s]\u001b[A\n",
            "  3% 7/250 [00:01<01:14,  3.27it/s]\u001b[A\n",
            "  3% 8/250 [00:02<01:18,  3.09it/s]\u001b[A\n",
            "  4% 9/250 [00:02<01:21,  2.94it/s]\u001b[A\n",
            "  4% 10/250 [00:03<01:27,  2.74it/s]\u001b[A\n",
            "  4% 11/250 [00:03<01:24,  2.83it/s]\u001b[A\n",
            "  5% 12/250 [00:03<01:23,  2.84it/s]\u001b[A\n",
            "  5% 13/250 [00:04<01:20,  2.96it/s]\u001b[A\n",
            "  6% 14/250 [00:04<01:18,  3.02it/s]\u001b[A\n",
            "  6% 15/250 [00:04<01:18,  2.98it/s]\u001b[A\n",
            "  6% 16/250 [00:05<01:17,  3.03it/s]\u001b[A\n",
            "  7% 17/250 [00:05<01:17,  3.01it/s]\u001b[A\n",
            "  7% 18/250 [00:05<01:17,  2.98it/s]\u001b[A\n",
            "  8% 19/250 [00:06<01:17,  2.98it/s]\u001b[A\n",
            "  8% 20/250 [00:06<01:18,  2.94it/s]\u001b[A\n",
            "  8% 21/250 [00:06<01:16,  2.98it/s]\u001b[A\n",
            "  9% 22/250 [00:07<01:17,  2.96it/s]\u001b[A\n",
            "  9% 23/250 [00:07<01:13,  3.07it/s]\u001b[A\n",
            " 10% 24/250 [00:07<01:15,  2.97it/s]\u001b[A\n",
            " 10% 25/250 [00:08<01:23,  2.69it/s]\u001b[A\n",
            " 10% 26/250 [00:08<01:24,  2.64it/s]\u001b[A\n",
            " 11% 27/250 [00:08<01:20,  2.78it/s]\u001b[A\n",
            " 11% 28/250 [00:09<01:16,  2.90it/s]\u001b[A\n",
            " 12% 29/250 [00:09<01:15,  2.94it/s]\u001b[A\n",
            " 12% 30/250 [00:09<01:14,  2.95it/s]\u001b[A\n",
            " 12% 31/250 [00:10<01:12,  3.02it/s]\u001b[A\n",
            " 13% 32/250 [00:10<01:11,  3.04it/s]\u001b[A\n",
            " 13% 33/250 [00:10<01:12,  2.99it/s]\u001b[A\n",
            " 14% 34/250 [00:11<01:10,  3.06it/s]\u001b[A\n",
            " 14% 35/250 [00:11<01:09,  3.07it/s]\u001b[A\n",
            " 14% 36/250 [00:11<01:15,  2.85it/s]\u001b[A\n",
            " 15% 37/250 [00:12<01:13,  2.90it/s]\u001b[A\n",
            " 15% 38/250 [00:12<01:11,  2.96it/s]\u001b[A\n",
            " 16% 39/250 [00:12<01:12,  2.91it/s]\u001b[A\n",
            " 16% 40/250 [00:13<01:16,  2.73it/s]\u001b[A\n",
            " 16% 41/250 [00:13<01:15,  2.77it/s]\u001b[A\n",
            " 17% 42/250 [00:14<01:14,  2.79it/s]\u001b[A\n",
            " 17% 43/250 [00:14<01:15,  2.75it/s]\u001b[A\n",
            " 18% 44/250 [00:14<01:12,  2.85it/s]\u001b[A\n",
            " 18% 45/250 [00:15<01:13,  2.80it/s]\u001b[A\n",
            " 18% 46/250 [00:15<01:10,  2.89it/s]\u001b[A\n",
            " 19% 47/250 [00:15<01:12,  2.81it/s]\u001b[A\n",
            " 19% 48/250 [00:16<01:10,  2.88it/s]\u001b[A\n",
            " 20% 49/250 [00:16<01:08,  2.95it/s]\u001b[A\n",
            " 20% 50/250 [00:16<01:08,  2.92it/s]\u001b[A\n",
            " 20% 51/250 [00:17<01:06,  3.01it/s]\u001b[A\n",
            " 21% 52/250 [00:17<01:05,  3.01it/s]\u001b[A\n",
            " 21% 53/250 [00:17<01:08,  2.87it/s]\u001b[A\n",
            " 22% 54/250 [00:18<01:08,  2.86it/s]\u001b[A\n",
            " 22% 55/250 [00:18<01:06,  2.93it/s]\u001b[A\n",
            " 22% 56/250 [00:18<01:06,  2.93it/s]\u001b[A\n",
            " 23% 57/250 [00:19<01:06,  2.90it/s]\u001b[A\n",
            " 23% 58/250 [00:19<01:04,  3.00it/s]\u001b[A\n",
            " 24% 59/250 [00:19<01:05,  2.93it/s]\u001b[A\n",
            " 24% 60/250 [00:20<01:15,  2.52it/s]\u001b[A\n",
            " 24% 61/250 [00:20<01:10,  2.69it/s]\u001b[A\n",
            " 25% 62/250 [00:21<01:12,  2.58it/s]\u001b[A\n",
            " 25% 63/250 [00:21<01:07,  2.76it/s]\u001b[A\n",
            " 26% 64/250 [00:21<01:06,  2.80it/s]\u001b[A\n",
            " 26% 65/250 [00:22<01:07,  2.74it/s]\u001b[A\n",
            " 26% 66/250 [00:22<01:06,  2.77it/s]\u001b[A\n",
            " 27% 67/250 [00:22<01:04,  2.82it/s]\u001b[A\n",
            " 27% 68/250 [00:23<01:01,  2.95it/s]\u001b[A\n",
            " 28% 69/250 [00:23<01:10,  2.55it/s]\u001b[A\n",
            " 28% 70/250 [00:24<01:10,  2.56it/s]\u001b[A\n",
            " 28% 71/250 [00:24<01:05,  2.71it/s]\u001b[A\n",
            " 29% 72/250 [00:24<01:03,  2.81it/s]\u001b[A\n",
            " 29% 73/250 [00:25<01:02,  2.85it/s]\u001b[A\n",
            " 30% 74/250 [00:25<01:09,  2.54it/s]\u001b[A\n",
            " 30% 75/250 [00:25<01:06,  2.64it/s]\u001b[A\n",
            " 30% 76/250 [00:26<01:04,  2.69it/s]\u001b[A\n",
            " 31% 77/250 [00:26<01:01,  2.83it/s]\u001b[A\n",
            " 31% 78/250 [00:26<00:58,  2.93it/s]\u001b[A\n",
            " 32% 79/250 [00:27<01:03,  2.68it/s]\u001b[A\n",
            " 32% 80/250 [00:27<01:06,  2.56it/s]\u001b[A\n",
            " 32% 81/250 [00:28<01:02,  2.72it/s]\u001b[A\n",
            " 33% 82/250 [00:28<00:59,  2.84it/s]\u001b[A\n",
            " 33% 83/250 [00:28<01:01,  2.71it/s]\u001b[A\n",
            " 34% 84/250 [00:29<01:00,  2.75it/s]\u001b[A\n",
            " 34% 85/250 [00:29<00:58,  2.82it/s]\u001b[A\n",
            " 34% 86/250 [00:29<01:03,  2.57it/s]\u001b[A\n",
            " 35% 87/250 [00:30<01:01,  2.66it/s]\u001b[A\n",
            " 35% 88/250 [00:30<01:03,  2.57it/s]\u001b[A\n",
            " 36% 89/250 [00:31<00:58,  2.73it/s]\u001b[A\n",
            " 36% 90/250 [00:31<01:07,  2.36it/s]\u001b[A\n",
            " 36% 91/250 [00:31<01:03,  2.52it/s]\u001b[A\n",
            " 37% 92/250 [00:32<00:59,  2.65it/s]\u001b[A\n",
            " 37% 93/250 [00:32<01:00,  2.58it/s]\u001b[A\n",
            " 38% 94/250 [00:33<00:59,  2.62it/s]\u001b[A\n",
            " 38% 95/250 [00:33<00:59,  2.61it/s]\u001b[A\n",
            " 38% 96/250 [00:34<01:07,  2.29it/s]\u001b[A\n",
            " 39% 97/250 [00:34<01:01,  2.51it/s]\u001b[A\n",
            " 39% 98/250 [00:34<00:56,  2.68it/s]\u001b[A\n",
            " 40% 99/250 [00:34<00:53,  2.84it/s]\u001b[A\n",
            " 40% 100/250 [00:35<00:51,  2.89it/s]\u001b[A\n",
            " 40% 101/250 [00:35<00:50,  2.94it/s]\u001b[A\n",
            " 41% 102/250 [00:35<00:50,  2.92it/s]\u001b[A\n",
            " 41% 103/250 [00:36<00:51,  2.87it/s]\u001b[A\n",
            " 42% 104/250 [00:36<00:50,  2.87it/s]\u001b[A\n",
            " 42% 105/250 [00:36<00:49,  2.96it/s]\u001b[A\n",
            " 42% 106/250 [00:37<00:55,  2.57it/s]\u001b[A\n",
            " 43% 107/250 [00:37<00:58,  2.43it/s]\u001b[A\n",
            " 43% 108/250 [00:38<00:54,  2.59it/s]\u001b[A\n",
            " 44% 109/250 [00:38<00:51,  2.71it/s]\u001b[A\n",
            " 44% 110/250 [00:38<00:51,  2.71it/s]\u001b[A\n",
            " 44% 111/250 [00:39<00:50,  2.76it/s]\u001b[A\n",
            " 45% 112/250 [00:39<00:52,  2.64it/s]\u001b[A\n",
            " 45% 113/250 [00:40<00:49,  2.74it/s]\u001b[A\n",
            " 46% 114/250 [00:40<00:47,  2.84it/s]\u001b[A\n",
            " 46% 115/250 [00:40<00:49,  2.75it/s]\u001b[A\n",
            " 46% 116/250 [00:41<00:51,  2.60it/s]\u001b[A\n",
            " 47% 117/250 [00:41<00:49,  2.67it/s]\u001b[A\n",
            " 47% 118/250 [00:41<00:50,  2.60it/s]\u001b[A\n",
            " 48% 119/250 [00:42<00:49,  2.67it/s]\u001b[A\n",
            " 48% 120/250 [00:42<00:47,  2.71it/s]\u001b[A\n",
            " 48% 121/250 [00:43<00:46,  2.75it/s]\u001b[A\n",
            " 49% 122/250 [00:43<00:45,  2.82it/s]\u001b[A\n",
            " 49% 123/250 [00:43<00:43,  2.91it/s]\u001b[A\n",
            " 50% 124/250 [00:43<00:42,  3.00it/s]\u001b[A\n",
            " 50% 125/250 [00:44<00:41,  3.05it/s]\u001b[A\n",
            " 50% 126/250 [00:44<00:40,  3.07it/s]\u001b[A\n",
            " 51% 127/250 [00:44<00:41,  2.99it/s]\u001b[A\n",
            " 51% 128/250 [00:45<00:49,  2.48it/s]\u001b[A\n",
            " 52% 129/250 [00:45<00:45,  2.64it/s]\u001b[A\n",
            " 52% 130/250 [00:46<00:46,  2.57it/s]\u001b[A\n",
            " 52% 131/250 [00:46<00:46,  2.57it/s]\u001b[A\n",
            " 53% 132/250 [00:47<00:44,  2.64it/s]\u001b[A\n",
            " 53% 133/250 [00:47<00:44,  2.65it/s]\u001b[A\n",
            " 54% 134/250 [00:47<00:43,  2.64it/s]\u001b[A\n",
            " 54% 135/250 [00:48<00:41,  2.74it/s]\u001b[A\n",
            " 54% 136/250 [00:48<00:39,  2.88it/s]\u001b[A\n",
            " 55% 137/250 [00:48<00:40,  2.78it/s]\u001b[A\n",
            " 55% 138/250 [00:49<00:38,  2.90it/s]\u001b[A\n",
            " 56% 139/250 [00:49<00:40,  2.73it/s]\u001b[A\n",
            " 56% 140/250 [00:49<00:38,  2.83it/s]\u001b[A\n",
            " 56% 141/250 [00:50<00:37,  2.94it/s]\u001b[A\n",
            " 57% 142/250 [00:50<00:43,  2.51it/s]\u001b[A\n",
            " 57% 143/250 [00:51<00:42,  2.54it/s]\u001b[A\n",
            " 58% 144/250 [00:51<00:39,  2.66it/s]\u001b[A\n",
            " 58% 145/250 [00:51<00:38,  2.72it/s]\u001b[A\n",
            " 58% 146/250 [00:52<00:36,  2.85it/s]\u001b[A\n",
            " 59% 147/250 [00:52<00:35,  2.91it/s]\u001b[A\n",
            " 59% 148/250 [00:52<00:33,  3.01it/s]\u001b[A\n",
            " 60% 149/250 [00:53<00:34,  2.94it/s]\u001b[A\n",
            " 60% 150/250 [00:53<00:33,  3.00it/s]\u001b[A\n",
            " 60% 151/250 [00:53<00:33,  2.97it/s]\u001b[A\n",
            " 61% 152/250 [00:54<00:34,  2.88it/s]\u001b[A\n",
            " 61% 153/250 [00:54<00:33,  2.93it/s]\u001b[A\n",
            " 62% 154/250 [00:54<00:33,  2.88it/s]\u001b[A\n",
            " 62% 155/250 [00:55<00:31,  3.00it/s]\u001b[A\n",
            " 62% 156/250 [00:55<00:34,  2.75it/s]\u001b[A\n",
            " 63% 157/250 [00:55<00:33,  2.77it/s]\u001b[A\n",
            " 63% 158/250 [00:56<00:33,  2.71it/s]\u001b[A\n",
            " 64% 159/250 [00:56<00:32,  2.83it/s]\u001b[A\n",
            " 64% 160/250 [00:56<00:31,  2.84it/s]\u001b[A\n",
            " 64% 161/250 [00:57<00:31,  2.84it/s]\u001b[A\n",
            " 65% 162/250 [00:57<00:30,  2.87it/s]\u001b[A\n",
            " 65% 163/250 [00:57<00:30,  2.86it/s]\u001b[A\n",
            " 66% 164/250 [00:58<00:30,  2.84it/s]\u001b[A\n",
            " 66% 165/250 [00:58<00:29,  2.84it/s]\u001b[A\n",
            " 66% 166/250 [00:59<00:30,  2.76it/s]\u001b[A\n",
            " 67% 167/250 [00:59<00:28,  2.88it/s]\u001b[A\n",
            " 67% 168/250 [00:59<00:28,  2.87it/s]\u001b[A\n",
            " 68% 169/250 [01:00<00:30,  2.68it/s]\u001b[A\n",
            " 68% 170/250 [01:00<00:29,  2.70it/s]\u001b[A\n",
            " 68% 171/250 [01:00<00:27,  2.85it/s]\u001b[A\n",
            " 69% 172/250 [01:01<00:26,  2.94it/s]\u001b[A\n",
            " 69% 173/250 [01:01<00:26,  2.93it/s]\u001b[A\n",
            " 70% 174/250 [01:01<00:25,  2.98it/s]\u001b[A\n",
            " 70% 175/250 [01:02<00:24,  3.03it/s]\u001b[A\n",
            " 70% 176/250 [01:02<00:24,  3.02it/s]\u001b[A\n",
            " 71% 177/250 [01:02<00:24,  3.02it/s]\u001b[A\n",
            " 71% 178/250 [01:03<00:24,  2.99it/s]\u001b[A\n",
            " 72% 179/250 [01:03<00:23,  3.03it/s]\u001b[A\n",
            " 72% 180/250 [01:03<00:23,  2.95it/s]\u001b[A\n",
            " 72% 181/250 [01:04<00:22,  3.04it/s]\u001b[A\n",
            " 73% 182/250 [01:04<00:22,  3.05it/s]\u001b[A\n",
            " 73% 183/250 [01:04<00:22,  2.99it/s]\u001b[A\n",
            " 74% 184/250 [01:05<00:22,  2.93it/s]\u001b[A\n",
            " 74% 185/250 [01:05<00:22,  2.94it/s]\u001b[A\n",
            " 74% 186/250 [01:05<00:21,  3.01it/s]\u001b[A\n",
            " 75% 187/250 [01:06<00:20,  3.01it/s]\u001b[A\n",
            " 75% 188/250 [01:06<00:20,  3.00it/s]\u001b[A\n",
            " 76% 189/250 [01:06<00:20,  2.97it/s]\u001b[A\n",
            " 76% 190/250 [01:07<00:19,  3.02it/s]\u001b[A\n",
            " 76% 191/250 [01:07<00:19,  3.05it/s]\u001b[A\n",
            " 77% 192/250 [01:07<00:19,  2.93it/s]\u001b[A\n",
            " 77% 193/250 [01:08<00:19,  2.95it/s]\u001b[A\n",
            " 78% 194/250 [01:08<00:18,  3.04it/s]\u001b[A\n",
            " 78% 195/250 [01:08<00:17,  3.06it/s]\u001b[A\n",
            " 78% 196/250 [01:09<00:17,  3.09it/s]\u001b[A\n",
            " 79% 197/250 [01:09<00:17,  3.00it/s]\u001b[A\n",
            " 79% 198/250 [01:09<00:18,  2.88it/s]\u001b[A\n",
            " 80% 199/250 [01:10<00:17,  2.94it/s]\u001b[A\n",
            " 80% 200/250 [01:10<00:17,  2.89it/s]\u001b[A\n",
            " 80% 201/250 [01:10<00:17,  2.85it/s]\u001b[A\n",
            " 81% 202/250 [01:11<00:16,  2.82it/s]\u001b[A\n",
            " 81% 203/250 [01:11<00:16,  2.83it/s]\u001b[A\n",
            " 82% 204/250 [01:12<00:17,  2.65it/s]\u001b[A\n",
            " 82% 205/250 [01:12<00:16,  2.69it/s]\u001b[A\n",
            " 82% 206/250 [01:12<00:15,  2.77it/s]\u001b[A\n",
            " 83% 207/250 [01:13<00:15,  2.76it/s]\u001b[A\n",
            " 83% 208/250 [01:13<00:14,  2.80it/s]\u001b[A\n",
            " 84% 209/250 [01:13<00:14,  2.80it/s]\u001b[A\n",
            " 84% 210/250 [01:14<00:13,  2.92it/s]\u001b[A\n",
            " 84% 211/250 [01:14<00:13,  2.97it/s]\u001b[A\n",
            " 85% 212/250 [01:14<00:12,  3.06it/s]\u001b[A\n",
            " 85% 213/250 [01:15<00:12,  3.00it/s]\u001b[A\n",
            " 86% 214/250 [01:15<00:11,  3.03it/s]\u001b[A\n",
            " 86% 215/250 [01:15<00:11,  3.01it/s]\u001b[A\n",
            " 86% 216/250 [01:16<00:11,  3.00it/s]\u001b[A\n",
            " 87% 217/250 [01:16<00:11,  2.94it/s]\u001b[A\n",
            " 87% 218/250 [01:16<00:11,  2.91it/s]\u001b[A\n",
            " 88% 219/250 [01:17<00:10,  3.01it/s]\u001b[A\n",
            " 88% 220/250 [01:17<00:09,  3.09it/s]\u001b[A\n",
            " 88% 221/250 [01:17<00:09,  3.10it/s]\u001b[A\n",
            " 89% 222/250 [01:18<00:09,  2.94it/s]\u001b[A\n",
            " 89% 223/250 [01:18<00:09,  2.90it/s]\u001b[A\n",
            " 90% 224/250 [01:18<00:08,  3.01it/s]\u001b[A\n",
            " 90% 225/250 [01:19<00:08,  2.93it/s]\u001b[A\n",
            " 90% 226/250 [01:19<00:08,  2.92it/s]\u001b[A\n",
            " 91% 227/250 [01:19<00:07,  2.99it/s]\u001b[A\n",
            " 91% 228/250 [01:20<00:07,  2.96it/s]\u001b[A\n",
            " 92% 229/250 [01:20<00:07,  2.74it/s]\u001b[A\n",
            " 92% 230/250 [01:20<00:07,  2.78it/s]\u001b[A\n",
            " 92% 231/250 [01:21<00:07,  2.65it/s]\u001b[A\n",
            " 93% 232/250 [01:21<00:07,  2.32it/s]\u001b[A\n",
            " 93% 233/250 [01:22<00:06,  2.47it/s]\u001b[A\n",
            " 94% 234/250 [01:22<00:06,  2.64it/s]\u001b[A\n",
            " 94% 235/250 [01:22<00:05,  2.76it/s]\u001b[A\n",
            " 94% 236/250 [01:23<00:05,  2.37it/s]\u001b[A\n",
            " 95% 237/250 [01:23<00:05,  2.41it/s]\u001b[A\n",
            " 95% 238/250 [01:24<00:04,  2.40it/s]\u001b[A\n",
            " 96% 239/250 [01:24<00:04,  2.47it/s]\u001b[A\n",
            " 96% 240/250 [01:24<00:03,  2.56it/s]\u001b[A\n",
            " 96% 241/250 [01:25<00:03,  2.61it/s]\u001b[A\n",
            " 97% 242/250 [01:25<00:03,  2.59it/s]\u001b[A\n",
            " 97% 243/250 [01:26<00:02,  2.77it/s]\u001b[A\n",
            " 98% 244/250 [01:26<00:02,  2.84it/s]\u001b[A\n",
            " 98% 245/250 [01:26<00:01,  2.96it/s]\u001b[A\n",
            " 98% 246/250 [01:27<00:01,  2.98it/s]\u001b[A\n",
            " 99% 247/250 [01:27<00:00,  3.03it/s]\u001b[A\n",
            " 99% 248/250 [01:27<00:00,  2.93it/s]\u001b[A\n",
            "100% 249/250 [01:28<00:00,  2.83it/s]\u001b[A\n",
            "100% 250/250 [01:28<00:00,  2.85it/s]\u001b[A\n",
            "{'eval_loss': 0.8636467456817627, 'eval_runtime': 88.8537, 'eval_samples_per_second': 22.509, 'eval_steps_per_second': 2.814, 'epoch': 2.4}\n",
            "\n",
            " 80% 1200/1500 [1:54:01<15:24,  3.08s/it]\n",
            "{'loss': 0.9576, 'learning_rate': 2.0714285714285718e-05, 'epoch': 2.42}\n",
            "{'loss': 0.9216, 'learning_rate': 2e-05, 'epoch': 2.44}\n",
            "{'loss': 0.7945, 'learning_rate': 1.928571428571429e-05, 'epoch': 2.46}\n",
            "{'loss': 0.6735, 'learning_rate': 1.8571428571428572e-05, 'epoch': 2.48}\n",
            "{'loss': 0.6116, 'learning_rate': 1.785714285714286e-05, 'epoch': 2.5}\n",
            "{'loss': 0.9678, 'learning_rate': 1.7142857142857145e-05, 'epoch': 2.52}\n",
            "{'loss': 0.9235, 'learning_rate': 1.642857142857143e-05, 'epoch': 2.54}\n",
            "{'loss': 0.8019, 'learning_rate': 1.5714285714285715e-05, 'epoch': 2.56}\n",
            "{'loss': 0.6657, 'learning_rate': 1.5e-05, 'epoch': 2.58}\n",
            "{'loss': 0.6063, 'learning_rate': 1.4285714285714285e-05, 'epoch': 2.6}\n",
            "{'loss': 0.9531, 'learning_rate': 1.357142857142857e-05, 'epoch': 2.62}\n",
            "{'loss': 0.9374, 'learning_rate': 1.2857142857142857e-05, 'epoch': 2.64}\n",
            "{'loss': 0.8106, 'learning_rate': 1.2142857142857144e-05, 'epoch': 2.66}\n",
            "{'loss': 0.6609, 'learning_rate': 1.1428571428571429e-05, 'epoch': 2.68}\n",
            "{'loss': 0.6144, 'learning_rate': 1.0714285714285714e-05, 'epoch': 2.7}\n",
            "{'loss': 0.9598, 'learning_rate': 1e-05, 'epoch': 2.72}\n",
            "{'loss': 0.9326, 'learning_rate': 9.285714285714286e-06, 'epoch': 2.74}\n",
            "{'loss': 0.7993, 'learning_rate': 8.571428571428573e-06, 'epoch': 2.76}\n",
            "{'loss': 0.6691, 'learning_rate': 7.857142857142858e-06, 'epoch': 2.78}\n",
            "{'loss': 0.623, 'learning_rate': 7.142857142857143e-06, 'epoch': 2.8}\n",
            " 93% 1400/1500 [2:11:24<05:08,  3.09s/it]\n",
            "  0% 0/250 [00:00<?, ?it/s]\u001b[A\n",
            "  1% 2/250 [00:00<00:39,  6.25it/s]\u001b[A\n",
            "  1% 3/250 [00:00<00:54,  4.50it/s]\u001b[A\n",
            "  2% 4/250 [00:01<01:09,  3.56it/s]\u001b[A\n",
            "  2% 5/250 [00:01<01:14,  3.28it/s]\u001b[A\n",
            "  2% 6/250 [00:01<01:15,  3.25it/s]\u001b[A\n",
            "  3% 7/250 [00:01<01:14,  3.27it/s]\u001b[A\n",
            "  3% 8/250 [00:02<01:18,  3.10it/s]\u001b[A\n",
            "  4% 9/250 [00:02<01:21,  2.95it/s]\u001b[A\n",
            "  4% 10/250 [00:03<01:27,  2.74it/s]\u001b[A\n",
            "  4% 11/250 [00:03<01:24,  2.83it/s]\u001b[A\n",
            "  5% 12/250 [00:03<01:23,  2.84it/s]\u001b[A\n",
            "  5% 13/250 [00:04<01:19,  2.97it/s]\u001b[A\n",
            "  6% 14/250 [00:04<01:17,  3.06it/s]\u001b[A\n",
            "  6% 15/250 [00:04<01:17,  3.02it/s]\u001b[A\n",
            "  6% 16/250 [00:05<01:16,  3.07it/s]\u001b[A\n",
            "  7% 17/250 [00:05<01:17,  3.02it/s]\u001b[A\n",
            "  7% 18/250 [00:05<01:17,  2.98it/s]\u001b[A\n",
            "  8% 19/250 [00:06<01:17,  2.97it/s]\u001b[A\n",
            "  8% 20/250 [00:06<01:18,  2.93it/s]\u001b[A\n",
            "  8% 21/250 [00:06<01:17,  2.97it/s]\u001b[A\n",
            "  9% 22/250 [00:07<01:17,  2.94it/s]\u001b[A\n",
            "  9% 23/250 [00:07<01:14,  3.05it/s]\u001b[A\n",
            " 10% 24/250 [00:07<01:16,  2.96it/s]\u001b[A\n",
            " 10% 25/250 [00:08<01:23,  2.69it/s]\u001b[A\n",
            " 10% 26/250 [00:08<01:24,  2.65it/s]\u001b[A\n",
            " 11% 27/250 [00:08<01:20,  2.77it/s]\u001b[A\n",
            " 11% 28/250 [00:09<01:16,  2.89it/s]\u001b[A\n",
            " 12% 29/250 [00:09<01:15,  2.93it/s]\u001b[A\n",
            " 12% 30/250 [00:09<01:14,  2.95it/s]\u001b[A\n",
            " 12% 31/250 [00:10<01:12,  3.03it/s]\u001b[A\n",
            " 13% 32/250 [00:10<01:11,  3.06it/s]\u001b[A\n",
            " 13% 33/250 [00:10<01:11,  3.03it/s]\u001b[A\n",
            " 14% 34/250 [00:11<01:09,  3.12it/s]\u001b[A\n",
            " 14% 35/250 [00:11<01:08,  3.13it/s]\u001b[A\n",
            " 14% 36/250 [00:11<01:13,  2.92it/s]\u001b[A\n",
            " 15% 37/250 [00:12<01:11,  2.99it/s]\u001b[A\n",
            " 15% 38/250 [00:12<01:09,  3.04it/s]\u001b[A\n",
            " 16% 39/250 [00:12<01:11,  2.97it/s]\u001b[A\n",
            " 16% 40/250 [00:13<01:15,  2.79it/s]\u001b[A\n",
            " 16% 41/250 [00:13<01:14,  2.81it/s]\u001b[A\n",
            " 17% 42/250 [00:14<01:14,  2.81it/s]\u001b[A\n",
            " 17% 43/250 [00:14<01:14,  2.77it/s]\u001b[A\n",
            " 18% 44/250 [00:14<01:11,  2.88it/s]\u001b[A\n",
            " 18% 45/250 [00:15<01:12,  2.83it/s]\u001b[A\n",
            " 18% 46/250 [00:15<01:09,  2.92it/s]\u001b[A\n",
            " 19% 47/250 [00:15<01:11,  2.83it/s]\u001b[A\n",
            " 19% 48/250 [00:16<01:09,  2.90it/s]\u001b[A\n",
            " 20% 49/250 [00:16<01:07,  2.97it/s]\u001b[A\n",
            " 20% 50/250 [00:16<01:07,  2.94it/s]\u001b[A\n",
            " 20% 51/250 [00:17<01:05,  3.02it/s]\u001b[A\n",
            " 21% 52/250 [00:17<01:05,  3.03it/s]\u001b[A\n",
            " 21% 53/250 [00:17<01:08,  2.90it/s]\u001b[A\n",
            " 22% 54/250 [00:18<01:08,  2.87it/s]\u001b[A\n",
            " 22% 55/250 [00:18<01:06,  2.95it/s]\u001b[A\n",
            " 22% 56/250 [00:18<01:05,  2.94it/s]\u001b[A\n",
            " 23% 57/250 [00:19<01:06,  2.90it/s]\u001b[A\n",
            " 23% 58/250 [00:19<01:04,  3.00it/s]\u001b[A\n",
            " 24% 59/250 [00:19<01:05,  2.92it/s]\u001b[A\n",
            " 24% 60/250 [00:20<01:15,  2.52it/s]\u001b[A\n",
            " 24% 61/250 [00:20<01:10,  2.67it/s]\u001b[A\n",
            " 25% 62/250 [00:21<01:13,  2.56it/s]\u001b[A\n",
            " 25% 63/250 [00:21<01:08,  2.74it/s]\u001b[A\n",
            " 26% 64/250 [00:21<01:06,  2.78it/s]\u001b[A\n",
            " 26% 65/250 [00:22<01:08,  2.71it/s]\u001b[A\n",
            " 26% 66/250 [00:22<01:07,  2.74it/s]\u001b[A\n",
            " 27% 67/250 [00:22<01:05,  2.79it/s]\u001b[A\n",
            " 27% 68/250 [00:23<01:02,  2.92it/s]\u001b[A\n",
            " 28% 69/250 [00:23<01:10,  2.55it/s]\u001b[A\n",
            " 28% 70/250 [00:24<01:10,  2.57it/s]\u001b[A\n",
            " 28% 71/250 [00:24<01:05,  2.72it/s]\u001b[A\n",
            " 29% 72/250 [00:24<01:03,  2.82it/s]\u001b[A\n",
            " 29% 73/250 [00:24<01:01,  2.87it/s]\u001b[A\n",
            " 30% 74/250 [00:25<01:08,  2.57it/s]\u001b[A\n",
            " 30% 75/250 [00:25<01:05,  2.69it/s]\u001b[A\n",
            " 30% 76/250 [00:26<01:03,  2.74it/s]\u001b[A\n",
            " 31% 77/250 [00:26<00:59,  2.89it/s]\u001b[A\n",
            " 31% 78/250 [00:26<00:57,  2.98it/s]\u001b[A\n",
            " 32% 79/250 [00:27<01:03,  2.71it/s]\u001b[A\n",
            " 32% 80/250 [00:27<01:05,  2.60it/s]\u001b[A\n",
            " 32% 81/250 [00:27<01:01,  2.76it/s]\u001b[A\n",
            " 33% 82/250 [00:28<00:57,  2.90it/s]\u001b[A\n",
            " 33% 83/250 [00:28<01:00,  2.76it/s]\u001b[A\n",
            " 34% 84/250 [00:28<00:59,  2.79it/s]\u001b[A\n",
            " 34% 85/250 [00:29<00:57,  2.86it/s]\u001b[A\n",
            " 34% 86/250 [00:29<01:03,  2.60it/s]\u001b[A\n",
            " 35% 87/250 [00:30<01:00,  2.70it/s]\u001b[A\n",
            " 35% 88/250 [00:30<01:02,  2.60it/s]\u001b[A\n",
            " 36% 89/250 [00:30<00:58,  2.77it/s]\u001b[A\n",
            " 36% 90/250 [00:31<01:07,  2.38it/s]\u001b[A\n",
            " 36% 91/250 [00:31<01:02,  2.54it/s]\u001b[A\n",
            " 37% 92/250 [00:32<00:59,  2.66it/s]\u001b[A\n",
            " 37% 93/250 [00:32<01:00,  2.59it/s]\u001b[A\n",
            " 38% 94/250 [00:32<00:59,  2.63it/s]\u001b[A\n",
            " 38% 95/250 [00:33<00:59,  2.62it/s]\u001b[A\n",
            " 38% 96/250 [00:33<01:07,  2.30it/s]\u001b[A\n",
            " 39% 97/250 [00:34<01:01,  2.51it/s]\u001b[A\n",
            " 39% 98/250 [00:34<00:56,  2.68it/s]\u001b[A\n",
            " 40% 99/250 [00:34<00:52,  2.85it/s]\u001b[A\n",
            " 40% 100/250 [00:35<00:51,  2.91it/s]\u001b[A\n",
            " 40% 101/250 [00:35<00:50,  2.96it/s]\u001b[A\n",
            " 41% 102/250 [00:35<00:50,  2.92it/s]\u001b[A\n",
            " 41% 103/250 [00:36<00:51,  2.85it/s]\u001b[A\n",
            " 42% 104/250 [00:36<00:50,  2.87it/s]\u001b[A\n",
            " 42% 105/250 [00:36<00:48,  2.96it/s]\u001b[A\n",
            " 42% 106/250 [00:37<00:55,  2.58it/s]\u001b[A\n",
            " 43% 107/250 [00:37<00:58,  2.45it/s]\u001b[A\n",
            " 43% 108/250 [00:38<00:53,  2.63it/s]\u001b[A\n",
            " 44% 109/250 [00:38<00:50,  2.79it/s]\u001b[A\n",
            " 44% 110/250 [00:38<00:50,  2.79it/s]\u001b[A\n",
            " 44% 111/250 [00:39<00:48,  2.87it/s]\u001b[A\n",
            " 45% 112/250 [00:39<00:50,  2.74it/s]\u001b[A\n",
            " 45% 113/250 [00:39<00:48,  2.85it/s]\u001b[A\n",
            " 46% 114/250 [00:40<00:46,  2.93it/s]\u001b[A\n",
            " 46% 115/250 [00:40<00:47,  2.82it/s]\u001b[A\n",
            " 46% 116/250 [00:40<00:50,  2.65it/s]\u001b[A\n",
            " 47% 117/250 [00:41<00:48,  2.73it/s]\u001b[A\n",
            " 47% 118/250 [00:41<00:49,  2.67it/s]\u001b[A\n",
            " 48% 119/250 [00:41<00:47,  2.74it/s]\u001b[A\n",
            " 48% 120/250 [00:42<00:46,  2.77it/s]\u001b[A\n",
            " 48% 121/250 [00:42<00:46,  2.80it/s]\u001b[A\n",
            " 49% 122/250 [00:42<00:44,  2.88it/s]\u001b[A\n",
            " 49% 123/250 [00:43<00:42,  2.99it/s]\u001b[A\n",
            " 50% 124/250 [00:43<00:41,  3.07it/s]\u001b[A\n",
            " 50% 125/250 [00:43<00:40,  3.11it/s]\u001b[A\n",
            " 50% 126/250 [00:44<00:39,  3.13it/s]\u001b[A\n",
            " 51% 127/250 [00:44<00:40,  3.03it/s]\u001b[A\n",
            " 51% 128/250 [00:45<00:48,  2.51it/s]\u001b[A\n",
            " 52% 129/250 [00:45<00:45,  2.67it/s]\u001b[A\n",
            " 52% 130/250 [00:45<00:46,  2.60it/s]\u001b[A\n",
            " 52% 131/250 [00:46<00:45,  2.60it/s]\u001b[A\n",
            " 53% 132/250 [00:46<00:44,  2.67it/s]\u001b[A\n",
            " 53% 133/250 [00:46<00:43,  2.68it/s]\u001b[A\n",
            " 54% 134/250 [00:47<00:43,  2.67it/s]\u001b[A\n",
            " 54% 135/250 [00:47<00:41,  2.76it/s]\u001b[A\n",
            " 54% 136/250 [00:47<00:39,  2.89it/s]\u001b[A\n",
            " 55% 137/250 [00:48<00:40,  2.79it/s]\u001b[A\n",
            " 55% 138/250 [00:48<00:38,  2.92it/s]\u001b[A\n",
            " 56% 139/250 [00:49<00:40,  2.75it/s]\u001b[A\n",
            " 56% 140/250 [00:49<00:38,  2.85it/s]\u001b[A\n",
            " 56% 141/250 [00:49<00:36,  2.95it/s]\u001b[A\n",
            " 57% 142/250 [00:50<00:42,  2.52it/s]\u001b[A\n",
            " 57% 143/250 [00:50<00:41,  2.55it/s]\u001b[A\n",
            " 58% 144/250 [00:50<00:39,  2.67it/s]\u001b[A\n",
            " 58% 145/250 [00:51<00:38,  2.73it/s]\u001b[A\n",
            " 58% 146/250 [00:51<00:36,  2.86it/s]\u001b[A\n",
            " 59% 147/250 [00:51<00:35,  2.92it/s]\u001b[A\n",
            " 59% 148/250 [00:52<00:33,  3.03it/s]\u001b[A\n",
            " 60% 149/250 [00:52<00:34,  2.95it/s]\u001b[A\n",
            " 60% 150/250 [00:52<00:33,  3.01it/s]\u001b[A\n",
            " 60% 151/250 [00:53<00:33,  2.97it/s]\u001b[A\n",
            " 61% 152/250 [00:53<00:34,  2.88it/s]\u001b[A\n",
            " 61% 153/250 [00:53<00:33,  2.93it/s]\u001b[A\n",
            " 62% 154/250 [00:54<00:33,  2.89it/s]\u001b[A\n",
            " 62% 155/250 [00:54<00:31,  3.03it/s]\u001b[A\n",
            " 62% 156/250 [00:55<00:34,  2.76it/s]\u001b[A\n",
            " 63% 157/250 [00:55<00:33,  2.80it/s]\u001b[A\n",
            " 63% 158/250 [00:55<00:33,  2.75it/s]\u001b[A\n",
            " 64% 159/250 [00:56<00:31,  2.87it/s]\u001b[A\n",
            " 64% 160/250 [00:56<00:31,  2.89it/s]\u001b[A\n",
            " 64% 161/250 [00:56<00:30,  2.88it/s]\u001b[A\n",
            " 65% 162/250 [00:57<00:30,  2.93it/s]\u001b[A\n",
            " 65% 163/250 [00:57<00:29,  2.95it/s]\u001b[A\n",
            " 66% 164/250 [00:57<00:29,  2.92it/s]\u001b[A\n",
            " 66% 165/250 [00:58<00:29,  2.91it/s]\u001b[A\n",
            " 66% 166/250 [00:58<00:29,  2.82it/s]\u001b[A\n",
            " 67% 167/250 [00:58<00:28,  2.95it/s]\u001b[A\n",
            " 67% 168/250 [00:59<00:27,  2.94it/s]\u001b[A\n",
            " 68% 169/250 [00:59<00:29,  2.73it/s]\u001b[A\n",
            " 68% 170/250 [00:59<00:29,  2.75it/s]\u001b[A\n",
            " 68% 171/250 [01:00<00:27,  2.89it/s]\u001b[A\n",
            " 69% 172/250 [01:00<00:26,  2.98it/s]\u001b[A\n",
            " 69% 173/250 [01:00<00:25,  2.96it/s]\u001b[A\n",
            " 70% 174/250 [01:01<00:25,  3.00it/s]\u001b[A\n",
            " 70% 175/250 [01:01<00:24,  3.04it/s]\u001b[A\n",
            " 70% 176/250 [01:01<00:24,  3.03it/s]\u001b[A\n",
            " 71% 177/250 [01:02<00:24,  3.04it/s]\u001b[A\n",
            " 71% 178/250 [01:02<00:23,  3.01it/s]\u001b[A\n",
            " 72% 179/250 [01:02<00:23,  3.06it/s]\u001b[A\n",
            " 72% 180/250 [01:03<00:23,  2.98it/s]\u001b[A\n",
            " 72% 181/250 [01:03<00:22,  3.07it/s]\u001b[A\n",
            " 73% 182/250 [01:03<00:22,  3.08it/s]\u001b[A\n",
            " 73% 183/250 [01:04<00:22,  3.02it/s]\u001b[A\n",
            " 74% 184/250 [01:04<00:22,  2.96it/s]\u001b[A\n",
            " 74% 185/250 [01:04<00:21,  2.96it/s]\u001b[A\n",
            " 74% 186/250 [01:05<00:21,  3.04it/s]\u001b[A\n",
            " 75% 187/250 [01:05<00:20,  3.02it/s]\u001b[A\n",
            " 75% 188/250 [01:05<00:20,  3.02it/s]\u001b[A\n",
            " 76% 189/250 [01:06<00:20,  2.99it/s]\u001b[A\n",
            " 76% 190/250 [01:06<00:19,  3.04it/s]\u001b[A\n",
            " 76% 191/250 [01:06<00:19,  3.06it/s]\u001b[A\n",
            " 77% 192/250 [01:07<00:19,  2.92it/s]\u001b[A\n",
            " 77% 193/250 [01:07<00:19,  2.93it/s]\u001b[A\n",
            " 78% 194/250 [01:07<00:18,  3.04it/s]\u001b[A\n",
            " 78% 195/250 [01:08<00:17,  3.06it/s]\u001b[A\n",
            " 78% 196/250 [01:08<00:17,  3.10it/s]\u001b[A\n",
            " 79% 197/250 [01:08<00:17,  3.01it/s]\u001b[A\n",
            " 79% 198/250 [01:09<00:18,  2.89it/s]\u001b[A\n",
            " 80% 199/250 [01:09<00:17,  2.95it/s]\u001b[A\n",
            " 80% 200/250 [01:09<00:17,  2.92it/s]\u001b[A\n",
            " 80% 201/250 [01:10<00:16,  2.91it/s]\u001b[A\n",
            " 81% 202/250 [01:10<00:16,  2.94it/s]\u001b[A\n",
            " 81% 203/250 [01:10<00:15,  2.95it/s]\u001b[A\n",
            " 82% 204/250 [01:11<00:16,  2.81it/s]\u001b[A\n",
            " 82% 205/250 [01:11<00:15,  2.85it/s]\u001b[A\n",
            " 82% 206/250 [01:11<00:15,  2.89it/s]\u001b[A\n",
            " 83% 207/250 [01:12<00:15,  2.85it/s]\u001b[A\n",
            " 83% 208/250 [01:12<00:14,  2.87it/s]\u001b[A\n",
            " 84% 209/250 [01:13<00:14,  2.85it/s]\u001b[A\n",
            " 84% 210/250 [01:13<00:13,  2.96it/s]\u001b[A\n",
            " 84% 211/250 [01:13<00:12,  3.01it/s]\u001b[A\n",
            " 85% 212/250 [01:13<00:12,  3.09it/s]\u001b[A\n",
            " 85% 213/250 [01:14<00:12,  3.02it/s]\u001b[A\n",
            " 86% 214/250 [01:14<00:11,  3.06it/s]\u001b[A\n",
            " 86% 215/250 [01:14<00:11,  3.06it/s]\u001b[A\n",
            " 86% 216/250 [01:15<00:11,  3.07it/s]\u001b[A\n",
            " 87% 217/250 [01:15<00:10,  3.01it/s]\u001b[A\n",
            " 87% 218/250 [01:15<00:10,  2.95it/s]\u001b[A\n",
            " 88% 219/250 [01:16<00:10,  3.05it/s]\u001b[A\n",
            " 88% 220/250 [01:16<00:09,  3.12it/s]\u001b[A\n",
            " 88% 221/250 [01:16<00:09,  3.14it/s]\u001b[A\n",
            " 89% 222/250 [01:17<00:09,  2.99it/s]\u001b[A\n",
            " 89% 223/250 [01:17<00:09,  2.95it/s]\u001b[A\n",
            " 90% 224/250 [01:17<00:08,  3.03it/s]\u001b[A\n",
            " 90% 225/250 [01:18<00:08,  2.95it/s]\u001b[A\n",
            " 90% 226/250 [01:18<00:08,  2.94it/s]\u001b[A\n",
            " 91% 227/250 [01:18<00:07,  3.01it/s]\u001b[A\n",
            " 91% 228/250 [01:19<00:07,  2.98it/s]\u001b[A\n",
            " 92% 229/250 [01:19<00:07,  2.76it/s]\u001b[A\n",
            " 92% 230/250 [01:20<00:07,  2.81it/s]\u001b[A\n",
            " 92% 231/250 [01:20<00:07,  2.68it/s]\u001b[A\n",
            " 93% 232/250 [01:21<00:07,  2.33it/s]\u001b[A\n",
            " 93% 233/250 [01:21<00:06,  2.48it/s]\u001b[A\n",
            " 94% 234/250 [01:21<00:06,  2.64it/s]\u001b[A\n",
            " 94% 235/250 [01:22<00:05,  2.78it/s]\u001b[A\n",
            " 94% 236/250 [01:22<00:05,  2.39it/s]\u001b[A\n",
            " 95% 237/250 [01:22<00:05,  2.43it/s]\u001b[A\n",
            " 95% 238/250 [01:23<00:04,  2.42it/s]\u001b[A\n",
            " 96% 239/250 [01:23<00:04,  2.49it/s]\u001b[A\n",
            " 96% 240/250 [01:24<00:03,  2.59it/s]\u001b[A\n",
            " 96% 241/250 [01:24<00:03,  2.65it/s]\u001b[A\n",
            " 97% 242/250 [01:24<00:03,  2.65it/s]\u001b[A\n",
            " 97% 243/250 [01:25<00:02,  2.83it/s]\u001b[A\n",
            " 98% 244/250 [01:25<00:02,  2.90it/s]\u001b[A\n",
            " 98% 245/250 [01:25<00:01,  3.01it/s]\u001b[A\n",
            " 98% 246/250 [01:26<00:01,  3.05it/s]\u001b[A\n",
            " 99% 247/250 [01:26<00:00,  3.11it/s]\u001b[A\n",
            " 99% 248/250 [01:26<00:00,  3.00it/s]\u001b[A\n",
            "100% 249/250 [01:27<00:00,  2.87it/s]\u001b[A\n",
            "100% 250/250 [01:27<00:00,  2.89it/s]\u001b[A\n",
            "{'eval_loss': 0.8630636930465698, 'eval_runtime': 87.923, 'eval_samples_per_second': 22.747, 'eval_steps_per_second': 2.843, 'epoch': 2.8}\n",
            "\n",
            " 93% 1400/1500 [2:12:52<05:08,  3.09s/it]\n",
            "{'loss': 0.9679, 'learning_rate': 6.428571428571429e-06, 'epoch': 2.82}\n",
            "{'loss': 0.9239, 'learning_rate': 5.7142857142857145e-06, 'epoch': 2.84}\n",
            "{'loss': 0.7882, 'learning_rate': 5e-06, 'epoch': 2.86}\n",
            "{'loss': 0.6622, 'learning_rate': 4.285714285714286e-06, 'epoch': 2.88}\n",
            "{'loss': 0.5944, 'learning_rate': 3.5714285714285714e-06, 'epoch': 2.9}\n",
            "{'loss': 0.9536, 'learning_rate': 2.8571428571428573e-06, 'epoch': 2.92}\n",
            "{'loss': 0.9169, 'learning_rate': 2.142857142857143e-06, 'epoch': 2.94}\n",
            "{'loss': 0.8119, 'learning_rate': 1.4285714285714286e-06, 'epoch': 2.96}\n",
            "{'loss': 0.6587, 'learning_rate': 7.142857142857143e-07, 'epoch': 2.98}\n",
            "{'loss': 0.6153, 'learning_rate': 0.0, 'epoch': 3.0}\n",
            "100% 1500/1500 [2:21:38<00:00,  3.10s/it]There were missing keys in the checkpoint model loaded: ['base_model.model.model.embed_tokens.weight', 'base_model.model.model.layers.0.self_attn.q_proj.weight', 'base_model.model.model.layers.0.self_attn.k_proj.weight', 'base_model.model.model.layers.0.self_attn.v_proj.weight', 'base_model.model.model.layers.0.self_attn.o_proj.weight', 'base_model.model.model.layers.0.self_attn.rotary_emb.inv_freq', 'base_model.model.model.layers.0.mlp.gate_proj.weight', 'base_model.model.model.layers.0.mlp.down_proj.weight', 'base_model.model.model.layers.0.mlp.up_proj.weight', 'base_model.model.model.layers.0.input_layernorm.weight', 'base_model.model.model.layers.0.post_attention_layernorm.weight', 'base_model.model.model.layers.1.self_attn.q_proj.weight', 'base_model.model.model.layers.1.self_attn.k_proj.weight', 'base_model.model.model.layers.1.self_attn.v_proj.weight', 'base_model.model.model.layers.1.self_attn.o_proj.weight', 'base_model.model.model.layers.1.self_attn.rotary_emb.inv_freq', 'base_model.model.model.layers.1.mlp.gate_proj.weight', 'base_model.model.model.layers.1.mlp.down_proj.weight', 'base_model.model.model.layers.1.mlp.up_proj.weight', 'base_model.model.model.layers.1.input_layernorm.weight', 'base_model.model.model.layers.1.post_attention_layernorm.weight', 'base_model.model.model.layers.2.self_attn.q_proj.weight', 'base_model.model.model.layers.2.self_attn.k_proj.weight', 'base_model.model.model.layers.2.self_attn.v_proj.weight', 'base_model.model.model.layers.2.self_attn.o_proj.weight', 'base_model.model.model.layers.2.self_attn.rotary_emb.inv_freq', 'base_model.model.model.layers.2.mlp.gate_proj.weight', 'base_model.model.model.layers.2.mlp.down_proj.weight', 'base_model.model.model.layers.2.mlp.up_proj.weight', 'base_model.model.model.layers.2.input_layernorm.weight', 'base_model.model.model.layers.2.post_attention_layernorm.weight', 'base_model.model.model.layers.3.self_attn.q_proj.weight', 'base_model.model.model.layers.3.self_attn.k_proj.weight', 'base_model.model.model.layers.3.self_attn.v_proj.weight', 'base_model.model.model.layers.3.self_attn.o_proj.weight', 'base_model.model.model.layers.3.self_attn.rotary_emb.inv_freq', 'base_model.model.model.layers.3.mlp.gate_proj.weight', 'base_model.model.model.layers.3.mlp.down_proj.weight', 'base_model.model.model.layers.3.mlp.up_proj.weight', 'base_model.model.model.layers.3.input_layernorm.weight', 'base_model.model.model.layers.3.post_attention_layernorm.weight', 'base_model.model.model.layers.4.self_attn.q_proj.weight', 'base_model.model.model.layers.4.self_attn.k_proj.weight', 'base_model.model.model.layers.4.self_attn.v_proj.weight', 'base_model.model.model.layers.4.self_attn.o_proj.weight', 'base_model.model.model.layers.4.self_attn.rotary_emb.inv_freq', 'base_model.model.model.layers.4.mlp.gate_proj.weight', 'base_model.model.model.layers.4.mlp.down_proj.weight', 'base_model.model.model.layers.4.mlp.up_proj.weight', 'base_model.model.model.layers.4.input_layernorm.weight', 'base_model.model.model.layers.4.post_attention_layernorm.weight', 'base_model.model.model.layers.5.self_attn.q_proj.weight', 'base_model.model.model.layers.5.self_attn.k_proj.weight', 'base_model.model.model.layers.5.self_attn.v_proj.weight', 'base_model.model.model.layers.5.self_attn.o_proj.weight', 'base_model.model.model.layers.5.self_attn.rotary_emb.inv_freq', 'base_model.model.model.layers.5.mlp.gate_proj.weight', 'base_model.model.model.layers.5.mlp.down_proj.weight', 'base_model.model.model.layers.5.mlp.up_proj.weight', 'base_model.model.model.layers.5.input_layernorm.weight', 'base_model.model.model.layers.5.post_attention_layernorm.weight', 'base_model.model.model.layers.6.self_attn.q_proj.weight', 'base_model.model.model.layers.6.self_attn.k_proj.weight', 'base_model.model.model.layers.6.self_attn.v_proj.weight', 'base_model.model.model.layers.6.self_attn.o_proj.weight', 'base_model.model.model.layers.6.self_attn.rotary_emb.inv_freq', 'base_model.model.model.layers.6.mlp.gate_proj.weight', 'base_model.model.model.layers.6.mlp.down_proj.weight', 'base_model.model.model.layers.6.mlp.up_proj.weight', 'base_model.model.model.layers.6.input_layernorm.weight', 'base_model.model.model.layers.6.post_attention_layernorm.weight', 'base_model.model.model.layers.7.self_attn.q_proj.weight', 'base_model.model.model.layers.7.self_attn.k_proj.weight', 'base_model.model.model.layers.7.self_attn.v_proj.weight', 'base_model.model.model.layers.7.self_attn.o_proj.weight', 'base_model.model.model.layers.7.self_attn.rotary_emb.inv_freq', 'base_model.model.model.layers.7.mlp.gate_proj.weight', 'base_model.model.model.layers.7.mlp.down_proj.weight', 'base_model.model.model.layers.7.mlp.up_proj.weight', 'base_model.model.model.layers.7.input_layernorm.weight', 'base_model.model.model.layers.7.post_attention_layernorm.weight', 'base_model.model.model.layers.8.self_attn.q_proj.weight', 'base_model.model.model.layers.8.self_attn.k_proj.weight', 'base_model.model.model.layers.8.self_attn.v_proj.weight', 'base_model.model.model.layers.8.self_attn.o_proj.weight', 'base_model.model.model.layers.8.self_attn.rotary_emb.inv_freq', 'base_model.model.model.layers.8.mlp.gate_proj.weight', 'base_model.model.model.layers.8.mlp.down_proj.weight', 'base_model.model.model.layers.8.mlp.up_proj.weight', 'base_model.model.model.layers.8.input_layernorm.weight', 'base_model.model.model.layers.8.post_attention_layernorm.weight', 'base_model.model.model.layers.9.self_attn.q_proj.weight', 'base_model.model.model.layers.9.self_attn.k_proj.weight', 'base_model.model.model.layers.9.self_attn.v_proj.weight', 'base_model.model.model.layers.9.self_attn.o_proj.weight', 'base_model.model.model.layers.9.self_attn.rotary_emb.inv_freq', 'base_model.model.model.layers.9.mlp.gate_proj.weight', 'base_model.model.model.layers.9.mlp.down_proj.weight', 'base_model.model.model.layers.9.mlp.up_proj.weight', 'base_model.model.model.layers.9.input_layernorm.weight', 'base_model.model.model.layers.9.post_attention_layernorm.weight', 'base_model.model.model.layers.10.self_attn.q_proj.weight', 'base_model.model.model.layers.10.self_attn.k_proj.weight', 'base_model.model.model.layers.10.self_attn.v_proj.weight', 'base_model.model.model.layers.10.self_attn.o_proj.weight', 'base_model.model.model.layers.10.self_attn.rotary_emb.inv_freq', 'base_model.model.model.layers.10.mlp.gate_proj.weight', 'base_model.model.model.layers.10.mlp.down_proj.weight', 'base_model.model.model.layers.10.mlp.up_proj.weight', 'base_model.model.model.layers.10.input_layernorm.weight', 'base_model.model.model.layers.10.post_attention_layernorm.weight', 'base_model.model.model.layers.11.self_attn.q_proj.weight', 'base_model.model.model.layers.11.self_attn.k_proj.weight', 'base_model.model.model.layers.11.self_attn.v_proj.weight', 'base_model.model.model.layers.11.self_attn.o_proj.weight', 'base_model.model.model.layers.11.self_attn.rotary_emb.inv_freq', 'base_model.model.model.layers.11.mlp.gate_proj.weight', 'base_model.model.model.layers.11.mlp.down_proj.weight', 'base_model.model.model.layers.11.mlp.up_proj.weight', 'base_model.model.model.layers.11.input_layernorm.weight', 'base_model.model.model.layers.11.post_attention_layernorm.weight', 'base_model.model.model.layers.12.self_attn.q_proj.weight', 'base_model.model.model.layers.12.self_attn.k_proj.weight', 'base_model.model.model.layers.12.self_attn.v_proj.weight', 'base_model.model.model.layers.12.self_attn.o_proj.weight', 'base_model.model.model.layers.12.self_attn.rotary_emb.inv_freq', 'base_model.model.model.layers.12.mlp.gate_proj.weight', 'base_model.model.model.layers.12.mlp.down_proj.weight', 'base_model.model.model.layers.12.mlp.up_proj.weight', 'base_model.model.model.layers.12.input_layernorm.weight', 'base_model.model.model.layers.12.post_attention_layernorm.weight', 'base_model.model.model.layers.13.self_attn.q_proj.weight', 'base_model.model.model.layers.13.self_attn.k_proj.weight', 'base_model.model.model.layers.13.self_attn.v_proj.weight', 'base_model.model.model.layers.13.self_attn.o_proj.weight', 'base_model.model.model.layers.13.self_attn.rotary_emb.inv_freq', 'base_model.model.model.layers.13.mlp.gate_proj.weight', 'base_model.model.model.layers.13.mlp.down_proj.weight', 'base_model.model.model.layers.13.mlp.up_proj.weight', 'base_model.model.model.layers.13.input_layernorm.weight', 'base_model.model.model.layers.13.post_attention_layernorm.weight', 'base_model.model.model.layers.14.self_attn.q_proj.weight', 'base_model.model.model.layers.14.self_attn.k_proj.weight', 'base_model.model.model.layers.14.self_attn.v_proj.weight', 'base_model.model.model.layers.14.self_attn.o_proj.weight', 'base_model.model.model.layers.14.self_attn.rotary_emb.inv_freq', 'base_model.model.model.layers.14.mlp.gate_proj.weight', 'base_model.model.model.layers.14.mlp.down_proj.weight', 'base_model.model.model.layers.14.mlp.up_proj.weight', 'base_model.model.model.layers.14.input_layernorm.weight', 'base_model.model.model.layers.14.post_attention_layernorm.weight', 'base_model.model.model.layers.15.self_attn.q_proj.weight', 'base_model.model.model.layers.15.self_attn.k_proj.weight', 'base_model.model.model.layers.15.self_attn.v_proj.weight', 'base_model.model.model.layers.15.self_attn.o_proj.weight', 'base_model.model.model.layers.15.self_attn.rotary_emb.inv_freq', 'base_model.model.model.layers.15.mlp.gate_proj.weight', 'base_model.model.model.layers.15.mlp.down_proj.weight', 'base_model.model.model.layers.15.mlp.up_proj.weight', 'base_model.model.model.layers.15.input_layernorm.weight', 'base_model.model.model.layers.15.post_attention_layernorm.weight', 'base_model.model.model.layers.16.self_attn.q_proj.weight', 'base_model.model.model.layers.16.self_attn.k_proj.weight', 'base_model.model.model.layers.16.self_attn.v_proj.weight', 'base_model.model.model.layers.16.self_attn.o_proj.weight', 'base_model.model.model.layers.16.self_attn.rotary_emb.inv_freq', 'base_model.model.model.layers.16.mlp.gate_proj.weight', 'base_model.model.model.layers.16.mlp.down_proj.weight', 'base_model.model.model.layers.16.mlp.up_proj.weight', 'base_model.model.model.layers.16.input_layernorm.weight', 'base_model.model.model.layers.16.post_attention_layernorm.weight', 'base_model.model.model.layers.17.self_attn.q_proj.weight', 'base_model.model.model.layers.17.self_attn.k_proj.weight', 'base_model.model.model.layers.17.self_attn.v_proj.weight', 'base_model.model.model.layers.17.self_attn.o_proj.weight', 'base_model.model.model.layers.17.self_attn.rotary_emb.inv_freq', 'base_model.model.model.layers.17.mlp.gate_proj.weight', 'base_model.model.model.layers.17.mlp.down_proj.weight', 'base_model.model.model.layers.17.mlp.up_proj.weight', 'base_model.model.model.layers.17.input_layernorm.weight', 'base_model.model.model.layers.17.post_attention_layernorm.weight', 'base_model.model.model.layers.18.self_attn.q_proj.weight', 'base_model.model.model.layers.18.self_attn.k_proj.weight', 'base_model.model.model.layers.18.self_attn.v_proj.weight', 'base_model.model.model.layers.18.self_attn.o_proj.weight', 'base_model.model.model.layers.18.self_attn.rotary_emb.inv_freq', 'base_model.model.model.layers.18.mlp.gate_proj.weight', 'base_model.model.model.layers.18.mlp.down_proj.weight', 'base_model.model.model.layers.18.mlp.up_proj.weight', 'base_model.model.model.layers.18.input_layernorm.weight', 'base_model.model.model.layers.18.post_attention_layernorm.weight', 'base_model.model.model.layers.19.self_attn.q_proj.weight', 'base_model.model.model.layers.19.self_attn.k_proj.weight', 'base_model.model.model.layers.19.self_attn.v_proj.weight', 'base_model.model.model.layers.19.self_attn.o_proj.weight', 'base_model.model.model.layers.19.self_attn.rotary_emb.inv_freq', 'base_model.model.model.layers.19.mlp.gate_proj.weight', 'base_model.model.model.layers.19.mlp.down_proj.weight', 'base_model.model.model.layers.19.mlp.up_proj.weight', 'base_model.model.model.layers.19.input_layernorm.weight', 'base_model.model.model.layers.19.post_attention_layernorm.weight', 'base_model.model.model.layers.20.self_attn.q_proj.weight', 'base_model.model.model.layers.20.self_attn.k_proj.weight', 'base_model.model.model.layers.20.self_attn.v_proj.weight', 'base_model.model.model.layers.20.self_attn.o_proj.weight', 'base_model.model.model.layers.20.self_attn.rotary_emb.inv_freq', 'base_model.model.model.layers.20.mlp.gate_proj.weight', 'base_model.model.model.layers.20.mlp.down_proj.weight', 'base_model.model.model.layers.20.mlp.up_proj.weight', 'base_model.model.model.layers.20.input_layernorm.weight', 'base_model.model.model.layers.20.post_attention_layernorm.weight', 'base_model.model.model.layers.21.self_attn.q_proj.weight', 'base_model.model.model.layers.21.self_attn.k_proj.weight', 'base_model.model.model.layers.21.self_attn.v_proj.weight', 'base_model.model.model.layers.21.self_attn.o_proj.weight', 'base_model.model.model.layers.21.self_attn.rotary_emb.inv_freq', 'base_model.model.model.layers.21.mlp.gate_proj.weight', 'base_model.model.model.layers.21.mlp.down_proj.weight', 'base_model.model.model.layers.21.mlp.up_proj.weight', 'base_model.model.model.layers.21.input_layernorm.weight', 'base_model.model.model.layers.21.post_attention_layernorm.weight', 'base_model.model.model.layers.22.self_attn.q_proj.weight', 'base_model.model.model.layers.22.self_attn.k_proj.weight', 'base_model.model.model.layers.22.self_attn.v_proj.weight', 'base_model.model.model.layers.22.self_attn.o_proj.weight', 'base_model.model.model.layers.22.self_attn.rotary_emb.inv_freq', 'base_model.model.model.layers.22.mlp.gate_proj.weight', 'base_model.model.model.layers.22.mlp.down_proj.weight', 'base_model.model.model.layers.22.mlp.up_proj.weight', 'base_model.model.model.layers.22.input_layernorm.weight', 'base_model.model.model.layers.22.post_attention_layernorm.weight', 'base_model.model.model.layers.23.self_attn.q_proj.weight', 'base_model.model.model.layers.23.self_attn.k_proj.weight', 'base_model.model.model.layers.23.self_attn.v_proj.weight', 'base_model.model.model.layers.23.self_attn.o_proj.weight', 'base_model.model.model.layers.23.self_attn.rotary_emb.inv_freq', 'base_model.model.model.layers.23.mlp.gate_proj.weight', 'base_model.model.model.layers.23.mlp.down_proj.weight', 'base_model.model.model.layers.23.mlp.up_proj.weight', 'base_model.model.model.layers.23.input_layernorm.weight', 'base_model.model.model.layers.23.post_attention_layernorm.weight', 'base_model.model.model.layers.24.self_attn.q_proj.weight', 'base_model.model.model.layers.24.self_attn.k_proj.weight', 'base_model.model.model.layers.24.self_attn.v_proj.weight', 'base_model.model.model.layers.24.self_attn.o_proj.weight', 'base_model.model.model.layers.24.self_attn.rotary_emb.inv_freq', 'base_model.model.model.layers.24.mlp.gate_proj.weight', 'base_model.model.model.layers.24.mlp.down_proj.weight', 'base_model.model.model.layers.24.mlp.up_proj.weight', 'base_model.model.model.layers.24.input_layernorm.weight', 'base_model.model.model.layers.24.post_attention_layernorm.weight', 'base_model.model.model.layers.25.self_attn.q_proj.weight', 'base_model.model.model.layers.25.self_attn.k_proj.weight', 'base_model.model.model.layers.25.self_attn.v_proj.weight', 'base_model.model.model.layers.25.self_attn.o_proj.weight', 'base_model.model.model.layers.25.self_attn.rotary_emb.inv_freq', 'base_model.model.model.layers.25.mlp.gate_proj.weight', 'base_model.model.model.layers.25.mlp.down_proj.weight', 'base_model.model.model.layers.25.mlp.up_proj.weight', 'base_model.model.model.layers.25.input_layernorm.weight', 'base_model.model.model.layers.25.post_attention_layernorm.weight', 'base_model.model.model.layers.26.self_attn.q_proj.weight', 'base_model.model.model.layers.26.self_attn.k_proj.weight', 'base_model.model.model.layers.26.self_attn.v_proj.weight', 'base_model.model.model.layers.26.self_attn.o_proj.weight', 'base_model.model.model.layers.26.self_attn.rotary_emb.inv_freq', 'base_model.model.model.layers.26.mlp.gate_proj.weight', 'base_model.model.model.layers.26.mlp.down_proj.weight', 'base_model.model.model.layers.26.mlp.up_proj.weight', 'base_model.model.model.layers.26.input_layernorm.weight', 'base_model.model.model.layers.26.post_attention_layernorm.weight', 'base_model.model.model.layers.27.self_attn.q_proj.weight', 'base_model.model.model.layers.27.self_attn.k_proj.weight', 'base_model.model.model.layers.27.self_attn.v_proj.weight', 'base_model.model.model.layers.27.self_attn.o_proj.weight', 'base_model.model.model.layers.27.self_attn.rotary_emb.inv_freq', 'base_model.model.model.layers.27.mlp.gate_proj.weight', 'base_model.model.model.layers.27.mlp.down_proj.weight', 'base_model.model.model.layers.27.mlp.up_proj.weight', 'base_model.model.model.layers.27.input_layernorm.weight', 'base_model.model.model.layers.27.post_attention_layernorm.weight', 'base_model.model.model.layers.28.self_attn.q_proj.weight', 'base_model.model.model.layers.28.self_attn.k_proj.weight', 'base_model.model.model.layers.28.self_attn.v_proj.weight', 'base_model.model.model.layers.28.self_attn.o_proj.weight', 'base_model.model.model.layers.28.self_attn.rotary_emb.inv_freq', 'base_model.model.model.layers.28.mlp.gate_proj.weight', 'base_model.model.model.layers.28.mlp.down_proj.weight', 'base_model.model.model.layers.28.mlp.up_proj.weight', 'base_model.model.model.layers.28.input_layernorm.weight', 'base_model.model.model.layers.28.post_attention_layernorm.weight', 'base_model.model.model.layers.29.self_attn.q_proj.weight', 'base_model.model.model.layers.29.self_attn.k_proj.weight', 'base_model.model.model.layers.29.self_attn.v_proj.weight', 'base_model.model.model.layers.29.self_attn.o_proj.weight', 'base_model.model.model.layers.29.self_attn.rotary_emb.inv_freq', 'base_model.model.model.layers.29.mlp.gate_proj.weight', 'base_model.model.model.layers.29.mlp.down_proj.weight', 'base_model.model.model.layers.29.mlp.up_proj.weight', 'base_model.model.model.layers.29.input_layernorm.weight', 'base_model.model.model.layers.29.post_attention_layernorm.weight', 'base_model.model.model.layers.30.self_attn.q_proj.weight', 'base_model.model.model.layers.30.self_attn.k_proj.weight', 'base_model.model.model.layers.30.self_attn.v_proj.weight', 'base_model.model.model.layers.30.self_attn.o_proj.weight', 'base_model.model.model.layers.30.self_attn.rotary_emb.inv_freq', 'base_model.model.model.layers.30.mlp.gate_proj.weight', 'base_model.model.model.layers.30.mlp.down_proj.weight', 'base_model.model.model.layers.30.mlp.up_proj.weight', 'base_model.model.model.layers.30.input_layernorm.weight', 'base_model.model.model.layers.30.post_attention_layernorm.weight', 'base_model.model.model.layers.31.self_attn.q_proj.weight', 'base_model.model.model.layers.31.self_attn.k_proj.weight', 'base_model.model.model.layers.31.self_attn.v_proj.weight', 'base_model.model.model.layers.31.self_attn.o_proj.weight', 'base_model.model.model.layers.31.self_attn.rotary_emb.inv_freq', 'base_model.model.model.layers.31.mlp.gate_proj.weight', 'base_model.model.model.layers.31.mlp.down_proj.weight', 'base_model.model.model.layers.31.mlp.up_proj.weight', 'base_model.model.model.layers.31.input_layernorm.weight', 'base_model.model.model.layers.31.post_attention_layernorm.weight', 'base_model.model.model.norm.weight', 'base_model.model.lm_head.0.weight'].\n",
            "{'train_runtime': 8499.7291, 'train_samples_per_second': 17.648, 'train_steps_per_second': 0.176, 'train_loss': 0.8647502457300822, 'epoch': 3.0}\n",
            "100% 1500/1500 [2:21:38<00:00,  5.67s/it]\n",
            "\n",
            " If there's a warning about missing keys above, please disregard :)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "evWTxSsMl6mR"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}