[Dacon Summarization] Transformer.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 5,
  "metadata": {
    "kernelspec": {
      "display_name": "eunil",
      "language": "python",
      "name": "eunil"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.13"
    },
    "colab": {
      "name": "[Baseline]Transformer.ipynb",
      "provenance": []
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "b4QLV01DZip9"
      },
      "source": [
        "baseline.ipynb<br>\n",
        ".. └ data<br>\n",
        ".... ├ train.json<br>\n",
        ".... ├ test.json<br>\n",
        ".... └ sample_submission.csv<br>"
      ],
      "id": "b4QLV01DZip9"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bwQV_cCSZiqD"
      },
      "source": [
        "# 사용 패키지"
      ],
      "id": "bwQV_cCSZiqD"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Fx8i5l8CZuBP"
      },
      "source": [
        "# %%capture\n",
        "# !pip install konlpy\n",
        "# !pip install transformer"
      ],
      "id": "Fx8i5l8CZuBP",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FQ3ku8RRatfA"
      },
      "source": [
        "# %%capture\n",
        "# !pip install kobert-transformers"
      ],
      "id": "FQ3ku8RRatfA",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0T9gHcBDZiqE"
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import json\n",
        "import os\n",
        "import random\n",
        "import math\n",
        "import copy\n",
        "import gc\n",
        "from tqdm import tqdm\n",
        "from glob import glob\n",
        "# from konlpy.tag import Mecab\n",
        "\n",
        "import torch\n",
        "from torch import nn\n",
        "from torch.utils.data import Dataset, DataLoader\n",
        "\n",
        "import warnings\n",
        "warnings.filterwarnings(action='ignore')"
      ],
      "id": "0T9gHcBDZiqE",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sasAH5WUZiqF"
      },
      "source": [
        "## 랜덤 시드 고정"
      ],
      "id": "sasAH5WUZiqF"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nlI2Lp_uZiqG"
      },
      "source": [
        "def seed_everything(seed: int = 42):\n",
        "    random.seed(seed)\n",
        "    np.random.seed(seed)\n",
        "    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n",
        "    torch.manual_seed(seed)\n",
        "    torch.cuda.manual_seed(seed)  # type: ignore\n",
        "    torch.backends.cudnn.deterministic = True  # type: ignore\n",
        "    torch.backends.cudnn.benchmark = True  # type: ignore"
      ],
      "id": "nlI2Lp_uZiqG",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rUgt2igqZiqH"
      },
      "source": [
        "seed_everything(42)"
      ],
      "id": "rUgt2igqZiqH",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-ZCaIYlfa0up"
      },
      "source": [
        "checkpoint = 'monologg/kobert'"
      ],
      "id": "-ZCaIYlfa0up",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jIsEox33ZiqH"
      },
      "source": [
        "# 데이터 로드"
      ],
      "id": "jIsEox33ZiqH"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yihtJbTBZiqI"
      },
      "source": [
        "# DIR = \"./data\"\n",
        "TRAIN_SOURCE = os.path.join(\"train.json\")\n",
        "TEST_SOURCE = os.path.join(\"test.json\")"
      ],
      "id": "yihtJbTBZiqI",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3BAFoZQHZiqJ"
      },
      "source": [
        "with open(TRAIN_SOURCE) as f:\n",
        "    TRAIN_DATA = json.loads(f.read())\n",
        "    \n",
        "with open(TEST_SOURCE) as f:\n",
        "    TEST_DATA = json.loads(f.read())"
      ],
      "id": "3BAFoZQHZiqJ",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Jn5BqKR0ZiqJ"
      },
      "source": [
        "train = pd.DataFrame(columns=['uid', 'title', 'region', 'context', 'summary'])\n",
        "uid = 1000\n",
        "for data in TRAIN_DATA:\n",
        "    for agenda in data['context'].keys():\n",
        "        context = ''\n",
        "        for line in data['context'][agenda]:\n",
        "            context += data['context'][agenda][line]\n",
        "            context += ' '\n",
        "        train.loc[uid, 'uid'] = uid\n",
        "        train.loc[uid, 'title'] = data['title']\n",
        "        train.loc[uid, 'region'] = data['region']\n",
        "        train.loc[uid, 'context'] = context[:-1]\n",
        "        train.loc[uid, 'summary'] = data['label'][agenda]['summary']\n",
        "        uid += 1\n",
        "\n",
        "test = pd.DataFrame(columns=['uid', 'title', 'region', 'context'])\n",
        "uid = 2000\n",
        "for data in TEST_DATA:\n",
        "    for agenda in data['context'].keys():\n",
        "        context = ''\n",
        "        for line in data['context'][agenda]:\n",
        "            context += data['context'][agenda][line]\n",
        "            context += ' '\n",
        "        test.loc[uid, 'uid'] = uid\n",
        "        test.loc[uid, 'title'] = data['title']\n",
        "        test.loc[uid, 'region'] = data['region']\n",
        "        test.loc[uid, 'context'] = context[:-1]\n",
        "        uid += 1"
      ],
      "id": "Jn5BqKR0ZiqJ",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dTuSTdS7ZiqK"
      },
      "source": [
        "train['total'] = train.title + ' ' + train.region + ' ' + train.context\n",
        "test['total'] = test.title + ' ' + test.region + ' ' + test.context"
      ],
      "id": "dTuSTdS7ZiqK",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 368
        },
        "id": "6xR13dzOZiqK",
        "outputId": "17ef5cad-2a2b-47c6-87ea-cbb63dc2de2f"
      },
      "source": [
        "train.head()"
      ],
      "id": "6xR13dzOZiqK",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>uid</th>\n",
              "      <th>title</th>\n",
              "      <th>region</th>\n",
              "      <th>context</th>\n",
              "      <th>summary</th>\n",
              "      <th>total</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1000</th>\n",
              "      <td>1000</td>\n",
              "      <td>제207회 완주군의회(임시회) 제 1 차 본회의회의록</td>\n",
              "      <td>완주</td>\n",
              "      <td>의석을 정돈하여 주시기 바랍니다. 성원이 되었으므로 제207회 완주군의회 임시회 제...</td>\n",
              "      <td>제207회 완주군의회 임시회 제1차 본회의 개의 선포.</td>\n",
              "      <td>제207회 완주군의회(임시회) 제 1 차 본회의회의록 완주 의석을 정돈하여 주시기 ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1001</th>\n",
              "      <td>1001</td>\n",
              "      <td>제207회 완주군의회(임시회) 제 1 차 본회의회의록</td>\n",
              "      <td>완주</td>\n",
              "      <td>의사팀장 수고하셨습니다. 먼저 의사일정 제1항 제207회 완주군의회 임시회 회기 결...</td>\n",
              "      <td>제207회 완주군의회 임시회 회기는 8월 26일부터 9월 4일까지 10일간으로 가결됨.</td>\n",
              "      <td>제207회 완주군의회(임시회) 제 1 차 본회의회의록 완주 의사팀장 수고하셨습니다....</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1002</th>\n",
              "      <td>1002</td>\n",
              "      <td>제207회 완주군의회(임시회) 제 1 차 본회의회의록</td>\n",
              "      <td>완주</td>\n",
              "      <td>다음은 의사일정 제2항 제207회 완주군의회 임시회 회의록 서명의원 선출의 건을 상...</td>\n",
              "      <td>제207회 완주군의회 임시회 회의록 서명의원으로 최등원 의원과 박웅배 의원이 선출됨.</td>\n",
              "      <td>제207회 완주군의회(임시회) 제 1 차 본회의회의록 완주 다음은 의사일정 제2항 ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1003</th>\n",
              "      <td>1003</td>\n",
              "      <td>제207회 완주군의회(임시회) 제 1 차 본회의회의록</td>\n",
              "      <td>완주</td>\n",
              "      <td>다음은 의사일정 제3항 본회의 휴회의 건을 상정합니다. 상임의원회 의정활동을 위하여...</td>\n",
              "      <td>8월 27일부터 9월 3일까지 8일간 휴회가 가결됨. 제2차 본회의는 9월 4일 오...</td>\n",
              "      <td>제207회 완주군의회(임시회) 제 1 차 본회의회의록 완주 다음은 의사일정 제3항 ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1004</th>\n",
              "      <td>1004</td>\n",
              "      <td>제251회 완주군의회(제1차 정례회) 제1차 본 회 의 회 의 록</td>\n",
              "      <td>완주</td>\n",
              "      <td>의석을 정돈하여 주시기 바랍니다. 성원이 되었으므로 제251회 완주군의회 제1차 정...</td>\n",
              "      <td>제251회 완주군의회 제1차 정례회 제1차 본회의 개의 선포.</td>\n",
              "      <td>제251회 완주군의회(제1차 정례회) 제1차 본 회 의 회 의 록 완주 의석을 정돈...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "       uid  ...                                              total\n",
              "1000  1000  ...  제207회 완주군의회(임시회) 제 1 차 본회의회의록 완주 의석을 정돈하여 주시기 ...\n",
              "1001  1001  ...  제207회 완주군의회(임시회) 제 1 차 본회의회의록 완주 의사팀장 수고하셨습니다....\n",
              "1002  1002  ...  제207회 완주군의회(임시회) 제 1 차 본회의회의록 완주 다음은 의사일정 제2항 ...\n",
              "1003  1003  ...  제207회 완주군의회(임시회) 제 1 차 본회의회의록 완주 다음은 의사일정 제3항 ...\n",
              "1004  1004  ...  제251회 완주군의회(제1차 정례회) 제1차 본 회 의 회 의 록 완주 의석을 정돈...\n",
              "\n",
              "[5 rows x 6 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sg8JvdJJZiqL",
        "outputId": "3be677c2-0115-4720-f1fc-cd17939c6bcf"
      },
      "source": [
        "train.shape"
      ],
      "id": "sg8JvdJJZiqL",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(2994, 6)"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 285
        },
        "id": "C7uyvwBRZiqM",
        "outputId": "9c0abde8-ad41-40af-e443-0a56942461ed"
      },
      "source": [
        "test.head()"
      ],
      "id": "C7uyvwBRZiqM",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>uid</th>\n",
              "      <th>title</th>\n",
              "      <th>region</th>\n",
              "      <th>context</th>\n",
              "      <th>total</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>2000</th>\n",
              "      <td>2000</td>\n",
              "      <td>제235회    본회의 제1차(2012.06.21.)</td>\n",
              "      <td>음성</td>\n",
              "      <td>의석을 정돈하여 주시기 바랍니다. 성원이 되었으므로 지금부터 음성군의회 제235회 ...</td>\n",
              "      <td>제235회    본회의 제1차(2012.06.21.) 음성 의석을 정돈하여 주시기 ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2001</th>\n",
              "      <td>2001</td>\n",
              "      <td>제235회    본회의 제1차(2012.06.21.)</td>\n",
              "      <td>음성</td>\n",
              "      <td>의사일정 제1항, 음성군의회 제235회 제1차 정례회 회기결정의 건을 상정합니다. ...</td>\n",
              "      <td>제235회    본회의 제1차(2012.06.21.) 음성 의사일정 제1항, 음성군...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2002</th>\n",
              "      <td>2002</td>\n",
              "      <td>제235회    본회의 제1차(2012.06.21.)</td>\n",
              "      <td>음성</td>\n",
              "      <td>의사일정 제2항, 회의록 서명의원 선출의 건을 상정합니다. 제235회 제1차 정례회...</td>\n",
              "      <td>제235회    본회의 제1차(2012.06.21.) 음성 의사일정 제2항, 회의록...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2003</th>\n",
              "      <td>2003</td>\n",
              "      <td>제235회    본회의 제1차(2012.06.21.)</td>\n",
              "      <td>음성</td>\n",
              "      <td>의사일정 제3항, 예산결산특별위원회 구성의 건을 상정합니다. 예산결산특별위원회 구성...</td>\n",
              "      <td>제235회    본회의 제1차(2012.06.21.) 음성 의사일정 제3항, 예산결...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2004</th>\n",
              "      <td>2004</td>\n",
              "      <td>제235회    본회의 제1차(2012.06.21.)</td>\n",
              "      <td>음성</td>\n",
              "      <td>의사일정 제4항, 환경분야 현지확인 특별위원회 구성결의안을 상정합니다. 대표발의하신...</td>\n",
              "      <td>제235회    본회의 제1차(2012.06.21.) 음성 의사일정 제4항, 환경분...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "       uid  ...                                              total\n",
              "2000  2000  ...  제235회    본회의 제1차(2012.06.21.) 음성 의석을 정돈하여 주시기 ...\n",
              "2001  2001  ...  제235회    본회의 제1차(2012.06.21.) 음성 의사일정 제1항, 음성군...\n",
              "2002  2002  ...  제235회    본회의 제1차(2012.06.21.) 음성 의사일정 제2항, 회의록...\n",
              "2003  2003  ...  제235회    본회의 제1차(2012.06.21.) 음성 의사일정 제3항, 예산결...\n",
              "2004  2004  ...  제235회    본회의 제1차(2012.06.21.) 음성 의사일정 제4항, 환경분...\n",
              "\n",
              "[5 rows x 5 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "K3HxWqf8ZiqM",
        "outputId": "0553667b-91e8-453f-e6ee-a5959082c237"
      },
      "source": [
        "test.shape"
      ],
      "id": "K3HxWqf8ZiqM",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(506, 5)"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sF690Rh-ZiqM"
      },
      "source": [
        "## 하이퍼파라미터"
      ],
      "id": "sF690Rh-ZiqM"
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Cdxcv-UPjD2f",
        "outputId": "49866b8f-b7a0-4c9d-8394-48cfabc0e060"
      },
      "source": [
        "if torch.cuda.is_available():    \n",
        "\n",
        "    # Tell PyTorch to use the GPU.    \n",
        "    device = torch.device(\"cuda\")\n",
        "\n",
        "    print('There are %d GPU(s) available.' % torch.cuda.device_count())\n",
        "\n",
        "    print('We will use the GPU:', torch.cuda.get_device_name(0))\n",
        "\n",
        "# If not...\n",
        "else:\n",
        "    print('No GPU available, using the CPU instead.')\n",
        "    device = torch.device(\"cpu\")"
      ],
      "id": "Cdxcv-UPjD2f",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "There are 1 GPU(s) available.\n",
            "We will use the GPU: Tesla K80\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zyqoOYY4ZiqN"
      },
      "source": [
        "encoder_len = 500\n",
        "decoder_len = 50\n",
        "max_vocab_size = 20000\n",
        "batch_size = 32\n",
        "num_layers = 6\n",
        "d_model = 512\n",
        "dff = 2048\n",
        "num_heads = 8\n",
        "dropout_rate = 0.1\n",
        "epochs = 20\n",
        "learning_rate = 1e-4\n",
        "# device = torch.device(\"cuda:0\")"
      ],
      "id": "zyqoOYY4ZiqN",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iMADlTnyZiqN"
      },
      "source": [
        "## train, validation 분리"
      ],
      "id": "iMADlTnyZiqN"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B2TF6oLoZiqO"
      },
      "source": [
        "# df_train = train.iloc[:-200]\n",
        "# df_val = train.iloc[-200:]"
      ],
      "id": "B2TF6oLoZiqO",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zzjL32ERfSIk"
      },
      "source": [
        "import sklearn\n",
        "from sklearn.model_selection import train_test_split\n",
        "df_train, df_val = train_test_split(train,test_size=0.3, shuffle=True, random_state=11)"
      ],
      "id": "zzjL32ERfSIk",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1rhgbYBRZiqO"
      },
      "source": [
        "# 토크나이징"
      ],
      "id": "1rhgbYBRZiqO"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qqauqJQRhWw7"
      },
      "source": [
        "%%capture\n",
        "!apt-get update\n",
        "!apt-get install g++ openjdk-8-jdk \n",
        "!pip3 install konlpy JPype1-py3\n",
        "!bash <(curl -s https://raw.githubusercontent.com/konlpy/konlpy/master/scripts/mecab.sh)"
      ],
      "id": "qqauqJQRhWw7",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NWdjvHsweDlI"
      },
      "source": [
        "import konlpy\n",
        "from konlpy.tag import Mecab"
      ],
      "id": "NWdjvHsweDlI",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a4qDtyyleGwI"
      },
      "source": [
        "mecab = Mecab()"
      ],
      "id": "a4qDtyyleGwI",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SarqjmjVZiqO"
      },
      "source": [
        "class Mecab_Tokenizer():\n",
        "    def __init__(self, max_length, mode, max_vocab_size=-1):\n",
        "        self.text_tokenizer = Mecab()\n",
        "        self.mode = mode\n",
        "        self.txt2idx = {'pad_':0, 'unk_':1}\n",
        "        self.idx2txt = {0:'pad_', 1:'unk_'}\n",
        "        self.max_length = max_length\n",
        "        self.word_count = {}\n",
        "        self.max_vocab_size = max_vocab_size\n",
        "        \n",
        "        # 띄어쓰기를 찾기 위한 태그 목록\n",
        "        self.font_blank_tag = [\n",
        "            '', 'EC', 'EC+JKO', 'EF', 'EP+EC', 'EP+EP+EC', 'EP+ETM', 'EP+ETN+JKO', 'ETM', 'ETN', 'ETN+JKO', 'ETN+JX', 'IC', 'JC', 'JKB', 'JKB+JX', 'JKO',\n",
        "            'JKQ', 'JKS', 'JX', 'MAG', 'MAG+JX', 'MAG+XSV+EP+EC', 'MAJ','MM', 'MM+EC', 'NNB', 'NNB+JKB', 'NNB+JKO', 'NNB+VCP+EC', 'NNBC', 'NNG', 'NNG+JX+JKO',\n",
        "            'NNG+VCP+EC', 'NNP', 'NNP+JX', 'NP', 'NP+JKO', 'NP+JKS', 'NP+JX', 'NP+VCP+EC', 'NR', 'SC', 'SF', 'SL', 'SN', 'SSC', 'SSO', 'SY', 'UNKNOWN',\n",
        "            'VA+EC', 'VA+EC+VX+ETM', 'VA+ETM', 'VA+ETN+JKB+JX', 'VCN+EC', 'VCN+ETM', 'VCP', 'VCP+EC', 'VCP+EP+EC', 'VCP+EP+ETM', 'VCP+ETM', 'VCP+ETN',\n",
        "            'VV+EC', 'VV+EC+JX', 'VV+EC+VX+EC', 'VV+EC+VX+ETM', 'VV+EP+EC', 'VV+EP+ETM', 'VV+ETM', 'VV+ETN', 'VX+EC', 'VX+EC+VX+EP+EC', 'VX+EP+ETM',\n",
        "            'VX+ETM', 'XPN', 'XR', 'XSA+EC', 'XSA+EC+VX+ETM', 'XSA+ETM', 'XSN', 'XSV+EC', 'XSV+EP+EC', 'XSV+ETM', 'XSV+ETN', 'XSV+JKO'\n",
        "        ]\n",
        "        self.back_blank_tag = [\n",
        "            '', 'IC', 'MAG', 'MAG+JX', 'MAG+XSV+EP+EC', 'MAJ', 'MM', 'MM+EC', 'NNB', 'NNB+JKB', 'NNB+VCP', 'NNB+VCP+EC', 'NNB+VCP+EF', 'NNBC', 'NNBC+VCP+EC',\n",
        "            'NNG', 'NNG+JC', 'NNG+JX+JKO', 'NNG+VCP', 'NNG+VCP+EC', 'NNG+VCP+ETM', 'NNP', 'NNP+JX', 'NP', 'NP+JKG', 'NP+JKO', 'NP+JKS', 'NP+JX', 'NP+VCP+EC', 'NP+VCP+EF',\n",
        "            'NR', 'SC', 'SL', 'SN', 'SSC', 'SSO', 'SY', 'VA', 'VA+EC', 'VA+EC+VX+ETM', 'VA+EF', 'VA+ETM', 'VA+ETN', 'VA+ETN+JKB+JX', 'VCN', 'VCN+EC', 'VCN+EF', 'VCN+ETM',\n",
        "            'VCN+ETN', 'VCP', 'VCP+EF', 'VV', 'VV+EC', 'VV+EC+JX', 'VV+EC+VX', 'VV+EC+VX+EC', 'VV+EC+VX+EF', 'VV+EC+VX+EP+EC', 'VV+EC+VX+ETM', 'VV+EF', 'VV+EP', 'VV+EP+EC',\n",
        "            'VV+EP+ETM', 'VV+ETM', 'VV+ETN', 'VV+ETN+VCP+EF', 'VX', 'VX+ETM', 'XPN', 'XR', 'XSA+ETN+VCP+EF', 'XSN'\n",
        "        ]\n",
        "        \n",
        "    def morpheme(self, sentence_list):\n",
        "        new_sentence = []\n",
        "        for i, sentence in tqdm(enumerate(sentence_list)):\n",
        "            temp = []\n",
        "            if self.mode == 'dec':\n",
        "                temp.append('sos_')\n",
        "            for t in self.text_tokenizer.pos(sentence):\n",
        "                temp.append('_'.join(t))\n",
        "            if self.mode == 'dec':\n",
        "                temp.append('eos_')\n",
        "            new_sentence.append(' '.join(temp))\n",
        "            \n",
        "        return new_sentence\n",
        "    \n",
        "    def fit(self, sentence_list):\n",
        "        for sentence in tqdm(sentence_list):\n",
        "            for word in sentence.split(' '):\n",
        "                try:\n",
        "                    self.word_count[word] += 1\n",
        "                except:\n",
        "                    self.word_count[word] = 1\n",
        "        self.word_count = dict(sorted(self.word_count.items(), key=self.sort_target, reverse=True))\n",
        "        \n",
        "        self.txt2idx = {'pad_':0, 'unk_':1}\n",
        "        self.idx2txt = {0:'pad_', 1:'unk_'}\n",
        "        if self.max_vocab_size == -1:\n",
        "            for i, word in enumerate(list(self.word_count.keys())):\n",
        "                self.txt2idx[word]=i+2\n",
        "                self.idx2txt[i+2]=word\n",
        "        else:\n",
        "            for i, word in enumerate(list(self.word_count.keys())[:self.max_vocab_size]):\n",
        "                self.txt2idx[word]=i+2\n",
        "                self.idx2txt[i+2]=word\n",
        "        \n",
        "    def sort_target(self, x):\n",
        "        return x[1]\n",
        "            \n",
        "    def txt2token(self, sentence_list):\n",
        "        tokens = []\n",
        "        for sentence in tqdm(sentence_list):\n",
        "            token = [0]*self.max_length\n",
        "            for i, w in enumerate(sentence.split(' ')):\n",
        "                if i == self.max_length:\n",
        "                    break\n",
        "                try:\n",
        "                    token[i] = self.txt2idx[w]\n",
        "                except:\n",
        "                    token[i] = self.txt2idx['unk_']\n",
        "            tokens.append(token)\n",
        "        return np.array(tokens)\n",
        "    \n",
        "    def convert(self, token):\n",
        "        sentence = []\n",
        "        for j, i in enumerate(token):\n",
        "            if self.mode == 'enc':\n",
        "                if i != self.txt2idx['pad_']:\n",
        "                    sentence.append(self.idx2txt[i].split('_')[0])\n",
        "            elif self.mode == 'dec':\n",
        "                if i == self.txt2idx['eos_'] or i == self.txt2idx['pad_']:\n",
        "                    break\n",
        "                elif i != 0:\n",
        "                    sentence.append(self.idx2txt[i].split('_')[0])\n",
        "                    # 앞뒤 태그를 확인하여 띄어쓰기 추가\n",
        "                    if self.idx2txt[i].split('_')[1] in self.font_blank_tag:\n",
        "                        try:\n",
        "                            if self.idx2txt[token[j+1]].split('_')[1] in self.back_blank_tag:\n",
        "                                sentence.append(' ')\n",
        "                        except:\n",
        "                            pass\n",
        "        sentence = \"\".join(sentence)\n",
        "        if self.mode == 'enc':\n",
        "            sentence = sentence[:-1]\n",
        "        elif self.mode == 'dec':\n",
        "            sentence = sentence[3:-1]\n",
        "            \n",
        "        return sentence"
      ],
      "id": "SarqjmjVZiqO",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ysyqzu3hZiqP"
      },
      "source": [
        "src_tokenizer = Mecab_Tokenizer(encoder_len, mode='enc', max_vocab_size=max_vocab_size)\n",
        "tar_tokenizer = Mecab_Tokenizer(decoder_len, mode='dec', max_vocab_size=max_vocab_size)"
      ],
      "id": "ysyqzu3hZiqP",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dvsHiG2TZiqP",
        "outputId": "739cec08-f7b3-4530-ceac-82a1e01c9218"
      },
      "source": [
        "train_src = src_tokenizer.morpheme(df_train.total)\n",
        "val_src = src_tokenizer.morpheme(df_val.total)\n",
        "test_src = src_tokenizer.morpheme(test.total)\n",
        "\n",
        "train_tar = tar_tokenizer.morpheme(df_train.summary)\n",
        "val_tar = tar_tokenizer.morpheme(df_val.summary)"
      ],
      "id": "dvsHiG2TZiqP",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "2095it [00:07, 262.54it/s]\n",
            "899it [00:03, 267.54it/s]\n",
            "506it [00:01, 260.23it/s]\n",
            "2095it [00:00, 4588.74it/s]\n",
            "899it [00:00, 4633.00it/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ABdlgaU7ZiqQ"
      },
      "source": [
        "# train_src_len = []\n",
        "# for m in train_src:\n",
        "#     m_len = len(m.split(' '))\n",
        "#     train_src_len.append(m_len)\n",
        "# print('train_src_max_len :', max(train_src_len))\n",
        "# plt.hist(train_src_len, bins=30)\n",
        "# plt.show()\n",
        "\n",
        "# train_tar_len = []\n",
        "# for m in train_tar:\n",
        "#     m_len = len(m.split(' '))\n",
        "#     train_tar_len.append(m_len)\n",
        "# print('train_tar_max_len :', max(train_tar_len))\n",
        "# plt.hist(train_tar_len, bins=30)\n",
        "# plt.show()"
      ],
      "id": "ABdlgaU7ZiqQ",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GfCO_sbnZiqQ",
        "outputId": "09585a5f-9e47-45ac-a2b3-0d00a4d2a405"
      },
      "source": [
        "src_tokenizer.fit(train_src)\n",
        "tar_tokenizer.fit(train_tar)"
      ],
      "id": "GfCO_sbnZiqQ",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 2095/2095 [00:00<00:00, 4453.80it/s]\n",
            "100%|██████████| 2095/2095 [00:00<00:00, 72565.81it/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "myKgGLrWZiqQ",
        "outputId": "f635c2bb-94f4-4eb7-e95b-caf8e001cc33"
      },
      "source": [
        "train_src_tokens = src_tokenizer.txt2token(train_src)\n",
        "val_src_tokens = src_tokenizer.txt2token(val_src)\n",
        "test_src_tokens = src_tokenizer.txt2token(test_src)\n",
        "\n",
        "train_tar_tokens = tar_tokenizer.txt2token(train_tar)\n",
        "val_tar_tokens = tar_tokenizer.txt2token(val_tar)"
      ],
      "id": "myKgGLrWZiqQ",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 2095/2095 [00:00<00:00, 6439.59it/s]\n",
            "100%|██████████| 899/899 [00:00<00:00, 5612.74it/s]\n",
            "100%|██████████| 506/506 [00:00<00:00, 6362.19it/s]\n",
            "100%|██████████| 2095/2095 [00:00<00:00, 77597.53it/s]\n",
            "100%|██████████| 899/899 [00:00<00:00, 65476.82it/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vKwqTtZiZiqQ"
      },
      "source": [
        "input_vocab_size = len(src_tokenizer.txt2idx)\n",
        "target_vocab_size = len(tar_tokenizer.txt2idx)"
      ],
      "id": "vKwqTtZiZiqQ",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0GC-2AmcZiqR",
        "outputId": "5e467920-d3f2-4dd5-b67c-cbddd67b7bef"
      },
      "source": [
        "input_vocab_size, target_vocab_size"
      ],
      "id": "0GC-2AmcZiqR",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(20002, 4312)"
            ]
          },
          "metadata": {},
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "id": "8Fd7GBybZiqR",
        "outputId": "7b843998-e862-4d99-f790-d74f3c72af9a"
      },
      "source": [
        "df_train.summary.iloc[0]"
      ],
      "id": "8Fd7GBybZiqR",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'제176회 제2차 정례회 제6차 본회의 개의 선포.'"
            ]
          },
          "metadata": {},
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9q60syo_ZiqR",
        "outputId": "fe15707b-1d6f-40a6-a79e-e94144d0aaca"
      },
      "source": [
        "train_tar_tokens[0], tar_tokenizer.convert(train_tar_tokens[0])"
      ],
      "id": "9q60syo_ZiqR",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(array([  3,   8, 920,  16,   8,  38,  25,  82,   8,  79,  25,  47,   5,\n",
              "         44,   5,  52,   2,   4,   0,   0,   0,   0,   0,   0,   0,   0,\n",
              "          0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
              "          0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0]),\n",
              " ' 제 176 회 제 2 차 정례회 제 6 차 본회의개의선포.')"
            ]
          },
          "metadata": {},
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "V244DPf4ZiqS"
      },
      "source": [
        "# 데이터셋"
      ],
      "id": "V244DPf4ZiqS"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hMVQir2BZiqS"
      },
      "source": [
        "class CustomDataset(Dataset):\n",
        "    def __init__(self, src_tokens, tar_tokens, mode='train'):\n",
        "        self.mode = mode\n",
        "        self.src_tokens = src_tokens\n",
        "        if self.mode == 'train':\n",
        "            self.tar_tokens = tar_tokens\n",
        "            \n",
        "    def __len__(self):\n",
        "        return len(self.src_tokens)\n",
        "    \n",
        "    def __getitem__(self, i):\n",
        "        src_token = self.src_tokens[i]\n",
        "        if self.mode == 'train':\n",
        "            tar_token = self.tar_tokens[i]\n",
        "            return {\n",
        "                'src_token' : torch.tensor(src_token, dtype=torch.long),\n",
        "                'tar_token' : torch.tensor(tar_token, dtype=torch.long),\n",
        "            }\n",
        "        else:\n",
        "            return {\n",
        "                'src_token' : torch.tensor(src_token, dtype=torch.long)\n",
        "            }"
      ],
      "id": "hMVQir2BZiqS",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eBGCvJgnZiqS"
      },
      "source": [
        "train_dataset = CustomDataset(train_src_tokens, train_tar_tokens)\n",
        "val_dataset = CustomDataset(val_src_tokens, val_tar_tokens)\n",
        "test_dataset = CustomDataset(test_src_tokens, None, 'test')\n",
        "\n",
        "train_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, num_workers=1, shuffle=True)\n",
        "val_dataloader = torch.utils.data.DataLoader(val_dataset, batch_size=batch_size, num_workers=1, shuffle=False)\n",
        "test_dataloader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, num_workers=1, shuffle=False)"
      ],
      "id": "eBGCvJgnZiqS",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CvtLPYNoZiqS"
      },
      "source": [
        "# 모델 Transformer\n",
        "\n",
        "https://www.tensorflow.org/text/tutorials/transformer 를 pytorch코드로 수정하여 작성하였습니다."
      ],
      "id": "CvtLPYNoZiqS"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Dc7o26Txp_wh"
      },
      "source": [
        "def get_angles(pos, i, d_model):\n",
        "    angle_rates = 1 / np.power(10000, (2 * (i//2)) / np.float32(d_model))\n",
        "    return pos * angle_rates"
      ],
      "id": "Dc7o26Txp_wh",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "K8l-XKICp_wi"
      },
      "source": [
        "def positional_encoding(position, d_model):\n",
        "    angle_rads = get_angles(np.arange(position)[:, np.newaxis],\n",
        "                            np.arange(d_model)[np.newaxis, :],\n",
        "                            d_model)\n",
        "\n",
        "    # apply sin to even indices in the array; 2i\n",
        "    angle_rads[:, 0::2] = np.sin(angle_rads[:, 0::2])\n",
        "\n",
        "    # apply cos to odd indices in the array; 2i+1\n",
        "    angle_rads[:, 1::2] = np.cos(angle_rads[:, 1::2])\n",
        "\n",
        "    pos_encoding = angle_rads[np.newaxis, ...]\n",
        "\n",
        "    return torch.tensor(pos_encoding, dtype=torch.float32)"
      ],
      "id": "K8l-XKICp_wi",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zhX4lBp-ZiqT"
      },
      "source": [
        "def create_padding_mask(seq):\n",
        "    seq = torch.tensor(torch.eq(seq, 0), dtype=torch.float32)\n",
        "\n",
        "    # add extra dimensions to add the padding\n",
        "    # to the attention logits.\n",
        "    seq = seq.unsqueeze(1).unsqueeze(2)\n",
        "    return seq  # (batch_size, 1, 1, seq_len)"
      ],
      "id": "zhX4lBp-ZiqT",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AbyU7V-6ZiqT"
      },
      "source": [
        "def create_look_ahead_mask(size):\n",
        "    mask = torch.ones(size, size).triu(diagonal=1)\n",
        "    return mask  # (seq_len, seq_len)"
      ],
      "id": "AbyU7V-6ZiqT",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MrkZ_SWsp_wi"
      },
      "source": [
        "def scaled_dot_product_attention(q, k, v, mask):\n",
        "    matmul_qk = torch.matmul(q, torch.transpose(k, -2, -1))  # (..., seq_len_q, seq_len_k)\n",
        "    \n",
        "    # scale matmul_qk\n",
        "    dk = k.size()[-1]\n",
        "    scaled_attention_logits = matmul_qk / math.sqrt(dk)\n",
        "    \n",
        "    # add the mask to the scaled tensor.\n",
        "    if mask is not None:\n",
        "        scaled_attention_logits += (mask * -1e9)\n",
        "\n",
        "    # softmax is normalized on the last axis (seq_len_k) so that the scores\n",
        "    # add up to 1.\n",
        "    attention_weights = torch.nn.functional.softmax(scaled_attention_logits, dim=-1)  # (..., seq_len_q, seq_len_k)\n",
        "\n",
        "    output = torch.matmul(attention_weights, v)  # (..., seq_len_q, depth_v)\n",
        "\n",
        "    return output, attention_weights"
      ],
      "id": "MrkZ_SWsp_wi",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i2MnjnFOZiqU"
      },
      "source": [
        "def print_out(q, k, v):\n",
        "    temp_out, temp_attn = scaled_dot_product_attention(\n",
        "      q, k, v, None)\n",
        "    print('Attention weights are:')\n",
        "    print(temp_attn)\n",
        "    print('Output is:')\n",
        "    print(temp_out)"
      ],
      "id": "i2MnjnFOZiqU",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "shEpA1pqZiqU",
        "outputId": "ff539fa3-2699-4247-fcd1-608c63f24248"
      },
      "source": [
        "np.set_printoptions(suppress=True)\n",
        "\n",
        "temp_k = torch.tensor([[10, 0, 0],\n",
        "                      [0, 10, 0],\n",
        "                      [0, 0, 10],\n",
        "                      [0, 0, 10]], dtype=torch.float32)  # (4, 3)\n",
        "\n",
        "temp_v = torch.tensor([[1, 0],\n",
        "                      [10, 0],\n",
        "                      [100, 5],\n",
        "                      [1000, 6]], dtype=torch.float32)  # (4, 2)\n",
        "\n",
        "# This `query` aligns with the second `key`,\n",
        "# so the second `value` is returned.\n",
        "temp_q = torch.tensor([[0, 10, 0]], dtype=torch.float32)  # (1, 3)\n",
        "print_out(temp_q, temp_k, temp_v)"
      ],
      "id": "shEpA1pqZiqU",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Attention weights are:\n",
            "tensor([[8.4333e-26, 1.0000e+00, 8.4333e-26, 8.4333e-26]])\n",
            "Output is:\n",
            "tensor([[1.0000e+01, 9.2766e-25]])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "--0Clh8jZiqU",
        "outputId": "04638f9a-7dfb-4364-ca62-4343018fdf8e"
      },
      "source": [
        "temp_q = torch.tensor([[10, 10, 0]], dtype=torch.float32)  # (1, 3)\n",
        "print_out(temp_q, temp_k, temp_v)"
      ],
      "id": "--0Clh8jZiqU",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Attention weights are:\n",
            "tensor([[5.0000e-01, 5.0000e-01, 4.2166e-26, 4.2166e-26]])\n",
            "Output is:\n",
            "tensor([[5.5000e+00, 4.6383e-25]])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hNJJzkjUZiqV",
        "outputId": "eaba3c90-5cd0-4b00-b368-bd7534ae5428"
      },
      "source": [
        "temp_q = torch.tensor([[0, 0, 10],\n",
        "                      [0, 10, 0],\n",
        "                      [10, 10, 0]], dtype=torch.float32)  # (3, 3)\n",
        "print_out(temp_q, temp_k, temp_v)"
      ],
      "id": "hNJJzkjUZiqV",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Attention weights are:\n",
            "tensor([[4.2166e-26, 4.2166e-26, 5.0000e-01, 5.0000e-01],\n",
            "        [8.4333e-26, 1.0000e+00, 8.4333e-26, 8.4333e-26],\n",
            "        [5.0000e-01, 5.0000e-01, 4.2166e-26, 4.2166e-26]])\n",
            "Output is:\n",
            "tensor([[5.5000e+02, 5.5000e+00],\n",
            "        [1.0000e+01, 9.2766e-25],\n",
            "        [5.5000e+00, 4.6383e-25]])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O_UgxaO8ZiqV"
      },
      "source": [
        "class MultiHeadAttention(nn.Module):\n",
        "    def __init__(self, d_model, num_heads):\n",
        "        super(MultiHeadAttention, self).__init__()\n",
        "        self.num_heads = num_heads\n",
        "        self.d_model = d_model\n",
        "\n",
        "        assert d_model % self.num_heads == 0\n",
        "\n",
        "        self.depth = d_model // self.num_heads\n",
        "\n",
        "        self.wq = nn.Linear(d_model, d_model)\n",
        "        self.wk = nn.Linear(d_model, d_model)\n",
        "        self.wv = nn.Linear(d_model, d_model)\n",
        "\n",
        "        self.wo = nn.Linear(d_model, d_model)\n",
        "        \n",
        "    def forward(self, v, k, q, mask):\n",
        "        batch_size = q.size()[0]\n",
        "        \n",
        "        q = self.wq(q).view(batch_size, -1, self.num_heads, self.depth).transpose(1, 2)\n",
        "        k = self.wk(k).view(batch_size, -1, self.num_heads, self.depth).transpose(1, 2)\n",
        "        v = self.wv(v).view(batch_size, -1, self.num_heads, self.depth).transpose(1, 2)\n",
        "        \n",
        "        # scaled_attention.shape == (batch_size, num_heads, seq_len_q, depth)\n",
        "        # attention_weights.shape == (batch_size, num_heads, seq_len_q, seq_len_k)\n",
        "        scaled_attention, attention_weights = scaled_dot_product_attention(q, k, v, mask)\n",
        "        \n",
        "        scaled_attention = scaled_attention.transpose(1,2).contiguous().view(batch_size, -1, self.num_heads * self.depth)\n",
        "                \n",
        "        output = self.wo(scaled_attention)  # (batch_size, seq_len_q, d_model)\n",
        "\n",
        "        return output, attention_weights"
      ],
      "id": "O_UgxaO8ZiqV",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LDMMzx6sZiqV",
        "outputId": "96366518-da0e-43fa-a4a8-3331f00622d5"
      },
      "source": [
        "temp_mha = MultiHeadAttention(d_model=512, num_heads=8)\n",
        "y = torch.rand(1, 60, 512)  # (batch_size, encoder_sequence, d_model)\n",
        "out, attn = temp_mha(y, k=y, q=y, mask=None)\n",
        "out.shape, attn.shape"
      ],
      "id": "LDMMzx6sZiqV",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(torch.Size([1, 60, 512]), torch.Size([1, 8, 60, 60]))"
            ]
          },
          "metadata": {},
          "execution_count": 44
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xR6HoS26ZiqV"
      },
      "source": [
        "class FFN(nn.Module):\n",
        "    def __init__(self, d_model, dff):\n",
        "        super(FFN, self).__init__()\n",
        "        self.layer1 = nn.Linear(d_model, dff)\n",
        "        self.activation = nn.ReLU()\n",
        "        self.fc = nn.Linear(dff, d_model)\n",
        "\n",
        "    def forward(self, x):\n",
        "        x = self.layer1(x)\n",
        "        x = self.activation(x)\n",
        "        x = self.fc(x)\n",
        "\n",
        "        return x"
      ],
      "id": "xR6HoS26ZiqV",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1CW6zuuvZiqV"
      },
      "source": [
        "class EncoderLayer(nn.Module):\n",
        "    def __init__(self, d_model, num_heads, dff, maximum_position_encoding, rate=0.1):\n",
        "        super(EncoderLayer, self).__init__()\n",
        "\n",
        "        self.mha = MultiHeadAttention(d_model, num_heads)\n",
        "        self.ffn = FFN(d_model, dff)\n",
        "        \n",
        "        self.layernorm1 = nn.LayerNorm([maximum_position_encoding, d_model])\n",
        "        self.layernorm2 = nn.LayerNorm([maximum_position_encoding, d_model])\n",
        "        \n",
        "        self.dropout1 = nn.Dropout(rate)\n",
        "        self.dropout2 = nn.Dropout(rate)\n",
        "\n",
        "\n",
        "    def forward(self, x, mask):\n",
        "        attn_output, _ = self.mha(x, x, x, mask)  # (batch_size, input_seq_len, d_model)\n",
        "        attn_output = self.dropout1(attn_output)\n",
        "        out1 = self.layernorm1(x + attn_output)  # (batch_size, input_seq_len, d_model)\n",
        "\n",
        "        ffn_output = self.ffn(out1)  # (batch_size, input_seq_len, d_model)\n",
        "        ffn_output = self.dropout2(ffn_output)\n",
        "        out2 = self.layernorm2(out1 + ffn_output)  # (batch_size, input_seq_len, d_model)\n",
        "\n",
        "        return out2"
      ],
      "id": "1CW6zuuvZiqV",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ip7GEe46ZiqV"
      },
      "source": [
        "# sample_encoder_layer = EncoderLayer(512, 8, 2048, encoder_len)\n",
        "\n",
        "# sample_encoder_layer_output = sample_encoder_layer(\n",
        "#     torch.rand(64, encoder_len, 512), None)\n",
        "\n",
        "# sample_encoder_layer_output.shape  # (batch_size, input_seq_len, d_model)"
      ],
      "id": "ip7GEe46ZiqV",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ogXv1pz9ZiqW"
      },
      "source": [
        "class DecoderLayer(nn.Module):\n",
        "    def __init__(self, d_model, num_heads, dff, maximum_position_encoding, rate=0.1):\n",
        "        super(DecoderLayer, self).__init__()\n",
        "\n",
        "        self.mha1 = MultiHeadAttention(d_model, num_heads)\n",
        "        self.mha2 = MultiHeadAttention(d_model, num_heads)\n",
        "        \n",
        "        self.ffn = FFN(d_model, dff)\n",
        "        \n",
        "        self.dropout1 = nn.Dropout(rate)\n",
        "        self.dropout2 = nn.Dropout(rate)\n",
        "        self.dropout3 = nn.Dropout(rate)\n",
        "        \n",
        "        self.layernorms1 = nn.ModuleList([copy.deepcopy(nn.LayerNorm([i+1, d_model])) for i in range(maximum_position_encoding)])\n",
        "        self.layernorms2 = nn.ModuleList([copy.deepcopy(nn.LayerNorm([i+1, d_model])) for i in range(maximum_position_encoding)])\n",
        "        self.layernorms3 = nn.ModuleList([copy.deepcopy(nn.LayerNorm([i+1, d_model])) for i in range(maximum_position_encoding)])\n",
        "\n",
        "    def forward(self, x, enc_output, look_ahead_mask, padding_mask):\n",
        "        # enc_output.shape == (batch_size, input_seq_len, d_model)\n",
        "        attn1, attn_weights_block1 = self.mha1(x, x, x, look_ahead_mask)  # (batch_size, target_seq_len, d_model)\n",
        "        attn1 = self.dropout1(attn1)\n",
        "        out1 = self.layernorms1[x.size(1)-1](attn1 + x)\n",
        "        \n",
        "        attn2, attn_weights_block2 = self.mha2(enc_output, enc_output, out1, padding_mask)  # (batch_size, target_seq_len, d_model)\n",
        "        attn2 = self.dropout2(attn2)\n",
        "        out2 = self.layernorms2[x.size(1)-1](attn2 + out1)  # (batch_size, target_seq_len, d_model)\n",
        "        \n",
        "        ffn_output = self.ffn(out2)  # (batch_size, target_seq_len, d_model)\n",
        "        ffn_output = self.dropout3(ffn_output)\n",
        "        out3 = self.layernorms3[x.size(1)-1](ffn_output + out2)  # (batch_size, target_seq_len, d_model)\n",
        "        \n",
        "        return out3, attn_weights_block1, attn_weights_block2"
      ],
      "id": "ogXv1pz9ZiqW",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QDXaP_0EZiqW"
      },
      "source": [
        "# sample_decoder_layer = DecoderLayer(512, 8, 2048, decoder_len)\n",
        "\n",
        "# sample_decoder_layer_output, _, _ = sample_decoder_layer(\n",
        "#     torch.rand(64, decoder_len, 512), sample_encoder_layer_output,\n",
        "#     None, None)\n",
        "\n",
        "# sample_decoder_layer_output.shape  # (batch_size, target_seq_len, d_model)"
      ],
      "id": "QDXaP_0EZiqW",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4JtoBGiPZiqW"
      },
      "source": [
        "def clones(module, N):\n",
        "    return nn.ModuleList([copy.deepcopy(module) for i in range(N)])"
      ],
      "id": "4JtoBGiPZiqW",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Pt4eCFTzZiqW"
      },
      "source": [
        "class Encoder(nn.Module):\n",
        "    def __init__(self, num_layers, d_model, num_heads, dff, input_vocab_size, maximum_position_encoding, device, rate=0.1):\n",
        "        super(Encoder, self).__init__()\n",
        "\n",
        "        self.d_model = d_model\n",
        "        self.num_layers = num_layers\n",
        "\n",
        "        self.embedding = nn.Embedding(input_vocab_size, d_model)\n",
        "        self.pos_encoding = positional_encoding(maximum_position_encoding, d_model).to(device)\n",
        "        \n",
        "        self.dec_layers = clones(EncoderLayer(d_model, num_heads, dff, maximum_position_encoding, rate), num_layers)\n",
        "        self.dropout = nn.Dropout(rate)\n",
        "\n",
        "    def forward(self, x, mask, enc_output=None):\n",
        "        if enc_output == None:\n",
        "            seq_len = x.size()[1]\n",
        "            attention_weights = {}\n",
        "            x = self.embedding(x)  # (batch_size, target_seq_len, d_model)\n",
        "            x *= torch.sqrt(torch.tensor(self.d_model, dtype=torch.float32))\n",
        "            x += self.pos_encoding[:, :seq_len, :]\n",
        "            x = self.dropout(x)\n",
        "            for i in range(self.num_layers):\n",
        "                x = self.dec_layers[i](x, mask)\n",
        "        else:\n",
        "            x = enc_output\n",
        "            \n",
        "        return x"
      ],
      "id": "Pt4eCFTzZiqW",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "by5CFZ57ZiqW"
      },
      "source": [
        "# sample_encoder = Encoder(num_layers=2, d_model=512, num_heads=8,\n",
        "#                          dff=2048, input_vocab_size=input_vocab_size,\n",
        "#                          maximum_position_encoding=encoder_len,\n",
        "#                          device='cpu')\n",
        "\n",
        "# temp_input = torch.randint(low=0, high=input_vocab_size, size=(64, encoder_len))\n",
        "\n",
        "# sample_encoder_output = sample_encoder(temp_input, mask=None, enc_output=None)\n",
        "\n",
        "# print(sample_encoder_output.shape)  # (batch_size, input_seq_len, d_model)"
      ],
      "id": "by5CFZ57ZiqW",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hpB7QSh4ZiqW"
      },
      "source": [
        "class Decoder(nn.Module):\n",
        "    def __init__(self, num_layers, d_model, num_heads, dff, target_vocab_size, maximum_position_encoding, device, rate=0.1):\n",
        "        super(Decoder, self).__init__()\n",
        "\n",
        "        self.d_model = d_model\n",
        "        self.num_layers = num_layers\n",
        "\n",
        "        self.embedding = nn.Embedding(target_vocab_size, d_model)\n",
        "        self.pos_encoding = positional_encoding(maximum_position_encoding, d_model).to(device)\n",
        "        \n",
        "        self.dec_layers = clones(DecoderLayer(d_model, num_heads, dff, maximum_position_encoding, rate), num_layers)\n",
        "        self.dropout = nn.Dropout(rate)\n",
        "        \n",
        "    def forward(self, x, enc_output, look_ahead_mask, padding_mask):\n",
        "        seq_len = x.size()[1]\n",
        "        attention_weights = {}\n",
        "        x = self.embedding(x)  # (batch_size, target_seq_len, d_model)\n",
        "        x *= torch.sqrt(torch.tensor(self.d_model, dtype=torch.float32))\n",
        "        x += self.pos_encoding[:, :seq_len, :]\n",
        "        x = self.dropout(x)\n",
        "        \n",
        "        for i in range(self.num_layers):\n",
        "            x, block1, block2 = self.dec_layers[i](x, enc_output, look_ahead_mask, padding_mask)\n",
        "\n",
        "            attention_weights['decoder_layer{}_block1'.format(i+1)] = block1\n",
        "            attention_weights['decoder_layer{}_block2'.format(i+1)] = block2\n",
        "            \n",
        "        # x.shape == (batch_size, target_seq_len, d_model)\n",
        "        return x, attention_weights"
      ],
      "id": "hpB7QSh4ZiqW",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8Qm56p5EZiqX"
      },
      "source": [
        "# sample_decoder = Decoder(num_layers=2, d_model=512, num_heads=8,\n",
        "#                          dff=2048, target_vocab_size=target_vocab_size,\n",
        "#                          maximum_position_encoding=decoder_len,\n",
        "#                          device='cpu')\n",
        "\n",
        "# temp_input = torch.randint(low=0, high=target_vocab_size, size=(64, decoder_len))\n",
        "\n",
        "# output, attn = sample_decoder(temp_input,\n",
        "#                               enc_output=sample_encoder_output,\n",
        "#                               look_ahead_mask=None,\n",
        "#                               padding_mask=None)\n",
        "\n",
        "# output.shape, attn['decoder_layer2_block2'].shape"
      ],
      "id": "8Qm56p5EZiqX",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XAvjYbVnZiqX"
      },
      "source": [
        "class Transformer(nn.Module):\n",
        "    def __init__(self, num_layers, d_model, num_heads, dff, input_vocab_size,\n",
        "               target_vocab_size, pe_input, pe_target, device, rate=0.1):\n",
        "        super().__init__()\n",
        "        self.device = device\n",
        "        self.encoder = Encoder(num_layers, d_model, num_heads, dff,\n",
        "                                 input_vocab_size, pe_input, device, rate)\n",
        "\n",
        "        self.decoder = Decoder(num_layers, d_model, num_heads, dff,\n",
        "                               target_vocab_size, pe_target, device, rate)\n",
        "\n",
        "        self.final_layer = nn.Linear(d_model, target_vocab_size)\n",
        "\n",
        "    def forward(self, inputs):\n",
        "        inp, tar, enc_output = inputs\n",
        "\n",
        "        enc_padding_mask, look_ahead_mask, dec_padding_mask = self.create_masks(inp, tar)\n",
        "\n",
        "        enc_output = self.encoder(inp, enc_padding_mask, enc_output)  # (batch_size, inp_seq_len, d_model)\n",
        "\n",
        "        # dec_output.shape == (batch_size, tar_seq_len, d_model)\n",
        "        dec_output, attention_weights = self.decoder(\n",
        "            tar, enc_output, look_ahead_mask, dec_padding_mask)\n",
        "\n",
        "        final_output = self.final_layer(dec_output)  # (batch_size, tar_seq_len, target_vocab_size)\n",
        "\n",
        "        return final_output, attention_weights, enc_output\n",
        "\n",
        "    def create_masks(self, inp, tar):\n",
        "        # Encoder padding mask\n",
        "        enc_padding_mask = create_padding_mask(inp)\n",
        "\n",
        "        # Used in the 2nd attention block in the decoder.\n",
        "        # This padding mask is used to mask the encoder outputs.\n",
        "        dec_padding_mask = create_padding_mask(inp)\n",
        "\n",
        "        # Used in the 1st attention block in the decoder.\n",
        "        # It is used to pad and mask future tokens in the input received by\n",
        "        # the decoder.\n",
        "        look_ahead_mask = create_look_ahead_mask(tar.size(1))\n",
        "        dec_target_padding_mask = create_padding_mask(tar)\n",
        "        look_ahead_mask = torch.maximum(dec_target_padding_mask.to(self.device), look_ahead_mask.to(self.device))\n",
        "\n",
        "        return enc_padding_mask, look_ahead_mask, dec_padding_mask"
      ],
      "id": "XAvjYbVnZiqX",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gqgfPF-mZiqX"
      },
      "source": [
        "# sample_transformer = Transformer(\n",
        "#     num_layers=2, d_model=512, num_heads=8, dff=2048,\n",
        "#     input_vocab_size=input_vocab_size, target_vocab_size=target_vocab_size,\n",
        "#     pe_input=encoder_len, pe_target=decoder_len, device='cpu')\n",
        "\n",
        "# temp_input = torch.randint(low=0, high=input_vocab_size, size=(64, encoder_len))\n",
        "# temp_target = torch.randint(low=0, high=target_vocab_size, size=(64, decoder_len))\n",
        "\n",
        "# fn_out, _, _ = sample_transformer([temp_input, temp_target, None])\n",
        "\n",
        "# fn_out.shape  # (batch_size, tar_seq_len, target_vocab_size)"
      ],
      "id": "gqgfPF-mZiqX",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rTrAbkkeZiqX"
      },
      "source": [
        "transformer = Transformer(\n",
        "    num_layers=num_layers,\n",
        "    d_model=d_model,\n",
        "    num_heads=num_heads,\n",
        "    dff=dff,\n",
        "    input_vocab_size=input_vocab_size,\n",
        "    target_vocab_size=target_vocab_size,\n",
        "    pe_input=encoder_len,\n",
        "    pe_target=decoder_len-1,\n",
        "    device=device,\n",
        "    rate=dropout_rate\n",
        ")\n",
        "\n",
        "transformer = transformer.to(device)"
      ],
      "id": "rTrAbkkeZiqX",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-IoBw-P3iCZE"
      },
      "source": [
        "# from transformers import BertModel"
      ],
      "id": "-IoBw-P3iCZE",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fobD_AJYb6cj"
      },
      "source": [
        "# model = BertModel.from_pretrained('monologg/kobert')"
      ],
      "id": "fobD_AJYb6cj",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ask0tjVkb76g"
      },
      "source": [
        "# transformer = model"
      ],
      "id": "Ask0tjVkb76g",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hRTvv0amZiqY"
      },
      "source": [
        "# 옵티마이저, 손실함수"
      ],
      "id": "hRTvv0amZiqY"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CyxvWuZXZiqY"
      },
      "source": [
        "optimizer = torch.optim.Adam(transformer.parameters(), lr=learning_rate)\n",
        "criterion = nn.CrossEntropyLoss()"
      ],
      "id": "CyxvWuZXZiqY",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jKE3m_4xZiqY"
      },
      "source": [
        "## 손실함수 및 평가함수 정의"
      ],
      "id": "jKE3m_4xZiqY"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mm8_QKnkZiqY"
      },
      "source": [
        "def loss_function(real, pred):\n",
        "    mask = torch.logical_not(torch.eq(real, 0))\n",
        "    loss_ = criterion(pred.permute(0,2,1), real)\n",
        "    mask = torch.tensor(mask, dtype=loss_.dtype)\n",
        "    loss_ = mask * loss_\n",
        "\n",
        "    return torch.sum(loss_)/torch.sum(mask)\n",
        "\n",
        "def accuracy_function(real, pred):\n",
        "    accuracies = torch.eq(real, torch.argmax(pred, dim=2))\n",
        "    mask = torch.logical_not(torch.eq(real, 0))\n",
        "    accuracies = torch.logical_and(mask, accuracies)\n",
        "    accuracies = torch.tensor(accuracies, dtype=torch.float32)\n",
        "    mask = torch.tensor(mask, dtype=torch.float32)\n",
        "    \n",
        "    return torch.sum(accuracies)/torch.sum(mask)"
      ],
      "id": "mm8_QKnkZiqY",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "39UOS3RAZiqY"
      },
      "source": [
        "## 학습 정의"
      ],
      "id": "39UOS3RAZiqY"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bltg9W1zZiqY"
      },
      "source": [
        "def train_step(batch_item, epoch, batch, training):\n",
        "    src = batch_item['src_token'].to(device)\n",
        "    tar = batch_item['tar_token'].to(device)\n",
        "    \n",
        "    tar_inp = tar[:, :-1]\n",
        "    tar_real = tar[:, 1:]\n",
        "    \n",
        "    if training is True:\n",
        "        transformer.train()\n",
        "        optimizer.zero_grad()\n",
        "        with torch.cuda.amp.autocast():\n",
        "            output, _, _ = transformer([src, tar_inp, None])\n",
        "            loss = loss_function(tar_real, output)\n",
        "        acc = accuracy_function(tar_real, output)\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "        lr = optimizer.param_groups[0][\"lr\"]\n",
        "        return loss, acc, round(lr, 10)\n",
        "    else:\n",
        "        transformer.eval()\n",
        "        with torch.no_grad():\n",
        "            output, _, _ = transformer([src, tar_inp, None])\n",
        "            loss = loss_function(tar_real, output)\n",
        "        acc = accuracy_function(tar_real, output)\n",
        "        return loss, acc"
      ],
      "id": "bltg9W1zZiqY",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Mcuc8mJIZiqZ"
      },
      "source": [
        "## 학습"
      ],
      "id": "Mcuc8mJIZiqZ"
    },
    {
      "cell_type": "code",
      "metadata": {
        "tags": [],
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "o_lJy4ORZiqZ",
        "outputId": "08ba9452-dec9-42c7-8297-c9f7fe9a3994"
      },
      "source": [
        "loss_plot, val_loss_plot = [], []\n",
        "acc_plot, val_acc_plot = [], []\n",
        "\n",
        "for epoch in range(epochs):\n",
        "    gc.collect()\n",
        "    total_loss, total_val_loss = 0, 0\n",
        "    total_acc, total_val_acc = 0, 0\n",
        "    \n",
        "    tqdm_dataset = tqdm(enumerate(train_dataloader))\n",
        "    training = True\n",
        "    for batch, batch_item in tqdm_dataset:\n",
        "        batch_loss, batch_acc, lr = train_step(batch_item, epoch, batch, training)\n",
        "        total_loss += batch_loss\n",
        "        total_acc += batch_acc\n",
        "        \n",
        "        tqdm_dataset.set_postfix({\n",
        "            'Epoch': epoch + 1,\n",
        "            'LR' : lr,                                                                \n",
        "            'Loss': '{:06f}'.format(batch_loss.item()),\n",
        "            'Total Loss' : '{:06f}'.format(total_loss/(batch+1)),\n",
        "            'Total ACC' : '{:06f}'.format(total_acc/(batch+1))\n",
        "        })\n",
        "    loss_plot.append(total_loss/(batch+1))\n",
        "    acc_plot.append(total_acc/(batch+1))\n",
        "    \n",
        "    tqdm_dataset = tqdm(enumerate(val_dataloader))\n",
        "    training = False\n",
        "    for batch, batch_item in tqdm_dataset:\n",
        "        batch_loss, batch_acc = train_step(batch_item, epoch, batch, training)\n",
        "        total_val_loss += batch_loss\n",
        "        total_val_acc += batch_acc\n",
        "        \n",
        "        tqdm_dataset.set_postfix({\n",
        "            'Epoch': epoch + 1,\n",
        "            'Val Loss': '{:06f}'.format(batch_loss.item()),\n",
        "            'Total Val Loss' : '{:06f}'.format(total_val_loss/(batch+1)),\n",
        "            'Total Val ACC' : '{:06f}'.format(total_val_acc/(batch+1))\n",
        "        })\n",
        "    val_loss_plot.append(total_val_loss/(batch+1))\n",
        "    val_acc_plot.append(total_val_acc/(batch+1))"
      ],
      "id": "o_lJy4ORZiqZ",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "66it [04:31,  4.11s/it, Epoch=1, LR=0.0001, Loss=2.454237, Total Loss=4.182315, Total ACC=0.115228]\n",
            "29it [00:19,  1.46it/s, Epoch=1, Val Loss=1.976757, Total Val Loss=3.069115, Total Val ACC=0.290053]\n",
            "66it [04:31,  4.11s/it, Epoch=2, LR=0.0001, Loss=2.870906, Total Loss=2.689737, Total ACC=0.360938]\n",
            "29it [00:19,  1.47it/s, Epoch=2, Val Loss=1.259266, Total Val Loss=2.430090, Total Val ACC=0.418013]\n",
            "66it [04:31,  4.11s/it, Epoch=3, LR=0.0001, Loss=2.727366, Total Loss=2.251300, Total ACC=0.440217]\n",
            "29it [00:19,  1.47it/s, Epoch=3, Val Loss=0.996184, Total Val Loss=2.173330, Total Val ACC=0.465065]\n",
            "66it [04:32,  4.13s/it, Epoch=4, LR=0.0001, Loss=1.461271, Total Loss=2.009227, Total ACC=0.481138]\n",
            "29it [00:19,  1.46it/s, Epoch=4, Val Loss=0.841526, Total Val Loss=2.012888, Total Val ACC=0.493326]\n",
            "66it [04:33,  4.15s/it, Epoch=5, LR=0.0001, Loss=2.118657, Total Loss=1.851978, Total ACC=0.510453]\n",
            "29it [00:19,  1.46it/s, Epoch=5, Val Loss=0.762108, Total Val Loss=1.916907, Total Val ACC=0.514181]\n",
            "66it [04:34,  4.16s/it, Epoch=6, LR=0.0001, Loss=2.094017, Total Loss=1.725316, Total ACC=0.531975]\n",
            "29it [00:19,  1.46it/s, Epoch=6, Val Loss=0.683927, Total Val Loss=1.839007, Total Val ACC=0.527345]\n",
            "66it [04:34,  4.16s/it, Epoch=7, LR=0.0001, Loss=1.825892, Total Loss=1.621555, Total ACC=0.552071]\n",
            "29it [00:19,  1.46it/s, Epoch=7, Val Loss=0.660682, Total Val Loss=1.784673, Total Val ACC=0.538827]\n",
            "66it [04:34,  4.16s/it, Epoch=8, LR=0.0001, Loss=0.888333, Total Loss=1.520470, Total ACC=0.571493]\n",
            "29it [00:19,  1.47it/s, Epoch=8, Val Loss=0.591826, Total Val Loss=1.727968, Total Val ACC=0.550184]\n",
            "66it [04:33,  4.14s/it, Epoch=9, LR=0.0001, Loss=1.426681, Total Loss=1.439465, Total ACC=0.585878]\n",
            "29it [00:19,  1.47it/s, Epoch=9, Val Loss=0.560404, Total Val Loss=1.693547, Total Val ACC=0.559934]\n",
            "66it [04:34,  4.15s/it, Epoch=10, LR=0.0001, Loss=1.900959, Total Loss=1.369747, Total ACC=0.600200]\n",
            "29it [00:19,  1.46it/s, Epoch=10, Val Loss=0.546634, Total Val Loss=1.662170, Total Val ACC=0.565371]\n",
            "66it [04:33,  4.15s/it, Epoch=11, LR=0.0001, Loss=1.681881, Total Loss=1.292211, Total ACC=0.615776]\n",
            "29it [00:19,  1.46it/s, Epoch=11, Val Loss=0.507015, Total Val Loss=1.627704, Total Val ACC=0.573461]\n",
            "66it [04:33,  4.15s/it, Epoch=12, LR=0.0001, Loss=0.763860, Total Loss=1.217555, Total ACC=0.630991]\n",
            "29it [00:19,  1.47it/s, Epoch=12, Val Loss=0.505375, Total Val Loss=1.619830, Total Val ACC=0.577376]\n",
            "66it [04:33,  4.15s/it, Epoch=13, LR=0.0001, Loss=1.440058, Total Loss=1.156683, Total ACC=0.644752]\n",
            "29it [00:19,  1.47it/s, Epoch=13, Val Loss=0.513579, Total Val Loss=1.598325, Total Val ACC=0.582967]\n",
            "66it [04:33,  4.14s/it, Epoch=14, LR=0.0001, Loss=1.259139, Total Loss=1.093556, Total ACC=0.659451]\n",
            "29it [00:19,  1.47it/s, Epoch=14, Val Loss=0.504909, Total Val Loss=1.585154, Total Val ACC=0.586863]\n",
            "66it [04:32,  4.13s/it, Epoch=15, LR=0.0001, Loss=1.046642, Total Loss=1.034604, Total ACC=0.674518]\n",
            "29it [00:19,  1.47it/s, Epoch=15, Val Loss=0.474748, Total Val Loss=1.568858, Total Val ACC=0.593032]\n",
            "66it [04:32,  4.13s/it, Epoch=16, LR=0.0001, Loss=0.871864, Total Loss=0.971660, Total ACC=0.689056]\n",
            "29it [00:19,  1.47it/s, Epoch=16, Val Loss=0.486965, Total Val Loss=1.563954, Total Val ACC=0.598144]\n",
            "66it [04:31,  4.12s/it, Epoch=17, LR=0.0001, Loss=0.552922, Total Loss=0.915020, Total ACC=0.703106]\n",
            "29it [00:19,  1.47it/s, Epoch=17, Val Loss=0.458644, Total Val Loss=1.560501, Total Val ACC=0.597486]\n",
            "66it [04:32,  4.13s/it, Epoch=18, LR=0.0001, Loss=1.230603, Total Loss=0.863493, Total ACC=0.719311]\n",
            "29it [00:19,  1.47it/s, Epoch=18, Val Loss=0.469824, Total Val Loss=1.553924, Total Val ACC=0.599814]\n",
            "66it [04:32,  4.13s/it, Epoch=19, LR=0.0001, Loss=0.986857, Total Loss=0.810409, Total ACC=0.733605]\n",
            "29it [00:19,  1.47it/s, Epoch=19, Val Loss=0.462223, Total Val Loss=1.551800, Total Val ACC=0.601533]\n",
            "66it [04:32,  4.13s/it, Epoch=20, LR=0.0001, Loss=0.903593, Total Loss=0.759614, Total ACC=0.748824]\n",
            "29it [00:19,  1.46it/s, Epoch=20, Val Loss=0.462727, Total Val Loss=1.561260, Total Val ACC=0.602581]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2kA28bcBZiqZ"
      },
      "source": [
        "## 학습 결과"
      ],
      "id": "2kA28bcBZiqZ"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jjsyTHYmZiqZ",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 279
        },
        "outputId": "ec4535ef-d0a1-4b58-eaed-c1ba50e0370f"
      },
      "source": [
        "plt.plot(loss_plot, label='train_loss')\n",
        "plt.plot(val_loss_plot, label='val_loss')\n",
        "plt.xlabel('epoch')\n",
        "plt.ylabel('loss')\n",
        "plt.legend()\n",
        "plt.show()"
      ],
      "id": "jjsyTHYmZiqZ",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxU9b3/8dcnyWTPJCEJ2VnCmrCFHRXQqrWKVBQXbNWKtZerVWtb6y3drOXqbe9mf/XWinvd6lLcqOIKKFIFZAlr2AWyQTayAdm/vz/OSRjCJASSmUkyn+fjcR5zcs53Zj4Zkrw55/s93yPGGJRSSvmvAF8XoJRSyrc0CJRSys9pECillJ/TIFBKKT+nQaCUUn4uyNcFnK34+HgzaNAgX5ehlFK9yoYNG0qNMQnu9vW6IBg0aBDr16/3dRlKKdWriMjB9vbpqSGllPJzGgRKKeXnNAiUUsrP9bo+AqVU39PQ0EB+fj61tbW+LqXXCw0NJS0tDYfD0ennaBAopXwuPz+fqKgoBg0ahIj4upxeyxhDWVkZ+fn5DB48uNPP01NDSimfq62tJS4uTkOgi0SEuLi4sz6y0iBQSvUIGgLd41w+R78Jgl2Hq/n9+7nU1DX6uhSllOpR/CYI8sqP88Rn+9l1uMrXpSilVI/iN0GQleIEYEehBoFS6lQVFRX85S9/OevnzZo1i4qKirN+3vz581myZMlZP89T/CYIkqNDiQ5zsKNIg0Apdar2gqCxseNTycuWLSMmJsZTZXmN3wwfFRGykp3sKKr2dSlKqQ787h/bu/3IPSvFyW+/Pard/QsXLmTfvn1kZ2fjcDgIDQ0lNjaWnTt3snv3bq6++mry8vKora3l3nvvZcGCBcDJuc9qamq44oormD59Ol988QWpqam88847hIWFnbG25cuX87Of/YzGxkYmT57M448/TkhICAsXLmTp0qUEBQVx2WWX8T//8z/8/e9/53e/+x2BgYFER0ezatWqbvl8/CYIADKTnfxt3UGamg2BATpCQSll+cMf/sC2bdvIycnh008/5corr2Tbtm2tY/GfffZZ+vXrx4kTJ5g8eTLXXnstcXFxp7zGnj17eOWVV3jqqae44YYbeOONN7j55ps7fN/a2lrmz5/P8uXLGT58ON/73vd4/PHHueWWW3jrrbfYuXMnItJ6+mnRokV8+OGHpKamntMpqfb4VRBkpTipbWjm69JjDO0f6etylFJudPQ/d2+ZMmXKKRdkPfroo7z11lsA5OXlsWfPntOCYPDgwWRnZwMwceJEDhw4cMb32bVrF4MHD2b48OEA3HrrrTz22GPcfffdhIaGcvvttzN79mxmz54NwAUXXMD8+fO54YYbmDt3bnd8q4Af9REAZCZHAWg/gVKqQxEREa3rn376KZ988glffvklmzdvZvz48W4v2AoJCWldDwwMPGP/QkeCgoJYt24d1113He+++y6XX345AIsXL+ahhx4iLy+PiRMnUlZWds7v4cqvgmBY/ygcgUKuBoFSykVUVBTV1e77DysrK4mNjSU8PJydO3eyZs2abnvfESNGcODAAfbu3QvAiy++yIUXXkhNTQ2VlZXMmjWLP/7xj2zevBmAffv2MXXqVBYtWkRCQgJ5eXndUofHTw2JSCCwHigwxsxusy8EeAGYCJQB84wxBzxVS3BQAEP7R+kQUqXUKeLi4rjgggsYPXo0YWFhJCYmtu67/PLLWbx4MZmZmYwYMYJp06Z12/uGhoby3HPPcf3117d2Ft9xxx2Ul5czZ84camtrMcbwyCOPAHD//fezZ88ejDFccskljBs3rlvqEGNMt7xQu28g8lNgEuB0EwQ/BMYaY+4QkRuBa4wx8zp6vUmTJpmu3KHsp6/n8PmeUr761aXn/BpKqe6Vm5tLZmamr8voM9x9niKywRgzyV17j54aEpE04Erg6XaazAGet9eXAJeIhyccyUp2UlJdR0l1nSffRimleg1P9xH8P+DfgOZ29qcCeQDGmEagEohr20hEFojIehFZX1JS0qWCspKtK4y1n0Ap5Wl33XUX2dnZpyzPPfecr8s6jcf6CERkNlBsjNkgIhd15bWMMU8CT4J1aqgrr5XpEgQzhyd05aWUUqpDjz32mK9L6BRPHhFcAFwlIgeAV4GLReSlNm0KgHQAEQkCorE6jT0mNiKY5OhQHUKqlFI2jwWBMeYXxpg0Y8wg4EZghTGm7WV2S4Fb7fXr7Dae7b3GOj2kp4aUUsri9esIRGSRiFxlf/kMECcie4GfAgu9UUNWipN9JceobWjyxtsppVSP5pUpJowxnwKf2usPuGyvBa73Rg2uMpOdNDUbdh+pZmxa7585UCmlusKvrixuoSOHlFJdFRnZ/nxlBw4cYPTo0V6spmv8MggG9AsnIjhQrzBWSin8bPbRFgEBwshkJ7l6bwKlep73F8Lhrd37mklj4Io/dNhk4cKFpKenc9dddwHw4IMPEhQUxMqVKzl69CgNDQ089NBDzJkz56zeura2ljvvvJP169cTFBTEI488wje+8Q22b9/ObbfdRn19Pc3NzbzxxhukpKRwww03kJ+fT1NTE7/5zW+YN6/DyRa6hV8GAVgzkb6zqRBjDB6+mFkp1QvMmzePH//4x61B8Prrr/Phhx/yox/9CKfTSWlpKdOmTeOqq646q78Zjz32GCLC1q1b2blzJ5dddhm7d+9m8eLF3Hvvvdx0003U19fT1NTEsmXLSElJ4b333gOsCe+8wW+DICs5mpfWHCL/6AnS+4X7uhylVIsz/M/dU8aPH09xcTGFhYWUlJQQGxtLUlISP/nJT1i1ahUBAQEUFBRw5MgRkpKSOv26q1ev5p577gFg5MiRDBw4kN27d3Peeefx8MMPk5+fz9y5cxk2bBhjxozhvvvu4+c//zmzZ89mxowZnvp2T+GXfQRw8mb227WfQCllu/7661myZAmvvfYa8+bN4+WXX6akpIQNGzaQk5NDYmKi23sRnIvvfve7LF26lLCwMGbNmsWKFSsYPnw4GzduZMyYMfz6179m0aJF3fJeZ+K3QTAiMYoA0ZvUKKVOmjdvHq+++ipLlizh+uuvp7Kykv79++NwOFi5ciUHDx4869ecMWMGL7/8MgC7d+/m0KFDjBgxgv3795ORkcGPfvQj5syZw5YtWygsLCQ8PJybb76Z+++/n40bN3b3t+iW354aCgsOZHB8hA4hVUq1GjVqFNXV1aSmppKcnMxNN93Et7/9bcaMGcOkSZMYOXLkWb/mD3/4Q+68807GjBlDUFAQf/3rXwkJCeH111/nxRdfxOFwkJSUxC9/+Uu++uor7r//fgICAnA4HDz++OMe+C5P5/H7EXS3rt6PwNXdf9vIpkMV/HPhxd3yekqpc6P3I+hePep+BD1dVoqTgooTVJ5o8HUpSinlM357aghOnZJ6WsZpt0FQSqkObd26lVtuueWUbSEhIaxdu9ZHFZ0bvw6CURoESvUYvfGanjFjxpCTk+PrMk5xLqf7/frUUEJUCPGRwTrVhFI+FhoaSllZ2Tn9EVMnGWMoKysjNDT0rJ7n10cEIkJmslOHkCrlY2lpaeTn59PVW9EqK1TT0tLO6jl+HQRgzUT63D8P0NDUjCPQrw+QlPIZh8PB4MGDfV2G3/L7v3yZyU7qm5rZV1Lj61KUUson/D4IWqaa0AvLlFL+yu+DICM+guCgAO0wVkr5Lb8PgqDAAEYkRum9CZRSfstjQSAioSKyTkQ2i8h2EfmdmzbzRaRERHLs5QeeqqcjWfbIIR26ppTyR548IqgDLjbGjAOygctFZJqbdq8ZY7Lt5WkP1tOuzOQoyo/Vc6Sqzhdvr5RSPuWxIDCWlqE4Dnvpkf/lzkqJBrTDWCnlnzzaRyAigSKSAxQDHxtj3E3Aca2IbBGRJSKS7sl62jMyOQrQexMopfyTR4PAGNNkjMkG0oApIjK6TZN/AIOMMWOBj4Hn3b2OiCwQkfUist4TVx46Qx2k9wvTIFBK+SWvjBoyxlQAK4HL22wvM8a0nJh/GpjYzvOfNMZMMsZMSkhI8EiNWclOcnUIqVLKD3ly1FCCiMTY62HAN4Gdbdoku3x5FZDrqXrOJDPZyddlxzhe3+irEpRSyic8OddQMvC8iARiBc7rxph3RWQRsN4YsxT4kYhcBTQC5cB8D9bToaxkJ8bAzsPVTBgQ66sylFLK6zwWBMaYLcB4N9sfcFn/BfALT9VwNlpuUrOjsEqDQCnlV/z+yuIWabFhRIUG6RBSpZTf0SCw6b0JlFL+SoPARVayk12Hq2lq7pHXvSmllEdoELjISnFyvL6Jg2XHfF2KUkp5jQaBi6zWm9nrTKRKKf+hQeBiaP9IggKEHUWVvi5FKaW8RoPARagjkCEJkXqTGqWUX9EgaCMrxamnhpRSfkWDoI3M5CgOV9VSfqze16UopZRXaBC0kZWs9yZQSvkXDYI2MlvuTaD9BEopP6FB0EZcZAiJzhA9IlBK+Q0NAjeydKoJpZQf0SBwIzPZyd7iGuoam3xdilJKeZwGgRtZKU4amw17jtT4uhSllPI4DQI3Wu9NoKeHlFJ+QIPAjUFxEYQ5ArXDWCnlFzQI3AgMEEYmR+kQUqWUX/CfIGhqhL3LwXTuXgOZyU5yi6ownWyvlFK9lf8EQc7L8NJcyP+qU82zkp1U1TZSUHHCw4UppZRveSwIRCRURNaJyGYR2S4iv3PTJkREXhORvSKyVkQGeaoeRl8LIdGw5vFONXe9mb1SSvVlnjwiqAMuNsaMA7KBy0VkWps2twNHjTFDgT8C/+mxakIiYcItsOMdqCw4Y/ORSVGI6E1qlFJ9n8eCwFhaBuI77KXtCfc5wPP2+hLgEhERT9XElH+xSvjq6TM2jQgJYlBchN6kRinV53m0j0BEAkUkBygGPjbGrG3TJBXIAzDGNAKVQJzHCoodBCNmwYa/QsOZz/1nJeu9CZRSfZ9Hg8AY02SMyQbSgCkiMvpcXkdEFojIehFZX1JS0rWipt4BJ8phy+tnbJqV4uRQ+XGqaxu69p5KKdWDeWXUkDGmAlgJXN5mVwGQDiAiQUA0UObm+U8aYyYZYyYlJCR0rZhB0yFxtNVpfIahoS1TUu88rEcFSqm+y5OjhhJEJMZeDwO+Cexs02wpcKu9fh2wwnh64L4ITLsTSnLh6886bNpykxodOaSU6ss8eUSQDKwUkS3AV1h9BO+KyCIRucpu8wwQJyJ7gZ8CCz1Yz0mjr4PweFizuMNmic4QYsMdGgRKqT4tyFMvbIzZAox3s/0Bl/Va4HpP1dAuRyhMug1W/Q+U74d+GW6biYh1M/vDGgRKqb7Lf64sbmvS7RAQCGuf7LBZZpKTnYeraWxq9lJhSinlXf4bBM5kGHUNbHoJatv/H39WipP6xma+Lj3mxeKUUsp7/DcIAKbeCfXVkPO3dptkpei9CZRSfZt/B0HaREibDOuegGb3p36GJEQSHBigQaCU6rP8OwjAusCsfD/s+cjtbkdgAMMSI3XkkFKqz9IgyJoDUSmw5i/tNmm5N4FSSvVFGgSBDpjyA+visiM73DbJSnZSWlNPcXWtl4tTSinP0yAAmHgbBIXCWvcXmLV2GOvpIaVUH6RBABDeD8beAFteg+Plp+3OTLKCQGciVUr1RRoELabeAY211hTVbUSHO0iNCdORQ0qpPkmDoEXiKBg807ppTdPp005rh7FSqq/SIHA19U6oKoDcf5y2KyvFyf6SGk7UN/mgMKWU8hwNAlfDvwWxg912Go9Li6bZwLKtRT4oTCmlPEeDwFVAIEz9V8hbCwUbT9l10Yj+jB8Qw8PLcjl6rN5HBSqlVPfTIGgr+yYIjjrtqCAwQPj93DFUnWjg4WW5PipOKaW6nwZBW6FOGH8TbHsTqg+fsmtkkpN/vTCDJRvy+WJvqY8KVEqp7qVB4M6UBdDcCF89c9quey4exqC4cH751lZqG7TjWCnV+2kQuBM3xOo4Xv8sNJw6rUSoI5D/uGYMB8qO838r9vioQKWU6j4aBO2ZegccL4Vtb5y26/yh8Vw3MY0nPtvPTr2NpVKql+tUEIjIvSLiFMszIrJRRC7zdHE+lXERJGTC2sfBmNN2/2pWJs4wBwvf2EpT8+n7lVKqt+jsEcH3jTFVwGVALHAL8IeOniAi6SKyUkR2iMh2EbnXTZuLRKRSRHLs5QF3r+UTIjDtDji8FQ5+cdru2IhgHpidRU5eBS+vPeiDApVSqnt0NgjEfpwFvGiM2e6yrT2NwH3GmCxgGnCXiGS5afe5MSbbXhZ1sh7vGHMDhMVaRwVuzMlOYcaweP7rg10UVZ7wcnFKKdU9OhsEG0TkI6wg+FBEogD393a0GWOKjDEb7fVqIBdI7UqxXhccDhPnw8734Ojp/+sXER6+egyNzc389p3t3q9PKaW6QWeD4HZgITDZGHMccAC3dfZNRGQQMB5Y62b3eSKyWUTeF5FR7Tx/gYisF5H1JSUlnX3b7jH5B4DAV0+53T0gLpyfXDqcj3Yc4YNth922UUqpnqyzQXAesMsYUyEiNwO/Bio780QRiQTeAH5s9zO42ggMNMaMA/4PeNvdaxhjnjTGTDLGTEpISOhkyd0kOg2yroINL0Bdjdsmt08fTFayk98u3UZV7ekzlyqlVE/W2SB4HDguIuOA+4B9wAtnepKIOLBC4GVjzJtt9xtjqowxNfb6MsAhIvGdLd5rpt4JdZWw+RW3u4MCA/j93DGUVNfx3x/s8nJxSinVNZ0NgkZjjAHmAH82xjwGRHX0BBER4Bkg1xjzSDttkux2iMgUu56yzhbvNelTIGU8rH0Cmt13jYxLj2H++YN5ae1BNhw8/S5nSinVU3U2CKpF5BdYw0bfE5EArH6Cjlxgt7/YZXjoLBG5Q0TusNtcB2wTkc3Ao8CNduD0LCJw3t1Qtgfe+wk0u59a4r7LhpMSHcYv3txKfWOHfelKKdVjBHWy3Tzgu1jXExwWkQHAf3f0BGPMas4wxNQY82fgz52swbdGXwvFO+Dz/4UTFTD3SQgKOaVJREgQ/371KL7/1/U8uWofd188zEfFKqVU53XqiMAYcxh4GYgWkdlArTHmjH0EfYoIXPIAXPYQ7Hgb/jbPbefxxSMTuXJsMo+u2Mv+Evedy0op1ZN0doqJG4B1wPXADcBaEbnOk4X1WOffA3Meg68/gxevhuOn9wf89ttZhAQF8Ku3ttETz3QppZSrzvYR/ArrGoJbjTHfA6YAv/FcWT3c+JvhhhegaDM8NwuqTr19Zf+oUH45K5Mv95exZEO+j4pUSqnO6WwQBBhjil2+LjuL5/ZNmd+Gm5ZAZR48exmU7Ttl97xJ6UweFMvDy3IpranzUZFKKXVmnf1j/oGIfCgi80VkPvAesMxzZfUSGRfCrUutvoJnL7cmqLMF2Le2PFbXyEPv7vBhkUop1bHOdhbfDzwJjLWXJ40xP/dkYb1G6kT4/gcQ6IDnroSDX7buGto/ih9eNJS3cwr5bLeXp8ZQSqlO6vTpHWPMG8aYn9rLW54sqtdJGAHf/xAiE+DFa2D3R627fviNIWQkRPCrt7ZyvL7Rh0UqpZR7HQaBiFSLSJWbpVpE9NZcrmLS4bYPIGE4vPod2PJ3AEKCAvn9NWPIP3qCP32it7ZUSvU8HQaBMSbKGON0s0QZY5zeKrLXiEyAW9+F9Gnw5r/AOmvG0qkZcXxnSjpPr/6aZVuLzvAiSinlXf498scTQp1w8xIYcQUs+xl8+p9gDL+YlUl2egx3/W0jT3++X68vUEr1GBoEnuAIgxtehHHfgU//Az5YiDM4kJd/MJUrRifx0Hu5PLh0u97rWCnVI3R2riF1tgKDYM5frFtdrvkLnKggdM6f+fN3JvCH2J08uWo/BRW1PPqdbMKD9Z9BKeU7ekTgSQEB8K3/gG/8Gra8Cq/dTEDtUX45K5PfXTWKFTuP8J0n11BSrRecKaV8R4PA00Tgwvvhyv+FPR/B/02AdU9x69Q0nrhlEruOVDP38X+yTyeoU0r5iAaBt0z+AdyxGhJHW53IT8zgm6E7eW3BeZyob2LuX75g3dd6QxullPdpEHhT4ii49R8w7yWoPwYvXMW4f97F0pvSiIsM5uan1/KPzYW+rlIp5Wc0CLxNxJqw7q511v0N9q0k5aWZLBu1gqlpIdzzyiYe/3SfDi9VSnmNBoGvOEJhxn1wzwYYfS2ha/7EC8fu5N8HbeW/PtjBr9/eRmOT3u5SKeV5GgS+5kyGaxbDD5Yj0anccvj3/DPu92xft4IFL27gWJ3OT6SU8iwNgp4ibRLc/glc8wQpUsbbIQ8we9+D3Ln4XYqran1dnVKqD/NYEIhIuoisFJEdIrJdRO5100ZE5FER2SsiW0Rkgqfq6RUCAmDcjXDPepj+U652rGVx+QLefPQ+9hboNNZKKc/w5BFBI3CfMSYLmAbcJSJZbdpcAQyzlwXA4x6sp/cIiYJLf0vA3V/RMPhi7mh8idCnzmfnipegucnX1Sml+hiPBYExpsgYs9FerwZygdQ2zeYALxjLGiBGRJI9VVOv028w0fNfpeSa12kICGPkqruo/v0I6t/9N8jfADqySCnVDbzSRyAig4DxwNo2u1KBPJev8zk9LBCRBSKyXkTWl5T43ymShHHfot99a3kh9UG+qB0I65+Bpy/GPJoNyxfBke2+LlEp1YuJp8eri0gk8BnwsDHmzTb73gX+YIxZbX+9HPi5MWZ9e683adIks359u7v7vK35lfzv0nXEF3zMjWHrmNi0BTFNkJAJo6+F0XMhboivy1RK9TAissEYM8ndPo9OeykiDuAN4OW2IWArANJdvk6zt6l2jEmL5rk7L+WDbaP5yfu5HC8/zE9SdnBt0FrCVj4EKx+ClAknQ8GZ4uuSlVI9nMeOCEREgOeBcmPMj9tpcyVwNzALmAo8aoyZ0tHr+vsRgau6xib++s8D/HnFXk40NPHDCaHcGb+ZsF1vQ1EOIDDwfCsUsuZARLyvS1ZK+UhHRwSeDILpwOfAVqDlEtlfAgMAjDGL7bD4M3A5cBy4raPTQqBB4E5pTR1//Hg3r6w7RFSog3svGcYtwxtw5L4NW5dA6S6QQMi4yLpz2tBLoF+Gr8tWSnmRT4LAUzQI2rfzcBUPv5fL53tKyYiP4JezMrlkZAJSvAO2LYEd70D5fqtx7GAYeqm1DJ4BwRG+LV4p5VEaBH7EGMPKXcU89F4u+0uOccHQOH59ZRaZyU6rQdk+2LcC9n4CX6+ChuMQGAwDzjsZDP0zrcnxlFJ9hgaBH2poaublNQf5f8v3UHWigXmT0/npN0eQEBVyslFjHRz60gqFvcuheIe1PSrFOn009FLIuNC63aZSqlfTIPBjFcfreXT5Xl748gChjkBumjqA2y4YTFJ06OmNKwtg33IrGPZ9CnWVIAGQNtkKhSGXQNIYCAr29rehlOoiDQLF/pIaHvl4N8u2FhEYIFw1LpUFMzMYkRTl/glNjVCw3jpS2PsJFG4CDAQ4IDELkrMheZz1mJgFjjCvfj9KqbOjQaBa5ZUf55nVX/PaV3mcaGjiwuEJ/OvMDM4bEod01C9wrNTqUyjKgaLN1nLiqLVPAiFhpBUMKXZAJI6GkEjvfFNKqTPSIFCnqThez0trDvLXLw5QWlPP6FQnC2YOYdboJIICOzHziDFQmXcyFIo2Q2EOHCu2GwjED7OPGuwlaSyExXj0+1JKuadBoNpV29DEW5sKeGrVfvaXHiMtNozbpw/mhknpRIScw4XnVUWnhkPRZqjKP7m/X4Z15XPKeGtJHqdHDkp5gQaBOqPmZsMnuUd4ctV+1h88SnSYg1umDeR75w+kf5SbjuWzcazUDoUcq6+hMMc6mgBAIGGEHQx2QCSN1j4HpbqZBoE6KxsOHuXJVfv4aMcRHIEBzB2fyg9mZDC0fzf+z72m2AqEwk1QuBEKNp48rRQQZF3L4BoO/bN0tJJSXaBBoM7J/pIanln9NUs25FPX2MylmYncNHUAM4bFd64f4WwYA1WFdjDY4VC46WSHdGAwJI6CfkMgZgDEpNuPAyE6TY8glDoDDQLVJaU1dbzw5UFeWnOQ8mP1JESFcM34VK6bmMbwxHaGn3YHY6DioHW0ULjJOr1UcdC63qG54dS2Ef3tYHCzRKdDcLjn6lSqF9AgUN2ivrGZlbuKWbIhn5U7i2lsNoxNi+baCWlcNS6F2AgvnbppboLqw1BxyFoqD51crzgEFXlugiLBCoToVHCm2Y+p1tGEMxWikiAg0Dv1K+UDGgSq25XV1PFOTiFLNuSzo6gKR6BwaWYi101MY+bwBBzdferobDQ3Q80Rl2A4aHVOVxyyjiaqCqC+5tTnSCBEJbsEhJvACI+HAB9+X0p1gQaB8qjthZW8saGAd3IKKDtWT3xkCFdnp3DdpDRGJjl9Xd7pjIHaSisQKgus4a0tAVGZf3J7U92pzwsMtgLhlFNPA60jjZgBelShejQNAuUVDU3NfLqrhCUb8lixs5iGJsPoVCfXTUjjquxU+nnr1FF3MAaOl50aDFX51mmnliON1ovnbAEOl6BIt0LCtZ/CmaJBoXxGg0B5XfmxepbmFLBkYz7bCqxTRxeP7M/cCWl8Y0R/goP6wCmW+uNWULjtpzhknZ5yJYHW6KagEAgMsYbDBoZYX7dua1kPhqDQNm1CrSuzw+OsJawfhPez1kOjdepw1SENAuVTuUVVvLEhn7dzCiitqSc23MHssSlcMyGV8ekxHc9x1Js1nDgZFBWHrPWGE9BYa512aqy3H+2lqd5l3XVbrdW2sRZMk/v3Cgiyg8EOifBYl3WX4AiJsm5CFBxxct0RriHiBzQIVI/Q2NTM53tLeXNjAR9tP0xdYzOD4yO4Znwq14xPJb2fDvHskDFQV22dsjpRDsfLrfXTlqOnft1eeLSSk+HQukS6PEaeGh6hTgiJth+dpz4GR2mHeg+lQaB6nOraBt7fdpg3N+azZn85AJMHxTJ3QhqzxiQTHebwcYV9REvH+PEyKzjqq6H+mL3UnFyvqzn169Z1l8e6ms6FSkiUdaqqbUiEOK1TXAGBVn9KQBAE2o9t19vdF2idYnN9DAiy7pvhdl+bdgFB1mm3wNuu/IgAABIhSURBVGDrNXvykZAx0NQAjSeso8OGE1YoR8Sd08tpEKgeraDiBG9vKuCtTQXsLa4hOCiASzP7M3d8GheO8PFQVHWSMdYfo7oqqK2yHyvbfN32sfJkm7pq6xRXc6N1nUdzo6+/IyuQWkIhMNjun2mzzXVdAq3QkQArRFrX3X3dZhtifd8NtdZpvgb7D3zjiZPbWrfb66b51Hqn/wQuffCcvlWfBIGIPAvMBoqNMaPd7L8IeAf42t70pjFm0ZleV4Og7zLGsLWgkjc3FvCPzYWUHaunX0Qw3x6bzDUT0hiXFt13+xP8kTHWxYGuwdDkut5g73fZZ1raN9nrTdYfS7fb7LYt21raNNXbS4PVF9PU0GZbO+uNddbrmmar9tZ1lwXTcZsABzhCISjMfrSXlkEEp2232waFWG0SR1v3/DgHvgqCmUAN8EIHQfAzY8zss3ldDQL/0NDUzKrdJby5qYCPdxyhvrGZjIQILstK4hsjEpg4MLb75ztSqg/rKAjOYcL5zjHGrBKRQZ56fdW3OQIDuCQzkUsyE6mqbeD9rUW8k1PI05/vZ/Fn+3CGBjFzeAIXj+zPhcMTiIsM8XXJSvVaHguCTjpPRDYDhVhHB9vdNRKRBcACgAEDBnixPNUTOEMdzJs8gHmTB1Bd28DqPaWs2FnMyl0lvLulCBEYlxbDxSP7c/HI/oxKceopJKXOgkc7i+0jgnfbOTXkBJqNMTUiMgv4kzFm2JleU08NqRbNzYbthVWs2FnMil3FbMmvwBjoHxXCRSOso4XpwxKIPJc7rSnVx/hs1FBHQeCm7QFgkjGmtKN2GgSqPaU1dXy2q4QVu4pZtbuE6tpGHIHC5EH9uHhkf74xsj8Z8RF6tKD8Uo8MAhFJAo4YY4yITAGWAAPNGQrSIFCd0dDUzMaDR1mxq5iVO4vZfcSabTQ5OpRpGXFMy+jHtIw4BvQL12BQfsFXo4ZeAS4C4oEjwG8BB4AxZrGI3A3cCTQCJ4CfGmO+ONPrahCoc5F/9Dif7iphzf4y1uwvp7TGmlk0pTUY4jhvSBxpsWEaDKpP0gvKlHJhjGFfSQ1f7i9nzf4y1u4vo7SmHoDUmDCmZvTjPDscdNoL1VdoECjVAWMMe4tr+HJ/WesRQ/kxKxjSYsNajximZfQjLVaDQfVOGgRKnYXmZsOe4ho7FKzl6HHr1pcZ8RHMHJ7AhcMTmJYRR1iw3l9A9Q4aBEp1QXOzYXdxNV/sLWPVHqufobahmeCgAKYM6seFwxOYOTyB4YmR2r+geiwNAqW6UW1DE18dKGfV7hI+213SOiIpyRnKzOHxXDi8P9OHxhMdrjOoqp5Dg0ApDyqsOMHne6xQWL2nlKraRgIEstNjuHB4f2YOj2dsWgyBAXq0oHxHg0ApL2lsamZzfgWf7S7ls90lrVc7x4Q7mD40npnDEpg+LJ6UmDBfl6r8jAaBUj5Sfqye1XtL+WxXCav2lFBSbV2/kJEQwYyh8VwwNJ7zhsQRFaqnkZRnaRAo1QMYY9h9pIbP95Swem8pa/eXc6KhicAAITs9hulD45kxLJ5x6TF6Mx7V7TQIlOqB6hqb2HiwgtV7rb6FLQWVGAORIUFMy4hjxjDriGFIgs6PpLpOg0CpXqDieD1f7Cvj8z2lrN5bQl75CcCaBuOCofFMHxbP5EH9tH9BnRMNAqV6oYNlx6xQ2FPKF/us0UhgDVMdPyDGXmIZkxpNqEMvbFMd88kdypRSXTMwLoKBcRHcPG0gTc2GbQWVbDx0lE2HKtiUd5T3tx0GIChAyEpxMj7dCoYJA2JJ76eT56nO0yMCpXqpkuo6cvIq2GSHw+b8Co7XNwEQFxHcesQwPj2GsekxeoMeP6dHBEr1QQlRIXwzK5FvZiUC1jUMu4/UsCnPPmo4dJRPcosBCBAYnhjFhIGxTBoYy8SBsXovBtVKjwiU6sMqjzeQk2+FwsZDFWw6eJTqOquvIT4yhIkDY5hoB8OoFO1r6Mv0iEApPxUd7uBCe7ZUODmz6vqD5Ww4eJSNB4/y4fYjAAQHBjAmLZqJA61+hokDY0mICvFl+cpL9IhAKT9XUl3HxkNWKGw4eJQt+ZXUNzUDMDAunIkDYplgHzUMT4zSOZN6KR0+qpTqtLrGJrYVVLHx4FH7yKGi9dae4cGBjEmNJntADNlpMYxLjyE5OlT7GnoBDQKl1DkzxpBXfoINh8rJOVRBTn4luYVVrUcN/aNCGJceQ7a9jEmLxqlzJ/U42keglDpnIsKAuHAGxIVzzfg0wDpqyC2qZnNeBTl5FWzOq+DjHUfs9jAkIZJxaTFkp0eTnR7LiKQogoN0/qSeymNHBCLyLDAbKDbGjHazX4A/AbOA48B8Y8zGM72uHhEo1TNVHK9nS35lazjk5FVQZt/7OTgogFEpTiYMODl8tb8z1McV+xefnBoSkZlADfBCO0EwC7gHKwimAn8yxkw90+tqECjVOxhjKKg40XrEkJNXwZb8SuoarVNKabFhrUNXJwyIZWRSFEE666rH+OTUkDFmlYgM6qDJHKyQMMAaEYkRkWRjTJGnalJKeY+IkBYbTlpsOLPHpgBQ39jMjqIq1h8oZ+Oho3y5r4x3cgoBiAgOJHtATOsopfEDYokO074Gb/BlH0EqkOfydb697bQgEJEFwAKAAQMGeKU4pVT3Cw4KaO1UhpNHDS3XNGw4dJTHPt1HU7N1pmJ4YuQp1zUMjtcpuT2hV3QWG2OeBJ4E69SQj8tRSnUT16OGOdmpAByra2RzfoU9fPUo720p4pV11v8ZnaFBjE2zRiaNS4tmTFoMKTp8tct8GQQFQLrL12n2NqWUH4sICeL8IfGcPyQesK6G3ldSY13sVlDJlvwKnlq1n0b7qCEuIpgxadGMTY1mbFoMY9OitSP6LPkyCJYCd4vIq1idxZXaP6CUaisgQBiWGMWwxChutLfVNjSx63A1W/KtDuitBZWs2l2CnQ0kOkMYkxpjHzVEMyY1mrhInS6jPR4LAhF5BbgIiBeRfOC3gAPAGLMYWIY1Ymgv1vDR2zxVi1Kqbwl1BDIu3bqyucXx+kZ2FFa1BsOW/AqW7zxCy8DI1JgwRqU4yUpxkpXsJDPZSVqs3rcB9MpipVQfVl3bwLaCKrYWWEcOO4qq+Lr0WGs4RIUGkZlsBUNWshUSwxIjCQnqe7Ow6pXFSim/FBXq4LwhcZw3JK512/H6RnYeria3qIodhVXkFlXx+vq81pv6BAUIQxIiyUpxkpkcRVZyNJnJUX361JIGgVLKr4QHBzHBvqVni+Zmw8Hy463BsKOoii/3lfHWppPjVxKdIWQlOxmVEs2oFOuxr9wSVINAKeX3AgKEwfERDI6P4Mqxya3by4/Vk1tU1Xr0sL2wilV7Sluvc4gKDTo1HFKdDE2I7HVXSGsQKKVUO/pFBHPB0HguGBrfuq1lxNL2wiq2F1ayvbCKv607SG2DNXVGcFAAI5Oi7I5pKyAyk5yEBffcfgcNAqWUOgvuRiw1NjXzdemxU8Jh2dbDrRfCBQhkJES2jlay+h6cJESF9IhTSzpqSCmlPKBl+ozt9imlHYWV5BZVU1BxorVNXEQwI5OjyExqCQgnQ/tHemTKbh01pJRSXuY6fca3RiW1bq883kDu4arWvofcompeWHOQentWVkegPWop2WmFhB0Q8R4ctaRBoJRSXhQd7mBaRhzTMk4OaW1sauZA2TF2FFW3BsQ/95XypsuopYSoEBbMyOBfZmZ0e00aBEop5WNBgQEM7R/F0P5RXDUupXX7KaOWiqro7/TMUYEGgVJK9VDuRi15Qu8a7KqUUqrbaRAopZSf0yBQSik/p0GglFJ+ToNAKaX8nAaBUkr5OQ0CpZTycxoESinl53rdpHMiUgIcPMenxwOl3VhOd+vp9UHPr1Hr6xqtr2t6cn0DjTEJ7nb0uiDoChFZ397sez1BT68Pen6NWl/XaH1d09Pra4+eGlJKKT+nQaCUUn7O34LgSV8XcAY9vT7o+TVqfV2j9XVNT6/PLb/qI1BKKXU6fzsiUEop1YYGgVJK+bk+GQQicrmI7BKRvSKy0M3+EBF5zd6/VkQGebG2dBFZKSI7RGS7iNzrps1FIlIpIjn28oC36rPf/4CIbLXfe72b/SIij9qf3xYRmeDF2ka4fC45IlIlIj9u08brn5+IPCsixSKyzWVbPxH5WET22I+x7Tz3VrvNHhG51Yv1/beI7LT/Dd8SkZh2ntvhz4MH63tQRApc/h1ntfPcDn/fPVjfay61HRCRnHae6/HPr8uMMX1qAQKBfUAGEAxsBrLatPkhsNhevxF4zYv1JQMT7PUoYLeb+i4C3vXhZ3gAiO9g/yzgfUCAacBaH/5bH8a6UMannx8wE5gAbHPZ9l/AQnt9IfCfbp7XD9hvP8ba67Fequ8yIMhe/0939XXm58GD9T0I/KwTPwMd/r57qr42+/8XeMBXn19Xl754RDAF2GuM2W+MqQdeBea0aTMHeN5eXwJcIiLijeKMMUXGmI32ejWQC6R647270RzgBWNZA8SISLIP6rgE2GeMOdcrzbuNMWYVUN5ms+vP2fPA1W6e+i3gY2NMuTHmKPAxcLk36jPGfGSMabS/XAOkdff7dlY7n19ndOb3vcs6qs/+23ED8Ep3v6+39MUgSAXyXL7O5/Q/tK1t7F+ESiDOK9W5sE9JjQfWutl9nohsFpH3RWSUVwsDA3wkIhtEZIGb/Z35jL3hRtr/5fPl59ci0RhTZK8fBhLdtOkpn+X3sY7y3DnTz4Mn3W2funq2nVNrPeHzmwEcMcbsaWe/Lz+/TumLQdAriEgk8AbwY2NMVZvdG7FOd4wD/g9428vlTTfGTACuAO4SkZlefv8zEpFg4Crg7252+/rzO42xzhH0yLHaIvIroBF4uZ0mvvp5eBwYAmQDRVinX3qi79Dx0UCP/33qi0FQAKS7fJ1mb3PbRkSCgGigzCvVWe/pwAqBl40xb7bdb4ypMsbU2OvLAIeIxHurPmNMgf1YDLyFdfjtqjOfsaddAWw0xhxpu8PXn5+LIy2nzOzHYjdtfPpZish8YDZwkx1Wp+nEz4NHGGOOGGOajDHNwFPtvK+vP78gYC7wWnttfPX5nY2+GARfAcNEZLD9v8YbgaVt2iwFWkZnXAesaO+XoLvZ5xOfAXKNMY+00yappc9CRKZg/Tt5JahEJEJEolrWsToUt7VpthT4nj16aBpQ6XIKxFva/V+YLz+/Nlx/zm4F3nHT5kPgMhGJtU99XGZv8zgRuRz4N+AqY8zxdtp05ufBU/W59jtd0877dub33ZMuBXYaY/Ld7fTl53dWfN1b7YkFa1TLbqzRBL+yty3C+oEHCMU6pbAXWAdkeLG26VinCLYAOfYyC7gDuMNuczewHWsExBrgfC/Wl2G/72a7hpbPz7U+AR6zP9+twCQv//tGYP1hj3bZ5tPPDyuUioAGrPPUt2P1Oy0H9gCfAP3stpOAp12e+337Z3EvcJsX69uLdX695eewZSRdCrCso58HL9X3ov3ztQXrj3ty2/rsr0/7ffdGffb2v7b83Lm09frn19VFp5hQSik/1xdPDSmllDoLGgRKKeXnNAiUUsrPaRAopZSf0yBQSik/p0GglBfZM6O+6+s6lHKlQaCUUn5Og0ApN0TkZhFZZ88h/4SIBIpIjYj8Uaz7SCwXkQS7bbaIrHGZ1z/W3j5URD6xJ7/bKCJD7JePFJEl9r0AXvbWzLdKtUeDQKk2RCQTmAdcYIzJBpqAm7CuaF5vjBkFfAb81n7KC8DPjTFjsa6Ebdn+MvCYsSa/Ox/rylSwZpz9MZCFdeXpBR7/ppTqQJCvC1CqB7oEmAh8Zf9nPQxrwrhmTk4u9hLwpohEAzHGmM/s7c8Df7fnl0k1xrwFYIypBbBfb52x56ax72o1CFjt+W9LKfc0CJQ6nQDPG2N+ccpGkd+0aXeu87PUuaw3ob+Hysf01JBSp1sOXCci/aH13sMDsX5frrPbfBdYbYypBI6KyAx7+y3AZ8a6+1y+iFxtv0aIiIR79btQqpP0fyJKtWGM2SEiv8a6q1QA1oyTdwHHgCn2vmKsfgSwpphebP+h3w/cZm+/BXhCRBbZr3G9F78NpTpNZx9VqpNEpMYYE+nrOpTqbnpqSCml/JweESillJ/TIwKllPJzGgRKKeXnNAiUUsrPaRAopZSf0yBQSik/9/8BRAUoV8iE95gAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CPYpFFSzZiqZ",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 279
        },
        "outputId": "3977a319-03ac-4bb8-993a-4add61fcecb2"
      },
      "source": [
        "plt.plot(acc_plot, label='train_acc')\n",
        "plt.plot(val_acc_plot, label='val_acc')\n",
        "plt.xlabel('epoch')\n",
        "plt.ylabel('acc')\n",
        "plt.legend()\n",
        "plt.show()"
      ],
      "id": "CPYpFFSzZiqZ",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXyU1b348c83O1nJMmFJggkQIOxgBFQ2ERVtFffdVtsr7a222t4u3NarXtv7q11+vbe9P2uvdfdqtWK1tKXFhc0FlYQl7AQQzATIyoSsZJnz++N5EoYwCQHyzCSZ7/v1ymtmnufMzDdPJuf7zDnnOUeMMSillApdYcEOQCmlVHBpIlBKqRCniUAppUKcJgKllApxmgiUUirERQQ7gDOVlpZmsrOzgx2GUkr1K4WFhZXGGJe/ff0uEWRnZ1NQUBDsMJRSql8RkYNd7dOmIaWUCnGaCJRSKsRpIlBKqRDX7/oI/GlpacHtdtPU1BTsUPqtmJgYMjMziYyMDHYoSqkAGxCJwO12k5CQQHZ2NiIS7HD6HWMMVVVVuN1ucnJygh2OUirABkTTUFNTE6mpqZoEzpKIkJqaqt+olApRAyIRAJoEzpEeP6VC14BJBEopNRB5vYat7hp+/W4xOw8fc+Q9BkQfgVJKDSQNza18UFzJql3lrNpVTnntcUQgJT6KvGGJvf5+mgh6gcfj4ZVXXuEb3/jGGT3vqquu4pVXXmHw4MEORaaU6i9Kqhs6Kv71+6tobvWSEB3B3DEuFoxLZ/5YF6nx0Y68tyaCXuDxePjtb397SiJobW0lIqLrQ7xixQqnQ1NK9VGtbV42lXh4b2c5q3aVsaesDoCRaXF8adZ5LMhL54LsFCLDnW/BH3CJ4N//sp0dh3q3HW388EQeuXpCl/uXLl3Kvn37mDp1KpGRkcTExJCcnMyuXbvYs2cP1157LSUlJTQ1NfHAAw+wZMkS4MS8SXV1dVx55ZXMnj2bjz76iIyMDP785z8zaNAgv+/3+9//nqeeeorm5mZGjx7NSy+9RGxsLGVlZXz9619n//79ADz55JNcdNFFvPjii/zyl79ERJg8eTIvvfRSrx4fpVTP1DS0sGaPdda/dk8FnoYWIsKEGTkp3JyfxYJx6Yx0xQc8rgGXCILh8ccfZ9u2bWzevJk1a9bwhS98gW3btnWMyX/22WdJSUmhsbGRCy64gBtuuIHU1NSTXqO4uJg//OEP/P73v+fmm2/mjTfe4M477/T7ftdffz333nsvAA899BDPPPMM3/zmN/nWt77FvHnzePPNN2lra6Ouro7t27fzk5/8hI8++oi0tDSqq6udPRhKqQ5er2HH4WOs3VPB2t0VFH5+lDavISUuikvHDeHSvHRm56aRGBPcCzkHXCLo7sw9UGbMmHHShVm/+c1vePPNNwEoKSmhuLj4lESQk5PD1KlTATj//PM5cOBAl6+/bds2HnroITweD3V1dVxxxRUArFq1ihdffBGA8PBwkpKSePHFF7nppptIS0sDICUlpdd+T6XUqSpqj/N+cQXr9lTwfnElVfXNAIwflsg/zxvFgrx0pmQOJjys7wzZHnCJoC+Ii4vruL9mzRreffdd1q9fT2xsLPPnz/d74VZ09IlOoPDwcBobG7t8/bvvvpu33nqLKVOm8Pzzz7NmzZpejV8p1XPNrV4KDx5lnV35b7ebplPiopibm8bcMS7m5LpwJTjT0dsbNBH0goSEBGpra/3uq6mpITk5mdjYWHbt2sXHH398zu9XW1vLsGHDaGlp4eWXXyYjIwOASy+9lCeffJIHH3ywo2lowYIFXHfddXznO98hNTWV6upq/Vag1Dk6WFXPuj0VrN1Tyfp9ldQ3txERJkw/L5nvXTGWubkuJgxPJKwPnfV3RxNBL0hNTeXiiy9m4sSJDBo0iCFDhnTsW7RoEb/73e/Iy8tj7NixzJo165zf78c//jEzZ87E5XIxc+bMjiT061//miVLlvDMM88QHh7Ok08+yYUXXsiPfvQj5s2bR3h4ONOmTeP5558/5xiUCiVNLW18uLeStXuss/4DVQ0AZCYP4tppGcwd4+KiUakkBLmt/2yJMSbYMZyR/Px803mFsp07d5KXlxekiAYOPY5KnVDT2MLqXeWs3H6EtXsqaGhuY1BkOLNGpjBvjIu5Y1zkpMX1m+lZRKTQGJPvb59+I1BKKVv5sSZW7ijj7e1HWL+vilavwZUQzXXTMrh8wlBmjUwhOiI82GH2Ok0Efdh9993Hhx9+eNK2Bx54gHvuuSdIESk18HxWWc/K7UdYuf0Imz73AJCdGstXZ+dw+YShTMsa3G/a+s+Wo4lARBYBvwbCgaeNMY932v+fwCX2w1gg3Rij8y3YnnjiiWCHoNSAY4xhW+kxVm4/wts7jnRc0TsxI5F/uWwMV0wcSm56fL9p8ukNjiUCEQkHngAuA9zABhFZbozZ0V7GGPNtn/LfBKY5FY9SKnQ1t3opOFjN29vLeGdHGaWeRsIEZuSk8MjV47ls/BAyk2ODHWbQOPmNYAaw1xizH0BEXgUWAzu6KH8b8IiD8SilQoQxhv2V9bxvX9S1fn8VDc1tREeEMSfXxQMLc1mYN4SUuKhgh9onOJkIMoASn8duYKa/giJyHpADrOpi/xJgCcCIESN6N0ql1IDgaWjmo31V9lW9lZR6rIsys1NjuWF6JrNz05g9Oo24aO0a7ayvHJFbgWXGmDZ/O40xTwFPgTV8NJCBKaX6ppY2L5tLPLy/p4J1xZUUuT14DSTERHDxqDS+ccko5ox2MSI1dJt8esrJRFAKZPk8zrS3+XMrcJ+DsfQp8fHx1NXVBTsMpfoVYwwHqxqsM/7iStbvq6LueCthAlOzBvPNBbnMHZPGlMzBRARg6uaBxMlEsAHIFZEcrARwK3B750IiMg5IBtY7GItSqh+qP97KR/uqWLvHmra5pNpq7slMHsQ1U4czNzeNC0elkTSof17R21c4lgiMMa0icj+wEmv46LPGmO0i8hhQYIxZbhe9FXjV9NYlzn9fCke29spLdRg6Ca58vMvdS5cuJSsri/vus77UPProo0RERLB69WqOHj1KS0sLP/nJT1i8ePFp36quro7Fixf7fZ6/dQW6WoNAqf7IGMOesjrW7ilnze4KNhyopqXNEBsVzkWjUlkyZyRzcl2clxobUsM7nTbwppgIQiLYtGkTDz74IGvXrgVg/PjxrFy5kqSkJBITE6msrGTWrFkUFxcjIt02DbW2ttLQ0HDK83bs2MF111130roCKSkp3HLLLVx44YUnTTSXlJR0Vr+mTjGhguFYUwsf7a1kze4K1u6p4HCNNTvv2CEJzBvrYv4YF+dnJw/IK3oDKbSmmOimwnbKtGnTKC8v59ChQ1RUVJCcnMzQoUP59re/zbp16wgLC6O0tJSysjKGDh3a7WsZY/jhD394yvNWrVrld10Bf2sQKNWXGWMt1tJe8W88eJRWryEhOoLZuWk8cKk1j8/wwf5X6FO9b+AlgiC56aabWLZsGUeOHOGWW27h5ZdfpqKigsLCQiIjI8nOzva7DkFnZ/s8pfqymsYW3i+u6Kj8K2qPAzBheCJL5o5k/th0po0YHJD1edWpNBH0kltuuYV7772XyspK1q5dyx//+EfS09OJjIxk9erVHDx4sEevU1NT4/d5Xa0r4G8NAv1WoILNGENxeR2rdlnr8xYetJZoTBoUyZzcNOaNcTFvjIv0xJhgh6rQRNBrJkyYQG1tLRkZGQwbNow77riDq6++mkmTJpGfn8+4ceN69DpdPW/ChAl+1xXoag0CpQKtqaWN9fuqOir/9gu68oYl8vV5I1kwLl2HdvZRA6+zWJ01PY7qTB3yNLJqVzmrd5Xz4b5Kmlq8DIoM5+LRqSwYN4RLxrkYlqRt/X1BaHUWK6Uc09rmZVOJp6Py33XEWh0vK2UQt+Rnccm4dGaNTCUmUkf49CeaCIJk69at3HXXXSdti46O5pNPPglSREr5V1V3nHXFFazeZXX01jS2EBEm5Gcn88OrxrFgXDqjXKE1bfNAM2ASgTGmX30QJ02axObNm4MdRof+1kSonNPmNRS5PazZXcGa3eUUldZgDKTGRXFpXjqXjhvCnDFpJPbT9XnVqQZEIoiJiaGqqorU1NR+lQz6CmMMVVVVxMToCI5QVV3fzLo9VsW/dk8FRxtaEHsOnwcvHcMl41xMHJ404FfqClUDIhFkZmbidrupqKgIdij9VkxMDJmZmcEOQwWI12soKq1hze5yVu+uoMjt6Tjrnz82nfljXczNdZGs8/WHhAGRCCIjI8nJyQl2GEr1aUfrm1nnc1FXdX0zIjAl0zrrnz/WxaQMPesPRQMiESilTnW8tY3CA0d5f28lH+6tZKvd1p8SF8W8MS7mj3UxJ9elq3QpTQRKDRRer2HXkVo+2Gstz7jhQDVNLV4iwoRpI6yz/nljXUzWs37ViSYCpfqxQ55GPiiu5AP7rL+qvhmA3PR4br1gBHNy05g5MpV4XZ5RdUM/HUr1I8eaWvh4XxUf7LUq//0V9QC4EqKZO8bF7NFpXDw6jaFJOgJM9ZwmAqX6MGMMOw/XsmpXGat3V7C5xEOb11qoZWZOCrfPGMGcXBdjhugFXersaSJQqo9pbG7jo32VHZO3tS/UMiUziW/MH8XFo9OYPiKZqAidvE31Dk0ESvUB7ZO3rdpVzod7Kzne6iUuKpw5uS6+fZk1rj89QZt7lDM0ESgVBG1ew+YSD6t3lfPernJ2Hj4GwIiUWG6bMYJL89KZkZOiyzOqgNBEoFSAHGtq4f09lby3q4w1u60LusLDhPzz2idvG8IoV5y29auA00SglINKqht4d2cZ7+4s45P91bR6DYNjI5k/xsWCvCHMy3WRFKuTt6ng0kSgVC/yeg2b3R7e3VHGezvL2V1mzdc/Oj2er87JYWHeEKZl6Spdqm/RRKDUOWpobuWD4kre3VnGql3lVNZZTT4zslN46At5LMwbQnZaXLDDVKpLjiYCEVkE/BoIB542xjzup8zNwKOAAbYYY253MialekPZsSbe21nOuzvLOkb5JERHMH9cOgvz0pk/Jl2bfFS/4VgiEJFw4AngMsANbBCR5caYHT5lcoF/BS42xhwVkXSn4lHqXBhj2HH4WEflX+SuAawlGm+fOYKFeUO4IDtFx/YPZG2t0NoIrcehxb5tbYSWJmi1f3y3d5Rr338c2prtn1bwtkBby4nbjvvt+zqXa4VLH4bJN/f6r+bkN4IZwF5jzH4AEXkVWAzs8ClzL/CEMeYogDGm3MF4lDojR+ubeX9vJWt3V/B+cQXltcc7Fmv53hVjWZg3RK/odYq3DeoroPaI9VN3BOoqAAMSBmEREBYOEm7d+t6XcJ/9Yfb+COt+SyM010NLAzTXWfeb7fstDfbjep8y9Xa5Bmg7fva/j4RDRAyER0J4lHUbFmHfRtrbfe5HRPsvlzC01w6xLycTQQZQ4vPYDczsVGYMgIh8iNV89Kgx5h+dX0hElgBLAEaMGOFIsEq1tnnZ4vawdncFa4srOxZrSRoUyezcNOblurhkXDquhOhgh9p/tbXaFfxhqCuzbmvLfB7bFX99ORiv8/FIGETFQ1QcRMZat1HxEJsCUVnW/Y7tcRA5yKrQ238iY/w8HmRV5JH2bcQgCO/b3bHBji4CyAXmA5nAOhGZZIzx+BYyxjwFPAWQn5+vi+uqXnPI08i6PRWsK67gg+JKjjW1Emaf9T9waS7zxriYnDmYcJ22uWcaPVBTAp4SqHFDzef2fftxfYWfCl4gLs06240fCkMnQsIwiB9i3SYMtX7i0q2K27RZ3xi8rfZ9r8/99u3eTmXarNvIWLtityv/iGjQb3SOJoJSIMvncaa9zZcb+MQY0wJ8JiJ7sBLDBgfjUiGsqaWNTz+rZt0ea5Wu4vI6AIYmxnDlxGEdM3hqR68fXq911t65gu+o9Evg+LGTnxMeDUmZMDgLci+DhOEnKvb2ij8+3Wr66LFgn78OPE4e0Q1ArojkYCWAW4HOI4LeAm4DnhORNKymov0OxqRCjDGGfRX1HRX/J59V0dTiJSoijJk5KdxyQRZzx7jITQ+xtn6vF47XQEO1/VMFjZ3vV0HD0ZP3eVtOfp2YJEgaAcnnQfbsE5V+0gjrNjYNwrQDva9zLBEYY1pF5H5gJVb7/7PGmO0i8hhQYIxZbu+7XER2AG3A94wxVU7FpEJDbVMLH+2rYu2eCtburqDU0wjASFcct80YwdwxLmblpDIoagDO49PWanWsHjsEx0rh2GH79pDVDt9QZVfsR7tugw+LgEEpVjt5bCqkjoLYGdbjxAwYPAKSsqxKPyYxsL+fcoQY07+a3PPz801BQUGww1B9iNdrDe1ca5/1bzx4lFavIT46gotHpzJ3jIu5uS6yUmKDHeq5aWmyKvNjh3wq+kNQe+jEtrqyUyv4iEGQONxqb49LO1HB+1b2HfdTIDpR280HIBEpNMbk+9unjW2qX6qqO84H9tDOdcUVVNZZSzROzEhkydyRzBvjYvp5yUT21akcWpo6NcFUnzhT933s2yzTXHfq60QnWpV84nBIz7PO2BOGWbft2wcla8WuuqWJQPULXq9h4+dHO876t5bWYAykxEUxNzeNuWNczMl19Z2hnS2NULUPKndDZTFU7rEet1fuLQ1dPzc60aq8Y1OtM3jXWPuMPRUSh9kVvF3ha9OM6gWaCFSf9nlVA8sKS3hjYymlnkbCw4TpIwbznYVjmDfWxcThSYQFc2hnfZVVyftW+BW7wfM51qwpAGK1q6eOts7aY1NPVPQnNc3Y2yOigvf7qJCkiUD1OfXHW1mx9TCvF7r59LNqRGBOrovvLxrL/LHpJA0K4NBOY6zmmtrD1hDJyuITlX7FbqvZpl1EDKTmQsb5MPV2SMuFtLFWZ2vkoMDFrNQZ0kSg+gRjDJ9+Vs3rhW5WbD1MQ3MbOWlxfO+KsVw/PYNhSQ5UpG0t9pWsh0+MqmkfaeO7rbXp5OfFpkHaGMi72mq2SRtjVfpJI3SopOqXNBGooCr1NPKnQjfLNro5WNVAXFQ4V08ezk35mZx/XvK5j+1vroey7XCkCMp3Qk2pPcrmsHWVK51GzYVHW+3wCcMhY7rd8Tr8RAdsWq7VnKPUAKKJQAVcU0sbK7cf4fUCNx/uq8QYmDUyhW8tyOXKSUOJjTrLj2V9JRzeAke2WhX/4SKo2ktHZR+dZF3klDAMhk2xKvv2Sj/Rruh1hI0KQZoIVEAYYy3W/nqhm79sOURtUysZgwfxrQW53DA9kxGpZzDG3xg4euBEZd9e8dcePlEmaQQMnQSTbrRuh062LoDSSl6pU2giUI46Wt/MnzaV8tqGz9lTVkdMZBhXThzGTednMmtk6ulH/BgDnoNQsgFKC60K/8jWE3PaSLjVTp8zz6rwh02GIRO1+UapM6CJQPU6r9ewfn8Vr24oYeW2IzS3eZmSNZj/c90kvjhlGIkx3Yz6aa6HQ5vAvcGq/N0brCmJwZo1cshEa2GO9rP89DwdkaPUOdJEoHpN2bEmXi8o4bWCEkqqG0mMieD2mSO45YIs8ob5ufDJGKjeD+4CcH9qVfpHtlnTBQOkjILRl0LmBdZP+vg+P6+7Uv2R/lepc9La5mX17gpe2/A5q3aV4zVw4chUvnv5WK6YMJSYSJ+J3Y7XwaGNUPKpXflvgIZKa19UvDX+fva3IWsGZORDXGpwfimlQowmAnVWDlbV88eCEl4vcFNee5y0+Gi+Nm8UN+dnkZMW53O2v8Gu+D+1hnG2T4iWmgtjrvA528+zlhRUSgWcJgLVY+3DPl/bUMJH+6oIE5g/Np1bL8jiklHxRB7ZAjtXnKj8O872EyDzfJjzXfts/3ztzFWqD9FEoE7rYFU9//vxQV4vdONpaCFzcAyPzolncVopyVVr4INP4Y1t1rKAYM2pk3s5ZF0AWTPBNU7P9pXqwzQRKL+8XsPaPRW8uP4A6/aUMSnsAA8PK2FuxmekHt2CbCizCkbGWVfgXvwAZM6wmnm0bV+pfkUTgTqJp6GZ1zd8zofr32dkbSH3RO3kydhdxLTVQSWQnAMj51sVftZMHcmj1ACg/8EKjGHPjo1s/+AvDCr9iBtkO/dKHUSCGTwSGXkj5MyF82ZDwpBgR6uU6mWaCELV0QO07F3DkS1vE3doPWO81YwBPNHpkLMIxi+EnDlIUmawI1VKOUwTQahoqoHd/4DP1tG6bw0RtW4igRiTxKbIKUSPnceUOdcweFiuzsejVIjRRDDQHdoEBc9iti5DWhqoD0vg/ZZxfORdiMmew2Vz53JJriu4q3wppYJKE8FA1FwP296Agmfh0Ca84TGsjprH/6u7iIMx47j5omzunTmCrJQzmPFTKTVgOZoIRGQR8GsgHHjaGPN4p/13A78ASu1N/88Y87STMQ1oZTug8DnY8iocP0Zr6lj+MvQBHjk4kUiSeXDxGG46P/PkaR+UUiHPsUQgIuHAE8BlgBvYICLLjTE7OhV9zRhzv1NxDHgtTbBzuXX2//l6CI+iLW8xf464goc2xtPSZvjKnBzuWzC6+1k/lVIhy8lvBDOAvcaY/QAi8iqwGOicCNTZqNpnnf1vetlaQD1lJOayH/NO1EL+/b0jlHoauXy8ix9elUd2Wlywo1VK9WFOJoIMoMTnsRuY6afcDSIyF9gDfNsYU9K5gIgsAZYAjBgxwoFQ+4m2Fti9wjr7378GwiJg7FWQ/xW2Rk3lsb/tZMOBzxg3NIFX/mkmF41OC3bESql+INidxX8B/mCMOS4iXwNeABZ0LmSMeQp4CiA/P9903j/gtTbDhqfhw19D3RFIyoIFD8G0uygzg/nFyt28sfEjUmKj+On1k7g5P4twHQWklOohJxNBKZDl8ziTE53CABhjqnwePg383MF4+h9jYPub8N6/W2v05syFa34DoxfS1AZPv7+f367ZTGubYcnckdx3ifYDKKXOnJOJYAOQKyI5WAngVuB23wIiMswY077i+DXATgfj6V8Oroe3H4LSAkifAHe+AaMXYozhb1sP89MVuyj1NLJowlD+9apxnJeq/QBKqbPjWCIwxrSKyP3ASqzho88aY7aLyGNAgTFmOfAtEbkGaAWqgbudiqffqCyGdx+FXX+FhGGw+AmYchuEhVPk9vDYX3ZQcPAoecMS+cVNk7lolPYDKKXOjRjTv5rc8/PzTUFBQbDD6H11FbDmp1D4vLVI++wHYNZ9EBVLm9fwy7d38+SafaTFR/Hdy8dyk/YDKKXOgIgUGmPy/e0Ldmexam6A9U/Ah/8FLY2Qfw/MWwrxLsCaFvqbf9jE+8WV3DYjix9elUeC9gMopXqRJoJg8bbB5ldg9X9A7WEY90VY+Cik5XYU2Xn4GEteKqCs5jg/vX4St80I4aGzSinHaCIINGNg73vwzsNQvh0y8uHG5+C8C08qtnzLIX6wrIjEQRG8+rVZTB+RHKSAlVIDnSaCQDpcBO/8m3UxWHI23PQ8jL/2pGmfW9u8/Hzlbp5at58LspN54o7ppCfEBCtipVQI0EQQCMbAul9azUCDBsOixyH/qxARdVKx6vpmvvmHjXy4t4ovXXgeD31hPFERYUEKWikVKjQROM3rhX/8AD59CibdDFf9wkoGnWwrreFrLxVSUXecn984mZvzs/y8mFJK9T5NBE5qbYa3/hm2LYML74fLfgxhp57hv7WplB+8UURKXBSvf+1CpmSdmiiUUsopmgic0lwPr90F+96zRgNd/OApS0C2tHn56YpdPPvhZ8zISeG3d0wnLT46KOEqpUKXJgInNFTDyzfBoY1wzX/D9C+dUqSy7jj3v7KRj/dXc/dF2fzoC3lEhmt/gFIq8HqUCETkOmCVMabGfjwYmG+MecvJ4PqlGje8dL01SdzNL0Le1acUKXJ7+PpLhVTVN/Orm6dw/fTMwMeplFK2np6CPtKeBACMMR7gEWdC6scq9sAzV8CxQ9YkcX6SwLJCNzf+bj0iwhv/fJEmAaVU0PW0achfwtBmJV+lhfC/N0JYONzzNxg25aTdLW1efvLXHbyw/iAXjUrlv2+bRqr2Byil+oCeVuYFIvIrrDWIAe4DCp0JqR/atwpevRPiUuGutyB11ClFfv6PXbyw/iD/NDuHpVeOI0L7A5RSfURPa6NvAs3Aa8CrQBNWMlDb/gQv32xdKfyVt/0mgc0lHp754DNumzGCh744XpOAUqpP6dE3AmNMPbDU4Vj6nw1Pw9++C1kz4fZXYdCp8wE1t3r5wbIi0hNi+NerxgUhSKWU6l6PTk1F5B17pFD742QRWelcWH2cMbDmZ/C3f4ExV8Bdb/pNAgC/W7uP3WW1/OTaibqMpFKqT+ppH0GaPVIIAGPMURFJdyimvs13yogpt1nXCYT7r+CLy2r571XFXDNlOAvHDwlwoEop1TM9baz2ikjHZPgikg30r6XNekNrM/zpn6wkcOH9sPi3XSaBNq/h+28UER8dwSNXjw9woEop1XM9/UbwI+ADEVkLCDAHWOJYVH1Rcz28dqc1Qmjhv8PFD5wyZYSvFz46wKbPPfzXLVN1mKhSqk/raWfxP0QkH6vy3wS8BTQ6GVif8/6vYN/qLqeM8FVS3cAvVu7mkrEuFk8dHqAAlVLq7PR0iol/Ah4AMoHNwCxgPbDAudD6kLYW2PSS1TF8miRgjOGHb24lTOA/rpuEdPOtQSml+oKe9hE8AFwAHDTGXAJMAzzdP2UA2f13qCuD8+8+bdFlhW7eL65k6VV5DB88yPnYlFLqHPU0ETQZY5oARCTaGLMLGHu6J4nIIhHZLSJ7RaTL6xBE5AYRMXbzU99T+BwkZsDoy7otVl7bxI//uoMZ2SncoQvNK6X6iZ4mArd9HcFbwDsi8mfgYHdPEJFwrCkprgTGA7eJyCnDZ0QkAesbxydnEnjAVH9mdRBP/xKEd9+S9sift9PU6uXxGyYRFqZNQkqp/qGnncXX2XcfFZHVQBLwj9M8bQaw1xizH0BEXgUWAzs6lfsx8DPgez0NOqA2vggSBtPu6rbYP7Yd5u/bjvD9RWMZ6YoPUHBKKXXuznjSG2PMWmPMcmNM82mKZgAlPo/d9rYOIjIdyDLG/JU8NzoAABGESURBVK27FxKRJSJSICIFFRUVZxry2WtrgU3/C7lXQFJGl8VqGlr4tz9vZ8LwRO6dMzJw8SmlVC8I2uxnIhIG/Ar4l9OVNcY8ZYzJN8bku1wu54Nrt+tvUF8O+fd0W+w/Vuygur6Zn90wWVcZU0r1O07WWqVAls/jTHtbuwRgIrBGRA5gDUld3qc6jAufh6QsGL2wyyIfFFfyxwI3X5s7kokZSYGLTSmleomTiWADkCsiOSISBdwKLG/faYypMcakGWOyjTHZwMfANcaYAgdj6rnq/bB/tdVJHBbut0hDcytL/1TEyLQ4vnVpboADVEqp3uFYIjDGtAL3AyuBncAfjTHbReQxEbnGqfftNYUvgITDtDu7LPLLlXtwH23k8RsmExPpP1kopVRf5+hyk8aYFcCKTtse7qLsfCdjOSOtzbD5ZRizCBL9TxGx8fOjPPfRZ9w16zxm5KQEOECllOo92rPpz+6/QX1Fl53Ex1vb+MGyIoYlxvD9Rae9rk4ppfo0XYDen4LnrE7iUf6nUvrt6n0Ul9fx3N0XkKCLzSil+jn9RtBZ1T74bC1M/7LfTuJdR47x2zV7uXbqcC4ZF5pr8yilBhZNBJ0VPt9lJ3Gb1/CDZUUkxETy8NUTAh+bUko5QJuGfLUetzqJx14JicNO2f3ch5+xxV3Db26bRkpcVBACVEqp3qffCHzt+is0VMH5p3YSf17VwC/f3s3CvHSunnxqklBKqf5KE4Gvgudg8Ai/ncTLNrppbvXy42sn6mIzSqkBRRNBu8q9cOB9u5P41MOypcTDmCEJDEvSxWaUUgOLJoJ2G5+HsAi/000bY9ji9jAlc3Dg41JKKYdpIgC7k/gVq5M4Ycgpu0uqG/E0tDA5SyeVU0oNPJoIAHb+pctOYoDNbmt5Zv1GoJQaiDQRgN1JfB6MvMTv7qISD9ERYYwdmhDgwJRSynmaCCqL4eAHcL7/TmKALW4PE4Yn6qIzSqkBSWu2wuetTuKp/qebbm3zsq30GJO1WUgpNUCFdiJoabKuJB73Bb+dxADF5XU0trQxNUsTgVJqYArtRLBzOTQe7bKTGKDI7iienKkjhpRSA1NoJ4LC5yE5G3LmdVlkc0kNiTERZKfGBSwspZQKpNBNBBW74eCHcP7dXXYSg/WNYHLmYMLCdFoJpdTAFLqJoPB5CIvsspMYoKmljV1HapmiF5IppQaw0EwELY3WlcR5X4R4V5fFth86RpvX6IghpdSAFpqJYMdyaPJYzULd2FJidRTriCGl1EAWmomg8DlIGQnZc7stVuT2MCQxmiGJMQEKTCmlAs/RRCAii0Rkt4jsFZGlfvZ/XUS2ishmEflARMY7GQ8A5Tvh8/Wn7SQG2OKu0fmFlFIDnmOJQETCgSeAK4HxwG1+KvpXjDGTjDFTgZ8Dv3Iqng6FL1idxFNu77ZYTUMLn1XWM0WbhZRSA5yT3whmAHuNMfuNMc3Aq8Bi3wLGmGM+D+MA42A8Vifxllcg7+puO4kBikp1xlGlVGhwcvH6DKDE57EbmNm5kIjcB3wHiAJOXSPSKrMEWAIwYsSIs49o+1vQVAP5XV9J3K7IXQPAJL2iWCk1wAW9s9gY84QxZhTwA+ChLso8ZYzJN8bku1zdn8l3q/A5SBkF2XNOW3RziYeRaXEkDYo8+/dTSql+wMlEUApk+TzOtLd15VXgWseiKdsBJZ9YncQ9WHzeuqJYvw0opQY+JxPBBiBXRHJEJAq4FVjuW0BEcn0efgEodiyaHW9BeBRMveO0RY/UNFF27Lh2FCulQoJjfQTGmFYRuR9YCYQDzxpjtovIY0CBMWY5cL+ILARagKPAl52Kh3lLYfy1EJd62qJbOmYc1USglBr4nOwsxhizAljRadvDPvcfcPL9TxIWBkN6dpnClhIPEWHChOGJDgellFLBF/TO4r6oyF3D2KEJxESGBzsUpZRynCaCTrxewxa3R/sHlFIhQxNBJweq6qltamWKjhhSSoUITQSdtHcU6zcCpVSo0ETQyZaSGgZFhjPaFR/sUJRSKiA0EXSyxe1hUkYSEeF6aJRSoUFrOx8tbV62HzqmVxQrpUKKJgIfu4/U0tzq1f4BpVRI0UTgo6OjWK8oVkqFEE0EPraUeEiOjSQrZVCwQ1FKqYDRROCjyF3D5MzBSA9mJ1VKqYFCE4GtobmVPWW12j+glAo5mghs20qP4TXoFcVKqZCjicC2pUSnnlZKhSZNBLYtbg8ZgwfhSogOdihKKRVQmghs1oyj2iyklAo9mgiA6vpmSqobtVlIKRWSNBGgF5IppUKbJgKgqKQGEZikI4aUUiFIEwHWN4LRrnjiox1dwlkppfqkkE8ExhiK3B7tH1BKhayQTwSlnkYq65qZqiOGlFIhytFEICKLRGS3iOwVkaV+9n9HRHaISJGIvCci5zkZjz9F7hpALyRTSoUuxxKBiIQDTwBXAuOB20RkfKdim4B8Y8xkYBnwc6fi6cqWEg9R4WGMG5YQ6LdWSqk+wclvBDOAvcaY/caYZuBVYLFvAWPMamNMg/3wYyDTwXj82uL2kDcsgeiI8EC/tVJK9QlOJoIMoMTnsdve1pWvAn/3t0NElohIgYgUVFRU9FqAbV7DVneNzjiqlAppfaKzWETuBPKBX/jbb4x5yhiTb4zJd7lcvfa++yvqqG9u0/4BpVRIc3LgfCmQ5fM40952EhFZCPwImGeMOe5gPKfYbM84qiOGlFKhzMlvBBuAXBHJEZEo4FZguW8BEZkG/A9wjTGm3MFY/Cpy1xAfHcHItPhAv7VSSvUZjiUCY0wrcD+wEtgJ/NEYs11EHhORa+xivwDigddFZLOILO/i5Ryxxe1hUkYSYWG6NKVSKnQ5OqeCMWYFsKLTtod97i908v27c7y1jZ2Hj/GV2TnBCkEppfqEPtFZHAw7D9fS0maYqh3FSqkQF7KJoMieenqyDh1VSoW4kE0Em0s8pMVHMzwpJtihKKVUUIVsIihy1zAlMwkR7ShWSoW2kEwEtU0t7Kuo0yuKlVKKEE0EW0trMAYm64pkSikVmolgS4k19bSuUayUUiGaCIrcHkakxJIcFxXsUJRSKuhCMhFsKfFos5BSStlCLhGU1zZxqKaJqdpRrJRSQAgmgqISXZpSKaV8hV4icHsIE5iYkRjsUJRSqk8IuUSw2V3DmCEJxEY5Ot+eUkr1GyGVCIwxFLk9OmxUKaV8hFQiKKluxNPQwmRdkUwppTqEVCLYbM84qt8IlFLqhJBKBEUlHqIjwhg7NCHYoSilVJ8RUolgi9vDhOGJRIaH1K+tlFLdCpkasbXNy7bSY3r9gFJKdRIyiaC4vI7Glja9olgppToJmUTQsTSlzjGklFInCZlEkBwbxWXjh5CdGhfsUJRSqk8JmctrL58wlMsnDA12GEop1ec4+o1ARBaJyG4R2SsiS/3snysiG0WkVURudDIWpZRS/jmWCEQkHHgCuBIYD9wmIuM7FfscuBt4xak4lFJKdc/JpqEZwF5jzH4AEXkVWAzsaC9gjDlg7/M6GIdSSqluONk0lAGU+Dx229vOmIgsEZECESmoqKjoleCUUkpZ+sWoIWPMU8aYfGNMvsvlCnY4Sik1oDiZCEqBLJ/HmfY2pZRSfYiTiWADkCsiOSISBdwKLHfw/ZRSSp0FxxKBMaYVuB9YCewE/miM2S4ij4nINQAicoGIuIGbgP8Rke1OxaOUUso/McYEO4YzIiIVwMGzfHoaUNmL4fQ2je/caHznrq/HqPGdvfOMMX47WftdIjgXIlJgjMkPdhxd0fjOjcZ37vp6jBqfM/rFqCGllFLO0USglFIhLtQSwVPBDuA0NL5zo/Gdu74eo8bngJDqI1BKKXWqUPtGoJRSqhNNBEopFeIGZCLowToI0SLymr3/ExHJDmBsWSKyWkR2iMh2EXnAT5n5IlIjIpvtn4cDFZ/9/gdEZKv93gV+9ouI/MY+fkUiMj2AsY31OS6bReSYiDzYqUzAj5+IPCsi5SKyzWdbioi8IyLF9m1yF8/9sl2mWES+HKDYfiEiu+y/35si4ncx79N9FhyO8VERKfX5O17VxXO7/X93ML7XfGI7ICKbu3huQI7hOTHGDKgfIBzYB4wEooAtwPhOZb4B/M6+fyvwWgDjGwZMt+8nAHv8xDcf+GsQj+EBIK2b/VcBfwcEmAV8EsS/9RGsC2WCevyAucB0YJvPtp8DS+37S4Gf+XleCrDfvk227ycHILbLgQj7/s/8xdaTz4LDMT4KfLcHn4Fu/9+diq/T/v8LPBzMY3guPwPxG0HHOgjGmGagfR0EX4uBF+z7y4BLRUQCEZwx5rAxZqN9vxZr+o2zmp47iBYDLxrLx8BgERkWhDguBfYZY872SvNeY4xZB1R32uz7OXsBuNbPU68A3jHGVBtjjgLvAIucjs0Y87axpoEB+BhrUsig6eL49URP/t/PWXfx2XXHzcAfevt9A2UgJoKerIPQUcb+Z6gBUgMSnQ+7SWoa8Imf3ReKyBYR+buITAhoYGCAt0WkUESW+Nnfa2tNnKNb6fqfL5jHr90QY8xh+/4RYIifMn3hWH4F6xueP6f7LDjtfrv56tkumtb6wvGbA5QZY4q72B/sY3haAzER9AsiEg+8ATxojDnWafdGrOaOKcB/A28FOLzZxpjpWMuM3icicwP8/qdlz2h7DfC6n93BPn6nMFYbQZ8bqy0iPwJagZe7KBLMz8KTwChgKnAYq/mlL7qN7r8N9Pn/p4GYCHqyDkJHGRGJAJKAqoBEZ71nJFYSeNkY86fO+40xx4wxdfb9FUCkiKQFKj5jTKl9Ww68ifX121dfWGviSmCjMaas845gHz8fZe1NZvZtuZ8yQTuWInI38EXgDjtRnaIHnwXHGGPKjDFtxhgv8Psu3juon0W7/rgeeK2rMsE8hj01EBNBT9ZBWA60j864EVjV1T9Cb7PbE58BdhpjftVFmaHtfRYiMgPr7xSQRCUicSKS0H4fq1NxW6diy4Ev2aOHZgE1Pk0ggdLlWVgwj18nvp+zLwN/9lNmJXC5iCTbTR+X29scJSKLgO8D1xhjGroo05PPgpMx+vY7XdfFewd73ZOFwC5jjNvfzmAfwx4Ldm+1Ez9Yo1r2YI0m+JG97TGsDz1ADFaTwl7gU2BkAGObjdVEUARstn+uAr4OfN0ucz+wHWsExMfARQGMb6T9vlvsGNqPn298AjxhH9+tQH6A/75xWBV7ks+2oB4/rKR0GGjBaqf+Kla/03tAMfAukGKXzQee9nnuV+zP4l7gngDFtherbb39M9g+im44sKK7z0IAj99L9uerCKtyH9Y5RvvxKf/vgYjP3v58++fOp2xQjuG5/OgUE0opFeIGYtOQUkqpM6CJQCmlQpwmAqWUCnGaCJRSKsRpIlBKqRCniUCpALJnRv1rsONQypcmAqWUCnGaCJTyQ0TuFJFP7Tnk/0dEwkWkTkT+U6x1JN4TEZdddqqIfOwzt3+yvX20iLxrT363UURG2S8fLyLL7PUAXg7UzLdKdUUTgVKdiEgecAtwsTFmKtAG3IF1RXOBMWYCsBZ4xH7Ki8APjDGTsa6Ebd/+MvCEsSa/uwjrylSwZpx9EBiPdeXpxY7/Ukp1IyLYASjVB10KnA9ssE/WB2FNGOflxORi/wv8SUSSgMHGmLX29heA1+35ZTKMMW8CGGOaAOzX+9TYc9PYq1plAx84/2sp5Z8mAqVOJcALxph/PWmjyL91Kne287Mc97nfhv4fqiDTpiGlTvUecKOIpEPH2sPnYf2/3GiXuR34wBhTAxwVkTn29ruAtcZafc4tItfarxEtIrEB/S2U6iE9E1GqE2PMDhF5CGtVqTCsGSfvA+qBGfa+cqx+BLCmmP6dXdHvB+6xt98F/I+IPGa/xk0B/DWU6jGdfVSpHhKROmNMfLDjUKq3adOQUkqFOP1GoJRSIU6/ESilVIjTRKCUUiFOE4FSSoU4TQRKKRXiNBEopVSI+/8dMLVT/QhBmwAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7zN6cl-BZiqZ"
      },
      "source": [
        "## 추론"
      ],
      "id": "7zN6cl-BZiqZ"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sBZl2W5vZiqZ"
      },
      "source": [
        "def evaluate(tokens):\n",
        "    transformer.to(device)\n",
        "    decoder_input = torch.tensor([tar_tokenizer.txt2idx['sos_']] * tokens.size(0), dtype=torch.long).to(device)\n",
        "    output = decoder_input.unsqueeze(1).to(device)\n",
        "    enc_output = None\n",
        "    for i in range(decoder_len-1):        \n",
        "        # predictions.shape == (batch_size, seq_len, vocab_size)\n",
        "        with torch.no_grad():\n",
        "            predictions, attention_weights, enc_output = transformer([tokens, output, enc_output])\n",
        "        \n",
        "        # select the last token from the seq_len dimension\n",
        "        predictions_ = predictions[: ,-1:, :]  # (batch_size, 1, vocab_size)\n",
        "        \n",
        "        predicted_id = torch.tensor(torch.argmax(predictions_, axis=-1), dtype=torch.int32)\n",
        "        \n",
        "        output = torch.cat([output, predicted_id], dim=-1)\n",
        "    output = output.cpu().numpy()\n",
        "    \n",
        "    summary_list = []\n",
        "    token_list = []\n",
        "    for token in output:\n",
        "        summary = tar_tokenizer.convert(token)\n",
        "        summary_list.append(summary)\n",
        "        token_list.append(token)\n",
        "    return summary_list, token_list"
      ],
      "id": "sBZl2W5vZiqZ",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sslbMWVuZiqa",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ea806d89-9051-4e94-e860-cfed36367001"
      },
      "source": [
        "tqdm_dataset = tqdm(enumerate(val_dataloader))\n",
        "preds = []\n",
        "tokens = []\n",
        "for batch, batch_item in tqdm_dataset:\n",
        "    output = evaluate(batch_item['src_token'].to(device))\n",
        "    preds.extend(output[0])\n",
        "    tokens.extend(output[1])"
      ],
      "id": "sslbMWVuZiqa",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "29it [02:57,  6.11s/it]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "tags": [],
        "id": "G3NfSKLcZiqa",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "42ca2ffa-ba67-478e-b991-333d066beeb9"
      },
      "source": [
        "for i, (a, p) in enumerate(zip(df_val.summary, preds)):\n",
        "    print('정답 :', a)\n",
        "    print('예측 :', p)\n",
        "    print('=================================================================================')\n",
        "    if i == 10:\n",
        "        break"
      ],
      "id": "G3NfSKLcZiqa",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "정답 : 소상공인 경영환경 개선사업의 경우 지속적으로 사업을 더 늘려서 많은 업소가 혜택을 볼 수 있도록 할 것. 신재생에너지 보급확대 사업에도 소상공인 혜택이 빠지지 않도록 할 것.\n",
            "예측 :  수도 사업소 소관 2018 년도 행정 사무 감사 시정 , 건의사항 조치 결과 보고.\n",
            "=================================================================================\n",
            "정답 : 제155회 완주군의회 임시회 회기는 10월 27일부터 11월 3일까지 8일간으로 가결됨.\n",
            "예측 :  제 완주군 의회 회 완주군 의회 임시회 회기는 11 월 10 일부터 11 월 11 일까지 3 일 간으로 가결됨.\n",
            "=================================================================================\n",
            "정답 : 2001년도 제1회 추가경정 세입세출예산안 제안 설명의 건.\n",
            "예측 :  2005 년도 세입 세출 예산안 제안 설명 . 해당 안건은 가결됨.\n",
            "=================================================================================\n",
            "정답 : 2013년 중기지방재정계획 보고.\n",
            "예측 :  2012 년도 행정 사무 감사 시정 및 건의사항 조치 결과 보고.\n",
            "=================================================================================\n",
            "정답 : 제124회 완주군의회 임시회 제1차 본회의 개의 선포.\n",
            "예측 :  완주군 의회 행정 사무 감사 특별 위원회 위원으로 송지용 의원 , 박웅배 의원 , 박종관 의원 , 홍의환 의원 , 홍의환 의원 , 임원규 의원 , 임원규 의원 , 임원규 의원 , 임원규 의원 , 임원규 의원 , 임원규 의원이 선임됨.\n",
            "=================================================================================\n",
            "정답 : 공유재산 관리계획 변경계획안 제안설명.\n",
            "예측 :  2016 년도 공유 재산 관리 계획안은 음성군 공유 재산 관리 계획을 받고자 음성군 공유 재산 심의회의의결을 받고자 제안됨. 해당 안건은 가결됨.\n",
            "=================================================================================\n",
            "정답 : 제134회 음성군의회 임시회 회기는 10월 7일부터 10월 9일까지 3일간으로 가결됨.\n",
            "예측 :  음성군 의회 회의 규칙 일부 개정 규칙 안은 음성군 의회 회의 회기는 10 월 23 일부터 10 월 26 일까지 7 일 간으로 가결됨.\n",
            "=================================================================================\n",
            "정답 : 음성군 지방공무원 정원 조례 중 개정조례안은 보건복지부의 기초생활보호대상 배치기준에 의거 2명의 사회복지직 정원이 승인되었기에 증원하여 복지행정 서비스 수준의 형평을 유지하고자 발의함. 해당 안건은 가결됨. 음성군 리장 정원 조례 중 개정조례안과 반 설치 조례 중 개정조례안은 인구 이동, 공장 입주, 아파트단지 건설 등 제반 여건과 생활환경이 변화함에 따라 행정구역을 적정 규모로 조정함으로써 주민 생활 편의 도모 및 효율적인 행정을 추진하고자 발의함. 해당 안건은 각각 가결됨.\n",
            "예측 :  음성군 지방 공무원 정원 조례 일부 개정 조례 안은 행정 기구 설치 조례에 따른 기구 설치 조례의일부 개정으로 인한 조례의일부 개정을 명확히 하고 , 반을 명확히 하고자 제안됨. 해당 안건은 가결됨. 음성군 지\n",
            "=================================================================================\n",
            "정답 : 주요사업 현지확인 특별위원회 운영 계획안이 가결됨.\n",
            "예측 :  주요 사업 현지 확인 특별 위원회 운영 계획안은 9 월 23 일부터 9 월 24 일까지 2 일 간 운영함.\n",
            "=================================================================================\n",
            "정답 : 제110회 임시회 휴회는 선진 시군 비교 견학 의사일정으로 갈음함.\n",
            "예측 :  제 120 회 임시회 휴회의건은 회기 중 선진 시군 비교 견학 활동을 하는 의사일정으로 갈음함.\n",
            "=================================================================================\n",
            "정답 : 제273회 제2차 정례회 회의록 서명의원으로 한동완 의원과 우성수 의원이 선출됨.\n",
            "예측 :  회의록 서명 의원으로 한동완 의원 , 이대웅 의원이 선출됨.\n",
            "=================================================================================\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ljh5sTo9Ziqa"
      },
      "source": [
        "## 제출"
      ],
      "id": "Ljh5sTo9Ziqa"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Qr6EPtJwZiqa",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "85e0cafd-71fd-44c7-f131-91fd21037bbd"
      },
      "source": [
        "tqdm_dataset = tqdm(enumerate(test_dataloader))\n",
        "preds = []\n",
        "tokens = []\n",
        "for batch, batch_item in tqdm_dataset:\n",
        "    output = evaluate(batch_item['src_token'].to(device))\n",
        "    preds.extend(output[0])\n",
        "    tokens.extend(output[1])"
      ],
      "id": "Qr6EPtJwZiqa",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "16it [01:39,  6.22s/it]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sKAwjvVEZiqa"
      },
      "source": [
        "submission = pd.read_csv('sample_submission.csv')"
      ],
      "id": "sKAwjvVEZiqa",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CxSzjwVZZiqa"
      },
      "source": [
        "submission['summary'] = preds"
      ],
      "id": "CxSzjwVZZiqa",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tnCd7yTPZiqb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 202
        },
        "outputId": "f5b2dd4a-568e-49cf-e197-3aea6aed886e"
      },
      "source": [
        "submission.head()"
      ],
      "id": "tnCd7yTPZiqb",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>uid</th>\n",
              "      <th>summary</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>id_2000-AGENDA_1</td>\n",
              "      <td>음성군 의회 제 1 차 정례회 제 1 차 본회의개의선포 . 운영 위원회 위원으로 ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>id_2000-AGENDA_2</td>\n",
              "      <td>음성군 의회 정례회의운영에 관한 조례 안은 음성군의회기는 6 월 23 일부터 6 ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>id_2000-AGENDA_3</td>\n",
              "      <td>회의록 서명 의원으로 한동완 의원 , 이대웅 의원이 선출됨.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>id_2000-AGENDA_4</td>\n",
              "      <td>예산 결산 특별 위원회 위원으로 이한철 의원 , 반광홍 의원 , 정태완 의원 , ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>id_2000-AGENDA_5</td>\n",
              "      <td>음성군 의회 운영 조례 안은 음성군 의회 운영의효율 적인 활동을 위해 주요 시설을...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                uid                                            summary\n",
              "0  id_2000-AGENDA_1   음성군 의회 제 1 차 정례회 제 1 차 본회의개의선포 . 운영 위원회 위원으로 ...\n",
              "1  id_2000-AGENDA_2   음성군 의회 정례회의운영에 관한 조례 안은 음성군의회기는 6 월 23 일부터 6 ...\n",
              "2  id_2000-AGENDA_3                  회의록 서명 의원으로 한동완 의원 , 이대웅 의원이 선출됨.\n",
              "3  id_2000-AGENDA_4   예산 결산 특별 위원회 위원으로 이한철 의원 , 반광홍 의원 , 정태완 의원 , ...\n",
              "4  id_2000-AGENDA_5   음성군 의회 운영 조례 안은 음성군 의회 운영의효율 적인 활동을 위해 주요 시설을..."
            ]
          },
          "metadata": {},
          "execution_count": 73
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "esS-YiTfZiqb"
      },
      "source": [
        "submission.to_csv('dacon_baseline.csv', index=False)"
      ],
      "id": "esS-YiTfZiqb",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qyW9iDzeeZBN",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "42e0981c-4aa8-4af3-a228-485c38f52852"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "id": "qyW9iDzeeZBN",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qyoeN6xFaQhL"
      },
      "source": [
        "submission.to_csv(\"/content/drive/My Drive/dacon_baseline_submission_1014.csv\")"
      ],
      "id": "qyoeN6xFaQhL",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-zrLrmGXZiqb"
      },
      "source": [
        "제출 API 사용법 => https://dacon.io/forum/403557"
      ],
      "id": "-zrLrmGXZiqb"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8B0M5qSxZiqb"
      },
      "source": [
        "# from dacon_submit_api import dacon_submit_api \n",
        "\n",
        "# result = dacon_submit_api.post_submission_file(\n",
        "#     'dacon_baseline.csv', \n",
        "#     '개인 Token', \n",
        "#     '235813',\n",
        "#     'BASELINE', \n",
        "#     'DACON_Baseline'\n",
        "# )"
      ],
      "id": "8B0M5qSxZiqb",
      "execution_count": null,
      "outputs": []
    }
  ]
}