BERT_Classifier.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2021-01-27T23:07:01.589882Z",
     "iopub.status.busy": "2021-01-27T23:07:01.589201Z",
     "iopub.status.idle": "2021-01-27T23:07:01.612220Z",
     "shell.execute_reply": "2021-01-27T23:07:01.611614Z"
    },
    "papermill": {
     "duration": 0.044266,
     "end_time": "2021-01-27T23:07:01.612336",
     "exception": false,
     "start_time": "2021-01-27T23:07:01.568070",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/kaggle/input/stanford-natural-language-inference-corpus/README.txt\n",
      "/kaggle/input/stanford-natural-language-inference-corpus/snli_1.0_test.csv\n",
      "/kaggle/input/stanford-natural-language-inference-corpus/snli_1.0_train.csv\n",
      "/kaggle/input/stanford-natural-language-inference-corpus/snli_1.0_dev.csv\n"
     ]
    }
   ],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "import os\n",
    "for dirname, _, filenames in os.walk('/kaggle/input'):\n",
    "    for filename in filenames:\n",
    "        print(os.path.join(dirname, filename))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a",
    "execution": {
     "iopub.execute_input": "2021-01-27T23:07:01.651664Z",
     "iopub.status.busy": "2021-01-27T23:07:01.651067Z",
     "iopub.status.idle": "2021-01-27T23:07:09.376328Z",
     "shell.execute_reply": "2021-01-27T23:07:09.375443Z"
    },
    "papermill": {
     "duration": 7.749912,
     "end_time": "2021-01-27T23:07:09.376448",
     "exception": false,
     "start_time": "2021-01-27T23:07:01.626536",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "snli_train = pd.read_csv('/kaggle/input/stanford-natural-language-inference-corpus/snli_1.0_train.csv',nrows=300000)\n",
    "snli_test = pd.read_csv('/kaggle/input/stanford-natural-language-inference-corpus/snli_1.0_test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-01-27T23:07:09.417996Z",
     "iopub.status.busy": "2021-01-27T23:07:09.417118Z",
     "iopub.status.idle": "2021-01-27T23:07:09.422381Z",
     "shell.execute_reply": "2021-01-27T23:07:09.421869Z"
    },
    "papermill": {
     "duration": 0.032241,
     "end_time": "2021-01-27T23:07:09.422484",
     "exception": false,
     "start_time": "2021-01-27T23:07:09.390243",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "A person on a horse jumps over a broken down airplane. A person is training his horse for a competition. neutral\n",
      "A person on a horse jumps over a broken down airplane. A person is at a diner, ordering an omelette. contradiction\n",
      "A person on a horse jumps over a broken down airplane. A person is outdoors, on a horse. entailment\n"
     ]
    }
   ],
   "source": [
    "print(snli_train['sentence1'][0],snli_train['sentence2'][0], snli_train['gold_label'][0])\n",
    "print(snli_train['sentence1'][1],snli_train['sentence2'][1], snli_train['gold_label'][1])\n",
    "print(snli_train['sentence1'][2],snli_train['sentence2'][2], snli_train['gold_label'][2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-01-27T23:07:09.458929Z",
     "iopub.status.busy": "2021-01-27T23:07:09.458192Z",
     "iopub.status.idle": "2021-01-27T23:07:09.623684Z",
     "shell.execute_reply": "2021-01-27T23:07:09.624213Z"
    },
    "papermill": {
     "duration": 0.187934,
     "end_time": "2021-01-27T23:07:09.624346",
     "exception": false,
     "start_time": "2021-01-27T23:07:09.436412",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "entailment       100064\n",
       "contradiction     99903\n",
       "neutral           99652\n",
       "Name: gold_label, dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Filter and rename the important columns\n",
    "train_df = snli_train.filter(['sentence1','sentence2','gold_label'], axis=1)\n",
    "test_df = snli_test.filter(['sentence1','sentence2','gold_label'], axis=1)\n",
    "train_df = train_df.rename(columns={\"sentence1\": \"premise\", \"sentence2\": \"hypothesis\"})\n",
    "test_df = test_df.rename(columns={\"sentence1\": \"premise\", \"sentence2\": \"hypothesis\"})\n",
    "\n",
    "# exclude sentences which could not be categorised\n",
    "train_df = train_df[train_df['gold_label'] != \"-\"]\n",
    "test_df = test_df[test_df['gold_label'] != \"-\"]\n",
    "\n",
    "train_df['gold_label'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-01-27T23:07:09.659947Z",
     "iopub.status.busy": "2021-01-27T23:07:09.659267Z",
     "iopub.status.idle": "2021-01-27T23:07:20.851455Z",
     "shell.execute_reply": "2021-01-27T23:07:20.850320Z"
    },
    "papermill": {
     "duration": 11.212089,
     "end_time": "2021-01-27T23:07:20.851590",
     "exception": false,
     "start_time": "2021-01-27T23:07:09.639501",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import transformers\n",
    "\n",
    "max_length = 128  # Maximum length of input sentence to the model.\n",
    "batch_size = 256\n",
    "epochs = 40"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-01-27T23:07:20.891676Z",
     "iopub.status.busy": "2021-01-27T23:07:20.890856Z",
     "iopub.status.idle": "2021-01-27T23:07:20.894706Z",
     "shell.execute_reply": "2021-01-27T23:07:20.894181Z"
    },
    "papermill": {
     "duration": 0.026665,
     "end_time": "2021-01-27T23:07:20.894818",
     "exception": false,
     "start_time": "2021-01-27T23:07:20.868153",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "torch.cuda.is_available()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-01-27T23:07:20.956213Z",
     "iopub.status.busy": "2021-01-27T23:07:20.941976Z",
     "iopub.status.idle": "2021-01-27T23:07:21.175296Z",
     "shell.execute_reply": "2021-01-27T23:07:21.174667Z"
    },
    "papermill": {
     "duration": 0.264988,
     "end_time": "2021-01-27T23:07:21.175436",
     "exception": false,
     "start_time": "2021-01-27T23:07:20.910448",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "train_df[\"label\"] = train_df[\"gold_label\"].apply(\n",
    "    lambda x: 0 if x == \"contradiction\" else 1 if x == \"entailment\" else 2\n",
    ")\n",
    "y_train = tf.keras.utils.to_categorical(train_df.label, num_classes=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-01-27T23:07:21.214595Z",
     "iopub.status.busy": "2021-01-27T23:07:21.213758Z",
     "iopub.status.idle": "2021-01-27T23:07:21.755982Z",
     "shell.execute_reply": "2021-01-27T23:07:21.756630Z"
    },
    "papermill": {
     "duration": 0.563114,
     "end_time": "2021-01-27T23:07:21.756812",
     "exception": false,
     "start_time": "2021-01-27T23:07:21.193698",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8697dd928e5a46c8a1b183cec2135099",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=231508.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "tokenizer = transformers.DistilBertTokenizer.from_pretrained(\"distilbert-base-uncased\", do_lower_case=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-01-27T23:07:23.034497Z",
     "iopub.status.busy": "2021-01-27T23:07:22.352094Z",
     "iopub.status.idle": "2021-01-27T23:07:23.039226Z",
     "shell.execute_reply": "2021-01-27T23:07:23.038224Z"
    },
    "papermill": {
     "duration": 1.26467,
     "end_time": "2021-01-27T23:07:23.039359",
     "exception": false,
     "start_time": "2021-01-27T23:07:21.774689",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "sentence_pairs = train_df[['premise', 'hypothesis']].values.astype('str')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-01-27T23:07:23.855962Z",
     "iopub.status.busy": "2021-01-27T23:07:23.855174Z",
     "iopub.status.idle": "2021-01-27T23:12:07.632748Z",
     "shell.execute_reply": "2021-01-27T23:12:07.631766Z"
    },
    "papermill": {
     "duration": 284.576455,
     "end_time": "2021-01-27T23:12:07.632899",
     "exception": false,
     "start_time": "2021-01-27T23:07:23.056444",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
      "/opt/conda/lib/python3.7/site-packages/transformers/tokenization_utils_base.py:2022: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
      "  FutureWarning,\n"
     ]
    }
   ],
   "source": [
    "encoded = tokenizer.batch_encode_plus(\n",
    "    sentence_pairs.tolist(),\n",
    "    add_special_tokens=True,\n",
    "    max_length=max_length,\n",
    "    return_attention_mask=True,\n",
    "    return_token_type_ids=True,\n",
    "    pad_to_max_length=True,\n",
    "    return_tensors=\"tf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-01-27T23:12:07.674045Z",
     "iopub.status.busy": "2021-01-27T23:12:07.672723Z",
     "iopub.status.idle": "2021-01-27T23:12:07.842937Z",
     "shell.execute_reply": "2021-01-27T23:12:07.842400Z"
    },
    "papermill": {
     "duration": 0.192736,
     "end_time": "2021-01-27T23:12:07.843056",
     "exception": false,
     "start_time": "2021-01-27T23:12:07.650320",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "input_ids = np.array(encoded[\"input_ids\"], dtype=\"int32\")\n",
    "attention_masks = np.array(encoded[\"attention_mask\"], dtype=\"int32\")\n",
    "token_type_ids = np.array(encoded[\"token_type_ids\"], dtype=\"int32\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-01-27T23:12:07.890227Z",
     "iopub.status.busy": "2021-01-27T23:12:07.889505Z",
     "iopub.status.idle": "2021-01-27T23:12:23.654847Z",
     "shell.execute_reply": "2021-01-27T23:12:23.655744Z"
    },
    "papermill": {
     "duration": 15.795278,
     "end_time": "2021-01-27T23:12:23.655935",
     "exception": false,
     "start_time": "2021-01-27T23:12:07.860657",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c0130d035f494419a5f3f60c44b2c870",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=442.0, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "851a6809ab8241b48bb5e18f86fb32f4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=363423424.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some layers from the model checkpoint at distilbert-base-uncased were not used when initializing TFDistilBertModel: ['vocab_transform', 'vocab_layer_norm', 'activation_13', 'vocab_projector']\n",
      "- This IS expected if you are initializing TFDistilBertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing TFDistilBertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "All the layers of TFDistilBertModel were initialized from the model checkpoint at distilbert-base-uncased.\n",
      "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFDistilBertModel for predictions without further training.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"functional_1\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_ids (InputLayer)          [(None, 128)]        0                                            \n",
      "__________________________________________________________________________________________________\n",
      "attention_masks (InputLayer)    [(None, 128)]        0                                            \n",
      "__________________________________________________________________________________________________\n",
      "tf_distil_bert_model (TFDistilB ((None, 128, 768),)  66362880    input_ids[0][0]                  \n",
      "                                                                 attention_masks[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dense (Dense)                   (None, 128, 100)     76900       tf_distil_bert_model[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 128, 100)     10100       dense[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 128, 100)     10100       dense_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "flatten (Flatten)               (None, 12800)        0           dense_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_3 (Dense)                 (None, 3)            38403       flatten[0][0]                    \n",
      "==================================================================================================\n",
      "Total params: 66,498,383\n",
      "Trainable params: 135,503\n",
      "Non-trainable params: 66,362,880\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "import keras\n",
    "from keras import datasets, layers, models\n",
    "\n",
    "def create_bert_model():\n",
    "    # Encoded token ids from BERT tokenizer.\n",
    "    input_ids = layers.Input(shape=(max_length,), dtype=tf.int32, name=\"input_ids\")\n",
    "    # Attention masks indicates to the model which tokens should be attended to.\n",
    "    attention_masks = layers.Input(shape=(max_length,), dtype=tf.int32, name=\"attention_masks\")\n",
    "    # Token type ids are binary masks identifying different sequences in the model.\n",
    "#     token_type_ids = layers.Input(shape=(max_length,), dtype=tf.int32, name=\"token_type_ids\")\n",
    "    \n",
    "    bert_model = transformers.TFDistilBertModel.from_pretrained('distilbert-base-uncased', return_dict=False)\n",
    "    bert_model.trainable = False\n",
    "\n",
    "    sequence_output = bert_model(input_ids, attention_mask=attention_masks)\n",
    "    \n",
    "    y_1= layers.Dense(100,activation='tanh')(sequence_output[0])\n",
    "    y_2= layers.Dense(100,activation='tanh')(y_1)\n",
    "    y_3= layers.Dense(100,activation='tanh')(y_2)\n",
    "\n",
    "    output=layers.Flatten()(y_3)\n",
    "    output = layers.Dense(3, activation='softmax')(output)\n",
    "\n",
    "    model = keras.models.Model([input_ids, attention_masks], output)\n",
    "    model.summary()\n",
    "    return model\n",
    "model = create_bert_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-01-27T23:12:23.715589Z",
     "iopub.status.busy": "2021-01-27T23:12:23.712239Z",
     "iopub.status.idle": "2021-01-28T06:33:25.112359Z",
     "shell.execute_reply": "2021-01-28T06:33:25.113080Z"
    },
    "papermill": {
     "duration": 26461.435783,
     "end_time": "2021-01-28T06:33:25.113263",
     "exception": false,
     "start_time": "2021-01-27T23:12:23.677480",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/40\n",
      "937/937 [==============================] - 660s 705ms/step - loss: 1.1059 - accuracy: 0.3587 - val_loss: 1.0887 - val_accuracy: 0.3912\n",
      "Epoch 2/40\n",
      "937/937 [==============================] - 660s 704ms/step - loss: 1.0808 - accuracy: 0.4036 - val_loss: 1.0667 - val_accuracy: 0.4309\n",
      "Epoch 3/40\n",
      "937/937 [==============================] - 658s 702ms/step - loss: 1.0621 - accuracy: 0.4338 - val_loss: 1.0493 - val_accuracy: 0.4538\n",
      "Epoch 4/40\n",
      "937/937 [==============================] - 658s 703ms/step - loss: 1.0461 - accuracy: 0.4560 - val_loss: 1.0352 - val_accuracy: 0.4699\n",
      "Epoch 5/40\n",
      "937/937 [==============================] - 658s 702ms/step - loss: 1.0345 - accuracy: 0.4688 - val_loss: 1.0241 - val_accuracy: 0.4818\n",
      "Epoch 6/40\n",
      "937/937 [==============================] - 659s 703ms/step - loss: 1.0249 - accuracy: 0.4797 - val_loss: 1.0146 - val_accuracy: 0.4922\n",
      "Epoch 7/40\n",
      "937/937 [==============================] - 663s 707ms/step - loss: 1.0164 - accuracy: 0.4899 - val_loss: 1.0060 - val_accuracy: 0.5015\n",
      "Epoch 8/40\n",
      "937/937 [==============================] - 658s 703ms/step - loss: 1.0092 - accuracy: 0.4967 - val_loss: 0.9986 - val_accuracy: 0.5086\n",
      "Epoch 9/40\n",
      "937/937 [==============================] - 658s 703ms/step - loss: 1.0022 - accuracy: 0.5035 - val_loss: 0.9914 - val_accuracy: 0.5155\n",
      "Epoch 10/40\n",
      "937/937 [==============================] - 659s 703ms/step - loss: 0.9962 - accuracy: 0.5102 - val_loss: 0.9849 - val_accuracy: 0.5221\n",
      "Epoch 11/40\n",
      "937/937 [==============================] - 659s 703ms/step - loss: 0.9902 - accuracy: 0.5156 - val_loss: 0.9787 - val_accuracy: 0.5285\n",
      "Epoch 12/40\n",
      "937/937 [==============================] - 659s 703ms/step - loss: 0.9842 - accuracy: 0.5218 - val_loss: 0.9729 - val_accuracy: 0.5333\n",
      "Epoch 13/40\n",
      "937/937 [==============================] - 659s 703ms/step - loss: 0.9788 - accuracy: 0.5263 - val_loss: 0.9672 - val_accuracy: 0.5390\n",
      "Epoch 14/40\n",
      "937/937 [==============================] - 659s 703ms/step - loss: 0.9739 - accuracy: 0.5298 - val_loss: 0.9619 - val_accuracy: 0.5436\n",
      "Epoch 15/40\n",
      "937/937 [==============================] - 659s 703ms/step - loss: 0.9684 - accuracy: 0.5350 - val_loss: 0.9567 - val_accuracy: 0.5479\n",
      "Epoch 16/40\n",
      "937/937 [==============================] - 659s 703ms/step - loss: 0.9642 - accuracy: 0.5386 - val_loss: 0.9518 - val_accuracy: 0.5521\n",
      "Epoch 17/40\n",
      "937/937 [==============================] - 658s 703ms/step - loss: 0.9596 - accuracy: 0.5425 - val_loss: 0.9471 - val_accuracy: 0.5551\n",
      "Epoch 18/40\n",
      "937/937 [==============================] - 659s 703ms/step - loss: 0.9552 - accuracy: 0.5465 - val_loss: 0.9426 - val_accuracy: 0.5597\n",
      "Epoch 19/40\n",
      "937/937 [==============================] - 669s 714ms/step - loss: 0.9516 - accuracy: 0.5496 - val_loss: 0.9391 - val_accuracy: 0.5614\n",
      "Epoch 20/40\n",
      "937/937 [==============================] - 666s 710ms/step - loss: 0.9471 - accuracy: 0.5521 - val_loss: 0.9343 - val_accuracy: 0.5649\n",
      "Epoch 21/40\n",
      "937/937 [==============================] - 659s 703ms/step - loss: 0.9437 - accuracy: 0.5556 - val_loss: 0.9303 - val_accuracy: 0.5683\n",
      "Epoch 22/40\n",
      "937/937 [==============================] - 659s 704ms/step - loss: 0.9400 - accuracy: 0.5589 - val_loss: 0.9263 - val_accuracy: 0.5711\n",
      "Epoch 23/40\n",
      "937/937 [==============================] - 662s 707ms/step - loss: 0.9359 - accuracy: 0.5618 - val_loss: 0.9226 - val_accuracy: 0.5732\n",
      "Epoch 24/40\n",
      "937/937 [==============================] - 660s 705ms/step - loss: 0.9326 - accuracy: 0.5639 - val_loss: 0.9190 - val_accuracy: 0.5760\n",
      "Epoch 25/40\n",
      "937/937 [==============================] - 663s 708ms/step - loss: 0.9294 - accuracy: 0.5660 - val_loss: 0.9155 - val_accuracy: 0.5785\n",
      "Epoch 26/40\n",
      "937/937 [==============================] - 665s 710ms/step - loss: 0.9263 - accuracy: 0.5692 - val_loss: 0.9123 - val_accuracy: 0.5804\n",
      "Epoch 27/40\n",
      "937/937 [==============================] - 668s 713ms/step - loss: 0.9233 - accuracy: 0.5715 - val_loss: 0.9090 - val_accuracy: 0.5821\n",
      "Epoch 28/40\n",
      "937/937 [==============================] - 662s 707ms/step - loss: 0.9199 - accuracy: 0.5729 - val_loss: 0.9058 - val_accuracy: 0.5852\n",
      "Epoch 29/40\n",
      "937/937 [==============================] - 658s 702ms/step - loss: 0.9170 - accuracy: 0.5762 - val_loss: 0.9030 - val_accuracy: 0.5870\n",
      "Epoch 30/40\n",
      "937/937 [==============================] - 658s 703ms/step - loss: 0.9147 - accuracy: 0.5769 - val_loss: 0.9001 - val_accuracy: 0.5894\n",
      "Epoch 31/40\n",
      "937/937 [==============================] - 659s 703ms/step - loss: 0.9127 - accuracy: 0.5790 - val_loss: 0.8971 - val_accuracy: 0.5915\n",
      "Epoch 32/40\n",
      "937/937 [==============================] - 665s 709ms/step - loss: 0.9094 - accuracy: 0.5816 - val_loss: 0.8944 - val_accuracy: 0.5932\n",
      "Epoch 33/40\n",
      "937/937 [==============================] - 659s 704ms/step - loss: 0.9064 - accuracy: 0.5834 - val_loss: 0.8917 - val_accuracy: 0.5948\n",
      "Epoch 34/40\n",
      "937/937 [==============================] - 665s 709ms/step - loss: 0.9038 - accuracy: 0.5856 - val_loss: 0.8893 - val_accuracy: 0.5973\n",
      "Epoch 35/40\n",
      "937/937 [==============================] - 663s 707ms/step - loss: 0.9018 - accuracy: 0.5863 - val_loss: 0.8867 - val_accuracy: 0.5985\n",
      "Epoch 36/40\n",
      "937/937 [==============================] - 662s 707ms/step - loss: 0.8992 - accuracy: 0.5879 - val_loss: 0.8843 - val_accuracy: 0.6000\n",
      "Epoch 37/40\n",
      "937/937 [==============================] - 659s 703ms/step - loss: 0.8969 - accuracy: 0.5899 - val_loss: 0.8819 - val_accuracy: 0.6018\n",
      "Epoch 38/40\n",
      "937/937 [==============================] - 659s 703ms/step - loss: 0.8950 - accuracy: 0.5913 - val_loss: 0.8795 - val_accuracy: 0.6036\n",
      "Epoch 39/40\n",
      "937/937 [==============================] - 659s 703ms/step - loss: 0.8929 - accuracy: 0.5935 - val_loss: 0.8774 - val_accuracy: 0.6044\n",
      "Epoch 40/40\n",
      "937/937 [==============================] - 659s 703ms/step - loss: 0.8918 - accuracy: 0.5936 - val_loss: 0.8752 - val_accuracy: 0.6064\n"
     ]
    }
   ],
   "source": [
    "model.compile(optimizer='Adadelta',loss='categorical_crossentropy',metrics = ['accuracy'])\n",
    "history = model.fit([input_ids, attention_masks],\n",
    "                    y_train,\n",
    "                    epochs=epochs,\n",
    "                    batch_size=batch_size,\n",
    "                    validation_split=0.2,\n",
    "                    verbose=1,\n",
    "                    use_multiprocessing=True,\n",
    "                    workers=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-01-28T06:33:51.012544Z",
     "iopub.status.busy": "2021-01-28T06:33:51.011606Z",
     "iopub.status.idle": "2021-01-28T06:33:51.412720Z",
     "shell.execute_reply": "2021-01-28T06:33:51.411286Z"
    },
    "papermill": {
     "duration": 13.57721,
     "end_time": "2021-01-28T06:33:51.412860",
     "exception": false,
     "start_time": "2021-01-28T06:33:37.835650",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#!pip install -q pyyaml h5py\n",
    "import h5py\n",
    "model.save_weights('bert_weights_300000_40epochs.hdf5')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-01-28T06:34:17.970184Z",
     "iopub.status.busy": "2021-01-28T06:34:17.968682Z",
     "iopub.status.idle": "2021-01-28T06:34:49.105529Z",
     "shell.execute_reply": "2021-01-28T06:34:49.104473Z"
    },
    "papermill": {
     "duration": 44.634086,
     "end_time": "2021-01-28T06:34:49.105668",
     "exception": false,
     "start_time": "2021-01-28T06:34:04.471582",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "sentence_pairs_test = test_df[['premise', 'hypothesis']].values.astype('str')\n",
    "\n",
    "\n",
    "encoded_test = tokenizer.batch_encode_plus(\n",
    "    sentence_pairs_test.tolist(),\n",
    "    add_special_tokens=True,\n",
    "    max_length=max_length,\n",
    "    return_attention_mask=True,\n",
    "    return_token_type_ids=True,\n",
    "    pad_to_max_length=True,\n",
    "    return_tensors=\"tf\")\n",
    "\n",
    "input_ids = np.array(encoded_test[\"input_ids\"], dtype=\"int32\")\n",
    "attention_masks = np.array(encoded_test[\"attention_mask\"], dtype=\"int32\")\n",
    "token_type_ids = np.array(encoded_test[\"token_type_ids\"], dtype=\"int32\")\n",
    "\n",
    "test_pred = model.predict([input_ids, attention_masks], batch_size=batch_size)\n",
    "\n",
    "test_results_bert = pd.DataFrame(test_pred)\n",
    "test_results_bert.to_csv(\"Bert_test_results_300000_40epochs.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-01-28T06:35:15.061582Z",
     "iopub.status.busy": "2021-01-28T06:35:15.059217Z",
     "iopub.status.idle": "2021-01-28T06:35:15.065041Z",
     "shell.execute_reply": "2021-01-28T06:35:15.066213Z"
    },
    "papermill": {
     "duration": 13.145046,
     "end_time": "2021-01-28T06:35:15.066388",
     "exception": false,
     "start_time": "2021-01-28T06:35:01.921342",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "pd.DataFrame.from_dict(history.history).to_csv('bert_history_300000_40epochs.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-01-28T06:35:41.880991Z",
     "iopub.status.busy": "2021-01-28T06:35:41.880277Z",
     "iopub.status.idle": "2021-01-28T06:35:42.178284Z",
     "shell.execute_reply": "2021-01-28T06:35:42.178883Z"
    },
    "papermill": {
     "duration": 13.575591,
     "end_time": "2021-01-28T06:35:42.179066",
     "exception": false,
     "start_time": "2021-01-28T06:35:28.603475",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3dd3yV9dn48c+VEMggkwxCEhL23gEHVnG1uOqoWqpWra2j2lb9dWi3HU+rT/e22tLHrThAVEBFizhAVgKEvQJZJCEkIZOs6/fHfQOH5AQOkJOTcb1fr7xyzv2973Ou3JBz5btFVTHGGGNaCwp0AMYYY7omSxDGGGO8sgRhjDHGK0sQxhhjvLIEYYwxxitLEMYYY7yyBGEMICL/JyK/9PHcXBG5xN8xGRNoliCMMcZ4ZQnCmB5ERPoEOgbTc1iCMN2G27TzXRHZICI1IvJvEUkSkcUiUiUiS0Uk1uP8z4vIJhGpEJFlIjLGo2yKiKxzr3sJCG31XleKSLZ77SciMtHHGK8QkSwROSQieSLySKvy89zXq3DLb3ePh4nI70Rkr4hUishH7rFZIpLv5T5c4j5+REReEZFnReQQcLuIzBCRFe57FInIX0Wkr8f140TkXRE5KCLFIvIDERkoIrUiMsDjvGkiUioiIb787KbnsQRhupsvAJcCI4GrgMXAD4B4nP/P3wIQkZHAC8ADQAKwCHhDRPq6H5YLgGeAOOBl93Vxr50KzAXuBgYA/wQWikg/H+KrAW4FYoArgK+LyDXu6w524/2LG9NkINu97rfANOBcN6bvAS0+3pOrgVfc93wOaAYedO/JOcDFwL1uDJHAUmAJMAgYDrynqvuBZcCNHq97C/Ciqjb6GIfpYSxBmO7mL6parKoFwIfAp6qapaqHgfnAFPe8LwJvqeq77gfcb4EwnA/gs4EQ4I+q2qiqrwCrPd7jTuCfqvqpqjar6lPAYfe6E1LVZaq6UVVbVHUDTpK6wC2+GViqqi+471umqtkiEgTcAdyvqgXue37i/ky+WKGqC9z3rFPVtaq6UlWbVDUXJ8EdieFKYL+q/k5V61W1SlU/dcuewkkKiEgw8CWcJGp6KUsQprsp9nhc5+V5f/fxIGDvkQJVbQHygBS3rECPX6lyr8fjdODbbhNNhYhUAGnudSckImeJyH/dpplK4B6cv+RxX2OXl8vicZq4vJX5Iq9VDCNF5E0R2e82O/3KhxgAXgfGishQnFpapaquOs2YTA9gCcL0VIU4H/QAiIjgfDgWAEVAinvsiMEej/OA/1HVGI+vcFV9wYf3fR5YCKSpajTwOHDkffKAYV6uOQDUt1NWA4R7/BzBOM1TnlovyfwPYCswQlWjcJrgThYDqloPzMOp6XwZqz30epYgTE81D7hCRC52O1m/jdNM9AmwAmgCviUifUTkOmCGx7VPAve4tQERkQi38znSh/eNBA6qar2IzABu8ih7DrhERG5033eAiEx2azdzgd+LyCARCRaRc9w+j+1AqPv+IcCPgJP1hUQCh4BqERkNfN2j7E1goIg8ICL9RCRSRM7yKH8auB34PPCsDz+v6cEsQZgeSVW34bSn/wXnL/SrgKtUtUFVG4DrcD4Iy3H6K17zuHYNTj/EX93yne65vrgX+LmIVAE/wUlUR153H3A5TrI6iNNBPckt/g6wEacv5CDwGBCkqpXua/4Lp/ZTAxw3qsmL7+AkpiqcZPeSRwxVOM1HVwH7gR3AhR7lH+N0jq9z+y9MLya2YZAxxpOIvA88r6r/CnQsJrAsQRhjjhKR6cC7OH0oVYGOxwSWNTEZYwAQkadw5kg8YMnBgNUgjDHGtMNqEMYYY7zqUQt7xcfHa0ZGRqDDMMaYbmPt2rUHVLX13BqghyWIjIwM1qxZE+gwjDGm2xCRve2VWROTMcYYryxBGGOM8coShDHGGK96VB+EN42NjeTn51NfXx/oUPwqNDSU1NRUQkJsbxdjTMfo8QkiPz+fyMhIMjIyOH7xzp5DVSkrKyM/P58hQ4YEOhxjTA/R45uY6uvrGTBgQI9NDgAiwoABA3p8LckY07l6fIIAenRyOKI3/IzGmM7V45uYjDGmR2pugtItULAO6srhvAc6/C0sQfhZRUUFzz//PPfee+8pXXf55Zfz/PPPExMT46fIjDHdRksLHNzlJIPCdVCYBUUboKnOKe8/EM79FgR1bKOQJQg/q6io4O9//3ubBNHc3ExwcHC71y1atMjfoRljAk0VKvOgdBtUl0DdQagtc78Oul9lcKgQGtwFdvuEQfIkyPwKDJoKKVMhdkiHJwfwc4IQkdnAn4Bg4F+q+qiXc2YBfwRCgAOqeoGv13YHDz/8MLt27WLy5MmEhITQv39/kpOTyc7OZvPmzVxzzTXk5eVRX1/P/fffz1133QUcWzakurqayy67jPPOO49PPvmElJQUXn/9dcLCwgL8kxljTkljPZRuhf0boTgH9udA8Uaorzz+vKAQCI+D8AHOV8IoGHK+kxRSpkL8KAjunL/t/fYu7ubqf8PZ3jAfWC0iC1V1s8c5McDfgdmquk9EEn299nT87I1NbC48dCYv0cbYQVH89Kpx7ZY/+uij5OTkkJ2dzbJly7jiiivIyck5Ohx17ty5xMXFUVdXx/Tp0/nCF77AgAEDjnuNHTt28MILL/Dkk09y44038uqrr3LLLbd06M9hjOlAVcXOh//+nGPJ4MB20GanPCQCksbCuOtg4HhIHAeRA52E0C8SusigE3+moRnATlXdDSAiLwJXA54f8jcBr7l79aKqJadwbbc0Y8aM4+Yq/PnPf2b+/PkA5OXlsWPHjjYJYsiQIUyePBmAadOmkZub22nxGmNOoP4QlO2AAzugeNOxZFBTcuycqFQnCYy+AgZOcL781CTU0fyZIFKAPI/n+cBZrc4ZCYSIyDIgEviTqj7t47UAiMhdwF0AgwcPPmFAJ/pLv7NEREQcfbxs2TKWLl3KihUrCA8PZ9asWV7nMvTr1+/o4+DgYOrq6jolVmMM0NwIhwqgPBdKtzs1gQPbnaRQVXjsvOC+kDAaRlwKSeOdRJA0zmku6qb8mSC81ZFab1/XB5gGXAyEAStEZKWP1zoHVZ8AngDIzMzsctvjRUZGUlXlfffGyspKYmNjCQ8PZ+vWraxcubKTozPGAFBzwBkhVLkPKvKgMt/pPK7Ig6oijvv46RsJCSNh6AUQPwLiRzr9AnFDILhnLXXjzwSRD6R5PE8FCr2cc0BVa4AaEVkOTPLx2m5hwIABzJw5k/HjxxMWFkZSUtLRstmzZ/P4448zceJERo0axdlnnx3ASI3pReorYe8nsGe581Wcc6wsKASiUyA6DYbOguhUiEmDmMFOIogc2GX6CPzNb3tSi0gfYDtO7aAAWA3cpKqbPM4ZA/wV+BzQF1gFzAG2nuxabzIzM7X1hkFbtmxhzJgxHfRTdW296Wc1xmeqUFMK+zdA7kdOQijMAm2BPqEw+GxnlNDgc5y+gf6JENT+EPSeRkTWqmqmtzK/1SBUtUlEvgG8jTNUda6qbhKRe9zyx1V1i4gsATYALTjDWXPcoNtc669YjTE9RE2ZM7u4xP0q3ep8rzvolAf1gdTpcP53naSQkgkhoYGN+SSq6hvZVVpDc4uiqihOzlNVWhQUJSQ4iOkZHd/X4dfBtKq6CFjU6tjjrZ7/BviNL9caYwzgNBGVbPVIBpud556jh/pFOZ3GY66CxDHOV0om9OsfuLhPQlXJLatl7d5y1u0rZ93ecrYVV3Gyhp74/v1Y86NLOjwem0ltjOnaDhVCwVrna3+OkxAO5R8rD4mAxNEw4rPO98QxkDAGogZ16b6CmsNN7C2rZW9ZDbtKq8nOq2DdvgoO1jQAEBnahymDY5k9fiBjk6PoFxKM4PxIghAkgPu4bx//DJm1BGGM6TrqK53+gYK1zqiigrXuKCKczuOE0ZB+rlsjGOskhOjBXXZOgaqSX17HhvxKdpdWk+smhNyyWg5UHz7u3KEJEVw8OpGp6bFMS49leEJ/goICm+AsQRhjAudQEez7xBlRtPcTp3ZwZEjpgOFuP8E05ytpfJfvLyirPsyG/Eqy8yrYkF/B+vzKozUCgOToUAbHhXPx6ETS48NJj4sgfUA46QPCiQztekNkLUEYYzqHKlTsdZPBx873g7udsr79Ie0sGHetmxCmQlhsYOM9gfrGZvYcqGFHSTU7i6vYUVJNTmEleQedSawiMDIxkkvGJDIxNYZJqTGMSOpPaEj3Gh1lCaKL6d+/P9XV1YEOw5gz09IMZbucoaVF693vG46NJgqLhcHnQuZXnSajgRM7bQG6E1FVahqaKa9poLKukYraRirqGiivbaSoos5JCCXV7C2rocWt6AQJpA+IYPygaG45K51JaTGMT4mmf7/A/zxnqvv/BMaYwGusd+YX7HwXCrOdiWeNtU5ZcF+nz2D0Fc6KpOkznb6EAPcbVNY1Oh3D7oihLUVVVNQ20NTifchQnyBhSHwEY5IjuWrSIIYn9mdEYn+GxEd0u5qBryxB+NlDDz1Eenr60f0gHnnkEUSE5cuXU15eTmNjI7/85S+5+uqrAxypMaeorhy2vwPb3oIdS6GxxmkqSp4EU29z1iJKnujMPu7TN6ChtrQoe8pq3GTgJIXtJc7w0SCBUQOjuGh0AvH9+xETHkJMWF/ne/iR7yHEhvclJLhrdob7S+9KEIsfdtZi70gDJ8Bl7W9VMWfOHB544IGjCWLevHksWbKEBx98kKioKA4cOMDZZ5/N5z//edtX2nRtqs6CdTvega1vQu7HzvLVkckw6YtODSHjM9Cn30lfyp/qGprZuv8QW4qq2FxUyebCQ2zdX0Vtg7PUdmRoH6YOjuWKiclMS49lUlpMj2gO8ge7K342ZcoUSkpKKCwspLS0lNjYWJKTk3nwwQdZvnw5QUFBFBQUUFxczMCBAwMdrjGOhlpnRFGxx34GxZvgsLu5TcJoZw/kUVfAoCkBbS7aV1bLJ7sOsGJ3GRsLKsk9cKx/ILJfH8YMiuLGzDTGJkcxZXAMw7rA8NHuoncliBP8pe9P119/Pa+88gr79+9nzpw5PPfcc5SWlrJ27VpCQkLIyMjwusy3MZ2iuuRYJ/L+DU4yOLjLWasInGajpHEw4Xrn+5ALIH54wMItOVTPit1lfLzzAJ/sKiO/3Bk5lBDZjylpMVw1cRBjB0UxNjmK1Ngwq5mfgd6VIAJkzpw53HnnnRw4cIAPPviAefPmkZiYSEhICP/973/Zu3dvoEM0vcWhQshffSwZFG2A6v3HymMGOyOKxn/B2eQmaRzEZASshnCk7yB7XwXZeRWs3F3GjhJnlF9UaB/OGTaAu84fyrnDBjAsob8lgw5mCaITjBs3jqqqKlJSUkhOTubmm2/mqquuIjMzk8mTJzN69OhAh2h6qqr9x1Ywzf3IqRkASJDTeTz0AichJE90+tMCPPegvKaB7PwKstyEsD6vgsq6RgD69+vD1PRYrp+WyrnD4hk7KIpgayryK0sQnWTjxmOd4/Hx8axYscLreTYHwpyR2oOwexnkfgh7PnS2wwRn4br0cyHzK86y1knjICSsU0PbWVLFp3sOcrDamVdQUdtARV0j5bUNVNQe+w7OyKKRSZFcPmEgU9Jimez2HVhC6FyWIIzpzlSdpqId78COd53mI21xdj1LPwemftkZWRSgiWg7iqt4a2MRizYWsb342B8/4X2DiXWHkMaG9yUlJozY8L4MigljcloME1OjibCRRQFn/wLGdDeHq2DXf48lhSN9CIOmOPscDL/UeRygmcmtk4IITE+P45GrxnLR6CSSovvRr0/PnFjW0/SKBKGqPb7zyl87A5oAa2mGA9uPLXddsNYZbtrSBP2iYfhFzjLXwy9xdkLrRBW1Dew5UENuWQ17SmvYU1bL5sJKdpXWHE0KP/v8OGaPH0hSVNdeZM941+MTRGhoKGVlZQwYMKDHJglVpaysjNBQ+yXs9uornb6D/FXOcteFWdDgNs30i4aUKTDzARh2EaTNgGD/rwCqquw+UMOa3IOsyS1nZ2k1uQdqKHf7C8DpM0iNDWdoQgS3npNhSaGH6PEJIjU1lfz8fEpLSwMdil+FhoaSmpoa6DDMqVJ1dkPb8Y6zXEXeSqd2EBTijCqa9CVIzXRWOI0b1inDTZuaW9hUeIjVuQdZ7SaFMnfJ6tjwEEYPjGL2+GSGxkeQER/BkPgI0uLCrNmoB+rxCSIkJIQhQ4YEOgxjjjlc5Yw02vEu7FwKhwqc40kT4NxvOn0IqZmdumRFeU0D72zez+Kc/azac/DoshSD48K5YFQCMzLiyMyIY1hCRI+tiZu2enyCMKZLaKiF7Usg51UnMTQfdkYaDZsFsx52+hCiBnVqSGXVh3lnczGLNhbxya4ymluUtLgwbpiWyvQhcWSmxzEw2pqJejNLEMb4S9Nh2PmekxS2LXZWO+2f5MxFGH0lDD67U/oQjlBVCirqWLatlMU5RazcfZDmFiV9QDh3nT+UKyYkM25QlNUQzFGWIIzpSC3Nzqzlja/Aljecxe3C4mDiDc7yFekzIahz2uor6xrZkF9B9r4K1udXkJ1XeXQf5KHxEXz9gmFcNmEgY5MtKRjvLEEY0xFKtsD6F2DDPKgqcmYuj77SSQpDL/B7TaGhqYUtRYecRLCvguz8CnaX1hwtH5YQwfkj45mcFsNZQwYwMsnWLTInZwnCmNNVXQo5rziJoWg9SDCMuBRm/xpGXgYh/mm/V1Vyy2pZn+esV5SdV8HmwkM0NDurr8b378fktBium5LC5LRYJqRGEx3WeU1ZpuewBGGML1ShutgZklqyxWlG2vGus2FO8mSY/ZhTW+if4Je3r6xtZNn2Et7bUsLyHaVH1ywKCwlmQmo0t8/MYHJaDJPSYhgUHWq1A9MhLEEY01pLizNRrTjHSQYlW5zEUFd+7JzoNGdI6qQ5zn7LfrC7tJr3tpSwdEsxa/aW09yiDIjoy0WjE5meEcfktBhGJPanTy/bBtN0HksQxhzR1AAb58FHfzx+FdTEMTD2Gkgc6zxOHAMR8R3+9i0tSnZ+BUty9rN0czG7Dzh9CKOSIrn7/KFcPCaJyWkxtqKp6TSWIIxpqIG1T8GKvzqT1pImwLVPQMZMiEoBPzbXtLQo6/aVs2jjfhbnFFFUWU9IsHD20AHcdm4GF41OJC0u3G/vb8yJWIIwvVftQVj1JHz6ONQdhMHnwlV/ciat+TEpNLcoa3IPsjjHSQrFhw7TNziI80cm8N3PjeLiMUnWqWy6BEsQpndpaXb2TNi0ANY97UxeGzkbznvQmbjmJ6rK5qJDLMgq4PXsQkqqDtOvTxCzRiVw+YRkLhqdSGSoJQXTtViCMD3f4WrY/V9nNvP2JVBbBkF9YNx1cN4Dzu5qflJUWceCrEIWZBWwrbiKkGBh1qhEPj9pEBeNTrRNcUyX5tf/nSIyG/gTEAz8S1UfbVU+C3gd2OMeek1Vf+6W5QJVQDPQpKqZ/ozV9DBV+2HrW05S2LPcWfsoNNrZO2HkbKcZKSzGL29dWdvI25v3M39dASv3lKEKUwfH8ItrxnPlhGRiI/r65X2N6Wh+SxAiEgz8DbgUyAdWi8hCVd3c6tQPVfXKdl7mQlU94K8YTQ+jCnmrnD6Fza87cxRiM2D6V2HUZc5ezH6Y0Vzb0MSa3HI+3nWAFbvKyCmopEUhY0A49188gmunpJA+IKLD39cYf/NnDWIGsFNVdwOIyIvA1UDrBGHMmWk6DDmvOYmhKNvZWOfsr8OUWyBhdId3ODc2t5CdV8EnO8v4eNcBsvaV09ishAQLU9Ji+dbFI7hgZAKT02Jswprp1vyZIFKAPI/n+cBZXs47R0TWA4XAd1R1k3tcgXdERIF/quoT3t5ERO4C7gIYPHhwR8VuuoNDRbBmLqz9D9SUQvwouOL3MPGL0K9/h75V9eEmlm8v5d3Nxby/tYTKukZEYPygaO44bwjnDotnekYs4X2tT8H0HP783+ztT6fWGyevA9JVtVpELgcWACPcspmqWigiicC7IrJVVZe3eUEncTwBkJmZaRsz9wYF62Dl32HTfGdU0sjZcNbdMHRWh9YWSg7V8+6WYt7dXMwnO8toaG4hNjyES8YkccmYRM4dFk90uI08Mj2XPxNEPpDm8TwVp5ZwlKoe8ni8SET+LiLxqnpAVQvd4yUiMh+nyapNgjC9REszbFsEK/4G+1Y4m+3MuAtm3AlxQzvsbWoON/F6diEvr80ja18F4Oyqdus56Vw6Nolp6bG2tIXpNfyZIFYDI0RkCFAAzAFu8jxBRAYCxaqqIjIDCALKRCQCCFLVKvfxZ4Gf+zFW01UdroKs5+DTf0B5LkQPhs/9CqZ8GUKjOuxtdhRX8ezKvby2roCqw02MHhjJdz47kkvHDrSlsU2v5bcEoapNIvIN4G2cYa5zVXWTiNzjlj8OXA98XUSagDpgjpsskoD57i9lH+B5VV3ir1hNF1SZ73Q6r33a2XQndQZc8jNnj4Xgjvlv29DUwjub9/PMir18uucgfYODuGJiMrecnc7UwdbBbIyo9pxm+8zMTF2zZk2gwzBnongzfPJn2PiyM2x17Ofh7PsgbXqHvUV+eS0vrc7jxdV5lFYdJi0ujJvPSueGaakM6N+vw97HmO5ARNa2N8/MhlyYwFOFvR/Dx3+CHe9ASDhMv9MZqhqb3iFv0djcwtLNxbywOo8Pd5QCcNGoRG45O53zRybYCqnGeGEJwgROSzNsfdNJDAVrITweLvyRM7EtPK5D3mLPgRpeXL2PV9fmc6C6geToUL510QhunJ5GSkxYh7yHMT2VJQjT+VRhy0JY+jM4uAtihzjzFybfBCFn/qFd39jM25v28+KqPFbsLiM4SLhodCJfmpHGBSMTrbZgjI8sQZjOVbIVFn8P9nwAiePghqdgzFUQFHxGL6uqZOVV8PKafN5cX0jV4SbS4sL47udGcf20VJKi/LM/tDE9mSUI0znqK2HZY7Dqn9A3Ai7/LUz7yhmPSCo+VM9r6wp4ZW0eu0prCAsJ5rIJA7l+WipnDxlAkNUWjDltliCMf7W0wPoXYOkjznIYU2+Fi39yRlt2qir/3VbC0yv2snx7KS0K0zNiufv8YVw+MZn+toS2MR3CfpOM/xSsc5qT8ldD6nS46SVImXpGL7lydxm/eXsba/eWkxwdyr2zhnP9tFQy4m21VGM6miUI0/EKs2D5b50RShGJcM0/YOIcCDr9JSpyCir537e3sXx7KUlR/fjVtRO4ITOVEFv2whi/sQRhOk7ealj+v85chn7RcMFDcM59zkY9p2lXaTW/f2c7b20sIiY8hB9cPppbz8kgNOTMOrWNMSdnCcKcudyPncSwexmExcFFP3IW0juDxLCzpJonl+/mlXX59OsTxLcuGs7Xzh9KlO3bbEynsQRhTt+eD2HZr51Z0BGJcOkvIPOO096LoaSqnjfWF7Egq4CNBZX0DQ7i1nPSue/C4cTbEhjGdDpLEObU1VXAOz+ErGchchDMfgym3XZak9xqDjfx9qb9zM8q4OOdB2hRmJASzY+vHMtVk5JJjLT5C8YEiiUIc2q2LYE3H4DqEjjvQbjgYQg59Q/xdfvKeeqTXN7ZVExdYzOpsWHcd+Fwrp6cwvDEjt0NzhhzeixBGN/UHoTFD8HGec4M6DnPn/KQ1SPzFx5ftptVuQeJCu3DdVNTuHZKCtPSY215bWO6GEsQ5uQ2L4S3vg11B50aw2e+DX36+nx5Y3MLC7MLeWL5brYVVzEoOpSfXDmWL05PI8ImtRnTZdlvp2lfdSks+g5sXgADJ8KXX4OBE3y+vOZwEy+s2sfcj/ZQWFnP6IGR/OGLk7hy4iCbv2BMN2AJwni3aQG89f+cLT8v/gmc+y0I9m2Iqary8pp8fr14C+W1jZw1JI7/uXYCs0YlWDOSMd2IJQhzvNqDTq0h51UYNMWZBZ04xufLd5ZU84P5G1m15yAzMuL4/uWjmTI41o8BG2P8xRKEOWbbYnjjfidJXPgjOO8Bn2sN9Y3N/GPZLv6xbBdhfYN57AsTuGFamq2makw3ZgnCOPMalnwf1j8PSePh5lcgeaLPl6/YVcYP529k94Earpk8iB9dOdYmthnTA1iC6O12LoXXvwnVxfCZ7zjrJ/k4Qqm8poFfLdrCy2vzGRwXztN3zOD8kQl+DtgY01ksQfRWDTXwzo9gzVyIHwVznoWUab5d2tTCc5/u5U/v7aC6vol7Zw3jWxePsAX0jOlhLEH0Rvlr4bU74eBuOOcbcNGPfZoNraq8vWk/jy7eSm5ZLecNj+fHV45l1MDITgjaGNPZLEH0Js1N8OHv4IPHIDIZblsIQ8736dKsfeX8z1tbWLO3nJFJ/fm/r0zngpE2bNWYnswSRG9Rtgvm3+3s7jbhBmdP6LCYk16Wd7CWx5Zs5c0NRSRE9uPR6yZw/bRU+thEN2N6PEsQPZ0qrHsKlvwAgvvAF/4NE64/6WWHm5r583s7eHL5HoKDhPsvHsFd5w+1pTGM6UXst70nqzkAC78J2xY5TUnX/AOiU0962abCSr49bz1b91dx3dQUvve50QyMtmW3jeltLEH0VHuWw6t3Ogvsfe5XcNbXT7ondFNzC49/sIs/vbeDmPC+zL09k4tGJ3VSwMaYrsanBCEirwJzgcWq2uLfkMwZaW6CDx6F5b+FAcPhlld8WmBvV2k13563nuy8Cq6cmMwvrh5PbITvK7YaY3oeX2sQ/wC+AvxZRF4G/k9Vt/ovLHNaKvLg1a9B3kqYfAtc/r/QN+KEl7S0KE+tyOWxJVsJDQnmL1+awlWTBnVOvMaYLs2nBKGqS4GlIhINfAl4V0TygCeBZ1W10Y8xGl9seQNevw9aWnzuiC6oqOO7L6/nk11lXDgqgce+MJHEKOtrMMY4fB6rKCIDgNuBrwFZwJ+AqcC7J7hmtohsE5GdIvKwl/JZIlIpItnu1098vda4GuuczXxeugXihsI9y0+aHFSV+Vn5zP7DctbnVfDodROYe/t0Sw7GmOP42gfxGjAaeAa4SlWL3KKXRGRNO9cEA38DLgXygdUislBVN7c69UNVvfI0r+3dKgvg+RuhOMeZEX3xT/QECbIAABk5SURBVE+6jlJFbQM/nJ/DWxuLyEyP5Q9fnExaXHgnBWyM6U587YP4q6q+761AVTPbuWYGsFNVdwOIyIvA1YAvH/Jncm3vULYLnr4G6iuc1VdHXHrSSz7YXsp3X15PeW0D35s9irvPH0awLcdtjGmHr01MY0Tk6LRbEYkVkXtPck0KkOfxPN891to5IrJeRBaLyLhTvBYRuUtE1ojImtLS0pP+ID1C8Sb4z2XQWAO3vXHS5FDX0MxPX8/htrmriA4LYf69M7l31nBLDsaYE/I1QdypqhVHnqhqOXDnSa7x9umjrZ6vA9JVdRLwF2DBKVx7JJYnVDVTVTMTEnrBUtP5a+E/l4MEwVcWw6DJJzx9Q34FV/7lQ55asZc7Zg7hjW+ex/iU6E4K1hjTnfnaxBQkIqKqCkf7CE42SD4fSPN4ngoUep6gqoc8Hi8Skb+LSLwv1/ZKez6EF+ZARDzc+jrEZrR7alNzC/9Y5kx6i+/fj+e+dhYzh8d3XqzGmG7P1wTxNjBPRB7H+Uv+HmDJSa5ZDYwQkSFAATAHuMnzBBEZCBSrqorIDJwaTRlQcbJre51tS2DerRA3BL68AKKS2z11b1kND76Uzbp9FVw1aRC/vHo80eG+bR1qjDFH+JogHgLuBr6O0/zzDvCvE12gqk0i8g2c5BIMzFXVTSJyj1v+OHA98HURaQLqgDluLcXrtaf80/UUG19xVmIdOAFufhUiBng9TVWZtyaPn7+xmaAg4U9zJnP1ZK9dN8YYc1Lithr1CJmZmbpmjddRt93X2qfgjfsh/Vz40osQGuX1tLLqwzz82kbe3VzMOUMH8NsbJ5ESE9bJwRpjuhsRWdveaFRf50GMAH4NjAWOzqZS1aEdEqHxLvt5eONbMPxSuPFp6Ot9vsL7W4v53isbOVTXyA8vH8NXzxtCkI1QMsacIV+bmP4D/BT4A3AhzrpM9gnkT5sXOktnDJ0Fc56DPv3anNLY3MLP3tjEsyv3MXpgJM98dQZjkr3XMIwx5lT5Osw1TFXfw2mS2quqjwAX+S+sXm7X+/DqVyElE+Y87zU5NLcoD76UzbMr9/G184aw4L6ZlhyMMR3K1xpEvYgEATvczuMCINF/YfVi+1bCizdD/Ci4eZ7X1VhbWpSHXt3AmxuK+P5lo7n7gmEBCNQY09P5WoN4AAgHvgVMA24BbvNXUL1W0Xp47kaIGgRffg3CYtucoqr8dOEmXlmbz/0Xj7DkYIzxm5PWINxJcTeq6neBapz+B9PRSrfDM9dBv0hnnkP/thU0VeXRxVt5ZuVe7j5/KA9cMiIAgRpjeouT1iBUtRmYJiLWKe0vFfvgmWtAxJkhHZPm9bQ/vbeDfy7fza3npPPwZaOxfxJjjD/52geRBbzu7iZXc+Sgqr7ml6h6k6piePpqaKiG2xdB/HCvp/3zg138cekOrp+WyiNXjbPkYIzxO18TRBzOEhieI5cUsARxJhpq4LnrnSRx6+swcLzX055ekcuvF2/lyonJPPaFiTbHwRjTKXzdctT6HTqaKiy4F/ZvhJvmQdp0r6fNW5PHT17fxKVjk/jDFyfbEt3GmE7j60zq/+BluW1VvaPDI+otPvwtbF4Al/wMRn7W6ylvb9rPw69u4DMj4vnrTVMICfZ5h1hjjDljvjYxvenxOBS4Flt++/RtXQTv/xIm3Agz7/d6yqo9B/nmC1lMTI3hn1+eRr8+wZ0cpDGmt/O1ielVz+ci8gKw1C8R9XQlW+C1O2HQFPj8n52RS61s21/F155aTWpsGHNvn054X1/zuDHGdJzTbbMYAQzuyEB6hdqDzoY/fSOcJTRC2q62WlBRx21zVxHWN5in75hBXMTJ9mUyxhj/8LUPoorj+yD24+wRYXzV3AQv3w6HCp3hrFGD2pxSXtPArf/+lJqGJl6+5xxSY72v3mqMMZ3B1yamSH8H0uO980PY8wFc/XevI5ZqG5q446nV5JXX8cwdMxg90BbeM8YElk9NTCJyrYhEezyPEZFr/BdWD7PuGfj0cTj7Pphyc5vixuYWvvF8FuvzKvjznCmcNdT7jnHGGNOZfO2D+KmqVh55oqoVOPtDmJPJWw1vPghDL4RLf96mWFX5wWsbeX9rCb+4Zjyzxw8MQJDGGNOWrwnC23k2tOZkmhpgwdchKhlu+A8Et71lf3h3Oy+vzeeBS0Zw81npAQjSGGO88zVBrBGR34vIMBEZKiJ/ANb6M7AeYcVfoGwHXPF7r0t3L91czJ/f38kXM9O4/2JbmdUY07X4miC+CTQALwHzgDrgPn8F1SOU74UPfgNjroIRl7YpzjtYy/+bl834lCh+drUtvmeM6Xp8HcVUAzzs51h6liUPgwTB7EfbFB1uaube59YB8I+bpxEaYrOkjTFdj6+jmN4VkRiP57Ei8rb/wurmti6CbYtg1kMQndqm+BdvbmZjQSW/u3EyaXE218EY0zX52sQU745cAkBVy7E9qb1rqIXFD0HCGDj73jbFr2cX8OzKfdx9wVAuHZsUgACNMcY3viaIFhE5urSGiGTgZXVXAyz/DVTugyt/D8EhxxXtLKni+69tZEZGHN/97KgABWiMMb7xdajqD4GPROQD9/n5wF3+CakbK90Gn/wFJt0E6eceV1RzuIl7nl1HeN9g/nLTFPrY0t3GmC7O107qJSKSiZMUsoHXcUYymSNU4a1vQ9/wNhPiVJUfzt/I7tJqnvnqWSRFhQYoSGOM8Z2vi/V9DbgfSMVJEGcDKzh+C9LebeMrkPuhM+ehf8JxRc+v2seC7EK+felIZg6PD1CAxhhzanxt57gfmA7sVdULgSlAqd+i6m7qK+HtH8CgqTDt9uOKcgoq+dnCzcwalcB9Fw4PTHzGGHMafO2DqFfVehFBRPqp6lYRsV7WI97/H6g9ADfPg6Dj5zQ8tmQrUWEh/OHGyQTZftLGmG7E1wSR786DWAC8KyLl2Jajjv0bYfWTMP1rzi5xHrYXV/HhjgN893OjiLWNf4wx3YxPTUyqeq2qVqjqI8CPgX8DJ13uW0Rmi8g2EdkpIu3OxBaR6SLSLCLXexzLFZGNIpItImt8iTMgPvkLhITDhT9sUzT3oz2EhgRx0wzbfM8Y0/2c8oqsqvrByc8CEQkG/gZcCuQDq0Vkoapu9nLeY4C3mdkXquqBU42x01SXQM5rkPkVCIs5rqis+jCvZRVw/bRUqz0YY7olfw7GnwHsVNXdqtoAvAhc7eW8bwKvAiV+jMU/1v4ftDTCjLZTQl5YtY+Gpha+cm5Gp4dljDEdwZ8JIgXI83ie7x47SkRSgGuBx71cr8A7IrJWRNqdlCcid4nIGhFZU1raiQOrmhthzVwYdhHEH79Ud0NTC0+v2Mv5IxMYkWS7tRpjuid/JghvQ3ZaL8/xR+AhVW32cu5MVZ0KXAbcJyLne3sTVX1CVTNVNTMhIcHbKf6xZSFUFcGMu9sUvbWxkJKqw3z1vCGdF48xxnQwf+4Klw+keTxPpe3Ip0zgRXcvhHjgchFpUtUFqloIoKolIjIfp8lquR/jPTWfPgGxGW32elBV/v3RHoYn9uf8ETYpzhjTffmzBrEaGCEiQ0SkLzAHWOh5gqoOUdUMVc0AXgHuVdUFIhIhIpEAIhIBfBbI8WOsp6ZoPeSthOl3tpn3sDq3nJyCQ9wxc4htAmSM6db8VoNQ1SYR+QbO6KRgYK6qbhKRe9xyb/0ORyQB890P2D7A86q6xF+xnrJPn3CGtk65pU3Rvz/aTUx4CNdOSfFyoTHGdB/+bGJCVRcBi1od85oYVPV2j8e7gUn+jO201ZTBxpdh8k1thrbuK6vlnc3F3DtrGGF9bZc4Y0z3ZmtOn6p1T0HzYa9DW59akUuwCF8+O6PTwzLGmI5mCeJUNDfB6n9DxmcgaexxRVX1jby0Oo8rJiYzMNqW8zbGdH+WIE7FtkVwKB/Oaju09eU1+VQfbrKhrcaYHsMSxKlY9QREp8HIy4473Nyi/OeTPWSmxzIxNaadi40xpnuxBOGr4k3OhkDTvwrBx/ftL91STN7BOqs9GGN6FEsQvlr1BPQJham3tSn690d7SIkJ49KxSQEIzBhj/MMShC/qymHDPJhwPYTHHVeUU1DJqj0H+crMDPoE2+00xvQc9onmi6xnobHW67pLr2cXEBIs3DAtzcuFxhjTfVmCOJmWZlj1JAw+B5InHlekqizO2c/M4fFEh4cEKEBjjPEPSxAnU7AWKvZC5h1tijYVHiK/vI7Lxg8MQGDGGONfliBOJvcj5/uwi9oULc4pIjhIuHSsJQhjTM9jCeJk9n4MCaMh4vilu480L501JI4421LUGNMDWYI4keYm2LcS0me2KdpRUs3u0hprXjLG9FiWIE5k/3poqIaMtgli8cb9iMDnxlmCMMb0TJYgTuRI/0P6eW2KFucUMW1wLIlRtjCfMaZnsgRxIrkfw4DhEHn8DOncAzVs3V/FbGteMsb0YJYg2tPSDPtWeO1/WJyzH8AShDGmR7ME0Z79G+HwIcho27y0JKeIianRpMaGByAwY4zpHJYg2rP3Y+d7qxpEQUUd6/MrrfZgjOnxLEG0J/djiB0C0SnHHV7iNi9dNj45EFEZY0ynsQThTUsL7PvE6/DWJTlFjB4YyZD4iAAEZowxnccShDclm50lvlsNby2pqmfN3nJrXjLG9AqWILw50v/Qqgbx9qZiVK15yRjTO1iC8Cb3I4geDDGDjzu8JKeIofERjEzqH6DAjDGm81iCaE3VqUG0qj2U1zSwcvdBZo8fiIgEKDhjjOk8liBaK90KtWVthre+u7mY5ha15iVjTK9hCaK1I+svtapBLM4pIjU2jPEpUQEIyhhjOp8liNb2fgyRg5w5EK5D9Y18tPMAs8dZ85IxpvewBOFJ1ZkglzETPBLB+1tKaGxWLptgw1uNMb2HJQhPZTuhpqRN/8PinCKSovoxJS02QIEZY0znswTh6Wj/w2eOHqptaOKD7aV8btxAgoKseckY03v4NUGIyGwR2SYiO0Xk4ROcN11EmkXk+lO9tkPt/Rj6J8GAYUcPZe2roL6xhQtHJ3ZKCMYY01X4LUGISDDwN+AyYCzwJREZ2855jwFvn+q1HepI/0P68f0PWfvKAZhqzUvGmF7GnzWIGcBOVd2tqg3Ai8DVXs77JvAqUHIa13ac8j1QVdhmeGvWvgqGJUQQHR7i17c3xpiuxp8JIgXI83ie7x47SkRSgGuBx0/1Wo/XuEtE1ojImtLS0tOPNvfI/g/HFuhTVbLzKphstQdjTC/kzwThrUdXWz3/I/CQqjafxrXOQdUnVDVTVTMTEhJOI0zX3o8hPB4SRh09lHewjrKaBqYMjjn91zXGmG6qjx9fOx9I83ieChS2OicTeNGdfBYPXC4iTT5e27FyP4L0c4/vf8hz+h8sQRhjeiN/1iBWAyNEZIiI9AXmAAs9T1DVIaqaoaoZwCvAvaq6wJdrO1T5XqjMa7P/dNa+CsJCghmVFOm3tzbGmK7KbzUIVW0SkW/gjE4KBuaq6iYRucctb93vcNJr/RVre/tPZ+VVMDE1mj7BNl3EGNP7+LOJCVVdBCxqdcxrYlDV2092rd/kfgxhsZB4bCRtfWMzmwsrueO8ISe40Bhjei770xhg70cw+FwIOnY7NhUeorFZbXkNY0yv5dcaRLfQWA9J42HEZ487fGSCnHVQG2N6K0sQIaEw57k2h7PzKkiJCSMpKjQAQRljTOBZE1M7svZVMDnNag/GmN7LEoQXJYfqKaios+YlY0yvZgnCi6y8CsD6H4wxvZslCC+y8yoICRbGDYoOdCjGGBMwliC8yNpXztjkKEJDggMdijHGBIwliFaamlvYkF9pHdTGmF7PEkQr24urqW1oZspgmyBnjOndLEG0km0d1MYYA1iCaCNrXzlxEX0ZHBce6FCMMSagLEG0kpVXwZS0GES87VlkjDG9hyUID5V1jewsqbYOamOMwRLEcdYf7X+wDmpjjLEE4SE7rwIRmJhmE+SMMcYShIesfeWMSOxPVGhIoEMxxpiAswThUlWy8mwFV2OMOcIShCu3rJaK2kbrfzDGGJclCFd2nu0gZ4wxnixBuLL2VRDRN5gRiZGBDsUYY7oESxCurH0VTEqLITjIJsgZYwxYggCgvrGZLUWHrIPaGGM8WIIAcgoqaWpR66A2xhgPliBwmpcAq0EYY4wHSxBAVl45aXFhJET2C3QoxhjTZViCwKlBTE6z5iVjjPHUJ9ABBFpDUwvnDY9n5vD4QIdijDFdSq9PEH37BPGbGyYFOgxjjOlyrInJGGOMV5YgjDHGeOXXBCEis0Vkm4jsFJGHvZRfLSIbRCRbRNaIyHkeZbkisvFImT/jNMYY05bf+iBEJBj4G3ApkA+sFpGFqrrZ47T3gIWqqiIyEZgHjPYov1BVD/grRmOMMe3zZw1iBrBTVXeragPwInC15wmqWq2q6j6NABRjjDFdgj8TRAqQ5/E83z12HBG5VkS2Am8Bd3gUKfCOiKwVkbvaexMRucttnlpTWlraQaEbY4zxZ4LwtixqmxqCqs5X1dHANcAvPIpmqupU4DLgPhE539ubqOoTqpqpqpkJCQkdEbcxxhj8myDygTSP56lAYXsnq+pyYJiIxLvPC93vJcB8nCYrY4wxncSfE+VWAyNEZAhQAMwBbvI8QUSGA7vcTuqpQF+gTEQigCBVrXIffxb4+cnecO3atQdEZO9pxhsPdNUOcYvt9Fhsp8diOz3dNbb09i7yW4JQ1SYR+QbwNhAMzFXVTSJyj1v+OPAF4FYRaQTqgC+6ySIJmC8iR2J8XlWX+PCep93GJCJrVDXzdK/3J4vt9Fhsp8diOz09MTa/LrWhqouARa2OPe7x+DHgMS/X7QZs/QtjjAkgm0ltjDHGK0sQxzwR6ABOwGI7PRbb6bHYTk+Pi02OzVMzxhhjjrEahDHGGK8sQRhjjPGq1yeIk604G0hdbUVbEZkrIiUikuNxLE5E3hWRHe73gOzd2k5sj4hIgXv/skXk8gDElSYi/xWRLSKySUTud48H/L6dILaucN9CRWSViKx3Y/uZe7wr3Lf2Ygv4ffOIMVhEskTkTff5ad23Xt0H4a44ux2PFWeBL7VacTZgRCQXyOwqK9q6y51UA0+r6nj32P8CB1X1UTfBxqrqQ10ktkeAalX9bWfH4xFXMpCsqutEJBJYi7OszO0E+L6dILYbCfx9EyBCVatFJAT4CLgfuI7A37f2YptNgO/bESLy/4BMIEpVrzzd39PeXoM46Yqz5hh3OZSDrQ5fDTzlPn4K5wOm07UTW8CpapGqrnMfVwFbcBatDPh9O0FsAaeOavdpiPuldI371l5sXYKIpAJXAP/yOHxa9623JwifVpwNIJ9WtA2wJFUtAucDB0gMcDytfUOcTanmBqr56wgRyQCmAJ/Sxe5bq9igC9w3t5kkGygB3lXVLnPf2okNusB9A/4IfA9o8Th2WvettycIn1acDSCfVrQ17foHMAyYDBQBvwtUICLSH3gVeEBVDwUqDm+8xNYl7puqNqvqZJyFPmeIyPhAxOFNO7EF/L6JyJVAiaqu7YjX6+0J4pRWnO1s3WRF22K3LftIm3ZJgOM5SlWL3V/kFuBJAnT/3HbqV4HnVPU193CXuG/eYusq9+0IVa0AluG08XeJ+3aEZ2xd5L7NBD7v9l++CFwkIs9ymvettyeIoyvOikhfnBVnFwY4JgBEJMLtOESOrWibc+KrAmIhcJv7+Dbg9QDGcpwjvxCuawnA/XM7NP8NbFHV33sUBfy+tRdbF7lvCSIS4z4OAy4BttI17pvX2LrCfVPV76tqqqpm4Hyeva+qt3C6901Ve/UXcDnOSKZdwA8DHY9HXEOB9e7Xpq4QG/ACTtW5Eaf29VVgAM7e4jvc73FdKLZngI3ABvcXJDkAcZ2H02y5Ach2vy7vCvftBLF1hfs2EchyY8gBfuIe7wr3rb3YAn7fWsU5C3jzTO5brx7maowxpn29vYnJGGNMOyxBGGOM8coShDHGGK8sQRhjjPHKEoQxxhivLEEY0wWIyKwjK28a01VYgjDGGOOVJQhjToGI3OLuBZAtIv90F22rFpHficg6EXlPRBLccyeLyEp38bb5RxZvE5HhIrLU3U9gnYgMc1++v4i8IiJbReQ5d6azMQFjCcIYH4nIGOCLOIsoTgaagZuBCGCdOgsrfgD81L3kaeAhVZ2IM8P2yPHngL+p6iTgXJwZ4OCspvoAMBZnJv1Mv/9QxpxAn0AHYEw3cjEwDVjt/nEfhrPoWQvwknvOs8BrIhINxKjqB+7xp4CX3fW1UlR1PoCq1gO4r7dKVfPd59lABs5mNMYEhCUIY3wnwFOq+v3jDor8uNV5J1q/5kTNRoc9Hjdjv58mwKyJyRjfvQdcLyKJcHSf33Sc36Pr3XNuAj5S1UqgXEQ+4x7/MvCBOvst5IvINe5r9BOR8E79KYzxkf2FYoyPVHWziPwIZ5e/IJyVY+8DaoBxIrIWqMTppwBnWeXH3QSwG/iKe/zLwD9F5Ofua9zQiT+GMT6z1VyNOUMiUq2q/QMdhzEdzZqYjDHGeGU1CGOMMV5ZDcIYY4xXliCMMcZ4ZQnCGGOMV5YgjDHGeGUJwhhjjFf/H+ixZRpyr5CeAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.plot(history.history['accuracy'])\n",
    "plt.plot(history.history['val_accuracy'])\n",
    "plt.title('model accuracy')\n",
    "plt.ylabel('accuracy')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['train', 'val'], loc='upper left')\n",
    "# plt.show()\n",
    "plt.savefig('bert_accuracy_300000_40epochs.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 13.473901,
     "end_time": "2021-01-28T06:36:36.423128",
     "exception": false,
     "start_time": "2021-01-28T06:36:22.949227",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  },
  "papermill": {
   "duration": 26995.119394,
   "end_time": "2021-01-28T06:36:52.178724",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2021-01-27T23:06:57.059330",
   "version": "2.1.0"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "19208161d5c646dc81ff74ca99e3837b": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "369c9ff2e0a8470cb6156a3910c1801a": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "39ae0a94b17d4f70b2f5a3612c35a489": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "4850c062d3cf4f42bb44d6163e9ec939": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": "initial"
      }
     },
     "4ae9fdaf72fe48229da8b6ac02067105": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "51aa3455e0ad4e16911f374c2182ff46": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "Downloading: 100%",
       "description_tooltip": null,
       "layout": "IPY_MODEL_e7c6251f4a22431aaf020ebb26ccf9c7",
       "max": 231508,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_5fb6f585b6474265bdffc2278db22b96",
       "value": 231508
      }
     },
     "5fb6f585b6474265bdffc2278db22b96": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": "initial"
      }
     },
     "695550d3012c464fbdccfa1d60e0b7ab": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": "initial"
      }
     },
     "707e09b536364dd2a205cdebf1eb9ec2": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "Downloading: 100%",
       "description_tooltip": null,
       "layout": "IPY_MODEL_bcb0eaf2fa4a4ab4b68beeffc9ff4225",
       "max": 442,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_4850c062d3cf4f42bb44d6163e9ec939",
       "value": 442
      }
     },
     "77673f43325c49a7a39677b5979cafc0": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_864cd133e62c436dab66f71794bd3426",
       "placeholder": "​",
       "style": "IPY_MODEL_c5610523d33f4aad850cd1bc792de4fb",
       "value": " 363M/363M [00:10&lt;00:00, 35.1MB/s]"
      }
     },
     "851a6809ab8241b48bb5e18f86fb32f4": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_c2f70c54288a47738f0d7fb7623ac371",
        "IPY_MODEL_77673f43325c49a7a39677b5979cafc0"
       ],
       "layout": "IPY_MODEL_4ae9fdaf72fe48229da8b6ac02067105"
      }
     },
     "864cd133e62c436dab66f71794bd3426": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "8697dd928e5a46c8a1b183cec2135099": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_51aa3455e0ad4e16911f374c2182ff46",
        "IPY_MODEL_ccee7d8c30c740f0981765ffd673ac68"
       ],
       "layout": "IPY_MODEL_beea61486ccf420aae463d8bb4293c06"
      }
     },
     "992b30de88114363870c82f35afddd05": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "a7cb3f38c7564f998e5f1f1f7773148f": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_992b30de88114363870c82f35afddd05",
       "placeholder": "​",
       "style": "IPY_MODEL_39ae0a94b17d4f70b2f5a3612c35a489",
       "value": " 442/442 [00:11&lt;00:00, 39.6B/s]"
      }
     },
     "bcb0eaf2fa4a4ab4b68beeffc9ff4225": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "beea61486ccf420aae463d8bb4293c06": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "c0130d035f494419a5f3f60c44b2c870": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_707e09b536364dd2a205cdebf1eb9ec2",
        "IPY_MODEL_a7cb3f38c7564f998e5f1f1f7773148f"
       ],
       "layout": "IPY_MODEL_369c9ff2e0a8470cb6156a3910c1801a"
      }
     },
     "c2f70c54288a47738f0d7fb7623ac371": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "Downloading: 100%",
       "description_tooltip": null,
       "layout": "IPY_MODEL_ccf2f7f42dc1432ea7a79ce6b0ad8646",
       "max": 363423424,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_695550d3012c464fbdccfa1d60e0b7ab",
       "value": 363423424
      }
     },
     "c5610523d33f4aad850cd1bc792de4fb": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "c7187d482d4b4558977feda4b879494f": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "ccee7d8c30c740f0981765ffd673ac68": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_c7187d482d4b4558977feda4b879494f",
       "placeholder": "​",
       "style": "IPY_MODEL_19208161d5c646dc81ff74ca99e3837b",
       "value": " 232k/232k [00:00&lt;00:00, 1.60MB/s]"
      }
     },
     "ccf2f7f42dc1432ea7a79ce6b0ad8646": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "e7c6251f4a22431aaf020ebb26ccf9c7": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}