ELMo_Classifier.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2021-01-27T23:08:42.242518Z",
     "iopub.status.busy": "2021-01-27T23:08:42.241957Z",
     "iopub.status.idle": "2021-01-27T23:08:42.310530Z",
     "shell.execute_reply": "2021-01-27T23:08:42.309695Z"
    },
    "papermill": {
     "duration": 0.091166,
     "end_time": "2021-01-27T23:08:42.310672",
     "exception": false,
     "start_time": "2021-01-27T23:08:42.219506",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/kaggle/input/embedding/elmo_3/tfhub_module.pb\n",
      "/kaggle/input/embedding-elmo/saved_model.pb\n",
      "/kaggle/input/embedding-elmo/tfhub_module.pb\n",
      "/kaggle/input/embedding-elmo/variables/variables.index\n",
      "/kaggle/input/embedding-elmo/variables/variables.data-00000-of-00001\n",
      "/kaggle/input/stanford-natural-language-inference-corpus/README.txt\n",
      "/kaggle/input/stanford-natural-language-inference-corpus/snli_1.0_test.csv\n",
      "/kaggle/input/stanford-natural-language-inference-corpus/snli_1.0_train.csv\n",
      "/kaggle/input/stanford-natural-language-inference-corpus/snli_1.0_dev.csv\n",
      "/kaggle/input/embedded-sentences-elmo/train_hypothesis_embedding.csv\n",
      "/kaggle/input/embedded-sentences-elmo/train_premise_embedding.csv\n",
      "/kaggle/input/embedded-sentences-elmo/test_hypothesis_embedding.csv\n",
      "/kaggle/input/embedded-sentences-elmo/test_premise_embedding.csv\n"
     ]
    }
   ],
   "source": [
    "# This Python 3 environment comes with many helpful analytics libraries installed\n",
    "# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n",
    "# For example, here's several helpful packages to load\n",
    "# !pip install --upgrade tensorflow==1.15.5\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "# Input data files are available in the read-only \"../input/\" directory\n",
    "# For example, running this (by clicking run or  Shift+Enter) will list all files under the input directory\n",
    "# pressing\n",
    "import os\n",
    "for dirname, _, filenames in os.walk('/kaggle/input'):\n",
    "    for filename in filenames:\n",
    "        print(os.path.join(dirname, filename))\n",
    "\n",
    "# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n",
    "# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a",
    "execution": {
     "iopub.execute_input": "2021-01-27T23:08:42.347323Z",
     "iopub.status.busy": "2021-01-27T23:08:42.346758Z",
     "iopub.status.idle": "2021-01-27T23:08:49.144323Z",
     "shell.execute_reply": "2021-01-27T23:08:49.144934Z"
    },
    "papermill": {
     "duration": 6.817777,
     "end_time": "2021-01-27T23:08:49.145120",
     "exception": false,
     "start_time": "2021-01-27T23:08:42.327343",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "snli_train = pd.read_csv('/kaggle/input/stanford-natural-language-inference-corpus/snli_1.0_train.csv', nrows=300000)\n",
    "snli_test = pd.read_csv('/kaggle/input/stanford-natural-language-inference-corpus/snli_1.0_test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-01-27T23:08:49.206043Z",
     "iopub.status.busy": "2021-01-27T23:08:49.205215Z",
     "iopub.status.idle": "2021-01-27T23:08:49.213316Z",
     "shell.execute_reply": "2021-01-27T23:08:49.213927Z"
    },
    "papermill": {
     "duration": 0.044769,
     "end_time": "2021-01-27T23:08:49.214108",
     "exception": false,
     "start_time": "2021-01-27T23:08:49.169339",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "A person on a horse jumps over a broken down airplane. A person is training his horse for a competition. neutral\n",
      "A person on a horse jumps over a broken down airplane. A person is at a diner, ordering an omelette. contradiction\n",
      "A person on a horse jumps over a broken down airplane. A person is outdoors, on a horse. entailment\n"
     ]
    }
   ],
   "source": [
    "print(snli_train['sentence1'][0],snli_train['sentence2'][0], snli_train['gold_label'][0])\n",
    "print(snli_train['sentence1'][1],snli_train['sentence2'][1], snli_train['gold_label'][1])\n",
    "print(snli_train['sentence1'][2],snli_train['sentence2'][2], snli_train['gold_label'][2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-01-27T23:08:49.285000Z",
     "iopub.status.busy": "2021-01-27T23:08:49.284180Z",
     "iopub.status.idle": "2021-01-27T23:08:49.503224Z",
     "shell.execute_reply": "2021-01-27T23:08:49.503667Z"
    },
    "papermill": {
     "duration": 0.258914,
     "end_time": "2021-01-27T23:08:49.503797",
     "exception": false,
     "start_time": "2021-01-27T23:08:49.244883",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0         A person on a horse jumps over a broken down a...\n",
      "1         A person on a horse jumps over a broken down a...\n",
      "2         A person on a horse jumps over a broken down a...\n",
      "3                     Children smiling and waving at camera\n",
      "4                     Children smiling and waving at camera\n",
      "                                ...                        \n",
      "299995    Three young Asian ladies pose for a picture at...\n",
      "299996    Three young Asian ladies pose for a picture at...\n",
      "299997                         A woman in pain in her room.\n",
      "299998                         A woman in pain in her room.\n",
      "299999                         A woman in pain in her room.\n",
      "Name: premise, Length: 299616, dtype: object\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "# Filter and rename the important columns\n",
    "train_df = snli_train.filter(['sentence1','sentence2','gold_label'], axis=1)\n",
    "test_df = snli_test.filter(['sentence1','sentence2','gold_label'], axis=1)\n",
    "train_df = train_df.rename(columns={\"sentence1\": \"premise\", \"sentence2\": \"hypothesis\"})\n",
    "test_df = test_df.rename(columns={\"sentence1\": \"premise\", \"sentence2\": \"hypothesis\"})\n",
    "\n",
    "# exclude sentences which could not be categorised\n",
    "train_df = train_df[train_df['gold_label'] != \"-\"]\n",
    "test_df = test_df[test_df['gold_label'] != \"-\"]\n",
    "\n",
    "# exclude NaN values\n",
    "train_df = train_df[train_df['hypothesis'] == train_df['hypothesis']]\n",
    "test_df = test_df[test_df['hypothesis'] == test_df['hypothesis']]\n",
    "\n",
    "train_df['gold_label'].value_counts()\n",
    "print(train_df['premise'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-01-27T23:08:49.541837Z",
     "iopub.status.busy": "2021-01-27T23:08:49.541135Z",
     "iopub.status.idle": "2021-01-27T23:08:59.115612Z",
     "shell.execute_reply": "2021-01-27T23:08:59.114159Z"
    },
    "papermill": {
     "duration": 9.595128,
     "end_time": "2021-01-27T23:08:59.115789",
     "exception": false,
     "start_time": "2021-01-27T23:08:49.520661",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import transformers\n",
    "\n",
    "max_length = 128  # Maximum length of input sentence to the model.\n",
    "batch_size = 64\n",
    "epochs = 40"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-01-27T23:08:59.189396Z",
     "iopub.status.busy": "2021-01-27T23:08:59.187932Z",
     "iopub.status.idle": "2021-01-27T23:08:59.192571Z",
     "shell.execute_reply": "2021-01-27T23:08:59.193344Z"
    },
    "papermill": {
     "duration": 0.047525,
     "end_time": "2021-01-27T23:08:59.193525",
     "exception": false,
     "start_time": "2021-01-27T23:08:59.146000",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "torch.cuda.is_available()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-01-27T23:08:59.264447Z",
     "iopub.status.busy": "2021-01-27T23:08:59.263120Z",
     "iopub.status.idle": "2021-01-27T23:08:59.596805Z",
     "shell.execute_reply": "2021-01-27T23:08:59.597564Z"
    },
    "papermill": {
     "duration": 0.370677,
     "end_time": "2021-01-27T23:08:59.597745",
     "exception": false,
     "start_time": "2021-01-27T23:08:59.227068",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "train_df[\"label\"] = train_df[\"gold_label\"].apply(\n",
    "    lambda x: 0 if x == \"contradiction\" else 1 if x == \"entailment\" else 2\n",
    ")\n",
    "y_train = tf.keras.utils.to_categorical(train_df.label, num_classes=3)\n",
    "\n",
    "test_df[\"label\"] = test_df[\"gold_label\"].apply(\n",
    "    lambda x: 0 if x == \"contradiction\" else 1 if x == \"entailment\" else 2\n",
    ")\n",
    "y_test = tf.keras.utils.to_categorical(test_df.label, num_classes=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-01-27T23:08:59.661628Z",
     "iopub.status.busy": "2021-01-27T23:08:59.660694Z",
     "iopub.status.idle": "2021-01-27T23:09:16.490405Z",
     "shell.execute_reply": "2021-01-27T23:09:16.492152Z"
    },
    "papermill": {
     "duration": 16.865536,
     "end_time": "2021-01-27T23:09:16.492390",
     "exception": false,
     "start_time": "2021-01-27T23:08:59.626854",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.3.1\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "print(tf.__version__)\n",
    "import tensorflow_hub as hub\n",
    "\n",
    "elmo = hub.load('../input/embedding-elmo')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-01-27T23:09:16.561781Z",
     "iopub.status.busy": "2021-01-27T23:09:16.560931Z",
     "iopub.status.idle": "2021-01-27T23:09:16.627878Z",
     "shell.execute_reply": "2021-01-27T23:09:16.628999Z"
    },
    "papermill": {
     "duration": 0.108779,
     "end_time": "2021-01-27T23:09:16.629172",
     "exception": false,
     "start_time": "2021-01-27T23:09:16.520393",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import keras\n",
    "from keras import backend as K\n",
    "from keras.engine import Layer\n",
    "import tensorflow_hub as hub\n",
    "import tensorflow as tf\n",
    "\n",
    "class ElmoEmbeddingLayer(Layer):\n",
    "    def __init__(self, **kwargs):\n",
    "        super(ElmoEmbeddingLayer, self).__init__(**kwargs)\n",
    "        self.dimensions = 1024\n",
    "        self.trainable=True\n",
    "\n",
    "    def build(self, input_shape):\n",
    "        self.elmo = elmo\n",
    "        super(ElmoEmbeddingLayer, self).build(input_shape)\n",
    "\n",
    "    def call(self, x, mask=None):\n",
    "        result = self.elmo.signatures['default'](tf.squeeze(tf.cast(x, tf.string), axis=1))['default']\n",
    "                    #  as_dict=True)\n",
    "        return result\n",
    "\n",
    "    def compute_output_shape(self, input_shape):\n",
    "        return (input_shape[0], self.dimensions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-01-27T23:09:16.703646Z",
     "iopub.status.busy": "2021-01-27T23:09:16.702837Z",
     "iopub.status.idle": "2021-01-27T23:10:01.986869Z",
     "shell.execute_reply": "2021-01-27T23:10:01.986373Z"
    },
    "papermill": {
     "duration": 45.322574,
     "end_time": "2021-01-27T23:10:01.986969",
     "exception": false,
     "start_time": "2021-01-27T23:09:16.664395",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "1100\n",
      "1200\n",
      "1300\n",
      "1400\n",
      "1500\n",
      "1600\n",
      "1700\n",
      "1800\n",
      "1900\n",
      "2000\n",
      "2100\n",
      "2200\n",
      "2300\n",
      "2400\n",
      "2500\n",
      "2600\n",
      "2700\n",
      "2800\n",
      "2900\n",
      "3000\n",
      "3100\n",
      "3200\n",
      "3300\n",
      "3400\n",
      "3500\n",
      "3600\n",
      "3700\n",
      "3800\n",
      "3900\n",
      "4000\n",
      "4100\n",
      "4200\n",
      "4300\n",
      "4400\n",
      "4500\n",
      "4600\n",
      "4700\n",
      "4800\n",
      "4900\n",
      "5000\n",
      "5100\n",
      "5200\n",
      "5300\n",
      "5400\n",
      "5500\n",
      "5600\n",
      "5700\n",
      "5800\n",
      "5900\n",
      "6000\n",
      "6100\n",
      "6200\n",
      "6300\n",
      "6400\n",
      "6500\n",
      "6600\n",
      "6700\n",
      "6800\n",
      "6900\n",
      "7000\n",
      "7100\n",
      "7200\n",
      "7300\n",
      "7400\n",
      "7500\n",
      "7600\n",
      "7700\n",
      "7800\n",
      "7900\n",
      "8000\n",
      "8100\n",
      "8200\n",
      "8300\n",
      "8400\n",
      "8500\n",
      "8600\n",
      "8700\n",
      "8800\n",
      "8900\n",
      "9000\n",
      "9100\n",
      "9200\n",
      "9300\n",
      "9400\n",
      "9500\n",
      "9600\n",
      "9700\n",
      "tf.Tensor(\n",
      "[[-0.17454977  0.04414361  0.12465099 ... -0.23744154  0.16813906\n",
      "  -0.03587628]\n",
      " [-0.17454977  0.04414361  0.12465099 ... -0.23744154  0.16813906\n",
      "  -0.03587628]\n",
      " [-0.17454977  0.04414361  0.12465099 ... -0.23744154  0.16813906\n",
      "  -0.03587628]\n",
      " ...\n",
      " [-0.21110037 -0.2673509  -0.05703427 ... -0.12372881  0.11923107\n",
      "   0.09973644]\n",
      " [-0.21110037 -0.2673509  -0.05703427 ... -0.12372881  0.11923107\n",
      "   0.09973644]\n",
      " [-0.21110037 -0.2673509  -0.05703427 ... -0.12372881  0.11923107\n",
      "   0.09973644]], shape=(9824, 1024), dtype=float32)\n",
      "0\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "1100\n",
      "1200\n",
      "1300\n",
      "1400\n",
      "1500\n",
      "1600\n",
      "1700\n",
      "1800\n",
      "1900\n",
      "2000\n",
      "2100\n",
      "2200\n",
      "2300\n",
      "2400\n",
      "2500\n",
      "2600\n",
      "2700\n",
      "2800\n",
      "2900\n",
      "3000\n",
      "3100\n",
      "3200\n",
      "3300\n",
      "3400\n",
      "3500\n",
      "3600\n",
      "3700\n",
      "3800\n",
      "3900\n",
      "4000\n",
      "4100\n",
      "4200\n",
      "4300\n",
      "4400\n",
      "4500\n",
      "4600\n",
      "4700\n",
      "4800\n",
      "4900\n",
      "5000\n",
      "5100\n",
      "5200\n",
      "5300\n",
      "5400\n",
      "5500\n",
      "5600\n",
      "5700\n",
      "5800\n",
      "5900\n",
      "6000\n",
      "6100\n",
      "6200\n",
      "6300\n",
      "6400\n",
      "6500\n",
      "6600\n",
      "6700\n",
      "6800\n",
      "6900\n",
      "7000\n",
      "7100\n",
      "7200\n",
      "7300\n",
      "7400\n",
      "7500\n",
      "7600\n",
      "7700\n",
      "7800\n",
      "7900\n",
      "8000\n",
      "8100\n",
      "8200\n",
      "8300\n",
      "8400\n",
      "8500\n",
      "8600\n",
      "8700\n",
      "8800\n",
      "8900\n",
      "9000\n",
      "9100\n",
      "9200\n",
      "9300\n",
      "9400\n",
      "9500\n",
      "9600\n",
      "9700\n",
      "tf.Tensor(\n",
      "[[-2.1524490e-01  4.3576318e-01 -2.5881174e-01 ... -4.1109510e-02\n",
      "   3.1052342e-01  1.0945505e-01]\n",
      " [-2.3627442e-01  2.2816630e-01 -2.0924484e-02 ... -3.3460283e-01\n",
      "   1.5864201e-01  1.4084952e-01]\n",
      " [-1.3826080e-01 -3.7522790e-01 -4.0751759e-02 ... -2.9288754e-01\n",
      "  -2.2028100e-01 -2.0341623e-01]\n",
      " ...\n",
      " [-1.5908624e-01 -1.0206465e-01 -1.0300703e-02 ... -1.9536532e-01\n",
      "   1.2823698e-01  1.5334642e-01]\n",
      " [-1.3842811e-01  1.3652274e-01  4.1444283e-02 ... -1.9144337e-01\n",
      "  -2.6506526e-02 -1.0367196e-01]\n",
      " [-4.0211946e-01 -2.7555227e-04 -2.9735301e-02 ... -1.9581757e-01\n",
      "   7.0776701e-02 -1.9876730e-01]], shape=(9824, 1024), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "embed_batch_size = 100\n",
    "\n",
    "def batch_embed(data, dataset):\n",
    "    full_embed = elmo.signatures['default'](tf.cast(data[dataset][0:embed_batch_size], tf.string))['default']\n",
    "\n",
    "    for i in range(len(data) // embed_batch_size):\n",
    "        print(i * embed_batch_size)\n",
    "        current = (i+1)*embed_batch_size\n",
    "        tensor = tf.cast(data[dataset][current:current + embed_batch_size], tf.string)\n",
    "        embed = elmo.signatures['default'](tensor)['default']\n",
    "        full_embed = tf.concat([full_embed, embed], axis=0)\n",
    "\n",
    "    print(full_embed)\n",
    "    \n",
    "    return full_embed\n",
    "\n",
    "test_premise_embed = batch_embed(test_df, 'premise')\n",
    "test_hypothesis_embed = batch_embed(test_df, 'hypothesis')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-01-27T23:10:02.606831Z",
     "iopub.status.busy": "2021-01-27T23:10:02.606008Z",
     "iopub.status.idle": "2021-01-27T23:27:52.899581Z",
     "shell.execute_reply": "2021-01-27T23:27:52.900136Z"
    },
    "papermill": {
     "duration": 1070.359312,
     "end_time": "2021-01-27T23:27:52.900299",
     "exception": false,
     "start_time": "2021-01-27T23:10:02.540987",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "1100\n",
      "1200\n",
      "1300\n",
      "1400\n",
      "1500\n",
      "1600\n",
      "1700\n",
      "1800\n",
      "1900\n",
      "2000\n",
      "2100\n",
      "2200\n",
      "2300\n",
      "2400\n",
      "2500\n",
      "2600\n",
      "2700\n",
      "2800\n",
      "2900\n",
      "3000\n",
      "3100\n",
      "3200\n",
      "3300\n",
      "3400\n",
      "3500\n",
      "3600\n",
      "3700\n",
      "3800\n",
      "3900\n",
      "4000\n",
      "4100\n",
      "4200\n",
      "4300\n",
      "4400\n",
      "4500\n",
      "4600\n",
      "4700\n",
      "4800\n",
      "4900\n",
      "5000\n",
      "5100\n",
      "5200\n",
      "5300\n",
      "5400\n",
      "5500\n",
      "5600\n",
      "5700\n",
      "5800\n",
      "5900\n",
      "6000\n",
      "6100\n",
      "6200\n",
      "6300\n",
      "6400\n",
      "6500\n",
      "6600\n",
      "6700\n",
      "6800\n",
      "6900\n",
      "7000\n",
      "7100\n",
      "7200\n",
      "7300\n",
      "7400\n",
      "7500\n",
      "7600\n",
      "7700\n",
      "7800\n",
      "7900\n",
      "8000\n",
      "8100\n",
      "8200\n",
      "8300\n",
      "8400\n",
      "8500\n",
      "8600\n",
      "8700\n",
      "8800\n",
      "8900\n",
      "9000\n",
      "9100\n",
      "9200\n",
      "9300\n",
      "9400\n",
      "9500\n",
      "9600\n",
      "9700\n",
      "9800\n",
      "9900\n",
      "10000\n",
      "10100\n",
      "10200\n",
      "10300\n",
      "10400\n",
      "10500\n",
      "10600\n",
      "10700\n",
      "10800\n",
      "10900\n",
      "11000\n",
      "11100\n",
      "11200\n",
      "11300\n",
      "11400\n",
      "11500\n",
      "11600\n",
      "11700\n",
      "11800\n",
      "11900\n",
      "12000\n",
      "12100\n",
      "12200\n",
      "12300\n",
      "12400\n",
      "12500\n",
      "12600\n",
      "12700\n",
      "12800\n",
      "12900\n",
      "13000\n",
      "13100\n",
      "13200\n",
      "13300\n",
      "13400\n",
      "13500\n",
      "13600\n",
      "13700\n",
      "13800\n",
      "13900\n",
      "14000\n",
      "14100\n",
      "14200\n",
      "14300\n",
      "14400\n",
      "14500\n",
      "14600\n",
      "14700\n",
      "14800\n",
      "14900\n",
      "15000\n",
      "15100\n",
      "15200\n",
      "15300\n",
      "15400\n",
      "15500\n",
      "15600\n",
      "15700\n",
      "15800\n",
      "15900\n",
      "16000\n",
      "16100\n",
      "16200\n",
      "16300\n",
      "16400\n",
      "16500\n",
      "16600\n",
      "16700\n",
      "16800\n",
      "16900\n",
      "17000\n",
      "17100\n",
      "17200\n",
      "17300\n",
      "17400\n",
      "17500\n",
      "17600\n",
      "17700\n",
      "17800\n",
      "17900\n",
      "18000\n",
      "18100\n",
      "18200\n",
      "18300\n",
      "18400\n",
      "18500\n",
      "18600\n",
      "18700\n",
      "18800\n",
      "18900\n",
      "19000\n",
      "19100\n",
      "19200\n",
      "19300\n",
      "19400\n",
      "19500\n",
      "19600\n",
      "19700\n",
      "19800\n",
      "19900\n",
      "20000\n",
      "20100\n",
      "20200\n",
      "20300\n",
      "20400\n",
      "20500\n",
      "20600\n",
      "20700\n",
      "20800\n",
      "20900\n",
      "21000\n",
      "21100\n",
      "21200\n",
      "21300\n",
      "21400\n",
      "21500\n",
      "21600\n",
      "21700\n",
      "21800\n",
      "21900\n",
      "22000\n",
      "22100\n",
      "22200\n",
      "22300\n",
      "22400\n",
      "22500\n",
      "22600\n",
      "22700\n",
      "22800\n",
      "22900\n",
      "23000\n",
      "23100\n",
      "23200\n",
      "23300\n",
      "23400\n",
      "23500\n",
      "23600\n",
      "23700\n",
      "23800\n",
      "23900\n",
      "24000\n",
      "24100\n",
      "24200\n",
      "24300\n",
      "24400\n",
      "24500\n",
      "24600\n",
      "24700\n",
      "24800\n",
      "24900\n",
      "25000\n",
      "25100\n",
      "25200\n",
      "25300\n",
      "25400\n",
      "25500\n",
      "25600\n",
      "25700\n",
      "25800\n",
      "25900\n",
      "26000\n",
      "26100\n",
      "26200\n",
      "26300\n",
      "26400\n",
      "26500\n",
      "26600\n",
      "26700\n",
      "26800\n",
      "26900\n",
      "27000\n",
      "27100\n",
      "27200\n",
      "27300\n",
      "27400\n",
      "27500\n",
      "27600\n",
      "27700\n",
      "27800\n",
      "27900\n",
      "28000\n",
      "28100\n",
      "28200\n",
      "28300\n",
      "28400\n",
      "28500\n",
      "28600\n",
      "28700\n",
      "28800\n",
      "28900\n",
      "29000\n",
      "29100\n",
      "29200\n",
      "29300\n",
      "29400\n",
      "29500\n",
      "29600\n",
      "29700\n",
      "29800\n",
      "29900\n",
      "30000\n",
      "30100\n",
      "30200\n",
      "30300\n",
      "30400\n",
      "30500\n",
      "30600\n",
      "30700\n",
      "30800\n",
      "30900\n",
      "31000\n",
      "31100\n",
      "31200\n",
      "31300\n",
      "31400\n",
      "31500\n",
      "31600\n",
      "31700\n",
      "31800\n",
      "31900\n",
      "32000\n",
      "32100\n",
      "32200\n",
      "32300\n",
      "32400\n",
      "32500\n",
      "32600\n",
      "32700\n",
      "32800\n",
      "32900\n",
      "33000\n",
      "33100\n",
      "33200\n",
      "33300\n",
      "33400\n",
      "33500\n",
      "33600\n",
      "33700\n",
      "33800\n",
      "33900\n",
      "34000\n",
      "34100\n",
      "34200\n",
      "34300\n",
      "34400\n",
      "34500\n",
      "34600\n",
      "34700\n",
      "34800\n",
      "34900\n",
      "35000\n",
      "35100\n",
      "35200\n",
      "35300\n",
      "35400\n",
      "35500\n",
      "35600\n",
      "35700\n",
      "35800\n",
      "35900\n",
      "36000\n",
      "36100\n",
      "36200\n",
      "36300\n",
      "36400\n",
      "36500\n",
      "36600\n",
      "36700\n",
      "36800\n",
      "36900\n",
      "37000\n",
      "37100\n",
      "37200\n",
      "37300\n",
      "37400\n",
      "37500\n",
      "37600\n",
      "37700\n",
      "37800\n",
      "37900\n",
      "38000\n",
      "38100\n",
      "38200\n",
      "38300\n",
      "38400\n",
      "38500\n",
      "38600\n",
      "38700\n",
      "38800\n",
      "38900\n",
      "39000\n",
      "39100\n",
      "39200\n",
      "39300\n",
      "39400\n",
      "39500\n",
      "39600\n",
      "39700\n",
      "39800\n",
      "39900\n",
      "40000\n",
      "40100\n",
      "40200\n",
      "40300\n",
      "40400\n",
      "40500\n",
      "40600\n",
      "40700\n",
      "40800\n",
      "40900\n",
      "41000\n",
      "41100\n",
      "41200\n",
      "41300\n",
      "41400\n",
      "41500\n",
      "41600\n",
      "41700\n",
      "41800\n",
      "41900\n",
      "42000\n",
      "42100\n",
      "42200\n",
      "42300\n",
      "42400\n",
      "42500\n",
      "42600\n",
      "42700\n",
      "42800\n",
      "42900\n",
      "43000\n",
      "43100\n",
      "43200\n",
      "43300\n",
      "43400\n",
      "43500\n",
      "43600\n",
      "43700\n",
      "43800\n",
      "43900\n",
      "44000\n",
      "44100\n",
      "44200\n",
      "44300\n",
      "44400\n",
      "44500\n",
      "44600\n",
      "44700\n",
      "44800\n",
      "44900\n",
      "45000\n",
      "45100\n",
      "45200\n",
      "45300\n",
      "45400\n",
      "45500\n",
      "45600\n",
      "45700\n",
      "45800\n",
      "45900\n",
      "46000\n",
      "46100\n",
      "46200\n",
      "46300\n",
      "46400\n",
      "46500\n",
      "46600\n",
      "46700\n",
      "46800\n",
      "46900\n",
      "47000\n",
      "47100\n",
      "47200\n",
      "47300\n",
      "47400\n",
      "47500\n",
      "47600\n",
      "47700\n",
      "47800\n",
      "47900\n",
      "48000\n",
      "48100\n",
      "48200\n",
      "48300\n",
      "48400\n",
      "48500\n",
      "48600\n",
      "48700\n",
      "48800\n",
      "48900\n",
      "49000\n",
      "49100\n",
      "49200\n",
      "49300\n",
      "49400\n",
      "49500\n",
      "49600\n",
      "49700\n",
      "49800\n",
      "49900\n",
      "50000\n",
      "50100\n",
      "50200\n",
      "50300\n",
      "50400\n",
      "50500\n",
      "50600\n",
      "50700\n",
      "50800\n",
      "50900\n",
      "51000\n",
      "51100\n",
      "51200\n",
      "51300\n",
      "51400\n",
      "51500\n",
      "51600\n",
      "51700\n",
      "51800\n",
      "51900\n",
      "52000\n",
      "52100\n",
      "52200\n",
      "52300\n",
      "52400\n",
      "52500\n",
      "52600\n",
      "52700\n",
      "52800\n",
      "52900\n",
      "53000\n",
      "53100\n",
      "53200\n",
      "53300\n",
      "53400\n",
      "53500\n",
      "53600\n",
      "53700\n",
      "53800\n",
      "53900\n",
      "54000\n",
      "54100\n",
      "54200\n",
      "54300\n",
      "54400\n",
      "54500\n",
      "54600\n",
      "54700\n",
      "54800\n",
      "54900\n",
      "55000\n",
      "55100\n",
      "55200\n",
      "55300\n",
      "55400\n",
      "55500\n",
      "55600\n",
      "55700\n",
      "55800\n",
      "55900\n",
      "56000\n",
      "56100\n",
      "56200\n",
      "56300\n",
      "56400\n",
      "56500\n",
      "56600\n",
      "56700\n",
      "56800\n",
      "56900\n",
      "57000\n",
      "57100\n",
      "57200\n",
      "57300\n",
      "57400\n",
      "57500\n",
      "57600\n",
      "57700\n",
      "57800\n",
      "57900\n",
      "58000\n",
      "58100\n",
      "58200\n",
      "58300\n",
      "58400\n",
      "58500\n",
      "58600\n",
      "58700\n",
      "58800\n",
      "58900\n",
      "59000\n",
      "59100\n",
      "59200\n",
      "59300\n",
      "59400\n",
      "59500\n",
      "59600\n",
      "59700\n",
      "59800\n",
      "59900\n",
      "60000\n",
      "60100\n",
      "60200\n",
      "60300\n",
      "60400\n",
      "60500\n",
      "60600\n",
      "60700\n",
      "60800\n",
      "60900\n",
      "61000\n",
      "61100\n",
      "61200\n",
      "61300\n",
      "61400\n",
      "61500\n",
      "61600\n",
      "61700\n",
      "61800\n",
      "61900\n",
      "62000\n",
      "62100\n",
      "62200\n",
      "62300\n",
      "62400\n",
      "62500\n",
      "62600\n",
      "62700\n",
      "62800\n",
      "62900\n",
      "63000\n",
      "63100\n",
      "63200\n",
      "63300\n",
      "63400\n",
      "63500\n",
      "63600\n",
      "63700\n",
      "63800\n",
      "63900\n",
      "64000\n",
      "64100\n",
      "64200\n",
      "64300\n",
      "64400\n",
      "64500\n",
      "64600\n",
      "64700\n",
      "64800\n",
      "64900\n",
      "65000\n",
      "65100\n",
      "65200\n",
      "65300\n",
      "65400\n",
      "65500\n",
      "65600\n",
      "65700\n",
      "65800\n",
      "65900\n",
      "66000\n",
      "66100\n",
      "66200\n",
      "66300\n",
      "66400\n",
      "66500\n",
      "66600\n",
      "66700\n",
      "66800\n",
      "66900\n",
      "67000\n",
      "67100\n",
      "67200\n",
      "67300\n",
      "67400\n",
      "67500\n",
      "67600\n",
      "67700\n",
      "67800\n",
      "67900\n",
      "68000\n",
      "68100\n",
      "68200\n",
      "68300\n",
      "68400\n",
      "68500\n",
      "68600\n",
      "68700\n",
      "68800\n",
      "68900\n",
      "69000\n",
      "69100\n",
      "69200\n",
      "69300\n",
      "69400\n",
      "69500\n",
      "69600\n",
      "69700\n",
      "69800\n",
      "69900\n",
      "70000\n",
      "70100\n",
      "70200\n",
      "70300\n",
      "70400\n",
      "70500\n",
      "70600\n",
      "70700\n",
      "70800\n",
      "70900\n",
      "71000\n",
      "71100\n",
      "71200\n",
      "71300\n",
      "71400\n",
      "71500\n",
      "71600\n",
      "71700\n",
      "71800\n",
      "71900\n",
      "72000\n",
      "72100\n",
      "72200\n",
      "72300\n",
      "72400\n",
      "72500\n",
      "72600\n",
      "72700\n",
      "72800\n",
      "72900\n",
      "73000\n",
      "73100\n",
      "73200\n",
      "73300\n",
      "73400\n",
      "73500\n",
      "73600\n",
      "73700\n",
      "73800\n",
      "73900\n",
      "74000\n",
      "74100\n",
      "74200\n",
      "74300\n",
      "74400\n",
      "74500\n",
      "74600\n",
      "74700\n",
      "74800\n",
      "74900\n",
      "75000\n",
      "75100\n",
      "75200\n",
      "75300\n",
      "75400\n",
      "75500\n",
      "75600\n",
      "75700\n",
      "75800\n",
      "75900\n",
      "76000\n",
      "76100\n",
      "76200\n",
      "76300\n",
      "76400\n",
      "76500\n",
      "76600\n",
      "76700\n",
      "76800\n",
      "76900\n",
      "77000\n",
      "77100\n",
      "77200\n",
      "77300\n",
      "77400\n",
      "77500\n",
      "77600\n",
      "77700\n",
      "77800\n",
      "77900\n",
      "78000\n",
      "78100\n",
      "78200\n",
      "78300\n",
      "78400\n",
      "78500\n",
      "78600\n",
      "78700\n",
      "78800\n",
      "78900\n",
      "79000\n",
      "79100\n",
      "79200\n",
      "79300\n",
      "79400\n",
      "79500\n",
      "79600\n",
      "79700\n",
      "79800\n",
      "79900\n",
      "80000\n",
      "80100\n",
      "80200\n",
      "80300\n",
      "80400\n",
      "80500\n",
      "80600\n",
      "80700\n",
      "80800\n",
      "80900\n",
      "81000\n",
      "81100\n",
      "81200\n",
      "81300\n",
      "81400\n",
      "81500\n",
      "81600\n",
      "81700\n",
      "81800\n",
      "81900\n",
      "82000\n",
      "82100\n",
      "82200\n",
      "82300\n",
      "82400\n",
      "82500\n",
      "82600\n",
      "82700\n",
      "82800\n",
      "82900\n",
      "83000\n",
      "83100\n",
      "83200\n",
      "83300\n",
      "83400\n",
      "83500\n",
      "83600\n",
      "83700\n",
      "83800\n",
      "83900\n",
      "84000\n",
      "84100\n",
      "84200\n",
      "84300\n",
      "84400\n",
      "84500\n",
      "84600\n",
      "84700\n",
      "84800\n",
      "84900\n",
      "85000\n",
      "85100\n",
      "85200\n",
      "85300\n",
      "85400\n",
      "85500\n",
      "85600\n",
      "85700\n",
      "85800\n",
      "85900\n",
      "86000\n",
      "86100\n",
      "86200\n",
      "86300\n",
      "86400\n",
      "86500\n",
      "86600\n",
      "86700\n",
      "86800\n",
      "86900\n",
      "87000\n",
      "87100\n",
      "87200\n",
      "87300\n",
      "87400\n",
      "87500\n",
      "87600\n",
      "87700\n",
      "87800\n",
      "87900\n",
      "88000\n",
      "88100\n",
      "88200\n",
      "88300\n",
      "88400\n",
      "88500\n",
      "88600\n",
      "88700\n",
      "88800\n",
      "88900\n",
      "89000\n",
      "89100\n",
      "89200\n",
      "89300\n",
      "89400\n",
      "89500\n",
      "89600\n",
      "89700\n",
      "89800\n",
      "89900\n",
      "90000\n",
      "90100\n",
      "90200\n",
      "90300\n",
      "90400\n",
      "90500\n",
      "90600\n",
      "90700\n",
      "90800\n",
      "90900\n",
      "91000\n",
      "91100\n",
      "91200\n",
      "91300\n",
      "91400\n",
      "91500\n",
      "91600\n",
      "91700\n",
      "91800\n",
      "91900\n",
      "92000\n",
      "92100\n",
      "92200\n",
      "92300\n",
      "92400\n",
      "92500\n",
      "92600\n",
      "92700\n",
      "92800\n",
      "92900\n",
      "93000\n",
      "93100\n",
      "93200\n",
      "93300\n",
      "93400\n",
      "93500\n",
      "93600\n",
      "93700\n",
      "93800\n",
      "93900\n",
      "94000\n",
      "94100\n",
      "94200\n",
      "94300\n",
      "94400\n",
      "94500\n",
      "94600\n",
      "94700\n",
      "94800\n",
      "94900\n",
      "95000\n",
      "95100\n",
      "95200\n",
      "95300\n",
      "95400\n",
      "95500\n",
      "95600\n",
      "95700\n",
      "95800\n",
      "95900\n",
      "96000\n",
      "96100\n",
      "96200\n",
      "96300\n",
      "96400\n",
      "96500\n",
      "96600\n",
      "96700\n",
      "96800\n",
      "96900\n",
      "97000\n",
      "97100\n",
      "97200\n",
      "97300\n",
      "97400\n",
      "97500\n",
      "97600\n",
      "97700\n",
      "97800\n",
      "97900\n",
      "98000\n",
      "98100\n",
      "98200\n",
      "98300\n",
      "98400\n",
      "98500\n",
      "98600\n",
      "98700\n",
      "98800\n",
      "98900\n",
      "99000\n",
      "99100\n",
      "99200\n",
      "99300\n",
      "99400\n",
      "99500\n",
      "99600\n",
      "99700\n",
      "99800\n",
      "99900\n",
      "100000\n",
      "100100\n",
      "100200\n",
      "100300\n",
      "100400\n",
      "100500\n",
      "100600\n",
      "100700\n",
      "100800\n",
      "100900\n",
      "101000\n",
      "101100\n",
      "101200\n",
      "101300\n",
      "101400\n",
      "101500\n",
      "101600\n",
      "101700\n",
      "101800\n",
      "101900\n",
      "102000\n",
      "102100\n",
      "102200\n",
      "102300\n",
      "102400\n",
      "102500\n",
      "102600\n",
      "102700\n",
      "102800\n",
      "102900\n",
      "103000\n",
      "103100\n",
      "103200\n",
      "103300\n",
      "103400\n",
      "103500\n",
      "103600\n",
      "103700\n",
      "103800\n",
      "103900\n",
      "104000\n",
      "104100\n",
      "104200\n",
      "104300\n",
      "104400\n",
      "104500\n",
      "104600\n",
      "104700\n",
      "104800\n",
      "104900\n",
      "105000\n",
      "105100\n",
      "105200\n",
      "105300\n",
      "105400\n",
      "105500\n",
      "105600\n",
      "105700\n",
      "105800\n",
      "105900\n",
      "106000\n",
      "106100\n",
      "106200\n",
      "106300\n",
      "106400\n",
      "106500\n",
      "106600\n",
      "106700\n",
      "106800\n",
      "106900\n",
      "107000\n",
      "107100\n",
      "107200\n",
      "107300\n",
      "107400\n",
      "107500\n",
      "107600\n",
      "107700\n",
      "107800\n",
      "107900\n",
      "108000\n",
      "108100\n",
      "108200\n",
      "108300\n",
      "108400\n",
      "108500\n",
      "108600\n",
      "108700\n",
      "108800\n",
      "108900\n",
      "109000\n",
      "109100\n",
      "109200\n",
      "109300\n",
      "109400\n",
      "109500\n",
      "109600\n",
      "109700\n",
      "109800\n",
      "109900\n",
      "110000\n",
      "110100\n",
      "110200\n",
      "110300\n",
      "110400\n",
      "110500\n",
      "110600\n",
      "110700\n",
      "110800\n",
      "110900\n",
      "111000\n",
      "111100\n",
      "111200\n",
      "111300\n",
      "111400\n",
      "111500\n",
      "111600\n",
      "111700\n",
      "111800\n",
      "111900\n",
      "112000\n",
      "112100\n",
      "112200\n",
      "112300\n",
      "112400\n",
      "112500\n",
      "112600\n",
      "112700\n",
      "112800\n",
      "112900\n",
      "113000\n",
      "113100\n",
      "113200\n",
      "113300\n",
      "113400\n",
      "113500\n",
      "113600\n",
      "113700\n",
      "113800\n",
      "113900\n",
      "114000\n",
      "114100\n",
      "114200\n",
      "114300\n",
      "114400\n",
      "114500\n",
      "114600\n",
      "114700\n",
      "114800\n",
      "114900\n",
      "115000\n",
      "115100\n",
      "115200\n",
      "115300\n",
      "115400\n",
      "115500\n",
      "115600\n",
      "115700\n",
      "115800\n",
      "115900\n",
      "116000\n",
      "116100\n",
      "116200\n",
      "116300\n",
      "116400\n",
      "116500\n",
      "116600\n",
      "116700\n",
      "116800\n",
      "116900\n",
      "117000\n",
      "117100\n",
      "117200\n",
      "117300\n",
      "117400\n",
      "117500\n",
      "117600\n",
      "117700\n",
      "117800\n",
      "117900\n",
      "118000\n",
      "118100\n",
      "118200\n",
      "118300\n",
      "118400\n",
      "118500\n",
      "118600\n",
      "118700\n",
      "118800\n",
      "118900\n",
      "119000\n",
      "119100\n",
      "119200\n",
      "119300\n",
      "119400\n",
      "119500\n",
      "119600\n",
      "119700\n",
      "119800\n",
      "119900\n",
      "120000\n",
      "120100\n",
      "120200\n",
      "120300\n",
      "120400\n",
      "120500\n",
      "120600\n",
      "120700\n",
      "120800\n",
      "120900\n",
      "121000\n",
      "121100\n",
      "121200\n",
      "121300\n",
      "121400\n",
      "121500\n",
      "121600\n",
      "121700\n",
      "121800\n",
      "121900\n",
      "122000\n",
      "122100\n",
      "122200\n",
      "122300\n",
      "122400\n",
      "122500\n",
      "122600\n",
      "122700\n",
      "122800\n",
      "122900\n",
      "123000\n",
      "123100\n",
      "123200\n",
      "123300\n",
      "123400\n",
      "123500\n",
      "123600\n",
      "123700\n",
      "123800\n",
      "123900\n",
      "124000\n",
      "124100\n",
      "124200\n",
      "124300\n",
      "124400\n",
      "124500\n",
      "124600\n",
      "124700\n",
      "124800\n",
      "124900\n",
      "125000\n",
      "125100\n",
      "125200\n",
      "125300\n",
      "125400\n",
      "125500\n",
      "125600\n",
      "125700\n",
      "125800\n",
      "125900\n",
      "126000\n",
      "126100\n",
      "126200\n",
      "126300\n",
      "126400\n",
      "126500\n",
      "126600\n",
      "126700\n",
      "126800\n",
      "126900\n",
      "127000\n",
      "127100\n",
      "127200\n",
      "127300\n",
      "127400\n",
      "127500\n",
      "127600\n",
      "127700\n",
      "127800\n",
      "127900\n",
      "128000\n",
      "128100\n",
      "128200\n",
      "128300\n",
      "128400\n",
      "128500\n",
      "128600\n",
      "128700\n",
      "128800\n",
      "128900\n",
      "129000\n",
      "129100\n",
      "129200\n",
      "129300\n",
      "129400\n",
      "129500\n",
      "129600\n",
      "129700\n",
      "129800\n",
      "129900\n",
      "130000\n",
      "130100\n",
      "130200\n",
      "130300\n",
      "130400\n",
      "130500\n",
      "130600\n",
      "130700\n",
      "130800\n",
      "130900\n",
      "131000\n",
      "131100\n",
      "131200\n",
      "131300\n",
      "131400\n",
      "131500\n",
      "131600\n",
      "131700\n",
      "131800\n",
      "131900\n",
      "132000\n",
      "132100\n",
      "132200\n",
      "132300\n",
      "132400\n",
      "132500\n",
      "132600\n",
      "132700\n",
      "132800\n",
      "132900\n",
      "133000\n",
      "133100\n",
      "133200\n",
      "133300\n",
      "133400\n",
      "133500\n",
      "133600\n",
      "133700\n",
      "133800\n",
      "133900\n",
      "134000\n",
      "134100\n",
      "134200\n",
      "134300\n",
      "134400\n",
      "134500\n",
      "134600\n",
      "134700\n",
      "134800\n",
      "134900\n",
      "135000\n",
      "135100\n",
      "135200\n",
      "135300\n",
      "135400\n",
      "135500\n",
      "135600\n",
      "135700\n",
      "135800\n",
      "135900\n",
      "136000\n",
      "136100\n",
      "136200\n",
      "136300\n",
      "136400\n",
      "136500\n",
      "136600\n",
      "136700\n",
      "136800\n",
      "136900\n",
      "137000\n",
      "137100\n",
      "137200\n",
      "137300\n",
      "137400\n",
      "137500\n",
      "137600\n",
      "137700\n",
      "137800\n",
      "137900\n",
      "138000\n",
      "138100\n",
      "138200\n",
      "138300\n",
      "138400\n",
      "138500\n",
      "138600\n",
      "138700\n",
      "138800\n",
      "138900\n",
      "139000\n",
      "139100\n",
      "139200\n",
      "139300\n",
      "139400\n",
      "139500\n",
      "139600\n",
      "139700\n",
      "139800\n",
      "139900\n",
      "140000\n",
      "140100\n",
      "140200\n",
      "140300\n",
      "140400\n",
      "140500\n",
      "140600\n",
      "140700\n",
      "140800\n",
      "140900\n",
      "141000\n",
      "141100\n",
      "141200\n",
      "141300\n",
      "141400\n",
      "141500\n",
      "141600\n",
      "141700\n",
      "141800\n",
      "141900\n",
      "142000\n",
      "142100\n",
      "142200\n",
      "142300\n",
      "142400\n",
      "142500\n",
      "142600\n",
      "142700\n",
      "142800\n",
      "142900\n",
      "143000\n",
      "143100\n",
      "143200\n",
      "143300\n",
      "143400\n",
      "143500\n",
      "143600\n",
      "143700\n",
      "143800\n",
      "143900\n",
      "144000\n",
      "144100\n",
      "144200\n",
      "144300\n",
      "144400\n",
      "144500\n",
      "144600\n",
      "144700\n",
      "144800\n",
      "144900\n",
      "145000\n",
      "145100\n",
      "145200\n",
      "145300\n",
      "145400\n",
      "145500\n",
      "145600\n",
      "145700\n",
      "145800\n",
      "145900\n",
      "146000\n",
      "146100\n",
      "146200\n",
      "146300\n",
      "146400\n",
      "146500\n",
      "146600\n",
      "146700\n",
      "146800\n",
      "146900\n",
      "147000\n",
      "147100\n",
      "147200\n",
      "147300\n",
      "147400\n",
      "147500\n",
      "147600\n",
      "147700\n",
      "147800\n",
      "147900\n",
      "148000\n",
      "148100\n",
      "148200\n",
      "148300\n",
      "148400\n",
      "148500\n",
      "148600\n",
      "148700\n",
      "148800\n",
      "148900\n",
      "149000\n",
      "149100\n",
      "149200\n",
      "149300\n",
      "149400\n",
      "149500\n",
      "149600\n",
      "149700\n",
      "149800\n",
      "149900\n",
      "150000\n",
      "150100\n",
      "150200\n",
      "150300\n",
      "150400\n",
      "150500\n",
      "150600\n",
      "150700\n",
      "150800\n",
      "150900\n",
      "151000\n",
      "151100\n",
      "151200\n",
      "151300\n",
      "151400\n",
      "151500\n",
      "151600\n",
      "151700\n",
      "151800\n",
      "151900\n",
      "152000\n",
      "152100\n",
      "152200\n",
      "152300\n",
      "152400\n",
      "152500\n",
      "152600\n",
      "152700\n",
      "152800\n",
      "152900\n",
      "153000\n",
      "153100\n",
      "153200\n",
      "153300\n",
      "153400\n",
      "153500\n",
      "153600\n",
      "153700\n",
      "153800\n",
      "153900\n",
      "154000\n",
      "154100\n",
      "154200\n",
      "154300\n",
      "154400\n",
      "154500\n",
      "154600\n",
      "154700\n",
      "154800\n",
      "154900\n",
      "155000\n",
      "155100\n",
      "155200\n",
      "155300\n",
      "155400\n",
      "155500\n",
      "155600\n",
      "155700\n",
      "155800\n",
      "155900\n",
      "156000\n",
      "156100\n",
      "156200\n",
      "156300\n",
      "156400\n",
      "156500\n",
      "156600\n",
      "156700\n",
      "156800\n",
      "156900\n",
      "157000\n",
      "157100\n",
      "157200\n",
      "157300\n",
      "157400\n",
      "157500\n",
      "157600\n",
      "157700\n",
      "157800\n",
      "157900\n",
      "158000\n",
      "158100\n",
      "158200\n",
      "158300\n",
      "158400\n",
      "158500\n",
      "158600\n",
      "158700\n",
      "158800\n",
      "158900\n",
      "159000\n",
      "159100\n",
      "159200\n",
      "159300\n",
      "159400\n",
      "159500\n",
      "159600\n",
      "159700\n",
      "159800\n",
      "159900\n",
      "160000\n",
      "160100\n",
      "160200\n",
      "160300\n",
      "160400\n",
      "160500\n",
      "160600\n",
      "160700\n",
      "160800\n",
      "160900\n",
      "161000\n",
      "161100\n",
      "161200\n",
      "161300\n",
      "161400\n",
      "161500\n",
      "161600\n",
      "161700\n",
      "161800\n",
      "161900\n",
      "162000\n",
      "162100\n",
      "162200\n",
      "162300\n",
      "162400\n",
      "162500\n",
      "162600\n",
      "162700\n",
      "162800\n",
      "162900\n",
      "163000\n",
      "163100\n",
      "163200\n",
      "163300\n",
      "163400\n",
      "163500\n",
      "163600\n",
      "163700\n",
      "163800\n",
      "163900\n",
      "164000\n",
      "164100\n",
      "164200\n",
      "164300\n",
      "164400\n",
      "164500\n",
      "164600\n",
      "164700\n",
      "164800\n",
      "164900\n",
      "165000\n",
      "165100\n",
      "165200\n",
      "165300\n",
      "165400\n",
      "165500\n",
      "165600\n",
      "165700\n",
      "165800\n",
      "165900\n",
      "166000\n",
      "166100\n",
      "166200\n",
      "166300\n",
      "166400\n",
      "166500\n",
      "166600\n",
      "166700\n",
      "166800\n",
      "166900\n",
      "167000\n",
      "167100\n",
      "167200\n",
      "167300\n",
      "167400\n",
      "167500\n",
      "167600\n",
      "167700\n",
      "167800\n",
      "167900\n",
      "168000\n",
      "168100\n",
      "168200\n",
      "168300\n",
      "168400\n",
      "168500\n",
      "168600\n",
      "168700\n",
      "168800\n",
      "168900\n",
      "169000\n",
      "169100\n",
      "169200\n",
      "169300\n",
      "169400\n",
      "169500\n",
      "169600\n",
      "169700\n",
      "169800\n",
      "169900\n",
      "170000\n",
      "170100\n",
      "170200\n",
      "170300\n",
      "170400\n",
      "170500\n",
      "170600\n",
      "170700\n",
      "170800\n",
      "170900\n",
      "171000\n",
      "171100\n",
      "171200\n",
      "171300\n",
      "171400\n",
      "171500\n",
      "171600\n",
      "171700\n",
      "171800\n",
      "171900\n",
      "172000\n",
      "172100\n",
      "172200\n",
      "172300\n",
      "172400\n",
      "172500\n",
      "172600\n",
      "172700\n",
      "172800\n",
      "172900\n",
      "173000\n",
      "173100\n",
      "173200\n",
      "173300\n",
      "173400\n",
      "173500\n",
      "173600\n",
      "173700\n",
      "173800\n",
      "173900\n",
      "174000\n",
      "174100\n",
      "174200\n",
      "174300\n",
      "174400\n",
      "174500\n",
      "174600\n",
      "174700\n",
      "174800\n",
      "174900\n",
      "175000\n",
      "175100\n",
      "175200\n",
      "175300\n",
      "175400\n",
      "175500\n",
      "175600\n",
      "175700\n",
      "175800\n",
      "175900\n",
      "176000\n",
      "176100\n",
      "176200\n",
      "176300\n",
      "176400\n",
      "176500\n",
      "176600\n",
      "176700\n",
      "176800\n",
      "176900\n",
      "177000\n",
      "177100\n",
      "177200\n",
      "177300\n",
      "177400\n",
      "177500\n",
      "177600\n",
      "177700\n",
      "177800\n",
      "177900\n",
      "178000\n",
      "178100\n",
      "178200\n",
      "178300\n",
      "178400\n",
      "178500\n",
      "178600\n",
      "178700\n",
      "178800\n",
      "178900\n",
      "179000\n",
      "179100\n",
      "179200\n",
      "179300\n",
      "179400\n",
      "179500\n",
      "179600\n",
      "179700\n",
      "179800\n",
      "179900\n",
      "180000\n",
      "180100\n",
      "180200\n",
      "180300\n",
      "180400\n",
      "180500\n",
      "180600\n",
      "180700\n",
      "180800\n",
      "180900\n",
      "181000\n",
      "181100\n",
      "181200\n",
      "181300\n",
      "181400\n",
      "181500\n",
      "181600\n",
      "181700\n",
      "181800\n",
      "181900\n",
      "182000\n",
      "182100\n",
      "182200\n",
      "182300\n",
      "182400\n",
      "182500\n",
      "182600\n",
      "182700\n",
      "182800\n",
      "182900\n",
      "183000\n",
      "183100\n",
      "183200\n",
      "183300\n",
      "183400\n",
      "183500\n",
      "183600\n",
      "183700\n",
      "183800\n",
      "183900\n",
      "184000\n",
      "184100\n",
      "184200\n",
      "184300\n",
      "184400\n",
      "184500\n",
      "184600\n",
      "184700\n",
      "184800\n",
      "184900\n",
      "185000\n",
      "185100\n",
      "185200\n",
      "185300\n",
      "185400\n",
      "185500\n",
      "185600\n",
      "185700\n",
      "185800\n",
      "185900\n",
      "186000\n",
      "186100\n",
      "186200\n",
      "186300\n",
      "186400\n",
      "186500\n",
      "186600\n",
      "186700\n",
      "186800\n",
      "186900\n",
      "187000\n",
      "187100\n",
      "187200\n",
      "187300\n",
      "187400\n",
      "187500\n",
      "187600\n",
      "187700\n",
      "187800\n",
      "187900\n",
      "188000\n",
      "188100\n",
      "188200\n",
      "188300\n",
      "188400\n",
      "188500\n",
      "188600\n",
      "188700\n",
      "188800\n",
      "188900\n",
      "189000\n",
      "189100\n",
      "189200\n",
      "189300\n",
      "189400\n",
      "189500\n",
      "189600\n",
      "189700\n",
      "189800\n",
      "189900\n",
      "190000\n",
      "190100\n",
      "190200\n",
      "190300\n",
      "190400\n",
      "190500\n",
      "190600\n",
      "190700\n",
      "190800\n",
      "190900\n",
      "191000\n",
      "191100\n",
      "191200\n",
      "191300\n",
      "191400\n",
      "191500\n",
      "191600\n",
      "191700\n",
      "191800\n",
      "191900\n",
      "192000\n",
      "192100\n",
      "192200\n",
      "192300\n",
      "192400\n",
      "192500\n",
      "192600\n",
      "192700\n",
      "192800\n",
      "192900\n",
      "193000\n",
      "193100\n",
      "193200\n",
      "193300\n",
      "193400\n",
      "193500\n",
      "193600\n",
      "193700\n",
      "193800\n",
      "193900\n",
      "194000\n",
      "194100\n",
      "194200\n",
      "194300\n",
      "194400\n",
      "194500\n",
      "194600\n",
      "194700\n",
      "194800\n",
      "194900\n",
      "195000\n",
      "195100\n",
      "195200\n",
      "195300\n",
      "195400\n",
      "195500\n",
      "195600\n",
      "195700\n",
      "195800\n",
      "195900\n",
      "196000\n",
      "196100\n",
      "196200\n",
      "196300\n",
      "196400\n",
      "196500\n",
      "196600\n",
      "196700\n",
      "196800\n",
      "196900\n",
      "197000\n",
      "197100\n",
      "197200\n",
      "197300\n",
      "197400\n",
      "197500\n",
      "197600\n",
      "197700\n",
      "197800\n",
      "197900\n",
      "198000\n",
      "198100\n",
      "198200\n",
      "198300\n",
      "198400\n",
      "198500\n",
      "198600\n",
      "198700\n",
      "198800\n",
      "198900\n",
      "199000\n",
      "199100\n",
      "199200\n",
      "199300\n",
      "199400\n",
      "199500\n",
      "199600\n",
      "199700\n",
      "199800\n",
      "199900\n",
      "200000\n",
      "200100\n",
      "200200\n",
      "200300\n",
      "200400\n",
      "200500\n",
      "200600\n",
      "200700\n",
      "200800\n",
      "200900\n",
      "201000\n",
      "201100\n",
      "201200\n",
      "201300\n",
      "201400\n",
      "201500\n",
      "201600\n",
      "201700\n",
      "201800\n",
      "201900\n",
      "202000\n",
      "202100\n",
      "202200\n",
      "202300\n",
      "202400\n",
      "202500\n",
      "202600\n",
      "202700\n",
      "202800\n",
      "202900\n",
      "203000\n",
      "203100\n",
      "203200\n",
      "203300\n",
      "203400\n",
      "203500\n",
      "203600\n",
      "203700\n",
      "203800\n",
      "203900\n",
      "204000\n",
      "204100\n",
      "204200\n",
      "204300\n",
      "204400\n",
      "204500\n",
      "204600\n",
      "204700\n",
      "204800\n",
      "204900\n",
      "205000\n",
      "205100\n",
      "205200\n",
      "205300\n",
      "205400\n",
      "205500\n",
      "205600\n",
      "205700\n",
      "205800\n",
      "205900\n",
      "206000\n",
      "206100\n",
      "206200\n",
      "206300\n",
      "206400\n",
      "206500\n",
      "206600\n",
      "206700\n",
      "206800\n",
      "206900\n",
      "207000\n",
      "207100\n",
      "207200\n",
      "207300\n",
      "207400\n",
      "207500\n",
      "207600\n",
      "207700\n",
      "207800\n",
      "207900\n",
      "208000\n",
      "208100\n",
      "208200\n",
      "208300\n",
      "208400\n",
      "208500\n",
      "208600\n",
      "208700\n",
      "208800\n",
      "208900\n",
      "209000\n",
      "209100\n",
      "209200\n",
      "209300\n",
      "209400\n",
      "209500\n",
      "209600\n",
      "209700\n",
      "209800\n",
      "209900\n",
      "210000\n",
      "210100\n",
      "210200\n",
      "210300\n",
      "210400\n",
      "210500\n",
      "210600\n",
      "210700\n",
      "210800\n",
      "210900\n",
      "211000\n",
      "211100\n",
      "211200\n",
      "211300\n",
      "211400\n",
      "211500\n",
      "211600\n",
      "211700\n",
      "211800\n",
      "211900\n",
      "212000\n",
      "212100\n",
      "212200\n",
      "212300\n",
      "212400\n",
      "212500\n",
      "212600\n",
      "212700\n",
      "212800\n",
      "212900\n",
      "213000\n",
      "213100\n",
      "213200\n",
      "213300\n",
      "213400\n",
      "213500\n",
      "213600\n",
      "213700\n",
      "213800\n",
      "213900\n",
      "214000\n",
      "214100\n",
      "214200\n",
      "214300\n",
      "214400\n",
      "214500\n",
      "214600\n",
      "214700\n",
      "214800\n",
      "214900\n",
      "215000\n",
      "215100\n",
      "215200\n",
      "215300\n",
      "215400\n",
      "215500\n",
      "215600\n",
      "215700\n",
      "215800\n",
      "215900\n",
      "216000\n",
      "216100\n",
      "216200\n",
      "216300\n",
      "216400\n",
      "216500\n",
      "216600\n",
      "216700\n",
      "216800\n",
      "216900\n",
      "217000\n",
      "217100\n",
      "217200\n",
      "217300\n",
      "217400\n",
      "217500\n",
      "217600\n",
      "217700\n",
      "217800\n",
      "217900\n",
      "218000\n",
      "218100\n",
      "218200\n",
      "218300\n",
      "218400\n",
      "218500\n",
      "218600\n",
      "218700\n",
      "218800\n",
      "218900\n",
      "219000\n",
      "219100\n",
      "219200\n",
      "219300\n",
      "219400\n",
      "219500\n",
      "219600\n",
      "219700\n",
      "219800\n",
      "219900\n",
      "220000\n",
      "220100\n",
      "220200\n",
      "220300\n",
      "220400\n",
      "220500\n",
      "220600\n",
      "220700\n",
      "220800\n",
      "220900\n",
      "221000\n",
      "221100\n",
      "221200\n",
      "221300\n",
      "221400\n",
      "221500\n",
      "221600\n",
      "221700\n",
      "221800\n",
      "221900\n",
      "222000\n",
      "222100\n",
      "222200\n",
      "222300\n",
      "222400\n",
      "222500\n",
      "222600\n",
      "222700\n",
      "222800\n",
      "222900\n",
      "223000\n",
      "223100\n",
      "223200\n",
      "223300\n",
      "223400\n",
      "223500\n",
      "223600\n",
      "223700\n",
      "223800\n",
      "223900\n",
      "224000\n",
      "224100\n",
      "224200\n",
      "224300\n",
      "224400\n",
      "224500\n",
      "224600\n",
      "224700\n",
      "224800\n",
      "224900\n",
      "225000\n",
      "225100\n",
      "225200\n",
      "225300\n",
      "225400\n",
      "225500\n",
      "225600\n",
      "225700\n",
      "225800\n",
      "225900\n",
      "226000\n",
      "226100\n",
      "226200\n",
      "226300\n",
      "226400\n",
      "226500\n",
      "226600\n",
      "226700\n",
      "226800\n",
      "226900\n",
      "227000\n",
      "227100\n",
      "227200\n",
      "227300\n",
      "227400\n",
      "227500\n",
      "227600\n",
      "227700\n",
      "227800\n",
      "227900\n",
      "228000\n",
      "228100\n",
      "228200\n",
      "228300\n",
      "228400\n",
      "228500\n",
      "228600\n",
      "228700\n",
      "228800\n",
      "228900\n",
      "229000\n",
      "229100\n",
      "229200\n",
      "229300\n",
      "229400\n",
      "229500\n",
      "229600\n",
      "229700\n",
      "229800\n",
      "229900\n",
      "230000\n",
      "230100\n",
      "230200\n",
      "230300\n",
      "230400\n",
      "230500\n",
      "230600\n",
      "230700\n",
      "230800\n",
      "230900\n",
      "231000\n",
      "231100\n",
      "231200\n",
      "231300\n",
      "231400\n",
      "231500\n",
      "231600\n",
      "231700\n",
      "231800\n",
      "231900\n",
      "232000\n",
      "232100\n",
      "232200\n",
      "232300\n",
      "232400\n",
      "232500\n",
      "232600\n",
      "232700\n",
      "232800\n",
      "232900\n",
      "233000\n",
      "233100\n",
      "233200\n",
      "233300\n",
      "233400\n",
      "233500\n",
      "233600\n",
      "233700\n",
      "233800\n",
      "233900\n",
      "234000\n",
      "234100\n",
      "234200\n",
      "234300\n",
      "234400\n",
      "234500\n",
      "234600\n",
      "234700\n",
      "234800\n",
      "234900\n",
      "235000\n",
      "235100\n",
      "235200\n",
      "235300\n",
      "235400\n",
      "235500\n",
      "235600\n",
      "235700\n",
      "235800\n",
      "235900\n",
      "236000\n",
      "236100\n",
      "236200\n",
      "236300\n",
      "236400\n",
      "236500\n",
      "236600\n",
      "236700\n",
      "236800\n",
      "236900\n",
      "237000\n",
      "237100\n",
      "237200\n",
      "237300\n",
      "237400\n",
      "237500\n",
      "237600\n",
      "237700\n",
      "237800\n",
      "237900\n",
      "238000\n",
      "238100\n",
      "238200\n",
      "238300\n",
      "238400\n",
      "238500\n",
      "238600\n",
      "238700\n",
      "238800\n",
      "238900\n",
      "239000\n",
      "239100\n",
      "239200\n",
      "239300\n",
      "239400\n",
      "239500\n",
      "239600\n",
      "239700\n",
      "239800\n",
      "239900\n",
      "240000\n",
      "240100\n",
      "240200\n",
      "240300\n",
      "240400\n",
      "240500\n",
      "240600\n",
      "240700\n",
      "240800\n",
      "240900\n",
      "241000\n",
      "241100\n",
      "241200\n",
      "241300\n",
      "241400\n",
      "241500\n",
      "241600\n",
      "241700\n",
      "241800\n",
      "241900\n",
      "242000\n",
      "242100\n",
      "242200\n",
      "242300\n",
      "242400\n",
      "242500\n",
      "242600\n",
      "242700\n",
      "242800\n",
      "242900\n",
      "243000\n",
      "243100\n",
      "243200\n",
      "243300\n",
      "243400\n",
      "243500\n",
      "243600\n",
      "243700\n",
      "243800\n",
      "243900\n",
      "244000\n",
      "244100\n",
      "244200\n",
      "244300\n",
      "244400\n",
      "244500\n",
      "244600\n",
      "244700\n",
      "244800\n",
      "244900\n",
      "245000\n",
      "245100\n",
      "245200\n",
      "245300\n",
      "245400\n",
      "245500\n",
      "245600\n",
      "245700\n",
      "245800\n",
      "245900\n",
      "246000\n",
      "246100\n",
      "246200\n",
      "246300\n",
      "246400\n",
      "246500\n",
      "246600\n",
      "246700\n",
      "246800\n",
      "246900\n",
      "247000\n",
      "247100\n",
      "247200\n",
      "247300\n",
      "247400\n",
      "247500\n",
      "247600\n",
      "247700\n",
      "247800\n",
      "247900\n",
      "248000\n",
      "248100\n",
      "248200\n",
      "248300\n",
      "248400\n",
      "248500\n",
      "248600\n",
      "248700\n",
      "248800\n",
      "248900\n",
      "249000\n",
      "249100\n",
      "249200\n",
      "249300\n",
      "249400\n",
      "249500\n",
      "249600\n",
      "249700\n",
      "249800\n",
      "249900\n",
      "250000\n",
      "250100\n",
      "250200\n",
      "250300\n",
      "250400\n",
      "250500\n",
      "250600\n",
      "250700\n",
      "250800\n",
      "250900\n",
      "251000\n",
      "251100\n",
      "251200\n",
      "251300\n",
      "251400\n",
      "251500\n",
      "251600\n",
      "251700\n",
      "251800\n",
      "251900\n",
      "252000\n",
      "252100\n",
      "252200\n",
      "252300\n",
      "252400\n",
      "252500\n",
      "252600\n",
      "252700\n",
      "252800\n",
      "252900\n",
      "253000\n",
      "253100\n",
      "253200\n",
      "253300\n",
      "253400\n",
      "253500\n",
      "253600\n",
      "253700\n",
      "253800\n",
      "253900\n",
      "254000\n",
      "254100\n",
      "254200\n",
      "254300\n",
      "254400\n",
      "254500\n",
      "254600\n",
      "254700\n",
      "254800\n",
      "254900\n",
      "255000\n",
      "255100\n",
      "255200\n",
      "255300\n",
      "255400\n",
      "255500\n",
      "255600\n",
      "255700\n",
      "255800\n",
      "255900\n",
      "256000\n",
      "256100\n",
      "256200\n",
      "256300\n",
      "256400\n",
      "256500\n",
      "256600\n",
      "256700\n",
      "256800\n",
      "256900\n",
      "257000\n",
      "257100\n",
      "257200\n",
      "257300\n",
      "257400\n",
      "257500\n",
      "257600\n",
      "257700\n",
      "257800\n",
      "257900\n",
      "258000\n",
      "258100\n",
      "258200\n",
      "258300\n",
      "258400\n",
      "258500\n",
      "258600\n",
      "258700\n",
      "258800\n",
      "258900\n",
      "259000\n",
      "259100\n",
      "259200\n",
      "259300\n",
      "259400\n",
      "259500\n",
      "259600\n",
      "259700\n",
      "259800\n",
      "259900\n",
      "260000\n",
      "260100\n",
      "260200\n",
      "260300\n",
      "260400\n",
      "260500\n",
      "260600\n",
      "260700\n",
      "260800\n",
      "260900\n",
      "261000\n",
      "261100\n",
      "261200\n",
      "261300\n",
      "261400\n",
      "261500\n",
      "261600\n",
      "261700\n",
      "261800\n",
      "261900\n",
      "262000\n",
      "262100\n",
      "262200\n",
      "262300\n",
      "262400\n",
      "262500\n",
      "262600\n",
      "262700\n",
      "262800\n",
      "262900\n",
      "263000\n",
      "263100\n",
      "263200\n",
      "263300\n",
      "263400\n",
      "263500\n",
      "263600\n",
      "263700\n",
      "263800\n",
      "263900\n",
      "264000\n",
      "264100\n",
      "264200\n",
      "264300\n",
      "264400\n",
      "264500\n",
      "264600\n",
      "264700\n",
      "264800\n",
      "264900\n",
      "265000\n",
      "265100\n",
      "265200\n",
      "265300\n",
      "265400\n",
      "265500\n",
      "265600\n",
      "265700\n",
      "265800\n",
      "265900\n",
      "266000\n",
      "266100\n",
      "266200\n",
      "266300\n",
      "266400\n",
      "266500\n",
      "266600\n",
      "266700\n",
      "266800\n",
      "266900\n",
      "267000\n",
      "267100\n",
      "267200\n",
      "267300\n",
      "267400\n",
      "267500\n",
      "267600\n",
      "267700\n",
      "267800\n",
      "267900\n",
      "268000\n",
      "268100\n",
      "268200\n",
      "268300\n",
      "268400\n",
      "268500\n",
      "268600\n",
      "268700\n",
      "268800\n",
      "268900\n",
      "269000\n",
      "269100\n",
      "269200\n",
      "269300\n",
      "269400\n",
      "269500\n",
      "269600\n",
      "269700\n",
      "269800\n",
      "269900\n",
      "270000\n",
      "270100\n",
      "270200\n",
      "270300\n",
      "270400\n",
      "270500\n",
      "270600\n",
      "270700\n",
      "270800\n",
      "270900\n",
      "271000\n",
      "271100\n",
      "271200\n",
      "271300\n",
      "271400\n",
      "271500\n",
      "271600\n",
      "271700\n",
      "271800\n",
      "271900\n",
      "272000\n",
      "272100\n",
      "272200\n",
      "272300\n",
      "272400\n",
      "272500\n",
      "272600\n",
      "272700\n",
      "272800\n",
      "272900\n",
      "273000\n",
      "273100\n",
      "273200\n",
      "273300\n",
      "273400\n",
      "273500\n",
      "273600\n",
      "273700\n",
      "273800\n",
      "273900\n",
      "274000\n",
      "274100\n",
      "274200\n",
      "274300\n",
      "274400\n",
      "274500\n",
      "274600\n",
      "274700\n",
      "274800\n",
      "274900\n",
      "275000\n",
      "275100\n",
      "275200\n",
      "275300\n",
      "275400\n",
      "275500\n",
      "275600\n",
      "275700\n",
      "275800\n",
      "275900\n",
      "276000\n",
      "276100\n",
      "276200\n",
      "276300\n",
      "276400\n",
      "276500\n",
      "276600\n",
      "276700\n",
      "276800\n",
      "276900\n",
      "277000\n",
      "277100\n",
      "277200\n",
      "277300\n",
      "277400\n",
      "277500\n",
      "277600\n",
      "277700\n",
      "277800\n",
      "277900\n",
      "278000\n",
      "278100\n",
      "278200\n",
      "278300\n",
      "278400\n",
      "278500\n",
      "278600\n",
      "278700\n",
      "278800\n",
      "278900\n",
      "279000\n",
      "279100\n",
      "279200\n",
      "279300\n",
      "279400\n",
      "279500\n",
      "279600\n",
      "279700\n",
      "279800\n",
      "279900\n",
      "280000\n",
      "280100\n",
      "280200\n",
      "280300\n",
      "280400\n",
      "280500\n",
      "280600\n",
      "280700\n",
      "280800\n",
      "280900\n",
      "281000\n",
      "281100\n",
      "281200\n",
      "281300\n",
      "281400\n",
      "281500\n",
      "281600\n",
      "281700\n",
      "281800\n",
      "281900\n",
      "282000\n",
      "282100\n",
      "282200\n",
      "282300\n",
      "282400\n",
      "282500\n",
      "282600\n",
      "282700\n",
      "282800\n",
      "282900\n",
      "283000\n",
      "283100\n",
      "283200\n",
      "283300\n",
      "283400\n",
      "283500\n",
      "283600\n",
      "283700\n",
      "283800\n",
      "283900\n",
      "284000\n",
      "284100\n",
      "284200\n",
      "284300\n",
      "284400\n",
      "284500\n",
      "284600\n",
      "284700\n",
      "284800\n",
      "284900\n",
      "285000\n",
      "285100\n",
      "285200\n",
      "285300\n",
      "285400\n",
      "285500\n",
      "285600\n",
      "285700\n",
      "285800\n",
      "285900\n",
      "286000\n",
      "286100\n",
      "286200\n",
      "286300\n",
      "286400\n",
      "286500\n",
      "286600\n",
      "286700\n",
      "286800\n",
      "286900\n",
      "287000\n",
      "287100\n",
      "287200\n",
      "287300\n",
      "287400\n",
      "287500\n",
      "287600\n",
      "287700\n",
      "287800\n",
      "287900\n",
      "288000\n",
      "288100\n",
      "288200\n",
      "288300\n",
      "288400\n",
      "288500\n",
      "288600\n",
      "288700\n",
      "288800\n",
      "288900\n",
      "289000\n",
      "289100\n",
      "289200\n",
      "289300\n",
      "289400\n",
      "289500\n",
      "289600\n",
      "289700\n",
      "289800\n",
      "289900\n",
      "290000\n",
      "290100\n",
      "290200\n",
      "290300\n",
      "290400\n",
      "290500\n",
      "290600\n",
      "290700\n",
      "290800\n",
      "290900\n",
      "291000\n",
      "291100\n",
      "291200\n",
      "291300\n",
      "291400\n",
      "291500\n",
      "291600\n",
      "291700\n",
      "291800\n",
      "291900\n",
      "292000\n",
      "292100\n",
      "292200\n",
      "292300\n",
      "292400\n",
      "292500\n",
      "292600\n",
      "292700\n",
      "292800\n",
      "292900\n",
      "293000\n",
      "293100\n",
      "293200\n",
      "293300\n",
      "293400\n",
      "293500\n",
      "293600\n",
      "293700\n",
      "293800\n",
      "293900\n",
      "294000\n",
      "294100\n",
      "294200\n",
      "294300\n",
      "294400\n",
      "294500\n",
      "294600\n",
      "294700\n",
      "294800\n",
      "294900\n",
      "295000\n",
      "295100\n",
      "295200\n",
      "295300\n",
      "295400\n",
      "295500\n",
      "295600\n",
      "295700\n",
      "295800\n",
      "295900\n",
      "296000\n",
      "296100\n",
      "296200\n",
      "296300\n",
      "296400\n",
      "296500\n",
      "296600\n",
      "296700\n",
      "296800\n",
      "296900\n",
      "297000\n",
      "297100\n",
      "297200\n",
      "297300\n",
      "297400\n",
      "297500\n",
      "297600\n",
      "297700\n",
      "297800\n",
      "297900\n",
      "298000\n",
      "298100\n",
      "298200\n",
      "298300\n",
      "298400\n",
      "298500\n",
      "298600\n",
      "298700\n",
      "298800\n",
      "298900\n",
      "299000\n",
      "299100\n",
      "299200\n",
      "299300\n",
      "299400\n",
      "299500\n",
      "tf.Tensor(\n",
      "[[-0.2609103  -0.2573713  -0.42536086 ... -0.28415424 -0.1476488\n",
      "   0.06535861]\n",
      " [-0.2609103  -0.2573713  -0.42536086 ... -0.28415424 -0.1476488\n",
      "   0.06535861]\n",
      " [-0.2609103  -0.2573713  -0.42536086 ... -0.28415424 -0.1476488\n",
      "   0.06535861]\n",
      " ...\n",
      " [-0.4165533  -0.18832478 -0.10225577 ... -0.5118836  -0.00708764\n",
      "   0.1011979 ]\n",
      " [-0.4165533  -0.18832478 -0.10225577 ... -0.5118836  -0.00708764\n",
      "   0.1011979 ]\n",
      " [-0.4165533  -0.18832478 -0.10225577 ... -0.5118836  -0.00708764\n",
      "   0.1011979 ]], shape=(299616, 1024), dtype=float32)\n",
      "0\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "1100\n",
      "1200\n",
      "1300\n",
      "1400\n",
      "1500\n",
      "1600\n",
      "1700\n",
      "1800\n",
      "1900\n",
      "2000\n",
      "2100\n",
      "2200\n",
      "2300\n",
      "2400\n",
      "2500\n",
      "2600\n",
      "2700\n",
      "2800\n",
      "2900\n",
      "3000\n",
      "3100\n",
      "3200\n",
      "3300\n",
      "3400\n",
      "3500\n",
      "3600\n",
      "3700\n",
      "3800\n",
      "3900\n",
      "4000\n",
      "4100\n",
      "4200\n",
      "4300\n",
      "4400\n",
      "4500\n",
      "4600\n",
      "4700\n",
      "4800\n",
      "4900\n",
      "5000\n",
      "5100\n",
      "5200\n",
      "5300\n",
      "5400\n",
      "5500\n",
      "5600\n",
      "5700\n",
      "5800\n",
      "5900\n",
      "6000\n",
      "6100\n",
      "6200\n",
      "6300\n",
      "6400\n",
      "6500\n",
      "6600\n",
      "6700\n",
      "6800\n",
      "6900\n",
      "7000\n",
      "7100\n",
      "7200\n",
      "7300\n",
      "7400\n",
      "7500\n",
      "7600\n",
      "7700\n",
      "7800\n",
      "7900\n",
      "8000\n",
      "8100\n",
      "8200\n",
      "8300\n",
      "8400\n",
      "8500\n",
      "8600\n",
      "8700\n",
      "8800\n",
      "8900\n",
      "9000\n",
      "9100\n",
      "9200\n",
      "9300\n",
      "9400\n",
      "9500\n",
      "9600\n",
      "9700\n",
      "9800\n",
      "9900\n",
      "10000\n",
      "10100\n",
      "10200\n",
      "10300\n",
      "10400\n",
      "10500\n",
      "10600\n",
      "10700\n",
      "10800\n",
      "10900\n",
      "11000\n",
      "11100\n",
      "11200\n",
      "11300\n",
      "11400\n",
      "11500\n",
      "11600\n",
      "11700\n",
      "11800\n",
      "11900\n",
      "12000\n",
      "12100\n",
      "12200\n",
      "12300\n",
      "12400\n",
      "12500\n",
      "12600\n",
      "12700\n",
      "12800\n",
      "12900\n",
      "13000\n",
      "13100\n",
      "13200\n",
      "13300\n",
      "13400\n",
      "13500\n",
      "13600\n",
      "13700\n",
      "13800\n",
      "13900\n",
      "14000\n",
      "14100\n",
      "14200\n",
      "14300\n",
      "14400\n",
      "14500\n",
      "14600\n",
      "14700\n",
      "14800\n",
      "14900\n",
      "15000\n",
      "15100\n",
      "15200\n",
      "15300\n",
      "15400\n",
      "15500\n",
      "15600\n",
      "15700\n",
      "15800\n",
      "15900\n",
      "16000\n",
      "16100\n",
      "16200\n",
      "16300\n",
      "16400\n",
      "16500\n",
      "16600\n",
      "16700\n",
      "16800\n",
      "16900\n",
      "17000\n",
      "17100\n",
      "17200\n",
      "17300\n",
      "17400\n",
      "17500\n",
      "17600\n",
      "17700\n",
      "17800\n",
      "17900\n",
      "18000\n",
      "18100\n",
      "18200\n",
      "18300\n",
      "18400\n",
      "18500\n",
      "18600\n",
      "18700\n",
      "18800\n",
      "18900\n",
      "19000\n",
      "19100\n",
      "19200\n",
      "19300\n",
      "19400\n",
      "19500\n",
      "19600\n",
      "19700\n",
      "19800\n",
      "19900\n",
      "20000\n",
      "20100\n",
      "20200\n",
      "20300\n",
      "20400\n",
      "20500\n",
      "20600\n",
      "20700\n",
      "20800\n",
      "20900\n",
      "21000\n",
      "21100\n",
      "21200\n",
      "21300\n",
      "21400\n",
      "21500\n",
      "21600\n",
      "21700\n",
      "21800\n",
      "21900\n",
      "22000\n",
      "22100\n",
      "22200\n",
      "22300\n",
      "22400\n",
      "22500\n",
      "22600\n",
      "22700\n",
      "22800\n",
      "22900\n",
      "23000\n",
      "23100\n",
      "23200\n",
      "23300\n",
      "23400\n",
      "23500\n",
      "23600\n",
      "23700\n",
      "23800\n",
      "23900\n",
      "24000\n",
      "24100\n",
      "24200\n",
      "24300\n",
      "24400\n",
      "24500\n",
      "24600\n",
      "24700\n",
      "24800\n",
      "24900\n",
      "25000\n",
      "25100\n",
      "25200\n",
      "25300\n",
      "25400\n",
      "25500\n",
      "25600\n",
      "25700\n",
      "25800\n",
      "25900\n",
      "26000\n",
      "26100\n",
      "26200\n",
      "26300\n",
      "26400\n",
      "26500\n",
      "26600\n",
      "26700\n",
      "26800\n",
      "26900\n",
      "27000\n",
      "27100\n",
      "27200\n",
      "27300\n",
      "27400\n",
      "27500\n",
      "27600\n",
      "27700\n",
      "27800\n",
      "27900\n",
      "28000\n",
      "28100\n",
      "28200\n",
      "28300\n",
      "28400\n",
      "28500\n",
      "28600\n",
      "28700\n",
      "28800\n",
      "28900\n",
      "29000\n",
      "29100\n",
      "29200\n",
      "29300\n",
      "29400\n",
      "29500\n",
      "29600\n",
      "29700\n",
      "29800\n",
      "29900\n",
      "30000\n",
      "30100\n",
      "30200\n",
      "30300\n",
      "30400\n",
      "30500\n",
      "30600\n",
      "30700\n",
      "30800\n",
      "30900\n",
      "31000\n",
      "31100\n",
      "31200\n",
      "31300\n",
      "31400\n",
      "31500\n",
      "31600\n",
      "31700\n",
      "31800\n",
      "31900\n",
      "32000\n",
      "32100\n",
      "32200\n",
      "32300\n",
      "32400\n",
      "32500\n",
      "32600\n",
      "32700\n",
      "32800\n",
      "32900\n",
      "33000\n",
      "33100\n",
      "33200\n",
      "33300\n",
      "33400\n",
      "33500\n",
      "33600\n",
      "33700\n",
      "33800\n",
      "33900\n",
      "34000\n",
      "34100\n",
      "34200\n",
      "34300\n",
      "34400\n",
      "34500\n",
      "34600\n",
      "34700\n",
      "34800\n",
      "34900\n",
      "35000\n",
      "35100\n",
      "35200\n",
      "35300\n",
      "35400\n",
      "35500\n",
      "35600\n",
      "35700\n",
      "35800\n",
      "35900\n",
      "36000\n",
      "36100\n",
      "36200\n",
      "36300\n",
      "36400\n",
      "36500\n",
      "36600\n",
      "36700\n",
      "36800\n",
      "36900\n",
      "37000\n",
      "37100\n",
      "37200\n",
      "37300\n",
      "37400\n",
      "37500\n",
      "37600\n",
      "37700\n",
      "37800\n",
      "37900\n",
      "38000\n",
      "38100\n",
      "38200\n",
      "38300\n",
      "38400\n",
      "38500\n",
      "38600\n",
      "38700\n",
      "38800\n",
      "38900\n",
      "39000\n",
      "39100\n",
      "39200\n",
      "39300\n",
      "39400\n",
      "39500\n",
      "39600\n",
      "39700\n",
      "39800\n",
      "39900\n",
      "40000\n",
      "40100\n",
      "40200\n",
      "40300\n",
      "40400\n",
      "40500\n",
      "40600\n",
      "40700\n",
      "40800\n",
      "40900\n",
      "41000\n",
      "41100\n",
      "41200\n",
      "41300\n",
      "41400\n",
      "41500\n",
      "41600\n",
      "41700\n",
      "41800\n",
      "41900\n",
      "42000\n",
      "42100\n",
      "42200\n",
      "42300\n",
      "42400\n",
      "42500\n",
      "42600\n",
      "42700\n",
      "42800\n",
      "42900\n",
      "43000\n",
      "43100\n",
      "43200\n",
      "43300\n",
      "43400\n",
      "43500\n",
      "43600\n",
      "43700\n",
      "43800\n",
      "43900\n",
      "44000\n",
      "44100\n",
      "44200\n",
      "44300\n",
      "44400\n",
      "44500\n",
      "44600\n",
      "44700\n",
      "44800\n",
      "44900\n",
      "45000\n",
      "45100\n",
      "45200\n",
      "45300\n",
      "45400\n",
      "45500\n",
      "45600\n",
      "45700\n",
      "45800\n",
      "45900\n",
      "46000\n",
      "46100\n",
      "46200\n",
      "46300\n",
      "46400\n",
      "46500\n",
      "46600\n",
      "46700\n",
      "46800\n",
      "46900\n",
      "47000\n",
      "47100\n",
      "47200\n",
      "47300\n",
      "47400\n",
      "47500\n",
      "47600\n",
      "47700\n",
      "47800\n",
      "47900\n",
      "48000\n",
      "48100\n",
      "48200\n",
      "48300\n",
      "48400\n",
      "48500\n",
      "48600\n",
      "48700\n",
      "48800\n",
      "48900\n",
      "49000\n",
      "49100\n",
      "49200\n",
      "49300\n",
      "49400\n",
      "49500\n",
      "49600\n",
      "49700\n",
      "49800\n",
      "49900\n",
      "50000\n",
      "50100\n",
      "50200\n",
      "50300\n",
      "50400\n",
      "50500\n",
      "50600\n",
      "50700\n",
      "50800\n",
      "50900\n",
      "51000\n",
      "51100\n",
      "51200\n",
      "51300\n",
      "51400\n",
      "51500\n",
      "51600\n",
      "51700\n",
      "51800\n",
      "51900\n",
      "52000\n",
      "52100\n",
      "52200\n",
      "52300\n",
      "52400\n",
      "52500\n",
      "52600\n",
      "52700\n",
      "52800\n",
      "52900\n",
      "53000\n",
      "53100\n",
      "53200\n",
      "53300\n",
      "53400\n",
      "53500\n",
      "53600\n",
      "53700\n",
      "53800\n",
      "53900\n",
      "54000\n",
      "54100\n",
      "54200\n",
      "54300\n",
      "54400\n",
      "54500\n",
      "54600\n",
      "54700\n",
      "54800\n",
      "54900\n",
      "55000\n",
      "55100\n",
      "55200\n",
      "55300\n",
      "55400\n",
      "55500\n",
      "55600\n",
      "55700\n",
      "55800\n",
      "55900\n",
      "56000\n",
      "56100\n",
      "56200\n",
      "56300\n",
      "56400\n",
      "56500\n",
      "56600\n",
      "56700\n",
      "56800\n",
      "56900\n",
      "57000\n",
      "57100\n",
      "57200\n",
      "57300\n",
      "57400\n",
      "57500\n",
      "57600\n",
      "57700\n",
      "57800\n",
      "57900\n",
      "58000\n",
      "58100\n",
      "58200\n",
      "58300\n",
      "58400\n",
      "58500\n",
      "58600\n",
      "58700\n",
      "58800\n",
      "58900\n",
      "59000\n",
      "59100\n",
      "59200\n",
      "59300\n",
      "59400\n",
      "59500\n",
      "59600\n",
      "59700\n",
      "59800\n",
      "59900\n",
      "60000\n",
      "60100\n",
      "60200\n",
      "60300\n",
      "60400\n",
      "60500\n",
      "60600\n",
      "60700\n",
      "60800\n",
      "60900\n",
      "61000\n",
      "61100\n",
      "61200\n",
      "61300\n",
      "61400\n",
      "61500\n",
      "61600\n",
      "61700\n",
      "61800\n",
      "61900\n",
      "62000\n",
      "62100\n",
      "62200\n",
      "62300\n",
      "62400\n",
      "62500\n",
      "62600\n",
      "62700\n",
      "62800\n",
      "62900\n",
      "63000\n",
      "63100\n",
      "63200\n",
      "63300\n",
      "63400\n",
      "63500\n",
      "63600\n",
      "63700\n",
      "63800\n",
      "63900\n",
      "64000\n",
      "64100\n",
      "64200\n",
      "64300\n",
      "64400\n",
      "64500\n",
      "64600\n",
      "64700\n",
      "64800\n",
      "64900\n",
      "65000\n",
      "65100\n",
      "65200\n",
      "65300\n",
      "65400\n",
      "65500\n",
      "65600\n",
      "65700\n",
      "65800\n",
      "65900\n",
      "66000\n",
      "66100\n",
      "66200\n",
      "66300\n",
      "66400\n",
      "66500\n",
      "66600\n",
      "66700\n",
      "66800\n",
      "66900\n",
      "67000\n",
      "67100\n",
      "67200\n",
      "67300\n",
      "67400\n",
      "67500\n",
      "67600\n",
      "67700\n",
      "67800\n",
      "67900\n",
      "68000\n",
      "68100\n",
      "68200\n",
      "68300\n",
      "68400\n",
      "68500\n",
      "68600\n",
      "68700\n",
      "68800\n",
      "68900\n",
      "69000\n",
      "69100\n",
      "69200\n",
      "69300\n",
      "69400\n",
      "69500\n",
      "69600\n",
      "69700\n",
      "69800\n",
      "69900\n",
      "70000\n",
      "70100\n",
      "70200\n",
      "70300\n",
      "70400\n",
      "70500\n",
      "70600\n",
      "70700\n",
      "70800\n",
      "70900\n",
      "71000\n",
      "71100\n",
      "71200\n",
      "71300\n",
      "71400\n",
      "71500\n",
      "71600\n",
      "71700\n",
      "71800\n",
      "71900\n",
      "72000\n",
      "72100\n",
      "72200\n",
      "72300\n",
      "72400\n",
      "72500\n",
      "72600\n",
      "72700\n",
      "72800\n",
      "72900\n",
      "73000\n",
      "73100\n",
      "73200\n",
      "73300\n",
      "73400\n",
      "73500\n",
      "73600\n",
      "73700\n",
      "73800\n",
      "73900\n",
      "74000\n",
      "74100\n",
      "74200\n",
      "74300\n",
      "74400\n",
      "74500\n",
      "74600\n",
      "74700\n",
      "74800\n",
      "74900\n",
      "75000\n",
      "75100\n",
      "75200\n",
      "75300\n",
      "75400\n",
      "75500\n",
      "75600\n",
      "75700\n",
      "75800\n",
      "75900\n",
      "76000\n",
      "76100\n",
      "76200\n",
      "76300\n",
      "76400\n",
      "76500\n",
      "76600\n",
      "76700\n",
      "76800\n",
      "76900\n",
      "77000\n",
      "77100\n",
      "77200\n",
      "77300\n",
      "77400\n",
      "77500\n",
      "77600\n",
      "77700\n",
      "77800\n",
      "77900\n",
      "78000\n",
      "78100\n",
      "78200\n",
      "78300\n",
      "78400\n",
      "78500\n",
      "78600\n",
      "78700\n",
      "78800\n",
      "78900\n",
      "79000\n",
      "79100\n",
      "79200\n",
      "79300\n",
      "79400\n",
      "79500\n",
      "79600\n",
      "79700\n",
      "79800\n",
      "79900\n",
      "80000\n",
      "80100\n",
      "80200\n",
      "80300\n",
      "80400\n",
      "80500\n",
      "80600\n",
      "80700\n",
      "80800\n",
      "80900\n",
      "81000\n",
      "81100\n",
      "81200\n",
      "81300\n",
      "81400\n",
      "81500\n",
      "81600\n",
      "81700\n",
      "81800\n",
      "81900\n",
      "82000\n",
      "82100\n",
      "82200\n",
      "82300\n",
      "82400\n",
      "82500\n",
      "82600\n",
      "82700\n",
      "82800\n",
      "82900\n",
      "83000\n",
      "83100\n",
      "83200\n",
      "83300\n",
      "83400\n",
      "83500\n",
      "83600\n",
      "83700\n",
      "83800\n",
      "83900\n",
      "84000\n",
      "84100\n",
      "84200\n",
      "84300\n",
      "84400\n",
      "84500\n",
      "84600\n",
      "84700\n",
      "84800\n",
      "84900\n",
      "85000\n",
      "85100\n",
      "85200\n",
      "85300\n",
      "85400\n",
      "85500\n",
      "85600\n",
      "85700\n",
      "85800\n",
      "85900\n",
      "86000\n",
      "86100\n",
      "86200\n",
      "86300\n",
      "86400\n",
      "86500\n",
      "86600\n",
      "86700\n",
      "86800\n",
      "86900\n",
      "87000\n",
      "87100\n",
      "87200\n",
      "87300\n",
      "87400\n",
      "87500\n",
      "87600\n",
      "87700\n",
      "87800\n",
      "87900\n",
      "88000\n",
      "88100\n",
      "88200\n",
      "88300\n",
      "88400\n",
      "88500\n",
      "88600\n",
      "88700\n",
      "88800\n",
      "88900\n",
      "89000\n",
      "89100\n",
      "89200\n",
      "89300\n",
      "89400\n",
      "89500\n",
      "89600\n",
      "89700\n",
      "89800\n",
      "89900\n",
      "90000\n",
      "90100\n",
      "90200\n",
      "90300\n",
      "90400\n",
      "90500\n",
      "90600\n",
      "90700\n",
      "90800\n",
      "90900\n",
      "91000\n",
      "91100\n",
      "91200\n",
      "91300\n",
      "91400\n",
      "91500\n",
      "91600\n",
      "91700\n",
      "91800\n",
      "91900\n",
      "92000\n",
      "92100\n",
      "92200\n",
      "92300\n",
      "92400\n",
      "92500\n",
      "92600\n",
      "92700\n",
      "92800\n",
      "92900\n",
      "93000\n",
      "93100\n",
      "93200\n",
      "93300\n",
      "93400\n",
      "93500\n",
      "93600\n",
      "93700\n",
      "93800\n",
      "93900\n",
      "94000\n",
      "94100\n",
      "94200\n",
      "94300\n",
      "94400\n",
      "94500\n",
      "94600\n",
      "94700\n",
      "94800\n",
      "94900\n",
      "95000\n",
      "95100\n",
      "95200\n",
      "95300\n",
      "95400\n",
      "95500\n",
      "95600\n",
      "95700\n",
      "95800\n",
      "95900\n",
      "96000\n",
      "96100\n",
      "96200\n",
      "96300\n",
      "96400\n",
      "96500\n",
      "96600\n",
      "96700\n",
      "96800\n",
      "96900\n",
      "97000\n",
      "97100\n",
      "97200\n",
      "97300\n",
      "97400\n",
      "97500\n",
      "97600\n",
      "97700\n",
      "97800\n",
      "97900\n",
      "98000\n",
      "98100\n",
      "98200\n",
      "98300\n",
      "98400\n",
      "98500\n",
      "98600\n",
      "98700\n",
      "98800\n",
      "98900\n",
      "99000\n",
      "99100\n",
      "99200\n",
      "99300\n",
      "99400\n",
      "99500\n",
      "99600\n",
      "99700\n",
      "99800\n",
      "99900\n",
      "100000\n",
      "100100\n",
      "100200\n",
      "100300\n",
      "100400\n",
      "100500\n",
      "100600\n",
      "100700\n",
      "100800\n",
      "100900\n",
      "101000\n",
      "101100\n",
      "101200\n",
      "101300\n",
      "101400\n",
      "101500\n",
      "101600\n",
      "101700\n",
      "101800\n",
      "101900\n",
      "102000\n",
      "102100\n",
      "102200\n",
      "102300\n",
      "102400\n",
      "102500\n",
      "102600\n",
      "102700\n",
      "102800\n",
      "102900\n",
      "103000\n",
      "103100\n",
      "103200\n",
      "103300\n",
      "103400\n",
      "103500\n",
      "103600\n",
      "103700\n",
      "103800\n",
      "103900\n",
      "104000\n",
      "104100\n",
      "104200\n",
      "104300\n",
      "104400\n",
      "104500\n",
      "104600\n",
      "104700\n",
      "104800\n",
      "104900\n",
      "105000\n",
      "105100\n",
      "105200\n",
      "105300\n",
      "105400\n",
      "105500\n",
      "105600\n",
      "105700\n",
      "105800\n",
      "105900\n",
      "106000\n",
      "106100\n",
      "106200\n",
      "106300\n",
      "106400\n",
      "106500\n",
      "106600\n",
      "106700\n",
      "106800\n",
      "106900\n",
      "107000\n",
      "107100\n",
      "107200\n",
      "107300\n",
      "107400\n",
      "107500\n",
      "107600\n",
      "107700\n",
      "107800\n",
      "107900\n",
      "108000\n",
      "108100\n",
      "108200\n",
      "108300\n",
      "108400\n",
      "108500\n",
      "108600\n",
      "108700\n",
      "108800\n",
      "108900\n",
      "109000\n",
      "109100\n",
      "109200\n",
      "109300\n",
      "109400\n",
      "109500\n",
      "109600\n",
      "109700\n",
      "109800\n",
      "109900\n",
      "110000\n",
      "110100\n",
      "110200\n",
      "110300\n",
      "110400\n",
      "110500\n",
      "110600\n",
      "110700\n",
      "110800\n",
      "110900\n",
      "111000\n",
      "111100\n",
      "111200\n",
      "111300\n",
      "111400\n",
      "111500\n",
      "111600\n",
      "111700\n",
      "111800\n",
      "111900\n",
      "112000\n",
      "112100\n",
      "112200\n",
      "112300\n",
      "112400\n",
      "112500\n",
      "112600\n",
      "112700\n",
      "112800\n",
      "112900\n",
      "113000\n",
      "113100\n",
      "113200\n",
      "113300\n",
      "113400\n",
      "113500\n",
      "113600\n",
      "113700\n",
      "113800\n",
      "113900\n",
      "114000\n",
      "114100\n",
      "114200\n",
      "114300\n",
      "114400\n",
      "114500\n",
      "114600\n",
      "114700\n",
      "114800\n",
      "114900\n",
      "115000\n",
      "115100\n",
      "115200\n",
      "115300\n",
      "115400\n",
      "115500\n",
      "115600\n",
      "115700\n",
      "115800\n",
      "115900\n",
      "116000\n",
      "116100\n",
      "116200\n",
      "116300\n",
      "116400\n",
      "116500\n",
      "116600\n",
      "116700\n",
      "116800\n",
      "116900\n",
      "117000\n",
      "117100\n",
      "117200\n",
      "117300\n",
      "117400\n",
      "117500\n",
      "117600\n",
      "117700\n",
      "117800\n",
      "117900\n",
      "118000\n",
      "118100\n",
      "118200\n",
      "118300\n",
      "118400\n",
      "118500\n",
      "118600\n",
      "118700\n",
      "118800\n",
      "118900\n",
      "119000\n",
      "119100\n",
      "119200\n",
      "119300\n",
      "119400\n",
      "119500\n",
      "119600\n",
      "119700\n",
      "119800\n",
      "119900\n",
      "120000\n",
      "120100\n",
      "120200\n",
      "120300\n",
      "120400\n",
      "120500\n",
      "120600\n",
      "120700\n",
      "120800\n",
      "120900\n",
      "121000\n",
      "121100\n",
      "121200\n",
      "121300\n",
      "121400\n",
      "121500\n",
      "121600\n",
      "121700\n",
      "121800\n",
      "121900\n",
      "122000\n",
      "122100\n",
      "122200\n",
      "122300\n",
      "122400\n",
      "122500\n",
      "122600\n",
      "122700\n",
      "122800\n",
      "122900\n",
      "123000\n",
      "123100\n",
      "123200\n",
      "123300\n",
      "123400\n",
      "123500\n",
      "123600\n",
      "123700\n",
      "123800\n",
      "123900\n",
      "124000\n",
      "124100\n",
      "124200\n",
      "124300\n",
      "124400\n",
      "124500\n",
      "124600\n",
      "124700\n",
      "124800\n",
      "124900\n",
      "125000\n",
      "125100\n",
      "125200\n",
      "125300\n",
      "125400\n",
      "125500\n",
      "125600\n",
      "125700\n",
      "125800\n",
      "125900\n",
      "126000\n",
      "126100\n",
      "126200\n",
      "126300\n",
      "126400\n",
      "126500\n",
      "126600\n",
      "126700\n",
      "126800\n",
      "126900\n",
      "127000\n",
      "127100\n",
      "127200\n",
      "127300\n",
      "127400\n",
      "127500\n",
      "127600\n",
      "127700\n",
      "127800\n",
      "127900\n",
      "128000\n",
      "128100\n",
      "128200\n",
      "128300\n",
      "128400\n",
      "128500\n",
      "128600\n",
      "128700\n",
      "128800\n",
      "128900\n",
      "129000\n",
      "129100\n",
      "129200\n",
      "129300\n",
      "129400\n",
      "129500\n",
      "129600\n",
      "129700\n",
      "129800\n",
      "129900\n",
      "130000\n",
      "130100\n",
      "130200\n",
      "130300\n",
      "130400\n",
      "130500\n",
      "130600\n",
      "130700\n",
      "130800\n",
      "130900\n",
      "131000\n",
      "131100\n",
      "131200\n",
      "131300\n",
      "131400\n",
      "131500\n",
      "131600\n",
      "131700\n",
      "131800\n",
      "131900\n",
      "132000\n",
      "132100\n",
      "132200\n",
      "132300\n",
      "132400\n",
      "132500\n",
      "132600\n",
      "132700\n",
      "132800\n",
      "132900\n",
      "133000\n",
      "133100\n",
      "133200\n",
      "133300\n",
      "133400\n",
      "133500\n",
      "133600\n",
      "133700\n",
      "133800\n",
      "133900\n",
      "134000\n",
      "134100\n",
      "134200\n",
      "134300\n",
      "134400\n",
      "134500\n",
      "134600\n",
      "134700\n",
      "134800\n",
      "134900\n",
      "135000\n",
      "135100\n",
      "135200\n",
      "135300\n",
      "135400\n",
      "135500\n",
      "135600\n",
      "135700\n",
      "135800\n",
      "135900\n",
      "136000\n",
      "136100\n",
      "136200\n",
      "136300\n",
      "136400\n",
      "136500\n",
      "136600\n",
      "136700\n",
      "136800\n",
      "136900\n",
      "137000\n",
      "137100\n",
      "137200\n",
      "137300\n",
      "137400\n",
      "137500\n",
      "137600\n",
      "137700\n",
      "137800\n",
      "137900\n",
      "138000\n",
      "138100\n",
      "138200\n",
      "138300\n",
      "138400\n",
      "138500\n",
      "138600\n",
      "138700\n",
      "138800\n",
      "138900\n",
      "139000\n",
      "139100\n",
      "139200\n",
      "139300\n",
      "139400\n",
      "139500\n",
      "139600\n",
      "139700\n",
      "139800\n",
      "139900\n",
      "140000\n",
      "140100\n",
      "140200\n",
      "140300\n",
      "140400\n",
      "140500\n",
      "140600\n",
      "140700\n",
      "140800\n",
      "140900\n",
      "141000\n",
      "141100\n",
      "141200\n",
      "141300\n",
      "141400\n",
      "141500\n",
      "141600\n",
      "141700\n",
      "141800\n",
      "141900\n",
      "142000\n",
      "142100\n",
      "142200\n",
      "142300\n",
      "142400\n",
      "142500\n",
      "142600\n",
      "142700\n",
      "142800\n",
      "142900\n",
      "143000\n",
      "143100\n",
      "143200\n",
      "143300\n",
      "143400\n",
      "143500\n",
      "143600\n",
      "143700\n",
      "143800\n",
      "143900\n",
      "144000\n",
      "144100\n",
      "144200\n",
      "144300\n",
      "144400\n",
      "144500\n",
      "144600\n",
      "144700\n",
      "144800\n",
      "144900\n",
      "145000\n",
      "145100\n",
      "145200\n",
      "145300\n",
      "145400\n",
      "145500\n",
      "145600\n",
      "145700\n",
      "145800\n",
      "145900\n",
      "146000\n",
      "146100\n",
      "146200\n",
      "146300\n",
      "146400\n",
      "146500\n",
      "146600\n",
      "146700\n",
      "146800\n",
      "146900\n",
      "147000\n",
      "147100\n",
      "147200\n",
      "147300\n",
      "147400\n",
      "147500\n",
      "147600\n",
      "147700\n",
      "147800\n",
      "147900\n",
      "148000\n",
      "148100\n",
      "148200\n",
      "148300\n",
      "148400\n",
      "148500\n",
      "148600\n",
      "148700\n",
      "148800\n",
      "148900\n",
      "149000\n",
      "149100\n",
      "149200\n",
      "149300\n",
      "149400\n",
      "149500\n",
      "149600\n",
      "149700\n",
      "149800\n",
      "149900\n",
      "150000\n",
      "150100\n",
      "150200\n",
      "150300\n",
      "150400\n",
      "150500\n",
      "150600\n",
      "150700\n",
      "150800\n",
      "150900\n",
      "151000\n",
      "151100\n",
      "151200\n",
      "151300\n",
      "151400\n",
      "151500\n",
      "151600\n",
      "151700\n",
      "151800\n",
      "151900\n",
      "152000\n",
      "152100\n",
      "152200\n",
      "152300\n",
      "152400\n",
      "152500\n",
      "152600\n",
      "152700\n",
      "152800\n",
      "152900\n",
      "153000\n",
      "153100\n",
      "153200\n",
      "153300\n",
      "153400\n",
      "153500\n",
      "153600\n",
      "153700\n",
      "153800\n",
      "153900\n",
      "154000\n",
      "154100\n",
      "154200\n",
      "154300\n",
      "154400\n",
      "154500\n",
      "154600\n",
      "154700\n",
      "154800\n",
      "154900\n",
      "155000\n",
      "155100\n",
      "155200\n",
      "155300\n",
      "155400\n",
      "155500\n",
      "155600\n",
      "155700\n",
      "155800\n",
      "155900\n",
      "156000\n",
      "156100\n",
      "156200\n",
      "156300\n",
      "156400\n",
      "156500\n",
      "156600\n",
      "156700\n",
      "156800\n",
      "156900\n",
      "157000\n",
      "157100\n",
      "157200\n",
      "157300\n",
      "157400\n",
      "157500\n",
      "157600\n",
      "157700\n",
      "157800\n",
      "157900\n",
      "158000\n",
      "158100\n",
      "158200\n",
      "158300\n",
      "158400\n",
      "158500\n",
      "158600\n",
      "158700\n",
      "158800\n",
      "158900\n",
      "159000\n",
      "159100\n",
      "159200\n",
      "159300\n",
      "159400\n",
      "159500\n",
      "159600\n",
      "159700\n",
      "159800\n",
      "159900\n",
      "160000\n",
      "160100\n",
      "160200\n",
      "160300\n",
      "160400\n",
      "160500\n",
      "160600\n",
      "160700\n",
      "160800\n",
      "160900\n",
      "161000\n",
      "161100\n",
      "161200\n",
      "161300\n",
      "161400\n",
      "161500\n",
      "161600\n",
      "161700\n",
      "161800\n",
      "161900\n",
      "162000\n",
      "162100\n",
      "162200\n",
      "162300\n",
      "162400\n",
      "162500\n",
      "162600\n",
      "162700\n",
      "162800\n",
      "162900\n",
      "163000\n",
      "163100\n",
      "163200\n",
      "163300\n",
      "163400\n",
      "163500\n",
      "163600\n",
      "163700\n",
      "163800\n",
      "163900\n",
      "164000\n",
      "164100\n",
      "164200\n",
      "164300\n",
      "164400\n",
      "164500\n",
      "164600\n",
      "164700\n",
      "164800\n",
      "164900\n",
      "165000\n",
      "165100\n",
      "165200\n",
      "165300\n",
      "165400\n",
      "165500\n",
      "165600\n",
      "165700\n",
      "165800\n",
      "165900\n",
      "166000\n",
      "166100\n",
      "166200\n",
      "166300\n",
      "166400\n",
      "166500\n",
      "166600\n",
      "166700\n",
      "166800\n",
      "166900\n",
      "167000\n",
      "167100\n",
      "167200\n",
      "167300\n",
      "167400\n",
      "167500\n",
      "167600\n",
      "167700\n",
      "167800\n",
      "167900\n",
      "168000\n",
      "168100\n",
      "168200\n",
      "168300\n",
      "168400\n",
      "168500\n",
      "168600\n",
      "168700\n",
      "168800\n",
      "168900\n",
      "169000\n",
      "169100\n",
      "169200\n",
      "169300\n",
      "169400\n",
      "169500\n",
      "169600\n",
      "169700\n",
      "169800\n",
      "169900\n",
      "170000\n",
      "170100\n",
      "170200\n",
      "170300\n",
      "170400\n",
      "170500\n",
      "170600\n",
      "170700\n",
      "170800\n",
      "170900\n",
      "171000\n",
      "171100\n",
      "171200\n",
      "171300\n",
      "171400\n",
      "171500\n",
      "171600\n",
      "171700\n",
      "171800\n",
      "171900\n",
      "172000\n",
      "172100\n",
      "172200\n",
      "172300\n",
      "172400\n",
      "172500\n",
      "172600\n",
      "172700\n",
      "172800\n",
      "172900\n",
      "173000\n",
      "173100\n",
      "173200\n",
      "173300\n",
      "173400\n",
      "173500\n",
      "173600\n",
      "173700\n",
      "173800\n",
      "173900\n",
      "174000\n",
      "174100\n",
      "174200\n",
      "174300\n",
      "174400\n",
      "174500\n",
      "174600\n",
      "174700\n",
      "174800\n",
      "174900\n",
      "175000\n",
      "175100\n",
      "175200\n",
      "175300\n",
      "175400\n",
      "175500\n",
      "175600\n",
      "175700\n",
      "175800\n",
      "175900\n",
      "176000\n",
      "176100\n",
      "176200\n",
      "176300\n",
      "176400\n",
      "176500\n",
      "176600\n",
      "176700\n",
      "176800\n",
      "176900\n",
      "177000\n",
      "177100\n",
      "177200\n",
      "177300\n",
      "177400\n",
      "177500\n",
      "177600\n",
      "177700\n",
      "177800\n",
      "177900\n",
      "178000\n",
      "178100\n",
      "178200\n",
      "178300\n",
      "178400\n",
      "178500\n",
      "178600\n",
      "178700\n",
      "178800\n",
      "178900\n",
      "179000\n",
      "179100\n",
      "179200\n",
      "179300\n",
      "179400\n",
      "179500\n",
      "179600\n",
      "179700\n",
      "179800\n",
      "179900\n",
      "180000\n",
      "180100\n",
      "180200\n",
      "180300\n",
      "180400\n",
      "180500\n",
      "180600\n",
      "180700\n",
      "180800\n",
      "180900\n",
      "181000\n",
      "181100\n",
      "181200\n",
      "181300\n",
      "181400\n",
      "181500\n",
      "181600\n",
      "181700\n",
      "181800\n",
      "181900\n",
      "182000\n",
      "182100\n",
      "182200\n",
      "182300\n",
      "182400\n",
      "182500\n",
      "182600\n",
      "182700\n",
      "182800\n",
      "182900\n",
      "183000\n",
      "183100\n",
      "183200\n",
      "183300\n",
      "183400\n",
      "183500\n",
      "183600\n",
      "183700\n",
      "183800\n",
      "183900\n",
      "184000\n",
      "184100\n",
      "184200\n",
      "184300\n",
      "184400\n",
      "184500\n",
      "184600\n",
      "184700\n",
      "184800\n",
      "184900\n",
      "185000\n",
      "185100\n",
      "185200\n",
      "185300\n",
      "185400\n",
      "185500\n",
      "185600\n",
      "185700\n",
      "185800\n",
      "185900\n",
      "186000\n",
      "186100\n",
      "186200\n",
      "186300\n",
      "186400\n",
      "186500\n",
      "186600\n",
      "186700\n",
      "186800\n",
      "186900\n",
      "187000\n",
      "187100\n",
      "187200\n",
      "187300\n",
      "187400\n",
      "187500\n",
      "187600\n",
      "187700\n",
      "187800\n",
      "187900\n",
      "188000\n",
      "188100\n",
      "188200\n",
      "188300\n",
      "188400\n",
      "188500\n",
      "188600\n",
      "188700\n",
      "188800\n",
      "188900\n",
      "189000\n",
      "189100\n",
      "189200\n",
      "189300\n",
      "189400\n",
      "189500\n",
      "189600\n",
      "189700\n",
      "189800\n",
      "189900\n",
      "190000\n",
      "190100\n",
      "190200\n",
      "190300\n",
      "190400\n",
      "190500\n",
      "190600\n",
      "190700\n",
      "190800\n",
      "190900\n",
      "191000\n",
      "191100\n",
      "191200\n",
      "191300\n",
      "191400\n",
      "191500\n",
      "191600\n",
      "191700\n",
      "191800\n",
      "191900\n",
      "192000\n",
      "192100\n",
      "192200\n",
      "192300\n",
      "192400\n",
      "192500\n",
      "192600\n",
      "192700\n",
      "192800\n",
      "192900\n",
      "193000\n",
      "193100\n",
      "193200\n",
      "193300\n",
      "193400\n",
      "193500\n",
      "193600\n",
      "193700\n",
      "193800\n",
      "193900\n",
      "194000\n",
      "194100\n",
      "194200\n",
      "194300\n",
      "194400\n",
      "194500\n",
      "194600\n",
      "194700\n",
      "194800\n",
      "194900\n",
      "195000\n",
      "195100\n",
      "195200\n",
      "195300\n",
      "195400\n",
      "195500\n",
      "195600\n",
      "195700\n",
      "195800\n",
      "195900\n",
      "196000\n",
      "196100\n",
      "196200\n",
      "196300\n",
      "196400\n",
      "196500\n",
      "196600\n",
      "196700\n",
      "196800\n",
      "196900\n",
      "197000\n",
      "197100\n",
      "197200\n",
      "197300\n",
      "197400\n",
      "197500\n",
      "197600\n",
      "197700\n",
      "197800\n",
      "197900\n",
      "198000\n",
      "198100\n",
      "198200\n",
      "198300\n",
      "198400\n",
      "198500\n",
      "198600\n",
      "198700\n",
      "198800\n",
      "198900\n",
      "199000\n",
      "199100\n",
      "199200\n",
      "199300\n",
      "199400\n",
      "199500\n",
      "199600\n",
      "199700\n",
      "199800\n",
      "199900\n",
      "200000\n",
      "200100\n",
      "200200\n",
      "200300\n",
      "200400\n",
      "200500\n",
      "200600\n",
      "200700\n",
      "200800\n",
      "200900\n",
      "201000\n",
      "201100\n",
      "201200\n",
      "201300\n",
      "201400\n",
      "201500\n",
      "201600\n",
      "201700\n",
      "201800\n",
      "201900\n",
      "202000\n",
      "202100\n",
      "202200\n",
      "202300\n",
      "202400\n",
      "202500\n",
      "202600\n",
      "202700\n",
      "202800\n",
      "202900\n",
      "203000\n",
      "203100\n",
      "203200\n",
      "203300\n",
      "203400\n",
      "203500\n",
      "203600\n",
      "203700\n",
      "203800\n",
      "203900\n",
      "204000\n",
      "204100\n",
      "204200\n",
      "204300\n",
      "204400\n",
      "204500\n",
      "204600\n",
      "204700\n",
      "204800\n",
      "204900\n",
      "205000\n",
      "205100\n",
      "205200\n",
      "205300\n",
      "205400\n",
      "205500\n",
      "205600\n",
      "205700\n",
      "205800\n",
      "205900\n",
      "206000\n",
      "206100\n",
      "206200\n",
      "206300\n",
      "206400\n",
      "206500\n",
      "206600\n",
      "206700\n",
      "206800\n",
      "206900\n",
      "207000\n",
      "207100\n",
      "207200\n",
      "207300\n",
      "207400\n",
      "207500\n",
      "207600\n",
      "207700\n",
      "207800\n",
      "207900\n",
      "208000\n",
      "208100\n",
      "208200\n",
      "208300\n",
      "208400\n",
      "208500\n",
      "208600\n",
      "208700\n",
      "208800\n",
      "208900\n",
      "209000\n",
      "209100\n",
      "209200\n",
      "209300\n",
      "209400\n",
      "209500\n",
      "209600\n",
      "209700\n",
      "209800\n",
      "209900\n",
      "210000\n",
      "210100\n",
      "210200\n",
      "210300\n",
      "210400\n",
      "210500\n",
      "210600\n",
      "210700\n",
      "210800\n",
      "210900\n",
      "211000\n",
      "211100\n",
      "211200\n",
      "211300\n",
      "211400\n",
      "211500\n",
      "211600\n",
      "211700\n",
      "211800\n",
      "211900\n",
      "212000\n",
      "212100\n",
      "212200\n",
      "212300\n",
      "212400\n",
      "212500\n",
      "212600\n",
      "212700\n",
      "212800\n",
      "212900\n",
      "213000\n",
      "213100\n",
      "213200\n",
      "213300\n",
      "213400\n",
      "213500\n",
      "213600\n",
      "213700\n",
      "213800\n",
      "213900\n",
      "214000\n",
      "214100\n",
      "214200\n",
      "214300\n",
      "214400\n",
      "214500\n",
      "214600\n",
      "214700\n",
      "214800\n",
      "214900\n",
      "215000\n",
      "215100\n",
      "215200\n",
      "215300\n",
      "215400\n",
      "215500\n",
      "215600\n",
      "215700\n",
      "215800\n",
      "215900\n",
      "216000\n",
      "216100\n",
      "216200\n",
      "216300\n",
      "216400\n",
      "216500\n",
      "216600\n",
      "216700\n",
      "216800\n",
      "216900\n",
      "217000\n",
      "217100\n",
      "217200\n",
      "217300\n",
      "217400\n",
      "217500\n",
      "217600\n",
      "217700\n",
      "217800\n",
      "217900\n",
      "218000\n",
      "218100\n",
      "218200\n",
      "218300\n",
      "218400\n",
      "218500\n",
      "218600\n",
      "218700\n",
      "218800\n",
      "218900\n",
      "219000\n",
      "219100\n",
      "219200\n",
      "219300\n",
      "219400\n",
      "219500\n",
      "219600\n",
      "219700\n",
      "219800\n",
      "219900\n",
      "220000\n",
      "220100\n",
      "220200\n",
      "220300\n",
      "220400\n",
      "220500\n",
      "220600\n",
      "220700\n",
      "220800\n",
      "220900\n",
      "221000\n",
      "221100\n",
      "221200\n",
      "221300\n",
      "221400\n",
      "221500\n",
      "221600\n",
      "221700\n",
      "221800\n",
      "221900\n",
      "222000\n",
      "222100\n",
      "222200\n",
      "222300\n",
      "222400\n",
      "222500\n",
      "222600\n",
      "222700\n",
      "222800\n",
      "222900\n",
      "223000\n",
      "223100\n",
      "223200\n",
      "223300\n",
      "223400\n",
      "223500\n",
      "223600\n",
      "223700\n",
      "223800\n",
      "223900\n",
      "224000\n",
      "224100\n",
      "224200\n",
      "224300\n",
      "224400\n",
      "224500\n",
      "224600\n",
      "224700\n",
      "224800\n",
      "224900\n",
      "225000\n",
      "225100\n",
      "225200\n",
      "225300\n",
      "225400\n",
      "225500\n",
      "225600\n",
      "225700\n",
      "225800\n",
      "225900\n",
      "226000\n",
      "226100\n",
      "226200\n",
      "226300\n",
      "226400\n",
      "226500\n",
      "226600\n",
      "226700\n",
      "226800\n",
      "226900\n",
      "227000\n",
      "227100\n",
      "227200\n",
      "227300\n",
      "227400\n",
      "227500\n",
      "227600\n",
      "227700\n",
      "227800\n",
      "227900\n",
      "228000\n",
      "228100\n",
      "228200\n",
      "228300\n",
      "228400\n",
      "228500\n",
      "228600\n",
      "228700\n",
      "228800\n",
      "228900\n",
      "229000\n",
      "229100\n",
      "229200\n",
      "229300\n",
      "229400\n",
      "229500\n",
      "229600\n",
      "229700\n",
      "229800\n",
      "229900\n",
      "230000\n",
      "230100\n",
      "230200\n",
      "230300\n",
      "230400\n",
      "230500\n",
      "230600\n",
      "230700\n",
      "230800\n",
      "230900\n",
      "231000\n",
      "231100\n",
      "231200\n",
      "231300\n",
      "231400\n",
      "231500\n",
      "231600\n",
      "231700\n",
      "231800\n",
      "231900\n",
      "232000\n",
      "232100\n",
      "232200\n",
      "232300\n",
      "232400\n",
      "232500\n",
      "232600\n",
      "232700\n",
      "232800\n",
      "232900\n",
      "233000\n",
      "233100\n",
      "233200\n",
      "233300\n",
      "233400\n",
      "233500\n",
      "233600\n",
      "233700\n",
      "233800\n",
      "233900\n",
      "234000\n",
      "234100\n",
      "234200\n",
      "234300\n",
      "234400\n",
      "234500\n",
      "234600\n",
      "234700\n",
      "234800\n",
      "234900\n",
      "235000\n",
      "235100\n",
      "235200\n",
      "235300\n",
      "235400\n",
      "235500\n",
      "235600\n",
      "235700\n",
      "235800\n",
      "235900\n",
      "236000\n",
      "236100\n",
      "236200\n",
      "236300\n",
      "236400\n",
      "236500\n",
      "236600\n",
      "236700\n",
      "236800\n",
      "236900\n",
      "237000\n",
      "237100\n",
      "237200\n",
      "237300\n",
      "237400\n",
      "237500\n",
      "237600\n",
      "237700\n",
      "237800\n",
      "237900\n",
      "238000\n",
      "238100\n",
      "238200\n",
      "238300\n",
      "238400\n",
      "238500\n",
      "238600\n",
      "238700\n",
      "238800\n",
      "238900\n",
      "239000\n",
      "239100\n",
      "239200\n",
      "239300\n",
      "239400\n",
      "239500\n",
      "239600\n",
      "239700\n",
      "239800\n",
      "239900\n",
      "240000\n",
      "240100\n",
      "240200\n",
      "240300\n",
      "240400\n",
      "240500\n",
      "240600\n",
      "240700\n",
      "240800\n",
      "240900\n",
      "241000\n",
      "241100\n",
      "241200\n",
      "241300\n",
      "241400\n",
      "241500\n",
      "241600\n",
      "241700\n",
      "241800\n",
      "241900\n",
      "242000\n",
      "242100\n",
      "242200\n",
      "242300\n",
      "242400\n",
      "242500\n",
      "242600\n",
      "242700\n",
      "242800\n",
      "242900\n",
      "243000\n",
      "243100\n",
      "243200\n",
      "243300\n",
      "243400\n",
      "243500\n",
      "243600\n",
      "243700\n",
      "243800\n",
      "243900\n",
      "244000\n",
      "244100\n",
      "244200\n",
      "244300\n",
      "244400\n",
      "244500\n",
      "244600\n",
      "244700\n",
      "244800\n",
      "244900\n",
      "245000\n",
      "245100\n",
      "245200\n",
      "245300\n",
      "245400\n",
      "245500\n",
      "245600\n",
      "245700\n",
      "245800\n",
      "245900\n",
      "246000\n",
      "246100\n",
      "246200\n",
      "246300\n",
      "246400\n",
      "246500\n",
      "246600\n",
      "246700\n",
      "246800\n",
      "246900\n",
      "247000\n",
      "247100\n",
      "247200\n",
      "247300\n",
      "247400\n",
      "247500\n",
      "247600\n",
      "247700\n",
      "247800\n",
      "247900\n",
      "248000\n",
      "248100\n",
      "248200\n",
      "248300\n",
      "248400\n",
      "248500\n",
      "248600\n",
      "248700\n",
      "248800\n",
      "248900\n",
      "249000\n",
      "249100\n",
      "249200\n",
      "249300\n",
      "249400\n",
      "249500\n",
      "249600\n",
      "249700\n",
      "249800\n",
      "249900\n",
      "250000\n",
      "250100\n",
      "250200\n",
      "250300\n",
      "250400\n",
      "250500\n",
      "250600\n",
      "250700\n",
      "250800\n",
      "250900\n",
      "251000\n",
      "251100\n",
      "251200\n",
      "251300\n",
      "251400\n",
      "251500\n",
      "251600\n",
      "251700\n",
      "251800\n",
      "251900\n",
      "252000\n",
      "252100\n",
      "252200\n",
      "252300\n",
      "252400\n",
      "252500\n",
      "252600\n",
      "252700\n",
      "252800\n",
      "252900\n",
      "253000\n",
      "253100\n",
      "253200\n",
      "253300\n",
      "253400\n",
      "253500\n",
      "253600\n",
      "253700\n",
      "253800\n",
      "253900\n",
      "254000\n",
      "254100\n",
      "254200\n",
      "254300\n",
      "254400\n",
      "254500\n",
      "254600\n",
      "254700\n",
      "254800\n",
      "254900\n",
      "255000\n",
      "255100\n",
      "255200\n",
      "255300\n",
      "255400\n",
      "255500\n",
      "255600\n",
      "255700\n",
      "255800\n",
      "255900\n",
      "256000\n",
      "256100\n",
      "256200\n",
      "256300\n",
      "256400\n",
      "256500\n",
      "256600\n",
      "256700\n",
      "256800\n",
      "256900\n",
      "257000\n",
      "257100\n",
      "257200\n",
      "257300\n",
      "257400\n",
      "257500\n",
      "257600\n",
      "257700\n",
      "257800\n",
      "257900\n",
      "258000\n",
      "258100\n",
      "258200\n",
      "258300\n",
      "258400\n",
      "258500\n",
      "258600\n",
      "258700\n",
      "258800\n",
      "258900\n",
      "259000\n",
      "259100\n",
      "259200\n",
      "259300\n",
      "259400\n",
      "259500\n",
      "259600\n",
      "259700\n",
      "259800\n",
      "259900\n",
      "260000\n",
      "260100\n",
      "260200\n",
      "260300\n",
      "260400\n",
      "260500\n",
      "260600\n",
      "260700\n",
      "260800\n",
      "260900\n",
      "261000\n",
      "261100\n",
      "261200\n",
      "261300\n",
      "261400\n",
      "261500\n",
      "261600\n",
      "261700\n",
      "261800\n",
      "261900\n",
      "262000\n",
      "262100\n",
      "262200\n",
      "262300\n",
      "262400\n",
      "262500\n",
      "262600\n",
      "262700\n",
      "262800\n",
      "262900\n",
      "263000\n",
      "263100\n",
      "263200\n",
      "263300\n",
      "263400\n",
      "263500\n",
      "263600\n",
      "263700\n",
      "263800\n",
      "263900\n",
      "264000\n",
      "264100\n",
      "264200\n",
      "264300\n",
      "264400\n",
      "264500\n",
      "264600\n",
      "264700\n",
      "264800\n",
      "264900\n",
      "265000\n",
      "265100\n",
      "265200\n",
      "265300\n",
      "265400\n",
      "265500\n",
      "265600\n",
      "265700\n",
      "265800\n",
      "265900\n",
      "266000\n",
      "266100\n",
      "266200\n",
      "266300\n",
      "266400\n",
      "266500\n",
      "266600\n",
      "266700\n",
      "266800\n",
      "266900\n",
      "267000\n",
      "267100\n",
      "267200\n",
      "267300\n",
      "267400\n",
      "267500\n",
      "267600\n",
      "267700\n",
      "267800\n",
      "267900\n",
      "268000\n",
      "268100\n",
      "268200\n",
      "268300\n",
      "268400\n",
      "268500\n",
      "268600\n",
      "268700\n",
      "268800\n",
      "268900\n",
      "269000\n",
      "269100\n",
      "269200\n",
      "269300\n",
      "269400\n",
      "269500\n",
      "269600\n",
      "269700\n",
      "269800\n",
      "269900\n",
      "270000\n",
      "270100\n",
      "270200\n",
      "270300\n",
      "270400\n",
      "270500\n",
      "270600\n",
      "270700\n",
      "270800\n",
      "270900\n",
      "271000\n",
      "271100\n",
      "271200\n",
      "271300\n",
      "271400\n",
      "271500\n",
      "271600\n",
      "271700\n",
      "271800\n",
      "271900\n",
      "272000\n",
      "272100\n",
      "272200\n",
      "272300\n",
      "272400\n",
      "272500\n",
      "272600\n",
      "272700\n",
      "272800\n",
      "272900\n",
      "273000\n",
      "273100\n",
      "273200\n",
      "273300\n",
      "273400\n",
      "273500\n",
      "273600\n",
      "273700\n",
      "273800\n",
      "273900\n",
      "274000\n",
      "274100\n",
      "274200\n",
      "274300\n",
      "274400\n",
      "274500\n",
      "274600\n",
      "274700\n",
      "274800\n",
      "274900\n",
      "275000\n",
      "275100\n",
      "275200\n",
      "275300\n",
      "275400\n",
      "275500\n",
      "275600\n",
      "275700\n",
      "275800\n",
      "275900\n",
      "276000\n",
      "276100\n",
      "276200\n",
      "276300\n",
      "276400\n",
      "276500\n",
      "276600\n",
      "276700\n",
      "276800\n",
      "276900\n",
      "277000\n",
      "277100\n",
      "277200\n",
      "277300\n",
      "277400\n",
      "277500\n",
      "277600\n",
      "277700\n",
      "277800\n",
      "277900\n",
      "278000\n",
      "278100\n",
      "278200\n",
      "278300\n",
      "278400\n",
      "278500\n",
      "278600\n",
      "278700\n",
      "278800\n",
      "278900\n",
      "279000\n",
      "279100\n",
      "279200\n",
      "279300\n",
      "279400\n",
      "279500\n",
      "279600\n",
      "279700\n",
      "279800\n",
      "279900\n",
      "280000\n",
      "280100\n",
      "280200\n",
      "280300\n",
      "280400\n",
      "280500\n",
      "280600\n",
      "280700\n",
      "280800\n",
      "280900\n",
      "281000\n",
      "281100\n",
      "281200\n",
      "281300\n",
      "281400\n",
      "281500\n",
      "281600\n",
      "281700\n",
      "281800\n",
      "281900\n",
      "282000\n",
      "282100\n",
      "282200\n",
      "282300\n",
      "282400\n",
      "282500\n",
      "282600\n",
      "282700\n",
      "282800\n",
      "282900\n",
      "283000\n",
      "283100\n",
      "283200\n",
      "283300\n",
      "283400\n",
      "283500\n",
      "283600\n",
      "283700\n",
      "283800\n",
      "283900\n",
      "284000\n",
      "284100\n",
      "284200\n",
      "284300\n",
      "284400\n",
      "284500\n",
      "284600\n",
      "284700\n",
      "284800\n",
      "284900\n",
      "285000\n",
      "285100\n",
      "285200\n",
      "285300\n",
      "285400\n",
      "285500\n",
      "285600\n",
      "285700\n",
      "285800\n",
      "285900\n",
      "286000\n",
      "286100\n",
      "286200\n",
      "286300\n",
      "286400\n",
      "286500\n",
      "286600\n",
      "286700\n",
      "286800\n",
      "286900\n",
      "287000\n",
      "287100\n",
      "287200\n",
      "287300\n",
      "287400\n",
      "287500\n",
      "287600\n",
      "287700\n",
      "287800\n",
      "287900\n",
      "288000\n",
      "288100\n",
      "288200\n",
      "288300\n",
      "288400\n",
      "288500\n",
      "288600\n",
      "288700\n",
      "288800\n",
      "288900\n",
      "289000\n",
      "289100\n",
      "289200\n",
      "289300\n",
      "289400\n",
      "289500\n",
      "289600\n",
      "289700\n",
      "289800\n",
      "289900\n",
      "290000\n",
      "290100\n",
      "290200\n",
      "290300\n",
      "290400\n",
      "290500\n",
      "290600\n",
      "290700\n",
      "290800\n",
      "290900\n",
      "291000\n",
      "291100\n",
      "291200\n",
      "291300\n",
      "291400\n",
      "291500\n",
      "291600\n",
      "291700\n",
      "291800\n",
      "291900\n",
      "292000\n",
      "292100\n",
      "292200\n",
      "292300\n",
      "292400\n",
      "292500\n",
      "292600\n",
      "292700\n",
      "292800\n",
      "292900\n",
      "293000\n",
      "293100\n",
      "293200\n",
      "293300\n",
      "293400\n",
      "293500\n",
      "293600\n",
      "293700\n",
      "293800\n",
      "293900\n",
      "294000\n",
      "294100\n",
      "294200\n",
      "294300\n",
      "294400\n",
      "294500\n",
      "294600\n",
      "294700\n",
      "294800\n",
      "294900\n",
      "295000\n",
      "295100\n",
      "295200\n",
      "295300\n",
      "295400\n",
      "295500\n",
      "295600\n",
      "295700\n",
      "295800\n",
      "295900\n",
      "296000\n",
      "296100\n",
      "296200\n",
      "296300\n",
      "296400\n",
      "296500\n",
      "296600\n",
      "296700\n",
      "296800\n",
      "296900\n",
      "297000\n",
      "297100\n",
      "297200\n",
      "297300\n",
      "297400\n",
      "297500\n",
      "297600\n",
      "297700\n",
      "297800\n",
      "297900\n",
      "298000\n",
      "298100\n",
      "298200\n",
      "298300\n",
      "298400\n",
      "298500\n",
      "298600\n",
      "298700\n",
      "298800\n",
      "298900\n",
      "299000\n",
      "299100\n",
      "299200\n",
      "299300\n",
      "299400\n",
      "299500\n",
      "tf.Tensor(\n",
      "[[-0.19842073 -0.19370967 -0.14263028 ... -0.21206917 -0.09008998\n",
      "   0.18410945]\n",
      " [-0.20716491  0.00182543 -0.02729532 ... -0.01221294  0.11878032\n",
      "   0.29838005]\n",
      " [-0.08811823 -0.08947874 -0.28831694 ... -0.25582212  0.16233759\n",
      "   0.2839969 ]\n",
      " ...\n",
      " [-0.49217376  0.07384131 -0.32119355 ... -0.19626263  0.25841707\n",
      "   0.04164412]\n",
      " [-0.48063675  0.06226829 -0.313448   ... -0.39579505  0.33588988\n",
      "   0.42171335]\n",
      " [-0.17588721  0.00797858 -0.11338039 ... -0.08468726  0.24348748\n",
      "   0.06529083]], shape=(299616, 1024), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "premise_embed = batch_embed(train_df, 'premise')\n",
    "hypothesis_embed = batch_embed(train_df, 'hypothesis')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-01-27T23:27:54.899474Z",
     "iopub.status.busy": "2021-01-27T23:27:54.898531Z",
     "iopub.status.idle": "2021-01-27T23:27:54.966446Z",
     "shell.execute_reply": "2021-01-27T23:27:54.967853Z"
    },
    "papermill": {
     "duration": 1.078967,
     "end_time": "2021-01-27T23:27:54.968133",
     "exception": false,
     "start_time": "2021-01-27T23:27:53.889166",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"functional_1\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            [(None, 1024)]       0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_2 (InputLayer)            [(None, 1024)]       0                                            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate (Concatenate)       (None, 2048)         0           input_1[0][0]                    \n",
      "                                                                 input_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense (Dense)                   (None, 300)          614700      concatenate[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 300)          90300       dense[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 300)          90300       dense_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "flatten (Flatten)               (None, 300)          0           dense_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_3 (Dense)                 (None, 3)            903         flatten[0][0]                    \n",
      "==================================================================================================\n",
      "Total params: 796,203\n",
      "Trainable params: 796,203\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "import keras\n",
    "from keras import datasets, layers, models\n",
    "\n",
    "def create_elmo_model():\n",
    "    # Encoded token ids from BERT tokenizer.\n",
    "    premise = layers.Input(shape=(max_length,), dtype=tf.int32, name=\"premise\")\n",
    "\n",
    "    # Attention masks indicates to the model which tokens should be attended to.\n",
    "    hypothesis = layers.Input(shape=(max_length,), dtype=tf.int32, name=\"hypothesis\")\n",
    "    # Token type ids are binary masks identifying different sequences in the model.\n",
    "#     token_type_ids = layers.Input(shape=(max_length,), dtype=tf.int32, name=\"token_type_ids\")\n",
    "    \n",
    "    conc=layers.concatenate([premise,hypothesis])\n",
    "    y_1= tf.layers.Dense(300,activation='tanh')(conc)\n",
    "    y_2= tf.layers.Dense(300,activation='tanh')(y_1)\n",
    "    y_3= tf.layers.Dense(300,activation='tanh')(y_2)\n",
    "\n",
    "    output=tf.layers.Flatten()(y_3)\n",
    "    output = tf.layers.Dense(3, activation='softmax')(output)\n",
    "\n",
    "    model = keras.models.Model([premise, hypothesis], output)\n",
    "    model.summary()\n",
    "    return model\n",
    "\n",
    "def create_elmo_model2():\n",
    "    premise = layers.Input(shape=(1024,), dtype=\"float\")\n",
    "    hypothesis = layers.Input(shape=(1024,), dtype=\"float\")\n",
    "    conc=layers.concatenate([premise,hypothesis])\n",
    "    y_1= layers.Dense(300,activation='tanh')(conc)\n",
    "    y_2= layers.Dense(300,activation='tanh')(y_1)\n",
    "    y_3= layers.Dense(300,activation='tanh')(y_2)\n",
    "    flat =layers.Flatten()(y_3)\n",
    "\n",
    "    output = layers.Dense(3, activation='softmax')(flat)\n",
    "\n",
    "    model = tf.keras.models.Model([premise, hypothesis], output)\n",
    "    model.summary()\n",
    "    return model\n",
    "model = create_elmo_model2()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-01-27T23:27:56.995072Z",
     "iopub.status.busy": "2021-01-27T23:27:56.993378Z",
     "iopub.status.idle": "2021-01-27T23:36:18.568033Z",
     "shell.execute_reply": "2021-01-27T23:36:18.568604Z"
    },
    "papermill": {
     "duration": 502.612811,
     "end_time": "2021-01-27T23:36:18.568739",
     "exception": false,
     "start_time": "2021-01-27T23:27:55.955928",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/40\n",
      "3746/3746 [==============================] - 12s 3ms/step - loss: 1.0555 - accuracy: 0.4457 - val_loss: 1.0108 - val_accuracy: 0.5070\n",
      "Epoch 2/40\n",
      "3746/3746 [==============================] - 12s 3ms/step - loss: 0.9843 - accuracy: 0.5303 - val_loss: 0.9620 - val_accuracy: 0.5443\n",
      "Epoch 3/40\n",
      "3746/3746 [==============================] - 13s 3ms/step - loss: 0.9466 - accuracy: 0.5548 - val_loss: 0.9332 - val_accuracy: 0.5620\n",
      "Epoch 4/40\n",
      "3746/3746 [==============================] - 12s 3ms/step - loss: 0.9232 - accuracy: 0.5692 - val_loss: 0.9144 - val_accuracy: 0.5742\n",
      "Epoch 5/40\n",
      "3746/3746 [==============================] - 12s 3ms/step - loss: 0.9072 - accuracy: 0.5792 - val_loss: 0.9013 - val_accuracy: 0.5825\n",
      "Epoch 6/40\n",
      "3746/3746 [==============================] - 12s 3ms/step - loss: 0.8956 - accuracy: 0.5869 - val_loss: 0.8916 - val_accuracy: 0.5882\n",
      "Epoch 7/40\n",
      "3746/3746 [==============================] - 12s 3ms/step - loss: 0.8867 - accuracy: 0.5922 - val_loss: 0.8840 - val_accuracy: 0.5932\n",
      "Epoch 8/40\n",
      "3746/3746 [==============================] - 13s 3ms/step - loss: 0.8796 - accuracy: 0.5972 - val_loss: 0.8779 - val_accuracy: 0.5980\n",
      "Epoch 9/40\n",
      "3746/3746 [==============================] - 12s 3ms/step - loss: 0.8738 - accuracy: 0.6007 - val_loss: 0.8733 - val_accuracy: 0.6021\n",
      "Epoch 10/40\n",
      "3746/3746 [==============================] - 12s 3ms/step - loss: 0.8690 - accuracy: 0.6041 - val_loss: 0.8686 - val_accuracy: 0.6043\n",
      "Epoch 11/40\n",
      "3746/3746 [==============================] - 13s 3ms/step - loss: 0.8648 - accuracy: 0.6071 - val_loss: 0.8650 - val_accuracy: 0.6072\n",
      "Epoch 12/40\n",
      "3746/3746 [==============================] - 12s 3ms/step - loss: 0.8611 - accuracy: 0.6095 - val_loss: 0.8618 - val_accuracy: 0.6079\n",
      "Epoch 13/40\n",
      "3746/3746 [==============================] - 12s 3ms/step - loss: 0.8579 - accuracy: 0.6115 - val_loss: 0.8594 - val_accuracy: 0.6118\n",
      "Epoch 14/40\n",
      "3746/3746 [==============================] - 13s 3ms/step - loss: 0.8551 - accuracy: 0.6132 - val_loss: 0.8565 - val_accuracy: 0.6121\n",
      "Epoch 15/40\n",
      "3746/3746 [==============================] - 12s 3ms/step - loss: 0.8526 - accuracy: 0.6149 - val_loss: 0.8545 - val_accuracy: 0.6146\n",
      "Epoch 16/40\n",
      "3746/3746 [==============================] - 12s 3ms/step - loss: 0.8502 - accuracy: 0.6162 - val_loss: 0.8525 - val_accuracy: 0.6155\n",
      "Epoch 17/40\n",
      "3746/3746 [==============================] - 12s 3ms/step - loss: 0.8481 - accuracy: 0.6173 - val_loss: 0.8506 - val_accuracy: 0.6163\n",
      "Epoch 18/40\n",
      "3746/3746 [==============================] - 12s 3ms/step - loss: 0.8462 - accuracy: 0.6188 - val_loss: 0.8488 - val_accuracy: 0.6177\n",
      "Epoch 19/40\n",
      "3746/3746 [==============================] - 13s 3ms/step - loss: 0.8444 - accuracy: 0.6198 - val_loss: 0.8476 - val_accuracy: 0.6172\n",
      "Epoch 20/40\n",
      "3746/3746 [==============================] - 12s 3ms/step - loss: 0.8427 - accuracy: 0.6208 - val_loss: 0.8463 - val_accuracy: 0.6193\n",
      "Epoch 21/40\n",
      "3746/3746 [==============================] - 12s 3ms/step - loss: 0.8412 - accuracy: 0.6215 - val_loss: 0.8448 - val_accuracy: 0.6181\n",
      "Epoch 22/40\n",
      "3746/3746 [==============================] - 13s 3ms/step - loss: 0.8397 - accuracy: 0.6223 - val_loss: 0.8432 - val_accuracy: 0.6203\n",
      "Epoch 23/40\n",
      "3746/3746 [==============================] - 12s 3ms/step - loss: 0.8384 - accuracy: 0.6226 - val_loss: 0.8420 - val_accuracy: 0.6210\n",
      "Epoch 24/40\n",
      "3746/3746 [==============================] - 12s 3ms/step - loss: 0.8370 - accuracy: 0.6236 - val_loss: 0.8411 - val_accuracy: 0.6218\n",
      "Epoch 25/40\n",
      "3746/3746 [==============================] - 13s 3ms/step - loss: 0.8358 - accuracy: 0.6241 - val_loss: 0.8402 - val_accuracy: 0.6231\n",
      "Epoch 26/40\n",
      "3746/3746 [==============================] - 12s 3ms/step - loss: 0.8347 - accuracy: 0.6245 - val_loss: 0.8391 - val_accuracy: 0.6224\n",
      "Epoch 27/40\n",
      "3746/3746 [==============================] - 12s 3ms/step - loss: 0.8336 - accuracy: 0.6252 - val_loss: 0.8383 - val_accuracy: 0.6230\n",
      "Epoch 28/40\n",
      "3746/3746 [==============================] - 13s 4ms/step - loss: 0.8325 - accuracy: 0.6260 - val_loss: 0.8372 - val_accuracy: 0.6237\n",
      "Epoch 29/40\n",
      "3746/3746 [==============================] - 12s 3ms/step - loss: 0.8315 - accuracy: 0.6268 - val_loss: 0.8366 - val_accuracy: 0.6243\n",
      "Epoch 30/40\n",
      "3746/3746 [==============================] - 12s 3ms/step - loss: 0.8305 - accuracy: 0.6272 - val_loss: 0.8360 - val_accuracy: 0.6265\n",
      "Epoch 31/40\n",
      "3746/3746 [==============================] - 14s 4ms/step - loss: 0.8296 - accuracy: 0.6277 - val_loss: 0.8350 - val_accuracy: 0.6264\n",
      "Epoch 32/40\n",
      "3746/3746 [==============================] - 12s 3ms/step - loss: 0.8287 - accuracy: 0.6285 - val_loss: 0.8344 - val_accuracy: 0.6260\n",
      "Epoch 33/40\n",
      "3746/3746 [==============================] - 12s 3ms/step - loss: 0.8279 - accuracy: 0.6293 - val_loss: 0.8338 - val_accuracy: 0.6255\n",
      "Epoch 34/40\n",
      "3746/3746 [==============================] - 14s 4ms/step - loss: 0.8271 - accuracy: 0.6298 - val_loss: 0.8329 - val_accuracy: 0.6259\n",
      "Epoch 35/40\n",
      "3746/3746 [==============================] - 12s 3ms/step - loss: 0.8262 - accuracy: 0.6301 - val_loss: 0.8318 - val_accuracy: 0.6275\n",
      "Epoch 36/40\n",
      "3746/3746 [==============================] - 12s 3ms/step - loss: 0.8255 - accuracy: 0.6305 - val_loss: 0.8317 - val_accuracy: 0.6264\n",
      "Epoch 37/40\n",
      "3746/3746 [==============================] - 14s 4ms/step - loss: 0.8247 - accuracy: 0.6309 - val_loss: 0.8307 - val_accuracy: 0.6291\n",
      "Epoch 38/40\n",
      "3746/3746 [==============================] - 12s 3ms/step - loss: 0.8240 - accuracy: 0.6316 - val_loss: 0.8305 - val_accuracy: 0.6292\n",
      "Epoch 39/40\n",
      "3746/3746 [==============================] - 13s 3ms/step - loss: 0.8233 - accuracy: 0.6317 - val_loss: 0.8295 - val_accuracy: 0.6294\n",
      "Epoch 40/40\n",
      "3746/3746 [==============================] - 14s 4ms/step - loss: 0.8226 - accuracy: 0.6323 - val_loss: 0.8292 - val_accuracy: 0.6295\n"
     ]
    }
   ],
   "source": [
    "model.compile(optimizer='Adadelta',loss='categorical_crossentropy',metrics = ['accuracy'])\n",
    "history = model.fit([np.array(premise_embed), np.array(hypothesis_embed)],\n",
    "                    y_train,\n",
    "                    epochs=epochs,\n",
    "                    batch_size=batch_size,\n",
    "                    validation_split=0.2,\n",
    "                    verbose=1,\n",
    "                    use_multiprocessing=True,\n",
    "                    workers=-1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-01-27T23:36:25.176915Z",
     "iopub.status.busy": "2021-01-27T23:36:25.176353Z",
     "iopub.status.idle": "2021-01-27T23:36:26.187591Z",
     "shell.execute_reply": "2021-01-27T23:36:26.187070Z"
    },
    "papermill": {
     "duration": 4.254928,
     "end_time": "2021-01-27T23:36:26.187702",
     "exception": false,
     "start_time": "2021-01-27T23:36:21.932774",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "77/77 [==============================] - 0s 2ms/step - loss: 0.8159 - accuracy: 0.6340\n",
      "[0.8159200549125671, 0.6339576840400696]\n",
      "tf.Tensor([2 1 0 ... 0 1 2], shape=(9824,), dtype=int64)\n",
      "tf.Tensor([1 1 0 ... 0 1 2], shape=(9824,), dtype=int64)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7eff1009d750>"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAl4AAAJ1CAYAAAAISv5zAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3debhVVf0/8Pe6gKmgJc7irCDOllOmpjilZjngPGVpaKallmlpmmlZWZqZZmhqmvOMU2aaNqiJ5pAz5IjlmIokIBf27w+ILzPUr7M3eF6vnvM89+y99r571eny4f1ZrFuqqgoAAK3X0fQDAAC0C4UXAEBNFF4AADVReAEA1EThBQBQE4UXAEBNurb6G/Tc5xL7VTBbeOC0nZp+BJhonrm6NP0IMNFi83crTX3veT58SCN1wsgHf9rInCVeAAA1UXgBANSk5a1GAIDpKu2VAbXXbAEAGiTxAgCaUxpb198IiRcAQE0kXgBAc6zxAgCgFRReAAA10WoEAJpjcT0AAK0g8QIAmmNxPQAArSDxAgCaY40XAACtoPACAKiJViMA0ByL6wEAaAWJFwDQHIvrAQBoBYkXANAca7wAAGgFhRcAQE20GgGA5lhcDwBAK0i8AIDmWFwPAEArSLwAgOZY4wUAQCsovAAAaqLVCAA0x+J6AABaQeIFADRH4gUAQCtIvACA5nTYTgIAgBZQeAEA1ESrEQBojsX1AAC0gsQLAGiO39UIAEArSLwAgOZY4wUAQCsovAAAaqLVCAA0x+J6AABaQeIFADTH4noAAFpB4gUANMcaLwAAWkHhBQBQE61GAKA5FtcDANAKEi8AoDkW1wMA0AoSLwCgOdZ4AQDQCgovAICaaDUCAM2xuB4AgFaQeAEAzbG4HgCAVpB4AQDNkXgBANAKCi8AgJpoNQIAzbGdBAAArSDxAgCaY3E9AACtIPECAJpjjRcAAK2g8AIAqIlWIwDQHIvrAQBoBYkXANAci+sBAGgFiRcA0Jgi8QIAoBUUXgAANdFqBAAao9UIAEBLSLwAgOa0V+Al8QIAqIvECwBojDVeAAC0hMILAKAmWo0AQGO0GgEAaAmJFwDQGIkXAECbK6VsXUp5qpQytJRy9HTGbFpKeaiU8lgp5a5Zua/ECwBozOyYeJVSuiQ5M8mWSYYlGVxKGVRV1eOTjPlQkrOSbF1V1QullEVm5d4SLwCAya2XZGhVVc9UVfVeksuSbD/FmD2TXFNV1QtJUlXVq7NyY4UXAMDkeiV5cZL3wyYcm1SfJAuUUu4spTxQStl3Vm6s1QgANKehTmMpZUCSAZMcGlhV1cB/n57GJdUU77smWTvJ5knmSXJPKeXeqqqentH3VXjV7KFTP50RozozdlyVzrHjsvnxt2bVpT+UU/dbL93n7poXXv9XDjzrT3lnVOc0r+8oJXd8+xP5x5sjs8ep49fxHb/bWtlijcXz1xfeysE/vydJsuuGy2aB7h/Iz3/zVG1zY87y2isv55QTj8mb/3wjpZRsu/3O2WHXvfL7O36TX/3iZ3nx+Wdz+jkXp8/Kq07z+hHvDM+Pv3dCnntmaEopOfwbJ2SV1dbML846LYPv/VNW6L1Sjvzmd5Ikv/31DRkxfHh22HWvOqfIHOR73z429/zx91lggZ654PLrkiRDn34yP/reiRn57rtZbPEl8s0Tv5/uPXrM0rVJcvYZp+bPd/8hK/bpm2NOODlJcuvNg/LO229n5z32qWdizLYmFFkDp3N6WJKlJnm/ZJK/T2PM61VV/SvJv0opv0+yZpIZFl5ajQ349HdvzybH3pLNj781SXL6/uvnhCseykbfuDk33f9iDv3kKtO99qBPrJSn/z584vv55umW9XovlI2PuSVdOkpWXvKDmbtbl+yx8fL5xe0z/N+eNtfRpUs+f+hXc84l1+XHA3+VG665LM8/+7csu/yK+eZ3T8tqa609w+vP/vEPsvb6G+bcS6/PWb+8Mksvs1z+NeKdPP7Xh3P2hVdl3NhxefZvQzJ69Kj89uZB2W6nXWuaGXOibbbbIaf85OzJjv3gpONz4BcPywWXXZuN+22eyy46f5avHTHinTz6yEM5/9JrM27suPxt6NMZPWpUfn3D9dlhl91bNg/+c6WURl4zMThJ71LKcqWUuZLsnmTQFGOuT7JxKaVrKWXeJOsneWJmN55p4VVK6VtKOaqU8pNSyukTvl55Ztcx63ovPn/ufnL8mrw7H305n1p3qWmOW2KBebLlWkvkorv+NvFYVVWZq+v4/xnn7tYlnWOrHPrJlTPwN0+lc+yUqSj8nwUXWji9Vxr/f+V5u3fPUsssnzdeezVLL7t8llpm2Rle+69/jchfH34gW39qxyRJt27d0mO++VNKRzo7x6SqqowePSpdu3bNVRdfkO132TNdu3Zr8YyYk635kXUy3/wfnOzYiy88lzU/sk6SZN31Nshdv7ttlq/tKB3pHDP5Z/HSi85P/9338llkpqqq6kxySJJbM76YuqKqqsdKKQeVUg6aMOaJJL9O8kiS+5KcW1XVozO79wwLr1LKURm/kr9MuOngCV9fOr09LZixKsnVR/XLHd/eOp/pt0KS5Ilhb2Wbj4xfs7f9ektniZ7zTvPa7+69dr512YMZN+7/CqoRozozaPCLueukbfLCayMy/N338uHlF8wtf3mp5XPh/ePlf7yUvw15MiutuvqsjX9pWD74oQXyo+8cly/ut2tOO/lbGTXy3czbvXs22nSLfHG/3bLYEr0yb/ceefrJx7LBxv1aPAPej5ZbfsX86fe/S5L87vbf5NVXXp7la+ft3j0f32zLHLDXzll8iSXTo8d8efLxR7PRJpu16nH5L82miVeqqrq5qqo+VVWtUFXVdyYcO7uqqrMnGXNKVVWrVFW1WlVVP56V+c5sjdf+SVatqmrMFP8lnZrksSTfm9ZFky5Ym3f9/fOB3j7o/7bNt2/Ly2+NzELzfyDXHLVZnv778Bx6zp/zvX3WzpE7rJ5fPzgsYzrHTXXdVmstkdeGj8rDz72ZDftOvlXIGTc9kTNuGp9unr7/ejn56keyzyYrpN/qi+WxF9/Kj65/rJa5MWca+e67OemYr+TALx2Z7t2nXj8zLWPHjs3Qp5/MwYcfnb6rrpGf/fj7ufyi8/KZAYdkl70+m132+myS5LSTv5V9Djg4twy6Jn8ZfE+WW6F39txvwEzuDuMdddyJ+ckPT84vzz07G35803Tr9p8lVXvu+7nsue/nkiQ/OOm4fO6gQ3LjdVdl8J/vyQor9sm++x/YiseGGZpZq3FckiWmcXzxCeemqaqqgVVVrVNV1TqKrsm9/NbIJMnrw0fnpvuHZe0VFsyQfwxP/x/8Lpsd9+tcfc/zefbVEVNdt36fhbPNR5bMQ6d+Oud+ccNsvMqiOfugDSYbs/oyCyRJ/vby8Oy20XL53E//lJWX/FCWX3S+1k+MOVJn55iceMwR6bfVttlo0y1m+bqFFlk0Cy28aPquukaSZONNt8zQp5+cbMzQp8f/ZWDJpZbJ7b++IceceEqef2ZoXnrx+f/dBHhfW2bZ5fOjn56Tcy66IptvtW2W6DXtZRgz8/RT4z+LSy29TG69+YaccPKP8uzfhmTYCz6L1G9middhSW4vpQzJ/+1nsXSSFTO+98l/YN4PdElHKRkxqjPzfqBL+q2+WE659tEsNP8H8vrw0Skl+cr2q+WCO4ZMde2JVzycE694OEmyYd9Fcsi2K+egs++ZbMw3+q+Rw8+7L127dKRLx/gYddy4KvN8oEvrJ8ccp6qqnHbyt7L0Msun/+6ztP3MRD0XXCgLL7JoXnz+uSy1zLJ58IE/Z+lll59szIXnnJkvf+24dHZ2Zty48X9PKx0dGT1q1P9sDry/vfnPN7JAzwUzbty4XHjez/Pp/v/dP9A47+wz8tVvfGv8Z3Hs2CTjP4ujRo38Xz4u/6XZcef6Vpph4VVV1a9LKX0yfgfXXhm/vmtYksFVVY2t4fneVxaef+5cdNjHkyRdO0quuuf53P7Xf+TArVbK/lv0TpLceP+Lufj3zyRJFvvQPDn9gPWz2w/vnOm9t117yTz47BsTE7XBQ1/PH7+7bR578a089sJbrZkQc7THHnkwt//6xiy7Qu8c/Jnxf6Dtd+ChGTPmvfzstO/l7bfezHFHHpLle6+U7552dt547dX8+Hsn5MQfnZkkOfjwo/ODE76eMZ1jsvgSS+aIb3x74r3v/v0d6bPyallw4fFt8ZVXWyMH7dM/y63QJ8v3Xqn+yTLbO+GYI/PQA4Pz9ltvZedPbp7PDjg4I999N9dedVmS5OObbpFtJ/xjjtdfezU/OOn4/OD0n0332k9u3z9J8oc7b0/fVVbLQhM+i6uuvmb2233HrLBin6zYp28DM6Xdlapq7b9867nPJf5pHbOFB07bqelHgInmmUsSzexjsfm7NRY7LbjvpY3UCW9cuEcjc7aPFwBATexcDwA0p72WeEm8AADqovACAKiJViMA0Jh2205C4gUAUBOJFwDQGIkXAAAtIfECABoj8QIAoCUUXgAANdFqBACa016dRokXAEBdJF4AQGMsrgcAoCUkXgBAYyReAAC0hMILAKAmWo0AQGO0GgEAaAmJFwDQGIkXAAAtIfECAJrTXoGXxAsAoC4KLwCAmmg1AgCNsbgeAICWkHgBAI2ReAEA0BISLwCgMRIvAABaQuEFAFATrUYAoDnt1WmUeAEA1EXiBQA0xuJ6AABaQuIFADRG4gUAQEsovAAAaqLVCAA0RqsRAICWkHgBAI2ReAEA0BISLwCgOe0VeEm8AADqovACAKiJViMA0BiL6wEAaAmJFwDQGIkXAAAtIfECABrTZoGXxAsAoC4KLwCAmmg1AgCNsbgeAICWkHgBAI1ps8BL4gUAUBeFFwBATbQaAYDGWFwPAEBLSLwAgMa0WeAl8QIAqIvECwBoTEdHe0VeEi8AgJoovAAAaqLVCAA0xuJ6AABaQuIFADTGBqoAALSExAsAaEybBV4SLwCAuii8AABqotUIADTG4noAAFpC4gUANEbiBQBAS0i8AIDGtFngJfECAKiLwgsAoCZajQBAYyyuBwCgJSReAEBj2izwkngBANRF4gUANMYaLwAAWkLhBQBQE61GAKAxbdZplHgBANRF4gUANMbiegAAWkLiBQA0ps0CL4kXAEBdFF4AADXRagQAGmNxPQAALdHyxOvZgbu3+lvALFnsY19q+hFgojcH/7TpR4DZQpsFXhIvAIC6KLwAgMaUUhp5zcJzbV1KeaqUMrSUcvQ0zm9aSnm7lPLQhNdxszJfi+sBACZRSumS5MwkWyYZlmRwKWVQVVWPTzH0D1VVbfef3FviBQAwufWSDK2q6pmqqt5LclmS7f8XN1Z4AQCNKaWZ10z0SvLiJO+HTTg2pQ1KKQ+XUm4ppaw6K/NVeAEAbaeUMqCUcv8krwGTnp7GJdUU7/+SZJmqqtZMckaS62bl+1rjBQA0pqkNVKuqGphk4HROD0uy1CTvl0zy9ymuHz7J1zeXUs4qpSxUVdXrM/q+Ei8AgMkNTtK7lLJcKWWuJLsnGTTpgFLKYmVC1VhKWS/ja6o3ZnZjiRcA0JjZcQPVqqo6SymHJLk1SZck51VV9Vgp5aAJ589OsnOSL5RSOpOMTLJ7VVVTtiOnovACAJhCVVU3J7l5imNnT/L1T5P8x7+CQqsRAKAmEi8AoDFNLa5visQLAKAmEi8AoDESLwAAWkLiBQA0ps0CL4kXAEBdFF4AADXRagQAGmNxPQAALSHxAgAa02aBl8QLAKAuEi8AoDHWeAEA0BIKLwCAmmg1AgCNabNOo8QLAKAuEi8AoDEdbRZ5SbwAAGoi8QIAGtNmgZfECwCgLgovAICaaDUCAI2xcz0AAC0h8QIAGtPRXoGXxAsAoC4SLwCgMdZ4AQDQEgovAICaaDUCAI1ps06jxAsAoC4SLwCgMSXtFXlJvAAAaiLxAgAaYwNVAABaQuEFAFATrUYAoDF2rgcAoCUkXgBAY9os8JJ4AQDUReIFADSmo80iL4kXAEBNFF4AADXRagQAGtNmnUaJFwBAXSReAEBjbKAKAEBLSLwAgMa0WeAl8QIAqIvCCwCgJlqNAEBj7FwPAEBLSLwAgMa0V94l8QIAqI3ECwBojA1UAQBoCYUXAEBNtBoBgMZ0tFenUeIFAFAXiRcA0BiL6wEAaAmJFwDQmDYLvCReAAB1UXgBANREqxEAaIzF9QAAtITECwBojA1UAQBoCYkXANAYa7wAAGgJhRcAQE20GgGAxrRXo1HiBQBQG4kXANCYDovrAQBoBYkXANCYNgu8JF4AAHVReAEA1ESrEQBojJ3rAQBoCYkXANCYNgu8JF4AAHWReAEAjbGBKgAALaHwqtGJxx+TT/TbMLv3/9Rkxy+/9FfZefttsttO2+Unp50yzWsvueiC7LbTdtm9/6dy7NFfyejRo5MkZ/z4h9lzl+1z/LFHTRx7843X57KLL2zdRHjf+GCPeXLJKfvnoWuOzYNXH5v111gua/Tplbt++ZXce9nR+ePFX8s6qy4z3es7OkruufSoXH36QROPnfSl7XPf5V/PuSfuM/HYHp9cN1/cY9NWToU53HHHfj2bbrxBdtp+u6nO/fL8X2TNVVfKm2/+c5rXXvTLC7Ljpz+ZnbbfLkd99YiJPx9P+9Ep2XnHT+WYr39t4tgbBl2Xiy/6ZWsmAbNA4VWjT356h5x+1sDJjt0/+M/5/Z2355Irr8/l19yYvT/zuamue/WVV3L5pb/KLy+5KpddfUPGjh2X2359c0a8804eefihXHLl9Rk3dlyGDnk6o0aNyo2DrsvOu+5R17SYg/3wazvnN3c/nrV2Oinr7XZynnzm5XznsB3ynYG35KO7fy8n/uzGfOewHaZ7/SF79stTz74y8f38PebOR9dcLuvtdnK6dHRk1RWXyNwf6JZ9PvXR/PzK39cxJeZQ2++wU37283OnOv7yP/6Re+6+O4svvsQ0r3vllVdyycUX5tIrrs4119+YcePG5tc335R33nknDz/0YK669oaMGzs2Q55+KqNGjcqg667Nrrvv2erp8B8opZlXUxReNfrI2utm/vk/NNmxq6+4LJ/57Ocz11xzJUl69lxwmteOHTs2o0ePSmdnZ0aNGpmFFl4kpaMjnWPGpKqqjB49Kl27ds2vfvmL7LbH3unarVvL58Ocbb7uc2ejj6yQC669J0kypnNs3h4xMlWVzN997iTjE7F/vPb2NK/vtciHsvVGq+b8a++eeGzcuCpzdRu/dHSeD3TLmM6xOfwzm+esy+5MZ+e4Fs+IOdna66yb+T/4wamOn/L9k3P4V46c4V5PY8eOzehR438+jhw1Kgsvskg6OkrGTPj5OGr06HTt2jUXnHdu9tx7n3Tz85EGKbwa9sLzz+WhvzyQz+69Ww7cf588/uhfpxqzyKKLZu99P5tPb715tt3y4+nRY7589GMbpnv37um3+ZbZe7edskSvJdOjR488/tij2aTf5g3MhDnNcr0WzOtvjsjAE/bOPZcelbOO2zPzzj1XjvzhVfnuYTtkyC0n5uTDd8xxZ1w/zetPObJ/jjn9uowbV008NuLd0bnu9ody72VH57m/v5HhI0Zm7VWWyY13Tv25hpm5847bs8iii2Slvn2nO2bRRRfNZ/b7XD6xRb9sselGma9Hj3xsw43SvXuPbLHlVtmt/w7p1WvJ9Jhvvjz26KPpt9kWNc6AWVFKaeTVlP+68CqlfPZ/+SDtauzYzgx/Z3jOu+iyfOmwI/P1rx2eqqomGzN8+Nu56847ct1Nt+Xm39yVkSNH5pabBiVJ9v3sAbn4imtz2FeOytln/SQHHnxorrvmynz9yMPzi3N+1sSUmEN07dola/VdKudc+YdssMf38+7I0fnq57bMgF02ztd+dE16b/PNfO2HV+dnx+811bXbbLxaXv3nO3nwiRenOnfqL3+bj+7+vRx96rU57uDtcuLPbsp+O26QX33/cznqgE/UMTXeB0aOHJlzBp6dgw/58gzHDX/77fzujttz829uz22/+0NGjhyZG28Y/5eFz+7/+VxxzfX56teOzplnnJ6DD/1Srrnqyhx5xJcz8Oyz6pgGTOX/J/E6YXonSikDSin3l1Luv+AXA6c3jCSLLLpY+m22ZUopWXX1NdLR0ZG33nxzsjH33XtPlujVKwv07Jmu3bql3+Zb5JGHHpxszFNPPp4kWXqZZXPzjdfn5FNOyzNDh+SF55+rayrMYV565c289OpbGfzo80mSa3/7UNbqu1T22m79XHf7Q0mSq297cJqL6zdYa/lst8nqefKmE3Lh9z6bTdftk/NO2neyMWuutGSSZMjzr2av7dbP3kedl1VXXCIrLL1wi2fG+8GwF1/ISy8Ny647bZ9tttwsr7zycnbfeae8/tprk427996702vJJdOzZ89069Ytm2+xVR5+cPKfj088Mf7n4zLLLJsbBl2XU049PUOHDsnzfj7OFjoaejVlhvt4lVIemd6pJItO77qqqgYmGZgkb48cV01vHMkm/TbP/YPvzdrrrpfnn382Y8aMyYcWWGCyMYstvngefeThjBo5Mh+Ye+4M/vO9WXnV1SYbc/aZP8k3vvntdI7pzLix49fSlNKRUaNG1TYX5iyvvPFOhr38Znovs0iGPP9qNl1vpTz5zMtZrtdC2Xjt3vnDA0Oy6Xp9MvSF16a69rgzBuW4M8anrhuv3TuH7bt5Pnfs5P+S9riDt8shJ12abl27pEvH+Fh/3Lgq8849V+snxxyvd5+Vcucf7pn4fpstN8slV1yVBRboOdm4xRZfIo88/HBGjhyZueeeO3++956sstrkPx/PPOP0HPetb6ezszPjxo5NknSUjowa6ecj9ZvZBqqLJvlEkjenOF6S3D31cGbk2KO/kgfuvy9vvfVWtttq03z+C4fk0zvslBOPPza79/9UunXrluNPPDmllLz26qv5zgnH5sdnDsxqq6+Zzbf4RPbZo3+6dOmSlfqunB377zrxvnfe8dussurqWXiRRZIkq6+5VvbY+dNZsfdK6bPS9NdGwBHfvzLnf3e/zNW1S5576fUMOP5XufHOR3LKkTuna9eOjB7dmUNOujRJsvjCH8xZx+2ZHQ+deQv7U5uukQcee37iwvw/P/JcBl/xjTw65KX89emXWjon5kxHffWI3D/4vrz11pvZcrOP5wtfPDQ79d9lmmNfffWVnHDcsTnz7HOyxhprZsutPpHdd9kxXbp0Td+VV87Ou+w2cewdt/82q622ehZZZHxWsMZaH07/HT6VPn36zHDtGLRKmXI90WQnS/lFkvOrqvrjNM5dUlXVTP9NrsSL2cViH/tS048AE705+KdNPwJMNHfXNLba/EvXPdlInfCTHfo2MucZJl5VVe0/g3M2QgEA+A/4XY0AQGM62utXNdrHCwCgLhIvAKAxEi8AAFpC4QUAUBOtRgCgMU3+3sQmSLwAAGoi8QIAGmNxPQAALaHwAgAaU0ozr5k/V9m6lPJUKWVoKeXoGYxbt5QytpSy86zMV+EFADCJUkqXJGcm2SbJKkn2KKWsMp1x309y66zeW+EFADC59ZIMrarqmaqq3ktyWZLtpzHu0CRXJ3l1Vm9scT0A0JiOhraTKKUMSDJgkkMDq6oaOOHrXklenOTcsCTrT3F9ryQ7Jtksybqz+n0VXgBA25lQZA2czulpVYPVFO9/nOSoqqrG/id7kSm8AIDGzKZrnoYlWWqS90sm+fsUY9ZJctmEomuhJNuWUjqrqrpuRjdWeAEATG5wkt6llOWSvJRk9yR7Tjqgqqrl/v11KeWCJDfOrOhKFF4AQINmx98YVFVVZynlkIz/14pdkpxXVdVjpZSDJpw/+7+9t8ILAGAKVVXdnOTmKY5Ns+Cqqmq/Wb3vbNpaBQB4/5F4AQCNaWo7iaZIvAAAaiLxAgAa02aBl8QLAKAuEi8AoDEdEi8AAFpB4QUAUBOtRgCgMbaTAACgJSReAEBj2izwkngBANRF4gUANMZ2EgAAtITCCwCgJlqNAEBjStqr1yjxAgCoicQLAGiMxfUAALSExAsAaIzECwCAllB4AQDURKsRAGhMabNf1ijxAgCoicQLAGiMxfUAALSExAsAaEybLfGSeAEA1EXhBQBQE61GAKAxHW3Wa5R4AQDUROIFADTGdhIAALSExAsAaEybLfGSeAEA1EXhBQBQE61GAKAxHWmvXqPECwCgJhIvAKAxFtcDANASEi8AoDE2UAUAoCUUXgAANdFqBAAa09Fmq+slXgAANZF4AQCNabPAS+IFAFAXiRcA0BhrvAAAaAmFFwBATbQaAYDGtFmnUeIFAFAXiRcA0Jh2S4Dabb4AAI2ReAEAjSlttshL4gUAUBOFFwBATbQaAYDGtFejUeIFAFAbiRcA0Bi/qxEAgJaQeAEAjWmvvEviBQBQG4UXAEBNtBoBgMa02dp6iRcAQF0kXgBAY/yuRgAAWkLiBQA0pt0SoHabLwBAYxReAAA10WoEABpjcT0AAC0h8QIAGtNeeZfECwCgNhIvAKAx7bbGq+WF1zujxrT6W8AseemPpzf9CDDRAv1/3vQjwEQjrz+w6UdoG1qNAAA10WoEABrTbglQu80XAKAxEi8AoDHttrhe4gUAUBOJFwDQmPbKuyReAAC1UXgBANREqxEAaEybra2XeAEA1EXiBQA0pqPNltdLvAAAaiLxAgAaY40XAAAtofACAKiJViMA0JhicT0AAK0g8QIAGmNxPQAALSHxAgAaYwNVAABaQuEFAFATrUYAoDEW1wMA0BISLwCgMRIvAIA2V0rZupTyVCllaCnl6Gmc376U8kgp5aFSyv2llI1m5b4SLwCgMbPjrwwqpXRJcmaSLZMMSzK4lDKoqqrHJxl2e5JBVVVVpZQ1klyRpO/M7i3xAgCY3HpJhlZV9UxVVe8luSzJ9pMOqKpqRFVV1YS33ZNUmQUKLwCAyfVK8uIk74dNODaZUsqOpZQnk9yU5HOzcmOFFwDQmI7SzKuUMmDC2qx/vwZM8ljT6n9OlWhVVXVtVVV9k+yQ5MRZma81XgBA26mqamCSgdM5PSzJUpO8XzLJ32dwr9+XUlBYxN4AABCsSURBVFYopSxUVdXrM/q+Ei8AoDGlof/MxOAkvUspy5VS5kqye5JBkz13KSuWMn4zjFLKR5LMleSNmd1Y4gUAMImqqjpLKYckuTVJlyTnVVX1WCnloAnnz07SP8m+pZQxSUYm2W2SxfbTpfACABozu26gWlXVzUlunuLY2ZN8/f0k3/9P76vVCABQE4UXAEBNtBoBgMbMjjvXt5LECwCgJhIvAKAxHe0VeEm8AADqIvECABpjjRcAAC2h8AIAqIlWIwDQmNl15/pWkXgBANRE4gUANKbNAi+JFwBAXSReAEBjOtpskZfECwCgJgovAICaaDUCAI1pr0ajxAsAoDYSLwCgOW0WeUm8AABqIvECABpT2izykngBANRE4QUAUBOtRgCgMW22cb3ECwCgLhIvAKAxbRZ4SbwAAOoi8QIAmtNmkZfECwCgJgovAICaaDUCAI2xcz0AAC0h8QIAGmMDVQAAWkLiBQA0ps0CL4kXAEBdFF4AADXRagQAmtNmvUaJFwBATSReAEBjbKAKAEBLKLwAAGqi1QgANMbO9QAAtITECwBoTJsFXhIvAIC6SLwAgOa0WeQl8QIAqInCCwCgJlqNAEBj7FwPAEBLSLwAgMbYQBUAgJaQeAEAjWmzwEviBQBQF4UXAEBNtBoBgOa0Wa9R4gUAUBOJFwDQGBuoAgDQEhKvGr36ysv53gnH5M03Xk/p6Mgnd+if/rvtneFvv50Tjz0yr/zj71l08SVy3Hd+mPnmn3+ya98bPTqHfeGzGfPeexk7dmw+vtkW2e/zX0ySDPzpabnvnj9mxT4r5ejjv5skue2WGzJ8+Nvpv9vetc+TOcNJ3zomd//hrizQs2cuvnJQkuTYo47IC88/myR55513Mt988+XCy66d6tp33hmek799XP72tyEpKTnm+JOy+ppr5czTf5R7/vSH9F6pb44/8XtJkltuHJThw9/ObnvuU9/kmON8sPtc+dkhm2SVpRdIVSUHnXFXtt9guWy77tJ5r3Ncnn15eAb85M68/a/3JrtuyYW659zD+mXRD82bcVWV8259Imfe+GiS5KR9189Way+VR559Iwf8+HdJkj027Z2ePT4wcQzNs4EqLdOlS5cc9KWv5PzLr89Pz/1Vrr/q8jz37N9y6YW/yEfWXT8XXnVjPrLu+rn0wl9MdW23uebKj356bs751VUZeNEVGXzPn/L4ow9nxIh38thfH8q5F1+dcWPH5ZmhT2f0qFG59abrs33/3RqYJXOKT35qx5z204GTHTvp+6fmwsuuzYWXXZt+m2+ZTTbbcprXnnbKyfnoxzbK5dfclIsuvybLLr98RrzzTv768IP51RXXZdy4sRk65OmMGjUqN91wbfrvsnsdU2IO9sMDPpbf/OXFrPXFK7LeYVflyWFv5vaHhmXtQ6/Mel++KkNeejtH9v/wVNd1jq1y9Hn35sOHXJFNvnZdDtx21fRd6kOZf9658tG+i2a9L1+VLh0lqy7TM3PP1SX7bLZSfn7L4w3MEMZTeNVowYUWTp++qyRJ5u3ePcssu1xef/XV3P2H32WrbT+dJNlq20/nT7+/Y6prSymZZ955kySdnZ3p7OxMSUlH6UjnmDGpqiqjR49O167dcvnFF2THXfdK167dapsbc54Pr71O5v/gB6d5rqqq3H7brdlq622nOvevESPy0F/uz6d26J8k6dZtrsw33/wpHR0Z0znhszhqdLp27ZqLLzwvu+6xd7p281lk+uabp1s2WnXxXHDbk0mSMZ3j8va/3svtDw3L2HFVkuS+p19Jr4W6T3Xty2++m4eeeT1JMmLkmDw57K0s0bN7xlVV5uo2/o+4eebqmjGd43L4jmvmrBv/ms6x42qaGUxtpoVXKaVvKWXzUkqPKY5v3brHev97+e8vZejTT2bl1VbPm//8ZxZcaOEk44uzt9785zSvGTt2bAbss0v6b7Np1l5vg6y82hqZt3v3bNxvixy4765ZbIle6d6jR556/NFs+PF+dU6H95mH/vJAevZcMEstvexU51566cV8aIGeOelbx2TfPXbKd7/9zYwc+W66d++efpttlc/ssVOW6NUrPXrMlyceezQf33Tz+ifAHGW5xebP62+PysAvbZp7Tuufsw75eOb9wOQrYfbdvG9ufeDFGd5n6UV6ZK3lF8zgp1/NiJFjct3dz+be0/rnuVeGZ/i772XtFRfJjfc938qp8F8oDb2aMsPCq5TypSTXJzk0yaOllO0nOf3dVj7Y+9nId9/Nt75+RA4+7Gvp3r3HzC+YoEuXLhl40ZW5fNBtefLxR/Ps34YkSXbf53MZeNGV+cKXv5rzf/7T7Dfgi7np+qvz7WO+ml+dN3Amd4Wp3XbrTdlyGmlXMv4vAE8/+Xh22nm3XHjpNZlnnnly4fnnJkn23m//XHjZtfnSEUdl4M9+ks9/4ZAMuvaqHHPU4Tn/3LPrnAJzkK5dStZaYaGc8+vHs8HhV+fdUZ35av+1Jp7/2i4fzthx43LZXUOme4/uc3fNpUdtlSPPvSfvjByTJDn12ofz0cOvztHn35vj9lonJ14yOPtt2Te/OnKLHLXL1G1LqMPMEq/PJ1m7qqodkmya5JullC9PODfdgrGUMqCUcn8p5f6LLzj3f/Ok7xOdnWPyra8fkc0/8cls3G+LJMkCPXvmjddfS5K88fpr+dACPWd4jx7zzZ+1PrJOBt/7p8mOD3nqiSTJkksvk9tuuSHHfeeHefaZoRn2gr/hMes6Oztz5x2/zRZbbTPN84sssmgWXmTRrLr6mkmSfptvlaefnHzNzFMT3i+9zLK55cbr853vn5Znhg7Jiy8819JnZ8700uv/ykuv/yuDn341SXLt3c9krRUWSpLs1a9Ptl1nmez3o6mXYPxb1y4dufTorXL5XUNy/b3PTnV+zeUWTJIM+fvb2atfn+x9ym+z6jI9s8Li8081lga0WeQ1s8KrS1VVI5KkqqrnMr742qaUcmpm8NhVVQ2sqmqdqqrW2Wu/A/5XzzrHq6oqP/zO8Vl62eWyy577Tjz+sY03zW9uHv+vyn5z86B8bOOp24RvvfnPjHhneJJk9KhReWDwvVlqmeUmG3P+wDOz34AvZmxnZ8ZNWMPQ0VEyevSoVk2J96HBf74nyyy7XBZZdLFpnl9woYWz6KKL5fnnxv8Bd/9992bZ5VaYbMzAs87I579waDo7OzN23PjPYunoyKhRPotM7ZW3RmbY6yPSu9f4NYebrtErT774Vrb88FL5Sv+1svN3fp2R73VO9/qzD90kT734Vn4y6K/TPH/cXuvmxEvuT7euHenSMf6PrnHjqqnamVCHmRVeL5dSJua9E4qw7ZIslGT1Vj7Y+9GjDz+Y2265MQ/ef18G7LNLBuyzS/589x+y+77754H77sm+O2+XB+67J3vsu3+S5PXXXs3XDz84SfLG66/nK188IAfs1T8Hf26PrL3eBtlgo00m3vuPd92RviuvmoUWXiQ95ps/q6y+Rg7Ya6eUlKzQe6VG5svs7bivfzWf32+PPP/8c/n01v0y6LqrkyS//c0tU7UZX3vt1Rxx6IET3x9x1DH51jFfy9677pAhTz+Zz+w/YOK5u37326yy6mpZeOFFMt9882e1NdbMXrtun1KS3n361jM55jhHnPOnnH/E5rnv9J2z5nIL5gdX/iWnHbhh5punW2484ZO597T++ckXNk6SLN5z3lz7zfGJ7MdWXix79euTTdZYIvee1j/3ntY/n1h7qYn3/dT6y+aBIa/lH/98N2//6738+clXMvj0nVMl+etz015PS71KQ/9pbL5VVU3/ZClLJumsqurlaZzbsKqqP03jsskMe3P09L8B1GjeufztltlHrz0tw2D2MfL6AxurRJ78x7uN1Al9F5+3kTnP8E+iqqqGzeDcTIsuAAD+jwgAAGiMnesBAGgJiRcA0Jg2C7wkXgAAdZF4AQDNabPIS+IFAFAThRcAQE20GgGAxjS5i3wTJF4AADWReAEAjbGBKgAALSHxAgAa02aBl8QLAKAuCi8AgJpoNQIAzWmzXqPECwCgJhIvAKAxNlAFAKAlJF4AQGNsoAoAQEsovAAAaqLVCAA0ps06jRIvAIC6SLwAgOa0WeQl8QIAqInECwBojA1UAQBoCYUXAEBNtBoBgMbYuR4AgJaQeAEAjWmzwEviBQBQF4kXANAYa7wAANpcKWXrUspTpZShpZSjp3F+r1LKIxNed5dS1pyV+yq8AAAmUUrpkuTMJNskWSXJHqWUVaYY9mySTaqqWiPJiUkGzsq9tRoBgAbNlr3G9ZIMrarqmSQppVyWZPskj/97QFVVd08y/t4kS87KjSVeAEDbKaUMKKXcP8lrwCSneyV5cZL3wyYcm579k9wyK99X4gUANKapxfVVVQ3M9NuD03qqapoDS+mX8YXXRrPyfRVeAACTG5ZkqUneL5nk71MOKqWskeTcJNtUVfXGrNxYqxEAaExp6DUTg5P0LqUsV0qZK8nuSQZN9tylLJ3kmiT7VFX19KzOV+IFADCJqqo6SymHJLk1SZck51VV9Vgp5aAJ589OclySBZOcVcb3SzurqlpnZvdWeAEATKGqqpuT3DzFsbMn+fqAJAf8p/dVeAEAjbFzPQAALSHxAgAaU2bPDVRbRuIFAFATiRcA0Jz2CrwkXgAAdVF4AQDURKsRAGhMm3UaJV4AAHWReAEAjbGBKgAALSHxAgAaYwNVAABaQuEFAFATrUYAoDnt1WmUeAEA1EXiBQA0ps0CL4kXAEBdJF4AQGNsoAoAQEsovAAAaqLVCAA0xs71AAC0hMQLAGiMxfUAALSEwgsAoCYKLwCAmii8AABqYnE9ANAYi+sBAGgJiRcA0BgbqAIA0BISLwCgMdZ4AQDQEgovAICaaDUCAI1ps06jxAsAoC4SLwCgOW0WeUm8AABqIvECABpjA1UAAFpC4QUAUBOtRgCgMXauBwCgJSReAEBj2izwkngBANRF4gUANKfNIi+JFwBATRReAAA10WoEABpj53oAAFpC4gUANMYGqgAAtESpqqrpZ2AWlFIGVFU1sOnngMTnkdmHzyJzGonXnGNA0w8Ak/B5ZHbhs8gcReEFAFAThRcAQE0UXnMOaxiYnfg8MrvwWWSOYnE9AEBNJF4AADVReM3mSilbl1KeKqUMLaUc3fTz0N5KKeeVUl4tpTza9LPQ3kopS5VSfldKeaKU8lgp5ctNPxPMCq3G2VgppUuSp5NsmWRYksFJ9qiq6vFGH4y2VUr5eJIRSS6sqmq1pp+H9lVKWTzJ4lVV/aWUMl+SB5Ls4OcjszuJ1+xtvSRDq6p6pqqq95JclmT7hp+JNlZV1e+T/LPp54Cqqv5RVdVfJnz9TpInkvRq9qlg5hRes7deSV6c5P2w+MECMJlSyrJJPpzkz80+Ccycwmv2Nq1fHao3DDBBKaVHkquTHFZV1fCmnwdmRuE1exuWZKlJ3i+Z5O8NPQvAbKWU0i3ji66Lq6q6punngVmh8Jq9DU7Su5SyXCllriS7JxnU8DMBNK6UUpL8IskTVVWd2vTzwKxSeM3GqqrqTHJIklszfuHoFVVVPdbsU9HOSimXJrknyUqllGGllP2bfiba1oZJ9kmyWSnloQmvbZt+KJgZ20kAANRE4gUAUBOFFwBATRReAAA1UXgBANRE4QUAUBOFFwBATRReAAA1UXgBANTk/wH021fM2IhGNAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 792x792 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "results = model.evaluate([test_premise_embed, test_hypothesis_embed], y_test, batch_size = 128)\n",
    "print(results)\n",
    "\n",
    "predictions = model.predict([test_premise_embed, test_hypothesis_embed])\n",
    "print(tf.argmax(y_test, 1))\n",
    "print(tf.argmax(predictions, 1))\n",
    "cf_matrix = confusion_matrix(tf.argmax(y_test,1), tf.argmax(predictions,1), labels=[0,1,2])\n",
    "cm_sum = np.sum(cf_matrix, axis=1, keepdims=True)\n",
    "f = plt.figure(figsize=(11,11))\n",
    "ax= plt.subplot()\n",
    "sns.heatmap(cf_matrix/cm_sum, annot=True, ax=ax, fmt='0.1%', cmap='Blues') #annot=True to annotate cells"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-01-27T23:36:33.684682Z",
     "iopub.status.busy": "2021-01-27T23:36:33.678142Z",
     "iopub.status.idle": "2021-01-27T23:36:33.843836Z",
     "shell.execute_reply": "2021-01-27T23:36:33.844414Z"
    },
    "papermill": {
     "duration": 3.499069,
     "end_time": "2021-01-27T23:36:33.844545",
     "exception": false,
     "start_time": "2021-01-27T23:36:30.345476",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x7efedc40dc50>"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEWCAYAAABxMXBSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deXydZZ3//9c7+96mSZfQnVKgFKHQUFYVBJRFKAgyZfuBoyIiM6DODMyMo4z6nWEcHXBEraCMqKyyVgfQUvYBpAspbSnQUkqbNs3WZm1OkpN8fn/cd8ppmjYnJacnyfk8H4/7ce79fO4bej65ruu+rltmhnPOORevtGQH4JxzbnjxxOGcc25APHE455wbEE8czjnnBsQTh3POuQHxxOGcc25APHE41w9Jv5b0/Tj33SjpjETH5FwyeeJwzjk3IJ44nEsRkjKSHYMbGTxxuBEhrCL6e0lvSmqV9CtJ4yU9JalZ0jOSimP2P1/SGkkNkp6XNCtm2zGSVoTHPQjk9Pquz0qqCI99RdJRccZ4rqQ3JDVJ2izpll7bTwnP1xBuvzpcnyvpR5I+kNQo6eVw3amSKvu4D2eE87dIeljS7yQ1AVdLmifp1fA7qiTdISkr5vjZkhZL2i6pWtI/SZogaaekkpj95kqqlZQZz7W7kcUThxtJLgLOBA4FzgOeAv4JKCX4f/1vASQdCtwP3AiMBZ4E/iApK/wRfRz4LTAG+H14XsJjjwXuBr4ClAC/ABZJyo4jvlbg/wNGA+cCX5V0QXjeKWG8PwljmgNUhMf9EJgLnBTG9A9Ad5z3ZD7wcPid9wJdwNfDe3IicDpwXRhDIfAM8DRwEHAIsMTMtgHPA5fEnPcK4AEz64wzDjeCeOJwI8lPzKzazLYALwF/MbM3zKwdeAw4Jtzvr4D/NbPF4Q/fD4Fcgh/mE4BM4HYz6zSzh4GlMd/xZeAXZvYXM+sys3uA9vC4fTKz581slZl1m9mbBMnrk+Hmy4FnzOz+8HvrzaxCUhrw18ANZrYl/M5XwmuKx6tm9nj4nW1mttzMXjOzqJltJEh8PTF8FthmZj8ys4iZNZvZX8Jt9xAkCySlA5cSJFeXgjxxuJGkOma+rY/lgnD+IOCDng1m1g1sBiaG27bY7qN/fhAzPxX4ZljV0yCpAZgcHrdPko6X9FxYxdMIXEvwlz/hOd7r47BSgqqyvrbFY3OvGA6V9EdJ28Lqq3+LIwaAJ4AjJB1MUKprNLPX9zMmN8x54nCpaCtBAgBAkgh+NLcAVcDEcF2PKTHzm4H/Z2ajY6Y8M7s/ju+9D1gETDazUcBCoOd7NgMz+jimDojsZVsrkBdzHekE1Vyxeg9//XPgbWCmmRURVOX1FwNmFgEeIigZXYmXNlKaJw6Xih4CzpV0eti4+02C6qZXgFeBKPC3kjIkfQ6YF3PsXcC1YelBkvLDRu/COL63ENhuZhFJ84DLYrbdC5wh6ZLwe0skzQlLQ3cD/yXpIEnpkk4M21TeBXLC788EvgX019ZSCDQBLZIOB74as+2PwARJN0rKllQo6fiY7b8BrgbOB34Xx/W6EcoTh0s5ZvYOQX39Twj+oj8POM/MOsysA/gcwQ/kDoL2kEdjjl1G0M5xR7h9fbhvPK4DviupGfg2QQLrOe8m4ByCJLadoGH86HDz3wGrCNpatgP/AaSZWWN4zl8SlJZagd2esurD3xEkrGaCJPhgTAzNBNVQ5wHbgHXAaTHb/4+gUX5F2D7iUpT8RU7OuXhJeha4z8x+mexYXPJ44nDOxUXSccBigjaa5mTH45LHq6qcc/2SdA9BH48bPWk4L3E455wbEC9xOOecG5CUGPSstLTUpk2bluwwnHNuWFm+fHmdmfXuG5QaiWPatGksW7Ys2WE459ywIumDvtZ7VZVzzrkB8cThnHNuQDxxOOecG5CUaOPoS2dnJ5WVlUQikWSHklA5OTlMmjSJzEx/345zbnCkbOKorKyksLCQadOmsftAqCOHmVFfX09lZSXTp09PdjjOuREiZauqIpEIJSUlIzZpAEiipKRkxJeqnHMHVsomDmBEJ40eqXCNzrkDK2WrqpxzbiQwM1raozTs7GR7awc7doZTayc7dnZw8dxJTC3JH9Tv9MSRJA0NDdx3331cd911AzrunHPO4b777mP06NEJisw5dyB0dRutHVEad3ZS29JObXM7dS3t1DV3UBezvLOji65uI9rdTbdBtLubri4j2h1MzZFOOrv6HnNQgmOnFnviGCkaGhr42c9+tkfi6OrqIj09fa/HPfnkk4kOzTm3F93dRltnF60dUXa2h58dXbREojRFOmlpj9IcidISidLSHq6LRGntCNY1twefre1RWju69vo9xXmZlBZkU1qQzUGjs8hIE+npCj4l0tNERrpIkyjKzaQ4L5PivKxgys+iOC+TMflZFOVkkpY2+NXVnjiS5Oabb+a9995jzpw5ZGZmUlBQQFlZGRUVFbz11ltccMEFbN68mUgkwg033MA111wDfDh8SktLC2effTannHIKr7zyChMnTuSJJ54gNzc3yVfm3NDX1W3Ut7ZT09ROTXOE7a2dNOzsoLGtk8a2Thp2dtIQzjfu7KClvYudYZKIR5qgIDuDwpxMCrIzKMjJYHReFpPG5FGYnUF+dgYF2RkUZ3RwUPdWinMzGJOXweicdEblpJEhwLrBOqCrFaKRcGr/8LOzLfjs7oT2KLR1QW0UunumrmA66XoYP3tQ758nDuBf/7CGt7Y2Deo5jzioiO+ct/f/WLfeeiurV6+moqKC559/nnPPPZfVq1fvemz27rvvZsyYMbS1tXHcccdx0UUXUVJSsts51q1bx/33389dd93FJZdcwiOPPMIVV1wxqNfh3HAT6exiW2OErQ1tbA0/qxoj1DZHqA4TRV1LB13de1bvpAmKcjMZlZvJ6NxMinIzmVycS2FOJvlZ6eRlZ+z+mZVBXlY6hTkZ4RQkirys9L0/mNLdBRueh5UPwNo/QLTto12w0iEtI2bqtTznsv7PMUCeOIaIefPm7dbX4r//+7957LHHANi8eTPr1q3bI3FMnz6dOXPmADB37lw2btx4wOJ1LtHMjEhnN83tnWFVUJTGtk6a2jppinSG88G6Ha0dbG1sY2tDhLqW9j3OVZKfxbiiHMYXZTOrrJDxRTmMK8xmXPhZWpBNUW4mhdkZ8VfttDXA1hVQ8zYUlUHmDMiZAVk5fe9fvSZIFm8+BC3bIGd08KN+8CchPQuUFiQBKZwPp/QsyMiGjBzIzAk+e5bTsyHtwD8cm9DEIeks4MdAOvBLM7u1j31OBW4HMoE6M/ukpMnAb4AJQDdwp5n9ONz/FuDLQG14in8ys49U8b+vksGBkp//YePV888/zzPPPMOrr75KXl4ep556ap99MbKzs3fNp6en09b2Ef9ycS6BuruN+tYOqpsibGuMUN0coboxwramoCTQFOmktf3D9oGW9ijdBpNVzXF6h3R1Y4huE12kYaSRlpZGdlYm47KzmVI4hoLpYygaXULxmLGMLS1l4pgCxhflkJO593bDuHRFoXYtVC6FyuXBZ907fe9beBCUzAinQ8AMVj0E21YFJYCZn4GjF8ChnwkSwDCUsMQhKR34KXAmUAkslbTIzN6K2Wc08DPgLDPbJGlcuCkKfNPMVkgqBJZLWhxz7G1m9sNExX4gFBYW0tzc9xs4GxsbKS4uJi8vj7fffpvXXnvtAEfn3MB0dnVT3RShuilCVWOQGHo+t/UkiqYI0V7VQ2mC0oJsxhflMCo3k/GFORRmwRHRtcxufZVDGv6PMTvf3/eXdwNt4VTTa1t2UTDljILc0cFf+bHzueGyGbQ3QaQp+GxvgvbmYDnSANVvQWdrcM68EphYDh/7PEwqD9oPWqqhfj3UvxdO64NqqJ31wTEHHQtn/ycc+TnILx2EO55ciSxxzAPWm9kGAEkPAPOBt2L2uQx41Mw2AZhZTfhZBVSF882S1gITex07rJWUlHDyySdz5JFHkpuby/jx43dtO+uss1i4cCFHHXUUhx12GCeccEISI3WpqLOrm53tXezsjNLa3kVbRxeNbZ1h6SBCVWMb2xrb2dYUfNa3ttP7LdS5memUjc6hbFQOxx88hrJROYwv+nCaUJRDaUHwxBCtdUG9/7tPw/rFEGmEtEyYdjIc+hU4+DTIyg8bjGOm7q7gs6sj/OFvDH/sG4OpZ11bQ/DZ8EE43wAdLX1ffEYOZBeGSacQcorgmCuCJDGpHIqnB9VJsQrGwYSP7XmunduhcyeMmjQo/12GioS9c1zSxQQliS+Fy1cCx5vZ9TH79FRRzQYKgR+b2W96nWca8CJwpJk1hVVVVwNNwDKCksmOPr7/GuAagClTpsz94IPd30eydu1aZs2aNRiXOuSl0rW6PXVEu6lqbKNyRxt1Le27dRTb1WEsfKqotSN4emhv/QJ6jM7LZEJRDhNGBQlgfDhfNiqHslG5TMgziqL1qKUamrcFf5G31gZ/ge+sD35QW+s+XLbwaaW80qAK59DPBMkipyhxN6YrGiaYhiARZI8KEkVGVuK+c5iRtNzMynuvT2SJo68Wpt7/N2YAc4HTgVzgVUmvmdm7AJIKgEeAG82s57GnnwPfC8/1PeBHwF/v8UVmdwJ3ApSXlycmOzp3gEW7uolEu4l0BqWA9mgXbR3dRKLBcnVThModbWzesZPKHW1Ubt/JtqYIez5AZByW08icnG0cl7GFGVQyPr2GzqIi2rJL6cwdSzRvLN15Y7GCcaQVTCCnoIiyjFbGpjeT3b4dWreEP/510FgLW2qCBNFcDe2NewavNMgdE1T15JVA6SGQd3y4XAqT5wVVOgeqsTc9A/JLgskNSCITRyUwOWZ5ErC1j33qzKwVaJX0InA08K6kTIKkca+ZPdpzgJlV98xLugv4Y4Lid+6Aao50Ut0UYWtD0CawtbEt/IywrTF4pLQ5Eu33PBJMKMphyugcPj3ZODSvnekZ2ylTHWNaN5DbuJ6M+ndRRzP0PHORPw5KpkOkBppXQc32+APPHhX+AI+DcbOCkkLheCiYEPM5IUgaSXgCyA2+RCaOpcBMSdOBLcACgjaNWE8Ad0jKALKA44HbFDwA/StgrZn9V+wBksrCNhCAC4HVCbwG5wZF485ONu/YyeawBLCtKUJNU/uuRuPqpkifPYmDnsM5TCvJ56QZpRTnZZGXaYy2Zoq6d1DY1UBhdAf5nfXkdmynsKOWvMg20poqobYKqjt3P2H+WBh7ePBUz7jDYeys4Mc+b8zu+0U7gqql1hpoCUsS7S1B6SC/NJzGBsvD9Mkgt/8SljjMLCrpeuBPBI/j3m1mayRdG25faGZrJT0NvEnwbMQvzWy1pFOAK4FVkirCU/Y8dvsDSXMIqqo2Al9J1DU4F6/2aBdbGyJU7tjJpu3BtDn83FS/k6ZeJYWs9DTGFWUzoSiHWWVFnHrYOMYXZYftBLm7GpKzMtJgx0Z456lweitsE+jeM4i0jOBR0FGTYPLxwWfRRBg1OZgfNRFyi+O7oIysYP9REz/6zXEjTkL7cYQ/9E/2Wrew1/J/Av/Za93L9N1GgpldOchhOheXnR1R1mxt4v26Viq379zVlrB5exvVzZHdnirKSk9j0phcpozJo3xyIXO1jtk7/8K4lrfIHDOVrAmz0LhZMHYSFE3avQqnuxu2LIc3wmRREz5MWHoYHHYOFIwP/tovGBt85o8L5nNG7/m0j3MJ4D3HnetDpLOLt6qaWFXZyJuVjaza0sD6mhbSLYoB3cqgbFQuk4pzOfmQUiaPyWVycR6TinOZUpLHeBpI27AE1v0Z1j4XPBaalgHjjoANi2HVfR9+WWY+lM4MqpDS0mHd4qCKSOkw9ST4zL/BoWcFHcqcGwI8cQwTBQUFtLTs5blzt9/aOrrYWN/KhtpW3q9rYUNdK29XNfNudTP53c0ckfYB83IquSh3CzOLN1LStpE0i2JZBSi7GLpHw87RYKNhZzFU58Cm12Dbm8EXFJbB7Atg5qdh+ic/fLx053aofQdq3w4+696BjS8FfQtmnB6ULGaeEX/VknMHkCcOlxIa2zp5Z1szb29r4t3qZt6va2VLzXa6mrcxnh2MVwPjtYNjc5q5PLOamQXvU9SxLTi4G0grCzp4jT8fMnJQpCHoSNa2I+gHULc+mO9ogQlHwenfCZLF+Nl9Vx/ljYGpJwaTc8OMJ44kuemmm5g6dequ93HccsstSOLFF19kx44ddHZ28v3vf5/58+cnOdLhpavbeK+2hbVVTby/ZRvbt6yno+Y98tsqmaxapqiGE9PrmaAdFFoL7PFAUDYUTIUJpwSJomcqGNfX1zmXkjxxADx1czAA2WCa8DE4e48xHXdZsGABN954467E8dBDD/H000/z9a9/naKiIurq6jjhhBM4//zz/b3h+9DSHmXl5gaWbdxB1foVTKn6MydaBaeomvmKGQssE6KZBdjoaWSM+RgqOijoW1BYFjNNCKqG/H47t0+eOJLkmGOOoaamhq1bt1JbW0txcTFlZWV8/etf58UXXyQtLY0tW7ZQXV3NhAkTkh3ukFHV2MbSjTtYvnE7yz7YQce2tZytv3Bu+mscllZJt8T2MXNIm3AC0bIZZJRMh9FToXgaGZ4UnBsUnjhgnyWDRLr44ot5+OGH2bZtGwsWLODee++ltraW5cuXk5mZybRp0/ocTj1VdIfVTq+/X8fK96tZvXEbOxqbKFYzZ2VW8PPspUzJeh9DdE0+ET52I2mzzqe0cHz/J3fO7TdPHEm0YMECvvzlL1NXV8cLL7zAQw89xLhx48jMzOS5556j98CMI52ZsXn5U2S+9ANyWjaR3hVhinUwU51c3rNT+I4cQ6jsRJh9HTrifDIKvVTm3IHiiSOJZs+eTXNzMxMnTqSsrIzLL7+c8847j/LycubMmcPhhx+e7BATrq6lnZfX1bF+1Suc/P4dnGgVVFopr2Qew6gxoygtHsX4kmJGFxWhzNxgyOusfDT15OCta865A84TR5KtWvVho3xpaSmvvvpqn/uNlD4cXd3Gso3bee6dWl5aV0tj1Xt8I+P3fCP9/2hLK6Ri1t8z/vSvcW6J919wbqjyxOESLtrVzevvb+fJ1VU8vbqaupZ2StNauKX4ac7O+QNKS4MTbiT/lBuZkzs62eE65/rhicMlRGe0i6VrN/DaqrWsXf8eWZHtlGU08b2STmZPbGPytmfQzhaYcxmc+k8+mJ5zw0hKJw4zG/F9JBL1hsc+dUbY/OpDdLz+a6a0VHASXZzUs63npWoNadBRCtM/AZ/6VjCkt3NuWEnZxJGTk0N9fT0lJSUjNnmYGfX19eTk5CT0e9o2VbB5yULKNv2BydbCZhvLi2MuYuKUGRw8/WCyinpGcx3nL/NxbgRI2cQxadIkKisrqa2tTXYoCZWTk8OkSZMG/8RtDWx56XfYG79hUts7TLUMXsk6kfaPXcEJp1/A6fmJTVbOueRJ2cSRmZnJ9OnTkx3G8NJaR/Ttp6hZ+igl215mIh2stak8VnYD0z91NafOnD5iS2/OuQ+lbOJwcapbD+/8L11r/5e0ytfJwDAr4ansM8k49go+/okzmJWX1f95nHMjRkITh6SzgB8TvDr2l2a2x9gekk4FbgcygToz++S+jpU0BngQmEbw6thLzGxHIq8j5bTWwWs/g7V/gLp3AVjHNJ6OXsiW8adx9hmfZv7h47104VyKSljikJQO/BQ4E6gElkpaZGZvxewzGvgZcJaZbZI0Lo5jbwaWmNmtkm4Ol29K1HWklM62IGG8dBvWuZP3C47l/u6rebLjGGbNms21n5xB+bQxyY7SOZdkiSxxzAPWm9kGAEkPAPOBt2L2uQx41Mw2AZhZTRzHzgdODfe7B3geTxwfTXc3vPkAPPt9aNrCmsKT+XrLhbzXPpH5Rx/E3Z+cwWETCpMdpXNuiEhk4pgIbI5ZrgSO77XPoUCmpOeBQuDHZvabfo4db2ZVAGZW1VNK6U3SNcA1AFOmTPloVzKSvfccLP4X2LaKD3IO46aOf+GNHbNZcPxk7v7EwUwqzkt2hM65ISaRiaOvCvDevdEygLnA6UAu8Kqk1+I8dp/M7E7gToDy8vID2AtumKhZC3/+F1i/mLqM8Xy343qe7T6ZKz5+MD85ZTpjC/d4NZ5zzgGJTRyVwOSY5UnA1j72qTOzVqBV0ovA0f0cWy2pLCxtlAE1uPh1RuDFH2Av385Ocri98zKe0Llc/qnD+N5J0xiVl5nsCJ1zQ1wiE8dSYKak6cAWYAFBm0asJ4A7JGUQDEpxPHAb8PY+jl0EXAXcGn4+kcBrGFk2vUb3418jbft6Hun6BAuzv8AlnzmaZ4+fSkG2P5ntnItPwn4tzCwq6XrgTwSP1N5tZmskXRtuX2hmayU9DbwJdBM8drsaoK9jw1PfCjwk6YvAJuDzibqGEaO9GZZ8F3v9LqpVys0dN3HoyRfwx08fRk5merKjc84NMzqgg+AlSXl5uS1btizZYSTHusXYH26Apq38OvppHiy6mu9dcgLH+WO1zrl+SFpuZuW913v9xEi1czs8/Y/w5gNsSpvE19u/w5EnnMmjZx9OXpb/Z3fO7T//BRmJ3n4S+8MNdO+s52fRC3g0/1K+f1k5Jx9SmuzInHMjgCeOkaStAZ6+GVbez/tp07g+ciMfm/txFn12FoU5/rSUc25weOIYKdYvwZ64HmupZmH3hfwm/RL+7aq5fOrw8cmOzDk3wnjiGO7am4OOfMv/h6rMKVwbuYUxh57AHy4+2jvxOecSwhPHcLbxZXj8OqxhE7/V+fyg7WK++dmjuPqkaT5yrXMuYTxxDEddUXj2e/B/t7M9eyLXtP8LTePKefjSYzh8QlGyo3POjXCeOIab1jp4+Avw/os8lfUZvtH4V1xy4mH84zmzvDOfc+6A8MQxnFQuh4eupLu1nu+lf43Ho6dyx1VHc/osbwB3zh04njiGAzNY/mt46h+I5o/nS+n/j5XRKTz4lRM5dLy/J8M5d2B54hjqOtvgyb+DN35H+9TT+HztF3i/I4f7v3y8Jw3nXFJ44hjKdnwAD10JVSvZecI3uHDNx9nc0sFvvziPIyeOSnZ0zrkU5YljqKpcBvdeDN3dtF50L5c8N4qNO1r49RfmMXdqcbKjc86lsLRkB+D6ULcO7v085Iym9epnuOKlYt6tbuYXV87lxBklyY7OOZfiPHEMNc3V8LvPgdKILHiYL/6hnjcrG7njsmM59bA+X6/unHMHlFdVDSWRpqB6qrWejisX8ZX/3c5f3t/O7X81h8/MnpDs6JxzDvASx9AR7QgawqvXwCX38N0V2bzwbi3/8bmjmD9nYrKjc865XTxxDAVmsOh62PA8nP/fPBU5kt+9tokvf3w6lxw3OdnROefcbhKaOCSdJekdSesl3dzH9lMlNUqqCKdvh+sPi1lXIalJ0o3htlskbYnZdk4ir+GAeOYWePNBOO1bbJ76Of7hkTc5evJo/v4zhyc7Muec20PC2jgkpQM/Bc4EKoGlkhaZ2Vu9dn3JzD4bu8LM3gHmxJxnC/BYzC63mdkPExX7AfWXX8D/3Q7lf03nyd/gb+98DQx+suAYsjK8QOicG3oS+cs0D1hvZhvMrAN4AJi/H+c5HXjPzD4Y1OiGgreegKdugsPOhXN+yA8Xv8sbmxq49aKjmFKSl+zonHOuT4lMHBOBzTHLleG63k6UtFLSU5Jm97F9AXB/r3XXS3pT0t2S+uwNJ+kaScskLautrd2vC0iomrXwyJdh0nFw0S95Yf12fvHCBi6dN4VzjypLdnTOObdXiUwcfb1JyHotrwCmmtnRwE+Ax3c7gZQFnA/8Pmb1z4EZBFVZVcCP+vpyM7vTzMrNrHzs2LH7dwWJYgb/+3eQmQsL7qMmksY3HqzgsPGFfOe8I5IdnXPO7VMiE0clEPtI0CRga+wOZtZkZi3h/JNApqTSmF3OBlaYWXXMMdVm1mVm3cBdBFViw8uqh+GDl+GM79CVV8qND1bQ2hHljsuO8XdqOOeGvEQmjqXATEnTw5LDAmBR7A6SJih8x6mkeWE89TG7XEqvaipJsfU4FwKrExB74kSa4M//DAcdA8dexc+eW88r79Xz3fOPZKaPduucGwYS9lSVmUUlXQ/8CUgH7jazNZKuDbcvBC4GviopCrQBC8zMACTlETyR9ZVep/6BpDkE1V4b+9g+tD1/K7TUwKX38/oHjdz2zLucf/RBfL58UrIjc865uCj8nR7RysvLbdmyZckOI+gVvvDjcOyVNJ35Qz5z24tkZaTxx785hcKczGRH55xzu5G03MzKe6/3saoOlJ4G8ZxRcPp3+NVL71PVGOGx607ypOGcG1a8h9mB8uaDsOkVOOMWGijg7pff56zZEzhmir9bwzk3vHjiOBDaGuDP34KJc+GYK7nzxQ20dET5+pmHJjsy55wbMK+qOhCe/3dorYPLf0/9zk5+/cpGPnvUQRw2wZ+ics4NP17iSLRtq+D1O6H8r+GgY1j4wntEOru48YyZyY7MOef2iyeOROruDhrEc4vhU9+iuinCb179gAuOmciMsQXJjs455/aLJ45EWnk/bH4NzvhXyBvDz55bT7TbuOF0L20454YvTxyJ0hmBZ74Dk+bBnMvZ0tDG/a9v5vNzJzG1JD/Z0Tnn3H7zxvFEWfMYtNbCRb+EtDTueHY9AH/jpQ3n3DDnJY5EWXoXlB4K0z/Jpvqd/H7ZZhbMm8zE0bnJjsw55z4STxyJsGUFbFkOx30JJP772XWkp4mvnXZIsiNzzrmPzBNHIiz9FWTmw9EL2FDbwqMrKrnihKmML8pJdmTOOfeRxZU4JD0i6VxJnmj6s3M7rH4YjroEckbx4yXryM5I56unzkh2ZM45NyjiTQQ/By4D1km6VdLhCYxpeKu4F6IROO5LvFvdzKKVW7nqpGmUFmQnOzLnnBsUcSUOM3vGzC4HjiV4B8ZiSa9I+oIkH9q1R3d3UE015USYcCS3LX6X/KwMvvKJg5MdmXPODZq4q54klQBXA18C3gB+TJBIFicksuHovSWw43047kvUtbTz9JptXHHCVIrzs5IdmXPODZq4+nFIehQ4HPgtcMLGIu8AABedSURBVJ6ZVYWbHpQ0BN6QNES8fhfkj4NZ5/NcRTVm8Nmjyvo/zjnnhpF4Sxx3mNkRZvbvMUkDgL7eDtVD0lmS3pG0XtLNfWw/VVKjpIpw+nbMto2SVoXrl8WsHyNpsaR14efQeKHFjo2w7s8w9yrIyGLJ2homFOUw+6CiZEfmnHODKt7EMUvS6J4FScWSrtvXAZLSgZ8CZwNHAJdKOqKPXV8ysznh9N1e204L18cmp5uBJWY2E1gSLiffsrtBaTD3C7RHu3hpXS2fmjUOScmOzDnnBlW8iePLZtbQs2BmO4Av93PMPGC9mW0wsw7gAWD+/oW5m/nAPeH8PcAFg3DOj6YzAit+C4efA6Mm8tqG7bR2dHHGrHHJjsw55wZdvIkjTTF/Ooelif5afCcCm2OWK8N1vZ0oaaWkpyTNjllvwJ8lLZd0Tcz68T3VZeFnn7/Okq6RtEzSstra2n5C/YjWPAZt2+G4IJcuWVtNTmYaJ80oTez3OudcEsQ7yOGfgIckLST4Qb8WeLqfY/qqo7FeyyuAqWbWIukc4HGgZxTAk81sq6RxBI//vm1mL8YZL2Z2J3AnQHl5ee/vHVy7xqX6BGbGkrU1nHLIWHIy0xP6tc45lwzxljhuAp4Fvgp8jaBt4R/6OaYSmByzPAnYGruDmTWZWUs4/ySQKak0XN4aftYAjxFUfQFUSyoDCD9r4ryGxOg1LtXb25rZ0tDm1VTOuREr3g6A3Wb2czO72MwuMrNfmFlXP4ctBWZKmi4pC1gALIrdQdKEniowSfPCeOol5UsqDNfnA58GVoeHLQKuCuevAp6I5xoSZukvd41LBfDs20Ee+9ThnjiccyNTvP04ZgL/TvB01K6R+sxsr12izSwq6XqCaq504G4zWyPp2nD7QuBi4KuSokAbsMDMTNJ44LEwp2QA95lZT9XYrQTVZl8ENgGfH8gFD6qd22H1I3D0pZAzCoBn1lZz1KRRjPMBDZ1zI1S8bRz/A3wHuA04DfgCfbdh7Casfnqy17qFMfN3AHf0cdwG4Oi9nLMeOD3OuBPrjd8F41LNCxrF61raqdjcwI2nH5rkwJxzLnHibePINbMlgMzsAzO7BfhU4sIaJlY/HLwadnzwMNizb9dgBqd7+4ZzbgSLN3FEwiHV10m6XtKF7OUx2JQRbYfqt2DaybtWLVlbTdko7y3unBvZ4k0cNwJ5wN8Cc4Er+LCBOjXVvAXdnVA2B4BIZxcvravjU4d7b3Hn3MjWbxtH2NnvEjP7e6CFoH3Dba0IPsuCppjXNtSzs6OLM2aNT2JQzjmXeP2WOMLHbufK/4zeXdXK4Emq4mkALFlbQ25mOifOKEluXM45l2DxPlX1BvCEpN8DrT0rzezRhEQ1HFRVBKUNKewtXs0pM0u9t7hzbsSLt41jDFBP8CTVeeH02UQFNeR1dUL1ml3tG29va2ZrY8R7izvnUkJcJQ4z83aNWDVroatjV/vGkrXVAJzmvcWdcykg3p7j/8OeAxRiZn896BENB1Urg8+DjgHgmbU1HD15NOMKvbe4c27ki7eN448x8znAhfQasDClVFVAViEUT6e2uZ2VlQ18/QzvLe6cSw3xVlU9Erss6X7gmYRENBxUrQyqqdLSeM57izvnUky8jeO9zQSmDGYgw0ZXFLat3tW+8czaag4alcMRZd5b3DmXGuJt42hm9zaObQTv6Eg9de9CtA0OmrOrt/hFcyd6b3HnXMqIt6qqMNGBDBtVH/YYf3VDPW2dXZzuvcWdcykkrqoqSRdKGhWzPFrSBYkLawirWhm8uKnkEJasrSYvK50TD/be4s651BFvG8d3zKyxZ8HMGgjez5F6tlbAhI9BWjoVmxuYO7XYe4s751JKvImjr/3ifZR35Ojugm2r4KCgx3hdcwdlo7zvhnMutcSbOJZJ+i9JMyQdLOk2YHkiAxuS6tdDZyuUHY2ZUd/aTklBdrKjcs65AyrexPE3QAfwIPAQwfvBv9bfQZLOkvSOpPWSbu5j+6mSGiVVhNO3w/WTJT0naa2kNZJuiDnmFklbYo45J85r+Oh6eoyXzaEpEqWzyyjJzzpgX++cc0NBvE9VtQJ7/PDvS/gej58CZwKVwFJJi8zsrV67vmRmvQdMjALfNLMVkgqB5ZIWxxx7m5n9cCDxDIqtFZCRC6WHUr89AkCplziccykm3qeqFksaHbNcLOlP/Rw2D1hvZhvMrAN4AJgfz/eZWZWZrQjnm4G1wMR4jk2oqpUw4UhIz6CupQPwxOGcSz3xVlWVhk9SAWBmO+j/neMTgc0xy5X0/eN/oqSVkp6SNLv3RknTgGOAv8Ssvl7Sm5LullTc15dLukbSMknLamtr+wk1Dt3dHw41AtS3tANQUuBVVc651BJv4uiWtGuIkfDHfI/Rcnvpqyt172NWAFPN7GjgJ8Dju51AKgAeAW40s6Zw9c+BGcAcoAr4UV9fbmZ3mlm5mZWPHTu2n1DjsON96Gje9Q6OutagxOGJwzmXauJ9pPafgZclvRAufwK4pp9jKoHJMcuT6DWibkwywMyelPQzSaVmVicpkyBp3Bv7pkEzq+6Zl3QXu4/cmzhb3wg+wxJHXXM7EozJ88ThnEstcZU4zOxpoBx4h+DJqm8SPFm1L0uBmZKmS8oCFgCLYneQNKHnXeaS5oXx1IfrfgWsNbP/6nVMWczihcDqeK7hI6taCelZMG4WAPWt7RTnZZGRvr/jRDrn3PAU7yCHXwJuICg1VAAnAK8SvEq2T2YWlXQ98CcgHbjbzNZIujbcvhC4GPiqpChBIlpgZibpFOBKYJWkcHAo/snMngR+IGkOQbXXRuArA7zm/VNVAeNnQ3omAPUtHf4ornMuJcVbVXUDcBzwmpmdJulw4F/7Oyj8oX+y17qFMfN3AHf0cdzL9N1GgpldGWfMg8csKHHM/tyuVfUtHd6+4ZxLSfHWs0TMLAIgKdvM3gYOS1xYQ8yOjRBp3NW+AVDX0u6P4jrnUlK8JY7KsB/H48BiSTtIpVfH7nrH+JxdqzxxOOdSVbw9xy8MZ2+R9BwwCng6YVENNVUVkJYJ444AoCPaTVMk6m0czrmUNOARbs3shf73GmGqVgZPU2UEJYz61qDzX2mhlzicc6nHnyXtj1kwRlVM+0Z9ONyIlzicc6nIE0d/Giuhbfse7RuAD6nunEtJnjj6s+sd47GJo2eAQy9xOOdSjyeO/lStBKUHnf9CPQMc+lNVzrlU5ImjP1srYOzhkJm7a1V9awc5mWnkZfm7xp1zqccTx76YBVVVMe0bELRxlORnEw6z5ZxzKcUTx740V0Fr7W5PVEHQxuHtG865VOWJY19i3jEeq957jTvnUpgnjn3ZWgFKC14XG8MHOHTOpTJPHPuSMwoOPRuy8netMjPqW9u9D4dzLmUNeMiRlHLidcEUo6ktSmeXeVWVcy5leYljgOp6xqnyqirnXIryxDFAH45T5SUO51xqSmjikHSWpHckrZd0cx/bT5XUKKkinL7d37GSxkhaLGld+FmcyGvo7cNxqrzE4ZxLTQlLHJLSgZ8CZwNHAJdKOqKPXV8ysznh9N04jr0ZWGJmM4El4fIB48ONOOdSXSJLHPOA9Wa2wcw6gAeA+YNw7HzgnnD+HuCCQYy5X3UtHUhQnJd5IL/WOeeGjEQmjonA5pjlynBdbydKWinpKUk9Iwnu69jxZlYFEH6O6+vLJV0jaZmkZbW1tR/lOnZT19JOcV4WGenePOScS02J/PXrayAn67W8AphqZkcDPyF4p3m8x+6Tmd1pZuVmVj527NiBHLpP9S0d/gIn51xKS2TiqAQmxyxPArbG7mBmTWbWEs4/CWRKKu3n2GpJZQDhZ01iwu9bfasPN+KcS22JTBxLgZmSpkvKAhYAi2J3kDRB4RCzkuaF8dT3c+wi4Kpw/irgiQRewx58uBHnXKpLWM9xM4tKuh74E5AO3G1mayRdG25fCFwMfFVSFGgDFpiZAX0eG576VuAhSV8ENgGfT9Q19KXWBzh0zqW4hA45ElY/Pdlr3cKY+TuAO+I9NlxfD5w+uJHGpz3aRXMk6r3GnXMpzR8NGoDtrWGvcS9xOOdSmCeOAahr7hluxEsczrnU5YljAHoGOPQSh3MulXniGICeAQ7HeuJwzqUwTxwDUO8DHDrnnCeOgahraScnM428rPRkh+Kcc0njiWMA6ls6KC3IJuyz6JxzKckTxwDUtXZ4w7hzLuV54hiAuuZ2Sv1RXOdcivPEMQD1re3eMO6cS3meOOJkZrvaOJxzLpV54ohTY1sn0W7zNg7nXMrzxBGnurDznw9w6JxLdZ444rSr81++lzicc6nNE0ec6sORcUsLvcThnEttnjjiVOclDuecAzxxxK2upQMJivMykx2Kc84llSeOONW3tDMmL4uMdL9lzrnUltBfQUlnSXpH0npJN+9jv+MkdUm6OFw+TFJFzNQk6cZw2y2StsRsOyeR19CjrsU7/znnHCTwneOS0oGfAmcClcBSSYvM7K0+9vsP4E8968zsHWBOzPYtwGMxh91mZj9MVOx9qW/p8PYN55wjsSWOecB6M9tgZh3AA8D8Pvb7G+ARoGYv5zkdeM/MPkhMmPGpb+3wEodzzpHYxDER2ByzXBmu20XSROBCYOE+zrMAuL/XuuslvSnpbknFfR0k6RpJyyQtq62tHXj0vdS1tPtwI845R2ITR18vrbBey7cDN5lZV58nkLKA84Hfx6z+OTCDoCqrCvhRX8ea2Z1mVm5m5WPHjh1o7LuJdHbRHIl6r3HnnCOBbRwEJYzJMcuTgK299ikHHghfjFQKnCMpamaPh9vPBlaYWXXPAbHzku4C/piA2HezPez85+NUOedcYhPHUmCmpOkEjdsLgMtidzCz6T3zkn4N/DEmaQBcSq9qKkllZlYVLl4IrB780HdXH45TVeLv4nDOucQlDjOLSrqe4GmpdOBuM1sj6dpw+77aNZCUR/BE1ld6bfqBpDkE1V4b+9g+6Hp6jZcWeonDOecSWeLAzJ4Enuy1rs+EYWZX91reCZT0sd+VgxhiXHYlDn8c1znnvOd4POp3tXF4VZVzznniiEN9Szu5menkZye0gOacc8OCJ4441LV45z/nnOvhiSMOwThV3r7hnHPgiSMu9S0dlPqjuM45B3jiiIsPN+Kccx/yxNGP7m5juw9w6Jxzu3ji6EdTpJNot3kbh3POhTxx9GNX5z8vcTjnHOCJo1914ThV3sbhnHMBTxz92DXAoZc4nHMO8MTRr/rWoKrKXxvrnHMBTxz9qGtuR4Ix3o/DOecATxz9qmvtYExeFulpfb3Q0DnnUo8njn7Ut7R7+4ZzzsXwxNGPupYOb99wzrkYnjj6Ud/S7m/+c865GAlNHJLOkvSOpPWSbt7HfsdJ6pJ0ccy6jZJWSaqQtCxm/RhJiyWtCz+LE3kN9S0d/q5x55yLkbDEISkd+ClwNnAEcKmkI/ay338QvJu8t9PMbI6ZlcesuxlYYmYzgSXhckJEOrtobo96r3HnnIuRyBLHPGC9mW0wsw7gAWB+H/v9DfAIUBPneecD94Tz9wAXfNRA96bnlbHea9w55z6UyMQxEdgcs1wZrttF0kTgQmBhH8cb8GdJyyVdE7N+vJlVAYSf4/r6cknXSFomaVltbe1+XUB9OE6VD3DonHMfSmTi6Kvjg/Vavh24ycy6+tj3ZDM7lqCq62uSPjGQLzezO82s3MzKx44dO5BDd/HhRpxzbk8ZCTx3JTA5ZnkSsLXXPuXAA5IASoFzJEXN7HEz2wpgZjWSHiOo+noRqJZUZmZVksqIv4prwGp7Rsb1x3Gdc26XRJY4lgIzJU2XlAUsABbF7mBm081smplNAx4GrjOzxyXlSyoEkJQPfBpYHR62CLgqnL8KeCJRF9BT4igt9BKHc871SFiJw8yikq4neFoqHbjbzNZIujbc3le7Ro/xwGNhSSQDuM/Mng633Qo8JOmLwCbg84m6hvqWdnIz08nLSmTBzDnnhpeE/iKa2ZPAk73W9ZkwzOzqmPkNwNF72a8eOH3woty7Q8YVcN7RZQfiq5xzbtjwP6X3YcG8KSyYNyXZYTjn3JDiQ44455wbEE8czjnnBsQTh3POuQHxxOGcc25APHE455wbEE8czjnnBsQTh3POuQHxxOGcc25AZNZ7wNqRR1It8MF+Hl4K1A1iOIPJY9s/Htv+8dj2z3CObaqZ7TG8eEokjo9C0rJebyAcMjy2/eOx7R+Pbf+MxNi8qso559yAeOJwzjk3IJ44+ndnsgPYB49t/3hs+8dj2z8jLjZv43DOOTcgXuJwzjk3IJ44nHPODYgnjn2QdJakdyStl3RzsuOJJWmjpFWSKiQtS3Isd0uqkbQ6Zt0YSYslrQs/i4dQbLdI2hLeuwpJ5yQptsmSnpO0VtIaSTeE65N+7/YRW9LvnaQcSa9LWhnG9q/h+qFw3/YWW9LvWxhHuqQ3JP0xXN6ve+ZtHHshKR14FzgTqASWApea2VtJDSwkaSNQbmZJ71gk6RNAC/AbMzsyXPcDYLuZ3Rom3WIzu2mIxHYL0GJmPzzQ8fSKrQwoM7MVkgqB5cAFwNUk+d7tI7ZLSPK9kyQg38xaJGUCLwM3AJ8j+fdtb7GdxdD4f+4bQDlQZGaf3d9/p17i2Lt5wHoz22BmHcADwPwkxzQkmdmLwPZeq+cD94Tz9xD86Bxwe4ltSDCzKjNbEc43A2uBiQyBe7eP2JLOAi3hYmY4GUPjvu0ttqSTNAk4F/hlzOr9umeeOPZuIrA5ZrmSIfIPJ2TAnyUtl3RNsoPpw3gzq4LgRwgYl+R4erte0pthVVZSqtFiSZoGHAP8hSF273rFBkPg3oVVLhVADbDYzIbMfdtLbJD8+3Y78A9Ad8y6/bpnnjj2Tn2sGxJ/OYRONrNjgbOBr4VVMi4+PwdmAHOAKuBHyQxGUgHwCHCjmTUlM5be+ohtSNw7M+sysznAJGCepCOTEUdf9hJbUu+bpM8CNWa2fDDO54lj7yqByTHLk4CtSYplD2a2NfysAR4jqFobSqrDevKe+vKaJMezi5lVh/+4u4G7SOK9C+vBHwHuNbNHw9VD4t71FdtQundhPA3A8wRtCEPivvWIjW0I3LeTgfPDttEHgE9J+h37ec88cezdUmCmpOmSsoAFwKIkxwSApPywwRJJ+cCngdX7PuqAWwRcFc5fBTyRxFh20/MPJXQhSbp3YUPqr4C1ZvZfMZuSfu/2FttQuHeSxkoaHc7nAmcAbzM07lufsSX7vpnZP5rZJDObRvBb9qyZXcH+3jMz82kvE3AOwZNV7wH/nOx4YuI6GFgZTmuSHRtwP0Hxu5OgpPZFoARYAqwLP8cModh+C6wC3gz/4ZQlKbZTCKo/3wQqwumcoXDv9hFb0u8dcBTwRhjDauDb4fqhcN/2FlvS71tMjKcCf/wo98wfx3XOOTcgXlXlnHNuQDxxOOecGxBPHM455wbEE4dzzrkB8cThnHNuQDxxODfESTq1ZzRT54YCTxzOOecGxBOHc4NE0hXhuxgqJP0iHOyuRdKPJK2QtETS2HDfOZJeCwe9e6xn0DtJh0h6JnyfwwpJM8LTF0h6WNLbku4Ne3Y7lxSeOJwbBJJmAX9FMPjkHKALuBzIB1ZYMCDlC8B3wkN+A9xkZkcR9CjuWX8v8FMzOxo4iaDXOwSj094IHEEwcsDJCb8o5/YiI9kBODdCnA7MBZaGhYFcggHjuoEHw31+BzwqaRQw2sxeCNffA/w+HH9sopk9BmBmEYDwfK+bWWW4XAFMI3hJkHMHnCcO5waHgHvM7B93Wyn9S6/99jXGz76qn9pj5rvwf7suibyqyrnBsQS4WNI42PUu56kE/8YuDve5DHjZzBqBHZI+Hq6/EnjBgvddVEq6IDxHtqS8A3oVzsXB/2pxbhCY2VuSvkXwVsY0gtF4vwa0ArMlLQcaCdpBIBjCemGYGDYAXwjXXwn8QtJ3w3N8/gBehnNx8dFxnUsgSS1mVpDsOJwbTF5V5ZxzbkC8xOGcc25AvMThnHNuQDxxOOecGxBPHM455wbEE4dzzrkB8cThnHNuQP5/8Py575KeX9kAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(history.history['accuracy'])\n",
    "plt.plot(history.history['val_accuracy'])\n",
    "plt.title('model accuracy')\n",
    "plt.ylabel('accuracy')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['train', 'val'], loc='upper left')\n",
    "# plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-01-27T23:36:40.781032Z",
     "iopub.status.busy": "2021-01-27T23:36:40.780169Z",
     "iopub.status.idle": "2021-01-27T23:36:40.793075Z",
     "shell.execute_reply": "2021-01-27T23:36:40.792579Z"
    },
    "papermill": {
     "duration": 3.574735,
     "end_time": "2021-01-27T23:36:40.793174",
     "exception": false,
     "start_time": "2021-01-27T23:36:37.218439",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.savefig('Elmo_accuracy_300000_40epochs.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-01-27T23:36:47.404287Z",
     "iopub.status.busy": "2021-01-27T23:36:47.403406Z",
     "iopub.status.idle": "2021-01-27T23:36:47.445903Z",
     "shell.execute_reply": "2021-01-27T23:36:47.445467Z"
    },
    "papermill": {
     "duration": 3.378968,
     "end_time": "2021-01-27T23:36:47.446000",
     "exception": false,
     "start_time": "2021-01-27T23:36:44.067032",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "model.save(\"Elmo_300000_40epochs.hdf5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-01-27T23:36:54.195720Z",
     "iopub.status.busy": "2021-01-27T23:36:54.194867Z",
     "iopub.status.idle": "2021-01-27T23:36:54.534486Z",
     "shell.execute_reply": "2021-01-27T23:36:54.533450Z"
    },
    "papermill": {
     "duration": 3.870023,
     "end_time": "2021-01-27T23:36:54.534601",
     "exception": false,
     "start_time": "2021-01-27T23:36:50.664578",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "test_results_elmo = pd.DataFrame(predictions)\n",
    "test_results_elmo.to_csv(\"Elmo_test_results_300000_40epochs.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-01-27T23:37:28.567142Z",
     "iopub.status.busy": "2021-01-27T23:37:28.566318Z",
     "iopub.status.idle": "2021-01-27T23:37:28.569112Z",
     "shell.execute_reply": "2021-01-27T23:37:28.568699Z"
    },
    "papermill": {
     "duration": 3.299041,
     "end_time": "2021-01-27T23:37:28.569203",
     "exception": false,
     "start_time": "2021-01-27T23:37:25.270162",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "pd.DataFrame.from_dict(history.history).to_csv('history_300000_40epochs.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 4.004351,
     "end_time": "2021-01-27T23:37:35.838203",
     "exception": false,
     "start_time": "2021-01-27T23:37:31.833852",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  },
  "papermill": {
   "duration": 1742.717261,
   "end_time": "2021-01-27T23:37:40.786270",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2021-01-27T23:08:38.069009",
   "version": "2.1.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}