Text_Deep_Learning.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Text_Deep_Learning.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "U0A5sddZXEJ4",
        "alnczQ9if68e"
      ]
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zNONFRCwckat"
      },
      "source": [
        "## Base Code"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "a4SV90uyrkVM"
      },
      "source": [
        "### Prerequisites\n",
        "\n",
        "Python 3.6+ is required because of certain performance optimization steps, such as the use of f-strings.  \n",
        "GPU usage is required, e.g. the LSTM layers are specific CuDNN ones."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zG_VtnVo7wgr",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f087c6f3-9dce-41f0-9016-a25042ef5207"
      },
      "source": [
        "### Parameters ###\n",
        "%tensorflow_version 1.x  # Set the tensorflow version\n",
        "use_google_drive = True #@param {type:\"boolean\"}\n",
        "gpu_fraction_usage = 0.9 #@param {type:\"slider\", min:0, max:1, step:0.1}\n",
        "###            ###\n",
        "\n",
        "# Upgrade gensim package\n",
        "!pip install gensim --upgrade --quiet\n",
        "\n",
        "# Connect to Google Drive via the bundled client which is the simplest out of many approaches\n",
        "if use_google_drive == True:\n",
        "    from google.colab import drive\n",
        "    drive.mount('/content/gdrive')\n",
        "\n",
        "# GPU usage settings for Tensorflow backend\n",
        "from tensorflow import ConfigProto, Session\n",
        "from keras import backend as K\n",
        "if K.backend() == \"tensorflow\":\n",
        "    from keras.backend.tensorflow_backend import set_session\n",
        "    config = ConfigProto()\n",
        "    config.gpu_options.allow_growth = True\n",
        "    config.gpu_options.per_process_gpu_memory_fraction = gpu_fraction_usage\n",
        "    set_session(Session(config=config))\n",
        "else:\n",
        "    raise ValueError(\"Keras is not using the 'tensorflow' backend\")"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "`%tensorflow_version` only switches the major version: 1.x or 2.x.\n",
            "You set: `1.x  # Set the tensorflow version`. This will be interpreted as: `1.x`.\n",
            "\n",
            "\n",
            "TensorFlow is already loaded. Please restart the runtime to change versions.\n",
            "Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount(\"/content/gdrive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "D4gscjCDrsK_"
      },
      "source": [
        "### Preprocessing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2Ih7BEO27wgz"
      },
      "source": [
        "from keras.datasets import imdb\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "from keras.preprocessing.text import Tokenizer\n",
        "from keras.models import Sequential, load_model\n",
        "from keras.layers import Dense, Embedding, LSTM, CuDNNLSTM, Bidirectional, Dropout \n",
        "from keras.layers import SimpleRNN, GRU, GlobalAveragePooling1D, Conv1D, Flatten\n",
        "from keras.layers import BatchNormalization\n",
        "from keras.optimizers import Adam, SGD, Adagrad\n",
        "from keras.callbacks import Callback, EarlyStopping, ModelCheckpoint\n",
        "from keras.utils import plot_model, np_utils\n",
        "from gensim.models import Word2Vec, KeyedVectors\n",
        "from sklearn.preprocessing import LabelEncoder\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.metrics import confusion_matrix\n",
        "from seaborn import heatmap\n",
        "###            ###\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import time\n",
        "import math\n",
        "import os\n",
        "import matplotlib.pyplot as plt\n",
        "###            ###\n",
        "import random\n",
        "from tensorflow import set_random_seed\n",
        "\n",
        "### Parameters ###\n",
        "random_state = 22 #@param {type:\"slider\", min:0, max:100, step:1}\n",
        "dataset_name = \"SemEval-2017 Task 4\" #@param [\"IMDb Large Movie Review Dataset\", \"Movie Review Subjectivity Dataset\", \"Movie Review Polarity Dataset\", \"SCH Dataset\", \"Finegrained Sentiment Dataset\", \"SemEval-2017 Task 4\"]\n",
        "feature_count = 15000 #@param {type:\"slider\", min:0, max:140000, step:5000}\n",
        "###            ###\n",
        "\n",
        "# Reproducibility\n",
        "random.seed(random_state)  # Python's seed\n",
        "np.random.seed(random_state)  # Numpy's seed\n",
        "set_random_seed(random_state)  # Tensorflow's seed\n",
        "\n",
        "def load_dataset():\n",
        "    ''' Dataset Loading '''\n",
        "    if dataset_name == \"IMDb Large Movie Review Dataset\":\n",
        "        (train_data, train_labels), (test_data, test_labels) = imdb.load_data(seed=random_state, num_words=feature_count)\n",
        "\n",
        "    elif dataset_name == \"Movie Review Subjectivity Dataset\":\n",
        "        data = [\"\" for i in range(10000)]\n",
        "        labels = [\"\" for i in range(10000)]\n",
        "        count = 0\n",
        "        with open('./gdrive/My Drive/Colab Datasets/Movie Review Subjectivity Dataset/plot.tok.gt9.5000', 'r', encoding='iso-8859-15') as file:\n",
        "            for line in file:\n",
        "                data[count] = line.rstrip('\\n')\n",
        "                labels[count] = 0\n",
        "                count += 1\n",
        "        with open('./gdrive/My Drive/Colab Datasets/Movie Review Subjectivity Dataset/quote.tok.gt9.5000', 'r', encoding='iso-8859-15') as file:\n",
        "            for line in file:\n",
        "                data[count] = line.rstrip('\\n')\n",
        "                labels[count] = 1\n",
        "                count += 1   \n",
        "        train_data, test_data, train_labels, test_labels = train_test_split(data, labels, test_size=0.1, random_state=random_state, shuffle=True)   \n",
        "        del data, labels\n",
        "\n",
        "    elif dataset_name == \"Movie Review Polarity Dataset\":\n",
        "        data = [\"\" for i in range(10662)]\n",
        "        labels = [\"\" for i in range(10662)]\n",
        "        count = 0\n",
        "        with open('./gdrive/My Drive/Colab Datasets/Movie Review Polarity Dataset/rt-polarity.neg', 'r', encoding='iso-8859-15') as file:\n",
        "            for line in file:\n",
        "                data[count] = line.rstrip('\\n')\n",
        "                labels[count] = 0\n",
        "                count += 1\n",
        "        with open('./gdrive/My Drive/Colab Datasets/Movie Review Polarity Dataset/rt-polarity.pos', 'r', encoding='iso-8859-15') as file:\n",
        "            for line in file:\n",
        "                data[count] = line.rstrip('\\n')\n",
        "                labels[count] = 1\n",
        "                count += 1    \n",
        "        train_data, test_data, train_labels, test_labels = train_test_split(data, labels, test_size=0.1, random_state=random_state, shuffle=True)  \n",
        "\n",
        "    elif dataset_name == \"SCH Dataset\":\n",
        "        df1 = pd.read_csv('./gdrive/My Drive/Colab Datasets/SCH Dataset/final_data_q13.csv', delimiter=\"|\", header=None)\n",
        "        df2 = pd.read_csv('./gdrive/My Drive/Colab Datasets/SCH Dataset/final_data_q14.csv', delimiter=\"|\", header=None)\n",
        "        df3 = pd.read_csv('./gdrive/My Drive/Colab Datasets/SCH Dataset/final_data_q15.csv', delimiter=\"|\", header=None)\n",
        "        df4 = pd.read_csv('./gdrive/My Drive/Colab Datasets/SCH Dataset/final_data_q16.csv', delimiter=\"|\", header=None)\n",
        "\n",
        "        df =  df1.append([df2, df3, df4], ignore_index=True)\n",
        "        df.rename(columns={0: \"Labels\", 1: \"Data_GR\", 2: \"Data_ENG\"}, inplace=True)\n",
        "        df.dropna(0, inplace=True)    \n",
        "        df[\"Labels\"] = df.apply(lambda row: row['Labels'] if isinstance(row['Labels'], str)==True else str(int(row['Labels'])), axis=1)\n",
        "        df = df.sample(frac=1., random_state=random_state).reset_index(drop=True)\n",
        "\n",
        "        print(f\"--Dataset Info:\\n{df.describe(include='all')}\\n\\n{df.head(4)}\\n\\n{df.iloc[:,0].value_counts()}\\n--\\n\")                  \n",
        "        \n",
        "        train_data, test_data, train_labels, test_labels = train_test_split(df.loc[:,\"Data_ENG\"].values, df.loc[:,\"Labels\"].values, test_size=0.1, random_state=random_state, shuffle=True)\n",
        "\n",
        "    elif dataset_name == \"Finegrained Sentiment Dataset\":\n",
        "        data = [[] for i in range(294)]\n",
        "        labels = [\"\" for i in range(294)]\n",
        "        count = 0\n",
        "        with open('./gdrive/My Drive/Colab Datasets/Finegrained Sentiment Dataset/finegrained.txt', 'r', encoding='iso-8859-15') as file:\n",
        "            for line in file:\n",
        "                if len(line.split(\"_\")) == 3:\n",
        "                    labels[count] = line.split(\"_\")[1]                  \n",
        "                elif len(line.strip()) == 0:\n",
        "                    data[count] = ' '.join(data[count])\n",
        "                    count += 1\n",
        "                else:\n",
        "                    temp = [x.strip() for x in line.split(\"\\t\")]\n",
        "                    if len(temp[1]) > 1:\n",
        "                        # \"nr\" label is ignored\n",
        "                        if temp[0] in [\"neg\", \"neu\", \"pos\", \"mix\"]:\n",
        "                            data[count].append(temp[0])              \n",
        "        \n",
        "        encoder = LabelEncoder()\n",
        "        encoder.fit(labels)\n",
        "        labels = encoder.transform(labels)\n",
        "\n",
        "        train_data, test_data, train_labels, test_labels = train_test_split(data, labels, test_size=0.1, random_state=random_state, shuffle=True)   \n",
        "        del data, labels \n",
        "\n",
        "    elif dataset_name == \"SemEval-2017 Task 4\": \n",
        "        data = [\"\" for i in range(53570)]\n",
        "        labels = [\"\" for i in range(53570)]\n",
        "        count = 0\n",
        "\n",
        "        data_path = \"./gdrive/My Drive/Colab Datasets/SemEval-2017 Task 4/GOLD/Subtask_A\"    \n",
        "        # data_dir_list = os.listdir(data_path)\n",
        "        for data_file in [\"sms-2013test-A.tsv\", \"livejournal-2014test-A.tsv\", \"twitter-2016devtest-A.txt\", \"twitter-2016dev-A.txt\", \"twitter-2016train-A.txt\", \"twitter-2013dev-A.txt\", \"twitter-2013test-A.txt\", \"twitter-2013train-A.txt\", \"twitter-2014test-A.txt\", \"twitter-2015test-A.txt\", \"twitter-2015train-A.txt\", \"twitter-2014sarcasm-A.txt\",  \"twitter-2016test-A.txt\"]:  # Order Matters\n",
        "            data_file = data_path + \"/\" + data_file            \n",
        "            if data_file.endswith(\".tsv\"):\n",
        "                print(f\"Loaded Subset {data_file}\")\n",
        "                with open(data_file, 'r', encoding='utf-8') as file:\n",
        "                    for line in file:\n",
        "                        line_split = line.rstrip('\\n').split('\\t')\n",
        "                        if len(line_split) >= 4:\n",
        "                            data[count] = line_split[3]\n",
        "                            labels[count] = line_split[2]\n",
        "                            count += 1                                       \n",
        "            elif data_file.endswith(\".txt\"):\n",
        "                print(f\"Loaded Subset {data_file}\")\n",
        "                with open(data_file, 'r', encoding='utf-8') as file:\n",
        "                    for line in file:\n",
        "                        line_split = line.rstrip('\\n').split('\\t')\n",
        "                        if len(line_split) >= 3:\n",
        "                            data[count] = line_split[2]\n",
        "                            labels[count] = line_split[1]\n",
        "                            count += 1               \n",
        "\n",
        "        encoder = LabelEncoder()\n",
        "        encoder.fit(labels)\n",
        "        labels = encoder.transform(labels)\n",
        "\n",
        "        train_data, test_data, train_labels, test_labels = train_test_split(data, labels, test_size=0.1, random_state=random_state, shuffle=True)  \n",
        "        del data, labels\n",
        "\n",
        "    else:\n",
        "        raise ValueError(f\"Dataset is not implemented yet\")\n",
        "\n",
        "    # Print Dataset Information\n",
        "    print(f\"{dataset_name} Loaded. Training entries: {len(train_data)}, labels: {len(train_labels)}\")\n",
        "    for i in range(4): print(train_data[i], train_labels[i])\n",
        "    print()\n",
        "\n",
        "    return train_data, test_data, train_labels, test_labels"
      ],
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gwo6jCo37wg6"
      },
      "source": [
        "# NLP Functions\n",
        "def imdb_specific_word_index():\n",
        "    ''' A dictionary mapping words to an integer index '''\n",
        "    word_index = imdb.get_word_index()\n",
        "\n",
        "    # The first indices are reserved\n",
        "    word_index = {k:(v+3) for k,v in word_index.items()}\n",
        "    #word_index[\"<PAD>\"] = 0  # not inserting this key makes the word_index have the same length as in non-IMDb datasets, i.e. for 20000 features we end up with a length of 19999\n",
        "    word_index[\"<START>\"] = 1\n",
        "    word_index[\"<UNK>\"] = 2  # unknown/OOV word\n",
        "    word_index[\"<UNUSED>\"] = 3\n",
        "\n",
        "    reverse_word_index = dict([(value, key) for (key, value) in word_index.items()])\n",
        "    \n",
        "    return word_index, reverse_word_index\n",
        "    \n",
        "def other_datasets_word_index(tokenizer):\n",
        "    ''' A dictionary mapping words to an integer index '''\n",
        "    word_index = tokenizer.word_index\n",
        "    \n",
        "    reverse_word_index = dict([(value, key) for (key, value) in word_index.items()])\n",
        "    \n",
        "    return word_index, reverse_word_index\n",
        "\n",
        "def reduce_word_index_features(word_index, reverse_word_index, train_data):\n",
        "    ''' Update the word index from its generic version to one that matches the number of features we selected '''\n",
        "    word_index = {k: v for k, v in word_index.items() if v < feature_count} \n",
        "    reverse_word_index = {k: v for k, v in reverse_word_index.items() if k < feature_count} \n",
        "\n",
        "    print(\"\\n\".join([decode_review(instance, reverse_word_index, mode=\"join\") for instance in train_data[0:4]]))\n",
        "\n",
        "    return word_index, reverse_word_index\n",
        "\n",
        "def sequence_padding(train_data, test_data, embeddings_sequence_length):\n",
        "    train_data = pad_sequences(train_data,\n",
        "                               padding='pre',  # Using 'pre' instead of 'post' on truncating leads to higher accuracy\n",
        "                               truncating='pre',\n",
        "                               maxlen=embeddings_sequence_length)\n",
        "\n",
        "    test_data = pad_sequences(test_data,\n",
        "                              padding='pre',\n",
        "                              truncating='pre',\n",
        "                              maxlen=embeddings_sequence_length)\n",
        "    \n",
        "    return train_data, test_data   \n",
        "\n",
        "def print_dataset_length_stats(train_data):        \n",
        "    maxim = 0\n",
        "    total = 0\n",
        "    count = 0\n",
        "    for i in train_data:\n",
        "        length = len(i)\n",
        "        if length > maxim:\n",
        "            maxim = length\n",
        "        count += 1\n",
        "        total += length\n",
        "    print(f\"General stats regarding the length of instances of the dataset (to help choose embeddings_sequence_length) - avg:{total/count:.2f} max:{maxim}\\n\")   \n",
        "\n",
        "def decode_review(text, reverse_word_index, mode):\n",
        "    if mode == \"join\":\n",
        "        return ' '.join([reverse_word_index.get(i, '?') for i in text])\n",
        "    else:\n",
        "        return [reverse_word_index.get(i, '?') for i in text]\n",
        "\n",
        "def encode_review(text, word_index):\n",
        "    text = text.split()\n",
        "    return [word_index.get(i, 2) for i in text]  # \"2\" refers to unknown/OOV word        "
      ],
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HCFF9vWhvURq"
      },
      "source": [
        "**Regarding Word2Vec, Embeddings and Google Drive**\n",
        "\n",
        "Converting the Word2Vec bin file to a text file of bigger size is even worse since we are using Google Drive.  \n",
        "Ideally the Google Drive file must be as small as possible because the bottleneck is caused by Download Speed not by the loading process itself.\n",
        "\n",
        "Location: USA  \n",
        "Download: 134.43 Mbit/s  \n",
        "Upload: 178.00 Mbit/s\n",
        "\n",
        "Load time for the zipped file is 170 to 293 seconds.\n",
        "\n",
        "---\n",
        "\n",
        "So here is a great idea: download the file from Drive to Colab [[1](https://stackoverflow.com/questions/48735600/file-download-from-google-drive-to-colaboratory)] [[2](https://medium.freecodecamp.org/how-to-transfer-large-files-to-google-colab-and-remote-jupyter-notebooks-26ca252892fa)] at the very start so that we can maintain access for the entire Runtime.\n",
        "\n",
        "Download time is 73 to 80 seconds.\n",
        "\n",
        "---\n",
        "\n",
        "**Regarding alternatives of Word2Vec**\n",
        "\n",
        "FastText is simply Word2Vec with subword n-grams but requires more RAM and training time\n",
        "\n",
        "---\n",
        "\n",
        "**Regarding simple Word2Vec similarity examples (line 25)**\n",
        "\n",
        "Other than the 3.6GB of RAM that we are using to load the Word2Vec embeddings, we can use simple similarity examples on the model to ensure its working will lead to an extra 3.6GB being used [[1](https://stackoverflow.com/questions/50478046/memory-error-when-using-gensim-for-loading-word2vec)] [[2](https://github.com/RaRe-Technologies/gensim/issues/293#issuecomment-175026483)].  \n",
        "\"I can load the model fine, and can retrieve the word vectors of words fine, but it seems to explode just when I try to access similarity-related functions (including 'doesn't match') etc.\"  \n",
        "\n",
        "If you want to enable the similarity examples, change line 25."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kl5HwN6M7wg_"
      },
      "source": [
        "# Word Embeddings Functions\n",
        "def load_word2vec_pretrained():  \n",
        "    ''' Loads word2vec files from Google Drive, which are used later on '''\n",
        "    time_counter = time.time()\n",
        "    print(f\"Downloading data from Google Drive to Colab hard drive...\")\n",
        "    !mkdir my_data\n",
        "    !cp -i '/content/gdrive/My Drive/Colab Datasets/GoogleNews-vectors-negative300.bin.gz' '/content/my_data/GoogleNews-vectors-negative300.bin.gz'\n",
        "    print(f\"Download completed in {time.time()-time_counter:.2f}sec, displaying information\")\n",
        "    !ls '/content/my_data' -l --block-size=MB\n",
        "\n",
        "    time_counter = time.time()\n",
        "    print(f\"Loading file...\")\n",
        "    word2vec = KeyedVectors.load_word2vec_format('/content/my_data/GoogleNews-vectors-negative300.bin.gz', binary=True, unicode_errors='strict')    \n",
        "    print(f\"Loading completed in {time.time()-time_counter:.2f}sec\")\n",
        "    \n",
        "    # Download, Unzip, Convert from bin to text file, Upload\n",
        "    # word2vec = gzip.open('/content/gdrive/My Drive/Colab Datasets/GoogleNews-vectors-negative300.bin.gz', 'rb')\n",
        "    # word2vec.save_word2vec_format('/content/gdrive/My Drive/Colab Datasets/GoogleNews-vectors-negative300.txt', binary=False)\n",
        "    # word2vec_text_mode = KeyedVectors.load_word2vec_format('/content/gdrive/My Drive/Colab Datasets/GoogleNews-vectors-negative300.txt')    \n",
        "    \n",
        "    dog = word2vec['dog']\n",
        "    print(f\"\\nWord2Vec Embeddings Dimension: {dog.shape}\")\n",
        "    print(f\"Example Values: {dog[:10]}\")\n",
        "    \n",
        "    if False:\n",
        "        # Some predefined functions that show content related information for given words\n",
        "        print(f\"Test 1: {word2vec.most_similar(positive=['woman', 'king'], negative=['man'])}\")\n",
        "        print(f\"Test 2: {word2vec.most_similar('hyundai', topn=10)}\")\n",
        "        print(f\"Test 3: {word2vec.doesnt_match('breakfast cereal dinner lunch'.split())}\")  # Raises Warning\n",
        "        print(f\"Test 4: {word2vec.similarity('woman', 'man')}\")\n",
        "    else:\n",
        "        print(f\"Examples of Word2Vec and similarity function usage won't be run in order to preserve RAM.\\n\")\n",
        "        \n",
        "    return word2vec\n",
        "\n",
        "def find_common_words(embeddings_vocab, original_vocab):\n",
        "    ''' Find words in common between two vocabularies '''\n",
        "    return set(filter(lambda x: x in embeddings_vocab, original_vocab))  # filter is not a good format, if the variable is used once it becomes empty (i.e. a generator), use set instead\n",
        "\n",
        "def find_oov_words(embeddings_vocab, original_vocab):\n",
        "    ''' Find words not in common between two vocabularies (OOV words) '''\n",
        "    return set(filter(lambda x: x not in embeddings_vocab, original_vocab))  # filter is not a good format, if the variable is used once it becomes empty (i.e. a generator), use set instead\n",
        "\n",
        "def manage_oov_words(embeddings, vocab):\n",
        "    ''' Find words in common, meaning the remaining words are out-of-vocabulary (OOV) ones '''\n",
        "    words_oov = find_oov_words(embeddings.vocab, vocab)  # Find words in common, meaning the remaining words are out-of-vocabulary (OOV) ones\n",
        "    print(f\"Impossible to remove all occurances of {len(words_oov)} words since time complexity would be too high. Out-of-vocabulary words will be kept and will be assigned vectors...\")\n",
        "    if len(words_oov) >= 3:\n",
        "        to_print = iter(words_oov)\n",
        "        print(f\"Some examples: {next(to_print)}, {next(to_print)}, {next(to_print)}\")\n",
        "    \n",
        "    #encoding_oov = [vocab[word] for word in words_oov]\n",
        "    #for instance in data_container:\n",
        "        #[word for word in instance if word not in encoding_oov]  # looking up a set is the absolute fastest in python\n",
        "\n",
        "def assign_embeddings(embeddings, embeddings_dimension, vocab, mode):\n",
        "    ''' Create an embeddings (weight) matrix for the Embedding layer from a loaded embedding ''' \n",
        "    words_in_common = find_common_words(embeddings.vocab, vocab)\n",
        "    words_oov = find_oov_words(embeddings.vocab, vocab)  # Find words in common, meaning the remaining words are out-of-vocabulary (OOV) ones\n",
        "    print(f\"Number of vocabulary words that cannot be found in the Word2Vec embeddings: {len(words_oov)}\")  \n",
        "    \n",
        "    # \"Total vocabulary size plus 0 for unknown words 'len(vocab) + 1\" is not entirely true, the index of 0 is simply not used leading to, for example a length of 19999 for 20000 features:\n",
        "    vocab_size = len(vocab) + 1\n",
        "    # Initialize the weight matrix with 0s\n",
        "    embed_final_matrix = np.zeros((vocab_size, embeddings_dimension))\n",
        "    # Store vectors using an integer mapping, for example from the Tokenizer\n",
        "    \n",
        "    if mode == \"zeros\":\n",
        "        for word in words_in_common:\n",
        "            embed_final_matrix[vocab[word]] = embeddings[word]\n",
        "    elif mode == \"random\":\n",
        "        np.random.seed(random_state)\n",
        "        for word, i in vocab.items():\n",
        "            if word in words_oov:\n",
        "                embed_final_matrix[i] = np.random.uniform(low=-0.5, high=0.5, size=embeddings_dimension)\n",
        "            else:\n",
        "                embed_final_matrix[i] = embeddings[word]\n",
        "    else:\n",
        "        raise ValueError(f\"{mode} is not a valid mode parameter.\")\n",
        "    \n",
        "    print(f\"Embeddings assignment completed.\\n\")      \n",
        "    return embed_final_matrix"
      ],
      "execution_count": 30,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cyPF2xpUK-C9"
      },
      "source": [
        "### Preprocessing Run"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NeUcmuIq7whK",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "45bc0db4-5877-48a2-a935-ac10db046793"
      },
      "source": [
        "### Parameters ###\n",
        "remove_first = False #@param {type:\"boolean\"}\n",
        "embeddings_mode = \"One-hot Encoding\" #@param [\"One-hot Encoding\", \"Tokenizing\", \"Word2Vec Pretrained\", \"Word2Vec Training\"]\n",
        "embeddings_sequence_length = 50 #@param {type:\"integer\"}\n",
        "trainable = False #@param {type:\"boolean\"}\n",
        "outofvocab_mode = \"random\" #@param [\"zeros\", \"random\"]\n",
        "###            ###\n",
        "\n",
        "# Load Dataset\n",
        "train_data, test_data, train_labels, test_labels = load_dataset()\n",
        "print_dataset_length_stats(train_data)\n",
        "\n",
        "# Remove the <START> symbol from all instances\n",
        "if remove_first == True:\n",
        "    if train_data[0][0] == 1:  # word_index[\"<START>\"] = 1\n",
        "        for i in range(len(train_data)):\n",
        "            train_data[i] = train_data[i][1:]\n",
        "        for i in range(len(test_data)):\n",
        "            test_data[i] = test_data[i][1:]\n",
        "\n",
        "# Create the Embeddings            \n",
        "if embeddings_mode == \"One-hot Encoding\":  \n",
        "    ''' \n",
        "        ONE-HOT ENCODING\n",
        "        Description: Not traditional One-hot, but instead [3, 62, 5, 90, ...] \n",
        "        Embedding_Layer: Yes\n",
        "    '''   \n",
        "    if dataset_name != \"IMDb Large Movie Review Dataset\":  # keras.imdb is already One-hot Encoded\n",
        "        tokenizer = Tokenizer(num_words=feature_count, \n",
        "                              lower=True, \n",
        "                              split=' ', \n",
        "                              oov_token=\"<UNK>\")\n",
        "        tokenizer.fit_on_texts(train_data)\n",
        "        # 'texts_to_sequences' list of strings as input and sequence of integers as output, 'texts_to_matrix' is meant to return a matrix of counts/tf-idfs\n",
        "        train_data = tokenizer.texts_to_sequences(train_data)\n",
        "        test_data = tokenizer.texts_to_sequences(test_data)   \n",
        "    \n",
        "    # Word_Index Stuff\n",
        "    if dataset_name == \"IMDb Large Movie Review Dataset\":  # keras.imdb is already One-hot Encoded\n",
        "        word_index, reverse_word_index = imdb_specific_word_index()\n",
        "    else:\n",
        "        word_index, reverse_word_index = other_datasets_word_index(tokenizer)\n",
        "\n",
        "    word_index, reverse_word_index = reduce_word_index_features(word_index, reverse_word_index, train_data)  # Update the word index to match the number of features we selected       \n",
        "  \n",
        "    # Peform Sequence Padding\n",
        "    train_data, test_data = sequence_padding(train_data, test_data, embeddings_sequence_length)\n",
        "\n",
        "elif embeddings_mode == \"Tokenizing\":  \n",
        "    ''' \n",
        "        TOKENIZING\n",
        "        Description: Unlike other modes this isn't exactly an embedding, leads to a collection of floats [0.00, 0.02, 0.12, 0.04, ...] based on tf-idf \n",
        "        Embedding_Layer: No\n",
        "    '''    \n",
        "    if dataset_name == \"IMDb Large Movie Review Dataset\":  # keras.imdb is already One-hot Encoded so let's transform it back to text\n",
        "        word_index, reverse_word_index = imdb_specific_word_index()\n",
        "        train_data = [decode_review(instance, reverse_word_index, mode=\"join\") for instance in train_data]\n",
        "        test_data = [decode_review(instance, reverse_word_index, mode=\"join\") for instance in test_data]\n",
        "    \n",
        "    tokenizer = Tokenizer(num_words=feature_count, \n",
        "                          lower=True, \n",
        "                          split=' ', \n",
        "                          )\n",
        "    tokenizer.fit_on_texts(train_data)\n",
        "    # 'texts_to_matrix' list of strings as input, 'sequences_to_matrix' list of integer word indices as input \n",
        "    train_data = tokenizer.texts_to_matrix(train_data, mode='tfidf')\n",
        "    test_data = tokenizer.texts_to_matrix(test_data, mode='tfidf')\n",
        "\n",
        "elif embeddings_mode == \"Word2Vec Pretrained\":\n",
        "    ''' \n",
        "        WORD2VEC PRETRAINED\n",
        "        Description: A much more advanced form of embeddings that is created through training a model unlike previous modes. Implements the CBOW and the Skip-gram models in order to learn word embeddings.\n",
        "        Embedding_Layer: Yes\n",
        "    '''\n",
        "    word2vec = load_word2vec_pretrained()  # This line can be commented out, if it was already loaded in the current session\n",
        "    embeddings_dimension = word2vec.vector_size\n",
        "    \n",
        "    if dataset_name != \"IMDb Large Movie Review Dataset\":  # keras.imdb is already One-hot Encoded\n",
        "        tokenizer = Tokenizer(num_words=feature_count, \n",
        "                              lower=True, \n",
        "                              split=' ', \n",
        "                              oov_token=\"<UNK>\")\n",
        "        tokenizer.fit_on_texts(train_data)\n",
        "        # 'texts_to_sequences' list of strings as input and sequence of integers as output, 'texts_to_matrix' is meant to return a matrix of counts/tf-idfs\n",
        "        train_data = tokenizer.texts_to_sequences(train_data)\n",
        "        test_data = tokenizer.texts_to_sequences(test_data)   \n",
        "    \n",
        "    # Word_Index Stuff\n",
        "    if dataset_name == \"IMDb Large Movie Review Dataset\":  # keras.imdb is already One-hot Encoded\n",
        "        word_index, reverse_word_index = imdb_specific_word_index()\n",
        "    else:\n",
        "        word_index, reverse_word_index = other_datasets_word_index(tokenizer)\n",
        "\n",
        "    word_index, reverse_word_index = reduce_word_index_features(word_index, reverse_word_index, train_data)  # Update the word index to match the number of features we selected       \n",
        "\n",
        "    # Peform Sequence Padding  \n",
        "    train_data, test_data = sequence_padding(train_data, test_data, embeddings_sequence_length)\n",
        "   \n",
        "    manage_oov_words(word2vec, word_index)\n",
        "    embedding_vectors = assign_embeddings(word2vec, embeddings_dimension, word_index, mode=outofvocab_mode)\n",
        "\n",
        "elif embeddings_mode == \"Word2Vec Training\":\n",
        "    ''' \n",
        "        Description: A much more advanced form of embeddings that is created through training a model unlike previous modes. Implements the CBOW and the Skip-gram models in order to learn word embeddings.\n",
        "        Embedding_Layer: Yes\n",
        "    '''\n",
        "    if dataset_name != \"IMDb Large Movie Review Dataset\":  # keras.imdb is already One-hot Encoded so let's transform it back to text\n",
        "        tokenizer = Tokenizer(num_words=feature_count, \n",
        "                              lower=True, \n",
        "                              split=' ', \n",
        "                              oov_token=\"<UNK>\")\n",
        "        tokenizer.fit_on_texts(train_data)\n",
        "        # 'texts_to_sequences' list of strings as input and sequence of integers as output, 'texts_to_matrix' is meant to return a matrix of counts/tf-idfs\n",
        "        train_data = tokenizer.texts_to_sequences(train_data)\n",
        "        test_data = tokenizer.texts_to_sequences(test_data)  \n",
        "        \n",
        "    # Word_Index Stuff\n",
        "    if dataset_name == \"IMDb Large Movie Review Dataset\":  # keras.imdb is already One-hot Encoded so let's transform it back to text\n",
        "        word_index, reverse_word_index = imdb_specific_word_index()\n",
        "    else:\n",
        "        word_index, reverse_word_index = other_datasets_word_index(tokenizer)\n",
        "\n",
        "    word_index, reverse_word_index = reduce_word_index_features(word_index, reverse_word_index, train_data)  # Update the word index to match the number of features we selected       \n",
        "      \n",
        "    # Train a Model\n",
        "    time_counter = time.time()    \n",
        "    print(f\"\\nTraining a new custom Word2Vec model on the dataset...\")\n",
        "    word2vec = Word2Vec([decode_review(instance, reverse_word_index, mode=\"don't join\") for instance in train_data],  # Default is 5 epochs \n",
        "                         size=300, sg=0, window=5, min_count=1, iter=5,\n",
        "                         seed=random_state, alpha=0.025, workers=4)\n",
        "    embeddings_dimension = word2vec.wv.vector_size\n",
        "    print(f\"Training completed in {time.time()-time_counter:.2f}sec. Vocabulary size: {len(word2vec.wv.vocab)}\\n\")  \n",
        "\n",
        "    # Peform Sequence Padding  \n",
        "    train_data, test_data = sequence_padding(train_data, test_data, embeddings_sequence_length)\n",
        "    \n",
        "    manage_oov_words(word2vec.wv, word_index)   \n",
        "    embedding_vectors = assign_embeddings(word2vec.wv, embeddings_dimension, word_index, mode=outofvocab_mode)    \n",
        "    \n",
        "    # If we want to store the model\n",
        "    #if False:\n",
        "    #    model.wv.save_word2vec_format(\"My_Word2Vec.txt\", binary=False)    \n",
        "\n",
        "# Print the resulting instances       \n",
        "for i in range(4): print(type(train_data[i]), list(train_data[i]))"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Loaded Subset ./gdrive/My Drive/Colab Datasets/SemEval-2017 Task 4/GOLD/Subtask_A/sms-2013test-A.tsv\n",
            "Loaded Subset ./gdrive/My Drive/Colab Datasets/SemEval-2017 Task 4/GOLD/Subtask_A/livejournal-2014test-A.tsv\n",
            "Loaded Subset ./gdrive/My Drive/Colab Datasets/SemEval-2017 Task 4/GOLD/Subtask_A/twitter-2016devtest-A.txt\n",
            "Loaded Subset ./gdrive/My Drive/Colab Datasets/SemEval-2017 Task 4/GOLD/Subtask_A/twitter-2016dev-A.txt\n",
            "Loaded Subset ./gdrive/My Drive/Colab Datasets/SemEval-2017 Task 4/GOLD/Subtask_A/twitter-2016train-A.txt\n",
            "Loaded Subset ./gdrive/My Drive/Colab Datasets/SemEval-2017 Task 4/GOLD/Subtask_A/twitter-2013dev-A.txt\n",
            "Loaded Subset ./gdrive/My Drive/Colab Datasets/SemEval-2017 Task 4/GOLD/Subtask_A/twitter-2013test-A.txt\n",
            "Loaded Subset ./gdrive/My Drive/Colab Datasets/SemEval-2017 Task 4/GOLD/Subtask_A/twitter-2013train-A.txt\n",
            "Loaded Subset ./gdrive/My Drive/Colab Datasets/SemEval-2017 Task 4/GOLD/Subtask_A/twitter-2014test-A.txt\n",
            "Loaded Subset ./gdrive/My Drive/Colab Datasets/SemEval-2017 Task 4/GOLD/Subtask_A/twitter-2015test-A.txt\n",
            "Loaded Subset ./gdrive/My Drive/Colab Datasets/SemEval-2017 Task 4/GOLD/Subtask_A/twitter-2015train-A.txt\n",
            "Loaded Subset ./gdrive/My Drive/Colab Datasets/SemEval-2017 Task 4/GOLD/Subtask_A/twitter-2014sarcasm-A.txt\n",
            "Loaded Subset ./gdrive/My Drive/Colab Datasets/SemEval-2017 Task 4/GOLD/Subtask_A/twitter-2016test-A.txt\n",
            "SemEval-2017 Task 4 Loaded. Training entries: 48213, labels: 48213\n",
            "cant wait to see the iPad HD .. 2\n",
            "@DebsODo: @lex_looper might want to double check your Saudi Arabia info on this one. 1. https://t.co/sToO95DdWt 2. https://t.co/T9j1IOsD3X 1\n",
            "@Pixe1ina getting more people who never would have touched a Dark SOuls in the 1st place 1\n",
            "@curtismufc10 Well Curtis... you also NEED a night out in Thompsons .... need a Sunday night session again soon bro!!!!! 2\n",
            "\n",
            "General stats regarding the length of instances of the dataset (to help choose embeddings_sequence_length) - avg:113.49 max:338\n",
            "\n",
            "cant wait to see the ipad hd\n",
            "<UNK> <UNK> looper might want to double check your saudi arabia info on this one 1 https t co <UNK> 2 https t co <UNK>\n",
            "<UNK> getting more people who never would have touched a dark souls in the 1st place\n",
            "<UNK> well curtis you also need a night out in <UNK> need a sunday night session again soon bro\n",
            "<class 'numpy.ndarray'> [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 745, 140, 3, 42, 2, 768, 2288]\n",
            "<class 'numpy.ndarray'> [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 9451, 267, 85, 3, 1045, 275, 71, 635, 769, 1167, 9, 26, 58, 92, 37, 6, 8, 1, 61, 37, 6, 8, 1]\n",
            "<class 'numpy.ndarray'> [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 244, 110, 158, 98, 197, 115, 25, 5124, 7, 577, 697, 5, 2, 47, 225]\n",
            "<class 'numpy.ndarray'> [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 139, 508, 15, 195, 163, 7, 41, 38, 5, 1, 163, 7, 51, 41, 1677, 171, 466, 1337]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yxp-o330wt9U",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "446f8c2c-2a73-4aa5-bfc7-40d15915a6dc"
      },
      "source": [
        "# In case of Multi-Class Classification, labels must be One-hot Encoded\n",
        "if len(set(train_labels)) > 2:\n",
        "    train_labels = np_utils.to_categorical(train_labels)\n",
        "    test_labels = np_utils.to_categorical(test_labels)\n",
        "    multiclass = True\n",
        "    print(f\"The current task is: Multi-clas Classification. Labels converted to One-hot Encoding.\")  \n",
        "else:\n",
        "    multiclass = False    \n",
        "    print(f\"The current task is: Binary Classification\")    "
      ],
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "The current task is: Multi-clas Classification. Labels converted to One-hot Encoding.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xHf4bh6UIT8a"
      },
      "source": [
        "### Training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oo08HBNi7whY",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "62ead3c7-ca82-4a53-d14d-bec7050a7f25"
      },
      "source": [
        "# For Multi-class tasks in particular, the following changes need to be performed\n",
        "# 1. Dense(1), the value of 1 on the final layer refers to the number of categories\n",
        "# 2. activation = \"softmax\"\n",
        "# 3. loss = \"categorical_crossentropy\" https://stackoverflow.com/a/46038271\n",
        "\n",
        "if embeddings_mode == \"Tokenizing\":\n",
        "    model = Sequential()\n",
        "    model.add(Dense(units=16, input_shape=(feature_count,)))  # when Embedding layer is not used, the first layer must include the input_shape parameter which refers to the vocabulary/feature size\n",
        "    model.add(Dense(units=16, activation='relu', use_bias=True))\n",
        "    model.add(Dense(units=16, activation='relu', use_bias=True))\n",
        "    model.add(Dense(1, activation='sigmoid'))\n",
        "\n",
        "elif embeddings_mode == \"One-hot Encoding\":\n",
        "    model = Sequential()\n",
        "    model.add(Embedding(input_dim=feature_count, output_dim=16, input_length=embeddings_sequence_length))    \n",
        "    model.add(GlobalAveragePooling1D()) \n",
        "    model.add(Dense(units=16, activation='relu', use_bias=True))\n",
        "    model.add(Dense(units=16, activation='relu', use_bias=True))\n",
        "    model.add(Dense(1, activation='sigmoid'))\n",
        "\n",
        "elif embeddings_mode == \"Word2Vec Pretrained\" or embeddings_mode == \"Word2Vec Training\":  # Word2Vec Embeddings\n",
        "    model = Sequential()\n",
        "    model.add(Embedding(input_dim=feature_count, output_dim=embeddings_dimension, weights=[embedding_vectors], input_length=embeddings_sequence_length, trainable=trainable))\n",
        "    model.add(GlobalAveragePooling1D())\n",
        "    model.add(Dense(units=32, activation='relu', use_bias=True))\n",
        "    model.add(Dense(units=32, activation='relu', use_bias=True))\n",
        "    model.add(Dense(1, activation='sigmoid'))\n",
        "\n",
        "model.summary()\n",
        "\n",
        "model.compile(optimizer=Adam(lr=0.0005),\n",
        "              loss='binary_crossentropy',\n",
        "              metrics=['accuracy'])"
      ],
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"sequential_2\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "embedding_2 (Embedding)      (None, 50, 16)            240000    \n",
            "_________________________________________________________________\n",
            "global_average_pooling1d_2 ( (None, 16)                0         \n",
            "_________________________________________________________________\n",
            "dense_4 (Dense)              (None, 16)                272       \n",
            "_________________________________________________________________\n",
            "dense_5 (Dense)              (None, 16)                272       \n",
            "_________________________________________________________________\n",
            "dense_6 (Dense)              (None, 1)                 17        \n",
            "=================================================================\n",
            "Total params: 240,561\n",
            "Trainable params: 240,561\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ItAKl8fSqIhu",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 487
        },
        "outputId": "18703062-d7ea-42b4-af51-8eee205300b8"
      },
      "source": [
        "plot_model(model, show_shapes=True, show_layer_names=True, dpi=72)"
      ],
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgAAAAHWCAIAAACOlaYCAAAABmJLR0QA/wD/AP+gvaeTAAAgAElEQVR4nOzdfVwTV744/jMR8jiUTUgICGiCLhYVQ0VXtN5ra+Vaa21pV0GkYoV12T7wQnCvINBqsVi8rXgvoZC1FLulqGxd3dUXi4qtqCiIbiutlEUtrCIGCChNIMQ8ze+P+Ta/NCQhQCDgfN5/+CInZ858zmTMJ/NwzmAEQSAAAADUQ3N1AAAAAFwDEgAAAFAUJAAAAKAoN1cH4EoYhrk6BACAi1H5OiilEwCi9mcPxgeGYZNoN5tc0Y4exX8FwikgAACgKEgAAABAUZAAAACAoiABAAAARUECAGCiWLVq1ccff+zqKIag0+lyc3OTkpJwHMcwLC8vjyyvqanx9/en0+mbN28eu7WLRCLsZ0uXLjWVNzc3h4eHM5nM8PDwmzdvIoSOHj1aWVk5dpE8JggKo3j3wfhw1W727rvvtrW1DXcp+9Hq9frIyMj6+nqCIKRSqUgk4nK5PT095LttbW0JCQkji9ZBVts3Go3z5s3LyspSq9VvvvlmaGgoWZ6RkfHpp5/ab5DiXwJwBADA46m8vNzpbe7Zs8fb23vhwoXky+zsbCaTuXv3bqevaFiuX7/e2Ni4Y8cOFou1e/fuhoaG69evI4SysrJ27dr1ww8/uDa8iQwSAAATQklJCZPJzMrKSktLwzDsjTfeCA4OxnE8OzubrJCamophWEREBI7jYrH4yy+/RAhFRUVhGHb79u379++LRCIcx8nK0dHRzc3NAQEBb7/9NkJozZo1KSkpo4zQYDDIZLK4uDhTCZfLLSoqKiws/PHHHwfXP3v2bGhoKI7jEonk9OnTCCFbXUMIHTt2LCgoyNPTMz4+XqvVDiuwhoYGsVjMZrMRQjwez9/fv6GhASHEYrEiIyMPHDgwsv5SgqsPQVyJ4t0H48Px3SwhISEzM5MgCKFQePHiRaPR+Nlnn7HZbFMFDodz5swZtVotk8mYTKZcLifbv3XrFkEQ3377LYfDIWvqdDqEkHNPAdXV1SGElEol+VIqlZ48eZIgiOjo6LVr1xK/PAWkUCg4HE5ZWZlSqZTJZBwOp7Oz01bX5HI5i8U6efJkT09PWFhYfn6+rRjIzMFkMkNCQsrLy8nCjz76SCKRmOqEhITs27eP/LukpEQsFo+sv1QARwAATFAYhi1btkytVuv1elOhr68vi8VKTEz08vKqrq4ez3haW1sZDIaHh4dFuVQqPX/+fG1trXlhVVWVUCjcsGGDh4cHGe3XX39teteia9XV1QEBAS+++CKPx3vppZcuXLhgK4aKigq5XN7R0ZGcnBwbG2v19I7RaKTT6eTffD7/zp07BJXGNg8LJAAAJiVvb++enp7xXOPAwACDwRhcLhAI8vPzt23bZl7Y2dkpEAhML4VCYWdnp62WFQrFzZs3yXt7du7c2dvba6umWCzGcdzT0zMhISEwMJDMOnw+v6+vz1Snt7dXKBSSfzMYDKPRqNFoHO4ltUACAGDyIQiivb3dz89vPFfKZrNtnZ1fv369UCgkL0uQfHx8FAqF6WVHR4ePj4+tlrlcbkhIiOm8RFVVlSPx6PV6FouFEJJIJK2trWQO6O7ubm9vnzVrFllHq9XSaDQmk+lIgxQECQCAyaSvr0+j0UilUq1Wu3z5coQQjuOXLl3S6XTt7e2majQajUajNTU1qdVqZ61aJBJpNBrz39rmCgsLzQcxrFixoqurq6ysTKVSyWSyhw8frlixwlbLzzzzTHNz86FDh/r7+9Vqta0jgMbGxoyMDLVarVQqi4qKuru7n3vuOYSQRCKZO3duTk6OWq3OzMwMDQ2dN28euYhCoZg+fTrFZ3yzx2VXHyYAincfjA8Hd7O0tDQ6nU7eyoIQmj59em9v75w5cxBCsbGxZB0Oh8Pj8dzd3SUSSXV1NVn4zjvvMJnMoKCgxMREhNDmzZvJ8qioKAaDERMTQxDE6tWrk5OTRxmtXq+fOnVqXV0dQRD79+/HcZzH4xUXF5sqHDx40Pw+/TNnzoSEhLDZbIlEUlVVRRDE9u3bbXXtyJEjQUFBDAZj0aJF165dUygUAoFAKpWaByCXy4OCgphMJo7jS5YsqampMb3V1NS0YMECBoMRHh7e3NxsKk9JSbHfcYp/CVBr6lcLVJv5FriEE3czHMfr6+tnz57tlNassh9tTk7OgwcP9u3bN3YBkHQ6XWxsrEQiyczMHHEjer0+ODj4xIkTwcHBtupQ/EsATgE5zezZszEM6+7udrD+3bt3AwMDMQyzuEJlUe7c6QGUSuW8efNwHOdyuatWrbp9+7b9+uM5OcGf//xnLpeLYdjMmTPv3r3r3Mbz8vLIqQtmzJhB3s44SRmNRheuPT09vaWl5cqVK2O9IplMxufzU1NTR9NIVlZWRkaGnW9/QOnDH+d2/+HDhwghhULh+CJyuRwhNDAw4GD56CkUisTERKVS+eDBg/Xr1y9atMjpq7BjyMkJjh8/7twPxXyNUqlUKBQ6sXEHOatHr732GkLIz8/vn//8p1MatGrIaLVabU5OTktLy9jF4BRHjhz5xz/+MWQ1in8HwhGAk7m7uzte2da1qdFcs6qrq2tsbLT1Lp/Pl8lkHh4eXC43Pj7+6tWrBoNhxOsarrGYnGCirXHslJaWEgRx7969+fPnuzAMd3f3jIwMsVjswhgcER0dvWrVKldHMdFBArDJYmx6SkoKhmEBAQEcDsfd3T0oKGjGjBkcDsfT07O0tNS0VEhICIPBmDlz5uHDh622gxA6ffp0aGgoOZrRfI2Dy03TAyC7w+grKiqefPJJJpMZEBCwY8cO0z1w9vX393t5eU2ZMsVWBddOTmB1pVbXaGulFmu04/e//z2Xy2WxWBs3bjQajatXr8YwTCQS3b9//69//aunpyd52t3io0xKSsIwrLKycu3atRkZGY5scwAmFlcfgriSne5bHZvu5+dXXl6u1WqPHDni7u5++/btR48eZWZm/uY3vyF+PgX03XffDQwMHDhwwN3d/d///vfgdjo7O5lMZkFBwcDAwK1bt9DPp3pslZumByBsDKPXaDQeHh5lZWX9/f2pqanh4eEOdj81NTUpKcl+nXGenMDiFJDVlVpdo9WVWqzRzimgpKQkuVx+69Ytd3f3Gzdu9Pf3e3p6/u1vfyPfTUxMlMvlVncJoVBYWlra29ubm5trq1OT63/Z5Ip29KjWXwtwBGCdrbHp3t7e7u7uy5Yt0+l0fn5+dDo9PDz8p59+Mi3o6+vLZDK3bNni7+9//vz5we1UVlYKhcK33nqLvJvNtKCt8sEshtG3tbWpVKqXX36ZzWa/8MILVqflGqytre3UqVPvvffesDaLSyYnGLxSp68xPz/fx8dn5syZPB5PpVKx2eyYmJhDhw4hhHQ6nU6n8/HxsbVLiEQiT0/PtLS00YcBwDhzc3UAE5RpbDr50s4YFlu8vLx6e3sxDLNoRy6XT5s2bXB9W+VD8vHxYTKZf//731955ZWKigpH7hFUqVRbtmw5evQol8sdwRptGf/JCZyyRnJrnD17VqlUkgcNCKGEhIT//M//VKlUFy5cePXVV9HodonJNRBpckULRgMSgHXk2PTvvvtuZIsTBNHW1hYQENDf32/Rzp/+9CfzIfLma7RaPiQcx3Nzc7ds2RIfH//UU099+umn9uurVKr4+Pj9+/c79/Y4YtwnJ3DWGj///POmpqbr16/7+vqaWluwYEFQUNDx48f/9a9/kfPdj2aXICbPneZUuy+e4tkOTgFZ5+DY9MEGBgY0Gk1+fr5Op4uIiBjczrPPPnvz5s3S0tK+vr6KigrTgrbKh6RWq8vLy2/cuKHRaGpra+0fASiVyoSEhA8++MCJ3/7jPznB4DVaXamdNRIE0dvbSw6dffToEYPBwHG8ubnZfExGQkLCwYMHeTweeZ18xLsEABOXKy9AuJr97luMTd+6dStCyM/P78aNG+RtNiEhId9//71QKMQwLD09fWBgYOXKlV5eXgwGIyws7NKlS1bbIQiiqKjI39+fy+WSd3ZHRkaSNQeXm6YH2Lt3r61h9BqNZvHixeSniWFYYGCgaZKAwYqLiy12gIsXL9qqPM6TE3z++efkKalf//rXd+/etdVfq2u0tVLTGsmpCyz6/vrrrxMEcefOnVmzZnE4nJiYmBkzZsyYMcNgMBAE8eDBAzab3draamuXSEpKQgj5+vqaPmurJtf/sskV7ehRrb8WqHW4Z+HxONrt6enZtm3bJ5984u7ubjAYsrOzv//++2PHjo3P2sdhcgJXrZEgiK1bt/7f//3fKNuZXLvZ5Ip29KjWXwtwCmjS++qrr+7evdvb26vVapubmy9cuCAQCDBr7t27N3jxe/fuOV7ZqvGfnGCs13jx4sX+/v5du3a9/PLLY7oiAFwLEsCkt3r1aqFQSJ7EWLly5dNPP11QUGD1cM/f33/w4v7+/o5XtrBx48b+/v7nn3/+m2++GYOeuWyNhYWFPj4+GIaZLjAAE51Ol5ubm5SURM6tlJeXR5bX1NT4+/vT6fTNmzeP3dpFIpHpN8rSpUtN5c3NzeHh4UwmMzw8/ObNmwiho0ePVlZWjl0kj4mxObM0OVC8+2B8OHE3G3IypdEvZT9avV4fGRlZX19PEIRUKhWJRFwut6enh3zX/JnAY8Rq+0ajcd68eVlZWWq1+s033wwNDSXLMzIyPv30U/sNUvxLAI4AAJg0Rja1kRMnRNqzZ4+3t/fChQvJl9nZ2Uwmk7xN1oWuX7/e2Ni4Y8cOFou1e/fuhoaG69evI4SysrJ27dpl9bnBgAQJAABXOnv2bGhoKI7jEonk9OnTyIHJlMgTICObEGnIKZjsMBgMMpksLi7OVMLlcouKigoLC62OPx/cNTvzWQ2eMstxDQ0NYrGYvF2Nx+P5+/s3NDQghFgsVmRk5IEDB0bWX0pw9SGIK1G8+2B82NnNFAoFh8MpKytTKpUymYzD4XR2dhIOTKY04gmRRhMt+RwFpVJJvpRKpSdPniQIIjo6eu3atcQvTwHZ6prV+Z2szrNkFZk5yAkTy8vLycKPPvpIIpGY6oSEhOzbt4/8u6SkRCwWj6y/VABHAAC4TFVVlVAo3LBhg4eHBzm10ddff+3gsmM9BdNgra2tDAbDw8PDolwqlZ4/f762tta80H7XLOZ3sjXP0mAVFRVyubyjoyM5OTk2Ntbq6R2j0Uin08m/+Xz+nTt3CArf6GkfJAAAXKazs1MgEJheCoXCzs7O4TYyblMwDQwMMBiMweUCgSA/P3/btm3mhcPqmmmeJQzDdu7caWeUtVgsxnHc09MzISEhMDCQzDp8Pt/8UfW9vb1CoZD8m8FgGI1Gi4fuARNIAAC4jI+Pj/kEUB0dHT4+PsNqgRjHKZjYbLats/Pr168XCoWmqxFomF0j51kynZeoqqpyJB69Xs9isRBCEomktbWVzAHd3d3t7e2mR2JotVoajcZkMh1pkIIgAQDgMitWrOjq6iorK1OpVDKZ7OHDh+Qko45MpjT6CZGGSyQSaTQa89/a5goLC80fH22ra1Y5OM9SY2NjRkaGWq1WKpVFRUXd3d3PPfccQkgikcydOzcnJ0etVmdmZoaGhs6bN49cRKFQTJ8+neIzvtnjsqsPEwDFuw/Gh/3d7MyZMyEhIWw2WyKRVFVVkYVDTqY0sgmRCGtTMDkerV6vnzp1al1dHUEQ5NxKPB6vuLjYVOHgwYPm9+kP7pqt+Z2IQfMsKRQKgUAglUrNA5DL5UFBQeQDM5YsWVJTU2N6q6mpacGCBQwGIzw8vLm52VSekpIy4v5SAaXnwaD4NCBgfIzFbjZ2EyLZjzYnJ+fBgwf79u1z+not6HS62NhYiUSSmZk54kb0en1wcPCJEyfszH1L8S8BOAUEwKQ0/lMwIYTS09NbWlquXLky1iuSyWR8Pj81NXU0jWRlZWVkZDj3uRePGUpnP4onfzA+nL6bbdy48YsvvvDz8ztx4sT8+fOd2DJyIFqdTvfhhx/GxMSIxWLnrtq5ysvLn3jiiVWrVtmvRvEvAWp3ntqfPRgfk2s3m1zRjh7V+msBTgEBAABFQQIAAACKggQAAAAURenzXzA8BABA6e9AKnceUNyuXbtM/wJAQXAKCAAAKAoSAAAAUBQkAAAAoChIAAAAQFGQAAAAgKIgAQAAAEVBAgAAAIqCBAAAABQFCQAAACgKEgAAAFAUJAAAAKAoSAAAAEBRkAAAAICiIAEAAABFQQIAAACKggQAAAAUBQkAAAAoChIAAABQFCQAAACgKEgAAABAUZAAAACAoiABAAAARUECAAAAioIEAAAAFOXm6gAAGFdqtfry5cvk3y0tLQihs2fPki+XLFnCZrNdFhkA4w4jCMLVMQAwfvR6vUAg0Gq1bm5u5M6PYZher6fT6QqFws0NfhIBCoFTQIBa3Nzc1q1b9+jRI6VSqVKpVCqVUql89OjRunXr4NsfUA0kAEA5r7/+OovFMi9hs9mbN292VTwAuAqcAgKUQxCEUChUKBSmEj6f39XVhWGYC6MCYPzBEQCgHAzD4uLi6HQ6+dLd3X3Tpk3w7Q8oCBIAoKK4uDjTGX86nb5p0ybXxgOAS8ApIEBRIpHozp07CKGAgIC7d++6OhwAXACOAABFxcfHM5lMBoOxZcsWV8cCgGvAEQCgqJaWlrlz5yKEbty4ERgY6OpwAHABSACAuubMmYNh2I0bN1wdCACuAQngF+BWEAAeb/CNZw6GPlqC/YM6Ojo6EEI+Pj6mEgybTD+JJle0EwH8wrMAO9AvwP8oiptcO8DkinYigC1mAe4CAgAAioIEAAAAFAUJAAAAKAoSAAAjsWrVqo8//tjVUYyWTqfLzc1NSkrCcRzDsLy8PLK8pqbG39+fTqeP6SSpIpEI+9nSpUtN5c3NzeHh4UwmMzw8/ObNmwiho0ePVlZWjl0k1EUAM7BBKM5VO8C7777b1tY23KVGGa1er4+MjKyvrycIQiqVikQiLpfb09NDvtvW1paQkDCa9odktX2j0Thv3rysrCy1Wv3mm2+GhoaS5RkZGZ9++uko1wj/wS3AEQAArldeXj7+K92zZ4+3t/fChQvJl9nZ2Uwmc/fu3eMfibnr1683Njbu2LGDxWLt3r27oaHh+vXrCKGsrKxdu3b98MMPrg3vMQMJAIBhKykpYTKZWVlZaWlpGIa98cYbwcHBOI5nZ2eTFVJTUzEMi4iIwHFcLBZ/+eWXCKGoqCgMw27fvn3//n2RSITjOFk5Ojq6ubk5ICDg7bffRgitWbMmJSVlrLtgMBhkMllcXJyphMvlFhUVFRYW/vjjj4Prnz17NjQ0FMdxiURy+vRphJCtviOEjh07FhQU5OnpGR8fr9VqhxVYQ0ODWCwmH87M4/H8/f0bGhoQQiwWKzIy8sCBAyPrL7DO1YcgEwtsEIpzfAdISEjIzMwkCEIoFF68eNFoNH722WdsNttUgcPhnDlzRq1Wy2QyJpMpl8vJ9m/dukUQxLfffsvhcMiaOp0OITTOp4Dq6uoQQkqlknwplUpPnjxJEER0dPTatWuJX54CUigUHA6nrKxMqVTKZDIOh9PZ2Wmr73K5nMVinTx5sqenJywsLD8/31YMZOZgMpkhISHl5eVk4UcffSSRSEx1QkJC9u3bR/5dUlIiFotH3GUC/oMPAkcAADgBhmHLli1Tq9V6vd5U6Ovry2KxEhMTvby8qqurXRedFa2trQwGw8PDw6JcKpWeP3++trbWvLCqqkooFG7YsMHDw4Psztdff21616Lv1dXVAQEBL774Io/He+mlly5cuGArhoqKCrlc3tHRkZycHBsba/X0jtFoND26h8/n37lzh4CRXM4DCQCAMeft7d3T0+PqKH5hYGCAwWAMLhcIBPn5+du2bTMv7OzsFAgEppdCobCzs9NWywqF4ubNm+S9PTt37uzt7bVVUywW4zju6emZkJAQGBhIZh0+n9/X12eq09vbKxQKyb8ZDIbRaNRoNA73EgwBEgAAY4sgiPb2dj8/P1cH8gtsNtvW2fn169cLhULyugXJx8fH/BHKHR0d5hMoWeByuSEhIaaTDFVVVY7Eo9frWSwWQkgikbS2tpI5oLu7u729fdasWWQdrVZLo9GYTKYjDQJHQAIAYKz09fVpNBqpVKrVapcvX44QwnH80qVLOp2uvb3dVI1Go9FotKamJrVaPW6xiUQijUZj/lvbXGFhofkohxUrVnR1dZWVlalUKplM9vDhwxUrVthq+Zlnnmlubj506FB/f79arbZ1BNDY2JiRkaFWq5VKZVFRUXd393PPPYcQkkgkc+fOzcnJUavVmZmZoaGh8+bNIxdRKBTTp0+HCd2cyWVXHyYk2CAU5+AOkJaWRqfTyTtVEELTp0/v7e2dM2cOQig2Npasw+FweDyeu7u7RCKprq4mC9955x0mkxkUFJSYmIgQ2rx5M1keFRXFYDBiYmIIgli9enVycrITo7VKr9dPnTq1rq6OIIj9+/fjOM7j8YqLi00VDh48aH6f/pkzZ0JCQthstkQiqaqqIghi+/bttvp+5MiRoKAgBoOxaNGia9euKRQKgUAglUrNA5DL5UFBQUwmE8fxJUuW1NTUmN5qampasGABg8EIDw9vbm42laekpDi4ZWyB/+AWYG68X4DJAinOiTsAjuP19fWzZ892SmtWjTLanJycBw8e7Nu3z4khWaXT6WJjYyUSSWZm5ogb0ev1wcHBJ06cCA4OHnEj8B/cApwCGkOzZ8/GMKy7u9vB+nfv3g0MDMQwzOIyl0W5cychUCqV8+bNw3Gcy+WuWrXq9u3bdirn5eWRcwaYO3XqlCMrmhRbw7mMRqOrQ7AnPT29paXlypUrY70imUzG5/NTU1NH00hWVlZGRsZovv2BFS49/phwnLtBHj58iBBSKBSOLyKXyxFCAwMDDpaPnkKhSExMVCqVDx48WL9+/aJFi+zXl0qlQqGQ/Fun0x0/fryystKRFU2KreGsHeC1115DCPn5+f3zn/90SoNWjT5arVabk5PT0tLilHjGzpEjR/7xj3+Mvh34xrMATwQbc+7u7o5XtnWBazQXvurq6jw8PMiztIPx+XyZTEb+HR8f//zzzxsMhilTpjjSspubW2Rk5LCCmeBbw1lKS0tLS0vHdBVO4e7unpGR4eoohhYdHe3qEB5PcArIUYNHt6ekpGAYFhAQwOFw3N3dg4KCZsyYweFwPD09zf/zh4SEMBiMmTNnHj582FZTp0+fDg0NJYdEmq90cLlpEgJkdyx+RUXFk08+yWQyAwICduzYYbqRzr7+/n4vLy/y29+RCQkyMjJM454ev60BwOPP1YcgE4utDWJrdLufn195eblWqz1y5Ii7u/vt27cfPXqUmZn5m9/8hvj5pMd33303MDBw4MABd3f3f//734Ob6uzsZDKZBQUFAwMDt27dQj+f3LBVbpqEgLAxFl+j0Xh4eJSVlfX396empoaHhzvY/dTU1KSkJPt1pFKp+f6j0+lMbz0GW2Ny/Y+YXNFOBLDFLMARgEPsjG739vZ2d3dftmyZTqfz8/Oj0+nh4eE//fSTqYKvry+TydyyZYu/v//58+cHN1VZWSkUCt966y3yljjTgrbKB7MYi9/W1qZSqV5++WU2m/3CCy9YndtrsLa2tlOnTr333ntD1jRdA9ixY4fFW4/N1gCACuAagENMo9vJl3ZGwdjh5eXV29uLYZhFU3K5fNq0aYPr2yofko+PD5PJ/Pvf//7KK69UVFQ4cieiSqXasmXL0aNHuVyu4yvas2fPCMIjTditMbnGGU2uaMFEA0cADhnZ6HZzBEG0tbUFBAQMborL5ZqPszdfqdXyIeE4npubu2XLFi6XW1tbW1hYaL++SqWKj4/fv3//uN1jN5G3xjgdezvD5Ip2IhjB/vN4gwTgEAdHt1s1MDCg0Wjy8/N1Ol1ERMTgpp599tmbN2+Wlpb29fVVVFSYFrRVPiS1Wl1eXn7jxg2NRlNbW2v/N69SqUxISPjggw9G9u3/6NGj119/3fH6E3xrAEAtrk7JE4udDWIxup0giK1btyKE/Pz8bty4Qd5YEhIS8v333wuFQgzD0tPTBwYGVq5c6eXlxWAwwsLCLl26ZKupoqIif39/LpdL3j8eGRlJ1hxcbpqEYO/evbbG4ms0msWLF5OfL4ZhgYGBpqkIBisuLrbYJS5evEjYmJCAnDPAov7SpUsfm60xuf5HTK5oJwLYYhZgYPQvPB4jxXt6erZt2/bJJ5+4u7sbDIbs7Ozvv//+2LFjro7LNYa1NSbXDjC5op0IYItZgFNAj6Gvvvrq7t27vb29Wq22ubn5woULAoEAs+bevXuuDnbMDd4aTz31lKuDAmBCgATwGFq9erVQKJw1axaHw1m5cuXTTz9dUFBg9QDQ39/f1cGOucFbIz093dVBjROdTpebm5uUlETO4JSXl0eW19TU+Pv70+n0zZs3j93aRSKR6afG0qVLTeXNzc3h4eFMJjM8PPzmzZv2G2lqalq+fLnFfFMGgyEtLY3P57PZ7Dlz5hiNxqNHj1ZWVo5JNx5v43e2aTKADUJxTtwB3n333RE85ndYS9mPVq/XR0ZG1tfXEwQhlUpFIhGXy+3p6SHfNX/k7xix2r7RaJw3b15WVpZarX7zzTdDQ0PttHDo0KH09HQej2cx31RaWtqiRYtaWlqUSmVMTAw5GjEjI+PTTz+1HxL8B7cAm+MXYP+gOCfuALNmzRpBAhjWUvajzc7O/v3vf0/+LZVKP//8c19f361bt5IlrkoA33zzzZQpU/r7+wmC6OnpwTDs22+/td+On5+feQJQqVQcDufq1asW1dRqdUBAQGNjo52m4D+4BTgFBICjzp49GxoaiuO4RCI5ffo0QigqKgrDsNu3b9+/f18kEplukYqOjm5ubg4ICCBPgEREROA4LhaLTc9ZtLqgaam3334bOTYdky0Gg0Emk8XFxZlKuFxuUVFRYWEZObYAACAASURBVGGh1bHQg7tmZ26lwdM3Oa6hoUEsFpPP0uHxeP7+/g0NDcNq4erVq0ajccGCBRblLBYrMjLywIEDw2qN6lydgSYW2CAUZ2cHUCgUHA6nrKxMqVTKZDIOh9PZ2UkucuvWLYIgvv32Ww6HQ1bW6XQIIfK3PIfDOXPmjFqtlslkTCZTLpeb1mWxoPlSo4y2rq4OIaRUKsmXUqn05MmTBEFER0evXbuW+OURgK2uWZ1byda8WIORmYOcvK+8vJws/OijjyQSialOSEjIvn377HfT4gigtLSUxWLNnz+ffIhFcnKy0Wgk3yopKRGLxXaagv/gFuAIAACHVFVVCYXCDRs2eHh4JCYmenl5ff311w4u6+vry2KxyKWqq6vHMsz/p7W1lcFgeHh4WJRLpdLz58/X1taaF9rvmsXcSnbmxbJQUVEhl8s7OjqSk5NjY2N/+OGHwXWMRiOdTh9W14xGI4fDKSkp6erqOnfuXHFx8fHjx8m3+Hz+nTt3CLjR02GQAABwSGdnp0AgML0UCoWdnZ3DbcTb27unp8epcVk3MDDAYDAGlwsEgvz8/G3btpkXDqtrpnmxMAzbuXOnnVHxYrEYx3FPT8+EhITAwEAy6/D5fPMn0ff29gqFwmF1TSAQ6HQ6iUTCYrEkEsnixYuvXbtGvsVgMIxGo8Uj5IAdkAAAcIiPj4/5ZEQdHR0+Pj7DaoEgiPb2dj8/P2eHZgWbzbZ1dn79+vVCodB0NQINs2sjmxdLr9ezWCyEkEQiaW1tJXNAd3d3e3v7cB/PEBYWplKpTFcyDAYDh8Mh/9ZqtTQajclkDqtBKoMEAIBDVqxY0dXVVVZWplKpZDLZw4cPyUlhcRy/dOmSTqdrb283VabRaDQarampSa1WI4T6+vo0Go1UKtVqtcuXLyfrDF7QYqnREIlEGo3G/Le2ucLCQvMHKdvqmlUOzovV2NiYkZGhVquVSmVRUVF3d/dzzz2HEJJIJHPnzs3JyVGr1ZmZmaGhofPmzRtW17y9vclZQLq7u+vr6+vq6iIiIsi3FArF9OnTYYbUYXDZ1YcJCTYIxdnfAc6cORMSEsJmsyUSSVVVFVn4zjvvMJnMoKCgxMREhNDmzZvJ8qioKAaDERMTw+FweDyeu7u7RCIxn4bI6oKmpQgb0zE5GK1er586dWpdXR3x8wxOPB6vuLjYVOHgwYPmt2kO7pqtuZWIQdM3KRQKgUAglUrNA5DL5UFBQeTDG5YsWVJTU2N6q6mpacGCBQwGIzw8vLm5mSAIqy0QBJGSkhIQEIAQwnF88eLF7e3tZHlPT8+rr77KYrGmTZtmPsgxJSVlxFuMmmBmjF+AqUIobix2ABzH6+vrx2IWUvvR5uTkPHjwYN++fU5frwWdThcbGyuRSDIzM13VAkJIr9cHBwefOHHCzry28B/cApwCAmDMGY3G8V9penp6S0vLlStXxnpFMpmMz+enpqa6sAWEUFZWVkZGxrg90+LxAPnwF+AHAsU5fQfYuHHjF1984efnd+LEifnz5zuxZeRAtDqd7sMPP4yJiRGLxc5d9URTXl7+xBNPrFq1yn41+A9uATbHL8D+QXGTaweYXNFOBLDFLMApIAAAoChIAAAAQFGQAAAAgKLgjNgvwBASAB5v8I1nDhIAoK5du3aZ/gWAguAUEAAAUBQkAAAAoChIAAAAQFGQAAAAgKIgAQAAAEVBAgAAAIqCBAAAABQFCQAAACgKEgAAAFAUJAAAAKAoSAAAAEBRkAAAAICiIAEAAABFQQIAAACKggQAAAAUBQkAAAAoChIAAABQFCQAAACgKEgAAABAUZAAAACAoiABAAAARUECAAAAioIEAAAAFAUJAAAAKMrN1QEAMK7UavXly5fJv1taWhBCZ8+eJV8uWbKEzWa7LDIAxh1GEISrYwBg/Oj1eoFAoNVq3dzcyJ0fwzC9Xk+n0xUKhZsb/CQCFAKngAC1uLm5rVu37tGjR0qlUqVSqVQqpVL56NGjdevWwbc/oBpIAIByXn/9dRaLZV7CZrM3b97sqngAcBU4BQQohyAIoVCoUChMJXw+v6urC8MwF0YFwPiDIwBAORiGxcXF0el08qW7u/umTZvg2x9QECQAQEVxcXGmM/50On3Tpk2ujQcAl4BTQICiRCLRnTt3EEIBAQF37951dTgAuAAcAQCKio+PZzKZDAZjy5Ytro4FANeAIwBAUS0tLXPnzkUI3bhxIzAw0NXhAOACkAAAdc2ZMwfDsBs3brg6EABchPjZzp07XR0LAACAsbVz507T176bxRu7du1yUVQAjLeOjg6EkI+Pj6sDmUDIb4DJ8j0wuaKdCCy2FYx9B9QFX/2A4uAuIAAAoChIAAAAQFGQAAAAgKIgAQAARmLVqlUff/yxq6MYgk6ny83NbW1tzcvLw3Ecw7C8vDzyrZqaGn9/fzqdPqYTwYpEIuxnS5cuJQubm5vDw8OZTGZ4ePjNmzeHbKSpqWn58uWnTp0yLzQYDGlpaXw+n81mz5kzx2g0Hj16tLKycljhQQIAAIxEZWXlW2+95azWdu7cee/ePWe1RjIYDFFRUc8995xYLE5NTc3NzRWJRO+///6DBw8QQkuXLq2rq4uLizt48KBz12tuxYoVpnsua2pqEEIEQURFRUVERDx8+DAsLCw6Otp+C4cPH/78888bGhosyjMzM8+fP3/16tXOzk6JRGI0GteuXVtTU1NSUuJ4eJAAAACuV15e7vQ29+zZ4+3tvXDhQlNJdnY2k8ncvXu309fluOvXrzc2Nu7YsYPFYu3evbuhoeH69et26sfExHzwwQcWT7Do6+srKCgoKCgQi8UeHh6HDh0iJzfMysratWvXDz/84GAwkAAAAMNWUlLCZDKzsrIQQmlpaRiGvfHGG8HBwTiOZ2dnI4RSU1MxDIuIiMBxXCwWf/nll+SCUVFRGIbdvn37/v37IpEIx3GEUHR0dHNzc0BAwNtvv40QWrNmTUpKyigjNBgMMpksLi7OvJDL5RYVFRUWFv74448W9c+ePRsaGorjuEQiOX36NFlotWsIoWPHjgUFBXl6esbHx2u12mEF1tDQIBaLyadP83g8f3//wb/uh3T16lWj0bhgwQKLchaLFRkZeeDAAQfbgQQAABi2+Pj41157jfx77969QqEwNjb2hx9++Pjjj/fu3YsQysvL43A427dvVygU6enpcXFx5LC7v/zlL+RSU6dO/dvf/kb+XVZWhhBqa2srKChACJ08eXL//v2jjPDatWv379+fN2+eRfnLL7/8yiuvpKenmxd2d3dHRkZu375dLpe/+eabv/3tb7u6umx1raOj47XXXsvLy2ttbf3uu+/+9Kc/2Qnj8uXLHh4eLBZr3rx5ZN97eno4HI6pwq9+9auenp7h9q69vR0hFBYW5uHhwePxtm7daprU56mnnjpx4oSD7UACAAA4B4Zhy5YtU6vVer2eLPH19WWxWImJiV5eXtXV1eMZTGtrK4PB8PDwGPyWVCo9f/58bW2tqaSqqkooFG7YsMHDw4OM9uuvvzZfxLxr1dXVAQEBL774Io/He+mlly5cuGAnjIqKCrlc3tHRkZycTCYSiwpGo9H0bCLHGY1GDodTUlLS1dV17ty54uLi48ePk2/x+fw7d+44OMkbJAAAwJjz9vYewe/c0RgYGGAwGFbfEggE+fn527ZtM5V0dnYKBALTS6FQ2NnZaatlhUJx8+ZN8saenTt39vb22glDLBbjOO7p6ZmQkBAYGFhbW8vn8/v6+kwVent7hULhMDr2cxd0Op1EImGxWBKJZPHixdeuXSPfYjAYRqNRo9E40g4kAADA2CIIor293c/PbzxXymaz7ZydX79+vVAoNF2Z8PHxMX9GdEdHh51pQrhcbkhIiOnenqqqKgdD0uv15Pd1a2srmQO6u7vb29tnzZrlYAsmYWFhKpXKdCXDYDCYTitptVoajcZkMh1pBxIAAGCs9PX1aTQaqVSq1WqXL19OFuI4funSJZ1OR57IRgjRaDQajdbU1KRWq521apFIpNFozH9rWygsLDSNY1ixYkVXV1dZWZlKpZLJZA8fPlyxYoWtBZ955pnm5uZDhw719/er1Wo7RwCNjY0ZGRlqtVqpVBYVFXV3dz/33HMSiWTu3Lk5OTlqtTozMzM0NHTwhYoheXt7R0ZGpqWldXd319fX19XVRUREkG8pFIrp06c7+oxr8+mgzacJBQBQkIPfA2lpaXQ6nc1m7927d/v27Qih6dOn9/b2zpkzByEUGxtLEASHw+HxeO7u7hKJpLq62rTsO++8w2Qyg4KCEhMTEUKbN28mCCIqKorBYMTExBAEsXr16uTk5FFGq9frp06dWldXR77cv38/juM8Hq+4uNhU5+DBgwkJCeTfZ86cCQkJYbPZEomkqqqKLLTVtSNHjgQFBTEYjEWLFl27dk2hUAgEAqlUahGDXC4PCgpiMpk4ji9ZsqSmpoYsb2pqWrBgAYPBCA8Pb25uJgttNZKSkhIQEIAQwnF88eLF7e3tZHlPT8+rr77KYrGmTZtWUFBgXt/O1rPYYv//A2FgYlUAgBO/B3Acr6+vnz179uibssV+tDk5OQ8ePNi3b9/YBUDS6XSxsbESiSQzM9O1jej1+uDg4BMnTgQHB1utYLHFRnUKaNu2bXQ6nbwX2KrZs2djGNbd3T3Kdh5vFt0fzQh7q0PGEUJ3794NDAzEMMzOpSGlUjlv3jwcx7lc7qpVq27fvj3k6k6fPh0REUH+yuPxeCEhIX/+85+tdmqwUe4bYrH46aefHjLCsWOaWgDDMA8Pj9WrV//rX/8aQTvmHXTuR28eIY1G4/F4y5cvP3LkyMjaHxmj0Tieq7OQnp7e0tJy5cqVsV6RTCbj8/mpqakubyQrKysjI8PWt/9go3oewL59++xf2b98+TKXyx19O483i+4PdzYPk8OHD3/33XdWB5VMmzbt8uXLvr6+dhbXarVLliy5dOmSXq9/8803X3vttbq6Ojv1P/vss8TExD179vzpT3/y9/d/8ODB1atXTYuM6b5RW1sbFhb217/+taWlxVWP801NTaXT6e+///79+/f//e9/b926dc2aNU1NTeSATMeZd9C5H70pwo6ODpVK1dzcnJeXFxMT09jYOA5DYTdu3Njf3//888+fOHFi/vz5Y706q6ZMmfKXv/zlww8/9Pb2FovFY7eipKSkidBIeXn5smXLVq1a5fgiIzkCqKura2xsdLy+u7v7CNbiKsPtnWuZR2t1yLjJkBeF+Hy+TCbz8PDgcrnx8fFXr141GAy2Kg8MDKSmpm7fvn3btm2BgYF0Ot3Hx2fNmjU5OTnDin9k+8bhw4f37NkTEBDwxRdfjGDxYRlyf6DRaIGBge++++7t27cHjy8dU6bY7H/0CCEPD48FCxYcOnTov//7vz/44IM7d+6MdWylpaUEQdy7d89V3/4kd3f3jIyMMf32nziio6OH9e2PHE8AFRUVTz75JJPJDAgI2LFjx+D7lqwOpCaFhIQwGIyZM2cePnyYLPn973/P5XJZLNbGjRsdP0i0WIq80k1eQP/qq6+8vLzI+8wsRmknJSVhGFZZWbl27dqMjAyra7faOwdHe9sa8m5rg9jZUMiBEfaOfBbmTp8+HRoaymQyQ0JCHNzOCKH+/n4vL68pU6aQLwcPzb98+fLDhw9jY2Mdac25+4bBYGhsbAwKCoqMjDRPAK7dH8ihT6af/1a7PPqP3lZsjtu+fbvRaBy8dkBRjlxP12g0Hh4eZWVl/f39qamp4eHhprc2bdqUmZmpUCg4HE5ZWZlSqZTJZBwOp7OzkyCIhw8fIoS+++67gYGBAwcOuLu7//vf/yYIIikpSS6X37p1y93d/caNG6Z2bF25Jlks9eOPP9JotO+++458Nzk5+fvvv5fL5SwW6+TJkz09PWFhYfn5+QRBCIXC0tLS3t7e3Nzcwe1Y7Z3VdmzhcDhnzpxRq9UymYzJZMrlclsbxFa5efcTEhJMfwuFwosXLxqNxs8++4zNZtv/LEh+fn6VlZXk352dnUwms6CgYGBg4NatWwihgYEB+xuZlJqampSUZKdCcXExQkij0dipM0b7RlVVFfny3LlzCCHTbR7jvz9IpVKhUGgwGG7fvh0RETFv3jyDwUDY+JRtbQfzDtr/6ImhPn3zj55ERmjxuQiFQjv/1ybX3YCTK9qJwGKLOXQE0NbWplKpXn75ZTab/cILLww+zrU/kNrX15fJZG7ZssXf3//8+fMIofz8fB8fn5kzZ/J4PJVK5WCuslgqMDDwpZdeys/PRwip1ep79+7NnTvX1ihtkUjk6emZlpY2uB2rvRvWaG80aMi7rQ0y5Ihzq8yHoQ/5WZirrKwUCoVvvfUWeSOaY5sZtbW1nTp16r333rNThxg00Hz79u1MJnPKlCnkRjZx+r5x+PDhyMhIhNB//Md/CASC0tJSstwl+0NnZ+eUKVMkEgmNRjt69CiNRrPV5dF/9MiB/4mO0Gg0ZJwAOLQf+Pj4MJnMv//97wMDAxUVFYPv63JwILWXl1dvb69KpVq/fj2fz6fT6XbGW1uwutTWrVvLysp6enoOHTq0ceNG5MAo7cHtWO3dsEZ7myOHvNvaIMMacW7VkJ+FOblcPm3atGG1r1KptmzZcvToUftXaEUiEUKotbXVVPI///M/W7duXbhwITlhlolz9w2tVnvs2LGFCxdiGObm5qZQKMrLy3U6Hfnu+O8P5O/rvr6+U6dO/frXv7bT5dF/9LZiGxalUvnTTz/ZPyf+3nvvYZPEe++9N4minQgsftg5lABwHM/Nzd2yZQuXy62trS0sLBy8Xw45kJogiLa2toCAgM8//7ypqen69esDAwOOT4Jhdally5YFBwd/8sknp06dWrNmDXJglPbgdqz2bmSjvYmfh7zb2iDDGnFu1ZCfhTkul2u+uiGpVKr4+Pj9+/cPeRvZ008/zefzTTd92uHcfaOysvK3v/2t6XM5evRod3e36d7HCbI/WO3y6D96W7ENy2effebm5vb888/bqTOJTqrAKaDh2rlzp/ln7VACUKvV5eXl5OnR2trawb877A+kHhgY0Gg0+fn5Op0uIiLi0aNHDAYDx/Hm5mYHZyxCCNlaauvWrXv27AkPDyePaoccpT24Hau9c3y0N8liyLutDTKsEedWDflZmHv22Wdv3rxZWlra19dXUVFhv2WlUpmQkPDBBx84chMxi8WSSqV5eXl79+69e/euVqu9deuW1cdQOHffKC8v/+Mf/2h6+corr8ycOdP8UvBE2B+sdnn0H72t2OwjCOLRo0cIofb2dplMtmPHjqysLPs3BAMKMc8Mdi4CL168mKyPYVhgYCA5sPuPf/yju7s7m83+6KOPrA6kHhgYWLlypZeXF4PBCAsLu3TpEkEQd+7cmTVrFofDiYmJmTFjxowZM1JTU03t2Epcg5ciL7g9evRoxowZDx48MNW0GKVN3l3r6+tra+1qtdpq7yzasZNUrQ55t7pBrJabb8YhR9jb+iwIG0PGi4qK/P39uVwuOXt7ZGSkrV6Q13XNXbx4kXzL1tD8c+fOrVy58le/+hWDwZg2bdqaNWvKy8vJt8Zi34iMjMQwLDg4+N69e2Qjr7zyCnnjzVNPPTXO+8Of//xncqrhmTNnnjt3zmLLWO2y1ULThnJzcxtycgVbn77Vj76goEAoFNLpdDIXcjic8PBw8u5MOybXb+rJFe1EYLHFHEoA3d3dmzZt0mq1BEHo9fp33333lVdeGdswx9Hoe8fhcBobG8cmOkuP92cxEUzkLTwOsU2ur9TJFe1EMJK7gL766qu7d+/29vZqtdrm5uYLFy489dRTIzjaGNK9e/esXrhw+tOizTnSuyEDG7ch76P/LFyykSeRcdvbR2AixwYmI4cSwOrVq4VCIXmkvHLlyqefftrigWrO4u/vbzVr+fv7j8XqSI70zk5gpiHv33zzzdgFOaxo7XPJRp5Exm1vH4GJHNskpdPpcnNzk5KSyHmT8vLyyPKamhp/f386nb558+YxDcDq/F0GgyEtLY3P57PZ7Dlz5hiNxqNHj454phA7HJq3hMPhmAZqPn5G2bvS0lLTrejj4PH+LCaCibyFJ3Jsg+3cuZMc4TFuCw6XwWCIiorKyMhYuHDhrFmz9u3b9/7777/++us8Hm/p0qV1dXW7du0afG3MiWzN35WZmXn+/PmrV6/y+fzExESj0bh27drMzEy5XB4fH+/EAGA8CABgTJSXl4/zgsO1Z88eb2/vhQsXki+zs7OZTOY4zJRnYnUSp76+voKCgoKCArFY7OHhcejQIfJOh6ysrF27dlm9127EIAEAABxldTqjqKgoDMNu3759//59kUhEjjmPjo5ubm4OCAh4++23rc6XZXUpiwUHT0LlRAaDQSaTxcXFmUq4XG5RUVFhYaHVIdaD+25nyiYHZxKz6urVq0ajccGCBRblLBYrMjLywIEDw2ptCLauDgMAKMjO94Cd6YwQQrdu3SII4ttvv+VwOARBkMOz29rayAqD58uyutTgBUcc7ZDIqcuVSiX5UiqVnjx5kiCI6OjotWvXEgTR1tZmel6Yrb5bnbJpWDOJEYMmcSotLWWxWPPnzycfzpGcnGw0Gsm3SkpKxGLxyPpLGsldQAAAMLLpjEws5ssaszAd1draymAwyMEc5qRS6fnz52tra80L7fcd++WUTcOdScyC0WjkcDglJSVdXV3nzp0rLi4+fvw4+Rafz79z5w4xaCauEYMEAABwiFOmM0I/z5flvLhGaGBggMFgDC4XCAT5+fnbtm0zLxxW30c8k5gpAJ1OJ5FIWCyWRCJZvHjxtWvXyLcYDIbRaHR8AoUhQQIAADjEKdMZET/Pl+XU0EaCzWbbOju/fv16oVBoerYHGmbfRzaTmElYWJhKpTJdhzAYDBwOh/xbq9XSaDQmkzmsBu2ABAAAcIid6YxwHL906ZJOp2tvbydLaDQajUZrampSq9VkicV8WVaXsrrgGBGJRBqNpq+vz+q7hYWF5s9nHtZUTsOdScyCt7d3ZGRkWlpad3d3fX19XV1dREQE+ZZCoSAffDSsBu2ABAAAcAifzz927NjevXt9fHxkMtnx48e9vLzIt1JSUv7whz/MnTv35MmT/f398fHxNBpt7dq1a9as+d3vfkfWWb169RNPPFFSUvK3v/3tiSeesLoUQsh8wRdffHHr1q1j1J0FCxZMnTqVfKbm//7v/+7YsWPTpk2ffvop+a6vry/5dDY7fU9LS+vs7Hzttdd++umnF198ESH0+uuvI4T8/f0///zz9957z8vLa/ny5T/++GN3d7e3t3dBQYFFDKmpqdOmTWtvb1+3bt2SJUvu379Pln/yyScEQUybNm3dunUffvjhb37zG7L8+++/f+mll5y4ETDT9YRdu3aZ/gUAUNMYfQ/gOF5fXz+CBxjYN8poc3JyHjx4sG/fPieGZJVOp4uNjZVIJJmZmSNuRK/XBwcHnzhxwpH5em2x2GJwBAAAGA/jNl+W49LT01taWq5cuTLWK5LJZHw+PzU1dTSNZGVlZWRkjObbfzBIAACAsTXO82U5bsqUKX/5y1+++uor82fbjYWkpKTCwkKLEb/DUl5evmzZMqdPTOTQXEAAADBi4zxf1rC4u7tnZGS4OoqhRUdHj0WzcAQAAAAUBQkAAAAoChIAAABQlfkkQa6OBQAAwNgynwwOc+K8QgBMLjD2BVAcnAICAACKggQAAAAUBQkAAAAoChIAAABQFCQAAACgKEgAAABAUZAAAACAoiABAAAARUECAAAAioIEAAAAFAUJAAAAKAoSAAAAUBQkAAAAoChIAAAAQFGQAAAAgKIgAQAAAEVBAgAAAIqCBAAAABQFCQAAACgKEgAAAFAUJAAAAKAoSAAAAEBRkAAAAICiIAEAAABFubk6AADGlVqtvnz5Mvl3S0sLQujs2bPkyyVLlrDZbJdFBsC4wwiCcHUMAIwfvV4vEAi0Wq2bmxu582MYptfr6XS6QqFwc4OfRIBC4BQQoBY3N7d169Y9evRIqVSqVCqVSqVUKh89erRu3Tr49gdUAwkAUM7rr7/OYrHMS9hs9ubNm10VDwCuAqeAAOUQBCEUChUKhamEz+d3dXVhGObCqAAYf3AEACgHw7C4uDg6nU6+dHd337RpE3z7AwqCBACoKC4uznTGn06nb9q0ybXxAOAScAoIUJRIJLpz5w5CKCAg4O7du64OBwAXgCMAQFHx8fFMJpPBYGzZssXVsQDgGnAEACiqpaVl7ty5CKEbN24EBga6OhwAXAASAKCuOXPmYBh248YNVwcCgGtAAphY4F4U8HiDL5wJBYY+TjjwP2Q0MGwYv2k6OjoQQj4+PmMZkT3DivYxAL9vJhpq7X8TH9W+EZxucm3AyRXt6FGtvxMf3AUEAAAUBQkAAAAoChIAAABQFCQAQFGrVq36+OOPXR3FEHQ6XW5ublJSEo7jGIbl5eWR5TU1Nf7+/nQ6fawnMW1qalq+fPmpU6fMCw0GQ1paGp/PZ7PZc+bMMRqNR48eraysHNNIwJggwEQCn8gouWoDvvvuu21tbcNdyn60er0+MjKyvr6eIAipVCoSibhcbk9PD/luW1tbQkLCyKJ10KFDh9LT03k8XmVlpXl5WlraokWLWlpalEplTEyMTqcjCCIjI+PTTz+13yDs3hMNHAEA4ATl5eVOb3PPnj3e3t4LFy4kX2ZnZzOZzN27dzt9RbbExMR88MEHFs9O6OvrKygoKCgoEIvFHh4ehw4dIqfVy8rK2rVr1w8//DBu4YHRgwQAqKikpITJZGZlZaWlpWEY9sYbbwQHB+M4np2dTVZITU3FMCwiIgLHcbFY/OWXXyKEoqKiMAy7ffv2/fv3RSIRjuNk5ejo6Obm5oCAgLfffhshtGbNmpSUlFFGaDAYZDJZXFycqYTL5RYVIa/bAQAAIABJREFUFRUWFv7444+D6589ezY0NBTHcYlEcvr0aYSQra4hhI4dOxYUFOTp6RkfH6/VaocV2NWrV41G44IFCyzKWSxWZGTkgQMHhtUacDFXH4KAX4BPZJQc34AJCQmZmZkEQQiFwosXLxqNxs8++4zNZpsqcDicM2fOqNVqmUzGZDLlcjnZ/q1btwiC+PbbbzkcDllTp9MhhJx7Cqiurg4hpFQqyZdSqfTkyZMEQURHR69du5b45SkghULB4XDKysqUSqVMJuNwOJ2dnba6JpfLWSzWyZMne3p6wsLC8vPz7Qfp5+dnfgqotLSUxWLNnz8fx3Eul5ucnGw0Gsm3SkpKxGLxyPoLXAKOAABACCEMw5YtW6ZWq/V6vanQ19eXxWIlJiZ6eXlVV1ePZzytra0MBsPDw8OiXCqVnj9/vra21rywqqpKKBRu2LDBw8ODjPbrr782vWvRterq6oCAgBdffJHH47300ksXLlwYVmBGo5HD4ZSUlHR1dZ07d664uPj48ePkW3w+/86dOwQM9Zo8IAEAMDRvb++enp7xXOPAwACDwRhcLhAI8vPzt23bZl7Y2dkpEAhML4VCYWdnp62WFQrFzZs3MQzDMGznzp29vb3DCkwgEOh0OolEwmKxJBLJ4sWLr127Rr7FYDCMRqNGoxlWg8CFIAEAMASCINrb2/38/MZzpWw229bZ+fXr1wuFQvKyBMnHx8f8EccdHR12JjjicrkhISGmkwBVVVXDCiwsLEylUpmuQxgMBg6HQ/6t1WppNBqTyRxWg8CFIAEAYFNfX59Go5FKpVqtdvny5QghHMcvXbqk0+na29tN1Wg0Go1Ga2pqUqvVzlq1SCTSaDR9fX1W3y0sLDQfxLBixYqurq6ysjKVSiWTyR4+fLhixQpbLT/zzDPNzc2HDh3q7+9Xq9XDPQLw9vaOjIxMS0vr7u6ur6+vq6uLiIgg31IoFNOnT4cZ3yYTl119ANbAJzJKDm7AtLQ0Op3OZrPJ/wXTp0/v7e2dM2cOQig2Npasw+FweDyeu7u7RCKprq4mC9955x0mkxkUFJSYmIgQ2rx5M1keFRXFYDBiYmIIgli9enVycvIoo9Xr9VOnTq2rqyMIYv/+/TiO83i84uJiU4WDBw+ajwM4c+ZMSEgIm82WSCRVVVUEQWzfvt1W144cORIUFMRgMBYtWnTt2jWFQiEQCKRSqUUMKSkpAQEBCCEcxxcvXtze3k6W9/T0vPrqqywWa9q0aQUFBeb17Xccdu+JBibnm1hgusRRcuIGxHG8vr5+9uzZTmnNKvvR5uTkPHjwYN++fWMXAEmn08XGxkokkszMzBE3otfrg4ODT5w4ERwcbKsO7N4TDZwCmnzu3r0bGBiIYdg4X21LTk5OT0+3UyEvL4+csQDDMBqNxuPxli9ffuTIkXGL0OmMRqML156ent7S0nLlypWxXpFMJuPz+ampqaNpJCsrKyMjw863P5iIXHr8ASw5+InI5XKE0MDAwFjHY1JfX8/lctPS0uxXk0qlQqGQIAilUnn16tWYmBiEUFZW1rjESBDOO8nw2muvIYT8/Pz++c9/OqVBq4aMVqvV5uTktLS0jF0MTnHkyJF//OMfQ1aDL5yJBo7IJhYHj5E7Ozt9fHwGBgbG544LvV6flJSkVCoDAgJyc3Pt1CwoKHj//ffJJ22Rtm/fnpeX9+OPP06fPn3sI51kJxkmV7SjR7X+TnxwCmgyOX36dGhoKJPJDAkJMRVaDOu3NQGAVquNioricDh8Pv/gwYODF7Sz3vz8/MTERIu7Oxyc8GD79u1Go5GcnMDBaAeHOqxoAQCOcunxB7Bk5xPp7OxkMpkFBQUDAwO3bt1CCA0MDFgd1m91AoAjR46sXLlSrVbfuHHjww8/dHw+gJaWlh07dhAEERsb6/gpIHNCoZCcdMHBaC1CtbXgcDfgBDS5oh09qvV34oMjgEmjsrJSKBS+9dZbTCbTNA2ZnWH9FhMA4Dj+zTffnDlz5sknn/zjH//o+HwA2dnZ9q/9Dkmj0dBoNMejtQjV/oIAgBFzc3UAwFFyuXzatGkWhaZh/eRLO8N/Xnjhha1bt/7hD39wc3M7ePCggwuWlZX913/91xNPPDHisJVK5U8//SQWix2P1iLUFStWON5NhNDkGog0uaIFjxk4Apg0uFyu+XB/U6GDw/oxDMvIyLh3797vfve7t99+28EFv/jiiw0bNpB3dpaVle3duzc8PHxYYX/22Wdubm7PP/+849FahDqsbqJJdZJhckU7esPac8A4gAQwaTz77LM3b94sLS3t6+urqKggCx0f1v/JJ59UVVUZDIaFCxdiGObggubzAJPXAMhpiu0gCOLRo0cIofb2dplMtmPHjqysLF9fX8ejtQh1WN0EAAyD634NACvsfyJFRUX+/v5cLpe8Sz0yMpIYNKzf1gQAJ0+enDp1qpubW1BQEDlVgMWCQ8ZmcRF48IQHBQUFQqGQTqeTZ/w5HE54eHhpaal5HUeiHRyq49FOrl16ckU7elTr78QHt+VOLHCj9ChNrg04uaIdPar1d+KDU0AAIYTu3buHWXPv3j1XhwYAGCuQAABCCPn7+1s9QvT393d1aJSm0+lyc3OTkpLISZby8vLI8pqaGn9/fzqdvnnz5jENoKmpafny5adOnTIvNBgMaWlpfD6fzWbPmTPHaDQePXq0srJyTCMBYwESAADW7dy5cwQHQCNbyiqDwRAVFfXcc89JpdLc3FyRSPT+++8/ePAAIbR06dK6urq4uDjTSOmxcPjw4c8//7yhocGiPDMz8/z581evXu3s7JRIJEajce3atTU1NSUlJWMXDBgLkAAAsK68vHzclrJqz5493t7eCxcuJF9mZ2czmczdu3c7q/0hxcTEfPDBBywWy7ywr6+voKCgoKBALBZ7eHgcOnTIzc0NIZSVlbVr164ffvhh3MIDowcJAFDI2bNnQ0NDcRyXSCTk9ERRUVEYht2+ffv+/fsikcg0xDo6Orq5uTkgIIC8FhIREYHjuFgsNj2I0eqCpqXI4QsOTpdklcFgkMlkcXFxphIul1tUVFRYWGh6HKP9rtmaFQqNbmKlq1evGo3GBQsWWJSzWKzIyMgDBw4MqzXgYuN3wxFwAHwio2RnAyoUCg6HU1ZWplQqZTIZh8Pp7OwkF7l16xZBEN9++y2HwyEr63Q6hFBbWxtBEBwO58yZM2q1WiaTMZlMuVxuWpfFguZLjTJacryFUqkkX0ql0pMnTxIEER0dvXbtWoIg2traTE8Es9U1q7NCOT6xEsnPz898OEhpaSmLxZo/fz6O41wuNzk52Wg0km+VlJSIxeKR9Re4BBwBAKqoqqoSCoUbNmzw8PBITEz08vL6+uuvHVzW19eXxWKRS1VXV49lmP9Pa2srg8Hw8PCwKJdKpefPn6+trTUvtN81i1mhRjmxktFo5HA4JSUlXV1d586dKy4uPn78OPkWn8+/c+cOATd6Th6QAABVdHZ2CgQC00uhUNjZ2TncRry9vXt6epwal3UDAwMMBmNwuUAgyM/P37Ztm3nhsLpmmlgJw7CdO3cOd1i1QCDQ6XQSiYTFYkkkksWLF1+7do18i8FgGI3GcX5QHRgNSACAKnx8fMwnU+ro6PDx8RlWCwRBtLe3+/n5OTs0K9hstq2z8+vXrxcKhaarEWiYXRvWxEqDhYWFqVQq03UIg8HA4XDIv7VaLY1GG5+HFAGngAQAqGLFihVdXV1lZWUqlUomkz18+JCcVRTH8UuXLul0uvb2dlNlGo1Go9GamprUajVCqK+vT6PRSKVSrVa7fPlyss7gBS2WGg2RSKTRaPr6+qy+W1hY+PHHHw/ZNatGObGSt7d3ZGRkWlpad3d3fX19XV1dREQE+ZZCoZg+fTrMbzqZuOzqA7AGPpFRsr8Bz5w5ExISwmazJRKJaZahd955h8lkBgUFJSYmIoQ2b95MlkdFRTEYjJiYGA6Hw+Px3N3dJRJJdXW1qTWrC5qWIqxNl+R4tHq9furUqXV1dQRB7N+/H8dxHo9XXFxsqnDw4EHTRWCrXbM1KxQxaGIlhUIhEAikUqlFDCkpKQEBAQghHMcXL17c3t5Olvf09Lz66qssFmvatGkFBQXm9UfcX+ASMDXHxAKTpYzSWGxAHMfr6+tnz57t3GbRUNHm5OQ8ePBg3759Tl+vBZ1OFxsbK5FIMjMzR9yIXq8PDg4+ceJEcHCwrTqwe080cAoIgKEZjcbxX2l6enpLS8uVK1fGekUymYzP56empo6mkaysrIyMDDvf/mACgoQ8scBPpFFy+gbcuHHjF1984efnd+LEifnz5zuxZeRAtDqd7sMPP4yJiSEfqTZhlZeXP/HEE6tWrbJfDXbviQY+j4kF/oeM0uTagJMr2tGjWn8nPjgFBAAAFAUJAAAAKAoSAAAAUBSckptYYBANeLzBF86EAgkAUNeuXbtM/wJAQXAKCAAAKAoSAAAAUBQkAAAAoChIAAAAQFGQAAAAgKIgAQAAAEVBAgAAAIqCBAAAABQFCQAAACgKEgAAAFAUJAAAAKAoSAAAAEBRkAAAAICiIAEAAABFQQIAAACKggQAAAAUBQkAAAAoChIAAABQFCQAAACgKEgAAABAUZAAAACAoiABAAAARUECAAAAioIEAAAAFOXm6gAAGFdqtfry5cvk3y0tLQihs2fPki+XLFnCZrNdFhkA4w4jCMLVMQAwfvR6vUAg0Gq1bm5u5M6PYZher6fT6QqFwu3/a+/uY5q6/seBn9tgn+5lS2tLQUBa/yBDRkp8iGMx2YaQzT2FGAQrGw51Ycs0Tl3AQZ3G53yNmNgKjWNgxmCQmZlAHApGccOPFdzMEj8SRDGKtZSKutIV7NP9/nG/v/vrSlta2tJ29/36w3jPPefe97ktPb2nPe/GwVsiwCAwBQSYJS4ubu3atS9evDCZTBMTExMTEyaT6cWLF2vXroVXf8A0MAAAxvnkk094PJ5rCZ/PLysri1Q8AEQKTAEBxiFJUiKRGI1GukQkEo2NjWEYFsGoAJh7cAcAGAfDsNLSUjabTW3Omzdvw4YN8OoPGAgGAMBEpaWl9Iw/m83esGFDZOMBICJgCggwlFQqffDgAUIoNTX14cOHkQ4HgAiAOwDAUBs3buRyuRwO59NPP410LABEBtwBAIYaHh5+9dVXEUK3bt1atGhRpMMBIAJgAADMlZmZiWHYrVu3Ih0IAJEBA0B0ge+igH83eMGJKrD0MerAX0gwMCyA9zSjo6MIocTExHBG5EtA0f4LwPubaMOs51/0Y9orQsjF1gWMrWiDx7T+Rj/4FhAAADAUDAAAAMBQMAAAAABDwQAAGGr16tUnT56MdBQzsNlsR44c2bp1K0EQGIbV1NRQ5b29vSkpKWw2O9xJTAcGBnJzc8+fP+9a6HA4KisrRSIRn8/PzMx0Op1nzpzp7OwMayQgLEgQTeARCVKkLuA333wzMjISaCvf0drt9oKCgr6+PpIkVSqVVCoVCATj4+PU3pGRkU2bNs0uWj+1tLTs2rVLKBR2dna6lldWVq5YsWJ4eNhkMikUCpvNRpJkVVXVd9995/uA8PSONnAHAEAItLW1hfyYhw4dSkhIWL58ObW5b98+Lpe7f//+kJ/IG4VCcfjwYbffTjCbzWq1Wq1Wy2Sy+Pj4lpYWKq2eUqncu3fv7du35yw8EDwYAAATNTQ0cLlcpVJZWVmJYdjnn3+ekZFBEMS+ffuoCjt27MAwLD8/nyAImUz2008/IYSKioowDLt79+7jx4+lUilBEFTl4uLiwcHB1NTULVu2IIQ++OCD7du3Bxmhw+HQaDSlpaV0iUAgqKurq62tvXfv3vT6Fy9ezM7OJghCLpdfuHABIeStawihn3/+OT09/eWXX964caPVag0osP7+fqfTuWzZMrdyHo9XUFBw6tSpgI4GIizStyDgH+ARCZL/F3DTpk3V1dUkSUokkt9++83pdJ4+fZrP59MVcBzv6uqyWCwajYbL5er1eur4Q0NDJEnevHkTx3Gqps1mQwiFdgpIq9UihEwmE7WpUqk6OjpIkiwuLi4sLCT/OQVkNBpxHG9ubjaZTBqNBsdxg8HgrWt6vZ7H43V0dIyPjy9duvTEiRO+g0xOTnadAmpqauLxeEuWLCEIQiAQbNu2zel0UrsaGhpkMtns+gsiAu4AAEAIIQzD3njjDYvFYrfb6cKkpCQej1deXj5//vyenp65jOf+/fscDic+Pt6tXKVSXbly5dq1a66F3d3dEolk/fr18fHxVLSXLl2i97p1raenJzU19f333xcKhR9++OGvv/4aUGBOpxPH8YaGhrGxscuXL9fX1589e5baJRKJHjx4QMJSr9gBAwAAM0tISBgfH5/LM05OTnI4nOnlYrH4xIkTO3fudC00GAxisZjelEgkBoPB25GNRuOdO3cwDMMwbM+ePc+fPw8oMLFYbLPZ5HI5j8eTy+U5OTk3btygdnE4HKfTOTU1FdABQQTBAADADEiS1Ol0ycnJc3lSPp/vbXZ+3bp1EomE+liCkpiY6PoTx6Ojoz4SHAkEgqysLHoSoLu7O6DAli5dOjExQX8O4XA4cByn/m+1WlksFpfLDeiAIIJgAADAK7PZPDU1pVKprFZrbm4uQoggiKtXr9psNp1OR1djsVgsFmtgYMBisYTq1FKpdGpqymw2e9xbW1vruoghLy9vbGysubl5YmJCo9E8e/YsLy/P25HffPPNwcHBlpaWv//+22KxBHoHkJCQUFBQUFlZ+eTJk76+Pq1Wm5+fT+0yGo1paWmQ8S2WROzTB+AJPCJB8vMCVlZWstlsPp9P/RWkpaU9f/48MzMTIVRSUkLVwXFcKBTOmzdPLpf39PRQhbt37+Zyuenp6eXl5QihsrIyqryoqIjD4SgUCpIk33vvvW3btgUZrd1uX7BggVarJUny+PHjBEEIhcL6+nq6QmNjo+s6gK6urqysLD6fL5fLu7u7SZKsqKjw1rXW1tb09HQOh7NixYobN24YjUaxWKxSqdxi2L59e2pqKkKIIIicnBydTkeVj4+Pr1mzhsfjLVy4UK1Wu9b33XF4ekcbSM4XXSBdYpBCeAEJgujr61u8eHFIjuaR72gPHjz49OnTY8eOhS8Ais1mKykpkcvl1dXVsz6I3W7PyMhob2/PyMjwVgee3tEGpoBiz8OHDxctWoRh2Nx82iaVSrH/Z+XKlT5q1tTUUBkLMAxjsVhCoTA3N7e1tXUOggwTp9MZwbPv2rVreHj4+vXr4T6RRqMRiUQ7duwI5iBKpbKqqsrHqz+IRhG9/wDu/HxE9Ho9QmhycjLc8ZAkGVC+AZVKJZFISJI0mUz9/f0KhQIhpFQqwxadu1A9pT/66COEUHJy8u+//x6SA3o0Y7RWq/XgwYPDw8PhiyEkWltbf/nllxmrwQtOtIE7suji5z2ywWBITEycnJycg29cbN68ub6+3s/KarX6wIED1C9tUSoqKmpqau7du5eWlhaeAP8htiYZYiva4DGtv9EPpoBiyYULF7Kzs7lcblZWFl3otqzfWwIAq9VaVFSE47hIJGpsbJzeMNBg/Ex4UFFR4XQ6qeQEfkY7PdTgowUAeBDR+w/gzscjYjAYuFyuWq2enJwcGhpCCE1OTnpc1u8xAUBra+vbb79tsVhu3bp19OhR//MBUC/N1KjT1tbmO356CsiVRCKhki74Ga1bqN4aBnoBo1BsRRs8pvU3+sEdQMzo7OyUSCRffPEFl8ul05D5WNbvlgCAIIg//vijq6vrlVde+eqrr/zPB3Du3Dm9Xj86Orpt27aSkpJZpHucmppisVj+R+sWqu+GAIBZi4t0AMBfer1+4cKFboX0sn5q08fyn3fffffLL7/87LPP4uLiGhsb/W8ok8mo/2zatOl//ud/rl27FtA3I00m019//UUdxM+TuoWal5fnf7QIodhaiBRb0YJ/GbgDiBkCgcB1uT9d6OeyfgzDqqqqHj16tHnz5i1btswuH4DdbnfLDj+j06dPx8XFvfPOO/5H6xZqQN1EMTXJEFvRBi+gZw6YAzAAxIy33nrrzp07TU1NZrP53LlzVKH/y/q//fbb7u5uh8OxfPlyDMP8bPjf//63qqrKYrGYTKa6uronT56sWrXKd5wkSb548QIhpNPpNBrN119/rVQqk5KS/I/WLdSAugkACECk3xOAf/D9iNTV1aWkpAgEAupb6gUFBeS0Zf3eEgB0dHQsWLAgLi4uPT2dShXg1tDjGfV6fXp6OvWpw+uvv97b20vvmp7wQK1WSyQSNptNzfjjOP7aa681NTW51vEn2umh+hntjBcw2sRWtMFjWn+jH3wtN7rAF6WDFFsXMLaiDR7T+hv9YAoIIITQo0ePME8ePXoU6dAAAOECAwBACKGUlBSPd4gpKSmRDo3RbDbbkSNHtm7dSiVZqqmpocp7e3tTUlLYbHZZWVlYAxgYGMjNzT1//rxrocPhqKysFIlEfD4/MzPT6XSeOXOms7MzrJGAcIABAADP9uzZM4sboNm18sjhcBQVFa1atUqlUh05ckQqlR44cODp06cIoZUrV2q12tLSUnqldDj8+OOP33///Z9//ulWXl1dfeXKlf7+foPBIJfLnU5nYWFhb29vQ0ND+IIB4QADAACetbW1zVkrjw4dOpSQkLB8+XJqc9++fVwud//+/aE6/owUCsXhw4fdvvhrNpvVarVarZbJZPHx8S0tLXFxcQghpVK5d+/eWawTBBEEAwBgkIsXL2ZnZxMEIZfLqfRERUVFGIbdvXv38ePHUqmUXmJdXFw8ODiYmppKfRaSn59PEIRMJqN/iNFjQ7oVtXzBz3RJHjkcDo1GU1paSpcIBIK6urra2lr65xh9d81bVigUXGKl/v5+p9O5bNkyt3Iej1dQUHDq1KmAjgYibO6+cAT8AI9IkHxcQKPRiON4c3OzyWTSaDQ4jhsMBqrJ0NAQSZI3b97EcZyqbLPZEEIjIyMkSeI43tXVZbFYNBoNl8vV6/X0udwaurYKMlqtVosQMplM1KZKpero6CBJsri4uLCwkCTJkZEROlO3t655zArlf2IlSnJycmdnJ73Z1NTE4/GWLFlCEIRAINi2bZvT6aR2NTQ0yGSy2fUXRATcAQCm6O7ulkgk69evj4+PLy8vnz9//qVLl/xsm5SUxOPxqFY9PT3hDPP/3L9/n8PhxMfHu5WrVKorV65cu3bNtdB319yyQgWZWMnpdOI43tDQMDY2dvny5fr6+rNnz1K7RCLRgwcPSPiiZ+yAAQAwhcFgEIvF9KZEIjEYDIEeJCEhYXx8PKRxeTY5OcnhcKaXi8XiEydO7Ny507UwoK7RiZUwDNuzZ0+gy6rFYrHNZpPL5TweTy6X5+Tk3Lhxg9rF4XCcTufc/FAdCAkYAABTJCYmuiZTGh0dTUxMDOgIJEnqdLrk5ORQh+YBn8/3Nju/bt06iURCfxqBAuza7NJA0ZYuXToxMUF/DuFwOHAcp/5vtVpZLNYc/EgRCBUYAABT5OXljY2NNTc3T0xMaDSaZ8+eUVlFCYK4evWqzWbT6XR0ZRaLxWKxBgYGLBYLQshsNk9NTalUKqvVmpubS9WZ3tCtVTCkUunU1JTZbPa4t7a29uTJkzN2zaMgEyslJCQUFBRUVlY+efKkr69Pq9Xm5+dTu4xGY1paGuQ3jSUR+/QBeAKPSJB8X8Curq6srCw+ny+Xy+ksQ7t37+Zyuenp6eXl5QihsrIyqryoqIjD4SgUChzHhULhvHnz5HJ5T08PfTSPDelWpKd0Sf5Ha7fbFyxYoNVqSZI8fvw4QRBCobC+vp6u0NjY6PpzzdO75i0rFDktsZLRaBSLxSqVyi2G7du3p6amIoQIgsjJydHpdFT5+Pj4mjVreDzewoUL1Wq1a/1Z9xdEBKTmiC6QLCVI4biABEH09fUF9CsIfvId7cGDB58+fXrs2LGQn9eNzWYrKSmRy+XV1dWzPojdbs/IyGhvb8/IyPBWB57e0QamgACYmdPpnPuT7tq1a3h4+Pr16+E+kUajEYlEO3bsCOYgSqWyqqrKx6s/iEIwIEcXeIsUpJBfwI8//viHH35ITk5ub29fsmRJCI+M/IjWZrMdPXpUoVDQv8sWndra2l566aXVq1f7rgZP72gDj0d0gb+QIMXWBYytaIPHtP5GP5gCAgAAhoIBAAAAGAoGAAAAYCiYkosusIgG/LvBC05UgQEAMNfevXvpfwFgIJgCAgAAhoIBAAAAGAoGAAAAYCgYAAAAgKFgAAAAAIaCAQAAABgKBgAAAGAoGAAAAIChYAAAAACGggEAAAAYCgYAAABgKBgAAACAoWAAAAAAhoIBAAAAGAoGAAAAYCgYAAAAgKFgAAAAAIaCAQAAABgKBgAAAGAoGAAAAIChYAAAAACGggEAAAAYCgYAAABgKBgAAACAoeIiHQAAc8pisfznP/+h/j88PIwQunjxIrX5+uuv8/n8iEUGwJzDSJKMdAwAzB273S4Wi61Wa1xcHPXkxzDMbrez2Wyj0RgXB2+JAIPAFBBglri4uLVr17548cJkMk1MTExMTJhMphcvXqxduxZe/QHTwAAAGOeTTz7h8XiuJXw+v6ysLFLxABApMAUEGIckSYlEYjQa6RKRSDQ2NoZhWASjAmDuwR0AYBwMw0pLS9lsNrU5b968DRs2wKs/YCAYAAATlZaW0jP+bDZ7w4YNkY0HgIiAKSDAUFKp9MGDBwih1NTUhw8fRjocACIA7gAAQ23cuJHL5XI4nE8//TTSsQAQGXAHABhqeHj41VdfRQjdunVr0aJFkQ4HgAiAAQAwV2ZmJoZht27dinQgAEQGDADRBb6LAv7d4AUnqsDSx6gDfyHBwLAA3tOMjo4ihBITE8MZkS8BRfsvAO9vog2znn/Rj2mvCCEXWxcwtqINHtP6G/3gW0AAAMBQMABltKS3AAANg0lEQVQAAABDwQAAAAAMBQMAYKjVq1efPHky0lHMwGazHTlyZOvWrQRBYBhWU1NDlff29qakpLDZ7HAnMR0YGMjNzT1//rxrocPhqKysFIlEfD4/MzPT6XSeOXOms7MzrJGAsCBBNIFHJEiRuoDffPPNyMhIoK18R2u32wsKCvr6+kiSVKlUUqlUIBCMj49Te0dGRjZt2jS7aP3U0tKya9cuoVDY2dnpWl5ZWblixYrh4WGTyaRQKGw2G0mSVVVV3333ne8DwtM72sAdAAAh0NbWFvJjHjp0KCEhYfny5dTmvn37uFzu/v37Q34ibxQKxeHDh91+O8FsNqvVarVaLZPJ4uPjW1paqLR6SqVy7969t2/fnrPwQPBgAABM1NDQwOVylUplZWUlhmGff/55RkYGQRD79u2jKuzYsQPDsPz8fIIgZDLZTz/9hBAqKirCMOzu3buPHz+WSqUEQVCVi4uLBwcHU1NTt2zZghD64IMPtm/fHmSEDodDo9GUlpbSJQKBoK6urra29t69e9PrX7x4MTs7myAIuVx+4cIFhJC3riGEfv755/T09Jdffnnjxo1WqzWgwPr7+51O57Jly9zKeTxeQUHBqVOnAjoaiLBI34KAf4BHJEj+X8BNmzZVV1eTJCmRSH777Ten03n69Gk+n09XwHG8q6vLYrFoNBoul6vX66njDw0NkSR58+ZNHMepmjabDSEU2ikgrVaLEDKZTNSmSqXq6OggSbK4uLiwsJD85xSQ0WjEcby5udlkMmk0GhzHDQaDt67p9Xoej9fR0TE+Pr506dITJ074DjI5Odl1CqipqYnH4y1ZsoQgCIFAsG3bNqfTSe1qaGiQyWSz6y+ICLgDAAAhhDAMe+ONNywWi91upwuTkpJ4PF55efn8+fN7enrmMp779+9zOJz4+Hi3cpVKdeXKlWvXrrkWdnd3SySS9evXx8fHU9FeunSJ3uvWtZ6entTU1Pfff18oFH744Ye//vprQIE5nU4cxxsaGsbGxi5fvlxfX3/27Flql0gkevDgAQlLvWIHDAAAzCwhIWF8fHwuzzg5OcnhcKaXi8XiEydO7Ny507XQYDCIxWJ6UyKRGAwGb0c2Go137tzBMAzDsD179jx//jygwMRisc1mk8vlPB5PLpfn5OTcuHGD2sXhcJxO59TUVEAHBBEEAwAAMyBJUqfTJScnz+VJ+Xy+t9n5devWSSQS6mMJSmJioutPHI+OjvpIcCQQCLKysuhJgO7u7oACW7p06cTEBP05hMPhwHGc+r/VamWxWFwuN6ADggiCAQAAr8xm89TUlEqlslqtubm5CCGCIK5evWqz2XQ6HV2NxWKxWKyBgQGLxRKqU0ul0qmpKbPZ7HFvbW2t6yKGvLy8sbGx5ubmiYkJjUbz7NmzvLw8b0d+8803BwcHW1pa/v77b4vFEugdQEJCQkFBQWVl5ZMnT/r6+rRabX5+PrXLaDSmpaVBxrdYErFPH4An8IgEyc8LWFlZyWaz+Xw+9VeQlpb2/PnzzMxMhFBJSQlVB8dxoVA4b948uVze09NDFe7evZvL5aanp5eXlyOEysrKqPKioiIOh6NQKEiSfO+997Zt2xZktHa7fcGCBVqtliTJ48ePEwQhFArr6+vpCo2Nja7rALq6urKysvh8vlwu7+7uJkmyoqLCW9daW1vT09M5HM6KFStu3LhhNBrFYrFKpXKLYfv27ampqQghgiBycnJ0Oh1VPj4+vmbNGh6Pt3DhQrVa7Vrfd8fh6R1tIDlfdIF0iUEK4QUkCKKvr2/x4sUhOZpHvqM9ePDg06dPjx07Fr4AKDabraSkRC6XV1dXz/ogdrs9IyOjvb09IyPDWx14ekcbmAKKPQ8fPly0aBGGYXPzadv0df/eatbU1FAZCzAMY7FYQqEwNze3tbV1DoIMEx+dnQO7du0aHh6+fv16uE+k0WhEItGOHTuCOYhSqayqqvLx6g+iUUTvP4A7Px8RvV6PEJqcnAx3PKSXdf/eqFQqiURCkqTJZOrv71coFAghpVI5B3FSQvWU/uijjxBCycnJv//+e0gO6NGM0Vqt1oMHDw4PD4cvhpBobW395ZdfZqwGLzjRBu7Ioouf98gGgyExMXFycjLc37gwm82JiYk9PT3TV356pFarDxw4QP3SFqWioqKmpubevXtpaWlhC/P/i61JhtiKNnhM62/0gymgWHLhwoXs7Gwul5uVlUUXui3r95YAwGq1FhUV4TguEokaGxunN/R4Rm/r/pHfCQ8qKiqcTieVnMDPaKeH6me0AIDARPT+A7jz8YgYDAYul6tWqycnJ4eGhhBCk5OTHpf1e0wA0Nra+vbbb1ssllu3bh09etTPfAA+1v17RE8BuZJIJFTSBT+jdQvVW8NAL2AUiq1og8e0/kY/uAOIGZ2dnRKJ5IsvvuByuXQaMh/L+t0SABAE8ccff3R1db3yyitfffWVn/kAfKz799/U1BSLxfI/WrdQfTcEAMxaXKQDAP7S6/ULFy50K6SX9VObPpb/vPvuu19++eVnn30WFxfX2NjoZ0N63T9CiF73v2bNGv/DNplMf/31l0wm8z9at1Dz8vL87yZCKLYWIsVWtOBfBu4AYoZAIHBd7k8X+rmsH8OwqqqqR48ebd68ecuWLX429LHu30+nT5+Oi4t75513/I/WLdSAuoliapIhtqINXkDPHDAHYACIGW+99dadO3eamprMZvO5c+eoQv+X9X/77bfd3d0Oh2P58uUYhvnZ0Me6f29Iknzx4gVCSKfTaTSar7/+WqlUJiUl+R+tW6gBdRMAEIBIvycA/+D7Eamrq0tJSREIBNS31AsKCshpy/q9JQDo6OhYsGBBXFxceno6lSrAraG3k3pb9z894YFarZZIJGw2m5rxx3H8tddea2pqcq3jT7TTQ/U/2th6SsdWtMFjWn+jH3wtN7rAF6WDFFsXMLaiDR7T+hv9YAoIIITQo0ePME8ePXoU6dAAAOECAwBACKGUlBSPd4gpKSmRDg38H5vNduTIka1bt1IJl2pqaqjy3t7elJQUNptdVlYW1gAGBgZyc3PPnz9Pl5w5c6azszOsJwVhBQMAAJ7t2bNnFjdAs2s1I4fDUVRUtGrVKpVKdeTIEalUeuDAgadPnyKEVq5cqdVqS0tL6VXT4fDjjz9+//33f/75p2thYWFhb29vQ0ND+M4LwgoGAAA8a2trm7NWMzp06FBCQsLy5cupzX379nG53P3794fjXB4pFIrDhw/zeDy3cqVSuXfv3tu3b89ZJCCEYAAADHLx4sXs7GyCIORyOZWeqKioCMOwu3fvPn78WCqV0kusi4uLBwcHU1NTqc9C8vPzCYKQyWT0DzF6bEi3opYv+JkuaUYOh0Oj0ZSWltIlAoGgrq6utraWXqLhu5veMkShoJMs8Xi8goKCU6dOzapnINLm7gtHwA/wiATJxwU0Go04jjc3N5tMJo1Gg+O4wWCgmgwNDZEkefPmTRzHqco2mw0hNDIyQpIkjuNdXV0Wi0Wj0XC5XL1eT5/LraFrqyCjdaXVahFCJpOJ2lSpVB0dHSRJFhcXFxYWkiQ5MjJC/zqYt256zBDlf5IlSnJycmdnp1thQ0ODTCYLYX/BnIE7AMAU3d3dEolk/fr18fHx5eXl8+fPv3Tpkp9tk5KSeDwe1aqnpyecYXpw//59DocTHx/vVq5Sqa5cuXLt2jXXQt/ddMsQFZIkSyKR6MGDByR8vzMGwQAAmMJgMIjFYnpTIpEYDIZAD5KQkDA+Ph7SuGY2OTnJ4XCml4vF4hMnTuzcudO1MKBu0kmWMAzbs2fP7JZYczgcp9M5N79PB0ILBgDAFImJia7JlEZHRxMTEwM6AkmSOp0uOTk51KHNgM/ne5udX7dunUQioT+ZQAF2M6AkS95YrVYWixXu3yYC4QADAGCKvLy8sbGx5ubmiYkJjUbz7NkzKqsoQRBXr1612Ww6nY6uzGKxWCzWwMCAxWJBCJnN5qmpKZVKZbVac3NzqTrTG7q1ChWpVDo1NWU2mz3ura2tPXny5Izd9CgkSZaMRmNaWhqkNY1JEfv0AXgCj0iQfF/Arq6urKwsPp8vl8vpLEO7d+/mcrnp6enl5eUIobKyMqq8qKiIw+EoFAocx4VC4bx58+RyeU9PD300jw3pVqSndEkBRUuz2+0LFizQarUkSR4/fpwgCKFQWF9fT1dobGykPwT22E1vGaLIaUmWjEajWCxWqVRuMWzfvj01NRUhRBBETk6OTqdz3eW7m4H2F8wZSM0RXSBZSpDCcQEJgujr61u8eHFoD4sCifbgwYNPnz49duxYyGNwY7PZSkpK5HJ5dXW1P/XtdntGRkZ7e3tGRsaMleHpHW1gCgiAmTmdzsgGsGvXruHh4evXr4f7RBqNRiQS7dixw8/6SqWyqqrKn1d/EIVgQI4u8BYpSCG/gB9//PEPP/yQnJzc3t6+ZMmSEB4ZBRitzWY7evSoQqGgfl4tGrS1tb300kurV6/2sz48vaMNPB7RBf5CghRbFzC2og0e0/ob/WAKCAAAGAoGAAAAYCgYAAAAgKFgSi66wGoa8O8GLzhRBQYAAABgKJgCAgAAhoIBAAAAGAoGAAAAYKj/BUOTU9tAf+fIAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<IPython.core.display.Image object>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qorXCi0nLKFx"
      },
      "source": [
        "### Training Run"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HScBiexx7whg",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 340
        },
        "outputId": "fe2f18b6-c438-4f2a-a84c-2cb1f50116fc"
      },
      "source": [
        "### Parameters ###\n",
        "batch_size = 256 #@param {type:\"integer\"}\n",
        "epochs = 4 #@param {type:\"slider\", min:0, max:100, step:1}\n",
        "validation_split = 0.2 #@param {type:\"slider\", min:0, max:1, step:0.1}\n",
        "###            ###\n",
        "\n",
        "history = model.fit(x=train_data,\n",
        "                    y=train_labels,\n",
        "                    batch_size=batch_size,\n",
        "                    epochs=epochs,\n",
        "                    validation_split=validation_split,\n",
        "                    verbose=1)"
      ],
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "error",
          "ename": "ValueError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-35-e555c9a89ddd>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     10\u001b[0m                     \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mepochs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m                     \u001b[0mvalidation_split\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalidation_split\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 12\u001b[0;31m                     verbose=1)\n\u001b[0m",
            "\u001b[0;32m/tensorflow-1.15.2/python3.6/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_freq, max_queue_size, workers, use_multiprocessing, **kwargs)\u001b[0m\n\u001b[1;32m   1152\u001b[0m             \u001b[0msample_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msample_weight\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1153\u001b[0m             \u001b[0mclass_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mclass_weight\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1154\u001b[0;31m             batch_size=batch_size)\n\u001b[0m\u001b[1;32m   1155\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1156\u001b[0m         \u001b[0;31m# Prepare validation data.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/tensorflow-1.15.2/python3.6/keras/engine/training.py\u001b[0m in \u001b[0;36m_standardize_user_data\u001b[0;34m(self, x, y, sample_weight, class_weight, check_array_lengths, batch_size)\u001b[0m\n\u001b[1;32m    619\u001b[0m                 \u001b[0mfeed_output_shapes\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    620\u001b[0m                 \u001b[0mcheck_batch_axis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m  \u001b[0;31m# Don't enforce the batch size.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 621\u001b[0;31m                 exception_prefix='target')\n\u001b[0m\u001b[1;32m    622\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    623\u001b[0m             \u001b[0;31m# Generate sample-wise weight values given the `sample_weight` and\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/tensorflow-1.15.2/python3.6/keras/engine/training_utils.py\u001b[0m in \u001b[0;36mstandardize_input_data\u001b[0;34m(data, names, shapes, check_batch_axis, exception_prefix)\u001b[0m\n\u001b[1;32m    143\u001b[0m                             \u001b[0;34m': expected '\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mnames\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m' to have shape '\u001b[0m \u001b[0;34m+\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    144\u001b[0m                             \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m' but got array with shape '\u001b[0m \u001b[0;34m+\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 145\u001b[0;31m                             str(data_shape))\n\u001b[0m\u001b[1;32m    146\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    147\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mValueError\u001b[0m: Error when checking target: expected dense_6 to have shape (1,) but got array with shape (3,)"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TlcbXYIs4T1l"
      },
      "source": [
        "# Evaluation Phase\n",
        "results = model.evaluate(test_data, test_labels)\n",
        "print()\n",
        "\n",
        "for i in range(len(model.metrics_names)):\n",
        "    if model.metrics_names[i] == \"acc\":\n",
        "        print(f\"Metric - {model.metrics_names[i]}: {results[i]*100:.3f}\")\n",
        "    else:\n",
        "        print(f\"Metric - {model.metrics_names[i]}: {results[i]}\")  \n",
        "\n",
        "if multiclass == False: \n",
        "    predictions = [1 * (x[0]>=0.5) for x in model.predict(test_data)]\n",
        "    cm = confusion_matrix(test_labels, predictions)\n",
        "else:    \n",
        "    predictions = np.argmax(model.predict(test_data), axis=1)\n",
        "    cm = confusion_matrix(test_labels.argmax(axis=1), predictions)\n",
        "print()\n",
        "print(cm)\n",
        "\n",
        "heatmap(cm, fmt=\"g\", annot=True, square=True, cmap=plt.cm.Blues)\n",
        "plt.tight_layout()\n",
        "plt.ylabel('True label')\n",
        "plt.xlabel('Predicted label')\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tAGmRQq3ukdo"
      },
      "source": [
        "## Model Analysis"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "U0A5sddZXEJ4"
      },
      "source": [
        "### Model Analysis: Out-of-Vocabulary (OOV) words in Embeddings\n",
        "\n",
        "Obviously, assigning all OOV words the same \"UNK\" vector will decrease performance [[1](https://stackoverflow.com/questions/45495190/initializing-out-of-vocabulary-oov-tokens)].\n",
        "\n",
        "Other ideas:\n",
        "- Simply remove all OOV words; too high time complexity.\n",
        "- Assign the average of its surrounding words [[2](https://stackoverflow.com/questions/41517969/how-to-get-random-word2vec-vector-for-unknow-word)], but that would require running through the entire dataset and performing calculations, thus too high time complexity.\n",
        "- Assign a unique random vector (uniform -0.5 to 0.5) to each word  [[1](https://stackoverflow.com/questions/45495190/initializing-out-of-vocabulary-oov-tokens)].\n",
        "\n",
        "| Approach (Using LSTM, 10 epochs) | Trainable On/Off | Performance |\n",
        "| - | - | - | \n",
        "| Pretrained, All OOV words set to a vector of zeros | False | 85.420 |\n",
        "| Pretrained, All OOV words set to a vector of zeros | True | 87.500 |\n",
        "| Pretrained, Each OOV word set to a unique random vector | False | 84.608 |\n",
        "| Pretrained, Each OOV word set to a unique random vector | True | 87.568 |\n",
        "| My own custom Word2Vec and use random vectors for OOV | False | 86.004 |\n",
        "| My own custom Word2Vec and use random vectors for OOV | True | 87.612 |"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "alnczQ9if68e"
      },
      "source": [
        "### Model Analysis: **Overfitting**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DwOsGSNJ7whz"
      },
      "source": [
        "def plot_model_history():\n",
        "    plt.clf()  # clear figure\n",
        "\n",
        "    plt.style.use('ggplot')\n",
        "\n",
        "    acc = history.history['accuracy']\n",
        "    val_acc = history.history['val_accuracy']\n",
        "    loss = history.history['loss']\n",
        "    val_loss = history.history['val_loss']\n",
        "    epochs = range(1, len(acc) + 1)\n",
        "\n",
        "    plt.figure(figsize=(14, 4))\n",
        "    plt.subplot(1, 2, 1)\n",
        "    plt.plot(epochs, acc, 'b', label='Training acc')\n",
        "    plt.plot(epochs, val_acc, 'g', label='Validation acc')\n",
        "    plt.title('Training and validation accuracy')\n",
        "    plt.legend()\n",
        "    plt.subplot(1, 2, 2)\n",
        "    plt.plot(epochs, loss, 'bo', label='Training loss')\n",
        "    plt.plot(epochs, val_loss, 'g', label='Validation loss')\n",
        "    plt.title('Training and validation loss')\n",
        "    plt.legend()\n",
        "    plt.show()\n",
        "\n",
        "print(history.history.keys())    \n",
        "plot_model_history()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wTyPYITNYDw0"
      },
      "source": [
        "After a certain number of Epochs the line continues to improve for the training data but does not for the validation data. This is a result of **overfiting**. After that point, the model over-optimizes and learns representations specific to the training data.\n",
        "\n",
        "We should simply stop training after a certain epoch. To prevent overfitting, the best solution is to use more training data or reduce the number of parameters (capacity). A model trained on more data will naturally generalize better. When that is no longer possible, the next best solution is to use techniques like regularization - two common such techniques are weight regularization and dropout. Concluding, the most common way to prevent overfitting are:\n",
        "- Get more training data.\n",
        "- Reduce the capacity of the network.\n",
        "- Add weight regularization.\n",
        "- Add dropout."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "beQ4rpC0codS"
      },
      "source": [
        "## Comparative Analysis of Parameters"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "K-zfSDox4_U8"
      },
      "source": [
        "### Preprocessing Parameters\n",
        "\n",
        "Manually change the parameter at hand and note down the resulting accuracy. \n",
        "Remember that we can't use the test set since it is considered cheating (data leak). However, the parameters tested here are of very general scope so we will use the test set anyway. The following neural network structure was used:\n",
        "\n",
        "```\n",
        "elif embeddings_mode == \"One-hot Encoding\":\n",
        "    model = keras.Sequential()\n",
        "    model.add(Embedding(input_dim=feature_count, output_dim=16, input_length=embeddings_sequence_length))  \n",
        "    model.add(CuDNNLSTM(units=neurons))\n",
        "    model.add(Dropout(0.1, seed=random_state)) \n",
        "    ...\n",
        "    \n",
        "validation_split = 0.1\n",
        "```\n",
        "\n",
        "| Feature Count | Performance |\n",
        "| - | - |\n",
        "| 5000 | 86.456 |\n",
        "| 10000 | 87.048 |\n",
        "| 20000 | 86.868 |\n",
        "| 30000 | 86.504 |\n",
        "\n",
        "| Feature Count (Big Seq Length) | Performance |\n",
        "| - | - |\n",
        "| 10000 | 86.708 |\n",
        "| 20000 | 87.004 |\n",
        "| 30000 | 86.812 |\n",
        "\n",
        "| Embeddings Sequence Length | Performance |\n",
        "| - | - |\n",
        "| 64 | 81.88 |\n",
        "| 128 | 85.28 |\n",
        "| 256 | 87.388 |\n",
        "| 512 | 85.896 |\n",
        "\n",
        "| Remove First | Performance |\n",
        "| - | - |\n",
        "| True | 0.8664 |\n",
        "| False | 0.8691 |"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cy74T7xPgvou"
      },
      "source": [
        "### **Model** Parameters (Grid Search) \n",
        "\n",
        "Use GridSearchCV to automatically try all possible parameters combinations. Remember that we can't use the test set since it is considered cheating (data leak).\n",
        "\n",
        "| Parameter | Values | Best Performing |\n",
        "| - | - | - |\n",
        "| Batch Size | batch_size = [32, 64, 128, 256, 512, 1024] | (useless benchmark) curve with 128 being the peak, high deviation, big drop off at large values |\n",
        "| Epochs | epochs = [1, 2, 3, 5, 10] | (useless benchmark) curve with 3 being the peak |\n",
        "| Validation Set Split | validation_split = [0.0, 0.1, 0.25, 0.5] | the lower the better, 0.1 performs best |\n",
        "| Neurons | neurons = [8, 16, 32, 64] | the higher the better |\n",
        "| Embeddings Initializer | init_mode = ['uniform', 'lecun_uniform', 'normal', 'zero', 'glorot_normal', 'glorot_uniform', 'he_normal', 'he_uniform'] | 'uniform' 'normal' 'glorot_normal' 'he_normal' 'he_uniform' |\n",
        "| Activation Function | activation = ['softmax', 'softplus', 'softsign', 'relu', 'tanh', 'sigmoid', 'hard_sigmoid', 'linear'] | 'linear' 'relu' 'tanh' 'softsign'|\n",
        "| Optimizer | optimizer = ['Adam', 'SGD', 'RMSprop', 'Adagrad', 'Adadelta', 'Adamax', 'Nadam'] | 'Adam' 'Adagrad' 'Adamax' |\n",
        "\n",
        "Some General Information [[1](https://www.dlology.com/blog/how-to-choose-last-layer-activation-and-loss-function/)]:\n",
        "\n",
        "| Problem type | Last-layer activation | Loss function | Example |\n",
        "| - | - | - | - |\n",
        "| Binary classification | sigmoid | binary_crossentropy | Dog vs. cat, Sentiment analysis |\n",
        "| Multi-class, single-label classification | softmax | categorical_crossentropy | MNIST |\n",
        "| Multi-class, multi-label classification | sigmoid | binary_crossentropy | News tags classification, one blog can have multiple tags |\n",
        "| Regression to arbitrary values | None | mse | Predict house price (an integer/float) |\n",
        "| Regression to values between 0 and 1 | sigmoid | mse or binary_crossentropy | Engine health assessment where 0 is broken, 1 is new |"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GLK_rNemh0wb"
      },
      "source": [
        "### Parameters ###\n",
        "run_this_section = False #@param {type:\"boolean\"}\n",
        "###            ###\n",
        "\n",
        "if run_this_section == True:\n",
        "    from keras.wrappers.scikit_learn import KerasClassifier\n",
        "    from sklearn.model_selection import GridSearchCV\n",
        "\n",
        "    def create_model(neurons=16, init_mode='uniform', activation='relu', optimizer='Adam'):\n",
        "        model = Sequential()\n",
        "        model.add(Embedding(input_dim=feature_count, output_dim=16, input_length=embeddings_sequence_length))  \n",
        "        model.add(CuDNNLSTM(units=neurons))  # CuDNNLSTM: Tensorflow's default LSTM is awful at utilizing GPU/parallelization, [1] https://stackoverflow.com/a/47610747, [2] https://news.ycombinator.com/item?id=14538086\n",
        "        model.add(Dropout(0.1, seed=random_state))\n",
        "        model.add(Dense(units=neurons, activation=activation, use_bias=True))\n",
        "        model.add(Dense(1, activation='sigmoid'))  # the value of 1 doesn't refer to the neuron count, e.g. for multi-class it refers to the number of categories\n",
        "\n",
        "        model.compile(optimizer='Adam',\n",
        "                      #optimizer=keras.optimizers.Adam(lr=lr),\n",
        "                      loss='binary_crossentropy',\n",
        "                      metrics=['accuracy'])   \n",
        "        return model\n",
        "\n",
        "    param_grid = dict(batch_size = [128],\n",
        "                      epochs = [10],\n",
        "                      validation_split = [0.0],\n",
        "                      #neurons = [8, 16, 32, 64],\n",
        "                      #init_mode = ['uniform', 'lecun_uniform', 'normal', 'zero', 'glorot_normal', 'glorot_uniform', 'he_normal', 'he_uniform'],\n",
        "                      #activation = ['softmax', 'softplus', 'softsign', 'relu', 'tanh', 'sigmoid', 'hard_sigmoid', 'linear'],\n",
        "                      #kernel_constraint = [1, 2, 3, 4, 5],\n",
        "                      #optimizer = ['Adam', 'SGD', 'RMSprop', 'Adagrad', 'Adadelta', 'Adamax', 'Nadam'],\n",
        "                      #lr = [0.01, 0.001, 0.0005]\n",
        "                     )\n",
        "\n",
        "    # Make use of Keras' scikit-learn API, also supports the parameters of the 'fit', 'predict', 'predict_proba', and 'score' methods\n",
        "    model = KerasClassifier(build_fn=create_model, verbose=0)\n",
        "\n",
        "    grid = GridSearchCV(estimator=model, param_grid=param_grid, cv=4, verbose=1, n_jobs=-1)  # Where 'cv' is the number of cross-validation folds\n",
        "\n",
        "    grid_result = grid.fit(train_data, train_labels)\n",
        "\n",
        "    print(f\"\\n- - - - - BEST PARAMETERS - - - - -\")\n",
        "    print(f\"{grid_result.best_score_:.8f} using {grid_result.best_params_}\")\n",
        "\n",
        "    print(f\"\\n- - - - - DETAILS - - - - -\")\n",
        "    for i in range(len(grid_result.cv_results_['params'])):         \n",
        "        print(f\"mean - {grid_result.cv_results_['mean_test_score'][i]:.10f}; std - {grid_result.cv_results_['std_test_score'][i]:.10f} - {grid_result.cv_results_['params'][i]}\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KRVyRrAuCWrv"
      },
      "source": [
        "## Advanced Code\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "43Gurxa0JZcC"
      },
      "source": [
        "### Code"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CEERonQcDfYS",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "17fc177d-f8f5-4be3-f246-7648e18bdb01"
      },
      "source": [
        "# Adam print the weights when using decay\n",
        "def adam_weights_calculator(epochs=5):\n",
        "    '''\n",
        "        Prints the learning rate per epoch when using decay\n",
        "    '''\n",
        "    for i in range(epochs):\n",
        "        beta_1 = 0.9 \n",
        "        beta_2 = 0.999\n",
        "        lr = 0.001\n",
        "        decay = 0.01\n",
        "\n",
        "        lr = lr * (1. / (1. + decay * i))\n",
        "        t = i + 1\n",
        "        lr_t = lr * math.sqrt(1. - math.pow(beta_2, t)) / (1. - math.pow(beta_1, t))\n",
        "        print(f\"Adam LR on epoch {i+1}: {lr_t:.6f}\")\n",
        "\n",
        "adam_weights_calculator(5)"
      ],
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Adam LR on epoch 1: 0.000316\n",
            "Adam LR on epoch 2: 0.000233\n",
            "Adam LR on epoch 3: 0.000198\n",
            "Adam LR on epoch 4: 0.000178\n",
            "Adam LR on epoch 5: 0.000166\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ABwHfx9ij2hy",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6fefaa8b-f6ce-4871-c204-63ac47067c8a"
      },
      "source": [
        "# For Multi-class tasks in particular, the following changes need to be performed\n",
        "# 1. Dense(1), the value of 1 on the final layer refers to the number of categories\n",
        "# 2. activation = \"softmax\"\n",
        "# 3. loss = \"categorical_crossentropy\" https://stackoverflow.com/a/46038271\n",
        "\n",
        "#model.add(CuDNNLSTM(units=128))  # CuDNNLSTM: Tensorflow's default LSTM is awful at utilizing GPU/parallelization, [1] https://stackoverflow.com/a/47610747, [2] https://news.ycombinator.com/item?id=14538086\n",
        "\n",
        "model = Sequential()\n",
        "model.add(Embedding(input_dim=feature_count, output_dim=100, input_length=embeddings_sequence_length))  \n",
        "#model.add(Embedding(input_dim=feature_count, output_dim=embeddings_dimension, weights=[embedding_vectors], input_length=embeddings_sequence_length, trainable=trainable))\n",
        "#model.add(Dropout(0.2, seed=random_state))  # Maybe remove the seed since it makes it biased\n",
        "#model.add(Conv1D(filters=250, kernel_size=3, padding='valid', activation='relu', strides=1))\n",
        "#model.add(GlobalAveragePooling1D())\n",
        "#model.add(Flatten())\n",
        "model.add(CuDNNLSTM(units=128, return_sequences=True))\n",
        "model.add(CuDNNLSTM(units=128, return_sequences=True))\n",
        "#model.add(MultiplicativeLSTM(units=128))\n",
        "model.add(CuDNNLSTM(units=128))\n",
        "#model.add(Flatten())\n",
        "#model.add(BatchNormalization())\n",
        "#model.add(CuDNNLSTM(units=60, kernel_regularizer=keras.regularizers.l2(0.00001)))\n",
        "model.add(Dropout(0.5, seed=random_state))  \n",
        "#model.add(Dense(units=128, activation='relu', use_bias=True))\n",
        "#model.add(Dense(units=128, activation='relu', use_bias=True))\n",
        "#model.add(Flatten())\n",
        "model.add(Dense(3, activation='softmax'))\n",
        "\n",
        "model.summary()\n",
        "\n",
        "model.compile(#optimizer=Adagrad(lr=0.005),\n",
        "              optimizer=Adam(lr=0.00005),  # 0.0005 or even better 0.00005\n",
        "              loss='categorical_crossentropy',\n",
        "              metrics=['accuracy'])\n",
        "\n",
        "early_stop = EarlyStopping(monitor='val_loss', mode='min', baseline=0.90, patience=3, verbose=1)\n",
        "model_save = ModelCheckpoint('./gdrive/My Drive/Colab Saved Models/best_model_text.h5', monitor='val_loss', mode='min', save_best_only=True)"
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"sequential_3\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "embedding_3 (Embedding)      (None, 50, 100)           1500000   \n",
            "_________________________________________________________________\n",
            "cu_dnnlstm_1 (CuDNNLSTM)     (None, 50, 128)           117760    \n",
            "_________________________________________________________________\n",
            "cu_dnnlstm_2 (CuDNNLSTM)     (None, 50, 128)           132096    \n",
            "_________________________________________________________________\n",
            "cu_dnnlstm_3 (CuDNNLSTM)     (None, 128)               132096    \n",
            "_________________________________________________________________\n",
            "dropout_1 (Dropout)          (None, 128)               0         \n",
            "_________________________________________________________________\n",
            "dense_7 (Dense)              (None, 3)                 387       \n",
            "=================================================================\n",
            "Total params: 1,882,339\n",
            "Trainable params: 1,882,339\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1N4bJDWmj7z_",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 570
        },
        "outputId": "0e6e9cba-ed7d-46b1-971c-182aabc5500a"
      },
      "source": [
        "plot_model(model, show_shapes=True, show_layer_names=True, dpi=72)"
      ],
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXIAAAIpCAIAAAAw5yT3AAAABmJLR0QA/wD/AP+gvaeTAAAgAElEQVR4nOzdfVwU5do48HuAfWF3kJa3BZfNBWrNF1xSSyR7NMUSzaRSCFFUSM1jHhU7ikBpeEj95Mv5uCpkiqlpUD6ZcJAE7aigWJlpYTz4AiniAguyLrAsu8vO7495mmd/sLvMwsCyen3/6AP33nPPNcN0OTN7zdwYQRAIAACY42TvAAAAjxtIKwAAhkFaAQAwzIVmPwzD+jQOAMDAR/NWLN20Qn9EAHoMwzAHOswcK9reo39uARdBAACGQVoBADAM0goAgGGQVgAADIO0AhxSRETEnj177B1FN/R6/ZYtW1asWIHjOIZhO3bsINtLSkr8/f3ZbPaiRYv6bu0SiQT7y4QJE6j2ioqK0NBQLpcbGhp68+ZNhNDx48cLCgqYXDdBD/2eAPSYvQ6zjz76qLq62talrEdrMBgiIyN/+ukngiDkcrlEIhEIBI2NjeSn1dXVCQkJPYuWJrPjG43GUaNGpaamajSav/3tbyEhIWR7cnLygQMHrA9I/68DZysAoJycHMbH/OSTT3x8fF544QXy17S0NC6Xu2nTJsZXZJNr167duHFj/fr1rq6umzZtun79+rVr1xBCqampGzdu/OOPPxhZC6QV4HiysrK4XG5qauq6deswDFu2bNmwYcNwHE9LSyM7JCYmYhg2depUHMcDAgK++eYbhFBUVBSGYbdv337w4IFEIsFxnOwcHR1dUVEhFovff/99hNDMmTNXr17dywg7OjoyMzPj4uKoFoFAkJGRsXfv3jt37nTtf+bMmZCQEBzHZTLZ6dOnEUKWNg0h9O2330qlUnd39/j4eJ1OZ1Ng169fDwgI4PF4CCEPDw9/f//r168jhFxdXSMjI/ft29ez7e2M5lkN/Z4A9Bj9wywhISElJYUgCKFQWFxcbDQav/jiCx6PR3Xg8/mFhYUajSYzM5PL5SoUCnL8W7duEQTx66+/8vl8sqder0cIMXsRdPnyZYSQWq0mf5XL5Xl5eQRBREdHz549m/j/L4KUSiWfzz969Kharc7MzOTz+XV1dZY2TaFQuLq65uXlNTY2jhkzZteuXZZiIPMRl8sNDg7OyckhG7dt2yaTyag+wcHB27dvJ3/OysoKCAjo2fZ2Amcr4HGAYdjEiRM1Go3BYKAa/fz8XF1dly5d6unpee7cuf6Mp6qqisPhuLm5dWqXy+Xnz58vLS01bSwqKhIKhXPnznVzcyOj/eGHH6hPO23auXPnxGLx66+/7uHh8cYbb1y4cMFSDPn5+QqFora2duXKlbGxsWYvcIxGI5vNJn/28vK6e/cuwUTdMKQV8Pjz8fFpbGzszzW2tbVxOJyu7d7e3rt27VqzZo1pY11dnbe3N/WrUCisq6uzNLJSqbx58yb5/c6GDRtUKpWlngEBATiOu7u7JyQkBAYGkrnMy8urpaWF6qNSqYRCIfkzh8MxGo1arZb2VloEaQU85giCqKmpEYlE/blSHo9n6a7HO++8IxQKyds9JF9fX6VSSf1aW1vr6+traWSBQBAcHExdbhQVFdGJx2AwuLq6IoRkMllVVRWZWRoaGmpqaoYOHUr20el0Tk5OXC6XzoDWQVoBj62WlhatViuXy3U63eTJkxFCOI5fvHhRr9fX1NRQ3ZycnJycnMrLyzUaDVOrlkgkWq3W9LzA1N69e02LbsLDw+vr648ePdrc3JyZmdnU1BQeHm5p5EmTJlVUVBw7dqy1tVWj0Vg6W7lx40ZycrJGo1Gr1RkZGQ0NDVOmTEEIyWSykSNHpqenazSalJSUkJCQUaNGkYsolcohQ4Yw864Cmvdg6PcEoMdoHmbr1q1js9nk1xkIoSFDhqhUqhEjRiCEYmNjyT58Pt/Dw4PFYslksnPnzpGNH374IZfLlUqlS5cuRQgtWrSIbI+KiuJwODExMQRBzJgxY+XKlb2M1mAwDB48+PLlywRB7Ny5E8dxDw+P/fv3Ux0OHjxoWldSWFgYHBzM4/FkMllRURFBEGvXrrW0adnZ2VKplMPhjBs37sqVK0ql0tvbWy6XmwagUCikUimXy8VxPCwsrKSkhPqovLx87NixHA4nNDS0oqKCal+9erX1DaefBOg+2f2kPQMO7ILBwwzH8Z9++mn48OGMjGaW9WjT09MfPny4ffv2vguApNfrY2NjZTJZSkpKjwcxGAzDhg3Lzc0dNmyYpT70/zr2uQgaPnw4hmENDQ00+9+7dy8wMBDDsE73kzq1M1vQrdVqR4wYwePxcBx/6aWXfvzxR+v9+7Oc/NChQwKBAMOwZ5555t69e8wOvmPHDrLYPCgoiPyi1EEZjUY7rj0pKamysrLbw6b3MjMzvby8EhMTezNIampqcnKylZxiG5pnNfR70tHU1IQQUiqV9BdRKBQIoba2Nprtvdfc3Lxw4cJHjx49evTo3Xfftf6VPuO6LSc/ceIEs38U0zXK5XKhUMjg4DQxtUXz5s1DCIlEol9++YWRAc3qNlqdTpeenl5ZWdl3MTAiOzv71KlT3XazIV3Q7dcHaUWlUtFfpLa21mz6sNROR2lpaVlZGZ2ehYWFbDbbaDT2YC09M3To0H5OK6ZrdPS00j8cK9reo7+9zFwEdaomXr16NYZhYrGYz+ezWCypVBoUFMTn893d3Y8cOUItFRwczOFwnnnmma+++srsOAih06dPh4SEkJWCpmvs2k4VdCOrhc/5+fnPPfccl8sVi8Xr16+nvl2zxGg01tXVHTp0KDo62spNcvuWk5tdqdk1WlpppzVasWTJEoFA4OrqOn/+fKPROGPGDAzDJBLJgwcP/vu//9vd3Z28ndHpT7lixQoMwwoKCmbPnp2cnGx9FcDh9T5Rma0mFolEOTk5Op0uOzubxWLdvn27vb09JSXlxRdfJP46W/ntt9/a2tr27dvHYrH+/PPPruPU1dVxudzdu3e3tbXdunUL/XVWYqmdKugmLBQ+a7VaNze3o0ePtra2JiYmhoaGdrvhr732GkJoypQpZD21Ff1cTt7pbMXsSs2u0exKO63RytnKihUrFArFrVu3WCxWWVlZa2uru7v7d999R366dOlShUJh9pAQCoVHjhxRqVRbtmyxtFH0D8iBwLGi7T3628vA2YqlamIfHx8WizVx4kS9Xi8Sidhsdmho6KNHj6gF/fz8uFzu4sWL/f39z58/33WcgoICoVC4fPly8nsyakFL7V11Knyurq5ubm6eNWsWj8ebPn262Ye+OsnPz6+trQ0PDx89ejSZDWmySzl515UyvsZdu3b5+vo+88wzHh4ezc3NPB4vJibm2LFjCCG9Xq/X6319fS0dEhKJxN3dfd26db0PAwxkNrx53xKqmpj81UoljyWenp4qlQrDsE7jKBSKp59+umt/S+3d8vX15XK5J0+efPPNN/Pz8+l8++js7CwUCpOSkvbt25ebm7tgwYIerLer/i8nZ2SNzc3NixcvPnPmjFqtJk9wEEIJCQn/9V//1dzcfOHChbfeegv17pBwrKljHCvafsNAWiGriX/77beeLU4QRHV1tVgsbm1t7TTOZ599ZlrUbLpGs+3dwnF8y5Ytixcvjo+Pf/755w8cOGBTnD1Yo6Wh+rmcnKk1Hj58uLy8/Nq1a35+ftRoY8eOlUqlJ06c+J//+R/yfSK9OSQY3M997Ukr5urXCT1oVhN31dbWptVqd+3apdfrp06d2nWcV1555ebNm0eOHGlpacnPz6cWtNTeLY1Gk5OTU1ZWptVqS0tLrZ+tnDx5ct++fVqt9tGjR7t27VIqla+88gr9dZnV/+XkXddodqVW1kgQhEqlIstS29vbORwOjuMVFRWmNUQJCQkHDx708PBwdnZGvTgkwGOCkbs1naqJV61ahRASiURlZWXkVy3BwcG///67UCjEMCwpKamtre21117z9PTkcDhjxoy5ePGi2XEIgsjIyPD39xcIBGQlQmRkJNmzaztV0L1161ZLhc9arXb8+PHkhmMYFhgYSJV1d3XlypWAgAAOh8Pj8V588cWzZ89a2QP9XE5++PBhgUCAEHr22Wfv3btnaXvNrtHSSqk1ksXmnY6ThQsXEgRx9+7doUOH8vn8mJiYoKCgoKCgjo4OgiAePnzI4/GqqqosHRIrVqxACPn5+VF/a7PoH5ADgWNF23s2pAvGRxzIGhoaFixYoNPpCIIwGAwfffTRm2++2W9r5/P5N27c6LfV9ecajUbj3//+996P41iHmWNF23v0t/fJeoL57Nmz9+7dU6lUOp2uoqLiwoUL3t7emDn379/vuvj9+/fpdzar/8vJ+3qNxcXFra2tGzdunDVrVp+uCDiQJyutzJgxQygUkqfxr7322ksvvbR7926z6dbf37/r4v7+/vQ7dzJ//vzW1tZp06ZdvXq1D7bMbmvcu3evr68vhmHUjRtAgQk9GDv/AaDHGDzMejZHh01LWY8WJvQA4HHTszk6GJzZAyb0AMABdJ31otuHqshLgJ49GNWbmT1gQg9a6PcEoMesHGaWZr1A3T1U1eMHo3oTLUzoAYADsD7rhXX9P7MHTOgBgAOwadYLS/rtUSyY0AMAB2DTrBdmEf34KBZM6AGAA7A06wWdh6p6/2CUrWBCD1ro9wSgx6wfZl1nvSBoPFTVswejCBoze1iJFib0gAk9wEDRF4dZ383sARN6WAIXQeDxZ5eZPZ7kCT3gbAUMIIwfZvPnz//yyy9FIlFubu7o0aMZHBnRiFav13/66acxMTEBAQHMrppZOTk5gwYNioiIsN6N/l8H0goYQBzrMHOsaHsPLoIAAHYDaQUAwDBIKwAAhtlwb6WvQwEADHAM37IFT7iNGzdS/wXAOrgIAgAwDNIKAIBhkFYAAAyDtAIAYBikFQAAwyCtAAAYBmkFAMAwSCsAAIZBWgEAMAzSCgCAYZBWAAAMg7QCAGAYpBUAAMMgrQAAGAZpBQDAMEgrAACGQVoBADAM0goAgGGQVgAADIO0AgBgGKQVAADDIK0AABgGaQUAwDBIKwAAhrnYOwAwcGk0mkuXLpE/V1ZWIoTOnDlD/hoWFsbj8ewWGRjYYFZDYJHBYPD29tbpdC4uLuRxgmGYwWBgs9lKpdLFBf5NAubBRRCwyMXFZc6cOe3t7Wq1urm5ubm5Wa1Wt7e3z5kzB3IKsALSCrBm4cKFrq6upi08Hm/RokX2igc4BLgIAtYQBCEUCpVKJdXi5eVVX1+PYZgdowIDHJytAGswDIuLi2Oz2eSvLBZrwYIFkFOAdZBWQDfi4uKoOylsNnvBggX2jQcMfHARBLonkUju3r2LEBKLxffu3bN3OGCgg7MV0L34+Hgul8vhcBYvXmzvWIADgLMV0L3KysqRI0cihMrKygIDA+0dDhjoIK0AWkaMGIFhWFlZmb0DAQ7AtrQCXwEA8GSyKVHYXCsJZzdPptraWoSQn5+fAx0AGAYn48yw9XzC5rMV+Ds9yRzrAHCsaAcyW/ckfBMEAGAYpBUAAMMgrQAAGAZpBfSJiIiIPXv22DsKJun1+i1btqxYsQLHcQzDduzYQbaXlJT4+/uz2ew+fbBbIpFgf5kwYQLVXlFRERoayuVyQ0NDb968iRA6fvx4QUFB30VCC2ELW/uDx4y9DoCPPvqourra1qUYjNZgMERGRv70008EQcjlcolEIhAIGhsbyU+rq6sTEhKYWpdZZsc3Go2jRo1KTU3VaDR/+9vfQkJCyPbk5OQDBw4wuHZb9yScrQAHkJOTY98APvnkEx8fnxdeeIH8NS0tjcvlbtq0yb5RXbt27caNG+vXr3d1dd20adP169evXbuGEEpNTd24ceMff/xhr8AgrQDmZWVlcbnc1NTUdevWYRi2bNmyYcOG4TielpZGdkhMTMQwbOrUqTiOBwQEfPPNNwihqKgoDMNu37794MEDiUSC4zjZOTo6uqKiQiwWv//++wihmTNnrl69uj83p6OjIzMzMy4ujmoRCAQZGRl79+69c+dO1/5nzpwJCQnBcVwmk50+fRohZGk/IIS+/fZbqVTq7u4eHx+v0+lsCuz69esBAQHkS4U9PDz8/f2vX7+OEHJ1dY2MjNy3b1/PtpcBfXouBB4z9A+AhISElJQUgiCEQmFxcbHRaPziiy94PB7Vgc/nFxYWajSazMxMLperUCjI8W/dukUQxK+//srn88meer0eIWTHi6DLly8jhNRqNfmrXC7Py8sjCCI6Onr27NnE/38RpFQq+Xz+0aNH1Wp1ZmYmn8+vq6sjLOwHhULh6uqal5fX2Ng4ZsyYXbt2WYqBzEdcLjc4ODgnJ4ds3LZtm0wmo/oEBwdv376d/DkrKysgIICRzSfgIggMTBiGTZw4UaPRGAwGqtHPz8/V1XXp0qWenp7nzp2zX3TdqKqq4nA4bm5undrlcvn58+dLS0tNG4uKioRC4dy5c93c3MhN++GHH6hPO+2Hc+fOicXi119/3cPD44033rhw4YKlGPLz8xUKRW1t7cqVK2NjY81e4BiNRuqFW15eXnfv3iXsVA0IaQXYn4+PT2Njo72jsKitrY3D4XRt9/b23rVr15o1a0wb6+rqvL29qV+FQmFdXZ2lkZVK5c2bN8nvdzZs2KBSqSz1DAgIwHHc3d09ISEhMDCQzGVeXl4tLS1UH5VKJRQKyZ85HI7RaNRqtbS3kkmQVoCdEQRRU1MjEonsHYhFPB7P0l2Pd955RygUkveGSL6+vqav/q2trfX19bU0skAgCA4Opq4dioqK6MRjMBjI95bLZLKqqioyszQ0NNTU1AwdOpTso9PpnJycuFwunQEZB2kF2E1LS4tWq5XL5TqdbvLkyQghHMcvXryo1+tramqobk5OTk5OTuXl5RqNxi5xSiQSrVZrel5gau/evaYVOuHh4fX19UePHm1ubs7MzGxqagoPD7c08qRJkyoqKo4dO9ba2qrRaCydrdy4cSM5OVmj0ajV6oyMjIaGhilTpiCEZDLZyJEj09PTNRpNSkpKSEjIqFGjyEWUSuWQIUPs9sqBPr1zAx4zNA+AdevWsdlsatrDIUOGqFSqESNGIIRiY2PJPnw+38PDg8ViyWSyc+fOkY0ffvghl8uVSqVLly5FCC1atIhsj4qK4nA4MTExBEHMmDFj5cqVDEbbLYPBMHjw4MuXLxMEsXPnThzHPTw89u/fT3U4ePCgaV1JYWFhcHAwj8eTyWRFRUUEQaxdu9bSfsjOzpZKpRwOZ9y4cVeuXFEqld7e3nK53DQAhUIhlUq5XC6O42FhYSUlJdRH5eXlY8eO5XA4oaGhFRUVVPvq1atp7iU6bN2T8AQzsAGDBwCO4z/99NPw4cMZGc0sBqNNT09/+PDh9u3bGRnNCr1eHxsbK5PJUlJSejyIwWAYNmxYbm7usGHDGInKkZ5gHj58OIZhDQ0NNPvfu3cvMDAQw7BON6I6tTNbNq7VakeMGMHj8XAcf+mll3788UcrnXfs2EFWdpv6/vvv6azIIfYGs4xGo71DoCspKamystL6X58RmZmZXl5eiYmJvRkkNTU1OTmZqZzSE316LmRdU1MTQkipVNJfRKFQIITa2tpotvdec3PzwoULHz169OjRo3fffbfbWgC5XC4UCsmf9Xr9iRMnCgoK6KzIIfYGUwfAvHnzEEIikeiXX35hZECzmD1cdTpdenp6ZWUlg2P2hezs7FOnTjE7ps2Jok9Ht478H0mlUtFfhHxHWdf/YSy101FaWlpWVkanZ2FhIZvNNhqNVvqYphWbOMTeYPYA6GuOFe1AZuueZPIiqFMZ8urVqzEME4vFfD6fxWJJpdKgoCA+n+/u7n7kyBFqqeDgYA6H88wzz3z11Vdmx0EInT59OiQkhCwxNF1j13aqbBxZrZjOz89/7rnnuFyuWCxev3499bWcJUajsa6u7tChQ9HR0eTddTol5MnJyVT11+O0NwDoBlNJy2wZskgkysnJ0el02dnZLBbr9u3b7e3tKSkpL774IvHXv8+//fZbW1vbvn37WCzWn3/+2XWcuro6Lpe7e/futra2W7duob/+HbbUTpWNExYqprVarZub29GjR1tbWxMTE0NDQ7vd8Ndeew0hNGXKFLIQ2wq5XG66e/V6PfXRY7A3bD1g7Muxoh3IbN2TjJ2tWCpD9vHxYbFYEydO1Ov1IpGIzWaHhoY+evSIWtDPz4/L5S5evNjf3//8+fNdxykoKBAKhcuXLye/YKMWtNTeVaeK6erq6ubm5lmzZvF4vOnTp5t9WqyT/Pz82tra8PDw0aNHk///W0FdBK1fv77TR4/H3gDAOpvfvG8JVYZM/mqlBMgST09PlUqFYVincRQKxdNPP921v6X2bvn6+nK53JMnT7755pv5+fl0vuN0dnYWCoVJSUn79u3Lzc2lOQ/xJ5980oPwSAN2bzjWpC6OFe1jg7GzlZ6VIVMIgqiurhaLxV3HEQgEptXQpms0294tHMe3bNmyePFigUBQWlq6d+9em+LswRptNZD3Ri9OpfubY0U7kNl6UDGWVmiWIXfV1tam1Wp37dql1+unTp3adZxXXnnl5s2bR44caWlpyc/Ppxa01N4tjUaTk5NTVlam1WpLS0ut//t88uTJffv2abXaR48e7dq1S6lUvvLKK/TX1d7evnDhQvr9B/jeAIAWW5OWlU87lSGvWrUKISQSicrKysgvF4KDg3///XehUIhhWFJSUltb22uvvebp6cnhcMaMGXPx4kWz4xAEkZGR4e/vLxAIyHqHyMhIsmfXdqpsfOvWrZYqprVa7fjx48nNxzAsMDCQKh7v6sqVKwEBARwOh8fjvfjii2fPniXbzZaQk5XdnfbwhAkTCIJ4PPaGrQeMfTlWtAOZrXvySSzeb2xsXLNmzeeff85isTo6OtLS0n7//fdvv/3W3nHZh017w7EOAMeKdiBzpOJ9ezl79uy9e/dUKpVOp6uoqLhw4YK3tzdmzv379+0dbJ/rujeef/55ewcFHNuTmFZmzJghFAqHDh3K5/Nfe+21l156affu3WbP5fz9/e0dbJ/rujeSkpLsHVQ/cZQ5OqwrLy+fPHmy6aNnlkbov9k/+vQSCzxmGDwAejZHh01LWY/WsebosOTYsWNJSUkeHh7Uo2eWRujN7B82JwrbekNaebIxeAAMHTq0B2nFpqWsR5uWlrZkyRLyZ7lcfvjwYT8/v1WrVpEt9korV69edXZ2bm1tJQiisbERw7Bff/2126FEIhGVViyNYKldo9GIxeIbN25YGd/Wv/uTeBEE+kLXWSwQjTk6yEuATjN7WFqQwZk9HG6Ojt6P0K+zf9iUhGztDx4zlg4AS7NYEDTm6DA7s4fZBW2d2cPK4eqIc3RYYXq2YmmE3sz+Yev/+HC2AhhgfRaLbvX/zB6OOEdHz1gaoU9n/4C0Ahhg0ywWVvTbzB6OOEcHTZZG6M/ZPyCtAAbYNIuFJUQ/zuzhiHN00GRphP6c/QPSCmCAlVks6MzR0XVmD7MLMjizhyPO0UGTpRH6dfYPm+7E2NofPGasHABdZ7EgdTtHh9mZPSwtaNPMHlaidaw5OsyOQFq9erVYLEYI4Tg+fvz4mpoasyNYGpkapMd70qwn8Zkg0GN9cQD03cwe1qN1oDk6GJnlwxI6s3/AM0HA8dhlZg8HmqODkVk+LOmL2T/gbAXYgPEDYP78+V9++aVIJMrNzR09ejSDIyMa0er1+k8//TQmJiYgIIDZVTuKnJycQYMGRUREWO9m698d0gqwgWMdAI4V7UAGF0EAADuDtAIAYBikFQAAw2y+t9J3oQAABqw+vGULnlgbN26k/guAdXARBABgGKQVAADDIK0AABgGaQUAwDBIKwAAhkFaAQAwDNIKAIBhkFYAAAyDtAIAYBikFQAAwyCtAAAYBmkFAMAwSCsAAIZBWgEAMAzSCgCAYZBWAAAMg7QCAGAYpBUAAMMgrQAAGAZpBQDAMEgrAACGQVoBADAM0goAgGGQVgAADHOxdwBg4NJoNJcuXSJ/rqysRAidOXOG/DUsLIzH49ktMjCwwayGwCKDweDt7a3T6VxcXMjjBMMwg8HAZrOVSqWLC/ybBMyDiyBgkYuLy5w5c9rb29VqdXNzc3Nzs1qtbm9vnzNnDuQUYAWkFWDNwoULXV1dTVt4PN6iRYvsFQ9wCHARBKwhCEIoFCqVSqrFy8urvr4ewzA7RgUGODhbAdZgGBYXF8dms8lfWSzWggULIKcA6yCtgG7ExcVRd1LYbPaCBQvsGw8Y+OAiCHRPIpHcvXsXISQWi+/du2fvcMBAB2croHvx8fFcLpfD4SxevNjesQAHAGcroHuVlZUjR45ECJWVlQUGBto7HDDQQVoBtIwYMQLDsLKyMnsHAhwB8ZcNGzbYOxYAgKPasGEDlUxcOn2wceNGO0UFBrTa2lqEkK+vr/Vu5PHjKEeRY0U7kHXah1CCDWjpNqEAQIFvggAADIO0AgBgGKQVAADDIK0A+4iIiNizZ4+9o+iGXq/fsmVLVVXVjh07cBzHMGzHjh3kRyUlJf7+/mw2u0+f55ZIJNhfJkyYQDZWVFSEhoZyudzQ0NCbN292O0h5efnkyZO///5700azg5htPH78eEFBgU1hQ1oB9lFQULB8+XKmRtuwYcP9+/eZGo3U0dERFRU1ZcqUgICAxMTELVu2SCSSf/7znw8fPkQITZgw4fLly3FxcQcPHmR2vabCw8Opb21LSkoQQgRBREVFTZ06tampacyYMdHR0dZH+Oqrrw4fPnz9+nXTRrODWBp59uzZJSUlWVlZ9MOGtAIeBzk5OYyP+cknn/j4+LzwwgtUS1paGpfL3bRpE+Prou/atWs3btxYv369q6vrpk2brl+/fu3aNSv9Y2JiNm/e3OmlOWYHsTJyamrqxo0b//jjD5pBQloBdpCVlcXlclNTUxFC69atwzBs2bJlw4YNw3E8LS0NIZSYmIhh2NSpU3EcDwgI+Oabb8gFo6KiMAy7ffv2gwcPJBIJjuMIoSFt4vwAACAASURBVOjo6IqKCrFY/P777yOEZs6cuXr16l5G2NHRkZmZGRcXZ9ooEAgyMjL27t17586dTv3PnDkTEhKC47hMJjt9+jTZaHbTEELffvutVCp1d3ePj4/X6XQ2BXb9+vWAgADyRcIeHh7+/v6dzkR6PIiVkV1dXSMjI/ft20dzfEgrwA7i4+PnzZtH/rx161ahUBgbG/vHH3/s2bNn69atCKEdO3bw+fy1a9cqlcqkpKS4uDiyHu/rr78mlxo8ePB3331H/nz06FGEUHV19e7duxFCeXl5O3fu7GWEV65cefDgwahRozq1z5o1680330xKSjJtbGhoiIyMXLt2rUKh+Nvf/vb222/X19db2rTa2tp58+bt2LGjqqrqt99+++yzz6yEcenSJTc3N1dX11GjRpHb3tjYyOfzqQ5PPfVUY2OjrVtndhDrIz///PO5ubk0x4e0AgYKDMMmTpyo0WgMBgPZ4ufn5+rqunTpUk9Pz3PnzvVnMFVVVRwOx83NretHcrn8/PnzpaWlVEtRUZFQKJw7d66bmxsZ7Q8//GC6iOmmnTt3TiwWv/766x4eHm+88caFCxeshJGfn69QKGpra1euXEmmp04djEYj9ZKtHjM7SKdGLy+vu3fv0nyEENIKcAA+Pj49+De5N9ra2jgcjtmPvL29d+3atWbNGqqlrq7O29ub+lUoFNbV1VkaWalU3rx5k/xyZ8OGDSqVykoYAQEBOI67u7snJCQEBgaWlpZ6eXm1tLRQHVQqlVAotGHDEEIImR3E+sgcDsdoNGq1WjrjQ1oBAx1BEDU1NSKRqD9XyuPxrNz1eOedd4RCIXXHx9fX1/R1v7W1tVaedRAIBMHBwdT3O0VFRTRDMhgMrq6uMpmsqqqK/P+/oaGhpqZm6NChNEegmB3E+sg6nc7JyYnL5dIZH9IKGLhaWlq0Wq1cLtfpdJMnTyYbcRy/ePGiXq+vqakhW5ycnJycnMrLyzUaDVOrlkgkWq3W9F/vTvbu3UvV3YSHh9fX1x89erS5uTkzM7OpqSk8PNzSgpMmTaqoqDh27Fhra6tGo7FytnLjxo3k5GSNRqNWqzMyMhoaGqZMmSKTyUaOHJmenq7RaFJSUkJCQrreAOqW2UGsj6xUKocMGUL3NcamL0YwfbQZgB6geRStW7eOzWbzeLytW7euXbsWITRkyBCVSjVixAiEUGxsLEEQfD7fw8ODxWLJZLJz585Ry3744YdcLlcqlS5duhQhtGjRIoIgoqKiOBxOTEwMQRAzZsxYuXJlL6M1GAyDBw++fPky+evOnTtxHPfw8Ni/fz/V5+DBgwkJCeTPhYWFwcHBPB5PJpMVFRWRjZY2LTs7WyqVcjiccePGXblyRalUent7y+XyTjEoFAqpVMrlcnEcDwsLKykpIdvLy8vHjh3L4XBCQ0MrKirIRkuDrF69WiwWI4RwHB8/fnxNTY2VQcw2UuNY2aud9uT/vcYJHhIHvcfgUYTj+E8//TR8+PDeD2WJ9WjT09MfPny4ffv2vguApNfrY2NjZTJZSkqKfQexxGAwDBs2LDc3d9iwYWY7dNqT/XoRdO/evcDAQAzDaN74MbVmzRo2m01WOvQRszXOlhQWFr766qsCgcDFxcXd3X348OFUtUJXVOk3hmFOTk4eHh6TJ0/Ozs7u2sFsbXi3leNUh6CgoMuXL3da+61bt2bOnOnp6cnlcp9++mnye8pJkyZh5nz33XfUaEeOHOk01OjRozEMGzVqlF6vp7OXesloNPbDWixJSkqqrKz88ccf+3pFmZmZXl5eiYmJdh/EktTU1OTkZEs5xQxLpzF9RKFQIITa2tp6sOyCBQtSUlLMfvTRRx9VV1f3JrBjx44lJSV5eHgUFBR02/nw4cNsNnvbtm137txpb2+vra09fPhwRkaGlUXkcrlQKCQIQq1W//zzzzExMeRfy7SDRCIRCASNjY1kS3V1NXWObf1T0/G7ev7555csWVJfX9/S0nLixImdO3cSBLFy5cqWlpaOjo7PP//c09NTp9O1trbm5uaeOHGCHM3Z2fnll182Hae4uNjZ2TkoKMjKZjJ1FJFVLSKR6Jdffun9aJZ0G61Op0tPT6+srOy7GAa+7OzsU6dOWe/TaU/29y3bPpq5qve122ZrnM1qa2tLTExct27dmjVrAgMD2Wy2UCicP3/+e++9R2dFbm5uY8eOPXbs2D/+8Y/NmzeTE2WQrNeG96xyXK/XX7t2LSkpydvbm8/nR0ZGrlq1CiH0r3/9i8/nOzn97wHAYrF4PN7MmTMjIyPJlunTpxcXF1dUVFBDyeXyt956y6a199iRI0cIgrh///7o0aP7Z41msVis5OTkgIAAO8Zgd9HR0RERETYtQjetXLx4MTQ0lMfjubu7p6SkmK2htuL06dMhISFcLjc4OJhqNFvabKnemaLT6aKiovh8vpeXF/mUF1W7TZ7Gi8ViPp/PYrGkUmlQUBCfz3d3d+96Pk9T10rw0tLShoaG2NhYs/3p75m1a9cajUbTSycrteHdfmoJuR+or0Lpmzt37qBBg/bv30/+WlNTU1dX9+KLL9o6DngC0UorjY2NM2fOjIqKamhoKCsr0+l0ZmuoLamvr4+MjFy8eLFKpbp06RLVbra02WyjqRMnTqjV6oaGhvPnz5MlUlTtNkEQIpFo+/btKpXqyy+//PPPPwsLC5uamlasWEGWdfdA10rwqqoqhJBEIjHbn/6e8fLy8vHx6TSbl9nacJqfWnLo0CG5XP7yyy9nZWW1trbSXArH8djY2MOHD5O3UTIyMlasWGHTesETi1ZaOXv2LI/HS0xM5PF4YrH4008/tWkdBQUFQqFw+fLl5FdlXTt0rdq21IgQwnH86tWrhYWFzz333AcffNB1NB8fHxaLNXHiRL1eLxKJ2Gx2aGjoo0ePbIrZCoIgEENXc1qtlroMoXStDaf/qVnjxo27ffv2smXLDhw4IJVKr169SnPBZcuW1dfXnzx5sr29vaioiLo+AsA6WmnlwYMHvalxVCgUTz/9dI8X72T69OmrVq167733JBLJmTNnmBqWPvI85fbt270cR61WP3r0qOt1e9facPqfWsLhcObOnXvx4sVp06bR/7IgODg4LCxs//792dnZMTExzs7OdJb6+OOPzX7HNAB9/PHHDhTtQPbxxx+bHgO00opQKCS/wekZgUBgWtrcSxiGJScn379//9133yUfhO9nEyZMIOuOTBs7Ojo+/PBDm8b54osvXFxcpk2b1vWjTrXhNn3aSUtLi+n9qVmzZlVXV9MP8r333isqKpLL5fHx8TQXcaCiSigBZUqnWcZopZVXX321qakpLS2toaFBr9eTKaZrDbUlr7zyys2bN48cOdLS0pKfn0/z6LTk888/Lyoq6ujoeOGFFzAMQ31Tu20Fl8vds2fPgQMHkpOTq6qq9Hp9ZWVlWloaVcphac8QBNHe3o4QqqmpyczMXL9+fWpqqp+fn9m1mNaG2/opQRAqlYosQkUIHTp06OzZs1qt9u7du3v27Jk0aRL9jZ0zZ45AIJg4ceKgQYPoLwWedKb5xkrm/s9//jN27FgejycSicjCB7M11JZkZGT4+/sLBAKyHiEyMpKwUNpsqd75gw8+IL8EnTRp0uDBg11cXKRSKVUlTdZuk1skEonKysrIp6SCg4N///13oVCIYVhSUpKVCC3VOFuqBL9w4cK0adOeeuopJycngUDw0ksv5eXlkR913TO7d+8WCoVsNpu8k8Ln80NDQ8nvUEnWa8O7rRwnO3T6yy5cuJAgCJ1OFxsbKxaLXVxcfH19Fy1a9PDhQ2qQefPmka/YCAwMPH36tNlg1q5dW1VVRRBESkoK+Y6fkJAQnU5ndjc61r//jhXtQAbF+6APOdZR5FjRDmR9WLx///59s7dzGH93cY8N/AgBeAwwmVb8/f3NniD5+/szuJbeGPgRggHlcZ3QQ61Wjxo1CsdxgUAQERFBfa2Zl5c3YsQIHo83cuTIU6dOkY0woQd4fPR4jg6mJvd4jCf00Ol0YWFhCoWisrLyqaeeIu94tra2xsTE/P3vf1cqlUuWLImOjia/A4EJPcDjo8fPeTE1ucdjPKGHl5dXZmamm5ubQCCIj4//+eefOzo6ampqWltb58yZw+fz586d29LSQmVnmNADDFxmZ73odo4Os5N7WHr2iqnJPR7vCT1Mtba2enp6Ojs7BwYGDh069N///rdWqz1x4sQzzzwTGBhI9rF1Qg94OxxgkpWjSKlU8vn8o0ePqtXqzMxMPp9fV1dHfoQQunXrFkEQv/76K5/PJwiCLAKiXnbB5/MLCws1Gk1mZiaXy1UoFGaX6rpgj6MlX1ujVqupFrlcTtYQREdHz549mzB5N4WVTRMKhcXFxUaj8YsvvuDxeARBKBQKV1fXvLy8xsbGMWPG7Nq1y0qEZD4in9HNyckhCGLbtm0ymYzqEBwcvH379m63VCQSWXrjR2Ji4ooVK8ifL1265OLighBycXExfSMfQRBZWVkBAQGWxrfzixHAE6vbWS+s6+fJPZ6QCT2qq6u///57svReqVRGRkbm5ua2tLRkZ2dHR0eTd5FIMKEHGIhsmvXCiv6Z3OPxntCD1NzcvHjx4uPHjwsEAoRQbm6un59fREQEn89/++23vb29Tecbgwk9wEBk06wXlhD9NbnH4z2hB0Koubk5Pj5+586d1Ksku25vW1sb9TNM6AEGIiuzXtCZo6Pr5B5mn71i6gGxx3tCD7VanZCQsHnzZtPX044fP/7GjRtFRUVtbW0nT54sLy8PCwujPoUJPYDdWD+KzM56QdCYo8Ps5B6WnkqjP7nHEzuhB/XSP0pxcTFBEJ999llQUBCXy3322WcPHDjQaRyY0APYRx8dRX00uQdM6EHTgJ7QA4Ae6//JPWBCD4qtE3q49EUQADBo/vz5ra2t06ZNy83N7c8X8Ts7O3/99deffvqpj49Pn758n5GXBPfdm4ZzcnImTpxo08v3Ia2Age7IkSM9njihl8gJPeyy6oGj28eOuoKLIAAAwyCtAAAYBmkFAMA002+e7R0LAMBRma9bAcAKKGsC9MFFEACAYZBWAAAMg7QCAGAYpBUAAMMgrQAAGAZpBQDAMEgrAACGQVoBADAM0goAgGGQVgAADIO0AgBgGKQVAADDIK0AABgGaQUAwDBIKwAAhkFaAQAwDNIKAIBhkFYAAAyDtAIAYBikFQAAwyCtAAAYBmkFAMAwSCsAAIZBWgEAMMzF3gGAgUuj0Vy6dIn8ubKyEiF05swZ8tewsDAej2e3yMDABrMaAosMBoO3t7dOp3NxcSGPEwzDDAYDm81WKpUuLvBvEjAPLoKARS4uLnPmzGlvb1er1c3Nzc3NzWq1ur29fc6cOZBTgBWQVoA1CxcudHV1NW3h8XiLFi2yVzzAIcBFELCGIAihUKhUKqkWLy+v+vp6DMPsGBUY4OBsBViDYVhcXBybzSZ/ZbFYCxYsgJwCrIO0AroRFxdH3Ulhs9kLFiywbzxg4IOLINA9iURy9+5dhJBYLL537569wwEDHZytgO7Fx8dzuVwOh7N48WJ7xwIcAJytgO5VVlaOHDkSIVRWVhYYGGjvcMBAB2kF0DJixAgMw8rKyuwdCHAExF82bNhg71gAAI5qw4YNVDJx6fTBxo0b7RQVGNBqa2sRQr6+vta7kcePoxxFjhXtQNZpH0IJNqCl24QCAAW+CQIAMAzSCgCAYZBWAAAMg7QC7CMiImLPnj32jqIber1+y5YtVVVVO3bswHEcw7AdO3aQH5WUlPj7+7PZ7D59nlsikWB/mTBhAtlYUVERGhrK5XJDQ0Nv3rzZ7SDl5eWTJ0/+/vvvqRa1Wj1q1CgcxwUCQURExO3bt8n2vLy8ESNG8Hi8kSNHnjp1imw8fvx4QUGBTWFDWgH2UVBQsHz5cqZG27Bhw/3795kajdTR0REVFTVlypSAgIDExMQtW7ZIJJJ//vOfDx8+RAhNmDDh8uXLcXFxBw8eZHa9psLDw6lvbUtKShBCBEFERUVNnTq1qalpzJgx0dHR1kf46quvDh8+fP36ddNGnU4XFhamUCgqKyufeuqpefPmIYRaW1tjYmL+/ve/K5XKJUuWREdHazQahNDs2bNLSkqysrLohw1pBTwOcnJyGB/zk08+8fHxeeGFF6iWtLQ0Lpe7adMmxtdF37Vr127cuLF+/XpXV9dNmzZdv3792rVrVvrHxMRs3ry500tzvLy8MjMz3dzcBAJBfHz8zz//3NHRUVNT09raOmfOHD6fP3fu3JaWFipTp6ambty48Y8//qAZJKQVYAdZWVlcLjc1NRUhtG7dOgzDli1bNmzYMBzH09LSEEKJiYkYhk2dOhXH8YCAgG+++YZcMCoqCsOw27dvP3jwQCKR4DiOEIqOjq6oqBCLxe+//z5CaObMmatXr+5lhB0dHZmZmXFxcaaNAoEgIyNj7969d+7c6dT/zJkzISEhOI7LZLLTp0+TjWY3DSH07bffSqVSd3f3+Ph4nU5nU2DXr18PCAggXyTs4eHh7+/f6UzEVq2trZ6ens7OzoGBgUOHDv33v/+t1WpPnDjxzDPPUA9quLq6RkZG7tu3j+6gplW2pnVyAPQA/aMoISEhJSWF/FkoFBYXFxuNxi+++ILH45GNfD6/sLBQo9FkZmZyuVyFQkG2I4Ru3bpFEMSvv/7K5/MJgtDr9Qih6upqBqO9fPkyQkitVlMtcrk8Ly+PIIjo6OjZs2cTBFFdXZ2QkEAQhFKp5PP5R48eVavVmZmZfD6/rq7O0qYpFApXV9e8vLzGxsYxY8bs2rXLSoRkPuJyucHBwTk5OQRBbNu2TSaTUR2Cg4O3b9/e7ZaKRKKCggKzHyUmJq5YsYL8+dKlS+RLMFxcXM6dO2faLSsrKyAgwNL4nfYknK2AgQLDsIkTJ2o0GoPBQLb4+fm5urouXbrU09Pz3Llz/RlMVVUVh8Nxc3Pr+pFcLj9//nxpaSnVUlRUJBQK586d6+bmRkb7ww8/mC5iumnnzp0Ti8Wvv/66h4fHG2+8ceHCBSth5OfnKxSK2tralStXxsbGdr0MMRqN1Eu2eqC6uvr777//+OOPEUJKpTIyMjI3N7elpSU7Ozs6Opq8i0Ty8vK6e/cuQe8RQkgrwAH4+Pg0Njb25xrb2to4HI7Zj7y9vXft2rVmzRqqpa6uztvbm/pVKBTW1dVZGlmpVN68eZP8cmfDhg0qlcpKGAEBATiOu7u7JyQkBAYGlpaWenl5tbS0UB1UKpVQKLRhw0w0NzcvXrz4+PHjAoEAIZSbm+vn5xcREcHn899++21vb+/c3FyqM4fDMRqNWq2WzsiQVsBARxBETU2NSCTqz5XyeDwrdz3eeecdoVBI3fHx9fU1fd1vbW2tlWcdBAJBcHAwdb1QVFREMySDweDq6iqTyaqqqsjM0tDQUFNTM3ToUJojmGpubo6Pj9+5c+ewYcPIlq7b29bWRv2s0+mcnJy4XC6dwSGtgIGrpaVFq9XK5XKdTjd58mSyEcfxixcv6vX6mpoassXJycnJyam8vJz8QpQREolEq9Wanhd0snfvXqruJjw8vL6+/ujRo83NzZmZmU1NTeHh4ZYWnDRpUkVFxbFjx1pbWzUajZWzlRs3biQnJ2s0GrVanZGR0dDQMGXKFJlMNnLkyPT0dI1Gk5KSEhISMmrUKFu3Tq1WJyQkbN68mcopCKHx48ffuHGjqKiora3t5MmT5eXlYWFh1KdKpXLIkCF0X2Ns6aYLAD1A8yhat24dm83m8Xhbt25du3YtQmjIkCEqlWrEiBEIodjYWIIg+Hy+h4cHi8WSyWSmtw8//PBDLpcrlUqXLl2KEFq0aBFBEFFRURwOJyYmhiCIGTNmrFy5spfRGgyGwYMHX758mfx1586dOI57eHjs37+f6nPw4EHyli1BEIWFhcHBwTweTyaTFRUVkY2WNi07O1sqlXI4nHHjxl25ckWpVHp7e8vl8k4xKBQKqVTK5XJxHA8LCyspKSHby8vLx44dy+FwQkNDKyoqyEZLg6xevVosFiOEcBwfP358TU0NQRD79+/vlAeKi4sJgvjss8+CgoK4XO6zzz574MCBTuNY2aud9uT/vcYJHhIHvcfgUYTj+E8//TR8+PDeD2WJ9WjT09MfPny4ffv2vguApNfrY2NjZTJZSkqKfQexxGAwDBs2LDc31/TsxlSnPdmvF0H37t0LDAzEMIzmjR9Ta9asYbPZZKUD4yzVMltRWFj46quvCgQCFxcXd3f34cOHU9UKXVGl3xiGOTk5eXh4TJ48OTs7u2sHs7Xh3VaOUx2CgoLIb0ZN3bp1a+bMmZ6enlwu9+mnn/76668RQpMmTcLM+e6776jRjhw50mmo0aNHYxg2atQo8jvdvmY0GvthLZYkJSVVVlb++OOPfb2izMxMLy+vxMREuw9iSWpqanJysqWcYgadE0IGKRQKhFBbW1sPll2wYAFV6dDJRx991IOyBYpSqVy6dKlarX748OE777wzbtw46/0PHz7MZrO3bdt2586d9vb22traw4cPZ2RkWFlELpcLhUKCINRq9c8//xwTE0P+tUw7SCQSgUDQ2NhItlBlEd1+ajp+V88///ySJUvq6+tbWlpOnDixc+dOgiBWrlzZ0tLS0dHx+eefe3p66nS61tbW3NzcEydOkKM5Ozu//PLLpuMUFxc7OzsHBQVZ2UymjiKynFwkEv3yyy+9H82SbqPV6XTp6emVlZV9F8PAl52dferUKet97Fy30kczV/WydttsLbOlzm1tbYmJievWrVuzZk1gYCCbzRYKhfPnz3/vvfforMvNzW3s2LHHjh37xz/+sXnzZnKiDJL12vCeVY7r9fpr164lJSV5e3vz+fzIyMhVq1YhhP71r3/x+Xwnp/89AFgsFo/HmzlzZmRkJNkyffr04uLiiooKaii5XP7WW2/ZtPYeO3LkCEEQ9+/fHz16dP+s0SwWi5WcnBwQEGDHGOwuOjo6IiLCpkXoppWLFy+GhobyeDx3d/eUlBSzNdRWnD59OiQkhCwWpBrNljZbqnem6HS6qKgoPp/v5eVFPuVF1W6Tp/FisZjP57NYLKlUGhQUxOfz3d3du57PW0LVMpO/dq0ELy0tbWhoiI2NNbs4/T2zdu1ao9FoeulkpTa8208tIfcD9VUofXPnzh00aBB1b6+mpqauru7FF1+0dRzwBKKVVhobG2fOnBkVFdXQ0FBWVqbT6cjrc4TQ4MGDv/vuO+uL19fXR0ZGLl68WKVSXbp0iWrfunWrUCgkawf37NmzdetWS42mTpw4oVarGxoazp8/T5ZIHT16FP1Vuy0SibZv365Sqb788ss///yzsLCwqalpxYoVu3fvprlHiouL33nnHerXvLy8nTt3mnaoqqpCCEkkErOL098zXl5ePj4+nWbzmjVr1ptvvpmUlGR2EeufWnLo0CG5XP7yyy9nZWW1trbSXArH8djY2MOHD5O3UTIyMlasWGHTesETi1ZaOXv2LI/HS0xM5PF4YrH4008/tWkdBQUFQqFw+fLl5FdlXTt0rdq21IgQwnH86tWrhYWFzz333AcffNB1NB8fHxaLNXHiRL1eLxKJ2Gx2aGjoo0eP6IRqWstsCUEQiKGrOa1WS12GULrWhtP/1Kxx48bdvn172bJlBw4ckEqlV69epbngsmXL6uvrT5482d7eXlRURF0fAWAdrbTy4MGD3tQ4KhSKp59+useLdzJ9+vRVq1a99957EonkzJkzTA2LutQyW0Kep9D5tsg6tVr96NGjrtftXWvD6X9qCYfDmTt37sWLF6dNm0b/y4Lg4OCwsLD9+/dnZ2fHxMRQF4bWffzxx2a/YxqAPv74YweKdiDr9C8xrbQiFArJb3B6RiAQmJY29xKGYcnJyffv33/33XfJB+EZ0bWW2ZIJEyaQdUemjR0dHR9++KFNa/ziiy9cXFymTZvW9aNOteE2fdpJS0uL6f2pWbNmVVdX0w/yvffeKyoqksvl8fHxNBdxoKJKKAFlSqdZxmillVdffbWpqSktLa2hoUGv15MppmsNtSWvvPLKzZs3jxw50tLSkp+fT/PotOTzzz8vKirq6Oh44YUXMAxDTNRum61ltoTL5e7Zs+fAgQPJyclVVVV6vb6ysjItLY0q5bC0ZwiCaG9vRwjV1NRkZmauX78+NTXVz8/P7FpMa8Nt/ZQgCJVKRRahIoQOHTp09uxZrVZ79+7dPXv2TJo0qdttpMyZM0cgEEycOHHQoEH0lwJPOtN8YyVz/+c//xk7diyPxxOJRGThg9kaaksyMjL8/f0FAgFZjxAZGUlYKG22VO/8wQcfkF+CTpo0afDgwS4uLlKplKqSJmu3yS0SiURlZWXk81fBwcG///67UCjEMCwpKclSeJZqmQnLleAXLlyYNm3aU0895eTkJBAIXnrpJfJlHGb3zO7du4VCIZvNJu+k8Pn80NBQ8jtUkvXa8G4rx8kOnTZh4cKFBEHodLrY2FixWOzi4uLr67to0aKHDx9Sg8ybN4/P5yOEAgMDT58+bTaYtWvXVlVVEQSRkpJCvj0oJCREp9OZ3ZOO9e+/Y0U7kEHxPuhDjnUUOVa0A1kfFu/fv3/f7O0cxt9d3GMDP0IAHgNMphV/f3+zJ0j+/v4MrqU3Bn6EYECBCT0QTOgBHic9nqODqck9YEIPmNADPG56/JwXU5N7wIQeMKEHcABmZ73odo4Os5N7WHr2iqnJPWBCD5jQAwwUVo4iK7NeoO7m6DA7uUfXpbou2ONoYUIPmNADOIBuZ72wrp8n94AJPWBCD+AAbJr1wor+mdwDJvSACT2AA7Bp1gtLiP6a3AMm9EAwoQcY+KzMekFnjo6uk3uYffaKqck9YEIPmNADDBTWjyKzs14QNOboMDu5h6Wn0uhP7gETelBgQg8wcPXRUdRHk3vA1xJ57QAAIABJREFUhB40DegJPQDosf6f3AMm9KDYOqGHS18EAQCD5s+f39raOm3atNzc3P58Eb+zs/PXX3/96aef+vj49OnL9xl5SXDfvWk4Jydn4sSJNr18H9IKGOiOHDlCf+IEZpETethl1QNHt48ddQUXQQAAhkFaAQAwDNIKAIBppt882zsWAICjMl+3AoAVUNYE6IOLIAAAwyCtAAAYBmkFAMAwSCsAAIZBWgEAMAzSCgCAYZBWAAAMg7QCAGAYpBUAAMMgrQAAGAZpBQDAMEgrAACGQVoBADAM0goAgGGQVgAADIO0AgBgGKQVAADDIK0AABgGaQUAwDBIKwAAhkFaAQAwDNIKAIBhkFYAAAyDtAIAYJiLvQMAA5dGo7l06RL5c2VlJULozJkz5K9hYWE8Hs9ukYGBDWY1BBYZDAZvb2+dTufi4kIeJxiGGQwGNputVCpdXODfJGAeXAQBi1xcXObMmdPe3q5Wq5ubm5ubm9VqdXt7+5w5cyCnACsgrQBrFi5c6OrqatrC4/EWLVpkr3iAQ4CLIGANQRBCoVCpVFItXl5e9fX1GIbZMSowwMHZCrAGw7C4uDg2m03+ymKxFixYADkFWAdpBXQjLi6OupPCZrMXLFhg33jAwAcXQaB7Eonk7t27CCGxWHzv3j17hwMGOjhbAd2Lj4/ncrkcDmfx4sX2jgU4ADhbAd2rrKwcOXIkQqisrCwwMNDe4YCBDtIKoGXEiBEYhpWVldk7EOAIiL9s2LDB3rEAABzVhg0bqGTi0umDjRs32ikqMKDV1tYihHx9fa13I48fRzmKHCvagazTPoQSbEBLtwkFAAp8EwQAYBikFQAAwyCtAAAYBmkF2EdERMSePXvsHUU39Hr9li1bqqqqduzYgeM4hmE7duwgPyopKfH392ez2X36PLdEIsH+MmHCBLKxoqIiNDSUy+WGhobevHmz20HKy8snT578/fffUy1qtXrUqFE4jgsEgoiIiNu3b5PteXl5I0aM4PF4I0eOPHXqFNl4/PjxgoICm8KGtALso6CgYPny5UyNtmHDhvv37zM1GqmjoyMqKmrKlCkBAQGJiYlbtmyRSCT//Oc/Hz58iBCaMGHC5cuX4+LiDh48yOx6TYWHh1Pf2paUlCCECIKIioqaOnVqU1PTmDFjoqOjrY/w1VdfHT58+Pr166aNOp0uLCxMoVBUVlY+9dRT8+bNQwi1trbGxMT8/e9/VyqVS5YsiY6O1mg0CKHZs2eXlJRkZWXRDxvSCngc5OTkMD7mJ5984uPj88ILL1AtaWlpXC5306ZNjK+LvmvXrt24cWP9+vWurq6bNm26fv36tWvXrPSPiYnZvHlzp5fmeHl5ZWZmurm5CQSC+Pj4n3/+uaOjo6amprW1dc6cOXw+f+7cuS0tLVSmTk1N3bhx4x9//EEzSEgrwA6ysrK4XG5qaipCaN26dRiGLVu2bNiwYTiOp6WlIYQSExMxDJs6dSqO4wEBAd988w25YFRUFIZht2/ffvDggUQiwXEcIRQdHV1RUSEWi99//32E0MyZM1evXt3LCDs6OjIzM+Pi4kwbBQJBRkbG3r1779y506n/mTNnQkJCcByXyWSnT58mG81uGkLo22+/lUql7u7u8fHxOp3OpsCuX78eEBBAvkjYw8PD39+/05mIrVpbWz09PZ2dnQMDA4cOHfrvf/9bq9WeOHHimWeeoR7UcHV1jYyM3LdvH91BTatsTevkAOgB+kdRQkJCSkoK+bNQKCwuLjYajV988QWPxyMb+Xx+YWGhRqPJzMzkcrkKhYJsRwjdunWLIIhff/2Vz+cTBKHX6xFC1dXVDEZ7+fJlhJBaraZa5HJ5Xl4eQRDR0dGzZ88mCKK6ujohIYEgCKVSyefzjx49qlarMzMz+Xx+XV2dpU1TKBSurq55eXmNjY1jxozZtWuXlQjJfMTlcoODg3NycgiC2LZtm0wmozoEBwdv37692y0ViUQFBQVmP0pMTFyxYgX586VLl8iXYLi4uJw7d860W1ZWVkBAgKXxO+1JOFsBAwWGYRMnTtRoNAaDgWzx8/NzdXVdunSpp6fnuXPn+jOYqqoqDofj5ubW9SO5XH7+/PnS0lKqpaioSCgUzp07183NjYz2hx9+MF3EdNPOnTsnFotff/11Dw+PN95448KFC1bCyM/PVygUtbW1K1eujI2N7XoZYjQaqZds9UB1dfX333//8ccfI4SUSmVkZGRubm5LS0t2dnZ0dDR5F4nk5eV19+5dgt4jhJBWgAPw8fFpbGzszzW2tbVxOByzH3l7e+/atWvNmjVUS11dnbe3N/WrUCisq6uzNLJSqbx58yb55c6GDRtUKpWVMAICAnAcd3d3T0hICAwMLC0t9fLyamlpoTqoVCqhUGjDhplobm5evHjx8ePHBQIBQig3N9fPzy8iIoLP57/99tve3t65ublUZw6HYzQatVotnZEhrYCBjiCImpoakUjUnyvl8XhW7nq88847QqGQuuPj6+tr+rrf2tpaK886CASC4OBg6nqhqKiIZkgGg8HV1VUmk1VVVZGZpaGhoaamZujQoTRHMNXc3BwfH79z585hw4aRLV23t62tjfpZp9M5OTlxuVw6g0NaAQNXS0uLVquVy+U6nW7y5MlkI47jFy9e1Ov1NTU1ZIuTk5OTk1N5eTn5hSgjJBKJVqs1PS/oZO/evVTdTXh4eH19/dGjR5ubmzMzM5uamsLDwy0tOGnSpIqKimPHjrW2tmo0GitnKzdu3EhOTtZoNGq1OiMjo6GhYcqUKTKZbOTIkenp6RqNJiUlJSQkZNSoUbZunVqtTkhI2Lx5M5VTEELjx4+/ceNGUVFRW1vbyZMny8vLw8LCqE+VSuWQIUPovsbY0k0XAHqA5lG0bt06NpvN4/G2bt26du1ahNCQIUNUKtWIESMQQrGxsQRB8Pl8Dw8PFoslk8lMbx9++OGHXC5XKpUuXboUIbRo0SKCIKKiojgcTkxMDEEQM2bMWLlyZS+jNRgMgwcPvnz5Mvnrzp07cRz38PDYv38/1efgwYPkLVuCIAoLC4ODg3k8nkwmKyoqIhstbVp2drZUKuVwOOPGjbty5YpSqfT29pbL5Z1iUCgUUqmUy+XiOB4WFlZSUkK2l5eXjx07lsPhhIaGVlRUkI2WBlm9erVYLEYI4Tg+fvz4mpoagiD279/fKQ8UFxcTBPHZZ58FBQVxudxnn332wIEDncaxslc77cn/e40TPCQOeo/BowjH8Z9++mn48OG9H8oS69Gmp6c/fPhw+/btfRcASa/Xx8bGymSylJQU+w5iicFgGDZsWG5urunZjalOe7JfL4Lu3bsXGBiIYRjNGz+m1qxZw2azyUoHxmm1WrJmGcfxl1566ccff+x2kcLCwldffVUgELi4uLi7uw8fPpyqVuiKKv3GMMzJycnDw2Py5MnZ2dldO5itDe+2cpzqEBQURH4zaurWrVszZ8709PTkcrlPP/30119/jRCaNGkSZs53331HjXbkyJFOQ40ePRrDsFGjRpHf6fY1o9HYD2uxJCkpqbKyks7B0EuZmZleXl6JiYl2H8SS1NTU5ORkSznFDDonhAxSKBQIoba2th4su2DBAqrSoZOPPvqoB2ULlObm5oULFz569OjRo0fvvvuule/nSYcPH2az2du2bbtz5057e3ttbe3hw4czMjKsLCKXy4VCIUEQarX6559/jomJIf9aph0kEolAIGhsbCRbqLKIbj81Hb+r559/fsmSJfX19S0tLSdOnNi5cydBECtXrmxpaeno6Pj88889PT11Ol1ra2tubu6JEyfI0ZydnV9++WXTcYqLi52dnYOCgqxsJlNHEVlOLhKJfvnll96PZkm30ep0uvT09MrKyr6LYeDLzs4+deqU9T52rlvpo5mrelm7jeP4wYMHBw0aNGjQoKioKPL601Lntra2xMTEdevWrVmzJjAwkM1mC4XC+fPnv/fee3TW5ebmNnbs2GPHjv3jH//YvHkzOVEGyXpteM8qx/V6/bVr15KSkry9vfl8fmRk5KpVqxBC//rXv/h8vpPT/x4ALBaLx+PNnDkzMjKSbJk+fXpxcXFFRQU1lFwuf+utt2xae48dOXKEIIj79++PHj26f9ZoFovFSk5ODggIsGMMdhcdHR0REWHTInTTysWLF0NDQ3k8nru7e0pKitkaaitOnz4dEhJCFgtSjWZLmy3VO1N0Ol1UVBSfz/fy8iKf8qJqt8nTeLFYzOfzWSyWVCoNCgri8/nu7u5dz+e7MhqNdXV1hw4dio6OptJf10rw0tLShoaG2NhYs4PQ3zNr1641Go2ml05WasO7/dQScj9QX4XSN3fu3EGDBlH39mpqaurq6l588UVbxwFPIFpppbGxcebMmVFRUQ0NDWVlZTqdjrw+RwgNHjz4u+++s754fX19ZGTk4sWLVSrVpUuXqPatW7cKhUKydnDPnj1bt2611GjqxIkTarW6oaHh/PnzZInU0aNH0V+12yKRaPv27SqV6ssvv/zzzz8LCwubmppWrFixe/fubjdz+vTpvr6+tbW127Ztoxrz8vJ27txp2q2qqgohJJFIzA5Cf894eXn5+Ph0ms1r1qxZb775ZlJSktlFrH9qyaFDh+Ry+csvv5yVldXa2kpzKRzHY2NjDx8+TN5GycjIWLFihU3rBU8sWmnl7NmzPB4vMTGRx+OJxeJPP/3UpnUUFBQIhcLly5eTX5V17dC1attSI0IIx/GrV68WFhY+99xzH3zwQdfRfHx8WCzWxIkT9Xq9SCRis9mhoaGPHj3qNs78/Pza2trw8PDRo0c3NTVZ6kZeHzFyNafVaqnLEErX2nD6n5o1bty427dvL1u27MCBA1Kp9OrVqzQXXLZsWX19/cmTJ9v/X3t3H9TEnT4A/LshJiFZ1CAYkKBAWzxExGnrG+2MSm0PWtvjpgIiigpa7SlXpT1fEI/KnafOTWWOKEbrW8tpsXXqCCpCpIevtdazc3daBrVwCggYQQghhCSwvz/WXy4lm80Lm+yGPp8/HLPZl2c3ycPu5vnm6etTqVTm6yMA6DmUVh49ejSUGseWlpbx48e7vPggb7755rp161avXh0WFnbhwgWmVosQ8vHxkclkmzZtEggElmXLg5DnKeZfvnGZRqPp6uqyvm63rg13/FlbhELhokWLrl69mpCQ4PiXBTExMXFxcQcPHiwtLU1LS/Px8XFkqW3btlF+x8RB27Zt86JouYwcVWTmUFqRyWTkNziukUqllqXNQ4RhWG5ublNT04oVK8iB8IyjuV+LEHr11VfJuiPLif39/Vu3bnVqK0ePHuXz+QkJCdZPDaoNd+rZQbRareX9qd/85jeNjY2OB7l69WqVSqVQKDIzMx1cxIuKKqEElCmDuow5lFbeeOONp0+fFhQUPHnyxGg0kinGuobalrlz5969e7ekpESr1Z49e9bBd6ctn376qUql6u/vnzZtGoZhiIna7dOnTx84cECv13d1dRUVFanV6rlz59qaWSQS7d2799ChQ7m5uQ0NDUajsb6+vqCgwFzKYevIEATR19eHEGpublYqlZs3b87LywsODqbcimVtuLPPEgTR2dlJFqEihD777LPq6mq9Xv/gwYO9e/fOmTPH5oGwkpycLJVKZ8+ePXLkSMeXAr90lvmGJnP/4x//ePnll8VicUhICFn4QFlDbcu+ffvkcrlUKiXrEZKSkggbpc226p0/+ugj8kvQOXPmjBs3js/nR0ZGmqukydptco9CQkJu375Njr+KiYn5z3/+I5PJMAzbtGmTrfBu3rwZHh4uFArFYvH06dOrq6vNT9mqBL906VJCQsLo0aN5PJ5UKn3llVfIH+OgPDJ79uyRyWQCgYC8kyKRSGbOnEl+h0qirw23WzlOzjDolV22bBlBEAaDIT09PTQ0lM/nBwUFLV++vKOjw7ySxYsXSyQShFBERERlZSVlMBs2bGhoaCAIYsuWLeSvB02dOtVgMFAeSe/6++9d0XIZFO8DN/Kud5F3Rctlbizeb2pqorydw/hvF7uM+xECMAwwmVbkcjnlCZJcLmdwK0PB/QiB1yGbfmRnZ7PS8QOx0a/DLvi9FcBRLvfocEdzD1vMTT8UCgUrHT9Y6ddhF6QVwFEuj/NyR3MPWwY1/fB8xw9W+nXYBWkFeA5l1wu7PToom3vYGnvljuYetlg3/aAft2W9+zQj4IbS9MOSW/p12GXrKyIAXEDzLqLpeoHs9eigbO5hvZT1gi5H64hBTT9oOn7Q7D5lMxOnmn4Q7u/XYRc09ADssNv1gh6LzT1ssdX0g3LcFv3uYz8fAedU0w8aburXYRekFeAhTnW9oOH55h622Gr6QTluy31NP2xxX78OuyCtAA9xquuFLQQbzT1soWn6YT1uyzNNP8zc2q/DLkgrwENoul440qPDurkH5dgrdzT3sIW+6cegcVtuavpBye39OuyyddMFABfQv4sou14QDvTooGzuYWtUmuPNPYb4nrds+mF33Bbl7tsaAUc43PTDM/067IIxQcCN3PQuclNzj6FH611NP2yx26/DLjYbegDgMnabe9jiXU0/bHG6X4c9kFYA1y1ZsqSnpychIcHxn8v0GB8fny+//LK6upr8hWP3yc7OLi4uHlRNy4gTJ07Mnj2b2YFLfAbXBYA7lJSUONI4gS1k0w+2o3Bdamoq4+uEsxUAAMMgrQAAGAZpBQDANMtvntmOBQDgrajrVgCgAWVNwHFwEQQAYBikFQAAwyCtAAAYBmkFAMAwSCsAAIZBWgEAMAzSCgCAYZBWAAAMg7QCAGAYpBUAAMMgrQAAGAZpBQDAMEgrAACGQVoBADAM0goAgGGQVgAADIO0AgBgGKQVAADDIK0AABgGaQUAwDBIKwAAhkFaAQAwDNIKAIBhkFYAAAzjsx0A4C6dTnft2jXy//X19QihCxcukA/j4uLEYjFrkQFug66GwCaTyRQYGGgwGPh8Pvk+wTDMZDIJBAK1Ws3nw98kQA0ugoBNfD4/OTm5r69Po9F0d3d3d3drNJq+vr7k5GTIKYAGpBVAZ9myZb6+vpZTxGLx8uXL2YoHeAW4CAJ0CIKQyWRqtdo8JSAg4PHjxxiGsRgV4Dg4WwF0MAzLyMgQCATkwxEjRixduhRyCqAHaQXYkZGRYb6TIhAIli5dym48gPvgIgjYFxYW9uDBA4RQaGjow4cP2Q4HcB2crQD7MjMzRSKRUChcuXIl27EALwBnK8C++vr6yZMnI4Ru374dERHBdjiA6yCtAIdER0djGHb79m22AwFegCtpBb5cAGDoOPJx5lCtJEeOCCBh2M/+5LS2tiKEgoKC2IuIzqBof5m487eZKy8GvC24xrteEe+K1k24cxDgmyAAAMMgrQAAGAZpBQDAMEgrYEgSExP37t3LdhT2GY3GnTt3Zmdn4ziOYdju3bvJ6VeuXJHL5QKBwN3Dsmtra+Pj48+fP2+eotFopkyZguO4VCpNTEy8f/8+Ob28vDw6OlosFk+ePPncuXPkxJMnT1ZUVLg1QiYR3MCdSACJrVfkj3/8Y2Njo7NL0UdrMpmSkpJu3LhBEIRCoQgLC5NKpe3t7eSzjY2NWVlZrkXroOPHj2/atMnf37+iosI8Ua1Wr1q1SqPRdHR0LFy4cMaMGQRBaLVaiUSiVCq1Wu3f/vY3HMd7enrI+XNzcw8dOkSzFe58iDgTB2eOCCCx9YpMnDiR8bRSUFDw3nvvkf9XKBSff/55cHDwunXryCkeSCukkJAQy7RiqaqqisfjmUymuro6hBCZ8sjfo6irqyPn0el0oaGhd+7csbV+7nyI4CIIuO7w4cMikSgvLw8htHHjRgzD3n///aioKBzHCwoKyHlycnIwDHv99ddxHA8PD//qq68QQikpKRiG3b9//9GjR2FhYTiOkzOnpqbW1dWFhoauXbsWIfT222+vX79+iEH29/crlcqMjAzzFKlUum/fvuLi4p9++sl6/gsXLkydOhXH8djY2MrKSppdQwh9/fXXkZGRo0aNyszMNBgMLgfZ09MzZswYHx+fiIiIiRMnnjlzRq/Xnzp16vnnnzePlvD19U1KSjpw4IDLW/EctvPaM9yJBJAcfEWysrK2bNlC/l8mk12+fHlgYODo0aNisdg8j0Qiqaqq0ul0SqVSJBK1tLSQ67937x5BED/88INEIiHnNBqNCCFmz1auX7+OENJoNORDhUJRXl5OEERqauqCBQuIn5+tqNVqiURy7NgxjUajVColEklbW5utXWtpafH19S0vL29vb3/ppZeKiorog6Q5W8nJycnOzib/f+3aNfKXKPh8fk1NjeVshw8fDg8Pd+EgeBicrQCGYRg2e/ZsnU5nMpnME4ODg319fVetWjVmzJiamhpPxtPQ0CAUCv38/AZNVygUFy9e/Pbbby0nqlQqmUy2aNEiPz8/MtpvvvnG/OygXaupqQkNDZ0/f76/v/8777xz6dIl1yJsbGw8f/78tm3bEEJqtTopKamsrEyr1ZaWlqampnZ0dJjnDAgIePDgAcGNmjcakFaAR40dO7a9vd2TW+zt7RUKhdbTAwMDi4qKPvzwQ8uJbW1tgYGB5ocymaytrc3WmtVq9d27dzEMwzAsPz+/s7PThfC6u7tXrlx58uRJqVSKECorKwsODk5MTJRIJO+++25gYGBZWZl5ZqFQODAwoNfrXdiQJ0FaAZ5DEERzc3NISIgnNyoWi23d9Vi4cKFMJiNv95CCgoIsf7i3tbWVZhiUVCqNiYkxn/mrVCpnY+vu7s7MzCwsLIyKiiKnWIfa29tr/r/BYODxeCKRyNkNeRikFeAJWq1Wr9crFAqDwRAfH48QwnH86tWrRqOxubnZPBuPx+PxeLW1tTqdjqlNh4WF6fV6rVZL+WxxcbFl3c28efMeP3587Nix7u5upVL59OnTefPm2VrznDlz6urqjh8/3tPTo9PpnD1b0Wg0WVlZO3bsMOcUhNCsWbPu3LmjUql6e3tPnz5dW1sbFxdnflatVk+YMIE7QwptYu2uzs9xJxJAcuQV2bhxo0AgEIvFu3bt2rBhA0JowoQJnZ2d0dHRCKH09HRyNolE4u/vP2LEiNjYWPM9yK1bt4pEosjIyFWrViGEli9fTk5PSUkRCoVpaWkEQbz11lsffPDBEKM1mUzjxo27fv06QRCFhYU4jvv7+x88eNA8w5EjRyy/YK6qqoqJiRGLxbGxsSqViiAIml0rLS2NjIwUCoUzZsy4efOmWq0ODAxUKBSDYli/fn1oaChCCMfxWbNmNTc3EwRx8ODBQR/Gy5cvEwSxf//+5557TiQSvfDCC4MKVdavX09zQLjzIeLKkEfuDL4EJAZfERzHb9y4MWnSJEbWRok+2u3bt3d0dHzyySfuC4BkNBrT09NjY2O3bNnC+MpNJlNUVFRZWZnl2Y0l7nyIvOki6OHDhxERERiGcfCWlXVpNqXdu3eTxeMYhvF4PH9///j4+NLSUs8EyaKBgQEWt75p06b6+vrvvvvO3RtSKpUBAQE5OTnuWHleXl5ubq6tnMItrJ4r/Y+DkbS0tCCEent73R2PJbvl5JSl2bYoFAqZTEYQhEaj+f7779PS0hBCeXl5jIXrGLs7xdR7Y/HixQihkJCQf/7zn4yskJLdaA0Gw/bt2+vr690Xg1uVlpaeO3eOfh4OfZzZDuAZB48I+RtlHk4rDpaT0xQ7WTKnFbM//OEPPj4+//3vf10P0Xl2d4o771FHeFe0bsKdg+AdF0GVlZVTp04ViUQxMTHklOzsbAzDKioqFixYkJubi6hqrinLxknWMztYTu44xwvPN2zYMDAwUFlZyf2dAsAhbOe1Z2giaWtrE4lEe/bs6e3tvXfvHvr/sxWZTFZSUtLZ2blz505bNdeUZeO2ZkZDKyd3+WyF3BeyBJ47O8Wd94YjvCtaN+HOQfCCs5WKigqZTLZmzRqRSGT+e0sKCwsbNWrUxo0baWqurcvG6Qu0WaHX63m8Z6/FsNkp8IvFoV/et6WlpWX8+PH08zhSc20uG3eqQNsDNBpNV1dXeHj4oOms75QXlF1Z8K5ohzcvSCtSqdSynpqS3ZprwqJs3KkCbQ84evQon89PSEgYNJ31nSK4UQThCO6UbLCIO4nVCy6C5s6de/fu3ZKSEq1We/bsWcp5aGqurcvGbc3ssXJygiD6+voQQs3NzUqlcvPmzXl5ecHBwV69UwD8D6t3dv6HPpJ9+/bJ5XKpVEoWQSQlJWVnZyOEgoODr169Ss5jXXNN2CgbtzWzI+XklChLsykLz/fs2SOTyQQCAXknRSKRzJw5s6SkhHyWUzvFnfeGI7wrWjfhzkHgyqmjm05iPVA27nme2SnvuqzwrmjdhDsHwQsugoaIkbLxpqYmjEpTU9PQV+4CdmvhAaA3nNPKkiVLenp6EhISbt26NcRVyeVyypM9uVzOSKiOY3CnflGgoYdHee56ixZ3IgEkpl4R1xp0OLsgfbTQ0MPDOBMHZ44IIDH1irjWoMPZBemjhYYeHjacL4KA+zg4/shy8JGt0Ux2F0RD6+wBDT1YwHZee4Y7kQASzSvi+PijQYOPKEczObLgUKKFhh6eB2crwGlDGX/k+c4e0NDD8yCtAKcxMv7IY509oKGH50FaAU4b+vgjwoOdPaChh+dBWgFOc3z8kfXgI+vRTA4u6DJo6MEC1u7q/Bx3IgEk+lfE8fFHloOPbI1msrug3c4eNNFCQw/P48ogAu4MZwAkd7wi7hvNBA09EJc+RHARBDyKldFM0NDDw7iS3riTaAGJ8VdkyZIlf//730NCQsrKyl588UUG14wciNZoNP71r39NS0uz/hU+r3DixImRI0cmJibSzMOdDxFn4uDMEQEk73pFvCtaN+HOQYCLIAAAwyCtAAAYBmkFAMAwrlyMeUGFDwCcx5WPM0fiABz38ccfm/8FgB4xRIQpAAAOL0lEQVRcBAEAGAZpBQDAMEgrAACGQVoBADAM0goAgGGQVgAADIO0AgBgGKQVAADDIK0AABgGaQUAwDBIKwAAhkFaAQAwDNIKAIBhkFYAAAyDtAIAYBikFQAAwyCtAAAYBmkFAMAwSCsAAIZBWgEAMAzSCgCAYZBWAAAMg7QCAGAYpBUAAMP4bAcAuEun0127do38f319PULowoUL5MO4uDixWMxaZIDboKshsMlkMgUGBhoMBj6fT75PMAwzmUwCgUCtVvP58DcJUIOLIGATn89PTk7u6+vTaDTd3d3d3d0ajaavry85ORlyCqABaQXQWbZsma+vr+UUsVi8fPlytuIBXgEuggAdgiBkMplarTZPCQgIePz4MYZhLEYFOA7OVgAdDMMyMjIEAgH5cMSIEUuXLoWcAuhBWgF2ZGRkmO+kCASCpUuXshsP4D64CAL2hYWFPXjwACEUGhr68OFDtsMBXAdnK8C+zMxMkUgkFApXrlzJdizAC8DZCrCvvr5+8uTJCKHbt29HRESwHQ7gOkgrwCHR0dEYht2+fZvtQIA3ILxKfn4+2wcMAE/Lz89n+5PnHO+rlczPz//444/ZjsJbkYfOhQPY2tqKEAoKCmI6IjouRzuceOPue19aAazwcEIBXg2+CQIAMAzSCgCAYZBWAAAMg7QCqCUmJu7du5ftKOwwGo07d+5saGjYvXs3juMYhu3evZt86sqVK3K5XCAQuHu8dW1tbXx8/Pnz58mHGo1mypQpOI5LpdLExMT79++b5ywvL4+OjhaLxZMnTz537hxC6OTJkxUVFW4NjxWQVgC1ioqKNWvWMLW2/Pz8pqYmptZG6u/vT0lJee2118LDw3Nycnbu3BkWFvbnP/+5o6MDIfTqq69ev349IyPjyJEjzG7X0hdffPH555//61//Mk8xGAxxcXEtLS319fWjR49evHgxOb2npyctLe33v/+9Wq1+7733UlNTdTrdggULrly5cvjwYfdFyApIK8ATTpw4wfg6//KXv4wdO3batGnmKQUFBSKR6E9/+hPj27IlLS1tx44dlj9JExAQoFQq/fz8pFJpZmbm999/39/fjxBqbm7u6elJTk6WSCSLFi3SarVkns3Ly/v4449//PFHj8XsAZBWAIXDhw+LRKK8vDyE0MaNGzEMe//996OionAcLygoQAjl5ORgGPb666/jOB4eHv7VV1+RC6akpGAYdv/+/UePHoWFheE4jhBKTU2tq6sLDQ1du3YtQujtt99ev379ECPs7+9XKpUZGRmWE6VS6b59+4qLi3/66adB81+4cGHq1Kk4jsfGxlZWVpITKXcNIfT1119HRkaOGjUqMzPTYDC4HGRPT8+YMWN8fHwQQhERERMnTjxz5oxerz916tTzzz9PDoPw9fVNSko6cOCAy1vhIrbr8ZyTn5/vdRWHnOL4AczKytqyZQv5f5lMdvny5YGBgaNHj4rFYnKiRCKpqqrS6XRKpVIkErW0tJDTEUL37t0jCOKHH36QSCQEQRiNRoRQY2Mjg9Fev34dIaTRaMxTFApFeXk5QRCpqakLFiwgCKKxsTErK4sgCLVaLZFIjh07ptFolEqlRCJpa2uztWstLS2+vr7l5eXt7e0vvfRSUVGR3ThDQkIqKiqsp+fk5GRnZ5sfXrt2jfyJCT6fX1NTY55++PDh8PBwFw4CZ8HZCnAUhmGzZ8/W6XQmk4mcEhwc7Ovru2rVqjFjxtTU1HgymIaGBqFQ6OfnZ/2UQqG4ePHit99+a56iUqlkMtmiRYv8/PzIaL/55hvLRSx3raamJjQ0dP78+f7+/u+8886lS5dci7CxsfH8+fPbtm0jH6rV6qSkpLKyMq1WW1pampqaSt4DQggFBAQ8ePCAGEaj8yCtAAaMHTu2vb3dk1vs7e0VCoWUTwUGBhYVFX344YfmKW1tbYGBgeaHMpmsra3N1prVavXdu3cxDMMwLD8/v7Oz04Xwuru7V65cefLkSalUSk4pKysLDg5OTEyUSCTvvvtuYGBgWVkZ+ZRQKBwYGNDr9S5siJsgrYChIgiiubk5JCTEkxsVi8U0dz0WLlwok8nMd3yCgoIsf463tbWVZiyCVCqNiYkxn8+rVCpnY+vu7s7MzCwsLIyKijJPtI62t7fX/BSPxxOJRM5uiLMgrQDXabVavV6vUCgMBkN8fDw5Ecfxq1evGo3G5uZmcgqPx+PxeLW1tTqdjqlNh4WF6fV6rVZra4bi4mJz3c28efMeP3587Nix7u5upVL59OnTefPm2Vpwzpw5dXV1x48f7+np0el0zp6taDSarKysHTt2WOYUhNCsWbPu3LmjUql6e3tPnz5dW1sbFxdHPqVWqydMmDCsfiGYtbs6LvHG21ec4uAB3Lhxo0AgEIvFu3bt2rBhA0JowoQJnZ2d0dHRCKH09HSCICQSib+//4gRI2JjYy1vQG7dulUkEkVGRq5atQohtHz5coIgUlJShEJhWloaQRBvvfXWBx98MMRoTSbTuHHjrl+/Tj4sLCzEcdzf3//gwYPmeY4cOULesiUIoqqqKiYmRiwWx8bGqlQqcqKtXSstLY2MjBQKhTNmzLh586ZarQ4MDFQoFNZhrF+/PjQ0FCGE4/isWbOam5sPHjw46CN2+fJlcub9+/c/99xzIpHohRdeOHTokOVKaA6IN77nvexnnGCk/BAxeABxHL9x48akSZOGvipb6KPdvn17R0fHJ5984r4ASEajMT09PTY2dsuWLYyv3GQyRUVFlZWVDTq7MfPG9/wwvAh6+PBhREQEhmGeuQf25MkT7OfIv4GUzDXmGIbxeDx/f//4+PjS0lIPxOkOAwMDLG5906ZN9fX13333nbs3pFQqAwICcnJy3LHyvLy83NxcWznFSw3DtDJ+/HhzQ3LPKCgoMJ/+/e53v6NpeUHWmMtkMoIgurq6qqqqgoKC0tLStm7d6smAh27JkiU9PT0JCQm3bt1iKwYfH58vv/yyurq6oaHBrRvKzs4uLi4e1N2RESdOnJg9e/bw6xI5PH/GyZN3vwICAsxJoaOjo6mpibxKt8vPz+/ll18+fvy4XC7fsWPHihUrJkyY4M5ImVRSUlJSUsJ2FGjEiBG5ublsR+G61NRUtkNwi2F1tlJZWTl16lSRSBQTE2M5fVAtNmXJtsFgSElJkUgkAQEB5OA0Fyq4lUplVlYW+X/HS9Q3bNgwMDBAVpQ7EipT0QLgJsMnrTx+/DgpKWnlypWdnZ2WF0Gtra2LFy/evXt3Q0PDv//97/379+/atUsmk6Wnp//444979+7dtWsXQujUqVMajebJkycXL15sb2+3XspuAEajsaKiYv78+eTD8vLywsJCRyIPCAgYO3bsw4cPHQyVkWgBcJ/hk1YqKipkMtmaNWtEIhE5wo1EU4ttWbKN4/itW7eqqqp+9atfffTRRy5UcH/xxRe//e1veTxXDqler+fxeA6GihAaerQAuM/wubfS0tIyfvx46+nmWmzyoa06qDfffHPdunWrV6/m8/lHjhxxcClLBw4cOHPmjAuRazSarq6u8PBwxzc6lGi3bdtmHqjiFbwrWnfwuj42w+dsRSqVWhZoW053pBYbw7Dc3NympqYVK1asXbvW2Qru6urqKVOmjB492oXIjx49yufzExISHN/oUKL1otoqb6wEY5zX5RQ0nNLK3Llz7969W1JSotVqz549a57uYC32p59+qlKp+vv7p02bhmGYsxXchYWF2dnZDoZKEERfXx9CqLm5WalUbt68OS8vLzg42PGNDjFaANyL7VzsHPo/X/v27ZPL5VKplPylv6SkJHL6oFpsypLt8vLycePG8fn8yMhIsrh70FI0UdXW1r7xxhuDJlKWqO/Zs0cmkwkEAvIWjEQimTlzZklJiXkGR0IlCMLlaL3r7793Resm3ngQoHj/l8W7DqB3Resm3ngQhs9FkFs1NTVhVBj/2WcAhgFIKw6Ry+WUJ3tyuZzt0AAFstFHdna257t86PV6smsHjuOvvPKKecjScO3dQQnSCnCRyz063NHcw5K50YdCofB8lw+TyTR9+vTW1tZHjx5NmjQpLS2NnD5ce3dQgrQCXORyjw53NPewNKjRh4e7fOA4fuTIkZEjR44cOTIlJaW5udl8+3JY9u6gBGkF/A9l1wu7PToom3tQLjVoQUY6ewxi3eiDpssH5S7bGoeFnBl4NTAw0NbW9tlnn6WmpprLFIdn7w5KHv3eaci88cs2TqE5gDRdL5C9Hh2UzT2sl7Je0OVobRnU6IOmywfNLlM2MHGq0cevf/1rhNBrr71mPoYk+t4dlLzxPQ9nK+AZu10v6LHY3MOSrUYf1l0+kL1dHjQOy6mBV2fPnm1tbZ03b96LL7749OlT8/Th17uDEqQV8IxTXS9oeL65hyVbjT6su3wgdzb68PHxkclkmzZtEggE5sYdaDj27qAEaQU841TXC1sINpp7WKJp9DGoywfySKOPQScmw693ByVIK+AZmq4XjvTosG7uYb0U5YLMom/0Ydnlg36XrTk48Or06dMHDhzQ6/VdXV1FRUVqtXru3LnmZ4dh7w5KbN3UcY033r7iFPoDSNn1gnCgRwdlcw/KpSwXtNvZw4WX27LRh90uH5S7bGscFuFYo4+bN2+Gh4cLhUKxWDx9+vTq6mrLZ+l7dzB1EFgHY4J+Wdx0AN3U3MO1aDnb6MNu7w5K3vieh4sgwAx2m3tY4myjj2HZu4MSpBUwVFxo7mGJm40+hmvvDkrD50cnAVs40tzDEgcbfQzX3h2U4GwFAMAwSCsAAIZBWgEAMI3tb7id442/Qg7AEEHdCgDglw4uggAADIO0AgBgGKQVAADD/g/oLUUh1jpFJAAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<IPython.core.display.Image object>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4Bczh_cgQ622",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "fa5c8d55-6300-444d-8fe2-cc72ba0b2038"
      },
      "source": [
        "# Training Phase\n",
        "history = model.fit(x=train_data,\n",
        "                    y=train_labels,\n",
        "                    batch_size=128,\n",
        "                    epochs=20,\n",
        "                    validation_data=(test_data, test_labels),  \n",
        "                    # The above isn't really \"cheating\", it's the equivalent of trying different parameters manually till we find the best for the specific  \n",
        "                    # model we are building like research papers do.\n",
        "                    #validation_split=0.1,\n",
        "                    callbacks=[early_stop, model_save],\n",
        "                    verbose=1)\n",
        "\n",
        "model = load_model('./gdrive/My Drive/Colab Saved Models/best_model_text.h5')\n",
        "#model = load_model('best_model_text.h5', custom_objects={'MultiplicativeLSTM': MultiplicativeLSTM})  # Handling saved models changes if we are using custom (3rd-party) layers"
      ],
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /tensorflow-1.15.2/python3.6/keras/backend/tensorflow_backend.py:422: The name tf.global_variables is deprecated. Please use tf.compat.v1.global_variables instead.\n",
            "\n",
            "Train on 48213 samples, validate on 5357 samples\n",
            "Epoch 1/20\n",
            "48213/48213 [==============================] - 13s 266us/step - loss: 1.0246 - accuracy: 0.4455 - val_loss: 0.9927 - val_accuracy: 0.4650\n",
            "Epoch 2/20\n",
            "48213/48213 [==============================] - 6s 124us/step - loss: 0.9141 - accuracy: 0.5466 - val_loss: 0.8501 - val_accuracy: 0.5985\n",
            "Epoch 3/20\n",
            "48213/48213 [==============================] - 6s 123us/step - loss: 0.7919 - accuracy: 0.6333 - val_loss: 0.8106 - val_accuracy: 0.6190\n",
            "Epoch 4/20\n",
            "48213/48213 [==============================] - 6s 124us/step - loss: 0.7325 - accuracy: 0.6674 - val_loss: 0.8020 - val_accuracy: 0.6289\n",
            "Epoch 5/20\n",
            "48213/48213 [==============================] - 6s 122us/step - loss: 0.6846 - accuracy: 0.6962 - val_loss: 0.8073 - val_accuracy: 0.6300\n",
            "Epoch 6/20\n",
            "48213/48213 [==============================] - 6s 123us/step - loss: 0.6475 - accuracy: 0.7155 - val_loss: 0.8133 - val_accuracy: 0.6377\n",
            "Epoch 7/20\n",
            "48213/48213 [==============================] - 6s 122us/step - loss: 0.6139 - accuracy: 0.7330 - val_loss: 0.8314 - val_accuracy: 0.6386\n",
            "Epoch 00007: early stopping\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NZUXZS5IkBOj",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 447
        },
        "outputId": "1a81dd33-30b1-47ac-9a88-263dbad5db20"
      },
      "source": [
        "# Evaluation Phase\n",
        "results = model.evaluate(test_data, test_labels)\n",
        "print()\n",
        "\n",
        "for i in range(len(model.metrics_names)):\n",
        "    if model.metrics_names[i] == \"acc\":\n",
        "        print(f\"Metric of the best model - {model.metrics_names[i]}: {results[i]*100:.3f}\")\n",
        "    else:\n",
        "        print(f\"Metric of the best model - {model.metrics_names[i]}: {results[i]}\")  \n",
        "\n",
        "if multiclass == False: \n",
        "    predictions = [1 * (x[0]>=0.5) for x in model.predict(test_data)]\n",
        "    cm = confusion_matrix(test_labels, predictions)\n",
        "else:    \n",
        "    predictions = np.argmax(model.predict(test_data), axis=1)\n",
        "    cm = confusion_matrix(test_labels.argmax(axis=1), predictions)\n",
        "print()\n",
        "print(cm)\n",
        "\n",
        "heatmap(cm, fmt=\"g\", annot=True, square=True, cmap=plt.cm.Blues)\n",
        "plt.tight_layout()\n",
        "plt.ylabel('True label')\n",
        "plt.xlabel('Predicted label')\n",
        "plt.show() "
      ],
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "5357/5357 [==============================] - 1s 191us/step\n",
            "\n",
            "Metric of the best model - loss: 0.8020134239315163\n",
            "Metric of the best model - accuracy: 0.6288967728614807\n",
            "\n",
            "[[ 296  335  160]\n",
            " [ 197 1601  693]\n",
            " [  64  539 1472]]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWMAAAEmCAYAAAC3V/E+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de7xWY/7/8ddn787nUu3OKnLIuYlyKIkojJhRchgNkUNoVDMyJPJlMIMZfoxJmgrTASFEUpJDkXLq4LCF2umkc3anvffn98e9yi3t9t63fXevde/3cx7r0X1f63St+9G8u1zrWtcyd0dERFIrI9UVEBERhbGISCgojEVEQkBhLCISAgpjEZEQUBiLiISAwlhEJI6ZjTSzVWY2f7fy683sczNbYGb3xZXfbGbZZvaFmZ0RV941KMs2s8FFnlfjjEVEfmJmHYHNwBh3PzwoOwW4BTjL3beZWX13X2VmrYGxwHFAI+AN4KDgUF8CXYAcYA5wobsvLOy85ZJ1QSIiUeTuM82s+W7F1wD3uPu2YJtVQXl3YFxQ/o2ZZRMLZoBsd18MYGbjgm2jF8ZfrsxVk70U7MjTz/hrZdWsmOoqpI261cpZaR6v8jHXlfgv+NaPH7kK6BtXNNzdhxex20FABzO7C9gKDHL3OUBjYHbcdjlBGcDS3crb7e0EoQ1jEZFkCIK3qPDdXTmgDtAeOBaYYGYtS7NeCmMRiS7bZ2MQcoCJHrvJ9oGZFQB1gWVA07jtmgRl7KV8jzSaQkSiy6zkS2JeAE6JndIOAioAPwCTgF5mVtHMWgCtgA+I3bBrZWYtzKwC0CvYtlBqGYtIdCWhZWxmY4FOQF0zywGGAiOBkcFwt+1A76CVvMDMJhC7MZcH9HP3/OA41wFTgExgpLsv2Ot5wzq0TTfwSodu4P16uoFXekr9Bt6xA0r8F3zLnAdKtQ6lRS1jEYmufddnnHQKYxGJrozMVNeg1CiMRSS6Er8hFzoKYxGJLnVTiIiEgFrGIiIhoJaxiEgIqGUsIhICahmLiISAwlhEJAQy1E0hIpJ6ahmLiISAbuCJiISAWsYiIiGglrGISAioZSwiEgJqGYuIhIBaxiIiIaCWsYhICKhlLCISAmoZi4iEgFrGIiIhoDAWEQmBNOqmSJ9/VkSk7LGMki9FHdJspJmtMrP5e1g30MzczOoG383MHjKzbDP71MzaxG3b28y+CpbeRZ1XYSwi0ZWRWfKlaKOArrsXmllT4HRgSVxxN6BVsPQF/h1sWwcYCrQDjgOGmlntvV5KcWomIhJKZiVfiuDuM4G1e1j1IPAXwOPKugNjPGY2UMvMGgJnAFPdfa27rwOmsoeAj6cwFpHIMrNElr5m9mHc0rcY5+kOLHP3T3Zb1RhYGvc9JygrrLxQuoEnIpFlCdzAc/fhwPASnKMK8FdiXRRJo5axiESXJbCU3AFAC+ATM/sWaALMM7MGwDKgady2TYKywsoLpTAWkchKpJuipNz9M3ev7+7N3b05sS6HNu6+ApgEXBqMqmgPbHD35cAU4HQzqx3cuDs9KCuUuilEJLISCddiHHMs0Amoa2Y5wFB3f6KQzScDZwLZQC5wGYC7rzWzO4E5wXbD3H1PNwV3URgXYvXKFTx49xDWr10DZnT97e85p8dFfJP9BY/cfxdbc7dQv2EjBg25iypVqwHwzddf8sg//o/cH38kwzJ4YPhTVKhYMcVXklrbt2/j1v5XsGPHdgry8zn+5FPp9cdreOTvd5D9xULAadhkf66/6Q4qV67C9NcmMeY//6RO3foAdDv3ArqcdV5qLyIE7r7jVt59+y1q16nDUxNe3FX+zLinmThhLBmZGZxwUkf69R8EwJiRj/Pyi8+RkZnJjYNupt0JJ6Wq6kmVjDB29wuLWN887rMD/QrZbiQwsrjnVRgXIjMzk8uvHcCBBx9Kbu6P3HjFRRx9bDseum8Yl197I0cc3Zapr7zAxLGjueSKfuTn5fHAnbcy4NY7aXHgwWzcsJ7Mcvp5y5evwB0P/IfKlauQl7eDW27owzHHnchl1w7c9Y/Yfx+9n1efH8/vLroMgBM7nc6V/Qenstqhc+Zvz+X3PS/izqE37yqbO+d93nlrOqPHTaRChQqsW7sGgG8WZzPt9ck89cwkfli9iv7XXMG4518hM7NYY2wjJRlhnCpJ6zM2s0PM7Kbg6ZSHgs+HJut8pa1O3XoceHCsulWqVKXp/i1Ys3o13y9dwuFH/QaAo9u25723pgHw0ZxZND+gFS0OPBiAGjVrpeVf/pIyMypXrgJAfl4eeXl5mNmuIHZ3tm/bllaPtSbD0W3aUqNmzZ+VvfDseC754xVUqFABgNp19gPg7RlvcurpZ1KhQgUaNW5Ck6ZNWbTgs31e531i39zA2yeSEsZmdhMwjtilfxAsBow1s8g1eVYu/56vv/qCg1sfTrPmLZn9zgwA3p0xlR9WrQRg2dIlYMZtA6+lf58Lee5/o1JX4ZDJz89nwJW9uOx3p3FU23YcdOgRADx871AuP78Ly5Z+y1nnXbBr+1lvT+fGK3py3+1/5odVK1JV7dBbsuRbPvloLlde2ot+V/beFbirV68kq0GDXdvVz2rA6uDvabrZFzfw9pVktYz7AMe6+z3u/lSw3EPsscA+he0UPxh7/JPF7mpJqi25ufxtyCCuvH4QVapW44bBtzP5+Qn86YqL2JKbS7ny5YFY4Cz89CMGDrmLex8Zyay3p/PJ3PdTXPtwyMzM5IHHx/H4hNfI/nwB332TDcD1N93BiAlTaNysBe+8+ToAxx7fkf/872UeHDGBo37TjofuuS2VVQ+1/Px8Nm7cwPDRY+nXfyBDBg8k1oVZdiiMi1YANNpDecNg3R65+3B3b+vubS/4w+VJqlrx5eXt4G9DBtGpSzdOOPlUAJru34I7H/g3/xzxPzqe1pUGjZoAULd+fQ4/qg01a9WmUqXKtG1/El9/+Xkqqx86VatV5/Cj2/LRB+/tKsvMzOSkU05n9tux7p7qNWtRPvjP7tPOPI/FX+k3LEz9+lmcfMppmBmtDz8SswzWr19HvXpZrFzx039RrFq5gnr1s1JY0+RRGBftT8A0M3vVzIYHy2vANKB/ks5Zqtydh+69g6b7t+DcC/6wq3z9utjolIKCAsaPeZxu3c8HoM1xJ/Dt4my2bt1Cfl4e8z+eS9PmLVNS9zDZsH4dP27eBMC2bVv5ZO5sGjfdn+XLYnOtuDtz3ptJ46YtAFi7ZvWufee89xaNmzXf53WOig6dTmXehx8AsOS7b8nL20GtWrU56eRTmPb6ZLZv3873y3LIWbqEQw87IsW1TY50CuOk3O5399fM7CBi3RI7n8deBsxx9/xknLO0LfzsY96c8grNW7bihstj/ZmXXnkd3+cs5ZXnxwNwfMfOnHZmdwCqVa/BuRdcwoC+l2BmtG1/Esce3yFl9Q+LdWtW8/C9QykoyKegwDmxUxd+074Dt/Tvw5bcH3F3mh9wEFf9KTZKYPLEccx57y0yMjOpXqMm1990R4qvIByG/nUQH304h/Xr13Nut870uaofZ3c/j7vvGMIlPbtTvlx5br39LsyMlgccSOcuXbn4/HPILJfJgJtuTd+byeHN1hKzsPYxfbkyN5wVi5gdefoZf62smmV7rHhpqlutXKnGZ90/jivxX/AfRvUKZYRrIKyIRFaYux1KSmEsIpGlMBYRCYP0yWKFsYhEl1rGIiIhoDAWEQkBhbGISAgojEVEwiB9slhhLCLRlZGRPm+OUxiLSGSpm0JEJAzSJ4sVxiISXWoZi4iEgMJYRCQEFMYiIiGQTmGcPuNCRKTsScLboc1spJmtMrP5cWV/N7PPzexTM3vezGrFrbvZzLLN7AszOyOuvGtQll2cFzErjEUkspL02qVRQNfdyqYCh7v7kcCXwM3B+VsDvYDDgn0eNbNMM8sEHgG6Aa2BC4NtC6UwFpHISkYYu/tMYO1uZa+7e17wdTbQJPjcHRjn7tvc/Rsgm9jr5o4Dst19sbtvB8YF2xZKYSwikWWWyGJ9zezDuKVvCU97OfBq8LkxsDRuXU5QVlh5oXQDT0QiK5EbeO4+HBie4PluAfKApxPZf28UxiISWftyMIWZ/RE4GzjVf3qT8zKgadxmTYIy9lK+R+qmEJHIStINvD2dpyvwF+Acd8+NWzUJ6GVmFc2sBdAK+ACYA7QysxZmVoHYTb5JezuHWsYiElnJaBmb2VigE1DXzHKAocRGT1QEpgaBPtvdr3b3BWY2AVhIrPuin7vnB8e5DpgCZAIj3X3B3s6rMBaRyMrIKP00dvcL91D8xF62vwu4aw/lk4HJxT2vwlhEIiuNHsBTGItIdKXT49AKYxGJrDTKYoWxiESXWsYiIiGgMBYRCYE0ymKFsYhEl1rGIiIhkIxxxqmiMBaRyEqjhrHCWESiS90UIiIhkEZZrDAWkehSy3gfqF4ptFWLlJadBqS6CpF378MDU12FtHHDSS1K9XhplMXhDWMRkaKoZSwiEgJplMUKYxGJLrWMRURCII2yWGEsItGllrGISAgojEVEQiCNslhhLCLRpZaxiEgIpFEWK4xFJLrSqWWckeoKiIgkyqzkS9HHtJFmtsrM5seV1TGzqWb2VfBn7aDczOwhM8s2s0/NrE3cPr2D7b8ys95FnVdhLCKRlWFW4qUYRgFddysbDExz91bAtOA7QDegVbD0Bf4NsfAGhgLtgOOAoTsDvNBrKdYVi4iEUDJaxu4+E1i7W3F3YHTweTRwblz5GI+ZDdQys4bAGcBUd1/r7uuAqfwy4H9GYSwikWVmiSx9zezDuKVvMU6V5e7Lg88rgKzgc2Ngadx2OUFZYeWF0g08EYmsRF6B5+7DgeGJntPd3cw80f0Lo5axiERWIi3jBK0Muh8I/lwVlC8DmsZt1yQoK6y8UApjEYmsZPQZF2ISsHNERG/gxbjyS4NRFe2BDUF3xhTgdDOrHdy4Oz0oK5S6KUQksozSH2dsZmOBTkBdM8shNiriHmCCmfUBvgN6BptPBs4EsoFc4DIAd19rZncCc4Lthrn77jcFf6bQMDazh4FC+0Xc/YaiL0tEJHkyE+k0LoK7X1jIqlP3sK0D/Qo5zkhgZHHPu7eW8YfFPYiISCqk0QN4hYexu4+O/25mVdw9N/lVEhEpnmI+xBEJRd7AM7PjzWwh8Hnw/SgzezTpNRMRKcI+vIGXdMUZTfFPYk+TrAFw90+AjsmslIhIcezDoW1JV6zRFO6+dLeLyE9OdUREii/E2VpixQnjpWZ2AuBmVh7oDyxKbrVERIpWpvqMgauJDd1oDHwPHE0hQzlERPYlS2AJqyJbxu7+A3DxPqiLiEiJhLkPuKSKM5qipZm9ZGargwmXXzSzlvuiciIie5NhJV/CqjjdFP8DJgANgUbAM8DYZFZKRKQ40mk0RXHCuIq7P+nuecHyFFAp2RUTESlKOo0z3tvcFHWCj6+a2WBgHLG5Ki4gNjmGiEhKhbmlW1J7u4E3l1j47rzaq+LWOXBzsiolIlIcYe4DLqm9zU3RYl9WRESkpMpKy3gXMzscaE1cX7G7j0lWpUREiiN9orgYYWxmQ4lNtNyaWF9xN+AdQGEsIilV1p7AO5/YpMor3P0y4CigZlJrJSJSDGViNEWcLe5eYGZ5ZlaD2Iv4mha1Uzq4984hzHpnJrVq12HUuOcByP7yCx64ZxhbtuTSoGFjbh12D1WrVWPqay8z7slRu/ZdnP0lw5+cQKuDDklR7VPnsaEX063j4axeu4m2Pe7eVX5Nr5O5qmcH8guc196ezy3/ir1GbNDlp/PH7seTX1DAwPue5Y1Zi/Z6nLJqW+5m3hz1T9Ys+xYzo/Mfb6RcxUrMGPMQO7ZtpUbdLLpc+RcqVK7KysVf8OaYf8V2dOe47pfQss2Jqb2AJChrfcYfmlkt4HFiIyw2A7OSWquQ6HpWd87rcSF3337LrrK/3zWUa/oP5Og2xzJ50vOMe+q/9Ln6erp0PZsuXc8GYkF865/7l8kgBnjypdk8Nv4tRtx56a6yjm1bcXanIzjugnvYviOPerWrAXBIywb0OKMNbc6/i4b1ajL5ses44txhFBT4Ho9Tlr099jGaHf4bul57K/l5O8jbvo1J99/MCT2vpPHBR7Lw7Sl89NqztDuvN3Ua70/PIQ+TkZnJj+vXMP72a2l+VHsyMjNTfRmlKo2yuOhuCne/1t3Xu/tjQBegd9BdkfaOatOW6jV+3iOTs+Q7jjqmLQBt2x3PzDff+MV+015/lc5duu2TOobRu/O+Zu2Gn78Upm+PDvzjv1PZviMPgNXrNgNwdqcjeWbKPLbvyOO779fw9dIfOPbw5oUep6zalvsj33/5GYd26ApAZrnyVKxSjfUrl9HooCMAaHpYG76e+y4A5StW2hW8+Tt2pFdqxckwK/ESVoWGsZm12X0B6gDlgs8JMbNIB3nzlgfwzlvTAZjxxhRWrVzxi23enPoanc8ou2G8JwfuX58TjzmAmWMG8fqI/vymdTMAGterSc6Kdbu2W7ZqHY3q65bE7jb+sILK1WsyfeT9jL+9H9NHPciObVup02h/vvko9h+qX8+Zyea1q3fts2Lx5/xvSF/GDr2aTn+4Pu1axZBefcZ7axnfv5flH7/inHcUtsLM+prZh2b24VOjRvyKUyTPX4YM48XnxtP30p7k5uZSvlz5n61fOP9TKlaqRMsDWqWohuFULjODOjWr0vHSf/DXB1/gqfsuT3WVIsUL8ln9XTaHnXI2F9z+COUrVGLe5PF0vmwA8998mQnDrmP71i1klPup57FBy0O46M7h9Lj1IeZOHk/eju0pvILkSKe5Kfb20McpiR7UzD4tbBWQtZdzDgeGAyzfsN0TPX8y7d+8Jf94eDgAS7/7ltnvzvzZ+umvv8qpp5+ZiqqF2rKV63lh2scAfLjgOwoKnLq1q7Fs9QaaNKi9a7vG9Wvz/aoNqapmaFWtXZdqtevSoGXsPsQBbTswb/J42p3Xm3MGxm5url+Rw3efffCLfes0akb5ipVZu+xb6jc/aJ/WO9kyQxyuJVWcoW2JyAIuBX67h2VNks65T6xbG6t+QUEBT44czjm/67lrXUFBATOmvU7n07umqnqh9dKMTzn52FgQHNisPhXKl+OHdZt5Zcan9DijDRXKl2P/RvtxYLN6zJn/bWorG0JVa9ahWp16rFuxFICcRR9Ru1EzcjeuB8ALCvjw5bEcdvJZAGxcvYKC/Njb0Tb+sJJ1y5dSfb9C20GRlYwpNM3sRjNbYGbzzWysmVUysxZm9r6ZZZvZeDOrEGxbMfieHaxvnui1FOsJvAS8DFRz9493X2FmM5J0zlI37Na/8PHcOWxYv57zzz6Vy67sx5YtubzwzDgAOpxyKt1+e+6u7T/5aC71shrQqHGZGPlXqNF/+yMdftOKurWqkf3andz52GRGvzCL/9x+MR8+81e278jnitueBGDR4hU89/pHfPTcLeTlF/CneyZQUOB7PU5Z1eGia5k6/D4K8ndQo25DOl8+gC/em8Znb74EwAFtTuTQk04HYPlX85n76gQyMsthZpx8yXVUrp5+ffGlPTeFmTUGbgBau/sWM5sA9ALOBB5093Fm9hjQB/h38Oc6dz/QzHoB9xKbTK3k53YPZW9AaLspoqZlpwGprkLk3fvwwFRXIW3ccFKLUo3PgS99UeKcuP+3BxdahyCMZxN7uG0j8ALwMPA00MDd88zseOB2dz/DzKYEn2eZWTlgBVDPEwjW4rzpw8zsEjO7LfjezMyOK+mJRERKWyLdFPEDBYKl787jufsyYgMUlgDLgQ3Enq9Y7+55wWY5xN4JSvDn0mDfvGD7/RK5luJ0UzwKFACdgWHAJuA54NhETigiUloSuX8XP1Dgl8ez2kB3oAWwntibjfbJTaDihHE7d29jZh8BuPu6nZ3XIiKplISHOE4DvnH31QBmNhE4EahlZuWC1m8TYFmw/TJi00PkBN0UNUlwkEJxRlPsMLNMYhPKY2b1iLWURURSKiOBpQhLgPZmVsVig5JPBRYCbxKbNA2gN/Bi8HlS8J1g/fRE+ot3XktRHgKeB+qb2V3Eps/UrC0iknKl/QSeu78PPAvMAz4jlpHDgZuAAWaWTaxP+IlglyeA/YLyAcDgRK+lyG4Kd3/azOYS+xfCgHPdfVGiJxQRKS3JmGvC3YcCQ3crXgz8YuCCu28FepTGeYszuXwzIBd4Kb7M3ZeURgVERBKVRg/gFesG3iv89GLSSsTuMn4BHJbEeomIFKlMvJB0J3c/Iv57MGPbtUmrkYhIMYV5SsySKvHj0O4+z8zaJaMyIiIlkUZZXKw+4/jnaTOANsD3SauRiEgxlaluCqB63Oc8Yn3IzyWnOiIixWekTxrvNYyDhz2qu/ugfVQfEZFiKxMt452P/plZ+r1SVkTSQpkIY+ADYv3DH5vZJGITZvy4c6W7T0xy3URE9irMr1EqqeL0GVciNvFFZ34ab+yAwlhEUqqstIzrByMp5vNTCO+kid9FJOXSqGG81zDOBKrBHm9XKoxFJOXKykMfy9192D6riYhICZWVboo0ukwRSUdp1DDeaxifus9qISKSgMw0SuNCw9jd1+7LioiIlFRZ6aYQEQm1snIDT0Qk1NIoixXGIhJdahmLiIRAGmWxwlhEoqs4r7ePCoWxiERWWZsoSEQklNInitOrlS8iZUyGWYmXophZLTN71sw+N7NFZna8mdUxs6lm9lXwZ+1gWzOzh8ws28w+DV7YnNi1JLqjiEiqWQJLMfwLeM3dDwGOAhYBg4Fp7t4KmBZ8B+gGtAqWvsC/E70WhbGIRJZZyZe9H89qAh2BJwDcfbu7rwe6A6ODzUYD5wafuwNjPGY2UMvMGiZyLQpjEYksM0tk6WtmH8YtfeMO2QJYDfzXzD4ysxFmVhXIcvflwTYrgKzgc2Ngadz+OUFZiekGnohEViKtSXcfDgwvZHU5Yq+bu97d3zezf/FTl8TO/d3MSn1Od7WMRSSyEmkZFyEHyHH394PvzxIL55U7ux+CP1cF65cBTeP2bxKUlVhoW8a1q1ZIdRXSwjNPDkl1FSKvx8AnU12FtHHDlEGlerzSHtrm7ivMbKmZHezuXxCbSnhhsPQG7gn+fDHYZRJwnZmNA9oBG+K6M0oktGEsIlKUJD30cT3wtJlVABYDlxHrRZhgZn2A74CewbaTgTOBbCA32DYhCmMRiaxk9LO6+8dA2z2s+sULN9zdgX6lcV6FsYhElh6HFhEJgfSJYoWxiERYGjWMFcYiEl0ZadQ2VhiLSGSpZSwiEgJ67ZKISAiom0JEJATSqGGsMBaR6FIYi4iEgKmbQkQk9TLSJ4sVxiISXWoZi4iEgPqMRURCQC1jEZEQUJ+xiEgIqGUsIhIC6jMWEQmBNMpihbGIRJcmChIRCYH0iWKFsYhEWRqlscJYRCJLoylEREIgjbqMyUh1BUREEmUJLMU6rlmmmX1kZi8H31uY2ftmlm1m482sQlBeMfieHaxvnui1KIxFJLqSlcbQH1gU9/1e4EF3PxBYB/QJyvsA64LyB4PtEqIwFpHIsgT+V+QxzZoAZwEjgu8GdAaeDTYZDZwbfO4efCdYf2qwfYkpjEUksswSWayvmX0Yt/Td7bD/BP4CFATf9wPWu3te8D0HaBx8bgwsBQjWbwi2LzHdwBORyEqkCeruw4Hhezye2dnAKnefa2adfk3dSkphLCLRVfqjKU4EzjGzM4FKQA3gX0AtMysXtH6bAMuC7ZcBTYEcMysH1ATWJHJidVOISGRlmJV42Rt3v9ndm7h7c6AXMN3dLwbeBM4PNusNvBh8nhR8J1g/3d09oWtJZCcRkTBI3mCKX7gJGGBm2cT6hJ8Iyp8A9gvKBwCDEz2BuilEJLqS+NCHu88AZgSfFwPH7WGbrUCP0jifwlhEIkuPQ4uIhEA6PQ6tMBaRyEqjLFYYl8TGjRu547Zbyc7+EjPjjjvv5qijjwFg9KiRPPD3e5nxzixq166T4pqGy/9d3ZOKlSuTkZFJRmYmN973OK+OHcGCD97BMjKoVrMWva77KzXr1CV38ybGP3IPa1Yso1yFClzQbzANm7VM9SWkxGMDzqBbuwNYvT6XtleN+tm6/r9vyz19O9GkxyOs2biFG88/lgs6HwpAucwMDmlah6YXPErVSuUZ8edu1K9VFccZOflTHnlhXgquJknSKI0VxiVw39/u4sSTOnD/Px9ix/btbNm6FYAVy5cz6913adiwUYprGF7X3PEvqtWotev7Kd0vpNuFVwDw9ivPMvWZUZx/1SCmPfckjVocyGU33cXKnO+YOOJBrrn9n6mqdko9+foCHpv0ESP+fObPypvUq86pbfZnycqNu8oefHYODz47B4Az27Xk+t+1Zd2mrVQsn8ng4TP4OHsV1SqX573/9wemzfuOz5ckNBQ2dNKpzzhpQ9vM7BAzO9XMqu1W3jVZ50ymTZs2MXfuHM77fWyoYfkKFahRowYAf7/3b9w48M8k+Eh6mVSpStVdn7dv28rOJs7KnG9pdXgbALKa7M+6VSvYtH5tKqqYcu/Oz2Htpq2/KL/vqlO45YmZFDactecphzJhRmyOmxVrf+Tj7FUAbN6yg8+XrqVR3Wp73C+KEnkcOqySEsZmdgOxQdHXA/PNrHvc6ruTcc5kW5aTQ+3adbjtlpvp+ftzuf22W8jNzeXN6W9QP6s+Bx9ySKqrGFpmMHzYQB788xXMen3SrvLJTz/OsL6/Z97MqXTtFZsEq1HzA/ns/ZkALPlqIetWr2T9mtUpqXcYnX38AXz/wyY+W7zn36RyxXJ0aducF9756hfrmmXV4OgD6jPn8+XJruY+sw/HGSddslrGVwK/cfdzgU7AEDPrH6wr9PeIn8Djicf3+Oh4yuTn5/H5ooX06HUhE557gcqVK/PYow8zYvh/uPa6/kUfoAy77v8eYcA/nuCKW//Ou689z9cLPgbgzIuv5Lbhz9GmYxfeeXUiAJ3Pu5gtP27m/oGX887kiTRu0YqMDD2bBLGg/Uuv9gwb826h25zV/gBmLfiedbu1qKtWKs/YIefw58feZFPu9mRXdd9JozRO1t/yDHffDODu3xIL5G5m9gW2DjcAAAkvSURBVAB7+Tncfbi7t3X3tn2u3H0ipdTKympAVlYDjjzyKAC6nN6VRQsXsmxZDj1/151uXTqzcuUKep3/O35YrZZcvJr71QOges3aHNGuA0uyF/1sfZsOXfhs9ltArPui13U3M/D+kVx4wy1s3rie/bLUFw/QsmEt9m9Qkw/+3ZvPR19J43rVmfXIH8iqXWXXNj1OPoRnZvz89y2XmcHYIecwfvoiXnz3ly3mKEvGFJqpkqwwXmlmR+/8EgTz2UBd4IgknTOp6tarR1aDBnz7zWIA3p89i0Nbt2bG27N4dep0Xp06naysBox7diJ169VLcW3DY9vWLWzdkrvr8xefzKFhs5as/n7prm3mz3mH+o2bAbDlx03k7dgBwPtvvEzL1kf9rH+5LFvw7Q/sf8GjHNL7cQ7p/TjLVm/i+H5PsnJd7PetUaUCJx3ZhJfe+/pn+z024Ay+WLqWhybOTUW1kyqd+oyTNZriUiAvviCY7ehSM/tPks6ZdIP/OoSbbxrEjh07aNKkKcP+72+prlLobV6/jv/edwsABfn5tOlwGocc045R993K6u+XYmbUrteA868aCMDKnO8Y+/DdmBkNmjan57UJP+ofeaMHn0WHI5tSt2Zlsp+6ijuffJfRU+YXuv05J7Zi2tzvyN22Y1fZCYc15uLTDuOzxauZ/eilAAz979tMmfNN0uu/L4Q4W0vMEpxgKOm25hHOikXMG5+vTHUVIq/HwCdTXYW0sWXKoFLNzy9X5pY4Jw7KqhLKDNc4YxGJrDD3AZeUwlhEIivMfcAlpTAWkchKoyxWGItIhKVRGiuMRSSy1GcsIhIC6jMWEQmBNMpihbGIRFgapbHCWEQiKyON+ikUxiISWekTxQpjEYmwNGoYJ+9NHyIiyVe6ExqbWVMze9PMFprZgp3zsJtZHTObamZfBX/WDsrNzB4ys2wz+9TM2iR6JQpjEYmsJEyhmQcMdPfWQHugn5m1BgYD09y9FTAt+A7QDWgVLH2Bfyd6LQpjEYms0n7Rh7svd/d5wedNwCKgMdAdGB1sNho4N/jcHRjjMbOBWmbWMJFrURiLSGQl0jKOf71bsOzxtUJm1hw4BngfyHL3nS8PXAFkBZ8bA0vjdssJykpMN/BEJLISeRza3YcDe33JZvBW++eAP7n7xvg3v7u7m1mpz7eulrGIRFcSXkhqZuWJBfHT7j4xKF65s/sh+HNVUL4MaBq3e5OgrMQUxiISWaWdxRZrAj8BLHL3B+JWTQJ6B597Ay/GlV8ajKpoD2yI684oEXVTiEhkJWGc8YnAH4DPzOzjoOyvwD3ABDPrA3wH9AzWTQbOBLKBXOCyRE+sMBaRyCrtKTTd/R0Kb0CfuoftHehXGudWGItIdKXRE3gKYxGJrDTKYoWxiERXOs1NoTAWkcjSa5dEREIgnVrGGmcsIhICahmLSGSlU8tYYSwikaU+YxGREFDLWEQkBNIoixXGIhJhaZTGCmMRiSz1GYuIhEBG+mSxwlhEIkxhLCKSeuqmEBEJgXQa2maxuZElEWbWN3i5ofwK+h1Lh37HaNPcFL/OHl/xLSWm37F06HeMMIWxiEgIKIxFREJAYfzrqH+udOh3LB36HSNMN/BEREJALWMRkRBQGIuIhIDCOEFm1tXMvjCzbDMbnOr6RJGZjTSzVWY2P9V1iSoza2pmb5rZQjNbYGb9U10nSYz6jBNgZpnAl0AXIAeYA1zo7gtTWrGIMbOOwGZgjLsfnur6RJGZNQQauvs8M6sOzAXO1d/F6FHLODHHAdnuvtjdtwPjgO4prlPkuPtMYG2q6xFl7r7c3ecFnzcBi4DGqa2VJEJhnJjGwNK47zno/wCSYmbWHDgGeD+1NZFEKIxF0oCZVQOeA/7k7htTXR8pOYVxYpYBTeO+NwnKRPY5MytPLIifdveJqa6PJEZhnJg5QCsza2FmFYBewKQU10nKIDMz4Algkbs/kOr6SOIUxglw9zzgOmAKsRsmE9x9QWprFT1mNhaYBRxsZjlm1ifVdYqgE4E/AJ3N7ONgOTPVlZKS09A2EZEQUMtYRCQEFMYiIiGgMBYRCQGFsYhICCiMRURCQGFcBplZfjAEar6ZPWNmVX7FsUaZ2fnB5xFm1nov23YysxMSOMe3Zla3uOW7bbO5hOe63cwGlbSOIr+Wwrhs2uLuRwczpW0Hro5faWblEjmou19RxGxhnYASh7FIWaAwlreBA4NW69tmNglYaGaZZvZ3M5tjZp+a2VUQe+LLzP5fMJfzG0D9nQcysxlm1jb43NXM5pnZJ2Y2LZjE5mrgxqBV3sHM6pnZc8E55pjZicG++5nZ68H8vCMAK+oizOwFM5sb7NN3t3UPBuXTzKxeUHaAmb0W7PO2mR1SGj+mSKISagFJeghawN2A14KiNsDh7v5NEGgb3P1YM6sIvGtmrxObFexgoDWQBSwERu523HrA40DH4Fh13H2tmT0GbHb3fwTb/Q940N3fMbNmxJ5oPBQYCrzj7sPM7CygOE/mXR6cozIwx8yec/c1QFXgQ3e/0cxuC459HbGXd17t7l+ZWTvgUaBzAj+jSKlQGJdNlc3s4+Dz28TmNjgB+MDdvwnKTweO3NkfDNQEWgEdgbHung98b2bT93D89sDMncdy98LmLD4NaB2bXgGAGsHsYx2B3wX7vmJm64pxTTeY2XnB56ZBXdcABcD4oPwpYGJwjhOAZ+LOXbEY5xBJGoVx2bTF3Y+OLwhC6cf4IuB6d5+y23alOe9BBtDe3bfuoS7FZmadiAX78e6ea2YzgEqFbO7Bedfv/huIpJL6jKUwU4BrgukZMbODzKwqMBO4IOhTbgicsod9ZwMdzaxFsG+doHwTUD1uu9eB63d+MbOd4TgTuCgo6wbULqKuNYF1QRAfQqxlvlMGsLN1fxGx7o+NwDdm1iM4h5nZUUWcQySpFMZSmBHE+oPnWeyFof8h9l9SzwNfBevGEJt17WfcfTXQl1iXwCf81E3wEnDezht4wA1A2+AG4UJ+GtVxB7EwX0Csu2JJEXV9DShnZouAe4j9Y7DTj8BxwTV0BoYF5RcDfYL6LUCvzZIU06xtIiIhoJaxiEgIKIxFREJAYSwiEgIKYxGREFAYi4iEgMJYRCQEFMYiIiHw/wH3QtRcXan/mwAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lbJo7jLmrZ5h"
      },
      "source": [
        "### Comparative Analysis of different Neural Networks\n",
        "\n",
        "General Good Settings: feature_count=15000, embeddings_sequence_length=50, batch_size=256, epochs=5.\n",
        "\n",
        "* My Parameters for IMDb Dataset (average length of instance is 238, max is 2494): random_state=22, feature_count=20000, remove_first=False, embeddings_mode=Word2Vec Pretrained, embeddings_sequence_length=256, Each OOV word set to a unique random vector, trainable=True, batch_size=256, epochs=EarlyStopping, activation=sigmoid, optimizer='adam', learning_rate=0.00005, validation_split=0.0, neurons=128.\n",
        "\n",
        "* My Parameters For Subjectivity Dataset (average length of instance is 21.5, max is 111): Obviously we want a significantly shorter sequence length, a bit smaller batch size and a higher learning rate since the nature of the dataset is very different the the IMDb one. embeddings_sequence_length=32, batch_size=128, learning_rate=0.0001.\n",
        "\n",
        "* My Parameters For Polarity Dataset (average length of instance is 18.8, max is 51): Obviously we want a significantly shorter sequence length, a bit smaller batch size and a higher learning rate since the nature of the dataset is very different the the IMDb one. embeddings_sequence_length=32, batch_size=128, learning_rate=0.00005.\n",
        "\n",
        "It is worth noting that the results are kind of random/useless because of the small size of the dataset as well as the fast convergence. Obviously, here we train different models on different number of epochs instead of comparing a set of models on the same number of epochs.\n",
        "\n",
        "| Model | Author | Performance (loss /acc) | Epochs | Embeddings Dimension | Note |\n",
        "| - | - | - | - | - | - |\n",
        "| Single LSTM | Me | 0.341 / 86.052 | 11 | 100 | This is run with One-hot Encoding - It reaches convergence too fast for Dropout or Deep to matter |\n",
        "| Single LSTM with Dropout | Me | 0.339 / 86.556 | 8 | 100 | This is run with One-hot Encoding |\n",
        "| Deep Stacked LSTMs with Dropout | Me | 0.346 / 85.188 | 4 | 100 | This is run with One-hot Encoding |\n",
        "| Multilayer Perceptron | Me | 0.294 / 87.708 | 17 | 300 | Now we move to Word2Vec Pretrained |\n",
        "| Multilayer Perceptron with Average Pooling | Me | 0.287 / 88.404 | 29 | 300 | - |\n",
        "| Single LSTM | Me | 0.303 / 87.240 | 9 | 300 | - |\n",
        "| Single LSTM with Dropout | Me | 0.307 / 87.048 | 12 | 300 | - |\n",
        "| Bidirectional LSTM | Me | 0.297 / 87.456 | 10 | 300 | - |\n",
        "| Bidirectional LSTM with Dropout | Me | 0.312 / 86.828 | 9 | 300 | - |\n",
        "| Deep Stacked LSTMs  | Me | 0.326 / 86.396 | 8 | 300 | - |\n",
        "| Deep Stacked LSTMs with Dropout | Me | 0.327 / 86.938 | 11 | 300 | - |\n",
        "| Simple RNN | Me | 0.342 / 85.844 | 12 | 300 | - |\n",
        "| Bidirectional Simple RNN | Me | 0.344 / 85.640 | 14 | 300 | - |\n",
        "| GRU | Me | 0.298 / 87.776 | 7 | 300 | - |\n",
        "| GRU with Dropout | Me | 0.298 / 87.572 | 11 | 300 | - |\n",
        "| CNN with Average Pooling | Me | 0.286 / 88.424 | 4 | 300 | lr=0.0005 |\n",
        "| LSTM inspired from Paper | [Liu et al.](https://aclweb.org/anthology/D15-1280) | 0.281 / 88.544 | 1 | 300 | We use the exact parameters from the paper with batch_size=128 and lr=0.05. Paper's accuracy: 88.50 |\n",
        "| mLSTM inspired from Paper | [Krause et al.](https://arxiv.org/pdf/1609.07959.pdf) | 0.356 / 87.84 | 4 | 300 | (Useless since we lack processing power)) We use similar parameters to those from the paper but no dropout, units=128, batch_size=512. [Code](https://github.com/titu1994/Keras-Multiplicative-LSTM). Paper's accuracy: - |\n",
        "| mLSTM inspired from Paper | [Krause et al.](https://arxiv.org/pdf/1609.07959.pdf) | 0.320 / 85.95 | 4 | 300 | (Useless since we lack processing power) We don't perform all the extra things the paper does, units=1000, batch_size=512. Paper's accuracy: - |\n",
        "\n",
        "| Model (on just 1 fold of cross_val) | Author | Performance | Epochs | Embeddings Dimension | Note |\n",
        "| - | - | - | - | - | - |\n",
        "| Single LSTM | Me | 90.400 | 7 | 100 | This is run with One-hot Encoding - It reaches convergence too fast for Dropout or Deep to matter |\n",
        "| Single LSTM with Dropout | Me | 90.400 | 9 | 100 | This is run with One-hot Encoding |\n",
        "| Deep Stacked LSTMs with Dropout | Me | 89.500 | 6 | 100 | This is run with One-hot Encoding |\n",
        "| Multilayer Perceptron | Me | 91.400 | 8 | 300 | Now we move to Word2Vec Pretrained |\n",
        "| Multilayer Perceptron with Average Pooling | Me | 92.300 | 10 | 300 | - |\n",
        "| Single LSTM | Me | 90.400 | 6 | 300 | - |\n",
        "| Single LSTM with Dropout | Me | 90.900 | 6 | 300 | - |\n",
        "| Bidirectional LSTM | Me | 90.500 | 5 | 300 | - |\n",
        "| Bidirectional LSTM with Dropout | Me | 90.200 | 5 | 300 | - |\n",
        "| Deep Stacked LSTMs  | Me | 90.400 | 4 | 300 | - |\n",
        "| Deep Stacked LSTMs with Dropout | Me | 90.700 | 6 | 300 | - |\n",
        "| Simple RNN | Me | 89.200 | 7 | 300 | - |\n",
        "| Bidirectional Simple RNN | Me | 90.300 | 7 | 300 | - |\n",
        "| GRU | Me | 90.800 | 7 | 300 | - |\n",
        "| GRU with Dropout | Me | 91.200 | 7 | 300 | - |\n",
        "| CNN with Average Pooling | Me | 91.700 | 14 | 300 | lr=0.0005 |\n",
        "| LSTM inspired from Paper | [Liu et al.](https://aclweb.org/anthology/D15-1280) | 91.100 | 1 | 300 | We use the exact parameters from the paper with units=60, batch_size=128 and lr=0.05. Paper's accuracy: - |\n",
        "| mLSTM inspired from Paper | [Krause et al.](https://arxiv.org/pdf/1609.07959.pdf) | 91.900 | 3 | 300 | (Useless since we lack processing power)) We use similar parameters to those from the paper but no dropout, units=128, batch_size=512. [Code](https://github.com/titu1994/Keras-Multiplicative-LSTM). Paper's accuracy: - |\n",
        "| mLSTM inspired from Paper | [Krause et al.](https://arxiv.org/pdf/1609.07959.pdf) | 90.000 | 4 | 300 | (Useless since we lack processing power) We don't perform all the extra things the paper does, units=1000, batch_size=128. Paper's accuracy: - |\n",
        "\n",
        "| Model (on just 1 fold of cross_val) | Author | Performance | Epochs | Embeddings Dimension | Note |\n",
        "| - | - | - | - | - | - |\n",
        "| Single LSTM | Me | 72.352 | 8 | 100 | This is run with One-hot Encoding - It reaches convergence too fast for Dropout or Deep to matter |\n",
        "| Single LSTM with Dropout | Me | 72.638 | 10 | 100 | This is run with One-hot Encoding |\n",
        "| Deep Stacked LSTMs with Dropout | Me | 71.516 | 7 | 100 | This is run with One-hot Encoding |\n",
        "| Multilayer Perceptron | Me | 77.226 | 14 | 300 | Now we move to Word2Vec Pretrained |\n",
        "| Multilayer Perceptron with Average Pooling | Me | 78.257 | 13 | 300 | - |\n",
        "| Single LSTM | Me | 77.130 | 8 | 300 | - |\n",
        "| Single LSTM with Dropout | Me | 75.351 | 6 | 300 | - |\n",
        "| Bidirectional LSTM | Me | 78.913 | 5 | 300 | - |\n",
        "| Bidirectional LSTM with Dropout | Me | 76.859 | 7 | 300 | - |\n",
        "| Deep Stacked LSTMs  | Me | 72.541 | 1 | 300 | - |\n",
        "| Deep Stacked LSTMs with Dropout | Me | 72.259 | 4 | 300 | - |\n",
        "| Simple RNN | Me | 75.914 | 11 | 300 | - |\n",
        "| Bidirectional Simple RNN | Me | 73.482 | 8 | 300 | - |\n",
        "| GRU | Me | 78.163 | 8 | 300 | - |\n",
        "| GRU with Dropout | Me | 78.738 | 12 | 300 | - |\n",
        "| CNN with Average Pooling | Me | 79.007 | 17 | 300 | lr=0.0005 |\n",
        "| LSTM inspired from Paper | [Liu et al.](https://aclweb.org/anthology/D15-1280) | 79.663 | 2 | 300 | We use the exact parameters from the paper with units=60, batch_size=128 and lr=0.005. Paper's accuracy: - |\n",
        "| mLSTM inspired from Paper | [Krause et al.](https://arxiv.org/pdf/1609.07959.pdf) | 80.506 | 2 | 300 | (Useless since we lack processing power)) We use similar parameters to those from the paper but no dropout, units=128, batch_size=512. [Code](https://github.com/titu1994/Keras-Multiplicative-LSTM). Paper's accuracy: - |\n",
        "| mLSTM inspired from Paper | [Krause et al.](https://arxiv.org/pdf/1609.07959.pdf) | 78.164 | 7 | 300 | (Useless since we lack processing power) We don't perform all the extra things the paper does, units=1000, batch_size=128. Paper's accuracy: - |"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KFeiK5pBQc3x"
      },
      "source": [
        "## Evaluate on External Data"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "trwlJUSBXskX"
      },
      "source": [
        "### Preprocessing Run\n",
        "\n",
        "FIRST, execute the entire preprocessing phase again (there are more efficient ways, but this works universally)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WcULC7A4XrGJ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "14f0ab48-ce96-4662-f2e7-b1b97e17c1a9"
      },
      "source": [
        "###   INPUT    ###\n",
        "test_sentence = [\"very bad movie, wow awful such a bad performance from the actors.\"]\n",
        "###            ###\n",
        "\n",
        "### Parameters ###\n",
        "remove_first = False #@param {type:\"boolean\"}\n",
        "embeddings_mode = \"One-hot Encoding\" #@param [\"One-hot Encoding\", \"Tokenizing\", \"Word2Vec Pretrained\", \"Word2Vec Training\"]\n",
        "embeddings_sequence_length = 50 #@param {type:\"integer\"}\n",
        "trainable = False #@param {type:\"boolean\"}\n",
        "outofvocab_mode = \"random\" #@param [\"zeros\", \"random\"]\n",
        "###            ###\n",
        "\n",
        "# Load Dataset\n",
        "train_data, test_data, train_labels, test_labels = load_dataset()\n",
        "print_dataset_length_stats(train_data)\n",
        "\n",
        "# Remove the <START> symbol from all instances\n",
        "if remove_first == True:\n",
        "    if train_data[0][0] == word_index[\"<START>\"]:\n",
        "        for i in range(len(train_data)):\n",
        "            train_data[i] = train_data[i][1:]\n",
        "        for i in range(len(test_data)):\n",
        "            test_data[i] = test_data[i][1:]\n",
        "\n",
        "# Create the Embeddings            \n",
        "if embeddings_mode == \"One-hot Encoding\":  \n",
        "    ''' \n",
        "        ONE-HOT ENCODING\n",
        "        Description: Not traditional One-hot, but instead [3, 62, 5, 90, ...] \n",
        "        Embedding_Layer: Yes\n",
        "    '''   \n",
        "    if dataset_name != \"IMDb Large Movie Review Dataset\":  # keras.imdb is already One-hot Encoded\n",
        "        tokenizer = Tokenizer(num_words=feature_count, \n",
        "                              lower=True, \n",
        "                              split=' ', \n",
        "                              oov_token=\"<UNK>\")\n",
        "        tokenizer.fit_on_texts(train_data)\n",
        "        # 'texts_to_sequences' list of strings as input and sequence of integers as output, 'texts_to_matrix' is meant to return a matrix of counts/tf-idfs\n",
        "        train_data = tokenizer.texts_to_sequences(train_data)\n",
        "        test_data = tokenizer.texts_to_sequences(test_data)  \n",
        "    \n",
        "    # Word_Index Stuff\n",
        "    if dataset_name == \"IMDb Large Movie Review Dataset\":  # keras.imdb is already One-hot Encoded\n",
        "        word_index, reverse_word_index = imdb_specific_word_index()\n",
        "    else:\n",
        "        word_index, reverse_word_index = other_datasets_word_index(tokenizer)\n",
        "\n",
        "    word_index, reverse_word_index = reduce_word_index_features(word_index, reverse_word_index, train_data)  # Update the word index to match the number of features we selected       \n",
        "\n",
        "    # NEW\n",
        "    test_sentence = encode_review(test_sentence[0], word_index)\n",
        "    test_sentence = np.array([test_sentence])  # Convert back to 2D array even if there is no use yet\n",
        "\n",
        "    # Peform Sequence Padding\n",
        "    train_data, test_data = sequence_padding(train_data, test_data, embeddings_sequence_length)\n",
        "    # NEW\n",
        "    test_sentence, _ = sequence_padding(test_sentence, test_sentence, embeddings_sequence_length) \n",
        "\n",
        "elif embeddings_mode == \"Tokenizing\":  \n",
        "    ''' \n",
        "        TOKENIZING\n",
        "        Description: Unlike other modes this isn't exactly an embedding, leads to a collection of floats [0.00, 0.02, 0.12, 0.04, ...] based on tf-idf \n",
        "        Embedding_Layer: No\n",
        "    '''    \n",
        "    if dataset_name == \"IMDb Large Movie Review Dataset\":  # keras.imdb is already One-hot Encoded so let's transform it back to text\n",
        "        word_index, reverse_word_index = imdb_specific_word_index()\n",
        "        train_data = [decode_review(instance, reverse_word_index, mode=\"join\") for instance in train_data]\n",
        "        test_data = [decode_review(instance, reverse_word_index, mode=\"join\") for instance in test_data]\n",
        "    \n",
        "    tokenizer = Tokenizer(num_words=feature_count, \n",
        "                          lower=True, \n",
        "                          split=' ', \n",
        "                          )\n",
        "    tokenizer.fit_on_texts(train_data)\n",
        "    # 'texts_to_matrix' list of strings as input, 'sequences_to_matrix' list of integer word indices as input \n",
        "    train_data = tokenizer.texts_to_matrix(train_data, mode='tfidf')\n",
        "    test_data = tokenizer.texts_to_matrix(test_data, mode='tfidf')\n",
        "\n",
        "     # NEW\n",
        "    test_sentence = tokenizer.texts_to_matrix(test_sentence, mode='tfidf')\n",
        "  \n",
        "elif embeddings_mode == \"Word2Vec Pretrained\":\n",
        "    ''' \n",
        "        WORD2VEC PRETRAINED\n",
        "        Description: A much more advanced form of embeddings that is created through training a model unlike previous modes. Implements the CBOW and the Skip-gram models in order to learn word embeddings.\n",
        "        Embedding_Layer: Yes\n",
        "    '''\n",
        "    word2vec = load_word2vec_pretrained()  # This line can be commented out, if it was already loaded in the current session\n",
        "    embeddings_dimension = word2vec.vector_size\n",
        "    \n",
        "    if dataset_name != \"IMDb Large Movie Review Dataset\":  # keras.imdb is already One-hot Encoded\n",
        "        tokenizer = Tokenizer(num_words=feature_count, \n",
        "                              lower=True, \n",
        "                              split=' ', \n",
        "                              oov_token=\"<UNK>\")\n",
        "        tokenizer.fit_on_texts(train_data)\n",
        "        # 'texts_to_sequences' list of strings as input and sequence of integers as output, 'texts_to_matrix' is meant to return a matrix of counts/tf-idfs\n",
        "        train_data = tokenizer.texts_to_sequences(train_data)\n",
        "        test_data = tokenizer.texts_to_sequences(test_data)   \n",
        "    \n",
        "    # Word_Index Stuff\n",
        "    if dataset_name == \"IMDb Large Movie Review Dataset\":  # keras.imdb is already One-hot Encoded\n",
        "        word_index, reverse_word_index = imdb_specific_word_index()\n",
        "    else:\n",
        "        word_index, reverse_word_index = other_datasets_word_index(tokenizer)\n",
        "\n",
        "    word_index, reverse_word_index = reduce_word_index_features(word_index, reverse_word_index, train_data)  # Update the word index to match the number of features we selected       \n",
        "\n",
        "    # NEW\n",
        "    test_sentence = encode_review(test_sentence[0], word_index)\n",
        "    test_sentence = np.array([test_sentence])  # Convert back to 2D array even if there is no use yet\n",
        "\n",
        "    # Peform Sequence Padding  \n",
        "    train_data, test_data = sequence_padding(train_data, test_data, embeddings_sequence_length)\n",
        "    # NEW\n",
        "    test_sentence, _ = sequence_padding(test_sentence, test_sentence, embeddings_sequence_length)  \n",
        "\n",
        "    manage_oov_words(word2vec, word_index)\n",
        "    embedding_vectors = assign_embeddings(word2vec, embeddings_dimension, word_index, mode=outofvocab_mode)\n",
        "\n",
        "elif embeddings_mode == \"Word2Vec Training\":\n",
        "    ''' \n",
        "        Description: A much more advanced form of embeddings that is created through training a model unlike previous modes. Implements the CBOW and the Skip-gram models in order to learn word embeddings.\n",
        "        Embedding_Layer: Yes\n",
        "    '''\n",
        "    if dataset_name != \"IMDb Large Movie Review Dataset\":  # keras.imdb is already One-hot Encoded so let's transform it back to text\n",
        "        tokenizer = Tokenizer(num_words=feature_count, \n",
        "                              lower=True, \n",
        "                              split=' ', \n",
        "                              oov_token=\"<UNK>\")\n",
        "        tokenizer.fit_on_texts(train_data)\n",
        "        # 'texts_to_sequences' list of strings as input and sequence of integers as output, 'texts_to_matrix' is meant to return a matrix of counts/tf-idfs\n",
        "        train_data = tokenizer.texts_to_sequences(train_data)\n",
        "        test_data = tokenizer.texts_to_sequences(test_data)  \n",
        "        \n",
        "    # Word_Index Stuff\n",
        "    if dataset_name == \"IMDb Large Movie Review Dataset\":  # keras.imdb is already One-hot Encoded so let's transform it back to text\n",
        "        word_index, reverse_word_index = imdb_specific_word_index()\n",
        "    else:\n",
        "        word_index, reverse_word_index = other_datasets_word_index(tokenizer)\n",
        "\n",
        "    word_index, reverse_word_index = reduce_word_index_features(word_index, reverse_word_index, train_data)  # Update the word index to match the number of features we selected       \n",
        "      \n",
        "    # Train a Model\n",
        "    time_counter = time.time()    \n",
        "    print(f\"\\nTraining a new custom Word2Vec model on the dataset...\")\n",
        "    word2vec = Word2Vec([decode_review(instance, reverse_word_index, mode=\"don't join\") for instance in train_data],  # Default is 5 epochs \n",
        "                         size=300, sg=0, window=5, min_count=1, iter=5,\n",
        "                         seed=random_state, alpha=0.025, workers=4)\n",
        "    embeddings_dimension = word2vec.wv.vector_size\n",
        "    print(f\"Training completed in {time.time()-time_counter:.2f}sec. Vocabulary size: {len(word2vec.wv.vocab)}\\n\")  \n",
        "\n",
        "    # Peform Sequence Padding  \n",
        "    train_data, test_data = sequence_padding(train_data, test_data, embeddings_sequence_length) \n",
        "    \n",
        "    manage_oov_words(word2vec.wv, word_index)   \n",
        "    embedding_vectors = assign_embeddings(word2vec.wv, embeddings_dimension, word_index, mode=outofvocab_mode)    \n",
        "    \n",
        "    # If we want to store the model\n",
        "    if False:\n",
        "        model.wv.save_word2vec_format(\"My_Word2Vec.txt\", binary=False)    \n",
        "\n",
        "# Print the resulting instances       \n",
        "for i in range(4): print(type(train_data[i]), list(train_data[i]))"
      ],
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Loaded Subset ./gdrive/My Drive/Colab Datasets/SemEval-2017 Task 4/GOLD/Subtask_A/sms-2013test-A.tsv\n",
            "Loaded Subset ./gdrive/My Drive/Colab Datasets/SemEval-2017 Task 4/GOLD/Subtask_A/livejournal-2014test-A.tsv\n",
            "Loaded Subset ./gdrive/My Drive/Colab Datasets/SemEval-2017 Task 4/GOLD/Subtask_A/twitter-2016devtest-A.txt\n",
            "Loaded Subset ./gdrive/My Drive/Colab Datasets/SemEval-2017 Task 4/GOLD/Subtask_A/twitter-2016dev-A.txt\n",
            "Loaded Subset ./gdrive/My Drive/Colab Datasets/SemEval-2017 Task 4/GOLD/Subtask_A/twitter-2016train-A.txt\n",
            "Loaded Subset ./gdrive/My Drive/Colab Datasets/SemEval-2017 Task 4/GOLD/Subtask_A/twitter-2013dev-A.txt\n",
            "Loaded Subset ./gdrive/My Drive/Colab Datasets/SemEval-2017 Task 4/GOLD/Subtask_A/twitter-2013test-A.txt\n",
            "Loaded Subset ./gdrive/My Drive/Colab Datasets/SemEval-2017 Task 4/GOLD/Subtask_A/twitter-2013train-A.txt\n",
            "Loaded Subset ./gdrive/My Drive/Colab Datasets/SemEval-2017 Task 4/GOLD/Subtask_A/twitter-2014test-A.txt\n",
            "Loaded Subset ./gdrive/My Drive/Colab Datasets/SemEval-2017 Task 4/GOLD/Subtask_A/twitter-2015test-A.txt\n",
            "Loaded Subset ./gdrive/My Drive/Colab Datasets/SemEval-2017 Task 4/GOLD/Subtask_A/twitter-2015train-A.txt\n",
            "Loaded Subset ./gdrive/My Drive/Colab Datasets/SemEval-2017 Task 4/GOLD/Subtask_A/twitter-2014sarcasm-A.txt\n",
            "Loaded Subset ./gdrive/My Drive/Colab Datasets/SemEval-2017 Task 4/GOLD/Subtask_A/twitter-2016test-A.txt\n",
            "SemEval-2017 Task 4 Loaded. Training entries: 48213, labels: 48213\n",
            "cant wait to see the iPad HD .. 2\n",
            "@DebsODo: @lex_looper might want to double check your Saudi Arabia info on this one. 1. https://t.co/sToO95DdWt 2. https://t.co/T9j1IOsD3X 1\n",
            "@Pixe1ina getting more people who never would have touched a Dark SOuls in the 1st place 1\n",
            "@curtismufc10 Well Curtis... you also NEED a night out in Thompsons .... need a Sunday night session again soon bro!!!!! 2\n",
            "\n",
            "General stats regarding the length of instances of the dataset (to help choose embeddings_sequence_length) - avg:113.49 max:338\n",
            "\n",
            "cant wait to see the ipad hd\n",
            "<UNK> <UNK> looper might want to double check your saudi arabia info on this one 1 https t co <UNK> 2 https t co <UNK>\n",
            "<UNK> getting more people who never would have touched a dark souls in the 1st place\n",
            "<UNK> well curtis you also need a night out in <UNK> need a sunday night session again soon bro\n",
            "<class 'numpy.ndarray'> [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 745, 140, 3, 42, 2, 768, 2288]\n",
            "<class 'numpy.ndarray'> [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 9451, 267, 85, 3, 1045, 275, 71, 635, 769, 1167, 9, 26, 58, 92, 37, 6, 8, 1, 61, 37, 6, 8, 1]\n",
            "<class 'numpy.ndarray'> [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 244, 110, 158, 98, 197, 115, 25, 5124, 7, 577, 697, 5, 2, 47, 225]\n",
            "<class 'numpy.ndarray'> [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 139, 508, 15, 195, 163, 7, 41, 38, 5, 1, 163, 7, 51, 41, 1677, 171, 466, 1337]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ss1DzWYyra82"
      },
      "source": [
        "Decide on a final embeddings mode out of the 4. Then the above code can be run once at the start, and a different code can be run each time for a prediction."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "stFluykKQtWH"
      },
      "source": [
        "### Code"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "15cSyU7LSj39"
      },
      "source": [
        "# Load Model\n",
        "model = load_model('./gdrive/My Drive/Colab Saved Models/best_model_text.h5')\n",
        "#model = load_model('best_model_text.h5', custom_objects={'MultiplicativeLSTM': MultiplicativeLSTM})  # Handling saved models changes if we are using custom (3rd-party) layers"
      ],
      "execution_count": 42,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RNO6Gw-0Qwf-",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "cbc3e688-cb98-4976-9bc7-93a3fac0646c"
      },
      "source": [
        "# Evaluation Phase\n",
        "def evaluate_single_sentence(model, test_sentence, multiclass):\n",
        "    if multiclass == False: \n",
        "        probability = model.predict(test_sentence)\n",
        "        predictions = [1 * (x[0]>=0.5) for x in probability]  \n",
        "\n",
        "        # 0 stands for negative, 1 stands for positive\n",
        "        if predictions[0] == 1:\n",
        "            return (\"positive\", probability) \n",
        "        elif predictions[0] == 0:\n",
        "            return (\"negative\", probability)\n",
        "        else:\n",
        "            return (\"no prediction\", None)\n",
        "\n",
        "    else:    \n",
        "        probability = model.predict(test_sentence)\n",
        "        predictions = np.argmax(probability, axis=1)\n",
        "    \n",
        "        # 0 stands for negative, 1 stands for neutral, 2 stands for positive\n",
        "        if predictions[0] == 2:\n",
        "            return (\"positive\", probability) \n",
        "        elif predictions[0] == 1:\n",
        "            return (\"neutral\", probability)\n",
        "        elif predictions[0] == 0:\n",
        "            return (\"negative\", probability)            \n",
        "        else:\n",
        "            return (\"no prediction\", None)        \n",
        "\n",
        "print(evaluate_single_sentence(model, test_sentence, multiclass))"
      ],
      "execution_count": 44,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "('negative', array([[0.5037171 , 0.25885844, 0.23742439]], dtype=float32))\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}