Bagging and Hyperparameter tuning .ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bagging and Hyperparameter tuning  \n",
    "\n",
    "<img src=\"./images/bags.jpg\" alt=\"The bagging operation\" width=\"400\" align='center'/>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## The Spam vs. not-Spam Dataset\n",
    "<img src=\"./images/spam_or_not_spam.png\" alt=\"Spam vs. not-Spam\" width=\"300\" align='center'/>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### About the (Spam vs. not-Spam) dataset:\n",
    "* The dataset tags email messages as spam or not-spam.\n",
    "* Classes (=categories): there are two possible classes: not_spam, spam\n",
    "* Attributes (=features): there are 57 features, including two types of features:\n",
    "  * word frequencies - the feature name contains a word, with the suffix '_wordFreq' e.g.: 'make_wordFreq'  \n",
    "    * The word could appear 0 times in a specific email, once or a few times.\n",
    "    * The values of this feature could be between 0 and 1 (the frequency is relative).\n",
    "  * Capital Letter pattern attributes - 3 features regarding capital letter patterns. e.g.: 'capitalLet_long' - the length of the longest sequence of capital letters in the email."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# IMPORT (PACKAGES) CELL\n",
    "# --------------------------------------------------------\n",
    "\n",
    "import sys\n",
    "import os\n",
    "import pathlib \n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import random\n",
    "get_ipython().run_line_magic('matplotlib', 'inline')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### load the 'spam vs not-spam' dataset\n",
    "<img src=\"./images/load_dataframe.jpg\" alt=\"load dataframe\" width=\"100\" align='center'/>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loadDataset(fileName):\n",
    "    df_spam=pd.read_csv(fileName)\n",
    "    return (df_spam)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing your implementation of the 'loadDataset' method ...\n",
      "----> The 'spam vs not-spam' dataframe object was loaded successfuly :-) \n",
      "\n",
      "The beginning (the head) of the dataframe:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>make_wordFreq</th>\n",
       "      <th>address_wordFreq</th>\n",
       "      <th>all_wordFreq</th>\n",
       "      <th>3d_wordFreq</th>\n",
       "      <th>our_wordFreq</th>\n",
       "      <th>over_wordFreq</th>\n",
       "      <th>remove_wordFreq</th>\n",
       "      <th>internet_wordFreq</th>\n",
       "      <th>order_wordFreq</th>\n",
       "      <th>mail_wordFreq</th>\n",
       "      <th>...</th>\n",
       "      <th>semicol_wordFreq</th>\n",
       "      <th>paren_wordFreq</th>\n",
       "      <th>bracket_wordFreq</th>\n",
       "      <th>bang_wordFreq</th>\n",
       "      <th>dollar_wordFreq</th>\n",
       "      <th>pound_wordFreq</th>\n",
       "      <th>capitalLet_avg</th>\n",
       "      <th>capitalLet_long</th>\n",
       "      <th>capitalLet_total</th>\n",
       "      <th>class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.29</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.178</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.044</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.666</td>\n",
       "      <td>10</td>\n",
       "      <td>180</td>\n",
       "      <td>not_spam</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.46</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.125</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.510</td>\n",
       "      <td>10</td>\n",
       "      <td>74</td>\n",
       "      <td>not_spam</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.718</td>\n",
       "      <td>11</td>\n",
       "      <td>55</td>\n",
       "      <td>not_spam</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.33</td>\n",
       "      <td>0.44</td>\n",
       "      <td>0.37</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.14</td>\n",
       "      <td>0.11</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.07</td>\n",
       "      <td>0.97</td>\n",
       "      <td>1.16</td>\n",
       "      <td>...</td>\n",
       "      <td>0.006</td>\n",
       "      <td>0.159</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.069</td>\n",
       "      <td>0.221</td>\n",
       "      <td>0.11</td>\n",
       "      <td>3.426</td>\n",
       "      <td>72</td>\n",
       "      <td>819</td>\n",
       "      <td>spam</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.00</td>\n",
       "      <td>2.08</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.12</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.04</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.263</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.428</td>\n",
       "      <td>4</td>\n",
       "      <td>20</td>\n",
       "      <td>spam</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 58 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   make_wordFreq  address_wordFreq  all_wordFreq  3d_wordFreq  our_wordFreq  \\\n",
       "0           0.00              0.00          0.29          0.0          0.00   \n",
       "1           0.46              0.00          0.00          0.0          0.00   \n",
       "2           0.00              0.00          0.00          0.0          0.00   \n",
       "3           0.33              0.44          0.37          0.0          0.14   \n",
       "4           0.00              2.08          0.00          0.0          3.12   \n",
       "\n",
       "   over_wordFreq  remove_wordFreq  internet_wordFreq  order_wordFreq  \\\n",
       "0           0.00             0.00               0.00            0.00   \n",
       "1           0.00             0.00               0.00            0.00   \n",
       "2           0.00             0.00               0.00            0.00   \n",
       "3           0.11             0.00               0.07            0.97   \n",
       "4           0.00             1.04               0.00            0.00   \n",
       "\n",
       "   mail_wordFreq  ...  semicol_wordFreq  paren_wordFreq  bracket_wordFreq  \\\n",
       "0           0.00  ...             0.000           0.178               0.0   \n",
       "1           0.00  ...             0.000           0.125               0.0   \n",
       "2           0.00  ...             0.000           0.000               0.0   \n",
       "3           1.16  ...             0.006           0.159               0.0   \n",
       "4           0.00  ...             0.000           0.000               0.0   \n",
       "\n",
       "   bang_wordFreq  dollar_wordFreq  pound_wordFreq  capitalLet_avg  \\\n",
       "0          0.044            0.000            0.00           1.666   \n",
       "1          0.000            0.000            0.00           1.510   \n",
       "2          0.000            0.000            0.00           1.718   \n",
       "3          0.069            0.221            0.11           3.426   \n",
       "4          0.263            0.000            0.00           1.428   \n",
       "\n",
       "   capitalLet_long  capitalLet_total     class  \n",
       "0               10               180  not_spam  \n",
       "1               10                74  not_spam  \n",
       "2               11                55  not_spam  \n",
       "3               72               819      spam  \n",
       "4                4                20      spam  \n",
       "\n",
       "[5 rows x 58 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# -----------------------------------------------------------------\n",
    "folderName = '.' + os.sep + 'data' \n",
    "datasetCsvFileName = folderName + os.sep + 'spam_notSpam.csv'\n",
    "# --------------------- \n",
    "print (\"Testing your implementation of the 'loadDataset' method ...\")\n",
    "dataset_forTesting = loadDataset(datasetCsvFileName)\n",
    "assert dataset_forTesting is not None, 'spam_notSpam object not initialized'\n",
    "assert isinstance(dataset_forTesting, pd.DataFrame), 'spam_notSpam object is not a dataframe'\n",
    "assert dataset_forTesting.empty == False, 'spam_notSpam dataframe object is empty'\n",
    "assert len(dataset_forTesting.columns)==58, 'spam_notSpam dataframe object is missing columns'\n",
    "assert len(dataset_forTesting.index)==4601, 'spam_notSpam dataframe object is missing rows'\n",
    "print (\"----> The 'spam vs not-spam' dataframe object was loaded successfuly :-) \\n\")\n",
    "dataset_forTesting=None\n",
    "print ('The beginning (the head) of the dataframe:')\n",
    "loadDataset(datasetCsvFileName).head()\n",
    "# --------------------------------------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# return values (comma separated):\n",
    "# - X_featureVectores - a dataframe containing all feature vectors. \n",
    "#                       It should contain the input dataframe after removing the class column.\n",
    "#                       The index of the X_featureVectores should be the same as the input dataframe parameter.\n",
    "# - y_Categories      - a series of containing all class values per instance.\n",
    "#                       The index of the y_Categories series should be the same as the input dataframe.\n",
    "# --------------------- \n",
    "def separateTo_X_and_y(dataset, classColName):\n",
    "    xInstances=dataset.drop(columns=classColName)\n",
    "    yCategories=dataset.loc[:,classColName]\n",
    "    return xInstances,yCategories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing your implementation of the 'separateTo_X_and_y' method ...\n",
      "----> The 'separateTo_X_and_y' test passed successfully :-) \n",
      "\n"
     ]
    }
   ],
   "source": [
    "# --------------------------------------------------------\n",
    "folderName = '.' + os.sep + 'data' \n",
    "datasetCsvFileName = folderName + os.sep + 'spam_notSpam.csv'\n",
    "# --------------------------------------------------------\n",
    "\n",
    "# --------------------------------------------------------\n",
    "print (\"Testing your implementation of the 'separateTo_X_and_y' method ...\")\n",
    "# --------------------- \n",
    "dataset_forTesting = loadDataset(datasetCsvFileName)\n",
    "X_vectors, y_categories = separateTo_X_and_y(dataset_forTesting, 'class')\n",
    "# ---------------------\n",
    "sameIndexes = lambda obj1,obj2: False not in (obj1.index.tolist()[n] == obj2.index.tolist()[n] for n in range(len(obj1.index))) \n",
    "assert X_vectors is not None, 'X_vectors object not initialized'\n",
    "assert isinstance(X_vectors, pd.DataFrame), 'X_vectors object is not a dataframe'\n",
    "assert X_vectors.empty == False, 'X_vectors dataframe object is empty'\n",
    "assert sameIndexes(X_vectors,dataset_forTesting), 'X_vectors should share the same indexes as the dataset'\n",
    "assert y_categories is not None, 'y_categories object not initialized'\n",
    "assert isinstance(y_categories, pd.Series), 'y_categories object is not a series'\n",
    "assert y_categories.empty == False, 'y_categories dataframe object is empty'\n",
    "assert sameIndexes(X_vectors,y_categories), 'X_vectors should share the same indexes as y_categories'\n",
    "print (\"----> The 'separateTo_X_and_y' test passed successfully :-) \\n\")\n",
    "dataset_forTesting = None\n",
    "X_vectors = None\n",
    "y_categories = None\n",
    "# --------------------------------------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# input parameters:\n",
    "# - y_Categories -  a dataframe structure, containing the dataset.\n",
    "# - positiveClassName - the name of the category (string) of the positive class\n",
    "#                       * if the value of a cell does not equal positiveClassName, \n",
    "#                         it should be considered as negative.\n",
    "# ------------\n",
    "# return value: \n",
    "# - y_NumCategories - a series of containing all class values with numeric values per instance.\n",
    "#                       * each cell value, which equals the negativeClassName will recieve a value of 0 in the\n",
    "#                         output series.\n",
    "#                       * each cell value, which equals the positiveClassName will recieve a value of 1 in the\n",
    "#                         output series.\n",
    "#                       * the index should be the same of the input series.\n",
    "# --------------------- \n",
    "def datasetCategoriesToNums(y_Categories, positiveClassName):\n",
    "    y_NumCategories=pd.Series(index=y_Categories.index)\n",
    "    for index in y_Categories.index:\n",
    "        if y_Categories.iloc[index]==positiveClassName:\n",
    "            y_NumCategories[index]=1\n",
    "        else:\n",
    "            y_NumCategories[index]=0\n",
    "    return y_NumCategories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing your implementation of the 'datasetCategoriesToNums' method ...\n",
      "----> The 'datasetCategoriesToNums' test passed successfully :-) \n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Test of:\n",
    "### --- Graded tests for the 'datasetCategoriesToNums' method \n",
    "# --------------------------------------------------------\n",
    "\n",
    "# --------------------------------------------------------\n",
    "folderName = '.' + os.sep + 'data' \n",
    "datasetCsvFileName = folderName + os.sep + 'spam_notSpam.csv'\n",
    "# --------------------------------------------------------\n",
    "\n",
    "# --------------------------------------------------------\n",
    "print (\"Testing your implementation of the 'datasetCategoriesToNums' method ...\")\n",
    "# --------------------- \n",
    "dataset_forTesting = loadDataset(datasetCsvFileName)\n",
    "X_vectors, y_categoriesB4 = separateTo_X_and_y(dataset_forTesting, 'class')\n",
    "y_categories = datasetCategoriesToNums(y_categoriesB4, 'spam')\n",
    "# --------------------- \n",
    "allValidVals = lambda arrValues,validVals: False not in (val in validVals for val in arrValues) \n",
    "sameIndexes = lambda obj1,obj2: False not in (obj1.index.tolist()[n] == obj2.index.tolist()[n] for n in range(len(obj1.index))) \n",
    "assert y_categories is not None, 'y_categories object not initialized'\n",
    "assert isinstance(y_categories, pd.Series), 'y_categories object is not a series'\n",
    "assert y_categories.empty == False, 'y_categories dataframe object is empty'\n",
    "assert sameIndexes(y_categories,y_categoriesB4), 'y_categories should share the same indexes as y_categoriesB4'\n",
    "assert allValidVals(np.unique(y_categories.values),(0,1)), 'classes should be only 0 or 1 (as an integer number)'\n",
    "print (\"----> The 'datasetCategoriesToNums' test passed successfully :-) \\n\")\n",
    "dataset_forTesting = None\n",
    "X_vectors = None\n",
    "y_categories = None\n",
    "# --------------------------------------------------------"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### split the dataset to a train-set and a test-set\n",
    "<img src=\"./images/train-test-split.png\" alt=\"train-test split\" width=\"100\" align='center'/>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Information after train-test split:\n",
      "The train-set includes 3680 instances and 3680 corresponding categories\n",
      "\n",
      "The test-set includes 921 instances and 921 corresponding categories\n",
      "\n",
      "First few rows of unified train-set:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>make_wordFreq</th>\n",
       "      <th>address_wordFreq</th>\n",
       "      <th>all_wordFreq</th>\n",
       "      <th>3d_wordFreq</th>\n",
       "      <th>our_wordFreq</th>\n",
       "      <th>over_wordFreq</th>\n",
       "      <th>remove_wordFreq</th>\n",
       "      <th>internet_wordFreq</th>\n",
       "      <th>order_wordFreq</th>\n",
       "      <th>mail_wordFreq</th>\n",
       "      <th>...</th>\n",
       "      <th>semicol_wordFreq</th>\n",
       "      <th>paren_wordFreq</th>\n",
       "      <th>bracket_wordFreq</th>\n",
       "      <th>bang_wordFreq</th>\n",
       "      <th>dollar_wordFreq</th>\n",
       "      <th>pound_wordFreq</th>\n",
       "      <th>capitalLet_avg</th>\n",
       "      <th>capitalLet_long</th>\n",
       "      <th>capitalLet_total</th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2776</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.23</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.139</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.279</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>1.736</td>\n",
       "      <td>10</td>\n",
       "      <td>66</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>908</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.336</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>2.352</td>\n",
       "      <td>15</td>\n",
       "      <td>40</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4540</th>\n",
       "      <td>0.44</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.89</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.44</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.944</td>\n",
       "      <td>0.145</td>\n",
       "      <td>0.072</td>\n",
       "      <td>2.451</td>\n",
       "      <td>28</td>\n",
       "      <td>152</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>788</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.186</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>2.823</td>\n",
       "      <td>38</td>\n",
       "      <td>240</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2186</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.284</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>1.357</td>\n",
       "      <td>5</td>\n",
       "      <td>19</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 58 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      make_wordFreq  address_wordFreq  all_wordFreq  3d_wordFreq  \\\n",
       "2776           0.00               0.0          1.23          0.0   \n",
       "908            0.00               0.0          0.00          0.0   \n",
       "4540           0.44               0.0          0.00          0.0   \n",
       "788            0.00               0.0          0.00          0.0   \n",
       "2186           0.00               0.0          0.00          0.0   \n",
       "\n",
       "      our_wordFreq  over_wordFreq  remove_wordFreq  internet_wordFreq  \\\n",
       "2776          0.00            0.0              0.0                0.0   \n",
       "908           0.00            0.0              0.0                0.0   \n",
       "4540          0.89            0.0              0.0                0.0   \n",
       "788           0.00            0.0              0.0                0.0   \n",
       "2186          0.00            0.0              0.0                0.0   \n",
       "\n",
       "      order_wordFreq  mail_wordFreq  ...  semicol_wordFreq  paren_wordFreq  \\\n",
       "2776             0.0           0.00  ...               0.0           0.139   \n",
       "908              0.0           0.00  ...               0.0           0.336   \n",
       "4540             0.0           0.44  ...               0.0           0.000   \n",
       "788              0.0           0.00  ...               0.0           0.186   \n",
       "2186             0.0           0.00  ...               0.0           0.000   \n",
       "\n",
       "      bracket_wordFreq  bang_wordFreq  dollar_wordFreq  pound_wordFreq  \\\n",
       "2776               0.0          0.279            0.000           0.000   \n",
       "908                0.0          0.000            0.000           0.000   \n",
       "4540               0.0          0.944            0.145           0.072   \n",
       "788                0.0          0.000            0.000           0.000   \n",
       "2186               0.0          1.284            0.000           0.000   \n",
       "\n",
       "      capitalLet_avg  capitalLet_long  capitalLet_total    0  \n",
       "2776           1.736               10                66  0.0  \n",
       "908            2.352               15                40  0.0  \n",
       "4540           2.451               28               152  1.0  \n",
       "788            2.823               38               240  1.0  \n",
       "2186           1.357                5                19  1.0  \n",
       "\n",
       "[5 rows x 58 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# --------------------------------------------------------\n",
    "# method name1: trainTestSplit\n",
    "# --------------------------------------------------------\n",
    "\n",
    "# --------------------------------------------------------\n",
    "# What does the method do?\n",
    "# --- the method split the input dataset into train and test. \n",
    "#     - It does so using the sklearn built in method\n",
    "# ------------\n",
    "# input parameters:\n",
    "# - X_vectors - a dataframe containing all feature vectors. \n",
    "# - y_categories - a series of containing all class values per instance.\n",
    "# - test_size_ratio - a number (0<number<1) of the wanted ratio of the dataset out of the dataset \n",
    "# - rand_state - a number, in order to guarantee reproducible results \n",
    "# ------------\n",
    "# return values (comma separated):\n",
    "# - X_train -  a dataframe containing all feature vectors of the train set\n",
    "# - X_test -  a dataframe containing all feature vectors of the test set\n",
    "# - y_train - a series of containing all class values per train instance\n",
    "# - y_test - a series of containing all class values per test instance\n",
    "# --------------------- \n",
    "def trainTestSplit(X_vectors, y_categories, test_size_ratio, rand_state):\n",
    "    return train_test_split(X_vectors, y_categories, test_size=test_size_ratio, random_state=rand_state)\n",
    "\n",
    "\n",
    "# --------------------------------------------------------\n",
    "# method name: reAttachTrainSet\n",
    "# --------------------------------------------------------\n",
    "\n",
    "# --------------------------------------------------------\n",
    "# What does the method do?\n",
    "# --- the method split the input dataset into train and test. \n",
    "#     - It does so using the sklearn built in method\n",
    "# ------------\n",
    "# input parameters:\n",
    "# - X_train -  a dataframe containing all feature vectors of the train set\n",
    "# - y_train - a series of containing all class values per train instance\n",
    "# ------------\n",
    "# return value:\n",
    "# - train_set - a reattached dataframe, consisting both feature vectors and categories.\n",
    "# --------------------- \n",
    "def reAttachTrainSet(X_train, y_train):\n",
    "    return pd.concat((X_train, y_train), axis=1)\n",
    "\n",
    "# --------------------------------------------------------\n",
    "# Temporary imports:\n",
    "import sklearn\n",
    "from sklearn.model_selection import train_test_split\n",
    "# --------\n",
    "folderName = '.' + os.sep + 'data' \n",
    "datasetCsvFileName = folderName + os.sep + 'spam_notSpam.csv'\n",
    "# --------------------- \n",
    "dataset_forTesting = loadDataset(datasetCsvFileName)\n",
    "X_vectors, y_categories = separateTo_X_and_y(dataset_forTesting, 'class')\n",
    "y_categories = datasetCategoriesToNums(y_categories, 'spam')\n",
    "X_train, X_test, y_train, y_test = trainTestSplit(X_vectors, y_categories, 0.2, 25)\n",
    "# --------------------------------------------------------\n",
    "\n",
    "# --------------------------------------------------------\n",
    "if 'datasets' in sys.modules:\n",
    "    del (datasets)\n",
    "if 'train_test_split' in sys.modules:\n",
    "    del (train_test_split)\n",
    "sys_modules = list(sys.modules.keys())\n",
    "for mdl in sys_modules:\n",
    "    if mdl.startswith('sklearn.'):\n",
    "        del(sys.modules[mdl]) \n",
    "del (sklearn)\n",
    "if 'sklearn' in sys.modules:\n",
    "    del (sys.modules['sklearn'])\n",
    "# --------------------------------------------------------\n",
    "\n",
    "# --------------------------------------------------------\n",
    "# display train-test split information\n",
    "# --------------------------------------------------------\n",
    "print ('Information after train-test split:')\n",
    "print('The train-set includes %d instances and %d corresponding categories\\n' %(X_train.shape[0],y_train.shape[0]))\n",
    "print('The test-set includes %d instances and %d corresponding categories\\n' %(X_test.shape[0],y_test.shape[0]))\n",
    "\n",
    "# --------------------------------------------------------\n",
    "## concatinate the X_train and y_train:\n",
    "# --------------------------------------------------------\n",
    "train_set = reAttachTrainSet(X_train, y_train)\n",
    "\n",
    "# --------------------------------------------------------\n",
    "dataset_forTesting = None\n",
    "X_vectors = None\n",
    "y_categories = None\n",
    "X_train = None\n",
    "X_test = None\n",
    "y_train = None\n",
    "y_test = None\n",
    "# --------------------------------------------------------\n",
    "\n",
    "# --------------------------------------------------------\n",
    "# Display the first few rows of the training-set:\n",
    "# --------------------------------------------------------\n",
    "print('First few rows of unified train-set:')\n",
    "train_set.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Building a Bagging Classifier\n",
    "<img src=\"./images/bagging-classifier.jpeg\" alt=\"bagging-classifier\" width=\"400\" align='center'/>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "** The following cells perform 2 methods:\n",
    "* step 1 - instance bootstrap\n",
    "* step 2 - feature bootstrap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# --------------------------------------------------------\n",
    "# method name: instanceBootStrap\n",
    "# --------------------------------------------------------\n",
    "\n",
    "# --------------------------------------------------------\n",
    "# What does the method do?\n",
    "# --- the method samples the dataset uniformly with replacements. \n",
    "#     * this means that there is a chance we could get the same instance more than once\n",
    "# ------------\n",
    "# input parameters:\n",
    "# - X_train - a dataframe containing all training feature vectors. \n",
    "# - y_train - a series of containing all training class values per instance.\n",
    "# - sampleRatio - the ratio of the sampeling out of training set \n",
    "#               * we will derive the number of instances to sample from sampleRatio\n",
    "# ------------\n",
    "# return values (comma separated):\n",
    "# - X_trainSampled - a dataframe containing sampled training feature vectors. \n",
    "# - y_trainSampled - a series of containing Sampled training class values per instance.\n",
    "# Note: X_trainSampled should have the same index as y_trainSampled \n",
    "# --------------------- \n",
    "def instanceBootStrap(X_train, y_train, sampleRatio):\n",
    "    helpLi=list(X_train.index.values) \n",
    "    howMany=int(len(y_train)*sampleRatio)\n",
    "    helpLi2=[]\n",
    "    X_trainSampled=pd.DataFrame()\n",
    "    y_trainSampled=pd.Series()\n",
    "    for x in range(howMany):\n",
    "        value=random.randrange(len(helpLi))\n",
    "        helpLi2.append(helpLi[value])\n",
    "    \n",
    "    X_trainSampled=X_train.loc[helpLi2]\n",
    "    y_trainSampled=y_train.loc[helpLi2]\n",
    "    \n",
    "    return X_trainSampled,y_trainSampled  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing your implementation of the 'instanceBootStrap' method ...\n",
      "----> The 'instanceBootStrap' test passed successfully :-) \n",
      "\n"
     ]
    }
   ],
   "source": [
    "# --------------------------------------------------------\n",
    "# Test of:\n",
    "### --- Graded tests for the 'instanceBootStrap' method \n",
    "# Important Note: additional test might also be taken from our side.\n",
    "\n",
    "# --------------------------------------------------------\n",
    "# Temporary imports:\n",
    "import sklearn\n",
    "from sklearn.model_selection import train_test_split\n",
    "# --------\n",
    "folderName = '.' + os.sep + 'data' \n",
    "datasetCsvFileName = folderName + os.sep + 'spam_notSpam.csv'\n",
    "# --------------------------------------------------------\n",
    "print (\"Testing your implementation of the 'instanceBootStrap' method ...\")\n",
    "dataset_forTesting = loadDataset(datasetCsvFileName)\n",
    "X_vectors, y_categories = separateTo_X_and_y(dataset_forTesting, 'class')\n",
    "y_categories = datasetCategoriesToNums(y_categories, 'spam')\n",
    "X_train, X_test, y_train, y_test = trainTestSplit(X_vectors, y_categories, 0.2, 35)\n",
    "# --------------------------------------------------------\n",
    "\n",
    "# --------------------------------------------------------\n",
    "if 'datasets' in sys.modules:\n",
    "    del (datasets)\n",
    "if 'train_test_split' in sys.modules:\n",
    "    del (train_test_split)\n",
    "sys_modules = list(sys.modules.keys())\n",
    "for mdl in sys_modules:\n",
    "    if mdl.startswith('sklearn.'):\n",
    "        del(sys.modules[mdl]) \n",
    "del (sklearn)\n",
    "if 'sklearn' in sys.modules:\n",
    "    del (sys.modules['sklearn'])\n",
    "# --------------------------------------------------------\n",
    "\n",
    "# --------------------------------------------------------\n",
    "allValidVals = lambda arrValues,validVals: False not in (val in validVals for val in arrValues) \n",
    "sameIndexes = lambda obj1,obj2: False not in (obj1.index.tolist()[n] == obj2.index.tolist()[n] for n in range(len(obj1.index))) \n",
    "X_train_100 = X_train.iloc[:100,:]\n",
    "y_train_100 = y_train.iloc[:100]\n",
    "sampleRatio = 0.5\n",
    "X_trainSampled, y_trainSampled = instanceBootStrap(X_train_100, y_train_100, sampleRatio)\n",
    "# --------------------------------------------------------\n",
    "\n",
    "assert X_trainSampled is not None, 'X_trainSampled object not initialized'\n",
    "assert isinstance(X_trainSampled, pd.DataFrame), 'X_trainSampled object is not a dataframe'\n",
    "assert X_trainSampled.empty == False, 'X_trainSampled dataframe object is empty'\n",
    "assert y_trainSampled is not None, 'y_trainSampled object not initialized'\n",
    "assert isinstance(y_trainSampled, pd.Series), 'y_trainSampled object is not a series'\n",
    "assert y_trainSampled.empty == False, 'y_trainSampled series object is empty'\n",
    "assert sameIndexes(X_trainSampled,y_trainSampled), 'X_trainSampled should share the same indexes as y_trainSampled'\n",
    "assert allValidVals(np.unique(y_categories.values),(0,1)), \"y_trainSampled's value should be only 0 or 1 (as an integer number)\"\n",
    "print (\"----> The 'instanceBootStrap' test passed successfully :-) \\n\")\n",
    "dataset_forTesting = None\n",
    "X_vectors = None\n",
    "y_categories = None\n",
    "X_train = None\n",
    "X_test = None\n",
    "y_train = None\n",
    "y_test = None\n",
    "X_train_100 = None\n",
    "y_train_100 = None\n",
    "sampleRatio = None\n",
    "X_trainSampled = None\n",
    "y_trainSampled = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# --------------------------------------------------------\n",
    "# method name: featureBootStrap\n",
    "# --------------------------------------------------------\n",
    "\n",
    "# --------------------------------------------------------\n",
    "# What does the method do?\n",
    "# --- the method samples the featureset uniformly with replacements. \n",
    "#     * this means that there is a chance we could choose the same feature twice \n",
    "#     * note that in the output dataframe, any feature which was already chosen,\n",
    "#            will be disregarded. In other words, such a case will result in less\n",
    "#            output features. For instance if we have 50 input features, and the sample\n",
    "#            ratio is 0.5, we should expect 25 sampled feature columns. But if one of the features\n",
    "#            was selected twice, we expect only 24 feature columns.\n",
    "# ------------\n",
    "# input parameters:\n",
    "# - X_instances - a dataframe containing all feature vectors from the dataset.\n",
    "# - sampleRatio - the ratio of the sampeling out of feature set,\n",
    "#               * we will derive the number of features from sampleRatio\n",
    "# ------------\n",
    "# return value:\n",
    "# - X_instances_featureSampled - a dataframe containing feature vectors with sampled features. \n",
    "#                               * note the instances refer to the same instances as the input dataframe.\n",
    "#                               * the difference is in the columns (the selected features). \n",
    "# --------------------- \n",
    "def featureBootStrap(X_instances, sampleRatio):\n",
    "    size=int((X_instances.shape[1])*sampleRatio)\n",
    "    helpLi=[]\n",
    "    X_instances_featureSampled=pd.DataFrame()\n",
    "    for x in range(size):\n",
    "        chosenOne=X_instances.iloc[:,random.randrange(X_instances.shape[1])]\n",
    "        helpLi.append(chosenOne.name)\n",
    "        if helpLi[x] not in X_instances_featureSampled:\n",
    "            X_instances_featureSampled[helpLi[x]]=X_instances.loc[:,helpLi[x]].values\n",
    "    return X_instances_featureSampled\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing your implementation of the 'featureBootStrap' method ...\n",
      "\n",
      "----> The 'featureBootStrap' test passed successfully :-) \n",
      "\n"
     ]
    }
   ],
   "source": [
    "# --------------------------------------------------------\n",
    "# Test of:\n",
    "### --- Graded tests for the 'featureBootStrap' method \n",
    "# --------------------------------------------------------\n",
    "folderName = '.' + os.sep + 'data' \n",
    "datasetCsvFileName = folderName + os.sep + 'spam_notSpam.csv'\n",
    "# --------------------------------------------------------\n",
    "print (\"Testing your implementation of the 'featureBootStrap' method ...\")\n",
    "dataset_forTesting = loadDataset(datasetCsvFileName)\n",
    "X_vectors, y_categories = separateTo_X_and_y(dataset_forTesting, 'class')\n",
    "# --------------------------------------------------------\n",
    "\n",
    "# --------------------------------------------------------\n",
    "allValidVals = lambda arrValues,validVals: False not in (val in validVals for val in arrValues) \n",
    "sameIndexes = lambda obj1,obj2: False not in (obj1.index.tolist()[n] == obj2.index.tolist()[n] for n in range(len(obj1.index))) \n",
    "\n",
    "columns = X_vectors.columns\n",
    "columns_50 = columns[:50] \n",
    "X_vectors_columns_50 = X_vectors[columns_50]\n",
    "sampleRatio = 0.5\n",
    "X_Sampled = featureBootStrap(X_vectors_columns_50, sampleRatio)\n",
    "# --------------------------------------------------------\n",
    "\n",
    "assert X_Sampled is not None, 'X_Sampled object not initialized'\n",
    "assert isinstance(X_Sampled, pd.DataFrame), 'X_Sampled object is not a dataframe'\n",
    "assert X_Sampled.empty == False, 'X_Sampled dataframe object is empty'\n",
    "assert X_Sampled.shape[1] >10, 'X_Sampled dataframe object has a wrong number of sampled features'\n",
    "\n",
    "print (\"\\n----> The 'featureBootStrap' test passed successfully :-) \\n\")\n",
    "dataset_forTesting = None\n",
    "X_vectors = None\n",
    "y_categories = None\n",
    "columns = None\n",
    "columns_50 = None\n",
    "X_vectors_columns_50 = None\n",
    "sampleRatio = None\n",
    "X_Sampled = None\n",
    "# --------------------------------------------------------\n",
    "\n",
    "# --------------------------------------------------------"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Building a model\n",
    "The following cells perform the following:\n",
    "* step 1 - decision stumps (and other classification utilities)\n",
    "* step 2 - Bagging (fit)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### decision stumps (and other classification utilities)\n",
    "<img src=\"./images/treeStumps.jpg\" alt=\"bagging-classifier\" width=\"400\" align='center'/>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "A few prediction examples:\n",
      "test instance [0]: actual: 0.0, prediction(classifier1): 0.0, prediction(classifier2): 0.0\n",
      "test instance [1]: actual: 1.0, prediction(classifier1): 0.0, prediction(classifier2): 0.0\n",
      "test instance [2]: actual: 0.0, prediction(classifier1): 0.0, prediction(classifier2): 0.0\n",
      "test instance [3]: actual: 0.0, prediction(classifier1): 1.0, prediction(classifier2): 0.0\n",
      "test instance [4]: actual: 0.0, prediction(classifier1): 0.0, prediction(classifier2): 0.0\n"
     ]
    }
   ],
   "source": [
    "# --------------------------------------------------------\n",
    "# class 1: DecisionStump\n",
    "# --------------------------------------------------------\n",
    "\n",
    "# --------------------------------------------------------\n",
    "# What does the method do?\n",
    "# --- wraps an api for decision stumps, so we have a unified api\n",
    "# ------------\n",
    "class DecisionStump():\n",
    "    def __init__(self):\n",
    "        self._decisionStump = DecisionTreeClassifier(max_depth=1)\n",
    "        \n",
    "    def fit(self,X_train,y_train):\n",
    "        self._decisionStump.fit(X_train,y_train)\n",
    "\n",
    "    def predict(self,X_test):\n",
    "        return self._decisionStump.predict(X_test)\n",
    "\n",
    "\n",
    "# --------------------------------------------------------\n",
    "# class2: ClassiferInstGen\n",
    "# --------------------------------------------------------\n",
    "\n",
    "# --------------------------------------------------------\n",
    "# What does the method do?\n",
    "# --- utility classifier to generate objects of the classification algorithm class   \n",
    "# ------------\n",
    "class ClassiferInstGen():\n",
    "    def __init__(self,classifierPyClass):\n",
    "        self._classifierPyClass = classifierPyClass\n",
    "        \n",
    "    def getNewClassifierPyObj(self):\n",
    "        return self._classifierPyClass()\n",
    "\n",
    "# --------------------------------------------------------\n",
    "# class3: BootstrapFeatureClassifer\n",
    "# --------------------------------------------------------\n",
    "\n",
    "# --------------------------------------------------------\n",
    "# What does the method do?\n",
    "# --- container classifier to save model and bootstrap features    \n",
    "# ------------\n",
    "class BootstrapFeatureClassifer():\n",
    "    def __init__(self,trainModel,featureNames):\n",
    "        self.model = trainModel\n",
    "        self.featureNames = featureNames\n",
    "\n",
    "        \n",
    "# --------------------------------------------------------\n",
    "# method name1: trainBootstrapedClassificationModel\n",
    "# --------------------------------------------------------\n",
    "\n",
    "# --------------------------------------------------------\n",
    "# What does the method do?\n",
    "# --- trains a classification model which matches the sklearn API of fit and predict.\n",
    "#     - It does so using the sklearn built in method\n",
    "# ------------\n",
    "# input parameters:\n",
    "# - X_sampled_train -  a dataframe containing all feature vectors of the train set\n",
    "# - y_train - a series of containing all class values per train instance\n",
    "# Note: the X_sampled_train, y_train parameters could be instance bootsraped, feature bootsraped, both or none\n",
    "# - classificationAlgo_pyClass - the python 'class' parameter of a classification algorithm \n",
    "#                                For instance, the above 'DecisionStump' class.\n",
    "#                                Note: passing the pyton class is similar to passing a method\n",
    "#                                      as a parameter. Each call to: classificationAlgo_pyClass()\n",
    "#                                      creates a new object (also called instance of the class)\n",
    "#                                      of the type of the class.  \n",
    "# ------------\n",
    "# return value:\n",
    "# - featuresTrainModelObj - an object including trained model and feature names \n",
    "# --------------------- \n",
    "def trainBootstrapedClassificationModel(X_sampled_train, y_train,classificationAlgo_pyClass):\n",
    "    classiferInstGen = ClassiferInstGen(classificationAlgo_pyClass)\n",
    "    classificationObj = classiferInstGen.getNewClassifierPyObj()\n",
    "    classificationObj.fit(X_sampled_train, y_train)\n",
    "    featuresTrainModelObj = BootstrapFeatureClassifer(classificationObj,X_sampled_train.columns)\n",
    "    return featuresTrainModelObj\n",
    "\n",
    "# --------------------------------------------------------\n",
    "# method name2: classifierPredict\n",
    "# --------------------------------------------------------\n",
    "\n",
    "# --------------------------------------------------------\n",
    "# What does the method do?\n",
    "# --- predict test examples using a classification model (which corresponds to sklearn classifier APIs)\n",
    "#     - It does so using the sklearn built in method\n",
    "# ------------\n",
    "# input parameters:\n",
    "# - X_test -  a dataframe containing feature vectors of the test set\n",
    "# - classification_obj - trained classificaion model (which corresponds to sklearn classifier APIs)\n",
    "# ------------\n",
    "# return value:\n",
    "# - yHat - a series of the predictions for each test instance  \n",
    "# --------------------- trainBootstrapedFeatureModel(X_sampled_train, y_train,classificationAlgo_pyClass)\n",
    "def classifierPredict(X_test, featuresTrainModelObj):\n",
    "    X_adapted = X_test[featuresTrainModelObj.featureNames]\n",
    "    predictions = featuresTrainModelObj.model.predict(X_adapted)\n",
    "    yHat = pd.Series(data=predictions,index=X_test.index)\n",
    "    return yHat\n",
    "\n",
    "# --------------------------------------------------------\n",
    "# Temporary imports:\n",
    "import sklearn\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "# --------\n",
    "folderName = '.' + os.sep + 'data' \n",
    "datasetCsvFileName = folderName + os.sep + 'spam_notSpam.csv'\n",
    "# --------------------- \n",
    "dataset_forTesting = loadDataset(datasetCsvFileName)\n",
    "X_vectors, y_categories = separateTo_X_and_y(dataset_forTesting, 'class')\n",
    "y_categories = datasetCategoriesToNums(y_categories, 'spam')\n",
    "X_train, X_test, y_train, y_test = trainTestSplit(X_vectors, y_categories, 0.2, 11)\n",
    "X_train_1st_k = X_train.iloc[:1000,:] \n",
    "y_train_1st_k = y_train.iloc[:1000]\n",
    "X_train_2nd_k = X_train.iloc[1000:2000,:] \n",
    "y_train_2nd_k = y_train.iloc[1000:2000]\n",
    "classificationModel1=trainBootstrapedClassificationModel(X_train_1st_k, y_train_1st_k,DecisionStump)\n",
    "classificationModel2=trainBootstrapedClassificationModel(X_train_2nd_k, y_train_2nd_k,DecisionStump)\n",
    "# --------------------------------------------------------\n",
    "\n",
    "\n",
    "# --------------------------------------------------------\n",
    "# show usages of functions\n",
    "# --------------------------------------------------------\n",
    "print ('A few prediction examples:')\n",
    "nExamples=5\n",
    "\n",
    "y_hat1 = classifierPredict(X_test.iloc[:nExamples,:], classificationModel1)\n",
    "y_hat2 = classifierPredict(X_test.iloc[:nExamples,:], classificationModel2)\n",
    "for nInd in range(nExamples):\n",
    "    print ('test instance [%d]: actual: %r, prediction(classifier1): %r, prediction(classifier2): %r' %(nInd,y_test.iloc[nInd],y_hat1.iloc[nInd],y_hat2.iloc[nInd]))\n",
    "# --------------------------------------------------------\n",
    "if 'datasets' in sys.modules:\n",
    "    del (datasets)\n",
    "if 'train_test_split' in sys.modules:\n",
    "    del (train_test_split)\n",
    "if 'DecisionTreeClassifier' in sys.modules:\n",
    "    del (DecisionTreeClassifier)\n",
    "sys_modules = list(sys.modules.keys())\n",
    "for mdl in sys_modules:\n",
    "    if mdl.startswith('sklearn.'):\n",
    "        del(sys.modules[mdl]) \n",
    "del (sklearn)\n",
    "if 'sklearn' in sys.modules:\n",
    "    del (sys.modules['sklearn'])\n",
    "# --------------------------------------------------------\n",
    "\n",
    "# --------------------------------------------------------\n",
    "dataset_forTesting = None\n",
    "X_vectors = None\n",
    "y_categories = None\n",
    "X_train = None\n",
    "X_test = None\n",
    "y_train = None\n",
    "y_test = None\n",
    "y_hat1 = None\n",
    "y_hat2 = None\n",
    "X_train_1st_k = None \n",
    "y_train_1st_k = None\n",
    "X_train_2nd_k = None \n",
    "y_train_2nd_k = None\n",
    "classificationModel1=None\n",
    "classificationModel2=None\n",
    "\n",
    "# --------------------------------------------------------"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Bagging (fit)\n",
    "<img src=\"./images/bags.jpg\" alt=\"The bagging operation\" width=\"200\" align='center'/>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#--------------------------------------------------------\n",
    "# method name: baggingFit\n",
    "# --------------------------------------------------------\n",
    "\n",
    "# --------------------------------------------------------\n",
    "# The following is expected:\n",
    "# --- the method needs to return the a list of models created after bootstraping instances & features.\n",
    "# ------------\n",
    "# input parameters:\n",
    "# - X_train - a dataframe containing all feature vectors of the train set\n",
    "# - y_train - a series of containing all class values per train instance\n",
    "# - instanceSampleRatio - the ratio of the sampeling out of training set, \n",
    "#                        * we will pass it on as a parameter, in order to sample instances.\n",
    "#                        - if instanceSampleRatio<=0, no instance bootstrap is done, and we leave\n",
    "#                          the training instances with no change.\n",
    "# - featureSampleRatio - the ratio of the sampeling out of feature set,\n",
    "#                        * we will pass it on as a parameter, in order to sample features.\n",
    "#                        - if featureSampleRatio<=0, no feature bootstrap is done, and we leave\n",
    "#                          the features with no change.\n",
    "# - classificationAlgo_pyClass - the python 'class' parameter of a classification algorithm \n",
    "#                                For instance, the above 'DecisionStump' class.\n",
    "#                                Note: passing the pyton class is similar to passing a method\n",
    "#                                      as a parameter.    \n",
    "# - numModels - number of models to train in bagging\n",
    "# Note: the X_train, y_train parameters could be instance bootsraped, feature bootsraped or both\n",
    "# ------------\n",
    "# return value:\n",
    "# - a list of trained models \n",
    "#  notes:\n",
    "#        * the number of models are detrminded by the input parameter \n",
    "# ---------------------\n",
    "def baggingFit(X_train, y_train, instanceSampleRatio, featureSampleRatio, classificationAlgo_pyClass, numModels):\n",
    "    res_li=[]\n",
    "    for x in range(numModels):\n",
    "        if instanceSampleRatio>0:\n",
    "            X_train_bootstraped,y_train_corrolated_labels=instanceBootStrap(X_train, y_train, instanceSampleRatio)\n",
    "            classificationModel=trainBootstrapedClassificationModel(X_train_bootstraped, y_train_corrolated_labels,classificationAlgo_pyClass)\n",
    "            res_li.append(classificationModel)\n",
    "        if featureSampleRatio>0:\n",
    "            X_train_bootstraped,y_train_corrolated_labels=instanceBootStrap(X_train, y_train, featureSampleRatio)\n",
    "            classificationModel=trainBootstrapedClassificationModel(X_train_bootstraped, y_train_corrolated_labels,classificationAlgo_pyClass)\n",
    "            res_li.append(classificationModel)\n",
    "    return (res_li)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing your implementation of the 'baggingFit' method ...\n",
      "check basic 'baggingFit' output validation ...\n",
      "trying to create inner bagged models prediction ...\n",
      "----> The 'baggingFit' test passed successfully :-) \n",
      "\n",
      "\n",
      "----------------------\n",
      "A few prediction examples:\n",
      "test instance [0]: actual: 1.0, predictions: '[bag-model 0]: 0.0; [bag-model 1]: 1.0; [bag-model 2]: 0.0; [bag-model 3]: 0.0; [bag-model 4]: 1.0; [bag-model 5]: 1.0'\n",
      "test instance [1]: actual: 1.0, predictions: '[bag-model 0]: 0.0; [bag-model 1]: 1.0; [bag-model 2]: 0.0; [bag-model 3]: 0.0; [bag-model 4]: 1.0; [bag-model 5]: 1.0'\n",
      "test instance [2]: actual: 0.0, predictions: '[bag-model 0]: 0.0; [bag-model 1]: 0.0; [bag-model 2]: 0.0; [bag-model 3]: 0.0; [bag-model 4]: 0.0; [bag-model 5]: 0.0'\n",
      "test instance [3]: actual: 1.0, predictions: '[bag-model 0]: 0.0; [bag-model 1]: 1.0; [bag-model 2]: 0.0; [bag-model 3]: 0.0; [bag-model 4]: 1.0; [bag-model 5]: 1.0'\n",
      "test instance [4]: actual: 0.0, predictions: '[bag-model 0]: 0.0; [bag-model 1]: 0.0; [bag-model 2]: 0.0; [bag-model 3]: 0.0; [bag-model 4]: 0.0; [bag-model 5]: 0.0'\n",
      "test instance [5]: actual: 0.0, predictions: '[bag-model 0]: 0.0; [bag-model 1]: 1.0; [bag-model 2]: 0.0; [bag-model 3]: 0.0; [bag-model 4]: 1.0; [bag-model 5]: 1.0'\n",
      "test instance [6]: actual: 1.0, predictions: '[bag-model 0]: 0.0; [bag-model 1]: 1.0; [bag-model 2]: 0.0; [bag-model 3]: 0.0; [bag-model 4]: 1.0; [bag-model 5]: 1.0'\n",
      "test instance [7]: actual: 0.0, predictions: '[bag-model 0]: 0.0; [bag-model 1]: 1.0; [bag-model 2]: 0.0; [bag-model 3]: 0.0; [bag-model 4]: 1.0; [bag-model 5]: 1.0'\n",
      "test instance [8]: actual: 0.0, predictions: '[bag-model 0]: 0.0; [bag-model 1]: 0.0; [bag-model 2]: 0.0; [bag-model 3]: 0.0; [bag-model 4]: 0.0; [bag-model 5]: 0.0'\n",
      "test instance [9]: actual: 0.0, predictions: '[bag-model 0]: 0.0; [bag-model 1]: 1.0; [bag-model 2]: 0.0; [bag-model 3]: 0.0; [bag-model 4]: 1.0; [bag-model 5]: 1.0'\n",
      "test instance [10]: actual: 0.0, predictions: '[bag-model 0]: 0.0; [bag-model 1]: 0.0; [bag-model 2]: 0.0; [bag-model 3]: 0.0; [bag-model 4]: 0.0; [bag-model 5]: 0.0'\n",
      "test instance [11]: actual: 0.0, predictions: '[bag-model 0]: 0.0; [bag-model 1]: 0.0; [bag-model 2]: 0.0; [bag-model 3]: 0.0; [bag-model 4]: 1.0; [bag-model 5]: 1.0'\n",
      "test instance [12]: actual: 1.0, predictions: '[bag-model 0]: 0.0; [bag-model 1]: 1.0; [bag-model 2]: 0.0; [bag-model 3]: 0.0; [bag-model 4]: 1.0; [bag-model 5]: 1.0'\n",
      "test instance [13]: actual: 1.0, predictions: '[bag-model 0]: 1.0; [bag-model 1]: 1.0; [bag-model 2]: 1.0; [bag-model 3]: 1.0; [bag-model 4]: 1.0; [bag-model 5]: 1.0'\n",
      "test instance [14]: actual: 0.0, predictions: '[bag-model 0]: 1.0; [bag-model 1]: 1.0; [bag-model 2]: 1.0; [bag-model 3]: 1.0; [bag-model 4]: 1.0; [bag-model 5]: 1.0'\n",
      "test instance [15]: actual: 1.0, predictions: '[bag-model 0]: 0.0; [bag-model 1]: 1.0; [bag-model 2]: 0.0; [bag-model 3]: 0.0; [bag-model 4]: 1.0; [bag-model 5]: 1.0'\n",
      "test instance [16]: actual: 0.0, predictions: '[bag-model 0]: 0.0; [bag-model 1]: 0.0; [bag-model 2]: 0.0; [bag-model 3]: 0.0; [bag-model 4]: 0.0; [bag-model 5]: 0.0'\n",
      "test instance [17]: actual: 0.0, predictions: '[bag-model 0]: 0.0; [bag-model 1]: 0.0; [bag-model 2]: 0.0; [bag-model 3]: 0.0; [bag-model 4]: 0.0; [bag-model 5]: 0.0'\n",
      "test instance [18]: actual: 0.0, predictions: '[bag-model 0]: 0.0; [bag-model 1]: 1.0; [bag-model 2]: 0.0; [bag-model 3]: 0.0; [bag-model 4]: 1.0; [bag-model 5]: 1.0'\n",
      "test instance [19]: actual: 0.0, predictions: '[bag-model 0]: 0.0; [bag-model 1]: 0.0; [bag-model 2]: 0.0; [bag-model 3]: 0.0; [bag-model 4]: 0.0; [bag-model 5]: 0.0'\n"
     ]
    }
   ],
   "source": [
    "# --------------------------------------------------------\n",
    "# Test of:\n",
    "### --- Graded tests for the 'baggingFit' method \n",
    "# Important Note: additional test might also be taken from our side.\n",
    "\n",
    "# --------------------------------------------------------\n",
    "# Temporary imports:\n",
    "import sklearn\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "# --------\n",
    "folderName = '.' + os.sep + 'data' \n",
    "datasetCsvFileName = folderName + os.sep + 'spam_notSpam.csv'\n",
    "# --------------------- \n",
    "print (\"Testing your implementation of the 'baggingFit' method ...\")\n",
    "dataset_forTesting = loadDataset(datasetCsvFileName)\n",
    "X_vectors, y_categories = separateTo_X_and_y(dataset_forTesting, 'class')\n",
    "y_categories = datasetCategoriesToNums(y_categories, 'spam')\n",
    "X_train, X_test, y_train, y_test = trainTestSplit(X_vectors, y_categories, 0.2, 25)\n",
    "# --------------------------------------------------------\n",
    "\n",
    "# --------------------------------------------------------\n",
    "allValidVals = lambda arrValues,validVals: False not in (val in validVals for val in arrValues) \n",
    "sameIndexes = lambda obj1,obj2: False not in (obj1.index.tolist()[n] == obj2.index.tolist()[n] for n in range(len(obj1.index))) \n",
    "innerElementTupple = lambda arrValues: False not in (isinstance(elem,tuple) for elem in arrValues)\n",
    "# --------------------------------------------------------\n",
    "instanceSampleRatio = 0.1\n",
    "featureSampleRatio = 0.5\n",
    "numModels = 3\n",
    "classification_PyClass = DecisionStump\n",
    "baggedModels =  baggingFit(X_train, y_train, instanceSampleRatio, featureSampleRatio, classification_PyClass, numModels)\n",
    "# --------------------------------------------------------\n",
    "\n",
    "print (\"check basic 'baggingFit' output validation ...\")\n",
    "assert baggedModels is not None, 'baggedModels object not initialized'\n",
    "assert isinstance(baggedModels, list), 'baggedModels object is not a list'\n",
    "assert None not in baggedModels, 'baggedModels should not include None elements'\n",
    "assert False not in [isinstance(elem,BootstrapFeatureClassifer) for elem in baggedModels], 'baggedModels should include only DecisionStump objects'\n",
    "\n",
    "print ('trying to create inner bagged models prediction ...')\n",
    "nExamples=20\n",
    "y_hat_arr = [classifierPredict(X_test.iloc[:nExamples],model) for model in baggedModels]\n",
    "assert False not in [allValidVals(y_hat.values,(0,1)) for y_hat in y_hat_arr], 'something went wrong with bagging inner models - invalid values found (valid values: 0 or 1)'\n",
    "\n",
    "print (\"----> The 'baggingFit' test passed successfully :-) \\n\")\n",
    "\n",
    "print ('\\n----------------------')\n",
    "print ('A few prediction examples:')\n",
    "for nInd in range(nExamples):\n",
    "    predictions = '; '.join('[bag-model %d]: %r' %(nPred,y_hat_arr[nPred].iloc[nInd]) for nPred in range(len(y_hat_arr)))\n",
    "    print ('test instance [%d]: actual: %r, predictions: %r' %(nInd,y_test.iloc[nInd],predictions))\n",
    "\n",
    "\n",
    "# --------------------------------------------------------\n",
    "if 'datasets' in sys.modules:\n",
    "    del (datasets)\n",
    "if 'train_test_split' in sys.modules:\n",
    "    del (train_test_split)\n",
    "if 'DecisionTreeClassifier' in sys.modules:\n",
    "    del (DecisionTreeClassifier)\n",
    "sys_modules = list(sys.modules.keys())\n",
    "for mdl in sys_modules:\n",
    "    if mdl.startswith('sklearn.'):\n",
    "        del(sys.modules[mdl]) \n",
    "del (sklearn)\n",
    "if 'sklearn' in sys.modules:\n",
    "    del (sys.modules['sklearn'])\n",
    "# --------------------------------------------------------\n",
    "\n",
    "# --------------------------------------------------------\n",
    "dataset_forTesting = None\n",
    "X_vectors = None\n",
    "y_categories = None\n",
    "X_train = None\n",
    "X_test = None\n",
    "y_train = None\n",
    "y_test = None\n",
    "y_hat = None\n",
    "\n",
    "instanceSampleRatio = None\n",
    "featureSampleRatio = None\n",
    "numModels = None\n",
    "classification_PyClass = None\n",
    "baggedModels = None\n",
    "\n",
    "nExamples = None\n",
    "y_hat_arr = None\n",
    "# --------------------------------------------------------\n",
    "\n",
    "# --------------------------------------------------------"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prediction\n",
    "The following cells perform the following:\n",
    "* step 1 - predict (voting)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# --------------------------------------------------------\n",
    "# method name: baggingPredict\n",
    "# --------------------------------------------------------\n",
    "\n",
    "# --------------------------------------------------------\n",
    "# The following is expected:\n",
    "# --- the method needs to return the predicted value for each test instances, \n",
    "#     based on the bagged trained models \n",
    "# ------------\n",
    "# input parameters:\n",
    "# - X_test -  a dataframe containing feature vectors of the test set\n",
    "# - baggingModels - the list of bagged models returned from 'baggingFit'\n",
    "# ------------\n",
    "# return value:\n",
    "# - y_hat - a series of the prediction, with the same index as X_test \n",
    "#  notes:\n",
    "#        * the number of models are detrminded by the input parameter \n",
    "# ---------------------\n",
    "def baggingPredict(X_test, baggingModels):\n",
    "    y_hat=pd.Series(index=X_test.index)\n",
    "    y_help=pd.DataFrame(index=X_test.index)\n",
    "    for baggedModel in baggingModels:\n",
    "        y_test=classifierPredict(X_test, baggedModel)\n",
    "        y_help[baggedModel]=y_test\n",
    "    counter=0\n",
    "    for index in y_help.index:\n",
    "        for cat in y_help[baggingModels]:\n",
    "            if y_help.loc[index,cat]==0:\n",
    "                counter-=1\n",
    "            else:\n",
    "                counter+=1\n",
    "        if counter>0:\n",
    "            y_hat[index]=1\n",
    "        else:\n",
    "            y_hat[index]=0\n",
    "        counter=0\n",
    "    return (y_hat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "check basic 'baggingPredict' output validation (part 1) ...\n",
      "----> The 'baggingPredict' test passed successfully :-) \n",
      "\n",
      "check basic 'baggingPredict' output validation (part 2) ...\n",
      "----> The 'baggingPredict' test passed successfully :-) \n",
      "\n",
      "\n",
      "----------------------\n",
      "A few prediction examples:\n",
      "test instance [0]: actual: 1.0, predictions: 1.0\n",
      "test instance [1]: actual: 1.0, predictions: 1.0\n",
      "test instance [2]: actual: 0.0, predictions: 0.0\n",
      "test instance [3]: actual: 1.0, predictions: 1.0\n",
      "test instance [4]: actual: 0.0, predictions: 0.0\n",
      "test instance [5]: actual: 0.0, predictions: 0.0\n",
      "test instance [6]: actual: 1.0, predictions: 1.0\n",
      "test instance [7]: actual: 0.0, predictions: 1.0\n",
      "test instance [8]: actual: 0.0, predictions: 0.0\n",
      "test instance [9]: actual: 0.0, predictions: 1.0\n",
      "test instance [10]: actual: 0.0, predictions: 0.0\n",
      "test instance [11]: actual: 0.0, predictions: 0.0\n",
      "test instance [12]: actual: 1.0, predictions: 1.0\n",
      "test instance [13]: actual: 1.0, predictions: 1.0\n",
      "test instance [14]: actual: 0.0, predictions: 1.0\n",
      "test instance [15]: actual: 1.0, predictions: 1.0\n",
      "test instance [16]: actual: 0.0, predictions: 0.0\n",
      "test instance [17]: actual: 0.0, predictions: 0.0\n",
      "test instance [18]: actual: 0.0, predictions: 0.0\n",
      "test instance [19]: actual: 0.0, predictions: 0.0\n"
     ]
    }
   ],
   "source": [
    "# --------------------------------------------------------\n",
    "# Test of:\n",
    "### --- Graded tests for the 'baggingPredict' method \n",
    "# --------------------------------------------------------\n",
    "# Temporary imports:\n",
    "import sklearn\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "# --------\n",
    "folderName = '.' + os.sep + 'data' \n",
    "datasetCsvFileName = folderName + os.sep + 'spam_notSpam.csv'\n",
    "# --------------------- \n",
    "dataset_forTesting = loadDataset(datasetCsvFileName)\n",
    "X_vectors, y_categories = separateTo_X_and_y(dataset_forTesting, 'class')\n",
    "y_categories = datasetCategoriesToNums(y_categories, 'spam')\n",
    "X_train, X_test, y_train, y_test = trainTestSplit(X_vectors, y_categories, 0.2, 25)\n",
    "# --------------------------------------------------------\n",
    "\n",
    "# --------------------------------------------------------\n",
    "allValidVals = lambda arrValues,validVals: False not in (val in validVals for val in arrValues) \n",
    "sameIndexes = lambda obj1,obj2: False not in (obj1.index.tolist()[n] == obj2.index.tolist()[n] for n in range(len(obj1.index))) \n",
    "innerElementTupple = lambda arrValues: False not in (isinstance(elem,tuple) for elem in arrValues)\n",
    "# --------------------------------------------------------\n",
    "\n",
    "# --------------------------------------------------------\n",
    "\n",
    "print (\"check basic 'baggingPredict' output validation (part 1) ...\")\n",
    "instanceSampleRatio = 0.1\n",
    "featureSampleRatio = 0.5\n",
    "numModels = 10\n",
    "classification_PyClass = DecisionStump\n",
    "baggedModels =  baggingFit(X_train, y_train, instanceSampleRatio, featureSampleRatio, classification_PyClass, numModels)\n",
    "yHat = baggingPredict(X_test, baggedModels)\n",
    "assert yHat is not None, 'yHat object not initialized'\n",
    "assert isinstance(yHat, pd.Series), 'yHat object is not a pandas series'\n",
    "assert None not in yHat.values, 'yHat should not include None elements'\n",
    "\n",
    "print (\"----> The 'baggingPredict' test passed successfully :-) \\n\")\n",
    "# --------------------------------------------------------\n",
    "print (\"check basic 'baggingPredict' output validation (part 2) ...\")\n",
    "assert allValidVals(yHat.values,(0,1)), 'prediction err - something went wrong with bagging inner models - invalid values found (valid values: 0 or 1)'\n",
    "\n",
    "print (\"----> The 'baggingPredict' test passed successfully :-) \\n\")\n",
    "print ('\\n----------------------')\n",
    "print ('A few prediction examples:')\n",
    "nExamples = 20\n",
    "for nInd in range(nExamples):\n",
    "    print ('test instance [%d]: actual: %r, predictions: %r' %(nInd,y_test.iloc[nInd],yHat.iloc[nInd]))\n",
    "\n",
    "\n",
    "# --------------------------------------------------------\n",
    "if 'datasets' in sys.modules:\n",
    "    del (datasets)\n",
    "if 'train_test_split' in sys.modules:\n",
    "    del (train_test_split)\n",
    "if 'DecisionTreeClassifier' in sys.modules:\n",
    "    del (DecisionTreeClassifier)\n",
    "sys_modules = list(sys.modules.keys())\n",
    "for mdl in sys_modules:\n",
    "    if mdl.startswith('sklearn.'):\n",
    "        del(sys.modules[mdl]) \n",
    "del (sklearn)\n",
    "if 'sklearn' in sys.modules:\n",
    "    del (sys.modules['sklearn'])\n",
    "# --------------------------------------------------------\n",
    "\n",
    "# --------------------------------------------------------\n",
    "dataset_forTesting = None\n",
    "X_vectors = None\n",
    "y_categories = None\n",
    "X_train = None\n",
    "X_test = None\n",
    "y_train = None\n",
    "y_test = None\n",
    "y_hat = None\n",
    "yHat = None\n",
    "\n",
    "instanceSampleRatio = None\n",
    "featureSampleRatio = None\n",
    "numModels = None\n",
    "classification_PyClass = None\n",
    "baggedModels = None\n",
    "\n",
    "nExamples = None\n",
    "y_hat_arr = None\n",
    "# --------------------------------------------------------\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluation\n",
    "The following cells perform the following:\n",
    "* step 1 - evaluate confusion matrix\n",
    "* step 2 - evaluate precision\n",
    "* step 3 - evaluate recall"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### evaluate confusion matrix\n",
    "<img src=\"./images/confusion_matrix.jpg\" alt=\"confusion_matrix\" width=\"300\" align='center'/>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "confusion matrix:\n",
      "\tpred-0\tpred-1\n",
      "is-0\tTN=534\tFP=11\n",
      "is-1\tFN=184\tTP=192\n"
     ]
    }
   ],
   "source": [
    "# --------------------------------------------------------\n",
    "# methos: getConfusionMatrix\n",
    "# --------------------------------------------------------\n",
    "\n",
    "# --------------------------------------------------------\n",
    "# What does the method do?\n",
    "# --- compute the 4 values of the confusion matrix: TN, FP, FN, TP\n",
    "#     - It does so using the sklearn built in method\n",
    "# ------------\n",
    "# input parameters:\n",
    "# - y_hat -  a series of containing all predicted class values per test instance\n",
    "# - y_test - a series of containing all class values per test instance\n",
    "# ------------\n",
    "# return values (comma seperated):\n",
    "# - TN - True negatives - number of instances, for which the actual value (from y_test) \n",
    "#                         is 0 (negative class, not_spam in our dataset) and \n",
    "#                         the predicted value (from y_hat) is also 0.\n",
    "# - FP - True negatives - number of instances, for which the actual value (from y_test) is 0, but \n",
    "#                         the predicted value (from y_hat) is 1 (positive class, spam in our dataset).\n",
    "# - FN - False negatives - number of instances, for which the actual value (from y_test) is 1, but                          \n",
    "#                         the predicted value (from y_hat) is 0.\n",
    "# - TP - False negatives - number of instances, for which the actual value (from y_test) is 1 and                          \n",
    "#                         the predicted value (from y_hat) is also 1.\n",
    "# --------------------- \n",
    "def getConfusionMatrix(y_hat,y_test):\n",
    "    TN = len([1 for ind in range(len(y_test)) if y_test.iloc[ind]==0 and y_hat.iloc[ind]==0])\n",
    "    FP = len([1 for ind in range(len(y_test)) if y_test.iloc[ind]==0 and y_hat.iloc[ind]==1])\n",
    "    FN = len([1 for ind in range(len(y_test)) if y_test.iloc[ind]==1 and y_hat.iloc[ind]==0])\n",
    "    TP = len([1 for ind in range(len(y_test)) if y_test.iloc[ind]==1 and y_hat.iloc[ind]==1])\n",
    "    return TN,FP,FN,TP\n",
    "\n",
    "\n",
    "# --------------------------------------------------------\n",
    "# Temporary imports:\n",
    "import sklearn\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "# --------\n",
    "folderName = '.' + os.sep + 'data' \n",
    "datasetCsvFileName = folderName + os.sep + 'spam_notSpam.csv'\n",
    "# --------------------- \n",
    "dataset_forTesting = loadDataset(datasetCsvFileName)\n",
    "X_vectors, y_categories = separateTo_X_and_y(dataset_forTesting, 'class')\n",
    "y_categories = datasetCategoriesToNums(y_categories, 'spam')\n",
    "X_train, X_test, y_train, y_test = trainTestSplit(X_vectors, y_categories, 0.2, 11)\n",
    "instanceSampleRatio = 0.1\n",
    "featureSampleRatio = 0.5\n",
    "numModels = 10\n",
    "classification_PyClass = DecisionStump\n",
    "baggedModels =  baggingFit(X_train, y_train, instanceSampleRatio, featureSampleRatio, classification_PyClass, numModels)\n",
    "y_hat = baggingPredict(X_test, baggedModels)\n",
    "# --------------------------------------------------------\n",
    "\n",
    "# --------------------------------------------------------\n",
    "numTN,numFP,numFN,numTP = getConfusionMatrix(y_hat,y_test)\n",
    "print ('confusion matrix:')\n",
    "print ('\\tpred-0\\tpred-1')\n",
    "print ('is-0\\tTN=%d\\tFP=%d' %(numTN,numFP))\n",
    "print ('is-1\\tFN=%d\\tTP=%d' %(numFN,numTP))\n",
    "# --------------------------------------------------------\n",
    "if 'datasets' in sys.modules:\n",
    "    del (datasets)\n",
    "if 'train_test_split' in sys.modules:\n",
    "    del (train_test_split)\n",
    "if 'DecisionTreeClassifier' in sys.modules:\n",
    "    del (DecisionTreeClassifier)\n",
    "sys_modules = list(sys.modules.keys())\n",
    "for mdl in sys_modules:\n",
    "    if mdl.startswith('sklearn.'):\n",
    "        del(sys.modules[mdl]) \n",
    "del (sklearn)\n",
    "if 'sklearn' in sys.modules:\n",
    "    del (sys.modules['sklearn'])\n",
    "# --------------------------------------------------------\n",
    "\n",
    "# --------------------------------------------------------\n",
    "dataset_forTesting = None\n",
    "X_vectors = None\n",
    "y_categories = None\n",
    "X_train = None\n",
    "X_test = None\n",
    "y_train = None\n",
    "y_test = None\n",
    "\n",
    "yHat = None\n",
    "\n",
    "instanceSampleRatio = None\n",
    "featureSampleRatio = None\n",
    "numModels = None\n",
    "classification_PyClass = None\n",
    "baggedModels = None\n",
    "\n",
    "nExamples = None\n",
    "\n",
    "# --------------------------------------------------------"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### evaluate precision\n",
    "<img src=\"./images/confusion_matrix-precision-recall.jpg\" alt=\"confusion_matrix-precision-recall\" width=\"300\" align='center'/>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# --------------------------------------------------------\n",
    "# method name: getPrecision\n",
    "# --------------------------------------------------------\n",
    "\n",
    "# --------------------------------------------------------\n",
    "# The following is expected:\n",
    "# --- the method needs to return the precision for class 1 (spam in out dataset)\n",
    "# ------------\n",
    "# input parameters:\n",
    "# - y_hat -  a series of containing all predicted class values per test instance\n",
    "# - y_test - a series of containing all class values per test instance\n",
    "# ------------\n",
    "# return value:\n",
    "# - precision value\n",
    "# ---------------------\n",
    "def getPrecision(y_hat,y_test):\n",
    "    test=getConfusionMatrix(y_hat,y_test)\n",
    "    #TN,FP,FN,TP\n",
    "    precision=(test[3]/(test[3]+test[1]))\n",
    "    return (precision)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "check basic 'getPrecision' output validation ...\n",
      "pricision=0.9455445544554455\n",
      "----> The 'getPrecision' test passed successfully :-) \n",
      "\n",
      "\n",
      "----------------------\n"
     ]
    }
   ],
   "source": [
    "# --------------------------------------------------------\n",
    "# Test of:\n",
    "### --- Graded tests for the 'getPrecision' method \n",
    "# --------------------------------------------------------\n",
    "# Temporary imports:\n",
    "import sklearn\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "# --------\n",
    "folderName = '.' + os.sep + 'data' \n",
    "datasetCsvFileName = folderName + os.sep + 'spam_notSpam.csv'\n",
    "# --------------------- \n",
    "dataset_forTesting = loadDataset(datasetCsvFileName)\n",
    "X_vectors, y_categories = separateTo_X_and_y(dataset_forTesting, 'class')\n",
    "y_categories = datasetCategoriesToNums(y_categories, 'spam')\n",
    "X_train, X_test, y_train, y_test = trainTestSplit(X_vectors, y_categories, 0.2, 25)\n",
    "instanceSampleRatio = 0.1\n",
    "featureSampleRatio = 0.5\n",
    "numModels = 5\n",
    "classification_PyClass = DecisionStump\n",
    "baggedModels =  baggingFit(X_train, y_train, instanceSampleRatio, featureSampleRatio, classification_PyClass, numModels)\n",
    "yHat = baggingPredict(X_test, baggedModels)\n",
    "# --------------------------------------------------------\n",
    "\n",
    "# --------------------------------------------------------\n",
    "allValidVals = lambda arrValues,validVals: False not in (val in validVals for val in arrValues) \n",
    "sameIndexes = lambda obj1,obj2: False not in (obj1.index.tolist()[n] == obj2.index.tolist()[n] for n in range(len(obj1.index))) \n",
    "innerElementTupple = lambda arrValues: False not in (isinstance(elem,tuple) for elem in arrValues)\n",
    "# --------------------------------------------------------\n",
    "\n",
    "# --------------------------------------------------------\n",
    "\n",
    "print (\"check basic 'getPrecision' output validation ...\")\n",
    "precision = getPrecision(yHat,y_test)\n",
    "assert precision is not None, 'precision not initialized'\n",
    "assert precision>0, 'precision should not be 0'\n",
    "assert precision<=1, 'precision should not be more than 1 (=100%)'\n",
    "assert precision>0.7, 'precision should not be >0.7 (more than 70%)'\n",
    "print ('pricision=%r' %(precision))\n",
    "print (\"----> The 'getPrecision' test passed successfully :-) \\n\")\n",
    "print ('\\n----------------------')\n",
    "\n",
    "\n",
    "# --------------------------------------------------------\n",
    "if 'datasets' in sys.modules:\n",
    "    del (datasets)\n",
    "if 'train_test_split' in sys.modules:\n",
    "    del (train_test_split)\n",
    "if 'DecisionTreeClassifier' in sys.modules:\n",
    "    del (DecisionTreeClassifier)\n",
    "sys_modules = list(sys.modules.keys())\n",
    "for mdl in sys_modules:\n",
    "    if mdl.startswith('sklearn.'):\n",
    "        del(sys.modules[mdl]) \n",
    "del (sklearn)\n",
    "if 'sklearn' in sys.modules:\n",
    "    del (sys.modules['sklearn'])\n",
    "# --------------------------------------------------------\n",
    "\n",
    "# --------------------------------------------------------\n",
    "dataset_forTesting = None\n",
    "X_vectors = None\n",
    "y_categories = None\n",
    "X_train = None\n",
    "X_test = None\n",
    "y_train = None\n",
    "y_test = None\n",
    "y_hat = None\n",
    "yHat = None\n",
    "\n",
    "instanceSampleRatio = None\n",
    "featureSampleRatio = None\n",
    "numModels = None\n",
    "classification_PyClass = None\n",
    "baggedModels = None\n",
    "\n",
    "nExamples = None\n",
    "y_hat_arr = None\n",
    "# --------------------------------------------------------\n",
    "\n",
    "# --------------------------------------------------------"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### evaluate recall\n",
    "<img src=\"./images/confusion_matrix-precision-recall.jpg\" alt=\"confusion_matrix-precision-recall\" width=\"300\" align='center'/>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# --------------------------------------------------------\n",
    "# method name: getRecall\n",
    "# --------------------------------------------------------\n",
    "\n",
    "# --------------------------------------------------------\n",
    "# The following is expected:\n",
    "# --- the method needs to return the recall for class 1 (spam in out dataset)\n",
    "# ------------\n",
    "# input parameters:\n",
    "# - y_hat -  a series of containing all predicted class values per test instance\n",
    "# - y_test - a series of containing all class values per test instance\n",
    "# ------------\n",
    "# return value:\n",
    "# - recall value\n",
    "# ---------------------\n",
    "def getRecall(y_hat,y_test):\n",
    "    test=getConfusionMatrix(y_hat,y_test)\n",
    "    #TN,FP,FN,TP\n",
    "    recall=test[3]/(test[3]+test[2])\n",
    "    return recall"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "check basic 'getRecall' output validation ...\n",
      "recall=0.7298850574712644\n",
      "----> The 'getRecall' test passed successfully :-) \n",
      "\n",
      "\n",
      "----------------------\n"
     ]
    }
   ],
   "source": [
    "# --------------------------------------------------------\n",
    "# Test of:\n",
    "### --- Graded tests for the 'getRecall' method \n",
    "# --------------------------------------------------------\n",
    "# Temporary imports:\n",
    "import sklearn\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "# --------\n",
    "folderName = '.' + os.sep + 'data' \n",
    "datasetCsvFileName = folderName + os.sep + 'spam_notSpam.csv'\n",
    "# --------------------- \n",
    "dataset_forTesting = loadDataset(datasetCsvFileName)\n",
    "X_vectors, y_categories = separateTo_X_and_y(dataset_forTesting, 'class')\n",
    "y_categories = datasetCategoriesToNums(y_categories, 'spam')\n",
    "X_train, X_test, y_train, y_test = trainTestSplit(X_vectors, y_categories, 0.2, 25)\n",
    "instanceSampleRatio = 0.1\n",
    "featureSampleRatio = 0.5\n",
    "numModels = 10\n",
    "classification_PyClass = DecisionStump\n",
    "baggedModels =  baggingFit(X_train, y_train, instanceSampleRatio, featureSampleRatio, classification_PyClass, numModels)\n",
    "yHat = baggingPredict(X_test, baggedModels)\n",
    "# --------------------------------------------------------\n",
    "\n",
    "# --------------------------------------------------------\n",
    "allValidVals = lambda arrValues,validVals: False not in (val in validVals for val in arrValues) \n",
    "sameIndexes = lambda obj1,obj2: False not in (obj1.index.tolist()[n] == obj2.index.tolist()[n] for n in range(len(obj1.index))) \n",
    "innerElementTupple = lambda arrValues: False not in (isinstance(elem,tuple) for elem in arrValues)\n",
    "# --------------------------------------------------------\n",
    "\n",
    "# --------------------------------------------------------\n",
    "\n",
    "print (\"check basic 'getRecall' output validation ...\")\n",
    "recall = getRecall(yHat,y_test)\n",
    "assert recall is not None, 'precision not initialized'\n",
    "assert recall>0, 'recall should not be 0'\n",
    "assert recall<=1, 'recall should not be more than 1 (=100%)'\n",
    "assert recall>0.3, 'recall should not be >0.5 (more than 50%)'\n",
    "print ('recall=%r' %(recall))\n",
    "print (\"----> The 'getRecall' test passed successfully :-) \\n\")\n",
    "\n",
    "print ('\\n----------------------')\n",
    "\n",
    "\n",
    "# --------------------------------------------------------\n",
    "if 'datasets' in sys.modules:\n",
    "    del (datasets)\n",
    "if 'train_test_split' in sys.modules:\n",
    "    del (train_test_split)\n",
    "if 'DecisionTreeClassifier' in sys.modules:\n",
    "    del (DecisionTreeClassifier)\n",
    "sys_modules = list(sys.modules.keys())\n",
    "for mdl in sys_modules:\n",
    "    if mdl.startswith('sklearn.'):\n",
    "        del(sys.modules[mdl]) \n",
    "del (sklearn)\n",
    "if 'sklearn' in sys.modules:\n",
    "    del (sys.modules['sklearn'])\n",
    "# --------------------------------------------------------\n",
    "\n",
    "# --------------------------------------------------------\n",
    "dataset_forTesting = None\n",
    "X_vectors = None\n",
    "y_categories = None\n",
    "X_train = None\n",
    "X_test = None\n",
    "y_train = None\n",
    "y_test = None\n",
    "y_hat = None\n",
    "yHat = None\n",
    "\n",
    "instanceSampleRatio = None\n",
    "featureSampleRatio = None\n",
    "numModels = None\n",
    "classification_PyClass = None\n",
    "baggedModels = None\n",
    "\n",
    "nExamples = None\n",
    "y_hat_arr = None\n",
    "# --------------------------------------------------------\n",
    "\n",
    "# --------------------------------------------------------"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### The following cells perform the following:\n",
    "* step 1 - evaluate accuracy\n",
    "* step 2 - split the data-set to train-set, validation-set and test-set\n",
    "* step 3 - bagging hyperparameters tuning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### evaluate accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy: 0.7806731813246471\n"
     ]
    }
   ],
   "source": [
    "# --------------------------------------------------------\n",
    "# methos: getConfusionMatrix\n",
    "# --------------------------------------------------------\n",
    "\n",
    "# --------------------------------------------------------\n",
    "# What does the method do?\n",
    "# --- computes the accuracy of the classifier\n",
    "# ------------\n",
    "# input parameters:\n",
    "# - y_hat -  a series of containing all predicted class values per test instance\n",
    "# - y_test - a series of containing all class values per test instance\n",
    "# ------------\n",
    "# return value:\n",
    "# - accuracy value\n",
    "# --------------------- \n",
    "def getAccuracy(y_hat,y_test):\n",
    "    correct = float(len([1 for ind in range(len(y_test)) if y_test.iloc[ind]==y_hat.iloc[ind]]))\n",
    "    return correct/float(len(y_test))\n",
    "\n",
    "\n",
    "# --------------------------------------------------------\n",
    "# Temporary imports:\n",
    "import sklearn\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "# --------\n",
    "folderName = '.' + os.sep + 'data' \n",
    "datasetCsvFileName = folderName + os.sep + 'spam_notSpam.csv'\n",
    "# --------------------- \n",
    "dataset_forTesting = loadDataset(datasetCsvFileName)\n",
    "X_vectors, y_categories = separateTo_X_and_y(dataset_forTesting, 'class')\n",
    "y_categories = datasetCategoriesToNums(y_categories, 'spam')\n",
    "X_train, X_test, y_train, y_test = trainTestSplit(X_vectors, y_categories, 0.2, 11)\n",
    "instanceSampleRatio = 0.1\n",
    "featureSampleRatio = 0.5\n",
    "numModels = 5\n",
    "classification_PyClass = DecisionStump\n",
    "baggedModels =  baggingFit(X_train, y_train, instanceSampleRatio, featureSampleRatio, classification_PyClass, numModels)\n",
    "y_hat = baggingPredict(X_test, baggedModels)\n",
    "# --------------------------------------------------------\n",
    "\n",
    "# --------------------------------------------------------\n",
    "accuracy = getAccuracy(y_hat,y_test)\n",
    "print ('accuracy: %r' %(accuracy))\n",
    "# --------------------------------------------------------\n",
    "if 'datasets' in sys.modules:\n",
    "    del (datasets)\n",
    "if 'train_test_split' in sys.modules:\n",
    "    del (train_test_split)\n",
    "if 'DecisionTreeClassifier' in sys.modules:\n",
    "    del (DecisionTreeClassifier)\n",
    "sys_modules = list(sys.modules.keys())\n",
    "for mdl in sys_modules:\n",
    "    if mdl.startswith('sklearn.'):\n",
    "        del(sys.modules[mdl]) \n",
    "del (sklearn)\n",
    "if 'sklearn' in sys.modules:\n",
    "    del (sys.modules['sklearn'])\n",
    "# --------------------------------------------------------\n",
    "\n",
    "# --------------------------------------------------------\n",
    "dataset_forTesting = None\n",
    "X_vectors = None\n",
    "y_categories = None\n",
    "X_train = None\n",
    "X_test = None\n",
    "y_train = None\n",
    "y_test = None\n",
    "\n",
    "yHat = None\n",
    "\n",
    "instanceSampleRatio = None\n",
    "featureSampleRatio = None\n",
    "numModels = None\n",
    "classification_PyClass = None\n",
    "baggedModels = None\n",
    "\n",
    "nExamples = None\n",
    "\n",
    "# --------------------------------------------------------"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### split the data-set to train-set, validation-set and test-set\n",
    "<img src=\"./images/train-validation-test-split.png\" alt=\"train-validation-test split\" width=\"300\" align='center'/>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Information after train-test split:\n",
      "* The train-set includes 2760 instances and 2760 corresponding categories\n",
      "* The validation-set includes 920 instances and 920 corresponding categories\n",
      "* The test-set includes 921 instances and 921 corresponding categories\n"
     ]
    }
   ],
   "source": [
    "#--------------------------------------------------------\n",
    "# method name: trainValidationTestSplit\n",
    "# --------------------------------------------------------\n",
    "\n",
    "# --------------------------------------------------------\n",
    "# What does the method do?\n",
    "# --- the method split the input dataset into train and test. \n",
    "#     - It does so using the sklearn built in method\n",
    "# ------------\n",
    "# input parameters:\n",
    "# - X_vectors - a dataframe containing all feature vectors. \n",
    "# - y_categories - a series of containing all class values per instance.\n",
    "# - test_size_ratio - a number (0<number<1) of the wanted ratio of the dataset out of the dataset \n",
    "# - rand_state - a number, in order to guarantee reproducible results \n",
    "# ------------\n",
    "# return values (comma separated):\n",
    "# - X_train -  a dataframe containing all feature vectors of the train set\n",
    "# - X_test -  a dataframe containing all feature vectors of the test set\n",
    "# - y_train - a series of containing all class values per train instance\n",
    "# - y_test - a series of containing all class values per test instance\n",
    "# --------------------- \n",
    "def trainValidationTestSplit(X_vectors, y_categories, validation_or_test_size_ratio, rand_state):\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X_vectors, y_categories, test_size=validation_or_test_size_ratio, random_state=rand_state)\n",
    "    validationRatio = validation_or_test_size_ratio / (1-validation_or_test_size_ratio)\n",
    "    X_train, X_validation, y_train, y_validation =  train_test_split(X_train, y_train, test_size=validationRatio, random_state=rand_state)\n",
    "    return X_train, X_validation, X_test, y_train, y_validation, y_test  \n",
    "\n",
    "# --------------------------------------------------------\n",
    "# Temporary imports:\n",
    "import sklearn\n",
    "from sklearn.model_selection import train_test_split\n",
    "# --------\n",
    "folderName = '.' + os.sep + 'data' \n",
    "datasetCsvFileName = folderName + os.sep + 'spam_notSpam.csv'\n",
    "# --------------------- \n",
    "dataset_forTesting = loadDataset(datasetCsvFileName)\n",
    "X_vectors, y_categories = separateTo_X_and_y(dataset_forTesting, 'class')\n",
    "y_categories = datasetCategoriesToNums(y_categories, 'spam')\n",
    "X_train, X_validation, X_test, y_train, y_validation, y_test =  trainValidationTestSplit(X_vectors, y_categories, 0.2, 43)\n",
    "\n",
    "# --------------------------------------------------------\n",
    "\n",
    "# --------------------------------------------------------\n",
    "if 'datasets' in sys.modules:\n",
    "    del (datasets)\n",
    "if 'train_test_split' in sys.modules:\n",
    "    del (train_test_split)\n",
    "sys_modules = list(sys.modules.keys())\n",
    "for mdl in sys_modules:\n",
    "    if mdl.startswith('sklearn.'):\n",
    "        del(sys.modules[mdl]) \n",
    "del (sklearn)\n",
    "if 'sklearn' in sys.modules:\n",
    "    del (sys.modules['sklearn'])\n",
    "# --------------------------------------------------------\n",
    "\n",
    "# --------------------------------------------------------\n",
    "# display train-test split information\n",
    "# --------------------------------------------------------\n",
    "print ('Information after train-test split:')\n",
    "print('* The train-set includes %d instances and %d corresponding categories' %(X_train.shape[0],y_train.shape[0]))\n",
    "print('* The validation-set includes %d instances and %d corresponding categories' %(X_validation.shape[0],y_validation.shape[0]))\n",
    "print('* The test-set includes %d instances and %d corresponding categories' %(X_test.shape[0],y_test.shape[0]))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### bagging hyperparameters tuning (using grid search)\n",
    "<img src=\"./images/grid_search.png\" alt=\"train-validation-test split\" width=\"100\" align='center'/>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# --------------------------------------------------------\n",
    "# method name: baggingFit\n",
    "# --------------------------------------------------------\n",
    "\n",
    "# --------------------------------------------------------\n",
    "# The following is expected:\n",
    "# --- the method chooses the best permutation of bagging model hyperparameters, using grid search\n",
    "# ------------\n",
    "# input parameters:\n",
    "# - X_train - a dataframe containing all feature vectors of the train set\n",
    "# - y_train - a series of containing all class values per train instance\n",
    "# - instanceSampleRatio - the ratio of the sampeling out of training set, \n",
    "#                        * we will pass it on as a parameter, in order to sample instances.\n",
    "#                        - if instanceSampleRatio<=0, no instance bootstrap is done, and we leave\n",
    "#                          the training instances with no change.\n",
    "# - featureSampleRatio - the ratio of the sampeling out of feature set,\n",
    "#                        * we will pass it on as a parameter, in order to sample features.\n",
    "#                        - if featureSampleRatio<=0, no feature bootstrap is done, and we leave\n",
    "#                          the features with no change.\n",
    "# - classificationAlgo_pyClass_Arr - an array of the python 'class' parameters of a classification algorithm to choose from,\n",
    "#                                For instance, the above [DecisionStump, NaiveBayes].\n",
    "#                                Note: passing the pyton class is similar to passing a method\n",
    "#                                      as a parameter.    \n",
    "# - numModels_Arr - an array of the options of the number of bagging models to train, for which we need to choose from\n",
    "#                   For instance: [3,5,10]\n",
    "# Note: the X_train, y_train parameters could be instance bootsraped, feature bootsraped or both\n",
    "# ------------\n",
    "# return values (comma seperated):\n",
    "# - allBaggedModels - an array of all trainedBaggedModels,\n",
    "# - best_baggedModels - array of the baggedModels ensembles (from baggeningFit), reaching the highest accuracy\n",
    "# - bestAccuracy - the highest accuracy (number) of all baggedModels ensemble\n",
    "# - best_classificationAlgo_pyClass - the classificationAlgo_pyClass (e.g. DecisionStump) of\n",
    "#                                     baggedModels ensemble with highest accuracy\n",
    "# - best_numModels - the num of 'bagged' Models (e.g. 5) of\n",
    "#                    baggedModels ensemble with highest accuracy\n",
    "def gridSearchBaggingModel(X_train, y_train, X_validation, y_validation, instanceSampleRatio, \n",
    "                           featureSampleRatio, classificationAlgo_pyClass_Arr, numModels_Arr):\n",
    "    help_df=pd.DataFrame(columns=['baggedModels','acc','numModels'])\n",
    "    allBaggedModels=[]\n",
    "    best_baggedModels=[]\n",
    "    counter=0\n",
    "    for x in numModels_Arr:   \n",
    "        for y in classificationAlgo_pyClass_Arr:\n",
    "            fit=baggingFit(X_train, y_train, instanceSampleRatio, featureSampleRatio, y, x)\n",
    "            predict=baggingPredict(X_validation,fit)\n",
    "            acc=getAccuracy(predict,y_validation)\n",
    "            for m in fit:\n",
    "                help_df.loc[counter,'baggedModels']=m\n",
    "                help_df.loc[counter,'algo']=y\n",
    "                help_df.loc[counter,'acc']=acc\n",
    "                help_df.loc[counter,'numModels']=x\n",
    "                counter+=1\n",
    "                allBaggedModels.append(m)\n",
    "    bestAccuracy=help_df['acc'].max()\n",
    "    best_baggedModels=help_df.loc[help_df['acc']==bestAccuracy,'baggedModels']\n",
    "    best_baggedModels=list(best_baggedModels)\n",
    "    best_classificationAlgo_pyClass=help_df.loc[help_df['acc']==bestAccuracy,'algo'].iloc[0]\n",
    "    best_numModels=help_df.loc[help_df['acc']==bestAccuracy,'numModels'].iloc[0]\n",
    "    return (allBaggedModels,best_baggedModels,bestAccuracy,best_classificationAlgo_pyClass,best_numModels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "check basic 'gridSearchBaggingModel' output validation ...\n",
      "----> The 'gridSearchBaggingModel' test passed successfully :-) \n",
      "\n",
      "\n",
      "----------------------\n",
      "Best bagged Classification algo: 'GaussianNB', num of models: 40, accuracy: 0.8391304347826087\n"
     ]
    }
   ],
   "source": [
    "# --------------------------------------------------------\n",
    "# Test of:\n",
    "### --- Graded tests for the 'getRecall' method \n",
    "# --------------------------------------------------------\n",
    "# Temporary imports:\n",
    "import sklearn\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "# --------\n",
    "folderName = '.' + os.sep + 'data' \n",
    "datasetCsvFileName = folderName + os.sep + 'spam_notSpam.csv'\n",
    "# --------------------- \n",
    "dataset_forTesting = loadDataset(datasetCsvFileName)\n",
    "X_vectors, y_categories = separateTo_X_and_y(dataset_forTesting, 'class')\n",
    "y_categories = datasetCategoriesToNums(y_categories, 'spam')\n",
    "X_train, X_validation, X_test, y_train, y_validation, y_test =  trainValidationTestSplit(X_vectors, y_categories, 0.2, 19)\n",
    "# --------------------------------------------------------\n",
    "\n",
    "# --------------------------------------------------------\n",
    "allValidVals = lambda arrValues,validVals: False not in (val in validVals for val in arrValues) \n",
    "sameIndexes = lambda obj1,obj2: False not in (obj1.index.tolist()[n] == obj2.index.tolist()[n] for n in range(len(obj1.index))) \n",
    "innerElementTupple = lambda arrValues: False not in (isinstance(elem,tuple) for elem in arrValues)\n",
    "# --------------------------------------------------------\n",
    "\n",
    "# --------------------------------------------------------\n",
    "instanceSampleRatio = 0.1\n",
    "featureSampleRatio = 0.5\n",
    "numModels_Arr = [5,10,40]\n",
    "classificationAlgo_pyClass_Arr = [DecisionStump,GaussianNB]\n",
    "allBaggedModels,best_baggedModels,bestAccuracy, best_classificationAlgo_pyClass,best_numOfModels = gridSearchBaggingModel(\n",
    "                        X_train, y_train, X_validation, y_validation, instanceSampleRatio, featureSampleRatio, \n",
    "                                                         classificationAlgo_pyClass_Arr, numModels_Arr)\n",
    "\n",
    "print (\"check basic 'gridSearchBaggingModel' output validation ...\")\n",
    "assert allBaggedModels is not None, 'allBaggedModels not initialized'\n",
    "assert best_baggedModels is not None, 'best_baggedModels not initialized'\n",
    "assert bestAccuracy is not None, 'bestAccuracy not initialized'\n",
    "assert best_classificationAlgo_pyClass is not None, 'best_classificationAlgo_pyClass not initialized'\n",
    "assert best_numOfModels is not None, 'best_numOfModels not initialized'\n",
    "assert isinstance(allBaggedModels, list), 'allBaggedModels object is not a list'\n",
    "assert isinstance(best_baggedModels, list), 'best_baggedModels object is not a list'\n",
    "assert bestAccuracy>0, 'bestAccuracy should not be 0'\n",
    "assert bestAccuracy<=1, 'bestAccuracy should not be more than 1 (=100%)'\n",
    "assert bestAccuracy>0.6, 'bestAccuracy should not be >0.6 (more than 60%)'\n",
    "assert None not in allBaggedModels, 'allBaggedModels should not include None elements'\n",
    "assert None not in best_baggedModels, 'best_baggedModels should not include None elements'\n",
    "\n",
    "print (\"----> The 'gridSearchBaggingModel' test passed successfully :-) \\n\")\n",
    "\n",
    "print ('\\n----------------------')\n",
    "print ('Best bagged Classification algo: %r, num of models: %r, accuracy: %r' %(best_classificationAlgo_pyClass.__name__,best_numOfModels,bestAccuracy))\n",
    "\n",
    "\n",
    "# --------------------------------------------------------\n",
    "if 'datasets' in sys.modules:\n",
    "    del (datasets)\n",
    "if 'train_test_split' in sys.modules:\n",
    "    del (train_test_split)\n",
    "if 'DecisionTreeClassifier' in sys.modules:\n",
    "    del (DecisionTreeClassifier)\n",
    "sys_modules = list(sys.modules.keys())\n",
    "for mdl in sys_modules:\n",
    "    if mdl.startswith('sklearn.'):\n",
    "        del(sys.modules[mdl]) \n",
    "del (sklearn)\n",
    "if 'sklearn' in sys.modules:\n",
    "    del (sys.modules['sklearn'])\n",
    "# --------------------------------------------------------\n",
    "\n",
    "# --------------------------------------------------------\n",
    "dataset_forTesting = None\n",
    "X_vectors = None\n",
    "y_categories = None\n",
    "X_train = None\n",
    "X_test = None\n",
    "y_train = None\n",
    "y_test = None\n",
    "y_hat = None\n",
    "yHat = None\n",
    "\n",
    "instanceSampleRatio = None\n",
    "featureSampleRatio = None\n",
    "numModels = None\n",
    "classification_PyClass = None\n",
    "baggedModels = None\n",
    "\n",
    "nExamples = None\n",
    "y_hat_arr = None\n",
    "# --------------------------------------------------------\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}