hp_catboost.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "8FLud1n-3pVm"
   },
   "source": [
    "# CatBoost"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "sO8VPU6n3vES"
   },
   "source": [
    "## Importing the libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "clDSsF7P33NU"
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "#  Make sure all print() lines are printed, not just the last one\n",
    "from IPython.core.interactiveshell import InteractiveShell\n",
    "InteractiveShell.ast_node_interactivity = \"all\"\n",
    "\n",
    "# Make sure matplotlib charts and graphs are displayed in the cell outputs\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "zGpwK5XD386E"
   },
   "source": [
    "## Importing the dataset\n",
    "\n",
    "Normally would clean data, but from CatBoost tutorial:\n",
    "\n",
    "> Pay attention that our features are of differnt types - some of them are numeric, some are categorical, and some are even just strings, which normally should be handled in some specific way (for example encoded with bag-of-words representation). But in our case we could treat these string features just as categorical one - all the heavy lifting is done inside CatBoost. How cool is that? :)\n",
    "\n",
    "Also, they deal with missing data by:\n",
    ">As we cat see, Age, Cabin and Embarked indeed have some missing values, so let's fill them with some number way out of their distributions - so the model would be able to easily distinguish between them and take it into account:\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Zcksk88u4Ae8"
   },
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "   Id  MSSubClass MSZoning  LotFrontage  LotArea Street Alley LotShape  \\\n",
       "0   1          60       RL         65.0     8450   Pave   NaN      Reg   \n",
       "1   2          20       RL         80.0     9600   Pave   NaN      Reg   \n",
       "2   3          60       RL         68.0    11250   Pave   NaN      IR1   \n",
       "3   4          70       RL         60.0     9550   Pave   NaN      IR1   \n",
       "4   5          60       RL         84.0    14260   Pave   NaN      IR1   \n",
       "\n",
       "  LandContour Utilities  ... PoolArea PoolQC Fence MiscFeature MiscVal MoSold  \\\n",
       "0         Lvl    AllPub  ...        0    NaN   NaN         NaN       0      2   \n",
       "1         Lvl    AllPub  ...        0    NaN   NaN         NaN       0      5   \n",
       "2         Lvl    AllPub  ...        0    NaN   NaN         NaN       0      9   \n",
       "3         Lvl    AllPub  ...        0    NaN   NaN         NaN       0      2   \n",
       "4         Lvl    AllPub  ...        0    NaN   NaN         NaN       0     12   \n",
       "\n",
       "  YrSold  SaleType  SaleCondition  SalePrice  \n",
       "0   2008        WD         Normal     208500  \n",
       "1   2007        WD         Normal     181500  \n",
       "2   2008        WD         Normal     223500  \n",
       "3   2006        WD        Abnorml     140000  \n",
       "4   2008        WD         Normal     250000  \n",
       "\n",
       "[5 rows x 81 columns]"
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Id</th>\n      <th>MSSubClass</th>\n      <th>MSZoning</th>\n      <th>LotFrontage</th>\n      <th>LotArea</th>\n      <th>Street</th>\n      <th>Alley</th>\n      <th>LotShape</th>\n      <th>LandContour</th>\n      <th>Utilities</th>\n      <th>...</th>\n      <th>PoolArea</th>\n      <th>PoolQC</th>\n      <th>Fence</th>\n      <th>MiscFeature</th>\n      <th>MiscVal</th>\n      <th>MoSold</th>\n      <th>YrSold</th>\n      <th>SaleType</th>\n      <th>SaleCondition</th>\n      <th>SalePrice</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1</td>\n      <td>60</td>\n      <td>RL</td>\n      <td>65.0</td>\n      <td>8450</td>\n      <td>Pave</td>\n      <td>NaN</td>\n      <td>Reg</td>\n      <td>Lvl</td>\n      <td>AllPub</td>\n      <td>...</td>\n      <td>0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>0</td>\n      <td>2</td>\n      <td>2008</td>\n      <td>WD</td>\n      <td>Normal</td>\n      <td>208500</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>2</td>\n      <td>20</td>\n      <td>RL</td>\n      <td>80.0</td>\n      <td>9600</td>\n      <td>Pave</td>\n      <td>NaN</td>\n      <td>Reg</td>\n      <td>Lvl</td>\n      <td>AllPub</td>\n      <td>...</td>\n      <td>0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>0</td>\n      <td>5</td>\n      <td>2007</td>\n      <td>WD</td>\n      <td>Normal</td>\n      <td>181500</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>3</td>\n      <td>60</td>\n      <td>RL</td>\n      <td>68.0</td>\n      <td>11250</td>\n      <td>Pave</td>\n      <td>NaN</td>\n      <td>IR1</td>\n      <td>Lvl</td>\n      <td>AllPub</td>\n      <td>...</td>\n      <td>0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>0</td>\n      <td>9</td>\n      <td>2008</td>\n      <td>WD</td>\n      <td>Normal</td>\n      <td>223500</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>4</td>\n      <td>70</td>\n      <td>RL</td>\n      <td>60.0</td>\n      <td>9550</td>\n      <td>Pave</td>\n      <td>NaN</td>\n      <td>IR1</td>\n      <td>Lvl</td>\n      <td>AllPub</td>\n      <td>...</td>\n      <td>0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>0</td>\n      <td>2</td>\n      <td>2006</td>\n      <td>WD</td>\n      <td>Abnorml</td>\n      <td>140000</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>5</td>\n      <td>60</td>\n      <td>RL</td>\n      <td>84.0</td>\n      <td>14260</td>\n      <td>Pave</td>\n      <td>NaN</td>\n      <td>IR1</td>\n      <td>Lvl</td>\n      <td>AllPub</td>\n      <td>...</td>\n      <td>0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>0</td>\n      <td>12</td>\n      <td>2008</td>\n      <td>WD</td>\n      <td>Normal</td>\n      <td>250000</td>\n    </tr>\n  </tbody>\n</table>\n<p>5 rows × 81 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 2
    },
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "     Id  MSSubClass MSZoning  LotFrontage  LotArea Street Alley LotShape  \\\n",
       "0  1461          20       RH         80.0    11622   Pave   NaN      Reg   \n",
       "1  1462          20       RL         81.0    14267   Pave   NaN      IR1   \n",
       "2  1463          60       RL         74.0    13830   Pave   NaN      IR1   \n",
       "3  1464          60       RL         78.0     9978   Pave   NaN      IR1   \n",
       "4  1465         120       RL         43.0     5005   Pave   NaN      IR1   \n",
       "\n",
       "  LandContour Utilities  ... ScreenPorch PoolArea PoolQC  Fence MiscFeature  \\\n",
       "0         Lvl    AllPub  ...         120        0    NaN  MnPrv         NaN   \n",
       "1         Lvl    AllPub  ...           0        0    NaN    NaN        Gar2   \n",
       "2         Lvl    AllPub  ...           0        0    NaN  MnPrv         NaN   \n",
       "3         Lvl    AllPub  ...           0        0    NaN    NaN         NaN   \n",
       "4         HLS    AllPub  ...         144        0    NaN    NaN         NaN   \n",
       "\n",
       "  MiscVal MoSold  YrSold  SaleType  SaleCondition  \n",
       "0       0      6    2010        WD         Normal  \n",
       "1   12500      6    2010        WD         Normal  \n",
       "2       0      3    2010        WD         Normal  \n",
       "3       0      6    2010        WD         Normal  \n",
       "4       0      1    2010        WD         Normal  \n",
       "\n",
       "[5 rows x 80 columns]"
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Id</th>\n      <th>MSSubClass</th>\n      <th>MSZoning</th>\n      <th>LotFrontage</th>\n      <th>LotArea</th>\n      <th>Street</th>\n      <th>Alley</th>\n      <th>LotShape</th>\n      <th>LandContour</th>\n      <th>Utilities</th>\n      <th>...</th>\n      <th>ScreenPorch</th>\n      <th>PoolArea</th>\n      <th>PoolQC</th>\n      <th>Fence</th>\n      <th>MiscFeature</th>\n      <th>MiscVal</th>\n      <th>MoSold</th>\n      <th>YrSold</th>\n      <th>SaleType</th>\n      <th>SaleCondition</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1461</td>\n      <td>20</td>\n      <td>RH</td>\n      <td>80.0</td>\n      <td>11622</td>\n      <td>Pave</td>\n      <td>NaN</td>\n      <td>Reg</td>\n      <td>Lvl</td>\n      <td>AllPub</td>\n      <td>...</td>\n      <td>120</td>\n      <td>0</td>\n      <td>NaN</td>\n      <td>MnPrv</td>\n      <td>NaN</td>\n      <td>0</td>\n      <td>6</td>\n      <td>2010</td>\n      <td>WD</td>\n      <td>Normal</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1462</td>\n      <td>20</td>\n      <td>RL</td>\n      <td>81.0</td>\n      <td>14267</td>\n      <td>Pave</td>\n      <td>NaN</td>\n      <td>IR1</td>\n      <td>Lvl</td>\n      <td>AllPub</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>Gar2</td>\n      <td>12500</td>\n      <td>6</td>\n      <td>2010</td>\n      <td>WD</td>\n      <td>Normal</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>1463</td>\n      <td>60</td>\n      <td>RL</td>\n      <td>74.0</td>\n      <td>13830</td>\n      <td>Pave</td>\n      <td>NaN</td>\n      <td>IR1</td>\n      <td>Lvl</td>\n      <td>AllPub</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>NaN</td>\n      <td>MnPrv</td>\n      <td>NaN</td>\n      <td>0</td>\n      <td>3</td>\n      <td>2010</td>\n      <td>WD</td>\n      <td>Normal</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>1464</td>\n      <td>60</td>\n      <td>RL</td>\n      <td>78.0</td>\n      <td>9978</td>\n      <td>Pave</td>\n      <td>NaN</td>\n      <td>IR1</td>\n      <td>Lvl</td>\n      <td>AllPub</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>0</td>\n      <td>6</td>\n      <td>2010</td>\n      <td>WD</td>\n      <td>Normal</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>1465</td>\n      <td>120</td>\n      <td>RL</td>\n      <td>43.0</td>\n      <td>5005</td>\n      <td>Pave</td>\n      <td>NaN</td>\n      <td>IR1</td>\n      <td>HLS</td>\n      <td>AllPub</td>\n      <td>...</td>\n      <td>144</td>\n      <td>0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>0</td>\n      <td>1</td>\n      <td>2010</td>\n      <td>WD</td>\n      <td>Normal</td>\n    </tr>\n  </tbody>\n</table>\n<p>5 rows × 80 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 2
    }
   ],
   "source": [
    "train_df = pd.read_csv(\"hp_train.csv\")  #   NB:  Dependent variable in LAST COLUMN\n",
    "test_df = pd.read_csv(\"hp_test.csv\")\n",
    "train_df.head()\n",
    "test_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "LotFrontage      259\n",
       "Alley           1369\n",
       "MasVnrType         8\n",
       "MasVnrArea         8\n",
       "BsmtQual          37\n",
       "BsmtCond          37\n",
       "BsmtExposure      38\n",
       "BsmtFinType1      37\n",
       "BsmtFinType2      38\n",
       "Electrical         1\n",
       "FireplaceQu      690\n",
       "GarageType        81\n",
       "GarageYrBlt       81\n",
       "GarageFinish      81\n",
       "GarageQual        81\n",
       "GarageCond        81\n",
       "PoolQC          1453\n",
       "Fence           1179\n",
       "MiscFeature     1406\n",
       "dtype: int64"
      ]
     },
     "metadata": {},
     "execution_count": 3
    }
   ],
   "source": [
    "# See how much NaN\n",
    "\n",
    "null_value_stats = train_df.isnull().sum(axis=0)\n",
    "null_value_stats[null_value_stats != 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Let's fill them with some number way out of their distributions - so the model would be able to easily distinguish between them and take it into account\n",
    "# Do for both train and test datasets\n",
    "\n",
    "train_df.fillna(-999, inplace=True)\n",
    "test_df.fillna(-999, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "XNn2RnST6_Q-"
   },
   "source": [
    "## Splitting the dataset into the Training set and Test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ajhBL-er7Gry"
   },
   "outputs": [],
   "source": [
    "# Separate featrues and label values\n",
    "\n",
    "# NOTE: dependent variable IN LAST COLUMN\n",
    "\n",
    "X = train_df.iloc[:, :-1]  #   Note: this differs from Udemy course where .value is added\n",
    "y = train_df.iloc[:, -1]  #   Note: this differs from Udemy course where .value is added\n",
    "\n",
    "# NB: convert y into type float\n",
    "y = y.astype(float)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "pandas.core.frame.DataFrame"
      ]
     },
     "metadata": {},
     "execution_count": 6
    }
   ],
   "source": [
    "type(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Id                 int64\nMSSubClass         int64\nMSZoning          object\nLotFrontage      float64\nLotArea            int64\n                  ...   \nMiscVal            int64\nMoSold             int64\nYrSold             int64\nSaleType          object\nSaleCondition     object\nLength: 80, dtype: object\n[ 0  1  2  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24\n 25 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49\n 50 51 52 53 54 55 56 57 58 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74\n 75 76 77 78 79]\n"
     ]
    }
   ],
   "source": [
    "# See data types - categorical, strings, numbers\n",
    "\n",
    "print(X.dtypes)\n",
    "\n",
    "categorical_features_indices = np.where(X.dtypes != np.float)[0]  # Try float, int as well as object\n",
    "print(categorical_features_indices)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "#  Note difference below from Udemy templates: using X_validation instead of X_test\n",
    "X_train, X_validation, y_train, y_validation = train_test_split(\n",
    "    X, y, train_size=0.75, random_state=42\n",
    ")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#y_train"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "1Y89ctGZ7Mcx"
   },
   "source": [
    "## Training CatBoost on the Training set\n",
    "Create the model itself"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make necessary imports\n",
    "from catboost import CatBoostRegressor, Pool, cv\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "# # To show plots\n",
    "# from ipywidgets import interact  \n",
    "# import ipywidgets as widgets  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "output_type": "error",
     "ename": "CatBoostError",
     "evalue": "Bad value for num_feature[non_default_doc_idx=0,feature_idx=2]=\"RL\": Cannot convert 'b'RL'' to float",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m_catboost.pyx\u001b[0m in \u001b[0;36m_catboost.get_float_feature\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32m_catboost.pyx\u001b[0m in \u001b[0;36m_catboost._FloatOrNan\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32m_catboost.pyx\u001b[0m in \u001b[0;36m_catboost._FloatOrNanFromString\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: Cannot convert 'b'RL'' to float",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mCatBoostError\u001b[0m                             Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-11-79583a79ab8f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;31m# train the model\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m model.fit(\n\u001b[0m\u001b[1;32m     11\u001b[0m     \u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m     \u001b[0my_train\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/opt/anaconda3/lib/python3.8/site-packages/catboost/core.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X, y, cat_features, sample_weight, baseline, use_best_model, eval_set, verbose, logging_level, plot, column_description, verbose_eval, metric_period, silent, early_stopping_rounds, save_snapshot, snapshot_file, snapshot_interval, init_model)\u001b[0m\n\u001b[1;32m   4738\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_check_is_regressor_loss\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'loss_function'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   4739\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 4740\u001b[0;31m         return self._fit(X, y, cat_features, None, None, None, sample_weight, None, None, None, None, baseline,\n\u001b[0m\u001b[1;32m   4741\u001b[0m                          \u001b[0muse_best_model\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0meval_set\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogging_level\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mplot\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcolumn_description\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   4742\u001b[0m                          \u001b[0mverbose_eval\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmetric_period\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msilent\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mearly_stopping_rounds\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/opt/anaconda3/lib/python3.8/site-packages/catboost/core.py\u001b[0m in \u001b[0;36m_fit\u001b[0;34m(self, X, y, cat_features, text_features, embedding_features, pairs, sample_weight, group_id, group_weight, subgroup_id, pairs_weight, baseline, use_best_model, eval_set, verbose, logging_level, plot, column_description, verbose_eval, metric_period, silent, early_stopping_rounds, save_snapshot, snapshot_file, snapshot_interval, init_model)\u001b[0m\n\u001b[1;32m   1775\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mCatBoostError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"y may be None only when X is an instance of catboost.Pool or string\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1776\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1777\u001b[0;31m         train_params = self._prepare_train_params(\n\u001b[0m\u001b[1;32m   1778\u001b[0m             \u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcat_features\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtext_features\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0membedding_features\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpairs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgroup_id\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1779\u001b[0m             \u001b[0mgroup_weight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msubgroup_id\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpairs_weight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbaseline\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/opt/anaconda3/lib/python3.8/site-packages/catboost/core.py\u001b[0m in \u001b[0;36m_prepare_train_params\u001b[0;34m(self, X, y, cat_features, text_features, embedding_features, pairs, sample_weight, group_id, group_weight, subgroup_id, pairs_weight, baseline, use_best_model, eval_set, verbose, logging_level, plot, column_description, verbose_eval, metric_period, silent, early_stopping_rounds, save_snapshot, snapshot_file, snapshot_interval, init_model)\u001b[0m\n\u001b[1;32m   1665\u001b[0m         \u001b[0membedding_features\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_process_feature_indices\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0membedding_features\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'embedding_features'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1666\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1667\u001b[0;31m         train_pool = _build_train_pool(X, y, cat_features, text_features, embedding_features, pairs,\n\u001b[0m\u001b[1;32m   1668\u001b[0m                                        \u001b[0msample_weight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgroup_id\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgroup_weight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msubgroup_id\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpairs_weight\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1669\u001b[0m                                        baseline, column_description)\n",
      "\u001b[0;32m~/opt/anaconda3/lib/python3.8/site-packages/catboost/core.py\u001b[0m in \u001b[0;36m_build_train_pool\u001b[0;34m(X, y, cat_features, text_features, embedding_features, pairs, sample_weight, group_id, group_weight, subgroup_id, pairs_weight, baseline, column_description)\u001b[0m\n\u001b[1;32m    982\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0my\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    983\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mCatBoostError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"y has not initialized in fit(): X is not catboost.Pool object, y must be not None in fit().\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 984\u001b[0;31m         train_pool = Pool(X, y, cat_features=cat_features, text_features=text_features, embedding_features=embedding_features, pairs=pairs, weight=sample_weight, group_id=group_id,\n\u001b[0m\u001b[1;32m    985\u001b[0m                           group_weight=group_weight, subgroup_id=subgroup_id, pairs_weight=pairs_weight, baseline=baseline)\n\u001b[1;32m    986\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mtrain_pool\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/opt/anaconda3/lib/python3.8/site-packages/catboost/core.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, data, label, cat_features, text_features, embedding_features, column_description, pairs, delimiter, has_header, ignore_csv_quoting, weight, group_id, group_weight, subgroup_id, pairs_weight, baseline, feature_names, thread_count)\u001b[0m\n\u001b[1;32m    453\u001b[0m                     )\n\u001b[1;32m    454\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 455\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_init\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcat_features\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtext_features\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0membedding_features\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpairs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgroup_id\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgroup_weight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msubgroup_id\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpairs_weight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbaseline\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeature_names\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mthread_count\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    456\u001b[0m         \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mPool\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    457\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/opt/anaconda3/lib/python3.8/site-packages/catboost/core.py\u001b[0m in \u001b[0;36m_init\u001b[0;34m(self, data, label, cat_features, text_features, embedding_features, pairs, weight, group_id, group_weight, subgroup_id, pairs_weight, baseline, feature_names, thread_count)\u001b[0m\n\u001b[1;32m    964\u001b[0m             \u001b[0mbaseline\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbaseline\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0msamples_count\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    965\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_check_baseline_shape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbaseline\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msamples_count\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 966\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_init_pool\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcat_features\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtext_features\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0membedding_features\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpairs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgroup_id\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgroup_weight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msubgroup_id\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpairs_weight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbaseline\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeature_names\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mthread_count\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    967\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    968\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m_catboost.pyx\u001b[0m in \u001b[0;36m_catboost._PoolBase._init_pool\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32m_catboost.pyx\u001b[0m in \u001b[0;36m_catboost._PoolBase._init_pool\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32m_catboost.pyx\u001b[0m in \u001b[0;36m_catboost._PoolBase._init_features_order_layout_pool\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32m_catboost.pyx\u001b[0m in \u001b[0;36m_catboost._set_features_order_data_pd_data_frame\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32m_catboost.pyx\u001b[0m in \u001b[0;36m_catboost.create_num_factor_data\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32m_catboost.pyx\u001b[0m in \u001b[0;36m_catboost.get_float_feature\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mCatBoostError\u001b[0m: Bad value for num_feature[non_default_doc_idx=0,feature_idx=2]=\"RL\": Cannot convert 'b'RL'' to float"
     ]
    }
   ],
   "source": [
    "#  Identify columns with categorical features\n",
    "# categorical_features_indices = np.where(X.dtypes != np.float)[\n",
    "#      0\n",
    "# ]  \n",
    "\n",
    "# specify the training parameters\n",
    "model = CatBoostRegressor(iterations=50, loss_function='RMSE')\n",
    "\n",
    "# train the model\n",
    "model.fit(\n",
    "    X_train,\n",
    "    y_train\n",
    "#     cat_features=categorical_features_indices,\n",
    "#    eval_set=(X_validation, y_validation),\n",
    "#    verbose=False,\n",
    "#    plot=True\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model cross-validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Best validation accuracy score: {:.2f}±{:.2f} on step {}'.format(\n",
    "    np.max(cv_data['test-Accuracy-mean']),\n",
    "    cv_data['test-Accuracy-std'][np.argmax(cv_data['test-Accuracy-mean'])],\n",
    "    np.argmax(cv_data['test-Accuracy-mean'])\n",
    "))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Precise validation accuracy score: {}'.format(np.max(cv_data['test-Accuracy-mean'])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv_params = model.get_params()\n",
    "\n",
    "cv_data = cv(\n",
    "    Pool(X, y, cat_features=categorical_features_indices),\n",
    "    cv_params,\n",
    "    plot=True\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# make the prediction using the resulting model\n",
    "y_pred = model.predict(y_validation)\n",
    "print(y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# R^2 score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "HxsRWlURDr6S"
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import r2_score\n",
    "\n",
    "r2_score(y_validation, y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Make competition submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import data\n",
    "data = pd.read_csv(\"hp_test.csv\")\n",
    "# data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Keep only the features as decided through the EDA\n",
    "keep_numerical = [\n",
    "    \"GrLivArea\",\n",
    "    \"GarageArea\",\n",
    "    \"TotalBsmtSF\",\n",
    "    \"1stFlrSF\",\n",
    "    \"TotRmsAbvGrd\",\n",
    "    \"YearBuilt\",\n",
    "    \"YearRemodAdd\",\n",
    "]\n",
    "keep_categorical = [\n",
    "    \"OverallQual\",\n",
    "    \"Neighborhood\",\n",
    "    \"GarageCars\",\n",
    "    \"ExterQual\",\n",
    "    \"BsmtQual\",\n",
    "    \"KitchenQual\",\n",
    "    \"FullBath\",\n",
    "    \"GarageFinish\",\n",
    "    \"FireplaceQu\",\n",
    "    \"Foundation\",\n",
    "    \"GarageType\",\n",
    "]\n",
    "data_subset = data[keep_categorical + keep_numerical]\n",
    "# data_subset.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# qgrid_widget = qgrid.show_grid(pd.DataFrame(data_subset),\n",
    "#                                show_toolbar=True,\n",
    "#                               grid_options={'forceFitColumns': False}  #   Many columns in dataframe become unusable for filter without this\n",
    "#                               )\n",
    "# qgrid_widget"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Deal with missing data\n",
    "data_subset.drop(columns=\"FireplaceQu\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Encode categorical features\n",
    "data_subset_enc = pd.get_dummies(data_subset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# qgrid_widget = qgrid.show_grid(pd.DataFrame(data_subset_enc),\n",
    "#                                show_toolbar=True,\n",
    "#                               grid_options={'forceFitColumns': False}  #   Many columns in dataframe become unusable for filter without this\n",
    "#                               )\n",
    "# qgrid_widget"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_subset_enc.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#  Now prepare for competition submission\n",
    "#####  Confirm why first column is dropped ##########\n",
    "X_competition = data_subset_enc.iloc[:, :].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.shape(X_competition)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Now only have numeric data, so impute missing values\n",
    "\n",
    "from sklearn.impute import SimpleImputer\n",
    "\n",
    "imputer = SimpleImputer(missing_values=np.nan, strategy=\"mean\")\n",
    "imputer.fit(X_competition)\n",
    "X_competition = imputer.transform(X_competition)\n",
    "# np.shape(X_competition)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_competition = sc_y.inverse_transform(\n",
    "    classifier.predict(sc_X.transform(X_competition))\n",
    ")\n",
    "# np.shape(y_competition)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dataset.iloc[0:10, 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Concatenate prediction and passenger ID columns\n",
    "# submission = pd.DataFrame(pd.concat([dataset.iloc[:, 0],  y_competition.iloc[:, :]], axis=1))\n",
    "submission = pd.DataFrame(y_competition)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission.columns = [\"SalePrice\"]\n",
    "IdColumn = pd.read_csv(\"hp_test.csv\")\n",
    "# submission = pd.concat(IdColumn[\"Id\"], submission[\"SalePrice\"])\n",
    "# submission\n",
    "submission = pd.concat([IdColumn[\"Id\"], submission[\"SalePrice\"]], axis=1)\n",
    "# print(submission)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Write to file\n",
    "submission.to_csv(\"hp_catboost.csv\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "EnbCjHgQ8XPn"
   },
   "source": [
    "## Applying k-Fold Cross Validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "yYbfiITD8ZAz"
   },
   "outputs": [],
   "source": [
    "# from sklearn.model_selection import cross_val_score\n",
    "\n",
    "# accuracies = cross_val_score(estimator=classifier, X=X_train, y=y_train, cv=5)\n",
    "# print(\"Accuracy: {:.2f} %\".format(accuracies.mean() * 100))\n",
    "# print(\"Standard Deviation: {:.2f} %\".format(accuracies.std() * 100))"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "catboost.ipynb",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6-final"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}