gbm_usage_example.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/r_merkulov/anaconda/lib/python3.6/site-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import lightgbm as lgb\n",
    "import category_encoders as ce\n",
    "import functools\n",
    "from sklearn.cross_validation import KFold\n",
    "import xgboost as xgb\n",
    "import catboost\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df = pd.read_csv(\n",
    "    filepath_or_buffer='/home/data/avito-demand-prediction/train.csv', \n",
    "    usecols=['item_id', 'user_id', 'region', 'city', 'parent_category_name', 'category_name', 'param_1',\n",
    "            'param_2', 'param_3', 'price', 'item_seq_number', 'user_type', 'image_top_1', 'deal_probability'],\n",
    "    nrows=10000,\n",
    "    index_col='item_id'\n",
    ")\n",
    "train_df.fillna({\n",
    "        'param_1': '<UNKNOWN>',\n",
    "        'param_2': '<UNKNOWN>',\n",
    "        'param_3': '<UNKNOWN>'\n",
    "    }, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 13)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>region</th>\n",
       "      <th>city</th>\n",
       "      <th>parent_category_name</th>\n",
       "      <th>category_name</th>\n",
       "      <th>param_1</th>\n",
       "      <th>param_2</th>\n",
       "      <th>param_3</th>\n",
       "      <th>price</th>\n",
       "      <th>item_seq_number</th>\n",
       "      <th>user_type</th>\n",
       "      <th>image_top_1</th>\n",
       "      <th>deal_probability</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>item_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>b912c3c6a6ad</th>\n",
       "      <td>e00f8ff2eaf9</td>\n",
       "      <td>Свердловская область</td>\n",
       "      <td>Екатеринбург</td>\n",
       "      <td>Личные вещи</td>\n",
       "      <td>Товары для детей и игрушки</td>\n",
       "      <td>Постельные принадлежности</td>\n",
       "      <td>&lt;UNKNOWN&gt;</td>\n",
       "      <td>&lt;UNKNOWN&gt;</td>\n",
       "      <td>400.0</td>\n",
       "      <td>2</td>\n",
       "      <td>Private</td>\n",
       "      <td>1008.0</td>\n",
       "      <td>0.12789</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2dac0150717d</th>\n",
       "      <td>39aeb48f0017</td>\n",
       "      <td>Самарская область</td>\n",
       "      <td>Самара</td>\n",
       "      <td>Для дома и дачи</td>\n",
       "      <td>Мебель и интерьер</td>\n",
       "      <td>Другое</td>\n",
       "      <td>&lt;UNKNOWN&gt;</td>\n",
       "      <td>&lt;UNKNOWN&gt;</td>\n",
       "      <td>3000.0</td>\n",
       "      <td>19</td>\n",
       "      <td>Private</td>\n",
       "      <td>692.0</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ba83aefab5dc</th>\n",
       "      <td>91e2f88dd6e3</td>\n",
       "      <td>Ростовская область</td>\n",
       "      <td>Ростов-на-Дону</td>\n",
       "      <td>Бытовая электроника</td>\n",
       "      <td>Аудио и видео</td>\n",
       "      <td>Видео, DVD и Blu-ray плееры</td>\n",
       "      <td>&lt;UNKNOWN&gt;</td>\n",
       "      <td>&lt;UNKNOWN&gt;</td>\n",
       "      <td>4000.0</td>\n",
       "      <td>9</td>\n",
       "      <td>Private</td>\n",
       "      <td>3032.0</td>\n",
       "      <td>0.43177</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>02996f1dd2ea</th>\n",
       "      <td>bf5cccea572d</td>\n",
       "      <td>Татарстан</td>\n",
       "      <td>Набережные Челны</td>\n",
       "      <td>Личные вещи</td>\n",
       "      <td>Товары для детей и игрушки</td>\n",
       "      <td>Автомобильные кресла</td>\n",
       "      <td>&lt;UNKNOWN&gt;</td>\n",
       "      <td>&lt;UNKNOWN&gt;</td>\n",
       "      <td>2200.0</td>\n",
       "      <td>286</td>\n",
       "      <td>Company</td>\n",
       "      <td>796.0</td>\n",
       "      <td>0.80323</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7c90be56d2ab</th>\n",
       "      <td>ef50846afc0b</td>\n",
       "      <td>Волгоградская область</td>\n",
       "      <td>Волгоград</td>\n",
       "      <td>Транспорт</td>\n",
       "      <td>Автомобили</td>\n",
       "      <td>С пробегом</td>\n",
       "      <td>ВАЗ (LADA)</td>\n",
       "      <td>2110</td>\n",
       "      <td>40000.0</td>\n",
       "      <td>3</td>\n",
       "      <td>Private</td>\n",
       "      <td>2264.0</td>\n",
       "      <td>0.20797</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   user_id                 region              city  \\\n",
       "item_id                                                               \n",
       "b912c3c6a6ad  e00f8ff2eaf9   Свердловская область      Екатеринбург   \n",
       "2dac0150717d  39aeb48f0017      Самарская область            Самара   \n",
       "ba83aefab5dc  91e2f88dd6e3     Ростовская область    Ростов-на-Дону   \n",
       "02996f1dd2ea  bf5cccea572d              Татарстан  Набережные Челны   \n",
       "7c90be56d2ab  ef50846afc0b  Волгоградская область         Волгоград   \n",
       "\n",
       "             parent_category_name               category_name  \\\n",
       "item_id                                                         \n",
       "b912c3c6a6ad          Личные вещи  Товары для детей и игрушки   \n",
       "2dac0150717d      Для дома и дачи           Мебель и интерьер   \n",
       "ba83aefab5dc  Бытовая электроника               Аудио и видео   \n",
       "02996f1dd2ea          Личные вещи  Товары для детей и игрушки   \n",
       "7c90be56d2ab            Транспорт                  Автомобили   \n",
       "\n",
       "                                  param_1     param_2    param_3    price  \\\n",
       "item_id                                                                     \n",
       "b912c3c6a6ad    Постельные принадлежности   <UNKNOWN>  <UNKNOWN>    400.0   \n",
       "2dac0150717d                       Другое   <UNKNOWN>  <UNKNOWN>   3000.0   \n",
       "ba83aefab5dc  Видео, DVD и Blu-ray плееры   <UNKNOWN>  <UNKNOWN>   4000.0   \n",
       "02996f1dd2ea         Автомобильные кресла   <UNKNOWN>  <UNKNOWN>   2200.0   \n",
       "7c90be56d2ab                   С пробегом  ВАЗ (LADA)       2110  40000.0   \n",
       "\n",
       "              item_seq_number user_type  image_top_1  deal_probability  \n",
       "item_id                                                                 \n",
       "b912c3c6a6ad                2   Private       1008.0           0.12789  \n",
       "2dac0150717d               19   Private        692.0           0.00000  \n",
       "ba83aefab5dc                9   Private       3032.0           0.43177  \n",
       "02996f1dd2ea              286   Company        796.0           0.80323  \n",
       "7c90be56d2ab                3   Private       2264.0           0.20797  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def encode_categorical(df, columns_to_encode):\n",
    "    \n",
    "    encoders = {}\n",
    "    \n",
    "    for c in columns_to_encode:\n",
    "        le = LabelEncoder()\n",
    "        df[c] = le.fit_transform(df[c])\n",
    "        encoders[c] = le\n",
    "        \n",
    "    return df, encoders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_ids_by_folds():\n",
    "    folds = [\n",
    "        (list(train_df.iloc[train].index), list(train_df.iloc[valid].index)) \n",
    "        for (train, valid) in KFold(train_df.shape[0], n_folds=5, shuffle=True)\n",
    "    ]\n",
    "    \n",
    "    return folds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "FOLDS = get_ids_by_folds()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "CATEGORICAL_FEATURES = ['user_id', 'region', 'city', 'parent_category_name', 'category_name', 'param_1', \n",
    "                        'param_2', 'param_3', 'user_type', 'image_top_1']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "df, LABEL_ENCODERS = encode_categorical(train_df, columns_to_encode=CATEGORICAL_FEATURES)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>region</th>\n",
       "      <th>city</th>\n",
       "      <th>parent_category_name</th>\n",
       "      <th>category_name</th>\n",
       "      <th>param_1</th>\n",
       "      <th>param_2</th>\n",
       "      <th>param_3</th>\n",
       "      <th>price</th>\n",
       "      <th>item_seq_number</th>\n",
       "      <th>user_type</th>\n",
       "      <th>image_top_1</th>\n",
       "      <th>deal_probability</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>item_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>b912c3c6a6ad</th>\n",
       "      <td>8471</td>\n",
       "      <td>19</td>\n",
       "      <td>147</td>\n",
       "      <td>4</td>\n",
       "      <td>41</td>\n",
       "      <td>167</td>\n",
       "      <td>4</td>\n",
       "      <td>87</td>\n",
       "      <td>400.0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>654</td>\n",
       "      <td>0.12789</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2dac0150717d</th>\n",
       "      <td>2185</td>\n",
       "      <td>17</td>\n",
       "      <td>429</td>\n",
       "      <td>2</td>\n",
       "      <td>22</td>\n",
       "      <td>81</td>\n",
       "      <td>4</td>\n",
       "      <td>87</td>\n",
       "      <td>3000.0</td>\n",
       "      <td>19</td>\n",
       "      <td>1</td>\n",
       "      <td>454</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ba83aefab5dc</th>\n",
       "      <td>5440</td>\n",
       "      <td>16</td>\n",
       "      <td>420</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>49</td>\n",
       "      <td>4</td>\n",
       "      <td>87</td>\n",
       "      <td>4000.0</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>1867</td>\n",
       "      <td>0.43177</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>02996f1dd2ea</th>\n",
       "      <td>7178</td>\n",
       "      <td>21</td>\n",
       "      <td>314</td>\n",
       "      <td>4</td>\n",
       "      <td>41</td>\n",
       "      <td>31</td>\n",
       "      <td>4</td>\n",
       "      <td>87</td>\n",
       "      <td>2200.0</td>\n",
       "      <td>286</td>\n",
       "      <td>0</td>\n",
       "      <td>527</td>\n",
       "      <td>0.80323</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7c90be56d2ab</th>\n",
       "      <td>9068</td>\n",
       "      <td>4</td>\n",
       "      <td>97</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>189</td>\n",
       "      <td>48</td>\n",
       "      <td>17</td>\n",
       "      <td>40000.0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1423</td>\n",
       "      <td>0.20797</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              user_id  region  city  parent_category_name  category_name  \\\n",
       "item_id                                                                    \n",
       "b912c3c6a6ad     8471      19   147                     4             41   \n",
       "2dac0150717d     2185      17   429                     2             22   \n",
       "ba83aefab5dc     5440      16   420                     0              2   \n",
       "02996f1dd2ea     7178      21   314                     4             41   \n",
       "7c90be56d2ab     9068       4    97                     6              0   \n",
       "\n",
       "              param_1  param_2  param_3    price  item_seq_number  user_type  \\\n",
       "item_id                                                                        \n",
       "b912c3c6a6ad      167        4       87    400.0                2          1   \n",
       "2dac0150717d       81        4       87   3000.0               19          1   \n",
       "ba83aefab5dc       49        4       87   4000.0                9          1   \n",
       "02996f1dd2ea       31        4       87   2200.0              286          0   \n",
       "7c90be56d2ab      189       48       17  40000.0                3          1   \n",
       "\n",
       "              image_top_1  deal_probability  \n",
       "item_id                                      \n",
       "b912c3c6a6ad          654           0.12789  \n",
       "2dac0150717d          454           0.00000  \n",
       "ba83aefab5dc         1867           0.43177  \n",
       "02996f1dd2ea          527           0.80323  \n",
       "7c90be56d2ab         1423           0.20797  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "TARGET_ENCODING_GROUPPING = [\n",
    "    [\"region\"],\n",
    "    [\"region\", \"user_type\"],\n",
    "    [\"image_top_1\"],\n",
    "    [\"parent_category_name\"],\n",
    "]\n",
    "\n",
    "COLS_TO_USE_FOR_TE = list(\n",
    "    functools.reduce(\n",
    "        lambda x1, x2: set(x1) | set(x2), \n",
    "        TARGET_ENCODING_GROUPPING\n",
    "    )\n",
    ")\n",
    "\n",
    "TRAIN_FEATURES = [\n",
    "    'user_id', 'region', 'city', 'parent_category_name', 'category_name', 'param_1',\n",
    "    'param_2', 'param_3', 'price', 'item_seq_number', 'user_type', 'image_top_1'\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def internal_validation_step(df, y, train_indices, n_fold=3):\n",
    "    \n",
    "    internal_folds = KFold(len(train_indices), n_folds=n_fold, random_state=42)\n",
    "    \n",
    "    target_encoded = []\n",
    "    \n",
    "    for internal_fold_train, internal_fold_valid in internal_folds:\n",
    "        \n",
    "        df_tr = df[COLS_TO_USE_FOR_TE]\\\n",
    "            .loc[[train_indices[i] for i in internal_fold_train]]\\\n",
    "            .copy(deep=True)\n",
    "        df_te = df[COLS_TO_USE_FOR_TE]\\\n",
    "            .loc[[train_indices[i] for i in internal_fold_valid]]\\\n",
    "            .copy(deep=True)\n",
    "        \n",
    "        for t in TARGET_ENCODING_GROUPPING:\n",
    "            \n",
    "            te = ce.TargetEncoder(cols=t, smoothing=100)\\\n",
    "                .fit(df_tr, y[[train_indices[i] for i in internal_fold_train]])\n",
    "            df_te = te.transform(df_te)\n",
    "\n",
    "        target_encoded.append(df_te)\n",
    "        \n",
    "    target_features = pd.concat(target_encoded)\n",
    "        \n",
    "    return target_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lightgbm_dataset_getter(X, y):\n",
    "    return lgb.Dataset(X, y, free_raw_data=False)\n",
    "\n",
    "def xgboost_dataset_getter(X, y):\n",
    "    return X, y\n",
    "\n",
    "def catboost_dataset_getter(X, y, categorical_features):\n",
    "    return catboost.Pool(X, y, cat_features=categorical_features)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fit_boosting(\n",
    "    df, y, train_indices, val_indices, fold_number, boosting_mode, boosting_params=None, \n",
    "    use_target_encoded_features=True\n",
    "):\n",
    "    train_dataset = df[TRAIN_FEATURES].loc[train_indices]\n",
    "    valid_dataset = df[TRAIN_FEATURES].loc[val_indices]\n",
    "    \n",
    "    if use_target_encoded_features:\n",
    "\n",
    "        target_encoded_features = internal_validation_step(df, y, train_indices)\n",
    "        train_dataset = train_dataset.join(target_encoded_features.loc[train_indices], rsuffix='_te')\n",
    "        \n",
    "        cols_to_use_for_target_encoding = list(\n",
    "            functools.reduce(\n",
    "                lambda x1, x2: set(x1) | set(x2), \n",
    "                TARGET_ENCODING_GROUPPING\n",
    "            )\n",
    "        )\n",
    "\n",
    "        df_te = df[COLS_TO_USE_FOR_TE]\\\n",
    "            .loc[val_indices]\\\n",
    "            .copy(deep=True)\n",
    "\n",
    "        for t in TARGET_ENCODING_GROUPPING:\n",
    "            te = ce.TargetEncoder(cols=t, smoothing=100).\\\n",
    "                fit(\n",
    "                    df[COLS_TO_USE_FOR_TE].loc[train_indices], \n",
    "                    y[train_indices]\n",
    "                )\n",
    "            df_te = te.transform(df_te)\n",
    "\n",
    "        valid_dataset = valid_dataset.join(df_te, rsuffix='_te')\n",
    "\n",
    "    cols = sorted(list(train_dataset.columns))\n",
    "    \n",
    "    assert sorted(list(train_dataset.columns)) == sorted(list(valid_dataset.columns))\n",
    "    \n",
    "    if boosting_mode == 'lightgbm':\n",
    "        \n",
    "        train_dataset = lightgbm_dataset_getter(train_dataset[cols], y.loc[train_indices])\n",
    "        valid_dataset = lightgbm_dataset_getter(valid_dataset[cols], y.loc[val_indices])\n",
    "        \n",
    "        gbm = lgb.train(\n",
    "            boosting_params,\n",
    "            train_dataset,\n",
    "            num_boost_round=10000,\n",
    "            early_stopping_rounds=200,\n",
    "            valid_sets=(train_dataset, valid_dataset),\n",
    "            valid_names=('train', 'valid'),\n",
    "            feature_name=cols,\n",
    "            categorical_feature=CATEGORICAL_FEATURES,\n",
    "            verbose_eval=100\n",
    "        )\n",
    "        \n",
    "    elif boosting_mode == 'catboost':\n",
    "        \n",
    "        cat_features = [cols.index(i) for i in CATEGORICAL_FEATURES]\n",
    "        \n",
    "        train_dataset = catboost_dataset_getter(train_dataset[cols], y.loc[train_indices], cat_features)\n",
    "        valid_dataset = catboost_dataset_getter(valid_dataset[cols], y.loc[val_indices], cat_features)\n",
    "        \n",
    "        gbm = catboost.CatBoostClassifier(**boosting_params)\n",
    "        \n",
    "        gbm.fit(train_dataset, eval_set=valid_dataset, plot=True)\n",
    "        \n",
    "    elif boosting_mode == 'xgboost':\n",
    "        \n",
    "        train_dataset = xgboost_dataset_getter(train_dataset[cols], y.loc[train_indices])\n",
    "        valid_dataset = xgboost_dataset_getter(valid_dataset[cols], y.loc[val_indices])\n",
    "        \n",
    "        gbm = xgb.XGBRegressor(**boosting_params)\n",
    "    \n",
    "        gbm.fit(\n",
    "            train_dataset[0], train_dataset[1], early_stopping_rounds=100, eval_metric='rmse',\n",
    "            eval_set=[valid_dataset]\n",
    "        )\n",
    "    else:\n",
    "        raise ValueError('Not supported {} boosting implementation!'.format(boosting_mode)) \n",
    "    \n",
    "    return gbm, cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {\n",
    "    'boosting_type': 'gbdt',\n",
    "    'objective': 'xentropy',\n",
    "    'metric': ['rmse', 'xentropy'],\n",
    "    'num_leaves': 31,\n",
    "    'learning_rate': 0.05,\n",
    "    'feature_fraction': 0.6,\n",
    "    'bagging_fraction': 0.6,\n",
    "    'bagging_freq': 5,\n",
    "    'cat_smooth': 150,\n",
    "    'num_threads': 8\n",
    "}\n",
    "    \n",
    "catboost_params = {\n",
    "    'iterations': 10000,\n",
    "    'learning_rate': 0.05,\n",
    "    'loss_function': 'CrossEntropy',\n",
    "    'custom_metric': ['RMSE'],\n",
    "    'eval_metric': 'RMSE',\n",
    "    'subsample': 0.6,\n",
    "    'logging_level': 'Verbose',\n",
    "    'metric_period': 100,\n",
    "    'depth': 5,\n",
    "    'rsm': 0.6,\n",
    "    'thread_count': 8,\n",
    "    'od_type': 'Iter',\n",
    "    'od_wait': 200,\n",
    "    'use_best_model': True,\n",
    "    'bootstrap_type': 'Bernoulli'\n",
    "}\n",
    "    \n",
    "xgb_params = {\n",
    "    'learning_rate': 0.2,\n",
    "    'max_depth': 5,\n",
    "    'subsample': 0.6,\n",
    "    'colsample_bytree': 0.6,\n",
    "    'objective': 'reg:logistic',\n",
    "    'n_estimators': 10000,\n",
    "    'nthread': 8,\n",
    "    'silent': False\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 0 started at 2018-07-13 23:41:02.861412\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/r_merkulov/anaconda/lib/python3.6/site-packages/lightgbm/basic.py:1038: UserWarning: categorical_feature in Dataset is overrided. New categorical_feature is ['category_name', 'city', 'image_top_1', 'param_1', 'param_2', 'param_3', 'parent_category_name', 'region', 'user_id', 'user_type']\n",
      "  warnings.warn('categorical_feature in Dataset is overrided. New categorical_feature is {}'.format(sorted(list(categorical_feature))))\n",
      "/home/r_merkulov/anaconda/lib/python3.6/site-packages/lightgbm/basic.py:681: UserWarning: categorical_feature in param dict is overrided.\n",
      "  warnings.warn('categorical_feature in param dict is overrided.')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training until validation scores don't improve for 200 rounds.\n",
      "[100]\ttrain's xentropy: 0.332427\ttrain's rmse: 0.221003\tvalid's xentropy: 0.363779\tvalid's rmse: 0.234333\n",
      "[200]\ttrain's xentropy: 0.308982\ttrain's rmse: 0.204225\tvalid's xentropy: 0.367326\tvalid's rmse: 0.23739\n",
      "Early stopping, best iteration is:\n",
      "[44]\ttrain's xentropy: 0.350217\ttrain's rmse: 0.231359\tvalid's xentropy: 0.36309\tvalid's rmse: 0.23316\n",
      "Fold 0 finished at 2018-07-13 23:41:33.257564\n",
      "\n",
      "\n",
      "\n",
      "Fold 1 started at 2018-07-13 23:41:33.257712\n",
      "Training until validation scores don't improve for 200 rounds.\n",
      "[100]\ttrain's xentropy: 0.330938\ttrain's rmse: 0.218465\tvalid's xentropy: 0.380755\tvalid's rmse: 0.250356\n",
      "[200]\ttrain's xentropy: 0.30841\ttrain's rmse: 0.2026\tvalid's xentropy: 0.38402\tvalid's rmse: 0.251425\n",
      "Early stopping, best iteration is:\n",
      "[84]\ttrain's xentropy: 0.335158\ttrain's rmse: 0.221091\tvalid's xentropy: 0.379822\tvalid's rmse: 0.250203\n",
      "Fold 1 finished at 2018-07-13 23:42:03.482667\n",
      "\n",
      "\n",
      "\n",
      "Fold 2 started at 2018-07-13 23:42:03.482813\n",
      "Training until validation scores don't improve for 200 rounds.\n",
      "[100]\ttrain's xentropy: 0.334025\ttrain's rmse: 0.220097\tvalid's xentropy: 0.36279\tvalid's rmse: 0.239676\n",
      "[200]\ttrain's xentropy: 0.311378\ttrain's rmse: 0.204434\tvalid's xentropy: 0.364795\tvalid's rmse: 0.240593\n",
      "Early stopping, best iteration is:\n",
      "[69]\ttrain's xentropy: 0.342315\ttrain's rmse: 0.224994\tvalid's xentropy: 0.361335\tvalid's rmse: 0.238568\n",
      "Fold 2 finished at 2018-07-13 23:42:33.896028\n",
      "\n",
      "\n",
      "\n",
      "Fold 3 started at 2018-07-13 23:42:33.896175\n",
      "Training until validation scores don't improve for 200 rounds.\n",
      "[100]\ttrain's xentropy: 0.331258\ttrain's rmse: 0.218549\tvalid's xentropy: 0.374319\tvalid's rmse: 0.245585\n",
      "[200]\ttrain's xentropy: 0.30902\ttrain's rmse: 0.203079\tvalid's xentropy: 0.376906\tvalid's rmse: 0.247149\n",
      "Early stopping, best iteration is:\n",
      "[63]\ttrain's xentropy: 0.341614\ttrain's rmse: 0.224804\tvalid's xentropy: 0.373388\tvalid's rmse: 0.244952\n",
      "Fold 3 finished at 2018-07-13 23:43:03.917193\n",
      "\n",
      "\n",
      "\n",
      "Fold 4 started at 2018-07-13 23:43:03.917333\n",
      "Training until validation scores don't improve for 200 rounds.\n",
      "[100]\ttrain's xentropy: 0.337008\ttrain's rmse: 0.222174\tvalid's xentropy: 0.355884\tvalid's rmse: 0.235769\n",
      "[200]\ttrain's xentropy: 0.3141\ttrain's rmse: 0.206164\tvalid's xentropy: 0.357154\tvalid's rmse: 0.236226\n",
      "Early stopping, best iteration is:\n",
      "[65]\ttrain's xentropy: 0.346288\ttrain's rmse: 0.227653\tvalid's xentropy: 0.355188\tvalid's rmse: 0.235199\n",
      "Fold 4 finished at 2018-07-13 23:43:34.106560\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "clfs = []\n",
    "for ind, (train_inds, val_inds) in enumerate(FOLDS):\n",
    "    print('Fold {} started at {}'.format(ind, datetime.datetime.now()))\n",
    "    clf, _ = fit_boosting(\n",
    "        df=df, y=df.deal_probability, train_indices=train_inds, val_indices=val_inds, fold_number=ind, \n",
    "        boosting_mode='lightgbm', boosting_params=params, use_target_encoded_features=True\n",
    "    )\n",
    "    clfs.append(clf)\n",
    "    print('Fold {} finished at {}\\n\\n\\n'.format(ind, datetime.datetime.now()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}