2.1 - Optuna LGBM tuning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "5c20a5b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.metrics import log_loss\n",
    "import lightgbm as lgbm\n",
    "# import optuna.integration.lightgbm as lgb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "b2d8ca71",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((432587, 40), (144196, 40), (432587,), (144196,))"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# reading data\n",
    "X_train = pd.read_feather('X_train.f').set_index('sku')\n",
    "X_val = pd.read_feather('X_val.f').set_index('sku')\n",
    "y_train = pd.read_feather('y_train.f').set_index('sku').iloc[:, 0]\n",
    "y_val = pd.read_feather('y_val.f').set_index('sku').iloc[:, 0]\n",
    "\n",
    "X_train.shape, X_val.shape, y_train.shape, y_val.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "2fd46204",
   "metadata": {},
   "outputs": [],
   "source": [
    "# tuned the model first using Optuna and extracted the best params\n",
    "\n",
    "# dtrain = lgb.Dataset(\n",
    "#     data=X_train,\n",
    "#     label=y_train\n",
    "# )\n",
    "# dval = lgb.Dataset(\n",
    "#     data=X_val,\n",
    "#     label=y_val\n",
    "# )\n",
    "\n",
    "# params = {\n",
    "#     'objective': 'multiclass',\n",
    "#     'num_classes': 30,\n",
    "#     'metric': 'multi_logloss',\n",
    "#     'boosting': 'gbdt',\n",
    "#     'random_seed': 0,\n",
    "#     'deterministic': True\n",
    "# }  \n",
    "\n",
    "# # using Optuna LightGBM Tuner\n",
    "# model = lgb.train(\n",
    "#     params,\n",
    "#     dtrain,\n",
    "#     valid_sets=[dtrain, dval],\n",
    "#     early_stopping_rounds=20\n",
    "# )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "e454d03f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] Auto-choosing row-wise multi-threading, the overhead of testing was 0.068409 seconds.\n",
      "You can set `force_row_wise=true` to remove the overhead.\n",
      "And if memory is not enough, you can set `force_col_wise=true`.\n",
      "[LightGBM] [Info] Total Bins 6640\n",
      "[LightGBM] [Info] Number of data points in the train set: 432587, number of used features: 40\n",
      "[LightGBM] [Info] Start training from score -3.336781\n",
      "[LightGBM] [Info] Start training from score -3.308508\n",
      "[LightGBM] [Info] Start training from score -3.334118\n",
      "[LightGBM] [Info] Start training from score -3.400128\n",
      "[LightGBM] [Info] Start training from score -3.461701\n",
      "[LightGBM] [Info] Start training from score -3.520495\n",
      "[LightGBM] [Info] Start training from score -3.374346\n",
      "[LightGBM] [Info] Start training from score -3.283230\n",
      "[LightGBM] [Info] Start training from score -3.260140\n",
      "[LightGBM] [Info] Start training from score -3.278189\n",
      "[LightGBM] [Info] Start training from score -3.360135\n",
      "[LightGBM] [Info] Start training from score -3.478267\n",
      "[LightGBM] [Info] Start training from score -3.519792\n",
      "[LightGBM] [Info] Start training from score -3.317780\n",
      "[LightGBM] [Info] Start training from score -3.230939\n",
      "[LightGBM] [Info] Start training from score -3.257614\n",
      "[LightGBM] [Info] Start training from score -3.262009\n",
      "[LightGBM] [Info] Start training from score -3.361800\n",
      "[LightGBM] [Info] Start training from score -3.477369\n",
      "[LightGBM] [Info] Start training from score -3.528260\n",
      "[LightGBM] [Info] Start training from score -3.353765\n",
      "[LightGBM] [Info] Start training from score -3.314468\n",
      "[LightGBM] [Info] Start training from score -3.324373\n",
      "[LightGBM] [Info] Start training from score -3.329782\n",
      "[LightGBM] [Info] Start training from score -3.422971\n",
      "[LightGBM] [Info] Start training from score -3.582297\n",
      "[LightGBM] [Info] Start training from score -3.656462\n",
      "[LightGBM] [Info] Start training from score -3.553944\n",
      "[LightGBM] [Info] Start training from score -3.595517\n",
      "[LightGBM] [Info] Start training from score -3.816494\n",
      "[1]\ttraining's multi_logloss: 3.34949\tvalid_1's multi_logloss: 3.35096\n",
      "Training until validation scores don't improve for 20 rounds\n",
      "[2]\ttraining's multi_logloss: 3.32102\tvalid_1's multi_logloss: 3.32325\n",
      "[3]\ttraining's multi_logloss: 3.29586\tvalid_1's multi_logloss: 3.29885\n",
      "[4]\ttraining's multi_logloss: 3.27408\tvalid_1's multi_logloss: 3.27797\n",
      "[5]\ttraining's multi_logloss: 3.25728\tvalid_1's multi_logloss: 3.26205\n",
      "[6]\ttraining's multi_logloss: 3.24153\tvalid_1's multi_logloss: 3.24709\n",
      "[7]\ttraining's multi_logloss: 3.22924\tvalid_1's multi_logloss: 3.23547\n",
      "[8]\ttraining's multi_logloss: 3.21809\tvalid_1's multi_logloss: 3.22512\n",
      "[9]\ttraining's multi_logloss: 3.20939\tvalid_1's multi_logloss: 3.21717\n",
      "[10]\ttraining's multi_logloss: 3.20026\tvalid_1's multi_logloss: 3.20875\n",
      "[11]\ttraining's multi_logloss: 3.19189\tvalid_1's multi_logloss: 3.20116\n",
      "[12]\ttraining's multi_logloss: 3.18404\tvalid_1's multi_logloss: 3.19392\n",
      "[13]\ttraining's multi_logloss: 3.17733\tvalid_1's multi_logloss: 3.18792\n",
      "[14]\ttraining's multi_logloss: 3.17127\tvalid_1's multi_logloss: 3.18256\n",
      "[15]\ttraining's multi_logloss: 3.16604\tvalid_1's multi_logloss: 3.178\n",
      "[16]\ttraining's multi_logloss: 3.16127\tvalid_1's multi_logloss: 3.17387\n",
      "[17]\ttraining's multi_logloss: 3.15698\tvalid_1's multi_logloss: 3.17028\n",
      "[18]\ttraining's multi_logloss: 3.15307\tvalid_1's multi_logloss: 3.16706\n",
      "[19]\ttraining's multi_logloss: 3.14918\tvalid_1's multi_logloss: 3.16383\n",
      "[20]\ttraining's multi_logloss: 3.14547\tvalid_1's multi_logloss: 3.16078\n",
      "[21]\ttraining's multi_logloss: 3.14241\tvalid_1's multi_logloss: 3.15843\n",
      "[22]\ttraining's multi_logloss: 3.13913\tvalid_1's multi_logloss: 3.15591\n",
      "[23]\ttraining's multi_logloss: 3.13619\tvalid_1's multi_logloss: 3.15363\n",
      "[24]\ttraining's multi_logloss: 3.1336\tvalid_1's multi_logloss: 3.15177\n",
      "[25]\ttraining's multi_logloss: 3.13113\tvalid_1's multi_logloss: 3.15\n",
      "[26]\ttraining's multi_logloss: 3.12875\tvalid_1's multi_logloss: 3.14826\n",
      "[27]\ttraining's multi_logloss: 3.12646\tvalid_1's multi_logloss: 3.14665\n",
      "[28]\ttraining's multi_logloss: 3.12433\tvalid_1's multi_logloss: 3.1452\n",
      "[29]\ttraining's multi_logloss: 3.12224\tvalid_1's multi_logloss: 3.14378\n",
      "[30]\ttraining's multi_logloss: 3.12022\tvalid_1's multi_logloss: 3.14243\n",
      "[31]\ttraining's multi_logloss: 3.11826\tvalid_1's multi_logloss: 3.14111\n",
      "[32]\ttraining's multi_logloss: 3.11646\tvalid_1's multi_logloss: 3.13998\n",
      "[33]\ttraining's multi_logloss: 3.11463\tvalid_1's multi_logloss: 3.13877\n",
      "[34]\ttraining's multi_logloss: 3.11294\tvalid_1's multi_logloss: 3.13775\n",
      "[35]\ttraining's multi_logloss: 3.11124\tvalid_1's multi_logloss: 3.13668\n",
      "[36]\ttraining's multi_logloss: 3.10968\tvalid_1's multi_logloss: 3.13578\n",
      "[37]\ttraining's multi_logloss: 3.10814\tvalid_1's multi_logloss: 3.13491\n",
      "[38]\ttraining's multi_logloss: 3.10665\tvalid_1's multi_logloss: 3.13413\n",
      "[39]\ttraining's multi_logloss: 3.10515\tvalid_1's multi_logloss: 3.13332\n",
      "[40]\ttraining's multi_logloss: 3.10379\tvalid_1's multi_logloss: 3.13265\n",
      "[41]\ttraining's multi_logloss: 3.10251\tvalid_1's multi_logloss: 3.13206\n",
      "[42]\ttraining's multi_logloss: 3.10121\tvalid_1's multi_logloss: 3.13143\n",
      "[43]\ttraining's multi_logloss: 3.09998\tvalid_1's multi_logloss: 3.13078\n",
      "[44]\ttraining's multi_logloss: 3.09877\tvalid_1's multi_logloss: 3.13026\n",
      "[45]\ttraining's multi_logloss: 3.09759\tvalid_1's multi_logloss: 3.12977\n",
      "[46]\ttraining's multi_logloss: 3.09639\tvalid_1's multi_logloss: 3.12923\n",
      "[47]\ttraining's multi_logloss: 3.09524\tvalid_1's multi_logloss: 3.12872\n",
      "[48]\ttraining's multi_logloss: 3.09411\tvalid_1's multi_logloss: 3.12826\n",
      "[49]\ttraining's multi_logloss: 3.09306\tvalid_1's multi_logloss: 3.12789\n",
      "[50]\ttraining's multi_logloss: 3.09198\tvalid_1's multi_logloss: 3.12748\n",
      "[51]\ttraining's multi_logloss: 3.09093\tvalid_1's multi_logloss: 3.12711\n",
      "[52]\ttraining's multi_logloss: 3.08992\tvalid_1's multi_logloss: 3.12675\n",
      "[53]\ttraining's multi_logloss: 3.08892\tvalid_1's multi_logloss: 3.12643\n",
      "[54]\ttraining's multi_logloss: 3.08799\tvalid_1's multi_logloss: 3.12609\n",
      "[55]\ttraining's multi_logloss: 3.08708\tvalid_1's multi_logloss: 3.12587\n",
      "[56]\ttraining's multi_logloss: 3.0861\tvalid_1's multi_logloss: 3.12554\n",
      "[57]\ttraining's multi_logloss: 3.08526\tvalid_1's multi_logloss: 3.12527\n",
      "[58]\ttraining's multi_logloss: 3.08432\tvalid_1's multi_logloss: 3.12498\n",
      "[59]\ttraining's multi_logloss: 3.08338\tvalid_1's multi_logloss: 3.12477\n",
      "[60]\ttraining's multi_logloss: 3.08252\tvalid_1's multi_logloss: 3.12451\n",
      "[61]\ttraining's multi_logloss: 3.08166\tvalid_1's multi_logloss: 3.1243\n",
      "[62]\ttraining's multi_logloss: 3.08075\tvalid_1's multi_logloss: 3.12405\n",
      "[63]\ttraining's multi_logloss: 3.0799\tvalid_1's multi_logloss: 3.12385\n",
      "[64]\ttraining's multi_logloss: 3.07911\tvalid_1's multi_logloss: 3.12368\n",
      "[65]\ttraining's multi_logloss: 3.07828\tvalid_1's multi_logloss: 3.12348\n",
      "[66]\ttraining's multi_logloss: 3.07749\tvalid_1's multi_logloss: 3.12328\n",
      "[67]\ttraining's multi_logloss: 3.07667\tvalid_1's multi_logloss: 3.12308\n",
      "[68]\ttraining's multi_logloss: 3.07589\tvalid_1's multi_logloss: 3.12293\n",
      "[69]\ttraining's multi_logloss: 3.07509\tvalid_1's multi_logloss: 3.1228\n",
      "[70]\ttraining's multi_logloss: 3.07428\tvalid_1's multi_logloss: 3.12265\n",
      "[71]\ttraining's multi_logloss: 3.07354\tvalid_1's multi_logloss: 3.12251\n",
      "[72]\ttraining's multi_logloss: 3.07279\tvalid_1's multi_logloss: 3.12235\n",
      "[73]\ttraining's multi_logloss: 3.07202\tvalid_1's multi_logloss: 3.12217\n",
      "[74]\ttraining's multi_logloss: 3.07127\tvalid_1's multi_logloss: 3.12203\n",
      "[75]\ttraining's multi_logloss: 3.07056\tvalid_1's multi_logloss: 3.12193\n",
      "[76]\ttraining's multi_logloss: 3.06981\tvalid_1's multi_logloss: 3.12178\n",
      "[77]\ttraining's multi_logloss: 3.06906\tvalid_1's multi_logloss: 3.12165\n",
      "[78]\ttraining's multi_logloss: 3.06835\tvalid_1's multi_logloss: 3.12157\n",
      "[79]\ttraining's multi_logloss: 3.06765\tvalid_1's multi_logloss: 3.12149\n",
      "[80]\ttraining's multi_logloss: 3.06696\tvalid_1's multi_logloss: 3.12139\n",
      "[81]\ttraining's multi_logloss: 3.06625\tvalid_1's multi_logloss: 3.12134\n",
      "[82]\ttraining's multi_logloss: 3.06555\tvalid_1's multi_logloss: 3.12122\n",
      "[83]\ttraining's multi_logloss: 3.06487\tvalid_1's multi_logloss: 3.12109\n",
      "[84]\ttraining's multi_logloss: 3.06419\tvalid_1's multi_logloss: 3.12105\n",
      "[85]\ttraining's multi_logloss: 3.06355\tvalid_1's multi_logloss: 3.12095\n",
      "[86]\ttraining's multi_logloss: 3.06284\tvalid_1's multi_logloss: 3.1209\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[87]\ttraining's multi_logloss: 3.06218\tvalid_1's multi_logloss: 3.12085\n",
      "[88]\ttraining's multi_logloss: 3.06149\tvalid_1's multi_logloss: 3.12076\n",
      "[89]\ttraining's multi_logloss: 3.06081\tvalid_1's multi_logloss: 3.12069\n",
      "[90]\ttraining's multi_logloss: 3.06018\tvalid_1's multi_logloss: 3.12061\n",
      "[91]\ttraining's multi_logloss: 3.05954\tvalid_1's multi_logloss: 3.12058\n",
      "[92]\ttraining's multi_logloss: 3.05887\tvalid_1's multi_logloss: 3.12053\n",
      "[93]\ttraining's multi_logloss: 3.05823\tvalid_1's multi_logloss: 3.12048\n",
      "[94]\ttraining's multi_logloss: 3.05757\tvalid_1's multi_logloss: 3.12043\n",
      "[95]\ttraining's multi_logloss: 3.05695\tvalid_1's multi_logloss: 3.12038\n",
      "[96]\ttraining's multi_logloss: 3.05632\tvalid_1's multi_logloss: 3.12034\n",
      "[97]\ttraining's multi_logloss: 3.05569\tvalid_1's multi_logloss: 3.12025\n",
      "[98]\ttraining's multi_logloss: 3.05507\tvalid_1's multi_logloss: 3.12023\n",
      "[99]\ttraining's multi_logloss: 3.05449\tvalid_1's multi_logloss: 3.12017\n",
      "[100]\ttraining's multi_logloss: 3.05387\tvalid_1's multi_logloss: 3.12012\n",
      "[101]\ttraining's multi_logloss: 3.05322\tvalid_1's multi_logloss: 3.12008\n",
      "[102]\ttraining's multi_logloss: 3.05262\tvalid_1's multi_logloss: 3.12003\n",
      "[103]\ttraining's multi_logloss: 3.05203\tvalid_1's multi_logloss: 3.12001\n",
      "[104]\ttraining's multi_logloss: 3.05142\tvalid_1's multi_logloss: 3.11998\n",
      "[105]\ttraining's multi_logloss: 3.05083\tvalid_1's multi_logloss: 3.11997\n",
      "[106]\ttraining's multi_logloss: 3.05026\tvalid_1's multi_logloss: 3.11995\n",
      "[107]\ttraining's multi_logloss: 3.04964\tvalid_1's multi_logloss: 3.11994\n",
      "[108]\ttraining's multi_logloss: 3.04907\tvalid_1's multi_logloss: 3.11993\n",
      "[109]\ttraining's multi_logloss: 3.04846\tvalid_1's multi_logloss: 3.11992\n",
      "[110]\ttraining's multi_logloss: 3.04787\tvalid_1's multi_logloss: 3.11987\n",
      "[111]\ttraining's multi_logloss: 3.04729\tvalid_1's multi_logloss: 3.11984\n",
      "[112]\ttraining's multi_logloss: 3.0467\tvalid_1's multi_logloss: 3.11984\n",
      "[113]\ttraining's multi_logloss: 3.04613\tvalid_1's multi_logloss: 3.11983\n",
      "[114]\ttraining's multi_logloss: 3.04556\tvalid_1's multi_logloss: 3.11978\n",
      "[115]\ttraining's multi_logloss: 3.04499\tvalid_1's multi_logloss: 3.11976\n",
      "[116]\ttraining's multi_logloss: 3.04445\tvalid_1's multi_logloss: 3.11975\n",
      "[117]\ttraining's multi_logloss: 3.04387\tvalid_1's multi_logloss: 3.11976\n",
      "[118]\ttraining's multi_logloss: 3.04328\tvalid_1's multi_logloss: 3.11977\n",
      "[119]\ttraining's multi_logloss: 3.04268\tvalid_1's multi_logloss: 3.11974\n",
      "[120]\ttraining's multi_logloss: 3.04208\tvalid_1's multi_logloss: 3.11976\n",
      "[121]\ttraining's multi_logloss: 3.04151\tvalid_1's multi_logloss: 3.11973\n",
      "[122]\ttraining's multi_logloss: 3.04091\tvalid_1's multi_logloss: 3.11971\n",
      "[123]\ttraining's multi_logloss: 3.04035\tvalid_1's multi_logloss: 3.11971\n",
      "[124]\ttraining's multi_logloss: 3.03976\tvalid_1's multi_logloss: 3.11968\n",
      "[125]\ttraining's multi_logloss: 3.03918\tvalid_1's multi_logloss: 3.11962\n",
      "[126]\ttraining's multi_logloss: 3.03855\tvalid_1's multi_logloss: 3.11959\n",
      "[127]\ttraining's multi_logloss: 3.03798\tvalid_1's multi_logloss: 3.11959\n",
      "[128]\ttraining's multi_logloss: 3.03745\tvalid_1's multi_logloss: 3.11958\n",
      "[129]\ttraining's multi_logloss: 3.03688\tvalid_1's multi_logloss: 3.11958\n",
      "[130]\ttraining's multi_logloss: 3.03634\tvalid_1's multi_logloss: 3.11959\n",
      "[131]\ttraining's multi_logloss: 3.03576\tvalid_1's multi_logloss: 3.11961\n",
      "[132]\ttraining's multi_logloss: 3.03521\tvalid_1's multi_logloss: 3.11957\n",
      "[133]\ttraining's multi_logloss: 3.03467\tvalid_1's multi_logloss: 3.11956\n",
      "[134]\ttraining's multi_logloss: 3.03407\tvalid_1's multi_logloss: 3.11948\n",
      "[135]\ttraining's multi_logloss: 3.03357\tvalid_1's multi_logloss: 3.11948\n",
      "[136]\ttraining's multi_logloss: 3.033\tvalid_1's multi_logloss: 3.11946\n",
      "[137]\ttraining's multi_logloss: 3.03245\tvalid_1's multi_logloss: 3.11944\n",
      "[138]\ttraining's multi_logloss: 3.03191\tvalid_1's multi_logloss: 3.11946\n",
      "[139]\ttraining's multi_logloss: 3.03136\tvalid_1's multi_logloss: 3.11946\n",
      "[140]\ttraining's multi_logloss: 3.03085\tvalid_1's multi_logloss: 3.11943\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[140]\ttraining's multi_logloss: 3.03085\tvalid_1's multi_logloss: 3.11943\n"
     ]
    }
   ],
   "source": [
    "dtrain = lgbm.Dataset(\n",
    "    data=X_train,\n",
    "    label=y_train\n",
    ")\n",
    "dval = lgbm.Dataset(\n",
    "    data=X_val,\n",
    "    label=y_val\n",
    ")\n",
    "\n",
    "best_params = {\n",
    "    'objective': 'multiclass',\n",
    "    'num_classes': 30,\n",
    "    'metric': 'multi_logloss',\n",
    "    'boosting': 'gbdt',\n",
    "    'random_seed': 0,\n",
    "    'deterministic': True,\n",
    "    'feature_pre_filter': False,\n",
    "    'lambda_l1': 0.00037397899681396743,\n",
    "    'lambda_l2': 8.75904356834923,\n",
    "    'num_leaves': 19,\n",
    "    'feature_fraction': 0.8,\n",
    "    'bagging_fraction': 1.0,\n",
    "    'bagging_freq': 0,\n",
    "    'min_child_samples': 25,\n",
    "    'num_iterations': 140,\n",
    "    'early_stopping_round': 20\n",
    "}  \n",
    "\n",
    "model = lgbm.train(\n",
    "    best_params,\n",
    "    dtrain,\n",
    "    valid_sets=[dtrain, dval],\n",
    "    early_stopping_rounds=20\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "94ce492b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log loss (train): 3.030849956990119\n",
      "Log loss (val): 3.1194339837202962\n",
      "RPS (train): 3.626246687440311\n",
      "RPS (val): 3.6679233366046873\n",
      "CPU times: user 6min 9s, sys: 2.22 s, total: 6min 11s\n",
      "Wall time: 1min 51s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "# predictions on best model\n",
    "probs_train = model.predict(X_train)\n",
    "probs_val = model.predict(X_val)\n",
    "\n",
    "log_train = log_loss(y_train, probs_train)\n",
    "log_val = log_loss(y_val, probs_val)\n",
    "print('Log loss (train):', log_train)\n",
    "print('Log loss (val):', log_val)\n",
    "\n",
    "# scoring_function expects target in [1, 30] not in [0, 29]\n",
    "rps_train = scoring_function(y_train + 1, probs_train)\n",
    "rps_val = scoring_function(y_val + 1, probs_val)\n",
    "print('RPS (train):', rps_train)\n",
    "print('RPS (val):', rps_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "3200a417",
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.DataFrame(probs_val).to_csv('2.1-probs-tuned-lgbm.csv')"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6"
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}