EVAL_BASIC_TUNED.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import nltk\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import SnowballStemmer, PorterStemmer, WordNetLemmatizer\n",
    "import matplotlib.pyplot as plt\n",
    "from wordcloud import WordCloud\n",
    "from math import log, sqrt\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "from scipy.sparse import csr_matrix\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "\n",
    "from sklearn.metrics import accuracy_score,confusion_matrix,recall_score,precision_score, f1_score\n",
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfTransformer, TfidfVectorizer\n",
    "from sklearn.model_selection import GridSearchCV, KFold\n",
    "\n",
    "from imblearn.over_sampling import SMOTE, ADASYN, RandomOverSampler\n",
    "from imblearn.pipeline import make_pipeline\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "from sklearn.pipeline import Pipeline\n",
    "%matplotlib inline  \n",
    "from imblearn.over_sampling import SMOTE, ADASYN, RandomOverSampler\n",
    "from sklearn.metrics import fbeta_score, make_scorer\n",
    "import xgboost\n",
    "from xgboost import XGBClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data1 = pd.read_csv(\"spam.csv\",encoding='latin-1')\n",
    "\n",
    "\n",
    "#spajanje zadnja cetri stupca kako bi ostavili poruke u potpunosti spojenima\n",
    "data1 = data1.rename(columns={\"Unnamed: 2\":\"two\", \"Unnamed: 3\":\"tri\",\"Unnamed: 4\":\"cetr\"})\n",
    "a =  data1[data1[\"two\"].notnull()][\"v2\"].map(str) + data1[data1[\"two\"].notnull()][\"two\"].map(str)\n",
    "data1.loc[data1[\"two\"].notnull(),\"v2\"] = a\n",
    "a =  data1[data1[\"tri\"].notnull()][\"v2\"].map(str) + data1[data1[\"tri\"].notnull()][\"tri\"].map(str)\n",
    "data1.loc[data1[\"tri\"].notnull(),\"v2\"] = a\n",
    "a =  data1[data1[\"cetr\"].notnull()][\"v2\"].map(str) + data1[data1[\"cetr\"].notnull()][\"cetr\"].map(str)\n",
    "data1.loc[data1[\"cetr\"].notnull(),\"v2\"] = a\n",
    "data1 = data1[[\"v1\",\"v2\"]]\n",
    "#data1.to_csv(\"proba.csv\")\n",
    "\n",
    "\n",
    "\n",
    "data1 = data1.rename(columns={\"v1\":\"category_class\", \"v2\":\"sms\"})\n",
    "codes = {'ham':0, 'spam':1}\n",
    "data1[\"class\"] = data1[\"category_class\"].map(codes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train,test = train_test_split(data1,test_size=0.25,random_state=28)\n",
    "features_train = train[\"sms\"]\n",
    "features_test = test[\"sms\"]\n",
    "labels_train = train[\"class\"]\n",
    "labels_test = test[\"class\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "count_vect = CountVectorizer()\n",
    "count_vect.fit(features_train)\n",
    "count_vectorizer_train = count_vect.transform(features_train)\n",
    "count_vectorizer_test = count_vect.transform(features_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ftwo_scorer = make_scorer(fbeta_score, beta=0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def crosValBestModel(clf, parameters):\n",
    "    grid_search = GridSearchCV(clf, parameters, cv = 10, scoring=ftwo_scorer, n_jobs=2, verbose=10)         \n",
    "    grid_search.fit(count_vectorizer_train, labels_train)\n",
    "    best = grid_search.best_estimator_\n",
    "    preds = best.predict(count_vectorizer_test)\n",
    "   \n",
    "    print(\"Best: %f using %s\" % (grid_search.best_score_, grid_search.best_params_))\n",
    "    means = grid_search.cv_results_['mean_test_score']\n",
    "    stds = grid_search.cv_results_['std_test_score']\n",
    "    params = grid_search.cv_results_['params']\n",
    "    for mean, stdev, param in zip(means, stds, params):\n",
    "        print(\"%f (%f) with: %r\" % (mean, stdev, param))\n",
    "    \n",
    "    print(\"Accuracy :\",accuracy_score(labels_test,preds))\n",
    "    print(\"Recall :\",recall_score(labels_test,preds))\n",
    "    print(\"Presicion :\",precision_score(labels_test,preds))\n",
    "    print(\"F0.5 score :\",fbeta_score(labels_test,preds, beta=0.5))\n",
    "    #print(\"F1 score :\",f1_score(labels_test,preds))\n",
    "    print(\"CONFUSION MATRIX\",confusion_matrix(labels_test,preds))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "============================================================\n",
      "___________ MNB______________________\n",
      "Accuracy : 0.992103374013\n",
      "Recall : 0.967213114754\n",
      "Presicion : 0.972527472527\n",
      "F0.5 score : 0.971459934138\n",
      "CONFUSION MATRIX [[1205    5]\n",
      " [   6  177]]\n",
      "============================================================\n",
      "============================================================\n",
      "___________ Logistic______________________\n",
      "Accuracy : 0.985642498205\n",
      "Recall : 0.907103825137\n",
      "Presicion : 0.98224852071\n",
      "F0.5 score : 0.966239813737\n",
      "CONFUSION MATRIX [[1207    3]\n",
      " [  17  166]]\n",
      "============================================================\n",
      "============================================================\n",
      "___________ SVC______________________\n",
      "Accuracy : 0.984924623116\n",
      "Recall : 0.912568306011\n",
      "Presicion : 0.970930232558\n",
      "F0.5 score : 0.958668197474\n",
      "CONFUSION MATRIX [[1205    5]\n",
      " [  16  167]]\n",
      "============================================================\n",
      "============================================================\n",
      "___________ KNN______________________\n",
      "Accuracy : 0.916726489591\n",
      "Recall : 0.366120218579\n",
      "Presicion : 1.0\n",
      "F0.5 score : 0.742793791574\n",
      "CONFUSION MATRIX [[1210    0]\n",
      " [ 116   67]]\n",
      "============================================================\n",
      "============================================================\n",
      "___________ RandomForrest______________________\n",
      "Accuracy : 0.969849246231\n",
      "Recall : 0.770491803279\n",
      "Presicion : 1.0\n",
      "F0.5 score : 0.943775100402\n",
      "CONFUSION MATRIX [[1210    0]\n",
      " [  42  141]]\n",
      "============================================================\n"
     ]
    }
   ],
   "source": [
    "clfs = [('MNB',MultinomialNB()), \n",
    "        ('Logistic',LogisticRegression()), \n",
    "        ('SVC',SVC(kernel='linear')), \n",
    "        ('KNN',KNeighborsClassifier()), \n",
    "        ('RandomForrest',RandomForestClassifier(random_state=7))]\n",
    "for name, clf in clfs:\n",
    "    clf.fit(count_vectorizer_train, labels_train)\n",
    "    preds = clf.predict(count_vectorizer_test)\n",
    "    \n",
    "    print('============================================================')\n",
    "    print('___________ '+name+'______________________')\n",
    "    print(\"Accuracy :\",accuracy_score(labels_test,preds))\n",
    "    print(\"Recall :\",recall_score(labels_test,preds))\n",
    "    print(\"Presicion :\",precision_score(labels_test,preds))\n",
    "    print(\"F0.5 score :\",fbeta_score(labels_test,preds, beta=0.5))\n",
    "    print(\"CONFUSION MATRIX\",confusion_matrix(labels_test,preds))\n",
    "    print('============================================================')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 10 folds for each of 10 candidates, totalling 100 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=2)]: Done   1 tasks      | elapsed:    1.2s\n",
      "[Parallel(n_jobs=2)]: Done   4 tasks      | elapsed:    2.2s\n",
      "[Parallel(n_jobs=2)]: Done   9 tasks      | elapsed:    2.3s\n",
      "[Parallel(n_jobs=2)]: Done  14 tasks      | elapsed:    2.4s\n",
      "[Parallel(n_jobs=2)]: Batch computation too fast (0.1795s.) Setting batch_size=2.\n",
      "[Parallel(n_jobs=2)]: Done  22 tasks      | elapsed:    2.6s\n",
      "[Parallel(n_jobs=2)]: Batch computation too fast (0.1257s.) Setting batch_size=6.\n",
      "[Parallel(n_jobs=2)]: Done  52 tasks      | elapsed:    3.2s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: 0.964588 using {'C': 8.9000000000000004}\n",
      "0.943039 (0.019904) with: {'C': 0.10000000000000001}\n",
      "0.957030 (0.013104) with: {'C': 1.2000000000000002}\n",
      "0.960315 (0.011266) with: {'C': 2.3000000000000003}\n",
      "0.962275 (0.011110) with: {'C': 3.4000000000000004}\n",
      "0.962275 (0.011110) with: {'C': 4.5}\n",
      "0.962275 (0.011110) with: {'C': 5.5999999999999996}\n",
      "0.962726 (0.010557) with: {'C': 6.7000000000000002}\n",
      "0.964165 (0.009889) with: {'C': 7.8000000000000007}\n",
      "0.964588 (0.010044) with: {'C': 8.9000000000000004}\n",
      "0.964588 (0.010044) with: {'C': 10.0}\n",
      "Accuracy : 0.987796123475\n",
      "Recall : 0.918032786885\n",
      "Presicion : 0.988235294118\n",
      "F0.5 score : 0.973348783314\n",
      "CONFUSION MATRIX [[1208    2]\n",
      " [  15  168]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=2)]: Done 100 out of 100 | elapsed:    4.1s finished\n"
     ]
    }
   ],
   "source": [
    "parameters = {\n",
    "    'C': np.linspace(0.1, 10, num=10)\n",
    "}\n",
    "crosValBestModel(LogisticRegression(), parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 10 folds for each of 10 candidates, totalling 100 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=2)]: Done   1 tasks      | elapsed:    2.1s\n",
      "[Parallel(n_jobs=2)]: Done   4 tasks      | elapsed:    3.9s\n",
      "[Parallel(n_jobs=2)]: Done   9 tasks      | elapsed:    5.6s\n",
      "[Parallel(n_jobs=2)]: Done  14 tasks      | elapsed:    8.1s\n",
      "[Parallel(n_jobs=2)]: Done  21 tasks      | elapsed:   10.7s\n",
      "[Parallel(n_jobs=2)]: Done  28 tasks      | elapsed:   14.1s\n",
      "[Parallel(n_jobs=2)]: Done  37 tasks      | elapsed:   17.5s\n",
      "[Parallel(n_jobs=2)]: Done  46 tasks      | elapsed:   21.2s\n",
      "[Parallel(n_jobs=2)]: Done  57 tasks      | elapsed:   25.6s\n",
      "[Parallel(n_jobs=2)]: Done  68 tasks      | elapsed:   30.0s\n",
      "[Parallel(n_jobs=2)]: Done  81 tasks      | elapsed:   35.2s\n",
      "[Parallel(n_jobs=2)]: Done  94 tasks      | elapsed:   40.3s\n",
      "[Parallel(n_jobs=2)]: Done 100 out of 100 | elapsed:   42.7s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: 0.968620 using {'C': 2.3000000000000003}\n",
      "0.967356 (0.008416) with: {'C': 0.10000000000000001}\n",
      "0.967196 (0.010602) with: {'C': 1.2000000000000002}\n",
      "0.968620 (0.009923) with: {'C': 2.3000000000000003}\n",
      "0.968620 (0.009923) with: {'C': 3.4000000000000004}\n",
      "0.968620 (0.009923) with: {'C': 4.5}\n",
      "0.968620 (0.009923) with: {'C': 5.5999999999999996}\n",
      "0.968620 (0.009923) with: {'C': 6.7000000000000002}\n",
      "0.968620 (0.009923) with: {'C': 7.8000000000000007}\n",
      "0.968620 (0.009923) with: {'C': 8.9000000000000004}\n",
      "0.968620 (0.009923) with: {'C': 10.0}\n",
      "Accuracy : 0.984924623116\n",
      "Recall : 0.912568306011\n",
      "Presicion : 0.970930232558\n",
      "F0.5 score : 0.958668197474\n",
      "CONFUSION MATRIX [[1205    5]\n",
      " [  16  167]]\n"
     ]
    }
   ],
   "source": [
    "parameters = {\n",
    "    'C': np.linspace(0.1, 10, num=10)\n",
    "}\n",
    "crosValBestModel(SVC(kernel='linear'), parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "parameters = {\n",
    "    'learning_rate': [0.01,0.01,0.1,1,10],\n",
    "    'max_depth' : [1,3,5,7],\n",
    "    'n_estimators' : [100,300,500,700]\n",
    "    'reg_lambda' : [0.01,0.01,0.1,1,10]\n",
    "}\n",
    "\n",
    "crosValBestModel(XGBClassifier(), parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 10 folds for each of 4 candidates, totalling 40 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=2)]: Done   1 tasks      | elapsed:    1.6s\n",
      "[Parallel(n_jobs=2)]: Done   4 tasks      | elapsed:    3.1s\n",
      "[Parallel(n_jobs=2)]: Done   9 tasks      | elapsed:    4.0s\n",
      "[Parallel(n_jobs=2)]: Done  14 tasks      | elapsed:    5.4s\n",
      "[Parallel(n_jobs=2)]: Done  21 tasks      | elapsed:    6.9s\n",
      "[Parallel(n_jobs=2)]: Done  28 tasks      | elapsed:    8.9s\n",
      "[Parallel(n_jobs=2)]: Done  37 tasks      | elapsed:   11.3s\n",
      "[Parallel(n_jobs=2)]: Done  40 out of  40 | elapsed:   12.3s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: 0.793919 using {'n_neighbors': 3}\n",
      "0.788413 (0.035278) with: {'n_neighbors': 2}\n",
      "0.793919 (0.037728) with: {'n_neighbors': 3}\n",
      "0.720075 (0.053815) with: {'n_neighbors': 4}\n",
      "0.727183 (0.048220) with: {'n_neighbors': 5}\n",
      "Accuracy : 0.928930366116\n",
      "Recall : 0.459016393443\n",
      "Presicion : 1.0\n",
      "F0.5 score : 0.809248554913\n",
      "CONFUSION MATRIX [[1210    0]\n",
      " [  99   84]]\n"
     ]
    }
   ],
   "source": [
    "parameters = {\n",
    "    'n_neighbors': [2,3,4,5]\n",
    "}\n",
    "\n",
    "crosValBestModel(KNeighborsClassifier(), parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 10 folds for each of 18 candidates, totalling 180 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=2)]: Done   1 tasks      | elapsed:    8.5s\n",
      "[Parallel(n_jobs=2)]: Done   4 tasks      | elapsed:   16.6s\n",
      "[Parallel(n_jobs=2)]: Done   9 tasks      | elapsed:   36.9s\n",
      "[Parallel(n_jobs=2)]: Done  14 tasks      | elapsed:   57.5s\n",
      "[Parallel(n_jobs=2)]: Done  21 tasks      | elapsed:  1.6min\n",
      "[Parallel(n_jobs=2)]: Done  28 tasks      | elapsed:  2.1min\n",
      "[Parallel(n_jobs=2)]: Done  37 tasks      | elapsed:  2.8min\n",
      "[Parallel(n_jobs=2)]: Done  46 tasks      | elapsed:  3.4min\n",
      "[Parallel(n_jobs=2)]: Done  57 tasks      | elapsed:  4.4min\n",
      "[Parallel(n_jobs=2)]: Done  68 tasks      | elapsed:  5.0min\n",
      "[Parallel(n_jobs=2)]: Done  81 tasks      | elapsed:  6.1min\n",
      "[Parallel(n_jobs=2)]: Done  94 tasks      | elapsed:  7.1min\n",
      "[Parallel(n_jobs=2)]: Done 109 tasks      | elapsed:  8.2min\n",
      "[Parallel(n_jobs=2)]: Done 124 tasks      | elapsed:  9.3min\n",
      "[Parallel(n_jobs=2)]: Done 141 tasks      | elapsed: 10.6min\n",
      "[Parallel(n_jobs=2)]: Done 158 tasks      | elapsed: 11.8min\n",
      "[Parallel(n_jobs=2)]: Done 177 tasks      | elapsed: 13.4min\n",
      "[Parallel(n_jobs=2)]: Done 180 out of 180 | elapsed: 13.6min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: 0.955723 using {'max_depth': 200, 'max_features': 'sqrt', 'n_estimators': 300}\n",
      "0.950871 (0.016830) with: {'max_depth': 200, 'max_features': 'auto', 'n_estimators': 200}\n",
      "0.954113 (0.014126) with: {'max_depth': 200, 'max_features': 'auto', 'n_estimators': 250}\n",
      "0.953608 (0.014635) with: {'max_depth': 200, 'max_features': 'auto', 'n_estimators': 300}\n",
      "0.952980 (0.015716) with: {'max_depth': 200, 'max_features': 'sqrt', 'n_estimators': 200}\n",
      "0.953073 (0.014395) with: {'max_depth': 200, 'max_features': 'sqrt', 'n_estimators': 250}\n",
      "0.955723 (0.012392) with: {'max_depth': 200, 'max_features': 'sqrt', 'n_estimators': 300}\n",
      "0.955035 (0.014728) with: {'max_depth': 250, 'max_features': 'auto', 'n_estimators': 200}\n",
      "0.952010 (0.015651) with: {'max_depth': 250, 'max_features': 'auto', 'n_estimators': 250}\n",
      "0.954200 (0.012966) with: {'max_depth': 250, 'max_features': 'auto', 'n_estimators': 300}\n",
      "0.953304 (0.017687) with: {'max_depth': 250, 'max_features': 'sqrt', 'n_estimators': 200}\n",
      "0.951586 (0.014806) with: {'max_depth': 250, 'max_features': 'sqrt', 'n_estimators': 250}\n",
      "0.955152 (0.013412) with: {'max_depth': 250, 'max_features': 'sqrt', 'n_estimators': 300}\n",
      "0.952833 (0.017733) with: {'max_depth': 300, 'max_features': 'auto', 'n_estimators': 200}\n",
      "0.953685 (0.013333) with: {'max_depth': 300, 'max_features': 'auto', 'n_estimators': 250}\n",
      "0.953995 (0.015820) with: {'max_depth': 300, 'max_features': 'auto', 'n_estimators': 300}\n",
      "0.954013 (0.013656) with: {'max_depth': 300, 'max_features': 'sqrt', 'n_estimators': 200}\n",
      "0.955336 (0.011570) with: {'max_depth': 300, 'max_features': 'sqrt', 'n_estimators': 250}\n",
      "0.952730 (0.013383) with: {'max_depth': 300, 'max_features': 'sqrt', 'n_estimators': 300}\n",
      "Accuracy : 0.977745872218\n",
      "Recall : 0.830601092896\n",
      "Presicion : 1.0\n",
      "F0.5 score : 0.960809102402\n",
      "CONFUSION MATRIX [[1210    0]\n",
      " [  31  152]]\n"
     ]
    }
   ],
   "source": [
    "n_estimators = [200, 250, 300]\n",
    "max_depth =  [200, 250, 300]\n",
    "max_features = ['auto', 'sqrt']\n",
    "parameters = dict(n_estimators=n_estimators,max_depth=max_depth, max_features = max_features )\n",
    "\n",
    "crosValBestModel(RandomForestClassifier(), parameters)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}