EVAL_PIPELINE_TOK2_TUNED.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import nltk\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import SnowballStemmer, PorterStemmer, WordNetLemmatizer\n",
    "import matplotlib.pyplot as plt\n",
    "from math import log, sqrt\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "from scipy.sparse import csr_matrix\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "\n",
    "from sklearn.metrics import accuracy_score,confusion_matrix,recall_score,precision_score, f1_score\n",
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfTransformer, TfidfVectorizer\n",
    "from sklearn.model_selection import GridSearchCV, KFold\n",
    "\n",
    "from imblearn.over_sampling import SMOTE, ADASYN, RandomOverSampler\n",
    "from imblearn.pipeline import make_pipeline\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "from sklearn.pipeline import Pipeline\n",
    "%matplotlib inline  \n",
    "from imblearn.over_sampling import SMOTE, ADASYN, RandomOverSampler\n",
    "from sklearn.metrics import fbeta_score, make_scorer\n",
    "import xgboost\n",
    "from xgboost import XGBClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data1 = pd.read_csv(\"spam.csv\",encoding='latin-1')\n",
    "\n",
    "\n",
    "#spajanje zadnja cetri stupca kako bi ostavili poruke u potpunosti spojenima\n",
    "data1 = data1.rename(columns={\"Unnamed: 2\":\"two\", \"Unnamed: 3\":\"tri\",\"Unnamed: 4\":\"cetr\"})\n",
    "a =  data1[data1[\"two\"].notnull()][\"v2\"].map(str) + data1[data1[\"two\"].notnull()][\"two\"].map(str)\n",
    "data1.loc[data1[\"two\"].notnull(),\"v2\"] = a\n",
    "a =  data1[data1[\"tri\"].notnull()][\"v2\"].map(str) + data1[data1[\"tri\"].notnull()][\"tri\"].map(str)\n",
    "data1.loc[data1[\"tri\"].notnull(),\"v2\"] = a\n",
    "a =  data1[data1[\"cetr\"].notnull()][\"v2\"].map(str) + data1[data1[\"cetr\"].notnull()][\"cetr\"].map(str)\n",
    "data1.loc[data1[\"cetr\"].notnull(),\"v2\"] = a\n",
    "data1 = data1[[\"v1\",\"v2\"]]\n",
    "#data1.to_csv(\"proba.csv\")\n",
    "\n",
    "\n",
    "\n",
    "data1 = data1.rename(columns={\"v1\":\"category_class\", \"v2\":\"sms\"})\n",
    "codes = {'ham':0, 'spam':1}\n",
    "data1[\"class\"] = data1[\"category_class\"].map(codes)\n",
    "maxLen = max(data1['sms'].apply(len).tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       upper_letters      length  numeric_chars  non_alpha_num\n",
      "class                                                         \n",
      "0           4.173472   71.627979       0.305285      17.500518\n",
      "1          15.483266  139.148594      15.812584      29.132530\n"
     ]
    }
   ],
   "source": [
    "# **NOVO - dodani featuri u dataframe i imas statistike prosjecne za svaki\n",
    "from nltk.corpus import words\n",
    "swords = set(words.words())\n",
    "def countUpperLetters(message):\n",
    "    return sum(1 for c in message if c.isupper())\n",
    "def countDigits(message):\n",
    "    return sum(c.isdigit() for c in message)\n",
    "def countNonAlphaNumerical(message):\n",
    "    return sum(not c.isalnum() for c in message)\n",
    "\n",
    "data1[\"upper_letters\"] = data1[\"sms\"].apply(countUpperLetters)\n",
    "data1['length'] = data1['sms'].apply(len)\n",
    "data1[\"numeric_chars\"] = data1[\"sms\"].apply(countDigits)\n",
    "data1[\"non_alpha_num\"] = data1[\"sms\"].apply(countNonAlphaNumerical)\n",
    "print(data1.groupby(\"class\").mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train,test = train_test_split(data1,test_size=0.25,random_state=28)\n",
    "features_train = train[\"sms\"]\n",
    "features_test = test[\"sms\"]\n",
    "labels_train = train[\"class\"]\n",
    "labels_test = test[\"class\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def dodajFeatureVratiArray(current_array,pandas_column):\n",
    "    return np.hstack((current_array,pandas_column.values.reshape(-1,1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def outSystem(count_vect = CountVectorizer(analyzer = 'char', ngram_range=(2,5)) ):\n",
    "    count_vect.fit(train['sms'])\n",
    "    count_vectorizer_train = count_vect.transform(train['sms']).todense()\n",
    "    count_vectorizer_test = count_vect.transform(test['sms']).todense()\n",
    "        \n",
    "    train_X = dodajFeatureVratiArray(count_vectorizer_train,train[\"numeric_chars\"])\n",
    "    train_X = dodajFeatureVratiArray(train_X,train[\"length\"])\n",
    "    train_X = dodajFeatureVratiArray(train_X,train[\"non_alpha_num\"])\n",
    "    train_X = dodajFeatureVratiArray(train_X,train[\"upper_letters\"])\n",
    "    \n",
    "    test_X = dodajFeatureVratiArray(count_vectorizer_test,test[\"numeric_chars\"])\n",
    "    test_X = dodajFeatureVratiArray(test_X,test[\"length\"])\n",
    "    test_X = dodajFeatureVratiArray(test_X,test[\"non_alpha_num\"])\n",
    "    test_X = dodajFeatureVratiArray(test_X,test[\"upper_letters\"])\n",
    "    \n",
    "    return(csr_matrix(train_X), csr_matrix(test_X))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data, test_data = outSystem()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ftwo_scorer = make_scorer(fbeta_score, beta=0.5)\n",
    "\n",
    "def crosValBestModel(clf, d_train, d_test, parameters):\n",
    "    grid_search = GridSearchCV(clf, parameters, cv = 10, scoring=ftwo_scorer, n_jobs=2, verbose=10)\n",
    "    grid_search.fit(d_train, labels_train)                \n",
    "    best = grid_search.best_estimator_\n",
    "    preds = best.predict(d_test)\n",
    "   \n",
    "    print(\"Best: %f using %s\" % (grid_search.best_score_, grid_search.best_params_))\n",
    "    means = grid_search.cv_results_['mean_test_score']\n",
    "    stds = grid_search.cv_results_['std_test_score']\n",
    "    params = grid_search.cv_results_['params']\n",
    "    for mean, stdev, param in zip(means, stds, params):\n",
    "        print(\"%f (%f) with: %r\" % (mean, stdev, param))\n",
    "    \n",
    "    print(\"Accuracy :\",accuracy_score(labels_test,preds))\n",
    "    print(\"Recall :\",recall_score(labels_test,preds))\n",
    "    print(\"Presicion :\",precision_score(labels_test,preds))\n",
    "    print(\"F0.5 score :\",fbeta_score(labels_test,preds, beta=0.5))\n",
    "    print(\"CONFUSION MATRIX\",confusion_matrix(labels_test,preds))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "============================================================\n",
      "___________ MNB______________________\n",
      "Accuracy : 0.991385498923\n",
      "Recall : 0.956284153005\n",
      "Presicion : 0.977653631285\n",
      "F0.5 score : 0.973303670745\n",
      "CONFUSION MATRIX [[1206    4]\n",
      " [   8  175]]\n",
      "============================================================\n",
      "============================================================\n",
      "___________ Logistic______________________\n",
      "Accuracy : 0.992103374013\n",
      "Recall : 0.950819672131\n",
      "Presicion : 0.988636363636\n",
      "F0.5 score : 0.98083427283\n",
      "CONFUSION MATRIX [[1208    2]\n",
      " [   9  174]]\n",
      "============================================================\n",
      "============================================================\n",
      "___________ SVC______________________\n",
      "Accuracy : 0.991385498923\n",
      "Recall : 0.96174863388\n",
      "Presicion : 0.972375690608\n",
      "F0.5 score : 0.970231532525\n",
      "CONFUSION MATRIX [[1205    5]\n",
      " [   7  176]]\n",
      "============================================================\n",
      "============================================================\n",
      "___________ KNN______________________\n",
      "Accuracy : 0.987078248385\n",
      "Recall : 0.934426229508\n",
      "Presicion : 0.966101694915\n",
      "F0.5 score : 0.959595959596\n",
      "CONFUSION MATRIX [[1204    6]\n",
      " [  12  171]]\n",
      "============================================================\n",
      "============================================================\n",
      "___________ RandomForrest______________________\n",
      "Accuracy : 0.990667623833\n",
      "Recall : 0.928961748634\n",
      "Presicion : 1.0\n",
      "F0.5 score : 0.98493626883\n",
      "CONFUSION MATRIX [[1210    0]\n",
      " [  13  170]]\n",
      "============================================================\n"
     ]
    }
   ],
   "source": [
    "clfs = [('MNB',MultinomialNB()), \n",
    "        ('Logistic',LogisticRegression()),\n",
    "        ('SVC',SVC(kernel='linear')), \n",
    "        ('KNN',KNeighborsClassifier()),\n",
    "        ('RandomForrest',RandomForestClassifier(n_estimators=300, max_depth=300,random_state=7))]\n",
    "        #('RandomForrest',RandomForestClassifier(random_state=7))]\n",
    "for name, clf in clfs:\n",
    "    clf.fit(train_data, labels_train)\n",
    "    preds = clf.predict(test_data)\n",
    "    \n",
    "    print('============================================================')\n",
    "    print('___________ '+name+'______________________')\n",
    "    print(\"Accuracy :\",accuracy_score(labels_test,preds))\n",
    "    print(\"Recall :\",recall_score(labels_test,preds))\n",
    "    print(\"Presicion :\",precision_score(labels_test,preds))\n",
    "    print(\"F0.5 score :\",fbeta_score(labels_test,preds, beta=0.5))\n",
    "    print(\"CONFUSION MATRIX\",confusion_matrix(labels_test,preds))\n",
    "    print('============================================================')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 10 folds for each of 10 candidates, totalling 100 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=2)]: Done   1 tasks      | elapsed:    2.2s\n",
      "[Parallel(n_jobs=2)]: Done   4 tasks      | elapsed:    4.3s\n",
      "[Parallel(n_jobs=2)]: Done   9 tasks      | elapsed:    6.1s\n",
      "[Parallel(n_jobs=2)]: Done  14 tasks      | elapsed:    8.5s\n",
      "[Parallel(n_jobs=2)]: Done  21 tasks      | elapsed:   11.3s\n",
      "[Parallel(n_jobs=2)]: Done  28 tasks      | elapsed:   15.3s\n",
      "[Parallel(n_jobs=2)]: Done  37 tasks      | elapsed:   18.9s\n",
      "[Parallel(n_jobs=2)]: Done  46 tasks      | elapsed:   23.2s\n",
      "[Parallel(n_jobs=2)]: Done  57 tasks      | elapsed:   27.6s\n",
      "[Parallel(n_jobs=2)]: Done  68 tasks      | elapsed:   33.8s\n",
      "[Parallel(n_jobs=2)]: Done  81 tasks      | elapsed:   40.0s\n",
      "[Parallel(n_jobs=2)]: Done  94 tasks      | elapsed:   48.1s\n",
      "[Parallel(n_jobs=2)]: Done 100 out of 100 | elapsed:   51.2s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: 0.971218 using {'C': 3.4000000000000004}\n",
      "0.969520 (0.008374) with: {'C': 0.10000000000000001}\n",
      "0.969943 (0.009136) with: {'C': 1.2000000000000002}\n",
      "0.970792 (0.009238) with: {'C': 2.3000000000000003}\n",
      "0.971218 (0.009415) with: {'C': 3.4000000000000004}\n",
      "0.971218 (0.009415) with: {'C': 4.5}\n",
      "0.971218 (0.009415) with: {'C': 5.5999999999999996}\n",
      "0.971218 (0.009415) with: {'C': 6.7000000000000002}\n",
      "0.971218 (0.009415) with: {'C': 7.8000000000000007}\n",
      "0.971218 (0.009415) with: {'C': 8.9000000000000004}\n",
      "0.970795 (0.009548) with: {'C': 10.0}\n",
      "Accuracy : 0.992821249103\n",
      "Recall : 0.956284153005\n",
      "Presicion : 0.988700564972\n",
      "F0.5 score : 0.982042648709\n",
      "CONFUSION MATRIX [[1208    2]\n",
      " [   8  175]]\n"
     ]
    }
   ],
   "source": [
    "clf = LogisticRegression()\n",
    "parameters = {\n",
    "    'C': np.linspace(0.1, 10, num=10)\n",
    "}\n",
    "crosValBestModel(clf, train_data, test_data, parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 10 folds for each of 10 candidates, totalling 100 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=2)]: Done   1 tasks      | elapsed:    8.6s\n"
     ]
    }
   ],
   "source": [
    "clf = SVC(kernel='linear')\n",
    "parameters = {\n",
    "    'C': np.linspace(0.1, 10, num=10)\n",
    "}\n",
    "crosValBestModel(clf, train_data, test_data, parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 10 folds for each of 4 candidates, totalling 40 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=2)]: Done   1 tasks      | elapsed:    7.3s\n",
      "[Parallel(n_jobs=2)]: Done   4 tasks      | elapsed:   10.5s\n",
      "[Parallel(n_jobs=2)]: Done   9 tasks      | elapsed:   15.5s\n",
      "[Parallel(n_jobs=2)]: Done  14 tasks      | elapsed:   20.7s\n",
      "[Parallel(n_jobs=2)]: Done  21 tasks      | elapsed:   27.7s\n",
      "[Parallel(n_jobs=2)]: Done  28 tasks      | elapsed:   35.0s\n",
      "[Parallel(n_jobs=2)]: Done  37 tasks      | elapsed:   45.5s\n",
      "[Parallel(n_jobs=2)]: Done  40 out of  40 | elapsed:   48.9s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: 0.956579 using {'n_neighbors': 2}\n",
      "0.956579 (0.010647) with: {'n_neighbors': 2}\n",
      "0.950772 (0.017047) with: {'n_neighbors': 3}\n",
      "0.951649 (0.013465) with: {'n_neighbors': 4}\n",
      "0.943809 (0.008413) with: {'n_neighbors': 5}\n",
      "Accuracy : 0.987796123475\n",
      "Recall : 0.907103825137\n",
      "Presicion : 1.0\n",
      "F0.5 score : 0.979929161747\n",
      "CONFUSION MATRIX [[1210    0]\n",
      " [  17  166]]\n"
     ]
    }
   ],
   "source": [
    "clf = KNeighborsClassifier()\n",
    "parameters = {\n",
    "    'n_neighbors': [2,3,4,5]\n",
    "}\n",
    "\n",
    "crosValBestModel(clf, train_data, test_data, parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf = XGBClassifier()\n",
    "\n",
    "parameters = {\n",
    "    'learning_rate': [0.01,0.01,0.1,1,10],\n",
    "    'max_depth' : [1,3,5,7],\n",
    "    'n_estimators' : [100,300,500,700]\n",
    "    'reg_lambda' : [0.01,0.01,0.1,1,10]\n",
    "}\n",
    "\n",
    "crosValBestModel(clf, train_data, test_data, parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 10 folds for each of 18 candidates, totalling 180 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=2)]: Done   1 tasks      | elapsed:   13.6s\n",
      "[Parallel(n_jobs=2)]: Done   4 tasks      | elapsed:   26.4s\n",
      "[Parallel(n_jobs=2)]: Done   9 tasks      | elapsed:  1.1min\n",
      "[Parallel(n_jobs=2)]: Done  14 tasks      | elapsed:  1.8min\n",
      "[Parallel(n_jobs=2)]: Done  21 tasks      | elapsed:  3.1min\n",
      "[Parallel(n_jobs=2)]: Done  28 tasks      | elapsed:  4.3min\n",
      "[Parallel(n_jobs=2)]: Done  37 tasks      | elapsed:  5.6min\n",
      "[Parallel(n_jobs=2)]: Done  46 tasks      | elapsed:  7.0min\n",
      "[Parallel(n_jobs=2)]: Done  57 tasks      | elapsed:  9.0min\n",
      "[Parallel(n_jobs=2)]: Done  68 tasks      | elapsed: 10.5min\n",
      "[Parallel(n_jobs=2)]: Done  81 tasks      | elapsed: 12.5min\n",
      "[Parallel(n_jobs=2)]: Done  94 tasks      | elapsed: 14.6min\n",
      "[Parallel(n_jobs=2)]: Done 109 tasks      | elapsed: 16.6min\n",
      "[Parallel(n_jobs=2)]: Done 124 tasks      | elapsed: 19.1min\n",
      "[Parallel(n_jobs=2)]: Done 141 tasks      | elapsed: 21.6min\n",
      "[Parallel(n_jobs=2)]: Done 158 tasks      | elapsed: 24.4min\n",
      "[Parallel(n_jobs=2)]: Done 177 tasks      | elapsed: 27.5min\n",
      "[Parallel(n_jobs=2)]: Done 180 out of 180 | elapsed: 28.0min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: 0.976977 using {'max_depth': 250, 'max_features': 'auto', 'n_estimators': 250}\n",
      "0.975376 (0.005038) with: {'max_depth': 200, 'max_features': 'auto', 'n_estimators': 200}\n",
      "0.974220 (0.009182) with: {'max_depth': 200, 'max_features': 'auto', 'n_estimators': 250}\n",
      "0.973670 (0.010738) with: {'max_depth': 200, 'max_features': 'auto', 'n_estimators': 300}\n",
      "0.976163 (0.006424) with: {'max_depth': 200, 'max_features': 'sqrt', 'n_estimators': 200}\n",
      "0.973249 (0.010710) with: {'max_depth': 200, 'max_features': 'sqrt', 'n_estimators': 250}\n",
      "0.974538 (0.010701) with: {'max_depth': 200, 'max_features': 'sqrt', 'n_estimators': 300}\n",
      "0.976005 (0.009370) with: {'max_depth': 250, 'max_features': 'auto', 'n_estimators': 200}\n",
      "0.976977 (0.007162) with: {'max_depth': 250, 'max_features': 'auto', 'n_estimators': 250}\n",
      "0.973975 (0.007668) with: {'max_depth': 250, 'max_features': 'auto', 'n_estimators': 300}\n",
      "0.973715 (0.010254) with: {'max_depth': 250, 'max_features': 'sqrt', 'n_estimators': 200}\n",
      "0.975034 (0.009734) with: {'max_depth': 250, 'max_features': 'sqrt', 'n_estimators': 250}\n",
      "0.973761 (0.009688) with: {'max_depth': 250, 'max_features': 'sqrt', 'n_estimators': 300}\n",
      "0.973273 (0.010280) with: {'max_depth': 300, 'max_features': 'auto', 'n_estimators': 200}\n",
      "0.973982 (0.007446) with: {'max_depth': 300, 'max_features': 'auto', 'n_estimators': 250}\n",
      "0.975755 (0.006235) with: {'max_depth': 300, 'max_features': 'auto', 'n_estimators': 300}\n",
      "0.974992 (0.010328) with: {'max_depth': 300, 'max_features': 'sqrt', 'n_estimators': 200}\n",
      "0.974341 (0.008658) with: {'max_depth': 300, 'max_features': 'sqrt', 'n_estimators': 250}\n",
      "0.974193 (0.009575) with: {'max_depth': 300, 'max_features': 'sqrt', 'n_estimators': 300}\n",
      "Accuracy : 0.990667623833\n",
      "Recall : 0.928961748634\n",
      "Presicion : 1.0\n",
      "F0.5 score : 0.98493626883\n",
      "CONFUSION MATRIX [[1210    0]\n",
      " [  13  170]]\n"
     ]
    }
   ],
   "source": [
    "clf = RandomForestClassifier()\n",
    "\n",
    "parameters = {\n",
    "    'n_estimators': [200, 250, 300],\n",
    "    'max_depth': [200, 250, 300],\n",
    "    'max_features': ['auto', 'sqrt']\n",
    "}\n",
    "\n",
    "crosValBestModel(clf,train_data, test_data, parameters)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}