实战篇/5.Titanic/Kaggle Titanic Best Score.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "id": "BD7C146F60204DF78902EF24E9EAF441",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "id": "3537FD7F7ECF4A1A81D1360049815A24",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "pd.set_option('display.max_columns', 80)\n",
    "pd.set_option('display.max_rows', 1000)\n",
    "pd.set_option('max_colwidth',1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "id": "9C3E410292A24FA08808AF2FC77ACAF0",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "trainData = pd.read_csv(\"train.csv\")\n",
    "testData = pd.read_csv(\"test.csv\")\n",
    "submitData = pd.read_csv(\"gender_submission.csv\")\n",
    "y_train = trainData['Survived']\n",
    "trainData.drop(\"Survived\", axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "id": "A2A4AD423E8F4B4B8C6E6A30C9E42E04",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "allData = pd.concat([trainData, testData], axis=0, ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "71A5588F249647C6B589A5B799186CBE",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "allData.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "id": "FA609CF9F65749999732D9D55E9FBB25",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "allData = pd.get_dummies(allData, columns=['Pclass'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "id": "E5C565C1ACC64AA388EF2A1CD6BA6730",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "allData = pd.get_dummies(allData, columns=['Sex'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "id": "AAEBC8578F9F4E4A843AAB70A0A4C3E0",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "allData['SibSp_Parch'] = allData['SibSp'] + allData['Parch']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "id": "9376D9E7B1A54BAF871202E93A70D701",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "allData = pd.get_dummies(allData, columns=['SibSp', 'Parch', 'SibSp_Parch'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "id": "B100242365AA4D498CA15115105E3F66",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "allData = pd.get_dummies(allData, columns=['Embarked'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "84FAB5292EA6429E849C8B1ECACEAA47",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "allData['NAME1'] = allData['Name'].str.extract('.+,(.+)').str.extract('^(.+?)\\.').str.strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "503A4DEBA17F4489A810CCF9E936D8DB",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "allData.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "id": "0EB5B2B8A0A84574854584AD779AF1CF",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "#将姓名分类处理()\n",
    "allData['NAME1'].replace(['Capt', 'Col', 'Major', 'Dr', 'Rev'], 'Officer' , inplace = True)\n",
    "allData['NAME1'].replace(['Jonkheer', 'Don', 'Sir', 'the Countess', 'Dona', 'Lady'], 'Royalty' , inplace = True)\n",
    "allData['NAME1'].replace(['Mme', 'Ms', 'Mrs'], 'Mrs')\n",
    "allData['NAME1'].replace(['Mlle', 'Miss'], 'Miss')\n",
    "allData['NAME1'].replace(['Mr'], 'Mr' , inplace = True)\n",
    "allData['NAME1'].replace(['Master'], 'Master' , inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "id": "8C483AEFB3084DE28D66D9C09ADC0139",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "allData = pd.get_dummies(allData, columns=['NAME1'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "id": "E97841F63F264C03858A87FCE17571A5",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "#从姓名中提取出姓\n",
    "allData['NAME2'] = allData['Name'].apply(lambda x: x.split('.')[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "id": "3B596747321E4E42843882E11AE0AA72",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# 计算数量,然后合并数据集\n",
    "Name2_sum = allData['NAME2'].value_counts().reset_index()\n",
    "Name2_sum.columns=['NAME2', 'Name2_sum']\n",
    "allData = pd.merge(allData, Name2_sum, how='left', on='NAME2')\n",
    "\n",
    "\n",
    "\n",
    "allData.loc[allData['Name2_sum'] == 1, 'Name2_new'] = 'one'\n",
    "allData.loc[allData['Name2_sum'] > 1, 'Name2_new'] = allData['NAME2']\n",
    "allData.drop('NAME2', axis=1, inplace=True)\n",
    "\n",
    "allData = pd.get_dummies(allData, columns=['Name2_new'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "id": "994DBD1FBD6C4B988E8575EBE40A148B",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "allData.drop('Name', axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "1F9CEE418C09405F8268CA7EDF60EFD4",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# 从上面的分析,发现该特征train集无miss值,test有一个缺失值,先查看\n",
    "allData.loc[allData['Fare'].isnull()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "86C7ACFCC04745B2A10F53A9058A6AC1",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# 票价与pclass和Embarked有关,所以用train分组后的平均数填充\n",
    "trainData.groupby(by=['Pclass', 'Embarked']).Fare.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "id": "80A7E28EF7AE4240B50F186871FDE6BE",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# 用pclass=3和Embarked=S的平均数14.644083来填充\n",
    "allData[\"Fare\"].fillna(14.644083, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "DD0129D8049A4931AC8103D3CFB7705B",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "allData.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "id": "CCA10F0E36A446648DCEF43BD7CA914F",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Ticket提取字符列\n",
    "allData['Ticket_Letter'] = allData['Ticket'].str.split().str[0]\n",
    "# str.isnumeric()  如果S中只有数字字符，则返回True，否则返回False\n",
    "allData['Ticket_Letter'] = allData['Ticket_Letter'].apply(lambda x:np.nan if x.isnumeric() else x)\n",
    "allData.drop('Ticket', axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "id": "103046A3837743E1822506179BFD21FC",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "allData = pd.get_dummies(allData, columns=['Ticket_Letter'], drop_first=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "id": "5B48EE10AFD54D5BB45401E428CED510",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# 使用年龄是否缺失来构造特征\n",
    "allData.loc[allData['Age'].isnull(), 'age_nan'] = 1\n",
    "allData.loc[allData['Age'].notnull(), 'age_nan'] = 0\n",
    "allData = pd.get_dummies(allData, columns=['age_nan'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "4345BBDB97154DD19AC0E7144F5ACB74",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "allData.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "B1671523277747CAABDDEAE5DEBFF326",
    "mdEditEnable": false
   },
   "source": [
    "### 通过建立模型预测缺失的Age特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "id": "3D088505509C45639195179442B47B1D",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# 不要Cabin是因为还没处理，先去掉\n",
    "missing_age = allData.drop(['Cabin'], axis=1)\n",
    "# 将Age完整的项作为训练集、将Age缺失的项作为测试集\n",
    "missing_age_train = missing_age[missing_age['Age'].notnull()]\n",
    "missing_age_test = missing_age[missing_age['Age'].isnull()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "id": "EB909D4734A94B778004AAECF3A2C17A",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# 分离X ,y\n",
    "missing_age_X_train = missing_age_train.drop('Age', axis=1)\n",
    "missing_age_Y_train = missing_age_train['Age']\n",
    "missing_age_X_test = missing_age_test.drop('Age', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "id": "671F22C3893F46639D6A245569055CF0",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# 标准化数据\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "ss = StandardScaler()\n",
    "ss.fit(missing_age_X_train)\n",
    "missing_age_X_train = ss.transform(missing_age_X_train)\n",
    "missing_age_test = ss.transform(missing_age_X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ADF5BD77E7F545B88FA1C4A9DC0DDF74",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# 使用贝叶斯岭回归预测年龄\n",
    "from sklearn.linear_model import BayesianRidge\n",
    "bayes = BayesianRidge()\n",
    "bayes.fit(missing_age_X_train, missing_age_Y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "id": "9C4277BB0A5F4A5EA10B907874030CDF",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# 用预测值填充\n",
    "allData.loc[(allData['Age'].isnull()), 'Age'] = bayes.predict(missing_age_X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "id": "1FA3E2B0C133461AA20B3C4D3C2D071B",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# 数据离散化，将年龄划分四个阶段10以下,10-18,18-30,30-50,50以上\n",
    "allData['Age'] = pd.cut(allData['Age'], bins=[0, 10, 18, 30, 50, 100], labels=[1, 2, 3, 4, 5])\n",
    "# 分列处理\n",
    "allData = pd.get_dummies(allData, columns=['Age'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "id": "815979BE493640468F6FC4A733AF962D",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Cabin项缺失太多，根据是否缺失给予标记\n",
    "allData.loc[allData['Cabin'].isnull(), 'Cabin_nan'] = 1\n",
    "allData.loc[allData['Cabin'].notnull(), 'Cabin_nan'] = 0\n",
    "# 分列处理\n",
    "allData = pd.get_dummies(allData, columns=['Cabin_nan'])\n",
    "allData.drop('Cabin', axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "E988C1B7EFDA488A82348072A088CC65",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "len(y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "id": "EB39D89303A44FFDB052833EBD3109F7",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# 分离数据\n",
    "trainData = allData[:891]\n",
    "testData = allData[891:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "F43A93AF87434865875F9231CF19128A",
    "mdEditEnable": false
   },
   "source": [
    "> 线性模型需要标准化后的数据建模，而树类模型不需要标准化的数据\n",
    "  在处理时，注意要将训练集的数据transform到test集上"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "id": "486E4161C94F42EE9A1221AB0C558EDD",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "ss2 = StandardScaler()\n",
    "ss2.fit(trainData)\n",
    "trainData_sd = ss2.transform(trainData)\n",
    "testData_sd = ss2.transform(testData)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# XGBOOST 生成新特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "temp_train = trainData.copy()\n",
    "temp_test = testData.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from xgboost.sklearn import XGBClassifier\n",
    "xgb = XGBClassifier(booster='gbtree', \n",
    "                    learning_rate =0.1,\n",
    "                    objective='binary:logitraw',   # binary:logistic\n",
    "                    gamma=0.05, \n",
    "                    subsample=0.4, \n",
    "                    reg_alpha=1e-05,\n",
    "                    n_estimators=50,\n",
    "                    metric=['auc', 'l2'],\n",
    "                    colsample_bytree=0.7, \n",
    "                    silent=1, \n",
    "                    nthread=4)\n",
    "\n",
    "xgb.fit(temp_train.values, y_train)\n",
    "new_feature= xgb.apply(temp_train.values)\n",
    "trainData = np.hstack((temp_train, new_feature))\n",
    "\n",
    "new_feature_test = xgb.apply(temp_test.values)\n",
    "testData = np.hstack((temp_test, new_feature_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import lightgbm as lgb\n",
    "gbm = lgb.LGBMClassifier(learning_rate=0.1, \n",
    "                   boosting_type='gbdt', \n",
    "                   objective='binary',\n",
    "                   n_estimators=100,\n",
    "                   metric=['auc', 'l2'], \n",
    "                   max_depth=7, \n",
    "                   bagging_fraction=0.7, \n",
    "                   is_unbalance=True)\n",
    "\n",
    "gbm.fit(temp_train.values, y_train)\n",
    "new_feature = gbm.apply(temp_train.values)\n",
    "trainData = np.hstack((trainData, new_feature))\n",
    "\n",
    "new_feature_test = gbm.apply(temp_test.values)\n",
    "testData = np.hstack((testData, new_feature_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "rf = RandomForestClassifier(n_estimators=200, \n",
    "                            min_samples_split=90, \n",
    "                            min_samples_leaf=3,\n",
    "                            max_depth=8,\n",
    "                            oob_score=True,\n",
    "                            criterion='gini')\n",
    "\n",
    "rf.fit(temp_train.values, y_train)\n",
    "new_feature = rf.apply(temp_train.values)\n",
    "trainData = np.hstack((trainData, new_feature))\n",
    "\n",
    "new_feature_test = rf.apply(temp_test.values)\n",
    "testData = np.hstack((testData, new_feature_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "0474DBE9D02C48508FDCB12DEE14FEF0",
    "mdEditEnable": false
   },
   "source": [
    "# Voting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "4966D080CAE24D2B8E73048093317D4E",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "from sklearn.ensemble import VotingClassifier\n",
    "\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "lr1 = LogisticRegression(C=0.1, max_iter=1000)\n",
    "lr2 = LogisticRegression(C=0.01, max_iter=1000)\n",
    "\n",
    "import xgboost as xgb\n",
    "xgb_model1 = xgb.XGBClassifier(max_depth=6, min_samples_leaf=3, n_estimators=2000, metric=['l2', 'auc'])\n",
    "\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "rf1 = RandomForestClassifier(n_estimators=200,min_samples_leaf=3,max_depth=8,oob_score=True, criterion='gini')\n",
    "rf2 = RandomForestClassifier(n_estimators=200,min_samples_leaf=3,max_depth=8,oob_score=True, criterion='entropy')\n",
    "\n",
    "\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "gbdt = GradientBoostingClassifier(learning_rate=0.01,min_samples_leaf=2,max_depth=6,n_estimators=200)\n",
    "\n",
    "import lightgbm as lgb\n",
    "gbm = lgb.LGBMClassifier(boosting_type='gbdt',\n",
    "                         objective='binary',\n",
    "                         metric=['l2', 'auc'],\n",
    "                         num_leaves=100, \n",
    "                         min_data_in_leaf=100,\n",
    "                         learning_rate=0.02,\n",
    "                         bagging_fraction=0.8,\n",
    "                         bagging_freq=5,\n",
    "                         lambda_l1=0.4,\n",
    "                         lambda_l2=0.6, \n",
    "                         max_depth=6,\n",
    "                         is_unbalance=True)\n",
    "\n",
    "vot = VotingClassifier(estimators=[('lr1', lr1), ('lr2', lr2), ('rf1', rf1), ('rf2', rf2), \n",
    "                                   ('gbdt',gbdt), ('xgb1',xgb_model1), ('lgb', gbm)], voting='soft')\n",
    "vot.fit(trainData, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "id": "F12C2908728F4667988F815CF626A2D6",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "submitData[\"Survived\"] = vot.predict(testData)\n",
    "submitData.to_csv('voting.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bagging"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import BaggingClassifier\n",
    "bagging = BaggingClassifier(base_estimator=vot, n_estimators=3)\n",
    "bagging.fit(trainData, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "submitData[\"Survived\"] = bagging.predict(testData)\n",
    "submitData.to_csv('bagging.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "B2CBE5B330084E95B862D9653DE4277D",
    "mdEditEnable": false
   },
   "source": [
    "# LightGBM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "9E7EFBE804C54FDC815ACD9ABC944ECA",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import lightgbm as lgb\n",
    "lgb_train = lgb.Dataset(trainData, y_train)  \n",
    "lgb_eval = lgb.Dataset(trainData[600:], y_train[600:], reference=lgb_train)  \n",
    "# specify your configurations as a dict  \n",
    "params = {  \n",
    "    'boosting_type': 'gbdt',  \n",
    "    'objective': 'binary',  \n",
    "    'metric': ['auc', 'l2'],  # 'map@2', \n",
    "    'num_leaves': 100, # 4\n",
    "    'min_data_in_leaf': 100,\n",
    "    'learning_rate': 0.02,  \n",
    "#     'feature_fraction': 0.3,  \n",
    "    'bagging_fraction': 0.8,  \n",
    "    'bagging_freq': 5,  \n",
    "    'lambda_l1': 0.4,  \n",
    "    'lambda_l2': 0.6,\n",
    "    'max_depth':6,\n",
    "#     'min_gain_to_split': 0.2,  \n",
    "    'verbose': 5,  \n",
    "    'is_unbalance': True\n",
    "}  \n",
    "  \n",
    "print('Start training...')  \n",
    "gbm = lgb.train(params,  \n",
    "                lgb_train,  \n",
    "                num_boost_round=8000,  \n",
    "                valid_sets=lgb_eval,  \n",
    "                early_stopping_rounds=500)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "id": "12CB48DFC22E4FEA94D3B185FADD5E26",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "pb = gbm.predict(testData, num_iteration=gbm.best_iteration)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "submitData[\"Survived\"].loc[pb > 0.5] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "submitData[\"Survived\"].loc[pb < 0.5] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "submitData.to_csv('lgb.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "29D3670D40F846A2A7733D6CC6A23C31",
    "mdEditEnable": false,
    "scrolled": false
   },
   "source": [
    "# NN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "CD366786F0F74DFA87F6AC6656368B11",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import OneHotEncoder\n",
    "enc = OneHotEncoder()\n",
    "enc.fit(y_train.reshape(-1, 1))\n",
    "y_hot = enc.transform(y_train.reshape(-1, 1))\n",
    "\n",
    "\n",
    "#构建LM神经网络模型\n",
    "from keras.models import Sequential #导入神经网络初始化函数\n",
    "from keras.layers.core import Dense, Activation #导入神经网络层函数、激活函数\n",
    "from keras.layers import Dropout\n",
    "from keras.metrics import top_k_categorical_accuracy\n",
    "from keras.callbacks import EarlyStopping\n",
    "netfile = 'net.model' #构建的神经网络模型存储路径\n",
    "\n",
    "def acc_top2(y_true, y_pred):\n",
    "    return top_k_categorical_accuracy(y_true, y_pred, k=2)\n",
    "\n",
    "net = Sequential()\n",
    "net.add(Dense(input_dim = 241, output_dim = 256))\n",
    "net.add(Activation('relu'))\n",
    "net.add(Dense(input_dim = 256, output_dim = 512))\n",
    "net.add(Activation('relu'))\n",
    "net.add(Dense(input_dim = 512, output_dim = 1024))\n",
    "net.add(Activation('relu'))\n",
    "net.add(Dropout(0.3))\n",
    "net.add(Dense(input_dim = 1024, output_dim = 1024))\n",
    "net.add(Activation('relu'))\n",
    "net.add(Dropout(0.3))\n",
    "net.add(Dense(input_dim = 1024, output_dim = 1024))\n",
    "net.add(Dense(input_dim = 1024, output_dim = 2))\n",
    "net.add(Activation('softmax'))\n",
    "net.compile(loss = 'categorical_crossentropy', optimizer = 'adam', metrics=['accuracy']) # accuracy\n",
    "early_stopping = EarlyStopping(monitor='val_loss', patience=500, verbose=10)\n",
    "\n",
    "net.fit(trainData_sd, y_hot, epochs=8000, batch_size=64, validation_data=(trainData_sd, y_hot), callbacks=[early_stopping])\n",
    "net.save_weights(netfile) #保存模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "id": "EC474ED6104C43018F31BED4EF7FC3E8",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "pred = net.predict(testData_sd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "id": "8C634526871942DE8CAF56874716809C",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "pred = pred.argsort()[np.arange(len(pred)), -1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "id": "9119CE5F1ADB42BD83A5327CD2CF7374",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "submitData[\"Survived\"] = pred\n",
    "submitData.to_csv('nn.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "9C6031256D27407EBBFA2E2A86055773",
    "scrolled": false
   },
   "source": [
    "# Blend"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn.cross_validation import StratifiedKFold\n",
    "from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from xgboost.sklearn import XGBClassifier\n",
    "import lightgbm as lgb\n",
    "\n",
    "def blend(X, y, X_submission, n_folds):\n",
    "    skf = list(StratifiedKFold(y, n_folds))\n",
    "\n",
    "    clfs = [RandomForestClassifier(n_estimators=200,min_samples_leaf=3,max_depth=8,oob_score=True, criterion='gini'),\n",
    "            RandomForestClassifier(n_estimators=200,min_samples_leaf=3,max_depth=8,oob_score=True, criterion='entropy'),\n",
    "            GradientBoostingClassifier(learning_rate=0.01,min_samples_leaf=2,max_depth=6,n_estimators=200),\n",
    "            XGBClassifier(learning_rate =0.05, n_estimators=300, max_depth=6, min_samples_leaf=3, num_round=2000),\n",
    "            KNeighborsClassifier(n_neighbors=5, weights='uniform', p=1)]\n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "    dataset_blend_train = np.zeros((X.shape[0], len(clfs)))\n",
    "\n",
    "    dataset_blend_test = np.zeros((X_submission.shape[0], len(clfs)))\n",
    "\n",
    "    for j, clf in enumerate(clfs):\n",
    "        print (j, clf)\n",
    "        dataset_blend_test_j = np.zeros((X_submission.shape[0], len(skf)))\n",
    "        for i, (train, test) in enumerate(skf):\n",
    "            print (\"Fold\", i)\n",
    "            X_train = X[train]\n",
    "            y_train = y[train]\n",
    "            X_test = X[test]\n",
    "            y_test = y[test]\n",
    "            clf.fit(X_train, y_train)\n",
    "            y_submission = clf.predict_proba(X_test)[:, 1]\n",
    "            dataset_blend_train[test, j] = y_submission\n",
    "            dataset_blend_test_j[:, i] = clf.predict_proba(X_submission)[:, 1]\n",
    "        dataset_blend_test[:, j] = dataset_blend_test_j.mean(1)\n",
    "    print(\"Blending.\")\n",
    "    \n",
    "    from sklearn.linear_model import LogisticRegression\n",
    "    lr1 = LogisticRegression(C=0.1, max_iter=1000)\n",
    "    lr2 = LogisticRegression(C=0.01, max_iter=1000)\n",
    "    \n",
    "    from sklearn.ensemble import VotingClassifier\n",
    "    clf = VotingClassifier(estimators=[('lr1', lr1), ('lr2', lr2)], voting='hard')\n",
    "    clf.fit(dataset_blend_train, y)\n",
    "#     y_submission = clf.predict_proba(dataset_blend_test)[:, 1]\n",
    "#     y_submission = clf.predict(dataset_blend_test)\n",
    "    return clf.predict(dataset_blend_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "pred = blend(trainData, y_train, testData, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "submitData[\"Survived\"] = pred\n",
    "submitData.to_csv('blend.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}