Sentiment  Twitter RNN.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sentiment Analysis with an RNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#https://www.kaggle.com/ngyptr/python-nltk-sentiment-analysis/data\n",
    "data = pd.read_csv('Sentiment.csv')\n",
    "# Keeping only the neccessary columns\n",
    "data = data[['text','sentiment']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>RT @NancyLeeGrahn: How did everyone feel about...</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>RT @ScottWalker: Didn't catch the full #GOPdeb...</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>RT @TJMShow: No mention of Tamir Rice and the ...</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>RT @RobGeorge: That Carly Fiorina is trending ...</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>RT @DanScavino: #GOPDebate w/ @realDonaldTrump...</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>RT @GregAbbott_TX: @TedCruz: \"On my first day ...</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>RT @warriorwoman91: I liked her and was happy ...</td>\n",
       "      <td>Negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Going on #MSNBC Live with @ThomasARoberts arou...</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text sentiment\n",
       "0  RT @NancyLeeGrahn: How did everyone feel about...   Neutral\n",
       "1  RT @ScottWalker: Didn't catch the full #GOPdeb...  Positive\n",
       "2  RT @TJMShow: No mention of Tamir Rice and the ...   Neutral\n",
       "3  RT @RobGeorge: That Carly Fiorina is trending ...  Positive\n",
       "4  RT @DanScavino: #GOPDebate w/ @realDonaldTrump...  Positive\n",
       "5  RT @GregAbbott_TX: @TedCruz: \"On my first day ...  Positive\n",
       "6  RT @warriorwoman91: I liked her and was happy ...  Negative\n",
       "7  Going on #MSNBC Live with @ThomasARoberts arou...   Neutral"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[:8]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from string import punctuation\n",
    "\n",
    "def pd_series_to_text(series, split=True):\n",
    "    review_list = []\n",
    "    for line in series:\n",
    "        \n",
    "        line_text = [e.lower() for e in line.split() if len(e) >= 3]\n",
    "        line_text = [word for word in line_text\n",
    "            if 'http' not in word\n",
    "            and not word.startswith('@')\n",
    "            and not word.startswith('#')\n",
    "            and word != 'RT']\n",
    "        line_text = ''.join([c for c in str(line_text) if c not in punctuation])\n",
    "        review_list.append(line_text)\n",
    "    if not split:\n",
    "        reviews = ' '.join([c for c in review_list if len(c) > 2])\n",
    "    else:\n",
    "        reviews = review_list\n",
    "    return reviews\n",
    "\n",
    "def pd_series_to_sentiment(series):\n",
    "    senti_list = []\n",
    "    for line in series:\n",
    "        if line=='Positive':\n",
    "            senti = 2\n",
    "        #elif line=='Neutral':\n",
    "        #    senti = 1\n",
    "        else:\n",
    "            senti = 0\n",
    "        senti_list.append(senti)\n",
    "    return senti_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "reviews = pd_series_to_text(data['text'],True)\n",
    "words = pd_series_to_text(data['text'],False).split()\n",
    "sentiment = pd_series_to_sentiment(data['sentiment'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0, 2, 0, 2, 2]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentiment[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['how did everyone feel about the climate change question last night exactly',\n",
       " 'didnt catch the full last night here are some scotts best lines seconds',\n",
       " 'mention tamir rice and the was held cleveland wow',\n",
       " 'that carly fiorina trending hours after her debate above any the men justcompleted says shes',\n",
       " 'delivered the highest ratings the history presidential debates']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reviews[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "13871 13871\n"
     ]
    }
   ],
   "source": [
    "print(len(reviews),len(sentiment))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Encoding the words and removing empty lines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "counts = Counter(words)\n",
    "vocab = sorted(counts, key=counts.get, reverse=True)\n",
    "vocab_to_int = {word: ii for ii, word in enumerate(vocab, 1)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "reviews_ints = []\n",
    "for each in reviews:\n",
    "    reviews_ints.append([vocab_to_int[word] for word in each.split()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Zero-length reviews: 0\n",
      "Maximum review length: 23\n"
     ]
    }
   ],
   "source": [
    "review_lens = Counter([len(x) for x in reviews_ints])\n",
    "print(\"Zero-length reviews: {}\".format(review_lens[0]))\n",
    "print(\"Maximum review length: {}\".format(max(review_lens)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "non_zero_idx = [ii for ii, review in enumerate(reviews_ints) if len(review) != 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "reviews_ints = [reviews_ints[ii] for ii in non_zero_idx]\n",
    "sentiment = np.asarray([sentiment[ii] for ii in non_zero_idx])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### create featurevectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "seq_len = 23\n",
    "features = np.zeros((len(reviews_ints), seq_len), dtype=int)\n",
    "for i, row in enumerate(reviews_ints):\n",
    "    features[i, -len(row):] = np.array(row)[:seq_len]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training, Validation, Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "split_frac = 0.8\n",
    "split_idx = int(len(features) * split_frac)\n",
    "train_x = features[:split_idx]\n",
    "train_y = sentiment[:split_idx]\n",
    "val_x = features[split_idx:]\n",
    "val_y = sentiment[split_idx:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_idx = int(len(val_x) * 0.5)\n",
    "test_x =  val_x[test_idx:]\n",
    "val_x = val_x[:test_idx]\n",
    "test_y =  val_y[test_idx:]\n",
    "val_y = val_y[:test_idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\t\t\tFeature Shapes:\n",
      "Features: \t\t(13871, 23) \n",
      "Train set: \t\t(11096, 23) \n",
      "Validation set: \t(1387, 23) \n",
      "Test set: \t\t(1388, 23)\n"
     ]
    }
   ],
   "source": [
    "print(\"\\t\\t\\tFeature Shapes:\")\n",
    "print(\"Features: \\t\\t{}\".format(features.shape), \n",
    "      \"\\nTrain set: \\t\\t{}\".format(train_x.shape), \n",
    "      \"\\nValidation set: \\t{}\".format(val_x.shape),\n",
    "      \"\\nTest set: \\t\\t{}\".format(test_x.shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Build the graph"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* lstm_size: Number of units in the hidden layers in the LSTM cells. Usually larger is better performance wise. Common values are 128, 256, 512, etc.\n",
    "* lstm_layers: Number of LSTM layers in the network. I'd start with 1, then add more if I'm underfitting.\n",
    "* batch_size: The number of reviews to feed the network in one training pass. Typically this should be set as high as you can go without running out of memory.\n",
    "* learning_rate: Learning rate\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lstm_size = 256\n",
    "lstm_layers = 1\n",
    "batch_size = 500\n",
    "learning_rate = 0.005"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "n_words = len(vocab_to_int) + 1 # Adding 1 because we use 0's for padding, dictionary started at 1\n",
    "\n",
    "# Create the graph object\n",
    "graph = tf.Graph()\n",
    "# Add nodes to the graph\n",
    "with graph.as_default():\n",
    "    inputs_ = tf.placeholder(tf.int32, [None, None], name='inputs')\n",
    "    labels_ = tf.placeholder(tf.int32, [None, None], name='labels')\n",
    "    keep_prob = tf.placeholder(tf.float32, name='keep_prob')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Size of the embedding vectors (number of units in the embedding layer)\n",
    "embed_size = 300 \n",
    "\n",
    "with graph.as_default():\n",
    "    embedding = tf.Variable(tf.random_uniform((n_words, embed_size), -1, 1))\n",
    "    embed = tf.nn.embedding_lookup(embedding, inputs_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with graph.as_default():\n",
    "    # Your basic LSTM cell\n",
    "    lstm = tf.contrib.rnn.BasicLSTMCell(lstm_size)\n",
    "    \n",
    "    # Add dropout to the cell\n",
    "    drop = tf.contrib.rnn.DropoutWrapper(lstm, output_keep_prob=keep_prob)\n",
    "    \n",
    "    # Stack up multiple LSTM layers, for deep learning\n",
    "    cell = tf.contrib.rnn.MultiRNNCell([drop] * lstm_layers)\n",
    "    \n",
    "    # Getting an initial state of all zeros\n",
    "    initial_state = cell.zero_state(batch_size, tf.float32)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with graph.as_default():\n",
    "    outputs, final_state = tf.nn.dynamic_rnn(cell, embed,\n",
    "                                             initial_state=initial_state)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "with graph.as_default():\n",
    "    predictions = tf.contrib.layers.fully_connected(outputs[:, -1], 1, activation_fn=tf.sigmoid)\n",
    "    cost = tf.losses.mean_squared_error(labels_, predictions)\n",
    "    \n",
    "    optimizer = tf.train.AdamOptimizer(learning_rate).minimize(cost)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with graph.as_default():\n",
    "    correct_pred = tf.equal(tf.cast(tf.round(predictions), tf.int32), labels_)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# batches\n",
    "def get_batches(x, y, batch_size=100):\n",
    "    \n",
    "    n_batches = len(x)//batch_size\n",
    "    x, y = x[:n_batches*batch_size], y[:n_batches*batch_size]\n",
    "    for ii in range(0, len(x), batch_size):\n",
    "        yield x[ii:ii+batch_size], y[ii:ii+batch_size]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0/20 Iteration: 5 Train loss: 0.570\n",
      "Epoch: 0/20 Iteration: 10 Train loss: 0.498\n",
      "Epoch: 0/20 Iteration: 15 Train loss: 0.536\n",
      "Epoch: 0/20 Iteration: 20 Train loss: 0.445\n",
      "Epoch: 1/20 Iteration: 25 Train loss: 0.404\n",
      "Val acc: 0.719\n",
      "Epoch: 1/20 Iteration: 30 Train loss: 0.431\n",
      "Epoch: 1/20 Iteration: 35 Train loss: 0.445\n",
      "Epoch: 1/20 Iteration: 40 Train loss: 0.256\n",
      "Epoch: 2/20 Iteration: 45 Train loss: 0.424\n",
      "Epoch: 2/20 Iteration: 50 Train loss: 0.316\n",
      "Val acc: 0.738\n",
      "Epoch: 2/20 Iteration: 55 Train loss: 0.306\n",
      "Epoch: 2/20 Iteration: 60 Train loss: 0.316\n",
      "Epoch: 2/20 Iteration: 65 Train loss: 0.251\n",
      "Epoch: 3/20 Iteration: 70 Train loss: 0.275\n",
      "Epoch: 3/20 Iteration: 75 Train loss: 0.293\n",
      "Val acc: 0.715\n",
      "Epoch: 3/20 Iteration: 80 Train loss: 0.242\n",
      "Epoch: 3/20 Iteration: 85 Train loss: 0.321\n",
      "Epoch: 4/20 Iteration: 90 Train loss: 0.274\n",
      "Epoch: 4/20 Iteration: 95 Train loss: 0.221\n",
      "Epoch: 4/20 Iteration: 100 Train loss: 0.251\n",
      "Val acc: 0.692\n",
      "Epoch: 4/20 Iteration: 105 Train loss: 0.398\n",
      "Epoch: 4/20 Iteration: 110 Train loss: 0.194\n",
      "Epoch: 5/20 Iteration: 115 Train loss: 0.222\n",
      "Epoch: 5/20 Iteration: 120 Train loss: 0.219\n",
      "Epoch: 5/20 Iteration: 125 Train loss: 0.228\n",
      "Val acc: 0.695\n",
      "Epoch: 5/20 Iteration: 130 Train loss: 0.213\n",
      "Epoch: 6/20 Iteration: 135 Train loss: 0.195\n",
      "Epoch: 6/20 Iteration: 140 Train loss: 0.193\n",
      "Epoch: 6/20 Iteration: 145 Train loss: 0.223\n",
      "Epoch: 6/20 Iteration: 150 Train loss: 0.200\n",
      "Val acc: 0.675\n",
      "Epoch: 7/20 Iteration: 155 Train loss: 0.234\n",
      "Epoch: 7/20 Iteration: 160 Train loss: 0.181\n",
      "Epoch: 7/20 Iteration: 165 Train loss: 0.190\n",
      "Epoch: 7/20 Iteration: 170 Train loss: 0.202\n",
      "Epoch: 7/20 Iteration: 175 Train loss: 0.180\n",
      "Val acc: 0.700\n",
      "Epoch: 8/20 Iteration: 180 Train loss: 0.182\n",
      "Epoch: 8/20 Iteration: 185 Train loss: 0.187\n",
      "Epoch: 8/20 Iteration: 190 Train loss: 0.179\n",
      "Epoch: 8/20 Iteration: 195 Train loss: 0.263\n",
      "Epoch: 9/20 Iteration: 200 Train loss: 0.215\n",
      "Val acc: 0.711\n",
      "Epoch: 9/20 Iteration: 205 Train loss: 0.162\n",
      "Epoch: 9/20 Iteration: 210 Train loss: 0.202\n",
      "Epoch: 9/20 Iteration: 215 Train loss: 0.382\n",
      "Epoch: 9/20 Iteration: 220 Train loss: 0.155\n",
      "Epoch: 10/20 Iteration: 225 Train loss: 0.189\n",
      "Val acc: 0.708\n",
      "Epoch: 10/20 Iteration: 230 Train loss: 0.177\n",
      "Epoch: 10/20 Iteration: 235 Train loss: 0.214\n",
      "Epoch: 10/20 Iteration: 240 Train loss: 0.206\n",
      "Epoch: 11/20 Iteration: 245 Train loss: 0.166\n",
      "Epoch: 11/20 Iteration: 250 Train loss: 0.175\n",
      "Val acc: 0.703\n",
      "Epoch: 11/20 Iteration: 255 Train loss: 0.193\n",
      "Epoch: 11/20 Iteration: 260 Train loss: 0.193\n",
      "Epoch: 12/20 Iteration: 265 Train loss: 0.208\n",
      "Epoch: 12/20 Iteration: 270 Train loss: 0.155\n",
      "Epoch: 12/20 Iteration: 275 Train loss: 0.175\n",
      "Val acc: 0.693\n",
      "Epoch: 12/20 Iteration: 280 Train loss: 0.197\n",
      "Epoch: 12/20 Iteration: 285 Train loss: 0.178\n",
      "Epoch: 13/20 Iteration: 290 Train loss: 0.169\n",
      "Epoch: 13/20 Iteration: 295 Train loss: 0.177\n",
      "Epoch: 13/20 Iteration: 300 Train loss: 0.171\n",
      "Val acc: 0.706\n",
      "Epoch: 13/20 Iteration: 305 Train loss: 0.259\n",
      "Epoch: 14/20 Iteration: 310 Train loss: 0.203\n",
      "Epoch: 14/20 Iteration: 315 Train loss: 0.157\n",
      "Epoch: 14/20 Iteration: 320 Train loss: 0.197\n",
      "Epoch: 14/20 Iteration: 325 Train loss: 0.378\n",
      "Val acc: 0.693\n",
      "Epoch: 14/20 Iteration: 330 Train loss: 0.155\n",
      "Epoch: 15/20 Iteration: 335 Train loss: 0.180\n",
      "Epoch: 15/20 Iteration: 340 Train loss: 0.163\n",
      "Epoch: 15/20 Iteration: 345 Train loss: 0.203\n",
      "Epoch: 15/20 Iteration: 350 Train loss: 0.206\n",
      "Val acc: 0.696\n",
      "Epoch: 16/20 Iteration: 355 Train loss: 0.163\n",
      "Epoch: 16/20 Iteration: 360 Train loss: 0.168\n",
      "Epoch: 16/20 Iteration: 365 Train loss: 0.190\n",
      "Epoch: 16/20 Iteration: 370 Train loss: 0.193\n",
      "Epoch: 17/20 Iteration: 375 Train loss: 0.207\n",
      "Val acc: 0.697\n",
      "Epoch: 17/20 Iteration: 380 Train loss: 0.152\n",
      "Epoch: 17/20 Iteration: 385 Train loss: 0.168\n",
      "Epoch: 17/20 Iteration: 390 Train loss: 0.201\n",
      "Epoch: 17/20 Iteration: 395 Train loss: 0.168\n",
      "Epoch: 18/20 Iteration: 400 Train loss: 0.166\n",
      "Val acc: 0.695\n",
      "Epoch: 18/20 Iteration: 405 Train loss: 0.176\n",
      "Epoch: 18/20 Iteration: 410 Train loss: 0.160\n",
      "Epoch: 18/20 Iteration: 415 Train loss: 0.258\n",
      "Epoch: 19/20 Iteration: 420 Train loss: 0.195\n",
      "Epoch: 19/20 Iteration: 425 Train loss: 0.155\n",
      "Val acc: 0.706\n",
      "Epoch: 19/20 Iteration: 430 Train loss: 0.196\n",
      "Epoch: 19/20 Iteration: 435 Train loss: 0.370\n",
      "Epoch: 19/20 Iteration: 440 Train loss: 0.155\n"
     ]
    }
   ],
   "source": [
    "epochs = 20\n",
    "\n",
    "with graph.as_default():\n",
    "    saver = tf.train.Saver()\n",
    "\n",
    "with tf.Session(graph=graph) as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    iteration = 1\n",
    "    for e in range(epochs):\n",
    "        state = sess.run(initial_state)\n",
    "        \n",
    "        for ii, (x, y) in enumerate(get_batches(train_x, train_y, batch_size), 1):\n",
    "            feed = {inputs_: x,\n",
    "                    labels_: y[:, None],\n",
    "                    keep_prob: 0.5,\n",
    "                    initial_state: state}\n",
    "            loss, state, _ = sess.run([cost, final_state, optimizer], feed_dict=feed)\n",
    "            \n",
    "            if iteration%5==0:\n",
    "                print(\"Epoch: {}/{}\".format(e, epochs),\n",
    "                      \"Iteration: {}\".format(iteration),\n",
    "                      \"Train loss: {:.3f}\".format(loss))\n",
    "\n",
    "            if iteration%25==0:\n",
    "                val_acc = []\n",
    "                val_state = sess.run(cell.zero_state(batch_size, tf.float32))\n",
    "                for x, y in get_batches(val_x, val_y, batch_size):\n",
    "                    feed = {inputs_: x,\n",
    "                            labels_: y[:, None],\n",
    "                            keep_prob: 1,\n",
    "                            initial_state: val_state}\n",
    "                    batch_acc, val_state = sess.run([accuracy, final_state], feed_dict=feed)\n",
    "                    val_acc.append(batch_acc)\n",
    "                print(\"Val acc: {:.3f}\".format(np.mean(val_acc)))\n",
    "            iteration +=1\n",
    "    saver.save(sess, \"checkpoints/sentiment.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from checkpoints\\sentiment.ckpt\n",
      "Test accuracy: 0.746\n"
     ]
    }
   ],
   "source": [
    "test_acc = []\n",
    "with tf.Session(graph=graph) as sess:\n",
    "    saver.restore(sess, tf.train.latest_checkpoint('checkpoints'))\n",
    "    test_state = sess.run(cell.zero_state(batch_size, tf.float32))\n",
    "    for ii, (x, y) in enumerate(get_batches(test_x, test_y, batch_size), 1):\n",
    "        feed = {inputs_: x,\n",
    "                labels_: y[:, None],\n",
    "                keep_prob: 1,\n",
    "                initial_state: test_state}\n",
    "        batch_acc, test_state = sess.run([accuracy, final_state], feed_dict=feed)\n",
    "        test_acc.append(batch_acc)\n",
    "    print(\"Test accuracy: {:.3f}\".format(np.mean(test_acc)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}