Sentiment_Analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 597,
   "source": [
    "# importing the needed packages/modules/libraries\r\n",
    "\r\n",
    "import numpy as np\r\n",
    "import pandas as pd\r\n",
    "import string\r\n",
    "import random\r\n",
    "import math\r\n",
    "import nltk\r\n",
    "from nltk import pos_tag, NaiveBayesClassifier, FreqDist\r\n",
    "from nltk.tokenize import word_tokenize\r\n",
    "from nltk.corpus import stopwords\r\n",
    "from nltk.stem import WordNetLemmatizer\r\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\r\n",
    "from sklearn.svm import SVC"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 598,
   "source": [
    "# loading the tweets.csv file into a dataframe\r\n",
    "\r\n",
    "df=pd.read_csv(\"tweets.csv\")"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 599,
   "source": [
    "# analyzing the head of the dataframe\r\n",
    "\r\n",
    "df.head()"
   ],
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "             tweet_id airline_sentiment  airline_sentiment_confidence  \\\n",
       "0  570306133677760513           neutral                        1.0000   \n",
       "1  570301130888122368          positive                        0.3486   \n",
       "2  570301083672813571           neutral                        0.6837   \n",
       "3  570301031407624196          negative                        1.0000   \n",
       "4  570300817074462722          negative                        1.0000   \n",
       "\n",
       "  negativereason  negativereason_confidence         airline  \\\n",
       "0            NaN                        NaN  Virgin America   \n",
       "1            NaN                     0.0000  Virgin America   \n",
       "2            NaN                        NaN  Virgin America   \n",
       "3     Bad Flight                     0.7033  Virgin America   \n",
       "4     Can't Tell                     1.0000  Virgin America   \n",
       "\n",
       "  airline_sentiment_gold        name negativereason_gold  retweet_count  \\\n",
       "0                    NaN     cairdin                 NaN              0   \n",
       "1                    NaN    jnardino                 NaN              0   \n",
       "2                    NaN  yvonnalynn                 NaN              0   \n",
       "3                    NaN    jnardino                 NaN              0   \n",
       "4                    NaN    jnardino                 NaN              0   \n",
       "\n",
       "                                                text tweet_coord  \\\n",
       "0                @VirginAmerica What @dhepburn said.         NaN   \n",
       "1  @VirginAmerica plus you've added commercials t...         NaN   \n",
       "2  @VirginAmerica I didn't today... Must mean I n...         NaN   \n",
       "3  @VirginAmerica it's really aggressive to blast...         NaN   \n",
       "4  @VirginAmerica and it's a really big bad thing...         NaN   \n",
       "\n",
       "               tweet_created tweet_location               user_timezone  \n",
       "0  2015-02-24 11:35:52 -0800            NaN  Eastern Time (US & Canada)  \n",
       "1  2015-02-24 11:15:59 -0800            NaN  Pacific Time (US & Canada)  \n",
       "2  2015-02-24 11:15:48 -0800      Lets Play  Central Time (US & Canada)  \n",
       "3  2015-02-24 11:15:36 -0800            NaN  Pacific Time (US & Canada)  \n",
       "4  2015-02-24 11:14:45 -0800            NaN  Pacific Time (US & Canada)  "
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>airline_sentiment</th>\n",
       "      <th>airline_sentiment_confidence</th>\n",
       "      <th>negativereason</th>\n",
       "      <th>negativereason_confidence</th>\n",
       "      <th>airline</th>\n",
       "      <th>airline_sentiment_gold</th>\n",
       "      <th>name</th>\n",
       "      <th>negativereason_gold</th>\n",
       "      <th>retweet_count</th>\n",
       "      <th>text</th>\n",
       "      <th>tweet_coord</th>\n",
       "      <th>tweet_created</th>\n",
       "      <th>tweet_location</th>\n",
       "      <th>user_timezone</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>570306133677760513</td>\n",
       "      <td>neutral</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Virgin America</td>\n",
       "      <td>NaN</td>\n",
       "      <td>cairdin</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>@VirginAmerica What @dhepburn said.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-02-24 11:35:52 -0800</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Eastern Time (US &amp; Canada)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>570301130888122368</td>\n",
       "      <td>positive</td>\n",
       "      <td>0.3486</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>Virgin America</td>\n",
       "      <td>NaN</td>\n",
       "      <td>jnardino</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>@VirginAmerica plus you've added commercials t...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-02-24 11:15:59 -0800</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Pacific Time (US &amp; Canada)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>570301083672813571</td>\n",
       "      <td>neutral</td>\n",
       "      <td>0.6837</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Virgin America</td>\n",
       "      <td>NaN</td>\n",
       "      <td>yvonnalynn</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>@VirginAmerica I didn't today... Must mean I n...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-02-24 11:15:48 -0800</td>\n",
       "      <td>Lets Play</td>\n",
       "      <td>Central Time (US &amp; Canada)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>570301031407624196</td>\n",
       "      <td>negative</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>Bad Flight</td>\n",
       "      <td>0.7033</td>\n",
       "      <td>Virgin America</td>\n",
       "      <td>NaN</td>\n",
       "      <td>jnardino</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>@VirginAmerica it's really aggressive to blast...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-02-24 11:15:36 -0800</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Pacific Time (US &amp; Canada)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>570300817074462722</td>\n",
       "      <td>negative</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>Can't Tell</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>Virgin America</td>\n",
       "      <td>NaN</td>\n",
       "      <td>jnardino</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>@VirginAmerica and it's a really big bad thing...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-02-24 11:14:45 -0800</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Pacific Time (US &amp; Canada)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "metadata": {},
     "execution_count": 599
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 600,
   "source": [
    "# getting the texts and the categories\r\n",
    "\r\n",
    "texts=df[\"text\"].values\r\n",
    "categories=df[\"airline_sentiment\"].values"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 601,
   "source": [
    "# tokenizing each text\r\n",
    "\r\n",
    "texts=[word_tokenize(text) for text in texts]"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 602,
   "source": [
    "# making a combined documents array with each entry as a tuple of the text and its category\r\n",
    "\r\n",
    "documents=[(texts[i], categories[i]) for i in range(len(texts))]"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 603,
   "source": [
    "# getting the english stop words, punctuation marks, and numbers\r\n",
    "\r\n",
    "stop_words=stopwords.words(\"english\")\r\n",
    "\r\n",
    "punctuation_marks=string.punctuation\r\n",
    "\r\n",
    "numbers=np.arange(0, 10)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 604,
   "source": [
    "# converting the punctuation marks from string to an array \r\n",
    "\r\n",
    "punctuation_marks=[punctuation_mark for punctuation_mark in punctuation_marks]"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 605,
   "source": [
    "# adding all the punctuation marks and numbers in the stop words\r\n",
    "\r\n",
    "stop_words=np.concatenate((stop_words, punctuation_marks, numbers), axis=0)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 606,
   "source": [
    "# making a word net lemmatizer object\r\n",
    "\r\n",
    "lemmatizer=WordNetLemmatizer()"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 607,
   "source": [
    "# defining the names corresponding to the values needed by the pos argument of the lemmatize function\r\n",
    "\r\n",
    "ADJ, ADV, VERB, NOUN=('a', 's', 'v', 'n')"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 608,
   "source": [
    "# takes the pos tag and returns the value as needed by the pos argument of the lemmatize function\r\n",
    "\r\n",
    "def get_simple_pos_tag(tag) :\r\n",
    "    if(tag.startswith('J')) :\r\n",
    "        return ADJ\r\n",
    "    elif(tag.startswith('V')) :\r\n",
    "        return VERB\r\n",
    "    elif(tag.startswith('R')) :\r\n",
    "        return ADV\r\n",
    "    else :\r\n",
    "        return NOUN"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 609,
   "source": [
    "# returns a clean review of the words by removing the stop words and lemmatizing the remaining(to get to the corresponding root word)\r\n",
    "\r\n",
    "def clean_words(words) :\r\n",
    "    cleaned_words=[]\r\n",
    "\r\n",
    "    pos_tag_values=pos_tag(words)\r\n",
    "\r\n",
    "    for i in range(len(words)) :\r\n",
    "        word=words[i]\r\n",
    "\r\n",
    "        if(word.lower() not in stop_words) :\r\n",
    "            number_present=False\r\n",
    "\r\n",
    "            for char in word :\r\n",
    "                if(char.isnumeric()) :\r\n",
    "                    number_present=True\r\n",
    "\r\n",
    "                    break\r\n",
    "\r\n",
    "            if(number_present) :\r\n",
    "                continue\r\n",
    "            \r\n",
    "            root_word=lemmatizer.lemmatize(word, pos=get_simple_pos_tag(pos_tag_values[i][1]))\r\n",
    "\r\n",
    "            cleaned_words.append(root_word)\r\n",
    "\r\n",
    "    return cleaned_words"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 610,
   "source": [
    "# getting the cleaned data in the similar format like documents\r\n",
    "\r\n",
    "cleaned_data=[]\r\n",
    "\r\n",
    "for (words, category) in documents :\r\n",
    "    cleaned_data.append((clean_words(words), category))"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "# doing a random shuffle on the cleaned data(although it is cleaned already)\r\n",
    "\r\n",
    "random.shuffle(cleaned_data)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "# defining the 75% mark for making the train and test split\r\n",
    "\r\n",
    "total=len(cleaned_data)\r\n",
    "\r\n",
    "limit_75=math.floor(total*0.75)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "# splitting the data into train and test\r\n",
    "\r\n",
    "train_data=cleaned_data[: limit_75]\r\n",
    "test_data=cleaned_data[limit_75:]"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "# getting all the words in the training data\r\n",
    "\r\n",
    "all_words=[]\r\n",
    "\r\n",
    "for document in train_data :\r\n",
    "    all_words+=document[0]"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "# findind the freq of each word and then choosing the top 2000 words with the highest frequencies\r\n",
    "\r\n",
    "words_freqs=FreqDist(all_words)\r\n",
    "\r\n",
    "top_words=words_freqs.most_common(2000)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "# making the features from the top words\r\n",
    "\r\n",
    "features=[word[0] for word in top_words]"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "# returns a feature dictionary corresponding to the words which tells whether a particular feature is present in the given set of words or not\r\n",
    "\r\n",
    "def get_feature_dictionary(words) :\r\n",
    "    feature_dict={}\r\n",
    "\r\n",
    "    words_set=set(words) \r\n",
    "\r\n",
    "    for word in features :\r\n",
    "        feature_dict[word]=(word in words_set)\r\n",
    "\r\n",
    "    return feature_dict"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "# converting the train and test data into the format as required by the nltk classifiers\r\n",
    "\r\n",
    "train_data=[(get_feature_dictionary(document), category) for (document, category) in train_data]\r\n",
    "test_data=[(get_feature_dictionary(document), category) for (document, category) in test_data]"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "# making an nltk naive bayes classifier and training it\r\n",
    "\r\n",
    "classifier=NaiveBayesClassifier.train(train_data)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "# testing the classifier's on the test data\r\n",
    "\r\n",
    "nltk.classify.accuracy(classifier, test_data)"
   ],
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "0.7674863387978142"
      ]
     },
     "metadata": {},
     "execution_count": 589
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "# modifiying the cleaned data to convert it to the format as required by the vectorizer\r\n",
    "\r\n",
    "data_modified=np.array([(\" \".join(document), category)  for (document, category) in cleaned_data])"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "# splitting the data into train and test(making sure to use the same limit, and the corresponding data will be the same as it has not been shuffled again)\r\n",
    "\r\n",
    "x_train=data_modified[:, 0][: limit_75]\r\n",
    "x_test=data_modified[:, 0][limit_75: ]\r\n",
    "\r\n",
    "y_train=data_modified[:, 1][: limit_75]\r\n",
    "y_test=data_modified[:, 1][limit_75: ]"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "# making an object of the tf-idf vectorizer(since we want to give consideration to the document frequency too) and we don't consider those words which occur in less than 5 documents and more than 15% of the documents\r\n",
    "\r\n",
    "tfidf_vec=TfidfVectorizer(max_features=2000, min_df=5, max_df=0.15)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "# fitting the training data into the vectorizer and getting the transformed training data\r\n",
    "\r\n",
    "x_train_modified=tfidf_vec.fit_transform(x_train)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "# transforming the testing data\r\n",
    "\r\n",
    "x_test_modified=tfidf_vec.transform(x_test)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "# making a linear svc classifier object\r\n",
    "\r\n",
    "classifier=SVC(kernel=\"linear\")"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "# fitting the classifier with the training data and testing it on the testing data\r\n",
    "\r\n",
    "classifier.fit(x_train_modified, y_train)\r\n",
    "\r\n",
    "classifier.score(x_test_modified, y_test)"
   ],
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "0.7844262295081967"
      ]
     },
     "metadata": {},
     "execution_count": 596
    }
   ],
   "metadata": {}
  }
 ],
 "metadata": {
  "orig_nbformat": 4,
  "language_info": {
   "name": "python",
   "version": "3.8.8",
   "mimetype": "text/x-python",
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "pygments_lexer": "ipython3",
   "nbconvert_exporter": "python",
   "file_extension": ".py"
  },
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.8.8 64-bit ('base': conda)"
  },
  "interpreter": {
   "hash": "970ece05f2fd367a433285c0204e778ad1644066d163bed046b3b0abfdd35b59"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}