Twitter Hashtag Hackathon .ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Used libraries: \n",
    "    ** pandas to store extracted information to DataFrame\n",
    "    ** matplotlib and seaborn to plot the data we need to look to\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### The Returned information will be stored in JSON "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting tweepy\n",
      "  Downloading tweepy-3.5.0-py2.py3-none-any.whl\n",
      "Collecting requests-oauthlib>=0.4.1 (from tweepy)\n",
      "  Downloading requests_oauthlib-0.8.0-py2.py3-none-any.whl\n",
      "Requirement already satisfied: six>=1.7.3 in /Users/mozilla/virtualenvs/ddm-data/lib/python3.4/site-packages (from tweepy)\n",
      "Requirement already satisfied: requests>=2.4.3 in /Users/mozilla/virtualenvs/ddm-data/lib/python3.4/site-packages (from tweepy)\n",
      "Collecting oauthlib>=0.6.2 (from requests-oauthlib>=0.4.1->tweepy)\n",
      "  Downloading oauthlib-2.0.2.tar.gz (125kB)\n",
      "\u001b[K    100% |████████████████████████████████| 133kB 1.7MB/s ta 0:00:01\n",
      "\u001b[?25hBuilding wheels for collected packages: oauthlib\n",
      "  Running setup.py bdist_wheel for oauthlib ... \u001b[?25l-\b \bdone\n",
      "\u001b[?25h  Stored in directory: /Users/mozilla/Library/Caches/pip/wheels/84/98/7a/fba7268f61097bea6081cbe5480bc439b38975748ea7684fd5\n",
      "Successfully built oauthlib\n",
      "Installing collected packages: oauthlib, requests-oauthlib, tweepy\n",
      "Successfully installed oauthlib-2.0.2 requests-oauthlib-0.8.0 tweepy-3.5.0\n"
     ]
    }
   ],
   "source": [
    "!pip install tweepy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting mpltools\n",
      "  Downloading mpltools-0.2.0.tar.gz\n",
      "Requirement already satisfied: matplotlib>=1.0 in /Users/mozilla/virtualenvs/ddm-data/lib/python3.4/site-packages (from mpltools)\n",
      "Collecting configobj>=4.4.0 (from mpltools)\n",
      "  Downloading configobj-5.0.6.tar.gz\n",
      "Collecting future>=0.12.4 (from mpltools)\n",
      "  Downloading future-0.16.0.tar.gz (824kB)\n",
      "\u001b[K    100% |████████████████████████████████| 829kB 1.4MB/s ta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: six>=1.10 in /Users/mozilla/virtualenvs/ddm-data/lib/python3.4/site-packages (from matplotlib>=1.0->mpltools)\n",
      "Requirement already satisfied: python-dateutil in /Users/mozilla/virtualenvs/ddm-data/lib/python3.4/site-packages (from matplotlib>=1.0->mpltools)\n",
      "Requirement already satisfied: pytz in /Users/mozilla/virtualenvs/ddm-data/lib/python3.4/site-packages (from matplotlib>=1.0->mpltools)\n",
      "Requirement already satisfied: cycler>=0.10 in /Users/mozilla/virtualenvs/ddm-data/lib/python3.4/site-packages (from matplotlib>=1.0->mpltools)\n",
      "Requirement already satisfied: pyparsing!=2.0.0,!=2.0.4,!=2.1.2,!=2.1.6,>=1.5.6 in /Users/mozilla/virtualenvs/ddm-data/lib/python3.4/site-packages (from matplotlib>=1.0->mpltools)\n",
      "Requirement already satisfied: numpy>=1.7.1 in /Users/mozilla/virtualenvs/ddm-data/lib/python3.4/site-packages (from matplotlib>=1.0->mpltools)\n",
      "Building wheels for collected packages: mpltools, configobj, future\n",
      "  Running setup.py bdist_wheel for mpltools ... \u001b[?25l-\b \bdone\n",
      "\u001b[?25h  Stored in directory: /Users/mozilla/Library/Caches/pip/wheels/f0/a2/41/c777a80b8ad5ecb753c6141f9211d32da4c4ce5966ddd0465a\n",
      "  Running setup.py bdist_wheel for configobj ... \u001b[?25l-\b \bdone\n",
      "\u001b[?25h  Stored in directory: /Users/mozilla/Library/Caches/pip/wheels/87/76/48/1564f8466fbd36402af5ac4972ffb56a6ef7f143892ef57fe5\n",
      "  Running setup.py bdist_wheel for future ... \u001b[?25l-\b \b\\\b \b|\b \bdone\n",
      "\u001b[?25h  Stored in directory: /Users/mozilla/Library/Caches/pip/wheels/c2/50/7c/0d83b4baac4f63ff7a765bd16390d2ab43c93587fac9d6017a\n",
      "Successfully built mpltools configobj future\n",
      "Installing collected packages: configobj, future, mpltools\n",
      "Successfully installed configobj-5.0.6 future-0.16.0 mpltools-0.2.0\n"
     ]
    }
   ],
   "source": [
    "!pip install mpltools"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting imread\n",
      "  Downloading imread-0.6-cp34-cp34m-macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl (1.6MB)\n",
      "\u001b[K    100% |████████████████████████████████| 1.6MB 769kB/s ta 0:00:01\n",
      "\u001b[?25hInstalling collected packages: imread\n",
      "Successfully installed imread-0.6\n"
     ]
    }
   ],
   "source": [
    "!pip install imread"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting Pillow==2.6.0\n",
      "  Downloading Pillow-2.6.0-cp34-cp34m-macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64.whl (2.8MB)\n",
      "\u001b[K    100% |████████████████████████████████| 2.8MB 503kB/s ta 0:00:01\n",
      "\u001b[?25hInstalling collected packages: Pillow\n",
      "  Found existing installation: Pillow 4.1.1\n",
      "    Uninstalling Pillow-4.1.1:\n",
      "      Successfully uninstalled Pillow-4.1.1\n",
      "Successfully installed Pillow-2.6.0\n"
     ]
    }
   ],
   "source": [
    "!pip install Pillow==2.6.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting wordcloud\n",
      "  Downloading wordcloud-1.3.1.tar.gz (169kB)\n",
      "\u001b[K    100% |████████████████████████████████| 174kB 1.5MB/s ta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: matplotlib in /Users/mozilla/virtualenvs/ddm-data/lib/python3.4/site-packages (from wordcloud)\n",
      "Requirement already satisfied: numpy>=1.6.1 in /Users/mozilla/virtualenvs/ddm-data/lib/python3.4/site-packages (from wordcloud)\n",
      "Requirement already satisfied: pillow in /Users/mozilla/virtualenvs/ddm-data/lib/python3.4/site-packages (from wordcloud)\n",
      "Requirement already satisfied: pyparsing!=2.0.0,!=2.0.4,!=2.1.2,!=2.1.6,>=1.5.6 in /Users/mozilla/virtualenvs/ddm-data/lib/python3.4/site-packages (from matplotlib->wordcloud)\n",
      "Requirement already satisfied: python-dateutil in /Users/mozilla/virtualenvs/ddm-data/lib/python3.4/site-packages (from matplotlib->wordcloud)\n",
      "Requirement already satisfied: cycler>=0.10 in /Users/mozilla/virtualenvs/ddm-data/lib/python3.4/site-packages (from matplotlib->wordcloud)\n",
      "Requirement already satisfied: pytz in /Users/mozilla/virtualenvs/ddm-data/lib/python3.4/site-packages (from matplotlib->wordcloud)\n",
      "Requirement already satisfied: six>=1.10 in /Users/mozilla/virtualenvs/ddm-data/lib/python3.4/site-packages (from matplotlib->wordcloud)\n",
      "Building wheels for collected packages: wordcloud\n",
      "  Running setup.py bdist_wheel for wordcloud ... \u001b[?25l-\b \b\\\b \b|\b \b/\b \bdone\n",
      "\u001b[?25h  Stored in directory: /Users/mozilla/Library/Caches/pip/wheels/d9/4c/ac/e63c45f2ce09860e9459a410953039c30296e89d9f7234675f\n",
      "Successfully built wordcloud\n",
      "Installing collected packages: wordcloud\n",
      "Successfully installed wordcloud-1.3.1\n"
     ]
    }
   ],
   "source": [
    "!pip install wordcloud"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/mozilla/virtualenvs/ddm-data/lib/python3.4/site-packages/mpltools/style/__init__.py:42: FutureWarning: \n",
      "\n",
      "    The style-sheet functionality in mpltools has been integrated into\n",
      "    Matplotlib >= 1.4. This module will be removed in a future release.\n",
      "\n",
      "    Note that style-sheets used by `matplotlib.style` use the standard\n",
      "    Matplotlib rc-file syntax instead of the INI format used by `mpltools`.\n",
      "    This mostly means un-quoting strings and changing '=' to ':'.\n",
      "\n",
      "\n",
      "  \"\"\", FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "from tweepy import OAuthHandler\n",
    "from tweepy import API\n",
    "from tweepy import Cursor\n",
    "import json\n",
    "import pandas as pd\n",
    "import matplotlib as mpl\n",
    "#mpl.use('Agg')\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.ticker import MaxNLocator\n",
    "from matplotlib import rcParams\n",
    "from mpltools import style\n",
    "from matplotlib import dates\n",
    "from datetime import datetime\n",
    "import seaborn as sns\n",
    "import time\n",
    "import os\n",
    "from scipy.misc import imread\n",
    "from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator\n",
    "import random"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Plots looking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/mozilla/virtualenvs/ddm-data/lib/python3.4/site-packages/matplotlib/__init__.py:913: UserWarning: axes.color_cycle is deprecated and replaced with axes.prop_cycle; please use the latter.\n",
      "  warnings.warn(self.msg_depr % (key, alt_key))\n"
     ]
    }
   ],
   "source": [
    "# Seaborn plots\n",
    "sns.set_palette(\"deep\", desat=.6)\n",
    "sns.set_context(rc={\"figure.figsize\": (8, 4)})\n",
    "# for R lovers :)\n",
    "style.use('ggplot')\n",
    "rcParams['axes.labelsize'] = 9\n",
    "rcParams['xtick.labelsize'] = 9\n",
    "rcParams['ytick.labelsize'] = 9\n",
    "rcParams['legend.fontsize'] = 7\n",
    "# rcParams['font.family'] = 'serif'\n",
    "rcParams['font.serif'] = ['Computer Modern Roman']\n",
    "rcParams['text.usetex'] = False\n",
    "rcParams['figure.figsize'] = 20, 10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Twitter Keys API"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "access_token = \"243331145-IRZR4i65iS1mTUPeX6cLfJ3k8WNCKzLyazKPZD5g\"\n",
    "access_token_secret = \"8OHEpXqwUe2YAdGdhqNwKcktdOSQ7MBb7ksC9HEW8nfHS\"\n",
    "consumer_key = \"SXVzb0MovBsovihhBBorTLHnj\"\n",
    "consumer_secret = \"yYDGxMX2J0d5rE1TC2W7dFCvlRobj9YdmGcv44iGjcKOXGzipl\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "MAX_TWEETS = 8000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# This handles Twitter authentication and the connection to Twitter Streaming API\n",
    "auth = OAuthHandler(consumer_key, consumer_secret)\n",
    "auth.set_access_token(access_token, access_token_secret)\n",
    "api = API(auth, wait_on_rate_limit=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "data = Cursor(api.search, q='jsconfeu').items(MAX_TWEETS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tweepy.cursor.ItemIterator at 0x115367da0>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "jsconf_data = []\n",
    "# You will use this line in production instead of this\n",
    "# current_working_dir = os.path.dirname(os.path.realpath(__file__))\n",
    "current_working_dir = \"./\"\n",
    "log_tweets = current_working_dir  + str(time.time()) + '_jsconfeutweets.txt'\n",
    "with open(log_tweets, 'w') as outfile:\n",
    "    for tweet in data:\n",
    "        jsconf_data.append(json.loads(json.dumps(tweet._json)))\n",
    "        outfile.write(json.dumps(tweet._json))\n",
    "        outfile.write(\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'text': 'RT @m_p_pfeiffer: A key point we hope ppl take away from @CSSconfeu/@jsconfeu is this: \"Diversity improves your product, your team, you\". 👌…', 'favorited': False, 'created_at': 'Thu May 11 17:48:43 +0000 2017', 'retweeted_status': {'possibly_sensitive': False, 'text': 'A key point we hope ppl take away from @CSSconfeu/@jsconfeu is this: \"Diversity improves your product, your team, y… https://t.co/Gv1qO75Nd4', 'favorited': False, 'created_at': 'Sat May 06 10:52:26 +0000 2017', 'in_reply_to_status_id': None, 'in_reply_to_user_id': None, 'metadata': {'result_type': 'recent', 'iso_language_code': 'en'}, 'retweet_count': 18, 'source': '<a href=\"http://twitter.com/download/iphone\" rel=\"nofollow\">Twitter for iPhone</a>', 'lang': 'en', 'coordinates': None, 'place': None, 'geo': None, 'in_reply_to_user_id_str': None, 'in_reply_to_screen_name': None, 'is_quote_status': False, 'truncated': True, 'contributors': None, 'in_reply_to_status_id_str': None, 'id_str': '860809476215099394', 'user': {'name': 'Michael P. Pfeiffer', 'profile_sidebar_fill_color': 'DDEEF6', 'default_profile': False, 'created_at': 'Thu Apr 28 13:47:05 +0000 2011', 'verified': False, 'followers_count': 2500, 'notifications': False, 'favourites_count': 1648, 'url': 'https://t.co/7tErgFioQq', 'translator_type': 'none', 'location': '✈︎ on the go', 'id': 289345259, 'lang': 'en', 'friends_count': 760, 'protected': False, 'has_extended_profile': False, 'description': 'Lead Emoji Engineer ☕️ // Founder @CSSconfeu // 🌍 working remotely // recently visited all 7 continents in a single year 🐧', 'profile_background_color': '285C53', 'profile_image_url': 'http://pbs.twimg.com/profile_images/742278916132503552/BMHDjF4l_normal.jpg', 'profile_background_image_url_https': 'https://pbs.twimg.com/profile_background_images/378800000165347381/v1YvrRrt.png', 'utc_offset': 3600, 'profile_banner_url': 'https://pbs.twimg.com/profile_banners/289345259/1484641375', 'profile_text_color': '333333', 'following': False, 'screen_name': 'm_p_pfeiffer', 'time_zone': 'London', 'profile_link_color': '00A6C6', 'is_translator': False, 'statuses_count': 739, 'default_profile_image': False, 'follow_request_sent': False, 'geo_enabled': False, 'profile_image_url_https': 'https://pbs.twimg.com/profile_images/742278916132503552/BMHDjF4l_normal.jpg', 'id_str': '289345259', 'is_translation_enabled': False, 'contributors_enabled': False, 'profile_use_background_image': True, 'profile_background_image_url': 'http://pbs.twimg.com/profile_background_images/378800000165347381/v1YvrRrt.png', 'entities': {'description': {'urls': []}, 'url': {'urls': [{'indices': [0, 23], 'display_url': 'contains.me', 'url': 'https://t.co/7tErgFioQq', 'expanded_url': 'http://www.contains.me'}]}}, 'listed_count': 69, 'profile_background_tile': False, 'profile_sidebar_border_color': 'FFFFFF'}, 'entities': {'symbols': [], 'hashtags': [], 'user_mentions': [{'indices': [39, 49], 'name': 'CSSconf EU', 'id_str': '1433730475', 'id': 1433730475, 'screen_name': 'CSSconfeu'}, {'indices': [50, 59], 'name': 'JSConf EU', 'id_str': '64462097', 'id': 64462097, 'screen_name': 'jsconfeu'}], 'urls': [{'indices': [117, 140], 'display_url': 'twitter.com/i/web/status/8…', 'url': 'https://t.co/Gv1qO75Nd4', 'expanded_url': 'https://twitter.com/i/web/status/860809476215099394'}]}, 'retweeted': False, 'favorite_count': 39, 'id': 860809476215099394}, 'in_reply_to_user_id': None, 'metadata': {'result_type': 'recent', 'iso_language_code': 'en'}, 'retweet_count': 18, 'source': '<a href=\"http://twitter.com\" rel=\"nofollow\">Twitter Web Client</a>', 'lang': 'en', 'coordinates': None, 'in_reply_to_status_id': None, 'place': None, 'geo': None, 'in_reply_to_user_id_str': None, 'in_reply_to_screen_name': None, 'is_quote_status': False, 'truncated': False, 'contributors': None, 'in_reply_to_status_id_str': None, 'id_str': '862726173830270976', 'user': {'name': 'Anjana Menon', 'profile_sidebar_fill_color': 'EFEFEF', 'default_profile': False, 'created_at': 'Sun May 31 23:16:39 +0000 2009', 'verified': False, 'followers_count': 2635, 'notifications': False, 'favourites_count': 1143, 'url': 'https://t.co/DeB451EPvI', 'translator_type': 'none', 'location': 'India', 'id': 43777167, 'lang': 'en', 'friends_count': 787, 'protected': False, 'has_extended_profile': True, 'description': 'Wannabe globetrotter. Budding polyglot. True Blue. Music buff. @Zomato alum. Content design @PayPal. All views are mine :)', 'profile_background_color': '131516', 'profile_image_url': 'http://pbs.twimg.com/profile_images/785403178590277633/Oh68ObWN_normal.jpg', 'profile_background_image_url_https': 'https://pbs.twimg.com/profile_background_images/474079309997424640/UD5BfN1F.jpeg', 'utc_offset': 3600, 'profile_banner_url': 'https://pbs.twimg.com/profile_banners/43777167/1466312973', 'profile_text_color': '333333', 'following': False, 'screen_name': 'AnjiMenon', 'time_zone': 'London', 'profile_link_color': '04CFCF', 'is_translator': False, 'statuses_count': 18951, 'default_profile_image': False, 'follow_request_sent': False, 'geo_enabled': True, 'profile_image_url_https': 'https://pbs.twimg.com/profile_images/785403178590277633/Oh68ObWN_normal.jpg', 'id_str': '43777167', 'is_translation_enabled': False, 'contributors_enabled': False, 'profile_use_background_image': True, 'profile_background_image_url': 'http://pbs.twimg.com/profile_background_images/474079309997424640/UD5BfN1F.jpeg', 'entities': {'description': {'urls': []}, 'url': {'urls': [{'indices': [0, 23], 'display_url': 'unsplash.com/@anjimenon', 'url': 'https://t.co/DeB451EPvI', 'expanded_url': 'https://unsplash.com/@anjimenon'}]}}, 'listed_count': 58, 'profile_background_tile': True, 'profile_sidebar_border_color': 'FFFFFF'}, 'entities': {'symbols': [], 'hashtags': [], 'user_mentions': [{'indices': [3, 16], 'name': 'Michael P. Pfeiffer', 'id_str': '289345259', 'id': 289345259, 'screen_name': 'm_p_pfeiffer'}, {'indices': [57, 67], 'name': 'CSSconf EU', 'id_str': '1433730475', 'id': 1433730475, 'screen_name': 'CSSconfeu'}, {'indices': [68, 77], 'name': 'JSConf EU', 'id_str': '64462097', 'id': 64462097, 'screen_name': 'jsconfeu'}], 'urls': []}, 'retweeted': False, 'favorite_count': 0, 'id': 862726173830270976}\n"
     ]
    }
   ],
   "source": [
    "print(jsconf_data[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Create the dataframe we will use\n",
    "tweets = pd.DataFrame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# We want to know when a tweet was sent\n",
    "tweets['created_at'] = map(lambda tweet: time.strftime('%Y-%m-%d %H:%M:%S', time.strptime(tweet['created_at'],'%a %b %d %H:%M:%S +0000 %Y')), jsconf_data)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Who is the tweet owner\n",
    "tweets['user'] = map(lambda tweet: tweet['user']['name'], jsconf_data)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# How many follower this user has\n",
    "tweets['user_followers_count'] = map(lambda tweet: tweet['user']['followers_count'], jsconf_data)\n",
    "# What is the tweet's content\n",
    "tweets['text'] = map(lambda tweet: tweet['text'].encode('utf-8'), jsconf_data)\n",
    "# If available what is the language the tweet is written in\n",
    "tweets['lang'] = map(lambda tweet: tweet['lang'], jsconf_data)\n",
    "# If available, where was the tweet sent from ?\n",
    "tweets['Location'] = map(lambda tweet: tweet['place']['country'] if tweet['place'] != None else None, jsconf_data)\n",
    "# How many times this tweet was retweeted and favorited\n",
    "tweets['retweet_count'] = map(lambda tweet: tweet['retweet_count'], jsconf_data)\n",
    "tweets['favorite_count'] = map(lambda tweet: tweet['favorite_count'], jsconf_data)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                          user         user_followers_count  \\\n",
      "0  <map object at 0x13c5fb6a0>  <map object at 0x13c61d208>   \n",
      "1  <map object at 0x13c5fb6a0>  <map object at 0x13c61d208>   \n",
      "2  <map object at 0x13c5fb6a0>  <map object at 0x13c61d208>   \n",
      "3  <map object at 0x13c5fb6a0>  <map object at 0x13c61d208>   \n",
      "4  <map object at 0x13c5fb6a0>  <map object at 0x13c61d208>   \n",
      "\n",
      "                          text                         lang  \\\n",
      "0  <map object at 0x13c61d320>  <map object at 0x13c61b8d0>   \n",
      "1  <map object at 0x13c61d320>  <map object at 0x13c61b8d0>   \n",
      "2  <map object at 0x13c61d320>  <map object at 0x13c61b8d0>   \n",
      "3  <map object at 0x13c61d320>  <map object at 0x13c61b8d0>   \n",
      "4  <map object at 0x13c61d320>  <map object at 0x13c61b8d0>   \n",
      "\n",
      "                      Location                retweet_count  \\\n",
      "0  <map object at 0x13c61b860>  <map object at 0x13c61b940>   \n",
      "1  <map object at 0x13c61b860>  <map object at 0x13c61b940>   \n",
      "2  <map object at 0x13c61b860>  <map object at 0x13c61b940>   \n",
      "3  <map object at 0x13c61b860>  <map object at 0x13c61b940>   \n",
      "4  <map object at 0x13c61b860>  <map object at 0x13c61b940>   \n",
      "\n",
      "                favorite_count                   created_at  \n",
      "0  <map object at 0x13c61b7f0>  <map object at 0x13c5fb5f8>  \n",
      "1  <map object at 0x13c61b7f0>  <map object at 0x13c5fb5f8>  \n",
      "2  <map object at 0x13c61b7f0>  <map object at 0x13c5fb5f8>  \n",
      "3  <map object at 0x13c61b7f0>  <map object at 0x13c5fb5f8>  \n",
      "4  <map object at 0x13c61b7f0>  <map object at 0x13c5fb5f8>  \n"
     ]
    }
   ],
   "source": [
    "print( tweets.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}