boxmojo-cleaning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# BoxMojo Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Data wrangling\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "\n",
    "# Time\n",
    "import time\n",
    "import datetime\n",
    "from datetime import datetime as dt\n",
    "import dateutil.parser\n",
    "\n",
    "# Plotting\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "# URL Grabbing\n",
    "import requests\n",
    "\n",
    "# Scraping / Searching\n",
    "from bs4 import BeautifulSoup\n",
    "import re\n",
    "\n",
    "# Misc\n",
    "import pickle\n",
    "from pprint import pprint\n",
    "\n",
    "%matplotlib inline\n",
    "pd.set_option('display.float_format', lambda x: '%.2f' % x) # reduces the amount of decimals when printing the df's"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Inputting the Data\n",
    "\n",
    "I was having issues with the way the data was formatted using scrapy, so I just readline by line, then evaluate the JSON since it works as a python dictionary. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "filename = 'test_run.jl'\n",
    "filename_r = './bomojo_scrap/' + filename\n",
    "\n",
    "with open(filename_r, 'r') as fh:\n",
    "    fstr = fh.readlines()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# this was a test string to find an issue with the eval statement, use this if you can figure it out! :)\n",
    "# test_str = '{\"The Boxtrolls\": {\"domestic_gross_adj\": \"$52,992,000\", \"foreign_unadj\": \"\\u00a0$58,447,728\", \"worldwide_gross_unadj\": \"$109,285,033\", \"release_date\": \"September 26, 2014\", \"genre\": \"Animation\", \"runtime\": \"1 hrs. 37 min.\", \"mpaa_rating\": \"PG\", \"production_budget_adj\": \"$60 million\", \"opening_weekend_adj\": \"\\u00a0$18,493,900\", \"number_of_theaters\": \"\\u00a03,464 theaters\", \"close_date\": \"\\u00a0February 5, 2015\", \"days_in_theater\": \"\\u00a0133 days / 19 weeks\", \"directors\": [\"Graham Annable\", \"Anthony Stacchi\"], \"writers\": [\"Irena Brignull\", \"Adam Pava\"], \"actors\": [\"Ben Kingsley\", \"Toni Collette\", \"Elle Fanning\", \"Jared Harris\", \"Simon Pegg\", \"Nick Frost\", \"Richard Ayoade \", \"Tracy Morgan\"], \"producers\": [\"Travis Knight\"], \"composers\": [\"Dario Marianelli\"]}}'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Lines causing eval errors:\n",
      "\n",
      "{\"The Boxtrolls\": {\"domestic_gross_adj\": \"$52,992,000\", \"foreign_unadj\": \"\\u00a0$58,447,728\", \"worldwide_gross_unadj\": \"$109,285,033\", \"release_date\": \"September 26, 2014\", \"genre\": \"Animation\", \"runtime\": \"1 hrs. 37 min.\", \"mpaa_rating\": \"PG\", \"production_budget_adj\": \"$60 million\", \"opening_weekend_adj\": \"\\u00a0$18,493,900\", \"number_of_theaters\": \"\\u00a03,464 theaters\", \"close_date\": \"\\u00a0February 5, 2015\", \"days_in_theater\": \"\\u00a0133 days / 19 weeks\", \"directors\": [\"Graham Annable\", \"Anthony Stacchi\"], \"writers\": [\"Irena Brig\"null\"\", \"Adam Pava\"], \"actors\": [\"Ben Kingsley\", \"Toni Collette\", \"Elle Fanning\", \"Jared Harris\", \"Simon Pegg\", \"Nick Frost\", \"Richard Ayoade \", \"Tracy Morgan\"], \"producers\": [\"Travis Knight\"], \"composers\": [\"Dario Marianelli\"]}}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# grabs all the movies\n",
    "movies = fstr\n",
    "\n",
    "# replaces all 'null' strings with \"null\" because when evaluating the expression, it can't use the keyword null\n",
    "movies = [x.replace('null','\"null\"') for x in movies]\n",
    "\n",
    "# evaluates the expression to be a list of dicts!\n",
    "# movies_eval = [eval(x) for x in movies]\n",
    "movies_eval = []\n",
    "\n",
    "print('Lines causing eval errors:\\n')\n",
    "for movie_json in movies:\n",
    "    try:\n",
    "        movies_eval.append(eval(movie_json))\n",
    "    except:\n",
    "        print(movie_json)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# creates a DataFrame for each of the dictionaries in the eval and concats them all\n",
    "df = pd.concat([pd.DataFrame(x) for x in movies_eval], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# transposes the columns and the index\n",
    "df = df.T"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Checking the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['actors', 'close_date', 'composers', 'days_in_theater', 'directors',\n",
       "       'domestic_gross_adj', 'foreign_unadj', 'genre', 'mpaa_rating',\n",
       "       'number_of_theaters', 'opening_weekend_adj', 'producers',\n",
       "       'production_budget_adj', 'release_date', 'runtime',\n",
       "       'worldwide_gross_unadj', 'writers'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check columns for any problems\n",
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "actors                   9965\n",
       "close_date               9965\n",
       "composers                9965\n",
       "days_in_theater          9965\n",
       "directors                9965\n",
       "domestic_gross_adj       9965\n",
       "foreign_unadj            9965\n",
       "genre                    9965\n",
       "mpaa_rating              9965\n",
       "number_of_theaters       9965\n",
       "opening_weekend_adj      9965\n",
       "producers                9965\n",
       "production_budget_adj    9965\n",
       "release_date             9965\n",
       "runtime                  9965\n",
       "worldwide_gross_unadj    9965\n",
       "writers                  9965\n",
       "dtype: int64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# make sure all the data was imported\n",
    "df.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>The Other Side of the Mountain Part II</th>\n",
       "      <th>Universal Soldier</th>\n",
       "      <th>Newtown</th>\n",
       "      <th>Mommie Dearest</th>\n",
       "      <th>G.B.F.</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>actors</th>\n",
       "      <td>[]</td>\n",
       "      <td>[Jean-Claude Van Damme, Dolph Lundgren]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Natasha Lyonne, Megan Mullally]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>close_date</th>\n",
       "      <td>null</td>\n",
       "      <td>null</td>\n",
       "      <td>null</td>\n",
       "      <td>null</td>\n",
       "      <td>null</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>composers</th>\n",
       "      <td>[]</td>\n",
       "      <td>[Tim Simonec]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>days_in_theater</th>\n",
       "      <td>null</td>\n",
       "      <td>null</td>\n",
       "      <td>14 days / 2 weeks</td>\n",
       "      <td>null</td>\n",
       "      <td>null</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>directors</th>\n",
       "      <td>[]</td>\n",
       "      <td>[Roland Emmerich]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>domestic_gross_adj</th>\n",
       "      <td>$42,753,500</td>\n",
       "      <td>$75,661,200</td>\n",
       "      <td>$13,900</td>\n",
       "      <td>$59,219,100</td>\n",
       "      <td>null</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>foreign_unadj</th>\n",
       "      <td>null</td>\n",
       "      <td>null</td>\n",
       "      <td>null</td>\n",
       "      <td>null</td>\n",
       "      <td>null</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>genre</th>\n",
       "      <td>Romance</td>\n",
       "      <td>Sci-Fi Action</td>\n",
       "      <td>Documentary</td>\n",
       "      <td>Drama</td>\n",
       "      <td>Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mpaa_rating</th>\n",
       "      <td>PG</td>\n",
       "      <td>R</td>\n",
       "      <td>Unrated</td>\n",
       "      <td>PG</td>\n",
       "      <td>R</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>number_of_theaters</th>\n",
       "      <td>null</td>\n",
       "      <td>1,925 theaters</td>\n",
       "      <td>3 theaters</td>\n",
       "      <td>null</td>\n",
       "      <td>null</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>opening_weekend_adj</th>\n",
       "      <td>null</td>\n",
       "      <td>$20,962,400</td>\n",
       "      <td>$6,700</td>\n",
       "      <td>null</td>\n",
       "      <td>null</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>producers</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>production_budget_adj</th>\n",
       "      <td>N/A</td>\n",
       "      <td>N/A</td>\n",
       "      <td>N/A</td>\n",
       "      <td>N/A</td>\n",
       "      <td>N/A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>release_date</th>\n",
       "      <td>January 1, 1978</td>\n",
       "      <td>July 10, 1992</td>\n",
       "      <td>October 7, 2016</td>\n",
       "      <td>September 18, 1981</td>\n",
       "      <td>January 17, 2014</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>runtime</th>\n",
       "      <td>1 hrs. 40 min.</td>\n",
       "      <td>1 hrs. 42 min.</td>\n",
       "      <td>1 hrs. 25 min.</td>\n",
       "      <td>2 hrs. 9 min.</td>\n",
       "      <td>1 hrs. 33 min.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>worldwide_gross_unadj</th>\n",
       "      <td>null</td>\n",
       "      <td>null</td>\n",
       "      <td>null</td>\n",
       "      <td>null</td>\n",
       "      <td>null</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>writers</th>\n",
       "      <td>[]</td>\n",
       "      <td>[Dean Devlin]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      The Other Side of the Mountain Part II  \\\n",
       "actors                                                    []   \n",
       "close_date                                              null   \n",
       "composers                                                 []   \n",
       "days_in_theater                                         null   \n",
       "directors                                                 []   \n",
       "domestic_gross_adj                               $42,753,500   \n",
       "foreign_unadj                                           null   \n",
       "genre                                                Romance   \n",
       "mpaa_rating                                               PG   \n",
       "number_of_theaters                                      null   \n",
       "opening_weekend_adj                                     null   \n",
       "producers                                                 []   \n",
       "production_budget_adj                                    N/A   \n",
       "release_date                                 January 1, 1978   \n",
       "runtime                                       1 hrs. 40 min.   \n",
       "worldwide_gross_unadj                                   null   \n",
       "writers                                                   []   \n",
       "\n",
       "                                             Universal Soldier  \\\n",
       "actors                 [Jean-Claude Van Damme, Dolph Lundgren]   \n",
       "close_date                                                null   \n",
       "composers                                        [Tim Simonec]   \n",
       "days_in_theater                                           null   \n",
       "directors                                    [Roland Emmerich]   \n",
       "domestic_gross_adj                                 $75,661,200   \n",
       "foreign_unadj                                             null   \n",
       "genre                                            Sci-Fi Action   \n",
       "mpaa_rating                                                  R   \n",
       "number_of_theaters                              1,925 theaters   \n",
       "opening_weekend_adj                                $20,962,400   \n",
       "producers                                                   []   \n",
       "production_budget_adj                                      N/A   \n",
       "release_date                                     July 10, 1992   \n",
       "runtime                                         1 hrs. 42 min.   \n",
       "worldwide_gross_unadj                                     null   \n",
       "writers                                          [Dean Devlin]   \n",
       "\n",
       "                                  Newtown      Mommie Dearest  \\\n",
       "actors                                 []                  []   \n",
       "close_date                           null                null   \n",
       "composers                              []                  []   \n",
       "days_in_theater         14 days / 2 weeks                null   \n",
       "directors                              []                  []   \n",
       "domestic_gross_adj                $13,900         $59,219,100   \n",
       "foreign_unadj                        null                null   \n",
       "genre                         Documentary               Drama   \n",
       "mpaa_rating                       Unrated                  PG   \n",
       "number_of_theaters             3 theaters                null   \n",
       "opening_weekend_adj                $6,700                null   \n",
       "producers                              []                  []   \n",
       "production_budget_adj                 N/A                 N/A   \n",
       "release_date              October 7, 2016  September 18, 1981   \n",
       "runtime                    1 hrs. 25 min.       2 hrs. 9 min.   \n",
       "worldwide_gross_unadj                null                null   \n",
       "writers                                []                  []   \n",
       "\n",
       "                                                 G.B.F.  \n",
       "actors                 [Natasha Lyonne, Megan Mullally]  \n",
       "close_date                                         null  \n",
       "composers                                            []  \n",
       "days_in_theater                                    null  \n",
       "directors                                            []  \n",
       "domestic_gross_adj                                 null  \n",
       "foreign_unadj                                      null  \n",
       "genre                                            Comedy  \n",
       "mpaa_rating                                           R  \n",
       "number_of_theaters                                 null  \n",
       "opening_weekend_adj                                null  \n",
       "producers                                            []  \n",
       "production_budget_adj                               N/A  \n",
       "release_date                           January 17, 2014  \n",
       "runtime                                  1 hrs. 33 min.  \n",
       "worldwide_gross_unadj                              null  \n",
       "writers                                              []  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# taking a peek into the data\n",
    "df.head().T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "actors                   object\n",
       "close_date               object\n",
       "composers                object\n",
       "days_in_theater          object\n",
       "directors                object\n",
       "domestic_gross_adj       object\n",
       "foreign_unadj            object\n",
       "genre                    object\n",
       "mpaa_rating              object\n",
       "number_of_theaters       object\n",
       "opening_weekend_adj      object\n",
       "producers                object\n",
       "production_budget_adj    object\n",
       "release_date             object\n",
       "runtime                  object\n",
       "worldwide_gross_unadj    object\n",
       "writers                  object\n",
       "dtype: object"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# looks like I need to manually convert a lot of these into their respective format\n",
    "df.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>The Karate Kid</th>\n",
       "      <th>Kicking and Screaming</th>\n",
       "      <th>After Midnight</th>\n",
       "      <th>Together</th>\n",
       "      <th>Three of Hearts</th>\n",
       "      <th>When a Stranger Calls</th>\n",
       "      <th>Legend</th>\n",
       "      <th>The Animation Show</th>\n",
       "      <th>Freaky Friday</th>\n",
       "      <th>Happily Ever After</th>\n",
       "      <th>Renegades</th>\n",
       "      <th>Hero</th>\n",
       "      <th>Running Scared</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>actors</th>\n",
       "      <td>[]</td>\n",
       "      <td>[Parker Posey]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Camilla Belle, Katie Cassidy, Clark Gregg]</td>\n",
       "      <td>[Emily Browning, Christopher Eccleston, Taron ...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Lindsay Lohan, Jamie Lee Curtis]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Kiefer Sutherland, Lou Diamond Phillips]</td>\n",
       "      <td>[Donnie Yen, Jet Li, Zhang Ziyi]</td>\n",
       "      <td>[Paul Walker, Vera Farmiga]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>close_date</th>\n",
       "      <td>null</td>\n",
       "      <td>April 11, 1996</td>\n",
       "      <td>August 7, 2005</td>\n",
       "      <td>October 9, 2003</td>\n",
       "      <td>December 8, 2005</td>\n",
       "      <td>March 19, 2006</td>\n",
       "      <td>January 7, 2016</td>\n",
       "      <td>null</td>\n",
       "      <td>January 15, 2004</td>\n",
       "      <td>null</td>\n",
       "      <td>null</td>\n",
       "      <td>November 25, 2004</td>\n",
       "      <td>April 2, 2006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>composers</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Carter Burwell]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Rolfe Kent]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Michael Kamen]</td>\n",
       "      <td>[Tan Dun]</td>\n",
       "      <td>[Mark Isham]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>days_in_theater</th>\n",
       "      <td>null</td>\n",
       "      <td>null</td>\n",
       "      <td>119 days / 17 weeks</td>\n",
       "      <td>null</td>\n",
       "      <td>51 days / 7.3 weeks</td>\n",
       "      <td>49 days / 7 weeks</td>\n",
       "      <td>49 days / 7 weeks</td>\n",
       "      <td>59 days / 8.4 weeks</td>\n",
       "      <td>163 days / 23.3 weeks</td>\n",
       "      <td>null</td>\n",
       "      <td>null</td>\n",
       "      <td>null</td>\n",
       "      <td>42 days / 6 weeks</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>directors</th>\n",
       "      <td>[]</td>\n",
       "      <td>[Noah Baumbach]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Simon West]</td>\n",
       "      <td>[Brian Helgeland]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Mark S. Waters]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Zhang Yimou]</td>\n",
       "      <td>[Wayne Kramer]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>domestic_gross_adj</th>\n",
       "      <td>$233,796,000</td>\n",
       "      <td>$1,428,700</td>\n",
       "      <td>$68,900</td>\n",
       "      <td>$1,652,500</td>\n",
       "      <td>$28,900</td>\n",
       "      <td>$63,204,700</td>\n",
       "      <td>$1,862,500</td>\n",
       "      <td>$332,700</td>\n",
       "      <td>$158,113,900</td>\n",
       "      <td>$6,893,600</td>\n",
       "      <td>$19,642,600</td>\n",
       "      <td>$74,813,500</td>\n",
       "      <td>$9,053,000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>foreign_unadj</th>\n",
       "      <td>null</td>\n",
       "      <td>null</td>\n",
       "      <td>$1,525,534</td>\n",
       "      <td>$13,535,226</td>\n",
       "      <td>null</td>\n",
       "      <td>$19,106,773</td>\n",
       "      <td>$41,100,000</td>\n",
       "      <td>null</td>\n",
       "      <td>$50,616,000</td>\n",
       "      <td>null</td>\n",
       "      <td>null</td>\n",
       "      <td>$123,684,413</td>\n",
       "      <td>$2,524,892</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>genre</th>\n",
       "      <td>Sports Action</td>\n",
       "      <td>Comedy</td>\n",
       "      <td>Foreign</td>\n",
       "      <td>Foreign</td>\n",
       "      <td>Documentary</td>\n",
       "      <td>Horror Thriller</td>\n",
       "      <td>Crime Drama</td>\n",
       "      <td>Animation</td>\n",
       "      <td>Fantasy Comedy</td>\n",
       "      <td>Animation</td>\n",
       "      <td>Action Comedy</td>\n",
       "      <td>Foreign / Action</td>\n",
       "      <td>Action / Crime</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mpaa_rating</th>\n",
       "      <td>PG</td>\n",
       "      <td>R</td>\n",
       "      <td>Unrated</td>\n",
       "      <td>PG</td>\n",
       "      <td>Unrated</td>\n",
       "      <td>PG-13</td>\n",
       "      <td>R</td>\n",
       "      <td>Unrated</td>\n",
       "      <td>PG</td>\n",
       "      <td>G</td>\n",
       "      <td>R</td>\n",
       "      <td>PG-13</td>\n",
       "      <td>R</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>number_of_theaters</th>\n",
       "      <td>1,111 theaters</td>\n",
       "      <td>26 theaters</td>\n",
       "      <td>3 theaters</td>\n",
       "      <td>47 theaters</td>\n",
       "      <td>1 theaters</td>\n",
       "      <td>3,004 theaters</td>\n",
       "      <td>107 theaters</td>\n",
       "      <td>6 theaters</td>\n",
       "      <td>3,067 theaters</td>\n",
       "      <td>1,018 theaters</td>\n",
       "      <td>1,627 theaters</td>\n",
       "      <td>2,175 theaters</td>\n",
       "      <td>1,611 theaters</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>opening_weekend_adj</th>\n",
       "      <td>$12,953,800</td>\n",
       "      <td>null</td>\n",
       "      <td>$10,600</td>\n",
       "      <td>$99,300</td>\n",
       "      <td>$6,000</td>\n",
       "      <td>$28,534,700</td>\n",
       "      <td>$86,300</td>\n",
       "      <td>null</td>\n",
       "      <td>$31,850,100</td>\n",
       "      <td>$3,669,000</td>\n",
       "      <td>$6,700,000</td>\n",
       "      <td>$25,078,500</td>\n",
       "      <td>$4,466,300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>producers</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[John Davis, Wyck Godfrey]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Ted Field, Joe Roth]</td>\n",
       "      <td>[Quentin Tarantino]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>production_budget_adj</th>\n",
       "      <td>N/A</td>\n",
       "      <td>N/A</td>\n",
       "      <td>N/A</td>\n",
       "      <td>N/A</td>\n",
       "      <td>N/A</td>\n",
       "      <td>$15 million</td>\n",
       "      <td>N/A</td>\n",
       "      <td>N/A</td>\n",
       "      <td>$20 million</td>\n",
       "      <td>N/A</td>\n",
       "      <td>N/A</td>\n",
       "      <td>$31 million</td>\n",
       "      <td>$15 million</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>release_date</th>\n",
       "      <td>June 22, 1984</td>\n",
       "      <td>October 6, 1995</td>\n",
       "      <td>December 3, 2004</td>\n",
       "      <td>May 30, 2003</td>\n",
       "      <td>October 19, 2005</td>\n",
       "      <td>February 3, 2006</td>\n",
       "      <td>November 20, 2015</td>\n",
       "      <td>February 18, 2005</td>\n",
       "      <td>August 6, 2003</td>\n",
       "      <td>May 28, 1993</td>\n",
       "      <td>June 2, 1989</td>\n",
       "      <td>August 27, 2004</td>\n",
       "      <td>February 24, 2006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>runtime</th>\n",
       "      <td>2 hrs. 7 min.</td>\n",
       "      <td>1 hrs. 36 min.</td>\n",
       "      <td>1 hrs. 29 min.</td>\n",
       "      <td>1 hrs. 56 min.</td>\n",
       "      <td>1 hrs. 37 min.</td>\n",
       "      <td>1 hrs. 40 min.</td>\n",
       "      <td>2 hrs. 11 min.</td>\n",
       "      <td>1 hrs. 25 min.</td>\n",
       "      <td>1 hrs. 37 min.</td>\n",
       "      <td>1 hrs. 14 min.</td>\n",
       "      <td>1 hrs. 46 min.</td>\n",
       "      <td>1 hrs. 39 min.</td>\n",
       "      <td>1 hrs. 50 min.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>worldwide_gross_unadj</th>\n",
       "      <td>null</td>\n",
       "      <td>null</td>\n",
       "      <td>$1,576,003</td>\n",
       "      <td>$14,687,167</td>\n",
       "      <td>null</td>\n",
       "      <td>$66,966,987</td>\n",
       "      <td>$42,972,994</td>\n",
       "      <td>null</td>\n",
       "      <td>$160,846,332</td>\n",
       "      <td>null</td>\n",
       "      <td>null</td>\n",
       "      <td>$177,394,432</td>\n",
       "      <td>$9,380,029</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>writers</th>\n",
       "      <td>[Robert Mark Kamen]</td>\n",
       "      <td>[Noah Baumbach]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Brian Helgeland]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Leslie Dixon, Heather Hach]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Wayne Kramer]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            The Karate Kid Kicking and Screaming  \\\n",
       "actors                                  []        [Parker Posey]   \n",
       "close_date                            null        April 11, 1996   \n",
       "composers                               []                    []   \n",
       "days_in_theater                       null                  null   \n",
       "directors                               []       [Noah Baumbach]   \n",
       "domestic_gross_adj            $233,796,000            $1,428,700   \n",
       "foreign_unadj                         null                  null   \n",
       "genre                        Sports Action                Comedy   \n",
       "mpaa_rating                             PG                     R   \n",
       "number_of_theaters          1,111 theaters           26 theaters   \n",
       "opening_weekend_adj            $12,953,800                  null   \n",
       "producers                               []                    []   \n",
       "production_budget_adj                  N/A                   N/A   \n",
       "release_date                 June 22, 1984       October 6, 1995   \n",
       "runtime                      2 hrs. 7 min.        1 hrs. 36 min.   \n",
       "worldwide_gross_unadj                 null                  null   \n",
       "writers                [Robert Mark Kamen]       [Noah Baumbach]   \n",
       "\n",
       "                             After Midnight          Together  \\\n",
       "actors                                   []                []   \n",
       "close_date                   August 7, 2005   October 9, 2003   \n",
       "composers                                []                []   \n",
       "days_in_theater         119 days / 17 weeks              null   \n",
       "directors                                []                []   \n",
       "domestic_gross_adj                  $68,900        $1,652,500   \n",
       "foreign_unadj                    $1,525,534       $13,535,226   \n",
       "genre                               Foreign           Foreign   \n",
       "mpaa_rating                         Unrated                PG   \n",
       "number_of_theaters               3 theaters       47 theaters   \n",
       "opening_weekend_adj                 $10,600           $99,300   \n",
       "producers                                []                []   \n",
       "production_budget_adj                   N/A               N/A   \n",
       "release_date               December 3, 2004      May 30, 2003   \n",
       "runtime                      1 hrs. 29 min.    1 hrs. 56 min.   \n",
       "worldwide_gross_unadj            $1,576,003       $14,687,167   \n",
       "writers                                  []                []   \n",
       "\n",
       "                            Three of Hearts  \\\n",
       "actors                                   []   \n",
       "close_date                 December 8, 2005   \n",
       "composers                                []   \n",
       "days_in_theater         51 days / 7.3 weeks   \n",
       "directors                                []   \n",
       "domestic_gross_adj                  $28,900   \n",
       "foreign_unadj                          null   \n",
       "genre                           Documentary   \n",
       "mpaa_rating                         Unrated   \n",
       "number_of_theaters               1 theaters   \n",
       "opening_weekend_adj                  $6,000   \n",
       "producers                                []   \n",
       "production_budget_adj                   N/A   \n",
       "release_date               October 19, 2005   \n",
       "runtime                      1 hrs. 37 min.   \n",
       "worldwide_gross_unadj                  null   \n",
       "writers                                  []   \n",
       "\n",
       "                                             When a Stranger Calls  \\\n",
       "actors                 [Camilla Belle, Katie Cassidy, Clark Gregg]   \n",
       "close_date                                          March 19, 2006   \n",
       "composers                                                       []   \n",
       "days_in_theater                                  49 days / 7 weeks   \n",
       "directors                                             [Simon West]   \n",
       "domestic_gross_adj                                     $63,204,700   \n",
       "foreign_unadj                                          $19,106,773   \n",
       "genre                                              Horror Thriller   \n",
       "mpaa_rating                                                  PG-13   \n",
       "number_of_theaters                                  3,004 theaters   \n",
       "opening_weekend_adj                                    $28,534,700   \n",
       "producers                               [John Davis, Wyck Godfrey]   \n",
       "production_budget_adj                                  $15 million   \n",
       "release_date                                      February 3, 2006   \n",
       "runtime                                             1 hrs. 40 min.   \n",
       "worldwide_gross_unadj                                  $66,966,987   \n",
       "writers                                                         []   \n",
       "\n",
       "                                                                  Legend  \\\n",
       "actors                 [Emily Browning, Christopher Eccleston, Taron ...   \n",
       "close_date                                               January 7, 2016   \n",
       "composers                                               [Carter Burwell]   \n",
       "days_in_theater                                        49 days / 7 weeks   \n",
       "directors                                              [Brian Helgeland]   \n",
       "domestic_gross_adj                                            $1,862,500   \n",
       "foreign_unadj                                                $41,100,000   \n",
       "genre                                                        Crime Drama   \n",
       "mpaa_rating                                                            R   \n",
       "number_of_theaters                                          107 theaters   \n",
       "opening_weekend_adj                                              $86,300   \n",
       "producers                                                             []   \n",
       "production_budget_adj                                                N/A   \n",
       "release_date                                           November 20, 2015   \n",
       "runtime                                                   2 hrs. 11 min.   \n",
       "worldwide_gross_unadj                                        $42,972,994   \n",
       "writers                                                [Brian Helgeland]   \n",
       "\n",
       "                         The Animation Show  \\\n",
       "actors                                   []   \n",
       "close_date                             null   \n",
       "composers                                []   \n",
       "days_in_theater         59 days / 8.4 weeks   \n",
       "directors                                []   \n",
       "domestic_gross_adj                 $332,700   \n",
       "foreign_unadj                          null   \n",
       "genre                             Animation   \n",
       "mpaa_rating                         Unrated   \n",
       "number_of_theaters               6 theaters   \n",
       "opening_weekend_adj                    null   \n",
       "producers                                []   \n",
       "production_budget_adj                   N/A   \n",
       "release_date              February 18, 2005   \n",
       "runtime                      1 hrs. 25 min.   \n",
       "worldwide_gross_unadj                  null   \n",
       "writers                                  []   \n",
       "\n",
       "                                           Freaky Friday Happily Ever After  \\\n",
       "actors                 [Lindsay Lohan, Jamie Lee Curtis]                 []   \n",
       "close_date                              January 15, 2004               null   \n",
       "composers                                   [Rolfe Kent]                 []   \n",
       "days_in_theater                    163 days / 23.3 weeks               null   \n",
       "directors                               [Mark S. Waters]                 []   \n",
       "domestic_gross_adj                          $158,113,900         $6,893,600   \n",
       "foreign_unadj                                $50,616,000               null   \n",
       "genre                                     Fantasy Comedy          Animation   \n",
       "mpaa_rating                                           PG                  G   \n",
       "number_of_theaters                        3,067 theaters     1,018 theaters   \n",
       "opening_weekend_adj                          $31,850,100         $3,669,000   \n",
       "producers                                             []                 []   \n",
       "production_budget_adj                        $20 million                N/A   \n",
       "release_date                              August 6, 2003       May 28, 1993   \n",
       "runtime                                   1 hrs. 37 min.     1 hrs. 14 min.   \n",
       "worldwide_gross_unadj                       $160,846,332               null   \n",
       "writers                     [Leslie Dixon, Heather Hach]                 []   \n",
       "\n",
       "                                                       Renegades  \\\n",
       "actors                 [Kiefer Sutherland, Lou Diamond Phillips]   \n",
       "close_date                                                  null   \n",
       "composers                                        [Michael Kamen]   \n",
       "days_in_theater                                             null   \n",
       "directors                                                     []   \n",
       "domestic_gross_adj                                   $19,642,600   \n",
       "foreign_unadj                                               null   \n",
       "genre                                              Action Comedy   \n",
       "mpaa_rating                                                    R   \n",
       "number_of_theaters                                1,627 theaters   \n",
       "opening_weekend_adj                                   $6,700,000   \n",
       "producers                                  [Ted Field, Joe Roth]   \n",
       "production_budget_adj                                        N/A   \n",
       "release_date                                        June 2, 1989   \n",
       "runtime                                           1 hrs. 46 min.   \n",
       "worldwide_gross_unadj                                       null   \n",
       "writers                                                       []   \n",
       "\n",
       "                                                   Hero  \\\n",
       "actors                 [Donnie Yen, Jet Li, Zhang Ziyi]   \n",
       "close_date                            November 25, 2004   \n",
       "composers                                     [Tan Dun]   \n",
       "days_in_theater                                    null   \n",
       "directors                                 [Zhang Yimou]   \n",
       "domestic_gross_adj                          $74,813,500   \n",
       "foreign_unadj                              $123,684,413   \n",
       "genre                                  Foreign / Action   \n",
       "mpaa_rating                                       PG-13   \n",
       "number_of_theaters                       2,175 theaters   \n",
       "opening_weekend_adj                         $25,078,500   \n",
       "producers                           [Quentin Tarantino]   \n",
       "production_budget_adj                       $31 million   \n",
       "release_date                            August 27, 2004   \n",
       "runtime                                  1 hrs. 39 min.   \n",
       "worldwide_gross_unadj                      $177,394,432   \n",
       "writers                                              []   \n",
       "\n",
       "                                    Running Scared  \n",
       "actors                 [Paul Walker, Vera Farmiga]  \n",
       "close_date                           April 2, 2006  \n",
       "composers                             [Mark Isham]  \n",
       "days_in_theater                  42 days / 6 weeks  \n",
       "directors                           [Wayne Kramer]  \n",
       "domestic_gross_adj                      $9,053,000  \n",
       "foreign_unadj                           $2,524,892  \n",
       "genre                               Action / Crime  \n",
       "mpaa_rating                                      R  \n",
       "number_of_theaters                  1,611 theaters  \n",
       "opening_weekend_adj                     $4,466,300  \n",
       "producers                                       []  \n",
       "production_budget_adj                  $15 million  \n",
       "release_date                     February 24, 2006  \n",
       "runtime                             1 hrs. 50 min.  \n",
       "worldwide_gross_unadj                   $9,380,029  \n",
       "writers                             [Wayne Kramer]  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# looks like I have a lot of duplicated indexes, so will just reset index to handle\n",
    "    df[df.index.duplicated()].T"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Cleaning the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# reset the index\n",
    "df = df.reset_index()\n",
    "\n",
    "# Replaces all unknown values\n",
    "df.replace('null', np.nan, inplace=True)\n",
    "df.replace('N/A', np.nan, inplace=True)\n",
    "df['genre'].replace('Unknown', np.nan, inplace=True)\n",
    "df['mpaa_rating'].replace('Unknown', np.nan, inplace=True)\n",
    "df['release_date'].replace('TBD', np.nan, inplace=True)\n",
    "df['foreign_unadj'].replace('n/a', np.nan, inplace=True)\n",
    "\n",
    "# remove things like '2018'\n",
    "df['release_date'].replace(re.compile(r'^\\d+$'), np.nan, inplace=True)\n",
    "\n",
    "# remove things like 'July 1994'\n",
    "df['release_date'].replace(re.compile(r'^^\\w+\\s+\\d+$'), np.nan, inplace=True)\n",
    "\n",
    "# removes encoded characters \n",
    "# .replace('\\\\xa0', '', inplace=True) didn't work as well as passing in the argument re.compile(r'^\\\\xa0')\n",
    "df['close_date'] = df['close_date'].str.strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Removes all empty arrays.\n",
    "# **This will break if done twice!\n",
    "df['actors'] = df['actors'].apply(lambda x: np.nan if len(x)==0 else x)\n",
    "df['composers'] = df['composers'].apply(lambda x: np.nan if len(x)==0 else x)\n",
    "df['directors'] = df['directors'].apply(lambda x: np.nan if len(x)==0 else x)\n",
    "df['producers'] = df['producers'].apply(lambda x: np.nan if len(x)==0 else x)\n",
    "df['writers'] = df['writers'].apply(lambda x: np.nan if len(x)==0 else x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>actors</th>\n",
       "      <th>close_date</th>\n",
       "      <th>composers</th>\n",
       "      <th>days_in_theater</th>\n",
       "      <th>directors</th>\n",
       "      <th>domestic_gross_adj</th>\n",
       "      <th>foreign_unadj</th>\n",
       "      <th>genre</th>\n",
       "      <th>mpaa_rating</th>\n",
       "      <th>number_of_theaters</th>\n",
       "      <th>opening_weekend_adj</th>\n",
       "      <th>producers</th>\n",
       "      <th>production_budget_adj</th>\n",
       "      <th>release_date</th>\n",
       "      <th>runtime</th>\n",
       "      <th>worldwide_gross_unadj</th>\n",
       "      <th>writers</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [index, actors, close_date, composers, days_in_theater, directors, domestic_gross_adj, foreign_unadj, genre, mpaa_rating, number_of_theaters, opening_weekend_adj, producers, production_budget_adj, release_date, runtime, worldwide_gross_unadj, writers]\n",
       "Index: []"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Checking encoded characters are removed\n",
    "mask = df['close_date'] == '\\xa0December 13, 2007'\n",
    "df[mask]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Transformation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# changing the release date to a datetime\n",
    "df['release_date'] = pd.to_datetime(df['release_date'], format=\"%B %d, %Y\");\n",
    "df['close_date'] = pd.to_datetime(df['close_date'], format=\"%B %d, %Y\");"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# remove everything after days and the whitespace before it\n",
    "df['days_in_theater'] = df['days_in_theater'].replace(re.compile('\\sdays.*'), '')\n",
    "\n",
    "# kill the commas\n",
    "df['days_in_theater'] = df['days_in_theater'].replace(re.compile(','), '')\n",
    "\n",
    "# strip any whitespace\n",
    "df['days_in_theater'] = df['days_in_theater'].str.strip()\n",
    "\n",
    "# convert values to ints\n",
    "# df['days_in_theater'] = df[df['days_in_theater'].notnull()]['days_in_theater'].apply(lambda days: int(days))\n",
    "df['days_in_theater'] = pd.to_numeric(df['days_in_theater'], errors='coerce', downcast='integer')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# remove the dollar signs\n",
    "df['domestic_gross_adj'] = df['domestic_gross_adj'].replace(re.compile('\\$'), '')\n",
    "\n",
    "# remove the commas\n",
    "df['domestic_gross_adj'] = df['domestic_gross_adj'].replace(re.compile(','), '')\n",
    "\n",
    "# kill any encoded whitespace, or whitespace\n",
    "df['domestic_gross_adj'] = df['domestic_gross_adj'].str.strip()\n",
    "\n",
    "# convert to int\n",
    "# df['domestic_gross_adj'] = df[df['domestic_gross_adj'].notnull()]['domestic_gross_adj'].apply(lambda days: int(days))\n",
    "df['domestic_gross_adj'] = pd.to_numeric(df['domestic_gross_adj'], errors='coerce')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# remove the dollar signs\n",
    "df['foreign_unadj'] = df['foreign_unadj'].replace(re.compile('\\$'), '')\n",
    "\n",
    "# remove the commas\n",
    "df['foreign_unadj'] = df['foreign_unadj'].replace(re.compile(','), '')\n",
    "\n",
    "# kill any encoded whitespace, or whitespace\n",
    "df['foreign_unadj'] = df['foreign_unadj'].str.strip()\n",
    "\n",
    "# convert to int\n",
    "# df['foreign_unadj'] = df[df['foreign_unadj'].notnull()]['foreign_unadj'].apply(lambda days: int(days))\n",
    "df['foreign_unadj'] = pd.to_numeric(df['foreign_unadj'], errors='coerce')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df['number_of_theaters'] = df['number_of_theaters'].replace(re.compile(r'\\stheaters'), '')\n",
    "df['number_of_theaters'] = df['number_of_theaters'].replace(re.compile(','), '')\n",
    "df['number_of_theaters'] = df['number_of_theaters'].str.strip()\n",
    "# df['number_of_theaters'] = df[df['number_of_theaters'].notnull()]['number_of_theaters'].apply(lambda days: int(days)) \n",
    "df['number_of_theaters'] = pd.to_numeric(df['number_of_theaters'], errors='coerce')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df['opening_weekend_adj'] = df['opening_weekend_adj'].replace(re.compile('\\$'), '')\n",
    "df['opening_weekend_adj'] = df['opening_weekend_adj'].replace(re.compile(','), '')\n",
    "df['opening_weekend_adj'] = df['opening_weekend_adj'].str.strip()\n",
    "df['opening_weekend_adj'] = pd.to_numeric(df['opening_weekend_adj'], errors='coerce')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# # prototyping the millionOrThousand function\n",
    "\n",
    "# string = '1223'\n",
    "# million = re.compile(r'(\\d+)\\s(million)')\n",
    "# thousand = re.compile(r'(^\\d+$)')\n",
    "\n",
    "# millions = re.findall(million, string)\n",
    "# thousands = re.findall(thousand, string)\n",
    "\n",
    "# if millions:\n",
    "#     print('million')\n",
    "# if thousands:\n",
    "#     print('thousand')\n",
    "    \n",
    "# thousands[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def millionOrThousand(string):\n",
    "    '''Returns a integer \n",
    "    '''\n",
    "    strng = str(string)\n",
    "    million = re.compile(r'(\\d+)\\s(million)')\n",
    "    thousand = re.compile(r'(^\\d+$)')\n",
    "    \n",
    "    thousands = re.findall(thousand, strng)\n",
    "    millions = re.findall(million, strng)\n",
    "    \n",
    "    if millions:\n",
    "        return int(float(millions[0][0]) * 1000000)\n",
    "    elif thousands:\n",
    "        return int(thousands[0])\n",
    "\n",
    "# # production budget -> integer\n",
    "df['production_budget_adj'] = df['production_budget_adj'].replace(re.compile('\\$'), '')\n",
    "df['production_budget_adj'] = df['production_budget_adj'].replace(re.compile(','), '')\n",
    "df['production_budget_adj'] = df['production_budget_adj'].str.strip()\n",
    "df['production_budget_adj'] = (df['production_budget_adj'].dropna().apply(millionOrThousand).astype(np.int))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def hoursAndMins(string):\n",
    "    string = str(string)\n",
    "    r_hrsmins = re.compile(r'(^\\d+)\\shrs\\.\\s(\\d+)')\n",
    "    hrsmins = r_hrsmins.findall(string)\n",
    "\n",
    "    if hrsmins:\n",
    "        hrs = int(hrsmins[0][0]) * 60\n",
    "        mins = int(hrsmins[0][1])\n",
    "        return int(hrs + mins)\n",
    "    else:\n",
    "        return np.nan\n",
    "    \n",
    "df['runtime'] = df['runtime'].apply(hoursAndMins)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# worldwide gross -> integer\n",
    "\n",
    "# remove the dollar signs\n",
    "df['worldwide_gross_unadj'] = df['worldwide_gross_unadj'].replace(re.compile('\\$'), '')\n",
    "\n",
    "# remove the commas\n",
    "df['worldwide_gross_unadj'] = df['worldwide_gross_unadj'].replace(re.compile(','), '')\n",
    "\n",
    "# kill any encoded whitespace, or whitespace\n",
    "df['worldwide_gross_unadj'] = df['worldwide_gross_unadj'].str.strip()\n",
    "\n",
    "# convert to int\n",
    "# df['domestic_gross_adj'] = df[df['domestic_gross_adj'].notnull()]['domestic_gross_adj'].apply(lambda days: int(days))\n",
    "df['worldwide_gross_unadj'] = pd.to_numeric(df['worldwide_gross_unadj'], errors='coerce')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Confirming the data looks good"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['index', 'actors', 'close_date', 'composers', 'days_in_theater',\n",
       "       'directors', 'domestic_gross_adj', 'foreign_unadj', 'genre',\n",
       "       'mpaa_rating', 'number_of_theaters', 'opening_weekend_adj', 'producers',\n",
       "       'production_budget_adj', 'release_date', 'runtime',\n",
       "       'worldwide_gross_unadj', 'writers'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check columns for any problems\n",
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "index                            object\n",
       "actors                           object\n",
       "close_date               datetime64[ns]\n",
       "composers                        object\n",
       "days_in_theater                 float64\n",
       "directors                        object\n",
       "domestic_gross_adj              float64\n",
       "foreign_unadj                   float64\n",
       "genre                            object\n",
       "mpaa_rating                      object\n",
       "number_of_theaters              float64\n",
       "opening_weekend_adj             float64\n",
       "producers                        object\n",
       "production_budget_adj           float64\n",
       "release_date             datetime64[ns]\n",
       "runtime                         float64\n",
       "worldwide_gross_unadj           float64\n",
       "writers                          object\n",
       "dtype: object"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>days_in_theater</th>\n",
       "      <th>domestic_gross_adj</th>\n",
       "      <th>foreign_unadj</th>\n",
       "      <th>number_of_theaters</th>\n",
       "      <th>opening_weekend_adj</th>\n",
       "      <th>production_budget_adj</th>\n",
       "      <th>runtime</th>\n",
       "      <th>worldwide_gross_unadj</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>5745.00</td>\n",
       "      <td>9470.00</td>\n",
       "      <td>4047.00</td>\n",
       "      <td>9086.00</td>\n",
       "      <td>8417.00</td>\n",
       "      <td>2419.00</td>\n",
       "      <td>9557.00</td>\n",
       "      <td>5080.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>85.60</td>\n",
       "      <td>36773478.15</td>\n",
       "      <td>60885921.46</td>\n",
       "      <td>958.04</td>\n",
       "      <td>9238846.09</td>\n",
       "      <td>48340876.05</td>\n",
       "      <td>104.52</td>\n",
       "      <td>86222892.04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>103.56</td>\n",
       "      <td>77886268.39</td>\n",
       "      <td>119104852.31</td>\n",
       "      <td>1217.06</td>\n",
       "      <td>19096666.27</td>\n",
       "      <td>47608964.42</td>\n",
       "      <td>22.05</td>\n",
       "      <td>170096362.46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>-168.00</td>\n",
       "      <td>100.00</td>\n",
       "      <td>29.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>100.00</td>\n",
       "      <td>220.00</td>\n",
       "      <td>37.00</td>\n",
       "      <td>29.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>28.00</td>\n",
       "      <td>115525.00</td>\n",
       "      <td>2481470.50</td>\n",
       "      <td>9.00</td>\n",
       "      <td>16000.00</td>\n",
       "      <td>15000000.00</td>\n",
       "      <td>91.00</td>\n",
       "      <td>1231547.25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>70.00</td>\n",
       "      <td>2838650.00</td>\n",
       "      <td>16002944.00</td>\n",
       "      <td>129.00</td>\n",
       "      <td>228700.00</td>\n",
       "      <td>32000000.00</td>\n",
       "      <td>100.00</td>\n",
       "      <td>19594619.50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>119.00</td>\n",
       "      <td>40450025.00</td>\n",
       "      <td>64375453.00</td>\n",
       "      <td>1862.75</td>\n",
       "      <td>11485800.00</td>\n",
       "      <td>68000000.00</td>\n",
       "      <td>113.00</td>\n",
       "      <td>96617784.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>3934.00</td>\n",
       "      <td>1234649200.00</td>\n",
       "      <td>2027457462.00</td>\n",
       "      <td>4468.00</td>\n",
       "      <td>246541600.00</td>\n",
       "      <td>300000000.00</td>\n",
       "      <td>583.00</td>\n",
       "      <td>2787965087.00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       days_in_theater  domestic_gross_adj  foreign_unadj  number_of_theaters  \\\n",
       "count          5745.00             9470.00        4047.00             9086.00   \n",
       "mean             85.60         36773478.15    60885921.46              958.04   \n",
       "std             103.56         77886268.39   119104852.31             1217.06   \n",
       "min            -168.00              100.00          29.00                1.00   \n",
       "25%              28.00           115525.00     2481470.50                9.00   \n",
       "50%              70.00          2838650.00    16002944.00              129.00   \n",
       "75%             119.00         40450025.00    64375453.00             1862.75   \n",
       "max            3934.00       1234649200.00  2027457462.00             4468.00   \n",
       "\n",
       "       opening_weekend_adj  production_budget_adj  runtime  \\\n",
       "count              8417.00                2419.00  9557.00   \n",
       "mean            9238846.09            48340876.05   104.52   \n",
       "std            19096666.27            47608964.42    22.05   \n",
       "min                 100.00                 220.00    37.00   \n",
       "25%               16000.00            15000000.00    91.00   \n",
       "50%              228700.00            32000000.00   100.00   \n",
       "75%            11485800.00            68000000.00   113.00   \n",
       "max           246541600.00           300000000.00   583.00   \n",
       "\n",
       "       worldwide_gross_unadj  \n",
       "count                5080.00  \n",
       "mean             86222892.04  \n",
       "std             170096362.46  \n",
       "min                    29.00  \n",
       "25%               1231547.25  \n",
       "50%              19594619.50  \n",
       "75%              96617784.00  \n",
       "max            2787965087.00  "
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>close_date</th>\n",
       "      <th>release_date</th>\n",
       "      <th>genre</th>\n",
       "      <th>mpaa_rating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>5982</td>\n",
       "      <td>9829</td>\n",
       "      <td>9308</td>\n",
       "      <td>9927</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
       "      <td>1691</td>\n",
       "      <td>2778</td>\n",
       "      <td>65</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>top</th>\n",
       "      <td>2011-09-29 00:00:00</td>\n",
       "      <td>2013-09-06 00:00:00</td>\n",
       "      <td>Foreign</td>\n",
       "      <td>R</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>freq</th>\n",
       "      <td>20</td>\n",
       "      <td>19</td>\n",
       "      <td>1619</td>\n",
       "      <td>3217</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first</th>\n",
       "      <td>1982-03-18 00:00:00</td>\n",
       "      <td>1921-03-06 00:00:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>last</th>\n",
       "      <td>2017-04-06 00:00:00</td>\n",
       "      <td>2023-06-30 00:00:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 close_date         release_date    genre mpaa_rating\n",
       "count                  5982                 9829     9308        9927\n",
       "unique                 1691                 2778       65           8\n",
       "top     2011-09-29 00:00:00  2013-09-06 00:00:00  Foreign           R\n",
       "freq                     20                   19     1619        3217\n",
       "first   1982-03-18 00:00:00  1921-03-06 00:00:00      NaN         NaN\n",
       "last    2017-04-06 00:00:00  2023-06-30 00:00:00      NaN         NaN"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[['close_date', 'release_date', 'genre', 'mpaa_rating']].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>actors</th>\n",
       "      <th>close_date</th>\n",
       "      <th>composers</th>\n",
       "      <th>days_in_theater</th>\n",
       "      <th>directors</th>\n",
       "      <th>domestic_gross_adj</th>\n",
       "      <th>foreign_unadj</th>\n",
       "      <th>genre</th>\n",
       "      <th>mpaa_rating</th>\n",
       "      <th>number_of_theaters</th>\n",
       "      <th>opening_weekend_adj</th>\n",
       "      <th>producers</th>\n",
       "      <th>production_budget_adj</th>\n",
       "      <th>release_date</th>\n",
       "      <th>runtime</th>\n",
       "      <th>worldwide_gross_unadj</th>\n",
       "      <th>writers</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8577</th>\n",
       "      <td>Star Wars</td>\n",
       "      <td>[Kenny Baker, Anthony Daniels, Alec Guinness, ...</td>\n",
       "      <td>NaT</td>\n",
       "      <td>[John Williams]</td>\n",
       "      <td>nan</td>\n",
       "      <td>[George Lucas]</td>\n",
       "      <td>1234649200.00</td>\n",
       "      <td>314400000.00</td>\n",
       "      <td>Sci-Fi Fantasy</td>\n",
       "      <td>PG</td>\n",
       "      <td>1750.00</td>\n",
       "      <td>nan</td>\n",
       "      <td>[George Lucas]</td>\n",
       "      <td>11000000.00</td>\n",
       "      <td>1977-05-25</td>\n",
       "      <td>121.00</td>\n",
       "      <td>775398007.00</td>\n",
       "      <td>[George Lucas]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          index                                             actors close_date  \\\n",
       "8577  Star Wars  [Kenny Baker, Anthony Daniels, Alec Guinness, ...        NaT   \n",
       "\n",
       "            composers  days_in_theater       directors  domestic_gross_adj  \\\n",
       "8577  [John Williams]              nan  [George Lucas]       1234649200.00   \n",
       "\n",
       "      foreign_unadj           genre mpaa_rating  number_of_theaters  \\\n",
       "8577   314400000.00  Sci-Fi Fantasy          PG             1750.00   \n",
       "\n",
       "      opening_weekend_adj       producers  production_budget_adj release_date  \\\n",
       "8577                  nan  [George Lucas]            11000000.00   1977-05-25   \n",
       "\n",
       "      runtime  worldwide_gross_unadj         writers  \n",
       "8577   121.00           775398007.00  [George Lucas]  "
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# the original star wars film\n",
    "# http://www.boxofficemojo.com/movies/?id=starwars4.htm&adjust_yr=2017&p=.htm\n",
    "df[df['domestic_gross_adj'] == 1234649200.00]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>actors</th>\n",
       "      <th>close_date</th>\n",
       "      <th>composers</th>\n",
       "      <th>days_in_theater</th>\n",
       "      <th>directors</th>\n",
       "      <th>domestic_gross_adj</th>\n",
       "      <th>foreign_unadj</th>\n",
       "      <th>genre</th>\n",
       "      <th>mpaa_rating</th>\n",
       "      <th>number_of_theaters</th>\n",
       "      <th>opening_weekend_adj</th>\n",
       "      <th>producers</th>\n",
       "      <th>production_budget_adj</th>\n",
       "      <th>release_date</th>\n",
       "      <th>runtime</th>\n",
       "      <th>worldwide_gross_unadj</th>\n",
       "      <th>writers</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3177</th>\n",
       "      <td>The Twilight Saga: Eclipse</td>\n",
       "      <td>[Kristen Stewart, Robert Pattinson, Taylor Lau...</td>\n",
       "      <td>2010-10-21</td>\n",
       "      <td>[Howard Shore]</td>\n",
       "      <td>114.00</td>\n",
       "      <td>[David Slade]</td>\n",
       "      <td>326993600.00</td>\n",
       "      <td>397959596.00</td>\n",
       "      <td>Romance</td>\n",
       "      <td>PG-13</td>\n",
       "      <td>4468.00</td>\n",
       "      <td>72736500.00</td>\n",
       "      <td>[Marty Bowen, Wyck Godfrey, Karen Rosenfelt]</td>\n",
       "      <td>68000000.00</td>\n",
       "      <td>2010-06-30</td>\n",
       "      <td>124.00</td>\n",
       "      <td>698491347.00</td>\n",
       "      <td>[Melissa Rosenberg]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           index  \\\n",
       "3177  The Twilight Saga: Eclipse   \n",
       "\n",
       "                                                 actors close_date  \\\n",
       "3177  [Kristen Stewart, Robert Pattinson, Taylor Lau... 2010-10-21   \n",
       "\n",
       "           composers  days_in_theater      directors  domestic_gross_adj  \\\n",
       "3177  [Howard Shore]           114.00  [David Slade]        326993600.00   \n",
       "\n",
       "      foreign_unadj    genre mpaa_rating  number_of_theaters  \\\n",
       "3177   397959596.00  Romance       PG-13             4468.00   \n",
       "\n",
       "      opening_weekend_adj                                     producers  \\\n",
       "3177          72736500.00  [Marty Bowen, Wyck Godfrey, Karen Rosenfelt]   \n",
       "\n",
       "      production_budget_adj release_date  runtime  worldwide_gross_unadj  \\\n",
       "3177            68000000.00   2010-06-30   124.00           698491347.00   \n",
       "\n",
       "                  writers  \n",
       "3177  [Melissa Rosenberg]  "
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# lol... twilight\n",
    "df[df['number_of_theaters'] == 4468.00]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>actors</th>\n",
       "      <th>close_date</th>\n",
       "      <th>composers</th>\n",
       "      <th>days_in_theater</th>\n",
       "      <th>directors</th>\n",
       "      <th>domestic_gross_adj</th>\n",
       "      <th>foreign_unadj</th>\n",
       "      <th>genre</th>\n",
       "      <th>mpaa_rating</th>\n",
       "      <th>number_of_theaters</th>\n",
       "      <th>opening_weekend_adj</th>\n",
       "      <th>producers</th>\n",
       "      <th>production_budget_adj</th>\n",
       "      <th>release_date</th>\n",
       "      <th>runtime</th>\n",
       "      <th>worldwide_gross_unadj</th>\n",
       "      <th>writers</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3438</th>\n",
       "      <td>Star Wars: The Force Awakens</td>\n",
       "      <td>[John Boyega, Daisy Ridley, Adam Driver, Oscar...</td>\n",
       "      <td>2016-06-02</td>\n",
       "      <td>[John Williams]</td>\n",
       "      <td>168.00</td>\n",
       "      <td>[J.J. Abrams]</td>\n",
       "      <td>935195600.00</td>\n",
       "      <td>1131561399.00</td>\n",
       "      <td>Sci-Fi Fantasy</td>\n",
       "      <td>PG-13</td>\n",
       "      <td>4134.00</td>\n",
       "      <td>246541600.00</td>\n",
       "      <td>[J.J. Abrams, Bryan Burk, Kathleen Kennedy]</td>\n",
       "      <td>245000000.00</td>\n",
       "      <td>2015-12-18</td>\n",
       "      <td>136.00</td>\n",
       "      <td>2068223624.00</td>\n",
       "      <td>[J.J. Abrams, Michael Arndt, Lawrence Kasdan]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             index  \\\n",
       "3438  Star Wars: The Force Awakens   \n",
       "\n",
       "                                                 actors close_date  \\\n",
       "3438  [John Boyega, Daisy Ridley, Adam Driver, Oscar... 2016-06-02   \n",
       "\n",
       "            composers  days_in_theater      directors  domestic_gross_adj  \\\n",
       "3438  [John Williams]           168.00  [J.J. Abrams]        935195600.00   \n",
       "\n",
       "      foreign_unadj           genre mpaa_rating  number_of_theaters  \\\n",
       "3438  1131561399.00  Sci-Fi Fantasy       PG-13             4134.00   \n",
       "\n",
       "      opening_weekend_adj                                    producers  \\\n",
       "3438         246541600.00  [J.J. Abrams, Bryan Burk, Kathleen Kennedy]   \n",
       "\n",
       "      production_budget_adj release_date  runtime  worldwide_gross_unadj  \\\n",
       "3438           245000000.00   2015-12-18   136.00          2068223624.00   \n",
       "\n",
       "                                            writers  \n",
       "3438  [J.J. Abrams, Michael Arndt, Lawrence Kasdan]  "
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# the star wars franchise seems incredibly successful\n",
    "df[df['opening_weekend_adj'] == 246541600.00]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "index                    9965\n",
       "actors                   4648\n",
       "close_date               5982\n",
       "composers                2552\n",
       "days_in_theater          5745\n",
       "directors                3491\n",
       "domestic_gross_adj       9470\n",
       "foreign_unadj            4047\n",
       "genre                    9308\n",
       "mpaa_rating              9927\n",
       "number_of_theaters       9086\n",
       "opening_weekend_adj      8417\n",
       "producers                2926\n",
       "production_budget_adj    2419\n",
       "release_date             9829\n",
       "runtime                  9557\n",
       "worldwide_gross_unadj    5080\n",
       "writers                  2375\n",
       "dtype: int64"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# make sure all the data was imported\n",
    "df.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count   9557.00\n",
       "mean     104.52\n",
       "std       22.05\n",
       "min       37.00\n",
       "25%       91.00\n",
       "50%      100.00\n",
       "75%      113.00\n",
       "max      583.00\n",
       "Name: runtime, dtype: float64"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['runtime'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>actors</th>\n",
       "      <th>close_date</th>\n",
       "      <th>composers</th>\n",
       "      <th>days_in_theater</th>\n",
       "      <th>directors</th>\n",
       "      <th>domestic_gross_adj</th>\n",
       "      <th>foreign_unadj</th>\n",
       "      <th>genre</th>\n",
       "      <th>mpaa_rating</th>\n",
       "      <th>number_of_theaters</th>\n",
       "      <th>opening_weekend_adj</th>\n",
       "      <th>producers</th>\n",
       "      <th>production_budget_adj</th>\n",
       "      <th>release_date</th>\n",
       "      <th>runtime</th>\n",
       "      <th>worldwide_gross_unadj</th>\n",
       "      <th>writers</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5107</th>\n",
       "      <td>Dekalog (2016 re-release)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaT</td>\n",
       "      <td>NaN</td>\n",
       "      <td>35.00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>107800.00</td>\n",
       "      <td>nan</td>\n",
       "      <td>Drama</td>\n",
       "      <td>Unrated</td>\n",
       "      <td>5.00</td>\n",
       "      <td>18800.00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>nan</td>\n",
       "      <td>2016-09-02</td>\n",
       "      <td>583.00</td>\n",
       "      <td>nan</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          index actors close_date composers  days_in_theater  \\\n",
       "5107  Dekalog (2016 re-release)    NaN        NaT       NaN            35.00   \n",
       "\n",
       "     directors  domestic_gross_adj  foreign_unadj  genre mpaa_rating  \\\n",
       "5107       NaN           107800.00            nan  Drama     Unrated   \n",
       "\n",
       "      number_of_theaters  opening_weekend_adj producers  \\\n",
       "5107                5.00             18800.00       NaN   \n",
       "\n",
       "      production_budget_adj release_date  runtime  worldwide_gross_unadj  \\\n",
       "5107                    nan   2016-09-02   583.00                    nan   \n",
       "\n",
       "     writers  \n",
       "5107     NaN  "
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# the 583 is a movie called dekalog, a polish tv series\n",
    "df[df['runtime'] == 583.00]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "R                3217\n",
       "Unrated          2826\n",
       "PG-13            2007\n",
       "PG               1353\n",
       "G                 262\n",
       "Not Yet Rated     239\n",
       "NC-17              21\n",
       "M                   2\n",
       "Name: mpaa_rating, dtype: int64"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['mpaa_rating'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Looks good, Pickle it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open('movies.pkl', 'wb') as picklefile:\n",
    "    pickle.dump(df, picklefile)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}