AB_testing_project.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pageviews = 5000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>metric</th>\n",
       "      <th>baseline_val</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>unique cookies to view page per day:</td>\n",
       "      <td>40000.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>unique cookies to click \"start free trial\" per...</td>\n",
       "      <td>3200.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>enrollments per day:</td>\n",
       "      <td>660.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>click-through-probability on \"start free trial\":</td>\n",
       "      <td>0.080000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>probability of enrolling, given click:</td>\n",
       "      <td>0.206250</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>probability of payment, given enroll:</td>\n",
       "      <td>0.530000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>probability of payment, given click</td>\n",
       "      <td>0.109313</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              metric  baseline_val\n",
       "0               unique cookies to view page per day:  40000.000000\n",
       "1  unique cookies to click \"start free trial\" per...   3200.000000\n",
       "2                               enrollments per day:    660.000000\n",
       "3   click-through-probability on \"start free trial\":      0.080000\n",
       "4             probability of enrolling, given click:      0.206250\n",
       "5              probability of payment, given enroll:      0.530000\n",
       "6                probability of payment, given click      0.109313"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_basevals = pd.read_csv(\"data/baseline_vals.csv\", index_col=False,header = None, names = ['metric','baseline_val'])\n",
    "df_basevals.metric = df_basevals.metric.map(lambda x: x.lower())\n",
    "df_basevals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0202"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "round(np.sqrt((.206250*(1-.206250))/(5000*3200/40000)),4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0551"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "round(np.sqrt((.53*(1-.53))/(5000*660/40000)),4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0156"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "round(np.sqrt((.109313*(1-.109313))/(5000*3200/40000)),4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Pageviews without Bonferroni Correction ##"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Gross Conversion ###\n",
    "\n",
    "- Baseline Conversion: 20.625%\n",
    "- Minimum Detectable Effect: 1%\n",
    "- alpha: 5%\n",
    "- beta: 20%\n",
    "- 1 - beta: 80%\n",
    "- sample size = 25,835 enrollments/group\n",
    "- Number of groups = 2 (experiment and control)\n",
    "- total sample size =  51,670 enrollments\n",
    "- clicks/pageview: 3200/40000 = .08 clicks/pageview\n",
    "- pageviews = 645,875\n",
    "\n",
    "\n",
    "\n",
    "### Retention ###\n",
    "\n",
    "- Baseline Conversion: 53%\n",
    "- Minimum Detectable Effect: 1%\n",
    "- alpha: 5%\n",
    "- beta: 20%\n",
    "- 1 - beta: 80%\n",
    "- sample size = 39,155 enrollments/group\n",
    "- Number of groups = 2 (experiment and control)\n",
    "- total sample size = 78,230 enrollments\n",
    "- enrollments/pageview: 660/40000 = .0165 enrollments/pageview\n",
    "- pageviews = 78,230/.0165 = 4,741,212\n",
    "\n",
    "### Net Conversion ###\n",
    "\n",
    "- Baseline Conversion: 10.9313%\n",
    "- Minimum Detectable Effect: .75%\n",
    "- alpha: 5%\n",
    "- beta: 20%\n",
    "- 1 - beta: 80%\n",
    "- sample size = 27,413 enrollments/group\n",
    "- Number of groups = 2 (experiment and control)\n",
    "- total sample size = 54,826\n",
    "- clicks/pageview: 3200/40000 = .08 clicks/pageview\n",
    "- pageviews = 685,325\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Duration and Exposure ###"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "118.5303"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "4741212.0/40000"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If we divert 100% off traffic, given 40,000 page views per day, the experiment would take 119 days.  That is a long time.  If we eliminate retention, we are left with Gross Conversion and Net Conversion.  This reduces the number of required pageviews to 685,325, and an 18 day experiment with 100% diversion.  There may be other experiments to run, so let's say 50% diversion for 35 days."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "17.133125"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "685325.0/40000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df_control = pd.read_csv(\"data/Final Project Results - Control.csv\")\n",
    "df_experiment = pd.read_csv(\"data/Final Project Results - Experiment.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Control</th>\n",
       "      <th>Experiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>cookies</th>\n",
       "      <td>345543</td>\n",
       "      <td>344660</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>clicks</th>\n",
       "      <td>28378</td>\n",
       "      <td>28325</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>enrollments</th>\n",
       "      <td>3785</td>\n",
       "      <td>3423</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>payments</th>\n",
       "      <td>2033</td>\n",
       "      <td>1945</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             Control  Experiment\n",
       "cookies       345543      344660\n",
       "clicks         28378       28325\n",
       "enrollments     3785        3423\n",
       "payments        2033        1945"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results = {\"Control\":pd.Series([df_control.Pageviews.sum(),df_control.Clicks.sum(),\n",
    "                                  df_control.Enrollments.sum(),df_control.Payments.sum()],\n",
    "                                  index = [\"cookies\",\"clicks\",\"enrollments\",\"payments\"]),\n",
    "           \"Experiment\":pd.Series([df_experiment.Pageviews.sum(),df_experiment.Clicks.sum(),\n",
    "                               df_experiment.Enrollments.sum(),df_experiment.Payments.sum()],\n",
    "                               index = [\"cookies\",\"clicks\",\"enrollments\",\"payments\"])}\n",
    "df_results = pd.DataFrame(results)\n",
    "df_results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sanity Checks"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Count Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Control</th>\n",
       "      <th>Experiment</th>\n",
       "      <th>Total</th>\n",
       "      <th>Prob</th>\n",
       "      <th>StdErr</th>\n",
       "      <th>MargErr</th>\n",
       "      <th>CI_lower</th>\n",
       "      <th>CI_upper</th>\n",
       "      <th>Obs_val</th>\n",
       "      <th>Pass_Sanity</th>\n",
       "      <th>Diff</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>cookies</th>\n",
       "      <td>345543</td>\n",
       "      <td>344660</td>\n",
       "      <td>690203</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.000602</td>\n",
       "      <td>0.001180</td>\n",
       "      <td>0.498820</td>\n",
       "      <td>0.501180</td>\n",
       "      <td>0.499360</td>\n",
       "      <td>True</td>\n",
       "      <td>0.001279</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>clicks</th>\n",
       "      <td>28378</td>\n",
       "      <td>28325</td>\n",
       "      <td>56703</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.002100</td>\n",
       "      <td>0.004116</td>\n",
       "      <td>0.495884</td>\n",
       "      <td>0.504116</td>\n",
       "      <td>0.499533</td>\n",
       "      <td>True</td>\n",
       "      <td>0.000935</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>enrollments</th>\n",
       "      <td>3785</td>\n",
       "      <td>3423</td>\n",
       "      <td>7208</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.005889</td>\n",
       "      <td>0.011543</td>\n",
       "      <td>0.488457</td>\n",
       "      <td>0.511543</td>\n",
       "      <td>0.474889</td>\n",
       "      <td>False</td>\n",
       "      <td>0.050222</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>payments</th>\n",
       "      <td>2033</td>\n",
       "      <td>1945</td>\n",
       "      <td>3978</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.007928</td>\n",
       "      <td>0.015538</td>\n",
       "      <td>0.484462</td>\n",
       "      <td>0.515538</td>\n",
       "      <td>0.488939</td>\n",
       "      <td>True</td>\n",
       "      <td>0.022122</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             Control  Experiment   Total  Prob    StdErr   MargErr  CI_lower  \\\n",
       "cookies       345543      344660  690203   0.5  0.000602  0.001180  0.498820   \n",
       "clicks         28378       28325   56703   0.5  0.002100  0.004116  0.495884   \n",
       "enrollments     3785        3423    7208   0.5  0.005889  0.011543  0.488457   \n",
       "payments        2033        1945    3978   0.5  0.007928  0.015538  0.484462   \n",
       "\n",
       "             CI_upper   Obs_val Pass_Sanity      Diff  \n",
       "cookies      0.501180  0.499360        True  0.001279  \n",
       "clicks       0.504116  0.499533        True  0.000935  \n",
       "enrollments  0.511543  0.474889       False  0.050222  \n",
       "payments     0.515538  0.488939        True  0.022122  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_results['Total']=df_results.Control + df_results.Experiment\n",
    "df_results['Prob'] = 0.5\n",
    "df_results['StdErr'] = np.sqrt((df_results.Prob * (1- df_results.Prob))/df_results.Total)\n",
    "df_results[\"MargErr\"] = 1.96 * df_results.StdErr\n",
    "df_results[\"CI_lower\"] = df_results.Prob - df_results.MargErr\n",
    "df_results[\"CI_upper\"] = df_results.Prob + df_results.MargErr\n",
    "df_results[\"Obs_val\"] = df_results.Experiment/df_results.Total\n",
    "df_results[\"Pass_Sanity\"] = df_results.apply(lambda x: (x.Obs_val > x.CI_lower) and (x.Obs_val < x.CI_upper),axis=1)\n",
    "df_results['Diff'] = abs((df_results.Experiment - df_results.Control)/df_results.Total)\n",
    "\n",
    "df_results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Other Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0.082125813574576823, 0.082182440666163759, 0.081266986844116651, 0.083097894488210866, 0.00046706827655464432, 0.0009154538220471028)\n"
     ]
    }
   ],
   "source": [
    "# click through probability (clicks/cookies)\n",
    "\n",
    "control_cookies = df_results.loc['cookies','Control']\n",
    "control_clicks = df_results.loc['clicks','Control']\n",
    "\n",
    "exp_cookies = df_results.loc['cookies','Experiment']\n",
    "exp_clicks = df_results.loc['clicks', 'Experiment']\n",
    "\n",
    "## control value \n",
    "cont_p_hat = control_clicks/control_cookies\n",
    "\n",
    "## observed value (experimental value)\n",
    "exp_p_hat = exp_clicks/exp_cookies\n",
    "\n",
    "## Standard Error\n",
    "SE_ClickProb = np.sqrt((cont_p_hat * (1- cont_p_hat))/control_cookies)\n",
    "\n",
    "\n",
    "## margin of error for 95% confidence interval (z = 1.96)\n",
    "\n",
    "ME_ClickProb = SE_ClickProb * 1.96\n",
    "\n",
    "## CI\n",
    "upper_ClickProb = exp_p_hat + ME_ClickProb\n",
    "lower_ClickProb = exp_p_hat - ME_ClickProb\n",
    "\n",
    "## Sane in the membrane (yes, it passes)\n",
    "print(cont_p_hat,exp_p_hat,lower_ClickProb,upper_ClickProb, SE_ClickProb, ME_ClickProb)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluation Metric Results Calculations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df_control_notnull = df_control[pd.isnull(df_control.Enrollments) != True]\n",
    "df_experiment_notnull = df_experiment[pd.isnull(df_control.Enrollments) != True]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Control</th>\n",
       "      <th>Experiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>cookies</th>\n",
       "      <td>212163</td>\n",
       "      <td>211362</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>clicks</th>\n",
       "      <td>17293</td>\n",
       "      <td>17260</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>enrollments</th>\n",
       "      <td>3785</td>\n",
       "      <td>3423</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>payments</th>\n",
       "      <td>2033</td>\n",
       "      <td>1945</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             Control  Experiment\n",
       "cookies       212163      211362\n",
       "clicks         17293       17260\n",
       "enrollments     3785        3423\n",
       "payments        2033        1945"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_notnull = {\"Control\":pd.Series([df_control_notnull.Pageviews.sum(),df_control_notnull.Clicks.sum(),\n",
    "                                  df_control_notnull.Enrollments.sum(),df_control_notnull.Payments.sum()],\n",
    "                                  index = [\"cookies\",\"clicks\",\"enrollments\",\"payments\"]),\n",
    "           \"Experiment\":pd.Series([df_experiment_notnull.Pageviews.sum(),df_experiment_notnull.Clicks.sum(),\n",
    "                               df_experiment_notnull.Enrollments.sum(),df_experiment_notnull.Payments.sum()],\n",
    "                               index = [\"cookies\",\"clicks\",\"enrollments\",\"payments\"])}\n",
    "df_results_notnull = pd.DataFrame(results_notnull)\n",
    "df_results_notnull"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Control</th>\n",
       "      <th>Experiment</th>\n",
       "      <th>Total</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>cookies</th>\n",
       "      <td>212163</td>\n",
       "      <td>211362</td>\n",
       "      <td>423525</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>clicks</th>\n",
       "      <td>17293</td>\n",
       "      <td>17260</td>\n",
       "      <td>34553</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>enrollments</th>\n",
       "      <td>3785</td>\n",
       "      <td>3423</td>\n",
       "      <td>7208</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>payments</th>\n",
       "      <td>2033</td>\n",
       "      <td>1945</td>\n",
       "      <td>3978</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             Control  Experiment   Total\n",
       "cookies       212163      211362  423525\n",
       "clicks         17293       17260   34553\n",
       "enrollments     3785        3423    7208\n",
       "payments        2033        1945    3978"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_results_notnull['Total']=df_results_notnull.Control + df_results_notnull.Experiment\n",
    "\n",
    "df_results_notnull"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# experiment values\n",
    "\n",
    "enrollments_exp = df_results_notnull.loc[\"enrollments\"].Experiment\n",
    "clicks_exp = df_results_notnull.loc[\"clicks\"].Experiment\n",
    "payments_exp = df_results_notnull.loc[\"payments\"].Experiment\n",
    "\n",
    "# control values\n",
    "\n",
    "enrollments_cont = df_results_notnull.loc[\"enrollments\"].Control\n",
    "clicks_cont = df_results_notnull.loc[\"clicks\"].Control\n",
    "payments_cont = df_results_notnull.loc[\"payments\"].Control\n",
    "\n",
    "\n",
    "\n",
    "# metrics\n",
    "\n",
    "GrossConversion_exp = enrollments_exp/clicks_exp\n",
    "NetConversion_exp = payments_exp/clicks_exp\n",
    "GrossConversion_cont = enrollments_cont/clicks_cont\n",
    "NetConversion_cont = payments_cont/clicks_cont\n",
    "\n",
    "GrossConversion = (enrollments_exp + enrollments_cont)/(clicks_cont + clicks_exp)\n",
    "NetConversion = (payments_cont + payments_exp)/(clicks_cont + clicks_exp)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GrossConversion: 0.208607067404 \n",
      "NetConversion:0.115127485312\n"
     ]
    }
   ],
   "source": [
    "print('GrossConversion: {} \\nNetConversion:{}'.format(GrossConversion,NetConversion))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.2188746891805933"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "GrossConversion_cont"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.19831981460023174"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "GrossConversion_exp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def stats_prop(p_hat,z_score,N_cont,N_exp,diff):\n",
    "    std_err = np.sqrt((p_hat * (1- p_hat ))*(1/N_cont + 1/N_exp))\n",
    "    marg_err = z_score * std_err\n",
    "    ci_lower = diff - marg_err\n",
    "    ci_upper = diff + marg_err\n",
    "    \n",
    "    return std_err,marg_err,ci_lower,ci_upper\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-0.020554874580361565"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "GrossConversion_diff = GrossConversion_exp - GrossConversion_cont\n",
    "GrossConversion_diff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "se_gross,me_gross,cil_gross,ciu_gross = stats_prop(GrossConversion,1.96,clicks_cont,\n",
    "                                                   clicks_exp,GrossConversion_diff)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0.0043716753852259364, 0.0085684837550428355, -0.029123358335404401, -0.01198639082531873)\n"
     ]
    }
   ],
   "source": [
    "print(se_gross,me_gross,cil_gross,ciu_gross)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-0.0048737226745441675"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "NetConversion_diff = NetConversion_exp - NetConversion_cont\n",
    "NetConversion_diff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "se_net,me_net,cil_net,ciu_net = stats_prop(NetConversion,1.96,clicks_cont,\n",
    "                                           clicks_exp,NetConversion_diff)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0.0034341335129324238, 0.0067309016853475505, -0.011604624359891718, 0.001857179010803383)\n"
     ]
    }
   ],
   "source": [
    "print(se_net,me_net,cil_net,ciu_net)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df_SignTest = pd.merge(df_control_notnull,df_experiment_notnull,on=\"Date\")\n",
    "df_SignTest['GrossConversion_cont'] = df_SignTest.Enrollments_x/df_SignTest.Clicks_x\n",
    "df_SignTest['GrossConversion_exp'] = df_SignTest.Enrollments_y/df_SignTest.Clicks_y\n",
    "df_SignTest['NetConversion_cont'] = df_SignTest.Payments_x/df_SignTest.Clicks_x\n",
    "df_SignTest['NetConversion_exp'] = df_SignTest.Payments_y/df_SignTest.Clicks_y\n",
    "\n",
    "cols = ['Date','GrossConversion_cont','GrossConversion_exp','NetConversion_cont','NetConversion_exp']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df_SignTest = df_SignTest[cols]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>GrossConversion_cont</th>\n",
       "      <th>GrossConversion_exp</th>\n",
       "      <th>NetConversion_cont</th>\n",
       "      <th>NetConversion_exp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Sat, Oct 11</td>\n",
       "      <td>0.195051</td>\n",
       "      <td>0.153061</td>\n",
       "      <td>0.101892</td>\n",
       "      <td>0.049563</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Sun, Oct 12</td>\n",
       "      <td>0.188703</td>\n",
       "      <td>0.147771</td>\n",
       "      <td>0.089859</td>\n",
       "      <td>0.115924</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Mon, Oct 13</td>\n",
       "      <td>0.183718</td>\n",
       "      <td>0.164027</td>\n",
       "      <td>0.104510</td>\n",
       "      <td>0.089367</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Tue, Oct 14</td>\n",
       "      <td>0.186603</td>\n",
       "      <td>0.166868</td>\n",
       "      <td>0.125598</td>\n",
       "      <td>0.111245</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Wed, Oct 15</td>\n",
       "      <td>0.194743</td>\n",
       "      <td>0.168269</td>\n",
       "      <td>0.076464</td>\n",
       "      <td>0.112981</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          Date  GrossConversion_cont  GrossConversion_exp  NetConversion_cont  \\\n",
       "0  Sat, Oct 11              0.195051             0.153061            0.101892   \n",
       "1  Sun, Oct 12              0.188703             0.147771            0.089859   \n",
       "2  Mon, Oct 13              0.183718             0.164027            0.104510   \n",
       "3  Tue, Oct 14              0.186603             0.166868            0.125598   \n",
       "4  Wed, Oct 15              0.194743             0.168269            0.076464   \n",
       "\n",
       "   NetConversion_exp  \n",
       "0           0.049563  \n",
       "1           0.115924  \n",
       "2           0.089367  \n",
       "3           0.111245  \n",
       "4           0.112981  "
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_SignTest.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_SignTest['GC_Sign'] = df_SignTest.GrossConversion_cont - df_SignTest.GrossConversion_exp\n",
    "df_SignTest['NC_Sign'] = df_SignTest.NetConversion_cont - df_SignTest.NetConversion_exp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "23"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_SignTest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "19"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_SignTest[df_SignTest.GC_Sign > 0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "13"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_SignTest[df_SignTest.NC_Sign > 0])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}