IMM_UsingProb.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Import packages \n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "# Set Working Directory\n",
    "import os\n",
    "# os.chdir(r'C:\\Personal\\IMM') # absolute path, using \\ and r prefix\n",
    "# wd = os.getcwd()\n",
    "\n",
    "import json\n",
    "from pprint import pprint\n",
    "import timeit\n",
    "from decimal import *\n",
    "import numba\n",
    "from numba import jit\n",
    "import traceback\n",
    "\n",
    "import functools\n",
    "from balsa.matrices import read_mdf, to_mdf\n",
    "\n",
    "from balsa.cheval import sample_from_weights\n",
    "\n",
    "# set random seed to be used\n",
    "seed = 12345"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\u001b[1;31mSignature:\u001b[0m \u001b[0msample_from_weights\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mweights\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrandomizer\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mastype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'category'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mn_threads\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mn_draws\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msqueeze\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
       "\u001b[1;31mDocstring:\u001b[0m\n",
       "Args:\n",
       "    weights (pd.DataFrame):\n",
       "    randomizer (inr, RandomState, or None):\n",
       "    astype (type or str):\n",
       "    n_threads (int):\n",
       "\n",
       "Returns (pd.Series):\n",
       "\u001b[1;31mFile:\u001b[0m      c:\\users\\mzd\\appdata\\local\\continuum\\anaconda3.6\\lib\\site-packages\\balsa\\cheval\\api.py\n",
       "\u001b[1;31mType:\u001b[0m      function\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sample_from_weights?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dirListing = 'c:\\\\personal\\\\IMM'\n",
    "fpath_pk = os.listdir(dirListing)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 0: Batch in the *.bin file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "record_type1 = np.dtype([\n",
    "    (\"Production Zone\", \"i4\"),\n",
    "    (\"Destination Zone\", \"i4\"),\n",
    "    (\"Market Segment\", \"i4\"),\n",
    "    (\"Segment 0 Drive-Alone Non-Toll\", \"f4\"),\n",
    "    (\"Segment 0 Drive-Alone Toll Trips\", \"f4\"),\n",
    "    (\"Segment 0 2-Person Non-Toll HOV Trips\", \"f4\"),\n",
    "    (\"Segment 0 2-Person Non-Toll non-HOV Trips\", \"f4\"),\n",
    "    (\"Segment 0 2-Person Toll HOV Trips\", \"f4\"),\n",
    "    (\"Segment 0 2-Person Toll Non-HOV Trips\", \"f4\"),\n",
    "    (\"Segment 0 3+ Person Non-Toll HOV Trips\", \"f4\"),\n",
    "    (\"Segment 0 3+ Person Non-Toll non-HOV Trips\", \"f4\"),\n",
    "    (\"Segment 0 3+ Person Toll HOV Trips\", \"f4\"),\n",
    "    (\"Segment 0 3+ Person Toll Non-HOV Trips\", \"f4\"),\n",
    "    (\"Segment 0 Non-Motorized Walk Trips\", \"f4\"),\n",
    "    (\"Segment 0 Non-Motorized Bicycle Trips\", \"f4\"),\n",
    "    (\"Segment 0 Taxi Trips\", \"f4\"),\n",
    "    (\"Segment 0 Uber Trips\", \"f4\"),\n",
    "    (\"Segment 0 GO Bus Walk Access Trips\", \"f4\"),\n",
    "    (\"Segment 0 GO Bus Drive Access Trips\", \"f4\"),\n",
    "    (\"Segment 0 Bus/Streetcar Walk Access Trips\", \"f4\"),\n",
    "    (\"Segment 0 Bus/Streetcar Drive Access Trips\", \"f4\"),\n",
    "    (\"Segment 0 Rapid Bus Walk Access Trips\", \"f4\"),\n",
    "    (\"Segment 0 Rapid Bus Drive Access Trips\", \"f4\"),\n",
    "    (\"Segment 0 Walk Access - GO Rail #1\", \"f4\"),\n",
    "    (\"Segment 0 Walk Access - GO Rail #2\", \"f4\"),\n",
    "    (\"Segment 0 Bus Access - GO Rail #1\", \"f4\"),\n",
    "    (\"Segment 0 Bus Access - GO Rail #2\", \"f4\"),\n",
    "    (\"Segment 0 Park-n-Ride Access - GO Rail #1\", \"f4\"),\n",
    "    (\"Segment 0 Park-n-Ride Access - GO Rail #2\", \"f4\"),\n",
    "    (\"Segment 0 Park-n-Ride Access - GO Rail #3\", \"f4\"),\n",
    "    (\"Segment 0 Park-n-Ride Access - GO Rail #4\", \"f4\"),\n",
    "    (\"Segment 0 Kiss-n-Ride Access - GO Rail #1\", \"f4\"),\n",
    "    (\"Segment 0 Kiss-n-Ride Access - GO Rail #2\", \"f4\"),\n",
    "    (\"Segment 0 Kiss-n-Ride Access - GO Rail #3\", \"f4\"),\n",
    "    (\"Segment 0 Kiss-n-Ride Access - GO Rail #4\", \"f4\"),\n",
    "    (\"Segment 0 Uber Access - GO Rail #1\", \"f4\"),\n",
    "    (\"Segment 0 Uber Access - GO Rail #2\", \"f4\"),\n",
    "    (\"Segment 0 Uber Access - GO Rail #3\", \"f4\"),\n",
    "    (\"Segment 0 Uber Access - GO Rail #4\", \"f4\"),\n",
    "    (\"Segment 0 Walk Access - TTC Subway #1\", \"f4\"),\n",
    "    (\"Segment 0 Walk Access - TTC Subway #2\", \"f4\"),\n",
    "    (\"Segment 0 Bus Access - TTC Subway #1\", \"f4\"),\n",
    "    (\"Segment 0 Bus Access - TTC Subway #2\", \"f4\"),\n",
    "    (\"Segment 0 Park-n-Ride Access - TTC Subway #1\", \"f4\"),\n",
    "    (\"Segment 0 Park-n-Ride Access - TTC Subway #2\", \"f4\"),\n",
    "    (\"Segment 0 Park-n-Ride Access - TTC Subway #3\", \"f4\"),\n",
    "    (\"Segment 0 Park-n-Ride Access - TTC Subway #4\", \"f4\"),\n",
    "    (\"Segment 0 Kiss-n-Ride Access - TTC Subway #1\", \"f4\"),\n",
    "    (\"Segment 0 Kiss-n-Ride Access - TTC Subway #2\", \"f4\"),\n",
    "    (\"Segment 0 Kiss-n-Ride Access - TTC Subway #3\", \"f4\"),\n",
    "    (\"Segment 0 Kiss-n-Ride Access - TTC Subway #4\", \"f4\"),\n",
    "    (\"Segment 0 Uber Access - TTC Subway #1\", \"f4\"),\n",
    "    (\"Segment 0 Uber Access - TTC Subway #2\", \"f4\"),\n",
    "    (\"Segment 0 Uber Access - TTC Subway #3\", \"f4\"),\n",
    "    (\"Segment 0 Uber Access - TTC Subway #4\", \"f4\"),\n",
    "    (\"Segment 1 Drive-Alone Non-Toll\", \"f4\"),\n",
    "    (\"Segment 1 Drive-Alone Toll Trips\", \"f4\"),\n",
    "    (\"Segment 1 2-Person Non-Toll HOV Trips\", \"f4\"),\n",
    "    (\"Segment 1 2-Person Non-Toll non-HOV Trips\", \"f4\"),\n",
    "    (\"Segment 1 2-Person Toll HOV Trips\", \"f4\"),\n",
    "    (\"Segment 1 2-Person Toll Non-HOV Trips\", \"f4\"),\n",
    "    (\"Segment 1 3+ Person Non-Toll HOV Trips\", \"f4\"),\n",
    "    (\"Segment 1 3+ Person Non-Toll non-HOV Trips\", \"f4\"),\n",
    "    (\"Segment 1 3+ Person Toll HOV Trips\", \"f4\"),\n",
    "    (\"Segment 1 3+ Person Toll Non-HOV Trips\", \"f4\"),\n",
    "    (\"Segment 1 Non-Motorized Walk Trips\", \"f4\"),\n",
    "    (\"Segment 1 Non-Motorized Bicycle Trips\", \"f4\"),\n",
    "    (\"Segment 1 Taxi Trips\", \"f4\"),\n",
    "    (\"Segment 1 Uber Trips\", \"f4\"),\n",
    "    (\"Segment 1 GO Bus Walk Access Trips\", \"f4\"),\n",
    "    (\"Segment 1 GO Bus Drive Access Trips\", \"f4\"),\n",
    "    (\"Segment 1 Bus/Streetcar Walk Access Trips\", \"f4\"),\n",
    "    (\"Segment 1 Bus/Streetcar Drive Access Trips\", \"f4\"),\n",
    "    (\"Segment 1 Rapid Bus Walk Access Trips\", \"f4\"),\n",
    "    (\"Segment 1 Rapid Bus Drive Access Trips\", \"f4\"),\n",
    "    (\"Segment 1 Walk Access - GO Rail #1\", \"f4\"),\n",
    "    (\"Segment 1 Walk Access - GO Rail #2\", \"f4\"),\n",
    "    (\"Segment 1 Bus Access - GO Rail #1\", \"f4\"),\n",
    "    (\"Segment 1 Bus Access - GO Rail #2\", \"f4\"),\n",
    "    (\"Segment 1 Park-n-Ride Access - GO Rail #1\", \"f4\"),\n",
    "    (\"Segment 1 Park-n-Ride Access - GO Rail #2\", \"f4\"),\n",
    "    (\"Segment 1 Park-n-Ride Access - GO Rail #3\", \"f4\"),\n",
    "    (\"Segment 1 Park-n-Ride Access - GO Rail #4\", \"f4\"),\n",
    "    (\"Segment 1 Kiss-n-Ride Access - GO Rail #1\", \"f4\"),\n",
    "    (\"Segment 1 Kiss-n-Ride Access - GO Rail #2\", \"f4\"),\n",
    "    (\"Segment 1 Kiss-n-Ride Access - GO Rail #3\", \"f4\"),\n",
    "    (\"Segment 1 Kiss-n-Ride Access - GO Rail #4\", \"f4\"),\n",
    "    (\"Segment 1 Uber Access - GO Rail #1\", \"f4\"),\n",
    "    (\"Segment 1 Uber Access - GO Rail #2\", \"f4\"),\n",
    "    (\"Segment 1 Uber Access - GO Rail #3\", \"f4\"),\n",
    "    (\"Segment 1 Uber Access - GO Rail #4\", \"f4\"),\n",
    "    (\"Segment 1 Walk Access - TTC Subway #1\", \"f4\"),\n",
    "    (\"Segment 1 Walk Access - TTC Subway #2\", \"f4\"),\n",
    "    (\"Segment 1 Bus Access - TTC Subway #1\", \"f4\"),\n",
    "    (\"Segment 1 Bus Access - TTC Subway #2\", \"f4\"),\n",
    "    (\"Segment 1 Park-n-Ride Access - TTC Subway #1\", \"f4\"),\n",
    "    (\"Segment 1 Park-n-Ride Access - TTC Subway #2\", \"f4\"),\n",
    "    (\"Segment 1 Park-n-Ride Access - TTC Subway #3\", \"f4\"),\n",
    "    (\"Segment 1 Park-n-Ride Access - TTC Subway #4\", \"f4\"),\n",
    "    (\"Segment 1 Kiss-n-Ride Access - TTC Subway #1\", \"f4\"),\n",
    "    (\"Segment 1 Kiss-n-Ride Access - TTC Subway #2\", \"f4\"),\n",
    "    (\"Segment 1 Kiss-n-Ride Access - TTC Subway #3\", \"f4\"),\n",
    "    (\"Segment 1 Kiss-n-Ride Access - TTC Subway #4\", \"f4\"),\n",
    "    (\"Segment 1 Uber Access - TTC Subway #1\", \"f4\"),\n",
    "    (\"Segment 1 Uber Access - TTC Subway #2\", \"f4\"),\n",
    "    (\"Segment 1 Uber Access - TTC Subway #3\", \"f4\"),\n",
    "    (\"Segment 1 Uber Access - TTC Subway #4\", \"f4\"),\n",
    "    (\"Segment 2 Drive-Alone Non-Toll\", \"f4\"),\n",
    "    (\"Segment 2 Drive-Alone Toll Trips\", \"f4\"),\n",
    "    (\"Segment 2 2-Person Non-Toll HOV Trips\", \"f4\"),\n",
    "    (\"Segment 2 2-Person Non-Toll non-HOV Trips\", \"f4\"),\n",
    "    (\"Segment 2 2-Person Toll HOV Trips\", \"f4\"),\n",
    "    (\"Segment 2 2-Person Toll Non-HOV Trips\", \"f4\"),\n",
    "    (\"Segment 2 3+ Person Non-Toll HOV Trips\", \"f4\"),\n",
    "    (\"Segment 2 3+ Person Non-Toll non-HOV Trips\", \"f4\"),\n",
    "    (\"Segment 2 3+ Person Toll HOV Trips\", \"f4\"),\n",
    "    (\"Segment 2 3+ Person Toll Non-HOV Trips\", \"f4\"),\n",
    "    (\"Segment 2 Non-Motorized Walk Trips\", \"f4\"),\n",
    "    (\"Segment 2 Non-Motorized Bicycle Trips\", \"f4\"),\n",
    "    (\"Segment 2 Taxi Trips\", \"f4\"),\n",
    "    (\"Segment 2 Uber Trips\", \"f4\"),\n",
    "    (\"Segment 2 GO Bus Walk Access Trips\", \"f4\"),\n",
    "    (\"Segment 2 GO Bus Drive Access Trips\", \"f4\"),\n",
    "    (\"Segment 2 Bus/Streetcar Walk Access Trips\", \"f4\"),\n",
    "    (\"Segment 2 Bus/Streetcar Drive Access Trips\", \"f4\"),\n",
    "    (\"Segment 2 Rapid Bus Walk Access Trips\", \"f4\"),\n",
    "    (\"Segment 2 Rapid Bus Drive Access Trips\", \"f4\"),\n",
    "    (\"Segment 2 Walk Access - GO Rail #1\", \"f4\"),\n",
    "    (\"Segment 2 Walk Access - GO Rail #2\", \"f4\"),\n",
    "    (\"Segment 2 Bus Access - GO Rail #1\", \"f4\"),\n",
    "    (\"Segment 2 Bus Access - GO Rail #2\", \"f4\"),\n",
    "    (\"Segment 2 Park-n-Ride Access - GO Rail #1\", \"f4\"),\n",
    "    (\"Segment 2 Park-n-Ride Access - GO Rail #2\", \"f4\"),\n",
    "    (\"Segment 2 Park-n-Ride Access - GO Rail #3\", \"f4\"),\n",
    "    (\"Segment 2 Park-n-Ride Access - GO Rail #4\", \"f4\"),\n",
    "    (\"Segment 2 Kiss-n-Ride Access - GO Rail #1\", \"f4\"),\n",
    "    (\"Segment 2 Kiss-n-Ride Access - GO Rail #2\", \"f4\"),\n",
    "    (\"Segment 2 Kiss-n-Ride Access - GO Rail #3\", \"f4\"),\n",
    "    (\"Segment 2 Kiss-n-Ride Access - GO Rail #4\", \"f4\"),\n",
    "    (\"Segment 2 Uber Access - GO Rail #1\", \"f4\"),\n",
    "    (\"Segment 2 Uber Access - GO Rail #2\", \"f4\"),\n",
    "    (\"Segment 2 Uber Access - GO Rail #3\", \"f4\"),\n",
    "    (\"Segment 2 Uber Access - GO Rail #4\", \"f4\"),\n",
    "    (\"Segment 2 Walk Access - TTC Subway #1\", \"f4\"),\n",
    "    (\"Segment 2 Walk Access - TTC Subway #2\", \"f4\"),\n",
    "    (\"Segment 2 Bus Access - TTC Subway #1\", \"f4\"),\n",
    "    (\"Segment 2 Bus Access - TTC Subway #2\", \"f4\"),\n",
    "    (\"Segment 2 Park-n-Ride Access - TTC Subway #1\", \"f4\"),\n",
    "    (\"Segment 2 Park-n-Ride Access - TTC Subway #2\", \"f4\"),\n",
    "    (\"Segment 2 Park-n-Ride Access - TTC Subway #3\", \"f4\"),\n",
    "    (\"Segment 2 Park-n-Ride Access - TTC Subway #4\", \"f4\"),\n",
    "    (\"Segment 2 Kiss-n-Ride Access - TTC Subway #1\", \"f4\"),\n",
    "    (\"Segment 2 Kiss-n-Ride Access - TTC Subway #2\", \"f4\"),\n",
    "    (\"Segment 2 Kiss-n-Ride Access - TTC Subway #3\", \"f4\"),\n",
    "    (\"Segment 2 Kiss-n-Ride Access - TTC Subway #4\", \"f4\"),\n",
    "    (\"Segment 2 Uber Access - TTC Subway #1\", \"f4\"),\n",
    "    (\"Segment 2 Uber Access - TTC Subway #2\", \"f4\"),\n",
    "    (\"Segment 2 Uber Access - TTC Subway #3\", \"f4\"),\n",
    "    (\"Segment 2 Uber Access - TTC Subway #4\", \"f4\"),\n",
    "    (\"Segment 3 Drive-Alone Non-Toll\", \"f4\"),\n",
    "    (\"Segment 3 Drive-Alone Toll Trips\", \"f4\"),\n",
    "    (\"Segment 3 2-Person Non-Toll HOV Trips\", \"f4\"),\n",
    "    (\"Segment 3 2-Person Non-Toll non-HOV Trips\", \"f4\"),\n",
    "    (\"Segment 3 2-Person Toll HOV Trips\", \"f4\"),\n",
    "    (\"Segment 3 2-Person Toll Non-HOV Trips\", \"f4\"),\n",
    "    (\"Segment 3 3+ Person Non-Toll HOV Trips\", \"f4\"),\n",
    "    (\"Segment 3 3+ Person Non-Toll non-HOV Trips\", \"f4\"),\n",
    "    (\"Segment 3 3+ Person Toll HOV Trips\", \"f4\"),\n",
    "    (\"Segment 3 3+ Person Toll Non-HOV Trips\", \"f4\"),\n",
    "    (\"Segment 3 Non-Motorized Walk Trips\", \"f4\"),\n",
    "    (\"Segment 3 Non-Motorized Bicycle Trips\", \"f4\"),\n",
    "    (\"Segment 3 Taxi Trips\", \"f4\"),\n",
    "    (\"Segment 3 Uber Trips\", \"f4\"),\n",
    "    (\"Segment 3 GO Bus Walk Access Trips\", \"f4\"),\n",
    "    (\"Segment 3 GO Bus Drive Access Trips\", \"f4\"),\n",
    "    (\"Segment 3 Bus/Streetcar Walk Access Trips\", \"f4\"),\n",
    "    (\"Segment 3 Bus/Streetcar Drive Access Trips\", \"f4\"),\n",
    "    (\"Segment 3 Rapid Bus Walk Access Trips\", \"f4\"),\n",
    "    (\"Segment 3 Rapid Bus Drive Access Trips\", \"f4\"),\n",
    "    (\"Segment 3 Walk Access - GO Rail #1\", \"f4\"),\n",
    "    (\"Segment 3 Walk Access - GO Rail #2\", \"f4\"),\n",
    "    (\"Segment 3 Bus Access - GO Rail #1\", \"f4\"),\n",
    "    (\"Segment 3 Bus Access - GO Rail #2\", \"f4\"),\n",
    "    (\"Segment 3 Park-n-Ride Access - GO Rail #1\", \"f4\"),\n",
    "    (\"Segment 3 Park-n-Ride Access - GO Rail #2\", \"f4\"),\n",
    "    (\"Segment 3 Park-n-Ride Access - GO Rail #3\", \"f4\"),\n",
    "    (\"Segment 3 Park-n-Ride Access - GO Rail #4\", \"f4\"),\n",
    "    (\"Segment 3 Kiss-n-Ride Access - GO Rail #1\", \"f4\"),\n",
    "    (\"Segment 3 Kiss-n-Ride Access - GO Rail #2\", \"f4\"),\n",
    "    (\"Segment 3 Kiss-n-Ride Access - GO Rail #3\", \"f4\"),\n",
    "    (\"Segment 3 Kiss-n-Ride Access - GO Rail #4\", \"f4\"),\n",
    "    (\"Segment 3 Uber Access - GO Rail #1\", \"f4\"),\n",
    "    (\"Segment 3 Uber Access - GO Rail #2\", \"f4\"),\n",
    "    (\"Segment 3 Uber Access - GO Rail #3\", \"f4\"),\n",
    "    (\"Segment 3 Uber Access - GO Rail #4\", \"f4\"),\n",
    "    (\"Segment 3 Walk Access - TTC Subway #1\", \"f4\"),\n",
    "    (\"Segment 3 Walk Access - TTC Subway #2\", \"f4\"),\n",
    "    (\"Segment 3 Bus Access - TTC Subway #1\", \"f4\"),\n",
    "    (\"Segment 3 Bus Access - TTC Subway #2\", \"f4\"),\n",
    "    (\"Segment 3 Park-n-Ride Access - TTC Subway #1\", \"f4\"),\n",
    "    (\"Segment 3 Park-n-Ride Access - TTC Subway #2\", \"f4\"),\n",
    "    (\"Segment 3 Park-n-Ride Access - TTC Subway #3\", \"f4\"),\n",
    "    (\"Segment 3 Park-n-Ride Access - TTC Subway #4\", \"f4\"),\n",
    "    (\"Segment 3 Kiss-n-Ride Access - TTC Subway #1\", \"f4\"),\n",
    "    (\"Segment 3 Kiss-n-Ride Access - TTC Subway #2\", \"f4\"),\n",
    "    (\"Segment 3 Kiss-n-Ride Access - TTC Subway #3\", \"f4\"),\n",
    "    (\"Segment 3 Kiss-n-Ride Access - TTC Subway #4\", \"f4\"),\n",
    "    (\"Segment 3 Uber Access - TTC Subway #1\", \"f4\"),\n",
    "    (\"Segment 3 Uber Access - TTC Subway #2\", \"f4\"),\n",
    "    (\"Segment 3 Uber Access - TTC Subway #3\", \"f4\"),\n",
    "    (\"Segment 3 Uber Access - TTC Subway #4\", \"f4\")\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# bring in vehicle type file. \n",
    "veh_type = pd.read_csv(os.path.join(dirListing, \"sample_veh_proportions_gghm_zone_csd.csv\")).set_index('ggh_zone')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # add in a dummy column of a repeating sequence so that the df can be unstacked\n",
    "# seq = cycle(['1', '2', '3', '4'])\n",
    "# hh_vehprob['probname'] = [next(seq) for count in range(hh_vehprob.shape[0])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# bring in Bill's probabilities filea\n",
    "hbw_prob = pd.DataFrame(np.fromfile(os.path.join(dirListing, \"hbw_peak_tresodat.bin\"), dtype = record_type1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function for setting market segments\n",
    "def market_segment(df):\n",
    "    '''\n",
    "    This function takes a dataframe and assigns the market segment to it that is used in the GGHM.\n",
    "    \n",
    "    Arguments: dataframe\n",
    "    \n",
    "    return: dataframe with market segment\n",
    "    '''\n",
    "    \n",
    "    if {'hhinc', 'auto_suff'}.issubset(df.columns):\n",
    "        \n",
    "        # create segments\n",
    "        df.loc[(df['hhinc'] <= 60000) & (df['auto_suff'] == 0), 'market_seg'] = 0\n",
    "        df.loc[(df['hhinc'] > 60000) & (df['auto_suff'] == 0), 'market_seg'] = 1\n",
    "        df.loc[(df['hhinc'] <= 60000) & (df['auto_suff'] == 1), 'market_seg'] = 2\n",
    "        df.loc[(df['hhinc'] > 60000) & (df['auto_suff'] == 1), 'market_seg'] = 3\n",
    "        df.loc[(df['hhinc'] <= 60000) & (df['auto_suff'] == 2), 'market_seg'] = 4\n",
    "        df.loc[(df['hhinc'] > 60000) & (df['auto_suff'] == 2), 'market_seg'] = 5\n",
    "        # set dtype\n",
    "        df['market_seg'] = df['market_seg'].astype('int8')\n",
    "    else:\n",
    "        print(\"The requisite fields are not there to run the function\")\n",
    "    \n",
    "    return(df)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 27.7 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# bring in the GGHMV4's trip_out and household file. \n",
    "hh = pd.read_csv(r\"c:\\personal\\IMM\\households_out.csv\")\n",
    "trips = pd.read_csv(r\"c:\\personal\\IMM\\trips_out.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 9.53 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# attach market segment for each household\n",
    "hh = market_segment(hh)\n",
    "\n",
    "# merge in household info so that we can estiamte the market segment for each later on\n",
    "trips = pd.merge(trips, hh, how = 'left', on = 'hhid')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# set dictionary of Dtypes\n",
    "dtype_trips = {}\n",
    "dtype_hh = {}\n",
    "\n",
    "# dictionary for storing hhold column dtypes\n",
    "dtype_hh = {'hhid':'int32',\n",
    "              'taz':'int16',\n",
    "              'hhinc':'int32',\n",
    "              'dtype':'int8',\n",
    "              'hhsize':'int8',\n",
    "              'nveh':'int8',\n",
    "              'auto_suff': 'int8'\n",
    "              }\n",
    "\n",
    "# dictionary for storing trips column dtypes\n",
    "dtype_trips = {'hhid':'int32',\n",
    "              'pid':'int8',\n",
    "              'tour_id':'int8',\n",
    "              'subtour_id':'int8',\n",
    "              'trip_id':'int8',\n",
    "              'activity_i':'category',\n",
    "              'activity_j':'category',\n",
    "              'taz_i': 'int16',\n",
    "              'taz_j': 'int16',\n",
    "              'tour_direction':'category',\n",
    "              'purpose': 'category',\n",
    "              'trip_direction': 'category',\n",
    "              'peak_factor': 'float64',\n",
    "               'taz':'int16',\n",
    "              'hhinc':'int32',\n",
    "              'dtype':'int8',\n",
    "              'hhsize':'int8',\n",
    "              'nveh':'int8',\n",
    "              'auto_suff': 'int8',\n",
    "              'market_seg': 'int8'}\n",
    "\n",
    "# dictionary for storing trips column dtypes for a processed trip file\n",
    "dtype_trips_processed = {'hhid':'int32',\n",
    "              'pid':'int8',\n",
    "              'tour_id':'int8',\n",
    "              'subtour_id':'int8',\n",
    "              'trip_id':'int8',\n",
    "              'activity_i':'category',\n",
    "              'activity_j':'category',\n",
    "              'taz_i': 'int16',\n",
    "              'taz_j': 'int16',\n",
    "              'tour_direction':'category',\n",
    "              'purpose': 'category',\n",
    "              'trip_direction': 'category',\n",
    "              'peak_factor': 'float64',\n",
    "              'hhinc': 'int32',\n",
    "              'market_seg': 'int8',\n",
    "              'dtype': 'int8',\n",
    "              'hhsize': 'int8',\n",
    "              'nveh': 'int8',\n",
    "              'auto_suff': 'int8',          \n",
    "              'veh_type': 'category'}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 9.3 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# set dtypes for trips dataframe to reduce memory requirements\n",
    "\n",
    "for key, value in dtype_trips.items():\n",
    "    trips[key] = trips[key].astype(value)\n",
    "\n",
    "for key, value in dtype_hh.items():\n",
    "    hh[key] = hh[key].astype(value)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Functions to assign vehicle trips"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def veh_type_assign(df, segment, sampledf, sampledf_all_mseg, veh_type):\n",
    "    \n",
    "    '''\n",
    "    This function takes the first trip purpose (hbw generally) and assigns a veh type to each household\n",
    "    \n",
    "    Args: The df that needs to be assigned a vehicle type\n",
    "          Segment counter\n",
    "          Dictionary name to collect DFs created for each zone within a market segment\n",
    "          Dictionary name to collect DFs across zones and market segments\n",
    "          File that has probabilities for the four veh type and 6 market segments for every zone\n",
    "    \n",
    "    Return: a dataframe that is the same length as the trip purpose df with veh type assigned to it\n",
    "    \n",
    "    '''\n",
    "\n",
    "    # subset the groups by market segment and sort. Then create\n",
    "    # the column indexes to slice the vehicle type dataframe.\n",
    "    # The melt the dataframe to get it ready for sampling \n",
    "    g1 = df.loc[df['market_seg'] == segment].sort_values('taz')\n",
    "    g2 = g1.groupby(['taz', 'market_seg']).size().reset_index(name = 'counts')\n",
    "\n",
    "    # get column indices to slice\n",
    "    segment = int(float(segment))\n",
    "    s_pos = segment*4\n",
    "    e_pos = segment*4+4\n",
    "    col_ind = list(range(s_pos, e_pos))\n",
    "    veh_type1 = veh_type.iloc[:, col_ind].reset_index()\n",
    "\n",
    "    # melt the dataframe to be sampled from.\n",
    "    veh_melt1 = pd.melt(veh_type1, id_vars = 'ggh_zone').sort_values('ggh_zone')\n",
    "\n",
    "    # get unique zones to loop over\n",
    "    uniq_zones = g1['taz'].unique()\n",
    "\n",
    "    for zone in uniq_zones :\n",
    "\n",
    "        # sample to produce the required records from the distribution in the zone of question\n",
    "        # get the weights to sample from\n",
    "        wts = veh_melt1.loc[veh_melt1['ggh_zone'] == zone]\n",
    "        wts = wts['value']\n",
    "\n",
    "        # number of observations to be sampled\n",
    "        no_sam = g2.loc[g2['taz'] == zone]\n",
    "        no_sam = int(float(no_sam.iloc[0]['counts']))\n",
    "\n",
    "        # sample with replacement and collect each individual zones results in a dictionary\n",
    "        sdf = veh_melt1.sample(n = no_sam, weights = wts, replace = True, random_state = seed)\n",
    "        columns = ['ggh_zone', 'value']\n",
    "        sdf.drop(columns, axis = 1, inplace = True)\n",
    "\n",
    "        # concatenate to the appropriate households within the zone and belonging to the market segment\n",
    "        # in question\n",
    "        sdf.reset_index(drop=True, inplace=True)\n",
    "        hh_df_zone = df.loc[(df['taz'] == zone) & (df['market_seg'] == segment) ]\n",
    "        hh_df_zone.reset_index(drop=True, inplace=True)\n",
    "        sdf = pd.concat([hh_df_zone, sdf], axis = 1)\n",
    "        \n",
    "        # collect in dictionary\n",
    "        sampledf[zone] = sdf\n",
    "\n",
    "    # now concatenate across segments\n",
    "    sampledf_all_mseg[segment] = pd.concat(sampledf.values(), ignore_index = True)\n",
    "    \n",
    "        # run check            \n",
    "    try:\n",
    "        assert len(sampledf_all_mseg[segment]) == len(g1)\n",
    "\n",
    "    except AssertionError as e:\n",
    "        e.args += ('The length of the sampled dataframe is wrong.')\n",
    "        raise\n",
    "\n",
    "    # get master dataframe and cbind it to the hbw_trips df\n",
    "    veh_type_df = pd.concat(sampledf_all_mseg.values(), ignore_index = True)\n",
    "    veh_type_df.rename(columns = {'variable': 'veh_type'}, inplace = True)\n",
    "\n",
    "    return(veh_type_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 4min 37s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# dictionary to collect dataframe by market segment\n",
    "zonedf_mseg = {}\n",
    "\n",
    "# run for all market segments in the household file\n",
    "for segment in hh['market_seg'].unique():\n",
    "   \n",
    "    # collect the dataframes for each zone\n",
    "    zonedf = {}    # for each zone\n",
    "    veh_assigned = veh_type_assign(hh, segment, zonedf, zonedf_mseg, veh_type)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Attach the VehType information to the trips file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 9.47 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "veh_assigned = veh_assigned[['hhid', 'veh_type']]\n",
    "trips = pd.merge(trips, veh_assigned, on = ['hhid', 'hhid'], how = 'left')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Now start with Bill's files. Currently, it is only HBW"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Start with the first batch (156 zones) that Bill has computed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp_df1 = hbw_prob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Production Zone</th>\n",
       "      <th>Destination Zone</th>\n",
       "      <th>Market Segment</th>\n",
       "      <th>Segment 0 Drive-Alone Non-Toll</th>\n",
       "      <th>Segment 0 Drive-Alone Toll Trips</th>\n",
       "      <th>Segment 0 2-Person Non-Toll HOV Trips</th>\n",
       "      <th>Segment 0 2-Person Non-Toll non-HOV Trips</th>\n",
       "      <th>Segment 0 2-Person Toll HOV Trips</th>\n",
       "      <th>Segment 0 2-Person Toll Non-HOV Trips</th>\n",
       "      <th>Segment 0 3+ Person Non-Toll HOV Trips</th>\n",
       "      <th>...</th>\n",
       "      <th>Segment 3 Park-n-Ride Access - TTC Subway #3</th>\n",
       "      <th>Segment 3 Park-n-Ride Access - TTC Subway #4</th>\n",
       "      <th>Segment 3 Kiss-n-Ride Access - TTC Subway #1</th>\n",
       "      <th>Segment 3 Kiss-n-Ride Access - TTC Subway #2</th>\n",
       "      <th>Segment 3 Kiss-n-Ride Access - TTC Subway #3</th>\n",
       "      <th>Segment 3 Kiss-n-Ride Access - TTC Subway #4</th>\n",
       "      <th>Segment 3 Uber Access - TTC Subway #1</th>\n",
       "      <th>Segment 3 Uber Access - TTC Subway #2</th>\n",
       "      <th>Segment 3 Uber Access - TTC Subway #3</th>\n",
       "      <th>Segment 3 Uber Access - TTC Subway #4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1001</td>\n",
       "      <td>1001</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1001</td>\n",
       "      <td>1001</td>\n",
       "      <td>3</td>\n",
       "      <td>46.399994</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.7</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1001</td>\n",
       "      <td>1001</td>\n",
       "      <td>4</td>\n",
       "      <td>44.499996</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1001</td>\n",
       "      <td>1001</td>\n",
       "      <td>5</td>\n",
       "      <td>74.400002</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.8</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1001</td>\n",
       "      <td>1001</td>\n",
       "      <td>6</td>\n",
       "      <td>73.099998</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.6</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 211 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Production Zone  Destination Zone  Market Segment  \\\n",
       "0             1001              1001               1   \n",
       "1             1001              1001               3   \n",
       "2             1001              1001               4   \n",
       "3             1001              1001               5   \n",
       "4             1001              1001               6   \n",
       "\n",
       "   Segment 0 Drive-Alone Non-Toll  Segment 0 Drive-Alone Toll Trips  \\\n",
       "0                        0.000002                               0.0   \n",
       "1                       46.399994                               0.0   \n",
       "2                       44.499996                               0.0   \n",
       "3                       74.400002                               0.0   \n",
       "4                       73.099998                               0.0   \n",
       "\n",
       "   Segment 0 2-Person Non-Toll HOV Trips  \\\n",
       "0                                    0.0   \n",
       "1                                    0.0   \n",
       "2                                    0.0   \n",
       "3                                    0.0   \n",
       "4                                    0.0   \n",
       "\n",
       "   Segment 0 2-Person Non-Toll non-HOV Trips  \\\n",
       "0                                        0.0   \n",
       "1                                        3.7   \n",
       "2                                        2.5   \n",
       "3                                        2.8   \n",
       "4                                        2.6   \n",
       "\n",
       "   Segment 0 2-Person Toll HOV Trips  Segment 0 2-Person Toll Non-HOV Trips  \\\n",
       "0                                0.0                                    0.0   \n",
       "1                                0.0                                    0.0   \n",
       "2                                0.0                                    0.0   \n",
       "3                                0.0                                    0.0   \n",
       "4                                0.0                                    0.0   \n",
       "\n",
       "   Segment 0 3+ Person Non-Toll HOV Trips  \\\n",
       "0                                     0.0   \n",
       "1                                     0.0   \n",
       "2                                     0.0   \n",
       "3                                     0.0   \n",
       "4                                     0.0   \n",
       "\n",
       "                   ...                    \\\n",
       "0                  ...                     \n",
       "1                  ...                     \n",
       "2                  ...                     \n",
       "3                  ...                     \n",
       "4                  ...                     \n",
       "\n",
       "   Segment 3 Park-n-Ride Access - TTC Subway #3  \\\n",
       "0                                           0.0   \n",
       "1                                           0.0   \n",
       "2                                           0.0   \n",
       "3                                           0.0   \n",
       "4                                           0.0   \n",
       "\n",
       "   Segment 3 Park-n-Ride Access - TTC Subway #4  \\\n",
       "0                                           0.0   \n",
       "1                                           0.0   \n",
       "2                                           0.0   \n",
       "3                                           0.0   \n",
       "4                                           0.0   \n",
       "\n",
       "   Segment 3 Kiss-n-Ride Access - TTC Subway #1  \\\n",
       "0                                           0.0   \n",
       "1                                           0.0   \n",
       "2                                           0.0   \n",
       "3                                           0.0   \n",
       "4                                           0.0   \n",
       "\n",
       "   Segment 3 Kiss-n-Ride Access - TTC Subway #2  \\\n",
       "0                                           0.0   \n",
       "1                                           0.0   \n",
       "2                                           0.0   \n",
       "3                                           0.0   \n",
       "4                                           0.0   \n",
       "\n",
       "   Segment 3 Kiss-n-Ride Access - TTC Subway #3  \\\n",
       "0                                           0.0   \n",
       "1                                           0.0   \n",
       "2                                           0.0   \n",
       "3                                           0.0   \n",
       "4                                           0.0   \n",
       "\n",
       "   Segment 3 Kiss-n-Ride Access - TTC Subway #4  \\\n",
       "0                                           0.0   \n",
       "1                                           0.0   \n",
       "2                                           0.0   \n",
       "3                                           0.0   \n",
       "4                                           0.0   \n",
       "\n",
       "   Segment 3 Uber Access - TTC Subway #1  \\\n",
       "0                                    0.0   \n",
       "1                                    0.0   \n",
       "2                                    0.0   \n",
       "3                                    0.0   \n",
       "4                                    0.0   \n",
       "\n",
       "   Segment 3 Uber Access - TTC Subway #2  \\\n",
       "0                                    0.0   \n",
       "1                                    0.0   \n",
       "2                                    0.0   \n",
       "3                                    0.0   \n",
       "4                                    0.0   \n",
       "\n",
       "   Segment 3 Uber Access - TTC Subway #3  \\\n",
       "0                                    0.0   \n",
       "1                                    0.0   \n",
       "2                                    0.0   \n",
       "3                                    0.0   \n",
       "4                                    0.0   \n",
       "\n",
       "   Segment 3 Uber Access - TTC Subway #4  \n",
       "0                                    0.0  \n",
       "1                                    0.0  \n",
       "2                                    0.0  \n",
       "3                                    0.0  \n",
       "4                                    0.0  \n",
       "\n",
       "[5 rows x 211 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "temp_df1.head(n=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "trad_auto = 0\n",
    "av_auto = 1\n",
    "trad_uber = 2\n",
    "av_uber = 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The trips_out file contains a peak hour factor column that decides whether a trip is sampled\n",
    "# in the peak or off-peak period. In order to discretely select the peak records and vice-versa\n",
    "# an uniform random number generator is run and the values are attached to the trips_out file.\n",
    "# If the (1-peak_factor) value in the record is greater than that of the random value than \n",
    "# the record is in the off-peak and vice-versa.\n",
    "\n",
    "np.random.seed(seed)\n",
    "random = pd.DataFrame(np.random.uniform(size = len(trips)))\n",
    "random.columns = ['rnum']\n",
    "\n",
    "# attach the random number generator and calculate peak_flag. A value of 1 in this flag\n",
    "# means that this is a peak period trip record.\n",
    "trips = pd.concat([trips, random], axis = 1)\n",
    "trips['peak_flag'] = np.where((1 - trips['peak_factor']) > trips['rnum'], 0, 1)\n",
    "trips[['peak_flag']] = trips[['peak_flag']].astype('int8')   # save some memory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Automatic pdb calling has been turned ON\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'trips' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-1-43b2cbd775f9>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[0mget_ipython\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmagic\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'pdb on'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mtrips\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mhead\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m: name 'trips' is not defined"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> \u001b[1;32m<ipython-input-1-43b2cbd775f9>\u001b[0m(2)\u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32m      1 \u001b[1;33m\u001b[0mget_ipython\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmagic\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'pdb on'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[1;32m----> 2 \u001b[1;33m\u001b[0mtrips\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mhead\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[0m\n",
      "*** NameError: name 'trips' is not defined\n"
     ]
    }
   ],
   "source": [
    "%pdb on\n",
    "trips.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trips_hbw = trips.loc[(trips['purpose'] == 'HBW') & (trips['peak_flag'] == 1)]\n",
    "trips_hbw.head(n=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gr = trips_hbw.groupby(['taz_i', 'taz_j', 'market_seg', 'veh_type']).size().reset_index(name = 'counts')\n",
    "gr.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gr.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "attempt = trips_hbw.groupby(['taz_i','hhid','veh_type']).size().reset_index()\n",
    "attempt = attempt.loc[attempt['taz_i'] == 1001]\n",
    "attempt1 = attempt.groupby('veh_type').size().reset_index()\n",
    "attempt1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = attempt.loc[attempt['taz_i'] == 1001]\n",
    "b = a.groupby('veh_type').size().reset_index(name = 'counts')\n",
    "b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "h = trips.loc[trips['taz_i'] == 1001]\n",
    "h = h.drop_duplicates(subset='hhid', keep=\"last\")\n",
    "len(h)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "h.head(n=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "hh = market_segment(hh)\n",
    "hh.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "h1 = hh.groupby(['market_seg', 'taz']).size().reset_index().sort_values('taz')\n",
    "h1.head(n=10)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "# group the household by taz and market segment\n",
    "h_gr = hh.groupby(['taz', 'market_seg']).size().reset_index(name = 'counts')\n",
    "\n",
    "# transfer the vehicle type to the grouped segment so that we can sample from it.\n",
    "gr = pd.merge(h_gr, veh_type, left_on = ['taz', 'market_seg'], right_on = ['taz', 'market_seg'], how = 'left')\n",
    "\n",
    "# group by and sample the dataframe using the weights and sort values to concatenate to the household dataframe\n",
    "s1 = gr.groupby(['taz', 'market_seg']).apply(lambda x: x.sample(n = x['counts'].min(), weights = x['value'], random_state = seed, replace = True)).reset_index(drop=True).sort_values(['taz', 'market_seg'])\n",
    "\n",
    "# attach the market segment to the veh_type. Also add in an integer based market seg\n",
    "s1['veh_type1'] = s1['variable'].str.split('_').str[2]\n",
    "s1['veh_type2'] = s1['variable'].str.split('_').str[3]\n",
    "s1['veh_type'] = s1['veh_type1'] + '_' + s1['veh_type2']\n",
    "\n",
    "# drop unncessary columns\n",
    "s1 = s1['veh_type']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "# concatenate the sampled vehicle information to each household\n",
    "hh_veh = hh.sort_values(['taz', 'market_seg'])\n",
    "hh_veh.reset_index(drop=True, inplace=True)\n",
    "s1.reset_index(drop = True, inplace = True)\n",
    "hh_veh = pd.concat([hh_veh, s1], axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}