Refining.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from scipy import stats\n",
    "import matplotlib.pyplot as plt\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\norouziy\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\IPython\\core\\interactiveshell.py:3020: DtypeWarning: Columns (3,6,18) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv('./travistorrent_8_2_2017.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# selecting the required columns from dataset\n",
    "df = df[['gh_pushed_at', 'gh_first_commit_created_at', 'tr_status', 'git_diff_src_churn', 'git_diff_test_churn', 'gh_sloc', 'gh_lang', 'gh_team_size', 'tr_build_id', 'tr_log_num_tests_run', 'tr_log_num_tests_ok', 'tr_log_num_tests_failed', 'gh_test_lines_per_kloc', 'gh_project_name', 'gh_test_cases_per_kloc']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not os.path.exists('figs'):\n",
    "    os.makedirs('figs')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3XucXXV57/HPM3vu19wmIUwSEyDlqlwaKJQWK4gKbblYbfX0pdRyGtvSFmvPqdBWq7U91bZoyzmtlhYVtRUpFkGLlYugRytIwFwJkItAhoRcDMkkmWQms/fTP9ZvT3aGvWevuax9mfV9v17z2nv99lp7Pys7mSe/u7k7IiIi42modgAiIlL7lCxERKQsJQsRESlLyUJERMpSshARkbKULEREpCwlCxERKUvJQkREylKyEBGRshqrHcBUzJs3z5cuXVrtMERE6sqTTz65x917J3JNXSeLpUuXsmrVqmqHISJSV8zshYleo2YoEREpS8lCRETKUrIQEZGylCxERKQsJQsRESlLyUJERMpSshARkbJSnyyGR3J8+YkXyeW0vayISCmpTxbf27yHD3xlHY9t/XG1QxERqVmpTxYHhkYA2LL7YJUjERGpXalPFoeH88niUJUjERGpXUoWw1kAtu5RshARKSX1yWLwaEgWaoYSESkp9ckiX7N4ad9hjoTEISIix0t9shgMycIdfqSmKBGRopQsho/VJraqk1tEpKjUJ4sjR7P0drUA6rcQESmlrnfKmw6DwyPM7WimqcE010JEpAQli+EsrU0Z5nW2aPisiEgJqW+GOjycpb05w0m9HWzdfQh3rRElIjJW6pPFYD5ZzOvg4NAIuw8MVTskEZGak2iyMLPnzWydma02s1WhbI6ZPWhmm8Lj7FBuZnarmW02s7Vmdl6SseUdOZqlrbmRk+d3ArBZ/RYiIq9SiZrFG9z9HHdfEY5vAh529+XAw+EY4ApgefhZCXyqArExOJylramBhT2tAKpZiIgUUY1mqKuBO8LzO4BrCso/75HHgFlmtjDpYAaHR2hvbqStOerrPzysWdwiImMlnSwceMDMnjSzlaFsgbvvAAiP80N5H7Ct4Nr+UJaow0eztDVn6GjOAMdP0hMRkUjSQ2cvdvftZjYfeNDMnhnnXCtS9qqhSSHprARYsmTJlII7ms1xNOu0N2VoG00WI1N6TxGRmSjRmoW7bw+Pu4B7gAuAnfnmpfC4K5zeDywuuHwRsL3Ie97m7ivcfUVvb++U4jscFg5sa87QnGkg02CqWYiIFJFYsjCzDjPryj8H3gSsB+4DrgunXQfcG57fB7w7jIq6ENifb65KSr5/oq05g5nR3pRRshARKSLJZqgFwD1mlv+cf3X3/zSzJ4C7zOx64EXg7eH8+4Ergc3AIPCeBGMDjvVPtIcmqLbmjDq4RUSKSCxZuPtW4Owi5T8GLitS7sANScVTTL5/oq0p+mNob86MboYkIiLHpHoG9+FX1SwaR/fkFhGRY9KdLAo6uAE6mtVnISJSTKqTRT4xtDUd67NQshARebVUJ4v8ntutIVm0q4NbRKSoVCeLXFiOPNMQzQdsb25k8Kj6LERExkp1ssjmosdMNLxXQ2dFREpIdbLI5aKaRUP4U2hvynBoSMlCRGSsVCeL7KuaoTIcPpodTSIiIhJJd7IISSHfDNXeEk3OOzKi2oWISKFUJ4t8B3dDQc0CtEy5iMhYqU4WY2sW+fkW6uQWETleqpNFvmuiwY4NnQXVLERExkp3shg7GkobIImIFJXqZDF2NFR+jSg1Q4mIHC/dySJfszB1cIuIjCfVySLfDFW43AegPS1ERMZIdbIYbYYKNYuOlqhmcWhIfRYiIoVSnSyOdXBHyaIzTMo7eETJQkSkULqThUPIEwB0NDdiBgeOHK1eUCIiNSjVySLrPtpfAVENo6ulkQHVLEREjpPqZJHL+ehIqLyu1iYGVLMQETlOqpNFNnd8zQKgq7WRA6pZiIgcJ93Jwn10JFRed2uT+ixERMZIdbLI5Xx0JFReV2sjA4dVsxARKZTqZDG2gxtCM9SQahYiIoVSnSzGDp0F6G5rUp+FiMgY6U4WRUdDRR3c7tpaVUQkL/FkYWYZM/uhmX09HC8zs8fNbJOZfdnMmkN5SzjeHF5fmnRsxUdDNZHNOYe1PpSIyKhK1CxuBDYWHH8c+KS7LwdeAa4P5dcDr7j7KcAnw3mJynrxmgWgTm4RkQKJJgszWwT8PPDP4diAS4G7wyl3ANeE51eHY8Lrl4XzE5MrUbMALfkhIlKobLIwsw4zawjPf8LMrjKzppjv/7fAHwK5cDwX2Ofu+f+29wN94XkfsA0gvL4/nJ+YrFN0NBSgJT9ERArEqVl8B2g1sz7gYeA9wOfKXWRmvwDscvcnC4uLnOoxXit835VmtsrMVu3evbtcGOOKOriPL+sOyUI1CxGRY+IkC3P3QeCtwP9192uBM2JcdzFwlZk9D9xJ1Pz0t8AsM2sM5ywCtofn/cBigPB6D7B37Ju6+23uvsLdV/T29sYIo7RckT6L7tAMpZqFiMgxsZKFmV0E/CrwH6GscZzzAXD3m919kbsvBd4BfMvdfxV4BHhbOO064N7w/L5wTHj9W57w+NVSo6FANQsRkUJxksWNwM3APe6+wcxOIvqFP1kfAN5vZpuJ+iRuD+W3A3ND+fuBm6bwGbEUq1l0jTZDqWYhIpJXtoYALHD3q/IH7r7VzP7/RD7E3R8FHs1fD1xQ5JwjwNsn8r5TVaxm0d6cwUxbq4qIFIpTs7g5ZlndyTqvWkjQzOhobuSgkoWIyKiSNQszuwK4Eugzs1sLXuoGZsRv0lzOyRQZg9XZ0qiahYhIgfGaobYDq4CrgMLhrweA308yqEop1gwF0NGS4dCQlvsQEckrmSzcfQ2wxszuAQ65exaitZ6AlgrFl6icO8UmiXe2NHJANQsRkVFx+iweANoKjtuAh5IJp7JyRXbKA+hQM5SIyHHiJItWdz+YPwjP25MLqXJKNUOpz0JE5HhxksUhMzsvf2BmPwkcTi6kyik2GgpCM5TmWYiIjIozz+J9wL+ZWX5ZjoXAryQXUuWUGg3V0dLIoWElCxGRvDjLdjxhZqcBpxIt9veMu8+ItTBKNkO1qhlKRKRQnCXK24mW6LjR3dcBS8OKsnWv2HIfEDVDHc06QyMaPisiAvH6LD4LDAMXheN+4M8Ti6iCSiWLjuYMAAfVbyEiAsRLFie7+18BRwHc/TDF956oO6WboaKVZzUxT0QkEidZDJtZG2EjIjM7GRhKNKoKyZUcDRVqFuq3EBEB4o2G+lPgP4HFZvYvRJsa/VqSQVVKdpzRUIBGRImIBHFGQz1oZk8BFxI1P93o7nsSj6wCsjkvOc8C1GchIpI33qqzp7n7MwUT8naExyVmthjY6+4vJB5hgkot9zGaLNQMJSICjF+zeD+wErilxOtzzWyNu79r+sOqjNKrzoZmKCULERFg/FVnV4bHN5Q6x8weSCKoSsk5xVedbVXNQkSkUNk+CzNrAn4LuCQUPQr8o7sfdfc3JRhb4nLuZIqMB+to1j7cIiKF4oyG+hTQBPxDOH5XKPufSQVVKdFoqFfXLDINpsUERUQKxEkW57v72QXH3zKzNUkFVEm5EqOhAHramth/eEYsgSUiMmVxJuVlw0Q8AMzsJGBGTG3OlhgNBdDV2sjAESULERGIV7P438AjZraVaJ7Fa4D3JBpVhZQaDQWqWYiIFBo3WZhZA9FGR8s5fonyGbLcR+lmqO62JrbtHaxwRCIitWncZih3zwG3uPuQu6919zUzJVFAWBuqxJKIPW1NDKhmISICxOuzeMDMfsmKTUioc6VGQwF0t6oZSkQkL06fxfuBDqKO7vzy5O7u3YlGlrBczoHiq85CVLM4NJxlJJujsdhkDBGRFImzkGBXJQKptKxHyaJkzaIt+qMZODLCnI7misUlIlKLYv2X2czeamafMLNbzOyamNe0mtkPzGyNmW0ws4+E8mVm9riZbTKzL5tZcyhvCcebw+tLJ3tTcWRj1CwA9VuIiBBvD+5/AH4TWAesB37TzP4+xnsPAZeGCX3nAG8xswuBjwOfdPflwCvA9eH864FX3P0U4JPhvMTk8jWLUqOhwm556rcQEYlXs3g98GZ3/6y7fxa4Evi5chd55GA4bAo/DlwK3B3K7wDyNZWrwzHh9cuS7FTP1yxKNUP1tIeahSbmiYjEShbPAksKjhcDa+O8uZllzGw1sAt4ENgC7HP3/KJL/UBfeN4HbAMIr+8H5hZ5z5VmtsrMVu3evTtOGEWFXEGpdJRvhlLNQkQkXrKYC2w0s0fN7FHgaaDXzO4zs/vGu9Dds+5+DrAIuAA4vdhp4bHYr21/VYH7be6+wt1X9Pb2xgi/uPxoqHLNUAOHtZigiEicobMfmuqHuPu+kGguBGaZWWOoPSwCtofT+olqLf1m1gj0AHun+tmlZMv0WahmISJyTJyhs9+ezBubWS9wNCSKNuCNRJ3WjwBvA+4ErgPuDZfcF46/H17/lru/qmYxXUbnWZRoh2ptaqApY0oWIiLEq1lM1kLgDjPLEDV33eXuXzezp4E7zezPgR8Ct4fzbwe+YGabiWoU70gwtrI1CzOLlvxQB7eISHLJwt3XAucWKd9K1H8xtvwI8Pak4hmr3Ggo0JIfIiJ5JTu4zezh8JjofIdqyeWix1KT8iBaeVaT8kRExq9ZLDSz1wNXmdmdjBmt5O5PJRpZwvKT8sbJFXS3NbF/cLhCEYmI1K7xksWHgJuIRix9Ysxr+cl1datcnwVEI6K0p4WIyDjJwt3vBu42sw+6+0crGFNFlBsNBdDd2qg+CxER4g2d/aiZXQVcEooedfevJxtW8uLWLPYfPoq7MwO38xARiS3OQoJ/CdxINHP7aeDGUFbXsnFqFm1NZHPO4HC2UmGJiNSkOENnfx44J2yxipndQTQ/4uYkA0tafjRUuZoFRLO4O1qSnJIiIlLb4m4BN6vgeU8SgVTasWao0ueMrg+liXkiknJx/rv8l8APzewRouGzl1DntQo4NnR2vL6I0ZrFoJKFiKRbnA7uL4VFAM8nShYfcPeXkw4sabk4M7gLtlYVEUmzWA3x7r6DaKG/GSNbZoly0MqzIiJ5cfssZpysx5lnoX24RUQgxckizmiortao4qWahYik3bjJwswazGx9pYKppDijoRozDXS2aBa3iMi4ySLMrVhjZkvGO68exVnuA2BuZzO7Dw5VIiQRkZoVp4N7IbDBzH4AHMoXuvtViUVVAbkYfRYAr5nbwQs/PjTuOSIiM12cZPGRxKOogjijoQCWzW3nqRde0fpQIpJqsfbgNrPXAMvd/SEzawcyyYeWrLg1i6XzOjg4NMLug0PM72qtRGgiIjUnzkKCvwHcDfxjKOoDvppkUJWQjTEaCmDZvA4Ant+jfS1EJL3iDJ29AbgYGABw903A/CSDqoQ4o6GgMFmo30JE0itOshhy99G9Rc2skWinvLoWdzRU36w2GhuMH6mTW0RSLE6y+LaZ/RHQZmaXA/8GfC3ZsJIXt4O7MdPAkjntGhElIqkWJ1ncBOwG1gHvBe4H/iTJoCohbgc3wImz2tix/0jSIYmI1Kw4o6FyYcOjx4man5519/pvhsonizI1C4AF3a3815Y9SYckIlKzyiYLM/t54NPAFqIlypeZ2Xvd/RtJB5ek0dFQMWoWC3ta2XVgiGzOyzZbiYjMRHEm5d0CvMHdNwOY2cnAfwD1nSw8Xp8FwIKeVrI5Z8/BIRZ0a66FiKRPnD6LXflEEWwFdiUUT8VkQ9WiMUayWBgSxMvqtxCRlCqZLMzsrWb2VqJ1oe43s18zs+uIRkI9Ue6NzWyxmT1iZhvNbIOZ3RjK55jZg2a2KTzODuVmZrea2WYzW2tm503TPRY1kh8NlSmfLE7oiZKFOrlFJK3Gq1n8YvhpBXYCrwd+jmhk1OwY7z0C/IG7nw5cCNxgZmcQja562N2XAw+HY4ArgOXhZyXwqYnezETkh87GqVnkk8XOASULEUmnkn0W7v6eqbxx2Ip1R3h+wMw2Ei0VcjVR0gG4A3gU+EAo/3wYafWYmc0ys4XhfabdSMxJeQBz2ptpyphqFiKSWnFGQy0DfhdYWnj+RJYoN7OlwLlEw28X5BOAu+8ws/zSIX3AtoLL+kNZIskiN4GaRUODsaC7VTULEUmtOKOhvgrcTtRXkZvoB5hZJ/AV4H3uPjDOMt/FXnjVfA4zW0nUTMWSJZPfk2kk5gzuvIU9rby07/CkP09EpJ7FSRZH3P3Wyby5mTURJYp/cfd/D8U7881LZraQYyOr+oHFBZcvAraPfU93vw24DWDFihWTnhyYnzMRd4+KxXPaeWzLjyf7cSIidS3O0Nm/M7M/NbOLzOy8/E+5iyz6LXw7sNHdP1Hw0n3AdeH5dcC9BeXvDqOiLgT2J9VfAVHNYiIT7JbMaWfHwBGGRrJJhSQiUrPi1CxeC7wLuJRjzVAejsdzcbhunZmtDmV/BHwMuMvMrgdeBN4eXrsfuBLYDAwCU+pgLyeby8Xqr8hbPLsdd3jplcOc1NuZYGQiIrUnTrK4FjipcJnyONz9uxTvhwC4rMj5TrR3RkWM5DzWUh95S+a2A/Di3kElCxFJnTjNUGuAWUkHUmm5nMeakJe3ZE6ULLbt1Y55IpI+cWoWC4BnzOwJYChfOJGhs7VoJOcTaobq7WyhpbGBF5UsRCSF4iSLP008iiqY6AqyDQ3G4jntShYikkpx9rP4diUCqbSoZhGnFe6YRbPb6H9Fcy1EJH3K/rY0swNmNhB+jphZ1swGKhFckrI5Z4K5ggVd0b4WIiJpE6dm0VV4bGbXABckFlGFTKZmsaC7hT0HhxjJ5mjMTDDTiIjUsQn/xnP3r1J+jkXNy01i17v53a24w48PTWgUsYhI3YuzkOBbCw4bgBUUWbOp3oxMcFIeMLpL3s6BI9oxT0RSJc5oqF8seD4CPE+0nHhdm8x+2vO7WgDYOaB+CxFJlzh9Fokuu1EtE51nAcdqFrsOaKlyEUmXksnCzD40znXu7h9NIJ6KiUZDTSxZzOtsxkw1CxFJn/FqFoeKlHUA1wNzgbpOFiPZidcsGjMNzOtsYZc2QRKRlBlvW9Vb8s/NrAu4kWgl2DuBW0pdVy+yPvE+C4j6LbRjnoikzbhDZ81sjpn9ObCWKLGc5+4fcPdd411XD7KTmGcBUb/Fy2qGEpGUKfnb0sz+GngCOAC81t0/7O6vVCyyhE1086O8k3s72LL7IMMjE95hVkSkbo33X+s/AE4E/gTYXrDkx4GZsdzHxOdZALxu0SyGR3I8t/NAAlGJiNSm8fosZvR6FiPZiY+GAjh7UbS1x5r+fZzV1zPdYYmI1KQZnRDGk/OJj4YCWDynjdntTazdtj+BqEREalNqk8Vk+yzMjNctmsWa/n0JRCUiUptSmyyyk5jBnXfmid1s3qVObhFJj9Qmi5Gsk5nE0FmA0xZ2M5Jztuw+OM1RiYjUptQmi6nULE47Idri49mXNSJKRNIhtcliZBJrQ+Utm9dBU8Z4RslCRFIitclisqOhAJoyDZzc28kzL9f9dBMRkVhSmyxGsrlJjYbKO+2ELjVDiUhqpDZZTKXPAuDUE7rZsf8I+wePTmNUIiK1KbXJYiTnZDJTq1kAPKtlP0QkBRJLFmb2GTPbZWbrC8rmmNmDZrYpPM4O5WZmt5rZZjNba2bnJRVXXjbnZGwqNYv8iCj1W4jIzJdkzeJzwFvGlN0EPOzuy4GHwzHAFcDy8LMS+FSCcQGT21a10MKeVrpaGzUiSkRSIbFk4e7fAfaOKb4auCM8vwO4pqD88x55DJhlZguTii2Xc4BJT8qDaNmP00/oVie3iKRCpfssFrj7DoDwOD+U9wHbCs7rD2WJGAnJonEKfRYQNUU9+/KB0eQjIjJT1UoHd7Hf2kV/A5vZSjNbZWardu/ePakPy47WLKaWLM5ePIsDQyPq5BaRGa/SyWJnvnkpPOa3Z+0HFhectwjYXuwN3P02d1/h7it6e3snFcRILloAcCp9FgAXnzIXgO9t3jOl9xERqXWVThb3AdeF59cB9xaUvzuMiroQ2J9vrkpCvmbRMIXRUAALe9o4qbeD7ypZiMgMl+TQ2S8B3wdONbN+M7se+BhwuZltAi4PxwD3A1uBzcA/Ab+dVFwwfX0WAD9zyjx+8KO9Wq5cRGa0ktuqTpW7v7PES5cVOdeBG5KKZazcNPVZAPzcqb18/vsv8L0te3jDqfPLXyAiUodqpYO7okZrFtOQLC4+ZR6dLY18c/3LU34vEZFalcpkkZ2GeRZ5LY0ZLj1tPg88vZORrJqiRGRmSmWymM6aBcAbz1jA3kPDbNyhIbQiMjOlMllkw9DZyW5+NNa5i2cBsKZ/37S8n4hIrUlpsogep6tmsWh2G7Pbm1irZCEiM1Qqk0V+Ut50jIaCaJ2osxfPYm3//ml5PxGRWpPKZJGd5j4LgNctmsVzOw/whe8/z1F1dIvIDJPKZDEyjfMs8n52+TzMjA/eu4F/f6p/2t5XRKQWpDJZHKtZTN/tn790Dhs+8mZ+YkEnX3jsBaJ5hiIiM0Mqk8VINqwNNc1339qU4V0Xvob1Lw3w0MZd5S8QEakTqUwWOZ/+mkXeL/3kIs48sZsb/vUp1mzT6CgRmRlSmSyS6LPIa29u5IvX/xSz25v44L3r+dYzO9l/+Oi0f46ISCWlMllkp2k/i1JmdzRz0xWnsbZ/P7/+uVW887bH2D+ohCEi9SuxVWdrWb7PIomaRd415/QxPJJjaCTHR7/+NH9x/9P81dvOTuzzRESSlMpkMV3bqo7HzPiV85cAsHX3Ib742Av83mXLWTS7PbHPFBFJSiqboaZ7IcFy3vv6kzCDP/nqem2SJCJ1KZU1i/xoqCRrFoUW9rTxkavO4o/uWcfvfukpBoezdLc18f/eeS42xa1dRUQqIZXJIt9nkcTQ2VL+x08t4cjRLH/29adHy645p4/Lz1jA0WyOgcNHmdvZUrF4REQmIpXJYrTPYhr24J6IX/+ZZcztbKajuZGP/+czvP+u1bzn4mV8c/3LvLD3EH921VlkGoxrz+2btuXTRUSmQyqTRaX7LApdfU4fAKee0MX771rNrQ9vYkF3Cyf2tPGHX1kLwFMvvsKfXX1WxZrJRETKSWWyGN38qIr9BYvntHPXey/iyNEczY0N7Bsc5vEf7WX1tn3c9p2trO3fzxkLu7npitOY3dFctThFRCClyaKaNYtCZkZbcwaAuZ0tXPnahVxx1gmcvrCLTz+6lbue3DY6wU9EpJpSOXS2Wn0WcZgZ1567iG/+/iX8wutO5PPff563f/q/ePRZLUwoItWT6mRR7ZpFOb976Sk0mLFl9yFWfuFJPvHgc2zedbDaYYlIClk977uwYsUKX7Vq1YSve3r7AKu37eOXVyyiMVP7+fKVQ8O878ur+c6m3bjD6Qu7WT6/k1Pmd/Lavh5+4oQu+ma1VTtMEakTZvaku6+Y0DVpTBb1aufAEe5ft4P71+1g58AQL+4dBKClsYEb3nAKv7xiMSf0tFY5ShGpdUoWKbNz4Ajb9g7y6W9v4aGNu2htauDXfnoZ3W2NdLU28cbT57OwRzUOETmekkWKPb/nEB/+2gYefXb3aFljg/Ebl5zEsnkdtDQ28OYzT6C1KVPFKEWkFkwmWdTU0Fkzewvwd0AG+Gd3/1iVQ6obS+d18Ln3XMDQSBZ32L7vMJ98aBOfenTL6DnL5nVw9qIezl82h7eceQI7B4bom91GT1vTlD576+6DdLY0Mr9bTWAiM1XN1CzMLAM8B1wO9ANPAO9096dLXaOaRXm7Bo4wnM3x3M4D/O1Dm9g1MMTLA0dGX+/tamF+VwttTRleu6gHgGd2HOCZlwcA6G5r4k1nLOCNpy+go6WRDdv3c+vDmxkayXL+0jk0Zhr42prtmMGstiaWz+/izL5ujKmNNDvthC5OX9hN4bzJuZ3NalYTmQZ13QxlZhcBH3b3N4fjmwHc/S9LXaNkMXHuzsYdB3jw6Z3M6WjivjXbyTkMDmfpDx3mC3pauWDZHBobjG17B/nu5j0czR77e3LO4lmc3NvJf23Zw+BwlmvP7WNORzM7B47w5Auv8NIrh6cUY9adweFs0deWzeuo+SHPIpXwe5ct5xfPPnFS19Z7M1QfsK3guB/4qbEnmdlKYCXAkiVLKhPZDGJmnHFiN2ec2A3Auy5aWvaa/YNHefLFvWRz0NaU4aKT5ya6bpW7s2H7ADv2HzmufNOuA2x4aQCnNv6DI1JNU20+nqhaShbFfvu86reCu98G3AZRzSLpoAR62pu49LQFFfs8M+Osvh7O6us5rvzyMyoXg4gcr5ZmpPUDiwuOFwHbqxSLiIgUqKVk8QSw3MyWmVkz8A7gvirHJCIi1FAzlLuPmNnvAN8kGjr7GXffUOWwRESEGkoWAO5+P3B/teMQEZHj1VIzlIiI1CglCxERKUvJQkREylKyEBGRsmpmuY/JMLPdwAuTvHwesGcaw6kFuqf6oHuqfTPtfuD4e3qNu/dO5OK6ThZTYWarJro2Sq3TPdUH3VPtm2n3A1O/JzVDiYhIWUoWIiJSVpqTxW3VDiABuqf6oHuqfTPtfmCK95TaPgsREYkvzTULERGJKZXJwszeYmbPmtlmM7up2vFMlpk9b2brzGy1ma0KZXPM7EEz2xQeZ1c7zvGY2WfMbJeZrS8oK3oPFrk1fG9rzey86kVeXIn7+bCZvRS+p9VmdmXBazeH+3nWzN5cnajHZ2aLzewRM9toZhvM7MZQXs/fU6l7qtvvysxazewHZrYm3NNHQvkyM3s8fE9fDqt6Y2Yt4XhzeH3puB/g7qn6IVrRdgtwEtAMrAHOqHZck7yX54F5Y8r+CrgpPL8J+Hi14yxzD5cA5wHry90DcCXwDaKNsi4EHq92/DHv58PA/ypy7hnh718LsCz8vcxU+x6KxLkQOC887wKeC7HX8/dU6p7q9rsKf96d4XkT8Hj4878LeEco/zTwW+H5bwOfDs/fAXx5vPdPY83iAmCzu29192HgTuDqKsc0na4G7gjP7wCuqWIsZbn7d4C9Y4pL3cNdbCijAAAE5UlEQVTVwOc98hgwy8wWVibSeErcTylXA3e6+5C7/wjYTPT3s6a4+w53fyo8PwBsJNoGuZ6/p1L3VErNf1fhz/tgOGwKPw5cCtwdysd+T/nv727gMjMruV9yGpNFsb2+x/tLUssceMDMngx7kwMscPcdEP2DAOZXLbrJK3UP9fzd/U5okvlMQdNg3d1PaKo4l+h/rTPiexpzT1DH35WZZcxsNbALeJCoBrTP3UfCKYVxj95TeH0/MLfUe6cxWcTa67tOXOzu5wFXADeY2SXVDihh9frdfQo4GTgH2AHcEsrr6n7MrBP4CvA+dx8Y79QiZTV5X0Xuqa6/K3fPuvs5RNtSXwCcXuy08Dihe0pjspgxe327+/bwuAu4h+gvx858lT887qpehJNW6h7q8rtz953hH3EO+CeONV/Uzf2YWRPRL9V/cfd/D8V1/T0Vu6eZ8F0BuPs+4FGiPotZZpbf6K4w7tF7Cq/3ME4TahqTxYzY69vMOsysK/8ceBOwnuhergunXQfcW50Ip6TUPdwHvDuMtrkQ2J9vBqllY9rrryX6niC6n3eEUSnLgOXADyodXzmhHft2YKO7f6Lgpbr9nkrdUz1/V2bWa2azwvM24I1EfTGPAG8Lp439nvLf39uAb3no7S6q2j341fghGq3xHFF73h9XO55J3sNJRKMz1gAb8vdB1Ob4MLApPM6pdqxl7uNLRNX9o0T/07m+1D0QVZv/Pnxv64AV1Y4/5v18IcS7NvwDXVhw/h+H+3kWuKLa8Ze4p58hap5YC6wOP1fW+fdU6p7q9rsCXgf8MMS+HvhQKD+JKLFtBv4NaAnlreF4c3j9pPHeXzO4RUSkrDQ2Q4mIyAQpWYiISFlKFiIiUpaShYiIlKVkISIiZSlZSOqY2dyCVUVfHrPKaHOVYvqsmZ1ajc8WiUNDZyXVzOzDwEF3/5tqxyJSy1SzEClgZteFPQFWm9k/mFlDKL/NzFaFfQI+VHB+v5n9hZk9ZmZPmNl5ZvaAmW0xs98o8v5dZvaNsOfAejN7Wyj/rpmdY2bXFtRynjOzTeH1883s22HRyG+Y2YJK/ZmIgJKFyCgzO4toiYef9mgxtkai5WAg2rdhBXA2cLmZnVFw6fPufiHwGNESEtcCPw18tMjHXBnOP9vdzyJaGXSUu98TPvtcopnEf2NmLcDfAb/k7j8JfLHEe4skprH8KSKp8UbgfGBVWNa/jWPLUr/TzK4n+jdzItFmOE+H1/Jri60DGt39EHDIzHJm1unH9hiAaCmGj5nZx4Cvufv3SsRyM9GaSv9oZucAZwIPhbgyREuJiFSMkoXIMQZ8xt0/eFyh2XLgRuACd99nZl8kWlcnbyg85gqe54+P+zfm7hvNbAVRDeOvzezr7v5/xnzem4k2pnl9QVxr3f1np3R3IlOgZiiRYx4CftnM5sHoqKklQDdwABgIq5JOev9lM+sj6lD/AvAJoi1YC19fBtwKvN3dj4Tip4E+M7sgnNNsZmdONgaRyVDNQiRw93UWbXL/UOjYPgr8JrCK6Bf2emArUKrpKI6ziZqhcsBweP9Cvw7MAe4LTU4vuvtVoSP81rAsfSPRpjwbphCHyIRo6KyIiJSlZigRESlLyUJERMpSshARkbKULEREpCwlCxERKUvJQkREylKyEBGRspQsRESkrP8GKRExVN6BF48AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Draw projects vs team size plot\n",
    "projects = df.groupby(['gh_team_size']).gh_project_name.nunique()\n",
    "plt.plot(projects)\n",
    "plt.xlabel('Team size')\n",
    "plt.ylabel('Number of projects')\n",
    "plt.savefig('./figs/projects.eps', format='eps')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3702595, 15)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Removing teams with size greater than 60, missing values. Creating needed features."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# removing projects with team size higher than 60 as there are only one sample project per team size\n",
    "df = df.loc[df['gh_team_size'] < 60]\n",
    "\n",
    "# Convert gh_pushed_at and gh_first_commit_created_at to delta time\n",
    "df['gh_pushed_at'] = pd.to_datetime(df['gh_pushed_at'])\n",
    "df['gh_first_commit_created_at'] = pd.to_datetime(df['gh_first_commit_created_at'])\n",
    "\n",
    "# Get date diff from them\n",
    "df['date_diff'] = df['gh_pushed_at'].sub(df['gh_first_commit_created_at'], axis=0)\n",
    "df[\"date_diff\"] = df[\"date_diff\"] / np.timedelta64(1, 'm')\n",
    "\n",
    "# Dropping rows where travis status is canceled or started and mapping the rest to 0 and 1\n",
    "df = df[df.tr_status != 'canceled']\n",
    "df = df[df.tr_status != 'started']\n",
    "df['tr_status'] = df['tr_status'].map({'passed': 1, 'failed': 0, 'errored': 0})\n",
    "\n",
    "# Source and test code ratio and their ratio over time\n",
    "df[\"src_diff_ratio\"] = df[\"git_diff_src_churn\"] / df[\"gh_sloc\"]\n",
    "df[\"src_date\"] = df[\"src_diff_ratio\"] / df[\"date_diff\"]\n",
    "\n",
    "total_tests = (df[\"gh_test_lines_per_kloc\"] * (df[\"gh_sloc\"] / 1000)) * (df[\"gh_test_cases_per_kloc\"] * (df[\"gh_sloc\"] / 1000))\n",
    "df[\"test_diff_ratio\"] = df[\"git_diff_test_churn\"] / total_tests\n",
    "df[\"test_date\"] = df[\"test_diff_ratio\"] / df[\"date_diff\"]\n",
    "\n",
    "df.replace('', np.nan, inplace=True) # remove empty cells\n",
    "df.replace([np.inf, -np.inf], np.nan, inplace=True) # remove infinity values\n",
    "\n",
    "# removing rows with null value\n",
    "df = df[pd.notnull(df['tr_log_num_tests_run'])]\n",
    "df = df[pd.notnull(df['tr_log_num_tests_ok'])]\n",
    "df = df[pd.notnull(df['tr_log_num_tests_failed'])]\n",
    "\n",
    "# getting test failure and success ratio\n",
    "df[\"test_failed_ratio\"] = df[\"tr_log_num_tests_failed\"] / df[\"tr_log_num_tests_run\"]\n",
    "df[\"test_ok_ratio\"] = df[\"tr_log_num_tests_ok\"] / df[\"tr_log_num_tests_run\"]\n",
    "\n",
    "# sorting data by team size\n",
    "df = df.sort_values('gh_team_size')\n",
    "\n",
    "# save the refined dataset\n",
    "df.to_csv('./refined.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1550252, 22)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}