05.08_Pandas_Aggregation_and_Grouping.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-28T01:00:12.078228Z",
     "start_time": "2019-10-28T01:00:11.180444Z"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "class display(object):\n",
    "    \"\"\"Display HTML representation of multiple objects\"\"\"\n",
    "    template = \"\"\"<div style=\"float: left; padding: 10px;\">\n",
    "    <p style='font-family:\"Courier New\", Courier, monospace'>{0}</p>{1}\n",
    "    </div>\"\"\"\n",
    "    def __init__(self, *args):\n",
    "        self.args = args\n",
    "        \n",
    "    def _repr_html_(self):\n",
    "        return '\\n'.join(self.template.format(a, eval(a)._repr_html_())\n",
    "                         for a in self.args)\n",
    "    \n",
    "    def __repr__(self):\n",
    "        return '\\n\\n'.join(a + '\\n' + repr(eval(a))\n",
    "                           for a in self.args)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Planets Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-28T01:01:55.835151Z",
     "start_time": "2019-10-28T01:01:51.089355Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1035, 6)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import seaborn as sns\n",
    "# import data\n",
    "planets = sns.load_dataset('planets')\n",
    "planets.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-28T01:03:14.093703Z",
     "start_time": "2019-10-28T01:03:14.058529Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>method</th>\n",
       "      <th>number</th>\n",
       "      <th>orbital_period</th>\n",
       "      <th>mass</th>\n",
       "      <th>distance</th>\n",
       "      <th>year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Radial Velocity</td>\n",
       "      <td>1</td>\n",
       "      <td>269.300</td>\n",
       "      <td>7.10</td>\n",
       "      <td>77.40</td>\n",
       "      <td>2006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Radial Velocity</td>\n",
       "      <td>1</td>\n",
       "      <td>874.774</td>\n",
       "      <td>2.21</td>\n",
       "      <td>56.95</td>\n",
       "      <td>2008</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Radial Velocity</td>\n",
       "      <td>1</td>\n",
       "      <td>763.000</td>\n",
       "      <td>2.60</td>\n",
       "      <td>19.84</td>\n",
       "      <td>2011</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Radial Velocity</td>\n",
       "      <td>1</td>\n",
       "      <td>326.030</td>\n",
       "      <td>19.40</td>\n",
       "      <td>110.62</td>\n",
       "      <td>2007</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Radial Velocity</td>\n",
       "      <td>1</td>\n",
       "      <td>516.220</td>\n",
       "      <td>10.50</td>\n",
       "      <td>119.47</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            method  number  orbital_period   mass  distance  year\n",
       "0  Radial Velocity       1         269.300   7.10     77.40  2006\n",
       "1  Radial Velocity       1         874.774   2.21     56.95  2008\n",
       "2  Radial Velocity       1         763.000   2.60     19.84  2011\n",
       "3  Radial Velocity       1         326.030  19.40    110.62  2007\n",
       "4  Radial Velocity       1         516.220  10.50    119.47  2009"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "planets.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Simple Aggregation in Pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-28T01:04:19.121435Z",
     "start_time": "2019-10-28T01:04:19.061619Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    0.374540\n",
       "1    0.950714\n",
       "2    0.731994\n",
       "3    0.598658\n",
       "4    0.156019\n",
       "dtype: float64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rng = np.random.RandomState(42)\n",
    "ser = pd.Series(rng.rand(5))\n",
    "ser"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-28T01:04:26.505636Z",
     "start_time": "2019-10-28T01:04:26.484216Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.811925491708157"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ser.sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-28T01:04:30.694607Z",
     "start_time": "2019-10-28T01:04:30.679989Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5623850983416314"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ser.mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For a `DataFrame`, by default the aggregations return results within each column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-28T01:05:39.627883Z",
     "start_time": "2019-10-28T01:05:39.600450Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>A</th>\n",
       "      <th>B</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.155995</td>\n",
       "      <td>0.020584</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.058084</td>\n",
       "      <td>0.969910</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.866176</td>\n",
       "      <td>0.832443</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.601115</td>\n",
       "      <td>0.212339</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.708073</td>\n",
       "      <td>0.181825</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          A         B\n",
       "0  0.155995  0.020584\n",
       "1  0.058084  0.969910\n",
       "2  0.866176  0.832443\n",
       "3  0.601115  0.212339\n",
       "4  0.708073  0.181825"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame({'A':rng.rand(5),\n",
    "                   'B':rng.rand(5)})\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-28T01:06:00.800469Z",
     "start_time": "2019-10-28T01:06:00.788637Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "A    0.477888\n",
       "B    0.443420\n",
       "dtype: float64"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-28T01:06:13.472098Z",
     "start_time": "2019-10-28T01:06:13.460361Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "A    0.477888\n",
       "B    0.443420\n",
       "dtype: float64"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "By specifying the `axis` argument, we can instead aggregate within each row"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-28T01:07:11.772358Z",
     "start_time": "2019-10-28T01:07:11.753184Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    0.088290\n",
       "1    0.513997\n",
       "2    0.849309\n",
       "3    0.406727\n",
       "4    0.444949\n",
       "dtype: float64"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.mean(axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Pandas `Series` and `DataFrame` include all of the common aggregates of Numpy's aggregations.  \n",
    "In addition, there is a convenience method `describe()` that computes several common aggregates for each column."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-28T01:13:22.959903Z",
     "start_time": "2019-10-28T01:13:22.874132Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>number</th>\n",
       "      <th>orbital_period</th>\n",
       "      <th>mass</th>\n",
       "      <th>distance</th>\n",
       "      <th>year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>498.00000</td>\n",
       "      <td>498.000000</td>\n",
       "      <td>498.000000</td>\n",
       "      <td>498.000000</td>\n",
       "      <td>498.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>1.73494</td>\n",
       "      <td>835.778671</td>\n",
       "      <td>2.509320</td>\n",
       "      <td>52.068213</td>\n",
       "      <td>2007.377510</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>1.17572</td>\n",
       "      <td>1469.128259</td>\n",
       "      <td>3.636274</td>\n",
       "      <td>46.596041</td>\n",
       "      <td>4.167284</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.00000</td>\n",
       "      <td>1.328300</td>\n",
       "      <td>0.003600</td>\n",
       "      <td>1.350000</td>\n",
       "      <td>1989.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.00000</td>\n",
       "      <td>38.272250</td>\n",
       "      <td>0.212500</td>\n",
       "      <td>24.497500</td>\n",
       "      <td>2005.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1.00000</td>\n",
       "      <td>357.000000</td>\n",
       "      <td>1.245000</td>\n",
       "      <td>39.940000</td>\n",
       "      <td>2009.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>2.00000</td>\n",
       "      <td>999.600000</td>\n",
       "      <td>2.867500</td>\n",
       "      <td>59.332500</td>\n",
       "      <td>2011.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>6.00000</td>\n",
       "      <td>17337.500000</td>\n",
       "      <td>25.000000</td>\n",
       "      <td>354.000000</td>\n",
       "      <td>2014.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          number  orbital_period        mass    distance         year\n",
       "count  498.00000      498.000000  498.000000  498.000000   498.000000\n",
       "mean     1.73494      835.778671    2.509320   52.068213  2007.377510\n",
       "std      1.17572     1469.128259    3.636274   46.596041     4.167284\n",
       "min      1.00000        1.328300    0.003600    1.350000  1989.000000\n",
       "25%      1.00000       38.272250    0.212500   24.497500  2005.000000\n",
       "50%      1.00000      357.000000    1.245000   39.940000  2009.000000\n",
       "75%      2.00000      999.600000    2.867500   59.332500  2011.000000\n",
       "max      6.00000    17337.500000   25.000000  354.000000  2014.000000"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "planets.dropna().describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-28T01:15:23.142031Z",
     "start_time": "2019-10-28T01:15:23.106454Z"
    }
   },
   "source": [
    "The following table summarizes some other built-in Pandas aggregations:\n",
    "\n",
    "| Aggregation              | Description                     |\n",
    "|--------------------------|---------------------------------|\n",
    "| ``count()``              | Total number of items           |\n",
    "| ``first()``, ``last()``  | First and last item             |\n",
    "| ``mean()``, ``median()`` | Mean and median                 |\n",
    "| ``min()``, ``max()``     | Minimum and maximum             |\n",
    "| ``std()``, ``var()``     | Standard deviation and variance |\n",
    "| ``mad()``                | Mean absolute deviation         |\n",
    "| ``prod()``               | Product of all items     \n",
    "|\n",
    "| ``sum()``                | Sum of all items                |\n",
    "\n",
    "These are all methods of ``DataFrame`` and ``Series`` objects."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## GroupBy: Split, Apply, Combine"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Split, apply, combine"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![](../PythonDataScienceHandbook/notebooks/figures/03.08-split-apply-combine.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This makes clear what the `groupby` accomplishes: \n",
    "+ The **split** step involves breaking up and grouping a `DataFrame` depending on the value of the specified key.\n",
    "+ The **apply** step involves computing some function, usually an aggregate, transformation, or filltering, within the individual groups\n",
    "+ The **conbine** step merges the results of these operations into an output array."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-28T01:28:32.794988Z",
     "start_time": "2019-10-28T01:28:32.750289Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>key</th>\n",
       "      <th>data</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>B</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>C</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>B</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>C</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  key  data\n",
       "0   A     0\n",
       "1   B     1\n",
       "2   C     2\n",
       "3   A     3\n",
       "4   B     4\n",
       "5   C     5"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame({'key': ['A', 'B', 'C', 'A', 'B', 'C'],\n",
    "                   'data': range(6)}, columns=['key', 'data'])\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The most basic split-apply-combine operation can be computed with the `groupby()` method of `DataFrame`, passing the name of the desired key column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-28T01:52:35.926411Z",
     "start_time": "2019-10-28T01:52:35.900686Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<pandas.core.groupby.generic.DataFrameGroupBy object at 0x0000021E9FF3EB70>"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.groupby('key')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The result is not a set of `DataFrame`s, but a `DataFrameGroupBy` object.  \n",
    "We can think of it as a special view of the `DataFrame`, which is poised to dig into the groups but does not actual computation **until the aggregation is applied**  \n",
    "To produce a result, we can apply an aggregate to it, which will perform the appropriate apply/combine steps to produce the desired result:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-28T01:35:37.622995Z",
     "start_time": "2019-10-28T01:35:37.591521Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>data</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>key</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>A</th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>B</th>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C</th>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     data\n",
       "key      \n",
       "A       3\n",
       "B       5\n",
       "C       7"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.groupby('key').sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### The GroupBy object\n",
    "Actually, we can treat the `GroupBy` object as it's a collection of `DataFrame`. and does not care how it done under the hood(it's difficult to express)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Column indexing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The `GroupBy` object supports column indexing in the same way as the `DataFrame`, and return s a modified `GroupBy` object"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-28T01:42:36.851551Z",
     "start_time": "2019-10-28T01:42:36.842139Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<pandas.core.groupby.generic.DataFrameGroupBy object at 0x0000021EA05C6F60>"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "planets.groupby('method')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-28T01:43:21.900229Z",
     "start_time": "2019-10-28T01:43:21.875294Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<pandas.core.groupby.generic.SeriesGroupBy object at 0x0000021EA05DAA20>"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "planets.groupby('method')['orbital_period']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-28T01:45:44.678522Z",
     "start_time": "2019-10-28T01:45:44.656774Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "method\n",
       "Astrometry                         631.180000\n",
       "Eclipse Timing Variations         4343.500000\n",
       "Imaging                          27500.000000\n",
       "Microlensing                      3300.000000\n",
       "Orbital Brightness Modulation        0.342887\n",
       "Pulsar Timing                       66.541900\n",
       "Pulsation Timing Variations       1170.000000\n",
       "Radial Velocity                    360.200000\n",
       "Transit                              5.714932\n",
       "Transit Timing Variations           57.011000\n",
       "Name: orbital_period, dtype: float64"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "planets.groupby('method')['orbital_period'].median()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Iteration over groups\n",
    "The `GroupBy` object supports direct iteration over the groups, returning each group as a `Series` or `DataFrame`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-28T01:54:09.870695Z",
     "start_time": "2019-10-28T01:54:09.844927Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Astrometry                     shape=(2, 6)\n",
      "Eclipse Timing Variations      shape=(9, 6)\n",
      "Imaging                        shape=(38, 6)\n",
      "Microlensing                   shape=(23, 6)\n",
      "Orbital Brightness Modulation  shape=(3, 6)\n",
      "Pulsar Timing                  shape=(5, 6)\n",
      "Pulsation Timing Variations    shape=(1, 6)\n",
      "Radial Velocity                shape=(553, 6)\n",
      "Transit                        shape=(397, 6)\n",
      "Transit Timing Variations      shape=(4, 6)\n"
     ]
    }
   ],
   "source": [
    "for method, group in planets.groupby('method'):\n",
    "    print(\"{:30s} shape={}\".format(method, group.shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Dispatch methods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-28T02:02:07.037342Z",
     "start_time": "2019-10-28T02:02:06.945313Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>min</th>\n",
       "      <th>25%</th>\n",
       "      <th>50%</th>\n",
       "      <th>75%</th>\n",
       "      <th>max</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>method</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Astrometry</th>\n",
       "      <td>2.0</td>\n",
       "      <td>2011.500000</td>\n",
       "      <td>2.121320</td>\n",
       "      <td>2010.0</td>\n",
       "      <td>2010.75</td>\n",
       "      <td>2011.5</td>\n",
       "      <td>2012.25</td>\n",
       "      <td>2013.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Eclipse Timing Variations</th>\n",
       "      <td>9.0</td>\n",
       "      <td>2010.000000</td>\n",
       "      <td>1.414214</td>\n",
       "      <td>2008.0</td>\n",
       "      <td>2009.00</td>\n",
       "      <td>2010.0</td>\n",
       "      <td>2011.00</td>\n",
       "      <td>2012.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Imaging</th>\n",
       "      <td>38.0</td>\n",
       "      <td>2009.131579</td>\n",
       "      <td>2.781901</td>\n",
       "      <td>2004.0</td>\n",
       "      <td>2008.00</td>\n",
       "      <td>2009.0</td>\n",
       "      <td>2011.00</td>\n",
       "      <td>2013.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Microlensing</th>\n",
       "      <td>23.0</td>\n",
       "      <td>2009.782609</td>\n",
       "      <td>2.859697</td>\n",
       "      <td>2004.0</td>\n",
       "      <td>2008.00</td>\n",
       "      <td>2010.0</td>\n",
       "      <td>2012.00</td>\n",
       "      <td>2013.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Orbital Brightness Modulation</th>\n",
       "      <td>3.0</td>\n",
       "      <td>2011.666667</td>\n",
       "      <td>1.154701</td>\n",
       "      <td>2011.0</td>\n",
       "      <td>2011.00</td>\n",
       "      <td>2011.0</td>\n",
       "      <td>2012.00</td>\n",
       "      <td>2013.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pulsar Timing</th>\n",
       "      <td>5.0</td>\n",
       "      <td>1998.400000</td>\n",
       "      <td>8.384510</td>\n",
       "      <td>1992.0</td>\n",
       "      <td>1992.00</td>\n",
       "      <td>1994.0</td>\n",
       "      <td>2003.00</td>\n",
       "      <td>2011.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pulsation Timing Variations</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2007.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2007.0</td>\n",
       "      <td>2007.00</td>\n",
       "      <td>2007.0</td>\n",
       "      <td>2007.00</td>\n",
       "      <td>2007.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Radial Velocity</th>\n",
       "      <td>553.0</td>\n",
       "      <td>2007.518987</td>\n",
       "      <td>4.249052</td>\n",
       "      <td>1989.0</td>\n",
       "      <td>2005.00</td>\n",
       "      <td>2009.0</td>\n",
       "      <td>2011.00</td>\n",
       "      <td>2014.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Transit</th>\n",
       "      <td>397.0</td>\n",
       "      <td>2011.236776</td>\n",
       "      <td>2.077867</td>\n",
       "      <td>2002.0</td>\n",
       "      <td>2010.00</td>\n",
       "      <td>2012.0</td>\n",
       "      <td>2013.00</td>\n",
       "      <td>2014.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Transit Timing Variations</th>\n",
       "      <td>4.0</td>\n",
       "      <td>2012.500000</td>\n",
       "      <td>1.290994</td>\n",
       "      <td>2011.0</td>\n",
       "      <td>2011.75</td>\n",
       "      <td>2012.5</td>\n",
       "      <td>2013.25</td>\n",
       "      <td>2014.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                               count         mean       std     min      25%  \\\n",
       "method                                                                         \n",
       "Astrometry                       2.0  2011.500000  2.121320  2010.0  2010.75   \n",
       "Eclipse Timing Variations        9.0  2010.000000  1.414214  2008.0  2009.00   \n",
       "Imaging                         38.0  2009.131579  2.781901  2004.0  2008.00   \n",
       "Microlensing                    23.0  2009.782609  2.859697  2004.0  2008.00   \n",
       "Orbital Brightness Modulation    3.0  2011.666667  1.154701  2011.0  2011.00   \n",
       "Pulsar Timing                    5.0  1998.400000  8.384510  1992.0  1992.00   \n",
       "Pulsation Timing Variations      1.0  2007.000000       NaN  2007.0  2007.00   \n",
       "Radial Velocity                553.0  2007.518987  4.249052  1989.0  2005.00   \n",
       "Transit                        397.0  2011.236776  2.077867  2002.0  2010.00   \n",
       "Transit Timing Variations        4.0  2012.500000  1.290994  2011.0  2011.75   \n",
       "\n",
       "                                  50%      75%     max  \n",
       "method                                                  \n",
       "Astrometry                     2011.5  2012.25  2013.0  \n",
       "Eclipse Timing Variations      2010.0  2011.00  2012.0  \n",
       "Imaging                        2009.0  2011.00  2013.0  \n",
       "Microlensing                   2010.0  2012.00  2013.0  \n",
       "Orbital Brightness Modulation  2011.0  2012.00  2013.0  \n",
       "Pulsar Timing                  1994.0  2003.00  2011.0  \n",
       "Pulsation Timing Variations    2007.0  2007.00  2007.0  \n",
       "Radial Velocity                2009.0  2011.00  2014.0  \n",
       "Transit                        2012.0  2013.00  2014.0  \n",
       "Transit Timing Variations      2012.5  2013.25  2014.0  "
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "planets.groupby('method')['year'].describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Aggregate, filter, transform, apply"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-28T06:07:19.059943Z",
     "start_time": "2019-10-28T06:07:19.026046Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>key</th>\n",
       "      <th>data1</th>\n",
       "      <th>data2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>B</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>C</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>B</td>\n",
       "      <td>4</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>C</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  key  data1  data2\n",
       "0   A      0      5\n",
       "1   B      1      0\n",
       "2   C      2      3\n",
       "3   A      3      3\n",
       "4   B      4      7\n",
       "5   C      5      9"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rng = np.random.RandomState(0)\n",
    "df = pd.DataFrame({'key': ['A', 'B', 'C', 'A', 'B', 'C'],\n",
    "                   'data1': range(6),\n",
    "                   'data2': rng.randint(0, 10, 6)},\n",
    "                   columns = ['key', 'data1', 'data2'])\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Aggregation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-28T06:09:31.619705Z",
     "start_time": "2019-10-28T06:09:31.570201Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"3\" halign=\"left\">data1</th>\n",
       "      <th colspan=\"3\" halign=\"left\">data2</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>min</th>\n",
       "      <th>std</th>\n",
       "      <th>max</th>\n",
       "      <th>min</th>\n",
       "      <th>std</th>\n",
       "      <th>max</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>key</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>A</th>\n",
       "      <td>0</td>\n",
       "      <td>2.12132</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>1.414214</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>B</th>\n",
       "      <td>1</td>\n",
       "      <td>2.12132</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>4.949747</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C</th>\n",
       "      <td>2</td>\n",
       "      <td>2.12132</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>4.242641</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    data1              data2              \n",
       "      min      std max   min       std max\n",
       "key                                       \n",
       "A       0  2.12132   3     3  1.414214   5\n",
       "B       1  2.12132   4     0  4.949747   7\n",
       "C       2  2.12132   5     3  4.242641   9"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.groupby('key').aggregate(['min', np.std, 'max'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Another pattern is to pass a dictionary mapping column names to operations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-28T06:12:06.469379Z",
     "start_time": "2019-10-28T06:12:06.439468Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>data1</th>\n",
       "      <th colspan=\"2\" halign=\"left\">data2</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>max</th>\n",
       "      <th>min</th>\n",
       "      <th>std</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>key</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>A</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>1.414214</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>B</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>4.949747</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C</th>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>4.242641</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    data1 data2          \n",
       "      max   min       std\n",
       "key                      \n",
       "A       3     3  1.414214\n",
       "B       4     0  4.949747\n",
       "C       5     3  4.242641"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.groupby('key').aggregate({\"data1\":max,\n",
    "                             \"data2\":[min, np.std]})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Filltering\n",
    "the filter function should return a Boolean value specifying whether the group passes the filtering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-28T06:16:08.268286Z",
     "start_time": "2019-10-28T06:16:08.238522Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"float: left; padding: 10px;\">\n",
       "    <p style='font-family:\"Courier New\", Courier, monospace'>df</p><div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>key</th>\n",
       "      <th>data1</th>\n",
       "      <th>data2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>B</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>C</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>B</td>\n",
       "      <td>4</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>C</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "    </div>\n",
       "<div style=\"float: left; padding: 10px;\">\n",
       "    <p style='font-family:\"Courier New\", Courier, monospace'>df.groupby('key').std()</p><div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>data1</th>\n",
       "      <th>data2</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>key</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>A</th>\n",
       "      <td>2.12132</td>\n",
       "      <td>1.414214</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>B</th>\n",
       "      <td>2.12132</td>\n",
       "      <td>4.949747</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C</th>\n",
       "      <td>2.12132</td>\n",
       "      <td>4.242641</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "    </div>\n",
       "<div style=\"float: left; padding: 10px;\">\n",
       "    <p style='font-family:\"Courier New\", Courier, monospace'>df.groupby('key').filter(filter_func)</p><div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>key</th>\n",
       "      <th>data1</th>\n",
       "      <th>data2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>B</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>C</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>B</td>\n",
       "      <td>4</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>C</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "    </div>"
      ],
      "text/plain": [
       "df\n",
       "  key  data1  data2\n",
       "0   A      0      5\n",
       "1   B      1      0\n",
       "2   C      2      3\n",
       "3   A      3      3\n",
       "4   B      4      7\n",
       "5   C      5      9\n",
       "\n",
       "df.groupby('key').std()\n",
       "       data1     data2\n",
       "key                   \n",
       "A    2.12132  1.414214\n",
       "B    2.12132  4.949747\n",
       "C    2.12132  4.242641\n",
       "\n",
       "df.groupby('key').filter(filter_func)\n",
       "  key  data1  data2\n",
       "1   B      1      0\n",
       "2   C      2      3\n",
       "4   B      4      7\n",
       "5   C      5      9"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def filter_func(x):\n",
    "    return x['data2'].std() > 4\n",
    "\n",
    "display('df', \"df.groupby('key').std()\", \"df.groupby('key').filter(filter_func)\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Transformation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Transformation can return some transformed version of the full data to recombine. For such a transformation, the output is the same shape as the input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-28T06:19:56.648468Z",
     "start_time": "2019-10-28T06:19:56.608467Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>data1</th>\n",
       "      <th>data2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-1.5</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-1.5</td>\n",
       "      <td>-3.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-1.5</td>\n",
       "      <td>-3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.5</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.5</td>\n",
       "      <td>3.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1.5</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   data1  data2\n",
       "0   -1.5    1.0\n",
       "1   -1.5   -3.5\n",
       "2   -1.5   -3.0\n",
       "3    1.5   -1.0\n",
       "4    1.5    3.5\n",
       "5    1.5    3.0"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.groupby('key').transform(lambda x: x - x.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-28T06:23:42.858087Z",
     "start_time": "2019-10-28T06:23:42.818247Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>data1</th>\n",
       "      <th>data2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   data1  data2\n",
       "0      1      6\n",
       "1      2      1\n",
       "2      3      4\n",
       "3      4      4\n",
       "4      5      8\n",
       "5      6     10"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def printer(x):\n",
    "    return x+1\n",
    "df.groupby('key').transform(printer)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Apply() method"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The `apply()` method lets you apply an arbitrary function to the group results. The function should take a DataFrame, and return either a Pandas object (e.g., DataFrame, Series) or a scalar; the combine operation will be tailored to the type of output returned."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-28T06:47:46.851375Z",
     "start_time": "2019-10-28T06:47:46.796195Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"float: left; padding: 10px;\">\n",
       "    <p style='font-family:\"Courier New\", Courier, monospace'>df</p><div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>key</th>\n",
       "      <th>data1</th>\n",
       "      <th>data2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>B</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>C</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>B</td>\n",
       "      <td>4</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>C</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "    </div>\n",
       "<div style=\"float: left; padding: 10px;\">\n",
       "    <p style='font-family:\"Courier New\", Courier, monospace'>df.groupby('key').apply(norm_by_data2)</p><div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>data1</th>\n",
       "      <th>data2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.5</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3.5</td>\n",
       "      <td>3.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5.5</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4.5</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6.5</td>\n",
       "      <td>10.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>8.5</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "    </div>"
      ],
      "text/plain": [
       "df\n",
       "  key  data1  data2\n",
       "0   A      0      5\n",
       "1   B      1      0\n",
       "2   C      2      3\n",
       "3   A      3      3\n",
       "4   B      4      7\n",
       "5   C      5      9\n",
       "\n",
       "df.groupby('key').apply(norm_by_data2)\n",
       "   data1  data2\n",
       "0    1.5    9.0\n",
       "1    3.5    3.5\n",
       "2    5.5    9.0\n",
       "3    4.5    7.0\n",
       "4    6.5   10.5\n",
       "5    8.5   15.0"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def norm_by_data2(x):\n",
    "    # x is a DataFrame of group values\n",
    "#     x['data1'] /= x['data2'].sum()\n",
    "    x += x.mean()\n",
    "    return x\n",
    "display('df', \"df.groupby('key').apply(norm_by_data2)\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Specifying the split key"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### A list, array, series, or index providing the groups key"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-28T06:50:58.006000Z",
     "start_time": "2019-10-28T06:50:57.965866Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"float: left; padding: 10px;\">\n",
       "    <p style='font-family:\"Courier New\", Courier, monospace'>df</p><div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>key</th>\n",
       "      <th>data1</th>\n",
       "      <th>data2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>B</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>C</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>B</td>\n",
       "      <td>4</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>C</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "    </div>\n",
       "<div style=\"float: left; padding: 10px;\">\n",
       "    <p style='font-family:\"Courier New\", Courier, monospace'>df.groupby(L).sum()</p><div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>data1</th>\n",
       "      <th>data2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "    </div>"
      ],
      "text/plain": [
       "df\n",
       "  key  data1  data2\n",
       "0   A      0      5\n",
       "1   B      1      0\n",
       "2   C      2      3\n",
       "3   A      3      3\n",
       "4   B      4      7\n",
       "5   C      5      9\n",
       "\n",
       "df.groupby(L).sum()\n",
       "   data1  data2\n",
       "0      7     17\n",
       "1      4      3\n",
       "2      4      7"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# the key can be any series or list \n",
    "# with a length matching that of the DataFrame\n",
    "L = [0, 1, 0, 1, 2, 0]\n",
    "display('df', 'df.groupby(L).sum()')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-28T06:54:42.706050Z",
     "start_time": "2019-10-28T06:54:42.670986Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"float: left; padding: 10px;\">\n",
       "    <p style='font-family:\"Courier New\", Courier, monospace'>df</p><div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>key</th>\n",
       "      <th>data1</th>\n",
       "      <th>data2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>B</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>C</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>B</td>\n",
       "      <td>4</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>C</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "    </div>\n",
       "<div style=\"float: left; padding: 10px;\">\n",
       "    <p style='font-family:\"Courier New\", Courier, monospace'>df.groupby(l).sum()</p><div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>data1</th>\n",
       "      <th>data2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>a</th>\n",
       "      <td>11</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>b</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>c</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "    </div>"
      ],
      "text/plain": [
       "df\n",
       "  key  data1  data2\n",
       "0   A      0      5\n",
       "1   B      1      0\n",
       "2   C      2      3\n",
       "3   A      3      3\n",
       "4   B      4      7\n",
       "5   C      5      9\n",
       "\n",
       "df.groupby(l).sum()\n",
       "   data1  data2\n",
       "a     11     24\n",
       "b      1      0\n",
       "c      3      3"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "l = np.random.randint(2,4, 6)\n",
    "# print(l)\n",
    "l = ['a','b','a','c','a','a']\n",
    "display('df', 'df.groupby(l).sum()')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-28T06:55:22.606839Z",
     "start_time": "2019-10-28T06:55:22.576583Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"float: left; padding: 10px;\">\n",
       "    <p style='font-family:\"Courier New\", Courier, monospace'>df</p><div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>key</th>\n",
       "      <th>data1</th>\n",
       "      <th>data2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>B</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>C</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>B</td>\n",
       "      <td>4</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>C</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "    </div>\n",
       "<div style=\"float: left; padding: 10px;\">\n",
       "    <p style='font-family:\"Courier New\", Courier, monospace'>df.groupby(df[\"key\"]).sum()</p><div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>data1</th>\n",
       "      <th>data2</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>key</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>A</th>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>B</th>\n",
       "      <td>5</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C</th>\n",
       "      <td>7</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "    </div>"
      ],
      "text/plain": [
       "df\n",
       "  key  data1  data2\n",
       "0   A      0      5\n",
       "1   B      1      0\n",
       "2   C      2      3\n",
       "3   A      3      3\n",
       "4   B      4      7\n",
       "5   C      5      9\n",
       "\n",
       "df.groupby(df[\"key\"]).sum()\n",
       "     data1  data2\n",
       "key              \n",
       "A        3      8\n",
       "B        5      7\n",
       "C        7     12"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "display('df', 'df.groupby(df[\"key\"]).sum()')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### A dictionary or series mapping index to group"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Another method is to provide a dictionary that maps index values to the group keys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-28T06:58:57.795832Z",
     "start_time": "2019-10-28T06:58:57.765535Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>data1</th>\n",
       "      <th>data2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>consonant</th>\n",
       "      <td>12</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vowel</th>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           data1  data2\n",
       "consonant     12     19\n",
       "vowel          3      8"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2 = df.set_index('key')\n",
    "mapping = {'A':'vowel', 'B':'consonant', 'C':'consonant'}\n",
    "df2.groupby(mapping).sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Any python function"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Similar to mapping, we can pass any Python function that will input the index value and output the group"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-28T07:01:23.695789Z",
     "start_time": "2019-10-28T07:01:23.650750Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"float: left; padding: 10px;\">\n",
       "    <p style='font-family:\"Courier New\", Courier, monospace'>df2</p><div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>data1</th>\n",
       "      <th>data2</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>key</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>A</th>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>B</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C</th>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>A</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>B</th>\n",
       "      <td>4</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C</th>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "    </div>\n",
       "<div style=\"float: left; padding: 10px;\">\n",
       "    <p style='font-family:\"Courier New\", Courier, monospace'>df2.groupby(str.lower).mean()</p><div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>data1</th>\n",
       "      <th>data2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>a</th>\n",
       "      <td>1.5</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>b</th>\n",
       "      <td>2.5</td>\n",
       "      <td>3.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>c</th>\n",
       "      <td>3.5</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "    </div>"
      ],
      "text/plain": [
       "df2\n",
       "     data1  data2\n",
       "key              \n",
       "A        0      5\n",
       "B        1      0\n",
       "C        2      3\n",
       "A        3      3\n",
       "B        4      7\n",
       "C        5      9\n",
       "\n",
       "df2.groupby(str.lower).mean()\n",
       "   data1  data2\n",
       "a    1.5    4.0\n",
       "b    2.5    3.5\n",
       "c    3.5    6.0"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "display('df2', 'df2.groupby(str.lower).mean()')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### A list of valid keys\n",
    "Any of the preceding key choice can be combined to group on a multi-index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-28T07:03:56.369207Z",
     "start_time": "2019-10-28T07:03:56.355511Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>data1</th>\n",
       "      <th>data2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>a</th>\n",
       "      <th>vowel</th>\n",
       "      <td>1.5</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>b</th>\n",
       "      <th>consonant</th>\n",
       "      <td>2.5</td>\n",
       "      <td>3.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>c</th>\n",
       "      <th>consonant</th>\n",
       "      <td>3.5</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             data1  data2\n",
       "a vowel        1.5    4.0\n",
       "b consonant    2.5    3.5\n",
       "c consonant    3.5    6.0"
      ]
     },
     "execution_count": 131,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2.groupby([str.lower, mapping]).mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Grouping example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-28T07:09:31.965541Z",
     "start_time": "2019-10-28T07:09:31.931250Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>decade</th>\n",
       "      <th>1980s</th>\n",
       "      <th>1990s</th>\n",
       "      <th>2000s</th>\n",
       "      <th>2010s</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>method</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Astrometry</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Eclipse Timing Variations</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>10.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Imaging</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>21.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Microlensing</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Orbital Brightness Modulation</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pulsar Timing</th>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pulsation Timing Variations</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Radial Velocity</th>\n",
       "      <td>1.0</td>\n",
       "      <td>52.0</td>\n",
       "      <td>475.0</td>\n",
       "      <td>424.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Transit</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>64.0</td>\n",
       "      <td>712.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Transit Timing Variations</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "decade                         1980s  1990s  2000s  2010s\n",
       "method                                                   \n",
       "Astrometry                       0.0    0.0    0.0    2.0\n",
       "Eclipse Timing Variations        0.0    0.0    5.0   10.0\n",
       "Imaging                          0.0    0.0   29.0   21.0\n",
       "Microlensing                     0.0    0.0   12.0   15.0\n",
       "Orbital Brightness Modulation    0.0    0.0    0.0    5.0\n",
       "Pulsar Timing                    0.0    9.0    1.0    1.0\n",
       "Pulsation Timing Variations      0.0    0.0    1.0    0.0\n",
       "Radial Velocity                  1.0   52.0  475.0  424.0\n",
       "Transit                          0.0    0.0   64.0  712.0\n",
       "Transit Timing Variations        0.0    0.0    0.0    9.0"
      ]
     },
     "execution_count": 149,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "decade = 10 * (planets['year'] // 10)\n",
    "decade = decade.astype(str) + 's'\n",
    "decade.name = 'decade'\n",
    "planets.groupby(['method', decade])['number'].sum().unstack().fillna(0)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}