output_for_hdch.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Output for app and downloadable jsons\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import json\n",
    "import re\n",
    "from textwrap import wrap\n",
    "import matplotlib.ticker\n",
    "import os\n",
    "import seaborn as sns\n",
    "\n",
    "idx = pd.IndexSlice"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### This notebook is just about putting the files together for the app and creating json file for downloads\n",
    "1. Output has to be in form dict or list or list of dicts\n",
    "2. if its going to a js library it will have to be parsed on the far side\n",
    "3. string output for dates is OK for js parser\n",
    "4. dict output can go directly to python templates"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Although this may seem like boring stuff, this where the automation process starts:\n",
    "\n",
    "1. Indentify attributes you want to measure/compare/visualise\n",
    "2. Create away to index those attributes through a variable \n",
    "3. Introduce that variable as an argument to a function.\n",
    "4. make sure that function does what you want\n",
    "\n",
    "#### so some of the charts here will become interactive on the app... and update as the data updates"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### We need the directories and file names for the JSON data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "here = os.getcwd()\n",
    "# the data is stored in the \"data\" folder:\n",
    "data_folder = here + '/data/JSON'\n",
    "os.listdir(data_folder)\n",
    "# names withoout a file extension are folders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## open up the dfs\n",
    "j_data = here + '/data/JSON/'\n",
    "def get_jsons(file_name):\n",
    "    a = pd.read_json(j_data + file_name, orient='index')\n",
    "    return a\n",
    "# we need these files\n",
    "dfs = ['m_b_2016.json','m_b_2017.json', 'rain2016.json','m_b_201724.json', 'rain2017.json']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "m_b_2016 = get_jsons(dfs[0])\n",
    "m_b_2017 = get_jsons(dfs[1])\n",
    "rain_2016 = get_jsons(dfs[2])\n",
    "m_b_201724 = get_jsons(dfs[3])\n",
    "rain_2017 = get_jsons(dfs[4])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### There are utilities for sorting by week and colony color\n",
    "\n",
    "They were pieced together in the \"Preparing...\" notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "utilities_folder = here + '/data/utilities/'\n",
    "os.listdir(utilities_folder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get what we need from the utilities folder:\n",
    "\n",
    "def get_jsons_x(folder,file_name):\n",
    "    with open(folder + file_name, 'r') as f:\n",
    "        a = json.load(f)\n",
    "    return a\n",
    "date_week16 = get_jsons_x(utilities_folder, 'dateWeek2016JsonObj.json')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2016 utilities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "colony_map_16 = get_jsons_x(utilities_folder, 'colony_map_16.json')\n",
    "three_plate_16 = get_jsons_x(utilities_folder, 'threeP2416.json')\n",
    "week_date16 = get_jsons_x(utilities_folder,'weekDate2016JsonObj.json')\n",
    "places = get_jsons_x(utilities_folder,'locations.json')\n",
    "mediums = get_jsons_x(utilities_folder, 'mediums2017.json')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2017 utilities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "week_date = get_jsons_x(utilities_folder,'weekDate2017JsonObj.json')\n",
    "date_week = get_jsons_x(utilities_folder,'dateWeek2017JsonObj.json')\n",
    "three_p_24_17 = get_jsons_x(utilities_folder,'threeP2417.json')\n",
    "colony_map = get_jsons_x(utilities_folder,'colony_map_17.json')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Example of how the utilities work with the dataframes:\n",
    "\n",
    "#### Combining the utility \"three_plate_16\" with the colony count data\n",
    "\n",
    "There are three indexable properties\n",
    "\n",
    "1. the first index goes through the colony colors\n",
    "2. the second index goes through the plate numbers\n",
    "3. the third goes through the locations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "m_b_2016.set_index(['Date', 'Location'], inplace=True)\n",
    "m_b_2016.sort_index(axis=0, level='Date', ascending=True, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "m_b_2016[three_plate_16[2][2]][:1]#<----should now be sorted by date"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# by changing the index numbers change location and colony\n",
    "a_color_week = m_b_2016.loc[idx[week_date16['Week one'], places[0]],\n",
    "                            three_plate_16[0]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# this will give all the values for a color group for the entire project:\n",
    "# m_b_2016.loc[m_b_2016.Location == places [2], three_plate_16[3]]\n",
    "a_color_week.apply('mean', axis=1)#<------ gives the average of the row"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a_color_week.apply('mean', axis=1).item()#<---- gives the number that we need"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# iterate through the colonies:\n",
    "def colony_avg(df, dates, place, plates):\n",
    "    a = df.loc[idx[dates, place], plates]\n",
    "    b = df.loc[idx[dates, place], 'P1_qty_sample'].item()\n",
    "    c = 100/b \n",
    "    d = a.apply('mean', axis=1).item()\n",
    "    if d == 0:\n",
    "        d = 1/len(plates)\n",
    "        f = colony_map_16[plates[0]]\n",
    "        e = d*c\n",
    "        return e, f, b\n",
    "    elif d > 0:\n",
    "        e =d*c\n",
    "        f = 'none'\n",
    "        return e, f, b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# iterate by week\n",
    "def weekly_avg(df, dates, place, plat_s):\n",
    "    d = []\n",
    "    e = []\n",
    "    for plates in plat_s:\n",
    "        a, b, c = colony_avg(df, dates, place, plates)\n",
    "        d.append(a)\n",
    "        if b != 'none':\n",
    "            e.append(b)\n",
    "    f = ', '.join(e)\n",
    "        \n",
    "    return [{date_week16[dates]:d}, {'BDL':f}, {'dilution':c}]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# add admin data\n",
    "def place_weekly_avg(df, dates, place, plat_s):\n",
    "    a = weekly_avg(df, dates, place, plat_s)\n",
    "    a.append({'num samps':len(plat_s[0])})\n",
    "    return a\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# group by place then week\n",
    "def place_by_week(df, dates, places, plates):\n",
    "    d = []\n",
    "    for place in places:\n",
    "        b = []\n",
    "        for date in dates:            \n",
    "            a= place_weekly_avg(df, date, place, plates)\n",
    "            b.append(a)\n",
    "        c = {place:b}\n",
    "        d.append(c)\n",
    "    return d\n",
    "    \n",
    "bar_charts = place_by_week(m_b_2016, list(date_week16.keys()), places[:], three_plate_16)\n",
    "bar_charts[0]['MRD'][0][0]#<--- gives the average colony count per group/week/location---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bar_charts[0]['MRD'][0][1]#<--- changing this number gives other data, colony types below detetectable limit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bar_charts[0]['MRD'][0][2]#<----- dilution factor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bar_charts[0]['MRD'][0][3]#<------ how many samples were taken"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "columns_2017 = m_b_2017.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "colony_map_17_24 = {'P1_24h_big_blue': 'big_blue','P1_fluo_halo_colonies': 'UV Fluo','P1_24h_med_blue': 'med_blue',\n",
    "                    'P1_24h_turq': 'turq','P1_24h_green': 'green','P1_24h_pink': 'pink','P1_24h_other': 'other',\n",
    "                    'P2_24h_big_blue': 'big_blue','P2_fluo_halo_colonies': 'UV Fluo','P2_24h_med_blue': 'med_blue',\n",
    "                    'P2_24h_turq': 'turq','P2_24h_green': 'green','P2_24h_pink': 'pink','P2_24h_other': 'other',\n",
    "                    'P3_24h_big_blue': 'big_blue','P3_flucolony_map_17_24o_halo_colonies': 'UV Fluo','P3_24h_med_blue': 'med_blue',\n",
    "                    'P3_24h_turq': 'turq','P3_24h_green': 'green','P3_24h_pink': 'pink','P3_24h_other': 'other'}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 2017 data\n",
    "m_b_201724.set_index(['Date', 'Location'], inplace=True)\n",
    "m_b_201724.sort_index(axis=0, level='Date', ascending=True, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def colony_avg_17(df, dates, place, plates):\n",
    "    a = df.loc[idx[dates, place], plates]\n",
    "    b = df.loc[idx[dates, place], 'P1_qty_sample'].item()\n",
    "    g = df.loc[idx[dates, place], 'Samples'].item()\n",
    "    c = 100/b \n",
    "    d = a.apply('mean', axis=1).item()\n",
    "    if g == 0:\n",
    "        e = 0\n",
    "        f = 'no sample'\n",
    "        return e, f, b, g        \n",
    "    elif d == 0:\n",
    "        d = 1/len(plates)\n",
    "        f = colony_map_17_24[plates[0]]\n",
    "        e = d*c\n",
    "        return e, f, b, g\n",
    "    elif d > 0:\n",
    "        e =d*c\n",
    "        f = 'none'\n",
    "        return e, f, b, g"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# iterate by week\n",
    "def weekly_avg_17(df, dates, place, plat_s):\n",
    "    d = []\n",
    "    e = []\n",
    "    for plates in plat_s:\n",
    "        a, b, c, g = colony_avg_17(df, dates, place, plates)\n",
    "        d.append(a)\n",
    "        if b != 'none':\n",
    "            e.append(b)\n",
    "    if g == 0:\n",
    "        f = e[0]\n",
    "    elif g != 0:\n",
    "        f = ', '.join(e)\n",
    "        \n",
    "    return [{date_week[dates]:d}, {'BDL':f}, {'dilution':c}, {'num samps':g}]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# add admin data\n",
    "def place_weekly_avg_17(df, dates, place, plat_s):\n",
    "    a = weekly_avg_17(df, dates, place, plat_s)   \n",
    "    return a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# group by place then week\n",
    "def place_by_week_17(df, dates, places, plates):\n",
    "    d = []\n",
    "    for place in places:\n",
    "        b = []\n",
    "        for date in dates:            \n",
    "            a= place_weekly_avg_17(df, date, place, plates)\n",
    "            b.append(a)\n",
    "        c = {place:b}\n",
    "        d.append(c)\n",
    "    return d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a_i = m_b_201724.loc[m_b_201724.medium == 'easy_gel'].copy()\n",
    "a_i.loc[idx['2017-07-24','MRD'], 'Samples'] = 0\n",
    "bar_chart_17 = place_by_week_17(a_i, list(date_week.keys()), places[:], three_p_24_17)\n",
    "week_days = list(week_date.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# so to get the average weekly total of all colony forming units\n",
    "# I will add up the averages of the individual colors(already calculated to make the average grid)\n",
    "# and we will grab only the coliforms --- using UV for big_blue\n",
    "\n",
    "def total_cfu(c, v, q):\n",
    "    # c = list of the weeks of the study 'Week one', 'Week two' ....\n",
    "    # v = list of the locations indentified for summary\n",
    "    # q = the results from the grid layout fundtion\n",
    "    # place to store the results\n",
    "    b = {}\n",
    "    # go through the locations one after another\n",
    "    for n, x in enumerate(v):\n",
    "        # place to store interim results\n",
    "        e = []\n",
    "        # go through the results of each location, week by week\n",
    "        for i, w in enumerate(c):\n",
    "            # q[n][v[n]][i][0][w] translates to\n",
    "            # four_eight_ave[0][places_48[0][0][0]['Week one']]\n",
    "            # if this was the first location through the function\n",
    "            a = q[n][v[n]][i][0][w]\n",
    "            if type(a) == dict:\n",
    "                # if there is no value or it is 'bdl'\n",
    "                # take the average of the current results and append to e\n",
    "                d=np.mean(e)\n",
    "            else:\n",
    "                # if the result is anything else sum it up\n",
    "                # this is the list of results for the bar charts\n",
    "                d = sum(a)\n",
    "            # take d and put it in e\n",
    "            e.append(d)\n",
    "        # get the standard deviation\n",
    "        f = np.std(e)\n",
    "        # make a string to indentify the std\n",
    "        # this will be component of the error bars\n",
    "        string = x + '_std'\n",
    "        # put all of that in a dict so it acan be called in a plotting function\n",
    "        b.update({x:e, string:f})\n",
    "    \n",
    "    return b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "t_cfu = total_cfu(week_days, places, bar_charts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rain_2016.sort_values('Date', inplace=True)\n",
    "date_list = sorted(list(week_date16.values()))\n",
    "samp_date = sorted(list(date_week16.keys()))\n",
    "samp_date = [pd.to_datetime(x) for x in samp_date]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "t_cfu17 = total_cfu(week_days, places, bar_chart_17)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rain_2017['Date'] = pd.to_datetime(rain_2017['Date'])\n",
    "rain_2017.sort_values('Date', inplace=True)\n",
    "date_list17 = sorted(list(week_date.values()))\n",
    "samp_date17 = sorted(list(date_week.keys()))\n",
    "samp_date17 = [pd.to_datetime(x) for x in samp_date17]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def not_simple_plot(arange, values, datelist, ticklocs, places,rain, xtiks,sup_title, title, filename):\n",
    "    collss=['darkred', 'darkslategray', 'teal']\n",
    "    nticks = 8\n",
    "    fig, ax2 = plt.subplots(figsize=(10,6), edgecolor='midnightblue')    \n",
    "    x = datelist\n",
    "    y = values\n",
    "    locs = ticklocs\n",
    "    labels = week_days\n",
    "    x_one = rain['Date']\n",
    "    y_one = rain['Rain']\n",
    "    #ax2.bar(x_one.values,y_one.values)\n",
    "    ylim = [1]\n",
    "    for i, n in enumerate(places):\n",
    "        string = n + '_std'\n",
    "        std = y[string]\n",
    "        ratio = std/max(y[n])\n",
    "        for j, o in enumerate(x):\n",
    "            t, c, j = ax2.errorbar(x[j], y[n][j], yerr=y[n][j]*ratio, capsize=2, alpha=0.4, ecolor='magenta',)\n",
    "            for k in j:\n",
    "                k.set_linestyle('--')\n",
    "            for cap in c:\n",
    "                cap.set_marker(\"o\")\n",
    "        ax2.plot(x, y[n], label=n, color=collss[i], linewidth=3)\n",
    "        if max(y[n]) > max(ylim):\n",
    "            ylim.append(max(y[n]))\n",
    "        ax2.set_ylim(bottom=0, top=max(ylim)+100)\n",
    "    ax1 = ax2.twinx() \n",
    "    ax1.bar(x_one.values, y_one.values,  color='b', alpha=0.2, label=\"mm of rain\")\n",
    "    \n",
    "      \n",
    "    ax1.spines['bottom'].set_color('midnightblue')\n",
    "    ax1.spines['top'].set_color('midnightblue')\n",
    "    ax1.spines['left'].set_color('midnightblue')\n",
    "    ax1.spines['right'].set_color('midnightblue')\n",
    "    ax1.grid(which='major', color='midnightblue', linestyle='--', alpha=0.2, axis='y')\n",
    "    ax1.tick_params(axis='both', which='both', color='midnightblue')\n",
    "    ax1.set_ylabel('Millimeters of rain per day', labelpad=10, fontsize=14)\n",
    "    #ax1.set_ylim(top=max(ylim))\n",
    "    \n",
    "    ax2.set_ylabel('Colony forming units per 100mL', labelpad=10, fontsize=14)\n",
    "    fig.legend( loc=(0.74, 0.55), frameon=True, fontsize=12, edgecolor='midnightblue', framealpha=0.8 )\n",
    "    \n",
    "    #plt.ylim(ymax=max(ylim))    \n",
    "    plt.xticks(locs, labels, fontsize=11, )\n",
    "    plt.suptitle(sup_title, fontsize=16, family='sans')\n",
    "    plt.title(title, fontsize=16, family='sans', y=1.03)\n",
    "    plt.subplots_adjust(top=0.85)\n",
    "    plt.savefig(filename)\n",
    "    \n",
    "    \n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "not_simple_plot(8, t_cfu17,samp_date17,  date_list17, places,rain_2017, week_days, 'Weekly average total colony forming units', 'June - July 2017; medium: Easygel+, 24hrs, n=69', 'data/charts/2017_AVG_Total_CFU_week.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "not_simple_plot(8, t_cfu,samp_date,  date_list, places,rain_2016, week_days, 'Weekly average total colony forming units', 'June - July 2016; medium: Easygel, 24hrs, n=72', 'data/charts/2016_AVG_Total_CFU_week_16.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def total_ecoli_16(c, v, q):\n",
    "    b = {}\n",
    "    for n, x in enumerate(v):\n",
    "        e = []\n",
    "        for i, w in enumerate(c):        \n",
    "            if type(q[n][v[n]][i][0][w]) == list:\n",
    "                a = q[n][v[n]][i][0][w][0]#<---- changed\n",
    "                \n",
    "                d = np.mean(a)\n",
    "                \n",
    "            elif type(q[n][v[n]][i][0][w]) == dict:\n",
    "                d=np.mean(e)\n",
    "            e.append(d)\n",
    "        f = np.std(e)\n",
    "        string = x + '_std'\n",
    "        b.update({x:e, string:f})\n",
    "        #print(b)\n",
    "    \n",
    "    return b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "total_big_blue_16 = total_ecoli_16(week_days,places, bar_charts,)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# will have to modify the fucntion a little to grab the 1st (uv) and 3rd element of the list\n",
    "# something like this:\n",
    "# four_eight_ave[2]['SVT'][3][0][weeks_l[3]][0:3:2]\n",
    "# [0:3:2] <-- this last bit means from items 0:3 grab every other item (not including 3)\n",
    "# and we have to account for the fact that what ever is coming down may not be a list\n",
    "def total_big_blue_17(c, v, q):\n",
    "    b = {}\n",
    "    for n, x in enumerate(v):\n",
    "        e = []\n",
    "        for i, w in enumerate(c):        \n",
    "            if type(q[n][v[n]][i][0][w]) == list:\n",
    "                a = q[n][v[n]][i][0][w][1]#<---- changed\n",
    "                #print(a)\n",
    "                d = np.sum(a)\n",
    "            elif type(q[n][v[n]][i][0][w]) == dict:\n",
    "                d=np.mean(e)\n",
    "            e.append(d)\n",
    "        f = np.std(e)\n",
    "        string = x + '_std'\n",
    "        b.update({x:e, string:f})\n",
    "        #print(b)\n",
    "    \n",
    "    return b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "total_big_blue17 = total_big_blue_17(week_days, places, bar_chart_17)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_plots_micro(df):\n",
    "    i = []\n",
    "    for b in places:\n",
    "        a = df[b]\n",
    "        for c, d in enumerate(a):\n",
    "            e = a[c]\n",
    "            f = week_days[c]\n",
    "            g = b\n",
    "            h = {'Date':f, 'cfu':e, 'Location':b}\n",
    "            i.append(h)\n",
    "            \n",
    "            \n",
    "            \n",
    "    return i\n",
    "plot_b_blue_17 = make_plots_micro(total_big_blue17)\n",
    "plot_b_blue_17 = make_plots_micro(total_big_blue_16)\n",
    "plot_total_17 = make_plots_micro(t_cfu17)\n",
    "plot_total_16 = make_plots_micro(t_cfu)       "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "not_simple_plot(8, total_big_blue_16,samp_date,  date_list, places,rain_2016, week_days, 'Weekly average Bioindicator colony forming units','June - August 2016; medium: Easygel, 24hrs, n=72', 'data/charts/2016_AVG_BigBlue_CFU_week.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "not_simple_plot(8, total_big_blue17,samp_date17,  date_list17, places,rain_2017, week_days,'Weekly average Bioindicator colony forming units','June - july 2017; medium: Easygel+, 24hrs, n=69', 'data/charts/2017_AVG_BigBlue_CFU_week.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def total_uv_17(c, v, q):\n",
    "    b = {}\n",
    "    for n, x in enumerate(v):\n",
    "        e = []\n",
    "        for i, w in enumerate(c):        \n",
    "            if type(q[n][v[n]][i][0][w]) == list:\n",
    "                a = q[n][v[n]][i][0][w][0]#<---- changed\n",
    "                #print(a)\n",
    "                d = np.sum(a)\n",
    "            elif type(q[n][v[n]][i][0][w]) == dict:\n",
    "                d=np.mean(e)\n",
    "            e.append(d)\n",
    "        f = np.std(e)\n",
    "        string = x + '_std'\n",
    "        b.update({x:e, string:f})\n",
    "        #print(b)\n",
    "    \n",
    "    return b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "total_uv17 = total_uv_17(week_days, places, bar_chart_17)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "not_simple_plot(8, total_uv17,samp_date17,  date_list17, places,rain_2017, week_days,'Weekly average UV+ Bioindicator colony forming units','June - july 2017; medium: Easygel+, 24hrs, n=69', 'data/charts/2017_AVG_UltraViolet_CFU_week.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dates_2017 = sorted(list(date_week.keys()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "place_names={'MRD':'Baye de Montreux', 'SVT':'Quai de Vernex', 'VNX':'Parc de Vernex'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    " def make_series(d,g, file_name):\n",
    "        a=[]\n",
    "        for b, c in enumerate(d):\n",
    "            i = []\n",
    "            for e, f in enumerate(g[c]):\n",
    "                f = f.astype(float)\n",
    "                i.append({'x':dates_2017[e], 'y':round(f, 0), 'location': place_names[c]})\n",
    "            a.append(i)\n",
    "        with open('data/JSON/APP_OUTPUT/' + file_name, 'wb') as h:\n",
    "            h.write(json.dumps(a).encode('utf-8'))\n",
    "#make_series(places, t_cfu17, 't_cfu_17.json')            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "make_series(places, total_uv17, 'total_uv17.json')\n",
    "make_series(places,total_big_blue17, 'big_blue17.json' )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_rain_days(df):\n",
    "    a =list(df['Date'].dt.strftime(\"%Y-%m-%d\"))\n",
    "    b = list(df['Rain'])\n",
    "    c = list(zip(a, b))\n",
    "    d = d1= [[x[0], x[1]] for x in c]\n",
    "    return d\n",
    "rain_17 = make_rain_days(rain_2017)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dates_2016 = sorted(list(date_week16.keys()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    " def make_series_16(d,g, file_name):\n",
    "        a=[]\n",
    "        for b, c in enumerate(d):\n",
    "            i = []\n",
    "            for e, f in enumerate(g[c]):\n",
    "                i.append({'x':dates_2016[e], 'y':round(f, 0), 'location': place_names[c]})\n",
    "            a.append(i)\n",
    "        with open('data/JSON/APP_OUTPUT/' + file_name, 'wb') as h:\n",
    "            h.write(json.dumps(a).encode('utf-8'))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "make_series_16(places, total_big_blue_16, 'big_blue16.json')\n",
    "make_series_16(places, t_cfu, 't_cfu_16.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rain_2016.sort_values(by='Date', inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rain_16 = make_rain_days(rain_2016)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "hotel_nights = {\"June 2016\":44451, \"July 2016\":63896, \"August 2016\":60500, \"June 2017\":51409, \"July\":63675}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rain_2016['rolling'] = rain_2016.rolling(3, min_periods= 1, on='Date').sum()['Rain'].round(2)\n",
    "rain_2016.fillna(0, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rain_2017['rolling'] = rain_2017.rolling(3, min_periods= 1, on='Date').sum()['Rain'].round(2)\n",
    "rain_2017.fillna(0, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_rolling(b, df):\n",
    "    e = {}\n",
    "    for a in b:\n",
    "#         print(a)\n",
    "#         print(df)\n",
    "        c = df.loc[df.Date == a, 'rolling'].item()\n",
    "        d = {a:c}\n",
    "        e.update(d)\n",
    "    return e\n",
    "r_2016_date = get_rolling(dates_2016, rain_2016)\n",
    "r_2017_date = get_rolling(dates_2017, rain_2017)\n",
    "print(r_2016_date, r_2017_date)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rain_bug(c, v, q, g, h,t,k):\n",
    "    b = []\n",
    "    for n, x in enumerate(v):\n",
    "        #print(x)\n",
    "        e = []\n",
    "        for i, w in enumerate(c):\n",
    "            if type(q[n][v[n]][i][0][w]) == list:\n",
    "                a = q[n][v[n]][i][0][w][g]#<---- changed\n",
    "                d = np.sum(a)\n",
    "            e.append(d)\n",
    "\n",
    "            b.append({'x':d, 'y':h[t[w]], 'location':x, 'date': t[w], 'name':k})\n",
    "\n",
    "    return b\n",
    "uv_rain_17 = rain_bug(week_days, places, bar_chart_17, 0, r_2017_date, week_date, 'Bioindicator UV+ 17')\n",
    "bb_rain_17 = rain_bug(week_days, places, bar_chart_17, 1, r_2017_date, week_date, 'Bioindicator 17')\n",
    "mb_rain_17 = rain_bug(week_days, places, bar_chart_17, 2, r_2017_date, week_date, 'Coliform 17')\n",
    "bb_rain_16 = rain_bug(week_days, places, bar_charts, 0, r_2016_date, week_date16, 'Bioindicator 16')\n",
    "mb_rain_16 = rain_bug(week_days, places, bar_charts, 1, r_2016_date, week_date16, 'Coliform 16')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "hotel_nights = {2016:{6:44451, 7:63896, 8:60500}, 2017:{6:51409, 7:63675}}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_x_y_date(b):\n",
    "    xs = []\n",
    "    ys = []\n",
    "    sz = []\n",
    "    ds = []\n",
    "    dct = []\n",
    "    n = []\n",
    "    for a in b:\n",
    "        if a['date'] in dates_2016:\n",
    "            #print('true')\n",
    "            xs.append(a['location'])\n",
    "            ys.append(a['x'])\n",
    "            sz.append((a['y']+10))\n",
    "            ds.append(date_week16[a['date']])\n",
    "            dct.append({date_week16[a['date']]:a['x']})\n",
    "            n.append({'week':date_week16[a['date']], 'cfu':a['x'], 'name':a['name']})\n",
    "        elif a['date'] in dates_2017:\n",
    "            xs.append(a['location'])\n",
    "            ys.append(a['x'])\n",
    "            sz.append((a['y']+10))\n",
    "            ds.append(date_week[a['date']])\n",
    "            dct.append({date_week[a['date']]:a['x']})\n",
    "            n.append({'week':date_week[a['date']], 'cfu':a['x'], 'name':a['name']})\n",
    "    return xs, ys, sz, ds, dct, n\n",
    "r_uvx_2017 = get_x_y_date(uv_rain_17)\n",
    "r_bbx_2017 = get_x_y_date(bb_rain_17)\n",
    "r_mbx_2017 = get_x_y_date(mb_rain_17)\n",
    "r_bbx_2016 = get_x_y_date(bb_rain_16)\n",
    "r_mbx_2016 = get_x_y_date(mb_rain_16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sns\n",
    "list_o_data = [r_uvx_2017,r_bbx_2017, r_mbx_2017, r_bbx_2016, r_mbx_2016]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.ticker as mtick\n",
    "def make_dfs(h,n):\n",
    "    colors = ['#00008b', '#1e90ff', '#40e0d0', 'g', 'y']\n",
    "    fig, ax = plt.subplots(figsize=(12,8))\n",
    "    b = []\n",
    "    for dat in h:\n",
    "        for j in dat[n]:\n",
    "            b.append(j)\n",
    "    c = pd.DataFrame(b)\n",
    "    sns.set_palette(colors)\n",
    "    ax=sns.swarmplot(x='week', y='cfu', hue='name', data=c,  alpha=0.8, size= 14, linewidth=1, edgecolor='w')\n",
    "    plt.axvline(x=1.42, color='b', linestyle='-.', alpha=0.8)\n",
    "    plt.axvline(x=3.7, color='b', linestyle='-.', alpha=0.8)\n",
    "    ax.text(2.56, 1100,  \"Jazz 2016\", ha=\"center\", va=\"center\", size=14, color='b')\n",
    "    ax.annotate(\"\", xy=(1.42, 1060), xycoords='data',\n",
    "               xytext=(3.7, 1060), textcoords='data',\n",
    "               arrowprops=dict(arrowstyle=\"<|-|>\", connectionstyle=\"arc3\", color='b', alpha=0.8))\n",
    "    plt.axvline(x=2.4, color='r', linestyle='-.', alpha=0.8)\n",
    "    plt.axvline(x=4.58, color='r', linestyle='-.', alpha=0.8)\n",
    "    ax.text(3.4, 1240,  \"Jazz 2017\", ha=\"center\", va=\"center\", size=14, color='r')\n",
    "    ax.annotate(\"\", xy=(2.4, 1200), xycoords='data',\n",
    "               xytext=(4.58, 1200), textcoords='data',\n",
    "               arrowprops=dict(arrowstyle=\"<|-|>\", connectionstyle=\"arc3\", color='r', alpha=0.8))\n",
    "    ax.xaxis.grid(which=\"major\", color='b', linewidth=0.7, alpha=0.1)\n",
    "    ax.set_xlabel(\"Sample weeks 2016 and 2017\", size=16, labelpad=20)\n",
    "    ax.set_ylabel(\"Colony forming units per 100mL\", size=16, labelpad=20)\n",
    "    plt.suptitle(\"Montreux large events and the occurence of bacteria in lake water\" , fontsize=16, family='sans')\n",
    "    plt.title(\"Colonies grouped by year and week number\", fontsize=16, family='sans', y=1.03)\n",
    "    ax.legend(loc='upper right', fontsize=14)\n",
    "    plt.savefig('data/charts/YEAR_OVER_YEAR_CFUINCREASE_JAZZ_2016_2017.svg')\n",
    "   \n",
    "    \n",
    "make_dfs(list_o_data, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_x_y_hotel(b):\n",
    "    xs = []\n",
    "    ys = []\n",
    "    sz = []\n",
    "    ds = []\n",
    "    dct = []\n",
    "    n = []\n",
    "    for a in b:\n",
    "        f = pd.to_datetime(a['date'])\n",
    "        d = f.month\n",
    "#         print(d)\n",
    "        e = f.year\n",
    "#         print(e)\n",
    "        if a['date'] in dates_2016:\n",
    "            #print('true')\n",
    "            xs.append(a['location'])\n",
    "            ys.append(a['x'])\n",
    "            sz.append((a['y']+10))\n",
    "            ds.append(date_week16[a['date']])\n",
    "            dct.append({date_week16[a['date']]:a['x']})\n",
    "            n.append({'week':date_week16[a['date']], 'cfu':a['x'], 'name':a['name'], 'nights':hotel_nights[e][d], 'rain':a['y']})\n",
    "        elif a['date'] in dates_2017:\n",
    "            xs.append(a['location'])\n",
    "            ys.append(a['x'])\n",
    "            sz.append((a['y']+10))\n",
    "            ds.append(date_week[a['date']])\n",
    "            dct.append({date_week[a['date']]:a['x']})\n",
    "            n.append({'week':date_week[a['date']], 'cfu':a['x'], 'name':a['name'], 'nights':hotel_nights[e][d], 'rain':a['y']})\n",
    "    return xs, ys, sz, ds, dct, n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "h_uvx_2017 = get_x_y_hotel(uv_rain_17)\n",
    "h_bbx_2017 = get_x_y_hotel(bb_rain_17)\n",
    "h_mbx_2017 = get_x_y_hotel(mb_rain_17)\n",
    "h_bbx_2016 = get_x_y_hotel(bb_rain_16)\n",
    "h_mbx_2016 = get_x_y_hotel(mb_rain_16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "hotel_night=[h_uvx_2017,h_bbx_2017,h_mbx_2017,h_bbx_2016,h_mbx_2016]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "class NpEncoder(json.JSONEncoder):\n",
    "    def default(self, obj):\n",
    "        if isinstance(obj, np.integer):\n",
    "            return int(obj)\n",
    "        elif isinstance(obj, np.floating):\n",
    "            return float(obj)\n",
    "        elif isinstance(obj, np.ndarray):\n",
    "            return obj.tolist()\n",
    "        else:\n",
    "            return super(NpEncoder, self).default(obj)\n",
    "\n",
    "def convert(o):\n",
    "    if isinstance(o, np.int64): return int(o)  \n",
    "    raise TypeError\n",
    "        \n",
    "\n",
    "#json.dumps(hotel_night, indent = 4, default=convert)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_dfs_h(h,n):\n",
    "    colors = ['#00008b', '#1e90ff', '#40e0d0', 'g', 'y']\n",
    "    fig, ax = plt.subplots(figsize=(12,8))\n",
    "    b = []\n",
    "    for dat in h:\n",
    "        for j in dat[n]:\n",
    "            b.append(j)\n",
    "    c = pd.DataFrame(b)\n",
    "    d = sorted(list(c['nights'].unique()))\n",
    "    d = [f\"{x:,d}\" for x in d]\n",
    "    sns.set_palette(colors)\n",
    "    fmt = '{%x:,.0f}'\n",
    "    tick = mtick.StrMethodFormatter(fmt)\n",
    "    ax.yaxis.set_major_formatter(mtick.FuncFormatter(lambda y,  p: format(int(y), ',')))\n",
    "    ax=sns.swarmplot(x='nights', y='cfu', hue='name', data=c,  alpha=0.8, size= 8, linewidth=1, edgecolor='w')\n",
    "    #ax=sns.scatterplot(x='nights', y='cfu', hue='name', data=c,  alpha=0.8, linewidth=3, edgecolor='w')\n",
    "    ax.set_xlabel(\"Hotel nights sold per month, Swiss federal office of statistics\", size=16, labelpad=20)\n",
    "    ax.set_ylabel(\"Colony forming units per 100mL\", size=16, labelpad=20)\n",
    "    plt.suptitle(\"Montreux hotel nights sold / occurence of coliform bacteria\" , fontsize=16, family='sans', )\n",
    "    plt.title(\"Grouped by hotel nights sold per month and colony color\", fontsize=16, family='sans', y=1.03)\n",
    "    ax.legend(loc='upper left', fontsize=14)\n",
    "    #locs, labels = plt.xticks()\n",
    "    #plt.xticks(locs,d, size=14)\n",
    "    #plt.yticks(size=14)\n",
    "    plt.savefig('data/charts/CFU_INCREASE_HOTEL_NIGHTS.svg')\n",
    "    return c    \n",
    "\n",
    "c=make_dfs_h(hotel_night, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_full(x):\n",
    "    pd.set_option('display.max_rows', len(x))\n",
    "    print(x)\n",
    "    pd.reset_option('display.max_rows')\n",
    "    \n",
    "#print_full(c)\n",
    "c.corr(method ='pearson') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_dfs_r(h,n):\n",
    "    colors = ['#00008b', '#1e90ff', '#40e0d0', 'g', 'y']\n",
    "    fig, ax = plt.subplots(figsize=(12,8))\n",
    "    b = []\n",
    "    for dat in h:\n",
    "        for j in dat[n]:\n",
    "            b.append(j)\n",
    "    c = pd.DataFrame(b)\n",
    "    sns.set_palette(colors)\n",
    "    fmt = '{%x:,.0f}'\n",
    "    tick = mtick.StrMethodFormatter(fmt)\n",
    "    ax.yaxis.set_major_formatter(\n",
    "        mtick.FuncFormatter(lambda y,  p: format(int(y), ',')))    \n",
    "    ax=sns.swarmplot(x='rain', y='cfu', hue='name', data=c,  alpha=0.8, size= 8, linewidth=1, edgecolor='w')\n",
    "    #ax=sns.scatterplot(x='nights', y='cfu', hue='name', data=c,  alpha=0.8, linewidth=3, edgecolor='w')\n",
    "    ax.set_xlabel(\"Montreux 72 hour rainfall total in millimeters\", size=16, labelpad=20)\n",
    "    ax.set_ylabel(\"Colony forming units per 100mL\", size=16, labelpad=20)\n",
    "    plt.suptitle(\"Rainfall and the occurence of coliform bacteria in lake water\" , fontsize=16,family='sans')\n",
    "    plt.title(\"Grouped by 72 hour rainfall total and colony color\", fontsize=16, family='sans', y=1.03)\n",
    "    ax.legend(loc='upper left', fontsize=14)\n",
    "    plt.xticks(size=14)\n",
    "    plt.yticks(size=14)\n",
    "    plt.savefig('data/charts/CFU_TOTAL_RAINFALL_YEAROVERYEAR.svg')\n",
    "    \n",
    "make_dfs_r(hotel_night, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}