create_simulations.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from simulation.notebook_utils import *\n",
    "from simulation.utils import *\n",
    "\n",
    "import busybee"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# [!] CHANGE `24` to `1` for shorter simulations\n",
    "SIM_TIME_MS = 24 * 3_600 * 1_000\n",
    "\n",
    "#\n",
    "# Global config for both ONLINE and OFFLINE\n",
    "#\n",
    "\n",
    "DEFAULT_CONFIG = NotebookSimulationConfig(\n",
    "    loopix_kwargs= {\n",
    "        'user_rate_pull': 1,\n",
    "        'user_rate_payload': 2,\n",
    "        'user_rate_drop': 2,\n",
    "        'user_rate_loop': 2,\n",
    "        'user_rate_delay': 3,\n",
    "        'mix_rate_loop': 2,\n",
    "        'mix_rate_loop_delay':3,\n",
    "    },\n",
    "    app_kwargs= {\n",
    "        'init_rate_per_second': 1/30,\n",
    "        'multi_message': 1,\n",
    "        'heavy_user_percentage': 20,\n",
    "        'heavy_user_weight': 4,\n",
    "    }\n",
    ")\n",
    "\n",
    "SLOW_RATE = 1/30\n",
    "\n",
    "CONFIG_NAMES_TO_CONFIGS = {\n",
    "#     'r15': DEFAULT_CONFIG,\n",
    "    'rs': DEFAULT_CONFIG.derive_new(\n",
    "        delta_app_kwargs = {\n",
    "            'init_rate_per_second': SLOW_RATE,\n",
    "            'multi_message': 1,\n",
    "        }),\n",
    "    'rs-m2': DEFAULT_CONFIG.derive_new(\n",
    "        delta_app_kwargs = {\n",
    "            'init_rate_per_second': SLOW_RATE,\n",
    "            'multi_message': 2,\n",
    "        }),\n",
    "    'rs-m4': DEFAULT_CONFIG.derive_new(\n",
    "        delta_app_kwargs = {\n",
    "            'init_rate_per_second': SLOW_RATE,\n",
    "            'multi_message': 4,\n",
    "        }),\n",
    "    'rs-m8': DEFAULT_CONFIG.derive_new(\n",
    "        delta_app_kwargs = {\n",
    "            'init_rate_per_second': SLOW_RATE,\n",
    "            'multi_message': 8,\n",
    "        }),\n",
    "    'rs-m16': DEFAULT_CONFIG.derive_new(\n",
    "        delta_app_kwargs = {\n",
    "            'init_rate_per_second': SLOW_RATE,\n",
    "            'multi_message': 16,\n",
    "        }),\n",
    "    'rs-m32': DEFAULT_CONFIG.derive_new(\n",
    "        delta_app_kwargs = {\n",
    "            'init_rate_per_second': SLOW_RATE,\n",
    "            'multi_message': 32,\n",
    "        }),\n",
    "    'rs-l1': DEFAULT_CONFIG.derive_new(\n",
    "        delta_loopix_kwargs = {\n",
    "            'user_rate_payload': 2,\n",
    "            'user_rate_drop': 2,\n",
    "            'user_rate_loop': 2,\n",
    "            'user_rate_delay': 3,\n",
    "            'mix_rate_loop': 2,\n",
    "            'mix_rate_loop_delay':3,\n",
    "        },\n",
    "        delta_app_kwargs = {\n",
    "            'init_rate_per_second': SLOW_RATE,\n",
    "        }),\n",
    "    'rs-l2': DEFAULT_CONFIG.derive_new(\n",
    "        delta_loopix_kwargs = {\n",
    "            'user_rate_payload': 2/2,\n",
    "            'user_rate_drop': 2/2,\n",
    "            'user_rate_loop': 2/2,\n",
    "            'user_rate_delay': 3/2,\n",
    "            'mix_rate_loop': 2/2,\n",
    "            'mix_rate_loop_delay':3/2,\n",
    "        },\n",
    "        delta_app_kwargs = {\n",
    "            'init_rate_per_second': SLOW_RATE,\n",
    "        }),\n",
    "    'rs-l4': DEFAULT_CONFIG.derive_new(\n",
    "        delta_loopix_kwargs = {\n",
    "            'user_rate_payload': 2/4,\n",
    "            'user_rate_drop': 2/4,\n",
    "            'user_rate_loop': 2/4,\n",
    "            'user_rate_delay': 3/4,\n",
    "            'mix_rate_loop': 2/4,\n",
    "            'mix_rate_loop_delay':3/4,\n",
    "        },\n",
    "        delta_app_kwargs = {\n",
    "            'init_rate_per_second': SLOW_RATE,\n",
    "        }),\n",
    "    'rs-l8': DEFAULT_CONFIG.derive_new(\n",
    "        delta_loopix_kwargs = {\n",
    "            'user_rate_payload': 2/8,\n",
    "            'user_rate_drop': 2/8,\n",
    "            'user_rate_loop': 2/8,\n",
    "            'user_rate_delay': 3/8,\n",
    "            'mix_rate_loop': 2/8,\n",
    "            'mix_rate_loop_delay':3/8,\n",
    "        },\n",
    "        delta_app_kwargs = {\n",
    "            'init_rate_per_second': SLOW_RATE,\n",
    "        }), \n",
    "    'rs-l16': DEFAULT_CONFIG.derive_new(\n",
    "        delta_loopix_kwargs = {\n",
    "            'user_rate_payload': 2/16,\n",
    "            'user_rate_drop': 2/16,\n",
    "            'user_rate_loop': 2/16,\n",
    "            'user_rate_delay': 3/16,\n",
    "            'mix_rate_loop': 2/16,\n",
    "            'mix_rate_loop_delay':3/16,\n",
    "        },\n",
    "        delta_app_kwargs = {\n",
    "            'init_rate_per_second': SLOW_RATE,\n",
    "        }),  \n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded schedules\n"
     ]
    }
   ],
   "source": [
    "DEFAULT_SCHEDULES = ['gen_os10', 'gen_os15', 'gen_os20']\n",
    "\n",
    "all_schedules = {k: read_compressed_bool_schedules(\"input/schedules_%s.txt.gz\" % k) for k in DEFAULT_SCHEDULES}\n",
    "all_schedules['online'] = None\n",
    "print(\"Loaded schedules\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pickle_to_file(args):\n",
    "    pickle_down({args[0]: args[1]})\n",
    "\n",
    "def create_sims(groupsizes, configs, schedules, strategies):\n",
    "    sims = dict()\n",
    "    for m, group_sizes in groupsizes:\n",
    "\n",
    "        for config_name in configs:\n",
    "            config = CONFIG_NAMES_TO_CONFIGS[config_name]\n",
    "\n",
    "            for schedule_name in schedules:\n",
    "                schedule = all_schedules[schedule_name]\n",
    "\n",
    "                for strategy_name in strategies:\n",
    "                        sim = create_scenario(m, group_sizes, config, strategy_name, SIM_TIME_MS, schedule if schedule else [])\n",
    "                        name = get_name_for_sim(sim, config_name, schedule_name, strategy_name)\n",
    "\n",
    "                        sims[name] = sim\n",
    "                        sim.name = name.replace('.input', '').upper()\n",
    "\n",
    "\n",
    "    _ = busybee.map(pickle_to_file, [(k, v) for k, v in sims.items()], processes=4)\n",
    "    print(\"Saved %d files\" % len(sims))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BusyBee: Start processing 21 items with 4 processes...\n",
      "BusyBee: 11/21, 52.4% (avg: 67ms cpu, rem: 0.2s)\n",
      "BusyBee: Finished processing 21 items in 0.9s (avg: 73ms cpu)\n",
      "Saved 21 files\n"
     ]
    }
   ],
   "source": [
    "#\n",
    "# ONLINE scenarios\n",
    "#\n",
    "\n",
    "ONLINE_GROUP_SIZES = [\n",
    "    (32, [32]),\n",
    "    (45, [45]),\n",
    "    (64, [64]),\n",
    "    (91, [91]),\n",
    "    (128, [128]),\n",
    "    (181, [181]),\n",
    "    (256, [256]),\n",
    "]\n",
    "ONLINE_CONFIGS = ['rs']\n",
    "ONLINE_SCHEDULES = ['online']\n",
    "ONLINE_STRATEGIES = ['unicast', 'rollercoaster-k1-p1-notimeout', 'rollercoaster-k2-p2-notimeout']\n",
    "\n",
    "create_sims(ONLINE_GROUP_SIZES, ONLINE_CONFIGS, ONLINE_SCHEDULES, ONLINE_STRATEGIES)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BusyBee: Start processing 24 items with 4 processes...\n",
      "BusyBee: 12/24, 50.0% (avg: 362ms cpu, rem: 1.1s)\n",
      "BusyBee: 18/24, 75.0% (avg: 372ms cpu, rem: 0.6s)\n",
      "BusyBee: Finished processing 24 items in 6.6s (avg: 363ms cpu)\n",
      "Saved 24 files\n",
      "BusyBee: Start processing 15 items with 4 processes...\n",
      "BusyBee:  8/15, 53.3% (avg: 657ms cpu, rem: 1.1s)\n",
      "BusyBee: Finished processing 15 items in 4.8s (avg: 607ms cpu)\n",
      "Saved 15 files\n"
     ]
    }
   ],
   "source": [
    "#\n",
    "# OFFLINE scenarios\n",
    "#\n",
    "\n",
    "OFFLINE_GROUP_SIZES = [\n",
    "    (128, [128]),\n",
    "    (256, [256]),\n",
    "]\n",
    "OFFLINE_SCHEDULES = ['gen_os10', 'gen_os15', 'gen_os20']\n",
    "OFFLINE_CONFIGS = ['rs']\n",
    "OFFLINE_SCHEDULES = DEFAULT_SCHEDULES[:]\n",
    "OFFLINE_STRATEGIES = [\n",
    "    'unicast',\n",
    "    'rollercoaster-k2-p2-notimeout',\n",
    "    'rollercoaster-k1-p1-timeout10x-dropoffline',\n",
    "    'rollercoaster-k2-p2-timeout10x-dropoffline',\n",
    "]\n",
    "\n",
    "# and for the histograms\n",
    "OFFLINE_HIST_GROUP_SIZES = [(128, [128])]\n",
    "OFFLINE_HIST_STRATEGIES = OFFLINE_STRATEGIES + ['rollercoaster-k2-p2-timeout10x'] \n",
    "\n",
    "create_sims(OFFLINE_GROUP_SIZES, OFFLINE_CONFIGS, OFFLINE_SCHEDULES, OFFLINE_STRATEGIES)\n",
    "create_sims(OFFLINE_HIST_GROUP_SIZES, OFFLINE_CONFIGS, OFFLINE_SCHEDULES, OFFLINE_HIST_STRATEGIES)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BusyBee: Start processing 30 items with 4 processes...\n",
      "BusyBee: 11/30, 36.7% (avg: 519ms cpu, rem: 2.5s)\n",
      "BusyBee: 15/30, 50.0% (avg: 455ms cpu, rem: 1.7s)\n",
      "BusyBee: Finished processing 30 items in 9.1s (avg: 370ms cpu)\n",
      "Saved 30 files\n"
     ]
    }
   ],
   "source": [
    "#\n",
    "# MULTIGROUP scenarios\n",
    "#\n",
    "\n",
    "MULTIGROUP_SIZE = 128\n",
    "MULTIGROUP_GROUP_SIZES = [\n",
    "    (MULTIGROUP_SIZE, [MULTIGROUP_SIZE]*1),\n",
    "    (MULTIGROUP_SIZE, [MULTIGROUP_SIZE]*2),\n",
    "    (MULTIGROUP_SIZE, [MULTIGROUP_SIZE]*4),\n",
    "    (MULTIGROUP_SIZE, [MULTIGROUP_SIZE]*8),\n",
    "    (MULTIGROUP_SIZE, [MULTIGROUP_SIZE]*16),\n",
    "]\n",
    "MULTIGROUP_CONFIGS = ['rs']\n",
    "MULTIGROUP_SCHEDULES = ['online', 'gen_os15']\n",
    "MULTIGROUP_STRATEGIES = [\n",
    "    'unicast',\n",
    "    'rollercoaster-k1-p1-timeout10x-dropoffline', \n",
    "    'rollercoaster-k2-p2-timeout10x-dropoffline', \n",
    "]\n",
    "\n",
    "create_sims(MULTIGROUP_GROUP_SIZES, MULTIGROUP_CONFIGS, MULTIGROUP_SCHEDULES, MULTIGROUP_STRATEGIES)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BusyBee: Start processing 36 items with 4 processes...\n",
      "BusyBee: 18/36, 50.0% (avg: 258ms cpu, rem: 1.2s)\n",
      "BusyBee: 24/36, 66.7% (avg: 240ms cpu, rem: 0.7s)\n",
      "BusyBee: Finished processing 36 items in 7.0s (avg: 215ms cpu)\n",
      "Saved 36 files\n"
     ]
    }
   ],
   "source": [
    "#\n",
    "# MULTIMESSAGE scenarios\n",
    "#\n",
    "\n",
    "MULTIMESSAGE_GROUP_SIZES = [\n",
    "    (128, [128])\n",
    "]\n",
    "MULTIMESSAGE_CONFIGS = ['rs', 'rs-m2', 'rs-m4', 'rs-m8', 'rs-m16', 'rs-m32']\n",
    "MULTIMESSAGE_SCHEDULES = ['online', 'gen_os15']\n",
    "MULTIMESSAGE_STRATEGIES = [\n",
    "    'unicast',\n",
    "    'rollercoaster-k1-p1-timeout10x-dropoffline', \n",
    "    'rollercoaster-k2-p2-timeout10x-dropoffline', \n",
    "]\n",
    "\n",
    "create_sims(MULTIMESSAGE_GROUP_SIZES, MULTIMESSAGE_CONFIGS, MULTIMESSAGE_SCHEDULES, MULTIMESSAGE_STRATEGIES)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BusyBee: Start processing 180 items with 4 processes...\n",
      "BusyBee:  20/180, 11.1% (avg: 302ms cpu, rem: 12.1s)\n",
      "BusyBee:  32/180, 17.8% (avg: 354ms cpu, rem: 13.1s)\n",
      "BusyBee:  55/180, 30.6% (avg: 306ms cpu, rem: 9.6s)\n",
      "BusyBee:  63/180, 35.0% (avg: 330ms cpu, rem: 9.7s)\n",
      "BusyBee:  90/180, 50.0% (avg: 315ms cpu, rem: 7.1s)\n",
      "BusyBee:  91/180, 50.6% (avg: 314ms cpu, rem: 7.0s)\n",
      "BusyBee: 107/180, 59.4% (avg: 319ms cpu, rem: 5.8s)\n",
      "BusyBee: 130/180, 72.2% (avg: 300ms cpu, rem: 3.8s)\n",
      "BusyBee: 144/180, 80.0% (avg: 307ms cpu, rem: 2.8s)\n",
      "BusyBee: 164/180, 91.1% (avg: 302ms cpu, rem: 1.2s)\n",
      "BusyBee: 176/180, 97.8% (avg: 310ms cpu, rem: 0.3s)\n",
      "BusyBee: Finished processing 180 items in 54.7s (avg: 311ms cpu)\n",
      "Saved 180 files\n"
     ]
    }
   ],
   "source": [
    "#\n",
    "# P scenarios\n",
    "#\n",
    "\n",
    "P_GROUP_SIZES = [\n",
    "    (128, [128]),\n",
    "]\n",
    "P_CONFIGS = ['rs', 'rs-l2', 'rs-l4', 'rs-l8', 'rs-l16']\n",
    "P_SCHEDULES = ['online', 'gen_os10', 'gen_os15', 'gen_os20']\n",
    "P_STRATEGIES = [\n",
    "    'rollercoaster-k1-p1-timeout10x-dropoffline', \n",
    "    'rollercoaster-k2-p2-timeout10x-dropoffline', \n",
    "    'rollercoaster-k4-p4-timeout10x-dropoffline', \n",
    "    'rollercoaster-k8-p8-timeout10x-dropoffline', \n",
    "    'rollercoaster-k16-p16-timeout10x-dropoffline', \n",
    "    'rollercoaster-k2-p1-timeout10x-dropoffline', \n",
    "    'rollercoaster-k4-p1-timeout10x-dropoffline', \n",
    "    'rollercoaster-k8-p1-timeout10x-dropoffline', \n",
    "    'rollercoaster-k16-p1-timeout10x-dropoffline', \n",
    "]\n",
    "\n",
    "create_sims(P_GROUP_SIZES, P_CONFIGS, P_SCHEDULES, P_STRATEGIES)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}