twitter_different_dist.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "from detectors import *\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import time\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def time_fit(det, data, labels):\n",
    "    time_begin = time.time()\n",
    "        \n",
    "    det = det.fit(np.array(data), targets=np.array(labels))\n",
    "    \n",
    "    return det, time.time() - time_begin\n",
    "\n",
    "def time_test(det, data):\n",
    "    time_begin = time.time()\n",
    "        \n",
    "    result = det.predict_proba(np.array(data))\n",
    "    \n",
    "    return det, time.time() - time_begin, result\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "modes = ['bert_768', 'bow_50', 'bow_768']\n",
    "subsets = {}\n",
    "\n",
    "result_pickle = 'data/results/twitter_diff_dist.pickle'\n",
    "\n",
    "for mode in modes:\n",
    "    with open('data/twitter/twitter_{mode}_diff_dist.pickle'.format(mode=mode), 'rb') as handle:\n",
    "        subsets[mode] = pickle.load(handle)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# initiate detectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "detectors = {\n",
    "    'csdd': CosineSimilarityDriftDetector(),\n",
    "    'kts' : KernelTwoSampleDriftDetector(),\n",
    "    'aks' : AlibiKSDetector(),\n",
    "    'ammd': AlibiMMDDetector(),\n",
    "    'lsdd': AlibiLSDDDetector(),\n",
    "    'cdbd': CDBDDetector()\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# tests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "if os.path.isfile(result_pickle):  # Do not overwrite\n",
    "    print('Loading result pickle: ', result_pickle)\n",
    "    with open(result_pickle, 'rb') as handle:\n",
    "        results_124, results_192, results_480 = pickle.load(handle)\n",
    "else:\n",
    "    results_124 = {mode: {detector: {} for detector in detectors} for mode in modes}\n",
    "    results_192 = {mode: {detector: {} for detector in detectors} for mode in modes}\n",
    "    results_480 = {mode: {detector: {} for detector in detectors} for mode in modes}\n",
    "\n",
    "for detector in detectors:\n",
    "    for mode in modes:\n",
    "        if not detector in results_124[mode]:\n",
    "            results_124[mode][detector] = {}\n",
    "            results_192[mode][detector] = {}\n",
    "            results_480[mode][detector] = {}\n",
    "        if 'predictions' in results_124[mode][detector]: # skip already computed\n",
    "            continue\n",
    "        \n",
    "        results_124[mode][detector]['predictions'] = []\n",
    "        results_192[mode][detector]['predictions'] = []\n",
    "        results_480[mode][detector]['predictions'] = []\n",
    "        \n",
    "        results_124[mode][detector]['time_detect'] = []\n",
    "        \n",
    "        for permutation_num in range(8):\n",
    "            det, t = time_fit(detectors[detector],\n",
    "                              subsets[mode]['100'][0][permutation_num] + subsets[mode]['100'][1][permutation_num],\n",
    "                              [0 for x in subsets[mode]['100'][0][permutation_num]] + [1 for x in subsets[mode]['100'][1][permutation_num]]\n",
    "                             )\n",
    "            results_124[mode][detector]['time_fit'] = t\n",
    "            \n",
    "            permutation_124 = subsets[mode]['124'][0][permutation_num] + subsets[mode]['124'][1][permutation_num]\n",
    "            det_2, t, res = time_test(det, permutation_124)\n",
    "            results_124[mode][detector]['predictions'].append(res)\n",
    "            results_124[mode][detector]['time_detect'].append(t)\n",
    "            \n",
    "            permutation_192 = subsets[mode]['192'][0][permutation_num] + subsets[mode]['192'][1][permutation_num]\n",
    "            det_2, t, res = time_test(det, permutation_192)\n",
    "            results_192[mode][detector]['predictions'].append(res)\n",
    "            \n",
    "            permutation_480 = subsets[mode]['480'][0][permutation_num] + subsets[mode]['480'][1][permutation_num]\n",
    "            det_2, t, res = time_test(det, permutation_480)\n",
    "            results_480[mode][detector]['predictions'].append(res)\n",
    "\n",
    "with open(result_pickle, 'wb') as handle:\n",
    "    pickle.dump((results_124, results_192, results_480), handle)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# plot graphs TODO"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for mode in modes:\n",
    "    for detector in detectors:\n",
    "        res = [np.mean(r[mode][detector]['predictions']) for r in (results_124, results_192, results_480)]\n",
    "        plt.plot(res)\n",
    "        plt.xticks(ticks=range(0,3), labels=['124 + 24h', '192 + 24h', '480 + 24h'], rotation=60)\n",
    "        plt.savefig('figures/by_dataset/twitter_{mode}_diff_dist_{detector}.pdf'.format(mode=mode, detector=detector), format='pdf')\n",
    "        plt.show()\n",
    "    for detector in detectors:\n",
    "        res = [np.mean(r[mode][detector]['predictions']) for r in (results_124, results_192, results_480)]\n",
    "        plt.plot(res)\n",
    "    plt.xticks(ticks=range(0,3), labels=['124 + 24h', '192 + 24h', '480 + 24h'], rotation=60)\n",
    "    plt.savefig('figures/by_dataset/twitter_{mode}_diff_dist_all.pdf'.format(mode=mode), format='pdf')\n",
    "    plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "language_info": {
   "name": "python",
   "pygments_lexer": "ipython3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}