srw_feateng_v3.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "srw-feateng-v3",
      "provenance": [],
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "a216f87b43d44e5498e1987b2cf4f635": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_6f40505482d642b696db8a7cebafb340",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_543ab62af55c4d4cbdaa7d1d4e3cec02",
              "IPY_MODEL_c10f36895ec7437386d8f5fed4415d41"
            ]
          }
        },
        "6f40505482d642b696db8a7cebafb340": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "543ab62af55c4d4cbdaa7d1d4e3cec02": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_665f10f7c0d54edb90009dd99251bfa9",
            "_dom_classes": [],
            "description": "",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 35836,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 35836,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a9b33fd422cd4aaea652126f758c1aaf"
          }
        },
        "c10f36895ec7437386d8f5fed4415d41": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_db8082bb5c7942a4b1da52e045609972",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "100% 35836/35836 [01:00&lt;00:00, 593.96it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_73bf74eeef0f4954998b0fd901e88096"
          }
        },
        "665f10f7c0d54edb90009dd99251bfa9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a9b33fd422cd4aaea652126f758c1aaf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "db8082bb5c7942a4b1da52e045609972": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "73bf74eeef0f4954998b0fd901e88096": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d880cc6109aa4f35ac1272bd219109a0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_e784a9883d95416b83901e074452e320",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_b8e77023d9ca4932993de18bb28fcd71",
              "IPY_MODEL_a659387f0d3c45c88adeb57f7975ee2b"
            ]
          }
        },
        "e784a9883d95416b83901e074452e320": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b8e77023d9ca4932993de18bb28fcd71": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_c4b3d0f8da55425f982a8465881718c3",
            "_dom_classes": [],
            "description": "",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 6556,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 6556,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2ace0c65f0644970ab2d6e75f45b8d02"
          }
        },
        "a659387f0d3c45c88adeb57f7975ee2b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_b419bf81408948f4a9308aaaf6ddc868",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "100% 6556/6556 [00:42&lt;00:00, 154.73it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_cce34f136d76466ca0c1653eb139d345"
          }
        },
        "c4b3d0f8da55425f982a8465881718c3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2ace0c65f0644970ab2d6e75f45b8d02": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b419bf81408948f4a9308aaaf6ddc868": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "cce34f136d76466ca0c1653eb139d345": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/qwiksilva/cs224w-github-rec/blob/master/srw_feateng_v3.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2tFBL6vcou2i",
        "colab_type": "code",
        "outputId": "30f15a8c-cce8-44e2-dd17-e18c91f94c6c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/gdrive')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tbUKqIUio7-S",
        "colab_type": "code",
        "outputId": "b0647b9c-be1f-432e-ab84-2364f5ecd6ed",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187
        }
      },
      "source": [
        "!pip install chars2vec"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting chars2vec\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/04/0a/8c327aae23e0532d239ec7b30446aca765eb5d9547b4c4b09cdd82e49797/chars2vec-0.1.7.tar.gz (8.1MB)\n",
            "\u001b[K     |████████████████████████████████| 8.1MB 2.8MB/s \n",
            "\u001b[?25hBuilding wheels for collected packages: chars2vec\n",
            "  Building wheel for chars2vec (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for chars2vec: filename=chars2vec-0.1.7-cp36-none-any.whl size=8111095 sha256=a21de6ee9ef962a22f67788ed8427825b5ba6f3b3bf8283c6fa92cad7f1eebb5\n",
            "  Stored in directory: /root/.cache/pip/wheels/97/b6/65/d7e778ef1213ec77d315aea0f536068b96e36cc94c02abbfde\n",
            "Successfully built chars2vec\n",
            "Installing collected packages: chars2vec\n",
            "Successfully installed chars2vec-0.1.7\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "L4t4o_1PpQ82",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%tensorflow_version 1.x\n",
        "import tensorflow as tf\n",
        "import chars2vec\n",
        "c2v = chars2vec.load_model('eng_100')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dWAGX9uXpGnx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pandas as pd\n",
        "import os\n",
        "import numpy as np\n",
        "import seaborn as sns\n",
        "import joblib\n",
        "\n",
        "from tqdm import tqdm_notebook as tqdm\n",
        "from collections import defaultdict\n",
        "from absl import logging\n",
        "# Reduce logging output.\n",
        "logging.set_verbosity(logging.ERROR)\n",
        "\n",
        "%matplotlib inline\n",
        "sns.set(color_codes=True)\n",
        "\n",
        "home = \"/gdrive/My Drive/Colab Notebooks/cs224w-data/final-data-12062019\""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RQDIlKwspI9Q",
        "colab_type": "code",
        "outputId": "981f3969-4590-4abd-b34a-1ecaa444f094",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 170
        }
      },
      "source": [
        "filepath_df = pd.read_csv(os.path.join(home, 'final_filepath_df_12062019.csv'), index_col=0)\n",
        "filepath_df.info()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "Int64Index: 149858 entries, 13986 to 38381\n",
            "Data columns (total 4 columns):\n",
            "additions    149858 non-null int64\n",
            "changes      149858 non-null int64\n",
            "deletions    149858 non-null int64\n",
            "filename     149858 non-null object\n",
            "dtypes: int64(3), object(1)\n",
            "memory usage: 5.7+ MB\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZB-kY98DpWni",
        "colab_type": "code",
        "outputId": "3441c015-6f5e-40cb-c16a-d6e65c4c516c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 119
        }
      },
      "source": [
        "comments_df = pd.read_csv(os.path.join(home, 'final_comments_df_12062019.csv'), index_col=0, parse_dates=['pr_created_at', 'comment_created_at'])\n",
        "print(comments_df.pr_id.nunique())\n",
        "\n",
        "# Drop extra bots\n",
        "extra_bot_id = []\n",
        "for _, (id, name) in comments_df.loc[:, ['commenter_id', 'commenter_login']].drop_duplicates().iterrows():\n",
        "    if 'k8s-' in name or 'bot' in name:\n",
        "        print(name, id)\n",
        "        extra_bot_id.append(id)\n",
        "comments_df.drop(comments_df[comments_df.commenter_id.isin(extra_bot_id)].index, inplace=True)\n",
        "print(comments_df.pr_id.nunique())\n",
        "\n",
        "# Only consider PR in filepath_df\n",
        "comments_df = comments_df.loc[comments_df.pr_id.isin(filepath_df.index.values)].copy()\n",
        "print(comments_df.shape)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "27750\n",
            "k8s-oncall 13734568\n",
            "k8s-reviewable 24305385\n",
            "k8s-slack 14020429\n",
            "27750\n",
            "(384665, 10)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "orxpyLaYpzej",
        "colab_type": "code",
        "outputId": "0ff60937-2e73-4d0b-b949-1c6dd55037f5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "# Filter train/test\n",
        "\n",
        "start_train = '2018-01-01'\n",
        "start_test = '2018-06-01'\n",
        "end_test = '2019-01-01'\n",
        "\n",
        "train_filter = (comments_df.comment_created_at >= start_train) & (comments_df.comment_created_at < start_test) & (comments_df.pr_created_at >= start_train) & (comments_df.pr_created_at < start_test)\n",
        "test_filter = (comments_df.comment_created_at >= start_test)  & (comments_df.comment_created_at < end_test) & (comments_df.pr_created_at >= start_test) & (comments_df.pr_created_at < end_test)\n",
        "\n",
        "train_comments_df = comments_df.loc[train_filter].copy()\n",
        "test_comments_df = comments_df.loc[test_filter].copy()\n",
        "print(train_comments_df.shape, test_comments_df.shape)\n",
        "\n",
        "# More clean\n",
        "bad_train_pr_list_in_2018 = joblib.load(os.path.join(home, 'bad_train_pr_list_in_2018.joblib'))\n",
        "bad_test_pr_list_in_2018 = joblib.load(os.path.join(home, 'bad_test_pr_list_in_2018.joblib'))  \n",
        "train_comments_df.drop(train_comments_df[train_comments_df.pr_id.isin(bad_train_pr_list_in_2018)].index, inplace=True) # Bad no non-owner commenters\n",
        "test_comments_df.drop(test_comments_df[test_comments_df.pr_id.isin(bad_test_pr_list_in_2018)].index, inplace=True)\n",
        "test_comments_df = test_comments_df.loc[test_comments_df.commenter_id.isin(train_comments_df.commenter_id.unique())].copy() # No new users entering the graph\n",
        "print(train_comments_df.shape, test_comments_df.shape)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(31153, 10) (44001, 10)\n",
            "(31064, 10) (37927, 10)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IlIlaQI2F77R",
        "colab_type": "code",
        "outputId": "49a0c13e-bab4-4d1d-c5b7-d974f3658960",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "train_user_list = set(train_comments_df.commenter_id.unique().tolist())\n",
        "test_user_list = set(test_comments_df.commenter_id.unique().tolist())\n",
        "train_pr_list = set(train_comments_df.pr_id.unique().tolist())\n",
        "test_pr_list = set(test_comments_df.pr_id.unique().tolist())\n",
        "print(len(train_user_list), len(test_user_list), len(train_pr_list), len(test_pr_list))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "690 366 2719 3837\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nptuBIv5_3s6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def generate_pr_user_feat_df(core_df):\n",
        "    combine = [core_df.groupby(['commenter_id', 'pr_id'])['comment_created_at'].min(),\n",
        "               core_df.groupby(['commenter_id', 'pr_id'])['comment_created_at'].max(),\n",
        "               core_df.groupby(['commenter_id', 'pr_id'])['comment'].count()]\n",
        "    colnames = ['first_comment', 'last_comment', 'num_comment']\n",
        "    pr_user_feat_df = pd.concat(combine, axis=1)\n",
        "    pr_user_feat_df.columns = colnames\n",
        "    user_gb_list = [core_df.groupby(['user_id', 'pr_id'])['username'].nunique().groupby('user_id').count(),\n",
        "                    core_df.groupby(['commenter_id', 'pr_id'])['commenter_login'].nunique().groupby('commenter_id').count()]\n",
        "    user_df = pd.concat(user_gb_list, axis=1)\n",
        "    user_df.columns = ['num_pr_submitted', 'num_pr_commented']\n",
        "    user_df.rename_axis('commenter_id')\n",
        "    pr_user_feat_df = pr_user_feat_df.join(user_df, on='commenter_id').fillna(value=0)\n",
        "    pr_user_feat_df = pr_user_feat_df.join(core_df.groupby('pr_id')['pr_created_at'].max(), on='pr_id')\n",
        "    pr_user_feat_df['first_comment_age'] = (pr_user_feat_df['first_comment'] - pr_user_feat_df['pr_created_at']).apply(lambda x: x.total_seconds()/86400)\n",
        "    pr_user_feat_df['last_comment_age'] = (pr_user_feat_df['last_comment'] - pr_user_feat_df['pr_created_at']).apply(lambda x: x.total_seconds()/86400)\n",
        "    return pr_user_feat_df\n",
        "\n",
        "train_pr_user_df = generate_pr_user_feat_df(train_comments_df)\n",
        "test_pr_user_df = generate_pr_user_feat_df(test_comments_df)\n",
        "train_pr_owner_dict = train_comments_df.groupby('pr_id')['user_id'].min()\n",
        "test_pr_owner_dict = test_comments_df.groupby('pr_id')['user_id'].min()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m0_voRYI-luC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def generate_evaluation_dict(pr_user_df, owner_lookup):\n",
        "    out = defaultdict(list)\n",
        "    for (commenter_id, pr_id), last_comment_ts in pr_user_df['last_comment'].iteritems():\n",
        "        if commenter_id != owner_lookup[pr_id]:\n",
        "            out[pr_id].append((commenter_id, last_comment_ts))\n",
        "    for pr_id, pr_value in out.items():\n",
        "        out[pr_id] = [user_id for user_id, ts in sorted(pr_value, key=lambda x: x[1])]\n",
        "    return out\n",
        "\n",
        "train_dict = generate_evaluation_dict(train_pr_user_df, train_pr_owner_dict)\n",
        "test_dict = generate_evaluation_dict(test_pr_user_df, test_pr_owner_dict)\n",
        "\n",
        "# bad_train_pr_list_in_2018 = train_pr_list - train_dict.keys() # No non-owner commenters\n",
        "# print(len(bad_train_pr_list_in_2018))\n",
        "# joblib.dump(bad_train_pr_list_in_2018, os.path.join(home, 'bad_train_pr_list_in_2018.joblib'))\n",
        "# bad_test_pr_list_in_2018 = test_pr_list - test_dict.keys()\n",
        "# print(len(bad_test_pr_list_in_2018))\n",
        "# joblib.dump(bad_test_pr_list_in_2018, os.path.join(home, 'bad_test_pr_list_in_2018.joblib'))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PyQWPoSO_py3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "filepath_df = filepath_df.loc[train_pr_list | test_pr_list, :].copy()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Jm_cdv6kp-uC",
        "colab_type": "code",
        "outputId": "7b85473d-91d4-4f97-c8fb-7cc398aa7d3f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        }
      },
      "source": [
        "# Compile description\n",
        "lookup = train_comments_df.groupby('pr_id')['pr_created_at'].max()\n",
        "title_list = []\n",
        "for pr_id, ts in tqdm(lookup.iteritems()):\n",
        "    title = train_comments_df.loc[(train_comments_df.pr_id==pr_id) & (train_comments_df.pr_created_at==ts), 'title'].values[0]\n",
        "    title_list.append(title)\n",
        "train_title_df = lookup.reset_index()\n",
        "train_title_df['title'] = title_list\n",
        "\n",
        "lookup = test_comments_df.groupby('pr_id')['pr_created_at'].max()\n",
        "title_list = []\n",
        "for pr_id, ts in tqdm(lookup.iteritems()):\n",
        "    title = test_comments_df.loc[(test_comments_df.pr_id==pr_id) & (test_comments_df.pr_created_at==ts), 'title'].values[0]\n",
        "    title_list.append(title)\n",
        "test_title_df = lookup.reset_index()\n",
        "test_title_df['title'] = title_list\n",
        "\n",
        "print(train_title_df.shape, test_title_df.shape)\n",
        "title_df = pd.concat([train_title_df, test_title_df]).set_index('pr_id').drop_duplicates()\n",
        "print(title_df.shape)\n",
        "\n",
        "process = title_df.groupby('pr_id')['title'].unique()\n",
        "def combine(title_list):\n",
        "    out = []\n",
        "    for title in title_list:\n",
        "        words = title.lower().split()\n",
        "        out.extend(words)\n",
        "    return ' '.join(out)\n",
        "final_title_compilation = process.apply(lambda x: combine(x))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2719it [00:04, 620.69it/s]\n",
            "3837it [00:06, 619.57it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "(2719, 3) (3837, 3)\n",
            "(6556, 2)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Og-y6BypqfID",
        "colab_type": "code",
        "outputId": "07cd2539-caf6-4fd4-bf5b-ac0dc5d71141",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 115,
          "referenced_widgets": [
            "a216f87b43d44e5498e1987b2cf4f635",
            "6f40505482d642b696db8a7cebafb340",
            "543ab62af55c4d4cbdaa7d1d4e3cec02",
            "c10f36895ec7437386d8f5fed4415d41",
            "665f10f7c0d54edb90009dd99251bfa9",
            "a9b33fd422cd4aaea652126f758c1aaf",
            "db8082bb5c7942a4b1da52e045609972",
            "73bf74eeef0f4954998b0fd901e88096",
            "d880cc6109aa4f35ac1272bd219109a0",
            "e784a9883d95416b83901e074452e320",
            "b8e77023d9ca4932993de18bb28fcd71",
            "a659387f0d3c45c88adeb57f7975ee2b",
            "c4b3d0f8da55425f982a8465881718c3",
            "2ace0c65f0644970ab2d6e75f45b8d02",
            "b419bf81408948f4a9308aaaf6ddc868",
            "cce34f136d76466ca0c1653eb139d345"
          ]
        }
      },
      "source": [
        "# Run char2vec on filepath\n",
        "pr_filepath_list = filepath_df.loc[:, 'filename'].values\n",
        "split_out = []\n",
        "for filepath in tqdm(pr_filepath_list):\n",
        "    components = filepath.split('/')\n",
        "    vec = c2v.vectorize_words(components)\n",
        "    split_out.append(np.mean(vec, axis=0))\n",
        "filepath_df['c2v_filepath'] = split_out\n",
        "\n",
        "# Compile PR features for cosine similarity\n",
        "filepath_summary_df = pd.concat([filepath_df.groupby('pr_id')['additions', 'changes', 'deletions'].mean(), \n",
        "                                 filepath_df.groupby('pr_id')['c2v_filepath'].apply(lambda x: np.mean(x, axis=0)),\n",
        "                                 filepath_df.groupby('pr_id')['filename'].count(),\n",
        "                                 final_title_compilation], axis=1)\n",
        "filepath_summary_df.columns = ['average_additions', 'average_changes', 'average_deletions', 'average_c2v_filepath', 'filepath_count', 'title']\n",
        "\n",
        "# Run char2vec on title\n",
        "title_list = filepath_summary_df.loc[:, 'title'].values\n",
        "split_out = []\n",
        "for filepath in tqdm(title_list):\n",
        "    components = filepath.split()\n",
        "    vec = c2v.vectorize_words(components)\n",
        "    split_out.append(np.mean(vec, axis=0))\n",
        "filepath_summary_df['c2v_title'] = split_out"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "a216f87b43d44e5498e1987b2cf4f635",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=0, max=35836), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "d880cc6109aa4f35ac1272bd219109a0",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=0, max=6556), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nwWmagh5vUZs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "del c2v"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7Q3AoXadqt-b",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.preprocessing import StandardScaler\n",
        "\n",
        "def generate_pr_feat_matrix(pr_id_list, filepath_summary_df, train=True, scaler_dict={}):\n",
        "    col_list = ['average_additions', 'average_changes', 'average_deletions', 'average_c2v_filepath', 'filepath_count', 'c2v_title']\n",
        "    extract = filepath_summary_df.loc[pr_id_list, col_list]\n",
        "    print(extract.shape)\n",
        "    collection = []\n",
        "    for feat_name in extract.columns:\n",
        "        column = extract.loc[:, feat_name].values\n",
        "        if 'c2v' in feat_name:\n",
        "            column = np.vstack(column.tolist())\n",
        "        else:\n",
        "            if train:\n",
        "                scaler = StandardScaler()\n",
        "                column = scaler.fit_transform(column.reshape(-1, 1)) # Potential leakage\n",
        "                scaler_dict[feat_name] = scaler\n",
        "            else:\n",
        "                column = scaler_dict[feat_name].transform(column.reshape(-1, 1))\n",
        "        collection.append(column)\n",
        "    pr_feat_matrix = np.hstack(collection)\n",
        "    pr_list = extract.index.values.tolist() # Match up with cosine\n",
        "    pr_cosine_index_map = dict(zip(pr_list, range(len(pr_list))))\n",
        "    return pr_feat_matrix, pr_cosine_index_map, scaler_dict"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HB5N1_CUrw_E",
        "colab_type": "code",
        "outputId": "5fedbcd6-c0e7-4d88-e8aa-9d308e92773d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        }
      },
      "source": [
        "%%time\n",
        "pr_list = set(train_pr_list) | set(test_pr_list)\n",
        "pr_feat_matrix, pr_cosine_index_map, scaler_dict = generate_pr_feat_matrix(pr_list, filepath_summary_df)\n",
        "print(pr_feat_matrix.shape)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(6556, 6)\n",
            "(6556, 204)\n",
            "CPU times: user 38.3 ms, sys: 1.98 ms, total: 40.3 ms\n",
            "Wall time: 40.3 ms\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "65R0KOJa2Edd",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# filepath_summary_df = pd.read_csv(os.path.join('/gdrive/My Drive/Colab Notebooks/cs224w-data/', 'fp_summary_1209.csv'), index_col=0)\n",
        "# pr_feat_matrix = joblib.load(os.path.join('/gdrive/My Drive/Colab Notebooks/cs224w-data/', 'pr_feat_matrix_12092019.joblib'))\n",
        "# pr_cosine_index_map = joblib.load(os.path.join('/gdrive/My Drive/Colab Notebooks/cs224w-data/', 'pr_cosine_index_map_12092019.joblib'))\n",
        "# joblib.dump(pr_feat_matrix, 'pr_feat_matrix_12092019.joblib')\n",
        "# joblib.dump(pr_cosine_index_map, 'pr_cosine_index_map_12092019.joblib')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Mdk2Mjjf4e1Y",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.metrics.pairwise import cosine_similarity\n",
        "cosine_sim_matrix = cosine_similarity(pr_feat_matrix, pr_feat_matrix) # Take a lot of RAM here, need to crash to ask Colab for 25GB\n",
        "\n",
        "# Split half if crash\n",
        "# half = range(pr_feat_matrix.shape[0]//2)\n",
        "# other = range(pr_feat_matrix.shape[0]//2, pr_feat_matrix.shape[0])\n",
        "# print(half, other)\n",
        "# half_cosine_sim_matrix = cosine_similarity(pr_feat_matrix[half, ], pr_feat_matrix)\n",
        "# other_cosine_sim_matrix = cosine_similarity(pr_feat_matrix[other, ], pr_feat_matrix)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "alJw5EJMsEcl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from collections import defaultdict\n",
        "from scipy import sparse\n",
        "from tqdm import tqdm\n",
        "from sklearn.preprocessing import StandardScaler\n",
        "\n",
        "def generate_cosine_sim_feat(similarity_matrix, pr_index_map, pr_user_feat_df, feat_name):\n",
        "    users_to_pr = defaultdict(list)\n",
        "    pr_to_users = defaultdict(list)\n",
        "    for commenter_id, pr_id in pr_user_feat_df.index:\n",
        "        users_to_pr[commenter_id].append(pr_id)\n",
        "        pr_to_users[pr_id].append(commenter_id)\n",
        "\n",
        "    out = []\n",
        "    for user_id, source_pr_id in tqdm(pr_user_feat_df.index):\n",
        "        interacted_pr_list = users_to_pr[user_id]\n",
        "        if len(interacted_pr_list) == 0: \n",
        "            print(user_id)\n",
        "        elif len(interacted_pr_list) == 1: \n",
        "            out.append(0.0)\n",
        "        else:\n",
        "            cumm = []\n",
        "            for target_pr_id in interacted_pr_list:\n",
        "                if source_pr_id != target_pr_id:\n",
        "                    sim = similarity_matrix[pr_index_map[source_pr_id], pr_index_map[target_pr_id]]\n",
        "                    cumm.append(sim)\n",
        "            # if len(cumm) == 0:\n",
        "            #     print(user_id, source_pr_id, target_pr_id, interacted_pr_list)\n",
        "            out.append(sum(cumm)/len(cumm))\n",
        "    pr_user_feat_df[feat_name] = out\n",
        "\n",
        "    return pr_user_feat_df\n",
        "    \n",
        "\n",
        "def convert_adj_matrix(mat, row_map, column_map, row_name, col_name):\n",
        "    m, n = mat.shape\n",
        "    print(\"Number of %s = %d\" %(row_name, m))\n",
        "    print(\"Number of %s = %d\" %(col_name, n))\n",
        "    out = sparse.lil_matrix((m+n, m+n), dtype=type(mat[0, 0]))\n",
        "    # out = np.zeros((m+n, m+n), dtype=type(mat[0, 0])) # Sparse matrix, lower dtype\n",
        "\n",
        "    out[:m, m:] = mat\n",
        "    out[m:, :m] = mat.T\n",
        "    map_values = range(m + n)\n",
        "    map_keys = list(zip(list(row_map.keys()), [row_name]*len(row_map))) + list(zip(list(column_map.keys()), [col_name]*len(column_map)))\n",
        "    total_map = dict(zip(map_keys, map_values))\n",
        "    return out, total_map\n",
        "\n",
        "\n",
        "def process_sparse_matrix(feat, pr_user_feat_df):\n",
        "    series = pr_user_feat_df.loc[:, feat]\n",
        "\n",
        "    user_list = series.reset_index().commenter_id.unique().tolist()\n",
        "    pr_list = series.reset_index().pr_id.unique().tolist()\n",
        "    user_index_map = dict(zip(user_list, range(len(user_list))))\n",
        "    pr_index_map = dict(zip(pr_list, range(len(pr_list))))\n",
        "\n",
        "    # out = np.zeros((len(pr_index_map), len(user_index_map)), dtype=)\n",
        "    out = sparse.lil_matrix((len(pr_index_map), len(user_index_map)), dtype=type(series[0]))\n",
        "    for (user_id, pr_id), feat_value in series.iteritems():\n",
        "        out[pr_index_map[pr_id], user_index_map[user_id]] = feat_value\n",
        "\n",
        "    return convert_adj_matrix(out, pr_index_map, user_index_map, \"PR\", \"User\")\n",
        "\n",
        "\n",
        "def main(pr_user_feat_df, train=True, scaler_dict=None):\n",
        "    global cosine_sim_matrix, pr_cosine_index_map\n",
        "    cosine_feat_name = 'neighbor_PR_average_cosine_sim'\n",
        "    pr_user_feat_df = generate_cosine_sim_feat(cosine_sim_matrix, pr_cosine_index_map, pr_user_feat_df, cosine_feat_name)\n",
        "\n",
        "    selected = {\n",
        "        'num_comment': np.int16, \n",
        "        'num_pr_submitted': np.int16,\n",
        "        'num_pr_commented': np.int16,\n",
        "        'first_comment_age': np.float32,\n",
        "        'last_comment_age': np.float32,\n",
        "        cosine_feat_name: np.float32,\n",
        "    }\n",
        "    pr_user_feat_df = pr_user_feat_df.loc[:, selected.keys()].astype(selected).copy()\n",
        "\n",
        "    def rescale(colname, df, scaler=None):\n",
        "        if not scaler:\n",
        "            scaler = StandardScaler()\n",
        "            df.loc[:, colname] = scaler.fit_transform(df.loc[:, colname].values.reshape(-1, 1)).reshape(-1)\n",
        "        else:\n",
        "            df.loc[:, colname] = scaler.transform(df.loc[:, colname].values.reshape(-1, 1)).reshape(-1)\n",
        "        return scaler\n",
        "\n",
        "    srw_output = {}\n",
        "    node_mapping = None\n",
        "    if train:\n",
        "        train_scalers = {} \n",
        "    for feat in selected.keys():\n",
        "        print(feat, pr_user_feat_df.loc[:, feat].mean(), pr_user_feat_df.loc[:, feat].std())\n",
        "        if feat != cosine_feat_name:\n",
        "            if train:\n",
        "                train_scalers[feat] = rescale(feat, pr_user_feat_df)\n",
        "            else:\n",
        "                _ = rescale(feat, pr_user_feat_df, scaler_dict[feat])\n",
        "        print(feat, pr_user_feat_df.loc[:, feat].mean(), pr_user_feat_df.loc[:, feat].std())\n",
        "        output, node_mapping = process_sparse_matrix(feat, pr_user_feat_df)\n",
        "        srw_output[feat] = output\n",
        "        # joblib.dump(output, '%s_sparse_lil.joblib' %feat)\n",
        "\n",
        "    if train:\n",
        "        return srw_output, node_mapping, train_scalers\n",
        "    else:\n",
        "        return srw_output, node_mapping"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DcqbFuqFuobK",
        "colab_type": "code",
        "outputId": "b47684a4-c735-48f6-d75b-bac4b765c998",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 544
        }
      },
      "source": [
        "train_srw_output, train_node_mapping, train_scalers = main(train_pr_user_df, train=True)\n",
        "for feat_name, sparse_mat in train_srw_output.items():\n",
        "    print(feat_name, sparse_mat.shape)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 9153/9153 [00:00<00:00, 17239.40it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "num_comment 3.393859936632798 4.409555527810614\n",
            "num_comment 4.1898092119088995e-16 1.000054631374828\n",
            "Number of PR = 2719\n",
            "Number of User = 690\n",
            "num_pr_submitted 21.608106631705454 23.324147872122925\n",
            "num_pr_submitted -1.4220145994966702e-15 1.0000546313748258\n",
            "Number of PR = 2719\n",
            "Number of User = 690\n",
            "num_pr_commented 112.15415710695947 126.1668109762065\n",
            "num_pr_commented -1.1223647622811139e-15 1.0000546313747958\n",
            "Number of PR = 2719\n",
            "Number of User = 690\n",
            "first_comment_age 4.455038070678711 10.988587379455566\n",
            "first_comment_age 5.446339557124702e-08 1.0000513792037964\n",
            "Number of PR = 2719\n",
            "Number of User = 690\n",
            "last_comment_age 8.868515014648438 15.646407127380371\n",
            "last_comment_age -3.46765780534497e-08 1.0000548362731934\n",
            "Number of PR = 2719\n",
            "Number of User = 690\n",
            "neighbor_PR_average_cosine_sim 0.5762746334075928 0.14846810698509216\n",
            "neighbor_PR_average_cosine_sim 0.5762746334075928 0.14846810698509216\n",
            "Number of PR = 2719\n",
            "Number of User = 690\n",
            "num_comment (3409, 3409)\n",
            "num_pr_submitted (3409, 3409)\n",
            "num_pr_commented (3409, 3409)\n",
            "first_comment_age (3409, 3409)\n",
            "last_comment_age (3409, 3409)\n",
            "neighbor_PR_average_cosine_sim (3409, 3409)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_j7IhPGyuqyh",
        "colab_type": "code",
        "outputId": "de49dae1-8a10-4274-8909-f50cc4e90527",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 544
        }
      },
      "source": [
        "test_srw_output, test_node_mapping = main(test_pr_user_df, train=False, scaler_dict=train_scalers)\n",
        "for feat_name, sparse_mat in test_srw_output.items():\n",
        "    print(feat_name, sparse_mat.shape)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 12313/12313 [00:01<00:00, 10703.96it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "num_comment 3.080240396329083 4.217802419761555\n",
            "num_comment -0.07112659581952308 0.9565664424688908\n",
            "Number of PR = 3837\n",
            "Number of User = 366\n",
            "num_pr_submitted 28.631527653699344 30.601134587218887\n",
            "num_pr_submitted 0.3011387494046489 1.3120653554872068\n",
            "Number of PR = 3837\n",
            "Number of User = 366\n",
            "num_pr_commented 183.6388370015431 199.45433255639102\n",
            "num_pr_commented 0.5666195780632681 1.5809643398089193\n",
            "Number of PR = 3837\n",
            "Number of User = 366\n",
            "first_comment_age 5.807437419891357 14.945874214172363\n",
            "first_comment_age 0.12308144569396973 1.36019766330719\n",
            "Number of PR = 3837\n",
            "Number of User = 366\n",
            "last_comment_age 10.727508544921875 21.2080078125\n",
            "last_comment_age 0.11882064491510391 1.3555282354354858\n",
            "Number of PR = 3837\n",
            "Number of User = 366\n",
            "neighbor_PR_average_cosine_sim 0.5818747878074646 0.10965071618556976\n",
            "neighbor_PR_average_cosine_sim 0.5818747878074646 0.10965071618556976\n",
            "Number of PR = 3837\n",
            "Number of User = 366\n",
            "num_comment (4203, 4203)\n",
            "num_pr_submitted (4203, 4203)\n",
            "num_pr_commented (4203, 4203)\n",
            "first_comment_age (4203, 4203)\n",
            "last_comment_age (4203, 4203)\n",
            "neighbor_PR_average_cosine_sim (4203, 4203)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j9sE130TuthE",
        "colab_type": "code",
        "outputId": "34dd2073-eccf-4a90-908d-1f85d2f7b02f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "joblib.dump(train_srw_output, os.path.join(home, 'train_srw_output_v4_2018_only.joblib'))\n",
        "joblib.dump(train_node_mapping, os.path.join(home, 'train_node_mapping_v4_2018_only.joblib'))\n",
        "joblib.dump(test_srw_output, os.path.join(home, 'test_srw_output_v4_2018_only.joblib'))\n",
        "joblib.dump(test_node_mapping, os.path.join(home, 'test_node_mapping_v4_2018_only.joblib'))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['/gdrive/My Drive/Colab Notebooks/cs224w-data/final-data-12062019/test_node_mapping_v4_2018_only.joblib']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 132
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "n_OWSDCn7GnE",
        "colab_type": "code",
        "outputId": "4459fa08-eccd-40cf-b059-177478a2ad17",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "joblib.dump(train_comments_df, os.path.join(home, 'train_comments_df_v4_2018_only.joblib'))\n",
        "joblib.dump(test_comments_df, os.path.join(home, 'test_comments_df_v4_2018_only.joblib'))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['/gdrive/My Drive/Colab Notebooks/cs224w-data/final-data-12062019/test_comments_df_v4_2018_only.joblib']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 133
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PwOg4ayq7TG6",
        "colab_type": "code",
        "outputId": "77638710-3259-4024-ecc9-8e6afe0aa032",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "joblib.dump(filepath_summary_df, os.path.join(home, 'filepath_feat_df_v4_2018_only.joblib'))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['/gdrive/My Drive/Colab Notebooks/cs224w-data/final-data-12062019/filepath_feat_df_v4_2018_only.joblib']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 136
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UkKuQAMFIkWG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def dump_stuff(obj, fname):\n",
        "    return joblib.dump(obj, os.path.join(home, '%s_v4_2018_only.joblib'%fname))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sIyM6JV3JTrO",
        "colab_type": "code",
        "outputId": "180e35b7-3302-42c6-b30f-d251fb2808ba",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "dump_stuff(train_dict, 'train_dict')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['/gdrive/My Drive/Colab Notebooks/cs224w-data/final-data-12062019/train_dict_v4_2018_only.joblib']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 140
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8sPO5RO_I8_D",
        "colab_type": "code",
        "outputId": "0bae9f82-9267-47b7-a4c1-b05c63a790eb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "dump_stuff(test_dict, 'test_dict')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['/gdrive/My Drive/Colab Notebooks/cs224w-data/final-data-12062019/test_dict_v4_2018_only.joblib']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 141
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lsBR57X9Jk2p",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}