train_openweb_original.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "id": "initial_id",
   "metadata": {
    "collapsed": true,
    "ExecuteTime": {
     "end_time": "2024-11-23T07:27:01.258528Z",
     "start_time": "2024-11-23T07:26:57.938513Z"
    }
   },
   "source": [
    "from datasets import load_dataset\n",
    "from tqdm import tqdm\n",
    "import ssl\n",
    "ssl._create_default_https_context = ssl._create_unverified_context\n",
    "dataset = load_dataset(\"openwebtext\", trust_remote_code=True) # 45 mins first time, after that 1.5 mins"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Loading dataset shards:   0%|          | 0/80 [00:00<?, ?it/s]"
      ],
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "18dcdc0520c94c85b5952fe6aef349be"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "execution_count": 1
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-11-23T07:27:53.658542Z",
     "start_time": "2024-11-23T07:27:53.402228Z"
    }
   },
   "cell_type": "code",
   "source": [
    "import json\n",
    "\n",
    "# Load vocab\n",
    "flyvec_embeddings_path = 'simple-flyvec-embeddings.json'\n",
    "with open(flyvec_embeddings_path, 'r') as file:\n",
    "    embeddings = json.load(file)\n",
    "\n",
    "\n",
    "vocab = {word: idx for idx, word in enumerate(embeddings.keys())}"
   ],
   "id": "7f03459056c43a0a",
   "outputs": [],
   "execution_count": 2
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-11-23T08:12:12.247401Z",
     "start_time": "2024-11-23T07:27:55.055610Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from encoder import Encoder\n",
    "from flyvec_model import FlyvecModel\n",
    "from collections import Counter\n",
    "import numpy as np\n",
    "import utils\n",
    "\n",
    "encoder = Encoder(vocab=vocab)\n",
    "\n",
    "\n",
    "# Create model\n",
    "model = FlyvecModel(\n",
    "    K_size=400,            # Number of neurons\n",
    "    vocab_size=len(vocab),  # Size of vocab\n",
    "    k=1,                    # Update top-k neurons\n",
    "    lr=.2,                  # Learning rate\n",
    "    norm_rate=1,            # Normalization rate\n",
    "    create_target_vector=True\n",
    ")\n",
    "\n",
    "window_size = 10\n",
    "\n",
    "id_counter = Counter()\n",
    "windows_count = 0\n",
    "passage_count = 0\n",
    "\n",
    "for passage in tqdm(dataset['train'], desc=\"Processing Passages\"):\n",
    "    passage_count += 1\n",
    "\n",
    "    text = passage['text']\n",
    "    preprocessed_text = encoder.preprocess(text, remove_stopwords=True)\n",
    "\n",
    "    words_arr = np.array(preprocessed_text)\n",
    "    words_arr = words_arr[:len(words_arr) - len(words_arr) % window_size]\n",
    "    train_data = words_arr.reshape(-1, window_size)\n",
    "\n",
    "    for window in train_data:\n",
    "        tokenized_window = encoder.tokenize(window.tolist())\n",
    "        one_hot = encoder.one_hot(tokenized_window, create_target_vector=True)\n",
    "        model.update(one_hot)\n",
    "        windows_count += 1\n",
    "        id_counter.update(tokenized_window)\n",
    "\n",
    "    # Save the model every 80,000 (1% of dataset)\n",
    "    if passage_count % 80000 == 0:\n",
    "        pct = passage_count // 80000\n",
    "        utils.save_model(model, f\"trained_models/original_openwebtext_checkpoints/model_checkpoint_{pct}pct.pt\")   "
   ],
   "id": "8694ca51d45abe7c",
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing Passages:   5%|▌         | 408220/8013769 [44:16<13:44:48, 153.68it/s]\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[0;31mKeyboardInterrupt\u001B[0m                         Traceback (most recent call last)",
      "Cell \u001B[0;32mIn[3], line 39\u001B[0m\n\u001B[1;32m     37\u001B[0m tokenized_window \u001B[38;5;241m=\u001B[39m encoder\u001B[38;5;241m.\u001B[39mtokenize(window\u001B[38;5;241m.\u001B[39mtolist())\n\u001B[1;32m     38\u001B[0m one_hot \u001B[38;5;241m=\u001B[39m encoder\u001B[38;5;241m.\u001B[39mone_hot(tokenized_window, create_target_vector\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mTrue\u001B[39;00m)\n\u001B[0;32m---> 39\u001B[0m \u001B[43mmodel\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mupdate\u001B[49m\u001B[43m(\u001B[49m\u001B[43mone_hot\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m     40\u001B[0m windows_count \u001B[38;5;241m+\u001B[39m\u001B[38;5;241m=\u001B[39m \u001B[38;5;241m1\u001B[39m\n\u001B[1;32m     41\u001B[0m id_counter\u001B[38;5;241m.\u001B[39mupdate(tokenized_window)\n",
      "File \u001B[0;32m~/PycharmProjects/flyvec_extensions/flyvec_model.py:25\u001B[0m, in \u001B[0;36mFlyvecModel.update\u001B[0;34m(self, x)\u001B[0m\n\u001B[1;32m     23\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21mupdate\u001B[39m(\u001B[38;5;28mself\u001B[39m, x):\n\u001B[1;32m     24\u001B[0m     \u001B[38;5;66;03m# Find indices of words in input (onehot vocab)\u001B[39;00m\n\u001B[0;32m---> 25\u001B[0m     active_indices \u001B[38;5;241m=\u001B[39m \u001B[43mx\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mnonzero\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\u001B[38;5;241m.\u001B[39msqueeze(\u001B[38;5;241m1\u001B[39m)\n\u001B[1;32m     27\u001B[0m     \u001B[38;5;66;03m# Calculate top k activations\u001B[39;00m\n\u001B[1;32m     28\u001B[0m     activations \u001B[38;5;241m=\u001B[39m torch\u001B[38;5;241m.\u001B[39msum(\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mW[:, active_indices], dim\u001B[38;5;241m=\u001B[39m\u001B[38;5;241m1\u001B[39m)\n",
      "\u001B[0;31mKeyboardInterrupt\u001B[0m: "
     ]
    }
   ],
   "execution_count": 3
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-11-23T08:12:14.290539Z",
     "start_time": "2024-11-23T08:12:14.271633Z"
    }
   },
   "cell_type": "code",
   "source": [
    "word_counter = {word: id_counter.get(id, 0) for word, id in vocab.items()}\n",
    "word_counter = dict(sorted(word_counter.items(), key=lambda x: x[1], reverse=True))\n",
    "word_counter"
   ],
   "id": "3db50022c470ed91",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'<UNK>': 38155221,\n",
       " '<NUM>': 8305884,\n",
       " 'new': 612615,\n",
       " 'like': 596510,\n",
       " 'people': 584814,\n",
       " 'time': 567202,\n",
       " 'first': 473229,\n",
       " 'year': 422146,\n",
       " 'years': 346766,\n",
       " 'may': 314285,\n",
       " 'way': 307553,\n",
       " 'well': 304598,\n",
       " 'last': 304433,\n",
       " 'back': 302307,\n",
       " 'world': 296989,\n",
       " 'see': 289925,\n",
       " 'state': 284012,\n",
       " 'know': 261798,\n",
       " 'work': 258707,\n",
       " 'made': 258177,\n",
       " 'game': 257141,\n",
       " 'think': 254245,\n",
       " 'still': 248900,\n",
       " 'right': 247901,\n",
       " 'going': 247409,\n",
       " 'government': 236486,\n",
       " 'day': 234180,\n",
       " 'take': 230045,\n",
       " 'says': 227783,\n",
       " 'want': 226943,\n",
       " 'team': 209455,\n",
       " 'say': 208699,\n",
       " 'long': 207710,\n",
       " 'trump': 205410,\n",
       " 'need': 204438,\n",
       " 'part': 199503,\n",
       " 'president': 190425,\n",
       " 'city': 188521,\n",
       " 'life': 187150,\n",
       " 'told': 180098,\n",
       " 'high': 179943,\n",
       " 'public': 179528,\n",
       " 'police': 175530,\n",
       " 'end': 175080,\n",
       " 'best': 170617,\n",
       " 'next': 170271,\n",
       " 'old': 164880,\n",
       " 'system': 164616,\n",
       " 'found': 161412,\n",
       " 'states': 159463,\n",
       " 'great': 159280,\n",
       " 'according': 157966,\n",
       " 'man': 157629,\n",
       " 'come': 157524,\n",
       " 'including': 156748,\n",
       " 'called': 155835,\n",
       " 'home': 154673,\n",
       " 'group': 154023,\n",
       " 'little': 150558,\n",
       " 'help': 150275,\n",
       " 'point': 150249,\n",
       " 'show': 150224,\n",
       " 'look': 149932,\n",
       " 'season': 149812,\n",
       " 'set': 149296,\n",
       " 'number': 149037,\n",
       " 'million': 148321,\n",
       " 'week': 147764,\n",
       " 'second': 147534,\n",
       " 'support': 147016,\n",
       " 'place': 146400,\n",
       " 'news': 145343,\n",
       " 'using': 144671,\n",
       " 'american': 144565,\n",
       " 'country': 144266,\n",
       " 'times': 143872,\n",
       " 'case': 143595,\n",
       " 'women': 143504,\n",
       " 'big': 143405,\n",
       " 'left': 142909,\n",
       " 'national': 142544,\n",
       " 'data': 142382,\n",
       " 'house': 142151,\n",
       " 'play': 139976,\n",
       " 'find': 139598,\n",
       " 'power': 138877,\n",
       " 'based': 137828,\n",
       " 'company': 136736,\n",
       " 'top': 135090,\n",
       " 'free': 133659,\n",
       " 'law': 133603,\n",
       " 'put': 132351,\n",
       " 'change': 131631,\n",
       " 'percent': 131075,\n",
       " 'party': 130883,\n",
       " 'later': 130125,\n",
       " 'united': 129372,\n",
       " 'today': 128192,\n",
       " 'money': 127876,\n",
       " 'far': 127730,\n",
       " 'let': 125322,\n",
       " 'story': 125258,\n",
       " 'white': 124181,\n",
       " 'family': 123698,\n",
       " 'school': 123619,\n",
       " 'making': 123396,\n",
       " 'war': 122820,\n",
       " 'able': 118116,\n",
       " 'away': 117726,\n",
       " 'days': 117564,\n",
       " 'media': 117124,\n",
       " 'real': 116022,\n",
       " 'political': 115134,\n",
       " 'health': 114245,\n",
       " 'games': 114138,\n",
       " 'working': 114094,\n",
       " 'least': 113031,\n",
       " 'com': 112304,\n",
       " 'mr': 111913,\n",
       " 'came': 111461,\n",
       " 'run': 111391,\n",
       " 'court': 111335,\n",
       " 'full': 111137,\n",
       " 'name': 110151,\n",
       " 'actually': 110113,\n",
       " 'men': 109510,\n",
       " 'important': 109306,\n",
       " 'report': 108544,\n",
       " 'keep': 107998,\n",
       " 'order': 107896,\n",
       " 'start': 107862,\n",
       " 'open': 107596,\n",
       " 'early': 107465,\n",
       " 'took': 107221,\n",
       " 'university': 107188,\n",
       " 'give': 106121,\n",
       " 'small': 105956,\n",
       " 'post': 105923,\n",
       " 'past': 105498,\n",
       " 'children': 105278,\n",
       " 'business': 104919,\n",
       " 'line': 104459,\n",
       " 'level': 104263,\n",
       " 'security': 104108,\n",
       " 'players': 103991,\n",
       " 'social': 103953,\n",
       " 'human': 102399,\n",
       " 'known': 101398,\n",
       " 'community': 101326,\n",
       " 'given': 101125,\n",
       " 'video': 99974,\n",
       " 'read': 99529,\n",
       " 'campaign': 99111,\n",
       " 'seen': 98943,\n",
       " 'black': 98528,\n",
       " 'obama': 98523,\n",
       " 'example': 98327,\n",
       " 'hard': 97889,\n",
       " 'water': 97620,\n",
       " 'future': 97320,\n",
       " 'went': 97202,\n",
       " 'general': 96748,\n",
       " 'likely': 96578,\n",
       " 'control': 95930,\n",
       " 'feel': 95378,\n",
       " 'local': 95321,\n",
       " 'night': 95311,\n",
       " 'series': 95028,\n",
       " 'history': 94952,\n",
       " 'side': 94871,\n",
       " 'sure': 94810,\n",
       " 'following': 94725,\n",
       " 'person': 94477,\n",
       " 'asked': 94372,\n",
       " 'york': 94321,\n",
       " 'months': 94061,\n",
       " 'course': 93313,\n",
       " 'members': 93092,\n",
       " 'process': 93000,\n",
       " 'head': 92556,\n",
       " 'large': 92523,\n",
       " 'possible': 92482,\n",
       " 'across': 92251,\n",
       " 'means': 92243,\n",
       " 'love': 91898,\n",
       " 'month': 91281,\n",
       " 'office': 91058,\n",
       " 'research': 91003,\n",
       " 'young': 90609,\n",
       " 'kind': 90471,\n",
       " 'looking': 89822,\n",
       " 'service': 89660,\n",
       " 'project': 89285,\n",
       " 'believe': 89205,\n",
       " 'north': 89040,\n",
       " 'thought': 88963,\n",
       " 'book': 88284,\n",
       " 'makes': 88024,\n",
       " 'problem': 87854,\n",
       " 'call': 87797,\n",
       " 'live': 87278,\n",
       " 'area': 86914,\n",
       " 'major': 86863,\n",
       " 'market': 86738,\n",
       " 'federal': 86074,\n",
       " 'rights': 84811,\n",
       " 'half': 84460,\n",
       " 'issue': 84331,\n",
       " 'recent': 84315,\n",
       " 'america': 84247,\n",
       " 'started': 84242,\n",
       " 'added': 84226,\n",
       " 'bill': 83518,\n",
       " 'international': 83426,\n",
       " 'policy': 83172,\n",
       " 'ago': 82999,\n",
       " 'care': 82930,\n",
       " 'trying': 82798,\n",
       " 'idea': 82445,\n",
       " 'department': 81951,\n",
       " 'comes': 81884,\n",
       " 'experience': 81881,\n",
       " 'military': 81821,\n",
       " 'deal': 81764,\n",
       " 'player': 81340,\n",
       " 'south': 81310,\n",
       " 'move': 81074,\n",
       " 'john': 80218,\n",
       " 'class': 80075,\n",
       " 'program': 80023,\n",
       " 'space': 79947,\n",
       " 'study': 79635,\n",
       " 'try': 79604,\n",
       " 'job': 79451,\n",
       " 'clear': 79427,\n",
       " 'question': 79200,\n",
       " 'current': 79102,\n",
       " 'death': 78733,\n",
       " 'saying': 78724,\n",
       " 'third': 78517,\n",
       " 'face': 78300,\n",
       " 'single': 78073,\n",
       " 'bit': 78044,\n",
       " 'taken': 77906,\n",
       " 'center': 77718,\n",
       " 'energy': 77685,\n",
       " 'list': 77662,\n",
       " 'taking': 77444,\n",
       " 'food': 77365,\n",
       " 'image': 77249,\n",
       " 'film': 76646,\n",
       " 'pay': 76570,\n",
       " 'action': 76530,\n",
       " 'close': 75890,\n",
       " 'body': 75474,\n",
       " 'plan': 75447,\n",
       " 'wanted': 75347,\n",
       " 'coming': 75286,\n",
       " 'hand': 75198,\n",
       " 'al': 75085,\n",
       " 'development': 75039,\n",
       " 'low': 74253,\n",
       " 'tax': 74168,\n",
       " 'true': 74025,\n",
       " 'non': 73950,\n",
       " 'clinton': 73881,\n",
       " 'washington': 73716,\n",
       " 'act': 73542,\n",
       " 'create': 73084,\n",
       " 'due': 72795,\n",
       " 'points': 72752,\n",
       " 'form': 72701,\n",
       " 'league': 72633,\n",
       " 'air': 72631,\n",
       " 'reported': 72111,\n",
       " 'continue': 71968,\n",
       " 'de': 71835,\n",
       " 'win': 71734,\n",
       " 'bad': 71719,\n",
       " 'building': 71474,\n",
       " 'final': 70688,\n",
       " 'force': 70572,\n",
       " 'students': 70315,\n",
       " 'officials': 70267,\n",
       " 'tell': 70124,\n",
       " 'attack': 70000,\n",
       " 'issues': 69937,\n",
       " 'similar': 69933,\n",
       " 'result': 69896,\n",
       " 'decision': 69811,\n",
       " 'self': 69716,\n",
       " 'whole': 69694,\n",
       " 'companies': 69662,\n",
       " 'wrote': 69527,\n",
       " 'access': 69255,\n",
       " 'matter': 69217,\n",
       " 'services': 68916,\n",
       " 'hours': 68857,\n",
       " 'released': 68857,\n",
       " 'release': 68775,\n",
       " 'car': 68629,\n",
       " 'reason': 68599,\n",
       " 'twitter': 68581,\n",
       " 'running': 68461,\n",
       " 'election': 68428,\n",
       " 'china': 68348,\n",
       " 'street': 68261,\n",
       " 'special': 68214,\n",
       " 'view': 68060,\n",
       " 'short': 67759,\n",
       " 'press': 67706,\n",
       " 'director': 67554,\n",
       " 'woman': 67395,\n",
       " 'role': 67381,\n",
       " 'key': 67321,\n",
       " 'return': 67304,\n",
       " 'statement': 67295,\n",
       " 'front': 67102,\n",
       " 'especially': 67068,\n",
       " 'music': 66926,\n",
       " 'light': 66891,\n",
       " 'simply': 66782,\n",
       " 'began': 66729,\n",
       " 'cost': 66632,\n",
       " 'economic': 66581,\n",
       " 'age': 66575,\n",
       " 'evidence': 66463,\n",
       " 'version': 66114,\n",
       " 'lead': 66031,\n",
       " 'talk': 65924,\n",
       " 'value': 65886,\n",
       " 'term': 65836,\n",
       " 'west': 65829,\n",
       " 'mean': 65396,\n",
       " 'record': 65307,\n",
       " 'hit': 65293,\n",
       " 'red': 65261,\n",
       " 'lost': 65137,\n",
       " 'source': 65105,\n",
       " 'countries': 64993,\n",
       " 'online': 64815,\n",
       " 'turn': 64801,\n",
       " 'code': 64724,\n",
       " 'led': 64723,\n",
       " 'industry': 64674,\n",
       " 'late': 64459,\n",
       " 'earlier': 64347,\n",
       " 'event': 64277,\n",
       " 'groups': 64259,\n",
       " 'march': 64230,\n",
       " 'share': 64200,\n",
       " 'playing': 64176,\n",
       " 'pretty': 64064,\n",
       " 'needed': 63957,\n",
       " 'technology': 63828,\n",
       " 'include': 63726,\n",
       " 'recently': 63579,\n",
       " 'near': 63441,\n",
       " 'shows': 63339,\n",
       " 'article': 63152,\n",
       " 'site': 63146,\n",
       " 'star': 63017,\n",
       " 'needs': 63008,\n",
       " 'type': 62838,\n",
       " 'room': 62777,\n",
       " 'minister': 62512,\n",
       " 'position': 62190,\n",
       " 'strong': 62173,\n",
       " 'saw': 62132,\n",
       " 'test': 61743,\n",
       " 'played': 61715,\n",
       " 'changes': 61595,\n",
       " 'sense': 61582,\n",
       " 'vote': 61232,\n",
       " 'field': 60941,\n",
       " 'google': 60889,\n",
       " 'friends': 60866,\n",
       " 'total': 60758,\n",
       " 'photo': 60476,\n",
       " 'anti': 60351,\n",
       " 'mind': 59918,\n",
       " 'watch': 59781,\n",
       " 'forward': 59755,\n",
       " 'held': 59528,\n",
       " 'build': 59491,\n",
       " 'results': 59482,\n",
       " 'price': 59465,\n",
       " 'higher': 59357,\n",
       " 'fight': 59333,\n",
       " 'global': 59283,\n",
       " 'fire': 59198,\n",
       " 'facebook': 59120,\n",
       " 'weeks': 59099,\n",
       " 'longer': 58993,\n",
       " 'official': 58581,\n",
       " 'words': 58270,\n",
       " 'republican': 58208,\n",
       " 'science': 58139,\n",
       " 'june': 58054,\n",
       " 'common': 58034,\n",
       " 'private': 57878,\n",
       " 'rate': 57866,\n",
       " 'potential': 57781,\n",
       " 'network': 57589,\n",
       " 'soon': 57362,\n",
       " 'god': 57280,\n",
       " 'administration': 57224,\n",
       " 'foreign': 57111,\n",
       " 'inside': 57087,\n",
       " 'nearly': 57073,\n",
       " 'created': 57047,\n",
       " 'page': 56896,\n",
       " 'hope': 56781,\n",
       " 'billion': 56775,\n",
       " 'understand': 56768,\n",
       " 'minutes': 56601,\n",
       " 'design': 56525,\n",
       " 'original': 56448,\n",
       " 'average': 56427,\n",
       " 'financial': 56396,\n",
       " 'middle': 56347,\n",
       " 'july': 56211,\n",
       " 'add': 56146,\n",
       " 'bring': 56085,\n",
       " 'workers': 55905,\n",
       " 'legal': 55840,\n",
       " 'living': 55836,\n",
       " 'child': 55788,\n",
       " 'problems': 55637,\n",
       " 'internet': 55481,\n",
       " 'received': 55429,\n",
       " 'works': 55348,\n",
       " 'yes': 55332,\n",
       " 'easy': 55332,\n",
       " 'interest': 55124,\n",
       " 'increase': 55104,\n",
       " 'member': 55101,\n",
       " 'board': 55055,\n",
       " 'currently': 55007,\n",
       " 'ways': 54974,\n",
       " 'maybe': 54921,\n",
       " 'email': 54919,\n",
       " 'americans': 54825,\n",
       " 'content': 54766,\n",
       " 'questions': 54760,\n",
       " 'model': 54709,\n",
       " 'leave': 54529,\n",
       " 'expected': 54522,\n",
       " 'shot': 54472,\n",
       " 'reports': 54463,\n",
       " 'users': 54451,\n",
       " 'chief': 54353,\n",
       " 'trade': 54086,\n",
       " 'college': 53984,\n",
       " 'april': 53980,\n",
       " 'phone': 53785,\n",
       " 'friday': 53780,\n",
       " 'personal': 53775,\n",
       " 'announced': 53687,\n",
       " 'response': 53555,\n",
       " 'involved': 53499,\n",
       " 'russia': 53421,\n",
       " 'society': 53275,\n",
       " 'risk': 53250,\n",
       " 'allow': 53061,\n",
       " 'follow': 52985,\n",
       " 'medical': 52837,\n",
       " 'fans': 52836,\n",
       " 'finally': 52812,\n",
       " 'step': 52793,\n",
       " 'published': 52774,\n",
       " 'defense': 52702,\n",
       " 'killed': 52493,\n",
       " 'effect': 52442,\n",
       " 'performance': 52227,\n",
       " 'wrong': 52085,\n",
       " 'county': 52061,\n",
       " 'land': 52060,\n",
       " 'club': 52016,\n",
       " 'cases': 52014,\n",
       " 'cut': 51984,\n",
       " 'east': 51851,\n",
       " 'nation': 51764,\n",
       " 'worked': 51677,\n",
       " 'note': 51538,\n",
       " 'lives': 51413,\n",
       " 'monday': 51402,\n",
       " 'justice': 51364,\n",
       " 'terms': 51306,\n",
       " 'amount': 51293,\n",
       " 'oil': 51243,\n",
       " 'ability': 51103,\n",
       " 'race': 51012,\n",
       " 'david': 51002,\n",
       " 'road': 50885,\n",
       " 'committee': 50840,\n",
       " 'takes': 50762,\n",
       " 'goal': 50736,\n",
       " 'sign': 50664,\n",
       " 'difficult': 50497,\n",
       " 'park': 50497,\n",
       " 'tuesday': 50484,\n",
       " 'leader': 50454,\n",
       " 'period': 50388,\n",
       " 'ground': 50354,\n",
       " 'simple': 50192,\n",
       " 'rest': 50174,\n",
       " 'gave': 50126,\n",
       " 'entire': 50108,\n",
       " 'meeting': 50106,\n",
       " 'staff': 50005,\n",
       " 'january': 49989,\n",
       " 'moment': 49899,\n",
       " 'check': 49655,\n",
       " 'happened': 49569,\n",
       " 'september': 49561,\n",
       " 'bank': 49549,\n",
       " 'central': 49540,\n",
       " 'tv': 49463,\n",
       " 'production': 49401,\n",
       " 'size': 49396,\n",
       " 'events': 49359,\n",
       " 'teams': 49358,\n",
       " 'education': 49355,\n",
       " 'turned': 49343,\n",
       " 'sex': 49334,\n",
       " 'forces': 49314,\n",
       " 'character': 49267,\n",
       " 'chance': 49183,\n",
       " 'morning': 49131,\n",
       " 'november': 49129,\n",
       " 'plans': 49078,\n",
       " 'built': 48949,\n",
       " 'goes': 48945,\n",
       " 'union': 48917,\n",
       " 'father': 48839,\n",
       " 'career': 48780,\n",
       " 'european': 48685,\n",
       " 'israel': 48671,\n",
       " 'democratic': 48659,\n",
       " 'images': 48626,\n",
       " 'agency': 48534,\n",
       " 'situation': 48294,\n",
       " 'movement': 48151,\n",
       " 'canada': 48141,\n",
       " 'decided': 48065,\n",
       " 'uk': 48023,\n",
       " 'website': 47975,\n",
       " 'ask': 47962,\n",
       " 'significant': 47896,\n",
       " 'felt': 47875,\n",
       " 'cause': 47853,\n",
       " 'areas': 47837,\n",
       " 'green': 47724,\n",
       " 'word': 47701,\n",
       " 'economy': 47676,\n",
       " 'couple': 47630,\n",
       " 'training': 47589,\n",
       " 'congress': 47510,\n",
       " 'popular': 47501,\n",
       " 'buy': 47458,\n",
       " 'russian': 47446,\n",
       " 'http': 47335,\n",
       " 'growth': 47289,\n",
       " 'numbers': 47274,\n",
       " 'october': 47224,\n",
       " 'language': 47118,\n",
       " 'paul': 47040,\n",
       " 'thursday': 47028,\n",
       " 'previous': 46956,\n",
       " 'focus': 46921,\n",
       " 'british': 46819,\n",
       " 'quickly': 46764,\n",
       " 'wednesday': 46695,\n",
       " 'heard': 46660,\n",
       " 'lower': 46660,\n",
       " 'spent': 46647,\n",
       " 'mother': 46637,\n",
       " 'account': 46335,\n",
       " 'huge': 46307,\n",
       " 'sunday': 46262,\n",
       " 'range': 46253,\n",
       " 'climate': 46253,\n",
       " 'church': 46160,\n",
       " 'included': 45969,\n",
       " 'rules': 45923,\n",
       " 'population': 45889,\n",
       " 'figure': 45866,\n",
       " 'systems': 45828,\n",
       " 'art': 45798,\n",
       " 'knew': 45705,\n",
       " 'wants': 45647,\n",
       " 'tried': 45488,\n",
       " 'california': 45292,\n",
       " 'december': 45267,\n",
       " 'user': 45262,\n",
       " 'europe': 45243,\n",
       " 'hold': 45243,\n",
       " 'looks': 45228,\n",
       " 'san': 45226,\n",
       " 'august': 45098,\n",
       " 'quality': 45041,\n",
       " 'interview': 45020,\n",
       " 'attention': 44930,\n",
       " 'offer': 44902,\n",
       " 'impact': 44843,\n",
       " 'allowed': 44778,\n",
       " 'happen': 44718,\n",
       " 'council': 44699,\n",
       " 'present': 44645,\n",
       " 'jobs': 44628,\n",
       " 'written': 44617,\n",
       " 'features': 44586,\n",
       " 'earth': 44541,\n",
       " 'talking': 44495,\n",
       " 'town': 44477,\n",
       " 'dead': 44398,\n",
       " 'mark': 44355,\n",
       " 'century': 44170,\n",
       " 'brought': 44164,\n",
       " 'investigation': 44149,\n",
       " 'conference': 44111,\n",
       " 'base': 44089,\n",
       " 'address': 44049,\n",
       " 'learn': 43893,\n",
       " 'round': 43884,\n",
       " 'senate': 43764,\n",
       " 'majority': 43755,\n",
       " 'summer': 43699,\n",
       " 'individual': 43645,\n",
       " 'natural': 43512,\n",
       " 'guy': 43510,\n",
       " 'india': 43510,\n",
       " 'worth': 43467,\n",
       " 'specific': 43400,\n",
       " 'leaders': 43375,\n",
       " 'officers': 43290,\n",
       " 'latest': 43231,\n",
       " 'parents': 43213,\n",
       " 'football': 43183,\n",
       " 'drug': 43109,\n",
       " 'hands': 43105,\n",
       " 'fall': 43021,\n",
       " 'app': 42922,\n",
       " 'donald': 42856,\n",
       " 'officer': 42775,\n",
       " 'date': 42765,\n",
       " 'considered': 42745,\n",
       " 'violence': 42729,\n",
       " 'daily': 42706,\n",
       " 'stay': 42608,\n",
       " 'remember': 42564,\n",
       " 'advertisement': 42562,\n",
       " 'starting': 42521,\n",
       " 'file': 42515,\n",
       " 'review': 42502,\n",
       " 'store': 42345,\n",
       " 'success': 42235,\n",
       " 'approach': 42156,\n",
       " 'son': 42116,\n",
       " 'dr': 42086,\n",
       " 'complete': 42054,\n",
       " 'gun': 42047,\n",
       " 'army': 42035,\n",
       " 'message': 41981,\n",
       " 'product': 41877,\n",
       " 'movie': 41840,\n",
       " 'heart': 41789,\n",
       " 'modern': 41780,\n",
       " 'levels': 41770,\n",
       " 'writing': 41752,\n",
       " 'certainly': 41744,\n",
       " 'paid': 41613,\n",
       " 'reading': 41445,\n",
       " 'details': 41437,\n",
       " 'continued': 41426,\n",
       " 'style': 41416,\n",
       " 'standard': 41396,\n",
       " 'pass': 41347,\n",
       " 'stories': 41310,\n",
       " 'wall': 41225,\n",
       " 'executive': 41206,\n",
       " 'completely': 41201,\n",
       " 'income': 41199,\n",
       " 'attacks': 41198,\n",
       " 'saturday': 41194,\n",
       " 'growing': 41159,\n",
       " 'addition': 41155,\n",
       " 'answer': 41141,\n",
       " 'safety': 41138,\n",
       " 'london': 41110,\n",
       " 'claims': 41028,\n",
       " 'leading': 41000,\n",
       " 'intelligence': 40990,\n",
       " 'michael': 40975,\n",
       " 'products': 40946,\n",
       " 'software': 40918,\n",
       " 'opportunity': 40913,\n",
       " 'biggest': 40807,\n",
       " 'nature': 40771,\n",
       " 'sound': 40749,\n",
       " 'exactly': 40705,\n",
       " 'culture': 40669,\n",
       " 'student': 40634,\n",
       " 'required': 40612,\n",
       " 'damage': 40607,\n",
       " 'practice': 40598,\n",
       " 'february': 40528,\n",
       " 'friend': 40485,\n",
       " 'kids': 40431,\n",
       " 'paper': 40404,\n",
       " 'multiple': 40392,\n",
       " 'st': 40323,\n",
       " 'additional': 40248,\n",
       " 'district': 40246,\n",
       " 'stand': 40227,\n",
       " 'ready': 40204,\n",
       " 'king': 40167,\n",
       " 'named': 40114,\n",
       " 'region': 40103,\n",
       " 'function': 40039,\n",
       " 'title': 40027,\n",
       " 'interesting': 40018,\n",
       " 'giving': 39981,\n",
       " 'thinking': 39979,\n",
       " 'drive': 39958,\n",
       " 'apple': 39907,\n",
       " 'prime': 39852,\n",
       " 'james': 39845,\n",
       " 'choice': 39840,\n",
       " 'stage': 39809,\n",
       " 'organization': 39781,\n",
       " 'consider': 39656,\n",
       " 'changed': 39646,\n",
       " 'write': 39638,\n",
       " 'coach': 39583,\n",
       " 'deep': 39559,\n",
       " 'republicans': 39503,\n",
       " 'break': 39397,\n",
       " 'expect': 39379,\n",
       " 'pick': 39369,\n",
       " 'effort': 39227,\n",
       " 'effects': 39214,\n",
       " 'reality': 39174,\n",
       " 'looked': 39062,\n",
       " 'match': 39061,\n",
       " 'blue': 39039,\n",
       " 'speed': 39036,\n",
       " 'laws': 39017,\n",
       " 'credit': 38980,\n",
       " 'cover': 38885,\n",
       " 'web': 38847,\n",
       " 'eventually': 38830,\n",
       " 'capital': 38829,\n",
       " 'costs': 38823,\n",
       " 'fun': 38768,\n",
       " 'meet': 38748,\n",
       " 'parts': 38737,\n",
       " 'increased': 38720,\n",
       " 'relationship': 38718,\n",
       " 'sexual': 38691,\n",
       " 'search': 38444,\n",
       " 'computer': 38413,\n",
       " 'ball': 38386,\n",
       " 'card': 38380,\n",
       " 'designed': 38339,\n",
       " 'showed': 38327,\n",
       " 'sort': 38305,\n",
       " 'happy': 38285,\n",
       " 'chinese': 38260,\n",
       " 'wife': 38241,\n",
       " 'western': 38204,\n",
       " 'books': 38135,\n",
       " 'analysis': 38112,\n",
       " 'calls': 38093,\n",
       " 'remains': 38089,\n",
       " 'hour': 38011,\n",
       " 'voters': 37898,\n",
       " 'died': 37850,\n",
       " 'thousands': 37828,\n",
       " 'texas': 37808,\n",
       " 'alone': 37768,\n",
       " 'civil': 37754,\n",
       " 'guys': 37730,\n",
       " 'amp': 37595,\n",
       " 'nuclear': 37501,\n",
       " 'lack': 37474,\n",
       " 'includes': 37405,\n",
       " 'fast': 37369,\n",
       " 'comments': 37353,\n",
       " 'efforts': 37317,\n",
       " 'judge': 37243,\n",
       " 'claim': 37242,\n",
       " 'la': 37234,\n",
       " 'moving': 37230,\n",
       " 'senior': 37219,\n",
       " 'met': 37190,\n",
       " 'speech': 37155,\n",
       " 'hear': 37121,\n",
       " 'limited': 37113,\n",
       " 'budget': 37108,\n",
       " 'loss': 37022,\n",
       " 'scene': 37001,\n",
       " 'studies': 36988,\n",
       " 'ones': 36966,\n",
       " 'politics': 36951,\n",
       " 'directly': 36933,\n",
       " 'presidential': 36861,\n",
       " 'environment': 36856,\n",
       " 'property': 36841,\n",
       " 'pressure': 36840,\n",
       " 'target': 36793,\n",
       " 'religious': 36743,\n",
       " 'update': 36730,\n",
       " 'battle': 36716,\n",
       " 'stuff': 36697,\n",
       " 'secretary': 36662,\n",
       " 'gas': 36657,\n",
       " 'track': 36561,\n",
       " 'letter': 36540,\n",
       " 'super': 36475,\n",
       " 'crime': 36467,\n",
       " 'freedom': 36449,\n",
       " 'piece': 36379,\n",
       " 'ahead': 36226,\n",
       " 'schools': 36148,\n",
       " 'dark': 36118,\n",
       " 'beginning': 36019,\n",
       " 'manager': 36007,\n",
       " 'democrats': 35996,\n",
       " 'debate': 35956,\n",
       " 'remain': 35847,\n",
       " 'table': 35714,\n",
       " 'device': 35671,\n",
       " 'compared': 35634,\n",
       " 'subject': 35617,\n",
       " 'moved': 35530,\n",
       " 'thanks': 35446,\n",
       " 'largest': 35435,\n",
       " 'poor': 35399,\n",
       " 'passed': 35372,\n",
       " 'comment': 35370,\n",
       " 'save': 35325,\n",
       " 'brain': 35320,\n",
       " 'contact': 35298,\n",
       " 'box': 35267,\n",
       " 'conditions': 35168,\n",
       " 'visit': 35152,\n",
       " 'receive': 35150,\n",
       " 'text': 35135,\n",
       " 'blood': 35063,\n",
       " 'conservative': 35039,\n",
       " 'reasons': 34947,\n",
       " 'associated': 34940,\n",
       " 'born': 34857,\n",
       " 'characters': 34814,\n",
       " 'clearly': 34780,\n",
       " 'reach': 34746,\n",
       " 'failed': 34708,\n",
       " 'challenge': 34585,\n",
       " 'french': 34493,\n",
       " 'safe': 34470,\n",
       " 'helped': 34460,\n",
       " 'spending': 34458,\n",
       " 'eyes': 34458,\n",
       " 'independent': 34432,\n",
       " 'crisis': 34431,\n",
       " 'candidate': 34427,\n",
       " 'screen': 34375,\n",
       " 'english': 34354,\n",
       " 'wide': 34297,\n",
       " 'families': 34262,\n",
       " 'core': 34243,\n",
       " 'voice': 34242,\n",
       " 'powerful': 34224,\n",
       " 'hospital': 34204,\n",
       " 'mass': 34177,\n",
       " 'difference': 34117,\n",
       " 'sports': 34116,\n",
       " 'treatment': 34092,\n",
       " 'researchers': 34073,\n",
       " 'digital': 34066,\n",
       " 'cup': 34020,\n",
       " 'windows': 34014,\n",
       " 'bitcoin': 33947,\n",
       " 'physical': 33857,\n",
       " 'resources': 33827,\n",
       " 'actions': 33814,\n",
       " 'greater': 33792,\n",
       " 'shown': 33790,\n",
       " 'rule': 33778,\n",
       " 'section': 33729,\n",
       " 'previously': 33720,\n",
       " 'signed': 33694,\n",
       " 'feature': 33661,\n",
       " 'protect': 33658,\n",
       " 'charge': 33581,\n",
       " 'management': 33560,\n",
       " 'threat': 33524,\n",
       " 'author': 33437,\n",
       " 'george': 33432,\n",
       " 'syria': 33431,\n",
       " 'weapons': 33381,\n",
       " 'german': 33327,\n",
       " 'status': 33312,\n",
       " 'fear': 33262,\n",
       " 'ice': 33241,\n",
       " 'forced': 33229,\n",
       " 'records': 33207,\n",
       " 'germany': 33203,\n",
       " 'brown': 33201,\n",
       " 'sources': 33181,\n",
       " 'australia': 33140,\n",
       " 'fighting': 32947,\n",
       " 'allows': 32934,\n",
       " 'block': 32806,\n",
       " 'rise': 32799,\n",
       " 'followed': 32781,\n",
       " 'projects': 32771,\n",
       " 'developed': 32752,\n",
       " 'benefits': 32686,\n",
       " 'sea': 32639,\n",
       " 'girl': 32631,\n",
       " 'basic': 32605,\n",
       " 'ideas': 32588,\n",
       " 'positive': 32565,\n",
       " 'easily': 32551,\n",
       " 'customers': 32527,\n",
       " 'picture': 32512,\n",
       " 'primary': 32492,\n",
       " 'uses': 32440,\n",
       " 'attorney': 32438,\n",
       " 'attempt': 32406,\n",
       " 'option': 32384,\n",
       " 'begin': 32373,\n",
       " 'fully': 32346,\n",
       " 'gives': 32320,\n",
       " 'association': 32282,\n",
       " 'posted': 32281,\n",
       " 'female': 32248,\n",
       " 'eu': 32228,\n",
       " 'mobile': 32216,\n",
       " 'decades': 32208,\n",
       " 'individuals': 32161,\n",
       " 'cities': 32125,\n",
       " 'criminal': 32108,\n",
       " 'plus': 32085,\n",
       " 'speaking': 32058,\n",
       " 'contract': 32012,\n",
       " 'photos': 32011,\n",
       " 'authorities': 31981,\n",
       " 'agreement': 31963,\n",
       " 'station': 31931,\n",
       " 'shooting': 31896,\n",
       " 'fine': 31880,\n",
       " 'highly': 31866,\n",
       " 'gold': 31749,\n",
       " 'perfect': 31749,\n",
       " 'link': 31743,\n",
       " 'truth': 31739,\n",
       " 'larger': 31717,\n",
       " 'reached': 31717,\n",
       " 'calling': 31709,\n",
       " 'adding': 31708,\n",
       " 'lines': 31673,\n",
       " 'professor': 31665,\n",
       " 'door': 31649,\n",
       " 'goals': 31622,\n",
       " 'smith': 31620,\n",
       " 'rates': 31545,\n",
       " 'options': 31530,\n",
       " 'avoid': 31419,\n",
       " 'noted': 31393,\n",
       " 'hillary': 31378,\n",
       " 'island': 31376,\n",
       " 'winning': 31301,\n",
       " 'continues': 31300,\n",
       " 'sales': 31240,\n",
       " 'material': 31185,\n",
       " 'nice': 31159,\n",
       " 'programs': 31141,\n",
       " 'commission': 31113,\n",
       " 'citizens': 31104,\n",
       " 'successful': 31083,\n",
       " 'pre': 31025,\n",
       " 'scale': 31022,\n",
       " 'activity': 31009,\n",
       " 'band': 30988,\n",
       " 'regular': 30961,\n",
       " 'color': 30925,\n",
       " 'creating': 30922,\n",
       " 'secret': 30873,\n",
       " 'direct': 30860,\n",
       " 'traditional': 30854,\n",
       " 'feet': 30843,\n",
       " 'iran': 30838,\n",
       " 'cent': 30826,\n",
       " 'unique': 30814,\n",
       " 'knowledge': 30800,\n",
       " 'places': 30770,\n",
       " ...}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 4
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-11-23T08:13:07.601554Z",
     "start_time": "2024-11-23T08:13:06.081881Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# Find words with embeddings most similar to the target word embedding\n",
    "target_word = 'bird'\n",
    "hash_length = 40\n",
    "top_N_closest = 20\n",
    "\n",
    "#model = utils.load_model('trained_models/.pt')\n",
    "import utils\n",
    "\n",
    "utils.calc_print_sim_words(\n",
    "    vocab=vocab,\n",
    "    word_counts=word_counter,\n",
    "    model=model,\n",
    "    word=target_word,\n",
    "    hash_len=hash_length,\n",
    "    top_N=top_N_closest,\n",
    "    create_target_vector=True\n",
    ")"
   ],
   "id": "24b1df8a53ae8ec5",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Word            Similarity Frequency \n",
      "-----------------------------------\n",
      "bird                1.000       9728\n",
      "adversity           0.885        993\n",
      "threeday            0.885          1\n",
      "citizens            0.880      31104\n",
      "here’s              0.880          0\n",
      "shop                0.880      17188\n",
      "jake                0.880       5190\n",
      "rewards             0.880       6772\n",
      "est                 0.880       4538\n",
      "urls                0.880       1261\n",
      "group’s             0.875          0\n",
      "responsibility      0.875      19924\n",
      "tournament          0.875      17125\n",
      "drawing             0.875      11858\n",
      "admitting           0.875       2856\n",
      "gaining             0.875       5233\n",
      "out”                0.875          0\n",
      "iso                 0.875       2720\n",
      "sizeable            0.875        819\n",
      "biblical            0.875       3602\n"
     ]
    }
   ],
   "execution_count": 8
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "",
   "id": "bb069bd342bbb7ec"
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}