w2v_ft_train.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Use extracted wiki + bank texts to train embeddings (ft)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gensim\n",
    "import re\n",
    "import nltk\n",
    "import multiprocessing\n",
    "path2 = '/opt/apps/cda/ez/sb_sandboxpuk/sandboxpuk_scripte/wiki/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import logging\n",
    "logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#list(text_preprocessed)[:3]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loading wiki text corpus\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 64 µs, sys: 138 µs, total: 202 µs\n",
      "Wall time: 120 µs\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "wiki_txt = open(path2 + 'wiki_bank_first.txt', 'r')\n",
    "\n",
    "class txt_wiki_all(object):\n",
    "    def __iter__(self):\n",
    "        for sen in wiki_txt:\n",
    "            \n",
    "            sen = re.sub('[^A-Za-zÄÖÜßäöüß]+', ' ',sen)\n",
    "            yield [a.lower() for a in sen.split()]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train plain wiki w2v"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "from gensim.models.word2vec import Word2Vec\n",
    "name = 'wiki_w2v_300.model'\n",
    "\n",
    "params = {'size': 300, 'window': 10, 'min_count': 50, 'iter':30,'max_vocab_size':500000,\n",
    "          'workers': max(1, 10 #multiprocessing.cpu_count() - 1\n",
    "                        ), 'sample': 1e-3,}\n",
    "word2vec = Word2Vec(sentences=list(text_preprocessed), **params)\n",
    "word2vec.save(path2 + name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_vectors = word2vec.wv\n",
    "\n",
    "from gensim.test.utils import get_tmpfile\n",
    "\n",
    "fname = get_tmpfile(\"vectors.kv\")\n",
    "print(fname)\n",
    "word_vectors.save(\"vectors.kv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "word2vec.wv.most_similar('karte')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "word2vec.most_similar(positive=['frau', 'könig'], negative=['mann'], topn=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "path='/opt/apps/cda/ez/sb_sandboxpuk/sandboxpuk_scripte/wiki/'\n",
    "model = 'wiki_w2v_300.model'\n",
    "from gensim.models.word2vec import Word2Vec\n",
    "word2vec = Word2Vec.load(path+model)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "kuko_text = []\n",
    "path = '/var/iophome/cb2lehk/feedbacks_v02/'\n",
    "\n",
    "with open(path+'data/KuKo2.txt') as fin:\n",
    "    for line in fin:\n",
    "        #print(line)\n",
    "        line = re.sub('[^A-Za-zÄÖÜßäöü]+', ' ',line)\n",
    "        line = [word.lower() for word in line.split()]\n",
    "        kuko_text.append(line)\n",
    "        \n",
    "nps_text = []\n",
    "with open(path+'data/NPS_LERNDATEN_FEEDBACK2KATEGORIE.txt') as fin:\n",
    "    for line in fin:\n",
    "        #print(line)\n",
    "        line = re.sub('[^A-Za-zÄÖÜßäöü]+', ' ',line.split('/t')[0])\n",
    "        line = [word.lower() for word in line.split()]\n",
    "        nps_text.append(line)\n",
    "#nps_text[1:2]\n",
    "data = spark.sql(\"select csp_feedbackaboutclientpotential from \\\n",
    "                 az_01_01_38_dynamics.csp_salespotential_v001 \\\n",
    "                 where length(csp_feedbackaboutclientpotential) > 2\").toPandas()\n",
    "\n",
    "data = data['csp_feedbackaboutclientpotential'].tolist()\n",
    "\n",
    "feedback_text = []\n",
    "\n",
    "for line in data:\n",
    "        #print(line)\n",
    "    line = re.sub('[^A-Za-zÄÖÜßäöü]+', ' ',line)\n",
    "    line = [word.lower() for word in line.split()]\n",
    "    feedback_text.append(line)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "text_all = []\n",
    "path = '/var/iophome/cb2lehk/feedbacks_v02/'\n",
    "\n",
    "with open('data/kuko01.txt') as fin:\n",
    "    for line in fin:\n",
    "        #print(line)\n",
    "        line = re.sub('[^A-Za-zÄÖÜßäöü]+', ' ',line)\n",
    "        line = [word.lower() for word in line.split()]\n",
    "        text_all.append(line)\n",
    "        \n",
    "with open('data/kuko02.txt') as fin:\n",
    "    for line in fin:\n",
    "        #print(line)\n",
    "        line = re.sub('[^A-Za-zÄÖÜßäöü]+', ' ',line)\n",
    "        line = [word.lower() for word in line.split()]\n",
    "        text_all.append(line)\n",
    "\n",
    "with open('data/kuko03.txt') as fin:\n",
    "    for line in fin:\n",
    "        #print(line)\n",
    "        line = re.sub('[^A-Za-zÄÖÜßäöü]+', ' ',line)\n",
    "        line = [word.lower() for word in line.split()]\n",
    "        text_all.append(line)\n",
    "\n",
    "        \n",
    "with open('data/nps_text082019.txt') as fin:\n",
    "    for line in fin:\n",
    "        #print(line)\n",
    "        line = re.sub('[^A-Za-zÄÖÜßäöü]+', ' ',line)\n",
    "        line = [word.lower() for word in line.split()]\n",
    "        text_all.append(line)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "text = nps_text[1:] + kuko_text[1:] + feedback_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "word2vec.build_vocab(text_all, update=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "word2vec.train(text_all,len(text_all),epochs=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nps = {}\n",
    "notin = {}\n",
    "with open('data/nps_text082019.txt') as fin:\n",
    "    for line in fin:\n",
    "        #print(line)\n",
    "        line = re.sub('[^A-Za-zÄÖÜßäöü]+', ' ',line)\n",
    "        line = [word.lower() for word in line.split()]\n",
    "        for word in line:\n",
    "            if word not in word2vec.wv.vocab:\n",
    "                if word in notin.keys():\n",
    "                    notin[word] = notin[word] + 1\n",
    "                else:\n",
    "                    notin[word] = 1\n",
    "            else:\n",
    "                if word in nps.keys():\n",
    "                    nps[word] = nps[word] + 1\n",
    "                else:\n",
    "                    nps[word] = 1\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nps['möglichst']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for key, value in sorted(notin.items(), key=lambda item: item[1])[-10:]:\n",
    "    print(\"%s: %s\" % (key, value))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "path2 = '/opt/apps/cda/ez/sb_sandboxpuk/sandboxpuk_scripte/wiki/'\n",
    "word2vec.save(path2 + model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "keep2 = ['zu', 'um', 'ab', 'im','da', 'er', 'an', 'in', 'dr', 'so','es','am','ob','cc']\n",
    "vocab2 = [word for word in word2vec.wv.vocab if len(word) == 2 and  word not in keep2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab_to_emb = {}\n",
    "for word in word2vec.wv.vocab:\n",
    "    if word not in vocab2:\n",
    "        vocab_to_emb[word] = word2vec.wv[word]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "path = '/opt/apps/cda/ez/sb_sandboxpuk/sandboxpuk_scripte/textmining_03/data/books/'\n",
    "\n",
    "with open(path+'vocab_to_emb_w2v_300.p', 'wb') as handle:\n",
    "    pickle.dump(vocab_to_emb, handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "    \n",
    "with open(path+'nps_vocab.p', 'wb') as handle:\n",
    "    pickle.dump(nps, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import embedding\n",
    "embedding.model_to_books(model_embed)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# FT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:31:55,491 : INFO : collecting all words and their counts\n",
      "2019-12-04 18:31:55,492 : INFO : PROGRESS: at sentence #0, processed 0 words, keeping 0 word types\n",
      "2019-12-04 18:31:55,636 : INFO : PROGRESS: at sentence #10000, processed 105525 words, keeping 9699 word types\n",
      "2019-12-04 18:31:55,779 : INFO : PROGRESS: at sentence #20000, processed 210250 words, keeping 14551 word types\n",
      "2019-12-04 18:31:55,916 : INFO : PROGRESS: at sentence #30000, processed 314166 words, keeping 17960 word types\n",
      "2019-12-04 18:31:56,055 : INFO : PROGRESS: at sentence #40000, processed 420151 words, keeping 17960 word types\n",
      "2019-12-04 18:31:56,195 : INFO : PROGRESS: at sentence #50000, processed 524837 words, keeping 17960 word types\n",
      "2019-12-04 18:31:56,334 : INFO : PROGRESS: at sentence #60000, processed 628902 words, keeping 17960 word types\n",
      "2019-12-04 18:31:56,479 : INFO : PROGRESS: at sentence #70000, processed 734878 words, keeping 17960 word types\n",
      "2019-12-04 18:31:56,620 : INFO : PROGRESS: at sentence #80000, processed 839353 words, keeping 17960 word types\n",
      "2019-12-04 18:31:56,758 : INFO : PROGRESS: at sentence #90000, processed 943905 words, keeping 17960 word types\n",
      "2019-12-04 18:31:56,898 : INFO : PROGRESS: at sentence #100000, processed 1048922 words, keeping 17960 word types\n",
      "2019-12-04 18:31:57,037 : INFO : PROGRESS: at sentence #110000, processed 1153797 words, keeping 17960 word types\n",
      "2019-12-04 18:31:57,176 : INFO : PROGRESS: at sentence #120000, processed 1258873 words, keeping 17960 word types\n",
      "2019-12-04 18:31:57,317 : INFO : PROGRESS: at sentence #130000, processed 1363684 words, keeping 17960 word types\n",
      "2019-12-04 18:31:57,456 : INFO : PROGRESS: at sentence #140000, processed 1468403 words, keeping 17960 word types\n",
      "2019-12-04 18:31:57,598 : INFO : PROGRESS: at sentence #150000, processed 1573656 words, keeping 17960 word types\n",
      "2019-12-04 18:31:57,737 : INFO : PROGRESS: at sentence #160000, processed 1678218 words, keeping 17960 word types\n",
      "2019-12-04 18:31:57,879 : INFO : PROGRESS: at sentence #170000, processed 1782828 words, keeping 17960 word types\n",
      "2019-12-04 18:31:58,019 : INFO : PROGRESS: at sentence #180000, processed 1888031 words, keeping 17960 word types\n",
      "2019-12-04 18:31:58,157 : INFO : PROGRESS: at sentence #190000, processed 1992924 words, keeping 17960 word types\n",
      "2019-12-04 18:31:58,294 : INFO : PROGRESS: at sentence #200000, processed 2097363 words, keeping 17960 word types\n",
      "2019-12-04 18:31:58,438 : INFO : PROGRESS: at sentence #210000, processed 2202788 words, keeping 17960 word types\n",
      "2019-12-04 18:31:58,578 : INFO : PROGRESS: at sentence #220000, processed 2307597 words, keeping 17960 word types\n",
      "2019-12-04 18:31:58,719 : INFO : PROGRESS: at sentence #230000, processed 2411795 words, keeping 17960 word types\n",
      "2019-12-04 18:31:58,859 : INFO : PROGRESS: at sentence #240000, processed 2517447 words, keeping 17960 word types\n",
      "2019-12-04 18:31:58,999 : INFO : PROGRESS: at sentence #250000, processed 2622130 words, keeping 17960 word types\n",
      "2019-12-04 18:31:59,137 : INFO : PROGRESS: at sentence #260000, processed 2726049 words, keeping 17960 word types\n",
      "2019-12-04 18:31:59,278 : INFO : PROGRESS: at sentence #270000, processed 2832194 words, keeping 17960 word types\n",
      "2019-12-04 18:31:59,422 : INFO : PROGRESS: at sentence #280000, processed 2936803 words, keeping 17960 word types\n",
      "2019-12-04 18:31:59,563 : INFO : PROGRESS: at sentence #290000, processed 3040951 words, keeping 17960 word types\n",
      "2019-12-04 18:31:59,738 : INFO : PROGRESS: at sentence #300000, processed 3146638 words, keeping 17960 word types\n",
      "2019-12-04 18:31:59,894 : INFO : PROGRESS: at sentence #310000, processed 3251339 words, keeping 17960 word types\n",
      "2019-12-04 18:32:00,031 : INFO : PROGRESS: at sentence #320000, processed 3355822 words, keeping 17960 word types\n",
      "2019-12-04 18:32:00,170 : INFO : PROGRESS: at sentence #330000, processed 3461048 words, keeping 17960 word types\n",
      "2019-12-04 18:32:00,309 : INFO : PROGRESS: at sentence #340000, processed 3565431 words, keeping 17960 word types\n",
      "2019-12-04 18:32:00,452 : INFO : PROGRESS: at sentence #350000, processed 3670790 words, keeping 17960 word types\n",
      "2019-12-04 18:32:00,593 : INFO : PROGRESS: at sentence #360000, processed 3775467 words, keeping 17960 word types\n",
      "2019-12-04 18:32:00,735 : INFO : PROGRESS: at sentence #370000, processed 3880199 words, keeping 17960 word types\n",
      "2019-12-04 18:32:00,877 : INFO : PROGRESS: at sentence #380000, processed 3985701 words, keeping 17960 word types\n",
      "2019-12-04 18:32:01,016 : INFO : PROGRESS: at sentence #390000, processed 4090123 words, keeping 17960 word types\n",
      "2019-12-04 18:32:01,156 : INFO : PROGRESS: at sentence #400000, processed 4195002 words, keeping 17960 word types\n",
      "2019-12-04 18:32:01,295 : INFO : PROGRESS: at sentence #410000, processed 4299914 words, keeping 17960 word types\n",
      "2019-12-04 18:32:01,434 : INFO : PROGRESS: at sentence #420000, processed 4404850 words, keeping 17960 word types\n",
      "2019-12-04 18:32:01,576 : INFO : PROGRESS: at sentence #430000, processed 4509215 words, keeping 17960 word types\n",
      "2019-12-04 18:32:01,717 : INFO : PROGRESS: at sentence #440000, processed 4614794 words, keeping 17960 word types\n",
      "2019-12-04 18:32:01,857 : INFO : PROGRESS: at sentence #450000, processed 4719493 words, keeping 17960 word types\n",
      "2019-12-04 18:32:01,997 : INFO : PROGRESS: at sentence #460000, processed 4823664 words, keeping 17960 word types\n",
      "2019-12-04 18:32:02,138 : INFO : PROGRESS: at sentence #470000, processed 4929152 words, keeping 17960 word types\n",
      "2019-12-04 18:32:02,278 : INFO : PROGRESS: at sentence #480000, processed 5034045 words, keeping 17960 word types\n",
      "2019-12-04 18:32:02,418 : INFO : PROGRESS: at sentence #490000, processed 5138171 words, keeping 17960 word types\n",
      "2019-12-04 18:32:02,559 : INFO : PROGRESS: at sentence #500000, processed 5244099 words, keeping 17960 word types\n",
      "2019-12-04 18:32:02,700 : INFO : PROGRESS: at sentence #510000, processed 5348406 words, keeping 17960 word types\n",
      "2019-12-04 18:32:02,841 : INFO : PROGRESS: at sentence #520000, processed 5453261 words, keeping 17960 word types\n",
      "2019-12-04 18:32:02,982 : INFO : PROGRESS: at sentence #530000, processed 5558428 words, keeping 17960 word types\n",
      "2019-12-04 18:32:03,122 : INFO : PROGRESS: at sentence #540000, processed 5663198 words, keeping 17960 word types\n",
      "2019-12-04 18:32:03,262 : INFO : PROGRESS: at sentence #550000, processed 5767933 words, keeping 17960 word types\n",
      "2019-12-04 18:32:03,402 : INFO : PROGRESS: at sentence #560000, processed 5873070 words, keeping 17960 word types\n",
      "2019-12-04 18:32:03,541 : INFO : PROGRESS: at sentence #570000, processed 5977566 words, keeping 17960 word types\n",
      "2019-12-04 18:32:03,681 : INFO : PROGRESS: at sentence #580000, processed 6082840 words, keeping 17960 word types\n",
      "2019-12-04 18:32:03,823 : INFO : PROGRESS: at sentence #590000, processed 6187378 words, keeping 17960 word types\n",
      "2019-12-04 18:32:03,964 : INFO : PROGRESS: at sentence #600000, processed 6292045 words, keeping 17960 word types\n",
      "2019-12-04 18:32:04,107 : INFO : PROGRESS: at sentence #610000, processed 6397415 words, keeping 17960 word types\n",
      "2019-12-04 18:32:04,248 : INFO : PROGRESS: at sentence #620000, processed 6502022 words, keeping 17960 word types\n",
      "2019-12-04 18:32:04,392 : INFO : PROGRESS: at sentence #630000, processed 6606583 words, keeping 17960 word types\n",
      "2019-12-04 18:32:04,533 : INFO : PROGRESS: at sentence #640000, processed 6711982 words, keeping 17960 word types\n",
      "2019-12-04 18:32:04,672 : INFO : PROGRESS: at sentence #650000, processed 6816790 words, keeping 17960 word types\n",
      "2019-12-04 18:32:04,812 : INFO : PROGRESS: at sentence #660000, processed 6921162 words, keeping 17960 word types\n",
      "2019-12-04 18:32:04,954 : INFO : PROGRESS: at sentence #670000, processed 7026721 words, keeping 17960 word types\n",
      "2019-12-04 18:32:05,093 : INFO : PROGRESS: at sentence #680000, processed 7131406 words, keeping 17960 word types\n",
      "2019-12-04 18:32:05,233 : INFO : PROGRESS: at sentence #690000, processed 7235307 words, keeping 17960 word types\n",
      "2019-12-04 18:32:05,374 : INFO : PROGRESS: at sentence #700000, processed 7341362 words, keeping 17960 word types\n",
      "2019-12-04 18:32:05,514 : INFO : PROGRESS: at sentence #710000, processed 7445984 words, keeping 17960 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:32:05,652 : INFO : PROGRESS: at sentence #720000, processed 7550108 words, keeping 17960 word types\n",
      "2019-12-04 18:32:05,791 : INFO : PROGRESS: at sentence #730000, processed 7656055 words, keeping 17960 word types\n",
      "2019-12-04 18:32:05,930 : INFO : PROGRESS: at sentence #740000, processed 7760535 words, keeping 17960 word types\n",
      "2019-12-04 18:32:06,069 : INFO : PROGRESS: at sentence #750000, processed 7865137 words, keeping 17960 word types\n",
      "2019-12-04 18:32:06,208 : INFO : PROGRESS: at sentence #760000, processed 7970139 words, keeping 17960 word types\n",
      "2019-12-04 18:32:06,351 : INFO : PROGRESS: at sentence #770000, processed 8074970 words, keeping 17960 word types\n",
      "2019-12-04 18:32:06,491 : INFO : PROGRESS: at sentence #780000, processed 8180048 words, keeping 17960 word types\n",
      "2019-12-04 18:32:06,631 : INFO : PROGRESS: at sentence #790000, processed 8284889 words, keeping 17960 word types\n",
      "2019-12-04 18:32:06,771 : INFO : PROGRESS: at sentence #800000, processed 8389571 words, keeping 17960 word types\n",
      "2019-12-04 18:32:06,912 : INFO : PROGRESS: at sentence #810000, processed 8494882 words, keeping 17960 word types\n",
      "2019-12-04 18:32:07,051 : INFO : PROGRESS: at sentence #820000, processed 8599413 words, keeping 17960 word types\n",
      "2019-12-04 18:32:07,191 : INFO : PROGRESS: at sentence #830000, processed 8704057 words, keeping 17960 word types\n",
      "2019-12-04 18:32:07,329 : INFO : PROGRESS: at sentence #840000, processed 8809227 words, keeping 17960 word types\n",
      "2019-12-04 18:32:07,468 : INFO : PROGRESS: at sentence #850000, processed 8914147 words, keeping 17960 word types\n",
      "2019-12-04 18:32:07,609 : INFO : PROGRESS: at sentence #860000, processed 9018569 words, keeping 17960 word types\n",
      "2019-12-04 18:32:07,753 : INFO : PROGRESS: at sentence #870000, processed 9124014 words, keeping 17960 word types\n",
      "2019-12-04 18:32:07,894 : INFO : PROGRESS: at sentence #880000, processed 9228821 words, keeping 17960 word types\n",
      "2019-12-04 18:32:08,034 : INFO : PROGRESS: at sentence #890000, processed 9333025 words, keeping 17960 word types\n",
      "2019-12-04 18:32:08,175 : INFO : PROGRESS: at sentence #900000, processed 9438588 words, keeping 17960 word types\n",
      "2019-12-04 18:32:08,315 : INFO : PROGRESS: at sentence #910000, processed 9543330 words, keeping 17960 word types\n",
      "2019-12-04 18:32:08,450 : INFO : PROGRESS: at sentence #920000, processed 9647213 words, keeping 17960 word types\n",
      "2019-12-04 18:32:08,585 : INFO : PROGRESS: at sentence #930000, processed 9753300 words, keeping 17960 word types\n",
      "2019-12-04 18:32:08,721 : INFO : PROGRESS: at sentence #940000, processed 9857914 words, keeping 17960 word types\n",
      "2019-12-04 18:32:08,860 : INFO : PROGRESS: at sentence #950000, processed 9962055 words, keeping 17960 word types\n",
      "2019-12-04 18:32:08,997 : INFO : PROGRESS: at sentence #960000, processed 10067803 words, keeping 17960 word types\n",
      "2019-12-04 18:32:09,138 : INFO : PROGRESS: at sentence #970000, processed 10172497 words, keeping 17960 word types\n",
      "2019-12-04 18:32:09,279 : INFO : PROGRESS: at sentence #980000, processed 10276959 words, keeping 17960 word types\n",
      "2019-12-04 18:32:09,423 : INFO : PROGRESS: at sentence #990000, processed 10382233 words, keeping 17960 word types\n",
      "2019-12-04 18:32:09,569 : INFO : PROGRESS: at sentence #1000000, processed 10486655 words, keeping 17960 word types\n",
      "2019-12-04 18:32:09,709 : INFO : PROGRESS: at sentence #1010000, processed 10591957 words, keeping 17960 word types\n",
      "2019-12-04 18:32:09,847 : INFO : PROGRESS: at sentence #1020000, processed 10696674 words, keeping 17960 word types\n",
      "2019-12-04 18:32:09,986 : INFO : PROGRESS: at sentence #1030000, processed 10801373 words, keeping 17960 word types\n",
      "2019-12-04 18:32:10,127 : INFO : PROGRESS: at sentence #1040000, processed 10906857 words, keeping 17960 word types\n",
      "2019-12-04 18:32:10,265 : INFO : PROGRESS: at sentence #1050000, processed 11011291 words, keeping 17960 word types\n",
      "2019-12-04 18:32:10,405 : INFO : PROGRESS: at sentence #1060000, processed 11116135 words, keeping 17960 word types\n",
      "2019-12-04 18:32:10,544 : INFO : PROGRESS: at sentence #1070000, processed 11221131 words, keeping 17960 word types\n",
      "2019-12-04 18:32:10,684 : INFO : PROGRESS: at sentence #1080000, processed 11326048 words, keeping 17960 word types\n",
      "2019-12-04 18:32:10,853 : INFO : PROGRESS: at sentence #1090000, processed 11430415 words, keeping 17960 word types\n",
      "2019-12-04 18:32:10,993 : INFO : PROGRESS: at sentence #1100000, processed 11535940 words, keeping 17960 word types\n",
      "2019-12-04 18:32:11,132 : INFO : PROGRESS: at sentence #1110000, processed 11640673 words, keeping 17960 word types\n",
      "2019-12-04 18:32:11,273 : INFO : PROGRESS: at sentence #1120000, processed 11744865 words, keeping 17960 word types\n",
      "2019-12-04 18:32:11,416 : INFO : PROGRESS: at sentence #1130000, processed 11850380 words, keeping 17960 word types\n",
      "2019-12-04 18:32:11,561 : INFO : PROGRESS: at sentence #1140000, processed 11955305 words, keeping 17960 word types\n",
      "2019-12-04 18:32:11,703 : INFO : PROGRESS: at sentence #1150000, processed 12059319 words, keeping 17960 word types\n",
      "2019-12-04 18:32:11,844 : INFO : PROGRESS: at sentence #1160000, processed 12165302 words, keeping 17960 word types\n",
      "2019-12-04 18:32:11,986 : INFO : PROGRESS: at sentence #1170000, processed 12269566 words, keeping 17960 word types\n",
      "2019-12-04 18:32:12,127 : INFO : PROGRESS: at sentence #1180000, processed 12374424 words, keeping 17960 word types\n",
      "2019-12-04 18:32:12,268 : INFO : PROGRESS: at sentence #1190000, processed 12479624 words, keeping 17960 word types\n",
      "2019-12-04 18:32:12,410 : INFO : PROGRESS: at sentence #1200000, processed 12584368 words, keeping 17960 word types\n",
      "2019-12-04 18:32:12,553 : INFO : PROGRESS: at sentence #1210000, processed 12689017 words, keeping 17960 word types\n",
      "2019-12-04 18:32:12,735 : INFO : PROGRESS: at sentence #1220000, processed 12794143 words, keeping 17960 word types\n",
      "2019-12-04 18:32:12,881 : INFO : PROGRESS: at sentence #1230000, processed 12898767 words, keeping 17960 word types\n",
      "2019-12-04 18:32:13,025 : INFO : PROGRESS: at sentence #1240000, processed 13004041 words, keeping 17960 word types\n",
      "2019-12-04 18:32:13,165 : INFO : PROGRESS: at sentence #1250000, processed 13108535 words, keeping 17960 word types\n",
      "2019-12-04 18:32:13,306 : INFO : PROGRESS: at sentence #1260000, processed 13213230 words, keeping 17960 word types\n",
      "2019-12-04 18:32:13,448 : INFO : PROGRESS: at sentence #1270000, processed 13318593 words, keeping 17960 word types\n",
      "2019-12-04 18:32:13,586 : INFO : PROGRESS: at sentence #1280000, processed 13423173 words, keeping 17960 word types\n",
      "2019-12-04 18:32:13,732 : INFO : PROGRESS: at sentence #1290000, processed 13527788 words, keeping 17960 word types\n",
      "2019-12-04 18:32:13,874 : INFO : PROGRESS: at sentence #1300000, processed 13633120 words, keeping 17960 word types\n",
      "2019-12-04 18:32:14,015 : INFO : PROGRESS: at sentence #1310000, processed 13737962 words, keeping 17960 word types\n",
      "2019-12-04 18:32:14,156 : INFO : PROGRESS: at sentence #1320000, processed 13842331 words, keeping 17960 word types\n",
      "2019-12-04 18:32:14,298 : INFO : PROGRESS: at sentence #1330000, processed 13947938 words, keeping 17960 word types\n",
      "2019-12-04 18:32:14,437 : INFO : PROGRESS: at sentence #1340000, processed 14052617 words, keeping 17960 word types\n",
      "2019-12-04 18:32:14,577 : INFO : PROGRESS: at sentence #1350000, processed 14156541 words, keeping 17960 word types\n",
      "2019-12-04 18:32:14,718 : INFO : PROGRESS: at sentence #1360000, processed 14262558 words, keeping 17960 word types\n",
      "2019-12-04 18:32:14,859 : INFO : PROGRESS: at sentence #1370000, processed 14367155 words, keeping 17960 word types\n",
      "2019-12-04 18:32:15,000 : INFO : PROGRESS: at sentence #1380000, processed 14471318 words, keeping 17960 word types\n",
      "2019-12-04 18:32:15,141 : INFO : PROGRESS: at sentence #1390000, processed 14577273 words, keeping 17960 word types\n",
      "2019-12-04 18:32:15,282 : INFO : PROGRESS: at sentence #1400000, processed 14681726 words, keeping 17960 word types\n",
      "2019-12-04 18:32:15,424 : INFO : PROGRESS: at sentence #1410000, processed 14786302 words, keeping 17960 word types\n",
      "2019-12-04 18:32:15,566 : INFO : PROGRESS: at sentence #1420000, processed 14891403 words, keeping 17960 word types\n",
      "2019-12-04 18:32:15,707 : INFO : PROGRESS: at sentence #1430000, processed 14996145 words, keeping 17960 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:32:15,853 : INFO : PROGRESS: at sentence #1440000, processed 15106801 words, keeping 19936 word types\n",
      "2019-12-04 18:32:16,006 : INFO : PROGRESS: at sentence #1450000, processed 15224879 words, keeping 22916 word types\n",
      "2019-12-04 18:32:16,142 : INFO : PROGRESS: at sentence #1460000, processed 15327721 words, keeping 24094 word types\n",
      "2019-12-04 18:32:16,272 : INFO : PROGRESS: at sentence #1470000, processed 15426969 words, keeping 24932 word types\n",
      "2019-12-04 18:32:16,409 : INFO : PROGRESS: at sentence #1480000, processed 15529298 words, keeping 25665 word types\n",
      "2019-12-04 18:32:16,551 : INFO : PROGRESS: at sentence #1490000, processed 15633776 words, keeping 26447 word types\n",
      "2019-12-04 18:32:16,681 : INFO : PROGRESS: at sentence #1500000, processed 15728489 words, keeping 27331 word types\n",
      "2019-12-04 18:32:16,810 : INFO : PROGRESS: at sentence #1510000, processed 15828857 words, keeping 28508 word types\n",
      "2019-12-04 18:32:16,935 : INFO : PROGRESS: at sentence #1520000, processed 15924477 words, keeping 29356 word types\n",
      "2019-12-04 18:32:17,061 : INFO : PROGRESS: at sentence #1530000, processed 16020899 words, keeping 30250 word types\n",
      "2019-12-04 18:32:17,188 : INFO : PROGRESS: at sentence #1540000, processed 16120460 words, keeping 31073 word types\n",
      "2019-12-04 18:32:17,326 : INFO : PROGRESS: at sentence #1550000, processed 16225124 words, keeping 32099 word types\n",
      "2019-12-04 18:32:17,465 : INFO : PROGRESS: at sentence #1560000, processed 16329964 words, keeping 33123 word types\n",
      "2019-12-04 18:32:17,599 : INFO : PROGRESS: at sentence #1570000, processed 16434696 words, keeping 34094 word types\n",
      "2019-12-04 18:32:17,735 : INFO : PROGRESS: at sentence #1580000, processed 16539432 words, keeping 35155 word types\n",
      "2019-12-04 18:32:17,875 : INFO : PROGRESS: at sentence #1590000, processed 16645719 words, keeping 36132 word types\n",
      "2019-12-04 18:32:18,010 : INFO : PROGRESS: at sentence #1600000, processed 16746838 words, keeping 37121 word types\n",
      "2019-12-04 18:32:18,149 : INFO : PROGRESS: at sentence #1610000, processed 16848611 words, keeping 38147 word types\n",
      "2019-12-04 18:32:18,318 : INFO : PROGRESS: at sentence #1620000, processed 16950875 words, keeping 39032 word types\n",
      "2019-12-04 18:32:18,506 : INFO : PROGRESS: at sentence #1630000, processed 17076154 words, keeping 40655 word types\n",
      "2019-12-04 18:32:18,657 : INFO : PROGRESS: at sentence #1640000, processed 17194372 words, keeping 41609 word types\n",
      "2019-12-04 18:32:18,803 : INFO : PROGRESS: at sentence #1650000, processed 17307476 words, keeping 42230 word types\n",
      "2019-12-04 18:32:18,946 : INFO : PROGRESS: at sentence #1660000, processed 17416862 words, keeping 42814 word types\n",
      "2019-12-04 18:32:19,085 : INFO : PROGRESS: at sentence #1670000, processed 17522815 words, keeping 43454 word types\n",
      "2019-12-04 18:32:19,231 : INFO : PROGRESS: at sentence #1680000, processed 17629648 words, keeping 44226 word types\n",
      "2019-12-04 18:32:19,371 : INFO : PROGRESS: at sentence #1690000, processed 17736298 words, keeping 44893 word types\n",
      "2019-12-04 18:32:19,512 : INFO : PROGRESS: at sentence #1700000, processed 17843617 words, keeping 45744 word types\n",
      "2019-12-04 18:32:19,658 : INFO : PROGRESS: at sentence #1710000, processed 17951059 words, keeping 46633 word types\n",
      "2019-12-04 18:32:19,802 : INFO : PROGRESS: at sentence #1720000, processed 18059720 words, keeping 47282 word types\n",
      "2019-12-04 18:32:19,947 : INFO : PROGRESS: at sentence #1730000, processed 18169538 words, keeping 47917 word types\n",
      "2019-12-04 18:32:20,092 : INFO : PROGRESS: at sentence #1740000, processed 18281853 words, keeping 48868 word types\n",
      "2019-12-04 18:32:20,244 : INFO : PROGRESS: at sentence #1750000, processed 18396075 words, keeping 50400 word types\n",
      "2019-12-04 18:32:20,394 : INFO : PROGRESS: at sentence #1760000, processed 18508062 words, keeping 51836 word types\n",
      "2019-12-04 18:32:20,546 : INFO : PROGRESS: at sentence #1770000, processed 18622748 words, keeping 53470 word types\n",
      "2019-12-04 18:32:20,700 : INFO : PROGRESS: at sentence #1780000, processed 18736329 words, keeping 55244 word types\n",
      "2019-12-04 18:32:20,873 : INFO : PROGRESS: at sentence #1790000, processed 18859658 words, keeping 56841 word types\n",
      "2019-12-04 18:32:21,035 : INFO : PROGRESS: at sentence #1800000, processed 18980242 words, keeping 58103 word types\n",
      "2019-12-04 18:32:21,194 : INFO : PROGRESS: at sentence #1810000, processed 19102935 words, keeping 59450 word types\n",
      "2019-12-04 18:32:21,341 : INFO : PROGRESS: at sentence #1820000, processed 19218620 words, keeping 59596 word types\n",
      "2019-12-04 18:32:21,494 : INFO : PROGRESS: at sentence #1830000, processed 19335319 words, keeping 59596 word types\n",
      "2019-12-04 18:32:21,624 : INFO : PROGRESS: at sentence #1840000, processed 19434888 words, keeping 59596 word types\n",
      "2019-12-04 18:32:21,754 : INFO : PROGRESS: at sentence #1850000, processed 19535636 words, keeping 59596 word types\n",
      "2019-12-04 18:32:21,886 : INFO : PROGRESS: at sentence #1860000, processed 19637871 words, keeping 59596 word types\n",
      "2019-12-04 18:32:22,021 : INFO : PROGRESS: at sentence #1870000, processed 19742023 words, keeping 59596 word types\n",
      "2019-12-04 18:32:22,146 : INFO : PROGRESS: at sentence #1880000, processed 19837437 words, keeping 59596 word types\n",
      "2019-12-04 18:32:22,274 : INFO : PROGRESS: at sentence #1890000, processed 19933609 words, keeping 59596 word types\n",
      "2019-12-04 18:32:22,401 : INFO : PROGRESS: at sentence #1900000, processed 20031105 words, keeping 59596 word types\n",
      "2019-12-04 18:32:22,529 : INFO : PROGRESS: at sentence #1910000, processed 20127990 words, keeping 59596 word types\n",
      "2019-12-04 18:32:22,663 : INFO : PROGRESS: at sentence #1920000, processed 20228408 words, keeping 59596 word types\n",
      "2019-12-04 18:32:22,799 : INFO : PROGRESS: at sentence #1930000, processed 20334504 words, keeping 59596 word types\n",
      "2019-12-04 18:32:22,932 : INFO : PROGRESS: at sentence #1940000, processed 20439343 words, keeping 59596 word types\n",
      "2019-12-04 18:32:23,066 : INFO : PROGRESS: at sentence #1950000, processed 20543018 words, keeping 59596 word types\n",
      "2019-12-04 18:32:23,205 : INFO : PROGRESS: at sentence #1960000, processed 20648169 words, keeping 59596 word types\n",
      "2019-12-04 18:32:23,345 : INFO : PROGRESS: at sentence #1970000, processed 20754036 words, keeping 59596 word types\n",
      "2019-12-04 18:32:23,479 : INFO : PROGRESS: at sentence #1980000, processed 20853971 words, keeping 59596 word types\n",
      "2019-12-04 18:32:23,616 : INFO : PROGRESS: at sentence #1990000, processed 20957060 words, keeping 59596 word types\n",
      "2019-12-04 18:32:23,757 : INFO : PROGRESS: at sentence #2000000, processed 21063760 words, keeping 59596 word types\n",
      "2019-12-04 18:32:23,924 : INFO : PROGRESS: at sentence #2010000, processed 21191776 words, keeping 59596 word types\n",
      "2019-12-04 18:32:24,076 : INFO : PROGRESS: at sentence #2020000, processed 21306993 words, keeping 59596 word types\n",
      "2019-12-04 18:32:24,220 : INFO : PROGRESS: at sentence #2030000, processed 21418566 words, keeping 59596 word types\n",
      "2019-12-04 18:32:24,358 : INFO : PROGRESS: at sentence #2040000, processed 21525339 words, keeping 59596 word types\n",
      "2019-12-04 18:32:24,496 : INFO : PROGRESS: at sentence #2050000, processed 21631446 words, keeping 59596 word types\n",
      "2019-12-04 18:32:24,637 : INFO : PROGRESS: at sentence #2060000, processed 21739548 words, keeping 59596 word types\n",
      "2019-12-04 18:32:24,777 : INFO : PROGRESS: at sentence #2070000, processed 21846196 words, keeping 59596 word types\n",
      "2019-12-04 18:32:24,919 : INFO : PROGRESS: at sentence #2080000, processed 21953980 words, keeping 59596 word types\n",
      "2019-12-04 18:32:25,062 : INFO : PROGRESS: at sentence #2090000, processed 22061982 words, keeping 59596 word types\n",
      "2019-12-04 18:32:25,204 : INFO : PROGRESS: at sentence #2100000, processed 22170618 words, keeping 59596 word types\n",
      "2019-12-04 18:32:25,353 : INFO : PROGRESS: at sentence #2110000, processed 22280833 words, keeping 59596 word types\n",
      "2019-12-04 18:32:25,508 : INFO : PROGRESS: at sentence #2120000, processed 22396721 words, keeping 59596 word types\n",
      "2019-12-04 18:32:25,655 : INFO : PROGRESS: at sentence #2130000, processed 22507647 words, keeping 59596 word types\n",
      "2019-12-04 18:32:25,805 : INFO : PROGRESS: at sentence #2140000, processed 22620670 words, keeping 59596 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:32:25,956 : INFO : PROGRESS: at sentence #2150000, processed 22733297 words, keeping 59596 word types\n",
      "2019-12-04 18:32:26,116 : INFO : PROGRESS: at sentence #2160000, processed 22854423 words, keeping 59596 word types\n",
      "2019-12-04 18:32:26,281 : INFO : PROGRESS: at sentence #2170000, processed 22974608 words, keeping 59596 word types\n",
      "2019-12-04 18:32:26,439 : INFO : PROGRESS: at sentence #2180000, processed 23095510 words, keeping 59596 word types\n",
      "2019-12-04 18:32:26,599 : INFO : PROGRESS: at sentence #2190000, processed 23218235 words, keeping 59596 word types\n",
      "2019-12-04 18:32:26,750 : INFO : PROGRESS: at sentence #2200000, processed 23335031 words, keeping 59596 word types\n",
      "2019-12-04 18:32:26,889 : INFO : PROGRESS: at sentence #2210000, processed 23442340 words, keeping 59596 word types\n",
      "2019-12-04 18:32:27,020 : INFO : PROGRESS: at sentence #2220000, processed 23541459 words, keeping 59596 word types\n",
      "2019-12-04 18:32:27,152 : INFO : PROGRESS: at sentence #2230000, processed 23643693 words, keeping 59596 word types\n",
      "2019-12-04 18:32:27,286 : INFO : PROGRESS: at sentence #2240000, processed 23746823 words, keeping 59596 word types\n",
      "2019-12-04 18:32:27,413 : INFO : PROGRESS: at sentence #2250000, processed 23844629 words, keeping 59596 word types\n",
      "2019-12-04 18:32:27,546 : INFO : PROGRESS: at sentence #2260000, processed 23945147 words, keeping 59596 word types\n",
      "2019-12-04 18:32:27,672 : INFO : PROGRESS: at sentence #2270000, processed 24039800 words, keeping 59596 word types\n",
      "2019-12-04 18:32:27,799 : INFO : PROGRESS: at sentence #2280000, processed 24136656 words, keeping 59596 word types\n",
      "2019-12-04 18:32:27,929 : INFO : PROGRESS: at sentence #2290000, processed 24234981 words, keeping 59596 word types\n",
      "2019-12-04 18:32:28,063 : INFO : PROGRESS: at sentence #2300000, processed 24337831 words, keeping 59596 word types\n",
      "2019-12-04 18:32:28,203 : INFO : PROGRESS: at sentence #2310000, processed 24443839 words, keeping 59596 word types\n",
      "2019-12-04 18:32:28,339 : INFO : PROGRESS: at sentence #2320000, processed 24548079 words, keeping 59596 word types\n",
      "2019-12-04 18:32:28,476 : INFO : PROGRESS: at sentence #2330000, processed 24653008 words, keeping 59596 word types\n",
      "2019-12-04 18:32:28,613 : INFO : PROGRESS: at sentence #2340000, processed 24758411 words, keeping 59596 word types\n",
      "2019-12-04 18:32:28,748 : INFO : PROGRESS: at sentence #2350000, processed 24861180 words, keeping 59596 word types\n",
      "2019-12-04 18:32:28,883 : INFO : PROGRESS: at sentence #2360000, processed 24962638 words, keeping 59596 word types\n",
      "2019-12-04 18:32:29,017 : INFO : PROGRESS: at sentence #2370000, processed 25064504 words, keeping 59596 word types\n",
      "2019-12-04 18:32:29,170 : INFO : PROGRESS: at sentence #2380000, processed 25183865 words, keeping 59596 word types\n",
      "2019-12-04 18:32:29,324 : INFO : PROGRESS: at sentence #2390000, processed 25305047 words, keeping 59596 word types\n",
      "2019-12-04 18:32:29,473 : INFO : PROGRESS: at sentence #2400000, processed 25420014 words, keeping 59596 word types\n",
      "2019-12-04 18:32:29,615 : INFO : PROGRESS: at sentence #2410000, processed 25530120 words, keeping 59596 word types\n",
      "2019-12-04 18:32:29,753 : INFO : PROGRESS: at sentence #2420000, processed 25636015 words, keeping 59596 word types\n",
      "2019-12-04 18:32:29,890 : INFO : PROGRESS: at sentence #2430000, processed 25741263 words, keeping 59596 word types\n",
      "2019-12-04 18:32:30,030 : INFO : PROGRESS: at sentence #2440000, processed 25849408 words, keeping 59596 word types\n",
      "2019-12-04 18:32:30,170 : INFO : PROGRESS: at sentence #2450000, processed 25957177 words, keeping 59596 word types\n",
      "2019-12-04 18:32:30,313 : INFO : PROGRESS: at sentence #2460000, processed 26063666 words, keeping 59596 word types\n",
      "2019-12-04 18:32:30,457 : INFO : PROGRESS: at sentence #2470000, processed 26172052 words, keeping 59596 word types\n",
      "2019-12-04 18:32:30,603 : INFO : PROGRESS: at sentence #2480000, processed 26282029 words, keeping 59596 word types\n",
      "2019-12-04 18:32:30,747 : INFO : PROGRESS: at sentence #2490000, processed 26392690 words, keeping 59596 word types\n",
      "2019-12-04 18:32:30,898 : INFO : PROGRESS: at sentence #2500000, processed 26508968 words, keeping 59596 word types\n",
      "2019-12-04 18:32:31,042 : INFO : PROGRESS: at sentence #2510000, processed 26618914 words, keeping 59596 word types\n",
      "2019-12-04 18:32:31,196 : INFO : PROGRESS: at sentence #2520000, processed 26733846 words, keeping 59596 word types\n",
      "2019-12-04 18:32:31,345 : INFO : PROGRESS: at sentence #2530000, processed 26843246 words, keeping 59596 word types\n",
      "2019-12-04 18:32:31,510 : INFO : PROGRESS: at sentence #2540000, processed 26969412 words, keeping 59596 word types\n",
      "2019-12-04 18:32:31,670 : INFO : PROGRESS: at sentence #2550000, processed 27090182 words, keeping 59596 word types\n",
      "2019-12-04 18:32:31,830 : INFO : PROGRESS: at sentence #2560000, processed 27211947 words, keeping 59596 word types\n",
      "2019-12-04 18:32:31,982 : INFO : PROGRESS: at sentence #2570000, processed 27330353 words, keeping 59596 word types\n",
      "2019-12-04 18:32:32,135 : INFO : PROGRESS: at sentence #2580000, processed 27449411 words, keeping 59596 word types\n",
      "2019-12-04 18:32:32,268 : INFO : PROGRESS: at sentence #2590000, processed 27549358 words, keeping 59596 word types\n",
      "2019-12-04 18:32:32,401 : INFO : PROGRESS: at sentence #2600000, processed 27649514 words, keeping 59596 word types\n",
      "2019-12-04 18:32:32,534 : INFO : PROGRESS: at sentence #2610000, processed 27751425 words, keeping 59596 word types\n",
      "2019-12-04 18:32:32,671 : INFO : PROGRESS: at sentence #2620000, processed 27856524 words, keeping 59596 word types\n",
      "2019-12-04 18:32:32,797 : INFO : PROGRESS: at sentence #2630000, processed 27952147 words, keeping 59596 word types\n",
      "2019-12-04 18:32:32,927 : INFO : PROGRESS: at sentence #2640000, processed 28049661 words, keeping 59596 word types\n",
      "2019-12-04 18:32:33,054 : INFO : PROGRESS: at sentence #2650000, processed 28146539 words, keeping 59596 word types\n",
      "2019-12-04 18:32:33,183 : INFO : PROGRESS: at sentence #2660000, processed 28242751 words, keeping 59596 word types\n",
      "2019-12-04 18:32:33,315 : INFO : PROGRESS: at sentence #2670000, processed 28342108 words, keeping 59596 word types\n",
      "2019-12-04 18:32:33,454 : INFO : PROGRESS: at sentence #2680000, processed 28447577 words, keeping 59596 word types\n",
      "2019-12-04 18:32:33,592 : INFO : PROGRESS: at sentence #2690000, processed 28552704 words, keeping 59596 word types\n",
      "2019-12-04 18:32:33,729 : INFO : PROGRESS: at sentence #2700000, processed 28656783 words, keeping 59596 word types\n",
      "2019-12-04 18:32:33,866 : INFO : PROGRESS: at sentence #2710000, processed 28762264 words, keeping 59596 word types\n",
      "2019-12-04 18:32:34,003 : INFO : PROGRESS: at sentence #2720000, processed 28867797 words, keeping 59596 word types\n",
      "2019-12-04 18:32:34,136 : INFO : PROGRESS: at sentence #2730000, processed 28968517 words, keeping 59596 word types\n",
      "2019-12-04 18:32:34,269 : INFO : PROGRESS: at sentence #2740000, processed 29070872 words, keeping 59596 word types\n",
      "2019-12-04 18:32:34,403 : INFO : PROGRESS: at sentence #2750000, processed 29173295 words, keeping 59596 word types\n",
      "2019-12-04 18:32:34,565 : INFO : PROGRESS: at sentence #2760000, processed 29302166 words, keeping 59596 word types\n",
      "2019-12-04 18:32:34,714 : INFO : PROGRESS: at sentence #2770000, processed 29418252 words, keeping 59596 word types\n",
      "2019-12-04 18:32:34,860 : INFO : PROGRESS: at sentence #2780000, processed 29530835 words, keeping 59596 word types\n",
      "2019-12-04 18:32:35,001 : INFO : PROGRESS: at sentence #2790000, processed 29639059 words, keeping 59596 word types\n",
      "2019-12-04 18:32:35,140 : INFO : PROGRESS: at sentence #2800000, processed 29745152 words, keeping 59596 word types\n",
      "2019-12-04 18:32:35,282 : INFO : PROGRESS: at sentence #2810000, processed 29852424 words, keeping 59596 word types\n",
      "2019-12-04 18:32:35,422 : INFO : PROGRESS: at sentence #2820000, processed 29959203 words, keeping 59596 word types\n",
      "2019-12-04 18:32:35,563 : INFO : PROGRESS: at sentence #2830000, processed 30066634 words, keeping 59596 word types\n",
      "2019-12-04 18:32:35,703 : INFO : PROGRESS: at sentence #2840000, processed 30173904 words, keeping 59596 word types\n",
      "2019-12-04 18:32:35,894 : INFO : PROGRESS: at sentence #2850000, processed 30282305 words, keeping 59596 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:32:36,037 : INFO : PROGRESS: at sentence #2860000, processed 30393024 words, keeping 59596 word types\n",
      "2019-12-04 18:32:36,184 : INFO : PROGRESS: at sentence #2870000, processed 30506503 words, keeping 59596 word types\n",
      "2019-12-04 18:32:36,333 : INFO : PROGRESS: at sentence #2880000, processed 30619744 words, keeping 59596 word types\n",
      "2019-12-04 18:32:36,481 : INFO : PROGRESS: at sentence #2890000, processed 30731804 words, keeping 59596 word types\n",
      "2019-12-04 18:32:36,632 : INFO : PROGRESS: at sentence #2900000, processed 30846317 words, keeping 59596 word types\n",
      "2019-12-04 18:32:36,785 : INFO : PROGRESS: at sentence #2910000, processed 30963700 words, keeping 59596 word types\n",
      "2019-12-04 18:32:36,942 : INFO : PROGRESS: at sentence #2920000, processed 31084267 words, keeping 59596 word types\n",
      "2019-12-04 18:32:37,103 : INFO : PROGRESS: at sentence #2930000, processed 31204922 words, keeping 59596 word types\n",
      "2019-12-04 18:32:37,263 : INFO : PROGRESS: at sentence #2940000, processed 31327939 words, keeping 59596 word types\n",
      "2019-12-04 18:32:37,412 : INFO : PROGRESS: at sentence #2950000, processed 31444996 words, keeping 59596 word types\n",
      "2019-12-04 18:32:37,558 : INFO : PROGRESS: at sentence #2960000, processed 31556820 words, keeping 59596 word types\n",
      "2019-12-04 18:32:37,690 : INFO : PROGRESS: at sentence #2970000, processed 31656538 words, keeping 59596 word types\n",
      "2019-12-04 18:32:37,825 : INFO : PROGRESS: at sentence #2980000, processed 31757736 words, keeping 59596 word types\n",
      "2019-12-04 18:32:37,960 : INFO : PROGRESS: at sentence #2990000, processed 31860272 words, keeping 59596 word types\n",
      "2019-12-04 18:32:38,089 : INFO : PROGRESS: at sentence #3000000, processed 31957724 words, keeping 59596 word types\n",
      "2019-12-04 18:32:38,225 : INFO : PROGRESS: at sentence #3010000, processed 32060265 words, keeping 59596 word types\n",
      "2019-12-04 18:32:38,349 : INFO : PROGRESS: at sentence #3020000, processed 32154171 words, keeping 59596 word types\n",
      "2019-12-04 18:32:38,478 : INFO : PROGRESS: at sentence #3030000, processed 32252331 words, keeping 59596 word types\n",
      "2019-12-04 18:32:38,605 : INFO : PROGRESS: at sentence #3040000, processed 32349338 words, keeping 59596 word types\n",
      "2019-12-04 18:32:38,737 : INFO : PROGRESS: at sentence #3050000, processed 32450946 words, keeping 59596 word types\n",
      "2019-12-04 18:32:38,875 : INFO : PROGRESS: at sentence #3060000, processed 32557259 words, keeping 59596 word types\n",
      "2019-12-04 18:32:39,010 : INFO : PROGRESS: at sentence #3070000, processed 32661531 words, keeping 59596 word types\n",
      "2019-12-04 18:32:39,145 : INFO : PROGRESS: at sentence #3080000, processed 32766037 words, keeping 59596 word types\n",
      "2019-12-04 18:32:39,284 : INFO : PROGRESS: at sentence #3090000, processed 32871452 words, keeping 59596 word types\n",
      "2019-12-04 18:32:39,421 : INFO : PROGRESS: at sentence #3100000, processed 32976000 words, keeping 59596 word types\n",
      "2019-12-04 18:32:39,554 : INFO : PROGRESS: at sentence #3110000, processed 33075654 words, keeping 59596 word types\n",
      "2019-12-04 18:32:39,691 : INFO : PROGRESS: at sentence #3120000, processed 33178915 words, keeping 59596 word types\n",
      "2019-12-04 18:32:39,837 : INFO : PROGRESS: at sentence #3130000, processed 33290423 words, keeping 59596 word types\n",
      "2019-12-04 18:32:39,999 : INFO : PROGRESS: at sentence #3140000, processed 33416108 words, keeping 59596 word types\n",
      "2019-12-04 18:32:40,148 : INFO : PROGRESS: at sentence #3150000, processed 33532007 words, keeping 59596 word types\n",
      "2019-12-04 18:32:40,293 : INFO : PROGRESS: at sentence #3160000, processed 33644225 words, keeping 59596 word types\n",
      "2019-12-04 18:32:40,429 : INFO : PROGRESS: at sentence #3170000, processed 33747676 words, keeping 59596 word types\n",
      "2019-12-04 18:32:40,568 : INFO : PROGRESS: at sentence #3180000, processed 33854015 words, keeping 59596 word types\n",
      "2019-12-04 18:32:40,710 : INFO : PROGRESS: at sentence #3190000, processed 33962098 words, keeping 59596 word types\n",
      "2019-12-04 18:32:40,854 : INFO : PROGRESS: at sentence #3200000, processed 34069434 words, keeping 59596 word types\n",
      "2019-12-04 18:32:40,994 : INFO : PROGRESS: at sentence #3210000, processed 34176832 words, keeping 59596 word types\n",
      "2019-12-04 18:32:41,140 : INFO : PROGRESS: at sentence #3220000, processed 34285232 words, keeping 59596 word types\n",
      "2019-12-04 18:32:41,295 : INFO : PROGRESS: at sentence #3230000, processed 34394077 words, keeping 59596 word types\n",
      "2019-12-04 18:32:41,435 : INFO : PROGRESS: at sentence #3240000, processed 34504363 words, keeping 59596 word types\n",
      "2019-12-04 18:32:41,588 : INFO : PROGRESS: at sentence #3250000, processed 34620780 words, keeping 59596 word types\n",
      "2019-12-04 18:32:41,735 : INFO : PROGRESS: at sentence #3260000, processed 34730853 words, keeping 59596 word types\n",
      "2019-12-04 18:32:41,885 : INFO : PROGRESS: at sentence #3270000, processed 34845202 words, keeping 59596 word types\n",
      "2019-12-04 18:32:42,034 : INFO : PROGRESS: at sentence #3280000, processed 34956045 words, keeping 59596 word types\n",
      "2019-12-04 18:32:42,192 : INFO : PROGRESS: at sentence #3290000, processed 35078978 words, keeping 59596 word types\n",
      "2019-12-04 18:32:42,350 : INFO : PROGRESS: at sentence #3300000, processed 35199429 words, keeping 59596 word types\n",
      "2019-12-04 18:32:42,508 : INFO : PROGRESS: at sentence #3310000, processed 35320837 words, keeping 59596 word types\n",
      "2019-12-04 18:32:42,665 : INFO : PROGRESS: at sentence #3320000, processed 35442043 words, keeping 59596 word types\n",
      "2019-12-04 18:32:42,819 : INFO : PROGRESS: at sentence #3330000, processed 35559670 words, keeping 59596 word types\n",
      "2019-12-04 18:32:42,958 : INFO : PROGRESS: at sentence #3340000, processed 35664257 words, keeping 59596 word types\n",
      "2019-12-04 18:32:43,089 : INFO : PROGRESS: at sentence #3350000, processed 35763270 words, keeping 59596 word types\n",
      "2019-12-04 18:32:43,222 : INFO : PROGRESS: at sentence #3360000, processed 35865646 words, keeping 59596 word types\n",
      "2019-12-04 18:32:43,355 : INFO : PROGRESS: at sentence #3370000, processed 35969565 words, keeping 59596 word types\n",
      "2019-12-04 18:32:43,480 : INFO : PROGRESS: at sentence #3380000, processed 36065146 words, keeping 59596 word types\n",
      "2019-12-04 18:32:43,610 : INFO : PROGRESS: at sentence #3390000, processed 36166028 words, keeping 59596 word types\n",
      "2019-12-04 18:32:43,737 : INFO : PROGRESS: at sentence #3400000, processed 36261202 words, keeping 59596 word types\n",
      "2019-12-04 18:32:43,864 : INFO : PROGRESS: at sentence #3410000, processed 36357844 words, keeping 59596 word types\n",
      "2019-12-04 18:32:44,004 : INFO : PROGRESS: at sentence #3420000, processed 36457042 words, keeping 59596 word types\n",
      "2019-12-04 18:32:44,144 : INFO : PROGRESS: at sentence #3430000, processed 36560859 words, keeping 59596 word types\n",
      "2019-12-04 18:32:44,287 : INFO : PROGRESS: at sentence #3440000, processed 36666201 words, keeping 59596 word types\n",
      "2019-12-04 18:32:44,428 : INFO : PROGRESS: at sentence #3450000, processed 36770613 words, keeping 59596 word types\n",
      "2019-12-04 18:32:44,566 : INFO : PROGRESS: at sentence #3460000, processed 36875490 words, keeping 59596 word types\n",
      "2019-12-04 18:32:44,702 : INFO : PROGRESS: at sentence #3470000, processed 36981391 words, keeping 59596 word types\n",
      "2019-12-04 18:32:44,835 : INFO : PROGRESS: at sentence #3480000, processed 37083151 words, keeping 59596 word types\n",
      "2019-12-04 18:32:44,969 : INFO : PROGRESS: at sentence #3490000, processed 37184744 words, keeping 59596 word types\n",
      "2019-12-04 18:32:45,105 : INFO : PROGRESS: at sentence #3500000, processed 37286802 words, keeping 59596 word types\n",
      "2019-12-04 18:32:45,262 : INFO : PROGRESS: at sentence #3510000, processed 37410025 words, keeping 59596 word types\n",
      "2019-12-04 18:32:45,415 : INFO : PROGRESS: at sentence #3520000, processed 37529409 words, keeping 59596 word types\n",
      "2019-12-04 18:32:45,561 : INFO : PROGRESS: at sentence #3530000, processed 37642994 words, keeping 59596 word types\n",
      "2019-12-04 18:32:45,703 : INFO : PROGRESS: at sentence #3540000, processed 37752800 words, keeping 59596 word types\n",
      "2019-12-04 18:32:45,840 : INFO : PROGRESS: at sentence #3550000, processed 37858776 words, keeping 59596 word types\n",
      "2019-12-04 18:32:45,978 : INFO : PROGRESS: at sentence #3560000, processed 37965305 words, keeping 59596 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:32:46,115 : INFO : PROGRESS: at sentence #3570000, processed 38072079 words, keeping 59596 word types\n",
      "2019-12-04 18:32:46,256 : INFO : PROGRESS: at sentence #3580000, processed 38179474 words, keeping 59596 word types\n",
      "2019-12-04 18:32:46,401 : INFO : PROGRESS: at sentence #3590000, processed 38286650 words, keeping 59596 word types\n",
      "2019-12-04 18:32:46,542 : INFO : PROGRESS: at sentence #3600000, processed 38395496 words, keeping 59596 word types\n",
      "2019-12-04 18:32:46,683 : INFO : PROGRESS: at sentence #3610000, processed 38505254 words, keeping 59596 word types\n",
      "2019-12-04 18:32:46,829 : INFO : PROGRESS: at sentence #3620000, processed 38616556 words, keeping 59596 word types\n",
      "2019-12-04 18:32:46,979 : INFO : PROGRESS: at sentence #3630000, processed 38731723 words, keeping 59596 word types\n",
      "2019-12-04 18:32:47,126 : INFO : PROGRESS: at sentence #3640000, processed 38843412 words, keeping 59596 word types\n",
      "2019-12-04 18:32:47,276 : INFO : PROGRESS: at sentence #3650000, processed 38957697 words, keeping 59596 word types\n",
      "2019-12-04 18:32:47,425 : INFO : PROGRESS: at sentence #3660000, processed 39068401 words, keeping 59596 word types\n",
      "2019-12-04 18:32:47,590 : INFO : PROGRESS: at sentence #3670000, processed 39194451 words, keeping 59596 word types\n",
      "2019-12-04 18:32:47,746 : INFO : PROGRESS: at sentence #3680000, processed 39314901 words, keeping 59596 word types\n",
      "2019-12-04 18:32:47,907 : INFO : PROGRESS: at sentence #3690000, processed 39437420 words, keeping 59596 word types\n",
      "2019-12-04 18:32:48,059 : INFO : PROGRESS: at sentence #3700000, processed 39554246 words, keeping 59596 word types\n",
      "2019-12-04 18:32:48,212 : INFO : PROGRESS: at sentence #3710000, processed 39671981 words, keeping 59596 word types\n",
      "2019-12-04 18:32:48,343 : INFO : PROGRESS: at sentence #3720000, processed 39771155 words, keeping 59596 word types\n",
      "2019-12-04 18:32:48,474 : INFO : PROGRESS: at sentence #3730000, processed 39871766 words, keeping 59596 word types\n",
      "2019-12-04 18:32:48,607 : INFO : PROGRESS: at sentence #3740000, processed 39973602 words, keeping 59596 word types\n",
      "2019-12-04 18:32:48,746 : INFO : PROGRESS: at sentence #3750000, processed 40078689 words, keeping 59596 word types\n",
      "2019-12-04 18:32:48,871 : INFO : PROGRESS: at sentence #3760000, processed 40173704 words, keeping 59596 word types\n",
      "2019-12-04 18:32:48,998 : INFO : PROGRESS: at sentence #3770000, processed 40270440 words, keeping 59596 word types\n",
      "2019-12-04 18:32:49,125 : INFO : PROGRESS: at sentence #3780000, processed 40367899 words, keeping 59596 word types\n",
      "2019-12-04 18:32:49,250 : INFO : PROGRESS: at sentence #3790000, processed 40464132 words, keeping 59596 word types\n",
      "2019-12-04 18:32:49,381 : INFO : PROGRESS: at sentence #3800000, processed 40563729 words, keeping 59596 word types\n",
      "2019-12-04 18:32:49,520 : INFO : PROGRESS: at sentence #3810000, processed 40670299 words, keeping 59596 word types\n",
      "2019-12-04 18:32:49,655 : INFO : PROGRESS: at sentence #3820000, processed 40775246 words, keeping 59596 word types\n",
      "2019-12-04 18:32:49,791 : INFO : PROGRESS: at sentence #3830000, processed 40879081 words, keeping 59596 word types\n",
      "2019-12-04 18:32:49,928 : INFO : PROGRESS: at sentence #3840000, processed 40984333 words, keeping 59596 word types\n",
      "2019-12-04 18:32:50,062 : INFO : PROGRESS: at sentence #3850000, processed 41090062 words, keeping 59596 word types\n",
      "2019-12-04 18:32:50,193 : INFO : PROGRESS: at sentence #3860000, processed 41190187 words, keeping 59596 word types\n",
      "2019-12-04 18:32:50,329 : INFO : PROGRESS: at sentence #3870000, processed 41292951 words, keeping 59596 word types\n",
      "2019-12-04 18:32:50,468 : INFO : PROGRESS: at sentence #3880000, processed 41397692 words, keeping 59596 word types\n",
      "2019-12-04 18:32:50,632 : INFO : PROGRESS: at sentence #3890000, processed 41526722 words, keeping 59596 word types\n",
      "2019-12-04 18:32:50,780 : INFO : PROGRESS: at sentence #3900000, processed 41642016 words, keeping 59596 word types\n",
      "2019-12-04 18:32:50,924 : INFO : PROGRESS: at sentence #3910000, processed 41754329 words, keeping 59596 word types\n",
      "2019-12-04 18:32:51,063 : INFO : PROGRESS: at sentence #3920000, processed 41861385 words, keeping 59596 word types\n",
      "2019-12-04 18:32:51,201 : INFO : PROGRESS: at sentence #3930000, processed 41967556 words, keeping 59596 word types\n",
      "2019-12-04 18:32:51,340 : INFO : PROGRESS: at sentence #3940000, processed 42075396 words, keeping 59596 word types\n",
      "2019-12-04 18:32:51,480 : INFO : PROGRESS: at sentence #3950000, processed 42181812 words, keeping 59596 word types\n",
      "2019-12-04 18:32:51,620 : INFO : PROGRESS: at sentence #3960000, processed 42289627 words, keeping 59596 word types\n",
      "2019-12-04 18:32:51,761 : INFO : PROGRESS: at sentence #3970000, processed 42397400 words, keeping 59596 word types\n",
      "2019-12-04 18:32:51,900 : INFO : PROGRESS: at sentence #3980000, processed 42505967 words, keeping 59596 word types\n",
      "2019-12-04 18:32:52,041 : INFO : PROGRESS: at sentence #3990000, processed 42616287 words, keeping 59596 word types\n",
      "2019-12-04 18:32:52,188 : INFO : PROGRESS: at sentence #4000000, processed 42731547 words, keeping 59596 word types\n",
      "2019-12-04 18:32:52,335 : INFO : PROGRESS: at sentence #4010000, processed 42843354 words, keeping 59596 word types\n",
      "2019-12-04 18:32:52,481 : INFO : PROGRESS: at sentence #4020000, processed 42955950 words, keeping 59596 word types\n",
      "2019-12-04 18:32:52,635 : INFO : PROGRESS: at sentence #4030000, processed 43068958 words, keeping 59596 word types\n",
      "2019-12-04 18:32:52,820 : INFO : PROGRESS: at sentence #4040000, processed 43189053 words, keeping 59596 word types\n",
      "2019-12-04 18:32:52,975 : INFO : PROGRESS: at sentence #4050000, processed 43309074 words, keeping 59596 word types\n",
      "2019-12-04 18:32:53,130 : INFO : PROGRESS: at sentence #4060000, processed 43430009 words, keeping 59596 word types\n",
      "2019-12-04 18:32:53,288 : INFO : PROGRESS: at sentence #4070000, processed 43553109 words, keeping 59596 word types\n",
      "2019-12-04 18:32:53,440 : INFO : PROGRESS: at sentence #4080000, processed 43670590 words, keeping 59596 word types\n",
      "2019-12-04 18:32:53,578 : INFO : PROGRESS: at sentence #4090000, processed 43778468 words, keeping 59596 word types\n",
      "2019-12-04 18:32:53,707 : INFO : PROGRESS: at sentence #4100000, processed 43878416 words, keeping 59596 word types\n",
      "2019-12-04 18:32:53,839 : INFO : PROGRESS: at sentence #4110000, processed 43979722 words, keeping 59596 word types\n",
      "2019-12-04 18:32:53,974 : INFO : PROGRESS: at sentence #4120000, processed 44082600 words, keeping 59596 word types\n",
      "2019-12-04 18:32:54,101 : INFO : PROGRESS: at sentence #4130000, processed 44180369 words, keeping 59596 word types\n",
      "2019-12-04 18:32:54,233 : INFO : PROGRESS: at sentence #4140000, processed 44281928 words, keeping 59596 word types\n",
      "2019-12-04 18:32:54,359 : INFO : PROGRESS: at sentence #4150000, processed 44376596 words, keeping 59596 word types\n",
      "2019-12-04 18:32:54,487 : INFO : PROGRESS: at sentence #4160000, processed 44473589 words, keeping 59596 word types\n",
      "2019-12-04 18:32:54,618 : INFO : PROGRESS: at sentence #4170000, processed 44571719 words, keeping 59596 word types\n",
      "2019-12-04 18:32:54,750 : INFO : PROGRESS: at sentence #4180000, processed 44674085 words, keeping 59596 word types\n",
      "2019-12-04 18:32:54,885 : INFO : PROGRESS: at sentence #4190000, processed 44779385 words, keeping 59596 word types\n",
      "2019-12-04 18:32:55,021 : INFO : PROGRESS: at sentence #4200000, processed 44884093 words, keeping 59596 word types\n",
      "2019-12-04 18:32:55,155 : INFO : PROGRESS: at sentence #4210000, processed 44988628 words, keeping 59596 word types\n",
      "2019-12-04 18:32:55,291 : INFO : PROGRESS: at sentence #4220000, processed 45094115 words, keeping 59596 word types\n",
      "2019-12-04 18:32:55,425 : INFO : PROGRESS: at sentence #4230000, processed 45197570 words, keeping 59596 word types\n",
      "2019-12-04 18:32:55,557 : INFO : PROGRESS: at sentence #4240000, processed 45298262 words, keeping 59596 word types\n",
      "2019-12-04 18:32:55,692 : INFO : PROGRESS: at sentence #4250000, processed 45400877 words, keeping 59596 word types\n",
      "2019-12-04 18:32:55,843 : INFO : PROGRESS: at sentence #4260000, processed 45517782 words, keeping 59596 word types\n",
      "2019-12-04 18:32:55,999 : INFO : PROGRESS: at sentence #4270000, processed 45639983 words, keeping 59596 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:32:56,146 : INFO : PROGRESS: at sentence #4280000, processed 45755479 words, keeping 59596 word types\n",
      "2019-12-04 18:32:56,287 : INFO : PROGRESS: at sentence #4290000, processed 45866348 words, keeping 59596 word types\n",
      "2019-12-04 18:32:56,423 : INFO : PROGRESS: at sentence #4300000, processed 45970746 words, keeping 59596 word types\n",
      "2019-12-04 18:32:56,562 : INFO : PROGRESS: at sentence #4310000, processed 46076775 words, keeping 59596 word types\n",
      "2019-12-04 18:32:56,702 : INFO : PROGRESS: at sentence #4320000, processed 46185163 words, keeping 59596 word types\n",
      "2019-12-04 18:32:56,841 : INFO : PROGRESS: at sentence #4330000, processed 46292863 words, keeping 59596 word types\n",
      "2019-12-04 18:32:56,980 : INFO : PROGRESS: at sentence #4340000, processed 46399707 words, keeping 59596 word types\n",
      "2019-12-04 18:32:57,120 : INFO : PROGRESS: at sentence #4350000, processed 46507858 words, keeping 59596 word types\n",
      "2019-12-04 18:32:57,262 : INFO : PROGRESS: at sentence #4360000, processed 46617432 words, keeping 59596 word types\n",
      "2019-12-04 18:32:57,404 : INFO : PROGRESS: at sentence #4370000, processed 46727735 words, keeping 59596 word types\n",
      "2019-12-04 18:32:57,555 : INFO : PROGRESS: at sentence #4380000, processed 46844184 words, keeping 59596 word types\n",
      "2019-12-04 18:32:57,699 : INFO : PROGRESS: at sentence #4390000, processed 46954051 words, keeping 59596 word types\n",
      "2019-12-04 18:32:57,848 : INFO : PROGRESS: at sentence #4400000, processed 47068755 words, keeping 59596 word types\n",
      "2019-12-04 18:32:57,994 : INFO : PROGRESS: at sentence #4410000, processed 47178797 words, keeping 59596 word types\n",
      "2019-12-04 18:32:58,156 : INFO : PROGRESS: at sentence #4420000, processed 47304144 words, keeping 59596 word types\n",
      "2019-12-04 18:32:58,312 : INFO : PROGRESS: at sentence #4430000, processed 47424445 words, keeping 59596 word types\n",
      "2019-12-04 18:32:58,467 : INFO : PROGRESS: at sentence #4440000, processed 47546268 words, keeping 59596 word types\n",
      "2019-12-04 18:32:58,615 : INFO : PROGRESS: at sentence #4450000, processed 47665623 words, keeping 59596 word types\n",
      "2019-12-04 18:32:58,765 : INFO : PROGRESS: at sentence #4460000, processed 47784519 words, keeping 59596 word types\n",
      "2019-12-04 18:32:58,895 : INFO : PROGRESS: at sentence #4470000, processed 47885831 words, keeping 59596 word types\n",
      "2019-12-04 18:32:59,024 : INFO : PROGRESS: at sentence #4480000, processed 47985433 words, keeping 59596 word types\n",
      "2019-12-04 18:32:59,156 : INFO : PROGRESS: at sentence #4490000, processed 48087416 words, keeping 59596 word types\n",
      "2019-12-04 18:32:59,291 : INFO : PROGRESS: at sentence #4500000, processed 48192438 words, keeping 59596 word types\n",
      "2019-12-04 18:32:59,415 : INFO : PROGRESS: at sentence #4510000, processed 48288461 words, keeping 59596 word types\n",
      "2019-12-04 18:32:59,545 : INFO : PROGRESS: at sentence #4520000, processed 48386966 words, keeping 59596 word types\n",
      "2019-12-04 18:32:59,671 : INFO : PROGRESS: at sentence #4530000, processed 48483195 words, keeping 59596 word types\n",
      "2019-12-04 18:32:59,796 : INFO : PROGRESS: at sentence #4540000, processed 48579091 words, keeping 59596 word types\n",
      "2019-12-04 18:32:59,925 : INFO : PROGRESS: at sentence #4550000, processed 48678755 words, keeping 59596 word types\n",
      "2019-12-04 18:33:00,059 : INFO : PROGRESS: at sentence #4560000, processed 48783436 words, keeping 59596 word types\n",
      "2019-12-04 18:33:00,193 : INFO : PROGRESS: at sentence #4570000, processed 48888531 words, keeping 59596 word types\n",
      "2019-12-04 18:33:00,331 : INFO : PROGRESS: at sentence #4580000, processed 48993171 words, keeping 59596 word types\n",
      "2019-12-04 18:33:00,467 : INFO : PROGRESS: at sentence #4590000, processed 49097921 words, keeping 59596 word types\n",
      "2019-12-04 18:33:00,605 : INFO : PROGRESS: at sentence #4600000, processed 49204238 words, keeping 59596 word types\n",
      "2019-12-04 18:33:00,736 : INFO : PROGRESS: at sentence #4610000, processed 49304830 words, keeping 59596 word types\n",
      "2019-12-04 18:33:00,868 : INFO : PROGRESS: at sentence #4620000, processed 49407095 words, keeping 59596 word types\n",
      "2019-12-04 18:33:01,000 : INFO : PROGRESS: at sentence #4630000, processed 49509328 words, keeping 59596 word types\n",
      "2019-12-04 18:33:01,160 : INFO : PROGRESS: at sentence #4640000, processed 49636408 words, keeping 59596 word types\n",
      "2019-12-04 18:33:01,311 : INFO : PROGRESS: at sentence #4650000, processed 49753546 words, keeping 59596 word types\n",
      "2019-12-04 18:33:01,456 : INFO : PROGRESS: at sentence #4660000, processed 49866194 words, keeping 59596 word types\n",
      "2019-12-04 18:33:01,596 : INFO : PROGRESS: at sentence #4670000, processed 49975239 words, keeping 59596 word types\n",
      "2019-12-04 18:33:01,734 : INFO : PROGRESS: at sentence #4680000, processed 50081164 words, keeping 59596 word types\n",
      "2019-12-04 18:33:01,874 : INFO : PROGRESS: at sentence #4690000, processed 50188239 words, keeping 59596 word types\n",
      "2019-12-04 18:33:02,013 : INFO : PROGRESS: at sentence #4700000, processed 50294914 words, keeping 59596 word types\n",
      "2019-12-04 18:33:02,156 : INFO : PROGRESS: at sentence #4710000, processed 50402191 words, keeping 59596 word types\n",
      "2019-12-04 18:33:02,296 : INFO : PROGRESS: at sentence #4720000, processed 50509526 words, keeping 59596 word types\n",
      "2019-12-04 18:33:02,435 : INFO : PROGRESS: at sentence #4730000, processed 50618133 words, keeping 59596 word types\n",
      "2019-12-04 18:33:02,576 : INFO : PROGRESS: at sentence #4740000, processed 50728410 words, keeping 59596 word types\n",
      "2019-12-04 18:33:02,722 : INFO : PROGRESS: at sentence #4750000, processed 50841026 words, keeping 59596 word types\n",
      "2019-12-04 18:33:02,871 : INFO : PROGRESS: at sentence #4760000, processed 50954958 words, keeping 59596 word types\n",
      "2019-12-04 18:33:03,019 : INFO : PROGRESS: at sentence #4770000, processed 51067343 words, keeping 59596 word types\n",
      "2019-12-04 18:33:03,167 : INFO : PROGRESS: at sentence #4780000, processed 51181788 words, keeping 59596 word types\n",
      "2019-12-04 18:33:03,317 : INFO : PROGRESS: at sentence #4790000, processed 51297250 words, keeping 59596 word types\n",
      "2019-12-04 18:33:03,476 : INFO : PROGRESS: at sentence #4800000, processed 51419114 words, keeping 59596 word types\n",
      "2019-12-04 18:33:03,631 : INFO : PROGRESS: at sentence #4810000, processed 51539469 words, keeping 59596 word types\n",
      "2019-12-04 18:33:03,790 : INFO : PROGRESS: at sentence #4820000, processed 51662892 words, keeping 59596 word types\n",
      "2019-12-04 18:33:03,935 : INFO : PROGRESS: at sentence #4830000, processed 51777645 words, keeping 59596 word types\n",
      "2019-12-04 18:33:04,083 : INFO : PROGRESS: at sentence #4840000, processed 51893429 words, keeping 59596 word types\n",
      "2019-12-04 18:33:04,213 : INFO : PROGRESS: at sentence #4850000, processed 51992949 words, keeping 59596 word types\n",
      "2019-12-04 18:33:04,343 : INFO : PROGRESS: at sentence #4860000, processed 52093937 words, keeping 59596 word types\n",
      "2019-12-04 18:33:04,473 : INFO : PROGRESS: at sentence #4870000, processed 52196494 words, keeping 59596 word types\n",
      "2019-12-04 18:33:04,601 : INFO : PROGRESS: at sentence #4880000, processed 52297318 words, keeping 59596 word types\n",
      "2019-12-04 18:33:04,728 : INFO : PROGRESS: at sentence #4890000, processed 52396083 words, keeping 59596 word types\n",
      "2019-12-04 18:33:04,851 : INFO : PROGRESS: at sentence #4900000, processed 52491133 words, keeping 59596 word types\n",
      "2019-12-04 18:33:04,977 : INFO : PROGRESS: at sentence #4910000, processed 52588913 words, keeping 59596 word types\n",
      "2019-12-04 18:33:05,104 : INFO : PROGRESS: at sentence #4920000, processed 52685874 words, keeping 59596 word types\n",
      "2019-12-04 18:33:05,235 : INFO : PROGRESS: at sentence #4930000, processed 52786931 words, keeping 59596 word types\n",
      "2019-12-04 18:33:05,374 : INFO : PROGRESS: at sentence #4940000, processed 52893141 words, keeping 59596 word types\n",
      "2019-12-04 18:33:05,509 : INFO : PROGRESS: at sentence #4950000, processed 52997812 words, keeping 59596 word types\n",
      "2019-12-04 18:33:05,642 : INFO : PROGRESS: at sentence #4960000, processed 53101875 words, keeping 59596 word types\n",
      "2019-12-04 18:33:05,779 : INFO : PROGRESS: at sentence #4970000, processed 53206923 words, keeping 59596 word types\n",
      "2019-12-04 18:33:05,915 : INFO : PROGRESS: at sentence #4980000, processed 53312273 words, keeping 59596 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:33:06,046 : INFO : PROGRESS: at sentence #4990000, processed 53412248 words, keeping 59596 word types\n",
      "2019-12-04 18:33:06,178 : INFO : PROGRESS: at sentence #5000000, processed 53515034 words, keeping 59596 word types\n",
      "2019-12-04 18:33:06,321 : INFO : PROGRESS: at sentence #5010000, processed 53624396 words, keeping 59596 word types\n",
      "2019-12-04 18:33:06,480 : INFO : PROGRESS: at sentence #5020000, processed 53751184 words, keeping 59596 word types\n",
      "2019-12-04 18:33:06,627 : INFO : PROGRESS: at sentence #5030000, processed 53867086 words, keeping 59596 word types\n",
      "2019-12-04 18:33:06,770 : INFO : PROGRESS: at sentence #5040000, processed 53977964 words, keeping 59596 word types\n",
      "2019-12-04 18:33:06,908 : INFO : PROGRESS: at sentence #5050000, processed 54083655 words, keeping 59596 word types\n",
      "2019-12-04 18:33:07,048 : INFO : PROGRESS: at sentence #5060000, processed 54189678 words, keeping 59596 word types\n",
      "2019-12-04 18:33:07,191 : INFO : PROGRESS: at sentence #5070000, processed 54298142 words, keeping 59596 word types\n",
      "2019-12-04 18:33:07,332 : INFO : PROGRESS: at sentence #5080000, processed 54405276 words, keeping 59596 word types\n",
      "2019-12-04 18:33:07,474 : INFO : PROGRESS: at sentence #5090000, processed 54512612 words, keeping 59596 word types\n",
      "2019-12-04 18:33:07,616 : INFO : PROGRESS: at sentence #5100000, processed 54621167 words, keeping 59596 word types\n",
      "2019-12-04 18:33:07,758 : INFO : PROGRESS: at sentence #5110000, processed 54729375 words, keeping 59596 word types\n",
      "2019-12-04 18:33:07,899 : INFO : PROGRESS: at sentence #5120000, processed 54839814 words, keeping 59596 word types\n",
      "2019-12-04 18:33:08,050 : INFO : PROGRESS: at sentence #5130000, processed 54955892 words, keeping 59596 word types\n",
      "2019-12-04 18:33:08,195 : INFO : PROGRESS: at sentence #5140000, processed 55066277 words, keeping 59596 word types\n",
      "2019-12-04 18:33:08,344 : INFO : PROGRESS: at sentence #5150000, processed 55180318 words, keeping 59596 word types\n",
      "2019-12-04 18:33:08,491 : INFO : PROGRESS: at sentence #5160000, processed 55291929 words, keeping 59596 word types\n",
      "2019-12-04 18:33:08,650 : INFO : PROGRESS: at sentence #5170000, processed 55413794 words, keeping 59596 word types\n",
      "2019-12-04 18:33:08,806 : INFO : PROGRESS: at sentence #5180000, processed 55533883 words, keeping 59596 word types\n",
      "2019-12-04 18:33:08,964 : INFO : PROGRESS: at sentence #5190000, processed 55655043 words, keeping 59596 word types\n",
      "2019-12-04 18:33:09,121 : INFO : PROGRESS: at sentence #5200000, processed 55777234 words, keeping 59596 word types\n",
      "2019-12-04 18:33:09,272 : INFO : PROGRESS: at sentence #5210000, processed 55894331 words, keeping 59596 word types\n",
      "2019-12-04 18:33:09,410 : INFO : PROGRESS: at sentence #5220000, processed 56000708 words, keeping 59596 word types\n",
      "2019-12-04 18:33:09,541 : INFO : PROGRESS: at sentence #5230000, processed 56099612 words, keeping 59596 word types\n",
      "2019-12-04 18:33:09,675 : INFO : PROGRESS: at sentence #5240000, processed 56201951 words, keeping 59596 word types\n",
      "2019-12-04 18:33:09,809 : INFO : PROGRESS: at sentence #5250000, processed 56305473 words, keeping 59596 word types\n",
      "2019-12-04 18:33:09,935 : INFO : PROGRESS: at sentence #5260000, processed 56403773 words, keeping 59596 word types\n",
      "2019-12-04 18:33:10,065 : INFO : PROGRESS: at sentence #5270000, processed 56502809 words, keeping 59596 word types\n",
      "2019-12-04 18:33:10,193 : INFO : PROGRESS: at sentence #5280000, processed 56597938 words, keeping 59596 word types\n",
      "2019-12-04 18:33:10,324 : INFO : PROGRESS: at sentence #5290000, processed 56694603 words, keeping 59596 word types\n",
      "2019-12-04 18:33:10,456 : INFO : PROGRESS: at sentence #5300000, processed 56793240 words, keeping 59596 word types\n",
      "2019-12-04 18:33:10,591 : INFO : PROGRESS: at sentence #5310000, processed 56896301 words, keeping 59596 word types\n",
      "2019-12-04 18:33:10,728 : INFO : PROGRESS: at sentence #5320000, processed 57001993 words, keeping 59596 word types\n",
      "2019-12-04 18:33:10,866 : INFO : PROGRESS: at sentence #5330000, processed 57106549 words, keeping 59596 word types\n",
      "2019-12-04 18:33:11,016 : INFO : PROGRESS: at sentence #5340000, processed 57211512 words, keeping 59596 word types\n",
      "2019-12-04 18:33:11,152 : INFO : PROGRESS: at sentence #5350000, processed 57317178 words, keeping 59596 word types\n",
      "2019-12-04 18:33:11,286 : INFO : PROGRESS: at sentence #5360000, processed 57419455 words, keeping 59596 word types\n",
      "2019-12-04 18:33:11,418 : INFO : PROGRESS: at sentence #5370000, processed 57521046 words, keeping 59596 word types\n",
      "2019-12-04 18:33:11,549 : INFO : PROGRESS: at sentence #5380000, processed 57622688 words, keeping 59596 word types\n",
      "2019-12-04 18:33:11,705 : INFO : PROGRESS: at sentence #5390000, processed 57743698 words, keeping 59596 word types\n",
      "2019-12-04 18:33:11,857 : INFO : PROGRESS: at sentence #5400000, processed 57864226 words, keeping 59596 word types\n",
      "2019-12-04 18:33:12,001 : INFO : PROGRESS: at sentence #5410000, processed 57978751 words, keeping 59596 word types\n",
      "2019-12-04 18:33:12,144 : INFO : PROGRESS: at sentence #5420000, processed 58088968 words, keeping 59596 word types\n",
      "2019-12-04 18:33:12,280 : INFO : PROGRESS: at sentence #5430000, processed 58194618 words, keeping 59596 word types\n",
      "2019-12-04 18:33:12,417 : INFO : PROGRESS: at sentence #5440000, processed 58300937 words, keeping 59596 word types\n",
      "2019-12-04 18:33:12,556 : INFO : PROGRESS: at sentence #5450000, processed 58408054 words, keeping 59596 word types\n",
      "2019-12-04 18:33:12,695 : INFO : PROGRESS: at sentence #5460000, processed 58515510 words, keeping 59596 word types\n",
      "2019-12-04 18:33:12,834 : INFO : PROGRESS: at sentence #5470000, processed 58622430 words, keeping 59596 word types\n",
      "2019-12-04 18:33:12,974 : INFO : PROGRESS: at sentence #5480000, processed 58731092 words, keeping 59596 word types\n",
      "2019-12-04 18:33:13,114 : INFO : PROGRESS: at sentence #5490000, processed 58840936 words, keeping 59596 word types\n",
      "2019-12-04 18:33:13,255 : INFO : PROGRESS: at sentence #5500000, processed 58951649 words, keeping 59596 word types\n",
      "2019-12-04 18:33:13,403 : INFO : PROGRESS: at sentence #5510000, processed 59067912 words, keeping 59596 word types\n",
      "2019-12-04 18:33:13,546 : INFO : PROGRESS: at sentence #5520000, processed 59178208 words, keeping 59596 word types\n",
      "2019-12-04 18:33:13,694 : INFO : PROGRESS: at sentence #5530000, processed 59293196 words, keeping 59596 word types\n",
      "2019-12-04 18:33:13,836 : INFO : PROGRESS: at sentence #5540000, processed 59401667 words, keeping 59596 word types\n",
      "2019-12-04 18:33:14,000 : INFO : PROGRESS: at sentence #5550000, processed 59529187 words, keeping 59596 word types\n",
      "2019-12-04 18:33:14,154 : INFO : PROGRESS: at sentence #5560000, processed 59649962 words, keeping 59596 word types\n",
      "2019-12-04 18:33:14,311 : INFO : PROGRESS: at sentence #5570000, processed 59771729 words, keeping 59596 word types\n",
      "2019-12-04 18:33:14,461 : INFO : PROGRESS: at sentence #5580000, processed 59889609 words, keeping 59596 word types\n",
      "2019-12-04 18:33:14,613 : INFO : PROGRESS: at sentence #5590000, processed 60008875 words, keeping 59596 word types\n",
      "2019-12-04 18:33:14,742 : INFO : PROGRESS: at sentence #5600000, processed 60107560 words, keeping 59596 word types\n",
      "2019-12-04 18:33:14,872 : INFO : PROGRESS: at sentence #5610000, processed 60207853 words, keeping 59596 word types\n",
      "2019-12-04 18:33:15,003 : INFO : PROGRESS: at sentence #5620000, processed 60309571 words, keeping 59596 word types\n",
      "2019-12-04 18:33:15,137 : INFO : PROGRESS: at sentence #5630000, processed 60414832 words, keeping 59596 word types\n",
      "2019-12-04 18:33:15,260 : INFO : PROGRESS: at sentence #5640000, processed 60510261 words, keeping 59596 word types\n",
      "2019-12-04 18:33:15,388 : INFO : PROGRESS: at sentence #5650000, processed 60607348 words, keeping 59596 word types\n",
      "2019-12-04 18:33:15,517 : INFO : PROGRESS: at sentence #5660000, processed 60704663 words, keeping 59596 word types\n",
      "2019-12-04 18:33:15,643 : INFO : PROGRESS: at sentence #5670000, processed 60800772 words, keeping 59596 word types\n",
      "2019-12-04 18:33:15,771 : INFO : PROGRESS: at sentence #5680000, processed 60899852 words, keeping 59596 word types\n",
      "2019-12-04 18:33:15,907 : INFO : PROGRESS: at sentence #5690000, processed 61005885 words, keeping 59596 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:33:16,044 : INFO : PROGRESS: at sentence #5700000, processed 61111126 words, keeping 59596 word types\n",
      "2019-12-04 18:33:16,178 : INFO : PROGRESS: at sentence #5710000, processed 61215117 words, keeping 59596 word types\n",
      "2019-12-04 18:33:16,314 : INFO : PROGRESS: at sentence #5720000, processed 61320312 words, keeping 59596 word types\n",
      "2019-12-04 18:33:16,451 : INFO : PROGRESS: at sentence #5730000, processed 61426340 words, keeping 59596 word types\n",
      "2019-12-04 18:33:16,583 : INFO : PROGRESS: at sentence #5740000, processed 61526431 words, keeping 59596 word types\n",
      "2019-12-04 18:33:16,718 : INFO : PROGRESS: at sentence #5750000, processed 61629231 words, keeping 59596 word types\n",
      "2019-12-04 18:33:16,850 : INFO : PROGRESS: at sentence #5760000, processed 61731633 words, keeping 59596 word types\n",
      "2019-12-04 18:33:17,012 : INFO : PROGRESS: at sentence #5770000, processed 61862151 words, keeping 59596 word types\n",
      "2019-12-04 18:33:17,159 : INFO : PROGRESS: at sentence #5780000, processed 61977563 words, keeping 59596 word types\n",
      "2019-12-04 18:33:17,314 : INFO : PROGRESS: at sentence #5790000, processed 62089940 words, keeping 59596 word types\n",
      "2019-12-04 18:33:17,453 : INFO : PROGRESS: at sentence #5800000, processed 62197514 words, keeping 59596 word types\n",
      "2019-12-04 18:33:17,589 : INFO : PROGRESS: at sentence #5810000, processed 62303670 words, keeping 59596 word types\n",
      "2019-12-04 18:33:17,728 : INFO : PROGRESS: at sentence #5820000, processed 62411180 words, keeping 59596 word types\n",
      "2019-12-04 18:33:17,865 : INFO : PROGRESS: at sentence #5830000, processed 62517694 words, keeping 59596 word types\n",
      "2019-12-04 18:33:18,005 : INFO : PROGRESS: at sentence #5840000, processed 62625299 words, keeping 59596 word types\n",
      "2019-12-04 18:33:18,144 : INFO : PROGRESS: at sentence #5850000, processed 62732878 words, keeping 59596 word types\n",
      "2019-12-04 18:33:18,284 : INFO : PROGRESS: at sentence #5860000, processed 62841316 words, keeping 59596 word types\n",
      "2019-12-04 18:33:18,428 : INFO : PROGRESS: at sentence #5870000, processed 62951858 words, keeping 59596 word types\n",
      "2019-12-04 18:33:18,577 : INFO : PROGRESS: at sentence #5880000, processed 63066298 words, keeping 59596 word types\n",
      "2019-12-04 18:33:18,723 : INFO : PROGRESS: at sentence #5890000, processed 63178537 words, keeping 59596 word types\n",
      "2019-12-04 18:33:18,872 : INFO : PROGRESS: at sentence #5900000, processed 63290964 words, keeping 59596 word types\n",
      "2019-12-04 18:33:19,021 : INFO : PROGRESS: at sentence #5910000, processed 63404937 words, keeping 59596 word types\n",
      "2019-12-04 18:33:19,176 : INFO : PROGRESS: at sentence #5920000, processed 63523396 words, keeping 59596 word types\n",
      "2019-12-04 18:33:19,333 : INFO : PROGRESS: at sentence #5930000, processed 63643623 words, keeping 59596 word types\n",
      "2019-12-04 18:33:19,490 : INFO : PROGRESS: at sentence #5940000, processed 63764520 words, keeping 59596 word types\n",
      "2019-12-04 18:33:19,647 : INFO : PROGRESS: at sentence #5950000, processed 63887494 words, keeping 59596 word types\n",
      "2019-12-04 18:33:19,796 : INFO : PROGRESS: at sentence #5960000, processed 64005020 words, keeping 59596 word types\n",
      "2019-12-04 18:33:19,938 : INFO : PROGRESS: at sentence #5970000, processed 64114944 words, keeping 59596 word types\n",
      "2019-12-04 18:33:20,069 : INFO : PROGRESS: at sentence #5980000, processed 64214857 words, keeping 59596 word types\n",
      "2019-12-04 18:33:20,200 : INFO : PROGRESS: at sentence #5990000, processed 64316074 words, keeping 59596 word types\n",
      "2019-12-04 18:33:20,332 : INFO : PROGRESS: at sentence #6000000, processed 64418672 words, keeping 59596 word types\n",
      "2019-12-04 18:33:20,458 : INFO : PROGRESS: at sentence #6010000, processed 64516405 words, keeping 59596 word types\n",
      "2019-12-04 18:33:20,590 : INFO : PROGRESS: at sentence #6020000, processed 64618689 words, keeping 59596 word types\n",
      "2019-12-04 18:33:20,714 : INFO : PROGRESS: at sentence #6030000, processed 64712761 words, keeping 59596 word types\n",
      "2019-12-04 18:33:20,841 : INFO : PROGRESS: at sentence #6040000, processed 64809991 words, keeping 59596 word types\n",
      "2019-12-04 18:33:20,967 : INFO : PROGRESS: at sentence #6050000, processed 64907650 words, keeping 59596 word types\n",
      "2019-12-04 18:33:21,100 : INFO : PROGRESS: at sentence #6060000, processed 65009827 words, keeping 59596 word types\n",
      "2019-12-04 18:33:21,236 : INFO : PROGRESS: at sentence #6070000, processed 65115685 words, keeping 59596 word types\n",
      "2019-12-04 18:33:21,371 : INFO : PROGRESS: at sentence #6080000, processed 65219993 words, keeping 59596 word types\n",
      "2019-12-04 18:33:21,506 : INFO : PROGRESS: at sentence #6090000, processed 65324530 words, keeping 59596 word types\n",
      "2019-12-04 18:33:21,643 : INFO : PROGRESS: at sentence #6100000, processed 65430074 words, keeping 59596 word types\n",
      "2019-12-04 18:33:21,778 : INFO : PROGRESS: at sentence #6110000, processed 65533889 words, keeping 59596 word types\n",
      "2019-12-04 18:33:21,910 : INFO : PROGRESS: at sentence #6120000, processed 65634230 words, keeping 59596 word types\n",
      "2019-12-04 18:33:22,047 : INFO : PROGRESS: at sentence #6130000, processed 65737351 words, keeping 59596 word types\n",
      "2019-12-04 18:33:22,194 : INFO : PROGRESS: at sentence #6140000, processed 65851350 words, keeping 59596 word types\n",
      "2019-12-04 18:33:22,353 : INFO : PROGRESS: at sentence #6150000, processed 65975252 words, keeping 59596 word types\n",
      "2019-12-04 18:33:22,504 : INFO : PROGRESS: at sentence #6160000, processed 66091100 words, keeping 59596 word types\n",
      "2019-12-04 18:33:22,649 : INFO : PROGRESS: at sentence #6170000, processed 66202807 words, keeping 59596 word types\n",
      "2019-12-04 18:33:22,785 : INFO : PROGRESS: at sentence #6180000, processed 66306105 words, keeping 59596 word types\n",
      "2019-12-04 18:33:22,926 : INFO : PROGRESS: at sentence #6190000, processed 66412622 words, keeping 59596 word types\n",
      "2019-12-04 18:33:23,069 : INFO : PROGRESS: at sentence #6200000, processed 66520836 words, keeping 59596 word types\n",
      "2019-12-04 18:33:23,248 : INFO : PROGRESS: at sentence #6210000, processed 66628361 words, keeping 59596 word types\n",
      "2019-12-04 18:33:23,387 : INFO : PROGRESS: at sentence #6220000, processed 66735664 words, keeping 59596 word types\n",
      "2019-12-04 18:33:23,527 : INFO : PROGRESS: at sentence #6230000, processed 66843815 words, keeping 59596 word types\n",
      "2019-12-04 18:33:23,672 : INFO : PROGRESS: at sentence #6240000, processed 66953083 words, keeping 59596 word types\n",
      "2019-12-04 18:33:23,819 : INFO : PROGRESS: at sentence #6250000, processed 67063194 words, keeping 59596 word types\n",
      "2019-12-04 18:33:23,974 : INFO : PROGRESS: at sentence #6260000, processed 67179648 words, keeping 59596 word types\n",
      "2019-12-04 18:33:24,120 : INFO : PROGRESS: at sentence #6270000, processed 67289699 words, keeping 59596 word types\n",
      "2019-12-04 18:33:24,271 : INFO : PROGRESS: at sentence #6280000, processed 67403567 words, keeping 59596 word types\n",
      "2019-12-04 18:33:24,417 : INFO : PROGRESS: at sentence #6290000, processed 67514372 words, keeping 59596 word types\n",
      "2019-12-04 18:33:24,577 : INFO : PROGRESS: at sentence #6300000, processed 67638771 words, keeping 59596 word types\n",
      "2019-12-04 18:33:24,732 : INFO : PROGRESS: at sentence #6310000, processed 67758867 words, keeping 59596 word types\n",
      "2019-12-04 18:33:24,890 : INFO : PROGRESS: at sentence #6320000, processed 67880903 words, keeping 59596 word types\n",
      "2019-12-04 18:33:25,044 : INFO : PROGRESS: at sentence #6330000, processed 68001209 words, keeping 59596 word types\n",
      "2019-12-04 18:33:25,196 : INFO : PROGRESS: at sentence #6340000, processed 68119099 words, keeping 59596 word types\n",
      "2019-12-04 18:33:25,329 : INFO : PROGRESS: at sentence #6350000, processed 68222316 words, keeping 59596 word types\n",
      "2019-12-04 18:33:25,460 : INFO : PROGRESS: at sentence #6360000, processed 68321631 words, keeping 59596 word types\n",
      "2019-12-04 18:33:25,591 : INFO : PROGRESS: at sentence #6370000, processed 68423869 words, keeping 59596 word types\n",
      "2019-12-04 18:33:25,725 : INFO : PROGRESS: at sentence #6380000, processed 68528245 words, keeping 59596 word types\n",
      "2019-12-04 18:33:25,848 : INFO : PROGRESS: at sentence #6390000, processed 68623090 words, keeping 59596 word types\n",
      "2019-12-04 18:33:25,981 : INFO : PROGRESS: at sentence #6400000, processed 68723551 words, keeping 59596 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:33:26,105 : INFO : PROGRESS: at sentence #6410000, processed 68819001 words, keeping 59596 word types\n",
      "2019-12-04 18:33:26,232 : INFO : PROGRESS: at sentence #6420000, processed 68915854 words, keeping 59596 word types\n",
      "2019-12-04 18:33:26,360 : INFO : PROGRESS: at sentence #6430000, processed 69014992 words, keeping 59596 word types\n",
      "2019-12-04 18:33:26,495 : INFO : PROGRESS: at sentence #6440000, processed 69119632 words, keeping 59596 word types\n",
      "2019-12-04 18:33:26,632 : INFO : PROGRESS: at sentence #6450000, processed 69224468 words, keeping 59596 word types\n",
      "2019-12-04 18:33:26,767 : INFO : PROGRESS: at sentence #6460000, processed 69329300 words, keeping 59596 word types\n",
      "2019-12-04 18:33:26,902 : INFO : PROGRESS: at sentence #6470000, processed 69433780 words, keeping 59596 word types\n",
      "2019-12-04 18:33:27,040 : INFO : PROGRESS: at sentence #6480000, processed 69540342 words, keeping 59596 word types\n",
      "2019-12-04 18:33:27,172 : INFO : PROGRESS: at sentence #6490000, processed 69641243 words, keeping 59596 word types\n",
      "2019-12-04 18:33:27,307 : INFO : PROGRESS: at sentence #6500000, processed 69743134 words, keeping 59596 word types\n",
      "2019-12-04 18:33:27,442 : INFO : PROGRESS: at sentence #6510000, processed 69845329 words, keeping 59596 word types\n",
      "2019-12-04 18:33:27,599 : INFO : PROGRESS: at sentence #6520000, processed 69970162 words, keeping 59596 word types\n",
      "2019-12-04 18:33:27,750 : INFO : PROGRESS: at sentence #6530000, processed 70088690 words, keeping 59596 word types\n",
      "2019-12-04 18:33:27,894 : INFO : PROGRESS: at sentence #6540000, processed 70201889 words, keeping 59596 word types\n",
      "2019-12-04 18:33:28,035 : INFO : PROGRESS: at sentence #6550000, processed 70311134 words, keeping 59596 word types\n",
      "2019-12-04 18:33:28,172 : INFO : PROGRESS: at sentence #6560000, processed 70417358 words, keeping 59596 word types\n",
      "2019-12-04 18:33:28,312 : INFO : PROGRESS: at sentence #6570000, processed 70523958 words, keeping 59596 word types\n",
      "2019-12-04 18:33:28,451 : INFO : PROGRESS: at sentence #6580000, processed 70630782 words, keeping 59596 word types\n",
      "2019-12-04 18:33:28,592 : INFO : PROGRESS: at sentence #6590000, processed 70738177 words, keeping 59596 word types\n",
      "2019-12-04 18:33:28,732 : INFO : PROGRESS: at sentence #6600000, processed 70845475 words, keeping 59596 word types\n",
      "2019-12-04 18:33:28,872 : INFO : PROGRESS: at sentence #6610000, processed 70954071 words, keeping 59596 word types\n",
      "2019-12-04 18:33:29,018 : INFO : PROGRESS: at sentence #6620000, processed 71063977 words, keeping 59596 word types\n",
      "2019-12-04 18:33:29,162 : INFO : PROGRESS: at sentence #6630000, processed 71176120 words, keeping 59596 word types\n",
      "2019-12-04 18:33:29,309 : INFO : PROGRESS: at sentence #6640000, processed 71290303 words, keeping 59596 word types\n",
      "2019-12-04 18:33:29,455 : INFO : PROGRESS: at sentence #6650000, processed 71402397 words, keeping 59596 word types\n",
      "2019-12-04 18:33:29,606 : INFO : PROGRESS: at sentence #6660000, processed 71516976 words, keeping 59596 word types\n",
      "2019-12-04 18:33:29,755 : INFO : PROGRESS: at sentence #6670000, processed 71630113 words, keeping 59596 word types\n",
      "2019-12-04 18:33:29,915 : INFO : PROGRESS: at sentence #6680000, processed 71753845 words, keeping 59596 word types\n",
      "2019-12-04 18:33:30,071 : INFO : PROGRESS: at sentence #6690000, processed 71874321 words, keeping 59596 word types\n",
      "2019-12-04 18:33:30,228 : INFO : PROGRESS: at sentence #6700000, processed 71997182 words, keeping 59596 word types\n",
      "2019-12-04 18:33:30,375 : INFO : PROGRESS: at sentence #6710000, processed 72112998 words, keeping 59596 word types\n",
      "2019-12-04 18:33:30,525 : INFO : PROGRESS: at sentence #6720000, processed 72229869 words, keeping 59596 word types\n",
      "2019-12-04 18:33:30,654 : INFO : PROGRESS: at sentence #6730000, processed 72329357 words, keeping 59596 word types\n",
      "2019-12-04 18:33:30,785 : INFO : PROGRESS: at sentence #6740000, processed 72430204 words, keeping 59596 word types\n",
      "2019-12-04 18:33:30,918 : INFO : PROGRESS: at sentence #6750000, processed 72532454 words, keeping 59596 word types\n",
      "2019-12-04 18:33:31,053 : INFO : PROGRESS: at sentence #6760000, processed 72636991 words, keeping 59596 word types\n",
      "2019-12-04 18:33:31,177 : INFO : PROGRESS: at sentence #6770000, processed 72731882 words, keeping 59596 word types\n",
      "2019-12-04 18:33:31,303 : INFO : PROGRESS: at sentence #6780000, processed 72828343 words, keeping 59596 word types\n",
      "2019-12-04 18:33:31,433 : INFO : PROGRESS: at sentence #6790000, processed 72925703 words, keeping 59596 word types\n",
      "2019-12-04 18:33:31,561 : INFO : PROGRESS: at sentence #6800000, processed 73022529 words, keeping 59596 word types\n",
      "2019-12-04 18:33:31,693 : INFO : PROGRESS: at sentence #6810000, processed 73122759 words, keeping 59596 word types\n",
      "2019-12-04 18:33:31,830 : INFO : PROGRESS: at sentence #6820000, processed 73228891 words, keeping 59596 word types\n",
      "2019-12-04 18:33:31,969 : INFO : PROGRESS: at sentence #6830000, processed 73333775 words, keeping 59596 word types\n",
      "2019-12-04 18:33:32,104 : INFO : PROGRESS: at sentence #6840000, processed 73437561 words, keeping 59596 word types\n",
      "2019-12-04 18:33:32,240 : INFO : PROGRESS: at sentence #6850000, processed 73542602 words, keeping 59596 word types\n",
      "2019-12-04 18:33:32,376 : INFO : PROGRESS: at sentence #6860000, processed 73648550 words, keeping 59596 word types\n",
      "2019-12-04 18:33:32,507 : INFO : PROGRESS: at sentence #6870000, processed 73748489 words, keeping 59596 word types\n",
      "2019-12-04 18:33:32,643 : INFO : PROGRESS: at sentence #6880000, processed 73851638 words, keeping 59596 word types\n",
      "2019-12-04 18:33:32,781 : INFO : PROGRESS: at sentence #6890000, processed 73957959 words, keeping 59596 word types\n",
      "2019-12-04 18:33:32,942 : INFO : PROGRESS: at sentence #6900000, processed 74086165 words, keeping 59596 word types\n",
      "2019-12-04 18:33:33,089 : INFO : PROGRESS: at sentence #6910000, processed 74201269 words, keeping 59596 word types\n",
      "2019-12-04 18:33:33,231 : INFO : PROGRESS: at sentence #6920000, processed 74312979 words, keeping 59596 word types\n",
      "2019-12-04 18:33:33,370 : INFO : PROGRESS: at sentence #6930000, processed 74419881 words, keeping 59596 word types\n",
      "2019-12-04 18:33:33,506 : INFO : PROGRESS: at sentence #6940000, processed 74526001 words, keeping 59596 word types\n",
      "2019-12-04 18:33:33,644 : INFO : PROGRESS: at sentence #6950000, processed 74633981 words, keeping 59596 word types\n",
      "2019-12-04 18:33:33,782 : INFO : PROGRESS: at sentence #6960000, processed 74740659 words, keeping 59596 word types\n",
      "2019-12-04 18:33:33,922 : INFO : PROGRESS: at sentence #6970000, processed 74848319 words, keeping 59596 word types\n",
      "2019-12-04 18:33:34,063 : INFO : PROGRESS: at sentence #6980000, processed 74956499 words, keeping 59596 word types\n",
      "2019-12-04 18:33:34,203 : INFO : PROGRESS: at sentence #6990000, processed 75065042 words, keeping 59596 word types\n",
      "2019-12-04 18:33:34,346 : INFO : PROGRESS: at sentence #7000000, processed 75175120 words, keeping 59596 word types\n",
      "2019-12-04 18:33:34,495 : INFO : PROGRESS: at sentence #7010000, processed 75290889 words, keeping 59596 word types\n",
      "2019-12-04 18:33:34,640 : INFO : PROGRESS: at sentence #7020000, processed 75402050 words, keeping 59596 word types\n",
      "2019-12-04 18:33:34,787 : INFO : PROGRESS: at sentence #7030000, processed 75515120 words, keeping 59596 word types\n",
      "2019-12-04 18:33:34,933 : INFO : PROGRESS: at sentence #7040000, processed 75627691 words, keeping 59596 word types\n",
      "2019-12-04 18:33:35,091 : INFO : PROGRESS: at sentence #7050000, processed 75748500 words, keeping 59596 word types\n",
      "2019-12-04 18:33:35,247 : INFO : PROGRESS: at sentence #7060000, processed 75868702 words, keeping 59596 word types\n",
      "2019-12-04 18:33:35,402 : INFO : PROGRESS: at sentence #7070000, processed 75989733 words, keeping 59596 word types\n",
      "2019-12-04 18:33:35,559 : INFO : PROGRESS: at sentence #7080000, processed 76112640 words, keeping 59596 word types\n",
      "2019-12-04 18:33:35,709 : INFO : PROGRESS: at sentence #7090000, processed 76229395 words, keeping 59596 word types\n",
      "2019-12-04 18:33:35,847 : INFO : PROGRESS: at sentence #7100000, processed 76336853 words, keeping 59596 word types\n",
      "2019-12-04 18:33:35,977 : INFO : PROGRESS: at sentence #7110000, processed 76436111 words, keeping 59596 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:33:36,108 : INFO : PROGRESS: at sentence #7120000, processed 76538103 words, keeping 59596 word types\n",
      "2019-12-04 18:33:36,241 : INFO : PROGRESS: at sentence #7130000, processed 76641385 words, keeping 59596 word types\n",
      "2019-12-04 18:33:36,365 : INFO : PROGRESS: at sentence #7140000, processed 76739033 words, keeping 59596 word types\n",
      "2019-12-04 18:33:36,494 : INFO : PROGRESS: at sentence #7150000, processed 76839809 words, keeping 59596 word types\n",
      "2019-12-04 18:33:36,619 : INFO : PROGRESS: at sentence #7160000, processed 76934461 words, keeping 59596 word types\n",
      "2019-12-04 18:33:36,744 : INFO : PROGRESS: at sentence #7170000, processed 77031418 words, keeping 59596 word types\n",
      "2019-12-04 18:33:36,871 : INFO : PROGRESS: at sentence #7180000, processed 77129581 words, keeping 59596 word types\n",
      "2019-12-04 18:33:36,997 : INFO : PROGRESS: at sentence #7190000, processed 77232315 words, keeping 59596 word types\n",
      "2019-12-04 18:33:37,129 : INFO : PROGRESS: at sentence #7200000, processed 77338219 words, keeping 59596 word types\n",
      "2019-12-04 18:33:37,259 : INFO : PROGRESS: at sentence #7210000, processed 77442540 words, keeping 59596 word types\n",
      "2019-12-04 18:33:37,394 : INFO : PROGRESS: at sentence #7220000, processed 77547465 words, keeping 59596 word types\n",
      "2019-12-04 18:33:37,526 : INFO : PROGRESS: at sentence #7230000, processed 77652756 words, keeping 59596 word types\n",
      "2019-12-04 18:33:37,656 : INFO : PROGRESS: at sentence #7240000, processed 77755822 words, keeping 59596 word types\n",
      "2019-12-04 18:33:37,788 : INFO : PROGRESS: at sentence #7250000, processed 77857134 words, keeping 59596 word types\n",
      "2019-12-04 18:33:37,919 : INFO : PROGRESS: at sentence #7260000, processed 77959120 words, keeping 59596 word types\n",
      "2019-12-04 18:33:38,068 : INFO : PROGRESS: at sentence #7270000, processed 78077854 words, keeping 59596 word types\n",
      "2019-12-04 18:33:38,222 : INFO : PROGRESS: at sentence #7280000, processed 78199395 words, keeping 59596 word types\n",
      "2019-12-04 18:33:38,368 : INFO : PROGRESS: at sentence #7290000, processed 78314440 words, keeping 59596 word types\n",
      "2019-12-04 18:33:38,509 : INFO : PROGRESS: at sentence #7300000, processed 78424608 words, keeping 59596 word types\n",
      "2019-12-04 18:33:38,645 : INFO : PROGRESS: at sentence #7310000, processed 78530285 words, keeping 59596 word types\n",
      "2019-12-04 18:33:38,782 : INFO : PROGRESS: at sentence #7320000, processed 78635411 words, keeping 59596 word types\n",
      "2019-12-04 18:33:38,923 : INFO : PROGRESS: at sentence #7330000, processed 78743819 words, keeping 59596 word types\n",
      "2019-12-04 18:33:39,066 : INFO : PROGRESS: at sentence #7340000, processed 78851513 words, keeping 59596 word types\n",
      "2019-12-04 18:33:39,207 : INFO : PROGRESS: at sentence #7350000, processed 78958295 words, keeping 59596 word types\n",
      "2019-12-04 18:33:39,350 : INFO : PROGRESS: at sentence #7360000, processed 79066538 words, keeping 59596 word types\n",
      "2019-12-04 18:33:39,494 : INFO : PROGRESS: at sentence #7370000, processed 79176364 words, keeping 59596 word types\n",
      "2019-12-04 18:33:39,637 : INFO : PROGRESS: at sentence #7380000, processed 79287040 words, keeping 59596 word types\n",
      "2019-12-04 18:33:39,789 : INFO : PROGRESS: at sentence #7390000, processed 79403346 words, keeping 59596 word types\n",
      "2019-12-04 18:33:39,932 : INFO : PROGRESS: at sentence #7400000, processed 79513235 words, keeping 59596 word types\n",
      "2019-12-04 18:33:40,081 : INFO : PROGRESS: at sentence #7410000, processed 79628047 words, keeping 59596 word types\n",
      "2019-12-04 18:33:40,226 : INFO : PROGRESS: at sentence #7420000, processed 79737670 words, keeping 59596 word types\n",
      "2019-12-04 18:33:40,391 : INFO : PROGRESS: at sentence #7430000, processed 79863508 words, keeping 59596 word types\n",
      "2019-12-04 18:33:40,546 : INFO : PROGRESS: at sentence #7440000, processed 79984303 words, keeping 59596 word types\n",
      "2019-12-04 18:33:40,702 : INFO : PROGRESS: at sentence #7450000, processed 80106116 words, keeping 59596 word types\n",
      "2019-12-04 18:33:40,852 : INFO : PROGRESS: at sentence #7460000, processed 80224667 words, keeping 59596 word types\n",
      "2019-12-04 18:33:41,003 : INFO : PROGRESS: at sentence #7470000, processed 80343772 words, keeping 59596 word types\n",
      "2019-12-04 18:33:41,133 : INFO : PROGRESS: at sentence #7480000, processed 80443923 words, keeping 59596 word types\n",
      "2019-12-04 18:33:41,263 : INFO : PROGRESS: at sentence #7490000, processed 80543918 words, keeping 59596 word types\n",
      "2019-12-04 18:33:41,395 : INFO : PROGRESS: at sentence #7500000, processed 80645951 words, keeping 59596 word types\n",
      "2019-12-04 18:33:41,530 : INFO : PROGRESS: at sentence #7510000, processed 80750977 words, keeping 59596 word types\n",
      "2019-12-04 18:33:41,652 : INFO : PROGRESS: at sentence #7520000, processed 80846679 words, keeping 59596 word types\n",
      "2019-12-04 18:33:41,783 : INFO : PROGRESS: at sentence #7530000, processed 80944354 words, keeping 59596 word types\n",
      "2019-12-04 18:33:41,912 : INFO : PROGRESS: at sentence #7540000, processed 81041167 words, keeping 59596 word types\n",
      "2019-12-04 18:33:42,041 : INFO : PROGRESS: at sentence #7550000, processed 81137370 words, keeping 59596 word types\n",
      "2019-12-04 18:33:42,173 : INFO : PROGRESS: at sentence #7560000, processed 81236694 words, keeping 59596 word types\n",
      "2019-12-04 18:33:42,313 : INFO : PROGRESS: at sentence #7570000, processed 81342055 words, keeping 59596 word types\n",
      "2019-12-04 18:33:42,452 : INFO : PROGRESS: at sentence #7580000, processed 81447139 words, keeping 59596 word types\n",
      "2019-12-04 18:33:42,589 : INFO : PROGRESS: at sentence #7590000, processed 81551649 words, keeping 59596 word types\n",
      "2019-12-04 18:33:42,726 : INFO : PROGRESS: at sentence #7600000, processed 81656621 words, keeping 59596 word types\n",
      "2019-12-04 18:33:42,863 : INFO : PROGRESS: at sentence #7610000, processed 81762293 words, keeping 59596 word types\n",
      "2019-12-04 18:33:42,995 : INFO : PROGRESS: at sentence #7620000, processed 81863052 words, keeping 59596 word types\n",
      "2019-12-04 18:33:43,129 : INFO : PROGRESS: at sentence #7630000, processed 81965371 words, keeping 59596 word types\n",
      "2019-12-04 18:33:43,263 : INFO : PROGRESS: at sentence #7640000, processed 82067962 words, keeping 59596 word types\n",
      "2019-12-04 18:33:43,424 : INFO : PROGRESS: at sentence #7650000, processed 82196213 words, keeping 59596 word types\n",
      "2019-12-04 18:33:43,572 : INFO : PROGRESS: at sentence #7660000, processed 82312615 words, keeping 59596 word types\n",
      "2019-12-04 18:33:43,715 : INFO : PROGRESS: at sentence #7670000, processed 82425061 words, keeping 59596 word types\n",
      "2019-12-04 18:33:43,855 : INFO : PROGRESS: at sentence #7680000, processed 82533683 words, keeping 59596 word types\n",
      "2019-12-04 18:33:43,994 : INFO : PROGRESS: at sentence #7690000, processed 82639637 words, keeping 59596 word types\n",
      "2019-12-04 18:33:44,133 : INFO : PROGRESS: at sentence #7700000, processed 82746945 words, keeping 59596 word types\n",
      "2019-12-04 18:33:44,271 : INFO : PROGRESS: at sentence #7710000, processed 82853603 words, keeping 59596 word types\n",
      "2019-12-04 18:33:44,411 : INFO : PROGRESS: at sentence #7720000, processed 82961042 words, keeping 59596 word types\n",
      "2019-12-04 18:33:44,551 : INFO : PROGRESS: at sentence #7730000, processed 83068234 words, keeping 59596 word types\n",
      "2019-12-04 18:33:44,691 : INFO : PROGRESS: at sentence #7740000, processed 83176636 words, keeping 59596 word types\n",
      "2019-12-04 18:33:44,836 : INFO : PROGRESS: at sentence #7750000, processed 83287404 words, keeping 59596 word types\n",
      "2019-12-04 18:33:44,983 : INFO : PROGRESS: at sentence #7760000, processed 83400750 words, keeping 59596 word types\n",
      "2019-12-04 18:33:45,131 : INFO : PROGRESS: at sentence #7770000, processed 83514036 words, keeping 59596 word types\n",
      "2019-12-04 18:33:45,280 : INFO : PROGRESS: at sentence #7780000, processed 83626113 words, keeping 59596 word types\n",
      "2019-12-04 18:33:45,430 : INFO : PROGRESS: at sentence #7790000, processed 83740617 words, keeping 59596 word types\n",
      "2019-12-04 18:33:45,582 : INFO : PROGRESS: at sentence #7800000, processed 83857616 words, keeping 59596 word types\n",
      "2019-12-04 18:33:45,742 : INFO : PROGRESS: at sentence #7810000, processed 83978641 words, keeping 59596 word types\n",
      "2019-12-04 18:33:45,898 : INFO : PROGRESS: at sentence #7820000, processed 84099140 words, keeping 59596 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:33:46,057 : INFO : PROGRESS: at sentence #7830000, processed 84222132 words, keeping 59596 word types\n",
      "2019-12-04 18:33:46,205 : INFO : PROGRESS: at sentence #7840000, processed 84339168 words, keeping 59596 word types\n",
      "2019-12-04 18:33:46,349 : INFO : PROGRESS: at sentence #7850000, processed 84451518 words, keeping 59596 word types\n",
      "2019-12-04 18:33:46,478 : INFO : PROGRESS: at sentence #7860000, processed 84551073 words, keeping 59596 word types\n",
      "2019-12-04 18:33:46,610 : INFO : PROGRESS: at sentence #7870000, processed 84652323 words, keeping 59596 word types\n",
      "2019-12-04 18:33:46,744 : INFO : PROGRESS: at sentence #7880000, processed 84754827 words, keeping 59596 word types\n",
      "2019-12-04 18:33:46,871 : INFO : PROGRESS: at sentence #7890000, processed 84852775 words, keeping 59596 word types\n",
      "2019-12-04 18:33:47,003 : INFO : PROGRESS: at sentence #7900000, processed 84954611 words, keeping 59596 word types\n",
      "2019-12-04 18:33:47,129 : INFO : PROGRESS: at sentence #7910000, processed 85048737 words, keeping 59596 word types\n",
      "2019-12-04 18:33:47,258 : INFO : PROGRESS: at sentence #7920000, processed 85146957 words, keeping 59596 word types\n",
      "2019-12-04 18:33:47,386 : INFO : PROGRESS: at sentence #7930000, processed 85243929 words, keeping 59596 word types\n",
      "2019-12-04 18:33:47,518 : INFO : PROGRESS: at sentence #7940000, processed 85345463 words, keeping 59596 word types\n",
      "2019-12-04 18:33:47,656 : INFO : PROGRESS: at sentence #7950000, processed 85451772 words, keeping 59596 word types\n",
      "2019-12-04 18:33:47,791 : INFO : PROGRESS: at sentence #7960000, processed 85556010 words, keeping 59596 word types\n",
      "2019-12-04 18:33:47,927 : INFO : PROGRESS: at sentence #7970000, processed 85660501 words, keeping 59596 word types\n",
      "2019-12-04 18:33:48,065 : INFO : PROGRESS: at sentence #7980000, processed 85765823 words, keeping 59596 word types\n",
      "2019-12-04 18:33:48,201 : INFO : PROGRESS: at sentence #7990000, processed 85870480 words, keeping 59596 word types\n",
      "2019-12-04 18:33:48,333 : INFO : PROGRESS: at sentence #8000000, processed 85970405 words, keeping 59596 word types\n",
      "2019-12-04 18:33:48,468 : INFO : PROGRESS: at sentence #8010000, processed 86073461 words, keeping 59596 word types\n",
      "2019-12-04 18:33:48,618 : INFO : PROGRESS: at sentence #8020000, processed 86184413 words, keeping 59596 word types\n",
      "2019-12-04 18:33:48,777 : INFO : PROGRESS: at sentence #8030000, processed 86310388 words, keeping 59596 word types\n",
      "2019-12-04 18:33:48,926 : INFO : PROGRESS: at sentence #8040000, processed 86426395 words, keeping 59596 word types\n",
      "2019-12-04 18:33:49,070 : INFO : PROGRESS: at sentence #8050000, processed 86538494 words, keeping 59596 word types\n",
      "2019-12-04 18:33:49,205 : INFO : PROGRESS: at sentence #8060000, processed 86642101 words, keeping 59596 word types\n",
      "2019-12-04 18:33:49,343 : INFO : PROGRESS: at sentence #8070000, processed 86748529 words, keeping 59596 word types\n",
      "2019-12-04 18:33:49,485 : INFO : PROGRESS: at sentence #8080000, processed 86856561 words, keeping 59596 word types\n",
      "2019-12-04 18:33:49,625 : INFO : PROGRESS: at sentence #8090000, processed 86963961 words, keeping 59596 word types\n",
      "2019-12-04 18:33:49,767 : INFO : PROGRESS: at sentence #8100000, processed 87071367 words, keeping 59596 word types\n",
      "2019-12-04 18:33:49,907 : INFO : PROGRESS: at sentence #8110000, processed 87179699 words, keeping 59596 word types\n",
      "2019-12-04 18:33:50,048 : INFO : PROGRESS: at sentence #8120000, processed 87288376 words, keeping 59596 word types\n",
      "2019-12-04 18:33:50,190 : INFO : PROGRESS: at sentence #8130000, processed 87398836 words, keeping 59596 word types\n",
      "2019-12-04 18:33:50,343 : INFO : PROGRESS: at sentence #8140000, processed 87514979 words, keeping 59596 word types\n",
      "2019-12-04 18:33:50,490 : INFO : PROGRESS: at sentence #8150000, processed 87625243 words, keeping 59596 word types\n",
      "2019-12-04 18:33:50,643 : INFO : PROGRESS: at sentence #8160000, processed 87739676 words, keeping 59596 word types\n",
      "2019-12-04 18:33:50,793 : INFO : PROGRESS: at sentence #8170000, processed 87850604 words, keeping 59596 word types\n",
      "2019-12-04 18:33:50,954 : INFO : PROGRESS: at sentence #8180000, processed 87973336 words, keeping 59596 word types\n",
      "2019-12-04 18:33:51,112 : INFO : PROGRESS: at sentence #8190000, processed 88093662 words, keeping 59596 word types\n",
      "2019-12-04 18:33:51,272 : INFO : PROGRESS: at sentence #8200000, processed 88215099 words, keeping 59596 word types\n",
      "2019-12-04 18:33:51,427 : INFO : PROGRESS: at sentence #8210000, processed 88336324 words, keeping 59596 word types\n",
      "2019-12-04 18:33:51,578 : INFO : PROGRESS: at sentence #8220000, processed 88453933 words, keeping 59596 word types\n",
      "2019-12-04 18:33:51,716 : INFO : PROGRESS: at sentence #8230000, processed 88558851 words, keeping 59596 word types\n",
      "2019-12-04 18:33:51,844 : INFO : PROGRESS: at sentence #8240000, processed 88657783 words, keeping 59596 word types\n",
      "2019-12-04 18:33:51,977 : INFO : PROGRESS: at sentence #8250000, processed 88760160 words, keeping 59596 word types\n",
      "2019-12-04 18:33:52,110 : INFO : PROGRESS: at sentence #8260000, processed 88863815 words, keeping 59596 word types\n",
      "2019-12-04 18:33:52,234 : INFO : PROGRESS: at sentence #8270000, processed 88959790 words, keeping 59596 word types\n",
      "2019-12-04 18:33:52,366 : INFO : PROGRESS: at sentence #8280000, processed 89060700 words, keeping 59596 word types\n",
      "2019-12-04 18:33:52,492 : INFO : PROGRESS: at sentence #8290000, processed 89155912 words, keeping 59596 word types\n",
      "2019-12-04 18:33:52,620 : INFO : PROGRESS: at sentence #8300000, processed 89252413 words, keeping 59596 word types\n",
      "2019-12-04 18:33:52,751 : INFO : PROGRESS: at sentence #8310000, processed 89351526 words, keeping 59596 word types\n",
      "2019-12-04 18:33:52,887 : INFO : PROGRESS: at sentence #8320000, processed 89455405 words, keeping 59596 word types\n",
      "2019-12-04 18:33:53,023 : INFO : PROGRESS: at sentence #8330000, processed 89560650 words, keeping 59596 word types\n",
      "2019-12-04 18:33:53,159 : INFO : PROGRESS: at sentence #8340000, processed 89665019 words, keeping 59596 word types\n",
      "2019-12-04 18:33:53,295 : INFO : PROGRESS: at sentence #8350000, processed 89770119 words, keeping 59596 word types\n",
      "2019-12-04 18:33:53,432 : INFO : PROGRESS: at sentence #8360000, processed 89875730 words, keeping 59596 word types\n",
      "2019-12-04 18:33:53,565 : INFO : PROGRESS: at sentence #8370000, processed 89977557 words, keeping 59596 word types\n",
      "2019-12-04 18:33:53,701 : INFO : PROGRESS: at sentence #8380000, processed 90079243 words, keeping 59596 word types\n",
      "2019-12-04 18:33:53,835 : INFO : PROGRESS: at sentence #8390000, processed 90181183 words, keeping 59596 word types\n",
      "2019-12-04 18:33:53,991 : INFO : PROGRESS: at sentence #8400000, processed 90304065 words, keeping 59596 word types\n",
      "2019-12-04 18:33:54,143 : INFO : PROGRESS: at sentence #8410000, processed 90423751 words, keeping 59596 word types\n",
      "2019-12-04 18:33:54,289 : INFO : PROGRESS: at sentence #8420000, processed 90537454 words, keeping 59596 word types\n",
      "2019-12-04 18:33:54,430 : INFO : PROGRESS: at sentence #8430000, processed 90647265 words, keeping 59596 word types\n",
      "2019-12-04 18:33:54,568 : INFO : PROGRESS: at sentence #8440000, processed 90753140 words, keeping 59596 word types\n",
      "2019-12-04 18:33:54,707 : INFO : PROGRESS: at sentence #8450000, processed 90859676 words, keeping 59596 word types\n",
      "2019-12-04 18:33:54,848 : INFO : PROGRESS: at sentence #8460000, processed 90966424 words, keeping 59596 word types\n",
      "2019-12-04 18:33:54,997 : INFO : PROGRESS: at sentence #8470000, processed 91073989 words, keeping 59596 word types\n",
      "2019-12-04 18:33:55,141 : INFO : PROGRESS: at sentence #8480000, processed 91181032 words, keeping 59596 word types\n",
      "2019-12-04 18:33:55,288 : INFO : PROGRESS: at sentence #8490000, processed 91290039 words, keeping 59596 word types\n",
      "2019-12-04 18:33:55,431 : INFO : PROGRESS: at sentence #8500000, processed 91399725 words, keeping 59596 word types\n",
      "2019-12-04 18:33:55,575 : INFO : PROGRESS: at sentence #8510000, processed 91510624 words, keeping 59596 word types\n",
      "2019-12-04 18:33:55,725 : INFO : PROGRESS: at sentence #8520000, processed 91626233 words, keeping 59596 word types\n",
      "2019-12-04 18:33:55,871 : INFO : PROGRESS: at sentence #8530000, processed 91737545 words, keeping 59596 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:33:56,018 : INFO : PROGRESS: at sentence #8540000, processed 91851990 words, keeping 59596 word types\n",
      "2019-12-04 18:33:56,161 : INFO : PROGRESS: at sentence #8550000, processed 91962218 words, keeping 59596 word types\n",
      "2019-12-04 18:33:56,321 : INFO : PROGRESS: at sentence #8560000, processed 92088653 words, keeping 59596 word types\n",
      "2019-12-04 18:33:56,475 : INFO : PROGRESS: at sentence #8570000, processed 92209240 words, keeping 59596 word types\n",
      "2019-12-04 18:33:56,623 : INFO : PROGRESS: at sentence #8580000, processed 92331470 words, keeping 59596 word types\n",
      "2019-12-04 18:33:56,769 : INFO : PROGRESS: at sentence #8590000, processed 92448742 words, keeping 59596 word types\n",
      "2019-12-04 18:33:56,918 : INFO : PROGRESS: at sentence #8600000, processed 92566644 words, keeping 59596 word types\n",
      "2019-12-04 18:33:57,046 : INFO : PROGRESS: at sentence #8610000, processed 92665770 words, keeping 59596 word types\n",
      "2019-12-04 18:33:57,175 : INFO : PROGRESS: at sentence #8620000, processed 92766226 words, keeping 59596 word types\n",
      "2019-12-04 18:33:57,304 : INFO : PROGRESS: at sentence #8630000, processed 92868036 words, keeping 59596 word types\n",
      "2019-12-04 18:33:57,440 : INFO : PROGRESS: at sentence #8640000, processed 92973198 words, keeping 59596 word types\n",
      "2019-12-04 18:33:57,564 : INFO : PROGRESS: at sentence #8650000, processed 93068364 words, keeping 59596 word types\n",
      "2019-12-04 18:33:57,689 : INFO : PROGRESS: at sentence #8660000, processed 93164962 words, keeping 59596 word types\n",
      "2019-12-04 18:33:57,817 : INFO : PROGRESS: at sentence #8670000, processed 93262446 words, keeping 59596 word types\n",
      "2019-12-04 18:33:57,940 : INFO : PROGRESS: at sentence #8680000, processed 93358756 words, keeping 59596 word types\n",
      "2019-12-04 18:33:58,067 : INFO : PROGRESS: at sentence #8690000, processed 93458329 words, keeping 59596 word types\n",
      "2019-12-04 18:33:58,202 : INFO : PROGRESS: at sentence #8700000, processed 93564863 words, keeping 59596 word types\n",
      "2019-12-04 18:33:58,336 : INFO : PROGRESS: at sentence #8710000, processed 93669718 words, keeping 59596 word types\n",
      "2019-12-04 18:33:58,469 : INFO : PROGRESS: at sentence #8720000, processed 93773518 words, keeping 59596 word types\n",
      "2019-12-04 18:33:58,605 : INFO : PROGRESS: at sentence #8730000, processed 93878855 words, keeping 59596 word types\n",
      "2019-12-04 18:33:58,745 : INFO : PROGRESS: at sentence #8740000, processed 93984539 words, keeping 59596 word types\n",
      "2019-12-04 18:33:58,877 : INFO : PROGRESS: at sentence #8750000, processed 94084744 words, keeping 59596 word types\n",
      "2019-12-04 18:33:59,012 : INFO : PROGRESS: at sentence #8760000, processed 94187553 words, keeping 59596 word types\n",
      "2019-12-04 18:33:59,148 : INFO : PROGRESS: at sentence #8770000, processed 94291352 words, keeping 59596 word types\n",
      "2019-12-04 18:33:59,314 : INFO : PROGRESS: at sentence #8780000, processed 94421108 words, keeping 59596 word types\n",
      "2019-12-04 18:33:59,463 : INFO : PROGRESS: at sentence #8790000, processed 94536373 words, keeping 59596 word types\n",
      "2019-12-04 18:33:59,608 : INFO : PROGRESS: at sentence #8800000, processed 94648690 words, keeping 59596 word types\n",
      "2019-12-04 18:33:59,747 : INFO : PROGRESS: at sentence #8810000, processed 94755876 words, keeping 59596 word types\n",
      "2019-12-04 18:33:59,884 : INFO : PROGRESS: at sentence #8820000, processed 94862020 words, keeping 59596 word types\n",
      "2019-12-04 18:34:00,023 : INFO : PROGRESS: at sentence #8830000, processed 94969889 words, keeping 59596 word types\n",
      "2019-12-04 18:34:00,162 : INFO : PROGRESS: at sentence #8840000, processed 95076371 words, keeping 59596 word types\n",
      "2019-12-04 18:34:00,302 : INFO : PROGRESS: at sentence #8850000, processed 95184007 words, keeping 59596 word types\n",
      "2019-12-04 18:34:00,442 : INFO : PROGRESS: at sentence #8860000, processed 95291758 words, keeping 59596 word types\n",
      "2019-12-04 18:34:00,584 : INFO : PROGRESS: at sentence #8870000, processed 95400330 words, keeping 59596 word types\n",
      "2019-12-04 18:34:00,727 : INFO : PROGRESS: at sentence #8880000, processed 95510772 words, keeping 59596 word types\n",
      "2019-12-04 18:34:00,876 : INFO : PROGRESS: at sentence #8890000, processed 95625798 words, keeping 59596 word types\n",
      "2019-12-04 18:34:01,021 : INFO : PROGRESS: at sentence #8900000, processed 95737617 words, keeping 59596 word types\n",
      "2019-12-04 18:34:01,169 : INFO : PROGRESS: at sentence #8910000, processed 95850319 words, keeping 59596 word types\n",
      "2019-12-04 18:34:01,319 : INFO : PROGRESS: at sentence #8920000, processed 95963575 words, keeping 59596 word types\n",
      "2019-12-04 18:34:01,475 : INFO : PROGRESS: at sentence #8930000, processed 96083304 words, keeping 59596 word types\n",
      "2019-12-04 18:34:01,631 : INFO : PROGRESS: at sentence #8940000, processed 96203333 words, keeping 59596 word types\n",
      "2019-12-04 18:34:01,789 : INFO : PROGRESS: at sentence #8950000, processed 96324217 words, keeping 59596 word types\n",
      "2019-12-04 18:34:01,948 : INFO : PROGRESS: at sentence #8960000, processed 96447134 words, keeping 59596 word types\n",
      "2019-12-04 18:34:02,098 : INFO : PROGRESS: at sentence #8970000, processed 96564693 words, keeping 59596 word types\n",
      "2019-12-04 18:34:02,239 : INFO : PROGRESS: at sentence #8980000, processed 96673050 words, keeping 59596 word types\n",
      "2019-12-04 18:34:02,372 : INFO : PROGRESS: at sentence #8990000, processed 96773024 words, keeping 59596 word types\n",
      "2019-12-04 18:34:02,507 : INFO : PROGRESS: at sentence #9000000, processed 96874284 words, keeping 59596 word types\n",
      "2019-12-04 18:34:02,641 : INFO : PROGRESS: at sentence #9010000, processed 96977024 words, keeping 59596 word types\n",
      "2019-12-04 18:34:02,770 : INFO : PROGRESS: at sentence #9020000, processed 97074671 words, keeping 59596 word types\n",
      "2019-12-04 18:34:02,903 : INFO : PROGRESS: at sentence #9030000, processed 97176585 words, keeping 59596 word types\n",
      "2019-12-04 18:34:03,029 : INFO : PROGRESS: at sentence #9040000, processed 97271361 words, keeping 59596 word types\n",
      "2019-12-04 18:34:03,155 : INFO : PROGRESS: at sentence #9050000, processed 97368169 words, keeping 59596 word types\n",
      "2019-12-04 18:34:03,283 : INFO : PROGRESS: at sentence #9060000, processed 97466206 words, keeping 59596 word types\n",
      "2019-12-04 18:34:03,416 : INFO : PROGRESS: at sentence #9070000, processed 97568449 words, keeping 59596 word types\n",
      "2019-12-04 18:34:03,553 : INFO : PROGRESS: at sentence #9080000, processed 97673801 words, keeping 59596 word types\n",
      "2019-12-04 18:34:03,687 : INFO : PROGRESS: at sentence #9090000, processed 97778502 words, keeping 59596 word types\n",
      "2019-12-04 18:34:03,821 : INFO : PROGRESS: at sentence #9100000, processed 97883203 words, keeping 59596 word types\n",
      "2019-12-04 18:34:03,962 : INFO : PROGRESS: at sentence #9110000, processed 97988726 words, keeping 59596 word types\n",
      "2019-12-04 18:34:04,108 : INFO : PROGRESS: at sentence #9120000, processed 98092111 words, keeping 59596 word types\n",
      "2019-12-04 18:34:04,241 : INFO : PROGRESS: at sentence #9130000, processed 98192815 words, keeping 59596 word types\n",
      "2019-12-04 18:34:04,374 : INFO : PROGRESS: at sentence #9140000, processed 98295562 words, keeping 59596 word types\n",
      "2019-12-04 18:34:04,521 : INFO : PROGRESS: at sentence #9150000, processed 98411886 words, keeping 59596 word types\n",
      "2019-12-04 18:34:04,674 : INFO : PROGRESS: at sentence #9160000, processed 98534229 words, keeping 59596 word types\n",
      "2019-12-04 18:34:04,860 : INFO : PROGRESS: at sentence #9170000, processed 98649870 words, keeping 59596 word types\n",
      "2019-12-04 18:34:05,002 : INFO : PROGRESS: at sentence #9180000, processed 98760760 words, keeping 59596 word types\n",
      "2019-12-04 18:34:05,138 : INFO : PROGRESS: at sentence #9190000, processed 98864844 words, keeping 59596 word types\n",
      "2019-12-04 18:34:05,276 : INFO : PROGRESS: at sentence #9200000, processed 98971101 words, keeping 59596 word types\n",
      "2019-12-04 18:34:05,415 : INFO : PROGRESS: at sentence #9210000, processed 99079600 words, keeping 59596 word types\n",
      "2019-12-04 18:34:05,553 : INFO : PROGRESS: at sentence #9220000, processed 99187160 words, keeping 59596 word types\n",
      "2019-12-04 18:34:05,694 : INFO : PROGRESS: at sentence #9230000, processed 99294236 words, keeping 59596 word types\n",
      "2019-12-04 18:34:05,833 : INFO : PROGRESS: at sentence #9240000, processed 99402334 words, keeping 59596 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:34:05,973 : INFO : PROGRESS: at sentence #9250000, processed 99511804 words, keeping 59596 word types\n",
      "2019-12-04 18:34:06,114 : INFO : PROGRESS: at sentence #9260000, processed 99622176 words, keeping 59596 word types\n",
      "2019-12-04 18:34:06,262 : INFO : PROGRESS: at sentence #9270000, processed 99738602 words, keeping 59596 word types\n",
      "2019-12-04 18:34:06,407 : INFO : PROGRESS: at sentence #9280000, processed 99848477 words, keeping 59596 word types\n",
      "2019-12-04 18:34:06,557 : INFO : PROGRESS: at sentence #9290000, processed 99963006 words, keeping 59596 word types\n",
      "2019-12-04 18:34:06,694 : INFO : PROGRESS: at sentence #9300000, processed 100073159 words, keeping 59596 word types\n",
      "2019-12-04 18:34:06,845 : INFO : PROGRESS: at sentence #9310000, processed 100198449 words, keeping 59596 word types\n",
      "2019-12-04 18:34:06,991 : INFO : PROGRESS: at sentence #9320000, processed 100318684 words, keeping 59596 word types\n",
      "2019-12-04 18:34:07,142 : INFO : PROGRESS: at sentence #9330000, processed 100440496 words, keeping 59596 word types\n",
      "2019-12-04 18:34:07,285 : INFO : PROGRESS: at sentence #9340000, processed 100559970 words, keeping 59596 word types\n",
      "2019-12-04 18:34:07,428 : INFO : PROGRESS: at sentence #9350000, processed 100678852 words, keeping 59596 word types\n",
      "2019-12-04 18:34:07,555 : INFO : PROGRESS: at sentence #9360000, processed 100780389 words, keeping 59596 word types\n",
      "2019-12-04 18:34:07,677 : INFO : PROGRESS: at sentence #9370000, processed 100879903 words, keeping 59596 word types\n",
      "2019-12-04 18:34:07,799 : INFO : PROGRESS: at sentence #9380000, processed 100982019 words, keeping 59596 word types\n",
      "2019-12-04 18:34:07,926 : INFO : PROGRESS: at sentence #9390000, processed 101087012 words, keeping 59596 word types\n",
      "2019-12-04 18:34:08,039 : INFO : PROGRESS: at sentence #9400000, processed 101182544 words, keeping 59596 word types\n",
      "2019-12-04 18:34:08,158 : INFO : PROGRESS: at sentence #9410000, processed 101281559 words, keeping 59596 word types\n",
      "2019-12-04 18:34:08,276 : INFO : PROGRESS: at sentence #9420000, processed 101377687 words, keeping 59596 word types\n",
      "2019-12-04 18:34:08,394 : INFO : PROGRESS: at sentence #9430000, processed 101473786 words, keeping 59596 word types\n",
      "2019-12-04 18:34:08,520 : INFO : PROGRESS: at sentence #9440000, processed 101573238 words, keeping 59596 word types\n",
      "2019-12-04 18:34:08,651 : INFO : PROGRESS: at sentence #9450000, processed 101677973 words, keeping 59596 word types\n",
      "2019-12-04 18:34:08,781 : INFO : PROGRESS: at sentence #9460000, processed 101782962 words, keeping 59596 word types\n",
      "2019-12-04 18:34:08,911 : INFO : PROGRESS: at sentence #9470000, processed 101887725 words, keeping 59596 word types\n",
      "2019-12-04 18:34:09,044 : INFO : PROGRESS: at sentence #9480000, processed 101992313 words, keeping 59596 word types\n",
      "2019-12-04 18:34:09,180 : INFO : PROGRESS: at sentence #9490000, processed 102098834 words, keeping 59596 word types\n",
      "2019-12-04 18:34:09,307 : INFO : PROGRESS: at sentence #9500000, processed 102199524 words, keeping 59596 word types\n",
      "2019-12-04 18:34:09,435 : INFO : PROGRESS: at sentence #9510000, processed 102301513 words, keeping 59596 word types\n",
      "2019-12-04 18:34:09,562 : INFO : PROGRESS: at sentence #9520000, processed 102403742 words, keeping 59596 word types\n",
      "2019-12-04 18:34:09,718 : INFO : PROGRESS: at sentence #9530000, processed 102530417 words, keeping 59596 word types\n",
      "2019-12-04 18:34:09,863 : INFO : PROGRESS: at sentence #9540000, processed 102647760 words, keeping 59596 word types\n",
      "2019-12-04 18:34:10,002 : INFO : PROGRESS: at sentence #9550000, processed 102760525 words, keeping 59596 word types\n",
      "2019-12-04 18:34:10,138 : INFO : PROGRESS: at sentence #9560000, processed 102869763 words, keeping 59596 word types\n",
      "2019-12-04 18:34:10,272 : INFO : PROGRESS: at sentence #9570000, processed 102975639 words, keeping 59596 word types\n",
      "2019-12-04 18:34:10,408 : INFO : PROGRESS: at sentence #9580000, processed 103082591 words, keeping 59596 word types\n",
      "2019-12-04 18:34:10,541 : INFO : PROGRESS: at sentence #9590000, processed 103189386 words, keeping 59596 word types\n",
      "2019-12-04 18:34:10,678 : INFO : PROGRESS: at sentence #9600000, processed 103296689 words, keeping 59596 word types\n",
      "2019-12-04 18:34:10,818 : INFO : PROGRESS: at sentence #9610000, processed 103403976 words, keeping 59596 word types\n",
      "2019-12-04 18:34:10,957 : INFO : PROGRESS: at sentence #9620000, processed 103512893 words, keeping 59596 word types\n",
      "2019-12-04 18:34:11,097 : INFO : PROGRESS: at sentence #9630000, processed 103622667 words, keeping 59596 word types\n",
      "2019-12-04 18:34:11,243 : INFO : PROGRESS: at sentence #9640000, processed 103735439 words, keeping 59596 word types\n",
      "2019-12-04 18:34:11,389 : INFO : PROGRESS: at sentence #9650000, processed 103849384 words, keeping 59596 word types\n",
      "2019-12-04 18:34:11,532 : INFO : PROGRESS: at sentence #9660000, processed 103961504 words, keeping 59596 word types\n",
      "2019-12-04 18:34:11,678 : INFO : PROGRESS: at sentence #9670000, processed 104076190 words, keeping 59596 word types\n",
      "2019-12-04 18:34:11,832 : INFO : PROGRESS: at sentence #9680000, processed 104191351 words, keeping 59596 word types\n",
      "2019-12-04 18:34:11,989 : INFO : PROGRESS: at sentence #9690000, processed 104313294 words, keeping 59596 word types\n",
      "2019-12-04 18:34:12,140 : INFO : PROGRESS: at sentence #9700000, processed 104433645 words, keeping 59596 word types\n",
      "2019-12-04 18:34:12,297 : INFO : PROGRESS: at sentence #9710000, processed 104556991 words, keeping 59596 word types\n",
      "2019-12-04 18:34:12,438 : INFO : PROGRESS: at sentence #9720000, processed 104671579 words, keeping 59596 word types\n",
      "2019-12-04 18:34:12,587 : INFO : PROGRESS: at sentence #9730000, processed 104788032 words, keeping 59596 word types\n",
      "2019-12-04 18:34:12,717 : INFO : PROGRESS: at sentence #9740000, processed 104887445 words, keeping 59596 word types\n",
      "2019-12-04 18:34:12,849 : INFO : PROGRESS: at sentence #9750000, processed 104988366 words, keeping 59596 word types\n",
      "2019-12-04 18:34:12,984 : INFO : PROGRESS: at sentence #9760000, processed 105090994 words, keeping 59596 word types\n",
      "2019-12-04 18:34:13,114 : INFO : PROGRESS: at sentence #9770000, processed 105192697 words, keeping 59596 word types\n",
      "2019-12-04 18:34:13,239 : INFO : PROGRESS: at sentence #9780000, processed 105290492 words, keeping 59596 word types\n",
      "2019-12-04 18:34:13,362 : INFO : PROGRESS: at sentence #9790000, processed 105385752 words, keeping 59596 word types\n",
      "2019-12-04 18:34:13,489 : INFO : PROGRESS: at sentence #9800000, processed 105483615 words, keeping 59596 word types\n",
      "2019-12-04 18:34:13,616 : INFO : PROGRESS: at sentence #9810000, processed 105580552 words, keeping 59596 word types\n",
      "2019-12-04 18:34:13,747 : INFO : PROGRESS: at sentence #9820000, processed 105681441 words, keeping 59596 word types\n",
      "2019-12-04 18:34:13,884 : INFO : PROGRESS: at sentence #9830000, processed 105787640 words, keeping 59596 word types\n",
      "2019-12-04 18:34:14,018 : INFO : PROGRESS: at sentence #9840000, processed 105892448 words, keeping 59596 word types\n",
      "2019-12-04 18:34:14,152 : INFO : PROGRESS: at sentence #9850000, processed 105996265 words, keeping 59596 word types\n",
      "2019-12-04 18:34:14,287 : INFO : PROGRESS: at sentence #9860000, processed 106101222 words, keeping 59596 word types\n",
      "2019-12-04 18:34:14,422 : INFO : PROGRESS: at sentence #9870000, processed 106206788 words, keeping 59596 word types\n",
      "2019-12-04 18:34:14,551 : INFO : PROGRESS: at sentence #9880000, processed 106306833 words, keeping 59596 word types\n",
      "2019-12-04 18:34:14,685 : INFO : PROGRESS: at sentence #9890000, processed 106409642 words, keeping 59596 word types\n",
      "2019-12-04 18:34:14,825 : INFO : PROGRESS: at sentence #9900000, processed 106518407 words, keeping 59596 word types\n",
      "2019-12-04 18:34:14,982 : INFO : PROGRESS: at sentence #9910000, processed 106645322 words, keeping 59596 word types\n",
      "2019-12-04 18:34:15,128 : INFO : PROGRESS: at sentence #9920000, processed 106761448 words, keeping 59596 word types\n",
      "2019-12-04 18:34:15,270 : INFO : PROGRESS: at sentence #9930000, processed 106871876 words, keeping 59596 word types\n",
      "2019-12-04 18:34:15,405 : INFO : PROGRESS: at sentence #9940000, processed 106978152 words, keeping 59596 word types\n",
      "2019-12-04 18:34:15,540 : INFO : PROGRESS: at sentence #9950000, processed 107084324 words, keeping 59596 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:34:15,679 : INFO : PROGRESS: at sentence #9960000, processed 107192592 words, keeping 59596 word types\n",
      "2019-12-04 18:34:15,819 : INFO : PROGRESS: at sentence #9970000, processed 107299582 words, keeping 59596 word types\n",
      "2019-12-04 18:34:15,958 : INFO : PROGRESS: at sentence #9980000, processed 107407130 words, keeping 59596 word types\n",
      "2019-12-04 18:34:16,099 : INFO : PROGRESS: at sentence #9990000, processed 107515468 words, keeping 59596 word types\n",
      "2019-12-04 18:34:16,241 : INFO : PROGRESS: at sentence #10000000, processed 107623817 words, keeping 59596 word types\n",
      "2019-12-04 18:34:16,382 : INFO : PROGRESS: at sentence #10010000, processed 107734247 words, keeping 59596 word types\n",
      "2019-12-04 18:34:16,531 : INFO : PROGRESS: at sentence #10020000, processed 107850271 words, keeping 59596 word types\n",
      "2019-12-04 18:34:16,677 : INFO : PROGRESS: at sentence #10030000, processed 107960640 words, keeping 59596 word types\n",
      "2019-12-04 18:34:16,826 : INFO : PROGRESS: at sentence #10040000, processed 108074344 words, keeping 59596 word types\n",
      "2019-12-04 18:34:16,970 : INFO : PROGRESS: at sentence #10050000, processed 108186342 words, keeping 59596 word types\n",
      "2019-12-04 18:34:17,129 : INFO : PROGRESS: at sentence #10060000, processed 108308085 words, keeping 59596 word types\n",
      "2019-12-04 18:34:17,284 : INFO : PROGRESS: at sentence #10070000, processed 108428066 words, keeping 59596 word types\n",
      "2019-12-04 18:34:17,442 : INFO : PROGRESS: at sentence #10080000, processed 108549285 words, keeping 59596 word types\n",
      "2019-12-04 18:34:17,597 : INFO : PROGRESS: at sentence #10090000, processed 108671604 words, keeping 59596 word types\n",
      "2019-12-04 18:34:17,748 : INFO : PROGRESS: at sentence #10100000, processed 108788475 words, keeping 59596 word types\n",
      "2019-12-04 18:34:17,887 : INFO : PROGRESS: at sentence #10110000, processed 108895254 words, keeping 59596 word types\n",
      "2019-12-04 18:34:18,018 : INFO : PROGRESS: at sentence #10120000, processed 108994150 words, keeping 59596 word types\n",
      "2019-12-04 18:34:18,152 : INFO : PROGRESS: at sentence #10130000, processed 109096527 words, keeping 59596 word types\n",
      "2019-12-04 18:34:18,290 : INFO : PROGRESS: at sentence #10140000, processed 109199984 words, keeping 59596 word types\n",
      "2019-12-04 18:34:18,420 : INFO : PROGRESS: at sentence #10150000, processed 109298142 words, keeping 59596 word types\n",
      "2019-12-04 18:34:18,554 : INFO : PROGRESS: at sentence #10160000, processed 109397492 words, keeping 59596 word types\n",
      "2019-12-04 18:34:18,680 : INFO : PROGRESS: at sentence #10170000, processed 109492507 words, keeping 59596 word types\n",
      "2019-12-04 18:34:18,808 : INFO : PROGRESS: at sentence #10180000, processed 109589304 words, keeping 59596 word types\n",
      "2019-12-04 18:34:18,939 : INFO : PROGRESS: at sentence #10190000, processed 109687748 words, keeping 59596 word types\n",
      "2019-12-04 18:34:19,072 : INFO : PROGRESS: at sentence #10200000, processed 109790742 words, keeping 59596 word types\n",
      "2019-12-04 18:34:19,208 : INFO : PROGRESS: at sentence #10210000, processed 109896636 words, keeping 59596 word types\n",
      "2019-12-04 18:34:19,343 : INFO : PROGRESS: at sentence #10220000, processed 110000917 words, keeping 59596 word types\n",
      "2019-12-04 18:34:19,479 : INFO : PROGRESS: at sentence #10230000, processed 110106140 words, keeping 59596 word types\n",
      "2019-12-04 18:34:19,618 : INFO : PROGRESS: at sentence #10240000, processed 110211652 words, keeping 59596 word types\n",
      "2019-12-04 18:34:19,753 : INFO : PROGRESS: at sentence #10250000, processed 110313948 words, keeping 59596 word types\n",
      "2019-12-04 18:34:19,888 : INFO : PROGRESS: at sentence #10260000, processed 110415630 words, keeping 59596 word types\n",
      "2019-12-04 18:34:20,023 : INFO : PROGRESS: at sentence #10270000, processed 110517334 words, keeping 59596 word types\n",
      "2019-12-04 18:34:20,177 : INFO : PROGRESS: at sentence #10280000, processed 110637782 words, keeping 59596 word types\n",
      "2019-12-04 18:34:20,330 : INFO : PROGRESS: at sentence #10290000, processed 110758445 words, keeping 59596 word types\n",
      "2019-12-04 18:34:20,478 : INFO : PROGRESS: at sentence #10300000, processed 110873248 words, keeping 59596 word types\n",
      "2019-12-04 18:34:20,627 : INFO : PROGRESS: at sentence #10310000, processed 110983465 words, keeping 59596 word types\n",
      "2019-12-04 18:34:20,768 : INFO : PROGRESS: at sentence #10320000, processed 111089129 words, keeping 59596 word types\n",
      "2019-12-04 18:34:20,911 : INFO : PROGRESS: at sentence #10330000, processed 111195075 words, keeping 59596 word types\n",
      "2019-12-04 18:34:21,051 : INFO : PROGRESS: at sentence #10340000, processed 111302546 words, keeping 59596 word types\n",
      "2019-12-04 18:34:21,194 : INFO : PROGRESS: at sentence #10350000, processed 111409978 words, keeping 59596 word types\n",
      "2019-12-04 18:34:21,338 : INFO : PROGRESS: at sentence #10360000, processed 111516779 words, keeping 59596 word types\n",
      "2019-12-04 18:34:21,482 : INFO : PROGRESS: at sentence #10370000, processed 111625484 words, keeping 59596 word types\n",
      "2019-12-04 18:34:21,621 : INFO : PROGRESS: at sentence #10380000, processed 111735269 words, keeping 59596 word types\n",
      "2019-12-04 18:34:21,766 : INFO : PROGRESS: at sentence #10390000, processed 111846071 words, keeping 59596 word types\n",
      "2019-12-04 18:34:21,946 : INFO : PROGRESS: at sentence #10400000, processed 111962435 words, keeping 59596 word types\n",
      "2019-12-04 18:34:22,093 : INFO : PROGRESS: at sentence #10410000, processed 112072480 words, keeping 59596 word types\n",
      "2019-12-04 18:34:22,243 : INFO : PROGRESS: at sentence #10420000, processed 112187549 words, keeping 59596 word types\n",
      "2019-12-04 18:34:22,391 : INFO : PROGRESS: at sentence #10430000, processed 112296198 words, keeping 59596 word types\n",
      "2019-12-04 18:34:22,557 : INFO : PROGRESS: at sentence #10440000, processed 112423324 words, keeping 59596 word types\n",
      "2019-12-04 18:34:22,715 : INFO : PROGRESS: at sentence #10450000, processed 112544169 words, keeping 59596 word types\n",
      "2019-12-04 18:34:22,875 : INFO : PROGRESS: at sentence #10460000, processed 112665953 words, keeping 59596 word types\n",
      "2019-12-04 18:34:23,026 : INFO : PROGRESS: at sentence #10470000, processed 112783850 words, keeping 59596 word types\n",
      "2019-12-04 18:34:23,184 : INFO : PROGRESS: at sentence #10480000, processed 112903108 words, keeping 59596 word types\n",
      "2019-12-04 18:34:23,317 : INFO : PROGRESS: at sentence #10490000, processed 113002139 words, keeping 59596 word types\n",
      "2019-12-04 18:34:23,452 : INFO : PROGRESS: at sentence #10500000, processed 113102509 words, keeping 59596 word types\n",
      "2019-12-04 18:34:23,594 : INFO : PROGRESS: at sentence #10510000, processed 113204110 words, keeping 59596 word types\n",
      "2019-12-04 18:34:23,751 : INFO : PROGRESS: at sentence #10520000, processed 113309443 words, keeping 59596 word types\n",
      "2019-12-04 18:34:23,876 : INFO : PROGRESS: at sentence #10530000, processed 113404737 words, keeping 59596 word types\n",
      "2019-12-04 18:34:24,012 : INFO : PROGRESS: at sentence #10540000, processed 113502009 words, keeping 59596 word types\n",
      "2019-12-04 18:34:24,150 : INFO : PROGRESS: at sentence #10550000, processed 113599249 words, keeping 59596 word types\n",
      "2019-12-04 18:34:24,284 : INFO : PROGRESS: at sentence #10560000, processed 113695420 words, keeping 59596 word types\n",
      "2019-12-04 18:34:24,419 : INFO : PROGRESS: at sentence #10570000, processed 113794506 words, keeping 59596 word types\n",
      "2019-12-04 18:34:24,555 : INFO : PROGRESS: at sentence #10580000, processed 113900413 words, keeping 59596 word types\n",
      "2019-12-04 18:34:24,692 : INFO : PROGRESS: at sentence #10590000, processed 114005722 words, keeping 59596 word types\n",
      "2019-12-04 18:34:24,829 : INFO : PROGRESS: at sentence #10600000, processed 114109697 words, keeping 59596 word types\n",
      "2019-12-04 18:34:24,967 : INFO : PROGRESS: at sentence #10610000, processed 114214930 words, keeping 59596 word types\n",
      "2019-12-04 18:34:25,107 : INFO : PROGRESS: at sentence #10620000, processed 114320866 words, keeping 59596 word types\n",
      "2019-12-04 18:34:25,256 : INFO : PROGRESS: at sentence #10630000, processed 114421022 words, keeping 59596 word types\n",
      "2019-12-04 18:34:25,395 : INFO : PROGRESS: at sentence #10640000, processed 114523615 words, keeping 59596 word types\n",
      "2019-12-04 18:34:25,532 : INFO : PROGRESS: at sentence #10650000, processed 114625993 words, keeping 59596 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:34:25,703 : INFO : PROGRESS: at sentence #10660000, processed 114756485 words, keeping 59596 word types\n",
      "2019-12-04 18:34:25,855 : INFO : PROGRESS: at sentence #10670000, processed 114871793 words, keeping 59596 word types\n",
      "2019-12-04 18:34:26,003 : INFO : PROGRESS: at sentence #10680000, processed 114984188 words, keeping 59596 word types\n",
      "2019-12-04 18:34:26,144 : INFO : PROGRESS: at sentence #10690000, processed 115091952 words, keeping 59596 word types\n",
      "2019-12-04 18:34:26,285 : INFO : PROGRESS: at sentence #10700000, processed 115198121 words, keeping 59596 word types\n",
      "2019-12-04 18:34:26,429 : INFO : PROGRESS: at sentence #10710000, processed 115305663 words, keeping 59596 word types\n",
      "2019-12-04 18:34:26,570 : INFO : PROGRESS: at sentence #10720000, processed 115412163 words, keeping 59596 word types\n",
      "2019-12-04 18:34:26,713 : INFO : PROGRESS: at sentence #10730000, processed 115519690 words, keeping 59596 word types\n",
      "2019-12-04 18:34:26,854 : INFO : PROGRESS: at sentence #10740000, processed 115627188 words, keeping 59596 word types\n",
      "2019-12-04 18:34:26,997 : INFO : PROGRESS: at sentence #10750000, processed 115735757 words, keeping 59596 word types\n",
      "2019-12-04 18:34:27,143 : INFO : PROGRESS: at sentence #10760000, processed 115846278 words, keeping 59596 word types\n",
      "2019-12-04 18:34:27,292 : INFO : PROGRESS: at sentence #10770000, processed 115960256 words, keeping 59596 word types\n",
      "2019-12-04 18:34:27,441 : INFO : PROGRESS: at sentence #10780000, processed 116073015 words, keeping 59596 word types\n",
      "2019-12-04 18:34:27,592 : INFO : PROGRESS: at sentence #10790000, processed 116185426 words, keeping 59596 word types\n",
      "2019-12-04 18:34:27,749 : INFO : PROGRESS: at sentence #10800000, processed 116299358 words, keeping 59596 word types\n",
      "2019-12-04 18:34:27,908 : INFO : PROGRESS: at sentence #10810000, processed 116417556 words, keeping 59596 word types\n",
      "2019-12-04 18:34:28,069 : INFO : PROGRESS: at sentence #10820000, processed 116537899 words, keeping 59596 word types\n",
      "2019-12-04 18:34:28,229 : INFO : PROGRESS: at sentence #10830000, processed 116658658 words, keeping 59596 word types\n",
      "2019-12-04 18:34:28,399 : INFO : PROGRESS: at sentence #10840000, processed 116781862 words, keeping 59596 word types\n",
      "2019-12-04 18:34:28,553 : INFO : PROGRESS: at sentence #10850000, processed 116899229 words, keeping 59596 word types\n",
      "2019-12-04 18:34:28,697 : INFO : PROGRESS: at sentence #10860000, processed 117009504 words, keeping 59596 word types\n",
      "2019-12-04 18:34:28,831 : INFO : PROGRESS: at sentence #10870000, processed 117109505 words, keeping 59596 word types\n",
      "2019-12-04 18:34:28,964 : INFO : PROGRESS: at sentence #10880000, processed 117210704 words, keeping 59596 word types\n",
      "2019-12-04 18:34:29,099 : INFO : PROGRESS: at sentence #10890000, processed 117313311 words, keeping 59596 word types\n",
      "2019-12-04 18:34:29,228 : INFO : PROGRESS: at sentence #10900000, processed 117410696 words, keeping 59596 word types\n",
      "2019-12-04 18:34:29,366 : INFO : PROGRESS: at sentence #10910000, processed 117513258 words, keeping 59596 word types\n",
      "2019-12-04 18:34:29,494 : INFO : PROGRESS: at sentence #10920000, processed 117607291 words, keeping 59596 word types\n",
      "2019-12-04 18:34:29,624 : INFO : PROGRESS: at sentence #10930000, processed 117704685 words, keeping 59596 word types\n",
      "2019-12-04 18:34:29,753 : INFO : PROGRESS: at sentence #10940000, processed 117802257 words, keeping 59596 word types\n",
      "2019-12-04 18:34:29,885 : INFO : PROGRESS: at sentence #10950000, processed 117904300 words, keeping 59596 word types\n",
      "2019-12-04 18:34:30,022 : INFO : PROGRESS: at sentence #10960000, processed 118010332 words, keeping 59596 word types\n",
      "2019-12-04 18:34:30,158 : INFO : PROGRESS: at sentence #10970000, processed 118114475 words, keeping 59596 word types\n",
      "2019-12-04 18:34:30,294 : INFO : PROGRESS: at sentence #10980000, processed 118219007 words, keeping 59596 word types\n",
      "2019-12-04 18:34:30,430 : INFO : PROGRESS: at sentence #10990000, processed 118324556 words, keeping 59596 word types\n",
      "2019-12-04 18:34:30,566 : INFO : PROGRESS: at sentence #11000000, processed 118428505 words, keeping 59596 word types\n",
      "2019-12-04 18:34:30,702 : INFO : PROGRESS: at sentence #11010000, processed 118528723 words, keeping 59596 word types\n",
      "2019-12-04 18:34:30,838 : INFO : PROGRESS: at sentence #11020000, processed 118631740 words, keeping 59596 word types\n",
      "2019-12-04 18:34:30,988 : INFO : PROGRESS: at sentence #11030000, processed 118745447 words, keeping 59596 word types\n",
      "2019-12-04 18:34:31,149 : INFO : PROGRESS: at sentence #11040000, processed 118869605 words, keeping 59596 word types\n",
      "2019-12-04 18:34:31,300 : INFO : PROGRESS: at sentence #11050000, processed 118985491 words, keeping 59596 word types\n",
      "2019-12-04 18:34:31,446 : INFO : PROGRESS: at sentence #11060000, processed 119097261 words, keeping 59596 word types\n",
      "2019-12-04 18:34:31,586 : INFO : PROGRESS: at sentence #11070000, processed 119200526 words, keeping 59596 word types\n",
      "2019-12-04 18:34:31,728 : INFO : PROGRESS: at sentence #11080000, processed 119307074 words, keeping 59596 word types\n",
      "2019-12-04 18:34:31,871 : INFO : PROGRESS: at sentence #11090000, processed 119415201 words, keeping 59596 word types\n",
      "2019-12-04 18:34:32,016 : INFO : PROGRESS: at sentence #11100000, processed 119522738 words, keeping 59596 word types\n",
      "2019-12-04 18:34:32,161 : INFO : PROGRESS: at sentence #11110000, processed 119630104 words, keeping 59596 word types\n",
      "2019-12-04 18:34:32,307 : INFO : PROGRESS: at sentence #11120000, processed 119738153 words, keeping 59596 word types\n",
      "2019-12-04 18:34:32,451 : INFO : PROGRESS: at sentence #11130000, processed 119847515 words, keeping 59596 word types\n",
      "2019-12-04 18:34:32,594 : INFO : PROGRESS: at sentence #11140000, processed 119957461 words, keeping 59596 word types\n",
      "2019-12-04 18:34:32,747 : INFO : PROGRESS: at sentence #11150000, processed 120073962 words, keeping 59596 word types\n",
      "2019-12-04 18:34:32,893 : INFO : PROGRESS: at sentence #11160000, processed 120184031 words, keeping 59596 word types\n",
      "2019-12-04 18:34:33,041 : INFO : PROGRESS: at sentence #11170000, processed 120298051 words, keeping 59596 word types\n",
      "2019-12-04 18:34:33,186 : INFO : PROGRESS: at sentence #11180000, processed 120408880 words, keeping 59596 word types\n",
      "2019-12-04 18:34:33,346 : INFO : PROGRESS: at sentence #11190000, processed 120532947 words, keeping 59596 word types\n",
      "2019-12-04 18:34:33,501 : INFO : PROGRESS: at sentence #11200000, processed 120653094 words, keeping 59596 word types\n",
      "2019-12-04 18:34:33,657 : INFO : PROGRESS: at sentence #11210000, processed 120774801 words, keeping 59596 word types\n",
      "2019-12-04 18:34:33,812 : INFO : PROGRESS: at sentence #11220000, processed 120895433 words, keeping 59596 word types\n",
      "2019-12-04 18:34:33,965 : INFO : PROGRESS: at sentence #11230000, processed 121013191 words, keeping 59596 word types\n",
      "2019-12-04 18:34:34,106 : INFO : PROGRESS: at sentence #11240000, processed 121116866 words, keeping 59596 word types\n",
      "2019-12-04 18:34:34,238 : INFO : PROGRESS: at sentence #11250000, processed 121216185 words, keeping 59596 word types\n",
      "2019-12-04 18:34:34,397 : INFO : PROGRESS: at sentence #11260000, processed 121318466 words, keeping 59596 word types\n",
      "2019-12-04 18:34:34,535 : INFO : PROGRESS: at sentence #11270000, processed 121422723 words, keeping 59596 word types\n",
      "2019-12-04 18:34:34,664 : INFO : PROGRESS: at sentence #11280000, processed 121518050 words, keeping 59596 word types\n",
      "2019-12-04 18:34:34,800 : INFO : PROGRESS: at sentence #11290000, processed 121618336 words, keeping 59596 word types\n",
      "2019-12-04 18:34:34,928 : INFO : PROGRESS: at sentence #11300000, processed 121713639 words, keeping 59596 word types\n",
      "2019-12-04 18:34:35,060 : INFO : PROGRESS: at sentence #11310000, processed 121810773 words, keeping 59596 word types\n",
      "2019-12-04 18:34:35,189 : INFO : PROGRESS: at sentence #11320000, processed 121909524 words, keeping 59596 word types\n",
      "2019-12-04 18:34:35,332 : INFO : PROGRESS: at sentence #11330000, processed 122014172 words, keeping 59596 word types\n",
      "2019-12-04 18:34:35,471 : INFO : PROGRESS: at sentence #11340000, processed 122118903 words, keeping 59596 word types\n",
      "2019-12-04 18:34:35,609 : INFO : PROGRESS: at sentence #11350000, processed 122223808 words, keeping 59596 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:34:35,745 : INFO : PROGRESS: at sentence #11360000, processed 122328280 words, keeping 59596 word types\n",
      "2019-12-04 18:34:35,886 : INFO : PROGRESS: at sentence #11370000, processed 122434700 words, keeping 59596 word types\n",
      "2019-12-04 18:34:36,020 : INFO : PROGRESS: at sentence #11380000, processed 122535876 words, keeping 59596 word types\n",
      "2019-12-04 18:34:36,155 : INFO : PROGRESS: at sentence #11390000, processed 122637675 words, keeping 59596 word types\n",
      "2019-12-04 18:34:36,291 : INFO : PROGRESS: at sentence #11400000, processed 122739812 words, keeping 59596 word types\n",
      "2019-12-04 18:34:36,450 : INFO : PROGRESS: at sentence #11410000, processed 122864284 words, keeping 59596 word types\n",
      "2019-12-04 18:34:36,603 : INFO : PROGRESS: at sentence #11420000, processed 122983008 words, keeping 59596 word types\n",
      "2019-12-04 18:34:36,749 : INFO : PROGRESS: at sentence #11430000, processed 123096270 words, keeping 59596 word types\n",
      "2019-12-04 18:34:36,889 : INFO : PROGRESS: at sentence #11440000, processed 123205662 words, keeping 59596 word types\n",
      "2019-12-04 18:34:37,026 : INFO : PROGRESS: at sentence #11450000, processed 123311754 words, keeping 59596 word types\n",
      "2019-12-04 18:34:37,167 : INFO : PROGRESS: at sentence #11460000, processed 123418438 words, keeping 59596 word types\n",
      "2019-12-04 18:34:37,308 : INFO : PROGRESS: at sentence #11470000, processed 123525176 words, keeping 59596 word types\n",
      "2019-12-04 18:34:37,448 : INFO : PROGRESS: at sentence #11480000, processed 123632622 words, keeping 59596 word types\n",
      "2019-12-04 18:34:37,586 : INFO : PROGRESS: at sentence #11490000, processed 123739869 words, keeping 59596 word types\n",
      "2019-12-04 18:34:37,727 : INFO : PROGRESS: at sentence #11500000, processed 123848526 words, keeping 59596 word types\n",
      "2019-12-04 18:34:37,869 : INFO : PROGRESS: at sentence #11510000, processed 123958463 words, keeping 59596 word types\n",
      "2019-12-04 18:34:38,013 : INFO : PROGRESS: at sentence #11520000, processed 124070202 words, keeping 59596 word types\n",
      "2019-12-04 18:34:38,161 : INFO : PROGRESS: at sentence #11530000, processed 124184665 words, keeping 59596 word types\n",
      "2019-12-04 18:34:38,307 : INFO : PROGRESS: at sentence #11540000, processed 124296646 words, keeping 59596 word types\n",
      "2019-12-04 18:34:38,457 : INFO : PROGRESS: at sentence #11550000, processed 124411326 words, keeping 59596 word types\n",
      "2019-12-04 18:34:38,606 : INFO : PROGRESS: at sentence #11560000, processed 124524011 words, keeping 59596 word types\n",
      "2019-12-04 18:34:38,766 : INFO : PROGRESS: at sentence #11570000, processed 124648153 words, keeping 59596 word types\n",
      "2019-12-04 18:34:38,923 : INFO : PROGRESS: at sentence #11580000, processed 124768547 words, keeping 59596 word types\n",
      "2019-12-04 18:34:39,080 : INFO : PROGRESS: at sentence #11590000, processed 124891385 words, keeping 59596 word types\n",
      "2019-12-04 18:34:39,226 : INFO : PROGRESS: at sentence #11600000, processed 125007309 words, keeping 59596 word types\n",
      "2019-12-04 18:34:39,376 : INFO : PROGRESS: at sentence #11610000, processed 125124666 words, keeping 59596 word types\n",
      "2019-12-04 18:34:39,504 : INFO : PROGRESS: at sentence #11620000, processed 125223907 words, keeping 59596 word types\n",
      "2019-12-04 18:34:39,634 : INFO : PROGRESS: at sentence #11630000, processed 125324875 words, keeping 59596 word types\n",
      "2019-12-04 18:34:39,766 : INFO : PROGRESS: at sentence #11640000, processed 125426700 words, keeping 59596 word types\n",
      "2019-12-04 18:34:39,900 : INFO : PROGRESS: at sentence #11650000, processed 125531693 words, keeping 59596 word types\n",
      "2019-12-04 18:34:40,022 : INFO : PROGRESS: at sentence #11660000, processed 125626457 words, keeping 59596 word types\n",
      "2019-12-04 18:34:40,147 : INFO : PROGRESS: at sentence #11670000, processed 125722999 words, keeping 59596 word types\n",
      "2019-12-04 18:34:40,271 : INFO : PROGRESS: at sentence #11680000, processed 125820386 words, keeping 59596 word types\n",
      "2019-12-04 18:34:40,398 : INFO : PROGRESS: at sentence #11690000, processed 125917050 words, keeping 59596 word types\n",
      "2019-12-04 18:34:40,528 : INFO : PROGRESS: at sentence #11700000, processed 126017245 words, keeping 59596 word types\n",
      "2019-12-04 18:34:40,665 : INFO : PROGRESS: at sentence #11710000, processed 126123365 words, keeping 59596 word types\n",
      "2019-12-04 18:34:40,801 : INFO : PROGRESS: at sentence #11720000, processed 126228261 words, keeping 59596 word types\n",
      "2019-12-04 18:34:40,935 : INFO : PROGRESS: at sentence #11730000, processed 126332046 words, keeping 59596 word types\n",
      "2019-12-04 18:34:41,071 : INFO : PROGRESS: at sentence #11740000, processed 126437061 words, keeping 59596 word types\n",
      "2019-12-04 18:34:41,209 : INFO : PROGRESS: at sentence #11750000, processed 126543059 words, keeping 59596 word types\n",
      "2019-12-04 18:34:41,342 : INFO : PROGRESS: at sentence #11760000, processed 126643045 words, keeping 59596 word types\n",
      "2019-12-04 18:34:41,475 : INFO : PROGRESS: at sentence #11770000, processed 126746109 words, keeping 59596 word types\n",
      "2019-12-04 18:34:41,614 : INFO : PROGRESS: at sentence #11780000, processed 126851807 words, keeping 59596 word types\n",
      "2019-12-04 18:34:41,777 : INFO : PROGRESS: at sentence #11790000, processed 126980512 words, keeping 59596 word types\n",
      "2019-12-04 18:34:41,925 : INFO : PROGRESS: at sentence #11800000, processed 127095646 words, keeping 59596 word types\n",
      "2019-12-04 18:34:42,072 : INFO : PROGRESS: at sentence #11810000, processed 127207410 words, keeping 59596 word types\n",
      "2019-12-04 18:34:42,211 : INFO : PROGRESS: at sentence #11820000, processed 127314382 words, keeping 59596 word types\n",
      "2019-12-04 18:34:42,350 : INFO : PROGRESS: at sentence #11830000, processed 127420557 words, keeping 59596 word types\n",
      "2019-12-04 18:34:42,491 : INFO : PROGRESS: at sentence #11840000, processed 127528544 words, keeping 59596 word types\n",
      "2019-12-04 18:34:42,630 : INFO : PROGRESS: at sentence #11850000, processed 127635111 words, keeping 59596 word types\n",
      "2019-12-04 18:34:42,771 : INFO : PROGRESS: at sentence #11860000, processed 127742718 words, keeping 59596 word types\n",
      "2019-12-04 18:34:42,912 : INFO : PROGRESS: at sentence #11870000, processed 127850657 words, keeping 59596 word types\n",
      "2019-12-04 18:34:43,052 : INFO : PROGRESS: at sentence #11880000, processed 127959426 words, keeping 59596 word types\n",
      "2019-12-04 18:34:43,193 : INFO : PROGRESS: at sentence #11890000, processed 128069463 words, keeping 59596 word types\n",
      "2019-12-04 18:34:43,341 : INFO : PROGRESS: at sentence #11900000, processed 128185239 words, keeping 59596 word types\n",
      "2019-12-04 18:34:43,487 : INFO : PROGRESS: at sentence #11910000, processed 128296610 words, keeping 59596 word types\n",
      "2019-12-04 18:34:43,633 : INFO : PROGRESS: at sentence #11920000, processed 128409387 words, keeping 59596 word types\n",
      "2019-12-04 18:34:43,780 : INFO : PROGRESS: at sentence #11930000, processed 128522219 words, keeping 59596 word types\n",
      "2019-12-04 18:34:43,936 : INFO : PROGRESS: at sentence #11940000, processed 128642777 words, keeping 59596 word types\n",
      "2019-12-04 18:34:44,091 : INFO : PROGRESS: at sentence #11950000, processed 128762932 words, keeping 59596 word types\n",
      "2019-12-04 18:34:44,246 : INFO : PROGRESS: at sentence #11960000, processed 128883795 words, keeping 59596 word types\n",
      "2019-12-04 18:34:44,403 : INFO : PROGRESS: at sentence #11970000, processed 129007097 words, keeping 59596 word types\n",
      "2019-12-04 18:34:44,551 : INFO : PROGRESS: at sentence #11980000, processed 129123688 words, keeping 59596 word types\n",
      "2019-12-04 18:34:44,691 : INFO : PROGRESS: at sentence #11990000, processed 129231424 words, keeping 59596 word types\n",
      "2019-12-04 18:34:44,821 : INFO : PROGRESS: at sentence #12000000, processed 129330852 words, keeping 59596 word types\n",
      "2019-12-04 18:34:44,954 : INFO : PROGRESS: at sentence #12010000, processed 129432694 words, keeping 59596 word types\n",
      "2019-12-04 18:34:45,093 : INFO : PROGRESS: at sentence #12020000, processed 129535707 words, keeping 59596 word types\n",
      "2019-12-04 18:34:45,220 : INFO : PROGRESS: at sentence #12030000, processed 129633251 words, keeping 59596 word types\n",
      "2019-12-04 18:34:45,350 : INFO : PROGRESS: at sentence #12040000, processed 129734406 words, keeping 59596 word types\n",
      "2019-12-04 18:34:45,475 : INFO : PROGRESS: at sentence #12050000, processed 129829053 words, keeping 59596 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:34:45,603 : INFO : PROGRESS: at sentence #12060000, processed 129926087 words, keeping 59596 word types\n",
      "2019-12-04 18:34:45,731 : INFO : PROGRESS: at sentence #12070000, processed 130024222 words, keeping 59596 word types\n",
      "2019-12-04 18:34:45,862 : INFO : PROGRESS: at sentence #12080000, processed 130126779 words, keeping 59596 word types\n",
      "2019-12-04 18:34:45,998 : INFO : PROGRESS: at sentence #12090000, processed 130232669 words, keeping 59596 word types\n",
      "2019-12-04 18:34:46,131 : INFO : PROGRESS: at sentence #12100000, processed 130337201 words, keeping 59596 word types\n",
      "2019-12-04 18:34:46,267 : INFO : PROGRESS: at sentence #12110000, processed 130441795 words, keeping 59596 word types\n",
      "2019-12-04 18:34:46,401 : INFO : PROGRESS: at sentence #12120000, processed 130547090 words, keeping 59596 word types\n",
      "2019-12-04 18:34:46,535 : INFO : PROGRESS: at sentence #12130000, processed 130650381 words, keeping 59596 word types\n",
      "2019-12-04 18:34:46,667 : INFO : PROGRESS: at sentence #12140000, processed 130751484 words, keeping 59596 word types\n",
      "2019-12-04 18:34:46,800 : INFO : PROGRESS: at sentence #12150000, processed 130853624 words, keeping 59596 word types\n",
      "2019-12-04 18:34:46,950 : INFO : PROGRESS: at sentence #12160000, processed 130971843 words, keeping 59596 word types\n",
      "2019-12-04 18:34:47,104 : INFO : PROGRESS: at sentence #12170000, processed 131093700 words, keeping 59596 word types\n",
      "2019-12-04 18:34:47,250 : INFO : PROGRESS: at sentence #12180000, processed 131208866 words, keeping 59596 word types\n",
      "2019-12-04 18:34:47,393 : INFO : PROGRESS: at sentence #12190000, processed 131319200 words, keeping 59596 word types\n",
      "2019-12-04 18:34:47,529 : INFO : PROGRESS: at sentence #12200000, processed 131424614 words, keeping 59596 word types\n",
      "2019-12-04 18:34:47,667 : INFO : PROGRESS: at sentence #12210000, processed 131529790 words, keeping 59596 word types\n",
      "2019-12-04 18:34:47,807 : INFO : PROGRESS: at sentence #12220000, processed 131638268 words, keeping 59596 word types\n",
      "2019-12-04 18:34:47,944 : INFO : PROGRESS: at sentence #12230000, processed 131745915 words, keeping 59596 word types\n",
      "2019-12-04 18:34:48,082 : INFO : PROGRESS: at sentence #12240000, processed 131852723 words, keeping 59596 word types\n",
      "2019-12-04 18:34:48,224 : INFO : PROGRESS: at sentence #12250000, processed 131960922 words, keeping 59596 word types\n",
      "2019-12-04 18:34:48,366 : INFO : PROGRESS: at sentence #12260000, processed 132070652 words, keeping 59596 word types\n",
      "2019-12-04 18:34:48,508 : INFO : PROGRESS: at sentence #12270000, processed 132181263 words, keeping 59596 word types\n",
      "2019-12-04 18:34:48,658 : INFO : PROGRESS: at sentence #12280000, processed 132297561 words, keeping 59596 word types\n",
      "2019-12-04 18:34:48,803 : INFO : PROGRESS: at sentence #12290000, processed 132407563 words, keeping 59596 word types\n",
      "2019-12-04 18:34:48,952 : INFO : PROGRESS: at sentence #12300000, processed 132522244 words, keeping 59596 word types\n",
      "2019-12-04 18:34:49,097 : INFO : PROGRESS: at sentence #12310000, processed 132632100 words, keeping 59596 word types\n",
      "2019-12-04 18:34:49,258 : INFO : PROGRESS: at sentence #12320000, processed 132757857 words, keeping 59596 word types\n",
      "2019-12-04 18:34:49,414 : INFO : PROGRESS: at sentence #12330000, processed 132878307 words, keeping 59596 word types\n",
      "2019-12-04 18:34:49,570 : INFO : PROGRESS: at sentence #12340000, processed 133000417 words, keeping 59596 word types\n",
      "2019-12-04 18:34:49,724 : INFO : PROGRESS: at sentence #12350000, processed 133118917 words, keeping 59596 word types\n",
      "2019-12-04 18:34:49,875 : INFO : PROGRESS: at sentence #12360000, processed 133238077 words, keeping 59596 word types\n",
      "2019-12-04 18:34:50,005 : INFO : PROGRESS: at sentence #12370000, processed 133338535 words, keeping 59596 word types\n",
      "2019-12-04 18:34:50,134 : INFO : PROGRESS: at sentence #12380000, processed 133438359 words, keeping 59596 word types\n",
      "2019-12-04 18:34:50,263 : INFO : PROGRESS: at sentence #12390000, processed 133540382 words, keeping 59596 word types\n",
      "2019-12-04 18:34:50,396 : INFO : PROGRESS: at sentence #12400000, processed 133645401 words, keeping 59596 word types\n",
      "2019-12-04 18:34:50,519 : INFO : PROGRESS: at sentence #12410000, processed 133741255 words, keeping 59596 word types\n",
      "2019-12-04 18:34:50,648 : INFO : PROGRESS: at sentence #12420000, processed 133838982 words, keeping 59596 word types\n",
      "2019-12-04 18:34:50,774 : INFO : PROGRESS: at sentence #12430000, processed 133935728 words, keeping 59596 word types\n",
      "2019-12-04 18:34:50,902 : INFO : PROGRESS: at sentence #12440000, processed 134031824 words, keeping 59596 word types\n",
      "2019-12-04 18:34:51,031 : INFO : PROGRESS: at sentence #12450000, processed 134131334 words, keeping 59596 word types\n",
      "2019-12-04 18:34:51,166 : INFO : PROGRESS: at sentence #12460000, processed 134236612 words, keeping 59596 word types\n",
      "2019-12-04 18:34:51,296 : INFO : PROGRESS: at sentence #12470000, processed 134341645 words, keeping 59596 word types\n",
      "2019-12-04 18:34:51,425 : INFO : PROGRESS: at sentence #12480000, processed 134446129 words, keeping 59596 word types\n",
      "2019-12-04 18:34:51,557 : INFO : PROGRESS: at sentence #12490000, processed 134551038 words, keeping 59596 word types\n",
      "2019-12-04 18:34:51,691 : INFO : PROGRESS: at sentence #12500000, processed 134656971 words, keeping 59596 word types\n",
      "2019-12-04 18:34:51,822 : INFO : PROGRESS: at sentence #12510000, processed 134757644 words, keeping 59596 word types\n",
      "2019-12-04 18:34:51,955 : INFO : PROGRESS: at sentence #12520000, processed 134859996 words, keeping 59596 word types\n",
      "2019-12-04 18:34:52,087 : INFO : PROGRESS: at sentence #12530000, processed 134962491 words, keeping 59596 word types\n",
      "2019-12-04 18:34:52,246 : INFO : PROGRESS: at sentence #12540000, processed 135090483 words, keeping 59596 word types\n",
      "2019-12-04 18:34:52,392 : INFO : PROGRESS: at sentence #12550000, processed 135207051 words, keeping 59596 word types\n",
      "2019-12-04 18:34:52,535 : INFO : PROGRESS: at sentence #12560000, processed 135319479 words, keeping 59596 word types\n",
      "2019-12-04 18:34:52,675 : INFO : PROGRESS: at sentence #12570000, processed 135428107 words, keeping 59596 word types\n",
      "2019-12-04 18:34:52,812 : INFO : PROGRESS: at sentence #12580000, processed 135534095 words, keeping 59596 word types\n",
      "2019-12-04 18:34:52,950 : INFO : PROGRESS: at sentence #12590000, processed 135641333 words, keeping 59596 word types\n",
      "2019-12-04 18:34:53,088 : INFO : PROGRESS: at sentence #12600000, processed 135748087 words, keeping 59596 word types\n",
      "2019-12-04 18:34:53,228 : INFO : PROGRESS: at sentence #12610000, processed 135855546 words, keeping 59596 word types\n",
      "2019-12-04 18:34:53,367 : INFO : PROGRESS: at sentence #12620000, processed 135962539 words, keeping 59596 word types\n",
      "2019-12-04 18:34:53,507 : INFO : PROGRESS: at sentence #12630000, processed 136071024 words, keeping 59596 word types\n",
      "2019-12-04 18:34:53,649 : INFO : PROGRESS: at sentence #12640000, processed 136181711 words, keeping 59596 word types\n",
      "2019-12-04 18:34:53,796 : INFO : PROGRESS: at sentence #12650000, processed 136294946 words, keeping 59596 word types\n",
      "2019-12-04 18:34:53,943 : INFO : PROGRESS: at sentence #12660000, processed 136408046 words, keeping 59596 word types\n",
      "2019-12-04 18:34:54,090 : INFO : PROGRESS: at sentence #12670000, processed 136520594 words, keeping 59596 word types\n",
      "2019-12-04 18:34:54,238 : INFO : PROGRESS: at sentence #12680000, processed 136635078 words, keeping 59596 word types\n",
      "2019-12-04 18:34:54,390 : INFO : PROGRESS: at sentence #12690000, processed 136751792 words, keeping 59596 word types\n",
      "2019-12-04 18:34:54,546 : INFO : PROGRESS: at sentence #12700000, processed 136872718 words, keeping 59596 word types\n",
      "2019-12-04 18:34:54,699 : INFO : PROGRESS: at sentence #12710000, processed 136993378 words, keeping 59596 word types\n",
      "2019-12-04 18:34:54,857 : INFO : PROGRESS: at sentence #12720000, processed 137116539 words, keeping 59596 word types\n",
      "2019-12-04 18:34:55,005 : INFO : PROGRESS: at sentence #12730000, processed 137233175 words, keeping 59596 word types\n",
      "2019-12-04 18:34:55,148 : INFO : PROGRESS: at sentence #12740000, processed 137346171 words, keeping 59596 word types\n",
      "2019-12-04 18:34:55,276 : INFO : PROGRESS: at sentence #12750000, processed 137445522 words, keeping 59596 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:34:55,405 : INFO : PROGRESS: at sentence #12760000, processed 137546904 words, keeping 59596 word types\n",
      "2019-12-04 18:34:55,537 : INFO : PROGRESS: at sentence #12770000, processed 137649371 words, keeping 59596 word types\n",
      "2019-12-04 18:34:55,665 : INFO : PROGRESS: at sentence #12780000, processed 137747675 words, keeping 59596 word types\n",
      "2019-12-04 18:34:55,798 : INFO : PROGRESS: at sentence #12790000, processed 137849189 words, keeping 59596 word types\n",
      "2019-12-04 18:34:55,922 : INFO : PROGRESS: at sentence #12800000, processed 137943365 words, keeping 59596 word types\n",
      "2019-12-04 18:34:56,054 : INFO : PROGRESS: at sentence #12810000, processed 138041511 words, keeping 59596 word types\n",
      "2019-12-04 18:34:56,184 : INFO : PROGRESS: at sentence #12820000, processed 138138644 words, keeping 59596 word types\n",
      "2019-12-04 18:34:56,319 : INFO : PROGRESS: at sentence #12830000, processed 138239936 words, keeping 59596 word types\n",
      "2019-12-04 18:34:56,457 : INFO : PROGRESS: at sentence #12840000, processed 138346043 words, keeping 59596 word types\n",
      "2019-12-04 18:34:56,594 : INFO : PROGRESS: at sentence #12850000, processed 138450489 words, keeping 59596 word types\n",
      "2019-12-04 18:34:56,730 : INFO : PROGRESS: at sentence #12860000, processed 138555035 words, keeping 59596 word types\n",
      "2019-12-04 18:34:56,867 : INFO : PROGRESS: at sentence #12870000, processed 138659987 words, keeping 59596 word types\n",
      "2019-12-04 18:34:57,004 : INFO : PROGRESS: at sentence #12880000, processed 138765189 words, keeping 59596 word types\n",
      "2019-12-04 18:34:57,137 : INFO : PROGRESS: at sentence #12890000, processed 138865085 words, keeping 59596 word types\n",
      "2019-12-04 18:34:57,272 : INFO : PROGRESS: at sentence #12900000, processed 138968102 words, keeping 59596 word types\n",
      "2019-12-04 18:34:57,416 : INFO : PROGRESS: at sentence #12910000, processed 139078415 words, keeping 59596 word types\n",
      "2019-12-04 18:34:57,587 : INFO : PROGRESS: at sentence #12920000, processed 139204802 words, keeping 59596 word types\n",
      "2019-12-04 18:34:57,736 : INFO : PROGRESS: at sentence #12930000, processed 139320679 words, keeping 59596 word types\n",
      "2019-12-04 18:34:57,881 : INFO : PROGRESS: at sentence #12940000, processed 139433130 words, keeping 59596 word types\n",
      "2019-12-04 18:34:58,016 : INFO : PROGRESS: at sentence #12950000, processed 139536718 words, keeping 59596 word types\n",
      "2019-12-04 18:34:58,154 : INFO : PROGRESS: at sentence #12960000, processed 139642887 words, keeping 59596 word types\n",
      "2019-12-04 18:34:58,295 : INFO : PROGRESS: at sentence #12970000, processed 139751076 words, keeping 59596 word types\n",
      "2019-12-04 18:34:58,435 : INFO : PROGRESS: at sentence #12980000, processed 139858482 words, keeping 59596 word types\n",
      "2019-12-04 18:34:58,576 : INFO : PROGRESS: at sentence #12990000, processed 139965730 words, keeping 59596 word types\n",
      "2019-12-04 18:34:58,718 : INFO : PROGRESS: at sentence #13000000, processed 140074126 words, keeping 59596 word types\n",
      "2019-12-04 18:34:58,859 : INFO : PROGRESS: at sentence #13010000, processed 140182829 words, keeping 59596 word types\n",
      "2019-12-04 18:34:59,002 : INFO : PROGRESS: at sentence #13020000, processed 140293189 words, keeping 59596 word types\n",
      "2019-12-04 18:34:59,157 : INFO : PROGRESS: at sentence #13030000, processed 140409511 words, keeping 59596 word types\n",
      "2019-12-04 18:34:59,305 : INFO : PROGRESS: at sentence #13040000, processed 140519540 words, keeping 59596 word types\n",
      "2019-12-04 18:34:59,456 : INFO : PROGRESS: at sentence #13050000, processed 140633833 words, keeping 59596 word types\n",
      "2019-12-04 18:34:59,606 : INFO : PROGRESS: at sentence #13060000, processed 140744992 words, keeping 59596 word types\n",
      "2019-12-04 18:34:59,771 : INFO : PROGRESS: at sentence #13070000, processed 140867482 words, keeping 59596 word types\n",
      "2019-12-04 18:34:59,931 : INFO : PROGRESS: at sentence #13080000, processed 140987806 words, keeping 59596 word types\n",
      "2019-12-04 18:35:00,094 : INFO : PROGRESS: at sentence #13090000, processed 141109291 words, keeping 59596 word types\n",
      "2019-12-04 18:35:00,253 : INFO : PROGRESS: at sentence #13100000, processed 141230615 words, keeping 59596 word types\n",
      "2019-12-04 18:35:00,407 : INFO : PROGRESS: at sentence #13110000, processed 141348152 words, keeping 59596 word types\n",
      "2019-12-04 18:35:00,548 : INFO : PROGRESS: at sentence #13120000, processed 141453470 words, keeping 59596 word types\n",
      "2019-12-04 18:35:00,678 : INFO : PROGRESS: at sentence #13130000, processed 141552292 words, keeping 59596 word types\n",
      "2019-12-04 18:35:00,813 : INFO : PROGRESS: at sentence #13140000, processed 141654678 words, keeping 59596 word types\n",
      "2019-12-04 18:35:00,947 : INFO : PROGRESS: at sentence #13150000, processed 141758397 words, keeping 59596 word types\n",
      "2019-12-04 18:35:01,073 : INFO : PROGRESS: at sentence #13160000, processed 141854302 words, keeping 59596 word types\n",
      "2019-12-04 18:35:01,204 : INFO : PROGRESS: at sentence #13170000, processed 141955257 words, keeping 59596 word types\n",
      "2019-12-04 18:35:01,331 : INFO : PROGRESS: at sentence #13180000, processed 142050620 words, keeping 59596 word types\n",
      "2019-12-04 18:35:01,459 : INFO : PROGRESS: at sentence #13190000, processed 142147027 words, keeping 59596 word types\n",
      "2019-12-04 18:35:01,590 : INFO : PROGRESS: at sentence #13200000, processed 142246188 words, keeping 59596 word types\n",
      "2019-12-04 18:35:01,727 : INFO : PROGRESS: at sentence #13210000, processed 142349663 words, keeping 59596 word types\n",
      "2019-12-04 18:35:01,865 : INFO : PROGRESS: at sentence #13220000, processed 142455158 words, keeping 59596 word types\n",
      "2019-12-04 18:35:02,002 : INFO : PROGRESS: at sentence #13230000, processed 142559511 words, keeping 59596 word types\n",
      "2019-12-04 18:35:02,139 : INFO : PROGRESS: at sentence #13240000, processed 142664670 words, keeping 59596 word types\n",
      "2019-12-04 18:35:02,277 : INFO : PROGRESS: at sentence #13250000, processed 142770192 words, keeping 59596 word types\n",
      "2019-12-04 18:35:02,410 : INFO : PROGRESS: at sentence #13260000, processed 142872261 words, keeping 59596 word types\n",
      "2019-12-04 18:35:02,544 : INFO : PROGRESS: at sentence #13270000, processed 142973818 words, keeping 59596 word types\n",
      "2019-12-04 18:35:02,680 : INFO : PROGRESS: at sentence #13280000, processed 143075741 words, keeping 59596 word types\n",
      "2019-12-04 18:35:02,837 : INFO : PROGRESS: at sentence #13290000, processed 143198124 words, keeping 59596 word types\n",
      "2019-12-04 18:35:02,992 : INFO : PROGRESS: at sentence #13300000, processed 143317891 words, keeping 59596 word types\n",
      "2019-12-04 18:35:03,139 : INFO : PROGRESS: at sentence #13310000, processed 143431855 words, keeping 59596 word types\n",
      "2019-12-04 18:35:03,281 : INFO : PROGRESS: at sentence #13320000, processed 143541751 words, keeping 59596 word types\n",
      "2019-12-04 18:35:03,419 : INFO : PROGRESS: at sentence #13330000, processed 143647569 words, keeping 59596 word types\n",
      "2019-12-04 18:35:03,559 : INFO : PROGRESS: at sentence #13340000, processed 143754172 words, keeping 59596 word types\n",
      "2019-12-04 18:35:03,699 : INFO : PROGRESS: at sentence #13350000, processed 143860969 words, keeping 59596 word types\n",
      "2019-12-04 18:35:03,840 : INFO : PROGRESS: at sentence #13360000, processed 143968575 words, keeping 59596 word types\n",
      "2019-12-04 18:35:03,980 : INFO : PROGRESS: at sentence #13370000, processed 144075442 words, keeping 59596 word types\n",
      "2019-12-04 18:35:04,123 : INFO : PROGRESS: at sentence #13380000, processed 144184391 words, keeping 59596 word types\n",
      "2019-12-04 18:35:04,266 : INFO : PROGRESS: at sentence #13390000, processed 144294084 words, keeping 59596 word types\n",
      "2019-12-04 18:35:04,411 : INFO : PROGRESS: at sentence #13400000, processed 144405041 words, keeping 59596 word types\n",
      "2019-12-04 18:35:04,563 : INFO : PROGRESS: at sentence #13410000, processed 144520764 words, keeping 59596 word types\n",
      "2019-12-04 18:35:04,709 : INFO : PROGRESS: at sentence #13420000, processed 144631703 words, keeping 59596 word types\n",
      "2019-12-04 18:35:04,859 : INFO : PROGRESS: at sentence #13430000, processed 144746488 words, keeping 59596 word types\n",
      "2019-12-04 18:35:05,005 : INFO : PROGRESS: at sentence #13440000, processed 144855634 words, keeping 59596 word types\n",
      "2019-12-04 18:35:05,169 : INFO : PROGRESS: at sentence #13450000, processed 144982931 words, keeping 59596 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:35:05,326 : INFO : PROGRESS: at sentence #13460000, processed 145103539 words, keeping 59596 word types\n",
      "2019-12-04 18:35:05,484 : INFO : PROGRESS: at sentence #13470000, processed 145225699 words, keeping 59596 word types\n",
      "2019-12-04 18:35:05,635 : INFO : PROGRESS: at sentence #13480000, processed 145343297 words, keeping 59596 word types\n",
      "2019-12-04 18:35:05,787 : INFO : PROGRESS: at sentence #13490000, processed 145461336 words, keeping 59596 word types\n",
      "2019-12-04 18:35:05,918 : INFO : PROGRESS: at sentence #13500000, processed 145560415 words, keeping 59596 word types\n",
      "2019-12-04 18:35:06,049 : INFO : PROGRESS: at sentence #13510000, processed 145660750 words, keeping 59596 word types\n",
      "2019-12-04 18:35:06,184 : INFO : PROGRESS: at sentence #13520000, processed 145762471 words, keeping 59596 word types\n",
      "2019-12-04 18:35:06,321 : INFO : PROGRESS: at sentence #13530000, processed 145867678 words, keeping 59596 word types\n",
      "2019-12-04 18:35:06,447 : INFO : PROGRESS: at sentence #13540000, processed 145962969 words, keeping 59596 word types\n",
      "2019-12-04 18:35:06,574 : INFO : PROGRESS: at sentence #13550000, processed 146059560 words, keeping 59596 word types\n",
      "2019-12-04 18:35:06,702 : INFO : PROGRESS: at sentence #13560000, processed 146156985 words, keeping 59596 word types\n",
      "2019-12-04 18:35:06,832 : INFO : PROGRESS: at sentence #13570000, processed 146253330 words, keeping 59596 word types\n",
      "2019-12-04 18:35:06,963 : INFO : PROGRESS: at sentence #13580000, processed 146352745 words, keeping 59596 word types\n",
      "2019-12-04 18:35:07,101 : INFO : PROGRESS: at sentence #13590000, processed 146459310 words, keeping 59596 word types\n",
      "2019-12-04 18:35:07,239 : INFO : PROGRESS: at sentence #13600000, processed 146564237 words, keeping 59596 word types\n",
      "2019-12-04 18:35:07,375 : INFO : PROGRESS: at sentence #13610000, processed 146668124 words, keeping 59596 word types\n",
      "2019-12-04 18:35:07,512 : INFO : PROGRESS: at sentence #13620000, processed 146773220 words, keeping 59596 word types\n",
      "2019-12-04 18:35:07,651 : INFO : PROGRESS: at sentence #13630000, processed 146879030 words, keeping 59596 word types\n",
      "2019-12-04 18:35:07,784 : INFO : PROGRESS: at sentence #13640000, processed 146979269 words, keeping 59596 word types\n",
      "2019-12-04 18:35:07,920 : INFO : PROGRESS: at sentence #13650000, processed 147082059 words, keeping 59596 word types\n",
      "2019-12-04 18:35:08,057 : INFO : PROGRESS: at sentence #13660000, processed 147185573 words, keeping 59596 word types\n",
      "2019-12-04 18:35:08,221 : INFO : PROGRESS: at sentence #13670000, processed 147315631 words, keeping 59596 word types\n",
      "2019-12-04 18:35:08,368 : INFO : PROGRESS: at sentence #13680000, processed 147430688 words, keeping 59596 word types\n",
      "2019-12-04 18:35:08,513 : INFO : PROGRESS: at sentence #13690000, processed 147543157 words, keeping 59596 word types\n",
      "2019-12-04 18:35:08,652 : INFO : PROGRESS: at sentence #13700000, processed 147650476 words, keeping 59596 word types\n",
      "2019-12-04 18:35:08,791 : INFO : PROGRESS: at sentence #13710000, processed 147756607 words, keeping 59596 word types\n",
      "2019-12-04 18:35:08,932 : INFO : PROGRESS: at sentence #13720000, processed 147864297 words, keeping 59596 word types\n",
      "2019-12-04 18:35:09,072 : INFO : PROGRESS: at sentence #13730000, processed 147970752 words, keeping 59596 word types\n",
      "2019-12-04 18:35:09,213 : INFO : PROGRESS: at sentence #13740000, processed 148078473 words, keeping 59596 word types\n",
      "2019-12-04 18:35:09,354 : INFO : PROGRESS: at sentence #13750000, processed 148186212 words, keeping 59596 word types\n",
      "2019-12-04 18:35:09,497 : INFO : PROGRESS: at sentence #13760000, processed 148294725 words, keeping 59596 word types\n",
      "2019-12-04 18:35:09,639 : INFO : PROGRESS: at sentence #13770000, processed 148405133 words, keeping 59596 word types\n",
      "2019-12-04 18:35:09,787 : INFO : PROGRESS: at sentence #13780000, processed 148520022 words, keeping 59596 word types\n",
      "2019-12-04 18:35:09,933 : INFO : PROGRESS: at sentence #13790000, processed 148632076 words, keeping 59596 word types\n",
      "2019-12-04 18:35:10,082 : INFO : PROGRESS: at sentence #13800000, processed 148744775 words, keeping 59596 word types\n",
      "2019-12-04 18:35:10,231 : INFO : PROGRESS: at sentence #13810000, processed 148858055 words, keeping 59596 word types\n",
      "2019-12-04 18:35:10,387 : INFO : PROGRESS: at sentence #13820000, processed 148977373 words, keeping 59596 word types\n",
      "2019-12-04 18:35:10,544 : INFO : PROGRESS: at sentence #13830000, processed 149097495 words, keeping 59596 word types\n",
      "2019-12-04 18:35:10,702 : INFO : PROGRESS: at sentence #13840000, processed 149218545 words, keeping 59596 word types\n",
      "2019-12-04 18:35:10,862 : INFO : PROGRESS: at sentence #13850000, processed 149341314 words, keeping 59596 word types\n",
      "2019-12-04 18:35:11,012 : INFO : PROGRESS: at sentence #13860000, processed 149458845 words, keeping 59596 word types\n",
      "2019-12-04 18:35:11,154 : INFO : PROGRESS: at sentence #13870000, processed 149567668 words, keeping 59596 word types\n",
      "2019-12-04 18:35:11,287 : INFO : PROGRESS: at sentence #13880000, processed 149667460 words, keeping 59596 word types\n",
      "2019-12-04 18:35:11,423 : INFO : PROGRESS: at sentence #13890000, processed 149768849 words, keeping 59596 word types\n",
      "2019-12-04 18:35:11,562 : INFO : PROGRESS: at sentence #13900000, processed 149871501 words, keeping 59596 word types\n",
      "2019-12-04 18:35:11,694 : INFO : PROGRESS: at sentence #13910000, processed 149969286 words, keeping 59596 word types\n",
      "2019-12-04 18:35:11,831 : INFO : PROGRESS: at sentence #13920000, processed 150071327 words, keeping 59596 word types\n",
      "2019-12-04 18:35:11,958 : INFO : PROGRESS: at sentence #13930000, processed 150165911 words, keeping 59596 word types\n",
      "2019-12-04 18:35:12,089 : INFO : PROGRESS: at sentence #13940000, processed 150262686 words, keeping 59596 word types\n",
      "2019-12-04 18:35:12,217 : INFO : PROGRESS: at sentence #13950000, processed 150360594 words, keeping 59596 word types\n",
      "2019-12-04 18:35:12,350 : INFO : PROGRESS: at sentence #13960000, processed 150462720 words, keeping 59596 word types\n",
      "2019-12-04 18:35:12,492 : INFO : PROGRESS: at sentence #13970000, processed 150568333 words, keeping 59596 word types\n",
      "2019-12-04 18:35:12,634 : INFO : PROGRESS: at sentence #13980000, processed 150673068 words, keeping 59596 word types\n",
      "2019-12-04 18:35:12,774 : INFO : PROGRESS: at sentence #13990000, processed 150777635 words, keeping 59596 word types\n",
      "2019-12-04 18:35:12,916 : INFO : PROGRESS: at sentence #14000000, processed 150883136 words, keeping 59596 word types\n",
      "2019-12-04 18:35:13,051 : INFO : PROGRESS: at sentence #14010000, processed 150986476 words, keeping 59596 word types\n",
      "2019-12-04 18:35:13,185 : INFO : PROGRESS: at sentence #14020000, processed 151087213 words, keeping 59596 word types\n",
      "2019-12-04 18:35:13,320 : INFO : PROGRESS: at sentence #14030000, processed 151190265 words, keeping 59596 word types\n",
      "2019-12-04 18:35:13,470 : INFO : PROGRESS: at sentence #14040000, processed 151305998 words, keeping 59596 word types\n",
      "2019-12-04 18:35:13,627 : INFO : PROGRESS: at sentence #14050000, processed 151428529 words, keeping 59596 word types\n",
      "2019-12-04 18:35:13,775 : INFO : PROGRESS: at sentence #14060000, processed 151544375 words, keeping 59596 word types\n",
      "2019-12-04 18:35:13,916 : INFO : PROGRESS: at sentence #14070000, processed 151655443 words, keeping 59596 word types\n",
      "2019-12-04 18:35:14,054 : INFO : PROGRESS: at sentence #14080000, processed 151759007 words, keeping 59596 word types\n",
      "2019-12-04 18:35:14,193 : INFO : PROGRESS: at sentence #14090000, processed 151865502 words, keeping 59596 word types\n",
      "2019-12-04 18:35:14,332 : INFO : PROGRESS: at sentence #14100000, processed 151973971 words, keeping 59596 word types\n",
      "2019-12-04 18:35:14,471 : INFO : PROGRESS: at sentence #14110000, processed 152081579 words, keeping 59596 word types\n",
      "2019-12-04 18:35:14,610 : INFO : PROGRESS: at sentence #14120000, processed 152188644 words, keeping 59596 word types\n",
      "2019-12-04 18:35:14,748 : INFO : PROGRESS: at sentence #14130000, processed 152296746 words, keeping 59596 word types\n",
      "2019-12-04 18:35:14,889 : INFO : PROGRESS: at sentence #14140000, processed 152406204 words, keeping 59596 word types\n",
      "2019-12-04 18:35:15,032 : INFO : PROGRESS: at sentence #14150000, processed 152516607 words, keeping 59596 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:35:15,181 : INFO : PROGRESS: at sentence #14160000, processed 152633097 words, keeping 59596 word types\n",
      "2019-12-04 18:35:15,322 : INFO : PROGRESS: at sentence #14170000, processed 152742960 words, keeping 59596 word types\n",
      "2019-12-04 18:35:15,466 : INFO : PROGRESS: at sentence #14180000, processed 152857362 words, keeping 59596 word types\n",
      "2019-12-04 18:35:15,610 : INFO : PROGRESS: at sentence #14190000, processed 152967634 words, keeping 59596 word types\n",
      "2019-12-04 18:35:15,769 : INFO : PROGRESS: at sentence #14200000, processed 153092772 words, keeping 59596 word types\n",
      "2019-12-04 18:35:15,926 : INFO : PROGRESS: at sentence #14210000, processed 153212840 words, keeping 59596 word types\n",
      "2019-12-04 18:35:16,083 : INFO : PROGRESS: at sentence #14220000, processed 153334774 words, keeping 59596 word types\n",
      "2019-12-04 18:35:16,236 : INFO : PROGRESS: at sentence #14230000, processed 153454311 words, keeping 59596 word types\n",
      "2019-12-04 18:35:16,386 : INFO : PROGRESS: at sentence #14240000, processed 153572978 words, keeping 59596 word types\n",
      "2019-12-04 18:35:16,519 : INFO : PROGRESS: at sentence #14250000, processed 153674910 words, keeping 59596 word types\n",
      "2019-12-04 18:35:16,649 : INFO : PROGRESS: at sentence #14260000, processed 153774516 words, keeping 59596 word types\n",
      "2019-12-04 18:35:16,780 : INFO : PROGRESS: at sentence #14270000, processed 153876489 words, keeping 59596 word types\n",
      "2019-12-04 18:35:16,914 : INFO : PROGRESS: at sentence #14280000, processed 153981441 words, keeping 59596 word types\n",
      "2019-12-04 18:35:17,035 : INFO : PROGRESS: at sentence #14290000, processed 154076617 words, keeping 59596 word types\n",
      "2019-12-04 18:35:17,163 : INFO : PROGRESS: at sentence #14300000, processed 154176259 words, keeping 59596 word types\n",
      "2019-12-04 18:35:17,287 : INFO : PROGRESS: at sentence #14310000, processed 154272142 words, keeping 59596 word types\n",
      "2019-12-04 18:35:17,414 : INFO : PROGRESS: at sentence #14320000, processed 154368361 words, keeping 59596 word types\n",
      "2019-12-04 18:35:17,544 : INFO : PROGRESS: at sentence #14330000, processed 154467786 words, keeping 59596 word types\n",
      "2019-12-04 18:35:17,680 : INFO : PROGRESS: at sentence #14340000, processed 154572552 words, keeping 59596 word types\n",
      "2019-12-04 18:35:17,815 : INFO : PROGRESS: at sentence #14350000, processed 154677555 words, keeping 59596 word types\n",
      "2019-12-04 18:35:17,949 : INFO : PROGRESS: at sentence #14360000, processed 154782112 words, keeping 59596 word types\n",
      "2019-12-04 18:35:18,086 : INFO : PROGRESS: at sentence #14370000, processed 154886954 words, keeping 59596 word types\n",
      "2019-12-04 18:35:18,223 : INFO : PROGRESS: at sentence #14380000, processed 154993451 words, keeping 59596 word types\n",
      "2019-12-04 18:35:18,357 : INFO : PROGRESS: at sentence #14390000, processed 155094120 words, keeping 59596 word types\n",
      "2019-12-04 18:35:18,489 : INFO : PROGRESS: at sentence #14400000, processed 155195890 words, keeping 59596 word types\n",
      "2019-12-04 18:35:18,620 : INFO : PROGRESS: at sentence #14410000, processed 155298367 words, keeping 59596 word types\n",
      "2019-12-04 18:35:18,779 : INFO : PROGRESS: at sentence #14420000, processed 155424604 words, keeping 59596 word types\n",
      "2019-12-04 18:35:18,928 : INFO : PROGRESS: at sentence #14430000, processed 155542209 words, keeping 59596 word types\n",
      "2019-12-04 18:35:19,069 : INFO : PROGRESS: at sentence #14440000, processed 155654920 words, keeping 59596 word types\n",
      "2019-12-04 18:35:19,207 : INFO : PROGRESS: at sentence #14450000, processed 155764326 words, keeping 59596 word types\n",
      "2019-12-04 18:35:19,344 : INFO : PROGRESS: at sentence #14460000, processed 155870161 words, keeping 59596 word types\n",
      "2019-12-04 18:35:19,482 : INFO : PROGRESS: at sentence #14470000, processed 155977103 words, keeping 59596 word types\n",
      "2019-12-04 18:35:19,620 : INFO : PROGRESS: at sentence #14480000, processed 156083848 words, keeping 59596 word types\n",
      "2019-12-04 18:35:19,762 : INFO : PROGRESS: at sentence #14490000, processed 156191204 words, keeping 59596 word types\n",
      "2019-12-04 18:35:19,904 : INFO : PROGRESS: at sentence #14500000, processed 156298489 words, keeping 59596 word types\n",
      "2019-12-04 18:35:20,045 : INFO : PROGRESS: at sentence #14510000, processed 156407166 words, keeping 59596 word types\n",
      "2019-12-04 18:35:20,187 : INFO : PROGRESS: at sentence #14520000, processed 156517086 words, keeping 59596 word types\n",
      "2019-12-04 18:35:20,331 : INFO : PROGRESS: at sentence #14530000, processed 156629545 words, keeping 59596 word types\n",
      "2019-12-04 18:35:20,479 : INFO : PROGRESS: at sentence #14540000, processed 156743703 words, keeping 59596 word types\n",
      "2019-12-04 18:35:20,627 : INFO : PROGRESS: at sentence #14550000, processed 156855582 words, keeping 59596 word types\n",
      "2019-12-04 18:35:20,779 : INFO : PROGRESS: at sentence #14560000, processed 156970438 words, keeping 59596 word types\n",
      "2019-12-04 18:35:20,927 : INFO : PROGRESS: at sentence #14570000, processed 157085131 words, keeping 59596 word types\n",
      "2019-12-04 18:35:21,084 : INFO : PROGRESS: at sentence #14580000, processed 157207603 words, keeping 59596 word types\n",
      "2019-12-04 18:35:21,238 : INFO : PROGRESS: at sentence #14590000, processed 157327998 words, keeping 59596 word types\n",
      "2019-12-04 18:35:21,395 : INFO : PROGRESS: at sentence #14600000, processed 157451256 words, keeping 59596 word types\n",
      "2019-12-04 18:35:21,540 : INFO : PROGRESS: at sentence #14610000, processed 157565954 words, keeping 59596 word types\n",
      "2019-12-04 18:35:21,690 : INFO : PROGRESS: at sentence #14620000, processed 157682620 words, keeping 59596 word types\n",
      "2019-12-04 18:35:21,820 : INFO : PROGRESS: at sentence #14630000, processed 157781999 words, keeping 59596 word types\n",
      "2019-12-04 18:35:21,951 : INFO : PROGRESS: at sentence #14640000, processed 157883061 words, keeping 59596 word types\n",
      "2019-12-04 18:35:22,081 : INFO : PROGRESS: at sentence #14650000, processed 157985454 words, keeping 59596 word types\n",
      "2019-12-04 18:35:22,214 : INFO : PROGRESS: at sentence #14660000, processed 158088059 words, keeping 59596 word types\n",
      "2019-12-04 18:35:22,340 : INFO : PROGRESS: at sentence #14670000, processed 158185026 words, keeping 59596 word types\n",
      "2019-12-04 18:35:22,464 : INFO : PROGRESS: at sentence #14680000, processed 158280534 words, keeping 59596 word types\n",
      "2019-12-04 18:35:22,593 : INFO : PROGRESS: at sentence #14690000, processed 158378347 words, keeping 59596 word types\n",
      "2019-12-04 18:35:22,721 : INFO : PROGRESS: at sentence #14700000, processed 158475146 words, keeping 59596 word types\n",
      "2019-12-04 18:35:22,853 : INFO : PROGRESS: at sentence #14710000, processed 158575993 words, keeping 59596 word types\n",
      "2019-12-04 18:35:22,992 : INFO : PROGRESS: at sentence #14720000, processed 158681991 words, keeping 59596 word types\n",
      "2019-12-04 18:35:23,128 : INFO : PROGRESS: at sentence #14730000, processed 158786795 words, keeping 59596 word types\n",
      "2019-12-04 18:35:23,263 : INFO : PROGRESS: at sentence #14740000, processed 158890760 words, keeping 59596 word types\n",
      "2019-12-04 18:35:23,401 : INFO : PROGRESS: at sentence #14750000, processed 158995788 words, keeping 59596 word types\n",
      "2019-12-04 18:35:23,539 : INFO : PROGRESS: at sentence #14760000, processed 159101377 words, keeping 59596 word types\n",
      "2019-12-04 18:35:23,672 : INFO : PROGRESS: at sentence #14770000, processed 159201367 words, keeping 59596 word types\n",
      "2019-12-04 18:35:23,807 : INFO : PROGRESS: at sentence #14780000, processed 159304358 words, keeping 59596 word types\n",
      "2019-12-04 18:35:23,946 : INFO : PROGRESS: at sentence #14790000, processed 159412409 words, keeping 59596 word types\n",
      "2019-12-04 18:35:24,107 : INFO : PROGRESS: at sentence #14800000, processed 159539683 words, keeping 59596 word types\n",
      "2019-12-04 18:35:24,252 : INFO : PROGRESS: at sentence #14810000, processed 159655575 words, keeping 59596 word types\n",
      "2019-12-04 18:35:24,391 : INFO : PROGRESS: at sentence #14820000, processed 159766158 words, keeping 59596 word types\n",
      "2019-12-04 18:35:24,528 : INFO : PROGRESS: at sentence #14830000, processed 159872712 words, keeping 59596 word types\n",
      "2019-12-04 18:35:24,666 : INFO : PROGRESS: at sentence #14840000, processed 159978850 words, keeping 59596 word types\n",
      "2019-12-04 18:35:24,806 : INFO : PROGRESS: at sentence #14850000, processed 160087069 words, keeping 59596 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:35:24,944 : INFO : PROGRESS: at sentence #14860000, processed 160193882 words, keeping 59596 word types\n",
      "2019-12-04 18:35:25,084 : INFO : PROGRESS: at sentence #14870000, processed 160301564 words, keeping 59596 word types\n",
      "2019-12-04 18:35:25,224 : INFO : PROGRESS: at sentence #14880000, processed 160409767 words, keeping 59596 word types\n",
      "2019-12-04 18:35:25,363 : INFO : PROGRESS: at sentence #14890000, processed 160518213 words, keeping 59596 word types\n",
      "2019-12-04 18:35:25,503 : INFO : PROGRESS: at sentence #14900000, processed 160628594 words, keeping 59596 word types\n",
      "2019-12-04 18:35:25,652 : INFO : PROGRESS: at sentence #14910000, processed 160744525 words, keeping 59596 word types\n",
      "2019-12-04 18:35:25,797 : INFO : PROGRESS: at sentence #14920000, processed 160855091 words, keeping 59596 word types\n",
      "2019-12-04 18:35:25,945 : INFO : PROGRESS: at sentence #14930000, processed 160968551 words, keeping 59596 word types\n",
      "2019-12-04 18:35:26,090 : INFO : PROGRESS: at sentence #14940000, processed 161080753 words, keeping 59596 word types\n",
      "2019-12-04 18:35:26,246 : INFO : PROGRESS: at sentence #14950000, processed 161202295 words, keeping 59596 word types\n",
      "2019-12-04 18:35:26,402 : INFO : PROGRESS: at sentence #14960000, processed 161322389 words, keeping 59596 word types\n",
      "2019-12-04 18:35:26,558 : INFO : PROGRESS: at sentence #14970000, processed 161443483 words, keeping 59596 word types\n",
      "2019-12-04 18:35:26,714 : INFO : PROGRESS: at sentence #14980000, processed 161565952 words, keeping 59596 word types\n",
      "2019-12-04 18:35:26,864 : INFO : PROGRESS: at sentence #14990000, processed 161682628 words, keeping 59596 word types\n",
      "2019-12-04 18:35:27,003 : INFO : PROGRESS: at sentence #15000000, processed 161789769 words, keeping 59596 word types\n",
      "2019-12-04 18:35:27,134 : INFO : PROGRESS: at sentence #15010000, processed 161888619 words, keeping 59596 word types\n",
      "2019-12-04 18:35:27,265 : INFO : PROGRESS: at sentence #15020000, processed 161991089 words, keeping 59596 word types\n",
      "2019-12-04 18:35:27,398 : INFO : PROGRESS: at sentence #15030000, processed 162094322 words, keeping 59596 word types\n",
      "2019-12-04 18:35:27,524 : INFO : PROGRESS: at sentence #15040000, processed 162192491 words, keeping 59596 word types\n",
      "2019-12-04 18:35:27,653 : INFO : PROGRESS: at sentence #15050000, processed 162292091 words, keeping 59596 word types\n",
      "2019-12-04 18:35:27,778 : INFO : PROGRESS: at sentence #15060000, processed 162387062 words, keeping 59596 word types\n",
      "2019-12-04 18:35:27,906 : INFO : PROGRESS: at sentence #15070000, processed 162483902 words, keeping 59596 word types\n",
      "2019-12-04 18:35:28,034 : INFO : PROGRESS: at sentence #15080000, processed 162582177 words, keeping 59596 word types\n",
      "2019-12-04 18:35:28,168 : INFO : PROGRESS: at sentence #15090000, processed 162685187 words, keeping 59596 word types\n",
      "2019-12-04 18:35:28,305 : INFO : PROGRESS: at sentence #15100000, processed 162791200 words, keeping 59596 word types\n",
      "2019-12-04 18:35:28,441 : INFO : PROGRESS: at sentence #15110000, processed 162895502 words, keeping 59596 word types\n",
      "2019-12-04 18:35:28,576 : INFO : PROGRESS: at sentence #15120000, processed 163000544 words, keeping 59596 word types\n",
      "2019-12-04 18:35:28,713 : INFO : PROGRESS: at sentence #15130000, processed 163105970 words, keeping 59596 word types\n",
      "2019-12-04 18:35:28,849 : INFO : PROGRESS: at sentence #15140000, processed 163208503 words, keeping 59596 word types\n",
      "2019-12-04 18:35:28,984 : INFO : PROGRESS: at sentence #15150000, processed 163310003 words, keeping 59596 word types\n",
      "2019-12-04 18:35:29,117 : INFO : PROGRESS: at sentence #15160000, processed 163411832 words, keeping 59596 word types\n",
      "2019-12-04 18:35:29,266 : INFO : PROGRESS: at sentence #15170000, processed 163531820 words, keeping 59596 word types\n",
      "2019-12-04 18:35:29,419 : INFO : PROGRESS: at sentence #15180000, processed 163652767 words, keeping 59596 word types\n",
      "2019-12-04 18:35:29,567 : INFO : PROGRESS: at sentence #15190000, processed 163767726 words, keeping 59596 word types\n",
      "2019-12-04 18:35:29,710 : INFO : PROGRESS: at sentence #15200000, processed 163877815 words, keeping 59596 word types\n",
      "2019-12-04 18:35:29,852 : INFO : PROGRESS: at sentence #15210000, processed 163983671 words, keeping 59596 word types\n",
      "2019-12-04 18:35:29,991 : INFO : PROGRESS: at sentence #15220000, processed 164089220 words, keeping 59596 word types\n",
      "2019-12-04 18:35:30,130 : INFO : PROGRESS: at sentence #15230000, processed 164196910 words, keeping 59596 word types\n",
      "2019-12-04 18:35:30,269 : INFO : PROGRESS: at sentence #15240000, processed 164304422 words, keeping 59596 word types\n",
      "2019-12-04 18:35:30,408 : INFO : PROGRESS: at sentence #15250000, processed 164411161 words, keeping 59596 word types\n",
      "2019-12-04 18:35:30,549 : INFO : PROGRESS: at sentence #15260000, processed 164519789 words, keeping 59596 word types\n",
      "2019-12-04 18:35:30,693 : INFO : PROGRESS: at sentence #15270000, processed 164629601 words, keeping 59596 word types\n",
      "2019-12-04 18:35:30,834 : INFO : PROGRESS: at sentence #15280000, processed 164740475 words, keeping 59596 word types\n",
      "2019-12-04 18:35:30,982 : INFO : PROGRESS: at sentence #15290000, processed 164856711 words, keeping 59596 word types\n",
      "2019-12-04 18:35:31,120 : INFO : PROGRESS: at sentence #15300000, processed 164966584 words, keeping 59596 word types\n",
      "2019-12-04 18:35:31,269 : INFO : PROGRESS: at sentence #15310000, processed 165081837 words, keeping 59596 word types\n",
      "2019-12-04 18:35:31,411 : INFO : PROGRESS: at sentence #15320000, processed 165190673 words, keeping 59596 word types\n",
      "2019-12-04 18:35:31,577 : INFO : PROGRESS: at sentence #15330000, processed 165317488 words, keeping 59596 word types\n",
      "2019-12-04 18:35:31,754 : INFO : PROGRESS: at sentence #15340000, processed 165438332 words, keeping 59596 word types\n",
      "2019-12-04 18:35:31,935 : INFO : PROGRESS: at sentence #15350000, processed 165560157 words, keeping 59596 word types\n",
      "2019-12-04 18:35:32,103 : INFO : PROGRESS: at sentence #15360000, processed 165678171 words, keeping 59596 word types\n",
      "2019-12-04 18:35:32,269 : INFO : PROGRESS: at sentence #15370000, processed 165797156 words, keeping 59596 word types\n",
      "2019-12-04 18:35:32,407 : INFO : PROGRESS: at sentence #15380000, processed 165896635 words, keeping 59596 word types\n",
      "2019-12-04 18:35:32,551 : INFO : PROGRESS: at sentence #15390000, processed 165996884 words, keeping 59596 word types\n",
      "2019-12-04 18:35:32,698 : INFO : PROGRESS: at sentence #15400000, processed 166098722 words, keeping 59596 word types\n",
      "2019-12-04 18:35:32,847 : INFO : PROGRESS: at sentence #15410000, processed 166203916 words, keeping 59596 word types\n",
      "2019-12-04 18:35:32,981 : INFO : PROGRESS: at sentence #15420000, processed 166299489 words, keeping 59596 word types\n",
      "2019-12-04 18:35:33,122 : INFO : PROGRESS: at sentence #15430000, processed 166396744 words, keeping 59596 word types\n",
      "2019-12-04 18:35:33,261 : INFO : PROGRESS: at sentence #15440000, processed 166493937 words, keeping 59596 word types\n",
      "2019-12-04 18:35:33,399 : INFO : PROGRESS: at sentence #15450000, processed 166590036 words, keeping 59596 word types\n",
      "2019-12-04 18:35:33,539 : INFO : PROGRESS: at sentence #15460000, processed 166689265 words, keeping 59596 word types\n",
      "2019-12-04 18:35:33,691 : INFO : PROGRESS: at sentence #15470000, processed 166794936 words, keeping 59596 word types\n",
      "2019-12-04 18:35:33,840 : INFO : PROGRESS: at sentence #15480000, processed 166900142 words, keeping 59596 word types\n",
      "2019-12-04 18:35:33,992 : INFO : PROGRESS: at sentence #15490000, processed 167004095 words, keeping 59596 word types\n",
      "2019-12-04 18:35:34,145 : INFO : PROGRESS: at sentence #15500000, processed 167109543 words, keeping 59596 word types\n",
      "2019-12-04 18:35:34,299 : INFO : PROGRESS: at sentence #15510000, processed 167215397 words, keeping 59596 word types\n",
      "2019-12-04 18:35:34,443 : INFO : PROGRESS: at sentence #15520000, processed 167315615 words, keeping 59596 word types\n",
      "2019-12-04 18:35:34,593 : INFO : PROGRESS: at sentence #15530000, processed 167418303 words, keeping 59596 word types\n",
      "2019-12-04 18:35:34,739 : INFO : PROGRESS: at sentence #15540000, processed 167520909 words, keeping 59596 word types\n",
      "2019-12-04 18:35:34,924 : INFO : PROGRESS: at sentence #15550000, processed 167650802 words, keeping 59596 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:35:35,088 : INFO : PROGRESS: at sentence #15560000, processed 167765978 words, keeping 59596 word types\n",
      "2019-12-04 18:35:35,247 : INFO : PROGRESS: at sentence #15570000, processed 167878522 words, keeping 59596 word types\n",
      "2019-12-04 18:35:35,401 : INFO : PROGRESS: at sentence #15580000, processed 167986517 words, keeping 59596 word types\n",
      "2019-12-04 18:35:35,550 : INFO : PROGRESS: at sentence #15590000, processed 168092670 words, keeping 59596 word types\n",
      "2019-12-04 18:35:35,705 : INFO : PROGRESS: at sentence #15600000, processed 168200027 words, keeping 59596 word types\n",
      "2019-12-04 18:35:35,858 : INFO : PROGRESS: at sentence #15610000, processed 168306510 words, keeping 59596 word types\n",
      "2019-12-04 18:35:36,016 : INFO : PROGRESS: at sentence #15620000, processed 168414239 words, keeping 59596 word types\n",
      "2019-12-04 18:35:36,169 : INFO : PROGRESS: at sentence #15630000, processed 168521387 words, keeping 59596 word types\n",
      "2019-12-04 18:35:36,325 : INFO : PROGRESS: at sentence #15640000, processed 168629948 words, keeping 59596 word types\n",
      "2019-12-04 18:35:36,480 : INFO : PROGRESS: at sentence #15650000, processed 168740654 words, keeping 59596 word types\n",
      "2019-12-04 18:35:36,644 : INFO : PROGRESS: at sentence #15660000, processed 168854508 words, keeping 59596 word types\n",
      "2019-12-04 18:35:36,810 : INFO : PROGRESS: at sentence #15670000, processed 168967392 words, keeping 59596 word types\n",
      "2019-12-04 18:35:36,976 : INFO : PROGRESS: at sentence #15680000, processed 169079689 words, keeping 59596 word types\n",
      "2019-12-04 18:35:37,146 : INFO : PROGRESS: at sentence #15690000, processed 169193729 words, keeping 59596 word types\n",
      "2019-12-04 18:35:37,314 : INFO : PROGRESS: at sentence #15700000, processed 169311615 words, keeping 59596 word types\n",
      "2019-12-04 18:35:37,491 : INFO : PROGRESS: at sentence #15710000, processed 169432212 words, keeping 59596 word types\n",
      "2019-12-04 18:35:37,663 : INFO : PROGRESS: at sentence #15720000, processed 169552926 words, keeping 59596 word types\n",
      "2019-12-04 18:35:37,841 : INFO : PROGRESS: at sentence #15730000, processed 169676028 words, keeping 59596 word types\n",
      "2019-12-04 18:35:38,007 : INFO : PROGRESS: at sentence #15740000, processed 169793308 words, keeping 59596 word types\n",
      "2019-12-04 18:35:38,164 : INFO : PROGRESS: at sentence #15750000, processed 169904056 words, keeping 59596 word types\n",
      "2019-12-04 18:35:38,309 : INFO : PROGRESS: at sentence #15760000, processed 170004078 words, keeping 59596 word types\n",
      "2019-12-04 18:35:38,452 : INFO : PROGRESS: at sentence #15770000, processed 170105169 words, keeping 59596 word types\n",
      "2019-12-04 18:35:38,593 : INFO : PROGRESS: at sentence #15780000, processed 170207825 words, keeping 59596 word types\n",
      "2019-12-04 18:35:38,729 : INFO : PROGRESS: at sentence #15790000, processed 170305130 words, keeping 59596 word types\n",
      "2019-12-04 18:35:38,874 : INFO : PROGRESS: at sentence #15800000, processed 170407663 words, keeping 59596 word types\n",
      "2019-12-04 18:35:39,008 : INFO : PROGRESS: at sentence #15810000, processed 170501616 words, keeping 59596 word types\n",
      "2019-12-04 18:35:39,149 : INFO : PROGRESS: at sentence #15820000, processed 170599360 words, keeping 59596 word types\n",
      "2019-12-04 18:35:39,293 : INFO : PROGRESS: at sentence #15830000, processed 170696809 words, keeping 59596 word types\n",
      "2019-12-04 18:35:39,439 : INFO : PROGRESS: at sentence #15840000, processed 170798696 words, keeping 59596 word types\n",
      "2019-12-04 18:35:39,595 : INFO : PROGRESS: at sentence #15850000, processed 170904633 words, keeping 59596 word types\n",
      "2019-12-04 18:35:39,752 : INFO : PROGRESS: at sentence #15860000, processed 171009010 words, keeping 59596 word types\n",
      "2019-12-04 18:35:39,906 : INFO : PROGRESS: at sentence #15870000, processed 171113486 words, keeping 59596 word types\n",
      "2019-12-04 18:35:40,057 : INFO : PROGRESS: at sentence #15880000, processed 171218946 words, keeping 59596 word types\n",
      "2019-12-04 18:35:40,203 : INFO : PROGRESS: at sentence #15890000, processed 171323139 words, keeping 59596 word types\n",
      "2019-12-04 18:35:40,346 : INFO : PROGRESS: at sentence #15900000, processed 171423087 words, keeping 59596 word types\n",
      "2019-12-04 18:35:40,496 : INFO : PROGRESS: at sentence #15910000, processed 171526256 words, keeping 59596 word types\n",
      "2019-12-04 18:35:40,654 : INFO : PROGRESS: at sentence #15920000, processed 171639289 words, keeping 59596 word types\n",
      "2019-12-04 18:35:40,828 : INFO : PROGRESS: at sentence #15930000, processed 171763960 words, keeping 59596 word types\n",
      "2019-12-04 18:35:40,989 : INFO : PROGRESS: at sentence #15940000, processed 171879605 words, keeping 59596 word types\n",
      "2019-12-04 18:35:41,146 : INFO : PROGRESS: at sentence #15950000, processed 171991721 words, keeping 59596 word types\n",
      "2019-12-04 18:35:41,294 : INFO : PROGRESS: at sentence #15960000, processed 172095070 words, keeping 59596 word types\n",
      "2019-12-04 18:35:41,447 : INFO : PROGRESS: at sentence #15970000, processed 172201514 words, keeping 59596 word types\n",
      "2019-12-04 18:35:41,607 : INFO : PROGRESS: at sentence #15980000, processed 172309658 words, keeping 59596 word types\n",
      "2019-12-04 18:35:41,768 : INFO : PROGRESS: at sentence #15990000, processed 172417161 words, keeping 59596 word types\n",
      "2019-12-04 18:35:41,926 : INFO : PROGRESS: at sentence #16000000, processed 172524484 words, keeping 59596 word types\n",
      "2019-12-04 18:35:42,083 : INFO : PROGRESS: at sentence #16010000, processed 172632707 words, keeping 59596 word types\n",
      "2019-12-04 18:35:42,240 : INFO : PROGRESS: at sentence #16020000, processed 172741891 words, keeping 59596 word types\n",
      "2019-12-04 18:35:42,395 : INFO : PROGRESS: at sentence #16030000, processed 172852116 words, keeping 59596 word types\n",
      "2019-12-04 18:35:42,564 : INFO : PROGRESS: at sentence #16040000, processed 172968334 words, keeping 59596 word types\n",
      "2019-12-04 18:35:42,729 : INFO : PROGRESS: at sentence #16050000, processed 173078366 words, keeping 59596 word types\n",
      "2019-12-04 18:35:42,900 : INFO : PROGRESS: at sentence #16060000, processed 173192595 words, keeping 59596 word types\n",
      "2019-12-04 18:35:43,078 : INFO : PROGRESS: at sentence #16070000, processed 173303384 words, keeping 59596 word types\n",
      "2019-12-04 18:35:43,259 : INFO : PROGRESS: at sentence #16080000, processed 173427085 words, keeping 59596 word types\n",
      "2019-12-04 18:35:43,438 : INFO : PROGRESS: at sentence #16090000, processed 173547354 words, keeping 59596 word types\n",
      "2019-12-04 18:35:43,618 : INFO : PROGRESS: at sentence #16100000, processed 173668959 words, keeping 59596 word types\n",
      "2019-12-04 18:35:43,789 : INFO : PROGRESS: at sentence #16110000, processed 173789709 words, keeping 59596 word types\n",
      "2019-12-04 18:35:43,956 : INFO : PROGRESS: at sentence #16120000, processed 173907415 words, keeping 59596 word types\n",
      "2019-12-04 18:35:44,104 : INFO : PROGRESS: at sentence #16130000, processed 174011423 words, keeping 59596 word types\n",
      "2019-12-04 18:35:44,247 : INFO : PROGRESS: at sentence #16140000, processed 174110842 words, keeping 59596 word types\n",
      "2019-12-04 18:35:44,395 : INFO : PROGRESS: at sentence #16150000, processed 174212925 words, keeping 59596 word types\n",
      "2019-12-04 18:35:44,540 : INFO : PROGRESS: at sentence #16160000, processed 174317144 words, keeping 59596 word types\n",
      "2019-12-04 18:35:44,677 : INFO : PROGRESS: at sentence #16170000, processed 174412668 words, keeping 59596 word types\n",
      "2019-12-04 18:35:44,822 : INFO : PROGRESS: at sentence #16180000, processed 174512960 words, keeping 59596 word types\n",
      "2019-12-04 18:35:44,961 : INFO : PROGRESS: at sentence #16190000, processed 174608439 words, keeping 59596 word types\n",
      "2019-12-04 18:35:45,088 : INFO : PROGRESS: at sentence #16200000, processed 174705257 words, keeping 59596 word types\n",
      "2019-12-04 18:35:45,224 : INFO : PROGRESS: at sentence #16210000, processed 174804251 words, keeping 59596 word types\n",
      "2019-12-04 18:35:45,374 : INFO : PROGRESS: at sentence #16220000, processed 174908703 words, keeping 59596 word types\n",
      "2019-12-04 18:35:45,524 : INFO : PROGRESS: at sentence #16230000, processed 175013404 words, keeping 59596 word types\n",
      "2019-12-04 18:35:45,673 : INFO : PROGRESS: at sentence #16240000, processed 175118288 words, keeping 59596 word types\n",
      "2019-12-04 18:35:45,822 : INFO : PROGRESS: at sentence #16250000, processed 175222800 words, keeping 59596 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:35:45,975 : INFO : PROGRESS: at sentence #16260000, processed 175329159 words, keeping 59596 word types\n",
      "2019-12-04 18:35:46,125 : INFO : PROGRESS: at sentence #16270000, processed 175430489 words, keeping 59596 word types\n",
      "2019-12-04 18:35:46,276 : INFO : PROGRESS: at sentence #16280000, processed 175532015 words, keeping 59596 word types\n",
      "2019-12-04 18:35:46,428 : INFO : PROGRESS: at sentence #16290000, processed 175634344 words, keeping 59596 word types\n",
      "2019-12-04 18:35:46,607 : INFO : PROGRESS: at sentence #16300000, processed 175758551 words, keeping 59596 word types\n",
      "2019-12-04 18:35:46,773 : INFO : PROGRESS: at sentence #16310000, processed 175877101 words, keeping 59596 word types\n",
      "2019-12-04 18:35:46,935 : INFO : PROGRESS: at sentence #16320000, processed 175990648 words, keeping 59596 word types\n",
      "2019-12-04 18:35:47,088 : INFO : PROGRESS: at sentence #16330000, processed 176100203 words, keeping 59596 word types\n",
      "2019-12-04 18:35:47,242 : INFO : PROGRESS: at sentence #16340000, processed 176206323 words, keeping 59596 word types\n",
      "2019-12-04 18:35:47,396 : INFO : PROGRESS: at sentence #16350000, processed 176312978 words, keeping 59596 word types\n",
      "2019-12-04 18:35:47,548 : INFO : PROGRESS: at sentence #16360000, processed 176419529 words, keeping 59596 word types\n",
      "2019-12-04 18:35:47,702 : INFO : PROGRESS: at sentence #16370000, processed 176526990 words, keeping 59596 word types\n",
      "2019-12-04 18:35:47,856 : INFO : PROGRESS: at sentence #16380000, processed 176634207 words, keeping 59596 word types\n",
      "2019-12-04 18:35:48,013 : INFO : PROGRESS: at sentence #16390000, processed 176743018 words, keeping 59596 word types\n",
      "2019-12-04 18:35:48,170 : INFO : PROGRESS: at sentence #16400000, processed 176852841 words, keeping 59596 word types\n",
      "2019-12-04 18:35:48,328 : INFO : PROGRESS: at sentence #16410000, processed 176964438 words, keeping 59596 word types\n",
      "2019-12-04 18:35:48,496 : INFO : PROGRESS: at sentence #16420000, processed 177079063 words, keeping 59596 word types\n",
      "2019-12-04 18:35:48,660 : INFO : PROGRESS: at sentence #16430000, processed 177190976 words, keeping 59596 word types\n",
      "2019-12-04 18:35:48,825 : INFO : PROGRESS: at sentence #16440000, processed 177305440 words, keeping 59596 word types\n",
      "2019-12-04 18:35:48,990 : INFO : PROGRESS: at sentence #16450000, processed 177417464 words, keeping 59596 word types\n",
      "2019-12-04 18:35:49,173 : INFO : PROGRESS: at sentence #16460000, processed 177542298 words, keeping 59596 word types\n",
      "2019-12-04 18:35:49,352 : INFO : PROGRESS: at sentence #16470000, processed 177662771 words, keeping 59596 word types\n",
      "2019-12-04 18:35:49,530 : INFO : PROGRESS: at sentence #16480000, processed 177785515 words, keeping 59596 word types\n",
      "2019-12-04 18:35:49,699 : INFO : PROGRESS: at sentence #16490000, processed 177901763 words, keeping 59596 word types\n",
      "2019-12-04 18:35:49,866 : INFO : PROGRESS: at sentence #16500000, processed 178019192 words, keeping 59596 word types\n",
      "2019-12-04 18:35:50,007 : INFO : PROGRESS: at sentence #16510000, processed 178118423 words, keeping 59596 word types\n",
      "2019-12-04 18:35:50,153 : INFO : PROGRESS: at sentence #16520000, processed 178219299 words, keeping 59596 word types\n",
      "2019-12-04 18:35:50,299 : INFO : PROGRESS: at sentence #16530000, processed 178320994 words, keeping 59596 word types\n",
      "2019-12-04 18:35:50,447 : INFO : PROGRESS: at sentence #16540000, processed 178426182 words, keeping 59596 word types\n",
      "2019-12-04 18:35:50,582 : INFO : PROGRESS: at sentence #16550000, processed 178521069 words, keeping 59596 word types\n",
      "2019-12-04 18:35:50,720 : INFO : PROGRESS: at sentence #16560000, processed 178617514 words, keeping 59596 word types\n",
      "2019-12-04 18:35:50,859 : INFO : PROGRESS: at sentence #16570000, processed 178715130 words, keeping 59596 word types\n",
      "2019-12-04 18:35:50,997 : INFO : PROGRESS: at sentence #16580000, processed 178811491 words, keeping 59596 word types\n",
      "2019-12-04 18:35:51,138 : INFO : PROGRESS: at sentence #16590000, processed 178911508 words, keeping 59596 word types\n",
      "2019-12-04 18:35:51,289 : INFO : PROGRESS: at sentence #16600000, processed 179017767 words, keeping 59596 word types\n",
      "2019-12-04 18:35:51,441 : INFO : PROGRESS: at sentence #16610000, processed 179122707 words, keeping 59596 word types\n",
      "2019-12-04 18:35:51,589 : INFO : PROGRESS: at sentence #16620000, processed 179226548 words, keeping 59596 word types\n",
      "2019-12-04 18:35:51,742 : INFO : PROGRESS: at sentence #16630000, processed 179331682 words, keeping 59596 word types\n",
      "2019-12-04 18:35:51,951 : INFO : PROGRESS: at sentence #16640000, processed 179437543 words, keeping 59596 word types\n",
      "2019-12-04 18:35:52,174 : INFO : PROGRESS: at sentence #16650000, processed 179537583 words, keeping 59596 word types\n",
      "2019-12-04 18:35:52,398 : INFO : PROGRESS: at sentence #16660000, processed 179640591 words, keeping 59596 word types\n",
      "2019-12-04 18:35:52,580 : INFO : PROGRESS: at sentence #16670000, processed 179745894 words, keeping 59596 word types\n",
      "2019-12-04 18:35:52,768 : INFO : PROGRESS: at sentence #16680000, processed 179874800 words, keeping 59596 word types\n",
      "2019-12-04 18:35:52,935 : INFO : PROGRESS: at sentence #16690000, processed 179989997 words, keeping 59596 word types\n",
      "2019-12-04 18:35:53,099 : INFO : PROGRESS: at sentence #16700000, processed 180101891 words, keeping 59596 word types\n",
      "2019-12-04 18:35:53,257 : INFO : PROGRESS: at sentence #16710000, processed 180208902 words, keeping 59596 word types\n",
      "2019-12-04 18:35:53,428 : INFO : PROGRESS: at sentence #16720000, processed 180315124 words, keeping 59596 word types\n",
      "2019-12-04 18:35:53,601 : INFO : PROGRESS: at sentence #16730000, processed 180423030 words, keeping 59596 word types\n",
      "2019-12-04 18:35:53,752 : INFO : PROGRESS: at sentence #16740000, processed 180529388 words, keeping 59596 word types\n",
      "2019-12-04 18:35:53,905 : INFO : PROGRESS: at sentence #16750000, processed 180637196 words, keeping 59596 word types\n",
      "2019-12-04 18:35:54,057 : INFO : PROGRESS: at sentence #16760000, processed 180744884 words, keeping 59596 word types\n",
      "2019-12-04 18:35:54,218 : INFO : PROGRESS: at sentence #16770000, processed 180853667 words, keeping 59596 word types\n",
      "2019-12-04 18:35:54,380 : INFO : PROGRESS: at sentence #16780000, processed 180963999 words, keeping 59596 word types\n",
      "2019-12-04 18:35:54,549 : INFO : PROGRESS: at sentence #16790000, processed 181079491 words, keeping 59596 word types\n",
      "2019-12-04 18:35:54,710 : INFO : PROGRESS: at sentence #16800000, processed 181191024 words, keeping 59596 word types\n",
      "2019-12-04 18:35:54,875 : INFO : PROGRESS: at sentence #16810000, processed 181303623 words, keeping 59596 word types\n",
      "2019-12-04 18:35:55,042 : INFO : PROGRESS: at sentence #16820000, processed 181416625 words, keeping 59596 word types\n",
      "2019-12-04 18:35:55,216 : INFO : PROGRESS: at sentence #16830000, processed 181536959 words, keeping 59596 word types\n",
      "2019-12-04 18:35:55,392 : INFO : PROGRESS: at sentence #16840000, processed 181657139 words, keeping 59596 word types\n",
      "2019-12-04 18:35:55,563 : INFO : PROGRESS: at sentence #16850000, processed 181778037 words, keeping 59596 word types\n",
      "2019-12-04 18:35:55,749 : INFO : PROGRESS: at sentence #16860000, processed 181901111 words, keeping 59596 word types\n",
      "2019-12-04 18:35:55,913 : INFO : PROGRESS: at sentence #16870000, processed 182018235 words, keeping 59596 word types\n",
      "2019-12-04 18:35:56,064 : INFO : PROGRESS: at sentence #16880000, processed 182125885 words, keeping 59596 word types\n",
      "2019-12-04 18:35:56,205 : INFO : PROGRESS: at sentence #16890000, processed 182225571 words, keeping 59596 word types\n",
      "2019-12-04 18:35:56,350 : INFO : PROGRESS: at sentence #16900000, processed 182327241 words, keeping 59596 word types\n",
      "2019-12-04 18:35:56,495 : INFO : PROGRESS: at sentence #16910000, processed 182430188 words, keeping 59596 word types\n",
      "2019-12-04 18:35:56,631 : INFO : PROGRESS: at sentence #16920000, processed 182527929 words, keeping 59596 word types\n",
      "2019-12-04 18:35:56,777 : INFO : PROGRESS: at sentence #16930000, processed 182629053 words, keeping 59596 word types\n",
      "2019-12-04 18:35:56,914 : INFO : PROGRESS: at sentence #16940000, processed 182723613 words, keeping 59596 word types\n",
      "2019-12-04 18:35:57,054 : INFO : PROGRESS: at sentence #16950000, processed 182820847 words, keeping 59596 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:35:57,201 : INFO : PROGRESS: at sentence #16960000, processed 182918976 words, keeping 59596 word types\n",
      "2019-12-04 18:35:57,349 : INFO : PROGRESS: at sentence #16970000, processed 183021361 words, keeping 59596 word types\n",
      "2019-12-04 18:35:57,501 : INFO : PROGRESS: at sentence #16980000, processed 183126937 words, keeping 59596 word types\n",
      "2019-12-04 18:35:57,637 : INFO : PROGRESS: at sentence #16990000, processed 183231598 words, keeping 59596 word types\n",
      "2019-12-04 18:35:57,772 : INFO : PROGRESS: at sentence #17000000, processed 183336158 words, keeping 59596 word types\n",
      "2019-12-04 18:35:57,909 : INFO : PROGRESS: at sentence #17010000, processed 183441422 words, keeping 59596 word types\n",
      "2019-12-04 18:35:58,044 : INFO : PROGRESS: at sentence #17020000, processed 183545004 words, keeping 59596 word types\n",
      "2019-12-04 18:35:58,177 : INFO : PROGRESS: at sentence #17030000, processed 183645853 words, keeping 59596 word types\n",
      "2019-12-04 18:35:58,312 : INFO : PROGRESS: at sentence #17040000, processed 183748232 words, keeping 59596 word types\n",
      "2019-12-04 18:35:58,464 : INFO : PROGRESS: at sentence #17050000, processed 183866016 words, keeping 59596 word types\n",
      "2019-12-04 18:35:58,621 : INFO : PROGRESS: at sentence #17060000, processed 183987955 words, keeping 59596 word types\n",
      "2019-12-04 18:35:58,768 : INFO : PROGRESS: at sentence #17070000, processed 184103187 words, keeping 59596 word types\n",
      "2019-12-04 18:35:58,912 : INFO : PROGRESS: at sentence #17080000, processed 184213702 words, keeping 59596 word types\n",
      "2019-12-04 18:35:59,050 : INFO : PROGRESS: at sentence #17090000, processed 184319177 words, keeping 59596 word types\n",
      "2019-12-04 18:35:59,192 : INFO : PROGRESS: at sentence #17100000, processed 184424293 words, keeping 59596 word types\n",
      "2019-12-04 18:35:59,337 : INFO : PROGRESS: at sentence #17110000, processed 184532742 words, keeping 59596 word types\n",
      "2019-12-04 18:35:59,479 : INFO : PROGRESS: at sentence #17120000, processed 184640379 words, keeping 59596 word types\n",
      "2019-12-04 18:35:59,622 : INFO : PROGRESS: at sentence #17130000, processed 184747272 words, keeping 59596 word types\n",
      "2019-12-04 18:35:59,766 : INFO : PROGRESS: at sentence #17140000, processed 184855262 words, keeping 59596 word types\n",
      "2019-12-04 18:35:59,909 : INFO : PROGRESS: at sentence #17150000, processed 184965038 words, keeping 59596 word types\n",
      "2019-12-04 18:36:00,057 : INFO : PROGRESS: at sentence #17160000, processed 185075564 words, keeping 59596 word types\n",
      "2019-12-04 18:36:00,211 : INFO : PROGRESS: at sentence #17170000, processed 185191924 words, keeping 59596 word types\n",
      "2019-12-04 18:36:00,368 : INFO : PROGRESS: at sentence #17180000, processed 185301869 words, keeping 59596 word types\n",
      "2019-12-04 18:36:00,529 : INFO : PROGRESS: at sentence #17190000, processed 185416534 words, keeping 59596 word types\n",
      "2019-12-04 18:36:00,682 : INFO : PROGRESS: at sentence #17200000, processed 185526453 words, keeping 59596 word types\n",
      "2019-12-04 18:36:00,852 : INFO : PROGRESS: at sentence #17210000, processed 185652031 words, keeping 59596 word types\n",
      "2019-12-04 18:36:01,015 : INFO : PROGRESS: at sentence #17220000, processed 185772396 words, keeping 59596 word types\n",
      "2019-12-04 18:36:01,175 : INFO : PROGRESS: at sentence #17230000, processed 185894466 words, keeping 59596 word types\n",
      "2019-12-04 18:36:01,331 : INFO : PROGRESS: at sentence #17240000, processed 186013245 words, keeping 59596 word types\n",
      "2019-12-04 18:36:01,487 : INFO : PROGRESS: at sentence #17250000, processed 186132381 words, keeping 59596 word types\n",
      "2019-12-04 18:36:01,620 : INFO : PROGRESS: at sentence #17260000, processed 186233114 words, keeping 59596 word types\n",
      "2019-12-04 18:36:01,755 : INFO : PROGRESS: at sentence #17270000, processed 186332992 words, keeping 59596 word types\n",
      "2019-12-04 18:36:01,888 : INFO : PROGRESS: at sentence #17280000, processed 186434852 words, keeping 59596 word types\n",
      "2019-12-04 18:36:02,025 : INFO : PROGRESS: at sentence #17290000, processed 186539972 words, keeping 59596 word types\n",
      "2019-12-04 18:36:02,151 : INFO : PROGRESS: at sentence #17300000, processed 186635841 words, keeping 59596 word types\n",
      "2019-12-04 18:36:02,280 : INFO : PROGRESS: at sentence #17310000, processed 186733878 words, keeping 59596 word types\n",
      "2019-12-04 18:36:02,408 : INFO : PROGRESS: at sentence #17320000, processed 186830318 words, keeping 59596 word types\n",
      "2019-12-04 18:36:02,538 : INFO : PROGRESS: at sentence #17330000, processed 186926312 words, keeping 59596 word types\n",
      "2019-12-04 18:36:02,670 : INFO : PROGRESS: at sentence #17340000, processed 187025839 words, keeping 59596 word types\n",
      "2019-12-04 18:36:02,807 : INFO : PROGRESS: at sentence #17350000, processed 187131110 words, keeping 59596 word types\n",
      "2019-12-04 18:36:02,945 : INFO : PROGRESS: at sentence #17360000, processed 187236099 words, keeping 59596 word types\n",
      "2019-12-04 18:36:03,084 : INFO : PROGRESS: at sentence #17370000, processed 187340660 words, keeping 59596 word types\n",
      "2019-12-04 18:36:03,223 : INFO : PROGRESS: at sentence #17380000, processed 187445529 words, keeping 59596 word types\n",
      "2019-12-04 18:36:03,360 : INFO : PROGRESS: at sentence #17390000, processed 187551606 words, keeping 59596 word types\n",
      "2019-12-04 18:36:03,491 : INFO : PROGRESS: at sentence #17400000, processed 187652137 words, keeping 59596 word types\n",
      "2019-12-04 18:36:03,628 : INFO : PROGRESS: at sentence #17410000, processed 187754469 words, keeping 59596 word types\n",
      "2019-12-04 18:36:03,766 : INFO : PROGRESS: at sentence #17420000, processed 187856919 words, keeping 59596 word types\n",
      "2019-12-04 18:36:03,933 : INFO : PROGRESS: at sentence #17430000, processed 187984427 words, keeping 59596 word types\n",
      "2019-12-04 18:36:04,082 : INFO : PROGRESS: at sentence #17440000, processed 188101412 words, keeping 59596 word types\n",
      "2019-12-04 18:36:04,225 : INFO : PROGRESS: at sentence #17450000, processed 188213825 words, keeping 59596 word types\n",
      "2019-12-04 18:36:04,365 : INFO : PROGRESS: at sentence #17460000, processed 188322675 words, keeping 59596 word types\n",
      "2019-12-04 18:36:04,503 : INFO : PROGRESS: at sentence #17470000, processed 188428535 words, keeping 59596 word types\n",
      "2019-12-04 18:36:04,643 : INFO : PROGRESS: at sentence #17480000, processed 188535864 words, keeping 59596 word types\n",
      "2019-12-04 18:36:04,782 : INFO : PROGRESS: at sentence #17490000, processed 188642521 words, keeping 59596 word types\n",
      "2019-12-04 18:36:04,921 : INFO : PROGRESS: at sentence #17500000, processed 188749922 words, keeping 59596 word types\n",
      "2019-12-04 18:36:05,061 : INFO : PROGRESS: at sentence #17510000, processed 188857031 words, keeping 59596 word types\n",
      "2019-12-04 18:36:05,200 : INFO : PROGRESS: at sentence #17520000, processed 188965410 words, keeping 59596 word types\n",
      "2019-12-04 18:36:05,346 : INFO : PROGRESS: at sentence #17530000, processed 189076077 words, keeping 59596 word types\n",
      "2019-12-04 18:36:05,493 : INFO : PROGRESS: at sentence #17540000, processed 189189076 words, keeping 59596 word types\n",
      "2019-12-04 18:36:05,640 : INFO : PROGRESS: at sentence #17550000, processed 189302476 words, keeping 59596 word types\n",
      "2019-12-04 18:36:05,787 : INFO : PROGRESS: at sentence #17560000, processed 189414986 words, keeping 59596 word types\n",
      "2019-12-04 18:36:05,935 : INFO : PROGRESS: at sentence #17570000, processed 189529416 words, keeping 59596 word types\n",
      "2019-12-04 18:36:06,088 : INFO : PROGRESS: at sentence #17580000, processed 189645533 words, keeping 59596 word types\n",
      "2019-12-04 18:36:06,244 : INFO : PROGRESS: at sentence #17590000, processed 189766949 words, keeping 59596 word types\n",
      "2019-12-04 18:36:06,399 : INFO : PROGRESS: at sentence #17600000, processed 189887527 words, keeping 59596 word types\n",
      "2019-12-04 18:36:06,559 : INFO : PROGRESS: at sentence #17610000, processed 190010993 words, keeping 59596 word types\n",
      "2019-12-04 18:36:06,707 : INFO : PROGRESS: at sentence #17620000, processed 190126934 words, keeping 59596 word types\n",
      "2019-12-04 18:36:06,852 : INFO : PROGRESS: at sentence #17630000, processed 190240794 words, keeping 59596 word types\n",
      "2019-12-04 18:36:06,980 : INFO : PROGRESS: at sentence #17640000, processed 190340090 words, keeping 59596 word types\n",
      "2019-12-04 18:36:07,110 : INFO : PROGRESS: at sentence #17650000, processed 190441480 words, keeping 59596 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:36:07,242 : INFO : PROGRESS: at sentence #17660000, processed 190543912 words, keeping 59596 word types\n",
      "2019-12-04 18:36:07,371 : INFO : PROGRESS: at sentence #17670000, processed 190643188 words, keeping 59596 word types\n",
      "2019-12-04 18:36:07,501 : INFO : PROGRESS: at sentence #17680000, processed 190743499 words, keeping 59596 word types\n",
      "2019-12-04 18:36:07,625 : INFO : PROGRESS: at sentence #17690000, processed 190838224 words, keeping 59596 word types\n",
      "2019-12-04 18:36:07,753 : INFO : PROGRESS: at sentence #17700000, processed 190936104 words, keeping 59596 word types\n",
      "2019-12-04 18:36:07,881 : INFO : PROGRESS: at sentence #17710000, processed 191033175 words, keeping 59596 word types\n",
      "2019-12-04 18:36:08,014 : INFO : PROGRESS: at sentence #17720000, processed 191134436 words, keeping 59596 word types\n",
      "2019-12-04 18:36:08,152 : INFO : PROGRESS: at sentence #17730000, processed 191240672 words, keeping 59596 word types\n",
      "2019-12-04 18:36:08,287 : INFO : PROGRESS: at sentence #17740000, processed 191345185 words, keeping 59596 word types\n",
      "2019-12-04 18:36:08,422 : INFO : PROGRESS: at sentence #17750000, processed 191449412 words, keeping 59596 word types\n",
      "2019-12-04 18:36:08,558 : INFO : PROGRESS: at sentence #17760000, processed 191554415 words, keeping 59596 word types\n",
      "2019-12-04 18:36:08,696 : INFO : PROGRESS: at sentence #17770000, processed 191659616 words, keeping 59596 word types\n",
      "2019-12-04 18:36:08,827 : INFO : PROGRESS: at sentence #17780000, processed 191759647 words, keeping 59596 word types\n",
      "2019-12-04 18:36:08,962 : INFO : PROGRESS: at sentence #17790000, processed 191862724 words, keeping 59596 word types\n",
      "2019-12-04 18:36:09,105 : INFO : PROGRESS: at sentence #17800000, processed 191972710 words, keeping 59596 word types\n",
      "2019-12-04 18:36:09,266 : INFO : PROGRESS: at sentence #17810000, processed 192099035 words, keeping 59596 word types\n",
      "2019-12-04 18:36:09,414 : INFO : PROGRESS: at sentence #17820000, processed 192214920 words, keeping 59596 word types\n",
      "2019-12-04 18:36:09,558 : INFO : PROGRESS: at sentence #17830000, processed 192326846 words, keeping 59596 word types\n",
      "2019-12-04 18:36:09,695 : INFO : PROGRESS: at sentence #17840000, processed 192431224 words, keeping 59596 word types\n",
      "2019-12-04 18:36:09,834 : INFO : PROGRESS: at sentence #17850000, processed 192537302 words, keeping 59596 word types\n",
      "2019-12-04 18:36:09,975 : INFO : PROGRESS: at sentence #17860000, processed 192645644 words, keeping 59596 word types\n",
      "2019-12-04 18:36:10,113 : INFO : PROGRESS: at sentence #17870000, processed 192752931 words, keeping 59596 word types\n",
      "2019-12-04 18:36:10,252 : INFO : PROGRESS: at sentence #17880000, processed 192860163 words, keeping 59596 word types\n",
      "2019-12-04 18:36:10,399 : INFO : PROGRESS: at sentence #17890000, processed 192968529 words, keeping 59596 word types\n",
      "2019-12-04 18:36:10,543 : INFO : PROGRESS: at sentence #17900000, processed 193077139 words, keeping 59596 word types\n",
      "2019-12-04 18:36:10,686 : INFO : PROGRESS: at sentence #17910000, processed 193187497 words, keeping 59596 word types\n",
      "2019-12-04 18:36:10,837 : INFO : PROGRESS: at sentence #17920000, processed 193303876 words, keeping 59596 word types\n",
      "2019-12-04 18:36:10,984 : INFO : PROGRESS: at sentence #17930000, processed 193413994 words, keeping 59596 word types\n",
      "2019-12-04 18:36:11,137 : INFO : PROGRESS: at sentence #17940000, processed 193528166 words, keeping 59596 word types\n",
      "2019-12-04 18:36:11,285 : INFO : PROGRESS: at sentence #17950000, processed 193639456 words, keeping 59596 word types\n",
      "2019-12-04 18:36:11,444 : INFO : PROGRESS: at sentence #17960000, processed 193761666 words, keeping 59596 word types\n",
      "2019-12-04 18:36:11,601 : INFO : PROGRESS: at sentence #17970000, processed 193881674 words, keeping 59596 word types\n",
      "2019-12-04 18:36:11,762 : INFO : PROGRESS: at sentence #17980000, processed 194003161 words, keeping 59596 word types\n",
      "2019-12-04 18:36:11,917 : INFO : PROGRESS: at sentence #17990000, processed 194125037 words, keeping 59596 word types\n",
      "2019-12-04 18:36:12,068 : INFO : PROGRESS: at sentence #18000000, processed 194242295 words, keeping 59596 word types\n",
      "2019-12-04 18:36:12,206 : INFO : PROGRESS: at sentence #18010000, processed 194348106 words, keeping 59596 word types\n",
      "2019-12-04 18:36:12,337 : INFO : PROGRESS: at sentence #18020000, processed 194446876 words, keeping 59596 word types\n",
      "2019-12-04 18:36:12,472 : INFO : PROGRESS: at sentence #18030000, processed 194549255 words, keeping 59596 word types\n",
      "2019-12-04 18:36:12,608 : INFO : PROGRESS: at sentence #18040000, processed 194652881 words, keeping 59596 word types\n",
      "2019-12-04 18:36:12,734 : INFO : PROGRESS: at sentence #18050000, processed 194749704 words, keeping 59596 word types\n",
      "2019-12-04 18:36:12,865 : INFO : PROGRESS: at sentence #18060000, processed 194850055 words, keeping 59596 word types\n",
      "2019-12-04 18:36:12,993 : INFO : PROGRESS: at sentence #18070000, processed 194945320 words, keeping 59596 word types\n",
      "2019-12-04 18:36:13,122 : INFO : PROGRESS: at sentence #18080000, processed 195041551 words, keeping 59596 word types\n",
      "2019-12-04 18:36:13,252 : INFO : PROGRESS: at sentence #18090000, processed 195140652 words, keeping 59596 word types\n",
      "2019-12-04 18:36:13,387 : INFO : PROGRESS: at sentence #18100000, processed 195244068 words, keeping 59596 word types\n",
      "2019-12-04 18:36:13,529 : INFO : PROGRESS: at sentence #18110000, processed 195349592 words, keeping 59596 word types\n",
      "2019-12-04 18:36:13,708 : INFO : PROGRESS: at sentence #18120000, processed 195453995 words, keeping 59596 word types\n",
      "2019-12-04 18:36:13,846 : INFO : PROGRESS: at sentence #18130000, processed 195559084 words, keeping 59596 word types\n",
      "2019-12-04 18:36:13,985 : INFO : PROGRESS: at sentence #18140000, processed 195664744 words, keeping 59596 word types\n",
      "2019-12-04 18:36:14,120 : INFO : PROGRESS: at sentence #18150000, processed 195766829 words, keeping 59596 word types\n",
      "2019-12-04 18:36:14,258 : INFO : PROGRESS: at sentence #18160000, processed 195868397 words, keeping 59596 word types\n",
      "2019-12-04 18:36:14,393 : INFO : PROGRESS: at sentence #18170000, processed 195970201 words, keeping 59596 word types\n",
      "2019-12-04 18:36:14,551 : INFO : PROGRESS: at sentence #18180000, processed 196092117 words, keeping 59596 word types\n",
      "2019-12-04 18:36:14,705 : INFO : PROGRESS: at sentence #18190000, processed 196212064 words, keeping 59596 word types\n",
      "2019-12-04 18:36:14,853 : INFO : PROGRESS: at sentence #18200000, processed 196326341 words, keeping 59596 word types\n",
      "2019-12-04 18:36:14,996 : INFO : PROGRESS: at sentence #18210000, processed 196436433 words, keeping 59596 word types\n",
      "2019-12-04 18:36:15,135 : INFO : PROGRESS: at sentence #18220000, processed 196542184 words, keeping 59596 word types\n",
      "2019-12-04 18:36:15,315 : INFO : PROGRESS: at sentence #18230000, processed 196648642 words, keeping 59596 word types\n",
      "2019-12-04 18:36:15,455 : INFO : PROGRESS: at sentence #18240000, processed 196755501 words, keeping 59596 word types\n",
      "2019-12-04 18:36:15,596 : INFO : PROGRESS: at sentence #18250000, processed 196863028 words, keeping 59596 word types\n",
      "2019-12-04 18:36:15,738 : INFO : PROGRESS: at sentence #18260000, processed 196969865 words, keeping 59596 word types\n",
      "2019-12-04 18:36:15,881 : INFO : PROGRESS: at sentence #18270000, processed 197078816 words, keeping 59596 word types\n",
      "2019-12-04 18:36:16,026 : INFO : PROGRESS: at sentence #18280000, processed 197188572 words, keeping 59596 word types\n",
      "2019-12-04 18:36:16,172 : INFO : PROGRESS: at sentence #18290000, processed 197299489 words, keeping 59596 word types\n",
      "2019-12-04 18:36:16,323 : INFO : PROGRESS: at sentence #18300000, processed 197415145 words, keeping 59596 word types\n",
      "2019-12-04 18:36:16,471 : INFO : PROGRESS: at sentence #18310000, processed 197526005 words, keeping 59596 word types\n",
      "2019-12-04 18:36:16,624 : INFO : PROGRESS: at sentence #18320000, processed 197640940 words, keeping 59596 word types\n",
      "2019-12-04 18:36:16,769 : INFO : PROGRESS: at sentence #18330000, processed 197749297 words, keeping 59596 word types\n",
      "2019-12-04 18:36:16,934 : INFO : PROGRESS: at sentence #18340000, processed 197877314 words, keeping 59596 word types\n",
      "2019-12-04 18:36:17,092 : INFO : PROGRESS: at sentence #18350000, processed 197997858 words, keeping 59596 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:36:17,252 : INFO : PROGRESS: at sentence #18360000, processed 198119799 words, keeping 59596 word types\n",
      "2019-12-04 18:36:17,404 : INFO : PROGRESS: at sentence #18370000, processed 198237384 words, keeping 59596 word types\n",
      "2019-12-04 18:36:17,557 : INFO : PROGRESS: at sentence #18380000, processed 198356081 words, keeping 59596 word types\n",
      "2019-12-04 18:36:17,687 : INFO : PROGRESS: at sentence #18390000, processed 198454934 words, keeping 59596 word types\n",
      "2019-12-04 18:36:17,819 : INFO : PROGRESS: at sentence #18400000, processed 198555192 words, keeping 59596 word types\n",
      "2019-12-04 18:36:17,953 : INFO : PROGRESS: at sentence #18410000, processed 198657053 words, keeping 59596 word types\n",
      "2019-12-04 18:36:18,090 : INFO : PROGRESS: at sentence #18420000, processed 198762279 words, keeping 59596 word types\n",
      "2019-12-04 18:36:18,216 : INFO : PROGRESS: at sentence #18430000, processed 198857481 words, keeping 59596 word types\n",
      "2019-12-04 18:36:18,345 : INFO : PROGRESS: at sentence #18440000, processed 198954379 words, keeping 59596 word types\n",
      "2019-12-04 18:36:18,475 : INFO : PROGRESS: at sentence #18450000, processed 199051792 words, keeping 59596 word types\n",
      "2019-12-04 18:36:18,604 : INFO : PROGRESS: at sentence #18460000, processed 199147934 words, keeping 59596 word types\n",
      "2019-12-04 18:36:18,736 : INFO : PROGRESS: at sentence #18470000, processed 199247160 words, keeping 59596 word types\n",
      "2019-12-04 18:36:18,876 : INFO : PROGRESS: at sentence #18480000, processed 199353750 words, keeping 59596 word types\n",
      "2019-12-04 18:36:19,013 : INFO : PROGRESS: at sentence #18490000, processed 199458623 words, keeping 59596 word types\n",
      "2019-12-04 18:36:19,152 : INFO : PROGRESS: at sentence #18500000, processed 199562483 words, keeping 59596 word types\n",
      "2019-12-04 18:36:19,293 : INFO : PROGRESS: at sentence #18510000, processed 199667737 words, keeping 59596 word types\n",
      "2019-12-04 18:36:19,436 : INFO : PROGRESS: at sentence #18520000, processed 199773661 words, keeping 59596 word types\n",
      "2019-12-04 18:36:19,572 : INFO : PROGRESS: at sentence #18530000, processed 199873802 words, keeping 59596 word types\n",
      "2019-12-04 18:36:19,709 : INFO : PROGRESS: at sentence #18540000, processed 199976621 words, keeping 59596 word types\n",
      "2019-12-04 18:36:19,847 : INFO : PROGRESS: at sentence #18550000, processed 200079657 words, keeping 59596 word types\n",
      "2019-12-04 18:36:20,012 : INFO : PROGRESS: at sentence #18560000, processed 200209785 words, keeping 59596 word types\n",
      "2019-12-04 18:36:20,161 : INFO : PROGRESS: at sentence #18570000, processed 200325015 words, keeping 59596 word types\n",
      "2019-12-04 18:36:20,307 : INFO : PROGRESS: at sentence #18580000, processed 200437579 words, keeping 59596 word types\n",
      "2019-12-04 18:36:20,447 : INFO : PROGRESS: at sentence #18590000, processed 200544925 words, keeping 59596 word types\n",
      "2019-12-04 18:36:20,588 : INFO : PROGRESS: at sentence #18600000, processed 200651188 words, keeping 59596 word types\n",
      "2019-12-04 18:36:20,730 : INFO : PROGRESS: at sentence #18610000, processed 200758744 words, keeping 59596 word types\n",
      "2019-12-04 18:36:20,870 : INFO : PROGRESS: at sentence #18620000, processed 200865149 words, keeping 59596 word types\n",
      "2019-12-04 18:36:21,012 : INFO : PROGRESS: at sentence #18630000, processed 200972835 words, keeping 59596 word types\n",
      "2019-12-04 18:36:21,155 : INFO : PROGRESS: at sentence #18640000, processed 201080510 words, keeping 59596 word types\n",
      "2019-12-04 18:36:21,297 : INFO : PROGRESS: at sentence #18650000, processed 201189221 words, keeping 59596 word types\n",
      "2019-12-04 18:36:21,442 : INFO : PROGRESS: at sentence #18660000, processed 201299554 words, keeping 59596 word types\n",
      "2019-12-04 18:36:21,624 : INFO : PROGRESS: at sentence #18670000, processed 201414291 words, keeping 59596 word types\n",
      "2019-12-04 18:36:21,813 : INFO : PROGRESS: at sentence #18680000, processed 201526328 words, keeping 59596 word types\n",
      "2019-12-04 18:36:21,963 : INFO : PROGRESS: at sentence #18690000, processed 201638945 words, keeping 59596 word types\n",
      "2019-12-04 18:36:22,113 : INFO : PROGRESS: at sentence #18700000, processed 201752397 words, keeping 59596 word types\n",
      "2019-12-04 18:36:22,268 : INFO : PROGRESS: at sentence #18710000, processed 201871538 words, keeping 59596 word types\n",
      "2019-12-04 18:36:22,424 : INFO : PROGRESS: at sentence #18720000, processed 201991645 words, keeping 59596 word types\n",
      "2019-12-04 18:36:22,582 : INFO : PROGRESS: at sentence #18730000, processed 202112908 words, keeping 59596 word types\n",
      "2019-12-04 18:36:22,740 : INFO : PROGRESS: at sentence #18740000, processed 202235380 words, keeping 59596 word types\n",
      "2019-12-04 18:36:22,891 : INFO : PROGRESS: at sentence #18750000, processed 202352902 words, keeping 59596 word types\n",
      "2019-12-04 18:36:23,034 : INFO : PROGRESS: at sentence #18760000, processed 202462268 words, keeping 59596 word types\n",
      "2019-12-04 18:36:23,165 : INFO : PROGRESS: at sentence #18770000, processed 202562140 words, keeping 59596 word types\n",
      "2019-12-04 18:36:23,296 : INFO : PROGRESS: at sentence #18780000, processed 202663365 words, keeping 59596 word types\n",
      "2019-12-04 18:36:23,430 : INFO : PROGRESS: at sentence #18790000, processed 202765981 words, keeping 59596 word types\n",
      "2019-12-04 18:36:23,556 : INFO : PROGRESS: at sentence #18800000, processed 202863772 words, keeping 59596 word types\n",
      "2019-12-04 18:36:23,690 : INFO : PROGRESS: at sentence #18810000, processed 202965939 words, keeping 59596 word types\n",
      "2019-12-04 18:36:23,814 : INFO : PROGRESS: at sentence #18820000, processed 203060488 words, keeping 59596 word types\n",
      "2019-12-04 18:36:23,943 : INFO : PROGRESS: at sentence #18830000, processed 203157338 words, keeping 59596 word types\n",
      "2019-12-04 18:36:24,073 : INFO : PROGRESS: at sentence #18840000, processed 203255141 words, keeping 59596 word types\n",
      "2019-12-04 18:36:24,207 : INFO : PROGRESS: at sentence #18850000, processed 203357292 words, keeping 59596 word types\n",
      "2019-12-04 18:36:24,345 : INFO : PROGRESS: at sentence #18860000, processed 203462910 words, keeping 59596 word types\n",
      "2019-12-04 18:36:24,481 : INFO : PROGRESS: at sentence #18870000, processed 203567460 words, keeping 59596 word types\n",
      "2019-12-04 18:36:24,617 : INFO : PROGRESS: at sentence #18880000, processed 203672136 words, keeping 59596 word types\n",
      "2019-12-04 18:36:24,754 : INFO : PROGRESS: at sentence #18890000, processed 203777599 words, keeping 59596 word types\n",
      "2019-12-04 18:36:24,890 : INFO : PROGRESS: at sentence #18900000, processed 203881075 words, keeping 59596 word types\n",
      "2019-12-04 18:36:25,024 : INFO : PROGRESS: at sentence #18910000, processed 203981667 words, keeping 59596 word types\n",
      "2019-12-04 18:36:25,160 : INFO : PROGRESS: at sentence #18920000, processed 204084861 words, keeping 59596 word types\n",
      "2019-12-04 18:36:25,309 : INFO : PROGRESS: at sentence #18930000, processed 204199751 words, keeping 59596 word types\n",
      "2019-12-04 18:36:25,467 : INFO : PROGRESS: at sentence #18940000, processed 204322795 words, keeping 59596 word types\n",
      "2019-12-04 18:36:25,616 : INFO : PROGRESS: at sentence #18950000, processed 204438813 words, keeping 59596 word types\n",
      "2019-12-04 18:36:25,760 : INFO : PROGRESS: at sentence #18960000, processed 204550098 words, keeping 59596 word types\n",
      "2019-12-04 18:36:25,897 : INFO : PROGRESS: at sentence #18970000, processed 204653485 words, keeping 59596 word types\n",
      "2019-12-04 18:36:26,037 : INFO : PROGRESS: at sentence #18980000, processed 204759960 words, keeping 59596 word types\n",
      "2019-12-04 18:36:26,180 : INFO : PROGRESS: at sentence #18990000, processed 204868351 words, keeping 59596 word types\n",
      "2019-12-04 18:36:26,322 : INFO : PROGRESS: at sentence #19000000, processed 204975986 words, keeping 59596 word types\n",
      "2019-12-04 18:36:26,464 : INFO : PROGRESS: at sentence #19010000, processed 205083015 words, keeping 59596 word types\n",
      "2019-12-04 18:36:26,608 : INFO : PROGRESS: at sentence #19020000, processed 205191194 words, keeping 59596 word types\n",
      "2019-12-04 18:36:26,752 : INFO : PROGRESS: at sentence #19030000, processed 205300710 words, keeping 59596 word types\n",
      "2019-12-04 18:36:26,894 : INFO : PROGRESS: at sentence #19040000, processed 205410890 words, keeping 59596 word types\n",
      "2019-12-04 18:36:27,046 : INFO : PROGRESS: at sentence #19050000, processed 205527390 words, keeping 59596 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:36:27,191 : INFO : PROGRESS: at sentence #19060000, processed 205637326 words, keeping 59596 word types\n",
      "2019-12-04 18:36:27,341 : INFO : PROGRESS: at sentence #19070000, processed 205751713 words, keeping 59596 word types\n",
      "2019-12-04 18:36:27,487 : INFO : PROGRESS: at sentence #19080000, processed 205861917 words, keeping 59596 word types\n",
      "2019-12-04 18:36:27,650 : INFO : PROGRESS: at sentence #19090000, processed 205986792 words, keeping 59596 word types\n",
      "2019-12-04 18:36:27,806 : INFO : PROGRESS: at sentence #19100000, processed 206106971 words, keeping 59596 word types\n",
      "2019-12-04 18:36:27,965 : INFO : PROGRESS: at sentence #19110000, processed 206228869 words, keeping 59596 word types\n",
      "2019-12-04 18:36:28,120 : INFO : PROGRESS: at sentence #19120000, processed 206348737 words, keeping 59596 word types\n",
      "2019-12-04 18:36:28,271 : INFO : PROGRESS: at sentence #19130000, processed 206467166 words, keeping 59596 word types\n",
      "2019-12-04 18:36:28,405 : INFO : PROGRESS: at sentence #19140000, processed 206569556 words, keeping 59596 word types\n",
      "2019-12-04 18:36:28,536 : INFO : PROGRESS: at sentence #19150000, processed 206668961 words, keeping 59596 word types\n",
      "2019-12-04 18:36:28,670 : INFO : PROGRESS: at sentence #19160000, processed 206771071 words, keeping 59596 word types\n",
      "2019-12-04 18:36:28,805 : INFO : PROGRESS: at sentence #19170000, processed 206875816 words, keeping 59596 word types\n",
      "2019-12-04 18:36:28,929 : INFO : PROGRESS: at sentence #19180000, processed 206970524 words, keeping 59596 word types\n",
      "2019-12-04 18:36:29,057 : INFO : PROGRESS: at sentence #19190000, processed 207070627 words, keeping 59596 word types\n",
      "2019-12-04 18:36:29,184 : INFO : PROGRESS: at sentence #19200000, processed 207166619 words, keeping 59596 word types\n",
      "2019-12-04 18:36:29,312 : INFO : PROGRESS: at sentence #19210000, processed 207262876 words, keeping 59596 word types\n",
      "2019-12-04 18:36:29,444 : INFO : PROGRESS: at sentence #19220000, processed 207362323 words, keeping 59596 word types\n",
      "2019-12-04 18:36:29,580 : INFO : PROGRESS: at sentence #19230000, processed 207466965 words, keeping 59596 word types\n",
      "2019-12-04 18:36:29,717 : INFO : PROGRESS: at sentence #19240000, processed 207571858 words, keeping 59596 word types\n",
      "2019-12-04 18:36:29,855 : INFO : PROGRESS: at sentence #19250000, processed 207676725 words, keeping 59596 word types\n",
      "2019-12-04 18:36:29,991 : INFO : PROGRESS: at sentence #19260000, processed 207781534 words, keeping 59596 word types\n",
      "2019-12-04 18:36:30,128 : INFO : PROGRESS: at sentence #19270000, processed 207887892 words, keeping 59596 word types\n",
      "2019-12-04 18:36:30,258 : INFO : PROGRESS: at sentence #19280000, processed 207988734 words, keeping 59596 word types\n",
      "2019-12-04 18:36:30,390 : INFO : PROGRESS: at sentence #19290000, processed 208090474 words, keeping 59596 word types\n",
      "2019-12-04 18:36:30,525 : INFO : PROGRESS: at sentence #19300000, processed 208192907 words, keeping 59596 word types\n",
      "2019-12-04 18:36:30,683 : INFO : PROGRESS: at sentence #19310000, processed 208318662 words, keeping 59596 word types\n",
      "2019-12-04 18:36:30,833 : INFO : PROGRESS: at sentence #19320000, processed 208436482 words, keeping 59596 word types\n",
      "2019-12-04 18:36:30,979 : INFO : PROGRESS: at sentence #19330000, processed 208549389 words, keeping 59596 word types\n",
      "2019-12-04 18:36:31,121 : INFO : PROGRESS: at sentence #19340000, processed 208658720 words, keeping 59596 word types\n",
      "2019-12-04 18:36:31,259 : INFO : PROGRESS: at sentence #19350000, processed 208764747 words, keeping 59596 word types\n",
      "2019-12-04 18:36:31,396 : INFO : PROGRESS: at sentence #19360000, processed 208871587 words, keeping 59596 word types\n",
      "2019-12-04 18:36:31,532 : INFO : PROGRESS: at sentence #19370000, processed 208978307 words, keeping 59596 word types\n",
      "2019-12-04 18:36:31,671 : INFO : PROGRESS: at sentence #19380000, processed 209085646 words, keeping 59596 word types\n",
      "2019-12-04 18:36:31,810 : INFO : PROGRESS: at sentence #19390000, processed 209193053 words, keeping 59596 word types\n",
      "2019-12-04 18:36:31,953 : INFO : PROGRESS: at sentence #19400000, processed 209301602 words, keeping 59596 word types\n",
      "2019-12-04 18:36:32,095 : INFO : PROGRESS: at sentence #19410000, processed 209411578 words, keeping 59596 word types\n",
      "2019-12-04 18:36:32,240 : INFO : PROGRESS: at sentence #19420000, processed 209523871 words, keeping 59596 word types\n",
      "2019-12-04 18:36:32,389 : INFO : PROGRESS: at sentence #19430000, processed 209638199 words, keeping 59596 word types\n",
      "2019-12-04 18:36:32,535 : INFO : PROGRESS: at sentence #19440000, processed 209750088 words, keeping 59596 word types\n",
      "2019-12-04 18:36:32,682 : INFO : PROGRESS: at sentence #19450000, processed 209864618 words, keeping 59596 word types\n",
      "2019-12-04 18:36:32,833 : INFO : PROGRESS: at sentence #19460000, processed 209978689 words, keeping 59596 word types\n",
      "2019-12-04 18:36:32,989 : INFO : PROGRESS: at sentence #19470000, processed 210101877 words, keeping 59596 word types\n",
      "2019-12-04 18:36:33,143 : INFO : PROGRESS: at sentence #19480000, processed 210222218 words, keeping 59596 word types\n",
      "2019-12-04 18:36:33,299 : INFO : PROGRESS: at sentence #19490000, processed 210345337 words, keeping 59596 word types\n",
      "2019-12-04 18:36:33,445 : INFO : PROGRESS: at sentence #19500000, processed 210460555 words, keeping 59596 word types\n",
      "2019-12-04 18:36:33,593 : INFO : PROGRESS: at sentence #19510000, processed 210577242 words, keeping 59596 word types\n",
      "2019-12-04 18:36:33,722 : INFO : PROGRESS: at sentence #19520000, processed 210676689 words, keeping 59596 word types\n",
      "2019-12-04 18:36:33,851 : INFO : PROGRESS: at sentence #19530000, processed 210777569 words, keeping 59596 word types\n",
      "2019-12-04 18:36:33,986 : INFO : PROGRESS: at sentence #19540000, processed 210879831 words, keeping 59596 word types\n",
      "2019-12-04 18:36:34,119 : INFO : PROGRESS: at sentence #19550000, processed 210983526 words, keeping 59596 word types\n",
      "2019-12-04 18:36:34,242 : INFO : PROGRESS: at sentence #19560000, processed 211079441 words, keeping 59596 word types\n",
      "2019-12-04 18:36:34,366 : INFO : PROGRESS: at sentence #19570000, processed 211175289 words, keeping 59596 word types\n",
      "2019-12-04 18:36:34,491 : INFO : PROGRESS: at sentence #19580000, processed 211272904 words, keeping 59596 word types\n",
      "2019-12-04 18:36:34,617 : INFO : PROGRESS: at sentence #19590000, processed 211369815 words, keeping 59596 word types\n",
      "2019-12-04 18:36:34,747 : INFO : PROGRESS: at sentence #19600000, processed 211470363 words, keeping 59596 word types\n",
      "2019-12-04 18:36:34,883 : INFO : PROGRESS: at sentence #19610000, processed 211576458 words, keeping 59596 word types\n",
      "2019-12-04 18:36:35,017 : INFO : PROGRESS: at sentence #19620000, processed 211681272 words, keeping 59596 word types\n",
      "2019-12-04 18:36:35,147 : INFO : PROGRESS: at sentence #19630000, processed 211784956 words, keeping 59596 word types\n",
      "2019-12-04 18:36:35,275 : INFO : PROGRESS: at sentence #19640000, processed 211890133 words, keeping 59596 word types\n",
      "2019-12-04 18:36:35,406 : INFO : PROGRESS: at sentence #19650000, processed 211995983 words, keeping 59596 word types\n",
      "2019-12-04 18:36:35,531 : INFO : PROGRESS: at sentence #19660000, processed 212095831 words, keeping 59596 word types\n",
      "2019-12-04 18:36:35,662 : INFO : PROGRESS: at sentence #19670000, processed 212198858 words, keeping 59596 word types\n",
      "2019-12-04 18:36:35,799 : INFO : PROGRESS: at sentence #19680000, processed 212306284 words, keeping 59596 word types\n",
      "2019-12-04 18:36:35,959 : INFO : PROGRESS: at sentence #19690000, processed 212433998 words, keeping 59596 word types\n",
      "2019-12-04 18:36:36,103 : INFO : PROGRESS: at sentence #19700000, processed 212549396 words, keeping 59596 word types\n",
      "2019-12-04 18:36:36,239 : INFO : PROGRESS: at sentence #19710000, processed 212660531 words, keeping 59596 word types\n",
      "2019-12-04 18:36:36,375 : INFO : PROGRESS: at sentence #19720000, processed 212767226 words, keeping 59596 word types\n",
      "2019-12-04 18:36:36,509 : INFO : PROGRESS: at sentence #19730000, processed 212873381 words, keeping 59596 word types\n",
      "2019-12-04 18:36:36,645 : INFO : PROGRESS: at sentence #19740000, processed 212981556 words, keeping 59596 word types\n",
      "2019-12-04 18:36:36,781 : INFO : PROGRESS: at sentence #19750000, processed 213088188 words, keeping 59596 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:36:36,918 : INFO : PROGRESS: at sentence #19760000, processed 213195992 words, keeping 59596 word types\n",
      "2019-12-04 18:36:37,056 : INFO : PROGRESS: at sentence #19770000, processed 213304105 words, keeping 59596 word types\n",
      "2019-12-04 18:36:37,191 : INFO : PROGRESS: at sentence #19780000, processed 213412637 words, keeping 59596 word types\n",
      "2019-12-04 18:36:37,332 : INFO : PROGRESS: at sentence #19790000, processed 213522843 words, keeping 59596 word types\n",
      "2019-12-04 18:36:37,482 : INFO : PROGRESS: at sentence #19800000, processed 213638715 words, keeping 59596 word types\n",
      "2019-12-04 18:36:37,628 : INFO : PROGRESS: at sentence #19810000, processed 213749612 words, keeping 59596 word types\n",
      "2019-12-04 18:36:37,777 : INFO : PROGRESS: at sentence #19820000, processed 213862754 words, keeping 59596 word types\n",
      "2019-12-04 18:36:37,925 : INFO : PROGRESS: at sentence #19830000, processed 213975211 words, keeping 59596 word types\n",
      "2019-12-04 18:36:38,083 : INFO : PROGRESS: at sentence #19840000, processed 214096577 words, keeping 59596 word types\n",
      "2019-12-04 18:36:38,241 : INFO : PROGRESS: at sentence #19850000, processed 214216626 words, keeping 59596 word types\n",
      "2019-12-04 18:36:38,398 : INFO : PROGRESS: at sentence #19860000, processed 214337684 words, keeping 59596 word types\n",
      "2019-12-04 18:36:38,555 : INFO : PROGRESS: at sentence #19870000, processed 214460396 words, keeping 59596 word types\n",
      "2019-12-04 18:36:38,706 : INFO : PROGRESS: at sentence #19880000, processed 214577123 words, keeping 59596 word types\n",
      "2019-12-04 18:36:38,844 : INFO : PROGRESS: at sentence #19890000, processed 214684201 words, keeping 59596 word types\n",
      "2019-12-04 18:36:38,973 : INFO : PROGRESS: at sentence #19900000, processed 214783265 words, keeping 59596 word types\n",
      "2019-12-04 18:36:39,106 : INFO : PROGRESS: at sentence #19910000, processed 214885540 words, keeping 59596 word types\n",
      "2019-12-04 18:36:39,241 : INFO : PROGRESS: at sentence #19920000, processed 214988789 words, keeping 59596 word types\n",
      "2019-12-04 18:36:39,367 : INFO : PROGRESS: at sentence #19930000, processed 215086527 words, keeping 59596 word types\n",
      "2019-12-04 18:36:39,497 : INFO : PROGRESS: at sentence #19940000, processed 215186756 words, keeping 59596 word types\n",
      "2019-12-04 18:36:39,622 : INFO : PROGRESS: at sentence #19950000, processed 215281588 words, keeping 59596 word types\n",
      "2019-12-04 18:36:39,750 : INFO : PROGRESS: at sentence #19960000, processed 215378449 words, keeping 59596 word types\n",
      "2019-12-04 18:36:39,877 : INFO : PROGRESS: at sentence #19970000, processed 215476904 words, keeping 59596 word types\n",
      "2019-12-04 18:36:40,010 : INFO : PROGRESS: at sentence #19980000, processed 215579816 words, keeping 59596 word types\n",
      "2019-12-04 18:36:40,155 : INFO : PROGRESS: at sentence #19990000, processed 215685778 words, keeping 59596 word types\n",
      "2019-12-04 18:36:40,288 : INFO : PROGRESS: at sentence #20000000, processed 215789970 words, keeping 59596 word types\n",
      "2019-12-04 18:36:40,422 : INFO : PROGRESS: at sentence #20010000, processed 215894891 words, keeping 59596 word types\n",
      "2019-12-04 18:36:40,558 : INFO : PROGRESS: at sentence #20020000, processed 216000463 words, keeping 59596 word types\n",
      "2019-12-04 18:36:40,691 : INFO : PROGRESS: at sentence #20030000, processed 216103040 words, keeping 59596 word types\n",
      "2019-12-04 18:36:40,824 : INFO : PROGRESS: at sentence #20040000, processed 216204495 words, keeping 59596 word types\n",
      "2019-12-04 18:36:40,955 : INFO : PROGRESS: at sentence #20050000, processed 216306334 words, keeping 59596 word types\n",
      "2019-12-04 18:36:41,108 : INFO : PROGRESS: at sentence #20060000, processed 216425921 words, keeping 59596 word types\n",
      "2019-12-04 18:36:41,261 : INFO : PROGRESS: at sentence #20070000, processed 216547030 words, keeping 59596 word types\n",
      "2019-12-04 18:36:41,406 : INFO : PROGRESS: at sentence #20080000, processed 216662019 words, keeping 59596 word types\n",
      "2019-12-04 18:36:41,547 : INFO : PROGRESS: at sentence #20090000, processed 216772187 words, keeping 59596 word types\n",
      "2019-12-04 18:36:41,685 : INFO : PROGRESS: at sentence #20100000, processed 216878109 words, keeping 59596 word types\n",
      "2019-12-04 18:36:41,821 : INFO : PROGRESS: at sentence #20110000, processed 216983364 words, keeping 59596 word types\n",
      "2019-12-04 18:36:41,962 : INFO : PROGRESS: at sentence #20120000, processed 217091335 words, keeping 59596 word types\n",
      "2019-12-04 18:36:42,106 : INFO : PROGRESS: at sentence #20130000, processed 217199062 words, keeping 59596 word types\n",
      "2019-12-04 18:36:42,247 : INFO : PROGRESS: at sentence #20140000, processed 217305653 words, keeping 59596 word types\n",
      "2019-12-04 18:36:42,388 : INFO : PROGRESS: at sentence #20150000, processed 217414139 words, keeping 59596 word types\n",
      "2019-12-04 18:36:42,530 : INFO : PROGRESS: at sentence #20160000, processed 217524094 words, keeping 59596 word types\n",
      "2019-12-04 18:36:42,675 : INFO : PROGRESS: at sentence #20170000, processed 217634813 words, keeping 59596 word types\n",
      "2019-12-04 18:36:42,827 : INFO : PROGRESS: at sentence #20180000, processed 217750938 words, keeping 59596 word types\n",
      "2019-12-04 18:36:42,972 : INFO : PROGRESS: at sentence #20190000, processed 217860943 words, keeping 59596 word types\n",
      "2019-12-04 18:36:43,127 : INFO : PROGRESS: at sentence #20200000, processed 217976121 words, keeping 59596 word types\n",
      "2019-12-04 18:36:43,274 : INFO : PROGRESS: at sentence #20210000, processed 218085171 words, keeping 59596 word types\n",
      "2019-12-04 18:36:43,440 : INFO : PROGRESS: at sentence #20220000, processed 218211547 words, keeping 59596 word types\n",
      "2019-12-04 18:36:43,599 : INFO : PROGRESS: at sentence #20230000, processed 218332250 words, keeping 59596 word types\n",
      "2019-12-04 18:36:43,761 : INFO : PROGRESS: at sentence #20240000, processed 218454254 words, keeping 59596 word types\n",
      "2019-12-04 18:36:44,058 : INFO : PROGRESS: at sentence #20250000, processed 218681456 words, keeping 94103 word types\n",
      "2019-12-04 18:36:44,308 : INFO : PROGRESS: at sentence #20260000, processed 218878243 words, keeping 113380 word types\n",
      "2019-12-04 18:36:44,559 : INFO : PROGRESS: at sentence #20270000, processed 219074777 words, keeping 131683 word types\n",
      "2019-12-04 18:36:44,814 : INFO : PROGRESS: at sentence #20280000, processed 219275748 words, keeping 147849 word types\n",
      "2019-12-04 18:36:45,080 : INFO : PROGRESS: at sentence #20290000, processed 219481505 words, keeping 163422 word types\n",
      "2019-12-04 18:36:45,347 : INFO : PROGRESS: at sentence #20300000, processed 219682623 words, keeping 177971 word types\n",
      "2019-12-04 18:36:45,601 : INFO : PROGRESS: at sentence #20310000, processed 219881818 words, keeping 191910 word types\n",
      "2019-12-04 18:36:45,868 : INFO : PROGRESS: at sentence #20320000, processed 220089193 words, keeping 203933 word types\n",
      "2019-12-04 18:36:46,126 : INFO : PROGRESS: at sentence #20330000, processed 220290421 words, keeping 215089 word types\n",
      "2019-12-04 18:36:46,402 : INFO : PROGRESS: at sentence #20340000, processed 220503921 words, keeping 228248 word types\n",
      "2019-12-04 18:36:46,685 : INFO : PROGRESS: at sentence #20350000, processed 220729759 words, keeping 242297 word types\n",
      "2019-12-04 18:36:46,933 : INFO : PROGRESS: at sentence #20360000, processed 220922627 words, keeping 253428 word types\n",
      "2019-12-04 18:36:47,197 : INFO : PROGRESS: at sentence #20370000, processed 221126574 words, keeping 263740 word types\n",
      "2019-12-04 18:36:47,473 : INFO : PROGRESS: at sentence #20380000, processed 221335836 words, keeping 275511 word types\n",
      "2019-12-04 18:36:47,737 : INFO : PROGRESS: at sentence #20390000, processed 221539617 words, keeping 286420 word types\n",
      "2019-12-04 18:36:48,002 : INFO : PROGRESS: at sentence #20400000, processed 221742074 words, keeping 296811 word types\n",
      "2019-12-04 18:36:48,263 : INFO : PROGRESS: at sentence #20410000, processed 221944257 words, keeping 306060 word types\n",
      "2019-12-04 18:36:48,527 : INFO : PROGRESS: at sentence #20420000, processed 222149435 words, keeping 315671 word types\n",
      "2019-12-04 18:36:48,793 : INFO : PROGRESS: at sentence #20430000, processed 222356003 words, keeping 324794 word types\n",
      "2019-12-04 18:36:49,055 : INFO : PROGRESS: at sentence #20440000, processed 222562972 words, keeping 333760 word types\n",
      "2019-12-04 18:36:49,313 : INFO : PROGRESS: at sentence #20450000, processed 222767528 words, keeping 342372 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:36:49,595 : INFO : PROGRESS: at sentence #20460000, processed 222969138 words, keeping 349872 word types\n",
      "2019-12-04 18:36:49,866 : INFO : PROGRESS: at sentence #20470000, processed 223184284 words, keeping 358855 word types\n",
      "2019-12-04 18:36:50,117 : INFO : PROGRESS: at sentence #20480000, processed 223385953 words, keeping 368211 word types\n",
      "2019-12-04 18:36:50,365 : INFO : PROGRESS: at sentence #20490000, processed 223585318 words, keeping 377475 word types\n",
      "2019-12-04 18:36:50,631 : INFO : PROGRESS: at sentence #20500000, processed 223794577 words, keeping 384737 word types\n",
      "2019-12-04 18:36:50,891 : INFO : PROGRESS: at sentence #20510000, processed 224005401 words, keeping 392793 word types\n",
      "2019-12-04 18:36:51,151 : INFO : PROGRESS: at sentence #20520000, processed 224209642 words, keeping 401193 word types\n",
      "2019-12-04 18:36:51,409 : INFO : PROGRESS: at sentence #20530000, processed 224408421 words, keeping 408691 word types\n",
      "2019-12-04 18:36:51,706 : INFO : PROGRESS: at sentence #20540000, processed 224644009 words, keeping 416889 word types\n",
      "2019-12-04 18:36:51,993 : INFO : PROGRESS: at sentence #20550000, processed 224873121 words, keeping 432939 word types\n",
      "2019-12-04 18:36:52,259 : INFO : PROGRESS: at sentence #20560000, processed 225082431 words, keeping 440809 word types\n",
      "2019-12-04 18:36:52,517 : INFO : PROGRESS: at sentence #20570000, processed 225282374 words, keeping 448536 word types\n",
      "2019-12-04 18:36:52,789 : INFO : PROGRESS: at sentence #20580000, processed 225496529 words, keeping 456074 word types\n",
      "2019-12-04 18:36:53,057 : INFO : PROGRESS: at sentence #20590000, processed 225709318 words, keeping 463933 word types\n",
      "2019-12-04 18:36:53,289 : INFO : PROGRESS: at sentence #20600000, processed 225886043 words, keeping 471987 word types\n",
      "2019-12-04 18:36:53,561 : INFO : PROGRESS: at sentence #20610000, processed 226101310 words, keeping 481373 word types\n",
      "2019-12-04 18:36:53,821 : INFO : PROGRESS: at sentence #20620000, processed 226304264 words, keeping 489728 word types\n",
      "2019-12-04 18:36:54,071 : INFO : PROGRESS: at sentence #20630000, processed 226495288 words, keeping 497950 word types\n",
      "2019-12-04 18:36:54,361 : INFO : pruned out 0 tokens with count <=1 (before 500001, after 500001)\n",
      "2019-12-04 18:36:54,597 : INFO : pruned out 251002 tokens with count <=2 (before 500001, after 248999)\n",
      "2019-12-04 18:36:54,782 : INFO : PROGRESS: at sentence #20640000, processed 226694908 words, keeping 255212 word types\n",
      "2019-12-04 18:36:55,034 : INFO : PROGRESS: at sentence #20650000, processed 226896143 words, keeping 264173 word types\n",
      "2019-12-04 18:36:55,292 : INFO : PROGRESS: at sentence #20660000, processed 227101349 words, keeping 273297 word types\n",
      "2019-12-04 18:36:55,552 : INFO : PROGRESS: at sentence #20670000, processed 227307854 words, keeping 282098 word types\n",
      "2019-12-04 18:36:55,809 : INFO : PROGRESS: at sentence #20680000, processed 227504613 words, keeping 290954 word types\n",
      "2019-12-04 18:36:56,079 : INFO : PROGRESS: at sentence #20690000, processed 227719773 words, keeping 301538 word types\n",
      "2019-12-04 18:36:56,470 : INFO : PROGRESS: at sentence #20700000, processed 228043277 words, keeping 320227 word types\n",
      "2019-12-04 18:36:56,960 : INFO : PROGRESS: at sentence #20710000, processed 228447851 words, keeping 340239 word types\n",
      "2019-12-04 18:36:57,579 : INFO : PROGRESS: at sentence #20720000, processed 228979741 words, keeping 362939 word types\n",
      "2019-12-04 18:36:58,107 : INFO : PROGRESS: at sentence #20730000, processed 229422444 words, keeping 375216 word types\n",
      "2019-12-04 18:36:58,574 : INFO : PROGRESS: at sentence #20740000, processed 229803212 words, keeping 382926 word types\n",
      "2019-12-04 18:36:59,038 : INFO : PROGRESS: at sentence #20750000, processed 230179173 words, keeping 390171 word types\n",
      "2019-12-04 18:36:59,430 : INFO : PROGRESS: at sentence #20760000, processed 230501700 words, keeping 395571 word types\n",
      "2019-12-04 18:36:59,745 : INFO : PROGRESS: at sentence #20770000, processed 230758754 words, keeping 400362 word types\n",
      "2019-12-04 18:37:00,158 : INFO : PROGRESS: at sentence #20780000, processed 231110024 words, keeping 405234 word types\n",
      "2019-12-04 18:37:00,485 : INFO : PROGRESS: at sentence #20790000, processed 231379615 words, keeping 409593 word types\n",
      "2019-12-04 18:37:00,747 : INFO : PROGRESS: at sentence #20800000, processed 231581052 words, keeping 413986 word types\n",
      "2019-12-04 18:37:01,017 : INFO : PROGRESS: at sentence #20810000, processed 231789773 words, keeping 420138 word types\n",
      "2019-12-04 18:37:01,273 : INFO : PROGRESS: at sentence #20820000, processed 231986346 words, keeping 426945 word types\n",
      "2019-12-04 18:37:01,551 : INFO : PROGRESS: at sentence #20830000, processed 232197995 words, keeping 434685 word types\n",
      "2019-12-04 18:37:01,818 : INFO : PROGRESS: at sentence #20840000, processed 232403145 words, keeping 442010 word types\n",
      "2019-12-04 18:37:02,121 : INFO : PROGRESS: at sentence #20850000, processed 232599420 words, keeping 448123 word types\n",
      "2019-12-04 18:37:02,359 : INFO : PROGRESS: at sentence #20860000, processed 232783764 words, keeping 454841 word types\n",
      "2019-12-04 18:37:02,625 : INFO : PROGRESS: at sentence #20870000, processed 232993633 words, keeping 462128 word types\n",
      "2019-12-04 18:37:02,898 : INFO : PROGRESS: at sentence #20880000, processed 233209357 words, keeping 471225 word types\n",
      "2019-12-04 18:37:03,164 : INFO : PROGRESS: at sentence #20890000, processed 233419907 words, keeping 479506 word types\n",
      "2019-12-04 18:37:03,424 : INFO : PROGRESS: at sentence #20900000, processed 233623835 words, keeping 486370 word types\n",
      "2019-12-04 18:37:03,678 : INFO : PROGRESS: at sentence #20910000, processed 233822333 words, keeping 493869 word types\n",
      "2019-12-04 18:37:04,174 : INFO : pruned out 239227 tokens with count <=3 (before 500011, after 260784)\n",
      "2019-12-04 18:37:04,192 : INFO : PROGRESS: at sentence #20920000, processed 234028141 words, keeping 261472 word types\n",
      "2019-12-04 18:37:04,442 : INFO : PROGRESS: at sentence #20930000, processed 234225550 words, keeping 271808 word types\n",
      "2019-12-04 18:37:04,697 : INFO : PROGRESS: at sentence #20940000, processed 234429906 words, keeping 279218 word types\n",
      "2019-12-04 18:37:04,952 : INFO : PROGRESS: at sentence #20950000, processed 234631398 words, keeping 288200 word types\n",
      "2019-12-04 18:37:05,201 : INFO : PROGRESS: at sentence #20960000, processed 234830808 words, keeping 296612 word types\n",
      "2019-12-04 18:37:05,452 : INFO : PROGRESS: at sentence #20970000, processed 235030949 words, keeping 305129 word types\n",
      "2019-12-04 18:37:05,706 : INFO : PROGRESS: at sentence #20980000, processed 235234342 words, keeping 313202 word types\n",
      "2019-12-04 18:37:05,970 : INFO : PROGRESS: at sentence #20990000, processed 235440874 words, keeping 322535 word types\n",
      "2019-12-04 18:37:06,228 : INFO : PROGRESS: at sentence #21000000, processed 235645913 words, keeping 331717 word types\n",
      "2019-12-04 18:37:06,495 : INFO : PROGRESS: at sentence #21010000, processed 235853534 words, keeping 340812 word types\n",
      "2019-12-04 18:37:06,743 : INFO : PROGRESS: at sentence #21020000, processed 236048163 words, keeping 347597 word types\n",
      "2019-12-04 18:37:07,010 : INFO : PROGRESS: at sentence #21030000, processed 236254764 words, keeping 355793 word types\n",
      "2019-12-04 18:37:07,268 : INFO : PROGRESS: at sentence #21040000, processed 236457127 words, keeping 364543 word types\n",
      "2019-12-04 18:37:07,532 : INFO : PROGRESS: at sentence #21050000, processed 236666610 words, keeping 372415 word types\n",
      "2019-12-04 18:37:07,794 : INFO : PROGRESS: at sentence #21060000, processed 236869056 words, keeping 380898 word types\n",
      "2019-12-04 18:37:08,055 : INFO : PROGRESS: at sentence #21070000, processed 237076181 words, keeping 389518 word types\n",
      "2019-12-04 18:37:08,312 : INFO : PROGRESS: at sentence #21080000, processed 237281339 words, keeping 397629 word types\n",
      "2019-12-04 18:37:08,563 : INFO : PROGRESS: at sentence #21090000, processed 237482754 words, keeping 405031 word types\n",
      "2019-12-04 18:37:08,817 : INFO : PROGRESS: at sentence #21100000, processed 237682544 words, keeping 412822 word types\n",
      "2019-12-04 18:37:09,070 : INFO : PROGRESS: at sentence #21110000, processed 237883925 words, keeping 420369 word types\n",
      "2019-12-04 18:37:09,323 : INFO : PROGRESS: at sentence #21120000, processed 238087503 words, keeping 428276 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:37:09,582 : INFO : PROGRESS: at sentence #21130000, processed 238298942 words, keeping 434888 word types\n",
      "2019-12-04 18:37:09,835 : INFO : PROGRESS: at sentence #21140000, processed 238503640 words, keeping 442559 word types\n",
      "2019-12-04 18:37:10,083 : INFO : PROGRESS: at sentence #21150000, processed 238705686 words, keeping 450829 word types\n",
      "2019-12-04 18:37:10,346 : INFO : PROGRESS: at sentence #21160000, processed 238914687 words, keeping 458852 word types\n",
      "2019-12-04 18:37:10,603 : INFO : PROGRESS: at sentence #21170000, processed 239117335 words, keeping 467394 word types\n",
      "2019-12-04 18:37:10,861 : INFO : PROGRESS: at sentence #21180000, processed 239323432 words, keeping 475313 word types\n",
      "2019-12-04 18:37:11,119 : INFO : PROGRESS: at sentence #21190000, processed 239527748 words, keeping 482513 word types\n",
      "2019-12-04 18:37:11,378 : INFO : PROGRESS: at sentence #21200000, processed 239731621 words, keeping 490731 word types\n",
      "2019-12-04 18:37:11,631 : INFO : PROGRESS: at sentence #21210000, processed 239933858 words, keeping 497211 word types\n",
      "2019-12-04 18:37:12,029 : INFO : pruned out 253608 tokens with count <=4 (before 500001, after 246393)\n",
      "2019-12-04 18:37:12,171 : INFO : PROGRESS: at sentence #21220000, processed 240145190 words, keeping 251156 word types\n",
      "2019-12-04 18:37:12,435 : INFO : PROGRESS: at sentence #21230000, processed 240350939 words, keeping 260703 word types\n",
      "2019-12-04 18:37:12,707 : INFO : PROGRESS: at sentence #21240000, processed 240561198 words, keeping 271133 word types\n",
      "2019-12-04 18:37:12,975 : INFO : PROGRESS: at sentence #21250000, processed 240770272 words, keeping 281569 word types\n",
      "2019-12-04 18:37:13,227 : INFO : PROGRESS: at sentence #21260000, processed 240964386 words, keeping 289450 word types\n",
      "2019-12-04 18:37:13,506 : INFO : PROGRESS: at sentence #21270000, processed 241171461 words, keeping 297632 word types\n",
      "2019-12-04 18:37:13,773 : INFO : PROGRESS: at sentence #21280000, processed 241376025 words, keeping 306258 word types\n",
      "2019-12-04 18:37:14,030 : INFO : PROGRESS: at sentence #21290000, processed 241575932 words, keeping 315040 word types\n",
      "2019-12-04 18:37:14,292 : INFO : PROGRESS: at sentence #21300000, processed 241781601 words, keeping 322890 word types\n",
      "2019-12-04 18:37:14,565 : INFO : PROGRESS: at sentence #21310000, processed 241995175 words, keeping 333201 word types\n",
      "2019-12-04 18:37:14,829 : INFO : PROGRESS: at sentence #21320000, processed 242196579 words, keeping 340979 word types\n",
      "2019-12-04 18:37:15,103 : INFO : PROGRESS: at sentence #21330000, processed 242403543 words, keeping 350167 word types\n",
      "2019-12-04 18:37:15,368 : INFO : PROGRESS: at sentence #21340000, processed 242607640 words, keeping 358899 word types\n",
      "2019-12-04 18:37:15,635 : INFO : PROGRESS: at sentence #21350000, processed 242814834 words, keeping 367721 word types\n",
      "2019-12-04 18:37:15,905 : INFO : PROGRESS: at sentence #21360000, processed 243017451 words, keeping 376056 word types\n",
      "2019-12-04 18:37:16,195 : INFO : PROGRESS: at sentence #21370000, processed 243220441 words, keeping 384385 word types\n",
      "2019-12-04 18:37:16,500 : INFO : PROGRESS: at sentence #21380000, processed 243429031 words, keeping 393405 word types\n",
      "2019-12-04 18:37:16,763 : INFO : PROGRESS: at sentence #21390000, processed 243643310 words, keeping 402377 word types\n",
      "2019-12-04 18:37:17,005 : INFO : PROGRESS: at sentence #21400000, processed 243841014 words, keeping 410111 word types\n",
      "2019-12-04 18:37:17,255 : INFO : PROGRESS: at sentence #21410000, processed 244039326 words, keeping 417588 word types\n",
      "2019-12-04 18:37:17,507 : INFO : PROGRESS: at sentence #21420000, processed 244244382 words, keeping 425799 word types\n",
      "2019-12-04 18:37:17,760 : INFO : PROGRESS: at sentence #21430000, processed 244450080 words, keeping 433106 word types\n",
      "2019-12-04 18:37:18,022 : INFO : PROGRESS: at sentence #21440000, processed 244653237 words, keeping 441439 word types\n",
      "2019-12-04 18:37:18,287 : INFO : PROGRESS: at sentence #21450000, processed 244859498 words, keeping 449293 word types\n",
      "2019-12-04 18:37:18,562 : INFO : PROGRESS: at sentence #21460000, processed 245073752 words, keeping 457167 word types\n",
      "2019-12-04 18:37:18,852 : INFO : PROGRESS: at sentence #21470000, processed 245287483 words, keeping 465211 word types\n",
      "2019-12-04 18:37:19,137 : INFO : PROGRESS: at sentence #21480000, processed 245505986 words, keeping 472828 word types\n",
      "2019-12-04 18:37:19,417 : INFO : PROGRESS: at sentence #21490000, processed 245717672 words, keeping 481589 word types\n",
      "2019-12-04 18:37:19,678 : INFO : PROGRESS: at sentence #21500000, processed 245901444 words, keeping 488348 word types\n",
      "2019-12-04 18:37:19,950 : INFO : PROGRESS: at sentence #21510000, processed 246107813 words, keeping 495906 word types\n",
      "2019-12-04 18:37:20,392 : INFO : pruned out 264626 tokens with count <=5 (before 500001, after 235375)\n",
      "2019-12-04 18:37:20,507 : INFO : PROGRESS: at sentence #21520000, processed 246307313 words, keeping 239142 word types\n",
      "2019-12-04 18:37:20,784 : INFO : PROGRESS: at sentence #21530000, processed 246509328 words, keeping 250101 word types\n",
      "2019-12-04 18:37:21,076 : INFO : PROGRESS: at sentence #21540000, processed 246730369 words, keeping 261330 word types\n",
      "2019-12-04 18:37:21,344 : INFO : PROGRESS: at sentence #21550000, processed 246929581 words, keeping 270797 word types\n",
      "2019-12-04 18:37:21,606 : INFO : PROGRESS: at sentence #21560000, processed 247131247 words, keeping 279906 word types\n",
      "2019-12-04 18:37:21,871 : INFO : PROGRESS: at sentence #21570000, processed 247337525 words, keeping 290081 word types\n",
      "2019-12-04 18:37:22,132 : INFO : PROGRESS: at sentence #21580000, processed 247534303 words, keeping 299065 word types\n",
      "2019-12-04 18:37:22,410 : INFO : PROGRESS: at sentence #21590000, processed 247748941 words, keeping 308841 word types\n",
      "2019-12-04 18:37:22,691 : INFO : PROGRESS: at sentence #21600000, processed 247964124 words, keeping 317597 word types\n",
      "2019-12-04 18:37:22,973 : INFO : PROGRESS: at sentence #21610000, processed 248168722 words, keeping 325739 word types\n",
      "2019-12-04 18:37:23,252 : INFO : PROGRESS: at sentence #21620000, processed 248384038 words, keeping 335091 word types\n",
      "2019-12-04 18:37:23,522 : INFO : PROGRESS: at sentence #21630000, processed 248591580 words, keeping 343799 word types\n",
      "2019-12-04 18:37:23,805 : INFO : PROGRESS: at sentence #21640000, processed 248812892 words, keeping 352930 word types\n",
      "2019-12-04 18:37:24,069 : INFO : PROGRESS: at sentence #21650000, processed 249025767 words, keeping 362382 word types\n",
      "2019-12-04 18:37:24,347 : INFO : PROGRESS: at sentence #21660000, processed 249236207 words, keeping 369796 word types\n",
      "2019-12-04 18:37:24,601 : INFO : PROGRESS: at sentence #21670000, processed 249431141 words, keeping 376877 word types\n",
      "2019-12-04 18:37:24,857 : INFO : PROGRESS: at sentence #21680000, processed 249636241 words, keeping 384278 word types\n",
      "2019-12-04 18:37:25,117 : INFO : PROGRESS: at sentence #21690000, processed 249824487 words, keeping 391581 word types\n",
      "2019-12-04 18:37:25,404 : INFO : PROGRESS: at sentence #21700000, processed 250034049 words, keeping 399927 word types\n",
      "2019-12-04 18:37:25,682 : INFO : PROGRESS: at sentence #21710000, processed 250244548 words, keeping 408854 word types\n",
      "2019-12-04 18:37:25,947 : INFO : PROGRESS: at sentence #21720000, processed 250449436 words, keeping 415942 word types\n",
      "2019-12-04 18:37:26,219 : INFO : PROGRESS: at sentence #21730000, processed 250661824 words, keeping 422868 word types\n",
      "2019-12-04 18:37:26,486 : INFO : PROGRESS: at sentence #21740000, processed 250865182 words, keeping 431612 word types\n",
      "2019-12-04 18:37:26,763 : INFO : PROGRESS: at sentence #21750000, processed 251061860 words, keeping 439544 word types\n",
      "2019-12-04 18:37:27,031 : INFO : PROGRESS: at sentence #21760000, processed 251268886 words, keeping 447501 word types\n",
      "2019-12-04 18:37:27,293 : INFO : PROGRESS: at sentence #21770000, processed 251476654 words, keeping 455201 word types\n",
      "2019-12-04 18:37:27,560 : INFO : PROGRESS: at sentence #21780000, processed 251684100 words, keeping 463020 word types\n",
      "2019-12-04 18:37:27,829 : INFO : PROGRESS: at sentence #21790000, processed 251890177 words, keeping 469902 word types\n",
      "2019-12-04 18:37:28,088 : INFO : PROGRESS: at sentence #21800000, processed 252092682 words, keeping 477053 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:37:28,349 : INFO : PROGRESS: at sentence #21810000, processed 252293701 words, keeping 484087 word types\n",
      "2019-12-04 18:37:28,613 : INFO : PROGRESS: at sentence #21820000, processed 252505804 words, keeping 491115 word types\n",
      "2019-12-04 18:37:28,869 : INFO : PROGRESS: at sentence #21830000, processed 252708558 words, keeping 497229 word types\n",
      "2019-12-04 18:37:29,238 : INFO : pruned out 270775 tokens with count <=6 (before 500001, after 229226)\n",
      "2019-12-04 18:37:29,421 : INFO : PROGRESS: at sentence #21840000, processed 252924328 words, keeping 236065 word types\n",
      "2019-12-04 18:37:29,664 : INFO : PROGRESS: at sentence #21850000, processed 253121403 words, keeping 246362 word types\n",
      "2019-12-04 18:37:29,931 : INFO : PROGRESS: at sentence #21860000, processed 253335997 words, keeping 256840 word types\n",
      "2019-12-04 18:37:30,200 : INFO : PROGRESS: at sentence #21870000, processed 253545302 words, keeping 266603 word types\n",
      "2019-12-04 18:37:30,474 : INFO : PROGRESS: at sentence #21880000, processed 253755320 words, keeping 277529 word types\n",
      "2019-12-04 18:37:30,746 : INFO : PROGRESS: at sentence #21890000, processed 253964349 words, keeping 287053 word types\n",
      "2019-12-04 18:37:31,007 : INFO : PROGRESS: at sentence #21900000, processed 254164182 words, keeping 296135 word types\n",
      "2019-12-04 18:37:31,275 : INFO : PROGRESS: at sentence #21910000, processed 254369402 words, keeping 305874 word types\n",
      "2019-12-04 18:37:31,544 : INFO : PROGRESS: at sentence #21920000, processed 254575707 words, keeping 315845 word types\n",
      "2019-12-04 18:37:31,813 : INFO : PROGRESS: at sentence #21930000, processed 254781869 words, keeping 324721 word types\n",
      "2019-12-04 18:37:32,081 : INFO : PROGRESS: at sentence #21940000, processed 254987738 words, keeping 333549 word types\n",
      "2019-12-04 18:37:32,352 : INFO : PROGRESS: at sentence #21950000, processed 255189628 words, keeping 341843 word types\n",
      "2019-12-04 18:37:32,619 : INFO : PROGRESS: at sentence #21960000, processed 255389105 words, keeping 350448 word types\n",
      "2019-12-04 18:37:32,898 : INFO : PROGRESS: at sentence #21970000, processed 255596777 words, keeping 359386 word types\n",
      "2019-12-04 18:37:33,169 : INFO : PROGRESS: at sentence #21980000, processed 255801940 words, keeping 368135 word types\n",
      "2019-12-04 18:37:33,499 : INFO : PROGRESS: at sentence #21990000, processed 256021062 words, keeping 377595 word types\n",
      "2019-12-04 18:37:33,770 : INFO : PROGRESS: at sentence #22000000, processed 256234530 words, keeping 386040 word types\n",
      "2019-12-04 18:37:34,049 : INFO : PROGRESS: at sentence #22010000, processed 256453898 words, keeping 394008 word types\n",
      "2019-12-04 18:37:34,316 : INFO : PROGRESS: at sentence #22020000, processed 256660150 words, keeping 402502 word types\n",
      "2019-12-04 18:37:34,577 : INFO : PROGRESS: at sentence #22030000, processed 256861380 words, keeping 410918 word types\n",
      "2019-12-04 18:37:34,840 : INFO : PROGRESS: at sentence #22040000, processed 257067879 words, keeping 419050 word types\n",
      "2019-12-04 18:37:35,107 : INFO : PROGRESS: at sentence #22050000, processed 257278098 words, keeping 427506 word types\n",
      "2019-12-04 18:37:35,365 : INFO : PROGRESS: at sentence #22060000, processed 257478023 words, keeping 435051 word types\n",
      "2019-12-04 18:37:35,618 : INFO : PROGRESS: at sentence #22070000, processed 257671839 words, keeping 442602 word types\n",
      "2019-12-04 18:37:35,874 : INFO : PROGRESS: at sentence #22080000, processed 257869171 words, keeping 451099 word types\n",
      "2019-12-04 18:37:36,136 : INFO : PROGRESS: at sentence #22090000, processed 258072892 words, keeping 458668 word types\n",
      "2019-12-04 18:37:36,397 : INFO : PROGRESS: at sentence #22100000, processed 258277513 words, keeping 467631 word types\n",
      "2019-12-04 18:37:36,671 : INFO : PROGRESS: at sentence #22110000, processed 258491077 words, keeping 476391 word types\n",
      "2019-12-04 18:37:36,921 : INFO : PROGRESS: at sentence #22120000, processed 258685551 words, keeping 483101 word types\n",
      "2019-12-04 18:37:37,185 : INFO : PROGRESS: at sentence #22130000, processed 258892804 words, keeping 491018 word types\n",
      "2019-12-04 18:37:37,441 : INFO : PROGRESS: at sentence #22140000, processed 259091144 words, keeping 498092 word types\n",
      "2019-12-04 18:37:37,785 : INFO : pruned out 274919 tokens with count <=7 (before 500006, after 225087)\n",
      "2019-12-04 18:37:37,984 : INFO : PROGRESS: at sentence #22150000, processed 259300624 words, keeping 232470 word types\n",
      "2019-12-04 18:37:38,267 : INFO : PROGRESS: at sentence #22160000, processed 259516770 words, keeping 243353 word types\n",
      "2019-12-04 18:37:38,536 : INFO : PROGRESS: at sentence #22170000, processed 259724738 words, keeping 252883 word types\n",
      "2019-12-04 18:37:38,816 : INFO : PROGRESS: at sentence #22180000, processed 259939780 words, keeping 262899 word types\n",
      "2019-12-04 18:37:39,116 : INFO : PROGRESS: at sentence #22190000, processed 260173052 words, keeping 275881 word types\n",
      "2019-12-04 18:37:39,386 : INFO : PROGRESS: at sentence #22200000, processed 260380687 words, keeping 284696 word types\n",
      "2019-12-04 18:37:39,652 : INFO : PROGRESS: at sentence #22210000, processed 260584015 words, keeping 293791 word types\n",
      "2019-12-04 18:37:39,917 : INFO : PROGRESS: at sentence #22220000, processed 260785539 words, keeping 303002 word types\n",
      "2019-12-04 18:37:40,184 : INFO : PROGRESS: at sentence #22230000, processed 260991253 words, keeping 313456 word types\n",
      "2019-12-04 18:37:40,440 : INFO : PROGRESS: at sentence #22240000, processed 261188465 words, keeping 321911 word types\n",
      "2019-12-04 18:37:40,713 : INFO : PROGRESS: at sentence #22250000, processed 261398768 words, keeping 330918 word types\n",
      "2019-12-04 18:37:40,976 : INFO : PROGRESS: at sentence #22260000, processed 261600560 words, keeping 338874 word types\n",
      "2019-12-04 18:37:41,244 : INFO : PROGRESS: at sentence #22270000, processed 261805469 words, keeping 347386 word types\n",
      "2019-12-04 18:37:41,508 : INFO : PROGRESS: at sentence #22280000, processed 262004714 words, keeping 356316 word types\n",
      "2019-12-04 18:37:41,791 : INFO : PROGRESS: at sentence #22290000, processed 262220077 words, keeping 366058 word types\n",
      "2019-12-04 18:37:42,081 : INFO : PROGRESS: at sentence #22300000, processed 262439386 words, keeping 374833 word types\n",
      "2019-12-04 18:37:42,368 : INFO : PROGRESS: at sentence #22310000, processed 262652334 words, keeping 384336 word types\n",
      "2019-12-04 18:37:42,639 : INFO : PROGRESS: at sentence #22320000, processed 262855308 words, keeping 392027 word types\n",
      "2019-12-04 18:37:42,915 : INFO : PROGRESS: at sentence #22330000, processed 263066186 words, keeping 401808 word types\n",
      "2019-12-04 18:37:43,245 : INFO : PROGRESS: at sentence #22340000, processed 263279436 words, keeping 410876 word types\n",
      "2019-12-04 18:37:43,548 : INFO : PROGRESS: at sentence #22350000, processed 263485385 words, keeping 418805 word types\n",
      "2019-12-04 18:37:43,822 : INFO : PROGRESS: at sentence #22360000, processed 263692247 words, keeping 426982 word types\n",
      "2019-12-04 18:37:44,080 : INFO : PROGRESS: at sentence #22370000, processed 263882342 words, keeping 433682 word types\n",
      "2019-12-04 18:37:44,354 : INFO : PROGRESS: at sentence #22380000, processed 264087230 words, keeping 441845 word types\n",
      "2019-12-04 18:37:44,630 : INFO : PROGRESS: at sentence #22390000, processed 264296834 words, keeping 449006 word types\n",
      "2019-12-04 18:37:44,895 : INFO : PROGRESS: at sentence #22400000, processed 264498308 words, keeping 456330 word types\n",
      "2019-12-04 18:37:45,164 : INFO : PROGRESS: at sentence #22410000, processed 264702246 words, keeping 463386 word types\n",
      "2019-12-04 18:37:45,417 : INFO : PROGRESS: at sentence #22420000, processed 264894416 words, keeping 470436 word types\n",
      "2019-12-04 18:37:45,686 : INFO : PROGRESS: at sentence #22430000, processed 265099048 words, keeping 478116 word types\n",
      "2019-12-04 18:37:45,947 : INFO : PROGRESS: at sentence #22440000, processed 265294417 words, keeping 485231 word types\n",
      "2019-12-04 18:37:46,215 : INFO : PROGRESS: at sentence #22450000, processed 265494566 words, keeping 492905 word types\n",
      "2019-12-04 18:37:46,735 : INFO : pruned out 277919 tokens with count <=8 (before 500003, after 222084)\n",
      "2019-12-04 18:37:46,788 : INFO : PROGRESS: at sentence #22460000, processed 265716150 words, keeping 223960 word types\n",
      "2019-12-04 18:37:47,050 : INFO : PROGRESS: at sentence #22470000, processed 265920253 words, keeping 233372 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:37:47,315 : INFO : PROGRESS: at sentence #22480000, processed 266126354 words, keeping 242786 word types\n",
      "2019-12-04 18:37:47,569 : INFO : PROGRESS: at sentence #22490000, processed 266324932 words, keeping 253104 word types\n",
      "2019-12-04 18:37:47,867 : INFO : PROGRESS: at sentence #22500000, processed 266561355 words, keeping 265982 word types\n",
      "2019-12-04 18:37:48,148 : INFO : PROGRESS: at sentence #22510000, processed 266782042 words, keeping 276426 word types\n",
      "2019-12-04 18:37:48,417 : INFO : PROGRESS: at sentence #22520000, processed 266990460 words, keeping 285739 word types\n",
      "2019-12-04 18:37:48,680 : INFO : PROGRESS: at sentence #22530000, processed 267196419 words, keeping 294903 word types\n",
      "2019-12-04 18:37:48,959 : INFO : PROGRESS: at sentence #22540000, processed 267415157 words, keeping 304409 word types\n",
      "2019-12-04 18:37:49,242 : INFO : PROGRESS: at sentence #22550000, processed 267636096 words, keeping 313898 word types\n",
      "2019-12-04 18:37:49,507 : INFO : PROGRESS: at sentence #22560000, processed 267842931 words, keeping 323041 word types\n",
      "2019-12-04 18:37:49,765 : INFO : PROGRESS: at sentence #22570000, processed 268041016 words, keeping 330423 word types\n",
      "2019-12-04 18:37:50,013 : INFO : PROGRESS: at sentence #22580000, processed 268232048 words, keeping 337318 word types\n",
      "2019-12-04 18:37:50,266 : INFO : PROGRESS: at sentence #22590000, processed 268429812 words, keeping 346220 word types\n",
      "2019-12-04 18:37:50,537 : INFO : PROGRESS: at sentence #22600000, processed 268642625 words, keeping 355104 word types\n",
      "2019-12-04 18:37:50,804 : INFO : PROGRESS: at sentence #22610000, processed 268852928 words, keeping 364013 word types\n",
      "2019-12-04 18:37:51,068 : INFO : PROGRESS: at sentence #22620000, processed 269060167 words, keeping 371819 word types\n",
      "2019-12-04 18:37:51,341 : INFO : PROGRESS: at sentence #22630000, processed 269263619 words, keeping 380483 word types\n",
      "2019-12-04 18:37:51,605 : INFO : PROGRESS: at sentence #22640000, processed 269467992 words, keeping 388785 word types\n",
      "2019-12-04 18:37:51,863 : INFO : PROGRESS: at sentence #22650000, processed 269667785 words, keeping 396644 word types\n",
      "2019-12-04 18:37:52,137 : INFO : PROGRESS: at sentence #22660000, processed 269886814 words, keeping 404436 word types\n",
      "2019-12-04 18:37:52,400 : INFO : PROGRESS: at sentence #22670000, processed 270088851 words, keeping 412308 word types\n",
      "2019-12-04 18:37:52,678 : INFO : PROGRESS: at sentence #22680000, processed 270302857 words, keeping 421757 word types\n",
      "2019-12-04 18:37:52,954 : INFO : PROGRESS: at sentence #22690000, processed 270516641 words, keeping 429832 word types\n",
      "2019-12-04 18:37:53,220 : INFO : PROGRESS: at sentence #22700000, processed 270725351 words, keeping 438843 word types\n",
      "2019-12-04 18:37:53,486 : INFO : PROGRESS: at sentence #22710000, processed 270935057 words, keeping 447335 word types\n",
      "2019-12-04 18:37:53,752 : INFO : PROGRESS: at sentence #22720000, processed 271140001 words, keeping 454565 word types\n",
      "2019-12-04 18:37:54,006 : INFO : PROGRESS: at sentence #22730000, processed 271337917 words, keeping 462460 word types\n",
      "2019-12-04 18:37:54,266 : INFO : PROGRESS: at sentence #22740000, processed 271539874 words, keeping 469615 word types\n",
      "2019-12-04 18:37:54,529 : INFO : PROGRESS: at sentence #22750000, processed 271747343 words, keeping 476647 word types\n",
      "2019-12-04 18:37:54,799 : INFO : PROGRESS: at sentence #22760000, processed 271962712 words, keeping 484446 word types\n",
      "2019-12-04 18:37:55,062 : INFO : PROGRESS: at sentence #22770000, processed 272168423 words, keeping 493837 word types\n",
      "2019-12-04 18:37:55,557 : INFO : pruned out 280341 tokens with count <=9 (before 500055, after 219714)\n",
      "2019-12-04 18:37:55,592 : INFO : PROGRESS: at sentence #22780000, processed 272367859 words, keeping 221036 word types\n",
      "2019-12-04 18:37:55,865 : INFO : PROGRESS: at sentence #22790000, processed 272579109 words, keeping 231962 word types\n",
      "2019-12-04 18:37:56,139 : INFO : PROGRESS: at sentence #22800000, processed 272792055 words, keeping 242088 word types\n",
      "2019-12-04 18:37:56,433 : INFO : PROGRESS: at sentence #22810000, processed 272996566 words, keeping 251530 word types\n",
      "2019-12-04 18:37:56,709 : INFO : PROGRESS: at sentence #22820000, processed 273210939 words, keeping 262204 word types\n",
      "2019-12-04 18:37:56,981 : INFO : PROGRESS: at sentence #22830000, processed 273423068 words, keeping 272077 word types\n",
      "2019-12-04 18:37:57,250 : INFO : PROGRESS: at sentence #22840000, processed 273631211 words, keeping 281823 word types\n",
      "2019-12-04 18:37:57,515 : INFO : PROGRESS: at sentence #22850000, processed 273834843 words, keeping 290147 word types\n",
      "2019-12-04 18:37:57,762 : INFO : PROGRESS: at sentence #22860000, processed 274025052 words, keeping 298672 word types\n",
      "2019-12-04 18:37:58,023 : INFO : PROGRESS: at sentence #22870000, processed 274225115 words, keeping 307632 word types\n",
      "2019-12-04 18:37:58,288 : INFO : PROGRESS: at sentence #22880000, processed 274429173 words, keeping 316062 word types\n",
      "2019-12-04 18:37:58,549 : INFO : PROGRESS: at sentence #22890000, processed 274631976 words, keeping 325555 word types\n",
      "2019-12-04 18:37:58,803 : INFO : PROGRESS: at sentence #22900000, processed 274826486 words, keeping 334547 word types\n",
      "2019-12-04 18:37:59,064 : INFO : PROGRESS: at sentence #22910000, processed 275025888 words, keeping 343762 word types\n",
      "2019-12-04 18:37:59,325 : INFO : PROGRESS: at sentence #22920000, processed 275230397 words, keeping 351761 word types\n",
      "2019-12-04 18:37:59,588 : INFO : PROGRESS: at sentence #22930000, processed 275431392 words, keeping 361042 word types\n",
      "2019-12-04 18:37:59,847 : INFO : PROGRESS: at sentence #22940000, processed 275632634 words, keeping 369919 word types\n",
      "2019-12-04 18:38:00,106 : INFO : PROGRESS: at sentence #22950000, processed 275832653 words, keeping 378654 word types\n",
      "2019-12-04 18:38:00,366 : INFO : PROGRESS: at sentence #22960000, processed 276036737 words, keeping 387058 word types\n",
      "2019-12-04 18:38:00,631 : INFO : PROGRESS: at sentence #22970000, processed 276241950 words, keeping 396201 word types\n",
      "2019-12-04 18:38:00,887 : INFO : PROGRESS: at sentence #22980000, processed 276441725 words, keeping 407115 word types\n",
      "2019-12-04 18:38:01,108 : INFO : PROGRESS: at sentence #22990000, processed 276612133 words, keeping 413317 word types\n",
      "2019-12-04 18:38:01,378 : INFO : PROGRESS: at sentence #23000000, processed 276818512 words, keeping 420984 word types\n",
      "2019-12-04 18:38:01,641 : INFO : PROGRESS: at sentence #23010000, processed 277023648 words, keeping 429160 word types\n",
      "2019-12-04 18:38:01,894 : INFO : PROGRESS: at sentence #23020000, processed 277220943 words, keeping 436182 word types\n",
      "2019-12-04 18:38:02,171 : INFO : PROGRESS: at sentence #23030000, processed 277432291 words, keeping 445617 word types\n",
      "2019-12-04 18:38:02,433 : INFO : PROGRESS: at sentence #23040000, processed 277637892 words, keeping 453282 word types\n",
      "2019-12-04 18:38:02,677 : INFO : PROGRESS: at sentence #23050000, processed 277827565 words, keeping 459875 word types\n",
      "2019-12-04 18:38:02,930 : INFO : PROGRESS: at sentence #23060000, processed 278026735 words, keeping 467044 word types\n",
      "2019-12-04 18:38:03,197 : INFO : PROGRESS: at sentence #23070000, processed 278233079 words, keeping 474398 word types\n",
      "2019-12-04 18:38:03,466 : INFO : PROGRESS: at sentence #23080000, processed 278441919 words, keeping 481439 word types\n",
      "2019-12-04 18:38:03,709 : INFO : PROGRESS: at sentence #23090000, processed 278630333 words, keeping 488290 word types\n",
      "2019-12-04 18:38:03,962 : INFO : PROGRESS: at sentence #23100000, processed 278826872 words, keeping 494602 word types\n",
      "2019-12-04 18:38:04,401 : INFO : pruned out 282139 tokens with count <=10 (before 500018, after 217879)\n",
      "2019-12-04 18:38:04,486 : INFO : PROGRESS: at sentence #23110000, processed 279034094 words, keeping 221238 word types\n",
      "2019-12-04 18:38:04,740 : INFO : PROGRESS: at sentence #23120000, processed 279236350 words, keeping 232683 word types\n",
      "2019-12-04 18:38:05,002 : INFO : PROGRESS: at sentence #23130000, processed 279446772 words, keeping 242406 word types\n",
      "2019-12-04 18:38:05,273 : INFO : PROGRESS: at sentence #23140000, processed 279659558 words, keeping 252232 word types\n",
      "2019-12-04 18:38:05,533 : INFO : PROGRESS: at sentence #23150000, processed 279864731 words, keeping 262198 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:38:05,795 : INFO : PROGRESS: at sentence #23160000, processed 280070646 words, keeping 271924 word types\n",
      "2019-12-04 18:38:06,056 : INFO : PROGRESS: at sentence #23170000, processed 280273916 words, keeping 281585 word types\n",
      "2019-12-04 18:38:06,369 : INFO : PROGRESS: at sentence #23180000, processed 280482826 words, keeping 290984 word types\n",
      "2019-12-04 18:38:06,695 : INFO : PROGRESS: at sentence #23190000, processed 280743516 words, keeping 314520 word types\n",
      "2019-12-04 18:38:06,991 : INFO : PROGRESS: at sentence #23200000, processed 280977986 words, keeping 326769 word types\n",
      "2019-12-04 18:38:07,279 : INFO : PROGRESS: at sentence #23210000, processed 281206685 words, keeping 345862 word types\n",
      "2019-12-04 18:38:07,547 : INFO : PROGRESS: at sentence #23220000, processed 281415266 words, keeping 355144 word types\n",
      "2019-12-04 18:38:07,821 : INFO : PROGRESS: at sentence #23230000, processed 281629550 words, keeping 366252 word types\n",
      "2019-12-04 18:38:08,083 : INFO : PROGRESS: at sentence #23240000, processed 281835427 words, keeping 375607 word types\n",
      "2019-12-04 18:38:08,374 : INFO : PROGRESS: at sentence #23250000, processed 282067376 words, keeping 389641 word types\n",
      "2019-12-04 18:38:08,632 : INFO : PROGRESS: at sentence #23260000, processed 282270739 words, keeping 397115 word types\n",
      "2019-12-04 18:38:08,906 : INFO : PROGRESS: at sentence #23270000, processed 282487187 words, keeping 406805 word types\n",
      "2019-12-04 18:38:09,166 : INFO : PROGRESS: at sentence #23280000, processed 282693115 words, keeping 414091 word types\n",
      "2019-12-04 18:38:09,419 : INFO : PROGRESS: at sentence #23290000, processed 282895536 words, keeping 422008 word types\n",
      "2019-12-04 18:38:09,668 : INFO : PROGRESS: at sentence #23300000, processed 283093160 words, keeping 430891 word types\n",
      "2019-12-04 18:38:09,924 : INFO : PROGRESS: at sentence #23310000, processed 283296758 words, keeping 438959 word types\n",
      "2019-12-04 18:38:10,193 : INFO : PROGRESS: at sentence #23320000, processed 283508617 words, keeping 447725 word types\n",
      "2019-12-04 18:38:10,445 : INFO : PROGRESS: at sentence #23330000, processed 283709370 words, keeping 455690 word types\n",
      "2019-12-04 18:38:10,702 : INFO : PROGRESS: at sentence #23340000, processed 283914387 words, keeping 463953 word types\n",
      "2019-12-04 18:38:10,954 : INFO : PROGRESS: at sentence #23350000, processed 284115220 words, keeping 471666 word types\n",
      "2019-12-04 18:38:11,216 : INFO : PROGRESS: at sentence #23360000, processed 284319627 words, keeping 479462 word types\n",
      "2019-12-04 18:38:11,479 : INFO : PROGRESS: at sentence #23370000, processed 284525118 words, keeping 487661 word types\n",
      "2019-12-04 18:38:11,742 : INFO : PROGRESS: at sentence #23380000, processed 284732054 words, keeping 495573 word types\n",
      "2019-12-04 18:38:12,154 : INFO : pruned out 284077 tokens with count <=11 (before 500002, after 215925)\n",
      "2019-12-04 18:38:12,287 : INFO : PROGRESS: at sentence #23390000, processed 284948388 words, keeping 221595 word types\n",
      "2019-12-04 18:38:12,549 : INFO : PROGRESS: at sentence #23400000, processed 285147742 words, keeping 230991 word types\n",
      "2019-12-04 18:38:12,801 : INFO : PROGRESS: at sentence #23410000, processed 285343863 words, keeping 240028 word types\n",
      "2019-12-04 18:38:13,062 : INFO : PROGRESS: at sentence #23420000, processed 285548020 words, keeping 249937 word types\n",
      "2019-12-04 18:38:13,335 : INFO : PROGRESS: at sentence #23430000, processed 285765443 words, keeping 260087 word types\n",
      "2019-12-04 18:38:13,598 : INFO : PROGRESS: at sentence #23440000, processed 285973840 words, keeping 270330 word types\n",
      "2019-12-04 18:38:13,882 : INFO : PROGRESS: at sentence #23450000, processed 286187880 words, keeping 281041 word types\n",
      "2019-12-04 18:38:14,143 : INFO : PROGRESS: at sentence #23460000, processed 286388633 words, keeping 289991 word types\n",
      "2019-12-04 18:38:14,419 : INFO : PROGRESS: at sentence #23470000, processed 286603691 words, keeping 303975 word types\n",
      "2019-12-04 18:38:14,706 : INFO : PROGRESS: at sentence #23480000, processed 286827975 words, keeping 313627 word types\n",
      "2019-12-04 18:38:14,962 : INFO : PROGRESS: at sentence #23490000, processed 287028145 words, keeping 323051 word types\n",
      "2019-12-04 18:38:15,222 : INFO : PROGRESS: at sentence #23500000, processed 287231610 words, keeping 331785 word types\n",
      "2019-12-04 18:38:15,470 : INFO : PROGRESS: at sentence #23510000, processed 287423704 words, keeping 339333 word types\n",
      "2019-12-04 18:38:15,722 : INFO : PROGRESS: at sentence #23520000, processed 287621493 words, keeping 347435 word types\n",
      "2019-12-04 18:38:15,985 : INFO : PROGRESS: at sentence #23530000, processed 287824299 words, keeping 355704 word types\n",
      "2019-12-04 18:38:16,252 : INFO : PROGRESS: at sentence #23540000, processed 288035914 words, keeping 365615 word types\n",
      "2019-12-04 18:38:16,498 : INFO : PROGRESS: at sentence #23550000, processed 288227145 words, keeping 374211 word types\n",
      "2019-12-04 18:38:16,753 : INFO : PROGRESS: at sentence #23560000, processed 288429339 words, keeping 382579 word types\n",
      "2019-12-04 18:38:17,035 : INFO : PROGRESS: at sentence #23570000, processed 288656177 words, keeping 392587 word types\n",
      "2019-12-04 18:38:17,308 : INFO : PROGRESS: at sentence #23580000, processed 288875362 words, keeping 402575 word types\n",
      "2019-12-04 18:38:17,561 : INFO : PROGRESS: at sentence #23590000, processed 289073287 words, keeping 410619 word types\n",
      "2019-12-04 18:38:17,836 : INFO : PROGRESS: at sentence #23600000, processed 289296509 words, keeping 418016 word types\n",
      "2019-12-04 18:38:18,087 : INFO : PROGRESS: at sentence #23610000, processed 289491358 words, keeping 425048 word types\n",
      "2019-12-04 18:38:18,355 : INFO : PROGRESS: at sentence #23620000, processed 289698647 words, keeping 433378 word types\n",
      "2019-12-04 18:38:18,597 : INFO : PROGRESS: at sentence #23630000, processed 289887873 words, keeping 439731 word types\n",
      "2019-12-04 18:38:18,857 : INFO : PROGRESS: at sentence #23640000, processed 290090451 words, keeping 447531 word types\n",
      "2019-12-04 18:38:19,135 : INFO : PROGRESS: at sentence #23650000, processed 290308784 words, keeping 456348 word types\n",
      "2019-12-04 18:38:19,400 : INFO : PROGRESS: at sentence #23660000, processed 290526572 words, keeping 464693 word types\n",
      "2019-12-04 18:38:19,649 : INFO : PROGRESS: at sentence #23670000, processed 290730699 words, keeping 471984 word types\n",
      "2019-12-04 18:38:19,920 : INFO : PROGRESS: at sentence #23680000, processed 290955176 words, keeping 480378 word types\n",
      "2019-12-04 18:38:20,170 : INFO : PROGRESS: at sentence #23690000, processed 291158744 words, keeping 487583 word types\n",
      "2019-12-04 18:38:20,437 : INFO : PROGRESS: at sentence #23700000, processed 291372411 words, keeping 495388 word types\n",
      "2019-12-04 18:38:20,862 : INFO : pruned out 285287 tokens with count <=12 (before 500001, after 214714)\n",
      "2019-12-04 18:38:20,956 : INFO : PROGRESS: at sentence #23710000, processed 291578931 words, keeping 218456 word types\n",
      "2019-12-04 18:38:21,207 : INFO : PROGRESS: at sentence #23720000, processed 291779662 words, keeping 228254 word types\n",
      "2019-12-04 18:38:21,475 : INFO : PROGRESS: at sentence #23730000, processed 291991641 words, keeping 239598 word types\n",
      "2019-12-04 18:38:21,745 : INFO : PROGRESS: at sentence #23740000, processed 292203053 words, keeping 249162 word types\n",
      "2019-12-04 18:38:22,006 : INFO : PROGRESS: at sentence #23750000, processed 292405333 words, keeping 258702 word types\n",
      "2019-12-04 18:38:22,274 : INFO : PROGRESS: at sentence #23760000, processed 292617042 words, keeping 269111 word types\n",
      "2019-12-04 18:38:22,552 : INFO : PROGRESS: at sentence #23770000, processed 292832699 words, keeping 277490 word types\n",
      "2019-12-04 18:38:22,852 : INFO : PROGRESS: at sentence #23780000, processed 293037803 words, keeping 286726 word types\n",
      "2019-12-04 18:38:23,119 : INFO : PROGRESS: at sentence #23790000, processed 293245962 words, keeping 296619 word types\n",
      "2019-12-04 18:38:23,372 : INFO : PROGRESS: at sentence #23800000, processed 293441024 words, keeping 304250 word types\n",
      "2019-12-04 18:38:23,636 : INFO : PROGRESS: at sentence #23810000, processed 293645086 words, keeping 314239 word types\n",
      "2019-12-04 18:38:23,898 : INFO : PROGRESS: at sentence #23820000, processed 293852430 words, keeping 322726 word types\n",
      "2019-12-04 18:38:24,179 : INFO : PROGRESS: at sentence #23830000, processed 294056524 words, keeping 332405 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:38:24,435 : INFO : PROGRESS: at sentence #23840000, processed 294254270 words, keeping 341807 word types\n",
      "2019-12-04 18:38:24,700 : INFO : PROGRESS: at sentence #23850000, processed 294461267 words, keeping 349830 word types\n",
      "2019-12-04 18:38:24,963 : INFO : PROGRESS: at sentence #23860000, processed 294666361 words, keeping 357993 word types\n",
      "2019-12-04 18:38:25,256 : INFO : PROGRESS: at sentence #23870000, processed 294872093 words, keeping 366549 word types\n",
      "2019-12-04 18:38:25,518 : INFO : PROGRESS: at sentence #23880000, processed 295072369 words, keeping 374898 word types\n",
      "2019-12-04 18:38:25,771 : INFO : PROGRESS: at sentence #23890000, processed 295268705 words, keeping 382594 word types\n",
      "2019-12-04 18:38:26,034 : INFO : PROGRESS: at sentence #23900000, processed 295474583 words, keeping 392014 word types\n",
      "2019-12-04 18:38:26,288 : INFO : PROGRESS: at sentence #23910000, processed 295673525 words, keeping 400095 word types\n",
      "2019-12-04 18:38:26,552 : INFO : PROGRESS: at sentence #23920000, processed 295877765 words, keeping 408217 word types\n",
      "2019-12-04 18:38:26,818 : INFO : PROGRESS: at sentence #23930000, processed 296084903 words, keeping 416097 word types\n",
      "2019-12-04 18:38:27,085 : INFO : PROGRESS: at sentence #23940000, processed 296275885 words, keeping 423311 word types\n",
      "2019-12-04 18:38:27,350 : INFO : PROGRESS: at sentence #23950000, processed 296482212 words, keeping 430614 word types\n",
      "2019-12-04 18:38:27,609 : INFO : PROGRESS: at sentence #23960000, processed 296686059 words, keeping 439610 word types\n",
      "2019-12-04 18:38:27,882 : INFO : PROGRESS: at sentence #23970000, processed 296897607 words, keeping 449686 word types\n",
      "2019-12-04 18:38:28,148 : INFO : PROGRESS: at sentence #23980000, processed 297105708 words, keeping 458211 word types\n",
      "2019-12-04 18:38:28,422 : INFO : PROGRESS: at sentence #23990000, processed 297329462 words, keeping 465038 word types\n",
      "2019-12-04 18:38:28,685 : INFO : PROGRESS: at sentence #24000000, processed 297538498 words, keeping 472053 word types\n",
      "2019-12-04 18:38:28,956 : INFO : PROGRESS: at sentence #24010000, processed 297749652 words, keeping 479825 word types\n",
      "2019-12-04 18:38:29,226 : INFO : PROGRESS: at sentence #24020000, processed 297959106 words, keeping 487186 word types\n",
      "2019-12-04 18:38:29,483 : INFO : PROGRESS: at sentence #24030000, processed 298160028 words, keeping 494739 word types\n",
      "2019-12-04 18:38:29,953 : INFO : pruned out 286263 tokens with count <=13 (before 500006, after 213743)\n",
      "2019-12-04 18:38:30,052 : INFO : PROGRESS: at sentence #24040000, processed 298363709 words, keeping 217929 word types\n",
      "2019-12-04 18:38:30,316 : INFO : PROGRESS: at sentence #24050000, processed 298562697 words, keeping 228773 word types\n",
      "2019-12-04 18:38:30,557 : INFO : PROGRESS: at sentence #24060000, processed 298746485 words, keeping 238191 word types\n",
      "2019-12-04 18:38:30,819 : INFO : PROGRESS: at sentence #24070000, processed 298948868 words, keeping 249569 word types\n",
      "2019-12-04 18:38:31,082 : INFO : PROGRESS: at sentence #24080000, processed 299151799 words, keeping 259670 word types\n",
      "2019-12-04 18:38:31,358 : INFO : PROGRESS: at sentence #24090000, processed 299367775 words, keeping 270079 word types\n",
      "2019-12-04 18:38:31,622 : INFO : PROGRESS: at sentence #24100000, processed 299569557 words, keeping 279722 word types\n",
      "2019-12-04 18:38:31,889 : INFO : PROGRESS: at sentence #24110000, processed 299778396 words, keeping 289331 word types\n",
      "2019-12-04 18:38:32,159 : INFO : PROGRESS: at sentence #24120000, processed 299992783 words, keeping 299053 word types\n",
      "2019-12-04 18:38:32,413 : INFO : PROGRESS: at sentence #24130000, processed 300197961 words, keeping 309089 word types\n",
      "2019-12-04 18:38:32,690 : INFO : PROGRESS: at sentence #24140000, processed 300413473 words, keeping 319313 word types\n",
      "2019-12-04 18:38:32,933 : INFO : PROGRESS: at sentence #24150000, processed 300607988 words, keeping 328234 word types\n",
      "2019-12-04 18:38:33,197 : INFO : PROGRESS: at sentence #24160000, processed 300817212 words, keeping 337771 word types\n",
      "2019-12-04 18:38:33,468 : INFO : PROGRESS: at sentence #24170000, processed 301030870 words, keeping 346760 word types\n",
      "2019-12-04 18:38:33,733 : INFO : PROGRESS: at sentence #24180000, processed 301244996 words, keeping 355893 word types\n",
      "2019-12-04 18:38:34,007 : INFO : PROGRESS: at sentence #24190000, processed 301460102 words, keeping 364336 word types\n",
      "2019-12-04 18:38:34,265 : INFO : PROGRESS: at sentence #24200000, processed 301662094 words, keeping 372714 word types\n",
      "2019-12-04 18:38:34,521 : INFO : PROGRESS: at sentence #24210000, processed 301863657 words, keeping 381924 word types\n",
      "2019-12-04 18:38:34,780 : INFO : PROGRESS: at sentence #24220000, processed 302063262 words, keeping 390033 word types\n",
      "2019-12-04 18:38:35,040 : INFO : PROGRESS: at sentence #24230000, processed 302265323 words, keeping 397252 word types\n",
      "2019-12-04 18:38:35,309 : INFO : PROGRESS: at sentence #24240000, processed 302472365 words, keeping 405745 word types\n",
      "2019-12-04 18:38:35,564 : INFO : PROGRESS: at sentence #24250000, processed 302672578 words, keeping 413339 word types\n",
      "2019-12-04 18:38:35,841 : INFO : PROGRESS: at sentence #24260000, processed 302887790 words, keeping 422047 word types\n",
      "2019-12-04 18:38:36,106 : INFO : PROGRESS: at sentence #24270000, processed 303096024 words, keeping 433822 word types\n",
      "2019-12-04 18:38:36,370 : INFO : PROGRESS: at sentence #24280000, processed 303304041 words, keeping 441826 word types\n",
      "2019-12-04 18:38:36,625 : INFO : PROGRESS: at sentence #24290000, processed 303503376 words, keeping 449624 word types\n",
      "2019-12-04 18:38:36,882 : INFO : PROGRESS: at sentence #24300000, processed 303705826 words, keeping 457563 word types\n",
      "2019-12-04 18:38:37,150 : INFO : PROGRESS: at sentence #24310000, processed 303917804 words, keeping 466184 word types\n",
      "2019-12-04 18:38:37,424 : INFO : PROGRESS: at sentence #24320000, processed 304131976 words, keeping 474796 word types\n",
      "2019-12-04 18:38:37,683 : INFO : PROGRESS: at sentence #24330000, processed 304336323 words, keeping 484009 word types\n",
      "2019-12-04 18:38:37,943 : INFO : PROGRESS: at sentence #24340000, processed 304537489 words, keeping 492319 word types\n",
      "2019-12-04 18:38:38,197 : INFO : PROGRESS: at sentence #24350000, processed 304735767 words, keeping 499691 word types\n",
      "2019-12-04 18:38:38,462 : INFO : pruned out 287200 tokens with count <=14 (before 500001, after 212801)\n",
      "2019-12-04 18:38:38,715 : INFO : PROGRESS: at sentence #24360000, processed 304936916 words, keeping 223847 word types\n",
      "2019-12-04 18:38:38,991 : INFO : PROGRESS: at sentence #24370000, processed 305151067 words, keeping 234846 word types\n",
      "2019-12-04 18:38:39,256 : INFO : PROGRESS: at sentence #24380000, processed 305354344 words, keeping 245827 word types\n",
      "2019-12-04 18:38:39,573 : INFO : PROGRESS: at sentence #24390000, processed 305566199 words, keeping 255982 word types\n",
      "2019-12-04 18:38:39,844 : INFO : PROGRESS: at sentence #24400000, processed 305775977 words, keeping 265503 word types\n",
      "2019-12-04 18:38:40,111 : INFO : PROGRESS: at sentence #24410000, processed 305987063 words, keeping 275646 word types\n",
      "2019-12-04 18:38:40,367 : INFO : PROGRESS: at sentence #24420000, processed 306189664 words, keeping 285532 word types\n",
      "2019-12-04 18:38:40,621 : INFO : PROGRESS: at sentence #24430000, processed 306390105 words, keeping 294176 word types\n",
      "2019-12-04 18:38:40,877 : INFO : PROGRESS: at sentence #24440000, processed 306590414 words, keeping 304336 word types\n",
      "2019-12-04 18:38:41,134 : INFO : PROGRESS: at sentence #24450000, processed 306791253 words, keeping 313273 word types\n",
      "2019-12-04 18:38:41,401 : INFO : PROGRESS: at sentence #24460000, processed 306996191 words, keeping 322569 word types\n",
      "2019-12-04 18:38:41,651 : INFO : PROGRESS: at sentence #24470000, processed 307196803 words, keeping 330462 word types\n",
      "2019-12-04 18:38:41,907 : INFO : PROGRESS: at sentence #24480000, processed 307394704 words, keeping 339358 word types\n",
      "2019-12-04 18:38:42,157 : INFO : PROGRESS: at sentence #24490000, processed 307591220 words, keeping 348017 word types\n",
      "2019-12-04 18:38:42,416 : INFO : PROGRESS: at sentence #24500000, processed 307797295 words, keeping 356454 word types\n",
      "2019-12-04 18:38:42,685 : INFO : PROGRESS: at sentence #24510000, processed 308009429 words, keeping 366179 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:38:42,948 : INFO : PROGRESS: at sentence #24520000, processed 308212334 words, keeping 374812 word types\n",
      "2019-12-04 18:38:43,206 : INFO : PROGRESS: at sentence #24530000, processed 308413671 words, keeping 383318 word types\n",
      "2019-12-04 18:38:43,470 : INFO : PROGRESS: at sentence #24540000, processed 308620500 words, keeping 390935 word types\n",
      "2019-12-04 18:38:43,741 : INFO : PROGRESS: at sentence #24550000, processed 308837675 words, keeping 399885 word types\n",
      "2019-12-04 18:38:43,996 : INFO : PROGRESS: at sentence #24560000, processed 309040897 words, keeping 407660 word types\n",
      "2019-12-04 18:38:44,256 : INFO : PROGRESS: at sentence #24570000, processed 309245957 words, keeping 416747 word types\n",
      "2019-12-04 18:38:44,506 : INFO : PROGRESS: at sentence #24580000, processed 309443559 words, keeping 425165 word types\n",
      "2019-12-04 18:38:44,759 : INFO : PROGRESS: at sentence #24590000, processed 309642461 words, keeping 433348 word types\n",
      "2019-12-04 18:38:45,008 : INFO : PROGRESS: at sentence #24600000, processed 309845139 words, keeping 441410 word types\n",
      "2019-12-04 18:38:45,266 : INFO : PROGRESS: at sentence #24610000, processed 310055294 words, keeping 449700 word types\n",
      "2019-12-04 18:38:45,504 : INFO : PROGRESS: at sentence #24620000, processed 310261389 words, keeping 458365 word types\n",
      "2019-12-04 18:38:45,740 : INFO : PROGRESS: at sentence #24630000, processed 310463605 words, keeping 465885 word types\n",
      "2019-12-04 18:38:45,977 : INFO : PROGRESS: at sentence #24640000, processed 310659885 words, keeping 473737 word types\n",
      "2019-12-04 18:38:46,223 : INFO : PROGRESS: at sentence #24650000, processed 310860562 words, keeping 481994 word types\n",
      "2019-12-04 18:38:46,471 : INFO : PROGRESS: at sentence #24660000, processed 311062780 words, keeping 489780 word types\n",
      "2019-12-04 18:38:46,716 : INFO : PROGRESS: at sentence #24670000, processed 311266783 words, keeping 498079 word types\n",
      "2019-12-04 18:38:47,044 : INFO : pruned out 287893 tokens with count <=15 (before 500002, after 212109)\n",
      "2019-12-04 18:38:47,257 : INFO : PROGRESS: at sentence #24680000, processed 311473986 words, keeping 220450 word types\n",
      "2019-12-04 18:38:47,525 : INFO : PROGRESS: at sentence #24690000, processed 311676147 words, keeping 231228 word types\n",
      "2019-12-04 18:38:47,802 : INFO : PROGRESS: at sentence #24700000, processed 311887124 words, keeping 242550 word types\n",
      "2019-12-04 18:38:48,064 : INFO : PROGRESS: at sentence #24710000, processed 312088174 words, keeping 252386 word types\n",
      "2019-12-04 18:38:48,347 : INFO : PROGRESS: at sentence #24720000, processed 312303346 words, keeping 262442 word types\n",
      "2019-12-04 18:38:48,623 : INFO : PROGRESS: at sentence #24730000, processed 312513535 words, keeping 272614 word types\n",
      "2019-12-04 18:38:48,890 : INFO : PROGRESS: at sentence #24740000, processed 312713943 words, keeping 282325 word types\n",
      "2019-12-04 18:38:49,152 : INFO : PROGRESS: at sentence #24750000, processed 312914351 words, keeping 291750 word types\n",
      "2019-12-04 18:38:49,427 : INFO : PROGRESS: at sentence #24760000, processed 313128286 words, keeping 301297 word types\n",
      "2019-12-04 18:38:49,707 : INFO : PROGRESS: at sentence #24770000, processed 313344539 words, keeping 311593 word types\n",
      "2019-12-04 18:38:49,967 : INFO : PROGRESS: at sentence #24780000, processed 313543330 words, keeping 320349 word types\n",
      "2019-12-04 18:38:50,229 : INFO : PROGRESS: at sentence #24790000, processed 313742179 words, keeping 330469 word types\n",
      "2019-12-04 18:38:50,500 : INFO : PROGRESS: at sentence #24800000, processed 313954425 words, keeping 338770 word types\n",
      "2019-12-04 18:38:50,749 : INFO : PROGRESS: at sentence #24810000, processed 314145688 words, keeping 346952 word types\n",
      "2019-12-04 18:38:51,016 : INFO : PROGRESS: at sentence #24820000, processed 314349520 words, keeping 355519 word types\n",
      "2019-12-04 18:38:51,283 : INFO : PROGRESS: at sentence #24830000, processed 314552756 words, keeping 364821 word types\n",
      "2019-12-04 18:38:51,544 : INFO : PROGRESS: at sentence #24840000, processed 314750928 words, keeping 374042 word types\n",
      "2019-12-04 18:38:51,808 : INFO : PROGRESS: at sentence #24850000, processed 314953965 words, keeping 382291 word types\n",
      "2019-12-04 18:38:52,075 : INFO : PROGRESS: at sentence #24860000, processed 315156900 words, keeping 390970 word types\n",
      "2019-12-04 18:38:52,349 : INFO : PROGRESS: at sentence #24870000, processed 315367694 words, keeping 399640 word types\n",
      "2019-12-04 18:38:52,615 : INFO : PROGRESS: at sentence #24880000, processed 315570017 words, keeping 408537 word types\n",
      "2019-12-04 18:38:52,885 : INFO : PROGRESS: at sentence #24890000, processed 315776450 words, keeping 416892 word types\n",
      "2019-12-04 18:38:53,156 : INFO : PROGRESS: at sentence #24900000, processed 315982185 words, keeping 426144 word types\n",
      "2019-12-04 18:38:53,443 : INFO : PROGRESS: at sentence #24910000, processed 316205366 words, keeping 433770 word types\n",
      "2019-12-04 18:38:53,731 : INFO : PROGRESS: at sentence #24920000, processed 316408757 words, keeping 442341 word types\n",
      "2019-12-04 18:38:53,988 : INFO : PROGRESS: at sentence #24930000, processed 316602863 words, keeping 450664 word types\n",
      "2019-12-04 18:38:54,263 : INFO : PROGRESS: at sentence #24940000, processed 316810693 words, keeping 458515 word types\n",
      "2019-12-04 18:38:54,532 : INFO : PROGRESS: at sentence #24950000, processed 317013052 words, keeping 466671 word types\n",
      "2019-12-04 18:38:54,809 : INFO : PROGRESS: at sentence #24960000, processed 317225140 words, keeping 474534 word types\n",
      "2019-12-04 18:38:55,061 : INFO : PROGRESS: at sentence #24970000, processed 317417678 words, keeping 482399 word types\n",
      "2019-12-04 18:38:55,333 : INFO : PROGRESS: at sentence #24980000, processed 317627972 words, keeping 491057 word types\n",
      "2019-12-04 18:38:55,591 : INFO : PROGRESS: at sentence #24990000, processed 317825459 words, keeping 498614 word types\n",
      "2019-12-04 18:38:55,929 : INFO : pruned out 288493 tokens with count <=16 (before 500001, after 211508)\n",
      "2019-12-04 18:38:56,136 : INFO : PROGRESS: at sentence #25000000, processed 318029767 words, keeping 220324 word types\n",
      "2019-12-04 18:38:56,407 : INFO : PROGRESS: at sentence #25010000, processed 318238466 words, keeping 231057 word types\n",
      "2019-12-04 18:38:56,681 : INFO : PROGRESS: at sentence #25020000, processed 318445885 words, keeping 240814 word types\n",
      "2019-12-04 18:38:56,949 : INFO : PROGRESS: at sentence #25030000, processed 318649853 words, keeping 251160 word types\n",
      "2019-12-04 18:38:57,224 : INFO : PROGRESS: at sentence #25040000, processed 318860121 words, keeping 261138 word types\n",
      "2019-12-04 18:38:57,486 : INFO : PROGRESS: at sentence #25050000, processed 319061013 words, keeping 270448 word types\n",
      "2019-12-04 18:38:57,754 : INFO : PROGRESS: at sentence #25060000, processed 319267609 words, keeping 279703 word types\n",
      "2019-12-04 18:38:58,022 : INFO : PROGRESS: at sentence #25070000, processed 319470586 words, keeping 289568 word types\n",
      "2019-12-04 18:38:58,294 : INFO : PROGRESS: at sentence #25080000, processed 319679523 words, keeping 299651 word types\n",
      "2019-12-04 18:38:58,565 : INFO : PROGRESS: at sentence #25090000, processed 319888132 words, keeping 309301 word types\n",
      "2019-12-04 18:38:58,836 : INFO : PROGRESS: at sentence #25100000, processed 320097891 words, keeping 318334 word types\n",
      "2019-12-04 18:38:59,112 : INFO : PROGRESS: at sentence #25110000, processed 320305782 words, keeping 328100 word types\n",
      "2019-12-04 18:38:59,400 : INFO : PROGRESS: at sentence #25120000, processed 320535746 words, keeping 340017 word types\n",
      "2019-12-04 18:38:59,675 : INFO : PROGRESS: at sentence #25130000, processed 320750113 words, keeping 349090 word types\n",
      "2019-12-04 18:38:59,942 : INFO : PROGRESS: at sentence #25140000, processed 320953780 words, keeping 357163 word types\n",
      "2019-12-04 18:39:00,215 : INFO : PROGRESS: at sentence #25150000, processed 321157775 words, keeping 366680 word types\n",
      "2019-12-04 18:39:00,480 : INFO : PROGRESS: at sentence #25160000, processed 321358771 words, keeping 374971 word types\n",
      "2019-12-04 18:39:00,746 : INFO : PROGRESS: at sentence #25170000, processed 321561946 words, keeping 382958 word types\n",
      "2019-12-04 18:39:01,008 : INFO : PROGRESS: at sentence #25180000, processed 321759724 words, keeping 391183 word types\n",
      "2019-12-04 18:39:01,279 : INFO : PROGRESS: at sentence #25190000, processed 321963765 words, keeping 399360 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:39:01,565 : INFO : PROGRESS: at sentence #25200000, processed 322183305 words, keeping 409054 word types\n",
      "2019-12-04 18:39:01,845 : INFO : PROGRESS: at sentence #25210000, processed 322393884 words, keeping 418797 word types\n",
      "2019-12-04 18:39:02,124 : INFO : PROGRESS: at sentence #25220000, processed 322609404 words, keeping 427183 word types\n",
      "2019-12-04 18:39:02,394 : INFO : PROGRESS: at sentence #25230000, processed 322814456 words, keeping 435426 word types\n",
      "2019-12-04 18:39:02,683 : INFO : PROGRESS: at sentence #25240000, processed 323039236 words, keeping 443925 word types\n",
      "2019-12-04 18:39:02,963 : INFO : PROGRESS: at sentence #25250000, processed 323251473 words, keeping 452782 word types\n",
      "2019-12-04 18:39:03,227 : INFO : PROGRESS: at sentence #25260000, processed 323452664 words, keeping 460989 word types\n",
      "2019-12-04 18:39:03,504 : INFO : PROGRESS: at sentence #25270000, processed 323659967 words, keeping 469333 word types\n",
      "2019-12-04 18:39:03,770 : INFO : PROGRESS: at sentence #25280000, processed 323873195 words, keeping 477362 word types\n",
      "2019-12-04 18:39:04,082 : INFO : PROGRESS: at sentence #25290000, processed 324120822 words, keeping 488069 word types\n",
      "2019-12-04 18:39:04,347 : INFO : PROGRESS: at sentence #25300000, processed 324326032 words, keeping 496877 word types\n",
      "2019-12-04 18:39:04,724 : INFO : pruned out 289065 tokens with count <=17 (before 500001, after 210936)\n",
      "2019-12-04 18:39:04,881 : INFO : PROGRESS: at sentence #25310000, processed 324525248 words, keeping 217177 word types\n",
      "2019-12-04 18:39:05,164 : INFO : PROGRESS: at sentence #25320000, processed 324739183 words, keeping 228432 word types\n",
      "2019-12-04 18:39:05,461 : INFO : PROGRESS: at sentence #25330000, processed 324969524 words, keeping 241341 word types\n",
      "2019-12-04 18:39:05,718 : INFO : PROGRESS: at sentence #25340000, processed 325167507 words, keeping 250470 word types\n",
      "2019-12-04 18:39:05,966 : INFO : PROGRESS: at sentence #25350000, processed 325350865 words, keeping 259949 word types\n",
      "2019-12-04 18:39:06,240 : INFO : PROGRESS: at sentence #25360000, processed 325558970 words, keeping 270770 word types\n",
      "2019-12-04 18:39:06,521 : INFO : PROGRESS: at sentence #25370000, processed 325778345 words, keeping 280738 word types\n",
      "2019-12-04 18:39:06,792 : INFO : PROGRESS: at sentence #25380000, processed 325987319 words, keeping 290627 word types\n",
      "2019-12-04 18:39:07,053 : INFO : PROGRESS: at sentence #25390000, processed 326187256 words, keeping 299625 word types\n",
      "2019-12-04 18:39:07,321 : INFO : PROGRESS: at sentence #25400000, processed 326385807 words, keeping 308761 word types\n",
      "2019-12-04 18:39:07,561 : INFO : PROGRESS: at sentence #25410000, processed 326569366 words, keeping 317694 word types\n",
      "2019-12-04 18:39:07,841 : INFO : PROGRESS: at sentence #25420000, processed 326782173 words, keeping 327028 word types\n",
      "2019-12-04 18:39:08,111 : INFO : PROGRESS: at sentence #25430000, processed 326986747 words, keeping 336461 word types\n",
      "2019-12-04 18:39:08,379 : INFO : PROGRESS: at sentence #25440000, processed 327192493 words, keeping 345288 word types\n",
      "2019-12-04 18:39:08,651 : INFO : PROGRESS: at sentence #25450000, processed 327403558 words, keeping 354442 word types\n",
      "2019-12-04 18:39:08,911 : INFO : PROGRESS: at sentence #25460000, processed 327606641 words, keeping 363856 word types\n",
      "2019-12-04 18:39:09,170 : INFO : PROGRESS: at sentence #25470000, processed 327805007 words, keeping 372003 word types\n",
      "2019-12-04 18:39:09,428 : INFO : PROGRESS: at sentence #25480000, processed 328003160 words, keeping 380035 word types\n",
      "2019-12-04 18:39:09,689 : INFO : PROGRESS: at sentence #25490000, processed 328204032 words, keeping 388558 word types\n",
      "2019-12-04 18:39:09,949 : INFO : PROGRESS: at sentence #25500000, processed 328403966 words, keeping 396558 word types\n",
      "2019-12-04 18:39:10,215 : INFO : PROGRESS: at sentence #25510000, processed 328617154 words, keeping 405496 word types\n",
      "2019-12-04 18:39:10,466 : INFO : PROGRESS: at sentence #25520000, processed 328814741 words, keeping 413989 word types\n",
      "2019-12-04 18:39:10,717 : INFO : PROGRESS: at sentence #25530000, processed 329012837 words, keeping 422256 word types\n",
      "2019-12-04 18:39:10,986 : INFO : PROGRESS: at sentence #25540000, processed 329225802 words, keeping 431538 word types\n",
      "2019-12-04 18:39:11,256 : INFO : PROGRESS: at sentence #25550000, processed 329435194 words, keeping 440276 word types\n",
      "2019-12-04 18:39:11,550 : INFO : PROGRESS: at sentence #25560000, processed 329639953 words, keeping 448205 word types\n",
      "2019-12-04 18:39:11,820 : INFO : PROGRESS: at sentence #25570000, processed 329850243 words, keeping 456796 word types\n",
      "2019-12-04 18:39:12,089 : INFO : PROGRESS: at sentence #25580000, processed 330056827 words, keeping 465115 word types\n",
      "2019-12-04 18:39:12,361 : INFO : PROGRESS: at sentence #25590000, processed 330261610 words, keeping 473834 word types\n",
      "2019-12-04 18:39:12,636 : INFO : PROGRESS: at sentence #25600000, processed 330470289 words, keeping 481040 word types\n",
      "2019-12-04 18:39:12,900 : INFO : PROGRESS: at sentence #25610000, processed 330668575 words, keeping 488743 word types\n",
      "2019-12-04 18:39:13,176 : INFO : PROGRESS: at sentence #25620000, processed 330878464 words, keeping 497365 word types\n",
      "2019-12-04 18:39:13,527 : INFO : pruned out 289594 tokens with count <=18 (before 500015, after 210421)\n",
      "2019-12-04 18:39:13,717 : INFO : PROGRESS: at sentence #25630000, processed 331087590 words, keeping 218032 word types\n",
      "2019-12-04 18:39:13,993 : INFO : PROGRESS: at sentence #25640000, processed 331289701 words, keeping 227977 word types\n",
      "2019-12-04 18:39:14,275 : INFO : PROGRESS: at sentence #25650000, processed 331495747 words, keeping 238745 word types\n",
      "2019-12-04 18:39:14,608 : INFO : PROGRESS: at sentence #25660000, processed 331712621 words, keeping 249028 word types\n",
      "2019-12-04 18:39:14,870 : INFO : PROGRESS: at sentence #25670000, processed 331912695 words, keeping 259547 word types\n",
      "2019-12-04 18:39:15,127 : INFO : PROGRESS: at sentence #25680000, processed 332113601 words, keeping 268549 word types\n",
      "2019-12-04 18:39:15,387 : INFO : PROGRESS: at sentence #25690000, processed 332314724 words, keeping 278542 word types\n",
      "2019-12-04 18:39:15,646 : INFO : PROGRESS: at sentence #25700000, processed 332517883 words, keeping 289161 word types\n",
      "2019-12-04 18:39:15,894 : INFO : PROGRESS: at sentence #25710000, processed 332713337 words, keeping 298510 word types\n",
      "2019-12-04 18:39:16,156 : INFO : PROGRESS: at sentence #25720000, processed 332917333 words, keeping 307738 word types\n",
      "2019-12-04 18:39:16,427 : INFO : PROGRESS: at sentence #25730000, processed 333130113 words, keeping 316896 word types\n",
      "2019-12-04 18:39:16,687 : INFO : PROGRESS: at sentence #25740000, processed 333332366 words, keeping 327409 word types\n",
      "2019-12-04 18:39:16,949 : INFO : PROGRESS: at sentence #25750000, processed 333535685 words, keeping 336590 word types\n",
      "2019-12-04 18:39:17,203 : INFO : PROGRESS: at sentence #25760000, processed 333733597 words, keeping 346630 word types\n",
      "2019-12-04 18:39:17,455 : INFO : PROGRESS: at sentence #25770000, processed 333929950 words, keeping 354846 word types\n",
      "2019-12-04 18:39:17,711 : INFO : PROGRESS: at sentence #25780000, processed 334131745 words, keeping 363610 word types\n",
      "2019-12-04 18:39:17,976 : INFO : PROGRESS: at sentence #25790000, processed 334341557 words, keeping 373743 word types\n",
      "2019-12-04 18:39:18,232 : INFO : PROGRESS: at sentence #25800000, processed 334534710 words, keeping 381987 word types\n",
      "2019-12-04 18:39:18,485 : INFO : PROGRESS: at sentence #25810000, processed 334731534 words, keeping 390459 word types\n",
      "2019-12-04 18:39:18,747 : INFO : PROGRESS: at sentence #25820000, processed 334942510 words, keeping 399364 word types\n",
      "2019-12-04 18:39:19,001 : INFO : PROGRESS: at sentence #25830000, processed 335141021 words, keeping 408210 word types\n",
      "2019-12-04 18:39:19,265 : INFO : PROGRESS: at sentence #25840000, processed 335352262 words, keeping 416507 word types\n",
      "2019-12-04 18:39:19,505 : INFO : PROGRESS: at sentence #25850000, processed 335542107 words, keeping 424690 word types\n",
      "2019-12-04 18:39:19,780 : INFO : PROGRESS: at sentence #25860000, processed 335766252 words, keeping 434243 word types\n",
      "2019-12-04 18:39:20,032 : INFO : PROGRESS: at sentence #25870000, processed 335963366 words, keeping 442250 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:39:20,303 : INFO : PROGRESS: at sentence #25880000, processed 336180290 words, keeping 451572 word types\n",
      "2019-12-04 18:39:20,554 : INFO : PROGRESS: at sentence #25890000, processed 336379508 words, keeping 460152 word types\n",
      "2019-12-04 18:39:20,827 : INFO : PROGRESS: at sentence #25900000, processed 336596631 words, keeping 469299 word types\n",
      "2019-12-04 18:39:21,095 : INFO : PROGRESS: at sentence #25910000, processed 336804523 words, keeping 477992 word types\n",
      "2019-12-04 18:39:21,356 : INFO : PROGRESS: at sentence #25920000, processed 337004655 words, keeping 485925 word types\n",
      "2019-12-04 18:39:21,663 : INFO : PROGRESS: at sentence #25930000, processed 337231445 words, keeping 494501 word types\n",
      "2019-12-04 18:39:22,111 : INFO : pruned out 289988 tokens with count <=19 (before 500002, after 210014)\n",
      "2019-12-04 18:39:22,203 : INFO : PROGRESS: at sentence #25940000, processed 337439613 words, keeping 214159 word types\n",
      "2019-12-04 18:39:22,474 : INFO : PROGRESS: at sentence #25950000, processed 337652520 words, keeping 225766 word types\n",
      "2019-12-04 18:39:22,730 : INFO : PROGRESS: at sentence #25960000, processed 337849390 words, keeping 236073 word types\n",
      "2019-12-04 18:39:23,003 : INFO : PROGRESS: at sentence #25970000, processed 338058121 words, keeping 248232 word types\n",
      "2019-12-04 18:39:23,277 : INFO : PROGRESS: at sentence #25980000, processed 338271202 words, keeping 259471 word types\n",
      "2019-12-04 18:39:23,553 : INFO : PROGRESS: at sentence #25990000, processed 338486872 words, keeping 270751 word types\n",
      "2019-12-04 18:39:23,835 : INFO : PROGRESS: at sentence #26000000, processed 338710233 words, keeping 282431 word types\n",
      "2019-12-04 18:39:24,100 : INFO : PROGRESS: at sentence #26010000, processed 338920762 words, keeping 291784 word types\n",
      "2019-12-04 18:39:24,364 : INFO : PROGRESS: at sentence #26020000, processed 339129662 words, keeping 301754 word types\n",
      "2019-12-04 18:39:24,630 : INFO : PROGRESS: at sentence #26030000, processed 339343143 words, keeping 311803 word types\n",
      "2019-12-04 18:39:24,898 : INFO : PROGRESS: at sentence #26040000, processed 339554806 words, keeping 321242 word types\n",
      "2019-12-04 18:39:25,168 : INFO : PROGRESS: at sentence #26050000, processed 339765707 words, keeping 330345 word types\n",
      "2019-12-04 18:39:25,428 : INFO : PROGRESS: at sentence #26060000, processed 339972876 words, keeping 339410 word types\n",
      "2019-12-04 18:39:25,683 : INFO : PROGRESS: at sentence #26070000, processed 340177232 words, keeping 349274 word types\n",
      "2019-12-04 18:39:25,957 : INFO : PROGRESS: at sentence #26080000, processed 340390733 words, keeping 358308 word types\n",
      "2019-12-04 18:39:26,222 : INFO : PROGRESS: at sentence #26090000, processed 340595838 words, keeping 367282 word types\n",
      "2019-12-04 18:39:26,473 : INFO : PROGRESS: at sentence #26100000, processed 340792765 words, keeping 376087 word types\n",
      "2019-12-04 18:39:26,726 : INFO : PROGRESS: at sentence #26110000, processed 340992601 words, keeping 385090 word types\n",
      "2019-12-04 18:39:26,985 : INFO : PROGRESS: at sentence #26120000, processed 341198687 words, keeping 393695 word types\n",
      "2019-12-04 18:39:27,238 : INFO : PROGRESS: at sentence #26130000, processed 341397810 words, keeping 401087 word types\n",
      "2019-12-04 18:39:27,504 : INFO : PROGRESS: at sentence #26140000, processed 341610148 words, keeping 409398 word types\n",
      "2019-12-04 18:39:27,770 : INFO : PROGRESS: at sentence #26150000, processed 341821510 words, keeping 418689 word types\n",
      "2019-12-04 18:39:28,030 : INFO : PROGRESS: at sentence #26160000, processed 342025403 words, keeping 426695 word types\n",
      "2019-12-04 18:39:28,295 : INFO : PROGRESS: at sentence #26170000, processed 342230177 words, keeping 435136 word types\n",
      "2019-12-04 18:39:28,548 : INFO : PROGRESS: at sentence #26180000, processed 342427399 words, keeping 443629 word types\n",
      "2019-12-04 18:39:28,814 : INFO : PROGRESS: at sentence #26190000, processed 342639542 words, keeping 454300 word types\n",
      "2019-12-04 18:39:29,084 : INFO : PROGRESS: at sentence #26200000, processed 342850930 words, keeping 462839 word types\n",
      "2019-12-04 18:39:29,351 : INFO : PROGRESS: at sentence #26210000, processed 343057396 words, keeping 470688 word types\n",
      "2019-12-04 18:39:29,608 : INFO : PROGRESS: at sentence #26220000, processed 343257181 words, keeping 478442 word types\n",
      "2019-12-04 18:39:29,871 : INFO : PROGRESS: at sentence #26230000, processed 343464185 words, keeping 486258 word types\n",
      "2019-12-04 18:39:30,133 : INFO : PROGRESS: at sentence #26240000, processed 343668341 words, keeping 494262 word types\n",
      "2019-12-04 18:39:30,633 : INFO : pruned out 290397 tokens with count <=20 (before 500001, after 209604)\n",
      "2019-12-04 18:39:30,675 : INFO : PROGRESS: at sentence #26250000, processed 343875835 words, keeping 211007 word types\n",
      "2019-12-04 18:39:30,942 : INFO : PROGRESS: at sentence #26260000, processed 344082952 words, keeping 221620 word types\n",
      "2019-12-04 18:39:31,202 : INFO : PROGRESS: at sentence #26270000, processed 344283936 words, keeping 231585 word types\n",
      "2019-12-04 18:39:31,464 : INFO : PROGRESS: at sentence #26280000, processed 344481450 words, keeping 241655 word types\n",
      "2019-12-04 18:39:31,729 : INFO : PROGRESS: at sentence #26290000, processed 344686832 words, keeping 252474 word types\n",
      "2019-12-04 18:39:32,010 : INFO : PROGRESS: at sentence #26300000, processed 344907546 words, keeping 263895 word types\n",
      "2019-12-04 18:39:32,273 : INFO : PROGRESS: at sentence #26310000, processed 345111610 words, keeping 274206 word types\n",
      "2019-12-04 18:39:32,537 : INFO : PROGRESS: at sentence #26320000, processed 345313253 words, keeping 284284 word types\n",
      "2019-12-04 18:39:32,802 : INFO : PROGRESS: at sentence #26330000, processed 345518451 words, keeping 293466 word types\n",
      "2019-12-04 18:39:33,077 : INFO : PROGRESS: at sentence #26340000, processed 345731140 words, keeping 305727 word types\n",
      "2019-12-04 18:39:33,353 : INFO : PROGRESS: at sentence #26350000, processed 345943102 words, keeping 314933 word types\n",
      "2019-12-04 18:39:33,629 : INFO : PROGRESS: at sentence #26360000, processed 346160785 words, keeping 325073 word types\n",
      "2019-12-04 18:39:33,911 : INFO : PROGRESS: at sentence #26370000, processed 346384141 words, keeping 335757 word types\n",
      "2019-12-04 18:39:34,173 : INFO : PROGRESS: at sentence #26380000, processed 346586764 words, keeping 344793 word types\n",
      "2019-12-04 18:39:34,442 : INFO : PROGRESS: at sentence #26390000, processed 346793821 words, keeping 353792 word types\n",
      "2019-12-04 18:39:34,712 : INFO : PROGRESS: at sentence #26400000, processed 347000008 words, keeping 362290 word types\n",
      "2019-12-04 18:39:34,968 : INFO : PROGRESS: at sentence #26410000, processed 347199219 words, keeping 371111 word types\n",
      "2019-12-04 18:39:35,254 : INFO : PROGRESS: at sentence #26420000, processed 347408667 words, keeping 379949 word types\n",
      "2019-12-04 18:39:35,509 : INFO : PROGRESS: at sentence #26430000, processed 347608135 words, keeping 389063 word types\n",
      "2019-12-04 18:39:35,769 : INFO : PROGRESS: at sentence #26440000, processed 347813584 words, keeping 396956 word types\n",
      "2019-12-04 18:39:36,027 : INFO : PROGRESS: at sentence #26450000, processed 348017153 words, keeping 405840 word types\n",
      "2019-12-04 18:39:36,279 : INFO : PROGRESS: at sentence #26460000, processed 348210084 words, keeping 413275 word types\n",
      "2019-12-04 18:39:36,552 : INFO : PROGRESS: at sentence #26470000, processed 348425432 words, keeping 421629 word types\n",
      "2019-12-04 18:39:36,824 : INFO : PROGRESS: at sentence #26480000, processed 348633860 words, keeping 430033 word types\n",
      "2019-12-04 18:39:37,096 : INFO : PROGRESS: at sentence #26490000, processed 348842233 words, keeping 438346 word types\n",
      "2019-12-04 18:39:37,361 : INFO : PROGRESS: at sentence #26500000, processed 349052432 words, keeping 446212 word types\n",
      "2019-12-04 18:39:37,632 : INFO : PROGRESS: at sentence #26510000, processed 349266204 words, keeping 454823 word types\n",
      "2019-12-04 18:39:37,902 : INFO : PROGRESS: at sentence #26520000, processed 349475303 words, keeping 463596 word types\n",
      "2019-12-04 18:39:38,172 : INFO : PROGRESS: at sentence #26530000, processed 349683195 words, keeping 472263 word types\n",
      "2019-12-04 18:39:38,446 : INFO : PROGRESS: at sentence #26540000, processed 349898575 words, keeping 480622 word types\n",
      "2019-12-04 18:39:38,708 : INFO : PROGRESS: at sentence #26550000, processed 350098681 words, keeping 488833 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:39:38,967 : INFO : PROGRESS: at sentence #26560000, processed 350298199 words, keeping 496455 word types\n",
      "2019-12-04 18:39:39,349 : INFO : pruned out 290608 tokens with count <=21 (before 500001, after 209393)\n",
      "2019-12-04 18:39:39,511 : INFO : PROGRESS: at sentence #26570000, processed 350509605 words, keeping 215899 word types\n",
      "2019-12-04 18:39:39,781 : INFO : PROGRESS: at sentence #26580000, processed 350720132 words, keeping 227624 word types\n",
      "2019-12-04 18:39:40,038 : INFO : PROGRESS: at sentence #26590000, processed 350921403 words, keeping 237323 word types\n",
      "2019-12-04 18:39:40,293 : INFO : PROGRESS: at sentence #26600000, processed 351122028 words, keeping 247855 word types\n",
      "2019-12-04 18:39:40,549 : INFO : PROGRESS: at sentence #26610000, processed 351319617 words, keeping 257297 word types\n",
      "2019-12-04 18:39:40,832 : INFO : PROGRESS: at sentence #26620000, processed 351540137 words, keeping 270239 word types\n",
      "2019-12-04 18:39:41,090 : INFO : PROGRESS: at sentence #26630000, processed 351744878 words, keeping 279634 word types\n",
      "2019-12-04 18:39:41,356 : INFO : PROGRESS: at sentence #26640000, processed 351946029 words, keeping 289087 word types\n",
      "2019-12-04 18:39:41,646 : INFO : PROGRESS: at sentence #26650000, processed 352144282 words, keeping 298911 word types\n",
      "2019-12-04 18:39:41,869 : INFO : PROGRESS: at sentence #26660000, processed 352314594 words, keeping 307276 word types\n",
      "2019-12-04 18:39:42,129 : INFO : PROGRESS: at sentence #26670000, processed 352514189 words, keeping 316089 word types\n",
      "2019-12-04 18:39:42,392 : INFO : PROGRESS: at sentence #26680000, processed 352724159 words, keeping 324741 word types\n",
      "2019-12-04 18:39:42,656 : INFO : PROGRESS: at sentence #26690000, processed 352925528 words, keeping 334106 word types\n",
      "2019-12-04 18:39:42,922 : INFO : PROGRESS: at sentence #26700000, processed 353129533 words, keeping 343158 word types\n",
      "2019-12-04 18:39:43,196 : INFO : PROGRESS: at sentence #26710000, processed 353337741 words, keeping 352566 word types\n",
      "2019-12-04 18:39:43,458 : INFO : PROGRESS: at sentence #26720000, processed 353537974 words, keeping 360795 word types\n",
      "2019-12-04 18:39:43,729 : INFO : PROGRESS: at sentence #26730000, processed 353746030 words, keeping 369141 word types\n",
      "2019-12-04 18:39:43,991 : INFO : PROGRESS: at sentence #26740000, processed 353950457 words, keeping 377433 word types\n",
      "2019-12-04 18:39:44,262 : INFO : PROGRESS: at sentence #26750000, processed 354163915 words, keeping 386512 word types\n",
      "2019-12-04 18:39:44,536 : INFO : PROGRESS: at sentence #26760000, processed 354378073 words, keeping 395568 word types\n",
      "2019-12-04 18:39:44,797 : INFO : PROGRESS: at sentence #26770000, processed 354582699 words, keeping 404497 word types\n",
      "2019-12-04 18:39:45,090 : INFO : PROGRESS: at sentence #26780000, processed 354815629 words, keeping 414548 word types\n",
      "2019-12-04 18:39:45,349 : INFO : PROGRESS: at sentence #26790000, processed 355015921 words, keeping 423572 word types\n",
      "2019-12-04 18:39:45,622 : INFO : PROGRESS: at sentence #26800000, processed 355231365 words, keeping 432496 word types\n",
      "2019-12-04 18:39:45,902 : INFO : PROGRESS: at sentence #26810000, processed 355455881 words, keeping 441412 word types\n",
      "2019-12-04 18:39:46,166 : INFO : PROGRESS: at sentence #26820000, processed 355660480 words, keeping 449829 word types\n",
      "2019-12-04 18:39:46,432 : INFO : PROGRESS: at sentence #26830000, processed 355864110 words, keeping 457386 word types\n",
      "2019-12-04 18:39:46,707 : INFO : PROGRESS: at sentence #26840000, processed 356084080 words, keeping 464962 word types\n",
      "2019-12-04 18:39:46,962 : INFO : PROGRESS: at sentence #26850000, processed 356287685 words, keeping 472651 word types\n",
      "2019-12-04 18:39:47,248 : INFO : PROGRESS: at sentence #26860000, processed 356506473 words, keeping 480855 word types\n",
      "2019-12-04 18:39:47,505 : INFO : PROGRESS: at sentence #26870000, processed 356705642 words, keeping 488707 word types\n",
      "2019-12-04 18:39:47,777 : INFO : PROGRESS: at sentence #26880000, processed 356923728 words, keeping 498125 word types\n",
      "2019-12-04 18:39:48,121 : INFO : pruned out 290858 tokens with count <=22 (before 500001, after 209143)\n",
      "2019-12-04 18:39:48,312 : INFO : PROGRESS: at sentence #26890000, processed 357119333 words, keeping 216591 word types\n",
      "2019-12-04 18:39:48,580 : INFO : PROGRESS: at sentence #26900000, processed 357324561 words, keeping 227086 word types\n",
      "2019-12-04 18:39:48,877 : INFO : PROGRESS: at sentence #26910000, processed 357515483 words, keeping 236565 word types\n",
      "2019-12-04 18:39:49,140 : INFO : PROGRESS: at sentence #26920000, processed 357719652 words, keeping 246952 word types\n",
      "2019-12-04 18:39:49,411 : INFO : PROGRESS: at sentence #26930000, processed 357930825 words, keeping 257202 word types\n",
      "2019-12-04 18:39:49,667 : INFO : PROGRESS: at sentence #26940000, processed 358128772 words, keeping 266991 word types\n",
      "2019-12-04 18:39:49,931 : INFO : PROGRESS: at sentence #26950000, processed 358334783 words, keeping 277788 word types\n",
      "2019-12-04 18:39:50,187 : INFO : PROGRESS: at sentence #26960000, processed 358534038 words, keeping 287256 word types\n",
      "2019-12-04 18:39:50,447 : INFO : PROGRESS: at sentence #26970000, processed 358736538 words, keeping 297694 word types\n",
      "2019-12-04 18:39:50,702 : INFO : PROGRESS: at sentence #26980000, processed 358937111 words, keeping 307260 word types\n",
      "2019-12-04 18:39:50,966 : INFO : PROGRESS: at sentence #26990000, processed 359144406 words, keeping 317233 word types\n",
      "2019-12-04 18:39:51,230 : INFO : PROGRESS: at sentence #27000000, processed 359352259 words, keeping 326231 word types\n",
      "2019-12-04 18:39:51,479 : INFO : PROGRESS: at sentence #27010000, processed 359550816 words, keeping 335765 word types\n",
      "2019-12-04 18:39:51,742 : INFO : PROGRESS: at sentence #27020000, processed 359761050 words, keeping 344550 word types\n",
      "2019-12-04 18:39:52,000 : INFO : PROGRESS: at sentence #27030000, processed 359961023 words, keeping 353042 word types\n",
      "2019-12-04 18:39:52,262 : INFO : PROGRESS: at sentence #27040000, processed 360165787 words, keeping 362103 word types\n",
      "2019-12-04 18:39:52,546 : INFO : PROGRESS: at sentence #27050000, processed 360392570 words, keeping 372908 word types\n",
      "2019-12-04 18:39:52,825 : INFO : PROGRESS: at sentence #27060000, processed 360616026 words, keeping 385631 word types\n",
      "2019-12-04 18:39:53,083 : INFO : PROGRESS: at sentence #27070000, processed 360818513 words, keeping 393708 word types\n",
      "2019-12-04 18:39:53,341 : INFO : PROGRESS: at sentence #27080000, processed 361020331 words, keeping 401577 word types\n",
      "2019-12-04 18:39:53,611 : INFO : PROGRESS: at sentence #27090000, processed 361233895 words, keeping 411414 word types\n",
      "2019-12-04 18:39:53,873 : INFO : PROGRESS: at sentence #27100000, processed 361439660 words, keeping 419292 word types\n",
      "2019-12-04 18:39:54,173 : INFO : PROGRESS: at sentence #27110000, processed 361683582 words, keeping 435831 word types\n",
      "2019-12-04 18:39:54,445 : INFO : PROGRESS: at sentence #27120000, processed 361901477 words, keeping 446926 word types\n",
      "2019-12-04 18:39:54,701 : INFO : PROGRESS: at sentence #27130000, processed 362102830 words, keeping 454570 word types\n",
      "2019-12-04 18:39:54,958 : INFO : PROGRESS: at sentence #27140000, processed 362304965 words, keeping 461909 word types\n",
      "2019-12-04 18:39:55,262 : INFO : PROGRESS: at sentence #27150000, processed 362505500 words, keeping 469924 word types\n",
      "2019-12-04 18:39:55,537 : INFO : PROGRESS: at sentence #27160000, processed 362725385 words, keeping 477765 word types\n",
      "2019-12-04 18:39:55,805 : INFO : PROGRESS: at sentence #27170000, processed 362933496 words, keeping 485340 word types\n",
      "2019-12-04 18:39:56,066 : INFO : PROGRESS: at sentence #27180000, processed 363134506 words, keeping 493455 word types\n",
      "2019-12-04 18:39:56,563 : INFO : pruned out 291153 tokens with count <=23 (before 500001, after 208848)\n",
      "2019-12-04 18:39:56,618 : INFO : PROGRESS: at sentence #27190000, processed 363341230 words, keeping 211087 word types\n",
      "2019-12-04 18:39:56,883 : INFO : PROGRESS: at sentence #27200000, processed 363548114 words, keeping 221343 word types\n",
      "2019-12-04 18:39:57,144 : INFO : PROGRESS: at sentence #27210000, processed 363754833 words, keeping 232268 word types\n",
      "2019-12-04 18:39:57,398 : INFO : PROGRESS: at sentence #27220000, processed 363954869 words, keeping 243369 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:39:57,657 : INFO : PROGRESS: at sentence #27230000, processed 364158419 words, keeping 251833 word types\n",
      "2019-12-04 18:39:57,973 : INFO : PROGRESS: at sentence #27240000, processed 364413471 words, keeping 261552 word types\n",
      "2019-12-04 18:39:58,230 : INFO : PROGRESS: at sentence #27250000, processed 364617563 words, keeping 271420 word types\n",
      "2019-12-04 18:39:58,494 : INFO : PROGRESS: at sentence #27260000, processed 364828410 words, keeping 280715 word types\n",
      "2019-12-04 18:39:58,753 : INFO : PROGRESS: at sentence #27270000, processed 365029924 words, keeping 289663 word types\n",
      "2019-12-04 18:39:59,018 : INFO : PROGRESS: at sentence #27280000, processed 365231938 words, keeping 298675 word types\n",
      "2019-12-04 18:39:59,286 : INFO : PROGRESS: at sentence #27290000, processed 365444820 words, keeping 309477 word types\n",
      "2019-12-04 18:39:59,538 : INFO : PROGRESS: at sentence #27300000, processed 365642263 words, keeping 319633 word types\n",
      "2019-12-04 18:39:59,792 : INFO : PROGRESS: at sentence #27310000, processed 365842564 words, keeping 327536 word types\n",
      "2019-12-04 18:40:00,056 : INFO : PROGRESS: at sentence #27320000, processed 366058497 words, keeping 337143 word types\n",
      "2019-12-04 18:40:00,309 : INFO : PROGRESS: at sentence #27330000, processed 366265264 words, keeping 346941 word types\n",
      "2019-12-04 18:40:00,566 : INFO : PROGRESS: at sentence #27340000, processed 366470439 words, keeping 356493 word types\n",
      "2019-12-04 18:40:00,843 : INFO : PROGRESS: at sentence #27350000, processed 366688618 words, keeping 366795 word types\n",
      "2019-12-04 18:40:01,110 : INFO : PROGRESS: at sentence #27360000, processed 366901150 words, keeping 375577 word types\n",
      "2019-12-04 18:40:01,369 : INFO : PROGRESS: at sentence #27370000, processed 367106730 words, keeping 384038 word types\n",
      "2019-12-04 18:40:01,629 : INFO : PROGRESS: at sentence #27380000, processed 367310102 words, keeping 391339 word types\n",
      "2019-12-04 18:40:01,888 : INFO : PROGRESS: at sentence #27390000, processed 367510614 words, keeping 399537 word types\n",
      "2019-12-04 18:40:02,155 : INFO : PROGRESS: at sentence #27400000, processed 367716248 words, keeping 407730 word types\n",
      "2019-12-04 18:40:02,404 : INFO : PROGRESS: at sentence #27410000, processed 367907966 words, keeping 415744 word types\n",
      "2019-12-04 18:40:02,672 : INFO : PROGRESS: at sentence #27420000, processed 368115497 words, keeping 425018 word types\n",
      "2019-12-04 18:40:02,948 : INFO : PROGRESS: at sentence #27430000, processed 368332120 words, keeping 434466 word types\n",
      "2019-12-04 18:40:03,207 : INFO : PROGRESS: at sentence #27440000, processed 368531001 words, keeping 442933 word types\n",
      "2019-12-04 18:40:03,461 : INFO : PROGRESS: at sentence #27450000, processed 368730017 words, keeping 449938 word types\n",
      "2019-12-04 18:40:03,723 : INFO : PROGRESS: at sentence #27460000, processed 368932215 words, keeping 457646 word types\n",
      "2019-12-04 18:40:03,988 : INFO : PROGRESS: at sentence #27470000, processed 369132972 words, keeping 464898 word types\n",
      "2019-12-04 18:40:04,266 : INFO : PROGRESS: at sentence #27480000, processed 369336309 words, keeping 473197 word types\n",
      "2019-12-04 18:40:04,528 : INFO : PROGRESS: at sentence #27490000, processed 369545137 words, keeping 480423 word types\n",
      "2019-12-04 18:40:04,800 : INFO : PROGRESS: at sentence #27500000, processed 369758459 words, keeping 488734 word types\n",
      "2019-12-04 18:40:05,065 : INFO : PROGRESS: at sentence #27510000, processed 369964391 words, keeping 496043 word types\n",
      "2019-12-04 18:40:05,455 : INFO : pruned out 291267 tokens with count <=24 (before 500001, after 208734)\n",
      "2019-12-04 18:40:05,596 : INFO : PROGRESS: at sentence #27520000, processed 370165424 words, keeping 214616 word types\n",
      "2019-12-04 18:40:05,859 : INFO : PROGRESS: at sentence #27530000, processed 370370747 words, keeping 224758 word types\n",
      "2019-12-04 18:40:06,133 : INFO : PROGRESS: at sentence #27540000, processed 370576046 words, keeping 234643 word types\n",
      "2019-12-04 18:40:06,405 : INFO : PROGRESS: at sentence #27550000, processed 370778574 words, keeping 245130 word types\n",
      "2019-12-04 18:40:06,689 : INFO : PROGRESS: at sentence #27560000, processed 370985271 words, keeping 256985 word types\n",
      "2019-12-04 18:40:06,947 : INFO : PROGRESS: at sentence #27570000, processed 371177448 words, keeping 266768 word types\n",
      "2019-12-04 18:40:07,221 : INFO : PROGRESS: at sentence #27580000, processed 371382722 words, keeping 276678 word types\n",
      "2019-12-04 18:40:07,496 : INFO : PROGRESS: at sentence #27590000, processed 371594961 words, keeping 286772 word types\n",
      "2019-12-04 18:40:07,766 : INFO : PROGRESS: at sentence #27600000, processed 371805294 words, keeping 296885 word types\n",
      "2019-12-04 18:40:08,040 : INFO : PROGRESS: at sentence #27610000, processed 372019655 words, keeping 306348 word types\n",
      "2019-12-04 18:40:08,304 : INFO : PROGRESS: at sentence #27620000, processed 372219257 words, keeping 316106 word types\n",
      "2019-12-04 18:40:08,589 : INFO : PROGRESS: at sentence #27630000, processed 372441777 words, keeping 326503 word types\n",
      "2019-12-04 18:40:08,873 : INFO : PROGRESS: at sentence #27640000, processed 372659628 words, keeping 337099 word types\n",
      "2019-12-04 18:40:09,147 : INFO : PROGRESS: at sentence #27650000, processed 372866865 words, keeping 346403 word types\n",
      "2019-12-04 18:40:09,430 : INFO : PROGRESS: at sentence #27660000, processed 373075796 words, keeping 355346 word types\n",
      "2019-12-04 18:40:09,698 : INFO : PROGRESS: at sentence #27670000, processed 373278833 words, keeping 364188 word types\n",
      "2019-12-04 18:40:09,962 : INFO : PROGRESS: at sentence #27680000, processed 373479044 words, keeping 373887 word types\n",
      "2019-12-04 18:40:10,242 : INFO : PROGRESS: at sentence #27690000, processed 373690442 words, keeping 384770 word types\n",
      "2019-12-04 18:40:10,517 : INFO : PROGRESS: at sentence #27700000, processed 373900817 words, keeping 394061 word types\n",
      "2019-12-04 18:40:10,777 : INFO : PROGRESS: at sentence #27710000, processed 374096898 words, keeping 401637 word types\n",
      "2019-12-04 18:40:11,049 : INFO : PROGRESS: at sentence #27720000, processed 374303019 words, keeping 410845 word types\n",
      "2019-12-04 18:40:11,323 : INFO : PROGRESS: at sentence #27730000, processed 374508106 words, keeping 419615 word types\n",
      "2019-12-04 18:40:11,587 : INFO : PROGRESS: at sentence #27740000, processed 374708441 words, keeping 428036 word types\n",
      "2019-12-04 18:40:11,845 : INFO : PROGRESS: at sentence #27750000, processed 374903376 words, keeping 435911 word types\n",
      "2019-12-04 18:40:12,105 : INFO : PROGRESS: at sentence #27760000, processed 375104766 words, keeping 444059 word types\n",
      "2019-12-04 18:40:12,366 : INFO : PROGRESS: at sentence #27770000, processed 375303974 words, keeping 450733 word types\n",
      "2019-12-04 18:40:12,637 : INFO : PROGRESS: at sentence #27780000, processed 375508842 words, keeping 458817 word types\n",
      "2019-12-04 18:40:12,916 : INFO : PROGRESS: at sentence #27790000, processed 375716774 words, keeping 466683 word types\n",
      "2019-12-04 18:40:13,198 : INFO : PROGRESS: at sentence #27800000, processed 375931109 words, keeping 475006 word types\n",
      "2019-12-04 18:40:13,468 : INFO : PROGRESS: at sentence #27810000, processed 376136515 words, keeping 482454 word types\n",
      "2019-12-04 18:40:13,723 : INFO : PROGRESS: at sentence #27820000, processed 376330460 words, keeping 489774 word types\n",
      "2019-12-04 18:40:13,990 : INFO : PROGRESS: at sentence #27830000, processed 376535950 words, keeping 498130 word types\n",
      "2019-12-04 18:40:14,342 : INFO : pruned out 291512 tokens with count <=25 (before 500001, after 208489)\n",
      "2019-12-04 18:40:14,576 : INFO : PROGRESS: at sentence #27840000, processed 376746552 words, keeping 217771 word types\n",
      "2019-12-04 18:40:14,865 : INFO : PROGRESS: at sentence #27850000, processed 376957534 words, keeping 228562 word types\n",
      "2019-12-04 18:40:15,135 : INFO : PROGRESS: at sentence #27860000, processed 377163700 words, keeping 238711 word types\n",
      "2019-12-04 18:40:15,416 : INFO : PROGRESS: at sentence #27870000, processed 377374544 words, keeping 249464 word types\n",
      "2019-12-04 18:40:15,686 : INFO : PROGRESS: at sentence #27880000, processed 377576213 words, keeping 259754 word types\n",
      "2019-12-04 18:40:15,944 : INFO : PROGRESS: at sentence #27890000, processed 377768502 words, keeping 269422 word types\n",
      "2019-12-04 18:40:16,199 : INFO : PROGRESS: at sentence #27900000, processed 377963011 words, keeping 278407 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:40:16,469 : INFO : PROGRESS: at sentence #27910000, processed 378169424 words, keeping 287576 word types\n",
      "2019-12-04 18:40:16,724 : INFO : PROGRESS: at sentence #27920000, processed 378363437 words, keeping 296055 word types\n",
      "2019-12-04 18:40:16,994 : INFO : PROGRESS: at sentence #27930000, processed 378564739 words, keeping 305728 word types\n",
      "2019-12-04 18:40:17,272 : INFO : PROGRESS: at sentence #27940000, processed 378775096 words, keeping 316549 word types\n",
      "2019-12-04 18:40:17,540 : INFO : PROGRESS: at sentence #27950000, processed 378977660 words, keeping 325067 word types\n",
      "2019-12-04 18:40:17,813 : INFO : PROGRESS: at sentence #27960000, processed 379183973 words, keeping 335739 word types\n",
      "2019-12-04 18:40:18,086 : INFO : PROGRESS: at sentence #27970000, processed 379389399 words, keeping 345123 word types\n",
      "2019-12-04 18:40:18,358 : INFO : PROGRESS: at sentence #27980000, processed 379593473 words, keeping 355166 word types\n",
      "2019-12-04 18:40:18,637 : INFO : PROGRESS: at sentence #27990000, processed 379800813 words, keeping 365786 word types\n",
      "2019-12-04 18:40:18,887 : INFO : PROGRESS: at sentence #28000000, processed 379989781 words, keeping 373931 word types\n",
      "2019-12-04 18:40:19,152 : INFO : PROGRESS: at sentence #28010000, processed 380191164 words, keeping 380619 word types\n",
      "2019-12-04 18:40:19,416 : INFO : PROGRESS: at sentence #28020000, processed 380391340 words, keeping 388214 word types\n",
      "2019-12-04 18:40:19,678 : INFO : PROGRESS: at sentence #28030000, processed 380590610 words, keeping 394818 word types\n",
      "2019-12-04 18:40:19,949 : INFO : PROGRESS: at sentence #28040000, processed 380795849 words, keeping 402787 word types\n",
      "2019-12-04 18:40:20,231 : INFO : PROGRESS: at sentence #28050000, processed 381015853 words, keeping 412978 word types\n",
      "2019-12-04 18:40:20,485 : INFO : PROGRESS: at sentence #28060000, processed 381207653 words, keeping 421938 word types\n",
      "2019-12-04 18:40:20,755 : INFO : PROGRESS: at sentence #28070000, processed 381411578 words, keeping 431373 word types\n",
      "2019-12-04 18:40:21,041 : INFO : PROGRESS: at sentence #28080000, processed 381623995 words, keeping 440969 word types\n",
      "2019-12-04 18:40:21,290 : INFO : PROGRESS: at sentence #28090000, processed 381812572 words, keeping 447754 word types\n",
      "2019-12-04 18:40:21,560 : INFO : PROGRESS: at sentence #28100000, processed 382020549 words, keeping 456208 word types\n",
      "2019-12-04 18:40:21,842 : INFO : PROGRESS: at sentence #28110000, processed 382240067 words, keeping 464561 word types\n",
      "2019-12-04 18:40:22,110 : INFO : PROGRESS: at sentence #28120000, processed 382446107 words, keeping 472335 word types\n",
      "2019-12-04 18:40:22,372 : INFO : PROGRESS: at sentence #28130000, processed 382647708 words, keeping 480001 word types\n",
      "2019-12-04 18:40:22,648 : INFO : PROGRESS: at sentence #28140000, processed 382860970 words, keeping 488481 word types\n",
      "2019-12-04 18:40:22,914 : INFO : PROGRESS: at sentence #28150000, processed 383064472 words, keeping 495708 word types\n",
      "2019-12-04 18:40:23,365 : INFO : pruned out 291694 tokens with count <=26 (before 500002, after 208308)\n",
      "2019-12-04 18:40:23,484 : INFO : PROGRESS: at sentence #28160000, processed 383268394 words, keeping 213178 word types\n",
      "2019-12-04 18:40:23,751 : INFO : PROGRESS: at sentence #28170000, processed 383463517 words, keeping 224140 word types\n",
      "2019-12-04 18:40:24,080 : INFO : PROGRESS: at sentence #28180000, processed 383673720 words, keeping 236226 word types\n",
      "2019-12-04 18:40:24,352 : INFO : PROGRESS: at sentence #28190000, processed 383873331 words, keeping 246740 word types\n",
      "2019-12-04 18:40:24,607 : INFO : PROGRESS: at sentence #28200000, processed 384067703 words, keeping 255939 word types\n",
      "2019-12-04 18:40:24,862 : INFO : PROGRESS: at sentence #28210000, processed 384263321 words, keeping 265409 word types\n",
      "2019-12-04 18:40:25,127 : INFO : PROGRESS: at sentence #28220000, processed 384465434 words, keeping 275018 word types\n",
      "2019-12-04 18:40:25,384 : INFO : PROGRESS: at sentence #28230000, processed 384655410 words, keeping 283998 word types\n",
      "2019-12-04 18:40:25,672 : INFO : PROGRESS: at sentence #28240000, processed 384864531 words, keeping 293710 word types\n",
      "2019-12-04 18:40:25,949 : INFO : PROGRESS: at sentence #28250000, processed 385080254 words, keeping 304315 word types\n",
      "2019-12-04 18:40:26,207 : INFO : PROGRESS: at sentence #28260000, processed 385275675 words, keeping 313806 word types\n",
      "2019-12-04 18:40:26,468 : INFO : PROGRESS: at sentence #28270000, processed 385481804 words, keeping 323374 word types\n",
      "2019-12-04 18:40:26,716 : INFO : PROGRESS: at sentence #28280000, processed 385683989 words, keeping 333905 word types\n",
      "2019-12-04 18:40:26,971 : INFO : PROGRESS: at sentence #28290000, processed 385886999 words, keeping 345155 word types\n",
      "2019-12-04 18:40:27,226 : INFO : PROGRESS: at sentence #28300000, processed 386091000 words, keeping 354120 word types\n",
      "2019-12-04 18:40:27,485 : INFO : PROGRESS: at sentence #28310000, processed 386295425 words, keeping 362851 word types\n",
      "2019-12-04 18:40:27,742 : INFO : PROGRESS: at sentence #28320000, processed 386498484 words, keeping 372039 word types\n",
      "2019-12-04 18:40:28,004 : INFO : PROGRESS: at sentence #28330000, processed 386706259 words, keeping 381000 word types\n",
      "2019-12-04 18:40:28,274 : INFO : PROGRESS: at sentence #28340000, processed 386920812 words, keeping 390168 word types\n",
      "2019-12-04 18:40:28,533 : INFO : PROGRESS: at sentence #28350000, processed 387126469 words, keeping 398686 word types\n",
      "2019-12-04 18:40:28,792 : INFO : PROGRESS: at sentence #28360000, processed 387328213 words, keeping 407333 word types\n",
      "2019-12-04 18:40:29,057 : INFO : PROGRESS: at sentence #28370000, processed 387534197 words, keeping 415319 word types\n",
      "2019-12-04 18:40:29,316 : INFO : PROGRESS: at sentence #28380000, processed 387734954 words, keeping 422005 word types\n",
      "2019-12-04 18:40:29,571 : INFO : PROGRESS: at sentence #28390000, processed 387939738 words, keeping 430532 word types\n",
      "2019-12-04 18:40:29,823 : INFO : PROGRESS: at sentence #28400000, processed 388143406 words, keeping 438824 word types\n",
      "2019-12-04 18:40:30,090 : INFO : PROGRESS: at sentence #28410000, processed 388355412 words, keeping 447643 word types\n",
      "2019-12-04 18:40:30,350 : INFO : PROGRESS: at sentence #28420000, processed 388565079 words, keeping 455639 word types\n",
      "2019-12-04 18:40:30,643 : INFO : PROGRESS: at sentence #28430000, processed 388804464 words, keeping 467591 word types\n",
      "2019-12-04 18:40:30,963 : INFO : PROGRESS: at sentence #28440000, processed 389070569 words, keeping 480024 word types\n",
      "2019-12-04 18:40:31,250 : INFO : PROGRESS: at sentence #28450000, processed 389301766 words, keeping 489828 word types\n",
      "2019-12-04 18:40:31,787 : INFO : pruned out 291963 tokens with count <=27 (before 500002, after 208039)\n",
      "2019-12-04 18:40:31,807 : INFO : PROGRESS: at sentence #28460000, processed 389538533 words, keeping 208894 word types\n",
      "2019-12-04 18:40:32,063 : INFO : PROGRESS: at sentence #28470000, processed 389735525 words, keeping 219722 word types\n",
      "2019-12-04 18:40:32,317 : INFO : PROGRESS: at sentence #28480000, processed 389933900 words, keeping 230198 word types\n",
      "2019-12-04 18:40:32,588 : INFO : PROGRESS: at sentence #28490000, processed 390144280 words, keeping 241959 word types\n",
      "2019-12-04 18:40:32,849 : INFO : PROGRESS: at sentence #28500000, processed 390344566 words, keeping 252237 word types\n",
      "2019-12-04 18:40:33,108 : INFO : PROGRESS: at sentence #28510000, processed 390541762 words, keeping 261900 word types\n",
      "2019-12-04 18:40:33,365 : INFO : PROGRESS: at sentence #28520000, processed 390740527 words, keeping 272073 word types\n",
      "2019-12-04 18:40:33,673 : INFO : PROGRESS: at sentence #28530000, processed 390987052 words, keeping 285652 word types\n",
      "2019-12-04 18:40:33,931 : INFO : PROGRESS: at sentence #28540000, processed 391179872 words, keeping 294813 word types\n",
      "2019-12-04 18:40:34,219 : INFO : PROGRESS: at sentence #28550000, processed 391402266 words, keeping 304409 word types\n",
      "2019-12-04 18:40:34,494 : INFO : PROGRESS: at sentence #28560000, processed 391611580 words, keeping 315983 word types\n",
      "2019-12-04 18:40:34,813 : INFO : PROGRESS: at sentence #28570000, processed 391870583 words, keeping 331016 word types\n",
      "2019-12-04 18:40:35,124 : INFO : PROGRESS: at sentence #28580000, processed 392124006 words, keeping 342147 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:40:35,386 : INFO : PROGRESS: at sentence #28590000, processed 392333493 words, keeping 351894 word types\n",
      "2019-12-04 18:40:35,636 : INFO : PROGRESS: at sentence #28600000, processed 392532156 words, keeping 360963 word types\n",
      "2019-12-04 18:40:35,888 : INFO : PROGRESS: at sentence #28610000, processed 392732740 words, keeping 369591 word types\n",
      "2019-12-04 18:40:36,151 : INFO : PROGRESS: at sentence #28620000, processed 392937005 words, keeping 379008 word types\n",
      "2019-12-04 18:40:36,410 : INFO : PROGRESS: at sentence #28630000, processed 393142116 words, keeping 387670 word types\n",
      "2019-12-04 18:40:36,660 : INFO : PROGRESS: at sentence #28640000, processed 393342123 words, keeping 397035 word types\n",
      "2019-12-04 18:40:36,901 : INFO : PROGRESS: at sentence #28650000, processed 393536058 words, keeping 405266 word types\n",
      "2019-12-04 18:40:37,160 : INFO : PROGRESS: at sentence #28660000, processed 393748249 words, keeping 414598 word types\n",
      "2019-12-04 18:40:37,404 : INFO : PROGRESS: at sentence #28670000, processed 393946626 words, keeping 422815 word types\n",
      "2019-12-04 18:40:37,656 : INFO : PROGRESS: at sentence #28680000, processed 394151127 words, keeping 431034 word types\n",
      "2019-12-04 18:40:37,900 : INFO : PROGRESS: at sentence #28690000, processed 394343346 words, keeping 438516 word types\n",
      "2019-12-04 18:40:38,149 : INFO : PROGRESS: at sentence #28700000, processed 394542575 words, keeping 446616 word types\n",
      "2019-12-04 18:40:38,405 : INFO : PROGRESS: at sentence #28710000, processed 394751158 words, keeping 454956 word types\n",
      "2019-12-04 18:40:38,693 : INFO : PROGRESS: at sentence #28720000, processed 394983090 words, keeping 467175 word types\n",
      "2019-12-04 18:40:38,955 : INFO : PROGRESS: at sentence #28730000, processed 395191722 words, keeping 475319 word types\n",
      "2019-12-04 18:40:39,219 : INFO : PROGRESS: at sentence #28740000, processed 395394551 words, keeping 483308 word types\n",
      "2019-12-04 18:40:39,473 : INFO : PROGRESS: at sentence #28750000, processed 395591419 words, keeping 490074 word types\n",
      "2019-12-04 18:40:39,722 : INFO : PROGRESS: at sentence #28760000, processed 395786724 words, keeping 497164 word types\n",
      "2019-12-04 18:40:40,086 : INFO : pruned out 292140 tokens with count <=28 (before 500001, after 207861)\n",
      "2019-12-04 18:40:40,250 : INFO : PROGRESS: at sentence #28770000, processed 395985611 words, keeping 214864 word types\n",
      "2019-12-04 18:40:40,537 : INFO : PROGRESS: at sentence #28780000, processed 396206377 words, keeping 226323 word types\n",
      "2019-12-04 18:40:40,803 : INFO : PROGRESS: at sentence #28790000, processed 396408133 words, keeping 237043 word types\n",
      "2019-12-04 18:40:41,065 : INFO : PROGRESS: at sentence #28800000, processed 396606511 words, keeping 247402 word types\n",
      "2019-12-04 18:40:41,322 : INFO : PROGRESS: at sentence #28810000, processed 396805774 words, keeping 257288 word types\n",
      "2019-12-04 18:40:41,613 : INFO : PROGRESS: at sentence #28820000, processed 397033089 words, keeping 268077 word types\n",
      "2019-12-04 18:40:41,879 : INFO : PROGRESS: at sentence #28830000, processed 397234682 words, keeping 278006 word types\n",
      "2019-12-04 18:40:42,151 : INFO : PROGRESS: at sentence #28840000, processed 397445582 words, keeping 288058 word types\n",
      "2019-12-04 18:40:42,403 : INFO : PROGRESS: at sentence #28850000, processed 397639964 words, keeping 296002 word types\n",
      "2019-12-04 18:40:42,649 : INFO : PROGRESS: at sentence #28860000, processed 397829159 words, keeping 304037 word types\n",
      "2019-12-04 18:40:42,906 : INFO : PROGRESS: at sentence #28870000, processed 398031074 words, keeping 312843 word types\n",
      "2019-12-04 18:40:43,165 : INFO : PROGRESS: at sentence #28880000, processed 398226893 words, keeping 321401 word types\n",
      "2019-12-04 18:40:43,433 : INFO : PROGRESS: at sentence #28890000, processed 398428041 words, keeping 330495 word types\n",
      "2019-12-04 18:40:43,702 : INFO : PROGRESS: at sentence #28900000, processed 398633481 words, keeping 339980 word types\n",
      "2019-12-04 18:40:43,958 : INFO : PROGRESS: at sentence #28910000, processed 398824300 words, keeping 349141 word types\n",
      "2019-12-04 18:40:44,225 : INFO : PROGRESS: at sentence #28920000, processed 399032496 words, keeping 358800 word types\n",
      "2019-12-04 18:40:44,492 : INFO : PROGRESS: at sentence #28930000, processed 399236717 words, keeping 367683 word types\n",
      "2019-12-04 18:40:44,770 : INFO : PROGRESS: at sentence #28940000, processed 399451758 words, keeping 376588 word types\n",
      "2019-12-04 18:40:45,029 : INFO : PROGRESS: at sentence #28950000, processed 399654422 words, keeping 385118 word types\n",
      "2019-12-04 18:40:45,301 : INFO : PROGRESS: at sentence #28960000, processed 399869091 words, keeping 393578 word types\n",
      "2019-12-04 18:40:45,573 : INFO : PROGRESS: at sentence #28970000, processed 400077806 words, keeping 401826 word types\n",
      "2019-12-04 18:40:45,835 : INFO : PROGRESS: at sentence #28980000, processed 400280855 words, keeping 409719 word types\n",
      "2019-12-04 18:40:46,170 : INFO : PROGRESS: at sentence #28990000, processed 400548710 words, keeping 420650 word types\n",
      "2019-12-04 18:40:46,445 : INFO : PROGRESS: at sentence #29000000, processed 400758943 words, keeping 428716 word types\n",
      "2019-12-04 18:40:46,705 : INFO : PROGRESS: at sentence #29010000, processed 400959067 words, keeping 436675 word types\n",
      "2019-12-04 18:40:46,970 : INFO : PROGRESS: at sentence #29020000, processed 401169929 words, keeping 444699 word types\n",
      "2019-12-04 18:40:47,231 : INFO : PROGRESS: at sentence #29030000, processed 401368652 words, keeping 452298 word types\n",
      "2019-12-04 18:40:47,504 : INFO : PROGRESS: at sentence #29040000, processed 401581609 words, keeping 462004 word types\n",
      "2019-12-04 18:40:47,768 : INFO : PROGRESS: at sentence #29050000, processed 401790263 words, keeping 469463 word types\n",
      "2019-12-04 18:40:48,027 : INFO : PROGRESS: at sentence #29060000, processed 401992767 words, keeping 476888 word types\n",
      "2019-12-04 18:40:48,287 : INFO : PROGRESS: at sentence #29070000, processed 402201166 words, keeping 484881 word types\n",
      "2019-12-04 18:40:48,547 : INFO : PROGRESS: at sentence #29080000, processed 402406215 words, keeping 492627 word types\n",
      "2019-12-04 18:40:49,048 : INFO : pruned out 292358 tokens with count <=29 (before 500003, after 207645)\n",
      "2019-12-04 18:40:49,066 : INFO : PROGRESS: at sentence #29090000, processed 402599593 words, keeping 208410 word types\n",
      "2019-12-04 18:40:49,317 : INFO : PROGRESS: at sentence #29100000, processed 402800664 words, keeping 219357 word types\n",
      "2019-12-04 18:40:49,569 : INFO : PROGRESS: at sentence #29110000, processed 403006090 words, keeping 229694 word types\n",
      "2019-12-04 18:40:49,853 : INFO : PROGRESS: at sentence #29120000, processed 403210856 words, keeping 239968 word types\n",
      "2019-12-04 18:40:50,123 : INFO : PROGRESS: at sentence #29130000, processed 403418368 words, keeping 250093 word types\n",
      "2019-12-04 18:40:50,381 : INFO : PROGRESS: at sentence #29140000, processed 403617966 words, keeping 259565 word types\n",
      "2019-12-04 18:40:50,657 : INFO : PROGRESS: at sentence #29150000, processed 403830500 words, keeping 270449 word types\n",
      "2019-12-04 18:40:50,914 : INFO : PROGRESS: at sentence #29160000, processed 404032365 words, keeping 279171 word types\n",
      "2019-12-04 18:40:51,177 : INFO : PROGRESS: at sentence #29170000, processed 404237417 words, keeping 289080 word types\n",
      "2019-12-04 18:40:51,443 : INFO : PROGRESS: at sentence #29180000, processed 404441657 words, keeping 299000 word types\n",
      "2019-12-04 18:40:51,713 : INFO : PROGRESS: at sentence #29190000, processed 404647382 words, keeping 308922 word types\n",
      "2019-12-04 18:40:51,980 : INFO : PROGRESS: at sentence #29200000, processed 404856146 words, keeping 318053 word types\n",
      "2019-12-04 18:40:52,234 : INFO : PROGRESS: at sentence #29210000, processed 405056209 words, keeping 327125 word types\n",
      "2019-12-04 18:40:52,499 : INFO : PROGRESS: at sentence #29220000, processed 405264265 words, keeping 336255 word types\n",
      "2019-12-04 18:40:52,754 : INFO : PROGRESS: at sentence #29230000, processed 405462358 words, keeping 346015 word types\n",
      "2019-12-04 18:40:53,010 : INFO : PROGRESS: at sentence #29240000, processed 405664121 words, keeping 355286 word types\n",
      "2019-12-04 18:40:53,267 : INFO : PROGRESS: at sentence #29250000, processed 405864326 words, keeping 363228 word types\n",
      "2019-12-04 18:40:53,529 : INFO : PROGRESS: at sentence #29260000, processed 406069495 words, keeping 371539 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:40:53,788 : INFO : PROGRESS: at sentence #29270000, processed 406268937 words, keeping 379454 word types\n",
      "2019-12-04 18:40:54,066 : INFO : PROGRESS: at sentence #29280000, processed 406487968 words, keeping 388111 word types\n",
      "2019-12-04 18:40:54,314 : INFO : PROGRESS: at sentence #29290000, processed 406675238 words, keeping 396250 word types\n",
      "2019-12-04 18:40:54,571 : INFO : PROGRESS: at sentence #29300000, processed 406877664 words, keeping 404221 word types\n",
      "2019-12-04 18:40:54,823 : INFO : PROGRESS: at sentence #29310000, processed 407076393 words, keeping 412696 word types\n",
      "2019-12-04 18:40:55,070 : INFO : PROGRESS: at sentence #29320000, processed 407270192 words, keeping 420369 word types\n",
      "2019-12-04 18:40:55,334 : INFO : PROGRESS: at sentence #29330000, processed 407478165 words, keeping 428807 word types\n",
      "2019-12-04 18:40:55,605 : INFO : PROGRESS: at sentence #29340000, processed 407687379 words, keeping 437624 word types\n",
      "2019-12-04 18:40:55,860 : INFO : PROGRESS: at sentence #29350000, processed 407884291 words, keeping 444785 word types\n",
      "2019-12-04 18:40:56,119 : INFO : PROGRESS: at sentence #29360000, processed 408086295 words, keeping 452867 word types\n",
      "2019-12-04 18:40:56,370 : INFO : PROGRESS: at sentence #29370000, processed 408281997 words, keeping 460365 word types\n",
      "2019-12-04 18:40:56,629 : INFO : PROGRESS: at sentence #29380000, processed 408483829 words, keeping 468484 word types\n",
      "2019-12-04 18:40:56,896 : INFO : PROGRESS: at sentence #29390000, processed 408696772 words, keeping 476696 word types\n",
      "2019-12-04 18:40:57,147 : INFO : PROGRESS: at sentence #29400000, processed 408896984 words, keeping 484164 word types\n",
      "2019-12-04 18:40:57,403 : INFO : PROGRESS: at sentence #29410000, processed 409100976 words, keeping 491172 word types\n",
      "2019-12-04 18:40:57,659 : INFO : PROGRESS: at sentence #29420000, processed 409310223 words, keeping 498539 word types\n",
      "2019-12-04 18:40:57,970 : INFO : pruned out 292516 tokens with count <=30 (before 500001, after 207485)\n",
      "2019-12-04 18:40:58,183 : INFO : PROGRESS: at sentence #29430000, processed 409509972 words, keeping 215969 word types\n",
      "2019-12-04 18:40:58,420 : INFO : PROGRESS: at sentence #29440000, processed 409699905 words, keeping 225667 word types\n",
      "2019-12-04 18:40:58,714 : INFO : PROGRESS: at sentence #29450000, processed 409897832 words, keeping 235128 word types\n",
      "2019-12-04 18:40:58,974 : INFO : PROGRESS: at sentence #29460000, processed 410099683 words, keeping 245679 word types\n",
      "2019-12-04 18:40:59,230 : INFO : PROGRESS: at sentence #29470000, processed 410300539 words, keeping 255298 word types\n",
      "2019-12-04 18:40:59,492 : INFO : PROGRESS: at sentence #29480000, processed 410508933 words, keeping 265872 word types\n",
      "2019-12-04 18:40:59,749 : INFO : PROGRESS: at sentence #29490000, processed 410715411 words, keeping 275592 word types\n",
      "2019-12-04 18:41:00,011 : INFO : PROGRESS: at sentence #29500000, processed 410924589 words, keeping 285180 word types\n",
      "2019-12-04 18:41:00,284 : INFO : PROGRESS: at sentence #29510000, processed 411139579 words, keeping 295898 word types\n",
      "2019-12-04 18:41:00,539 : INFO : PROGRESS: at sentence #29520000, processed 411345151 words, keeping 305866 word types\n",
      "2019-12-04 18:41:00,814 : INFO : PROGRESS: at sentence #29530000, processed 411576155 words, keeping 316025 word types\n",
      "2019-12-04 18:41:01,060 : INFO : PROGRESS: at sentence #29540000, processed 411775992 words, keeping 324888 word types\n",
      "2019-12-04 18:41:01,309 : INFO : PROGRESS: at sentence #29550000, processed 411975408 words, keeping 333974 word types\n",
      "2019-12-04 18:41:01,558 : INFO : PROGRESS: at sentence #29560000, processed 412178240 words, keeping 342728 word types\n",
      "2019-12-04 18:41:01,814 : INFO : PROGRESS: at sentence #29570000, processed 412381627 words, keeping 351453 word types\n",
      "2019-12-04 18:41:02,079 : INFO : PROGRESS: at sentence #29580000, processed 412585485 words, keeping 360476 word types\n",
      "2019-12-04 18:41:02,342 : INFO : PROGRESS: at sentence #29590000, processed 412791437 words, keeping 369808 word types\n",
      "2019-12-04 18:41:02,592 : INFO : PROGRESS: at sentence #29600000, processed 412992319 words, keeping 378932 word types\n",
      "2019-12-04 18:41:02,832 : INFO : PROGRESS: at sentence #29610000, processed 413186694 words, keeping 386713 word types\n",
      "2019-12-04 18:41:03,089 : INFO : PROGRESS: at sentence #29620000, processed 413405653 words, keeping 396951 word types\n",
      "2019-12-04 18:41:03,342 : INFO : PROGRESS: at sentence #29630000, processed 413613059 words, keeping 405694 word types\n",
      "2019-12-04 18:41:03,597 : INFO : PROGRESS: at sentence #29640000, processed 413816511 words, keeping 412213 word types\n",
      "2019-12-04 18:41:03,859 : INFO : PROGRESS: at sentence #29650000, processed 414015541 words, keeping 420055 word types\n",
      "2019-12-04 18:41:04,127 : INFO : PROGRESS: at sentence #29660000, processed 414223630 words, keeping 427849 word types\n",
      "2019-12-04 18:41:04,386 : INFO : PROGRESS: at sentence #29670000, processed 414427265 words, keeping 435974 word types\n",
      "2019-12-04 18:41:04,635 : INFO : PROGRESS: at sentence #29680000, processed 414625277 words, keeping 443260 word types\n",
      "2019-12-04 18:41:04,896 : INFO : PROGRESS: at sentence #29690000, processed 414841534 words, keeping 451547 word types\n",
      "2019-12-04 18:41:05,146 : INFO : PROGRESS: at sentence #29700000, processed 415037089 words, keeping 459332 word types\n",
      "2019-12-04 18:41:05,406 : INFO : PROGRESS: at sentence #29710000, processed 415241128 words, keeping 466141 word types\n",
      "2019-12-04 18:41:05,664 : INFO : PROGRESS: at sentence #29720000, processed 415442346 words, keeping 473197 word types\n",
      "2019-12-04 18:41:05,922 : INFO : PROGRESS: at sentence #29730000, processed 415647186 words, keeping 480929 word types\n",
      "2019-12-04 18:41:06,182 : INFO : PROGRESS: at sentence #29740000, processed 415857422 words, keeping 488874 word types\n",
      "2019-12-04 18:41:06,444 : INFO : PROGRESS: at sentence #29750000, processed 416065133 words, keeping 496798 word types\n",
      "2019-12-04 18:41:06,812 : INFO : pruned out 292751 tokens with count <=31 (before 500002, after 207251)\n",
      "2019-12-04 18:41:06,972 : INFO : PROGRESS: at sentence #29760000, processed 416269895 words, keeping 213685 word types\n",
      "2019-12-04 18:41:07,221 : INFO : PROGRESS: at sentence #29770000, processed 416465203 words, keeping 223246 word types\n",
      "2019-12-04 18:41:07,478 : INFO : PROGRESS: at sentence #29780000, processed 416667512 words, keeping 232280 word types\n",
      "2019-12-04 18:41:07,742 : INFO : PROGRESS: at sentence #29790000, processed 416874812 words, keeping 243100 word types\n",
      "2019-12-04 18:41:08,003 : INFO : PROGRESS: at sentence #29800000, processed 417080410 words, keeping 253193 word types\n",
      "2019-12-04 18:41:08,256 : INFO : PROGRESS: at sentence #29810000, processed 417279451 words, keeping 262433 word types\n",
      "2019-12-04 18:41:08,527 : INFO : PROGRESS: at sentence #29820000, processed 417490211 words, keeping 272080 word types\n",
      "2019-12-04 18:41:08,845 : INFO : PROGRESS: at sentence #29830000, processed 417694679 words, keeping 281851 word types\n",
      "2019-12-04 18:41:09,121 : INFO : PROGRESS: at sentence #29840000, processed 417915388 words, keeping 291082 word types\n",
      "2019-12-04 18:41:09,379 : INFO : PROGRESS: at sentence #29850000, processed 418114244 words, keeping 299822 word types\n",
      "2019-12-04 18:41:09,643 : INFO : PROGRESS: at sentence #29860000, processed 418325704 words, keeping 309836 word types\n",
      "2019-12-04 18:41:09,905 : INFO : PROGRESS: at sentence #29870000, processed 418534972 words, keeping 319181 word types\n",
      "2019-12-04 18:41:10,170 : INFO : PROGRESS: at sentence #29880000, processed 418746430 words, keeping 327718 word types\n",
      "2019-12-04 18:41:10,441 : INFO : PROGRESS: at sentence #29890000, processed 418960379 words, keeping 336375 word types\n",
      "2019-12-04 18:41:10,720 : INFO : PROGRESS: at sentence #29900000, processed 419181127 words, keeping 344700 word types\n",
      "2019-12-04 18:41:11,026 : INFO : PROGRESS: at sentence #29910000, processed 419425936 words, keeping 354201 word types\n",
      "2019-12-04 18:41:11,306 : INFO : PROGRESS: at sentence #29920000, processed 419647890 words, keeping 362747 word types\n",
      "2019-12-04 18:41:11,570 : INFO : PROGRESS: at sentence #29930000, processed 419856078 words, keeping 370874 word types\n",
      "2019-12-04 18:41:11,827 : INFO : PROGRESS: at sentence #29940000, processed 420061609 words, keeping 379201 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:41:12,120 : INFO : PROGRESS: at sentence #29950000, processed 420293528 words, keeping 390113 word types\n",
      "2019-12-04 18:41:12,388 : INFO : PROGRESS: at sentence #29960000, processed 420505168 words, keeping 398585 word types\n",
      "2019-12-04 18:41:12,654 : INFO : PROGRESS: at sentence #29970000, processed 420706112 words, keeping 406248 word types\n",
      "2019-12-04 18:41:12,928 : INFO : PROGRESS: at sentence #29980000, processed 420917688 words, keeping 414333 word types\n",
      "2019-12-04 18:41:13,205 : INFO : PROGRESS: at sentence #29990000, processed 421134925 words, keeping 423421 word types\n",
      "2019-12-04 18:41:13,467 : INFO : PROGRESS: at sentence #30000000, processed 421339795 words, keeping 431310 word types\n",
      "2019-12-04 18:41:13,739 : INFO : PROGRESS: at sentence #30010000, processed 421555752 words, keeping 440475 word types\n",
      "2019-12-04 18:41:14,023 : INFO : PROGRESS: at sentence #30020000, processed 421781765 words, keeping 449855 word types\n",
      "2019-12-04 18:41:14,297 : INFO : PROGRESS: at sentence #30030000, processed 421995651 words, keeping 457859 word types\n",
      "2019-12-04 18:41:14,571 : INFO : PROGRESS: at sentence #30040000, processed 422205664 words, keeping 466028 word types\n",
      "2019-12-04 18:41:14,844 : INFO : PROGRESS: at sentence #30050000, processed 422414290 words, keeping 474323 word types\n",
      "2019-12-04 18:41:15,129 : INFO : PROGRESS: at sentence #30060000, processed 422628504 words, keeping 483261 word types\n",
      "2019-12-04 18:41:15,379 : INFO : PROGRESS: at sentence #30070000, processed 422823844 words, keeping 491023 word types\n",
      "2019-12-04 18:41:15,642 : INFO : PROGRESS: at sentence #30080000, processed 423033742 words, keeping 498640 word types\n",
      "2019-12-04 18:41:15,971 : INFO : pruned out 293097 tokens with count <=32 (before 500002, after 206905)\n",
      "2019-12-04 18:41:16,188 : INFO : PROGRESS: at sentence #30090000, processed 423240742 words, keeping 215547 word types\n",
      "2019-12-04 18:41:16,453 : INFO : PROGRESS: at sentence #30100000, processed 423444382 words, keeping 225542 word types\n",
      "2019-12-04 18:41:16,718 : INFO : PROGRESS: at sentence #30110000, processed 423648380 words, keeping 235665 word types\n",
      "2019-12-04 18:41:16,984 : INFO : PROGRESS: at sentence #30120000, processed 423856087 words, keeping 246002 word types\n",
      "2019-12-04 18:41:17,242 : INFO : PROGRESS: at sentence #30130000, processed 424057612 words, keeping 255802 word types\n",
      "2019-12-04 18:41:17,502 : INFO : PROGRESS: at sentence #30140000, processed 424258595 words, keeping 266283 word types\n",
      "2019-12-04 18:41:17,773 : INFO : PROGRESS: at sentence #30150000, processed 424477982 words, keeping 276491 word types\n",
      "2019-12-04 18:41:18,036 : INFO : PROGRESS: at sentence #30160000, processed 424684180 words, keeping 285733 word types\n",
      "2019-12-04 18:41:18,311 : INFO : PROGRESS: at sentence #30170000, processed 424901276 words, keeping 295544 word types\n",
      "2019-12-04 18:41:18,587 : INFO : PROGRESS: at sentence #30180000, processed 425116909 words, keeping 305409 word types\n",
      "2019-12-04 18:41:18,874 : INFO : PROGRESS: at sentence #30190000, processed 425345287 words, keeping 316292 word types\n",
      "2019-12-04 18:41:19,145 : INFO : PROGRESS: at sentence #30200000, processed 425553356 words, keeping 325109 word types\n",
      "2019-12-04 18:41:19,411 : INFO : PROGRESS: at sentence #30210000, processed 425760374 words, keeping 333684 word types\n",
      "2019-12-04 18:41:19,671 : INFO : PROGRESS: at sentence #30220000, processed 425963920 words, keeping 342992 word types\n",
      "2019-12-04 18:41:19,947 : INFO : PROGRESS: at sentence #30230000, processed 426183591 words, keeping 353005 word types\n",
      "2019-12-04 18:41:20,212 : INFO : PROGRESS: at sentence #30240000, processed 426392740 words, keeping 361891 word types\n",
      "2019-12-04 18:41:20,472 : INFO : PROGRESS: at sentence #30250000, processed 426592744 words, keeping 370579 word types\n",
      "2019-12-04 18:41:20,735 : INFO : PROGRESS: at sentence #30260000, processed 426798797 words, keeping 379823 word types\n",
      "2019-12-04 18:41:20,996 : INFO : PROGRESS: at sentence #30270000, processed 427003987 words, keeping 388932 word types\n",
      "2019-12-04 18:41:21,262 : INFO : PROGRESS: at sentence #30280000, processed 427210724 words, keeping 397975 word types\n",
      "2019-12-04 18:41:21,530 : INFO : PROGRESS: at sentence #30290000, processed 427423344 words, keeping 406373 word types\n",
      "2019-12-04 18:41:21,805 : INFO : PROGRESS: at sentence #30300000, processed 427637202 words, keeping 415124 word types\n",
      "2019-12-04 18:41:22,070 : INFO : PROGRESS: at sentence #30310000, processed 427840729 words, keeping 423258 word types\n",
      "2019-12-04 18:41:22,334 : INFO : PROGRESS: at sentence #30320000, processed 428043737 words, keeping 432071 word types\n",
      "2019-12-04 18:41:22,585 : INFO : PROGRESS: at sentence #30330000, processed 428238472 words, keeping 439751 word types\n",
      "2019-12-04 18:41:22,844 : INFO : PROGRESS: at sentence #30340000, processed 428441160 words, keeping 447343 word types\n",
      "2019-12-04 18:41:23,114 : INFO : PROGRESS: at sentence #30350000, processed 428644006 words, keeping 455173 word types\n",
      "2019-12-04 18:41:23,381 : INFO : PROGRESS: at sentence #30360000, processed 428853751 words, keeping 462921 word types\n",
      "2019-12-04 18:41:23,641 : INFO : PROGRESS: at sentence #30370000, processed 429055797 words, keeping 470622 word types\n",
      "2019-12-04 18:41:23,903 : INFO : PROGRESS: at sentence #30380000, processed 429257018 words, keeping 478514 word types\n",
      "2019-12-04 18:41:24,163 : INFO : PROGRESS: at sentence #30390000, processed 429458999 words, keeping 485720 word types\n",
      "2019-12-04 18:41:24,428 : INFO : PROGRESS: at sentence #30400000, processed 429669620 words, keeping 493274 word types\n",
      "2019-12-04 18:41:24,684 : INFO : PROGRESS: at sentence #30410000, processed 429869559 words, keeping 499966 word types\n",
      "2019-12-04 18:41:24,961 : INFO : pruned out 293367 tokens with count <=33 (before 500003, after 206636)\n",
      "2019-12-04 18:41:25,224 : INFO : PROGRESS: at sentence #30420000, processed 430072096 words, keeping 217165 word types\n",
      "2019-12-04 18:41:25,490 : INFO : PROGRESS: at sentence #30430000, processed 430275450 words, keeping 227369 word types\n",
      "2019-12-04 18:41:25,754 : INFO : PROGRESS: at sentence #30440000, processed 430479169 words, keeping 238710 word types\n",
      "2019-12-04 18:41:26,017 : INFO : PROGRESS: at sentence #30450000, processed 430684750 words, keeping 249452 word types\n",
      "2019-12-04 18:41:26,258 : INFO : PROGRESS: at sentence #30460000, processed 430872171 words, keeping 259125 word types\n",
      "2019-12-04 18:41:26,519 : INFO : PROGRESS: at sentence #30470000, processed 431075456 words, keeping 268815 word types\n",
      "2019-12-04 18:41:26,781 : INFO : PROGRESS: at sentence #30480000, processed 431278077 words, keeping 279109 word types\n",
      "2019-12-04 18:41:27,047 : INFO : PROGRESS: at sentence #30490000, processed 431481209 words, keeping 288643 word types\n",
      "2019-12-04 18:41:27,317 : INFO : PROGRESS: at sentence #30500000, processed 431688913 words, keeping 298224 word types\n",
      "2019-12-04 18:41:27,583 : INFO : PROGRESS: at sentence #30510000, processed 431892612 words, keeping 308382 word types\n",
      "2019-12-04 18:41:27,841 : INFO : PROGRESS: at sentence #30520000, processed 432092987 words, keeping 318007 word types\n",
      "2019-12-04 18:41:28,111 : INFO : PROGRESS: at sentence #30530000, processed 432302750 words, keeping 327845 word types\n",
      "2019-12-04 18:41:28,376 : INFO : PROGRESS: at sentence #30540000, processed 432506282 words, keeping 337459 word types\n",
      "2019-12-04 18:41:28,643 : INFO : PROGRESS: at sentence #30550000, processed 432714869 words, keeping 345904 word types\n",
      "2019-12-04 18:41:28,898 : INFO : PROGRESS: at sentence #30560000, processed 432910455 words, keeping 353971 word types\n",
      "2019-12-04 18:41:29,162 : INFO : PROGRESS: at sentence #30570000, processed 433111875 words, keeping 361902 word types\n",
      "2019-12-04 18:41:29,428 : INFO : PROGRESS: at sentence #30580000, processed 433314089 words, keeping 370293 word types\n",
      "2019-12-04 18:41:29,698 : INFO : PROGRESS: at sentence #30590000, processed 433522955 words, keeping 379217 word types\n",
      "2019-12-04 18:41:29,959 : INFO : PROGRESS: at sentence #30600000, processed 433724075 words, keeping 387630 word types\n",
      "2019-12-04 18:41:30,224 : INFO : PROGRESS: at sentence #30610000, processed 433929800 words, keeping 396553 word types\n",
      "2019-12-04 18:41:30,486 : INFO : PROGRESS: at sentence #30620000, processed 434129611 words, keeping 404312 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:41:30,760 : INFO : PROGRESS: at sentence #30630000, processed 434340736 words, keeping 412802 word types\n",
      "2019-12-04 18:41:31,029 : INFO : PROGRESS: at sentence #30640000, processed 434547525 words, keeping 421219 word types\n",
      "2019-12-04 18:41:31,598 : INFO : PROGRESS: at sentence #30650000, processed 435025220 words, keeping 447060 word types\n",
      "2019-12-04 18:41:31,900 : INFO : PROGRESS: at sentence #30660000, processed 435254455 words, keeping 456101 word types\n",
      "2019-12-04 18:41:32,233 : INFO : PROGRESS: at sentence #30670000, processed 435515431 words, keeping 468348 word types\n",
      "2019-12-04 18:41:32,513 : INFO : PROGRESS: at sentence #30680000, processed 435728255 words, keeping 476948 word types\n",
      "2019-12-04 18:41:32,779 : INFO : PROGRESS: at sentence #30690000, processed 435930198 words, keeping 484696 word types\n",
      "2019-12-04 18:41:33,041 : INFO : PROGRESS: at sentence #30700000, processed 436131636 words, keeping 492184 word types\n",
      "2019-12-04 18:41:33,310 : INFO : PROGRESS: at sentence #30710000, processed 436339761 words, keeping 499753 word types\n",
      "2019-12-04 18:41:33,601 : INFO : pruned out 293713 tokens with count <=34 (before 500001, after 206288)\n",
      "2019-12-04 18:41:33,879 : INFO : PROGRESS: at sentence #30720000, processed 436559130 words, keeping 217289 word types\n",
      "2019-12-04 18:41:34,293 : INFO : PROGRESS: at sentence #30730000, processed 436854431 words, keeping 234097 word types\n",
      "2019-12-04 18:41:34,565 : INFO : PROGRESS: at sentence #30740000, processed 437060235 words, keeping 245517 word types\n",
      "2019-12-04 18:41:34,822 : INFO : PROGRESS: at sentence #30750000, processed 437259452 words, keeping 255165 word types\n",
      "2019-12-04 18:41:35,171 : INFO : PROGRESS: at sentence #30760000, processed 437554448 words, keeping 260463 word types\n",
      "2019-12-04 18:41:35,426 : INFO : PROGRESS: at sentence #30770000, processed 437753408 words, keeping 269927 word types\n",
      "2019-12-04 18:41:35,687 : INFO : PROGRESS: at sentence #30780000, processed 437953383 words, keeping 278991 word types\n",
      "2019-12-04 18:41:35,941 : INFO : PROGRESS: at sentence #30790000, processed 438149369 words, keeping 288231 word types\n",
      "2019-12-04 18:41:36,198 : INFO : PROGRESS: at sentence #30800000, processed 438349518 words, keeping 297004 word types\n",
      "2019-12-04 18:41:36,466 : INFO : PROGRESS: at sentence #30810000, processed 438559194 words, keeping 306710 word types\n",
      "2019-12-04 18:41:36,735 : INFO : PROGRESS: at sentence #30820000, processed 438768450 words, keeping 316068 word types\n",
      "2019-12-04 18:41:37,005 : INFO : PROGRESS: at sentence #30830000, processed 438979115 words, keeping 325357 word types\n",
      "2019-12-04 18:41:37,267 : INFO : PROGRESS: at sentence #30840000, processed 439185239 words, keeping 333184 word types\n",
      "2019-12-04 18:41:37,528 : INFO : PROGRESS: at sentence #30850000, processed 439386204 words, keeping 340827 word types\n",
      "2019-12-04 18:41:37,798 : INFO : PROGRESS: at sentence #30860000, processed 439594885 words, keeping 350351 word types\n",
      "2019-12-04 18:41:38,060 : INFO : PROGRESS: at sentence #30870000, processed 439799495 words, keeping 359576 word types\n",
      "2019-12-04 18:41:38,330 : INFO : PROGRESS: at sentence #30880000, processed 440008995 words, keeping 369387 word types\n",
      "2019-12-04 18:41:38,591 : INFO : PROGRESS: at sentence #30890000, processed 440213610 words, keeping 377014 word types\n",
      "2019-12-04 18:41:38,854 : INFO : PROGRESS: at sentence #30900000, processed 440416488 words, keeping 385351 word types\n",
      "2019-12-04 18:41:39,113 : INFO : PROGRESS: at sentence #30910000, processed 440616222 words, keeping 394410 word types\n",
      "2019-12-04 18:41:39,367 : INFO : PROGRESS: at sentence #30920000, processed 440814461 words, keeping 402361 word types\n",
      "2019-12-04 18:41:39,638 : INFO : PROGRESS: at sentence #30930000, processed 441029576 words, keeping 411524 word types\n",
      "2019-12-04 18:41:39,895 : INFO : PROGRESS: at sentence #30940000, processed 441230376 words, keeping 419608 word types\n",
      "2019-12-04 18:41:40,146 : INFO : PROGRESS: at sentence #30950000, processed 441428997 words, keeping 427575 word types\n",
      "2019-12-04 18:41:40,392 : INFO : PROGRESS: at sentence #30960000, processed 441624308 words, keeping 434916 word types\n",
      "2019-12-04 18:41:40,660 : INFO : PROGRESS: at sentence #30970000, processed 441838127 words, keeping 443259 word types\n",
      "2019-12-04 18:41:40,922 : INFO : PROGRESS: at sentence #30980000, processed 442042124 words, keeping 451679 word types\n",
      "2019-12-04 18:41:41,188 : INFO : PROGRESS: at sentence #30990000, processed 442249105 words, keeping 459717 word types\n",
      "2019-12-04 18:41:41,443 : INFO : PROGRESS: at sentence #31000000, processed 442443549 words, keeping 466841 word types\n",
      "2019-12-04 18:41:41,705 : INFO : PROGRESS: at sentence #31010000, processed 442648334 words, keeping 474591 word types\n",
      "2019-12-04 18:41:41,966 : INFO : PROGRESS: at sentence #31020000, processed 442852610 words, keeping 482043 word types\n",
      "2019-12-04 18:41:42,223 : INFO : PROGRESS: at sentence #31030000, processed 443049413 words, keeping 489052 word types\n",
      "2019-12-04 18:41:42,480 : INFO : PROGRESS: at sentence #31040000, processed 443246946 words, keeping 496933 word types\n",
      "2019-12-04 18:41:42,874 : INFO : pruned out 293937 tokens with count <=35 (before 500001, after 206064)\n",
      "2019-12-04 18:41:43,036 : INFO : PROGRESS: at sentence #31050000, processed 443452441 words, keeping 213143 word types\n",
      "2019-12-04 18:41:43,306 : INFO : PROGRESS: at sentence #31060000, processed 443660264 words, keeping 225097 word types\n",
      "2019-12-04 18:41:43,571 : INFO : PROGRESS: at sentence #31070000, processed 443864468 words, keeping 235080 word types\n",
      "2019-12-04 18:41:43,833 : INFO : PROGRESS: at sentence #31080000, processed 444066908 words, keeping 245536 word types\n",
      "2019-12-04 18:41:44,101 : INFO : PROGRESS: at sentence #31090000, processed 444275925 words, keeping 256054 word types\n",
      "2019-12-04 18:41:44,362 : INFO : PROGRESS: at sentence #31100000, processed 444483686 words, keeping 266727 word types\n",
      "2019-12-04 18:41:44,617 : INFO : PROGRESS: at sentence #31110000, processed 444684251 words, keeping 276030 word types\n",
      "2019-12-04 18:41:44,878 : INFO : PROGRESS: at sentence #31120000, processed 444892369 words, keeping 286990 word types\n",
      "2019-12-04 18:41:45,139 : INFO : PROGRESS: at sentence #31130000, processed 445099933 words, keeping 298505 word types\n",
      "2019-12-04 18:41:45,406 : INFO : PROGRESS: at sentence #31140000, processed 445308375 words, keeping 307943 word types\n",
      "2019-12-04 18:41:45,698 : INFO : PROGRESS: at sentence #31150000, processed 445541112 words, keeping 316156 word types\n",
      "2019-12-04 18:41:45,957 : INFO : PROGRESS: at sentence #31160000, processed 445749090 words, keeping 325300 word types\n",
      "2019-12-04 18:41:46,207 : INFO : PROGRESS: at sentence #31170000, processed 445951685 words, keeping 334569 word types\n",
      "2019-12-04 18:41:46,461 : INFO : PROGRESS: at sentence #31180000, processed 446151238 words, keeping 343061 word types\n",
      "2019-12-04 18:41:46,711 : INFO : PROGRESS: at sentence #31190000, processed 446348787 words, keeping 351429 word types\n",
      "2019-12-04 18:41:46,989 : INFO : PROGRESS: at sentence #31200000, processed 446568433 words, keeping 360366 word types\n",
      "2019-12-04 18:41:47,244 : INFO : PROGRESS: at sentence #31210000, processed 446763684 words, keeping 369019 word types\n",
      "2019-12-04 18:41:47,498 : INFO : PROGRESS: at sentence #31220000, processed 446964869 words, keeping 377301 word types\n",
      "2019-12-04 18:41:47,754 : INFO : PROGRESS: at sentence #31230000, processed 447165298 words, keeping 385804 word types\n",
      "2019-12-04 18:41:48,020 : INFO : PROGRESS: at sentence #31240000, processed 447374421 words, keeping 394340 word types\n",
      "2019-12-04 18:41:48,267 : INFO : PROGRESS: at sentence #31250000, processed 447573975 words, keeping 401973 word types\n",
      "2019-12-04 18:41:48,524 : INFO : PROGRESS: at sentence #31260000, processed 447771092 words, keeping 410017 word types\n",
      "2019-12-04 18:41:48,778 : INFO : PROGRESS: at sentence #31270000, processed 447969174 words, keeping 417942 word types\n",
      "2019-12-04 18:41:49,037 : INFO : PROGRESS: at sentence #31280000, processed 448170889 words, keeping 426320 word types\n",
      "2019-12-04 18:41:49,307 : INFO : PROGRESS: at sentence #31290000, processed 448381421 words, keeping 435198 word types\n",
      "2019-12-04 18:41:49,563 : INFO : PROGRESS: at sentence #31300000, processed 448581401 words, keeping 444364 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:41:49,821 : INFO : PROGRESS: at sentence #31310000, processed 448780177 words, keeping 452328 word types\n",
      "2019-12-04 18:41:50,080 : INFO : PROGRESS: at sentence #31320000, processed 448981163 words, keeping 460232 word types\n",
      "2019-12-04 18:41:50,362 : INFO : PROGRESS: at sentence #31330000, processed 449190212 words, keeping 468343 word types\n",
      "2019-12-04 18:41:50,638 : INFO : PROGRESS: at sentence #31340000, processed 449398452 words, keeping 475985 word types\n",
      "2019-12-04 18:41:50,895 : INFO : PROGRESS: at sentence #31350000, processed 449596781 words, keeping 483426 word types\n",
      "2019-12-04 18:41:51,167 : INFO : PROGRESS: at sentence #31360000, processed 449805933 words, keeping 490653 word types\n",
      "2019-12-04 18:41:51,427 : INFO : PROGRESS: at sentence #31370000, processed 450005361 words, keeping 498086 word types\n",
      "2019-12-04 18:41:51,784 : INFO : pruned out 294263 tokens with count <=36 (before 500003, after 205740)\n",
      "2019-12-04 18:41:51,987 : INFO : PROGRESS: at sentence #31380000, processed 450210171 words, keeping 214197 word types\n",
      "2019-12-04 18:41:52,255 : INFO : PROGRESS: at sentence #31390000, processed 450411636 words, keeping 225236 word types\n",
      "2019-12-04 18:41:52,523 : INFO : PROGRESS: at sentence #31400000, processed 450612030 words, keeping 235089 word types\n",
      "2019-12-04 18:41:52,804 : INFO : PROGRESS: at sentence #31410000, processed 450834160 words, keeping 248312 word types\n",
      "2019-12-04 18:41:53,075 : INFO : PROGRESS: at sentence #31420000, processed 451036879 words, keeping 258482 word types\n",
      "2019-12-04 18:41:53,343 : INFO : PROGRESS: at sentence #31430000, processed 451244652 words, keeping 268310 word types\n",
      "2019-12-04 18:41:53,603 : INFO : PROGRESS: at sentence #31440000, processed 451446895 words, keeping 277604 word types\n",
      "2019-12-04 18:41:53,859 : INFO : PROGRESS: at sentence #31450000, processed 451640203 words, keeping 286789 word types\n",
      "2019-12-04 18:41:54,118 : INFO : PROGRESS: at sentence #31460000, processed 451842976 words, keeping 295629 word types\n",
      "2019-12-04 18:41:54,387 : INFO : PROGRESS: at sentence #31470000, processed 452056210 words, keeping 305230 word types\n",
      "2019-12-04 18:41:54,642 : INFO : PROGRESS: at sentence #31480000, processed 452252861 words, keeping 313965 word types\n",
      "2019-12-04 18:41:54,899 : INFO : PROGRESS: at sentence #31490000, processed 452453284 words, keeping 323126 word types\n",
      "2019-12-04 18:41:55,151 : INFO : PROGRESS: at sentence #31500000, processed 452650446 words, keeping 332441 word types\n",
      "2019-12-04 18:41:55,416 : INFO : PROGRESS: at sentence #31510000, processed 452858357 words, keeping 341082 word types\n",
      "2019-12-04 18:41:55,700 : INFO : PROGRESS: at sentence #31520000, processed 453085371 words, keeping 351277 word types\n",
      "2019-12-04 18:41:55,964 : INFO : PROGRESS: at sentence #31530000, processed 453287893 words, keeping 360158 word types\n",
      "2019-12-04 18:41:56,215 : INFO : PROGRESS: at sentence #31540000, processed 453481311 words, keeping 368446 word types\n",
      "2019-12-04 18:41:56,487 : INFO : PROGRESS: at sentence #31550000, processed 453694724 words, keeping 377485 word types\n",
      "2019-12-04 18:41:56,734 : INFO : PROGRESS: at sentence #31560000, processed 453889445 words, keeping 385094 word types\n",
      "2019-12-04 18:41:56,997 : INFO : PROGRESS: at sentence #31570000, processed 454097154 words, keeping 394668 word types\n",
      "2019-12-04 18:41:57,245 : INFO : PROGRESS: at sentence #31580000, processed 454289522 words, keeping 402284 word types\n",
      "2019-12-04 18:41:57,512 : INFO : PROGRESS: at sentence #31590000, processed 454497043 words, keeping 411139 word types\n",
      "2019-12-04 18:41:57,769 : INFO : PROGRESS: at sentence #31600000, processed 454698261 words, keeping 418982 word types\n",
      "2019-12-04 18:41:58,022 : INFO : PROGRESS: at sentence #31610000, processed 454898726 words, keeping 426999 word types\n",
      "2019-12-04 18:41:58,285 : INFO : PROGRESS: at sentence #31620000, processed 455102411 words, keeping 434553 word types\n",
      "2019-12-04 18:41:58,537 : INFO : PROGRESS: at sentence #31630000, processed 455301608 words, keeping 442174 word types\n",
      "2019-12-04 18:41:58,779 : INFO : PROGRESS: at sentence #31640000, processed 455491647 words, keeping 450083 word types\n",
      "2019-12-04 18:41:59,029 : INFO : PROGRESS: at sentence #31650000, processed 455681940 words, keeping 456743 word types\n",
      "2019-12-04 18:41:59,295 : INFO : PROGRESS: at sentence #31660000, processed 455890976 words, keeping 464402 word types\n",
      "2019-12-04 18:41:59,550 : INFO : PROGRESS: at sentence #31670000, processed 456092637 words, keeping 471925 word types\n",
      "2019-12-04 18:41:59,814 : INFO : PROGRESS: at sentence #31680000, processed 456299385 words, keeping 479640 word types\n",
      "2019-12-04 18:42:00,075 : INFO : PROGRESS: at sentence #31690000, processed 456505132 words, keeping 486434 word types\n",
      "2019-12-04 18:42:00,340 : INFO : PROGRESS: at sentence #31700000, processed 456721612 words, keeping 493356 word types\n",
      "2019-12-04 18:42:00,833 : INFO : pruned out 294446 tokens with count <=37 (before 500001, after 205555)\n",
      "2019-12-04 18:42:00,864 : INFO : PROGRESS: at sentence #31710000, processed 456923701 words, keeping 206639 word types\n",
      "2019-12-04 18:42:01,126 : INFO : PROGRESS: at sentence #31720000, processed 457131784 words, keeping 217283 word types\n",
      "2019-12-04 18:42:01,381 : INFO : PROGRESS: at sentence #31730000, processed 457329845 words, keeping 228801 word types\n",
      "2019-12-04 18:42:01,638 : INFO : PROGRESS: at sentence #31740000, processed 457532466 words, keeping 238955 word types\n",
      "2019-12-04 18:42:01,905 : INFO : PROGRESS: at sentence #31750000, processed 457737304 words, keeping 249323 word types\n",
      "2019-12-04 18:42:02,160 : INFO : PROGRESS: at sentence #31760000, processed 457934816 words, keeping 259515 word types\n",
      "2019-12-04 18:42:02,420 : INFO : PROGRESS: at sentence #31770000, processed 458137487 words, keeping 268834 word types\n",
      "2019-12-04 18:42:02,678 : INFO : PROGRESS: at sentence #31780000, processed 458337163 words, keeping 278237 word types\n",
      "2019-12-04 18:42:02,931 : INFO : PROGRESS: at sentence #31790000, processed 458535423 words, keeping 287629 word types\n",
      "2019-12-04 18:42:03,195 : INFO : PROGRESS: at sentence #31800000, processed 458738639 words, keeping 296366 word types\n",
      "2019-12-04 18:42:03,461 : INFO : PROGRESS: at sentence #31810000, processed 458940941 words, keeping 305920 word types\n",
      "2019-12-04 18:42:03,733 : INFO : PROGRESS: at sentence #31820000, processed 459142760 words, keeping 314327 word types\n",
      "2019-12-04 18:42:03,997 : INFO : PROGRESS: at sentence #31830000, processed 459340231 words, keeping 323674 word types\n",
      "2019-12-04 18:42:04,270 : INFO : PROGRESS: at sentence #31840000, processed 459544183 words, keeping 333626 word types\n",
      "2019-12-04 18:42:04,531 : INFO : PROGRESS: at sentence #31850000, processed 459745429 words, keeping 342114 word types\n",
      "2019-12-04 18:42:04,765 : INFO : PROGRESS: at sentence #31860000, processed 459928138 words, keeping 347510 word types\n",
      "2019-12-04 18:42:05,032 : INFO : PROGRESS: at sentence #31870000, processed 460135990 words, keeping 356359 word types\n",
      "2019-12-04 18:42:05,279 : INFO : PROGRESS: at sentence #31880000, processed 460329361 words, keeping 365061 word types\n",
      "2019-12-04 18:42:05,542 : INFO : PROGRESS: at sentence #31890000, processed 460537704 words, keeping 373509 word types\n",
      "2019-12-04 18:42:05,813 : INFO : PROGRESS: at sentence #31900000, processed 460746755 words, keeping 382349 word types\n",
      "2019-12-04 18:42:06,069 : INFO : PROGRESS: at sentence #31910000, processed 460945289 words, keeping 390671 word types\n",
      "2019-12-04 18:42:06,335 : INFO : PROGRESS: at sentence #31920000, processed 461142687 words, keeping 398506 word types\n",
      "2019-12-04 18:42:06,612 : INFO : PROGRESS: at sentence #31930000, processed 461355815 words, keeping 406704 word types\n",
      "2019-12-04 18:42:06,884 : INFO : PROGRESS: at sentence #31940000, processed 461570372 words, keeping 416261 word types\n",
      "2019-12-04 18:42:07,154 : INFO : PROGRESS: at sentence #31950000, processed 461779905 words, keeping 424716 word types\n",
      "2019-12-04 18:42:07,413 : INFO : PROGRESS: at sentence #31960000, processed 461979766 words, keeping 432039 word types\n",
      "2019-12-04 18:42:07,689 : INFO : PROGRESS: at sentence #31970000, processed 462181577 words, keeping 439976 word types\n",
      "2019-12-04 18:42:08,032 : INFO : PROGRESS: at sentence #31980000, processed 462437273 words, keeping 450534 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:42:08,334 : INFO : PROGRESS: at sentence #31990000, processed 462651784 words, keeping 458740 word types\n",
      "2019-12-04 18:42:08,649 : INFO : PROGRESS: at sentence #32000000, processed 462889840 words, keeping 472162 word types\n",
      "2019-12-04 18:42:08,911 : INFO : PROGRESS: at sentence #32010000, processed 463091671 words, keeping 479465 word types\n",
      "2019-12-04 18:42:09,170 : INFO : PROGRESS: at sentence #32020000, processed 463294190 words, keeping 486853 word types\n",
      "2019-12-04 18:42:09,432 : INFO : PROGRESS: at sentence #32030000, processed 463500659 words, keeping 494508 word types\n",
      "2019-12-04 18:42:09,869 : INFO : pruned out 294694 tokens with count <=38 (before 500004, after 205310)\n",
      "2019-12-04 18:42:09,965 : INFO : PROGRESS: at sentence #32040000, processed 463712015 words, keeping 209064 word types\n",
      "2019-12-04 18:42:10,280 : INFO : PROGRESS: at sentence #32050000, processed 463927003 words, keeping 222691 word types\n",
      "2019-12-04 18:42:10,543 : INFO : PROGRESS: at sentence #32060000, processed 464132845 words, keeping 232974 word types\n",
      "2019-12-04 18:42:10,814 : INFO : PROGRESS: at sentence #32070000, processed 464347095 words, keeping 243658 word types\n",
      "2019-12-04 18:42:11,082 : INFO : PROGRESS: at sentence #32080000, processed 464561434 words, keeping 254924 word types\n",
      "2019-12-04 18:42:11,349 : INFO : PROGRESS: at sentence #32090000, processed 464770199 words, keeping 264876 word types\n",
      "2019-12-04 18:42:11,608 : INFO : PROGRESS: at sentence #32100000, processed 464969522 words, keeping 274463 word types\n",
      "2019-12-04 18:42:11,871 : INFO : PROGRESS: at sentence #32110000, processed 465177644 words, keeping 283731 word types\n",
      "2019-12-04 18:42:12,118 : INFO : PROGRESS: at sentence #32120000, processed 465371821 words, keeping 293060 word types\n",
      "2019-12-04 18:42:12,383 : INFO : PROGRESS: at sentence #32130000, processed 465580349 words, keeping 302831 word types\n",
      "2019-12-04 18:42:12,635 : INFO : PROGRESS: at sentence #32140000, processed 465776374 words, keeping 312299 word types\n",
      "2019-12-04 18:42:12,894 : INFO : PROGRESS: at sentence #32150000, processed 465975970 words, keeping 321288 word types\n",
      "2019-12-04 18:42:13,153 : INFO : PROGRESS: at sentence #32160000, processed 466181102 words, keeping 330867 word types\n",
      "2019-12-04 18:42:13,423 : INFO : PROGRESS: at sentence #32170000, processed 466395697 words, keeping 341141 word types\n",
      "2019-12-04 18:42:13,676 : INFO : PROGRESS: at sentence #32180000, processed 466596649 words, keeping 349688 word types\n",
      "2019-12-04 18:42:13,928 : INFO : PROGRESS: at sentence #32190000, processed 466794303 words, keeping 358990 word types\n",
      "2019-12-04 18:42:14,178 : INFO : PROGRESS: at sentence #32200000, processed 466992329 words, keeping 367624 word types\n",
      "2019-12-04 18:42:14,442 : INFO : PROGRESS: at sentence #32210000, processed 467202101 words, keeping 376114 word types\n",
      "2019-12-04 18:42:14,677 : INFO : PROGRESS: at sentence #32220000, processed 467391783 words, keeping 381877 word types\n",
      "2019-12-04 18:42:14,942 : INFO : PROGRESS: at sentence #32230000, processed 467598597 words, keeping 391393 word types\n",
      "2019-12-04 18:42:15,209 : INFO : PROGRESS: at sentence #32240000, processed 467804976 words, keeping 400053 word types\n",
      "2019-12-04 18:42:15,482 : INFO : PROGRESS: at sentence #32250000, processed 468004052 words, keeping 408168 word types\n",
      "2019-12-04 18:42:15,735 : INFO : PROGRESS: at sentence #32260000, processed 468200496 words, keeping 416396 word types\n",
      "2019-12-04 18:42:15,983 : INFO : PROGRESS: at sentence #32270000, processed 468396824 words, keeping 424079 word types\n",
      "2019-12-04 18:42:16,234 : INFO : PROGRESS: at sentence #32280000, processed 468592664 words, keeping 432702 word types\n",
      "2019-12-04 18:42:16,494 : INFO : PROGRESS: at sentence #32290000, processed 468796890 words, keeping 440402 word types\n",
      "2019-12-04 18:42:16,758 : INFO : PROGRESS: at sentence #32300000, processed 469005943 words, keeping 449653 word types\n",
      "2019-12-04 18:42:17,015 : INFO : PROGRESS: at sentence #32310000, processed 469209689 words, keeping 457972 word types\n",
      "2019-12-04 18:42:17,270 : INFO : PROGRESS: at sentence #32320000, processed 469411315 words, keeping 465923 word types\n",
      "2019-12-04 18:42:17,529 : INFO : PROGRESS: at sentence #32330000, processed 469617224 words, keeping 473971 word types\n",
      "2019-12-04 18:42:17,778 : INFO : PROGRESS: at sentence #32340000, processed 469813989 words, keeping 480856 word types\n",
      "2019-12-04 18:42:18,031 : INFO : PROGRESS: at sentence #32350000, processed 470016744 words, keeping 488607 word types\n",
      "2019-12-04 18:42:18,301 : INFO : PROGRESS: at sentence #32360000, processed 470224675 words, keeping 496679 word types\n",
      "2019-12-04 18:42:18,698 : INFO : pruned out 294904 tokens with count <=39 (before 500001, after 205097)\n",
      "2019-12-04 18:42:18,826 : INFO : PROGRESS: at sentence #32370000, processed 470423379 words, keeping 210652 word types\n",
      "2019-12-04 18:42:19,084 : INFO : PROGRESS: at sentence #32380000, processed 470623187 words, keeping 221182 word types\n",
      "2019-12-04 18:42:19,339 : INFO : PROGRESS: at sentence #32390000, processed 470818838 words, keeping 230638 word types\n",
      "2019-12-04 18:42:19,597 : INFO : PROGRESS: at sentence #32400000, processed 471020389 words, keeping 241669 word types\n",
      "2019-12-04 18:42:19,874 : INFO : PROGRESS: at sentence #32410000, processed 471244125 words, keeping 251196 word types\n",
      "2019-12-04 18:42:20,148 : INFO : PROGRESS: at sentence #32420000, processed 471459408 words, keeping 260838 word types\n",
      "2019-12-04 18:42:20,434 : INFO : PROGRESS: at sentence #32430000, processed 471682808 words, keeping 272784 word types\n",
      "2019-12-04 18:42:20,722 : INFO : PROGRESS: at sentence #32440000, processed 471905658 words, keeping 284312 word types\n",
      "2019-12-04 18:42:20,988 : INFO : PROGRESS: at sentence #32450000, processed 472109450 words, keeping 293432 word types\n",
      "2019-12-04 18:42:21,288 : INFO : PROGRESS: at sentence #32460000, processed 472342557 words, keeping 305781 word types\n",
      "2019-12-04 18:42:21,550 : INFO : PROGRESS: at sentence #32470000, processed 472547748 words, keeping 315736 word types\n",
      "2019-12-04 18:42:21,813 : INFO : PROGRESS: at sentence #32480000, processed 472751825 words, keeping 325633 word types\n",
      "2019-12-04 18:42:22,077 : INFO : PROGRESS: at sentence #32490000, processed 472955877 words, keeping 335294 word types\n",
      "2019-12-04 18:42:22,367 : INFO : PROGRESS: at sentence #32500000, processed 473179536 words, keeping 345727 word types\n",
      "2019-12-04 18:42:22,650 : INFO : PROGRESS: at sentence #32510000, processed 473403106 words, keeping 355863 word types\n",
      "2019-12-04 18:42:22,918 : INFO : PROGRESS: at sentence #32520000, processed 473607194 words, keeping 364421 word types\n",
      "2019-12-04 18:42:23,187 : INFO : PROGRESS: at sentence #32530000, processed 473815298 words, keeping 372589 word types\n",
      "2019-12-04 18:42:23,446 : INFO : PROGRESS: at sentence #32540000, processed 474014605 words, keeping 380860 word types\n",
      "2019-12-04 18:42:23,709 : INFO : PROGRESS: at sentence #32550000, processed 474217553 words, keeping 389223 word types\n",
      "2019-12-04 18:42:24,035 : INFO : PROGRESS: at sentence #32560000, processed 474457563 words, keeping 397001 word types\n",
      "2019-12-04 18:42:24,297 : INFO : PROGRESS: at sentence #32570000, processed 474655899 words, keeping 405705 word types\n",
      "2019-12-04 18:42:24,577 : INFO : PROGRESS: at sentence #32580000, processed 474862444 words, keeping 414403 word types\n",
      "2019-12-04 18:42:24,845 : INFO : PROGRESS: at sentence #32590000, processed 475054204 words, keeping 421780 word types\n",
      "2019-12-04 18:42:25,111 : INFO : PROGRESS: at sentence #32600000, processed 475255508 words, keeping 429958 word types\n",
      "2019-12-04 18:42:25,380 : INFO : PROGRESS: at sentence #32610000, processed 475465026 words, keeping 437727 word types\n",
      "2019-12-04 18:42:25,644 : INFO : PROGRESS: at sentence #32620000, processed 475669766 words, keeping 445649 word types\n",
      "2019-12-04 18:42:25,914 : INFO : PROGRESS: at sentence #32630000, processed 475881491 words, keeping 454846 word types\n",
      "2019-12-04 18:42:26,170 : INFO : PROGRESS: at sentence #32640000, processed 476086337 words, keeping 462291 word types\n",
      "2019-12-04 18:42:26,429 : INFO : PROGRESS: at sentence #32650000, processed 476293233 words, keeping 470717 word types\n",
      "2019-12-04 18:42:26,681 : INFO : PROGRESS: at sentence #32660000, processed 476494900 words, keeping 478333 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:42:26,942 : INFO : PROGRESS: at sentence #32670000, processed 476702172 words, keeping 485872 word types\n",
      "2019-12-04 18:42:27,211 : INFO : PROGRESS: at sentence #32680000, processed 476914236 words, keeping 494525 word types\n",
      "2019-12-04 18:42:27,667 : INFO : pruned out 295149 tokens with count <=40 (before 500004, after 204855)\n",
      "2019-12-04 18:42:27,743 : INFO : PROGRESS: at sentence #32690000, processed 477106214 words, keeping 208100 word types\n",
      "2019-12-04 18:42:28,002 : INFO : PROGRESS: at sentence #32700000, processed 477306467 words, keeping 217782 word types\n",
      "2019-12-04 18:42:28,269 : INFO : PROGRESS: at sentence #32710000, processed 477505393 words, keeping 227052 word types\n",
      "2019-12-04 18:42:28,539 : INFO : PROGRESS: at sentence #32720000, processed 477712477 words, keeping 237842 word types\n",
      "2019-12-04 18:42:28,799 : INFO : PROGRESS: at sentence #32730000, processed 477909992 words, keeping 248063 word types\n",
      "2019-12-04 18:42:29,064 : INFO : PROGRESS: at sentence #32740000, processed 478113954 words, keeping 258402 word types\n",
      "2019-12-04 18:42:29,322 : INFO : PROGRESS: at sentence #32750000, processed 478309318 words, keeping 268165 word types\n",
      "2019-12-04 18:42:29,596 : INFO : PROGRESS: at sentence #32760000, processed 478518500 words, keeping 278407 word types\n",
      "2019-12-04 18:42:29,858 : INFO : PROGRESS: at sentence #32770000, processed 478722251 words, keeping 287388 word types\n",
      "2019-12-04 18:42:30,112 : INFO : PROGRESS: at sentence #32780000, processed 478918081 words, keeping 295944 word types\n",
      "2019-12-04 18:42:30,374 : INFO : PROGRESS: at sentence #32790000, processed 479127864 words, keeping 306106 word types\n",
      "2019-12-04 18:42:30,631 : INFO : PROGRESS: at sentence #32800000, processed 479331951 words, keeping 315548 word types\n",
      "2019-12-04 18:42:30,907 : INFO : PROGRESS: at sentence #32810000, processed 479546990 words, keeping 325143 word types\n",
      "2019-12-04 18:42:31,170 : INFO : PROGRESS: at sentence #32820000, processed 479754677 words, keeping 335246 word types\n",
      "2019-12-04 18:42:31,427 : INFO : PROGRESS: at sentence #32830000, processed 479952629 words, keeping 344230 word types\n",
      "2019-12-04 18:42:31,683 : INFO : PROGRESS: at sentence #32840000, processed 480153791 words, keeping 352245 word types\n",
      "2019-12-04 18:42:31,955 : INFO : PROGRESS: at sentence #32850000, processed 480364538 words, keeping 362134 word types\n",
      "2019-12-04 18:42:32,219 : INFO : PROGRESS: at sentence #32860000, processed 480572003 words, keeping 371532 word types\n",
      "2019-12-04 18:42:32,478 : INFO : PROGRESS: at sentence #32870000, processed 480776547 words, keeping 379891 word types\n",
      "2019-12-04 18:42:32,742 : INFO : PROGRESS: at sentence #32880000, processed 480979864 words, keeping 388647 word types\n",
      "2019-12-04 18:42:33,004 : INFO : PROGRESS: at sentence #32890000, processed 481185032 words, keeping 396885 word types\n",
      "2019-12-04 18:42:33,265 : INFO : PROGRESS: at sentence #32900000, processed 481385123 words, keeping 405136 word types\n",
      "2019-12-04 18:42:33,548 : INFO : PROGRESS: at sentence #32910000, processed 481596914 words, keeping 413994 word types\n",
      "2019-12-04 18:42:33,855 : INFO : PROGRESS: at sentence #32920000, processed 481801304 words, keeping 421972 word types\n",
      "2019-12-04 18:42:34,118 : INFO : PROGRESS: at sentence #32930000, processed 482008744 words, keeping 429554 word types\n",
      "2019-12-04 18:42:34,375 : INFO : PROGRESS: at sentence #32940000, processed 482209228 words, keeping 437162 word types\n",
      "2019-12-04 18:42:34,636 : INFO : PROGRESS: at sentence #32950000, processed 482417647 words, keeping 445135 word types\n",
      "2019-12-04 18:42:34,895 : INFO : PROGRESS: at sentence #32960000, processed 482621644 words, keeping 452981 word types\n",
      "2019-12-04 18:42:35,159 : INFO : PROGRESS: at sentence #32970000, processed 482827595 words, keeping 460066 word types\n",
      "2019-12-04 18:42:35,419 : INFO : PROGRESS: at sentence #32980000, processed 483027264 words, keeping 468170 word types\n",
      "2019-12-04 18:42:35,686 : INFO : PROGRESS: at sentence #32990000, processed 483232653 words, keeping 476267 word types\n",
      "2019-12-04 18:42:35,952 : INFO : PROGRESS: at sentence #33000000, processed 483437638 words, keeping 484695 word types\n",
      "2019-12-04 18:42:36,224 : INFO : PROGRESS: at sentence #33010000, processed 483652057 words, keeping 492079 word types\n",
      "2019-12-04 18:42:36,754 : INFO : pruned out 295366 tokens with count <=41 (before 500001, after 204635)\n",
      "2019-12-04 18:42:36,773 : INFO : PROGRESS: at sentence #33020000, processed 483870276 words, keeping 205385 word types\n",
      "2019-12-04 18:42:37,042 : INFO : PROGRESS: at sentence #33030000, processed 484077935 words, keeping 216469 word types\n",
      "2019-12-04 18:42:37,307 : INFO : PROGRESS: at sentence #33040000, processed 484285454 words, keeping 227566 word types\n",
      "2019-12-04 18:42:37,567 : INFO : PROGRESS: at sentence #33050000, processed 484488454 words, keeping 238268 word types\n",
      "2019-12-04 18:42:37,822 : INFO : PROGRESS: at sentence #33060000, processed 484687996 words, keeping 249314 word types\n",
      "2019-12-04 18:42:38,081 : INFO : PROGRESS: at sentence #33070000, processed 484891208 words, keeping 259221 word types\n",
      "2019-12-04 18:42:38,348 : INFO : PROGRESS: at sentence #33080000, processed 485099935 words, keeping 269546 word types\n",
      "2019-12-04 18:42:38,600 : INFO : PROGRESS: at sentence #33090000, processed 485299256 words, keeping 279427 word types\n",
      "2019-12-04 18:42:38,858 : INFO : PROGRESS: at sentence #33100000, processed 485500204 words, keeping 289406 word types\n",
      "2019-12-04 18:42:39,127 : INFO : PROGRESS: at sentence #33110000, processed 485711078 words, keeping 299007 word types\n",
      "2019-12-04 18:42:39,390 : INFO : PROGRESS: at sentence #33120000, processed 485915130 words, keeping 308906 word types\n",
      "2019-12-04 18:42:39,657 : INFO : PROGRESS: at sentence #33130000, processed 486108343 words, keeping 317298 word types\n",
      "2019-12-04 18:42:39,910 : INFO : PROGRESS: at sentence #33140000, processed 486303370 words, keeping 326332 word types\n",
      "2019-12-04 18:42:40,199 : INFO : PROGRESS: at sentence #33150000, processed 486526273 words, keeping 336483 word types\n",
      "2019-12-04 18:42:40,461 : INFO : PROGRESS: at sentence #33160000, processed 486733213 words, keeping 346863 word types\n",
      "2019-12-04 18:42:40,726 : INFO : PROGRESS: at sentence #33170000, processed 486939106 words, keeping 355085 word types\n",
      "2019-12-04 18:42:40,988 : INFO : PROGRESS: at sentence #33180000, processed 487148111 words, keeping 363753 word types\n",
      "2019-12-04 18:42:41,238 : INFO : PROGRESS: at sentence #33190000, processed 487342532 words, keeping 372680 word types\n",
      "2019-12-04 18:42:41,509 : INFO : PROGRESS: at sentence #33200000, processed 487551577 words, keeping 380520 word types\n",
      "2019-12-04 18:42:41,751 : INFO : PROGRESS: at sentence #33210000, processed 487744280 words, keeping 388687 word types\n",
      "2019-12-04 18:42:41,999 : INFO : PROGRESS: at sentence #33220000, processed 487950629 words, keeping 397312 word types\n",
      "2019-12-04 18:42:42,268 : INFO : PROGRESS: at sentence #33230000, processed 488166924 words, keeping 406112 word types\n",
      "2019-12-04 18:42:42,524 : INFO : PROGRESS: at sentence #33240000, processed 488365459 words, keeping 413910 word types\n",
      "2019-12-04 18:42:42,790 : INFO : PROGRESS: at sentence #33250000, processed 488571104 words, keeping 421910 word types\n",
      "2019-12-04 18:42:43,040 : INFO : PROGRESS: at sentence #33260000, processed 488768103 words, keeping 429830 word types\n",
      "2019-12-04 18:42:43,307 : INFO : PROGRESS: at sentence #33270000, processed 488971617 words, keeping 437386 word types\n",
      "2019-12-04 18:42:43,569 : INFO : PROGRESS: at sentence #33280000, processed 489171957 words, keeping 444838 word types\n",
      "2019-12-04 18:42:43,833 : INFO : PROGRESS: at sentence #33290000, processed 489372962 words, keeping 452429 word types\n",
      "2019-12-04 18:42:44,100 : INFO : PROGRESS: at sentence #33300000, processed 489575964 words, keeping 460243 word types\n",
      "2019-12-04 18:42:44,371 : INFO : PROGRESS: at sentence #33310000, processed 489777430 words, keeping 468586 word types\n",
      "2019-12-04 18:42:44,639 : INFO : PROGRESS: at sentence #33320000, processed 489980043 words, keeping 476570 word types\n",
      "2019-12-04 18:42:44,900 : INFO : PROGRESS: at sentence #33330000, processed 490175780 words, keeping 484122 word types\n",
      "2019-12-04 18:42:45,170 : INFO : PROGRESS: at sentence #33340000, processed 490379766 words, keeping 491854 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:42:45,444 : INFO : PROGRESS: at sentence #33350000, processed 490579803 words, keeping 498850 word types\n",
      "2019-12-04 18:42:45,782 : INFO : pruned out 295627 tokens with count <=42 (before 500001, after 204374)\n",
      "2019-12-04 18:42:46,008 : INFO : PROGRESS: at sentence #33360000, processed 490777114 words, keeping 213538 word types\n",
      "2019-12-04 18:42:46,319 : INFO : PROGRESS: at sentence #33370000, processed 490975318 words, keeping 223573 word types\n",
      "2019-12-04 18:42:46,589 : INFO : PROGRESS: at sentence #33380000, processed 491177538 words, keeping 233360 word types\n",
      "2019-12-04 18:42:46,872 : INFO : PROGRESS: at sentence #33390000, processed 491390316 words, keeping 244456 word types\n",
      "2019-12-04 18:42:47,138 : INFO : PROGRESS: at sentence #33400000, processed 491587421 words, keeping 253906 word types\n",
      "2019-12-04 18:42:47,415 : INFO : PROGRESS: at sentence #33410000, processed 491794781 words, keeping 263812 word types\n",
      "2019-12-04 18:42:47,681 : INFO : PROGRESS: at sentence #33420000, processed 491995144 words, keeping 273264 word types\n",
      "2019-12-04 18:42:47,979 : INFO : PROGRESS: at sentence #33430000, processed 492209735 words, keeping 283757 word types\n",
      "2019-12-04 18:42:48,250 : INFO : PROGRESS: at sentence #33440000, processed 492414410 words, keeping 292168 word types\n",
      "2019-12-04 18:42:48,540 : INFO : PROGRESS: at sentence #33450000, processed 492638901 words, keeping 301949 word types\n",
      "2019-12-04 18:42:48,807 : INFO : PROGRESS: at sentence #33460000, processed 492840929 words, keeping 310632 word types\n",
      "2019-12-04 18:42:49,068 : INFO : PROGRESS: at sentence #33470000, processed 493037577 words, keeping 319975 word types\n",
      "2019-12-04 18:42:49,361 : INFO : PROGRESS: at sentence #33480000, processed 493261487 words, keeping 331920 word types\n",
      "2019-12-04 18:42:49,641 : INFO : PROGRESS: at sentence #33490000, processed 493474458 words, keeping 341910 word types\n",
      "2019-12-04 18:42:49,916 : INFO : PROGRESS: at sentence #33500000, processed 493687446 words, keeping 351616 word types\n",
      "2019-12-04 18:42:50,181 : INFO : PROGRESS: at sentence #33510000, processed 493889787 words, keeping 360124 word types\n",
      "2019-12-04 18:42:50,449 : INFO : PROGRESS: at sentence #33520000, processed 494096851 words, keeping 369314 word types\n",
      "2019-12-04 18:42:50,716 : INFO : PROGRESS: at sentence #33530000, processed 494299842 words, keeping 378090 word types\n",
      "2019-12-04 18:42:50,979 : INFO : PROGRESS: at sentence #33540000, processed 494505997 words, keeping 386797 word types\n",
      "2019-12-04 18:42:51,256 : INFO : PROGRESS: at sentence #33550000, processed 494716610 words, keeping 396690 word types\n",
      "2019-12-04 18:42:51,517 : INFO : PROGRESS: at sentence #33560000, processed 494914485 words, keeping 405029 word types\n",
      "2019-12-04 18:42:51,782 : INFO : PROGRESS: at sentence #33570000, processed 495118214 words, keeping 413467 word types\n",
      "2019-12-04 18:42:52,053 : INFO : PROGRESS: at sentence #33580000, processed 495325984 words, keeping 421846 word types\n",
      "2019-12-04 18:42:52,313 : INFO : PROGRESS: at sentence #33590000, processed 495524148 words, keeping 429632 word types\n",
      "2019-12-04 18:42:52,588 : INFO : PROGRESS: at sentence #33600000, processed 495734580 words, keeping 438007 word types\n",
      "2019-12-04 18:42:52,863 : INFO : PROGRESS: at sentence #33610000, processed 495946909 words, keeping 446151 word types\n",
      "2019-12-04 18:42:53,131 : INFO : PROGRESS: at sentence #33620000, processed 496149998 words, keeping 453664 word types\n",
      "2019-12-04 18:42:53,404 : INFO : PROGRESS: at sentence #33630000, processed 496358026 words, keeping 461616 word types\n",
      "2019-12-04 18:42:53,674 : INFO : PROGRESS: at sentence #33640000, processed 496562107 words, keeping 468717 word types\n",
      "2019-12-04 18:42:53,934 : INFO : PROGRESS: at sentence #33650000, processed 496762106 words, keeping 475881 word types\n",
      "2019-12-04 18:42:54,200 : INFO : PROGRESS: at sentence #33660000, processed 496964160 words, keeping 483025 word types\n",
      "2019-12-04 18:42:54,462 : INFO : PROGRESS: at sentence #33670000, processed 497165383 words, keeping 490931 word types\n",
      "2019-12-04 18:42:54,727 : INFO : PROGRESS: at sentence #33680000, processed 497369316 words, keeping 498575 word types\n",
      "2019-12-04 18:42:55,057 : INFO : pruned out 295915 tokens with count <=43 (before 500031, after 204116)\n",
      "2019-12-04 18:42:55,286 : INFO : PROGRESS: at sentence #33690000, processed 497577004 words, keeping 213434 word types\n",
      "2019-12-04 18:42:55,544 : INFO : PROGRESS: at sentence #33700000, processed 497775853 words, keeping 224281 word types\n",
      "2019-12-04 18:42:55,807 : INFO : PROGRESS: at sentence #33710000, processed 497977360 words, keeping 234916 word types\n",
      "2019-12-04 18:42:56,077 : INFO : PROGRESS: at sentence #33720000, processed 498184885 words, keeping 245407 word types\n",
      "2019-12-04 18:42:56,342 : INFO : PROGRESS: at sentence #33730000, processed 498386827 words, keeping 256422 word types\n",
      "2019-12-04 18:42:56,616 : INFO : PROGRESS: at sentence #33740000, processed 498592424 words, keeping 267205 word types\n",
      "2019-12-04 18:42:56,891 : INFO : PROGRESS: at sentence #33750000, processed 498794174 words, keeping 276734 word types\n",
      "2019-12-04 18:42:57,172 : INFO : PROGRESS: at sentence #33760000, processed 499007945 words, keeping 287577 word types\n",
      "2019-12-04 18:42:57,430 : INFO : PROGRESS: at sentence #33770000, processed 499204615 words, keeping 296950 word types\n",
      "2019-12-04 18:42:57,705 : INFO : PROGRESS: at sentence #33780000, processed 499413092 words, keeping 306434 word types\n",
      "2019-12-04 18:42:57,978 : INFO : PROGRESS: at sentence #33790000, processed 499620849 words, keeping 315910 word types\n",
      "2019-12-04 18:42:58,246 : INFO : PROGRESS: at sentence #33800000, processed 499825068 words, keeping 324727 word types\n",
      "2019-12-04 18:42:58,509 : INFO : PROGRESS: at sentence #33810000, processed 500020823 words, keeping 334424 word types\n",
      "2019-12-04 18:42:58,783 : INFO : PROGRESS: at sentence #33820000, processed 500229780 words, keeping 343145 word types\n",
      "2019-12-04 18:42:59,047 : INFO : PROGRESS: at sentence #33830000, processed 500426231 words, keeping 352326 word types\n",
      "2019-12-04 18:42:59,319 : INFO : PROGRESS: at sentence #33840000, processed 500633574 words, keeping 361666 word types\n",
      "2019-12-04 18:42:59,580 : INFO : PROGRESS: at sentence #33850000, processed 500835452 words, keeping 370016 word types\n",
      "2019-12-04 18:42:59,850 : INFO : PROGRESS: at sentence #33860000, processed 501038690 words, keeping 378611 word types\n",
      "2019-12-04 18:43:00,112 : INFO : PROGRESS: at sentence #33870000, processed 501234287 words, keeping 388113 word types\n",
      "2019-12-04 18:43:00,372 : INFO : PROGRESS: at sentence #33880000, processed 501433527 words, keeping 396221 word types\n",
      "2019-12-04 18:43:00,633 : INFO : PROGRESS: at sentence #33890000, processed 501635183 words, keeping 404525 word types\n",
      "2019-12-04 18:43:00,896 : INFO : PROGRESS: at sentence #33900000, processed 501840808 words, keeping 413057 word types\n",
      "2019-12-04 18:43:01,161 : INFO : PROGRESS: at sentence #33910000, processed 502051870 words, keeping 421384 word types\n",
      "2019-12-04 18:43:01,418 : INFO : PROGRESS: at sentence #33920000, processed 502254493 words, keeping 429486 word types\n",
      "2019-12-04 18:43:01,672 : INFO : PROGRESS: at sentence #33930000, processed 502454780 words, keeping 437710 word types\n",
      "2019-12-04 18:43:01,926 : INFO : PROGRESS: at sentence #33940000, processed 502654095 words, keeping 445777 word types\n",
      "2019-12-04 18:43:02,192 : INFO : PROGRESS: at sentence #33950000, processed 502866872 words, keeping 455287 word types\n",
      "2019-12-04 18:43:02,460 : INFO : PROGRESS: at sentence #33960000, processed 503079517 words, keeping 463577 word types\n",
      "2019-12-04 18:43:02,710 : INFO : PROGRESS: at sentence #33970000, processed 503273516 words, keeping 470557 word types\n",
      "2019-12-04 18:43:02,968 : INFO : PROGRESS: at sentence #33980000, processed 503476165 words, keeping 478804 word types\n",
      "2019-12-04 18:43:03,219 : INFO : PROGRESS: at sentence #33990000, processed 503674070 words, keeping 486227 word types\n",
      "2019-12-04 18:43:03,471 : INFO : PROGRESS: at sentence #34000000, processed 503873520 words, keeping 493525 word types\n",
      "2019-12-04 18:43:03,941 : INFO : pruned out 296068 tokens with count <=44 (before 500001, after 203933)\n",
      "2019-12-04 18:43:03,998 : INFO : PROGRESS: at sentence #34010000, processed 504078185 words, keeping 206535 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:43:04,256 : INFO : PROGRESS: at sentence #34020000, processed 504280706 words, keeping 217370 word types\n",
      "2019-12-04 18:43:04,519 : INFO : PROGRESS: at sentence #34030000, processed 504485071 words, keeping 228115 word types\n",
      "2019-12-04 18:43:04,785 : INFO : PROGRESS: at sentence #34040000, processed 504692262 words, keeping 238850 word types\n",
      "2019-12-04 18:43:05,038 : INFO : PROGRESS: at sentence #34050000, processed 504889082 words, keeping 248835 word types\n",
      "2019-12-04 18:43:05,300 : INFO : PROGRESS: at sentence #34060000, processed 505092826 words, keeping 258657 word types\n",
      "2019-12-04 18:43:05,559 : INFO : PROGRESS: at sentence #34070000, processed 505293214 words, keeping 267943 word types\n",
      "2019-12-04 18:43:05,811 : INFO : PROGRESS: at sentence #34080000, processed 505491754 words, keeping 277570 word types\n",
      "2019-12-04 18:43:06,069 : INFO : PROGRESS: at sentence #34090000, processed 505697464 words, keeping 286850 word types\n",
      "2019-12-04 18:43:06,334 : INFO : PROGRESS: at sentence #34100000, processed 505907849 words, keeping 297181 word types\n",
      "2019-12-04 18:43:06,589 : INFO : PROGRESS: at sentence #34110000, processed 506109155 words, keeping 307235 word types\n",
      "2019-12-04 18:43:06,854 : INFO : PROGRESS: at sentence #34120000, processed 506314676 words, keeping 316725 word types\n",
      "2019-12-04 18:43:07,123 : INFO : PROGRESS: at sentence #34130000, processed 506525005 words, keeping 326126 word types\n",
      "2019-12-04 18:43:07,400 : INFO : PROGRESS: at sentence #34140000, processed 506736766 words, keeping 335947 word types\n",
      "2019-12-04 18:43:07,678 : INFO : PROGRESS: at sentence #34150000, processed 506953424 words, keeping 345821 word types\n",
      "2019-12-04 18:43:07,934 : INFO : PROGRESS: at sentence #34160000, processed 507149055 words, keeping 354801 word types\n",
      "2019-12-04 18:43:08,198 : INFO : PROGRESS: at sentence #34170000, processed 507358157 words, keeping 364603 word types\n",
      "2019-12-04 18:43:08,452 : INFO : PROGRESS: at sentence #34180000, processed 507560858 words, keeping 373408 word types\n",
      "2019-12-04 18:43:08,698 : INFO : PROGRESS: at sentence #34190000, processed 507756570 words, keeping 382022 word types\n",
      "2019-12-04 18:43:08,957 : INFO : PROGRESS: at sentence #34200000, processed 507957924 words, keeping 390081 word types\n",
      "2019-12-04 18:43:09,227 : INFO : PROGRESS: at sentence #34210000, processed 508169942 words, keeping 397868 word types\n",
      "2019-12-04 18:43:09,473 : INFO : PROGRESS: at sentence #34220000, processed 508365979 words, keeping 405296 word types\n",
      "2019-12-04 18:43:09,726 : INFO : PROGRESS: at sentence #34230000, processed 508566066 words, keeping 412715 word types\n",
      "2019-12-04 18:43:09,972 : INFO : PROGRESS: at sentence #34240000, processed 508767898 words, keeping 420939 word types\n",
      "2019-12-04 18:43:10,209 : INFO : PROGRESS: at sentence #34250000, processed 508976122 words, keeping 429188 word types\n",
      "2019-12-04 18:43:10,444 : INFO : PROGRESS: at sentence #34260000, processed 509177208 words, keeping 437825 word types\n",
      "2019-12-04 18:43:10,690 : INFO : PROGRESS: at sentence #34270000, processed 509379432 words, keeping 445654 word types\n",
      "2019-12-04 18:43:10,924 : INFO : PROGRESS: at sentence #34280000, processed 509575317 words, keeping 453442 word types\n",
      "2019-12-04 18:43:11,170 : INFO : PROGRESS: at sentence #34290000, processed 509780046 words, keeping 461913 word types\n",
      "2019-12-04 18:43:11,415 : INFO : PROGRESS: at sentence #34300000, processed 509984137 words, keeping 470455 word types\n",
      "2019-12-04 18:43:11,661 : INFO : PROGRESS: at sentence #34310000, processed 510188079 words, keeping 477743 word types\n",
      "2019-12-04 18:43:11,914 : INFO : PROGRESS: at sentence #34320000, processed 510395781 words, keeping 484965 word types\n",
      "2019-12-04 18:43:12,186 : INFO : PROGRESS: at sentence #34330000, processed 510606870 words, keeping 492676 word types\n",
      "2019-12-04 18:43:12,713 : INFO : pruned out 296321 tokens with count <=45 (before 500004, after 203683)\n",
      "2019-12-04 18:43:12,717 : INFO : PROGRESS: at sentence #34340000, processed 510814090 words, keeping 203864 word types\n",
      "2019-12-04 18:43:13,001 : INFO : PROGRESS: at sentence #34350000, processed 511038824 words, keeping 215128 word types\n",
      "2019-12-04 18:43:13,270 : INFO : PROGRESS: at sentence #34360000, processed 511248256 words, keeping 225903 word types\n",
      "2019-12-04 18:43:13,538 : INFO : PROGRESS: at sentence #34370000, processed 511458718 words, keeping 236266 word types\n",
      "2019-12-04 18:43:13,793 : INFO : PROGRESS: at sentence #34380000, processed 511656494 words, keeping 247080 word types\n",
      "2019-12-04 18:43:14,055 : INFO : PROGRESS: at sentence #34390000, processed 511858535 words, keeping 256372 word types\n",
      "2019-12-04 18:43:14,317 : INFO : PROGRESS: at sentence #34400000, processed 512067621 words, keeping 266312 word types\n",
      "2019-12-04 18:43:14,580 : INFO : PROGRESS: at sentence #34410000, processed 512276458 words, keeping 276829 word types\n",
      "2019-12-04 18:43:14,836 : INFO : PROGRESS: at sentence #34420000, processed 512480119 words, keeping 286193 word types\n",
      "2019-12-04 18:43:15,096 : INFO : PROGRESS: at sentence #34430000, processed 512680138 words, keeping 296699 word types\n",
      "2019-12-04 18:43:15,364 : INFO : PROGRESS: at sentence #34440000, processed 512884867 words, keeping 305716 word types\n",
      "2019-12-04 18:43:15,618 : INFO : PROGRESS: at sentence #34450000, processed 513086453 words, keeping 314597 word types\n",
      "2019-12-04 18:43:15,895 : INFO : PROGRESS: at sentence #34460000, processed 513290039 words, keeping 323744 word types\n",
      "2019-12-04 18:43:16,164 : INFO : PROGRESS: at sentence #34470000, processed 513495287 words, keeping 333542 word types\n",
      "2019-12-04 18:43:16,425 : INFO : PROGRESS: at sentence #34480000, processed 513695914 words, keeping 342643 word types\n",
      "2019-12-04 18:43:16,681 : INFO : PROGRESS: at sentence #34490000, processed 513904672 words, keeping 351152 word types\n",
      "2019-12-04 18:43:16,941 : INFO : PROGRESS: at sentence #34500000, processed 514115169 words, keeping 359602 word types\n",
      "2019-12-04 18:43:17,191 : INFO : PROGRESS: at sentence #34510000, processed 514314373 words, keeping 368562 word types\n",
      "2019-12-04 18:43:17,470 : INFO : PROGRESS: at sentence #34520000, processed 514521521 words, keeping 377888 word types\n",
      "2019-12-04 18:43:17,747 : INFO : PROGRESS: at sentence #34530000, processed 514735718 words, keeping 386867 word types\n",
      "2019-12-04 18:43:17,996 : INFO : PROGRESS: at sentence #34540000, processed 514929780 words, keeping 394934 word types\n",
      "2019-12-04 18:43:18,274 : INFO : PROGRESS: at sentence #34550000, processed 515140409 words, keeping 404262 word types\n",
      "2019-12-04 18:43:18,542 : INFO : PROGRESS: at sentence #34560000, processed 515348101 words, keeping 413123 word types\n",
      "2019-12-04 18:43:18,837 : INFO : PROGRESS: at sentence #34570000, processed 515578491 words, keeping 423868 word types\n",
      "2019-12-04 18:43:19,110 : INFO : PROGRESS: at sentence #34580000, processed 515787802 words, keeping 432185 word types\n",
      "2019-12-04 18:43:19,379 : INFO : PROGRESS: at sentence #34590000, processed 515996846 words, keeping 439935 word types\n",
      "2019-12-04 18:43:19,646 : INFO : PROGRESS: at sentence #34600000, processed 516200082 words, keeping 447574 word types\n",
      "2019-12-04 18:43:19,903 : INFO : PROGRESS: at sentence #34610000, processed 516398568 words, keeping 454874 word types\n",
      "2019-12-04 18:43:20,184 : INFO : PROGRESS: at sentence #34620000, processed 516604991 words, keeping 463166 word types\n",
      "2019-12-04 18:43:20,463 : INFO : PROGRESS: at sentence #34630000, processed 516818259 words, keeping 471637 word types\n",
      "2019-12-04 18:43:20,742 : INFO : PROGRESS: at sentence #34640000, processed 517026018 words, keeping 479675 word types\n",
      "2019-12-04 18:43:21,022 : INFO : PROGRESS: at sentence #34650000, processed 517239938 words, keeping 487431 word types\n",
      "2019-12-04 18:43:21,313 : INFO : PROGRESS: at sentence #34660000, processed 517465256 words, keeping 498058 word types\n",
      "2019-12-04 18:43:21,685 : INFO : pruned out 296538 tokens with count <=46 (before 500001, after 203463)\n",
      "2019-12-04 18:43:21,898 : INFO : PROGRESS: at sentence #34670000, processed 517670856 words, keeping 212595 word types\n",
      "2019-12-04 18:43:22,214 : INFO : PROGRESS: at sentence #34680000, processed 517876670 words, keeping 223270 word types\n",
      "2019-12-04 18:43:22,475 : INFO : PROGRESS: at sentence #34690000, processed 518080111 words, keeping 233446 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:43:22,742 : INFO : PROGRESS: at sentence #34700000, processed 518288352 words, keeping 244150 word types\n",
      "2019-12-04 18:43:23,061 : INFO : PROGRESS: at sentence #34710000, processed 518524203 words, keeping 253155 word types\n",
      "2019-12-04 18:43:23,353 : INFO : PROGRESS: at sentence #34720000, processed 518744933 words, keeping 262822 word types\n",
      "2019-12-04 18:43:23,694 : INFO : PROGRESS: at sentence #34730000, processed 518975579 words, keeping 270479 word types\n",
      "2019-12-04 18:43:23,964 : INFO : PROGRESS: at sentence #34740000, processed 519182760 words, keeping 279759 word types\n",
      "2019-12-04 18:43:24,258 : INFO : PROGRESS: at sentence #34750000, processed 519400315 words, keeping 289327 word types\n",
      "2019-12-04 18:43:24,537 : INFO : PROGRESS: at sentence #34760000, processed 519609106 words, keeping 298532 word types\n",
      "2019-12-04 18:43:24,808 : INFO : PROGRESS: at sentence #34770000, processed 519815635 words, keeping 307268 word types\n",
      "2019-12-04 18:43:25,074 : INFO : PROGRESS: at sentence #34780000, processed 520018829 words, keeping 315488 word types\n",
      "2019-12-04 18:43:25,353 : INFO : PROGRESS: at sentence #34790000, processed 520229676 words, keeping 323732 word types\n",
      "2019-12-04 18:43:25,627 : INFO : PROGRESS: at sentence #34800000, processed 520440561 words, keeping 332100 word types\n",
      "2019-12-04 18:43:25,891 : INFO : PROGRESS: at sentence #34810000, processed 520644216 words, keeping 341530 word types\n",
      "2019-12-04 18:43:26,163 : INFO : PROGRESS: at sentence #34820000, processed 520847113 words, keeping 350559 word types\n",
      "2019-12-04 18:43:26,430 : INFO : PROGRESS: at sentence #34830000, processed 521053314 words, keeping 359071 word types\n",
      "2019-12-04 18:43:26,669 : INFO : PROGRESS: at sentence #34840000, processed 521239943 words, keeping 365662 word types\n",
      "2019-12-04 18:43:26,921 : INFO : PROGRESS: at sentence #34850000, processed 521439970 words, keeping 373272 word types\n",
      "2019-12-04 18:43:27,174 : INFO : PROGRESS: at sentence #34860000, processed 521639263 words, keeping 382198 word types\n",
      "2019-12-04 18:43:27,428 : INFO : PROGRESS: at sentence #34870000, processed 521840380 words, keeping 389443 word types\n",
      "2019-12-04 18:43:27,692 : INFO : PROGRESS: at sentence #34880000, processed 522047164 words, keeping 397399 word types\n",
      "2019-12-04 18:43:27,947 : INFO : PROGRESS: at sentence #34890000, processed 522250357 words, keeping 404678 word types\n",
      "2019-12-04 18:43:28,215 : INFO : PROGRESS: at sentence #34900000, processed 522460532 words, keeping 412963 word types\n",
      "2019-12-04 18:43:28,481 : INFO : PROGRESS: at sentence #34910000, processed 522668857 words, keeping 421612 word types\n",
      "2019-12-04 18:43:28,750 : INFO : PROGRESS: at sentence #34920000, processed 522873121 words, keeping 429176 word types\n",
      "2019-12-04 18:43:29,015 : INFO : PROGRESS: at sentence #34930000, processed 523079816 words, keeping 437205 word types\n",
      "2019-12-04 18:43:29,290 : INFO : PROGRESS: at sentence #34940000, processed 523303291 words, keeping 447242 word types\n",
      "2019-12-04 18:43:29,559 : INFO : PROGRESS: at sentence #34950000, processed 523510367 words, keeping 455602 word types\n",
      "2019-12-04 18:43:29,824 : INFO : PROGRESS: at sentence #34960000, processed 523715185 words, keeping 463275 word types\n",
      "2019-12-04 18:43:30,084 : INFO : PROGRESS: at sentence #34970000, processed 523914634 words, keeping 470528 word types\n",
      "2019-12-04 18:43:30,342 : INFO : PROGRESS: at sentence #34980000, processed 524117209 words, keeping 478667 word types\n",
      "2019-12-04 18:43:30,612 : INFO : PROGRESS: at sentence #34990000, processed 524328394 words, keeping 487109 word types\n",
      "2019-12-04 18:43:30,899 : INFO : PROGRESS: at sentence #35000000, processed 524536762 words, keeping 494352 word types\n",
      "2019-12-04 18:43:31,354 : INFO : pruned out 296659 tokens with count <=47 (before 500003, after 203344)\n",
      "2019-12-04 18:43:31,503 : INFO : PROGRESS: at sentence #35010000, processed 524788206 words, keeping 222113 word types\n",
      "2019-12-04 18:43:31,768 : INFO : PROGRESS: at sentence #35020000, processed 524989740 words, keeping 232989 word types\n",
      "2019-12-04 18:43:32,041 : INFO : PROGRESS: at sentence #35030000, processed 525195648 words, keeping 243171 word types\n",
      "2019-12-04 18:43:32,307 : INFO : PROGRESS: at sentence #35040000, processed 525403393 words, keeping 254464 word types\n",
      "2019-12-04 18:43:32,569 : INFO : PROGRESS: at sentence #35050000, processed 525607409 words, keeping 264419 word types\n",
      "2019-12-04 18:43:32,841 : INFO : PROGRESS: at sentence #35060000, processed 525814916 words, keeping 275288 word types\n",
      "2019-12-04 18:43:33,098 : INFO : PROGRESS: at sentence #35070000, processed 526012774 words, keeping 284734 word types\n",
      "2019-12-04 18:43:33,353 : INFO : PROGRESS: at sentence #35080000, processed 526209042 words, keeping 293051 word types\n",
      "2019-12-04 18:43:33,618 : INFO : PROGRESS: at sentence #35090000, processed 526416921 words, keeping 303009 word types\n",
      "2019-12-04 18:43:33,879 : INFO : PROGRESS: at sentence #35100000, processed 526617868 words, keeping 311852 word types\n",
      "2019-12-04 18:43:34,131 : INFO : PROGRESS: at sentence #35110000, processed 526815185 words, keeping 318870 word types\n",
      "2019-12-04 18:43:34,404 : INFO : PROGRESS: at sentence #35120000, processed 527026572 words, keeping 327418 word types\n",
      "2019-12-04 18:43:34,672 : INFO : PROGRESS: at sentence #35130000, processed 527233736 words, keeping 338407 word types\n",
      "2019-12-04 18:43:34,923 : INFO : PROGRESS: at sentence #35140000, processed 527426797 words, keeping 346572 word types\n",
      "2019-12-04 18:43:35,218 : INFO : PROGRESS: at sentence #35150000, processed 527636370 words, keeping 355644 word types\n",
      "2019-12-04 18:43:35,517 : INFO : PROGRESS: at sentence #35160000, processed 527835808 words, keeping 363691 word types\n",
      "2019-12-04 18:43:35,797 : INFO : PROGRESS: at sentence #35170000, processed 528057886 words, keeping 372393 word types\n",
      "2019-12-04 18:43:36,088 : INFO : PROGRESS: at sentence #35180000, processed 528287701 words, keeping 384169 word types\n",
      "2019-12-04 18:43:36,370 : INFO : PROGRESS: at sentence #35190000, processed 528507097 words, keeping 392347 word types\n",
      "2019-12-04 18:43:36,651 : INFO : PROGRESS: at sentence #35200000, processed 528734752 words, keeping 400305 word types\n",
      "2019-12-04 18:43:36,910 : INFO : PROGRESS: at sentence #35210000, processed 528936203 words, keeping 408986 word types\n",
      "2019-12-04 18:43:37,166 : INFO : PROGRESS: at sentence #35220000, processed 529135089 words, keeping 417109 word types\n",
      "2019-12-04 18:43:37,428 : INFO : PROGRESS: at sentence #35230000, processed 529335546 words, keeping 424968 word types\n",
      "2019-12-04 18:43:37,685 : INFO : PROGRESS: at sentence #35240000, processed 529535740 words, keeping 432963 word types\n",
      "2019-12-04 18:43:37,953 : INFO : PROGRESS: at sentence #35250000, processed 529743757 words, keeping 441662 word types\n",
      "2019-12-04 18:43:38,211 : INFO : PROGRESS: at sentence #35260000, processed 529940077 words, keeping 449571 word types\n",
      "2019-12-04 18:43:38,485 : INFO : PROGRESS: at sentence #35270000, processed 530155464 words, keeping 458013 word types\n",
      "2019-12-04 18:43:38,746 : INFO : PROGRESS: at sentence #35280000, processed 530360518 words, keeping 466098 word types\n",
      "2019-12-04 18:43:39,004 : INFO : PROGRESS: at sentence #35290000, processed 530558246 words, keeping 473447 word types\n",
      "2019-12-04 18:43:39,273 : INFO : PROGRESS: at sentence #35300000, processed 530760493 words, keeping 480696 word types\n",
      "2019-12-04 18:43:39,552 : INFO : PROGRESS: at sentence #35310000, processed 530969883 words, keeping 489174 word types\n",
      "2019-12-04 18:43:39,828 : INFO : PROGRESS: at sentence #35320000, processed 531182169 words, keeping 496927 word types\n",
      "2019-12-04 18:43:40,217 : INFO : pruned out 296874 tokens with count <=48 (before 500001, after 203127)\n",
      "2019-12-04 18:43:40,383 : INFO : PROGRESS: at sentence #35330000, processed 531386935 words, keeping 209554 word types\n",
      "2019-12-04 18:43:40,653 : INFO : PROGRESS: at sentence #35340000, processed 531590292 words, keeping 220153 word types\n",
      "2019-12-04 18:43:40,920 : INFO : PROGRESS: at sentence #35350000, processed 531796385 words, keeping 230298 word types\n",
      "2019-12-04 18:43:41,192 : INFO : PROGRESS: at sentence #35360000, processed 532004220 words, keeping 240744 word types\n",
      "2019-12-04 18:43:41,467 : INFO : PROGRESS: at sentence #35370000, processed 532216097 words, keeping 250374 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:43:41,731 : INFO : PROGRESS: at sentence #35380000, processed 532422843 words, keeping 259938 word types\n",
      "2019-12-04 18:43:41,993 : INFO : PROGRESS: at sentence #35390000, processed 532625294 words, keeping 269347 word types\n",
      "2019-12-04 18:43:42,275 : INFO : PROGRESS: at sentence #35400000, processed 532833212 words, keeping 279218 word types\n",
      "2019-12-04 18:43:42,559 : INFO : PROGRESS: at sentence #35410000, processed 533053316 words, keeping 289846 word types\n",
      "2019-12-04 18:43:42,825 : INFO : PROGRESS: at sentence #35420000, processed 533258300 words, keeping 299672 word types\n",
      "2019-12-04 18:43:43,083 : INFO : PROGRESS: at sentence #35430000, processed 533463848 words, keeping 307617 word types\n",
      "2019-12-04 18:43:43,341 : INFO : PROGRESS: at sentence #35440000, processed 533663791 words, keeping 316603 word types\n",
      "2019-12-04 18:43:43,617 : INFO : PROGRESS: at sentence #35450000, processed 533880417 words, keeping 327066 word types\n",
      "2019-12-04 18:43:43,903 : INFO : PROGRESS: at sentence #35460000, processed 534105395 words, keeping 337227 word types\n",
      "2019-12-04 18:43:44,174 : INFO : PROGRESS: at sentence #35470000, processed 534315430 words, keeping 346579 word types\n",
      "2019-12-04 18:43:44,460 : INFO : PROGRESS: at sentence #35480000, processed 534542067 words, keeping 356215 word types\n",
      "2019-12-04 18:43:44,742 : INFO : PROGRESS: at sentence #35490000, processed 534767205 words, keeping 365109 word types\n",
      "2019-12-04 18:43:45,029 : INFO : PROGRESS: at sentence #35500000, processed 534995864 words, keeping 374361 word types\n",
      "2019-12-04 18:43:45,306 : INFO : PROGRESS: at sentence #35510000, processed 535216989 words, keeping 383046 word types\n",
      "2019-12-04 18:43:45,576 : INFO : PROGRESS: at sentence #35520000, processed 535428246 words, keeping 391556 word types\n",
      "2019-12-04 18:43:45,839 : INFO : PROGRESS: at sentence #35530000, processed 535638937 words, keeping 400295 word types\n",
      "2019-12-04 18:43:46,114 : INFO : PROGRESS: at sentence #35540000, processed 535840148 words, keeping 407865 word types\n",
      "2019-12-04 18:43:46,389 : INFO : PROGRESS: at sentence #35550000, processed 536059654 words, keeping 417346 word types\n",
      "2019-12-04 18:43:46,658 : INFO : PROGRESS: at sentence #35560000, processed 536273924 words, keeping 425518 word types\n",
      "2019-12-04 18:43:46,921 : INFO : PROGRESS: at sentence #35570000, processed 536484146 words, keeping 434155 word types\n",
      "2019-12-04 18:43:47,198 : INFO : PROGRESS: at sentence #35580000, processed 536699679 words, keeping 442622 word types\n",
      "2019-12-04 18:43:47,475 : INFO : PROGRESS: at sentence #35590000, processed 536910551 words, keeping 450345 word types\n",
      "2019-12-04 18:43:47,739 : INFO : PROGRESS: at sentence #35600000, processed 537113574 words, keeping 458015 word types\n",
      "2019-12-04 18:43:48,012 : INFO : PROGRESS: at sentence #35610000, processed 537319014 words, keeping 465404 word types\n",
      "2019-12-04 18:43:48,292 : INFO : PROGRESS: at sentence #35620000, processed 537531396 words, keeping 473944 word types\n",
      "2019-12-04 18:43:48,564 : INFO : PROGRESS: at sentence #35630000, processed 537739063 words, keeping 482072 word types\n",
      "2019-12-04 18:43:48,822 : INFO : PROGRESS: at sentence #35640000, processed 537930469 words, keeping 489534 word types\n",
      "2019-12-04 18:43:49,074 : INFO : PROGRESS: at sentence #35650000, processed 538123855 words, keeping 496544 word types\n",
      "2019-12-04 18:43:49,459 : INFO : pruned out 297050 tokens with count <=49 (before 500001, after 202951)\n",
      "2019-12-04 18:43:49,612 : INFO : PROGRESS: at sentence #35660000, processed 538335120 words, keeping 208819 word types\n",
      "2019-12-04 18:43:49,889 : INFO : PROGRESS: at sentence #35670000, processed 538544562 words, keeping 219457 word types\n",
      "2019-12-04 18:43:50,174 : INFO : PROGRESS: at sentence #35680000, processed 538759517 words, keeping 230508 word types\n",
      "2019-12-04 18:43:50,455 : INFO : PROGRESS: at sentence #35690000, processed 538980730 words, keeping 241289 word types\n",
      "2019-12-04 18:43:50,725 : INFO : PROGRESS: at sentence #35700000, processed 539186824 words, keeping 251582 word types\n",
      "2019-12-04 18:43:50,985 : INFO : PROGRESS: at sentence #35710000, processed 539390569 words, keeping 261376 word types\n",
      "2019-12-04 18:43:51,270 : INFO : PROGRESS: at sentence #35720000, processed 539613245 words, keeping 271710 word types\n",
      "2019-12-04 18:43:51,547 : INFO : PROGRESS: at sentence #35730000, processed 539829994 words, keeping 281954 word types\n",
      "2019-12-04 18:43:51,791 : INFO : PROGRESS: at sentence #35740000, processed 540013749 words, keeping 291375 word types\n",
      "2019-12-04 18:43:52,091 : INFO : PROGRESS: at sentence #35750000, processed 540249998 words, keeping 301385 word types\n",
      "2019-12-04 18:43:52,363 : INFO : PROGRESS: at sentence #35760000, processed 540460478 words, keeping 312012 word types\n",
      "2019-12-04 18:43:52,663 : INFO : PROGRESS: at sentence #35770000, processed 540670898 words, keeping 321403 word types\n",
      "2019-12-04 18:43:52,911 : INFO : PROGRESS: at sentence #35780000, processed 540862086 words, keeping 329677 word types\n",
      "2019-12-04 18:43:53,186 : INFO : PROGRESS: at sentence #35790000, processed 541079261 words, keeping 338991 word types\n",
      "2019-12-04 18:43:53,452 : INFO : PROGRESS: at sentence #35800000, processed 541290268 words, keeping 348256 word types\n",
      "2019-12-04 18:43:53,712 : INFO : PROGRESS: at sentence #35810000, processed 541490687 words, keeping 355878 word types\n",
      "2019-12-04 18:43:53,974 : INFO : PROGRESS: at sentence #35820000, processed 541692780 words, keeping 364140 word types\n",
      "2019-12-04 18:43:54,249 : INFO : PROGRESS: at sentence #35830000, processed 541897013 words, keeping 373254 word types\n",
      "2019-12-04 18:43:54,515 : INFO : PROGRESS: at sentence #35840000, processed 542097569 words, keeping 381479 word types\n",
      "2019-12-04 18:43:54,776 : INFO : PROGRESS: at sentence #35850000, processed 542302272 words, keeping 389139 word types\n",
      "2019-12-04 18:43:55,036 : INFO : PROGRESS: at sentence #35860000, processed 542504954 words, keeping 397715 word types\n",
      "2019-12-04 18:43:55,299 : INFO : PROGRESS: at sentence #35870000, processed 542707176 words, keeping 406674 word types\n",
      "2019-12-04 18:43:55,545 : INFO : PROGRESS: at sentence #35880000, processed 542895962 words, keeping 414684 word types\n",
      "2019-12-04 18:43:55,791 : INFO : PROGRESS: at sentence #35890000, processed 543089131 words, keeping 425136 word types\n",
      "2019-12-04 18:43:56,066 : INFO : PROGRESS: at sentence #35900000, processed 543292402 words, keeping 433032 word types\n",
      "2019-12-04 18:43:56,336 : INFO : PROGRESS: at sentence #35910000, processed 543500011 words, keeping 440552 word types\n",
      "2019-12-04 18:43:56,602 : INFO : PROGRESS: at sentence #35920000, processed 543702597 words, keeping 448998 word types\n",
      "2019-12-04 18:43:56,870 : INFO : PROGRESS: at sentence #35930000, processed 543911318 words, keeping 457135 word types\n",
      "2019-12-04 18:43:57,129 : INFO : PROGRESS: at sentence #35940000, processed 544112439 words, keeping 463838 word types\n",
      "2019-12-04 18:43:57,415 : INFO : PROGRESS: at sentence #35950000, processed 544334853 words, keeping 474189 word types\n",
      "2019-12-04 18:43:57,670 : INFO : PROGRESS: at sentence #35960000, processed 544534185 words, keeping 481446 word types\n",
      "2019-12-04 18:43:57,924 : INFO : PROGRESS: at sentence #35970000, processed 544732726 words, keeping 489111 word types\n",
      "2019-12-04 18:43:58,176 : INFO : PROGRESS: at sentence #35980000, processed 544926081 words, keeping 496391 word types\n",
      "2019-12-04 18:43:58,567 : INFO : pruned out 297241 tokens with count <=50 (before 500001, after 202760)\n",
      "2019-12-04 18:43:58,750 : INFO : PROGRESS: at sentence #35990000, processed 545139860 words, keeping 210185 word types\n",
      "2019-12-04 18:43:59,087 : INFO : PROGRESS: at sentence #36000000, processed 545350582 words, keeping 221265 word types\n",
      "2019-12-04 18:43:59,353 : INFO : PROGRESS: at sentence #36010000, processed 545556872 words, keeping 231615 word types\n",
      "2019-12-04 18:43:59,615 : INFO : PROGRESS: at sentence #36020000, processed 545760270 words, keeping 242249 word types\n",
      "2019-12-04 18:43:59,872 : INFO : PROGRESS: at sentence #36030000, processed 545958520 words, keeping 253564 word types\n",
      "2019-12-04 18:44:00,155 : INFO : PROGRESS: at sentence #36040000, processed 546177348 words, keeping 264516 word types\n",
      "2019-12-04 18:44:00,420 : INFO : PROGRESS: at sentence #36050000, processed 546383327 words, keeping 274877 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:44:00,674 : INFO : PROGRESS: at sentence #36060000, processed 546580162 words, keeping 284618 word types\n",
      "2019-12-04 18:44:00,943 : INFO : PROGRESS: at sentence #36070000, processed 546787411 words, keeping 294248 word types\n",
      "2019-12-04 18:44:01,198 : INFO : PROGRESS: at sentence #36080000, processed 546981332 words, keeping 303111 word types\n",
      "2019-12-04 18:44:01,472 : INFO : PROGRESS: at sentence #36090000, processed 547189299 words, keeping 313042 word types\n",
      "2019-12-04 18:44:01,755 : INFO : PROGRESS: at sentence #36100000, processed 547390062 words, keeping 321648 word types\n",
      "2019-12-04 18:44:02,024 : INFO : PROGRESS: at sentence #36110000, processed 547595218 words, keeping 330957 word types\n",
      "2019-12-04 18:44:02,292 : INFO : PROGRESS: at sentence #36120000, processed 547799713 words, keeping 339493 word types\n",
      "2019-12-04 18:44:02,562 : INFO : PROGRESS: at sentence #36130000, processed 548005439 words, keeping 348582 word types\n",
      "2019-12-04 18:44:02,840 : INFO : PROGRESS: at sentence #36140000, processed 548220427 words, keeping 356981 word types\n",
      "2019-12-04 18:44:03,108 : INFO : PROGRESS: at sentence #36150000, processed 548425293 words, keeping 365466 word types\n",
      "2019-12-04 18:44:03,365 : INFO : PROGRESS: at sentence #36160000, processed 548622709 words, keeping 373365 word types\n",
      "2019-12-04 18:44:03,649 : INFO : PROGRESS: at sentence #36170000, processed 548826353 words, keeping 381800 word types\n",
      "2019-12-04 18:44:03,923 : INFO : PROGRESS: at sentence #36180000, processed 549035512 words, keeping 390760 word types\n",
      "2019-12-04 18:44:04,189 : INFO : PROGRESS: at sentence #36190000, processed 549235958 words, keeping 398932 word types\n",
      "2019-12-04 18:44:04,454 : INFO : PROGRESS: at sentence #36200000, processed 549438386 words, keeping 406952 word types\n",
      "2019-12-04 18:44:04,731 : INFO : PROGRESS: at sentence #36210000, processed 549645345 words, keeping 414766 word types\n",
      "2019-12-04 18:44:05,008 : INFO : PROGRESS: at sentence #36220000, processed 549854670 words, keeping 423787 word types\n",
      "2019-12-04 18:44:05,281 : INFO : PROGRESS: at sentence #36230000, processed 550066151 words, keeping 431524 word types\n",
      "2019-12-04 18:44:05,552 : INFO : PROGRESS: at sentence #36240000, processed 550277997 words, keeping 439829 word types\n",
      "2019-12-04 18:44:05,810 : INFO : PROGRESS: at sentence #36250000, processed 550478333 words, keeping 447109 word types\n",
      "2019-12-04 18:44:06,085 : INFO : PROGRESS: at sentence #36260000, processed 550690605 words, keeping 455037 word types\n",
      "2019-12-04 18:44:06,345 : INFO : PROGRESS: at sentence #36270000, processed 550891979 words, keeping 461785 word types\n",
      "2019-12-04 18:44:06,610 : INFO : PROGRESS: at sentence #36280000, processed 551097802 words, keeping 469078 word types\n",
      "2019-12-04 18:44:06,908 : INFO : PROGRESS: at sentence #36290000, processed 551331274 words, keeping 478775 word types\n",
      "2019-12-04 18:44:07,169 : INFO : PROGRESS: at sentence #36300000, processed 551532632 words, keeping 486595 word types\n",
      "2019-12-04 18:44:07,433 : INFO : PROGRESS: at sentence #36310000, processed 551735380 words, keeping 493549 word types\n",
      "2019-12-04 18:44:07,948 : INFO : pruned out 317842 tokens with count <=51 (before 500002, after 182160)\n",
      "2019-12-04 18:44:07,996 : INFO : PROGRESS: at sentence #36320000, processed 551946494 words, keeping 184531 word types\n",
      "2019-12-04 18:44:08,273 : INFO : PROGRESS: at sentence #36330000, processed 552158290 words, keeping 195440 word types\n",
      "2019-12-04 18:44:08,537 : INFO : PROGRESS: at sentence #36340000, processed 552357673 words, keeping 205285 word types\n",
      "2019-12-04 18:44:08,798 : INFO : PROGRESS: at sentence #36350000, processed 552553808 words, keeping 215289 word types\n",
      "2019-12-04 18:44:09,067 : INFO : PROGRESS: at sentence #36360000, processed 552756172 words, keeping 225251 word types\n",
      "2019-12-04 18:44:09,315 : INFO : PROGRESS: at sentence #36370000, processed 552945525 words, keeping 233917 word types\n",
      "2019-12-04 18:44:09,605 : INFO : PROGRESS: at sentence #36380000, processed 553166747 words, keeping 244478 word types\n",
      "2019-12-04 18:44:09,876 : INFO : PROGRESS: at sentence #36390000, processed 553375797 words, keeping 254333 word types\n",
      "2019-12-04 18:44:10,142 : INFO : PROGRESS: at sentence #36400000, processed 553578653 words, keeping 264117 word types\n",
      "2019-12-04 18:44:10,416 : INFO : PROGRESS: at sentence #36410000, processed 553786631 words, keeping 273930 word types\n",
      "2019-12-04 18:44:10,687 : INFO : PROGRESS: at sentence #36420000, processed 553995779 words, keeping 283744 word types\n",
      "2019-12-04 18:44:10,956 : INFO : PROGRESS: at sentence #36430000, processed 554193115 words, keeping 292225 word types\n",
      "2019-12-04 18:44:11,224 : INFO : PROGRESS: at sentence #36440000, processed 554392866 words, keeping 301573 word types\n",
      "2019-12-04 18:44:11,502 : INFO : PROGRESS: at sentence #36450000, processed 554600116 words, keeping 310665 word types\n",
      "2019-12-04 18:44:11,787 : INFO : PROGRESS: at sentence #36460000, processed 554803743 words, keeping 319131 word types\n",
      "2019-12-04 18:44:12,042 : INFO : PROGRESS: at sentence #36470000, processed 554995595 words, keeping 326951 word types\n",
      "2019-12-04 18:44:12,310 : INFO : PROGRESS: at sentence #36480000, processed 555200409 words, keeping 335772 word types\n",
      "2019-12-04 18:44:12,581 : INFO : PROGRESS: at sentence #36490000, processed 555405226 words, keeping 345015 word types\n",
      "2019-12-04 18:44:12,870 : INFO : PROGRESS: at sentence #36500000, processed 555628332 words, keeping 357307 word types\n",
      "2019-12-04 18:44:13,147 : INFO : PROGRESS: at sentence #36510000, processed 555841628 words, keeping 366082 word types\n",
      "2019-12-04 18:44:13,420 : INFO : PROGRESS: at sentence #36520000, processed 556050887 words, keeping 375567 word types\n",
      "2019-12-04 18:44:13,689 : INFO : PROGRESS: at sentence #36530000, processed 556259363 words, keeping 384400 word types\n",
      "2019-12-04 18:44:14,050 : INFO : PROGRESS: at sentence #36540000, processed 556530986 words, keeping 399265 word types\n",
      "2019-12-04 18:44:14,319 : INFO : PROGRESS: at sentence #36550000, processed 556737636 words, keeping 407799 word types\n",
      "2019-12-04 18:44:14,596 : INFO : PROGRESS: at sentence #36560000, processed 556953487 words, keeping 415749 word types\n",
      "2019-12-04 18:44:14,869 : INFO : PROGRESS: at sentence #36570000, processed 557163661 words, keeping 424135 word types\n",
      "2019-12-04 18:44:15,140 : INFO : PROGRESS: at sentence #36580000, processed 557364051 words, keeping 431537 word types\n",
      "2019-12-04 18:44:15,409 : INFO : PROGRESS: at sentence #36590000, processed 557575805 words, keeping 440070 word types\n",
      "2019-12-04 18:44:15,682 : INFO : PROGRESS: at sentence #36600000, processed 557784495 words, keeping 447608 word types\n",
      "2019-12-04 18:44:15,952 : INFO : PROGRESS: at sentence #36610000, processed 557990078 words, keeping 455105 word types\n",
      "2019-12-04 18:44:16,244 : INFO : PROGRESS: at sentence #36620000, processed 558213818 words, keeping 464174 word types\n",
      "2019-12-04 18:44:16,521 : INFO : PROGRESS: at sentence #36630000, processed 558421630 words, keeping 472166 word types\n",
      "2019-12-04 18:44:16,793 : INFO : PROGRESS: at sentence #36640000, processed 558633060 words, keeping 480249 word types\n",
      "2019-12-04 18:44:17,062 : INFO : PROGRESS: at sentence #36650000, processed 558842464 words, keeping 487884 word types\n",
      "2019-12-04 18:44:17,330 : INFO : PROGRESS: at sentence #36660000, processed 559049302 words, keeping 494866 word types\n",
      "2019-12-04 18:44:17,801 : INFO : pruned out 319297 tokens with count <=52 (before 500001, after 180704)\n",
      "2019-12-04 18:44:17,871 : INFO : PROGRESS: at sentence #36670000, processed 559245935 words, keeping 183538 word types\n",
      "2019-12-04 18:44:18,138 : INFO : PROGRESS: at sentence #36680000, processed 559451219 words, keeping 194460 word types\n",
      "2019-12-04 18:44:18,398 : INFO : PROGRESS: at sentence #36690000, processed 559650635 words, keeping 204981 word types\n",
      "2019-12-04 18:44:18,689 : INFO : PROGRESS: at sentence #36700000, processed 559890552 words, keeping 218249 word types\n",
      "2019-12-04 18:44:18,937 : INFO : PROGRESS: at sentence #36710000, processed 560091010 words, keeping 228137 word types\n",
      "2019-12-04 18:44:19,194 : INFO : PROGRESS: at sentence #36720000, processed 560298451 words, keeping 238733 word types\n",
      "2019-12-04 18:44:19,457 : INFO : PROGRESS: at sentence #36730000, processed 560512017 words, keeping 249869 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:44:19,713 : INFO : PROGRESS: at sentence #36740000, processed 560716465 words, keeping 259571 word types\n",
      "2019-12-04 18:44:19,985 : INFO : PROGRESS: at sentence #36750000, processed 560932426 words, keeping 268812 word types\n",
      "2019-12-04 18:44:20,254 : INFO : PROGRESS: at sentence #36760000, processed 561143525 words, keeping 277963 word types\n",
      "2019-12-04 18:44:20,521 : INFO : PROGRESS: at sentence #36770000, processed 561348665 words, keeping 286807 word types\n",
      "2019-12-04 18:44:20,792 : INFO : PROGRESS: at sentence #36780000, processed 561560103 words, keeping 296464 word types\n",
      "2019-12-04 18:44:21,068 : INFO : PROGRESS: at sentence #36790000, processed 561785058 words, keeping 305908 word types\n",
      "2019-12-04 18:44:21,334 : INFO : PROGRESS: at sentence #36800000, processed 561997774 words, keeping 315714 word types\n",
      "2019-12-04 18:44:21,604 : INFO : PROGRESS: at sentence #36810000, processed 562207609 words, keeping 325675 word types\n",
      "2019-12-04 18:44:21,881 : INFO : PROGRESS: at sentence #36820000, processed 562422829 words, keeping 335111 word types\n",
      "2019-12-04 18:44:22,142 : INFO : PROGRESS: at sentence #36830000, processed 562627147 words, keeping 344013 word types\n",
      "2019-12-04 18:44:22,434 : INFO : PROGRESS: at sentence #36840000, processed 562865618 words, keeping 354461 word types\n",
      "2019-12-04 18:44:22,707 : INFO : PROGRESS: at sentence #36850000, processed 563080364 words, keeping 362741 word types\n",
      "2019-12-04 18:44:22,985 : INFO : PROGRESS: at sentence #36860000, processed 563285581 words, keeping 371225 word types\n",
      "2019-12-04 18:44:23,246 : INFO : PROGRESS: at sentence #36870000, processed 563485653 words, keeping 379739 word types\n",
      "2019-12-04 18:44:23,532 : INFO : PROGRESS: at sentence #36880000, processed 563714534 words, keeping 389456 word types\n",
      "2019-12-04 18:44:23,824 : INFO : PROGRESS: at sentence #36890000, processed 563958393 words, keeping 400272 word types\n",
      "2019-12-04 18:44:24,127 : INFO : PROGRESS: at sentence #36900000, processed 564210959 words, keeping 411213 word types\n",
      "2019-12-04 18:44:24,479 : INFO : PROGRESS: at sentence #36910000, processed 564516194 words, keeping 424034 word types\n",
      "2019-12-04 18:44:24,776 : INFO : PROGRESS: at sentence #36920000, processed 564759054 words, keeping 434239 word types\n",
      "2019-12-04 18:44:25,034 : INFO : PROGRESS: at sentence #36930000, processed 564964800 words, keeping 441413 word types\n",
      "2019-12-04 18:44:25,292 : INFO : PROGRESS: at sentence #36940000, processed 565166412 words, keeping 448244 word types\n",
      "2019-12-04 18:44:25,541 : INFO : PROGRESS: at sentence #36950000, processed 565359990 words, keeping 454772 word types\n",
      "2019-12-04 18:44:25,802 : INFO : PROGRESS: at sentence #36960000, processed 565564222 words, keeping 464389 word types\n",
      "2019-12-04 18:44:26,061 : INFO : PROGRESS: at sentence #36970000, processed 565767945 words, keeping 472070 word types\n",
      "2019-12-04 18:44:26,324 : INFO : PROGRESS: at sentence #36980000, processed 565976133 words, keeping 480459 word types\n",
      "2019-12-04 18:44:26,604 : INFO : PROGRESS: at sentence #36990000, processed 566181466 words, keeping 488903 word types\n",
      "2019-12-04 18:44:26,865 : INFO : PROGRESS: at sentence #37000000, processed 566392174 words, keeping 497300 word types\n",
      "2019-12-04 18:44:27,294 : INFO : pruned out 319857 tokens with count <=53 (before 500001, after 180144)\n",
      "2019-12-04 18:44:27,467 : INFO : PROGRESS: at sentence #37010000, processed 566595345 words, keeping 187318 word types\n",
      "2019-12-04 18:44:27,724 : INFO : PROGRESS: at sentence #37020000, processed 566793119 words, keeping 197813 word types\n",
      "2019-12-04 18:44:27,975 : INFO : PROGRESS: at sentence #37030000, processed 566991834 words, keeping 208445 word types\n",
      "2019-12-04 18:44:28,241 : INFO : PROGRESS: at sentence #37040000, processed 567197245 words, keeping 218652 word types\n",
      "2019-12-04 18:44:28,503 : INFO : PROGRESS: at sentence #37050000, processed 567402547 words, keeping 228528 word types\n",
      "2019-12-04 18:44:28,760 : INFO : PROGRESS: at sentence #37060000, processed 567604642 words, keeping 238278 word types\n",
      "2019-12-04 18:44:29,021 : INFO : PROGRESS: at sentence #37070000, processed 567808596 words, keeping 247369 word types\n",
      "2019-12-04 18:44:29,283 : INFO : PROGRESS: at sentence #37080000, processed 568014245 words, keeping 256342 word types\n",
      "2019-12-04 18:44:29,541 : INFO : PROGRESS: at sentence #37090000, processed 568213129 words, keeping 265688 word types\n",
      "2019-12-04 18:44:29,802 : INFO : PROGRESS: at sentence #37100000, processed 568418685 words, keeping 275528 word types\n",
      "2019-12-04 18:44:30,076 : INFO : PROGRESS: at sentence #37110000, processed 568629667 words, keeping 286659 word types\n",
      "2019-12-04 18:44:30,355 : INFO : PROGRESS: at sentence #37120000, processed 568848592 words, keeping 297669 word types\n",
      "2019-12-04 18:44:30,607 : INFO : PROGRESS: at sentence #37130000, processed 569044361 words, keeping 306060 word types\n",
      "2019-12-04 18:44:30,868 : INFO : PROGRESS: at sentence #37140000, processed 569245965 words, keeping 314691 word types\n",
      "2019-12-04 18:44:31,131 : INFO : PROGRESS: at sentence #37150000, processed 569451612 words, keeping 323811 word types\n",
      "2019-12-04 18:44:31,381 : INFO : PROGRESS: at sentence #37160000, processed 569645839 words, keeping 330609 word types\n",
      "2019-12-04 18:44:31,654 : INFO : PROGRESS: at sentence #37170000, processed 569856465 words, keeping 339600 word types\n",
      "2019-12-04 18:44:31,935 : INFO : PROGRESS: at sentence #37180000, processed 570066787 words, keeping 348799 word types\n",
      "2019-12-04 18:44:32,190 : INFO : PROGRESS: at sentence #37190000, processed 570266887 words, keeping 357129 word types\n",
      "2019-12-04 18:44:32,460 : INFO : PROGRESS: at sentence #37200000, processed 570477094 words, keeping 366669 word types\n",
      "2019-12-04 18:44:32,723 : INFO : PROGRESS: at sentence #37210000, processed 570680891 words, keeping 374912 word types\n",
      "2019-12-04 18:44:32,994 : INFO : PROGRESS: at sentence #37220000, processed 570895849 words, keeping 383407 word types\n",
      "2019-12-04 18:44:33,250 : INFO : PROGRESS: at sentence #37230000, processed 571092453 words, keeping 391478 word types\n",
      "2019-12-04 18:44:33,505 : INFO : PROGRESS: at sentence #37240000, processed 571291726 words, keeping 399407 word types\n",
      "2019-12-04 18:44:33,763 : INFO : PROGRESS: at sentence #37250000, processed 571493277 words, keeping 406866 word types\n",
      "2019-12-04 18:44:34,022 : INFO : PROGRESS: at sentence #37260000, processed 571696104 words, keeping 415269 word types\n",
      "2019-12-04 18:44:34,276 : INFO : PROGRESS: at sentence #37270000, processed 571891509 words, keeping 423587 word types\n",
      "2019-12-04 18:44:34,532 : INFO : PROGRESS: at sentence #37280000, processed 572092219 words, keeping 431444 word types\n",
      "2019-12-04 18:44:34,800 : INFO : PROGRESS: at sentence #37290000, processed 572303656 words, keeping 440129 word types\n",
      "2019-12-04 18:44:35,117 : INFO : PROGRESS: at sentence #37300000, processed 572512283 words, keeping 447853 word types\n",
      "2019-12-04 18:44:35,372 : INFO : PROGRESS: at sentence #37310000, processed 572710250 words, keeping 455162 word types\n",
      "2019-12-04 18:44:35,628 : INFO : PROGRESS: at sentence #37320000, processed 572914927 words, keeping 463070 word types\n",
      "2019-12-04 18:44:35,889 : INFO : PROGRESS: at sentence #37330000, processed 573127578 words, keeping 470399 word types\n",
      "2019-12-04 18:44:36,143 : INFO : PROGRESS: at sentence #37340000, processed 573333771 words, keeping 478543 word types\n",
      "2019-12-04 18:44:36,403 : INFO : PROGRESS: at sentence #37350000, processed 573536233 words, keeping 485677 word types\n",
      "2019-12-04 18:44:36,664 : INFO : PROGRESS: at sentence #37360000, processed 573740836 words, keeping 493246 word types\n",
      "2019-12-04 18:44:37,168 : INFO : pruned out 320513 tokens with count <=54 (before 500001, after 179488)\n",
      "2019-12-04 18:44:37,359 : INFO : PROGRESS: at sentence #37370000, processed 574061137 words, keeping 190862 word types\n",
      "2019-12-04 18:44:37,637 : INFO : PROGRESS: at sentence #37380000, processed 574275384 words, keeping 203005 word types\n",
      "2019-12-04 18:44:37,910 : INFO : PROGRESS: at sentence #37390000, processed 574488707 words, keeping 213988 word types\n",
      "2019-12-04 18:44:38,160 : INFO : PROGRESS: at sentence #37400000, processed 574683548 words, keeping 223412 word types\n",
      "2019-12-04 18:44:38,408 : INFO : PROGRESS: at sentence #37410000, processed 574875605 words, keeping 233302 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:44:38,671 : INFO : PROGRESS: at sentence #37420000, processed 575081198 words, keeping 243886 word types\n",
      "2019-12-04 18:44:38,931 : INFO : PROGRESS: at sentence #37430000, processed 575279339 words, keeping 253749 word types\n",
      "2019-12-04 18:44:39,185 : INFO : PROGRESS: at sentence #37440000, processed 575474861 words, keeping 263170 word types\n",
      "2019-12-04 18:44:39,441 : INFO : PROGRESS: at sentence #37450000, processed 575673423 words, keeping 271742 word types\n",
      "2019-12-04 18:44:39,701 : INFO : PROGRESS: at sentence #37460000, processed 575874823 words, keeping 280414 word types\n",
      "2019-12-04 18:44:39,952 : INFO : PROGRESS: at sentence #37470000, processed 576071034 words, keeping 288789 word types\n",
      "2019-12-04 18:44:40,213 : INFO : PROGRESS: at sentence #37480000, processed 576273701 words, keeping 298562 word types\n",
      "2019-12-04 18:44:40,469 : INFO : PROGRESS: at sentence #37490000, processed 576471467 words, keeping 307971 word types\n",
      "2019-12-04 18:44:40,731 : INFO : PROGRESS: at sentence #37500000, processed 576670904 words, keeping 316014 word types\n",
      "2019-12-04 18:44:40,980 : INFO : PROGRESS: at sentence #37510000, processed 576864860 words, keeping 323757 word types\n",
      "2019-12-04 18:44:41,234 : INFO : PROGRESS: at sentence #37520000, processed 577064657 words, keeping 332128 word types\n",
      "2019-12-04 18:44:41,495 : INFO : PROGRESS: at sentence #37530000, processed 577262590 words, keeping 340834 word types\n",
      "2019-12-04 18:44:41,755 : INFO : PROGRESS: at sentence #37540000, processed 577467860 words, keeping 350483 word types\n",
      "2019-12-04 18:44:42,004 : INFO : PROGRESS: at sentence #37550000, processed 577654494 words, keeping 359116 word types\n",
      "2019-12-04 18:44:42,281 : INFO : PROGRESS: at sentence #37560000, processed 577859918 words, keeping 367466 word types\n",
      "2019-12-04 18:44:42,551 : INFO : PROGRESS: at sentence #37570000, processed 578065138 words, keeping 376317 word types\n",
      "2019-12-04 18:44:42,818 : INFO : PROGRESS: at sentence #37580000, processed 578269585 words, keeping 384686 word types\n",
      "2019-12-04 18:44:43,063 : INFO : PROGRESS: at sentence #37590000, processed 578459447 words, keeping 391862 word types\n",
      "2019-12-04 18:44:43,321 : INFO : PROGRESS: at sentence #37600000, processed 578657949 words, keeping 399538 word types\n",
      "2019-12-04 18:44:43,580 : INFO : PROGRESS: at sentence #37610000, processed 578857101 words, keeping 407375 word types\n",
      "2019-12-04 18:44:43,845 : INFO : PROGRESS: at sentence #37620000, processed 579064447 words, keeping 415528 word types\n",
      "2019-12-04 18:44:44,104 : INFO : PROGRESS: at sentence #37630000, processed 579264149 words, keeping 422984 word types\n",
      "2019-12-04 18:44:44,367 : INFO : PROGRESS: at sentence #37640000, processed 579464938 words, keeping 430693 word types\n",
      "2019-12-04 18:44:44,628 : INFO : PROGRESS: at sentence #37650000, processed 579668861 words, keeping 438423 word types\n",
      "2019-12-04 18:44:44,890 : INFO : PROGRESS: at sentence #37660000, processed 579869906 words, keeping 445637 word types\n",
      "2019-12-04 18:44:45,161 : INFO : PROGRESS: at sentence #37670000, processed 580076096 words, keeping 453108 word types\n",
      "2019-12-04 18:44:45,426 : INFO : PROGRESS: at sentence #37680000, processed 580279940 words, keeping 460161 word types\n",
      "2019-12-04 18:44:45,680 : INFO : PROGRESS: at sentence #37690000, processed 580482500 words, keeping 468013 word types\n",
      "2019-12-04 18:44:45,936 : INFO : PROGRESS: at sentence #37700000, processed 580685955 words, keeping 475385 word types\n",
      "2019-12-04 18:44:46,204 : INFO : PROGRESS: at sentence #37710000, processed 580881054 words, keeping 482589 word types\n",
      "2019-12-04 18:44:46,462 : INFO : PROGRESS: at sentence #37720000, processed 581079043 words, keeping 489474 word types\n",
      "2019-12-04 18:44:46,728 : INFO : PROGRESS: at sentence #37730000, processed 581282541 words, keeping 496818 word types\n",
      "2019-12-04 18:44:47,155 : INFO : pruned out 321030 tokens with count <=55 (before 500002, after 178972)\n",
      "2019-12-04 18:44:47,305 : INFO : PROGRESS: at sentence #37740000, processed 581486834 words, keeping 185106 word types\n",
      "2019-12-04 18:44:47,581 : INFO : PROGRESS: at sentence #37750000, processed 581692371 words, keeping 195415 word types\n",
      "2019-12-04 18:44:47,843 : INFO : PROGRESS: at sentence #37760000, processed 581895172 words, keeping 206463 word types\n",
      "2019-12-04 18:44:48,105 : INFO : PROGRESS: at sentence #37770000, processed 582092413 words, keeping 216797 word types\n",
      "2019-12-04 18:44:48,387 : INFO : PROGRESS: at sentence #37780000, processed 582308466 words, keeping 228313 word types\n",
      "2019-12-04 18:44:48,686 : INFO : PROGRESS: at sentence #37790000, processed 582535295 words, keeping 240628 word types\n",
      "2019-12-04 18:44:48,981 : INFO : PROGRESS: at sentence #37800000, processed 582763245 words, keeping 251809 word types\n",
      "2019-12-04 18:44:49,252 : INFO : PROGRESS: at sentence #37810000, processed 582969427 words, keeping 261083 word types\n",
      "2019-12-04 18:44:49,515 : INFO : PROGRESS: at sentence #37820000, processed 583170855 words, keeping 270325 word types\n",
      "2019-12-04 18:44:49,816 : INFO : PROGRESS: at sentence #37830000, processed 583402153 words, keeping 283548 word types\n",
      "2019-12-04 18:44:50,072 : INFO : PROGRESS: at sentence #37840000, processed 583598385 words, keeping 292390 word types\n",
      "2019-12-04 18:44:50,330 : INFO : PROGRESS: at sentence #37850000, processed 583794110 words, keeping 301622 word types\n",
      "2019-12-04 18:44:50,598 : INFO : PROGRESS: at sentence #37860000, processed 583997724 words, keeping 310717 word types\n",
      "2019-12-04 18:44:50,866 : INFO : PROGRESS: at sentence #37870000, processed 584208230 words, keeping 319437 word types\n",
      "2019-12-04 18:44:51,144 : INFO : PROGRESS: at sentence #37880000, processed 584421765 words, keeping 328822 word types\n",
      "2019-12-04 18:44:51,403 : INFO : PROGRESS: at sentence #37890000, processed 584613387 words, keeping 336887 word types\n",
      "2019-12-04 18:44:51,673 : INFO : PROGRESS: at sentence #37900000, processed 584822098 words, keeping 345668 word types\n",
      "2019-12-04 18:44:51,948 : INFO : PROGRESS: at sentence #37910000, processed 585028188 words, keeping 355144 word types\n",
      "2019-12-04 18:44:52,200 : INFO : PROGRESS: at sentence #37920000, processed 585223698 words, keeping 363404 word types\n",
      "2019-12-04 18:44:52,492 : INFO : PROGRESS: at sentence #37930000, processed 585446183 words, keeping 372638 word types\n",
      "2019-12-04 18:44:52,769 : INFO : PROGRESS: at sentence #37940000, processed 585654619 words, keeping 380910 word types\n",
      "2019-12-04 18:44:53,075 : INFO : PROGRESS: at sentence #37950000, processed 585888993 words, keeping 390246 word types\n",
      "2019-12-04 18:44:53,379 : INFO : PROGRESS: at sentence #37960000, processed 586117604 words, keeping 399955 word types\n",
      "2019-12-04 18:44:53,658 : INFO : PROGRESS: at sentence #37970000, processed 586329802 words, keeping 408254 word types\n",
      "2019-12-04 18:44:53,947 : INFO : PROGRESS: at sentence #37980000, processed 586540741 words, keeping 416896 word types\n",
      "2019-12-04 18:44:54,227 : INFO : PROGRESS: at sentence #37990000, processed 586758224 words, keeping 424474 word types\n",
      "2019-12-04 18:44:54,521 : INFO : PROGRESS: at sentence #38000000, processed 586978640 words, keeping 432426 word types\n",
      "2019-12-04 18:44:54,809 : INFO : PROGRESS: at sentence #38010000, processed 587212880 words, keeping 441806 word types\n",
      "2019-12-04 18:44:55,088 : INFO : PROGRESS: at sentence #38020000, processed 587438962 words, keeping 450533 word types\n",
      "2019-12-04 18:44:55,398 : INFO : PROGRESS: at sentence #38030000, processed 587675557 words, keeping 459857 word types\n",
      "2019-12-04 18:44:55,689 : INFO : PROGRESS: at sentence #38040000, processed 587898933 words, keeping 468493 word types\n",
      "2019-12-04 18:44:55,978 : INFO : PROGRESS: at sentence #38050000, processed 588111527 words, keeping 475859 word types\n",
      "2019-12-04 18:44:56,247 : INFO : PROGRESS: at sentence #38060000, processed 588316357 words, keeping 483350 word types\n",
      "2019-12-04 18:44:56,531 : INFO : PROGRESS: at sentence #38070000, processed 588533695 words, keeping 491852 word types\n",
      "2019-12-04 18:44:56,816 : INFO : PROGRESS: at sentence #38080000, processed 588747296 words, keeping 499491 word types\n",
      "2019-12-04 18:44:57,151 : INFO : pruned out 321416 tokens with count <=56 (before 500001, after 178585)\n",
      "2019-12-04 18:44:57,436 : INFO : PROGRESS: at sentence #38090000, processed 588965405 words, keeping 189491 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:44:57,713 : INFO : PROGRESS: at sentence #38100000, processed 589168482 words, keeping 199647 word types\n",
      "2019-12-04 18:44:58,015 : INFO : PROGRESS: at sentence #38110000, processed 589382131 words, keeping 210242 word types\n",
      "2019-12-04 18:44:58,286 : INFO : PROGRESS: at sentence #38120000, processed 589586081 words, keeping 220242 word types\n",
      "2019-12-04 18:44:58,594 : INFO : PROGRESS: at sentence #38130000, processed 589810935 words, keeping 233726 word types\n",
      "2019-12-04 18:44:58,845 : INFO : PROGRESS: at sentence #38140000, processed 589999143 words, keeping 242488 word types\n",
      "2019-12-04 18:44:59,109 : INFO : PROGRESS: at sentence #38150000, processed 590203975 words, keeping 252321 word types\n",
      "2019-12-04 18:44:59,382 : INFO : PROGRESS: at sentence #38160000, processed 590408627 words, keeping 262090 word types\n",
      "2019-12-04 18:44:59,661 : INFO : PROGRESS: at sentence #38170000, processed 590613876 words, keeping 271115 word types\n",
      "2019-12-04 18:44:59,949 : INFO : PROGRESS: at sentence #38180000, processed 590817424 words, keeping 279719 word types\n",
      "2019-12-04 18:45:00,227 : INFO : PROGRESS: at sentence #38190000, processed 591027146 words, keeping 288797 word types\n",
      "2019-12-04 18:45:00,502 : INFO : PROGRESS: at sentence #38200000, processed 591228663 words, keeping 297864 word types\n",
      "2019-12-04 18:45:00,782 : INFO : PROGRESS: at sentence #38210000, processed 591438274 words, keeping 307245 word types\n",
      "2019-12-04 18:45:01,042 : INFO : PROGRESS: at sentence #38220000, processed 591639662 words, keeping 316016 word types\n",
      "2019-12-04 18:45:01,317 : INFO : PROGRESS: at sentence #38230000, processed 591841170 words, keeping 324740 word types\n",
      "2019-12-04 18:45:01,600 : INFO : PROGRESS: at sentence #38240000, processed 592057216 words, keeping 333287 word types\n",
      "2019-12-04 18:45:01,856 : INFO : PROGRESS: at sentence #38250000, processed 592256742 words, keeping 342020 word types\n",
      "2019-12-04 18:45:02,117 : INFO : PROGRESS: at sentence #38260000, processed 592457288 words, keeping 350429 word types\n",
      "2019-12-04 18:45:02,378 : INFO : PROGRESS: at sentence #38270000, processed 592652395 words, keeping 358355 word types\n",
      "2019-12-04 18:45:02,658 : INFO : PROGRESS: at sentence #38280000, processed 592862019 words, keeping 366863 word types\n",
      "2019-12-04 18:45:02,934 : INFO : PROGRESS: at sentence #38290000, processed 593071063 words, keeping 374946 word types\n",
      "2019-12-04 18:45:03,230 : INFO : PROGRESS: at sentence #38300000, processed 593275462 words, keeping 382985 word types\n",
      "2019-12-04 18:45:03,514 : INFO : PROGRESS: at sentence #38310000, processed 593476148 words, keeping 390717 word types\n",
      "2019-12-04 18:45:03,789 : INFO : PROGRESS: at sentence #38320000, processed 593677486 words, keeping 398868 word types\n",
      "2019-12-04 18:45:04,062 : INFO : PROGRESS: at sentence #38330000, processed 593887903 words, keeping 407766 word types\n",
      "2019-12-04 18:45:04,321 : INFO : PROGRESS: at sentence #38340000, processed 594082338 words, keeping 415091 word types\n",
      "2019-12-04 18:45:04,615 : INFO : PROGRESS: at sentence #38350000, processed 594281964 words, keeping 422952 word types\n",
      "2019-12-04 18:45:04,930 : INFO : PROGRESS: at sentence #38360000, processed 594505904 words, keeping 431250 word types\n",
      "2019-12-04 18:45:05,245 : INFO : PROGRESS: at sentence #38370000, processed 594696680 words, keeping 438421 word types\n",
      "2019-12-04 18:45:05,514 : INFO : PROGRESS: at sentence #38380000, processed 594898936 words, keeping 446141 word types\n",
      "2019-12-04 18:45:05,782 : INFO : PROGRESS: at sentence #38390000, processed 595098401 words, keeping 453531 word types\n",
      "2019-12-04 18:45:06,067 : INFO : PROGRESS: at sentence #38400000, processed 595298787 words, keeping 461306 word types\n",
      "2019-12-04 18:45:06,384 : INFO : PROGRESS: at sentence #38410000, processed 595500261 words, keeping 469121 word types\n",
      "2019-12-04 18:45:06,659 : INFO : PROGRESS: at sentence #38420000, processed 595705671 words, keeping 476416 word types\n",
      "2019-12-04 18:45:06,939 : INFO : PROGRESS: at sentence #38430000, processed 595908151 words, keeping 483538 word types\n",
      "2019-12-04 18:45:07,202 : INFO : PROGRESS: at sentence #38440000, processed 596106866 words, keeping 490620 word types\n",
      "2019-12-04 18:45:07,484 : INFO : PROGRESS: at sentence #38450000, processed 596319883 words, keeping 498808 word types\n",
      "2019-12-04 18:45:07,824 : INFO : pruned out 321845 tokens with count <=57 (before 500004, after 178159)\n",
      "2019-12-04 18:45:08,070 : INFO : PROGRESS: at sentence #38460000, processed 596523550 words, keeping 188317 word types\n",
      "2019-12-04 18:45:08,337 : INFO : PROGRESS: at sentence #38470000, processed 596723562 words, keeping 198959 word types\n",
      "2019-12-04 18:45:08,605 : INFO : PROGRESS: at sentence #38480000, processed 596923903 words, keeping 209394 word types\n",
      "2019-12-04 18:45:08,881 : INFO : PROGRESS: at sentence #38490000, processed 597131328 words, keeping 220290 word types\n",
      "2019-12-04 18:45:09,134 : INFO : PROGRESS: at sentence #38500000, processed 597315773 words, keeping 229922 word types\n",
      "2019-12-04 18:45:09,420 : INFO : PROGRESS: at sentence #38510000, processed 597523731 words, keeping 241008 word types\n",
      "2019-12-04 18:45:09,699 : INFO : PROGRESS: at sentence #38520000, processed 597731555 words, keeping 251313 word types\n",
      "2019-12-04 18:45:09,958 : INFO : PROGRESS: at sentence #38530000, processed 597930734 words, keeping 261882 word types\n",
      "2019-12-04 18:45:10,241 : INFO : PROGRESS: at sentence #38540000, processed 598143833 words, keeping 272031 word types\n",
      "2019-12-04 18:45:10,515 : INFO : PROGRESS: at sentence #38550000, processed 598352197 words, keeping 281433 word types\n",
      "2019-12-04 18:45:10,800 : INFO : PROGRESS: at sentence #38560000, processed 598553226 words, keeping 291339 word types\n",
      "2019-12-04 18:45:11,063 : INFO : PROGRESS: at sentence #38570000, processed 598748681 words, keeping 299969 word types\n",
      "2019-12-04 18:45:11,318 : INFO : PROGRESS: at sentence #38580000, processed 598936013 words, keeping 307657 word types\n",
      "2019-12-04 18:45:11,579 : INFO : PROGRESS: at sentence #38590000, processed 599139371 words, keeping 316167 word types\n",
      "2019-12-04 18:45:11,852 : INFO : PROGRESS: at sentence #38600000, processed 599335238 words, keeping 324535 word types\n",
      "2019-12-04 18:45:12,153 : INFO : PROGRESS: at sentence #38610000, processed 599534536 words, keeping 333004 word types\n",
      "2019-12-04 18:45:12,411 : INFO : PROGRESS: at sentence #38620000, processed 599725945 words, keeping 340552 word types\n",
      "2019-12-04 18:45:12,704 : INFO : PROGRESS: at sentence #38630000, processed 599935840 words, keeping 349781 word types\n",
      "2019-12-04 18:45:12,966 : INFO : PROGRESS: at sentence #38640000, processed 600129311 words, keeping 358076 word types\n",
      "2019-12-04 18:45:13,224 : INFO : PROGRESS: at sentence #38650000, processed 600329270 words, keeping 366103 word types\n",
      "2019-12-04 18:45:13,485 : INFO : PROGRESS: at sentence #38660000, processed 600531288 words, keeping 374614 word types\n",
      "2019-12-04 18:45:13,755 : INFO : PROGRESS: at sentence #38670000, processed 600733603 words, keeping 382745 word types\n",
      "2019-12-04 18:45:14,073 : INFO : PROGRESS: at sentence #38680000, processed 600940203 words, keeping 390950 word types\n",
      "2019-12-04 18:45:14,351 : INFO : PROGRESS: at sentence #38690000, processed 601139460 words, keeping 398355 word types\n",
      "2019-12-04 18:45:14,619 : INFO : PROGRESS: at sentence #38700000, processed 601338921 words, keeping 406226 word types\n",
      "2019-12-04 18:45:14,904 : INFO : PROGRESS: at sentence #38710000, processed 601530691 words, keeping 413110 word types\n",
      "2019-12-04 18:45:15,177 : INFO : PROGRESS: at sentence #38720000, processed 601732617 words, keeping 421448 word types\n",
      "2019-12-04 18:45:15,454 : INFO : PROGRESS: at sentence #38730000, processed 601934458 words, keeping 428926 word types\n",
      "2019-12-04 18:45:15,739 : INFO : PROGRESS: at sentence #38740000, processed 602135817 words, keeping 436590 word types\n",
      "2019-12-04 18:45:16,063 : INFO : PROGRESS: at sentence #38750000, processed 602343080 words, keeping 444127 word types\n",
      "2019-12-04 18:45:16,337 : INFO : PROGRESS: at sentence #38760000, processed 602550809 words, keeping 451818 word types\n",
      "2019-12-04 18:45:16,620 : INFO : PROGRESS: at sentence #38770000, processed 602760782 words, keeping 459651 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:45:16,878 : INFO : PROGRESS: at sentence #38780000, processed 602945753 words, keeping 466643 word types\n",
      "2019-12-04 18:45:17,166 : INFO : PROGRESS: at sentence #38790000, processed 603150851 words, keeping 473868 word types\n",
      "2019-12-04 18:45:17,445 : INFO : PROGRESS: at sentence #38800000, processed 603357570 words, keeping 481849 word types\n",
      "2019-12-04 18:45:17,761 : INFO : PROGRESS: at sentence #38810000, processed 603570326 words, keeping 489284 word types\n",
      "2019-12-04 18:45:18,029 : INFO : PROGRESS: at sentence #38820000, processed 603770881 words, keeping 496650 word types\n",
      "2019-12-04 18:45:18,455 : INFO : pruned out 322206 tokens with count <=58 (before 500002, after 177796)\n",
      "2019-12-04 18:45:18,621 : INFO : PROGRESS: at sentence #38830000, processed 603979644 words, keeping 184006 word types\n",
      "2019-12-04 18:45:18,884 : INFO : PROGRESS: at sentence #38840000, processed 604169839 words, keeping 193979 word types\n",
      "2019-12-04 18:45:19,164 : INFO : PROGRESS: at sentence #38850000, processed 604376567 words, keeping 203964 word types\n",
      "2019-12-04 18:45:19,438 : INFO : PROGRESS: at sentence #38860000, processed 604581061 words, keeping 213779 word types\n",
      "2019-12-04 18:45:19,714 : INFO : PROGRESS: at sentence #38870000, processed 604785520 words, keeping 223503 word types\n",
      "2019-12-04 18:45:19,998 : INFO : PROGRESS: at sentence #38880000, processed 604997424 words, keeping 233579 word types\n",
      "2019-12-04 18:45:20,277 : INFO : PROGRESS: at sentence #38890000, processed 605197702 words, keeping 242696 word types\n",
      "2019-12-04 18:45:20,547 : INFO : PROGRESS: at sentence #38900000, processed 605399925 words, keeping 251689 word types\n",
      "2019-12-04 18:45:20,816 : INFO : PROGRESS: at sentence #38910000, processed 605608836 words, keeping 261003 word types\n",
      "2019-12-04 18:45:21,091 : INFO : PROGRESS: at sentence #38920000, processed 605813012 words, keeping 270093 word types\n",
      "2019-12-04 18:45:21,363 : INFO : PROGRESS: at sentence #38930000, processed 606018918 words, keeping 278867 word types\n",
      "2019-12-04 18:45:21,638 : INFO : PROGRESS: at sentence #38940000, processed 606216680 words, keeping 287153 word types\n",
      "2019-12-04 18:45:21,935 : INFO : PROGRESS: at sentence #38950000, processed 606432523 words, keeping 296305 word types\n",
      "2019-12-04 18:45:22,194 : INFO : PROGRESS: at sentence #38960000, processed 606631206 words, keeping 304982 word types\n",
      "2019-12-04 18:45:22,473 : INFO : PROGRESS: at sentence #38970000, processed 606832590 words, keeping 313343 word types\n",
      "2019-12-04 18:45:22,750 : INFO : PROGRESS: at sentence #38980000, processed 607040426 words, keeping 323732 word types\n",
      "2019-12-04 18:45:23,032 : INFO : PROGRESS: at sentence #38990000, processed 607249055 words, keeping 332315 word types\n",
      "2019-12-04 18:45:23,310 : INFO : PROGRESS: at sentence #39000000, processed 607467166 words, keeping 341025 word types\n",
      "2019-12-04 18:45:23,567 : INFO : PROGRESS: at sentence #39010000, processed 607656931 words, keeping 348688 word types\n",
      "2019-12-04 18:45:23,830 : INFO : PROGRESS: at sentence #39020000, processed 607852414 words, keeping 356048 word types\n",
      "2019-12-04 18:45:24,124 : INFO : PROGRESS: at sentence #39030000, processed 608061965 words, keeping 364277 word types\n",
      "2019-12-04 18:45:24,409 : INFO : PROGRESS: at sentence #39040000, processed 608262363 words, keeping 371880 word types\n",
      "2019-12-04 18:45:24,717 : INFO : PROGRESS: at sentence #39050000, processed 608468639 words, keeping 379465 word types\n",
      "2019-12-04 18:45:25,005 : INFO : PROGRESS: at sentence #39060000, processed 608671838 words, keeping 386950 word types\n",
      "2019-12-04 18:45:25,307 : INFO : PROGRESS: at sentence #39070000, processed 608890433 words, keeping 395761 word types\n",
      "2019-12-04 18:45:25,583 : INFO : PROGRESS: at sentence #39080000, processed 609097543 words, keeping 403056 word types\n",
      "2019-12-04 18:45:25,837 : INFO : PROGRESS: at sentence #39090000, processed 609292943 words, keeping 410472 word types\n",
      "2019-12-04 18:45:26,112 : INFO : PROGRESS: at sentence #39100000, processed 609508050 words, keeping 419158 word types\n",
      "2019-12-04 18:45:26,391 : INFO : PROGRESS: at sentence #39110000, processed 609722535 words, keeping 426848 word types\n",
      "2019-12-04 18:45:26,650 : INFO : PROGRESS: at sentence #39120000, processed 609919063 words, keeping 433765 word types\n",
      "2019-12-04 18:45:26,929 : INFO : PROGRESS: at sentence #39130000, processed 610122518 words, keeping 441128 word types\n",
      "2019-12-04 18:45:27,192 : INFO : PROGRESS: at sentence #39140000, processed 610323209 words, keeping 448261 word types\n",
      "2019-12-04 18:45:27,466 : INFO : PROGRESS: at sentence #39150000, processed 610525709 words, keeping 455306 word types\n",
      "2019-12-04 18:45:27,745 : INFO : PROGRESS: at sentence #39160000, processed 610728833 words, keeping 462552 word types\n",
      "2019-12-04 18:45:28,025 : INFO : PROGRESS: at sentence #39170000, processed 610932717 words, keeping 469363 word types\n",
      "2019-12-04 18:45:28,298 : INFO : PROGRESS: at sentence #39180000, processed 611141002 words, keeping 476265 word types\n",
      "2019-12-04 18:45:28,556 : INFO : PROGRESS: at sentence #39190000, processed 611326260 words, keeping 482919 word types\n",
      "2019-12-04 18:45:28,815 : INFO : PROGRESS: at sentence #39200000, processed 611521179 words, keeping 489760 word types\n",
      "2019-12-04 18:45:29,093 : INFO : PROGRESS: at sentence #39210000, processed 611724965 words, keeping 497848 word types\n",
      "2019-12-04 18:45:29,493 : INFO : pruned out 322473 tokens with count <=59 (before 500001, after 177528)\n",
      "2019-12-04 18:45:29,670 : INFO : PROGRESS: at sentence #39220000, processed 611917281 words, keeping 184255 word types\n",
      "2019-12-04 18:45:29,944 : INFO : PROGRESS: at sentence #39230000, processed 612113214 words, keeping 194236 word types\n",
      "2019-12-04 18:45:30,218 : INFO : PROGRESS: at sentence #39240000, processed 612318951 words, keeping 205438 word types\n",
      "2019-12-04 18:45:30,491 : INFO : PROGRESS: at sentence #39250000, processed 612518160 words, keeping 214928 word types\n",
      "2019-12-04 18:45:30,766 : INFO : PROGRESS: at sentence #39260000, processed 612725475 words, keeping 224324 word types\n",
      "2019-12-04 18:45:31,063 : INFO : PROGRESS: at sentence #39270000, processed 612935055 words, keeping 234987 word types\n",
      "2019-12-04 18:45:31,337 : INFO : PROGRESS: at sentence #39280000, processed 613137587 words, keeping 245001 word types\n",
      "2019-12-04 18:45:31,630 : INFO : PROGRESS: at sentence #39290000, processed 613359306 words, keeping 256772 word types\n",
      "2019-12-04 18:45:31,906 : INFO : PROGRESS: at sentence #39300000, processed 613567363 words, keeping 267418 word types\n",
      "2019-12-04 18:45:32,190 : INFO : PROGRESS: at sentence #39310000, processed 613775872 words, keeping 277274 word types\n",
      "2019-12-04 18:45:32,472 : INFO : PROGRESS: at sentence #39320000, processed 613970221 words, keeping 286498 word types\n",
      "2019-12-04 18:45:32,746 : INFO : PROGRESS: at sentence #39330000, processed 614174981 words, keeping 295299 word types\n",
      "2019-12-04 18:45:33,056 : INFO : PROGRESS: at sentence #39340000, processed 614384126 words, keeping 304699 word types\n",
      "2019-12-04 18:45:33,324 : INFO : PROGRESS: at sentence #39350000, processed 614588246 words, keeping 313477 word types\n",
      "2019-12-04 18:45:33,600 : INFO : PROGRESS: at sentence #39360000, processed 614791338 words, keeping 322138 word types\n",
      "2019-12-04 18:45:33,862 : INFO : PROGRESS: at sentence #39370000, processed 614988717 words, keeping 330192 word types\n",
      "2019-12-04 18:45:34,151 : INFO : PROGRESS: at sentence #39380000, processed 615197821 words, keeping 339009 word types\n",
      "2019-12-04 18:45:34,405 : INFO : PROGRESS: at sentence #39390000, processed 615387057 words, keeping 348500 word types\n",
      "2019-12-04 18:45:34,667 : INFO : PROGRESS: at sentence #39400000, processed 615582959 words, keeping 356908 word types\n",
      "2019-12-04 18:45:34,952 : INFO : PROGRESS: at sentence #39410000, processed 615789526 words, keeping 365335 word types\n",
      "2019-12-04 18:45:35,225 : INFO : PROGRESS: at sentence #39420000, processed 615992458 words, keeping 373779 word types\n",
      "2019-12-04 18:45:35,496 : INFO : PROGRESS: at sentence #39430000, processed 616187206 words, keeping 381520 word types\n",
      "2019-12-04 18:45:35,743 : INFO : PROGRESS: at sentence #39440000, processed 616377662 words, keeping 388515 word types\n",
      "2019-12-04 18:45:36,000 : INFO : PROGRESS: at sentence #39450000, processed 616569031 words, keeping 395997 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:45:36,267 : INFO : PROGRESS: at sentence #39460000, processed 616772551 words, keeping 403475 word types\n",
      "2019-12-04 18:45:36,534 : INFO : PROGRESS: at sentence #39470000, processed 616977898 words, keeping 411403 word types\n",
      "2019-12-04 18:45:36,807 : INFO : PROGRESS: at sentence #39480000, processed 617187246 words, keeping 420068 word types\n",
      "2019-12-04 18:45:37,068 : INFO : PROGRESS: at sentence #39490000, processed 617387254 words, keeping 427957 word types\n",
      "2019-12-04 18:45:37,395 : INFO : PROGRESS: at sentence #39500000, processed 617596379 words, keeping 436055 word types\n",
      "2019-12-04 18:45:37,674 : INFO : PROGRESS: at sentence #39510000, processed 617821586 words, keeping 444892 word types\n",
      "2019-12-04 18:45:37,919 : INFO : PROGRESS: at sentence #39520000, processed 618019425 words, keeping 452643 word types\n",
      "2019-12-04 18:45:38,168 : INFO : PROGRESS: at sentence #39530000, processed 618213223 words, keeping 460157 word types\n",
      "2019-12-04 18:45:38,417 : INFO : PROGRESS: at sentence #39540000, processed 618409441 words, keeping 467294 word types\n",
      "2019-12-04 18:45:38,679 : INFO : PROGRESS: at sentence #39550000, processed 618609175 words, keeping 474957 word types\n",
      "2019-12-04 18:45:38,924 : INFO : PROGRESS: at sentence #39560000, processed 618804936 words, keeping 481437 word types\n",
      "2019-12-04 18:45:39,178 : INFO : PROGRESS: at sentence #39570000, processed 619008410 words, keeping 487643 word types\n",
      "2019-12-04 18:45:39,432 : INFO : PROGRESS: at sentence #39580000, processed 619209258 words, keeping 494335 word types\n",
      "2019-12-04 18:45:39,896 : INFO : pruned out 322811 tokens with count <=60 (before 500001, after 177190)\n",
      "2019-12-04 18:45:39,949 : INFO : PROGRESS: at sentence #39590000, processed 619411615 words, keeping 179525 word types\n",
      "2019-12-04 18:45:40,209 : INFO : PROGRESS: at sentence #39600000, processed 619612409 words, keeping 189244 word types\n",
      "2019-12-04 18:45:40,468 : INFO : PROGRESS: at sentence #39610000, processed 619811220 words, keeping 199447 word types\n",
      "2019-12-04 18:45:40,722 : INFO : PROGRESS: at sentence #39620000, processed 620004824 words, keeping 209519 word types\n",
      "2019-12-04 18:45:40,984 : INFO : PROGRESS: at sentence #39630000, processed 620212016 words, keeping 219711 word types\n",
      "2019-12-04 18:45:41,234 : INFO : PROGRESS: at sentence #39640000, processed 620405141 words, keeping 229000 word types\n",
      "2019-12-04 18:45:41,499 : INFO : PROGRESS: at sentence #39650000, processed 620612333 words, keeping 238781 word types\n",
      "2019-12-04 18:45:41,753 : INFO : PROGRESS: at sentence #39660000, processed 620812163 words, keeping 248137 word types\n",
      "2019-12-04 18:45:42,002 : INFO : PROGRESS: at sentence #39670000, processed 621011094 words, keeping 257742 word types\n",
      "2019-12-04 18:45:42,262 : INFO : PROGRESS: at sentence #39680000, processed 621220218 words, keeping 266796 word types\n",
      "2019-12-04 18:45:42,531 : INFO : PROGRESS: at sentence #39690000, processed 621430573 words, keeping 275587 word types\n",
      "2019-12-04 18:45:42,783 : INFO : PROGRESS: at sentence #39700000, processed 621628815 words, keeping 284342 word types\n",
      "2019-12-04 18:45:43,034 : INFO : PROGRESS: at sentence #39710000, processed 621827464 words, keeping 293214 word types\n",
      "2019-12-04 18:45:43,279 : INFO : PROGRESS: at sentence #39720000, processed 622020094 words, keeping 301304 word types\n",
      "2019-12-04 18:45:43,541 : INFO : PROGRESS: at sentence #39730000, processed 622227412 words, keeping 311551 word types\n",
      "2019-12-04 18:45:43,797 : INFO : PROGRESS: at sentence #39740000, processed 622429723 words, keeping 320077 word types\n",
      "2019-12-04 18:45:44,067 : INFO : PROGRESS: at sentence #39750000, processed 622648136 words, keeping 328120 word types\n",
      "2019-12-04 18:45:44,322 : INFO : PROGRESS: at sentence #39760000, processed 622850685 words, keeping 336549 word types\n",
      "2019-12-04 18:45:44,581 : INFO : PROGRESS: at sentence #39770000, processed 623052657 words, keeping 344955 word types\n",
      "2019-12-04 18:45:44,840 : INFO : PROGRESS: at sentence #39780000, processed 623251419 words, keeping 353207 word types\n",
      "2019-12-04 18:45:45,100 : INFO : PROGRESS: at sentence #39790000, processed 623452177 words, keeping 361435 word types\n",
      "2019-12-04 18:45:45,361 : INFO : PROGRESS: at sentence #39800000, processed 623658441 words, keeping 369845 word types\n",
      "2019-12-04 18:45:45,622 : INFO : PROGRESS: at sentence #39810000, processed 623864763 words, keeping 378439 word types\n",
      "2019-12-04 18:45:45,870 : INFO : PROGRESS: at sentence #39820000, processed 624060367 words, keeping 385591 word types\n",
      "2019-12-04 18:45:46,124 : INFO : PROGRESS: at sentence #39830000, processed 624259880 words, keeping 393908 word types\n",
      "2019-12-04 18:45:46,377 : INFO : PROGRESS: at sentence #39840000, processed 624458055 words, keeping 401451 word types\n",
      "2019-12-04 18:45:46,638 : INFO : PROGRESS: at sentence #39850000, processed 624658939 words, keeping 408764 word types\n",
      "2019-12-04 18:45:46,890 : INFO : PROGRESS: at sentence #39860000, processed 624853627 words, keeping 416052 word types\n",
      "2019-12-04 18:45:47,156 : INFO : PROGRESS: at sentence #39870000, processed 625060633 words, keeping 424493 word types\n",
      "2019-12-04 18:45:47,416 : INFO : PROGRESS: at sentence #39880000, processed 625259403 words, keeping 432202 word types\n",
      "2019-12-04 18:45:47,669 : INFO : PROGRESS: at sentence #39890000, processed 625456512 words, keeping 439716 word types\n",
      "2019-12-04 18:45:47,922 : INFO : PROGRESS: at sentence #39900000, processed 625651240 words, keeping 446717 word types\n",
      "2019-12-04 18:45:48,181 : INFO : PROGRESS: at sentence #39910000, processed 625851277 words, keeping 453379 word types\n",
      "2019-12-04 18:45:48,442 : INFO : PROGRESS: at sentence #39920000, processed 626053643 words, keeping 461057 word types\n",
      "2019-12-04 18:45:48,715 : INFO : PROGRESS: at sentence #39930000, processed 626268506 words, keeping 468308 word types\n",
      "2019-12-04 18:45:48,978 : INFO : PROGRESS: at sentence #39940000, processed 626474551 words, keeping 474998 word types\n",
      "2019-12-04 18:45:49,240 : INFO : PROGRESS: at sentence #39950000, processed 626681301 words, keeping 482359 word types\n",
      "2019-12-04 18:45:49,490 : INFO : PROGRESS: at sentence #39960000, processed 626875895 words, keeping 489054 word types\n",
      "2019-12-04 18:45:49,760 : INFO : PROGRESS: at sentence #39970000, processed 627083334 words, keeping 495932 word types\n",
      "2019-12-04 18:45:50,199 : INFO : pruned out 323073 tokens with count <=61 (before 500001, after 176928)\n",
      "2019-12-04 18:45:50,302 : INFO : PROGRESS: at sentence #39980000, processed 627283810 words, keeping 180766 word types\n",
      "2019-12-04 18:45:50,561 : INFO : PROGRESS: at sentence #39990000, processed 627490471 words, keeping 190891 word types\n",
      "2019-12-04 18:45:50,812 : INFO : PROGRESS: at sentence #40000000, processed 627693129 words, keeping 201627 word types\n",
      "2019-12-04 18:45:51,064 : INFO : PROGRESS: at sentence #40010000, processed 627896591 words, keeping 211647 word types\n",
      "2019-12-04 18:45:51,321 : INFO : PROGRESS: at sentence #40020000, processed 628098666 words, keeping 222325 word types\n",
      "2019-12-04 18:45:51,578 : INFO : PROGRESS: at sentence #40030000, processed 628300877 words, keeping 231998 word types\n",
      "2019-12-04 18:45:51,834 : INFO : PROGRESS: at sentence #40040000, processed 628504222 words, keeping 241577 word types\n",
      "2019-12-04 18:45:52,075 : INFO : PROGRESS: at sentence #40050000, processed 628681628 words, keeping 249462 word types\n",
      "2019-12-04 18:45:52,331 : INFO : PROGRESS: at sentence #40060000, processed 628876816 words, keeping 258327 word types\n",
      "2019-12-04 18:45:52,594 : INFO : PROGRESS: at sentence #40070000, processed 629077835 words, keeping 267233 word types\n",
      "2019-12-04 18:45:52,846 : INFO : PROGRESS: at sentence #40080000, processed 629274629 words, keeping 275763 word types\n",
      "2019-12-04 18:45:53,098 : INFO : PROGRESS: at sentence #40090000, processed 629468501 words, keeping 284054 word types\n",
      "2019-12-04 18:45:53,360 : INFO : PROGRESS: at sentence #40100000, processed 629670412 words, keeping 293025 word types\n",
      "2019-12-04 18:45:53,614 : INFO : PROGRESS: at sentence #40110000, processed 629868077 words, keeping 301491 word types\n",
      "2019-12-04 18:45:53,877 : INFO : PROGRESS: at sentence #40120000, processed 630068757 words, keeping 309766 word types\n",
      "2019-12-04 18:45:54,135 : INFO : PROGRESS: at sentence #40130000, processed 630273793 words, keeping 318222 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:45:54,390 : INFO : PROGRESS: at sentence #40140000, processed 630474234 words, keeping 327477 word types\n",
      "2019-12-04 18:45:54,655 : INFO : PROGRESS: at sentence #40150000, processed 630679444 words, keeping 336409 word types\n",
      "2019-12-04 18:45:54,922 : INFO : PROGRESS: at sentence #40160000, processed 630892273 words, keeping 344356 word types\n",
      "2019-12-04 18:45:55,185 : INFO : PROGRESS: at sentence #40170000, processed 631101085 words, keeping 353112 word types\n",
      "2019-12-04 18:45:55,451 : INFO : PROGRESS: at sentence #40180000, processed 631304128 words, keeping 362640 word types\n",
      "2019-12-04 18:45:55,715 : INFO : PROGRESS: at sentence #40190000, processed 631511314 words, keeping 370809 word types\n",
      "2019-12-04 18:45:55,989 : INFO : PROGRESS: at sentence #40200000, processed 631732781 words, keeping 379182 word types\n",
      "2019-12-04 18:45:56,250 : INFO : PROGRESS: at sentence #40210000, processed 631945659 words, keeping 387943 word types\n",
      "2019-12-04 18:45:56,515 : INFO : PROGRESS: at sentence #40220000, processed 632143627 words, keeping 395091 word types\n",
      "2019-12-04 18:45:56,782 : INFO : PROGRESS: at sentence #40230000, processed 632346484 words, keeping 402985 word types\n",
      "2019-12-04 18:45:57,055 : INFO : PROGRESS: at sentence #40240000, processed 632564898 words, keeping 411729 word types\n",
      "2019-12-04 18:45:57,321 : INFO : PROGRESS: at sentence #40250000, processed 632777505 words, keeping 420612 word types\n",
      "2019-12-04 18:45:57,587 : INFO : PROGRESS: at sentence #40260000, processed 632980062 words, keeping 428523 word types\n",
      "2019-12-04 18:45:57,858 : INFO : PROGRESS: at sentence #40270000, processed 633186635 words, keeping 435639 word types\n",
      "2019-12-04 18:45:58,129 : INFO : PROGRESS: at sentence #40280000, processed 633385901 words, keeping 443161 word types\n",
      "2019-12-04 18:45:58,383 : INFO : PROGRESS: at sentence #40290000, processed 633579639 words, keeping 449905 word types\n",
      "2019-12-04 18:45:58,628 : INFO : PROGRESS: at sentence #40300000, processed 633774318 words, keeping 457030 word types\n",
      "2019-12-04 18:45:58,878 : INFO : PROGRESS: at sentence #40310000, processed 633974686 words, keeping 464285 word types\n",
      "2019-12-04 18:45:59,137 : INFO : PROGRESS: at sentence #40320000, processed 634182867 words, keeping 471406 word types\n",
      "2019-12-04 18:45:59,398 : INFO : PROGRESS: at sentence #40330000, processed 634383121 words, keeping 479175 word types\n",
      "2019-12-04 18:45:59,662 : INFO : PROGRESS: at sentence #40340000, processed 634587922 words, keeping 486392 word types\n",
      "2019-12-04 18:45:59,912 : INFO : PROGRESS: at sentence #40350000, processed 634788976 words, keeping 494036 word types\n",
      "2019-12-04 18:46:00,430 : INFO : pruned out 323311 tokens with count <=62 (before 500001, after 176690)\n",
      "2019-12-04 18:46:00,454 : INFO : PROGRESS: at sentence #40360000, processed 634990733 words, keeping 177724 word types\n",
      "2019-12-04 18:46:00,721 : INFO : PROGRESS: at sentence #40370000, processed 635191224 words, keeping 188100 word types\n",
      "2019-12-04 18:46:00,987 : INFO : PROGRESS: at sentence #40380000, processed 635392373 words, keeping 198652 word types\n",
      "2019-12-04 18:46:01,264 : INFO : PROGRESS: at sentence #40390000, processed 635600311 words, keeping 209851 word types\n",
      "2019-12-04 18:46:01,545 : INFO : PROGRESS: at sentence #40400000, processed 635814460 words, keeping 220813 word types\n",
      "2019-12-04 18:46:01,814 : INFO : PROGRESS: at sentence #40410000, processed 636017421 words, keeping 232084 word types\n",
      "2019-12-04 18:46:02,084 : INFO : PROGRESS: at sentence #40420000, processed 636220519 words, keeping 242295 word types\n",
      "2019-12-04 18:46:02,349 : INFO : PROGRESS: at sentence #40430000, processed 636420609 words, keeping 250928 word types\n",
      "2019-12-04 18:46:02,634 : INFO : PROGRESS: at sentence #40440000, processed 636629004 words, keeping 260262 word types\n",
      "2019-12-04 18:46:02,913 : INFO : PROGRESS: at sentence #40450000, processed 636828810 words, keeping 271407 word types\n",
      "2019-12-04 18:46:03,197 : INFO : PROGRESS: at sentence #40460000, processed 637027232 words, keeping 280004 word types\n",
      "2019-12-04 18:46:03,476 : INFO : PROGRESS: at sentence #40470000, processed 637235007 words, keeping 289534 word types\n",
      "2019-12-04 18:46:03,765 : INFO : PROGRESS: at sentence #40480000, processed 637449899 words, keeping 298297 word types\n",
      "2019-12-04 18:46:04,034 : INFO : PROGRESS: at sentence #40490000, processed 637656625 words, keeping 307676 word types\n",
      "2019-12-04 18:46:04,291 : INFO : PROGRESS: at sentence #40500000, processed 637855812 words, keeping 316249 word types\n",
      "2019-12-04 18:46:04,546 : INFO : PROGRESS: at sentence #40510000, processed 638054506 words, keeping 324815 word types\n",
      "2019-12-04 18:46:04,833 : INFO : PROGRESS: at sentence #40520000, processed 638273692 words, keeping 334429 word types\n",
      "2019-12-04 18:46:05,092 : INFO : PROGRESS: at sentence #40530000, processed 638466563 words, keeping 342423 word types\n",
      "2019-12-04 18:46:05,355 : INFO : PROGRESS: at sentence #40540000, processed 638671046 words, keeping 350314 word types\n",
      "2019-12-04 18:46:05,653 : INFO : PROGRESS: at sentence #40550000, processed 638888418 words, keeping 361439 word types\n",
      "2019-12-04 18:46:05,937 : INFO : PROGRESS: at sentence #40560000, processed 639087602 words, keeping 369433 word types\n",
      "2019-12-04 18:46:06,183 : INFO : PROGRESS: at sentence #40570000, processed 639290303 words, keeping 377280 word types\n",
      "2019-12-04 18:46:06,442 : INFO : PROGRESS: at sentence #40580000, processed 639505322 words, keeping 386705 word types\n",
      "2019-12-04 18:46:06,693 : INFO : PROGRESS: at sentence #40590000, processed 639711304 words, keeping 394920 word types\n",
      "2019-12-04 18:46:06,944 : INFO : PROGRESS: at sentence #40600000, processed 639926574 words, keeping 404232 word types\n",
      "2019-12-04 18:46:07,198 : INFO : PROGRESS: at sentence #40610000, processed 640142073 words, keeping 413825 word types\n",
      "2019-12-04 18:46:07,438 : INFO : PROGRESS: at sentence #40620000, processed 640342120 words, keeping 421595 word types\n",
      "2019-12-04 18:46:07,734 : INFO : PROGRESS: at sentence #40630000, processed 640543473 words, keeping 429671 word types\n",
      "2019-12-04 18:46:07,976 : INFO : PROGRESS: at sentence #40640000, processed 640749329 words, keeping 437215 word types\n",
      "2019-12-04 18:46:08,219 : INFO : PROGRESS: at sentence #40650000, processed 640952301 words, keeping 445161 word types\n",
      "2019-12-04 18:46:08,466 : INFO : PROGRESS: at sentence #40660000, processed 641162038 words, keeping 452190 word types\n",
      "2019-12-04 18:46:08,715 : INFO : PROGRESS: at sentence #40670000, processed 641360657 words, keeping 459294 word types\n",
      "2019-12-04 18:46:08,980 : INFO : PROGRESS: at sentence #40680000, processed 641571747 words, keeping 468207 word types\n",
      "2019-12-04 18:46:09,240 : INFO : PROGRESS: at sentence #40690000, processed 641774776 words, keeping 475630 word types\n",
      "2019-12-04 18:46:09,515 : INFO : PROGRESS: at sentence #40700000, processed 641997727 words, keeping 483876 word types\n",
      "2019-12-04 18:46:09,770 : INFO : PROGRESS: at sentence #40710000, processed 642209712 words, keeping 491036 word types\n",
      "2019-12-04 18:46:10,031 : INFO : PROGRESS: at sentence #40720000, processed 642423337 words, keeping 498307 word types\n",
      "2019-12-04 18:46:10,367 : INFO : pruned out 323579 tokens with count <=63 (before 500001, after 176422)\n",
      "2019-12-04 18:46:10,573 : INFO : PROGRESS: at sentence #40730000, processed 642622912 words, keeping 185238 word types\n",
      "2019-12-04 18:46:10,797 : INFO : PROGRESS: at sentence #40740000, processed 642795648 words, keeping 194858 word types\n",
      "2019-12-04 18:46:11,095 : INFO : PROGRESS: at sentence #40750000, processed 642992480 words, keeping 205118 word types\n",
      "2019-12-04 18:46:11,358 : INFO : PROGRESS: at sentence #40760000, processed 643200071 words, keeping 215674 word types\n",
      "2019-12-04 18:46:11,635 : INFO : PROGRESS: at sentence #40770000, processed 643417279 words, keeping 227652 word types\n",
      "2019-12-04 18:46:11,897 : INFO : PROGRESS: at sentence #40780000, processed 643625546 words, keeping 236853 word types\n",
      "2019-12-04 18:46:12,157 : INFO : PROGRESS: at sentence #40790000, processed 643832022 words, keeping 246435 word types\n",
      "2019-12-04 18:46:12,420 : INFO : PROGRESS: at sentence #40800000, processed 644043986 words, keeping 256288 word types\n",
      "2019-12-04 18:46:12,658 : INFO : PROGRESS: at sentence #40810000, processed 644238876 words, keeping 265475 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:46:12,911 : INFO : PROGRESS: at sentence #40820000, processed 644438157 words, keeping 274325 word types\n",
      "2019-12-04 18:46:13,169 : INFO : PROGRESS: at sentence #40830000, processed 644635669 words, keeping 282709 word types\n",
      "2019-12-04 18:46:13,423 : INFO : PROGRESS: at sentence #40840000, processed 644843439 words, keeping 291589 word types\n",
      "2019-12-04 18:46:13,681 : INFO : PROGRESS: at sentence #40850000, processed 645051651 words, keeping 301566 word types\n",
      "2019-12-04 18:46:13,930 : INFO : PROGRESS: at sentence #40860000, processed 645250120 words, keeping 309385 word types\n",
      "2019-12-04 18:46:14,176 : INFO : PROGRESS: at sentence #40870000, processed 645452809 words, keeping 319152 word types\n",
      "2019-12-04 18:46:14,423 : INFO : PROGRESS: at sentence #40880000, processed 645655184 words, keeping 327193 word types\n",
      "2019-12-04 18:46:14,724 : INFO : PROGRESS: at sentence #40890000, processed 645863158 words, keeping 338146 word types\n",
      "2019-12-04 18:46:14,977 : INFO : PROGRESS: at sentence #40900000, processed 646071107 words, keeping 346667 word types\n",
      "2019-12-04 18:46:15,234 : INFO : PROGRESS: at sentence #40910000, processed 646282906 words, keeping 355231 word types\n",
      "2019-12-04 18:46:15,478 : INFO : PROGRESS: at sentence #40920000, processed 646478167 words, keeping 362788 word types\n",
      "2019-12-04 18:46:15,725 : INFO : PROGRESS: at sentence #40930000, processed 646683499 words, keeping 371031 word types\n",
      "2019-12-04 18:46:15,974 : INFO : PROGRESS: at sentence #40940000, processed 646890548 words, keeping 379372 word types\n",
      "2019-12-04 18:46:16,228 : INFO : PROGRESS: at sentence #40950000, processed 647097355 words, keeping 388027 word types\n",
      "2019-12-04 18:46:16,479 : INFO : PROGRESS: at sentence #40960000, processed 647302944 words, keeping 395297 word types\n",
      "2019-12-04 18:46:16,733 : INFO : PROGRESS: at sentence #40970000, processed 647507461 words, keeping 402817 word types\n",
      "2019-12-04 18:46:16,977 : INFO : PROGRESS: at sentence #40980000, processed 647707581 words, keeping 410216 word types\n",
      "2019-12-04 18:46:17,235 : INFO : PROGRESS: at sentence #40990000, processed 647915619 words, keeping 418279 word types\n",
      "2019-12-04 18:46:17,522 : INFO : PROGRESS: at sentence #41000000, processed 648123127 words, keeping 425954 word types\n",
      "2019-12-04 18:46:17,777 : INFO : PROGRESS: at sentence #41010000, processed 648321387 words, keeping 433784 word types\n",
      "2019-12-04 18:46:18,070 : INFO : PROGRESS: at sentence #41020000, processed 648525935 words, keeping 440750 word types\n",
      "2019-12-04 18:46:18,331 : INFO : PROGRESS: at sentence #41030000, processed 648725533 words, keeping 448119 word types\n",
      "2019-12-04 18:46:18,605 : INFO : PROGRESS: at sentence #41040000, processed 648934664 words, keeping 456588 word types\n",
      "2019-12-04 18:46:18,880 : INFO : PROGRESS: at sentence #41050000, processed 649145515 words, keeping 464037 word types\n",
      "2019-12-04 18:46:19,209 : INFO : PROGRESS: at sentence #41060000, processed 649369374 words, keeping 471986 word types\n",
      "2019-12-04 18:46:19,479 : INFO : PROGRESS: at sentence #41070000, processed 649571803 words, keeping 479903 word types\n",
      "2019-12-04 18:46:19,767 : INFO : PROGRESS: at sentence #41080000, processed 649780552 words, keeping 488019 word types\n",
      "2019-12-04 18:46:20,040 : INFO : PROGRESS: at sentence #41090000, processed 649981618 words, keeping 495042 word types\n",
      "2019-12-04 18:46:20,542 : INFO : pruned out 323824 tokens with count <=64 (before 500001, after 176177)\n",
      "2019-12-04 18:46:20,621 : INFO : PROGRESS: at sentence #41100000, processed 650183551 words, keeping 179789 word types\n",
      "2019-12-04 18:46:20,900 : INFO : PROGRESS: at sentence #41110000, processed 650389712 words, keeping 191252 word types\n",
      "2019-12-04 18:46:21,181 : INFO : PROGRESS: at sentence #41120000, processed 650594253 words, keeping 201915 word types\n",
      "2019-12-04 18:46:21,458 : INFO : PROGRESS: at sentence #41130000, processed 650795846 words, keeping 211493 word types\n",
      "2019-12-04 18:46:21,733 : INFO : PROGRESS: at sentence #41140000, processed 650997156 words, keeping 220667 word types\n",
      "2019-12-04 18:46:21,984 : INFO : PROGRESS: at sentence #41150000, processed 651189512 words, keeping 230261 word types\n",
      "2019-12-04 18:46:22,250 : INFO : PROGRESS: at sentence #41160000, processed 651395624 words, keeping 239791 word types\n",
      "2019-12-04 18:46:22,520 : INFO : PROGRESS: at sentence #41170000, processed 651592158 words, keeping 248826 word types\n",
      "2019-12-04 18:46:22,808 : INFO : PROGRESS: at sentence #41180000, processed 651814110 words, keeping 261153 word types\n",
      "2019-12-04 18:46:23,090 : INFO : PROGRESS: at sentence #41190000, processed 652021304 words, keeping 270872 word types\n",
      "2019-12-04 18:46:23,377 : INFO : PROGRESS: at sentence #41200000, processed 652221168 words, keeping 280255 word types\n",
      "2019-12-04 18:46:23,650 : INFO : PROGRESS: at sentence #41210000, processed 652419968 words, keeping 288585 word types\n",
      "2019-12-04 18:46:23,919 : INFO : PROGRESS: at sentence #41220000, processed 652613854 words, keeping 296762 word types\n",
      "2019-12-04 18:46:24,199 : INFO : PROGRESS: at sentence #41230000, processed 652826078 words, keeping 305363 word types\n",
      "2019-12-04 18:46:24,492 : INFO : PROGRESS: at sentence #41240000, processed 653020631 words, keeping 313779 word types\n",
      "2019-12-04 18:46:24,772 : INFO : PROGRESS: at sentence #41250000, processed 653225357 words, keeping 322658 word types\n",
      "2019-12-04 18:46:25,053 : INFO : PROGRESS: at sentence #41260000, processed 653443706 words, keeping 333359 word types\n",
      "2019-12-04 18:46:25,328 : INFO : PROGRESS: at sentence #41270000, processed 653648835 words, keeping 341908 word types\n",
      "2019-12-04 18:46:25,589 : INFO : PROGRESS: at sentence #41280000, processed 653843299 words, keeping 349540 word types\n",
      "2019-12-04 18:46:25,865 : INFO : PROGRESS: at sentence #41290000, processed 654051599 words, keeping 358313 word types\n",
      "2019-12-04 18:46:26,121 : INFO : PROGRESS: at sentence #41300000, processed 654249073 words, keeping 366014 word types\n",
      "2019-12-04 18:46:26,381 : INFO : PROGRESS: at sentence #41310000, processed 654447466 words, keeping 373907 word types\n",
      "2019-12-04 18:46:26,636 : INFO : PROGRESS: at sentence #41320000, processed 654647078 words, keeping 381520 word types\n",
      "2019-12-04 18:46:26,896 : INFO : PROGRESS: at sentence #41330000, processed 654851296 words, keeping 389492 word types\n",
      "2019-12-04 18:46:27,170 : INFO : PROGRESS: at sentence #41340000, processed 655067071 words, keeping 398654 word types\n",
      "2019-12-04 18:46:27,425 : INFO : PROGRESS: at sentence #41350000, processed 655265493 words, keeping 406797 word types\n",
      "2019-12-04 18:46:27,678 : INFO : PROGRESS: at sentence #41360000, processed 655459588 words, keeping 413948 word types\n",
      "2019-12-04 18:46:27,936 : INFO : PROGRESS: at sentence #41370000, processed 655657046 words, keeping 420951 word types\n",
      "2019-12-04 18:46:28,247 : INFO : PROGRESS: at sentence #41380000, processed 655861193 words, keeping 428703 word types\n",
      "2019-12-04 18:46:28,516 : INFO : PROGRESS: at sentence #41390000, processed 656054581 words, keeping 436344 word types\n",
      "2019-12-04 18:46:28,769 : INFO : PROGRESS: at sentence #41400000, processed 656246433 words, keeping 443755 word types\n",
      "2019-12-04 18:46:29,032 : INFO : PROGRESS: at sentence #41410000, processed 656450576 words, keeping 451816 word types\n",
      "2019-12-04 18:46:29,289 : INFO : PROGRESS: at sentence #41420000, processed 656653736 words, keeping 459793 word types\n",
      "2019-12-04 18:46:29,558 : INFO : PROGRESS: at sentence #41430000, processed 656856679 words, keeping 467245 word types\n",
      "2019-12-04 18:46:29,813 : INFO : PROGRESS: at sentence #41440000, processed 657054921 words, keeping 475533 word types\n",
      "2019-12-04 18:46:30,079 : INFO : PROGRESS: at sentence #41450000, processed 657262165 words, keeping 483695 word types\n",
      "2019-12-04 18:46:30,357 : INFO : PROGRESS: at sentence #41460000, processed 657482246 words, keeping 491530 word types\n",
      "2019-12-04 18:46:30,615 : INFO : PROGRESS: at sentence #41470000, processed 657678657 words, keeping 498814 word types\n",
      "2019-12-04 18:46:30,952 : INFO : pruned out 324053 tokens with count <=65 (before 500003, after 175950)\n",
      "2019-12-04 18:46:31,184 : INFO : PROGRESS: at sentence #41480000, processed 657884622 words, keeping 185452 word types\n",
      "2019-12-04 18:46:31,456 : INFO : PROGRESS: at sentence #41490000, processed 658079789 words, keeping 195927 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:46:31,729 : INFO : PROGRESS: at sentence #41500000, processed 658286117 words, keeping 206755 word types\n",
      "2019-12-04 18:46:31,983 : INFO : PROGRESS: at sentence #41510000, processed 658486124 words, keeping 216023 word types\n",
      "2019-12-04 18:46:32,241 : INFO : PROGRESS: at sentence #41520000, processed 658683349 words, keeping 225454 word types\n",
      "2019-12-04 18:46:32,501 : INFO : PROGRESS: at sentence #41530000, processed 658885646 words, keeping 234959 word types\n",
      "2019-12-04 18:46:32,767 : INFO : PROGRESS: at sentence #41540000, processed 659092123 words, keeping 244206 word types\n",
      "2019-12-04 18:46:33,021 : INFO : PROGRESS: at sentence #41550000, processed 659288395 words, keeping 252392 word types\n",
      "2019-12-04 18:46:33,286 : INFO : PROGRESS: at sentence #41560000, processed 659486330 words, keeping 261448 word types\n",
      "2019-12-04 18:46:33,552 : INFO : PROGRESS: at sentence #41570000, processed 659690542 words, keeping 270079 word types\n",
      "2019-12-04 18:46:33,815 : INFO : PROGRESS: at sentence #41580000, processed 659893007 words, keeping 278458 word types\n",
      "2019-12-04 18:46:34,085 : INFO : PROGRESS: at sentence #41590000, processed 660093922 words, keeping 287822 word types\n",
      "2019-12-04 18:46:34,340 : INFO : PROGRESS: at sentence #41600000, processed 660294940 words, keeping 296611 word types\n",
      "2019-12-04 18:46:34,592 : INFO : PROGRESS: at sentence #41610000, processed 660487400 words, keeping 305314 word types\n",
      "2019-12-04 18:46:34,839 : INFO : PROGRESS: at sentence #41620000, processed 660678342 words, keeping 313316 word types\n",
      "2019-12-04 18:46:35,099 : INFO : PROGRESS: at sentence #41630000, processed 660880796 words, keeping 321994 word types\n",
      "2019-12-04 18:46:35,365 : INFO : PROGRESS: at sentence #41640000, processed 661094911 words, keeping 330631 word types\n",
      "2019-12-04 18:46:35,632 : INFO : PROGRESS: at sentence #41650000, processed 661303290 words, keeping 339525 word types\n",
      "2019-12-04 18:46:35,907 : INFO : PROGRESS: at sentence #41660000, processed 661518500 words, keeping 348176 word types\n",
      "2019-12-04 18:46:36,158 : INFO : PROGRESS: at sentence #41670000, processed 661715591 words, keeping 356099 word types\n",
      "2019-12-04 18:46:36,406 : INFO : PROGRESS: at sentence #41680000, processed 661909574 words, keeping 364346 word types\n",
      "2019-12-04 18:46:36,663 : INFO : PROGRESS: at sentence #41690000, processed 662111679 words, keeping 372170 word types\n",
      "2019-12-04 18:46:36,916 : INFO : PROGRESS: at sentence #41700000, processed 662311164 words, keeping 379548 word types\n",
      "2019-12-04 18:46:37,175 : INFO : PROGRESS: at sentence #41710000, processed 662507512 words, keeping 387412 word types\n",
      "2019-12-04 18:46:37,428 : INFO : PROGRESS: at sentence #41720000, processed 662706942 words, keeping 394983 word types\n",
      "2019-12-04 18:46:37,696 : INFO : PROGRESS: at sentence #41730000, processed 662922371 words, keeping 403627 word types\n",
      "2019-12-04 18:46:37,958 : INFO : PROGRESS: at sentence #41740000, processed 663127836 words, keeping 411083 word types\n",
      "2019-12-04 18:46:38,243 : INFO : PROGRESS: at sentence #41750000, processed 663338195 words, keeping 419147 word types\n",
      "2019-12-04 18:46:38,510 : INFO : PROGRESS: at sentence #41760000, processed 663542471 words, keeping 426237 word types\n",
      "2019-12-04 18:46:38,839 : INFO : PROGRESS: at sentence #41770000, processed 663752910 words, keeping 433685 word types\n",
      "2019-12-04 18:46:39,116 : INFO : PROGRESS: at sentence #41780000, processed 663963586 words, keeping 441000 word types\n",
      "2019-12-04 18:46:39,379 : INFO : PROGRESS: at sentence #41790000, processed 664167244 words, keeping 448585 word types\n",
      "2019-12-04 18:46:39,652 : INFO : PROGRESS: at sentence #41800000, processed 664373552 words, keeping 456186 word types\n",
      "2019-12-04 18:46:39,908 : INFO : PROGRESS: at sentence #41810000, processed 664569963 words, keeping 463523 word types\n",
      "2019-12-04 18:46:40,167 : INFO : PROGRESS: at sentence #41820000, processed 664766616 words, keeping 470716 word types\n",
      "2019-12-04 18:46:40,428 : INFO : PROGRESS: at sentence #41830000, processed 664967202 words, keeping 477209 word types\n",
      "2019-12-04 18:46:40,678 : INFO : PROGRESS: at sentence #41840000, processed 665159733 words, keeping 484162 word types\n",
      "2019-12-04 18:46:40,941 : INFO : PROGRESS: at sentence #41850000, processed 665356562 words, keeping 491405 word types\n",
      "2019-12-04 18:46:41,192 : INFO : PROGRESS: at sentence #41860000, processed 665552128 words, keeping 499119 word types\n",
      "2019-12-04 18:46:41,521 : INFO : pruned out 324273 tokens with count <=66 (before 500001, after 175728)\n",
      "2019-12-04 18:46:41,751 : INFO : PROGRESS: at sentence #41870000, processed 665755709 words, keeping 184944 word types\n",
      "2019-12-04 18:46:42,006 : INFO : PROGRESS: at sentence #41880000, processed 665951930 words, keeping 193977 word types\n",
      "2019-12-04 18:46:42,266 : INFO : PROGRESS: at sentence #41890000, processed 666151999 words, keeping 203977 word types\n",
      "2019-12-04 18:46:42,530 : INFO : PROGRESS: at sentence #41900000, processed 666354453 words, keeping 214015 word types\n",
      "2019-12-04 18:46:42,796 : INFO : PROGRESS: at sentence #41910000, processed 666554825 words, keeping 223543 word types\n",
      "2019-12-04 18:46:43,061 : INFO : PROGRESS: at sentence #41920000, processed 666753984 words, keeping 232787 word types\n",
      "2019-12-04 18:46:43,314 : INFO : PROGRESS: at sentence #41930000, processed 666947875 words, keeping 241661 word types\n",
      "2019-12-04 18:46:43,572 : INFO : PROGRESS: at sentence #41940000, processed 667146239 words, keeping 251233 word types\n",
      "2019-12-04 18:46:43,823 : INFO : PROGRESS: at sentence #41950000, processed 667339641 words, keeping 259354 word types\n",
      "2019-12-04 18:46:44,077 : INFO : PROGRESS: at sentence #41960000, processed 667533473 words, keeping 268474 word types\n",
      "2019-12-04 18:46:44,335 : INFO : PROGRESS: at sentence #41970000, processed 667731817 words, keeping 277553 word types\n",
      "2019-12-04 18:46:44,594 : INFO : PROGRESS: at sentence #41980000, processed 667932826 words, keeping 286092 word types\n",
      "2019-12-04 18:46:44,860 : INFO : PROGRESS: at sentence #41990000, processed 668142343 words, keeping 294283 word types\n",
      "2019-12-04 18:46:45,115 : INFO : PROGRESS: at sentence #42000000, processed 668340847 words, keeping 302276 word types\n",
      "2019-12-04 18:46:45,380 : INFO : PROGRESS: at sentence #42010000, processed 668544536 words, keeping 310874 word types\n",
      "2019-12-04 18:46:45,635 : INFO : PROGRESS: at sentence #42020000, processed 668740261 words, keeping 318562 word types\n",
      "2019-12-04 18:46:45,895 : INFO : PROGRESS: at sentence #42030000, processed 668943541 words, keeping 326632 word types\n",
      "2019-12-04 18:46:46,156 : INFO : PROGRESS: at sentence #42040000, processed 669142963 words, keeping 334666 word types\n",
      "2019-12-04 18:46:46,416 : INFO : PROGRESS: at sentence #42050000, processed 669344872 words, keeping 342676 word types\n",
      "2019-12-04 18:46:46,673 : INFO : PROGRESS: at sentence #42060000, processed 669543034 words, keeping 350548 word types\n",
      "2019-12-04 18:46:46,935 : INFO : PROGRESS: at sentence #42070000, processed 669746624 words, keeping 358835 word types\n",
      "2019-12-04 18:46:47,196 : INFO : PROGRESS: at sentence #42080000, processed 669949243 words, keeping 366747 word types\n",
      "2019-12-04 18:46:47,456 : INFO : PROGRESS: at sentence #42090000, processed 670149794 words, keeping 374543 word types\n",
      "2019-12-04 18:46:47,775 : INFO : PROGRESS: at sentence #42100000, processed 670373854 words, keeping 383503 word types\n",
      "2019-12-04 18:46:48,027 : INFO : PROGRESS: at sentence #42110000, processed 670567909 words, keeping 391596 word types\n",
      "2019-12-04 18:46:48,260 : INFO : PROGRESS: at sentence #42120000, processed 670745506 words, keeping 399119 word types\n",
      "2019-12-04 18:46:48,527 : INFO : PROGRESS: at sentence #42130000, processed 670948818 words, keeping 406878 word types\n",
      "2019-12-04 18:46:48,822 : INFO : PROGRESS: at sentence #42140000, processed 671180173 words, keeping 415106 word types\n",
      "2019-12-04 18:46:49,081 : INFO : PROGRESS: at sentence #42150000, processed 671376114 words, keeping 421856 word types\n",
      "2019-12-04 18:46:49,340 : INFO : PROGRESS: at sentence #42160000, processed 671575145 words, keeping 429636 word types\n",
      "2019-12-04 18:46:49,593 : INFO : PROGRESS: at sentence #42170000, processed 671770601 words, keeping 437667 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:46:49,846 : INFO : PROGRESS: at sentence #42180000, processed 671968525 words, keeping 444964 word types\n",
      "2019-12-04 18:46:50,101 : INFO : PROGRESS: at sentence #42190000, processed 672166271 words, keeping 451964 word types\n",
      "2019-12-04 18:46:50,357 : INFO : PROGRESS: at sentence #42200000, processed 672362866 words, keeping 458858 word types\n",
      "2019-12-04 18:46:50,622 : INFO : PROGRESS: at sentence #42210000, processed 672565113 words, keeping 467574 word types\n",
      "2019-12-04 18:46:50,891 : INFO : PROGRESS: at sentence #42220000, processed 672771270 words, keeping 475341 word types\n",
      "2019-12-04 18:46:51,163 : INFO : PROGRESS: at sentence #42230000, processed 672979871 words, keeping 482257 word types\n",
      "2019-12-04 18:46:51,435 : INFO : PROGRESS: at sentence #42240000, processed 673186671 words, keeping 489833 word types\n",
      "2019-12-04 18:46:51,695 : INFO : PROGRESS: at sentence #42250000, processed 673384662 words, keeping 496190 word types\n",
      "2019-12-04 18:46:52,137 : INFO : pruned out 324468 tokens with count <=67 (before 500004, after 175536)\n",
      "2019-12-04 18:46:52,214 : INFO : PROGRESS: at sentence #42260000, processed 673546124 words, keeping 182920 word types\n",
      "2019-12-04 18:46:52,484 : INFO : PROGRESS: at sentence #42270000, processed 673751722 words, keeping 193501 word types\n",
      "2019-12-04 18:46:52,744 : INFO : PROGRESS: at sentence #42280000, processed 673951367 words, keeping 203793 word types\n",
      "2019-12-04 18:46:53,010 : INFO : PROGRESS: at sentence #42290000, processed 674155979 words, keeping 214024 word types\n",
      "2019-12-04 18:46:53,286 : INFO : PROGRESS: at sentence #42300000, processed 674368986 words, keeping 225278 word types\n",
      "2019-12-04 18:46:53,554 : INFO : PROGRESS: at sentence #42310000, processed 674569022 words, keeping 235560 word types\n",
      "2019-12-04 18:46:53,823 : INFO : PROGRESS: at sentence #42320000, processed 674774890 words, keeping 244845 word types\n",
      "2019-12-04 18:46:54,108 : INFO : PROGRESS: at sentence #42330000, processed 674995672 words, keeping 256217 word types\n",
      "2019-12-04 18:46:54,364 : INFO : PROGRESS: at sentence #42340000, processed 675190093 words, keeping 264684 word types\n",
      "2019-12-04 18:46:54,635 : INFO : PROGRESS: at sentence #42350000, processed 675398012 words, keeping 274289 word types\n",
      "2019-12-04 18:46:54,928 : INFO : PROGRESS: at sentence #42360000, processed 675620573 words, keeping 285357 word types\n",
      "2019-12-04 18:46:55,194 : INFO : PROGRESS: at sentence #42370000, processed 675829112 words, keeping 294346 word types\n",
      "2019-12-04 18:46:55,449 : INFO : PROGRESS: at sentence #42380000, processed 676023540 words, keeping 302120 word types\n",
      "2019-12-04 18:46:55,711 : INFO : PROGRESS: at sentence #42390000, processed 676225797 words, keeping 310214 word types\n",
      "2019-12-04 18:46:55,989 : INFO : PROGRESS: at sentence #42400000, processed 676432593 words, keeping 318285 word types\n",
      "2019-12-04 18:46:56,241 : INFO : PROGRESS: at sentence #42410000, processed 676632194 words, keeping 326187 word types\n",
      "2019-12-04 18:46:56,489 : INFO : PROGRESS: at sentence #42420000, processed 676829989 words, keeping 334197 word types\n",
      "2019-12-04 18:46:56,749 : INFO : PROGRESS: at sentence #42430000, processed 677039436 words, keeping 342523 word types\n",
      "2019-12-04 18:46:56,998 : INFO : PROGRESS: at sentence #42440000, processed 677238362 words, keeping 349731 word types\n",
      "2019-12-04 18:46:57,247 : INFO : PROGRESS: at sentence #42450000, processed 677438003 words, keeping 357010 word types\n",
      "2019-12-04 18:46:57,512 : INFO : PROGRESS: at sentence #42460000, processed 677645582 words, keeping 364554 word types\n",
      "2019-12-04 18:46:57,787 : INFO : PROGRESS: at sentence #42470000, processed 677848603 words, keeping 372346 word types\n",
      "2019-12-04 18:46:58,064 : INFO : PROGRESS: at sentence #42480000, processed 678058660 words, keeping 379941 word types\n",
      "2019-12-04 18:46:58,329 : INFO : PROGRESS: at sentence #42490000, processed 678265690 words, keeping 387616 word types\n",
      "2019-12-04 18:46:58,610 : INFO : PROGRESS: at sentence #42500000, processed 678470964 words, keeping 395404 word types\n",
      "2019-12-04 18:46:58,871 : INFO : PROGRESS: at sentence #42510000, processed 678674223 words, keeping 403041 word types\n",
      "2019-12-04 18:46:59,141 : INFO : PROGRESS: at sentence #42520000, processed 678886052 words, keeping 411520 word types\n",
      "2019-12-04 18:46:59,454 : INFO : PROGRESS: at sentence #42530000, processed 679118298 words, keeping 421596 word types\n",
      "2019-12-04 18:46:59,714 : INFO : PROGRESS: at sentence #42540000, processed 679321294 words, keeping 428472 word types\n",
      "2019-12-04 18:46:59,936 : INFO : PROGRESS: at sentence #42550000, processed 679492412 words, keeping 434829 word types\n",
      "2019-12-04 18:47:00,202 : INFO : PROGRESS: at sentence #42560000, processed 679703846 words, keeping 442559 word types\n",
      "2019-12-04 18:47:00,465 : INFO : PROGRESS: at sentence #42570000, processed 679905556 words, keeping 450175 word types\n",
      "2019-12-04 18:47:00,731 : INFO : PROGRESS: at sentence #42580000, processed 680114768 words, keeping 458970 word types\n",
      "2019-12-04 18:47:00,983 : INFO : PROGRESS: at sentence #42590000, processed 680310922 words, keeping 466332 word types\n",
      "2019-12-04 18:47:01,245 : INFO : PROGRESS: at sentence #42600000, processed 680515054 words, keeping 473798 word types\n",
      "2019-12-04 18:47:01,505 : INFO : PROGRESS: at sentence #42610000, processed 680718412 words, keeping 480810 word types\n",
      "2019-12-04 18:47:01,756 : INFO : PROGRESS: at sentence #42620000, processed 680915056 words, keeping 489369 word types\n",
      "2019-12-04 18:47:02,014 : INFO : PROGRESS: at sentence #42630000, processed 681116499 words, keeping 496075 word types\n",
      "2019-12-04 18:47:02,446 : INFO : pruned out 324664 tokens with count <=68 (before 500001, after 175337)\n",
      "2019-12-04 18:47:02,556 : INFO : PROGRESS: at sentence #42640000, processed 681309252 words, keeping 179208 word types\n",
      "2019-12-04 18:47:02,831 : INFO : PROGRESS: at sentence #42650000, processed 681507438 words, keeping 189259 word types\n",
      "2019-12-04 18:47:03,093 : INFO : PROGRESS: at sentence #42660000, processed 681708025 words, keeping 199234 word types\n",
      "2019-12-04 18:47:03,350 : INFO : PROGRESS: at sentence #42670000, processed 681903551 words, keeping 209181 word types\n",
      "2019-12-04 18:47:03,610 : INFO : PROGRESS: at sentence #42680000, processed 682100172 words, keeping 218939 word types\n",
      "2019-12-04 18:47:03,911 : INFO : PROGRESS: at sentence #42690000, processed 682303856 words, keeping 228470 word types\n",
      "2019-12-04 18:47:04,176 : INFO : PROGRESS: at sentence #42700000, processed 682507555 words, keeping 237719 word types\n",
      "2019-12-04 18:47:04,437 : INFO : PROGRESS: at sentence #42710000, processed 682710075 words, keeping 247729 word types\n",
      "2019-12-04 18:47:04,719 : INFO : PROGRESS: at sentence #42720000, processed 682910417 words, keeping 257393 word types\n",
      "2019-12-04 18:47:04,982 : INFO : PROGRESS: at sentence #42730000, processed 683111757 words, keeping 266763 word types\n",
      "2019-12-04 18:47:05,237 : INFO : PROGRESS: at sentence #42740000, processed 683310976 words, keeping 275230 word types\n",
      "2019-12-04 18:47:05,501 : INFO : PROGRESS: at sentence #42750000, processed 683513211 words, keeping 284548 word types\n",
      "2019-12-04 18:47:05,757 : INFO : PROGRESS: at sentence #42760000, processed 683710345 words, keeping 293318 word types\n",
      "2019-12-04 18:47:06,034 : INFO : PROGRESS: at sentence #42770000, processed 683929408 words, keeping 304058 word types\n",
      "2019-12-04 18:47:06,306 : INFO : PROGRESS: at sentence #42780000, processed 684141556 words, keeping 312977 word types\n",
      "2019-12-04 18:47:06,573 : INFO : PROGRESS: at sentence #42790000, processed 684343261 words, keeping 321181 word types\n",
      "2019-12-04 18:47:06,835 : INFO : PROGRESS: at sentence #42800000, processed 684545364 words, keeping 329204 word types\n",
      "2019-12-04 18:47:07,099 : INFO : PROGRESS: at sentence #42810000, processed 684755368 words, keeping 337255 word types\n",
      "2019-12-04 18:47:07,346 : INFO : PROGRESS: at sentence #42820000, processed 684950183 words, keeping 344252 word types\n",
      "2019-12-04 18:47:07,601 : INFO : PROGRESS: at sentence #42830000, processed 685150406 words, keeping 352338 word types\n",
      "2019-12-04 18:47:07,864 : INFO : PROGRESS: at sentence #42840000, processed 685355740 words, keeping 360766 word types\n",
      "2019-12-04 18:47:08,112 : INFO : PROGRESS: at sentence #42850000, processed 685551531 words, keeping 368371 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:47:08,375 : INFO : PROGRESS: at sentence #42860000, processed 685762351 words, keeping 377081 word types\n",
      "2019-12-04 18:47:08,634 : INFO : PROGRESS: at sentence #42870000, processed 685962502 words, keeping 384490 word types\n",
      "2019-12-04 18:47:08,892 : INFO : PROGRESS: at sentence #42880000, processed 686160423 words, keeping 392165 word types\n",
      "2019-12-04 18:47:09,159 : INFO : PROGRESS: at sentence #42890000, processed 686367642 words, keeping 400554 word types\n",
      "2019-12-04 18:47:09,452 : INFO : PROGRESS: at sentence #42900000, processed 686599546 words, keeping 410280 word types\n",
      "2019-12-04 18:47:09,718 : INFO : PROGRESS: at sentence #42910000, processed 686804754 words, keeping 417998 word types\n",
      "2019-12-04 18:47:10,021 : INFO : PROGRESS: at sentence #42920000, processed 687002626 words, keeping 424810 word types\n",
      "2019-12-04 18:47:10,280 : INFO : PROGRESS: at sentence #42930000, processed 687214322 words, keeping 432702 word types\n",
      "2019-12-04 18:47:10,541 : INFO : PROGRESS: at sentence #42940000, processed 687419279 words, keeping 439987 word types\n",
      "2019-12-04 18:47:10,789 : INFO : PROGRESS: at sentence #42950000, processed 687614040 words, keeping 446893 word types\n",
      "2019-12-04 18:47:11,060 : INFO : PROGRESS: at sentence #42960000, processed 687827748 words, keeping 454949 word types\n",
      "2019-12-04 18:47:11,334 : INFO : PROGRESS: at sentence #42970000, processed 688046056 words, keeping 462512 word types\n",
      "2019-12-04 18:47:11,596 : INFO : PROGRESS: at sentence #42980000, processed 688250410 words, keeping 469374 word types\n",
      "2019-12-04 18:47:11,850 : INFO : PROGRESS: at sentence #42990000, processed 688450927 words, keeping 476238 word types\n",
      "2019-12-04 18:47:12,123 : INFO : PROGRESS: at sentence #43000000, processed 688663510 words, keeping 484327 word types\n",
      "2019-12-04 18:47:12,383 : INFO : PROGRESS: at sentence #43010000, processed 688867074 words, keeping 492057 word types\n",
      "2019-12-04 18:47:12,645 : INFO : PROGRESS: at sentence #43020000, processed 689070757 words, keeping 498565 word types\n",
      "2019-12-04 18:47:12,964 : INFO : pruned out 324856 tokens with count <=69 (before 500001, after 175145)\n",
      "2019-12-04 18:47:13,184 : INFO : PROGRESS: at sentence #43030000, processed 689278145 words, keeping 185343 word types\n",
      "2019-12-04 18:47:13,442 : INFO : PROGRESS: at sentence #43040000, processed 689479651 words, keeping 196117 word types\n",
      "2019-12-04 18:47:13,693 : INFO : PROGRESS: at sentence #43050000, processed 689678856 words, keeping 205942 word types\n",
      "2019-12-04 18:47:13,950 : INFO : PROGRESS: at sentence #43060000, processed 689880658 words, keeping 215881 word types\n",
      "2019-12-04 18:47:14,207 : INFO : PROGRESS: at sentence #43070000, processed 690087152 words, keeping 225074 word types\n",
      "2019-12-04 18:47:14,503 : INFO : PROGRESS: at sentence #43080000, processed 690293138 words, keeping 234776 word types\n",
      "2019-12-04 18:47:14,764 : INFO : PROGRESS: at sentence #43090000, processed 690497848 words, keeping 243679 word types\n",
      "2019-12-04 18:47:15,009 : INFO : PROGRESS: at sentence #43100000, processed 690693825 words, keeping 252182 word types\n",
      "2019-12-04 18:47:15,295 : INFO : PROGRESS: at sentence #43110000, processed 690908522 words, keeping 261407 word types\n",
      "2019-12-04 18:47:15,550 : INFO : PROGRESS: at sentence #43120000, processed 691110404 words, keeping 270381 word types\n",
      "2019-12-04 18:47:15,805 : INFO : PROGRESS: at sentence #43130000, processed 691310511 words, keeping 279429 word types\n",
      "2019-12-04 18:47:16,060 : INFO : PROGRESS: at sentence #43140000, processed 691512306 words, keeping 288012 word types\n",
      "2019-12-04 18:47:16,318 : INFO : PROGRESS: at sentence #43150000, processed 691711372 words, keeping 296431 word types\n",
      "2019-12-04 18:47:16,579 : INFO : PROGRESS: at sentence #43160000, processed 691916977 words, keeping 305041 word types\n",
      "2019-12-04 18:47:16,837 : INFO : PROGRESS: at sentence #43170000, processed 692118053 words, keeping 313450 word types\n",
      "2019-12-04 18:47:17,099 : INFO : PROGRESS: at sentence #43180000, processed 692323795 words, keeping 322472 word types\n",
      "2019-12-04 18:47:17,341 : INFO : PROGRESS: at sentence #43190000, processed 692514840 words, keeping 329857 word types\n",
      "2019-12-04 18:47:17,594 : INFO : PROGRESS: at sentence #43200000, processed 692715089 words, keeping 338029 word types\n",
      "2019-12-04 18:47:17,844 : INFO : PROGRESS: at sentence #43210000, processed 692913582 words, keeping 345690 word types\n",
      "2019-12-04 18:47:18,106 : INFO : PROGRESS: at sentence #43220000, processed 693118177 words, keeping 353599 word types\n",
      "2019-12-04 18:47:18,361 : INFO : PROGRESS: at sentence #43230000, processed 693315166 words, keeping 361138 word types\n",
      "2019-12-04 18:47:18,626 : INFO : PROGRESS: at sentence #43240000, processed 693525525 words, keeping 370389 word types\n",
      "2019-12-04 18:47:18,880 : INFO : PROGRESS: at sentence #43250000, processed 693724021 words, keeping 377710 word types\n",
      "2019-12-04 18:47:19,116 : INFO : PROGRESS: at sentence #43260000, processed 693912982 words, keeping 384661 word types\n",
      "2019-12-04 18:47:19,367 : INFO : PROGRESS: at sentence #43270000, processed 694117572 words, keeping 392586 word types\n",
      "2019-12-04 18:47:19,615 : INFO : PROGRESS: at sentence #43280000, processed 694312905 words, keeping 399620 word types\n",
      "2019-12-04 18:47:19,857 : INFO : PROGRESS: at sentence #43290000, processed 694505427 words, keeping 405907 word types\n",
      "2019-12-04 18:47:20,102 : INFO : PROGRESS: at sentence #43300000, processed 694705683 words, keeping 412994 word types\n",
      "2019-12-04 18:47:20,351 : INFO : PROGRESS: at sentence #43310000, processed 694910113 words, keeping 420415 word types\n",
      "2019-12-04 18:47:20,604 : INFO : PROGRESS: at sentence #43320000, processed 695112793 words, keeping 427686 word types\n",
      "2019-12-04 18:47:20,857 : INFO : PROGRESS: at sentence #43330000, processed 695311364 words, keeping 434220 word types\n",
      "2019-12-04 18:47:21,109 : INFO : PROGRESS: at sentence #43340000, processed 695515922 words, keeping 441126 word types\n",
      "2019-12-04 18:47:21,365 : INFO : PROGRESS: at sentence #43350000, processed 695718395 words, keeping 448667 word types\n",
      "2019-12-04 18:47:21,617 : INFO : PROGRESS: at sentence #43360000, processed 695913555 words, keeping 456184 word types\n",
      "2019-12-04 18:47:21,872 : INFO : PROGRESS: at sentence #43370000, processed 696111191 words, keeping 462876 word types\n",
      "2019-12-04 18:47:22,128 : INFO : PROGRESS: at sentence #43380000, processed 696313306 words, keeping 469345 word types\n",
      "2019-12-04 18:47:22,391 : INFO : PROGRESS: at sentence #43390000, processed 696517149 words, keeping 476493 word types\n",
      "2019-12-04 18:47:22,654 : INFO : PROGRESS: at sentence #43400000, processed 696717301 words, keeping 483802 word types\n",
      "2019-12-04 18:47:22,917 : INFO : PROGRESS: at sentence #43410000, processed 696919189 words, keeping 490111 word types\n",
      "2019-12-04 18:47:23,191 : INFO : PROGRESS: at sentence #43420000, processed 697120463 words, keeping 497406 word types\n",
      "2019-12-04 18:47:23,570 : INFO : pruned out 325070 tokens with count <=70 (before 500003, after 174933)\n",
      "2019-12-04 18:47:23,742 : INFO : PROGRESS: at sentence #43430000, processed 697324405 words, keeping 181994 word types\n",
      "2019-12-04 18:47:24,010 : INFO : PROGRESS: at sentence #43440000, processed 697531741 words, keeping 192412 word types\n",
      "2019-12-04 18:47:24,278 : INFO : PROGRESS: at sentence #43450000, processed 697730574 words, keeping 202032 word types\n",
      "2019-12-04 18:47:24,559 : INFO : PROGRESS: at sentence #43460000, processed 697930607 words, keeping 211751 word types\n",
      "2019-12-04 18:47:24,820 : INFO : PROGRESS: at sentence #43470000, processed 698130948 words, keeping 221786 word types\n",
      "2019-12-04 18:47:25,096 : INFO : PROGRESS: at sentence #43480000, processed 698345110 words, keeping 232432 word types\n",
      "2019-12-04 18:47:25,361 : INFO : PROGRESS: at sentence #43490000, processed 698550952 words, keeping 242432 word types\n",
      "2019-12-04 18:47:25,625 : INFO : PROGRESS: at sentence #43500000, processed 698759091 words, keeping 251305 word types\n",
      "2019-12-04 18:47:25,888 : INFO : PROGRESS: at sentence #43510000, processed 698964642 words, keeping 260607 word types\n",
      "2019-12-04 18:47:26,149 : INFO : PROGRESS: at sentence #43520000, processed 699168481 words, keeping 269844 word types\n",
      "2019-12-04 18:47:26,463 : INFO : PROGRESS: at sentence #43530000, processed 699415618 words, keeping 279372 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:47:26,722 : INFO : PROGRESS: at sentence #43540000, processed 699615359 words, keeping 287875 word types\n",
      "2019-12-04 18:47:26,994 : INFO : PROGRESS: at sentence #43550000, processed 699826583 words, keeping 297237 word types\n",
      "2019-12-04 18:47:27,250 : INFO : PROGRESS: at sentence #43560000, processed 700029324 words, keeping 306704 word types\n",
      "2019-12-04 18:47:27,520 : INFO : PROGRESS: at sentence #43570000, processed 700232250 words, keeping 315112 word types\n",
      "2019-12-04 18:47:27,786 : INFO : PROGRESS: at sentence #43580000, processed 700435889 words, keeping 322711 word types\n",
      "2019-12-04 18:47:28,038 : INFO : PROGRESS: at sentence #43590000, processed 700630958 words, keeping 330629 word types\n",
      "2019-12-04 18:47:28,292 : INFO : PROGRESS: at sentence #43600000, processed 700828600 words, keeping 338530 word types\n",
      "2019-12-04 18:47:28,559 : INFO : PROGRESS: at sentence #43610000, processed 701035508 words, keeping 347342 word types\n",
      "2019-12-04 18:47:28,825 : INFO : PROGRESS: at sentence #43620000, processed 701241527 words, keeping 355834 word types\n",
      "2019-12-04 18:47:29,089 : INFO : PROGRESS: at sentence #43630000, processed 701447847 words, keeping 364292 word types\n",
      "2019-12-04 18:47:29,350 : INFO : PROGRESS: at sentence #43640000, processed 701648052 words, keeping 372112 word types\n",
      "2019-12-04 18:47:29,614 : INFO : PROGRESS: at sentence #43650000, processed 701852221 words, keeping 380082 word types\n",
      "2019-12-04 18:47:29,869 : INFO : PROGRESS: at sentence #43660000, processed 702053132 words, keeping 387356 word types\n",
      "2019-12-04 18:47:30,129 : INFO : PROGRESS: at sentence #43670000, processed 702255598 words, keeping 395808 word types\n",
      "2019-12-04 18:47:30,377 : INFO : PROGRESS: at sentence #43680000, processed 702450145 words, keeping 402808 word types\n",
      "2019-12-04 18:47:30,640 : INFO : PROGRESS: at sentence #43690000, processed 702654449 words, keeping 411775 word types\n",
      "2019-12-04 18:47:30,905 : INFO : PROGRESS: at sentence #43700000, processed 702854360 words, keeping 419000 word types\n",
      "2019-12-04 18:47:31,173 : INFO : PROGRESS: at sentence #43710000, processed 703066067 words, keeping 429789 word types\n",
      "2019-12-04 18:47:31,424 : INFO : PROGRESS: at sentence #43720000, processed 703260719 words, keeping 436455 word types\n",
      "2019-12-04 18:47:31,677 : INFO : PROGRESS: at sentence #43730000, processed 703449098 words, keeping 442734 word types\n",
      "2019-12-04 18:47:31,948 : INFO : PROGRESS: at sentence #43740000, processed 703643874 words, keeping 449379 word types\n",
      "2019-12-04 18:47:32,204 : INFO : PROGRESS: at sentence #43750000, processed 703843229 words, keeping 456127 word types\n",
      "2019-12-04 18:47:32,471 : INFO : PROGRESS: at sentence #43760000, processed 704046657 words, keeping 463473 word types\n",
      "2019-12-04 18:47:32,730 : INFO : PROGRESS: at sentence #43770000, processed 704251006 words, keeping 470414 word types\n",
      "2019-12-04 18:47:32,998 : INFO : PROGRESS: at sentence #43780000, processed 704463050 words, keeping 477422 word types\n",
      "2019-12-04 18:47:33,273 : INFO : PROGRESS: at sentence #43790000, processed 704681690 words, keeping 485941 word types\n",
      "2019-12-04 18:47:33,535 : INFO : PROGRESS: at sentence #43800000, processed 704882258 words, keeping 492899 word types\n",
      "2019-12-04 18:47:34,076 : INFO : pruned out 325235 tokens with count <=71 (before 500001, after 174766)\n",
      "2019-12-04 18:47:34,097 : INFO : PROGRESS: at sentence #43810000, processed 705087328 words, keeping 175469 word types\n",
      "2019-12-04 18:47:34,366 : INFO : PROGRESS: at sentence #43820000, processed 705293663 words, keeping 185877 word types\n",
      "2019-12-04 18:47:34,630 : INFO : PROGRESS: at sentence #43830000, processed 705501918 words, keeping 195710 word types\n",
      "2019-12-04 18:47:34,888 : INFO : PROGRESS: at sentence #43840000, processed 705696279 words, keeping 205131 word types\n",
      "2019-12-04 18:47:35,163 : INFO : PROGRESS: at sentence #43850000, processed 705896868 words, keeping 214569 word types\n",
      "2019-12-04 18:47:35,443 : INFO : PROGRESS: at sentence #43860000, processed 706094565 words, keeping 224345 word types\n",
      "2019-12-04 18:47:35,706 : INFO : PROGRESS: at sentence #43870000, processed 706296715 words, keeping 234298 word types\n",
      "2019-12-04 18:47:35,973 : INFO : PROGRESS: at sentence #43880000, processed 706506226 words, keeping 244711 word types\n",
      "2019-12-04 18:47:36,230 : INFO : PROGRESS: at sentence #43890000, processed 706708238 words, keeping 254080 word types\n",
      "2019-12-04 18:47:36,490 : INFO : PROGRESS: at sentence #43900000, processed 706911487 words, keeping 262984 word types\n",
      "2019-12-04 18:47:36,752 : INFO : PROGRESS: at sentence #43910000, processed 707115075 words, keeping 271744 word types\n",
      "2019-12-04 18:47:37,019 : INFO : PROGRESS: at sentence #43920000, processed 707324745 words, keeping 280560 word types\n",
      "2019-12-04 18:47:37,294 : INFO : PROGRESS: at sentence #43930000, processed 707537553 words, keeping 290574 word types\n",
      "2019-12-04 18:47:37,554 : INFO : PROGRESS: at sentence #43940000, processed 707739311 words, keeping 299318 word types\n",
      "2019-12-04 18:47:37,820 : INFO : PROGRESS: at sentence #43950000, processed 707945592 words, keeping 308110 word types\n",
      "2019-12-04 18:47:38,081 : INFO : PROGRESS: at sentence #43960000, processed 708148719 words, keeping 316704 word types\n",
      "2019-12-04 18:47:38,335 : INFO : PROGRESS: at sentence #43970000, processed 708346134 words, keeping 324241 word types\n",
      "2019-12-04 18:47:38,576 : INFO : PROGRESS: at sentence #43980000, processed 708532273 words, keeping 331907 word types\n",
      "2019-12-04 18:47:38,829 : INFO : PROGRESS: at sentence #43990000, processed 708727754 words, keeping 339571 word types\n",
      "2019-12-04 18:47:39,078 : INFO : PROGRESS: at sentence #44000000, processed 708919200 words, keeping 347168 word types\n",
      "2019-12-04 18:47:39,341 : INFO : PROGRESS: at sentence #44010000, processed 709122965 words, keeping 355250 word types\n",
      "2019-12-04 18:47:39,611 : INFO : PROGRESS: at sentence #44020000, processed 709325529 words, keeping 362955 word types\n",
      "2019-12-04 18:47:39,873 : INFO : PROGRESS: at sentence #44030000, processed 709523292 words, keeping 370843 word types\n",
      "2019-12-04 18:47:40,125 : INFO : PROGRESS: at sentence #44040000, processed 709719203 words, keeping 378737 word types\n",
      "2019-12-04 18:47:40,393 : INFO : PROGRESS: at sentence #44050000, processed 709929385 words, keeping 388074 word types\n",
      "2019-12-04 18:47:40,649 : INFO : PROGRESS: at sentence #44060000, processed 710130761 words, keeping 395323 word types\n",
      "2019-12-04 18:47:40,908 : INFO : PROGRESS: at sentence #44070000, processed 710334430 words, keeping 402473 word types\n",
      "2019-12-04 18:47:41,181 : INFO : PROGRESS: at sentence #44080000, processed 710552060 words, keeping 410727 word types\n",
      "2019-12-04 18:47:41,447 : INFO : PROGRESS: at sentence #44090000, processed 710760061 words, keeping 418522 word types\n",
      "2019-12-04 18:47:41,750 : INFO : PROGRESS: at sentence #44100000, processed 710950220 words, keeping 425454 word types\n",
      "2019-12-04 18:47:42,018 : INFO : PROGRESS: at sentence #44110000, processed 711159054 words, keeping 432550 word types\n",
      "2019-12-04 18:47:42,271 : INFO : PROGRESS: at sentence #44120000, processed 711355471 words, keeping 439036 word types\n",
      "2019-12-04 18:47:42,526 : INFO : PROGRESS: at sentence #44130000, processed 711552173 words, keeping 446296 word types\n",
      "2019-12-04 18:47:42,802 : INFO : PROGRESS: at sentence #44140000, processed 711770154 words, keeping 453852 word types\n",
      "2019-12-04 18:47:43,056 : INFO : PROGRESS: at sentence #44150000, processed 711965064 words, keeping 460601 word types\n",
      "2019-12-04 18:47:43,300 : INFO : PROGRESS: at sentence #44160000, processed 712151865 words, keeping 466500 word types\n",
      "2019-12-04 18:47:43,554 : INFO : PROGRESS: at sentence #44170000, processed 712352729 words, keeping 472827 word types\n",
      "2019-12-04 18:47:43,795 : INFO : PROGRESS: at sentence #44180000, processed 712535319 words, keeping 479764 word types\n",
      "2019-12-04 18:47:44,042 : INFO : PROGRESS: at sentence #44190000, processed 712726782 words, keeping 486117 word types\n",
      "2019-12-04 18:47:44,299 : INFO : PROGRESS: at sentence #44200000, processed 712930751 words, keeping 493416 word types\n",
      "2019-12-04 18:47:44,781 : INFO : pruned out 325414 tokens with count <=72 (before 500003, after 174589)\n",
      "2019-12-04 18:47:44,850 : INFO : PROGRESS: at sentence #44210000, processed 713154650 words, keeping 179196 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:47:45,114 : INFO : PROGRESS: at sentence #44220000, processed 713363122 words, keeping 191463 word types\n",
      "2019-12-04 18:47:45,389 : INFO : PROGRESS: at sentence #44230000, processed 713579028 words, keeping 203264 word types\n",
      "2019-12-04 18:47:45,650 : INFO : PROGRESS: at sentence #44240000, processed 713782174 words, keeping 212978 word types\n",
      "2019-12-04 18:47:45,900 : INFO : PROGRESS: at sentence #44250000, processed 713980615 words, keeping 222782 word types\n",
      "2019-12-04 18:47:46,159 : INFO : PROGRESS: at sentence #44260000, processed 714187271 words, keeping 233335 word types\n",
      "2019-12-04 18:47:46,414 : INFO : PROGRESS: at sentence #44270000, processed 714387924 words, keeping 242916 word types\n",
      "2019-12-04 18:47:46,682 : INFO : PROGRESS: at sentence #44280000, processed 714594215 words, keeping 253194 word types\n",
      "2019-12-04 18:47:46,953 : INFO : PROGRESS: at sentence #44290000, processed 714794464 words, keeping 261560 word types\n",
      "2019-12-04 18:47:47,217 : INFO : PROGRESS: at sentence #44300000, processed 714997256 words, keeping 271621 word types\n",
      "2019-12-04 18:47:47,485 : INFO : PROGRESS: at sentence #44310000, processed 715200527 words, keeping 281008 word types\n",
      "2019-12-04 18:47:47,736 : INFO : PROGRESS: at sentence #44320000, processed 715386721 words, keeping 289180 word types\n",
      "2019-12-04 18:47:48,003 : INFO : PROGRESS: at sentence #44330000, processed 715589600 words, keeping 297798 word types\n",
      "2019-12-04 18:47:48,266 : INFO : PROGRESS: at sentence #44340000, processed 715791225 words, keeping 306476 word types\n",
      "2019-12-04 18:47:48,535 : INFO : PROGRESS: at sentence #44350000, processed 715998254 words, keeping 315242 word types\n",
      "2019-12-04 18:47:48,802 : INFO : PROGRESS: at sentence #44360000, processed 716195305 words, keeping 323433 word types\n",
      "2019-12-04 18:47:49,078 : INFO : PROGRESS: at sentence #44370000, processed 716414293 words, keeping 332333 word types\n",
      "2019-12-04 18:47:49,334 : INFO : PROGRESS: at sentence #44380000, processed 716620987 words, keeping 341056 word types\n",
      "2019-12-04 18:47:49,602 : INFO : PROGRESS: at sentence #44390000, processed 716826352 words, keeping 349301 word types\n",
      "2019-12-04 18:47:49,868 : INFO : PROGRESS: at sentence #44400000, processed 717031205 words, keeping 356332 word types\n",
      "2019-12-04 18:47:50,134 : INFO : PROGRESS: at sentence #44410000, processed 717233965 words, keeping 363761 word types\n",
      "2019-12-04 18:47:50,400 : INFO : PROGRESS: at sentence #44420000, processed 717442710 words, keeping 371880 word types\n",
      "2019-12-04 18:47:50,681 : INFO : PROGRESS: at sentence #44430000, processed 717652478 words, keeping 380583 word types\n",
      "2019-12-04 18:47:50,941 : INFO : PROGRESS: at sentence #44440000, processed 717856726 words, keeping 388537 word types\n",
      "2019-12-04 18:47:51,209 : INFO : PROGRESS: at sentence #44450000, processed 718067171 words, keeping 396564 word types\n",
      "2019-12-04 18:47:51,472 : INFO : PROGRESS: at sentence #44460000, processed 718269600 words, keeping 403865 word types\n",
      "2019-12-04 18:47:51,749 : INFO : PROGRESS: at sentence #44470000, processed 718473001 words, keeping 412065 word types\n",
      "2019-12-04 18:47:52,013 : INFO : PROGRESS: at sentence #44480000, processed 718675980 words, keeping 419225 word types\n",
      "2019-12-04 18:47:52,276 : INFO : PROGRESS: at sentence #44490000, processed 718889386 words, keeping 426981 word types\n",
      "2019-12-04 18:47:52,533 : INFO : PROGRESS: at sentence #44500000, processed 719091420 words, keeping 434294 word types\n",
      "2019-12-04 18:47:52,803 : INFO : PROGRESS: at sentence #44510000, processed 719300640 words, keeping 443205 word types\n",
      "2019-12-04 18:47:53,059 : INFO : PROGRESS: at sentence #44520000, processed 719500461 words, keeping 450318 word types\n",
      "2019-12-04 18:47:53,314 : INFO : PROGRESS: at sentence #44530000, processed 719699793 words, keeping 457577 word types\n",
      "2019-12-04 18:47:53,568 : INFO : PROGRESS: at sentence #44540000, processed 719900386 words, keeping 465625 word types\n",
      "2019-12-04 18:47:53,826 : INFO : PROGRESS: at sentence #44550000, processed 720102176 words, keeping 472742 word types\n",
      "2019-12-04 18:47:54,080 : INFO : PROGRESS: at sentence #44560000, processed 720302052 words, keeping 480328 word types\n",
      "2019-12-04 18:47:54,338 : INFO : PROGRESS: at sentence #44570000, processed 720506794 words, keeping 486960 word types\n",
      "2019-12-04 18:47:54,595 : INFO : PROGRESS: at sentence #44580000, processed 720713632 words, keeping 493877 word types\n",
      "2019-12-04 18:47:55,105 : INFO : pruned out 325572 tokens with count <=73 (before 500001, after 174429)\n",
      "2019-12-04 18:47:55,128 : INFO : PROGRESS: at sentence #44590000, processed 720916179 words, keeping 175330 word types\n",
      "2019-12-04 18:47:55,396 : INFO : PROGRESS: at sentence #44600000, processed 721117950 words, keeping 186352 word types\n",
      "2019-12-04 18:47:55,656 : INFO : PROGRESS: at sentence #44610000, processed 721315550 words, keeping 196605 word types\n",
      "2019-12-04 18:47:55,915 : INFO : PROGRESS: at sentence #44620000, processed 721515288 words, keeping 205624 word types\n",
      "2019-12-04 18:47:56,181 : INFO : PROGRESS: at sentence #44630000, processed 721719705 words, keeping 215883 word types\n",
      "2019-12-04 18:47:56,453 : INFO : PROGRESS: at sentence #44640000, processed 721927950 words, keeping 225779 word types\n",
      "2019-12-04 18:47:56,717 : INFO : PROGRESS: at sentence #44650000, processed 722129886 words, keeping 235489 word types\n",
      "2019-12-04 18:47:56,983 : INFO : PROGRESS: at sentence #44660000, processed 722338384 words, keeping 246341 word types\n",
      "2019-12-04 18:47:57,260 : INFO : PROGRESS: at sentence #44670000, processed 722554517 words, keeping 257074 word types\n",
      "2019-12-04 18:47:57,534 : INFO : PROGRESS: at sentence #44680000, processed 722760983 words, keeping 267220 word types\n",
      "2019-12-04 18:47:57,796 : INFO : PROGRESS: at sentence #44690000, processed 722964306 words, keeping 276419 word types\n",
      "2019-12-04 18:47:58,063 : INFO : PROGRESS: at sentence #44700000, processed 723172927 words, keeping 285818 word types\n",
      "2019-12-04 18:47:58,329 : INFO : PROGRESS: at sentence #44710000, processed 723377368 words, keeping 294695 word types\n",
      "2019-12-04 18:47:58,596 : INFO : PROGRESS: at sentence #44720000, processed 723576170 words, keeping 302957 word types\n",
      "2019-12-04 18:47:58,861 : INFO : PROGRESS: at sentence #44730000, processed 723780632 words, keeping 311913 word types\n",
      "2019-12-04 18:47:59,119 : INFO : PROGRESS: at sentence #44740000, processed 723980614 words, keeping 320060 word types\n",
      "2019-12-04 18:47:59,376 : INFO : PROGRESS: at sentence #44750000, processed 724182313 words, keeping 328778 word types\n",
      "2019-12-04 18:47:59,637 : INFO : PROGRESS: at sentence #44760000, processed 724383967 words, keeping 336936 word types\n",
      "2019-12-04 18:47:59,898 : INFO : PROGRESS: at sentence #44770000, processed 724585801 words, keeping 343897 word types\n",
      "2019-12-04 18:48:00,164 : INFO : PROGRESS: at sentence #44780000, processed 724792619 words, keeping 352427 word types\n",
      "2019-12-04 18:48:00,424 : INFO : PROGRESS: at sentence #44790000, processed 724992103 words, keeping 359587 word types\n",
      "2019-12-04 18:48:00,683 : INFO : PROGRESS: at sentence #44800000, processed 725190227 words, keeping 367663 word types\n",
      "2019-12-04 18:48:00,943 : INFO : PROGRESS: at sentence #44810000, processed 725391779 words, keeping 375318 word types\n",
      "2019-12-04 18:48:01,204 : INFO : PROGRESS: at sentence #44820000, processed 725594089 words, keeping 382885 word types\n",
      "2019-12-04 18:48:01,471 : INFO : PROGRESS: at sentence #44830000, processed 725794809 words, keeping 390365 word types\n",
      "2019-12-04 18:48:01,735 : INFO : PROGRESS: at sentence #44840000, processed 725995183 words, keeping 397666 word types\n",
      "2019-12-04 18:48:01,985 : INFO : PROGRESS: at sentence #44850000, processed 726187929 words, keeping 405177 word types\n",
      "2019-12-04 18:48:02,246 : INFO : PROGRESS: at sentence #44860000, processed 726390553 words, keeping 413165 word types\n",
      "2019-12-04 18:48:02,502 : INFO : PROGRESS: at sentence #44870000, processed 726586782 words, keeping 420558 word types\n",
      "2019-12-04 18:48:02,762 : INFO : PROGRESS: at sentence #44880000, processed 726782054 words, keeping 427668 word types\n",
      "2019-12-04 18:48:03,024 : INFO : PROGRESS: at sentence #44890000, processed 726983713 words, keeping 434676 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:48:03,280 : INFO : PROGRESS: at sentence #44900000, processed 727182440 words, keeping 441658 word types\n",
      "2019-12-04 18:48:03,540 : INFO : PROGRESS: at sentence #44910000, processed 727385963 words, keeping 449002 word types\n",
      "2019-12-04 18:48:03,795 : INFO : PROGRESS: at sentence #44920000, processed 727583362 words, keeping 455880 word types\n",
      "2019-12-04 18:48:04,054 : INFO : PROGRESS: at sentence #44930000, processed 727782081 words, keeping 463150 word types\n",
      "2019-12-04 18:48:04,303 : INFO : PROGRESS: at sentence #44940000, processed 727973906 words, keeping 469918 word types\n",
      "2019-12-04 18:48:04,555 : INFO : PROGRESS: at sentence #44950000, processed 728169373 words, keeping 476364 word types\n",
      "2019-12-04 18:48:04,805 : INFO : PROGRESS: at sentence #44960000, processed 728361262 words, keeping 482428 word types\n",
      "2019-12-04 18:48:05,068 : INFO : PROGRESS: at sentence #44970000, processed 728565967 words, keeping 489505 word types\n",
      "2019-12-04 18:48:05,320 : INFO : PROGRESS: at sentence #44980000, processed 728759360 words, keeping 496265 word types\n",
      "2019-12-04 18:48:05,778 : INFO : pruned out 325782 tokens with count <=74 (before 500027, after 174245)\n",
      "2019-12-04 18:48:05,889 : INFO : PROGRESS: at sentence #44990000, processed 728961098 words, keeping 178671 word types\n",
      "2019-12-04 18:48:06,162 : INFO : PROGRESS: at sentence #45000000, processed 729163319 words, keeping 189400 word types\n",
      "2019-12-04 18:48:06,439 : INFO : PROGRESS: at sentence #45010000, processed 729374195 words, keeping 200091 word types\n",
      "2019-12-04 18:48:06,708 : INFO : PROGRESS: at sentence #45020000, processed 729576340 words, keeping 210337 word types\n",
      "2019-12-04 18:48:06,975 : INFO : PROGRESS: at sentence #45030000, processed 729778516 words, keeping 219714 word types\n",
      "2019-12-04 18:48:07,245 : INFO : PROGRESS: at sentence #45040000, processed 729982260 words, keeping 229563 word types\n",
      "2019-12-04 18:48:07,508 : INFO : PROGRESS: at sentence #45050000, processed 730181995 words, keeping 239540 word types\n",
      "2019-12-04 18:48:07,781 : INFO : PROGRESS: at sentence #45060000, processed 730389844 words, keeping 249502 word types\n",
      "2019-12-04 18:48:08,043 : INFO : PROGRESS: at sentence #45070000, processed 730592535 words, keeping 257947 word types\n",
      "2019-12-04 18:48:08,306 : INFO : PROGRESS: at sentence #45080000, processed 730794327 words, keeping 267148 word types\n",
      "2019-12-04 18:48:08,571 : INFO : PROGRESS: at sentence #45090000, processed 730999273 words, keeping 275875 word types\n",
      "2019-12-04 18:48:08,831 : INFO : PROGRESS: at sentence #45100000, processed 731199323 words, keeping 284502 word types\n",
      "2019-12-04 18:48:09,097 : INFO : PROGRESS: at sentence #45110000, processed 731401808 words, keeping 293595 word types\n",
      "2019-12-04 18:48:09,366 : INFO : PROGRESS: at sentence #45120000, processed 731611330 words, keeping 302498 word types\n",
      "2019-12-04 18:48:09,621 : INFO : PROGRESS: at sentence #45130000, processed 731806911 words, keeping 310456 word types\n",
      "2019-12-04 18:48:09,939 : INFO : PROGRESS: at sentence #45140000, processed 732010959 words, keeping 318315 word types\n",
      "2019-12-04 18:48:10,211 : INFO : PROGRESS: at sentence #45150000, processed 732222808 words, keeping 326598 word types\n",
      "2019-12-04 18:48:10,479 : INFO : PROGRESS: at sentence #45160000, processed 732428558 words, keeping 335845 word types\n",
      "2019-12-04 18:48:10,745 : INFO : PROGRESS: at sentence #45170000, processed 732633872 words, keeping 343540 word types\n",
      "2019-12-04 18:48:11,013 : INFO : PROGRESS: at sentence #45180000, processed 732836456 words, keeping 351411 word types\n",
      "2019-12-04 18:48:11,272 : INFO : PROGRESS: at sentence #45190000, processed 733034813 words, keeping 359178 word types\n",
      "2019-12-04 18:48:11,534 : INFO : PROGRESS: at sentence #45200000, processed 733233858 words, keeping 366940 word types\n",
      "2019-12-04 18:48:11,793 : INFO : PROGRESS: at sentence #45210000, processed 733430380 words, keeping 374526 word types\n",
      "2019-12-04 18:48:12,062 : INFO : PROGRESS: at sentence #45220000, processed 733635028 words, keeping 382194 word types\n",
      "2019-12-04 18:48:12,330 : INFO : PROGRESS: at sentence #45230000, processed 733840569 words, keeping 390875 word types\n",
      "2019-12-04 18:48:12,605 : INFO : PROGRESS: at sentence #45240000, processed 734048879 words, keeping 399127 word types\n",
      "2019-12-04 18:48:12,862 : INFO : PROGRESS: at sentence #45250000, processed 734246316 words, keeping 406650 word types\n",
      "2019-12-04 18:48:13,125 : INFO : PROGRESS: at sentence #45260000, processed 734453956 words, keeping 414521 word types\n",
      "2019-12-04 18:48:13,438 : INFO : PROGRESS: at sentence #45270000, processed 734653539 words, keeping 422226 word types\n",
      "2019-12-04 18:48:13,695 : INFO : PROGRESS: at sentence #45280000, processed 734855586 words, keeping 430484 word types\n",
      "2019-12-04 18:48:13,952 : INFO : PROGRESS: at sentence #45290000, processed 735060391 words, keeping 438540 word types\n",
      "2019-12-04 18:48:14,209 : INFO : PROGRESS: at sentence #45300000, processed 735267563 words, keeping 446010 word types\n",
      "2019-12-04 18:48:14,463 : INFO : PROGRESS: at sentence #45310000, processed 735469644 words, keeping 452630 word types\n",
      "2019-12-04 18:48:14,701 : INFO : PROGRESS: at sentence #45320000, processed 735661340 words, keeping 459547 word types\n",
      "2019-12-04 18:48:14,958 : INFO : PROGRESS: at sentence #45330000, processed 735862906 words, keeping 466912 word types\n",
      "2019-12-04 18:48:15,205 : INFO : PROGRESS: at sentence #45340000, processed 736060159 words, keeping 473612 word types\n",
      "2019-12-04 18:48:15,456 : INFO : PROGRESS: at sentence #45350000, processed 736265076 words, keeping 480892 word types\n",
      "2019-12-04 18:48:15,699 : INFO : PROGRESS: at sentence #45360000, processed 736462165 words, keeping 486998 word types\n",
      "2019-12-04 18:48:15,955 : INFO : PROGRESS: at sentence #45370000, processed 736666415 words, keeping 493728 word types\n",
      "2019-12-04 18:48:16,454 : INFO : pruned out 325922 tokens with count <=75 (before 500003, after 174081)\n",
      "2019-12-04 18:48:16,470 : INFO : PROGRESS: at sentence #45380000, processed 736863581 words, keeping 174763 word types\n",
      "2019-12-04 18:48:16,730 : INFO : PROGRESS: at sentence #45390000, processed 737067432 words, keeping 185396 word types\n",
      "2019-12-04 18:48:16,982 : INFO : PROGRESS: at sentence #45400000, processed 737266957 words, keeping 195366 word types\n",
      "2019-12-04 18:48:17,238 : INFO : PROGRESS: at sentence #45410000, processed 737469415 words, keeping 205386 word types\n",
      "2019-12-04 18:48:17,493 : INFO : PROGRESS: at sentence #45420000, processed 737671600 words, keeping 215163 word types\n",
      "2019-12-04 18:48:17,733 : INFO : PROGRESS: at sentence #45430000, processed 737857395 words, keeping 223491 word types\n",
      "2019-12-04 18:48:17,979 : INFO : PROGRESS: at sentence #45440000, processed 738052404 words, keeping 231501 word types\n",
      "2019-12-04 18:48:18,228 : INFO : PROGRESS: at sentence #45450000, processed 738240743 words, keeping 240439 word types\n",
      "2019-12-04 18:48:18,477 : INFO : PROGRESS: at sentence #45460000, processed 738432223 words, keeping 248575 word types\n",
      "2019-12-04 18:48:18,729 : INFO : PROGRESS: at sentence #45470000, processed 738630416 words, keeping 257216 word types\n",
      "2019-12-04 18:48:18,973 : INFO : PROGRESS: at sentence #45480000, processed 738820430 words, keeping 266355 word types\n",
      "2019-12-04 18:48:19,232 : INFO : PROGRESS: at sentence #45490000, processed 739023643 words, keeping 276265 word types\n",
      "2019-12-04 18:48:19,491 : INFO : PROGRESS: at sentence #45500000, processed 739226286 words, keeping 285018 word types\n",
      "2019-12-04 18:48:19,747 : INFO : PROGRESS: at sentence #45510000, processed 739428015 words, keeping 293049 word types\n",
      "2019-12-04 18:48:20,001 : INFO : PROGRESS: at sentence #45520000, processed 739628765 words, keeping 301514 word types\n",
      "2019-12-04 18:48:20,260 : INFO : PROGRESS: at sentence #45530000, processed 739831997 words, keeping 309784 word types\n",
      "2019-12-04 18:48:20,552 : INFO : PROGRESS: at sentence #45540000, processed 740062422 words, keeping 320542 word types\n",
      "2019-12-04 18:48:20,805 : INFO : PROGRESS: at sentence #45550000, processed 740260067 words, keeping 328571 word types\n",
      "2019-12-04 18:48:21,051 : INFO : PROGRESS: at sentence #45560000, processed 740454457 words, keeping 336367 word types\n",
      "2019-12-04 18:48:21,302 : INFO : PROGRESS: at sentence #45570000, processed 740653517 words, keeping 344548 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:48:21,565 : INFO : PROGRESS: at sentence #45580000, processed 740864551 words, keeping 353179 word types\n",
      "2019-12-04 18:48:21,824 : INFO : PROGRESS: at sentence #45590000, processed 741067653 words, keeping 361288 word types\n",
      "2019-12-04 18:48:22,084 : INFO : PROGRESS: at sentence #45600000, processed 741269340 words, keeping 368609 word types\n",
      "2019-12-04 18:48:22,353 : INFO : PROGRESS: at sentence #45610000, processed 741486368 words, keeping 378895 word types\n",
      "2019-12-04 18:48:22,634 : INFO : PROGRESS: at sentence #45620000, processed 741708096 words, keeping 387444 word types\n",
      "2019-12-04 18:48:22,897 : INFO : PROGRESS: at sentence #45630000, processed 741913684 words, keeping 395485 word types\n",
      "2019-12-04 18:48:23,163 : INFO : PROGRESS: at sentence #45640000, processed 742123478 words, keeping 403229 word types\n",
      "2019-12-04 18:48:23,423 : INFO : PROGRESS: at sentence #45650000, processed 742327486 words, keeping 410784 word types\n",
      "2019-12-04 18:48:23,677 : INFO : PROGRESS: at sentence #45660000, processed 742528567 words, keeping 418291 word types\n",
      "2019-12-04 18:48:23,922 : INFO : PROGRESS: at sentence #45670000, processed 742722410 words, keeping 425470 word types\n",
      "2019-12-04 18:48:24,179 : INFO : PROGRESS: at sentence #45680000, processed 742931129 words, keeping 432733 word types\n",
      "2019-12-04 18:48:24,432 : INFO : PROGRESS: at sentence #45690000, processed 743134138 words, keeping 439434 word types\n",
      "2019-12-04 18:48:24,733 : INFO : PROGRESS: at sentence #45700000, processed 743373434 words, keeping 448305 word types\n",
      "2019-12-04 18:48:25,000 : INFO : PROGRESS: at sentence #45710000, processed 743586169 words, keeping 456084 word types\n",
      "2019-12-04 18:48:25,256 : INFO : PROGRESS: at sentence #45720000, processed 743787653 words, keeping 463607 word types\n",
      "2019-12-04 18:48:25,572 : INFO : PROGRESS: at sentence #45730000, processed 744041587 words, keeping 473443 word types\n",
      "2019-12-04 18:48:25,841 : INFO : PROGRESS: at sentence #45740000, processed 744257500 words, keeping 482070 word types\n",
      "2019-12-04 18:48:26,097 : INFO : PROGRESS: at sentence #45750000, processed 744463908 words, keeping 488412 word types\n",
      "2019-12-04 18:48:26,355 : INFO : PROGRESS: at sentence #45760000, processed 744672446 words, keeping 494943 word types\n",
      "2019-12-04 18:48:26,825 : INFO : pruned out 326078 tokens with count <=76 (before 500022, after 173944)\n",
      "2019-12-04 18:48:26,927 : INFO : PROGRESS: at sentence #45770000, processed 744896119 words, keeping 178373 word types\n",
      "2019-12-04 18:48:27,179 : INFO : PROGRESS: at sentence #45780000, processed 745094034 words, keeping 188599 word types\n",
      "2019-12-04 18:48:27,436 : INFO : PROGRESS: at sentence #45790000, processed 745293605 words, keeping 198444 word types\n",
      "2019-12-04 18:48:27,711 : INFO : PROGRESS: at sentence #45800000, processed 745512844 words, keeping 208887 word types\n",
      "2019-12-04 18:48:27,975 : INFO : PROGRESS: at sentence #45810000, processed 745721089 words, keeping 219147 word types\n",
      "2019-12-04 18:48:28,224 : INFO : PROGRESS: at sentence #45820000, processed 745924425 words, keeping 229907 word types\n",
      "2019-12-04 18:48:28,466 : INFO : PROGRESS: at sentence #45830000, processed 746128024 words, keeping 238746 word types\n",
      "2019-12-04 18:48:28,723 : INFO : PROGRESS: at sentence #45840000, processed 746341951 words, keeping 248216 word types\n",
      "2019-12-04 18:48:28,963 : INFO : PROGRESS: at sentence #45850000, processed 746544225 words, keeping 257090 word types\n",
      "2019-12-04 18:48:29,204 : INFO : PROGRESS: at sentence #45860000, processed 746744170 words, keeping 266081 word types\n",
      "2019-12-04 18:48:29,506 : INFO : PROGRESS: at sentence #45870000, processed 747007632 words, keeping 277903 word types\n",
      "2019-12-04 18:48:29,751 : INFO : PROGRESS: at sentence #45880000, processed 747209056 words, keeping 286804 word types\n",
      "2019-12-04 18:48:30,008 : INFO : PROGRESS: at sentence #45890000, processed 747422706 words, keeping 295911 word types\n",
      "2019-12-04 18:48:30,254 : INFO : PROGRESS: at sentence #45900000, processed 747624860 words, keeping 304435 word types\n",
      "2019-12-04 18:48:30,502 : INFO : PROGRESS: at sentence #45910000, processed 747825643 words, keeping 311926 word types\n",
      "2019-12-04 18:48:30,753 : INFO : PROGRESS: at sentence #45920000, processed 748031457 words, keeping 320658 word types\n",
      "2019-12-04 18:48:30,997 : INFO : PROGRESS: at sentence #45930000, processed 748236437 words, keeping 328814 word types\n",
      "2019-12-04 18:48:31,239 : INFO : PROGRESS: at sentence #45940000, processed 748434974 words, keeping 336837 word types\n",
      "2019-12-04 18:48:31,487 : INFO : PROGRESS: at sentence #45950000, processed 748635375 words, keeping 345303 word types\n",
      "2019-12-04 18:48:31,757 : INFO : PROGRESS: at sentence #45960000, processed 748852325 words, keeping 353262 word types\n",
      "2019-12-04 18:48:32,017 : INFO : PROGRESS: at sentence #45970000, processed 749062197 words, keeping 361272 word types\n",
      "2019-12-04 18:48:32,296 : INFO : PROGRESS: at sentence #45980000, processed 749269804 words, keeping 369679 word types\n",
      "2019-12-04 18:48:32,560 : INFO : PROGRESS: at sentence #45990000, processed 749476375 words, keeping 377587 word types\n",
      "2019-12-04 18:48:32,825 : INFO : PROGRESS: at sentence #46000000, processed 749685505 words, keeping 385609 word types\n",
      "2019-12-04 18:48:33,088 : INFO : PROGRESS: at sentence #46010000, processed 749893675 words, keeping 393737 word types\n",
      "2019-12-04 18:48:33,342 : INFO : PROGRESS: at sentence #46020000, processed 750098064 words, keeping 401241 word types\n",
      "2019-12-04 18:48:33,604 : INFO : PROGRESS: at sentence #46030000, processed 750307566 words, keeping 408759 word types\n",
      "2019-12-04 18:48:33,856 : INFO : PROGRESS: at sentence #46040000, processed 750506214 words, keeping 415943 word types\n",
      "2019-12-04 18:48:34,116 : INFO : PROGRESS: at sentence #46050000, processed 750713831 words, keeping 423591 word types\n",
      "2019-12-04 18:48:34,371 : INFO : PROGRESS: at sentence #46060000, processed 750914617 words, keeping 430713 word types\n",
      "2019-12-04 18:48:34,606 : INFO : PROGRESS: at sentence #46070000, processed 751096409 words, keeping 438257 word types\n",
      "2019-12-04 18:48:34,877 : INFO : PROGRESS: at sentence #46080000, processed 751307835 words, keeping 447820 word types\n",
      "2019-12-04 18:48:35,123 : INFO : PROGRESS: at sentence #46090000, processed 751497695 words, keeping 454469 word types\n",
      "2019-12-04 18:48:35,399 : INFO : PROGRESS: at sentence #46100000, processed 751717048 words, keeping 463821 word types\n",
      "2019-12-04 18:48:35,704 : INFO : PROGRESS: at sentence #46110000, processed 751959807 words, keeping 474020 word types\n",
      "2019-12-04 18:48:36,002 : INFO : PROGRESS: at sentence #46120000, processed 752199460 words, keeping 483437 word types\n",
      "2019-12-04 18:48:36,305 : INFO : PROGRESS: at sentence #46130000, processed 752434928 words, keeping 493154 word types\n",
      "2019-12-04 18:48:36,845 : INFO : pruned out 326191 tokens with count <=77 (before 500001, after 173810)\n",
      "2019-12-04 18:48:36,897 : INFO : PROGRESS: at sentence #46140000, processed 752655231 words, keeping 175607 word types\n",
      "2019-12-04 18:48:37,183 : INFO : PROGRESS: at sentence #46150000, processed 752881861 words, keeping 187472 word types\n",
      "2019-12-04 18:48:37,474 : INFO : PROGRESS: at sentence #46160000, processed 753104167 words, keeping 198950 word types\n",
      "2019-12-04 18:48:37,787 : INFO : PROGRESS: at sentence #46170000, processed 753321182 words, keeping 210492 word types\n",
      "2019-12-04 18:48:38,156 : INFO : PROGRESS: at sentence #46180000, processed 753614547 words, keeping 227593 word types\n",
      "2019-12-04 18:48:38,444 : INFO : PROGRESS: at sentence #46190000, processed 753840178 words, keeping 238615 word types\n",
      "2019-12-04 18:48:38,738 : INFO : PROGRESS: at sentence #46200000, processed 754068277 words, keeping 250561 word types\n",
      "2019-12-04 18:48:39,041 : INFO : PROGRESS: at sentence #46210000, processed 754301358 words, keeping 260859 word types\n",
      "2019-12-04 18:48:39,297 : INFO : PROGRESS: at sentence #46220000, processed 754499746 words, keeping 269657 word types\n",
      "2019-12-04 18:48:39,551 : INFO : PROGRESS: at sentence #46230000, processed 754700037 words, keeping 278006 word types\n",
      "2019-12-04 18:48:39,882 : INFO : PROGRESS: at sentence #46240000, processed 754927222 words, keeping 288957 word types\n",
      "2019-12-04 18:48:40,135 : INFO : PROGRESS: at sentence #46250000, processed 755122349 words, keeping 297882 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:48:40,408 : INFO : PROGRESS: at sentence #46260000, processed 755334445 words, keeping 307263 word types\n",
      "2019-12-04 18:48:40,682 : INFO : PROGRESS: at sentence #46270000, processed 755538172 words, keeping 315522 word types\n",
      "2019-12-04 18:48:40,951 : INFO : PROGRESS: at sentence #46280000, processed 755740277 words, keeping 324270 word types\n",
      "2019-12-04 18:48:41,207 : INFO : PROGRESS: at sentence #46290000, processed 755937053 words, keeping 332432 word types\n",
      "2019-12-04 18:48:41,469 : INFO : PROGRESS: at sentence #46300000, processed 756136511 words, keeping 340098 word types\n",
      "2019-12-04 18:48:41,741 : INFO : PROGRESS: at sentence #46310000, processed 756334719 words, keeping 347919 word types\n",
      "2019-12-04 18:48:41,980 : INFO : PROGRESS: at sentence #46320000, processed 756517885 words, keeping 355863 word types\n",
      "2019-12-04 18:48:42,242 : INFO : PROGRESS: at sentence #46330000, processed 756717882 words, keeping 364214 word types\n",
      "2019-12-04 18:48:42,516 : INFO : PROGRESS: at sentence #46340000, processed 756932908 words, keeping 374801 word types\n",
      "2019-12-04 18:48:42,797 : INFO : PROGRESS: at sentence #46350000, processed 757132220 words, keeping 382475 word types\n",
      "2019-12-04 18:48:43,055 : INFO : PROGRESS: at sentence #46360000, processed 757329849 words, keeping 389870 word types\n",
      "2019-12-04 18:48:43,316 : INFO : PROGRESS: at sentence #46370000, processed 757533221 words, keeping 397925 word types\n",
      "2019-12-04 18:48:43,552 : INFO : PROGRESS: at sentence #46380000, processed 757714382 words, keeping 404793 word types\n",
      "2019-12-04 18:48:43,801 : INFO : PROGRESS: at sentence #46390000, processed 757909420 words, keeping 412092 word types\n",
      "2019-12-04 18:48:44,050 : INFO : PROGRESS: at sentence #46400000, processed 758102198 words, keeping 419504 word types\n",
      "2019-12-04 18:48:44,360 : INFO : PROGRESS: at sentence #46410000, processed 758293280 words, keeping 426601 word types\n",
      "2019-12-04 18:48:44,620 : INFO : PROGRESS: at sentence #46420000, processed 758495764 words, keeping 434497 word types\n",
      "2019-12-04 18:48:44,875 : INFO : PROGRESS: at sentence #46430000, processed 758694194 words, keeping 441405 word types\n",
      "2019-12-04 18:48:45,141 : INFO : PROGRESS: at sentence #46440000, processed 758899070 words, keeping 448010 word types\n",
      "2019-12-04 18:48:45,404 : INFO : PROGRESS: at sentence #46450000, processed 759106701 words, keeping 455177 word types\n",
      "2019-12-04 18:48:45,663 : INFO : PROGRESS: at sentence #46460000, processed 759310487 words, keeping 462621 word types\n",
      "2019-12-04 18:48:45,965 : INFO : PROGRESS: at sentence #46470000, processed 759516372 words, keeping 470123 word types\n",
      "2019-12-04 18:48:46,221 : INFO : PROGRESS: at sentence #46480000, processed 759715839 words, keeping 477937 word types\n",
      "2019-12-04 18:48:46,485 : INFO : PROGRESS: at sentence #46490000, processed 759922536 words, keeping 485404 word types\n",
      "2019-12-04 18:48:46,743 : INFO : PROGRESS: at sentence #46500000, processed 760123030 words, keeping 492461 word types\n",
      "2019-12-04 18:48:47,017 : INFO : PROGRESS: at sentence #46510000, processed 760327345 words, keeping 498856 word types\n",
      "2019-12-04 18:48:47,370 : INFO : pruned out 326344 tokens with count <=78 (before 500001, after 173657)\n",
      "2019-12-04 18:48:47,587 : INFO : PROGRESS: at sentence #46520000, processed 760528952 words, keeping 182322 word types\n",
      "2019-12-04 18:48:47,861 : INFO : PROGRESS: at sentence #46530000, processed 760741827 words, keeping 193095 word types\n",
      "2019-12-04 18:48:48,121 : INFO : PROGRESS: at sentence #46540000, processed 760940656 words, keeping 202993 word types\n",
      "2019-12-04 18:48:48,371 : INFO : PROGRESS: at sentence #46550000, processed 761134473 words, keeping 211977 word types\n",
      "2019-12-04 18:48:48,626 : INFO : PROGRESS: at sentence #46560000, processed 761329133 words, keeping 220901 word types\n",
      "2019-12-04 18:48:48,890 : INFO : PROGRESS: at sentence #46570000, processed 761535003 words, keeping 229506 word types\n",
      "2019-12-04 18:48:49,175 : INFO : PROGRESS: at sentence #46580000, processed 761760560 words, keeping 241055 word types\n",
      "2019-12-04 18:48:49,461 : INFO : PROGRESS: at sentence #46590000, processed 761986977 words, keeping 252129 word types\n",
      "2019-12-04 18:48:49,750 : INFO : PROGRESS: at sentence #46600000, processed 762217001 words, keeping 263605 word types\n",
      "2019-12-04 18:48:50,016 : INFO : PROGRESS: at sentence #46610000, processed 762425968 words, keeping 273587 word types\n",
      "2019-12-04 18:48:50,269 : INFO : PROGRESS: at sentence #46620000, processed 762622648 words, keeping 282026 word types\n",
      "2019-12-04 18:48:50,522 : INFO : PROGRESS: at sentence #46630000, processed 762817869 words, keeping 290613 word types\n",
      "2019-12-04 18:48:50,775 : INFO : PROGRESS: at sentence #46640000, processed 763016756 words, keeping 298697 word types\n",
      "2019-12-04 18:48:51,032 : INFO : PROGRESS: at sentence #46650000, processed 763218157 words, keeping 306490 word types\n",
      "2019-12-04 18:48:51,277 : INFO : PROGRESS: at sentence #46660000, processed 763410394 words, keeping 315142 word types\n",
      "2019-12-04 18:48:51,535 : INFO : PROGRESS: at sentence #46670000, processed 763613185 words, keeping 323267 word types\n",
      "2019-12-04 18:48:51,822 : INFO : PROGRESS: at sentence #46680000, processed 763836937 words, keeping 331154 word types\n",
      "2019-12-04 18:48:52,106 : INFO : PROGRESS: at sentence #46690000, processed 764058448 words, keeping 340429 word types\n",
      "2019-12-04 18:48:52,375 : INFO : PROGRESS: at sentence #46700000, processed 764268185 words, keeping 348724 word types\n",
      "2019-12-04 18:48:52,633 : INFO : PROGRESS: at sentence #46710000, processed 764466324 words, keeping 356933 word types\n",
      "2019-12-04 18:48:52,909 : INFO : PROGRESS: at sentence #46720000, processed 764669535 words, keeping 363766 word types\n",
      "2019-12-04 18:48:53,175 : INFO : PROGRESS: at sentence #46730000, processed 764876362 words, keeping 370734 word types\n",
      "2019-12-04 18:48:53,433 : INFO : PROGRESS: at sentence #46740000, processed 765079698 words, keeping 378178 word types\n",
      "2019-12-04 18:48:53,697 : INFO : PROGRESS: at sentence #46750000, processed 765285489 words, keeping 385958 word types\n",
      "2019-12-04 18:48:53,943 : INFO : PROGRESS: at sentence #46760000, processed 765478414 words, keeping 392902 word types\n",
      "2019-12-04 18:48:54,197 : INFO : PROGRESS: at sentence #46770000, processed 765683176 words, keeping 400208 word types\n",
      "2019-12-04 18:48:54,456 : INFO : PROGRESS: at sentence #46780000, processed 765892875 words, keeping 407332 word types\n",
      "2019-12-04 18:48:54,685 : INFO : PROGRESS: at sentence #46790000, processed 766071601 words, keeping 412968 word types\n",
      "2019-12-04 18:48:54,910 : INFO : PROGRESS: at sentence #46800000, processed 766248149 words, keeping 418214 word types\n",
      "2019-12-04 18:48:55,128 : INFO : PROGRESS: at sentence #46810000, processed 766416065 words, keeping 423941 word types\n",
      "2019-12-04 18:48:55,358 : INFO : PROGRESS: at sentence #46820000, processed 766598989 words, keeping 429412 word types\n",
      "2019-12-04 18:48:55,612 : INFO : PROGRESS: at sentence #46830000, processed 766799717 words, keeping 436083 word types\n",
      "2019-12-04 18:48:55,869 : INFO : PROGRESS: at sentence #46840000, processed 767001376 words, keeping 442928 word types\n",
      "2019-12-04 18:48:56,124 : INFO : PROGRESS: at sentence #46850000, processed 767200891 words, keeping 449998 word types\n",
      "2019-12-04 18:48:56,367 : INFO : PROGRESS: at sentence #46860000, processed 767393114 words, keeping 456566 word types\n",
      "2019-12-04 18:48:56,627 : INFO : PROGRESS: at sentence #46870000, processed 767597693 words, keeping 462814 word types\n",
      "2019-12-04 18:48:56,874 : INFO : PROGRESS: at sentence #46880000, processed 767794295 words, keeping 469658 word types\n",
      "2019-12-04 18:48:57,121 : INFO : PROGRESS: at sentence #46890000, processed 767990906 words, keeping 476540 word types\n",
      "2019-12-04 18:48:57,385 : INFO : PROGRESS: at sentence #46900000, processed 768197579 words, keeping 483863 word types\n",
      "2019-12-04 18:48:57,641 : INFO : PROGRESS: at sentence #46910000, processed 768397088 words, keeping 490580 word types\n",
      "2019-12-04 18:48:57,893 : INFO : PROGRESS: at sentence #46920000, processed 768598230 words, keeping 497531 word types\n",
      "2019-12-04 18:48:58,261 : INFO : pruned out 326483 tokens with count <=79 (before 500002, after 173519)\n",
      "2019-12-04 18:48:58,420 : INFO : PROGRESS: at sentence #46930000, processed 768800341 words, keeping 179901 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:48:58,671 : INFO : PROGRESS: at sentence #46940000, processed 769001542 words, keeping 189991 word types\n",
      "2019-12-04 18:48:58,940 : INFO : PROGRESS: at sentence #46950000, processed 769213000 words, keeping 200401 word types\n",
      "2019-12-04 18:48:59,200 : INFO : PROGRESS: at sentence #46960000, processed 769412127 words, keeping 210178 word types\n",
      "2019-12-04 18:48:59,454 : INFO : PROGRESS: at sentence #46970000, processed 769610483 words, keeping 219763 word types\n",
      "2019-12-04 18:48:59,744 : INFO : PROGRESS: at sentence #46980000, processed 769841221 words, keeping 230736 word types\n",
      "2019-12-04 18:48:59,973 : INFO : PROGRESS: at sentence #46990000, processed 770022493 words, keeping 237946 word types\n",
      "2019-12-04 18:49:00,236 : INFO : PROGRESS: at sentence #47000000, processed 770233347 words, keeping 247374 word types\n",
      "2019-12-04 18:49:00,484 : INFO : PROGRESS: at sentence #47010000, processed 770430759 words, keeping 256779 word types\n",
      "2019-12-04 18:49:00,741 : INFO : PROGRESS: at sentence #47020000, processed 770633010 words, keeping 266064 word types\n",
      "2019-12-04 18:49:00,995 : INFO : PROGRESS: at sentence #47030000, processed 770835621 words, keeping 275519 word types\n",
      "2019-12-04 18:49:01,250 : INFO : PROGRESS: at sentence #47040000, processed 771035522 words, keeping 284026 word types\n",
      "2019-12-04 18:49:01,509 : INFO : PROGRESS: at sentence #47050000, processed 771237005 words, keeping 292185 word types\n",
      "2019-12-04 18:49:01,801 : INFO : PROGRESS: at sentence #47060000, processed 771470439 words, keeping 302303 word types\n",
      "2019-12-04 18:49:02,055 : INFO : PROGRESS: at sentence #47070000, processed 771668375 words, keeping 310844 word types\n",
      "2019-12-04 18:49:02,338 : INFO : PROGRESS: at sentence #47080000, processed 771891980 words, keeping 320996 word types\n",
      "2019-12-04 18:49:02,598 : INFO : PROGRESS: at sentence #47090000, processed 772091372 words, keeping 330020 word types\n",
      "2019-12-04 18:49:02,858 : INFO : PROGRESS: at sentence #47100000, processed 772294795 words, keeping 337827 word types\n",
      "2019-12-04 18:49:03,123 : INFO : PROGRESS: at sentence #47110000, processed 772503674 words, keeping 346881 word types\n",
      "2019-12-04 18:49:03,380 : INFO : PROGRESS: at sentence #47120000, processed 772704377 words, keeping 354861 word types\n",
      "2019-12-04 18:49:03,641 : INFO : PROGRESS: at sentence #47130000, processed 772913485 words, keeping 363504 word types\n",
      "2019-12-04 18:49:03,905 : INFO : PROGRESS: at sentence #47140000, processed 773121844 words, keeping 372114 word types\n",
      "2019-12-04 18:49:04,164 : INFO : PROGRESS: at sentence #47150000, processed 773326935 words, keeping 379993 word types\n",
      "2019-12-04 18:49:04,447 : INFO : PROGRESS: at sentence #47160000, processed 773549863 words, keeping 388006 word types\n",
      "2019-12-04 18:49:04,708 : INFO : PROGRESS: at sentence #47170000, processed 773756738 words, keeping 396474 word types\n",
      "2019-12-04 18:49:04,972 : INFO : PROGRESS: at sentence #47180000, processed 773966921 words, keeping 404925 word types\n",
      "2019-12-04 18:49:05,230 : INFO : PROGRESS: at sentence #47190000, processed 774165107 words, keeping 412122 word types\n",
      "2019-12-04 18:49:05,495 : INFO : PROGRESS: at sentence #47200000, processed 774373911 words, keeping 420217 word types\n",
      "2019-12-04 18:49:05,754 : INFO : PROGRESS: at sentence #47210000, processed 774573461 words, keeping 427644 word types\n",
      "2019-12-04 18:49:06,003 : INFO : PROGRESS: at sentence #47220000, processed 774767619 words, keeping 435144 word types\n",
      "2019-12-04 18:49:06,253 : INFO : PROGRESS: at sentence #47230000, processed 774964550 words, keeping 442232 word types\n",
      "2019-12-04 18:49:06,521 : INFO : PROGRESS: at sentence #47240000, processed 775175548 words, keeping 450377 word types\n",
      "2019-12-04 18:49:06,771 : INFO : PROGRESS: at sentence #47250000, processed 775369933 words, keeping 457394 word types\n",
      "2019-12-04 18:49:07,025 : INFO : PROGRESS: at sentence #47260000, processed 775569994 words, keeping 464264 word types\n",
      "2019-12-04 18:49:07,278 : INFO : PROGRESS: at sentence #47270000, processed 775771887 words, keeping 470541 word types\n",
      "2019-12-04 18:49:07,542 : INFO : PROGRESS: at sentence #47280000, processed 775984022 words, keeping 477148 word types\n",
      "2019-12-04 18:49:07,797 : INFO : PROGRESS: at sentence #47290000, processed 776184682 words, keeping 483551 word types\n",
      "2019-12-04 18:49:08,057 : INFO : PROGRESS: at sentence #47300000, processed 776388276 words, keeping 490211 word types\n",
      "2019-12-04 18:49:08,318 : INFO : PROGRESS: at sentence #47310000, processed 776591347 words, keeping 496854 word types\n",
      "2019-12-04 18:49:08,737 : INFO : pruned out 326637 tokens with count <=80 (before 500005, after 173368)\n",
      "2019-12-04 18:49:08,876 : INFO : PROGRESS: at sentence #47320000, processed 776791392 words, keeping 178465 word types\n",
      "2019-12-04 18:49:09,143 : INFO : PROGRESS: at sentence #47330000, processed 776997786 words, keeping 189169 word types\n",
      "2019-12-04 18:49:09,394 : INFO : PROGRESS: at sentence #47340000, processed 777188858 words, keeping 198199 word types\n",
      "2019-12-04 18:49:09,659 : INFO : PROGRESS: at sentence #47350000, processed 777392036 words, keeping 208229 word types\n",
      "2019-12-04 18:49:09,924 : INFO : PROGRESS: at sentence #47360000, processed 777593902 words, keeping 218036 word types\n",
      "2019-12-04 18:49:10,182 : INFO : PROGRESS: at sentence #47370000, processed 777792138 words, keeping 227560 word types\n",
      "2019-12-04 18:49:10,436 : INFO : PROGRESS: at sentence #47380000, processed 777986799 words, keeping 237028 word types\n",
      "2019-12-04 18:49:10,697 : INFO : PROGRESS: at sentence #47390000, processed 778186277 words, keeping 246239 word types\n",
      "2019-12-04 18:49:10,959 : INFO : PROGRESS: at sentence #47400000, processed 778389643 words, keeping 254584 word types\n",
      "2019-12-04 18:49:11,217 : INFO : PROGRESS: at sentence #47410000, processed 778589569 words, keeping 263211 word types\n",
      "2019-12-04 18:49:11,480 : INFO : PROGRESS: at sentence #47420000, processed 778791298 words, keeping 271796 word types\n",
      "2019-12-04 18:49:11,749 : INFO : PROGRESS: at sentence #47430000, processed 778998419 words, keeping 279909 word types\n",
      "2019-12-04 18:49:12,009 : INFO : PROGRESS: at sentence #47440000, processed 779196161 words, keeping 288318 word types\n",
      "2019-12-04 18:49:12,269 : INFO : PROGRESS: at sentence #47450000, processed 779394414 words, keeping 296315 word types\n",
      "2019-12-04 18:49:12,535 : INFO : PROGRESS: at sentence #47460000, processed 779592083 words, keeping 304641 word types\n",
      "2019-12-04 18:49:12,809 : INFO : PROGRESS: at sentence #47470000, processed 779795921 words, keeping 313664 word types\n",
      "2019-12-04 18:49:13,084 : INFO : PROGRESS: at sentence #47480000, processed 779996904 words, keeping 322774 word types\n",
      "2019-12-04 18:49:13,361 : INFO : PROGRESS: at sentence #47490000, processed 780203624 words, keeping 331178 word types\n",
      "2019-12-04 18:49:13,619 : INFO : PROGRESS: at sentence #47500000, processed 780396376 words, keeping 338743 word types\n",
      "2019-12-04 18:49:13,884 : INFO : PROGRESS: at sentence #47510000, processed 780591876 words, keeping 346025 word types\n",
      "2019-12-04 18:49:14,159 : INFO : PROGRESS: at sentence #47520000, processed 780803309 words, keeping 353628 word types\n",
      "2019-12-04 18:49:14,420 : INFO : PROGRESS: at sentence #47530000, processed 781002087 words, keeping 360983 word types\n",
      "2019-12-04 18:49:14,692 : INFO : PROGRESS: at sentence #47540000, processed 781208130 words, keeping 368759 word types\n",
      "2019-12-04 18:49:14,971 : INFO : PROGRESS: at sentence #47550000, processed 781415310 words, keeping 376336 word types\n",
      "2019-12-04 18:49:15,232 : INFO : PROGRESS: at sentence #47560000, processed 781610016 words, keeping 383687 word types\n",
      "2019-12-04 18:49:15,496 : INFO : PROGRESS: at sentence #47570000, processed 781808902 words, keeping 390411 word types\n",
      "2019-12-04 18:49:15,766 : INFO : PROGRESS: at sentence #47580000, processed 782008619 words, keeping 398305 word types\n",
      "2019-12-04 18:49:16,037 : INFO : PROGRESS: at sentence #47590000, processed 782212669 words, keeping 405913 word types\n",
      "2019-12-04 18:49:16,317 : INFO : PROGRESS: at sentence #47600000, processed 782423000 words, keeping 413584 word types\n",
      "2019-12-04 18:49:16,712 : INFO : PROGRESS: at sentence #47610000, processed 782689254 words, keeping 426185 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:49:16,976 : INFO : PROGRESS: at sentence #47620000, processed 782886211 words, keeping 433413 word types\n",
      "2019-12-04 18:49:17,239 : INFO : PROGRESS: at sentence #47630000, processed 783083464 words, keeping 439828 word types\n",
      "2019-12-04 18:49:17,489 : INFO : PROGRESS: at sentence #47640000, processed 783273360 words, keeping 446178 word types\n",
      "2019-12-04 18:49:17,760 : INFO : PROGRESS: at sentence #47650000, processed 783474614 words, keeping 453035 word types\n",
      "2019-12-04 18:49:18,023 : INFO : PROGRESS: at sentence #47660000, processed 783671376 words, keeping 459315 word types\n",
      "2019-12-04 18:49:18,294 : INFO : PROGRESS: at sentence #47670000, processed 783870006 words, keeping 465337 word types\n",
      "2019-12-04 18:49:18,555 : INFO : PROGRESS: at sentence #47680000, processed 784067139 words, keeping 471968 word types\n",
      "2019-12-04 18:49:18,816 : INFO : PROGRESS: at sentence #47690000, processed 784266147 words, keeping 479007 word types\n",
      "2019-12-04 18:49:19,091 : INFO : PROGRESS: at sentence #47700000, processed 784474141 words, keeping 486889 word types\n",
      "2019-12-04 18:49:19,403 : INFO : PROGRESS: at sentence #47710000, processed 784710337 words, keeping 495824 word types\n",
      "2019-12-04 18:49:19,893 : INFO : pruned out 326791 tokens with count <=81 (before 500001, after 173210)\n",
      "2019-12-04 18:49:20,000 : INFO : PROGRESS: at sentence #47720000, processed 784918456 words, keeping 177566 word types\n",
      "2019-12-04 18:49:20,269 : INFO : PROGRESS: at sentence #47730000, processed 785119444 words, keeping 187951 word types\n",
      "2019-12-04 18:49:20,601 : INFO : PROGRESS: at sentence #47740000, processed 785371535 words, keeping 203120 word types\n",
      "2019-12-04 18:49:20,896 : INFO : PROGRESS: at sentence #47750000, processed 785591504 words, keeping 214172 word types\n",
      "2019-12-04 18:49:21,161 : INFO : PROGRESS: at sentence #47760000, processed 785786329 words, keeping 223296 word types\n",
      "2019-12-04 18:49:21,428 : INFO : PROGRESS: at sentence #47770000, processed 785983681 words, keeping 232115 word types\n",
      "2019-12-04 18:49:21,710 : INFO : PROGRESS: at sentence #47780000, processed 786194482 words, keeping 241213 word types\n",
      "2019-12-04 18:49:21,986 : INFO : PROGRESS: at sentence #47790000, processed 786401572 words, keeping 250217 word types\n",
      "2019-12-04 18:49:22,242 : INFO : PROGRESS: at sentence #47800000, processed 786596843 words, keeping 259298 word types\n",
      "2019-12-04 18:49:22,503 : INFO : PROGRESS: at sentence #47810000, processed 786796044 words, keeping 268203 word types\n",
      "2019-12-04 18:49:22,760 : INFO : PROGRESS: at sentence #47820000, processed 786992210 words, keeping 276790 word types\n",
      "2019-12-04 18:49:23,021 : INFO : PROGRESS: at sentence #47830000, processed 787188147 words, keeping 284858 word types\n",
      "2019-12-04 18:49:23,287 : INFO : PROGRESS: at sentence #47840000, processed 787387663 words, keeping 292618 word types\n",
      "2019-12-04 18:49:23,547 : INFO : PROGRESS: at sentence #47850000, processed 787577594 words, keeping 300935 word types\n",
      "2019-12-04 18:49:23,835 : INFO : PROGRESS: at sentence #47860000, processed 787776719 words, keeping 308691 word types\n",
      "2019-12-04 18:49:24,115 : INFO : PROGRESS: at sentence #47870000, processed 787979200 words, keeping 316529 word types\n",
      "2019-12-04 18:49:24,405 : INFO : PROGRESS: at sentence #47880000, processed 788190541 words, keeping 325261 word types\n",
      "2019-12-04 18:49:24,677 : INFO : PROGRESS: at sentence #47890000, processed 788383816 words, keeping 333403 word types\n",
      "2019-12-04 18:49:24,953 : INFO : PROGRESS: at sentence #47900000, processed 788578007 words, keeping 341353 word types\n",
      "2019-12-04 18:49:25,244 : INFO : PROGRESS: at sentence #47910000, processed 788781295 words, keeping 349988 word types\n",
      "2019-12-04 18:49:25,522 : INFO : PROGRESS: at sentence #47920000, processed 788981801 words, keeping 357549 word types\n",
      "2019-12-04 18:49:25,801 : INFO : PROGRESS: at sentence #47930000, processed 789188120 words, keeping 365089 word types\n",
      "2019-12-04 18:49:26,075 : INFO : PROGRESS: at sentence #47940000, processed 789390162 words, keeping 373116 word types\n",
      "2019-12-04 18:49:26,357 : INFO : PROGRESS: at sentence #47950000, processed 789596235 words, keeping 380923 word types\n",
      "2019-12-04 18:49:26,626 : INFO : PROGRESS: at sentence #47960000, processed 789798592 words, keeping 388015 word types\n",
      "2019-12-04 18:49:26,908 : INFO : PROGRESS: at sentence #47970000, processed 790010301 words, keeping 396071 word types\n",
      "2019-12-04 18:49:27,174 : INFO : PROGRESS: at sentence #47980000, processed 790207085 words, keeping 403461 word types\n",
      "2019-12-04 18:49:27,441 : INFO : PROGRESS: at sentence #47990000, processed 790405754 words, keeping 410159 word types\n",
      "2019-12-04 18:49:27,722 : INFO : PROGRESS: at sentence #48000000, processed 790605316 words, keeping 417771 word types\n",
      "2019-12-04 18:49:27,974 : INFO : PROGRESS: at sentence #48010000, processed 790791049 words, keeping 423852 word types\n",
      "2019-12-04 18:49:28,237 : INFO : PROGRESS: at sentence #48020000, processed 790992145 words, keeping 430893 word types\n",
      "2019-12-04 18:49:28,496 : INFO : PROGRESS: at sentence #48030000, processed 791192411 words, keeping 438395 word types\n",
      "2019-12-04 18:49:28,825 : INFO : PROGRESS: at sentence #48040000, processed 791397241 words, keeping 445084 word types\n",
      "2019-12-04 18:49:29,097 : INFO : PROGRESS: at sentence #48050000, processed 791607855 words, keeping 451918 word types\n",
      "2019-12-04 18:49:29,379 : INFO : PROGRESS: at sentence #48060000, processed 791826346 words, keeping 459880 word types\n",
      "2019-12-04 18:49:29,654 : INFO : PROGRESS: at sentence #48070000, processed 792038644 words, keeping 467938 word types\n",
      "2019-12-04 18:49:29,914 : INFO : PROGRESS: at sentence #48080000, processed 792240170 words, keeping 474391 word types\n",
      "2019-12-04 18:49:30,175 : INFO : PROGRESS: at sentence #48090000, processed 792441214 words, keeping 482551 word types\n",
      "2019-12-04 18:49:30,439 : INFO : PROGRESS: at sentence #48100000, processed 792641575 words, keeping 488789 word types\n",
      "2019-12-04 18:49:30,699 : INFO : PROGRESS: at sentence #48110000, processed 792847648 words, keeping 495420 word types\n",
      "2019-12-04 18:49:31,160 : INFO : pruned out 326932 tokens with count <=82 (before 500009, after 173077)\n",
      "2019-12-04 18:49:31,245 : INFO : PROGRESS: at sentence #48120000, processed 793051954 words, keeping 176705 word types\n",
      "2019-12-04 18:49:31,537 : INFO : PROGRESS: at sentence #48130000, processed 793279442 words, keeping 188950 word types\n",
      "2019-12-04 18:49:31,802 : INFO : PROGRESS: at sentence #48140000, processed 793487670 words, keeping 199252 word types\n",
      "2019-12-04 18:49:32,060 : INFO : PROGRESS: at sentence #48150000, processed 793690208 words, keeping 209026 word types\n",
      "2019-12-04 18:49:32,330 : INFO : PROGRESS: at sentence #48160000, processed 793902317 words, keeping 218970 word types\n",
      "2019-12-04 18:49:32,608 : INFO : PROGRESS: at sentence #48170000, processed 794099537 words, keeping 229081 word types\n",
      "2019-12-04 18:49:32,881 : INFO : PROGRESS: at sentence #48180000, processed 794309146 words, keeping 239049 word types\n",
      "2019-12-04 18:49:33,146 : INFO : PROGRESS: at sentence #48190000, processed 794512664 words, keeping 247967 word types\n",
      "2019-12-04 18:49:33,405 : INFO : PROGRESS: at sentence #48200000, processed 794713980 words, keeping 256854 word types\n",
      "2019-12-04 18:49:33,656 : INFO : PROGRESS: at sentence #48210000, processed 794908849 words, keeping 265302 word types\n",
      "2019-12-04 18:49:33,907 : INFO : PROGRESS: at sentence #48220000, processed 795105756 words, keeping 273785 word types\n",
      "2019-12-04 18:49:34,158 : INFO : PROGRESS: at sentence #48230000, processed 795305567 words, keeping 283249 word types\n",
      "2019-12-04 18:49:34,424 : INFO : PROGRESS: at sentence #48240000, processed 795516572 words, keeping 292830 word types\n",
      "2019-12-04 18:49:34,716 : INFO : PROGRESS: at sentence #48250000, processed 795752137 words, keeping 303927 word types\n",
      "2019-12-04 18:49:34,969 : INFO : PROGRESS: at sentence #48260000, processed 795951768 words, keeping 311816 word types\n",
      "2019-12-04 18:49:35,221 : INFO : PROGRESS: at sentence #48270000, processed 796150856 words, keeping 319376 word types\n",
      "2019-12-04 18:49:35,481 : INFO : PROGRESS: at sentence #48280000, processed 796351574 words, keeping 327311 word types\n",
      "2019-12-04 18:49:35,752 : INFO : PROGRESS: at sentence #48290000, processed 796563651 words, keeping 336368 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:49:36,021 : INFO : PROGRESS: at sentence #48300000, processed 796777579 words, keeping 344510 word types\n",
      "2019-12-04 18:49:36,273 : INFO : PROGRESS: at sentence #48310000, processed 796974536 words, keeping 352041 word types\n",
      "2019-12-04 18:49:36,562 : INFO : PROGRESS: at sentence #48320000, processed 797204261 words, keeping 362049 word types\n",
      "2019-12-04 18:49:36,811 : INFO : PROGRESS: at sentence #48330000, processed 797396514 words, keeping 369283 word types\n",
      "2019-12-04 18:49:37,064 : INFO : PROGRESS: at sentence #48340000, processed 797595479 words, keeping 377343 word types\n",
      "2019-12-04 18:49:37,318 : INFO : PROGRESS: at sentence #48350000, processed 797798361 words, keeping 385136 word types\n",
      "2019-12-04 18:49:37,589 : INFO : PROGRESS: at sentence #48360000, processed 798013279 words, keeping 392983 word types\n",
      "2019-12-04 18:49:37,873 : INFO : PROGRESS: at sentence #48370000, processed 798229373 words, keeping 400696 word types\n",
      "2019-12-04 18:49:38,127 : INFO : PROGRESS: at sentence #48380000, processed 798435194 words, keeping 407836 word types\n",
      "2019-12-04 18:49:38,387 : INFO : PROGRESS: at sentence #48390000, processed 798637781 words, keeping 414788 word types\n",
      "2019-12-04 18:49:38,640 : INFO : PROGRESS: at sentence #48400000, processed 798833435 words, keeping 422442 word types\n",
      "2019-12-04 18:49:38,897 : INFO : PROGRESS: at sentence #48410000, processed 799033910 words, keeping 429811 word types\n",
      "2019-12-04 18:49:39,138 : INFO : PROGRESS: at sentence #48420000, processed 799224936 words, keeping 436376 word types\n",
      "2019-12-04 18:49:39,366 : INFO : PROGRESS: at sentence #48430000, processed 799407202 words, keeping 441970 word types\n",
      "2019-12-04 18:49:39,611 : INFO : PROGRESS: at sentence #48440000, processed 799596739 words, keeping 448089 word types\n",
      "2019-12-04 18:49:39,855 : INFO : PROGRESS: at sentence #48450000, processed 799783185 words, keeping 453578 word types\n",
      "2019-12-04 18:49:40,098 : INFO : PROGRESS: at sentence #48460000, processed 799973191 words, keeping 459677 word types\n",
      "2019-12-04 18:49:40,360 : INFO : PROGRESS: at sentence #48470000, processed 800175734 words, keeping 466596 word types\n",
      "2019-12-04 18:49:40,628 : INFO : PROGRESS: at sentence #48480000, processed 800383990 words, keeping 473943 word types\n",
      "2019-12-04 18:49:40,877 : INFO : PROGRESS: at sentence #48490000, processed 800577050 words, keeping 480274 word types\n",
      "2019-12-04 18:49:41,120 : INFO : PROGRESS: at sentence #48500000, processed 800767818 words, keeping 485980 word types\n",
      "2019-12-04 18:49:41,387 : INFO : PROGRESS: at sentence #48510000, processed 800980360 words, keeping 493106 word types\n",
      "2019-12-04 18:49:41,640 : INFO : PROGRESS: at sentence #48520000, processed 801178597 words, keeping 499430 word types\n",
      "2019-12-04 18:49:41,953 : INFO : pruned out 327065 tokens with count <=83 (before 500001, after 172936)\n",
      "2019-12-04 18:49:42,187 : INFO : PROGRESS: at sentence #48530000, processed 801377759 words, keeping 182396 word types\n",
      "2019-12-04 18:49:42,453 : INFO : PROGRESS: at sentence #48540000, processed 801584344 words, keeping 192995 word types\n",
      "2019-12-04 18:49:42,716 : INFO : PROGRESS: at sentence #48550000, processed 801787690 words, keeping 204305 word types\n",
      "2019-12-04 18:49:42,973 : INFO : PROGRESS: at sentence #48560000, processed 801986864 words, keeping 213509 word types\n",
      "2019-12-04 18:49:43,214 : INFO : PROGRESS: at sentence #48570000, processed 802172433 words, keeping 221354 word types\n",
      "2019-12-04 18:49:43,448 : INFO : PROGRESS: at sentence #48580000, processed 802352667 words, keeping 228631 word types\n",
      "2019-12-04 18:49:43,702 : INFO : PROGRESS: at sentence #48590000, processed 802547419 words, keeping 236908 word types\n",
      "2019-12-04 18:49:43,976 : INFO : PROGRESS: at sentence #48600000, processed 802746403 words, keeping 245606 word types\n",
      "2019-12-04 18:49:44,245 : INFO : PROGRESS: at sentence #48610000, processed 802950966 words, keeping 254614 word types\n",
      "2019-12-04 18:49:44,519 : INFO : PROGRESS: at sentence #48620000, processed 803156379 words, keeping 263891 word types\n",
      "2019-12-04 18:49:44,820 : INFO : PROGRESS: at sentence #48630000, processed 803357638 words, keeping 273075 word types\n",
      "2019-12-04 18:49:45,088 : INFO : PROGRESS: at sentence #48640000, processed 803565545 words, keeping 281662 word types\n",
      "2019-12-04 18:49:45,367 : INFO : PROGRESS: at sentence #48650000, processed 803785063 words, keeping 291175 word types\n",
      "2019-12-04 18:49:45,647 : INFO : PROGRESS: at sentence #48660000, processed 804002582 words, keeping 300617 word types\n",
      "2019-12-04 18:49:45,902 : INFO : PROGRESS: at sentence #48670000, processed 804199082 words, keeping 308586 word types\n",
      "2019-12-04 18:49:46,169 : INFO : PROGRESS: at sentence #48680000, processed 804406316 words, keeping 317291 word types\n",
      "2019-12-04 18:49:46,442 : INFO : PROGRESS: at sentence #48690000, processed 804616470 words, keeping 326310 word types\n",
      "2019-12-04 18:49:46,694 : INFO : PROGRESS: at sentence #48700000, processed 804813971 words, keeping 332585 word types\n",
      "2019-12-04 18:49:46,991 : INFO : PROGRESS: at sentence #48710000, processed 805045145 words, keeping 343137 word types\n",
      "2019-12-04 18:49:47,246 : INFO : PROGRESS: at sentence #48720000, processed 805245666 words, keeping 351429 word types\n",
      "2019-12-04 18:49:47,517 : INFO : PROGRESS: at sentence #48730000, processed 805459968 words, keeping 361011 word types\n",
      "2019-12-04 18:49:47,793 : INFO : PROGRESS: at sentence #48740000, processed 805676706 words, keeping 372294 word types\n",
      "2019-12-04 18:49:48,079 : INFO : PROGRESS: at sentence #48750000, processed 805903104 words, keeping 382666 word types\n",
      "2019-12-04 18:49:48,329 : INFO : PROGRESS: at sentence #48760000, processed 806096238 words, keeping 390636 word types\n",
      "2019-12-04 18:49:48,581 : INFO : PROGRESS: at sentence #48770000, processed 806294090 words, keeping 397752 word types\n",
      "2019-12-04 18:49:48,831 : INFO : PROGRESS: at sentence #48780000, processed 806490422 words, keeping 404999 word types\n",
      "2019-12-04 18:49:49,100 : INFO : PROGRESS: at sentence #48790000, processed 806705250 words, keeping 413190 word types\n",
      "2019-12-04 18:49:49,409 : INFO : PROGRESS: at sentence #48800000, processed 806905353 words, keeping 420749 word types\n",
      "2019-12-04 18:49:49,686 : INFO : PROGRESS: at sentence #48810000, processed 807121116 words, keeping 430072 word types\n",
      "2019-12-04 18:49:49,947 : INFO : PROGRESS: at sentence #48820000, processed 807325883 words, keeping 437783 word types\n",
      "2019-12-04 18:49:50,195 : INFO : PROGRESS: at sentence #48830000, processed 807520547 words, keeping 444240 word types\n",
      "2019-12-04 18:49:50,455 : INFO : PROGRESS: at sentence #48840000, processed 807723065 words, keeping 451153 word types\n",
      "2019-12-04 18:49:50,722 : INFO : PROGRESS: at sentence #48850000, processed 807933616 words, keeping 460583 word types\n",
      "2019-12-04 18:49:51,000 : INFO : PROGRESS: at sentence #48860000, processed 808142067 words, keeping 468964 word types\n",
      "2019-12-04 18:49:51,263 : INFO : PROGRESS: at sentence #48870000, processed 808343880 words, keeping 476153 word types\n",
      "2019-12-04 18:49:51,528 : INFO : PROGRESS: at sentence #48880000, processed 808545261 words, keeping 483156 word types\n",
      "2019-12-04 18:49:51,789 : INFO : PROGRESS: at sentence #48890000, processed 808750529 words, keeping 489966 word types\n",
      "2019-12-04 18:49:52,051 : INFO : PROGRESS: at sentence #48900000, processed 808953091 words, keeping 497831 word types\n",
      "2019-12-04 18:49:52,428 : INFO : pruned out 327218 tokens with count <=84 (before 500002, after 172784)\n",
      "2019-12-04 18:49:52,605 : INFO : PROGRESS: at sentence #48910000, processed 809150163 words, keeping 180235 word types\n",
      "2019-12-04 18:49:52,883 : INFO : PROGRESS: at sentence #48920000, processed 809366714 words, keeping 193729 word types\n",
      "2019-12-04 18:49:53,157 : INFO : PROGRESS: at sentence #48930000, processed 809574488 words, keeping 204568 word types\n",
      "2019-12-04 18:49:53,431 : INFO : PROGRESS: at sentence #48940000, processed 809787328 words, keeping 214845 word types\n",
      "2019-12-04 18:49:53,693 : INFO : PROGRESS: at sentence #48950000, processed 809988005 words, keeping 224485 word types\n",
      "2019-12-04 18:49:53,949 : INFO : PROGRESS: at sentence #48960000, processed 810184615 words, keeping 233633 word types\n",
      "2019-12-04 18:49:54,210 : INFO : PROGRESS: at sentence #48970000, processed 810386064 words, keeping 243822 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:49:54,499 : INFO : PROGRESS: at sentence #48980000, processed 810617589 words, keeping 254103 word types\n",
      "2019-12-04 18:49:54,765 : INFO : PROGRESS: at sentence #48990000, processed 810819354 words, keeping 263733 word types\n",
      "2019-12-04 18:49:55,035 : INFO : PROGRESS: at sentence #49000000, processed 811022789 words, keeping 273411 word types\n",
      "2019-12-04 18:49:55,298 : INFO : PROGRESS: at sentence #49010000, processed 811226057 words, keeping 282581 word types\n",
      "2019-12-04 18:49:55,560 : INFO : PROGRESS: at sentence #49020000, processed 811427741 words, keeping 291246 word types\n",
      "2019-12-04 18:49:55,823 : INFO : PROGRESS: at sentence #49030000, processed 811632291 words, keeping 300563 word types\n",
      "2019-12-04 18:49:56,083 : INFO : PROGRESS: at sentence #49040000, processed 811838244 words, keeping 309084 word types\n",
      "2019-12-04 18:49:56,328 : INFO : PROGRESS: at sentence #49050000, processed 812025739 words, keeping 316558 word types\n",
      "2019-12-04 18:49:56,583 : INFO : PROGRESS: at sentence #49060000, processed 812224246 words, keeping 324150 word types\n",
      "2019-12-04 18:49:56,838 : INFO : PROGRESS: at sentence #49070000, processed 812424978 words, keeping 331871 word types\n",
      "2019-12-04 18:49:57,098 : INFO : PROGRESS: at sentence #49080000, processed 812630093 words, keeping 339537 word types\n",
      "2019-12-04 18:49:57,369 : INFO : PROGRESS: at sentence #49090000, processed 812847946 words, keeping 347753 word types\n",
      "2019-12-04 18:49:57,626 : INFO : PROGRESS: at sentence #49100000, processed 813050064 words, keeping 356631 word types\n",
      "2019-12-04 18:49:57,875 : INFO : PROGRESS: at sentence #49110000, processed 813243866 words, keeping 364433 word types\n",
      "2019-12-04 18:49:58,128 : INFO : PROGRESS: at sentence #49120000, processed 813450219 words, keeping 372376 word types\n",
      "2019-12-04 18:49:58,383 : INFO : PROGRESS: at sentence #49130000, processed 813659041 words, keeping 380489 word types\n",
      "2019-12-04 18:49:58,628 : INFO : PROGRESS: at sentence #49140000, processed 813853437 words, keeping 388021 word types\n",
      "2019-12-04 18:49:58,873 : INFO : PROGRESS: at sentence #49150000, processed 814044274 words, keeping 395774 word types\n",
      "2019-12-04 18:49:59,130 : INFO : PROGRESS: at sentence #49160000, processed 814244773 words, keeping 403572 word types\n",
      "2019-12-04 18:49:59,403 : INFO : PROGRESS: at sentence #49170000, processed 814468055 words, keeping 413017 word types\n",
      "2019-12-04 18:49:59,693 : INFO : PROGRESS: at sentence #49180000, processed 814697970 words, keeping 422171 word types\n",
      "2019-12-04 18:49:59,965 : INFO : PROGRESS: at sentence #49190000, processed 814912745 words, keeping 430188 word types\n",
      "2019-12-04 18:50:00,217 : INFO : PROGRESS: at sentence #49200000, processed 815105494 words, keeping 437456 word types\n",
      "2019-12-04 18:50:00,474 : INFO : PROGRESS: at sentence #49210000, processed 815300788 words, keeping 444653 word types\n",
      "2019-12-04 18:50:00,708 : INFO : PROGRESS: at sentence #49220000, processed 815487782 words, keeping 451107 word types\n",
      "2019-12-04 18:50:00,970 : INFO : PROGRESS: at sentence #49230000, processed 815697995 words, keeping 458328 word types\n",
      "2019-12-04 18:50:01,220 : INFO : PROGRESS: at sentence #49240000, processed 815894420 words, keeping 465148 word types\n",
      "2019-12-04 18:50:01,493 : INFO : PROGRESS: at sentence #49250000, processed 816124316 words, keeping 475107 word types\n",
      "2019-12-04 18:50:01,763 : INFO : PROGRESS: at sentence #49260000, processed 816332340 words, keeping 482010 word types\n",
      "2019-12-04 18:50:02,021 : INFO : PROGRESS: at sentence #49270000, processed 816541178 words, keeping 489339 word types\n",
      "2019-12-04 18:50:02,274 : INFO : PROGRESS: at sentence #49280000, processed 816751446 words, keeping 497039 word types\n",
      "2019-12-04 18:50:02,666 : INFO : pruned out 327386 tokens with count <=85 (before 500001, after 172615)\n",
      "2019-12-04 18:50:02,828 : INFO : PROGRESS: at sentence #49290000, processed 816971227 words, keeping 180042 word types\n",
      "2019-12-04 18:50:03,098 : INFO : PROGRESS: at sentence #49300000, processed 817185016 words, keeping 193447 word types\n",
      "2019-12-04 18:50:03,373 : INFO : PROGRESS: at sentence #49310000, processed 817397675 words, keeping 204193 word types\n",
      "2019-12-04 18:50:03,628 : INFO : PROGRESS: at sentence #49320000, processed 817596046 words, keeping 214656 word types\n",
      "2019-12-04 18:50:03,900 : INFO : PROGRESS: at sentence #49330000, processed 817805395 words, keeping 224968 word types\n",
      "2019-12-04 18:50:04,185 : INFO : PROGRESS: at sentence #49340000, processed 818027102 words, keeping 236283 word types\n",
      "2019-12-04 18:50:04,469 : INFO : PROGRESS: at sentence #49350000, processed 818257000 words, keeping 247892 word types\n",
      "2019-12-04 18:50:04,730 : INFO : PROGRESS: at sentence #49360000, processed 818467788 words, keeping 258152 word types\n",
      "2019-12-04 18:50:04,996 : INFO : PROGRESS: at sentence #49370000, processed 818680076 words, keeping 268682 word types\n",
      "2019-12-04 18:50:05,246 : INFO : PROGRESS: at sentence #49380000, processed 818874202 words, keeping 277577 word types\n",
      "2019-12-04 18:50:05,497 : INFO : PROGRESS: at sentence #49390000, processed 819073707 words, keeping 286423 word types\n",
      "2019-12-04 18:50:05,756 : INFO : PROGRESS: at sentence #49400000, processed 819278425 words, keeping 296110 word types\n",
      "2019-12-04 18:50:06,008 : INFO : PROGRESS: at sentence #49410000, processed 819486641 words, keeping 304215 word types\n",
      "2019-12-04 18:50:06,253 : INFO : PROGRESS: at sentence #49420000, processed 819685967 words, keeping 312592 word types\n",
      "2019-12-04 18:50:06,494 : INFO : PROGRESS: at sentence #49430000, processed 819880326 words, keeping 320857 word types\n",
      "2019-12-04 18:50:06,755 : INFO : PROGRESS: at sentence #49440000, processed 820096124 words, keeping 329479 word types\n",
      "2019-12-04 18:50:07,007 : INFO : PROGRESS: at sentence #49450000, processed 820299286 words, keeping 337071 word types\n",
      "2019-12-04 18:50:07,284 : INFO : PROGRESS: at sentence #49460000, processed 820525018 words, keeping 346762 word types\n",
      "2019-12-04 18:50:07,539 : INFO : PROGRESS: at sentence #49470000, processed 820733163 words, keeping 356969 word types\n",
      "2019-12-04 18:50:07,790 : INFO : PROGRESS: at sentence #49480000, processed 820930470 words, keeping 363909 word types\n",
      "2019-12-04 18:50:08,053 : INFO : PROGRESS: at sentence #49490000, processed 821137031 words, keeping 371756 word types\n",
      "2019-12-04 18:50:08,325 : INFO : PROGRESS: at sentence #49500000, processed 821353815 words, keeping 380132 word types\n",
      "2019-12-04 18:50:08,586 : INFO : PROGRESS: at sentence #49510000, processed 821563275 words, keeping 387970 word types\n",
      "2019-12-04 18:50:08,849 : INFO : PROGRESS: at sentence #49520000, processed 821772826 words, keeping 395448 word types\n",
      "2019-12-04 18:50:09,109 : INFO : PROGRESS: at sentence #49530000, processed 821976906 words, keeping 403371 word types\n",
      "2019-12-04 18:50:09,391 : INFO : PROGRESS: at sentence #49540000, processed 822190469 words, keeping 410709 word types\n",
      "2019-12-04 18:50:09,647 : INFO : PROGRESS: at sentence #49550000, processed 822396275 words, keeping 418038 word types\n",
      "2019-12-04 18:50:09,951 : INFO : PROGRESS: at sentence #49560000, processed 822606774 words, keeping 426993 word types\n",
      "2019-12-04 18:50:10,212 : INFO : PROGRESS: at sentence #49570000, processed 822813031 words, keeping 434669 word types\n",
      "2019-12-04 18:50:10,471 : INFO : PROGRESS: at sentence #49580000, processed 823019894 words, keeping 441981 word types\n",
      "2019-12-04 18:50:10,731 : INFO : PROGRESS: at sentence #49590000, processed 823224651 words, keeping 449526 word types\n",
      "2019-12-04 18:50:10,981 : INFO : PROGRESS: at sentence #49600000, processed 823421331 words, keeping 455549 word types\n",
      "2019-12-04 18:50:11,286 : INFO : PROGRESS: at sentence #49610000, processed 823668526 words, keeping 466766 word types\n",
      "2019-12-04 18:50:11,571 : INFO : PROGRESS: at sentence #49620000, processed 823895020 words, keeping 474627 word types\n",
      "2019-12-04 18:50:11,944 : INFO : PROGRESS: at sentence #49630000, processed 824201797 words, keeping 488622 word types\n",
      "2019-12-04 18:50:12,239 : INFO : PROGRESS: at sentence #49640000, processed 824432616 words, keeping 497355 word types\n",
      "2019-12-04 18:50:12,607 : INFO : pruned out 327519 tokens with count <=86 (before 500002, after 172483)\n",
      "2019-12-04 18:50:12,844 : INFO : PROGRESS: at sentence #49650000, processed 824661037 words, keeping 182730 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:50:13,129 : INFO : PROGRESS: at sentence #49660000, processed 824877473 words, keeping 193349 word types\n",
      "2019-12-04 18:50:13,391 : INFO : PROGRESS: at sentence #49670000, processed 825080844 words, keeping 203028 word types\n",
      "2019-12-04 18:50:13,661 : INFO : PROGRESS: at sentence #49680000, processed 825292272 words, keeping 212853 word types\n",
      "2019-12-04 18:50:13,932 : INFO : PROGRESS: at sentence #49690000, processed 825502516 words, keeping 223114 word types\n",
      "2019-12-04 18:50:14,205 : INFO : PROGRESS: at sentence #49700000, processed 825717867 words, keeping 233048 word types\n",
      "2019-12-04 18:50:14,481 : INFO : PROGRESS: at sentence #49710000, processed 825925958 words, keeping 242631 word types\n",
      "2019-12-04 18:50:14,737 : INFO : PROGRESS: at sentence #49720000, processed 826126134 words, keeping 251540 word types\n",
      "2019-12-04 18:50:14,993 : INFO : PROGRESS: at sentence #49730000, processed 826327017 words, keeping 261873 word types\n",
      "2019-12-04 18:50:15,260 : INFO : PROGRESS: at sentence #49740000, processed 826533437 words, keeping 271027 word types\n",
      "2019-12-04 18:50:15,537 : INFO : PROGRESS: at sentence #49750000, processed 826752959 words, keeping 281098 word types\n",
      "2019-12-04 18:50:15,814 : INFO : PROGRESS: at sentence #49760000, processed 826970505 words, keeping 290644 word types\n",
      "2019-12-04 18:50:16,109 : INFO : PROGRESS: at sentence #49770000, processed 827211756 words, keeping 302647 word types\n",
      "2019-12-04 18:50:16,384 : INFO : PROGRESS: at sentence #49780000, processed 827430575 words, keeping 313220 word types\n",
      "2019-12-04 18:50:16,634 : INFO : PROGRESS: at sentence #49790000, processed 827621336 words, keeping 320686 word types\n",
      "2019-12-04 18:50:16,899 : INFO : PROGRESS: at sentence #49800000, processed 827826774 words, keeping 332030 word types\n",
      "2019-12-04 18:50:17,159 : INFO : PROGRESS: at sentence #49810000, processed 828026801 words, keeping 340830 word types\n",
      "2019-12-04 18:50:17,432 : INFO : PROGRESS: at sentence #49820000, processed 828239053 words, keeping 348863 word types\n",
      "2019-12-04 18:50:17,705 : INFO : PROGRESS: at sentence #49830000, processed 828455602 words, keeping 356921 word types\n",
      "2019-12-04 18:50:17,973 : INFO : PROGRESS: at sentence #49840000, processed 828664277 words, keeping 365890 word types\n",
      "2019-12-04 18:50:18,251 : INFO : PROGRESS: at sentence #49850000, processed 828883211 words, keeping 375318 word types\n",
      "2019-12-04 18:50:18,497 : INFO : PROGRESS: at sentence #49860000, processed 829071276 words, keeping 383062 word types\n",
      "2019-12-04 18:50:18,751 : INFO : PROGRESS: at sentence #49870000, processed 829271313 words, keeping 391042 word types\n",
      "2019-12-04 18:50:19,029 : INFO : PROGRESS: at sentence #49880000, processed 829486884 words, keeping 398452 word types\n",
      "2019-12-04 18:50:19,277 : INFO : PROGRESS: at sentence #49890000, processed 829681075 words, keeping 405612 word types\n",
      "2019-12-04 18:50:19,528 : INFO : PROGRESS: at sentence #49900000, processed 829879082 words, keeping 412567 word types\n",
      "2019-12-04 18:50:19,837 : INFO : PROGRESS: at sentence #49910000, processed 830077990 words, keeping 419543 word types\n",
      "2019-12-04 18:50:20,094 : INFO : PROGRESS: at sentence #49920000, processed 830282592 words, keeping 427265 word types\n",
      "2019-12-04 18:50:20,351 : INFO : PROGRESS: at sentence #49930000, processed 830484778 words, keeping 434618 word types\n",
      "2019-12-04 18:50:20,606 : INFO : PROGRESS: at sentence #49940000, processed 830684381 words, keeping 441864 word types\n",
      "2019-12-04 18:50:20,875 : INFO : PROGRESS: at sentence #49950000, processed 830893368 words, keeping 449724 word types\n",
      "2019-12-04 18:50:21,130 : INFO : PROGRESS: at sentence #49960000, processed 831096860 words, keeping 457009 word types\n",
      "2019-12-04 18:50:21,378 : INFO : PROGRESS: at sentence #49970000, processed 831291594 words, keeping 463453 word types\n",
      "2019-12-04 18:50:21,641 : INFO : PROGRESS: at sentence #49980000, processed 831498785 words, keeping 471932 word types\n",
      "2019-12-04 18:50:21,902 : INFO : PROGRESS: at sentence #49990000, processed 831703157 words, keeping 479096 word types\n",
      "2019-12-04 18:50:22,155 : INFO : PROGRESS: at sentence #50000000, processed 831900695 words, keeping 485558 word types\n",
      "2019-12-04 18:50:22,409 : INFO : PROGRESS: at sentence #50010000, processed 832100351 words, keeping 492102 word types\n",
      "2019-12-04 18:50:22,670 : INFO : PROGRESS: at sentence #50020000, processed 832304923 words, keeping 498875 word types\n",
      "2019-12-04 18:50:23,020 : INFO : pruned out 327637 tokens with count <=87 (before 500001, after 172364)\n",
      "2019-12-04 18:50:23,222 : INFO : PROGRESS: at sentence #50030000, processed 832501183 words, keeping 180355 word types\n",
      "2019-12-04 18:50:23,491 : INFO : PROGRESS: at sentence #50040000, processed 832713931 words, keeping 190850 word types\n",
      "2019-12-04 18:50:23,765 : INFO : PROGRESS: at sentence #50050000, processed 832921923 words, keeping 200367 word types\n",
      "2019-12-04 18:50:24,014 : INFO : PROGRESS: at sentence #50060000, processed 833111819 words, keeping 210842 word types\n",
      "2019-12-04 18:50:24,279 : INFO : PROGRESS: at sentence #50070000, processed 833313876 words, keeping 220612 word types\n",
      "2019-12-04 18:50:24,537 : INFO : PROGRESS: at sentence #50080000, processed 833514759 words, keeping 230733 word types\n",
      "2019-12-04 18:50:24,796 : INFO : PROGRESS: at sentence #50090000, processed 833722253 words, keeping 241045 word types\n",
      "2019-12-04 18:50:25,058 : INFO : PROGRESS: at sentence #50100000, processed 833925891 words, keeping 249899 word types\n",
      "2019-12-04 18:50:25,320 : INFO : PROGRESS: at sentence #50110000, processed 834135518 words, keeping 259505 word types\n",
      "2019-12-04 18:50:25,590 : INFO : PROGRESS: at sentence #50120000, processed 834348227 words, keeping 269179 word types\n",
      "2019-12-04 18:50:25,841 : INFO : PROGRESS: at sentence #50130000, processed 834546900 words, keeping 277947 word types\n",
      "2019-12-04 18:50:26,097 : INFO : PROGRESS: at sentence #50140000, processed 834751556 words, keeping 286447 word types\n",
      "2019-12-04 18:50:26,382 : INFO : PROGRESS: at sentence #50150000, processed 834976888 words, keeping 296509 word types\n",
      "2019-12-04 18:50:26,639 : INFO : PROGRESS: at sentence #50160000, processed 835181287 words, keeping 304716 word types\n",
      "2019-12-04 18:50:26,924 : INFO : PROGRESS: at sentence #50170000, processed 835387408 words, keeping 312834 word types\n",
      "2019-12-04 18:50:27,213 : INFO : PROGRESS: at sentence #50180000, processed 835613511 words, keeping 323347 word types\n",
      "2019-12-04 18:50:27,495 : INFO : PROGRESS: at sentence #50190000, processed 835835038 words, keeping 332566 word types\n",
      "2019-12-04 18:50:27,754 : INFO : PROGRESS: at sentence #50200000, processed 836037534 words, keeping 340666 word types\n",
      "2019-12-04 18:50:28,021 : INFO : PROGRESS: at sentence #50210000, processed 836239439 words, keeping 349369 word types\n",
      "2019-12-04 18:50:28,267 : INFO : PROGRESS: at sentence #50220000, processed 836428660 words, keeping 357077 word types\n",
      "2019-12-04 18:50:28,520 : INFO : PROGRESS: at sentence #50230000, processed 836628888 words, keeping 363967 word types\n",
      "2019-12-04 18:50:28,792 : INFO : PROGRESS: at sentence #50240000, processed 836833558 words, keeping 371261 word types\n",
      "2019-12-04 18:50:29,080 : INFO : PROGRESS: at sentence #50250000, processed 837035907 words, keeping 379027 word types\n",
      "2019-12-04 18:50:29,329 : INFO : PROGRESS: at sentence #50260000, processed 837234378 words, keeping 386073 word types\n",
      "2019-12-04 18:50:29,596 : INFO : PROGRESS: at sentence #50270000, processed 837445600 words, keeping 394026 word types\n",
      "2019-12-04 18:50:29,890 : INFO : PROGRESS: at sentence #50280000, processed 837649205 words, keeping 400702 word types\n",
      "2019-12-04 18:50:30,145 : INFO : PROGRESS: at sentence #50290000, processed 837844724 words, keeping 407879 word types\n",
      "2019-12-04 18:50:30,371 : INFO : PROGRESS: at sentence #50300000, processed 838019266 words, keeping 413874 word types\n",
      "2019-12-04 18:50:30,620 : INFO : PROGRESS: at sentence #50310000, processed 838208461 words, keeping 420504 word types\n",
      "2019-12-04 18:50:30,890 : INFO : PROGRESS: at sentence #50320000, processed 838417856 words, keeping 428608 word types\n",
      "2019-12-04 18:50:31,152 : INFO : PROGRESS: at sentence #50330000, processed 838621806 words, keeping 436352 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:50:31,408 : INFO : PROGRESS: at sentence #50340000, processed 838817331 words, keeping 442938 word types\n",
      "2019-12-04 18:50:31,655 : INFO : PROGRESS: at sentence #50350000, processed 839010038 words, keeping 448676 word types\n",
      "2019-12-04 18:50:31,917 : INFO : PROGRESS: at sentence #50360000, processed 839214885 words, keeping 455046 word types\n",
      "2019-12-04 18:50:32,172 : INFO : PROGRESS: at sentence #50370000, processed 839414901 words, keeping 461125 word types\n",
      "2019-12-04 18:50:32,428 : INFO : PROGRESS: at sentence #50380000, processed 839614799 words, keeping 466854 word types\n",
      "2019-12-04 18:50:32,696 : INFO : PROGRESS: at sentence #50390000, processed 839825169 words, keeping 473841 word types\n",
      "2019-12-04 18:50:32,982 : INFO : PROGRESS: at sentence #50400000, processed 840027456 words, keeping 481133 word types\n",
      "2019-12-04 18:50:33,257 : INFO : PROGRESS: at sentence #50410000, processed 840234532 words, keeping 488033 word types\n",
      "2019-12-04 18:50:33,541 : INFO : PROGRESS: at sentence #50420000, processed 840455737 words, keeping 495548 word types\n",
      "2019-12-04 18:50:34,023 : INFO : pruned out 327847 tokens with count <=88 (before 500082, after 172235)\n",
      "2019-12-04 18:50:34,127 : INFO : PROGRESS: at sentence #50430000, processed 840675962 words, keeping 176296 word types\n",
      "2019-12-04 18:50:34,397 : INFO : PROGRESS: at sentence #50440000, processed 840878554 words, keeping 187372 word types\n",
      "2019-12-04 18:50:34,665 : INFO : PROGRESS: at sentence #50450000, processed 841081295 words, keeping 196932 word types\n",
      "2019-12-04 18:50:34,928 : INFO : PROGRESS: at sentence #50460000, processed 841280550 words, keeping 207404 word types\n",
      "2019-12-04 18:50:35,196 : INFO : PROGRESS: at sentence #50470000, processed 841485080 words, keeping 217223 word types\n",
      "2019-12-04 18:50:35,473 : INFO : PROGRESS: at sentence #50480000, processed 841699152 words, keeping 227283 word types\n",
      "2019-12-04 18:50:35,741 : INFO : PROGRESS: at sentence #50490000, processed 841899790 words, keeping 236221 word types\n",
      "2019-12-04 18:50:36,018 : INFO : PROGRESS: at sentence #50500000, processed 842109925 words, keeping 245966 word types\n",
      "2019-12-04 18:50:36,287 : INFO : PROGRESS: at sentence #50510000, processed 842316963 words, keeping 255101 word types\n",
      "2019-12-04 18:50:36,549 : INFO : PROGRESS: at sentence #50520000, processed 842517685 words, keeping 264048 word types\n",
      "2019-12-04 18:50:36,812 : INFO : PROGRESS: at sentence #50530000, processed 842716372 words, keeping 272640 word types\n",
      "2019-12-04 18:50:37,043 : INFO : PROGRESS: at sentence #50540000, processed 842891698 words, keeping 279222 word types\n",
      "2019-12-04 18:50:37,268 : INFO : PROGRESS: at sentence #50550000, processed 843062536 words, keeping 285884 word types\n",
      "2019-12-04 18:50:37,531 : INFO : PROGRESS: at sentence #50560000, processed 843265246 words, keeping 294231 word types\n",
      "2019-12-04 18:50:37,803 : INFO : PROGRESS: at sentence #50570000, processed 843467671 words, keeping 303228 word types\n",
      "2019-12-04 18:50:38,065 : INFO : PROGRESS: at sentence #50580000, processed 843668757 words, keeping 311597 word types\n",
      "2019-12-04 18:50:38,329 : INFO : PROGRESS: at sentence #50590000, processed 843870265 words, keeping 320761 word types\n",
      "2019-12-04 18:50:38,608 : INFO : PROGRESS: at sentence #50600000, processed 844085241 words, keeping 329921 word types\n",
      "2019-12-04 18:50:38,875 : INFO : PROGRESS: at sentence #50610000, processed 844290015 words, keeping 337617 word types\n",
      "2019-12-04 18:50:39,148 : INFO : PROGRESS: at sentence #50620000, processed 844497218 words, keeping 346711 word types\n",
      "2019-12-04 18:50:39,362 : INFO : PROGRESS: at sentence #50630000, processed 844655143 words, keeping 352175 word types\n",
      "2019-12-04 18:50:39,509 : INFO : PROGRESS: at sentence #50640000, processed 844760310 words, keeping 353680 word types\n",
      "2019-12-04 18:50:39,655 : INFO : PROGRESS: at sentence #50650000, processed 844864834 words, keeping 355170 word types\n",
      "2019-12-04 18:50:39,799 : INFO : PROGRESS: at sentence #50660000, processed 844970160 words, keeping 355762 word types\n",
      "2019-12-04 18:50:39,944 : INFO : PROGRESS: at sentence #50670000, processed 845074939 words, keeping 355762 word types\n",
      "2019-12-04 18:50:40,087 : INFO : PROGRESS: at sentence #50680000, processed 845179351 words, keeping 355762 word types\n",
      "2019-12-04 18:50:40,231 : INFO : PROGRESS: at sentence #50690000, processed 845284939 words, keeping 355762 word types\n",
      "2019-12-04 18:50:40,374 : INFO : PROGRESS: at sentence #50700000, processed 845389437 words, keeping 355762 word types\n",
      "2019-12-04 18:50:40,518 : INFO : PROGRESS: at sentence #50710000, processed 845494240 words, keeping 355762 word types\n",
      "2019-12-04 18:50:40,661 : INFO : PROGRESS: at sentence #50720000, processed 845599361 words, keeping 355762 word types\n",
      "2019-12-04 18:50:40,805 : INFO : PROGRESS: at sentence #50730000, processed 845704209 words, keeping 355762 word types\n",
      "2019-12-04 18:50:40,953 : INFO : PROGRESS: at sentence #50740000, processed 845808442 words, keeping 355762 word types\n",
      "2019-12-04 18:50:41,097 : INFO : PROGRESS: at sentence #50750000, processed 845914143 words, keeping 355762 word types\n",
      "2019-12-04 18:50:41,243 : INFO : PROGRESS: at sentence #50760000, processed 846018924 words, keeping 355762 word types\n",
      "2019-12-04 18:50:41,388 : INFO : PROGRESS: at sentence #50770000, processed 846122725 words, keeping 355762 word types\n",
      "2019-12-04 18:50:41,533 : INFO : PROGRESS: at sentence #50780000, processed 846228515 words, keeping 355762 word types\n",
      "2019-12-04 18:50:41,681 : INFO : PROGRESS: at sentence #50790000, processed 846333361 words, keeping 355762 word types\n",
      "2019-12-04 18:50:41,827 : INFO : PROGRESS: at sentence #50800000, processed 846437420 words, keeping 355762 word types\n",
      "2019-12-04 18:50:41,978 : INFO : PROGRESS: at sentence #50810000, processed 846543488 words, keeping 355762 word types\n",
      "2019-12-04 18:50:42,122 : INFO : PROGRESS: at sentence #50820000, processed 846647873 words, keeping 355762 word types\n",
      "2019-12-04 18:50:42,267 : INFO : PROGRESS: at sentence #50830000, processed 846752587 words, keeping 355762 word types\n",
      "2019-12-04 18:50:42,411 : INFO : PROGRESS: at sentence #50840000, processed 846857657 words, keeping 355762 word types\n",
      "2019-12-04 18:50:42,555 : INFO : PROGRESS: at sentence #50850000, processed 846962338 words, keeping 355762 word types\n",
      "2019-12-04 18:50:42,699 : INFO : PROGRESS: at sentence #50860000, processed 847067269 words, keeping 355762 word types\n",
      "2019-12-04 18:50:42,844 : INFO : PROGRESS: at sentence #50870000, processed 847172224 words, keeping 355762 word types\n",
      "2019-12-04 18:50:42,988 : INFO : PROGRESS: at sentence #50880000, processed 847276954 words, keeping 355762 word types\n",
      "2019-12-04 18:50:43,132 : INFO : PROGRESS: at sentence #50890000, processed 847381999 words, keeping 355762 word types\n",
      "2019-12-04 18:50:43,278 : INFO : PROGRESS: at sentence #50900000, processed 847486647 words, keeping 355762 word types\n",
      "2019-12-04 18:50:43,422 : INFO : PROGRESS: at sentence #50910000, processed 847591421 words, keeping 355762 word types\n",
      "2019-12-04 18:50:43,567 : INFO : PROGRESS: at sentence #50920000, processed 847696741 words, keeping 355762 word types\n",
      "2019-12-04 18:50:43,712 : INFO : PROGRESS: at sentence #50930000, processed 847801486 words, keeping 355762 word types\n",
      "2019-12-04 18:50:43,859 : INFO : PROGRESS: at sentence #50940000, processed 847905952 words, keeping 355762 word types\n",
      "2019-12-04 18:50:44,003 : INFO : PROGRESS: at sentence #50950000, processed 848011425 words, keeping 355762 word types\n",
      "2019-12-04 18:50:44,146 : INFO : PROGRESS: at sentence #50960000, processed 848116249 words, keeping 355762 word types\n",
      "2019-12-04 18:50:44,290 : INFO : PROGRESS: at sentence #50970000, processed 848220413 words, keeping 355762 word types\n",
      "2019-12-04 18:50:44,434 : INFO : PROGRESS: at sentence #50980000, processed 848325902 words, keeping 355762 word types\n",
      "2019-12-04 18:50:44,578 : INFO : PROGRESS: at sentence #50990000, processed 848430747 words, keeping 355762 word types\n",
      "2019-12-04 18:50:44,723 : INFO : PROGRESS: at sentence #51000000, processed 848534584 words, keeping 355762 word types\n",
      "2019-12-04 18:50:44,869 : INFO : PROGRESS: at sentence #51010000, processed 848640699 words, keeping 355762 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:50:45,014 : INFO : PROGRESS: at sentence #51020000, processed 848745300 words, keeping 355762 word types\n",
      "2019-12-04 18:50:45,156 : INFO : PROGRESS: at sentence #51030000, processed 848849446 words, keeping 355762 word types\n",
      "2019-12-04 18:50:45,301 : INFO : PROGRESS: at sentence #51040000, processed 848955413 words, keeping 355762 word types\n",
      "2019-12-04 18:50:45,445 : INFO : PROGRESS: at sentence #51050000, processed 849059911 words, keeping 355762 word types\n",
      "2019-12-04 18:50:45,588 : INFO : PROGRESS: at sentence #51060000, processed 849164210 words, keeping 355762 word types\n",
      "2019-12-04 18:50:45,731 : INFO : PROGRESS: at sentence #51070000, processed 849269568 words, keeping 355762 word types\n",
      "2019-12-04 18:50:45,874 : INFO : PROGRESS: at sentence #51080000, processed 849374161 words, keeping 355762 word types\n",
      "2019-12-04 18:50:46,018 : INFO : PROGRESS: at sentence #51090000, processed 849479430 words, keeping 355762 word types\n",
      "2019-12-04 18:50:46,163 : INFO : PROGRESS: at sentence #51100000, processed 849584234 words, keeping 355762 word types\n",
      "2019-12-04 18:50:46,307 : INFO : PROGRESS: at sentence #51110000, processed 849688893 words, keeping 355762 word types\n",
      "2019-12-04 18:50:46,453 : INFO : PROGRESS: at sentence #51120000, processed 849794220 words, keeping 355762 word types\n",
      "2019-12-04 18:50:46,597 : INFO : PROGRESS: at sentence #51130000, processed 849898738 words, keeping 355762 word types\n",
      "2019-12-04 18:50:46,743 : INFO : PROGRESS: at sentence #51140000, processed 850003317 words, keeping 355762 word types\n",
      "2019-12-04 18:50:46,888 : INFO : PROGRESS: at sentence #51150000, processed 850108525 words, keeping 355762 word types\n",
      "2019-12-04 18:50:47,032 : INFO : PROGRESS: at sentence #51160000, processed 850213344 words, keeping 355762 word types\n",
      "2019-12-04 18:50:47,176 : INFO : PROGRESS: at sentence #51170000, processed 850317797 words, keeping 355762 word types\n",
      "2019-12-04 18:50:47,322 : INFO : PROGRESS: at sentence #51180000, processed 850423331 words, keeping 355762 word types\n",
      "2019-12-04 18:50:47,466 : INFO : PROGRESS: at sentence #51190000, processed 850528050 words, keeping 355762 word types\n",
      "2019-12-04 18:50:47,607 : INFO : PROGRESS: at sentence #51200000, processed 850632236 words, keeping 355762 word types\n",
      "2019-12-04 18:50:47,749 : INFO : PROGRESS: at sentence #51210000, processed 850737923 words, keeping 355762 word types\n",
      "2019-12-04 18:50:47,891 : INFO : PROGRESS: at sentence #51220000, processed 850842558 words, keeping 355762 word types\n",
      "2019-12-04 18:50:48,034 : INFO : PROGRESS: at sentence #51230000, processed 850946729 words, keeping 355762 word types\n",
      "2019-12-04 18:50:48,180 : INFO : PROGRESS: at sentence #51240000, processed 851052837 words, keeping 355762 word types\n",
      "2019-12-04 18:50:48,324 : INFO : PROGRESS: at sentence #51250000, processed 851157131 words, keeping 355762 word types\n",
      "2019-12-04 18:50:48,468 : INFO : PROGRESS: at sentence #51260000, processed 851261534 words, keeping 355762 word types\n",
      "2019-12-04 18:50:48,611 : INFO : PROGRESS: at sentence #51270000, processed 851367065 words, keeping 355762 word types\n",
      "2019-12-04 18:50:48,755 : INFO : PROGRESS: at sentence #51280000, processed 851471882 words, keeping 355762 word types\n",
      "2019-12-04 18:50:48,899 : INFO : PROGRESS: at sentence #51290000, processed 851576266 words, keeping 355762 word types\n",
      "2019-12-04 18:50:49,042 : INFO : PROGRESS: at sentence #51300000, processed 851681508 words, keeping 355762 word types\n",
      "2019-12-04 18:50:49,182 : INFO : PROGRESS: at sentence #51310000, processed 851786010 words, keeping 355762 word types\n",
      "2019-12-04 18:50:49,324 : INFO : PROGRESS: at sentence #51320000, processed 851891281 words, keeping 355762 word types\n",
      "2019-12-04 18:50:49,464 : INFO : PROGRESS: at sentence #51330000, processed 851996066 words, keeping 355762 word types\n",
      "2019-12-04 18:50:49,606 : INFO : PROGRESS: at sentence #51340000, processed 852100556 words, keeping 355762 word types\n",
      "2019-12-04 18:50:49,751 : INFO : PROGRESS: at sentence #51350000, processed 852206064 words, keeping 355762 word types\n",
      "2019-12-04 18:50:49,895 : INFO : PROGRESS: at sentence #51360000, processed 852310644 words, keeping 355762 word types\n",
      "2019-12-04 18:50:50,042 : INFO : PROGRESS: at sentence #51370000, processed 852415442 words, keeping 355762 word types\n",
      "2019-12-04 18:50:50,185 : INFO : PROGRESS: at sentence #51380000, processed 852520541 words, keeping 355762 word types\n",
      "2019-12-04 18:50:50,326 : INFO : PROGRESS: at sentence #51390000, processed 852625348 words, keeping 355762 word types\n",
      "2019-12-04 18:50:50,470 : INFO : PROGRESS: at sentence #51400000, processed 852729642 words, keeping 355762 word types\n",
      "2019-12-04 18:50:50,613 : INFO : PROGRESS: at sentence #51410000, processed 852835269 words, keeping 355762 word types\n",
      "2019-12-04 18:50:50,758 : INFO : PROGRESS: at sentence #51420000, processed 852940059 words, keeping 355762 word types\n",
      "2019-12-04 18:50:50,900 : INFO : PROGRESS: at sentence #51430000, processed 853043961 words, keeping 355762 word types\n",
      "2019-12-04 18:50:51,044 : INFO : PROGRESS: at sentence #51440000, processed 853149576 words, keeping 355762 word types\n",
      "2019-12-04 18:50:51,186 : INFO : PROGRESS: at sentence #51450000, processed 853254533 words, keeping 355762 word types\n",
      "2019-12-04 18:50:51,329 : INFO : PROGRESS: at sentence #51460000, processed 853358593 words, keeping 355762 word types\n",
      "2019-12-04 18:50:51,474 : INFO : PROGRESS: at sentence #51470000, processed 853464670 words, keeping 355762 word types\n",
      "2019-12-04 18:50:51,616 : INFO : PROGRESS: at sentence #51480000, processed 853569023 words, keeping 355762 word types\n",
      "2019-12-04 18:50:51,761 : INFO : PROGRESS: at sentence #51490000, processed 853673770 words, keeping 355762 word types\n",
      "2019-12-04 18:50:51,905 : INFO : PROGRESS: at sentence #51500000, processed 853778871 words, keeping 355762 word types\n",
      "2019-12-04 18:50:52,049 : INFO : PROGRESS: at sentence #51510000, processed 853883617 words, keeping 355762 word types\n",
      "2019-12-04 18:50:52,192 : INFO : PROGRESS: at sentence #51520000, processed 853988426 words, keeping 355762 word types\n",
      "2019-12-04 18:50:52,334 : INFO : PROGRESS: at sentence #51530000, processed 854093460 words, keeping 355762 word types\n",
      "2019-12-04 18:50:52,475 : INFO : PROGRESS: at sentence #51540000, processed 854198129 words, keeping 355762 word types\n",
      "2019-12-04 18:50:52,620 : INFO : PROGRESS: at sentence #51550000, processed 854303249 words, keeping 355762 word types\n",
      "2019-12-04 18:50:52,762 : INFO : PROGRESS: at sentence #51560000, processed 854407833 words, keeping 355762 word types\n",
      "2019-12-04 18:50:52,905 : INFO : PROGRESS: at sentence #51570000, processed 854512588 words, keeping 355762 word types\n",
      "2019-12-04 18:50:53,051 : INFO : PROGRESS: at sentence #51580000, processed 854617944 words, keeping 355762 word types\n",
      "2019-12-04 18:50:53,194 : INFO : PROGRESS: at sentence #51590000, processed 854722657 words, keeping 355762 word types\n",
      "2019-12-04 18:50:53,339 : INFO : PROGRESS: at sentence #51600000, processed 854827112 words, keeping 355762 word types\n",
      "2019-12-04 18:50:53,482 : INFO : PROGRESS: at sentence #51610000, processed 854932652 words, keeping 355762 word types\n",
      "2019-12-04 18:50:53,625 : INFO : PROGRESS: at sentence #51620000, processed 855037341 words, keeping 355762 word types\n",
      "2019-12-04 18:50:53,768 : INFO : PROGRESS: at sentence #51630000, processed 855141610 words, keeping 355762 word types\n",
      "2019-12-04 18:50:53,913 : INFO : PROGRESS: at sentence #51640000, processed 855247089 words, keeping 355762 word types\n",
      "2019-12-04 18:50:54,057 : INFO : PROGRESS: at sentence #51650000, processed 855351956 words, keeping 355762 word types\n",
      "2019-12-04 18:50:54,198 : INFO : PROGRESS: at sentence #51660000, processed 855455758 words, keeping 355762 word types\n",
      "2019-12-04 18:50:54,341 : INFO : PROGRESS: at sentence #51670000, processed 855561874 words, keeping 355762 word types\n",
      "2019-12-04 18:50:54,481 : INFO : PROGRESS: at sentence #51680000, processed 855666485 words, keeping 355762 word types\n",
      "2019-12-04 18:50:54,624 : INFO : PROGRESS: at sentence #51690000, processed 855770665 words, keeping 355762 word types\n",
      "2019-12-04 18:50:54,768 : INFO : PROGRESS: at sentence #51700000, processed 855876558 words, keeping 355762 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:50:54,911 : INFO : PROGRESS: at sentence #51710000, processed 855981114 words, keeping 355762 word types\n",
      "2019-12-04 18:50:55,055 : INFO : PROGRESS: at sentence #51720000, processed 856085445 words, keeping 355762 word types\n",
      "2019-12-04 18:50:55,198 : INFO : PROGRESS: at sentence #51730000, processed 856190759 words, keeping 355762 word types\n",
      "2019-12-04 18:50:55,340 : INFO : PROGRESS: at sentence #51740000, processed 856295310 words, keeping 355762 word types\n",
      "2019-12-04 18:50:55,483 : INFO : PROGRESS: at sentence #51750000, processed 856400611 words, keeping 355762 word types\n",
      "2019-12-04 18:50:55,626 : INFO : PROGRESS: at sentence #51760000, processed 856505402 words, keeping 355762 word types\n",
      "2019-12-04 18:50:55,770 : INFO : PROGRESS: at sentence #51770000, processed 856610167 words, keeping 355762 word types\n",
      "2019-12-04 18:50:55,913 : INFO : PROGRESS: at sentence #51780000, processed 856715344 words, keeping 355762 word types\n",
      "2019-12-04 18:50:56,055 : INFO : PROGRESS: at sentence #51790000, processed 856819875 words, keeping 355762 word types\n",
      "2019-12-04 18:50:56,200 : INFO : PROGRESS: at sentence #51800000, processed 856924466 words, keeping 355762 word types\n",
      "2019-12-04 18:50:56,343 : INFO : PROGRESS: at sentence #51810000, processed 857029736 words, keeping 355762 word types\n",
      "2019-12-04 18:50:56,484 : INFO : PROGRESS: at sentence #51820000, processed 857134551 words, keeping 355762 word types\n",
      "2019-12-04 18:50:56,628 : INFO : PROGRESS: at sentence #51830000, processed 857239026 words, keeping 355762 word types\n",
      "2019-12-04 18:50:56,770 : INFO : PROGRESS: at sentence #51840000, processed 857344340 words, keeping 355762 word types\n",
      "2019-12-04 18:50:56,915 : INFO : PROGRESS: at sentence #51850000, processed 857449273 words, keeping 355762 word types\n",
      "2019-12-04 18:50:57,057 : INFO : PROGRESS: at sentence #51860000, processed 857553443 words, keeping 355762 word types\n",
      "2019-12-04 18:50:57,199 : INFO : PROGRESS: at sentence #51870000, processed 857659137 words, keeping 355762 word types\n",
      "2019-12-04 18:50:57,342 : INFO : PROGRESS: at sentence #51880000, processed 857763683 words, keeping 355762 word types\n",
      "2019-12-04 18:50:57,487 : INFO : PROGRESS: at sentence #51890000, processed 857867881 words, keeping 355762 word types\n",
      "2019-12-04 18:50:57,631 : INFO : PROGRESS: at sentence #51900000, processed 857973890 words, keeping 355762 word types\n",
      "2019-12-04 18:50:57,774 : INFO : PROGRESS: at sentence #51910000, processed 858078363 words, keeping 355762 word types\n",
      "2019-12-04 18:50:57,918 : INFO : PROGRESS: at sentence #51920000, processed 858182766 words, keeping 355762 word types\n",
      "2019-12-04 18:50:58,062 : INFO : PROGRESS: at sentence #51930000, processed 858288236 words, keeping 355762 word types\n",
      "2019-12-04 18:50:58,205 : INFO : PROGRESS: at sentence #51940000, processed 858392984 words, keeping 355762 word types\n",
      "2019-12-04 18:50:58,346 : INFO : PROGRESS: at sentence #51950000, processed 858497411 words, keeping 355762 word types\n",
      "2019-12-04 18:50:58,489 : INFO : PROGRESS: at sentence #51960000, processed 858602810 words, keeping 355762 word types\n",
      "2019-12-04 18:50:58,632 : INFO : PROGRESS: at sentence #51970000, processed 858707265 words, keeping 355762 word types\n",
      "2019-12-04 18:50:58,777 : INFO : PROGRESS: at sentence #51980000, processed 858812453 words, keeping 355762 word types\n",
      "2019-12-04 18:50:58,920 : INFO : PROGRESS: at sentence #51990000, processed 858917196 words, keeping 355762 word types\n",
      "2019-12-04 18:50:59,062 : INFO : PROGRESS: at sentence #52000000, processed 859021797 words, keeping 355762 word types\n",
      "2019-12-04 18:50:59,203 : INFO : PROGRESS: at sentence #52010000, processed 859127223 words, keeping 355762 word types\n",
      "2019-12-04 18:50:59,349 : INFO : PROGRESS: at sentence #52020000, processed 859231766 words, keeping 355762 word types\n",
      "2019-12-04 18:50:59,493 : INFO : PROGRESS: at sentence #52030000, processed 859336672 words, keeping 355762 word types\n",
      "2019-12-04 18:50:59,639 : INFO : PROGRESS: at sentence #52040000, processed 859441717 words, keeping 355762 word types\n",
      "2019-12-04 18:50:59,824 : INFO : PROGRESS: at sentence #52050000, processed 859546461 words, keeping 355762 word types\n",
      "2019-12-04 18:50:59,967 : INFO : PROGRESS: at sentence #52060000, processed 859650834 words, keeping 355762 word types\n",
      "2019-12-04 18:51:00,113 : INFO : PROGRESS: at sentence #52070000, processed 859765188 words, keeping 356432 word types\n",
      "2019-12-04 18:51:00,270 : INFO : PROGRESS: at sentence #52080000, processed 859880807 words, keeping 356974 word types\n",
      "2019-12-04 18:51:00,420 : INFO : PROGRESS: at sentence #52090000, processed 859980256 words, keeping 357233 word types\n",
      "2019-12-04 18:51:00,554 : INFO : PROGRESS: at sentence #52100000, processed 860081348 words, keeping 357520 word types\n",
      "2019-12-04 18:51:00,688 : INFO : PROGRESS: at sentence #52110000, processed 860183842 words, keeping 357770 word types\n",
      "2019-12-04 18:51:00,820 : INFO : PROGRESS: at sentence #52120000, processed 860284493 words, keeping 358028 word types\n",
      "2019-12-04 18:51:00,951 : INFO : PROGRESS: at sentence #52130000, processed 860383459 words, keeping 358484 word types\n",
      "2019-12-04 18:51:01,074 : INFO : PROGRESS: at sentence #52140000, processed 860478459 words, keeping 358850 word types\n",
      "2019-12-04 18:51:01,206 : INFO : PROGRESS: at sentence #52150000, processed 860576270 words, keeping 359169 word types\n",
      "2019-12-04 18:51:01,334 : INFO : PROGRESS: at sentence #52160000, processed 860673236 words, keeping 359581 word types\n",
      "2019-12-04 18:51:01,466 : INFO : PROGRESS: at sentence #52170000, processed 860774265 words, keeping 360005 word types\n",
      "2019-12-04 18:51:01,603 : INFO : PROGRESS: at sentence #52180000, processed 860880469 words, keeping 360497 word types\n",
      "2019-12-04 18:51:01,741 : INFO : PROGRESS: at sentence #52190000, processed 860985129 words, keeping 360985 word types\n",
      "2019-12-04 18:51:01,878 : INFO : PROGRESS: at sentence #52200000, processed 861089321 words, keeping 361510 word types\n",
      "2019-12-04 18:51:02,015 : INFO : PROGRESS: at sentence #52210000, processed 861194250 words, keeping 362103 word types\n",
      "2019-12-04 18:51:02,152 : INFO : PROGRESS: at sentence #52220000, processed 861299639 words, keeping 362675 word types\n",
      "2019-12-04 18:51:02,285 : INFO : PROGRESS: at sentence #52230000, processed 861399594 words, keeping 363210 word types\n",
      "2019-12-04 18:51:02,420 : INFO : PROGRESS: at sentence #52240000, processed 861502560 words, keeping 363836 word types\n",
      "2019-12-04 18:51:02,563 : INFO : PROGRESS: at sentence #52250000, processed 861611887 words, keeping 364499 word types\n",
      "2019-12-04 18:51:02,724 : INFO : PROGRESS: at sentence #52260000, processed 861738562 words, keeping 365506 word types\n",
      "2019-12-04 18:51:02,873 : INFO : PROGRESS: at sentence #52270000, processed 861854364 words, keeping 365852 word types\n",
      "2019-12-04 18:51:03,018 : INFO : PROGRESS: at sentence #52280000, processed 861965470 words, keeping 366106 word types\n",
      "2019-12-04 18:51:03,158 : INFO : PROGRESS: at sentence #52290000, processed 862071044 words, keeping 366399 word types\n",
      "2019-12-04 18:51:03,295 : INFO : PROGRESS: at sentence #52300000, processed 862177081 words, keeping 366740 word types\n",
      "2019-12-04 18:51:03,434 : INFO : PROGRESS: at sentence #52310000, processed 862285531 words, keeping 367194 word types\n",
      "2019-12-04 18:51:03,572 : INFO : PROGRESS: at sentence #52320000, processed 862392682 words, keeping 367649 word types\n",
      "2019-12-04 18:51:03,711 : INFO : PROGRESS: at sentence #52330000, processed 862499993 words, keeping 368127 word types\n",
      "2019-12-04 18:51:03,864 : INFO : PROGRESS: at sentence #52340000, processed 862608513 words, keeping 368660 word types\n",
      "2019-12-04 18:51:04,004 : INFO : PROGRESS: at sentence #52350000, processed 862716764 words, keeping 369019 word types\n",
      "2019-12-04 18:51:04,145 : INFO : PROGRESS: at sentence #52360000, processed 862827182 words, keeping 369393 word types\n",
      "2019-12-04 18:51:04,295 : INFO : PROGRESS: at sentence #52370000, processed 862943312 words, keeping 370158 word types\n",
      "2019-12-04 18:51:04,440 : INFO : PROGRESS: at sentence #52380000, processed 863053700 words, keeping 371177 word types\n",
      "2019-12-04 18:51:04,591 : INFO : PROGRESS: at sentence #52390000, processed 863167648 words, keeping 372169 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:51:04,738 : INFO : PROGRESS: at sentence #52400000, processed 863279336 words, keeping 373322 word types\n",
      "2019-12-04 18:51:04,899 : INFO : PROGRESS: at sentence #52410000, processed 863401236 words, keeping 374586 word types\n",
      "2019-12-04 18:51:05,057 : INFO : PROGRESS: at sentence #52420000, processed 863521241 words, keeping 375391 word types\n",
      "2019-12-04 18:51:05,216 : INFO : PROGRESS: at sentence #52430000, processed 863642439 words, keeping 376254 word types\n",
      "2019-12-04 18:51:05,375 : INFO : PROGRESS: at sentence #52440000, processed 863764580 words, keeping 376822 word types\n",
      "2019-12-04 18:51:05,525 : INFO : PROGRESS: at sentence #52450000, processed 863881801 words, keeping 376822 word types\n",
      "2019-12-04 18:51:05,663 : INFO : PROGRESS: at sentence #52460000, processed 863988064 words, keeping 376822 word types\n",
      "2019-12-04 18:51:05,793 : INFO : PROGRESS: at sentence #52470000, processed 864086939 words, keeping 376822 word types\n",
      "2019-12-04 18:51:05,928 : INFO : PROGRESS: at sentence #52480000, processed 864189337 words, keeping 376822 word types\n",
      "2019-12-04 18:51:06,066 : INFO : PROGRESS: at sentence #52490000, processed 864292835 words, keeping 376822 word types\n",
      "2019-12-04 18:51:06,196 : INFO : PROGRESS: at sentence #52500000, processed 864391134 words, keeping 376822 word types\n",
      "2019-12-04 18:51:06,327 : INFO : PROGRESS: at sentence #52510000, processed 864490151 words, keeping 376822 word types\n",
      "2019-12-04 18:51:06,453 : INFO : PROGRESS: at sentence #52520000, processed 864585346 words, keeping 376822 word types\n",
      "2019-12-04 18:51:06,580 : INFO : PROGRESS: at sentence #52530000, processed 864681886 words, keeping 376822 word types\n",
      "2019-12-04 18:51:06,710 : INFO : PROGRESS: at sentence #52540000, processed 864780606 words, keeping 376822 word types\n",
      "2019-12-04 18:51:06,845 : INFO : PROGRESS: at sentence #52550000, processed 864883727 words, keeping 376822 word types\n",
      "2019-12-04 18:51:06,986 : INFO : PROGRESS: at sentence #52560000, processed 864989339 words, keeping 376822 word types\n",
      "2019-12-04 18:51:07,123 : INFO : PROGRESS: at sentence #52570000, processed 865093849 words, keeping 376822 word types\n",
      "2019-12-04 18:51:07,258 : INFO : PROGRESS: at sentence #52580000, processed 865198912 words, keeping 376822 word types\n",
      "2019-12-04 18:51:07,396 : INFO : PROGRESS: at sentence #52590000, processed 865304593 words, keeping 376822 word types\n",
      "2019-12-04 18:51:07,532 : INFO : PROGRESS: at sentence #52600000, processed 865406733 words, keeping 376822 word types\n",
      "2019-12-04 18:51:07,671 : INFO : PROGRESS: at sentence #52610000, processed 865508378 words, keeping 376822 word types\n",
      "2019-12-04 18:51:07,809 : INFO : PROGRESS: at sentence #52620000, processed 865610077 words, keeping 376822 word types\n",
      "2019-12-04 18:51:07,971 : INFO : PROGRESS: at sentence #52630000, processed 865731301 words, keeping 376822 word types\n",
      "2019-12-04 18:51:08,128 : INFO : PROGRESS: at sentence #52640000, processed 865851594 words, keeping 376822 word types\n",
      "2019-12-04 18:51:08,280 : INFO : PROGRESS: at sentence #52650000, processed 865966076 words, keeping 376822 word types\n",
      "2019-12-04 18:51:08,428 : INFO : PROGRESS: at sentence #52660000, processed 866076365 words, keeping 376822 word types\n",
      "2019-12-04 18:51:08,569 : INFO : PROGRESS: at sentence #52670000, processed 866181911 words, keeping 376822 word types\n",
      "2019-12-04 18:51:08,711 : INFO : PROGRESS: at sentence #52680000, processed 866288266 words, keeping 376822 word types\n",
      "2019-12-04 18:51:08,858 : INFO : PROGRESS: at sentence #52690000, processed 866395384 words, keeping 376822 word types\n",
      "2019-12-04 18:51:09,001 : INFO : PROGRESS: at sentence #52700000, processed 866502973 words, keeping 376822 word types\n",
      "2019-12-04 18:51:09,147 : INFO : PROGRESS: at sentence #52710000, processed 866609770 words, keeping 376822 word types\n",
      "2019-12-04 18:51:09,292 : INFO : PROGRESS: at sentence #52720000, processed 866718486 words, keeping 376822 word types\n",
      "2019-12-04 18:51:09,435 : INFO : PROGRESS: at sentence #52730000, processed 866828301 words, keeping 376822 word types\n",
      "2019-12-04 18:51:09,585 : INFO : PROGRESS: at sentence #52740000, processed 866939015 words, keeping 376822 word types\n",
      "2019-12-04 18:51:09,740 : INFO : PROGRESS: at sentence #52750000, processed 867055218 words, keeping 376822 word types\n",
      "2019-12-04 18:51:09,891 : INFO : PROGRESS: at sentence #52760000, processed 867165580 words, keeping 376822 word types\n",
      "2019-12-04 18:51:10,043 : INFO : PROGRESS: at sentence #52770000, processed 867280677 words, keeping 376822 word types\n",
      "2019-12-04 18:51:10,191 : INFO : PROGRESS: at sentence #52780000, processed 867388958 words, keeping 376822 word types\n",
      "2019-12-04 18:51:10,357 : INFO : PROGRESS: at sentence #52790000, processed 867516606 words, keeping 376822 word types\n",
      "2019-12-04 18:51:10,521 : INFO : PROGRESS: at sentence #52800000, processed 867637329 words, keeping 376822 word types\n",
      "2019-12-04 18:51:10,684 : INFO : PROGRESS: at sentence #52810000, processed 867759188 words, keeping 376822 word types\n",
      "2019-12-04 18:51:10,839 : INFO : PROGRESS: at sentence #52820000, processed 867876967 words, keeping 376822 word types\n",
      "2019-12-04 18:51:10,998 : INFO : PROGRESS: at sentence #52830000, processed 867996221 words, keeping 376822 word types\n",
      "2019-12-04 18:51:11,129 : INFO : PROGRESS: at sentence #52840000, processed 868094940 words, keeping 376822 word types\n",
      "2019-12-04 18:51:11,265 : INFO : PROGRESS: at sentence #52850000, processed 868195220 words, keeping 376822 word types\n",
      "2019-12-04 18:51:11,400 : INFO : PROGRESS: at sentence #52860000, processed 868296937 words, keeping 376822 word types\n",
      "2019-12-04 18:51:11,540 : INFO : PROGRESS: at sentence #52870000, processed 868402240 words, keeping 376822 word types\n",
      "2019-12-04 18:51:11,669 : INFO : PROGRESS: at sentence #52880000, processed 868497599 words, keeping 376822 word types\n",
      "2019-12-04 18:51:11,797 : INFO : PROGRESS: at sentence #52890000, processed 868594626 words, keeping 376822 word types\n",
      "2019-12-04 18:51:11,925 : INFO : PROGRESS: at sentence #52900000, processed 868692039 words, keeping 376822 word types\n",
      "2019-12-04 18:51:12,053 : INFO : PROGRESS: at sentence #52910000, processed 868788078 words, keeping 376822 word types\n",
      "2019-12-04 18:51:12,186 : INFO : PROGRESS: at sentence #52920000, processed 868887165 words, keeping 376822 word types\n",
      "2019-12-04 18:51:12,326 : INFO : PROGRESS: at sentence #52930000, processed 868993263 words, keeping 376822 word types\n",
      "2019-12-04 18:51:12,467 : INFO : PROGRESS: at sentence #52940000, processed 869098415 words, keeping 376822 word types\n",
      "2019-12-04 18:51:12,604 : INFO : PROGRESS: at sentence #52950000, processed 869202543 words, keeping 376822 word types\n",
      "2019-12-04 18:51:12,743 : INFO : PROGRESS: at sentence #52960000, processed 869307694 words, keeping 376822 word types\n",
      "2019-12-04 18:51:12,887 : INFO : PROGRESS: at sentence #52970000, processed 869413683 words, keeping 376822 word types\n",
      "2019-12-04 18:51:13,022 : INFO : PROGRESS: at sentence #52980000, processed 869513758 words, keeping 376822 word types\n",
      "2019-12-04 18:51:13,159 : INFO : PROGRESS: at sentence #52990000, processed 869616534 words, keeping 376822 word types\n",
      "2019-12-04 18:51:13,294 : INFO : PROGRESS: at sentence #53000000, processed 869719012 words, keeping 376822 word types\n",
      "2019-12-04 18:51:13,461 : INFO : PROGRESS: at sentence #53010000, processed 869849524 words, keeping 376822 word types\n",
      "2019-12-04 18:51:13,614 : INFO : PROGRESS: at sentence #53020000, processed 869964850 words, keeping 376822 word types\n",
      "2019-12-04 18:51:13,764 : INFO : PROGRESS: at sentence #53030000, processed 870077337 words, keeping 376822 word types\n",
      "2019-12-04 18:51:13,909 : INFO : PROGRESS: at sentence #53040000, processed 870184851 words, keeping 376822 word types\n",
      "2019-12-04 18:51:14,051 : INFO : PROGRESS: at sentence #53050000, processed 870291020 words, keeping 376822 word types\n",
      "2019-12-04 18:51:14,198 : INFO : PROGRESS: at sentence #53060000, processed 870398577 words, keeping 376822 word types\n",
      "2019-12-04 18:51:14,341 : INFO : PROGRESS: at sentence #53070000, processed 870505042 words, keeping 376822 word types\n",
      "2019-12-04 18:51:14,483 : INFO : PROGRESS: at sentence #53080000, processed 870612692 words, keeping 376822 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:51:14,629 : INFO : PROGRESS: at sentence #53090000, processed 870720269 words, keeping 376822 word types\n",
      "2019-12-04 18:51:14,774 : INFO : PROGRESS: at sentence #53100000, processed 870828766 words, keeping 376822 word types\n",
      "2019-12-04 18:51:14,918 : INFO : PROGRESS: at sentence #53110000, processed 870939216 words, keeping 376822 word types\n",
      "2019-12-04 18:51:15,061 : INFO : PROGRESS: at sentence #53120000, processed 871053714 words, keeping 376822 word types\n",
      "2019-12-04 18:51:15,205 : INFO : PROGRESS: at sentence #53130000, processed 871166015 words, keeping 376822 word types\n",
      "2019-12-04 18:51:15,350 : INFO : PROGRESS: at sentence #53140000, processed 871278399 words, keeping 376822 word types\n",
      "2019-12-04 18:51:15,496 : INFO : PROGRESS: at sentence #53150000, processed 871392282 words, keeping 376822 word types\n",
      "2019-12-04 18:51:15,652 : INFO : PROGRESS: at sentence #53160000, processed 871510797 words, keeping 376822 word types\n",
      "2019-12-04 18:51:15,809 : INFO : PROGRESS: at sentence #53170000, processed 871631005 words, keeping 376822 word types\n",
      "2019-12-04 18:51:15,962 : INFO : PROGRESS: at sentence #53180000, processed 871751941 words, keeping 376822 word types\n",
      "2019-12-04 18:51:16,114 : INFO : PROGRESS: at sentence #53190000, processed 871874855 words, keeping 376822 word types\n",
      "2019-12-04 18:51:16,259 : INFO : PROGRESS: at sentence #53200000, processed 871992453 words, keeping 376822 word types\n",
      "2019-12-04 18:51:16,399 : INFO : PROGRESS: at sentence #53210000, processed 872102290 words, keeping 376822 word types\n",
      "2019-12-04 18:51:16,527 : INFO : PROGRESS: at sentence #53220000, processed 872202225 words, keeping 376822 word types\n",
      "2019-12-04 18:51:16,656 : INFO : PROGRESS: at sentence #53230000, processed 872303413 words, keeping 376822 word types\n",
      "2019-12-04 18:51:16,789 : INFO : PROGRESS: at sentence #53240000, processed 872405997 words, keeping 376822 word types\n",
      "2019-12-04 18:51:16,912 : INFO : PROGRESS: at sentence #53250000, processed 872503789 words, keeping 376822 word types\n",
      "2019-12-04 18:51:17,040 : INFO : PROGRESS: at sentence #53260000, processed 872605984 words, keeping 376822 word types\n",
      "2019-12-04 18:51:17,159 : INFO : PROGRESS: at sentence #53270000, processed 872700138 words, keeping 376822 word types\n",
      "2019-12-04 18:51:17,285 : INFO : PROGRESS: at sentence #53280000, processed 872797333 words, keeping 376822 word types\n",
      "2019-12-04 18:51:17,408 : INFO : PROGRESS: at sentence #53290000, processed 872894995 words, keeping 376822 word types\n",
      "2019-12-04 18:51:17,538 : INFO : PROGRESS: at sentence #53300000, processed 872997213 words, keeping 376822 word types\n",
      "2019-12-04 18:51:17,676 : INFO : PROGRESS: at sentence #53310000, processed 873103100 words, keeping 376822 word types\n",
      "2019-12-04 18:51:17,807 : INFO : PROGRESS: at sentence #53320000, processed 873207432 words, keeping 376822 word types\n",
      "2019-12-04 18:51:17,941 : INFO : PROGRESS: at sentence #53330000, processed 873311916 words, keeping 376822 word types\n",
      "2019-12-04 18:51:18,078 : INFO : PROGRESS: at sentence #53340000, processed 873417405 words, keeping 376822 word types\n",
      "2019-12-04 18:51:18,217 : INFO : PROGRESS: at sentence #53350000, processed 873521230 words, keeping 376822 word types\n",
      "2019-12-04 18:51:18,354 : INFO : PROGRESS: at sentence #53360000, processed 873621580 words, keeping 376822 word types\n",
      "2019-12-04 18:51:18,491 : INFO : PROGRESS: at sentence #53370000, processed 873724719 words, keeping 376822 word types\n",
      "2019-12-04 18:51:18,637 : INFO : PROGRESS: at sentence #53380000, processed 873838846 words, keeping 376822 word types\n",
      "2019-12-04 18:51:18,793 : INFO : PROGRESS: at sentence #53390000, processed 873962603 words, keeping 376822 word types\n",
      "2019-12-04 18:51:18,944 : INFO : PROGRESS: at sentence #53400000, processed 874078507 words, keeping 376822 word types\n",
      "2019-12-04 18:51:19,088 : INFO : PROGRESS: at sentence #53410000, processed 874190172 words, keeping 376822 word types\n",
      "2019-12-04 18:51:19,222 : INFO : PROGRESS: at sentence #53420000, processed 874293512 words, keeping 376822 word types\n",
      "2019-12-04 18:51:19,366 : INFO : PROGRESS: at sentence #53430000, processed 874399917 words, keeping 376822 word types\n",
      "2019-12-04 18:51:19,513 : INFO : PROGRESS: at sentence #53440000, processed 874508244 words, keeping 376822 word types\n",
      "2019-12-04 18:51:19,654 : INFO : PROGRESS: at sentence #53450000, processed 874615768 words, keeping 376822 word types\n",
      "2019-12-04 18:51:19,795 : INFO : PROGRESS: at sentence #53460000, processed 874722971 words, keeping 376822 word types\n",
      "2019-12-04 18:51:19,938 : INFO : PROGRESS: at sentence #53470000, processed 874831192 words, keeping 376822 word types\n",
      "2019-12-04 18:51:20,082 : INFO : PROGRESS: at sentence #53480000, processed 874940493 words, keeping 376822 word types\n",
      "2019-12-04 18:51:20,231 : INFO : PROGRESS: at sentence #53490000, processed 875050586 words, keeping 376822 word types\n",
      "2019-12-04 18:51:20,385 : INFO : PROGRESS: at sentence #53500000, processed 875167067 words, keeping 376822 word types\n",
      "2019-12-04 18:51:20,532 : INFO : PROGRESS: at sentence #53510000, processed 875277023 words, keeping 376822 word types\n",
      "2019-12-04 18:51:20,688 : INFO : PROGRESS: at sentence #53520000, processed 875391131 words, keeping 376822 word types\n",
      "2019-12-04 18:51:20,837 : INFO : PROGRESS: at sentence #53530000, processed 875501843 words, keeping 376822 word types\n",
      "2019-12-04 18:51:21,000 : INFO : PROGRESS: at sentence #53540000, processed 875626135 words, keeping 376822 word types\n",
      "2019-12-04 18:51:21,158 : INFO : PROGRESS: at sentence #53550000, processed 875746330 words, keeping 376822 word types\n",
      "2019-12-04 18:51:21,319 : INFO : PROGRESS: at sentence #53560000, processed 875868387 words, keeping 376822 word types\n",
      "2019-12-04 18:51:21,473 : INFO : PROGRESS: at sentence #53570000, processed 875988575 words, keeping 376822 word types\n",
      "2019-12-04 18:51:21,625 : INFO : PROGRESS: at sentence #53580000, processed 876106505 words, keeping 376822 word types\n",
      "2019-12-04 18:51:21,760 : INFO : PROGRESS: at sentence #53590000, processed 876209644 words, keeping 376822 word types\n",
      "2019-12-04 18:51:21,890 : INFO : PROGRESS: at sentence #53600000, processed 876308922 words, keeping 376822 word types\n",
      "2019-12-04 18:51:22,023 : INFO : PROGRESS: at sentence #53610000, processed 876411210 words, keeping 376822 word types\n",
      "2019-12-04 18:51:22,157 : INFO : PROGRESS: at sentence #53620000, processed 876515593 words, keeping 376822 word types\n",
      "2019-12-04 18:51:22,281 : INFO : PROGRESS: at sentence #53630000, processed 876610408 words, keeping 376822 word types\n",
      "2019-12-04 18:51:22,410 : INFO : PROGRESS: at sentence #53640000, processed 876710866 words, keeping 376822 word types\n",
      "2019-12-04 18:51:22,533 : INFO : PROGRESS: at sentence #53650000, processed 876806409 words, keeping 376822 word types\n",
      "2019-12-04 18:51:22,661 : INFO : PROGRESS: at sentence #53660000, processed 876903076 words, keeping 376822 word types\n",
      "2019-12-04 18:51:22,788 : INFO : PROGRESS: at sentence #53670000, processed 877002383 words, keeping 376822 word types\n",
      "2019-12-04 18:51:22,923 : INFO : PROGRESS: at sentence #53680000, processed 877106979 words, keeping 376822 word types\n",
      "2019-12-04 18:51:23,057 : INFO : PROGRESS: at sentence #53690000, processed 877211869 words, keeping 376822 word types\n",
      "2019-12-04 18:51:23,194 : INFO : PROGRESS: at sentence #53700000, processed 877316675 words, keeping 376822 word types\n",
      "2019-12-04 18:51:23,327 : INFO : PROGRESS: at sentence #53710000, processed 877421128 words, keeping 376822 word types\n",
      "2019-12-04 18:51:23,463 : INFO : PROGRESS: at sentence #53720000, processed 877527736 words, keeping 376822 word types\n",
      "2019-12-04 18:51:23,594 : INFO : PROGRESS: at sentence #53730000, processed 877628669 words, keeping 376822 word types\n",
      "2019-12-04 18:51:23,725 : INFO : PROGRESS: at sentence #53740000, processed 877730542 words, keeping 376822 word types\n",
      "2019-12-04 18:51:23,859 : INFO : PROGRESS: at sentence #53750000, processed 877832664 words, keeping 376822 word types\n",
      "2019-12-04 18:51:24,018 : INFO : PROGRESS: at sentence #53760000, processed 877957725 words, keeping 376822 word types\n",
      "2019-12-04 18:51:24,192 : INFO : PROGRESS: at sentence #53770000, processed 878076081 words, keeping 376822 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:51:24,338 : INFO : PROGRESS: at sentence #53780000, processed 878189265 words, keeping 376822 word types\n",
      "2019-12-04 18:51:24,479 : INFO : PROGRESS: at sentence #53790000, processed 878298555 words, keeping 376822 word types\n",
      "2019-12-04 18:51:24,617 : INFO : PROGRESS: at sentence #53800000, processed 878404657 words, keeping 376822 word types\n",
      "2019-12-04 18:51:24,757 : INFO : PROGRESS: at sentence #53810000, processed 878511389 words, keeping 376822 word types\n",
      "2019-12-04 18:51:24,897 : INFO : PROGRESS: at sentence #53820000, processed 878618156 words, keeping 376822 word types\n",
      "2019-12-04 18:51:25,041 : INFO : PROGRESS: at sentence #53830000, processed 878725527 words, keeping 376822 word types\n",
      "2019-12-04 18:51:25,184 : INFO : PROGRESS: at sentence #53840000, processed 878832874 words, keeping 376822 word types\n",
      "2019-12-04 18:51:25,326 : INFO : PROGRESS: at sentence #53850000, processed 878941450 words, keeping 376822 word types\n",
      "2019-12-04 18:51:25,471 : INFO : PROGRESS: at sentence #53860000, processed 879051422 words, keeping 376822 word types\n",
      "2019-12-04 18:51:25,617 : INFO : PROGRESS: at sentence #53870000, processed 879163541 words, keeping 376822 word types\n",
      "2019-12-04 18:51:25,767 : INFO : PROGRESS: at sentence #53880000, processed 879277673 words, keeping 376822 word types\n",
      "2019-12-04 18:51:25,917 : INFO : PROGRESS: at sentence #53890000, processed 879389790 words, keeping 376822 word types\n",
      "2019-12-04 18:51:26,067 : INFO : PROGRESS: at sentence #53900000, processed 879504477 words, keeping 376822 word types\n",
      "2019-12-04 18:51:26,217 : INFO : PROGRESS: at sentence #53910000, processed 879617591 words, keeping 376822 word types\n",
      "2019-12-04 18:51:26,378 : INFO : PROGRESS: at sentence #53920000, processed 879741228 words, keeping 376822 word types\n",
      "2019-12-04 18:51:26,537 : INFO : PROGRESS: at sentence #53930000, processed 879861821 words, keeping 376822 word types\n",
      "2019-12-04 18:51:26,701 : INFO : PROGRESS: at sentence #53940000, processed 879984549 words, keeping 376822 word types\n",
      "2019-12-04 18:51:26,849 : INFO : PROGRESS: at sentence #53950000, processed 880100374 words, keeping 376822 word types\n",
      "2019-12-04 18:51:27,001 : INFO : PROGRESS: at sentence #53960000, processed 880217191 words, keeping 376822 word types\n",
      "2019-12-04 18:51:27,132 : INFO : PROGRESS: at sentence #53970000, processed 880316712 words, keeping 376822 word types\n",
      "2019-12-04 18:51:27,263 : INFO : PROGRESS: at sentence #53980000, processed 880417548 words, keeping 376822 word types\n",
      "2019-12-04 18:51:27,397 : INFO : PROGRESS: at sentence #53990000, processed 880519736 words, keeping 376822 word types\n",
      "2019-12-04 18:51:27,532 : INFO : PROGRESS: at sentence #54000000, processed 880624226 words, keeping 376822 word types\n",
      "2019-12-04 18:51:27,655 : INFO : PROGRESS: at sentence #54010000, processed 880719203 words, keeping 376822 word types\n",
      "2019-12-04 18:51:27,781 : INFO : PROGRESS: at sentence #54020000, processed 880815697 words, keeping 376822 word types\n",
      "2019-12-04 18:51:27,907 : INFO : PROGRESS: at sentence #54030000, processed 880912980 words, keeping 376822 word types\n",
      "2019-12-04 18:51:28,033 : INFO : PROGRESS: at sentence #54040000, processed 881009892 words, keeping 376822 word types\n",
      "2019-12-04 18:51:28,162 : INFO : PROGRESS: at sentence #54050000, processed 881110066 words, keeping 376822 word types\n",
      "2019-12-04 18:51:28,298 : INFO : PROGRESS: at sentence #54060000, processed 881216263 words, keeping 376822 word types\n",
      "2019-12-04 18:51:28,432 : INFO : PROGRESS: at sentence #54070000, processed 881321179 words, keeping 376822 word types\n",
      "2019-12-04 18:51:28,567 : INFO : PROGRESS: at sentence #54080000, processed 881424880 words, keeping 376822 word types\n",
      "2019-12-04 18:51:28,702 : INFO : PROGRESS: at sentence #54090000, processed 881530077 words, keeping 376822 word types\n",
      "2019-12-04 18:51:28,840 : INFO : PROGRESS: at sentence #54100000, processed 881635891 words, keeping 376822 word types\n",
      "2019-12-04 18:51:28,972 : INFO : PROGRESS: at sentence #54110000, processed 881735819 words, keeping 376822 word types\n",
      "2019-12-04 18:51:29,107 : INFO : PROGRESS: at sentence #54120000, processed 881838990 words, keeping 376822 word types\n",
      "2019-12-04 18:51:29,248 : INFO : PROGRESS: at sentence #54130000, processed 881945465 words, keeping 376822 word types\n",
      "2019-12-04 18:51:29,411 : INFO : PROGRESS: at sentence #54140000, processed 882073506 words, keeping 376822 word types\n",
      "2019-12-04 18:51:29,561 : INFO : PROGRESS: at sentence #54150000, processed 882188770 words, keeping 376822 word types\n",
      "2019-12-04 18:51:29,706 : INFO : PROGRESS: at sentence #54160000, processed 882300385 words, keeping 376822 word types\n",
      "2019-12-04 18:51:29,846 : INFO : PROGRESS: at sentence #54170000, processed 882407270 words, keeping 376822 word types\n",
      "2019-12-04 18:51:29,987 : INFO : PROGRESS: at sentence #54180000, processed 882513344 words, keeping 376822 word types\n",
      "2019-12-04 18:51:30,129 : INFO : PROGRESS: at sentence #54190000, processed 882621312 words, keeping 376822 word types\n",
      "2019-12-04 18:51:30,269 : INFO : PROGRESS: at sentence #54200000, processed 882727984 words, keeping 376822 word types\n",
      "2019-12-04 18:51:30,411 : INFO : PROGRESS: at sentence #54210000, processed 882835716 words, keeping 376822 word types\n",
      "2019-12-04 18:51:30,557 : INFO : PROGRESS: at sentence #54220000, processed 882943898 words, keeping 376822 word types\n",
      "2019-12-04 18:51:30,727 : INFO : PROGRESS: at sentence #54230000, processed 883052427 words, keeping 376822 word types\n",
      "2019-12-04 18:51:30,871 : INFO : PROGRESS: at sentence #54240000, processed 883162495 words, keeping 376822 word types\n",
      "2019-12-04 18:51:31,021 : INFO : PROGRESS: at sentence #54250000, processed 883278213 words, keeping 376822 word types\n",
      "2019-12-04 18:51:31,169 : INFO : PROGRESS: at sentence #54260000, processed 883389411 words, keeping 376822 word types\n",
      "2019-12-04 18:51:31,326 : INFO : PROGRESS: at sentence #54270000, processed 883502459 words, keeping 376822 word types\n",
      "2019-12-04 18:51:31,478 : INFO : PROGRESS: at sentence #54280000, processed 883615060 words, keeping 376822 word types\n",
      "2019-12-04 18:51:31,637 : INFO : PROGRESS: at sentence #54290000, processed 883735908 words, keeping 376822 word types\n",
      "2019-12-04 18:51:31,800 : INFO : PROGRESS: at sentence #54300000, processed 883856170 words, keeping 376822 word types\n",
      "2019-12-04 18:51:31,960 : INFO : PROGRESS: at sentence #54310000, processed 883977153 words, keeping 376822 word types\n",
      "2019-12-04 18:51:32,120 : INFO : PROGRESS: at sentence #54320000, processed 884099999 words, keeping 376822 word types\n",
      "2019-12-04 18:51:32,271 : INFO : PROGRESS: at sentence #54330000, processed 884216784 words, keeping 376822 word types\n",
      "2019-12-04 18:51:32,412 : INFO : PROGRESS: at sentence #54340000, processed 884324255 words, keeping 376822 word types\n",
      "2019-12-04 18:51:32,542 : INFO : PROGRESS: at sentence #54350000, processed 884423462 words, keeping 376822 word types\n",
      "2019-12-04 18:51:32,686 : INFO : PROGRESS: at sentence #54360000, processed 884525522 words, keeping 376822 word types\n",
      "2019-12-04 18:51:32,821 : INFO : PROGRESS: at sentence #54370000, processed 884628731 words, keeping 376822 word types\n",
      "2019-12-04 18:51:32,949 : INFO : PROGRESS: at sentence #54380000, processed 884726307 words, keeping 376822 word types\n",
      "2019-12-04 18:51:33,080 : INFO : PROGRESS: at sentence #54390000, processed 884827185 words, keeping 376822 word types\n",
      "2019-12-04 18:51:33,203 : INFO : PROGRESS: at sentence #54400000, processed 884921714 words, keeping 376822 word types\n",
      "2019-12-04 18:51:33,336 : INFO : PROGRESS: at sentence #54410000, processed 885018780 words, keeping 376822 word types\n",
      "2019-12-04 18:51:33,466 : INFO : PROGRESS: at sentence #54420000, processed 885116887 words, keeping 376822 word types\n",
      "2019-12-04 18:51:33,601 : INFO : PROGRESS: at sentence #54430000, processed 885219684 words, keeping 376822 word types\n",
      "2019-12-04 18:51:33,737 : INFO : PROGRESS: at sentence #54440000, processed 885325558 words, keeping 376822 word types\n",
      "2019-12-04 18:51:33,872 : INFO : PROGRESS: at sentence #54450000, processed 885429916 words, keeping 376822 word types\n",
      "2019-12-04 18:51:34,008 : INFO : PROGRESS: at sentence #54460000, processed 885534796 words, keeping 376822 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:51:34,145 : INFO : PROGRESS: at sentence #54470000, processed 885640124 words, keeping 376822 word types\n",
      "2019-12-04 18:51:34,283 : INFO : PROGRESS: at sentence #54480000, processed 885743148 words, keeping 376822 word types\n",
      "2019-12-04 18:51:34,420 : INFO : PROGRESS: at sentence #54490000, processed 885844541 words, keeping 376822 word types\n",
      "2019-12-04 18:51:34,566 : INFO : PROGRESS: at sentence #54500000, processed 885946527 words, keeping 376822 word types\n",
      "2019-12-04 18:51:34,732 : INFO : PROGRESS: at sentence #54510000, processed 886065524 words, keeping 376822 word types\n",
      "2019-12-04 18:51:34,890 : INFO : PROGRESS: at sentence #54520000, processed 886186833 words, keeping 376822 word types\n",
      "2019-12-04 18:51:35,040 : INFO : PROGRESS: at sentence #54530000, processed 886301857 words, keeping 376822 word types\n",
      "2019-12-04 18:51:35,182 : INFO : PROGRESS: at sentence #54540000, processed 886411954 words, keeping 376822 word types\n",
      "2019-12-04 18:51:35,326 : INFO : PROGRESS: at sentence #54550000, processed 886517686 words, keeping 376822 word types\n",
      "2019-12-04 18:51:35,466 : INFO : PROGRESS: at sentence #54560000, processed 886622912 words, keeping 376822 word types\n",
      "2019-12-04 18:51:35,609 : INFO : PROGRESS: at sentence #54570000, processed 886731196 words, keeping 376822 word types\n",
      "2019-12-04 18:51:35,751 : INFO : PROGRESS: at sentence #54580000, processed 886838887 words, keeping 376822 word types\n",
      "2019-12-04 18:51:35,892 : INFO : PROGRESS: at sentence #54590000, processed 886945631 words, keeping 376822 word types\n",
      "2019-12-04 18:51:36,035 : INFO : PROGRESS: at sentence #54600000, processed 887053914 words, keeping 376822 word types\n",
      "2019-12-04 18:51:36,179 : INFO : PROGRESS: at sentence #54610000, processed 887163702 words, keeping 376822 word types\n",
      "2019-12-04 18:51:36,327 : INFO : PROGRESS: at sentence #54620000, processed 887274467 words, keeping 376822 word types\n",
      "2019-12-04 18:51:36,481 : INFO : PROGRESS: at sentence #54630000, processed 887390817 words, keeping 376822 word types\n",
      "2019-12-04 18:51:36,625 : INFO : PROGRESS: at sentence #54640000, processed 887500596 words, keeping 376822 word types\n",
      "2019-12-04 18:51:36,780 : INFO : PROGRESS: at sentence #54650000, processed 887615457 words, keeping 376822 word types\n",
      "2019-12-04 18:51:36,933 : INFO : PROGRESS: at sentence #54660000, processed 887724975 words, keeping 376822 word types\n",
      "2019-12-04 18:51:37,101 : INFO : PROGRESS: at sentence #54670000, processed 887850957 words, keeping 376822 word types\n",
      "2019-12-04 18:51:37,267 : INFO : PROGRESS: at sentence #54680000, processed 887971716 words, keeping 376822 word types\n",
      "2019-12-04 18:51:37,435 : INFO : PROGRESS: at sentence #54690000, processed 888093567 words, keeping 376822 word types\n",
      "2019-12-04 18:51:37,592 : INFO : PROGRESS: at sentence #54700000, processed 888212082 words, keeping 376822 word types\n",
      "2019-12-04 18:51:37,749 : INFO : PROGRESS: at sentence #54710000, processed 888331200 words, keeping 376822 word types\n",
      "2019-12-04 18:51:37,883 : INFO : PROGRESS: at sentence #54720000, processed 888431268 words, keeping 376822 word types\n",
      "2019-12-04 18:51:38,014 : INFO : PROGRESS: at sentence #54730000, processed 888531293 words, keeping 376822 word types\n",
      "2019-12-04 18:51:38,148 : INFO : PROGRESS: at sentence #54740000, processed 888633252 words, keeping 376822 word types\n",
      "2019-12-04 18:51:38,282 : INFO : PROGRESS: at sentence #54750000, processed 888738336 words, keeping 376822 word types\n",
      "2019-12-04 18:51:38,405 : INFO : PROGRESS: at sentence #54760000, processed 888834050 words, keeping 376822 word types\n",
      "2019-12-04 18:51:38,532 : INFO : PROGRESS: at sentence #54770000, processed 888931682 words, keeping 376822 word types\n",
      "2019-12-04 18:51:38,660 : INFO : PROGRESS: at sentence #54780000, processed 889028476 words, keeping 376822 word types\n",
      "2019-12-04 18:51:38,789 : INFO : PROGRESS: at sentence #54790000, processed 889124708 words, keeping 376822 word types\n",
      "2019-12-04 18:51:38,918 : INFO : PROGRESS: at sentence #54800000, processed 889224075 words, keeping 376822 word types\n",
      "2019-12-04 18:51:39,055 : INFO : PROGRESS: at sentence #54810000, processed 889329394 words, keeping 376822 word types\n",
      "2019-12-04 18:51:39,191 : INFO : PROGRESS: at sentence #54820000, processed 889434584 words, keeping 376822 word types\n",
      "2019-12-04 18:51:39,325 : INFO : PROGRESS: at sentence #54830000, processed 889538901 words, keeping 376822 word types\n",
      "2019-12-04 18:51:39,463 : INFO : PROGRESS: at sentence #54840000, processed 889644041 words, keeping 376822 word types\n",
      "2019-12-04 18:51:39,602 : INFO : PROGRESS: at sentence #54850000, processed 889749641 words, keeping 376822 word types\n",
      "2019-12-04 18:51:39,735 : INFO : PROGRESS: at sentence #54860000, processed 889850415 words, keeping 376822 word types\n",
      "2019-12-04 18:51:39,870 : INFO : PROGRESS: at sentence #54870000, processed 889952807 words, keeping 376822 word types\n",
      "2019-12-04 18:51:40,007 : INFO : PROGRESS: at sentence #54880000, processed 890055286 words, keeping 376822 word types\n",
      "2019-12-04 18:51:40,168 : INFO : PROGRESS: at sentence #54890000, processed 890183730 words, keeping 376822 word types\n",
      "2019-12-04 18:51:40,317 : INFO : PROGRESS: at sentence #54900000, processed 890299982 words, keeping 376822 word types\n",
      "2019-12-04 18:51:40,462 : INFO : PROGRESS: at sentence #54910000, processed 890412366 words, keeping 376822 word types\n",
      "2019-12-04 18:51:40,604 : INFO : PROGRESS: at sentence #54920000, processed 890520977 words, keeping 376822 word types\n",
      "2019-12-04 18:51:40,746 : INFO : PROGRESS: at sentence #54930000, processed 890627047 words, keeping 376822 word types\n",
      "2019-12-04 18:51:40,889 : INFO : PROGRESS: at sentence #54940000, processed 890734366 words, keeping 376822 word types\n",
      "2019-12-04 18:51:41,031 : INFO : PROGRESS: at sentence #54950000, processed 890840989 words, keeping 376822 word types\n",
      "2019-12-04 18:51:41,175 : INFO : PROGRESS: at sentence #54960000, processed 890948408 words, keeping 376822 word types\n",
      "2019-12-04 18:51:41,319 : INFO : PROGRESS: at sentence #54970000, processed 891055625 words, keeping 376822 word types\n",
      "2019-12-04 18:51:41,465 : INFO : PROGRESS: at sentence #54980000, processed 891164024 words, keeping 376822 word types\n",
      "2019-12-04 18:51:41,609 : INFO : PROGRESS: at sentence #54990000, processed 891274766 words, keeping 376822 word types\n",
      "2019-12-04 18:51:41,757 : INFO : PROGRESS: at sentence #55000000, processed 891388101 words, keeping 376822 word types\n",
      "2019-12-04 18:51:41,907 : INFO : PROGRESS: at sentence #55010000, processed 891501344 words, keeping 376822 word types\n",
      "2019-12-04 18:51:42,054 : INFO : PROGRESS: at sentence #55020000, processed 891613530 words, keeping 376822 word types\n",
      "2019-12-04 18:51:42,208 : INFO : PROGRESS: at sentence #55030000, processed 891727955 words, keeping 376822 word types\n",
      "2019-12-04 18:51:42,360 : INFO : PROGRESS: at sentence #55040000, processed 891845124 words, keeping 376822 word types\n",
      "2019-12-04 18:51:42,519 : INFO : PROGRESS: at sentence #55050000, processed 891966052 words, keeping 376822 word types\n",
      "2019-12-04 18:51:42,679 : INFO : PROGRESS: at sentence #55060000, processed 892086646 words, keeping 376822 word types\n",
      "2019-12-04 18:51:42,840 : INFO : PROGRESS: at sentence #55070000, processed 892209550 words, keeping 376822 word types\n",
      "2019-12-04 18:51:42,989 : INFO : PROGRESS: at sentence #55080000, processed 892326605 words, keeping 376822 word types\n",
      "2019-12-04 18:51:43,133 : INFO : PROGRESS: at sentence #55090000, processed 892438821 words, keeping 376822 word types\n",
      "2019-12-04 18:51:43,262 : INFO : PROGRESS: at sentence #55100000, processed 892538424 words, keeping 376822 word types\n",
      "2019-12-04 18:51:43,418 : INFO : PROGRESS: at sentence #55110000, processed 892639690 words, keeping 376822 word types\n",
      "2019-12-04 18:51:43,549 : INFO : PROGRESS: at sentence #55120000, processed 892742183 words, keeping 376822 word types\n",
      "2019-12-04 18:51:43,675 : INFO : PROGRESS: at sentence #55130000, processed 892840020 words, keeping 376822 word types\n",
      "2019-12-04 18:51:43,805 : INFO : PROGRESS: at sentence #55140000, processed 892942068 words, keeping 376822 word types\n",
      "2019-12-04 18:51:43,929 : INFO : PROGRESS: at sentence #55150000, processed 893036081 words, keeping 376822 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:51:44,057 : INFO : PROGRESS: at sentence #55160000, processed 893134294 words, keeping 376822 word types\n",
      "2019-12-04 18:51:44,183 : INFO : PROGRESS: at sentence #55170000, processed 893231265 words, keeping 376822 word types\n",
      "2019-12-04 18:51:44,313 : INFO : PROGRESS: at sentence #55180000, processed 893332824 words, keeping 376822 word types\n",
      "2019-12-04 18:51:44,447 : INFO : PROGRESS: at sentence #55190000, processed 893439134 words, keeping 376822 word types\n",
      "2019-12-04 18:51:44,582 : INFO : PROGRESS: at sentence #55200000, processed 893543385 words, keeping 376822 word types\n",
      "2019-12-04 18:51:44,725 : INFO : PROGRESS: at sentence #55210000, processed 893647876 words, keeping 376822 word types\n",
      "2019-12-04 18:51:44,863 : INFO : PROGRESS: at sentence #55220000, processed 893753256 words, keeping 376822 word types\n",
      "2019-12-04 18:51:45,002 : INFO : PROGRESS: at sentence #55230000, processed 893857855 words, keeping 376822 word types\n",
      "2019-12-04 18:51:45,134 : INFO : PROGRESS: at sentence #55240000, processed 893957765 words, keeping 376822 word types\n",
      "2019-12-04 18:51:45,272 : INFO : PROGRESS: at sentence #55250000, processed 894060781 words, keeping 376822 word types\n",
      "2019-12-04 18:51:45,417 : INFO : PROGRESS: at sentence #55260000, processed 894171861 words, keeping 376822 word types\n",
      "2019-12-04 18:51:45,578 : INFO : PROGRESS: at sentence #55270000, processed 894297820 words, keeping 376822 word types\n",
      "2019-12-04 18:51:45,727 : INFO : PROGRESS: at sentence #55280000, processed 894413728 words, keeping 376822 word types\n",
      "2019-12-04 18:51:45,873 : INFO : PROGRESS: at sentence #55290000, processed 894525943 words, keeping 376822 word types\n",
      "2019-12-04 18:51:46,010 : INFO : PROGRESS: at sentence #55300000, processed 894629458 words, keeping 376822 word types\n",
      "2019-12-04 18:51:46,148 : INFO : PROGRESS: at sentence #55310000, processed 894735858 words, keeping 376822 word types\n",
      "2019-12-04 18:51:46,292 : INFO : PROGRESS: at sentence #55320000, processed 894843914 words, keeping 376822 word types\n",
      "2019-12-04 18:51:46,434 : INFO : PROGRESS: at sentence #55330000, processed 894951379 words, keeping 376822 word types\n",
      "2019-12-04 18:51:46,576 : INFO : PROGRESS: at sentence #55340000, processed 895058709 words, keeping 376822 word types\n",
      "2019-12-04 18:51:46,719 : INFO : PROGRESS: at sentence #55350000, processed 895167060 words, keeping 376822 word types\n",
      "2019-12-04 18:51:46,863 : INFO : PROGRESS: at sentence #55360000, processed 895275797 words, keeping 376822 word types\n",
      "2019-12-04 18:51:47,007 : INFO : PROGRESS: at sentence #55370000, processed 895386224 words, keeping 376822 word types\n",
      "2019-12-04 18:51:47,159 : INFO : PROGRESS: at sentence #55380000, processed 895502500 words, keeping 376822 word types\n",
      "2019-12-04 18:51:47,305 : INFO : PROGRESS: at sentence #55390000, processed 895612606 words, keeping 376822 word types\n",
      "2019-12-04 18:51:47,456 : INFO : PROGRESS: at sentence #55400000, processed 895727090 words, keeping 376822 word types\n",
      "2019-12-04 18:51:47,603 : INFO : PROGRESS: at sentence #55410000, processed 895837980 words, keeping 376822 word types\n",
      "2019-12-04 18:51:47,767 : INFO : PROGRESS: at sentence #55420000, processed 895960853 words, keeping 376822 word types\n",
      "2019-12-04 18:51:47,926 : INFO : PROGRESS: at sentence #55430000, processed 896081030 words, keeping 376822 word types\n",
      "2019-12-04 18:51:48,087 : INFO : PROGRESS: at sentence #55440000, processed 896202519 words, keeping 376822 word types\n",
      "2019-12-04 18:51:48,249 : INFO : PROGRESS: at sentence #55450000, processed 896323686 words, keeping 376822 word types\n",
      "2019-12-04 18:51:48,408 : INFO : PROGRESS: at sentence #55460000, processed 896441406 words, keeping 376822 word types\n",
      "2019-12-04 18:51:48,548 : INFO : PROGRESS: at sentence #55470000, processed 896546232 words, keeping 376822 word types\n",
      "2019-12-04 18:51:48,683 : INFO : PROGRESS: at sentence #55480000, processed 896645142 words, keeping 376822 word types\n",
      "2019-12-04 18:51:48,821 : INFO : PROGRESS: at sentence #55490000, processed 896747604 words, keeping 376822 word types\n",
      "2019-12-04 18:51:48,960 : INFO : PROGRESS: at sentence #55500000, processed 896851205 words, keeping 376822 word types\n",
      "2019-12-04 18:51:49,089 : INFO : PROGRESS: at sentence #55510000, processed 896947128 words, keeping 376822 word types\n",
      "2019-12-04 18:51:49,221 : INFO : PROGRESS: at sentence #55520000, processed 897048024 words, keeping 376822 word types\n",
      "2019-12-04 18:51:49,346 : INFO : PROGRESS: at sentence #55530000, processed 897143166 words, keeping 376822 word types\n",
      "2019-12-04 18:51:49,474 : INFO : PROGRESS: at sentence #55540000, processed 897239777 words, keeping 376822 word types\n",
      "2019-12-04 18:51:49,606 : INFO : PROGRESS: at sentence #55550000, processed 897338861 words, keeping 376822 word types\n",
      "2019-12-04 18:51:49,744 : INFO : PROGRESS: at sentence #55560000, processed 897442810 words, keeping 376822 word types\n",
      "2019-12-04 18:51:49,887 : INFO : PROGRESS: at sentence #55570000, processed 897548007 words, keeping 376822 word types\n",
      "2019-12-04 18:51:50,022 : INFO : PROGRESS: at sentence #55580000, processed 897652409 words, keeping 376822 word types\n",
      "2019-12-04 18:51:50,159 : INFO : PROGRESS: at sentence #55590000, processed 897757396 words, keeping 376822 word types\n",
      "2019-12-04 18:51:50,298 : INFO : PROGRESS: at sentence #55600000, processed 897863143 words, keeping 376822 word types\n",
      "2019-12-04 18:51:50,434 : INFO : PROGRESS: at sentence #55610000, processed 897964961 words, keeping 376822 word types\n",
      "2019-12-04 18:51:50,571 : INFO : PROGRESS: at sentence #55620000, processed 898066576 words, keeping 376822 word types\n",
      "2019-12-04 18:51:50,709 : INFO : PROGRESS: at sentence #55630000, processed 898168598 words, keeping 376822 word types\n",
      "2019-12-04 18:51:50,872 : INFO : PROGRESS: at sentence #55640000, processed 898291442 words, keeping 376822 word types\n",
      "2019-12-04 18:51:51,027 : INFO : PROGRESS: at sentence #55650000, processed 898411081 words, keeping 376822 word types\n",
      "2019-12-04 18:51:51,176 : INFO : PROGRESS: at sentence #55660000, processed 898524842 words, keeping 376822 word types\n",
      "2019-12-04 18:51:51,321 : INFO : PROGRESS: at sentence #55670000, processed 898634632 words, keeping 376822 word types\n",
      "2019-12-04 18:51:51,466 : INFO : PROGRESS: at sentence #55680000, processed 898740590 words, keeping 376822 word types\n",
      "2019-12-04 18:51:51,610 : INFO : PROGRESS: at sentence #55690000, processed 898847077 words, keeping 376822 word types\n",
      "2019-12-04 18:51:51,753 : INFO : PROGRESS: at sentence #55700000, processed 898953799 words, keeping 376822 word types\n",
      "2019-12-04 18:51:51,901 : INFO : PROGRESS: at sentence #55710000, processed 899061329 words, keeping 376822 word types\n",
      "2019-12-04 18:51:52,046 : INFO : PROGRESS: at sentence #55720000, processed 899168375 words, keeping 376822 word types\n",
      "2019-12-04 18:51:52,191 : INFO : PROGRESS: at sentence #55730000, processed 899277353 words, keeping 376822 word types\n",
      "2019-12-04 18:51:52,340 : INFO : PROGRESS: at sentence #55740000, processed 899387098 words, keeping 376822 word types\n",
      "2019-12-04 18:51:52,492 : INFO : PROGRESS: at sentence #55750000, processed 899498041 words, keeping 376822 word types\n",
      "2019-12-04 18:51:52,645 : INFO : PROGRESS: at sentence #55760000, processed 899613578 words, keeping 376822 word types\n",
      "2019-12-04 18:51:52,794 : INFO : PROGRESS: at sentence #55770000, processed 899725001 words, keeping 376822 word types\n",
      "2019-12-04 18:51:52,946 : INFO : PROGRESS: at sentence #55780000, processed 899839416 words, keeping 376822 word types\n",
      "2019-12-04 18:51:53,099 : INFO : PROGRESS: at sentence #55790000, processed 899949764 words, keeping 376822 word types\n",
      "2019-12-04 18:51:53,271 : INFO : PROGRESS: at sentence #55800000, processed 900076094 words, keeping 376822 word types\n",
      "2019-12-04 18:51:53,437 : INFO : PROGRESS: at sentence #55810000, processed 900196611 words, keeping 376822 word types\n",
      "2019-12-04 18:51:53,601 : INFO : PROGRESS: at sentence #55820000, processed 900318947 words, keeping 376822 word types\n",
      "2019-12-04 18:51:53,756 : INFO : PROGRESS: at sentence #55830000, processed 900436128 words, keeping 376822 word types\n",
      "2019-12-04 18:51:53,912 : INFO : PROGRESS: at sentence #55840000, processed 900553980 words, keeping 376822 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:51:54,043 : INFO : PROGRESS: at sentence #55850000, processed 900653064 words, keeping 376822 word types\n",
      "2019-12-04 18:51:54,176 : INFO : PROGRESS: at sentence #55860000, processed 900753582 words, keeping 376822 word types\n",
      "2019-12-04 18:51:54,309 : INFO : PROGRESS: at sentence #55870000, processed 900855438 words, keeping 376822 word types\n",
      "2019-12-04 18:51:54,446 : INFO : PROGRESS: at sentence #55880000, processed 900960569 words, keeping 376822 word types\n",
      "2019-12-04 18:51:54,571 : INFO : PROGRESS: at sentence #55890000, processed 901055691 words, keeping 376822 word types\n",
      "2019-12-04 18:51:54,698 : INFO : PROGRESS: at sentence #55900000, processed 901152290 words, keeping 376822 word types\n",
      "2019-12-04 18:51:54,825 : INFO : PROGRESS: at sentence #55910000, processed 901249748 words, keeping 376822 word types\n",
      "2019-12-04 18:51:54,950 : INFO : PROGRESS: at sentence #55920000, processed 901346085 words, keeping 376822 word types\n",
      "2019-12-04 18:51:55,078 : INFO : PROGRESS: at sentence #55930000, processed 901445646 words, keeping 376822 word types\n",
      "2019-12-04 18:51:55,215 : INFO : PROGRESS: at sentence #55940000, processed 901552228 words, keeping 376822 word types\n",
      "2019-12-04 18:51:55,349 : INFO : PROGRESS: at sentence #55950000, processed 901657050 words, keeping 376822 word types\n",
      "2019-12-04 18:51:55,483 : INFO : PROGRESS: at sentence #55960000, processed 901760884 words, keeping 376822 word types\n",
      "2019-12-04 18:51:55,620 : INFO : PROGRESS: at sentence #55970000, processed 901866237 words, keeping 376822 word types\n",
      "2019-12-04 18:51:55,760 : INFO : PROGRESS: at sentence #55980000, processed 901971899 words, keeping 376822 word types\n",
      "2019-12-04 18:51:55,894 : INFO : PROGRESS: at sentence #55990000, processed 902072066 words, keeping 376822 word types\n",
      "2019-12-04 18:51:56,031 : INFO : PROGRESS: at sentence #56000000, processed 902174908 words, keeping 376822 word types\n",
      "2019-12-04 18:51:56,168 : INFO : PROGRESS: at sentence #56010000, processed 902278846 words, keeping 376822 word types\n",
      "2019-12-04 18:51:56,348 : INFO : PROGRESS: at sentence #56020000, processed 902408411 words, keeping 376822 word types\n",
      "2019-12-04 18:51:56,499 : INFO : PROGRESS: at sentence #56030000, processed 902523719 words, keeping 376822 word types\n",
      "2019-12-04 18:51:56,644 : INFO : PROGRESS: at sentence #56040000, processed 902636025 words, keeping 376822 word types\n",
      "2019-12-04 18:51:56,781 : INFO : PROGRESS: at sentence #56050000, processed 902743233 words, keeping 376822 word types\n",
      "2019-12-04 18:51:56,920 : INFO : PROGRESS: at sentence #56060000, processed 902849369 words, keeping 376822 word types\n",
      "2019-12-04 18:51:57,060 : INFO : PROGRESS: at sentence #56070000, processed 902957231 words, keeping 376822 word types\n",
      "2019-12-04 18:51:57,198 : INFO : PROGRESS: at sentence #56080000, processed 903063757 words, keeping 376822 word types\n",
      "2019-12-04 18:51:57,339 : INFO : PROGRESS: at sentence #56090000, processed 903171394 words, keeping 376822 word types\n",
      "2019-12-04 18:51:57,481 : INFO : PROGRESS: at sentence #56100000, processed 903279133 words, keeping 376822 word types\n",
      "2019-12-04 18:51:57,623 : INFO : PROGRESS: at sentence #56110000, processed 903387761 words, keeping 376822 word types\n",
      "2019-12-04 18:51:57,768 : INFO : PROGRESS: at sentence #56120000, processed 903498156 words, keeping 376822 word types\n",
      "2019-12-04 18:51:57,919 : INFO : PROGRESS: at sentence #56130000, processed 903613210 words, keeping 376822 word types\n",
      "2019-12-04 18:51:58,068 : INFO : PROGRESS: at sentence #56140000, processed 903724984 words, keeping 376822 word types\n",
      "2019-12-04 18:51:58,218 : INFO : PROGRESS: at sentence #56150000, processed 903837720 words, keeping 376822 word types\n",
      "2019-12-04 18:51:58,369 : INFO : PROGRESS: at sentence #56160000, processed 903950970 words, keeping 376822 word types\n",
      "2019-12-04 18:51:58,528 : INFO : PROGRESS: at sentence #56170000, processed 904070721 words, keeping 376822 word types\n",
      "2019-12-04 18:51:58,686 : INFO : PROGRESS: at sentence #56180000, processed 904190777 words, keeping 376822 word types\n",
      "2019-12-04 18:51:58,845 : INFO : PROGRESS: at sentence #56190000, processed 904311621 words, keeping 376822 word types\n",
      "2019-12-04 18:51:59,007 : INFO : PROGRESS: at sentence #56200000, processed 904434663 words, keeping 376822 word types\n",
      "2019-12-04 18:51:59,160 : INFO : PROGRESS: at sentence #56210000, processed 904552183 words, keeping 376822 word types\n",
      "2019-12-04 18:51:59,302 : INFO : PROGRESS: at sentence #56220000, processed 904660409 words, keeping 376822 word types\n",
      "2019-12-04 18:51:59,436 : INFO : PROGRESS: at sentence #56230000, processed 904760393 words, keeping 376822 word types\n",
      "2019-12-04 18:51:59,569 : INFO : PROGRESS: at sentence #56240000, processed 904861625 words, keeping 376822 word types\n",
      "2019-12-04 18:51:59,704 : INFO : PROGRESS: at sentence #56250000, processed 904964361 words, keeping 376822 word types\n",
      "2019-12-04 18:51:59,838 : INFO : PROGRESS: at sentence #56260000, processed 905062126 words, keeping 376822 word types\n",
      "2019-12-04 18:51:59,974 : INFO : PROGRESS: at sentence #56270000, processed 905163975 words, keeping 376822 word types\n",
      "2019-12-04 18:52:00,100 : INFO : PROGRESS: at sentence #56280000, processed 905258647 words, keeping 376822 word types\n",
      "2019-12-04 18:52:00,230 : INFO : PROGRESS: at sentence #56290000, processed 905355500 words, keeping 376822 word types\n",
      "2019-12-04 18:52:00,358 : INFO : PROGRESS: at sentence #56300000, processed 905453506 words, keeping 376822 word types\n",
      "2019-12-04 18:52:00,497 : INFO : PROGRESS: at sentence #56310000, processed 905555839 words, keeping 376822 word types\n",
      "2019-12-04 18:52:00,635 : INFO : PROGRESS: at sentence #56320000, processed 905661142 words, keeping 376822 word types\n",
      "2019-12-04 18:52:00,769 : INFO : PROGRESS: at sentence #56330000, processed 905765833 words, keeping 376822 word types\n",
      "2019-12-04 18:52:00,904 : INFO : PROGRESS: at sentence #56340000, processed 905870548 words, keeping 376822 word types\n",
      "2019-12-04 18:52:01,043 : INFO : PROGRESS: at sentence #56350000, processed 905976051 words, keeping 376822 word types\n",
      "2019-12-04 18:52:01,179 : INFO : PROGRESS: at sentence #56360000, processed 906079522 words, keeping 376822 word types\n",
      "2019-12-04 18:52:01,313 : INFO : PROGRESS: at sentence #56370000, processed 906180172 words, keeping 376822 word types\n",
      "2019-12-04 18:52:01,449 : INFO : PROGRESS: at sentence #56380000, processed 906282874 words, keeping 376822 word types\n",
      "2019-12-04 18:52:01,601 : INFO : PROGRESS: at sentence #56390000, processed 906399284 words, keeping 376822 word types\n",
      "2019-12-04 18:52:01,758 : INFO : PROGRESS: at sentence #56400000, processed 906521682 words, keeping 376822 word types\n",
      "2019-12-04 18:52:01,909 : INFO : PROGRESS: at sentence #56410000, processed 906637281 words, keeping 376822 word types\n",
      "2019-12-04 18:52:02,055 : INFO : PROGRESS: at sentence #56420000, processed 906748181 words, keeping 376822 word types\n",
      "2019-12-04 18:52:02,194 : INFO : PROGRESS: at sentence #56430000, processed 906852287 words, keeping 376822 word types\n",
      "2019-12-04 18:52:02,334 : INFO : PROGRESS: at sentence #56440000, processed 906958433 words, keeping 376822 word types\n",
      "2019-12-04 18:52:02,478 : INFO : PROGRESS: at sentence #56450000, processed 907066973 words, keeping 376822 word types\n",
      "2019-12-04 18:52:02,620 : INFO : PROGRESS: at sentence #56460000, processed 907174509 words, keeping 376822 word types\n",
      "2019-12-04 18:52:02,762 : INFO : PROGRESS: at sentence #56470000, processed 907281617 words, keeping 376822 word types\n",
      "2019-12-04 18:52:02,905 : INFO : PROGRESS: at sentence #56480000, processed 907389688 words, keeping 376822 word types\n",
      "2019-12-04 18:52:03,049 : INFO : PROGRESS: at sentence #56490000, processed 907499179 words, keeping 376822 word types\n",
      "2019-12-04 18:52:03,193 : INFO : PROGRESS: at sentence #56500000, processed 907609479 words, keeping 376822 word types\n",
      "2019-12-04 18:52:03,346 : INFO : PROGRESS: at sentence #56510000, processed 907725966 words, keeping 376822 word types\n",
      "2019-12-04 18:52:03,494 : INFO : PROGRESS: at sentence #56520000, processed 907835890 words, keeping 376822 word types\n",
      "2019-12-04 18:52:03,645 : INFO : PROGRESS: at sentence #56530000, processed 907950534 words, keeping 376822 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:52:03,793 : INFO : PROGRESS: at sentence #56540000, processed 908060582 words, keeping 376822 word types\n",
      "2019-12-04 18:52:03,959 : INFO : PROGRESS: at sentence #56550000, processed 908185866 words, keeping 376822 word types\n",
      "2019-12-04 18:52:04,118 : INFO : PROGRESS: at sentence #56560000, processed 908306097 words, keeping 376822 word types\n",
      "2019-12-04 18:52:04,278 : INFO : PROGRESS: at sentence #56570000, processed 908427907 words, keeping 376822 word types\n",
      "2019-12-04 18:52:04,432 : INFO : PROGRESS: at sentence #56580000, processed 908547367 words, keeping 376822 word types\n",
      "2019-12-04 18:52:04,586 : INFO : PROGRESS: at sentence #56590000, processed 908666275 words, keeping 376822 word types\n",
      "2019-12-04 18:52:04,720 : INFO : PROGRESS: at sentence #56600000, processed 908767766 words, keeping 376822 word types\n",
      "2019-12-04 18:52:04,851 : INFO : PROGRESS: at sentence #56610000, processed 908867199 words, keeping 376822 word types\n",
      "2019-12-04 18:52:04,985 : INFO : PROGRESS: at sentence #56620000, processed 908969371 words, keeping 376822 word types\n",
      "2019-12-04 18:52:05,120 : INFO : PROGRESS: at sentence #56630000, processed 909074303 words, keeping 376822 word types\n",
      "2019-12-04 18:52:05,245 : INFO : PROGRESS: at sentence #56640000, processed 909169917 words, keeping 376822 word types\n",
      "2019-12-04 18:52:05,372 : INFO : PROGRESS: at sentence #56650000, processed 909268930 words, keeping 376822 word types\n",
      "2019-12-04 18:52:05,495 : INFO : PROGRESS: at sentence #56660000, processed 909365169 words, keeping 376822 word types\n",
      "2019-12-04 18:52:05,619 : INFO : PROGRESS: at sentence #56670000, processed 909461134 words, keeping 376822 word types\n",
      "2019-12-04 18:52:05,747 : INFO : PROGRESS: at sentence #56680000, processed 909560611 words, keeping 376822 word types\n",
      "2019-12-04 18:52:05,885 : INFO : PROGRESS: at sentence #56690000, processed 909665404 words, keeping 376822 word types\n",
      "2019-12-04 18:52:06,019 : INFO : PROGRESS: at sentence #56700000, processed 909770321 words, keeping 376822 word types\n",
      "2019-12-04 18:52:06,153 : INFO : PROGRESS: at sentence #56710000, processed 909875136 words, keeping 376822 word types\n",
      "2019-12-04 18:52:06,288 : INFO : PROGRESS: at sentence #56720000, processed 909979698 words, keeping 376822 word types\n",
      "2019-12-04 18:52:06,426 : INFO : PROGRESS: at sentence #56730000, processed 910086117 words, keeping 376822 word types\n",
      "2019-12-04 18:52:06,558 : INFO : PROGRESS: at sentence #56740000, processed 910186846 words, keeping 376822 word types\n",
      "2019-12-04 18:52:06,692 : INFO : PROGRESS: at sentence #56750000, processed 910288869 words, keeping 376822 word types\n",
      "2019-12-04 18:52:06,826 : INFO : PROGRESS: at sentence #56760000, processed 910391135 words, keeping 376822 word types\n",
      "2019-12-04 18:52:07,021 : INFO : PROGRESS: at sentence #56770000, processed 910517922 words, keeping 376822 word types\n",
      "2019-12-04 18:52:07,176 : INFO : PROGRESS: at sentence #56780000, processed 910635217 words, keeping 376822 word types\n",
      "2019-12-04 18:52:07,324 : INFO : PROGRESS: at sentence #56790000, processed 910747938 words, keeping 376822 word types\n",
      "2019-12-04 18:52:07,465 : INFO : PROGRESS: at sentence #56800000, processed 910857120 words, keeping 376822 word types\n",
      "2019-12-04 18:52:07,605 : INFO : PROGRESS: at sentence #56810000, processed 910962977 words, keeping 376822 word types\n",
      "2019-12-04 18:52:07,749 : INFO : PROGRESS: at sentence #56820000, processed 911069975 words, keeping 376822 word types\n",
      "2019-12-04 18:52:07,891 : INFO : PROGRESS: at sentence #56830000, processed 911176788 words, keeping 376822 word types\n",
      "2019-12-04 18:52:08,054 : INFO : PROGRESS: at sentence #56840000, processed 911284119 words, keeping 376822 word types\n",
      "2019-12-04 18:52:08,195 : INFO : PROGRESS: at sentence #56850000, processed 911391312 words, keeping 376822 word types\n",
      "2019-12-04 18:52:08,337 : INFO : PROGRESS: at sentence #56860000, processed 911500228 words, keeping 376822 word types\n",
      "2019-12-04 18:52:08,479 : INFO : PROGRESS: at sentence #56870000, processed 911610073 words, keeping 376822 word types\n",
      "2019-12-04 18:52:08,625 : INFO : PROGRESS: at sentence #56880000, processed 911722873 words, keeping 376822 word types\n",
      "2019-12-04 18:52:08,775 : INFO : PROGRESS: at sentence #56890000, processed 911836819 words, keeping 376822 word types\n",
      "2019-12-04 18:52:08,925 : INFO : PROGRESS: at sentence #56900000, processed 911948936 words, keeping 376822 word types\n",
      "2019-12-04 18:52:09,078 : INFO : PROGRESS: at sentence #56910000, processed 912063563 words, keeping 376822 word types\n",
      "2019-12-04 18:52:09,230 : INFO : PROGRESS: at sentence #56920000, processed 912178796 words, keeping 376822 word types\n",
      "2019-12-04 18:52:09,388 : INFO : PROGRESS: at sentence #56930000, processed 912300703 words, keeping 376822 word types\n",
      "2019-12-04 18:52:09,546 : INFO : PROGRESS: at sentence #56940000, processed 912421088 words, keeping 376822 word types\n",
      "2019-12-04 18:52:09,707 : INFO : PROGRESS: at sentence #56950000, processed 912544401 words, keeping 376822 word types\n",
      "2019-12-04 18:52:09,855 : INFO : PROGRESS: at sentence #56960000, processed 912658924 words, keeping 376822 word types\n",
      "2019-12-04 18:52:10,008 : INFO : PROGRESS: at sentence #56970000, processed 912775466 words, keeping 376822 word types\n",
      "2019-12-04 18:52:10,138 : INFO : PROGRESS: at sentence #56980000, processed 912874788 words, keeping 376822 word types\n",
      "2019-12-04 18:52:10,277 : INFO : PROGRESS: at sentence #56990000, processed 912975760 words, keeping 376822 word types\n",
      "2019-12-04 18:52:10,413 : INFO : PROGRESS: at sentence #57000000, processed 913078359 words, keeping 376822 word types\n",
      "2019-12-04 18:52:10,542 : INFO : PROGRESS: at sentence #57010000, processed 913179861 words, keeping 376822 word types\n",
      "2019-12-04 18:52:10,707 : INFO : PROGRESS: at sentence #57020000, processed 913277879 words, keeping 376822 word types\n",
      "2019-12-04 18:52:10,828 : INFO : PROGRESS: at sentence #57030000, processed 913373010 words, keeping 376822 word types\n",
      "2019-12-04 18:52:10,954 : INFO : PROGRESS: at sentence #57040000, processed 913471004 words, keeping 376822 word types\n",
      "2019-12-04 18:52:11,081 : INFO : PROGRESS: at sentence #57050000, processed 913567918 words, keeping 376822 word types\n",
      "2019-12-04 18:52:11,212 : INFO : PROGRESS: at sentence #57060000, processed 913668819 words, keeping 376822 word types\n",
      "2019-12-04 18:52:11,349 : INFO : PROGRESS: at sentence #57070000, processed 913775008 words, keeping 376822 word types\n",
      "2019-12-04 18:52:11,481 : INFO : PROGRESS: at sentence #57080000, processed 913879835 words, keeping 376822 word types\n",
      "2019-12-04 18:52:11,615 : INFO : PROGRESS: at sentence #57090000, processed 913983614 words, keeping 376822 word types\n",
      "2019-12-04 18:52:11,749 : INFO : PROGRESS: at sentence #57100000, processed 914088607 words, keeping 376822 word types\n",
      "2019-12-04 18:52:11,884 : INFO : PROGRESS: at sentence #57110000, processed 914194132 words, keeping 376822 word types\n",
      "2019-12-04 18:52:12,016 : INFO : PROGRESS: at sentence #57120000, processed 914294205 words, keeping 376822 word types\n",
      "2019-12-04 18:52:12,151 : INFO : PROGRESS: at sentence #57130000, processed 914396981 words, keeping 376822 word types\n",
      "2019-12-04 18:52:12,290 : INFO : PROGRESS: at sentence #57140000, processed 914505925 words, keeping 376822 word types\n",
      "2019-12-04 18:52:12,450 : INFO : PROGRESS: at sentence #57150000, processed 914632718 words, keeping 376822 word types\n",
      "2019-12-04 18:52:12,599 : INFO : PROGRESS: at sentence #57160000, processed 914748913 words, keeping 376822 word types\n",
      "2019-12-04 18:52:12,741 : INFO : PROGRESS: at sentence #57170000, processed 914859291 words, keeping 376822 word types\n",
      "2019-12-04 18:52:12,881 : INFO : PROGRESS: at sentence #57180000, processed 914965526 words, keeping 376822 word types\n",
      "2019-12-04 18:52:13,024 : INFO : PROGRESS: at sentence #57190000, processed 915071648 words, keeping 376822 word types\n",
      "2019-12-04 18:52:13,164 : INFO : PROGRESS: at sentence #57200000, processed 915179929 words, keeping 376822 word types\n",
      "2019-12-04 18:52:13,307 : INFO : PROGRESS: at sentence #57210000, processed 915286997 words, keeping 376822 word types\n",
      "2019-12-04 18:52:13,447 : INFO : PROGRESS: at sentence #57220000, processed 915394488 words, keeping 376822 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:52:13,593 : INFO : PROGRESS: at sentence #57230000, processed 915502900 words, keeping 376822 word types\n",
      "2019-12-04 18:52:13,735 : INFO : PROGRESS: at sentence #57240000, processed 915611171 words, keeping 376822 word types\n",
      "2019-12-04 18:52:13,876 : INFO : PROGRESS: at sentence #57250000, processed 915721630 words, keeping 376822 word types\n",
      "2019-12-04 18:52:14,025 : INFO : PROGRESS: at sentence #57260000, processed 915837636 words, keeping 376822 word types\n",
      "2019-12-04 18:52:14,168 : INFO : PROGRESS: at sentence #57270000, processed 915948079 words, keeping 376822 word types\n",
      "2019-12-04 18:52:14,313 : INFO : PROGRESS: at sentence #57280000, processed 916061802 words, keeping 376822 word types\n",
      "2019-12-04 18:52:14,457 : INFO : PROGRESS: at sentence #57290000, processed 916173694 words, keeping 376822 word types\n",
      "2019-12-04 18:52:14,651 : INFO : PROGRESS: at sentence #57300000, processed 916295472 words, keeping 376822 word types\n",
      "2019-12-04 18:52:14,805 : INFO : PROGRESS: at sentence #57310000, processed 916415507 words, keeping 376822 word types\n",
      "2019-12-04 18:52:14,965 : INFO : PROGRESS: at sentence #57320000, processed 916536620 words, keeping 376822 word types\n",
      "2019-12-04 18:52:15,127 : INFO : PROGRESS: at sentence #57330000, processed 916658943 words, keeping 376822 word types\n",
      "2019-12-04 18:52:15,273 : INFO : PROGRESS: at sentence #57340000, processed 916775883 words, keeping 376822 word types\n",
      "2019-12-04 18:52:15,420 : INFO : PROGRESS: at sentence #57350000, processed 916882557 words, keeping 376822 word types\n",
      "2019-12-04 18:52:15,551 : INFO : PROGRESS: at sentence #57360000, processed 916981468 words, keeping 376822 word types\n",
      "2019-12-04 18:52:15,687 : INFO : PROGRESS: at sentence #57370000, processed 917083928 words, keeping 376822 word types\n",
      "2019-12-04 18:52:15,821 : INFO : PROGRESS: at sentence #57380000, processed 917187321 words, keeping 376822 word types\n",
      "2019-12-04 18:52:15,948 : INFO : PROGRESS: at sentence #57390000, processed 917285518 words, keeping 376822 word types\n",
      "2019-12-04 18:52:16,076 : INFO : PROGRESS: at sentence #57400000, processed 917384855 words, keeping 376822 word types\n",
      "2019-12-04 18:52:16,199 : INFO : PROGRESS: at sentence #57410000, processed 917479826 words, keeping 376822 word types\n",
      "2019-12-04 18:52:16,323 : INFO : PROGRESS: at sentence #57420000, processed 917576628 words, keeping 376822 word types\n",
      "2019-12-04 18:52:16,450 : INFO : PROGRESS: at sentence #57430000, processed 917675045 words, keeping 376822 word types\n",
      "2019-12-04 18:52:16,583 : INFO : PROGRESS: at sentence #57440000, processed 917778153 words, keeping 376822 word types\n",
      "2019-12-04 18:52:16,720 : INFO : PROGRESS: at sentence #57450000, processed 917883942 words, keeping 376822 word types\n",
      "2019-12-04 18:52:16,854 : INFO : PROGRESS: at sentence #57460000, processed 917988361 words, keeping 376822 word types\n",
      "2019-12-04 18:52:16,989 : INFO : PROGRESS: at sentence #57470000, processed 918093453 words, keeping 376822 word types\n",
      "2019-12-04 18:52:17,128 : INFO : PROGRESS: at sentence #57480000, processed 918199074 words, keeping 376822 word types\n",
      "2019-12-04 18:52:17,261 : INFO : PROGRESS: at sentence #57490000, processed 918301322 words, keeping 376822 word types\n",
      "2019-12-04 18:52:17,397 : INFO : PROGRESS: at sentence #57500000, processed 918402919 words, keeping 376822 word types\n",
      "2019-12-04 18:52:17,536 : INFO : PROGRESS: at sentence #57510000, processed 918504686 words, keeping 376822 word types\n",
      "2019-12-04 18:52:17,689 : INFO : PROGRESS: at sentence #57520000, processed 918625278 words, keeping 376822 word types\n",
      "2019-12-04 18:52:17,842 : INFO : PROGRESS: at sentence #57530000, processed 918745886 words, keeping 376822 word types\n",
      "2019-12-04 18:52:17,994 : INFO : PROGRESS: at sentence #57540000, processed 918860650 words, keeping 376822 word types\n",
      "2019-12-04 18:52:18,138 : INFO : PROGRESS: at sentence #57550000, processed 918970834 words, keeping 376822 word types\n",
      "2019-12-04 18:52:18,279 : INFO : PROGRESS: at sentence #57560000, processed 919076555 words, keeping 376822 word types\n",
      "2019-12-04 18:52:18,420 : INFO : PROGRESS: at sentence #57570000, processed 919182448 words, keeping 376822 word types\n",
      "2019-12-04 18:52:18,564 : INFO : PROGRESS: at sentence #57580000, processed 919289938 words, keeping 376822 word types\n",
      "2019-12-04 18:52:18,708 : INFO : PROGRESS: at sentence #57590000, processed 919397347 words, keeping 376822 word types\n",
      "2019-12-04 18:52:18,850 : INFO : PROGRESS: at sentence #57600000, processed 919504188 words, keeping 376822 word types\n",
      "2019-12-04 18:52:18,993 : INFO : PROGRESS: at sentence #57610000, processed 919612816 words, keeping 376822 word types\n",
      "2019-12-04 18:52:19,141 : INFO : PROGRESS: at sentence #57620000, processed 919722658 words, keeping 376822 word types\n",
      "2019-12-04 18:52:19,291 : INFO : PROGRESS: at sentence #57630000, processed 919833482 words, keeping 376822 word types\n",
      "2019-12-04 18:52:19,449 : INFO : PROGRESS: at sentence #57640000, processed 919949807 words, keeping 376822 word types\n",
      "2019-12-04 18:52:19,603 : INFO : PROGRESS: at sentence #57650000, processed 920059833 words, keeping 376822 word types\n",
      "2019-12-04 18:52:19,803 : INFO : PROGRESS: at sentence #57660000, processed 920174912 words, keeping 376822 word types\n",
      "2019-12-04 18:52:19,951 : INFO : PROGRESS: at sentence #57670000, processed 920283478 words, keeping 376822 word types\n",
      "2019-12-04 18:52:20,121 : INFO : PROGRESS: at sentence #57680000, processed 920410851 words, keeping 376822 word types\n",
      "2019-12-04 18:52:20,279 : INFO : PROGRESS: at sentence #57690000, processed 920531485 words, keeping 376822 word types\n",
      "2019-12-04 18:52:20,439 : INFO : PROGRESS: at sentence #57700000, processed 920653430 words, keeping 376822 word types\n",
      "2019-12-04 18:52:20,595 : INFO : PROGRESS: at sentence #57710000, processed 920771288 words, keeping 376822 word types\n",
      "2019-12-04 18:52:20,750 : INFO : PROGRESS: at sentence #57720000, processed 920890559 words, keeping 376822 word types\n",
      "2019-12-04 18:52:20,885 : INFO : PROGRESS: at sentence #57730000, processed 920989496 words, keeping 376822 word types\n",
      "2019-12-04 18:52:21,019 : INFO : PROGRESS: at sentence #57740000, processed 921089823 words, keeping 376822 word types\n",
      "2019-12-04 18:52:21,153 : INFO : PROGRESS: at sentence #57750000, processed 921191473 words, keeping 376822 word types\n",
      "2019-12-04 18:52:21,298 : INFO : PROGRESS: at sentence #57760000, processed 921296777 words, keeping 376822 word types\n",
      "2019-12-04 18:52:21,429 : INFO : PROGRESS: at sentence #57770000, processed 921392069 words, keeping 376822 word types\n",
      "2019-12-04 18:52:21,559 : INFO : PROGRESS: at sentence #57780000, processed 921489382 words, keeping 376822 word types\n",
      "2019-12-04 18:52:21,686 : INFO : PROGRESS: at sentence #57790000, processed 921586583 words, keeping 376822 word types\n",
      "2019-12-04 18:52:21,813 : INFO : PROGRESS: at sentence #57800000, processed 921682780 words, keeping 376822 word types\n",
      "2019-12-04 18:52:21,944 : INFO : PROGRESS: at sentence #57810000, processed 921781809 words, keeping 376822 word types\n",
      "2019-12-04 18:52:22,078 : INFO : PROGRESS: at sentence #57820000, processed 921887784 words, keeping 376822 word types\n",
      "2019-12-04 18:52:22,213 : INFO : PROGRESS: at sentence #57830000, processed 921993110 words, keeping 376822 word types\n",
      "2019-12-04 18:52:22,350 : INFO : PROGRESS: at sentence #57840000, processed 922097050 words, keeping 376822 word types\n",
      "2019-12-04 18:52:22,490 : INFO : PROGRESS: at sentence #57850000, processed 922202298 words, keeping 376822 word types\n",
      "2019-12-04 18:52:22,631 : INFO : PROGRESS: at sentence #57860000, processed 922308209 words, keeping 376822 word types\n",
      "2019-12-04 18:52:22,768 : INFO : PROGRESS: at sentence #57870000, processed 922408324 words, keeping 376822 word types\n",
      "2019-12-04 18:52:22,909 : INFO : PROGRESS: at sentence #57880000, processed 922510985 words, keeping 376822 word types\n",
      "2019-12-04 18:52:23,050 : INFO : PROGRESS: at sentence #57890000, processed 922613358 words, keeping 376822 word types\n",
      "2019-12-04 18:52:23,219 : INFO : PROGRESS: at sentence #57900000, processed 922743900 words, keeping 376822 word types\n",
      "2019-12-04 18:52:23,372 : INFO : PROGRESS: at sentence #57910000, processed 922859168 words, keeping 376822 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:52:23,521 : INFO : PROGRESS: at sentence #57920000, processed 922971603 words, keeping 376822 word types\n",
      "2019-12-04 18:52:23,669 : INFO : PROGRESS: at sentence #57930000, processed 923079335 words, keeping 376822 word types\n",
      "2019-12-04 18:52:23,817 : INFO : PROGRESS: at sentence #57940000, processed 923185453 words, keeping 376822 word types\n",
      "2019-12-04 18:52:23,963 : INFO : PROGRESS: at sentence #57950000, processed 923293008 words, keeping 376822 word types\n",
      "2019-12-04 18:52:24,106 : INFO : PROGRESS: at sentence #57960000, processed 923399517 words, keeping 376822 word types\n",
      "2019-12-04 18:52:24,258 : INFO : PROGRESS: at sentence #57970000, processed 923507058 words, keeping 376822 word types\n",
      "2019-12-04 18:52:24,408 : INFO : PROGRESS: at sentence #57980000, processed 923614611 words, keeping 376822 word types\n",
      "2019-12-04 18:52:24,554 : INFO : PROGRESS: at sentence #57990000, processed 923723187 words, keeping 376822 word types\n",
      "2019-12-04 18:52:24,699 : INFO : PROGRESS: at sentence #58000000, processed 923833648 words, keeping 376822 word types\n",
      "2019-12-04 18:52:24,848 : INFO : PROGRESS: at sentence #58010000, processed 923947718 words, keeping 376822 word types\n",
      "2019-12-04 18:52:24,998 : INFO : PROGRESS: at sentence #58020000, processed 924060420 words, keeping 376822 word types\n",
      "2019-12-04 18:52:25,153 : INFO : PROGRESS: at sentence #58030000, processed 924172788 words, keeping 376822 word types\n",
      "2019-12-04 18:52:25,310 : INFO : PROGRESS: at sentence #58040000, processed 924286701 words, keeping 376822 word types\n",
      "2019-12-04 18:52:25,471 : INFO : PROGRESS: at sentence #58050000, processed 924405000 words, keeping 376822 word types\n",
      "2019-12-04 18:52:25,638 : INFO : PROGRESS: at sentence #58060000, processed 924525246 words, keeping 376822 word types\n",
      "2019-12-04 18:52:25,803 : INFO : PROGRESS: at sentence #58070000, processed 924646091 words, keeping 376822 word types\n",
      "2019-12-04 18:52:25,973 : INFO : PROGRESS: at sentence #58080000, processed 924769284 words, keeping 376822 word types\n",
      "2019-12-04 18:52:26,129 : INFO : PROGRESS: at sentence #58090000, processed 924886572 words, keeping 376822 word types\n",
      "2019-12-04 18:52:26,276 : INFO : PROGRESS: at sentence #58100000, processed 924996892 words, keeping 376822 word types\n",
      "2019-12-04 18:52:26,409 : INFO : PROGRESS: at sentence #58110000, processed 925096848 words, keeping 376822 word types\n",
      "2019-12-04 18:52:26,542 : INFO : PROGRESS: at sentence #58120000, processed 925198056 words, keeping 376822 word types\n",
      "2019-12-04 18:52:26,677 : INFO : PROGRESS: at sentence #58130000, processed 925300631 words, keeping 376822 word types\n",
      "2019-12-04 18:52:26,809 : INFO : PROGRESS: at sentence #58140000, processed 925398047 words, keeping 376822 word types\n",
      "2019-12-04 18:52:26,941 : INFO : PROGRESS: at sentence #58150000, processed 925500587 words, keeping 376822 word types\n",
      "2019-12-04 18:52:27,064 : INFO : PROGRESS: at sentence #58160000, processed 925594660 words, keeping 376822 word types\n",
      "2019-12-04 18:52:27,193 : INFO : PROGRESS: at sentence #58170000, processed 925692003 words, keeping 376822 word types\n",
      "2019-12-04 18:52:27,319 : INFO : PROGRESS: at sentence #58180000, processed 925789549 words, keeping 376822 word types\n",
      "2019-12-04 18:52:27,448 : INFO : PROGRESS: at sentence #58190000, processed 925891635 words, keeping 376822 word types\n",
      "2019-12-04 18:52:27,584 : INFO : PROGRESS: at sentence #58200000, processed 925997708 words, keeping 376822 word types\n",
      "2019-12-04 18:52:27,720 : INFO : PROGRESS: at sentence #58210000, processed 926101865 words, keeping 376822 word types\n",
      "2019-12-04 18:52:27,854 : INFO : PROGRESS: at sentence #58220000, processed 926206394 words, keeping 376822 word types\n",
      "2019-12-04 18:52:27,990 : INFO : PROGRESS: at sentence #58230000, processed 926311962 words, keeping 376822 word types\n",
      "2019-12-04 18:52:28,126 : INFO : PROGRESS: at sentence #58240000, processed 926415844 words, keeping 376822 word types\n",
      "2019-12-04 18:52:28,260 : INFO : PROGRESS: at sentence #58250000, processed 926516108 words, keeping 376822 word types\n",
      "2019-12-04 18:52:28,395 : INFO : PROGRESS: at sentence #58260000, processed 926619122 words, keeping 376822 word types\n",
      "2019-12-04 18:52:28,543 : INFO : PROGRESS: at sentence #58270000, processed 926732807 words, keeping 376822 word types\n",
      "2019-12-04 18:52:28,704 : INFO : PROGRESS: at sentence #58280000, processed 926857062 words, keeping 376822 word types\n",
      "2019-12-04 18:52:28,852 : INFO : PROGRESS: at sentence #58290000, processed 926972845 words, keeping 376822 word types\n",
      "2019-12-04 18:52:28,996 : INFO : PROGRESS: at sentence #58300000, processed 927084612 words, keeping 376822 word types\n",
      "2019-12-04 18:52:29,134 : INFO : PROGRESS: at sentence #58310000, processed 927187938 words, keeping 376822 word types\n",
      "2019-12-04 18:52:29,274 : INFO : PROGRESS: at sentence #58320000, processed 927294452 words, keeping 376822 word types\n",
      "2019-12-04 18:52:29,418 : INFO : PROGRESS: at sentence #58330000, processed 927402600 words, keeping 376822 word types\n",
      "2019-12-04 18:52:29,558 : INFO : PROGRESS: at sentence #58340000, processed 927510201 words, keeping 376822 word types\n",
      "2019-12-04 18:52:29,698 : INFO : PROGRESS: at sentence #58350000, processed 927617433 words, keeping 376822 word types\n",
      "2019-12-04 18:52:29,841 : INFO : PROGRESS: at sentence #58360000, processed 927725526 words, keeping 376822 word types\n",
      "2019-12-04 18:52:29,984 : INFO : PROGRESS: at sentence #58370000, processed 927834862 words, keeping 376822 word types\n",
      "2019-12-04 18:52:30,130 : INFO : PROGRESS: at sentence #58380000, processed 927944888 words, keeping 376822 word types\n",
      "2019-12-04 18:52:30,283 : INFO : PROGRESS: at sentence #58390000, processed 928061396 words, keeping 376822 word types\n",
      "2019-12-04 18:52:30,430 : INFO : PROGRESS: at sentence #58400000, processed 928171457 words, keeping 376822 word types\n",
      "2019-12-04 18:52:30,580 : INFO : PROGRESS: at sentence #58410000, processed 928285378 words, keeping 376822 word types\n",
      "2019-12-04 18:52:30,729 : INFO : PROGRESS: at sentence #58420000, processed 928396221 words, keeping 376822 word types\n",
      "2019-12-04 18:52:30,892 : INFO : PROGRESS: at sentence #58430000, processed 928520370 words, keeping 376822 word types\n",
      "2019-12-04 18:52:31,050 : INFO : PROGRESS: at sentence #58440000, processed 928640471 words, keeping 376822 word types\n",
      "2019-12-04 18:52:31,206 : INFO : PROGRESS: at sentence #58450000, processed 928762358 words, keeping 376822 word types\n",
      "2019-12-04 18:52:31,361 : INFO : PROGRESS: at sentence #58460000, processed 928882800 words, keeping 376822 word types\n",
      "2019-12-04 18:52:31,510 : INFO : PROGRESS: at sentence #58470000, processed 929000664 words, keeping 376822 word types\n",
      "2019-12-04 18:52:31,642 : INFO : PROGRESS: at sentence #58480000, processed 929104185 words, keeping 376822 word types\n",
      "2019-12-04 18:52:31,781 : INFO : PROGRESS: at sentence #58490000, processed 929203601 words, keeping 376822 word types\n",
      "2019-12-04 18:52:31,913 : INFO : PROGRESS: at sentence #58500000, processed 929305756 words, keeping 376822 word types\n",
      "2019-12-04 18:52:32,048 : INFO : PROGRESS: at sentence #58510000, processed 929410174 words, keeping 376822 word types\n",
      "2019-12-04 18:52:32,173 : INFO : PROGRESS: at sentence #58520000, processed 929505297 words, keeping 376822 word types\n",
      "2019-12-04 18:52:32,303 : INFO : PROGRESS: at sentence #58530000, processed 929605693 words, keeping 376822 word types\n",
      "2019-12-04 18:52:32,428 : INFO : PROGRESS: at sentence #58540000, processed 929700931 words, keeping 376822 word types\n",
      "2019-12-04 18:52:32,554 : INFO : PROGRESS: at sentence #58550000, processed 929798127 words, keeping 376822 word types\n",
      "2019-12-04 18:52:32,682 : INFO : PROGRESS: at sentence #58560000, processed 929896899 words, keeping 376822 word types\n",
      "2019-12-04 18:52:32,816 : INFO : PROGRESS: at sentence #58570000, processed 930001447 words, keeping 376822 word types\n",
      "2019-12-04 18:52:32,995 : INFO : PROGRESS: at sentence #58580000, processed 930106278 words, keeping 376822 word types\n",
      "2019-12-04 18:52:33,131 : INFO : PROGRESS: at sentence #58590000, processed 930211239 words, keeping 376822 word types\n",
      "2019-12-04 18:52:33,264 : INFO : PROGRESS: at sentence #58600000, processed 930315683 words, keeping 376822 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:52:33,401 : INFO : PROGRESS: at sentence #58610000, processed 930422079 words, keeping 376822 word types\n",
      "2019-12-04 18:52:33,535 : INFO : PROGRESS: at sentence #58620000, processed 930523226 words, keeping 376822 word types\n",
      "2019-12-04 18:52:33,671 : INFO : PROGRESS: at sentence #58630000, processed 930625038 words, keeping 376822 word types\n",
      "2019-12-04 18:52:33,808 : INFO : PROGRESS: at sentence #58640000, processed 930727192 words, keeping 376822 word types\n",
      "2019-12-04 18:52:34,016 : INFO : PROGRESS: at sentence #58650000, processed 930851699 words, keeping 376822 word types\n",
      "2019-12-04 18:52:34,171 : INFO : PROGRESS: at sentence #58660000, processed 930970438 words, keeping 376822 word types\n",
      "2019-12-04 18:52:34,332 : INFO : PROGRESS: at sentence #58670000, processed 931083611 words, keeping 376822 word types\n",
      "2019-12-04 18:52:34,479 : INFO : PROGRESS: at sentence #58680000, processed 931192968 words, keeping 376822 word types\n",
      "2019-12-04 18:52:34,625 : INFO : PROGRESS: at sentence #58690000, processed 931299126 words, keeping 376822 word types\n",
      "2019-12-04 18:52:34,774 : INFO : PROGRESS: at sentence #58700000, processed 931405792 words, keeping 376822 word types\n",
      "2019-12-04 18:52:34,930 : INFO : PROGRESS: at sentence #58710000, processed 931512534 words, keeping 376822 word types\n",
      "2019-12-04 18:52:35,072 : INFO : PROGRESS: at sentence #58720000, processed 931619957 words, keeping 376822 word types\n",
      "2019-12-04 18:52:35,216 : INFO : PROGRESS: at sentence #58730000, processed 931727229 words, keeping 376822 word types\n",
      "2019-12-04 18:52:35,359 : INFO : PROGRESS: at sentence #58740000, processed 931835920 words, keeping 376822 word types\n",
      "2019-12-04 18:52:35,503 : INFO : PROGRESS: at sentence #58750000, processed 931945826 words, keeping 376822 word types\n",
      "2019-12-04 18:52:35,652 : INFO : PROGRESS: at sentence #58760000, processed 932057678 words, keeping 376822 word types\n",
      "2019-12-04 18:52:35,800 : INFO : PROGRESS: at sentence #58770000, processed 932172054 words, keeping 376822 word types\n",
      "2019-12-04 18:52:35,949 : INFO : PROGRESS: at sentence #58780000, processed 932284124 words, keeping 376822 word types\n",
      "2019-12-04 18:52:36,102 : INFO : PROGRESS: at sentence #58790000, processed 932398719 words, keeping 376822 word types\n",
      "2019-12-04 18:52:36,253 : INFO : PROGRESS: at sentence #58800000, processed 932511480 words, keeping 376822 word types\n",
      "2019-12-04 18:52:36,416 : INFO : PROGRESS: at sentence #58810000, processed 932635552 words, keeping 376822 word types\n",
      "2019-12-04 18:52:36,573 : INFO : PROGRESS: at sentence #58820000, processed 932756051 words, keeping 376822 word types\n",
      "2019-12-04 18:52:36,737 : INFO : PROGRESS: at sentence #58830000, processed 932878851 words, keeping 376822 word types\n",
      "2019-12-04 18:52:36,887 : INFO : PROGRESS: at sentence #58840000, processed 932994746 words, keeping 376822 word types\n",
      "2019-12-04 18:52:37,041 : INFO : PROGRESS: at sentence #58850000, processed 933111961 words, keeping 376822 word types\n",
      "2019-12-04 18:52:37,172 : INFO : PROGRESS: at sentence #58860000, processed 933211276 words, keeping 376822 word types\n",
      "2019-12-04 18:52:37,304 : INFO : PROGRESS: at sentence #58870000, processed 933312113 words, keeping 376822 word types\n",
      "2019-12-04 18:52:37,439 : INFO : PROGRESS: at sentence #58880000, processed 933414186 words, keeping 376822 word types\n",
      "2019-12-04 18:52:37,576 : INFO : PROGRESS: at sentence #58890000, processed 933519093 words, keeping 376822 word types\n",
      "2019-12-04 18:52:37,701 : INFO : PROGRESS: at sentence #58900000, processed 933613754 words, keeping 376822 word types\n",
      "2019-12-04 18:52:37,825 : INFO : PROGRESS: at sentence #58910000, processed 933710323 words, keeping 376822 word types\n",
      "2019-12-04 18:52:37,978 : INFO : PROGRESS: at sentence #58920000, processed 933807687 words, keeping 376822 word types\n",
      "2019-12-04 18:52:38,104 : INFO : PROGRESS: at sentence #58930000, processed 933904437 words, keeping 376822 word types\n",
      "2019-12-04 18:52:38,250 : INFO : PROGRESS: at sentence #58940000, processed 934004564 words, keeping 376822 word types\n",
      "2019-12-04 18:52:38,382 : INFO : PROGRESS: at sentence #58950000, processed 934110727 words, keeping 376822 word types\n",
      "2019-12-04 18:52:38,516 : INFO : PROGRESS: at sentence #58960000, processed 934215626 words, keeping 376822 word types\n",
      "2019-12-04 18:52:38,650 : INFO : PROGRESS: at sentence #58970000, processed 934319426 words, keeping 376822 word types\n",
      "2019-12-04 18:52:38,787 : INFO : PROGRESS: at sentence #58980000, processed 934424386 words, keeping 376822 word types\n",
      "2019-12-04 18:52:38,943 : INFO : PROGRESS: at sentence #58990000, processed 934530387 words, keeping 376822 word types\n",
      "2019-12-04 18:52:39,092 : INFO : PROGRESS: at sentence #59000000, processed 934630426 words, keeping 376822 word types\n",
      "2019-12-04 18:52:39,230 : INFO : PROGRESS: at sentence #59010000, processed 934733375 words, keeping 376822 word types\n",
      "2019-12-04 18:52:39,368 : INFO : PROGRESS: at sentence #59020000, processed 934839286 words, keeping 376822 word types\n",
      "2019-12-04 18:52:39,532 : INFO : PROGRESS: at sentence #59030000, processed 934967949 words, keeping 376822 word types\n",
      "2019-12-04 18:52:39,682 : INFO : PROGRESS: at sentence #59040000, processed 935083123 words, keeping 376822 word types\n",
      "2019-12-04 18:52:39,828 : INFO : PROGRESS: at sentence #59050000, processed 935194795 words, keeping 376822 word types\n",
      "2019-12-04 18:52:39,967 : INFO : PROGRESS: at sentence #59060000, processed 935301754 words, keeping 376822 word types\n",
      "2019-12-04 18:52:40,105 : INFO : PROGRESS: at sentence #59070000, processed 935407952 words, keeping 376822 word types\n",
      "2019-12-04 18:52:40,249 : INFO : PROGRESS: at sentence #59080000, processed 935515898 words, keeping 376822 word types\n",
      "2019-12-04 18:52:40,390 : INFO : PROGRESS: at sentence #59090000, processed 935622509 words, keeping 376822 word types\n",
      "2019-12-04 18:52:40,533 : INFO : PROGRESS: at sentence #59100000, processed 935730110 words, keeping 376822 word types\n",
      "2019-12-04 18:52:40,677 : INFO : PROGRESS: at sentence #59110000, processed 935838129 words, keeping 376822 word types\n",
      "2019-12-04 18:52:40,822 : INFO : PROGRESS: at sentence #59120000, processed 935946885 words, keeping 376822 word types\n",
      "2019-12-04 18:52:40,964 : INFO : PROGRESS: at sentence #59130000, processed 936056827 words, keeping 376822 word types\n",
      "2019-12-04 18:52:41,114 : INFO : PROGRESS: at sentence #59140000, processed 936172621 words, keeping 376822 word types\n",
      "2019-12-04 18:52:41,264 : INFO : PROGRESS: at sentence #59150000, processed 936283971 words, keeping 376822 word types\n",
      "2019-12-04 18:52:41,414 : INFO : PROGRESS: at sentence #59160000, processed 936396744 words, keeping 376822 word types\n",
      "2019-12-04 18:52:41,564 : INFO : PROGRESS: at sentence #59170000, processed 936509578 words, keeping 376822 word types\n",
      "2019-12-04 18:52:41,724 : INFO : PROGRESS: at sentence #59180000, processed 936630127 words, keeping 376822 word types\n",
      "2019-12-04 18:52:41,885 : INFO : PROGRESS: at sentence #59190000, processed 936750318 words, keeping 376822 word types\n",
      "2019-12-04 18:52:42,046 : INFO : PROGRESS: at sentence #59200000, processed 936871364 words, keeping 376822 word types\n",
      "2019-12-04 18:52:42,207 : INFO : PROGRESS: at sentence #59210000, processed 936994478 words, keeping 376822 word types\n",
      "2019-12-04 18:52:42,360 : INFO : PROGRESS: at sentence #59220000, processed 937111076 words, keeping 376822 word types\n",
      "2019-12-04 18:52:42,504 : INFO : PROGRESS: at sentence #59230000, processed 937218791 words, keeping 376822 word types\n",
      "2019-12-04 18:52:42,636 : INFO : PROGRESS: at sentence #59240000, processed 937318143 words, keeping 376822 word types\n",
      "2019-12-04 18:52:42,771 : INFO : PROGRESS: at sentence #59250000, processed 937420017 words, keeping 376822 word types\n",
      "2019-12-04 18:52:42,907 : INFO : PROGRESS: at sentence #59260000, processed 937523092 words, keeping 376822 word types\n",
      "2019-12-04 18:52:43,037 : INFO : PROGRESS: at sentence #59270000, processed 937620617 words, keeping 376822 word types\n",
      "2019-12-04 18:52:43,170 : INFO : PROGRESS: at sentence #59280000, processed 937721658 words, keeping 376822 word types\n",
      "2019-12-04 18:52:43,295 : INFO : PROGRESS: at sentence #59290000, processed 937816385 words, keeping 376822 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:52:43,422 : INFO : PROGRESS: at sentence #59300000, processed 937913358 words, keeping 376822 word types\n",
      "2019-12-04 18:52:43,549 : INFO : PROGRESS: at sentence #59310000, processed 938011552 words, keeping 376822 word types\n",
      "2019-12-04 18:52:43,680 : INFO : PROGRESS: at sentence #59320000, processed 938114144 words, keeping 376822 word types\n",
      "2019-12-04 18:52:43,817 : INFO : PROGRESS: at sentence #59330000, processed 938219980 words, keeping 376822 word types\n",
      "2019-12-04 18:52:43,950 : INFO : PROGRESS: at sentence #59340000, processed 938324552 words, keeping 376822 word types\n",
      "2019-12-04 18:52:44,085 : INFO : PROGRESS: at sentence #59350000, processed 938429162 words, keeping 376822 word types\n",
      "2019-12-04 18:52:44,222 : INFO : PROGRESS: at sentence #59360000, processed 938534413 words, keeping 376822 word types\n",
      "2019-12-04 18:52:44,356 : INFO : PROGRESS: at sentence #59370000, processed 938637759 words, keeping 376822 word types\n",
      "2019-12-04 18:52:44,493 : INFO : PROGRESS: at sentence #59380000, processed 938738871 words, keeping 376822 word types\n",
      "2019-12-04 18:52:44,631 : INFO : PROGRESS: at sentence #59390000, processed 938840995 words, keeping 376822 word types\n",
      "2019-12-04 18:52:44,784 : INFO : PROGRESS: at sentence #59400000, processed 938959364 words, keeping 376822 word types\n",
      "2019-12-04 18:52:44,938 : INFO : PROGRESS: at sentence #59410000, processed 939081177 words, keeping 376822 word types\n",
      "2019-12-04 18:52:45,085 : INFO : PROGRESS: at sentence #59420000, processed 939196323 words, keeping 376822 word types\n",
      "2019-12-04 18:52:45,229 : INFO : PROGRESS: at sentence #59430000, processed 939306517 words, keeping 376822 word types\n",
      "2019-12-04 18:52:45,371 : INFO : PROGRESS: at sentence #59440000, processed 939412009 words, keeping 376822 word types\n",
      "2019-12-04 18:52:45,515 : INFO : PROGRESS: at sentence #59450000, processed 939517075 words, keeping 376822 word types\n",
      "2019-12-04 18:52:45,661 : INFO : PROGRESS: at sentence #59460000, processed 939625628 words, keeping 376822 word types\n",
      "2019-12-04 18:52:45,803 : INFO : PROGRESS: at sentence #59470000, processed 939733323 words, keeping 376822 word types\n",
      "2019-12-04 18:52:45,944 : INFO : PROGRESS: at sentence #59480000, processed 939840101 words, keeping 376822 word types\n",
      "2019-12-04 18:52:46,087 : INFO : PROGRESS: at sentence #59490000, processed 939948310 words, keeping 376822 word types\n",
      "2019-12-04 18:52:46,232 : INFO : PROGRESS: at sentence #59500000, processed 940058025 words, keeping 376822 word types\n",
      "2019-12-04 18:52:46,375 : INFO : PROGRESS: at sentence #59510000, processed 940168695 words, keeping 376822 word types\n",
      "2019-12-04 18:52:46,526 : INFO : PROGRESS: at sentence #59520000, processed 940284952 words, keeping 376822 word types\n",
      "2019-12-04 18:52:46,669 : INFO : PROGRESS: at sentence #59530000, processed 940394937 words, keeping 376822 word types\n",
      "2019-12-04 18:52:46,820 : INFO : PROGRESS: at sentence #59540000, processed 940509693 words, keeping 376822 word types\n",
      "2019-12-04 18:52:46,964 : INFO : PROGRESS: at sentence #59550000, processed 940619434 words, keeping 376822 word types\n",
      "2019-12-04 18:52:47,126 : INFO : PROGRESS: at sentence #59560000, processed 940745237 words, keeping 376822 word types\n",
      "2019-12-04 18:52:47,283 : INFO : PROGRESS: at sentence #59570000, processed 940865858 words, keeping 376822 word types\n",
      "2019-12-04 18:52:47,447 : INFO : PROGRESS: at sentence #59580000, processed 940987823 words, keeping 376822 word types\n",
      "2019-12-04 18:52:47,601 : INFO : PROGRESS: at sentence #59590000, processed 941106318 words, keeping 376822 word types\n",
      "2019-12-04 18:52:47,756 : INFO : PROGRESS: at sentence #59600000, processed 941225490 words, keeping 376822 word types\n",
      "2019-12-04 18:52:47,890 : INFO : PROGRESS: at sentence #59610000, processed 941325877 words, keeping 376822 word types\n",
      "2019-12-04 18:52:48,021 : INFO : PROGRESS: at sentence #59620000, processed 941425744 words, keeping 376822 word types\n",
      "2019-12-04 18:52:48,155 : INFO : PROGRESS: at sentence #59630000, processed 941527712 words, keeping 376822 word types\n",
      "2019-12-04 18:52:48,290 : INFO : PROGRESS: at sentence #59640000, processed 941632833 words, keeping 376822 word types\n",
      "2019-12-04 18:52:48,412 : INFO : PROGRESS: at sentence #59650000, processed 941728560 words, keeping 376822 word types\n",
      "2019-12-04 18:52:48,538 : INFO : PROGRESS: at sentence #59660000, processed 941826296 words, keeping 376822 word types\n",
      "2019-12-04 18:52:48,664 : INFO : PROGRESS: at sentence #59670000, processed 941923008 words, keeping 376822 word types\n",
      "2019-12-04 18:52:48,819 : INFO : PROGRESS: at sentence #59680000, processed 942019181 words, keeping 376822 word types\n",
      "2019-12-04 18:52:48,949 : INFO : PROGRESS: at sentence #59690000, processed 942118737 words, keeping 376822 word types\n",
      "2019-12-04 18:52:49,086 : INFO : PROGRESS: at sentence #59700000, processed 942223902 words, keeping 376822 word types\n",
      "2019-12-04 18:52:49,219 : INFO : PROGRESS: at sentence #59710000, processed 942328998 words, keeping 376822 word types\n",
      "2019-12-04 18:52:49,354 : INFO : PROGRESS: at sentence #59720000, processed 942433499 words, keeping 376822 word types\n",
      "2019-12-04 18:52:49,491 : INFO : PROGRESS: at sentence #59730000, processed 942538402 words, keeping 376822 word types\n",
      "2019-12-04 18:52:49,629 : INFO : PROGRESS: at sentence #59740000, processed 942644214 words, keeping 376822 word types\n",
      "2019-12-04 18:52:49,764 : INFO : PROGRESS: at sentence #59750000, processed 942744954 words, keeping 376822 word types\n",
      "2019-12-04 18:52:49,900 : INFO : PROGRESS: at sentence #59760000, processed 942847354 words, keeping 376822 word types\n",
      "2019-12-04 18:52:50,043 : INFO : PROGRESS: at sentence #59770000, processed 942949857 words, keeping 376822 word types\n",
      "2019-12-04 18:52:50,210 : INFO : PROGRESS: at sentence #59780000, processed 943077905 words, keeping 376822 word types\n",
      "2019-12-04 18:52:50,361 : INFO : PROGRESS: at sentence #59790000, processed 943194458 words, keeping 376822 word types\n",
      "2019-12-04 18:52:50,508 : INFO : PROGRESS: at sentence #59800000, processed 943306857 words, keeping 376822 word types\n",
      "2019-12-04 18:52:50,654 : INFO : PROGRESS: at sentence #59810000, processed 943415466 words, keeping 376822 word types\n",
      "2019-12-04 18:52:50,800 : INFO : PROGRESS: at sentence #59820000, processed 943521378 words, keeping 376822 word types\n",
      "2019-12-04 18:52:51,002 : INFO : PROGRESS: at sentence #59830000, processed 943628787 words, keeping 376822 word types\n",
      "2019-12-04 18:52:51,149 : INFO : PROGRESS: at sentence #59840000, processed 943735413 words, keeping 376822 word types\n",
      "2019-12-04 18:52:51,293 : INFO : PROGRESS: at sentence #59850000, processed 943842941 words, keeping 376822 word types\n",
      "2019-12-04 18:52:51,439 : INFO : PROGRESS: at sentence #59860000, processed 943949914 words, keeping 376822 word types\n",
      "2019-12-04 18:52:51,585 : INFO : PROGRESS: at sentence #59870000, processed 944058386 words, keeping 376822 word types\n",
      "2019-12-04 18:52:51,730 : INFO : PROGRESS: at sentence #59880000, processed 944169095 words, keeping 376822 word types\n",
      "2019-12-04 18:52:51,888 : INFO : PROGRESS: at sentence #59890000, processed 944282380 words, keeping 376822 word types\n",
      "2019-12-04 18:52:52,047 : INFO : PROGRESS: at sentence #59900000, processed 944395501 words, keeping 376822 word types\n",
      "2019-12-04 18:52:52,197 : INFO : PROGRESS: at sentence #59910000, processed 944507928 words, keeping 376822 word types\n",
      "2019-12-04 18:52:52,348 : INFO : PROGRESS: at sentence #59920000, processed 944622459 words, keeping 376822 word types\n",
      "2019-12-04 18:52:52,508 : INFO : PROGRESS: at sentence #59930000, processed 944739264 words, keeping 376822 word types\n",
      "2019-12-04 18:52:52,670 : INFO : PROGRESS: at sentence #59940000, processed 944860198 words, keeping 376822 word types\n",
      "2019-12-04 18:52:52,830 : INFO : PROGRESS: at sentence #59950000, processed 944980859 words, keeping 376822 word types\n",
      "2019-12-04 18:52:52,994 : INFO : PROGRESS: at sentence #59960000, processed 945103895 words, keeping 376822 word types\n",
      "2019-12-04 18:52:53,150 : INFO : PROGRESS: at sentence #59970000, processed 945220641 words, keeping 376822 word types\n",
      "2019-12-04 18:52:53,303 : INFO : PROGRESS: at sentence #59980000, processed 945333508 words, keeping 376822 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:52:53,436 : INFO : PROGRESS: at sentence #59990000, processed 945432864 words, keeping 376822 word types\n",
      "2019-12-04 18:52:53,569 : INFO : PROGRESS: at sentence #60000000, processed 945534312 words, keeping 376822 word types\n",
      "2019-12-04 18:52:53,706 : INFO : PROGRESS: at sentence #60010000, processed 945636685 words, keeping 376822 word types\n",
      "2019-12-04 18:52:53,836 : INFO : PROGRESS: at sentence #60020000, processed 945734816 words, keeping 376822 word types\n",
      "2019-12-04 18:52:53,971 : INFO : PROGRESS: at sentence #60030000, processed 945836522 words, keeping 376822 word types\n",
      "2019-12-04 18:52:54,096 : INFO : PROGRESS: at sentence #60040000, processed 945930627 words, keeping 376822 word types\n",
      "2019-12-04 18:52:54,226 : INFO : PROGRESS: at sentence #60050000, processed 946028833 words, keeping 376822 word types\n",
      "2019-12-04 18:52:54,353 : INFO : PROGRESS: at sentence #60060000, processed 946125950 words, keeping 376822 word types\n",
      "2019-12-04 18:52:54,484 : INFO : PROGRESS: at sentence #60070000, processed 946227309 words, keeping 376822 word types\n",
      "2019-12-04 18:52:54,622 : INFO : PROGRESS: at sentence #60080000, processed 946333445 words, keeping 376822 word types\n",
      "2019-12-04 18:52:54,756 : INFO : PROGRESS: at sentence #60090000, processed 946437873 words, keeping 376822 word types\n",
      "2019-12-04 18:52:54,892 : INFO : PROGRESS: at sentence #60100000, processed 946542351 words, keeping 376822 word types\n",
      "2019-12-04 18:52:55,028 : INFO : PROGRESS: at sentence #60110000, processed 946647422 words, keeping 376822 word types\n",
      "2019-12-04 18:52:55,163 : INFO : PROGRESS: at sentence #60120000, processed 946752509 words, keeping 376822 word types\n",
      "2019-12-04 18:52:55,295 : INFO : PROGRESS: at sentence #60130000, processed 946852416 words, keeping 376822 word types\n",
      "2019-12-04 18:52:55,430 : INFO : PROGRESS: at sentence #60140000, processed 946955420 words, keeping 376822 word types\n",
      "2019-12-04 18:52:55,572 : INFO : PROGRESS: at sentence #60150000, processed 947065978 words, keeping 376822 word types\n",
      "2019-12-04 18:52:55,732 : INFO : PROGRESS: at sentence #60160000, processed 947192151 words, keeping 376822 word types\n",
      "2019-12-04 18:52:55,878 : INFO : PROGRESS: at sentence #60170000, processed 947308074 words, keeping 376822 word types\n",
      "2019-12-04 18:52:56,020 : INFO : PROGRESS: at sentence #60180000, processed 947420529 words, keeping 376822 word types\n",
      "2019-12-04 18:52:56,155 : INFO : PROGRESS: at sentence #60190000, processed 947524042 words, keeping 376822 word types\n",
      "2019-12-04 18:52:56,292 : INFO : PROGRESS: at sentence #60200000, processed 947630356 words, keeping 376822 word types\n",
      "2019-12-04 18:52:56,431 : INFO : PROGRESS: at sentence #60210000, processed 947738414 words, keeping 376822 word types\n",
      "2019-12-04 18:52:56,573 : INFO : PROGRESS: at sentence #60220000, processed 947845879 words, keeping 376822 word types\n",
      "2019-12-04 18:52:56,713 : INFO : PROGRESS: at sentence #60230000, processed 947953105 words, keeping 376822 word types\n",
      "2019-12-04 18:52:56,854 : INFO : PROGRESS: at sentence #60240000, processed 948061512 words, keeping 376822 word types\n",
      "2019-12-04 18:52:56,997 : INFO : PROGRESS: at sentence #60250000, processed 948170238 words, keeping 376822 word types\n",
      "2019-12-04 18:52:57,140 : INFO : PROGRESS: at sentence #60260000, processed 948280575 words, keeping 376822 word types\n",
      "2019-12-04 18:52:57,293 : INFO : PROGRESS: at sentence #60270000, processed 948396722 words, keeping 376822 word types\n",
      "2019-12-04 18:52:57,441 : INFO : PROGRESS: at sentence #60280000, processed 948506861 words, keeping 376822 word types\n",
      "2019-12-04 18:52:57,591 : INFO : PROGRESS: at sentence #60290000, processed 948621286 words, keeping 376822 word types\n",
      "2019-12-04 18:52:57,739 : INFO : PROGRESS: at sentence #60300000, processed 948732390 words, keeping 376822 word types\n",
      "2019-12-04 18:52:57,898 : INFO : PROGRESS: at sentence #60310000, processed 948854908 words, keeping 376822 word types\n",
      "2019-12-04 18:52:58,056 : INFO : PROGRESS: at sentence #60320000, processed 948975193 words, keeping 376822 word types\n",
      "2019-12-04 18:52:58,212 : INFO : PROGRESS: at sentence #60330000, processed 949096669 words, keeping 376822 word types\n",
      "2019-12-04 18:52:58,366 : INFO : PROGRESS: at sentence #60340000, processed 949218018 words, keeping 376822 word types\n",
      "2019-12-04 18:52:58,516 : INFO : PROGRESS: at sentence #60350000, processed 949335562 words, keeping 376822 word types\n",
      "2019-12-04 18:52:58,652 : INFO : PROGRESS: at sentence #60360000, processed 949440826 words, keeping 376822 word types\n",
      "2019-12-04 18:52:58,782 : INFO : PROGRESS: at sentence #60370000, processed 949539633 words, keeping 376822 word types\n",
      "2019-12-04 18:52:58,917 : INFO : PROGRESS: at sentence #60380000, processed 949642044 words, keeping 376822 word types\n",
      "2019-12-04 18:52:59,051 : INFO : PROGRESS: at sentence #60390000, processed 949745738 words, keeping 376822 word types\n",
      "2019-12-04 18:52:59,176 : INFO : PROGRESS: at sentence #60400000, processed 949841689 words, keeping 376822 word types\n",
      "2019-12-04 18:52:59,306 : INFO : PROGRESS: at sentence #60410000, processed 949942635 words, keeping 376822 word types\n",
      "2019-12-04 18:52:59,428 : INFO : PROGRESS: at sentence #60420000, processed 950037881 words, keeping 376822 word types\n",
      "2019-12-04 18:52:59,553 : INFO : PROGRESS: at sentence #60430000, processed 950134363 words, keeping 376822 word types\n",
      "2019-12-04 18:52:59,679 : INFO : PROGRESS: at sentence #60440000, processed 950233642 words, keeping 376822 word types\n",
      "2019-12-04 18:52:59,811 : INFO : PROGRESS: at sentence #60450000, processed 950337046 words, keeping 376822 word types\n",
      "2019-12-04 18:52:59,946 : INFO : PROGRESS: at sentence #60460000, processed 950442509 words, keeping 376822 word types\n",
      "2019-12-04 18:53:00,080 : INFO : PROGRESS: at sentence #60470000, processed 950546905 words, keeping 376822 word types\n",
      "2019-12-04 18:53:00,216 : INFO : PROGRESS: at sentence #60480000, processed 950652134 words, keeping 376822 word types\n",
      "2019-12-04 18:53:00,353 : INFO : PROGRESS: at sentence #60490000, processed 950757585 words, keeping 376822 word types\n",
      "2019-12-04 18:53:00,487 : INFO : PROGRESS: at sentence #60500000, processed 950859588 words, keeping 376822 word types\n",
      "2019-12-04 18:53:00,619 : INFO : PROGRESS: at sentence #60510000, processed 950961213 words, keeping 376822 word types\n",
      "2019-12-04 18:53:00,753 : INFO : PROGRESS: at sentence #60520000, processed 951063089 words, keeping 376822 word types\n",
      "2019-12-04 18:53:00,909 : INFO : PROGRESS: at sentence #60530000, processed 951185567 words, keeping 376822 word types\n",
      "2019-12-04 18:53:01,064 : INFO : PROGRESS: at sentence #60540000, processed 951305324 words, keeping 376822 word types\n",
      "2019-12-04 18:53:01,212 : INFO : PROGRESS: at sentence #60550000, processed 951419231 words, keeping 376822 word types\n",
      "2019-12-04 18:53:01,356 : INFO : PROGRESS: at sentence #60560000, processed 951529146 words, keeping 376822 word types\n",
      "2019-12-04 18:53:01,496 : INFO : PROGRESS: at sentence #60570000, processed 951634910 words, keeping 376822 word types\n",
      "2019-12-04 18:53:01,634 : INFO : PROGRESS: at sentence #60580000, processed 951741579 words, keeping 376822 word types\n",
      "2019-12-04 18:53:01,775 : INFO : PROGRESS: at sentence #60590000, processed 951848274 words, keeping 376822 word types\n",
      "2019-12-04 18:53:01,916 : INFO : PROGRESS: at sentence #60600000, processed 951955930 words, keeping 376822 word types\n",
      "2019-12-04 18:53:02,056 : INFO : PROGRESS: at sentence #60610000, processed 952062861 words, keeping 376822 word types\n",
      "2019-12-04 18:53:02,199 : INFO : PROGRESS: at sentence #60620000, processed 952171803 words, keeping 376822 word types\n",
      "2019-12-04 18:53:02,342 : INFO : PROGRESS: at sentence #60630000, processed 952281451 words, keeping 376822 word types\n",
      "2019-12-04 18:53:02,486 : INFO : PROGRESS: at sentence #60640000, processed 952392479 words, keeping 376822 word types\n",
      "2019-12-04 18:53:02,639 : INFO : PROGRESS: at sentence #60650000, processed 952508126 words, keeping 376822 word types\n",
      "2019-12-04 18:53:02,786 : INFO : PROGRESS: at sentence #60660000, processed 952619188 words, keeping 376822 word types\n",
      "2019-12-04 18:53:02,936 : INFO : PROGRESS: at sentence #60670000, processed 952733920 words, keeping 376822 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:53:03,082 : INFO : PROGRESS: at sentence #60680000, processed 952843247 words, keeping 376822 word types\n",
      "2019-12-04 18:53:03,247 : INFO : PROGRESS: at sentence #60690000, processed 952970325 words, keeping 376822 word types\n",
      "2019-12-04 18:53:03,402 : INFO : PROGRESS: at sentence #60700000, processed 953090899 words, keeping 376822 word types\n",
      "2019-12-04 18:53:03,560 : INFO : PROGRESS: at sentence #60710000, processed 953213123 words, keeping 376822 word types\n",
      "2019-12-04 18:53:03,712 : INFO : PROGRESS: at sentence #60720000, processed 953330658 words, keeping 376822 word types\n",
      "2019-12-04 18:53:03,864 : INFO : PROGRESS: at sentence #60730000, processed 953448636 words, keeping 376822 word types\n",
      "2019-12-04 18:53:03,995 : INFO : PROGRESS: at sentence #60740000, processed 953547766 words, keeping 376822 word types\n",
      "2019-12-04 18:53:04,131 : INFO : PROGRESS: at sentence #60750000, processed 953648111 words, keeping 376822 word types\n",
      "2019-12-04 18:53:04,263 : INFO : PROGRESS: at sentence #60760000, processed 953749835 words, keeping 376822 word types\n",
      "2019-12-04 18:53:04,400 : INFO : PROGRESS: at sentence #60770000, processed 953855063 words, keeping 376822 word types\n",
      "2019-12-04 18:53:04,522 : INFO : PROGRESS: at sentence #60780000, processed 953950397 words, keeping 376822 word types\n",
      "2019-12-04 18:53:04,645 : INFO : PROGRESS: at sentence #60790000, processed 954046919 words, keeping 376822 word types\n",
      "2019-12-04 18:53:04,772 : INFO : PROGRESS: at sentence #60800000, processed 954144295 words, keeping 376822 word types\n",
      "2019-12-04 18:53:04,897 : INFO : PROGRESS: at sentence #60810000, processed 954240665 words, keeping 376822 word types\n",
      "2019-12-04 18:53:05,027 : INFO : PROGRESS: at sentence #60820000, processed 954340117 words, keeping 376822 word types\n",
      "2019-12-04 18:53:05,164 : INFO : PROGRESS: at sentence #60830000, processed 954446622 words, keeping 376822 word types\n",
      "2019-12-04 18:53:05,298 : INFO : PROGRESS: at sentence #60840000, processed 954551572 words, keeping 376822 word types\n",
      "2019-12-04 18:53:05,432 : INFO : PROGRESS: at sentence #60850000, processed 954655463 words, keeping 376822 word types\n",
      "2019-12-04 18:53:05,567 : INFO : PROGRESS: at sentence #60860000, processed 954760550 words, keeping 376822 word types\n",
      "2019-12-04 18:53:05,705 : INFO : PROGRESS: at sentence #60870000, processed 954866409 words, keeping 376822 word types\n",
      "2019-12-04 18:53:05,839 : INFO : PROGRESS: at sentence #60880000, processed 954966622 words, keeping 376822 word types\n",
      "2019-12-04 18:53:05,981 : INFO : PROGRESS: at sentence #60890000, processed 955069368 words, keeping 376822 word types\n",
      "2019-12-04 18:53:06,118 : INFO : PROGRESS: at sentence #60900000, processed 955172959 words, keeping 376822 word types\n",
      "2019-12-04 18:53:06,284 : INFO : PROGRESS: at sentence #60910000, processed 955303035 words, keeping 376822 word types\n",
      "2019-12-04 18:53:06,433 : INFO : PROGRESS: at sentence #60920000, processed 955418056 words, keeping 376822 word types\n",
      "2019-12-04 18:53:06,580 : INFO : PROGRESS: at sentence #60930000, processed 955530554 words, keeping 376822 word types\n",
      "2019-12-04 18:53:06,722 : INFO : PROGRESS: at sentence #60940000, processed 955637846 words, keeping 376822 word types\n",
      "2019-12-04 18:53:06,864 : INFO : PROGRESS: at sentence #60950000, processed 955743917 words, keeping 376822 word types\n",
      "2019-12-04 18:53:07,012 : INFO : PROGRESS: at sentence #60960000, processed 955851669 words, keeping 376822 word types\n",
      "2019-12-04 18:53:07,154 : INFO : PROGRESS: at sentence #60970000, processed 955958103 words, keeping 376822 word types\n",
      "2019-12-04 18:53:07,299 : INFO : PROGRESS: at sentence #60980000, processed 956065836 words, keeping 376822 word types\n",
      "2019-12-04 18:53:07,453 : INFO : PROGRESS: at sentence #60990000, processed 956173546 words, keeping 376822 word types\n",
      "2019-12-04 18:53:07,600 : INFO : PROGRESS: at sentence #61000000, processed 956282124 words, keeping 376822 word types\n",
      "2019-12-04 18:53:07,745 : INFO : PROGRESS: at sentence #61010000, processed 956392502 words, keeping 376822 word types\n",
      "2019-12-04 18:53:07,896 : INFO : PROGRESS: at sentence #61020000, processed 956507403 words, keeping 376822 word types\n",
      "2019-12-04 18:53:08,048 : INFO : PROGRESS: at sentence #61030000, processed 956619417 words, keeping 376822 word types\n",
      "2019-12-04 18:53:08,199 : INFO : PROGRESS: at sentence #61040000, processed 956732201 words, keeping 376822 word types\n",
      "2019-12-04 18:53:08,352 : INFO : PROGRESS: at sentence #61050000, processed 956845463 words, keeping 376822 word types\n",
      "2019-12-04 18:53:08,511 : INFO : PROGRESS: at sentence #61060000, processed 956964829 words, keeping 376822 word types\n",
      "2019-12-04 18:53:08,672 : INFO : PROGRESS: at sentence #61070000, processed 957084984 words, keeping 376822 word types\n",
      "2019-12-04 18:53:08,835 : INFO : PROGRESS: at sentence #61080000, processed 957205900 words, keeping 376822 word types\n",
      "2019-12-04 18:53:08,996 : INFO : PROGRESS: at sentence #61090000, processed 957328708 words, keeping 376822 word types\n",
      "2019-12-04 18:53:09,150 : INFO : PROGRESS: at sentence #61100000, processed 957446234 words, keeping 376822 word types\n",
      "2019-12-04 18:53:09,295 : INFO : PROGRESS: at sentence #61110000, processed 957554996 words, keeping 376822 word types\n",
      "2019-12-04 18:53:09,430 : INFO : PROGRESS: at sentence #61120000, processed 957654837 words, keeping 376822 word types\n",
      "2019-12-04 18:53:09,567 : INFO : PROGRESS: at sentence #61130000, processed 957756213 words, keeping 376822 word types\n",
      "2019-12-04 18:53:09,706 : INFO : PROGRESS: at sentence #61140000, processed 957858891 words, keeping 376822 word types\n",
      "2019-12-04 18:53:09,838 : INFO : PROGRESS: at sentence #61150000, processed 957956616 words, keeping 376822 word types\n",
      "2019-12-04 18:53:09,975 : INFO : PROGRESS: at sentence #61160000, processed 958058658 words, keeping 376822 word types\n",
      "2019-12-04 18:53:10,100 : INFO : PROGRESS: at sentence #61170000, processed 958153299 words, keeping 376822 word types\n",
      "2019-12-04 18:53:10,227 : INFO : PROGRESS: at sentence #61180000, processed 958250006 words, keeping 376822 word types\n",
      "2019-12-04 18:53:10,354 : INFO : PROGRESS: at sentence #61190000, processed 958347966 words, keeping 376822 word types\n",
      "2019-12-04 18:53:10,489 : INFO : PROGRESS: at sentence #61200000, processed 958450060 words, keeping 376822 word types\n",
      "2019-12-04 18:53:10,631 : INFO : PROGRESS: at sentence #61210000, processed 958555658 words, keeping 376822 word types\n",
      "2019-12-04 18:53:10,769 : INFO : PROGRESS: at sentence #61220000, processed 958660406 words, keeping 376822 word types\n",
      "2019-12-04 18:53:10,910 : INFO : PROGRESS: at sentence #61230000, processed 958765008 words, keeping 376822 word types\n",
      "2019-12-04 18:53:11,053 : INFO : PROGRESS: at sentence #61240000, processed 958870494 words, keeping 376822 word types\n",
      "2019-12-04 18:53:11,192 : INFO : PROGRESS: at sentence #61250000, processed 958973842 words, keeping 376822 word types\n",
      "2019-12-04 18:53:11,326 : INFO : PROGRESS: at sentence #61260000, processed 959074640 words, keeping 376822 word types\n",
      "2019-12-04 18:53:11,467 : INFO : PROGRESS: at sentence #61270000, processed 959177554 words, keeping 376822 word types\n",
      "2019-12-04 18:53:11,622 : INFO : PROGRESS: at sentence #61280000, processed 959293322 words, keeping 376822 word types\n",
      "2019-12-04 18:53:11,784 : INFO : PROGRESS: at sentence #61290000, processed 959415939 words, keeping 376822 word types\n",
      "2019-12-04 18:53:11,940 : INFO : PROGRESS: at sentence #61300000, processed 959531759 words, keeping 376822 word types\n",
      "2019-12-04 18:53:12,084 : INFO : PROGRESS: at sentence #61310000, processed 959642773 words, keeping 376822 word types\n",
      "2019-12-04 18:53:12,224 : INFO : PROGRESS: at sentence #61320000, processed 959746449 words, keeping 376822 word types\n",
      "2019-12-04 18:53:12,371 : INFO : PROGRESS: at sentence #61330000, processed 959852893 words, keeping 376822 word types\n",
      "2019-12-04 18:53:12,541 : INFO : PROGRESS: at sentence #61340000, processed 959961316 words, keeping 376822 word types\n",
      "2019-12-04 18:53:12,686 : INFO : PROGRESS: at sentence #61350000, processed 960068938 words, keeping 376822 word types\n",
      "2019-12-04 18:53:12,832 : INFO : PROGRESS: at sentence #61360000, processed 960176039 words, keeping 376822 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:53:12,976 : INFO : PROGRESS: at sentence #61370000, processed 960284102 words, keeping 376822 word types\n",
      "2019-12-04 18:53:13,121 : INFO : PROGRESS: at sentence #61380000, processed 960393532 words, keeping 376822 word types\n",
      "2019-12-04 18:53:13,271 : INFO : PROGRESS: at sentence #61390000, processed 960504002 words, keeping 376822 word types\n",
      "2019-12-04 18:53:13,426 : INFO : PROGRESS: at sentence #61400000, processed 960620483 words, keeping 376822 word types\n",
      "2019-12-04 18:53:13,575 : INFO : PROGRESS: at sentence #61410000, processed 960730262 words, keeping 376822 word types\n",
      "2019-12-04 18:53:13,730 : INFO : PROGRESS: at sentence #61420000, processed 960844784 words, keeping 376822 word types\n",
      "2019-12-04 18:53:13,879 : INFO : PROGRESS: at sentence #61430000, processed 960955003 words, keeping 376822 word types\n",
      "2019-12-04 18:53:14,048 : INFO : PROGRESS: at sentence #61440000, processed 961080179 words, keeping 376822 word types\n",
      "2019-12-04 18:53:14,211 : INFO : PROGRESS: at sentence #61450000, processed 961200260 words, keeping 376822 word types\n",
      "2019-12-04 18:53:14,378 : INFO : PROGRESS: at sentence #61460000, processed 961322231 words, keeping 376822 word types\n",
      "2019-12-04 18:53:14,535 : INFO : PROGRESS: at sentence #61470000, processed 961441622 words, keeping 376822 word types\n",
      "2019-12-04 18:53:14,692 : INFO : PROGRESS: at sentence #61480000, processed 961560388 words, keeping 376822 word types\n",
      "2019-12-04 18:53:14,832 : INFO : PROGRESS: at sentence #61490000, processed 961662257 words, keeping 376822 word types\n",
      "2019-12-04 18:53:14,970 : INFO : PROGRESS: at sentence #61500000, processed 961761879 words, keeping 376822 word types\n",
      "2019-12-04 18:53:15,105 : INFO : PROGRESS: at sentence #61510000, processed 961863820 words, keeping 376822 word types\n",
      "2019-12-04 18:53:15,247 : INFO : PROGRESS: at sentence #61520000, processed 961968790 words, keeping 376822 word types\n",
      "2019-12-04 18:53:15,373 : INFO : PROGRESS: at sentence #61530000, processed 962064075 words, keeping 376822 word types\n",
      "2019-12-04 18:53:15,505 : INFO : PROGRESS: at sentence #61540000, processed 962163610 words, keeping 376822 word types\n",
      "2019-12-04 18:53:15,634 : INFO : PROGRESS: at sentence #61550000, processed 962259495 words, keeping 376822 word types\n",
      "2019-12-04 18:53:15,763 : INFO : PROGRESS: at sentence #61560000, processed 962355751 words, keeping 376822 word types\n",
      "2019-12-04 18:53:15,897 : INFO : PROGRESS: at sentence #61570000, processed 962455165 words, keeping 376822 word types\n",
      "2019-12-04 18:53:16,037 : INFO : PROGRESS: at sentence #61580000, processed 962559848 words, keeping 376822 word types\n",
      "2019-12-04 18:53:16,175 : INFO : PROGRESS: at sentence #61590000, processed 962664966 words, keeping 376822 word types\n",
      "2019-12-04 18:53:16,314 : INFO : PROGRESS: at sentence #61600000, processed 962769524 words, keeping 376822 word types\n",
      "2019-12-04 18:53:16,454 : INFO : PROGRESS: at sentence #61610000, processed 962874259 words, keeping 376822 word types\n",
      "2019-12-04 18:53:16,592 : INFO : PROGRESS: at sentence #61620000, processed 962980852 words, keeping 376822 word types\n",
      "2019-12-04 18:53:16,728 : INFO : PROGRESS: at sentence #61630000, processed 963081447 words, keeping 376822 word types\n",
      "2019-12-04 18:53:16,864 : INFO : PROGRESS: at sentence #61640000, processed 963183219 words, keeping 376822 word types\n",
      "2019-12-04 18:53:17,005 : INFO : PROGRESS: at sentence #61650000, processed 963285713 words, keeping 376822 word types\n",
      "2019-12-04 18:53:17,169 : INFO : PROGRESS: at sentence #61660000, processed 963411940 words, keeping 376822 word types\n",
      "2019-12-04 18:53:17,325 : INFO : PROGRESS: at sentence #61670000, processed 963529603 words, keeping 376822 word types\n",
      "2019-12-04 18:53:17,474 : INFO : PROGRESS: at sentence #61680000, processed 963642322 words, keeping 376822 word types\n",
      "2019-12-04 18:53:17,620 : INFO : PROGRESS: at sentence #61690000, processed 963751661 words, keeping 376822 word types\n",
      "2019-12-04 18:53:17,760 : INFO : PROGRESS: at sentence #61700000, processed 963857536 words, keeping 376822 word types\n",
      "2019-12-04 18:53:17,903 : INFO : PROGRESS: at sentence #61710000, processed 963964402 words, keeping 376822 word types\n",
      "2019-12-04 18:53:18,045 : INFO : PROGRESS: at sentence #61720000, processed 964071258 words, keeping 376822 word types\n",
      "2019-12-04 18:53:18,191 : INFO : PROGRESS: at sentence #61730000, processed 964178566 words, keeping 376822 word types\n",
      "2019-12-04 18:53:18,338 : INFO : PROGRESS: at sentence #61740000, processed 964285827 words, keeping 376822 word types\n",
      "2019-12-04 18:53:18,483 : INFO : PROGRESS: at sentence #61750000, processed 964394601 words, keeping 376822 word types\n",
      "2019-12-04 18:53:18,630 : INFO : PROGRESS: at sentence #61760000, processed 964504456 words, keeping 376822 word types\n",
      "2019-12-04 18:53:18,778 : INFO : PROGRESS: at sentence #61770000, processed 964617034 words, keeping 376822 word types\n",
      "2019-12-04 18:53:18,931 : INFO : PROGRESS: at sentence #61780000, processed 964731073 words, keeping 376822 word types\n",
      "2019-12-04 18:53:19,087 : INFO : PROGRESS: at sentence #61790000, processed 964843004 words, keeping 376822 word types\n",
      "2019-12-04 18:53:19,243 : INFO : PROGRESS: at sentence #61800000, processed 964957982 words, keeping 376822 word types\n",
      "2019-12-04 18:53:19,397 : INFO : PROGRESS: at sentence #61810000, processed 965072772 words, keeping 376822 word types\n",
      "2019-12-04 18:53:19,559 : INFO : PROGRESS: at sentence #61820000, processed 965195018 words, keeping 376822 word types\n",
      "2019-12-04 18:53:19,722 : INFO : PROGRESS: at sentence #61830000, processed 965315360 words, keeping 376822 word types\n",
      "2019-12-04 18:53:19,885 : INFO : PROGRESS: at sentence #61840000, processed 965438707 words, keeping 376822 word types\n",
      "2019-12-04 18:53:20,034 : INFO : PROGRESS: at sentence #61850000, processed 965553365 words, keeping 376822 word types\n",
      "2019-12-04 18:53:20,186 : INFO : PROGRESS: at sentence #61860000, processed 965669908 words, keeping 376822 word types\n",
      "2019-12-04 18:53:20,317 : INFO : PROGRESS: at sentence #61870000, processed 965769347 words, keeping 376822 word types\n",
      "2019-12-04 18:53:20,450 : INFO : PROGRESS: at sentence #61880000, processed 965870371 words, keeping 376822 word types\n",
      "2019-12-04 18:53:20,586 : INFO : PROGRESS: at sentence #61890000, processed 965972793 words, keeping 376822 word types\n",
      "2019-12-04 18:53:20,723 : INFO : PROGRESS: at sentence #61900000, processed 966075217 words, keeping 376822 word types\n",
      "2019-12-04 18:53:20,853 : INFO : PROGRESS: at sentence #61910000, processed 966172389 words, keeping 376822 word types\n",
      "2019-12-04 18:53:20,977 : INFO : PROGRESS: at sentence #61920000, processed 966267836 words, keeping 376822 word types\n",
      "2019-12-04 18:53:21,132 : INFO : PROGRESS: at sentence #61930000, processed 966365680 words, keeping 376822 word types\n",
      "2019-12-04 18:53:21,259 : INFO : PROGRESS: at sentence #61940000, processed 966462511 words, keeping 376822 word types\n",
      "2019-12-04 18:53:21,389 : INFO : PROGRESS: at sentence #61950000, processed 966563327 words, keeping 376822 word types\n",
      "2019-12-04 18:53:21,526 : INFO : PROGRESS: at sentence #61960000, processed 966669345 words, keeping 376822 word types\n",
      "2019-12-04 18:53:21,661 : INFO : PROGRESS: at sentence #61970000, processed 966774222 words, keeping 376822 word types\n",
      "2019-12-04 18:53:21,796 : INFO : PROGRESS: at sentence #61980000, processed 966878132 words, keeping 376822 word types\n",
      "2019-12-04 18:53:21,933 : INFO : PROGRESS: at sentence #61990000, processed 966983047 words, keeping 376822 word types\n",
      "2019-12-04 18:53:22,074 : INFO : PROGRESS: at sentence #62000000, processed 967088713 words, keeping 376822 word types\n",
      "2019-12-04 18:53:22,208 : INFO : PROGRESS: at sentence #62010000, processed 967188708 words, keeping 376822 word types\n",
      "2019-12-04 18:53:22,345 : INFO : PROGRESS: at sentence #62020000, processed 967291663 words, keeping 376822 word types\n",
      "2019-12-04 18:53:22,488 : INFO : PROGRESS: at sentence #62030000, processed 967399877 words, keeping 376822 word types\n",
      "2019-12-04 18:53:22,652 : INFO : PROGRESS: at sentence #62040000, processed 967527033 words, keeping 376822 word types\n",
      "2019-12-04 18:53:22,803 : INFO : PROGRESS: at sentence #62050000, processed 967643041 words, keeping 376822 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:53:22,946 : INFO : PROGRESS: at sentence #62060000, processed 967753507 words, keeping 376822 word types\n",
      "2019-12-04 18:53:23,085 : INFO : PROGRESS: at sentence #62070000, processed 967860067 words, keeping 376822 word types\n",
      "2019-12-04 18:53:23,225 : INFO : PROGRESS: at sentence #62080000, processed 967966182 words, keeping 376822 word types\n",
      "2019-12-04 18:53:23,367 : INFO : PROGRESS: at sentence #62090000, processed 968074445 words, keeping 376822 word types\n",
      "2019-12-04 18:53:23,508 : INFO : PROGRESS: at sentence #62100000, processed 968181281 words, keeping 376822 word types\n",
      "2019-12-04 18:53:23,648 : INFO : PROGRESS: at sentence #62110000, processed 968288975 words, keeping 376822 word types\n",
      "2019-12-04 18:53:23,792 : INFO : PROGRESS: at sentence #62120000, processed 968397220 words, keeping 376822 word types\n",
      "2019-12-04 18:53:23,935 : INFO : PROGRESS: at sentence #62130000, processed 968505604 words, keeping 376822 word types\n",
      "2019-12-04 18:53:24,082 : INFO : PROGRESS: at sentence #62140000, processed 968615983 words, keeping 376822 word types\n",
      "2019-12-04 18:53:24,238 : INFO : PROGRESS: at sentence #62150000, processed 968731961 words, keeping 376822 word types\n",
      "2019-12-04 18:53:24,387 : INFO : PROGRESS: at sentence #62160000, processed 968842460 words, keeping 376822 word types\n",
      "2019-12-04 18:53:24,538 : INFO : PROGRESS: at sentence #62170000, processed 968955997 words, keeping 376822 word types\n",
      "2019-12-04 18:53:24,689 : INFO : PROGRESS: at sentence #62180000, processed 969068196 words, keeping 376822 word types\n",
      "2019-12-04 18:53:24,851 : INFO : PROGRESS: at sentence #62190000, processed 969189763 words, keeping 376822 word types\n",
      "2019-12-04 18:53:25,010 : INFO : PROGRESS: at sentence #62200000, processed 969309822 words, keeping 376822 word types\n",
      "2019-12-04 18:53:25,172 : INFO : PROGRESS: at sentence #62210000, processed 969430842 words, keeping 376822 word types\n",
      "2019-12-04 18:53:25,336 : INFO : PROGRESS: at sentence #62220000, processed 969553347 words, keeping 376822 word types\n",
      "2019-12-04 18:53:25,490 : INFO : PROGRESS: at sentence #62230000, processed 969670080 words, keeping 376822 word types\n",
      "2019-12-04 18:53:25,635 : INFO : PROGRESS: at sentence #62240000, processed 969777173 words, keeping 376822 word types\n",
      "2019-12-04 18:53:25,769 : INFO : PROGRESS: at sentence #62250000, processed 969876044 words, keeping 376822 word types\n",
      "2019-12-04 18:53:25,910 : INFO : PROGRESS: at sentence #62260000, processed 969978436 words, keeping 376822 word types\n",
      "2019-12-04 18:53:26,050 : INFO : PROGRESS: at sentence #62270000, processed 970081675 words, keeping 376822 word types\n",
      "2019-12-04 18:53:26,182 : INFO : PROGRESS: at sentence #62280000, processed 970179786 words, keeping 376822 word types\n",
      "2019-12-04 18:53:26,314 : INFO : PROGRESS: at sentence #62290000, processed 970279451 words, keeping 376822 word types\n",
      "2019-12-04 18:53:26,439 : INFO : PROGRESS: at sentence #62300000, processed 970374367 words, keeping 376822 word types\n",
      "2019-12-04 18:53:26,566 : INFO : PROGRESS: at sentence #62310000, processed 970471230 words, keeping 376822 word types\n",
      "2019-12-04 18:53:26,694 : INFO : PROGRESS: at sentence #62320000, processed 970569551 words, keeping 376822 word types\n",
      "2019-12-04 18:53:26,829 : INFO : PROGRESS: at sentence #62330000, processed 970672571 words, keeping 376822 word types\n",
      "2019-12-04 18:53:26,968 : INFO : PROGRESS: at sentence #62340000, processed 970778590 words, keeping 376822 word types\n",
      "2019-12-04 18:53:27,104 : INFO : PROGRESS: at sentence #62350000, processed 970882863 words, keeping 376822 word types\n",
      "2019-12-04 18:53:27,251 : INFO : PROGRESS: at sentence #62360000, processed 970987930 words, keeping 376822 word types\n",
      "2019-12-04 18:53:27,390 : INFO : PROGRESS: at sentence #62370000, processed 971093444 words, keeping 376822 word types\n",
      "2019-12-04 18:53:27,523 : INFO : PROGRESS: at sentence #62380000, processed 971195854 words, keeping 376822 word types\n",
      "2019-12-04 18:53:27,657 : INFO : PROGRESS: at sentence #62390000, processed 971297387 words, keeping 376822 word types\n",
      "2019-12-04 18:53:27,793 : INFO : PROGRESS: at sentence #62400000, processed 971399222 words, keeping 376822 word types\n",
      "2019-12-04 18:53:27,948 : INFO : PROGRESS: at sentence #62410000, processed 971519324 words, keeping 376822 word types\n",
      "2019-12-04 18:53:28,104 : INFO : PROGRESS: at sentence #62420000, processed 971640184 words, keeping 376822 word types\n",
      "2019-12-04 18:53:28,255 : INFO : PROGRESS: at sentence #62430000, processed 971755157 words, keeping 376822 word types\n",
      "2019-12-04 18:53:28,399 : INFO : PROGRESS: at sentence #62440000, processed 971865156 words, keeping 376822 word types\n",
      "2019-12-04 18:53:28,541 : INFO : PROGRESS: at sentence #62450000, processed 971971097 words, keeping 376822 word types\n",
      "2019-12-04 18:53:28,683 : INFO : PROGRESS: at sentence #62460000, processed 972076711 words, keeping 376822 word types\n",
      "2019-12-04 18:53:28,829 : INFO : PROGRESS: at sentence #62470000, processed 972184302 words, keeping 376822 word types\n",
      "2019-12-04 18:53:29,011 : INFO : PROGRESS: at sentence #62480000, processed 972291850 words, keeping 376822 word types\n",
      "2019-12-04 18:53:29,162 : INFO : PROGRESS: at sentence #62490000, processed 972398514 words, keeping 376822 word types\n",
      "2019-12-04 18:53:29,315 : INFO : PROGRESS: at sentence #62500000, processed 972507153 words, keeping 376822 word types\n",
      "2019-12-04 18:53:29,462 : INFO : PROGRESS: at sentence #62510000, processed 972616953 words, keeping 376822 word types\n",
      "2019-12-04 18:53:29,607 : INFO : PROGRESS: at sentence #62520000, processed 972727810 words, keeping 376822 word types\n",
      "2019-12-04 18:53:29,760 : INFO : PROGRESS: at sentence #62530000, processed 972844099 words, keeping 376822 word types\n",
      "2019-12-04 18:53:29,907 : INFO : PROGRESS: at sentence #62540000, processed 972953942 words, keeping 376822 word types\n",
      "2019-12-04 18:53:30,063 : INFO : PROGRESS: at sentence #62550000, processed 973069203 words, keeping 376822 word types\n",
      "2019-12-04 18:53:30,210 : INFO : PROGRESS: at sentence #62560000, processed 973178069 words, keeping 376822 word types\n",
      "2019-12-04 18:53:30,377 : INFO : PROGRESS: at sentence #62570000, processed 973304919 words, keeping 376822 word types\n",
      "2019-12-04 18:53:30,537 : INFO : PROGRESS: at sentence #62580000, processed 973425729 words, keeping 376822 word types\n",
      "2019-12-04 18:53:30,697 : INFO : PROGRESS: at sentence #62590000, processed 973547560 words, keeping 376822 word types\n",
      "2019-12-04 18:53:30,851 : INFO : PROGRESS: at sentence #62600000, processed 973665650 words, keeping 376822 word types\n",
      "2019-12-04 18:53:31,008 : INFO : PROGRESS: at sentence #62610000, processed 973784578 words, keeping 376822 word types\n",
      "2019-12-04 18:53:31,144 : INFO : PROGRESS: at sentence #62620000, processed 973883965 words, keeping 376822 word types\n",
      "2019-12-04 18:53:31,279 : INFO : PROGRESS: at sentence #62630000, processed 973984275 words, keeping 376822 word types\n",
      "2019-12-04 18:53:31,415 : INFO : PROGRESS: at sentence #62640000, processed 974086046 words, keeping 376822 word types\n",
      "2019-12-04 18:53:31,557 : INFO : PROGRESS: at sentence #62650000, processed 974191177 words, keeping 376822 word types\n",
      "2019-12-04 18:53:31,684 : INFO : PROGRESS: at sentence #62660000, processed 974286818 words, keeping 376822 word types\n",
      "2019-12-04 18:53:31,810 : INFO : PROGRESS: at sentence #62670000, processed 974384086 words, keeping 376822 word types\n",
      "2019-12-04 18:53:31,938 : INFO : PROGRESS: at sentence #62680000, processed 974481213 words, keeping 376822 word types\n",
      "2019-12-04 18:53:32,065 : INFO : PROGRESS: at sentence #62690000, processed 974577365 words, keeping 376822 word types\n",
      "2019-12-04 18:53:32,196 : INFO : PROGRESS: at sentence #62700000, processed 974676641 words, keeping 376822 word types\n",
      "2019-12-04 18:53:32,333 : INFO : PROGRESS: at sentence #62710000, processed 974782182 words, keeping 376822 word types\n",
      "2019-12-04 18:53:32,470 : INFO : PROGRESS: at sentence #62720000, processed 974887484 words, keeping 376822 word types\n",
      "2019-12-04 18:53:32,605 : INFO : PROGRESS: at sentence #62730000, processed 974991454 words, keeping 376822 word types\n",
      "2019-12-04 18:53:32,740 : INFO : PROGRESS: at sentence #62740000, processed 975096893 words, keeping 376822 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:53:32,880 : INFO : PROGRESS: at sentence #62750000, processed 975202764 words, keeping 376822 word types\n",
      "2019-12-04 18:53:33,019 : INFO : PROGRESS: at sentence #62760000, processed 975302905 words, keeping 376822 word types\n",
      "2019-12-04 18:53:33,154 : INFO : PROGRESS: at sentence #62770000, processed 975405609 words, keeping 376822 word types\n",
      "2019-12-04 18:53:33,292 : INFO : PROGRESS: at sentence #62780000, processed 975508221 words, keeping 376822 word types\n",
      "2019-12-04 18:53:33,455 : INFO : PROGRESS: at sentence #62790000, processed 975638309 words, keeping 376822 word types\n",
      "2019-12-04 18:53:33,601 : INFO : PROGRESS: at sentence #62800000, processed 975753353 words, keeping 376822 word types\n",
      "2019-12-04 18:53:33,742 : INFO : PROGRESS: at sentence #62810000, processed 975865943 words, keeping 376822 word types\n",
      "2019-12-04 18:53:33,884 : INFO : PROGRESS: at sentence #62820000, processed 975973833 words, keeping 376822 word types\n",
      "2019-12-04 18:53:34,025 : INFO : PROGRESS: at sentence #62830000, processed 976080089 words, keeping 376822 word types\n",
      "2019-12-04 18:53:34,163 : INFO : PROGRESS: at sentence #62840000, processed 976187409 words, keeping 376822 word types\n",
      "2019-12-04 18:53:34,304 : INFO : PROGRESS: at sentence #62850000, processed 976293898 words, keeping 376822 word types\n",
      "2019-12-04 18:53:34,453 : INFO : PROGRESS: at sentence #62860000, processed 976401611 words, keeping 376822 word types\n",
      "2019-12-04 18:53:34,604 : INFO : PROGRESS: at sentence #62870000, processed 976508751 words, keeping 376822 word types\n",
      "2019-12-04 18:53:34,748 : INFO : PROGRESS: at sentence #62880000, processed 976617399 words, keeping 376822 word types\n",
      "2019-12-04 18:53:34,895 : INFO : PROGRESS: at sentence #62890000, processed 976728050 words, keeping 376822 word types\n",
      "2019-12-04 18:53:35,049 : INFO : PROGRESS: at sentence #62900000, processed 976841926 words, keeping 376822 word types\n",
      "2019-12-04 18:53:35,202 : INFO : PROGRESS: at sentence #62910000, processed 976954803 words, keeping 376822 word types\n",
      "2019-12-04 18:53:35,352 : INFO : PROGRESS: at sentence #62920000, processed 977067099 words, keeping 376822 word types\n",
      "2019-12-04 18:53:35,505 : INFO : PROGRESS: at sentence #62930000, processed 977181091 words, keeping 376822 word types\n",
      "2019-12-04 18:53:35,663 : INFO : PROGRESS: at sentence #62940000, processed 977299038 words, keeping 376822 word types\n",
      "2019-12-04 18:53:35,821 : INFO : PROGRESS: at sentence #62950000, processed 977419522 words, keeping 376822 word types\n",
      "2019-12-04 18:53:35,978 : INFO : PROGRESS: at sentence #62960000, processed 977540371 words, keeping 376822 word types\n",
      "2019-12-04 18:53:36,142 : INFO : PROGRESS: at sentence #62970000, processed 977663453 words, keeping 376822 word types\n",
      "2019-12-04 18:53:36,296 : INFO : PROGRESS: at sentence #62980000, processed 977780729 words, keeping 376822 word types\n",
      "2019-12-04 18:53:36,440 : INFO : PROGRESS: at sentence #62990000, processed 977891474 words, keeping 376822 word types\n",
      "2019-12-04 18:53:36,573 : INFO : PROGRESS: at sentence #63000000, processed 977991414 words, keeping 376822 word types\n",
      "2019-12-04 18:53:36,704 : INFO : PROGRESS: at sentence #63010000, processed 978092538 words, keeping 376822 word types\n",
      "2019-12-04 18:53:36,837 : INFO : PROGRESS: at sentence #63020000, processed 978195234 words, keeping 376822 word types\n",
      "2019-12-04 18:53:36,964 : INFO : PROGRESS: at sentence #63030000, processed 978292462 words, keeping 376822 word types\n",
      "2019-12-04 18:53:37,098 : INFO : PROGRESS: at sentence #63040000, processed 978395163 words, keeping 376822 word types\n",
      "2019-12-04 18:53:37,219 : INFO : PROGRESS: at sentence #63050000, processed 978488960 words, keeping 376822 word types\n",
      "2019-12-04 18:53:37,346 : INFO : PROGRESS: at sentence #63060000, processed 978586649 words, keeping 376822 word types\n",
      "2019-12-04 18:53:37,473 : INFO : PROGRESS: at sentence #63070000, processed 978684172 words, keeping 376822 word types\n",
      "2019-12-04 18:53:37,604 : INFO : PROGRESS: at sentence #63080000, processed 978786094 words, keeping 376822 word types\n",
      "2019-12-04 18:53:37,741 : INFO : PROGRESS: at sentence #63090000, processed 978892022 words, keeping 376822 word types\n",
      "2019-12-04 18:53:37,874 : INFO : PROGRESS: at sentence #63100000, processed 978996382 words, keeping 376822 word types\n",
      "2019-12-04 18:53:38,006 : INFO : PROGRESS: at sentence #63110000, processed 979100783 words, keeping 376822 word types\n",
      "2019-12-04 18:53:38,145 : INFO : PROGRESS: at sentence #63120000, processed 979206272 words, keeping 376822 word types\n",
      "2019-12-04 18:53:38,280 : INFO : PROGRESS: at sentence #63130000, processed 979310496 words, keeping 376822 word types\n",
      "2019-12-04 18:53:38,414 : INFO : PROGRESS: at sentence #63140000, processed 979410467 words, keeping 376822 word types\n",
      "2019-12-04 18:53:38,553 : INFO : PROGRESS: at sentence #63150000, processed 979513583 words, keeping 376822 word types\n",
      "2019-12-04 18:53:38,703 : INFO : PROGRESS: at sentence #63160000, processed 979626688 words, keeping 376822 word types\n",
      "2019-12-04 18:53:38,867 : INFO : PROGRESS: at sentence #63170000, processed 979751326 words, keeping 376822 word types\n",
      "2019-12-04 18:53:39,018 : INFO : PROGRESS: at sentence #63180000, processed 979867117 words, keeping 376822 word types\n",
      "2019-12-04 18:53:39,170 : INFO : PROGRESS: at sentence #63190000, processed 979979024 words, keeping 376822 word types\n",
      "2019-12-04 18:53:39,308 : INFO : PROGRESS: at sentence #63200000, processed 980082423 words, keeping 376822 word types\n",
      "2019-12-04 18:53:39,450 : INFO : PROGRESS: at sentence #63210000, processed 980188893 words, keeping 376822 word types\n",
      "2019-12-04 18:53:39,592 : INFO : PROGRESS: at sentence #63220000, processed 980297011 words, keeping 376822 word types\n",
      "2019-12-04 18:53:39,734 : INFO : PROGRESS: at sentence #63230000, processed 980404560 words, keeping 376822 word types\n",
      "2019-12-04 18:53:39,875 : INFO : PROGRESS: at sentence #63240000, processed 980511876 words, keeping 376822 word types\n",
      "2019-12-04 18:53:40,019 : INFO : PROGRESS: at sentence #63250000, processed 980620076 words, keeping 376822 word types\n",
      "2019-12-04 18:53:40,162 : INFO : PROGRESS: at sentence #63260000, processed 980729241 words, keeping 376822 word types\n",
      "2019-12-04 18:53:40,306 : INFO : PROGRESS: at sentence #63270000, processed 980839415 words, keeping 376822 word types\n",
      "2019-12-04 18:53:40,459 : INFO : PROGRESS: at sentence #63280000, processed 980955786 words, keeping 376822 word types\n",
      "2019-12-04 18:53:40,604 : INFO : PROGRESS: at sentence #63290000, processed 981065802 words, keeping 376822 word types\n",
      "2019-12-04 18:53:40,755 : INFO : PROGRESS: at sentence #63300000, processed 981179919 words, keeping 376822 word types\n",
      "2019-12-04 18:53:40,903 : INFO : PROGRESS: at sentence #63310000, processed 981290776 words, keeping 376822 word types\n",
      "2019-12-04 18:53:41,065 : INFO : PROGRESS: at sentence #63320000, processed 981414487 words, keeping 376822 word types\n",
      "2019-12-04 18:53:41,225 : INFO : PROGRESS: at sentence #63330000, processed 981534777 words, keeping 376822 word types\n",
      "2019-12-04 18:53:41,386 : INFO : PROGRESS: at sentence #63340000, processed 981656462 words, keeping 376822 word types\n",
      "2019-12-04 18:53:41,545 : INFO : PROGRESS: at sentence #63350000, processed 981777112 words, keeping 376822 word types\n",
      "2019-12-04 18:53:41,703 : INFO : PROGRESS: at sentence #63360000, processed 981894868 words, keeping 376822 word types\n",
      "2019-12-04 18:53:41,845 : INFO : PROGRESS: at sentence #63370000, processed 981998789 words, keeping 376822 word types\n",
      "2019-12-04 18:53:41,976 : INFO : PROGRESS: at sentence #63380000, processed 982098183 words, keeping 376822 word types\n",
      "2019-12-04 18:53:42,110 : INFO : PROGRESS: at sentence #63390000, processed 982200263 words, keeping 376822 word types\n",
      "2019-12-04 18:53:42,244 : INFO : PROGRESS: at sentence #63400000, processed 982304548 words, keeping 376822 word types\n",
      "2019-12-04 18:53:42,369 : INFO : PROGRESS: at sentence #63410000, processed 982400160 words, keeping 376822 word types\n",
      "2019-12-04 18:53:42,497 : INFO : PROGRESS: at sentence #63420000, processed 982500339 words, keeping 376822 word types\n",
      "2019-12-04 18:53:42,621 : INFO : PROGRESS: at sentence #63430000, processed 982595756 words, keeping 376822 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:53:42,747 : INFO : PROGRESS: at sentence #63440000, processed 982692623 words, keeping 376822 word types\n",
      "2019-12-04 18:53:42,876 : INFO : PROGRESS: at sentence #63450000, processed 982791568 words, keeping 376822 word types\n",
      "2019-12-04 18:53:43,012 : INFO : PROGRESS: at sentence #63460000, processed 982896085 words, keeping 376822 word types\n",
      "2019-12-04 18:53:43,148 : INFO : PROGRESS: at sentence #63470000, processed 983000766 words, keeping 376822 word types\n",
      "2019-12-04 18:53:43,283 : INFO : PROGRESS: at sentence #63480000, processed 983105651 words, keeping 376822 word types\n",
      "2019-12-04 18:53:43,416 : INFO : PROGRESS: at sentence #63490000, processed 983210132 words, keeping 376822 word types\n",
      "2019-12-04 18:53:43,555 : INFO : PROGRESS: at sentence #63500000, processed 983316523 words, keeping 376822 word types\n",
      "2019-12-04 18:53:43,690 : INFO : PROGRESS: at sentence #63510000, processed 983417848 words, keeping 376822 word types\n",
      "2019-12-04 18:53:43,826 : INFO : PROGRESS: at sentence #63520000, processed 983519394 words, keeping 376822 word types\n",
      "2019-12-04 18:53:43,959 : INFO : PROGRESS: at sentence #63530000, processed 983621686 words, keeping 376822 word types\n",
      "2019-12-04 18:53:44,120 : INFO : PROGRESS: at sentence #63540000, processed 983746018 words, keeping 376822 word types\n",
      "2019-12-04 18:53:44,275 : INFO : PROGRESS: at sentence #63550000, processed 983864583 words, keeping 376822 word types\n",
      "2019-12-04 18:53:44,460 : INFO : PROGRESS: at sentence #63560000, processed 983978055 words, keeping 376822 word types\n",
      "2019-12-04 18:53:44,604 : INFO : PROGRESS: at sentence #63570000, processed 984087514 words, keeping 376822 word types\n",
      "2019-12-04 18:53:44,743 : INFO : PROGRESS: at sentence #63580000, processed 984193659 words, keeping 376822 word types\n",
      "2019-12-04 18:53:44,883 : INFO : PROGRESS: at sentence #63590000, processed 984300391 words, keeping 376822 word types\n",
      "2019-12-04 18:53:45,025 : INFO : PROGRESS: at sentence #63600000, processed 984406894 words, keeping 376822 word types\n",
      "2019-12-04 18:53:45,169 : INFO : PROGRESS: at sentence #63610000, processed 984514366 words, keeping 376822 word types\n",
      "2019-12-04 18:53:45,312 : INFO : PROGRESS: at sentence #63620000, processed 984621585 words, keeping 376822 word types\n",
      "2019-12-04 18:53:45,455 : INFO : PROGRESS: at sentence #63630000, processed 984730338 words, keeping 376822 word types\n",
      "2019-12-04 18:53:45,602 : INFO : PROGRESS: at sentence #63640000, processed 984840219 words, keeping 376822 word types\n",
      "2019-12-04 18:53:45,748 : INFO : PROGRESS: at sentence #63650000, processed 984951869 words, keeping 376822 word types\n",
      "2019-12-04 18:53:45,900 : INFO : PROGRESS: at sentence #63660000, processed 985066406 words, keeping 376822 word types\n",
      "2019-12-04 18:53:46,048 : INFO : PROGRESS: at sentence #63670000, processed 985178373 words, keeping 376822 word types\n",
      "2019-12-04 18:53:46,201 : INFO : PROGRESS: at sentence #63680000, processed 985292920 words, keeping 376822 word types\n",
      "2019-12-04 18:53:46,351 : INFO : PROGRESS: at sentence #63690000, processed 985405094 words, keeping 376822 word types\n",
      "2019-12-04 18:53:46,516 : INFO : PROGRESS: at sentence #63700000, processed 985529683 words, keeping 376822 word types\n",
      "2019-12-04 18:53:46,674 : INFO : PROGRESS: at sentence #63710000, processed 985650210 words, keeping 376822 word types\n",
      "2019-12-04 18:53:46,835 : INFO : PROGRESS: at sentence #63720000, processed 985772953 words, keeping 376822 word types\n",
      "2019-12-04 18:53:46,986 : INFO : PROGRESS: at sentence #63730000, processed 985889102 words, keeping 376822 word types\n",
      "2019-12-04 18:53:47,139 : INFO : PROGRESS: at sentence #63740000, processed 986006625 words, keeping 376822 word types\n",
      "2019-12-04 18:53:47,274 : INFO : PROGRESS: at sentence #63750000, processed 986105744 words, keeping 376822 word types\n",
      "2019-12-04 18:53:47,409 : INFO : PROGRESS: at sentence #63760000, processed 986206641 words, keeping 376822 word types\n",
      "2019-12-04 18:53:47,537 : INFO : PROGRESS: at sentence #63770000, processed 986308354 words, keeping 376822 word types\n",
      "2019-12-04 18:53:47,670 : INFO : PROGRESS: at sentence #63780000, processed 986413551 words, keeping 376822 word types\n",
      "2019-12-04 18:53:47,794 : INFO : PROGRESS: at sentence #63790000, processed 986508387 words, keeping 376822 word types\n",
      "2019-12-04 18:53:47,921 : INFO : PROGRESS: at sentence #63800000, processed 986604875 words, keeping 376822 word types\n",
      "2019-12-04 18:53:48,048 : INFO : PROGRESS: at sentence #63810000, processed 986702436 words, keeping 376822 word types\n",
      "2019-12-04 18:53:48,176 : INFO : PROGRESS: at sentence #63820000, processed 986798821 words, keeping 376822 word types\n",
      "2019-12-04 18:53:48,305 : INFO : PROGRESS: at sentence #63830000, processed 986898985 words, keeping 376822 word types\n",
      "2019-12-04 18:53:48,442 : INFO : PROGRESS: at sentence #63840000, processed 987005130 words, keeping 376822 word types\n",
      "2019-12-04 18:53:48,577 : INFO : PROGRESS: at sentence #63850000, processed 987110143 words, keeping 376822 word types\n",
      "2019-12-04 18:53:48,713 : INFO : PROGRESS: at sentence #63860000, processed 987213928 words, keeping 376822 word types\n",
      "2019-12-04 18:53:48,847 : INFO : PROGRESS: at sentence #63870000, processed 987319012 words, keeping 376822 word types\n",
      "2019-12-04 18:53:48,983 : INFO : PROGRESS: at sentence #63880000, processed 987424933 words, keeping 376822 word types\n",
      "2019-12-04 18:53:49,115 : INFO : PROGRESS: at sentence #63890000, processed 987524917 words, keeping 376822 word types\n",
      "2019-12-04 18:53:49,251 : INFO : PROGRESS: at sentence #63900000, processed 987627953 words, keeping 376822 word types\n",
      "2019-12-04 18:53:49,389 : INFO : PROGRESS: at sentence #63910000, processed 987733368 words, keeping 376822 word types\n",
      "2019-12-04 18:53:49,553 : INFO : PROGRESS: at sentence #63920000, processed 987862215 words, keeping 376822 word types\n",
      "2019-12-04 18:53:49,703 : INFO : PROGRESS: at sentence #63930000, processed 987977367 words, keeping 376822 word types\n",
      "2019-12-04 18:53:49,892 : INFO : PROGRESS: at sentence #63940000, processed 988089231 words, keeping 376822 word types\n",
      "2019-12-04 18:53:50,036 : INFO : PROGRESS: at sentence #63950000, processed 988196279 words, keeping 376822 word types\n",
      "2019-12-04 18:53:50,177 : INFO : PROGRESS: at sentence #63960000, processed 988302476 words, keeping 376822 word types\n",
      "2019-12-04 18:53:50,322 : INFO : PROGRESS: at sentence #63970000, processed 988410413 words, keeping 376822 word types\n",
      "2019-12-04 18:53:50,465 : INFO : PROGRESS: at sentence #63980000, processed 988516818 words, keeping 376822 word types\n",
      "2019-12-04 18:53:50,607 : INFO : PROGRESS: at sentence #63990000, processed 988624563 words, keeping 376822 word types\n",
      "2019-12-04 18:53:50,751 : INFO : PROGRESS: at sentence #64000000, processed 988732254 words, keeping 376822 word types\n",
      "2019-12-04 18:53:50,897 : INFO : PROGRESS: at sentence #64010000, processed 988841063 words, keeping 376822 word types\n",
      "2019-12-04 18:53:51,043 : INFO : PROGRESS: at sentence #64020000, processed 988951336 words, keeping 376822 word types\n",
      "2019-12-04 18:53:51,194 : INFO : PROGRESS: at sentence #64030000, processed 989066855 words, keeping 376822 word types\n",
      "2019-12-04 18:53:51,343 : INFO : PROGRESS: at sentence #64040000, processed 989178399 words, keeping 376822 word types\n",
      "2019-12-04 18:53:51,495 : INFO : PROGRESS: at sentence #64050000, processed 989290974 words, keeping 376822 word types\n",
      "2019-12-04 18:53:51,688 : INFO : PROGRESS: at sentence #64060000, processed 989403991 words, keeping 376822 word types\n",
      "2019-12-04 18:53:51,845 : INFO : PROGRESS: at sentence #64070000, processed 989524353 words, keeping 376822 word types\n",
      "2019-12-04 18:53:52,002 : INFO : PROGRESS: at sentence #64080000, processed 989644540 words, keeping 376822 word types\n",
      "2019-12-04 18:53:52,164 : INFO : PROGRESS: at sentence #64090000, processed 989765510 words, keeping 376822 word types\n",
      "2019-12-04 18:53:52,321 : INFO : PROGRESS: at sentence #64100000, processed 989888589 words, keeping 376822 word types\n",
      "2019-12-04 18:53:52,473 : INFO : PROGRESS: at sentence #64110000, processed 990005611 words, keeping 376822 word types\n",
      "2019-12-04 18:53:52,618 : INFO : PROGRESS: at sentence #64120000, processed 990113201 words, keeping 376822 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:53:52,751 : INFO : PROGRESS: at sentence #64130000, processed 990212905 words, keeping 376822 word types\n",
      "2019-12-04 18:53:52,887 : INFO : PROGRESS: at sentence #64140000, processed 990314520 words, keeping 376822 word types\n",
      "2019-12-04 18:53:53,022 : INFO : PROGRESS: at sentence #64150000, processed 990417536 words, keeping 376822 word types\n",
      "2019-12-04 18:53:53,156 : INFO : PROGRESS: at sentence #64160000, processed 990515242 words, keeping 376822 word types\n",
      "2019-12-04 18:53:53,296 : INFO : PROGRESS: at sentence #64170000, processed 990616440 words, keeping 376822 word types\n",
      "2019-12-04 18:53:53,419 : INFO : PROGRESS: at sentence #64180000, processed 990710983 words, keeping 376822 word types\n",
      "2019-12-04 18:53:53,548 : INFO : PROGRESS: at sentence #64190000, processed 990808134 words, keeping 376822 word types\n",
      "2019-12-04 18:53:53,686 : INFO : PROGRESS: at sentence #64200000, processed 990906279 words, keeping 376822 word types\n",
      "2019-12-04 18:53:53,818 : INFO : PROGRESS: at sentence #64210000, processed 991008783 words, keeping 376822 word types\n",
      "2019-12-04 18:53:53,950 : INFO : PROGRESS: at sentence #64220000, processed 991114283 words, keeping 376822 word types\n",
      "2019-12-04 18:53:54,126 : INFO : PROGRESS: at sentence #64230000, processed 991219027 words, keeping 376822 word types\n",
      "2019-12-04 18:53:54,287 : INFO : PROGRESS: at sentence #64240000, processed 991323594 words, keeping 376822 word types\n",
      "2019-12-04 18:53:54,423 : INFO : PROGRESS: at sentence #64250000, processed 991428907 words, keeping 376822 word types\n",
      "2019-12-04 18:53:54,548 : INFO : PROGRESS: at sentence #64260000, processed 991532361 words, keeping 376822 word types\n",
      "2019-12-04 18:53:54,672 : INFO : PROGRESS: at sentence #64270000, processed 991633225 words, keeping 376822 word types\n",
      "2019-12-04 18:53:54,807 : INFO : PROGRESS: at sentence #64280000, processed 991735649 words, keeping 376822 word types\n",
      "2019-12-04 18:53:54,956 : INFO : PROGRESS: at sentence #64290000, processed 991853492 words, keeping 376822 word types\n",
      "2019-12-04 18:53:55,106 : INFO : PROGRESS: at sentence #64300000, processed 991975340 words, keeping 376822 word types\n",
      "2019-12-04 18:53:55,252 : INFO : PROGRESS: at sentence #64310000, processed 992090566 words, keeping 376822 word types\n",
      "2019-12-04 18:53:55,391 : INFO : PROGRESS: at sentence #64320000, processed 992201087 words, keeping 376822 word types\n",
      "2019-12-04 18:53:55,528 : INFO : PROGRESS: at sentence #64330000, processed 992306460 words, keeping 376822 word types\n",
      "2019-12-04 18:53:55,663 : INFO : PROGRESS: at sentence #64340000, processed 992411589 words, keeping 376822 word types\n",
      "2019-12-04 18:53:55,805 : INFO : PROGRESS: at sentence #64350000, processed 992520121 words, keeping 376822 word types\n",
      "2019-12-04 18:53:55,943 : INFO : PROGRESS: at sentence #64360000, processed 992627815 words, keeping 376822 word types\n",
      "2019-12-04 18:53:56,084 : INFO : PROGRESS: at sentence #64370000, processed 992734581 words, keeping 376822 word types\n",
      "2019-12-04 18:53:56,225 : INFO : PROGRESS: at sentence #64380000, processed 992842579 words, keeping 376822 word types\n",
      "2019-12-04 18:53:56,367 : INFO : PROGRESS: at sentence #64390000, processed 992952427 words, keeping 376822 word types\n",
      "2019-12-04 18:53:56,511 : INFO : PROGRESS: at sentence #64400000, processed 993062939 words, keeping 376822 word types\n",
      "2019-12-04 18:53:56,665 : INFO : PROGRESS: at sentence #64410000, processed 993179362 words, keeping 376822 word types\n",
      "2019-12-04 18:53:56,811 : INFO : PROGRESS: at sentence #64420000, processed 993289288 words, keeping 376822 word types\n",
      "2019-12-04 18:53:56,966 : INFO : PROGRESS: at sentence #64430000, processed 993403888 words, keeping 376822 word types\n",
      "2019-12-04 18:53:57,117 : INFO : PROGRESS: at sentence #64440000, processed 993513883 words, keeping 376822 word types\n",
      "2019-12-04 18:53:57,284 : INFO : PROGRESS: at sentence #64450000, processed 993639489 words, keeping 376822 word types\n",
      "2019-12-04 18:53:57,445 : INFO : PROGRESS: at sentence #64460000, processed 993759830 words, keeping 376822 word types\n",
      "2019-12-04 18:53:57,604 : INFO : PROGRESS: at sentence #64470000, processed 993881890 words, keeping 376822 word types\n",
      "2019-12-04 18:53:57,758 : INFO : PROGRESS: at sentence #64480000, processed 994000602 words, keeping 376822 word types\n",
      "2019-12-04 18:53:57,914 : INFO : PROGRESS: at sentence #64490000, processed 994119778 words, keeping 376822 word types\n",
      "2019-12-04 18:53:58,047 : INFO : PROGRESS: at sentence #64500000, processed 994220462 words, keeping 376822 word types\n",
      "2019-12-04 18:53:58,179 : INFO : PROGRESS: at sentence #64510000, processed 994320398 words, keeping 376822 word types\n",
      "2019-12-04 18:53:58,314 : INFO : PROGRESS: at sentence #64520000, processed 994422252 words, keeping 376822 word types\n",
      "2019-12-04 18:53:58,450 : INFO : PROGRESS: at sentence #64530000, processed 994527331 words, keeping 376822 word types\n",
      "2019-12-04 18:53:58,574 : INFO : PROGRESS: at sentence #64540000, processed 994623076 words, keeping 376822 word types\n",
      "2019-12-04 18:53:58,701 : INFO : PROGRESS: at sentence #64550000, processed 994721117 words, keeping 376822 word types\n",
      "2019-12-04 18:53:58,829 : INFO : PROGRESS: at sentence #64560000, processed 994817651 words, keeping 376822 word types\n",
      "2019-12-04 18:53:58,955 : INFO : PROGRESS: at sentence #64570000, processed 994913719 words, keeping 376822 word types\n",
      "2019-12-04 18:53:59,084 : INFO : PROGRESS: at sentence #64580000, processed 995013174 words, keeping 376822 word types\n",
      "2019-12-04 18:53:59,217 : INFO : PROGRESS: at sentence #64590000, processed 995118415 words, keeping 376822 word types\n",
      "2019-12-04 18:53:59,353 : INFO : PROGRESS: at sentence #64600000, processed 995223507 words, keeping 376822 word types\n",
      "2019-12-04 18:53:59,490 : INFO : PROGRESS: at sentence #64610000, processed 995328039 words, keeping 376822 word types\n",
      "2019-12-04 18:53:59,626 : INFO : PROGRESS: at sentence #64620000, processed 995432848 words, keeping 376822 word types\n",
      "2019-12-04 18:53:59,764 : INFO : PROGRESS: at sentence #64630000, processed 995538951 words, keeping 376822 word types\n",
      "2019-12-04 18:53:59,897 : INFO : PROGRESS: at sentence #64640000, processed 995639470 words, keeping 376822 word types\n",
      "2019-12-04 18:54:00,034 : INFO : PROGRESS: at sentence #64650000, processed 995741860 words, keeping 376822 word types\n",
      "2019-12-04 18:54:00,170 : INFO : PROGRESS: at sentence #64660000, processed 995844252 words, keeping 376822 word types\n",
      "2019-12-04 18:54:00,329 : INFO : PROGRESS: at sentence #64670000, processed 995971790 words, keeping 376822 word types\n",
      "2019-12-04 18:54:00,479 : INFO : PROGRESS: at sentence #64680000, processed 996088790 words, keeping 376822 word types\n",
      "2019-12-04 18:54:00,625 : INFO : PROGRESS: at sentence #64690000, processed 996201237 words, keeping 376822 word types\n",
      "2019-12-04 18:54:00,766 : INFO : PROGRESS: at sentence #64700000, processed 996310066 words, keeping 376822 word types\n",
      "2019-12-04 18:54:00,903 : INFO : PROGRESS: at sentence #64710000, processed 996415866 words, keeping 376822 word types\n",
      "2019-12-04 18:54:01,043 : INFO : PROGRESS: at sentence #64720000, processed 996523220 words, keeping 376822 word types\n",
      "2019-12-04 18:54:01,183 : INFO : PROGRESS: at sentence #64730000, processed 996629838 words, keeping 376822 word types\n",
      "2019-12-04 18:54:01,326 : INFO : PROGRESS: at sentence #64740000, processed 996737297 words, keeping 376822 word types\n",
      "2019-12-04 18:54:01,464 : INFO : PROGRESS: at sentence #64750000, processed 996844351 words, keeping 376822 word types\n",
      "2019-12-04 18:54:01,602 : INFO : PROGRESS: at sentence #64760000, processed 996952818 words, keeping 376822 word types\n",
      "2019-12-04 18:54:01,743 : INFO : PROGRESS: at sentence #64770000, processed 997063479 words, keeping 376822 word types\n",
      "2019-12-04 18:54:01,886 : INFO : PROGRESS: at sentence #64780000, processed 997176492 words, keeping 376822 word types\n",
      "2019-12-04 18:54:02,033 : INFO : PROGRESS: at sentence #64790000, processed 997289855 words, keeping 376822 word types\n",
      "2019-12-04 18:54:02,181 : INFO : PROGRESS: at sentence #64800000, processed 997402329 words, keeping 376822 word types\n",
      "2019-12-04 18:54:02,330 : INFO : PROGRESS: at sentence #64810000, processed 997516898 words, keeping 376822 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:54:02,479 : INFO : PROGRESS: at sentence #64820000, processed 997633013 words, keeping 376822 word types\n",
      "2019-12-04 18:54:02,638 : INFO : PROGRESS: at sentence #64830000, processed 997754378 words, keeping 376822 word types\n",
      "2019-12-04 18:54:02,795 : INFO : PROGRESS: at sentence #64840000, processed 997874926 words, keeping 376822 word types\n",
      "2019-12-04 18:54:02,957 : INFO : PROGRESS: at sentence #64850000, processed 997998375 words, keeping 376822 word types\n",
      "2019-12-04 18:54:03,108 : INFO : PROGRESS: at sentence #64860000, processed 998114310 words, keeping 376822 word types\n",
      "2019-12-04 18:54:03,261 : INFO : PROGRESS: at sentence #64870000, processed 998228184 words, keeping 376822 word types\n",
      "2019-12-04 18:54:03,393 : INFO : PROGRESS: at sentence #64880000, processed 998327448 words, keeping 376822 word types\n",
      "2019-12-04 18:54:03,527 : INFO : PROGRESS: at sentence #64890000, processed 998428801 words, keeping 376822 word types\n",
      "2019-12-04 18:54:03,661 : INFO : PROGRESS: at sentence #64900000, processed 998531268 words, keeping 376822 word types\n",
      "2019-12-04 18:54:03,792 : INFO : PROGRESS: at sentence #64910000, processed 998630398 words, keeping 376822 word types\n",
      "2019-12-04 18:54:03,921 : INFO : PROGRESS: at sentence #64920000, processed 998730841 words, keeping 376822 word types\n",
      "2019-12-04 18:54:04,043 : INFO : PROGRESS: at sentence #64930000, processed 998825503 words, keeping 376822 word types\n",
      "2019-12-04 18:54:04,170 : INFO : PROGRESS: at sentence #64940000, processed 998923479 words, keeping 376822 word types\n",
      "2019-12-04 18:54:04,294 : INFO : PROGRESS: at sentence #64950000, processed 999020520 words, keeping 376822 word types\n",
      "2019-12-04 18:54:04,426 : INFO : PROGRESS: at sentence #64960000, processed 999121780 words, keeping 376822 word types\n",
      "2019-12-04 18:54:04,562 : INFO : PROGRESS: at sentence #64970000, processed 999228070 words, keeping 376822 word types\n",
      "2019-12-04 18:54:04,695 : INFO : PROGRESS: at sentence #64980000, processed 999332508 words, keeping 376822 word types\n",
      "2019-12-04 18:54:04,830 : INFO : PROGRESS: at sentence #64990000, processed 999436777 words, keeping 376822 word types\n",
      "2019-12-04 18:54:04,967 : INFO : PROGRESS: at sentence #65000000, processed 999541835 words, keeping 376822 word types\n",
      "2019-12-04 18:54:05,103 : INFO : PROGRESS: at sentence #65010000, processed 999646973 words, keeping 376822 word types\n",
      "2019-12-04 18:54:05,238 : INFO : PROGRESS: at sentence #65020000, processed 999747024 words, keeping 376822 word types\n",
      "2019-12-04 18:54:05,374 : INFO : PROGRESS: at sentence #65030000, processed 999850038 words, keeping 376822 word types\n",
      "2019-12-04 18:54:05,517 : INFO : PROGRESS: at sentence #65040000, processed 999960103 words, keeping 376822 word types\n",
      "2019-12-04 18:54:05,678 : INFO : PROGRESS: at sentence #65050000, processed 1000086449 words, keeping 376822 word types\n",
      "2019-12-04 18:54:05,827 : INFO : PROGRESS: at sentence #65060000, processed 1000202371 words, keeping 376822 word types\n",
      "2019-12-04 18:54:05,972 : INFO : PROGRESS: at sentence #65070000, processed 1000314396 words, keeping 376822 word types\n",
      "2019-12-04 18:54:06,110 : INFO : PROGRESS: at sentence #65080000, processed 1000418588 words, keeping 376822 word types\n",
      "2019-12-04 18:54:06,247 : INFO : PROGRESS: at sentence #65090000, processed 1000524767 words, keeping 376822 word types\n",
      "2019-12-04 18:54:06,388 : INFO : PROGRESS: at sentence #65100000, processed 1000632984 words, keeping 376822 word types\n",
      "2019-12-04 18:54:06,527 : INFO : PROGRESS: at sentence #65110000, processed 1000740307 words, keeping 376822 word types\n",
      "2019-12-04 18:54:06,669 : INFO : PROGRESS: at sentence #65120000, processed 1000847516 words, keeping 376822 word types\n",
      "2019-12-04 18:54:06,810 : INFO : PROGRESS: at sentence #65130000, processed 1000955901 words, keeping 376822 word types\n",
      "2019-12-04 18:54:06,951 : INFO : PROGRESS: at sentence #65140000, processed 1001064550 words, keeping 376822 word types\n",
      "2019-12-04 18:54:07,094 : INFO : PROGRESS: at sentence #65150000, processed 1001174877 words, keeping 376822 word types\n",
      "2019-12-04 18:54:07,246 : INFO : PROGRESS: at sentence #65160000, processed 1001291154 words, keeping 376822 word types\n",
      "2019-12-04 18:54:07,395 : INFO : PROGRESS: at sentence #65170000, processed 1001401374 words, keeping 376822 word types\n",
      "2019-12-04 18:54:07,544 : INFO : PROGRESS: at sentence #65180000, processed 1001515625 words, keeping 376822 word types\n",
      "2019-12-04 18:54:07,690 : INFO : PROGRESS: at sentence #65190000, processed 1001626836 words, keeping 376822 word types\n",
      "2019-12-04 18:54:07,852 : INFO : PROGRESS: at sentence #65200000, processed 1001749085 words, keeping 376822 word types\n",
      "2019-12-04 18:54:08,014 : INFO : PROGRESS: at sentence #65210000, processed 1001869154 words, keeping 376822 word types\n",
      "2019-12-04 18:54:08,172 : INFO : PROGRESS: at sentence #65220000, processed 1001990710 words, keeping 376822 word types\n",
      "2019-12-04 18:54:08,329 : INFO : PROGRESS: at sentence #65230000, processed 1002112392 words, keeping 376822 word types\n",
      "2019-12-04 18:54:08,480 : INFO : PROGRESS: at sentence #65240000, processed 1002229714 words, keeping 376822 word types\n",
      "2019-12-04 18:54:08,619 : INFO : PROGRESS: at sentence #65250000, processed 1002335433 words, keeping 376822 word types\n",
      "2019-12-04 18:54:08,751 : INFO : PROGRESS: at sentence #65260000, processed 1002434238 words, keeping 376822 word types\n",
      "2019-12-04 18:54:08,886 : INFO : PROGRESS: at sentence #65270000, processed 1002536574 words, keeping 376822 word types\n",
      "2019-12-04 18:54:09,020 : INFO : PROGRESS: at sentence #65280000, processed 1002640241 words, keeping 376822 word types\n",
      "2019-12-04 18:54:09,146 : INFO : PROGRESS: at sentence #65290000, processed 1002736879 words, keeping 376822 word types\n",
      "2019-12-04 18:54:09,277 : INFO : PROGRESS: at sentence #65300000, processed 1002837318 words, keeping 376822 word types\n",
      "2019-12-04 18:54:09,402 : INFO : PROGRESS: at sentence #65310000, processed 1002932694 words, keeping 376822 word types\n",
      "2019-12-04 18:54:09,527 : INFO : PROGRESS: at sentence #65320000, processed 1003028932 words, keeping 376822 word types\n",
      "2019-12-04 18:54:09,656 : INFO : PROGRESS: at sentence #65330000, processed 1003128034 words, keeping 376822 word types\n",
      "2019-12-04 18:54:09,790 : INFO : PROGRESS: at sentence #65340000, processed 1003231442 words, keeping 376822 word types\n",
      "2019-12-04 18:54:09,926 : INFO : PROGRESS: at sentence #65350000, processed 1003336996 words, keeping 376822 word types\n",
      "2019-12-04 18:54:10,063 : INFO : PROGRESS: at sentence #65360000, processed 1003441361 words, keeping 376822 word types\n",
      "2019-12-04 18:54:10,198 : INFO : PROGRESS: at sentence #65370000, processed 1003546402 words, keeping 376822 word types\n",
      "2019-12-04 18:54:10,335 : INFO : PROGRESS: at sentence #65380000, processed 1003652141 words, keeping 376822 word types\n",
      "2019-12-04 18:54:10,470 : INFO : PROGRESS: at sentence #65390000, processed 1003754150 words, keeping 376822 word types\n",
      "2019-12-04 18:54:10,607 : INFO : PROGRESS: at sentence #65400000, processed 1003855670 words, keeping 376822 word types\n",
      "2019-12-04 18:54:10,743 : INFO : PROGRESS: at sentence #65410000, processed 1003957584 words, keeping 376822 word types\n",
      "2019-12-04 18:54:10,897 : INFO : PROGRESS: at sentence #65420000, processed 1004079587 words, keeping 376822 word types\n",
      "2019-12-04 18:54:11,052 : INFO : PROGRESS: at sentence #65430000, processed 1004199478 words, keeping 376822 word types\n",
      "2019-12-04 18:54:11,199 : INFO : PROGRESS: at sentence #65440000, processed 1004313735 words, keeping 376822 word types\n",
      "2019-12-04 18:54:11,345 : INFO : PROGRESS: at sentence #65450000, processed 1004423747 words, keeping 376822 word types\n",
      "2019-12-04 18:54:11,484 : INFO : PROGRESS: at sentence #65460000, processed 1004529527 words, keeping 376822 word types\n",
      "2019-12-04 18:54:11,629 : INFO : PROGRESS: at sentence #65470000, processed 1004636013 words, keeping 376822 word types\n",
      "2019-12-04 18:54:11,774 : INFO : PROGRESS: at sentence #65480000, processed 1004742864 words, keeping 376822 word types\n",
      "2019-12-04 18:54:11,917 : INFO : PROGRESS: at sentence #65490000, processed 1004850423 words, keeping 376822 word types\n",
      "2019-12-04 18:54:12,061 : INFO : PROGRESS: at sentence #65500000, processed 1004957193 words, keeping 376822 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:54:12,207 : INFO : PROGRESS: at sentence #65510000, processed 1005066198 words, keeping 376822 word types\n",
      "2019-12-04 18:54:12,349 : INFO : PROGRESS: at sentence #65520000, processed 1005175897 words, keeping 376822 word types\n",
      "2019-12-04 18:54:12,495 : INFO : PROGRESS: at sentence #65530000, processed 1005286828 words, keeping 376822 word types\n",
      "2019-12-04 18:54:12,649 : INFO : PROGRESS: at sentence #65540000, processed 1005402469 words, keeping 376822 word types\n",
      "2019-12-04 18:54:12,796 : INFO : PROGRESS: at sentence #65550000, processed 1005513438 words, keeping 376822 word types\n",
      "2019-12-04 18:54:12,949 : INFO : PROGRESS: at sentence #65560000, processed 1005628306 words, keeping 376822 word types\n",
      "2019-12-04 18:54:13,096 : INFO : PROGRESS: at sentence #65570000, processed 1005736723 words, keeping 376822 word types\n",
      "2019-12-04 18:54:13,264 : INFO : PROGRESS: at sentence #65580000, processed 1005864736 words, keeping 376822 word types\n",
      "2019-12-04 18:54:13,421 : INFO : PROGRESS: at sentence #65590000, processed 1005985261 words, keeping 376822 word types\n",
      "2019-12-04 18:54:13,578 : INFO : PROGRESS: at sentence #65600000, processed 1006107232 words, keeping 376822 word types\n",
      "2019-12-04 18:54:13,729 : INFO : PROGRESS: at sentence #65610000, processed 1006224810 words, keeping 376822 word types\n",
      "2019-12-04 18:54:13,881 : INFO : PROGRESS: at sentence #65620000, processed 1006343406 words, keeping 376822 word types\n",
      "2019-12-04 18:54:14,013 : INFO : PROGRESS: at sentence #65630000, processed 1006442333 words, keeping 376822 word types\n",
      "2019-12-04 18:54:14,144 : INFO : PROGRESS: at sentence #65640000, processed 1006542585 words, keeping 376822 word types\n",
      "2019-12-04 18:54:14,278 : INFO : PROGRESS: at sentence #65650000, processed 1006644408 words, keeping 376822 word types\n",
      "2019-12-04 18:54:14,415 : INFO : PROGRESS: at sentence #65660000, processed 1006749620 words, keeping 376822 word types\n",
      "2019-12-04 18:54:14,541 : INFO : PROGRESS: at sentence #65670000, processed 1006844861 words, keeping 376822 word types\n",
      "2019-12-04 18:54:14,668 : INFO : PROGRESS: at sentence #65680000, processed 1006941695 words, keeping 376822 word types\n",
      "2019-12-04 18:54:14,834 : INFO : PROGRESS: at sentence #65690000, processed 1007039090 words, keeping 376822 word types\n",
      "2019-12-04 18:54:14,961 : INFO : PROGRESS: at sentence #65700000, processed 1007135277 words, keeping 376822 word types\n",
      "2019-12-04 18:54:15,091 : INFO : PROGRESS: at sentence #65710000, processed 1007234530 words, keeping 376822 word types\n",
      "2019-12-04 18:54:15,227 : INFO : PROGRESS: at sentence #65720000, processed 1007341074 words, keeping 376822 word types\n",
      "2019-12-04 18:54:15,363 : INFO : PROGRESS: at sentence #65730000, processed 1007445981 words, keeping 376822 word types\n",
      "2019-12-04 18:54:15,497 : INFO : PROGRESS: at sentence #65740000, processed 1007549826 words, keeping 376822 word types\n",
      "2019-12-04 18:54:15,636 : INFO : PROGRESS: at sentence #65750000, processed 1007655072 words, keeping 376822 word types\n",
      "2019-12-04 18:54:15,777 : INFO : PROGRESS: at sentence #65760000, processed 1007761026 words, keeping 376822 word types\n",
      "2019-12-04 18:54:15,911 : INFO : PROGRESS: at sentence #65770000, processed 1007861187 words, keeping 376822 word types\n",
      "2019-12-04 18:54:16,091 : INFO : PROGRESS: at sentence #65780000, processed 1007963952 words, keeping 376822 word types\n",
      "2019-12-04 18:54:16,230 : INFO : PROGRESS: at sentence #65790000, processed 1008067176 words, keeping 376822 word types\n",
      "2019-12-04 18:54:16,397 : INFO : PROGRESS: at sentence #65800000, processed 1008197155 words, keeping 376822 word types\n",
      "2019-12-04 18:54:16,548 : INFO : PROGRESS: at sentence #65810000, processed 1008312389 words, keeping 376822 word types\n",
      "2019-12-04 18:54:16,695 : INFO : PROGRESS: at sentence #65820000, processed 1008424904 words, keeping 376822 word types\n",
      "2019-12-04 18:54:16,840 : INFO : PROGRESS: at sentence #65830000, processed 1008532306 words, keeping 376822 word types\n",
      "2019-12-04 18:54:16,981 : INFO : PROGRESS: at sentence #65840000, processed 1008638554 words, keeping 376822 word types\n",
      "2019-12-04 18:54:17,126 : INFO : PROGRESS: at sentence #65850000, processed 1008746096 words, keeping 376822 word types\n",
      "2019-12-04 18:54:17,269 : INFO : PROGRESS: at sentence #65860000, processed 1008852522 words, keeping 376822 word types\n",
      "2019-12-04 18:54:17,410 : INFO : PROGRESS: at sentence #65870000, processed 1008960221 words, keeping 376822 word types\n",
      "2019-12-04 18:54:17,557 : INFO : PROGRESS: at sentence #65880000, processed 1009067859 words, keeping 376822 word types\n",
      "2019-12-04 18:54:17,703 : INFO : PROGRESS: at sentence #65890000, processed 1009176590 words, keeping 376822 word types\n",
      "2019-12-04 18:54:17,847 : INFO : PROGRESS: at sentence #65900000, processed 1009286917 words, keeping 376822 word types\n",
      "2019-12-04 18:54:17,999 : INFO : PROGRESS: at sentence #65910000, processed 1009401706 words, keeping 376822 word types\n",
      "2019-12-04 18:54:18,190 : INFO : PROGRESS: at sentence #65920000, processed 1009513714 words, keeping 376822 word types\n",
      "2019-12-04 18:54:18,344 : INFO : PROGRESS: at sentence #65930000, processed 1009626375 words, keeping 376822 word types\n",
      "2019-12-04 18:54:18,495 : INFO : PROGRESS: at sentence #65940000, processed 1009739791 words, keeping 376822 word types\n",
      "2019-12-04 18:54:18,656 : INFO : PROGRESS: at sentence #65950000, processed 1009858951 words, keeping 376822 word types\n",
      "2019-12-04 18:54:18,819 : INFO : PROGRESS: at sentence #65960000, processed 1009979087 words, keeping 376822 word types\n",
      "2019-12-04 18:54:18,984 : INFO : PROGRESS: at sentence #65970000, processed 1010100285 words, keeping 376822 word types\n",
      "2019-12-04 18:54:19,143 : INFO : PROGRESS: at sentence #65980000, processed 1010222813 words, keeping 376822 word types\n",
      "2019-12-04 18:54:19,298 : INFO : PROGRESS: at sentence #65990000, processed 1010340440 words, keeping 376822 word types\n",
      "2019-12-04 18:54:19,444 : INFO : PROGRESS: at sentence #66000000, processed 1010449650 words, keeping 376822 word types\n",
      "2019-12-04 18:54:19,578 : INFO : PROGRESS: at sentence #66010000, processed 1010549550 words, keeping 376822 word types\n",
      "2019-12-04 18:54:19,713 : INFO : PROGRESS: at sentence #66020000, processed 1010650733 words, keeping 376822 word types\n",
      "2019-12-04 18:54:19,851 : INFO : PROGRESS: at sentence #66030000, processed 1010753292 words, keeping 376822 word types\n",
      "2019-12-04 18:54:20,003 : INFO : PROGRESS: at sentence #66040000, processed 1010851106 words, keeping 376822 word types\n",
      "2019-12-04 18:54:20,144 : INFO : PROGRESS: at sentence #66050000, processed 1010953272 words, keeping 376822 word types\n",
      "2019-12-04 18:54:20,271 : INFO : PROGRESS: at sentence #66060000, processed 1011047857 words, keeping 376822 word types\n",
      "2019-12-04 18:54:20,398 : INFO : PROGRESS: at sentence #66070000, processed 1011144651 words, keeping 376822 word types\n",
      "2019-12-04 18:54:20,529 : INFO : PROGRESS: at sentence #66080000, processed 1011242463 words, keeping 376822 word types\n",
      "2019-12-04 18:54:20,662 : INFO : PROGRESS: at sentence #66090000, processed 1011344620 words, keeping 376822 word types\n",
      "2019-12-04 18:54:20,798 : INFO : PROGRESS: at sentence #66100000, processed 1011450259 words, keeping 376822 word types\n",
      "2019-12-04 18:54:20,935 : INFO : PROGRESS: at sentence #66110000, processed 1011554905 words, keeping 376822 word types\n",
      "2019-12-04 18:54:21,069 : INFO : PROGRESS: at sentence #66120000, processed 1011659477 words, keeping 376822 word types\n",
      "2019-12-04 18:54:21,205 : INFO : PROGRESS: at sentence #66130000, processed 1011765032 words, keeping 376822 word types\n",
      "2019-12-04 18:54:21,339 : INFO : PROGRESS: at sentence #66140000, processed 1011868451 words, keeping 376822 word types\n",
      "2019-12-04 18:54:21,494 : INFO : PROGRESS: at sentence #66150000, processed 1011969007 words, keeping 376822 word types\n",
      "2019-12-04 18:54:21,631 : INFO : PROGRESS: at sentence #66160000, processed 1012072164 words, keeping 376822 word types\n",
      "2019-12-04 18:54:21,782 : INFO : PROGRESS: at sentence #66170000, processed 1012187207 words, keeping 376822 word types\n",
      "2019-12-04 18:54:21,940 : INFO : PROGRESS: at sentence #66180000, processed 1012310166 words, keeping 376822 word types\n",
      "2019-12-04 18:54:22,088 : INFO : PROGRESS: at sentence #66190000, processed 1012426173 words, keeping 376822 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:54:22,232 : INFO : PROGRESS: at sentence #66200000, processed 1012537450 words, keeping 376822 word types\n",
      "2019-12-04 18:54:22,371 : INFO : PROGRESS: at sentence #66210000, processed 1012640882 words, keeping 376822 word types\n",
      "2019-12-04 18:54:22,511 : INFO : PROGRESS: at sentence #66220000, processed 1012747378 words, keeping 376822 word types\n",
      "2019-12-04 18:54:22,660 : INFO : PROGRESS: at sentence #66230000, processed 1012855779 words, keeping 376822 word types\n",
      "2019-12-04 18:54:22,808 : INFO : PROGRESS: at sentence #66240000, processed 1012963442 words, keeping 376822 word types\n",
      "2019-12-04 18:54:22,956 : INFO : PROGRESS: at sentence #66250000, processed 1013070390 words, keeping 376822 word types\n",
      "2019-12-04 18:54:23,105 : INFO : PROGRESS: at sentence #66260000, processed 1013178575 words, keeping 376822 word types\n",
      "2019-12-04 18:54:23,249 : INFO : PROGRESS: at sentence #66270000, processed 1013288081 words, keeping 376822 word types\n",
      "2019-12-04 18:54:23,394 : INFO : PROGRESS: at sentence #66280000, processed 1013398341 words, keeping 376822 word types\n",
      "2019-12-04 18:54:23,554 : INFO : PROGRESS: at sentence #66290000, processed 1013514811 words, keeping 376822 word types\n",
      "2019-12-04 18:54:23,717 : INFO : PROGRESS: at sentence #66300000, processed 1013624741 words, keeping 376822 word types\n",
      "2019-12-04 18:54:23,874 : INFO : PROGRESS: at sentence #66310000, processed 1013739110 words, keeping 376822 word types\n",
      "2019-12-04 18:54:24,032 : INFO : PROGRESS: at sentence #66320000, processed 1013849404 words, keeping 376822 word types\n",
      "2019-12-04 18:54:24,230 : INFO : PROGRESS: at sentence #66330000, processed 1013974194 words, keeping 376822 word types\n",
      "2019-12-04 18:54:24,390 : INFO : PROGRESS: at sentence #66340000, processed 1014094421 words, keeping 376822 word types\n",
      "2019-12-04 18:54:24,551 : INFO : PROGRESS: at sentence #66350000, processed 1014216252 words, keeping 376822 word types\n",
      "2019-12-04 18:54:24,708 : INFO : PROGRESS: at sentence #66360000, processed 1014336112 words, keeping 376822 word types\n",
      "2019-12-04 18:54:24,868 : INFO : PROGRESS: at sentence #66370000, processed 1014454561 words, keeping 376822 word types\n",
      "2019-12-04 18:54:25,005 : INFO : PROGRESS: at sentence #66380000, processed 1014556900 words, keeping 376822 word types\n",
      "2019-12-04 18:54:25,138 : INFO : PROGRESS: at sentence #66390000, processed 1014656354 words, keeping 376822 word types\n",
      "2019-12-04 18:54:25,273 : INFO : PROGRESS: at sentence #66400000, processed 1014758409 words, keeping 376822 word types\n",
      "2019-12-04 18:54:25,411 : INFO : PROGRESS: at sentence #66410000, processed 1014863233 words, keeping 376822 word types\n",
      "2019-12-04 18:54:25,537 : INFO : PROGRESS: at sentence #66420000, processed 1014957971 words, keeping 376822 word types\n",
      "2019-12-04 18:54:25,670 : INFO : PROGRESS: at sentence #66430000, processed 1015058035 words, keeping 376822 word types\n",
      "2019-12-04 18:54:25,796 : INFO : PROGRESS: at sentence #66440000, processed 1015153983 words, keeping 376822 word types\n",
      "2019-12-04 18:54:25,921 : INFO : PROGRESS: at sentence #66450000, processed 1015250277 words, keeping 376822 word types\n",
      "2019-12-04 18:54:26,051 : INFO : PROGRESS: at sentence #66460000, processed 1015349677 words, keeping 376822 word types\n",
      "2019-12-04 18:54:26,190 : INFO : PROGRESS: at sentence #66470000, processed 1015454354 words, keeping 376822 word types\n",
      "2019-12-04 18:54:26,328 : INFO : PROGRESS: at sentence #66480000, processed 1015559229 words, keeping 376822 word types\n",
      "2019-12-04 18:54:26,466 : INFO : PROGRESS: at sentence #66490000, processed 1015664070 words, keeping 376822 word types\n",
      "2019-12-04 18:54:26,602 : INFO : PROGRESS: at sentence #66500000, processed 1015768919 words, keeping 376822 word types\n",
      "2019-12-04 18:54:26,742 : INFO : PROGRESS: at sentence #66510000, processed 1015875247 words, keeping 376822 word types\n",
      "2019-12-04 18:54:26,882 : INFO : PROGRESS: at sentence #66520000, processed 1015976075 words, keeping 376822 word types\n",
      "2019-12-04 18:54:27,018 : INFO : PROGRESS: at sentence #66530000, processed 1016077847 words, keeping 376822 word types\n",
      "2019-12-04 18:54:27,157 : INFO : PROGRESS: at sentence #66540000, processed 1016180230 words, keeping 376822 word types\n",
      "2019-12-04 18:54:27,326 : INFO : PROGRESS: at sentence #66550000, processed 1016306076 words, keeping 376822 word types\n",
      "2019-12-04 18:54:27,487 : INFO : PROGRESS: at sentence #66560000, processed 1016423904 words, keeping 376822 word types\n",
      "2019-12-04 18:54:27,640 : INFO : PROGRESS: at sentence #66570000, processed 1016536811 words, keeping 376822 word types\n",
      "2019-12-04 18:54:27,790 : INFO : PROGRESS: at sentence #66580000, processed 1016646080 words, keeping 376822 word types\n",
      "2019-12-04 18:54:27,933 : INFO : PROGRESS: at sentence #66590000, processed 1016752082 words, keeping 376822 word types\n",
      "2019-12-04 18:54:28,080 : INFO : PROGRESS: at sentence #66600000, processed 1016858926 words, keeping 376822 word types\n",
      "2019-12-04 18:54:28,222 : INFO : PROGRESS: at sentence #66610000, processed 1016965655 words, keeping 376822 word types\n",
      "2019-12-04 18:54:28,368 : INFO : PROGRESS: at sentence #66620000, processed 1017072978 words, keeping 376822 word types\n",
      "2019-12-04 18:54:28,515 : INFO : PROGRESS: at sentence #66630000, processed 1017180378 words, keeping 376822 word types\n",
      "2019-12-04 18:54:28,660 : INFO : PROGRESS: at sentence #66640000, processed 1017288987 words, keeping 376822 word types\n",
      "2019-12-04 18:54:28,811 : INFO : PROGRESS: at sentence #66650000, processed 1017398892 words, keeping 376822 word types\n",
      "2019-12-04 18:54:28,963 : INFO : PROGRESS: at sentence #66660000, processed 1017511223 words, keeping 376822 word types\n",
      "2019-12-04 18:54:29,121 : INFO : PROGRESS: at sentence #66670000, processed 1017625595 words, keeping 376822 word types\n",
      "2019-12-04 18:54:29,278 : INFO : PROGRESS: at sentence #66680000, processed 1017737483 words, keeping 376822 word types\n",
      "2019-12-04 18:54:29,432 : INFO : PROGRESS: at sentence #66690000, processed 1017852097 words, keeping 376822 word types\n",
      "2019-12-04 18:54:29,586 : INFO : PROGRESS: at sentence #66700000, processed 1017966239 words, keeping 376822 word types\n",
      "2019-12-04 18:54:29,749 : INFO : PROGRESS: at sentence #66710000, processed 1018089337 words, keeping 376822 word types\n",
      "2019-12-04 18:54:29,913 : INFO : PROGRESS: at sentence #66720000, processed 1018209617 words, keeping 376822 word types\n",
      "2019-12-04 18:54:30,076 : INFO : PROGRESS: at sentence #66730000, processed 1018332703 words, keeping 376822 word types\n",
      "2019-12-04 18:54:30,229 : INFO : PROGRESS: at sentence #66740000, processed 1018447888 words, keeping 376822 word types\n",
      "2019-12-04 18:54:30,382 : INFO : PROGRESS: at sentence #66750000, processed 1018564526 words, keeping 376822 word types\n",
      "2019-12-04 18:54:30,517 : INFO : PROGRESS: at sentence #66760000, processed 1018663975 words, keeping 376822 word types\n",
      "2019-12-04 18:54:30,653 : INFO : PROGRESS: at sentence #66770000, processed 1018764942 words, keeping 376822 word types\n",
      "2019-12-04 18:54:30,788 : INFO : PROGRESS: at sentence #66780000, processed 1018867297 words, keeping 376822 word types\n",
      "2019-12-04 18:54:30,924 : INFO : PROGRESS: at sentence #66790000, processed 1018970676 words, keeping 376822 word types\n",
      "2019-12-04 18:54:31,051 : INFO : PROGRESS: at sentence #66800000, processed 1019066856 words, keeping 376822 word types\n",
      "2019-12-04 18:54:31,176 : INFO : PROGRESS: at sentence #66810000, processed 1019162576 words, keeping 376822 word types\n",
      "2019-12-04 18:54:31,307 : INFO : PROGRESS: at sentence #66820000, processed 1019260259 words, keeping 376822 word types\n",
      "2019-12-04 18:54:31,436 : INFO : PROGRESS: at sentence #66830000, processed 1019357204 words, keeping 376822 word types\n",
      "2019-12-04 18:54:31,566 : INFO : PROGRESS: at sentence #66840000, processed 1019457739 words, keeping 376822 word types\n",
      "2019-12-04 18:54:31,704 : INFO : PROGRESS: at sentence #66850000, processed 1019563871 words, keeping 376822 word types\n",
      "2019-12-04 18:54:31,840 : INFO : PROGRESS: at sentence #66860000, processed 1019668627 words, keeping 376822 word types\n",
      "2019-12-04 18:54:31,974 : INFO : PROGRESS: at sentence #66870000, processed 1019772399 words, keeping 376822 word types\n",
      "2019-12-04 18:54:32,111 : INFO : PROGRESS: at sentence #66880000, processed 1019877566 words, keeping 376822 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:54:32,249 : INFO : PROGRESS: at sentence #66890000, processed 1019983350 words, keeping 376822 word types\n",
      "2019-12-04 18:54:32,383 : INFO : PROGRESS: at sentence #66900000, processed 1020083140 words, keeping 376822 word types\n",
      "2019-12-04 18:54:32,521 : INFO : PROGRESS: at sentence #66910000, processed 1020186181 words, keeping 376822 word types\n",
      "2019-12-04 18:54:32,663 : INFO : PROGRESS: at sentence #66920000, processed 1020293721 words, keeping 376822 word types\n",
      "2019-12-04 18:54:32,827 : INFO : PROGRESS: at sentence #66930000, processed 1020421436 words, keeping 376822 word types\n",
      "2019-12-04 18:54:32,981 : INFO : PROGRESS: at sentence #66940000, processed 1020536989 words, keeping 376822 word types\n",
      "2019-12-04 18:54:33,126 : INFO : PROGRESS: at sentence #66950000, processed 1020647971 words, keeping 376822 word types\n",
      "2019-12-04 18:54:33,266 : INFO : PROGRESS: at sentence #66960000, processed 1020754541 words, keeping 376822 word types\n",
      "2019-12-04 18:54:33,412 : INFO : PROGRESS: at sentence #66970000, processed 1020860755 words, keeping 376822 word types\n",
      "2019-12-04 18:54:33,563 : INFO : PROGRESS: at sentence #66980000, processed 1020968940 words, keeping 376822 word types\n",
      "2019-12-04 18:54:33,706 : INFO : PROGRESS: at sentence #66990000, processed 1021075577 words, keeping 376822 word types\n",
      "2019-12-04 18:54:33,849 : INFO : PROGRESS: at sentence #67000000, processed 1021183384 words, keeping 376822 word types\n",
      "2019-12-04 18:54:33,998 : INFO : PROGRESS: at sentence #67010000, processed 1021291481 words, keeping 376822 word types\n",
      "2019-12-04 18:54:34,142 : INFO : PROGRESS: at sentence #67020000, processed 1021399989 words, keeping 376822 word types\n",
      "2019-12-04 18:54:34,287 : INFO : PROGRESS: at sentence #67030000, processed 1021510251 words, keeping 376822 word types\n",
      "2019-12-04 18:54:34,443 : INFO : PROGRESS: at sentence #67040000, processed 1021626117 words, keeping 376822 word types\n",
      "2019-12-04 18:54:34,592 : INFO : PROGRESS: at sentence #67050000, processed 1021736980 words, keeping 376822 word types\n",
      "2019-12-04 18:54:34,743 : INFO : PROGRESS: at sentence #67060000, processed 1021850260 words, keeping 376822 word types\n",
      "2019-12-04 18:54:34,898 : INFO : PROGRESS: at sentence #67070000, processed 1021962586 words, keeping 376822 word types\n",
      "2019-12-04 18:54:35,060 : INFO : PROGRESS: at sentence #67080000, processed 1022084002 words, keeping 376822 word types\n",
      "2019-12-04 18:54:35,228 : INFO : PROGRESS: at sentence #67090000, processed 1022204059 words, keeping 376822 word types\n",
      "2019-12-04 18:54:35,389 : INFO : PROGRESS: at sentence #67100000, processed 1022325152 words, keeping 376822 word types\n",
      "2019-12-04 18:54:35,548 : INFO : PROGRESS: at sentence #67110000, processed 1022447808 words, keeping 376822 word types\n",
      "2019-12-04 18:54:35,703 : INFO : PROGRESS: at sentence #67120000, processed 1022564474 words, keeping 376822 word types\n",
      "2019-12-04 18:54:35,848 : INFO : PROGRESS: at sentence #67130000, processed 1022671609 words, keeping 376822 word types\n",
      "2019-12-04 18:54:35,982 : INFO : PROGRESS: at sentence #67140000, processed 1022770615 words, keeping 376822 word types\n",
      "2019-12-04 18:54:36,118 : INFO : PROGRESS: at sentence #67150000, processed 1022872850 words, keeping 376822 word types\n",
      "2019-12-04 18:54:36,256 : INFO : PROGRESS: at sentence #67160000, processed 1022976179 words, keeping 376822 word types\n",
      "2019-12-04 18:54:36,387 : INFO : PROGRESS: at sentence #67170000, processed 1023074133 words, keeping 376822 word types\n",
      "2019-12-04 18:54:36,517 : INFO : PROGRESS: at sentence #67180000, processed 1023174090 words, keeping 376822 word types\n",
      "2019-12-04 18:54:36,641 : INFO : PROGRESS: at sentence #67190000, processed 1023268971 words, keeping 376822 word types\n",
      "2019-12-04 18:54:36,770 : INFO : PROGRESS: at sentence #67200000, processed 1023365783 words, keeping 376822 word types\n",
      "2019-12-04 18:54:36,899 : INFO : PROGRESS: at sentence #67210000, processed 1023464252 words, keeping 376822 word types\n",
      "2019-12-04 18:54:37,033 : INFO : PROGRESS: at sentence #67220000, processed 1023567117 words, keeping 376822 word types\n",
      "2019-12-04 18:54:37,170 : INFO : PROGRESS: at sentence #67230000, processed 1023673043 words, keeping 376822 word types\n",
      "2019-12-04 18:54:37,306 : INFO : PROGRESS: at sentence #67240000, processed 1023777299 words, keeping 376822 word types\n",
      "2019-12-04 18:54:37,443 : INFO : PROGRESS: at sentence #67250000, processed 1023882274 words, keeping 376822 word types\n",
      "2019-12-04 18:54:37,583 : INFO : PROGRESS: at sentence #67260000, processed 1023987811 words, keeping 376822 word types\n",
      "2019-12-04 18:54:37,718 : INFO : PROGRESS: at sentence #67270000, processed 1024090411 words, keeping 376822 word types\n",
      "2019-12-04 18:54:37,851 : INFO : PROGRESS: at sentence #67280000, processed 1024191858 words, keeping 376822 word types\n",
      "2019-12-04 18:54:37,987 : INFO : PROGRESS: at sentence #67290000, processed 1024293693 words, keeping 376822 word types\n",
      "2019-12-04 18:54:38,143 : INFO : PROGRESS: at sentence #67300000, processed 1024413384 words, keeping 376822 word types\n",
      "2019-12-04 18:54:38,301 : INFO : PROGRESS: at sentence #67310000, processed 1024534459 words, keeping 376822 word types\n",
      "2019-12-04 18:54:38,453 : INFO : PROGRESS: at sentence #67320000, processed 1024649455 words, keeping 376822 word types\n",
      "2019-12-04 18:54:38,599 : INFO : PROGRESS: at sentence #67330000, processed 1024759567 words, keeping 376822 word types\n",
      "2019-12-04 18:54:38,740 : INFO : PROGRESS: at sentence #67340000, processed 1024865504 words, keeping 376822 word types\n",
      "2019-12-04 18:54:38,882 : INFO : PROGRESS: at sentence #67350000, processed 1024970851 words, keeping 376822 word types\n",
      "2019-12-04 18:54:39,027 : INFO : PROGRESS: at sentence #67360000, processed 1025078690 words, keeping 376822 word types\n",
      "2019-12-04 18:54:39,170 : INFO : PROGRESS: at sentence #67370000, processed 1025186378 words, keeping 376822 word types\n",
      "2019-12-04 18:54:39,315 : INFO : PROGRESS: at sentence #67380000, processed 1025293048 words, keeping 376822 word types\n",
      "2019-12-04 18:54:39,459 : INFO : PROGRESS: at sentence #67390000, processed 1025401549 words, keeping 376822 word types\n",
      "2019-12-04 18:54:39,605 : INFO : PROGRESS: at sentence #67400000, processed 1025511436 words, keeping 376822 word types\n",
      "2019-12-04 18:54:39,751 : INFO : PROGRESS: at sentence #67410000, processed 1025622232 words, keeping 376822 word types\n",
      "2019-12-04 18:54:39,904 : INFO : PROGRESS: at sentence #67420000, processed 1025738293 words, keeping 376822 word types\n",
      "2019-12-04 18:54:40,051 : INFO : PROGRESS: at sentence #67430000, processed 1025848345 words, keeping 376822 word types\n",
      "2019-12-04 18:54:40,211 : INFO : PROGRESS: at sentence #67440000, processed 1025963502 words, keeping 376822 word types\n",
      "2019-12-04 18:54:40,394 : INFO : PROGRESS: at sentence #67450000, processed 1026072542 words, keeping 376822 word types\n",
      "2019-12-04 18:54:40,557 : INFO : PROGRESS: at sentence #67460000, processed 1026198967 words, keeping 376822 word types\n",
      "2019-12-04 18:54:40,712 : INFO : PROGRESS: at sentence #67470000, processed 1026319668 words, keeping 376822 word types\n",
      "2019-12-04 18:54:40,868 : INFO : PROGRESS: at sentence #67480000, processed 1026441770 words, keeping 376822 word types\n",
      "2019-12-04 18:54:41,018 : INFO : PROGRESS: at sentence #67490000, processed 1026559735 words, keeping 376822 word types\n",
      "2019-12-04 18:54:41,172 : INFO : PROGRESS: at sentence #67500000, processed 1026678872 words, keeping 376822 word types\n",
      "2019-12-04 18:54:41,302 : INFO : PROGRESS: at sentence #67510000, processed 1026778626 words, keeping 376822 word types\n",
      "2019-12-04 18:54:41,432 : INFO : PROGRESS: at sentence #67520000, processed 1026878819 words, keeping 376822 word types\n",
      "2019-12-04 18:54:41,563 : INFO : PROGRESS: at sentence #67530000, processed 1026980601 words, keeping 376822 word types\n",
      "2019-12-04 18:54:41,698 : INFO : PROGRESS: at sentence #67540000, processed 1027085855 words, keeping 376822 word types\n",
      "2019-12-04 18:54:41,820 : INFO : PROGRESS: at sentence #67550000, processed 1027181408 words, keeping 376822 word types\n",
      "2019-12-04 18:54:41,945 : INFO : PROGRESS: at sentence #67560000, processed 1027278716 words, keeping 376822 word types\n",
      "2019-12-04 18:54:42,070 : INFO : PROGRESS: at sentence #67570000, processed 1027375776 words, keeping 376822 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:54:42,194 : INFO : PROGRESS: at sentence #67580000, processed 1027472031 words, keeping 376822 word types\n",
      "2019-12-04 18:54:42,322 : INFO : PROGRESS: at sentence #67590000, processed 1027571327 words, keeping 376822 word types\n",
      "2019-12-04 18:54:42,458 : INFO : PROGRESS: at sentence #67600000, processed 1027676749 words, keeping 376822 word types\n",
      "2019-12-04 18:54:42,596 : INFO : PROGRESS: at sentence #67610000, processed 1027782016 words, keeping 376822 word types\n",
      "2019-12-04 18:54:42,731 : INFO : PROGRESS: at sentence #67620000, processed 1027885872 words, keeping 376822 word types\n",
      "2019-12-04 18:54:42,869 : INFO : PROGRESS: at sentence #67630000, processed 1027991466 words, keeping 376822 word types\n",
      "2019-12-04 18:54:43,006 : INFO : PROGRESS: at sentence #67640000, processed 1028097235 words, keeping 376822 word types\n",
      "2019-12-04 18:54:43,136 : INFO : PROGRESS: at sentence #67650000, processed 1028197628 words, keeping 376822 word types\n",
      "2019-12-04 18:54:43,266 : INFO : PROGRESS: at sentence #67660000, processed 1028300135 words, keeping 376822 word types\n",
      "2019-12-04 18:54:43,396 : INFO : PROGRESS: at sentence #67670000, processed 1028402832 words, keeping 376822 word types\n",
      "2019-12-04 18:54:43,555 : INFO : PROGRESS: at sentence #67680000, processed 1028532335 words, keeping 376822 word types\n",
      "2019-12-04 18:54:43,700 : INFO : PROGRESS: at sentence #67690000, processed 1028647662 words, keeping 376822 word types\n",
      "2019-12-04 18:54:43,843 : INFO : PROGRESS: at sentence #67700000, processed 1028760306 words, keeping 376822 word types\n",
      "2019-12-04 18:54:43,983 : INFO : PROGRESS: at sentence #67710000, processed 1028868350 words, keeping 376822 word types\n",
      "2019-12-04 18:54:44,122 : INFO : PROGRESS: at sentence #67720000, processed 1028974410 words, keeping 376822 word types\n",
      "2019-12-04 18:54:44,261 : INFO : PROGRESS: at sentence #67730000, processed 1029081815 words, keeping 376822 word types\n",
      "2019-12-04 18:54:44,400 : INFO : PROGRESS: at sentence #67740000, processed 1029188474 words, keeping 376822 word types\n",
      "2019-12-04 18:54:44,540 : INFO : PROGRESS: at sentence #67750000, processed 1029296043 words, keeping 376822 word types\n",
      "2019-12-04 18:54:44,681 : INFO : PROGRESS: at sentence #67760000, processed 1029403220 words, keeping 376822 word types\n",
      "2019-12-04 18:54:44,823 : INFO : PROGRESS: at sentence #67770000, processed 1029511752 words, keeping 376822 word types\n",
      "2019-12-04 18:54:44,967 : INFO : PROGRESS: at sentence #67780000, processed 1029622412 words, keeping 376822 word types\n",
      "2019-12-04 18:54:45,115 : INFO : PROGRESS: at sentence #67790000, processed 1029736060 words, keeping 376822 word types\n",
      "2019-12-04 18:54:45,263 : INFO : PROGRESS: at sentence #67800000, processed 1029849081 words, keeping 376822 word types\n",
      "2019-12-04 18:54:45,408 : INFO : PROGRESS: at sentence #67810000, processed 1029961377 words, keeping 376822 word types\n",
      "2019-12-04 18:54:45,556 : INFO : PROGRESS: at sentence #67820000, processed 1030075525 words, keeping 376822 word types\n",
      "2019-12-04 18:54:45,708 : INFO : PROGRESS: at sentence #67830000, processed 1030193151 words, keeping 376822 word types\n",
      "2019-12-04 18:54:45,868 : INFO : PROGRESS: at sentence #67840000, processed 1030313782 words, keeping 376822 word types\n",
      "2019-12-04 18:54:46,029 : INFO : PROGRESS: at sentence #67850000, processed 1030434557 words, keeping 376822 word types\n",
      "2019-12-04 18:54:46,189 : INFO : PROGRESS: at sentence #67860000, processed 1030557545 words, keeping 376822 word types\n",
      "2019-12-04 18:54:46,342 : INFO : PROGRESS: at sentence #67870000, processed 1030674829 words, keeping 376822 word types\n",
      "2019-12-04 18:54:46,487 : INFO : PROGRESS: at sentence #67880000, processed 1030785994 words, keeping 376822 word types\n",
      "2019-12-04 18:54:46,619 : INFO : PROGRESS: at sentence #67890000, processed 1030885985 words, keeping 376822 word types\n",
      "2019-12-04 18:54:46,752 : INFO : PROGRESS: at sentence #67900000, processed 1030987133 words, keeping 376822 word types\n",
      "2019-12-04 18:54:46,886 : INFO : PROGRESS: at sentence #67910000, processed 1031089663 words, keeping 376822 word types\n",
      "2019-12-04 18:54:47,014 : INFO : PROGRESS: at sentence #67920000, processed 1031187016 words, keeping 376822 word types\n",
      "2019-12-04 18:54:47,147 : INFO : PROGRESS: at sentence #67930000, processed 1031289489 words, keeping 376822 word types\n",
      "2019-12-04 18:54:47,271 : INFO : PROGRESS: at sentence #67940000, processed 1031383481 words, keeping 376822 word types\n",
      "2019-12-04 18:54:47,399 : INFO : PROGRESS: at sentence #67950000, processed 1031481516 words, keeping 376822 word types\n",
      "2019-12-04 18:54:47,527 : INFO : PROGRESS: at sentence #67960000, processed 1031578598 words, keeping 376822 word types\n",
      "2019-12-04 18:54:47,659 : INFO : PROGRESS: at sentence #67970000, processed 1031680456 words, keeping 376822 word types\n",
      "2019-12-04 18:54:47,799 : INFO : PROGRESS: at sentence #67980000, processed 1031786453 words, keeping 376822 word types\n",
      "2019-12-04 18:54:47,939 : INFO : PROGRESS: at sentence #67990000, processed 1031890958 words, keeping 376822 word types\n",
      "2019-12-04 18:54:48,079 : INFO : PROGRESS: at sentence #68000000, processed 1031995346 words, keeping 376822 word types\n",
      "2019-12-04 18:54:48,222 : INFO : PROGRESS: at sentence #68010000, processed 1032100785 words, keeping 376822 word types\n",
      "2019-12-04 18:54:48,350 : INFO : PROGRESS: at sentence #68020000, processed 1032205119 words, keeping 376822 word types\n",
      "2019-12-04 18:54:48,481 : INFO : PROGRESS: at sentence #68030000, processed 1032305110 words, keeping 376822 word types\n",
      "2019-12-04 18:54:48,617 : INFO : PROGRESS: at sentence #68040000, processed 1032408193 words, keeping 376822 word types\n",
      "2019-12-04 18:54:48,763 : INFO : PROGRESS: at sentence #68050000, processed 1032520855 words, keeping 376822 word types\n",
      "2019-12-04 18:54:48,925 : INFO : PROGRESS: at sentence #68060000, processed 1032645627 words, keeping 376822 word types\n",
      "2019-12-04 18:54:49,078 : INFO : PROGRESS: at sentence #68070000, processed 1032761354 words, keeping 376822 word types\n",
      "2019-12-04 18:54:49,223 : INFO : PROGRESS: at sentence #68080000, processed 1032873544 words, keeping 376822 word types\n",
      "2019-12-04 18:54:49,359 : INFO : PROGRESS: at sentence #68090000, processed 1032976979 words, keeping 376822 word types\n",
      "2019-12-04 18:54:49,500 : INFO : PROGRESS: at sentence #68100000, processed 1033083401 words, keeping 376822 word types\n",
      "2019-12-04 18:54:49,642 : INFO : PROGRESS: at sentence #68110000, processed 1033191504 words, keeping 376822 word types\n",
      "2019-12-04 18:54:49,786 : INFO : PROGRESS: at sentence #68120000, processed 1033298906 words, keeping 376822 word types\n",
      "2019-12-04 18:54:49,930 : INFO : PROGRESS: at sentence #68130000, processed 1033406148 words, keeping 376822 word types\n",
      "2019-12-04 18:54:50,073 : INFO : PROGRESS: at sentence #68140000, processed 1033514500 words, keeping 376822 word types\n",
      "2019-12-04 18:54:50,217 : INFO : PROGRESS: at sentence #68150000, processed 1033623597 words, keeping 376822 word types\n",
      "2019-12-04 18:54:50,361 : INFO : PROGRESS: at sentence #68160000, processed 1033733972 words, keeping 376822 word types\n",
      "2019-12-04 18:54:50,515 : INFO : PROGRESS: at sentence #68170000, processed 1033850137 words, keeping 376822 word types\n",
      "2019-12-04 18:54:50,661 : INFO : PROGRESS: at sentence #68180000, processed 1033960209 words, keeping 376822 word types\n",
      "2019-12-04 18:54:50,812 : INFO : PROGRESS: at sentence #68190000, processed 1034074448 words, keeping 376822 word types\n",
      "2019-12-04 18:54:50,962 : INFO : PROGRESS: at sentence #68200000, processed 1034185319 words, keeping 376822 word types\n",
      "2019-12-04 18:54:51,126 : INFO : PROGRESS: at sentence #68210000, processed 1034308644 words, keeping 376822 word types\n",
      "2019-12-04 18:54:51,285 : INFO : PROGRESS: at sentence #68220000, processed 1034429068 words, keeping 376822 word types\n",
      "2019-12-04 18:54:51,447 : INFO : PROGRESS: at sentence #68230000, processed 1034550525 words, keeping 376822 word types\n",
      "2019-12-04 18:54:51,608 : INFO : PROGRESS: at sentence #68240000, processed 1034671420 words, keeping 376822 word types\n",
      "2019-12-04 18:54:51,761 : INFO : PROGRESS: at sentence #68250000, processed 1034789246 words, keeping 376822 word types\n",
      "2019-12-04 18:54:51,900 : INFO : PROGRESS: at sentence #68260000, processed 1034893344 words, keeping 376822 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:54:52,035 : INFO : PROGRESS: at sentence #68270000, processed 1034992637 words, keeping 376822 word types\n",
      "2019-12-04 18:54:52,172 : INFO : PROGRESS: at sentence #68280000, processed 1035094942 words, keeping 376822 word types\n",
      "2019-12-04 18:54:52,310 : INFO : PROGRESS: at sentence #68290000, processed 1035198973 words, keeping 376822 word types\n",
      "2019-12-04 18:54:52,437 : INFO : PROGRESS: at sentence #68300000, processed 1035294615 words, keeping 376822 word types\n",
      "2019-12-04 18:54:52,567 : INFO : PROGRESS: at sentence #68310000, processed 1035395003 words, keeping 376822 word types\n",
      "2019-12-04 18:54:52,692 : INFO : PROGRESS: at sentence #68320000, processed 1035490476 words, keeping 376822 word types\n",
      "2019-12-04 18:54:52,821 : INFO : PROGRESS: at sentence #68330000, processed 1035587016 words, keeping 376822 word types\n",
      "2019-12-04 18:54:52,954 : INFO : PROGRESS: at sentence #68340000, processed 1035686229 words, keeping 376822 word types\n",
      "2019-12-04 18:54:53,091 : INFO : PROGRESS: at sentence #68350000, processed 1035790434 words, keeping 376822 word types\n",
      "2019-12-04 18:54:53,229 : INFO : PROGRESS: at sentence #68360000, processed 1035895390 words, keeping 376822 word types\n",
      "2019-12-04 18:54:53,366 : INFO : PROGRESS: at sentence #68370000, processed 1036000076 words, keeping 376822 word types\n",
      "2019-12-04 18:54:53,505 : INFO : PROGRESS: at sentence #68380000, processed 1036104660 words, keeping 376822 word types\n",
      "2019-12-04 18:54:53,645 : INFO : PROGRESS: at sentence #68390000, processed 1036210792 words, keeping 376822 word types\n",
      "2019-12-04 18:54:53,783 : INFO : PROGRESS: at sentence #68400000, processed 1036312387 words, keeping 376822 word types\n",
      "2019-12-04 18:54:53,920 : INFO : PROGRESS: at sentence #68410000, processed 1036413965 words, keeping 376822 word types\n",
      "2019-12-04 18:54:54,057 : INFO : PROGRESS: at sentence #68420000, processed 1036516133 words, keeping 376822 word types\n",
      "2019-12-04 18:54:54,214 : INFO : PROGRESS: at sentence #68430000, processed 1036639983 words, keeping 376822 word types\n",
      "2019-12-04 18:54:54,367 : INFO : PROGRESS: at sentence #68440000, processed 1036758943 words, keeping 376822 word types\n",
      "2019-12-04 18:54:54,512 : INFO : PROGRESS: at sentence #68450000, processed 1036872389 words, keeping 376822 word types\n",
      "2019-12-04 18:54:54,650 : INFO : PROGRESS: at sentence #68460000, processed 1036982061 words, keeping 376822 word types\n",
      "2019-12-04 18:54:54,791 : INFO : PROGRESS: at sentence #68470000, processed 1037088073 words, keeping 376822 word types\n",
      "2019-12-04 18:54:54,932 : INFO : PROGRESS: at sentence #68480000, processed 1037194680 words, keeping 376822 word types\n",
      "2019-12-04 18:54:55,073 : INFO : PROGRESS: at sentence #68490000, processed 1037301401 words, keeping 376822 word types\n",
      "2019-12-04 18:54:55,216 : INFO : PROGRESS: at sentence #68500000, processed 1037408807 words, keeping 376822 word types\n",
      "2019-12-04 18:54:55,357 : INFO : PROGRESS: at sentence #68510000, processed 1037516056 words, keeping 376822 word types\n",
      "2019-12-04 18:54:55,500 : INFO : PROGRESS: at sentence #68520000, processed 1037624863 words, keeping 376822 word types\n",
      "2019-12-04 18:54:55,680 : INFO : PROGRESS: at sentence #68530000, processed 1037734579 words, keeping 376822 word types\n",
      "2019-12-04 18:54:55,829 : INFO : PROGRESS: at sentence #68540000, processed 1037846136 words, keeping 376822 word types\n",
      "2019-12-04 18:54:55,981 : INFO : PROGRESS: at sentence #68550000, processed 1037960969 words, keeping 376822 word types\n",
      "2019-12-04 18:54:56,127 : INFO : PROGRESS: at sentence #68560000, processed 1038072834 words, keeping 376822 word types\n",
      "2019-12-04 18:54:56,278 : INFO : PROGRESS: at sentence #68570000, processed 1038187210 words, keeping 376822 word types\n",
      "2019-12-04 18:54:56,431 : INFO : PROGRESS: at sentence #68580000, processed 1038298604 words, keeping 376822 word types\n",
      "2019-12-04 18:54:56,598 : INFO : PROGRESS: at sentence #68590000, processed 1038423885 words, keeping 376822 word types\n",
      "2019-12-04 18:54:56,760 : INFO : PROGRESS: at sentence #68600000, processed 1038544373 words, keeping 376822 word types\n",
      "2019-12-04 18:54:56,918 : INFO : PROGRESS: at sentence #68610000, processed 1038667079 words, keeping 376822 word types\n",
      "2019-12-04 18:54:57,069 : INFO : PROGRESS: at sentence #68620000, processed 1038783506 words, keeping 376822 word types\n",
      "2019-12-04 18:54:57,217 : INFO : PROGRESS: at sentence #68630000, processed 1038901211 words, keeping 376822 word types\n",
      "2019-12-04 18:54:57,347 : INFO : PROGRESS: at sentence #68640000, processed 1039000310 words, keeping 376822 word types\n",
      "2019-12-04 18:54:57,477 : INFO : PROGRESS: at sentence #68650000, processed 1039101163 words, keeping 376822 word types\n",
      "2019-12-04 18:54:57,615 : INFO : PROGRESS: at sentence #68660000, processed 1039202890 words, keeping 376822 word types\n",
      "2019-12-04 18:54:57,752 : INFO : PROGRESS: at sentence #68670000, processed 1039307954 words, keeping 376822 word types\n",
      "2019-12-04 18:54:57,875 : INFO : PROGRESS: at sentence #68680000, processed 1039402859 words, keeping 376822 word types\n",
      "2019-12-04 18:54:58,003 : INFO : PROGRESS: at sentence #68690000, processed 1039499440 words, keeping 376822 word types\n",
      "2019-12-04 18:54:58,133 : INFO : PROGRESS: at sentence #68700000, processed 1039597088 words, keeping 376822 word types\n",
      "2019-12-04 18:54:58,259 : INFO : PROGRESS: at sentence #68710000, processed 1039693426 words, keeping 376822 word types\n",
      "2019-12-04 18:54:58,388 : INFO : PROGRESS: at sentence #68720000, processed 1039793132 words, keeping 376822 word types\n",
      "2019-12-04 18:54:58,525 : INFO : PROGRESS: at sentence #68730000, processed 1039899660 words, keeping 376822 word types\n",
      "2019-12-04 18:54:58,660 : INFO : PROGRESS: at sentence #68740000, processed 1040004733 words, keeping 376822 word types\n",
      "2019-12-04 18:54:58,795 : INFO : PROGRESS: at sentence #68750000, processed 1040108394 words, keeping 376822 word types\n",
      "2019-12-04 18:54:58,931 : INFO : PROGRESS: at sentence #68760000, processed 1040213433 words, keeping 376822 word types\n",
      "2019-12-04 18:54:59,071 : INFO : PROGRESS: at sentence #68770000, processed 1040319272 words, keeping 376822 word types\n",
      "2019-12-04 18:54:59,215 : INFO : PROGRESS: at sentence #68780000, processed 1040419394 words, keeping 376822 word types\n",
      "2019-12-04 18:54:59,358 : INFO : PROGRESS: at sentence #68790000, processed 1040522362 words, keeping 376822 word types\n",
      "2019-12-04 18:54:59,499 : INFO : PROGRESS: at sentence #68800000, processed 1040627597 words, keeping 376822 word types\n",
      "2019-12-04 18:54:59,667 : INFO : PROGRESS: at sentence #68810000, processed 1040756468 words, keeping 376822 word types\n",
      "2019-12-04 18:54:59,818 : INFO : PROGRESS: at sentence #68820000, processed 1040871679 words, keeping 376822 word types\n",
      "2019-12-04 18:54:59,966 : INFO : PROGRESS: at sentence #68830000, processed 1040983600 words, keeping 376822 word types\n",
      "2019-12-04 18:55:00,110 : INFO : PROGRESS: at sentence #68840000, processed 1041090681 words, keeping 376822 word types\n",
      "2019-12-04 18:55:00,254 : INFO : PROGRESS: at sentence #68850000, processed 1041196948 words, keeping 376822 word types\n",
      "2019-12-04 18:55:00,399 : INFO : PROGRESS: at sentence #68860000, processed 1041304853 words, keeping 376822 word types\n",
      "2019-12-04 18:55:00,543 : INFO : PROGRESS: at sentence #68870000, processed 1041411214 words, keeping 376822 word types\n",
      "2019-12-04 18:55:00,690 : INFO : PROGRESS: at sentence #68880000, processed 1041519038 words, keeping 376822 word types\n",
      "2019-12-04 18:55:00,833 : INFO : PROGRESS: at sentence #68890000, processed 1041626636 words, keeping 376822 word types\n",
      "2019-12-04 18:55:00,976 : INFO : PROGRESS: at sentence #68900000, processed 1041735440 words, keeping 376822 word types\n",
      "2019-12-04 18:55:01,121 : INFO : PROGRESS: at sentence #68910000, processed 1041845630 words, keeping 376822 word types\n",
      "2019-12-04 18:55:01,272 : INFO : PROGRESS: at sentence #68920000, processed 1041961239 words, keeping 376822 word types\n",
      "2019-12-04 18:55:01,424 : INFO : PROGRESS: at sentence #68930000, processed 1042072738 words, keeping 376822 word types\n",
      "2019-12-04 18:55:01,576 : INFO : PROGRESS: at sentence #68940000, processed 1042185525 words, keeping 376822 word types\n",
      "2019-12-04 18:55:01,728 : INFO : PROGRESS: at sentence #68950000, processed 1042298506 words, keeping 376822 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:55:01,889 : INFO : PROGRESS: at sentence #68960000, processed 1042418547 words, keeping 376822 word types\n",
      "2019-12-04 18:55:02,052 : INFO : PROGRESS: at sentence #68970000, processed 1042538702 words, keeping 376822 word types\n",
      "2019-12-04 18:55:02,213 : INFO : PROGRESS: at sentence #68980000, processed 1042659619 words, keeping 376822 word types\n",
      "2019-12-04 18:55:02,373 : INFO : PROGRESS: at sentence #68990000, processed 1042782715 words, keeping 376822 word types\n",
      "2019-12-04 18:55:02,526 : INFO : PROGRESS: at sentence #69000000, processed 1042900099 words, keeping 376822 word types\n",
      "2019-12-04 18:55:02,665 : INFO : PROGRESS: at sentence #69010000, processed 1043007715 words, keeping 376822 word types\n",
      "2019-12-04 18:55:02,794 : INFO : PROGRESS: at sentence #69020000, processed 1043107557 words, keeping 376822 word types\n",
      "2019-12-04 18:55:02,924 : INFO : PROGRESS: at sentence #69030000, processed 1043209014 words, keeping 376822 word types\n",
      "2019-12-04 18:55:03,059 : INFO : PROGRESS: at sentence #69040000, processed 1043311991 words, keeping 376822 word types\n",
      "2019-12-04 18:55:03,186 : INFO : PROGRESS: at sentence #69050000, processed 1043409784 words, keeping 376822 word types\n",
      "2019-12-04 18:55:03,316 : INFO : PROGRESS: at sentence #69060000, processed 1043511038 words, keeping 376822 word types\n",
      "2019-12-04 18:55:03,441 : INFO : PROGRESS: at sentence #69070000, processed 1043605609 words, keeping 376822 word types\n",
      "2019-12-04 18:55:03,572 : INFO : PROGRESS: at sentence #69080000, processed 1043702774 words, keeping 376822 word types\n",
      "2019-12-04 18:55:03,703 : INFO : PROGRESS: at sentence #69090000, processed 1043800857 words, keeping 376822 word types\n",
      "2019-12-04 18:55:03,842 : INFO : PROGRESS: at sentence #69100000, processed 1043903309 words, keeping 376822 word types\n",
      "2019-12-04 18:55:03,979 : INFO : PROGRESS: at sentence #69110000, processed 1044008691 words, keeping 376822 word types\n",
      "2019-12-04 18:55:04,116 : INFO : PROGRESS: at sentence #69120000, processed 1044113474 words, keeping 376822 word types\n",
      "2019-12-04 18:55:04,253 : INFO : PROGRESS: at sentence #69130000, processed 1044218037 words, keeping 376822 word types\n",
      "2019-12-04 18:55:04,394 : INFO : PROGRESS: at sentence #69140000, processed 1044323225 words, keeping 376822 word types\n",
      "2019-12-04 18:55:04,533 : INFO : PROGRESS: at sentence #69150000, processed 1044426862 words, keeping 376822 word types\n",
      "2019-12-04 18:55:04,669 : INFO : PROGRESS: at sentence #69160000, processed 1044527546 words, keeping 376822 word types\n",
      "2019-12-04 18:55:04,808 : INFO : PROGRESS: at sentence #69170000, processed 1044630125 words, keeping 376822 word types\n",
      "2019-12-04 18:55:04,963 : INFO : PROGRESS: at sentence #69180000, processed 1044747499 words, keeping 376822 word types\n",
      "2019-12-04 18:55:05,121 : INFO : PROGRESS: at sentence #69190000, processed 1044869449 words, keeping 376822 word types\n",
      "2019-12-04 18:55:05,271 : INFO : PROGRESS: at sentence #69200000, processed 1044984988 words, keeping 376822 word types\n",
      "2019-12-04 18:55:05,414 : INFO : PROGRESS: at sentence #69210000, processed 1045095544 words, keeping 376822 word types\n",
      "2019-12-04 18:55:05,550 : INFO : PROGRESS: at sentence #69220000, processed 1045200735 words, keeping 376822 word types\n",
      "2019-12-04 18:55:05,689 : INFO : PROGRESS: at sentence #69230000, processed 1045306175 words, keeping 376822 word types\n",
      "2019-12-04 18:55:05,831 : INFO : PROGRESS: at sentence #69240000, processed 1045414462 words, keeping 376822 word types\n",
      "2019-12-04 18:55:05,973 : INFO : PROGRESS: at sentence #69250000, processed 1045522343 words, keeping 376822 word types\n",
      "2019-12-04 18:55:06,115 : INFO : PROGRESS: at sentence #69260000, processed 1045629176 words, keeping 376822 word types\n",
      "2019-12-04 18:55:06,255 : INFO : PROGRESS: at sentence #69270000, processed 1045737206 words, keeping 376822 word types\n",
      "2019-12-04 18:55:06,396 : INFO : PROGRESS: at sentence #69280000, processed 1045846856 words, keeping 376822 word types\n",
      "2019-12-04 18:55:06,538 : INFO : PROGRESS: at sentence #69290000, processed 1045957238 words, keeping 376822 word types\n",
      "2019-12-04 18:55:06,688 : INFO : PROGRESS: at sentence #69300000, processed 1046073623 words, keeping 376822 word types\n",
      "2019-12-04 18:55:06,832 : INFO : PROGRESS: at sentence #69310000, processed 1046183541 words, keeping 376822 word types\n",
      "2019-12-04 18:55:06,981 : INFO : PROGRESS: at sentence #69320000, processed 1046298232 words, keeping 376822 word types\n",
      "2019-12-04 18:55:07,127 : INFO : PROGRESS: at sentence #69330000, processed 1046408176 words, keeping 376822 word types\n",
      "2019-12-04 18:55:07,290 : INFO : PROGRESS: at sentence #69340000, processed 1046533742 words, keeping 376822 word types\n",
      "2019-12-04 18:55:07,448 : INFO : PROGRESS: at sentence #69350000, processed 1046654036 words, keeping 376822 word types\n",
      "2019-12-04 18:55:07,608 : INFO : PROGRESS: at sentence #69360000, processed 1046775957 words, keeping 376822 word types\n",
      "2019-12-04 18:55:07,764 : INFO : PROGRESS: at sentence #69370000, processed 1046895042 words, keeping 376822 word types\n",
      "2019-12-04 18:55:07,919 : INFO : PROGRESS: at sentence #69380000, processed 1047014032 words, keeping 376822 word types\n",
      "2019-12-04 18:55:08,052 : INFO : PROGRESS: at sentence #69390000, processed 1047115028 words, keeping 376822 word types\n",
      "2019-12-04 18:55:08,182 : INFO : PROGRESS: at sentence #69400000, processed 1047214816 words, keeping 376822 word types\n",
      "2019-12-04 18:55:08,318 : INFO : PROGRESS: at sentence #69410000, processed 1047316719 words, keeping 376822 word types\n",
      "2019-12-04 18:55:08,453 : INFO : PROGRESS: at sentence #69420000, processed 1047421896 words, keeping 376822 word types\n",
      "2019-12-04 18:55:08,579 : INFO : PROGRESS: at sentence #69430000, processed 1047518126 words, keeping 376822 word types\n",
      "2019-12-04 18:55:08,703 : INFO : PROGRESS: at sentence #69440000, processed 1047615961 words, keeping 376822 word types\n",
      "2019-12-04 18:55:08,825 : INFO : PROGRESS: at sentence #69450000, processed 1047712183 words, keeping 376822 word types\n",
      "2019-12-04 18:55:08,950 : INFO : PROGRESS: at sentence #69460000, processed 1047808364 words, keeping 376822 word types\n",
      "2019-12-04 18:55:09,077 : INFO : PROGRESS: at sentence #69470000, processed 1047907736 words, keeping 376822 word types\n",
      "2019-12-04 18:55:09,208 : INFO : PROGRESS: at sentence #69480000, processed 1048012744 words, keeping 376822 word types\n",
      "2019-12-04 18:55:09,342 : INFO : PROGRESS: at sentence #69490000, processed 1048117908 words, keeping 376822 word types\n",
      "2019-12-04 18:55:09,475 : INFO : PROGRESS: at sentence #69500000, processed 1048222508 words, keeping 376822 word types\n",
      "2019-12-04 18:55:09,607 : INFO : PROGRESS: at sentence #69510000, processed 1048327360 words, keeping 376822 word types\n",
      "2019-12-04 18:55:09,745 : INFO : PROGRESS: at sentence #69520000, processed 1048433647 words, keeping 376822 word types\n",
      "2019-12-04 18:55:09,875 : INFO : PROGRESS: at sentence #69530000, processed 1048534133 words, keeping 376822 word types\n",
      "2019-12-04 18:55:10,007 : INFO : PROGRESS: at sentence #69540000, processed 1048636321 words, keeping 376822 word types\n",
      "2019-12-04 18:55:10,140 : INFO : PROGRESS: at sentence #69550000, processed 1048738699 words, keeping 376822 word types\n",
      "2019-12-04 18:55:10,302 : INFO : PROGRESS: at sentence #69560000, processed 1048866213 words, keeping 376822 word types\n",
      "2019-12-04 18:55:10,451 : INFO : PROGRESS: at sentence #69570000, processed 1048983143 words, keeping 376822 word types\n",
      "2019-12-04 18:55:10,596 : INFO : PROGRESS: at sentence #69580000, processed 1049095637 words, keeping 376822 word types\n",
      "2019-12-04 18:55:10,737 : INFO : PROGRESS: at sentence #69590000, processed 1049204500 words, keeping 376822 word types\n",
      "2019-12-04 18:55:10,869 : INFO : PROGRESS: at sentence #69600000, processed 1049310413 words, keeping 376822 word types\n",
      "2019-12-04 18:55:11,004 : INFO : PROGRESS: at sentence #69610000, processed 1049417660 words, keeping 376822 word types\n",
      "2019-12-04 18:55:11,137 : INFO : PROGRESS: at sentence #69620000, processed 1049524357 words, keeping 376822 word types\n",
      "2019-12-04 18:55:11,275 : INFO : PROGRESS: at sentence #69630000, processed 1049631684 words, keeping 376822 word types\n",
      "2019-12-04 18:55:11,413 : INFO : PROGRESS: at sentence #69640000, processed 1049738979 words, keeping 376822 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:55:11,552 : INFO : PROGRESS: at sentence #69650000, processed 1049847267 words, keeping 376822 word types\n",
      "2019-12-04 18:55:11,698 : INFO : PROGRESS: at sentence #69660000, processed 1049957891 words, keeping 376822 word types\n",
      "2019-12-04 18:55:11,858 : INFO : PROGRESS: at sentence #69670000, processed 1050070774 words, keeping 376822 word types\n",
      "2019-12-04 18:55:12,006 : INFO : PROGRESS: at sentence #69680000, processed 1050184259 words, keeping 376822 word types\n",
      "2019-12-04 18:55:12,157 : INFO : PROGRESS: at sentence #69690000, processed 1050296809 words, keeping 376822 word types\n",
      "2019-12-04 18:55:12,311 : INFO : PROGRESS: at sentence #69700000, processed 1050411244 words, keeping 376822 word types\n",
      "2019-12-04 18:55:12,466 : INFO : PROGRESS: at sentence #69710000, processed 1050527169 words, keeping 376822 word types\n",
      "2019-12-04 18:55:12,625 : INFO : PROGRESS: at sentence #69720000, processed 1050648700 words, keeping 376822 word types\n",
      "2019-12-04 18:55:12,781 : INFO : PROGRESS: at sentence #69730000, processed 1050769148 words, keeping 376822 word types\n",
      "2019-12-04 18:55:12,941 : INFO : PROGRESS: at sentence #69740000, processed 1050892662 words, keeping 376822 word types\n",
      "2019-12-04 18:55:13,090 : INFO : PROGRESS: at sentence #69750000, processed 1051008246 words, keeping 376822 word types\n",
      "2019-12-04 18:55:13,236 : INFO : PROGRESS: at sentence #69760000, processed 1051122740 words, keeping 376822 word types\n",
      "2019-12-04 18:55:13,365 : INFO : PROGRESS: at sentence #69770000, processed 1051222102 words, keeping 376822 word types\n",
      "2019-12-04 18:55:13,490 : INFO : PROGRESS: at sentence #69780000, processed 1051323209 words, keeping 376822 word types\n",
      "2019-12-04 18:55:13,617 : INFO : PROGRESS: at sentence #69790000, processed 1051425847 words, keeping 376822 word types\n",
      "2019-12-04 18:55:13,741 : INFO : PROGRESS: at sentence #69800000, processed 1051525661 words, keeping 376822 word types\n",
      "2019-12-04 18:55:13,865 : INFO : PROGRESS: at sentence #69810000, processed 1051625399 words, keeping 376822 word types\n",
      "2019-12-04 18:55:13,986 : INFO : PROGRESS: at sentence #69820000, processed 1051720171 words, keeping 376822 word types\n",
      "2019-12-04 18:55:14,108 : INFO : PROGRESS: at sentence #69830000, processed 1051818046 words, keeping 376822 word types\n",
      "2019-12-04 18:55:14,229 : INFO : PROGRESS: at sentence #69840000, processed 1051915092 words, keeping 376822 word types\n",
      "2019-12-04 18:55:14,357 : INFO : PROGRESS: at sentence #69850000, processed 1052016107 words, keeping 376822 word types\n",
      "2019-12-04 18:55:14,485 : INFO : PROGRESS: at sentence #69860000, processed 1052122440 words, keeping 376822 word types\n",
      "2019-12-04 18:55:14,614 : INFO : PROGRESS: at sentence #69870000, processed 1052226983 words, keeping 376822 word types\n",
      "2019-12-04 18:55:14,746 : INFO : PROGRESS: at sentence #69880000, processed 1052331279 words, keeping 376822 word types\n",
      "2019-12-04 18:55:14,893 : INFO : PROGRESS: at sentence #69890000, processed 1052436242 words, keeping 376822 word types\n",
      "2019-12-04 18:55:15,028 : INFO : PROGRESS: at sentence #69900000, processed 1052541507 words, keeping 376822 word types\n",
      "2019-12-04 18:55:15,161 : INFO : PROGRESS: at sentence #69910000, processed 1052641557 words, keeping 376822 word types\n",
      "2019-12-04 18:55:15,294 : INFO : PROGRESS: at sentence #69920000, processed 1052744544 words, keeping 376822 word types\n",
      "2019-12-04 18:55:15,436 : INFO : PROGRESS: at sentence #69930000, processed 1052854192 words, keeping 376822 word types\n",
      "2019-12-04 18:55:15,594 : INFO : PROGRESS: at sentence #69940000, processed 1052980844 words, keeping 376822 word types\n",
      "2019-12-04 18:55:15,741 : INFO : PROGRESS: at sentence #69950000, processed 1053096571 words, keeping 376822 word types\n",
      "2019-12-04 18:55:15,882 : INFO : PROGRESS: at sentence #69960000, processed 1053208038 words, keeping 376822 word types\n",
      "2019-12-04 18:55:16,019 : INFO : PROGRESS: at sentence #69970000, processed 1053313044 words, keeping 376822 word types\n",
      "2019-12-04 18:55:16,156 : INFO : PROGRESS: at sentence #69980000, processed 1053419100 words, keeping 376822 word types\n",
      "2019-12-04 18:55:16,299 : INFO : PROGRESS: at sentence #69990000, processed 1053527493 words, keeping 376822 word types\n",
      "2019-12-04 18:55:16,437 : INFO : PROGRESS: at sentence #70000000, processed 1053634750 words, keeping 376822 word types\n",
      "2019-12-04 18:55:16,573 : INFO : PROGRESS: at sentence #70010000, processed 1053741976 words, keeping 376822 word types\n",
      "2019-12-04 18:55:16,715 : INFO : PROGRESS: at sentence #70020000, processed 1053850540 words, keeping 376822 word types\n",
      "2019-12-04 18:55:16,857 : INFO : PROGRESS: at sentence #70030000, processed 1053958845 words, keeping 376822 word types\n",
      "2019-12-04 18:55:17,006 : INFO : PROGRESS: at sentence #70040000, processed 1054069223 words, keeping 376822 word types\n",
      "2019-12-04 18:55:17,157 : INFO : PROGRESS: at sentence #70050000, processed 1054185476 words, keeping 376822 word types\n",
      "2019-12-04 18:55:17,305 : INFO : PROGRESS: at sentence #70060000, processed 1054295593 words, keeping 376822 word types\n",
      "2019-12-04 18:55:17,457 : INFO : PROGRESS: at sentence #70070000, processed 1054409812 words, keeping 376822 word types\n",
      "2019-12-04 18:55:17,606 : INFO : PROGRESS: at sentence #70080000, processed 1054521189 words, keeping 376822 word types\n",
      "2019-12-04 18:55:17,769 : INFO : PROGRESS: at sentence #70090000, processed 1054643326 words, keeping 376822 word types\n",
      "2019-12-04 18:55:17,928 : INFO : PROGRESS: at sentence #70100000, processed 1054763283 words, keeping 376822 word types\n",
      "2019-12-04 18:55:18,087 : INFO : PROGRESS: at sentence #70110000, processed 1054884619 words, keeping 376822 word types\n",
      "2019-12-04 18:55:18,250 : INFO : PROGRESS: at sentence #70120000, processed 1055006751 words, keeping 376822 word types\n",
      "2019-12-04 18:55:18,404 : INFO : PROGRESS: at sentence #70130000, processed 1055123927 words, keeping 376822 word types\n",
      "2019-12-04 18:55:18,546 : INFO : PROGRESS: at sentence #70140000, processed 1055230056 words, keeping 376822 word types\n",
      "2019-12-04 18:55:18,679 : INFO : PROGRESS: at sentence #70150000, processed 1055328826 words, keeping 376822 word types\n",
      "2019-12-04 18:55:18,814 : INFO : PROGRESS: at sentence #70160000, processed 1055431092 words, keeping 376822 word types\n",
      "2019-12-04 18:55:18,950 : INFO : PROGRESS: at sentence #70170000, processed 1055534786 words, keeping 376822 word types\n",
      "2019-12-04 18:55:19,077 : INFO : PROGRESS: at sentence #70180000, processed 1055632192 words, keeping 376822 word types\n",
      "2019-12-04 18:55:19,205 : INFO : PROGRESS: at sentence #70190000, processed 1055731954 words, keeping 376822 word types\n",
      "2019-12-04 18:55:19,327 : INFO : PROGRESS: at sentence #70200000, processed 1055827205 words, keeping 376822 word types\n",
      "2019-12-04 18:55:19,455 : INFO : PROGRESS: at sentence #70210000, processed 1055923636 words, keeping 376822 word types\n",
      "2019-12-04 18:55:19,582 : INFO : PROGRESS: at sentence #70220000, processed 1056022506 words, keeping 376822 word types\n",
      "2019-12-04 18:55:19,726 : INFO : PROGRESS: at sentence #70230000, processed 1056125801 words, keeping 376822 word types\n",
      "2019-12-04 18:55:19,865 : INFO : PROGRESS: at sentence #70240000, processed 1056231386 words, keeping 376822 word types\n",
      "2019-12-04 18:55:20,002 : INFO : PROGRESS: at sentence #70250000, processed 1056335804 words, keeping 376822 word types\n",
      "2019-12-04 18:55:20,137 : INFO : PROGRESS: at sentence #70260000, processed 1056440921 words, keeping 376822 word types\n",
      "2019-12-04 18:55:20,276 : INFO : PROGRESS: at sentence #70270000, processed 1056546611 words, keeping 376822 word types\n",
      "2019-12-04 18:55:20,410 : INFO : PROGRESS: at sentence #70280000, processed 1056648581 words, keeping 376822 word types\n",
      "2019-12-04 18:55:20,545 : INFO : PROGRESS: at sentence #70290000, processed 1056750300 words, keeping 376822 word types\n",
      "2019-12-04 18:55:20,680 : INFO : PROGRESS: at sentence #70300000, processed 1056851938 words, keeping 376822 word types\n",
      "2019-12-04 18:55:20,836 : INFO : PROGRESS: at sentence #70310000, processed 1056973509 words, keeping 376822 word types\n",
      "2019-12-04 18:55:20,991 : INFO : PROGRESS: at sentence #70320000, processed 1057093749 words, keeping 376822 word types\n",
      "2019-12-04 18:55:21,139 : INFO : PROGRESS: at sentence #70330000, processed 1057208116 words, keeping 376822 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:55:21,284 : INFO : PROGRESS: at sentence #70340000, processed 1057318191 words, keeping 376822 word types\n",
      "2019-12-04 18:55:21,421 : INFO : PROGRESS: at sentence #70350000, processed 1057423961 words, keeping 376822 word types\n",
      "2019-12-04 18:55:21,562 : INFO : PROGRESS: at sentence #70360000, processed 1057530304 words, keeping 376822 word types\n",
      "2019-12-04 18:55:21,708 : INFO : PROGRESS: at sentence #70370000, processed 1057637431 words, keeping 376822 word types\n",
      "2019-12-04 18:55:21,853 : INFO : PROGRESS: at sentence #70380000, processed 1057744909 words, keeping 376822 word types\n",
      "2019-12-04 18:55:21,989 : INFO : PROGRESS: at sentence #70390000, processed 1057851641 words, keeping 376822 word types\n",
      "2019-12-04 18:55:22,127 : INFO : PROGRESS: at sentence #70400000, processed 1057960530 words, keeping 376822 word types\n",
      "2019-12-04 18:55:22,266 : INFO : PROGRESS: at sentence #70410000, processed 1058070386 words, keeping 376822 word types\n",
      "2019-12-04 18:55:22,408 : INFO : PROGRESS: at sentence #70420000, processed 1058181152 words, keeping 376822 word types\n",
      "2019-12-04 18:55:22,559 : INFO : PROGRESS: at sentence #70430000, processed 1058297088 words, keeping 376822 word types\n",
      "2019-12-04 18:55:22,703 : INFO : PROGRESS: at sentence #70440000, processed 1058407666 words, keeping 376822 word types\n",
      "2019-12-04 18:55:22,858 : INFO : PROGRESS: at sentence #70450000, processed 1058522530 words, keeping 376822 word types\n",
      "2019-12-04 18:55:23,008 : INFO : PROGRESS: at sentence #70460000, processed 1058631007 words, keeping 376822 word types\n",
      "2019-12-04 18:55:23,179 : INFO : PROGRESS: at sentence #70470000, processed 1058758809 words, keeping 376822 word types\n",
      "2019-12-04 18:55:23,343 : INFO : PROGRESS: at sentence #70480000, processed 1058879600 words, keeping 376822 word types\n",
      "2019-12-04 18:55:23,505 : INFO : PROGRESS: at sentence #70490000, processed 1059001469 words, keeping 376822 word types\n",
      "2019-12-04 18:55:23,662 : INFO : PROGRESS: at sentence #70500000, processed 1059119210 words, keeping 376822 word types\n",
      "2019-12-04 18:55:23,815 : INFO : PROGRESS: at sentence #70510000, processed 1059238017 words, keeping 376822 word types\n",
      "2019-12-04 18:55:23,943 : INFO : PROGRESS: at sentence #70520000, processed 1059336745 words, keeping 376822 word types\n",
      "2019-12-04 18:55:24,072 : INFO : PROGRESS: at sentence #70530000, processed 1059437070 words, keeping 376822 word types\n",
      "2019-12-04 18:55:24,206 : INFO : PROGRESS: at sentence #70540000, processed 1059538881 words, keeping 376822 word types\n",
      "2019-12-04 18:55:24,351 : INFO : PROGRESS: at sentence #70550000, processed 1059644235 words, keeping 376822 word types\n",
      "2019-12-04 18:55:24,476 : INFO : PROGRESS: at sentence #70560000, processed 1059739451 words, keeping 376822 word types\n",
      "2019-12-04 18:55:24,603 : INFO : PROGRESS: at sentence #70570000, processed 1059836375 words, keeping 376822 word types\n",
      "2019-12-04 18:55:24,724 : INFO : PROGRESS: at sentence #70580000, processed 1059933771 words, keeping 376822 word types\n",
      "2019-12-04 18:55:24,848 : INFO : PROGRESS: at sentence #70590000, processed 1060029926 words, keeping 376822 word types\n",
      "2019-12-04 18:55:24,980 : INFO : PROGRESS: at sentence #70600000, processed 1060129179 words, keeping 376822 word types\n",
      "2019-12-04 18:55:25,120 : INFO : PROGRESS: at sentence #70610000, processed 1060235442 words, keeping 376822 word types\n",
      "2019-12-04 18:55:25,260 : INFO : PROGRESS: at sentence #70620000, processed 1060340297 words, keeping 376822 word types\n",
      "2019-12-04 18:55:25,401 : INFO : PROGRESS: at sentence #70630000, processed 1060444351 words, keeping 376822 word types\n",
      "2019-12-04 18:55:25,540 : INFO : PROGRESS: at sentence #70640000, processed 1060549720 words, keeping 376822 word types\n",
      "2019-12-04 18:55:25,682 : INFO : PROGRESS: at sentence #70650000, processed 1060655598 words, keeping 376822 word types\n",
      "2019-12-04 18:55:25,816 : INFO : PROGRESS: at sentence #70660000, processed 1060755670 words, keeping 376822 word types\n",
      "2019-12-04 18:55:25,953 : INFO : PROGRESS: at sentence #70670000, processed 1060858497 words, keeping 376822 word types\n",
      "2019-12-04 18:55:26,094 : INFO : PROGRESS: at sentence #70680000, processed 1060961427 words, keeping 376822 word types\n",
      "2019-12-04 18:55:26,261 : INFO : PROGRESS: at sentence #70690000, processed 1061091563 words, keeping 376822 word types\n",
      "2019-12-04 18:55:26,408 : INFO : PROGRESS: at sentence #70700000, processed 1061206891 words, keeping 376822 word types\n",
      "2019-12-04 18:55:26,550 : INFO : PROGRESS: at sentence #70710000, processed 1061319450 words, keeping 376822 word types\n",
      "2019-12-04 18:55:26,693 : INFO : PROGRESS: at sentence #70720000, processed 1061426735 words, keeping 376822 word types\n",
      "2019-12-04 18:55:26,834 : INFO : PROGRESS: at sentence #70730000, processed 1061532974 words, keeping 376822 word types\n",
      "2019-12-04 18:55:26,977 : INFO : PROGRESS: at sentence #70740000, processed 1061640593 words, keeping 376822 word types\n",
      "2019-12-04 18:55:27,118 : INFO : PROGRESS: at sentence #70750000, processed 1061747007 words, keeping 376822 word types\n",
      "2019-12-04 18:55:27,258 : INFO : PROGRESS: at sentence #70760000, processed 1061854655 words, keeping 376822 word types\n",
      "2019-12-04 18:55:27,404 : INFO : PROGRESS: at sentence #70770000, processed 1061962325 words, keeping 376822 word types\n",
      "2019-12-04 18:55:27,548 : INFO : PROGRESS: at sentence #70780000, processed 1062070871 words, keeping 376822 word types\n",
      "2019-12-04 18:55:27,692 : INFO : PROGRESS: at sentence #70790000, processed 1062181322 words, keeping 376822 word types\n",
      "2019-12-04 18:55:27,843 : INFO : PROGRESS: at sentence #70800000, processed 1062295984 words, keeping 376822 word types\n",
      "2019-12-04 18:55:27,991 : INFO : PROGRESS: at sentence #70810000, processed 1062408126 words, keeping 376822 word types\n",
      "2019-12-04 18:55:28,141 : INFO : PROGRESS: at sentence #70820000, processed 1062520620 words, keeping 376822 word types\n",
      "2019-12-04 18:55:28,290 : INFO : PROGRESS: at sentence #70830000, processed 1062634213 words, keeping 376822 word types\n",
      "2019-12-04 18:55:28,448 : INFO : PROGRESS: at sentence #70840000, processed 1062752968 words, keeping 376822 word types\n",
      "2019-12-04 18:55:28,607 : INFO : PROGRESS: at sentence #70850000, processed 1062873171 words, keeping 376822 word types\n",
      "2019-12-04 18:55:28,765 : INFO : PROGRESS: at sentence #70860000, processed 1062994270 words, keeping 376822 word types\n",
      "2019-12-04 18:55:28,905 : INFO : collected 376822 word types from a corpus of 1063101444 raw words and 70868669 sentences\n",
      "2019-12-04 18:55:28,906 : INFO : Loading a fresh vocabulary\n",
      "2019-12-04 18:55:29,858 : INFO : effective_min_count=5 retains 207501 unique words (55% of original 376822, drops 169321)\n",
      "2019-12-04 18:55:29,859 : INFO : effective_min_count=5 leaves 1013813420 word corpus (99% of original 1014041061, drops 227641)\n",
      "2019-12-04 18:55:30,667 : INFO : deleting the raw counts dictionary of 376822 items\n",
      "2019-12-04 18:55:30,689 : INFO : sample=0.001 downsamples 53 most-common words\n",
      "2019-12-04 18:55:30,690 : INFO : downsampling leaves estimated 782537507 word corpus (77.2% of prior 1013813420)\n",
      "2019-12-04 18:55:59,317 : INFO : estimated required memory for 207501 words, 630830 buckets and 300 dimensions: 1424477292 bytes\n",
      "2019-12-04 18:55:59,345 : INFO : resetting layer weights\n",
      "2019-12-04 18:57:02,900 : INFO : Total number of ngrams is 630830\n",
      "2019-12-04 18:57:14,138 : INFO : training model with 8 workers on 207501 vocabulary and 300 features, using sg=1 hs=0 sample=0.001 negative=5 window=4\n",
      "2019-12-04 18:57:14,144 : WARNING : train() called with an empty iterator (if not intended, be sure to provide a corpus that offers restartable iteration = an iterable).\n",
      "2019-12-04 18:57:14,146 : INFO : worker thread finished; awaiting finish of 7 more threads\n",
      "2019-12-04 18:57:14,147 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2019-12-04 18:57:14,148 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2019-12-04 18:57:14,148 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2019-12-04 18:57:14,149 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2019-12-04 18:57:14,149 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2019-12-04 18:57:14,150 : INFO : worker thread finished; awaiting finish of 1 more threads\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:57:14,151 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2019-12-04 18:57:14,151 : INFO : EPOCH - 1 : training on 0 raw words (0 effective words) took 0.0s, 0 effective words/s\n",
      "2019-12-04 18:57:14,152 : WARNING : EPOCH - 1 : supplied example count (0) did not equal expected count (70868669)\n",
      "2019-12-04 18:57:14,156 : WARNING : train() called with an empty iterator (if not intended, be sure to provide a corpus that offers restartable iteration = an iterable).\n",
      "2019-12-04 18:57:14,158 : INFO : worker thread finished; awaiting finish of 7 more threads\n",
      "2019-12-04 18:57:14,158 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2019-12-04 18:57:14,159 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2019-12-04 18:57:14,159 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2019-12-04 18:57:14,160 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2019-12-04 18:57:14,160 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2019-12-04 18:57:14,161 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2019-12-04 18:57:14,162 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2019-12-04 18:57:14,162 : INFO : EPOCH - 2 : training on 0 raw words (0 effective words) took 0.0s, 0 effective words/s\n",
      "2019-12-04 18:57:14,163 : WARNING : EPOCH - 2 : supplied example count (0) did not equal expected count (70868669)\n",
      "2019-12-04 18:57:14,166 : WARNING : train() called with an empty iterator (if not intended, be sure to provide a corpus that offers restartable iteration = an iterable).\n",
      "2019-12-04 18:57:14,167 : INFO : worker thread finished; awaiting finish of 7 more threads\n",
      "2019-12-04 18:57:14,168 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2019-12-04 18:57:14,168 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2019-12-04 18:57:14,169 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2019-12-04 18:57:14,170 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2019-12-04 18:57:14,170 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2019-12-04 18:57:14,171 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2019-12-04 18:57:14,171 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2019-12-04 18:57:14,172 : INFO : EPOCH - 3 : training on 0 raw words (0 effective words) took 0.0s, 0 effective words/s\n",
      "2019-12-04 18:57:14,172 : WARNING : EPOCH - 3 : supplied example count (0) did not equal expected count (70868669)\n",
      "2019-12-04 18:57:14,176 : WARNING : train() called with an empty iterator (if not intended, be sure to provide a corpus that offers restartable iteration = an iterable).\n",
      "2019-12-04 18:57:14,178 : INFO : worker thread finished; awaiting finish of 7 more threads\n",
      "2019-12-04 18:57:14,179 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2019-12-04 18:57:14,180 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2019-12-04 18:57:14,180 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2019-12-04 18:57:14,181 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2019-12-04 18:57:14,182 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2019-12-04 18:57:14,183 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2019-12-04 18:57:14,184 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2019-12-04 18:57:14,184 : INFO : EPOCH - 4 : training on 0 raw words (0 effective words) took 0.0s, 0 effective words/s\n",
      "2019-12-04 18:57:14,185 : WARNING : EPOCH - 4 : supplied example count (0) did not equal expected count (70868669)\n",
      "2019-12-04 18:57:14,189 : WARNING : train() called with an empty iterator (if not intended, be sure to provide a corpus that offers restartable iteration = an iterable).\n",
      "2019-12-04 18:57:14,191 : INFO : worker thread finished; awaiting finish of 7 more threads\n",
      "2019-12-04 18:57:14,192 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2019-12-04 18:57:14,192 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2019-12-04 18:57:14,193 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2019-12-04 18:57:14,193 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2019-12-04 18:57:14,194 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2019-12-04 18:57:14,194 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2019-12-04 18:57:14,195 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2019-12-04 18:57:14,195 : INFO : EPOCH - 5 : training on 0 raw words (0 effective words) took 0.0s, 0 effective words/s\n",
      "2019-12-04 18:57:14,196 : WARNING : EPOCH - 5 : supplied example count (0) did not equal expected count (70868669)\n",
      "2019-12-04 18:57:14,201 : WARNING : train() called with an empty iterator (if not intended, be sure to provide a corpus that offers restartable iteration = an iterable).\n",
      "2019-12-04 18:57:14,203 : INFO : worker thread finished; awaiting finish of 7 more threads\n",
      "2019-12-04 18:57:14,203 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2019-12-04 18:57:14,204 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2019-12-04 18:57:14,204 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2019-12-04 18:57:14,205 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2019-12-04 18:57:14,206 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2019-12-04 18:57:14,206 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2019-12-04 18:57:14,207 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2019-12-04 18:57:14,207 : INFO : EPOCH - 6 : training on 0 raw words (0 effective words) took 0.0s, 0 effective words/s\n",
      "2019-12-04 18:57:14,208 : WARNING : EPOCH - 6 : supplied example count (0) did not equal expected count (70868669)\n",
      "2019-12-04 18:57:14,214 : WARNING : train() called with an empty iterator (if not intended, be sure to provide a corpus that offers restartable iteration = an iterable).\n",
      "2019-12-04 18:57:14,215 : INFO : worker thread finished; awaiting finish of 7 more threads\n",
      "2019-12-04 18:57:14,216 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2019-12-04 18:57:14,216 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2019-12-04 18:57:14,217 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2019-12-04 18:57:14,218 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2019-12-04 18:57:14,218 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2019-12-04 18:57:14,219 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2019-12-04 18:57:14,219 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2019-12-04 18:57:14,220 : INFO : EPOCH - 7 : training on 0 raw words (0 effective words) took 0.0s, 0 effective words/s\n",
      "2019-12-04 18:57:14,220 : WARNING : EPOCH - 7 : supplied example count (0) did not equal expected count (70868669)\n",
      "2019-12-04 18:57:14,226 : WARNING : train() called with an empty iterator (if not intended, be sure to provide a corpus that offers restartable iteration = an iterable).\n",
      "2019-12-04 18:57:14,227 : INFO : worker thread finished; awaiting finish of 7 more threads\n",
      "2019-12-04 18:57:14,227 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2019-12-04 18:57:14,228 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2019-12-04 18:57:14,229 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2019-12-04 18:57:14,229 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2019-12-04 18:57:14,230 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2019-12-04 18:57:14,230 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2019-12-04 18:57:14,231 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2019-12-04 18:57:14,231 : INFO : EPOCH - 8 : training on 0 raw words (0 effective words) took 0.0s, 0 effective words/s\n",
      "2019-12-04 18:57:14,232 : WARNING : EPOCH - 8 : supplied example count (0) did not equal expected count (70868669)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:57:14,238 : WARNING : train() called with an empty iterator (if not intended, be sure to provide a corpus that offers restartable iteration = an iterable).\n",
      "2019-12-04 18:57:14,239 : INFO : worker thread finished; awaiting finish of 7 more threads\n",
      "2019-12-04 18:57:14,240 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2019-12-04 18:57:14,240 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2019-12-04 18:57:14,241 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2019-12-04 18:57:14,241 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2019-12-04 18:57:14,242 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2019-12-04 18:57:14,242 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2019-12-04 18:57:14,243 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2019-12-04 18:57:14,244 : INFO : EPOCH - 9 : training on 0 raw words (0 effective words) took 0.0s, 0 effective words/s\n",
      "2019-12-04 18:57:14,244 : WARNING : EPOCH - 9 : supplied example count (0) did not equal expected count (70868669)\n",
      "2019-12-04 18:57:14,249 : WARNING : train() called with an empty iterator (if not intended, be sure to provide a corpus that offers restartable iteration = an iterable).\n",
      "2019-12-04 18:57:14,250 : INFO : worker thread finished; awaiting finish of 7 more threads\n",
      "2019-12-04 18:57:14,251 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2019-12-04 18:57:14,251 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2019-12-04 18:57:14,252 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2019-12-04 18:57:14,252 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2019-12-04 18:57:14,253 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2019-12-04 18:57:14,253 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2019-12-04 18:57:14,254 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2019-12-04 18:57:14,254 : INFO : EPOCH - 10 : training on 0 raw words (0 effective words) took 0.0s, 0 effective words/s\n",
      "2019-12-04 18:57:14,255 : WARNING : EPOCH - 10 : supplied example count (0) did not equal expected count (70868669)\n",
      "2019-12-04 18:57:14,260 : WARNING : train() called with an empty iterator (if not intended, be sure to provide a corpus that offers restartable iteration = an iterable).\n",
      "2019-12-04 18:57:14,261 : INFO : worker thread finished; awaiting finish of 7 more threads\n",
      "2019-12-04 18:57:14,262 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2019-12-04 18:57:14,262 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2019-12-04 18:57:14,263 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2019-12-04 18:57:14,263 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2019-12-04 18:57:14,264 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2019-12-04 18:57:14,264 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2019-12-04 18:57:14,265 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2019-12-04 18:57:14,265 : INFO : EPOCH - 11 : training on 0 raw words (0 effective words) took 0.0s, 0 effective words/s\n",
      "2019-12-04 18:57:14,266 : WARNING : EPOCH - 11 : supplied example count (0) did not equal expected count (70868669)\n",
      "2019-12-04 18:57:14,272 : WARNING : train() called with an empty iterator (if not intended, be sure to provide a corpus that offers restartable iteration = an iterable).\n",
      "2019-12-04 18:57:14,273 : INFO : worker thread finished; awaiting finish of 7 more threads\n",
      "2019-12-04 18:57:14,274 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2019-12-04 18:57:14,275 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2019-12-04 18:57:14,275 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2019-12-04 18:57:14,276 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2019-12-04 18:57:14,276 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2019-12-04 18:57:14,277 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2019-12-04 18:57:14,277 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2019-12-04 18:57:14,278 : INFO : EPOCH - 12 : training on 0 raw words (0 effective words) took 0.0s, 0 effective words/s\n",
      "2019-12-04 18:57:14,278 : WARNING : EPOCH - 12 : supplied example count (0) did not equal expected count (70868669)\n",
      "2019-12-04 18:57:14,284 : WARNING : train() called with an empty iterator (if not intended, be sure to provide a corpus that offers restartable iteration = an iterable).\n",
      "2019-12-04 18:57:14,286 : INFO : worker thread finished; awaiting finish of 7 more threads\n",
      "2019-12-04 18:57:14,287 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2019-12-04 18:57:14,287 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2019-12-04 18:57:14,288 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2019-12-04 18:57:14,288 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2019-12-04 18:57:14,289 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2019-12-04 18:57:14,289 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2019-12-04 18:57:14,290 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2019-12-04 18:57:14,290 : INFO : EPOCH - 13 : training on 0 raw words (0 effective words) took 0.0s, 0 effective words/s\n",
      "2019-12-04 18:57:14,291 : WARNING : EPOCH - 13 : supplied example count (0) did not equal expected count (70868669)\n",
      "2019-12-04 18:57:14,296 : WARNING : train() called with an empty iterator (if not intended, be sure to provide a corpus that offers restartable iteration = an iterable).\n",
      "2019-12-04 18:57:14,298 : INFO : worker thread finished; awaiting finish of 7 more threads\n",
      "2019-12-04 18:57:14,298 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2019-12-04 18:57:14,299 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2019-12-04 18:57:14,300 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2019-12-04 18:57:14,300 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2019-12-04 18:57:14,301 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2019-12-04 18:57:14,302 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2019-12-04 18:57:14,303 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2019-12-04 18:57:14,303 : INFO : EPOCH - 14 : training on 0 raw words (0 effective words) took 0.0s, 0 effective words/s\n",
      "2019-12-04 18:57:14,304 : WARNING : EPOCH - 14 : supplied example count (0) did not equal expected count (70868669)\n",
      "2019-12-04 18:57:14,310 : WARNING : train() called with an empty iterator (if not intended, be sure to provide a corpus that offers restartable iteration = an iterable).\n",
      "2019-12-04 18:57:14,311 : INFO : worker thread finished; awaiting finish of 7 more threads\n",
      "2019-12-04 18:57:14,312 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2019-12-04 18:57:14,313 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2019-12-04 18:57:14,313 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2019-12-04 18:57:14,314 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2019-12-04 18:57:14,314 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2019-12-04 18:57:14,315 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2019-12-04 18:57:14,315 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2019-12-04 18:57:14,316 : INFO : EPOCH - 15 : training on 0 raw words (0 effective words) took 0.0s, 0 effective words/s\n",
      "2019-12-04 18:57:14,316 : WARNING : EPOCH - 15 : supplied example count (0) did not equal expected count (70868669)\n",
      "2019-12-04 18:57:14,320 : WARNING : train() called with an empty iterator (if not intended, be sure to provide a corpus that offers restartable iteration = an iterable).\n",
      "2019-12-04 18:57:14,322 : INFO : worker thread finished; awaiting finish of 7 more threads\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:57:14,322 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2019-12-04 18:57:14,323 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2019-12-04 18:57:14,323 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2019-12-04 18:57:14,324 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2019-12-04 18:57:14,324 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2019-12-04 18:57:14,325 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2019-12-04 18:57:14,326 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2019-12-04 18:57:14,326 : INFO : EPOCH - 16 : training on 0 raw words (0 effective words) took 0.0s, 0 effective words/s\n",
      "2019-12-04 18:57:14,327 : WARNING : EPOCH - 16 : supplied example count (0) did not equal expected count (70868669)\n",
      "2019-12-04 18:57:14,331 : WARNING : train() called with an empty iterator (if not intended, be sure to provide a corpus that offers restartable iteration = an iterable).\n",
      "2019-12-04 18:57:14,333 : INFO : worker thread finished; awaiting finish of 7 more threads\n",
      "2019-12-04 18:57:14,333 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2019-12-04 18:57:14,334 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2019-12-04 18:57:14,334 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2019-12-04 18:57:14,335 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2019-12-04 18:57:14,335 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2019-12-04 18:57:14,336 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2019-12-04 18:57:14,336 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2019-12-04 18:57:14,337 : INFO : EPOCH - 17 : training on 0 raw words (0 effective words) took 0.0s, 0 effective words/s\n",
      "2019-12-04 18:57:14,337 : WARNING : EPOCH - 17 : supplied example count (0) did not equal expected count (70868669)\n",
      "2019-12-04 18:57:14,341 : WARNING : train() called with an empty iterator (if not intended, be sure to provide a corpus that offers restartable iteration = an iterable).\n",
      "2019-12-04 18:57:14,343 : INFO : worker thread finished; awaiting finish of 7 more threads\n",
      "2019-12-04 18:57:14,343 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2019-12-04 18:57:14,344 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2019-12-04 18:57:14,344 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2019-12-04 18:57:14,345 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2019-12-04 18:57:14,345 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2019-12-04 18:57:14,346 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2019-12-04 18:57:14,346 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2019-12-04 18:57:14,347 : INFO : EPOCH - 18 : training on 0 raw words (0 effective words) took 0.0s, 0 effective words/s\n",
      "2019-12-04 18:57:14,347 : WARNING : EPOCH - 18 : supplied example count (0) did not equal expected count (70868669)\n",
      "2019-12-04 18:57:14,352 : WARNING : train() called with an empty iterator (if not intended, be sure to provide a corpus that offers restartable iteration = an iterable).\n",
      "2019-12-04 18:57:14,354 : INFO : worker thread finished; awaiting finish of 7 more threads\n",
      "2019-12-04 18:57:14,354 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2019-12-04 18:57:14,355 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2019-12-04 18:57:14,355 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2019-12-04 18:57:14,356 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2019-12-04 18:57:14,356 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2019-12-04 18:57:14,357 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2019-12-04 18:57:14,357 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2019-12-04 18:57:14,358 : INFO : EPOCH - 19 : training on 0 raw words (0 effective words) took 0.0s, 0 effective words/s\n",
      "2019-12-04 18:57:14,359 : WARNING : EPOCH - 19 : supplied example count (0) did not equal expected count (70868669)\n",
      "2019-12-04 18:57:14,364 : WARNING : train() called with an empty iterator (if not intended, be sure to provide a corpus that offers restartable iteration = an iterable).\n",
      "2019-12-04 18:57:14,365 : INFO : worker thread finished; awaiting finish of 7 more threads\n",
      "2019-12-04 18:57:14,366 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2019-12-04 18:57:14,366 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2019-12-04 18:57:14,367 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2019-12-04 18:57:14,367 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2019-12-04 18:57:14,368 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2019-12-04 18:57:14,368 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2019-12-04 18:57:14,369 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2019-12-04 18:57:14,369 : INFO : EPOCH - 20 : training on 0 raw words (0 effective words) took 0.0s, 0 effective words/s\n",
      "2019-12-04 18:57:14,370 : WARNING : EPOCH - 20 : supplied example count (0) did not equal expected count (70868669)\n",
      "2019-12-04 18:57:14,371 : INFO : training on a 0 raw words (0 effective words) took 0.2s, 0 effective words/s\n",
      "2019-12-04 18:57:14,371 : WARNING : under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
      "2019-12-04 18:57:34,067 : INFO : saving FastText object under /opt/apps/cda/ez/sb_sandboxpuk/sandboxpuk_scripte/wiki/wiki_ft_300.model, separately None\n",
      "2019-12-04 18:57:34,068 : INFO : storing np array 'vectors' to /opt/apps/cda/ez/sb_sandboxpuk/sandboxpuk_scripte/wiki/wiki_ft_300.model.wv.vectors.npy\n",
      "2019-12-04 18:57:34,383 : INFO : storing np array 'vectors_vocab' to /opt/apps/cda/ez/sb_sandboxpuk/sandboxpuk_scripte/wiki/wiki_ft_300.model.wv.vectors_vocab.npy\n",
      "2019-12-04 18:57:34,700 : INFO : storing np array 'vectors_ngrams' to /opt/apps/cda/ez/sb_sandboxpuk/sandboxpuk_scripte/wiki/wiki_ft_300.model.wv.vectors_ngrams.npy\n",
      "2019-12-04 18:57:35,923 : INFO : not storing attribute vectors_norm\n",
      "2019-12-04 18:57:35,925 : INFO : not storing attribute vectors_vocab_norm\n",
      "2019-12-04 18:57:35,926 : INFO : not storing attribute vectors_ngrams_norm\n",
      "2019-12-04 18:57:35,926 : INFO : not storing attribute buckets_word\n",
      "2019-12-04 18:57:35,927 : INFO : storing np array 'syn1neg' to /opt/apps/cda/ez/sb_sandboxpuk/sandboxpuk_scripte/wiki/wiki_ft_300.model.trainables.syn1neg.npy\n",
      "2019-12-04 18:57:36,291 : INFO : storing np array 'vectors_vocab_lockf' to /opt/apps/cda/ez/sb_sandboxpuk/sandboxpuk_scripte/wiki/wiki_ft_300.model.trainables.vectors_vocab_lockf.npy\n",
      "2019-12-04 18:57:36,786 : INFO : storing np array 'vectors_ngrams_lockf' to /opt/apps/cda/ez/sb_sandboxpuk/sandboxpuk_scripte/wiki/wiki_ft_300.model.trainables.vectors_ngrams_lockf.npy\n",
      "2019-12-04 18:57:38,757 : INFO : saved /opt/apps/cda/ez/sb_sandboxpuk/sandboxpuk_scripte/wiki/wiki_ft_300.model\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 25min 30s, sys: 23.9 s, total: 25min 54s\n",
      "Wall time: 25min 43s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "from gensim.models import FastText\n",
    "name = 'wiki_ft_300.model'\n",
    "#sentences = txt_wiki_all()\n",
    "\n",
    "params = {'size': 300, 'window': 4, 'iter':100,'word_ngrams':1,'sg':1,'max_vocab_size':500000,\n",
    "          'workers': max(1, 8 #multiprocessing.cpu_count() - 1\n",
    "                        ), 'sample': 1e-3,}\n",
    "#ft = FastText(sentences = txt_wiki_all(), **params)\n",
    "\n",
    "ft = FastText(**params)\n",
    "ft.build_vocab(txt_wiki_all())\n",
    "ft.train(txt_wiki_all(), total_examples=ft.corpus_count, epochs=20)\n",
    "\n",
    "ft.save(path2 + name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gensim.models import FastText as ft\n",
    "ft = ft.load('data/wiki_ft_300.model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-04 18:57:38,765 : INFO : precomputing L2-norms of word weight vectors\n",
      "2019-12-04 18:57:39,430 : INFO : precomputing L2-norms of ngram weight vectors\n",
      "/opt/apps/anaconda3/lib/python3.6/site-packages/gensim/matutils.py:737: FutureWarning: Conversion of the second argument of issubdtype from `int` to `np.signedinteger` is deprecated. In future, it will be treated as `np.int64 == np.dtype(int).type`.\n",
      "  if np.issubdtype(vec.dtype, np.int):\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('kkarte', 0.6204562783241272),\n",
       " ('eckarte', 0.5813156366348267),\n",
       " ('kartei', 0.5756651759147644),\n",
       " ('karten', 0.5659630298614502),\n",
       " ('ihrekarte', 0.5561856031417847),\n",
       " ('karteikarten', 0.536436140537262),\n",
       " ('zahlkarte', 0.5272395014762878),\n",
       " ('neuekarte', 0.5252927541732788),\n",
       " ('bahnkarte', 0.5089269876480103),\n",
       " ('seekarte', 0.5052211284637451)]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ft.wv.most_similar('karte')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/apps/anaconda3/lib/python3.6/site-packages/ipykernel/__main__.py:1: DeprecationWarning: Call to deprecated `most_similar` (Method will be removed in 4.0.0, use self.wv.most_similar() instead).\n",
      "  if __name__ == '__main__':\n",
      "/opt/apps/anaconda3/lib/python3.6/site-packages/gensim/matutils.py:737: FutureWarning: Conversion of the second argument of issubdtype from `int` to `np.signedinteger` is deprecated. In future, it will be treated as `np.int64 == np.dtype(int).type`.\n",
      "  if np.issubdtype(vec.dtype, np.int):\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('königin', 0.39161479473114014),\n",
       " ('königsmarck', 0.3716628849506378),\n",
       " ('könige', 0.36920976638793945),\n",
       " ('königl', 0.36845505237579346),\n",
       " ('königen', 0.3553083539009094),\n",
       " ('königs', 0.3526017963886261),\n",
       " ('königsplatz', 0.3464611768722534),\n",
       " ('könemann', 0.3417457044124603),\n",
       " ('königsgut', 0.34161829948425293),\n",
       " ('königsmacher', 0.3284785747528076)]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ft.most_similar(positive=['mann', 'könig'], negative=['frau'], topn=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_vectors = word2vec.wv\n",
    "\n",
    "from gensim.test.utils import get_tmpfile\n",
    "from gensim.models import KeyedVectors\n",
    "\n",
    "fname = get_tmpfile(\"vectors.kv\")\n",
    "word_vectors.save(fname)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_vectors = KeyedVectors.load(fname, mmap='r')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "path2 = '/opt/apps/cda/ez/sb_sandboxpuk/sandboxpuk_scripte/wiki/data/'\n",
    "\n",
    "with open (path2+'wiki_.txt', 'rt') as myfile:  # Open file lorem.txt for reading text\n",
    "    i=1\n",
    "    a = []\n",
    "    for myline in myfile:                 # For each line, read it to a string \n",
    "        i+=1\n",
    "        a.append(myline)\n",
    "        if i < 6:\n",
    "            print(myline)  \n",
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a[-8]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "PySpark with Python Version 3.6",
   "language": "python",
   "name": "pyspark"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}