examples/chained_augmentation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Two augmentation chainer classes are currently available\n",
    "\n",
    "This are:\n",
    "* AugmentationStreamer\n",
    "* CallOnOneChain\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_First we import some augmentation classes and define a test corpus_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from seaqube.augmentation.char.qwerty import QwertyAugmentation\n",
    "from seaqube.augmentation.corpus.unigram import UnigramAugmentation\n",
    "from seaqube.augmentation.word.active2passive import Active2PassiveAugmentation\n",
    "from seaqube.augmentation.word.eda import EDAAugmentation\n",
    "from seaqube.augmentation.word.embedding import EmbeddingAugmentation\n",
    "from seaqube.augmentation.word.translation import TranslationAugmentation\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "TEST_CORPUS = [['till', 'this', 'moment', 'i', 'never', 'knew', 'myself', '.'],\n",
    "               ['seldom', ',', 'very', 'seldom', ',', 'does', 'complete', 'truth', 'belong', 'to', 'any', 'human',\n",
    "                'disclosure', ';', 'seldom', 'can', 'it', 'happen', 'that', 'something', 'is', 'not', 'a', 'little',\n",
    "                'disguised', 'or', 'a', 'little', 'mistaken', '.'],\n",
    "               ['i', 'declare', 'after', 'all', 'there', 'is', 'no', 'enjoyment', 'like', 'reading', '!', 'how', 'much',\n",
    "                'sooner', 'one', 'tires', 'of', 'anything', 'than', 'of', 'a', 'book', '!', '”'],\n",
    "               ['men', 'have', 'had', 'every', 'advantage', 'of', 'us', 'in', 'telling', 'their', 'own', 'story', '.',\n",
    "                'education', 'has', 'been', 'theirs', 'in', 'so', 'much', 'higher', 'a', 'degree'],\n",
    "               ['i', 'wish', ',', 'as', 'well', 'as', 'everybody', 'else', ',', 'to', 'be', 'perfectly', 'happy', ';',\n",
    "                'but', ',', 'like', 'everybody', 'else', ',', 'it', 'must', 'be', 'in', 'my', 'own', 'way', '.'],\n",
    "               ['there', 'are', 'people', ',', 'who', 'the', 'more', 'you', 'do', 'for', 'them', ',', 'the', 'less',\n",
    "                'they', 'will', 'do', 'for', 'themselves', '.'],\n",
    "               ['one', 'half', 'of', 'the', 'world', 'can', 'not', 'understand', 'the', 'pleasures', 'of', 'the',\n",
    "                'other', '.']]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## AugmentationStreamer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from seaqube.augmentation.base import AugmentationStreamer\n",
    "from seaqube.augmentation.reduction.unique_corpus import UniqueCorpusReduction\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Here we set up a augmentation stream. Every document will passed trought this augmentation line by line.\n",
    "# This means: a document _d_ will be in the first step translated.\n",
    "# In the second step, this translated document is feed to Qwerty. Now, qwerty returns 2 documents.\n",
    "# This 2 documents will be each feed to EDA. EDA geenrates 4 augmented documents for the two inputs, i.e. one line results in 8 lines output.\n",
    "# AugmentationStreamer can also reduce documents, here it reduce it, using the unique reducer.\n",
    "streamer = AugmentationStreamer([TranslationAugmentation(max_length=1), QwertyAugmentation(seed=424242, max_length=2), EDAAugmentation(max_length=4)], reduction_chain=[UniqueCorpusReduction()])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First, we run the `streamer` only on one line of the corpus."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100% (1 of 1) |##########################| Elapsed Time: 0:00:02 Time:  0:00:02\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[['trough', 'this', 'moment', 'i', 'never', 'knew', 'myself'],\n",
       " ['till', 'this', 'moment', 'i', 'never', 'knew', 'myself'],\n",
       " ['till', 'this', 'moment', 'i', 'neer', 'knew', 'myself'],\n",
       " ['till', 'this', 'moment', 'never', 'knew', 'myself'],\n",
       " ['neer', 'till', 'this', 'moment', 'i', 'never', 'knew', 'myself'],\n",
       " ['this', 'moment', 'i', 'never', 'knew', 'myself'],\n",
       " ['boulder', 'clay', 'this', 'moment', 'i', 'never', 'knew', 'myself']]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "augmented_doc = streamer([TEST_CORPUS[0]])\n",
    "augmented_doc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(augmented_doc) # after reducing documents can be less then 8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100% (7 of 7) |##########################| Elapsed Time: 0:00:18 Time:  0:00:18\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[['trough', 'this', 'moment', 'i', 'never', 'knew', 'myself'],\n",
       " ['till', 'this', 'moment', 'i', 'never', 'knew', 'myself'],\n",
       " ['till', 'this', 'moment', 'i', 'neer', 'knew', 'myself'],\n",
       " ['till', 'this', 'moment', 'never', 'knew', 'myself'],\n",
       " ['neer', 'till', 'this', 'moment', 'i', 'never', 'knew', 'myself'],\n",
       " ['this', 'moment', 'i', 'never', 'knew', 'myself'],\n",
       " ['boulder', 'clay', 'this', 'moment', 'i', 'never', 'knew', 'myself'],\n",
       " ['till', 'this', 'i', 'never', 'myself'],\n",
       " ['till', 'this', 'moment', 'i', 'neer', 'knew', 'mys', 'lf'],\n",
       " ['till', 'this', 'never', 'i', 'moment', 'knew', 'mys', 'lf'],\n",
       " ['till', 'this', 'moment', 'i', 'never', 'knew', 'myself'],\n",
       " ['till', 'this', 'here', 'and', 'now', 'i', 'never', 'knew', 'myself'],\n",
       " ['till', 'this', 'moment', 'i', 'never', 'knew', 'mys', 'lf'],\n",
       " ['till',\n",
       "  'this',\n",
       "  'moment',\n",
       "  'i',\n",
       "  'never',\n",
       "  'public',\n",
       "  'treasury',\n",
       "  'knew',\n",
       "  'myself'],\n",
       " ['public',\n",
       "  'treasury',\n",
       "  'till',\n",
       "  'this',\n",
       "  'moment',\n",
       "  'i',\n",
       "  'never',\n",
       "  'knew',\n",
       "  'mys',\n",
       "  'lf'],\n",
       " ['seldom',\n",
       "  'very',\n",
       "  'seldom',\n",
       "  'equal',\n",
       "  'does',\n",
       "  'complete',\n",
       "  'truth',\n",
       "  'belong',\n",
       "  'to',\n",
       "  'any',\n",
       "  'human',\n",
       "  'disflosure',\n",
       "  'seldom',\n",
       "  'can',\n",
       "  'it',\n",
       "  'happen',\n",
       "  'that',\n",
       "  'something',\n",
       "  'arrant',\n",
       "  'is',\n",
       "  'not',\n",
       "  'a',\n",
       "  'little',\n",
       "  'disguised',\n",
       "  'or',\n",
       "  'a',\n",
       "  'little',\n",
       "  'mistaken'],\n",
       " ['seleom',\n",
       "  'seldom',\n",
       "  'does',\n",
       "  'complete',\n",
       "  'truth',\n",
       "  'jelong',\n",
       "  'to',\n",
       "  'any',\n",
       "  'disclosure',\n",
       "  'seldom',\n",
       "  'can',\n",
       "  'it',\n",
       "  'happen',\n",
       "  'that',\n",
       "  'something',\n",
       "  'is',\n",
       "  'not',\n",
       "  'a',\n",
       "  'disguised',\n",
       "  'or',\n",
       "  'a',\n",
       "  'little',\n",
       "  'mistaken'],\n",
       " ['seldom',\n",
       "  'very',\n",
       "  'seldom',\n",
       "  'does',\n",
       "  'belong',\n",
       "  'to',\n",
       "  'complete',\n",
       "  'truth',\n",
       "  'belong',\n",
       "  'to',\n",
       "  'any',\n",
       "  'human',\n",
       "  'disflosure',\n",
       "  'seldom',\n",
       "  'can',\n",
       "  'it',\n",
       "  'happen',\n",
       "  'that',\n",
       "  'something',\n",
       "  'is',\n",
       "  'beaver',\n",
       "  'state',\n",
       "  'not',\n",
       "  'a',\n",
       "  'little',\n",
       "  'disguised',\n",
       "  'or',\n",
       "  'a',\n",
       "  'little',\n",
       "  'mistaken'],\n",
       " ['seleom',\n",
       "  'very',\n",
       "  'does',\n",
       "  'complete',\n",
       "  'jelong',\n",
       "  'to',\n",
       "  'any',\n",
       "  'human',\n",
       "  'disclosure',\n",
       "  'seldom',\n",
       "  'it',\n",
       "  'happen',\n",
       "  'that',\n",
       "  'something',\n",
       "  'is',\n",
       "  'not',\n",
       "  'a',\n",
       "  'little',\n",
       "  'disguised',\n",
       "  'or',\n",
       "  'a',\n",
       "  'mistaken'],\n",
       " ['seleom',\n",
       "  'very',\n",
       "  'seldom',\n",
       "  'does',\n",
       "  'complete',\n",
       "  'man',\n",
       "  'truth',\n",
       "  'jelong',\n",
       "  'to',\n",
       "  'any',\n",
       "  'human',\n",
       "  'disclosure',\n",
       "  'seldom',\n",
       "  'can',\n",
       "  'it',\n",
       "  'happen',\n",
       "  'that',\n",
       "  'something',\n",
       "  'is',\n",
       "  'whatever',\n",
       "  'not',\n",
       "  'a',\n",
       "  'little',\n",
       "  'disguised',\n",
       "  'or',\n",
       "  'a',\n",
       "  'little',\n",
       "  'mistaken'],\n",
       " ['seldom',\n",
       "  'very',\n",
       "  'seldom',\n",
       "  'does',\n",
       "  'complete',\n",
       "  'truth',\n",
       "  'not',\n",
       "  'to',\n",
       "  'any',\n",
       "  'human',\n",
       "  'disflosure',\n",
       "  'seldom',\n",
       "  'can',\n",
       "  'it',\n",
       "  'disguised',\n",
       "  'that',\n",
       "  'something',\n",
       "  'is',\n",
       "  'belong',\n",
       "  'a',\n",
       "  'little',\n",
       "  'happen',\n",
       "  'or',\n",
       "  'a',\n",
       "  'little',\n",
       "  'mistaken'],\n",
       " ['seleom',\n",
       "  'mistaken',\n",
       "  'seldom',\n",
       "  'human',\n",
       "  'complete',\n",
       "  'truth',\n",
       "  'jelong',\n",
       "  'to',\n",
       "  'any',\n",
       "  'does',\n",
       "  'disclosure',\n",
       "  'seldom',\n",
       "  'can',\n",
       "  'it',\n",
       "  'happen',\n",
       "  'that',\n",
       "  'something',\n",
       "  'is',\n",
       "  'not',\n",
       "  'a',\n",
       "  'little',\n",
       "  'disguised',\n",
       "  'or',\n",
       "  'a',\n",
       "  'little',\n",
       "  'very'],\n",
       " ['rarely',\n",
       "  'very',\n",
       "  'rarely',\n",
       "  'does',\n",
       "  'complete',\n",
       "  'truth',\n",
       "  'belong',\n",
       "  'to',\n",
       "  'any',\n",
       "  'man',\n",
       "  'disflosure',\n",
       "  'rarely',\n",
       "  'can',\n",
       "  'it',\n",
       "  'happen',\n",
       "  'that',\n",
       "  'something',\n",
       "  'is',\n",
       "  'not',\n",
       "  'a',\n",
       "  'little',\n",
       "  'disguised',\n",
       "  'or',\n",
       "  'a',\n",
       "  'little',\n",
       "  'mistaken'],\n",
       " ['i',\n",
       "  'declare',\n",
       "  'after',\n",
       "  'all',\n",
       "  'there',\n",
       "  'is',\n",
       "  'no',\n",
       "  'enjoyment',\n",
       "  'like',\n",
       "  'reading',\n",
       "  'how',\n",
       "  'much',\n",
       "  'sooner',\n",
       "  'one',\n",
       "  'tires',\n",
       "  'of',\n",
       "  'anything',\n",
       "  'than',\n",
       "  'of',\n",
       "  'a',\n",
       "  'adenine',\n",
       "  'later',\n",
       "  'on',\n",
       "  'book'],\n",
       " ['i',\n",
       "  'declare',\n",
       "  'after',\n",
       "  'all',\n",
       "  'there',\n",
       "  'of',\n",
       "  'no',\n",
       "  'enjoyment',\n",
       "  'like',\n",
       "  'than',\n",
       "  'how',\n",
       "  'much',\n",
       "  'sooner',\n",
       "  'one',\n",
       "  'tires',\n",
       "  'is',\n",
       "  'anything',\n",
       "  'reading',\n",
       "  'of',\n",
       "  'a',\n",
       "  'book'],\n",
       " ['i',\n",
       "  'declare',\n",
       "  'after',\n",
       "  'all',\n",
       "  'thege',\n",
       "  'is',\n",
       "  'no',\n",
       "  'enjoyment',\n",
       "  'like',\n",
       "  'reading',\n",
       "  'how',\n",
       "  'very',\n",
       "  'much',\n",
       "  'sooner',\n",
       "  'one',\n",
       "  'tores',\n",
       "  'og',\n",
       "  'anything',\n",
       "  'than',\n",
       "  'of',\n",
       "  'a',\n",
       "  'script'],\n",
       " ['i',\n",
       "  'declare',\n",
       "  'after',\n",
       "  'all',\n",
       "  'there',\n",
       "  'is',\n",
       "  'no',\n",
       "  'enjoyment',\n",
       "  'like',\n",
       "  'reading',\n",
       "  'much',\n",
       "  'how',\n",
       "  'sooner',\n",
       "  'anything',\n",
       "  'tires',\n",
       "  'of',\n",
       "  'one',\n",
       "  'than',\n",
       "  'of',\n",
       "  'a',\n",
       "  'book'],\n",
       " ['i',\n",
       "  'thege',\n",
       "  'is',\n",
       "  'no',\n",
       "  'like',\n",
       "  'reading',\n",
       "  'how',\n",
       "  'much',\n",
       "  'sooner',\n",
       "  'one',\n",
       "  'og',\n",
       "  'anything',\n",
       "  'than',\n",
       "  'of',\n",
       "  'book'],\n",
       " ['i',\n",
       "  'declare',\n",
       "  'after',\n",
       "  'all',\n",
       "  'thege',\n",
       "  'than',\n",
       "  'a',\n",
       "  'enjoyment',\n",
       "  'like',\n",
       "  'reading',\n",
       "  'how',\n",
       "  'much',\n",
       "  'sooner',\n",
       "  'one',\n",
       "  'tores',\n",
       "  'og',\n",
       "  'anything',\n",
       "  'is',\n",
       "  'of',\n",
       "  'no',\n",
       "  'book'],\n",
       " ['very',\n",
       "  'much',\n",
       "  'i',\n",
       "  'declare',\n",
       "  'announce',\n",
       "  'after',\n",
       "  'all',\n",
       "  'there',\n",
       "  'is',\n",
       "  'no',\n",
       "  'enjoyment',\n",
       "  'like',\n",
       "  'reading',\n",
       "  'how',\n",
       "  'much',\n",
       "  'sooner',\n",
       "  'one',\n",
       "  'tires',\n",
       "  'of',\n",
       "  'anything',\n",
       "  'than',\n",
       "  'of',\n",
       "  'a',\n",
       "  'book'],\n",
       " ['i',\n",
       "  'declare',\n",
       "  'after',\n",
       "  'all',\n",
       "  'thege',\n",
       "  'no',\n",
       "  'enjoyment',\n",
       "  'like',\n",
       "  'reading',\n",
       "  'how',\n",
       "  'much',\n",
       "  'sooner',\n",
       "  'tores',\n",
       "  'og',\n",
       "  'anything',\n",
       "  'than',\n",
       "  'a',\n",
       "  'book'],\n",
       " ['men',\n",
       "  'have',\n",
       "  'had',\n",
       "  'every',\n",
       "  'advqntage',\n",
       "  'of',\n",
       "  'us',\n",
       "  'in',\n",
       "  'telling',\n",
       "  'their',\n",
       "  'own',\n",
       "  'education',\n",
       "  'has',\n",
       "  'been',\n",
       "  'in',\n",
       "  'much',\n",
       "  'higher',\n",
       "  'degree'],\n",
       " ['men',\n",
       "  'have',\n",
       "  'had',\n",
       "  'every',\n",
       "  'advqntage',\n",
       "  'of',\n",
       "  'atomic',\n",
       "  'number',\n",
       "  'in',\n",
       "  'telling',\n",
       "  'their',\n",
       "  'own',\n",
       "  'story',\n",
       "  'department',\n",
       "  'of',\n",
       "  'education',\n",
       "  'has',\n",
       "  'been',\n",
       "  'theirs',\n",
       "  'in',\n",
       "  'so',\n",
       "  'much',\n",
       "  'higher',\n",
       "  'a',\n",
       "  'degree'],\n",
       " ['of',\n",
       "  'have',\n",
       "  'had',\n",
       "  'every',\n",
       "  'advqntage',\n",
       "  'men',\n",
       "  'us',\n",
       "  'in',\n",
       "  'telling',\n",
       "  'their',\n",
       "  'own',\n",
       "  'story',\n",
       "  'education',\n",
       "  'has',\n",
       "  'been',\n",
       "  'theirs',\n",
       "  'in',\n",
       "  'so',\n",
       "  'much',\n",
       "  'a',\n",
       "  'higher',\n",
       "  'degree'],\n",
       " ['men',\n",
       "  'have',\n",
       "  'had',\n",
       "  'every',\n",
       "  'reward',\n",
       "  'of',\n",
       "  'us',\n",
       "  'in',\n",
       "  'telling',\n",
       "  'their',\n",
       "  'own',\n",
       "  'tarradiddle',\n",
       "  'education',\n",
       "  'has',\n",
       "  'been',\n",
       "  'theirs',\n",
       "  'in',\n",
       "  'so',\n",
       "  'muxh',\n",
       "  'higher',\n",
       "  'a',\n",
       "  'degree'],\n",
       " ['men',\n",
       "  'have',\n",
       "  'had',\n",
       "  'every',\n",
       "  'advantage',\n",
       "  'of',\n",
       "  'usa',\n",
       "  'in',\n",
       "  'telling',\n",
       "  'their',\n",
       "  'own',\n",
       "  'report',\n",
       "  'education',\n",
       "  'has',\n",
       "  'been',\n",
       "  'theirs',\n",
       "  'in',\n",
       "  'so',\n",
       "  'muxh',\n",
       "  'higher',\n",
       "  'a',\n",
       "  'degree'],\n",
       " ['men',\n",
       "  'have',\n",
       "  'had',\n",
       "  'every',\n",
       "  'advantage',\n",
       "  'of',\n",
       "  'us',\n",
       "  'in',\n",
       "  'telling',\n",
       "  'their',\n",
       "  'own',\n",
       "  'story',\n",
       "  'education',\n",
       "  'has',\n",
       "  'been',\n",
       "  'theirs',\n",
       "  'in',\n",
       "  'possess',\n",
       "  'so',\n",
       "  'usa',\n",
       "  'muxh',\n",
       "  'higher',\n",
       "  'a',\n",
       "  'degree'],\n",
       " ['men',\n",
       "  'have',\n",
       "  'had',\n",
       "  'every',\n",
       "  'advqntage',\n",
       "  'of',\n",
       "  'us',\n",
       "  'in',\n",
       "  'telling',\n",
       "  'their',\n",
       "  'possess',\n",
       "  'own',\n",
       "  'usa',\n",
       "  'story',\n",
       "  'education',\n",
       "  'has',\n",
       "  'been',\n",
       "  'theirs',\n",
       "  'in',\n",
       "  'so',\n",
       "  'much',\n",
       "  'higher',\n",
       "  'a',\n",
       "  'degree'],\n",
       " ['men',\n",
       "  'every',\n",
       "  'had',\n",
       "  'have',\n",
       "  'advantage',\n",
       "  'of',\n",
       "  'us',\n",
       "  'in',\n",
       "  'telling',\n",
       "  'their',\n",
       "  'own',\n",
       "  'story',\n",
       "  'in',\n",
       "  'has',\n",
       "  'been',\n",
       "  'theirs',\n",
       "  'education',\n",
       "  'so',\n",
       "  'muxh',\n",
       "  'higher',\n",
       "  'a',\n",
       "  'degree'],\n",
       " ['i',\n",
       "  'regard',\n",
       "  'as',\n",
       "  'well',\n",
       "  'as',\n",
       "  'everybody',\n",
       "  'else',\n",
       "  'to',\n",
       "  'be',\n",
       "  'utterly',\n",
       "  'happy',\n",
       "  'but',\n",
       "  'lkke',\n",
       "  'everybody',\n",
       "  'else',\n",
       "  'it',\n",
       "  'must',\n",
       "  'be',\n",
       "  'in',\n",
       "  'my',\n",
       "  'own',\n",
       "  'way'],\n",
       " ['i',\n",
       "  'wish',\n",
       "  'as',\n",
       "  'well',\n",
       "  'as',\n",
       "  'everybody',\n",
       "  'else',\n",
       "  'to',\n",
       "  'be',\n",
       "  'happy',\n",
       "  'but',\n",
       "  'lkke',\n",
       "  'everybody',\n",
       "  'else',\n",
       "  'it',\n",
       "  'must',\n",
       "  'be',\n",
       "  'in',\n",
       "  'my',\n",
       "  'own',\n",
       "  'way'],\n",
       " ['i',\n",
       "  'swell',\n",
       "  'wish',\n",
       "  'as',\n",
       "  'well',\n",
       "  'as',\n",
       "  'everybody',\n",
       "  'else',\n",
       "  'to',\n",
       "  'be',\n",
       "  'perfectly',\n",
       "  'happy',\n",
       "  'but',\n",
       "  'lkke',\n",
       "  'everybody',\n",
       "  'else',\n",
       "  'it',\n",
       "  'must',\n",
       "  'swell',\n",
       "  'be',\n",
       "  'in',\n",
       "  'my',\n",
       "  'own',\n",
       "  'way'],\n",
       " ['i',\n",
       "  'wish',\n",
       "  'as',\n",
       "  'as',\n",
       "  'everybody',\n",
       "  'else',\n",
       "  'to',\n",
       "  'be',\n",
       "  'perfectly',\n",
       "  'happy',\n",
       "  'but',\n",
       "  'like',\n",
       "  'everybody',\n",
       "  'else',\n",
       "  'it',\n",
       "  'must',\n",
       "  'be',\n",
       "  'in',\n",
       "  'my',\n",
       "  'own',\n",
       "  'way'],\n",
       " ['must',\n",
       "  'as',\n",
       "  'as',\n",
       "  'well',\n",
       "  'wish',\n",
       "  'everybody',\n",
       "  'else',\n",
       "  'to',\n",
       "  'be',\n",
       "  'perfectly',\n",
       "  'happy',\n",
       "  'but',\n",
       "  'like',\n",
       "  'everybody',\n",
       "  'else',\n",
       "  'it',\n",
       "  'i',\n",
       "  'be',\n",
       "  'in',\n",
       "  'my',\n",
       "  'own',\n",
       "  'way'],\n",
       " ['i',\n",
       "  'wish',\n",
       "  'as',\n",
       "  'well',\n",
       "  'as',\n",
       "  'everybody',\n",
       "  'else',\n",
       "  'to',\n",
       "  'be',\n",
       "  'perfectly',\n",
       "  'happy',\n",
       "  'but',\n",
       "  'like',\n",
       "  'everybody',\n",
       "  'else',\n",
       "  'it',\n",
       "  'must',\n",
       "  'be',\n",
       "  'in',\n",
       "  'own',\n",
       "  'way'],\n",
       " ['i',\n",
       "  'wish',\n",
       "  'as',\n",
       "  'well',\n",
       "  'as',\n",
       "  'everybody',\n",
       "  'else',\n",
       "  'to',\n",
       "  'be',\n",
       "  'utterly',\n",
       "  'happy',\n",
       "  'but',\n",
       "  'same',\n",
       "  'everybody',\n",
       "  'else',\n",
       "  'it',\n",
       "  'must',\n",
       "  'be',\n",
       "  'in',\n",
       "  'my',\n",
       "  'own',\n",
       "  'way'],\n",
       " ['i',\n",
       "  'wish',\n",
       "  'as',\n",
       "  'well',\n",
       "  'as',\n",
       "  'everybody',\n",
       "  'else',\n",
       "  'to',\n",
       "  'equal',\n",
       "  'be',\n",
       "  'perfectly',\n",
       "  'happy',\n",
       "  'but',\n",
       "  'lkke',\n",
       "  'everybody',\n",
       "  'else',\n",
       "  'it',\n",
       "  'must',\n",
       "  'be',\n",
       "  'in',\n",
       "  'my',\n",
       "  'own',\n",
       "  'way'],\n",
       " ['there',\n",
       "  'are',\n",
       "  'people',\n",
       "  'who',\n",
       "  'the',\n",
       "  'more',\n",
       "  'do',\n",
       "  'for',\n",
       "  'them',\n",
       "  'the',\n",
       "  'less',\n",
       "  'they',\n",
       "  'will',\n",
       "  'do',\n",
       "  'for',\n",
       "  'themselves'],\n",
       " ['there',\n",
       "  'are',\n",
       "  'people',\n",
       "  'the',\n",
       "  'who',\n",
       "  'more',\n",
       "  'you',\n",
       "  'do',\n",
       "  'for',\n",
       "  'them',\n",
       "  'the',\n",
       "  'less',\n",
       "  'they',\n",
       "  'will',\n",
       "  'do',\n",
       "  'for',\n",
       "  'themselves'],\n",
       " ['there',\n",
       "  'are',\n",
       "  'people',\n",
       "  'who',\n",
       "  'the',\n",
       "  'more',\n",
       "  'you',\n",
       "  'do',\n",
       "  'for',\n",
       "  'the',\n",
       "  'less',\n",
       "  'they',\n",
       "  'will',\n",
       "  'do',\n",
       "  'themselves'],\n",
       " ['there',\n",
       "  'are',\n",
       "  'people',\n",
       "  'who',\n",
       "  'the',\n",
       "  'more',\n",
       "  'you',\n",
       "  'do',\n",
       "  'for',\n",
       "  'them',\n",
       "  'the',\n",
       "  'less',\n",
       "  'they',\n",
       "  'will',\n",
       "  'do',\n",
       "  'citizenry',\n",
       "  'for',\n",
       "  'themselves'],\n",
       " ['there',\n",
       "  'are',\n",
       "  'people',\n",
       "  'who',\n",
       "  'the',\n",
       "  'more',\n",
       "  'you',\n",
       "  'do',\n",
       "  'for',\n",
       "  'them',\n",
       "  'the',\n",
       "  'less',\n",
       "  'they',\n",
       "  'will',\n",
       "  'for',\n",
       "  'themselves'],\n",
       " ['there',\n",
       "  'citizenry',\n",
       "  'are',\n",
       "  'people',\n",
       "  'who',\n",
       "  'the',\n",
       "  'more',\n",
       "  'you',\n",
       "  'do',\n",
       "  'for',\n",
       "  'them',\n",
       "  'the',\n",
       "  'less',\n",
       "  'they',\n",
       "  'will',\n",
       "  'do',\n",
       "  'for',\n",
       "  'themselves'],\n",
       " ['there',\n",
       "  'are',\n",
       "  'people',\n",
       "  'who',\n",
       "  'the',\n",
       "  'more',\n",
       "  'you',\n",
       "  'do',\n",
       "  'for',\n",
       "  'them',\n",
       "  'the',\n",
       "  'less',\n",
       "  'will',\n",
       "  'do',\n",
       "  'for',\n",
       "  'themselves'],\n",
       " ['there',\n",
       "  'are',\n",
       "  'people',\n",
       "  'who',\n",
       "  'the',\n",
       "  'more',\n",
       "  'you',\n",
       "  'do',\n",
       "  'for',\n",
       "  'them',\n",
       "  'the',\n",
       "  'to',\n",
       "  'a',\n",
       "  'lesser',\n",
       "  'extent',\n",
       "  'they',\n",
       "  'will',\n",
       "  'do',\n",
       "  'for',\n",
       "  'themselves'],\n",
       " ['one',\n",
       "  'half',\n",
       "  'of',\n",
       "  'the',\n",
       "  'world',\n",
       "  'can',\n",
       "  'not',\n",
       "  'understand',\n",
       "  'the',\n",
       "  'of',\n",
       "  'the',\n",
       "  'other'],\n",
       " ['one',\n",
       "  'half',\n",
       "  'of',\n",
       "  'the',\n",
       "  'world',\n",
       "  'understand',\n",
       "  'can',\n",
       "  'not',\n",
       "  'the',\n",
       "  'pleasures',\n",
       "  'of',\n",
       "  'the',\n",
       "  'other'],\n",
       " ['one',\n",
       "  'half',\n",
       "  'of',\n",
       "  'the',\n",
       "  'the',\n",
       "  'can',\n",
       "  'not',\n",
       "  'understand',\n",
       "  'the',\n",
       "  'pleasures',\n",
       "  'of',\n",
       "  'world',\n",
       "  'other'],\n",
       " ['one',\n",
       "  'half',\n",
       "  'of',\n",
       "  'the',\n",
       "  'world',\n",
       "  'can',\n",
       "  'not',\n",
       "  'understand',\n",
       "  'early',\n",
       "  'the',\n",
       "  'pleasures',\n",
       "  'of',\n",
       "  'the',\n",
       "  'other'],\n",
       " ['one',\n",
       "  'half',\n",
       "  'of',\n",
       "  'the',\n",
       "  'world',\n",
       "  'humankind',\n",
       "  'can',\n",
       "  'not',\n",
       "  'understand',\n",
       "  'the',\n",
       "  'pleasures',\n",
       "  'of',\n",
       "  'the',\n",
       "  'other'],\n",
       " ['one',\n",
       "  'half',\n",
       "  'of',\n",
       "  'the',\n",
       "  'creation',\n",
       "  'can',\n",
       "  'not',\n",
       "  'understand',\n",
       "  'the',\n",
       "  'pleasures',\n",
       "  'of',\n",
       "  'the',\n",
       "  'other'],\n",
       " ['one',\n",
       "  'half',\n",
       "  'of',\n",
       "  'the',\n",
       "  'world',\n",
       "  'can',\n",
       "  'not',\n",
       "  'understand',\n",
       "  'the',\n",
       "  'pleasures',\n",
       "  'of',\n",
       "  'the',\n",
       "  'other'],\n",
       " ['one',\n",
       "  'can',\n",
       "  'not',\n",
       "  'of',\n",
       "  'the',\n",
       "  'world',\n",
       "  'half',\n",
       "  'understand',\n",
       "  'the',\n",
       "  'pleasures',\n",
       "  'of',\n",
       "  'the',\n",
       "  'other']]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "streamer(TEST_CORPUS) # apply the full corpus for the streamer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CallOnOneChain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus = [['the', 'quick', 'brown', 'fox', 'jumps', 'over', 'the', 'lazy', 'dog']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from seaqube.tools.chainer import CallOnOneChain\n",
    "from seaqube.nlp.tools import unique_2d_list"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### The usage is similar to the AugmentatinoStreamer. Only the reduceing is different."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipe = CallOnOneChain([TranslationAugmentation(max_length=1), UnigramAugmentation(corpus=TEST_CORPUS, seed=50, max_length=2, replace_threshold=0.9, find_threshold=0.85),\n",
    "                               QwertyAugmentation(seed=424242, max_length=2), unique_2d_list])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100% (7 of 7) |##########################| Elapsed Time: 0:00:18 Time:  0:00:18\n",
      " 50% (7 of 14) |############             | Elapsed Time: 0:00:00 ETA:  00:00:00"
     ]
    }
   ],
   "source": [
    "augmented_and_unique = pipe(TEST_CORPUS)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The implementation is differnt, here the full corpus is applied in the first augmentation, then the augmented corpus is applied on the next augmentation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['i',\n",
       "  'declare',\n",
       "  'after',\n",
       "  'all',\n",
       "  'there',\n",
       "  'is',\n",
       "  'no',\n",
       "  'enjoyment',\n",
       "  'like',\n",
       "  'reading',\n",
       "  '!',\n",
       "  'how',\n",
       "  'much',\n",
       "  'sooner',\n",
       "  'one',\n",
       "  'tires',\n",
       "  'of',\n",
       "  'anything',\n",
       "  'than',\n",
       "  'of',\n",
       "  'a',\n",
       "  'book',\n",
       "  '!',\n",
       "  '”'],\n",
       " ['till', 'this', 'moment', 'i', 'never', 'knew', 'myself', '.'],\n",
       " ['men',\n",
       "  'have',\n",
       "  'had',\n",
       "  'every',\n",
       "  'advantage',\n",
       "  'of',\n",
       "  'us',\n",
       "  'in',\n",
       "  'telling',\n",
       "  'their',\n",
       "  'own',\n",
       "  'story',\n",
       "  '.',\n",
       "  'education',\n",
       "  'has',\n",
       "  'been',\n",
       "  'theirs',\n",
       "  'in',\n",
       "  'so',\n",
       "  'much',\n",
       "  'higher',\n",
       "  'a',\n",
       "  'd2gree'],\n",
       " ['i',\n",
       "  'declare',\n",
       "  'after',\n",
       "  'all',\n",
       "  'there',\n",
       "  'is',\n",
       "  'no',\n",
       "  'enjoyment',\n",
       "  'like',\n",
       "  'feading',\n",
       "  '!',\n",
       "  'how',\n",
       "  'much',\n",
       "  'sooner',\n",
       "  'one',\n",
       "  'tires',\n",
       "  'of',\n",
       "  'ahything',\n",
       "  'than',\n",
       "  'of',\n",
       "  'a',\n",
       "  'book',\n",
       "  '!',\n",
       "  '”'],\n",
       " ['men',\n",
       "  'have',\n",
       "  'had',\n",
       "  'every',\n",
       "  'advantage',\n",
       "  ',',\n",
       "  'us',\n",
       "  'in',\n",
       "  'telling',\n",
       "  'their',\n",
       "  'own',\n",
       "  'story',\n",
       "  '.',\n",
       "  'education',\n",
       "  'has',\n",
       "  'been',\n",
       "  'theirs',\n",
       "  'in',\n",
       "  'so',\n",
       "  'much',\n",
       "  'higher',\n",
       "  'a',\n",
       "  'decree'],\n",
       " ['one',\n",
       "  'half',\n",
       "  'of',\n",
       "  'the',\n",
       "  'world',\n",
       "  'can',\n",
       "  'not',\n",
       "  'understand',\n",
       "  'hhe',\n",
       "  'pleasures',\n",
       "  'of',\n",
       "  'the',\n",
       "  'other',\n",
       "  '.'],\n",
       " [',',\n",
       "  'wish',\n",
       "  ',',\n",
       "  'as',\n",
       "  'well',\n",
       "  'as',\n",
       "  'everybody',\n",
       "  'else',\n",
       "  ',',\n",
       "  'to',\n",
       "  'be',\n",
       "  'perfectly',\n",
       "  'happy',\n",
       "  ';',\n",
       "  'but',\n",
       "  ',',\n",
       "  'like',\n",
       "  'everybody',\n",
       "  'else',\n",
       "  ',',\n",
       "  'it',\n",
       "  'must',\n",
       "  'be',\n",
       "  'in',\n",
       "  'my',\n",
       "  'own',\n",
       "  'way',\n",
       "  '.'],\n",
       " ['men',\n",
       "  'have',\n",
       "  'had',\n",
       "  'every',\n",
       "  'advantage',\n",
       "  'of',\n",
       "  'us',\n",
       "  'in',\n",
       "  'telming',\n",
       "  'their',\n",
       "  'own',\n",
       "  'story',\n",
       "  '.',\n",
       "  'education',\n",
       "  'has',\n",
       "  'be2n',\n",
       "  'theirs',\n",
       "  'in',\n",
       "  'so',\n",
       "  'much',\n",
       "  'higher',\n",
       "  'a',\n",
       "  'degree'],\n",
       " [',',\n",
       "  'half',\n",
       "  'of',\n",
       "  'the',\n",
       "  'world',\n",
       "  'can',\n",
       "  'not',\n",
       "  'understand',\n",
       "  'the',\n",
       "  'p>easures',\n",
       "  'of',\n",
       "  'the',\n",
       "  'other',\n",
       "  '.'],\n",
       " [',',\n",
       "  'eish',\n",
       "  ',',\n",
       "  'as',\n",
       "  'well',\n",
       "  'as',\n",
       "  'everybody',\n",
       "  'else',\n",
       "  ',',\n",
       "  'to',\n",
       "  'be',\n",
       "  'perfectmy',\n",
       "  'happy',\n",
       "  ';',\n",
       "  'but',\n",
       "  ',',\n",
       "  'like',\n",
       "  'everybody',\n",
       "  'else',\n",
       "  ',',\n",
       "  'it',\n",
       "  'just',\n",
       "  'be',\n",
       "  'in',\n",
       "  'my',\n",
       "  'own',\n",
       "  'way',\n",
       "  '.'],\n",
       " ['men',\n",
       "  'have',\n",
       "  'had',\n",
       "  'every',\n",
       "  'advantage',\n",
       "  ',',\n",
       "  'us',\n",
       "  'in',\n",
       "  'tellibg',\n",
       "  'their',\n",
       "  'own',\n",
       "  'story',\n",
       "  '.',\n",
       "  'education',\n",
       "  'has',\n",
       "  'been',\n",
       "  'hheirs',\n",
       "  'in',\n",
       "  'so',\n",
       "  'much',\n",
       "  'higher',\n",
       "  'a',\n",
       "  'degree'],\n",
       " ['i',\n",
       "  'wish',\n",
       "  ',',\n",
       "  'as',\n",
       "  'well',\n",
       "  'as',\n",
       "  'everybody',\n",
       "  'else',\n",
       "  ',',\n",
       "  'to',\n",
       "  'be',\n",
       "  'perfectly',\n",
       "  'happy',\n",
       "  ';',\n",
       "  'but',\n",
       "  ',',\n",
       "  'like',\n",
       "  'everybody',\n",
       "  'else',\n",
       "  ',',\n",
       "  'it',\n",
       "  'must',\n",
       "  'be',\n",
       "  'in',\n",
       "  'my',\n",
       "  'own',\n",
       "  'way',\n",
       "  '.'],\n",
       " ['i',\n",
       "  'eish',\n",
       "  ',',\n",
       "  'as',\n",
       "  'well',\n",
       "  'as',\n",
       "  'everybody',\n",
       "  'else',\n",
       "  ',',\n",
       "  'to',\n",
       "  'be',\n",
       "  'perfectmy',\n",
       "  'happy',\n",
       "  ';',\n",
       "  'but',\n",
       "  ',',\n",
       "  'like',\n",
       "  'everybody',\n",
       "  'else',\n",
       "  ',',\n",
       "  'it',\n",
       "  'just',\n",
       "  'be',\n",
       "  'in',\n",
       "  'my',\n",
       "  'own',\n",
       "  'way',\n",
       "  '.'],\n",
       " ['till', 'this', 'moment', ',', 'never', 'knew', 'myself', '.'],\n",
       " ['one',\n",
       "  'half',\n",
       "  'of',\n",
       "  'the',\n",
       "  'world',\n",
       "  'can',\n",
       "  'not',\n",
       "  'understand',\n",
       "  'the',\n",
       "  'pleasures',\n",
       "  'of',\n",
       "  'the',\n",
       "  'other',\n",
       "  '.'],\n",
       " [',',\n",
       "  'declare',\n",
       "  'after',\n",
       "  'all',\n",
       "  'there',\n",
       "  'is',\n",
       "  'no',\n",
       "  'enjoyment',\n",
       "  'like',\n",
       "  'reading',\n",
       "  '!',\n",
       "  'how',\n",
       "  'much',\n",
       "  'sooner',\n",
       "  'one',\n",
       "  'tires',\n",
       "  'of',\n",
       "  'anything',\n",
       "  'than',\n",
       "  'of',\n",
       "  'a',\n",
       "  'book',\n",
       "  '!',\n",
       "  '”'],\n",
       " [',',\n",
       "  'are',\n",
       "  'people',\n",
       "  ',',\n",
       "  'who',\n",
       "  'the',\n",
       "  'more',\n",
       "  'you',\n",
       "  'so',\n",
       "  'for',\n",
       "  'them',\n",
       "  ',',\n",
       "  'the',\n",
       "  'less',\n",
       "  'they',\n",
       "  'will',\n",
       "  'do',\n",
       "  'for',\n",
       "  'themselv4s',\n",
       "  '.'],\n",
       " [',',\n",
       "  'half',\n",
       "  'of',\n",
       "  'the',\n",
       "  'world',\n",
       "  'can',\n",
       "  'not',\n",
       "  'understand',\n",
       "  'the',\n",
       "  'pleasures',\n",
       "  'of',\n",
       "  'the',\n",
       "  'other',\n",
       "  '.'],\n",
       " [',',\n",
       "  'declare',\n",
       "  'after',\n",
       "  'all',\n",
       "  'there',\n",
       "  'is',\n",
       "  'no',\n",
       "  'enjoyment',\n",
       "  'like',\n",
       "  'feading',\n",
       "  '!',\n",
       "  'how',\n",
       "  'much',\n",
       "  'sooner',\n",
       "  'one',\n",
       "  'tires',\n",
       "  'of',\n",
       "  'ahything',\n",
       "  'than',\n",
       "  'of',\n",
       "  'a',\n",
       "  'book',\n",
       "  '!',\n",
       "  '”'],\n",
       " ['seldom',\n",
       "  ',',\n",
       "  'very',\n",
       "  'seldom',\n",
       "  ',',\n",
       "  'does',\n",
       "  'complehe',\n",
       "  'truth',\n",
       "  'belong',\n",
       "  'to',\n",
       "  'any',\n",
       "  'human',\n",
       "  'disclosure',\n",
       "  ';',\n",
       "  'seldom',\n",
       "  'can',\n",
       "  'it',\n",
       "  'happen',\n",
       "  'that',\n",
       "  'something',\n",
       "  'is',\n",
       "  'not',\n",
       "  'a',\n",
       "  'little',\n",
       "  'disguised',\n",
       "  'or',\n",
       "  'a',\n",
       "  'little',\n",
       "  'mistaken',\n",
       "  '.'],\n",
       " ['there',\n",
       "  'are',\n",
       "  'people',\n",
       "  ',',\n",
       "  'who',\n",
       "  'th2',\n",
       "  'more',\n",
       "  'you',\n",
       "  'do',\n",
       "  'for',\n",
       "  'them',\n",
       "  ',',\n",
       "  'the',\n",
       "  'less',\n",
       "  'they',\n",
       "  'will',\n",
       "  'do',\n",
       "  'for',\n",
       "  'yhemselves',\n",
       "  '.'],\n",
       " ['seldom',\n",
       "  ',',\n",
       "  'very',\n",
       "  'seldom',\n",
       "  ',',\n",
       "  'does',\n",
       "  'complete',\n",
       "  'truth',\n",
       "  'belong',\n",
       "  'to',\n",
       "  'any',\n",
       "  'human',\n",
       "  'disclosure',\n",
       "  ';',\n",
       "  'seldom',\n",
       "  'can',\n",
       "  'it',\n",
       "  'happen',\n",
       "  'that',\n",
       "  'someth8ng',\n",
       "  'is',\n",
       "  'not',\n",
       "  'a',\n",
       "  'little',\n",
       "  'disguised',\n",
       "  'or',\n",
       "  'a',\n",
       "  'little',\n",
       "  'mistaien',\n",
       "  '.'],\n",
       " ['there',\n",
       "  'are',\n",
       "  'people',\n",
       "  ',',\n",
       "  'who',\n",
       "  'the',\n",
       "  'more',\n",
       "  'you',\n",
       "  'do',\n",
       "  'for',\n",
       "  'them',\n",
       "  ',',\n",
       "  'the',\n",
       "  'less',\n",
       "  'they',\n",
       "  'will',\n",
       "  'do',\n",
       "  'for',\n",
       "  'themselves',\n",
       "  '.'],\n",
       " [',',\n",
       "  'are',\n",
       "  'people',\n",
       "  ',',\n",
       "  'who',\n",
       "  'the',\n",
       "  'more',\n",
       "  'you',\n",
       "  'do',\n",
       "  'for',\n",
       "  'them',\n",
       "  ',',\n",
       "  'the',\n",
       "  'less',\n",
       "  'they',\n",
       "  'will',\n",
       "  'do',\n",
       "  'for',\n",
       "  'themselves',\n",
       "  '.']]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "augmented_and_unique"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "24"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(augmented_and_unique) # 8 * 2 * 2 = 32, reducing makes it smaller "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}