Search_Engine_for_e_commerce_using_Sentence_Transformers.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-iNHt6FDdniF"
      },
      "source": [
        "# Installations and Imports"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mDXVk_TaIur-",
        "outputId": "0f6a3170-bc7a-48d3-c2e8-2284df2ef2ff"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: transformers in /usr/local/lib/python3.7/dist-packages (4.23.1)\n",
            "Requirement already satisfied: datasets in /usr/local/lib/python3.7/dist-packages (2.6.1)\n",
            "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /usr/local/lib/python3.7/dist-packages (from transformers) (0.13.1)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.10.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (0.10.1)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.7/dist-packages (from transformers) (6.0)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.21.6)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (21.3)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (4.13.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.8.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2022.6.2)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.64.1)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0,>=0.10.0->transformers) (4.1.1)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers) (3.0.9)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.7/dist-packages (from datasets) (3.8.3)\n",
            "Requirement already satisfied: responses<0.19 in /usr/local/lib/python3.7/dist-packages (from datasets) (0.18.0)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from datasets) (1.3.5)\n",
            "Requirement already satisfied: xxhash in /usr/local/lib/python3.7/dist-packages (from datasets) (3.1.0)\n",
            "Requirement already satisfied: multiprocess in /usr/local/lib/python3.7/dist-packages (from datasets) (0.70.13)\n",
            "Requirement already satisfied: dill<0.3.6 in /usr/local/lib/python3.7/dist-packages (from datasets) (0.3.5.1)\n",
            "Requirement already satisfied: fsspec[http]>=2021.11.1 in /usr/local/lib/python3.7/dist-packages (from datasets) (2022.10.0)\n",
            "Requirement already satisfied: pyarrow>=6.0.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (6.0.1)\n",
            "Requirement already satisfied: asynctest==0.13.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (0.13.0)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (1.8.1)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (4.0.2)\n",
            "Requirement already satisfied: charset-normalizer<3.0,>=2.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (2.1.1)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (1.2.0)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (22.1.0)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (1.3.1)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (6.0.2)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.25.11)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2022.9.24)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers) (3.9.0)\n",
            "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2022.5)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2.8.2)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.7.3->pandas->datasets) (1.15.0)\n"
          ]
        }
      ],
      "source": [
        "!pip install transformers datasets"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "nR4lW5kcIx0j"
      },
      "outputs": [],
      "source": [
        "import tensorflow as tf### models\n",
        "import numpy as np### math computations\n",
        "import matplotlib.pyplot as plt### plotting bar chart\n",
        "import sklearn### machine learning library\n",
        "import cv2## image processing\n",
        "from sklearn.metrics import confusion_matrix, roc_curve### metrics\n",
        "import seaborn as sns### visualizations\n",
        "import datetime\n",
        "import pathlib\n",
        "import io\n",
        "import pandas as pd\n",
        "import os\n",
        "import re\n",
        "import csv\n",
        "import string\n",
        "import time\n",
        "from numpy import random\n",
        "import gensim.downloader as api\n",
        "from PIL import Image\n",
        "import tensorflow_datasets as tfds\n",
        "import tensorflow_probability as tfp\n",
        "from tensorflow.keras.models import Model\n",
        "from tensorflow.keras.layers import Layer\n",
        "from tensorflow.keras.layers import Dense,Flatten,InputLayer,BatchNormalization,Dropout,Input,LayerNormalization\n",
        "from tensorflow.keras.losses import BinaryCrossentropy,CategoricalCrossentropy, SparseCategoricalCrossentropy\n",
        "from tensorflow.keras.metrics import Accuracy,TopKCategoricalAccuracy, CategoricalAccuracy, SparseCategoricalAccuracy\n",
        "from tensorflow.keras.optimizers import Adam\n",
        "from google.colab import drive\n",
        "from google.colab import files\n",
        "from datasets import load_dataset\n",
        "from transformers import AutoTokenizer,create_optimizer,TFAutoModel"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "sV60GYXSgSDK"
      },
      "outputs": [],
      "source": [
        "BATCH_SIZE=128\n",
        "MAX_LENGTH=64"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "O2od1HEDDvz3"
      },
      "source": [
        "# Dataset Preparation"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aSaOBWrxwzaE"
      },
      "source": [
        "## Downloads"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true,
          "base_uri": "https://localhost:8080/"
        },
        "id": "Z4IA5YKfIx26",
        "outputId": "c17dcbac-b255-45b1-dbb4-6bf3cbcf00bb"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Collecting aicrowd-cli\n",
            "  Downloading aicrowd_cli-0.1.15-py3-none-any.whl (51 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m51.1/51.1 kB\u001b[0m \u001b[31m1.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting click<8,>=7.1.2 (from aicrowd-cli)\n",
            "  Downloading click-7.1.2-py2.py3-none-any.whl (82 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m82.8/82.8 kB\u001b[0m \u001b[31m3.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting GitPython==3.1.18 (from aicrowd-cli)\n",
            "  Downloading GitPython-3.1.18-py3-none-any.whl (170 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m170.1/170.1 kB\u001b[0m \u001b[31m8.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: requests<3,>=2.25.1 in /usr/local/lib/python3.10/dist-packages (from aicrowd-cli) (2.27.1)\n",
            "Collecting requests-toolbelt<1,>=0.9.1 (from aicrowd-cli)\n",
            "  Downloading requests_toolbelt-0.10.1-py2.py3-none-any.whl (54 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m54.5/54.5 kB\u001b[0m \u001b[31m4.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting rich<11,>=10.0.0 (from aicrowd-cli)\n",
            "  Downloading rich-10.16.2-py3-none-any.whl (214 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m214.4/214.4 kB\u001b[0m \u001b[31m17.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: toml<1,>=0.10.2 in /usr/local/lib/python3.10/dist-packages (from aicrowd-cli) (0.10.2)\n",
            "Requirement already satisfied: tqdm<5,>=4.56.0 in /usr/local/lib/python3.10/dist-packages (from aicrowd-cli) (4.65.0)\n",
            "Collecting pyzmq==22.1.0 (from aicrowd-cli)\n",
            "  Downloading pyzmq-22.1.0.tar.gz (1.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m34.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting python-slugify<6,>=5.0.0 (from aicrowd-cli)\n",
            "  Downloading python_slugify-5.0.2-py2.py3-none-any.whl (6.7 kB)\n",
            "Collecting semver<3,>=2.13.0 (from aicrowd-cli)\n",
            "  Downloading semver-2.13.0-py2.py3-none-any.whl (12 kB)\n",
            "Collecting gitdb<5,>=4.0.1 (from GitPython==3.1.18->aicrowd-cli)\n",
            "  Downloading gitdb-4.0.10-py3-none-any.whl (62 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.7/62.7 kB\u001b[0m \u001b[31m6.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: text-unidecode>=1.3 in /usr/local/lib/python3.10/dist-packages (from python-slugify<6,>=5.0.0->aicrowd-cli) (1.3)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.25.1->aicrowd-cli) (1.26.16)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.25.1->aicrowd-cli) (2023.7.22)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.25.1->aicrowd-cli) (2.0.12)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.25.1->aicrowd-cli) (3.4)\n",
            "Collecting colorama<0.5.0,>=0.4.0 (from rich<11,>=10.0.0->aicrowd-cli)\n",
            "  Downloading colorama-0.4.6-py2.py3-none-any.whl (25 kB)\n",
            "Collecting commonmark<0.10.0,>=0.9.0 (from rich<11,>=10.0.0->aicrowd-cli)\n",
            "  Downloading commonmark-0.9.1-py2.py3-none-any.whl (51 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m51.1/51.1 kB\u001b[0m \u001b[31m4.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pygments<3.0.0,>=2.6.0 in /usr/local/lib/python3.10/dist-packages (from rich<11,>=10.0.0->aicrowd-cli) (2.14.0)\n",
            "Collecting smmap<6,>=3.0.1 (from gitdb<5,>=4.0.1->GitPython==3.1.18->aicrowd-cli)\n",
            "  Downloading smmap-5.0.0-py3-none-any.whl (24 kB)\n",
            "Building wheels for collected packages: pyzmq\n",
            "  Building wheel for pyzmq (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pyzmq: filename=pyzmq-22.1.0-cp310-cp310-linux_x86_64.whl size=1365937 sha256=be5047699a67736c5cc0eb3e9fe5430a474df20d7c516fecd289ee155918ab12\n",
            "  Stored in directory: /root/.cache/pip/wheels/2d/e1/f6/45f17238de3475af44fca25559ef8d2f70f2b230d941da499e\n",
            "Successfully built pyzmq\n",
            "Installing collected packages: commonmark, smmap, semver, pyzmq, python-slugify, colorama, click, rich, requests-toolbelt, gitdb, GitPython, aicrowd-cli\n",
            "  Attempting uninstall: pyzmq\n",
            "    Found existing installation: pyzmq 23.2.1\n",
            "    Uninstalling pyzmq-23.2.1:\n",
            "      Successfully uninstalled pyzmq-23.2.1\n",
            "  Attempting uninstall: python-slugify\n",
            "    Found existing installation: python-slugify 8.0.1\n",
            "    Uninstalling python-slugify-8.0.1:\n",
            "      Successfully uninstalled python-slugify-8.0.1\n",
            "  Attempting uninstall: click\n",
            "    Found existing installation: click 8.1.6\n",
            "    Uninstalling click-8.1.6:\n",
            "      Successfully uninstalled click-8.1.6\n",
            "  Attempting uninstall: rich\n",
            "    Found existing installation: rich 13.4.2\n",
            "    Uninstalling rich-13.4.2:\n",
            "      Successfully uninstalled rich-13.4.2\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "fiona 1.9.4.post1 requires click~=8.0, but you have click 7.1.2 which is incompatible.\n",
            "flask 2.2.5 requires click>=8.0, but you have click 7.1.2 which is incompatible.\n",
            "flax 0.7.0 requires rich>=11.1, but you have rich 10.16.2 which is incompatible.\n",
            "pip-tools 6.13.0 requires click>=8, but you have click 7.1.2 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[0mSuccessfully installed GitPython-3.1.18 aicrowd-cli-0.1.15 click-7.1.2 colorama-0.4.6 commonmark-0.9.1 gitdb-4.0.10 python-slugify-5.0.2 pyzmq-22.1.0 requests-toolbelt-0.10.1 rich-10.16.2 semver-2.13.0 smmap-5.0.0\n"
          ]
        },
        {
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "click",
                  "zmq"
                ]
              }
            }
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "!pip install aicrowd-cli"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true,
          "base_uri": "https://localhost:8080/"
        },
        "id": "BjoYZ8LlIx7m",
        "outputId": "b2165263-c9a8-4873-ca4f-1811b431dbc7"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Please login here: \u001b[34m\u001b[1m\u001b[4mhttps://api.aicrowd.com/auth/MBECItuxsEOhVpzCz1fhEk9sCzd-3Q9kPZhujNadKN0\u001b[0m\n",
            "/usr/bin/xdg-open: 882: www-browser: not found\n",
            "/usr/bin/xdg-open: 882: links2: not found\n",
            "/usr/bin/xdg-open: 882: elinks: not found\n",
            "/usr/bin/xdg-open: 882: links: not found\n",
            "/usr/bin/xdg-open: 882: lynx: not found\n",
            "/usr/bin/xdg-open: 882: w3m: not found\n",
            "xdg-open: no method available for opening 'https://api.aicrowd.com/auth/MBECItuxsEOhVpzCz1fhEk9sCzd-3Q9kPZhujNadKN0'\n",
            "\u001b[31mLogin Error: Couldn't login. Max retries exceeded\u001b[0m\u001b[33m\n",
            "Please try logging in again\u001b[0m\n"
          ]
        }
      ],
      "source": [
        "!aicrowd login"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true,
          "base_uri": "https://localhost:8080/"
        },
        "id": "9gGjEwWWIx5S",
        "outputId": "9623e0eb-970c-4f2a-8079-ae0fa4d85781"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\u001b[31mCLI Error: Config property aicrowd_api_key not set\u001b[0m\u001b[33m\n",
            "\u001b[31mPlease run\u001b[0m \u001b[33maicrowd login\u001b[0m \u001b[31mto login.\u001b[0m\u001b[0m\n"
          ]
        }
      ],
      "source": [
        "!aicrowd dataset download -c esci-challenge-for-improving-product-search"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WvuvZc1_x8Gd"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DczeG3N5x8hL"
      },
      "source": [
        "## Decompress and Copy"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "67uJtS-MtdmV",
        "outputId": "9dd7e299-c9a5-4287-f67b-0a8182061970"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Archive:  /content/train-v0.3.csv.zip\n",
            "  inflating: /content/dataset/data/processed/public/task_3_product_substitute_identification/train-v0.3.csv  \n"
          ]
        }
      ],
      "source": [
        "!unzip \"/content/train-v0.3.csv.zip\" -d \"/content/dataset/\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "plKnZRUSJJFc",
        "outputId": "95b10089-b493-4a83-e26a-ce4a15402962"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Archive:  /content/product_catalogue-v0.3.csv.zip\n",
            "  inflating: /content/dataset/data/processed/public/task_3_product_substitute_identification/product_catalogue-v0.3.csv  \n",
            "\n"
          ]
        }
      ],
      "source": [
        "!unzip \"/content/product_catalogue-v0.3.csv.zip\" -d \"/content/dataset/\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "amZh8KfKtqH0"
      },
      "outputs": [],
      "source": [
        "!cp /content/dataset/data/processed/public/task_3_product_substitute_identification/train-v0.3.csv /content/drive/MyDrive/datasets/kdd_cup"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "kCWUL9aFKC4U"
      },
      "outputs": [],
      "source": [
        "!cp /content/dataset/data/processed/public/task_3_product_substitute_identification/product_catalogue-v0.3.csv /content/drive/MyDrive/datasets/kdd_cup"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ogDlZqTX0Cf-"
      },
      "source": [
        "## Data Loading"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "gnQdyIhnx4Q7"
      },
      "outputs": [],
      "source": [
        "filepath_train='/content/drive/MyDrive/datasets/kdd_cup/train-v0.3_1.csv'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rZXDn4cBx4T6"
      },
      "outputs": [],
      "source": [
        "filepath_catalogue='/content/drive/MyDrive/datasets/kdd_cup/product_catalogue-v0.3.csv'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "H8I6OELSx4WT"
      },
      "outputs": [],
      "source": [
        "df_catalogue=pd.read_csv(filepath_catalogue)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "NSmz3WYrx4Yt"
      },
      "outputs": [],
      "source": [
        "df_train=pd.read_csv(filepath_train)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "WVRcwFpYx4bV",
        "outputId": "88d4c543-7188-4367-81e2-b561aee541ff"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-87e11232-c73e-4abb-8d69-46454d578872\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>query_id</th>\n",
              "      <th>query</th>\n",
              "      <th>query_locale</th>\n",
              "      <th>product_id</th>\n",
              "      <th>esci_label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>572650</th>\n",
              "      <td>26004</td>\n",
              "      <td>0係</td>\n",
              "      <td>jp</td>\n",
              "      <td>B00FNJ1CTQ</td>\n",
              "      <td>substitute</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>572651</th>\n",
              "      <td>26004</td>\n",
              "      <td>0係</td>\n",
              "      <td>jp</td>\n",
              "      <td>B00FW60P84</td>\n",
              "      <td>irrelevant</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>572652</th>\n",
              "      <td>26004</td>\n",
              "      <td>0係</td>\n",
              "      <td>jp</td>\n",
              "      <td>B00GAYCNDM</td>\n",
              "      <td>substitute</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>572653</th>\n",
              "      <td>26004</td>\n",
              "      <td>0係</td>\n",
              "      <td>jp</td>\n",
              "      <td>B00MFBKQKG</td>\n",
              "      <td>substitute</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>572654</th>\n",
              "      <td>26004</td>\n",
              "      <td>0係</td>\n",
              "      <td>jp</td>\n",
              "      <td>B01G1YXU28</td>\n",
              "      <td>irrelevant</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>781739</th>\n",
              "      <td>33803</td>\n",
              "      <td>針なしほっちきす</td>\n",
              "      <td>jp</td>\n",
              "      <td>B08XGQ9RH7</td>\n",
              "      <td>substitute</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>781740</th>\n",
              "      <td>33803</td>\n",
              "      <td>針なしほっちきす</td>\n",
              "      <td>jp</td>\n",
              "      <td>B0987RGRF2</td>\n",
              "      <td>exact</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>781741</th>\n",
              "      <td>33803</td>\n",
              "      <td>針なしほっちきす</td>\n",
              "      <td>jp</td>\n",
              "      <td>B099NFJWP6</td>\n",
              "      <td>exact</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>781742</th>\n",
              "      <td>33803</td>\n",
              "      <td>針なしほっちきす</td>\n",
              "      <td>jp</td>\n",
              "      <td>B09F3B413J</td>\n",
              "      <td>exact</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>781743</th>\n",
              "      <td>33803</td>\n",
              "      <td>針なしほっちきす</td>\n",
              "      <td>jp</td>\n",
              "      <td>B09F3F1KDP</td>\n",
              "      <td>exact</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>209094 rows × 5 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-87e11232-c73e-4abb-8d69-46454d578872')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-87e11232-c73e-4abb-8d69-46454d578872 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-87e11232-c73e-4abb-8d69-46454d578872');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "        query_id     query query_locale  product_id  esci_label\n",
              "572650     26004        0係           jp  B00FNJ1CTQ  substitute\n",
              "572651     26004        0係           jp  B00FW60P84  irrelevant\n",
              "572652     26004        0係           jp  B00GAYCNDM  substitute\n",
              "572653     26004        0係           jp  B00MFBKQKG  substitute\n",
              "572654     26004        0係           jp  B01G1YXU28  irrelevant\n",
              "...          ...       ...          ...         ...         ...\n",
              "781739     33803  針なしほっちきす           jp  B08XGQ9RH7  substitute\n",
              "781740     33803  針なしほっちきす           jp  B0987RGRF2       exact\n",
              "781741     33803  針なしほっちきす           jp  B099NFJWP6       exact\n",
              "781742     33803  針なしほっちきす           jp  B09F3B413J       exact\n",
              "781743     33803  針なしほっちきす           jp  B09F3F1KDP       exact\n",
              "\n",
              "[209094 rows x 5 columns]"
            ]
          },
          "execution_count": 73,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df_train.loc[df_train['query_locale'] == 'jp']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "nPOte2AIx4d6",
        "outputId": "19c71092-e058-49c6-a4e1-5312203b54a8"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-25acc418-7c65-4f46-8ecd-20fb6a846506\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>product_id</th>\n",
              "      <th>product_title</th>\n",
              "      <th>product_description</th>\n",
              "      <th>product_bullet_point</th>\n",
              "      <th>product_brand</th>\n",
              "      <th>product_color_name</th>\n",
              "      <th>product_locale</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>B079VKKJN7</td>\n",
              "      <td>11 Degrees de los Hombres Playera con Logo, Ne...</td>\n",
              "      <td>Esta playera con el logo de la marca Carrier d...</td>\n",
              "      <td>11 Degrees Negro Playera con logo\\nA estrenar ...</td>\n",
              "      <td>11 Degrees</td>\n",
              "      <td>Negro</td>\n",
              "      <td>es</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>B079Y9VRKS</td>\n",
              "      <td>Camiseta Eleven Degrees Core TS White (M)</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>11 Degrees</td>\n",
              "      <td>Blanco</td>\n",
              "      <td>es</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>B07DP4LM9H</td>\n",
              "      <td>11 Degrees de los Hombres Core Pull Over Hoodi...</td>\n",
              "      <td>La sudadera con capucha Core Pull Over de 11 G...</td>\n",
              "      <td>11 Degrees Azul Core Pull Over Hoodie\\nA estre...</td>\n",
              "      <td>11 Degrees</td>\n",
              "      <td>Azul</td>\n",
              "      <td>es</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>B07G37B9HP</td>\n",
              "      <td>11 Degrees Poli Panel Track Pant XL Black</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>11 Degrees</td>\n",
              "      <td>NaN</td>\n",
              "      <td>es</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>B07LCTGDHY</td>\n",
              "      <td>11 Degrees Gorra Trucker Negro OSFA (Talla úni...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>11 Degrees</td>\n",
              "      <td>Negro (</td>\n",
              "      <td>es</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1815211</th>\n",
              "      <td>B09FJYBVKS</td>\n",
              "      <td>スポーツブラ 揺れない ヨガウェア レディース トップス 通気性 ナイトブラ スポーツブラジ...</td>\n",
              "      <td>&lt;p&gt;【注意事項】&lt;/p&gt;&lt;p&gt;※商品内容以外のアクセサリー・小物等は付属しません.&lt;/p&gt;...</td>\n",
              "      <td>シンプルのデザインで合わせやすいです。コート、半袖、シャツを合わせて、家で着てもいい、寝てい...</td>\n",
              "      <td>Doworspaw</td>\n",
              "      <td>ベージュ</td>\n",
              "      <td>jp</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1815212</th>\n",
              "      <td>B09FTJ3S1N</td>\n",
              "      <td>ブラジャー レディース 補正ブラ Gabrioir 揺れない 大きいサイズ バストアップ美胸...</td>\n",
              "      <td>&lt;p&gt;【配送】&lt;/p&gt;&lt;p&gt;普通には2-3日後出荷できますが。出荷した後15-25日ぐらい届...</td>\n",
              "      <td>【いろんな場合に適用】軽い運動時、家事、外出、長時間移動、温泉旅行などに活用なセクシルームブ...</td>\n",
              "      <td>Gabrioir</td>\n",
              "      <td>パープル</td>\n",
              "      <td>jp</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1815213</th>\n",
              "      <td>B09G72LDQZ</td>\n",
              "      <td>レディース ブラジャー 補正ブラ トップス Gabrioir 大きいサイズ 揺れない 脇高設...</td>\n",
              "      <td>&lt;p&gt;【配送】&lt;/p&gt;&lt;p&gt;普通には2-3日後出荷できますが。出荷した後15-25日ぐらい届...</td>\n",
              "      <td>【いろんな場合に適用】軽い運動時、家事、外出、長時間移動、温泉旅行などに活用なセクシルームブ...</td>\n",
              "      <td>Gabrioir</td>\n",
              "      <td>ベージュ</td>\n",
              "      <td>jp</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1815214</th>\n",
              "      <td>B09GBGRTPB</td>\n",
              "      <td>ナイトブラ ノンワイヤーブラ ブラジャー レディース Gabrioir 大きいサイズ パット...</td>\n",
              "      <td>&lt;p&gt;【配送】&lt;/p&gt;&lt;p&gt;普通には2-3日後出荷できますが。出荷した後15-25日ぐらい届...</td>\n",
              "      <td>【いろんな場合に適用】軽い運動時、家事、外出、長時間移動、温泉旅行などに活用なセクシルームブ...</td>\n",
              "      <td>Gabrioir</td>\n",
              "      <td>パープル</td>\n",
              "      <td>jp</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1815215</th>\n",
              "      <td>B09GBKN83Q</td>\n",
              "      <td>インナーキャミソール レディース ブラトップ キャミソール Gabrioir ノーワイヤー ...</td>\n",
              "      <td>&lt;p&gt;【配送】&lt;/p&gt;&lt;p&gt;普通には2-3日後出荷できますが。出荷した後15-25日ぐらい届...</td>\n",
              "      <td>【いろんな場合に適用】軽い運動時、家事、外出、長時間移動、温泉旅行などに活用なセクシルームブ...</td>\n",
              "      <td>Gabrioir</td>\n",
              "      <td>ブラック</td>\n",
              "      <td>jp</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>1815216 rows × 7 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-25acc418-7c65-4f46-8ecd-20fb6a846506')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-25acc418-7c65-4f46-8ecd-20fb6a846506 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-25acc418-7c65-4f46-8ecd-20fb6a846506');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "         product_id                                      product_title  \\\n",
              "0        B079VKKJN7  11 Degrees de los Hombres Playera con Logo, Ne...   \n",
              "1        B079Y9VRKS          Camiseta Eleven Degrees Core TS White (M)   \n",
              "2        B07DP4LM9H  11 Degrees de los Hombres Core Pull Over Hoodi...   \n",
              "3        B07G37B9HP          11 Degrees Poli Panel Track Pant XL Black   \n",
              "4        B07LCTGDHY  11 Degrees Gorra Trucker Negro OSFA (Talla úni...   \n",
              "...             ...                                                ...   \n",
              "1815211  B09FJYBVKS  スポーツブラ 揺れない ヨガウェア レディース トップス 通気性 ナイトブラ スポーツブラジ...   \n",
              "1815212  B09FTJ3S1N  ブラジャー レディース 補正ブラ Gabrioir 揺れない 大きいサイズ バストアップ美胸...   \n",
              "1815213  B09G72LDQZ  レディース ブラジャー 補正ブラ トップス Gabrioir 大きいサイズ 揺れない 脇高設...   \n",
              "1815214  B09GBGRTPB  ナイトブラ ノンワイヤーブラ ブラジャー レディース Gabrioir 大きいサイズ パット...   \n",
              "1815215  B09GBKN83Q  インナーキャミソール レディース ブラトップ キャミソール Gabrioir ノーワイヤー ...   \n",
              "\n",
              "                                       product_description  \\\n",
              "0        Esta playera con el logo de la marca Carrier d...   \n",
              "1                                                      NaN   \n",
              "2        La sudadera con capucha Core Pull Over de 11 G...   \n",
              "3                                                      NaN   \n",
              "4                                                      NaN   \n",
              "...                                                    ...   \n",
              "1815211  <p>【注意事項】</p><p>※商品内容以外のアクセサリー・小物等は付属しません.</p>...   \n",
              "1815212  <p>【配送】</p><p>普通には2-3日後出荷できますが。出荷した後15-25日ぐらい届...   \n",
              "1815213  <p>【配送】</p><p>普通には2-3日後出荷できますが。出荷した後15-25日ぐらい届...   \n",
              "1815214  <p>【配送】</p><p>普通には2-3日後出荷できますが。出荷した後15-25日ぐらい届...   \n",
              "1815215  <p>【配送】</p><p>普通には2-3日後出荷できますが。出荷した後15-25日ぐらい届...   \n",
              "\n",
              "                                      product_bullet_point product_brand  \\\n",
              "0        11 Degrees Negro Playera con logo\\nA estrenar ...    11 Degrees   \n",
              "1                                                      NaN    11 Degrees   \n",
              "2        11 Degrees Azul Core Pull Over Hoodie\\nA estre...    11 Degrees   \n",
              "3                                                      NaN    11 Degrees   \n",
              "4                                                      NaN    11 Degrees   \n",
              "...                                                    ...           ...   \n",
              "1815211  シンプルのデザインで合わせやすいです。コート、半袖、シャツを合わせて、家で着てもいい、寝てい...     Doworspaw   \n",
              "1815212  【いろんな場合に適用】軽い運動時、家事、外出、長時間移動、温泉旅行などに活用なセクシルームブ...      Gabrioir   \n",
              "1815213  【いろんな場合に適用】軽い運動時、家事、外出、長時間移動、温泉旅行などに活用なセクシルームブ...      Gabrioir   \n",
              "1815214  【いろんな場合に適用】軽い運動時、家事、外出、長時間移動、温泉旅行などに活用なセクシルームブ...      Gabrioir   \n",
              "1815215  【いろんな場合に適用】軽い運動時、家事、外出、長時間移動、温泉旅行などに活用なセクシルームブ...      Gabrioir   \n",
              "\n",
              "        product_color_name product_locale  \n",
              "0                    Negro             es  \n",
              "1                   Blanco             es  \n",
              "2                     Azul             es  \n",
              "3                      NaN             es  \n",
              "4                  Negro (             es  \n",
              "...                    ...            ...  \n",
              "1815211               ベージュ             jp  \n",
              "1815212               パープル             jp  \n",
              "1815213               ベージュ             jp  \n",
              "1815214               パープル             jp  \n",
              "1815215               ブラック             jp  \n",
              "\n",
              "[1815216 rows x 7 columns]"
            ]
          },
          "execution_count": 12,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df_catalogue"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ZbXNRBzkx4fz"
      },
      "outputs": [],
      "source": [
        "def get_product_title(product_id):\n",
        "  options=[]\n",
        "  options.append(str(product_id))\n",
        "  matched_row=df_catalogue.loc[df_catalogue['product_id'].isin(options)]\n",
        "  #print(int(str(matched_row['product_title']).split(\"   \")[0]))\n",
        "  return matched_row['product_title'][int(str(matched_row['product_title']).split(\"   \")[0])]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "2g6O3lkDzlgG",
        "outputId": "9787857f-84b7-44b5-a0cc-57a6bb23f140"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'面部保护 防雾 耐用 5个 透明面罩 健康盾牌 塑料面巾 完全可见 透气 可重复使用 成人 口罩 方便 打招呼 防护 (Clear)'"
            ]
          },
          "execution_count": 64,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "get_product_title(\"B08N6NCKRX\")### irrelevant"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7VUizQVZzlm2"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LVQaVIEHVJ4W"
      },
      "source": [
        "## Dataset Generation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ST2lnPhgLj-L"
      },
      "outputs": [],
      "source": [
        "header = ['query', 'product', 'label']\n",
        "\n",
        "with open('/content/drive/MyDrive/datasets/kdd_cup/csv_file_9.csv', 'w', encoding='UTF8') as f:\n",
        "\n",
        "  # create the csv writer\n",
        "  writer = csv.writer(f)\n",
        "\n",
        "  # write the header\n",
        "  writer.writerow(header)\n",
        "\n",
        "  for i in range(781000,781636):\n",
        "\n",
        "    data=[df_train[i:i+1]['query'][i],get_product_title(df_train[i:i+1]['product_id'][i]),df_train[i:i+1]['esci_label'][i]]\n",
        "    # write the data\n",
        "    writer.writerow(data)\n",
        ""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xOX5gW2lWZYm"
      },
      "outputs": [],
      "source": [
        "filepath=\"/content/drive/MyDrive/datasets/kdd_cup/csv_file_9.csv\"\n",
        "train_data=pd.read_csv(filepath)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1481
        },
        "id": "99TbLjQZLkAr",
        "outputId": "1363a0a1-8649-4e34-e88a-d52af0fa1f88"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-57f28b4f-3a3a-40cb-8682-070a3a26c4f1\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>query</th>\n",
              "      <th>product</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>ベビーマグ ちゃん 効果 なし</td>\n",
              "      <td>ピジョン Pigeon マグマグ スパウト 200ml 5ヵ月頃から</td>\n",
              "      <td>exact</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>ベビーマグ ちゃん 効果 なし</td>\n",
              "      <td>コンビ テテオ teteo マグベビー バルーン Neo</td>\n",
              "      <td>exact</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>ベビーマグ ちゃん 効果 なし</td>\n",
              "      <td>コンビ テテオ teteo パーティマグ 離乳ナビセット Neo マルチカラー 1個アソート...</td>\n",
              "      <td>exact</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>ベビーマグ ちゃん 効果 なし</td>\n",
              "      <td>レック アンパンマン ワンタッチ ストローマグ 300ml</td>\n",
              "      <td>irrelevant</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>ベビーマグ ちゃん 効果 なし</td>\n",
              "      <td>ピジョン Pigeon マグマグ セット 3ヵ月頃から 発達にあわせた4種類の飲み方トレーニング</td>\n",
              "      <td>substitute</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>631</th>\n",
              "      <td>肩なしブラジャー</td>\n",
              "      <td>スポーツブラ Lalaly【2枚組】 スポーツブラ 速乾 伸張 パット付き 脇高 はみ肉スッ...</td>\n",
              "      <td>substitute</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>632</th>\n",
              "      <td>肩なしブラジャー</td>\n",
              "      <td>2枚組スポーツブラ T-wilker 通気性 揺れない パッド入り ノンワイヤーブラ ナイト...</td>\n",
              "      <td>exact</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>633</th>\n",
              "      <td>肩なしブラジャー</td>\n",
              "      <td>（AQUIVER）肩ひも無し ブラ ストラップレス ブラジャー 肩ひもなし 背中見せ インナ...</td>\n",
              "      <td>exact</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>634</th>\n",
              "      <td>肩なしブラジャー</td>\n",
              "      <td>ずれにくい 6色6サイズ展開 レディース チューブトップ ブラ ブラジャー ナイトブラ ベア...</td>\n",
              "      <td>exact</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>635</th>\n",
              "      <td>肩なしブラジャー</td>\n",
              "      <td>[ウイング/ワコール] ノンワイヤーブラジャー 脇高パッドでシルエットをキープ 【フィットト...</td>\n",
              "      <td>substitute</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>636 rows × 3 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-57f28b4f-3a3a-40cb-8682-070a3a26c4f1')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-57f28b4f-3a3a-40cb-8682-070a3a26c4f1 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-57f28b4f-3a3a-40cb-8682-070a3a26c4f1');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "               query                                            product  \\\n",
              "0    ベビーマグ ちゃん 効果 なし                 ピジョン Pigeon マグマグ スパウト 200ml 5ヵ月頃から   \n",
              "1    ベビーマグ ちゃん 効果 なし                       コンビ テテオ teteo マグベビー バルーン Neo   \n",
              "2    ベビーマグ ちゃん 効果 なし  コンビ テテオ teteo パーティマグ 離乳ナビセット Neo マルチカラー 1個アソート...   \n",
              "3    ベビーマグ ちゃん 効果 なし                      レック アンパンマン ワンタッチ ストローマグ 300ml   \n",
              "4    ベビーマグ ちゃん 効果 なし   ピジョン Pigeon マグマグ セット 3ヵ月頃から 発達にあわせた4種類の飲み方トレーニング   \n",
              "..               ...                                                ...   \n",
              "631         肩なしブラジャー  スポーツブラ Lalaly【2枚組】 スポーツブラ 速乾 伸張 パット付き 脇高 はみ肉スッ...   \n",
              "632         肩なしブラジャー  2枚組スポーツブラ T-wilker 通気性 揺れない パッド入り ノンワイヤーブラ ナイト...   \n",
              "633         肩なしブラジャー  （AQUIVER）肩ひも無し ブラ ストラップレス ブラジャー 肩ひもなし 背中見せ インナ...   \n",
              "634         肩なしブラジャー  ずれにくい 6色6サイズ展開 レディース チューブトップ ブラ ブラジャー ナイトブラ ベア...   \n",
              "635         肩なしブラジャー  [ウイング/ワコール] ノンワイヤーブラジャー 脇高パッドでシルエットをキープ 【フィットト...   \n",
              "\n",
              "          label  \n",
              "0         exact  \n",
              "1         exact  \n",
              "2         exact  \n",
              "3    irrelevant  \n",
              "4    substitute  \n",
              "..          ...  \n",
              "631  substitute  \n",
              "632       exact  \n",
              "633       exact  \n",
              "634       exact  \n",
              "635  substitute  \n",
              "\n",
              "[636 rows x 3 columns]"
            ]
          },
          "execution_count": 68,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "train_data"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TsB7knUXFgCB"
      },
      "source": [
        "## Data Preprocessing"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2k4skxqvaugD",
        "outputId": "8d2afd9e-61cd-4c59-8c2f-21db8ccf5302"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "['/content/drive/MyDrive/datasets/kdd_cup/csv_file_0.csv', '/content/drive/MyDrive/datasets/kdd_cup/csv_file_1.csv', '/content/drive/MyDrive/datasets/kdd_cup/csv_file_2.csv', '/content/drive/MyDrive/datasets/kdd_cup/csv_file_3.csv', '/content/drive/MyDrive/datasets/kdd_cup/csv_file_4.csv', '/content/drive/MyDrive/datasets/kdd_cup/csv_file_5.csv', '/content/drive/MyDrive/datasets/kdd_cup/csv_file_6.csv', '/content/drive/MyDrive/datasets/kdd_cup/csv_file_7.csv', '/content/drive/MyDrive/datasets/kdd_cup/csv_file_8.csv', '/content/drive/MyDrive/datasets/kdd_cup/csv_file_9.csv']\n"
          ]
        }
      ],
      "source": [
        "filepaths=[\"/content/drive/MyDrive/datasets/kdd_cup/csv_file_\"+str(i)+\".csv\" for i in range(10)]\n",
        "print(filepaths)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 185,
          "referenced_widgets": [
            "17e4888ada794ca58efe8139c1915bde",
            "e86f14c6ca0540c3b43bbf286c082fa5",
            "24dd3128d5dd42588c2818f545a1729f",
            "b6e5c71a5cdb46c58d61cb9075b538cd",
            "a14327416e534c2c82590d3bf9f53a56",
            "c7ef2649314844b996706998f548b8a2",
            "290b049faa1d47fe94918894589df21f",
            "be106420c9e248c3af0d20dd26347e36",
            "baf23296164e4fb49055190dc27d73e5",
            "78630f0a9bea42ea815111db44922a98",
            "1b983274bbaf48ddb0e27036e7e3d9cb",
            "557eb4180d0c41f093249bb0dab2a66b",
            "f61c4e25c8dd491faebb567c3414b487",
            "331daf2ca76b4752b10406fc20783c12",
            "107af6e8e531449881cf234a518d962a",
            "a974c19cb2d74f85aa2d6175b7b054b4",
            "f7a4ff94b0a94d129498dfbb41e81778",
            "23f81baef07346db9992662c4e1016a5",
            "60782d79f0244c6b9d1071c3effe5075",
            "3ef5dadae87e4f11b5c5ca5001ebb61d",
            "5362841eba8a4ee39b9e28ab7df0812a",
            "df0b06a1893a4193a8fd5c3683cdbf6a",
            "ccb197e162fd4b969b132ac985527a22",
            "b84d233573e24760839453ab88ecf8b4",
            "41dafa3fe83f495189a5b675ad4a46c4",
            "316e0036819541a798464be65a1c5a4b",
            "a465957f9c23410bbc42b99f53112905",
            "516475b9825445a89135230c0ebc689c",
            "23556057e0064150a3673b2752ac0543",
            "160f9a290bf444ebb76e1cb6e46c272b",
            "2a14f9fe2ab84481bbfae2f497a2e97b",
            "a222503085d142669593dd5fe8ef2526",
            "75db01a35994498389abbe7e713e75c4",
            "0af335553d2c42349e4641a0018dfe7a",
            "cbe1f26047bf4adb8a66b9d0bd1c0a77",
            "865f50d0e9f7418c97f9da9d4c0c021e",
            "4c32b2e5411a4727a4aa7f860cbca221",
            "4ea61ecd7e1a4678b546e60e02cdf400",
            "f7f0278a0c6c4fc28925e9d3c352c67a",
            "283d37bc43f949e4aa8ac1b50b2dd239",
            "994a985977974fe99398974b4db2c2e7",
            "d32142e28bdb475ca1e4e57e4a2a48a7",
            "bde0b764bd6a4281b16943fd93478f73",
            "4a2b49a2ae1c4dfab7daaff9a0fccc11"
          ]
        },
        "id": "F7yaOKv8bq8g",
        "outputId": "5ffa39c4-454c-48fb-cae3-5f44c03cbd27"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:datasets.builder:Using custom data configuration default-040c829cfb31fc59\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Downloading and preparing dataset csv/default to /root/.cache/huggingface/datasets/csv/default-040c829cfb31fc59/0.0.0/6b34fb8fcf56f7c8ba51dc895bfa2bfbe43546f190a60fcf74bb5e8afdcc2317...\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "17e4888ada794ca58efe8139c1915bde",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading data files:   0%|          | 0/1 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "557eb4180d0c41f093249bb0dab2a66b",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Extracting data files:   0%|          | 0/1 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "ccb197e162fd4b969b132ac985527a22",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "0 tables [00:00, ? tables/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Dataset csv downloaded and prepared to /root/.cache/huggingface/datasets/csv/default-040c829cfb31fc59/0.0.0/6b34fb8fcf56f7c8ba51dc895bfa2bfbe43546f190a60fcf74bb5e8afdcc2317. Subsequent calls will reuse this data.\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "0af335553d2c42349e4641a0018dfe7a",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "  0%|          | 0/1 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "dataset = load_dataset('csv', data_files=filepaths)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "j9O46mgZ7wCv",
        "outputId": "c8e2e183-d05d-43ff-a8db-297f47556144"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "DatasetDict({\n",
              "    train: Dataset({\n",
              "        features: ['query', 'product', 'label'],\n",
              "        num_rows: 731636\n",
              "    })\n",
              "})"
            ]
          },
          "execution_count": 6,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oo8deXsxdyBZ",
        "outputId": "67cf5d8d-8ff2-49b1-eefb-b9204e6b33d2"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{'query': '# 2 pencils not sharpened',\n",
              " 'product': 'Ticonderoga Beginner Pencils, Wood-Cased #2 HB Soft, With Eraser, Yellow, 12-Pack (13308)',\n",
              " 'label': 'exact'}"
            ]
          },
          "execution_count": 7,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "dataset['train'][0]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "BNcP-EFNZhlm"
      },
      "outputs": [],
      "source": [
        "def get_label(label):\n",
        "  if label=='exact':\n",
        "    return 1.0\n",
        "  elif label=='substitute':\n",
        "    return 0.7\n",
        "  elif label=='complement':\n",
        "    return 0.5\n",
        "  else:\n",
        "    return 0.0"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "zOMxWvGLZhoM"
      },
      "outputs": [],
      "source": [
        "model_id=\"sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2\"\n",
        "tokenizer = AutoTokenizer.from_pretrained(model_id)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "wWpNeVJta8na"
      },
      "outputs": [],
      "source": [
        "def preprocess(dataset):\n",
        "  if dataset['product']==None:\n",
        "    dataset['product']=dataset['query']\n",
        "\n",
        "  dataset['input_ids_query']=[]\n",
        "  dataset['token_type_ids_query']=[]\n",
        "  dataset['attention_mask_query']=[]\n",
        "\n",
        "  dataset['input_ids_product']=[]\n",
        "  dataset['token_type_ids_product']=[]\n",
        "  dataset['attention_mask_product']=[]\n",
        "\n",
        "  tokenized_output_query=tokenizer(dataset['query'],max_length=MAX_LENGTH,padding='max_length',truncation=True)\n",
        "  tokenized_output_product=tokenizer(dataset['product'],max_length=MAX_LENGTH,padding='max_length',truncation=True)\n",
        "\n",
        "  dataset['input_ids_query'].append(tokenized_output_query['input_ids'])\n",
        "  dataset['token_type_ids_query'].append(tokenized_output_query['token_type_ids'])\n",
        "  dataset['attention_mask_query'].append(tokenized_output_query['attention_mask'])\n",
        "\n",
        "  dataset['input_ids_product'].append(tokenized_output_product['input_ids'])\n",
        "  dataset['token_type_ids_product'].append(tokenized_output_product['token_type_ids'])\n",
        "  dataset['attention_mask_product'].append(tokenized_output_product['attention_mask'])\n",
        "\n",
        "  dataset['label']=get_label(dataset['label'])\n",
        "\n",
        "  return dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 49,
          "referenced_widgets": [
            "4d0c4ed8eac54185b5188b421126f2c4",
            "ea41ad65203a4c75ad4ce3410d3abcc7",
            "001f6a8fa56846628ce364e6c18f724b",
            "4942db5c2d26408fa12aa884f76f04b8",
            "b9957450854e408ab6c2c0ec41dca942",
            "fefe375e7e0d4e90a3f3dc5e2cfd6cf7",
            "904bd5a359064db5ad790932f1c79791",
            "78b4b45d7273428c9a48ed1676fe8263",
            "ba10cdcacab24304a1dbe3fc01868e89",
            "06462707efcb4e6589b0e8b3e9241d3f",
            "4aa639a3c9ff4743a8ab11b0d9d6c11c"
          ]
        },
        "id": "mJx4-Ln9a8q9",
        "outputId": "67202c69-bb19-4815-97bd-01e570c9053f"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "4d0c4ed8eac54185b5188b421126f2c4",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "  0%|          | 0/731636 [00:00<?, ?ex/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "prep_dataset=dataset.map(preprocess)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AhVLYQ3xa8tJ",
        "outputId": "d269194f-6d63-4805-e99a-dcf6b511ff74"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{'query': ['# 2 pencils not sharpened'],\n",
              " 'product': ['BIC Evolution Cased Pencil, #2 Lead, Gray Barrel, 24-Count (PGEBP241-BLK)'],\n",
              " 'label': [0.7],\n",
              " 'input_ids_query': [[[101,\n",
              "    1001,\n",
              "    1016,\n",
              "    14745,\n",
              "    2015,\n",
              "    2025,\n",
              "    26694,\n",
              "    102,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0]]],\n",
              " 'token_type_ids_query': [[[0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0]]],\n",
              " 'attention_mask_query': [[[1,\n",
              "    1,\n",
              "    1,\n",
              "    1,\n",
              "    1,\n",
              "    1,\n",
              "    1,\n",
              "    1,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0]]],\n",
              " 'input_ids_product': [[[101,\n",
              "    12170,\n",
              "    2278,\n",
              "    6622,\n",
              "    2553,\n",
              "    2094,\n",
              "    14745,\n",
              "    1010,\n",
              "    1001,\n",
              "    1016,\n",
              "    2599,\n",
              "    1010,\n",
              "    3897,\n",
              "    8460,\n",
              "    1010,\n",
              "    2484,\n",
              "    1011,\n",
              "    4175,\n",
              "    1006,\n",
              "    18720,\n",
              "    15878,\n",
              "    2361,\n",
              "    18827,\n",
              "    2487,\n",
              "    1011,\n",
              "    1038,\n",
              "    13687,\n",
              "    1007,\n",
              "    102,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0]]],\n",
              " 'token_type_ids_product': [[[0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0]]],\n",
              " 'attention_mask_product': [[[1,\n",
              "    1,\n",
              "    1,\n",
              "    1,\n",
              "    1,\n",
              "    1,\n",
              "    1,\n",
              "    1,\n",
              "    1,\n",
              "    1,\n",
              "    1,\n",
              "    1,\n",
              "    1,\n",
              "    1,\n",
              "    1,\n",
              "    1,\n",
              "    1,\n",
              "    1,\n",
              "    1,\n",
              "    1,\n",
              "    1,\n",
              "    1,\n",
              "    1,\n",
              "    1,\n",
              "    1,\n",
              "    1,\n",
              "    1,\n",
              "    1,\n",
              "    1,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0,\n",
              "    0]]]}"
            ]
          },
          "execution_count": 12,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "prep_dataset['train'][21:22]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "C9GpqXAwa8vV"
      },
      "outputs": [],
      "source": [
        "tf_dataset = prep_dataset[\"train\"].to_tf_dataset(\n",
        "    columns=['input_ids_query', 'token_type_ids_query', 'attention_mask_query','input_ids_product', 'token_type_ids_product', 'attention_mask_product', 'label'],\n",
        "    shuffle=True,\n",
        "    batch_size=BATCH_SIZE,\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "29icJztva816",
        "outputId": "d384be38-ff1d-4758-f168-9cb1e271caa4"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "<PrefetchDataset element_spec={'label': TensorSpec(shape=(None,), dtype=tf.float32, name=None), 'input_ids_query': TensorSpec(shape=(None, 1, 64), dtype=tf.int64, name=None), 'token_type_ids_query': TensorSpec(shape=(None, 1, 64), dtype=tf.int64, name=None), 'attention_mask_query': TensorSpec(shape=(None, 1, 64), dtype=tf.int64, name=None), 'input_ids_product': TensorSpec(shape=(None, 1, 64), dtype=tf.int64, name=None), 'token_type_ids_product': TensorSpec(shape=(None, 1, 64), dtype=tf.int64, name=None), 'attention_mask_product': TensorSpec(shape=(None, 1, 64), dtype=tf.int64, name=None)}>"
            ]
          },
          "execution_count": 14,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "tf_dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZYCAPfsOa84L",
        "outputId": "794a774b-2b77-443a-9067-d207b3ba6673"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'label': <tf.Tensor: shape=(128,), dtype=float32, numpy=\n",
            "array([1. , 1. , 1. , 1. , 1. , 0. , 0.7, 1. , 0. , 0.5, 1. , 0.7, 0.7,\n",
            "       0.7, 1. , 0. , 1. , 1. , 0.7, 0.7, 0.7, 0.7, 1. , 0. , 1. , 0.7,\n",
            "       1. , 0.7, 0.7, 0.7, 1. , 1. , 1. , 0.7, 0.7, 0.7, 0.7, 0. , 0.7,\n",
            "       0.5, 0. , 0.7, 1. , 0.7, 0.7, 1. , 0.7, 1. , 0. , 1. , 0.5, 1. ,\n",
            "       0.7, 1. , 0. , 1. , 0.7, 0.7, 1. , 0. , 1. , 1. , 0.5, 1. , 0.7,\n",
            "       0. , 1. , 1. , 1. , 1. , 0.5, 0.7, 1. , 0.7, 1. , 1. , 0.7, 1. ,\n",
            "       0.7, 0.7, 0.7, 1. , 1. , 1. , 1. , 0.7, 1. , 1. , 0.7, 0.7, 0. ,\n",
            "       0.7, 1. , 0.5, 0.7, 0.7, 1. , 0. , 1. , 1. , 1. , 1. , 1. , 0. ,\n",
            "       0.7, 1. , 1. , 1. , 0.7, 1. , 1. , 1. , 1. , 1. , 0.7, 1. , 0. ,\n",
            "       1. , 0. , 1. , 1. , 0.7, 0.7, 0.7, 1. , 1. , 1. , 0.7],\n",
            "      dtype=float32)>, 'input_ids_query': <tf.Tensor: shape=(128, 1, 64), dtype=int64, numpy=\n",
            "array([[[  101,  4942,  4313, ...,     0,     0,     0]],\n",
            "\n",
            "       [[  101,  1713, 30260, ...,     0,     0,     0]],\n",
            "\n",
            "       [[  101,   100,   100, ...,     0,     0,     0]],\n",
            "\n",
            "       ...,\n",
            "\n",
            "       [[  101, 11163,  8873, ...,     0,     0,     0]],\n",
            "\n",
            "       [[  101,  3019, 25742, ...,     0,     0,     0]],\n",
            "\n",
            "       [[  101,  2214,  3212, ...,     0,     0,     0]]])>, 'token_type_ids_query': <tf.Tensor: shape=(128, 1, 64), dtype=int64, numpy=\n",
            "array([[[0, 0, 0, ..., 0, 0, 0]],\n",
            "\n",
            "       [[0, 0, 0, ..., 0, 0, 0]],\n",
            "\n",
            "       [[0, 0, 0, ..., 0, 0, 0]],\n",
            "\n",
            "       ...,\n",
            "\n",
            "       [[0, 0, 0, ..., 0, 0, 0]],\n",
            "\n",
            "       [[0, 0, 0, ..., 0, 0, 0]],\n",
            "\n",
            "       [[0, 0, 0, ..., 0, 0, 0]]])>, 'attention_mask_query': <tf.Tensor: shape=(128, 1, 64), dtype=int64, numpy=\n",
            "array([[[1, 1, 1, ..., 0, 0, 0]],\n",
            "\n",
            "       [[1, 1, 1, ..., 0, 0, 0]],\n",
            "\n",
            "       [[1, 1, 1, ..., 0, 0, 0]],\n",
            "\n",
            "       ...,\n",
            "\n",
            "       [[1, 1, 1, ..., 0, 0, 0]],\n",
            "\n",
            "       [[1, 1, 1, ..., 0, 0, 0]],\n",
            "\n",
            "       [[1, 1, 1, ..., 0, 0, 0]]])>, 'input_ids_product': <tf.Tensor: shape=(128, 1, 64), dtype=int64, numpy=\n",
            "array([[[  101,  4654, 25808, ...,     0,     0,     0]],\n",
            "\n",
            "       [[  101,   100,  3590, ...,     0,     0,     0]],\n",
            "\n",
            "       [[  101,   100,   100, ...,   102,     0,     0]],\n",
            "\n",
            "       ...,\n",
            "\n",
            "       [[  101,  2004, 11638, ...,     0,     0,     0]],\n",
            "\n",
            "       [[  101,  8292,  3669, ...,     0,     0,     0]],\n",
            "\n",
            "       [[  101,  9733,  6827, ...,     0,     0,     0]]])>, 'token_type_ids_product': <tf.Tensor: shape=(128, 1, 64), dtype=int64, numpy=\n",
            "array([[[0, 0, 0, ..., 0, 0, 0]],\n",
            "\n",
            "       [[0, 0, 0, ..., 0, 0, 0]],\n",
            "\n",
            "       [[0, 0, 0, ..., 0, 0, 0]],\n",
            "\n",
            "       ...,\n",
            "\n",
            "       [[0, 0, 0, ..., 0, 0, 0]],\n",
            "\n",
            "       [[0, 0, 0, ..., 0, 0, 0]],\n",
            "\n",
            "       [[0, 0, 0, ..., 0, 0, 0]]])>, 'attention_mask_product': <tf.Tensor: shape=(128, 1, 64), dtype=int64, numpy=\n",
            "array([[[1, 1, 1, ..., 0, 0, 0]],\n",
            "\n",
            "       [[1, 1, 1, ..., 0, 0, 0]],\n",
            "\n",
            "       [[1, 1, 1, ..., 1, 0, 0]],\n",
            "\n",
            "       ...,\n",
            "\n",
            "       [[1, 1, 1, ..., 0, 0, 0]],\n",
            "\n",
            "       [[1, 1, 1, ..., 0, 0, 0]],\n",
            "\n",
            "       [[1, 1, 1, ..., 0, 0, 0]]])>}\n"
          ]
        }
      ],
      "source": [
        "for i in tf_dataset.take(1):\n",
        "  print(i)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "e-A8yipua89C"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "g1VJmjYjkCFr"
      },
      "source": [
        "# Modeling"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "F50N4e6sWM-l"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Lc8U3V04a8_L",
        "outputId": "97e70943-92ce-4beb-8557-af0208e3701d"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "All model checkpoint layers were used when initializing TFBertModel.\n",
            "\n",
            "All the layers of TFBertModel were initialized from the model checkpoint at sentence-transformers/multi-qa-MiniLM-L6-cos-v1.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFBertModel for predictions without further training.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Model: \"tf_bert_model_1\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " bert (TFBertMainLayer)      multiple                  22713216  \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 22,713,216\n",
            "Trainable params: 22,713,216\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ]
        }
      ],
      "source": [
        "model = TFAutoModel.from_pretrained(model_id)\n",
        "model.summary()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "GcWYm-Kja9Dr"
      },
      "outputs": [],
      "source": [
        "class SentenceTransformer(tf.keras.Model):\n",
        "  def __init__(self,model):\n",
        "    super(SentenceTransformer,self).__init__()\n",
        "    self.model=model\n",
        "    self.dense=Dense(1,activation='sigmoid')\n",
        "\n",
        "  def compile(self,optimizer,loss_fn):\n",
        "    super(SentenceTransformer,self).compile()\n",
        "    self.optimizer=optimizer\n",
        "    self.loss_fn=loss_fn\n",
        "    self.loss_metric=tf.keras.metrics.Mean(name='loss')\n",
        "\n",
        "  @property\n",
        "  def metrics(self):\n",
        "    return [self.loss_metric]\n",
        "\n",
        "  def mean_pooling(self, model_output, attention_mask):\n",
        "    token_embeddings = model_output[0]\n",
        "\n",
        "    input_mask_expanded = tf.cast(\n",
        "        tf.broadcast_to(tf.expand_dims(attention_mask, -1), tf.shape(token_embeddings)),\n",
        "        tf.float32\n",
        "    )\n",
        "    return tf.math.reduce_sum(token_embeddings * input_mask_expanded, axis=1)/tf.clip_by_value(tf.math.reduce_sum(input_mask_expanded, axis=1), 1e-9, tf.float32.max)\n",
        "\n",
        "  def train_step(self,train_data):\n",
        "\n",
        "    query={'input_ids':train_data['input_ids_query'][:,0,:],\n",
        "           'token_type_ids':train_data['token_type_ids_query'][:,0,:],\n",
        "           'attention_mask':train_data['attention_mask_query'][:,0,:]}\n",
        "\n",
        "    product={'input_ids':train_data['input_ids_product'][:,0,:],\n",
        "             'token_type_ids':train_data['token_type_ids_product'][:,0,:],\n",
        "             'attention_mask':train_data['attention_mask_product'][:,0,:]}\n",
        "\n",
        "    labels=train_data['label']\n",
        "\n",
        "    with tf.GradientTape() as recorder:\n",
        "      query_predictions=self.model(query)\n",
        "      pred_query=self.mean_pooling(query_predictions,train_data['attention_mask_query'][:,0,:])\n",
        "\n",
        "      product_predictions=self.model(product)\n",
        "      pred_product=self.mean_pooling(product_predictions,train_data['attention_mask_product'][:,0,:])\n",
        "\n",
        "      pred_concat=tf.concat([pred_query,pred_product,tf.abs(pred_query-pred_product)],axis=-1)\n",
        "\n",
        "      predictions=self.dense(pred_concat)\n",
        "      loss=self.loss_fn(labels,predictions)\n",
        "\n",
        "\n",
        "    partial_derivatives = recorder.gradient(loss,self.model.trainable_weights)\n",
        "    self.optimizer.apply_gradients(zip(partial_derivatives, self.model.trainable_weights))\n",
        "\n",
        "\n",
        "    self.loss_metric.update_state(loss)\n",
        "\n",
        "    return {'loss':self.loss_metric.result(),}"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DLgQ-j6kkc5J"
      },
      "source": [
        "# Model Training"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "yHLddxbd1w_7"
      },
      "outputs": [],
      "source": [
        "stransformer=SentenceTransformer(model)\n",
        "stransformer.compile(\n",
        "    optimizer=tf.keras.optimizers.Adam(learning_rate=2e-5,),\n",
        "    loss_fn=tf.keras.losses.BinaryCrossentropy(),\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 419
        },
        "id": "r9_azxQo1xBu",
        "outputId": "443c71b7-debd-45bd-8999-ef8f94b31703"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 1/5\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:Gradients do not exist for variables ['tf_bert_model/bert/pooler/dense/kernel:0', 'tf_bert_model/bert/pooler/dense/bias:0'] when minimizing the loss. If you're using `model.compile()`, did you forget to provide a `loss`argument?\n",
            "WARNING:tensorflow:Gradients do not exist for variables ['tf_bert_model/bert/pooler/dense/kernel:0', 'tf_bert_model/bert/pooler/dense/bias:0'] when minimizing the loss. If you're using `model.compile()`, did you forget to provide a `loss`argument?\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1574/5716 [=======>......................] - ETA: 34:49 - loss: 0.4792"
          ]
        },
        {
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "output_type": "error",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-69-3909a2cd7f8e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mEPOCHS\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mhistory\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mstransformer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtf_dataset\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mEPOCHS\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/utils/traceback_utils.py\u001b[0m in \u001b[0;36merror_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     62\u001b[0m     \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     63\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 64\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     65\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# pylint: disable=broad-except\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     66\u001b[0m       \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_process_traceback_frames\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__traceback__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[1;32m   1412\u001b[0m               \u001b[0mlogs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtmp_logs\u001b[0m  \u001b[0;31m# No error, now safe to assign to logs.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1413\u001b[0m               \u001b[0mend_step\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstep\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mdata_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep_increment\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1414\u001b[0;31m               \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_train_batch_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mend_step\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1415\u001b[0m               \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstop_training\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1416\u001b[0m                 \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/callbacks.py\u001b[0m in \u001b[0;36mon_train_batch_end\u001b[0;34m(self, batch, logs)\u001b[0m\n\u001b[1;32m    436\u001b[0m     \"\"\"\n\u001b[1;32m    437\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_should_call_train_batch_hooks\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 438\u001b[0;31m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_batch_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mModeKeys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTRAIN\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'end'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    439\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    440\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0mon_test_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/callbacks.py\u001b[0m in \u001b[0;36m_call_batch_hook\u001b[0;34m(self, mode, hook, batch, logs)\u001b[0m\n\u001b[1;32m    295\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_batch_begin_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    296\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'end'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 297\u001b[0;31m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_batch_end_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    298\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    299\u001b[0m       raise ValueError(\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/callbacks.py\u001b[0m in \u001b[0;36m_call_batch_end_hook\u001b[0;34m(self, mode, batch, logs)\u001b[0m\n\u001b[1;32m    316\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_batch_times\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_time\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    317\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 318\u001b[0;31m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_batch_hook_helper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhook_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    319\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    320\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_batch_times\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_num_batches_for_timing_check\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/callbacks.py\u001b[0m in \u001b[0;36m_call_batch_hook_helper\u001b[0;34m(self, hook_name, batch, logs)\u001b[0m\n\u001b[1;32m    354\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mcallback\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    355\u001b[0m       \u001b[0mhook\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcallback\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhook_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 356\u001b[0;31m       \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    357\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    358\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_check_timing\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/callbacks.py\u001b[0m in \u001b[0;36mon_train_batch_end\u001b[0;34m(self, batch, logs)\u001b[0m\n\u001b[1;32m   1032\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1033\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0mon_train_batch_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1034\u001b[0;31m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_batch_update_progbar\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1035\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1036\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0mon_test_batch_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/callbacks.py\u001b[0m in \u001b[0;36m_batch_update_progbar\u001b[0;34m(self, batch, logs)\u001b[0m\n\u001b[1;32m   1104\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mverbose\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1105\u001b[0m       \u001b[0;31m# Only block async when verbose = 1.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1106\u001b[0;31m       \u001b[0mlogs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_utils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msync_to_numpy_or_python_type\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1107\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprogbar\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mseen\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlogs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfinalize\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1108\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/utils/tf_utils.py\u001b[0m in \u001b[0;36msync_to_numpy_or_python_type\u001b[0;34m(tensors)\u001b[0m\n\u001b[1;32m    605\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndim\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    606\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 607\u001b[0;31m   \u001b[0;32mreturn\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnest\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmap_structure\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_to_single_numpy_or_python_type\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtensors\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    608\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    609\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/util/nest.py\u001b[0m in \u001b[0;36mmap_structure\u001b[0;34m(func, *structure, **kwargs)\u001b[0m\n\u001b[1;32m    914\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    915\u001b[0m   return pack_sequence_as(\n\u001b[0;32m--> 916\u001b[0;31m       \u001b[0mstructure\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mentries\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    917\u001b[0m       expand_composites=expand_composites)\n\u001b[1;32m    918\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/util/nest.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m    914\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    915\u001b[0m   return pack_sequence_as(\n\u001b[0;32m--> 916\u001b[0;31m       \u001b[0mstructure\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mentries\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    917\u001b[0m       expand_composites=expand_composites)\n\u001b[1;32m    918\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/utils/tf_utils.py\u001b[0m in \u001b[0;36m_to_single_numpy_or_python_type\u001b[0;34m(t)\u001b[0m\n\u001b[1;32m    599\u001b[0m     \u001b[0;31m# Don't turn ragged or sparse tensors to NumPy.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    600\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 601\u001b[0;31m       \u001b[0mt\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    602\u001b[0m     \u001b[0;31m# Strings, ragged and sparse tensors don't have .item(). Return them as-is.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    603\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndarray\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgeneric\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/ops.py\u001b[0m in \u001b[0;36mnumpy\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1157\u001b[0m     \"\"\"\n\u001b[1;32m   1158\u001b[0m     \u001b[0;31m# TODO(slebedev): Consider avoiding a copy for non-CPU or remote tensors.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1159\u001b[0;31m     \u001b[0mmaybe_arr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_numpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1160\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mmaybe_arr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmaybe_arr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndarray\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mmaybe_arr\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1161\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/ops.py\u001b[0m in \u001b[0;36m_numpy\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1123\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_numpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1124\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1125\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_numpy_internal\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1126\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1127\u001b[0m       \u001b[0;32mraise\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_status_to_exception\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;32mNone\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ],
      "source": [
        "EPOCHS=5\n",
        "history=stransformer.fit(tf_dataset,epochs=EPOCHS,)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Pq2ZKsl3gnyh"
      },
      "outputs": [],
      "source": [
        "model_path='/content/drive/MyDrive/stransformer/stransformers.h5'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4sLN9Z8ikmPm"
      },
      "outputs": [],
      "source": [
        "#stransformer.model.save_weights(model_path)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "sN6OeQGxgkQD"
      },
      "outputs": [],
      "source": [
        "model.load_weights(model_path)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "KiZs5mO-V5oa"
      },
      "outputs": [],
      "source": [
        "def mean_pooling(model_output, attention_mask):\n",
        "  token_embeddings = model_output[0]\n",
        "\n",
        "  input_mask_expanded = tf.cast(\n",
        "      tf.broadcast_to(tf.expand_dims(attention_mask, -1), tf.shape(token_embeddings)),\n",
        "      tf.float32\n",
        "  )\n",
        "  return tf.math.reduce_sum(token_embeddings * input_mask_expanded, axis=1)/tf.clip_by_value(tf.math.reduce_sum(input_mask_expanded, axis=1), 1e-9, tf.float32.max)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "o8bzhpIeyYBj"
      },
      "source": [
        "# Testing"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nfuNasWYvHgD"
      },
      "source": [
        "## Create Embeddings"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4DfIVhWJkmR3"
      },
      "outputs": [],
      "source": [
        "filepath_catalogue='/content/drive/MyDrive/datasets/kdd_cup/product_catalogue-v0.3.csv'\n",
        "df_catalogue=pd.read_csv(filepath_catalogue)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "dmnaGgIgjCCC",
        "outputId": "9be5e3d0-a0c8-4ffd-afa3-df37927baf7a"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'【国内正規品】PeakDesign ピークデザイン シェルS SH-S-1'"
            ]
          },
          "execution_count": 11,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df_catalogue['product_title'][1000000]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9hZZgysZYZ9G"
      },
      "outputs": [],
      "source": [
        "product_titles=[str(df_catalogue['product_title'][i]) for i in range(len(df_catalogue))]\n",
        "#print(product_titles)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "az23EZGaYZ_q",
        "outputId": "16013f76-a36b-405f-edc8-1999f86b998d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1815216\n"
          ]
        }
      ],
      "source": [
        "print(len(product_titles))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "P3MKdncGdX5g"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "-PjwSDTVbYV9"
      },
      "outputs": [],
      "source": [
        "embeddings=[]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rqbYMz26h024",
        "outputId": "bd426b91-5fcb-4c63-99c7-a4ebe2f4f2bb"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "2836"
            ]
          },
          "execution_count": 16,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "INFERENCE_BATCH_SIZE=640\n",
        "len(product_titles)//INFERENCE_BATCH_SIZE"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TmMmLRiHYaGF",
        "outputId": "5fc0f66e-5394-43d7-804f-4e73c65ecc0c"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "0\n",
            "100\n",
            "200\n",
            "300\n",
            "400\n",
            "500\n",
            "600\n",
            "700\n",
            "800\n",
            "900\n",
            "1000\n",
            "1100\n",
            "1200\n",
            "1300\n",
            "1400\n",
            "1500\n",
            "1600\n",
            "1700\n",
            "1800\n",
            "1900\n",
            "2000\n",
            "2100\n",
            "2200\n",
            "2300\n",
            "2400\n",
            "2500\n",
            "2600\n",
            "2700\n",
            "2800\n"
          ]
        }
      ],
      "source": [
        "for i in range(len(product_titles)//INFERENCE_BATCH_SIZE):\n",
        "  tokenized_output=tokenizer(\n",
        "      product_titles[INFERENCE_BATCH_SIZE*i:INFERENCE_BATCH_SIZE*(i+1)],max_length=MAX_LENGTH,padding='max_length',truncation=True,return_tensors=\"tf\")\n",
        "  model_output=model(tokenized_output)\n",
        "  embedding=mean_pooling(model_output,tokenized_output['attention_mask'])\n",
        "  embeddings.append(embedding)\n",
        "  if i%100==0:\n",
        "    print(i)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "N0X_C06wbEM2",
        "outputId": "eb43f0b9-9e52-435c-c26f-012e61982ae8"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[<tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[ 0.04175833,  0.4398206 ,  0.15303786, ..., -0.18008585,\n",
              "         -0.71677995,  0.2654784 ],\n",
              "        [-0.05360471, -0.13363229,  0.06446703, ..., -0.04043194,\n",
              "         -0.5303095 ,  0.6396754 ],\n",
              "        [-0.07222205,  0.568274  ,  0.04923436, ..., -0.3087162 ,\n",
              "         -0.8591781 ,  0.75944847],\n",
              "        ...,\n",
              "        [ 0.2783488 ,  0.7277557 , -0.19541022, ...,  0.25403762,\n",
              "         -0.77744997,  0.88280714],\n",
              "        [-0.11515236,  0.44915688, -0.32512447, ...,  0.15528241,\n",
              "         -0.28903574, -0.01429977],\n",
              "        [-0.71291393, -0.30155414,  0.60434943, ..., -0.52819324,\n",
              "         -0.06169818, -0.42003578]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[-0.25033033, -0.36277884,  0.04568261, ..., -0.61685425,\n",
              "         -0.22854248,  0.0749432 ],\n",
              "        [ 0.31579307,  0.3078013 , -0.15302154, ..., -0.0205166 ,\n",
              "         -0.24058373, -0.1389403 ],\n",
              "        [ 0.34076113,  0.09088283, -0.09920587, ...,  0.3154116 ,\n",
              "         -0.71782845,  0.14604795],\n",
              "        ...,\n",
              "        [ 0.34244695,  0.5600365 , -0.25037694, ...,  0.29202062,\n",
              "         -0.78201824,  1.0003353 ],\n",
              "        [ 0.48839852,  0.58567506, -0.23761684, ...,  0.39142865,\n",
              "         -0.55380523,  0.93402475],\n",
              "        [-0.12706415,  0.04007166,  0.32039976, ..., -0.16968857,\n",
              "         -0.5625253 , -0.40995255]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[ 0.14748684,  0.19580686,  0.07518169, ...,  0.23299414,\n",
              "         -0.38106498,  0.97672564],\n",
              "        [ 0.10406388, -0.15469515, -0.07299469, ..., -0.09553993,\n",
              "         -0.18333358,  0.48289955],\n",
              "        [-0.48539472, -0.39405754,  0.35476398, ..., -0.5138081 ,\n",
              "          0.3042271 , -0.12476353],\n",
              "        ...,\n",
              "        [ 0.48841166,  0.13147825, -0.26861796, ...,  0.2507595 ,\n",
              "         -0.445062  ,  0.91309756],\n",
              "        [ 0.43607992,  0.14310056, -0.2774595 , ...,  0.3000701 ,\n",
              "         -0.64802307,  0.58720326],\n",
              "        [ 0.5343046 , -0.00881386, -0.38213897, ...,  0.37829387,\n",
              "         -0.7968462 ,  0.6387348 ]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[ 0.42024356,  0.04089506, -0.26343155, ...,  0.320241  ,\n",
              "         -0.34386224,  0.21557738],\n",
              "        [ 0.70042986,  0.27202126, -0.12118231, ...,  0.4024516 ,\n",
              "         -0.71601355,  0.9871158 ],\n",
              "        [ 0.20603643,  0.21225566, -0.27889886, ...,  0.32165194,\n",
              "         -0.71019053,  0.9009069 ],\n",
              "        ...,\n",
              "        [ 0.03869297,  0.58761203, -0.24618867, ..., -0.17659199,\n",
              "         -0.24322106,  0.95189375],\n",
              "        [ 0.26066822,  0.09344871, -0.04015832, ...,  0.17433245,\n",
              "         -0.94286335,  0.76141584],\n",
              "        [-0.02870281,  0.15181287,  0.09849725, ...,  0.40340248,\n",
              "         -0.9012172 ,  0.69525766]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[-0.286191  ,  0.16769332,  0.46374464, ..., -0.19424438,\n",
              "          0.06105224,  0.45701525],\n",
              "        [ 0.7010249 ,  1.2779559 , -0.39571473, ...,  0.36589256,\n",
              "         -0.24720716,  1.2214389 ],\n",
              "        [ 0.10948884,  0.62776875, -0.36958   , ..., -0.1517321 ,\n",
              "         -0.16462211,  1.1436412 ],\n",
              "        ...,\n",
              "        [-0.04111135, -0.20377578,  0.12384884, ...,  0.2541174 ,\n",
              "         -0.7143339 ,  0.3983777 ],\n",
              "        [ 0.1415652 ,  0.05144472, -0.04741249, ...,  0.0705729 ,\n",
              "         -0.62784183,  0.20909509],\n",
              "        [-0.1373359 , -0.24972369, -0.02410387, ...,  0.04416026,\n",
              "         -0.5743125 ,  0.41214696]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[ 0.16799496, -0.32816967,  0.01181602, ...,  0.10041109,\n",
              "         -0.64877653,  0.3889101 ],\n",
              "        [-0.02450915, -0.20282096,  0.0083284 , ...,  0.18377303,\n",
              "         -0.52443844,  0.5755725 ],\n",
              "        [ 0.01421617,  0.03285762, -0.02940205, ...,  0.19213267,\n",
              "         -0.5942168 ,  0.5512716 ],\n",
              "        ...,\n",
              "        [-0.00806624,  0.42302358,  0.3290485 , ...,  0.11360864,\n",
              "         -0.65781355,  0.07665016],\n",
              "        [ 0.24988385,  0.46609062, -0.05153411, ...,  0.47314677,\n",
              "         -0.8938951 ,  0.5596481 ],\n",
              "        [ 0.12137385, -0.06866003, -0.12768006, ...,  0.04252189,\n",
              "         -0.7592595 ,  0.01903482]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[ 2.3139369e-02,  3.4977409e-01, -2.1248076e-03, ...,\n",
              "          2.0912278e-01, -6.8048084e-01,  6.3785213e-01],\n",
              "        [-8.7924369e-02,  4.5413840e-01,  2.0525348e-01, ...,\n",
              "          4.3102801e-02, -8.5152590e-01,  2.1857570e-01],\n",
              "        [ 2.3010924e-01,  4.1551170e-01,  3.4641664e-02, ...,\n",
              "          4.4237506e-01, -9.5529884e-01,  8.5906112e-01],\n",
              "        ...,\n",
              "        [ 1.7330341e-01,  5.5858982e-01,  1.3590802e-01, ...,\n",
              "          2.7698830e-01, -6.8018240e-01,  7.0290136e-01],\n",
              "        [ 2.1508950e-01, -8.3878031e-04, -5.2778214e-02, ...,\n",
              "          3.8234007e-01, -3.8288379e-01,  4.1337579e-01],\n",
              "        [ 2.2225678e-01,  6.1098218e-01, -6.3592836e-02, ...,\n",
              "          4.3762371e-01, -6.0720587e-01,  4.3495446e-01]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[-0.646252  , -0.71932673,  0.53551346, ..., -0.39160207,\n",
              "         -0.14408746, -0.28350443],\n",
              "        [ 0.70611435,  0.46755788,  0.17171729, ...,  0.6644606 ,\n",
              "         -0.66235614,  0.33803654],\n",
              "        [-0.50988173,  0.1057297 ,  0.40107605, ...,  0.13284951,\n",
              "         -0.6258956 ,  0.5444822 ],\n",
              "        ...,\n",
              "        [ 0.00541785, -0.115128  ,  0.2224784 , ...,  0.45370325,\n",
              "         -0.26409048,  0.8207882 ],\n",
              "        [ 0.07472134,  0.21213049,  0.03968048, ...,  0.00946861,\n",
              "         -0.60264266,  0.7194275 ],\n",
              "        [ 0.21142125, -0.12419219, -0.13894767, ...,  0.24269105,\n",
              "         -0.5978866 ,  0.7704548 ]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[ 0.01500008,  0.39985567, -0.01700629, ...,  0.36136168,\n",
              "         -0.53402525,  0.72195226],\n",
              "        [ 0.28185338,  0.53459483,  0.05878032, ..., -0.16678536,\n",
              "         -0.22839068,  1.0593886 ],\n",
              "        [ 0.16177693,  0.08386799,  0.08180445, ...,  0.20268714,\n",
              "         -0.713283  ,  0.43605784],\n",
              "        ...,\n",
              "        [ 0.07754885,  0.05648893, -0.07567939, ..., -0.18757747,\n",
              "         -0.43229648, -0.22383551],\n",
              "        [ 0.44943663,  0.5101748 , -0.20771615, ...,  0.4087045 ,\n",
              "         -0.6822468 ,  0.1559221 ],\n",
              "        [-0.14639053, -0.2177874 , -0.01756675, ..., -0.3538432 ,\n",
              "         -0.39450964, -0.37615493]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[ 0.370123  ,  0.56531143, -0.2859203 , ..., -0.10627373,\n",
              "         -0.7266577 ,  0.57767457],\n",
              "        [-0.25348648, -0.0205687 ,  0.11779017, ..., -0.5437002 ,\n",
              "         -0.4677617 , -0.61622155],\n",
              "        [-0.14639053, -0.2177874 , -0.01756675, ..., -0.3538432 ,\n",
              "         -0.39450964, -0.37615493],\n",
              "        ...,\n",
              "        [-0.01486018,  0.8658072 ,  0.14513871, ...,  0.2555863 ,\n",
              "         -0.6604221 ,  1.3605083 ],\n",
              "        [-0.275398  ,  0.05214844,  0.19720614, ..., -0.5928097 ,\n",
              "         -0.2562231 , -0.04542203],\n",
              "        [ 0.06585225,  0.55188924,  0.04813208, ..., -0.35324207,\n",
              "         -0.54707086,  0.09369139]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[-0.566859  ,  0.03334103,  0.46732602, ..., -0.26820275,\n",
              "         -0.16225775,  0.11631189],\n",
              "        [ 0.09178782,  0.2041869 ,  0.15012287, ..., -0.00689907,\n",
              "         -0.62904215,  0.68531024],\n",
              "        [-0.38941145,  0.31177565,  0.02137183, ..., -0.18622142,\n",
              "         -0.521308  ,  0.63888437],\n",
              "        ...,\n",
              "        [ 0.04305602,  0.6504234 , -0.12663138, ...,  0.534755  ,\n",
              "          0.01034129,  0.36399373],\n",
              "        [ 0.12109199,  0.18649685,  0.08178806, ..., -0.2866235 ,\n",
              "         -0.22596951,  0.7859608 ],\n",
              "        [ 0.14244571,  0.82605755,  0.02186577, ...,  0.13987163,\n",
              "         -0.457522  ,  0.72653013]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[ 0.01971628,  0.24909668,  0.332681  , ...,  0.09781835,\n",
              "         -0.5355704 ,  1.0896999 ],\n",
              "        [-0.35724515,  0.7342107 ,  0.23408698, ...,  0.18934625,\n",
              "         -0.0780261 ,  0.5420728 ],\n",
              "        [ 0.12834032,  0.98195297,  0.14281681, ...,  0.47438654,\n",
              "         -0.62614924,  1.1591474 ],\n",
              "        ...,\n",
              "        [-0.61155075,  0.80029696,  0.4696462 , ..., -0.21341865,\n",
              "         -0.19173215,  0.7937192 ],\n",
              "        [-0.11969849,  0.9270084 ,  0.35477567, ..., -0.03220062,\n",
              "         -0.4568102 ,  1.1192102 ],\n",
              "        [ 0.15940093,  0.97020066,  0.11175465, ..., -0.27326182,\n",
              "         -0.5957728 ,  0.8714527 ]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[ 0.10157666,  0.9047333 ,  0.04322327, ...,  0.09852891,\n",
              "         -0.38284534,  1.21513   ],\n",
              "        [ 0.02233633,  0.8738396 ,  0.12372379, ...,  0.01728655,\n",
              "         -0.24666472,  1.008563  ],\n",
              "        [ 0.14673252,  0.9494429 ,  0.04610442, ..., -0.11451961,\n",
              "         -0.41614172,  0.936616  ],\n",
              "        ...,\n",
              "        [-0.14836508, -0.34121507,  0.3488845 , ..., -0.18113215,\n",
              "         -0.5030347 ,  0.1497709 ],\n",
              "        [ 0.13120905,  0.2369727 , -0.14269298, ...,  0.18332572,\n",
              "         -0.6598623 ,  0.89777035],\n",
              "        [ 0.5659621 ,  0.52872664, -0.09020699, ...,  0.19810052,\n",
              "         -0.3595669 , -0.08044092]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[-0.26627168,  0.4282594 ,  0.34135833, ..., -0.03261119,\n",
              "         -0.71648556,  0.8759529 ],\n",
              "        [ 0.23294124,  0.417558  ,  0.09019086, ...,  0.1555465 ,\n",
              "         -0.43012065,  0.12572706],\n",
              "        [ 0.19895072,  0.18410844,  0.04166614, ...,  0.10346494,\n",
              "         -0.5329989 ,  0.38633493],\n",
              "        ...,\n",
              "        [-0.47337252, -0.40794116,  0.04851312, ..., -0.44143248,\n",
              "         -0.7932615 ,  0.59647024],\n",
              "        [-0.1133939 ,  0.10674752, -0.01672552, ..., -0.36275628,\n",
              "         -0.3728306 , -0.42068732],\n",
              "        [-0.65361303, -0.02561901,  0.05988749, ..., -0.6236122 ,\n",
              "         -0.5288415 ,  0.10669609]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[ 1.1490872e-01,  1.2124353e-01,  4.1173857e-01, ...,\n",
              "          1.7361213e-01, -6.5839195e-01,  5.8026415e-01],\n",
              "        [-3.9404258e-01, -9.4543219e-02,  3.7512209e-02, ...,\n",
              "         -2.5264459e-02, -6.2423831e-01,  5.7284796e-01],\n",
              "        [-3.2687244e-01, -4.9143523e-04, -1.0026325e-02, ...,\n",
              "         -2.2642723e-01, -4.7448558e-01,  4.8515418e-01],\n",
              "        ...,\n",
              "        [ 3.2483262e-01,  1.6334677e-01, -5.5577576e-01, ...,\n",
              "         -6.1663780e-03, -8.0985665e-01,  5.6609708e-01],\n",
              "        [ 2.1349227e-01,  3.0032581e-01, -4.4987958e-02, ...,\n",
              "          4.6920884e-01, -5.9497440e-01,  6.5547711e-01],\n",
              "        [-4.2232554e-02, -1.0509109e-01,  1.3962300e-01, ...,\n",
              "          5.8519572e-01, -7.1166307e-01,  3.6402977e-01]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[ 0.24688911,  0.2083171 ,  0.00615057, ...,  0.03520483,\n",
              "         -0.7128647 ,  0.5391904 ],\n",
              "        [ 0.39536873,  0.41643956, -0.01857961, ...,  0.17843254,\n",
              "         -0.79931355,  0.8440734 ],\n",
              "        [ 0.5177639 ,  0.07008002, -0.23908179, ..., -0.09356965,\n",
              "         -0.3334423 ,  0.57445   ],\n",
              "        ...,\n",
              "        [-0.07067601, -0.13010715,  0.19300392, ..., -0.11004839,\n",
              "         -0.51720726,  0.03396518],\n",
              "        [ 0.19038646,  0.3498009 , -0.06343661, ...,  0.18272282,\n",
              "         -0.58199227,  0.49969316],\n",
              "        [ 0.31493706,  0.1694332 ,  0.14490066, ...,  0.6568433 ,\n",
              "         -0.46218178,  0.27008015]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[-0.06583623,  0.14604574,  0.00673229, ...,  0.49180144,\n",
              "         -0.19037186,  0.4769562 ],\n",
              "        [ 0.2131216 ,  0.8030973 ,  0.02223831, ...,  0.43527305,\n",
              "         -0.682142  ,  1.3874313 ],\n",
              "        [-0.24200527, -0.19570503,  0.82811165, ..., -0.2624034 ,\n",
              "         -0.04067682, -0.18759875],\n",
              "        ...,\n",
              "        [ 0.5992238 ,  0.22622028, -0.15812862, ...,  0.47404814,\n",
              "         -0.42368928,  0.7459436 ],\n",
              "        [-0.33941984, -0.7586083 ,  0.3847668 , ..., -0.08621169,\n",
              "          0.07291491, -1.5789393 ],\n",
              "        [ 0.22075655,  0.5020302 , -0.10647847, ...,  0.46515366,\n",
              "         -0.40455118,  0.6570176 ]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[ 0.23648602,  0.11702654, -0.20512347, ...,  0.41427353,\n",
              "         -0.52074945,  0.26732093],\n",
              "        [ 0.127407  ,  0.2110875 , -0.28987536, ...,  0.5517178 ,\n",
              "         -0.43492645,  0.26112163],\n",
              "        [ 0.59176147,  0.08843001,  0.01885601, ...,  0.21823305,\n",
              "         -0.2659875 ,  0.23607948],\n",
              "        ...,\n",
              "        [ 0.5949431 ,  1.0990536 , -0.23828645, ...,  0.6064155 ,\n",
              "         -0.9516905 ,  0.8232104 ],\n",
              "        [ 0.02301317,  1.0176575 ,  0.13559593, ...,  0.3306479 ,\n",
              "         -0.6779507 ,  0.9546297 ],\n",
              "        [ 0.06756505,  0.8047561 ,  0.09765031, ...,  0.10274994,\n",
              "         -0.6419105 , -0.08558054]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[ 0.04294442,  0.41440418, -0.07512046, ..., -0.07347631,\n",
              "         -0.52332354,  0.4847638 ],\n",
              "        [ 0.2710858 ,  0.81507385, -0.36308318, ...,  0.28536677,\n",
              "         -0.86375666,  0.62337863],\n",
              "        [ 0.4642743 ,  0.6716567 , -0.03945766, ...,  0.48418248,\n",
              "         -0.7741654 ,  1.0063552 ],\n",
              "        ...,\n",
              "        [ 0.45695192,  0.0628484 , -0.2168539 , ...,  0.02491212,\n",
              "         -0.33348417, -0.19710474],\n",
              "        [ 0.18160863,  0.2840213 , -0.05385072, ..., -0.11252134,\n",
              "         -0.21932514, -0.02501918],\n",
              "        [-0.01095332,  0.588571  ,  0.05051978, ..., -0.05566474,\n",
              "         -0.69155717,  0.23114802]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[ 0.2791404 ,  0.40595725,  0.11480775, ..., -0.5035303 ,\n",
              "         -0.6115774 ,  0.39307648],\n",
              "        [ 0.39426807,  0.33540088,  0.14849414, ...,  0.09766593,\n",
              "         -0.31658375, -0.09469123],\n",
              "        [ 0.05244792,  0.05727427,  0.21003352, ..., -0.4842351 ,\n",
              "         -0.23643959, -0.3016082 ],\n",
              "        ...,\n",
              "        [ 0.5072082 ,  0.74538505, -0.09499668, ...,  0.59595495,\n",
              "         -0.01816199,  0.6747888 ],\n",
              "        [-0.00153396,  0.5001153 , -0.0685868 , ..., -0.16356705,\n",
              "         -0.73123354,  0.46462226],\n",
              "        [ 0.02112093,  0.05578696,  0.01951397, ..., -0.16294038,\n",
              "         -0.05869416,  0.15101688]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[-0.22604407, -0.34471193,  0.2814709 , ..., -0.3381662 ,\n",
              "         -0.20208563, -0.10536566],\n",
              "        [-0.05940239, -0.2854067 ,  0.31916422, ..., -0.14987619,\n",
              "         -0.23090896, -0.25686592],\n",
              "        [-0.23925832, -0.49337098,  0.5492798 , ..., -0.21327318,\n",
              "         -0.01636759, -0.14413574],\n",
              "        ...,\n",
              "        [ 0.01499754,  0.3521631 ,  0.0088081 , ...,  0.11955793,\n",
              "         -0.45359528,  0.31013495],\n",
              "        [ 0.10090962,  0.334304  ,  0.00819372, ...,  0.40696743,\n",
              "         -0.47200945,  0.57296145],\n",
              "        [ 0.01964979,  0.42006972,  0.04076331, ...,  0.6667238 ,\n",
              "         -0.5624147 ,  0.7917737 ]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[-0.20985857,  0.14177252,  0.06964822, ...,  0.14048697,\n",
              "         -0.57887155,  0.2710777 ],\n",
              "        [ 0.03044176,  0.28998834, -0.12894753, ..., -0.07987388,\n",
              "         -0.51949865,  0.21709022],\n",
              "        [-0.23092833,  0.61163396,  0.07136898, ...,  0.0472135 ,\n",
              "         -0.29334325,  0.73603815],\n",
              "        ...,\n",
              "        [ 0.188179  ,  0.5045909 ,  0.10049309, ...,  0.16793744,\n",
              "         -0.12253466,  0.27801794],\n",
              "        [ 0.39823723,  0.68124104, -0.15368856, ...,  0.70156085,\n",
              "         -0.7015736 ,  1.0942147 ],\n",
              "        [ 0.12372316,  1.049538  , -0.06727453, ...,  0.5157293 ,\n",
              "         -0.8061489 ,  1.0929339 ]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[ 0.21965924, -0.2493218 ,  0.12256566, ...,  0.17922617,\n",
              "         -0.66765654,  0.91267806],\n",
              "        [ 0.23020138, -0.33955625,  0.1733351 , ...,  0.23338686,\n",
              "         -0.58052015,  0.70666945],\n",
              "        [ 0.4104217 ,  0.13838242, -0.37406036, ...,  0.43434286,\n",
              "         -0.52905035,  0.7429536 ],\n",
              "        ...,\n",
              "        [ 0.15614037,  0.71359426,  0.07959738, ...,  0.43773472,\n",
              "         -0.57667947,  1.310644  ],\n",
              "        [ 0.42132553, -0.01929493, -0.15595295, ...,  0.48627794,\n",
              "         -0.26043135,  0.81319135],\n",
              "        [ 0.54887825,  1.3677843 , -0.24242453, ...,  0.70995873,\n",
              "         -0.6389373 ,  0.98212725]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[ 0.40913922,  0.6437877 , -0.16954151, ...,  0.73065805,\n",
              "         -0.64532757,  0.9785884 ],\n",
              "        [ 0.3097515 ,  0.44457015,  0.12385947, ...,  0.6791362 ,\n",
              "         -0.58877206,  1.2007908 ],\n",
              "        [ 0.0096737 ,  0.7055997 ,  0.01033811, ...,  0.3917299 ,\n",
              "         -0.6522756 ,  0.76251566],\n",
              "        ...,\n",
              "        [ 0.31845835,  0.05508452, -0.10967573, ...,  0.38035405,\n",
              "         -0.8006219 ,  0.93477607],\n",
              "        [ 0.3094072 , -0.2102557 ,  0.10989378, ...,  0.57816267,\n",
              "         -0.62174004,  0.55668694],\n",
              "        [ 0.09111734,  0.18056746,  0.10708551, ...,  0.52324903,\n",
              "         -0.7744313 ,  0.20597236]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[ 0.32492325,  0.6402603 , -0.04418265, ...,  0.69436234,\n",
              "         -0.5739316 ,  0.6300135 ],\n",
              "        [ 0.16450916,  0.7181147 ,  0.36038983, ...,  0.32265028,\n",
              "         -0.7178333 ,  0.7866748 ],\n",
              "        [ 0.49170193,  0.8241518 , -0.31141257, ...,  0.14548346,\n",
              "         -0.6988038 ,  1.0474403 ],\n",
              "        ...,\n",
              "        [ 0.42476422,  0.90140694, -0.24912468, ...,  0.5223128 ,\n",
              "          0.11187278,  0.6445696 ],\n",
              "        [ 0.39604723,  0.891434  , -0.18173157, ...,  0.57968765,\n",
              "         -0.57120574,  0.6188661 ],\n",
              "        [ 0.24298257,  0.54896396, -0.09961283, ...,  0.30053696,\n",
              "         -0.7100791 ,  1.2927234 ]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[ 0.3686855 ,  0.14158182, -0.06603931, ...,  0.65680903,\n",
              "         -0.26583555, -0.20703544],\n",
              "        [ 0.13039273,  0.47866523, -0.13116276, ...,  0.5029583 ,\n",
              "         -0.333032  ,  0.48123148],\n",
              "        [-0.23732598,  0.2736469 ,  0.02437121, ..., -0.11776209,\n",
              "         -0.52278656, -0.22755046],\n",
              "        ...,\n",
              "        [-0.08647965,  0.821133  , -0.26240712, ...,  0.4705316 ,\n",
              "         -0.84316033,  1.1565737 ],\n",
              "        [-0.00486945,  0.6837065 , -0.03362267, ...,  0.39370075,\n",
              "         -0.8021494 ,  0.86728776],\n",
              "        [ 0.02139072,  1.0516987 , -0.19311492, ...,  0.19469985,\n",
              "         -0.78733695,  0.85759485]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[ 0.15099739,  0.66440636, -0.01801643, ...,  0.49822155,\n",
              "         -0.60503495,  0.77278024],\n",
              "        [-0.08215621,  0.7576025 , -0.35166696, ...,  0.30989203,\n",
              "         -0.5801937 ,  0.83706933],\n",
              "        [-0.152631  ,  0.5895366 , -0.1759167 , ...,  0.2921711 ,\n",
              "         -0.7170673 ,  0.86878115],\n",
              "        ...,\n",
              "        [ 0.2449684 , -0.07164929,  0.05697073, ...,  0.5375358 ,\n",
              "         -0.37742734,  0.8079238 ],\n",
              "        [-0.29008788, -0.15579534, -0.01283896, ...,  0.7388498 ,\n",
              "         -0.5897742 ,  0.4515136 ],\n",
              "        [-0.10294172,  0.3555222 ,  0.22267294, ...,  0.19703956,\n",
              "         -0.5881062 ,  0.6083472 ]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[ 0.35946563,  0.29333267,  0.1422744 , ...,  0.38166472,\n",
              "         -0.41407412,  0.68468386],\n",
              "        [ 0.28643012, -0.27947205,  0.00391141, ...,  0.13479374,\n",
              "         -0.3167661 ,  0.78904045],\n",
              "        [-0.5684243 , -0.43954667,  0.26769784, ..., -0.21161228,\n",
              "         -0.2228092 ,  0.08609941],\n",
              "        ...,\n",
              "        [ 0.36932933,  0.16171601, -0.0992183 , ..., -0.10370647,\n",
              "         -0.91680545,  0.49948135],\n",
              "        [ 0.2432496 ,  0.3627526 ,  0.01810543, ..., -0.05479885,\n",
              "         -0.9364949 ,  0.5916867 ],\n",
              "        [ 0.25659117,  0.4389261 , -0.05896876, ..., -0.09829643,\n",
              "         -1.2642912 ,  0.5054443 ]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[ 0.37231156,  0.08294662, -0.16616859, ..., -0.12055435,\n",
              "         -0.78240275,  0.5643776 ],\n",
              "        [ 0.45664176,  0.6375481 , -0.19348007, ...,  0.19480202,\n",
              "         -0.85333276,  0.77435505],\n",
              "        [-0.0295205 ,  0.6420359 ,  0.1147195 , ..., -0.28298396,\n",
              "         -0.32668245,  0.74717194],\n",
              "        ...,\n",
              "        [ 0.29803252, -0.31292805, -0.08049765, ..., -0.47411028,\n",
              "         -0.32571203,  0.1301616 ],\n",
              "        [ 0.43356603,  0.1163137 , -0.32584718, ...,  0.4197551 ,\n",
              "         -0.40994936,  0.42267334],\n",
              "        [ 0.7577016 ,  0.01567647, -0.2828081 , ..., -0.02313686,\n",
              "         -0.642099  ,  0.18100622]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[ 0.5023431 ,  0.18121031, -0.1246812 , ...,  0.27034426,\n",
              "         -0.60975254,  0.14160097],\n",
              "        [ 0.6108162 ,  0.33550242, -0.31714678, ...,  0.5832409 ,\n",
              "         -0.58394194,  0.6782671 ],\n",
              "        [ 0.06752969,  0.00566939,  0.35637522, ...,  0.48818794,\n",
              "         -0.59925514,  0.40198234],\n",
              "        ...,\n",
              "        [-0.29386792,  0.74698246,  0.09412478, ...,  0.42301154,\n",
              "          0.11242285,  0.40774134],\n",
              "        [ 0.06243708,  0.582511  ,  0.00260057, ...,  0.5817374 ,\n",
              "          0.02051132,  0.5330948 ],\n",
              "        [-0.27442867,  0.8262154 ,  0.08992533, ...,  0.23895746,\n",
              "          0.21280804,  0.4768016 ]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[ 0.10108016,  0.7373512 , -0.06049845, ...,  0.7891657 ,\n",
              "          0.20169477,  0.9041737 ],\n",
              "        [-0.17353503,  0.8262605 , -0.09635602, ...,  0.2630303 ,\n",
              "          0.16502917,  0.16417976],\n",
              "        [ 0.12158553,  0.6139697 , -0.10976139, ...,  0.28408954,\n",
              "          0.06068013,  0.5421906 ],\n",
              "        ...,\n",
              "        [ 0.08483312,  0.6005356 ,  0.15147108, ...,  0.26565474,\n",
              "         -0.2816685 ,  0.6623235 ],\n",
              "        [ 0.3229027 ,  0.7145016 ,  0.05491697, ...,  0.13983545,\n",
              "         -0.30164415,  0.543282  ],\n",
              "        [ 0.1636998 ,  0.6326752 ,  0.02633532, ...,  0.30062017,\n",
              "         -0.38007036,  0.5441976 ]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[ 0.1738298 ,  0.69292086,  0.21938246, ...,  0.01665772,\n",
              "         -0.4841403 ,  0.13018875],\n",
              "        [-0.72749025, -0.01341574,  0.5376369 , ..., -0.6404275 ,\n",
              "         -0.06836149,  0.26137364],\n",
              "        [ 0.24599819,  0.48633352,  0.04608562, ..., -0.0399502 ,\n",
              "         -0.544908  ,  0.30180973],\n",
              "        ...,\n",
              "        [ 0.49150923,  0.35832238, -0.07855911, ...,  0.6586546 ,\n",
              "         -0.41610056,  0.6773295 ],\n",
              "        [-0.27097553,  0.19141173,  0.05843251, ..., -0.13107574,\n",
              "         -0.87420845,  0.15111946],\n",
              "        [-0.03067096,  0.4597472 , -0.0081422 , ...,  0.21551357,\n",
              "         -0.53990895,  0.10877921]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[ 0.40816453,  0.7242059 , -0.21497425, ...,  0.74188054,\n",
              "         -0.33964536,  0.7691854 ],\n",
              "        [ 0.4202708 ,  0.7680881 , -0.04916425, ...,  0.6465059 ,\n",
              "         -0.92108744,  0.8128672 ],\n",
              "        [ 0.6769749 ,  0.9002141 , -0.51365477, ...,  0.6296458 ,\n",
              "         -0.85555744,  0.6527981 ],\n",
              "        ...,\n",
              "        [ 0.16184385,  0.6983598 , -0.11803973, ..., -0.2649557 ,\n",
              "         -0.08203625, -0.00428872],\n",
              "        [ 0.35170203,  0.9224744 , -0.08616624, ...,  0.34510198,\n",
              "         -0.20744029,  1.0936215 ],\n",
              "        [ 0.5283233 ,  0.41355196, -0.3151552 , ...,  0.336158  ,\n",
              "         -0.6274787 ,  0.48428157]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[ 0.34417656,  0.44984102, -0.15906665, ...,  0.2240383 ,\n",
              "         -0.25925824,  0.4895668 ],\n",
              "        [ 0.4201514 ,  0.19050865,  0.19349426, ...,  0.15432328,\n",
              "         -0.55059576,  0.25777623],\n",
              "        [ 0.11814367,  0.18835554, -0.05639662, ..., -0.08989175,\n",
              "         -0.5292117 ,  0.70375556],\n",
              "        ...,\n",
              "        [ 0.08248366,  0.26830688,  0.04166511, ...,  0.3410406 ,\n",
              "         -0.47847757, -0.55114865],\n",
              "        [ 0.15517375,  0.34014007, -0.02817206, ...,  0.4790844 ,\n",
              "         -0.28118038, -0.04963386],\n",
              "        [ 0.15517375,  0.34014007, -0.02817206, ...,  0.4790844 ,\n",
              "         -0.28118038, -0.04963386]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[ 0.15517375,  0.34014007, -0.02817206, ...,  0.4790844 ,\n",
              "         -0.28118038, -0.04963386],\n",
              "        [ 0.15517375,  0.34014007, -0.02817206, ...,  0.4790844 ,\n",
              "         -0.28118038, -0.04963386],\n",
              "        [ 0.14516827,  0.55291486,  0.02075753, ...,  0.5595679 ,\n",
              "         -0.7079608 ,  0.37188727],\n",
              "        ...,\n",
              "        [ 0.0871079 ,  0.13035016,  0.06606723, ...,  0.20172855,\n",
              "         -0.5226581 ,  0.45579943],\n",
              "        [ 0.27784735,  0.18642333, -0.00269201, ...,  0.21619222,\n",
              "         -0.87275296,  0.7224734 ],\n",
              "        [-0.06386577,  0.02079103,  0.04732474, ..., -0.05860281,\n",
              "         -0.91304445,  1.0727803 ]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[ 0.20987271,  0.31435996,  0.10047122, ..., -0.01257294,\n",
              "         -0.8087982 ,  0.59859776],\n",
              "        [ 0.11936756,  0.0251148 , -0.12234222, ..., -0.06651466,\n",
              "         -0.59557015,  0.60717684],\n",
              "        [ 0.38855967,  0.24265152, -0.14351457, ...,  0.32160228,\n",
              "         -1.0505002 ,  0.44987294],\n",
              "        ...,\n",
              "        [-0.6380977 ,  0.15407082,  0.20129146, ..., -0.67529196,\n",
              "         -0.4066537 ,  0.4867354 ],\n",
              "        [ 0.36287686,  0.35677904, -0.288061  , ...,  0.3981217 ,\n",
              "         -0.824088  ,  0.5570968 ],\n",
              "        [-0.31820753,  0.20030646, -0.01016104, ...,  0.05243865,\n",
              "         -0.6429082 ,  0.3245317 ]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[-0.08369117, -0.19402127, -0.06810013, ..., -0.01525985,\n",
              "         -0.55764675,  0.10537972],\n",
              "        [-0.25108373,  0.17104302,  0.04734301, ..., -0.04227674,\n",
              "         -0.75507706,  0.53735113],\n",
              "        [ 0.01077597,  0.38262147, -0.1581204 , ...,  0.05184609,\n",
              "         -0.49940354,  0.31732172],\n",
              "        ...,\n",
              "        [ 0.07067408,  0.48063254, -0.23734164, ...,  0.40610048,\n",
              "         -0.71762246,  0.22710623],\n",
              "        [-0.06106582,  0.5852883 ,  0.19927536, ...,  0.38237327,\n",
              "         -0.15035896, -0.72111976],\n",
              "        [-0.08602911,  0.1268641 ,  0.15558307, ...,  0.13285013,\n",
              "         -0.09289515,  0.01989212]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[ 0.05978991,  0.2396311 ,  0.04924653, ...,  0.27211484,\n",
              "         -0.80442053,  0.6961332 ],\n",
              "        [-0.00437761,  0.5523109 , -0.10989753, ...,  0.5515091 ,\n",
              "         -0.43342724,  0.5574106 ],\n",
              "        [ 0.3096685 ,  0.6208367 , -0.47445053, ...,  0.47695026,\n",
              "         -0.34062675, -0.10462704],\n",
              "        ...,\n",
              "        [-0.1311244 ,  0.4791031 ,  0.4094357 , ..., -0.02299992,\n",
              "         -0.1201403 ,  0.34671146],\n",
              "        [-0.6583344 ,  0.21551935,  0.3831869 , ..., -0.17188403,\n",
              "         -0.5395626 ,  0.31486768],\n",
              "        [-1.0238988 ,  0.04092908,  0.7109073 , ..., -0.22895274,\n",
              "          0.36135194,  0.4070304 ]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[-0.07985907,  0.51052725, -0.03998213, ...,  0.09246536,\n",
              "          0.22587384, -0.26033193],\n",
              "        [-0.23217116,  0.7417168 ,  0.3079051 , ...,  0.05509806,\n",
              "         -0.11215814,  0.01567556],\n",
              "        [-0.5652295 ,  0.34927028,  0.43862647, ..., -0.12314934,\n",
              "          0.45933193, -0.65000856],\n",
              "        ...,\n",
              "        [ 0.7083567 ,  0.96381706, -0.40347198, ...,  0.36084577,\n",
              "         -0.28367004,  0.8823428 ],\n",
              "        [ 0.327139  ,  0.92294097, -0.08482131, ...,  0.03475928,\n",
              "         -0.5857273 ,  0.74184024],\n",
              "        [ 0.33586463,  0.7419777 , -0.44791046, ..., -0.13934532,\n",
              "         -0.20758635,  0.56303775]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[ 0.6187793 ,  1.131212  , -0.79093575, ...,  0.5422813 ,\n",
              "         -0.0334533 ,  0.99984   ],\n",
              "        [ 0.6403183 ,  0.74518067, -0.26462355, ...,  0.1320436 ,\n",
              "         -0.30016214,  1.1166171 ],\n",
              "        [-0.1025129 ,  0.51025337, -0.4288716 , ...,  0.42988297,\n",
              "         -0.6628003 ,  0.71169   ],\n",
              "        ...,\n",
              "        [ 0.45034808,  0.6154961 , -0.37978342, ...,  0.1651332 ,\n",
              "         -0.7925423 ,  0.6388703 ],\n",
              "        [ 0.3357759 ,  0.5625353 , -0.2904886 , ...,  0.45307395,\n",
              "         -0.57085806,  0.569012  ],\n",
              "        [ 0.29782453,  0.7309329 , -0.17436488, ...,  0.37811315,\n",
              "         -0.2746507 ,  0.48166442]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[ 0.6570019 ,  0.26681414, -0.08328229, ...,  0.45949835,\n",
              "         -0.5114346 ,  0.6698504 ],\n",
              "        [ 0.32506314,  0.5714409 , -0.20395012, ...,  0.29760414,\n",
              "         -0.54640144,  0.3095368 ],\n",
              "        [ 0.28587252,  0.43272698, -0.15402156, ...,  0.45018288,\n",
              "         -0.3493012 ,  0.97700316],\n",
              "        ...,\n",
              "        [-0.10458773,  0.27263522,  0.03391352, ...,  0.265528  ,\n",
              "         -0.52032584,  0.21883468],\n",
              "        [-0.6415881 , -0.55679286,  0.5186954 , ..., -0.48380375,\n",
              "         -0.40191653, -1.1443204 ],\n",
              "        [-0.03406422, -0.06340456, -0.04942336, ..., -0.1155649 ,\n",
              "         -0.54424125,  0.22335248]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[-0.1464122 ,  0.175546  ,  0.14668007, ..., -0.25077525,\n",
              "         -0.80843776,  0.17914599],\n",
              "        [-0.30585358, -0.29958656, -0.31153122, ..., -0.18554911,\n",
              "         -0.3320423 , -0.22027646],\n",
              "        [ 0.10153656, -0.17803892,  0.08439865, ..., -0.7002616 ,\n",
              "         -0.3350033 ,  0.14119451],\n",
              "        ...,\n",
              "        [ 0.3423663 ,  0.58563006, -0.19767687, ...,  0.351862  ,\n",
              "         -0.6414875 ,  0.7488497 ],\n",
              "        [ 0.09045681,  0.8667017 , -0.20097443, ...,  0.27162656,\n",
              "         -0.48677635,  0.21553153],\n",
              "        [ 0.23880702,  0.49379843,  0.04227067, ...,  0.18505928,\n",
              "         -0.47207844,  0.1621174 ]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[ 0.24985431,  0.72062624, -0.09577925, ...,  0.26217076,\n",
              "         -0.43697786,  0.13986957],\n",
              "        [ 0.18193382,  0.82216495, -0.12819901, ...,  0.3806292 ,\n",
              "         -0.44452175,  0.2924575 ],\n",
              "        [ 0.31577557,  0.88780427, -0.24447687, ...,  0.28638446,\n",
              "         -0.65911454,  0.32017955],\n",
              "        ...,\n",
              "        [ 0.3061867 ,  0.7309455 , -0.45630705, ..., -0.05198695,\n",
              "         -1.0109775 ,  1.1847556 ],\n",
              "        [ 0.09870414,  0.47968328, -0.37348637, ...,  0.23388076,\n",
              "         -1.1102897 ,  0.87830526],\n",
              "        [ 0.5282401 ,  0.34981543,  0.08564341, ...,  0.10961034,\n",
              "         -0.9094299 ,  0.688156  ]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[ 0.08889334,  0.16713107,  0.02340168, ..., -0.04486939,\n",
              "         -0.57340086,  0.2546112 ],\n",
              "        [ 0.5495925 ,  0.8308296 , -0.35876194, ...,  0.09227581,\n",
              "         -0.8254601 ,  1.072566  ],\n",
              "        [ 0.04765711, -0.00762183, -0.10532968, ..., -0.5129871 ,\n",
              "         -0.55642766,  1.1335884 ],\n",
              "        ...,\n",
              "        [ 0.35946718,  0.1157268 , -0.31456503, ...,  0.42901272,\n",
              "         -0.34281152,  0.517075  ],\n",
              "        [ 0.4252884 ,  0.2537548 , -0.1214812 , ...,  0.5182926 ,\n",
              "         -0.36434966,  0.48326865],\n",
              "        [ 0.8531362 ,  0.65186036, -0.38560647, ...,  0.4032962 ,\n",
              "         -0.5365142 ,  0.5070005 ]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[ 0.22424106,  0.32209015, -0.25759637, ...,  0.48941362,\n",
              "         -0.28314283,  0.2880593 ],\n",
              "        [ 0.27863944,  0.3669665 , -0.23716992, ...,  0.31560582,\n",
              "         -0.47233823,  0.3454033 ],\n",
              "        [ 0.89701015,  0.6393595 , -0.5385041 , ...,  0.1325096 ,\n",
              "          0.00839926,  0.86231303],\n",
              "        ...,\n",
              "        [ 0.40723428,  1.1456412 ,  0.17252569, ...,  0.64120245,\n",
              "         -0.4080873 ,  0.6245851 ],\n",
              "        [ 0.29404828,  0.60391766, -0.27502948, ...,  0.20598508,\n",
              "         -0.43503377,  0.51962286],\n",
              "        [ 0.5719483 ,  1.2043589 ,  0.09766899, ...,  0.61103356,\n",
              "         -0.27657253,  0.7623711 ]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[ 0.5404781 ,  0.10412516, -0.15971789, ...,  0.5487774 ,\n",
              "         -0.4789466 ,  0.4315443 ],\n",
              "        [ 0.4431393 ,  0.89538306,  0.10078054, ...,  0.56523526,\n",
              "         -0.57292175,  0.7123042 ],\n",
              "        [ 0.5155684 ,  0.729125  ,  0.06967655, ...,  0.43746307,\n",
              "         -0.7651194 ,  0.693876  ],\n",
              "        ...,\n",
              "        [ 0.126738  ,  0.39066768, -0.01525419, ...,  0.38665828,\n",
              "         -0.73031825,  0.44211316],\n",
              "        [ 0.35260284,  0.3038126 ,  0.05491549, ...,  0.52455026,\n",
              "         -0.52892333,  0.16472168],\n",
              "        [ 0.58558965,  0.63859826, -0.16164257, ...,  0.48919228,\n",
              "         -0.7328696 , -0.07020758]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[ 0.38854226,  0.3735644 , -0.07952414, ...,  0.3477415 ,\n",
              "         -0.80323374,  0.15743807],\n",
              "        [ 0.81519675,  0.38440478, -0.07338443, ...,  0.47631064,\n",
              "         -0.6049184 ,  0.08031452],\n",
              "        [ 0.3705597 ,  0.5914797 , -0.15254776, ...,  0.6428111 ,\n",
              "         -0.69904107,  0.42255345],\n",
              "        ...,\n",
              "        [ 0.4341535 ,  0.21441951, -0.18879057, ...,  0.23492326,\n",
              "         -0.4207737 ,  0.3380057 ],\n",
              "        [ 0.14280991,  0.81629163,  0.16387762, ...,  0.04422454,\n",
              "         -0.7028291 ,  0.42284337],\n",
              "        [ 0.07144222,  0.3247802 , -0.09924142, ...,  0.30164942,\n",
              "         -0.4941344 ,  0.9402394 ]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[-0.06649326,  0.4389569 ,  0.17363466, ...,  0.0581261 ,\n",
              "         -0.44034472,  0.61222106],\n",
              "        [-0.5465233 , -1.124553  ,  0.5579491 , ..., -0.5176564 ,\n",
              "         -0.31999108, -0.7401602 ],\n",
              "        [ 0.29242244,  0.46416104, -0.38462657, ...,  0.4199731 ,\n",
              "         -0.6811066 ,  0.35934466],\n",
              "        ...,\n",
              "        [ 0.11599897,  0.35312983,  0.31708175, ...,  0.1646039 ,\n",
              "         -0.6624576 ,  0.8220618 ],\n",
              "        [-0.12130418, -0.03373461,  0.13379844, ..., -0.17142142,\n",
              "         -0.57086664,  0.87227464],\n",
              "        [-0.21208723, -0.14657547,  0.39373428, ..., -0.15115935,\n",
              "         -0.5627362 ,  0.81661516]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[ 0.17324579,  0.18587479,  0.01235794, ...,  0.08199985,\n",
              "         -0.79323673,  0.84476435],\n",
              "        [-0.0194194 , -0.03639492,  0.16072005, ..., -0.01420284,\n",
              "         -0.61131185,  0.9438889 ],\n",
              "        [-0.11328482,  0.35194916,  0.1417093 , ...,  0.21020837,\n",
              "         -0.16866504,  0.10789575],\n",
              "        ...,\n",
              "        [ 0.00306409,  0.2718405 , -0.03297516, ...,  0.27758977,\n",
              "         -0.41364172,  0.17603049],\n",
              "        [ 0.05689452,  0.28557414, -0.00206023, ..., -0.30614856,\n",
              "         -0.8098319 ,  0.98681355],\n",
              "        [-0.5475038 , -0.07821836,  0.506874  , ...,  0.14888628,\n",
              "         -0.15133561, -0.43156022]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[-0.11136685,  0.18157418,  0.03490132, ..., -0.14957342,\n",
              "         -0.8297894 , -0.05549465],\n",
              "        [ 0.18945841, -0.23811372,  0.17549013, ...,  0.15159377,\n",
              "         -0.7684381 , -0.40012097],\n",
              "        [-0.02172724, -0.24847299,  0.08291249, ...,  0.43213442,\n",
              "         -0.47215346,  0.22293441],\n",
              "        ...,\n",
              "        [-0.03785382,  0.14414437,  0.21212645, ...,  0.27597964,\n",
              "         -0.7005139 ,  0.7931075 ],\n",
              "        [-0.00662961,  0.2087726 ,  0.15985899, ...,  0.3352664 ,\n",
              "         -0.6960561 ,  0.8023909 ],\n",
              "        [ 0.37969252, -0.30308047,  0.08839465, ...,  0.14982367,\n",
              "         -0.82759833,  0.6012589 ]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[-0.06725577,  0.12168062,  0.25466308, ..., -0.24961263,\n",
              "         -0.7948897 ,  0.02268899],\n",
              "        [ 0.23207699, -0.25463477,  0.1924538 , ...,  0.25350934,\n",
              "         -0.59244686, -0.08777143],\n",
              "        [ 0.41471642, -0.04427533,  0.02472221, ...,  0.2761019 ,\n",
              "         -0.44848195, -0.0177274 ],\n",
              "        ...,\n",
              "        [ 0.06006682,  0.7703579 , -0.34302044, ...,  0.30698764,\n",
              "         -1.119358  ,  1.3380312 ],\n",
              "        [ 0.12286592,  0.5696432 , -0.1489874 , ...,  0.33848146,\n",
              "         -0.76949126,  0.97884667],\n",
              "        [ 0.21382317,  0.87827337, -0.31646577, ...,  0.8203157 ,\n",
              "         -0.8789492 ,  1.3250222 ]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[ 0.01655052,  0.8294454 , -0.2711699 , ...,  0.22089316,\n",
              "         -0.8453393 ,  0.59815043],\n",
              "        [-0.02791062,  0.73354644,  0.05465506, ...,  0.5601142 ,\n",
              "         -0.76577634,  1.1185762 ],\n",
              "        [ 0.35094222,  0.9453306 , -0.22560298, ...,  0.645362  ,\n",
              "         -0.8098575 ,  1.4562751 ],\n",
              "        ...,\n",
              "        [ 0.7291266 ,  1.0342542 , -0.12219348, ...,  0.51998895,\n",
              "         -0.32327482,  1.1637083 ],\n",
              "        [ 0.3311365 ,  0.4480165 , -0.25034755, ...,  0.56198764,\n",
              "          0.07019245,  0.5453573 ],\n",
              "        [ 0.63570523,  0.7192222 , -0.07452877, ...,  0.489186  ,\n",
              "         -0.60083735,  1.4428623 ]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[ 0.30132467,  0.6814254 , -0.3586182 , ...,  0.21969886,\n",
              "         -0.3884564 ,  0.04407474],\n",
              "        [ 0.1330121 ,  0.7841201 , -0.03400195, ...,  0.33939165,\n",
              "         -0.8473749 ,  0.9824689 ],\n",
              "        [-0.06610477,  0.649222  , -0.03803795, ...,  0.2989933 ,\n",
              "         -0.71126825,  0.9370901 ],\n",
              "        ...,\n",
              "        [ 0.27523455,  0.03065219,  0.09987245, ...,  0.56753385,\n",
              "         -0.57749456,  0.18317372],\n",
              "        [ 0.33923715,  0.29027537, -0.02099422, ...,  0.23972706,\n",
              "         -0.37580958,  0.228851  ],\n",
              "        [ 0.18400098,  0.2718451 ,  0.28441903, ...,  0.72389674,\n",
              "         -0.35875407,  0.2154068 ]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[-0.04260314,  0.27292526,  0.14313366, ...,  0.6762733 ,\n",
              "         -0.4074256 ,  0.2035831 ],\n",
              "        [ 0.09715012, -0.01817725,  0.04655338, ..., -0.04273693,\n",
              "         -0.29251114,  0.28227884],\n",
              "        [ 0.2459273 ,  0.46582213, -0.1523506 , ...,  0.39800906,\n",
              "         -0.16327047, -0.10275068],\n",
              "        ...,\n",
              "        [-0.19713482,  0.37590456,  0.10348152, ..., -0.26014128,\n",
              "         -0.20619342,  0.11880876],\n",
              "        [ 0.21906745,  0.15181062, -0.15468563, ...,  0.03658875,\n",
              "         -0.3917877 ,  0.9844929 ],\n",
              "        [ 0.14974646,  0.5647074 ,  0.00944525, ...,  0.5494519 ,\n",
              "         -0.64025986,  1.1279323 ]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[ 0.1488743 ,  0.7695134 , -0.09584476, ...,  0.5348843 ,\n",
              "         -0.02777718,  0.6113664 ],\n",
              "        [ 0.3028332 ,  0.65920275, -0.03137356, ...,  0.52296364,\n",
              "         -0.3822064 ,  1.0325187 ],\n",
              "        [ 0.50339997,  0.7166225 , -0.04848012, ...,  0.33151257,\n",
              "         -0.42512125,  0.7508548 ],\n",
              "        ...,\n",
              "        [ 0.24253136,  0.68694174,  0.07925204, ...,  0.57947564,\n",
              "         -0.8018829 ,  0.65275246],\n",
              "        [ 0.4428961 ,  0.4162957 , -0.13917315, ...,  0.39692456,\n",
              "         -0.7584097 ,  0.76880133],\n",
              "        [ 0.8426641 ,  0.14576289, -0.12374816, ...,  0.3049525 ,\n",
              "         -0.9221351 ,  0.36942828]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[ 0.8071418 ,  0.26640934, -0.13567317, ...,  0.34043926,\n",
              "         -0.84350044,  0.47509143],\n",
              "        [ 0.53760326,  0.13882877,  0.05441089, ..., -0.33036914,\n",
              "         -0.3732426 ,  0.25845113],\n",
              "        [ 0.4224757 ,  0.60515267, -0.3027662 , ...,  0.18672107,\n",
              "         -0.668012  ,  0.35976458],\n",
              "        ...,\n",
              "        [ 0.32428685,  0.58369374,  0.02898304, ...,  0.00546183,\n",
              "         -0.79492086,  0.84840083],\n",
              "        [-0.12066573, -0.26398015,  0.37035453, ..., -0.18876553,\n",
              "         -0.10557153, -0.59349775],\n",
              "        [ 0.41036376, -0.14882454,  0.19757925, ...,  0.15744142,\n",
              "         -0.19232388, -0.76184595]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[ 0.41572863, -0.16554993,  0.13144578, ..., -0.05506627,\n",
              "         -0.2058861 , -0.9415229 ],\n",
              "        [ 0.41572863, -0.16554993,  0.13144578, ..., -0.05506627,\n",
              "         -0.2058861 , -0.9415229 ],\n",
              "        [-0.21408218,  0.0354009 ,  0.30035377, ..., -0.44883463,\n",
              "         -0.61365294,  0.16712396],\n",
              "        ...,\n",
              "        [-0.2858945 ,  0.2574596 ,  0.4313294 , ..., -0.02435965,\n",
              "         -0.35933793, -0.0729022 ],\n",
              "        [ 0.50991756,  0.5236222 ,  0.10997125, ...,  0.285472  ,\n",
              "         -0.27326745,  0.68530804],\n",
              "        [ 0.1042879 ,  0.41177768,  0.21446961, ...,  0.1969343 ,\n",
              "         -0.38787147,  0.49171862]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[ 0.28941447,  0.5364072 ,  0.08167557, ...,  0.17297956,\n",
              "         -0.5962058 ,  0.58968693],\n",
              "        [-0.02349837,  0.51279515,  0.12203777, ...,  0.2411263 ,\n",
              "         -0.23012708,  0.48079997],\n",
              "        [ 0.09753379,  0.5166145 ,  0.37607372, ...,  0.22335172,\n",
              "         -0.06080085,  0.3017902 ],\n",
              "        ...,\n",
              "        [ 0.12050737,  0.7641096 ,  0.00939304, ...,  0.53329617,\n",
              "         -0.79932314,  0.5140848 ],\n",
              "        [ 0.36584416,  0.4254596 ,  0.01205726, ...,  0.4074947 ,\n",
              "         -0.41240934,  0.708062  ],\n",
              "        [-0.31603983,  0.39955643,  0.17808063, ...,  0.08530295,\n",
              "         -0.40153033,  0.2179726 ]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[ 0.36121625,  0.94407904, -0.15520039, ...,  0.5414938 ,\n",
              "         -0.9997876 ,  0.73624057],\n",
              "        [ 0.5263732 ,  0.8590815 , -0.0573445 , ...,  0.6065876 ,\n",
              "         -0.6266865 ,  0.4518441 ],\n",
              "        [ 0.16620742,  0.9733031 , -0.09100347, ...,  0.68056506,\n",
              "         -1.2562368 ,  0.80877477],\n",
              "        ...,\n",
              "        [-0.2681111 , -0.24163142,  0.3387013 , ..., -0.3747528 ,\n",
              "         -0.48507482,  0.44308865],\n",
              "        [-0.13271394, -0.37013927,  0.24762833, ...,  0.07554104,\n",
              "         -0.6235151 ,  0.43298453],\n",
              "        [ 0.21147627, -0.20856375, -0.08537959, ...,  0.33191094,\n",
              "         -0.9155308 ,  0.52356815]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[ 0.3968767 ,  0.07090001, -0.10161608, ...,  0.31948227,\n",
              "         -0.7075718 ,  1.0667208 ],\n",
              "        [ 0.48978716, -0.11410373, -0.14721625, ...,  0.56121266,\n",
              "         -0.8147645 ,  0.4370043 ],\n",
              "        [ 0.00541744,  0.03330313,  0.03036296, ...,  0.10976648,\n",
              "         -0.48804805,  0.70448893],\n",
              "        ...,\n",
              "        [ 0.18680891,  0.5333333 , -0.06187884, ...,  0.4953989 ,\n",
              "         -0.6733924 ,  0.9306482 ],\n",
              "        [ 0.07840654,  0.12861717,  0.01215151, ...,  0.34381   ,\n",
              "         -0.46287957,  0.7128444 ],\n",
              "        [ 0.2435374 ,  0.610012  , -0.04794603, ...,  0.08257686,\n",
              "         -0.6675359 ,  0.63622636]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[ 0.1490556 ,  0.37437788,  0.07089726, ...,  0.07362735,\n",
              "         -0.43582022,  0.59937906],\n",
              "        [ 0.38451454,  0.70795757,  0.2521555 , ...,  0.29536393,\n",
              "         -0.51347303,  0.5904188 ],\n",
              "        [-0.11231422,  0.17284136,  0.22045738, ...,  0.27210516,\n",
              "         -0.34875655,  0.3945038 ],\n",
              "        ...,\n",
              "        [ 0.19156949,  0.6244567 ,  0.02565807, ...,  0.71009856,\n",
              "         -0.6306871 ,  0.28534585],\n",
              "        [ 0.10865514,  0.31599504,  0.12329948, ...,  0.5460105 ,\n",
              "         -0.46650225,  0.4020375 ],\n",
              "        [ 0.09801386,  0.34250456, -0.00846934, ...,  0.31068945,\n",
              "         -0.5218049 ,  0.6705441 ]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[ 0.23018676,  0.4462028 , -0.2133594 , ...,  0.48548695,\n",
              "         -1.0447772 ,  0.43320668],\n",
              "        [ 0.15341526,  0.67076087, -0.16568503, ...,  0.3897142 ,\n",
              "         -0.90945375,  0.43095508],\n",
              "        [ 0.20461397,  0.38946038,  0.11392039, ...,  0.708928  ,\n",
              "         -0.5408876 ,  0.25267258],\n",
              "        ...,\n",
              "        [ 0.15937337,  0.13119118, -0.01465843, ...,  0.00595213,\n",
              "         -0.38154694,  0.14657798],\n",
              "        [ 0.5738313 ,  0.543767  , -0.21693084, ...,  0.47962216,\n",
              "         -0.62617016,  0.9727691 ],\n",
              "        [-0.12079615, -0.21947145,  0.14594476, ..., -0.09407096,\n",
              "         -0.49360037,  0.5654154 ]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[ 0.10727261,  0.26883727,  0.3088202 , ..., -0.25411206,\n",
              "         -0.69887775,  0.19423874],\n",
              "        [ 0.28637493,  0.70141715,  0.24646626, ...,  0.26967323,\n",
              "         -0.5894451 ,  0.951314  ],\n",
              "        [ 0.24601184,  0.17035572,  0.14442214, ...,  0.19548726,\n",
              "         -0.5053323 ,  0.884899  ],\n",
              "        ...,\n",
              "        [ 0.16103458,  0.03369559,  0.10014198, ...,  0.14102739,\n",
              "         -1.1342207 ,  0.6962849 ],\n",
              "        [ 0.07979911, -0.12606373, -0.11513853, ..., -0.11375601,\n",
              "         -0.7266346 ,  0.25593227],\n",
              "        [-0.15097609,  0.3758086 ,  0.06013557, ..., -0.3097233 ,\n",
              "         -0.74151266,  0.35492936]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[-0.06590167, -0.24584723, -0.0949191 , ..., -0.07862891,\n",
              "         -0.6669496 ,  0.09451632],\n",
              "        [ 0.44697267,  0.84339464, -0.36830294, ...,  0.69724315,\n",
              "         -0.5501991 ,  0.37256157],\n",
              "        [ 0.37325647,  0.4956124 , -0.20061272, ...,  0.7521054 ,\n",
              "         -0.5416607 ,  0.91962266],\n",
              "        ...,\n",
              "        [ 0.3533726 ,  0.05205043, -0.1761618 , ...,  0.50137395,\n",
              "         -0.8221627 ,  0.67615217],\n",
              "        [ 0.30527878,  0.07822542,  0.14590162, ...,  0.27787802,\n",
              "         -0.8840883 ,  0.3291156 ],\n",
              "        [-0.28973898, -0.14922105,  0.32465112, ..., -0.23985627,\n",
              "          0.0054338 ,  0.728845  ]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[ 4.38436598e-01,  1.05310544e-01, -1.79547727e-01, ...,\n",
              "          5.28941154e-01, -8.42412353e-01,  6.37197793e-01],\n",
              "        [ 5.12322962e-01,  3.90972316e-01, -5.94207466e-01, ...,\n",
              "          1.51858568e-01, -2.37925455e-01,  2.16791958e-01],\n",
              "        [ 8.21340144e-01,  2.85423041e-04, -3.88494194e-01, ...,\n",
              "          6.38570607e-01, -2.18411922e-01,  6.16821766e-01],\n",
              "        ...,\n",
              "        [-9.56401229e-02,  1.25966263e+00, -3.63609903e-02, ...,\n",
              "         -1.44257367e-01, -8.31029713e-01,  6.65080309e-01],\n",
              "        [-8.13011527e-02,  3.08245599e-01,  1.17681839e-01, ...,\n",
              "         -2.63439506e-01, -6.20396852e-01,  1.10300615e-01],\n",
              "        [ 3.72155122e-02,  4.24469739e-01,  1.51009262e-02, ...,\n",
              "          4.12593216e-01, -8.19372952e-01,  2.66456664e-01]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[ 0.48129874,  0.16016784, -0.5691415 , ...,  0.97178876,\n",
              "         -0.53141856,  0.01047113],\n",
              "        [ 0.24972327,  0.3831345 , -0.25618732, ...,  0.38291633,\n",
              "         -0.63059896,  0.07866368],\n",
              "        [-0.04763549,  0.17098191,  0.28870496, ..., -0.35152435,\n",
              "         -0.49087378, -0.02847356],\n",
              "        ...,\n",
              "        [ 0.6708065 ,  0.64129156, -0.11832799, ...,  0.29898506,\n",
              "         -0.6471307 ,  0.92082196],\n",
              "        [ 0.774798  ,  0.59940034, -0.28722215, ...,  0.6142622 ,\n",
              "         -0.38883618,  0.5825647 ],\n",
              "        [ 0.56372935,  0.97804606, -0.1948443 , ...,  0.40362877,\n",
              "         -0.5435379 ,  0.26442775]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[ 0.3629208 ,  0.7922613 , -0.1722442 , ...,  0.6218373 ,\n",
              "         -0.87550604,  0.44041207],\n",
              "        [ 0.11774167,  0.26108506,  0.3247916 , ...,  0.3701537 ,\n",
              "         -0.54987216,  0.6205007 ],\n",
              "        [ 0.37840074,  0.8290572 ,  0.04586449, ...,  0.42665857,\n",
              "         -0.55506986,  0.50999004],\n",
              "        ...,\n",
              "        [ 0.15458529,  0.22799423,  0.23160206, ...,  0.43363166,\n",
              "         -0.50440174,  0.24949257],\n",
              "        [ 0.47988364,  0.14173171,  0.07829133, ...,  0.47739625,\n",
              "         -0.620689  , -0.06834707],\n",
              "        [ 0.16702494,  0.25452346,  0.13978285, ...,  0.5039698 ,\n",
              "         -0.63105357,  0.5272646 ]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[ 0.07510505, -0.02630998,  0.10323071, ...,  0.43307918,\n",
              "         -0.52154714,  0.34036046],\n",
              "        [ 0.45862055,  0.15199192,  0.08253706, ...,  0.49847957,\n",
              "         -0.583775  ,  0.12519506],\n",
              "        [ 0.24367774,  0.08308162,  0.11360378, ...,  0.5263312 ,\n",
              "         -0.3815117 ,  0.17766635],\n",
              "        ...,\n",
              "        [ 0.12427872,  0.7422735 , -0.01278383, ...,  0.22140294,\n",
              "         -0.5210907 ,  0.15514582],\n",
              "        [-0.48089072,  0.35506874,  0.28277606, ..., -0.7758032 ,\n",
              "         -0.28193992,  0.927873  ],\n",
              "        [ 0.45754355,  0.11124886, -0.10373747, ..., -0.1052884 ,\n",
              "         -0.33330962,  0.9842299 ]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[ 0.05767048,  0.6784576 ,  0.11798032, ...,  0.06327491,\n",
              "          0.02011503,  0.18097539],\n",
              "        [ 0.0458347 ,  0.62441677,  0.12430233, ...,  0.05168312,\n",
              "         -0.5940815 ,  0.19486046],\n",
              "        [-0.37624815,  0.37015012,  0.06212066, ..., -0.44116965,\n",
              "         -0.58932287,  0.13675879],\n",
              "        ...,\n",
              "        [ 0.39463633,  0.6391019 , -0.22980069, ...,  0.23239905,\n",
              "         -0.73994285,  1.2287415 ],\n",
              "        [ 0.34591419,  0.63935167, -0.20617235, ...,  0.23749623,\n",
              "         -0.7434813 ,  1.2184438 ],\n",
              "        [ 0.32645798,  0.46562687, -0.09097634, ..., -0.27737895,\n",
              "         -0.6587117 ,  0.82292676]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[ 0.5166099 ,  0.5589155 , -0.12654743, ...,  0.50015205,\n",
              "         -0.9846431 ,  0.91059643],\n",
              "        [ 0.5583706 ,  0.63716316, -0.13690181, ...,  0.46166742,\n",
              "         -0.953922  ,  0.9311324 ],\n",
              "        [ 0.5455674 ,  0.69437295, -0.20920247, ...,  0.25610343,\n",
              "         -1.0855858 ,  0.9145943 ],\n",
              "        ...,\n",
              "        [ 0.24306796,  0.29211944,  0.04529058, ...,  0.48041067,\n",
              "         -0.00887271,  0.20168845],\n",
              "        [ 0.22126931,  0.42204466,  0.13641359, ...,  0.6886028 ,\n",
              "         -0.01845839, -0.3738218 ],\n",
              "        [ 0.19991499,  0.30568367, -0.08364878, ...,  0.34717506,\n",
              "         -0.03802823, -0.44528425]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[ 0.00571867,  0.5689832 ,  0.11146595, ...,  0.494468  ,\n",
              "         -0.3811729 , -0.40087178],\n",
              "        [ 0.21047543,  0.8450965 , -0.00878942, ...,  0.76812536,\n",
              "          0.05733347,  0.56220543],\n",
              "        [ 0.32459033,  1.0109932 ,  0.12942563, ...,  0.56765944,\n",
              "          0.01996663,  0.04107491],\n",
              "        ...,\n",
              "        [ 0.0273682 ,  0.3609271 , -0.03880462, ...,  0.5354582 ,\n",
              "         -0.11057856,  0.27993917],\n",
              "        [ 0.1057364 ,  0.3018772 ,  0.14686969, ...,  0.6978316 ,\n",
              "          0.05919636,  0.25341067],\n",
              "        [ 0.12098888,  0.5470804 , -0.04074562, ...,  0.73170286,\n",
              "         -0.1277496 ,  0.23804708]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[ 0.00194599,  0.03980017,  0.01924944, ...,  0.57383925,\n",
              "         -0.08425857,  0.20431818],\n",
              "        [ 0.23505557,  0.25117046, -0.10832252, ...,  0.5990385 ,\n",
              "         -0.33925727,  0.0718202 ],\n",
              "        [ 0.14767109,  0.24051541,  0.11039366, ...,  0.43375218,\n",
              "         -0.13995771,  0.36328736],\n",
              "        ...,\n",
              "        [ 0.28035435,  0.51324606,  0.00728543, ...,  0.640002  ,\n",
              "          0.10868333,  0.67270356],\n",
              "        [-0.05035577,  1.0068604 , -0.1497363 , ...,  0.6850552 ,\n",
              "         -0.5648898 ,  1.3428581 ],\n",
              "        [ 0.21748847,  0.8230658 ,  0.0174663 , ...,  0.44952863,\n",
              "         -0.31135297,  1.0329726 ]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(2560, 384), dtype=float32, numpy=\n",
              " array([[ 0.33361128,  0.9949328 , -0.21699554, ...,  0.53282094,\n",
              "         -0.4805457 ,  1.1110184 ],\n",
              "        [ 0.14175911,  0.6773977 ,  0.03766262, ...,  0.45526415,\n",
              "         -0.17630497,  0.90425515],\n",
              "        [ 0.26826644,  1.4036442 , -0.1380129 , ...,  0.79485697,\n",
              "         -0.6117353 ,  1.0099295 ],\n",
              "        ...,\n",
              "        [ 0.02534301,  1.061204  , -0.14722796, ...,  0.02857254,\n",
              "         -0.69393456,  0.8082855 ],\n",
              "        [-0.11182072,  0.3292352 ,  0.10625602, ...,  0.4261853 ,\n",
              "         -0.47815022,  0.4932231 ],\n",
              "        [-0.13715905,  0.59341025,  0.23041216, ..., -0.0028236 ,\n",
              "         -0.46424943,  0.4001049 ]], dtype=float32)>]"
            ]
          },
          "execution_count": 128,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "embeddings"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "EwETIRKQbERR"
      },
      "outputs": [],
      "source": [
        "#np.savez_compressed('embeddings.npz', embeddings)\n",
        "#np.savez_compressed('product_titles.npz',product_titles)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "643hCYcJYaIi"
      },
      "outputs": [],
      "source": [
        "!cp '/content/embeddings.npz' '/content/drive/MyDrive/stransformer/'\n",
        "!cp '/content/product_titles.npz' '/content/drive/MyDrive/stransformer/'"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AUwuU3gLvL6p"
      },
      "source": [
        "## Use Embeddings"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "tgFI0P4KYaKn"
      },
      "outputs": [],
      "source": [
        "loaded_embedding=np.load('/content/drive/MyDrive/stransformer/embeddings.npz')\n",
        "loaded_embedding_array=np.array(loaded_embedding['arr_0'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Gn09aOMc4Mfn"
      },
      "outputs": [],
      "source": [
        "loaded_titles=np.load('/content/drive/MyDrive/stransformer/product_titles.npz')\n",
        "loaded_titles_array=np.array(loaded_titles['arr_0'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WSPWbmv9YaNe",
        "outputId": "bd3c08e9-2adb-4c59-af09-f04ef962aa5d"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "(2836, 640, 384)"
            ]
          },
          "execution_count": 91,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "loaded_embedding_array.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "z9MhfqLmYaPv",
        "outputId": "197bfe8a-cc83-45a2-8da8-69707489b859"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "(1815040, 384)\n"
          ]
        }
      ],
      "source": [
        "loaded_embedding_array=loaded_embedding_array.reshape(-1,loaded_embedding_array.shape[2])\n",
        "print(loaded_embedding_array.shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Iv-Lia6yYmkt"
      },
      "outputs": [],
      "source": [
        "\n",
        "tokenizer = AutoTokenizer.from_pretrained(model_id)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WybnyIrbkmUM",
        "outputId": "6058f748-e97a-4aa8-9dc3-f1e998a55635"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "(1, 384)\n"
          ]
        }
      ],
      "source": [
        "inputs = tokenizer([\"針なしほっちきす\"],max_length=MAX_LENGTH,padding='max_length',truncation=True,return_tensors=\"tf\")\n",
        "\n",
        "logits = model(**inputs)\n",
        "out_embedding=mean_pooling(logits,inputs['attention_mask'])\n",
        "print(out_embedding.shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Qa_j2X4uGJ5U",
        "outputId": "6817775b-6467-4b8f-fd3d-a36587c329e9"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "(1815040, 1)\n"
          ]
        }
      ],
      "source": [
        "u_dot_v=np.matmul(loaded_embedding_array,(np.array(out_embedding).T))\n",
        "print(u_dot_v.shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NLmXRkR_GJ7_",
        "outputId": "f3543847-b05c-4797-a1b0-c3899ef90f6a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "(1815040,)\n",
            "[11.215763   9.297123  10.870735  ...  5.05177    7.0463676  4.7366333]\n"
          ]
        }
      ],
      "source": [
        "u_magnitude=np.sqrt(np.sum(loaded_embedding_array*loaded_embedding_array,axis=-1))\n",
        "print(u_magnitude.shape)\n",
        "print(u_magnitude)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4Ke-taACGJ-b",
        "outputId": "0afe68fa-011e-45e3-dd3e-85e802f8785e"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "(1,)\n",
            "[6.5474906]\n"
          ]
        }
      ],
      "source": [
        "v_magnitude=np.sqrt(np.sum(out_embedding*out_embedding,axis=-1))\n",
        "print(v_magnitude.shape)\n",
        "print(v_magnitude)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "q6Btr8vjGKBI",
        "outputId": "5f649baa-0dd6-435b-a4c1-a81951199df8"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[[0.24800435 0.33141387 0.11507133 ... 0.12804486 0.35886842 0.08012675]]\n"
          ]
        }
      ],
      "source": [
        "cosine_similarity=u_dot_v.T/(u_magnitude*v_magnitude)\n",
        "print(cosine_similarity)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "faynJ8a7GKDe",
        "outputId": "59b77df1-0311-4e5a-cc5b-49b731580417"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[[  83865 1386827  779273 ... 1811568 1139688 1803600]]\n"
          ]
        }
      ],
      "source": [
        "sorted_indices=np.argsort(cosine_similarity,axis=-1)\n",
        "print(sorted_indices)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rMshUKwiKUy0",
        "outputId": "474498e7-4fc8-4847-b86d-aa2c40e3b115"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "0 ['知りすぎたのね']\n",
            "1 ['リバウンドしない収納']\n",
            "2 ['ヴェノム (吹替版)']\n",
            "3 ['ヴェノム (字幕版)']\n",
            "4 ['メルちゃん おせわパーツ おしょくじ&はみがきセット']\n",
            "5 ['スウィングしなけりゃ意味ないね']\n",
            "6 ['スウィングしなけりゃ意味ないね']\n",
            "7 ['スウィングしなけりゃ意味ないね']\n",
            "8 ['スウィングしなけりゃ意味ないね']\n",
            "9 ['スイングしなけりゃ意味ないね']\n",
            "10 ['スウィングしなけりゃ意味ないね']\n",
            "11 ['スイングしなけりゃ意味ないね']\n",
            "12 ['スウィングしなけりゃ意味ないね']\n",
            "13 ['スウィングしなけりゃ意味ないね']\n",
            "14 ['スイングしなけりゃ意味ないね']\n",
            "15 ['スウィングしなけりゃ意味ないね']\n",
            "16 ['すみっコぐらしの英会話']\n",
            "17 ['映画あたしンち']\n",
            "18 ['クラリネットピース スウィングしなけゃ意味ないね']\n",
            "19 ['すみっコぐらしのお弁当']\n",
            "20 ['嫁入り道具の花ふきん']\n",
            "21 ['ビジュアルヌード・ポーズＢＯＯＫ ａｃｔ松岡ちな']\n",
            "22 ['Best of Ramin Djawadi']\n",
            "23 ['CHARMINER 0045']\n",
            "24 ['キズナアイ 1/7 完成品フィギュア']\n"
          ]
        }
      ],
      "source": [
        "for i in range(25):\n",
        "  print(i,loaded_titles_array[sorted_indices[:,len(sorted_indices[0])-i-1]])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ipeY2AFZKU1W"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "sl6JAXp_KU3w"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ObgcYw87KU57"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "wfESJM7IGKFu"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "OVmI9EVKGKIa"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "hcBvfbLIGKLU"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "aVoZagmWvT9d"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Qp4St55AvT_s"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "DCOcbMo0vUCX"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "uQAXVW7AvUFR"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7HULCj4WZllJ"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Lb4K-W1pZlnB"
      },
      "outputs": [],
      "source": [
        "small_dataset = dataset.filter(lambda example: example[\"query\"].startswith(\"A\"))\n",
        "print(len(small_dataset))"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [
        "aSaOBWrxwzaE",
        "DczeG3N5x8hL",
        "ogDlZqTX0Cf-",
        "LVQaVIEHVJ4W"
      ],
      "machine_shape": "hm",
      "provenance": [],
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "001f6a8fa56846628ce364e6c18f724b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_78b4b45d7273428c9a48ed1676fe8263",
            "max": 731636,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_ba10cdcacab24304a1dbe3fc01868e89",
            "value": 731636
          }
        },
        "06462707efcb4e6589b0e8b3e9241d3f": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0af335553d2c42349e4641a0018dfe7a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_cbe1f26047bf4adb8a66b9d0bd1c0a77",
              "IPY_MODEL_865f50d0e9f7418c97f9da9d4c0c021e",
              "IPY_MODEL_4c32b2e5411a4727a4aa7f860cbca221"
            ],
            "layout": "IPY_MODEL_4ea61ecd7e1a4678b546e60e02cdf400"
          }
        },
        "107af6e8e531449881cf234a518d962a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5362841eba8a4ee39b9e28ab7df0812a",
            "placeholder": "​",
            "style": "IPY_MODEL_df0b06a1893a4193a8fd5c3683cdbf6a",
            "value": " 1/1 [00:00&lt;00:00,  3.90it/s]"
          }
        },
        "160f9a290bf444ebb76e1cb6e46c272b": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "20px"
          }
        },
        "17e4888ada794ca58efe8139c1915bde": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_e86f14c6ca0540c3b43bbf286c082fa5",
              "IPY_MODEL_24dd3128d5dd42588c2818f545a1729f",
              "IPY_MODEL_b6e5c71a5cdb46c58d61cb9075b538cd"
            ],
            "layout": "IPY_MODEL_a14327416e534c2c82590d3bf9f53a56"
          }
        },
        "1b983274bbaf48ddb0e27036e7e3d9cb": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "23556057e0064150a3673b2752ac0543": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "23f81baef07346db9992662c4e1016a5": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "24dd3128d5dd42588c2818f545a1729f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_be106420c9e248c3af0d20dd26347e36",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_baf23296164e4fb49055190dc27d73e5",
            "value": 1
          }
        },
        "283d37bc43f949e4aa8ac1b50b2dd239": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "290b049faa1d47fe94918894589df21f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2a14f9fe2ab84481bbfae2f497a2e97b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "316e0036819541a798464be65a1c5a4b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a222503085d142669593dd5fe8ef2526",
            "placeholder": "​",
            "style": "IPY_MODEL_75db01a35994498389abbe7e713e75c4",
            "value": " 75/? [00:09&lt;00:00, 13.55 tables/s]"
          }
        },
        "331daf2ca76b4752b10406fc20783c12": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_60782d79f0244c6b9d1071c3effe5075",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_3ef5dadae87e4f11b5c5ca5001ebb61d",
            "value": 1
          }
        },
        "3ef5dadae87e4f11b5c5ca5001ebb61d": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "41dafa3fe83f495189a5b675ad4a46c4": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_160f9a290bf444ebb76e1cb6e46c272b",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_2a14f9fe2ab84481bbfae2f497a2e97b",
            "value": 1
          }
        },
        "4942db5c2d26408fa12aa884f76f04b8": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_06462707efcb4e6589b0e8b3e9241d3f",
            "placeholder": "​",
            "style": "IPY_MODEL_4aa639a3c9ff4743a8ab11b0d9d6c11c",
            "value": " 731636/731636 [06:09&lt;00:00, 1794.13ex/s]"
          }
        },
        "4a2b49a2ae1c4dfab7daaff9a0fccc11": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4aa639a3c9ff4743a8ab11b0d9d6c11c": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4c32b2e5411a4727a4aa7f860cbca221": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bde0b764bd6a4281b16943fd93478f73",
            "placeholder": "​",
            "style": "IPY_MODEL_4a2b49a2ae1c4dfab7daaff9a0fccc11",
            "value": " 1/1 [00:00&lt;00:00, 10.19it/s]"
          }
        },
        "4d0c4ed8eac54185b5188b421126f2c4": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_ea41ad65203a4c75ad4ce3410d3abcc7",
              "IPY_MODEL_001f6a8fa56846628ce364e6c18f724b",
              "IPY_MODEL_4942db5c2d26408fa12aa884f76f04b8"
            ],
            "layout": "IPY_MODEL_b9957450854e408ab6c2c0ec41dca942"
          }
        },
        "4ea61ecd7e1a4678b546e60e02cdf400": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "516475b9825445a89135230c0ebc689c": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5362841eba8a4ee39b9e28ab7df0812a": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "557eb4180d0c41f093249bb0dab2a66b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_f61c4e25c8dd491faebb567c3414b487",
              "IPY_MODEL_331daf2ca76b4752b10406fc20783c12",
              "IPY_MODEL_107af6e8e531449881cf234a518d962a"
            ],
            "layout": "IPY_MODEL_a974c19cb2d74f85aa2d6175b7b054b4"
          }
        },
        "60782d79f0244c6b9d1071c3effe5075": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "75db01a35994498389abbe7e713e75c4": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "78630f0a9bea42ea815111db44922a98": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "78b4b45d7273428c9a48ed1676fe8263": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "865f50d0e9f7418c97f9da9d4c0c021e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_994a985977974fe99398974b4db2c2e7",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_d32142e28bdb475ca1e4e57e4a2a48a7",
            "value": 1
          }
        },
        "904bd5a359064db5ad790932f1c79791": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "994a985977974fe99398974b4db2c2e7": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a14327416e534c2c82590d3bf9f53a56": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a222503085d142669593dd5fe8ef2526": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a465957f9c23410bbc42b99f53112905": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": "hidden",
            "width": null
          }
        },
        "a974c19cb2d74f85aa2d6175b7b054b4": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b6e5c71a5cdb46c58d61cb9075b538cd": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_78630f0a9bea42ea815111db44922a98",
            "placeholder": "​",
            "style": "IPY_MODEL_1b983274bbaf48ddb0e27036e7e3d9cb",
            "value": " 1/1 [00:00&lt;00:00, 11.51it/s]"
          }
        },
        "b84d233573e24760839453ab88ecf8b4": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_516475b9825445a89135230c0ebc689c",
            "placeholder": "​",
            "style": "IPY_MODEL_23556057e0064150a3673b2752ac0543",
            "value": ""
          }
        },
        "b9957450854e408ab6c2c0ec41dca942": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ba10cdcacab24304a1dbe3fc01868e89": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "baf23296164e4fb49055190dc27d73e5": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "bde0b764bd6a4281b16943fd93478f73": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "be106420c9e248c3af0d20dd26347e36": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c7ef2649314844b996706998f548b8a2": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cbe1f26047bf4adb8a66b9d0bd1c0a77": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f7f0278a0c6c4fc28925e9d3c352c67a",
            "placeholder": "​",
            "style": "IPY_MODEL_283d37bc43f949e4aa8ac1b50b2dd239",
            "value": "100%"
          }
        },
        "ccb197e162fd4b969b132ac985527a22": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_b84d233573e24760839453ab88ecf8b4",
              "IPY_MODEL_41dafa3fe83f495189a5b675ad4a46c4",
              "IPY_MODEL_316e0036819541a798464be65a1c5a4b"
            ],
            "layout": "IPY_MODEL_a465957f9c23410bbc42b99f53112905"
          }
        },
        "d32142e28bdb475ca1e4e57e4a2a48a7": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "df0b06a1893a4193a8fd5c3683cdbf6a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e86f14c6ca0540c3b43bbf286c082fa5": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c7ef2649314844b996706998f548b8a2",
            "placeholder": "​",
            "style": "IPY_MODEL_290b049faa1d47fe94918894589df21f",
            "value": "Downloading data files: 100%"
          }
        },
        "ea41ad65203a4c75ad4ce3410d3abcc7": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fefe375e7e0d4e90a3f3dc5e2cfd6cf7",
            "placeholder": "​",
            "style": "IPY_MODEL_904bd5a359064db5ad790932f1c79791",
            "value": "100%"
          }
        },
        "f61c4e25c8dd491faebb567c3414b487": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f7a4ff94b0a94d129498dfbb41e81778",
            "placeholder": "​",
            "style": "IPY_MODEL_23f81baef07346db9992662c4e1016a5",
            "value": "Extracting data files: 100%"
          }
        },
        "f7a4ff94b0a94d129498dfbb41e81778": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f7f0278a0c6c4fc28925e9d3c352c67a": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fefe375e7e0d4e90a3f3dc5e2cfd6cf7": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}