xls_r_bemba_model_eval.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "xls-r-bemba-model-eval.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm",
      "authorship_tag": "ABX9TyNH0+mtvrckKZYsFRaFrvno",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/csikasote/xls-r-bem-exp/blob/main/xls_r_bemba_model_eval.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip -q install datasets transformers torchaudio librosa huggingface_hub jiwer\n",
        "!pip install https://github.com/kpu/kenlm/archive/master.zip\n",
        "!pip install pyctcdecode"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "u97KwaSBgsws",
        "outputId": "dd7ab0e5-2930-4715-b02c-486826f281ce"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[K     |████████████████████████████████| 311 kB 14.2 MB/s \n",
            "\u001b[K     |████████████████████████████████| 3.5 MB 63.5 MB/s \n",
            "\u001b[K     |████████████████████████████████| 67 kB 6.8 MB/s \n",
            "\u001b[K     |████████████████████████████████| 133 kB 99.1 MB/s \n",
            "\u001b[K     |████████████████████████████████| 243 kB 87.3 MB/s \n",
            "\u001b[K     |████████████████████████████████| 1.1 MB 74.5 MB/s \n",
            "\u001b[K     |████████████████████████████████| 6.8 MB 71.8 MB/s \n",
            "\u001b[K     |████████████████████████████████| 596 kB 71.1 MB/s \n",
            "\u001b[K     |████████████████████████████████| 895 kB 72.7 MB/s \n",
            "\u001b[K     |████████████████████████████████| 50 kB 7.9 MB/s \n",
            "\u001b[K     |████████████████████████████████| 144 kB 87.3 MB/s \n",
            "\u001b[K     |████████████████████████████████| 94 kB 4.1 MB/s \n",
            "\u001b[K     |████████████████████████████████| 271 kB 90.1 MB/s \n",
            "\u001b[?25h  Building wheel for python-Levenshtein (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting https://github.com/kpu/kenlm/archive/master.zip\n",
            "  Downloading https://github.com/kpu/kenlm/archive/master.zip (541 kB)\n",
            "\u001b[K     |████████████████████████████████| 541 kB 4.8 MB/s \n",
            "\u001b[?25hBuilding wheels for collected packages: kenlm\n",
            "  Building wheel for kenlm (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for kenlm: filename=kenlm-0.0.0-cp37-cp37m-linux_x86_64.whl size=2336488 sha256=f4bb558ef0d879ea9d58a2786d6d1126eb94e8cd7a84452790016c8bc37e8093\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-af5cou0g/wheels/3d/aa/02/7b4a2eab5d7a2a9391bd9680dbad6270808a147bc3b7047e4e\n",
            "Successfully built kenlm\n",
            "Installing collected packages: kenlm\n",
            "Successfully installed kenlm-0.0.0\n",
            "Collecting pyctcdecode\n",
            "  Downloading pyctcdecode-0.3.0-py2.py3-none-any.whl (43 kB)\n",
            "\u001b[K     |████████████████████████████████| 43 kB 2.2 MB/s \n",
            "\u001b[?25hCollecting pygtrie<3.0,>=2.1\n",
            "  Downloading pygtrie-2.4.2.tar.gz (35 kB)\n",
            "Requirement already satisfied: numpy<2.0.0,>=1.15.0 in /usr/local/lib/python3.7/dist-packages (from pyctcdecode) (1.21.5)\n",
            "Collecting hypothesis<7,>=6.14\n",
            "  Downloading hypothesis-6.36.2-py3-none-any.whl (376 kB)\n",
            "\u001b[K     |████████████████████████████████| 376 kB 32.0 MB/s \n",
            "\u001b[?25hRequirement already satisfied: sortedcontainers<3.0.0,>=2.1.0 in /usr/local/lib/python3.7/dist-packages (from hypothesis<7,>=6.14->pyctcdecode) (2.4.0)\n",
            "Requirement already satisfied: attrs>=19.2.0 in /usr/local/lib/python3.7/dist-packages (from hypothesis<7,>=6.14->pyctcdecode) (21.4.0)\n",
            "Building wheels for collected packages: pygtrie\n",
            "  Building wheel for pygtrie (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pygtrie: filename=pygtrie-2.4.2-py3-none-any.whl size=19063 sha256=26238bbc3b9b74251aa7909923513d542cf752e448da04c1e86b661fa5211e92\n",
            "  Stored in directory: /root/.cache/pip/wheels/d3/f8/ba/1d828b1603ea422686eb694253a43cb3a5901ea4696c1e0603\n",
            "Successfully built pygtrie\n",
            "Installing collected packages: pygtrie, hypothesis, pyctcdecode\n",
            "Successfully installed hypothesis-6.36.2 pyctcdecode-0.3.0 pygtrie-2.4.2\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Download BembaSpeech dataset from github repo\n",
        "!git clone https://github.com/csikasote/BembaSpeech.git \n",
        "%cd BembaSpeech\n",
        "!python prepare.py\n",
        "%cd /content"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ww1q503AYs5L",
        "outputId": "cafb3d4c-b3cf-49df-f03b-5e6bf2860c15"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'BembaSpeech'...\n",
            "remote: Enumerating objects: 35625, done.\u001b[K\n",
            "remote: Counting objects: 100% (242/242), done.\u001b[K\n",
            "remote: Compressing objects: 100% (239/239), done.\u001b[K\n",
            "remote: Total 35625 (delta 109), reused 7 (delta 1), pack-reused 35383\u001b[K\n",
            "Receiving objects: 100% (35625/35625), 2.44 GiB | 28.67 MiB/s, done.\n",
            "Resolving deltas: 100% (17681/17681), done.\n",
            "Checking out files: 100% (15140/15140), done.\n",
            "/content/BembaSpeech\n",
            "Preparing audio files and generating csv files for training the model ... \n",
            "\n",
            "N/A% (0 of 3) |                           | Elapsed Time: 0:00:00 ETA:  --:--:--> 'train.csv' has been processed successfully!\n",
            "> Training set size: 20.05 hrs\n",
            "> No of files: 11892\n",
            " 33% (1 of 3) |#########                  | Elapsed Time: 0:00:01 ETA:   0:00:02> 'dev.csv' has been created successfully!\n",
            "> Evaluation set size: 2.42 hrs\n",
            "> No of files: 1549\n",
            " 66% (2 of 3) |##################         | Elapsed Time: 0:00:02 ETA:   0:00:01> 'test.csv' has been created successfully!\n",
            "> Test set size: 2.05 hrs\n",
            "> No. of file: 977\n",
            "100% (3 of 3) |###########################| Elapsed Time: 0:00:02 Time:  0:00:02\n",
            "/content\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!wget https://raw.githubusercontent.com/csikasote/xls-r-bem-exp/main/eval.py"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Gjgkbuk9Yqe9",
        "outputId": "e58e228f-6f4c-4be8-c5bd-8cafdc8f2743"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2022-02-17 08:18:21--  https://raw.githubusercontent.com/csikasote/xsl-r-bem-exp/main/eval.py\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.108.133, 185.199.109.133, 185.199.110.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.108.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 4992 (4.9K) [text/plain]\n",
            "Saving to: ‘eval.py’\n",
            "\n",
            "eval.py             100%[===================>]   4.88K  --.-KB/s    in 0s      \n",
            "\n",
            "2022-02-17 08:18:22 (70.1 MB/s) - ‘eval.py’ saved [4992/4992]\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!python eval.py \\\n",
        "  --model_id csikasote/wav2vec2-large-xls-r-1b-bemba-fds \\\n",
        "  --dataset bembaspeech \\\n",
        "  --config bem \\\n",
        "  --split test \\\n",
        "  --path /content/BembaSpeech/BembaSpeech/csv/test.csv \\\n",
        "  --log_outputs"
      ],
      "metadata": {
        "id": "m5VxnywaYyCJ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a443d805-fa7f-47bf-daa3-1f31b3bdb5e6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Using custom data configuration default-13b84fa24be49824\n",
            "Reusing dataset csv (/root/.cache/huggingface/datasets/csv/default-13b84fa24be49824/0.0.0/6b9057d9e23d9d8a2f05b985917a0da84d70c5dae3d22ddd8a3f22fb01c69d9e)\n",
            "\r  0% 0/1 [00:00<?, ?it/s]\r100% 1/1 [00:00<00:00, 676.06it/s]\n",
            "tcmalloc: large alloc 1158389760 bytes == 0x558d92002000 @  0x7fa9b0d7a615 0x558b7ec833bc 0x558b7ed6418a 0x558b7ec861cd 0x558b7ec86120 0x558b7ecf9f33 0x558b7ec879da 0x558b7ecfa2c0 0x558b7ec879da 0x558b7ecf5eae 0x558b7ecf502f 0x558b7ec87aba 0x558b7ecf5eae 0x558b7ec879da 0x558b7ecf5eae 0x558b7ecf502f 0x558b7ec87aba 0x558b7ecf5eae 0x558b7ecf502f 0x558b7ec8836c 0x558b7ec88571 0x558b7ecf7633 0x558b7ecf502f 0x558b7ec87aba 0x558b7ecf5eae 0x558b7ec879da 0x558b7ecf5eae 0x558b7ecf502f 0x558b7ec87aba 0x558b7ecf5eae 0x558b7ecf502f\n",
            "tcmalloc: large alloc 1447993344 bytes == 0x558dd70bc000 @  0x7fa9b0d7a615 0x558b7ec833bc 0x558b7ed6418a 0x558b7ec861cd 0x558b7ec86120 0x558b7ecf9f33 0x558b7ec879da 0x558b7ecfa2c0 0x558b7ec879da 0x558b7ecf5eae 0x558b7ecf502f 0x558b7ec87aba 0x558b7ecf5eae 0x558b7ec879da 0x558b7ecf5eae 0x558b7ecf502f 0x558b7ec87aba 0x558b7ecf5eae 0x558b7ecf502f 0x558b7ec8836c 0x558b7ec88571 0x558b7ecf7633 0x558b7ecf502f 0x558b7ec87aba 0x558b7ecf5eae 0x558b7ec879da 0x558b7ecf5eae 0x558b7ecf502f 0x558b7ec87aba 0x558b7ecf5eae 0x558b7ecf502f\n",
            "tcmalloc: large alloc 1809997824 bytes == 0x558d5ac3a000 @  0x7fa9b0d7a615 0x558b7ec833bc 0x558b7ed6418a 0x558b7ec861cd 0x558b7ec86120 0x558b7ecf9f33 0x558b7ec879da 0x558b7ecfa2c0 0x558b7ec879da 0x558b7ecf5eae 0x558b7ecf502f 0x558b7ec87aba 0x558b7ecf5eae 0x558b7ec879da 0x558b7ecf5eae 0x558b7ecf502f 0x558b7ec87aba 0x558b7ecf5eae 0x558b7ecf502f 0x558b7ec8836c 0x558b7ec88571 0x558b7ecf7633 0x558b7ecf502f 0x558b7ec87aba 0x558b7ecf5eae 0x558b7ec879da 0x558b7ecf5eae 0x558b7ecf502f 0x558b7ec87aba 0x558b7ecf5eae 0x558b7ecf502f\n",
            "tcmalloc: large alloc 2262499328 bytes == 0x558dc6a60000 @  0x7fa9b0d7a615 0x558b7ec833bc 0x558b7ed6418a 0x558b7ec861cd 0x558b7ec86120 0x558b7ecf9f33 0x558b7ec879da 0x558b7ecfa2c0 0x558b7ec879da 0x558b7ecf5eae 0x558b7ecf502f 0x558b7ec87aba 0x558b7ecf5eae 0x558b7ec879da 0x558b7ecf5eae 0x558b7ecf502f 0x558b7ec87aba 0x558b7ecf5eae 0x558b7ecf502f 0x558b7ec8836c 0x558b7ec88571 0x558b7ecf7633 0x558b7ecf502f 0x558b7ec87aba 0x558b7ecf5eae 0x558b7ec879da 0x558b7ecf5eae 0x558b7ecf502f 0x558b7ec87aba 0x558b7ecf5eae 0x558b7ecf502f\n",
            "tcmalloc: large alloc 2828124160 bytes == 0x558e50a14000 @  0x7fa9b0d7a615 0x558b7ec833bc 0x558b7ed6418a 0x558b7ec861cd 0x558b7ec86120 0x558b7ecf9f33 0x558b7ec879da 0x558b7ecfa2c0 0x558b7ec879da 0x558b7ecf5eae 0x558b7ecf502f 0x558b7ec87aba 0x558b7ecf5eae 0x558b7ec879da 0x558b7ecf5eae 0x558b7ecf502f 0x558b7ec87aba 0x558b7ecf5eae 0x558b7ecf502f 0x558b7ec8836c 0x558b7ec88571 0x558b7ecf7633 0x558b7ecf502f 0x558b7ec87aba 0x558b7ecf5eae 0x558b7ec879da 0x558b7ecf5eae 0x558b7ecf502f 0x558b7ec87aba 0x558b7ecf5eae 0x558b7ecf502f\n",
            "tcmalloc: large alloc 3535159296 bytes == 0x558d5ac3a000 @  0x7fa9b0d7a615 0x558b7ec833bc 0x558b7ed6418a 0x558b7ec861cd 0x558b7ec86120 0x558b7ecf9f33 0x558b7ec879da 0x558b7ecfa2c0 0x558b7ec879da 0x558b7ecf5eae 0x558b7ecf502f 0x558b7ec87aba 0x558b7ecf5eae 0x558b7ec879da 0x558b7ecf5eae 0x558b7ecf502f 0x558b7ec87aba 0x558b7ecf5eae 0x558b7ecf502f 0x558b7ec8836c 0x558b7ec88571 0x558b7ecf7633 0x558b7ecf502f 0x558b7ec87aba 0x558b7ecf5eae 0x558b7ec879da 0x558b7ecf5eae 0x558b7ecf502f 0x558b7ec87aba 0x558b7ecf5eae 0x558b7ecf502f\n",
            "tcmalloc: large alloc 4418953216 bytes == 0x558ef9c60000 @  0x7fa9b0d7a615 0x558b7ec833bc 0x558b7ed6418a 0x558b7ec861cd 0x558b7ec86120 0x558b7ecf9f33 0x558b7ec879da 0x558b7ecfa2c0 0x558b7ec879da 0x558b7ecf5eae 0x558b7ecf502f 0x558b7ec87aba 0x558b7ecf5eae 0x558b7ec879da 0x558b7ecf5eae 0x558b7ecf502f 0x558b7ec87aba 0x558b7ecf5eae 0x558b7ecf502f 0x558b7ec8836c 0x558b7ec88571 0x558b7ecf7633 0x558b7ecf502f 0x558b7ec87aba 0x558b7ecf5eae 0x558b7ec879da 0x558b7ecf5eae 0x558b7ecf502f 0x558b7ec87aba 0x558b7ecf5eae 0x558b7ecf502f\n",
            "10ex [00:02,  7.13ex/s]/usr/local/lib/python3.7/dist-packages/transformers/pipelines/base.py:978: UserWarning: You seem to be using the pipelines sequentially on GPU. In order to maximize efficiency please use a dataset\n",
            "  UserWarning,\n",
            "977ex [02:42,  6.00ex/s]\n",
            "WER: 0.3408664758709168\n",
            "CER: 0.07910422732746925\n",
            "977ex [00:00, 18683.00ex/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "torch.cuda.empty_cache()"
      ],
      "metadata": {
        "id": "2JKR8zrCeVYw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "2RYMBVoTdO6B"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}