02. NLP with Transformers/04. Named Entity Recognition/NER.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "NER.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "fbaef0b360f942369f74f2ef60e6cdfe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_6843bd89bf5e490598001bd848f15ca4",
              "IPY_MODEL_b44bf50082c74e86851cdf55e0c80f9e",
              "IPY_MODEL_a21d3ce3d62d4aed90d7d77c724bba4d"
            ],
            "layout": "IPY_MODEL_7001f92dd038402cad0da75e98018d6c"
          }
        },
        "6843bd89bf5e490598001bd848f15ca4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1f9dda0de3ff4bc48556e13aedff0346",
            "placeholder": "​",
            "style": "IPY_MODEL_d3b987a3c6a14bfc9c683097cf4fd65c",
            "value": "100%"
          }
        },
        "b44bf50082c74e86851cdf55e0c80f9e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3d54f66080794fa1b34b8afde4acf812",
            "max": 3,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_0dfe02b320ee4f8bafc59d99b8146c4c",
            "value": 3
          }
        },
        "a21d3ce3d62d4aed90d7d77c724bba4d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8f38bcb7aec54645b433c38f87089dfe",
            "placeholder": "​",
            "style": "IPY_MODEL_8699b76e92604925ba2f431e1d04b711",
            "value": " 3/3 [00:00&lt;00:00, 58.78it/s]"
          }
        },
        "7001f92dd038402cad0da75e98018d6c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1f9dda0de3ff4bc48556e13aedff0346": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d3b987a3c6a14bfc9c683097cf4fd65c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3d54f66080794fa1b34b8afde4acf812": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0dfe02b320ee4f8bafc59d99b8146c4c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "8f38bcb7aec54645b433c38f87089dfe": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8699b76e92604925ba2f431e1d04b711": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "1ee3f160a7de46be9ca95e7ff81038b8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_8ae16087ce2f4677b86c98ba911e5a6a",
              "IPY_MODEL_2f83c788617b40aca8b4b15e833da89c",
              "IPY_MODEL_708b7c68de8c4db2a09174a7b5c607a2"
            ],
            "layout": "IPY_MODEL_599dbf81d8314a23a630c93325e42a50"
          }
        },
        "8ae16087ce2f4677b86c98ba911e5a6a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b1765223f9384bea84c12b7bb015433e",
            "placeholder": "​",
            "style": "IPY_MODEL_6e87e072e3794af8bef0525048140bf7",
            "value": "100%"
          }
        },
        "2f83c788617b40aca8b4b15e833da89c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4271d30d3c1f459992c08b60e8b08184",
            "max": 3,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_abe643c74b90491d8e24130bbe44c118",
            "value": 3
          }
        },
        "708b7c68de8c4db2a09174a7b5c607a2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_59104a504a074d348b0dbf80142e1f03",
            "placeholder": "​",
            "style": "IPY_MODEL_764b846740f94f92956bcb3fd7f114a8",
            "value": " 3/3 [00:00&lt;00:00, 68.35it/s]"
          }
        },
        "599dbf81d8314a23a630c93325e42a50": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b1765223f9384bea84c12b7bb015433e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6e87e072e3794af8bef0525048140bf7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4271d30d3c1f459992c08b60e8b08184": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "abe643c74b90491d8e24130bbe44c118": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "59104a504a074d348b0dbf80142e1f03": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "764b846740f94f92956bcb3fd7f114a8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "04d9cc8c508d458a99972f3f6080d6ed": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_507c47bc95f7433886d9c9fd93490253",
              "IPY_MODEL_5a897f8eb9c24dbc8f90fa390925d67d",
              "IPY_MODEL_d9e7373d30634e368fd9529754d17ca2"
            ],
            "layout": "IPY_MODEL_a721584bc87943ca80e7b3468a407a78"
          }
        },
        "507c47bc95f7433886d9c9fd93490253": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_611df15835ae4346bd292f8d9b6c1b7f",
            "placeholder": "​",
            "style": "IPY_MODEL_4d76cbc90f8640179593200e9f49f706",
            "value": "100%"
          }
        },
        "5a897f8eb9c24dbc8f90fa390925d67d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3d55b5a86ef44bc892173741d3249e51",
            "max": 3,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_2e107311239e4f9b96a3a22714730aa2",
            "value": 3
          }
        },
        "d9e7373d30634e368fd9529754d17ca2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0026815771354848b0983e4cffd15069",
            "placeholder": "​",
            "style": "IPY_MODEL_0bac203513ec443798f333d7257afbfe",
            "value": " 3/3 [00:00&lt;00:00, 68.71it/s]"
          }
        },
        "a721584bc87943ca80e7b3468a407a78": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "611df15835ae4346bd292f8d9b6c1b7f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4d76cbc90f8640179593200e9f49f706": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3d55b5a86ef44bc892173741d3249e51": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2e107311239e4f9b96a3a22714730aa2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "0026815771354848b0983e4cffd15069": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0bac203513ec443798f333d7257afbfe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0e3e204701824f53bcd6a2ff4504c04c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_26c2e404c72f4bbfa60289adafa05364",
              "IPY_MODEL_52e4db9ac4dd49e0b02bb7261ea70ce4",
              "IPY_MODEL_ea8400f423f444708458f5561c591807"
            ],
            "layout": "IPY_MODEL_b546fbda658240048b03e8a7e5a875be"
          }
        },
        "26c2e404c72f4bbfa60289adafa05364": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9a96a6a218b54698bbd4ebc61eec3867",
            "placeholder": "​",
            "style": "IPY_MODEL_c8d0dd8a642844a38754a0c99907c2a5",
            "value": "100%"
          }
        },
        "52e4db9ac4dd49e0b02bb7261ea70ce4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_72221fea0e2542c6b2008c477d11e592",
            "max": 3,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_b5bd03b2e47049aa8f2b73235484096f",
            "value": 3
          }
        },
        "ea8400f423f444708458f5561c591807": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_be70a9586348471eae0fea0ea4a5ed4d",
            "placeholder": "​",
            "style": "IPY_MODEL_414cefe4cabc4e38a49b76129afca3b6",
            "value": " 3/3 [00:00&lt;00:00, 54.36it/s]"
          }
        },
        "b546fbda658240048b03e8a7e5a875be": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9a96a6a218b54698bbd4ebc61eec3867": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c8d0dd8a642844a38754a0c99907c2a5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "72221fea0e2542c6b2008c477d11e592": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b5bd03b2e47049aa8f2b73235484096f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "be70a9586348471eae0fea0ea4a5ed4d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "414cefe4cabc4e38a49b76129afca3b6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "a7e4d91888d8452880ca5d502e6fd1fc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_ddef8dd0821b4b4b8ef9d68e45eaa8d8",
              "IPY_MODEL_adec86fa94d04afc9421867641102675",
              "IPY_MODEL_ed531099e8cf48f091b6e8f1d7954783"
            ],
            "layout": "IPY_MODEL_66a7f3e118f44989a8b205f95b36b8af"
          }
        },
        "ddef8dd0821b4b4b8ef9d68e45eaa8d8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_16b50d1599aa457caba85f60feeb7fc1",
            "placeholder": "​",
            "style": "IPY_MODEL_48a16903bc9049eaa04dbd46994ef07c",
            "value": "100%"
          }
        },
        "adec86fa94d04afc9421867641102675": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c59cceebba1146b5b982c27c837cbe31",
            "max": 13,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_edc7d77a31784f9f9af6a9a956cf4138",
            "value": 13
          }
        },
        "ed531099e8cf48f091b6e8f1d7954783": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9c38643343a44a21b9fa568d42609fa4",
            "placeholder": "​",
            "style": "IPY_MODEL_b2173e5cf3494fb6ab6f1648d5476f51",
            "value": " 13/13 [00:01&lt;00:00,  6.08ba/s]"
          }
        },
        "66a7f3e118f44989a8b205f95b36b8af": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "16b50d1599aa457caba85f60feeb7fc1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "48a16903bc9049eaa04dbd46994ef07c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c59cceebba1146b5b982c27c837cbe31": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "edc7d77a31784f9f9af6a9a956cf4138": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "9c38643343a44a21b9fa568d42609fa4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b2173e5cf3494fb6ab6f1648d5476f51": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "611a343b9fd44e2fa9da79f3e4a3be67": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_fd21136b5e0c421f9607036ac3b9b0f0",
              "IPY_MODEL_c11943147f4842969b0b87efb1b5770f",
              "IPY_MODEL_df4cd72a9ca644cf84cd0803890c665c"
            ],
            "layout": "IPY_MODEL_b6c10fccce624877b0e06dba89e90337"
          }
        },
        "fd21136b5e0c421f9607036ac3b9b0f0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8fc9e7f0aada42c7ad2be77ba79a0eb4",
            "placeholder": "​",
            "style": "IPY_MODEL_f5f4ee7329d245a3a7ec22ed3500458c",
            "value": "100%"
          }
        },
        "c11943147f4842969b0b87efb1b5770f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d68879f3d7574047925b423e55ab52fc",
            "max": 197,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_4e60888b644440f990ffca1b6e8acdf4",
            "value": 197
          }
        },
        "df4cd72a9ca644cf84cd0803890c665c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c8ed9e4d161840a096a025c03658e10f",
            "placeholder": "​",
            "style": "IPY_MODEL_e19a6cc0dcd84a68b104ef1e3da01479",
            "value": " 197/197 [00:37&lt;00:00,  6.20ba/s]"
          }
        },
        "b6c10fccce624877b0e06dba89e90337": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8fc9e7f0aada42c7ad2be77ba79a0eb4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f5f4ee7329d245a3a7ec22ed3500458c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d68879f3d7574047925b423e55ab52fc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4e60888b644440f990ffca1b6e8acdf4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "c8ed9e4d161840a096a025c03658e10f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e19a6cc0dcd84a68b104ef1e3da01479": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3891226f1781476da6ec6bc990d54c17": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_83125c6c822e47fc8bf2e037ce1a6bd1",
              "IPY_MODEL_678f0ab0c20441ee8d2449b3f3004ddf",
              "IPY_MODEL_8f66c4dc1be5414d87168b2156d2bf9e"
            ],
            "layout": "IPY_MODEL_544f06a4f4774ace861b61247545c349"
          }
        },
        "83125c6c822e47fc8bf2e037ce1a6bd1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7303ba1309bd4116b017b98d714a0845",
            "placeholder": "​",
            "style": "IPY_MODEL_b71fbbe8b54c4ce39cbac3e45ef7d81a",
            "value": "100%"
          }
        },
        "678f0ab0c20441ee8d2449b3f3004ddf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_25bc78c2202d48b7baf52127f97bf9d6",
            "max": 5,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_0a3a9858ca4c45afb5517206ee283e47",
            "value": 5
          }
        },
        "8f66c4dc1be5414d87168b2156d2bf9e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5bf46a946ead46dc99d738f081bde36e",
            "placeholder": "​",
            "style": "IPY_MODEL_3ac94d58b5ed47e89607bbd25ef90d6c",
            "value": " 5/5 [00:00&lt;00:00,  8.41ba/s]"
          }
        },
        "544f06a4f4774ace861b61247545c349": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7303ba1309bd4116b017b98d714a0845": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b71fbbe8b54c4ce39cbac3e45ef7d81a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "25bc78c2202d48b7baf52127f97bf9d6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0a3a9858ca4c45afb5517206ee283e47": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "5bf46a946ead46dc99d738f081bde36e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3ac94d58b5ed47e89607bbd25ef90d6c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5949b3fb5ac4445d9012cac6224f91ea": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_c471b1e2dc4b4c1f81855ca2d67e216b",
              "IPY_MODEL_087e3480402647f1972df8f1aee83f38",
              "IPY_MODEL_1663d4ccf7264ccc961a176b407649ff"
            ],
            "layout": "IPY_MODEL_6799dc79b5354409b22f899eddc045f7"
          }
        },
        "c471b1e2dc4b4c1f81855ca2d67e216b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e9d70a51c6424c78be717bc32e88057c",
            "placeholder": "​",
            "style": "IPY_MODEL_a9e4122594ab419cb1b3d2ccaf6ec69a",
            "value": "100%"
          }
        },
        "087e3480402647f1972df8f1aee83f38": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2de82a252b624c5b91a9047b8e78a7db",
            "max": 5,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_d2daac2054e34a3d87205d7daaf6672e",
            "value": 5
          }
        },
        "1663d4ccf7264ccc961a176b407649ff": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_dc76b7a2cc4d4a918f146f62996cbc6f",
            "placeholder": "​",
            "style": "IPY_MODEL_5e62eba6d2264d7abf47b4a41fa43704",
            "value": " 5/5 [00:00&lt;00:00,  9.09ba/s]"
          }
        },
        "6799dc79b5354409b22f899eddc045f7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e9d70a51c6424c78be717bc32e88057c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a9e4122594ab419cb1b3d2ccaf6ec69a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2de82a252b624c5b91a9047b8e78a7db": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d2daac2054e34a3d87205d7daaf6672e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "dc76b7a2cc4d4a918f146f62996cbc6f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5e62eba6d2264d7abf47b4a41fa43704": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "85d33f318076422b8f676e1ac06ce1ee": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_c816f0a6c4144438b6e0885318360275",
              "IPY_MODEL_0ac49ff2840046fba7b66c6cdf7ac61b",
              "IPY_MODEL_b885e371c96b4ffd9f189ac2465f7c06"
            ],
            "layout": "IPY_MODEL_c2e6ca8a1c7f42ab8d83df1565b5d6f1"
          }
        },
        "c816f0a6c4144438b6e0885318360275": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_84728e74ad574125ba8052f041aa4cbc",
            "placeholder": "​",
            "style": "IPY_MODEL_cb763db02b674284bc5702fca35349d0",
            "value": "100%"
          }
        },
        "0ac49ff2840046fba7b66c6cdf7ac61b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b891750ba8fd4e8b94a221fe2deec11d",
            "max": 3,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_89c10a91fa0f40138b560695027a6b79",
            "value": 3
          }
        },
        "b885e371c96b4ffd9f189ac2465f7c06": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d7f5689004ff49abb0a6c33842b1870d",
            "placeholder": "​",
            "style": "IPY_MODEL_6a79a1f8e4cf4c7baf5dd74027490bd6",
            "value": " 3/3 [00:00&lt;00:00,  7.68ba/s]"
          }
        },
        "c2e6ca8a1c7f42ab8d83df1565b5d6f1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "84728e74ad574125ba8052f041aa4cbc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cb763db02b674284bc5702fca35349d0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b891750ba8fd4e8b94a221fe2deec11d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "89c10a91fa0f40138b560695027a6b79": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "d7f5689004ff49abb0a6c33842b1870d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6a79a1f8e4cf4c7baf5dd74027490bd6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0529d904558b4b93b98c0c0d4b18c601": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_27394b6475ab4c7aa33f48c8631745a5",
              "IPY_MODEL_60cc0804c1b645bd997b1e24b9293ba9",
              "IPY_MODEL_99f297f0962447bd8b4cab2e0fb3c18d"
            ],
            "layout": "IPY_MODEL_ce09f6871fd54c4186b2cb8dab5c5ee2"
          }
        },
        "27394b6475ab4c7aa33f48c8631745a5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3f55bde1a92941229fbc0d99660b76e8",
            "placeholder": "​",
            "style": "IPY_MODEL_a243f4e51b4140289b34dfed8a7d6556",
            "value": "100%"
          }
        },
        "60cc0804c1b645bd997b1e24b9293ba9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f71a736eb0254be3beeda7dce990263f",
            "max": 3,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_01156ca6a3cd49e0b033e79ff8db1bae",
            "value": 3
          }
        },
        "99f297f0962447bd8b4cab2e0fb3c18d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9e6df46864b5407fbf670ed5820efa0c",
            "placeholder": "​",
            "style": "IPY_MODEL_f06fbf70067b4ad1a52687bbd1917b3e",
            "value": " 3/3 [00:00&lt;00:00,  7.91ba/s]"
          }
        },
        "ce09f6871fd54c4186b2cb8dab5c5ee2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3f55bde1a92941229fbc0d99660b76e8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a243f4e51b4140289b34dfed8a7d6556": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f71a736eb0254be3beeda7dce990263f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "01156ca6a3cd49e0b033e79ff8db1bae": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "9e6df46864b5407fbf670ed5820efa0c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f06fbf70067b4ad1a52687bbd1917b3e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "bacd6980f0554fcdbf55096376c54f60": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_173dfd34e0df41cb8a73aa89bf832f74",
              "IPY_MODEL_223cf648aedb4add9759bca64fcf9a15",
              "IPY_MODEL_05d38d8a92554199b94c537871de2990"
            ],
            "layout": "IPY_MODEL_67bd7d599436450a8d1e5ea5c1ab0511"
          }
        },
        "173dfd34e0df41cb8a73aa89bf832f74": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_71cb6d4dcf7d487f867f1e1d032afe36",
            "placeholder": "​",
            "style": "IPY_MODEL_e34b8434b575495aa8e92c145d6d5026",
            "value": "100%"
          }
        },
        "223cf648aedb4add9759bca64fcf9a15": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a1243998babc45d489e7d7a4ed42873c",
            "max": 2,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_fa991fa36b3946ed85c66f0548a6ed0b",
            "value": 2
          }
        },
        "05d38d8a92554199b94c537871de2990": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_486d7f4b43e84ef493b6eabd0815a9ec",
            "placeholder": "​",
            "style": "IPY_MODEL_5339e0133f454bebbe7627fd57fe3ca7",
            "value": " 2/2 [00:00&lt;00:00,  5.03ba/s]"
          }
        },
        "67bd7d599436450a8d1e5ea5c1ab0511": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "71cb6d4dcf7d487f867f1e1d032afe36": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e34b8434b575495aa8e92c145d6d5026": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "a1243998babc45d489e7d7a4ed42873c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fa991fa36b3946ed85c66f0548a6ed0b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "486d7f4b43e84ef493b6eabd0815a9ec": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5339e0133f454bebbe7627fd57fe3ca7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4df85b6812734ba8b21f1d122262be1a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_a4367abef48b4bd5b252c48861507163",
              "IPY_MODEL_c02de02ef1fc47ebaa352560b35f0f69",
              "IPY_MODEL_15aac8c5f948419b8da011f7dad87b8f"
            ],
            "layout": "IPY_MODEL_a41215d00b0549188b0d609b239e4d1f"
          }
        },
        "a4367abef48b4bd5b252c48861507163": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4e6e507ab015430d8f3b36adcde5dc20",
            "placeholder": "​",
            "style": "IPY_MODEL_3356ca5f770c4cd3b094e037fbd2ca3e",
            "value": "100%"
          }
        },
        "c02de02ef1fc47ebaa352560b35f0f69": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4ca50970819844688371a43bc14cdcaf",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_6c6be58b918247ae832cf99a2aec5ba1",
            "value": 1
          }
        },
        "15aac8c5f948419b8da011f7dad87b8f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_467f890e427649cca53ce7695e22f899",
            "placeholder": "​",
            "style": "IPY_MODEL_29dad38e38c14649812748bcbe7ccdc3",
            "value": " 1/1 [00:00&lt;00:00,  6.70ba/s]"
          }
        },
        "a41215d00b0549188b0d609b239e4d1f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4e6e507ab015430d8f3b36adcde5dc20": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3356ca5f770c4cd3b094e037fbd2ca3e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4ca50970819844688371a43bc14cdcaf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6c6be58b918247ae832cf99a2aec5ba1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "467f890e427649cca53ce7695e22f899": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "29dad38e38c14649812748bcbe7ccdc3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "babc68b676e64d649cb404571b72f390": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_411555aed8ca4e088623b1e687eac724",
              "IPY_MODEL_755ca233d7e74de58740efb3696b2149",
              "IPY_MODEL_9c1a8dd2cc7c46378c11f70a032531a0"
            ],
            "layout": "IPY_MODEL_83f73a9ff9af41ac9b614dca5ace58e2"
          }
        },
        "411555aed8ca4e088623b1e687eac724": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c6fd425c567b49438e45444626608bcd",
            "placeholder": "​",
            "style": "IPY_MODEL_54afa3a7bf4c4dfb886d85db551df5fb",
            "value": "100%"
          }
        },
        "755ca233d7e74de58740efb3696b2149": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_77cb43ba110f418583a5458e32bcdeb3",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_b4f1a608a8ac41a585af222281a16524",
            "value": 1
          }
        },
        "9c1a8dd2cc7c46378c11f70a032531a0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_906b785cff5b4c5e864f219f3fe81343",
            "placeholder": "​",
            "style": "IPY_MODEL_d378324bda714932953e030f40bf161a",
            "value": " 1/1 [00:00&lt;00:00,  6.31ba/s]"
          }
        },
        "83f73a9ff9af41ac9b614dca5ace58e2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c6fd425c567b49438e45444626608bcd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "54afa3a7bf4c4dfb886d85db551df5fb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "77cb43ba110f418583a5458e32bcdeb3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b4f1a608a8ac41a585af222281a16524": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "906b785cff5b4c5e864f219f3fe81343": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d378324bda714932953e030f40bf161a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b4f89c85d6184a619fbe54735251a6ac": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_358eec8f4da44c46aee2295d16b83b1b",
              "IPY_MODEL_0b5154b8e9af4b5f8be5c9d521b6b4e9",
              "IPY_MODEL_ff002853d6a246fa8e6e0e3cdc9370cd"
            ],
            "layout": "IPY_MODEL_47bf5b9785f044a391fa1703062fd18c"
          }
        },
        "358eec8f4da44c46aee2295d16b83b1b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3eaf9acb098d40e8a130ca5acd3b2688",
            "placeholder": "​",
            "style": "IPY_MODEL_8190da58411d454b909075919ec6bf68",
            "value": "100%"
          }
        },
        "0b5154b8e9af4b5f8be5c9d521b6b4e9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_83ff353ec26d4013bcd5e43ff3afdfaa",
            "max": 2,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_482ace85fab248eba12015cb5aaca8eb",
            "value": 2
          }
        },
        "ff002853d6a246fa8e6e0e3cdc9370cd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_32e41f0f9a854078bb0eba978512a704",
            "placeholder": "​",
            "style": "IPY_MODEL_a0f3411b7e724adaa7b9c48aad3c4108",
            "value": " 2/2 [00:00&lt;00:00,  5.17ba/s]"
          }
        },
        "47bf5b9785f044a391fa1703062fd18c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3eaf9acb098d40e8a130ca5acd3b2688": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8190da58411d454b909075919ec6bf68": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "83ff353ec26d4013bcd5e43ff3afdfaa": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "482ace85fab248eba12015cb5aaca8eb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "32e41f0f9a854078bb0eba978512a704": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a0f3411b7e724adaa7b9c48aad3c4108": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7f4b40288db043c78a17841cc6ffb488": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_1849d226524d4b1491d8d63f046ac1e4",
              "IPY_MODEL_76b37c8bf3e74827a746190954d13319",
              "IPY_MODEL_0476685a236e47b69c7fceb0c2c9219c"
            ],
            "layout": "IPY_MODEL_1664069b384544a4bf539e594fd10164"
          }
        },
        "1849d226524d4b1491d8d63f046ac1e4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d37c136c0f404f9ba744a1db35e28c6f",
            "placeholder": "​",
            "style": "IPY_MODEL_e58b0cb5e90642aea2263dc663948506",
            "value": "100%"
          }
        },
        "76b37c8bf3e74827a746190954d13319": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6056d5ce405b40d3bc3bcc43f4b46060",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_1eae7217ef4b49d78b3f586a40384c99",
            "value": 1
          }
        },
        "0476685a236e47b69c7fceb0c2c9219c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c10de464e7b748bc83c273baef2fc814",
            "placeholder": "​",
            "style": "IPY_MODEL_9ae208732a3f48a4b0c793280889f6ba",
            "value": " 1/1 [00:00&lt;00:00,  7.63ba/s]"
          }
        },
        "1664069b384544a4bf539e594fd10164": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d37c136c0f404f9ba744a1db35e28c6f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e58b0cb5e90642aea2263dc663948506": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6056d5ce405b40d3bc3bcc43f4b46060": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1eae7217ef4b49d78b3f586a40384c99": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "c10de464e7b748bc83c273baef2fc814": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9ae208732a3f48a4b0c793280889f6ba": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "cf3b86828c2545f2924e9ccd89c0e47d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_bbcde84ff19f46e2a9b67793cf1e28e2",
              "IPY_MODEL_97656fe4cb1e49ca9f6c418844c8ffa6",
              "IPY_MODEL_96e93f87f8e6437493691a962aaf8dfd"
            ],
            "layout": "IPY_MODEL_7f47d80c90ec493cb403de7a321f31c4"
          }
        },
        "bbcde84ff19f46e2a9b67793cf1e28e2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7925fe24ad754d9ebfeea9e648667fc9",
            "placeholder": "​",
            "style": "IPY_MODEL_193bd50ecbd245e9a212a16276c463df",
            "value": "100%"
          }
        },
        "97656fe4cb1e49ca9f6c418844c8ffa6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_58f8d1ad991f426e84cde8c98246f0d4",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_2a66903b36da4b2595d74b70d29f36a1",
            "value": 1
          }
        },
        "96e93f87f8e6437493691a962aaf8dfd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1c56fa411cf84c99a3a15904e4826b9b",
            "placeholder": "​",
            "style": "IPY_MODEL_8e6526d7f79b47368c09dbc82d9294ef",
            "value": " 1/1 [00:00&lt;00:00,  6.91ba/s]"
          }
        },
        "7f47d80c90ec493cb403de7a321f31c4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7925fe24ad754d9ebfeea9e648667fc9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "193bd50ecbd245e9a212a16276c463df": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "58f8d1ad991f426e84cde8c98246f0d4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2a66903b36da4b2595d74b70d29f36a1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "1c56fa411cf84c99a3a15904e4826b9b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8e6526d7f79b47368c09dbc82d9294ef": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "**Named Entity Recognition**\n",
        "- NER is a common NLP task that identifies entities like people, organizations or locations in text. These entities can be used for various applications such as gaining insights from documents, augmenting the quality of search engines, or building a structured database from a corpus."
      ],
      "metadata": {
        "id": "laTHUb0QFFTC"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### **Initialization**\n",
        "- I use these three lines of code on top of my each notebooks because it will help to prevent any problems while reloading the same project. And the third line of code helps to make visualization within the notebook."
      ],
      "metadata": {
        "id": "MVCsl_joGpMU"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#@ INITIALIZATION: \n",
        "%reload_ext autoreload\n",
        "%autoreload 2\n",
        "%matplotlib inline"
      ],
      "metadata": {
        "id": "NmeascpYGuzS"
      },
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Downloading Libraries and Dependencies**\n",
        "- I have downloaded all the libraries and dependencies required for the project in one particular cell."
      ],
      "metadata": {
        "id": "61WCv6OqG4WC"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#@ IMPORTING MODULES: UNCOMMENT BELOW:\n",
        "# !pip install transformers[sentencepiece]\n",
        "# !pip install datasets\n",
        "# !pip install seqeval\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "from torch.nn.functional import cross_entropy\n",
        "from transformers import Trainer\n",
        "from transformers import TrainingArguments\n",
        "from transformers import AutoConfig\n",
        "from transformers import AutoTokenizer\n",
        "from transformers import AutoModel\n",
        "from transformers import XLMRobertaConfig\n",
        "from transformers.modeling_outputs import TokenClassifierOutput\n",
        "from transformers.models.roberta.modeling_roberta import RobertaModel\n",
        "from transformers.models.roberta.modeling_roberta import RobertaPreTrainedModel\n",
        "from transformers import DataCollatorForTokenClassification\n",
        "from datasets import load_dataset\n",
        "from datasets import get_dataset_config_names\n",
        "from datasets import DatasetDict\n",
        "from datasets import concatenate_datasets\n",
        "from collections import defaultdict\n",
        "from collections import Counter\n",
        "from seqeval.metrics import classification_report\n",
        "from seqeval.metrics import f1_score\n",
        "from huggingface_hub import notebook_login\n",
        "from sklearn.metrics import ConfusionMatrixDisplay, confusion_matrix\n",
        "\n",
        "#@ IGNORING WARNINGS: \n",
        "import warnings\n",
        "warnings.filterwarnings(\"ignore\")"
      ],
      "metadata": {
        "id": "kgBudKpYG_RS"
      },
      "execution_count": 38,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### **The Dataset**\n",
        "- The dataset consists of Wikipedia articles in many languages. Each article is annotated with LOC(location), PER(person) and ORG(organization) tags in IOB2 format. "
      ],
      "metadata": {
        "id": "qTHKStWWHdQr"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-Po6IrfJE3K7",
        "outputId": "5baebc95-90ca-4cae-d4f6-5dbb5d6a883e"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "XTREME has 183 configurations\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['PAN-X.af', 'PAN-X.ar', 'PAN-X.bg', 'PAN-X.bn', 'PAN-X.de']"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ],
      "source": [
        "#@ LOADING PAN-X DATASET:\n",
        "xtreme_subsets = get_dataset_config_names(\"xtreme\")                     # Xtreme benchmarked. \n",
        "print(f\"XTREME has {len(xtreme_subsets)} configurations\")               # Inspecting number of configurations.\n",
        "panx_subsets = [s for s in xtreme_subsets if s.startswith(\"PAN\")]       # Initializing PAN-X dataset.\n",
        "panx_subsets[:5]                                                        # Inspection."
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@ LOADING PAN-X DATASET:\n",
        "langs = [\"de\", \"fr\", \"it\", \"en\"]                                                # Initialization.\n",
        "fracs = [0.629, 0.229, 0.084, 0.059]                                            # Initialization.\n",
        "panx_ch = defaultdict(DatasetDict)                                              # Initializing dictionary. \n",
        "for lang, frac in zip(langs, fracs):\n",
        "    ds = load_dataset(\"xtreme\", name=f\"PAN-X.{lang}\")                           # Loading monolingual corpus.\n",
        "    for split in ds:\n",
        "        panx_ch[lang][split] = (\n",
        "            ds[split]\n",
        "            .shuffle(seed=2022)                                                 # Shuffling. \n",
        "            .select(range(int(frac * ds[split].num_rows))))                     # Downsampling dataset. "
      ],
      "metadata": {
        "id": "8uqysJDUJ4QU",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 442,
          "referenced_widgets": [
            "fbaef0b360f942369f74f2ef60e6cdfe",
            "6843bd89bf5e490598001bd848f15ca4",
            "b44bf50082c74e86851cdf55e0c80f9e",
            "a21d3ce3d62d4aed90d7d77c724bba4d",
            "7001f92dd038402cad0da75e98018d6c",
            "1f9dda0de3ff4bc48556e13aedff0346",
            "d3b987a3c6a14bfc9c683097cf4fd65c",
            "3d54f66080794fa1b34b8afde4acf812",
            "0dfe02b320ee4f8bafc59d99b8146c4c",
            "8f38bcb7aec54645b433c38f87089dfe",
            "8699b76e92604925ba2f431e1d04b711",
            "1ee3f160a7de46be9ca95e7ff81038b8",
            "8ae16087ce2f4677b86c98ba911e5a6a",
            "2f83c788617b40aca8b4b15e833da89c",
            "708b7c68de8c4db2a09174a7b5c607a2",
            "599dbf81d8314a23a630c93325e42a50",
            "b1765223f9384bea84c12b7bb015433e",
            "6e87e072e3794af8bef0525048140bf7",
            "4271d30d3c1f459992c08b60e8b08184",
            "abe643c74b90491d8e24130bbe44c118",
            "59104a504a074d348b0dbf80142e1f03",
            "764b846740f94f92956bcb3fd7f114a8",
            "04d9cc8c508d458a99972f3f6080d6ed",
            "507c47bc95f7433886d9c9fd93490253",
            "5a897f8eb9c24dbc8f90fa390925d67d",
            "d9e7373d30634e368fd9529754d17ca2",
            "a721584bc87943ca80e7b3468a407a78",
            "611df15835ae4346bd292f8d9b6c1b7f",
            "4d76cbc90f8640179593200e9f49f706",
            "3d55b5a86ef44bc892173741d3249e51",
            "2e107311239e4f9b96a3a22714730aa2",
            "0026815771354848b0983e4cffd15069",
            "0bac203513ec443798f333d7257afbfe",
            "0e3e204701824f53bcd6a2ff4504c04c",
            "26c2e404c72f4bbfa60289adafa05364",
            "52e4db9ac4dd49e0b02bb7261ea70ce4",
            "ea8400f423f444708458f5561c591807",
            "b546fbda658240048b03e8a7e5a875be",
            "9a96a6a218b54698bbd4ebc61eec3867",
            "c8d0dd8a642844a38754a0c99907c2a5",
            "72221fea0e2542c6b2008c477d11e592",
            "b5bd03b2e47049aa8f2b73235484096f",
            "be70a9586348471eae0fea0ea4a5ed4d",
            "414cefe4cabc4e38a49b76129afca3b6"
          ]
        },
        "outputId": "426f40c3-4997-4e55-ef28-41153ae6cd22"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Reusing dataset xtreme (/root/.cache/huggingface/datasets/xtreme/PAN-X.de/1.0.0/349258adc25bb45e47de193222f95e68a44f7a7ab53c4283b3f007208a11bf7e)\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/3 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "fbaef0b360f942369f74f2ef60e6cdfe"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Loading cached shuffled indices for dataset at /root/.cache/huggingface/datasets/xtreme/PAN-X.de/1.0.0/349258adc25bb45e47de193222f95e68a44f7a7ab53c4283b3f007208a11bf7e/cache-dc382e99bd2b7098.arrow\n",
            "Loading cached shuffled indices for dataset at /root/.cache/huggingface/datasets/xtreme/PAN-X.de/1.0.0/349258adc25bb45e47de193222f95e68a44f7a7ab53c4283b3f007208a11bf7e/cache-c1148974c7445208.arrow\n",
            "Loading cached shuffled indices for dataset at /root/.cache/huggingface/datasets/xtreme/PAN-X.de/1.0.0/349258adc25bb45e47de193222f95e68a44f7a7ab53c4283b3f007208a11bf7e/cache-a6c757bb698e99d8.arrow\n",
            "Reusing dataset xtreme (/root/.cache/huggingface/datasets/xtreme/PAN-X.fr/1.0.0/349258adc25bb45e47de193222f95e68a44f7a7ab53c4283b3f007208a11bf7e)\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/3 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "1ee3f160a7de46be9ca95e7ff81038b8"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Loading cached shuffled indices for dataset at /root/.cache/huggingface/datasets/xtreme/PAN-X.fr/1.0.0/349258adc25bb45e47de193222f95e68a44f7a7ab53c4283b3f007208a11bf7e/cache-48607c59ace84d0b.arrow\n",
            "Loading cached shuffled indices for dataset at /root/.cache/huggingface/datasets/xtreme/PAN-X.fr/1.0.0/349258adc25bb45e47de193222f95e68a44f7a7ab53c4283b3f007208a11bf7e/cache-ad924b582fa43377.arrow\n",
            "Loading cached shuffled indices for dataset at /root/.cache/huggingface/datasets/xtreme/PAN-X.fr/1.0.0/349258adc25bb45e47de193222f95e68a44f7a7ab53c4283b3f007208a11bf7e/cache-a9387d1f61db954b.arrow\n",
            "Reusing dataset xtreme (/root/.cache/huggingface/datasets/xtreme/PAN-X.it/1.0.0/349258adc25bb45e47de193222f95e68a44f7a7ab53c4283b3f007208a11bf7e)\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/3 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "04d9cc8c508d458a99972f3f6080d6ed"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Loading cached shuffled indices for dataset at /root/.cache/huggingface/datasets/xtreme/PAN-X.it/1.0.0/349258adc25bb45e47de193222f95e68a44f7a7ab53c4283b3f007208a11bf7e/cache-1dd3d856388549da.arrow\n",
            "Loading cached shuffled indices for dataset at /root/.cache/huggingface/datasets/xtreme/PAN-X.it/1.0.0/349258adc25bb45e47de193222f95e68a44f7a7ab53c4283b3f007208a11bf7e/cache-d881eaa00d634cb2.arrow\n",
            "Loading cached shuffled indices for dataset at /root/.cache/huggingface/datasets/xtreme/PAN-X.it/1.0.0/349258adc25bb45e47de193222f95e68a44f7a7ab53c4283b3f007208a11bf7e/cache-2ebd40825d65723e.arrow\n",
            "Reusing dataset xtreme (/root/.cache/huggingface/datasets/xtreme/PAN-X.en/1.0.0/349258adc25bb45e47de193222f95e68a44f7a7ab53c4283b3f007208a11bf7e)\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/3 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "0e3e204701824f53bcd6a2ff4504c04c"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Loading cached shuffled indices for dataset at /root/.cache/huggingface/datasets/xtreme/PAN-X.en/1.0.0/349258adc25bb45e47de193222f95e68a44f7a7ab53c4283b3f007208a11bf7e/cache-bf644b9b954e614a.arrow\n",
            "Loading cached shuffled indices for dataset at /root/.cache/huggingface/datasets/xtreme/PAN-X.en/1.0.0/349258adc25bb45e47de193222f95e68a44f7a7ab53c4283b3f007208a11bf7e/cache-bdb2a5e0dfa211d1.arrow\n",
            "Loading cached shuffled indices for dataset at /root/.cache/huggingface/datasets/xtreme/PAN-X.en/1.0.0/349258adc25bb45e47de193222f95e68a44f7a7ab53c4283b3f007208a11bf7e/cache-b89fabe1a51f963b.arrow\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@ LOADING PAN-X DATASET:\n",
        "pd.DataFrame({lang:[panx_ch[lang][\"train\"].num_rows] for lang in langs},\n",
        "             index=[\"Number of training examples\"])                             # Creating dataframe."
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 81
        },
        "id": "VsY6uTTtql0D",
        "outputId": "f5b949c8-a647-4db6-b356-7ad96ebb5486"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                de    fr    it    en\n",
              "Number of training examples  12580  4580  1680  1180"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-0e56732b-3476-41a3-8158-88e659b46a55\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>de</th>\n",
              "      <th>fr</th>\n",
              "      <th>it</th>\n",
              "      <th>en</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>Number of training examples</th>\n",
              "      <td>12580</td>\n",
              "      <td>4580</td>\n",
              "      <td>1680</td>\n",
              "      <td>1180</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-0e56732b-3476-41a3-8158-88e659b46a55')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-0e56732b-3476-41a3-8158-88e659b46a55 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-0e56732b-3476-41a3-8158-88e659b46a55');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@ INSPECTING GERMAN CORPUS:\n",
        "element = panx_ch[\"de\"][\"train\"][0]                         # Initializing german corpus.\n",
        "for key, value in element.items():\n",
        "    print(f\"{key}: {value}\")                                # Inspection."
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ISI2tB7hiEnh",
        "outputId": "6d1e7ca4-906a-444c-81f2-dfddc145ff5e"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tokens: ['Brighton', '&', 'Hove', 'Albion', '-', 'Scunthorpe', 'United', '3:2']\n",
            "ner_tags: [3, 4, 4, 4, 0, 3, 4, 0]\n",
            "langs: ['de', 'de', 'de', 'de', 'de', 'de', 'de', 'de']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@ INSPECTING GERMAN CORPUS:\n",
        "element = panx_ch[\"de\"][\"train\"].features                   # Initializing german corpus.\n",
        "for key, value in element.items():\n",
        "    print(f\"{key}: {value}\")                                # Inspection.\n",
        "tags = panx_ch[\"de\"][\"train\"].features[\"ner_tags\"].feature  # Initializing ner tags.\n",
        "print(tags)                                                 # Inspection."
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Yy2NH-C_j0Xy",
        "outputId": "6cb64b1f-04f3-4b4f-db67-11bb4a9908c7"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tokens: Sequence(feature=Value(dtype='string', id=None), length=-1, id=None)\n",
            "ner_tags: Sequence(feature=ClassLabel(num_classes=7, names=['O', 'B-PER', 'I-PER', 'B-ORG', 'I-ORG', 'B-LOC', 'I-LOC'], id=None), length=-1, id=None)\n",
            "langs: Sequence(feature=Value(dtype='string', id=None), length=-1, id=None)\n",
            "ClassLabel(num_classes=7, names=['O', 'B-PER', 'I-PER', 'B-ORG', 'I-ORG', 'B-LOC', 'I-LOC'], id=None)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@ INITIALIZING TAG NAMES: \n",
        "def create_tag_names(batch):                                                    # Defining class. \n",
        "    return {\"ner_tag_str\": [tags.int2str(idx) for idx in batch[\"ner_tags\"]]}    # Converting integer to strings.\n",
        "panx_de = panx_ch[\"de\"].map(create_tag_names)                                   # Implementation of function."
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xwkNhBOblk_z",
        "outputId": "cfc7f834-d003-419d-f1bc-5c0bd3822ab3"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Loading cached processed dataset at /root/.cache/huggingface/datasets/xtreme/PAN-X.de/1.0.0/349258adc25bb45e47de193222f95e68a44f7a7ab53c4283b3f007208a11bf7e/cache-10c39e47e2cb8378.arrow\n",
            "Loading cached processed dataset at /root/.cache/huggingface/datasets/xtreme/PAN-X.de/1.0.0/349258adc25bb45e47de193222f95e68a44f7a7ab53c4283b3f007208a11bf7e/cache-4c27d77cfabd4001.arrow\n",
            "Loading cached processed dataset at /root/.cache/huggingface/datasets/xtreme/PAN-X.de/1.0.0/349258adc25bb45e47de193222f95e68a44f7a7ab53c4283b3f007208a11bf7e/cache-6010854ae6d92763.arrow\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@ INSPECTING TAG NAMES:\n",
        "de_example = panx_de[\"train\"][0]                                                # Initialization.\n",
        "pd.DataFrame([de_example[\"tokens\"], de_example[\"ner_tag_str\"]], \n",
        "             [\"Tokens\", \"Tags\"])                                                # Creating a dataframe."
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 112
        },
        "id": "lUC9860qpCuO",
        "outputId": "8536c923-e9d1-42e1-c964-0e6348880f5a"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "               0      1      2       3  4           5       6    7\n",
              "Tokens  Brighton      &   Hove  Albion  -  Scunthorpe  United  3:2\n",
              "Tags       B-ORG  I-ORG  I-ORG   I-ORG  O       B-ORG   I-ORG    O"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-834f34b0-4364-4037-aad7-62e5955e3f89\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>Tokens</th>\n",
              "      <td>Brighton</td>\n",
              "      <td>&amp;</td>\n",
              "      <td>Hove</td>\n",
              "      <td>Albion</td>\n",
              "      <td>-</td>\n",
              "      <td>Scunthorpe</td>\n",
              "      <td>United</td>\n",
              "      <td>3:2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Tags</th>\n",
              "      <td>B-ORG</td>\n",
              "      <td>I-ORG</td>\n",
              "      <td>I-ORG</td>\n",
              "      <td>I-ORG</td>\n",
              "      <td>O</td>\n",
              "      <td>B-ORG</td>\n",
              "      <td>I-ORG</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-834f34b0-4364-4037-aad7-62e5955e3f89')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-834f34b0-4364-4037-aad7-62e5955e3f89 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-834f34b0-4364-4037-aad7-62e5955e3f89');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@ CALCULATING FREQUENCIES OF ENTITIES: GENERAL:\n",
        "split2freqs = defaultdict(Counter)                              # Initialization.\n",
        "for split, dataset in panx_de.items():\n",
        "    for row in dataset[\"ner_tag_str\"]:\n",
        "        for tag in row:\n",
        "            if tag.startswith(\"B\"):\n",
        "                tag_type = tag.split(\"-\")[1]\n",
        "                split2freqs[split][tag_type] += 1\n",
        "pd.DataFrame.from_dict(split2freqs, orient=\"index\")             # Creating dataframe. "
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 143
        },
        "id": "jqHR1Tb4qtKx",
        "outputId": "05c1c478-1909-4a79-8798-747543a12786"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "             ORG   LOC   PER\n",
              "train       5465  6130  5812\n",
              "validation  2742  3070  2801\n",
              "test        2624  3070  3017"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-4672dde3-a55c-4d3a-a30d-0e6eea9059e8\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>ORG</th>\n",
              "      <th>LOC</th>\n",
              "      <th>PER</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>train</th>\n",
              "      <td>5465</td>\n",
              "      <td>6130</td>\n",
              "      <td>5812</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>validation</th>\n",
              "      <td>2742</td>\n",
              "      <td>3070</td>\n",
              "      <td>2801</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>test</th>\n",
              "      <td>2624</td>\n",
              "      <td>3070</td>\n",
              "      <td>3017</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-4672dde3-a55c-4d3a-a30d-0e6eea9059e8')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-4672dde3-a55c-4d3a-a30d-0e6eea9059e8 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-4672dde3-a55c-4d3a-a30d-0e6eea9059e8');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### **Multilingual Transformers**\n",
        "- Multilingual transformers involve similar architectures and training procedures as their multilingual counterparts, except that the corpus used for pretraining consists of documents in many languages. "
      ],
      "metadata": {
        "id": "LbytnjJkIR_l"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Tokenization**\n",
        "- XLM-R uses a tokenizer called SentencePiece that is trained on the raw text of all one hundred languages."
      ],
      "metadata": {
        "id": "8H-4BH_FJX06"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#@ INITIALIZING TOKENIZATION:\n",
        "bert_model_name = \"bert-base-cased\"                                 # Initializing bert checkpoint.\n",
        "xlmr_model_name = \"xlm-roberta-base\"                                # Initializing xlmr checkpoint.\n",
        "bert_tokenizer = AutoTokenizer.from_pretrained(bert_model_name)     # Initializing pretrained tokenizer.\n",
        "xlmr_tokenizer = AutoTokenizer.from_pretrained(xlmr_model_name)     # Initializing pretrained tokenizer. \n",
        "text = \"Jack Sparrow loves New York!\"                               # Initializing text example.\n",
        "bert_tokens = bert_tokenizer(text).tokens()                         # Initializing bert tokens.\n",
        "xlmr_tokens = xlmr_tokenizer(text).tokens()                         # Initializing xlmr tokens."
      ],
      "metadata": {
        "id": "oj1Whb5gs83v"
      },
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**The Tokenizer Pipeline**\n",
        "  - Normalization.\n",
        "  - Pretokenization. \n",
        "  - Tokenizer Model. \n",
        "  - Postprocessing."
      ],
      "metadata": {
        "id": "sMvoHUTiLyc1"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "**The SentencePiece Tokenizer**\n",
        "- The SentencePiece tokenizer is based on a type of subword segmentation called Unigram and encodes each input text as a sequence of Unicode characters."
      ],
      "metadata": {
        "id": "whzOPiEKOf0n"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#@ IMPLEMENTATION OF SENTENCEPIECE TOKENIZER:\n",
        "\"\".join(xlmr_tokens).replace(\"\\u2581\", \" \")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "GVNL3rkBLNH0",
        "outputId": "70768648-fa66-4c10-d555-3cf3095977b1"
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'<s> Jack Sparrow loves New York!</s>'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Transformers for Named Entity Recognition**"
      ],
      "metadata": {
        "id": "pljLS0pSQvCM"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#@ CUSTOM MODEL FOR TOKEN CLASSIFICATION:\n",
        "class XLMRobertaForTokenClassification(RobertaPreTrainedModel):                         # Defining class. \n",
        "    config_class = XLMRobertaConfig\n",
        "    def __init__(self, config):                                                         # Initializing constructor function.\n",
        "        super().__init__(config)\n",
        "        self.num_labels = config.num_labels                                             # Initialization.\n",
        "        self.roberta = RobertaModel(config, add_pooling_layer=False)                    # Initializing roberta model.\n",
        "        self.dropout = nn.Dropout(config.hidden_dropout_prob)                           # Initializing dropout layer.\n",
        "        self.classifier = nn.Linear(config.hidden_size, config.num_labels)              # Initializing linear layer.\n",
        "        self.init_weights()                                                             # Loading and initializing weights.\n",
        "    \n",
        "    def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, \n",
        "                labels=None, **kwargs):\n",
        "        outputs = self.roberta(input_ids, attention_mask=attention_mask, \n",
        "                               token_type_ids=token_type_ids, **kwargs)                 # Getting encoder representations. \n",
        "        sequence_output = self.dropout(outputs[0])                                      # Applying dropout.\n",
        "        logits = self.classifier(sequence_output)                                       # Applying classifier.\n",
        "        loss = None\n",
        "        if labels is not None:\n",
        "            loss_fct = nn.CrossEntropyLoss()                                            # Initializing cross entropy loss function.\n",
        "            loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))          # Calculating loss.\n",
        "        return TokenClassifierOutput(loss=loss, logits=logits, \n",
        "                                     hidden_states=outputs.hidden_states,\n",
        "                                     attentions=outputs.attentions)                     # Getting model output."
      ],
      "metadata": {
        "id": "my6K0Mq6P-82"
      },
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@ LOADING CUSTOM MODEL:\n",
        "index2tag = {idx: tag for idx, tag in enumerate(tags.names)}                            # Index to tags.\n",
        "tag2index = {tag: idx for idx, tag in enumerate(tags.names)}                            # Tags to index.\n",
        "xlmr_config = AutoConfig.from_pretrained(xlmr_model_name, \n",
        "                                         num_labels=tags.num_classes, \n",
        "                                         id2label=index2tag, label2id=tag2index)        # Initializing configurations. "
      ],
      "metadata": {
        "id": "dLFs8VTFoMmW"
      },
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@ LOADING MODEL WEIGHTS: \n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")       # Initializing gpu. \n",
        "xlmr_model = (XLMRobertaForTokenClassification.from_pretrained(\n",
        "    xlmr_model_name, config=xlmr_config).to(device))                        # Initializing pretrained model."
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "z9EpCsVPqcCZ",
        "outputId": "371fb5e6-c703-4d85-b4f5-bb18cf13b13b"
      },
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of the model checkpoint at xlm-roberta-base were not used when initializing XLMRobertaForTokenClassification: ['lm_head.layer_norm.weight', 'lm_head.layer_norm.bias', 'lm_head.dense.bias', 'lm_head.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'lm_head.bias']\n",
            "- This IS expected if you are initializing XLMRobertaForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing XLMRobertaForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of XLMRobertaForTokenClassification were not initialized from the model checkpoint at xlm-roberta-base and are newly initialized: ['roberta.embeddings.position_ids', 'classifier.weight', 'classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@ IMPLEMENTATION OF MODEL:\n",
        "input_ids = xlmr_tokenizer.encode(text, return_tensors=\"pt\")                # Initializing ids of tokens.\n",
        "pd.DataFrame([xlmr_tokens, input_ids[0].numpy()], \n",
        "             index=[\"Tokens\", \"Inputs\"])                                    # Creating a dataframe."
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 112
        },
        "id": "1o5FGUk5rZ8u",
        "outputId": "b0227e98-09bc-4d95-d7cc-6090ec299ac3"
      },
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "          0      1      2      3      4  5     6      7   8     9\n",
              "Tokens  <s>  ▁Jack  ▁Spar    row  ▁love  s  ▁New  ▁York   !  </s>\n",
              "Inputs    0  21763  37456  15555   5161  7  2356   5753  38     2"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-fefcc577-471b-4ed5-85a6-c67b1c0e47c2\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>Tokens</th>\n",
              "      <td>&lt;s&gt;</td>\n",
              "      <td>▁Jack</td>\n",
              "      <td>▁Spar</td>\n",
              "      <td>row</td>\n",
              "      <td>▁love</td>\n",
              "      <td>s</td>\n",
              "      <td>▁New</td>\n",
              "      <td>▁York</td>\n",
              "      <td>!</td>\n",
              "      <td>&lt;/s&gt;</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Inputs</th>\n",
              "      <td>0</td>\n",
              "      <td>21763</td>\n",
              "      <td>37456</td>\n",
              "      <td>15555</td>\n",
              "      <td>5161</td>\n",
              "      <td>7</td>\n",
              "      <td>2356</td>\n",
              "      <td>5753</td>\n",
              "      <td>38</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-fefcc577-471b-4ed5-85a6-c67b1c0e47c2')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-fefcc577-471b-4ed5-85a6-c67b1c0e47c2 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-fefcc577-471b-4ed5-85a6-c67b1c0e47c2');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@ INITIALIZING PREDICTIONS:\n",
        "outputs = xlmr_model(input_ids.to(device)).logits                           # Implementation of mode.\n",
        "predictions = torch.argmax(outputs, dim=-1)                                 # Generating predictions.\n",
        "print(f\"Number of tokens in sequence: {len(xlmr_tokens)}\")                  # Inspection.\n",
        "print(f\"Shape of outputs: {outputs.shape}\")                                 # Inspection.\n",
        "preds = [tags.names[p] for p in predictions[0].cpu().numpy()]               # Initializing predictions.\n",
        "pd.DataFrame([xlmr_tokens, preds], index=[\"Tokens\", \"Tags\"])                # Inspection."
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 146
        },
        "id": "kPjVa_XusHhC",
        "outputId": "1ac113f1-8dcb-452d-dac9-d4db80a2562a"
      },
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Number of tokens in sequence: 10\n",
            "Shape of outputs: torch.Size([1, 10, 7])\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "            0      1      2      3      4      5      6      7      8      9\n",
              "Tokens    <s>  ▁Jack  ▁Spar    row  ▁love      s   ▁New  ▁York      !   </s>\n",
              "Tags    I-LOC  I-LOC  I-LOC  I-LOC  I-LOC  I-LOC  I-LOC  I-LOC  I-LOC  I-LOC"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-ed62920a-3318-4019-83b1-1db6da78db21\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>Tokens</th>\n",
              "      <td>&lt;s&gt;</td>\n",
              "      <td>▁Jack</td>\n",
              "      <td>▁Spar</td>\n",
              "      <td>row</td>\n",
              "      <td>▁love</td>\n",
              "      <td>s</td>\n",
              "      <td>▁New</td>\n",
              "      <td>▁York</td>\n",
              "      <td>!</td>\n",
              "      <td>&lt;/s&gt;</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Tags</th>\n",
              "      <td>I-LOC</td>\n",
              "      <td>I-LOC</td>\n",
              "      <td>I-LOC</td>\n",
              "      <td>I-LOC</td>\n",
              "      <td>I-LOC</td>\n",
              "      <td>I-LOC</td>\n",
              "      <td>I-LOC</td>\n",
              "      <td>I-LOC</td>\n",
              "      <td>I-LOC</td>\n",
              "      <td>I-LOC</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-ed62920a-3318-4019-83b1-1db6da78db21')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-ed62920a-3318-4019-83b1-1db6da78db21 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-ed62920a-3318-4019-83b1-1db6da78db21');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@ INITIALIZING TEXT TAGGER: IMPORTANT: \n",
        "def tag_text(text, tags, model, tokenizer):                                 # Defining function. \n",
        "    tokens = tokenizer(text).tokens()                                       # Initializing tokenization.\n",
        "    input_ids = xlmr_tokenizer(text, \n",
        "                               return_tensors=\"pt\").input_ids.to(device)    # Encoding sequence into ids.\n",
        "    outputs = model(input_ids)[0]                                           # Generating predictions.\n",
        "    predictions = torch.argmax(outputs, dim=2)\n",
        "    preds = [tags.names[p] for p in predictions[0].cpu().numpy()]           # Initializing predictions.\n",
        "    return pd.DataFrame([tokens, preds], index=[\"Tokens\", \"Tags\"])          # Inspecting dataframe."
      ],
      "metadata": {
        "id": "orKUl3aNupt7"
      },
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### **Tokenizing Texts for NER**"
      ],
      "metadata": {
        "id": "dyTfPeORzucK"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#@ TOKENIZING TEXTS FOR NER: EXAMPLE:\n",
        "words, labels = de_example[\"tokens\"], de_example[\"ner_tags\"]                # Initialization.\n",
        "tokenized_input = xlmr_tokenizer(de_example[\"tokens\"], \n",
        "                                 is_split_into_words=True)                  # Tokenized inputs.\n",
        "tokens = xlmr_tokenizer.convert_ids_to_tokens(tokenized_input[\"input_ids\"]) # Initializing tokens.\n",
        "pd.DataFrame([tokens], index=[\"Tokens\"])                                    # Inspection."
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 81
        },
        "id": "bsKZ4Ko5xmkC",
        "outputId": "c4b0fe06-d1d5-4124-ba6f-82c9c8707af5"
      },
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "         0        1   2   3    4   5      6   7   8     9   10   11  12  \\\n",
              "Tokens  <s>  ▁Bright  on  ▁&  ▁Ho  ve  ▁Albi  on  ▁-  ▁Scu  nt  hor  pe   \n",
              "\n",
              "             13   14 15    16  \n",
              "Tokens  ▁United  ▁3:  2  </s>  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-4d881b38-fcc7-4cb8-914e-b4d9da17d6d7\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "      <th>10</th>\n",
              "      <th>11</th>\n",
              "      <th>12</th>\n",
              "      <th>13</th>\n",
              "      <th>14</th>\n",
              "      <th>15</th>\n",
              "      <th>16</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>Tokens</th>\n",
              "      <td>&lt;s&gt;</td>\n",
              "      <td>▁Bright</td>\n",
              "      <td>on</td>\n",
              "      <td>▁&amp;</td>\n",
              "      <td>▁Ho</td>\n",
              "      <td>ve</td>\n",
              "      <td>▁Albi</td>\n",
              "      <td>on</td>\n",
              "      <td>▁-</td>\n",
              "      <td>▁Scu</td>\n",
              "      <td>nt</td>\n",
              "      <td>hor</td>\n",
              "      <td>pe</td>\n",
              "      <td>▁United</td>\n",
              "      <td>▁3:</td>\n",
              "      <td>2</td>\n",
              "      <td>&lt;/s&gt;</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-4d881b38-fcc7-4cb8-914e-b4d9da17d6d7')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-4d881b38-fcc7-4cb8-914e-b4d9da17d6d7 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-4d881b38-fcc7-4cb8-914e-b4d9da17d6d7');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@ TOKENIZING TEXTS FOR NER:\n",
        "word_ids = tokenized_input.word_ids()                                       # Initializing word ids.\n",
        "pd.DataFrame([tokens, word_ids], index=[\"Tokens\", \"Word IDs\"])              # Inspection. "
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 112
        },
        "id": "C5XycREf0QGB",
        "outputId": "15d11dd5-def2-49d6-c40a-96b15d1f5660"
      },
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "            0        1   2   3    4   5      6   7   8     9   10   11  12  \\\n",
              "Tokens     <s>  ▁Bright  on  ▁&  ▁Ho  ve  ▁Albi  on  ▁-  ▁Scu  nt  hor  pe   \n",
              "Word IDs  None        0   0   1    2   2      3   3   4     5   5    5   5   \n",
              "\n",
              "               13   14 15    16  \n",
              "Tokens    ▁United  ▁3:  2  </s>  \n",
              "Word IDs        6    7  7  None  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-432204a3-b159-496e-8f4f-0a9d1be07d6b\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "      <th>10</th>\n",
              "      <th>11</th>\n",
              "      <th>12</th>\n",
              "      <th>13</th>\n",
              "      <th>14</th>\n",
              "      <th>15</th>\n",
              "      <th>16</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>Tokens</th>\n",
              "      <td>&lt;s&gt;</td>\n",
              "      <td>▁Bright</td>\n",
              "      <td>on</td>\n",
              "      <td>▁&amp;</td>\n",
              "      <td>▁Ho</td>\n",
              "      <td>ve</td>\n",
              "      <td>▁Albi</td>\n",
              "      <td>on</td>\n",
              "      <td>▁-</td>\n",
              "      <td>▁Scu</td>\n",
              "      <td>nt</td>\n",
              "      <td>hor</td>\n",
              "      <td>pe</td>\n",
              "      <td>▁United</td>\n",
              "      <td>▁3:</td>\n",
              "      <td>2</td>\n",
              "      <td>&lt;/s&gt;</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Word IDs</th>\n",
              "      <td>None</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>3</td>\n",
              "      <td>3</td>\n",
              "      <td>4</td>\n",
              "      <td>5</td>\n",
              "      <td>5</td>\n",
              "      <td>5</td>\n",
              "      <td>5</td>\n",
              "      <td>6</td>\n",
              "      <td>7</td>\n",
              "      <td>7</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-432204a3-b159-496e-8f4f-0a9d1be07d6b')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-432204a3-b159-496e-8f4f-0a9d1be07d6b button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-432204a3-b159-496e-8f4f-0a9d1be07d6b');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@ TOKENIZING TEXTS FOR NER:\n",
        "previous_word_idx = None                                            # Initialization.\n",
        "label_ids = []                                                      # Initialization.\n",
        "for word_idx in word_ids:\n",
        "    if word_idx is None or word_idx == previous_word_idx:\n",
        "        label_ids.append(-100)\n",
        "    elif word_idx != previous_word_idx:\n",
        "        label_ids.append(labels[word_idx])\n",
        "    previous_word_dx = word_idx\n",
        "labels = [index2tag[l] if l != -100 else \"IGN\" for l in label_ids]\n",
        "index = [\"Tokens\", \"Word IDs\", \"Label IDs\", \"Labels\"]\n",
        "pd.DataFrame([tokens, word_ids, label_ids, labels], index=index)    # Initializing dataframe."
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 174
        },
        "id": "3yNixZoQ2QmY",
        "outputId": "5dd9e691-d2ed-4958-cc7d-e59c169f58a0"
      },
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "             0        1      2      3      4      5      6      7   8      9   \\\n",
              "Tokens      <s>  ▁Bright     on     ▁&    ▁Ho     ve  ▁Albi     on  ▁-   ▁Scu   \n",
              "Word IDs   None        0      0      1      2      2      3      3   4      5   \n",
              "Label IDs  -100        3      3      4      4      4      4      4   0      3   \n",
              "Labels      IGN    B-ORG  B-ORG  I-ORG  I-ORG  I-ORG  I-ORG  I-ORG   O  B-ORG   \n",
              "\n",
              "              10     11     12       13   14 15    16  \n",
              "Tokens        nt    hor     pe  ▁United  ▁3:  2  </s>  \n",
              "Word IDs       5      5      5        6    7  7  None  \n",
              "Label IDs      3      3      3        4    0  0  -100  \n",
              "Labels     B-ORG  B-ORG  B-ORG    I-ORG    O  O   IGN  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-6adf88e1-27c3-4f74-8f88-10ea57cd8284\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "      <th>10</th>\n",
              "      <th>11</th>\n",
              "      <th>12</th>\n",
              "      <th>13</th>\n",
              "      <th>14</th>\n",
              "      <th>15</th>\n",
              "      <th>16</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>Tokens</th>\n",
              "      <td>&lt;s&gt;</td>\n",
              "      <td>▁Bright</td>\n",
              "      <td>on</td>\n",
              "      <td>▁&amp;</td>\n",
              "      <td>▁Ho</td>\n",
              "      <td>ve</td>\n",
              "      <td>▁Albi</td>\n",
              "      <td>on</td>\n",
              "      <td>▁-</td>\n",
              "      <td>▁Scu</td>\n",
              "      <td>nt</td>\n",
              "      <td>hor</td>\n",
              "      <td>pe</td>\n",
              "      <td>▁United</td>\n",
              "      <td>▁3:</td>\n",
              "      <td>2</td>\n",
              "      <td>&lt;/s&gt;</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Word IDs</th>\n",
              "      <td>None</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>3</td>\n",
              "      <td>3</td>\n",
              "      <td>4</td>\n",
              "      <td>5</td>\n",
              "      <td>5</td>\n",
              "      <td>5</td>\n",
              "      <td>5</td>\n",
              "      <td>6</td>\n",
              "      <td>7</td>\n",
              "      <td>7</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Label IDs</th>\n",
              "      <td>-100</td>\n",
              "      <td>3</td>\n",
              "      <td>3</td>\n",
              "      <td>4</td>\n",
              "      <td>4</td>\n",
              "      <td>4</td>\n",
              "      <td>4</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>3</td>\n",
              "      <td>3</td>\n",
              "      <td>3</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>-100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Labels</th>\n",
              "      <td>IGN</td>\n",
              "      <td>B-ORG</td>\n",
              "      <td>B-ORG</td>\n",
              "      <td>I-ORG</td>\n",
              "      <td>I-ORG</td>\n",
              "      <td>I-ORG</td>\n",
              "      <td>I-ORG</td>\n",
              "      <td>I-ORG</td>\n",
              "      <td>O</td>\n",
              "      <td>B-ORG</td>\n",
              "      <td>B-ORG</td>\n",
              "      <td>B-ORG</td>\n",
              "      <td>B-ORG</td>\n",
              "      <td>I-ORG</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "      <td>IGN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-6adf88e1-27c3-4f74-8f88-10ea57cd8284')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-6adf88e1-27c3-4f74-8f88-10ea57cd8284 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-6adf88e1-27c3-4f74-8f88-10ea57cd8284');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@ TOKENIZING TEXT FOR NER: IMPORTANT:\n",
        "def tokenize_and_align_labels(examples):                                            # Defining class. \n",
        "    tokenized_inputs = xlmr_tokenizer(examples[\"tokens\"], truncation=True, \n",
        "                                      is_split_into_words=True)                     # Initializing tokenized inputs. \n",
        "    labels = []                                                                     # Initialization.\n",
        "    for idx, label in enumerate(examples[\"ner_tags\"]):\n",
        "        word_ids = tokenized_inputs.word_ids(batch_index=idx)                       # Initializing word ids.\n",
        "        previous_word_idx = None                                                    # Initialization.\n",
        "        label_ids = []                                                              # Initialization.\n",
        "        for word_idx in word_ids:\n",
        "            if word_idx is None or word_idx == previous_word_idx:\n",
        "                label_ids.append(-100)                                              # -100 is ignored. \n",
        "            else:\n",
        "                label_ids.append(label[word_idx])                                   # Adding labels.\n",
        "            previous_word_idx = word_idx\n",
        "        labels.append(label_ids)\n",
        "    tokenized_inputs[\"labels\"] = labels                                             # Adding labels column.\n",
        "    return tokenized_inputs\n",
        "\n",
        "#@ ENCODING PAN-X DATASET:\n",
        "def encode_panx_dataset(corpus):                                                    # Defining function. \n",
        "    return corpus.map(tokenize_and_align_labels, batched=True,\n",
        "                      remove_columns=[\"langs\", \"ner_tags\", \"tokens\"])               # Encoding dataset.\n",
        "panx_de_encoded = encode_panx_dataset(panx_ch[\"de\"])                                # Implementation."
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 104,
          "referenced_widgets": [
            "a7e4d91888d8452880ca5d502e6fd1fc",
            "ddef8dd0821b4b4b8ef9d68e45eaa8d8",
            "adec86fa94d04afc9421867641102675",
            "ed531099e8cf48f091b6e8f1d7954783",
            "66a7f3e118f44989a8b205f95b36b8af",
            "16b50d1599aa457caba85f60feeb7fc1",
            "48a16903bc9049eaa04dbd46994ef07c",
            "c59cceebba1146b5b982c27c837cbe31",
            "edc7d77a31784f9f9af6a9a956cf4138",
            "9c38643343a44a21b9fa568d42609fa4",
            "b2173e5cf3494fb6ab6f1648d5476f51"
          ]
        },
        "id": "X2FjueJ54VM7",
        "outputId": "7dbf70d2-d591-465a-f8a2-5c3d8f2d6000"
      },
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/13 [00:00<?, ?ba/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "a7e4d91888d8452880ca5d502e6fd1fc"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Loading cached processed dataset at /root/.cache/huggingface/datasets/xtreme/PAN-X.de/1.0.0/349258adc25bb45e47de193222f95e68a44f7a7ab53c4283b3f007208a11bf7e/cache-5e1da0ee67f278e1.arrow\n",
            "Loading cached processed dataset at /root/.cache/huggingface/datasets/xtreme/PAN-X.de/1.0.0/349258adc25bb45e47de193222f95e68a44f7a7ab53c4283b3f007208a11bf7e/cache-c3a89d9efa0b632d.arrow\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### **Performance Measures**"
      ],
      "metadata": {
        "id": "vUtUUxsg8DdF"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#@ INSPECTING PERFORMANCE MEASURES: \n",
        "y_true = [[\"O\", \"O\", \"O\", \"B-MISC\", \"I-MISC\", \"I-MISC\", \"O\"],\n",
        "          [\"B-PER\", \"I-PER\", \"O\"]]                                      # Initializing sample. \n",
        "y_pred = [[\"O\", \"O\", \"B-MISC\", \"I-MISC\", \"I-MISC\", \"I-MISC\", \"O\"],\n",
        "          [\"B-PER\", \"I-PER\", \"O\"]]                                      # Initializing sample. \n",
        "print(classification_report(y_true, y_pred))                            # Inspecting classification report. "
      ],
      "metadata": {
        "id": "_VHLIQYw77Rn",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8a8f2f47-8954-446d-d5b1-ab9d8356259a"
      },
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "        MISC       0.00      0.00      0.00         1\n",
            "         PER       1.00      1.00      1.00         1\n",
            "\n",
            "   micro avg       0.50      0.50      0.50         2\n",
            "   macro avg       0.50      0.50      0.50         2\n",
            "weighted avg       0.50      0.50      0.50         2\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@ FUNCTION FOR PERFORMANCE MEASURES:\n",
        "def align_predictions(predictions, label_ids):                                  # Defining function.\n",
        "    preds = np.argmax(predictions, axis=2)                                      # Initializing predictions.\n",
        "    batch_size, seq_len = preds.shape                                           # Initialization.\n",
        "    labels_list, preds_list = [], []                                            # Initialization.\n",
        "    for batch_idx in range(batch_size):\n",
        "        example_labels, example_preds = [], []                                  # Initialization.\n",
        "        for seq_idx in range(seq_len):\n",
        "            if label_ids[batch_idx, seq_idx] != -100:\n",
        "                example_labels.append(index2tag[label_ids[batch_idx][seq_idx]])\n",
        "                example_preds.append(index2tag[preds[batch_idx][seq_idx]])\n",
        "        labels_list.append(example_labels)                                      # Append labels.\n",
        "        preds_list.append(example_preds)                                        # Append predictions. \n",
        "    return preds_list, labels_list"
      ],
      "metadata": {
        "id": "wAFkt53ifiR8"
      },
      "execution_count": 32,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### **Fine-Tuning XLM-Roberta**"
      ],
      "metadata": {
        "id": "JqAxqMJVlFGT"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#@ INITIALIZING TRAINING ARGUMENTS:\n",
        "num_epochs = 3                                                          # Initializing epochs.\n",
        "batch_size = 24                                                         # Initializing batch size.\n",
        "logging_steps = len(panx_de_encoded[\"train\"]) // batch_size             # Initialization.\n",
        "model_name = f\"{xlmr_model_name}-finetuned-panx-de\"                     # Initialization. \n",
        "training_args = TrainingArguments(\n",
        "    output_dir=model_name, log_level=\"error\", \n",
        "    num_train_epochs=num_epochs,\n",
        "    per_device_train_batch_size=batch_size, \n",
        "    per_device_eval_batch_size=batch_size, \n",
        "    evaluation_strategy=\"epoch\",\n",
        "    save_steps=1e6, weight_decay=0.01, disable_tqdm=False,\n",
        "    logging_steps=logging_steps, push_to_hub=False)                     # Initializing training arguments. "
      ],
      "metadata": {
        "id": "-j29pnzOfwTZ"
      },
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@ COMPUTING METRICS: \n",
        "def compute_metrics(eval_pred):                                         # Defining function.\n",
        "    y_pred, y_true = align_predictions(eval_pred.predictions,\n",
        "                                       eval_pred.label_ids)             # Implementation. \n",
        "    return {\"f1\": f1_score(y_true, y_pred)}                             # Getting f1 score."
      ],
      "metadata": {
        "id": "TvihxmuFoMAS"
      },
      "execution_count": 34,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@ INITIALIZING DATA COLLATOR:\n",
        "data_collator = DataCollatorForTokenClassification(xlmr_tokenizer)\n",
        "\n",
        "#@ MODEL INITIALIZER: \n",
        "def model_init():                                                       # Defining function.\n",
        "    return (XLMRobertaForTokenClassification\n",
        "            .from_pretrained(xlmr_model_name, config=xlmr_config)\n",
        "            .to(device))                                                # Initializing model."
      ],
      "metadata": {
        "id": "U95kilgep-fI"
      },
      "execution_count": 35,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@ TRAINING THE MODEL:\n",
        "trainer = Trainer(model_init=model_init, args=training_args,\n",
        "                  data_collator=data_collator, \n",
        "                  compute_metrics=compute_metrics,                      # Computing metrics.\n",
        "                  train_dataset=panx_de_encoded[\"train\"],               # Training dataset.\n",
        "                  eval_dataset=panx_de_encoded[\"validation\"],           # Validation dataset.\n",
        "                  tokenizer=xlmr_tokenizer)                             # Initializing trainer.\n",
        "trainer.train()                                                         # Training model."
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "xaVRLCYrqwnE",
        "outputId": "fface965-b539-4d2e-b884-f8add8e74d02"
      },
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='1575' max='1575' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [1575/1575 16:13, Epoch 3/3]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>F1</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>0.262000</td>\n",
              "      <td>0.164417</td>\n",
              "      <td>0.822531</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>0.127800</td>\n",
              "      <td>0.142015</td>\n",
              "      <td>0.838958</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>0.080800</td>\n",
              "      <td>0.137400</td>\n",
              "      <td>0.861608</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TrainOutput(global_step=1575, training_loss=0.15672146745144375, metrics={'train_runtime': 974.4037, 'train_samples_per_second': 38.731, 'train_steps_per_second': 1.616, 'total_flos': 857112519661200.0, 'train_loss': 0.15672146745144375, 'epoch': 3.0})"
            ]
          },
          "metadata": {},
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@ INITIALIZING MODEL EVALUATION:\n",
        "text_de = \"Jeff Dean ist ein Informatiker bei Google in Kalifornien\"        # Text example.\n",
        "tag_text(text_de, tags, trainer.model, xlmr_tokenizer)                      # Inspection. "
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 112
        },
        "id": "87TEVlC7sy-8",
        "outputId": "48c0a84c-c33c-4daa-c714-f6f233ea5b50"
      },
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "         0      1      2      3     4     5           6    7     8        9   \\\n",
              "Tokens  <s>  ▁Jeff    ▁De     an  ▁ist  ▁ein  ▁Informati  ker  ▁bei  ▁Google   \n",
              "Tags      O  B-PER  I-PER  I-PER     O     O           O    O     O    B-ORG   \n",
              "\n",
              "         10          11     12    13  \n",
              "Tokens  ▁in  ▁Kaliforni     en  </s>  \n",
              "Tags      O       B-LOC  I-LOC     O  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-45782520-38f2-46fb-b518-3cfe05c9c6ef\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "      <th>10</th>\n",
              "      <th>11</th>\n",
              "      <th>12</th>\n",
              "      <th>13</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>Tokens</th>\n",
              "      <td>&lt;s&gt;</td>\n",
              "      <td>▁Jeff</td>\n",
              "      <td>▁De</td>\n",
              "      <td>an</td>\n",
              "      <td>▁ist</td>\n",
              "      <td>▁ein</td>\n",
              "      <td>▁Informati</td>\n",
              "      <td>ker</td>\n",
              "      <td>▁bei</td>\n",
              "      <td>▁Google</td>\n",
              "      <td>▁in</td>\n",
              "      <td>▁Kaliforni</td>\n",
              "      <td>en</td>\n",
              "      <td>&lt;/s&gt;</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Tags</th>\n",
              "      <td>O</td>\n",
              "      <td>B-PER</td>\n",
              "      <td>I-PER</td>\n",
              "      <td>I-PER</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "      <td>B-ORG</td>\n",
              "      <td>O</td>\n",
              "      <td>B-LOC</td>\n",
              "      <td>I-LOC</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-45782520-38f2-46fb-b518-3cfe05c9c6ef')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-45782520-38f2-46fb-b518-3cfe05c9c6ef button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-45782520-38f2-46fb-b518-3cfe05c9c6ef');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 37
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### **Error Analysis**"
      ],
      "metadata": {
        "id": "fS05ZIcRvec5"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#@ INITIALIZING ERROR ANALYSIS FUNCTION:\n",
        "def forward_pass_with_label(batch):                                             # Defining function.\n",
        "    features = [dict(zip(batch, t)) for t in zip(*batch.values())]              # Converting into list of dict.\n",
        "    batch = data_collator(features)                                             # Padding the inputs.\n",
        "    input_ids = batch[\"input_ids\"].to(device)                                   # Initializing input ids.\n",
        "    attention_mask = batch[\"attention_mask\"].to(device)                         # Initializing attention mask. \n",
        "    labels = batch[\"labels\"].to(device)                                         # Initializing labels.\n",
        "    with torch.no_grad():\n",
        "        output = trainer.model(input_ids, attention_mask)                       # Passing data through model. \n",
        "        predicted_label = torch.argmax(output.logits, axis=-1).cpu().numpy()    # Getting predictions.\n",
        "    loss = cross_entropy(output.logits.view(-1, 7), labels.view(-1), \n",
        "                            reduction=\"none\")                                   # Calculating loss per token.\n",
        "    loss = loss.view(len(input_ids), -1).cpu().numpy()                          # Unflatten batch dimension. \n",
        "    return {\"loss\": loss, \"predicted_label\": predicted_label}\n",
        "\n",
        "#@ IMPLEMENTATION:\n",
        "valid_set = panx_de_encoded[\"validation\"]                                       # Initialization.\n",
        "valid_set = valid_set.map(forward_pass_with_label,batched=True,batch_size=32)   # Implementation.\n",
        "df = valid_set.to_pandas()                                                      # Creating dataframe. "
      ],
      "metadata": {
        "id": "Pal0VHOzzbFe",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 86,
          "referenced_widgets": [
            "611a343b9fd44e2fa9da79f3e4a3be67",
            "fd21136b5e0c421f9607036ac3b9b0f0",
            "c11943147f4842969b0b87efb1b5770f",
            "df4cd72a9ca644cf84cd0803890c665c",
            "b6c10fccce624877b0e06dba89e90337",
            "8fc9e7f0aada42c7ad2be77ba79a0eb4",
            "f5f4ee7329d245a3a7ec22ed3500458c",
            "d68879f3d7574047925b423e55ab52fc",
            "4e60888b644440f990ffca1b6e8acdf4",
            "c8ed9e4d161840a096a025c03658e10f",
            "e19a6cc0dcd84a68b104ef1e3da01479"
          ]
        },
        "outputId": "0307bd7c-e689-4359-fbec-5e7482f1ca93"
      },
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Parameter 'function'=<function forward_pass_with_label at 0x7f808eff4d40> of the transform datasets.arrow_dataset.Dataset._map_single couldn't be hashed properly, a random hash was used instead. Make sure your transforms and parameters are serializable with pickle or dill for the dataset fingerprinting and caching to work. If you reuse this transform, the caching mechanism will consider it to be different from the previous calls and recompute everything. This warning is only showed once. Subsequent hashing failures won't be showed.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/197 [00:00<?, ?ba/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "611a343b9fd44e2fa9da79f3e4a3be67"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@ MAPPING TOKENS AND LABELS TO STRINGS:\n",
        "index2tag[-100] = \"IGN\"                                                         # Initializaion.\n",
        "df[\"input_tokens\"] = df[\"input_ids\"].apply(\n",
        "    lambda x: xlmr_tokenizer.convert_ids_to_tokens(x))                          # Adding input tokens.\n",
        "df[\"predicted_label\"] = df[\"predicted_label\"].apply(\n",
        "    lambda x: [index2tag[i] for i in x])                                        # Adding predicted label. \n",
        "df[\"labels\"] = df[\"labels\"].apply(\n",
        "    lambda x: [index2tag[i] for i in x])                                        # Adding labels. \n",
        "df[\"loss\"] = df.apply(lambda x: x[\"loss\"][:len(x[\"input_ids\"])], axis=1)        # Adding loss.\n",
        "df[\"predicted_label\"] = df.apply(\n",
        "    lambda x: x[\"predicted_label\"][:len(x[\"input_ids\"])], axis=1)               # Adding predicted label.\n",
        "df.head(3)                                                                      # Inspection."
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 239
        },
        "id": "kLABBXY9ulUX",
        "outputId": "b27e3dda-7c38-4a51-95b0-b99dda515d92"
      },
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                           input_ids  \\\n",
              "0  [0, 622, 11328, 33, 104658, 643, 168, 24129, 5...   \n",
              "1                  [0, 15758, 6, 4, 152886, 1785, 2]   \n",
              "2           [0, 242, 5106, 5753, 6406, 5106, 242, 2]   \n",
              "\n",
              "                                      attention_mask  \\\n",
              "0  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...   \n",
              "1                              [1, 1, 1, 1, 1, 1, 1]   \n",
              "2                           [1, 1, 1, 1, 1, 1, 1, 1]   \n",
              "\n",
              "                                              labels  \\\n",
              "0  [IGN, O, O, IGN, IGN, O, O, O, O, O, B-PER, IG...   \n",
              "1                    [IGN, B-LOC, O, IGN, O, O, IGN]   \n",
              "2               [IGN, O, O, B-ORG, I-ORG, O, O, IGN]   \n",
              "\n",
              "                                                loss  \\\n",
              "0  [0.0, 0.00016616394, 0.0028389886, 0.0, 0.0, 0...   \n",
              "1  [0.0, 0.01020181, 0.0003118029, 0.0, 0.0007510...   \n",
              "2  [0.0, 0.00033194278, 0.00050543883, 0.43016806...   \n",
              "\n",
              "                                     predicted_label  \\\n",
              "0  [O, O, O, O, O, O, O, O, O, O, B-ORG, B-ORG, I...   \n",
              "1                          [O, B-LOC, O, O, O, O, O]   \n",
              "2                   [O, O, O, B-ORG, I-ORG, O, O, O]   \n",
              "\n",
              "                                        input_tokens  \n",
              "0  [<s>, ▁Die, ▁Autor, en, rechte, ▁für, ▁den, ▁T...  \n",
              "1             [<s>, ▁Japan, ▁, ,, ▁1.200, ▁km, </s>]  \n",
              "2        [<s>, ▁', ▁'', ▁York, ▁City, ▁'', ▁', </s>]  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-e522d601-695f-42d8-9a81-a0e53e05ba6d\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>input_ids</th>\n",
              "      <th>attention_mask</th>\n",
              "      <th>labels</th>\n",
              "      <th>loss</th>\n",
              "      <th>predicted_label</th>\n",
              "      <th>input_tokens</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>[0, 622, 11328, 33, 104658, 643, 168, 24129, 5...</td>\n",
              "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
              "      <td>[IGN, O, O, IGN, IGN, O, O, O, O, O, B-PER, IG...</td>\n",
              "      <td>[0.0, 0.00016616394, 0.0028389886, 0.0, 0.0, 0...</td>\n",
              "      <td>[O, O, O, O, O, O, O, O, O, O, B-ORG, B-ORG, I...</td>\n",
              "      <td>[&lt;s&gt;, ▁Die, ▁Autor, en, rechte, ▁für, ▁den, ▁T...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>[0, 15758, 6, 4, 152886, 1785, 2]</td>\n",
              "      <td>[1, 1, 1, 1, 1, 1, 1]</td>\n",
              "      <td>[IGN, B-LOC, O, IGN, O, O, IGN]</td>\n",
              "      <td>[0.0, 0.01020181, 0.0003118029, 0.0, 0.0007510...</td>\n",
              "      <td>[O, B-LOC, O, O, O, O, O]</td>\n",
              "      <td>[&lt;s&gt;, ▁Japan, ▁, ,, ▁1.200, ▁km, &lt;/s&gt;]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>[0, 242, 5106, 5753, 6406, 5106, 242, 2]</td>\n",
              "      <td>[1, 1, 1, 1, 1, 1, 1, 1]</td>\n",
              "      <td>[IGN, O, O, B-ORG, I-ORG, O, O, IGN]</td>\n",
              "      <td>[0.0, 0.00033194278, 0.00050543883, 0.43016806...</td>\n",
              "      <td>[O, O, O, B-ORG, I-ORG, O, O, O]</td>\n",
              "      <td>[&lt;s&gt;, ▁', ▁'', ▁York, ▁City, ▁'', ▁', &lt;/s&gt;]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-e522d601-695f-42d8-9a81-a0e53e05ba6d')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-e522d601-695f-42d8-9a81-a0e53e05ba6d button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-e522d601-695f-42d8-9a81-a0e53e05ba6d');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@ INSPECTING TOKENS:\n",
        "df_tokens = df.apply(pd.Series.explode)                                         # Unpacking list of tokens.\n",
        "df_tokens = df_tokens.query(\"labels != 'IGN'\")                                  # Dropping IGN tokens.\n",
        "df_tokens[\"loss\"] = df_tokens[\"loss\"].astype(float).round(2)                    # Casting to floats.\n",
        "df_tokens.head(10)                                                              # Inspecting dataframe."
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 363
        },
        "id": "U__kfgdQ1RDf",
        "outputId": "931be542-b76b-4f2f-f83f-560771e9c0d6"
      },
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "  input_ids attention_mask labels  loss predicted_label input_tokens\n",
              "0       622              1      O  0.00               O         ▁Die\n",
              "0     11328              1      O  0.00               O       ▁Autor\n",
              "0       643              1      O  0.00               O         ▁für\n",
              "0       168              1      O  0.00               O         ▁den\n",
              "0     24129              1      O  0.00               O        ▁Text\n",
              "0     50012              1      O  0.00               O      ▁liegen\n",
              "0      1079              1      O  0.00               O         ▁bei\n",
              "0      6921              1  B-PER  0.97           B-ORG          ▁Ir\n",
              "0     37837              1  I-PER  1.04           I-ORG        ▁Mill\n",
              "0         6              1      O  0.00               O            ▁"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-a1b837f2-a338-4dab-94a3-71543c5cc5cb\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>input_ids</th>\n",
              "      <th>attention_mask</th>\n",
              "      <th>labels</th>\n",
              "      <th>loss</th>\n",
              "      <th>predicted_label</th>\n",
              "      <th>input_tokens</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>622</td>\n",
              "      <td>1</td>\n",
              "      <td>O</td>\n",
              "      <td>0.00</td>\n",
              "      <td>O</td>\n",
              "      <td>▁Die</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>11328</td>\n",
              "      <td>1</td>\n",
              "      <td>O</td>\n",
              "      <td>0.00</td>\n",
              "      <td>O</td>\n",
              "      <td>▁Autor</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>643</td>\n",
              "      <td>1</td>\n",
              "      <td>O</td>\n",
              "      <td>0.00</td>\n",
              "      <td>O</td>\n",
              "      <td>▁für</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>168</td>\n",
              "      <td>1</td>\n",
              "      <td>O</td>\n",
              "      <td>0.00</td>\n",
              "      <td>O</td>\n",
              "      <td>▁den</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>24129</td>\n",
              "      <td>1</td>\n",
              "      <td>O</td>\n",
              "      <td>0.00</td>\n",
              "      <td>O</td>\n",
              "      <td>▁Text</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>50012</td>\n",
              "      <td>1</td>\n",
              "      <td>O</td>\n",
              "      <td>0.00</td>\n",
              "      <td>O</td>\n",
              "      <td>▁liegen</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1079</td>\n",
              "      <td>1</td>\n",
              "      <td>O</td>\n",
              "      <td>0.00</td>\n",
              "      <td>O</td>\n",
              "      <td>▁bei</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>6921</td>\n",
              "      <td>1</td>\n",
              "      <td>B-PER</td>\n",
              "      <td>0.97</td>\n",
              "      <td>B-ORG</td>\n",
              "      <td>▁Ir</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>37837</td>\n",
              "      <td>1</td>\n",
              "      <td>I-PER</td>\n",
              "      <td>1.04</td>\n",
              "      <td>I-ORG</td>\n",
              "      <td>▁Mill</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>6</td>\n",
              "      <td>1</td>\n",
              "      <td>O</td>\n",
              "      <td>0.00</td>\n",
              "      <td>O</td>\n",
              "      <td>▁</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-a1b837f2-a338-4dab-94a3-71543c5cc5cb')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-a1b837f2-a338-4dab-94a3-71543c5cc5cb button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-a1b837f2-a338-4dab-94a3-71543c5cc5cb');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 41
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@ INSPECTING AVERAGE LOSS IN VALIDATION SET:\n",
        "df_tokens.groupby(\"input_tokens\")[[\"loss\"]]\\\n",
        "         .agg([\"count\", \"mean\", \"sum\"])\\\n",
        "         .droplevel(level=0, axis=1)\\\n",
        "         .sort_values(by=\"sum\", ascending=False)\\\n",
        "         .reset_index()\\\n",
        "         .round(2)\\\n",
        "         .head(5)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "XK9NEZbF2tMx",
        "outputId": "cecd3cf7-4d97-418c-e8de-d5d4c08374d8"
      },
      "execution_count": 42,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "  input_tokens  count  mean     sum\n",
              "0         ▁der   1376  0.10  141.55\n",
              "1            ▁   5949  0.02  139.23\n",
              "2          ▁in    994  0.11  106.56\n",
              "3           ▁(    255  0.41  104.68\n",
              "4         ▁von    785  0.12   97.13"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-34d0f65d-bdd0-468a-b2ac-3fa8a3578a47\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>input_tokens</th>\n",
              "      <th>count</th>\n",
              "      <th>mean</th>\n",
              "      <th>sum</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>▁der</td>\n",
              "      <td>1376</td>\n",
              "      <td>0.10</td>\n",
              "      <td>141.55</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>▁</td>\n",
              "      <td>5949</td>\n",
              "      <td>0.02</td>\n",
              "      <td>139.23</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>▁in</td>\n",
              "      <td>994</td>\n",
              "      <td>0.11</td>\n",
              "      <td>106.56</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>▁(</td>\n",
              "      <td>255</td>\n",
              "      <td>0.41</td>\n",
              "      <td>104.68</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>▁von</td>\n",
              "      <td>785</td>\n",
              "      <td>0.12</td>\n",
              "      <td>97.13</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-34d0f65d-bdd0-468a-b2ac-3fa8a3578a47')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-34d0f65d-bdd0-468a-b2ac-3fa8a3578a47 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-34d0f65d-bdd0-468a-b2ac-3fa8a3578a47');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 42
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@ INSPECTING AVERAGE LOSS IN VALIDATION SET:\n",
        "df_tokens.groupby(\"labels\")[[\"loss\"]]\\\n",
        "         .agg([\"count\", \"mean\", \"sum\"])\\\n",
        "         .droplevel(level=0, axis=1)\\\n",
        "         .sort_values(by=\"sum\", ascending=False)\\\n",
        "         .reset_index()\\\n",
        "         .round(2)\\\n",
        "         .head(5)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "aCoPnafI46Tt",
        "outputId": "b078da5d-9bb7-487f-b4d1-c8319d8d3446"
      },
      "execution_count": 43,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "  labels  count  mean      sum\n",
              "0  I-ORG   3987  0.44  1769.77\n",
              "1  B-ORG   2742  0.58  1594.90\n",
              "2      O  43183  0.03  1373.00\n",
              "3  B-LOC   3070  0.35  1084.92\n",
              "4  I-LOC   1425  0.63   891.23"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-178b0568-a4fe-47f7-bee0-eb7db1035c10\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>labels</th>\n",
              "      <th>count</th>\n",
              "      <th>mean</th>\n",
              "      <th>sum</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>I-ORG</td>\n",
              "      <td>3987</td>\n",
              "      <td>0.44</td>\n",
              "      <td>1769.77</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>B-ORG</td>\n",
              "      <td>2742</td>\n",
              "      <td>0.58</td>\n",
              "      <td>1594.90</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>O</td>\n",
              "      <td>43183</td>\n",
              "      <td>0.03</td>\n",
              "      <td>1373.00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>B-LOC</td>\n",
              "      <td>3070</td>\n",
              "      <td>0.35</td>\n",
              "      <td>1084.92</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>I-LOC</td>\n",
              "      <td>1425</td>\n",
              "      <td>0.63</td>\n",
              "      <td>891.23</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-178b0568-a4fe-47f7-bee0-eb7db1035c10')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-178b0568-a4fe-47f7-bee0-eb7db1035c10 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-178b0568-a4fe-47f7-bee0-eb7db1035c10');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@ PLOTTING CONFUSION MATRIX:\n",
        "def plot_confusion_matrix(y_preds, y_true, labels):                             # Defining function. \n",
        "    cm = confusion_matrix(y_true, y_preds, normalize=\"true\")                    # Initializing confusion matrix.\n",
        "    fig, ax = plt.subplots(figsize=(6, 6))                                      # Initialization.\n",
        "    disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=labels)   # Initializing confusion matrix display.\n",
        "    disp.plot(cmap=\"Blues\", values_format=\".2f\", ax=ax, colorbar=False)         # Plotting.\n",
        "    plt.title(\"Normalized confusion matrix\")\n",
        "    plt.show()\n",
        "\n",
        "#@ IMPLEMENTATION:\n",
        "plot_confusion_matrix(df_tokens[\"predicted_label\"], df_tokens[\"labels\"], \n",
        "                      tags.names)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 404
        },
        "id": "bytkl_7o5gLk",
        "outputId": "2ca9dbcb-83e9-4e20-862f-8a448bdd5feb"
      },
      "execution_count": 44,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x432 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAGDCAYAAAA1cVfYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd3gU5drH8e+dBAIJJCEFSEJCgCNFpKP0JtIROwo20KMUjwVFFPBIR0XA7hEVjociHQWkSFNARSmhCYIFCCEJEFoEFEh2n/ePnSS7KUuIySZ5vT/XtRe7M8/M/PbJ7Nw7hR0xxqCUUkrlxquoAyillCretFAopZRySwuFUkopt7RQKKWUcksLhVJKKbe0UCillHJLC4X6f0NEvhaRf1rP7xeRNQU8/xgRMSLiU5DzvcoyRUT+KyJnRWTrX5hPGxE5WJDZioqIRIvIBRHxLuosfxdaKFSeicgRETkpIv5Ow/4pIl8XYawcGWPmGGM6F3WOAtAa6ARUMcbclN+ZGGM2G2NqFVyswmGtY7e4a2OMOWqMKWeMsXkq19+dFgp1rbyBp//qTKxvyrr+XV1V4Igx5mJRBykOPLk3pzLpB1Vdq9eBoSISlNNIEWkpIttEJMX6t6XTuK9FZIKIfAv8AVS3DuUMFpFfROS8iIwTkRoi8p2I/C4iC0SktDV9BRH5QkSSrUMxX4hIlVxy9BORb6znw6xDFemPVBH5xBoXKCLTRSRJRBJEZHz6IQ0R8RaRySJySkQOAT3cdYyIRInIEivfaRF51xruJSIviUictUc2U0QCrXHph7MeFpGj1rJGWuMeBT4GWli5xzi/L6flGhH5h/W8u4jst/oyQUSGWsPbi8gxp2nqWH+PcyKyT0R6OY37RETeE5EV1nx+EJEaubzn9Pz9RSTe+rsMFJEbRWSPNf93ndrXEJENVv+cEpE56euSiMwCooHl1vsd5jT/R0XkKLDBaZiPiASLyDERudWaRzkR+VVEHnL3t1LXyBijD33k6QEcAW4BlgDjrWH/BL62ngcDZ4EHAR+gj/U6xBr/NXAUqGuNLwUYYCkQYA2/DKwHqgOBwH7gYWv6EOAuwA8oDywEPnfK9zXwT+t5P+CbHN5DFJAIdLNefwZMA/yBisBWYIA1biBwwJomGPjKyuuTw3y9gd3AG9a8ygCtrXGPAL9a76mc1X+zrHEx1jw/AsoCDaw+qJPT+8jpfVnT/8N6ngS0sZ5XABpbz9sDx6znpaw8I4DSwM3AeaCWNf4T4DRwk/V3mgPMy2WdSM//gfWeOwOXgM+t/owETgLtrPb/wHEozRcIAzYBb2Zdx3KY/0yrX8s6DfOx2nQGjlvL+whYVNSflf9vjyIPoI+S8yCzUNwApFgfdOdC8SCwNcs0W4B+1vOvgbFZxhugldPrHcALTq+nOG9IskzbEDjr9Ppr3BQKayOTMX+gkrVRLuvUpg/wlfV8AzDQaVxnci8ULYDkXMatBwY7va4FpFob4fSNXhWn8VuB+3J6H7m8L+dCcRQYAARkadOezELRxtqwejmNnwuMtp5/AnzsNK47cCCXv0F6/kinYaeBe51eLwaeyWX624GdWdexHOZfPYdhPk7D3gH2AglYX0z0UXAPPfSkrpkx5kfgC+DFLKMigLgsw+JwfKtMF5/DLE84Pf8zh9flAETET0SmWYdwfsfxbTRI8n71y3TgoDHmNet1VRzfrpOsQyTncOxdVHR6P855s743Z1FAnDEmLYdxWfslDkeRqOQ07LjT8z+w3nM+3IVjwx4nIhtFpEUueeKNMfYsmZz/TteaJ69/w0oiMs86LPY7MBsIvcq8Ief1xtmHOL7AfGKMOZ2H+alroIVC5dco4DFcNy6JODa+zqJxfMtL91d+rvg5HN/GmxljAoC21nC52oQi8iJQE3jUaXA8jj2KUGNMkPUIMMbUtcYn4SgA6aLdLCIeiJacT7Zm7ZdoIA3XjWleXcRx6A0AEansPNIYs80YcxuOYvc5sCCXPFHiejFB1r9TYZmIYx2oZ/0NH8D175fb+pHremN9UfgQx+Gpwenna1TB0UKh8sUY8yswH3jKafBKoKaI9LVONN4LXI9j76MglMfx7fSciATjKFZXJSLdrJx3GGP+dHoPScAaYIqIBFgnnWuISDuryQLgKRGpIiIVyL4H5WwrjsLyqoj4i0gZEWlljZsLDBGRaiJSDsfGcn4uex9XsxuoKyINRaQMMNrpfZYWx/8fCTTGpAK/A/Yc5vEDjr2EYSJSSkTaA7cC8/KR51qVBy4AKSISCTyfZfwJHOdyrsUIHIXkERwXW8y8hr1MlQdaKNRfMRbHCUYArF3+nji++Z8GhgE9jTGnCmh5b+I4z3AK+B5Yncfp7sVxPuUnybzy6QNr3EM4Tujux3HifREQbo37CPgSx8Y5FsdJ6BwZxzX9t+I4WXsUOGYtF2AGMAvHobLDOE72PpnH7FmX8zOOfl8H/AJ8k6XJg8AR67DOQOD+HOZxxcraDUdfvg88ZIw5kJ9M12gM0BjHOa4VZO/TV4CXrEOBQ682MxFpAjyLI78NeA1H0XBX1NU1EutEkFJKKZUj3aNQSinllhYKpZRSbmmhUEop5ZYWCqWUUm5poVBKKeXW3/KXGKW0v5GywUUdI1/q16h49UbFkJdc9f/EKZWhJK8tJfU60ri4I5w+dSrHrv97Foqywfi2eLaoY+TLhsX5uvy+yJUpVXL//5PWOM+TEtzpqWk5/R/H4q9dq9xvd6KHnpRSSrmlhUIppZRbWiiUUkq5pYVCKaWUW1oolFJKuaWFQimllFtaKJRSSrmlhUIppZRbWiiUUkq5pYVCKaWUW1oolFJKuaWFQimllFtaKJRSSrmlhUIppZRbWiiUUkq5pYVCKaWUW3/LGxflV8fGVXnl8fZ4e3kxa82PvLlom8v4qLDyvPNMZ0IDynL2wiUGTF5N4ukLANx38/UMvc9xY5DJ87Yyb8N+j2b/6vufePmtJdjthj49m/OvB29xGX/5ShpPj5/N3oPHqBDgx3/GPkxUeAjxSadpf/+rVI8OA6Bx3Rhee763x3Kv37Kf4VMXY7fbeaBXC555uHOW3KkMHjOL3QfiqRDoz/Tx/YmOCOFMykX6vzidnT/FcV+PZkzyYOZ067bsZ8SUxdjsdh68Lefsg0ZnZp8xwcp+7iL9hk9n5/44+vTU7NeU+7v9DJ+yyMrdkiH9csg9aha7DhwlONCfGRMfIToiBICp//2S2cu24O3lxatD76Zji+s9mn3Dlv2MfHMJNptjXX/qoU7Zsv9r7Gx2H4gnONCfD8f3Izo8hNh9cTz32jwAjDE8/2g3erRvUKDZSvwehYhUEZGlIvKLiPwmIm+JSOmCXo6Xl/D6oJu5Z9TnNB/8P+5qV4taUa63Ux37aFvmrf+J1k/OZtLcH3j54dYABJXz5YW+zbnl2bl0HDKXF/o2J9Dft6Aj5spmszNy6iJmTx7AV7Nf5PN1sfx8+LhLm7lffE9geT++nf8Sj93bngn/WZ4xrmpkCGs/GcbaT4Z5tEjYbHaGvb6QBW8O4rt5I1myZgcHDiW5tJm9bAtB5f3YvngUg+7rwJj3lgLgW9qH4QN6MOapOzyW15nNZmfYpIUseGsQW+aPZPGXuWffsWQUg/p0YPS7VnZfH0YM6MFYzX7NuZ+ftICFbw3m+wUvsTiH9WXW0i0EBpQl9rPRDOrbgdHvOHIfOJTEkrWxbJk/kkVvD2boawuw2Tx3pzqbzc4LUxYyd+pAvpk7giVrd3DwsGv2Ocsdn9Gti15mwH3tGffeMgBq1whn7YyhfDXzBea/MYjnJ80nLc1WoPlKdKEQx/0SlwCfG2OuA2oC5YAJBb2sJjUrcyjpHHEnUkhNs7Nk00G6N6/h0qZWVAib9xwFYPOeeLo1rw5Ax8YxfL0zjnMXLpNy8TJf74zjliYxBR0xVzt/iiOmSihVI0MpXcqH225pxJff7HVps+abvdzT7UYAerRvwDc7fsGYor37b+z+OKpVCSXGyn1Hpyas2uSae9WmvdzXoxkAvW5uyKZtP2OMwb+sL80b1sC3dNHsNO/Y55r9zs7Zs6/cmJn9tpyy+2r2a8t9hOpRocRUsXJ3aszKjXtc2qzatIc+GbkbsXHbQYwxrNy4hzs7Nca3dCmqRoZSPSqUHfuOeCy7Y10Py1zXb2nM6ix9vnrzXu7t7jgqcWuHhmze7uhzvzKl8fFx3Gr40pU0CuOO4yW6UAA3A5eMMf8FMMbYgCHAIyLiV5ALCg8pR0Ly+YzXiacuEB5SzqXNvsPJ9Gx5HQA9W/yDAD9fKpQvQ3hIOY6dypw24XT2aQvT8eQUIipWyHgdHhbE8eSUXNv4+HgT4F+GsykXATiadIbO/V/nrn+9ww+7f/NY7qST54islJk7omIQScnnXNskpxBRMSgzd7mynLFyF6Wk5Lxlj6yk2QuKI5NT7koVSMqynieezGzjnDvbtBWzT1uYjiefI9JajwHCKwZlW/7xLH1evlyZjD7fse8IbfpOpN0Dr/D6sN4ZhaOglPRzFHWBHc4DjDG/i8hR4B/AnhynKiT/nrGJSQM70Lfj9Xy3L4GEU+ex2Yv2W/lfVTEkkK2LRxEc6M+eA/E8MmI6X816kfL+ZYo6mlLK0qRuDJs/HcHPR47z5NjZdGxxPWV8SxXY/Ev6HkWeicjjIrJdRLabK9f+zSfp9AUiw8pnvI4ILUeSdaI63fEzF3lo4he0e3oO42d+C8DvFy+TdPoCVUIzp40MyT5tYaocFkjiybMZr5OSz1E5LDDXNmlpNn6/eIkKgf74lvYhONAfgPq1o4iJCOFQ/EmP5A6vGETCiczciSfPER4W5NomLJDEk+cyc1/4MyNvUQoPy1v2hBOavaA4MjnlPnGW8CzreUTFzDbOubNNezL7tIWpclgQCScz99qSTp7LtvzKWfr8/IVL2fq8Zkxl/P18s52b+atKeqHYDzRxHiAiAUA08KvzcGPMh8aYpsaYplL62lfo2J+PUyOiAtGVAijl48WdbWux6odDLm2CA8og1uHBIffcyJy1+wBYH3uEDo2qEujvS6C/Lx0aVWV97JFrzpBfDWtHczj+FEcTT3MlNY2l63bSudUNLm06t7qBhascV3Gt+Ho3rRpfh4hw+uyFjJN6cQmnOHzsVMZVIoWtUZ1oDsUnE5d4iiupaXy2dgfd2tZzadO1TT3mrfgBgGUbdtGmaU1ECv4Y7bVqfL2VPcGRfcmaHXRt45q9W9vM7Es1+1/W+Pqq/HbUKffaWLq1re/SpmubeszNyL2Ttjc6cndrW58la2O5fCWVuIRT/HY0mSZ1YzyWPXNdd3xGP1sXS5csfd6l9Q3MX7kVgOVf7aJ1E8dnNC7xdMbJ6/ikM/wSd4Ko8OBsy/grpKhPWP4V1snsbcDbxpiZIuINfAD8box5LrfpvAKjjG+LZ695eZ2axjDxsfZ4ewlz1u5jyoKtDL+/Bbt+OcGqrYfo1eo6Xn64FcbAdz8e4/n/fMUV6w94f6e6PHuP40TUlAU/8Om6/F0em7D4yXxNt37Lfka99Rl2u517ezTj6Yc78/rHK2lQO5rOrW/g0uVUnho3m32/JBAU4Mf7ox+iamQoK77ezeSPV+Hj44WXlxfPPdKVzq1vuPoCsyhTKn/HTNd+u4+RbyzGZjf0vbU5z/XvwivTVtCwTjTd2tbj0uVUBo2eyd6fjxEU4MfH4/sTExkKQMPbR3H+4iVSU9MIKOfHorcHU7t6+DVnyO/2b+23+xgx1ZH9/lub89wjXZg4bQWNnLIPHOXIXiHAj48nZGZvcJtT9vJ+LM5n9vwq6uz5LTprvt3HiKmLsNkM9/dqztBHujLxgy9oWCea7u3qZ+TeczCeCgH+TJ/Qn5gqjtyTZ6xmzrLv8fH2YuKzd9GpVd18ZUhNy9/VUuu+28dLby7BZrfTt2dzhvTrwqsfOtb1rm0cff7EmFkZfT5tXD9iIkNZsGor78xah4+PN14iPPdIV7q3q3/V5WXVrtVNxO7YnmPHl+hCASAiUcD7QG0ce0grgaHGmMu5TZPfQlEc5LdQFLX8ForioBh8yf/bKeq9k78iv4WiqLkrFCX9ZDbGmHjg1qLOoZRS/1+V9HMUSimlCpkWCqWUUm5poVBKKeWWFgqllFJuaaFQSinllhYKpZRSbmmhUEop5ZYWCqWUUm5poVBKKeWWFgqllFJuaaFQSinllhYKpZRSbmmhUEop5ZYWCqWUUm5poVBKKeWWFgqllFJuaaFQSinllhYKpZRSbpX4W6HmR73qFVmz4F9FHSNfqg+YV9QR8iX+475FHSHfSvmU3O9Tdrsp6gj5UoJvmc2lVFtRR8gXu8l9XSm5nwCllFIeoYVCKaWUW1oolFJKuaWFQimllFtaKJRSSrmlhUIppZRbWiiUUkq5pYVCKaWUW1oolFJKuaWFQimllFtaKJRSSrmlhUIppZRbWiiUUkq5pYVCKaWUW1oolFJKuaWFQimllFtaKJRSSrmlhUIppZRbf8tboebX1z/8xNh3PsNmN9zboxmD77/FZfzlK2k8O3EOP/58jKAAP94d9TBR4cF8vnYH0+ZtyGh34LckvvjoOepeF+mx7B3qhTOu7414ewlzNv3Kuyv2uYyPDPbj7cdaEuBXGm8vYcLCnazfk0ijaiG83r8ZAIIw+fM9rIqN91juDVv2M/LNJdhsdh7o1YKnHurkMv7ylVT+NXY2uw/EExzoz4fj+xEdHkLsvjiee81x21hjDM8/2o0e7Rt4LDfAuu/2M3zKImx2Ow/e1pIh/Tpnyz5o1Cx2HThKcKA/MyY+QnRECABT//sls5dtwdvLi1eH3k3HFtd7NPv6LfsZPnUxdruj3595OHv2wWNmsftAPBUC/Zk+vj/RESGcSblI/xens/OnOO7r0YxJz/f2aO6S3Odf//ATo992bF/u69GMJx7Ivn0ZMmEOe38+RoUAP94b7di+APz0WyLDJy/g/MVLeIkXyz8cQhnfUgWWzWN7FCJiE5FdIrJbRGJFpGUu7UaLSILV9kcR6ZXD8PRHkIi0F5EU6/UBEZlcGPltNjsvv7mYTyY9ztr/vcCy9Tv55chxlzYLVnxPYPmybPx0JI/e045Xpy0H4PZOTVg1/XlWTX+eN0bcT1R4sEeLhJcIrzx4E32nbqDtiOXc0SyGmhGBLm2e6VWPZVvj6DRqJQP/8w2vPnQTAAcSztFl9CpueXklfaZs4PV+zfD28swNjW02Oy9MWcjcqQP5Zu4IlqzdwcHDSS5t5iz/nsDyfmxd9DID7mvPuPeWAVC7RjhrZwzlq5kvMP+NQTw/aT5paZ67l7HNZuf5SQtY+NZgvl/wEovX7ODAIdfss5ZuITCgLLGfjWZQ3w6MfmcpAAcOJbFkbSxb5o9k0duDGfraAmw2u0ezD3t9IQveHMR380ayJIfss5dtIai8H9sXj2LQfR0Y854ju29pH4YP6MGYp+7wWF7n3CW5z196YzH/e/1x1s90bF9+zrJ9mW9tXzbPHck/e7fjlQ8c25e0NBtPj5vNxOfuYf3MF1nw9hOU8vEu0HyePPT0pzGmoTGmATAceMVN2zeMMQ2Be4AZIuLlPNzpcc4avtlq3wjoKSKtCjr8rp+OUjUylOiIUEqX8uHWmxux5psfXdqs+fZH7uri2MB2b9eA72J/wWS5Yfmy9Tu59eZGBR3PrUbVQzh84jxHky+QarPz+Q9H6NKoiksbY6B8Wcc3kPJlS3H87J8A/HnFhs3ueA9lSnllez+FKXZ/HNWqhBET6ejzO25pzOpNe13arN68l3u7O/r81g4N2bz9Z4wx+JUpjY/1Ybl0JQ3wTHFLt2PfEapHhRJTxZH9zk6NWblxj0ubVZv20KeHY2/ttpsbsXHbQYwxrNy4hzs7Nca3dCmqRoZSPSqUHfuOeCy7o99DM/u9UxNWZen3VZv2cp+VvdfNDdm0zdHv/mV9ad6wBr6lPX+woiT3+a6fjhITGUrV9O1Lxxy2L9/8yN1dM7cv31rbl03bDlKnRgTX/8Px5bNCoD/e3gW7aS+qcxQBwNmrNTLG/ASkAaF5makx5k9gF1DgX9dPnDpHRMWgjNfhYYGcOJWSpU1KRhsfH2/K+5fhbMpFlzZffLWTXh0bF3Q8t8Ir+JF45o+M10ln/yC8gp9Lm8mf7+GuFtWInXoHc57twMjZ2zLGNaoewsYJPflqfE+G/W9rRuEobMeTzxHp3OcVg0hKTsnSJoXISk59Xq4MZ6w+37HvCG36TqTdA6/w+rDeGYXDE5KSU4isVCHjdUSlCtmyJ57MbOPj401AubKcSbmYfdqK2actTEknz2VZfhBJyedc2yS7ruvp2YtSSe7z4zltX7Ku67lsXw7FJ4PAA899QPdHJ/OfT9cXeD5Plv2yIrILKAOEAzdfbQIRaQbYgWRr0BARecB6ftYY0yFL+wrAdcCmAktdgHbuj6Osb2lqVQ8v6ijZ3NE8hvnfHuKD1T/RpEYo7z7eknYvfYExsPPQadqN/ILrwgN4+7GWbNibwOVUz+2W51eTujFs/nQEPx85zpNjZ9OxxfUFetxWqeLAZrOzfc9hln84hLJlStNnyPvUqxVF6yY1C2wZRXHoqTbQFZgpIrkdDxhiFZXJwL0m83iH86En5yLRRkR2AwnAl8aY41lnKCKPi8h2Edl+5vSpaw5fKTSIxJOZ36qSklOoFBqYpU1gRpu0NBvnL16iQqB/xvjlG2Lp1dGzh53AsQcREZy5BxFewY+ks3+4tOnbtgbLtsYBsOO3U/iW8iaknK9Lm1+SfufipTRqRwbhCZXDgkhw7vOT5wgPC8zSJpCEE059fuESwU59DlAzpjL+fr7ZjlcXpvCwQBJOZO40J544my17RMXMNmlpNn6/8CfBgf7Zpz2ZfdrCFF4xKMvyzxEe5vo3Dw9zXdfTsxelktznlXPavmRd13PZvoRXDOSmBtUJDipH2TKl6dD8en78+ViB5iuSQ0/GmC04DieFiciE9JPTTk3SC0IbY8zmPMxys3Xuoy7wqIg0zGGZHxpjmhpjmgaH5OlIlosGtaM4ciyZ+KTTXElNY/mGnXRqVdelTadWN7D4y60ArNy4m5aN/kF6LbTb7az4aje3FkGh2HX4NNUrlSc61J9S3l7c3iyGNTtdV6SE0xdpc31lAK4LD8C3lDenzl8mOtQ/4+R1lRB//hEeQPwpzxxiaFQnmkPxycQlOvr8s3WxdGlTz6VNl9Y3MH+lo8+Xf7WL1k2uQ0SISzydcfI6PukMv8SdyLhCxBMaX1+V344mE5dwiiupaSxZG0u3tvVd2nRtU4+5K34AYOmGnbS9sSYiQre29VmyNpbLV1KJSzjFb0eTaVI3xmPZM/vdkf2ztTvo1ta137u2qcc8K/uyDbto07QmuX/v84yS3OcNakdx+FgyR611ffn6nLcvi1Y7bV8aO7YvbW+qzcFDSfx56QppaTa+3/Ur18VUKtB8RXJ5rIjUBryB08aYkcDIgpivMeawiLwKvAD0KYh5pvPx8WbsM3fx0NBp2Ox2endvRs1q4Uydvop6taPo1OoGendvxrMT5tCu7wSCyvvxzqgHM6b/YfchwisGER1x7UXqr7LZDSNmb2Pu0I54ewlzN//GwcQUht1Rn12Hz7Bm1zFGz4tlcv9mPN65DgbD0x9vAeCmmhV5skddUm127HZ4cdZWzly47JHcPj7evPrc3dz7zPvY7Hb69mxO7erhvPrhChrWiaZrm3rcf2sLnhgzi5vuHkuFAD+mjesHwA+7f+OdWevw8fHGS4TXhvYmJKicR3KnZ580rDd3PfUeNpvh/l7NqVMjnIkffEHDOtF0b1efB29rycBRM2l8x2gqBPgzfUJ/AOrUCOf2WxrRvPcEfLy9eH1Y7wI/OXm17K8NvYd7nnofm93Q91ZHv78yzdHv3drW44FeLRg0eiZN7xpDUIAfH4/vnzF9w9tHcf7iJVJT01i5cS+L3h5MbQ8cbi3pfT7umbt40Nq+3Nu9GbWqhTNl+irq1Yqic+sbuLdHM56ZMIc2fRzbl3dHO7YvQeX9+Oe97en5+FREhA7N69CxRd2rLPHaiKeuYhERG5B+6YQAI4wxK3JoNxq4YIyZnMPwx8g8XwFwOxADDDXG9LTalQV+BVoZY47klKVBoyZmzcbv/8K7KTq1Bs8v6gj5Ev9x36KOkG+lfEru/0u1e+jCg4Lm5aFLsAvD+T9TizpCvnRq15xdsTty7HiP7VEYY/J0yYkxZrSb4TmNOwJ87dTuTwrhqiellPq7KrlflZRSSnmEFgqllFJuaaFQSinllhYKpZRSbmmhUEop5ZYWCqWUUm5poVBKKeWWFgqllFJuaaFQSinllhYKpZRSbmmhUEop5ZYWCqWUUm5poVBKKeWWFgqllFJuaaFQSinllhYKpZRSbmmhUEop5VaR3DO7qHl7Cf6+ebrhXrET91GB3grcYyp2m1jUEfLt7NqXijpCvpXUW4p66hbNhaFs6ZK5bfGS3NcV3aNQSinllhYKpZRSbmmhUEop5ZYWCqWUUm5poVBKKeWWFgqllFJuaaFQSinllhYKpZRSbmmhUEop5ZYWCqWUUm5poVBKKeWWFgqllFJuaaFQSinllhYKpZRSbmmhUEop5ZYWCqWUUm5poVBKKeWWFgqllFJuaaFQSinl1t/yntn5tX7Lfka+sQSb3c4DvVrw9EOdXMZfvpLKE2Nms/tgPMEB/nw0vh/RESF8/cMBxr2/jNQ0G6V8vBn95O20aVrTo9k3fP8T/35zCTabnftvbc6T2bKn8eS42ew5EE+FQH+mjXuY6PAQYvfH8fxr8wHHfYyHPtqV7u0aeCx3x6bVeWVwF7y9hFmrdvHm/O9cxkdVDOSdoT0JDfTj7PlLDHj1cxJPnSeqYiCzRt+Nl5fg4+3NR0u38d8vYj2WG2Ddd/sZPmURNrudB29ryZB+nV3GX76SyqBRs9h14CjBgf7MmPgI0REhAEz975fMXrYFby8vXh16Nx1bXK/Z85J7y35GTFls5W7BMw/nkHv0LHZb6/mMCf2JjgjhzLmL9Bs+nZ374+jTsxmTnu/tsczpivP2pcj2KETkQi7DR4tIgojsEpEfRWN+rncAACAASURBVKRXDsPTH0Ei0l5EUqzXB0RkcmHktdnsvDh5IfPeGMi3c0fw2ZodHDyc5NJmzrLvCQrwY9uilxnYpz1j31sGQHCQP3MmD2DTnOG8+/IDDB4zqzAius0+fPJCPp0ygE2fDuezdbEcPHzcpc2ny7cQVL4s3y/8NwPubc/495cDULt6OF9Of471/xvG3KkDef61BaSl2TyS28tLeP3JbtwzYi7N//kBd3WoS63oUJc2Ywd0ZN7avbQe8BGTZm/m5UdvBuD4mfN0fvoT2g78mE5PzuCZe1tSOaScR3KDo8+fn7SAhW8N5vsFL7F4zQ4OHHJdX2Yt3UJgQFliPxvNoL4dGP3OUgAOHEpiydpYtswfyaK3BzP0tQXYbHbNnofcwyYtZMFbg9gyfySLv8yee/ayLQSV92PHklEM6tOB0e86cvv6+jBiQA/GPnWHR7JmVdy3L8X10NMbxpiGwD3ADBHxch7u9DhnDd9stW8E9BSRVgUdKHZ/HDFVwoiJDKV0KR9u79SYVZv2urRZtXkv93a/CYBbOzRk8/afMcZQv1YUlcMCAceG99LlVC5fSS3oiLnauT+OalXCqJqe/ZbGfLnZNfuXm3+kdzdH9p4dGvCNld2vTGl8fLwBuHQlDRGPxaZJrQgOJZ4h7vg5UtPsLPl6H91bun5TqhUdxuZdRwDYvOsI3Vo4xqem2bmS6ihopUv54OXlweDAjn1HqB4VSkwVR5/f2akxKzfucWmzatMe+vRoBsBtNzdi47aDGGNYuXEPd3ZqjG/pUlSNDKV6VCg79h3R7FfNHUe1KqEZn9E7OzfJ9hlduXEv92XkbsimbY713L+sL80b1sDXt2gOshT37UtxLRQAGGN+AtKA0Ku1tdr/CewCIgs6S1LyOSIrBmW8jqgYRFJyikub48kpRFZytPHx8SagXBnOpFx0abP8q13Ur1kF39KlCjpirpKSU4iolJk9PCx79qTkc0RUqgA4spf3z8weu+8Ibe9/hQ4PvsqkYb0zCkdhCw8tT0Ly7xmvE0+dJzy0vEubfYdO0LN1LQB6tq5FgL8vFcqXBSAyLIBvpj3Gj58+xVvzv+P46Rx3YgtFUnIKkVZ/AkRUqpCtzxNPZrZxrC9lOZNyMfu0FbNPW5hKavak5HNZlh1EUvK5LG2yfkbLZvuMFoXivn0p1oVCRJoBdiDZGjTE6bDTVzm0rwBcB2zKYdzjIrJdRLafSk7OOtojDhxKYtx7y5j84r1Fsvz8alw3hk1zhrN6+nO8PXMdly57bm/oav794Tpa1a/Kxv/8k1b1q5KQ/Ds2u+NQR0Ly77Qe8BFN+r3HfZ3qExbkX8RplSo8hbl9Ka6FYoiI7AImA/caY4w13PnQUwen9m1EZDeQAHxpjDmedYbGmA+NMU2NMU1Dw8KuOVB4WBAJJzO/nSSePEe4tbuXrnJYIAknHG3S0mz8fuESwYH+VvuzPPzCx7z78oNUq3Lty/8rwsMCSTyRmT0pOXv28LAgEk+cBRzZz1/MzJ6uZkxl/Mv6ZjvuW1iSTp0nMiwg43VEaHmSTp13aXP89AUeGrOIdoM+ZvwMx3eH3y9eztbmpyPJtKgXVfihLeFhgSRY/QmQeOJstj6PqJjZxrG+/ElwoH/2aU9mn7YwldTs4WFBWZZ9jvCwoCxtsn5G/8y2nheF4r59KfJCISIT0vcSnAanF4Q2xpjNeZjNZmNMA6Au8KiINCzonI3qRHM4Ppm4xNNcSU3j87WxdG1Tz6VN1zY3MH/lVsCxC9i66XWICCnn/6Dvs9P49+BeNGtQvaCjXVXDOtEcOuaUfV0snVvf4NKmc5sbWLDKkf2Lr3bTqokje1zi6YyT1/FJZ/j16AmiwoM9kjv2YCI1IoOJrhxEKR8v7mxfl1VbfnZpExxQNuO8yZA+rZjz5W7AUVTKlHYcbw4sV4bmN0Txa/xpj+QGaHx9VX47mkxcwimupKaxZG0s3drWd2nTtU095q74AYClG3bS9saaiAjd2tZnydpYLl9JJS7hFL8dTaZJ3RjNftXc0RyKd8q9Zke2z2i3tvWYl5F7F22aOnIXteK+fSnyy2ONMSOBkQU0r8Mi8irwAtCnIOaZzsfHm1eG3k3vp9/HbrfTp2dzalcP59UPV9CwdjRd29bj/ltbMHjMLG68eywVAvz4cFw/AD5euJnDx04xecZqJs9YDcDCtwYTFlw+9wUWcPaJz95FnyH/wWbLzP7aRytpWDuKLm3q0bdnc/41djbN7xlHUIAf08Y+DMDW3Yd4Z/Y6Svl44yXCq8/dQ0iQZ64estkNw95dzeJX+uDt5cWcL3dxIO4Uwx9ux66fE1m15RdaN6jKy4/ejDGG7/Ye5fl3HP1bMzqU8QNuwRgQgXcXfs/+I5475Ojj482kYb2566n3sNkM9/dqTp0a4Uz84Asa1omme7v6PHhbSwaOmknjO0ZTIcCf6RP6A1CnRji339KI5r0n4OPtxevDeuPt7bnvdCU1u4+PN5Oev4e7n3ofm91w/61W7mkraFQnmm5t6/FArxYMHDWTJneOoUKAHx9buQEa3DaK8xcvkZqaxoqNe1n89mBqVw/3WPbivH2RzKM6niUiF4wx2bY4IjIauGCMmZzD8MfIPF8BcDsQAww1xvS02pUFfgVaGWOO5LTsxk2amk3fbf3rb6II2OxF8/f6qyp3f6WoI+Tb2bUvFXWEv52i2i4VhJL6GW3b8iZid2zPcfeqyPYocioS1vDRbobnNO4I8LVTuz8phKuelFLq76rIz1EopZQq3rRQKKWUcksLhVJKKbe0UCillHJLC4VSSim3tFAopZRySwuFUkopt7RQKKWUcksLhVJKKbe0UCillHJLC4VSSim3tFAopZRySwuFUkopt7RQKKWUcksLhVJKKbe0UCillHJLC4VSSim3ivye2era+JbyLuoI+VKSbydaud/soo6Qb3vevqeoI+RLsH+poo6QbyX0Tqi4i617FEoppdzKdY9CRN7BTZExxjxVKImUUkoVK+4OPW33WAqllFLFVq6FwhjzP+fXIuJnjPmj8CMppZQqTq56jkJEWojIfuCA9bqBiLxf6MmUUkoVC3k5mf0m0AU4DWCM2Q20LcxQSimlio88XfVkjInPMshWCFmUUkoVQ3n5fxTxItISMCJSCnga+KlwYymllCou8rJHMRB4AogEEoGG1mullFJ/A1fdozDGnALu90AWpZRSxVBernqqLiLLRSRZRE6KyFIRqe6JcEoppYpeXg49fQosAMKBCGAhMLcwQymllCo+8lIo/Iwxs4wxadZjNlCmsIMppZQqHtz91lOw9XSViLwIzMPx20/3Ais9kE0ppVQx4O5k9g4chUGs1wOcxhlgeGGFUkopVXy4+62nap4MopRSqnjK042LROQG4Hqczk0YY2YWViillFLFx1ULhYiMAtrjKBQrgW7AN4AWCqWU+hvIyx7F3UADYKcxpr+IVAJK7r0h/4L1W/Yz8o0l2Ox2HujVgqcf6uQy/vKVVJ4YM5vdB+MJDvDno/H9iI4I4esfDjDu/WWkptko5ePN6Cdvp03Tmh7Nvu67/Qyfsgib3c6Dt7VkSL/O2bIPGjWLXQeOEhzoz4yJjxAdEQLA1P9+yexlW/D28uLVoXfTscX1mjsPOtQLZ1zfG/H2EuZs+pV3V+xzGR8Z7Mfbj7UkwK803l7ChIU7Wb8nkUbVQni9fzMABGHy53tYFZv159YKz6atB5jw3ufY7Xbu6d6Mx/t0dBm/bc9vTHxvKQcPJTH1pQfo2q5BxrhHX/yQ3fvjaHJDNaZN/KfHMqcryZ/RDVv289KbS7DZ7NzfqwVP5ZD9X2Nns+dAPBUC/flwfD+iw0OI3RfH0NfmAWCM4flHu9G9fYOcFpFvebk89k9jjB1IE5EA4CQQdbWJRMQmIrtEZLeIxFq/F5Vb29YislVEDliPx53GjRaRBGte+0WkT5Zpn7Wm2Wsta6r1m1QFymaz8+Lkhcx7YyDfzh3BZ2t2cPBwkkubOcu+JyjAj22LXmZgn/aMfW8ZAMFB/syZPIBNc4bz7ssPMHjMrIKOd9Xsz09awMK3BvP9gpdYvGYHBw65Zp+1dAuBAWWJ/Ww0g/p2YPQ7SwE4cCiJJWtj2TJ/JIveHszQ1xZgs9k191V4ifDKgzfRd+oG2o5Yzh3NYqgZEejS5ple9Vi2NY5Oo1Yy8D/f8OpDNzmyJ5yjy+hV3PLySvpM2cDr/Zrh7SU5LabA2Wx2xr69hI9feYwVM4bxxYad/HrkuEub8IoVeGXYffTs2Cjb9P/s3Z5JL/b1SNasSvpn9MUpC/l06kA2zx3BZ2uzZ/90+fcElffjh0UvM+C+9oyzsteuEc6aGUPZMPMF5r0xiKGT5pOWVrC/25qXQrFdRIKAj3BcCRULbMnDdH8aYxoaYxrguELqlZwaiUhlHP+pb6AxpjbQGhggIj2cmr1hjGkI3AZMSy8EIjIQ6Aw0N8bUA27EUcjK5iHfNYndH0dMlTBiIkMpXcqH2zs1ZtWmvS5tVm3ey73dHR/2Wzs0ZPP2nzHGUL9WFJXDHBuJ2tXDuXQ5lctXUgs6Yq527DtC9ahQYqo4st/ZqTErN+5xzb5pD316OL7F3nZzIzZuO4gxhpUb93Bnp8b4li5F1chQqkeFsmPfEc19FY2qh3D4xHmOJl8g1Wbn8x+O0KVRFZc2xkD5so7vNOXLluL42T8B+POKDZvdcRfiMqW8MMbdbe8L1p4DR6kaGUJURAilS/nQo0Mj1n/nuidUpXIwtWtE4CXZi1eLxjXx9/P1VFwXJfkzGrs/jmrO2W9pzOos2Vdv3ktvp+zfWNn9ypTGx8cbgEtX0hAK/kvFVQuFMWawMeacMeYDoBPwsDGm/zUuJwA4m8u4J4BPjDGx1vJOAcOAF3PI8gvwB1DBGjQSGGSMOWeNv2KMedUY8/s15ruqpORzRFYMyngdUTGIpOQUlzbHk1OIrORo4+PjTUC5MpxJuejSZvlXu6hfswq+pQt8pydXSckpRFaqkPE6olKFbNkTT2a2cWQvy5mUi9mnrZh9Ws2dXXgFPxLPZN4QMunsH4RX8HNpM/nzPdzVohqxU+9gzrMdGDl7W8a4RtVD2DihJ1+N78mw/23NKByF7cSpFCqHZa7nlcICOXHKc/32V5Tkz+jx5HNEZMl+PEv2pCzZyztl37HvCG37TqT9A6/w+rDeGYWjoLj7D3eN3Y1L37C7UVZEduG4UiocuDmXdnWB/2UZtt0anlOmX4wxJ63DYOWMMYevkqPYOHAoiXHvLWPBW4OLOooqBu5oHsP8bw/xweqfaFIjlHcfb0m7l77AGNh56DTtRn7BdeEBvP1YSzbsTeByqucOnf1dldTPaJO6MWz6dAQ/HznOk2Nnc3OL6ynjW3CFzt0exRQ3j8l5mHf6oafaQFdgpkgO+6p5M0RE9gE/ABNyaiAiXazzGEdyOh8iIo+LyHYR2X4qOfmaA4SHBZFw8lzG68ST5wgPcz3mXDkskIQTjjZpaTZ+v3CJ4EB/q/1ZHn7hY959+UGqVQm75uX/FeFhgSScyNyhSzxxNlv2iIqZbRzZ/yQ40D/7tCezT6u5s0s6+wcRwZl7EOEV/Eg663rL+b5ta7BsaxwAO347hW8pb0LKuR62+SXpdy5eSqN2ZBCeUCk0kOPJmev5ieQUKoV6rt/+ipL8Ga0cFkRiluyVs2QPz5L9vFP2dDVjKuPv55vtXN5flWuhMMZ0cPPIbe8gt3ltAUKBMBGZYG3Qd1mj9wNNskzSBHA+MPqGMaYucBcwXUTKWIeXLohINWsZX1rnMX4ESueQ4UNjTFNjTNPQsGtfCRrVieZwfDJxiae5kprG52tj6dqmnkubrm1uYP7KrYBj97V10+sQEVLO/0HfZ6fx78G9aNbA8z+82/j6qvx2NJm4hFNcSU1jydpYurWtnyV7Peau+AGApRt20vbGmogI3drWZ8naWC5fSSUu4RS/HU2mSd0YzX0Vuw6fpnql8kSH+lPK24vbm8WwZucxlzYJpy/S5vrKAFwXHoBvKW9Onb9MdKh/xsnrKiH+/CM8gPhTF7MtozDUqx3FkYRTxCc51vMVX+3k5pbZdu6LpZL8GW1UJ5pDztnXxdIlS/YurW9ggXP2Jo7scYmnM05exyed4de4E0SFB2dbxl8hhXWiTEQuGGPKWc9r4/i/F5WMMbYs7cJx7Cn0MsbsEpEQYDUw1hizXERGAxeMMZOt9kuBlcaYaSIyGOgF3GeMOWftsawFxhtjvs4tW+MmTc2m77Ze83ta+90+XnpjCXa7nT49m/Ns/y68+uEKGtaOpmvbely6nMrgMbPY+/MxKgT48eG4fsREhjJlxpe8PXMt1aIyC9TCtwYTFlz+mjP4eOfp7rXZrPl2HyOmLsJmM9zfqzlDH+nKxA++oGGdaLq3q8+ly6kMHDWTPQfjqRDgz/QJ/YmpEgrA5BmrmbPse3y8vZj47F10auW5DUdxyF25X/6uBu9YP4KxfZvi7SXM3fwbby3/kWF31GfX4TOs2XWMmhGBTO7fDH/fUhgM4+bvZOO+JO5uWY0ne9Ql1WbHboepy/awOvbY1ReYgz1v33PN02z84Scmvvc5Nrvhrm43Mej+W3jrv6u5oVYVOra8gT0HjvKvUZ/w+4U/8S3lQ2hweVbMGAZA36ff5VD8Sf748zJBAf5MGNqbNjfWvuYMwf75O2xSHD6j+T2dtO67ffz7TcelvX16NmdIvy689uEKGtSJpmsbR/Z/WdmDAvyYZmVfuGor78xah4+PN14iPPtIV7q3q3/V5WXVrtVN7NyxPcejPoVZKGxA+ml7AUYYY1bk0rYtjkNa5a22bxpj/mONG41roWiC4yqpOjh+c2oo8E/gMnAB+BZHocj1DFx+C0VxkN9CofIvv4WiOMhPoSgO8lsoigMPXXdQ4NwVijz9hEd+GGPyfNrdGLMJx6WtOY0bneX1DqCW06DXrYdSSqlCkJc73ImIPCAiL1uvo0XkpsKPppRSqjjIy3GM94EWQPr/iD4PvFdoiZRSShUreTn01MwY01hEdgIYY86KSLaripRSSv3/lJc9ilQR8cZx4hgRCQP0f/4opdTfRF4KxdvAZ0BFEZmA4zLXiYWaSimlVLFx1UNPxpg5IrID6Ijj0tXbjTE/FXoypZRSxUJeblwUjeOH+JY7DzPGHC3MYEoppYqHvJzMXoHj/ITg+IG/asBBcvjRPqWUUv//5OXQk8sPjli/4FqyflpRKaVUvl3z70FYPy/erBCyKKWUKobyco7iWaeXXkBjILHQEimllCpW8nKOwvnnE9NwnLNYXDhxlFJKFTduC4X1H+3KG2OGeiiPUkqpYibXcxQi4mPdO6KVB/MopZQqZtztUWzFcT5il4gsAxYCGbfZMsYsKeRsSimlioG8nKMoA5wGbibz/1MYQAuFUkr9DbgrFBWtK55+JLNApCuh93BSSil1rdwVCm+gHK4FIp0WCqWU+ptwVyiSjDFjPZbEw7wkx1vDFnv2EnpDXi+vktnfAEc+7HP1RsVU+D3vF3WEfDm5+ImijpBvxpTMz6i7r//u/md2yf1kK6WUKjDuCkVHj6VQSilVbOVaKIwxZzwZRCmlVPF0zT8KqJRS6u9FC4VSSim3tFAopZRySwuFUkopt7RQKKWUcksLhVJKKbe0UCillHJLC4VSSim3tFAopZRySwuFUkopt7RQKKWUcksLhVJKKbe0UCillHJLC4VSSim3tFAopZRyy92tUFUW67fsZ/jUxdjtdh7o1YJnHu7sMv7ylVQGj5nF7gPxVAj0Z/r4/kRHhHAm5SL9X5zOzp/iuK9HMyY931uz59G67/YzfMoibHY7D97WkiH9suceNGoWuw4cJTjQnxkTHyE6IgSAqf/9ktnLtuDt5cWrQ++mY4vrPZp9w/c/8fKbS7DZ7fS9tTlPPtgpS/Y0nho3mz0HHX0+bezDRIWHZIw/dvwM7R54haGPdGNQ35s9mr1jo2heeawt3l7CrLX7eXPxDpfxVULL8f4znQj098XbSxgz8zvW7oijlI8XbwzuQKMaFbEbePHjTXz7Y4LHcm/Ysp+Rby7BZnOs5089lLXPU/nX2NnsPhBPcKA/H47vR3R4CF9vPcD495eRmmqjVClvRv3rdto0remx3OBYX/5tZb//1uY8mS17Gk+Om80e6zM6bdzDRIeHELs/judfmw84bsM69NGudG/XoECzeWyPQkQuuBnXWkS2isgB6/G407jRIpIgIrtEZL+I9Mky7bPWNHtFZLeITBWRUgWd32azM+z1hSx4cxDfzRvJkjU7OHAoyaXN7GVbCCrvx/bFoxh0XwfGvLcUAN/SPgwf0IMxT91R0LHypKRmt9nsPD9pAQvfGsz3C15icQ65Zy3dQmBAWWI/G82gvh0Y/Y4j94FDSSxZG8uW+SNZ9PZghr62AJvN7tHsI6YsZM6UAWycM5zP18Vy8PBxlzZzv9hCYPmybFnwbx6/tz3j31/uMn70O59zc3PPFjdw3N/89QHtuWfMMpr/aw53talJragKLm2e630jn3/zC+2GzOPRyauZPKA9AA93rgtAq6fncseozxnfvzWeuj29zWbnhSkLmTt1IN/MHcGStTs4eNh1fZmz/HsCy/uxddHLDLivPePeWwZASKA/s193/K3e+fcDPDFmlmdCO2UfPnkhn04ZwKZPh/NZDuvLp8u3EFS+LN8v/DcDnNaX2tXD+XL6c6z/3zDmTh3I868tIC3NVqD5ivzQk4hUBj4FBhpjagOtgQEi0sOp2RvGmIbAbcC09EIgIgOBzkBzY0w94EbgJFC2oHPG7o+jWpVQYiJDKV3Khzs6NWHVpr0ubVZt2st9PZoB0Ovmhmza9jPGGPzL+tK8YQ18SxfNDlxJzb5j3xGqR4USU8WR+85OjVm5cU+W3HvoY+W+7eZGbNx2EGMMKzfu4c5OjfEtXYqqkaFUjwplx74jHsu+86c4YqqEUdXq89s6NubLza59vnrzj/TufhMAPds3YPMOR5+nv6/o8BBqVavssczpmlxXiUPHzxF34ndS0+ws2fwz3W+q7trIQHm/0gAE+Ply/OxFAGpFBbN5zzEATqX8ScrFyzT6RyWP5Has52GZ6/ktjVm9KWuf7+Veq89v7dCQzdsdfV6vVhSVwwIBx4b30uVULl9J9UhugJ1W9vT15fZbsq8vX27+kd7drPWlQwO+sbL7lSmNj483AJeupBVKYS7yQgE8AXxijIkFMMacAoYBL2ZtaIz5BfgDSP96MxIYZIw5Z42/Yox51Rjze0GHTDp5jshKmd+qIioGkZR8zrVNcgoRFYMA8PHxJqBcWc6kXCzoKNespGZPSk5xzV2pAknJKS5tEk9mtnHOnW3aitmnLUzHk1OItPoTILxiEMezLP948jkiKjpl9y/DmZSLXPzjMu/NXs9zj3T1WF5n4SH+JJzKPACQePoC4SHlXNq8Ou8HererxY/T+7Pg5VsZ9uFGAH48fIquN1XD20uIrhhAwxoViQx1nbawHE8+l63Ps/7NjyenEFkpcz0vX65MtvX8i692Ua9WFXxLF/iBiVwlJacQUckpe1j27EnJ54hwWtfL+2dmj913hLb3v0KHB19l0rDeGYWjoBSHcxR1gf9lGbbdGu5CRBoDvxhjTopIAFDOGHM4LwuxDmc9DhAVHf3XEitViCbPWMXj97bH38+3qKPk6q42Nfl0wwHeW7qTG2tV5oMhnWn55Bxmr9tPzagKfDXlXuKTz7P1QBI2uynquHl24FASY99fxoI3Bxd1lGvSuG4Mm+YM5+cjx3lq3Bxubn49ZXwLrtAVhz2KvBgiIvuAH4AJOTUQkS7WeYwjItIy63hjzIfGmKbGmKahoWHXHCC8YhAJJ85mvE48eY7wsCDXNmGBJJ50fFNPS7Px+4U/CQ70v+ZlFbSSmj08LNA194mzhFuHB9JFVMxs45w727Qns09bmCqHBZJwMnOvLenkuYxDG5ltgkg86ZT94iWCA/2J3RfHuPeXceNdY/howUbenrmWGYs2eSx70umLLnsBESHlSDrteorxgU7X8/m3vwCw7eBxypTyJiSgLDa7YeT0b2g7ZB73T1xBYDlffks8iydUDgvK1udZ/+aVwwJJOJG5np+/cCljPU88eZZ+L37Mu/9+kGpVrn0b8VeEhwWSeMIpe3L27OFhQSQ6revnL17K9hmtGVMZ/7K+2c7l/VUeLxQiMsHaoO+yBu0HmmRp1gTY5/T6DWNMXeAuYLqIlLEOL10QkWoAxpgvrfMYPwKlCzp3ozrRHIpPJi7xFFdS0/hs7Q66ta3n0qZrm3rMW/EDAMs27KJN05qIp87kuVFSsze+viq/HU0mLsGRe8naWLq1re/Spmubesy1ci/dsJO2Nzpyd2tbnyVrY7l8JZW4hFP8djSZJnVjPJa9Ye1oDh9L5mjiaa6kprF0fSxdWt/g0qZL6xtYsHIrAF98vZvWTa5DRFj6n6fZtngU2xaP4rHe7XjqoU48cndbj2WP/eUENcKDiK4YQCkfL+5sU5NVW1133BOSL9C2fhUAalapgG9pb06l/EnZ0j74+ToOVLRvEEWazc7BeM8Uisz13NHnn62LpUsb1/W8S+sbmG/1+fKvdmX0ecr5P+j73DReGtyLZg2q5zT7QtWwTjSHjmVm/3xdLJ2zrC+d29zAglXW+vLVblpZ2eMST2ecvI5POsOvR08QFR5coPkk/eRZYRORC8aYbAcrRSQcx55CL2PMLhEJAVYDY40xy0VkNHDBGDPZar8UWGmMmSYig4FewH3GmHPi2LKtBcYbY77OLUvjJk3NN1u2XfN7WPvtPka+sRib3dD31uY8178Lr0xbQcM60XRrW49Ll1MZNHome38+RlCAHx+P709MZCgADW8fxfmLl0hNTSOgnB+L3h5M7erh15whv4o6u5dX/orOuc8Y1gAAGbpJREFUmm/3MWLqImw2w/29mjP0ka5M/OALGtaJpnu7+ly6nMrAUTMdl5gG+DN9Qn9iqjhyT56xmjnLvsfH24uJz95Fp1bZjmbmyaUr+buCZP13+3j57c+w2ezc17M5zzzcmUkfraRB7Si6tHH0+ZPjZvOj1ecfjHmYqlafp5s8fRX+ZX3zfXls+D3v52u6Tk2qMvHRNnh7eTFn/X6mLNzO8L7N2PXrSVZtPUytqAq89cTN+JcpjTGGUf/7lq92xRNVsTyLR9+G3W5IOnORp95ZT3zy+Wte/snFT+Qr97rv9vFS+iXJPZszpF8X/q+9e4+Oor7/P/58J0HklgAmKFcDtipSAQEVLyCKKCgFRatfL7WoPSrevlIBv0KL0Soogteq9V5Ba0XFn1pBBG2BKiohchEELwjYgBIUiCiXsHn//tgJ5DokcZPNyutxDsfszGd2XrtO9rU7s5m587Hodt4/eM6vuXUKSz/9L81SG/Lon4eS2Tqde56eyQOTZ9G+7Z5PElPvu5qM5k2qnKGwmq+ps99bxtj7o9vLBQN7csPQ07jr8el0Lba9XHvbnu3l0dui28uLMxbw4LOzqZeSTJIZf7j0dAac1HnvKyylzwnH8lFOdrm/qHEvimBeb2AS0AQw4D53fySYl0XJouhO9FtSHQEHRgC/B3YAW4F3iRZFhUcuq1sUUn3VLYq6oLpFURdUtyjirbpFURdUtyjiLawoau1gdkUlEcybS/SrreXNyyp1eyFwWLFJdwf/RESkBiTKwWwREYkTFYWIiIRSUYiISCgVhYiIhFJRiIhIKBWFiIiEUlGIiEgoFYWIiIRSUYiISCgVhYiIhFJRiIhIKBWFiIiEUlGIiEgoFYWIiIRSUYiISCgVhYiIhFJRiIhIqFq7wl1dYiT2pTmldtWvl7jvpxL1kqItThkT7wjV9t2ccfGOUC0W8pKYuL8BIiJSK1QUIiISSkUhIiKhVBQiIhJKRSEiIqFUFCIiEkpFISIioVQUIiISSkUhIiKhVBQiIhJKRSEiIqFUFCIiEkpFISIioVQUIiISSkUhIiKhVBQiIhJKRSEiIqFUFCIiEmqfvBRqdc1+bzk3T3qJSGEhvx18PMOHnlZi/o6dBQy7ZQqLVqyleVojnhp3Ge1aHQDAPU/P5NnX5pOclMSdI86l73FHKPvPODfA7PnLGT3p5SD7cdzwu3KyZ01h8YqvaJbWiKfuuJR2rQ7gu80/MPTmJ/lo+RouGHgsE0aeV6u5Ad6Zv5wx900jEink4kHHcf0l/cpkv/a2Z1m84iuapzXisduH0q7lAeQsW8ONd/0DAHdn5OUDOLNPl1rL3feYQxl//UCSk5KY8sYC7ntuTon5bQ9syoP/dw7pTRuxKX8bV97+Auvy8gFo0yKN+286h9Yt0nB3zhv1N776enOtZa/L20utfKIws4iZLTKzxWaWY2bHVzAuy8xGlDP9LDNbYmafmNlSMzur1PwRZrYiWMcCM7sk1o8hEilk5ISpvHj/1bw/9Y+8/NZCVqxaX2LMlFfnk5bagJxXshh24clkPfgqACtWrWfarBzmvzCGlx64mhF3TSUSKYx1xJ9d9kTNXZR91IQXmXr/MOa/MIaXZ5bN/uxr82napCELp93CsAtOJusv0ez166cw+sozue36s2stb3GRSCE3TXqR5++5iv88P5ppsxay8suS2Z97/X3SmjTkw5fGcuX/9OHPD70GwOGHtGTWUyP41+SbeOHeYYyc8AK7dkVqJXdSknH38EH8ZuTT9LzkXs7p24XDDm5RYsxtV5/BP2Z+xImXPsCEZ95m7BX9d897ZMx5PPj8XHr+9l5OvfJhNm76oVZyQ93fXmpr19M2d+/q7l2Am4HxlV3QzLoAE4HB7t4RGARMNLPOwfyrgH7AMe7eFegLhFwmvHoWLltNh7bpZLZJZ796KQzp143pc5aUGDNj7hIuOPNYAAafchRzFqzE3Zk+ZwlD+nWj/n71OLh1Oh3aprNw2epYR/zZZU/U3NHsa2jfJp3M1kH207ozY+7SEmOmz1nK/+zO3pW5Cz7F3WnUoD49ux5C/frx+cCfs3wN7dtk7M5+9qndeLNU9jfnLeX8M44B4Ncnd2VedjR7w/33IyUlGYDtO3dRA7+KFeresS2rcr9lzfpNFOyKMO3txZxxYscSYw7LbMG8nC8AmJezigHB/MMObkFKchL/zv4cgB+27WTbjoJay17Xt5d4HKNIBTZVYfwIYJy7fwkQ/Hc8MDKYPxoY5u75wfx8d38mhnkBWJ+3hdYHNtt9u9WBzVift6XEmHUb9oxJSUkmtXEDvtvyQ9llW5RdtiYlavZEzQ2wPm9zqfU3ZX3e5lJjttD6wKZAyezx9nXeZlq3aLr7dssWTcs8d1+Xyt6k8f67sy9ctppeF47jpIvHc/eo83YXR01rmZ5K7oY9Odfl5dMyI63EmGWfr2dg704ADOzdidRG+9MstSGHtE1ny9btTL79IuY8cR23DRtAUlLtlVxd315qqygaBLuFVgBPAH+uwrKdgIWlpmUDncwsFWji7qtilFNEfqLunTKZ9/fRvPXUCB6YPIvttfjOfG/+9PB0TujanjlPXMcJXduTu2ELkcJCUpKTOK5zJn96aDqnXPkQB7dqzoUDusc7bp1R27ueDgf6A5PNrPbqGjCzK8ws28yy8zbmVXn5lhlp5H6z54PQum82lXm30qrFnjG7dkXI37qN5mmNyi67oeyyNSlRsydqboCWGU1LrX8zLTOalhqTRu430XeNxbPH20EZTcndsOfd7PoNm8s8dweVyv791u1lsh+aeRCNGtYvs6+9pqzfmE/rFntytspILftJ6NvvueSPz3HS7x/k9sffAiB/63bW5W1h6efrWLN+E5FIIdPnLafLoa1qJTfU/e2l1nc9uft8IB3IMLM7gk8ai0IWWQ6UrvbuwLJgd9NWM+tQifU+5u493L1HRnpGlXN3O+Jgvlibx5rcjews2MW0WTkM6N25xJj+vY7k+Tc+AODVdz6i99GHYmYM6N2ZabNy2LGzgDW5G/libR7dO2VWOUN1JWr2RM0dzd6OVV8Vy/7WQvr3OrLEmAG9j+Qfu7MvolePaPZ4O6pjkH3dt+ws2MUrs3M4vVT200/8FS9M/xCA1/+1iBO7/xIzY826b3cfvP5q/Xd8tuYb2rZsXiu5c1b8l0PapNOuZTPqpSQzpG8XZrz7SYkxzdMa7n6Oh1/Uh+emZ+9eNq1xAw4IXnh7devAytUbaiU31P3txdy95ldittXdGwc/Hw78BzjQ3SOlxmUBW919YrFpXYEXgX7uvtrMMoHZwLnuvsjMrgZ+DZzv7vlm1hgY4u6TK8rTvXsPf/eD7Co/jrfeXcboe14iEnEuGtSTEZf1Z9xf/0nXju0446TObN9RwFW3TGbJyq9oltqIJ++4lMw26QBMfOpNnnvtfVKSkxj3h3Pod0KnKq//p0jU7HUhd3V/R2a9u4zR97xMpNC56Nc9ufGy0xn36Bsc1bEdA3ofuTv70k//S7PUhjxxx6Vkto5m7zL4Fr7/YTsFBbtIbdKQlx+4msM7tKxyhl2R6mWf/d4y/njfNCKFhVw4sCfDh57OnY+9QdeO7ejfK5r9mlun7M7+6J+Hktk6nakzPuTBKbNJSUkmyYwbL+vPGSd13uv6Smtxyphq5e7X8zDGXTeQ5CTjuenZTJryb26+7FQWrcxlxrufMOikXzH2ytNxh/cWf8nIe19lZ0H0ZahPj19w+zVnYGYsWpnLDXe/QkE1vrH13Zxx1coe7+3lhJ5Hk7Mwu9zmqa2iiABFh/ANGO3ub5QzLgu4AdhaNM3d25jZEOBWoB5QANzi7tOCZYzoge3Lg3kFwCR3f7aiPNUtCtk31cbvSE2pblHEW3WLoi6oblHEW1hR1Mr379y9Ul97cPcsIKuc6dOAaRUs48CE4J+IiMSYTuEhIiKhVBQiIhJKRSEiIqFUFCIiEkpFISIioVQUIiISSkUhIiKhVBQiIhJKRSEiIqFUFCIiEkpFISIioVQUIiISSkUhIiKhVBQiIhJKRSEiIqFUFCIiEkpFISIioVQUIiISqlYuhSqSyKKXZU9M9VISM/umuePjHaHamh19bbwjVMuOlWsrnKdPFCIiEkpFISIioVQUIiISSkUhIiKhVBQiIhJKRSEiIqFUFCIiEkpFISIioVQUIiISSkUhIiKhVBQiIhJKRSEiIqFUFCIiEkpFISIioVQUIiISSkUhIiKhVBQiIhJKRSEiIqF0KdQqmP3ecm6e9BKRwkJ+O/h4hg89rcT8HTsLGHbLFBatWEvztEY8Ne4y2rU6AIB7np7Js6/NJzkpiTtHnEvf445Q9p9xbmXX9lJVfY/ryPgbzyU5KYkpr77Hfc/MKjG/7UHNeHDsxaQ3bcym/B+5cuwzrNuwGYCsawdz2omdALj7yTd5ZVZOTLPF5ROFmW2tYHqWmY0oZ/pZZrbEzD4xs6Vmdlap+SPMbIWZLTKzBWZ2SawzRyKFjJwwlRfvv5r3p/6Rl99ayIpV60uMmfLqfNJSG5DzShbDLjyZrAdfBWDFqvVMm5XD/BfG8NIDVzPirqlEIoWxjvizy56ouZVd20tVJSUZd486j9/878P0PO92zjmtO4e1P6jEmNv+92z+8caHnHjheCY8MYOx1wwC4LQTOtH58Lb0uuhOTh06kWsv7kuTRvvHNl9M760GmFkXYCIw2N07AoOAiWbWOZh/FdAPOMbduwJ9gZhfUX7hstV0aJtOZpt09quXwpB+3Zg+Z0mJMTPmLuGCM48FYPApRzFnwUrcnelzljCkXzfq71ePg1un06FtOguXrY51xJ9d9kTNrezaXqqqe6dMVn21kTW531KwK8K0WTmccVLnEmMO69CSedkrAZiX/SkDeh8Znd7+IN776HMikUJ+3L6TZZ/l0ve4jjHNV+eLAhgBjHP3LwGC/44HRgbzRwPD3D0/mJ/v7s/EOsT6vC20PrDZ7tutDmzG+rwtJcas27BnTEpKMqmNG/Ddlh/KLtui7LI1KVGzJ2puUPbdy2p7qZSWGWnkfrNpT85vNtEyI63EmGWf5jLw5K4ADDy5C6mNG9AsrREff5bLqcd1pEH9ejRPa0SvHoeWeCyxkAjHKDoR/URRXDZwjZmlAk3cfVXtxxIRqT1/uv8VJoz6DRcOPJb3Pvqc3G82EYkU8q8PVtDtiIOZ+dSNbNy0lQVLvyRSGNvdZonwiSImzOwKM8s2s+y8jXlVXr4yjd+qxZ4xu3ZFyN+6jeZpjcouu6HssjUpUbMnam5Q9t3LanuplMp8Gvp64xYuGfUEJ118F7c//DoA+Vu3ATDp6Zn0vuhOhlz7FwzjizUbYpovrkVhZncEB6AXhQxbDnQvNa07sCzY3bTVzDrsbV3u/pi793D3HhnpGVXO2u2Ig/libR5rcjeys2AX02blMKB3yX2I/XsdyfNvfADAq+98RO+jD8XMGNC7M9Nm5bBjZwFrcjfyxdo8unfKrHKG6krU7ImaW9m1vVRVzvI1HNIug3atDqBeSjJD+nVjxtySx1eapzXCLHr4dfjQ03nu9feB6IHwZmmNAOj0i1Z0+mUr3vlgRUzzxXXXk7uPAcbsZdhE4EUze8fdV5tZJtHjEucG88cDD5nZ+e6eb2aNgSHuPjmWWVNSkpkw6jzOuf4hIhHnokE96XhIS8b99Z907diOM07qzG8HH89Vt0ym29lZNEttxJN3XApAx0NactapR9HzvDtISU7i7lHnkZxcex2dqNkTNbeya3upqkikkFETpvLyA9eQnGw899r7rFj1NTdfeSaLPlnLjLlLObH7Lxl7zSDc4b2PPmfkhKkA1EtJZvpjNwDw/Q/buWLsMzH/xpa5e0zvsFIrNdvq7o3LmZ4F3ADs/vqsu7cxsyHArUA9oAC4xd2nBcsY0QPblwfzCoBJ7v5sRevv3r2Hv/tBduwekIhIoNnR18Y7QrXsWDmVwh83lPuN0bgURbypKESkpvwci2KfOZgtIiLVo6IQEZFQKgoREQmlohARkVAqChERCaWiEBGRUCoKEREJpaIQEZFQKgoREQmlohARkVAqChERCaWiEBGRUCoKEREJpaIQEZFQKgoREQmlohARkVAqChERCaWiEBGRUPvkpVDNLA9YU4OrSAc21uD915REzQ2Jmz1Rc4Oyx0NN5j7Y3TPKm7FPFkVNM7Nsd+8R7xxVlai5IXGzJ2puUPZ4iFdu7XoSEZFQKgoREQmloqgZj8U7QDUlam5I3OyJmhuUPR7iklvHKEREJJQ+UYiISCgVRYyYWRsze9XMPjOzL8zsfjPbL05ZIma2yMwWm1mOmR1fwbgsM8sNxn5sZoPKmV70r6mZ9TGzLcHtFWY2sQYfw9ZEyVzZ5zsYe6KZfRhkWWFmV1Tw2Jab2QWllv1DsMzSYF33mFm9GOQv97muq3mL3X9VtvMR5Uw/y8yWmNknQcazSs0fEeRfZGYLzOySWGUvto6w7bzuZHZ3/fuJ/wADPgQuDW4nA08Cd8cpz9ZiP58OzKlgXBYwIvi5I9HvZycVn15qfB/gn8HPDYAVwAk1/RjqeuYqPN8HAWuBbsHtdGAhcGY5j+2XQD5QL7h9FfAm0DS4vR/wf0BqDT7XdTLvT9nOi03rAnwOtA9utw9udy6Wf2ZRXiAV+F08tvO6kFmfKGLjFGC7uz8N4O4RYDhwmZk1jGuy6MayaW+D3P0TYBfRF4O9cvdtwCKg9U9K9xPU0cxhz/c1wN/cPSfIsxEYRfQFtAR3/wz4EWgWTBoDDHP3zcH8ne5+p7vnxzh/ouat1HZezAhgnLt/CRD8dzwwMpg/mmj+/GB+vrs/E8O81RG3zCmxuBOhE9F3Wru5e76ZrQV+ASyp5TwNzGwRsD/QkmiRhTKzY4FCIC+YNNzMLg5+3uTuJ5ca34zou8i5MUtdRXUoc2Wf705A6V/c7GB6CWbWDfjM3TeYWSrQuOgFohbV9bxV3s6L6QSU3g2ZDVwT5G/i7qtiEzNm4pZZnyh+nra5e1d3PxzoD0w2M6tg7PDgl20icL4Hn1mBe4P76FrqBbeXmS0GcoGZ7v51jT2KitW1zFV5vvdmuJktAz4A7ihvgJmdHuyDXh12PKSWxDNvLJ93CaGiiI3lQPfiE4KGb0d0H2LcuPt8ortmMszsjqIDvcWGFL249nL3eZW4y3nu3oXou5vLzaxrTeQukmiZ9/J8l9lOgtvLit2+1907AecAT5rZ/sGuhK1m1j5Yx0x37wp8THTff0wkWt7iKrGdl1bhYyuWv0NNZC1PXc+sooiNt4GGRd8wMLNkYBLR/bs/xjOYmR1O9OD6t+4+pugd90+932C3wp3ATT/1vvaynoTKvJfn+yFgaFFRmdkBwF3AhHKyvkZ0t8LvgknjgUfMrGmwrBHd5RIziZa3uGps5xOBm80sM1g+k+g+/knB/PHAQ8EbPsyscU1866lIXc+sYxQx4O5uZmcDD5vZn4gW8HSi/xPjoUGxdyZG9JsPkSreR/H9/QBnlTPmr8AIM8t099XVyBlr8cpcqefb3dcH+R43sybB2Pvc/fUK7vc24O9m9jjwCNAI+MDMdgBbgXeBj2L0GMpIgLxV2c7/aGY3FN1w9zZmdhPwukW/slsAjHL3ovt7BGgMLDCzgmD+pNJ3WsPqTGb9ZbaIiITSricREQmlohARkVAqChERCaWiEBGRUCoKEREJpaKQfYrtOePox2b24k85F5eZ/c3Mzg1+fsLMjggZ26c6f5Uc/DVzmXNZVTS91JgKzwpbwfhyz1gqoqKQfU3RaR9+BewkesbN3cysWn9b5O6/d/flIUP6APE+3YZItagoZF82D/hF8G5/npm9Biw3s2Qzu9ui5/NfYmZXQvSvi83sL2a20sxmAy2K7sjM/m1mPYKf+1v0+giLzezt4C9oryI4R5WZ9TKzDDN7OVjHAjM7IVj2ADN7y8yWmdkTRP+QLJSZ/T8zWxgsc0WpefcG0982s4xg2iFm9mawzLzgr5pFKqS/zJZ9UvDJYQDRayYAdAN+5e5fBi+2W9z9aDOrD7xrZm8BRwGHAUcABxI9985Tpe43A3gc6B3cV3N3/87M/kr02gMTg3F/J3qepP+YWTui1xHoCNwC/MfdbzOzM4HLK/FwLgvW0YDoX+W+7O7fEv3L6Gx3H25mY4P7vpbodZevcvfPLHoG3oep2plXZR+jopB9TfHTPswjeoGp44EPi50W+zSgc9HxByCN6OnJewPPB6eJWGdm75Rz/z2BucWuGfBdBTlOBY6wPSc7TTWzxsE6hgTLvmFmlbnGwvXBKWQA2gZZvyV6CvYXgunPAtOCdRwPvFhs3fUrsQ7Zh6koZF+zrfSJ14IXzB+KTwKuc/eZpcadEcMcSUBPd99eTpZKM7M+REvnOHf/0cz+TcUn3/NgvZtjcZJF2XfoGIVIWTOBYcGJ1zCzQ82sEdELHp0fHMNoCZxczrLvA70tOMW2mTUPpn8PNCk27i3guqIbtufU53OBC4NpA9hzxbiKpBG9SNOPwbGGnsXmJQFFn4ouJLpLKx/40sx+E6zDzKzLXtYh+zgVhUhZTxA9/pBjZh8DjxL99P0K8FkwbzIwv/SC7p4HXEF0N89i9uz6eR04u+hgNnA90CM4WL6cPd++upVo0Swjugtq7V6yvgmkmNknRE+h/n6xeT8AxwSP4RSiZ3cFuIjodTkWE722xOBKPCeyD9PZY0VEJJQ+UYiISCgVhYiIhFJRiIhIKBWFiIiEUlGIiEgoFYWIiIRSUYiISCgVhYiIhPr/0rL2XEOUDNoAAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@ INSPECTING TOKEN SEQUENCES: \n",
        "def get_samples(df):                                                    # Defining function.\n",
        "    for _, row in df.iterrows():\n",
        "        labels, preds, tokens, losses = [], [], [], []                  # Initialization. \n",
        "        for i, mask in enumerate(row[\"attention_mask\"]):\n",
        "            if i not in {0, len(row[\"attention_mask\"])}:\n",
        "                labels.append(row[\"labels\"][i])                         # Adding labels. \n",
        "                preds.append(row[\"predicted_label\"][i])                 # Adding predicted labels.\n",
        "                tokens.append(row[\"input_tokens\"][i])                   # Adding tokens.\n",
        "                losses.append(f\"{row['loss'][i]:.2f}\")                  # Adding loss.\n",
        "        df_tmp = pd.DataFrame({\"tokens\": tokens, \"labels\": labels,\n",
        "                               \"preds\": preds, \"losses\": losses}).T      # Creating dataframe.\n",
        "        yield df_tmp            \n",
        "\n",
        "#@ IMPLEMENTATION:\n",
        "df[\"total_loss\"] = df[\"loss\"].apply(sum)                                # Adding loss.\n",
        "df_tmp = df.sort_values(by=\"total_loss\", ascending=False).head(5)       # Initializing dataframe.\n",
        "for sample in get_samples(df_tmp):\n",
        "    display(sample)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 835
        },
        "id": "iIdWYA2K7lTB",
        "outputId": "e657bea5-9272-4aaa-d201-41deafa5a8c8"
      },
      "execution_count": 45,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "          0     1      2      3     4     5      6      7      8      9   \\\n",
              "tokens    ▁'   ▁''     ▁Τ      Κ   ▁''    ▁'     ▁'    ▁''     ▁T    ▁''   \n",
              "labels     O     O      O    IGN     O     O  B-LOC  I-LOC  I-LOC  I-LOC   \n",
              "preds      O     O  B-ORG  B-ORG     O     O      O      O      O      O   \n",
              "losses  0.00  0.00   3.34   0.00  0.00  0.00  10.17   9.88   7.95   8.09   \n",
              "\n",
              "           10    11     12     13    14     15     16    17    18  \n",
              "tokens     ▁'    ri    ▁''     ▁'     k    ▁''     ▁'   ala  </s>  \n",
              "labels  I-LOC   IGN  I-LOC  I-LOC   IGN  I-LOC  I-LOC   IGN   IGN  \n",
              "preds       O     O      O      O     O      O      O     O     O  \n",
              "losses   8.39  0.00   8.69   8.98  0.00   9.30   9.46  0.00  0.00  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-4b7f5b7e-1d20-438b-a44c-c72e94dba12a\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "      <th>10</th>\n",
              "      <th>11</th>\n",
              "      <th>12</th>\n",
              "      <th>13</th>\n",
              "      <th>14</th>\n",
              "      <th>15</th>\n",
              "      <th>16</th>\n",
              "      <th>17</th>\n",
              "      <th>18</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>tokens</th>\n",
              "      <td>▁'</td>\n",
              "      <td>▁''</td>\n",
              "      <td>▁Τ</td>\n",
              "      <td>Κ</td>\n",
              "      <td>▁''</td>\n",
              "      <td>▁'</td>\n",
              "      <td>▁'</td>\n",
              "      <td>▁''</td>\n",
              "      <td>▁T</td>\n",
              "      <td>▁''</td>\n",
              "      <td>▁'</td>\n",
              "      <td>ri</td>\n",
              "      <td>▁''</td>\n",
              "      <td>▁'</td>\n",
              "      <td>k</td>\n",
              "      <td>▁''</td>\n",
              "      <td>▁'</td>\n",
              "      <td>ala</td>\n",
              "      <td>&lt;/s&gt;</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>labels</th>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "      <td>IGN</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "      <td>B-LOC</td>\n",
              "      <td>I-LOC</td>\n",
              "      <td>I-LOC</td>\n",
              "      <td>I-LOC</td>\n",
              "      <td>I-LOC</td>\n",
              "      <td>IGN</td>\n",
              "      <td>I-LOC</td>\n",
              "      <td>I-LOC</td>\n",
              "      <td>IGN</td>\n",
              "      <td>I-LOC</td>\n",
              "      <td>I-LOC</td>\n",
              "      <td>IGN</td>\n",
              "      <td>IGN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>preds</th>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "      <td>B-ORG</td>\n",
              "      <td>B-ORG</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>losses</th>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>3.34</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>10.17</td>\n",
              "      <td>9.88</td>\n",
              "      <td>7.95</td>\n",
              "      <td>8.09</td>\n",
              "      <td>8.39</td>\n",
              "      <td>0.00</td>\n",
              "      <td>8.69</td>\n",
              "      <td>8.98</td>\n",
              "      <td>0.00</td>\n",
              "      <td>9.30</td>\n",
              "      <td>9.46</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-4b7f5b7e-1d20-438b-a44c-c72e94dba12a')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-4b7f5b7e-1d20-438b-a44c-c72e94dba12a button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-4b7f5b7e-1d20-438b-a44c-c72e94dba12a');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "            0      1      2      3      4      5      6      7      8      9   \\\n",
              "tokens  ▁Schwa      n   dorf     ▁–  ▁Cham     ▁–   ▁Fur     th    ▁im  ▁Wald   \n",
              "labels   B-ORG    IGN    IGN  I-ORG  I-ORG  I-ORG  I-ORG    IGN  I-ORG  I-ORG   \n",
              "preds    B-LOC  B-LOC  I-LOC      O  B-LOC      O  B-LOC  I-LOC  I-LOC  I-LOC   \n",
              "losses    4.73   0.00   0.00   4.40   4.67   5.20   5.42   0.00   3.72   4.19   \n",
              "\n",
              "        ...     17     18     19     20          21        22     23     24  \\\n",
              "tokens  ...   ▁als  ▁Teil   ▁der  ▁Fern  verbindung  ▁München     ▁-  ▁Prag   \n",
              "labels  ...  I-ORG  I-ORG  I-ORG  I-ORG         IGN     I-ORG  I-ORG  I-ORG   \n",
              "preds   ...  I-LOC      O      O      O           O     B-LOC  I-LOC  B-LOC   \n",
              "losses  ...   2.22   3.81   5.11   4.21        0.00      1.77   2.02   3.28   \n",
              "\n",
              "           25    26  \n",
              "tokens     ▁)  </s>  \n",
              "labels  I-ORG   IGN  \n",
              "preds   I-LOC     O  \n",
              "losses   2.75  0.00  \n",
              "\n",
              "[4 rows x 27 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-f2380c59-1082-4747-a077-877278b5c3d1\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "      <th>...</th>\n",
              "      <th>17</th>\n",
              "      <th>18</th>\n",
              "      <th>19</th>\n",
              "      <th>20</th>\n",
              "      <th>21</th>\n",
              "      <th>22</th>\n",
              "      <th>23</th>\n",
              "      <th>24</th>\n",
              "      <th>25</th>\n",
              "      <th>26</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>tokens</th>\n",
              "      <td>▁Schwa</td>\n",
              "      <td>n</td>\n",
              "      <td>dorf</td>\n",
              "      <td>▁–</td>\n",
              "      <td>▁Cham</td>\n",
              "      <td>▁–</td>\n",
              "      <td>▁Fur</td>\n",
              "      <td>th</td>\n",
              "      <td>▁im</td>\n",
              "      <td>▁Wald</td>\n",
              "      <td>...</td>\n",
              "      <td>▁als</td>\n",
              "      <td>▁Teil</td>\n",
              "      <td>▁der</td>\n",
              "      <td>▁Fern</td>\n",
              "      <td>verbindung</td>\n",
              "      <td>▁München</td>\n",
              "      <td>▁-</td>\n",
              "      <td>▁Prag</td>\n",
              "      <td>▁)</td>\n",
              "      <td>&lt;/s&gt;</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>labels</th>\n",
              "      <td>B-ORG</td>\n",
              "      <td>IGN</td>\n",
              "      <td>IGN</td>\n",
              "      <td>I-ORG</td>\n",
              "      <td>I-ORG</td>\n",
              "      <td>I-ORG</td>\n",
              "      <td>I-ORG</td>\n",
              "      <td>IGN</td>\n",
              "      <td>I-ORG</td>\n",
              "      <td>I-ORG</td>\n",
              "      <td>...</td>\n",
              "      <td>I-ORG</td>\n",
              "      <td>I-ORG</td>\n",
              "      <td>I-ORG</td>\n",
              "      <td>I-ORG</td>\n",
              "      <td>IGN</td>\n",
              "      <td>I-ORG</td>\n",
              "      <td>I-ORG</td>\n",
              "      <td>I-ORG</td>\n",
              "      <td>I-ORG</td>\n",
              "      <td>IGN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>preds</th>\n",
              "      <td>B-LOC</td>\n",
              "      <td>B-LOC</td>\n",
              "      <td>I-LOC</td>\n",
              "      <td>O</td>\n",
              "      <td>B-LOC</td>\n",
              "      <td>O</td>\n",
              "      <td>B-LOC</td>\n",
              "      <td>I-LOC</td>\n",
              "      <td>I-LOC</td>\n",
              "      <td>I-LOC</td>\n",
              "      <td>...</td>\n",
              "      <td>I-LOC</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "      <td>B-LOC</td>\n",
              "      <td>I-LOC</td>\n",
              "      <td>B-LOC</td>\n",
              "      <td>I-LOC</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>losses</th>\n",
              "      <td>4.73</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>4.40</td>\n",
              "      <td>4.67</td>\n",
              "      <td>5.20</td>\n",
              "      <td>5.42</td>\n",
              "      <td>0.00</td>\n",
              "      <td>3.72</td>\n",
              "      <td>4.19</td>\n",
              "      <td>...</td>\n",
              "      <td>2.22</td>\n",
              "      <td>3.81</td>\n",
              "      <td>5.11</td>\n",
              "      <td>4.21</td>\n",
              "      <td>0.00</td>\n",
              "      <td>1.77</td>\n",
              "      <td>2.02</td>\n",
              "      <td>3.28</td>\n",
              "      <td>2.75</td>\n",
              "      <td>0.00</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>4 rows × 27 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-f2380c59-1082-4747-a077-877278b5c3d1')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-f2380c59-1082-4747-a077-877278b5c3d1 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-f2380c59-1082-4747-a077-877278b5c3d1');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "             0         1       2            3         4      5        6   \\\n",
              "tokens  ▁United  ▁Nations  ▁Multi  dimensional  ▁Integra    ted  ▁Stabil   \n",
              "labels    B-PER     I-PER   I-PER          IGN     I-PER    IGN    I-PER   \n",
              "preds     B-ORG     I-ORG   I-ORG        I-ORG     I-ORG  I-ORG    I-ORG   \n",
              "losses     6.07      6.35    6.53         0.00      6.37   0.00     6.38   \n",
              "\n",
              "             7         8      9      10        11        12         13     14  \n",
              "tokens  ization  ▁Mission    ▁in   ▁the  ▁Central  ▁African  ▁Republic   </s>  \n",
              "labels      IGN     I-PER  I-PER  I-PER     I-PER     I-PER      I-PER    IGN  \n",
              "preds     I-ORG     I-ORG  I-ORG  I-ORG     I-ORG     I-ORG      I-ORG  I-ORG  \n",
              "losses     0.00      6.15   6.13   6.56      6.67      6.32       6.30   0.00  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-140ebc34-0ceb-4a77-bd67-4c193c91c7a2\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "      <th>10</th>\n",
              "      <th>11</th>\n",
              "      <th>12</th>\n",
              "      <th>13</th>\n",
              "      <th>14</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>tokens</th>\n",
              "      <td>▁United</td>\n",
              "      <td>▁Nations</td>\n",
              "      <td>▁Multi</td>\n",
              "      <td>dimensional</td>\n",
              "      <td>▁Integra</td>\n",
              "      <td>ted</td>\n",
              "      <td>▁Stabil</td>\n",
              "      <td>ization</td>\n",
              "      <td>▁Mission</td>\n",
              "      <td>▁in</td>\n",
              "      <td>▁the</td>\n",
              "      <td>▁Central</td>\n",
              "      <td>▁African</td>\n",
              "      <td>▁Republic</td>\n",
              "      <td>&lt;/s&gt;</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>labels</th>\n",
              "      <td>B-PER</td>\n",
              "      <td>I-PER</td>\n",
              "      <td>I-PER</td>\n",
              "      <td>IGN</td>\n",
              "      <td>I-PER</td>\n",
              "      <td>IGN</td>\n",
              "      <td>I-PER</td>\n",
              "      <td>IGN</td>\n",
              "      <td>I-PER</td>\n",
              "      <td>I-PER</td>\n",
              "      <td>I-PER</td>\n",
              "      <td>I-PER</td>\n",
              "      <td>I-PER</td>\n",
              "      <td>I-PER</td>\n",
              "      <td>IGN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>preds</th>\n",
              "      <td>B-ORG</td>\n",
              "      <td>I-ORG</td>\n",
              "      <td>I-ORG</td>\n",
              "      <td>I-ORG</td>\n",
              "      <td>I-ORG</td>\n",
              "      <td>I-ORG</td>\n",
              "      <td>I-ORG</td>\n",
              "      <td>I-ORG</td>\n",
              "      <td>I-ORG</td>\n",
              "      <td>I-ORG</td>\n",
              "      <td>I-ORG</td>\n",
              "      <td>I-ORG</td>\n",
              "      <td>I-ORG</td>\n",
              "      <td>I-ORG</td>\n",
              "      <td>I-ORG</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>losses</th>\n",
              "      <td>6.07</td>\n",
              "      <td>6.35</td>\n",
              "      <td>6.53</td>\n",
              "      <td>0.00</td>\n",
              "      <td>6.37</td>\n",
              "      <td>0.00</td>\n",
              "      <td>6.38</td>\n",
              "      <td>0.00</td>\n",
              "      <td>6.15</td>\n",
              "      <td>6.13</td>\n",
              "      <td>6.56</td>\n",
              "      <td>6.67</td>\n",
              "      <td>6.32</td>\n",
              "      <td>6.30</td>\n",
              "      <td>0.00</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-140ebc34-0ceb-4a77-bd67-4c193c91c7a2')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-140ebc34-0ceb-4a77-bd67-4c193c91c7a2 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-140ebc34-0ceb-4a77-bd67-4c193c91c7a2');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "           0     1     2      3      4      5         6     7      8      9   \\\n",
              "tokens    ▁''     8     .  ▁Juli    ▁''     ▁:  ▁Protest  camp   ▁auf   ▁dem   \n",
              "labels  B-ORG   IGN   IGN  I-ORG  I-ORG  I-ORG     I-ORG   IGN  I-ORG  I-ORG   \n",
              "preds       O     O     O      O      O      O         O     O      O      O   \n",
              "losses   7.29  0.00  0.00   3.42   8.05   8.68      5.96  0.00   7.38   7.19   \n",
              "\n",
              "              10     11          12     13      14     15     16    17  \n",
              "tokens  ▁Gelände   ▁der  ▁Republika      n  ischen   ▁Gar     de  </s>  \n",
              "labels     I-ORG  I-ORG       I-ORG    IGN     IGN  I-ORG    IGN   IGN  \n",
              "preds          O      O       B-ORG  I-ORG   I-ORG  I-ORG  I-ORG     O  \n",
              "losses      6.39   6.01        4.58   0.00    0.00   0.02   0.00  0.00  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-ba889efa-f527-4a1d-a379-470afb15fa8a\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "      <th>10</th>\n",
              "      <th>11</th>\n",
              "      <th>12</th>\n",
              "      <th>13</th>\n",
              "      <th>14</th>\n",
              "      <th>15</th>\n",
              "      <th>16</th>\n",
              "      <th>17</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>tokens</th>\n",
              "      <td>▁''</td>\n",
              "      <td>8</td>\n",
              "      <td>.</td>\n",
              "      <td>▁Juli</td>\n",
              "      <td>▁''</td>\n",
              "      <td>▁:</td>\n",
              "      <td>▁Protest</td>\n",
              "      <td>camp</td>\n",
              "      <td>▁auf</td>\n",
              "      <td>▁dem</td>\n",
              "      <td>▁Gelände</td>\n",
              "      <td>▁der</td>\n",
              "      <td>▁Republika</td>\n",
              "      <td>n</td>\n",
              "      <td>ischen</td>\n",
              "      <td>▁Gar</td>\n",
              "      <td>de</td>\n",
              "      <td>&lt;/s&gt;</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>labels</th>\n",
              "      <td>B-ORG</td>\n",
              "      <td>IGN</td>\n",
              "      <td>IGN</td>\n",
              "      <td>I-ORG</td>\n",
              "      <td>I-ORG</td>\n",
              "      <td>I-ORG</td>\n",
              "      <td>I-ORG</td>\n",
              "      <td>IGN</td>\n",
              "      <td>I-ORG</td>\n",
              "      <td>I-ORG</td>\n",
              "      <td>I-ORG</td>\n",
              "      <td>I-ORG</td>\n",
              "      <td>I-ORG</td>\n",
              "      <td>IGN</td>\n",
              "      <td>IGN</td>\n",
              "      <td>I-ORG</td>\n",
              "      <td>IGN</td>\n",
              "      <td>IGN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>preds</th>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "      <td>B-ORG</td>\n",
              "      <td>I-ORG</td>\n",
              "      <td>I-ORG</td>\n",
              "      <td>I-ORG</td>\n",
              "      <td>I-ORG</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>losses</th>\n",
              "      <td>7.29</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>3.42</td>\n",
              "      <td>8.05</td>\n",
              "      <td>8.68</td>\n",
              "      <td>5.96</td>\n",
              "      <td>0.00</td>\n",
              "      <td>7.38</td>\n",
              "      <td>7.19</td>\n",
              "      <td>6.39</td>\n",
              "      <td>6.01</td>\n",
              "      <td>4.58</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.02</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-ba889efa-f527-4a1d-a379-470afb15fa8a')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-ba889efa-f527-4a1d-a379-470afb15fa8a button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-ba889efa-f527-4a1d-a379-470afb15fa8a');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "           0      1          2     3      4      5           6     7     8   \\\n",
              "tokens  ▁1988    ▁in  ▁Auckland  ▁und  ▁1989    ▁in  ▁Stavanger  ▁tru     g   \n",
              "labels  B-ORG  I-ORG      I-ORG     O  B-ORG  I-ORG       I-ORG     O   IGN   \n",
              "preds       O      O      B-LOC     O      O      O       B-LOC     O     O   \n",
              "losses   3.88   9.76       7.29  0.00   4.05   9.39        7.23  0.00  0.00   \n",
              "\n",
              "          9     10      11    12    13    14    15    16    17    18  \n",
              "tokens   ▁er  ▁mit  ▁einem  ▁sie     b   ten  ▁bzw     ▁     .  </s>  \n",
              "labels     O     O       O     O   IGN   IGN     O     O   IGN   IGN  \n",
              "preds      O     O       O     O     O     O     O     O     O     O  \n",
              "losses  0.00  0.00    0.00  0.00  0.00  0.00  0.00  0.00  0.00  0.00  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-83867cf8-0931-4190-b532-35b4e80bc902\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "      <th>10</th>\n",
              "      <th>11</th>\n",
              "      <th>12</th>\n",
              "      <th>13</th>\n",
              "      <th>14</th>\n",
              "      <th>15</th>\n",
              "      <th>16</th>\n",
              "      <th>17</th>\n",
              "      <th>18</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>tokens</th>\n",
              "      <td>▁1988</td>\n",
              "      <td>▁in</td>\n",
              "      <td>▁Auckland</td>\n",
              "      <td>▁und</td>\n",
              "      <td>▁1989</td>\n",
              "      <td>▁in</td>\n",
              "      <td>▁Stavanger</td>\n",
              "      <td>▁tru</td>\n",
              "      <td>g</td>\n",
              "      <td>▁er</td>\n",
              "      <td>▁mit</td>\n",
              "      <td>▁einem</td>\n",
              "      <td>▁sie</td>\n",
              "      <td>b</td>\n",
              "      <td>ten</td>\n",
              "      <td>▁bzw</td>\n",
              "      <td>▁</td>\n",
              "      <td>.</td>\n",
              "      <td>&lt;/s&gt;</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>labels</th>\n",
              "      <td>B-ORG</td>\n",
              "      <td>I-ORG</td>\n",
              "      <td>I-ORG</td>\n",
              "      <td>O</td>\n",
              "      <td>B-ORG</td>\n",
              "      <td>I-ORG</td>\n",
              "      <td>I-ORG</td>\n",
              "      <td>O</td>\n",
              "      <td>IGN</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "      <td>IGN</td>\n",
              "      <td>IGN</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "      <td>IGN</td>\n",
              "      <td>IGN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>preds</th>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "      <td>B-LOC</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "      <td>B-LOC</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>losses</th>\n",
              "      <td>3.88</td>\n",
              "      <td>9.76</td>\n",
              "      <td>7.29</td>\n",
              "      <td>0.00</td>\n",
              "      <td>4.05</td>\n",
              "      <td>9.39</td>\n",
              "      <td>7.23</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-83867cf8-0931-4190-b532-35b4e80bc902')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-83867cf8-0931-4190-b532-35b4e80bc902 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-83867cf8-0931-4190-b532-35b4e80bc902');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@ INSPECTING OPEN PARENTHESIS:\n",
        "df_tmp = df.loc[df[\"input_tokens\"].apply(lambda x: u\"\\u2581(\" in x)].head(2)\n",
        "for sample in get_samples(df_tmp):\n",
        "    display(sample)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 333
        },
        "id": "7TDQbsPJ_jDo",
        "outputId": "ef35b738-9c80-4685-898a-bbb1fc596ab8"
      },
      "execution_count": 52,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "            0      1      2     3      4      5      6         7      8      9\n",
              "tokens   ▁Pam    plo     na    ▁-    ▁Vi    ana     ▁(  ▁Navarra     ▁)   </s>\n",
              "labels  B-LOC    IGN    IGN     O  B-LOC    IGN  I-LOC     I-LOC  I-LOC    IGN\n",
              "preds   B-LOC  I-LOC  I-LOC     O  B-LOC  I-LOC  I-LOC     I-LOC  I-LOC  B-LOC\n",
              "losses   0.05   0.00   0.00  0.01   0.03   0.00   0.00      0.01   0.01   0.00"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-0d99a564-57b1-4284-83fd-c25e34169664\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>tokens</th>\n",
              "      <td>▁Pam</td>\n",
              "      <td>plo</td>\n",
              "      <td>na</td>\n",
              "      <td>▁-</td>\n",
              "      <td>▁Vi</td>\n",
              "      <td>ana</td>\n",
              "      <td>▁(</td>\n",
              "      <td>▁Navarra</td>\n",
              "      <td>▁)</td>\n",
              "      <td>&lt;/s&gt;</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>labels</th>\n",
              "      <td>B-LOC</td>\n",
              "      <td>IGN</td>\n",
              "      <td>IGN</td>\n",
              "      <td>O</td>\n",
              "      <td>B-LOC</td>\n",
              "      <td>IGN</td>\n",
              "      <td>I-LOC</td>\n",
              "      <td>I-LOC</td>\n",
              "      <td>I-LOC</td>\n",
              "      <td>IGN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>preds</th>\n",
              "      <td>B-LOC</td>\n",
              "      <td>I-LOC</td>\n",
              "      <td>I-LOC</td>\n",
              "      <td>O</td>\n",
              "      <td>B-LOC</td>\n",
              "      <td>I-LOC</td>\n",
              "      <td>I-LOC</td>\n",
              "      <td>I-LOC</td>\n",
              "      <td>I-LOC</td>\n",
              "      <td>B-LOC</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>losses</th>\n",
              "      <td>0.05</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.01</td>\n",
              "      <td>0.03</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.01</td>\n",
              "      <td>0.01</td>\n",
              "      <td>0.00</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-0d99a564-57b1-4284-83fd-c25e34169664')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-0d99a564-57b1-4284-83fd-c25e34169664 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-0d99a564-57b1-4284-83fd-c25e34169664');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "            0      1      2      3      4      5      6      7      8\n",
              "tokens   ▁Cle   mens  ▁VIII      .     ▁(   ▁Pap     st     ▁)   </s>\n",
              "labels  B-PER    IGN  I-PER    IGN  I-PER  I-PER    IGN  I-PER    IGN\n",
              "preds   B-PER  B-PER  I-PER  I-PER  I-PER  I-PER  I-PER  I-PER  I-PER\n",
              "losses   0.00   0.00   0.00   0.00   0.00   0.00   0.00   0.00   0.00"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-fcbebee9-58b1-4e82-9caa-f0a69821c0ae\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>tokens</th>\n",
              "      <td>▁Cle</td>\n",
              "      <td>mens</td>\n",
              "      <td>▁VIII</td>\n",
              "      <td>.</td>\n",
              "      <td>▁(</td>\n",
              "      <td>▁Pap</td>\n",
              "      <td>st</td>\n",
              "      <td>▁)</td>\n",
              "      <td>&lt;/s&gt;</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>labels</th>\n",
              "      <td>B-PER</td>\n",
              "      <td>IGN</td>\n",
              "      <td>I-PER</td>\n",
              "      <td>IGN</td>\n",
              "      <td>I-PER</td>\n",
              "      <td>I-PER</td>\n",
              "      <td>IGN</td>\n",
              "      <td>I-PER</td>\n",
              "      <td>IGN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>preds</th>\n",
              "      <td>B-PER</td>\n",
              "      <td>B-PER</td>\n",
              "      <td>I-PER</td>\n",
              "      <td>I-PER</td>\n",
              "      <td>I-PER</td>\n",
              "      <td>I-PER</td>\n",
              "      <td>I-PER</td>\n",
              "      <td>I-PER</td>\n",
              "      <td>I-PER</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>losses</th>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-fcbebee9-58b1-4e82-9caa-f0a69821c0ae')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-fcbebee9-58b1-4e82-9caa-f0a69821c0ae button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-fcbebee9-58b1-4e82-9caa-f0a69821c0ae');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### **Cross-Lingual Transfer**"
      ],
      "metadata": {
        "id": "ortyWHQ_BSzK"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#@ EVALUATION ON MULTIPLE LANGUAGES: IMPORTANT:\n",
        "def get_f1_score(trainer, dataset):                             # Defining function.\n",
        "    return trainer.predict(dataset).metrics[\"test_f1\"]          # Getting f1 score."
      ],
      "metadata": {
        "id": "xYCdLcY-AdgA"
      },
      "execution_count": 46,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@ MODEL PERFORMANCE ON TEST DATASET:\n",
        "f1_scores = defaultdict(dict)                                                   # Initializing dictionary.\n",
        "f1_scores[\"de\"][\"de\"] = get_f1_score(trainer, \n",
        "                                     panx_de_encoded[\"test\"])                   # Getting f1 score.\n",
        "print(f\"F1-score of [de] model on [de] dataset: {f1_scores['de']['de']:.3f}\")   # Inspection."
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "id": "C7TBcnOnB8o1",
        "outputId": "5d87b169-fe04-4709-99d0-56a28e07018c"
      },
      "execution_count": 47,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='263' max='263' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [263/263 00:37]\n",
              "    </div>\n",
              "    "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "F1-score of [de] model on [de] dataset: 0.864\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@ INSPECTING MODEL PERFORMANCE ON FRENCH: \n",
        "text_fr = \"Jeff Dean est informaticien chez Google en Californie\"               # Initializing text.\n",
        "tag_text(text_fr, tags, trainer.model, xlmr_tokenizer)                          # Inspection."
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 112
        },
        "id": "0hm8hY5AGMKi",
        "outputId": "60baf082-ccdc-404e-fb28-4295b347f343"
      },
      "execution_count": 48,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "         0      1      2      3     4            5    6      7        8    9   \\\n",
              "Tokens  <s>  ▁Jeff    ▁De     an  ▁est  ▁informatic  ien  ▁chez  ▁Google  ▁en   \n",
              "Tags      O  B-PER  I-PER  I-PER     O            O    O      O    B-ORG    O   \n",
              "\n",
              "           10     11     12    13  \n",
              "Tokens  ▁Cali    for    nie  </s>  \n",
              "Tags    B-LOC  B-LOC  B-LOC     O  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-a1980e95-3999-49cc-bba8-bf4470388a8f\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "      <th>10</th>\n",
              "      <th>11</th>\n",
              "      <th>12</th>\n",
              "      <th>13</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>Tokens</th>\n",
              "      <td>&lt;s&gt;</td>\n",
              "      <td>▁Jeff</td>\n",
              "      <td>▁De</td>\n",
              "      <td>an</td>\n",
              "      <td>▁est</td>\n",
              "      <td>▁informatic</td>\n",
              "      <td>ien</td>\n",
              "      <td>▁chez</td>\n",
              "      <td>▁Google</td>\n",
              "      <td>▁en</td>\n",
              "      <td>▁Cali</td>\n",
              "      <td>for</td>\n",
              "      <td>nie</td>\n",
              "      <td>&lt;/s&gt;</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Tags</th>\n",
              "      <td>O</td>\n",
              "      <td>B-PER</td>\n",
              "      <td>I-PER</td>\n",
              "      <td>I-PER</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "      <td>B-ORG</td>\n",
              "      <td>O</td>\n",
              "      <td>B-LOC</td>\n",
              "      <td>B-LOC</td>\n",
              "      <td>B-LOC</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-a1980e95-3999-49cc-bba8-bf4470388a8f')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-a1980e95-3999-49cc-bba8-bf4470388a8f button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-a1980e95-3999-49cc-bba8-bf4470388a8f');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 48
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@ EVALUATION OF PAN-X LANGUAGES: IMPORTANT:\n",
        "def evaluate_lang_performance(lang, trainer):               # Defining function.                  \n",
        "    panx_ds = encode_panx_dataset(panx_ch[lang])            # Encoding dataset.\n",
        "    return get_f1_score(trainer, panx_ds[\"test\"])           # F1 score on test dataset."
      ],
      "metadata": {
        "id": "4YmZp1FyG6EF"
      },
      "execution_count": 49,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@ IMPLEMENTATION ON FRENCH DATASET:\n",
        "f1_scores[\"de\"][\"fr\"] = evaluate_lang_performance(\"fr\", trainer)                # Implementation.\n",
        "print(f\"F1-score of [de] model on [fr] dataset: {f1_scores['de']['fr']:.3f}\")   # Inspection."
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 141,
          "referenced_widgets": [
            "3891226f1781476da6ec6bc990d54c17",
            "83125c6c822e47fc8bf2e037ce1a6bd1",
            "678f0ab0c20441ee8d2449b3f3004ddf",
            "8f66c4dc1be5414d87168b2156d2bf9e",
            "544f06a4f4774ace861b61247545c349",
            "7303ba1309bd4116b017b98d714a0845",
            "b71fbbe8b54c4ce39cbac3e45ef7d81a",
            "25bc78c2202d48b7baf52127f97bf9d6",
            "0a3a9858ca4c45afb5517206ee283e47",
            "5bf46a946ead46dc99d738f081bde36e",
            "3ac94d58b5ed47e89607bbd25ef90d6c"
          ]
        },
        "id": "lD1J6eZDH6mo",
        "outputId": "eef39ab6-ecf8-4014-a8de-b746ae7c1020"
      },
      "execution_count": 54,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/5 [00:00<?, ?ba/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "3891226f1781476da6ec6bc990d54c17"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Loading cached processed dataset at /root/.cache/huggingface/datasets/xtreme/PAN-X.fr/1.0.0/349258adc25bb45e47de193222f95e68a44f7a7ab53c4283b3f007208a11bf7e/cache-17b34755b551ad0b.arrow\n",
            "Loading cached processed dataset at /root/.cache/huggingface/datasets/xtreme/PAN-X.fr/1.0.0/349258adc25bb45e47de193222f95e68a44f7a7ab53c4283b3f007208a11bf7e/cache-2528fe72e754fa78.arrow\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='359' max='263' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [263/263 04:20]\n",
              "    </div>\n",
              "    "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "F1-score of [de] model on [fr] dataset: 0.729\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@ IMPLEMENTATION ON ITALIAN DATASET:\n",
        "f1_scores[\"de\"][\"it\"] = evaluate_lang_performance(\"it\", trainer)                # Implementation.\n",
        "print(f\"F1-score of [de] model on [it] dataset: {f1_scores['de']['it']:.3f}\")   # Inspection."
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 126
        },
        "id": "6dWOEgT7ISuS",
        "outputId": "ef969b24-fe36-4661-aad0-1c34082a255d"
      },
      "execution_count": 63,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Loading cached processed dataset at /root/.cache/huggingface/datasets/xtreme/PAN-X.it/1.0.0/349258adc25bb45e47de193222f95e68a44f7a7ab53c4283b3f007208a11bf7e/cache-0d011d2b65742086.arrow\n",
            "Loading cached processed dataset at /root/.cache/huggingface/datasets/xtreme/PAN-X.it/1.0.0/349258adc25bb45e47de193222f95e68a44f7a7ab53c4283b3f007208a11bf7e/cache-639b50d87e1e3dd7.arrow\n",
            "Loading cached processed dataset at /root/.cache/huggingface/datasets/xtreme/PAN-X.it/1.0.0/349258adc25bb45e47de193222f95e68a44f7a7ab53c4283b3f007208a11bf7e/cache-da2ec01a6417acae.arrow\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='550' max='263' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [263/263 19:24]\n",
              "    </div>\n",
              "    "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "F1-score of [de] model on [it] dataset: 0.658\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@ IMPLEMENTATION ON ENGLISH DATASET:\n",
        "f1_scores[\"de\"][\"en\"] = evaluate_lang_performance(\"en\", trainer)                # Implementation.\n",
        "print(f\"F1-score of [de] model on [en] dataset: {f1_scores['de']['en']:.3f}\")   # Inspection."
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 126
        },
        "id": "9waBA4hFIpy2",
        "outputId": "34cec260-ec74-4dc6-8fd0-a54b3595cfdb"
      },
      "execution_count": 64,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Loading cached processed dataset at /root/.cache/huggingface/datasets/xtreme/PAN-X.en/1.0.0/349258adc25bb45e47de193222f95e68a44f7a7ab53c4283b3f007208a11bf7e/cache-b66c836d4276ec27.arrow\n",
            "Loading cached processed dataset at /root/.cache/huggingface/datasets/xtreme/PAN-X.en/1.0.0/349258adc25bb45e47de193222f95e68a44f7a7ab53c4283b3f007208a11bf7e/cache-1bc4582bb21ac460.arrow\n",
            "Loading cached processed dataset at /root/.cache/huggingface/datasets/xtreme/PAN-X.en/1.0.0/349258adc25bb45e47de193222f95e68a44f7a7ab53c4283b3f007208a11bf7e/cache-dd87826dfc1b6121.arrow\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='575' max='263' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [263/263 19:29]\n",
              "    </div>\n",
              "    "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "F1-score of [de] model on [en] dataset: 0.591\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@ FUNCTION TO FINE-TUNE XLM-R MODEL: IMPORTANT:\n",
        "def train_on_subset(dataset, num_samples):                                          # Defining function. \n",
        "    train_ds = dataset[\"train\"].shuffle(seed=2022).select(range(num_samples))       # Initializing training dataset.\n",
        "    valid_ds = dataset[\"validation\"]                                                # Initializing validation dataset.\n",
        "    test_ds = dataset[\"test\"]                                                       # Initializing test dataset.\n",
        "    training_args.logging_steps = len(train_ds) // batch_size                       # Initializing logging steps.\n",
        "    trainer = Trainer(model_init=model_init, args=training_args, \n",
        "                      data_collator=data_collator, compute_metrics=compute_metrics,\n",
        "                      train_dataset=train_ds, eval_dataset=valid_ds,                # Training and validation dataset.\n",
        "                      tokenizer=xlmr_tokenizer)                                     # Initializing trainer. \n",
        "    trainer.train()                                                                 # Training the trainer model.\n",
        "    if training_args.push_to_hub:\n",
        "        trainer.push_to_hub(commit_message=\"Training completed!\")                   # Pushing to hub.\n",
        "    f1_score = get_f1_score(trainer, test_ds)                                       # Getting f1 scores.\n",
        "    return pd.DataFrame.from_dict({\"num_samples\": [len(train_ds)], \n",
        "                                   \"f1_score\": f1_score})                           # Getting dataframe."
      ],
      "metadata": {
        "id": "ngoJkw1PJBKh"
      },
      "execution_count": 50,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@ FINE-TUNING WITH FRENCH CORPUS:\n",
        "panx_fr_encoded = encode_panx_dataset(panx_ch[\"fr\"])                                # Encoding french dataset.\n",
        "training_args.push_to_hub = False\n",
        "metrics_df = train_on_subset(panx_fr_encoded, 200)                                  # Implementation of funtion.\n",
        "metrics_df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 349,
          "referenced_widgets": [
            "5949b3fb5ac4445d9012cac6224f91ea",
            "c471b1e2dc4b4c1f81855ca2d67e216b",
            "087e3480402647f1972df8f1aee83f38",
            "1663d4ccf7264ccc961a176b407649ff",
            "6799dc79b5354409b22f899eddc045f7",
            "e9d70a51c6424c78be717bc32e88057c",
            "a9e4122594ab419cb1b3d2ccaf6ec69a",
            "2de82a252b624c5b91a9047b8e78a7db",
            "d2daac2054e34a3d87205d7daaf6672e",
            "dc76b7a2cc4d4a918f146f62996cbc6f",
            "5e62eba6d2264d7abf47b4a41fa43704",
            "85d33f318076422b8f676e1ac06ce1ee",
            "c816f0a6c4144438b6e0885318360275",
            "0ac49ff2840046fba7b66c6cdf7ac61b",
            "b885e371c96b4ffd9f189ac2465f7c06",
            "c2e6ca8a1c7f42ab8d83df1565b5d6f1",
            "84728e74ad574125ba8052f041aa4cbc",
            "cb763db02b674284bc5702fca35349d0",
            "b891750ba8fd4e8b94a221fe2deec11d",
            "89c10a91fa0f40138b560695027a6b79",
            "d7f5689004ff49abb0a6c33842b1870d",
            "6a79a1f8e4cf4c7baf5dd74027490bd6",
            "0529d904558b4b93b98c0c0d4b18c601",
            "27394b6475ab4c7aa33f48c8631745a5",
            "60cc0804c1b645bd997b1e24b9293ba9",
            "99f297f0962447bd8b4cab2e0fb3c18d",
            "ce09f6871fd54c4186b2cb8dab5c5ee2",
            "3f55bde1a92941229fbc0d99660b76e8",
            "a243f4e51b4140289b34dfed8a7d6556",
            "f71a736eb0254be3beeda7dce990263f",
            "01156ca6a3cd49e0b033e79ff8db1bae",
            "9e6df46864b5407fbf670ed5820efa0c",
            "f06fbf70067b4ad1a52687bbd1917b3e"
          ]
        },
        "id": "CwyFCextNKU4",
        "outputId": "dc1718ec-57af-45b7-d249-20becc7f223a"
      },
      "execution_count": 51,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/5 [00:00<?, ?ba/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "5949b3fb5ac4445d9012cac6224f91ea"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/3 [00:00<?, ?ba/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "85d33f318076422b8f676e1ac06ce1ee"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/3 [00:00<?, ?ba/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "0529d904558b4b93b98c0c0d4b18c601"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='27' max='27' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [27/27 00:45, Epoch 3/3]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>F1</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>1.921700</td>\n",
              "      <td>1.412960</td>\n",
              "      <td>0.001899</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>1.428000</td>\n",
              "      <td>1.269929</td>\n",
              "      <td>0.058604</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>1.255600</td>\n",
              "      <td>1.164589</td>\n",
              "      <td>0.115176</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='96' max='96' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [96/96 00:11]\n",
              "    </div>\n",
              "    "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   num_samples  f1_score\n",
              "0          200  0.122155"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-153a2238-363b-4e94-911e-5b2153d104ae\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>num_samples</th>\n",
              "      <th>f1_score</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>200</td>\n",
              "      <td>0.122155</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-153a2238-363b-4e94-911e-5b2153d104ae')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-153a2238-363b-4e94-911e-5b2153d104ae button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-153a2238-363b-4e94-911e-5b2153d104ae');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 51
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@ FINE-TUNING WITH FRENCH CORPUS: INCREASING TRAINING SET:\n",
        "for num_samples in [400]:\n",
        "    metrics_df = metrics_df.append(\n",
        "        train_on_subset(panx_fr_encoded, num_samples), ignore_index=True)       # Implementation."
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 226
        },
        "id": "rfIufH2sNvi5",
        "outputId": "98bf46c7-1b0d-4a48-92bc-319b14ba6473"
      },
      "execution_count": 52,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Loading cached shuffled indices for dataset at /root/.cache/huggingface/datasets/xtreme/PAN-X.fr/1.0.0/349258adc25bb45e47de193222f95e68a44f7a7ab53c4283b3f007208a11bf7e/cache-a910b4a8a23320db.arrow\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='51' max='51' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [51/51 00:58, Epoch 3/3]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>F1</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>1.656500</td>\n",
              "      <td>1.177004</td>\n",
              "      <td>0.131535</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>1.063500</td>\n",
              "      <td>0.816487</td>\n",
              "      <td>0.481619</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>0.779300</td>\n",
              "      <td>0.703334</td>\n",
              "      <td>0.548086</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='96' max='96' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [96/96 00:12]\n",
              "    </div>\n",
              "    "
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@ PLOTTING F1 SCORES:\n",
        "fig, ax = plt.subplots()\n",
        "ax.axhline(f1_scores[\"de\"][\"fr\"], ls=\"--\", color=\"r\")\n",
        "metrics_df.set_index(\"num_samples\").plot(ax=ax)\n",
        "plt.legend([\"Zero-shot from de\", \"Fine-tuned on fr\"], loc=\"lower right\")\n",
        "plt.ylim((0, 1))\n",
        "plt.xlabel(\"Number of training samples\")\n",
        "plt.ylabel(\"F1 score\")\n",
        "plt.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 283
        },
        "id": "cq6jcaMBO1th",
        "outputId": "ddbdfc92-8315-45cf-dafe-8251d4a55820"
      },
      "execution_count": 55,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEKCAYAAAAfGVI8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXwV9b3/8dcnC4QlrAk57ATZExAl4IKyKJC4V2tV3G0rFat1qV71Vqz2tvdX61J7eylWey11adVqF2qRoAjigsoqJuwgQtCQsBMg+/f3x0zCyQYJcE5Czvv5eOTBzHfmzHwyOZzPmZnvfL7mnENERCJXVGMHICIijUuJQEQkwikRiIhEOCUCEZEIp0QgIhLhlAhERCJcyBKBmb1gZnlmllXHcjOz/zGzDWa20sxOD1UsIiJSt1CeEcwEMo6w/AKgv/8zBZgRwlhERKQOIUsEzrmFwK4jrHIZ8KLzfAJ0MLOuoYpHRERqF9OI++4ObA2az/Hbvqm+oplNwTtroE2bNiMGDRoUlgBFRJqLpUuX7nDOJda2rDETQb05554DngNIS0tzS5YsaeSIREROLmb2VV3LGrPX0DagZ9B8D79NRETCqDETwSzgRr/30JnAXudcjctCIiISWiG7NGRmfwHGAQlmlgP8FIgFcM49C8wGLgQ2AAeBW0IVi4iI1C1kicA5N/koyx3ww1DtX0RE6kdPFouIRDglAhGRCKdEICIS4ZQIREQinBKBiEiEUyIQEYlwSgQiIhFOiUBEJMIpEYiIRDglAhGRCKdEICIS4ZQIREQinBKBiEiEUyIQEYlwSgQiIhFOiUBEJMIpEYiIRDglAhGRCKdEICIS4ZQIREQinBKBiEiEUyIQEYlwSgQiIhFOiUBEJMIpEYiIRDglAhGRCKdEICIS4ZQIREQinBKBiEiEUyIQEYlwSgQiIhFOiUBEJMIpEYiIRDglAhGRCKdEICIS4UKaCMwsw8zWmtkGM3uwluW9zGy+mS03s5VmdmEo4xERkZpClgjMLBqYDlwADAEmm9mQaqs9DLzunDsNuAb4XajiERGR2oXyjGAUsME5t8k5Vwy8ClxWbR0HtPOn2wNfhzAeERGpRSgTQXdga9B8jt8W7FHgejPLAWYDd9a2ITObYmZLzGxJfn5+KGIVEYlYjX2zeDIw0znXA7gQeMnMasTknHvOOZfmnEtLTEwMe5AiIs1ZKBPBNqBn0HwPvy3Y94DXAZxzi4A4ICGEMYmISDWhTASLgf5mlmxmLfBuBs+qts4W4HwAMxuMlwh07UdEJIxClgicc6XAHUAmsBqvd1C2mf3MzC71V/sxcKuZfQ78BbjZOedCFZOIiNQUE8qNO+dm490EDm57JGh6FTA6lDGIiMiRNfbNYhERaWRKBCIiEU6JQEQkwikRiIhEOCUCEZEIp0QgIhLhlAhERCKcEoGISIQL6QNlIbF2LYwbV7Xtqqvg9tvh4EG4sJaxbW6+2fvZsQOuvLLm8qlT4eqrYetWuOGGmst//GO45BJv3z/4Qc3lDz8MEybAihVw9901l//3f8PZZ8PHH8N//mfN5c88A8OHw7vvws9/XnP5738PAwfCv/4FTz1Vc/lLL0HPnvDaazBjRs3lb7wBCQkwc6b3U93s2dC6Nfzud/D66zWXL1jg/fvkk/DWW1WXtWoFb7/tTf/Xf8G8eVWXd+4Mb77pTT/0ECxaVHV5jx7w8sve9N13e8cw2IAB8Nxz3vSUKbBuXdXlw4d7xw/g+ushJ6fq8rPOgv/3/7zpb38bdu6suvz882HaNG/6ggvg0KGqyy++GO67z5uu/r4Dvff03vOmT8b3XhCdEYiIRDg72Ur7pKWluSVLljR2GCIiJxUzW+qcS6ttmc4IREQinBKBiEiEUyIQEYlwSgQiIhFOiUBEJMIpEYiIRDglAhGRCKdEICIS4ZQIREQinBKBiEiEUyIQEYlwSgQiIhFOiUBEJMIpEYiIRDglAhGRCKdEICIS4ZQIREQinBKBiEiEUyIQEYlwSgQiIhFOiUBEJMIpEYiIRDglAhGRCBfSRGBmGWa21sw2mNmDdaxzlZmtMrNsM/tzKOMREZGaYkK1YTOLBqYDE4EcYLGZzXLOrQpapz/wEDDaObfbzLqEKh4REaldKM8IRgEbnHObnHPFwKvAZdXWuRWY7pzbDeCcywthPCIiUotQJoLuwNag+Ry/LdgAYICZfWRmn5hZRm0bMrMpZrbEzJbk5+eHKFwRkcjU2DeLY4D+wDhgMvC8mXWovpJz7jnnXJpzLi0xMTHMIYqING+hTATbgJ5B8z38tmA5wCznXIlz7ktgHV5iEBGRMKlXIjCzc8zsFn860cyS6/GyxUB/M0s2sxbANcCsauv8A+9sADNLwLtUtKmesYuIyAlw1ERgZj8FHsDr3QMQC7x8tNc550qBO4BMYDXwunMu28x+ZmaX+qtlAjvNbBUwH7jfObez4b+GiIgcK3POHXkFsxXAacAy59xpfttK59ywMMRXQ1pamluyZElj7FpEpNHsPVRCbLTRusWx9fo3s6XOubTaltVni8XOOWdmzt9Ym2OKQkREGmRHQRHvrNrOnKxcPt64g19cPpSr0noe/YUNVJ9E8LqZ/R7oYGa3At8Fnj/hkYiICNv2HCIzK5c52bks2byLcge9OrXmltHJDO9Zo1PlCXHERGBmBrwGDAL2AQOBR5xz74QkGhGRCLQxv4A5WblkZueyMmcvAAOT4rnjvP5kpAQY3DUe7+M4NI6YCPxLQrOdc0MBffiLiJwAzjmyv95HZnYuc7JyWZ9XAMCpPTvwQMYg0lOS6JvYNmzx1OfS0DIzG+mcWxzyaEREmqnycseyLbuZ41/2ydl9iCiDUcmduO6MIUxKCdCtQ6tGia0+ieAM4Doz+wo4ABjeyUKj9BoSETlZlJSV8+mmXczJ/obM7O3k7y8iNto4p18Cd57XjwmDk+jctmVjh1mvRJAe8ihERJqJwpIyPli/gzlZuby7ejt7D5XQKjaa8YMSSU8JMH5QF9rFxTZ2mFUcNRE4574ys1OBc/2mD5xzn4c2LBGRk8f+whLmr80nMyuX+WvzOFhcRru4GCYMSSIjJcCYAYnExUY3dph1OmoiMLO78MpF/81vetnMnnPO/TakkYmINGG7DhTz7qrtzMnO5cP1OyguKyehbUsuP607GakBzuzbmdjoxq7rWT/1uTT0PeAM59wBADN7HFgEKBGISET5Zu8h5mZ7D3h9+uVOyh1079CKG87qTUZqgNN7dSQ6KnTdPEOlPonAgLKg+TK/TUSk2du84wBz/G6eK7buAaBfl7bcPq4fGakBUrq1C2kf/3CoTyL4I/Cpmf3dn/8W8H+hC0lEpPE451iTu7/yAa81ufsBGNq9PfenDyQ9JYl+XeIbOcoTqz43i582swXAOX7TLc655SGNSkQkjMrLHSty9lSWdvhq50HMYGTvTky7eAjpKUn06Ni6scMMmfrcLD4TyHbOLfPn25nZGc65T0MenYhIiJSWlfPZl7uYk+1989++r4iYKOPsfgn8YMwpTBySRGJ84/fxD4f6XBqaAZweNF9QS5uISJNXWFLGxxt38PYXXh//3QdLiIuNYuyARDJSA5w3MIn2rZtWH/9wqNfNYhc0aIFzrtzMjq0gtohImB0oKmXB2nzmZOcyf00eBUWlxLeM4fzBXchI9fr4H2uN/+aiPr/9JjP7Ed5ZAMDtaDhJEWnC9hws5t3VeczJymXh+nyKS8vp3KYFl5zalfSUAGefkkCLmJOjj3841CcR3Ab8D/Aw4IB5wJRQBiUi0lB5+wrJXLWdzKxcFm3aSVm5o2v7OK4d1YsLUgOk9el0UvbxD4f69BrKwxt4XkSkSdmy86BXyjk7l2VbduMcJCe0YcqYvmSkBBjWo/1J38c/HOrTa+hXwM+BQ8AcYBhwj3PuqAPYi4icSM451ud5g7jMycpl1Tf7ABjStR33TBhARmqA/l3a6sO/gepzaWiSc+4/zOxyYDNwBbAQUCIQkZBzzrEyZ6/XzTMrl007DgAwondHfnLhYNJTAvTq3Hz7+IdDfRJBxToXAX91zu1VthWRUCordyzevIs5WbnMzc7l672FREcZZ/XtzC3nJDNpSBJJ7eIaO8xmoz6J4C0zW4N3aWiqmSUChaENS0QiTVFpGR9v3ElmVi7vrNrOzgPFtIiJYkz/RO6dNJAJg7vQoXWLxg6zWarPzeIH/fsEe51zZWZ2ELgs9KGJSHN3sLiUhevymZOVy7zVeewvKqVNi2jOG+zV8R83MJE2LSO7j3841OsIO+d2BU0fwBuyUkSkwfYeKuG9NV4p5/fX5VNYUk7H1rFcMDRARqrXx78pD+LSHCnVikjI5e8v4h1/EJePN+ygtNyR1K4lV6X1JCMlwKjkTsScJIO4NEdKBCISEjm7D5KZ7T3gtfirXTgHvTu35nvnJJOeGmB4jw5E6QGvJuGYEoGZDXLOrTnRwYjIyW1DXoH3gFdWLl9s2wvAoEA8PzqvPxmpAQYF4tXHvwk61jOCuUCvExmIiJx8nHNkf73Pe8ArO5cNeQUADO/ZgQcvGER6SoDkhDaNHKUcTZ2JwMz+p65FQIfQhCMiTV1ZuWPZlt2VT/du23OIKIMzkjtzw5m9mZSSRNf2rRo7TGmAI50R3AL8GCiqZdnk0IQjIk1RSVk5izbuZE52LnOzt7OjoIgW0VGc0z+Bu87vz4QhSXRqoz7+J6sjJYLFQJZz7uPqC8zs0ZBFJCJNQmFJWWUf/3dXb2dfYSmtW0QzfmAX0lMDjB+YSHxc5A3i0hwdKRFcSR1PEDvnkkMTjog0pv2FJby3Jo/M7Fzmr8nnUEkZ7VvFMnGI18f/3P7q498cHSkRtA1+kExEmqedBUW8u9p7wOujDTspLisnMb4l3x7RnYyUrpzRtxOx6uPfrB0pEfwDf1xiM3vTOfft8IQkIqH29Z5DzPXr+H/25S7KHfTo2Iqbzu5NRmqA03p2VB//CHKkRBD8Luh7LBs3swzgN0A08Afn3C/rWO/bwBvASOfckmPZl4gc2Zc7DlR28/x86x4A+ndpyw/H9yM9JUBKt3bq4x+hjpQIXB3T9WJm0cB0YCKQAyw2s1nOuVXV1osH7gI+beg+RKRuzjlWf7O/so7/2u37ARjWoz33pw8kPSVAvy5tGzlKaQqOlAhONbN9eGcGrfxp/HnnnGt3lG2PAjY45zYBmNmreFVLV1Vb77+Ax4H7Gxq8iFRVXu5YvnVP5dO9W3YdxAxG9unEIxcPIT01QPcO6uMvVdWZCJxzx9s1oDuwNWg+BzgjeAUzOx3o6Zz7t5nVmQjMbAowBaBXLz3QLBKstKycT7/0BnHJzM4lb38RsdHG2ackMHXcKUwYnERifMvGDlOasEYrOmdmUcDTwM1HW9c59xzwHEBaWlqDL1OJNDeFJWV8tGEHc7JyeWf1dvYcLCEuNopxA7qQkRpg/KAutG+lPv5SP6FMBNuAnkHzPfy2CvFAKrDAv0EVAGaZ2aW6YSxSU0FRKQvW5jEnK5f5a/I4UFxGfFwMEwYnkZ4SYOyARFq1UB9/abhQJoLFQH8zS8ZLANcA11YsdM7tBRIq5s1sAXCfkoDIYbsPFPPu6u1kZueycP0OikvLSWjbgkuHdycjNcBZfTvTIkZ9/OX4hCwROOdKzewOIBOv++gLzrlsM/sZsMQ5NytU+xY5mW3fV1jZx/+TTbsoK3d079CK68/w+viP6N2RaPXxlxMopPcInHOzgdnV2h6pY91xoYxFpCn7aueByp4+y7Z4ffz7JrbhtrF9yUjpSmp39fGX0NEIZSKNwDnHuu0FlQ94rf7G652d0q0dP544gAuGBujXJb6Ro5RIoUQgEibOOT7P2VvZzfPLHQcwgxG9OvLwRYNJTwnQs1Prxg5TIpASgUgIlZaVs3jzbjKzvQ//b/YWEhNlnHVKZ753TjKThiTRpV1cY4cpEU6JQOQEKyot4+MNOyv7+O86UEzLmCjGDEjkvkkDOX9wFzq01iAu0nQoEYicAAeLS3l/bT5vZ+Xy3po8CopKadsyhvMGeQ94jR2QSJuW+u8mTZPemSLHaO/BEuat8er4v78un6LScjq2juWioV3JSA1wdr/OtIzRA17S9CkRiDRA3v5C3lnlffgv2riT0nJHoF0ck0f1Ij0lwMg+HYnRIC5yklEiEDmKrbsOVt7sXfLVbpyDPp1b8/1z+5KRGmBY9/YaxEVOakoEIrXYkLe/so9/1javj//gru24+/wBZKQGGJDUVg94SbOhRCCC18c/a9s+5mR/w5ysXDbmHwDg9F4d+M8LB5GeEqB35zaNHKVIaCgRSMQqK3cs/Wp35QNe2/YcIjrKOCO5Ezed3YdJQwIE2quPvzR/SgQSUYpLy1m0ye/jvyqXHQXFtIiO4tz+Cdw1oT8TBifRqY36+EtkUSKQZu9QcRnvr8snMzuXd1dvZ39hKa1bRDN+UBcyUgKMG5hIfJwGcZHIpUQgzdK+whLeW+0N4rJgXR6FJeW0bxVLekqAjJQA5/RPIC5WffxFQIlAmpEdBUW8u2o7c7Jz+WjDDkrKHF3iW/KdET3JSA0wKrkTserjL1KDEoGc1L7ec6iyjv/izbsod9CzUytuGZ1MekqA03p2UB9/kaNQIpCTzqb8AuZk55KZlcvnOXsBGJgUzx3n9ScjJcDgrvHq4y/SAEoE0uQ551j1zT4y/Qe81m0vAODUnh14IGMQ6SlJ9E1s28hRipy8lAikSSovdyzfurvy6d6tuw4RZTAquROPXjKESSkBunVo1dhhijQLSgTSZJSUlfPppl3Myf6GudnbydtfRGy0MbpfAj8c148JQ5JIaNuyscMUaXaUCKRRFZaU8cH6HczJ8vr47z1UQqvYaMYNTCQjNcD4QV1opz7+IiGlRCBht7+whPlr88nMymX+2jwOFpcRHxfDxMFJpKcGGNM/kVYt1MdfJFyUCCQsdh0o5t3VXh3/D9fvoLisnIS2LfjWad3JSAlwZt/OtIhRH3+RxqBEICGTu7eQuau8Pv6ffrmLsnJH9w6tuOGs3mSkBji9V0ei1cdfpNEpEcgJtXnHAe8Br+xclm/ZA0C/Lm2ZOvYUMlIDpHRrpz7+Ik2MEoEcF+cca7f7g7hk5bImdz8AQ7u35/70gaSnJNGvS3wjRykiR6JEIA1WXu74PGdP5dO9m3cexAzSendk2sVDmDQkiZ6dWjd2mCJST0oEUi+lZeV8tnkXmVm5ZGZvJ3dfITFRxlmndObWMX2ZOCSJLvEaxEXkZKREIHUqKi3jow07/EFctrP7YAktY6IYOyCR/0gdyPmDkmjfWn38RU52SgRSxYGiUhaszWdOdi7z1+RRUFRKfMsYzhvsDeIydmAirVvobSPSnOh/tLDnYDHzVucxJzuXhevyKSotp1ObFlw8rCvpqQHOPqUzLWP0gJdIc6VEEKHy9hUyd9V2MrNzWbRxJ6Xljq7t45g8qhcZqQHSenckRoO4iEQEJYIIsnXXwcpBXJZu2Y1zkJzQhlvH9CUjJcCwHu3Vx18kAikRNGPOOTbkFVSWcs7+eh8AQ7q2454JA8hIDdC/S1t9+ItEuJAmAjPLAH4DRAN/cM79strye4HvA6VAPvBd59xXoYypuXPO8cW2vZUf/pvyDwAwondHfnLhYNJTAvTqrD7+InJYyBKBmUUD04GJQA6w2MxmOedWBa22HEhzzh00s6nAr4CrQxVTc1VW7liyeVflA15f7y0kOso4s28nbjm7D5NSAiS1Ux9/EaldKM8IRgEbnHObAMzsVeAyoDIROOfmB63/CXB9CONpVopLy/l44w4ys3OZm72dnQeKaRETxZj+CdwzcQATBifRsU2Lxg5TRE4CoUwE3YGtQfM5wBlHWP97wNu1LTCzKcAUgF69ep2o+E46B4tLWbgunzlZucxbk8f+wlLatIhm/KAuZKQGGDewC21b6raPiDRMk/jUMLPrgTRgbG3LnXPPAc8BpKWluTCG1uj2HirhvTVeHf/31+VTWFJOh9axZKQEyEgNMLpfAnGx6uMvIsculIlgG9AzaL6H31aFmU0AfgKMdc4VhTCek8aOgiLeWeV9+H+8cQclZY6kdi25Kq0nGSkBRiV3Uh9/ETlhQpkIFgP9zSwZLwFcA1wbvIKZnQb8HshwzuWFMJYmb9ueQ2T6PX0Wb96Fc9CrU2u+OzqZ9NQAw3t0IEqDuIhICIQsETjnSs3sDiATr/voC865bDP7GbDEOTcLeAJoC/zV78u+xTl3aahiamo25nt9/DOzc1mZsxeAQYF4fnRefzJSAwwKxKuPv4iEnDl3cl1yT0tLc0uWLGnsMI6Jc47sr/dVPt27Pq8AgOE9O5CRGiA9JUByQptGjlJEmiMzW+qcS6ttWZO4WdyclZc7lm3ZXfmAV87uQ0QZnJHcmevP7M2klCS6tm/V2GFKM1VSUkJOTg6FhYWNHYqESVxcHD169CA2tv4l4pUIQqCkrJxPNu1kTlYuc1dtJ39/ES2ioxjdrzN3ntePCYOT6Ny2ZWOHKREgJyeH+Ph4+vTpo8uMEcA5x86dO8nJySE5Obner1MiOEEKS8q8Pv7ZucxbncfeQyW0io1m/KBE0lMCjB/UhXZxGsRFwquwsFBJIIKYGZ07dyY/P79Br1MiOA77C0t4b00emdm5LFibz8HiMtrFxTBhSBIZKQHGDEhUH39pdEoCkeVY/t5KBA20s6CId1d7ffw/2rCT4rJyEtq25PLTupORGuDMvp2JVR9/ETmJKBHUwzd7DzE3eztvZ33DZ1/uotxBj46tuPGs3mSkBjitV0ei1cdfpIa///3vPPbYY1XaVq5cyb///W8uuOCCkO9/wYIFPPnkk7z11lv1fs0zzzzDlClTaN26ZpXeDz74gNtuu43Y2FgWLVpEq1bh6ejRtm1bCgoKQrZ9JYI6fLnjQGU3zxVb9wDQv0tbfji+H+kpAVK6tdMpt8hRXH755Vx++eWV88899xyvvPIK6enp9Xq9cw7nHFFR4TvLfuaZZ7j++utrTQSvvPIKDz30ENdfX7U+ZmlpKTExJ+/H6ckb+QnmnGNN7v7KB7zW5O4HYFiP9tyfPpD0lAD9urRt5ChFjtO4cTXbrroKbr8dDh6ECy+sufzmm72fHTvgyiurLluwoN67XrduHT/72c/4+OOPKz/Yn3jiCV5//XWKioq4/PLLeeyxx9i8eTPp6emcccYZLF26lNmzZ/O///u/vP3225gZDz/8MFdfXbNa/fvvv89dd90FeNfJFy5cCEBBQQFXXnklWVlZjBgxgpdffhkzY968edx3332UlpYycuRIZsyYwe9//3u+/vprxo8fT0JCAvPnHy6Q/Ic//IHXX3+dzMxM3n77bW699VamTZtGx44dWbNmDStXrmTq1KksWbKEmJgYnn76acaPH8/MmTP5xz/+wYEDB1i/fj333XcfxcXFvPTSS7Rs2ZLZs2fTqVOnKr/Ll19+ybXXXktBQQGXXXZZlWW1HbPjFdGJoLzcsSJnT2Vph692HsQMRvbpxCMXD2FSShI9OmoQF5HjVVJSwrXXXstTTz1VWUF47ty5rF+/ns8++wznHJdeeikLFy6kV69erF+/nj/96U+ceeaZvPnmm6xYsYLPP/+cHTt2MHLkSMaMGUPXrl2r7OPJJ59k+vTpjB49moKCAuLivDE4li9fTnZ2Nt26dWP06NF89NFHpKWlcfPNNzNv3jwGDBjAjTfeyIwZM7j77rt5+umnmT9/PgkJCVW2//3vf58PP/yQiy++mCuvvJIFCxawbNkysrKySE5O5qmnnsLM+OKLL1izZg2TJk1i3bp1AGRlZbF8+XIKCwvp168fjz/+OMuXL+eee+7hxRdf5O67766yr7vuuoupU6dy4403Mn369Mr2uo7ZmDFjjuvvE3GJoLSsnM++9Adxyc5l+74iYqONs09J4LaxpzBhcBKJ8erjL83Ukb7Bt2595OUJCQ06Awg2bdo0UlJSqnyTnzt3LnPnzuW0004DvG/u69evp1evXvTu3ZszzzwTgA8//JDJkycTHR1NUlISY8eOZfHixVx6adVqNKNHj+bee+/luuuu44orrqBHjx4AjBo1qnJ6+PDhbN68mfj4eJKTkxkwYAAAN910E9OnT6/xgXw0o0aNquyv/+GHH3LnnXcCMGjQIHr37l2ZCMaPH098fDzx8fG0b9+eSy65BIChQ4eycuXKGtv96KOPePPNNwG44YYbeOCBB454zJQI6mnZlt385dMtvLt6O7sPlhAXG8W4AV4d//GDutC+lfr4i4TCggULePPNN1m2bFmVduccDz30ED/4wQ+qtG/evJk2bY5eamX69Ok8//zzAMyePZsHH3yQiy66iNmzZzN69GgyMzMBaNny8Be76OhoSktLj/dXqlSfOKvHEBUVVTkfFRVVZzy13YOs65gdr4jp55jtj+M7dkAiz15/OsumTeTZG0bwrdO6KwmIhMju3bu55ZZbePHFF4mPj6+yLD09nRdeeKGyN8y2bdvIy6tZhPjcc8/ltddeo6ysjPz8fBYuXMioUaP44Q9/yIoVK1ixYgXdunVj48aNDB06lAceeICRI0eyZs2aOuMaOHAgmzdvZsOGDQC89NJLjB3rDYcSHx/P/v37G/y7nnvuubzyyiuAdz9ky5YtDBw4sMHbAe/s5tVXXwWo3CbU/5g1VMScEXwnrSdXj+xFi5iIyX0ije7ZZ58lLy+PqVOnVml/6KGHuPrqq1m9ejVnnXUW4HWRfPnll4mOrvoQ5uWXX86iRYs49dRTMTN+9atfEQgEauzrmWeeYf78+URFRZGSksIFF1zAokWLao0rLi6OP/7xj3znO9+pvFl82223ATBlyhQyMjLo1q1blZvFR3P77bczdepUhg4dSkxMDDNnzqxyJtAQv/nNb7j22mt5/PHHq9wsnjRpUq3HrEuXLse0nwqqPirSjK1evZrBgwc3dhgSZrX93Y9UfVRfj0VEIpwSgYhIhFMiEBGJcEoEIiIRTolARCTCKRGIiEQ4JQIRCano6GiGDx9e+bN582bOPvvsE7LtPXv28Lvf/e6EbOtoxo0bRyi6rk+ePJlhw+n1WKoAAA+SSURBVIbx61//+oRvu74i5oEyEWkcrVq1YsWKFVXaPv744xOy7YpEcPvtt5+Q7YVbbm4uixcvrnzCOVg4S1srEYhEiMf+lc2qr/ed0G0O6daOn16S0uDXVQy0smDBAh599FESEhJqlIleunQp9957LwUFBSQkJDBz5swaFUcffPBBNm7cyPDhw5k4cSIXXXRRlYFo7rjjjspKo3369OGmm27iX//6FyUlJfz1r39l0KBBHDhwgDvvvJOsrCxKSkp49NFHueyyyzh06BC33HILn3/+OYMGDeLQoUO1/i61lbNu2bJlnfsLNmnSJLZt28bw4cP57W9/y7Rp0xg+fHhlob0f//jHDT62x0KXhkQkpA4dOlR5WSh4kJoKy5cv55lnnmHVqlVs2rSJjz76iJKSEu68807eeOMNli5dyne/+11+8pOf1HjtL3/5S0455RRWrFjBE088cdRYEhISWLZsGVOnTuXJJ58E4Be/+AXnnXcen332GfPnz+f+++/nwIEDzJgxg9atW7N69Woee+wxli5dWmN7hYWF3Hzzzbz22mt88cUXlJaWMmPGjCPuL9isWbMq4z/33HMBKC4uZsmSJWFLAqAzApGIcSzf3E+E2i4NBautTHSHDh3Iyspi4sSJAJSVldU4GzgWV1xxBQAjRozgb3/7G+CVdp41a1blB3VhYSFbtmxh4cKF/OhHPwJg2LBhDBs2rMb21q5de8Ry1rXt72hqG3Qn1JQIRKRR1VYm2jlHSkpKjaJxW7durazlf9ttt5GRkVFleUxMDOXl5ZXzhYWFte4ruBy1c44333zzmCuFHklt+zua+pa2PpF0aUhEmpyBAweSn59fmQhKSkrIzs6mZ8+elaWnb7vttholo3v37s2qVasoKipiz549zJs376j7Sk9P57e//S0VBTiXL18OwJgxY/jzn/8MeCOM1TaAzJHKWZ9MlAhEpMlp0aIFb7zxBg888ACnnnoqw4cPr7WnUefOnRk9ejSpqancf//99OzZk6uuuorU1FSuuuqqypG8jmTatGmUlJQwbNgwUlJSmDZtGgBTp06loKCAwYMH88gjjzBixIgarw0uZz106FCioqIqy1mfTFSGWqQZUxnqyKQy1CIi0iBKBCIiEU6JQKSZO9ku/8rxOZa/txKBSDMWFxfHzp07lQwihHOOnTt3EhcX16DX6TkCkWasR48e5OTkkJ+f39ihSJjExcVVPqBXX0oEIs1YbGwsycnJjR2GNHEhvTRkZhlmttbMNpjZg7Usb2lmr/nLPzWzPqGMR0REagpZIjCzaGA6cAEwBJhsZkOqrfY9YLdzrh/wa+DxUMUjIiK1C+UZwShgg3Nuk3OuGHgVuKzaOpcBf/Kn3wDONzMLYUwiIlJNKO8RdAe2Bs3nAGfUtY5zrtTM9gKdgR3BK5nZFGCKP1tgZmuPMaaE6ttuIhRXwyiuhmuqsSmuhjmeuHrXteCkuFnsnHsOeO54t2NmS+p6xLoxKa6GUVwN11RjU1wNE6q4QnlpaBvQM2i+h99W6zpmFgO0B3aGMCYREakmlIlgMdDfzJLNrAVwDTCr2jqzgJv86SuB95yefBERCauQXRryr/nfAWQC0cALzrlsM/sZsMQ5Nwv4P+AlM9sA7MJLFqF03JeXQkRxNYziarimGpviapiQxHXSlaEWEZETS7WGREQinBKBiEiEa1aJwMx6mtl8M1tlZtlmdpff3snM3jGz9f6/Hf12M7P/8UtcrDSz08Mc1xNmtsbf99/NrIPf3sfMDpnZCv/n2TDH9aiZbQva/4VBr3nIP15rzSw9zHG9FhTTZjNb4beH63jFmdlnZva5H9djfnuyXyJlgx9jC789LCVUjhDXK/7fKcvMXjCzWL99nJntDTpej4Q5rplm9mXQ/of77eH6/1hXXB8ExfS1mf3Dbw/L8QqKL9rMlpvZW/586N9fzrlm8wN0BU73p+OBdXjlLX4FPOi3Pwg87k9fCLwNGHAm8GmY45oExPjtjwfF1QfIasTj9ShwXy3rDwE+B1oCycBGIDpccVVb5yngkTAfLwPa+tOxwKf+++Z14Bq//Vlgqj99O/CsP30N8FqY47rQX2bAX4LiGge81YjHayZwZS3rh+v/Y61xVVvnTeDGcB6voH3fC/y5Yp/heH81qzMC59w3zrll/vR+YDXe08vBpSz+BHzLn74MeNF5PgE6mFnXcMXlnJvrnCv1V/sE71mLsDnC8arLZcCrzrki59yXwAa8UiJhjcvMDLgK78MtbPz3SYE/G+v/OOA8vBIpUPP9FfISKnXF5Zyb7S9zwGeE//1V1/GqS7j+Px4xLjNrh/c3/ceJ3vfRmFkP4CLgD/68EYb3V7NKBMH806TT8LJ9knPuG39RLpDkT9dWBuNIH4QnOq5g38X7NlQh2T89fN/Mzg1lTHXEdYd/ev6C+ZfSaDrH61xgu3NufVBbWI6Xf9q+AsgD3sE7K9oTlNCDj0mVEipARQmVkMflnPs0aFkscAMwJ+glZ/mXRt42s5RQxHSUuH7hv79+bWYt/bawvb+OdLzwPmjnOef2BbWF5XgBzwD/AZT7850Jw/urWSYCM2uLd2p3d7U/Jv63o0bpM1tXXGb2E6AUeMVv+gbo5Zw7Df800f+WEq64ZgCnAMP9WJ4K1b4bGFeFyVQ9Gwjb8XLOlTnnhuN9ux4FDArFfhqqelxmlhq0+HfAQufcB/78MqC3c+5U4LeE8JtvHXE9hHfcRgKdgAdCtf8GxlWh+vsrLMfLzC4G8pxzS0Ox/SNpdonA//bzJvCKc+5vfvP2ilNM/988v70+ZTBCGRdmdjNwMXCdn6TwL73s9KeX4n3rHBCuuJxz2/3/KOXA8xy+/NMUjlcMcAXwWkVbOI9X0D73APOBs/AuYVQ8nBl8TMJeQiUorgx/vz8FEvESZMU6+youjTjnZgOxZpYQrrj8S3/OOVcE/JFGeH/VFheAfxxGAf8OWidcx2s0cKmZbcar1nwe8BvC8P5qVonAvz72f8Bq59zTQYuCS1ncBPwzqP1Gv7fCmcDeoEtIIY/LzDLwTgMvdc4dDGpPNG88B8ysL9Af2BTGuIKvy14OZPnTs4Br/N4KyX5cn4UrLt8EYI1zLido/XAdr0Q73LOrFTAR7/7FfLwSKVDz/RXyEip1xLXGzL4PpAOT/aResX6g4lqymY3C+xw44QnqCHFVfCkzvMswwe+vcPx/rDUuf/GVeDdpC4PWD8vxcs495Jzr4Zzrg3fz9z3n3HWE4/3lwnQnPBw/wDl4l31WAiv8nwvxrpvNA9YD7wKd3OHeA9PxvkF+AaSFOa4NeNf4KtoqegB8G8j225YBl4Q5rpf847HSf7N1DXrNT/zjtRa4IJxx+ctmArdVWz9cx2sYsNyPK4vDvZb64iXEDcBfgZZ+e5w/v8Ff3jfMcZX6f6uKY1jRfod/vD7H66Rwdpjjes9/f2UBL3O4B0+4/j/WGpe/bAHeWUvw+mE5XtX2OY7DvYZC/v5SiQkRkQjXrC4NiYhIwykRiIhEOCUCEZEIp0QgIhLhlAhERCKcEoEcMzNzZvZU0Px9ZvboCdr2TDO78uhrHvd+vmNmq81sfrX2PmZ27TFu8+N6rPMHMxtyLNtvTGa2wMya3KDucnyUCOR4FAFXhPqp1IYKegqzPr4H3OqcG1+tvQ9QayI42vadc2cfbafOue8751bVN0iRUFIikONRijeG6j3VF1T/Rm9mBf6/4/zCcP80s01m9kszu868+vBfmNkpQZuZYGZLzGydX4eloljYE2a22C9a9oOg7X5gZrOAGh+wZjbZ336WmT3utz2C9/Da/5nZE9Ve8kvgXPPqz99jZjeb2Swzew+YZ2ZtzWyemS3zt3tZHb/rAjN7w7xxJ14JekK18pu1mRWY2S/MK2r2iZkl+e2n+PNfmNnPK7Zb7fdqY2b/9l+bZWZXV/xu/jHKMrPnqu331/5xXW1mI83sb+aN1fFzf50+QfGu9uNvXcu+J5nZIv8Y/NW82lD4f9NV/t/nyeqvkyYo1E/I6af5/gAFQDtgM16dk/uAR/1lMwmqOQ8U+P+OA/bgjTnQEq9eymP+sruAZ4JePwfvy0p/vKqLccAU4GF/nZbAEryxEcYBB4DkWuLsBmzBq7kTg/dk67f8ZQuo5QlWqtWgB272Y6h4Kj0GaOdPJ+A93Wm1/K578erDRAGLgHOq7xfvKepL/OlfBf1+b+GVhwC4rWK71eL8NvB80Hx7/99OQW0vBW1/AYfHvbgL+Drob5GD9xR+Hz+m0f56L+CPT1ERt/87LwTa+O0PAI/4r18bdCw6NPb7VD9H/9EZgRwX51UFfRH4UQNetth5hceK8MoJzPXbv8D7EKrwunOu3HnlpjfhVaychFePZgVeaerOeIkC4DPnjZNQ3UhggXMu33nlel8BxjQg3grvOOd2+dMG/LeZrcQrW9Kdw+XNg33mnMtxXq2fFdV+vwrFeB/6AEuD1jkLr4QAeAOV1OYLYKKZPW5m5zrn9vrt480bteoLvOJlwaWTZwW9Njvob7GJw0XftjrnPvKnX8Y7cwp2Jt5ARR/5f4ubgN54ia8Q7yzrCuAg0uQ15FqqSF2ewavx88egtlL8S49mFgW0CFpWFDRdHjRfTtX3ZPX6Jw7vA/hO51xm8AIzG4d3RhBKwdu/Du8MY4RzrsS8ipFxtbwm+Hcto/b/cyXO//p8hHVq5ZxbZ96QjhcCPzezeXhnFb/DO+PY6t/AD44t+HhX/1tU7Lu2Yx/M8BLj5OoxmVeY7Xy8Qmh34CUiacJ0RiDHzf+W/DrejdcKm4ER/vSleKNANdR3zCzKv2/QF++SQyYw1Q6PvzvAzNocZTufAWPNLMG8KqWTgfeP8pr9eMNk1qU9Xu34EjMbj/dt+ET7BO/SD3jVKGsws27AQefcy8ATwOkc/tDf4V+3P5beV73M7Cx/+lrgw1piG21m/fw42vh/i7Z4l6dm4907OvUY9i1hpjMCOVGewvv2V+F54J9m9jnetf5j+ba+Be9DvB1exdFCM/sD3qWTZf4N0HwOD91XK+fcN2b2IF45XwP+7Zz755Feg1eZssyPfyawu9ryV4B/+ZdelnC4jPGJdDfwsnkDF83Bu+xS3VDgCTMrB0rwxrPdY2bP41XWzAUWH8O+1wI/NLMX8G6+zwhe6JzLN28sjb/Y4RHGHsZLoP80szi8Y30v0uSp+qhIE+X31DnknHNmdg3ejePLjva6E7DfPng3ylOPsqo0EzojEGm6RgD/65/57MEb11rkhNMZgYhIhNPNYhGRCKdEICIS4ZQIREQinBKBiEiEUyIQEYlw/x830YklYhuorgAAAABJRU5ErkJggg==\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### **Fine-Tuning Multiple Languages**"
      ],
      "metadata": {
        "id": "GB-e4K_BnFYk"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#@ FUNCTION TO CONCATENATE DATASET:\n",
        "def concatenate_splits(corpora):                                        # Defining function.\n",
        "    multi_corpus = DatasetDict()                                        # Initialization.\n",
        "    for split in corpora[0].keys():\n",
        "        multi_corpus[split] = concatenate_datasets(\n",
        "            [corpus[split] for corpus in corpora]).shuffle(seed=2022)   # Initializing multi corpus.\n",
        "    return multi_corpus\n",
        "\n",
        "#@ IMPLEMENTATION:\n",
        "panx_de_fr_encoded = concatenate_splits([panx_de_encoded,\n",
        "                                         panx_fr_encoded])              # Implementation."
      ],
      "metadata": {
        "id": "9XKeOPqinLK_"
      },
      "execution_count": 57,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@ TRAINING THE MODEL:\n",
        "training_args.logging_steps = len(panx_de_fr_encoded[\"train\"]) // batch_size        # Initializing training logs. \n",
        "training_args.output_dir = \"xlm-roberta-base-finetuned-panx-de-fr\"                  # Initialization.\n",
        "trainer = Trainer(model_init=model_init, args=training_args, \n",
        "                  data_collator=data_collator, compute_metrics=compute_metrics,\n",
        "                  train_dataset=panx_de_fr_encoded[\"train\"], \n",
        "                  eval_dataset=panx_de_fr_encoded[\"validation\"],                    # Validation dataset.\n",
        "                  tokenizer=xlmr_tokenizer)                                         # Initializing trainer.\n",
        "trainer.train()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "CnlUGRSIp4o9",
        "outputId": "20cae4f0-afc3-445c-e4f2-ba872160508e"
      },
      "execution_count": 58,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='2145' max='2145' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [2145/2145 21:53, Epoch 3/3]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>F1</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>0.285400</td>\n",
              "      <td>0.184471</td>\n",
              "      <td>0.826331</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>0.142900</td>\n",
              "      <td>0.160440</td>\n",
              "      <td>0.841947</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>0.090800</td>\n",
              "      <td>0.163271</td>\n",
              "      <td>0.860465</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TrainOutput(global_step=2145, training_loss=0.17303146682419143, metrics={'train_runtime': 1313.9794, 'train_samples_per_second': 39.179, 'train_steps_per_second': 1.632, 'total_flos': 1131251766906240.0, 'train_loss': 0.17303146682419143, 'epoch': 3.0})"
            ]
          },
          "metadata": {},
          "execution_count": 58
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@ INSPECTING MODEL PERFORMANCE: \n",
        "for lang in langs:\n",
        "    f1 = evaluate_lang_performance(lang, trainer)                       # Implementation.\n",
        "    print(f\"F1-score of [de-fr] model on [{lang}] dataset: {f1:.3f}\")   # Inspecting f1 score. "
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 422,
          "referenced_widgets": [
            "bacd6980f0554fcdbf55096376c54f60",
            "173dfd34e0df41cb8a73aa89bf832f74",
            "223cf648aedb4add9759bca64fcf9a15",
            "05d38d8a92554199b94c537871de2990",
            "67bd7d599436450a8d1e5ea5c1ab0511",
            "71cb6d4dcf7d487f867f1e1d032afe36",
            "e34b8434b575495aa8e92c145d6d5026",
            "a1243998babc45d489e7d7a4ed42873c",
            "fa991fa36b3946ed85c66f0548a6ed0b",
            "486d7f4b43e84ef493b6eabd0815a9ec",
            "5339e0133f454bebbe7627fd57fe3ca7",
            "4df85b6812734ba8b21f1d122262be1a",
            "a4367abef48b4bd5b252c48861507163",
            "c02de02ef1fc47ebaa352560b35f0f69",
            "15aac8c5f948419b8da011f7dad87b8f",
            "a41215d00b0549188b0d609b239e4d1f",
            "4e6e507ab015430d8f3b36adcde5dc20",
            "3356ca5f770c4cd3b094e037fbd2ca3e",
            "4ca50970819844688371a43bc14cdcaf",
            "6c6be58b918247ae832cf99a2aec5ba1",
            "467f890e427649cca53ce7695e22f899",
            "29dad38e38c14649812748bcbe7ccdc3",
            "babc68b676e64d649cb404571b72f390",
            "411555aed8ca4e088623b1e687eac724",
            "755ca233d7e74de58740efb3696b2149",
            "9c1a8dd2cc7c46378c11f70a032531a0",
            "83f73a9ff9af41ac9b614dca5ace58e2",
            "c6fd425c567b49438e45444626608bcd",
            "54afa3a7bf4c4dfb886d85db551df5fb",
            "77cb43ba110f418583a5458e32bcdeb3",
            "b4f1a608a8ac41a585af222281a16524",
            "906b785cff5b4c5e864f219f3fe81343",
            "d378324bda714932953e030f40bf161a",
            "b4f89c85d6184a619fbe54735251a6ac",
            "358eec8f4da44c46aee2295d16b83b1b",
            "0b5154b8e9af4b5f8be5c9d521b6b4e9",
            "ff002853d6a246fa8e6e0e3cdc9370cd",
            "47bf5b9785f044a391fa1703062fd18c",
            "3eaf9acb098d40e8a130ca5acd3b2688",
            "8190da58411d454b909075919ec6bf68",
            "83ff353ec26d4013bcd5e43ff3afdfaa",
            "482ace85fab248eba12015cb5aaca8eb",
            "32e41f0f9a854078bb0eba978512a704",
            "a0f3411b7e724adaa7b9c48aad3c4108",
            "7f4b40288db043c78a17841cc6ffb488",
            "1849d226524d4b1491d8d63f046ac1e4",
            "76b37c8bf3e74827a746190954d13319",
            "0476685a236e47b69c7fceb0c2c9219c",
            "1664069b384544a4bf539e594fd10164",
            "d37c136c0f404f9ba744a1db35e28c6f",
            "e58b0cb5e90642aea2263dc663948506",
            "6056d5ce405b40d3bc3bcc43f4b46060",
            "1eae7217ef4b49d78b3f586a40384c99",
            "c10de464e7b748bc83c273baef2fc814",
            "9ae208732a3f48a4b0c793280889f6ba",
            "cf3b86828c2545f2924e9ccd89c0e47d",
            "bbcde84ff19f46e2a9b67793cf1e28e2",
            "97656fe4cb1e49ca9f6c418844c8ffa6",
            "96e93f87f8e6437493691a962aaf8dfd",
            "7f47d80c90ec493cb403de7a321f31c4",
            "7925fe24ad754d9ebfeea9e648667fc9",
            "193bd50ecbd245e9a212a16276c463df",
            "58f8d1ad991f426e84cde8c98246f0d4",
            "2a66903b36da4b2595d74b70d29f36a1",
            "1c56fa411cf84c99a3a15904e4826b9b",
            "8e6526d7f79b47368c09dbc82d9294ef"
          ]
        },
        "id": "4av2k8ezrnZ8",
        "outputId": "33ddc403-e01e-4407-ec2f-11a328020962"
      },
      "execution_count": 60,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Loading cached processed dataset at /root/.cache/huggingface/datasets/xtreme/PAN-X.de/1.0.0/349258adc25bb45e47de193222f95e68a44f7a7ab53c4283b3f007208a11bf7e/cache-5536e31f64c74e10.arrow\n",
            "Loading cached processed dataset at /root/.cache/huggingface/datasets/xtreme/PAN-X.de/1.0.0/349258adc25bb45e47de193222f95e68a44f7a7ab53c4283b3f007208a11bf7e/cache-5e1da0ee67f278e1.arrow\n",
            "Loading cached processed dataset at /root/.cache/huggingface/datasets/xtreme/PAN-X.de/1.0.0/349258adc25bb45e47de193222f95e68a44f7a7ab53c4283b3f007208a11bf7e/cache-c3a89d9efa0b632d.arrow\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='419' max='263' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [263/263 01:00]\n",
              "    </div>\n",
              "    "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Loading cached processed dataset at /root/.cache/huggingface/datasets/xtreme/PAN-X.fr/1.0.0/349258adc25bb45e47de193222f95e68a44f7a7ab53c4283b3f007208a11bf7e/cache-de9189e48180c3aa.arrow\n",
            "Loading cached processed dataset at /root/.cache/huggingface/datasets/xtreme/PAN-X.fr/1.0.0/349258adc25bb45e47de193222f95e68a44f7a7ab53c4283b3f007208a11bf7e/cache-17b34755b551ad0b.arrow\n",
            "Loading cached processed dataset at /root/.cache/huggingface/datasets/xtreme/PAN-X.fr/1.0.0/349258adc25bb45e47de193222f95e68a44f7a7ab53c4283b3f007208a11bf7e/cache-2528fe72e754fa78.arrow\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "F1-score of [de-fr] model on [de] dataset: 0.867\n",
            "F1-score of [de-fr] model on [fr] dataset: 0.867\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/2 [00:00<?, ?ba/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "bacd6980f0554fcdbf55096376c54f60"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/1 [00:00<?, ?ba/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "4df85b6812734ba8b21f1d122262be1a"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/1 [00:00<?, ?ba/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "babc68b676e64d649cb404571b72f390"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "F1-score of [de-fr] model on [it] dataset: 0.799\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/2 [00:00<?, ?ba/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "b4f89c85d6184a619fbe54735251a6ac"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/1 [00:00<?, ?ba/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "7f4b40288db043c78a17841cc6ffb488"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/1 [00:00<?, ?ba/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "cf3b86828c2545f2924e9ccd89c0e47d"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "F1-score of [de-fr] model on [en] dataset: 0.664\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@ FINE-TUNING ON MULTIPLE LANGUAGES:\n",
        "corpora = [panx_de_encoded]                                                 # Initialization.\n",
        "for lang in langs[1:]:\n",
        "    training_args.output_dir = f\"xlm-roberta-base-finetuned-panx-{lang}\"\n",
        "    ds_encoded = encode_panx_dataset(panx_ch[lang])\n",
        "    metrics = train_on_subset(ds_encoded, ds_encoded[\"train\"].num_rows)\n",
        "    f1_scores[lang][lang] = metrics[\"f1_score\"][0]                          # Adding f1 scores. \n",
        "    corpora.append(ds_encoded)\n",
        "\n",
        "#@ INITIALIZING CONCATENATION:\n",
        "corpora_encoded = concatenate_splits(corpora)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QHqFreDetuET",
        "outputId": "f740799e-c7da-4d50-aff3-7553708b6d94"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Loading cached processed dataset at /root/.cache/huggingface/datasets/xtreme/PAN-X.fr/1.0.0/349258adc25bb45e47de193222f95e68a44f7a7ab53c4283b3f007208a11bf7e/cache-de9189e48180c3aa.arrow\n",
            "Loading cached processed dataset at /root/.cache/huggingface/datasets/xtreme/PAN-X.fr/1.0.0/349258adc25bb45e47de193222f95e68a44f7a7ab53c4283b3f007208a11bf7e/cache-17b34755b551ad0b.arrow\n",
            "Loading cached processed dataset at /root/.cache/huggingface/datasets/xtreme/PAN-X.fr/1.0.0/349258adc25bb45e47de193222f95e68a44f7a7ab53c4283b3f007208a11bf7e/cache-2528fe72e754fa78.arrow\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@ TRAINING THE MODEL:\n",
        "training_args.logging_steps = len(corpora_encoded[\"train\"]) // batch_size           # Initializing training logs. \n",
        "training_args.output_dir = \"xlm-roberta-base-finetuned-panx-all\"                    # Initialization.\n",
        "trainer = Trainer(model_init=model_init, args=training_args, \n",
        "                  data_collator=data_collator, compute_metrics=compute_metrics,\n",
        "                  train_dataset=corpora_encoded[\"train\"], \n",
        "                  eval_dataset=corpora_encoded[\"validation\"],                       # Validation dataset.\n",
        "                  tokenizer=xlmr_tokenizer)                                         # Initializing trainer.\n",
        "trainer.train()"
      ],
      "metadata": {
        "id": "jegLlinNuo_T"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@ SUMMARIZING MULTILINGUAL LEARNING:\n",
        "for idx, lang in enumerate(langs):\n",
        "    f1_scores[\"all\"][lang] = get_f1_score(trainer, corpora[idx][\"test\"])\n",
        "scores_data = {\"de\": f1_scores[\"de\"], \n",
        "               \"each\": {lang: f1_scores[lang][lang] for lang in langs},\n",
        "               \"all\": f1_scores[\"all\"]}                                             # Dictionary of f1 scores.\n",
        "f1_scores_df = pd.DataFrame(scores_data).T.round(4)                                 # Creating dataframe.\n",
        "f1_scores_df.rename_axis(index=\"Fine-tune on\", columns=\"Evaluated on\", \n",
        "                         inplace=True)\n",
        "f1_scores_df"
      ],
      "metadata": {
        "id": "LPyYVzgcwMXn"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}